diff --git a/.gitattributes b/.gitattributes index e0a5a1471c70adfc417ae6aeaac09dd2035469f5..33afc53ca84c252dfa2d6ac592a646596fc8b1aa 100644 --- a/.gitattributes +++ b/.gitattributes @@ -67,3 +67,4 @@ tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN3_235B__moonep__ma tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/nsys/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/nsys/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/nsys/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text +tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal/rep1/train.log filter=lfs diff=lfs merge=lfs -text diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..a2f67aaad0fcd6e75ce5a797913807c2c368151f --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1/mem_samples.csv @@ -0,0 +1,272 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 2231 +1, 2231 +2, 2231 +3, 2231 +4, 2231 +5, 2231 +6, 2231 +7, 2231 +0, 39451 +1, 39905 +2, 39927 +3, 39909 +4, 39437 +5, 39917 +6, 39931 +7, 39951 +0, 40043 +1, 40503 +2, 40511 +3, 40495 +4, 40045 +5, 40505 +6, 40515 +7, 40509 +0, 42733 +1, 43181 +2, 43181 +3, 43181 +4, 42733 +5, 43181 +6, 43181 +7, 43181 +0, 42971 +1, 43419 +2, 43419 +3, 43419 +4, 42971 +5, 43419 +6, 43419 +7, 43419 +0, 43083 +1, 43531 +2, 43531 +3, 43531 +4, 43083 +5, 43531 +6, 43531 +7, 43531 +0, 44029 +1, 44477 +2, 44477 +3, 44477 +4, 44029 +5, 44477 +6, 44477 +7, 44477 +0, 45123 +1, 45571 +2, 45571 +3, 45571 +4, 45123 +5, 45571 +6, 45571 +7, 45571 +0, 46707 +1, 47155 +2, 47155 +3, 47155 +4, 46707 +5, 47155 +6, 47155 +7, 47155 +0, 47807 +1, 48255 +2, 48255 +3, 48255 +4, 47807 +5, 48255 +6, 48255 +7, 48255 +0, 82679 +1, 82625 +2, 82645 +3, 82645 +4, 82659 +5, 82687 +6, 82727 +7, 82585 +0, 84631 +1, 84657 +2, 84657 +3, 84657 +4, 84651 +5, 84657 +6, 84657 +7, 84657 +0, 85729 +1, 85755 +2, 85755 +3, 85755 +4, 85749 +5, 85755 +6, 85755 +7, 85755 +0, 85729 +1, 85755 +2, 85755 +3, 85755 +4, 85749 +5, 85755 +6, 85755 +7, 85755 +0, 85729 +1, 85755 +2, 85755 +3, 85755 +4, 85749 +5, 85755 +6, 85755 +7, 85755 +0, 102939 +1, 103003 +2, 103007 +3, 102991 +4, 103019 +5, 103011 +6, 102985 +7, 102997 +0, 103781 +1, 103817 +2, 103871 +3, 103833 +4, 103863 +5, 103921 +6, 103893 +7, 103871 +0, 104525 +1, 104573 +2, 104573 +3, 104573 +4, 104565 +5, 104573 +6, 104573 +7, 104573 +0, 105139 +1, 105139 +2, 105201 +3, 105231 +4, 105241 +5, 105197 +6, 105225 +7, 105213 +0, 128441 +1, 128525 +2, 128505 +3, 128505 +4, 128527 +5, 128525 +6, 128505 +7, 128505 +0, 128441 +1, 128525 +2, 128505 +3, 128505 +4, 128527 +5, 128525 +6, 128505 +7, 128505 +0, 128441 +1, 128525 +2, 128505 +3, 128505 +4, 128527 +5, 128525 +6, 128505 +7, 128505 +0, 128441 +1, 128525 +2, 128505 +3, 128505 +4, 129621 +5, 128525 +6, 128505 +7, 128505 +0, 127495 +1, 127643 +2, 128205 +3, 127879 +4, 128675 +5, 127579 +6, 127559 +7, 127559 +0, 1966 +1, 2222 +2, 127559 +3, 3246 +4, 1966 +5, 1966 +6, 1966 +7, 1966 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1/run.env b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..4ec7f4c0f83a310d7abcac052dde346018e4b364 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1/run.env @@ -0,0 +1,70 @@ +backend=echo +config=G4 +family=tierA +repo=/workspace/Megatron-MoE-Echo +branch=token-drop-analysis +commit=b099fd84 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=896 +num_experts=128 +topk=32 +experts_per_rank=16 +expert_params=11010048 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=131072 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.068359375 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1/train.log b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ce25f9bc2f4a0b9072dadcd9b0e3b8c935b74649 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1/train.log @@ -0,0 +1,16897 @@ +[05:03:48] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + G4 / echo (slots/rank=16 policy=matched) + E=128 K=32 H'=896 m=16 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=static cap=1.068359375 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ b099fd84 +============================================================== +W0830 05:03:49.917000 470792 torch/distributed/run.py:766] +W0830 05:03:49.917000 470792 torch/distributed/run.py:766] ***************************************** +W0830 05:03:49.917000 470792 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 05:03:49.917000 470792 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 896 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 128 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.068359375 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 32 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 0261 is connected to is connected to is connected to is connected to 7777 peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7777 + + + +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 5 is connected to 74 peer ranks. is connected to Expected number of connected peer ranks is : [Gloo] Rank 777 peer ranks. + is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank2]:[W830 05:04:07.140743552 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 05:04:07.140743166 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 05:04:07.140813233 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 05:04:07.140834998 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 05:04:07.140845149 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 05:04:07.140855341 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 05:04:07.141791484 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 05:04:08.307953569 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.865 seconds +time to initialize megatron (seconds): 7.172 +[after megatron is initialized] datetime: 2026-08-30 05:04:13 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 3910275072 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (42479616 elements, 42479616 padded size): + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.final_layernorm.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight +Params for bucket 3 (42475520 elements, 42475520 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.pre_mlp_layernorm.weight +Params for bucket 4 (42475520 elements, 42475520 padded size): + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_proj.weight +Params for bucket 5 (42475520 elements, 42475520 padded size): + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.pre_mlp_layernorm.weight +Params for bucket 6 (42475520 elements, 42475520 padded size): + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_qkv.weight +Params for bucket 7 (42475520 elements, 42475520 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.weight +Params for bucket 8 (42475520 elements, 42475520 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_proj.weight +Params for bucket 9 (42475520 elements, 42475520 padded size): + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_qkv.weight +Params for bucket 10 (42475520 elements, 42475520 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight +Params for bucket 11 (42475520 elements, 42475520 padded size): + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.mlp.router.weight +Params for bucket 12 (42475520 elements, 42475520 padded size): + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_proj.weight +Params for bucket 13 (42475520 elements, 42475520 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 14 (42475520 elements, 42475520 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.pre_mlp_layernorm.weight +Params for bucket 15 (42475520 elements, 42475520 padded size): + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.linear_qkv.weight +Params for bucket 16 (42475520 elements, 42475520 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 17 (42475520 elements, 42475520 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.pre_mlp_layernorm.weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 67 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 +Params for bucket 3 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 +Params for bucket 23 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 +Params for bucket 24 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 +Params for bucket 25 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 +Params for bucket 28 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 +Params for bucket 29 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 36 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 +Params for bucket 37 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 +Params for bucket 42 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 +Params for bucket 43 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 +Params for bucket 44 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 46 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 +Params for bucket 47 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 +Params for bucket 48 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 +Params for bucket 49 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 +Params for bucket 50 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 +Params for bucket 51 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 +Params for bucket 52 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 +Params for bucket 53 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 +Params for bucket 54 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 +Params for bucket 55 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 +Params for bucket 56 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 +Params for bucket 59 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 +Params for bucket 60 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 +Params for bucket 61 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 +Params for bucket 62 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 +Params for bucket 63 (40370176 elements, 40370176 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 +Params for bucket 64 (40370176 elements, 40370176 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 +Params for bucket 66 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 +Params for bucket 67 (29360128 elements, 29360128 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 05:04:14 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 192 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 05:04:14 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (688.49, 695.71) + train/valid/test-data-iterators-setup ..........: (64.30, 72.75) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 05:04:14 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[48, 2776, 896, 457, 1357, 502, 747, 852, 921, 155, 996, 561, 1182, 583, 326, 487, 135, 444, 1177, 865, 1046, 110, 2956, 2088, 460, 975, 832, 1248, 1287, 388, 1995, 2383, 2787, 798, 233, 482, 184, 1944, 1574, 292, 2405, 885, 1059, 1011, 1220, 1572, 1040, 1203, 267, 201, 581, 2076, 744, 371, 546, 90, 536, 195, 1054, 1556, 1391, 358, 649, 1089, 1031, 2023, 445, 1103, 937, 455, 744, 1282, 908, 2335, 219, 1968, 696, 838, 2770, 679, 2262, 1212, 2768, 494, 2135, 480, 593, 958, 1343, 316, 749, 903, 1061, 561, 389, 611, 1996, 235, 1308, 472, 1663, 1203, 1798, 2457, 837, 840, 624, 2328, 624, 157, 2319, 963, 310, 1400, 741, 1077, 350, 524, 1202, 1287, 932, 1379, 1432, 956, 1299, 480, 335, 648] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[79, 2956, 568, 182, 1217, 425, 769, 1156, 1136, 291, 1028, 174, 953, 828, 309, 385, 124, 402, 314, 1101, 1058, 118, 2707, 1734, 263, 1062, 1145, 1025, 1490, 431, 1436, 2206, 3142, 484, 266, 958, 141, 2349, 1831, 722, 2931, 416, 1187, 583, 1091, 1250, 1266, 999, 175, 228, 228, 2144, 1386, 342, 285, 69, 587, 493, 916, 1613, 1522, 380, 1460, 840, 680, 2009, 462, 1059, 1312, 203, 704, 1509, 912, 1852, 152, 2179, 786, 674, 2880, 795, 2722, 1040, 2331, 461, 2377, 349, 823, 1082, 2009, 261, 548, 1037, 1266, 463, 404, 853, 2051, 229, 2306, 464, 1339, 1104, 1963, 2514, 823, 671, 593, 2129, 661, 453, 1923, 862, 265, 851, 1301, 1087, 147, 543, 832, 1232, 792, 1410, 1471, 680, 1578, 388, 609, 251] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[76, 2952, 694, 262, 1513, 618, 1031, 992, 1025, 182, 955, 181, 603, 562, 225, 525, 146, 647, 639, 919, 902, 147, 2914, 1759, 460, 702, 1022, 795, 1334, 422, 1610, 2017, 3080, 356, 206, 440, 121, 1948, 2197, 575, 2977, 808, 1488, 441, 771, 1133, 1184, 1302, 243, 370, 325, 2000, 852, 458, 499, 149, 325, 631, 711, 1631, 1865, 592, 1057, 1093, 514, 1945, 731, 837, 1397, 395, 791, 1551, 1039, 1566, 121, 1937, 676, 1405, 1955, 1046, 2077, 1063, 2890, 384, 2231, 485, 921, 803, 2187, 230, 762, 840, 1052, 376, 440, 858, 2159, 267, 1789, 547, 1502, 975, 1717, 1837, 998, 815, 587, 2342, 766, 479, 1770, 793, 177, 928, 1614, 1372, 368, 826, 783, 1310, 850, 1749, 1477, 998, 1764, 412, 691, 246] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[51, 2962, 486, 178, 1629, 530, 1087, 696, 956, 179, 1546, 284, 882, 666, 179, 296, 69, 810, 559, 545, 1006, 220, 3203, 1941, 551, 1196, 885, 1138, 864, 292, 1426, 1838, 3083, 743, 254, 224, 81, 2271, 1615, 427, 2349, 536, 1500, 1092, 1380, 1643, 1204, 1395, 187, 553, 265, 1864, 625, 464, 391, 65, 830, 267, 932, 1808, 1548, 487, 564, 1312, 859, 1836, 923, 1021, 1246, 103, 946, 1661, 1203, 1853, 216, 1691, 809, 1012, 2237, 810, 2848, 886, 3088, 592, 1753, 342, 550, 1810, 1547, 244, 703, 1309, 926, 664, 395, 1181, 2208, 163, 1559, 631, 1359, 764, 1227, 2110, 813, 925, 508, 2016, 502, 299, 2244, 1368, 344, 818, 1293, 1195, 307, 468, 1030, 1401, 801, 1699, 1434, 746, 1307, 441, 414, 305] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[128, 3053, 828, 361, 1245, 611, 692, 1162, 778, 324, 972, 388, 1142, 1006, 134, 413, 76, 489, 1473, 718, 818, 185, 2596, 1701, 840, 956, 1105, 1373, 1484, 312, 1738, 2054, 2803, 418, 300, 598, 253, 1995, 2029, 441, 2673, 300, 1548, 725, 1319, 1272, 875, 1260, 369, 361, 221, 1659, 1189, 598, 553, 104, 577, 311, 849, 1426, 1436, 700, 1145, 1174, 1123, 2130, 530, 1243, 785, 386, 596, 2042, 705, 2221, 469, 1756, 580, 553, 2311, 624, 2421, 1242, 3026, 409, 2030, 284, 1034, 989, 1213, 289, 519, 1046, 624, 661, 486, 1104, 1884, 238, 1902, 570, 1491, 932, 1575, 2465, 701, 941, 554, 2330, 766, 467, 2071, 644, 243, 1099, 1097, 956, 486, 576, 1053, 1496, 629, 1077, 1577, 614, 1356, 488, 423, 304] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[141, 2558, 988, 249, 1113, 448, 983, 919, 1184, 173, 857, 140, 931, 576, 155, 423, 246, 333, 1081, 736, 926, 110, 2822, 1473, 471, 777, 1198, 557, 1420, 563, 1395, 1552, 3062, 442, 134, 491, 158, 1928, 2291, 451, 3111, 253, 1039, 1009, 1419, 1203, 1139, 643, 240, 234, 272, 1952, 1042, 413, 648, 75, 513, 436, 1083, 1028, 1203, 661, 1013, 687, 575, 1738, 530, 983, 1419, 205, 690, 1588, 1018, 2034, 309, 1815, 636, 513, 1677, 1079, 3201, 1196, 2883, 665, 1986, 147, 951, 1193, 2571, 210, 551, 1186, 1186, 501, 578, 1070, 1622, 368, 1693, 687, 1700, 483, 2077, 2087, 1139, 967, 926, 2171, 656, 460, 1744, 1119, 272, 1225, 1504, 1451, 326, 686, 1375, 1319, 812, 1739, 1527, 821, 1480, 893, 559, 529] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[113, 2241, 727, 458, 1232, 233, 987, 685, 1242, 138, 1420, 629, 711, 647, 125, 431, 113, 642, 517, 935, 1080, 109, 2755, 1808, 482, 796, 864, 778, 1095, 121, 1712, 1818, 2940, 448, 351, 468, 480, 2422, 1571, 608, 2569, 332, 1246, 886, 1503, 1109, 1450, 801, 279, 510, 292, 2473, 801, 421, 547, 86, 546, 612, 1255, 1489, 2007, 291, 548, 1192, 488, 2195, 800, 1593, 813, 223, 849, 1518, 909, 2339, 231, 1743, 730, 803, 2068, 706, 2538, 1120, 2702, 627, 1482, 538, 862, 913, 1963, 336, 537, 1073, 915, 964, 397, 668, 2294, 273, 1816, 582, 1573, 702, 1931, 2417, 533, 1028, 486, 1741, 771, 555, 1783, 1262, 285, 1261, 834, 1398, 268, 482, 691, 1503, 1107, 1395, 1767, 747, 2315, 364, 673, 415] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[159, 2994, 594, 592, 1464, 573, 1045, 499, 1166, 153, 1871, 113, 1131, 1001, 173, 467, 115, 361, 1232, 695, 769, 446, 2851, 1928, 502, 841, 941, 991, 1290, 242, 1469, 1433, 2961, 509, 231, 218, 232, 2712, 2014, 503, 2559, 205, 1276, 831, 1172, 1312, 1106, 1111, 333, 202, 427, 1617, 696, 876, 685, 61, 735, 581, 1194, 1557, 1580, 602, 323, 961, 912, 1668, 513, 1097, 973, 235, 914, 1703, 927, 2110, 324, 2125, 589, 893, 2331, 721, 2409, 1802, 2839, 613, 2029, 126, 730, 1151, 1357, 294, 609, 1374, 783, 579, 463, 874, 2035, 267, 1729, 907, 1245, 710, 1828, 2892, 926, 969, 392, 2184, 750, 437, 2161, 1321, 226, 959, 958, 1347, 128, 397, 646, 1174, 778, 1173, 1460, 551, 1848, 362, 401, 361] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[48, 2776, 896, 457, 1357, 502, 747, 852, 921, 155, 996, 561, 1182, 583, 326, 487, 1639, 1147, 1149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 444, 1177, 865, 1046, 110, 2669, 2088, 460, 975, 832, 1248, 1287, 388, 1995, 2383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 798, 233, 482, 184, 1944, 1574, 292, 2405, 885, 1059, 1011, 1220, 1572, 1040, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 201, 581, 2076, 744, 371, 546, 90, 536, 195, 1054, 1556, 1391, 358, 649, 1089, 2543, 1310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 2023, 445, 1103, 937, 455, 744, 1282, 908, 2335, 219, 1968, 696, 838, 831, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2262, 1212, 1129, 494, 2135, 480, 593, 958, 1343, 316, 749, 903, 1061, 561, 389, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1996, 235, 1308, 472, 1663, 1203, 1798, 0, 837, 840, 624, 1245, 624, 157, 2319, 963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 1400, 741, 1077, 350, 524, 1202, 1287, 932, 1379, 1432, 956, 1299, 480, 335, 648, 1083, 790, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16781, 18102, 16146, 15557, 16494, 15196, 16284, 16512] avg=16384.0000 max=18102 max_rank=1 min=15196 min_rank=5 imbalance_ratio=1.104858 max_load_reduction=112970 max_load_reduction_pct=0.861893 moved_to_echo=64486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[128, 3053, 828, 361, 1245, 611, 692, 1162, 778, 324, 972, 388, 1142, 1006, 134, 413, 1792, 1150, 959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 489, 1473, 718, 818, 185, 2344, 1701, 840, 956, 1105, 1373, 1484, 312, 1738, 2054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 418, 300, 598, 253, 1995, 2029, 441, 2673, 300, 1548, 725, 1319, 1272, 875, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 361, 221, 1659, 1189, 598, 553, 104, 577, 311, 849, 1426, 1436, 700, 1145, 1174, 2558, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 2130, 530, 1243, 785, 386, 596, 2042, 705, 2221, 469, 1756, 580, 553, 693, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2421, 1242, 1234, 409, 2030, 284, 1034, 989, 1213, 289, 519, 1046, 624, 661, 486, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1884, 238, 1902, 570, 1491, 932, 1575, 0, 701, 941, 554, 1246, 766, 467, 2071, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 1099, 1097, 956, 486, 576, 1053, 1496, 629, 1077, 1577, 614, 1356, 488, 423, 304, 1084, 659, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17138, 17666, 16251, 16545, 16436, 15585, 15982, 15469] avg=16384.0000 max=17666 max_rank=1 min=15469 min_rank=7 imbalance_ratio=1.078247 max_load_reduction=113406 max_load_reduction_pct=0.865219 moved_to_echo=63472 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[76, 2952, 694, 262, 1513, 618, 1031, 992, 1025, 182, 955, 181, 603, 562, 225, 525, 1712, 858, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 647, 639, 919, 902, 147, 2631, 1759, 460, 702, 1022, 795, 1334, 422, 1610, 2017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 356, 206, 440, 121, 1948, 2197, 575, 2977, 808, 1488, 441, 771, 1133, 1184, 1302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 370, 325, 2000, 852, 458, 499, 149, 325, 631, 711, 1631, 1865, 592, 1057, 1093, 2811, 979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 1945, 731, 837, 1397, 395, 791, 1551, 1039, 1566, 121, 1937, 676, 1405, 587, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2077, 1063, 1178, 384, 2231, 485, 921, 803, 2187, 230, 762, 840, 1052, 376, 440, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2159, 267, 1789, 547, 1502, 975, 1717, 0, 998, 815, 587, 1252, 766, 479, 1770, 793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 928, 1614, 1372, 368, 826, 783, 1310, 850, 1749, 1477, 998, 1764, 412, 691, 246, 1090, 557, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15777, 16152, 16216, 16591, 16538, 15887, 16416, 17495] avg=16384.0000 max=17495 max_rank=7 min=15777 min_rank=0 imbalance_ratio=1.067810 max_load_reduction=113577 max_load_reduction_pct=0.866524 moved_to_echo=66336 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[79, 2956, 568, 182, 1217, 425, 769, 1156, 1136, 291, 1028, 174, 953, 828, 309, 385, 1380, 1174, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 402, 314, 1101, 1058, 118, 2444, 1734, 263, 1062, 1145, 1025, 1490, 431, 1436, 2206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 484, 266, 958, 141, 2349, 1831, 722, 2931, 416, 1187, 583, 1091, 1250, 1266, 999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 228, 228, 2144, 1386, 342, 285, 69, 587, 493, 916, 1613, 1522, 380, 1460, 840, 2867, 1340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 680, 2009, 462, 1059, 1312, 203, 704, 1509, 912, 1852, 152, 2179, 786, 674, 864, 795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2722, 1040, 951, 461, 2377, 349, 823, 1082, 2009, 261, 548, 1037, 1266, 463, 404, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2051, 229, 2306, 464, 1339, 1104, 1963, 0, 823, 671, 593, 1138, 661, 453, 1923, 862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 851, 1301, 1087, 147, 543, 832, 1232, 792, 1410, 1471, 680, 1578, 388, 609, 251, 991, 821, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16205, 16353, 16749, 16875, 16152, 16646, 16580, 15512] avg=16384.0000 max=16875 max_rank=3 min=15512 min_rank=7 imbalance_ratio=1.029968 max_load_reduction=114197 max_load_reduction_pct=0.871254 moved_to_echo=64890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[141, 2558, 988, 249, 1113, 448, 983, 919, 1184, 173, 857, 140, 931, 576, 155, 423, 1712, 975, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 333, 1081, 736, 926, 110, 2543, 1473, 471, 777, 1198, 557, 1420, 563, 1395, 1552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 442, 134, 491, 158, 1928, 2291, 451, 3111, 253, 1039, 1009, 1419, 1203, 1139, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 234, 272, 1952, 1042, 413, 648, 75, 513, 436, 1083, 1028, 1203, 661, 1013, 687, 2798, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 1738, 530, 983, 1419, 205, 690, 1588, 1018, 2034, 309, 1815, 636, 513, 497, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3201, 1196, 1171, 665, 1986, 147, 951, 1193, 2571, 210, 551, 1186, 1186, 501, 578, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1622, 368, 1693, 687, 1700, 483, 2077, 0, 1139, 967, 926, 1156, 656, 460, 1744, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 1225, 1504, 1451, 326, 686, 1375, 1319, 812, 1739, 1527, 821, 1480, 893, 559, 529, 1015, 482, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15223, 15381, 15975, 15410, 15629, 18363, 16797, 18294] avg=16384.0000 max=18363 max_rank=5 min=15223 min_rank=0 imbalance_ratio=1.120789 max_load_reduction=112709 max_load_reduction_pct=0.859901 moved_to_echo=69083 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[51, 2962, 486, 178, 1629, 530, 1087, 696, 956, 179, 1546, 284, 882, 666, 179, 296, 1829, 985, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 810, 559, 545, 1006, 220, 2892, 1941, 551, 1196, 885, 1138, 864, 292, 1426, 1838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 743, 254, 224, 81, 2271, 1615, 427, 2349, 536, 1500, 1092, 1380, 1643, 1204, 1395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 553, 265, 1864, 625, 464, 391, 65, 830, 267, 932, 1808, 1548, 487, 564, 1312, 2813, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 859, 1836, 923, 1021, 1246, 103, 946, 1661, 1203, 1853, 216, 1691, 809, 1012, 671, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2848, 886, 1259, 592, 1753, 342, 550, 1810, 1547, 244, 703, 1309, 926, 664, 395, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2208, 163, 1559, 631, 1359, 764, 1227, 0, 813, 925, 508, 1078, 502, 299, 2244, 1368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 818, 1293, 1195, 307, 468, 1030, 1401, 801, 1699, 1434, 746, 1307, 441, 414, 305, 938, 638, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16349, 16232, 16984, 16100, 16860, 17009, 15648, 15890] avg=16384.0000 max=17009 max_rank=5 min=15648 min_rank=6 imbalance_ratio=1.038147 max_load_reduction=114063 max_load_reduction_pct=0.870232 moved_to_echo=65407 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[113, 2241, 727, 458, 1232, 233, 987, 685, 1242, 138, 1420, 629, 711, 647, 125, 431, 1600, 1128, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 642, 517, 935, 1080, 109, 2487, 1808, 482, 796, 864, 778, 1095, 121, 1712, 1818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 448, 351, 468, 480, 2422, 1571, 608, 2569, 332, 1246, 886, 1503, 1109, 1450, 801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 510, 292, 2473, 801, 421, 547, 86, 546, 612, 1255, 1489, 2007, 291, 548, 1192, 2683, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 2195, 800, 1593, 813, 223, 849, 1518, 909, 2339, 231, 1743, 730, 803, 620, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2538, 1120, 1102, 627, 1482, 538, 862, 913, 1963, 336, 537, 1073, 915, 964, 397, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2294, 273, 1816, 582, 1573, 702, 1931, 0, 533, 1028, 486, 931, 771, 555, 1783, 1262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 1261, 834, 1398, 268, 482, 691, 1503, 1107, 1395, 1767, 747, 2315, 364, 673, 415, 810, 590, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15605, 15357, 16501, 17321, 16560, 16035, 16520, 17173] avg=16384.0000 max=17321 max_rank=3 min=15357 min_rank=1 imbalance_ratio=1.057190 max_load_reduction=113751 max_load_reduction_pct=0.867851 moved_to_echo=66288 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[159, 2994, 594, 592, 1464, 573, 1045, 499, 1166, 153, 1871, 113, 1131, 1001, 173, 467, 1681, 1351, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 361, 1232, 695, 769, 446, 2574, 1928, 502, 841, 941, 991, 1290, 242, 1469, 1433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 509, 231, 218, 232, 2712, 2014, 503, 2559, 205, 1276, 831, 1172, 1312, 1106, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 202, 427, 1617, 696, 876, 685, 61, 735, 581, 1194, 1557, 1580, 602, 323, 961, 2702, 1541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 912, 1668, 513, 1097, 973, 235, 914, 1703, 927, 2110, 324, 2125, 589, 893, 699, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2409, 1802, 1158, 613, 2029, 126, 730, 1151, 1357, 294, 609, 1374, 783, 579, 463, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2035, 267, 1729, 907, 1245, 710, 1828, 0, 926, 969, 392, 1168, 750, 437, 2161, 1321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 959, 958, 1347, 128, 397, 646, 1174, 778, 1173, 1460, 551, 1848, 362, 401, 361, 1016, 665, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17994, 15829, 16250, 16673, 16403, 16351, 16845, 14727] avg=16384.0000 max=17994 max_rank=0 min=14727 min_rank=7 imbalance_ratio=1.098267 max_load_reduction=113078 max_load_reduction_pct=0.862717 moved_to_echo=64326 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2176, 8576, 9344, 9984, 2432, 4608, 7680, 10752, 6784, 11648, 12160, 6144, 13056, 3840, 4224, 3072, 8064, 5248, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2176, 4224, 2048, 3968, 1664, 17664, 15232, 4096, 21632, 3840, 10368, 6656, 9984, 10496, 9344, 8832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[20480, 9600, 9216, 4352, 16128, 2816, 6528, 8960, 14208, 2304, 4992, 8832, 7936, 4864, 3584, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[16256, 2048, 14208, 4864, 11904, 6912, 14208, 0, 6784, 7168, 4736, 9216, 5504, 3328, 16128, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2176, 2688, 2688, 15872, 7424, 3968, 4224, 768, 4736, 3584, 8064, 12160, 12672, 4096, 6784, 8448, 21888, 10112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[896, 22528, 5888, 2816, 10880, 3968, 7424, 7040, 8448, 1664, 9728, 2560, 7552, 5888, 1664, 3456, 13440, 8832, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[6272, 15616, 4992, 8960, 8960, 2304, 6272, 12928, 7680, 16384, 2048, 15232, 5504, 6784, 5504, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1024, 4224, 7040, 6528, 7680, 1536, 20608, 14464, 4096, 7424, 8064, 7936, 10368, 2816, 12800, 15360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[3545, 3, 2947, 927, 3120, 2, 0, 1930, 1675, 19, 22, 2, 2072, 0, 0, 2627, 111, 17, 454, 135, 6, 4048, 337, 891, 16, 12, 2, 10, 12, 1151, 4091, 8, 4068, 59, 3371, 2, 0, 0, 242, 39, 1292, 0, 0, 85, 0, 42, 51, 5, 14, 702, 1313, 17, 35, 66, 495, 38, 394, 3786, 3011, 583, 432, 0, 2, 8, 132, 104, 2996, 4, 95, 0, 4096, 26, 1636, 2788, 1, 1414, 1173, 0, 580, 1, 3764, 2907, 2560, 3122, 3132, 51, 575, 3928, 68, 515, 366, 4078, 1877, 17, 0, 758, 10, 193, 4096, 2756, 3300, 2, 4062, 115, 1, 736, 1654, 428, 4019, 1024, 0, 1927, 157, 117, 621, 83, 322, 4065, 2, 283, 1507, 37, 1, 4019, 328, 0, 110, 1989] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[3556, 0, 2087, 258, 4030, 1, 604, 1422, 3948, 54, 4, 221, 1265, 226, 4, 486, 4, 216, 621, 0, 877, 4004, 203, 936, 1808, 467, 0, 1603, 357, 3358, 4072, 13, 2181, 855, 3117, 0, 93, 16, 0, 7, 1114, 491, 22, 300, 0, 23, 57, 43, 0, 7, 40, 0, 11, 983, 2, 3, 0, 4083, 315, 1266, 2606, 285, 23, 0, 45, 1552, 2858, 88, 240, 0, 4019, 6, 521, 2261, 802, 249, 729, 43, 3015, 0, 3351, 2389, 1272, 1331, 3093, 217, 16, 4094, 4, 224, 319, 4025, 1145, 8, 2, 487, 1, 0, 4081, 16, 3048, 37, 3658, 0, 148, 1473, 2969, 882, 3526, 2938, 4, 314, 5, 333, 1676, 22, 0, 1828, 200, 614, 848, 8, 0, 4063, 1428, 100, 0, 3799] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[3977, 0, 1241, 68, 2433, 5, 22, 1831, 4061, 50, 1243, 35, 3970, 625, 33, 2095, 40, 0, 2111, 6, 87, 1711, 633, 45, 1, 46, 0, 38, 1, 2683, 2901, 1, 664, 808, 2084, 0, 11, 3, 1, 0, 38, 3, 7, 0, 110, 10, 1032, 96, 0, 267, 481, 0, 37, 2276, 0, 0, 4, 3426, 688, 2257, 140, 2, 16, 239, 0, 0, 2284, 68, 1000, 58, 2438, 39, 2660, 1613, 9, 0, 219, 5, 4084, 0, 4050, 2211, 3499, 67, 3466, 1907, 4024, 3979, 0, 15, 2667, 4096, 70, 25, 0, 2, 0, 1382, 3914, 361, 3867, 4, 245, 22, 84, 623, 4067, 9, 1948, 1483, 41, 3977, 193, 882, 845, 1561, 0, 983, 2849, 2, 1323, 77, 0, 3864, 1306, 0, 0, 3912] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[3103, 0, 3339, 14, 2220, 0, 28, 4051, 3884, 223, 69, 183, 2736, 7, 8, 1257, 0, 0, 2650, 0, 151, 3876, 1515, 2859, 46, 168, 21, 505, 0, 1301, 1858, 0, 197, 1870, 3428, 1, 5, 20, 274, 0, 733, 1741, 36, 0, 5, 60, 701, 29, 0, 0, 522, 15, 306, 2862, 19, 20, 4, 4090, 264, 241, 1068, 128, 13, 0, 0, 498, 3762, 688, 100, 0, 3822, 3, 260, 659, 392, 344, 118, 0, 3904, 0, 3873, 1773, 2494, 501, 3990, 14, 1874, 4076, 0, 2772, 2187, 4091, 14, 42, 3, 680, 10, 79, 3867, 117, 3530, 0, 3990, 0, 65, 3680, 3647, 678, 1524, 2178, 937, 1191, 105, 6, 571, 0, 0, 3943, 209, 402, 276, 0, 39, 1059, 0, 0, 2, 1309] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1270, 173, 3446, 925, 2094, 0, 37, 3247, 3386, 176, 1818, 254, 1188, 207, 0, 2327, 5, 1, 3144, 40, 507, 1856, 516, 5, 15, 4, 0, 116, 0, 88, 2369, 0, 1052, 986, 3666, 0, 285, 0, 3382, 2, 19, 18, 0, 111, 0, 0, 299, 0, 43, 62, 98, 0, 2725, 142, 23, 58, 58, 4064, 134, 1441, 2867, 0, 3, 3, 1, 78, 3128, 1262, 0, 0, 3807, 43, 3103, 2159, 370, 347, 114, 0, 1144, 0, 4085, 1113, 921, 3800, 3584, 135, 669, 3650, 15, 106, 2659, 4047, 6, 123, 0, 3250, 0, 220, 3574, 60, 3530, 0, 4036, 0, 1, 3533, 162, 166, 2939, 1604, 227, 2149, 17, 1438, 798, 0, 0, 4066, 285, 438, 1670, 0, 0, 1524, 63, 17, 12, 4069] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[3736, 0, 3318, 653, 3891, 1, 0, 3772, 3468, 2256, 1124, 0, 461, 521, 0, 3667, 0, 0, 2172, 41, 515, 3858, 0, 303, 0, 0, 3, 23, 0, 934, 3850, 0, 3307, 53, 3006, 3, 3, 0, 0, 5, 90, 4, 0, 3, 0, 0, 68, 5, 1, 43, 521, 0, 169, 1881, 0, 0, 1, 4002, 259, 3093, 6, 5, 1, 9, 1, 0, 3528, 2, 295, 11, 4065, 0, 3018, 3202, 0, 135, 779, 0, 1549, 5, 4051, 775, 556, 1742, 2629, 848, 767, 3409, 2, 24, 744, 4064, 1863, 147, 4, 230, 9, 33, 4038, 1534, 3927, 0, 1882, 1, 1, 567, 776, 27, 1801, 97, 134, 2801, 34, 33, 3006, 89, 1, 3482, 115, 494, 2386, 15, 0, 3974, 995, 1236, 192, 3842] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[3514, 0, 4093, 543, 2684, 0, 51, 1774, 4089, 632, 0, 3, 2014, 1361, 0, 3929, 0, 20, 72, 0, 0, 2138, 293, 1553, 3, 422, 1, 3, 15, 3492, 2689, 1, 2212, 2304, 2724, 5, 324, 6, 0, 0, 2545, 0, 0, 128, 92, 87, 5, 71, 0, 2, 0, 3, 48, 3436, 0, 8, 1, 3969, 90, 3133, 895, 0, 28, 154, 1, 0, 3250, 166, 1680, 4, 3981, 0, 1303, 0, 15, 61, 1, 0, 2384, 0, 4040, 3191, 566, 2070, 3523, 117, 680, 4011, 0, 1235, 749, 4090, 35, 0, 1, 889, 1, 686, 3091, 1735, 3670, 66, 3597, 6, 4, 273, 4073, 4, 3422, 5, 297, 281, 119, 107, 0, 669, 53, 2635, 181, 90, 3867, 16, 0, 3217, 49, 59, 9, 3088] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[990, 14, 3154, 1945, 2336, 5, 301, 3891, 3720, 3235, 182, 3, 1718, 1518, 0, 2768, 28, 11, 3447, 3, 27, 3045, 16, 73, 0, 7, 6, 245, 47, 2984, 3998, 0, 3206, 1935, 3493, 0, 0, 40, 19, 0, 565, 871, 0, 1, 9, 1, 0, 8, 0, 33, 255, 0, 276, 3704, 1, 0, 1, 3471, 363, 3469, 261, 0, 84, 318, 0, 1, 3952, 393, 1347, 10, 2666, 0, 1608, 2923, 1, 25, 1, 10, 3879, 6, 4075, 1984, 134, 2, 3921, 18, 1411, 3612, 2, 21, 2274, 4054, 9, 392, 50, 51, 0, 2020, 4094, 1357, 1067, 6, 325, 0, 77, 2298, 3922, 10, 242, 677, 106, 657, 30, 52, 1019, 5, 4, 2484, 1975, 11, 31, 0, 3, 3487, 108, 17, 29, 4026] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[786, 3, 2947, 927, 3120, 2, 0, 1930, 0, 19, 22, 2, 2072, 0, 0, 2627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 17, 454, 135, 6, 4048, 337, 891, 16, 12, 2, 10, 12, 1151, 4091, 8, 2411, 882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 59, 3371, 2, 0, 0, 242, 39, 1292, 0, 0, 85, 0, 42, 51, 5, 4078, 3605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 702, 1313, 17, 35, 66, 495, 38, 394, 3786, 3011, 583, 432, 0, 2, 8, 4096, 793, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 104, 2996, 4, 95, 0, 4096, 26, 1636, 2788, 1, 1414, 1173, 0, 580, 1, 1118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2907, 2560, 3122, 3132, 51, 575, 2591, 68, 515, 366, 0, 1877, 17, 0, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 193, 0, 2756, 1900, 2, 4062, 115, 1, 736, 1654, 428, 4019, 1024, 0, 1927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 117, 621, 83, 322, 4065, 2, 283, 1507, 37, 1, 4019, 328, 0, 110, 1989, 1400, 348, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14457, 14594, 16939, 15944, 16164, 18539, 18827, 15608] avg=16384.0000 max=18827 max_rank=6 min=14457 min_rank=0 imbalance_ratio=1.149109 max_load_reduction=112245 max_load_reduction_pct=0.856361 moved_to_echo=69138 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[881, 0, 1241, 68, 2433, 5, 22, 1831, 0, 50, 1243, 35, 3970, 625, 33, 2095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 2111, 6, 87, 1711, 633, 45, 1, 46, 0, 38, 1, 2683, 2901, 1, 2705, 2137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 664, 808, 2084, 0, 11, 3, 1, 0, 38, 3, 7, 0, 110, 10, 1032, 96, 4096, 3878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 481, 0, 37, 2276, 0, 0, 4, 3426, 688, 2257, 140, 2, 16, 239, 3914, 1924, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2284, 68, 1000, 58, 2438, 39, 2660, 1613, 9, 0, 219, 5, 4084, 0, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2211, 3499, 67, 3466, 1907, 4024, 2625, 0, 15, 2667, 0, 70, 25, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1382, 0, 361, 2226, 4, 245, 22, 84, 623, 4067, 9, 1948, 1483, 41, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 882, 845, 1561, 0, 983, 2849, 2, 1323, 77, 0, 3864, 1306, 0, 0, 3912, 1641, 391, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14532, 15146, 12841, 15843, 15609, 20578, 16472, 20051] avg=16384.0000 max=20578 max_rank=5 min=12841 min_rank=2 imbalance_ratio=1.255981 max_load_reduction=110494 max_load_reduction_pct=0.843002 moved_to_echo=72710 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[828, 0, 3318, 653, 3891, 1, 0, 3772, 0, 2256, 1124, 0, 461, 521, 0, 3667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2172, 41, 515, 3858, 0, 303, 0, 0, 3, 23, 0, 934, 3850, 0, 2541, 1825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3307, 53, 3006, 3, 3, 0, 0, 5, 90, 4, 0, 3, 0, 0, 68, 5, 4064, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 43, 521, 0, 169, 1881, 0, 0, 1, 4002, 259, 3093, 6, 5, 1, 9, 4038, 1643, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3528, 2, 295, 11, 4065, 0, 3018, 3202, 0, 135, 779, 0, 1549, 5, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 775, 556, 1742, 2629, 848, 767, 2249, 2, 24, 744, 0, 1863, 147, 4, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 33, 0, 1534, 2261, 0, 1882, 1, 1, 567, 776, 27, 1801, 97, 134, 2801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 33, 3006, 89, 1, 3482, 115, 494, 2386, 15, 0, 3974, 995, 1236, 192, 3842, 1666, 367, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20492, 16065, 14491, 15843, 17560, 12580, 11924, 22117] avg=16384.0000 max=22117 max_rank=7 min=11924 min_rank=6 imbalance_ratio=1.349915 max_load_reduction=108955 max_load_reduction_pct=0.831261 moved_to_echo=64181 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[788, 0, 2087, 258, 4030, 1, 604, 1422, 0, 54, 4, 221, 1265, 226, 4, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 216, 621, 0, 877, 4004, 203, 936, 1808, 467, 0, 1603, 357, 3358, 4072, 13, 2419, 2077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 855, 3117, 0, 93, 16, 0, 7, 1114, 491, 22, 300, 0, 23, 57, 43, 4025, 3209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 40, 0, 11, 983, 2, 3, 0, 4083, 315, 1266, 2606, 285, 23, 0, 4081, 1871, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 1552, 2858, 88, 240, 0, 4019, 6, 521, 2261, 802, 249, 729, 43, 3015, 0, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2389, 1272, 1331, 3093, 217, 16, 2701, 4, 224, 319, 0, 1145, 8, 2, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 16, 1755, 37, 3658, 0, 148, 1473, 2969, 882, 3526, 2938, 4, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 333, 1676, 22, 0, 1828, 200, 614, 848, 8, 0, 4063, 1428, 100, 0, 3799, 1293, 349, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11450, 23035, 15553, 15718, 17593, 13208, 17721, 16794] avg=16384.0000 max=23035 max_rank=1 min=11450 min_rank=0 imbalance_ratio=1.405945 max_load_reduction=108037 max_load_reduction_pct=0.824257 moved_to_echo=65316 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[687, 0, 3339, 14, 2220, 0, 28, 4051, 0, 223, 69, 183, 2736, 7, 8, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2650, 0, 151, 3876, 1515, 2859, 46, 168, 21, 505, 0, 1301, 1858, 0, 2111, 2044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 1870, 3428, 1, 5, 20, 274, 0, 733, 1741, 36, 0, 5, 60, 701, 29, 4091, 3710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 15, 306, 2862, 19, 20, 4, 4090, 264, 241, 1068, 128, 13, 0, 3867, 1840, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 3762, 688, 100, 0, 3822, 3, 260, 659, 392, 344, 118, 0, 3904, 0, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1773, 2494, 501, 3990, 14, 1874, 2689, 0, 2772, 2187, 0, 14, 42, 3, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 79, 0, 117, 2032, 0, 3990, 0, 65, 3680, 3647, 678, 1524, 2178, 937, 1191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 6, 571, 0, 0, 3943, 209, 402, 276, 0, 39, 1059, 0, 0, 2, 1309, 1498, 305, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14822, 19105, 16901, 15422, 15710, 19033, 20128, 9951] avg=16384.0000 max=20128 max_rank=6 min=9951 min_rank=7 imbalance_ratio=1.228516 max_load_reduction=110944 max_load_reduction_pct=0.846436 moved_to_echo=64822 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[220, 14, 3154, 1945, 2336, 5, 301, 3891, 0, 3235, 182, 3, 1718, 1518, 0, 2768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 11, 3447, 3, 27, 3045, 16, 73, 0, 7, 6, 245, 47, 2984, 3998, 0, 673, 1957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 1935, 3493, 0, 0, 40, 19, 0, 565, 871, 0, 1, 9, 1, 0, 8, 4054, 3902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 255, 0, 276, 3704, 1, 0, 1, 3471, 363, 3469, 261, 0, 84, 318, 4094, 1763, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3952, 393, 1347, 10, 2666, 0, 1608, 2923, 1, 25, 1, 10, 3879, 6, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1984, 134, 2, 3921, 18, 1411, 2383, 2, 21, 2274, 0, 9, 392, 50, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2020, 0, 1357, 615, 6, 325, 0, 77, 2298, 3922, 10, 242, 677, 106, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 52, 1019, 5, 4, 2484, 1975, 11, 31, 0, 3, 3487, 108, 17, 29, 4026, 452, 97, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21290, 16567, 18104, 18266, 17850, 12652, 12312, 14031] avg=16384.0000 max=21290 max_rank=0 min=12312 min_rank=6 imbalance_ratio=1.299438 max_load_reduction=109782 max_load_reduction_pct=0.837570 moved_to_echo=56845 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[779, 0, 4093, 543, 2684, 0, 51, 1774, 0, 632, 0, 3, 2014, 1361, 0, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 72, 0, 0, 2138, 293, 1553, 3, 422, 1, 3, 15, 3492, 2689, 1, 2390, 2152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2212, 2304, 2724, 5, 324, 6, 0, 0, 2545, 0, 0, 128, 92, 87, 5, 71, 4090, 3870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 48, 3436, 0, 8, 1, 3969, 90, 3133, 895, 0, 28, 154, 3091, 1937, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3250, 166, 1680, 4, 3981, 0, 1303, 0, 15, 61, 1, 0, 2384, 0, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3191, 566, 2070, 3523, 117, 680, 2646, 0, 1235, 749, 0, 35, 0, 1, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 686, 0, 1735, 2113, 66, 3597, 6, 4, 273, 4073, 4, 3422, 5, 297, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 107, 0, 669, 53, 2635, 181, 90, 3867, 16, 0, 3217, 49, 59, 9, 3088, 1557, 345, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17863, 15244, 18463, 16965, 13987, 15702, 16563, 16285] avg=16384.0000 max=18463 max_rank=2 min=13987 min_rank=4 imbalance_ratio=1.126892 max_load_reduction=112609 max_load_reduction_pct=0.859138 moved_to_echo=62537 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[274, 173, 3446, 925, 2094, 0, 37, 3247, 0, 176, 1818, 254, 1188, 207, 0, 2327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 3144, 40, 507, 1856, 516, 5, 15, 4, 0, 116, 0, 88, 2369, 0, 868, 1782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 986, 3666, 0, 285, 0, 3382, 2, 19, 18, 0, 111, 0, 0, 299, 0, 4047, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 62, 98, 0, 2725, 142, 23, 58, 58, 4064, 134, 1441, 2867, 0, 3, 3, 3574, 1604, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 78, 3128, 1262, 0, 0, 3807, 43, 3103, 2159, 370, 347, 114, 0, 1144, 0, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1113, 921, 3800, 3584, 135, 669, 2399, 15, 106, 2659, 0, 6, 123, 0, 3250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 60, 2028, 0, 4036, 0, 1, 3533, 162, 166, 2939, 1604, 227, 2149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1438, 798, 0, 0, 4066, 285, 438, 1670, 0, 0, 1524, 63, 17, 12, 4069, 1502, 128, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16166, 11316, 17780, 17071, 16599, 18780, 17125, 16235] avg=16384.0000 max=18780 max_rank=5 min=11316 min_rank=1 imbalance_ratio=1.146240 max_load_reduction=112292 max_load_reduction_pct=0.856720 moved_to_echo=68739 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 1152, 3328, 128, 3712, 15360, 640, 128, 512, 30976, 5248, 15488, 8320, 512, 256, 768, 30848, 13440, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[16896, 8960, 24960, 128, 768, 128, 3968, 128, 6400, 3200, 128, 640, 256, 256, 2304, 384, 32640, 30080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[5248, 256, 23680, 5376, 22912, 128, 1152, 22016, 0, 6656, 4480, 768, 15488, 4480, 128, 19200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 16384, 12032, 12672, 27392, 3328, 10112, 20352, 128, 4992, 12032, 0, 5120, 768, 128, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 384, 14720, 256, 2176, 24576, 3584, 6784, 1920, 1152, 128, 2560, 512, 16000, 25856, 128, 16128, 14976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[768, 3072, 8576, 2432, 384, 23552, 5888, 2432, 12032, 256, 128, 25216, 4352, 1536, 384, 26112, 11136, 2432, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 4736, 0, 7936, 14976, 128, 21888, 256, 384, 13184, 21376, 2304, 19456, 10112, 1792, 13312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 2304, 25856, 2688, 4864, 128, 28928, 128, 14208, 15616, 1664, 2688, 3200, 128, 20608, 128, 8832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4096, 0, 2890, 22, 4, 4082, 0, 4092, 0, 4090, 0, 4081, 3477, 0, 2, 0, 146, 0, 3, 0, 31, 2018, 0, 0, 3123, 643, 0, 0, 0, 2412, 2, 14, 3877, 0, 0, 50, 0, 49, 2, 0, 3929, 0, 150, 4069, 3967, 244, 114, 0, 46, 4089, 0, 0, 4, 23, 16, 640, 10, 0, 19, 0, 3945, 3898, 21, 3408, 796, 1, 853, 6, 0, 288, 4095, 4, 4, 1063, 3457, 21, 0, 134, 0, 0, 23, 3197, 2436, 3371, 1157, 4038, 4054, 4096, 0, 19, 1090, 0, 0, 4, 2, 0, 0, 344, 4091, 0, 86, 0, 1, 936, 0, 0, 4071, 77, 0, 4003, 0, 0, 334, 2, 4089, 2351, 3614, 3782, 0, 9, 2, 0, 14, 7, 940, 0, 275, 37] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4081, 0, 4090, 3, 2675, 126, 5, 4063, 0, 4036, 0, 3062, 3495, 574, 0, 3, 0, 294, 0, 0, 0, 1, 23, 25, 41, 0, 2887, 31, 622, 4023, 1, 144, 3180, 2, 78, 0, 0, 3000, 1778, 0, 3974, 0, 1504, 3355, 4058, 2379, 3, 0, 0, 3952, 1, 594, 0, 0, 74, 0, 76, 0, 0, 549, 3672, 4094, 0, 1419, 1858, 0, 4022, 3, 1, 2978, 3715, 20, 29, 0, 1218, 778, 0, 3, 0, 0, 0, 1907, 1671, 132, 4093, 4089, 3796, 4096, 5, 2551, 0, 971, 0, 89, 20, 3, 0, 52, 0, 0, 5, 654, 0, 145, 4, 0, 1961, 21, 0, 11, 134, 0, 0, 7, 4019, 3670, 3963, 135, 21, 0, 13, 88, 2875, 66, 1, 0, 329, 2803] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1997, 0, 3798, 123, 0, 4068, 0, 61, 0, 4080, 0, 3957, 69, 1660, 24, 16, 5, 1730, 0, 2, 12, 0, 676, 539, 2, 928, 36, 36, 0, 4095, 0, 3025, 3053, 11, 9, 0, 5, 0, 3757, 0, 2803, 0, 0, 0, 4072, 125, 0, 211, 0, 3848, 0, 357, 0, 0, 86, 1, 16, 0, 3, 4004, 3642, 4096, 6, 4014, 1536, 0, 1619, 16, 275, 1277, 4079, 991, 1469, 3, 3790, 243, 23, 0, 4, 0, 438, 3752, 872, 0, 3275, 4055, 1127, 4071, 0, 3495, 10, 1976, 0, 653, 2747, 16, 0, 0, 4080, 0, 683, 1, 0, 380, 2, 22, 58, 0, 4, 3514, 0, 0, 3025, 624, 3972, 2713, 35, 158, 20, 0, 342, 573, 8, 0, 18, 1, 9, 3960] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[981, 0, 4012, 278, 988, 83, 70, 3523, 1, 3722, 0, 3553, 1642, 3557, 0, 578, 0, 111, 1, 2, 376, 27, 637, 0, 138, 0, 0, 1, 37, 4084, 67, 4007, 4096, 12, 0, 502, 54, 2, 1092, 0, 4091, 45, 51, 230, 4090, 1728, 355, 1, 1, 3528, 843, 0, 6, 0, 2, 0, 1331, 87, 0, 4085, 3861, 3903, 3332, 3111, 0, 46, 53, 0, 0, 3871, 3198, 2, 2555, 2, 2371, 367, 13, 91, 0, 0, 0, 4061, 1, 3198, 1, 3685, 3922, 4074, 0, 175, 0, 0, 35, 0, 529, 1, 1, 20, 3349, 283, 0, 404, 0, 0, 64, 0, 109, 1, 0, 4091, 111, 297, 388, 992, 4078, 4096, 2763, 570, 0, 30, 185, 4, 0, 5, 181, 0, 63, 3821] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[3860, 11, 42, 4043, 2569, 3574, 0, 72, 0, 4077, 6, 1985, 4040, 169, 30, 3223, 0, 0, 0, 132, 4048, 11, 0, 0, 1344, 3172, 807, 21, 2, 4022, 33, 0, 2904, 0, 15, 236, 3, 2834, 0, 1, 3255, 44, 1037, 141, 863, 536, 138, 539, 1, 3865, 4089, 0, 182, 0, 0, 2912, 0, 0, 0, 258, 4096, 3700, 1386, 4096, 3597, 262, 3778, 0, 53, 0, 2031, 0, 65, 14, 723, 1270, 0, 2, 8, 0, 1239, 435, 2, 156, 193, 4007, 3113, 4037, 0, 1738, 0, 47, 0, 0, 13, 0, 27, 0, 2699, 263, 0, 0, 0, 0, 118, 0, 3604, 233, 0, 4093, 19, 0, 0, 0, 4096, 4074, 0, 0, 0, 2683, 1365, 46, 142, 0, 0, 0, 52, 2351] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[3494, 0, 3429, 0, 551, 4069, 1, 3255, 0, 4038, 154, 3395, 60, 238, 318, 13, 114, 3649, 0, 0, 1097, 0, 289, 0, 1332, 4003, 9, 2, 17, 3233, 6, 25, 7, 28, 0, 0, 0, 1735, 2585, 458, 1210, 816, 25, 3, 3795, 141, 30, 0, 170, 4081, 20, 3183, 8, 22, 4046, 2191, 18, 0, 554, 4095, 2916, 3756, 385, 3587, 4065, 0, 3958, 0, 0, 5, 3813, 562, 0, 0, 4084, 0, 0, 8, 0, 1902, 91, 3836, 3370, 15, 2, 4058, 3272, 3559, 17, 289, 35, 106, 6, 0, 2600, 12, 0, 0, 1811, 64, 0, 103, 939, 0, 403, 608, 0, 17, 77, 1811, 0, 3, 922, 0, 3881, 1697, 0, 709, 0, 0, 0, 26, 805, 0, 0, 258, 0, 617] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4085, 0, 4042, 729, 40, 3947, 0, 1395, 2, 4086, 0, 4035, 7, 393, 0, 0, 1, 485, 1, 0, 47, 10, 0, 0, 2892, 3027, 3890, 0, 53, 3696, 3, 2182, 3080, 2, 114, 147, 19, 2358, 14, 0, 4093, 5, 0, 22, 924, 7, 0, 0, 46, 4094, 0, 2, 16, 10, 8, 0, 444, 0, 0, 2989, 4086, 4080, 29, 4024, 1, 1, 1744, 1, 50, 1362, 3955, 3259, 1501, 236, 2512, 13, 0, 0, 0, 18, 0, 3175, 723, 0, 13, 4096, 661, 4095, 0, 4090, 0, 9, 4, 0, 32, 0, 0, 1, 4079, 0, 0, 0, 38, 1, 0, 5, 120, 900, 8, 2784, 12, 0, 56, 0, 4066, 4091, 3484, 0, 6, 0, 1466, 103, 2784, 0, 2057, 10, 58, 3731] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[3872, 0, 4096, 0, 286, 4093, 0, 2810, 0, 3941, 0, 4096, 1656, 1, 0, 0, 584, 1, 0, 0, 0, 0, 0, 178, 401, 0, 3756, 224, 141, 4052, 0, 475, 2896, 0, 4, 3859, 0, 3381, 613, 0, 4084, 1, 2009, 2, 3843, 0, 93, 0, 0, 3838, 3, 150, 0, 0, 185, 0, 0, 0, 16, 0, 2721, 3570, 786, 809, 5, 0, 3290, 144, 0, 891, 3804, 222, 11, 4046, 3092, 42, 0, 2614, 37, 0, 0, 765, 7, 11, 191, 4081, 4089, 4091, 0, 4073, 0, 0, 0, 90, 3893, 5, 4, 0, 4094, 0, 82, 18, 0, 0, 8, 1, 8, 2443, 198, 107, 2, 3, 961, 3, 4044, 357, 4058, 785, 187, 8, 4094, 0, 0, 0, 2400, 0, 0, 187] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[874, 0, 4012, 278, 988, 83, 70, 3523, 1, 0, 0, 0, 1642, 3557, 0, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 1, 2, 376, 27, 637, 0, 138, 0, 0, 1, 37, 4084, 67, 4007, 3174, 1562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 502, 54, 2, 1092, 0, 4091, 45, 51, 230, 4090, 1728, 355, 1, 496, 126, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 241, 843, 0, 6, 0, 2, 0, 1331, 87, 0, 4085, 3861, 3903, 3332, 3111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 53, 0, 0, 3871, 3198, 2, 2555, 2, 2371, 367, 13, 91, 0, 0, 1865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 1, 3198, 1, 3189, 3922, 0, 0, 175, 0, 0, 35, 0, 529, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 3349, 283, 0, 404, 0, 0, 64, 0, 109, 1, 0, 4091, 111, 297, 4074, 3722, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 992, 4078, 4096, 2763, 570, 0, 30, 185, 4, 0, 5, 181, 0, 63, 3821, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15606, 14224, 16994, 20803, 14434, 15112, 16639, 17260] avg=16384.0000 max=20803 max_rank=3 min=14224 min_rank=1 imbalance_ratio=1.269714 max_load_reduction=110269 max_load_reduction_pct=0.841286 moved_to_echo=63445 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3439, 0, 4096, 0, 286, 4093, 0, 2810, 0, 0, 0, 0, 1656, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 584, 1, 0, 0, 0, 0, 0, 178, 401, 0, 3756, 224, 141, 4052, 0, 475, 3457, 1803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2896, 0, 4, 3859, 0, 3381, 613, 0, 4084, 1, 2009, 2, 3843, 0, 93, 0, 554, 144, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 3, 150, 0, 0, 185, 0, 0, 0, 16, 0, 2721, 3570, 786, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3290, 144, 0, 891, 3804, 222, 11, 4046, 3092, 42, 0, 2614, 37, 0, 2149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 765, 7, 11, 191, 3527, 4089, 0, 0, 4073, 0, 0, 0, 90, 3893, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4094, 0, 82, 18, 0, 0, 8, 1, 8, 2443, 198, 107, 2, 3, 4091, 3941, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 961, 3, 4044, 357, 4058, 785, 187, 8, 4094, 0, 0, 0, 2400, 0, 0, 187, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 15072, 21580, 8495, 20347, 16651, 15126, 17420] avg=16384.0000 max=21580 max_rank=2 min=8495 min_rank=3 imbalance_ratio=1.317139 max_load_reduction=109492 max_load_reduction_pct=0.835358 moved_to_echo=69544 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3646, 0, 2890, 22, 4, 4082, 0, 4092, 0, 0, 0, 0, 3477, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 3, 0, 31, 2018, 0, 0, 3123, 643, 0, 0, 0, 2412, 2, 14, 3679, 1795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 0, 50, 0, 49, 2, 0, 3929, 0, 150, 4069, 3967, 244, 114, 0, 544, 145, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 279, 0, 0, 4, 23, 16, 640, 10, 0, 19, 0, 3945, 3898, 21, 3408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 796, 1, 853, 6, 0, 288, 4095, 4, 4, 1063, 3457, 21, 0, 134, 0, 0, 2141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3197, 2436, 3371, 1157, 3494, 4054, 0, 0, 19, 1090, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 4091, 0, 86, 0, 1, 936, 0, 0, 4071, 77, 0, 4003, 0, 0, 4096, 4090, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 2, 4089, 2351, 3614, 3782, 0, 9, 2, 0, 14, 7, 940, 0, 275, 37, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18215, 13866, 17239, 12309, 12863, 18847, 21926, 15807] avg=16384.0000 max=21926 max_rank=6 min=12309 min_rank=3 imbalance_ratio=1.338257 max_load_reduction=109146 max_load_reduction_pct=0.832718 moved_to_echo=69443 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3633, 0, 4090, 3, 2675, 126, 5, 4063, 0, 0, 0, 0, 3495, 574, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 0, 0, 1, 23, 25, 41, 0, 2887, 31, 622, 4023, 1, 144, 3555, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3180, 2, 78, 0, 0, 3000, 1778, 0, 3974, 0, 1504, 3355, 4058, 2379, 3, 0, 551, 110, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 1, 594, 0, 0, 74, 0, 76, 0, 0, 549, 3672, 4094, 0, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1858, 0, 4022, 3, 1, 2978, 3715, 20, 29, 0, 1218, 778, 0, 3, 0, 0, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1907, 1671, 132, 4093, 3538, 3796, 0, 5, 2551, 0, 971, 0, 89, 20, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 5, 654, 0, 145, 4, 0, 1961, 21, 0, 11, 134, 0, 4096, 4036, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4019, 3670, 3963, 135, 21, 0, 13, 88, 2875, 66, 1, 0, 329, 2803, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18667, 12993, 24070, 10750, 16231, 18776, 11245, 18340] avg=16384.0000 max=24070 max_rank=2 min=10750 min_rank=3 imbalance_ratio=1.469116 max_load_reduction=107002 max_load_reduction_pct=0.816360 moved_to_echo=64592 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3436, 11, 42, 4043, 2569, 3574, 0, 72, 0, 0, 6, 0, 4040, 169, 30, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 4048, 11, 0, 0, 1344, 3172, 807, 21, 2, 4022, 33, 0, 3477, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2904, 0, 15, 236, 3, 2834, 0, 1, 3255, 44, 1037, 141, 863, 536, 138, 539, 540, 70, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 264, 4089, 0, 182, 0, 0, 2912, 0, 0, 0, 258, 4096, 3700, 1386, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3597, 262, 3778, 0, 53, 0, 2031, 0, 65, 14, 723, 1270, 0, 2, 8, 0, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1239, 435, 2, 156, 193, 3467, 3113, 0, 0, 1738, 0, 47, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 2699, 263, 0, 0, 0, 0, 118, 0, 3604, 233, 0, 4093, 19, 0, 4037, 4077, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4074, 0, 0, 0, 2683, 1365, 46, 142, 0, 0, 0, 52, 2351, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21215, 17942, 13249, 20984, 12845, 10403, 19294, 15140] avg=16384.0000 max=21215 max_rank=0 min=10403 min_rank=5 imbalance_ratio=1.294861 max_load_reduction=109857 max_load_reduction_pct=0.838142 moved_to_echo=57682 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3111, 0, 3429, 0, 551, 4069, 1, 3255, 0, 0, 154, 0, 60, 238, 318, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 3649, 0, 0, 1097, 0, 289, 0, 1332, 4003, 9, 2, 17, 3233, 6, 25, 3672, 1493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 28, 0, 0, 0, 1735, 2585, 458, 1210, 816, 25, 3, 3795, 141, 30, 0, 547, 120, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 278, 20, 3183, 8, 22, 4046, 2191, 18, 0, 554, 4095, 2916, 3756, 385, 3587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 3958, 0, 0, 5, 3813, 562, 0, 0, 4084, 0, 0, 8, 0, 1902, 1782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 3836, 3370, 15, 2, 3511, 3272, 0, 17, 289, 35, 106, 6, 0, 2600, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1811, 64, 0, 103, 939, 0, 403, 608, 0, 17, 77, 1811, 0, 3, 3559, 4038, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 922, 0, 3881, 1697, 0, 709, 0, 0, 0, 26, 805, 0, 0, 258, 0, 617, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15199, 18941, 11584, 25229, 20179, 17162, 13564, 9214] avg=16384.0000 max=25229 max_rank=3 min=9214 min_rank=7 imbalance_ratio=1.539856 max_load_reduction=105843 max_load_reduction_pct=0.807518 moved_to_echo=60119 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1778, 0, 3798, 123, 0, 4068, 0, 61, 0, 0, 0, 0, 69, 1660, 24, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1730, 0, 2, 12, 0, 676, 539, 2, 928, 36, 36, 0, 4095, 0, 3025, 3462, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3053, 11, 9, 0, 5, 0, 3757, 0, 2803, 0, 0, 0, 4072, 125, 0, 211, 546, 139, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 0, 357, 0, 0, 86, 1, 16, 0, 3, 4004, 3642, 4096, 6, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1536, 0, 1619, 16, 275, 1277, 4079, 991, 1469, 3, 3790, 243, 23, 0, 4, 0, 2076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 3752, 872, 0, 3275, 3509, 1127, 0, 0, 3495, 10, 1976, 0, 653, 2747, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 683, 1, 0, 380, 2, 22, 58, 0, 4, 3514, 0, 0, 4071, 4080, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3025, 624, 3972, 2713, 35, 158, 20, 0, 342, 573, 8, 0, 18, 1, 9, 3960, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11597, 16290, 14779, 16488, 17401, 21870, 17018, 15629] avg=16384.0000 max=21870 max_rank=5 min=11597 min_rank=0 imbalance_ratio=1.334839 max_load_reduction=109202 max_load_reduction_pct=0.833145 moved_to_echo=71918 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3637, 0, 4042, 729, 40, 3947, 0, 1395, 2, 0, 0, 0, 7, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 485, 1, 0, 47, 10, 0, 0, 2892, 3027, 3890, 0, 53, 3696, 3, 2182, 3683, 1774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3080, 2, 114, 147, 19, 2358, 14, 0, 4093, 5, 0, 22, 924, 7, 0, 0, 552, 142, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 280, 0, 2, 16, 10, 8, 0, 444, 0, 0, 2989, 4086, 4080, 29, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1744, 1, 50, 1362, 3955, 3259, 1501, 236, 2512, 13, 0, 0, 0, 18, 2119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3175, 723, 0, 13, 3544, 661, 0, 0, 4090, 0, 9, 4, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4079, 0, 0, 0, 38, 1, 0, 5, 120, 900, 8, 2784, 12, 0, 4095, 4086, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 4066, 4091, 3484, 0, 6, 0, 1466, 103, 2784, 0, 2057, 10, 58, 3731, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14192, 21744, 11577, 16014, 16772, 12251, 16260, 22262] avg=16384.0000 max=22262 max_rank=7 min=11577 min_rank=2 imbalance_ratio=1.358765 max_load_reduction=108810 max_load_reduction_pct=0.830154 moved_to_echo=67545 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[11904, 384, 19328, 256, 384, 10752, 28800, 5120, 5760, 5376, 21248, 2816, 128, 2944, 128, 1920, 14848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[384, 2176, 4992, 4352, 256, 128, 4480, 5760, 1920, 128, 640, 16000, 29056, 31104, 6016, 24576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[896, 6272, 128, 256, 5632, 2176, 1664, 768, 9344, 11776, 11392, 384, 896, 29696, 128, 9984, 28160, 12416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[23680, 128, 26496, 5248, 7168, 24064, 128, 19328, 128, 0, 256, 0, 14464, 6656, 384, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 512, 24320, 640, 896, 1280, 1024, 1536, 640, 640, 9984, 3712, 384, 20480, 384, 384, 32128, 32128, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[5760, 1664, 32256, 23168, 17920, 6144, 256, 2816, 7552, 896, 6656, 128, 5632, 384, 896, 17536, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[23168, 128, 256, 4864, 128, 13440, 9856, 512, 27520, 1024, 4864, 7936, 25728, 5248, 768, 768, 4352, 1024, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1792, 21248, 9088, 6912, 8960, 27904, 24064, 0, 128, 16512, 1152, 3200, 128, 896, 9856, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[8, 225, 4020, 4051, 0, 560, 29, 1879, 119, 3909, 163, 4091, 3183, 2755, 9, 0, 3294, 0, 47, 1424, 3749, 29, 5, 3, 0, 4096, 0, 0, 31, 24, 0, 3949, 188, 4, 0, 4094, 13, 1, 1, 0, 782, 3984, 813, 12, 364, 32, 182, 153, 0, 1, 0, 0, 3566, 1, 611, 1285, 3439, 1, 689, 3750, 55, 15, 0, 0, 7, 3203, 13, 36, 1, 0, 611, 4094, 4, 0, 43, 3622, 61, 91, 0, 156, 0, 0, 0, 3818, 6, 0, 116, 4048, 5, 451, 2144, 0, 1610, 142, 320, 1451, 11, 3908, 3327, 18, 395, 2217, 4090, 604, 7, 723, 3570, 0, 0, 0, 1533, 877, 1, 3116, 0, 0, 3553, 2717, 0, 161, 3, 3, 3998, 21, 4050, 0, 425, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 2, 2438, 0, 23, 482, 110, 8, 2126, 0, 17, 2853, 3647, 2665, 3, 0, 3847, 4010, 0, 2982, 1605, 1022, 10, 0, 0, 0, 0, 0, 1088, 133, 2751, 432, 1547, 1, 0, 0, 1, 11, 3923, 2341, 0, 0, 13, 3875, 678, 2617, 36, 0, 0, 3981, 115, 286, 4083, 4034, 3890, 0, 34, 4, 1179, 4055, 3958, 835, 1, 444, 13, 236, 0, 169, 4079, 0, 529, 0, 105, 569, 0, 85, 2, 0, 0, 0, 0, 0, 0, 3359, 3939, 0, 276, 4095, 12, 0, 402, 970, 0, 2732, 1678, 0, 0, 31, 3323, 0, 3909, 3986, 0, 46, 2, 2929, 5, 0, 0, 708, 372, 3321, 0, 3737, 446, 0, 733, 4, 10, 2, 901, 1, 2020, 28, 3236, 4, 3854, 18] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 836, 799, 650, 0, 0, 3135, 0, 3753, 142, 4, 4045, 4093, 3255, 18, 0, 20, 4071, 433, 3947, 4077, 419, 0, 2, 3, 0, 0, 2, 3692, 267, 22, 61, 3, 29, 355, 4063, 26, 2, 912, 85, 4, 2705, 1311, 4086, 3951, 864, 318, 0, 1, 3787, 0, 0, 1848, 2688, 4068, 69, 0, 3, 719, 3928, 0, 0, 0, 0, 0, 4096, 0, 1738, 3619, 88, 5, 1, 209, 4, 6, 949, 0, 195, 0, 2, 0, 0, 0, 1, 3943, 2, 2646, 4096, 4094, 0, 97, 0, 0, 2392, 891, 4092, 0, 46, 3167, 283, 35, 3507, 0, 46, 6, 860, 0, 1, 0, 0, 153, 6, 0, 580, 3645, 0, 3810, 0, 0, 0, 3939, 0, 4082, 0, 4, 0, 146, 19] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 29, 3727, 0, 77, 0, 40, 0, 4062, 4051, 0, 1651, 3832, 4095, 0, 0, 4096, 0, 0, 6, 4021, 0, 0, 24, 420, 3523, 1, 2, 0, 0, 2, 4095, 2, 0, 0, 2436, 0, 254, 609, 0, 0, 2, 3850, 296, 3960, 0, 1, 6, 804, 1376, 1446, 0, 4030, 6, 2212, 3503, 0, 0, 1, 1845, 0, 3, 21, 399, 4096, 0, 0, 0, 1023, 20, 4, 416, 8, 699, 4094, 478, 5, 190, 1012, 3123, 8, 275, 0, 4068, 270, 0, 18, 4088, 47, 1, 4052, 0, 0, 4096, 176, 0, 11, 1682, 4086, 11, 2063, 3527, 0, 0, 20, 0, 9, 4, 0, 58, 4092, 2290, 110, 3618, 0, 1, 4083, 536, 9, 0, 1, 6, 4070, 0, 3289, 3, 410, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1, 2, 2778, 2, 0, 126, 104, 8, 4096, 2308, 3793, 4087, 3758, 785, 1581, 0, 4092, 4000, 23, 2451, 4019, 3, 0, 0, 4, 0, 0, 40, 2325, 0, 3422, 2949, 0, 2, 296, 53, 5, 0, 0, 0, 0, 3821, 2847, 1471, 63, 1678, 69, 0, 0, 0, 0, 51, 4072, 0, 4019, 1790, 9, 260, 1924, 3908, 1, 15, 12, 1062, 57, 2, 0, 1, 0, 0, 0, 535, 248, 113, 0, 225, 0, 583, 3, 4090, 106, 5, 0, 459, 3444, 0, 3769, 4096, 4051, 0, 3340, 0, 0, 4089, 0, 437, 3, 1156, 2756, 9, 3923, 86, 10, 4082, 0, 266, 18, 2, 0, 0, 0, 1884, 0, 858, 0, 1, 262, 0, 0, 0, 1, 0, 3596, 143, 4096, 0, 4082, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 1, 43, 19, 4084, 3156, 3908, 4074, 1996, 4093, 0, 68, 4063, 1, 0, 3316, 537, 0, 50, 94, 278, 4096, 0, 0, 1273, 0, 3400, 4096, 0, 562, 0, 98, 1303, 1, 0, 0, 0, 2341, 7, 0, 316, 6, 0, 3, 0, 687, 0, 3737, 3783, 1, 73, 0, 34, 0, 2333, 3567, 1, 0, 0, 69, 3810, 564, 0, 0, 0, 0, 392, 31, 15, 0, 19, 217, 0, 0, 7, 0, 6, 16, 0, 4096, 3614, 0, 44, 3885, 1903, 1831, 109, 821, 1505, 3210, 15, 398, 0, 0, 4086, 0, 2645, 4041, 3374, 3340, 0, 3324, 2, 1, 42, 865, 3607, 935, 0, 3992, 2811, 0, 272, 660, 0, 260, 286, 0, 2416, 2, 1712, 16, 201, 12] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[29, 0, 2763, 697, 3392, 0, 144, 0, 4093, 608, 2563, 3868, 3716, 1308, 0, 20, 3555, 0, 18, 119, 256, 0, 49, 0, 0, 4076, 0, 0, 3200, 0, 19, 4052, 0, 862, 40, 17, 653, 29, 2479, 0, 0, 2, 1821, 0, 3809, 65, 0, 3, 6, 1, 74, 0, 2596, 2, 3779, 3566, 788, 0, 2597, 3955, 13, 2, 1, 155, 3, 3814, 55, 0, 48, 0, 1683, 163, 213, 0, 1024, 4093, 0, 0, 961, 4, 0, 0, 10, 4087, 119, 0, 1515, 4078, 3557, 1369, 215, 19, 0, 4085, 1323, 149, 0, 2190, 4025, 0, 4096, 0, 0, 4032, 637, 127, 1, 0, 0, 0, 182, 2797, 0, 2269, 12, 0, 4083, 0, 0, 0, 0, 0, 4096, 0, 4060, 18, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 140, 68, 728, 0, 4, 0, 1, 486, 4068, 0, 4026, 4096, 3463, 0, 4, 3377, 3670, 0, 1, 4094, 1, 8, 0, 0, 0, 16, 4, 0, 0, 3723, 29, 0, 141, 1184, 1184, 418, 3, 3815, 0, 0, 45, 489, 27, 3, 0, 0, 7, 200, 387, 2110, 3865, 2413, 3787, 24, 4033, 0, 0, 1, 4096, 0, 27, 0, 0, 3589, 22, 0, 0, 4048, 10, 971, 0, 1, 978, 1739, 4070, 2, 0, 172, 569, 3866, 0, 2, 3786, 4096, 0, 0, 3986, 1, 345, 3883, 0, 0, 4091, 54, 30, 0, 12, 4041, 0, 1231, 2411, 0, 21, 0, 0, 2592, 0, 0, 6, 3966, 0, 83, 3972, 329, 19, 13, 3571, 0, 0, 1665, 0, 3629, 0, 2149, 0, 785, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 836, 799, 650, 0, 0, 3135, 0, 3753, 142, 4, 0, 2482, 3255, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4071, 433, 3947, 4077, 419, 0, 2, 3, 0, 0, 2, 3692, 267, 22, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 29, 355, 4063, 26, 2, 912, 85, 4, 2705, 1311, 4086, 3951, 864, 318, 0, 1603, 8, 1459, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3787, 0, 0, 1848, 2688, 4068, 69, 0, 3, 719, 2325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1738, 3619, 88, 5, 1, 209, 4, 6, 949, 0, 195, 0, 2, 3766, 2586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3943, 2, 2646, 330, 4094, 0, 97, 0, 0, 2392, 891, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 2329, 283, 35, 3507, 0, 46, 6, 860, 0, 1, 0, 0, 153, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 580, 3645, 0, 3810, 0, 0, 0, 3939, 0, 4082, 0, 4, 0, 146, 19, 838, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15074, 17008, 23163, 15508, 17264, 18488, 7272, 17295] avg=16384.0000 max=23163 max_rank=2 min=7272 min_rank=6 imbalance_ratio=1.413757 max_load_reduction=107909 max_load_reduction_pct=0.823280 moved_to_echo=60319 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 2438, 0, 23, 482, 110, 8, 2126, 0, 17, 0, 2212, 2665, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2173, 4010, 0, 2982, 1605, 1022, 10, 0, 0, 0, 0, 0, 1088, 133, 2751, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1547, 1, 0, 0, 1, 11, 3923, 2341, 0, 0, 13, 3875, 678, 2617, 36, 0, 1655, 1674, 1028, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 115, 286, 4083, 4034, 3890, 0, 34, 4, 1179, 2400, 3958, 835, 1, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 236, 0, 169, 4079, 0, 529, 0, 105, 569, 0, 85, 2, 0, 0, 0, 3765, 1825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3359, 3939, 0, 276, 330, 12, 0, 402, 970, 0, 2732, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 2444, 0, 3909, 3986, 0, 46, 2, 2929, 5, 0, 0, 708, 372, 3321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3737, 446, 0, 733, 4, 10, 2, 901, 1, 2020, 28, 3236, 4, 3854, 18, 879, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10086, 16206, 20629, 25244, 11377, 13698, 17753, 16079] avg=16384.0000 max=25244 max_rank=3 min=10086 min_rank=0 imbalance_ratio=1.540771 max_load_reduction=105828 max_load_reduction_pct=0.807404 moved_to_echo=58907 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 2778, 2, 0, 126, 104, 8, 4096, 2308, 3793, 0, 2279, 785, 1581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2312, 4000, 23, 2451, 4019, 3, 0, 0, 4, 0, 0, 40, 2325, 0, 3422, 2949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 296, 53, 5, 0, 0, 0, 0, 3821, 2847, 1471, 63, 1678, 69, 0, 1595, 1780, 1474, 1266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 4072, 0, 4019, 1790, 9, 260, 1924, 2313, 1, 15, 12, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 2, 0, 1, 0, 0, 0, 535, 248, 113, 0, 225, 0, 583, 3, 4090, 3766, 2613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 5, 0, 459, 3444, 0, 3769, 330, 4051, 0, 3340, 0, 0, 4089, 0, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1156, 2027, 9, 3923, 86, 10, 4082, 0, 266, 18, 2, 0, 0, 0, 1884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 858, 0, 1, 262, 0, 0, 0, 1, 0, 3596, 143, 4096, 0, 4082, 0, 729, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17863, 21548, 16420, 15528, 12236, 20030, 13466, 13981] avg=16384.0000 max=21548 max_rank=1 min=12236 min_rank=4 imbalance_ratio=1.315186 max_load_reduction=109524 max_load_reduction_pct=0.835602 moved_to_echo=59713 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 2763, 697, 3392, 0, 144, 0, 4093, 608, 2563, 0, 2254, 1308, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2008, 0, 18, 119, 256, 0, 49, 0, 0, 4076, 0, 0, 3200, 0, 19, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 862, 40, 17, 653, 29, 2479, 0, 0, 2, 1821, 0, 3809, 65, 0, 3, 1614, 1547, 1394, 1252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 74, 0, 2596, 2, 3779, 3566, 788, 0, 2597, 2341, 13, 2, 1, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3814, 55, 0, 48, 0, 1683, 163, 213, 0, 1024, 4093, 0, 0, 961, 4, 3750, 2474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4087, 119, 0, 1515, 328, 3557, 1369, 215, 19, 0, 4085, 1323, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2190, 2960, 0, 4096, 0, 0, 4032, 637, 127, 1, 0, 0, 0, 182, 2797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2269, 12, 0, 4083, 0, 0, 0, 0, 0, 4096, 0, 4060, 18, 0, 0, 1065, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17871, 13797, 15587, 15921, 18285, 16776, 17022, 15813] avg=16384.0000 max=18285 max_rank=4 min=13797 min_rank=1 imbalance_ratio=1.116028 max_load_reduction=112787 max_load_reduction_pct=0.860497 moved_to_echo=67896 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 29, 3727, 0, 77, 0, 40, 0, 4062, 4051, 0, 0, 2324, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2314, 0, 0, 6, 4021, 0, 0, 24, 420, 3523, 1, 2, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2436, 0, 254, 609, 0, 0, 2, 3850, 296, 3960, 0, 1, 6, 753, 1782, 596, 1291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 1376, 1446, 0, 4030, 6, 2212, 3503, 0, 0, 1, 1092, 0, 3, 21, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1023, 20, 4, 416, 8, 699, 4094, 478, 5, 190, 1012, 3123, 3759, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 275, 0, 4068, 270, 0, 18, 329, 47, 1, 4052, 0, 0, 4096, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1682, 3005, 11, 2063, 3527, 0, 0, 20, 0, 9, 4, 0, 58, 4092, 2290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 3618, 0, 1, 4083, 536, 9, 0, 1, 6, 4070, 0, 3289, 3, 410, 0, 1081, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18405, 14408, 15838, 14893, 19982, 13340, 16772, 17434] avg=16384.0000 max=19982 max_rank=4 min=13340 min_rank=5 imbalance_ratio=1.219604 max_load_reduction=111090 max_load_reduction_pct=0.847549 moved_to_echo=67528 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 1, 43, 19, 4084, 3156, 3908, 0, 1211, 4093, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2295, 1, 0, 3316, 537, 0, 50, 94, 278, 4096, 0, 0, 1273, 0, 3400, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 0, 98, 1303, 1, 0, 0, 0, 2341, 7, 0, 316, 6, 0, 3, 1456, 1768, 1468, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, 0, 3737, 3783, 1, 73, 0, 34, 0, 2333, 2111, 1, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 564, 0, 0, 0, 0, 392, 31, 15, 0, 19, 217, 0, 0, 7, 0, 3572, 2606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 16, 0, 4096, 3614, 0, 44, 313, 1903, 1831, 109, 821, 1505, 3210, 15, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3005, 0, 2645, 4041, 3374, 3340, 0, 3324, 2, 1, 42, 865, 3607, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 2811, 0, 272, 660, 0, 260, 286, 0, 2416, 2, 1712, 16, 201, 12, 1081, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20677, 19436, 10001, 12829, 11233, 17881, 25181, 13834] avg=16384.0000 max=25181 max_rank=6 min=10001 min_rank=2 imbalance_ratio=1.536926 max_load_reduction=105891 max_load_reduction_pct=0.807884 moved_to_echo=68129 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[8, 225, 4020, 4051, 0, 560, 29, 1879, 119, 3909, 163, 0, 1931, 2755, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1861, 0, 47, 1424, 3749, 29, 5, 3, 0, 4096, 0, 0, 31, 24, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 4, 0, 4094, 13, 1, 1, 0, 782, 3984, 813, 12, 364, 32, 182, 153, 1531, 1433, 1475, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3566, 1, 611, 1285, 3439, 1, 689, 2219, 55, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3203, 13, 36, 1, 0, 611, 4094, 4, 0, 43, 3622, 61, 91, 0, 156, 3722, 2616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3818, 6, 0, 116, 326, 5, 451, 2144, 0, 1610, 142, 320, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3908, 2447, 18, 395, 2217, 4090, 604, 7, 723, 3570, 0, 0, 0, 1533, 877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3116, 0, 0, 3553, 2717, 0, 161, 3, 3, 3998, 21, 4050, 0, 425, 3, 880, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19658, 15218, 16134, 11882, 18280, 10389, 20400, 19111] avg=16384.0000 max=20400 max_rank=6 min=10389 min_rank=5 imbalance_ratio=1.245117 max_load_reduction=110672 max_load_reduction_pct=0.844360 moved_to_echo=68180 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 140, 68, 728, 0, 4, 0, 1, 486, 4068, 0, 0, 2476, 3463, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1905, 3670, 0, 1, 4094, 1, 8, 0, 0, 0, 16, 4, 0, 0, 3723, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 1184, 1184, 418, 3, 3815, 0, 0, 45, 489, 27, 3, 0, 0, 7, 1676, 1472, 1452, 1384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 387, 2110, 3865, 2413, 3787, 24, 4033, 0, 0, 1, 2420, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3589, 22, 0, 0, 4048, 10, 971, 0, 1, 978, 1739, 4070, 2, 0, 172, 569, 3670, 2574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3866, 0, 2, 3786, 4096, 0, 0, 316, 1, 345, 3883, 0, 0, 4091, 54, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2967, 0, 1231, 2411, 0, 21, 0, 0, 2592, 0, 0, 6, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 3972, 329, 19, 13, 3571, 0, 0, 1665, 0, 3629, 0, 2149, 0, 785, 0, 1074, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11438, 13451, 13300, 19267, 22415, 20470, 13206, 17525] avg=16384.0000 max=22415 max_rank=4 min=11438 min_rank=0 imbalance_ratio=1.368103 max_load_reduction=108657 max_load_reduction_pct=0.828987 moved_to_echo=73616 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[4096, 384, 128, 23680, 19456, 128, 8448, 2688, 13696, 4096, 14336, 1920, 3200, 24960, 4480, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[14976, 15872, 640, 14336, 22400, 1536, 128, 128, 768, 15872, 128, 128, 11648, 512, 13440, 19712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 9088, 21248, 384, 18304, 19840, 7552, 12288, 768, 8320, 6272, 128, 128, 1664, 13952, 12160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 1280, 16640, 6144, 7680, 1280, 3712, 1920, 22912, 18304, 10496, 0, 17280, 22528, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[11648, 12032, 128, 2048, 12928, 128, 4224, 5248, 896, 2432, 7040, 13824, 128, 1152, 2176, 8064, 29824, 18432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[256, 22144, 7296, 128, 16896, 7552, 128, 512, 6912, 128, 28032, 256, 22656, 128, 9984, 128, 7680, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1024, 10240, 3840, 8064, 26496, 10624, 18688, 14336, 4352, 384, 9472, 17280, 4096, 1024, 128, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1792, 1664, 1920, 12032, 2432, 384, 11776, 2432, 896, 12928, 11264, 9856, 13184, 5376, 640, 256, 11904, 11520, 10368, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4047, 1191, 3958, 0, 179, 0, 0, 404, 57, 0, 3578, 4085, 0, 0, 2385, 392, 0, 0, 0, 0, 3799, 0, 0, 4093, 4060, 7, 1288, 0, 0, 4091, 599, 159, 5, 3254, 1, 20, 476, 0, 3078, 4072, 4096, 2617, 22, 42, 0, 1331, 0, 295, 0, 4006, 1, 573, 4060, 118, 58, 3168, 1003, 4, 3093, 0, 1, 25, 2210, 4090, 32, 16, 171, 2565, 19, 6, 7, 34, 2, 0, 0, 374, 49, 8, 138, 4, 0, 758, 228, 1847, 3385, 1, 0, 3426, 402, 0, 8, 2701, 302, 0, 1926, 111, 1933, 3718, 3190, 0, 58, 82, 4092, 0, 0, 4, 1, 1, 0, 36, 108, 3764, 77, 3318, 0, 0, 0, 0, 40, 0, 19, 0, 0, 4060, 3978, 343, 42, 3597] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[903, 2, 3886, 0, 2123, 350, 47, 0, 3325, 0, 310, 18, 1, 2692, 6, 3534, 0, 0, 0, 111, 3365, 5, 4091, 3879, 1389, 176, 19, 0, 894, 657, 0, 0, 2825, 3907, 54, 0, 4096, 4089, 3098, 0, 405, 0, 4016, 0, 125, 32, 3372, 34, 0, 760, 3276, 4081, 675, 0, 0, 2339, 1198, 2, 0, 0, 1, 90, 3796, 640, 12, 1743, 0, 0, 1608, 10, 4094, 35, 1, 0, 0, 0, 0, 443, 166, 0, 3, 85, 3628, 0, 4096, 1724, 6, 2063, 597, 26, 1282, 67, 4089, 0, 719, 2, 0, 58, 0, 0, 2574, 2, 3659, 12, 0, 0, 581, 4048, 14, 20, 0, 4048, 0, 4093, 0, 1, 14, 1, 2688, 1111, 11, 3953, 3, 2565, 7, 416, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4040, 2616, 4093, 0, 532, 1, 11, 3, 3957, 171, 45, 4096, 1, 0, 3838, 1492, 0, 0, 0, 3200, 1, 0, 0, 4096, 102, 94, 4094, 0, 0, 4091, 1082, 12, 0, 4072, 0, 23, 1, 0, 3106, 4094, 0, 0, 4075, 24, 3369, 3901, 1064, 4070, 0, 4007, 1651, 4064, 86, 0, 10, 333, 0, 0, 0, 0, 3826, 35, 3587, 502, 0, 1604, 1, 24, 0, 53, 3510, 0, 4091, 0, 39, 1, 4, 282, 8, 0, 3065, 0, 0, 0, 4090, 3, 0, 161, 0, 0, 0, 4003, 3916, 0, 12, 0, 0, 0, 0, 159, 10, 0, 4086, 7, 67, 18, 0, 311, 0, 2723, 21, 4015, 0, 150, 0, 28, 0, 127, 4070, 0, 1, 0, 445, 2287, 37, 0, 6, 69] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4095, 4043, 4086, 0, 382, 313, 4, 578, 1, 0, 0, 1946, 192, 0, 0, 241, 4096, 0, 22, 666, 4070, 0, 0, 4095, 4082, 3422, 3805, 0, 8, 3097, 8, 0, 0, 1, 0, 1, 0, 0, 4090, 4095, 45, 636, 2431, 4015, 0, 0, 0, 3578, 403, 3605, 0, 3901, 1885, 359, 2, 3823, 0, 0, 62, 0, 3475, 80, 0, 3914, 1, 222, 13, 178, 0, 0, 886, 2, 0, 18, 0, 7, 340, 0, 0, 16, 0, 162, 0, 3711, 217, 1, 20, 1142, 14, 0, 0, 0, 3699, 0, 2611, 0, 0, 67, 752, 3644, 0, 0, 3907, 0, 0, 0, 0, 3701, 0, 1632, 4, 4076, 0, 4085, 313, 1, 0, 0, 0, 139, 199, 0, 37, 3411, 4093, 0, 3, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[3971, 3811, 4096, 135, 0, 9, 0, 1, 728, 4, 0, 4092, 0, 4, 0, 1510, 186, 0, 0, 4094, 6, 0, 0, 4071, 4096, 302, 530, 257, 855, 12, 3848, 0, 0, 290, 1796, 426, 0, 4, 4048, 218, 4061, 3961, 2531, 0, 0, 2, 2, 3731, 0, 3922, 12, 3995, 3990, 0, 202, 4080, 0, 2, 0, 1, 0, 124, 2818, 3459, 5, 15, 5, 107, 0, 0, 154, 5, 4096, 0, 411, 43, 0, 2, 1, 0, 0, 0, 5, 158, 2230, 1, 9, 3024, 0, 28, 1, 18, 4094, 1, 32, 0, 0, 597, 171, 3739, 0, 0, 4096, 27, 3900, 171, 0, 3291, 0, 0, 0, 3940, 106, 4096, 568, 0, 11, 4, 4005, 7, 0, 20, 122, 3929, 3307, 11, 0, 216] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[3813, 4096, 2965, 342, 0, 0, 147, 0, 4033, 1, 79, 3976, 0, 0, 0, 4096, 2, 68, 0, 4084, 43, 3, 0, 447, 2416, 0, 3581, 35, 0, 346, 4054, 420, 0, 3031, 0, 555, 568, 45, 4094, 584, 98, 3763, 3980, 0, 6, 0, 0, 21, 0, 4076, 1, 0, 4093, 0, 0, 3328, 0, 0, 3022, 0, 665, 0, 0, 4061, 0, 0, 194, 1, 748, 0, 40, 0, 1613, 0, 0, 97, 247, 11, 0, 13, 0, 3790, 667, 0, 143, 0, 994, 4096, 1076, 0, 0, 3909, 4058, 199, 4068, 17, 1, 404, 2634, 4, 0, 7, 3250, 3, 712, 0, 0, 2504, 0, 0, 1, 3821, 67, 4092, 0, 0, 8, 0, 2164, 78, 0, 3, 0, 4094, 85, 1766, 268, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4039, 3483, 3255, 12, 0, 254, 0, 0, 109, 0, 0, 22, 0, 25, 0, 4013, 3926, 0, 0, 1, 4064, 0, 4006, 9, 0, 233, 520, 263, 0, 3280, 2636, 0, 1938, 3063, 30, 0, 2, 0, 3268, 690, 2911, 3715, 2119, 15, 0, 20, 4096, 4095, 0, 3865, 3, 4096, 4090, 958, 0, 2803, 291, 0, 0, 0, 1540, 0, 3953, 3, 0, 36, 59, 0, 1, 58, 4088, 0, 678, 2, 3792, 12, 0, 14, 827, 884, 0, 0, 0, 0, 1530, 0, 0, 3, 0, 0, 1, 47, 3264, 0, 0, 2454, 851, 220, 3820, 66, 0, 0, 4089, 8, 1299, 0, 0, 1192, 0, 2881, 24, 4096, 0, 454, 302, 0, 0, 11, 3993, 0, 148, 5, 2, 4096, 4027, 0, 0, 24] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[3849, 2238, 4021, 0, 0, 652, 0, 0, 0, 0, 0, 4086, 0, 0, 1, 4079, 0, 3, 0, 3533, 2973, 67, 0, 3886, 3601, 4084, 2753, 87, 1611, 0, 0, 0, 0, 120, 785, 0, 3, 179, 4091, 518, 2895, 307, 4012, 3450, 0, 30, 0, 862, 783, 4096, 2, 3869, 3264, 713, 0, 2, 43, 0, 4, 0, 1174, 0, 0, 0, 36, 1, 0, 3516, 2, 4008, 2961, 37, 30, 1218, 0, 0, 1, 403, 975, 23, 0, 0, 11, 4077, 4091, 4032, 0, 3375, 20, 0, 12, 13, 3863, 38, 44, 0, 0, 0, 4085, 0, 5, 0, 3340, 0, 0, 1, 0, 3943, 0, 0, 0, 385, 0, 4071, 0, 0, 25, 4, 643, 0, 919, 0, 476, 4064, 3386, 0, 195, 17] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[903, 2, 914, 0, 2123, 350, 47, 0, 3325, 0, 310, 18, 1, 2692, 6, 3534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 3365, 5, 4091, 558, 1389, 176, 19, 0, 894, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2825, 3907, 54, 0, 4096, 4089, 0, 0, 405, 0, 1742, 0, 125, 32, 3372, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 3276, 4081, 675, 0, 0, 2339, 1198, 2, 0, 0, 1, 90, 3796, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1743, 0, 0, 1608, 10, 4094, 35, 1, 0, 0, 0, 0, 443, 166, 0, 3098, 676, 2903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 85, 3628, 0, 4096, 1724, 6, 2063, 597, 26, 1282, 67, 4089, 0, 719, 2, 1185, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 2574, 2, 3659, 12, 0, 0, 581, 4048, 14, 20, 0, 4048, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 14, 1, 2688, 1111, 11, 3953, 3, 2565, 7, 416, 0, 0, 2623, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14225, 11265, 20681, 16182, 14789, 20270, 16105, 17555] avg=16384.0000 max=20681 max_rank=2 min=11265 min_rank=1 imbalance_ratio=1.262268 max_load_reduction=110391 max_load_reduction_pct=0.842216 moved_to_echo=68719 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4095, 4043, 962, 0, 382, 313, 4, 578, 1, 0, 0, 1946, 192, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 22, 666, 4070, 0, 0, 589, 4082, 3422, 3805, 0, 8, 3097, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4095, 45, 636, 1055, 4015, 0, 0, 0, 3578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 398, 0, 3901, 1885, 359, 2, 3823, 0, 0, 62, 0, 3475, 80, 0, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 222, 13, 178, 0, 0, 886, 2, 0, 18, 0, 7, 340, 0, 0, 16, 4090, 3207, 3052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 3711, 217, 1, 20, 1142, 14, 0, 0, 0, 3699, 0, 2611, 0, 717, 737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 752, 3644, 0, 0, 3907, 0, 0, 0, 0, 3701, 0, 1632, 4, 4076, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 313, 1, 0, 0, 0, 139, 199, 0, 37, 3411, 4093, 0, 3, 4095, 2769, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12757, 23865, 13426, 18302, 12032, 13031, 18442, 19217] avg=16384.0000 max=23865 max_rank=1 min=12032 min_rank=4 imbalance_ratio=1.456604 max_load_reduction=107207 max_load_reduction_pct=0.817924 moved_to_echo=62722 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4047, 1191, 931, 0, 179, 0, 0, 404, 57, 0, 3578, 4085, 0, 0, 2385, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3799, 0, 0, 589, 4060, 7, 1288, 0, 0, 4091, 599, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3254, 1, 20, 476, 0, 0, 4072, 4096, 2617, 11, 42, 0, 1331, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 1, 573, 4060, 118, 58, 3168, 1003, 4, 3093, 0, 1, 25, 2210, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 16, 171, 2565, 19, 6, 7, 34, 2, 0, 0, 374, 49, 8, 138, 4, 3078, 3563, 2957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 758, 228, 1847, 3385, 1, 0, 3426, 402, 0, 8, 2701, 302, 0, 1926, 111, 6, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1933, 3718, 3190, 0, 58, 82, 4092, 0, 0, 4, 1, 1, 0, 36, 108, 3764, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 3318, 0, 0, 0, 0, 40, 0, 19, 0, 0, 4060, 3978, 343, 42, 3597, 2768, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17249, 14592, 16220, 18847, 13023, 15837, 16992, 18312] avg=16384.0000 max=18847 max_rank=3 min=13023 min_rank=4 imbalance_ratio=1.150330 max_load_reduction=112225 max_load_reduction_pct=0.856209 moved_to_echo=64164 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4040, 2616, 962, 0, 532, 1, 11, 3, 3957, 171, 45, 4096, 1, 0, 3838, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3200, 1, 0, 0, 589, 102, 94, 4094, 0, 0, 4091, 1082, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 23, 1, 0, 0, 4094, 0, 0, 1767, 24, 3369, 3901, 1064, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 1651, 4064, 86, 0, 10, 333, 0, 0, 0, 0, 3826, 35, 3587, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1604, 1, 24, 0, 53, 3510, 0, 4091, 0, 39, 1, 4, 282, 8, 0, 3106, 3564, 3058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3065, 0, 0, 0, 4090, 3, 0, 161, 0, 0, 0, 4003, 3916, 0, 12, 0, 1203, 737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 10, 0, 4086, 7, 67, 18, 0, 311, 0, 2723, 21, 4015, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 28, 0, 127, 4070, 0, 1, 0, 445, 2287, 37, 0, 6, 69, 2770, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21765, 13265, 22385, 14537, 19345, 17190, 12522, 10063] avg=16384.0000 max=22385 max_rank=2 min=10063 min_rank=7 imbalance_ratio=1.366272 max_load_reduction=108687 max_load_reduction_pct=0.829216 moved_to_echo=59120 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3971, 3811, 963, 135, 0, 9, 0, 1, 728, 4, 0, 4092, 0, 4, 0, 1510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 4094, 6, 0, 0, 586, 4096, 302, 530, 257, 855, 12, 3848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 1796, 426, 0, 4, 0, 218, 4061, 3961, 1098, 0, 0, 2, 2, 3731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 12, 3995, 3990, 0, 202, 4080, 0, 2, 0, 1, 0, 124, 2818, 3459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 15, 5, 107, 0, 0, 154, 5, 4096, 0, 411, 43, 0, 2, 1, 0, 4048, 3489, 3060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 158, 2230, 1, 9, 3024, 0, 28, 1, 18, 4094, 1, 32, 0, 747, 732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 597, 171, 3739, 0, 0, 4096, 27, 3900, 171, 0, 3291, 0, 0, 0, 3940, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 4096, 568, 0, 11, 4, 4005, 7, 0, 20, 122, 3929, 3307, 11, 0, 216, 2753, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15228, 14772, 15589, 19116, 15441, 11080, 20618, 19228] avg=16384.0000 max=20618 max_rank=6 min=11080 min_rank=5 imbalance_ratio=1.258423 max_load_reduction=110454 max_load_reduction_pct=0.842697 moved_to_echo=66367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3813, 4096, 698, 342, 0, 0, 147, 0, 4033, 1, 79, 3976, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 68, 0, 4084, 43, 3, 0, 65, 2416, 0, 3581, 35, 0, 346, 4054, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3031, 0, 555, 568, 45, 0, 584, 98, 3763, 1726, 0, 6, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 1, 0, 4093, 0, 0, 3328, 0, 0, 3022, 0, 665, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 1, 748, 0, 40, 0, 1613, 0, 0, 97, 247, 11, 0, 13, 4094, 3626, 2215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3790, 667, 0, 143, 0, 994, 4096, 1076, 0, 0, 3909, 4058, 199, 4068, 17, 1175, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 404, 2634, 4, 0, 7, 3250, 3, 712, 0, 0, 2504, 0, 0, 1, 3821, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 4092, 0, 0, 8, 0, 2164, 78, 0, 3, 0, 4094, 85, 1766, 268, 4087, 302, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21281, 15117, 10397, 15620, 12899, 24272, 14420, 17066] avg=16384.0000 max=24272 max_rank=5 min=10397 min_rank=2 imbalance_ratio=1.481445 max_load_reduction=106800 max_load_reduction_pct=0.814819 moved_to_echo=68657 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4039, 3483, 759, 12, 0, 254, 0, 0, 109, 0, 0, 22, 0, 25, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 0, 0, 1, 4064, 0, 4006, 0, 0, 233, 520, 263, 0, 3280, 2636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1938, 3063, 30, 0, 2, 0, 0, 690, 2911, 3715, 910, 15, 0, 20, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 3, 4096, 4090, 958, 0, 2803, 291, 0, 0, 0, 1540, 0, 3953, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 59, 0, 1, 58, 4088, 0, 678, 2, 3792, 12, 0, 14, 827, 884, 3268, 3441, 2434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1530, 0, 0, 3, 0, 0, 1, 47, 3264, 0, 0, 2454, 629, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 851, 220, 3820, 66, 0, 0, 4089, 8, 1299, 0, 0, 1192, 0, 2881, 24, 4096, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 302, 0, 0, 11, 3993, 0, 148, 5, 2, 4096, 4027, 0, 0, 24, 6, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12716, 18929, 21485, 18161, 19594, 7931, 19126, 13130] avg=16384.0000 max=21485 max_rank=2 min=7931 min_rank=5 imbalance_ratio=1.311340 max_load_reduction=109587 max_load_reduction_pct=0.836082 moved_to_echo=59781 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3849, 2238, 946, 0, 0, 652, 0, 0, 0, 0, 0, 4086, 0, 0, 1, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3533, 2973, 67, 0, 555, 3601, 4084, 2753, 87, 1611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 785, 0, 3, 179, 0, 518, 2895, 307, 1740, 3450, 0, 30, 0, 862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 783, 453, 2, 3869, 3264, 713, 0, 2, 43, 0, 4, 0, 1174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 1, 0, 3516, 2, 4008, 2961, 37, 30, 1218, 0, 0, 1, 403, 975, 23, 4091, 3643, 3004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4077, 4091, 4032, 0, 3375, 20, 0, 12, 13, 3863, 38, 44, 0, 1184, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 5, 0, 3340, 0, 0, 1, 0, 3943, 0, 0, 0, 385, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 25, 4, 643, 0, 919, 0, 476, 4064, 3386, 0, 195, 17, 2630, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15851, 19267, 10889, 10307, 23949, 21461, 12847, 16501] avg=16384.0000 max=23949 max_rank=4 min=10307 min_rank=3 imbalance_ratio=1.461731 max_load_reduction=107123 max_load_reduction_pct=0.817284 moved_to_echo=74758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[28800, 21504, 7168, 512, 3328, 1664, 256, 1024, 12288, 256, 4096, 22400, 256, 2816, 6272, 19456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1280, 3200, 4992, 24704, 22144, 2176, 384, 19968, 2560, 128, 6272, 128, 10752, 384, 16384, 16768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[8320, 128, 128, 15744, 18432, 128, 8192, 3584, 19840, 8320, 16640, 768, 3456, 15616, 12288, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[3072, 4864, 4608, 9856, 19840, 5888, 1152, 17408, 2176, 128, 1408, 10880, 27392, 256, 9472, 2688, 6912, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 3712, 512, 6400, 2432, 4224, 15744, 128, 10624, 1280, 4352, 640, 768, 1280, 2176, 1024, 28928, 25216, 22784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 24448, 1280, 128, 128, 256, 17664, 1408, 1408, 4096, 1152, 28544, 18944, 2560, 640, 12160, 16640, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2816, 5120, 14720, 7680, 2688, 128, 30592, 128, 6016, 256, 640, 19072, 128, 7296, 256, 28160, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[4864, 17792, 2688, 1152, 5248, 4352, 0, 14336, 14592, 15104, 10112, 7552, 3584, 5376, 8576, 16768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 4085, 66, 4063, 0, 3852, 0, 4, 3571, 0, 0, 204, 0, 0, 338, 4074, 371, 104, 2968, 4091, 27, 4088, 4069, 0, 493, 0, 4024, 0, 0, 4095, 0, 0, 0, 0, 27, 2040, 4, 0, 0, 4093, 2905, 11, 0, 670, 0, 0, 0, 0, 0, 3968, 3837, 229, 34, 4095, 1, 0, 0, 397, 2622, 31, 8, 2927, 0, 0, 59, 11, 0, 4093, 0, 0, 3922, 0, 2363, 1006, 2, 4030, 4055, 0, 3792, 1032, 3930, 0, 0, 32, 33, 12, 549, 0, 0, 2723, 98, 554, 0, 0, 0, 0, 2, 0, 0, 0, 3951, 1, 911, 4096, 0, 3944, 20, 30, 0, 2, 6, 3731, 0, 19, 0, 0, 18, 93, 5, 4083, 0, 0, 1284, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 15, 0, 0, 3967, 66, 4096, 0, 4093, 0, 4, 2128, 0, 0, 3080, 4019, 1080, 133, 4088, 0, 38, 916, 0, 23, 4096, 3355, 0, 1, 1903, 4087, 0, 101, 0, 114, 1, 1196, 0, 125, 4042, 1757, 0, 0, 4095, 675, 10, 8, 326, 565, 512, 16, 4077, 0, 3994, 3883, 14, 5, 29, 7, 0, 59, 3811, 1, 9, 1, 31, 0, 0, 0, 0, 0, 4072, 0, 4012, 0, 0, 3987, 4093, 310, 3441, 3247, 0, 144, 0, 3959, 48, 14, 0, 4047, 3413, 0, 2, 24, 100, 5, 16, 0, 4096, 0, 0, 1771, 0, 14, 0, 3440, 3097, 0, 212, 956, 2965, 0, 3089, 959, 2, 143, 355, 1, 443, 0, 0, 0, 0, 29, 3644, 0, 268, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 28, 909, 0, 0, 4092, 156, 2939, 0, 4096, 254, 0, 3, 0, 0, 858, 0, 1, 0, 2, 802, 3349, 4031, 2307, 0, 3689, 4047, 174, 4040, 0, 3859, 0, 355, 3673, 3, 0, 2155, 0, 3670, 4087, 2, 13, 1, 4096, 3623, 4079, 0, 5, 1, 0, 22, 83, 0, 4027, 4075, 2, 0, 2, 1950, 13, 0, 1158, 4035, 0, 0, 514, 0, 0, 1, 0, 38, 2, 0, 18, 2983, 1, 1575, 747, 219, 1110, 4026, 45, 4002, 4094, 3193, 0, 0, 0, 4083, 14, 3, 69, 0, 2021, 88, 646, 0, 0, 0, 10, 1085, 0, 59, 0, 4085, 3090, 9, 0, 28, 523, 0, 0, 0, 2, 4039, 3974, 0, 0, 1097, 1, 6, 0, 6, 2800, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1, 4, 4, 2, 0, 0, 0, 4066, 0, 188, 0, 0, 1, 0, 0, 3301, 0, 2672, 0, 21, 0, 4096, 0, 2183, 12, 4096, 1432, 0, 1655, 0, 3257, 0, 5, 4094, 2347, 193, 4068, 163, 294, 1876, 193, 0, 0, 4096, 4096, 4093, 2378, 6, 346, 3, 0, 3919, 0, 4088, 49, 0, 65, 286, 2736, 23, 890, 3982, 0, 2880, 0, 0, 0, 949, 0, 55, 0, 3877, 0, 242, 0, 0, 17, 6, 3, 4093, 170, 26, 3884, 26, 4087, 525, 0, 0, 3, 985, 3240, 0, 67, 4086, 1198, 4095, 9, 4090, 14, 103, 8, 0, 2, 0, 101, 4064, 9, 4095, 0, 0, 0, 270, 0, 1, 1, 1082, 0, 101, 3569, 61, 0, 0, 0, 3926, 0, 0, 3771, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[296, 25, 119, 2, 2205, 0, 0, 0, 0, 129, 0, 1403, 3141, 0, 0, 449, 1335, 0, 342, 681, 0, 3031, 1323, 4058, 0, 2633, 3931, 85, 0, 0, 4030, 0, 3561, 0, 78, 0, 3955, 0, 0, 2572, 9, 0, 3818, 1977, 3854, 3088, 0, 2618, 16, 0, 545, 3979, 7, 854, 3465, 1, 13, 4094, 0, 0, 0, 3077, 0, 0, 139, 0, 19, 0, 1666, 1506, 25, 2915, 0, 4078, 0, 742, 4063, 0, 25, 0, 39, 81, 3492, 0, 4096, 2737, 17, 35, 0, 435, 546, 1274, 8, 0, 0, 3691, 518, 653, 80, 0, 114, 190, 2144, 554, 3894, 11, 0, 9, 4087, 164, 3083, 79, 0, 0, 0, 4089, 0, 0, 4067, 19, 0, 723, 108, 3593, 1, 464, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[112, 0, 0, 0, 0, 22, 0, 3618, 0, 4093, 0, 0, 4096, 0, 42, 4031, 0, 0, 1, 819, 121, 4048, 5, 4081, 1, 3329, 4096, 1007, 4094, 0, 685, 1, 0, 12, 198, 0, 3957, 0, 0, 3306, 0, 40, 1, 4093, 1, 4029, 0, 0, 0, 0, 0, 273, 0, 0, 872, 8, 1919, 4009, 2385, 0, 0, 2015, 0, 0, 0, 64, 0, 748, 4096, 0, 615, 3838, 0, 181, 0, 244, 1330, 3703, 648, 0, 3889, 1, 3803, 2477, 3604, 886, 1, 9, 3544, 3308, 0, 0, 1675, 3405, 5, 4096, 0, 0, 651, 0, 0, 16, 542, 0, 0, 2528, 1, 3282, 495, 4026, 454, 0, 0, 99, 8, 4095, 0, 0, 3177, 1, 0, 0, 105, 0, 0, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 1186, 1396, 83, 0, 4036, 188, 4094, 0, 1031, 720, 69, 0, 0, 0, 2057, 0, 1337, 0, 0, 195, 0, 2, 3953, 0, 3701, 4079, 0, 3447, 0, 2748, 0, 59, 3977, 14, 15, 0, 1, 1421, 3277, 2, 696, 4, 4096, 2362, 1, 0, 0, 0, 0, 0, 31, 3136, 55, 0, 0, 1318, 0, 4067, 0, 2082, 0, 1689, 0, 0, 0, 0, 124, 4028, 4096, 69, 2969, 1, 94, 4096, 1500, 1716, 4073, 0, 0, 3604, 3, 4041, 150, 1335, 17, 29, 0, 8, 0, 0, 0, 0, 2529, 2679, 2844, 121, 2, 0, 0, 4096, 4, 0, 0, 0, 3012, 2, 0, 50, 4072, 3424, 1322, 0, 10, 3, 0, 0, 0, 4096, 2736, 0, 0, 8, 4068, 813, 4, 597, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4005, 60, 0, 4077, 11, 3530, 0, 4045, 136, 0, 0, 339, 8, 94, 2344, 9, 0, 17, 4, 3072, 753, 768, 0, 3519, 1697, 2, 3974, 0, 1921, 1013, 2, 4068, 0, 18, 10, 0, 177, 2878, 1, 1180, 0, 595, 0, 3920, 0, 254, 3, 3393, 0, 2330, 10, 4095, 2669, 0, 803, 0, 2844, 0, 0, 0, 4016, 695, 0, 3342, 1, 1, 782, 2782, 12, 3831, 0, 4, 4027, 0, 6, 0, 3380, 1, 3434, 0, 273, 4074, 3896, 44, 3, 21, 0, 5, 60, 265, 0, 2190, 0, 1790, 6, 0, 0, 0, 1008, 2, 91, 0, 2753, 3953, 0, 3953, 67, 4061, 0, 49, 0, 0, 2758, 242, 2, 0, 4085, 164, 0, 11, 0, 4094, 0, 0, 176, 14] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 4085, 66, 4063, 0, 3852, 0, 4, 3571, 0, 0, 204, 1495, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 4074, 371, 104, 2968, 4091, 27, 0, 1605, 0, 493, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 27, 2040, 4, 0, 0, 1216, 2905, 11, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 3837, 229, 34, 4095, 1, 0, 0, 397, 2622, 31, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2927, 0, 0, 59, 11, 0, 4093, 0, 0, 3922, 0, 2363, 1006, 2, 4030, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 3792, 1032, 312, 0, 0, 32, 33, 12, 549, 0, 0, 2723, 98, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3951, 1, 911, 4096, 0, 3944, 20, 30, 1382, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 3731, 0, 19, 0, 0, 18, 93, 5, 4083, 0, 0, 1284, 3, 4088, 2511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22538, 18095, 10968, 15743, 19133, 13192, 15560, 15843] avg=16384.0000 max=22538 max_rank=0 min=10968 min_rank=2 imbalance_ratio=1.375610 max_load_reduction=108534 max_load_reduction_pct=0.828049 moved_to_echo=63728 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[296, 25, 119, 2, 2205, 0, 0, 0, 0, 129, 0, 1403, 3141, 0, 0, 449, 722, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1335, 0, 342, 681, 0, 3031, 1323, 4058, 0, 0, 1551, 85, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3561, 0, 78, 0, 3955, 0, 0, 2572, 9, 0, 3818, 588, 3854, 3088, 0, 2618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 545, 3979, 7, 854, 3465, 1, 13, 4094, 0, 0, 0, 3077, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 19, 0, 1666, 1506, 25, 2915, 0, 4078, 0, 742, 4063, 0, 25, 0, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 81, 3492, 0, 325, 2737, 17, 35, 0, 435, 546, 1274, 8, 0, 0, 3691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 653, 80, 0, 114, 190, 2144, 554, 3894, 11, 0, 9, 4087, 164, 3083, 79, 667, 1182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4067, 19, 0, 723, 108, 3593, 1, 464, 0, 0, 2633, 2617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9645, 16436, 24141, 16562, 15865, 12680, 17429, 18314] avg=16384.0000 max=24141 max_rank=2 min=9645 min_rank=0 imbalance_ratio=1.473450 max_load_reduction=106931 max_load_reduction_pct=0.815819 moved_to_echo=64288 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[112, 0, 0, 0, 0, 22, 0, 3618, 0, 4093, 0, 0, 4096, 0, 42, 4031, 1495, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 819, 121, 4048, 5, 4081, 1, 0, 1616, 1007, 4094, 0, 685, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 198, 0, 3957, 0, 0, 3306, 0, 40, 1, 1216, 1, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 0, 872, 8, 1919, 4009, 2385, 0, 0, 2015, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 748, 4096, 0, 615, 3838, 0, 181, 0, 244, 1330, 3703, 648, 0, 716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 1, 3803, 2477, 285, 886, 1, 9, 3544, 3308, 0, 0, 1675, 3405, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 0, 0, 16, 542, 0, 0, 2528, 1, 3282, 495, 4026, 454, 0, 1382, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 8, 4095, 0, 0, 3177, 1, 0, 0, 105, 0, 0, 2, 0, 0, 3329, 2303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18525, 16479, 12760, 12013, 16183, 27384, 14609, 13119] avg=16384.0000 max=27384 max_rank=5 min=12013 min_rank=3 imbalance_ratio=1.671387 max_load_reduction=103688 max_load_reduction_pct=0.791077 moved_to_echo=71295 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 28, 909, 0, 0, 4092, 156, 2939, 0, 4096, 254, 0, 3, 0, 0, 858, 1496, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 802, 3349, 4031, 2307, 0, 0, 1596, 174, 4040, 0, 3859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 3673, 3, 0, 2155, 0, 3670, 4087, 2, 13, 1, 1217, 3623, 4079, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 22, 83, 0, 4027, 4075, 2, 0, 2, 1950, 13, 0, 1158, 4035, 0, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 0, 1, 0, 38, 2, 0, 18, 2983, 1, 1575, 747, 219, 1110, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 45, 4002, 4094, 253, 0, 0, 0, 4083, 14, 3, 69, 0, 2021, 88, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1085, 0, 59, 0, 4085, 3090, 9, 0, 28, 523, 0, 0, 1383, 1217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4039, 3974, 0, 0, 1097, 1, 6, 0, 6, 2800, 0, 0, 0, 0, 3689, 2040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15731, 20161, 22883, 15894, 7916, 19344, 11489, 17654] avg=16384.0000 max=22883 max_rank=2 min=7916 min_rank=4 imbalance_ratio=1.396667 max_load_reduction=108189 max_load_reduction_pct=0.825417 moved_to_echo=56403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 4, 4, 2, 0, 0, 0, 4066, 0, 188, 0, 0, 1, 0, 0, 3301, 1499, 1155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2672, 0, 21, 0, 4096, 0, 2183, 12, 0, 555, 0, 1655, 0, 3257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4094, 2347, 193, 4068, 163, 294, 1876, 193, 0, 0, 1210, 4096, 4093, 2378, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 346, 3, 0, 3919, 0, 4088, 49, 0, 65, 286, 2736, 23, 890, 3982, 0, 2880, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 949, 0, 55, 0, 3877, 0, 242, 0, 0, 17, 6, 3, 4093, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 26, 3884, 26, 316, 525, 0, 0, 3, 985, 3240, 0, 67, 4086, 1198, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4090, 14, 103, 8, 0, 2, 0, 101, 4064, 9, 4095, 0, 0, 0, 270, 1387, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1082, 0, 101, 3569, 61, 0, 0, 0, 3926, 0, 0, 3771, 1, 4096, 2616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10221, 14451, 25016, 19456, 9496, 18621, 14586, 19225] avg=16384.0000 max=25016 max_rank=2 min=9496 min_rank=4 imbalance_ratio=1.526855 max_load_reduction=106056 max_load_reduction_pct=0.809143 moved_to_echo=61928 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4005, 60, 0, 4077, 11, 3530, 0, 4045, 136, 0, 0, 339, 8, 94, 217, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2344, 9, 0, 17, 4, 3072, 753, 768, 0, 0, 670, 2, 3974, 0, 1921, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4068, 0, 18, 10, 0, 177, 2878, 1, 1180, 0, 178, 0, 3920, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3393, 0, 2330, 10, 4095, 2669, 0, 803, 0, 2844, 0, 0, 0, 4016, 695, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3342, 1, 1, 782, 2782, 12, 3831, 0, 4, 4027, 0, 6, 0, 3380, 1, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3434, 0, 273, 4074, 309, 44, 3, 21, 0, 5, 60, 265, 0, 2190, 0, 1790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1008, 2, 91, 0, 2753, 3953, 0, 3953, 67, 4061, 0, 49, 200, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2758, 242, 2, 0, 4085, 164, 0, 11, 0, 4094, 0, 0, 176, 14, 3519, 2489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17620, 14547, 12686, 21078, 18466, 12468, 16653, 17554] avg=16384.0000 max=21078 max_rank=3 min=12468 min_rank=5 imbalance_ratio=1.286499 max_load_reduction=109994 max_load_reduction_pct=0.839188 moved_to_echo=65141 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 15, 0, 0, 3967, 66, 4096, 0, 4093, 0, 4, 2128, 0, 0, 3080, 1495, 1116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 1080, 133, 4088, 0, 38, 916, 0, 23, 0, 1323, 0, 1, 1903, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 114, 1, 1196, 0, 125, 4042, 1757, 0, 0, 1217, 675, 10, 8, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 512, 16, 4077, 0, 3994, 3883, 14, 5, 29, 7, 0, 59, 3811, 1, 9, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 31, 0, 0, 0, 0, 0, 4072, 0, 4012, 0, 0, 3987, 4093, 310, 3441, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3247, 0, 144, 0, 313, 48, 14, 0, 4047, 3413, 0, 2, 24, 100, 5, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1771, 0, 14, 0, 3440, 3097, 0, 212, 956, 2965, 0, 3089, 1383, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 959, 2, 143, 355, 1, 443, 0, 0, 0, 0, 29, 3644, 0, 268, 2, 0, 4096, 2530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20060, 17611, 9572, 17418, 20534, 11373, 22032, 12472] avg=16384.0000 max=22032 max_rank=6 min=9572 min_rank=2 imbalance_ratio=1.344727 max_load_reduction=109040 max_load_reduction_pct=0.831909 moved_to_echo=66411 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1186, 1396, 83, 0, 4036, 188, 4094, 0, 1031, 720, 69, 0, 0, 0, 2057, 1496, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1337, 0, 0, 195, 0, 2, 3953, 0, 0, 1610, 0, 3447, 0, 2748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 3977, 14, 15, 0, 1, 1421, 3277, 2, 696, 4, 1217, 2362, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3136, 55, 0, 0, 1318, 0, 4067, 0, 2082, 0, 1689, 0, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 4028, 4096, 69, 2969, 1, 94, 4096, 1500, 1716, 4073, 0, 0, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3604, 3, 4041, 150, 106, 17, 29, 0, 8, 0, 0, 0, 0, 2529, 2679, 2844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 2, 0, 0, 4096, 4, 0, 0, 0, 3012, 2, 0, 50, 4072, 3424, 1322, 1383, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3, 0, 0, 0, 4096, 2736, 0, 0, 8, 4068, 813, 4, 597, 2, 3701, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16732, 13292, 13046, 12908, 23479, 16010, 18714, 16891] avg=16384.0000 max=23479 max_rank=4 min=12908 min_rank=3 imbalance_ratio=1.433044 max_load_reduction=107593 max_load_reduction_pct=0.820869 moved_to_echo=75094 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[7808, 5120, 896, 9728, 1536, 17792, 10112, 21504, 128, 0, 10624, 1280, 17792, 1920, 24704, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[22528, 256, 23552, 11904, 2304, 4352, 128, 128, 11776, 8192, 4480, 1664, 1792, 17152, 4096, 17792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[768, 8960, 768, 128, 8192, 256, 2944, 640, 18304, 19840, 1024, 15744, 5760, 19840, 7040, 4864, 9216, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 6912, 128, 1920, 10752, 8576, 768, 25600, 128, 8704, 15104, 2560, 15104, 13696, 4608, 12800, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1024, 3968, 640, 14720, 3200, 21120, 18944, 256, 4224, 12544, 14080, 128, 3072, 14464, 12416, 3712, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[4096, 19968, 2816, 256, 15360, 256, 5760, 24192, 2048, 2048, 3840, 8064, 17536, 19328, 2432, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1024, 128, 7040, 17664, 128, 640, 20096, 3072, 128, 896, 384, 26240, 896, 768, 5888, 128, 29184, 18048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[512, 1280, 10624, 256, 2304, 20352, 512, 26496, 0, 21632, 1152, 1536, 13056, 384, 128, 14080, 9984, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3674, 0, 4096, 3370, 3, 1666, 2308, 18, 4040, 3, 2, 1294, 630, 3444, 0, 0, 52, 1636, 4093, 913, 2904, 171, 2024, 837, 0, 57, 4083, 4085, 4080, 4039, 32, 3387, 26, 98, 0, 16, 0, 2706, 0, 4094, 2598, 1, 2919, 0, 712, 0, 792, 5, 4093, 5, 0, 157, 43, 1, 0, 0, 31, 385, 3540, 0, 270, 0, 0, 1316, 317, 0, 0, 0, 0, 1162, 705, 2747, 8, 0, 113, 427, 3, 0, 0, 1164, 8, 0, 0, 0, 131, 3301, 74, 0, 4096, 0, 3, 12, 4095, 2555, 0, 0, 2949, 2546, 0, 122, 3490, 396, 0, 1, 0, 3928, 7, 0, 0, 0, 2, 0, 464, 3858, 3142, 0, 6, 0, 4, 0, 1, 247, 119, 4088, 1014, 3018, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 18, 291, 1177, 4096, 129, 0, 5, 0, 1970, 169, 4008, 4000, 1, 0, 1, 81, 0, 32, 7, 0, 182, 3883, 3738, 2457, 0, 0, 4096, 0, 76, 5, 0, 0, 4096, 0, 2880, 1074, 0, 4, 3, 2, 1126, 0, 144, 0, 0, 46, 3267, 1, 0, 20, 0, 0, 3918, 4096, 255, 277, 2262, 1, 37, 27, 0, 2888, 49, 0, 0, 0, 206, 0, 0, 425, 3813, 0, 0, 148, 763, 4035, 536, 0, 0, 2576, 3223, 3521, 4095, 0, 0, 0, 4088, 1323, 0, 0, 3235, 0, 4090, 1089, 460, 0, 79, 0, 0, 0, 0, 2840, 1612, 1126, 11, 4049, 0, 432, 2737, 3573, 0, 6, 0, 2514, 0, 0, 1425, 4045, 0, 3529, 4091, 132, 254] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[3826, 1, 0, 0, 3610, 0, 3, 0, 44, 3, 3946, 11, 0, 4096, 159, 4084, 2725, 0, 153, 0, 24, 2, 0, 0, 67, 6, 4053, 4046, 168, 51, 3, 2302, 303, 0, 227, 0, 0, 753, 1, 3954, 720, 3807, 0, 1, 1, 1732, 0, 6, 31, 1, 0, 4088, 27, 3719, 4096, 94, 1569, 497, 950, 368, 0, 3502, 4059, 0, 2065, 0, 2189, 0, 4011, 0, 0, 7, 2551, 0, 48, 536, 0, 0, 7, 643, 236, 0, 0, 10, 20, 4065, 0, 0, 3900, 0, 1, 1998, 0, 0, 149, 24, 144, 4075, 3557, 110, 0, 2588, 3879, 5, 28, 13, 25, 1259, 3088, 0, 0, 527, 1404, 11, 1817, 2066, 4096, 1, 4096, 0, 4, 5, 4079, 0, 3601, 0, 227, 18] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4056, 0, 4088, 0, 0, 6, 0, 109, 75, 0, 233, 4092, 12, 30, 47, 4093, 4094, 0, 0, 7, 1, 0, 0, 56, 0, 3200, 15, 3451, 0, 3635, 0, 3529, 0, 0, 3930, 1266, 666, 4074, 14, 41, 4095, 410, 0, 1, 0, 0, 0, 42, 3, 4087, 717, 3879, 0, 4, 3383, 2425, 4034, 1296, 3226, 4095, 0, 0, 0, 0, 4096, 0, 565, 0, 6, 0, 41, 2799, 0, 0, 1, 3991, 0, 0, 422, 0, 1078, 0, 372, 0, 4090, 4096, 4067, 0, 0, 653, 0, 2754, 3436, 0, 0, 1, 7, 1995, 1733, 109, 17, 0, 12, 0, 1881, 40, 539, 0, 2, 0, 0, 18, 0, 4096, 1442, 0, 3212, 15, 3632, 1653, 0, 1, 0, 145, 7, 1528, 2, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4092, 0, 863, 1180, 0, 0, 459, 0, 46, 0, 0, 0, 0, 4068, 0, 4096, 4088, 7, 0, 0, 0, 0, 0, 0, 0, 251, 461, 3990, 2, 3174, 0, 3914, 0, 0, 0, 1875, 3, 3699, 2481, 610, 3894, 4095, 0, 6, 0, 0, 180, 9, 0, 0, 0, 4065, 139, 0, 2, 136, 3061, 0, 4096, 4096, 0, 3484, 3, 3, 4096, 0, 0, 1, 4074, 9, 50, 0, 175, 1, 1572, 4014, 0, 0, 72, 9, 1, 0, 0, 7, 3699, 3979, 1693, 1, 4093, 0, 80, 4088, 1498, 115, 142, 68, 0, 3850, 8, 38, 0, 4096, 0, 0, 0, 4065, 0, 3, 4096, 0, 0, 0, 0, 58, 2, 0, 3905, 125, 4081, 0, 335, 0, 1, 0, 2, 3030, 3212, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[266, 0, 0, 2497, 0, 131, 90, 4067, 0, 1, 0, 4027, 0, 18, 0, 3822, 3959, 4, 0, 1, 62, 13, 1591, 1349, 0, 0, 1534, 4027, 0, 0, 0, 3995, 25, 17, 0, 0, 509, 3128, 322, 0, 4096, 58, 0, 0, 0, 3930, 0, 15, 0, 4073, 1, 1570, 0, 0, 96, 0, 0, 0, 4058, 1909, 1, 4091, 0, 8, 4089, 18, 3328, 0, 0, 0, 155, 478, 0, 3217, 0, 3360, 0, 57, 2, 0, 2334, 0, 0, 0, 115, 3703, 4068, 1, 0, 5, 0, 4096, 0, 372, 1878, 1174, 1325, 4095, 0, 4092, 0, 6, 0, 17, 0, 11, 0, 2, 4096, 0, 0, 228, 0, 675, 3777, 0, 3759, 235, 3098, 2, 0, 1565, 4085, 738, 4096, 3353, 6, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[57, 0, 0, 4046, 12, 1, 306, 211, 392, 0, 1, 0, 0, 3830, 1659, 4067, 1337, 0, 0, 0, 0, 0, 0, 33, 0, 0, 4052, 3807, 15, 0, 2, 4084, 12, 0, 3, 0, 84, 4070, 0, 2801, 4002, 34, 2048, 0, 211, 246, 0, 28, 7, 17, 161, 3547, 0, 0, 3064, 779, 3586, 0, 4096, 3722, 40, 895, 0, 0, 4025, 0, 2, 40, 2, 0, 0, 29, 63, 0, 0, 2688, 0, 0, 0, 2, 3767, 109, 0, 3, 2448, 4090, 4038, 239, 562, 0, 0, 3955, 3471, 0, 0, 0, 3583, 4095, 0, 3752, 3, 4096, 0, 19, 1, 1176, 9, 186, 4075, 42, 433, 0, 2, 1, 4095, 0, 4089, 0, 0, 0, 0, 0, 4084, 0, 4093, 2340, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[3915, 0, 2, 2255, 0, 0, 26, 0, 0, 235, 24, 0, 0, 111, 20, 4046, 1599, 4015, 2, 361, 0, 0, 24, 0, 0, 63, 4085, 4074, 0, 0, 0, 0, 0, 21, 1, 0, 4095, 2722, 0, 218, 3133, 1993, 0, 178, 0, 1924, 2849, 369, 52, 0, 2185, 4064, 108, 1, 0, 4, 228, 1872, 3344, 3738, 0, 3699, 81, 435, 4011, 41, 119, 77, 51, 1006, 42, 0, 1, 1, 0, 4074, 13, 3, 0, 0, 4085, 0, 0, 0, 4073, 259, 4074, 2099, 813, 4, 0, 3906, 3879, 0, 0, 63, 1575, 1088, 0, 3890, 0, 4014, 800, 1412, 0, 0, 11, 20, 1, 465, 0, 0, 886, 4076, 0, 231, 0, 44, 3331, 0, 557, 2151, 3959, 75, 3494, 4096, 31, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3674, 0, 4096, 3370, 3, 1666, 2308, 18, 4040, 3, 2, 1294, 630, 3444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 1636, 4093, 913, 2904, 171, 2024, 837, 0, 57, 4083, 2599, 4080, 4039, 32, 3387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 98, 0, 16, 0, 2706, 0, 4094, 2598, 1, 2919, 0, 712, 0, 792, 5, 2072, 734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 0, 157, 43, 1, 0, 0, 31, 385, 1252, 0, 270, 0, 0, 1316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 0, 0, 1162, 705, 2747, 8, 0, 113, 427, 3, 0, 0, 1164, 2957, 3018, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 131, 344, 74, 0, 4096, 0, 3, 12, 4095, 2555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2949, 2546, 0, 122, 3490, 396, 0, 1, 0, 3928, 7, 0, 0, 0, 2, 0, 752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 3858, 3142, 0, 6, 0, 4, 0, 1, 247, 119, 4088, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[24548, 30907, 16773, 7553, 12837, 11318, 14193, 12943] avg=16384.0000 max=30907 max_rank=1 min=7553 min_rank=3 imbalance_ratio=1.886414 max_load_reduction=100165 max_load_reduction_pct=0.764198 moved_to_echo=51291 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3826, 1, 0, 0, 3610, 0, 3, 0, 44, 3, 3946, 11, 0, 4096, 159, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2725, 0, 153, 0, 24, 2, 0, 0, 67, 6, 4053, 2575, 168, 51, 3, 2302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 227, 0, 0, 753, 1, 3954, 720, 3807, 0, 1, 1, 1732, 0, 6, 556, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 1, 0, 4088, 27, 3719, 4096, 94, 1569, 497, 336, 368, 0, 3502, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2065, 0, 2189, 0, 4011, 0, 0, 7, 2551, 0, 48, 536, 0, 0, 7, 643, 3641, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 10, 20, 424, 0, 0, 3900, 0, 1, 1998, 0, 0, 149, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 4075, 3557, 110, 0, 2588, 3879, 5, 28, 13, 25, 1259, 3088, 0, 0, 527, 744, 1007, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1404, 11, 1817, 2066, 4096, 1, 3089, 0, 4, 5, 4079, 0, 3601, 0, 227, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19639, 12129, 12788, 22387, 15756, 6762, 21193, 20418] avg=16384.0000 max=22387 max_rank=3 min=6762 min_rank=5 imbalance_ratio=1.366394 max_load_reduction=108685 max_load_reduction_pct=0.829201 moved_to_echo=64129 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 863, 1180, 0, 0, 459, 0, 46, 0, 0, 0, 0, 4068, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 0, 0, 0, 0, 0, 251, 461, 2539, 2, 3174, 0, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1875, 3, 3699, 2481, 610, 3894, 4095, 0, 6, 0, 0, 180, 9, 2397, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 139, 0, 2, 136, 3061, 0, 1449, 4096, 0, 3484, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4074, 9, 50, 0, 175, 1, 1572, 4014, 0, 0, 72, 9, 3564, 3030, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 3699, 415, 1693, 1, 4093, 0, 80, 4088, 1498, 115, 142, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3850, 8, 38, 0, 4096, 0, 0, 0, 4065, 0, 3, 4096, 0, 0, 0, 734, 1003, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 2, 0, 3905, 125, 3078, 0, 335, 0, 1, 0, 2, 0, 3212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14660, 14436, 19966, 16438, 20917, 15900, 18037, 10718] avg=16384.0000 max=20917 max_rank=4 min=10718 min_rank=7 imbalance_ratio=1.276672 max_load_reduction=110155 max_load_reduction_pct=0.840416 moved_to_echo=65572 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4056, 0, 4088, 0, 0, 6, 0, 109, 75, 0, 233, 4092, 12, 30, 47, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 7, 1, 0, 0, 56, 0, 3200, 15, 2187, 0, 3635, 0, 3529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 1266, 666, 4074, 14, 41, 4095, 410, 0, 1, 0, 0, 0, 42, 1891, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4087, 717, 3879, 0, 4, 3383, 2425, 4034, 1296, 1136, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 565, 0, 6, 0, 41, 2799, 0, 0, 1, 3991, 0, 0, 422, 0, 3671, 1528, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 0, 372, 0, 4090, 425, 4067, 0, 0, 653, 0, 2754, 3436, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1995, 1733, 109, 17, 0, 12, 0, 1881, 40, 539, 0, 2, 0, 0, 18, 639, 897, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1442, 0, 3212, 15, 2735, 1653, 0, 1, 0, 145, 7, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16693, 16724, 17055, 25059, 17319, 16876, 8037, 13309] avg=16384.0000 max=25059 max_rank=3 min=8037 min_rank=6 imbalance_ratio=1.529480 max_load_reduction=106013 max_load_reduction_pct=0.808815 moved_to_echo=55541 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[57, 0, 0, 4046, 12, 1, 306, 211, 392, 0, 1, 0, 0, 3830, 1659, 3924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1337, 0, 0, 0, 0, 0, 0, 33, 0, 0, 4052, 2423, 15, 0, 2, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 3, 0, 84, 4070, 0, 2801, 4002, 34, 2048, 0, 211, 246, 0, 28, 2397, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 17, 161, 3547, 0, 0, 3064, 779, 3586, 0, 1449, 3722, 40, 895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 2, 40, 2, 0, 0, 29, 63, 0, 0, 2688, 0, 0, 0, 2, 3663, 2340, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3767, 109, 0, 3, 2448, 427, 4038, 239, 562, 0, 0, 3955, 3471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3583, 4095, 0, 3752, 3, 4096, 0, 19, 1, 1176, 9, 186, 4075, 42, 433, 0, 700, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4095, 0, 4089, 0, 0, 0, 0, 0, 4084, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14439, 11946, 16620, 17267, 13104, 19019, 22313, 16364] avg=16384.0000 max=22313 max_rank=6 min=11946 min_rank=1 imbalance_ratio=1.361877 max_load_reduction=108759 max_load_reduction_pct=0.829765 moved_to_echo=70800 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 18, 291, 1177, 4096, 129, 0, 5, 0, 1970, 169, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 1, 0, 1, 81, 0, 32, 7, 0, 182, 3883, 2378, 2457, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 5, 0, 0, 4096, 0, 2880, 1074, 0, 4, 3, 2, 1126, 0, 144, 2397, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 3267, 1, 0, 20, 0, 0, 3918, 1449, 255, 277, 2262, 1, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 2888, 49, 0, 0, 0, 206, 0, 0, 425, 3813, 0, 0, 148, 763, 2887, 4091, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 536, 0, 0, 2576, 336, 3521, 4095, 0, 0, 0, 4088, 1323, 0, 0, 3235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1089, 460, 0, 79, 0, 0, 0, 0, 2840, 1612, 1126, 11, 4049, 0, 688, 618, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 2737, 3573, 0, 6, 0, 1896, 0, 0, 1425, 4045, 0, 3529, 0, 132, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15818, 17118, 12479, 11533, 15547, 23745, 16803, 18029] avg=16384.0000 max=23745 max_rank=5 min=11533 min_rank=3 imbalance_ratio=1.449280 max_load_reduction=107327 max_load_reduction_pct=0.818840 moved_to_echo=74124 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3915, 0, 2, 2255, 0, 0, 26, 0, 0, 235, 24, 0, 0, 111, 20, 3903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1599, 4015, 2, 361, 0, 0, 24, 0, 0, 63, 4085, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1, 0, 4095, 2722, 0, 218, 3133, 1993, 0, 178, 0, 1924, 2849, 369, 1957, 732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 2185, 4064, 108, 1, 0, 4, 228, 1872, 1183, 3738, 0, 3699, 81, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 41, 119, 77, 51, 1006, 42, 0, 1, 1, 0, 4074, 13, 3, 0, 0, 232, 4096, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4073, 27, 4074, 2099, 813, 4, 0, 3906, 3879, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1575, 1088, 0, 3890, 0, 4014, 800, 1412, 0, 0, 11, 20, 1, 465, 0, 0, 749, 819, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 886, 4076, 0, 231, 0, 44, 2512, 0, 557, 2151, 3959, 75, 3494, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10491, 12742, 20192, 17650, 13971, 23023, 14987, 18016] avg=16384.0000 max=23023 max_rank=5 min=10491 min_rank=0 imbalance_ratio=1.405212 max_load_reduction=108049 max_load_reduction_pct=0.824348 moved_to_echo=69997 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[266, 0, 0, 2497, 0, 131, 90, 4067, 0, 1, 0, 4027, 0, 18, 0, 3687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 4, 0, 1, 62, 13, 1591, 1349, 0, 0, 1534, 2562, 0, 0, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 17, 0, 0, 509, 3128, 322, 0, 4096, 58, 0, 0, 0, 3930, 0, 15, 2375, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 1, 1570, 0, 0, 96, 0, 0, 0, 1436, 1909, 1, 4091, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 18, 3328, 0, 0, 0, 155, 478, 0, 3217, 0, 3360, 0, 57, 2, 0, 3317, 3353, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2334, 0, 0, 0, 115, 386, 4068, 1, 0, 5, 0, 4096, 0, 372, 1878, 1174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 4095, 0, 4092, 0, 6, 0, 17, 0, 11, 0, 2, 4096, 0, 0, 228, 741, 761, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 675, 3777, 0, 3759, 235, 2337, 2, 0, 1565, 4085, 738, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14784, 15070, 15199, 13185, 21621, 14429, 15509, 21275] avg=16384.0000 max=21621 max_rank=4 min=13185 min_rank=3 imbalance_ratio=1.319641 max_load_reduction=109451 max_load_reduction_pct=0.835045 moved_to_echo=72834 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[15616, 768, 384, 128, 17152, 2816, 21632, 6528, 13568, 768, 128, 24960, 17792, 3072, 2176, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[22784, 128, 9216, 256, 8192, 2304, 1024, 6272, 2816, 3328, 2176, 22912, 128, 128, 768, 2688, 23936, 21504, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[21888, 5760, 4352, 1408, 3072, 256, 3712, 2304, 128, 3840, 22272, 19968, 6784, 11008, 128, 25344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9600, 25856, 6400, 12672, 3584, 15360, 4736, 1536, 1920, 9344, 3456, 3200, 16512, 640, 4608, 896, 5760, 5120, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[384, 256, 4224, 3200, 5376, 25344, 2944, 14720, 23680, 10496, 4992, 256, 1024, 8960, 3840, 640, 16128, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3200, 15616, 17920, 2304, 19200, 512, 15744, 1664, 1024, 5504, 20480, 5120, 19840, 0, 3712, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[4224, 8192, 3200, 24704, 384, 3840, 10752, 3456, 12544, 8064, 9728, 18304, 640, 18048, 4224, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[19968, 128, 9088, 17536, 3712, 1920, 3584, 5632, 8704, 384, 4224, 9472, 768, 17664, 2176, 27264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 282, 0, 0, 1, 58, 1563, 180, 4095, 1, 0, 81, 4084, 3567, 0, 135, 0, 0, 371, 0, 0, 0, 0, 1183, 2339, 0, 4095, 4094, 4058, 0, 2, 0, 1, 0, 4081, 0, 1, 3941, 0, 153, 258, 51, 4088, 0, 3784, 4096, 3117, 2376, 1589, 2, 0, 0, 1, 454, 0, 0, 1, 0, 0, 859, 121, 0, 11, 9, 4024, 0, 9, 0, 4096, 0, 0, 0, 4095, 4049, 0, 37, 4054, 0, 0, 0, 4041, 0, 1, 3992, 3604, 3876, 2063, 1345, 777, 237, 0, 0, 0, 34, 6, 0, 91, 0, 3617, 0, 31, 3373, 3873, 1, 0, 1399, 0, 6, 0, 1, 5, 0, 21, 4096, 5, 4094, 1, 187, 0, 0, 3995, 2402, 385, 3966, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 3178, 4071, 1, 3632, 0, 549, 426, 4096, 288, 4, 1774, 0, 30, 1110, 34, 2174, 0, 703, 50, 3887, 1072, 2685, 7, 4066, 0, 0, 206, 0, 1, 0, 0, 0, 0, 4091, 0, 3191, 11, 0, 4073, 11, 1292, 4091, 0, 27, 0, 0, 445, 0, 1, 0, 2, 0, 4085, 50, 1832, 89, 0, 0, 49, 2884, 224, 31, 0, 0, 2, 1940, 2, 0, 4020, 1875, 1171, 1, 4090, 21, 1599, 2, 3628, 2936, 1, 3853, 3993, 2714, 14, 3589, 16, 0, 8, 0, 0, 0, 2984, 130, 0, 824, 0, 0, 0, 833, 2846, 0, 93, 0, 3626, 5, 373, 2472, 3457, 0, 0, 4096, 136, 4096, 0, 0, 1, 15, 4078, 1, 0, 3987, 76, 937, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 618, 3965, 0, 3812, 13, 18, 4078, 0, 4, 0, 0, 4002, 0, 28, 1, 4096, 3125, 1, 0, 4096, 1702, 73, 0, 4023, 87, 0, 41, 155, 2375, 0, 0, 1022, 4092, 3708, 4096, 119, 2552, 3522, 3413, 3514, 1995, 0, 0, 5, 0, 0, 0, 1, 0, 0, 474, 1961, 479, 0, 0, 0, 6, 0, 4091, 17, 85, 48, 1, 3861, 0, 14, 5, 4047, 3965, 224, 0, 3937, 222, 0, 0, 780, 943, 2946, 64, 0, 2, 2531, 4069, 0, 4096, 0, 0, 0, 0, 3074, 0, 1, 55, 20, 9, 911, 0, 4092, 0, 0, 0, 2, 48, 0, 729, 4026, 578, 100, 2, 0, 3953, 0, 3984, 134, 30, 1887, 9, 293, 3778, 2, 0, 131] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 50, 0, 29, 0, 3729, 4096, 0, 2931, 949, 481, 509, 0, 0, 59, 1808, 3745, 4, 0, 0, 1972, 0, 0, 0, 3560, 412, 4046, 28, 4091, 5, 0, 146, 166, 0, 3751, 0, 2122, 259, 0, 0, 18, 0, 3952, 17, 3725, 4085, 4096, 106, 0, 0, 0, 4, 1, 1, 181, 0, 3536, 4000, 31, 2, 0, 0, 0, 913, 13, 321, 11, 0, 3856, 1505, 0, 0, 4021, 4096, 144, 625, 4004, 157, 1, 0, 2948, 0, 4072, 640, 0, 169, 57, 4094, 3938, 3522, 0, 0, 0, 0, 3, 4085, 0, 0, 3845, 0, 0, 0, 4075, 403, 0, 1790, 0, 2668, 0, 2527, 8, 0, 0, 374, 1329, 4041, 0, 36, 0, 0, 664, 0, 390, 3023, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 1, 3442, 0, 3143, 4050, 0, 3902, 3944, 33, 104, 550, 3489, 141, 18, 0, 0, 0, 0, 1, 0, 0, 21, 4090, 3972, 14, 225, 953, 2656, 7, 4, 3894, 0, 4022, 3976, 999, 0, 3226, 0, 3661, 0, 1281, 4083, 1, 4063, 4059, 1931, 0, 14, 0, 0, 0, 2, 0, 1, 6, 264, 0, 0, 1771, 45, 0, 99, 0, 0, 36, 0, 2020, 4038, 0, 0, 0, 3779, 3539, 28, 1225, 3210, 0, 4061, 2, 4096, 3, 219, 6, 0, 4041, 0, 20, 0, 4036, 0, 41, 1558, 785, 2, 545, 0, 0, 9, 0, 0, 0, 4072, 372, 1, 4069, 0, 0, 0, 4084, 0, 0, 0, 0, 190, 4096, 62, 4037, 1, 35, 0, 2, 0, 0, 594, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 2, 134, 14, 0, 4094, 0, 3482, 2923, 299, 1, 103, 65, 0, 2004, 0, 212, 4091, 3, 0, 421, 3899, 458, 214, 6, 2087, 2833, 1113, 0, 2, 132, 0, 125, 4030, 0, 0, 3568, 3552, 127, 31, 3897, 3994, 0, 0, 1813, 2514, 4002, 4096, 117, 349, 64, 0, 20, 0, 0, 8, 24, 0, 3995, 0, 187, 0, 0, 0, 0, 36, 4096, 88, 0, 0, 0, 3, 0, 0, 4094, 1, 0, 4066, 3124, 4096, 33, 3942, 844, 1163, 497, 0, 2683, 4083, 0, 1, 0, 1, 0, 4, 2, 3957, 0, 1256, 0, 2542, 0, 1331, 2250, 4067, 2344, 0, 2, 0, 92, 0, 0, 0, 282, 0, 13, 1025, 4070, 0, 0, 1880, 0, 0, 0, 3335, 4076, 4, 584, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3780, 0, 0, 0, 0, 4, 3368, 4096, 4082, 2781, 0, 236, 0, 31, 7, 0, 0, 0, 150, 0, 1362, 347, 53, 0, 1437, 4043, 0, 18, 0, 4095, 0, 0, 3, 3, 5, 4096, 0, 0, 88, 0, 333, 0, 0, 0, 953, 3343, 4096, 3471, 0, 0, 0, 0, 0, 0, 84, 0, 0, 4073, 3400, 5, 0, 0, 0, 0, 4095, 0, 36, 0, 969, 4079, 0, 0, 0, 2075, 4096, 0, 413, 3136, 0, 0, 64, 4094, 0, 3555, 0, 70, 4073, 0, 2, 67, 2977, 1, 0, 22, 0, 4096, 37, 17, 0, 2193, 0, 1, 4096, 4096, 3860, 0, 0, 0, 0, 0, 2146, 3978, 0, 0, 8, 0, 4029, 1695, 0, 0, 0, 4087, 0, 3501, 4070, 777, 0, 718] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 3290, 4033, 4095, 0, 3865, 40, 0, 0, 19, 0, 2, 120, 586, 0, 6, 0, 0, 0, 0, 216, 4096, 0, 813, 4065, 0, 2, 2, 450, 0, 1794, 0, 0, 4094, 4061, 50, 3658, 0, 4012, 4096, 1294, 3654, 262, 3890, 0, 0, 131, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 1, 0, 3, 3, 0, 0, 0, 5, 4008, 0, 327, 0, 4096, 0, 1, 134, 4077, 4041, 3322, 0, 4084, 0, 180, 0, 0, 0, 35, 4096, 0, 10, 0, 167, 976, 3101, 0, 3798, 0, 0, 3998, 1812, 650, 1366, 4096, 1063, 441, 3988, 8, 6, 34, 3843, 0, 15, 412, 0, 705, 3216, 1962, 3245, 1608, 0, 0, 0, 263, 1092, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 282, 0, 0, 1, 58, 1563, 180, 4095, 1, 0, 81, 4084, 3567, 0, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 371, 0, 0, 0, 0, 1183, 2339, 0, 4095, 4094, 4058, 0, 2, 1239, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4081, 0, 1, 3941, 0, 153, 258, 51, 0, 0, 0, 0, 3047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2376, 1589, 2, 0, 0, 1, 454, 0, 0, 1, 0, 0, 859, 121, 0, 11, 3685, 4096, 3784, 1575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4024, 0, 9, 0, 4096, 0, 0, 0, 4095, 4049, 0, 37, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1952, 0, 1, 3992, 3604, 3876, 2063, 1345, 777, 237, 0, 0, 0, 34, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 3617, 0, 31, 3373, 3873, 1, 0, 1399, 0, 6, 0, 1, 5, 248, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 5, 4094, 1, 187, 0, 0, 3995, 2402, 385, 3966, 0, 0, 0, 2414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14514, 17615, 11533, 18554, 16688, 17887, 12715, 21566] avg=16384.0000 max=21566 max_rank=7 min=11533 min_rank=2 imbalance_ratio=1.316284 max_load_reduction=109506 max_load_reduction_pct=0.835464 moved_to_echo=68856 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 3178, 4071, 1, 3632, 0, 549, 426, 4096, 288, 4, 1774, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 1110, 34, 2174, 0, 703, 50, 3887, 1072, 2685, 7, 4066, 0, 0, 206, 1112, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4091, 0, 3191, 11, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 445, 0, 1, 0, 2, 0, 4085, 50, 1832, 89, 0, 0, 49, 3718, 4091, 1292, 1571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2884, 224, 31, 0, 0, 2, 1940, 2, 0, 4020, 1875, 1171, 1, 372, 21, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1754, 2936, 1, 3853, 3993, 2714, 14, 3589, 16, 0, 8, 0, 0, 0, 2984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 824, 0, 0, 0, 833, 2846, 0, 93, 0, 3626, 5, 373, 2472, 3457, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 136, 4096, 0, 0, 1, 15, 4078, 1, 0, 3987, 76, 937, 6, 2405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18561, 17233, 7305, 17252, 14142, 21864, 14881, 19834] avg=16384.0000 max=21864 max_rank=5 min=7305 min_rank=2 imbalance_ratio=1.334473 max_load_reduction=109208 max_load_reduction_pct=0.833191 moved_to_echo=70721 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 134, 14, 0, 4094, 0, 3482, 2923, 299, 1, 103, 65, 0, 2004, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 4091, 3, 0, 421, 3899, 458, 214, 6, 2087, 2833, 1113, 0, 2, 132, 0, 1209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 4030, 0, 0, 3568, 3552, 127, 31, 3897, 3994, 0, 0, 1813, 0, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 349, 64, 0, 20, 0, 0, 8, 24, 0, 3995, 0, 187, 0, 0, 0, 3696, 4002, 2514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 4096, 88, 0, 0, 0, 3, 0, 0, 4094, 1, 0, 370, 3124, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 1904, 844, 1163, 497, 0, 2683, 4083, 0, 1, 0, 1, 0, 4, 2, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1256, 0, 2542, 0, 1331, 2250, 4067, 2344, 0, 2, 0, 92, 0, 0, 0, 242, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 13, 1025, 4070, 0, 0, 1880, 0, 0, 0, 3335, 4076, 4, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13708, 16680, 25141, 14976, 15908, 15172, 14218, 15269] avg=16384.0000 max=25141 max_rank=2 min=13708 min_rank=0 imbalance_ratio=1.534485 max_load_reduction=105931 max_load_reduction_pct=0.808189 moved_to_echo=60567 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 618, 3965, 0, 3812, 13, 18, 4078, 0, 4, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 28, 1, 4096, 3125, 1, 0, 4096, 1702, 73, 0, 4023, 87, 0, 239, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 155, 2375, 0, 0, 1022, 4092, 3708, 4096, 119, 2552, 0, 3413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 474, 1961, 479, 0, 0, 0, 6, 0, 3579, 1995, 3514, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 17, 85, 48, 1, 3861, 0, 14, 5, 4047, 3965, 224, 0, 358, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 943, 2946, 64, 0, 2, 2531, 4069, 0, 4096, 0, 0, 0, 0, 3074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 55, 20, 9, 911, 0, 4092, 0, 0, 0, 2, 48, 0, 729, 4026, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 100, 2, 0, 3953, 0, 3984, 134, 30, 1887, 9, 293, 3778, 2, 0, 131, 2081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12628, 21557, 21573, 13371, 16938, 18103, 9940, 16962] avg=16384.0000 max=21573 max_rank=2 min=9940 min_rank=6 imbalance_ratio=1.316711 max_load_reduction=109499 max_load_reduction_pct=0.835411 moved_to_echo=61943 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 3442, 0, 3143, 4050, 0, 3902, 3944, 33, 104, 550, 3489, 141, 18, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 21, 4090, 3972, 14, 225, 953, 2656, 7, 4, 1256, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3894, 0, 4022, 3976, 999, 0, 3226, 0, 3661, 0, 1281, 0, 1, 0, 0, 1888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 2, 0, 1, 6, 264, 0, 0, 1771, 45, 0, 99, 2918, 4059, 4063, 1574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 2020, 4038, 0, 0, 0, 3779, 3539, 28, 1225, 292, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1979, 3, 219, 6, 0, 4041, 0, 20, 0, 4036, 0, 41, 1558, 785, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 0, 0, 9, 0, 0, 0, 4072, 372, 1, 4069, 0, 0, 0, 4084, 0, 251, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 4096, 62, 4037, 1, 35, 0, 2, 0, 0, 594, 0, 0, 2411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23427, 13297, 22948, 14816, 19018, 12692, 13446, 11428] avg=16384.0000 max=23427 max_rank=0 min=11428 min_rank=7 imbalance_ratio=1.429871 max_load_reduction=107645 max_load_reduction_pct=0.821266 moved_to_echo=56584 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 50, 0, 29, 0, 3729, 4096, 0, 2931, 949, 481, 509, 0, 0, 59, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1808, 3745, 4, 0, 0, 1972, 0, 0, 0, 3560, 412, 4046, 28, 4091, 5, 0, 904, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 166, 0, 3751, 0, 2122, 259, 0, 0, 18, 0, 0, 17, 0, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 4, 1, 1, 181, 0, 3536, 4000, 31, 2, 0, 0, 0, 3640, 4085, 3725, 1523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 913, 13, 321, 11, 0, 3856, 1505, 0, 0, 4021, 4096, 144, 625, 364, 157, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1424, 0, 4072, 640, 0, 169, 57, 4094, 3938, 3522, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 3845, 0, 0, 0, 4075, 403, 0, 1790, 0, 2668, 0, 2527, 8, 181, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 1329, 4041, 0, 36, 0, 0, 664, 0, 390, 3023, 0, 0, 0, 2333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13273, 20671, 10483, 20835, 16027, 17919, 19674, 12190] avg=16384.0000 max=20835 max_rank=3 min=10483 min_rank=2 imbalance_ratio=1.271667 max_load_reduction=110237 max_load_reduction_pct=0.841042 moved_to_echo=65810 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3780, 0, 0, 0, 0, 4, 3368, 4096, 4082, 2781, 0, 236, 0, 31, 7, 0, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 1362, 347, 53, 0, 1437, 4043, 0, 18, 0, 4095, 0, 0, 1258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 5, 4096, 0, 0, 88, 0, 333, 0, 0, 0, 953, 0, 0, 3392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 4073, 3400, 5, 0, 0, 0, 0, 2856, 4096, 3343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 36, 0, 969, 4079, 0, 0, 0, 2075, 4096, 0, 413, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 1969, 0, 3555, 0, 70, 4073, 0, 2, 67, 2977, 1, 0, 22, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 17, 0, 2193, 0, 1, 4096, 4096, 3860, 0, 0, 0, 0, 0, 2146, 3978, 255, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4029, 1695, 0, 0, 0, 4087, 0, 3501, 4070, 777, 0, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18997, 12763, 8873, 17857, 16043, 16896, 20758, 18885] avg=16384.0000 max=20758 max_rank=6 min=8873 min_rank=2 imbalance_ratio=1.266968 max_load_reduction=110314 max_load_reduction_pct=0.841629 moved_to_echo=72582 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 3290, 4033, 4095, 0, 3865, 40, 0, 0, 19, 0, 2, 608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 586, 0, 6, 0, 0, 0, 0, 216, 4096, 0, 813, 4065, 0, 2, 2, 1252, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 0, 1794, 0, 0, 4094, 4061, 50, 3658, 0, 4012, 0, 1294, 0, 0, 3803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 1, 0, 3, 3706, 262, 3654, 1578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 5, 4008, 0, 327, 0, 4096, 0, 1, 134, 371, 4041, 3322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1974, 0, 180, 0, 0, 0, 35, 4096, 0, 10, 0, 167, 976, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 0, 3998, 1812, 650, 1366, 4096, 1063, 441, 3988, 8, 6, 34, 3843, 0, 250, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 412, 0, 705, 3216, 1962, 3245, 1608, 0, 0, 0, 263, 1092, 0, 0, 0, 2420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15964, 11256, 23216, 13411, 16308, 10539, 25440, 14938] avg=16384.0000 max=25440 max_rank=6 min=10539 min_rank=5 imbalance_ratio=1.552734 max_load_reduction=105632 max_load_reduction_pct=0.805908 moved_to_echo=67225 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[12032, 4352, 4608, 256, 3072, 24064, 3456, 384, 128, 26240, 25728, 1664, 2560, 2816, 7680, 13184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[4736, 4480, 8320, 16000, 4608, 10880, 19968, 3840, 19072, 4480, 7936, 0, 7552, 0, 0, 20224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[3840, 128, 512, 3456, 128, 13824, 19968, 21376, 14720, 21760, 1664, 1408, 9472, 7936, 5760, 1920, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2304, 12544, 1280, 512, 3968, 10368, 4352, 384, 10880, 25344, 7680, 10496, 13312, 18944, 256, 256, 8576, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[2688, 2048, 256, 512, 128, 128, 640, 256, 512, 18048, 12032, 1920, 2944, 256, 128, 256, 27904, 26752, 25984, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[8704, 1408, 896, 16256, 1920, 2944, 12032, 31232, 8064, 640, 11264, 3712, 2944, 512, 15872, 11520, 1792, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[896, 640, 8704, 3456, 31616, 3840, 11520, 3712, 128, 14720, 2432, 8192, 24064, 1536, 1536, 896, 14080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 13440, 4736, 12160, 9088, 7680, 17664, 8832, 17280, 4864, 14976, 128, 256, 2560, 3968, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 186, 0, 4096, 22, 4096, 1, 0, 3975, 4087, 0, 0, 54, 145, 0, 74, 0, 0, 4096, 22, 0, 4095, 28, 45, 3938, 96, 63, 5, 3981, 2, 0, 0, 48, 0, 0, 3559, 0, 156, 0, 10, 4096, 77, 0, 31, 20, 0, 0, 731, 28, 125, 3910, 0, 4095, 78, 0, 0, 4, 0, 4047, 4, 26, 12, 1, 0, 1, 2, 0, 0, 48, 4, 0, 0, 125, 0, 4066, 3711, 3035, 0, 112, 0, 3381, 4056, 0, 1165, 197, 0, 2195, 0, 0, 0, 0, 3820, 4096, 4049, 4054, 0, 3377, 163, 0, 4026, 4071, 200, 2823, 0, 0, 421, 97, 0, 0, 4095, 4050, 3234, 526, 1, 0, 4096, 1035, 0, 0, 3818, 1, 578, 0, 246, 1932] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 3950, 718, 0, 861, 0, 2968, 0, 0, 4083, 0, 0, 0, 0, 0, 235, 3903, 4094, 9, 3992, 1054, 675, 0, 0, 3962, 0, 0, 48, 1, 1, 26, 0, 4005, 4074, 0, 301, 3861, 890, 0, 0, 0, 3709, 0, 8, 36, 0, 0, 0, 4095, 0, 87, 20, 2736, 0, 0, 3423, 13, 3, 547, 4, 3357, 8, 0, 0, 1, 89, 2487, 0, 0, 63, 3278, 2, 4072, 8, 0, 3369, 2, 677, 0, 0, 3729, 0, 0, 1720, 0, 2463, 0, 0, 0, 0, 2022, 4048, 29, 3, 0, 0, 3274, 1102, 4050, 0, 0, 3758, 637, 0, 281, 26, 2821, 1688, 3626, 100, 1, 4096, 148, 3855, 0, 4095, 69, 16, 3517, 0, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[405, 1401, 437, 3, 0, 92, 613, 1, 824, 53, 0, 207, 4076, 1569, 38, 0, 0, 2, 0, 4, 4007, 907, 0, 0, 2383, 3384, 4064, 0, 4016, 371, 0, 88, 1042, 0, 0, 0, 0, 1510, 2806, 4093, 1784, 0, 0, 4088, 0, 2719, 9, 0, 2903, 12, 0, 0, 0, 4051, 126, 41, 0, 1, 0, 0, 0, 1577, 0, 353, 0, 0, 0, 458, 0, 75, 0, 0, 267, 2634, 0, 54, 99, 3998, 3, 0, 490, 3, 330, 3335, 0, 3997, 4066, 3908, 10, 1679, 2934, 193, 0, 0, 196, 1424, 0, 38, 4093, 0, 240, 1043, 4096, 0, 0, 306, 0, 0, 3494, 34, 0, 0, 4, 3546, 3591, 3841, 7, 1, 4096, 119, 4096, 0, 3122, 3805, 5, 0, 1323, 3959] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 2, 4, 176, 0, 0, 4078, 0, 13, 2963, 0, 525, 4094, 0, 0, 0, 4082, 0, 32, 0, 394, 0, 2000, 61, 1989, 0, 3907, 40, 3967, 3781, 0, 3858, 3938, 44, 24, 0, 74, 0, 113, 65, 523, 0, 3158, 4008, 0, 4048, 0, 0, 3348, 1, 0, 4, 6, 0, 2231, 0, 0, 0, 2, 1376, 40, 0, 2198, 0, 3238, 126, 0, 3340, 1000, 0, 165, 7, 0, 12, 0, 0, 0, 3949, 38, 919, 0, 0, 71, 0, 1201, 1950, 0, 4096, 154, 3, 0, 0, 0, 0, 0, 1, 62, 3924, 0, 3834, 892, 29, 2068, 0, 0, 3750, 23, 4, 0, 0, 50, 0, 3435, 3943, 4014, 204, 3275, 0, 4092, 520, 3691, 52, 4054, 3016, 378, 1806, 0, 2519] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[2591, 97, 89, 42, 0, 1716, 3992, 195, 0, 406, 0, 3782, 4091, 4024, 3312, 0, 9, 0, 0, 0, 3923, 3779, 0, 2670, 121, 23, 1421, 0, 3470, 2726, 1121, 41, 3, 740, 0, 73, 0, 0, 810, 25, 1996, 0, 1, 2657, 0, 0, 1, 0, 3596, 0, 0, 0, 1, 0, 1182, 0, 55, 0, 3, 0, 2, 85, 1433, 3, 2, 197, 1, 0, 4096, 0, 0, 0, 0, 0, 16, 1, 0, 4065, 10, 1617, 0, 0, 1104, 4068, 3070, 433, 4093, 3459, 0, 3993, 22, 0, 0, 0, 1, 44, 11, 4095, 4066, 4073, 0, 100, 4027, 1010, 0, 3943, 3919, 44, 18, 0, 82, 0, 13, 2701, 3660, 0, 62, 0, 2976, 0, 4096, 0, 4096, 479, 1, 0, 1, 1000] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4068, 0, 186, 49, 0, 0, 343, 191, 4089, 4012, 0, 0, 2971, 1, 0, 0, 3002, 0, 222, 1, 22, 3153, 3554, 0, 4, 977, 105, 0, 4011, 0, 2809, 0, 0, 33, 0, 589, 0, 0, 897, 0, 937, 0, 2124, 0, 4091, 4055, 3, 0, 367, 0, 4038, 77, 9, 1538, 0, 0, 41, 0, 348, 0, 0, 3318, 2007, 19, 87, 0, 9, 0, 0, 32, 0, 0, 285, 62, 1, 0, 350, 2452, 0, 2578, 0, 9, 257, 4087, 0, 3688, 0, 3879, 4, 4025, 3548, 1952, 2165, 1435, 4093, 215, 13, 0, 624, 2196, 746, 15, 2763, 0, 0, 72, 3983, 67, 4096, 0, 14, 0, 2338, 1045, 1, 3979, 0, 0, 22, 3517, 4046, 3, 3418, 8, 0, 4001, 631, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3638, 3955, 1806, 4091, 4096, 4096, 3696, 26, 4096, 4000, 3213, 136, 0, 0, 813, 32, 0, 0, 2296, 20, 0, 3533, 0, 4096, 5, 1, 0, 0, 0, 4050, 0, 1651, 465, 0, 176, 3468, 0, 623, 0, 3635, 4095, 0, 7, 0, 0, 267, 1, 0, 0, 0, 5, 45, 689, 0, 0, 0, 0, 4, 0, 1614, 0, 0, 5, 0, 0, 13, 1, 695, 0, 24, 126, 135, 0, 0, 4096, 0, 351, 41, 32, 82, 0, 185, 3306, 0, 4096, 0, 0, 292, 0, 0, 0, 724, 4, 23, 4040, 4, 4029, 0, 3709, 3908, 0, 0, 2791, 128, 470, 0, 14, 318, 1, 2774, 3752, 3867, 26, 4096, 0, 3312, 421, 4067, 1, 3307, 1605, 308, 0, 64, 1389] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[2076, 0, 356, 338, 0, 0, 4096, 45, 0, 3501, 0, 4053, 3178, 0, 9, 0, 4060, 0, 0, 0, 0, 411, 7, 0, 4085, 2232, 277, 0, 4082, 393, 1554, 4096, 1070, 1125, 0, 4061, 0, 0, 0, 13, 356, 0, 0, 4096, 1, 3602, 3, 0, 7, 0, 54, 0, 0, 0, 3051, 54, 0, 0, 569, 0, 0, 0, 28, 0, 100, 230, 0, 44, 2, 0, 1, 9, 0, 0, 0, 4096, 0, 893, 3902, 1683, 0, 53, 152, 4095, 3465, 3433, 0, 4096, 0, 0, 323, 0, 0, 0, 3772, 0, 0, 667, 43, 72, 2263, 544, 3859, 3803, 0, 0, 0, 1039, 149, 0, 0, 11, 3846, 3940, 4034, 0, 0, 12, 3925, 3455, 4093, 1, 4096, 3911, 0, 0, 0, 21] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2591, 97, 89, 42, 0, 1716, 3992, 195, 0, 406, 0, 3782, 2507, 4024, 3312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3923, 3779, 0, 2670, 121, 23, 1421, 0, 52, 2726, 1121, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 740, 0, 73, 0, 0, 810, 25, 1996, 0, 1, 2657, 0, 0, 1, 0, 1584, 976, 529, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3596, 0, 0, 0, 1, 0, 1182, 0, 55, 0, 3, 0, 2, 85, 1433, 3, 4096, 4096, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 197, 1, 0, 4096, 0, 0, 0, 0, 0, 16, 1, 0, 4065, 10, 1617, 2622, 3418, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1104, 4068, 3070, 433, 4093, 2483, 0, 3993, 22, 0, 0, 0, 1, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4095, 4066, 4073, 0, 100, 2196, 1010, 0, 3943, 3919, 44, 18, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2172, 3660, 0, 62, 0, 0, 0, 0, 0, 0, 479, 1, 0, 1, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22753, 15886, 9592, 14906, 17679, 19311, 23557, 7388] avg=16384.0000 max=23557 max_rank=6 min=7388 min_rank=7 imbalance_ratio=1.437805 max_load_reduction=107515 max_load_reduction_pct=0.820274 moved_to_echo=67935 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2076, 0, 356, 338, 0, 0, 4096, 45, 0, 3501, 0, 4053, 1944, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 411, 7, 0, 4085, 2232, 277, 0, 57, 393, 1554, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1125, 0, 4061, 0, 0, 0, 13, 356, 0, 0, 4096, 1, 3602, 3, 0, 1234, 1161, 777, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 54, 0, 0, 0, 3051, 54, 0, 0, 569, 0, 0, 0, 28, 0, 4096, 4093, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 230, 0, 44, 2, 0, 1, 9, 0, 0, 0, 4096, 0, 893, 3902, 1683, 3457, 4025, 1568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 152, 4095, 3465, 3433, 0, 2935, 0, 0, 323, 0, 0, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, 43, 72, 2263, 544, 2099, 3803, 0, 0, 0, 1039, 149, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 3163, 4034, 0, 0, 12, 0, 3455, 0, 1, 0, 3911, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16418, 17172, 17691, 12420, 20010, 18228, 10690, 18443] avg=16384.0000 max=20010 max_rank=4 min=10690 min_rank=6 imbalance_ratio=1.221313 max_load_reduction=111062 max_load_reduction_pct=0.847336 moved_to_echo=67371 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3638, 3955, 1806, 4091, 4096, 4096, 3696, 26, 4096, 2451, 3213, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 813, 32, 0, 0, 2296, 20, 0, 3533, 0, 4096, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 1651, 465, 0, 176, 3468, 0, 623, 0, 3635, 4095, 0, 7, 0, 0, 1549, 1156, 735, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 1, 0, 0, 0, 5, 45, 689, 0, 0, 0, 0, 4, 0, 1614, 0, 3307, 4067, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 13, 1, 695, 0, 24, 126, 135, 0, 0, 4096, 0, 351, 2917, 0, 1585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 32, 82, 0, 185, 3306, 0, 2940, 0, 0, 292, 0, 0, 0, 724, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4040, 4, 4029, 0, 3709, 2132, 0, 0, 2791, 128, 470, 0, 14, 318, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2774, 3017, 3867, 26, 4096, 0, 0, 421, 0, 1, 0, 1605, 308, 0, 64, 1389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[35300, 10796, 21801, 10394, 9948, 7606, 17659, 17568] avg=16384.0000 max=35300 max_rank=0 min=7606 min_rank=5 imbalance_ratio=2.154541 max_load_reduction=95772 max_load_reduction_pct=0.730682 moved_to_echo=52781 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[405, 1401, 437, 3, 0, 92, 613, 1, 824, 53, 0, 207, 2497, 1569, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 4007, 907, 0, 0, 2383, 3384, 4064, 0, 60, 371, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 0, 0, 0, 0, 1510, 2806, 4093, 1784, 0, 0, 4088, 0, 2719, 9, 0, 1579, 1103, 694, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2903, 12, 0, 0, 0, 4051, 126, 41, 0, 1, 0, 0, 0, 1577, 0, 353, 3122, 4096, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 0, 75, 0, 0, 267, 2634, 0, 54, 99, 3998, 3, 0, 3609, 3956, 1662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 3, 330, 3335, 0, 3997, 4066, 2805, 10, 1679, 2934, 193, 0, 0, 196, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4093, 0, 240, 1043, 2234, 0, 0, 306, 0, 0, 3494, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2852, 3591, 3841, 7, 1, 0, 119, 0, 0, 0, 3805, 5, 0, 1323, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 15270, 21627, 16769, 16815, 21462, 11482, 19507] avg=16384.0000 max=21627 max_rank=2 min=8140 min_rank=0 imbalance_ratio=1.320007 max_load_reduction=109445 max_load_reduction_pct=0.834999 moved_to_echo=69266 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4068, 0, 186, 49, 0, 0, 343, 191, 4089, 4012, 0, 0, 1821, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3002, 0, 222, 1, 22, 3153, 3554, 0, 4, 977, 105, 0, 60, 0, 2809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 589, 0, 0, 897, 0, 937, 0, 2124, 0, 4091, 4055, 3, 0, 1150, 1095, 204, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 4038, 77, 9, 1538, 0, 0, 41, 0, 348, 0, 0, 3318, 2007, 19, 3418, 4046, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 9, 0, 0, 32, 0, 0, 285, 62, 1, 0, 350, 2452, 0, 2578, 20, 3951, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 257, 4087, 0, 3688, 0, 2784, 4, 4025, 3548, 1952, 2165, 1435, 4093, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 624, 2196, 746, 15, 1507, 0, 0, 72, 3983, 67, 4096, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2338, 841, 1, 3979, 0, 0, 0, 3517, 0, 3, 0, 8, 0, 4001, 631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14760, 13909, 15313, 19228, 10948, 28262, 13333, 15319] avg=16384.0000 max=28262 max_rank=5 min=10948 min_rank=4 imbalance_ratio=1.724976 max_load_reduction=102810 max_load_reduction_pct=0.784378 moved_to_echo=67862 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 3950, 718, 0, 861, 0, 2968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 235, 3903, 4094, 9, 3992, 1054, 10, 0, 0, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1, 1, 26, 0, 4005, 4074, 0, 301, 3861, 890, 0, 0, 0, 0, 1052, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3709, 0, 8, 36, 0, 0, 0, 4095, 0, 87, 20, 2736, 0, 0, 3423, 13, 4095, 3855, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 547, 4, 3357, 8, 0, 0, 1, 89, 2487, 0, 0, 63, 3278, 2, 4072, 3609, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3369, 2, 677, 0, 0, 2677, 0, 0, 1720, 0, 2463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2022, 4048, 29, 3, 0, 0, 3274, 1102, 4050, 0, 0, 3758, 637, 0, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 2269, 1688, 3626, 100, 1, 0, 148, 0, 0, 0, 69, 16, 3517, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8498, 21342, 14811, 22564, 18185, 10916, 19204, 15552] avg=16384.0000 max=22564 max_rank=3 min=8498 min_rank=0 imbalance_ratio=1.377197 max_load_reduction=108508 max_load_reduction_pct=0.827850 moved_to_echo=63857 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 186, 0, 4096, 22, 4096, 1, 0, 3975, 2504, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 74, 0, 0, 4096, 22, 0, 4095, 28, 45, 3938, 2, 63, 5, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 48, 0, 0, 3559, 0, 156, 0, 10, 4096, 77, 0, 31, 20, 1583, 55, 793, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 731, 28, 125, 3910, 0, 4095, 78, 0, 0, 4, 0, 4047, 4, 26, 3818, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 1, 2, 0, 0, 48, 4, 0, 0, 125, 0, 4066, 3711, 3035, 3608, 94, 1633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 3381, 4056, 0, 1165, 142, 0, 2195, 0, 0, 0, 0, 3820, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 4054, 0, 3377, 163, 0, 2196, 4071, 200, 2823, 0, 0, 421, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3257, 3234, 526, 1, 0, 0, 1035, 0, 0, 0, 1, 578, 0, 246, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14934, 16494, 10627, 17354, 16340, 18967, 21451, 14905] avg=16384.0000 max=21451 max_rank=6 min=10627 min_rank=2 imbalance_ratio=1.309265 max_load_reduction=109621 max_load_reduction_pct=0.836342 moved_to_echo=71663 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4, 176, 0, 0, 4078, 0, 13, 2963, 0, 525, 2508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 32, 0, 394, 0, 2000, 61, 1989, 0, 3907, 40, 59, 3781, 0, 3858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3938, 44, 24, 0, 74, 0, 113, 65, 523, 0, 3158, 4008, 0, 4048, 0, 0, 1586, 1156, 772, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3348, 1, 0, 4, 6, 0, 2231, 0, 0, 0, 2, 1376, 40, 0, 2198, 0, 4054, 3691, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3238, 126, 0, 3340, 1000, 0, 165, 7, 0, 12, 0, 0, 0, 3949, 38, 919, 3606, 3908, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 1201, 1950, 0, 2940, 154, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 3924, 0, 3834, 892, 29, 1128, 0, 0, 3750, 23, 4, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3435, 3171, 4014, 204, 3275, 0, 0, 520, 0, 52, 0, 3016, 378, 1806, 0, 2519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10269, 20203, 19610, 17437, 21147, 6320, 13696, 22390] avg=16384.0000 max=22390 max_rank=7 min=6320 min_rank=5 imbalance_ratio=1.366577 max_load_reduction=108682 max_load_reduction_pct=0.829178 moved_to_echo=63553 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[15488, 896, 384, 128, 8448, 14720, 5888, 6656, 20352, 6656, 17920, 5120, 384, 7424, 5504, 16128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[640, 256, 5376, 19072, 12672, 16896, 9344, 19712, 256, 11904, 8960, 2176, 4736, 1536, 12672, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[3456, 1152, 128, 7296, 5248, 128, 896, 128, 768, 5376, 256, 4352, 512, 26880, 7680, 14336, 23552, 20096, 10112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[4224, 18944, 12928, 17664, 4352, 5504, 13568, 12160, 1408, 17792, 8064, 1664, 12032, 896, 512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[10112, 2048, 1792, 5248, 128, 1792, 11776, 8320, 10496, 0, 9344, 27008, 5120, 14464, 128, 128, 10368, 7808, 5120, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[14208, 128, 4864, 256, 256, 9600, 6656, 9088, 256, 128, 1024, 4224, 128, 9088, 10752, 512, 30080, 28032, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[16640, 20864, 24192, 12288, 7552, 128, 0, 9216, 0, 128, 0, 12928, 1408, 9344, 2304, 14976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[9216, 1536, 1152, 4352, 4224, 3712, 21376, 4608, 17152, 15360, 128, 17536, 16256, 11776, 3584, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4093, 3959, 0, 13, 2888, 0, 0, 1264, 50, 17, 0, 0, 0, 1431, 1, 0, 512, 3292, 32, 2232, 3026, 121, 400, 3351, 0, 4088, 4095, 5, 0, 6, 133, 90, 34, 125, 4060, 4048, 0, 0, 0, 205, 1, 0, 0, 4096, 69, 5, 3909, 0, 0, 116, 0, 0, 3, 3909, 2, 3703, 0, 3669, 3979, 1, 1, 39, 4089, 0, 0, 72, 0, 302, 0, 0, 1, 4054, 8, 374, 4063, 0, 3722, 1, 0, 4086, 1137, 0, 0, 31, 3, 394, 0, 8, 0, 1695, 34, 3996, 0, 0, 20, 26, 0, 4087, 0, 0, 3769, 0, 0, 0, 136, 135, 162, 36, 0, 0, 4092, 4030, 204, 4061, 4096, 119, 0, 598, 0, 0, 4071, 924, 886, 363, 4094, 20] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4080, 3900, 3789, 0, 0, 231, 3741, 0, 3706, 38, 57, 4096, 0, 0, 4096, 0, 231, 0, 295, 4010, 4092, 0, 92, 36, 4041, 22, 5, 4057, 0, 98, 16, 0, 0, 0, 3475, 392, 4095, 0, 0, 0, 0, 0, 0, 4087, 4090, 29, 0, 1971, 0, 0, 0, 0, 0, 0, 0, 541, 4096, 3804, 258, 4096, 0, 472, 0, 2650, 0, 0, 111, 0, 0, 0, 2, 0, 3824, 0, 29, 4092, 0, 4090, 0, 0, 1, 0, 3, 4095, 3002, 51, 18, 0, 14, 0, 4046, 0, 2294, 3553, 0, 0, 0, 74, 21, 0, 0, 4066, 0, 0, 0, 241, 0, 3551, 141, 0, 9, 1655, 1, 86, 4072, 139, 0, 36, 0, 49, 4078, 0, 0, 970, 0, 3873, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4014, 0, 0, 3395, 0, 181, 0, 1, 1171, 186, 96, 0, 0, 0, 0, 3395, 2240, 1, 4032, 0, 4096, 4093, 3595, 3980, 0, 4096, 18, 828, 3981, 0, 0, 0, 788, 4096, 0, 39, 0, 3980, 0, 0, 7, 0, 848, 3699, 0, 0, 1043, 204, 2755, 4096, 501, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 4016, 0, 0, 0, 0, 55, 0, 4094, 0, 0, 0, 0, 17, 4091, 0, 84, 213, 58, 3305, 0, 4, 3364, 0, 0, 0, 3873, 30, 4089, 0, 0, 0, 3938, 0, 3696, 0, 0, 0, 0, 87, 12, 0, 2, 3945, 0, 0, 1, 0, 0, 243, 0, 0, 4096, 0, 0, 0, 4025, 0, 2, 16, 343, 1958, 0, 4090, 17, 0, 0, 3660, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 313, 0, 4037, 0, 31, 0, 0, 0, 4093, 0, 0, 4086, 82, 0, 1655, 935, 11, 0, 0, 4096, 521, 0, 0, 0, 1, 38, 0, 4096, 0, 0, 28, 1, 3, 96, 9, 0, 4096, 3932, 46, 98, 146, 0, 3922, 0, 4095, 0, 0, 297, 1606, 0, 0, 0, 0, 4096, 7, 1, 4096, 4, 0, 3769, 0, 42, 0, 4096, 16, 0, 3158, 0, 71, 0, 0, 10, 4040, 4, 0, 3894, 52, 3757, 39, 14, 0, 0, 1138, 4074, 3333, 4096, 12, 238, 0, 154, 4029, 91, 0, 4004, 0, 0, 0, 0, 25, 0, 0, 3989, 0, 11, 0, 2784, 0, 2728, 3, 0, 1, 4067, 99, 0, 4029, 4095, 1, 0, 8, 4096, 68, 4020, 0, 0, 2247, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4088, 0, 0, 3975, 456, 4019, 0, 2065, 86, 3981, 25, 1269, 177, 0, 0, 59, 5, 1, 1784, 0, 523, 0, 0, 486, 0, 4096, 107, 0, 4092, 138, 17, 11, 4067, 4090, 1, 212, 4, 4090, 2744, 0, 0, 3712, 20, 0, 7, 4088, 54, 1, 0, 0, 0, 0, 0, 1157, 0, 4049, 80, 21, 0, 743, 0, 195, 0, 0, 0, 2677, 0, 30, 204, 0, 3983, 3, 0, 3933, 0, 0, 0, 0, 1239, 1927, 0, 4096, 0, 0, 3943, 0, 126, 4096, 14, 33, 0, 3829, 576, 3315, 3842, 0, 3569, 0, 0, 20, 0, 0, 4094, 186, 0, 0, 547, 65, 0, 1, 89, 0, 3637, 0, 2, 4096, 2, 0, 2946, 1, 3713, 0, 459, 0, 3772, 1301, 3911, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[23, 0, 0, 30, 2, 0, 1, 0, 0, 3889, 0, 923, 4082, 418, 0, 0, 0, 0, 4074, 0, 0, 4057, 3991, 3218, 48, 0, 64, 307, 4096, 62, 0, 3172, 521, 3868, 0, 0, 22, 4090, 834, 1048, 0, 4086, 185, 4060, 2, 3032, 1000, 1, 1883, 3563, 38, 0, 0, 0, 0, 0, 72, 0, 430, 0, 4092, 0, 0, 153, 4072, 0, 0, 1623, 0, 0, 711, 0, 0, 1914, 0, 0, 2479, 0, 3853, 897, 0, 8, 0, 0, 27, 4092, 608, 4090, 2, 0, 0, 4096, 676, 6, 0, 0, 0, 0, 753, 1985, 11, 0, 4055, 0, 0, 13, 3943, 5, 3719, 7, 0, 171, 0, 4096, 0, 3475, 4096, 0, 29, 319, 0, 0, 890, 4084, 11, 1946, 2868, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 6, 0, 3840, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 4, 3836, 0, 1512, 0, 4096, 3450, 0, 859, 4076, 0, 20, 0, 44, 10, 0, 3547, 0, 3836, 4009, 1, 286, 940, 0, 3167, 0, 0, 2328, 3, 0, 3987, 0, 0, 35, 162, 15, 0, 0, 1118, 11, 0, 4092, 0, 4082, 0, 2818, 4, 1905, 617, 0, 0, 4089, 99, 0, 4096, 2, 3, 3601, 126, 4096, 0, 0, 4082, 2785, 0, 3981, 4087, 164, 0, 26, 0, 0, 4095, 17, 2962, 4068, 93, 982, 0, 88, 0, 0, 11, 3761, 0, 1, 0, 0, 4088, 0, 0, 0, 0, 312, 0, 0, 183, 0, 5, 4045, 64, 4096, 0, 4095, 0, 0, 4093, 4026, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4074, 0, 6, 126, 2, 0, 176, 26, 978, 3608, 0, 4096, 0, 282, 1088, 3084, 97, 4017, 0, 4047, 3777, 3274, 3229, 0, 274, 39, 0, 0, 4085, 1732, 0, 0, 2786, 4053, 15, 0, 0, 166, 4009, 415, 61, 0, 2674, 4075, 7, 28, 0, 0, 407, 0, 0, 0, 94, 4096, 25, 0, 0, 799, 0, 0, 3773, 107, 5, 0, 5, 0, 69, 3, 0, 4022, 4020, 3790, 0, 2838, 11, 0, 5, 0, 4016, 3, 1, 4073, 3306, 0, 0, 3358, 262, 3156, 1072, 10, 0, 4096, 245, 10, 1894, 0, 21, 56, 56, 0, 0, 0, 3187, 3, 271, 594, 0, 3868, 4, 26, 5, 0, 0, 0, 2, 2547, 4091, 0, 4, 0, 570, 0, 0, 41, 0, 0, 3875, 1874] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 313, 0, 4037, 0, 31, 0, 0, 0, 4093, 0, 0, 4086, 82, 0, 1655, 1309, 237, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 935, 11, 0, 0, 4096, 521, 0, 0, 0, 1, 38, 0, 1011, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 96, 9, 0, 2452, 3932, 46, 98, 146, 0, 3922, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 1606, 0, 0, 0, 0, 4096, 7, 1, 4096, 4, 0, 3769, 0, 42, 0, 2892, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 3158, 0, 71, 0, 0, 10, 3865, 4, 0, 3894, 52, 3757, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1138, 4074, 3333, 4096, 12, 238, 0, 154, 939, 91, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 3989, 0, 11, 0, 2784, 0, 2728, 3, 0, 1, 4096, 3085, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 99, 0, 3792, 4095, 1, 0, 8, 4096, 68, 4020, 0, 0, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16018, 6641, 14800, 17145, 18962, 18093, 16920, 22493] avg=16384.0000 max=22493 max_rank=7 min=6641 min_rank=1 imbalance_ratio=1.372864 max_load_reduction=108579 max_load_reduction_pct=0.828392 moved_to_echo=76468 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 3959, 0, 13, 2888, 0, 0, 1264, 50, 17, 0, 0, 0, 1431, 1294, 239, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 512, 3292, 32, 2232, 3026, 121, 400, 3351, 0, 4088, 1010, 5, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 90, 34, 125, 4060, 2423, 0, 0, 0, 205, 1, 0, 0, 4096, 69, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 0, 0, 116, 0, 0, 3, 3909, 2, 3703, 0, 3669, 3979, 1, 1, 39, 1217, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 72, 0, 302, 0, 0, 1, 3879, 8, 374, 4063, 0, 3722, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1137, 0, 0, 31, 3, 394, 0, 8, 0, 395, 34, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 26, 0, 4087, 0, 0, 3769, 0, 0, 0, 136, 135, 162, 36, 0, 0, 4094, 3085, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4030, 204, 3822, 4096, 119, 0, 598, 0, 0, 4071, 924, 886, 363, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15423, 18076, 11241, 20879, 16511, 10084, 15633, 23225] avg=16384.0000 max=23225 max_rank=7 min=10084 min_rank=5 imbalance_ratio=1.417542 max_load_reduction=107847 max_load_reduction_pct=0.822807 moved_to_echo=65453 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4074, 0, 6, 126, 2, 0, 176, 26, 978, 3608, 0, 4096, 0, 282, 1088, 3084, 53, 150, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 4017, 0, 4047, 3777, 3274, 3229, 0, 274, 39, 0, 0, 1008, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2786, 4053, 15, 0, 0, 100, 4009, 415, 61, 0, 2674, 4075, 7, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 0, 0, 94, 4096, 25, 0, 0, 799, 0, 0, 3773, 107, 5, 0, 2940, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 69, 3, 0, 4022, 4020, 3790, 0, 2715, 11, 0, 5, 0, 4016, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4073, 3306, 0, 0, 3358, 262, 3156, 1072, 10, 0, 955, 245, 10, 1894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 56, 56, 0, 0, 0, 3187, 3, 271, 594, 0, 3868, 4, 26, 5, 0, 3875, 3077, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2397, 4091, 0, 4, 0, 570, 0, 0, 41, 0, 0, 0, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17872, 21494, 18223, 12259, 18659, 18342, 15244, 8979] avg=16384.0000 max=21494 max_rank=1 min=8979 min_rank=7 imbalance_ratio=1.311890 max_load_reduction=109578 max_load_reduction_pct=0.836014 moved_to_echo=61224 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4014, 0, 0, 3395, 0, 181, 0, 1, 1171, 186, 96, 0, 0, 0, 0, 3395, 1272, 237, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2240, 1, 4032, 0, 4096, 4093, 3595, 3980, 0, 4096, 18, 828, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 788, 4096, 0, 39, 0, 2383, 0, 0, 7, 0, 848, 3699, 0, 0, 1043, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2755, 4096, 501, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 4016, 0, 0, 0, 2827, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 4094, 0, 0, 0, 0, 17, 3914, 0, 84, 213, 58, 3305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3364, 0, 0, 0, 3873, 30, 4089, 0, 0, 0, 917, 0, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 12, 0, 2, 3945, 0, 0, 1, 0, 0, 243, 0, 0, 4096, 3660, 2999, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3788, 0, 2, 16, 343, 1958, 0, 4090, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14125, 27961, 13107, 22713, 11740, 15973, 15239, 10214] avg=16384.0000 max=27961 max_rank=1 min=10214 min_rank=7 imbalance_ratio=1.706604 max_load_reduction=103111 max_load_reduction_pct=0.786674 moved_to_echo=53166 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 6, 0, 3840, 0, 0, 0, 0, 0, 0, 30, 0, 1226, 10, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3836, 0, 1512, 0, 4096, 3450, 0, 859, 1006, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 10, 0, 3547, 0, 2296, 4009, 1, 286, 940, 0, 3167, 0, 0, 2328, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 0, 35, 162, 15, 0, 0, 1118, 11, 0, 4092, 0, 4082, 0, 2940, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2818, 4, 1905, 617, 0, 0, 4089, 99, 0, 3919, 2, 3, 3601, 126, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 2785, 0, 3981, 4087, 164, 0, 26, 0, 0, 954, 17, 2962, 4068, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 982, 0, 88, 0, 0, 11, 3761, 0, 1, 0, 0, 4088, 0, 0, 0, 0, 4026, 3070, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 0, 0, 173, 0, 5, 4045, 64, 4096, 0, 4095, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5293, 14783, 16631, 16756, 21279, 23219, 16228, 16883] avg=16384.0000 max=23219 max_rank=5 min=5293 min_rank=0 imbalance_ratio=1.417175 max_load_reduction=107853 max_load_reduction_pct=0.822853 moved_to_echo=77609 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4080, 3900, 3789, 0, 0, 231, 3741, 0, 3706, 38, 57, 4096, 0, 0, 4096, 1313, 243, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 295, 4010, 4092, 0, 92, 36, 4041, 22, 5, 997, 0, 98, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3475, 392, 2442, 0, 0, 0, 0, 0, 0, 4087, 4090, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1971, 0, 0, 0, 0, 0, 0, 0, 541, 4096, 3804, 258, 4096, 0, 472, 0, 2909, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2650, 0, 0, 111, 0, 0, 0, 2, 0, 3655, 0, 29, 4092, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 4095, 3002, 51, 18, 0, 14, 0, 933, 0, 2294, 3553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 21, 0, 0, 4066, 0, 0, 0, 241, 0, 3551, 141, 0, 9, 3873, 3060, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1655, 1, 86, 3829, 139, 0, 36, 0, 49, 4078, 0, 0, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[29459, 13935, 14515, 18487, 14629, 13964, 15240, 10843] avg=16384.0000 max=29459 max_rank=0 min=10843 min_rank=7 imbalance_ratio=1.798035 max_load_reduction=101613 max_load_reduction_pct=0.775246 moved_to_echo=54676 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4088, 0, 0, 3975, 456, 4019, 0, 2065, 86, 3981, 25, 1269, 177, 0, 0, 59, 1308, 241, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 1784, 0, 523, 0, 0, 486, 0, 4096, 107, 0, 1010, 138, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4090, 1, 212, 4, 2448, 2744, 0, 0, 3712, 20, 0, 7, 4088, 54, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1157, 0, 4049, 80, 21, 0, 743, 0, 195, 0, 0, 2749, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2677, 0, 30, 204, 0, 3983, 3, 0, 3763, 0, 0, 0, 0, 1239, 1927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3943, 0, 126, 4096, 14, 33, 0, 892, 576, 3315, 3842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3569, 0, 0, 20, 0, 0, 4094, 186, 0, 0, 547, 65, 0, 1, 89, 0, 3911, 3082, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3637, 0, 2, 3855, 2, 0, 2946, 1, 3713, 0, 459, 0, 3772, 1301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21919, 8178, 21448, 9328, 13826, 20933, 15752, 19688] avg=16384.0000 max=21919 max_rank=0 min=8178 min_rank=1 imbalance_ratio=1.337830 max_load_reduction=109153 max_load_reduction_pct=0.832771 moved_to_echo=70199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[23, 0, 0, 30, 2, 0, 1, 0, 0, 3889, 0, 923, 4082, 418, 0, 0, 1308, 204, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 4057, 3991, 3218, 48, 0, 64, 307, 1011, 62, 0, 3172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 3868, 0, 0, 22, 2448, 834, 1048, 0, 4086, 185, 4060, 2, 3032, 1000, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1883, 3563, 38, 0, 0, 0, 0, 0, 72, 0, 430, 0, 4092, 0, 0, 153, 2940, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 1623, 0, 0, 711, 0, 0, 1831, 0, 0, 2479, 0, 3853, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 27, 4092, 608, 4090, 2, 0, 0, 955, 676, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 753, 1985, 11, 0, 4055, 0, 0, 13, 3943, 5, 3719, 7, 0, 171, 2868, 3085, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3271, 4096, 0, 29, 319, 0, 0, 890, 4084, 11, 1946, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10963, 20004, 21107, 13505, 15466, 10464, 20816, 18747] avg=16384.0000 max=21107 max_rank=2 min=10464 min_rank=5 imbalance_ratio=1.288269 max_load_reduction=109965 max_load_reduction_pct=0.838966 moved_to_echo=65493 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[17792, 2816, 2048, 9728, 256, 4480, 12928, 3968, 128, 27648, 128, 512, 18432, 256, 28160, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[3328, 4352, 10496, 7680, 20480, 18304, 15360, 7936, 4864, 19200, 256, 6144, 8064, 2048, 256, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[12288, 4480, 8064, 19328, 512, 4352, 3328, 9728, 2304, 20736, 256, 6400, 12544, 896, 1152, 13824, 9088, 1664, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[4608, 128, 1152, 6272, 128, 128, 30976, 256, 384, 640, 7680, 8192, 10496, 256, 128, 4352, 30464, 24576, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[11264, 13312, 640, 128, 256, 5504, 4224, 12160, 768, 18048, 4352, 4736, 27904, 384, 4608, 256, 21504, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[8448, 16256, 256, 7424, 4480, 17024, 15616, 1536, 512, 9216, 3840, 18944, 4224, 19456, 4608, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[13824, 8320, 384, 24960, 16640, 128, 7168, 1408, 14592, 4224, 17664, 5120, 5760, 9984, 0, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 19712, 7296, 1152, 16128, 21888, 5376, 15872, 1408, 128, 256, 7040, 1664, 16384, 17408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[49, 0, 4093, 41, 1, 0, 0, 0, 4096, 3120, 13, 0, 3, 301, 1, 0, 0, 0, 0, 0, 2692, 15, 4084, 6, 0, 1805, 0, 0, 15, 1, 1373, 0, 22, 0, 1, 677, 2749, 4013, 3823, 0, 0, 25, 322, 1544, 181, 9, 259, 5, 283, 1, 4096, 0, 0, 7, 0, 0, 0, 4096, 2, 0, 0, 2771, 0, 30, 4096, 0, 4096, 4095, 2, 0, 17, 0, 4096, 2720, 2781, 196, 0, 0, 0, 0, 13, 0, 0, 0, 4094, 1, 78, 4094, 2789, 0, 0, 0, 0, 1137, 0, 0, 0, 3715, 388, 3317, 4096, 3952, 4073, 4096, 135, 4095, 3306, 31, 0, 0, 0, 0, 95, 0, 0, 3656, 377, 113, 168, 0, 3949, 2390, 0, 4096, 3938, 0, 0, 156] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 3, 4051, 0, 4096, 0, 159, 0, 2052, 0, 0, 4096, 0, 0, 0, 0, 0, 79, 0, 0, 429, 0, 3986, 0, 0, 3563, 317, 4094, 0, 41, 3825, 30, 8, 0, 0, 4084, 0, 99, 0, 40, 4096, 188, 3934, 0, 339, 0, 4064, 0, 0, 0, 3921, 14, 4, 150, 142, 872, 0, 4088, 0, 178, 0, 3937, 3150, 0, 4078, 0, 0, 1656, 0, 2773, 17, 0, 3621, 3, 696, 0, 58, 4096, 0, 0, 4092, 123, 0, 0, 23, 35, 0, 0, 3914, 402, 0, 0, 2914, 30, 0, 0, 11, 772, 2301, 4090, 4074, 4040, 0, 4081, 0, 0, 3252, 0, 4035, 0, 45, 0, 857, 1, 0, 649, 0, 1511, 3439, 0, 3643, 5, 6, 285, 33, 1278, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 2454, 1, 0, 472, 0, 0, 0, 0, 116, 0, 4096, 4006, 0, 1, 31, 0, 0, 4, 0, 0, 6, 2894, 0, 0, 0, 1412, 1921, 2270, 460, 3957, 0, 14, 0, 0, 4007, 0, 1080, 679, 14, 63, 0, 0, 5, 0, 4041, 4044, 0, 0, 48, 2197, 0, 4090, 85, 0, 978, 194, 4027, 4096, 115, 4, 3289, 1928, 492, 0, 575, 10, 4096, 148, 0, 1862, 251, 3467, 4091, 4091, 1083, 160, 0, 0, 0, 0, 0, 0, 3660, 3865, 4067, 0, 0, 0, 1, 0, 3536, 4074, 0, 1233, 0, 33, 49, 352, 0, 2145, 11, 791, 3783, 1, 0, 592, 0, 3867, 1356, 69, 9, 4090, 0, 0, 0, 33, 2680, 3285, 0, 3852, 3507, 0, 0, 0, 0, 167, 539] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 25, 37, 1355, 108, 0, 347, 4096, 0, 288, 0, 12, 0, 0, 0, 3, 0, 643, 0, 93, 0, 1, 0, 0, 4096, 0, 4080, 0, 0, 246, 1675, 3864, 0, 0, 0, 0, 2440, 0, 488, 23, 10, 11, 0, 0, 1109, 3, 3420, 0, 0, 0, 2593, 0, 1209, 3961, 0, 0, 925, 0, 4011, 0, 4023, 2799, 0, 0, 4021, 9, 0, 1, 3268, 0, 4096, 116, 731, 14, 4066, 0, 4067, 4096, 0, 53, 4080, 0, 4067, 0, 220, 0, 59, 3994, 2630, 0, 0, 0, 0, 10, 0, 4007, 0, 0, 4096, 4094, 4096, 0, 4064, 0, 0, 0, 741, 4092, 0, 0, 0, 1, 0, 26, 3728, 0, 0, 112, 2455, 3776, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[78, 3126, 4063, 888, 4087, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 160, 0, 1, 4, 0, 13, 0, 0, 0, 1, 0, 1, 0, 0, 14, 0, 357, 1224, 45, 0, 0, 4096, 0, 0, 703, 332, 1787, 0, 0, 0, 4083, 2270, 0, 0, 25, 1114, 671, 0, 4096, 3969, 1, 3063, 0, 2978, 0, 1032, 936, 0, 3922, 0, 2263, 186, 3029, 152, 5, 4091, 0, 0, 3992, 0, 4079, 3297, 3, 157, 0, 0, 3673, 2805, 0, 62, 336, 4096, 0, 3733, 3581, 52, 2206, 145, 2523, 0, 248, 26, 0, 2, 4041, 2086, 0, 4096, 0, 3821, 0, 1, 0, 0, 5, 187, 3201, 0, 470, 0, 0, 0, 0, 4091, 3881, 620, 3902, 0, 0, 1661, 0, 1030, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4083, 133, 0, 9, 0, 0, 11, 155, 1023, 0, 17, 50, 4079, 19, 0, 0, 22, 0, 0, 1400, 0, 0, 0, 0, 0, 1121, 0, 0, 0, 3122, 0, 0, 22, 0, 3807, 2822, 89, 492, 2759, 0, 75, 3, 4085, 0, 224, 0, 36, 48, 0, 4095, 2775, 4096, 0, 4096, 10, 109, 4095, 0, 61, 0, 1467, 4096, 0, 247, 0, 4032, 1298, 0, 3941, 0, 0, 2675, 1054, 3, 4016, 630, 0, 0, 0, 4086, 349, 189, 1586, 0, 8, 0, 0, 4095, 0, 0, 279, 4094, 13, 55, 0, 0, 0, 1480, 3843, 0, 3910, 11, 4056, 0, 2321, 4072, 0, 1248, 0, 0, 0, 2627, 76, 0, 4092, 0, 3220, 4057, 0, 4096, 248, 4095, 3, 0, 361, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[3, 711, 4096, 3684, 11, 1, 2065, 0, 0, 0, 10, 4095, 0, 0, 357, 0, 35, 13, 199, 0, 1, 662, 3733, 2, 125, 0, 0, 0, 0, 8, 2985, 535, 0, 2398, 0, 4074, 0, 3561, 0, 4089, 0, 0, 68, 641, 2966, 0, 0, 8, 0, 273, 3896, 0, 3687, 0, 0, 0, 0, 4091, 1543, 0, 0, 16, 162, 0, 1374, 0, 4071, 4037, 4056, 191, 12, 0, 3741, 0, 3974, 0, 0, 3891, 3575, 0, 4074, 4, 31, 530, 3259, 407, 878, 3996, 85, 0, 0, 17, 2582, 743, 0, 0, 85, 430, 0, 3331, 27, 4077, 4022, 3011, 1097, 0, 2824, 0, 0, 59, 0, 67, 35, 22, 0, 1159, 13, 0, 3854, 0, 3229, 0, 19, 0, 2032, 1328, 0, 19] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4093, 0, 469, 3883, 0, 0, 0, 1581, 9, 0, 4095, 19, 0, 36, 0, 0, 0, 0, 43, 0, 45, 139, 0, 0, 379, 605, 4091, 0, 19, 3993, 0, 1646, 3500, 0, 4095, 3812, 0, 2, 4095, 0, 0, 1, 5, 104, 133, 314, 103, 0, 0, 4096, 0, 0, 143, 1, 5, 3, 4078, 0, 48, 5, 0, 2900, 237, 3130, 2928, 44, 153, 3780, 4058, 0, 0, 4092, 0, 0, 2, 0, 2806, 2580, 0, 1704, 3864, 4, 0, 29, 3849, 0, 178, 3603, 6, 0, 0, 3968, 0, 696, 0, 28, 0, 0, 0, 3718, 3966, 2001, 4095, 0, 0, 3367, 190, 0, 0, 0, 24, 173, 2, 0, 3470, 20, 0, 3492, 0, 3348, 3984, 3929, 0, 15, 0, 0, 951] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[49, 0, 4093, 41, 1, 0, 0, 0, 4096, 3120, 13, 0, 3, 301, 1, 0, 1931, 0, 533, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2692, 15, 4084, 6, 0, 1805, 0, 0, 15, 1, 1373, 0, 4096, 3812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 1, 677, 2749, 4013, 3823, 0, 0, 25, 322, 1544, 181, 9, 259, 5, 962, 921, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 1, 4096, 0, 0, 7, 0, 0, 0, 3134, 2, 0, 0, 2771, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 2, 0, 17, 0, 0, 2720, 850, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4094, 1, 78, 4094, 2789, 0, 0, 0, 0, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3715, 388, 3317, 4096, 2498, 4073, 0, 135, 4095, 3306, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 3656, 377, 113, 168, 0, 3714, 2390, 0, 4096, 3938, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14466, 17899, 15748, 10324, 16072, 12206, 25654, 18703] avg=16384.0000 max=25654 max_rank=6 min=10324 min_rank=3 imbalance_ratio=1.565796 max_load_reduction=105418 max_load_reduction_pct=0.804276 moved_to_echo=72635 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4051, 0, 4096, 0, 159, 0, 2052, 0, 0, 4096, 0, 0, 0, 0, 483, 1165, 545, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 429, 0, 3986, 0, 0, 3563, 317, 4094, 0, 41, 3825, 30, 4081, 3370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4084, 0, 99, 0, 40, 4096, 188, 3934, 0, 339, 0, 4064, 0, 960, 941, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 14, 4, 150, 142, 872, 0, 3128, 0, 178, 0, 3937, 3150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 1656, 0, 2773, 17, 0, 0, 3, 213, 0, 58, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 123, 0, 0, 23, 35, 0, 0, 3914, 402, 0, 0, 1749, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 772, 2301, 4090, 4074, 2554, 0, 0, 0, 0, 3252, 0, 4035, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 857, 1, 0, 649, 0, 1511, 3439, 0, 3426, 5, 6, 285, 33, 1278, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16901, 23815, 18970, 15496, 12894, 10368, 21134, 11494] avg=16384.0000 max=23815 max_rank=1 min=10368 min_rank=5 imbalance_ratio=1.453552 max_load_reduction=107257 max_load_reduction_pct=0.818306 moved_to_echo=55890 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2454, 1, 0, 472, 0, 0, 0, 0, 116, 0, 4096, 4006, 0, 1, 31, 2841, 1629, 1, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 6, 2894, 0, 0, 0, 1412, 1921, 2270, 460, 3957, 0, 3783, 3228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4007, 0, 1080, 679, 14, 63, 0, 0, 5, 0, 4041, 4044, 0, 946, 2, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 2197, 0, 4090, 85, 0, 978, 194, 3081, 4096, 115, 4, 3289, 1928, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 10, 4096, 148, 0, 1862, 251, 0, 4091, 1250, 1083, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3660, 3865, 4067, 0, 0, 0, 1, 0, 3536, 2445, 0, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 49, 352, 0, 2145, 8, 791, 0, 1, 0, 592, 0, 3867, 1356, 69, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 33, 2680, 3285, 0, 3623, 3507, 0, 0, 0, 0, 167, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15887, 19935, 15124, 20597, 13526, 18807, 9272, 17924] avg=16384.0000 max=20597 max_rank=3 min=9272 min_rank=6 imbalance_ratio=1.257141 max_load_reduction=110475 max_load_reduction_pct=0.842857 moved_to_echo=59529 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[78, 3126, 4063, 888, 4087, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 160, 0, 2834, 1009, 552, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 13, 0, 0, 0, 1, 0, 1, 0, 0, 14, 0, 357, 1224, 3821, 3715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4096, 0, 0, 703, 332, 1787, 0, 0, 0, 4083, 2270, 0, 0, 699, 955, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 1114, 671, 0, 4096, 3969, 1, 3063, 0, 2279, 0, 1032, 936, 0, 3922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2263, 186, 3029, 152, 5, 4091, 0, 0, 0, 0, 1245, 3297, 3, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3673, 2805, 0, 62, 336, 4096, 0, 3733, 3581, 52, 2206, 145, 1514, 0, 248, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4041, 2086, 0, 2589, 0, 0, 0, 1, 0, 0, 5, 187, 3201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 0, 0, 0, 0, 4091, 3881, 620, 3670, 0, 0, 1661, 0, 1030, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21169, 9151, 15202, 21108, 14428, 22477, 12112, 15425] avg=16384.0000 max=22477 max_rank=5 min=9151 min_rank=1 imbalance_ratio=1.371887 max_load_reduction=108595 max_load_reduction_pct=0.828514 moved_to_echo=64442 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 469, 3883, 0, 0, 0, 1581, 9, 0, 4095, 19, 0, 36, 0, 0, 1587, 535, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 45, 139, 0, 0, 379, 605, 4091, 0, 19, 3993, 0, 4095, 3809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1646, 3500, 0, 4095, 3812, 0, 2, 4095, 0, 0, 1, 5, 104, 133, 314, 103, 958, 924, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 143, 1, 5, 3, 3120, 0, 48, 5, 0, 2900, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3130, 2928, 44, 153, 3780, 4058, 0, 0, 0, 0, 0, 2, 0, 2806, 2580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1704, 3864, 4, 0, 29, 3849, 0, 178, 3603, 6, 0, 0, 2381, 0, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 3718, 2507, 2001, 0, 0, 0, 3367, 190, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 2, 0, 3470, 20, 0, 3492, 0, 3149, 3984, 3929, 0, 15, 0, 0, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16590, 17218, 19891, 10558, 19481, 16314, 11835, 19185] avg=16384.0000 max=19891 max_rank=2 min=10558 min_rank=3 imbalance_ratio=1.214050 max_load_reduction=111181 max_load_reduction_pct=0.848244 moved_to_echo=66815 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4083, 133, 0, 9, 0, 0, 11, 155, 1023, 0, 17, 50, 4079, 19, 0, 3, 1641, 530, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 1400, 0, 0, 0, 0, 0, 1121, 0, 0, 0, 3122, 0, 4056, 2489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 3807, 2822, 89, 492, 2759, 0, 75, 3, 4085, 0, 224, 0, 36, 967, 915, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 4095, 2775, 4096, 0, 4096, 10, 109, 3128, 0, 61, 0, 1467, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 4032, 1298, 0, 3941, 0, 0, 0, 1054, 0, 4016, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 349, 189, 1586, 0, 8, 0, 0, 4095, 0, 0, 279, 2453, 13, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1480, 3843, 0, 2465, 11, 0, 0, 2321, 4072, 0, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2627, 76, 0, 4092, 0, 3220, 4057, 0, 3850, 248, 4095, 3, 0, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11939, 12210, 16542, 23981, 15218, 13113, 15440, 22629] avg=16384.0000 max=23981 max_rank=3 min=11939 min_rank=0 imbalance_ratio=1.463684 max_load_reduction=107091 max_load_reduction_pct=0.817039 moved_to_echo=66400 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 25, 37, 1355, 108, 0, 347, 4096, 0, 288, 0, 12, 2844, 1052, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 643, 0, 93, 0, 1, 0, 0, 4096, 0, 4080, 0, 4094, 3042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 1675, 3864, 0, 0, 0, 0, 2440, 0, 488, 23, 10, 11, 0, 0, 930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1109, 3, 3420, 0, 0, 0, 2593, 0, 1209, 3031, 0, 0, 925, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 2799, 0, 0, 4021, 9, 0, 1, 0, 0, 1252, 116, 731, 14, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4096, 0, 53, 4080, 0, 4067, 0, 220, 0, 59, 3994, 1578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4007, 0, 0, 4096, 0, 4096, 0, 4064, 0, 0, 0, 741, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 26, 3728, 0, 0, 112, 2455, 3776, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14486, 16052, 9687, 16301, 17032, 22214, 21106, 14194] avg=16384.0000 max=22214 max_rank=5 min=9687 min_rank=2 imbalance_ratio=1.355835 max_load_reduction=108858 max_load_reduction_pct=0.830521 moved_to_echo=74546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 711, 4096, 3684, 11, 1, 2065, 0, 0, 0, 10, 4095, 0, 0, 357, 0, 2760, 1033, 550, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 13, 199, 0, 1, 662, 3733, 2, 125, 0, 0, 0, 0, 8, 2985, 535, 3011, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2398, 0, 4074, 0, 3561, 0, 4089, 0, 0, 68, 641, 2966, 0, 0, 8, 961, 950, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 3896, 0, 3687, 0, 0, 0, 0, 3130, 1543, 0, 0, 16, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1374, 0, 4071, 4037, 4056, 191, 12, 0, 0, 0, 1214, 0, 0, 3891, 3575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4, 31, 530, 3259, 407, 878, 3996, 85, 0, 0, 17, 1549, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 430, 0, 3331, 27, 2577, 4022, 0, 1097, 0, 2824, 0, 0, 59, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 22, 0, 1159, 13, 0, 3854, 0, 3037, 0, 19, 0, 2032, 1328, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19634, 14792, 19908, 12707, 22421, 15573, 14519, 11518] avg=16384.0000 max=22421 max_rank=4 min=11518 min_rank=7 imbalance_ratio=1.368469 max_load_reduction=108651 max_load_reduction_pct=0.828941 moved_to_echo=64031 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1536, 1536, 26496, 2816, 16000, 4480, 6912, 4992, 1536, 24064, 5760, 1536, 1920, 11520, 20224, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[19328, 6528, 15360, 15488, 12032, 15104, 1920, 256, 0, 7936, 6144, 8832, 1664, 11008, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[256, 14592, 16512, 5120, 16768, 128, 2304, 1408, 8064, 4352, 384, 24704, 4096, 4736, 640, 128, 13696, 9216, 3328, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 256, 128, 4608, 1408, 14848, 128, 128, 5760, 3456, 10112, 6400, 640, 23808, 1792, 31104, 27008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[256, 4992, 8576, 20736, 14080, 15232, 15104, 0, 5376, 6528, 21504, 256, 9216, 1664, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[8448, 128, 0, 13056, 512, 11648, 25984, 640, 24576, 10368, 10624, 9856, 6144, 8192, 256, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[1792, 6272, 1792, 28800, 9472, 8960, 5760, 11392, 8448, 384, 4864, 6400, 7808, 6784, 8704, 256, 7424, 5632, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[21760, 11264, 256, 6016, 15744, 12544, 5120, 12032, 18304, 512, 2304, 8064, 13696, 2048, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3651, 0, 0, 0, 3290, 3870, 1222, 21, 4007, 331, 1541, 79, 0, 0, 0, 1407, 1, 69, 0, 0, 0, 5, 0, 18, 0, 1560, 0, 105, 4034, 0, 0, 0, 412, 15, 0, 0, 0, 0, 0, 10, 0, 0, 4001, 2393, 0, 0, 7, 4096, 0, 0, 0, 3944, 20, 828, 3715, 0, 0, 0, 284, 140, 0, 69, 0, 0, 0, 1672, 3776, 0, 382, 793, 0, 4094, 347, 1, 2938, 31, 0, 0, 0, 1403, 0, 3399, 48, 473, 4095, 1, 3933, 0, 4060, 0, 1858, 0, 93, 0, 0, 0, 4049, 0, 4077, 1872, 992, 0, 4096, 18, 74, 2, 46, 125, 3732, 4096, 15, 4096, 0, 0, 3845, 4004, 0, 697, 0, 0, 3478, 251, 4096, 0, 2353, 3667, 0, 2849] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[9, 1, 3949, 3576, 83, 2471, 663, 5, 0, 0, 3825, 3938, 0, 0, 0, 0, 4042, 169, 11, 3918, 223, 366, 0, 238, 103, 23, 41, 0, 3, 0, 44, 0, 0, 269, 0, 3949, 12, 0, 4074, 41, 3992, 0, 0, 2928, 0, 1, 0, 1, 27, 0, 1302, 6, 0, 0, 4096, 62, 3161, 4095, 0, 0, 0, 0, 0, 4085, 0, 0, 3270, 1793, 9, 0, 4027, 3697, 0, 0, 0, 4096, 0, 460, 0, 72, 0, 4096, 1, 0, 4096, 0, 3668, 4028, 1324, 0, 0, 0, 0, 4028, 1, 0, 0, 1, 0, 1206, 0, 0, 4094, 296, 0, 1479, 0, 0, 4091, 0, 4096, 0, 1, 0, 3832, 0, 4022, 0, 9, 3751, 0, 0, 3546, 0, 0, 2160, 0, 20] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[18, 14, 3989, 5, 2246, 2578, 1174, 3555, 6, 873, 0, 3903, 149, 0, 1034, 0, 4096, 1573, 1, 4095, 152, 4096, 0, 0, 0, 466, 738, 1, 292, 4, 0, 399, 0, 0, 0, 66, 1388, 4081, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3962, 0, 0, 4043, 4069, 0, 0, 4030, 4095, 0, 25, 134, 0, 3852, 0, 2669, 0, 115, 0, 0, 1648, 2544, 0, 1970, 4092, 3728, 0, 0, 304, 11, 279, 165, 0, 1, 0, 662, 1, 4041, 0, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 3, 0, 4092, 3778, 95, 0, 770, 39, 0, 3925, 0, 67, 0, 3, 3210, 162, 0, 176, 0, 4096, 4096, 3316, 0, 0, 0, 3505, 4090, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4096, 167, 3855, 48, 4094, 4096, 89, 1067, 3329, 7, 4096, 0, 0, 0, 2, 4, 3889, 2, 0, 2067, 14, 1841, 0, 0, 130, 0, 286, 0, 295, 5, 403, 0, 0, 0, 0, 0, 0, 2983, 4, 22, 0, 0, 4096, 0, 0, 0, 0, 2767, 35, 0, 667, 3801, 0, 4083, 0, 0, 2412, 490, 1248, 0, 0, 11, 7, 2762, 3117, 3562, 2, 0, 0, 0, 116, 4093, 522, 0, 4, 0, 1, 1, 40, 3849, 3, 1391, 0, 509, 30, 6, 4016, 0, 0, 0, 122, 0, 0, 151, 0, 1783, 3199, 1010, 0, 3687, 0, 1109, 3912, 7, 4053, 0, 2, 0, 4039, 0, 479, 3531, 3462, 4088, 2591, 0, 58, 0, 4081, 0, 0, 0, 2, 814, 3213, 1052, 4, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 3678, 264, 0, 4055, 36, 17, 0, 192, 4096, 0, 0, 43, 0, 29, 301, 0, 86, 4, 0, 0, 0, 99, 0, 4033, 0, 0, 157, 0, 297, 58, 3506, 486, 0, 0, 0, 0, 4096, 660, 189, 0, 4095, 3666, 0, 0, 0, 31, 0, 322, 2925, 0, 3, 3579, 0, 0, 3483, 4072, 0, 0, 0, 0, 0, 3919, 283, 4096, 3191, 0, 0, 0, 0, 4096, 3381, 0, 3, 302, 3969, 1, 0, 3572, 22, 4092, 1, 3835, 4003, 2, 3924, 0, 0, 0, 0, 0, 0, 0, 4072, 2641, 275, 4094, 12, 1292, 0, 0, 4095, 0, 4096, 19, 0, 0, 3364, 1527, 631, 1, 1, 838, 4071, 0, 0, 0, 0, 0, 32, 0, 225, 34, 3716, 0, 2, 688] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4053, 0, 3651, 4051, 2, 120, 0, 0, 1, 0, 437, 0, 0, 0, 0, 0, 3, 0, 0, 1586, 0, 0, 0, 0, 3, 185, 1, 53, 0, 0, 0, 0, 0, 0, 56, 4095, 792, 0, 5, 110, 4093, 4087, 4094, 4072, 287, 3, 29, 4096, 2161, 288, 2247, 4078, 0, 4093, 1165, 2, 0, 3367, 0, 0, 0, 79, 2, 4083, 0, 4074, 0, 0, 0, 0, 0, 4089, 7, 184, 1528, 4094, 3613, 0, 0, 3991, 123, 1235, 0, 4096, 245, 1, 4039, 4032, 0, 3956, 0, 419, 0, 0, 28, 4078, 197, 1907, 0, 4088, 0, 0, 28, 0, 2064, 0, 0, 0, 2470, 6, 0, 2011, 718, 8, 3278, 7, 0, 0, 1, 4, 4008, 0, 0, 0, 0, 908, 0, 4007] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4096, 36, 4096, 0, 0, 594, 45, 6, 3835, 0, 3571, 0, 432, 0, 0, 176, 4064, 2, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1441, 0, 0, 2624, 0, 4, 0, 3780, 2318, 3, 0, 122, 4096, 0, 4053, 3684, 46, 0, 0, 1, 557, 0, 390, 4086, 1344, 4074, 174, 1, 473, 4096, 4095, 0, 101, 0, 0, 1426, 1, 0, 0, 0, 0, 0, 0, 8, 1, 18, 3935, 9, 1816, 161, 1, 3731, 1157, 4069, 2, 0, 3903, 0, 1, 4024, 0, 0, 230, 0, 8, 7, 31, 322, 4, 5, 0, 4096, 4076, 0, 1862, 1162, 1, 0, 0, 0, 4096, 3982, 269, 3761, 0, 4054, 1869, 60, 3483, 0, 0, 0, 3852, 0, 351, 401, 1927, 295, 0, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4075, 0, 4094, 947, 261, 20, 4, 0, 0, 1, 4092, 0, 0, 9, 0, 0, 2544, 0, 0, 0, 0, 0, 183, 921, 0, 312, 0, 0, 0, 1553, 0, 0, 385, 0, 0, 3456, 0, 1, 56, 0, 947, 0, 2791, 2871, 757, 13, 78, 4095, 0, 166, 1325, 2324, 984, 4082, 0, 929, 3865, 4096, 5, 140, 0, 0, 1, 3514, 4096, 4074, 5, 0, 0, 26, 15, 3460, 3950, 0, 1570, 0, 25, 0, 0, 0, 0, 0, 0, 0, 150, 0, 4086, 3920, 0, 4095, 349, 0, 3962, 228, 0, 0, 1743, 13, 7, 3287, 1, 8, 1079, 0, 4090, 275, 751, 2, 3297, 92, 0, 4059, 0, 3994, 812, 2084, 0, 0, 0, 588, 156, 3074, 0, 0, 3958, 4095, 0, 3699] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3651, 0, 0, 0, 3290, 3870, 1222, 21, 4007, 331, 1541, 79, 0, 0, 0, 1407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 69, 0, 0, 0, 5, 0, 18, 0, 1560, 0, 105, 4034, 0, 0, 0, 2849, 3019, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 412, 15, 0, 0, 0, 0, 0, 10, 0, 0, 4001, 2393, 0, 0, 7, 4096, 871, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3944, 20, 274, 3715, 0, 0, 0, 284, 140, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1672, 3776, 0, 382, 793, 0, 3223, 347, 1, 2938, 31, 0, 0, 0, 1403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3399, 48, 473, 4095, 1, 3933, 0, 4060, 0, 1858, 0, 93, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 4077, 1872, 992, 0, 4096, 18, 74, 2, 46, 125, 713, 4096, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3176, 4004, 0, 697, 0, 0, 3478, 251, 4096, 0, 2353, 3667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19419, 12214, 11853, 8446, 14566, 18581, 24271, 21722] avg=16384.0000 max=24271 max_rank=6 min=8446 min_rank=3 imbalance_ratio=1.481384 max_load_reduction=106801 max_load_reduction_pct=0.814827 moved_to_echo=79140 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 167, 2364, 48, 4094, 4096, 89, 1067, 3329, 7, 4096, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 2, 0, 2067, 14, 1841, 0, 0, 130, 0, 286, 0, 295, 5, 403, 0, 4091, 3267, 2735, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2983, 4, 22, 0, 0, 4096, 0, 0, 0, 0, 2767, 871, 486, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 667, 3801, 0, 1348, 0, 0, 2412, 490, 1248, 0, 0, 11, 7, 2762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3117, 3562, 2, 0, 0, 0, 116, 3222, 522, 0, 4, 0, 1, 1, 40, 3849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1391, 0, 509, 30, 6, 4016, 0, 0, 0, 122, 0, 0, 151, 0, 1783, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3199, 1010, 0, 3687, 0, 1109, 3912, 7, 4053, 0, 2, 0, 772, 0, 479, 3531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3462, 4088, 2141, 0, 58, 0, 4081, 0, 0, 0, 2, 814, 3213, 1052, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23459, 20030, 11261, 12781, 14436, 8429, 21761, 18915] avg=16384.0000 max=23459 max_rank=0 min=8429 min_rank=5 imbalance_ratio=1.431824 max_load_reduction=107613 max_load_reduction_pct=0.821022 moved_to_echo=63541 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 2256, 264, 0, 4055, 36, 17, 0, 192, 4096, 0, 0, 43, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 301, 0, 86, 4, 0, 0, 0, 99, 0, 4033, 0, 0, 157, 0, 297, 58, 688, 2721, 2398, 959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3506, 486, 0, 0, 0, 0, 4096, 660, 189, 0, 4095, 3666, 0, 0, 0, 31, 872, 463, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 2925, 0, 3, 1181, 0, 0, 3483, 4072, 0, 0, 0, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 4096, 3191, 0, 0, 0, 0, 3224, 3381, 0, 3, 302, 3969, 1, 0, 3572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4092, 1, 3835, 4003, 2, 3924, 0, 0, 0, 0, 0, 0, 0, 4072, 2641, 658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 4094, 12, 1292, 0, 0, 4095, 0, 4096, 19, 0, 0, 643, 1527, 631, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 838, 3363, 0, 0, 0, 0, 0, 32, 0, 225, 34, 3716, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15084, 11801, 18114, 15905, 22022, 23250, 16685, 8211] avg=16384.0000 max=23250 max_rank=5 min=8211 min_rank=7 imbalance_ratio=1.419067 max_load_reduction=107822 max_load_reduction_pct=0.822617 moved_to_echo=70168 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[9, 1, 2421, 3576, 83, 2471, 663, 5, 0, 0, 3825, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 169, 11, 3918, 223, 366, 0, 238, 103, 23, 41, 0, 3, 0, 44, 0, 20, 3309, 0, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 0, 3949, 12, 0, 4074, 41, 3992, 0, 0, 2928, 0, 1, 0, 1, 787, 498, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 1302, 6, 0, 0, 4096, 62, 3161, 4095, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3270, 1793, 9, 0, 4027, 2910, 0, 0, 0, 4096, 0, 460, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 3668, 4028, 1324, 0, 0, 0, 0, 4028, 1, 0, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1206, 0, 0, 4094, 296, 0, 1479, 0, 0, 782, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3166, 0, 4022, 0, 9, 3751, 0, 0, 3546, 0, 0, 2160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16992, 13540, 16599, 16834, 16637, 21861, 11954, 16655] avg=16384.0000 max=21861 max_rank=5 min=11954 min_rank=6 imbalance_ratio=1.334290 max_load_reduction=109211 max_load_reduction_pct=0.833214 moved_to_echo=67107 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4075, 0, 2510, 947, 261, 20, 4, 0, 0, 1, 4092, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2544, 0, 0, 0, 0, 0, 183, 921, 0, 312, 0, 0, 0, 1553, 0, 0, 3699, 2667, 2735, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 0, 0, 3456, 0, 1, 56, 0, 947, 0, 2791, 2871, 757, 13, 78, 4095, 736, 516, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 1325, 2324, 984, 1347, 0, 929, 3865, 4096, 5, 140, 0, 0, 1, 3514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4074, 5, 0, 0, 26, 15, 2724, 3950, 0, 1570, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 4086, 3920, 0, 4095, 349, 0, 3962, 228, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1743, 13, 7, 3287, 1, 8, 1079, 0, 4090, 275, 751, 2, 630, 92, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 671, 2084, 0, 0, 0, 588, 156, 3074, 0, 0, 3958, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11919, 15682, 16712, 18696, 16485, 16921, 16037, 18620] avg=16384.0000 max=18696 max_rank=3 min=11919 min_rank=0 imbalance_ratio=1.141113 max_load_reduction=112376 max_load_reduction_pct=0.857361 moved_to_echo=68063 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[18, 14, 2446, 5, 2246, 2578, 1174, 3555, 6, 873, 0, 3903, 149, 0, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1573, 1, 4095, 152, 4096, 0, 0, 0, 466, 738, 1, 292, 4, 0, 399, 4096, 3175, 2708, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 1388, 4081, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 541, 503, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3962, 0, 0, 1335, 4069, 0, 0, 4030, 4095, 0, 25, 134, 0, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2669, 0, 115, 0, 0, 1648, 2003, 0, 1970, 4092, 3728, 0, 0, 304, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 165, 0, 1, 0, 662, 1, 4041, 0, 0, 1, 0, 14, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4092, 3778, 95, 0, 770, 39, 0, 750, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3210, 134, 0, 176, 0, 4096, 4096, 3316, 0, 0, 0, 3505, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18001, 26932, 10686, 21503, 16540, 5190, 9594, 22626] avg=16384.0000 max=26932 max_rank=1 min=5190 min_rank=5 imbalance_ratio=1.643799 max_load_reduction=104140 max_load_reduction_pct=0.794525 moved_to_echo=53950 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4053, 0, 2231, 4051, 2, 120, 0, 0, 1, 0, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1586, 0, 0, 0, 0, 3, 185, 1, 53, 0, 0, 0, 0, 4007, 2002, 2746, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 4095, 792, 0, 5, 110, 4093, 4087, 4094, 4072, 287, 3, 29, 4096, 874, 463, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2161, 288, 2247, 4078, 0, 1347, 1165, 2, 0, 3367, 0, 0, 0, 79, 2, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 3215, 7, 184, 1528, 4094, 3613, 0, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 1235, 0, 4096, 245, 1, 4039, 4032, 0, 3956, 0, 419, 0, 0, 28, 4078, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 1907, 0, 4088, 0, 0, 28, 0, 2064, 0, 0, 0, 468, 6, 0, 2011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 718, 8, 2702, 7, 0, 0, 1, 4, 4008, 0, 0, 0, 0, 908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10895, 11543, 27200, 18819, 20706, 22784, 10769, 8356] avg=16384.0000 max=27200 max_rank=2 min=8356 min_rank=7 imbalance_ratio=1.660156 max_load_reduction=103872 max_load_reduction_pct=0.792480 moved_to_echo=62615 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 36, 2512, 0, 0, 594, 45, 6, 3835, 0, 3571, 0, 432, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 2, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1441, 0, 0, 2624, 4087, 3313, 2729, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3780, 2318, 3, 0, 122, 4096, 0, 4053, 3684, 46, 0, 0, 1, 1, 516, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 0, 390, 4086, 1344, 1345, 174, 1, 473, 4096, 4095, 0, 101, 0, 0, 1426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 1, 18, 3935, 9, 1816, 161, 1, 3731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1157, 4069, 2, 0, 3903, 0, 1, 4024, 0, 0, 230, 0, 8, 7, 31, 322, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 4096, 4076, 0, 1862, 1162, 1, 0, 0, 0, 783, 3982, 269, 3761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 1544, 60, 3483, 0, 0, 0, 3852, 0, 351, 401, 1927, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15303, 19330, 18647, 18088, 9680, 14056, 20001, 15967] avg=16384.0000 max=20001 max_rank=6 min=9680 min_rank=4 imbalance_ratio=1.220764 max_load_reduction=111071 max_load_reduction_pct=0.847404 moved_to_echo=59704 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[7552, 20224, 10368, 1920, 512, 896, 5888, 20608, 8320, 2176, 14080, 12288, 9472, 640, 384, 16640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2816, 896, 12928, 18304, 2432, 8192, 13312, 1024, 13440, 24320, 9728, 384, 128, 384, 128, 23680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[9472, 7040, 4096, 19584, 5120, 5248, 23040, 1664, 14464, 2560, 896, 128, 5632, 9728, 5632, 17536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[24192, 256, 16768, 8960, 9984, 17920, 3328, 4736, 11264, 1408, 21760, 7936, 640, 128, 1152, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[4224, 16256, 17024, 6272, 7808, 768, 8192, 8448, 14848, 3328, 8320, 1280, 18688, 16384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1664, 18560, 128, 8960, 16640, 768, 23680, 20096, 5504, 8064, 2560, 512, 4096, 4480, 4224, 8832, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[18944, 1920, 128, 11776, 512, 6400, 256, 1280, 256, 6656, 1152, 256, 6272, 1664, 768, 3200, 23552, 23552, 16640, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[4352, 896, 128, 15360, 4608, 7168, 12416, 1024, 13440, 4096, 23168, 19712, 1152, 128, 128, 15104, 5632, 3456, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[2, 3871, 0, 0, 56, 4094, 0, 46, 4088, 1744, 3968, 4096, 0, 0, 0, 1170, 63, 958, 3314, 20, 1148, 0, 272, 4094, 0, 0, 1958, 4090, 1, 0, 1, 0, 167, 0, 3471, 0, 2016, 0, 730, 0, 0, 4072, 408, 4092, 60, 232, 4088, 0, 200, 0, 0, 395, 0, 0, 0, 40, 477, 0, 0, 243, 0, 0, 23, 0, 8, 6, 1740, 2, 16, 0, 201, 0, 3989, 0, 4096, 42, 2, 0, 3463, 0, 45, 0, 0, 3937, 18, 3137, 0, 0, 2, 110, 4039, 3703, 0, 1, 3, 0, 23, 35, 6, 4090, 0, 0, 4005, 1658, 2790, 0, 0, 853, 4091, 0, 0, 0, 900, 3794, 0, 0, 1693, 7, 3957, 1, 3703, 3678, 0, 3847, 0, 3613, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 1990, 168, 0, 0, 2892, 0, 1569, 0, 2586, 0, 0, 65, 0, 0, 123, 50, 11, 0, 0, 0, 0, 783, 3982, 0, 0, 3068, 3846, 0, 836, 241, 0, 0, 4088, 3250, 3343, 13, 0, 984, 2589, 84, 0, 0, 0, 0, 4095, 0, 0, 93, 0, 0, 0, 368, 0, 0, 0, 99, 4020, 608, 4096, 0, 4089, 6, 707, 4090, 3943, 0, 2446, 4096, 0, 0, 2446, 2447, 41, 0, 0, 0, 0, 4089, 34, 299, 0, 0, 4076, 0, 3115, 2432, 0, 0, 9, 0, 83, 0, 797, 0, 376, 288, 0, 4096, 260, 0, 0, 3717, 4095, 3731, 1, 130, 77, 0, 0, 1175, 0, 3787, 2659, 8, 3453, 17, 3971, 8, 1403, 3264, 4096, 0, 0, 6, 215, 0, 1054] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[3510, 0, 0, 0, 0, 0, 2, 1, 3109, 0, 702, 3, 206, 0, 0, 0, 15, 121, 0, 22, 0, 3491, 28, 4052, 1486, 298, 4094, 3670, 0, 402, 531, 0, 54, 3, 4089, 76, 2187, 0, 0, 0, 0, 0, 283, 0, 515, 4067, 396, 3770, 0, 0, 0, 0, 4092, 0, 10, 0, 3949, 6, 36, 285, 0, 4096, 396, 0, 14, 3264, 3825, 0, 32, 0, 4086, 0, 3987, 0, 2, 0, 418, 1520, 251, 0, 0, 0, 4096, 92, 3, 1619, 0, 0, 0, 2, 683, 4040, 0, 3627, 0, 2467, 0, 4096, 0, 2319, 401, 0, 0, 1, 2529, 3948, 3532, 17, 0, 2, 3952, 2, 731, 4014, 3770, 2880, 0, 19, 4012, 0, 833, 1897, 0, 3850, 41, 3, 0, 142] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 3476, 167, 431, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 279, 0, 0, 0, 3891, 0, 0, 646, 0, 0, 1, 0, 3802, 0, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 0, 4090, 4092, 0, 2756, 0, 1337, 0, 4063, 0, 3510, 103, 524, 0, 51, 3940, 0, 0, 17, 1656, 0, 429, 301, 16, 4096, 0, 0, 0, 4092, 0, 1, 3093, 13, 58, 0, 3490, 0, 0, 0, 7, 26, 4039, 122, 0, 0, 3537, 13, 0, 3354, 0, 33, 8, 0, 0, 4087, 0, 3955, 2, 0, 0, 0, 1713, 0, 19, 0, 4038, 4072, 3908, 0, 0, 0, 0, 4094, 4096, 5, 232, 3868, 2839, 0, 0, 906, 4001, 0, 3712, 181, 1, 2649, 0, 0, 0, 748] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[2, 0, 681, 498, 0, 3143, 2535, 0, 4, 0, 9, 327, 7, 0, 2, 0, 1846, 0, 1744, 2, 4096, 210, 234, 0, 8, 648, 3820, 768, 0, 910, 0, 18, 4094, 4088, 3865, 871, 1, 0, 0, 0, 4, 4096, 117, 0, 0, 3610, 3707, 258, 0, 0, 0, 0, 766, 0, 0, 12, 0, 0, 1097, 9, 0, 3950, 0, 0, 1502, 4087, 0, 0, 1, 0, 1, 2571, 4051, 0, 1, 28, 0, 906, 208, 2, 0, 0, 0, 4096, 0, 23, 0, 0, 13, 0, 16, 4035, 0, 3500, 0, 0, 3, 0, 68, 3675, 4041, 224, 4096, 0, 2377, 2680, 3954, 3819, 2153, 3830, 1240, 0, 0, 4096, 0, 1, 0, 4096, 2271, 0, 3261, 0, 0, 4049, 3251, 28, 0, 761] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[40, 22, 7, 0, 1342, 0, 0, 0, 0, 44, 0, 0, 201, 3873, 0, 0, 0, 0, 820, 4096, 1058, 2439, 0, 837, 0, 0, 4050, 473, 0, 197, 0, 0, 0, 3669, 3921, 0, 0, 0, 0, 3835, 0, 2028, 2, 1, 3233, 13, 380, 4094, 0, 70, 4096, 3755, 3391, 0, 0, 1, 4096, 0, 0, 3009, 0, 3896, 0, 0, 3842, 2, 0, 0, 0, 0, 0, 1975, 3944, 0, 4061, 0, 0, 0, 0, 0, 1, 0, 3, 4081, 0, 3686, 0, 0, 0, 3009, 0, 0, 628, 4096, 0, 0, 0, 0, 143, 2655, 3541, 0, 17, 4096, 3729, 0, 4096, 12, 287, 4086, 4, 195, 2, 25, 486, 10, 42, 0, 470, 1, 4090, 1234, 50, 4081, 2911, 0, 0, 2492] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 183, 114, 5, 4086, 352, 141, 184, 3677, 78, 4065, 454, 0, 3813, 112, 0, 0, 630, 0, 0, 31, 0, 0, 33, 306, 0, 0, 102, 0, 672, 3493, 0, 0, 0, 3950, 14, 3664, 291, 0, 3543, 2153, 2268, 996, 304, 439, 3455, 0, 1165, 0, 0, 383, 3980, 0, 1, 272, 3456, 4095, 2000, 0, 0, 0, 2709, 32, 0, 406, 0, 3632, 0, 0, 4096, 286, 0, 4043, 4058, 0, 5, 0, 0, 3852, 0, 1, 18, 5, 3835, 541, 0, 0, 0, 1126, 0, 0, 0, 0, 3460, 1720, 0, 0, 0, 3541, 955, 4096, 48, 0, 0, 3766, 55, 0, 0, 825, 4096, 16, 147, 0, 0, 3112, 302, 10, 3820, 8, 25, 977, 4095, 792, 3933, 3668, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 4033, 2, 0, 0, 85, 0, 6, 12, 4067, 0, 0, 1, 4095, 0, 39, 454, 0, 4042, 2626, 0, 3877, 5, 4096, 0, 3, 4096, 0, 287, 7, 0, 6, 0, 3905, 4096, 171, 13, 0, 3698, 15, 0, 2859, 0, 0, 0, 4, 0, 4095, 0, 1, 4066, 0, 4, 0, 0, 76, 2829, 0, 0, 0, 0, 21, 0, 0, 0, 4092, 4086, 1, 2573, 128, 0, 0, 1984, 24, 0, 4061, 0, 3569, 4074, 0, 334, 22, 4096, 0, 3740, 4062, 0, 1346, 0, 1308, 385, 2529, 0, 3797, 0, 2, 0, 0, 16, 0, 772, 0, 3243, 0, 4040, 0, 0, 0, 0, 0, 4096, 0, 0, 670, 644, 394, 37, 4083, 0, 0, 0, 20, 0, 4096, 4017, 0, 0, 1039] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3510, 0, 0, 0, 0, 0, 2, 1, 3109, 0, 702, 3, 206, 0, 0, 0, 1702, 2027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 121, 0, 22, 0, 3491, 28, 4052, 1486, 298, 4094, 3670, 0, 402, 531, 0, 1823, 1455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 3, 0, 76, 2187, 0, 0, 0, 0, 0, 283, 0, 515, 3580, 396, 3770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 10, 0, 3949, 6, 36, 285, 0, 4096, 396, 0, 4089, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3264, 3825, 0, 32, 0, 4086, 0, 2472, 0, 2, 0, 418, 1520, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 92, 3, 1619, 0, 0, 0, 2, 683, 4040, 0, 3627, 0, 2467, 266, 0, 487, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2319, 401, 0, 0, 1, 0, 3948, 3532, 17, 0, 2, 3952, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 731, 4014, 3770, 2880, 0, 19, 4012, 0, 567, 1897, 0, 0, 41, 3, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11262, 21488, 10864, 17786, 15884, 17442, 18270, 18076] avg=16384.0000 max=21488 max_rank=1 min=10864 min_rank=2 imbalance_ratio=1.311523 max_load_reduction=109584 max_load_reduction_pct=0.836060 moved_to_echo=69672 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 3476, 167, 431, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 279, 0, 2628, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3891, 0, 0, 646, 0, 0, 1, 0, 3802, 0, 0, 0, 0, 0, 1255, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4090, 4092, 0, 2756, 0, 1337, 0, 3576, 0, 3510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 524, 0, 51, 3940, 0, 0, 17, 1656, 0, 429, 301, 16, 4096, 0, 0, 4095, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 3093, 13, 58, 0, 2165, 0, 0, 0, 7, 26, 4039, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3537, 13, 0, 3354, 0, 33, 8, 0, 0, 4087, 0, 3955, 2, 0, 1185, 983, 487, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1713, 0, 19, 0, 3055, 4072, 0, 0, 0, 0, 0, 4094, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 3868, 2839, 0, 0, 906, 4001, 0, 2527, 181, 1, 0, 0, 0, 0, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20663, 10868, 19364, 16508, 13616, 17696, 17054, 15303] avg=16384.0000 max=20663 max_rank=0 min=10868 min_rank=1 imbalance_ratio=1.261169 max_load_reduction=110409 max_load_reduction_pct=0.842354 moved_to_echo=63669 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 3871, 0, 0, 56, 4094, 0, 46, 4088, 1744, 3968, 4096, 0, 0, 0, 1170, 1877, 2025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 958, 3314, 20, 1148, 0, 272, 4094, 0, 0, 1958, 4090, 1, 0, 1, 0, 1822, 1455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 0, 2016, 0, 730, 0, 0, 4072, 408, 4092, 60, 205, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 395, 0, 0, 0, 40, 477, 0, 0, 243, 0, 0, 23, 0, 3471, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 1740, 2, 16, 0, 201, 0, 2474, 0, 4096, 42, 2, 0, 3463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 3937, 18, 3137, 0, 0, 2, 110, 4039, 3703, 0, 1, 3, 0, 1182, 975, 27, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 35, 6, 4090, 0, 0, 3030, 1658, 0, 0, 0, 853, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 900, 3794, 0, 0, 1693, 7, 3957, 1, 2521, 3678, 0, 0, 0, 3613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[27037, 19196, 15838, 5762, 12050, 17239, 13786, 20164] avg=16384.0000 max=27037 max_rank=0 min=5762 min_rank=3 imbalance_ratio=1.650208 max_load_reduction=104035 max_load_reduction_pct=0.793724 moved_to_echo=63239 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1990, 168, 0, 0, 2892, 0, 1569, 0, 2586, 0, 0, 65, 0, 0, 123, 2509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 11, 0, 0, 0, 0, 783, 3982, 0, 0, 3068, 3846, 0, 836, 241, 0, 0, 893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 3343, 13, 0, 984, 2589, 84, 0, 0, 0, 0, 3604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 368, 0, 0, 0, 99, 4020, 608, 4096, 0, 4089, 6, 707, 3250, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3943, 0, 2446, 4096, 0, 0, 2446, 1518, 41, 0, 0, 0, 0, 4089, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 4076, 0, 3115, 2432, 0, 0, 9, 0, 83, 0, 797, 0, 376, 1042, 905, 491, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 4096, 260, 0, 0, 2812, 4095, 0, 1, 130, 77, 0, 0, 1175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3787, 2659, 8, 3453, 17, 3971, 8, 1403, 2222, 4096, 0, 0, 6, 215, 0, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11902, 13710, 14705, 18558, 22703, 13661, 12934, 22899] avg=16384.0000 max=22899 max_rank=7 min=11902 min_rank=0 imbalance_ratio=1.397644 max_load_reduction=108173 max_load_reduction_pct=0.825294 moved_to_echo=72197 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[40, 22, 7, 0, 1342, 0, 0, 0, 0, 44, 0, 0, 201, 3873, 0, 0, 2509, 2149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 820, 4096, 1058, 2439, 0, 837, 0, 0, 4050, 473, 0, 197, 0, 0, 1932, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3669, 0, 0, 0, 0, 0, 3835, 0, 2028, 2, 1, 3233, 12, 380, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 4096, 3755, 3391, 0, 0, 1, 4096, 0, 0, 3009, 0, 3896, 0, 0, 3921, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 2, 0, 0, 0, 0, 0, 1975, 2446, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 4081, 0, 3686, 0, 0, 0, 3009, 0, 0, 628, 4096, 0, 0, 1306, 4, 1, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 2655, 3541, 0, 13, 4096, 0, 0, 4096, 12, 287, 4086, 4, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 25, 486, 10, 42, 0, 470, 1, 2784, 1234, 50, 0, 2911, 0, 0, 2492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10187, 17341, 17254, 27455, 12326, 16874, 19128, 10507] avg=16384.0000 max=27455 max_rank=3 min=10187 min_rank=0 imbalance_ratio=1.675720 max_load_reduction=103617 max_load_reduction_pct=0.790535 moved_to_echo=58835 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 183, 114, 5, 4086, 352, 141, 184, 3677, 78, 4065, 454, 0, 3813, 112, 0, 2533, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 0, 31, 0, 0, 33, 306, 0, 0, 102, 0, 672, 3493, 0, 1863, 1475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3664, 291, 0, 3543, 2153, 2268, 996, 304, 439, 3041, 0, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 3980, 0, 1, 272, 3456, 4095, 2000, 0, 0, 0, 2709, 32, 0, 3950, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 406, 0, 3632, 0, 0, 4096, 286, 0, 2507, 4058, 0, 5, 0, 0, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 18, 5, 3835, 541, 0, 0, 0, 1126, 0, 0, 0, 0, 3460, 1720, 0, 312, 0, 414, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3541, 955, 4096, 48, 0, 0, 0, 55, 0, 0, 825, 4096, 16, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3112, 302, 10, 3820, 8, 25, 665, 4095, 792, 0, 3668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21867, 8605, 17878, 22111, 18842, 11493, 13779, 16497] avg=16384.0000 max=22111 max_rank=3 min=8605 min_rank=1 imbalance_ratio=1.349548 max_load_reduction=108961 max_load_reduction_pct=0.831306 moved_to_echo=60611 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 681, 498, 0, 3143, 2535, 0, 4, 0, 9, 327, 7, 0, 2, 0, 1599, 2132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1846, 0, 1744, 2, 4096, 210, 234, 0, 8, 648, 3820, 768, 0, 910, 0, 18, 1917, 1478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4088, 0, 871, 1, 0, 0, 0, 4, 4096, 117, 0, 0, 3177, 3707, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 766, 0, 0, 12, 0, 0, 1097, 9, 0, 3950, 0, 0, 3865, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1502, 4087, 0, 0, 1, 0, 1, 2571, 2512, 0, 1, 28, 0, 906, 208, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 13, 0, 16, 4035, 0, 3500, 0, 0, 1043, 997, 433, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 68, 3675, 4041, 224, 3099, 0, 0, 2680, 3954, 3819, 2153, 3830, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 2271, 0, 2218, 0, 0, 0, 3251, 28, 0, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10939, 17699, 20413, 10477, 11819, 14217, 28786, 16722] avg=16384.0000 max=28786 max_rank=6 min=10477 min_rank=3 imbalance_ratio=1.756958 max_load_reduction=102286 max_load_reduction_pct=0.780380 moved_to_echo=71544 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4033, 2, 0, 0, 85, 0, 6, 12, 4067, 0, 0, 1, 4095, 0, 39, 2718, 2157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 0, 4042, 2626, 0, 3877, 5, 4096, 0, 3, 4096, 0, 287, 7, 0, 6, 1939, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 0, 171, 13, 0, 3698, 15, 0, 2859, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4066, 0, 4, 0, 0, 76, 2829, 0, 0, 0, 0, 21, 0, 0, 4096, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4086, 1, 2573, 128, 0, 0, 1224, 24, 0, 4061, 0, 3569, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 22, 4096, 0, 3740, 4062, 0, 1346, 0, 1308, 385, 2529, 0, 3797, 0, 2, 0, 792, 4, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 772, 0, 2451, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 670, 644, 394, 37, 4083, 0, 0, 0, 20, 0, 0, 4017, 0, 0, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17215, 22165, 14756, 12415, 23832, 22450, 7335, 10904] avg=16384.0000 max=23832 max_rank=4 min=7335 min_rank=6 imbalance_ratio=1.454590 max_load_reduction=107240 max_load_reduction_pct=0.818176 moved_to_echo=64521 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[512, 640, 8576, 8192, 12672, 128, 384, 3712, 17280, 6144, 2176, 8064, 128, 22912, 512, 768, 30848, 8832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[4352, 15872, 0, 4480, 7936, 384, 9600, 14080, 2304, 18176, 1920, 5760, 4352, 17280, 8576, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[5760, 19200, 10880, 7040, 1920, 17024, 14848, 1536, 13568, 15232, 896, 0, 13952, 3968, 0, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[384, 4224, 9600, 14080, 12928, 384, 14464, 13952, 0, 6784, 11776, 4864, 7424, 16128, 14592, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[9984, 19584, 13312, 2560, 9856, 4352, 4736, 7040, 17408, 4224, 8192, 4224, 512, 6144, 20096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[7680, 13696, 1152, 1024, 5504, 10624, 2688, 1920, 11008, 12672, 8832, 4992, 512, 16000, 512, 1408, 18176, 14080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[768, 128, 11776, 20224, 4352, 19072, 2432, 1408, 1152, 4480, 5248, 18560, 640, 23296, 1792, 2944, 6400, 4736, 2432, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2432, 1792, 13824, 6784, 6400, 10752, 1408, 17152, 1920, 1024, 24960, 13056, 384, 3072, 4352, 128, 12672, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 4091, 29, 188, 4070, 37, 78, 2456, 0, 0, 254, 0, 0, 3478, 0, 263, 3994, 6, 0, 938, 3826, 4, 3695, 2313, 0, 0, 0, 0, 0, 208, 8, 0, 0, 7, 1126, 0, 0, 4034, 3606, 0, 1921, 1911, 0, 3439, 4059, 747, 0, 0, 4077, 0, 48, 0, 526, 4055, 0, 111, 11, 0, 544, 0, 5, 0, 25, 4090, 2779, 4084, 2071, 3, 208, 4096, 0, 0, 3187, 52, 10, 3047, 526, 0, 4094, 4086, 5, 4095, 0, 0, 0, 0, 4070, 0, 1092, 0, 0, 0, 0, 0, 549, 0, 0, 0, 3997, 4087, 60, 0, 0, 13, 0, 3876, 0, 3310, 1592, 92, 4, 0, 0, 0, 3219, 0, 2365, 99, 1741, 0, 4051, 0, 0, 216, 10, 0, 8, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[293, 4073, 0, 0, 0, 0, 2309, 21, 0, 0, 2336, 0, 505, 3277, 0, 0, 4058, 0, 0, 3253, 3835, 2, 0, 0, 0, 0, 1924, 0, 0, 0, 3409, 0, 335, 2651, 0, 701, 2, 4092, 3988, 1, 905, 648, 114, 4096, 3873, 1, 3843, 1, 4092, 234, 4088, 0, 4068, 0, 7, 0, 968, 28, 0, 0, 4096, 7, 150, 12, 0, 4084, 0, 2, 0, 3841, 0, 1, 3943, 0, 0, 0, 3642, 1102, 2421, 183, 0, 4080, 0, 0, 0, 0, 0, 50, 3079, 0, 0, 3, 4082, 1344, 3950, 3383, 0, 0, 2095, 0, 2363, 41, 0, 0, 82, 366, 4096, 0, 0, 4041, 421, 0, 0, 47, 4036, 15, 0, 0, 0, 0, 160, 0, 626, 1196, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 3448, 0, 0, 3232, 3722, 18, 0, 1480, 1133, 672, 0, 510, 0, 2223, 692, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 24, 3521, 0, 1389, 4095, 0, 1102, 888, 0, 0, 4096, 2958, 0, 0, 0, 76, 1052, 1, 954, 741, 406, 3734, 0, 4049, 3184, 0, 4095, 47, 0, 0, 38, 0, 0, 25, 46, 0, 3818, 0, 0, 3365, 3287, 4003, 0, 78, 4079, 0, 0, 4081, 944, 3650, 0, 3672, 0, 17, 373, 0, 16, 0, 3559, 564, 0, 4096, 2751, 4093, 0, 11, 0, 22, 4044, 4096, 0, 953, 0, 0, 115, 2, 0, 1, 0, 4096, 0, 2331, 0, 0, 2527, 4051, 1, 534, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[167, 42, 3, 0, 0, 10, 0, 1413, 0, 0, 0, 4069, 3364, 3917, 0, 19, 3322, 4069, 0, 1906, 0, 2, 0, 1, 31, 699, 0, 0, 0, 1, 4083, 0, 3010, 0, 12, 1, 0, 4074, 3944, 150, 48, 1587, 0, 11, 4078, 1474, 3831, 4095, 4, 0, 0, 0, 2192, 3574, 1, 3672, 332, 204, 2, 0, 975, 3223, 0, 0, 0, 0, 70, 33, 17, 2, 0, 66, 4075, 2, 4, 0, 0, 186, 4096, 0, 120, 3344, 0, 418, 2821, 841, 0, 0, 4091, 0, 0, 3883, 0, 4063, 0, 3772, 0, 6, 4096, 4088, 0, 2638, 0, 0, 0, 3236, 3919, 0, 32, 0, 1697, 0, 15, 544, 2128, 0, 13, 0, 3962, 4096, 0, 0, 0, 0, 29, 1033, 24, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1814, 0, 2192, 4077, 788, 4082, 0, 0, 3788, 0, 55, 0, 0, 590, 0, 1, 2365, 0, 0, 0, 3, 301, 597, 4093, 1, 0, 0, 0, 0, 0, 4096, 2, 20, 0, 0, 32, 0, 3917, 2814, 0, 4096, 2779, 3616, 1, 2195, 3173, 4059, 0, 3891, 280, 3994, 0, 0, 4, 0, 0, 0, 29, 0, 1, 4088, 1918, 1260, 0, 0, 4096, 4067, 0, 0, 4093, 1427, 697, 2, 2006, 0, 50, 4091, 4096, 17, 0, 0, 0, 1, 100, 0, 0, 31, 3704, 0, 0, 0, 0, 0, 3535, 8, 0, 7, 0, 1729, 0, 0, 489, 0, 0, 0, 0, 0, 4095, 0, 1435, 1788, 13, 31, 0, 0, 0, 0, 6, 230, 3687, 11, 4093, 4085, 1, 337, 3975, 1350, 677] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4050, 0, 0, 3, 72, 58, 61, 982, 0, 0, 37, 1559, 1584, 2453, 72, 0, 4096, 3, 1682, 368, 595, 0, 0, 202, 0, 0, 507, 0, 0, 397, 4084, 0, 0, 0, 3914, 0, 0, 1789, 2029, 0, 3822, 3655, 0, 2860, 3257, 0, 1016, 3918, 6, 422, 0, 0, 186, 0, 0, 326, 0, 20, 250, 4096, 3154, 4068, 38, 3616, 3841, 4022, 939, 119, 0, 185, 23, 394, 285, 0, 0, 0, 823, 3567, 27, 0, 0, 4096, 0, 0, 3766, 0, 243, 2, 4073, 0, 0, 0, 0, 1959, 777, 4096, 1, 0, 2422, 0, 4095, 0, 2787, 0, 469, 0, 4044, 2987, 2, 0, 4043, 0, 0, 0, 0, 0, 3798, 0, 0, 15, 2, 3472, 642, 0, 19, 0, 4, 3726] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 89, 0, 270, 4096, 1, 4004, 365, 0, 4, 37, 1, 1335, 1514, 0, 4095, 0, 93, 3, 3229, 0, 0, 134, 0, 0, 1446, 37, 4, 0, 4085, 0, 204, 82, 68, 3526, 0, 706, 0, 0, 0, 4, 0, 0, 2362, 0, 0, 4071, 4059, 25, 0, 0, 4096, 3840, 0, 4096, 11, 1, 0, 0, 953, 3892, 1885, 0, 0, 357, 0, 3421, 0, 0, 0, 1945, 0, 0, 0, 1, 160, 4088, 1, 3778, 4, 4096, 0, 0, 0, 23, 0, 3811, 4096, 0, 0, 3, 0, 241, 1, 3107, 0, 0, 2849, 3608, 3998, 4096, 7, 91, 70, 100, 3278, 4096, 0, 0, 4083, 3943, 0, 3408, 1775, 5, 3366, 0, 0, 0, 19, 1, 360, 0, 2, 0, 0, 4061] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 1353, 0, 0, 17, 0, 591, 0, 0, 0, 0, 0, 12, 3670, 0, 0, 3502, 953, 4096, 0, 344, 4, 0, 0, 0, 0, 0, 0, 1, 4095, 3754, 2, 485, 3401, 919, 293, 1636, 4063, 7, 0, 4096, 4096, 0, 0, 66, 30, 0, 0, 769, 54, 4088, 262, 3205, 3979, 0, 3987, 85, 0, 0, 489, 0, 433, 78, 4, 0, 100, 4094, 0, 0, 3640, 0, 4096, 4078, 4028, 0, 0, 1, 232, 1787, 1329, 0, 653, 0, 0, 2368, 0, 4, 0, 4096, 1753, 1, 0, 4051, 3937, 441, 1985, 0, 0, 3551, 0, 1, 3740, 153, 3503, 3821, 0, 3977, 4094, 0, 5, 4018, 0, 0, 0, 2614, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[167, 42, 3, 0, 0, 10, 0, 1413, 0, 0, 0, 4069, 3364, 3917, 0, 19, 3304, 0, 972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3322, 4069, 0, 1906, 0, 2, 0, 1, 31, 699, 0, 0, 0, 1, 4083, 0, 0, 1, 491, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3010, 0, 12, 1, 0, 0, 3944, 150, 48, 1587, 0, 11, 774, 1474, 3831, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2192, 3574, 1, 3672, 332, 204, 2, 0, 975, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 33, 17, 2, 0, 66, 4075, 2, 4, 0, 0, 186, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 2853, 0, 418, 2821, 841, 0, 0, 4091, 0, 0, 3883, 0, 4063, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4088, 0, 2638, 0, 0, 0, 3236, 3489, 0, 32, 0, 1697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 544, 2128, 0, 13, 0, 3962, 4096, 0, 0, 0, 0, 29, 1033, 24, 0, 4096, 3102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17280, 15036, 18937, 14178, 8551, 22862, 15186, 19042] avg=16384.0000 max=22862 max_rank=5 min=8551 min_rank=4 imbalance_ratio=1.395386 max_load_reduction=108210 max_load_reduction_pct=0.825577 moved_to_echo=65641 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1353, 0, 0, 17, 0, 591, 0, 0, 0, 0, 0, 12, 3670, 0, 0, 57, 40, 969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3502, 953, 4096, 0, 344, 4, 0, 0, 0, 0, 0, 0, 1, 4095, 3754, 2, 57, 296, 99, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 3401, 919, 293, 1636, 0, 7, 0, 4096, 4096, 0, 0, 9, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 54, 4088, 262, 3205, 3979, 0, 3987, 85, 0, 0, 489, 0, 433, 78, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 0, 3640, 0, 4096, 4078, 4028, 0, 0, 1, 232, 1787, 1329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 2368, 0, 4, 0, 4096, 1753, 1, 0, 4051, 3937, 441, 1985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3740, 153, 3503, 3821, 0, 3537, 4094, 0, 5, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2614, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3551, 3094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6709, 17643, 14972, 17137, 23288, 19190, 22872, 9261] avg=16384.0000 max=23288 max_rank=4 min=6709 min_rank=0 imbalance_ratio=1.421387 max_load_reduction=107784 max_load_reduction_pct=0.822327 moved_to_echo=74611 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[293, 4073, 0, 0, 0, 0, 2309, 21, 0, 0, 2336, 0, 505, 3277, 0, 0, 3138, 1558, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 3253, 3835, 2, 0, 0, 0, 0, 1924, 0, 0, 0, 3409, 0, 2240, 1553, 599, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 2651, 0, 701, 2, 0, 3988, 1, 905, 648, 114, 4096, 735, 1, 3843, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2539, 234, 4088, 0, 4068, 0, 7, 0, 968, 28, 0, 0, 4096, 7, 150, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 0, 2, 0, 3841, 0, 1, 3943, 0, 0, 0, 3642, 1102, 2421, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3481, 0, 0, 0, 0, 0, 50, 3079, 0, 0, 3, 4082, 1344, 3950, 3383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2363, 41, 0, 0, 82, 366, 3646, 0, 0, 4041, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4036, 15, 0, 0, 0, 0, 160, 0, 626, 1196, 1, 0, 0, 0, 2095, 3116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18486, 21323, 18021, 16197, 15421, 19372, 10960, 11292] avg=16384.0000 max=21323 max_rank=1 min=10960 min_rank=6 imbalance_ratio=1.301453 max_load_reduction=109749 max_load_reduction_pct=0.837318 moved_to_echo=57045 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 89, 0, 270, 4096, 1, 4004, 365, 0, 4, 37, 1, 1335, 1514, 0, 1914, 136, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 93, 3, 3229, 0, 0, 134, 0, 0, 1446, 37, 4, 0, 4085, 0, 195, 1540, 602, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 82, 68, 3526, 0, 0, 0, 0, 0, 4, 0, 0, 448, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2519, 25, 0, 0, 4096, 3840, 0, 4096, 11, 1, 0, 0, 953, 3892, 1885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 3421, 0, 0, 0, 1945, 0, 0, 0, 1, 160, 4088, 1, 3778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3494, 0, 0, 0, 23, 0, 3811, 4096, 0, 0, 3, 0, 241, 1, 3107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3608, 3998, 4096, 7, 91, 70, 100, 2918, 4096, 0, 0, 4083, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3408, 1775, 5, 3366, 0, 0, 0, 19, 1, 360, 0, 2, 0, 0, 4061, 2849, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13935, 15823, 8403, 21318, 13420, 14780, 27010, 16383] avg=16384.0000 max=27010 max_rank=6 min=8403 min_rank=2 imbalance_ratio=1.648560 max_load_reduction=104062 max_load_reduction_pct=0.793930 moved_to_echo=71593 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 29, 188, 4070, 37, 78, 2456, 0, 0, 254, 0, 0, 3478, 0, 263, 3289, 1558, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 6, 0, 938, 3826, 4, 3695, 2313, 0, 0, 0, 0, 0, 208, 8, 0, 2240, 1547, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1126, 0, 0, 0, 3606, 0, 1921, 1911, 0, 3439, 770, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2530, 0, 48, 0, 526, 4055, 0, 111, 11, 0, 544, 0, 5, 0, 25, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2779, 286, 2071, 3, 208, 4096, 0, 0, 3187, 52, 10, 3047, 526, 0, 4094, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3493, 0, 0, 0, 0, 4070, 0, 1092, 0, 0, 0, 0, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 60, 0, 0, 13, 0, 3876, 0, 3310, 1592, 92, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3219, 0, 2365, 99, 1741, 0, 4051, 0, 0, 216, 10, 0, 8, 0, 3997, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20753, 19381, 13527, 11945, 24445, 9209, 13034, 18778] avg=16384.0000 max=24445 max_rank=4 min=9209 min_rank=5 imbalance_ratio=1.492004 max_load_reduction=106627 max_load_reduction_pct=0.813499 moved_to_echo=65466 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1814, 0, 2192, 4077, 788, 4082, 0, 0, 3788, 0, 55, 0, 0, 590, 0, 1, 1778, 1562, 934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2365, 0, 0, 0, 3, 301, 597, 4093, 1, 0, 0, 0, 0, 0, 4096, 2, 2247, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 32, 0, 0, 2814, 0, 4096, 2779, 3616, 1, 417, 3173, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2415, 280, 3994, 0, 0, 4, 0, 0, 0, 29, 0, 1, 4088, 1918, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 4067, 0, 0, 4093, 1427, 697, 2, 2006, 0, 50, 4091, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 100, 0, 0, 31, 3704, 0, 0, 0, 0, 0, 3535, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 489, 0, 0, 0, 0, 0, 4095, 0, 1435, 1788, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 6, 230, 3687, 11, 4093, 4085, 1, 337, 3975, 1350, 677, 1729, 2983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21661, 15181, 21007, 13989, 20833, 7379, 7827, 23195] avg=16384.0000 max=23195 max_rank=7 min=7379 min_rank=5 imbalance_ratio=1.415710 max_load_reduction=107877 max_load_reduction_pct=0.823036 moved_to_echo=59234 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 3448, 0, 0, 3232, 3722, 18, 0, 1480, 719, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1133, 672, 0, 510, 0, 2223, 692, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1122, 599, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3521, 0, 1389, 4095, 0, 1102, 169, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 0, 0, 0, 76, 1052, 1, 954, 741, 406, 3734, 0, 4049, 3184, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 38, 0, 0, 25, 46, 0, 3818, 0, 0, 3365, 3287, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 3480, 0, 0, 4081, 944, 3650, 0, 3672, 0, 17, 373, 0, 16, 0, 3559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 0, 0, 2751, 4093, 0, 11, 0, 22, 4044, 3646, 0, 953, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 4096, 0, 2331, 0, 0, 2527, 4051, 1, 534, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16718, 11497, 14373, 20128, 14629, 19870, 16199, 17658] avg=16384.0000 max=20128 max_rank=3 min=11497 min_rank=1 imbalance_ratio=1.228516 max_load_reduction=110944 max_load_reduction_pct=0.846436 moved_to_echo=68356 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4050, 0, 0, 3, 72, 58, 61, 982, 0, 0, 37, 1559, 1584, 2453, 72, 0, 2639, 1534, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 1682, 368, 595, 0, 0, 202, 0, 0, 507, 0, 0, 397, 4084, 0, 2206, 2, 602, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 0, 0, 0, 2029, 0, 3822, 3655, 0, 2860, 618, 0, 1016, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 422, 0, 0, 186, 0, 0, 326, 0, 20, 250, 4096, 3154, 4068, 38, 3616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 282, 939, 119, 0, 185, 23, 394, 285, 0, 0, 0, 823, 3567, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3494, 0, 0, 3766, 0, 243, 2, 4073, 0, 0, 0, 0, 1959, 777, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 2787, 0, 469, 0, 3600, 2987, 2, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 0, 15, 2, 3472, 642, 0, 19, 0, 4, 3726, 2422, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15530, 15188, 21832, 16180, 10485, 18410, 17984, 15463] avg=16384.0000 max=21832 max_rank=2 min=10485 min_rank=4 imbalance_ratio=1.332520 max_load_reduction=109240 max_load_reduction_pct=0.833435 moved_to_echo=62342 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 20864, 128, 640, 13056, 1920, 8064, 7680, 24320, 1792, 128, 4352, 8192, 15104, 5760, 19968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12416, 1024, 12288, 384, 14464, 16512, 128, 13184, 2176, 768, 4608, 4608, 17408, 16768, 3456, 11904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[640, 128, 0, 14592, 14720, 11008, 3072, 3712, 4480, 11648, 20864, 18688, 2688, 5632, 16128, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[6400, 9600, 2432, 4352, 9344, 8320, 3072, 8960, 7680, 0, 2688, 8960, 9216, 18816, 1664, 1792, 16896, 6400, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[26624, 5760, 5888, 7040, 11904, 2560, 4992, 6784, 128, 768, 3968, 128, 128, 4736, 27648, 128, 9216, 7552, 3712, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 4096, 13824, 128, 13696, 128, 8320, 7808, 4352, 10112, 9856, 1536, 1024, 5120, 1408, 8576, 24960, 17408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[6784, 1280, 11264, 3712, 256, 15872, 1536, 7296, 15616, 9984, 128, 3200, 12672, 16640, 16512, 9472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[4096, 6144, 6144, 4608, 1664, 0, 19968, 256, 16384, 18816, 3840, 11520, 3968, 5504, 12800, 16256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 499, 4096, 0, 0, 2890, 187, 2759, 0, 0, 3900, 4094, 0, 1, 1, 162, 3578, 0, 0, 0, 220, 20, 4096, 0, 4096, 0, 0, 2, 4096, 0, 1703, 4096, 0, 0, 0, 314, 0, 0, 2, 4092, 4086, 0, 4096, 2186, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 79, 1932, 0, 0, 1, 4096, 687, 3994, 4093, 167, 0, 1, 1614, 18, 4096, 4080, 65, 0, 1, 0, 2892, 0, 0, 70, 26, 0, 0, 0, 611, 0, 0, 0, 0, 3494, 0, 307, 30, 0, 0, 3968, 4096, 4032, 4, 2713, 5, 0, 68, 335, 4060, 1131, 139, 0, 3991, 3160, 0, 0, 0, 1116, 3665, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 1662, 1110] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[44, 0, 1, 1, 2372, 2, 0, 33, 930, 345, 0, 0, 0, 43, 4091, 0, 8, 1402, 0, 0, 36, 1795, 0, 0, 0, 1146, 0, 3189, 1310, 24, 12, 157, 39, 440, 0, 0, 3, 2, 0, 0, 0, 2239, 0, 4096, 192, 4096, 4096, 0, 9, 8, 3993, 111, 0, 4096, 1114, 0, 0, 20, 198, 0, 4071, 0, 0, 3755, 92, 0, 4094, 2627, 3411, 4067, 3357, 0, 3851, 229, 0, 0, 0, 0, 1773, 160, 0, 0, 0, 0, 0, 0, 0, 0, 3770, 0, 0, 3930, 1, 0, 22, 2, 4096, 3566, 0, 4094, 154, 403, 3, 87, 720, 0, 4060, 0, 4096, 0, 3766, 4082, 0, 73, 4096, 4096, 4090, 23, 1037, 259, 0, 0, 3409, 0, 0, 0, 3930, 27] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3278, 3036, 0, 0, 2374, 0, 10, 225, 655, 2376, 3755, 0, 0, 255, 0, 0, 4028, 0, 0, 0, 17, 3712, 48, 4050, 0, 4074, 0, 1212, 0, 3796, 0, 1029, 0, 0, 1235, 11, 1794, 989, 5, 686, 4043, 1, 3454, 312, 566, 2939, 3, 61, 9, 4096, 3942, 1, 83, 3289, 7, 3, 1, 2643, 0, 261, 0, 4006, 0, 4095, 12, 4082, 258, 0, 3604, 3923, 0, 0, 0, 2, 20, 0, 0, 0, 58, 3424, 0, 0, 0, 0, 1195, 5, 303, 3873, 347, 0, 91, 0, 4096, 0, 0, 2496, 236, 4096, 3531, 173, 0, 0, 0, 0, 13, 0, 72, 891, 0, 442, 0, 2952, 2710, 583, 243, 0, 634, 2047, 30, 0, 4043, 0, 4096, 26] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 3714, 3750, 3937, 0, 0, 4091, 136, 343, 0, 1, 0, 0, 0, 2581, 175, 2, 0, 4027, 0, 0, 0, 2, 662, 3968, 0, 0, 4096, 0, 276, 3965, 1774, 32, 0, 84, 0, 4096, 22, 313, 0, 0, 0, 3849, 4090, 0, 305, 0, 4068, 0, 0, 0, 712, 1, 0, 0, 352, 0, 0, 4096, 95, 2389, 267, 0, 1, 0, 3434, 0, 0, 0, 4086, 3, 62, 0, 4096, 0, 2851, 0, 3933, 0, 0, 4096, 0, 0, 0, 54, 6, 0, 0, 2087, 0, 644, 0, 0, 27, 3637, 4095, 3524, 0, 4000, 0, 0, 3, 0, 884, 0, 3991, 160, 3033, 4061, 0, 0, 2954, 4096, 4079, 0, 0, 0, 0, 614, 0, 0, 4025, 258, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 4036, 3932, 3891, 1695, 0, 0, 10, 0, 35, 0, 0, 4096, 2, 21, 0, 63, 509, 4095, 0, 0, 0, 785, 196, 3488, 0, 0, 730, 348, 0, 4095, 0, 0, 0, 2, 0, 68, 5, 231, 3, 1868, 2883, 4077, 51, 6, 44, 3712, 810, 0, 923, 4008, 46, 3, 0, 0, 1501, 0, 2798, 2249, 563, 3839, 3952, 398, 11, 0, 2589, 0, 0, 0, 3275, 3184, 669, 338, 4071, 0, 3051, 4096, 1314, 1516, 697, 962, 19, 0, 0, 878, 21, 250, 0, 2740, 600, 3822, 122, 63, 0, 0, 3537, 0, 2394, 10, 0, 0, 0, 1, 162, 0, 3931, 126, 2913, 3225, 0, 52, 3904, 209, 0, 2732, 0, 0, 0, 1, 0, 0, 4056, 0, 3460, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 3869, 1204, 22, 88, 4086, 0, 4095, 0, 0, 0, 350, 160, 0, 0, 0, 128, 0, 0, 4, 0, 3563, 127, 0, 4007, 0, 818, 0, 2063, 0, 0, 1641, 0, 0, 2, 13, 0, 0, 4064, 4096, 159, 38, 2929, 4, 155, 312, 4090, 1, 0, 0, 4084, 3998, 0, 0, 23, 21, 0, 0, 0, 0, 99, 0, 0, 4095, 4054, 3, 0, 410, 0, 4085, 3918, 4091, 0, 0, 3018, 3775, 0, 3565, 2, 0, 0, 0, 0, 0, 0, 3087, 0, 0, 3766, 0, 222, 4007, 0, 0, 716, 1, 0, 1280, 3736, 0, 0, 0, 0, 0, 1321, 4090, 0, 0, 2870, 494, 0, 1, 4078, 3336, 3987, 1, 4093, 275, 4, 0, 214, 4060, 0, 4092, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[150, 0, 0, 3607, 4067, 0, 0, 0, 4092, 0, 3826, 257, 1325, 318, 3720, 0, 0, 0, 0, 29, 563, 0, 272, 0, 3120, 482, 535, 2495, 1723, 89, 0, 4038, 0, 4067, 0, 107, 3, 4092, 0, 0, 0, 415, 4063, 961, 0, 4086, 0, 139, 19, 2414, 7, 4091, 714, 0, 286, 0, 8, 2, 98, 0, 3487, 8, 4096, 4096, 0, 1, 3520, 4081, 0, 4094, 465, 0, 4087, 3902, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3663, 2, 2, 3174, 0, 0, 4092, 0, 12, 3035, 0, 0, 3912, 9, 1, 1849, 0, 0, 2322, 207, 4096, 0, 0, 3998, 0, 0, 5, 4096, 43, 5, 0, 1635, 0, 0, 0, 2726, 0, 5, 63, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1642, 4, 25, 2, 0, 4096, 0, 0, 0, 0, 82, 0, 4096, 0, 3114, 0, 0, 4096, 0, 96, 0, 0, 0, 285, 0, 2848, 51, 148, 3888, 0, 0, 3313, 0, 4096, 18, 2970, 0, 3280, 0, 0, 0, 0, 4080, 3932, 6, 3782, 0, 0, 0, 4093, 0, 0, 0, 534, 749, 1, 959, 245, 0, 0, 0, 0, 0, 370, 0, 562, 1649, 3454, 0, 1650, 0, 0, 4045, 4063, 0, 71, 0, 177, 4093, 0, 0, 8, 0, 0, 0, 1206, 0, 0, 2171, 0, 0, 3827, 147, 0, 15, 2841, 2059, 2043, 3818, 1095, 0, 4096, 1, 2, 0, 4051, 4094, 0, 4, 0, 23, 48, 0, 661, 3929, 0, 4093, 3857, 4096, 609, 0, 0, 0, 674, 732, 0, 4029, 178] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 499, 4096, 0, 0, 2890, 187, 2759, 0, 0, 3900, 4094, 0, 987, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 162, 3578, 0, 0, 0, 220, 20, 4096, 0, 4096, 0, 0, 2, 4096, 119, 506, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1703, 4096, 0, 0, 0, 314, 0, 0, 2, 4092, 4086, 0, 3025, 2186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 79, 1932, 0, 0, 1, 4096, 1449, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, 3994, 4093, 167, 0, 1, 1614, 18, 0, 2160, 65, 0, 1, 0, 2892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 26, 0, 0, 0, 611, 0, 0, 0, 0, 3494, 0, 307, 30, 0, 4096, 2654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 4096, 4032, 4, 2713, 5, 0, 68, 335, 4060, 1131, 20, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1116, 2678, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 1662, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19591, 17368, 19504, 12549, 15692, 11288, 24423, 10657] avg=16384.0000 max=24423 max_rank=6 min=10657 min_rank=7 imbalance_ratio=1.490662 max_load_reduction=106649 max_load_reduction_pct=0.813667 moved_to_echo=62060 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[44, 0, 1, 1, 2372, 2, 0, 33, 930, 345, 0, 0, 0, 43, 4091, 0, 1101, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1402, 0, 0, 36, 1795, 0, 0, 0, 1146, 0, 3189, 1310, 24, 12, 157, 3526, 653, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 440, 0, 0, 3, 2, 0, 0, 0, 2239, 0, 4096, 192, 3025, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 3993, 111, 0, 4096, 1114, 0, 0, 20, 198, 0, 4071, 0, 0, 3755, 81, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 4094, 2627, 3411, 4067, 3357, 0, 0, 122, 0, 0, 0, 0, 1773, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3770, 0, 0, 3930, 1, 0, 22, 2, 3851, 3429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3566, 0, 4094, 154, 403, 3, 87, 720, 0, 4060, 0, 570, 0, 3766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 4096, 4096, 2989, 23, 1037, 259, 0, 0, 3409, 0, 0, 0, 3930, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9138, 13284, 14132, 18352, 19703, 15005, 21519, 19939] avg=16384.0000 max=21519 max_rank=6 min=9138 min_rank=0 imbalance_ratio=1.313416 max_load_reduction=109553 max_load_reduction_pct=0.835823 moved_to_echo=76166 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3278, 3036, 0, 0, 2374, 0, 10, 225, 655, 2376, 3755, 0, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 4028, 0, 0, 0, 17, 3712, 48, 4050, 0, 4074, 0, 1212, 11, 142, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 0, 1029, 0, 0, 1235, 11, 1794, 989, 5, 686, 4043, 1, 3454, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 2939, 3, 61, 9, 4096, 3942, 1, 83, 3289, 7, 3, 1, 2643, 0, 261, 1393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 4095, 12, 4082, 258, 0, 0, 2077, 0, 0, 0, 2, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 3424, 0, 0, 0, 0, 1195, 5, 303, 3873, 347, 0, 91, 0, 3604, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2496, 236, 4096, 3531, 173, 0, 0, 0, 0, 2, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 442, 0, 2952, 1980, 583, 243, 0, 634, 2047, 30, 0, 4043, 0, 4096, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16439, 18002, 17355, 19297, 14552, 13649, 14702, 17076] avg=16384.0000 max=19297 max_rank=3 min=13649 min_rank=5 imbalance_ratio=1.177795 max_load_reduction=111775 max_load_reduction_pct=0.852776 moved_to_echo=59979 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 3714, 3750, 3937, 0, 0, 4091, 136, 343, 0, 1, 0, 0, 1098, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2581, 175, 2, 0, 4027, 0, 0, 0, 2, 662, 3968, 0, 0, 4096, 0, 3436, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 3965, 1774, 32, 0, 84, 0, 4096, 22, 313, 0, 0, 0, 2842, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 0, 4068, 0, 0, 0, 712, 1, 0, 0, 352, 0, 0, 4096, 95, 2389, 1, 842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 0, 1, 0, 3434, 0, 0, 0, 0, 2, 62, 0, 4096, 0, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 0, 0, 4096, 0, 0, 0, 54, 6, 0, 0, 2087, 0, 644, 0, 0, 4086, 3412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 3637, 4095, 3524, 0, 4000, 0, 0, 3, 0, 884, 0, 555, 160, 3033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2954, 4096, 2981, 0, 0, 0, 0, 614, 0, 0, 4025, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17242, 19598, 17494, 12861, 10713, 18318, 19918, 14928] avg=16384.0000 max=19918 max_rank=6 min=10713 min_rank=4 imbalance_ratio=1.215698 max_load_reduction=111154 max_load_reduction_pct=0.848038 moved_to_echo=63877 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1642, 4, 25, 2, 0, 4096, 0, 0, 0, 0, 82, 0, 4096, 0, 3114, 0, 1102, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 96, 0, 0, 0, 285, 0, 2848, 51, 148, 3888, 0, 0, 3313, 3, 8, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 2970, 0, 3280, 0, 0, 0, 0, 4080, 3932, 6, 2792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 534, 749, 1, 959, 245, 0, 0, 0, 0, 0, 370, 1443, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 1649, 3454, 0, 1650, 0, 0, 0, 2151, 0, 71, 0, 177, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1206, 0, 0, 2171, 0, 0, 3827, 147, 0, 15, 2841, 4045, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2059, 2043, 3818, 1095, 0, 4096, 1, 2, 0, 4051, 4094, 0, 1, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 661, 3929, 0, 2991, 3857, 4096, 609, 0, 0, 0, 674, 732, 0, 4029, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14325, 15205, 21174, 9222, 13807, 14300, 21283, 21756] avg=16384.0000 max=21756 max_rank=7 min=9222 min_rank=3 imbalance_ratio=1.327881 max_load_reduction=109316 max_load_reduction_pct=0.834015 moved_to_echo=71146 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 3869, 1204, 22, 88, 4086, 0, 4095, 0, 0, 0, 350, 160, 898, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 4, 0, 3563, 127, 0, 4007, 0, 818, 0, 2063, 3521, 460, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1641, 0, 0, 2, 13, 0, 0, 4064, 4096, 159, 38, 2163, 4, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 4090, 1, 0, 0, 4084, 3998, 0, 0, 23, 21, 0, 0, 0, 0, 99, 1391, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4054, 3, 0, 410, 0, 0, 2075, 4091, 0, 0, 3018, 3775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3565, 2, 0, 0, 0, 0, 0, 0, 3087, 0, 0, 3766, 0, 222, 4007, 0, 4085, 2410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 716, 1, 0, 1280, 3736, 0, 0, 0, 0, 0, 1321, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 0, 1, 4078, 2438, 3987, 1, 4093, 275, 4, 0, 214, 4060, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14909, 15143, 12335, 14660, 21521, 21144, 7623, 23737] avg=16384.0000 max=23737 max_rank=7 min=7623 min_rank=6 imbalance_ratio=1.448792 max_load_reduction=107335 max_load_reduction_pct=0.818901 moved_to_echo=74025 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 4036, 3932, 3891, 1695, 0, 0, 10, 0, 35, 0, 0, 4096, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 21, 0, 63, 509, 4095, 0, 0, 0, 785, 196, 3488, 0, 0, 730, 348, 3384, 515, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 68, 5, 231, 3, 1868, 2883, 3010, 51, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 3712, 810, 0, 923, 4008, 46, 3, 0, 0, 1501, 0, 2798, 2249, 563, 3839, 1130, 892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 398, 11, 0, 2589, 0, 0, 0, 0, 1687, 669, 338, 4071, 0, 3051, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1314, 1516, 697, 962, 19, 0, 0, 878, 21, 250, 0, 2740, 600, 3822, 122, 63, 3275, 2710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3537, 0, 2394, 10, 0, 0, 0, 1, 162, 0, 547, 126, 2913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 3904, 209, 0, 2732, 0, 0, 0, 1, 0, 0, 4056, 0, 3460, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17872, 14503, 12222, 22518, 20862, 18989, 9690, 14416] avg=16384.0000 max=22518 max_rank=3 min=9690 min_rank=6 imbalance_ratio=1.374390 max_load_reduction=108554 max_load_reduction_pct=0.828201 moved_to_echo=63957 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[150, 0, 0, 3607, 4067, 0, 0, 0, 4092, 0, 3826, 257, 1325, 318, 3720, 0, 15, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 563, 0, 272, 0, 3120, 482, 535, 2495, 1723, 89, 0, 4038, 3529, 640, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 107, 3, 4092, 0, 0, 0, 415, 4063, 961, 0, 3009, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2414, 7, 4091, 714, 0, 286, 0, 8, 2, 98, 0, 3487, 8, 4096, 4096, 1389, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3520, 4081, 0, 4094, 465, 0, 0, 2059, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3663, 2, 2, 3174, 0, 0, 4092, 0, 4087, 3358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3035, 0, 0, 3912, 9, 1, 1849, 0, 0, 2322, 207, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 28, 5, 0, 1635, 0, 0, 0, 2726, 0, 5, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21556, 17969, 16856, 21613, 14222, 18379, 11914, 8563] avg=16384.0000 max=21613 max_rank=3 min=8563 min_rank=7 imbalance_ratio=1.319153 max_load_reduction=109459 max_load_reduction_pct=0.835106 moved_to_echo=53078 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 8448, 384, 3968, 5248, 9984, 384, 512, 6784, 13312, 1536, 25472, 7040, 5120, 4864, 15232, 17536, 3584, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[10368, 17024, 15616, 15360, 8064, 19072, 3584, 2176, 896, 4480, 15616, 2688, 2944, 384, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[512, 1280, 14976, 20736, 16128, 11264, 5504, 6656, 1024, 2688, 3456, 3712, 21120, 384, 21376, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[1920, 128, 128, 3712, 21888, 20224, 7936, 1920, 14464, 4736, 11008, 896, 6144, 6656, 19200, 4352, 6016, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 22272, 7552, 4224, 128, 7552, 1664, 4224, 1920, 8320, 16384, 15872, 7168, 19968, 13952, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1280, 17280, 8960, 8448, 1664, 16896, 10880, 128, 1152, 3584, 2304, 2048, 10368, 9088, 4864, 18944, 8320, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[8832, 1664, 896, 8576, 128, 1280, 640, 1024, 13952, 384, 384, 27008, 1152, 5120, 8448, 2944, 31232, 18816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[5120, 9088, 17536, 18560, 9472, 13952, 6144, 128, 0, 12416, 4992, 512, 8192, 3200, 18560, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4071, 0, 3, 97, 2757, 29, 3977, 1, 102, 0, 8, 1958, 4084, 6, 10, 0, 0, 0, 4066, 22, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 0, 0, 3510, 4015, 3808, 4070, 0, 0, 0, 1, 169, 3893, 0, 986, 0, 0, 716, 0, 0, 2135, 0, 31, 0, 4094, 4093, 1722, 0, 0, 40, 2492, 5, 780, 0, 0, 2975, 1232, 56, 0, 3598, 1557, 0, 0, 1512, 0, 0, 1, 4079, 0, 0, 4084, 0, 1, 4093, 0, 16, 3326, 513, 442, 418, 3906, 37, 6, 4096, 2417, 10, 14, 674, 0, 4086, 4041, 3150, 0, 4096, 19, 0, 3, 0, 0, 0, 0, 29, 0, 0, 728, 156, 0, 26, 4044, 28, 4095, 0, 0, 4094, 0, 0, 0, 1195, 0, 364] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1464, 0, 4076, 50, 4095, 4096, 0, 0, 103, 12, 0, 0, 3381, 0, 0, 405, 0, 0, 0, 0, 4083, 589, 0, 2, 0, 0, 961, 0, 3750, 72, 2748, 2, 0, 4023, 3867, 1176, 1, 0, 0, 0, 133, 1528, 0, 0, 336, 131, 3817, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 228, 712, 2975, 4096, 0, 164, 0, 2430, 255, 3792, 125, 0, 15, 0, 0, 1, 3929, 0, 0, 0, 0, 0, 0, 2152, 3658, 3904, 0, 0, 0, 2600, 0, 720, 4096, 0, 1490, 0, 4096, 5, 582, 2253, 3834, 4021, 4059, 2115, 3503, 221, 2, 0, 0, 0, 0, 3307, 71, 1, 75, 0, 3728, 620, 0, 516, 2994, 4096, 3968, 149, 0, 208, 0, 2, 0, 24, 69, 298] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4038, 0, 4096, 29, 614, 0, 315, 0, 0, 0, 526, 3137, 4096, 4077, 0, 0, 0, 0, 48, 20, 210, 1, 0, 0, 0, 0, 4028, 0, 4096, 0, 211, 4055, 4078, 3902, 0, 46, 0, 24, 39, 47, 18, 3773, 29, 0, 1, 20, 8, 4, 3985, 2077, 0, 0, 154, 4089, 0, 15, 0, 0, 4080, 0, 0, 0, 8, 2080, 0, 4096, 91, 0, 0, 1, 16, 0, 0, 28, 4015, 0, 0, 0, 0, 1085, 3301, 3985, 2, 4055, 3254, 2879, 0, 0, 0, 4, 0, 92, 3852, 0, 73, 1958, 0, 3554, 4093, 0, 4096, 4096, 217, 0, 0, 2, 6, 395, 0, 1, 18, 0, 4092, 215, 3516, 3658, 4036, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 84, 48] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[419, 0, 1434, 1494, 57, 0, 1, 0, 133, 0, 138, 0, 4096, 2821, 1523, 0, 0, 929, 3146, 0, 4009, 197, 0, 0, 0, 0, 3917, 4062, 2025, 0, 5, 4083, 1371, 126, 4045, 85, 1093, 563, 0, 0, 0, 111, 290, 298, 694, 0, 4028, 0, 3782, 3840, 0, 0, 0, 109, 73, 0, 0, 0, 0, 25, 0, 26, 432, 3070, 27, 3670, 785, 301, 0, 49, 0, 0, 0, 2684, 0, 172, 17, 0, 0, 4096, 0, 0, 1, 0, 4095, 3949, 0, 0, 3596, 0, 2670, 2380, 3313, 0, 3537, 3753, 267, 4061, 2937, 0, 1900, 4054, 0, 3747, 0, 0, 8, 1, 0, 2, 152, 194, 0, 0, 0, 712, 4083, 3944, 4013, 6, 5, 1182, 46, 15, 2098, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 3789, 0, 4096, 4, 3921, 220, 0, 138, 739, 358, 4092, 1994, 0, 53, 1, 0, 0, 0, 4096, 0, 4063, 77, 84, 0, 4096, 0, 4022, 0, 1933, 0, 1009, 4072, 3896, 1, 0, 13, 0, 0, 3853, 4096, 0, 3, 0, 0, 0, 0, 2756, 0, 1391, 0, 61, 4096, 0, 8, 0, 31, 3089, 0, 0, 24, 0, 2603, 0, 4054, 0, 0, 0, 1216, 3711, 668, 2, 0, 4096, 0, 0, 355, 0, 0, 0, 1280, 271, 0, 0, 31, 0, 0, 4004, 978, 0, 4057, 3321, 161, 12, 4078, 0, 4063, 4083, 3017, 0, 56, 273, 0, 7, 0, 2282, 0, 0, 2287, 0, 0, 4095, 0, 3177, 0, 1849, 185, 4096, 0, 0, 0, 1, 0, 0, 527, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[348, 0, 3678, 3296, 0, 0, 4096, 0, 69, 0, 0, 0, 4096, 657, 2378, 0, 0, 58, 4089, 1094, 3061, 0, 168, 0, 0, 1, 0, 0, 3473, 0, 0, 0, 4073, 4096, 1835, 0, 0, 554, 0, 0, 72, 4096, 75, 294, 15, 43, 1255, 0, 0, 0, 0, 0, 0, 0, 3835, 157, 113, 115, 0, 0, 0, 0, 0, 4007, 1, 3489, 0, 0, 4, 1, 4093, 0, 91, 4093, 4094, 0, 0, 14, 0, 0, 4096, 4096, 0, 0, 4083, 0, 0, 621, 0, 0, 1, 3990, 4040, 1, 3426, 0, 2, 3718, 4096, 4084, 0, 3933, 0, 3, 0, 0, 0, 0, 0, 0, 2, 0, 0, 528, 7, 20, 3800, 3852, 3948, 0, 0, 4096, 0, 2548, 18, 0, 267, 719] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4052, 0, 0, 0, 0, 3580, 4090, 0, 0, 0, 249, 4093, 558, 0, 1488, 0, 4074, 398, 4052, 1691, 3, 4095, 635, 3087, 0, 0, 0, 0, 0, 422, 1078, 3980, 1713, 2122, 0, 32, 0, 1, 0, 1260, 11, 131, 0, 0, 4096, 0, 0, 0, 0, 1059, 4090, 1306, 2003, 15, 209, 0, 0, 4014, 0, 2, 0, 1398, 3884, 0, 314, 0, 0, 5, 22, 0, 4094, 3764, 30, 4088, 3962, 784, 25, 0, 1832, 4094, 0, 414, 2879, 0, 249, 0, 0, 5, 4095, 18, 0, 20, 14, 0, 1065, 0, 2813, 3945, 0, 1, 1, 0, 0, 0, 0, 0, 0, 68, 1918, 5, 0, 2849, 0, 3809, 38, 3600, 2478, 4077, 1, 1, 0, 173, 397, 60, 0, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 243, 1093, 0, 0, 0, 0, 28, 0, 0, 4050, 4094, 0, 0, 0, 0, 4067, 8, 3930, 647, 1631, 0, 0, 4037, 3344, 0, 4085, 0, 13, 0, 58, 3271, 3842, 0, 0, 0, 0, 0, 4093, 4095, 1904, 0, 65, 996, 4096, 0, 4071, 4077, 0, 0, 4094, 0, 0, 0, 81, 3, 2862, 0, 0, 10, 0, 56, 1, 659, 0, 0, 45, 0, 4096, 0, 712, 0, 27, 0, 0, 0, 3087, 3947, 2512, 806, 2, 0, 0, 4079, 0, 0, 2, 1897, 0, 4096, 4089, 0, 0, 4095, 0, 0, 3388, 0, 0, 4094, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 1332, 230, 4096, 0, 164, 2326, 18, 0, 88, 4042, 0, 4092, 0, 15, 0, 4086] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4038, 0, 4096, 29, 614, 0, 315, 0, 0, 0, 526, 3137, 3789, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 20, 210, 1, 0, 0, 0, 0, 4028, 0, 4096, 0, 211, 4055, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 3902, 0, 46, 0, 24, 39, 47, 18, 3773, 29, 0, 1, 20, 8, 4, 389, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 2077, 0, 0, 154, 4089, 0, 15, 0, 0, 4080, 0, 0, 0, 8, 2080, 57, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 91, 0, 0, 1, 16, 0, 0, 28, 4015, 0, 0, 0, 0, 1085, 3381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3301, 3985, 2, 4055, 3254, 2879, 0, 0, 0, 4, 0, 20, 0, 0, 73, 1958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3554, 4093, 0, 4096, 4096, 217, 0, 0, 2, 6, 395, 0, 1, 18, 0, 1337, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 215, 3516, 3658, 4036, 0, 2358, 0, 0, 0, 0, 0, 0, 0, 84, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20621, 12734, 12620, 17016, 12713, 19531, 17830, 18007] avg=16384.0000 max=20621 max_rank=0 min=12620 min_rank=2 imbalance_ratio=1.258606 max_load_reduction=110451 max_load_reduction_pct=0.842674 moved_to_echo=68081 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4071, 0, 3, 97, 2757, 29, 3977, 1, 102, 0, 8, 1958, 3779, 6, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 22, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 0, 0, 3510, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 3808, 4070, 0, 0, 0, 1, 169, 3893, 0, 986, 0, 0, 716, 0, 0, 390, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2135, 0, 31, 0, 4094, 4093, 1722, 0, 0, 40, 2492, 5, 780, 0, 0, 2975, 2550, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1232, 56, 0, 3598, 1557, 0, 0, 1512, 0, 0, 1, 4079, 0, 0, 4084, 0, 2121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 16, 3326, 513, 442, 418, 3906, 37, 6, 839, 0, 10, 14, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4041, 3150, 0, 4096, 19, 0, 3, 0, 0, 0, 0, 29, 0, 0, 1340, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 728, 156, 0, 26, 4044, 28, 2365, 0, 0, 4094, 0, 0, 0, 1195, 0, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16798, 11766, 18289, 21213, 18240, 14295, 17471, 13000] avg=16384.0000 max=21213 max_rank=3 min=11766 min_rank=1 imbalance_ratio=1.294739 max_load_reduction=109859 max_load_reduction_pct=0.838158 moved_to_echo=63006 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 243, 1093, 0, 0, 0, 0, 28, 0, 0, 3741, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 8, 3930, 647, 1631, 0, 0, 4037, 3344, 0, 4085, 0, 13, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 3271, 3842, 0, 0, 0, 0, 0, 4093, 4095, 1904, 0, 65, 996, 4096, 0, 6, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 4077, 0, 0, 4094, 0, 0, 0, 81, 3, 2862, 0, 0, 10, 0, 56, 2552, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 659, 0, 0, 45, 0, 4096, 0, 712, 0, 27, 0, 0, 0, 3087, 3947, 3589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2512, 806, 2, 0, 0, 4079, 0, 0, 2, 1897, 0, 834, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3388, 0, 0, 4094, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 9, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1332, 230, 4096, 0, 164, 2326, 3, 0, 88, 4042, 0, 4092, 0, 15, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9200, 21829, 22668, 18306, 16163, 14227, 8205, 20474] avg=16384.0000 max=22668 max_rank=2 min=8205 min_rank=6 imbalance_ratio=1.383545 max_load_reduction=108404 max_load_reduction_pct=0.827057 moved_to_echo=59069 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1464, 0, 4076, 50, 4095, 4096, 0, 0, 103, 12, 0, 0, 3129, 0, 0, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 589, 0, 2, 0, 0, 961, 0, 3750, 72, 2748, 2, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 3867, 1176, 1, 0, 0, 0, 133, 1528, 0, 0, 336, 131, 3817, 0, 378, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 228, 712, 2975, 4096, 0, 164, 0, 2430, 0, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 3792, 125, 0, 15, 0, 0, 1, 3929, 0, 0, 0, 0, 0, 0, 2152, 3596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3658, 3904, 0, 0, 0, 2600, 0, 720, 4096, 0, 1490, 0, 0, 5, 582, 2253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3834, 4021, 4059, 2115, 3503, 221, 2, 0, 0, 0, 0, 3307, 71, 1, 75, 0, 1299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3728, 620, 0, 516, 2994, 4096, 2291, 149, 0, 208, 0, 2, 0, 24, 69, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17430, 12260, 15589, 15117, 13865, 19308, 22508, 14995] avg=16384.0000 max=22508 max_rank=6 min=12260 min_rank=1 imbalance_ratio=1.373779 max_load_reduction=108564 max_load_reduction_pct=0.828278 moved_to_echo=70676 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3789, 0, 4096, 4, 3921, 220, 0, 138, 739, 358, 3787, 1994, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4063, 77, 84, 0, 4096, 0, 4022, 0, 1933, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 4072, 3896, 1, 0, 13, 0, 0, 3853, 4096, 0, 3, 0, 0, 0, 0, 391, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2756, 0, 1391, 0, 61, 4096, 0, 8, 0, 31, 3089, 0, 0, 24, 0, 2603, 2526, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 1216, 3711, 668, 2, 0, 4096, 0, 0, 355, 0, 0, 2916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1280, 271, 0, 0, 31, 0, 0, 4004, 978, 0, 831, 0, 161, 12, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4083, 3017, 0, 56, 273, 0, 7, 0, 2282, 0, 0, 2287, 0, 0, 1341, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3177, 0, 1849, 185, 2364, 0, 0, 0, 1, 0, 0, 527, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19099, 18436, 17575, 16990, 17018, 11646, 18109, 12199] avg=16384.0000 max=19099 max_rank=0 min=11646 min_rank=5 imbalance_ratio=1.165710 max_load_reduction=111973 max_load_reduction_pct=0.854286 moved_to_echo=58972 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4052, 0, 0, 0, 0, 3580, 4090, 0, 0, 0, 249, 3787, 558, 0, 1488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 398, 4052, 1691, 3, 4095, 635, 3087, 0, 0, 0, 0, 0, 422, 1078, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 1713, 2122, 0, 32, 0, 1, 0, 1260, 11, 131, 0, 0, 4096, 0, 0, 389, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 4090, 1306, 2003, 15, 209, 0, 0, 4014, 0, 2, 0, 1398, 3884, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 5, 22, 0, 4094, 3764, 30, 4088, 3962, 784, 25, 0, 1832, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 414, 2879, 0, 249, 0, 0, 5, 4095, 18, 0, 0, 14, 0, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2813, 3945, 0, 1, 1, 0, 0, 0, 0, 0, 0, 68, 1918, 5, 0, 1334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2849, 0, 3809, 38, 3600, 2478, 2354, 1, 1, 0, 173, 397, 60, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17804, 19600, 13976, 17982, 18938, 12833, 10085, 19854] avg=16384.0000 max=19854 max_rank=7 min=10085 min_rank=6 imbalance_ratio=1.211792 max_load_reduction=111218 max_load_reduction_pct=0.848526 moved_to_echo=61710 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[419, 0, 1434, 1494, 57, 0, 1, 0, 133, 0, 138, 0, 3789, 2821, 1523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 929, 3146, 0, 4009, 197, 0, 0, 0, 0, 3917, 4062, 2025, 0, 5, 4083, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1371, 126, 4045, 85, 1093, 563, 0, 0, 0, 111, 290, 298, 694, 0, 4028, 0, 383, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3782, 3840, 0, 0, 0, 109, 73, 0, 0, 0, 0, 25, 0, 26, 432, 3070, 1481, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 3670, 785, 301, 0, 49, 0, 0, 0, 2684, 0, 172, 17, 0, 0, 4096, 2908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 3949, 0, 0, 3596, 0, 2670, 489, 0, 0, 3537, 3753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 4061, 2937, 0, 1900, 4054, 0, 3747, 0, 0, 8, 1, 0, 2, 152, 194, 1313, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 712, 4083, 3944, 2317, 6, 5, 1182, 46, 15, 2098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11809, 22438, 13329, 13243, 14709, 22090, 19046, 14408] avg=16384.0000 max=22438 max_rank=1 min=11809 min_rank=0 imbalance_ratio=1.369507 max_load_reduction=108634 max_load_reduction_pct=0.828812 moved_to_echo=70253 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[348, 0, 3678, 3296, 0, 0, 4096, 0, 69, 0, 0, 0, 3789, 657, 2378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 4089, 1094, 3061, 0, 168, 0, 0, 1, 0, 0, 3473, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4096, 1835, 0, 0, 554, 0, 0, 72, 4096, 75, 294, 15, 43, 1255, 0, 376, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3835, 157, 113, 115, 0, 0, 0, 0, 0, 4007, 2484, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3489, 0, 0, 4, 1, 4093, 0, 91, 4093, 4094, 0, 0, 14, 0, 0, 3546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4083, 0, 0, 621, 0, 0, 1, 818, 0, 1, 3426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3718, 4096, 4084, 0, 3933, 0, 3, 0, 0, 0, 0, 0, 0, 2, 0, 1292, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 7, 20, 3800, 3852, 2280, 0, 0, 4096, 0, 2548, 18, 0, 267, 719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18311, 12009, 17026, 11205, 19426, 17142, 17818, 18135] avg=16384.0000 max=19426 max_rank=4 min=11205 min_rank=3 imbalance_ratio=1.185669 max_load_reduction=111646 max_load_reduction_pct=0.851791 moved_to_echo=72521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[17664, 18176, 768, 7040, 14848, 14336, 512, 1792, 15616, 7040, 4224, 3840, 0, 256, 7680, 17920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[18688, 25088, 23680, 1408, 1152, 1280, 128, 256, 13440, 17792, 3456, 640, 1152, 6016, 13312, 128, 2816, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[16896, 1792, 14720, 4992, 24576, 17024, 16384, 256, 128, 13696, 256, 7168, 2176, 1792, 512, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 5120, 15872, 5248, 21120, 1536, 9984, 768, 3200, 4096, 16384, 4096, 25600, 128, 5376, 12800, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[4224, 26368, 30720, 12416, 9600, 20608, 512, 3840, 128, 128, 2304, 3712, 256, 4352, 256, 256, 9344, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[10368, 4096, 17152, 5248, 12800, 4224, 16000, 4352, 512, 256, 1536, 5760, 29696, 14208, 3968, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[16768, 10112, 2560, 4096, 9728, 18432, 5760, 512, 512, 1024, 19584, 4224, 896, 256, 1920, 21120, 11776, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[1536, 20224, 1024, 3968, 1664, 1408, 12032, 6400, 8576, 6912, 16384, 8320, 896, 512, 7296, 13184, 22144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[233, 2944, 699, 404, 1195, 420, 744, 949, 1376, 172, 1461, 325, 1101, 664, 116, 342, 166, 637, 1488, 591, 890, 144, 2621, 1649, 266, 875, 1101, 813, 1459, 490, 1673, 1679, 2789, 547, 307, 502, 104, 2086, 2062, 437, 2777, 368, 1441, 945, 1196, 1468, 1167, 1154, 311, 291, 252, 2158, 890, 576, 467, 115, 395, 860, 898, 1550, 1379, 757, 706, 1213, 801, 2272, 639, 1111, 848, 259, 455, 1983, 560, 1323, 179, 1845, 552, 989, 2945, 609, 2740, 1185, 2682, 510, 1598, 353, 834, 1222, 1531, 238, 560, 1160, 726, 410, 452, 873, 2101, 300, 1839, 559, 1725, 611, 1844, 2723, 592, 1004, 448, 1412, 574, 247, 2227, 787, 217, 1237, 987, 1693, 275, 522, 1594, 1311, 750, 1184, 1558, 884, 1746, 514, 566, 742] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[122, 2812, 827, 244, 1398, 396, 751, 1024, 800, 63, 1166, 224, 1324, 655, 143, 428, 249, 432, 1318, 1426, 670, 58, 2675, 1366, 450, 708, 878, 995, 1637, 490, 1135, 1904, 3364, 711, 350, 393, 456, 1893, 2137, 405, 2577, 327, 1294, 899, 1482, 1301, 885, 1399, 373, 135, 383, 1916, 1006, 466, 782, 220, 613, 191, 787, 1689, 1395, 715, 648, 1047, 893, 1616, 579, 867, 1287, 312, 1012, 1659, 1369, 2273, 374, 2253, 990, 540, 2323, 818, 2576, 1249, 2829, 253, 2140, 101, 620, 1265, 1549, 181, 928, 920, 865, 378, 367, 848, 1632, 276, 2197, 469, 1454, 889, 1638, 1902, 714, 961, 711, 2642, 668, 265, 1872, 1025, 294, 1524, 931, 1428, 358, 522, 696, 1404, 787, 1727, 1516, 432, 1490, 403, 389, 615] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[39, 2602, 463, 363, 1762, 362, 708, 607, 1047, 90, 1789, 224, 979, 616, 160, 477, 249, 870, 512, 777, 616, 181, 2622, 2107, 503, 1378, 997, 1059, 889, 311, 1246, 2251, 3261, 641, 459, 420, 169, 2123, 2026, 653, 2578, 350, 1250, 615, 1626, 1830, 960, 1319, 304, 315, 391, 2004, 674, 668, 557, 76, 1128, 348, 896, 1817, 1316, 603, 399, 895, 1081, 1459, 650, 1519, 1666, 303, 822, 1226, 1237, 1919, 317, 2175, 1016, 636, 2625, 650, 2562, 1316, 2709, 210, 1869, 304, 587, 793, 1659, 274, 665, 1167, 760, 584, 508, 665, 2230, 165, 1590, 530, 1254, 684, 1491, 2660, 696, 942, 590, 2095, 647, 310, 1912, 849, 244, 950, 856, 1256, 234, 403, 883, 1300, 1129, 1983, 1757, 697, 1988, 310, 462, 515] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[52, 3102, 472, 255, 1436, 313, 831, 895, 1431, 80, 1521, 126, 1034, 1091, 452, 382, 48, 414, 734, 434, 1129, 212, 2704, 1965, 834, 892, 889, 863, 1158, 396, 1815, 1861, 2843, 544, 360, 279, 78, 2372, 1937, 523, 3002, 214, 1478, 546, 1274, 1699, 1059, 922, 292, 322, 251, 1579, 582, 498, 363, 80, 678, 352, 1058, 1983, 1480, 362, 665, 1279, 662, 1861, 477, 1512, 850, 174, 679, 2227, 897, 2064, 206, 1735, 441, 1201, 2320, 1035, 2543, 747, 2598, 611, 2006, 160, 598, 1139, 1727, 204, 792, 1016, 773, 810, 382, 888, 2410, 319, 1713, 953, 1191, 836, 1476, 2877, 649, 949, 532, 2136, 750, 576, 2396, 1176, 214, 957, 1194, 1002, 153, 518, 888, 1459, 769, 1656, 1937, 706, 1322, 333, 528, 387] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[52, 2341, 450, 213, 1368, 264, 1031, 879, 942, 133, 1099, 232, 983, 684, 278, 267, 32, 340, 494, 685, 1184, 66, 3321, 1817, 657, 854, 965, 993, 1245, 144, 1300, 2077, 3458, 809, 212, 757, 278, 1857, 2592, 679, 2691, 145, 1093, 551, 1251, 1275, 1081, 620, 217, 366, 265, 1894, 726, 494, 649, 103, 635, 407, 562, 2336, 1895, 405, 997, 1119, 712, 2040, 615, 1591, 1488, 183, 764, 1859, 882, 2401, 422, 1710, 987, 770, 2055, 637, 2356, 947, 2726, 395, 2062, 147, 689, 1168, 1790, 140, 774, 1165, 819, 569, 470, 764, 2427, 206, 2123, 812, 1273, 823, 2086, 2316, 744, 545, 441, 2336, 690, 377, 2190, 1281, 257, 797, 1494, 1248, 426, 592, 756, 1351, 799, 1772, 1575, 653, 1632, 351, 466, 327] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[62, 2296, 774, 214, 1195, 281, 862, 614, 1380, 172, 1527, 234, 418, 907, 269, 397, 94, 510, 1589, 699, 942, 325, 2782, 2017, 567, 1164, 848, 1085, 1366, 333, 2061, 1603, 2793, 435, 541, 396, 121, 2269, 1723, 1057, 2378, 272, 1862, 992, 1572, 1256, 800, 896, 277, 369, 218, 1789, 607, 527, 612, 85, 440, 394, 817, 1815, 1801, 388, 930, 1025, 866, 1718, 849, 1175, 615, 306, 570, 1341, 532, 2355, 296, 1879, 854, 1073, 2496, 482, 2289, 1097, 2363, 507, 2083, 411, 1065, 950, 1630, 224, 777, 1054, 459, 648, 416, 781, 2463, 342, 1535, 615, 1559, 755, 1362, 2959, 769, 865, 621, 2161, 498, 473, 1985, 1010, 172, 1600, 1389, 1887, 208, 591, 1143, 1047, 819, 1565, 1553, 382, 2066, 307, 492, 674] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[389, 2702, 324, 486, 1591, 295, 707, 929, 1379, 142, 1439, 164, 1073, 893, 168, 380, 67, 507, 975, 639, 673, 213, 3055, 2131, 629, 847, 817, 993, 968, 487, 1710, 1533, 2696, 1113, 345, 780, 67, 1772, 1712, 700, 2492, 739, 1796, 824, 1686, 1359, 824, 944, 320, 353, 502, 2197, 990, 487, 305, 80, 619, 372, 1434, 1708, 1876, 438, 426, 1132, 663, 1594, 774, 1185, 1268, 296, 316, 1956, 866, 1953, 229, 1948, 411, 792, 1980, 1089, 2270, 1559, 2542, 518, 1635, 49, 773, 1645, 1727, 284, 824, 918, 990, 419, 373, 940, 1945, 484, 1124, 617, 1574, 324, 2503, 2406, 524, 1293, 371, 1646, 531, 338, 1838, 1087, 419, 1189, 602, 1730, 364, 386, 1083, 1287, 626, 1380, 1535, 1253, 1949, 532, 535, 418] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[44, 2609, 811, 329, 1564, 442, 1026, 716, 746, 115, 1547, 484, 1141, 494, 159, 370, 147, 346, 751, 584, 896, 40, 3344, 1861, 573, 884, 862, 702, 921, 206, 969, 1432, 2941, 481, 330, 525, 224, 1465, 1485, 540, 2750, 167, 959, 648, 1069, 1571, 1409, 878, 254, 366, 277, 1570, 738, 366, 766, 95, 818, 762, 1381, 1638, 1622, 503, 608, 822, 821, 2079, 360, 1345, 1599, 236, 712, 2081, 837, 2726, 277, 1400, 780, 896, 2421, 1353, 3025, 1102, 2577, 488, 1928, 213, 484, 1691, 2029, 295, 709, 864, 1277, 419, 350, 1135, 1828, 354, 1913, 810, 1931, 849, 2071, 3016, 747, 776, 669, 1989, 908, 475, 2126, 1311, 561, 553, 747, 1096, 156, 403, 960, 1502, 619, 1427, 1553, 982, 1166, 873, 703, 346] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[39, 2602, 463, 363, 1762, 362, 708, 607, 1047, 90, 1789, 224, 979, 616, 160, 477, 3182, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 870, 512, 777, 616, 181, 2622, 2107, 503, 1378, 997, 1059, 889, 311, 1246, 2251, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 641, 459, 420, 169, 2123, 2026, 653, 2578, 350, 1250, 615, 1626, 1830, 960, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 315, 391, 2004, 674, 668, 557, 76, 1128, 348, 896, 1817, 1316, 603, 399, 895, 2310, 1672, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 1459, 650, 1519, 1666, 303, 822, 1226, 1237, 1919, 317, 2175, 1016, 636, 315, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2562, 1316, 1875, 210, 1869, 304, 587, 793, 1659, 274, 665, 1167, 760, 584, 508, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 165, 1590, 530, 1254, 684, 1491, 0, 696, 942, 590, 2095, 647, 310, 1912, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 950, 856, 1256, 234, 403, 883, 1300, 1129, 1983, 1757, 697, 1988, 310, 462, 515, 821, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16458, 16606, 17098, 16465, 16991, 15798, 15072, 16584] avg=16384.0000 max=17098 max_rank=2 min=15072 min_rank=6 imbalance_ratio=1.043579 max_load_reduction=113974 max_load_reduction_pct=0.869553 moved_to_echo=64445 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[233, 2944, 699, 404, 1195, 420, 744, 949, 1376, 172, 1461, 325, 1101, 664, 116, 342, 2721, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 637, 1488, 591, 890, 144, 2621, 1649, 266, 875, 1101, 813, 1459, 490, 1673, 1679, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 547, 307, 502, 104, 2086, 2062, 437, 2777, 368, 1441, 945, 1196, 1468, 1167, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 291, 252, 2158, 890, 576, 467, 115, 395, 860, 898, 1550, 1379, 757, 706, 1213, 2592, 1711, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 801, 2272, 639, 1111, 848, 259, 455, 1983, 560, 1323, 179, 1845, 552, 989, 353, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2740, 1185, 1856, 510, 1598, 353, 834, 1222, 1531, 238, 560, 1160, 726, 410, 452, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1241, 300, 1839, 559, 1725, 611, 1844, 0, 592, 1004, 448, 1412, 574, 247, 2227, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 1237, 987, 1693, 275, 522, 1594, 1311, 750, 1184, 1558, 884, 1746, 514, 566, 742, 774, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16878, 16580, 16629, 17207, 14778, 16248, 15410, 17342] avg=16384.0000 max=17342 max_rank=7 min=14778 min_rank=4 imbalance_ratio=1.058472 max_load_reduction=113730 max_load_reduction_pct=0.867691 moved_to_echo=63778 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[52, 2341, 450, 213, 1368, 264, 1031, 879, 942, 133, 1099, 232, 983, 684, 278, 267, 3374, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 340, 494, 685, 1184, 66, 3321, 1817, 657, 854, 965, 993, 1245, 144, 1300, 2077, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 809, 212, 757, 278, 1857, 2592, 679, 2691, 145, 1093, 551, 1251, 1275, 1081, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 366, 265, 1894, 726, 494, 649, 103, 635, 407, 562, 2336, 1895, 405, 997, 1119, 1809, 1456, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 712, 2040, 615, 1591, 1488, 183, 764, 1859, 882, 2401, 422, 1710, 987, 770, 246, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2356, 947, 1886, 395, 2062, 147, 689, 1168, 1790, 140, 774, 1165, 819, 569, 470, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1433, 206, 2123, 812, 1273, 823, 2086, 0, 744, 545, 441, 2336, 690, 377, 2190, 1281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 797, 1494, 1248, 426, 592, 756, 1351, 799, 1772, 1575, 653, 1632, 351, 466, 327, 894, 801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15450, 16213, 15975, 16435, 17307, 16141, 17360, 16191] avg=16384.0000 max=17360 max_rank=6 min=15450 min_rank=0 imbalance_ratio=1.059570 max_load_reduction=113712 max_load_reduction_pct=0.867554 moved_to_echo=66999 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[62, 2296, 774, 214, 1195, 281, 862, 614, 1380, 172, 1527, 234, 418, 907, 269, 397, 2725, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 510, 1589, 699, 942, 325, 2782, 2017, 567, 1164, 848, 1085, 1366, 333, 2061, 1603, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 435, 541, 396, 121, 2269, 1723, 1057, 2378, 272, 1862, 992, 1572, 1256, 800, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 369, 218, 1789, 607, 527, 612, 85, 440, 394, 817, 1815, 1801, 388, 930, 1025, 2197, 1859, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 866, 1718, 849, 1175, 615, 306, 570, 1341, 532, 2355, 296, 1879, 854, 1073, 299, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2289, 1097, 1636, 507, 2083, 411, 1065, 950, 1630, 224, 777, 1054, 459, 648, 416, 781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1455, 342, 1535, 615, 1559, 755, 1362, 0, 769, 865, 621, 2161, 498, 473, 1985, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 1600, 1389, 1887, 208, 591, 1143, 1047, 819, 1565, 1553, 382, 2066, 307, 492, 674, 907, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15427, 18018, 16638, 16251, 15210, 16027, 16005, 17496] avg=16384.0000 max=18018 max_rank=1 min=15210 min_rank=4 imbalance_ratio=1.099731 max_load_reduction=113054 max_load_reduction_pct=0.862534 moved_to_echo=64738 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[52, 3102, 472, 255, 1436, 313, 831, 895, 1431, 80, 1521, 126, 1034, 1091, 452, 382, 2774, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 414, 734, 434, 1129, 212, 2704, 1965, 834, 892, 889, 863, 1158, 396, 1815, 1861, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 544, 360, 279, 78, 2372, 1937, 523, 3002, 214, 1478, 546, 1274, 1699, 1059, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 322, 251, 1579, 582, 498, 363, 80, 678, 352, 1058, 1983, 1480, 362, 665, 1279, 2042, 1809, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 1861, 477, 1512, 850, 174, 679, 2227, 897, 2064, 206, 1735, 441, 1201, 278, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2543, 747, 1798, 611, 2006, 160, 598, 1139, 1727, 204, 792, 1016, 773, 810, 382, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1423, 319, 1713, 953, 1191, 836, 1476, 0, 649, 949, 532, 2136, 750, 576, 2396, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 957, 1194, 1002, 153, 518, 888, 1459, 769, 1656, 1937, 706, 1322, 333, 528, 387, 888, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17315, 16385, 16356, 15774, 16299, 16194, 17075, 15674] avg=16384.0000 max=17315 max_rank=0 min=15674 min_rank=7 imbalance_ratio=1.056824 max_load_reduction=113757 max_load_reduction_pct=0.867897 moved_to_echo=65242 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[122, 2812, 827, 244, 1398, 396, 751, 1024, 800, 63, 1166, 224, 1324, 655, 143, 428, 3283, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 432, 1318, 1426, 670, 58, 2675, 1366, 450, 708, 878, 995, 1637, 490, 1135, 1904, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 711, 350, 393, 456, 1893, 2137, 405, 2577, 327, 1294, 899, 1482, 1301, 885, 1399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 135, 383, 1916, 1006, 466, 782, 220, 613, 191, 787, 1689, 1395, 715, 648, 1047, 2044, 1195, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 893, 1616, 579, 867, 1287, 312, 1012, 1659, 1369, 2273, 374, 2253, 990, 540, 279, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2576, 1249, 1958, 253, 2140, 101, 620, 1265, 1549, 181, 928, 920, 865, 378, 367, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 964, 276, 2197, 469, 1454, 889, 1638, 0, 714, 961, 711, 2642, 668, 265, 1872, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 1524, 931, 1428, 358, 522, 696, 1404, 787, 1727, 1516, 432, 1490, 403, 389, 615, 601, 831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16367, 16431, 16590, 15672, 17121, 16198, 16745, 15948] avg=16384.0000 max=17121 max_rank=4 min=15672 min_rank=3 imbalance_ratio=1.044983 max_load_reduction=113951 max_load_reduction_pct=0.869377 moved_to_echo=66012 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[44, 2609, 811, 329, 1564, 442, 1026, 716, 746, 115, 1547, 484, 1141, 494, 159, 370, 2870, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 346, 751, 584, 896, 40, 3344, 1861, 573, 884, 862, 702, 921, 206, 969, 1432, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 481, 330, 525, 224, 1465, 1485, 540, 2750, 167, 959, 648, 1069, 1571, 1409, 878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 366, 277, 1570, 738, 366, 766, 95, 818, 762, 1381, 1638, 1622, 503, 608, 822, 2131, 1896, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 821, 2079, 360, 1345, 1599, 236, 712, 2081, 837, 2726, 277, 1400, 780, 896, 290, 1353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3025, 1102, 1783, 488, 1928, 213, 484, 1691, 2029, 295, 709, 864, 1277, 419, 350, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 354, 1913, 810, 1931, 849, 2071, 0, 747, 776, 669, 1989, 908, 475, 2126, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 553, 747, 1096, 156, 403, 960, 1502, 619, 1427, 1553, 982, 1166, 873, 703, 346, 673, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16587, 14555, 14572, 16688, 17792, 17792, 18009, 15077] avg=16384.0000 max=18009 max_rank=6 min=14555 min_rank=1 imbalance_ratio=1.099182 max_load_reduction=113063 max_load_reduction_pct=0.862602 moved_to_echo=68670 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[389, 2702, 324, 486, 1591, 295, 707, 929, 1379, 142, 1439, 164, 1073, 893, 168, 380, 2635, 894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 507, 975, 639, 673, 213, 3055, 2131, 629, 847, 817, 993, 968, 487, 1710, 1533, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 1113, 345, 780, 67, 1772, 1712, 700, 2492, 739, 1796, 824, 1686, 1359, 824, 944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 353, 502, 2197, 990, 487, 305, 80, 619, 372, 1434, 1708, 1876, 438, 426, 1132, 1746, 1512, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 663, 1594, 774, 1185, 1268, 296, 316, 1956, 866, 1953, 229, 1948, 411, 792, 234, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2270, 1559, 1750, 518, 1635, 49, 773, 1645, 1727, 284, 824, 918, 990, 419, 373, 940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1142, 484, 1124, 617, 1574, 324, 2503, 0, 524, 1293, 371, 1646, 531, 338, 1838, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 419, 1189, 602, 1730, 364, 386, 1083, 1287, 626, 1380, 1535, 1253, 1949, 532, 535, 418, 720, 752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16590, 16284, 17214, 16580, 15574, 16674, 15396, 16760] avg=16384.0000 max=17214 max_rank=2 min=15396 min_rank=6 imbalance_ratio=1.050659 max_load_reduction=113858 max_load_reduction_pct=0.868668 moved_to_echo=64404 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[6528, 14720, 4992, 10368, 9728, 2176, 5376, 14336, 7296, 17024, 2304, 14976, 6144, 6912, 2304, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[20480, 9216, 14592, 3584, 15360, 1792, 5760, 9984, 13696, 1920, 6144, 8320, 6784, 4352, 3328, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1024, 21504, 4864, 2560, 11520, 2816, 6784, 6656, 9216, 1024, 11648, 2048, 8064, 6016, 1792, 3072, 23680, 7808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2432, 2560, 2560, 15232, 6272, 4096, 4608, 896, 5376, 3712, 7936, 14592, 12800, 4224, 5504, 8576, 16896, 13184, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1152, 4096, 7936, 5888, 7040, 1280, 23168, 14976, 4480, 7680, 7424, 7552, 9728, 2944, 12032, 14464, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[10112, 2560, 14080, 5376, 12032, 5888, 14592, 0, 5504, 7424, 4480, 16512, 5376, 3072, 16640, 8576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2432, 8832, 8320, 11392, 2176, 3968, 8064, 10752, 6400, 12800, 13056, 6016, 13440, 3712, 4224, 4096, 6400, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[640, 5376, 2944, 4096, 1536, 15872, 15744, 5120, 21248, 2688, 11264, 6144, 11264, 11776, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[3651, 52, 731, 1583, 860, 0, 736, 75, 3105, 0, 2171, 52, 3703, 2022, 27, 3740, 397, 0, 2187, 0, 3, 1640, 3636, 46, 7, 0, 0, 100, 2, 878, 3870, 179, 959, 972, 3330, 4, 12, 0, 1, 35, 3540, 191, 17, 7, 1149, 249, 568, 0, 0, 909, 4, 0, 402, 3152, 0, 0, 249, 4092, 2928, 2627, 1379, 0, 5, 2, 2, 1103, 4095, 2, 191, 0, 3376, 235, 261, 134, 8, 17, 7, 0, 1754, 13, 3784, 3471, 449, 267, 3055, 462, 3896, 4088, 0, 406, 2846, 4036, 1746, 753, 0, 822, 0, 757, 3577, 2300, 1623, 3, 350, 2, 0, 122, 1631, 0, 2290, 1502, 38, 1775, 91, 89, 36, 3, 5, 3702, 22, 0, 1263, 4, 0, 1291, 701, 12, 360, 4008] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[3650, 0, 962, 2259, 2457, 1, 200, 294, 2167, 82, 1370, 1, 1081, 2247, 4, 2789, 219, 239, 3821, 5, 17, 3855, 32, 71, 0, 51, 0, 305, 1, 2929, 3941, 1, 488, 3470, 2966, 0, 1, 0, 745, 36, 130, 26, 5, 0, 78, 36, 22, 1, 0, 197, 167, 0, 82, 2088, 0, 129, 5, 3179, 1101, 3165, 2198, 1, 24, 27, 8, 40, 3717, 13, 218, 1, 4014, 21, 1448, 482, 0, 55, 144, 0, 3637, 10, 4092, 3113, 2041, 3993, 3790, 0, 12, 4094, 51, 32, 881, 4096, 661, 2746, 57, 60, 0, 13, 3974, 2517, 1331, 5, 2271, 1, 0, 995, 1276, 24, 3497, 558, 3, 1021, 2982, 140, 163, 2, 0, 3150, 669, 0, 1954, 0, 0, 1907, 382, 8, 20, 3264] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2452, 1, 1598, 570, 3784, 0, 339, 3309, 1668, 1307, 1754, 212, 911, 428, 59, 885, 418, 50, 953, 30, 16, 509, 23, 985, 372, 1021, 0, 128, 13, 4061, 3946, 0, 3538, 1926, 4058, 0, 0, 1, 11, 1, 4, 14, 0, 0, 3, 0, 521, 167, 2, 0, 167, 0, 2307, 3054, 0, 4, 10, 4017, 1173, 87, 3017, 0, 554, 8, 6, 64, 1536, 1068, 315, 114, 3271, 8, 199, 828, 80, 175, 1237, 4, 4081, 0, 3817, 1498, 2432, 1111, 3209, 86, 432, 4080, 64, 553, 2017, 3866, 698, 462, 0, 235, 3, 55, 4070, 882, 2152, 0, 3452, 0, 236, 86, 440, 11, 3532, 2192, 187, 1458, 1970, 120, 27, 1, 0, 3263, 2964, 22, 3771, 79, 1, 2379, 40, 11, 0, 3676] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[3415, 2, 2921, 125, 1233, 0, 13, 203, 1113, 239, 3585, 16, 1003, 272, 89, 3594, 106, 0, 2968, 1, 686, 246, 1540, 8, 0, 120, 55, 375, 0, 856, 3473, 0, 3748, 3038, 4066, 4, 27, 14, 22, 0, 11, 2377, 7, 89, 731, 0, 2117, 0, 11, 600, 341, 0, 42, 3399, 2, 1, 11, 4022, 0, 1405, 2255, 0, 111, 32, 0, 1, 3826, 1, 350, 2, 3556, 1, 792, 4009, 1, 0, 745, 0, 3451, 0, 3439, 3828, 1890, 42, 4058, 65, 3125, 4094, 7, 85, 3383, 4079, 31, 2252, 1, 104, 4, 51, 4025, 620, 2128, 3, 2702, 14, 8, 1689, 3826, 2, 2604, 1757, 2332, 54, 341, 40, 137, 8, 0, 380, 2296, 0, 400, 7, 13, 19, 66, 49, 14, 1525] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2788, 0, 4094, 539, 2525, 0, 160, 3929, 3874, 1780, 0, 1039, 2285, 2086, 33, 1690, 0, 0, 506, 0, 10, 3985, 2, 1139, 115, 4, 0, 6, 0, 4078, 704, 31, 2324, 1340, 3966, 0, 0, 3, 23, 23, 1045, 303, 0, 5, 12, 28, 27, 6, 0, 360, 7, 1, 23, 445, 60, 1, 12, 3419, 0, 2028, 2474, 0, 26, 38, 0, 2, 3147, 912, 220, 8, 4090, 4, 2630, 0, 2410, 978, 474, 0, 87, 1, 4085, 1559, 23, 8, 3697, 0, 350, 3982, 0, 1484, 532, 3990, 15, 956, 0, 2, 19, 1727, 4095, 278, 3918, 96, 2833, 65, 0, 2404, 3683, 3, 1097, 7, 13, 3994, 477, 105, 3110, 7, 0, 1272, 600, 102, 1757, 0, 0, 3750, 24, 0, 510, 4079] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4078, 2, 1334, 424, 3338, 1, 118, 3488, 3228, 299, 124, 0, 320, 3, 40, 3958, 36, 14, 30, 0, 161, 2295, 1473, 2211, 263, 1473, 0, 1406, 0, 3697, 3977, 0, 1134, 352, 1235, 0, 0, 0, 1, 4, 474, 102, 0, 0, 2, 2923, 4, 2, 66, 42, 68, 0, 972, 1951, 12, 0, 35, 3921, 2053, 376, 236, 109, 79, 11, 31, 502, 3863, 224, 281, 0, 4089, 0, 4044, 37, 509, 565, 208, 17, 4037, 21, 4002, 282, 3620, 3033, 3281, 175, 2879, 3947, 2, 388, 2884, 1794, 22, 67, 8, 286, 1, 398, 4096, 496, 4095, 12, 3555, 101, 5, 228, 709, 88, 4030, 917, 1540, 3117, 293, 6, 45, 8, 0, 591, 3, 0, 1407, 91, 60, 1535, 18, 543, 9, 4022] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1094, 4, 4047, 3936, 3585, 0, 39, 1374, 3429, 436, 17, 4, 3856, 1985, 69, 1491, 107, 0, 2032, 6, 805, 3963, 619, 61, 15, 1544, 28, 298, 1, 1532, 4039, 0, 610, 363, 4017, 0, 0, 0, 1505, 8, 262, 1, 2, 0, 5, 152, 249, 1, 0, 17, 35, 0, 358, 2971, 4, 178, 1, 4058, 2544, 2870, 2183, 0, 3, 5, 0, 25, 3988, 79, 35, 1, 3208, 2, 136, 320, 57, 153, 147, 12, 2586, 0, 3976, 3926, 2639, 2157, 3066, 93, 144, 3908, 0, 445, 719, 4096, 259, 287, 5, 1143, 1, 4, 3932, 3191, 570, 33, 3269, 0, 13, 1485, 2210, 336, 3826, 888, 2, 561, 926, 67, 25, 17, 0, 2494, 475, 0, 1785, 3, 7, 920, 243, 7, 298, 3054] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[3462, 2, 4023, 135, 342, 0, 9, 4039, 985, 1144, 297, 9, 73, 692, 15, 2970, 41, 3, 2831, 19, 29, 3378, 33, 195, 0, 59, 19, 379, 86, 2623, 1477, 8, 3223, 310, 3663, 22, 0, 0, 109, 42, 2250, 216, 15, 0, 0, 1, 1903, 12, 0, 1338, 0, 1, 295, 2022, 1, 18, 0, 3996, 2720, 3960, 342, 0, 0, 0, 3, 237, 2666, 636, 63, 266, 3849, 0, 3396, 15, 62, 170, 482, 10, 3052, 1, 3923, 3137, 2854, 328, 1671, 40, 9, 3844, 0, 693, 3168, 4092, 0, 133, 6, 81, 0, 91, 3968, 2263, 1896, 0, 4083, 0, 180, 1513, 3642, 4, 3555, 5, 0, 3201, 340, 599, 404, 7, 6, 3060, 11, 0, 1388, 185, 2, 1537, 80, 36, 445, 3848] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 1334, 424, 3338, 1, 118, 3488, 3228, 299, 124, 0, 320, 3, 40, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 14, 30, 0, 161, 2295, 1473, 2211, 263, 1473, 0, 1406, 0, 3697, 3977, 0, 1123, 1013, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1134, 352, 1235, 0, 0, 0, 1, 4, 474, 102, 0, 0, 2, 2923, 4, 2, 3947, 2481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 42, 68, 0, 972, 1951, 12, 0, 35, 3921, 2053, 376, 236, 109, 79, 11, 3065, 1171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 502, 3863, 224, 281, 0, 4089, 0, 4044, 37, 509, 565, 208, 17, 4037, 21, 2574, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 3620, 3033, 3281, 175, 2879, 0, 2, 388, 2884, 671, 22, 67, 8, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 398, 0, 496, 4095, 12, 3555, 101, 5, 228, 709, 88, 3443, 917, 1540, 3117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 6, 45, 8, 0, 591, 3, 0, 1407, 91, 60, 1535, 18, 543, 9, 4022, 2831, 1615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13932, 19343, 12661, 14167, 21589, 17598, 18705, 13077] avg=16384.0000 max=21589 max_rank=4 min=12661 min_rank=2 imbalance_ratio=1.317688 max_load_reduction=109483 max_load_reduction_pct=0.835289 moved_to_echo=70969 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 962, 2259, 2457, 1, 200, 294, 2167, 82, 1370, 1, 1081, 2247, 4, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 239, 3821, 5, 17, 3855, 32, 71, 0, 51, 0, 305, 1, 2929, 3941, 1, 2564, 907, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 3470, 2966, 0, 1, 0, 745, 36, 130, 26, 5, 0, 78, 36, 22, 1, 4094, 2407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 167, 0, 82, 2088, 0, 129, 5, 3179, 1101, 3165, 2198, 1, 24, 27, 2743, 1196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 40, 3717, 13, 218, 1, 4014, 21, 1448, 482, 0, 55, 144, 0, 3637, 10, 1813, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3113, 2041, 3993, 3790, 0, 12, 0, 51, 32, 881, 1532, 661, 2746, 57, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 2517, 1331, 5, 2271, 1, 0, 995, 1276, 24, 2987, 558, 3, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2982, 140, 163, 2, 0, 3150, 669, 0, 1954, 0, 0, 1907, 382, 8, 20, 3264, 2896, 1567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13980, 19079, 14505, 16302, 16131, 18969, 13002, 19104] avg=16384.0000 max=19104 max_rank=7 min=13002 min_rank=6 imbalance_ratio=1.166016 max_load_reduction=111968 max_load_reduction_pct=0.854248 moved_to_echo=67206 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 52, 731, 1583, 860, 0, 736, 75, 3105, 0, 2171, 52, 3703, 2022, 27, 1146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 0, 2187, 0, 3, 1640, 3636, 46, 7, 0, 0, 100, 2, 878, 3870, 179, 2526, 906, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 959, 972, 3330, 4, 12, 0, 1, 35, 3540, 191, 17, 7, 1149, 249, 568, 0, 4088, 2166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 909, 4, 0, 402, 3152, 0, 0, 249, 4092, 2928, 2627, 1379, 0, 5, 2, 2745, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1103, 4095, 2, 191, 0, 3376, 235, 261, 134, 8, 17, 7, 0, 1754, 13, 2432, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3471, 449, 267, 3055, 462, 3896, 0, 0, 406, 2846, 1510, 1746, 753, 0, 822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 0, 2300, 1623, 3, 350, 2, 0, 122, 1631, 0, 1956, 1502, 38, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 89, 36, 3, 5, 3702, 22, 0, 1263, 4, 0, 1291, 701, 12, 360, 4008, 2678, 1411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16263, 16539, 17288, 19600, 13964, 19683, 12059, 15676] avg=16384.0000 max=19683 max_rank=5 min=12059 min_rank=6 imbalance_ratio=1.201355 max_load_reduction=111389 max_load_reduction_pct=0.849831 moved_to_echo=61382 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1598, 570, 3784, 0, 339, 3309, 1668, 1307, 1754, 212, 911, 428, 59, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 50, 953, 30, 16, 509, 23, 985, 372, 1021, 0, 128, 13, 4061, 3946, 0, 2420, 609, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3538, 1926, 4058, 0, 0, 1, 11, 1, 4, 14, 0, 0, 3, 0, 521, 167, 4080, 2466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 167, 0, 2307, 3054, 0, 4, 10, 4017, 1173, 87, 3017, 0, 554, 8, 1843, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 64, 1536, 1068, 315, 114, 3271, 8, 199, 828, 80, 175, 1237, 4, 4081, 0, 575, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1498, 2432, 1111, 3209, 86, 432, 0, 64, 553, 2017, 1446, 698, 462, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 55, 0, 882, 2152, 0, 3452, 0, 236, 86, 440, 11, 3017, 2192, 187, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1970, 120, 27, 1, 0, 3263, 2964, 22, 3771, 79, 1, 2379, 40, 11, 0, 3676, 2700, 1604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16212, 15592, 16790, 17360, 14076, 14243, 14171, 22628] avg=16384.0000 max=22628 max_rank=7 min=14076 min_rank=4 imbalance_ratio=1.381104 max_load_reduction=108444 max_load_reduction_pct=0.827362 moved_to_echo=65118 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 539, 2525, 0, 160, 3929, 3874, 1780, 0, 1039, 2285, 2086, 33, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 0, 10, 3985, 2, 1139, 115, 4, 0, 6, 0, 4078, 704, 31, 2500, 693, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2324, 1340, 3966, 0, 0, 3, 23, 23, 1045, 303, 0, 5, 12, 28, 27, 6, 3982, 2481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 7, 1, 23, 445, 60, 1, 12, 3419, 0, 2028, 2474, 0, 26, 38, 2095, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3147, 912, 220, 8, 4090, 4, 2630, 0, 2410, 978, 474, 0, 87, 1, 1102, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1559, 23, 8, 3697, 0, 350, 0, 0, 1484, 532, 1490, 15, 956, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1727, 0, 278, 3918, 96, 2833, 65, 0, 2404, 3683, 3, 934, 7, 13, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 105, 3110, 7, 0, 1272, 600, 102, 1757, 0, 0, 3750, 24, 0, 510, 4079, 2890, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22855, 13850, 15568, 12184, 16228, 10116, 19974, 20297] avg=16384.0000 max=22855 max_rank=0 min=10116 min_rank=5 imbalance_ratio=1.394958 max_load_reduction=108217 max_load_reduction_pct=0.825630 moved_to_echo=66615 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4023, 135, 342, 0, 9, 4039, 985, 1144, 297, 9, 73, 692, 15, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 3, 2831, 19, 29, 3378, 33, 195, 0, 59, 19, 379, 86, 2623, 1477, 8, 2561, 859, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3223, 310, 3663, 22, 0, 0, 109, 42, 2250, 216, 15, 0, 0, 1, 1903, 12, 3844, 2403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1338, 0, 1, 295, 2022, 1, 18, 0, 3996, 2720, 3960, 342, 0, 0, 0, 2603, 1147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 237, 2666, 636, 63, 266, 3849, 0, 3396, 15, 62, 170, 482, 10, 3052, 1, 1931, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3137, 2854, 328, 1671, 40, 9, 0, 0, 693, 3168, 1531, 0, 133, 6, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 2263, 1896, 0, 4083, 0, 180, 1513, 3642, 4, 3037, 5, 0, 3201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 599, 404, 7, 6, 3060, 11, 0, 1388, 185, 2, 1537, 80, 36, 445, 3848, 2776, 1565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12676, 14728, 18013, 18443, 17357, 13651, 19915, 16289] avg=16384.0000 max=19915 max_rank=6 min=12676 min_rank=0 imbalance_ratio=1.215515 max_load_reduction=111157 max_load_reduction_pct=0.848061 moved_to_echo=67212 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 2921, 125, 1233, 0, 13, 203, 1113, 239, 3585, 16, 1003, 272, 89, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 2968, 1, 686, 246, 1540, 8, 0, 120, 55, 375, 0, 856, 3473, 0, 2553, 849, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 3038, 4066, 4, 27, 14, 22, 0, 11, 2377, 7, 89, 731, 0, 2117, 0, 4094, 2438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 600, 341, 0, 42, 3399, 2, 1, 11, 4022, 0, 1405, 2255, 0, 111, 32, 2566, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3826, 1, 350, 2, 3556, 1, 792, 4009, 1, 0, 745, 0, 3451, 0, 2337, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3828, 1890, 42, 4058, 65, 3125, 0, 7, 85, 3383, 1526, 31, 2252, 1, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 51, 0, 620, 2128, 3, 2702, 14, 8, 1689, 3826, 2, 2225, 1757, 2332, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 40, 137, 8, 0, 380, 2296, 0, 400, 7, 13, 19, 66, 49, 14, 1525, 2433, 1587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11915, 13992, 22783, 15804, 19451, 20397, 17415, 9315] avg=16384.0000 max=22783 max_rank=2 min=9315 min_rank=7 imbalance_ratio=1.390564 max_load_reduction=108289 max_load_reduction_pct=0.826180 moved_to_echo=66578 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 4047, 3936, 3585, 0, 39, 1374, 3429, 436, 17, 4, 3856, 1985, 69, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 2032, 6, 805, 3963, 619, 61, 15, 1544, 28, 298, 1, 1532, 4039, 0, 2564, 271, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 363, 4017, 0, 0, 0, 1505, 8, 262, 1, 2, 0, 5, 152, 249, 1, 3908, 2381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 35, 0, 358, 2971, 4, 178, 1, 4058, 2544, 2870, 2183, 0, 3, 5, 823, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 3988, 79, 35, 1, 3208, 2, 136, 320, 57, 153, 147, 12, 2586, 0, 969, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 2639, 2157, 3066, 93, 144, 0, 0, 445, 719, 1532, 259, 287, 5, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 3191, 570, 33, 3269, 0, 13, 1485, 2210, 336, 3268, 888, 2, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 926, 67, 25, 17, 0, 2494, 475, 0, 1785, 3, 7, 920, 243, 7, 298, 3054, 2814, 1551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23239, 17949, 13464, 17212, 12276, 16415, 15831, 14686] avg=16384.0000 max=23239 max_rank=0 min=12276 min_rank=4 imbalance_ratio=1.418396 max_load_reduction=107833 max_load_reduction_pct=0.822701 moved_to_echo=59208 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 19712, 9600, 18176, 128, 1664, 16768, 19584, 5376, 9344, 1408, 13312, 9856, 384, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1408, 384, 15360, 128, 1792, 19968, 7424, 4736, 896, 4352, 128, 3072, 128, 20736, 25472, 256, 18816, 6144, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 3200, 0, 12672, 17792, 256, 22528, 256, 512, 8576, 17536, 512, 20992, 7936, 4224, 15232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[7424, 1280, 3968, 128, 128, 17920, 7040, 128, 13824, 384, 128, 13440, 1664, 768, 1664, 27520, 22144, 12544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 3584, 896, 128, 4608, 19200, 128, 384, 384, 30720, 12544, 16640, 14208, 128, 896, 128, 18560, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 2048, 26880, 2944, 1792, 512, 29568, 384, 12928, 5888, 3200, 2176, 3456, 128, 22784, 128, 13824, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[16128, 11776, 27392, 128, 128, 128, 2432, 256, 7808, 3328, 128, 128, 2048, 3456, 5504, 256, 32128, 19328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 20864, 16000, 11008, 25856, 1024, 10880, 0, 128, 4096, 16512, 11264, 3456, 7680, 128, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4096, 21, 3530, 2227, 0, 3190, 13, 2779, 0, 3528, 0, 1751, 2821, 0, 0, 164, 16, 344, 7, 1, 222, 0, 4, 417, 4029, 722, 0, 50, 1091, 4094, 819, 3806, 41, 105, 476, 2084, 21, 2620, 3, 9, 156, 0, 47, 3824, 2924, 2, 0, 0, 3, 4063, 521, 0, 201, 0, 0, 134, 719, 0, 1, 2783, 3411, 3617, 5, 2146, 883, 2, 4089, 8, 0, 284, 3798, 0, 65, 1318, 4061, 0, 0, 1, 13, 106, 375, 307, 3160, 2, 99, 4091, 215, 4047, 68, 428, 0, 0, 1, 0, 2756, 71, 244, 0, 3392, 1, 1641, 4, 0, 4, 180, 0, 2378, 190, 3, 4096, 0, 0, 2879, 0, 4078, 3908, 72, 79, 0, 0, 3465, 0, 3893, 0, 7, 0, 0, 652] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[4093, 37, 0, 246, 5, 3591, 0, 4094, 1, 3966, 0, 3990, 3991, 35, 60, 2, 0, 34, 0, 0, 12, 704, 7, 1, 1739, 3866, 92, 0, 661, 3611, 4004, 0, 392, 0, 2, 283, 190, 4093, 0, 24, 2857, 35, 0, 2809, 143, 2, 789, 0, 72, 2915, 1, 8, 4, 0, 16, 574, 50, 23, 328, 3642, 4090, 1368, 1, 4034, 2796, 13, 4096, 2, 0, 4089, 1811, 0, 325, 1, 4, 11, 0, 1, 116, 63, 3, 3709, 4093, 2, 218, 4059, 2832, 4090, 4, 2546, 0, 0, 0, 1, 100, 1634, 0, 16, 711, 8, 1468, 1025, 0, 0, 1, 19, 2382, 0, 0, 689, 2, 0, 3986, 4, 4080, 3700, 307, 7, 301, 2, 4075, 3, 0, 0, 0, 2, 149, 3929] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4096, 7, 4082, 5, 111, 2311, 0, 3985, 0, 4089, 0, 4075, 2321, 1863, 1, 2, 0, 139, 0, 11, 0, 0, 0, 155, 233, 2682, 420, 14, 113, 4080, 619, 352, 0, 6, 7, 109, 2, 4081, 0, 6, 3011, 0, 0, 73, 4022, 0, 844, 0, 1, 3951, 138, 0, 64, 0, 0, 0, 489, 0, 88, 22, 4096, 3987, 1, 4084, 3331, 1, 1062, 0, 11, 3704, 3178, 0, 774, 51, 3700, 0, 0, 0, 3, 0, 56, 3734, 809, 2, 597, 4087, 3946, 3966, 0, 4034, 0, 1557, 0, 0, 1690, 2, 7, 0, 4086, 0, 1798, 0, 0, 0, 29, 0, 15, 0, 0, 91, 1203, 0, 291, 0, 4086, 4096, 0, 117, 0, 17, 3988, 3782, 0, 0, 0, 0, 4, 2319] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[3962, 0, 3954, 0, 0, 4024, 4, 0, 0, 4078, 0, 6, 59, 0, 4, 2644, 0, 29, 0, 13, 1556, 0, 0, 1, 1, 25, 3967, 0, 47, 3212, 226, 11, 4096, 0, 0, 2334, 0, 1598, 0, 0, 4093, 0, 0, 151, 2378, 4088, 2051, 0, 1, 4090, 10, 135, 0, 1, 1645, 1976, 0, 0, 414, 4070, 3718, 3360, 2891, 1, 48, 0, 4096, 0, 0, 134, 3807, 0, 154, 356, 3826, 3719, 0, 0, 0, 0, 4, 4082, 111, 434, 6, 4082, 0, 4096, 18, 3060, 1, 13, 0, 13, 4091, 67, 0, 0, 3426, 2307, 27, 0, 3, 4, 0, 0, 2747, 5, 0, 3047, 38, 0, 1, 2133, 4096, 4026, 3, 0, 23, 0, 1794, 13, 191, 0, 1, 0, 44, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4096, 0, 3691, 2793, 0, 3690, 11, 114, 0, 3954, 0, 2816, 173, 0, 9, 4, 0, 0, 0, 826, 0, 0, 301, 2, 3546, 3853, 0, 0, 227, 3612, 0, 3873, 3, 0, 653, 3513, 1, 3165, 0, 0, 4084, 0, 5, 14, 873, 24, 0, 0, 6, 2980, 5, 217, 0, 0, 1259, 0, 1109, 0, 233, 3486, 4086, 3572, 9, 4087, 3726, 0, 3690, 27, 0, 26, 4024, 2, 490, 26, 34, 10, 0, 0, 0, 0, 37, 3877, 1863, 298, 3882, 4071, 974, 4096, 26, 3920, 1, 62, 0, 0, 4093, 0, 1, 0, 1416, 5, 78, 0, 22, 164, 387, 0, 3037, 3363, 0, 122, 0, 0, 0, 11, 4091, 3957, 253, 0, 275, 0, 0, 12, 9, 7, 20, 1, 135, 3476] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[138, 1, 4096, 0, 797, 4096, 6, 3926, 0, 4096, 1, 4051, 3839, 4046, 0, 0, 37, 2652, 6, 0, 0, 3, 0, 3181, 1962, 108, 419, 0, 0, 4086, 3537, 4060, 9, 20, 0, 0, 0, 118, 0, 0, 2191, 0, 5, 3119, 2862, 4, 15, 0, 0, 3751, 1, 35, 3, 0, 121, 15, 1, 0, 0, 31, 3807, 3908, 0, 3913, 61, 0, 1459, 22, 3, 172, 3925, 455, 1352, 20, 4085, 616, 0, 2, 36, 0, 5, 2675, 1092, 1962, 188, 4090, 1044, 4095, 0, 2136, 0, 1, 0, 579, 3247, 6, 0, 1, 4092, 13, 428, 0, 0, 79, 0, 1, 0, 9, 103, 3515, 320, 1, 3648, 2511, 4093, 2109, 2290, 0, 0, 0, 0, 13, 1266, 0, 170, 0, 0, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[253, 68, 3955, 5, 1725, 4046, 17, 1183, 0, 3590, 0, 4094, 3942, 3510, 25, 5, 367, 1107, 3, 0, 1158, 36, 0, 0, 433, 0, 2833, 0, 5, 3236, 2, 3785, 3985, 0, 2, 0, 4, 52, 170, 1, 3941, 0, 107, 0, 3123, 347, 2, 0, 0, 4023, 0, 56, 1, 0, 10, 69, 3911, 3, 236, 4096, 4089, 4073, 2, 4066, 0, 0, 409, 4, 43, 2, 4016, 10, 815, 0, 4096, 8, 1, 20, 2, 8, 152, 4096, 919, 19, 0, 2353, 149, 4064, 15, 3245, 28, 6, 13, 0, 591, 0, 86, 22, 3235, 7, 114, 61, 1, 0, 27, 0, 2881, 2822, 0, 4077, 0, 18, 341, 415, 4020, 3838, 394, 0, 15, 0, 3196, 0, 36, 0, 1508, 0, 3117, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[3853, 0, 4095, 85, 4, 4090, 65, 3395, 14, 4095, 0, 4088, 77, 34, 0, 15, 4, 0, 27, 294, 7, 10, 0, 8, 3931, 35, 3697, 109, 344, 4093, 657, 5, 0, 21, 241, 4011, 32, 1618, 296, 0, 3692, 14, 0, 5, 4081, 0, 122, 0, 0, 3847, 11, 1, 5, 19, 1796, 62, 0, 20, 295, 3050, 4093, 1243, 92, 4081, 708, 11, 3615, 0, 125, 3220, 3810, 0, 996, 0, 721, 31, 0, 15, 1, 0, 37, 4021, 848, 55, 580, 4094, 3809, 3994, 0, 4062, 0, 2, 0, 68, 1034, 164, 0, 1, 4094, 0, 0, 0, 55, 5, 0, 1, 2142, 0, 3, 4096, 5, 0, 799, 3, 4092, 3216, 1939, 1771, 4, 11, 103, 4, 0, 0, 1294, 0, 2, 1532] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 21, 3122, 2227, 0, 0, 13, 2779, 0, 0, 0, 1751, 2821, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 344, 7, 1, 222, 0, 4, 417, 4029, 722, 0, 50, 1091, 4094, 819, 3806, 1187, 884, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 105, 476, 2084, 21, 2620, 3, 9, 156, 0, 47, 3824, 2924, 2, 0, 0, 1832, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4063, 521, 0, 201, 0, 0, 134, 719, 0, 1, 2783, 937, 3617, 5, 2146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 883, 2, 4089, 8, 0, 284, 3798, 0, 65, 1318, 4061, 0, 0, 1, 13, 106, 1358, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 307, 3160, 2, 99, 3207, 215, 0, 68, 428, 0, 0, 1, 0, 2756, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 3392, 1, 1641, 4, 0, 4, 180, 0, 2378, 190, 3, 4096, 0, 0, 4047, 2295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2879, 0, 4078, 3908, 72, 79, 0, 0, 3465, 0, 3893, 0, 7, 0, 0, 652, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16994, 18043, 15377, 15130, 17273, 10689, 18475, 19091] avg=16384.0000 max=19091 max_rank=7 min=10689 min_rank=5 imbalance_ratio=1.165222 max_load_reduction=111981 max_load_reduction_pct=0.854347 moved_to_echo=65528 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4093, 37, 0, 246, 5, 0, 0, 4094, 1, 0, 0, 3990, 3991, 35, 60, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 12, 704, 7, 1, 1739, 3866, 92, 0, 661, 3611, 4004, 0, 1423, 877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 2, 283, 190, 4093, 0, 24, 2857, 35, 0, 2809, 143, 2, 789, 0, 2062, 1385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 2915, 1, 8, 4, 0, 16, 574, 50, 23, 328, 3642, 1123, 1368, 1, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2796, 13, 4096, 2, 0, 4089, 1811, 0, 325, 1, 4, 11, 0, 1, 116, 63, 1529, 1544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3709, 4093, 2, 218, 3182, 2832, 0, 4, 2546, 0, 0, 0, 1, 100, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 711, 8, 1468, 1025, 0, 0, 1, 19, 2382, 0, 0, 689, 2, 0, 4090, 2581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 4, 4080, 3700, 307, 7, 301, 2, 4075, 3, 0, 0, 0, 2, 149, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16554, 17031, 15066, 14159, 16401, 18324, 12992, 20545] avg=16384.0000 max=20545 max_rank=7 min=12992 min_rank=6 imbalance_ratio=1.253967 max_load_reduction=110527 max_load_reduction_pct=0.843254 moved_to_echo=68262 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 7, 3609, 5, 111, 0, 0, 3985, 0, 0, 0, 4075, 2321, 1863, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 11, 0, 0, 0, 155, 233, 2682, 420, 14, 113, 4080, 619, 352, 1425, 883, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 109, 2, 4081, 0, 6, 3011, 0, 0, 73, 4022, 0, 844, 0, 1327, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3951, 138, 0, 64, 0, 0, 0, 489, 0, 88, 22, 1125, 3987, 1, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3331, 1, 1062, 0, 11, 3704, 3178, 0, 774, 51, 3700, 0, 0, 0, 3, 0, 984, 1546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 3734, 809, 2, 597, 3204, 3946, 0, 0, 4034, 0, 1557, 0, 0, 1690, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4086, 0, 1798, 0, 0, 0, 29, 0, 15, 0, 0, 91, 1203, 0, 3966, 2661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 0, 4086, 4096, 0, 117, 0, 17, 3988, 3782, 0, 0, 0, 0, 4, 2319, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20075, 11531, 14916, 13950, 18345, 19631, 13856, 18768] avg=16384.0000 max=20075 max_rank=0 min=11531 min_rank=1 imbalance_ratio=1.225281 max_load_reduction=110997 max_load_reduction_pct=0.846840 moved_to_echo=70600 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[138, 1, 3621, 0, 797, 0, 6, 3926, 0, 0, 1, 4051, 3839, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 2652, 6, 0, 0, 3, 0, 3181, 1962, 108, 419, 0, 0, 4086, 3537, 4060, 1325, 884, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 20, 0, 0, 0, 118, 0, 0, 2191, 0, 5, 3119, 2862, 4, 15, 0, 2353, 1432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3751, 1, 35, 3, 0, 121, 15, 1, 0, 0, 31, 1045, 3908, 0, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 1459, 22, 3, 172, 3925, 455, 1352, 20, 4085, 616, 0, 2, 36, 0, 1743, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2675, 1092, 1962, 188, 3206, 1044, 0, 0, 2136, 0, 1, 0, 579, 3247, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 13, 428, 0, 0, 79, 0, 1, 0, 9, 103, 3515, 320, 1, 4095, 2664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3648, 2511, 4093, 2109, 2290, 0, 0, 0, 0, 13, 1266, 0, 170, 0, 0, 9, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20426, 22667, 12128, 12824, 15388, 16141, 15321, 16177] avg=16384.0000 max=22667 max_rank=1 min=12128 min_rank=2 imbalance_ratio=1.383484 max_load_reduction=108405 max_load_reduction_pct=0.827065 moved_to_echo=63027 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[253, 68, 3497, 5, 1725, 0, 17, 1183, 0, 0, 0, 4094, 3942, 3510, 25, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 1107, 3, 0, 1158, 36, 0, 0, 433, 0, 2833, 0, 5, 3236, 2, 3785, 1423, 508, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 2, 0, 4, 52, 170, 1, 3941, 0, 107, 0, 3123, 347, 2, 0, 2323, 1254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 56, 1, 0, 10, 69, 3911, 3, 236, 4096, 1123, 4073, 2, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 4, 43, 2, 4016, 10, 815, 0, 4096, 8, 1, 20, 2, 8, 1723, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 4096, 919, 19, 0, 1845, 149, 0, 15, 3245, 28, 6, 13, 0, 591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 22, 3235, 7, 114, 61, 1, 0, 27, 0, 2881, 2822, 0, 4077, 0, 18, 4064, 2336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 415, 4020, 3838, 394, 0, 15, 0, 3196, 0, 36, 0, 1508, 0, 3117, 5, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18324, 15289, 15311, 21669, 12700, 11078, 19751, 16950] avg=16384.0000 max=21669 max_rank=3 min=11078 min_rank=5 imbalance_ratio=1.322571 max_load_reduction=109403 max_load_reduction_pct=0.834679 moved_to_echo=60479 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3962, 0, 3489, 0, 0, 0, 4, 0, 0, 0, 0, 6, 59, 0, 4, 2644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 13, 1556, 0, 0, 1, 1, 25, 3967, 0, 47, 3212, 226, 11, 1297, 886, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2334, 0, 1598, 0, 0, 4093, 0, 0, 151, 2378, 4088, 2051, 0, 2312, 1425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 10, 135, 0, 1, 1645, 1976, 0, 0, 414, 4070, 1015, 3360, 2891, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 4096, 0, 0, 134, 3807, 0, 154, 356, 3826, 3719, 0, 0, 0, 0, 1712, 1406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4082, 111, 434, 6, 3196, 0, 0, 18, 3060, 1, 13, 0, 13, 4091, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3426, 2307, 27, 0, 3, 4, 0, 0, 2747, 5, 0, 3047, 38, 0, 4096, 2653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2133, 4096, 4026, 3, 0, 23, 0, 1794, 13, 191, 0, 1, 0, 44, 1, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10168, 11667, 24526, 19609, 19258, 15096, 18353, 12395] avg=16384.0000 max=24526 max_rank=2 min=10168 min_rank=0 imbalance_ratio=1.496948 max_load_reduction=106546 max_load_reduction_pct=0.812881 moved_to_echo=65102 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3853, 0, 3621, 85, 4, 0, 65, 3395, 14, 0, 0, 4088, 77, 34, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 27, 294, 7, 10, 0, 8, 3931, 35, 3697, 109, 344, 4093, 657, 5, 1424, 885, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 241, 4011, 32, 1618, 296, 0, 3692, 14, 0, 5, 4081, 0, 122, 0, 2348, 1430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3847, 11, 1, 5, 19, 1796, 62, 0, 20, 295, 3050, 1124, 1243, 92, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 708, 11, 3615, 0, 125, 3220, 3810, 0, 996, 0, 721, 31, 0, 15, 1, 0, 1742, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4021, 848, 55, 580, 3209, 3809, 0, 0, 4062, 0, 2, 0, 68, 1034, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 55, 5, 0, 1, 2142, 0, 3, 4096, 5, 0, 3994, 2665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 799, 3, 4092, 3216, 1939, 1771, 4, 11, 103, 4, 0, 0, 1294, 0, 2, 1532, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15251, 15936, 17911, 15646, 16540, 17889, 17061, 14838] avg=16384.0000 max=17911 max_rank=2 min=14838 min_rank=7 imbalance_ratio=1.093201 max_load_reduction=113161 max_load_reduction_pct=0.863350 moved_to_echo=66328 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 3264, 2793, 0, 0, 11, 114, 0, 0, 0, 2816, 173, 0, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 0, 0, 301, 2, 3546, 3853, 0, 0, 227, 3612, 0, 3873, 1422, 880, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 653, 3513, 1, 3165, 0, 0, 4084, 0, 5, 14, 873, 24, 0, 0, 2120, 1382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2980, 5, 217, 0, 0, 1259, 0, 1109, 0, 233, 3486, 1122, 3572, 9, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3726, 0, 3690, 27, 0, 26, 4024, 2, 490, 26, 34, 10, 0, 0, 0, 0, 1570, 1542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 3877, 1863, 298, 3882, 3191, 974, 0, 26, 3920, 1, 62, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1416, 5, 78, 0, 22, 164, 387, 0, 3037, 3363, 0, 122, 0, 0, 4096, 2572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4091, 3957, 253, 0, 275, 0, 0, 12, 9, 7, 20, 1, 135, 3476, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13280, 18908, 15837, 18085, 15167, 22224, 15263, 12308] avg=16384.0000 max=22224 max_rank=5 min=12308 min_rank=7 imbalance_ratio=1.356445 max_load_reduction=108848 max_load_reduction_pct=0.830444 moved_to_echo=64962 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[11648, 128, 22528, 128, 256, 11648, 28416, 512, 4992, 1792, 20608, 4480, 128, 128, 256, 256, 12416, 11904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 128, 24576, 2432, 5632, 1152, 128, 256, 640, 128, 15616, 6400, 128, 19840, 1664, 128, 32512, 20480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[24704, 256, 24320, 5376, 2688, 0, 128, 19584, 128, 0, 128, 24960, 17280, 9600, 128, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[768, 26624, 12928, 2816, 5632, 24320, 13056, 0, 256, 23552, 128, 1664, 128, 768, 17664, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[8576, 256, 1408, 12416, 256, 17408, 512, 128, 24064, 128, 256, 10112, 20480, 4480, 3840, 0, 16768, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[512, 4352, 128, 1152, 3072, 768, 384, 3840, 16000, 11392, 11520, 256, 2560, 30080, 9984, 16000, 11008, 6784, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 29696, 768, 512, 384, 128, 4864, 2944, 6400, 128, 1664, 21248, 8704, 25216, 3072, 26496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[12032, 5120, 32640, 28928, 5376, 2048, 640, 128, 16640, 3840, 5504, 128, 3072, 128, 3456, 12032, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 197, 3859, 4073, 0, 15, 378, 3, 70, 4083, 0, 43, 4093, 3332, 3433, 0, 1238, 407, 0, 3804, 4096, 24, 0, 0, 1311, 56, 2, 523, 0, 0, 298, 3702, 1, 0, 0, 1419, 152, 3, 883, 233, 439, 3840, 501, 4057, 0, 0, 0, 1, 4, 4095, 0, 0, 4095, 0, 4095, 3221, 154, 131, 176, 4068, 34, 344, 0, 0, 113, 505, 71, 4020, 13, 19, 18, 89, 4069, 410, 3999, 4002, 0, 0, 2, 25, 14, 64, 6, 18, 4000, 0, 1090, 4046, 225, 57, 3168, 0, 0, 4078, 0, 3379, 30, 681, 3405, 1582, 3, 0, 209, 269, 0, 0, 0, 235, 0, 0, 4043, 0, 269, 3319, 8, 0, 263, 81, 0, 1, 207, 0, 3952, 0, 3964, 24, 8, 33] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 567, 0, 3266, 0, 3074, 2611, 3810, 2550, 7, 3782, 4078, 3958, 3, 0, 192, 0, 7, 4093, 3832, 7, 3, 599, 53, 4058, 0, 2984, 230, 2, 1339, 4095, 295, 22, 4, 269, 7, 1, 1976, 0, 4, 14, 283, 493, 3460, 6, 0, 0, 0, 3043, 0, 2, 2688, 20, 0, 3784, 382, 0, 7, 4082, 0, 0, 497, 0, 0, 386, 201, 1744, 3320, 0, 587, 0, 0, 422, 5, 4024, 13, 0, 1232, 0, 26, 190, 0, 3996, 3764, 0, 138, 4096, 1941, 0, 54, 0, 93, 368, 347, 117, 0, 30, 4096, 73, 3634, 210, 0, 3563, 0, 0, 1, 25, 8, 5, 2116, 15, 3, 3902, 0, 0, 4090, 10, 0, 104, 1, 27, 4072, 0, 4092, 0, 17, 3375] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4085, 0, 0, 71, 0, 1396, 0, 0, 1499, 406, 4059, 3595, 0, 0, 6, 1, 0, 4092, 4008, 0, 1005, 0, 0, 3968, 96, 731, 118, 0, 241, 2, 5, 0, 0, 1, 8, 4077, 1661, 25, 20, 0, 4, 0, 619, 45, 4, 8, 0, 1877, 157, 0, 2458, 64, 4093, 3998, 7, 0, 17, 3843, 0, 10, 18, 2, 523, 4089, 4091, 0, 3859, 0, 1, 0, 2213, 96, 0, 3417, 0, 0, 0, 96, 477, 0, 292, 3781, 3842, 4, 5, 3737, 0, 0, 1089, 4, 1, 234, 1, 0, 0, 3159, 3890, 0, 4042, 4074, 361, 28, 28, 0, 3507, 43, 10, 0, 3354, 0, 1, 4062, 415, 0, 4083, 4095, 0, 0, 1, 0, 4093, 155, 4096, 7, 3316, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 34, 0, 0, 2586, 0, 146, 3013, 0, 0, 3647, 4096, 3993, 0, 0, 0, 0, 0, 3844, 240, 1155, 124, 227, 0, 38, 1, 0, 10, 0, 0, 800, 0, 0, 0, 1111, 258, 0, 45, 166, 0, 340, 85, 3480, 13, 0, 0, 4096, 67, 1631, 0, 288, 4028, 2, 4012, 2522, 275, 0, 3325, 3710, 1, 1, 1, 0, 200, 3926, 1, 5, 3712, 0, 1908, 0, 0, 0, 3602, 3791, 227, 0, 0, 39, 106, 5, 5, 4096, 4085, 37, 0, 4090, 0, 3977, 4019, 0, 603, 0, 0, 0, 9, 1406, 4030, 1, 2075, 4057, 0, 1706, 0, 7, 0, 942, 2, 282, 3439, 1, 46, 3847, 9, 1, 4093, 3983, 0, 70, 444, 0, 4090, 12, 4094, 0, 0, 581] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4011, 160, 25, 32, 0, 4094, 4088, 4002, 3016, 2626, 4086, 4095, 0, 119, 2192, 0, 0, 36, 4010, 0, 39, 50, 1, 544, 0, 5, 0, 0, 886, 20, 0, 10, 31, 4, 26, 1255, 4034, 66, 0, 789, 0, 234, 1, 0, 2, 50, 17, 258, 58, 872, 3191, 1, 4080, 2809, 0, 0, 0, 4022, 1, 0, 0, 23, 4077, 3223, 1244, 0, 4057, 0, 0, 0, 3903, 0, 1846, 4070, 0, 0, 27, 0, 7, 3, 0, 4096, 4091, 0, 4, 3929, 878, 142, 386, 1, 0, 27, 117, 0, 0, 3540, 4047, 2, 2247, 6, 1, 1215, 0, 6, 3411, 2, 5, 3026, 4074, 0, 0, 4093, 10, 0, 138, 17, 1, 63, 74, 0, 4094, 1, 4041, 5, 589, 265] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 51, 4096, 6, 0, 39, 3853, 0, 3436, 3589, 2422, 4060, 4093, 3100, 1, 0, 4083, 898, 3, 0, 2491, 0, 0, 0, 0, 4079, 59, 0, 100, 3, 28, 47, 451, 0, 12, 2820, 0, 0, 266, 2422, 13, 2647, 2, 0, 45, 0, 0, 0, 0, 4063, 0, 2, 4064, 528, 382, 3744, 3724, 0, 42, 3947, 41, 0, 0, 0, 0, 4095, 737, 6, 3355, 0, 3875, 7, 8, 0, 1344, 4093, 4, 11, 0, 0, 0, 0, 0, 4013, 1, 0, 354, 4055, 392, 0, 101, 0, 0, 1, 1, 4096, 0, 194, 4088, 302, 3994, 3323, 1278, 4092, 0, 11, 0, 1, 0, 0, 27, 0, 0, 2979, 0, 0, 883, 691, 0, 2, 3079, 0, 4095, 1, 121, 0, 1610, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 3954, 3689, 420, 27, 547, 1207, 0, 4095, 1981, 9, 2011, 3677, 4079, 0, 0, 4059, 3754, 90, 23, 3934, 2070, 101, 0, 0, 32, 0, 238, 117, 0, 28, 190, 2, 0, 18, 3962, 0, 0, 0, 0, 0, 1, 1, 4000, 3, 285, 4, 0, 0, 3779, 0, 346, 4090, 6, 4058, 4096, 287, 11, 0, 3997, 13, 0, 95, 0, 2375, 1044, 0, 0, 3698, 0, 417, 0, 4, 0, 0, 641, 307, 6, 38, 33, 0, 0, 0, 4084, 2616, 4, 2324, 4095, 4064, 0, 4052, 0, 0, 2906, 1236, 13, 3, 4060, 4084, 0, 4079, 1, 0, 629, 0, 7, 7, 8, 0, 26, 166, 0, 0, 4058, 31, 21, 2374, 99, 0, 0, 1, 0, 4079, 5, 3991, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 697, 4, 14, 4051, 21, 133, 9, 4047, 4030, 107, 602, 3933, 4093, 1, 0, 4095, 184, 37, 3879, 2053, 0, 0, 0, 0, 3840, 0, 0, 3734, 0, 4096, 0, 0, 0, 1, 118, 330, 0, 0, 0, 1, 0, 0, 10, 40, 0, 60, 5, 0, 0, 0, 660, 4092, 3, 17, 51, 3, 28, 0, 4095, 5, 0, 12, 0, 3763, 891, 0, 40, 2711, 0, 213, 3, 0, 4, 0, 3148, 2, 307, 135, 2899, 0, 459, 0, 3698, 3988, 0, 2828, 3917, 1, 968, 0, 5, 0, 2832, 4053, 4078, 0, 3953, 4095, 0, 4044, 6, 2, 808, 0, 8, 2462, 40, 0, 0, 4009, 5, 151, 3782, 839, 0, 4072, 3, 0, 421, 26, 0, 4086, 30, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4085, 0, 0, 71, 0, 1396, 0, 0, 1499, 406, 0, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 4092, 4008, 0, 1005, 0, 0, 3968, 96, 731, 118, 0, 241, 2, 710, 673, 510, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 8, 4077, 1661, 25, 20, 0, 4, 0, 619, 45, 4, 8, 4059, 3240, 1547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1877, 157, 0, 2458, 64, 4093, 3998, 7, 0, 17, 3133, 0, 10, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 4089, 4091, 0, 3859, 0, 1, 0, 2213, 96, 0, 3417, 0, 0, 0, 96, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 0, 292, 3781, 3842, 4, 5, 1680, 0, 0, 1089, 4, 1, 234, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3159, 3277, 0, 4042, 4074, 361, 28, 28, 0, 3507, 43, 10, 0, 3354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4062, 415, 0, 4083, 4095, 0, 0, 1, 0, 3420, 155, 4096, 7, 3316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7812, 16367, 15323, 15834, 18792, 11410, 21883, 23651] avg=16384.0000 max=23651 max_rank=7 min=7812 min_rank=0 imbalance_ratio=1.443542 max_load_reduction=107421 max_load_reduction_pct=0.819557 moved_to_echo=75736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 567, 0, 3266, 0, 3074, 2611, 3810, 2550, 7, 3782, 0, 391, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 7, 4093, 3832, 7, 3, 599, 53, 4058, 0, 2984, 230, 2, 1339, 4095, 754, 670, 559, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 22, 4, 269, 7, 1, 1976, 0, 4, 14, 283, 493, 3460, 6, 0, 0, 4078, 3567, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3043, 0, 2, 2688, 20, 0, 3784, 382, 0, 7, 3328, 0, 0, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 386, 201, 1744, 3320, 0, 587, 0, 0, 422, 5, 4024, 13, 0, 1232, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 190, 0, 3996, 3764, 0, 138, 1841, 1941, 0, 54, 0, 93, 368, 347, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 3450, 73, 3634, 210, 0, 3563, 0, 0, 1, 25, 8, 5, 2116, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3902, 0, 0, 4090, 10, 0, 104, 1, 27, 3402, 0, 4092, 0, 17, 3375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20061, 23694, 16175, 13751, 12363, 12875, 13130, 19023] avg=16384.0000 max=23694 max_rank=1 min=12363 min_rank=4 imbalance_ratio=1.446167 max_load_reduction=107378 max_load_reduction_pct=0.819229 moved_to_echo=57391 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 197, 3859, 4073, 0, 15, 378, 3, 70, 4083, 0, 43, 0, 329, 3433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1238, 407, 0, 3804, 4096, 24, 0, 0, 1311, 56, 2, 523, 0, 0, 298, 3702, 751, 650, 553, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1419, 152, 3, 883, 233, 439, 3840, 501, 4057, 0, 0, 0, 1, 4093, 3003, 1675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 0, 0, 4095, 0, 4095, 3221, 154, 131, 176, 3317, 34, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 505, 71, 4020, 13, 19, 18, 89, 4069, 410, 3999, 4002, 0, 0, 2, 25, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 64, 6, 18, 4000, 0, 1090, 1818, 225, 57, 3168, 0, 0, 4078, 0, 3379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 681, 2869, 1582, 3, 0, 209, 269, 0, 0, 0, 235, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 3319, 8, 0, 263, 81, 0, 1, 207, 0, 3302, 0, 3964, 24, 8, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16483, 17595, 20300, 19666, 17711, 17917, 9921, 11479] avg=16384.0000 max=20300 max_rank=2 min=9921 min_rank=6 imbalance_ratio=1.239014 max_load_reduction=110772 max_load_reduction_pct=0.845123 moved_to_echo=57028 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4011, 160, 25, 32, 0, 4094, 4088, 4002, 3016, 2626, 0, 404, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2192, 0, 0, 36, 4010, 0, 39, 50, 1, 544, 0, 5, 0, 0, 886, 20, 743, 674, 537, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 31, 4, 26, 1255, 4034, 66, 0, 789, 0, 234, 1, 0, 2, 50, 4086, 3691, 1627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 258, 58, 872, 3191, 1, 4080, 2809, 0, 0, 0, 3279, 1, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3223, 1244, 0, 4057, 0, 0, 0, 3903, 0, 1846, 4070, 0, 0, 27, 0, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 0, 4096, 4091, 0, 4, 1765, 878, 142, 386, 1, 0, 27, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3540, 3409, 2, 2247, 6, 1, 1215, 0, 6, 3411, 2, 5, 3026, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 10, 0, 138, 17, 1, 63, 74, 0, 3420, 1, 4041, 5, 589, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22577, 9951, 15906, 14589, 22871, 11517, 20944, 12717] avg=16384.0000 max=22871 max_rank=4 min=9951 min_rank=1 imbalance_ratio=1.395935 max_load_reduction=108201 max_load_reduction_pct=0.825508 moved_to_echo=68049 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 34, 0, 0, 2586, 0, 146, 3013, 0, 0, 3647, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3844, 240, 1155, 124, 227, 0, 38, 1, 0, 10, 0, 0, 800, 685, 673, 559, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1111, 258, 0, 45, 166, 0, 340, 85, 3480, 13, 0, 0, 4096, 4096, 3599, 1694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 1631, 0, 288, 4028, 2, 4012, 2522, 275, 0, 3325, 3025, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 3926, 1, 5, 3712, 0, 1908, 0, 0, 0, 3602, 3791, 227, 0, 0, 39, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 5, 5, 4096, 4085, 37, 0, 1837, 0, 3977, 4019, 0, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1406, 3395, 1, 2075, 4057, 0, 1706, 0, 7, 0, 942, 2, 282, 3439, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 3847, 9, 1, 4093, 3983, 0, 70, 444, 0, 3417, 12, 4094, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9820, 8569, 18983, 19178, 17833, 18770, 17322, 20597] avg=16384.0000 max=20597 max_rank=7 min=8569 min_rank=1 imbalance_ratio=1.257141 max_load_reduction=110475 max_load_reduction_pct=0.842857 moved_to_echo=74522 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 3954, 3689, 420, 27, 547, 1207, 0, 4095, 1981, 9, 2011, 0, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 3754, 90, 23, 3934, 2070, 101, 0, 0, 32, 0, 238, 117, 0, 28, 190, 738, 671, 559, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 18, 3962, 0, 0, 0, 0, 0, 1, 1, 4000, 3, 285, 4, 0, 3677, 3676, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3779, 0, 346, 4090, 6, 4058, 4096, 287, 11, 0, 3259, 13, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2375, 1044, 0, 0, 3698, 0, 417, 0, 4, 0, 0, 641, 307, 6, 38, 33, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 2616, 4, 2324, 1840, 4064, 0, 4052, 0, 0, 2906, 1236, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4060, 3440, 0, 4079, 1, 0, 629, 0, 7, 7, 8, 0, 26, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 31, 21, 2374, 99, 0, 0, 1, 0, 3408, 5, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18343, 16820, 17325, 20040, 8991, 23139, 12426, 13988] avg=16384.0000 max=23139 max_rank=5 min=8991 min_rank=4 imbalance_ratio=1.412292 max_load_reduction=107933 max_load_reduction_pct=0.823463 moved_to_echo=58544 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 51, 4096, 6, 0, 39, 3853, 0, 3436, 3589, 2422, 4060, 0, 303, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 898, 3, 0, 2491, 0, 0, 0, 0, 4079, 59, 0, 100, 3, 28, 47, 733, 678, 557, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 0, 12, 2820, 0, 0, 266, 2422, 13, 2647, 2, 0, 45, 0, 0, 0, 4093, 2797, 1683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 2, 4064, 528, 382, 3744, 3724, 0, 42, 3214, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 737, 6, 3355, 0, 3875, 7, 8, 0, 1344, 4093, 4, 11, 0, 0, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 1, 0, 354, 1815, 392, 0, 101, 0, 0, 1, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 3438, 302, 3994, 3323, 1278, 4092, 0, 11, 0, 1, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2979, 0, 0, 883, 691, 0, 2, 3079, 0, 3417, 1, 121, 0, 1610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21856, 13978, 17251, 19804, 17966, 10774, 16660, 12783] avg=16384.0000 max=21856 max_rank=0 min=10774 min_rank=5 imbalance_ratio=1.333984 max_load_reduction=109216 max_load_reduction_pct=0.833252 moved_to_echo=58183 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 697, 4, 14, 4051, 21, 133, 9, 4047, 4030, 107, 602, 0, 404, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 184, 37, 3879, 2053, 0, 0, 0, 0, 3840, 0, 0, 3734, 0, 4096, 0, 756, 672, 535, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 118, 330, 0, 0, 0, 1, 0, 0, 10, 40, 0, 60, 5, 3933, 3689, 1622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 4092, 3, 17, 51, 3, 28, 0, 3339, 5, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3763, 891, 0, 40, 2711, 0, 213, 3, 0, 4, 0, 3148, 2, 307, 135, 2899, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 0, 3698, 3988, 0, 2828, 1760, 1, 968, 0, 5, 0, 2832, 4053, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3953, 3449, 0, 4044, 6, 2, 808, 0, 8, 2462, 40, 0, 0, 4009, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 3782, 839, 0, 4072, 3, 0, 421, 26, 0, 3414, 30, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14120, 24098, 9809, 8210, 14545, 24670, 18786, 16834] avg=16384.0000 max=24670 max_rank=5 min=8210 min_rank=3 imbalance_ratio=1.505737 max_load_reduction=106402 max_load_reduction_pct=0.811783 moved_to_echo=74835 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 18816, 256, 2176, 28800, 640, 20864, 24320, 4864, 256, 3584, 25984, 128, 384, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[11136, 18176, 6400, 5888, 24832, 128, 7040, 128, 10240, 1024, 10880, 27264, 640, 384, 1536, 3200, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[768, 128, 128, 9728, 896, 5376, 8960, 2944, 512, 7680, 896, 12288, 4224, 384, 128, 4224, 32128, 27264, 13312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[640, 768, 384, 27904, 26496, 128, 6784, 14464, 7552, 5248, 12928, 128, 768, 10496, 5760, 11776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[15872, 5248, 256, 19840, 24704, 3328, 1280, 896, 1408, 16640, 256, 4608, 4352, 128, 7040, 8960, 5888, 5376, 4480, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[512, 30080, 1408, 128, 20096, 9088, 128, 768, 3840, 128, 27264, 256, 28544, 128, 5632, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 17024, 26752, 2048, 24192, 11776, 1920, 12416, 128, 128, 9472, 1408, 128, 3456, 21248, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 4992, 20352, 4736, 7424, 3328, 8704, 8320, 22656, 20352, 7168, 17280, 0, 3072, 3456, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[23, 4095, 3967, 0, 122, 0, 0, 51, 3778, 0, 0, 4089, 0, 0, 2, 0, 0, 0, 87, 4093, 402, 95, 52, 1, 15, 0, 610, 2, 304, 0, 3186, 82, 1, 3999, 586, 0, 0, 5, 4082, 0, 2, 2480, 700, 0, 9, 58, 4096, 3962, 69, 3949, 0, 4090, 4068, 0, 26, 9, 0, 4, 4067, 0, 3248, 1, 4092, 2497, 0, 0, 2, 35, 82, 0, 3, 18, 1749, 1897, 1210, 306, 0, 552, 165, 2041, 0, 0, 30, 0, 2043, 144, 0, 3766, 0, 0, 108, 3433, 4069, 0, 0, 0, 29, 4086, 1, 2510, 0, 0, 4094, 0, 13, 0, 10, 4027, 430, 0, 16, 4093, 1565, 12, 0, 0, 80, 1, 4094, 0, 405, 3843, 0, 3422, 463, 38, 3671, 1360] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[3044, 536, 3822, 46, 0, 0, 261, 0, 550, 0, 0, 3891, 0, 0, 1, 735, 52, 0, 0, 313, 0, 20, 0, 1733, 1, 0, 4079, 6, 0, 0, 128, 0, 3676, 0, 8, 155, 4004, 0, 4096, 170, 91, 3804, 3879, 4020, 150, 0, 4021, 423, 0, 4047, 1, 3833, 3944, 0, 2, 3736, 0, 6, 0, 268, 3, 0, 2493, 4096, 3, 0, 511, 79, 0, 0, 102, 1, 4005, 9, 3997, 301, 0, 151, 0, 3890, 0, 1, 0, 135, 4025, 300, 0, 52, 0, 276, 0, 2, 3414, 63, 0, 3210, 1190, 1418, 489, 1974, 0, 0, 4067, 1397, 2474, 0, 0, 4095, 0, 0, 0, 4092, 18, 3473, 0, 22, 8, 176, 4078, 0, 0, 3, 0, 4096, 306, 305, 2700, 20] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[3173, 3493, 4074, 0, 0, 559, 0, 0, 118, 0, 51, 3417, 0, 3, 4, 3470, 4016, 47, 28, 3627, 4092, 6, 0, 1681, 19, 0, 4000, 0, 2833, 2120, 4, 0, 0, 4, 7, 1, 41, 0, 4070, 369, 32, 2814, 3980, 823, 0, 257, 210, 0, 1, 4060, 0, 4060, 3837, 0, 188, 4063, 0, 36, 0, 33, 4088, 84, 0, 3643, 428, 0, 2, 0, 0, 4044, 207, 0, 1387, 0, 0, 0, 36, 0, 1, 4091, 1479, 454, 9, 1536, 4085, 2079, 24, 3802, 0, 0, 0, 0, 3889, 0, 528, 0, 82, 40, 0, 4030, 828, 0, 239, 0, 3967, 103, 0, 9, 0, 1534, 1, 10, 2262, 4095, 36, 55, 0, 0, 4096, 0, 1, 0, 128, 1004, 47, 754, 11, 2123] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4091, 3652, 3836, 0, 817, 2, 343, 0, 3331, 1, 0, 2785, 0, 0, 0, 205, 2312, 2, 15, 0, 357, 0, 18, 4095, 3676, 79, 767, 2690, 0, 3980, 772, 0, 0, 3, 158, 7, 44, 0, 4059, 68, 3355, 821, 1966, 366, 404, 2, 0, 3856, 1754, 3836, 0, 41, 3905, 0, 74, 4096, 0, 0, 375, 0, 2466, 0, 0, 515, 59, 0, 0, 596, 3, 32, 0, 0, 0, 257, 45, 3306, 8, 1499, 0, 4093, 0, 0, 2940, 488, 2025, 7, 232, 0, 0, 0, 0, 3998, 3095, 16, 8, 104, 25, 5, 3028, 2491, 0, 0, 3859, 1, 3985, 126, 0, 3764, 2, 0, 0, 4079, 12, 4086, 37, 2, 30, 57, 994, 0, 4015, 476, 1543, 4042, 31, 129, 3, 1442] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[2089, 797, 4095, 0, 3, 562, 4, 0, 3898, 7, 0, 4094, 0, 3808, 0, 3542, 4096, 0, 25, 0, 7, 0, 0, 4079, 2925, 0, 3814, 6, 1590, 3815, 1083, 0, 1, 789, 941, 400, 1442, 0, 4027, 0, 2324, 4071, 0, 0, 0, 1, 259, 0, 44, 4057, 0, 4095, 4094, 0, 48, 4095, 0, 0, 416, 0, 0, 0, 0, 4046, 1521, 669, 0, 414, 1, 1017, 0, 0, 0, 1, 3621, 0, 1635, 58, 0, 149, 0, 898, 61, 1626, 4033, 0, 0, 0, 0, 0, 0, 2130, 3751, 0, 0, 1821, 2, 3696, 0, 347, 0, 0, 4013, 0, 51, 117, 0, 0, 86, 0, 44, 4061, 0, 3319, 472, 0, 2, 9, 4064, 0, 21, 2148, 0, 4085, 1639, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4091, 3795, 4058, 4, 3919, 9, 1, 3, 1369, 99, 0, 0, 0, 0, 139, 0, 38, 0, 0, 0, 256, 34, 1, 4067, 0, 3976, 4096, 7, 3570, 3918, 0, 0, 3845, 0, 3662, 0, 256, 0, 4058, 3485, 0, 3801, 99, 96, 0, 0, 0, 112, 1202, 4096, 1244, 2980, 4059, 0, 39, 3735, 0, 0, 0, 0, 29, 592, 3434, 4092, 9, 975, 12, 3, 12, 3998, 218, 7, 7, 780, 0, 5, 0, 32, 404, 1077, 1, 21, 1518, 555, 4096, 0, 0, 0, 0, 3, 4, 161, 3534, 4, 0, 0, 19, 6, 4076, 3599, 962, 0, 3549, 0, 3, 1, 1, 4090, 0, 2275, 740, 2341, 0, 19, 14, 0, 0, 311, 69, 0, 347, 1, 695, 4061, 4091, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[356, 1919, 3701, 15, 567, 3338, 1923, 26, 4050, 0, 2, 4037, 0, 139, 0, 3745, 4059, 1, 0, 4031, 0, 0, 0, 3908, 1, 3, 4090, 0, 78, 0, 4081, 2, 0, 0, 4056, 3606, 0, 1, 4096, 0, 3749, 3774, 4094, 16, 2, 367, 1, 4090, 0, 1898, 0, 1265, 0, 0, 0, 63, 5, 0, 0, 4, 4070, 0, 0, 4065, 4, 86, 947, 4046, 0, 0, 0, 0, 504, 409, 77, 3882, 0, 222, 82, 44, 2, 0, 552, 3120, 174, 58, 22, 0, 0, 16, 206, 3009, 3689, 0, 0, 1, 68, 1960, 0, 4096, 1, 8, 0, 0, 4004, 4, 20, 1, 0, 0, 2, 2674, 153, 3577, 0, 35, 38, 0, 3952, 52, 37, 146, 2, 4059, 1708, 3, 2, 24] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[3533, 7, 4043, 1, 4, 231, 23, 0, 18, 0, 0, 2863, 0, 9, 0, 3942, 4041, 50, 0, 4059, 22, 0, 0, 0, 0, 0, 0, 2220, 0, 3061, 4089, 10, 0, 4078, 15, 21, 124, 0, 4094, 3828, 1427, 4084, 3978, 0, 52, 0, 851, 4096, 1324, 3552, 3, 4053, 1794, 0, 0, 4060, 133, 0, 0, 0, 164, 0, 3152, 229, 0, 0, 391, 16, 0, 0, 2708, 0, 1053, 206, 0, 967, 0, 78, 0, 2590, 0, 0, 846, 1025, 4061, 0, 3, 3708, 0, 10, 7, 3278, 4086, 3, 0, 0, 1, 773, 31, 2184, 0, 33, 4091, 823, 3880, 0, 0, 668, 175, 0, 0, 3811, 435, 4001, 0, 0, 3, 0, 3936, 0, 11, 4, 0, 3818, 0, 0, 0, 20] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3533, 7, 2221, 1, 4, 231, 23, 0, 18, 0, 0, 2863, 0, 9, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 50, 0, 4059, 22, 0, 0, 0, 0, 0, 0, 2220, 0, 3061, 4089, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 15, 21, 124, 0, 0, 3828, 1427, 3972, 3978, 0, 52, 0, 851, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1324, 0, 3, 4053, 1420, 0, 0, 4060, 133, 0, 0, 0, 164, 0, 3152, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 16, 0, 0, 2708, 0, 1053, 206, 0, 967, 0, 78, 0, 2590, 4094, 2691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 846, 1025, 4061, 0, 3, 3708, 0, 10, 7, 3278, 4086, 3, 0, 0, 1822, 861, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 773, 31, 2184, 0, 33, 4091, 823, 3880, 0, 0, 668, 175, 0, 0, 3050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 4001, 0, 0, 3, 0, 3936, 0, 11, 4, 0, 3818, 0, 0, 0, 20, 761, 0, 254, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12852, 17552, 22442, 14538, 14794, 19830, 15709, 13355] avg=16384.0000 max=22442 max_rank=2 min=12852 min_rank=0 imbalance_ratio=1.369751 max_load_reduction=108630 max_load_reduction_pct=0.828781 moved_to_echo=63688 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2089, 797, 2246, 0, 3, 562, 4, 0, 3898, 7, 0, 4094, 0, 3808, 0, 3542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 25, 0, 7, 0, 0, 4079, 2925, 0, 2986, 6, 1590, 3815, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 789, 941, 400, 1442, 0, 0, 0, 2324, 3955, 0, 0, 0, 1, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 4095, 3233, 0, 48, 4095, 0, 0, 416, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 669, 0, 414, 1, 1017, 0, 0, 0, 1, 3621, 0, 1635, 58, 0, 149, 4027, 3075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 898, 61, 1626, 4033, 0, 0, 0, 0, 0, 0, 2130, 3751, 0, 0, 1821, 1849, 982, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3696, 0, 347, 0, 0, 4013, 0, 51, 117, 0, 0, 86, 0, 44, 3245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3319, 472, 0, 2, 9, 4064, 0, 21, 2148, 0, 4085, 1639, 0, 1, 0, 816, 828, 584, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21050, 20612, 10112, 15977, 16188, 17428, 11601, 18104] avg=16384.0000 max=21050 max_rank=0 min=10112 min_rank=2 imbalance_ratio=1.284790 max_load_reduction=110022 max_load_reduction_pct=0.839401 moved_to_echo=63321 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[23, 4095, 2179, 0, 122, 0, 0, 51, 3778, 0, 0, 4089, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 4093, 402, 95, 52, 1, 15, 0, 479, 2, 304, 0, 3186, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3999, 586, 0, 0, 5, 0, 0, 2, 2412, 700, 0, 9, 58, 4096, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 4090, 3219, 0, 26, 9, 0, 4, 4067, 0, 3248, 1, 4092, 2497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 35, 82, 0, 3, 18, 1749, 1897, 1210, 306, 0, 552, 165, 2041, 4082, 2992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 2043, 144, 0, 3766, 0, 0, 108, 3433, 4069, 0, 0, 0, 1788, 957, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4086, 1, 2510, 0, 0, 4094, 0, 13, 0, 10, 4027, 430, 0, 16, 3276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1565, 12, 0, 0, 80, 1, 4094, 0, 405, 3843, 0, 3422, 463, 38, 3671, 1360, 817, 131, 577, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14339, 8798, 15830, 21322, 15134, 16610, 18492, 20547] avg=16384.0000 max=21322 max_rank=3 min=8798 min_rank=1 imbalance_ratio=1.301392 max_load_reduction=109750 max_load_reduction_pct=0.837326 moved_to_echo=70783 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4091, 3652, 2107, 0, 817, 2, 343, 0, 3331, 1, 0, 2785, 0, 0, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2312, 2, 15, 0, 357, 0, 18, 4095, 3676, 79, 602, 2690, 0, 3980, 772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 158, 7, 44, 0, 0, 68, 3355, 799, 1966, 366, 404, 2, 0, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1754, 0, 0, 41, 3090, 0, 74, 4096, 0, 0, 375, 0, 2466, 0, 0, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 596, 3, 32, 0, 0, 0, 257, 45, 3306, 8, 1499, 0, 4093, 4059, 2907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2940, 488, 2025, 7, 232, 0, 0, 0, 0, 3998, 3095, 16, 8, 104, 1729, 929, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 5, 3028, 2491, 0, 0, 3859, 1, 3985, 126, 0, 3764, 2, 0, 0, 3264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4086, 37, 2, 30, 57, 994, 0, 4015, 476, 1543, 4042, 31, 129, 3, 1442, 815, 165, 554, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17334, 18598, 11028, 12411, 16864, 15832, 20550, 18455] avg=16384.0000 max=20550 max_rank=6 min=11028 min_rank=2 imbalance_ratio=1.254272 max_load_reduction=110522 max_load_reduction_pct=0.843216 moved_to_echo=71701 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3044, 536, 2099, 46, 0, 0, 261, 0, 550, 0, 0, 3891, 0, 0, 1, 735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 313, 0, 20, 0, 1733, 1, 0, 3199, 6, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3676, 0, 8, 155, 4004, 0, 0, 170, 91, 3699, 3879, 4020, 150, 0, 4021, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3833, 3121, 0, 2, 3736, 0, 6, 0, 268, 3, 0, 2493, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 511, 79, 0, 0, 102, 1, 4005, 9, 3997, 301, 0, 151, 0, 3890, 4096, 3066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 135, 4025, 300, 0, 52, 0, 276, 0, 2, 3414, 63, 0, 3210, 1723, 981, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1190, 1418, 489, 1974, 0, 0, 4067, 1397, 2474, 0, 0, 4095, 0, 0, 0, 3275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3473, 0, 22, 8, 176, 4078, 0, 0, 3, 0, 4096, 306, 305, 2700, 20, 817, 880, 559, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11163, 5452, 24296, 17559, 20211, 14446, 20379, 17566] avg=16384.0000 max=24296 max_rank=2 min=5452 min_rank=1 imbalance_ratio=1.482910 max_load_reduction=106776 max_load_reduction_pct=0.814636 moved_to_echo=72602 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3173, 3493, 2238, 0, 0, 559, 0, 0, 118, 0, 51, 3417, 0, 3, 4, 3470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 47, 28, 3627, 4092, 6, 0, 1681, 19, 0, 3137, 0, 2833, 2120, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 1, 41, 0, 0, 369, 32, 2737, 3980, 823, 0, 257, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4060, 3037, 0, 188, 4063, 0, 36, 0, 33, 4088, 84, 0, 3643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 2, 0, 0, 4044, 207, 0, 1387, 0, 0, 0, 36, 0, 1, 4091, 4070, 3076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1479, 454, 9, 1536, 4085, 2079, 24, 3802, 0, 0, 0, 0, 3889, 0, 528, 0, 1836, 984, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 40, 0, 4030, 828, 0, 239, 0, 3967, 103, 0, 9, 0, 1534, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2262, 4095, 36, 55, 0, 0, 4096, 0, 1, 0, 128, 1004, 47, 754, 11, 2123, 1, 863, 544, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16526, 21610, 8461, 19233, 17342, 20961, 10842, 16097] avg=16384.0000 max=21610 max_rank=1 min=8461 min_rank=2 imbalance_ratio=1.318970 max_load_reduction=109462 max_load_reduction_pct=0.835129 moved_to_echo=65242 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[356, 1919, 2033, 15, 567, 3338, 1923, 26, 4050, 0, 2, 4037, 0, 139, 0, 3745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 1, 0, 4031, 0, 0, 0, 3908, 1, 3, 3207, 0, 78, 0, 4081, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 3606, 0, 1, 0, 0, 3749, 3670, 4094, 16, 2, 367, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1265, 0, 0, 0, 63, 5, 0, 0, 4, 4070, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 86, 947, 4046, 0, 0, 0, 0, 504, 409, 77, 3882, 0, 222, 82, 44, 4096, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 552, 3120, 174, 58, 22, 0, 0, 16, 206, 3009, 3689, 0, 0, 1, 1668, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 1960, 0, 4096, 1, 8, 0, 0, 4004, 4, 20, 1, 0, 0, 2, 2140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 3577, 0, 35, 38, 0, 3952, 52, 37, 146, 2, 4059, 1708, 3, 2, 24, 534, 883, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22150, 19371, 23652, 9472, 15838, 12976, 12304, 15309] avg=16384.0000 max=23652 max_rank=2 min=9472 min_rank=3 imbalance_ratio=1.443604 max_load_reduction=107420 max_load_reduction_pct=0.819550 moved_to_echo=56427 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4091, 3795, 2229, 4, 3919, 9, 1, 3, 1369, 99, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 256, 34, 1, 4067, 0, 3976, 3212, 7, 3570, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3845, 0, 3662, 0, 256, 0, 0, 3485, 0, 3696, 99, 96, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1202, 0, 1244, 2980, 3213, 0, 39, 3735, 0, 0, 0, 0, 29, 592, 3434, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 975, 12, 3, 12, 3998, 218, 7, 7, 780, 0, 5, 0, 32, 404, 1077, 4058, 3104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 21, 1518, 555, 4096, 0, 0, 0, 0, 3, 4, 161, 3534, 4, 0, 0, 1829, 992, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 6, 4076, 3599, 962, 0, 3549, 0, 3, 1, 1, 4090, 0, 2275, 740, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 14, 0, 0, 311, 69, 0, 347, 1, 695, 4061, 4091, 0, 0, 0, 467, 884, 575, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15658, 19079, 15251, 20560, 14701, 12989, 21195, 11639] avg=16384.0000 max=21195 max_rank=6 min=11639 min_rank=7 imbalance_ratio=1.293640 max_load_reduction=109877 max_load_reduction_pct=0.838295 moved_to_echo=60524 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[20480, 18304, 17408, 128, 5504, 4736, 2560, 128, 17152, 128, 128, 25216, 0, 3968, 256, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1536, 1408, 6016, 8576, 24576, 2688, 384, 11392, 0, 384, 384, 16128, 29568, 128, 640, 5248, 14336, 7168, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[18688, 128, 256, 16128, 5248, 256, 128, 19584, 6656, 4096, 16896, 4992, 8448, 16896, 13440, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2048, 1792, 1920, 5248, 128, 9216, 3328, 128, 8832, 3584, 8960, 8832, 1792, 2688, 768, 18048, 32640, 22400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[4480, 22656, 640, 128, 256, 640, 25344, 128, 4864, 6656, 2432, 28672, 8320, 1280, 6400, 4992, 5120, 4736, 3712, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[4480, 0, 1280, 24448, 20352, 0, 384, 23936, 256, 128, 4864, 384, 14080, 768, 13184, 23296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1536, 12032, 7680, 21248, 1792, 128, 23936, 2304, 18432, 384, 128, 16768, 768, 3840, 896, 20224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[7552, 8960, 9472, 4224, 6016, 128, 0, 7936, 11008, 24960, 18816, 5376, 640, 768, 9472, 16640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 869, 3141, 255, 0, 27, 0, 3411, 0, 1635, 327, 0, 69, 4, 0, 1353, 1, 1, 1, 0, 6, 4061, 3918, 3259, 71, 3785, 4076, 0, 3642, 0, 1336, 24, 152, 86, 13, 564, 0, 0, 4095, 2146, 345, 547, 836, 4096, 4060, 3930, 5, 271, 0, 197, 19, 3522, 585, 4082, 0, 11, 0, 0, 3970, 0, 0, 145, 1, 55, 0, 39, 0, 996, 3861, 0, 484, 2036, 0, 28, 4062, 261, 3572, 286, 3, 148, 3885, 1, 3783, 263, 242, 0, 115, 22, 2142, 1158, 0, 0, 0, 0, 160, 2873, 0, 2, 369, 0, 1904, 56, 0, 280, 3420, 4066, 14, 1, 4053, 496, 2, 2, 0, 0, 3055, 60, 0, 0, 4071, 39, 1302, 8, 1, 3480, 4078, 0, 888, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 68, 429, 0, 4044, 289, 3659, 6, 1065, 652, 0, 0, 0, 0, 2121, 218, 3776, 0, 2, 0, 4096, 119, 186, 0, 431, 4001, 0, 2467, 0, 3484, 0, 93, 4089, 0, 0, 827, 9, 595, 3762, 3920, 0, 1296, 4093, 0, 4087, 0, 11, 33, 4035, 0, 4001, 0, 4096, 3024, 21, 60, 0, 4096, 6, 0, 4045, 117, 6, 0, 0, 1, 0, 0, 4057, 0, 3789, 0, 286, 0, 82, 3455, 252, 37, 2, 4063, 0, 2556, 4096, 2953, 0, 344, 2, 0, 2068, 0, 3, 51, 0, 1881, 57, 12, 3896, 6, 744, 0, 0, 0, 0, 1519, 4095, 0, 0, 0, 38, 0, 92, 0, 913, 4028, 9, 0, 536, 4063, 0, 4, 0, 0, 3632, 14, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 25, 140, 0, 0, 4073, 26, 4080, 15, 195, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 3806, 0, 4072, 3470, 0, 673, 795, 3913, 2324, 1, 183, 142, 0, 0, 2527, 1, 0, 4060, 6, 158, 3368, 2778, 1696, 12, 4095, 2549, 2420, 0, 0, 0, 82, 10, 3959, 0, 3992, 1051, 0, 834, 4066, 0, 0, 354, 142, 0, 1057, 0, 1577, 6, 0, 0, 0, 599, 4096, 0, 1075, 3, 0, 3547, 0, 3549, 4039, 4083, 191, 3735, 0, 2931, 1, 95, 0, 49, 2, 0, 0, 64, 481, 405, 4074, 1, 0, 0, 191, 0, 4050, 1, 974, 4088, 1693, 233, 0, 14, 2969, 4087, 45, 0, 1, 1, 274, 0, 743, 86, 11, 3883, 0, 443, 721, 0, 0, 703, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[41, 0, 3, 7, 0, 50, 5, 4090, 0, 3878, 0, 3, 0, 0, 117, 0, 3748, 1, 0, 44, 0, 0, 3896, 2869, 0, 4052, 2592, 0, 4095, 40, 0, 0, 0, 3523, 0, 133, 4025, 0, 168, 0, 175, 4, 0, 0, 750, 4092, 0, 4, 0, 44, 0, 4084, 0, 148, 4071, 90, 3834, 0, 1614, 14, 23, 198, 4096, 0, 0, 0, 0, 430, 920, 65, 0, 4090, 0, 0, 3125, 0, 4093, 0, 3295, 1556, 4055, 1, 4087, 3, 3890, 168, 4, 0, 4094, 3733, 3566, 0, 289, 3992, 6, 4096, 0, 0, 186, 0, 104, 288, 0, 0, 149, 4019, 0, 238, 1, 3840, 132, 0, 4, 4, 1199, 4096, 0, 0, 4094, 0, 0, 0, 0, 2384, 154, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[2, 1, 3101, 824, 0, 3960, 19, 2490, 0, 4080, 0, 22, 0, 0, 27, 0, 0, 4, 0, 2168, 0, 3614, 823, 4067, 0, 3064, 607, 132, 1994, 195, 0, 0, 0, 4058, 6, 2282, 89, 0, 4096, 2699, 13, 7, 1692, 3868, 794, 4074, 0, 609, 41, 1, 0, 3588, 0, 1586, 1262, 14, 1, 577, 3918, 56, 120, 242, 4096, 0, 0, 0, 0, 39, 113, 90, 0, 2700, 10, 0, 4084, 2, 91, 26, 1229, 1517, 3966, 188, 3602, 4096, 4096, 0, 26, 0, 0, 20, 0, 0, 633, 2007, 33, 154, 0, 0, 0, 0, 778, 13, 0, 0, 4042, 3321, 8, 0, 0, 3981, 3332, 2552, 73, 1049, 940, 4096, 0, 0, 3313, 0, 0, 0, 0, 3715, 154, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[258, 0, 3882, 0, 0, 3, 0, 4030, 0, 3805, 4, 0, 4, 1, 0, 0, 47, 620, 0, 0, 291, 16, 3405, 0, 0, 4094, 3946, 0, 570, 55, 756, 520, 3844, 0, 149, 3962, 3780, 0, 4053, 2139, 399, 1, 0, 4009, 4, 4096, 19, 429, 88, 4030, 40, 2720, 567, 0, 4056, 111, 402, 0, 4053, 348, 0, 0, 1435, 0, 148, 38, 0, 1138, 110, 0, 0, 1622, 0, 3, 3028, 3928, 0, 1, 1420, 0, 4093, 0, 3753, 0, 3404, 0, 982, 11, 2235, 5, 0, 268, 2, 4090, 129, 116, 0, 0, 0, 0, 3642, 0, 1, 14, 930, 4078, 0, 2435, 29, 3651, 1026, 10, 2, 0, 4095, 1389, 2, 0, 4012, 2, 10, 18, 1, 4096, 0, 63, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1, 481, 4086, 313, 3, 3540, 37, 4050, 0, 4065, 1050, 0, 3882, 0, 68, 859, 0, 3951, 0, 0, 2, 4055, 1581, 4069, 1, 3495, 3968, 0, 3307, 0, 3295, 0, 3, 169, 1, 5, 0, 31, 1, 355, 103, 26, 3, 4096, 3599, 3814, 0, 137, 0, 3, 0, 3653, 0, 3269, 1670, 0, 3, 0, 4057, 0, 246, 0, 0, 0, 0, 0, 0, 106, 0, 72, 449, 4031, 1, 0, 18, 419, 739, 197, 0, 72, 4096, 24, 4069, 3663, 4095, 577, 2, 0, 744, 0, 0, 50, 0, 3302, 726, 3939, 0, 3902, 0, 78, 98, 7, 0, 0, 3644, 3285, 27, 16, 0, 11, 796, 63, 14, 222, 3383, 354, 0, 0, 4096, 70, 3467, 0, 11, 731, 0, 0, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[2, 0, 4095, 26, 1, 4065, 118, 3826, 0, 4096, 0, 0, 0, 0, 0, 937, 11, 2386, 0, 2, 32, 2152, 82, 2892, 68, 3252, 4051, 0, 6, 0, 3764, 0, 418, 1803, 118, 300, 0, 1, 1, 3767, 1, 0, 770, 3448, 4084, 3348, 0, 0, 0, 0, 0, 276, 0, 4054, 1372, 0, 28, 2, 114, 0, 2, 3109, 2950, 0, 0, 0, 6, 4, 3, 0, 0, 4027, 7, 1938, 3237, 1876, 3706, 1728, 3457, 972, 4083, 0, 2966, 64, 12, 199, 6, 94, 0, 0, 0, 24, 0, 778, 10, 3935, 0, 0, 0, 812, 0, 888, 0, 3, 4045, 4020, 2020, 3971, 2, 4054, 0, 0, 33, 120, 0, 2181, 0, 0, 4067, 44, 2851, 14, 0, 2914, 21, 50, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 869, 3141, 255, 0, 27, 0, 3411, 0, 1635, 327, 0, 69, 4, 0, 1353, 3104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 6, 4061, 3918, 3259, 71, 3315, 0, 0, 3642, 0, 1336, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 86, 13, 564, 0, 0, 4095, 2146, 345, 547, 836, 3932, 4060, 0, 5, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 19, 3522, 585, 4082, 0, 11, 0, 0, 3970, 0, 0, 145, 1, 55, 314, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 996, 3861, 0, 484, 2036, 0, 28, 4062, 261, 3572, 286, 3, 148, 1555, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 868, 1, 3783, 263, 242, 0, 115, 22, 2142, 1158, 0, 0, 0, 0, 160, 2873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 369, 0, 1904, 56, 0, 280, 3420, 4066, 14, 1, 4053, 496, 2, 2, 1462, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3055, 60, 0, 0, 4071, 39, 1302, 8, 1, 3480, 4078, 0, 888, 0, 3930, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14195, 19635, 17052, 13065, 18221, 11627, 16283, 20994] avg=16384.0000 max=20994 max_rank=7 min=11627 min_rank=5 imbalance_ratio=1.281372 max_load_reduction=110078 max_load_reduction_pct=0.839828 moved_to_echo=67125 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 25, 140, 0, 0, 4073, 26, 4080, 15, 195, 0, 0, 4096, 0, 0, 8, 2980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3806, 0, 4072, 3470, 0, 673, 697, 0, 2324, 1, 183, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2527, 1, 0, 4060, 6, 158, 3368, 2778, 1696, 12, 3931, 2549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 10, 3959, 0, 3992, 1051, 0, 834, 4066, 0, 0, 354, 142, 0, 1057, 66, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 6, 0, 0, 0, 599, 4096, 0, 1075, 3, 0, 3547, 0, 3549, 4039, 1634, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 913, 191, 3735, 0, 2931, 1, 95, 0, 49, 2, 0, 0, 64, 481, 405, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 191, 0, 4050, 1, 974, 4088, 1693, 233, 0, 14, 2969, 4087, 45, 1536, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 274, 0, 743, 86, 11, 3883, 0, 443, 721, 0, 0, 703, 4, 2420, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15638, 15368, 21086, 15777, 20980, 12941, 19914, 9368] avg=16384.0000 max=21086 max_rank=2 min=9368 min_rank=7 imbalance_ratio=1.286987 max_load_reduction=109986 max_load_reduction_pct=0.839127 moved_to_echo=63203 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[258, 0, 3882, 0, 0, 3, 0, 4030, 0, 3805, 4, 0, 4, 1, 0, 0, 3005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 620, 0, 0, 291, 16, 3405, 0, 0, 3585, 0, 0, 570, 55, 756, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3844, 0, 149, 3962, 3780, 0, 4053, 2139, 399, 1, 0, 3849, 4, 0, 19, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 4030, 40, 2720, 567, 0, 4056, 111, 402, 0, 4053, 348, 0, 0, 1435, 0, 340, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 38, 0, 1138, 110, 0, 0, 1622, 0, 3, 3028, 3928, 0, 1, 1420, 0, 1638, 861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 915, 0, 3753, 0, 3404, 0, 982, 11, 2235, 5, 0, 268, 2, 4090, 129, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3642, 0, 1, 14, 930, 4078, 0, 2435, 29, 3651, 1026, 10, 1540, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 1389, 2, 0, 4012, 2, 10, 18, 1, 4096, 0, 63, 1, 0, 4096, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14992, 9865, 22628, 18350, 13935, 15910, 17525, 17867] avg=16384.0000 max=22628 max_rank=2 min=9865 min_rank=1 imbalance_ratio=1.381104 max_load_reduction=108444 max_load_reduction_pct=0.827362 moved_to_echo=65237 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 481, 4086, 313, 3, 3540, 37, 4050, 0, 4065, 1050, 0, 3882, 0, 68, 859, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 0, 0, 2, 4055, 1581, 4069, 1, 3061, 0, 0, 3307, 0, 3295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 169, 1, 5, 0, 31, 1, 355, 103, 26, 3, 3932, 3599, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3653, 0, 3269, 1670, 0, 3, 0, 4057, 0, 246, 0, 0, 0, 290, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 72, 449, 4031, 1, 0, 18, 419, 739, 197, 0, 72, 1639, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 916, 24, 4069, 3663, 4095, 577, 2, 0, 744, 0, 0, 50, 0, 3302, 726, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 78, 98, 7, 0, 0, 3644, 3285, 27, 16, 0, 11, 796, 63, 1541, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 222, 3383, 354, 0, 0, 4096, 70, 3467, 0, 11, 731, 0, 0, 0, 3, 3814, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[25457, 23322, 8365, 13355, 8609, 22107, 13612, 16245] avg=16384.0000 max=25457 max_rank=0 min=8365 min_rank=2 imbalance_ratio=1.553772 max_load_reduction=105615 max_load_reduction_pct=0.805779 moved_to_echo=60573 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 68, 429, 0, 4044, 289, 3659, 6, 1065, 652, 0, 0, 0, 0, 2121, 3047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 3776, 0, 2, 0, 4096, 119, 186, 0, 379, 0, 0, 2467, 0, 3484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 4089, 0, 0, 827, 9, 595, 3762, 3920, 0, 1296, 3929, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4035, 0, 4001, 0, 4096, 3024, 21, 60, 0, 4096, 6, 0, 4045, 117, 6, 35, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4057, 0, 3789, 0, 286, 0, 82, 3455, 252, 37, 2, 1626, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 908, 0, 2556, 4096, 2953, 0, 344, 2, 0, 2068, 0, 3, 51, 0, 1881, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3896, 6, 744, 0, 0, 0, 0, 1519, 4095, 0, 0, 0, 38, 0, 92, 1529, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 913, 4028, 9, 0, 536, 4063, 0, 4, 0, 0, 3632, 14, 0, 0, 0, 4087, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15381, 14727, 18531, 23739, 14460, 14919, 11948, 17367] avg=16384.0000 max=23739 max_rank=3 min=11948 min_rank=6 imbalance_ratio=1.448914 max_load_reduction=107333 max_load_reduction_pct=0.818886 moved_to_echo=58694 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4095, 26, 1, 4065, 118, 3826, 0, 4096, 0, 0, 0, 0, 0, 937, 3085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2386, 0, 2, 32, 2152, 82, 2892, 68, 2848, 0, 0, 6, 0, 3764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 1803, 118, 300, 0, 1, 1, 3767, 1, 0, 770, 3310, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 0, 4054, 1372, 0, 28, 2, 114, 0, 2, 3109, 2950, 0, 270, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 3, 0, 0, 4027, 7, 1938, 3237, 1876, 3706, 1728, 3457, 972, 1634, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 913, 0, 2966, 64, 12, 199, 6, 94, 0, 0, 0, 24, 0, 778, 10, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 812, 0, 888, 0, 3, 4045, 4020, 2020, 3971, 2, 4054, 0, 0, 1536, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 120, 0, 2181, 0, 0, 4067, 44, 2851, 14, 0, 2914, 21, 50, 0, 0, 3348, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20251, 14243, 14573, 12315, 23479, 9001, 21485, 15725] avg=16384.0000 max=23479 max_rank=4 min=9001 min_rank=5 imbalance_ratio=1.433044 max_load_reduction=107593 max_load_reduction_pct=0.820869 moved_to_echo=69690 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 3101, 824, 0, 3960, 19, 2490, 0, 4080, 0, 22, 0, 0, 27, 0, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2168, 0, 3614, 823, 4067, 0, 2684, 0, 132, 1994, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 6, 2282, 89, 0, 4096, 2699, 13, 7, 1692, 3710, 794, 0, 0, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 1, 0, 3588, 0, 1586, 1262, 14, 1, 577, 3918, 56, 120, 242, 4096, 0, 254, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 113, 90, 0, 2700, 10, 0, 4084, 2, 91, 26, 1229, 1517, 1587, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 887, 188, 3602, 4096, 4096, 0, 26, 0, 0, 20, 0, 0, 633, 2007, 33, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 778, 13, 0, 0, 4042, 3321, 8, 0, 0, 3981, 3332, 2552, 1492, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 1049, 940, 4096, 0, 0, 3313, 0, 0, 0, 0, 3715, 154, 0, 0, 0, 4074, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14989, 15681, 20055, 15914, 11620, 15742, 19645, 17426] avg=16384.0000 max=20055 max_rank=2 min=11620 min_rank=4 imbalance_ratio=1.224060 max_load_reduction=111017 max_load_reduction_pct=0.846992 moved_to_echo=64433 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[41, 0, 3, 7, 0, 50, 5, 4090, 0, 3878, 0, 3, 0, 0, 117, 0, 1975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 1, 0, 44, 0, 0, 3896, 2869, 0, 3538, 0, 0, 4095, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3523, 0, 133, 4025, 0, 168, 0, 175, 4, 0, 0, 750, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 4084, 0, 148, 4071, 90, 3834, 0, 1614, 14, 23, 198, 4096, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 430, 920, 65, 0, 4090, 0, 0, 3125, 0, 4093, 0, 3295, 1556, 1629, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 896, 1, 4087, 3, 3890, 168, 4, 0, 4094, 3733, 3566, 0, 289, 3992, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 104, 288, 0, 0, 149, 4019, 0, 238, 1, 3840, 132, 0, 1530, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 1199, 4096, 0, 0, 4094, 0, 0, 0, 0, 2384, 154, 0, 1, 0, 4092, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10169, 18231, 8782, 18557, 19768, 28825, 10660, 16080] avg=16384.0000 max=28825 max_rank=5 min=8782 min_rank=2 imbalance_ratio=1.759338 max_load_reduction=102247 max_load_reduction_pct=0.780083 moved_to_echo=75333 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[4096, 10752, 128, 6144, 384, 22144, 17408, 17408, 896, 20224, 0, 2560, 16128, 512, 12800, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 2432, 16768, 12544, 128, 1280, 27904, 256, 11520, 128, 512, 21760, 4480, 128, 1664, 128, 29952, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[7296, 512, 28672, 12288, 21632, 1024, 1664, 256, 9344, 7040, 3584, 384, 1152, 14720, 3456, 19328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[4608, 16256, 384, 7296, 12800, 128, 13184, 18304, 7808, 2304, 4736, 26624, 15872, 0, 128, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[256, 8448, 128, 25856, 1152, 21248, 16512, 256, 5248, 4736, 21888, 512, 768, 7936, 12800, 1152, 1920, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 7808, 640, 1920, 6528, 5376, 128, 1280, 21888, 28672, 2304, 6784, 4224, 19072, 9472, 2816, 12288, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 1408, 18560, 1920, 128, 19840, 512, 29696, 128, 22912, 2048, 128, 8064, 128, 256, 5376, 20736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[256, 1664, 128, 2816, 5120, 4352, 1536, 26496, 128, 3456, 17664, 6656, 19328, 2560, 13056, 8320, 13056, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[2526, 0, 0, 0, 21, 0, 4, 35, 4067, 0, 0, 3, 204, 4011, 0, 4085, 3701, 2, 19, 15, 0, 0, 6, 0, 0, 0, 3853, 2, 0, 9, 0, 4095, 0, 0, 749, 469, 0, 4093, 237, 4036, 1386, 25, 4058, 0, 1, 140, 0, 0, 0, 0, 18, 3696, 0, 0, 1396, 0, 4095, 2292, 4096, 4089, 6, 3364, 0, 60, 4037, 43, 3373, 0, 0, 19, 0, 259, 0, 0, 58, 3682, 1, 0, 0, 0, 0, 191, 1, 0, 1928, 4091, 0, 4068, 0, 17, 71, 88, 3874, 0, 0, 1128, 3, 3005, 1505, 448, 1, 4003, 742, 0, 114, 0, 0, 962, 3862, 0, 271, 0, 520, 3445, 4047, 0, 4096, 13, 0, 0, 82, 0, 3976, 65, 3927, 4092, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[58, 0, 22, 1, 0, 2, 0, 0, 4086, 0, 0, 1556, 1, 121, 1, 3722, 4096, 64, 0, 0, 1345, 0, 131, 0, 0, 0, 37, 3187, 2, 0, 1, 4096, 9, 0, 0, 0, 116, 4095, 0, 4096, 4054, 0, 0, 0, 0, 38, 4051, 0, 0, 45, 5, 3821, 0, 0, 1393, 0, 4072, 3346, 4068, 3538, 2, 4087, 6, 975, 4096, 41, 0, 92, 0, 0, 1, 1, 0, 0, 0, 3720, 0, 0, 4039, 25, 52, 0, 1037, 0, 125, 3422, 4075, 399, 0, 0, 0, 4012, 0, 0, 177, 0, 3, 4025, 0, 4093, 14, 4001, 0, 1669, 518, 0, 0, 241, 3912, 27, 22, 0, 11, 1988, 0, 8, 1648, 0, 3499, 382, 378, 2798, 3980, 292, 4002, 3901, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[3989, 0, 51, 3815, 0, 0, 0, 0, 420, 19, 0, 0, 0, 8, 0, 3485, 3379, 26, 6, 0, 0, 0, 0, 65, 0, 1920, 8, 4096, 0, 4089, 0, 4096, 0, 0, 0, 28, 0, 3742, 1, 0, 4001, 3945, 0, 0, 0, 4092, 213, 0, 1, 0, 37, 4096, 0, 1, 618, 4, 32, 3928, 4096, 4096, 0, 0, 0, 0, 0, 1, 1, 2656, 4096, 594, 13, 2882, 479, 0, 5, 3851, 1, 0, 3790, 0, 40, 0, 0, 0, 4096, 1694, 57, 217, 272, 0, 0, 4078, 64, 81, 2003, 4079, 4, 4086, 72, 4096, 0, 3427, 0, 5, 0, 1117, 0, 0, 2442, 0, 66, 50, 0, 4091, 1368, 1, 4090, 4, 0, 467, 3, 0, 4062, 0, 3051, 1117, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4073, 0, 104, 0, 0, 111, 8, 36, 412, 0, 0, 3714, 0, 1346, 0, 3459, 4023, 65, 1, 53, 0, 0, 136, 4096, 32, 8, 0, 4015, 0, 4087, 0, 3871, 0, 2, 4016, 3166, 0, 19, 0, 0, 3942, 1350, 0, 12, 0, 2155, 0, 0, 266, 3184, 33, 4096, 0, 174, 14, 27, 117, 2873, 1863, 4083, 5, 572, 0, 81, 863, 1, 4045, 0, 718, 29, 14, 0, 318, 1642, 10, 622, 0, 0, 0, 0, 0, 0, 0, 345, 3755, 4096, 1067, 0, 0, 0, 0, 4022, 0, 0, 166, 0, 25, 3998, 0, 3787, 44, 0, 4034, 0, 265, 3830, 7, 0, 3930, 0, 0, 3819, 3, 60, 875, 0, 3945, 1900, 4085, 1901, 0, 0, 0, 79, 4090, 2707, 112, 163] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3, 0, 0, 2798, 81, 3923, 32, 0, 4091, 92, 0, 146, 0, 4072, 196, 4096, 568, 6, 0, 0, 1526, 0, 3200, 0, 0, 0, 3989, 3105, 159, 39, 0, 4004, 189, 0, 71, 1537, 152, 3694, 0, 4084, 3958, 66, 199, 0, 6, 0, 0, 2, 0, 599, 2513, 3367, 0, 443, 4096, 0, 1, 4090, 4077, 1824, 0, 2645, 0, 0, 4088, 10, 88, 2588, 132, 0, 54, 3616, 0, 0, 0, 3490, 0, 2, 0, 0, 170, 224, 323, 0, 0, 40, 0, 1172, 0, 359, 0, 2401, 3026, 1, 0, 807, 4078, 4093, 3974, 1616, 1, 4092, 0, 0, 101, 73, 896, 470, 0, 0, 0, 0, 2, 0, 3024, 137, 3913, 0, 0, 6, 0, 78, 0, 0, 4074, 4091, 23, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 24, 0, 0, 18, 0, 4096, 14, 0, 4, 0, 0, 4043, 2460, 3976, 0, 0, 0, 1596, 0, 0, 123, 0, 13, 3836, 2762, 848, 0, 376, 3731, 0, 0, 378, 0, 0, 4096, 98, 4022, 3940, 0, 336, 377, 0, 1, 1710, 0, 0, 37, 7, 3908, 1, 0, 4, 4074, 3104, 4090, 4096, 2006, 0, 42, 0, 0, 4094, 0, 2, 34, 0, 0, 4, 3827, 0, 26, 0, 4091, 5, 0, 1555, 171, 72, 1, 137, 0, 784, 3437, 2483, 1585, 0, 0, 1, 4034, 0, 0, 0, 235, 3476, 4013, 3044, 998, 120, 0, 0, 0, 0, 2429, 3681, 254, 0, 7, 105, 0, 23, 2493, 952, 0, 6, 47, 4061, 430, 181, 0, 1834, 3900, 4069, 4057, 0, 67] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[3985, 0, 0, 33, 0, 0, 0, 0, 0, 29, 0, 0, 0, 4095, 0, 4093, 1502, 0, 3, 0, 0, 0, 0, 322, 0, 2183, 4095, 3791, 0, 114, 0, 190, 0, 0, 105, 269, 3376, 3800, 0, 1, 4096, 1269, 0, 0, 0, 3, 0, 0, 109, 0, 756, 4096, 0, 7, 103, 159, 3308, 4092, 4081, 4080, 0, 0, 2, 0, 1979, 4, 4093, 316, 3456, 120, 438, 0, 0, 0, 0, 4096, 98, 0, 1, 1, 3974, 11, 2033, 1, 3539, 3129, 1302, 3886, 704, 63, 0, 2078, 202, 5, 87, 143, 2, 2950, 2, 50, 0, 0, 111, 0, 0, 0, 1, 0, 4096, 0, 0, 532, 2225, 4089, 2367, 0, 4059, 0, 4051, 0, 1, 0, 4068, 587, 4056, 3919, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[3999, 0, 2, 0, 3, 0, 0, 6, 26, 0, 2029, 0, 0, 2708, 106, 3952, 3784, 0, 1, 0, 0, 0, 0, 238, 1, 1177, 1, 2624, 28, 0, 1, 4096, 5, 0, 0, 143, 9, 4096, 8, 1, 3899, 4086, 1339, 178, 31, 4095, 3574, 0, 6, 0, 0, 4072, 7, 0, 468, 123, 2472, 0, 0, 3271, 16, 857, 0, 0, 2704, 0, 3772, 0, 352, 0, 27, 0, 0, 734, 0, 4032, 0, 0, 0, 0, 4015, 0, 26, 60, 3, 3954, 0, 92, 3729, 0, 188, 4028, 344, 6, 0, 0, 4095, 4095, 15, 3894, 0, 4023, 3977, 0, 1368, 5, 0, 0, 4042, 62, 81, 1797, 1, 3888, 38, 62, 4096, 0, 3847, 0, 0, 167, 1813, 0, 0, 12, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 24, 0, 0, 18, 0, 4096, 14, 0, 4, 0, 0, 4043, 2460, 3624, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 0, 0, 0, 1596, 0, 0, 123, 0, 13, 3836, 2762, 848, 0, 376, 3731, 1, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 4096, 98, 4022, 3940, 0, 336, 377, 0, 1, 1710, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 7, 296, 1, 0, 4, 4074, 3104, 4090, 4096, 2006, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 34, 0, 0, 4, 3827, 0, 26, 0, 4091, 5, 0, 1555, 171, 3368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 1, 137, 0, 784, 3437, 2483, 1585, 0, 0, 1, 4034, 0, 0, 0, 235, 1367, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3476, 2132, 3044, 998, 120, 0, 0, 0, 0, 2429, 3681, 254, 0, 7, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2493, 952, 0, 4, 47, 4061, 430, 181, 0, 1834, 3900, 0, 4057, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14527, 17776, 14959, 17757, 17177, 14581, 16246, 18049] avg=16384.0000 max=18049 max_rank=7 min=14527 min_rank=0 imbalance_ratio=1.101624 max_load_reduction=113023 max_load_reduction_pct=0.862297 moved_to_echo=66053 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3985, 0, 0, 33, 0, 0, 0, 0, 0, 29, 0, 0, 0, 4095, 0, 4093, 3614, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1502, 0, 3, 0, 0, 0, 0, 322, 0, 2183, 4095, 3791, 0, 114, 0, 190, 1015, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 269, 3376, 3800, 0, 1, 4096, 1269, 0, 0, 0, 3, 0, 0, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 756, 301, 0, 7, 103, 159, 3308, 4092, 4081, 4080, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1979, 4, 4093, 316, 3456, 120, 438, 0, 0, 0, 0, 4096, 98, 0, 1, 1, 3535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 11, 2033, 1, 3539, 3129, 1302, 3886, 704, 63, 0, 2078, 202, 5, 87, 143, 1008, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1560, 2, 50, 0, 0, 111, 0, 0, 0, 1, 0, 4096, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2225, 4089, 2367, 0, 2187, 0, 4051, 0, 1, 0, 4068, 587, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16109, 13597, 13776, 16998, 18137, 22607, 6354, 23494] avg=16384.0000 max=23494 max_rank=7 min=6354 min_rank=6 imbalance_ratio=1.433960 max_load_reduction=107578 max_load_reduction_pct=0.820755 moved_to_echo=70592 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3999, 0, 2, 0, 3, 0, 0, 6, 26, 0, 2029, 0, 0, 2708, 106, 3952, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3784, 0, 1, 0, 0, 0, 0, 238, 1, 1177, 1, 2624, 28, 0, 1, 4096, 1022, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 143, 9, 4096, 8, 1, 3899, 4086, 1339, 178, 31, 4095, 3574, 0, 861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 308, 7, 0, 468, 123, 2472, 0, 0, 3271, 16, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2704, 0, 3772, 0, 352, 0, 27, 0, 0, 734, 0, 4032, 0, 0, 0, 0, 3510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 26, 60, 3, 3954, 0, 92, 3729, 0, 188, 4028, 344, 6, 0, 0, 1395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2176, 15, 3894, 0, 4023, 3977, 0, 1368, 5, 0, 0, 4042, 62, 81, 1797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3888, 38, 62, 2213, 0, 3847, 0, 0, 167, 1813, 0, 0, 12, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13085, 13497, 22325, 7528, 15131, 17840, 25535, 16131] avg=16384.0000 max=25535 max_rank=6 min=7528 min_rank=3 imbalance_ratio=1.558533 max_load_reduction=105537 max_load_reduction_pct=0.805183 moved_to_echo=74637 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 2798, 81, 3923, 32, 0, 4091, 92, 0, 146, 0, 4072, 196, 4096, 3629, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 6, 0, 0, 1526, 0, 3200, 0, 0, 0, 3989, 3105, 159, 39, 0, 4004, 976, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 71, 1537, 152, 3694, 0, 4084, 3958, 66, 199, 0, 6, 0, 0, 2, 822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 599, 2513, 255, 0, 443, 4096, 0, 1, 4090, 4077, 1824, 0, 2645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 10, 88, 2588, 132, 0, 54, 3616, 0, 0, 0, 3490, 0, 2, 0, 0, 2902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 224, 323, 0, 0, 40, 0, 1172, 0, 359, 0, 2401, 3026, 1, 0, 807, 1394, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 2175, 3974, 1616, 1, 4092, 0, 0, 101, 73, 896, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3024, 137, 2115, 0, 0, 6, 0, 78, 0, 0, 0, 4091, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23369, 18096, 14780, 20543, 16970, 10362, 17476, 9476] avg=16384.0000 max=23369 max_rank=0 min=9476 min_rank=7 imbalance_ratio=1.426331 max_load_reduction=107703 max_load_reduction_pct=0.821709 moved_to_echo=54284 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2526, 0, 0, 0, 21, 0, 4, 35, 4067, 0, 0, 3, 204, 4011, 0, 4085, 3498, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3701, 2, 19, 15, 0, 0, 6, 0, 0, 0, 3853, 2, 0, 9, 0, 4095, 1022, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 749, 469, 0, 4093, 237, 4036, 1386, 25, 4058, 0, 1, 140, 0, 0, 861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 279, 0, 0, 1396, 0, 4095, 2292, 4096, 4089, 6, 3364, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 43, 3373, 0, 0, 19, 0, 259, 0, 0, 58, 3682, 1, 0, 0, 0, 3186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 1, 0, 1928, 4091, 0, 4068, 0, 17, 71, 88, 3874, 0, 0, 1128, 1023, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1597, 1505, 448, 1, 4003, 742, 0, 114, 0, 0, 962, 3862, 0, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 3445, 4047, 0, 2213, 13, 0, 0, 82, 0, 3976, 65, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18685, 13109, 16055, 19695, 14658, 16909, 13508, 18453] avg=16384.0000 max=19695 max_rank=3 min=13109 min_rank=1 imbalance_ratio=1.202087 max_load_reduction=111377 max_load_reduction_pct=0.849739 moved_to_echo=63528 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4073, 0, 104, 0, 0, 111, 8, 36, 412, 0, 0, 3714, 0, 1346, 0, 3459, 3644, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 65, 1, 53, 0, 0, 136, 4096, 32, 8, 0, 4015, 0, 4087, 0, 3871, 984, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4016, 3166, 0, 19, 0, 0, 3942, 1350, 0, 12, 0, 2155, 0, 0, 829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 3184, 33, 310, 0, 174, 14, 27, 117, 2873, 1863, 4083, 5, 572, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 863, 1, 4045, 0, 718, 29, 14, 0, 318, 1642, 10, 622, 0, 0, 0, 0, 3530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 3755, 4096, 1067, 0, 0, 0, 0, 4022, 0, 0, 166, 0, 1362, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2124, 0, 3787, 44, 0, 4034, 0, 265, 3830, 7, 0, 3930, 0, 0, 3819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 60, 875, 0, 2132, 1900, 4085, 1901, 0, 0, 0, 79, 0, 2707, 112, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17163, 21883, 15491, 13602, 11792, 15259, 21865, 14017] avg=16384.0000 max=21883 max_rank=1 min=11792 min_rank=4 imbalance_ratio=1.335632 max_load_reduction=109189 max_load_reduction_pct=0.833046 moved_to_echo=62933 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3989, 0, 51, 3815, 0, 0, 0, 0, 420, 19, 0, 0, 0, 8, 0, 3485, 2718, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3379, 26, 6, 0, 0, 0, 0, 65, 0, 1920, 8, 4096, 0, 4089, 0, 4096, 1020, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 3742, 1, 0, 4001, 3945, 0, 0, 0, 4092, 213, 0, 859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 37, 310, 0, 1, 618, 4, 32, 3928, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2656, 4096, 594, 13, 2882, 479, 0, 5, 3851, 1, 0, 3790, 0, 3530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 4096, 1694, 57, 217, 272, 0, 0, 4078, 64, 81, 2003, 4079, 1392, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2171, 72, 4096, 0, 3427, 0, 5, 0, 1117, 0, 0, 2442, 0, 66, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1368, 1, 2211, 4, 0, 467, 3, 0, 4062, 0, 0, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14761, 19228, 16881, 13123, 21899, 18406, 13450, 13324] avg=16384.0000 max=21899 max_rank=4 min=13123 min_rank=3 imbalance_ratio=1.336609 max_load_reduction=109173 max_load_reduction_pct=0.832924 moved_to_echo=67079 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[58, 0, 22, 1, 0, 2, 0, 0, 4086, 0, 0, 1556, 1, 121, 1, 3722, 3564, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 64, 0, 0, 1345, 0, 131, 0, 0, 0, 37, 3187, 2, 0, 1, 4096, 411, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 116, 4095, 0, 4096, 4054, 0, 0, 0, 0, 38, 4051, 0, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 5, 289, 0, 0, 1393, 0, 4072, 3346, 4068, 3538, 2, 4087, 6, 975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 41, 0, 92, 0, 0, 1, 1, 0, 0, 0, 3720, 0, 0, 4039, 25, 3293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 1037, 0, 125, 3422, 4075, 399, 0, 0, 0, 4012, 0, 0, 177, 0, 1371, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2138, 0, 4093, 14, 4001, 0, 1669, 518, 0, 0, 241, 3912, 27, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1988, 0, 8, 891, 0, 3499, 382, 378, 2798, 3980, 292, 0, 3901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13373, 13886, 16805, 21826, 15308, 15108, 16638, 18128] avg=16384.0000 max=21826 max_rank=3 min=13373 min_rank=0 imbalance_ratio=1.332153 max_load_reduction=109246 max_load_reduction_pct=0.833481 moved_to_echo=65182 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[18688, 0, 256, 6784, 128, 4096, 128, 128, 17280, 256, 2048, 5504, 256, 16384, 4352, 29440, 24320, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[2816, 20096, 12672, 256, 14080, 2048, 19584, 3200, 768, 3072, 19840, 4992, 0, 23936, 256, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[21888, 128, 15488, 5760, 8832, 768, 640, 10624, 896, 2432, 128, 27648, 128, 128, 9472, 256, 26880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[384, 3968, 3456, 2432, 128, 640, 8192, 4480, 17280, 24832, 26496, 27008, 128, 11648, 128, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[25088, 256, 128, 128, 4480, 0, 3584, 4864, 128, 5376, 15872, 23680, 1152, 8448, 384, 28288, 6528, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[11776, 16128, 8704, 19072, 256, 19584, 8960, 1792, 2432, 7552, 4608, 2048, 22400, 128, 640, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[8448, 512, 3584, 512, 14336, 23936, 9088, 11520, 4736, 512, 384, 24832, 7552, 128, 2560, 6400, 10368, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 128, 5376, 5632, 3712, 27648, 384, 16256, 29312, 10752, 6016, 640, 128, 10624, 9600, 128, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[3967, 0, 1, 0, 0, 4076, 3576, 4057, 0, 4063, 0, 0, 4093, 2521, 112, 4, 0, 0, 140, 619, 180, 109, 0, 8, 0, 3691, 0, 1703, 0, 2488, 0, 0, 0, 0, 5, 0, 0, 2, 3997, 4, 3744, 5, 4083, 3947, 3671, 3806, 3826, 1, 1656, 14, 74, 6, 0, 1, 99, 0, 0, 87, 24, 0, 3922, 0, 0, 0, 0, 0, 4008, 1, 0, 105, 0, 308, 343, 1, 15, 10, 366, 3989, 0, 2409, 15, 4049, 0, 4076, 586, 1, 4096, 38, 4090, 0, 4046, 52, 0, 0, 159, 3597, 18, 0, 0, 284, 245, 0, 7, 4089, 18, 0, 0, 1, 2, 0, 633, 3954, 1008, 1701, 45, 21, 4095, 84, 2, 68, 0, 3, 2202, 3745, 4064, 0, 41, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[3210, 1, 99, 12, 0, 0, 247, 4080, 461, 0, 0, 0, 0, 698, 4081, 0, 0, 230, 0, 0, 2, 1, 1, 0, 1310, 3714, 32, 2754, 652, 4096, 0, 0, 0, 427, 0, 3070, 0, 1, 3888, 1, 4088, 0, 3622, 3828, 9, 3796, 4091, 0, 3865, 0, 0, 0, 4, 0, 4046, 0, 0, 0, 0, 0, 0, 3, 1, 1, 4096, 0, 0, 0, 0, 71, 1, 0, 0, 4088, 589, 0, 4089, 4096, 0, 4060, 0, 0, 15, 3739, 0, 2, 0, 0, 1399, 0, 4083, 10, 0, 0, 1, 4075, 0, 0, 0, 4027, 0, 3027, 0, 4096, 22, 88, 3371, 0, 2678, 3810, 0, 1086, 0, 0, 420, 0, 3797, 0, 3762, 0, 0, 0, 0, 3975, 2071, 0, 6, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 4096, 0, 3932, 3458, 2926, 0, 0, 0, 2, 0, 20, 0, 0, 3966, 0, 0, 0, 3279, 1824, 0, 1, 4093, 17, 4085, 0, 2760, 0, 58, 0, 2, 0, 468, 784, 0, 4070, 3, 18, 0, 118, 0, 0, 4096, 4092, 0, 14, 0, 0, 1, 4, 0, 0, 0, 0, 4, 4096, 39, 0, 0, 277, 0, 3259, 0, 2469, 0, 92, 4096, 0, 474, 0, 0, 2359, 16, 1191, 1038, 1033, 0, 0, 3686, 0, 4096, 0, 0, 4095, 0, 3972, 0, 4094, 174, 0, 46, 0, 38, 4079, 3821, 0, 0, 873, 0, 232, 4090, 4073, 0, 1, 0, 0, 716, 3910, 3007, 0, 668, 12, 1, 3714, 67, 892, 4048, 0, 8, 0, 4034, 0, 0, 0, 3986] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[2942, 1, 0, 0, 19, 3593, 4084, 4091, 3849, 3937, 74, 0, 0, 2, 0, 1, 0, 3543, 33, 0, 935, 0, 2193, 0, 0, 4096, 185, 1, 11, 3949, 0, 3, 36, 0, 0, 0, 0, 90, 1, 1084, 0, 0, 1342, 109, 940, 0, 3452, 497, 219, 0, 1337, 0, 0, 0, 0, 0, 0, 157, 3565, 0, 0, 1, 0, 2, 4072, 3619, 32, 1, 829, 4035, 0, 9, 138, 4096, 4066, 4, 690, 4096, 4067, 3255, 634, 2455, 984, 4096, 0, 1238, 0, 419, 51, 0, 4, 0, 8, 170, 0, 4084, 4003, 165, 0, 1269, 4, 61, 0, 4096, 0, 475, 2946, 0, 0, 51, 4089, 376, 47, 0, 0, 0, 3007, 239, 4096, 695, 0, 3913, 0, 0, 164, 0, 0, 3850] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4085, 0, 6, 0, 74, 4096, 2583, 0, 0, 3858, 69, 101, 17, 0, 0, 3605, 0, 3970, 0, 8, 489, 4005, 0, 3, 0, 4090, 0, 15, 619, 0, 0, 1, 0, 0, 37, 0, 1519, 2943, 582, 3102, 84, 4095, 3891, 1072, 0, 4091, 3831, 20, 88, 0, 1, 1642, 10, 0, 0, 0, 521, 0, 19, 121, 0, 0, 25, 2, 1521, 72, 3420, 4, 0, 2611, 62, 1, 0, 2483, 2698, 22, 0, 4072, 2, 0, 1, 3268, 458, 4095, 0, 0, 4089, 3579, 890, 0, 3950, 5, 77, 0, 0, 3900, 0, 101, 4, 0, 22, 6, 0, 3925, 531, 1, 22, 20, 373, 3905, 1365, 3717, 1270, 0, 0, 418, 4016, 0, 2382, 4052, 0, 6, 230, 14, 4002, 18, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3193, 1, 2227, 191, 0, 3509, 4077, 813, 2986, 37, 0, 0, 0, 4017, 0, 0, 46, 110, 0, 0, 1690, 13, 0, 0, 4087, 2301, 0, 0, 74, 4096, 0, 0, 0, 1, 0, 1731, 4071, 376, 4096, 1, 4081, 3760, 10, 0, 93, 4081, 3932, 0, 3539, 0, 0, 0, 15, 0, 9, 0, 0, 0, 0, 0, 59, 3, 2, 3552, 7, 4054, 17, 0, 0, 0, 0, 0, 0, 1883, 4093, 0, 2, 4096, 0, 4095, 0, 3863, 3466, 4092, 3973, 74, 1723, 0, 5, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 676, 0, 0, 0, 4096, 0, 4067, 3, 0, 62, 0, 2907, 12, 5, 0, 1, 0, 4005, 4053, 0, 0, 0, 0, 0, 20, 0, 0, 4064, 682] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[10, 0, 22, 0, 53, 4081, 238, 3526, 5, 87, 609, 0, 88, 0, 2466, 0, 4079, 4096, 66, 6, 0, 6, 3159, 0, 80, 4087, 0, 130, 0, 4060, 0, 0, 2061, 17, 1, 0, 3211, 2, 4086, 3107, 741, 842, 0, 1480, 0, 3987, 1682, 1, 0, 0, 0, 67, 0, 0, 265, 0, 0, 4049, 3310, 0, 0, 0, 13, 0, 0, 0, 1173, 164, 0, 3749, 0, 0, 231, 4054, 3807, 0, 30, 1083, 0, 0, 0, 2779, 183, 17, 382, 34, 0, 4096, 3991, 0, 4096, 0, 50, 14, 0, 2, 0, 4091, 2396, 3927, 494, 1, 48, 4090, 749, 0, 0, 0, 429, 0, 552, 0, 0, 4034, 4095, 0, 3840, 4038, 0, 253, 0, 3641, 0, 464, 4070, 0, 22, 27] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4087, 0, 2, 1, 0, 0, 2569, 195, 268, 3448, 5, 0, 3734, 1, 17, 0, 3, 4069, 161, 7, 28, 3674, 1468, 0, 0, 553, 3617, 1, 23, 4096, 0, 0, 8, 0, 0, 1785, 17, 0, 3736, 3800, 3247, 3598, 24, 2754, 45, 3596, 4092, 4041, 4084, 0, 601, 498, 419, 1, 0, 557, 0, 11, 0, 30, 0, 0, 0, 23, 589, 201, 0, 0, 0, 0, 88, 0, 1, 6, 4095, 0, 0, 3888, 0, 0, 0, 4088, 1026, 6, 42, 0, 2980, 3588, 550, 4091, 4017, 65, 0, 0, 0, 3910, 1251, 2246, 0, 0, 0, 0, 275, 4095, 0, 1, 0, 0, 4096, 9, 10, 3712, 121, 7, 3946, 0, 3945, 3, 0, 29, 0, 1967, 0, 2, 0, 86, 2824, 222] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2052, 0, 1, 0, 0, 4076, 3576, 4057, 0, 4063, 0, 0, 4093, 2521, 112, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 619, 180, 109, 0, 8, 0, 3691, 0, 1703, 0, 2488, 0, 0, 1359, 126, 162, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2, 3997, 4, 3744, 5, 4083, 3947, 3671, 1241, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1656, 14, 74, 6, 0, 1, 99, 0, 0, 87, 24, 0, 3922, 0, 0, 0, 3826, 3850, 2565, 1753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 1, 0, 105, 0, 308, 343, 1, 15, 10, 366, 2630, 0, 2409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4049, 0, 4076, 586, 1, 4096, 38, 4090, 0, 196, 52, 0, 0, 159, 3597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 284, 245, 0, 7, 3963, 18, 0, 0, 1, 2, 0, 633, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1701, 45, 21, 4048, 84, 2, 68, 0, 3, 2202, 3745, 4064, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[24555, 10632, 20700, 17877, 10196, 20955, 9125, 17032] avg=16384.0000 max=24555 max_rank=0 min=9125 min_rank=6 imbalance_ratio=1.498718 max_load_reduction=106517 max_load_reduction_pct=0.812660 moved_to_echo=57308 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2114, 0, 6, 0, 74, 4096, 2583, 0, 0, 3858, 69, 101, 17, 0, 0, 3605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 0, 8, 489, 4005, 0, 3, 0, 4090, 0, 15, 619, 0, 0, 1, 1387, 121, 166, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 1519, 2943, 582, 3102, 84, 4095, 3891, 1072, 0, 1334, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 1, 1642, 10, 0, 0, 0, 521, 0, 19, 121, 0, 0, 25, 2, 3831, 3758, 2757, 1805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 72, 3420, 4, 0, 2611, 62, 1, 0, 2483, 2698, 22, 0, 2685, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3268, 458, 4095, 0, 0, 4089, 3579, 890, 0, 192, 5, 77, 0, 0, 3900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 4, 0, 22, 6, 0, 3804, 531, 1, 22, 20, 373, 3905, 1365, 3717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1270, 0, 0, 418, 3970, 0, 2382, 4052, 0, 6, 230, 14, 4002, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16523, 14920, 18679, 14580, 15581, 20554, 13871, 16364] avg=16384.0000 max=20554 max_rank=5 min=13871 min_rank=6 imbalance_ratio=1.254517 max_load_reduction=110518 max_load_reduction_pct=0.843185 moved_to_echo=66370 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2115, 0, 2, 1, 0, 0, 2569, 195, 268, 3448, 5, 0, 3734, 1, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4069, 161, 7, 28, 3674, 1468, 0, 0, 553, 3617, 1, 23, 4096, 0, 0, 1325, 126, 166, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1785, 17, 0, 3736, 3800, 3247, 3598, 24, 2754, 45, 1173, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 601, 498, 419, 1, 0, 557, 0, 11, 0, 30, 0, 0, 0, 23, 4092, 3822, 2423, 1806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 201, 0, 0, 0, 0, 88, 0, 1, 6, 4095, 0, 0, 2563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1026, 6, 42, 0, 2980, 3588, 550, 4091, 195, 65, 0, 0, 0, 3910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1251, 2246, 0, 0, 0, 0, 275, 3969, 0, 1, 0, 0, 4096, 9, 10, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 7, 3946, 0, 3899, 3, 0, 29, 0, 1967, 0, 2, 0, 86, 2824, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12355, 19363, 24228, 18367, 7543, 20541, 15569, 13106] avg=16384.0000 max=24228 max_rank=2 min=7543 min_rank=4 imbalance_ratio=1.478760 max_load_reduction=106844 max_load_reduction_pct=0.815155 moved_to_echo=56759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1660, 1, 99, 12, 0, 0, 247, 4080, 461, 0, 0, 0, 0, 698, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 2, 1, 1, 0, 1310, 3714, 32, 2754, 652, 4096, 0, 0, 1395, 126, 131, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 3070, 0, 1, 3888, 1, 4088, 0, 3622, 3828, 9, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 0, 0, 0, 4, 0, 4046, 0, 0, 0, 0, 0, 0, 3, 1, 1, 4091, 3885, 2558, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 71, 1, 0, 0, 4088, 589, 0, 4089, 2701, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3739, 0, 2, 0, 0, 1399, 0, 198, 10, 0, 0, 1, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 3027, 0, 3970, 22, 88, 3371, 0, 2678, 3810, 0, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 0, 3753, 0, 3762, 0, 0, 0, 0, 3975, 2071, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11339, 14488, 20172, 19873, 19695, 9439, 22079, 13987] avg=16384.0000 max=22079 max_rank=6 min=9439 min_rank=5 imbalance_ratio=1.347595 max_load_reduction=108993 max_load_reduction_pct=0.831551 moved_to_echo=65200 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 22, 0, 53, 4081, 238, 3526, 5, 87, 609, 0, 88, 0, 2466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4096, 66, 6, 0, 6, 3159, 0, 80, 4087, 0, 130, 0, 4060, 0, 0, 369, 126, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2061, 17, 1, 0, 3211, 2, 4086, 3107, 741, 842, 0, 1480, 0, 1300, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 265, 0, 0, 4049, 3310, 0, 0, 0, 13, 0, 1682, 3897, 2687, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1173, 164, 0, 3749, 0, 0, 231, 4054, 3807, 0, 30, 714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2779, 183, 17, 382, 34, 0, 4096, 3991, 0, 199, 0, 50, 14, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2396, 3927, 494, 1, 48, 3964, 749, 0, 0, 0, 429, 0, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 4095, 0, 3796, 4038, 0, 253, 0, 3641, 0, 464, 4070, 0, 22, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11181, 20308, 16849, 15974, 13922, 11747, 16651, 24440] avg=16384.0000 max=24440 max_rank=7 min=11181 min_rank=0 imbalance_ratio=1.491699 max_load_reduction=106632 max_load_reduction_pct=0.813538 moved_to_echo=66760 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 4096, 0, 3932, 3458, 2926, 0, 0, 0, 2, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 0, 3279, 1824, 0, 1, 4093, 17, 4085, 0, 2760, 0, 58, 353, 126, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 468, 784, 0, 4070, 3, 18, 0, 118, 0, 0, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1, 4, 0, 0, 0, 0, 4, 4096, 39, 0, 0, 277, 0, 4092, 3895, 2760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3259, 0, 2469, 0, 92, 4096, 0, 474, 0, 0, 2359, 16, 1191, 685, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3686, 0, 4096, 0, 0, 4095, 0, 3972, 0, 199, 174, 0, 46, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 3821, 0, 0, 873, 0, 232, 3964, 4073, 0, 1, 0, 0, 716, 3910, 3007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 12, 1, 3671, 67, 892, 4048, 0, 8, 0, 4034, 0, 0, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14443, 20605, 6799, 15182, 15674, 16306, 24676, 17387] avg=16384.0000 max=24676 max_rank=6 min=6799 min_rank=2 imbalance_ratio=1.506104 max_load_reduction=106396 max_load_reduction_pct=0.811737 moved_to_echo=74043 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1645, 1, 2227, 191, 0, 3509, 4077, 813, 2986, 37, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 110, 0, 0, 1690, 13, 0, 0, 4087, 2301, 0, 0, 74, 4096, 0, 0, 1400, 131, 133, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1731, 4071, 376, 4096, 1, 4081, 3760, 10, 0, 93, 1326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3539, 0, 0, 0, 15, 0, 9, 0, 0, 0, 0, 0, 59, 3, 2, 3552, 3932, 3902, 2755, 1415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4054, 17, 0, 0, 0, 0, 0, 0, 1883, 4093, 0, 2, 2696, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3863, 3466, 4092, 3973, 74, 1723, 0, 5, 0, 193, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 0, 0, 0, 3965, 0, 4067, 3, 0, 62, 0, 2907, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 3955, 4053, 0, 0, 0, 0, 0, 20, 0, 0, 4064, 682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19503, 14131, 19546, 19183, 16847, 17390, 11692, 12780] avg=16384.0000 max=19546 max_rank=2 min=11692 min_rank=6 imbalance_ratio=1.192993 max_load_reduction=111526 max_load_reduction_pct=0.850876 moved_to_echo=58709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1522, 1, 0, 0, 19, 3593, 4084, 4091, 3849, 3937, 74, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3543, 33, 0, 935, 0, 2193, 0, 0, 4096, 185, 1, 11, 3949, 0, 3, 1395, 126, 120, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 90, 1, 1084, 0, 0, 1342, 109, 940, 0, 0, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 1337, 0, 0, 0, 0, 0, 0, 157, 3565, 0, 0, 1, 0, 2, 3452, 3, 0, 1300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 3619, 32, 1, 829, 4035, 0, 9, 138, 4096, 4066, 4, 690, 2701, 4067, 3255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 634, 2455, 984, 4096, 0, 1238, 0, 419, 51, 0, 1, 0, 8, 170, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 165, 0, 1269, 4, 61, 0, 3970, 0, 475, 2946, 0, 0, 51, 4089, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 2972, 239, 4096, 695, 0, 3913, 0, 0, 164, 0, 0, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21173, 16625, 4099, 10036, 31614, 14140, 17409, 15976] avg=16384.0000 max=31614 max_rank=4 min=4099 min_rank=2 imbalance_ratio=1.929565 max_load_reduction=99458 max_load_reduction_pct=0.758804 moved_to_echo=79139 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[4224, 20096, 512, 640, 3328, 11136, 8704, 128, 5504, 26752, 3968, 8704, 1408, 25600, 0, 128, 9088, 1024, 896, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[768, 24192, 6144, 24320, 4992, 1408, 17024, 11776, 14976, 4096, 1408, 384, 256, 256, 256, 19712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2176, 512, 128, 7168, 9728, 3456, 24576, 11136, 16128, 12416, 13184, 13312, 4864, 8960, 0, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[13568, 8064, 11136, 256, 1024, 14720, 256, 896, 768, 16640, 21760, 128, 6400, 17408, 5120, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[2560, 6528, 8576, 512, 30080, 8576, 11136, 9216, 0, 9600, 2432, 12288, 14464, 128, 7040, 8832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9472, 10496, 2432, 10240, 1664, 3200, 640, 31616, 5504, 4736, 6400, 128, 7680, 8576, 13568, 15872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[11136, 128, 2432, 256, 4352, 19456, 21376, 20224, 10496, 15488, 768, 128, 7936, 7296, 6784, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[13568, 128, 2048, 2304, 512, 128, 4480, 640, 640, 4352, 11136, 256, 4096, 128, 384, 3584, 29056, 27136, 18560, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3757, 5, 0, 1179, 0, 0, 0, 0, 0, 0, 2, 3052, 2358, 0, 1, 0, 4092, 66, 0, 16, 2401, 0, 129, 3095, 2527, 0, 4001, 0, 16, 9, 0, 3425, 3, 241, 0, 0, 572, 0, 0, 0, 0, 0, 80, 3, 0, 4006, 0, 0, 283, 26, 0, 689, 0, 26, 4017, 74, 0, 0, 1196, 6, 0, 0, 4065, 555, 697, 4096, 235, 0, 3834, 0, 135, 2386, 0, 291, 7, 2389, 5, 4063, 0, 959, 109, 2, 3477, 19, 2622, 0, 0, 3593, 0, 17, 4092, 0, 0, 0, 0, 0, 8, 3053, 0, 0, 0, 4087, 3731, 0, 0, 9, 2480, 0, 3662, 3, 0, 2, 192, 237, 3989, 970, 2763, 0, 4092, 4088, 3588, 23, 4045, 4042, 3, 3737, 0, 3267] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 174, 25, 1453, 0, 4091, 4096, 2894, 0, 0, 522, 80, 0, 0, 3502, 3907, 0, 333, 17, 0, 4096, 0, 4084, 0, 0, 0, 4014, 1, 0, 4092, 0, 0, 230, 3675, 0, 3, 159, 0, 0, 4096, 0, 32, 4093, 0, 0, 0, 0, 0, 678, 129, 0, 0, 5, 0, 1, 17, 25, 4096, 0, 0, 3, 47, 55, 6, 0, 0, 33, 0, 0, 0, 0, 4096, 1488, 295, 0, 2, 4033, 853, 4059, 4096, 0, 4083, 38, 0, 3429, 0, 0, 1, 0, 0, 0, 4006, 3804, 484, 0, 2397, 4095, 0, 0, 3919, 7, 0, 0, 2, 11, 0, 4094, 2902, 4079, 1, 55, 43, 4096, 1, 2790, 0, 4088, 4095, 26, 0, 0, 744] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1814, 0, 8, 18, 0, 32, 0, 1902, 4059, 78, 40, 32, 4096, 0, 5, 4045, 4092, 1096, 4064, 35, 3650, 0, 3863, 4, 3914, 0, 530, 3308, 26, 3, 0, 3531, 492, 52, 1576, 7, 52, 2, 2207, 3644, 9, 3360, 0, 0, 4056, 0, 2447, 0, 0, 4, 3669, 4040, 708, 0, 100, 0, 49, 0, 2263, 2367, 0, 10, 3841, 0, 0, 26, 391, 0, 3, 0, 0, 4, 0, 4096, 0, 3907, 258, 1, 0, 0, 1029, 2289, 0, 3745, 746, 2, 3955, 76, 22, 0, 0, 17, 0, 0, 86, 0, 0, 2, 1670, 1250, 1, 2, 0, 0, 4019, 53, 29, 0, 4094, 340, 3735, 4088, 59, 451, 3290, 4091, 269, 44, 3476, 22, 0, 188, 44, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4089, 2, 156, 18, 91, 0, 308, 0, 0, 4077, 0, 1994, 2024, 0, 0, 0, 4013, 0, 0, 819, 16, 344, 12, 1268, 3998, 4044, 4090, 336, 3949, 0, 0, 0, 3842, 0, 0, 139, 0, 2, 0, 5, 0, 168, 0, 254, 0, 3908, 6, 1391, 68, 17, 363, 693, 1893, 0, 0, 3902, 1, 80, 143, 3383, 1, 3190, 0, 0, 225, 43, 17, 45, 3995, 0, 0, 412, 0, 0, 0, 2, 0, 1875, 0, 3939, 0, 3024, 18, 5, 4015, 3533, 10, 3553, 0, 0, 841, 3, 0, 2, 0, 446, 2, 81, 0, 4093, 346, 4096, 4096, 79, 0, 0, 0, 0, 0, 0, 847, 37, 676, 3329, 3067, 0, 3089, 37, 4081, 2344, 3578, 191, 4026, 0, 33, 569, 3188, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 2809, 0, 912, 4080, 0, 3858, 3886, 703, 0, 0, 4068, 0, 0, 7, 0, 0, 9, 0, 169, 0, 4063, 3468, 4032, 0, 0, 0, 285, 10, 22, 4, 0, 0, 3391, 2458, 1244, 0, 1088, 1, 75, 4084, 0, 0, 4096, 0, 73, 3936, 0, 0, 3941, 0, 0, 48, 1079, 188, 31, 0, 2785, 7, 36, 220, 93, 49, 4064, 0, 45, 450, 53, 37, 6, 32, 1, 1449, 3, 3815, 1, 0, 2240, 3784, 3979, 489, 0, 4086, 0, 0, 4096, 0, 0, 0, 4035, 0, 928, 9, 0, 133, 5, 3, 3912, 3316, 8, 453, 0, 1137, 232, 6, 0, 82, 3971, 898, 3980, 1, 3908, 8, 4096, 0, 832, 0, 4095, 194, 0, 1411, 0, 3475] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 39, 0, 2632, 0, 309, 4083, 3517, 4096, 0, 4095, 3358, 0, 0, 180, 0, 0, 0, 142, 381, 4092, 6, 55, 1336, 0, 374, 0, 1475, 1528, 2668, 22, 2430, 0, 0, 716, 2, 0, 0, 0, 705, 3, 42, 0, 0, 0, 8, 0, 3480, 0, 0, 0, 43, 0, 341, 998, 0, 2, 32, 940, 3, 0, 3470, 1910, 10, 0, 0, 0, 4075, 27, 254, 3810, 0, 0, 1, 4095, 8, 4095, 3754, 0, 0, 2, 0, 3920, 0, 3828, 1, 244, 0, 3801, 0, 0, 0, 0, 458, 4068, 0, 0, 21, 4091, 0, 4021, 4096, 37, 1, 0, 2445, 1738, 3459, 0, 5, 0, 5, 6, 4088, 1834, 633, 845, 4096, 2318, 2335, 0, 4086, 3739, 131, 1, 105, 972] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3250, 0, 522, 0, 0, 3910, 4096, 0, 17, 0, 2564, 0, 0, 0, 3491, 54, 0, 3, 938, 0, 4094, 0, 37, 363, 0, 2744, 3730, 0, 0, 3253, 0, 0, 4004, 9, 0, 68, 0, 0, 954, 120, 0, 3, 4096, 0, 4084, 720, 31, 0, 133, 0, 0, 0, 3143, 0, 1, 775, 4094, 748, 1550, 0, 1, 4081, 4060, 0, 0, 0, 624, 0, 0, 0, 1, 4096, 0, 4, 0, 1, 4058, 760, 5, 3201, 0, 2711, 3, 0, 40, 0, 0, 0, 0, 1, 7, 3992, 64, 1717, 0, 978, 1959, 0, 0, 3650, 3399, 1, 0, 2, 574, 0, 3936, 1634, 3626, 0, 4071, 3246, 4092, 3, 4096, 0, 4096, 4095, 332, 0, 0, 256] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[146, 0, 30, 212, 0, 0, 3949, 0, 0, 3572, 0, 4090, 379, 4019, 8, 0, 0, 3974, 2, 14, 0, 1358, 1168, 0, 0, 121, 4096, 16, 14, 0, 0, 1756, 1375, 1, 0, 1218, 8, 446, 1, 90, 4095, 0, 2273, 3961, 0, 90, 3, 2, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 3178, 0, 0, 231, 4092, 0, 7, 190, 0, 0, 5, 0, 0, 0, 0, 0, 0, 60, 4088, 0, 1559, 0, 0, 847, 0, 4079, 2325, 0, 4090, 0, 22, 4045, 0, 0, 0, 125, 1, 0, 3075, 0, 3979, 136, 1362, 3169, 0, 0, 3484, 3961, 3995, 0, 0, 0, 0, 3737, 3969, 3796, 30, 529, 13, 4071, 0, 4088, 39, 3902, 3883, 1, 228, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3757, 5, 0, 1179, 0, 0, 0, 0, 0, 0, 2, 3052, 2358, 0, 1, 0, 1988, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 66, 0, 16, 2401, 0, 129, 3095, 2527, 0, 1834, 0, 16, 9, 0, 3425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 241, 0, 0, 572, 0, 0, 0, 0, 0, 80, 3, 0, 4006, 0, 0, 4092, 1399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 26, 0, 689, 0, 26, 4017, 74, 0, 0, 1196, 6, 0, 0, 4065, 555, 2199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 697, 4096, 235, 0, 3834, 0, 135, 2386, 0, 291, 7, 2389, 5, 4063, 0, 959, 2646, 1511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 2, 3477, 19, 2622, 0, 0, 3593, 0, 17, 4092, 0, 0, 0, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3053, 0, 0, 0, 4087, 3731, 0, 0, 9, 2480, 0, 3662, 3, 0, 2, 598, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 237, 0, 970, 2763, 0, 0, 4088, 2390, 23, 0, 4042, 3, 3737, 0, 3267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12621, 17610, 10396, 13136, 23254, 14531, 17812, 21712] avg=16384.0000 max=23254 max_rank=4 min=10396 min_rank=2 imbalance_ratio=1.419312 max_load_reduction=107818 max_load_reduction_pct=0.822586 moved_to_echo=77309 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1814, 0, 8, 18, 0, 32, 0, 1902, 4059, 78, 40, 32, 1920, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 4045, 4092, 1096, 4064, 35, 3650, 0, 1771, 4, 3914, 0, 530, 3308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 3, 0, 3531, 492, 52, 1576, 7, 52, 2, 2207, 3644, 9, 3360, 0, 0, 3290, 1202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 2447, 0, 0, 4, 3669, 4040, 708, 0, 100, 0, 49, 0, 2263, 2367, 2059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3841, 0, 0, 26, 391, 0, 3, 0, 0, 4, 0, 4096, 0, 3907, 2274, 1415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 1, 0, 0, 1029, 2289, 0, 3745, 746, 2, 3955, 76, 22, 0, 0, 17, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 2, 1670, 1250, 1, 2, 0, 0, 4019, 53, 29, 0, 44, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 340, 0, 4088, 59, 451, 0, 4091, 180, 44, 0, 22, 0, 188, 44, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10165, 30610, 19453, 21762, 15967, 12185, 7328, 13602] avg=16384.0000 max=30610 max_rank=1 min=7328 min_rank=6 imbalance_ratio=1.868286 max_load_reduction=100462 max_load_reduction_pct=0.766464 moved_to_echo=49082 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 2809, 0, 912, 4080, 0, 3858, 3886, 703, 0, 0, 2019, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 7, 0, 0, 9, 0, 169, 0, 1863, 3468, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 10, 22, 4, 0, 0, 3391, 2458, 1244, 0, 1088, 1, 75, 4084, 0, 0, 4096, 1416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 73, 3936, 0, 0, 3941, 0, 0, 48, 1079, 188, 31, 0, 2785, 7, 2195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 220, 93, 49, 4064, 0, 45, 450, 53, 37, 6, 32, 1, 1449, 3, 3815, 2679, 1507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2240, 3784, 3979, 489, 0, 4086, 0, 0, 4096, 0, 0, 0, 4035, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 928, 9, 0, 133, 5, 3, 3912, 3316, 8, 453, 0, 1137, 232, 6, 0, 82, 138, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 898, 0, 1, 3908, 8, 0, 0, 555, 0, 0, 194, 0, 1411, 0, 3475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18551, 13616, 18174, 18379, 14539, 22849, 10543, 14421] avg=16384.0000 max=22849 max_rank=5 min=10543 min_rank=6 imbalance_ratio=1.394592 max_load_reduction=108223 max_load_reduction_pct=0.825676 moved_to_echo=62352 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 39, 0, 2632, 0, 309, 4083, 3517, 4096, 0, 4095, 3358, 0, 0, 180, 185, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 381, 4092, 6, 55, 1336, 0, 173, 0, 1475, 1528, 2668, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2430, 0, 0, 716, 2, 0, 0, 0, 705, 3, 42, 0, 0, 0, 8, 0, 4096, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3480, 0, 0, 0, 43, 0, 341, 998, 0, 2, 32, 940, 3, 0, 3470, 1910, 2252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4075, 27, 254, 3810, 0, 0, 1, 4095, 8, 4095, 3754, 0, 2672, 1548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3920, 0, 3828, 1, 244, 0, 3801, 0, 0, 0, 0, 458, 4068, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4091, 0, 4021, 4096, 37, 1, 0, 2445, 1738, 3459, 0, 5, 0, 389, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 1834, 633, 845, 0, 2318, 1556, 0, 0, 3739, 131, 1, 105, 972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22782, 11878, 9416, 13471, 24349, 16712, 20319, 12145] avg=16384.0000 max=24349 max_rank=4 min=9416 min_rank=2 imbalance_ratio=1.486145 max_load_reduction=106723 max_load_reduction_pct=0.814232 moved_to_echo=73525 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 174, 25, 1453, 0, 4091, 4096, 2894, 0, 0, 2040, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 80, 0, 0, 3502, 3907, 0, 333, 17, 0, 1869, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 1, 0, 4092, 0, 0, 230, 3675, 0, 3, 159, 0, 0, 4096, 0, 32, 4096, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 678, 129, 0, 0, 5, 0, 1, 17, 25, 4096, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 47, 55, 6, 0, 0, 33, 0, 0, 0, 0, 4096, 1488, 295, 2674, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4033, 853, 4059, 4096, 0, 4083, 38, 0, 3429, 0, 0, 1, 0, 0, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 3804, 484, 0, 2397, 4095, 0, 0, 3919, 7, 0, 0, 2, 11, 0, 469, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2902, 0, 1, 55, 43, 0, 1, 1851, 0, 0, 4095, 26, 0, 0, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19156, 14314, 21812, 11291, 10242, 21064, 19381, 13812] avg=16384.0000 max=21812 max_rank=2 min=10242 min_rank=4 imbalance_ratio=1.331299 max_load_reduction=109260 max_load_reduction_pct=0.833588 moved_to_echo=64499 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[146, 0, 30, 212, 0, 0, 3949, 0, 0, 3572, 0, 4090, 379, 4019, 8, 0, 2036, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 2, 14, 0, 1358, 1168, 0, 0, 121, 1877, 16, 14, 0, 0, 1756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1375, 1, 0, 1218, 8, 446, 1, 90, 4095, 0, 2273, 3961, 0, 90, 3, 2, 4071, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 3178, 0, 0, 231, 4092, 2091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 190, 0, 0, 5, 0, 0, 0, 0, 0, 0, 60, 4088, 0, 1559, 2553, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 847, 0, 4079, 2325, 0, 4090, 0, 22, 4045, 0, 0, 0, 125, 1, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3075, 0, 3979, 136, 1362, 3169, 0, 0, 3484, 3961, 3995, 0, 0, 0, 0, 681, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3737, 3969, 0, 30, 529, 13, 0, 0, 2723, 39, 0, 3883, 1, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18706, 10300, 18983, 17786, 9902, 16218, 24025, 15152] avg=16384.0000 max=24025 max_rank=6 min=9902 min_rank=4 imbalance_ratio=1.466370 max_load_reduction=107047 max_load_reduction_pct=0.816704 moved_to_echo=65297 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4089, 2, 156, 18, 91, 0, 308, 0, 0, 4077, 0, 1994, 2024, 0, 0, 0, 2033, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 819, 16, 344, 12, 1268, 3998, 4044, 1875, 336, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 0, 0, 139, 0, 2, 0, 5, 0, 168, 0, 254, 0, 3908, 6, 1391, 4081, 1392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 17, 363, 693, 1893, 0, 0, 3902, 1, 80, 143, 3383, 1, 3190, 0, 0, 1690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 43, 17, 45, 3995, 0, 0, 412, 0, 0, 0, 2, 0, 1875, 0, 3939, 2634, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3024, 18, 5, 4015, 3533, 10, 3553, 0, 0, 841, 3, 0, 2, 0, 446, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 81, 0, 4093, 346, 4096, 4096, 79, 0, 0, 0, 0, 0, 0, 847, 37, 596, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 3329, 0, 0, 3089, 37, 0, 2344, 2384, 191, 0, 0, 33, 569, 3188, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15006, 20674, 15188, 15424, 14350, 16048, 14455, 19927] avg=16384.0000 max=20674 max_rank=1 min=14350 min_rank=4 imbalance_ratio=1.261841 max_load_reduction=110398 max_load_reduction_pct=0.842270 moved_to_echo=64780 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3250, 0, 522, 0, 0, 3910, 4096, 0, 17, 0, 2035, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2564, 0, 0, 0, 3491, 54, 0, 3, 938, 0, 1876, 0, 37, 363, 0, 2744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3730, 0, 0, 3253, 0, 0, 4004, 9, 0, 68, 0, 0, 954, 120, 0, 3, 4092, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4084, 720, 31, 0, 133, 0, 0, 0, 3143, 0, 1, 775, 4094, 748, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1550, 0, 1, 4081, 4060, 0, 0, 0, 624, 0, 0, 0, 1, 4096, 0, 4, 2679, 1373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4058, 760, 5, 3201, 0, 2711, 3, 0, 40, 0, 0, 0, 0, 1, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3992, 64, 1717, 0, 978, 1959, 0, 0, 3650, 3399, 1, 0, 2, 574, 0, 683, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 1634, 0, 0, 4071, 3246, 0, 3, 2728, 0, 0, 4095, 332, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14085, 12070, 17650, 19823, 18469, 11465, 17209, 20301] avg=16384.0000 max=20301 max_rank=7 min=11465 min_rank=5 imbalance_ratio=1.239075 max_load_reduction=110771 max_load_reduction_pct=0.845116 moved_to_echo=67444 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2560, 4480, 4480, 4224, 20096, 128, 896, 7168, 768, 384, 128, 6528, 128, 27904, 5248, 14592, 20864, 11520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[15744, 256, 128, 13056, 1152, 512, 9216, 6272, 6144, 256, 5888, 7936, 1152, 19712, 128, 1536, 32000, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[20736, 13440, 0, 7040, 15232, 4736, 0, 12928, 14464, 384, 0, 20096, 640, 6144, 3456, 12928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[19456, 4224, 128, 5120, 13952, 10880, 5504, 4864, 12672, 4224, 13184, 3840, 17536, 1920, 3200, 11264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[8064, 128, 256, 1536, 4608, 0, 14848, 4352, 4992, 17408, 128, 27008, 24320, 7808, 128, 256, 14336, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[384, 3072, 14720, 9344, 19840, 19840, 128, 26112, 896, 3968, 20608, 128, 128, 128, 4736, 4608, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[20224, 128, 7040, 6144, 2048, 4224, 16896, 9216, 768, 256, 5760, 7808, 128, 4096, 17024, 13824, 16768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1024, 14336, 4096, 14592, 512, 17024, 26752, 4736, 128, 11520, 12416, 6912, 11392, 128, 1536, 128, 3712, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 719, 0, 0, 14, 0, 1, 0, 0, 162, 0, 2800, 0, 0, 4093, 2, 1347, 1, 0, 4096, 4093, 2535, 4020, 0, 2589, 0, 774, 4096, 0, 0, 0, 2925, 603, 0, 2804, 0, 4096, 1, 0, 66, 2, 0, 8, 0, 3989, 3388, 44, 0, 4096, 267, 0, 0, 0, 0, 1, 4094, 4096, 0, 476, 4002, 0, 0, 0, 3407, 1, 0, 3770, 0, 0, 0, 0, 31, 4096, 0, 46, 2680, 1, 4092, 573, 4, 2503, 0, 0, 0, 173, 4, 187, 0, 0, 0, 3659, 0, 2648, 0, 0, 0, 0, 247, 3833, 0, 0, 4056, 0, 0, 12, 611, 0, 245, 0, 0, 4019, 1992, 525, 0, 4091, 2700, 3, 0, 73, 3924, 0, 16, 4037, 21, 0, 2396, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[3987, 30, 490, 3398, 0, 0, 0, 0, 0, 3720, 0, 4058, 7, 3, 0, 200, 0, 7, 4095, 7, 4045, 2348, 275, 392, 159, 736, 2153, 0, 3850, 5, 7, 0, 1, 254, 6, 4092, 515, 2385, 236, 0, 0, 4, 0, 0, 0, 3877, 3633, 0, 4057, 0, 3, 0, 103, 0, 0, 3, 2435, 0, 0, 46, 4066, 0, 0, 0, 4096, 0, 0, 4076, 0, 0, 2648, 0, 79, 4093, 14, 515, 3, 0, 3288, 0, 4, 0, 0, 886, 135, 1654, 35, 4081, 1, 55, 0, 4096, 0, 4052, 3882, 0, 0, 110, 0, 0, 0, 0, 4094, 31, 1, 2, 561, 1870, 0, 0, 3, 3234, 303, 3745, 280, 3068, 0, 0, 234, 0, 4096, 1, 3767, 3575, 4093, 616, 5, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1793, 0, 1122, 26, 0, 0, 11, 5, 0, 4087, 0, 1319, 140, 0, 0, 3316, 47, 4, 300, 0, 3758, 4, 4068, 12, 2584, 0, 0, 0, 3892, 15, 2941, 34, 0, 0, 0, 19, 1487, 32, 3318, 0, 0, 4095, 587, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 296, 0, 7, 124, 2, 0, 4075, 0, 867, 2839, 4089, 0, 582, 2136, 0, 0, 125, 59, 32, 3491, 2, 0, 2, 3062, 0, 0, 1, 1271, 11, 0, 2614, 3319, 3516, 2824, 811, 104, 95, 4054, 0, 1202, 236, 11, 0, 868, 0, 4033, 0, 0, 2705, 1, 0, 0, 4095, 1, 51, 0, 0, 2, 4094, 4096, 0, 3783, 4096, 1034, 895, 66, 3035, 536, 805, 3851, 3749, 4026, 0, 286] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4092, 0, 0, 3282, 0, 3, 3, 221, 0, 5, 67, 0, 3728, 1, 0, 2, 0, 278, 0, 2713, 2590, 2, 0, 0, 79, 2768, 1, 1178, 3892, 0, 0, 0, 3578, 0, 1, 77, 0, 2060, 3339, 0, 0, 4091, 0, 3855, 0, 1106, 0, 0, 231, 823, 64, 147, 4066, 0, 0, 4094, 2326, 0, 0, 115, 185, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3945, 4, 0, 4050, 0, 0, 1476, 50, 4096, 0, 0, 1522, 4096, 0, 272, 3104, 0, 7, 12, 1718, 2290, 3168, 0, 4009, 2005, 14, 3578, 0, 55, 6, 0, 0, 2, 0, 0, 1265, 1632, 89, 423, 0, 0, 1, 5, 3824, 0, 3984, 4096, 1, 4096, 0, 3065, 0, 4096, 4096, 4096, 181, 1579, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4094, 94, 0, 1, 0, 3, 0, 4087, 0, 56, 0, 677, 581, 0, 0, 3835, 109, 556, 0, 0, 3978, 0, 0, 2959, 0, 0, 52, 0, 4096, 0, 59, 0, 3588, 358, 0, 4086, 42, 0, 0, 0, 841, 0, 0, 2334, 0, 2060, 75, 0, 2687, 3446, 7, 62, 0, 0, 36, 1746, 3581, 0, 0, 0, 3284, 71, 0, 0, 1360, 13, 10, 3889, 0, 69, 218, 3, 0, 4011, 0, 0, 3397, 2747, 3387, 1739, 0, 643, 69, 0, 0, 1850, 2227, 220, 0, 0, 9, 4089, 0, 4091, 4060, 0, 0, 0, 0, 311, 13, 0, 3948, 0, 13, 580, 2269, 2114, 26, 2, 0, 3845, 4038, 0, 0, 4091, 4096, 0, 3871, 0, 4055, 0, 4082, 0, 0, 34, 2042, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[6, 0, 0, 2, 34, 1465, 1, 2980, 0, 0, 0, 0, 50, 2, 0, 889, 0, 0, 44, 11, 4028, 337, 3507, 991, 1, 0, 0, 4091, 4069, 159, 2, 1, 997, 2045, 0, 93, 57, 1, 86, 1, 48, 2929, 0, 4092, 212, 125, 0, 0, 26, 4082, 5, 0, 0, 618, 15, 2, 4072, 1332, 3792, 5, 4092, 0, 15, 0, 3963, 0, 3376, 129, 0, 0, 3876, 0, 2, 4090, 0, 2, 3835, 1334, 2232, 19, 98, 0, 0, 0, 3862, 3341, 818, 3, 1, 0, 973, 4085, 0, 2112, 3068, 0, 1024, 0, 0, 0, 0, 6, 0, 19, 0, 0, 519, 4096, 0, 656, 0, 0, 3817, 1, 0, 3602, 4096, 0, 4037, 0, 4077, 0, 4096, 207, 4092, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[424, 0, 732, 3991, 0, 2033, 0, 0, 0, 250, 0, 0, 0, 1, 0, 705, 194, 0, 0, 4096, 3635, 4093, 0, 3882, 425, 0, 8, 4096, 4095, 0, 0, 0, 0, 0, 4, 0, 3171, 3746, 10, 0, 0, 3796, 0, 3019, 0, 317, 0, 0, 1, 4, 0, 0, 0, 0, 0, 3683, 3910, 786, 70, 45, 3989, 8, 318, 0, 4096, 0, 929, 0, 0, 0, 20, 25, 4087, 3274, 0, 0, 126, 13, 4089, 0, 0, 0, 190, 0, 340, 0, 0, 795, 0, 150, 29, 3901, 38, 3468, 7, 1, 0, 0, 4025, 1666, 0, 0, 2018, 154, 0, 53, 42, 0, 0, 19, 0, 2, 102, 3390, 0, 3763, 4096, 1, 1580, 0, 2706, 3827, 4096, 4096, 2130, 4095, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1118, 0, 0, 203, 0, 0, 3424, 3587, 1, 4052, 0, 1, 32, 6, 162, 46, 0, 0, 0, 142, 4081, 2647, 0, 0, 268, 4088, 0, 0, 3533, 38, 12, 0, 0, 3918, 3142, 10, 0, 4094, 0, 82, 1579, 3906, 382, 3, 0, 2603, 4082, 0, 4096, 0, 0, 463, 0, 11, 10, 247, 0, 3701, 0, 0, 3015, 0, 0, 0, 2574, 2, 0, 33, 0, 219, 0, 0, 162, 4096, 96, 28, 3755, 0, 3808, 1, 0, 501, 0, 0, 0, 1626, 4096, 0, 455, 1, 0, 3636, 12, 456, 0, 2, 35, 173, 1716, 529, 0, 0, 725, 436, 0, 171, 3993, 1351, 3834, 0, 0, 519, 4004, 3995, 0, 3323, 4096, 589, 3167, 0, 4078, 0, 2346, 0, 9, 0, 3627, 12] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4094, 94, 0, 1, 0, 3, 0, 4087, 0, 56, 0, 677, 581, 0, 0, 3835, 4096, 1439, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 556, 0, 0, 3978, 0, 0, 2959, 0, 0, 52, 0, 2784, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3588, 358, 0, 4086, 42, 0, 0, 0, 841, 0, 0, 2334, 0, 2060, 75, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2687, 3446, 7, 62, 0, 0, 36, 1746, 3581, 0, 0, 0, 3284, 71, 0, 0, 2931, 1169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1360, 13, 10, 3889, 0, 69, 218, 3, 0, 2884, 0, 0, 3397, 2747, 3387, 1739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 643, 69, 0, 0, 1850, 2227, 220, 0, 0, 9, 4089, 0, 4091, 4060, 0, 552, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 13, 0, 3948, 0, 13, 580, 2269, 2114, 26, 2, 0, 3845, 4091, 2616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 3871, 0, 0, 0, 650, 0, 0, 34, 2042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19464, 10497, 13959, 19020, 19716, 17953, 19828, 10635] avg=16384.0000 max=19828 max_rank=6 min=10497 min_rank=1 imbalance_ratio=1.210205 max_load_reduction=111244 max_load_reduction_pct=0.848724 moved_to_echo=68132 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1793, 0, 1122, 26, 0, 0, 11, 5, 0, 4087, 0, 1319, 140, 0, 0, 3316, 4096, 1078, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4, 300, 0, 3758, 4, 4068, 12, 2584, 0, 0, 0, 2646, 15, 2941, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1487, 32, 3318, 0, 0, 4095, 587, 0, 0, 0, 4089, 0, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 7, 124, 2, 0, 4075, 0, 867, 2839, 578, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 582, 2136, 0, 0, 125, 59, 32, 2511, 2, 0, 2, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1271, 11, 0, 2614, 3319, 3516, 2824, 811, 104, 95, 4054, 0, 1202, 236, 11, 480, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 868, 0, 4033, 0, 0, 2705, 1, 0, 0, 4095, 1, 51, 0, 0, 2, 3783, 1957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 1034, 895, 66, 0, 536, 129, 3851, 3749, 4026, 0, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17091, 16413, 14127, 9898, 12600, 20685, 17496, 22762] avg=16384.0000 max=22762 max_rank=7 min=9898 min_rank=3 imbalance_ratio=1.389282 max_load_reduction=108310 max_load_reduction_pct=0.826340 moved_to_echo=73543 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 2, 34, 1465, 1, 2980, 0, 0, 0, 0, 50, 2, 0, 889, 4096, 1447, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 11, 4028, 337, 3507, 991, 1, 0, 0, 4091, 2766, 159, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 997, 2045, 0, 93, 57, 1, 86, 1, 48, 2929, 0, 4092, 212, 125, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4082, 5, 0, 0, 618, 15, 2, 4072, 1332, 3792, 5, 4092, 0, 15, 0, 2941, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 3376, 129, 0, 0, 3876, 0, 2, 2941, 0, 2, 3835, 1334, 2232, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 3862, 3341, 818, 3, 1, 0, 973, 4085, 0, 2112, 3068, 0, 563, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 0, 0, 0, 0, 6, 0, 19, 0, 0, 519, 4096, 0, 656, 0, 0, 3602, 2630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3817, 1, 0, 0, 0, 0, 4037, 0, 0, 0, 652, 207, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11475, 15938, 11272, 22158, 21709, 19066, 12552, 16902] avg=16384.0000 max=22158 max_rank=3 min=11272 min_rank=2 imbalance_ratio=1.352417 max_load_reduction=108914 max_load_reduction_pct=0.830948 moved_to_echo=70229 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3987, 30, 490, 3398, 0, 0, 0, 0, 0, 3720, 0, 4058, 7, 3, 0, 200, 0, 1453, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4095, 7, 4045, 2348, 275, 392, 159, 736, 2153, 0, 2617, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 254, 6, 4092, 515, 2385, 236, 0, 0, 4, 0, 0, 0, 3877, 3633, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 3, 0, 103, 0, 0, 3, 2435, 0, 0, 46, 4066, 0, 0, 0, 2705, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 0, 0, 2648, 0, 79, 2943, 14, 515, 3, 0, 3288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 886, 135, 1654, 35, 4081, 1, 55, 0, 4096, 0, 4052, 3882, 0, 563, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 4094, 31, 1, 2, 561, 1870, 0, 0, 3, 3234, 3068, 2643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 3745, 280, 0, 0, 0, 234, 0, 0, 1, 600, 3575, 4093, 616, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17808, 16846, 15590, 14516, 17662, 19579, 15617, 13454] avg=16384.0000 max=19579 max_rank=5 min=13454 min_rank=7 imbalance_ratio=1.195007 max_load_reduction=111493 max_load_reduction_pct=0.850624 moved_to_echo=66312 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[424, 0, 732, 3991, 0, 2033, 0, 0, 0, 250, 0, 0, 0, 1, 0, 705, 4096, 961, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 4096, 3635, 4093, 0, 3882, 425, 0, 8, 4096, 2784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3171, 3746, 10, 0, 0, 3796, 0, 3019, 0, 317, 0, 0, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 3683, 3910, 786, 70, 45, 3989, 8, 318, 0, 2941, 1168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 929, 0, 0, 0, 20, 25, 4087, 2354, 0, 0, 126, 13, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 340, 0, 0, 795, 0, 150, 29, 3901, 38, 3468, 7, 1, 451, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 1666, 0, 0, 2018, 154, 0, 53, 42, 0, 0, 19, 0, 2, 3763, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 3390, 0, 0, 0, 1, 1580, 0, 0, 3827, 652, 4096, 2130, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13696, 23213, 14532, 16923, 15739, 9513, 13487, 23969] avg=16384.0000 max=23969 max_rank=7 min=9513 min_rank=5 imbalance_ratio=1.462952 max_load_reduction=107103 max_load_reduction_pct=0.817131 moved_to_echo=62708 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1118, 0, 0, 203, 0, 0, 3424, 3587, 1, 4052, 0, 1, 32, 6, 162, 46, 4096, 1448, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 4081, 2647, 0, 0, 268, 4088, 0, 0, 2402, 38, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 3142, 10, 0, 4094, 0, 82, 1579, 3906, 382, 3, 0, 2603, 4082, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 463, 0, 11, 10, 247, 0, 3701, 0, 0, 3015, 0, 0, 0, 1684, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2574, 2, 0, 33, 0, 219, 0, 0, 162, 2945, 96, 28, 3755, 0, 3808, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 0, 0, 0, 1626, 4096, 0, 455, 1, 0, 3636, 12, 456, 0, 2, 564, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 173, 1716, 529, 0, 0, 725, 436, 0, 171, 3993, 1351, 3834, 0, 0, 519, 3323, 2630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 3995, 0, 0, 0, 589, 3167, 0, 0, 0, 374, 0, 9, 0, 3627, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18464, 13678, 24388, 14235, 13623, 11472, 19435, 15777] avg=16384.0000 max=24388 max_rank=2 min=11472 min_rank=5 imbalance_ratio=1.488525 max_load_reduction=106684 max_load_reduction_pct=0.813934 moved_to_echo=60307 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 3282, 0, 3, 3, 221, 0, 5, 67, 0, 3728, 1, 0, 2, 4096, 1088, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 0, 2713, 2590, 2, 0, 0, 79, 2768, 1, 1178, 2637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3578, 0, 1, 77, 0, 2060, 3339, 0, 0, 4091, 0, 3855, 0, 1106, 0, 0, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 823, 64, 147, 4066, 0, 0, 4094, 2326, 0, 0, 115, 185, 0, 0, 0, 2945, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3945, 4, 0, 2905, 0, 0, 1476, 50, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1522, 4096, 0, 272, 3104, 0, 7, 12, 1718, 2290, 3168, 0, 4009, 2005, 14, 561, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3578, 0, 55, 6, 0, 0, 2, 0, 0, 1265, 1632, 89, 423, 0, 0, 1, 3984, 1977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3824, 0, 0, 0, 1, 4096, 0, 0, 0, 644, 4096, 4096, 181, 1579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17095, 12246, 18691, 16110, 12477, 22919, 13012, 18522] avg=16384.0000 max=22919 max_rank=5 min=12246 min_rank=1 imbalance_ratio=1.398865 max_load_reduction=108153 max_load_reduction_pct=0.825142 moved_to_echo=66930 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 719, 0, 0, 14, 0, 1, 0, 0, 162, 0, 2800, 0, 0, 4093, 2700, 1393, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1347, 1, 0, 4096, 4093, 2535, 4020, 0, 2589, 0, 774, 2784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2925, 603, 0, 2804, 0, 4096, 1, 0, 66, 2, 0, 8, 0, 3989, 3388, 44, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 267, 0, 0, 0, 0, 1, 4094, 4096, 0, 476, 4002, 0, 0, 0, 11, 1169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3407, 1, 0, 3770, 0, 0, 0, 0, 31, 2945, 0, 46, 2680, 1, 4092, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2503, 0, 0, 0, 173, 4, 187, 0, 0, 0, 3659, 0, 2648, 0, 0, 564, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 3833, 0, 0, 4056, 0, 0, 12, 611, 0, 245, 0, 0, 4019, 4091, 2531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1992, 525, 0, 0, 0, 3, 0, 73, 0, 0, 4, 4037, 21, 0, 2396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15979, 22241, 18513, 18212, 17546, 9885, 19645, 9051] avg=16384.0000 max=22241 max_rank=1 min=9051 min_rank=7 imbalance_ratio=1.357483 max_load_reduction=108831 max_load_reduction_pct=0.830315 moved_to_echo=56127 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[23680, 128, 4992, 14080, 0, 384, 10880, 128, 4480, 22528, 128, 640, 15360, 7296, 25088, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[11136, 7296, 3200, 11264, 5376, 16512, 7040, 128, 2560, 18944, 1024, 13312, 256, 14080, 15360, 128, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 6528, 4480, 896, 7296, 15104, 10752, 8192, 1280, 2048, 3456, 30720, 128, 22144, 13312, 128, 4352, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[19712, 128, 3072, 11008, 128, 3584, 3456, 11008, 128, 12288, 256, 6144, 7424, 128, 256, 13184, 27392, 10368, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[18432, 19584, 384, 0, 0, 1664, 17920, 256, 0, 4480, 3712, 19968, 18304, 8960, 17920, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[4736, 1152, 6144, 10496, 128, 128, 17664, 768, 128, 2176, 13824, 9600, 4608, 768, 128, 11648, 29824, 18816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[11136, 12544, 384, 768, 4224, 640, 384, 9856, 20480, 10112, 3968, 768, 26752, 128, 1280, 2944, 16768, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 2304, 4480, 7040, 30336, 13568, 10496, 12288, 3584, 10240, 2304, 10240, 21504, 256, 3072, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 4021, 0, 0, 10, 0, 35, 4096, 0, 0, 4080, 203, 0, 0, 0, 1, 0, 0, 0, 0, 8, 4096, 0, 12, 190, 0, 3653, 41, 0, 0, 0, 13, 55, 32, 8, 3, 476, 4033, 4096, 8, 1, 0, 4096, 0, 2752, 0, 0, 0, 1, 4048, 0, 0, 1, 9, 3929, 0, 4092, 971, 0, 0, 3934, 168, 11, 4041, 0, 2691, 1240, 0, 0, 827, 3730, 3221, 16, 1698, 0, 9, 4067, 81, 0, 4080, 4094, 0, 4029, 0, 635, 0, 998, 4096, 0, 1, 0, 15, 0, 2, 0, 0, 0, 0, 4096, 0, 4025, 4084, 4091, 0, 0, 4092, 0, 1, 0, 0, 0, 3, 0, 0, 5, 2, 0, 2959, 2, 4096, 685, 3067, 4093, 127, 0, 2889, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[543, 0, 0, 1388, 4096, 1, 0, 47, 3868, 250, 27, 4092, 0, 38, 0, 0, 4, 3250, 0, 0, 62, 0, 0, 65, 1, 17, 0, 203, 10, 0, 4089, 0, 0, 4084, 41, 1873, 0, 20, 0, 4030, 0, 3882, 13, 4061, 0, 2970, 1469, 0, 3838, 3, 3, 0, 1944, 0, 252, 4060, 171, 3996, 502, 0, 885, 2681, 4096, 0, 928, 3052, 3023, 0, 15, 4095, 0, 1, 3388, 0, 44, 0, 3, 7, 4094, 0, 3103, 4089, 0, 4064, 237, 332, 3, 0, 0, 0, 111, 990, 2867, 0, 300, 0, 0, 1883, 0, 4096, 0, 0, 4096, 4046, 60, 22, 3925, 0, 0, 0, 2107, 40, 0, 0, 0, 1, 0, 16, 41, 0, 3780, 3, 1121, 3979, 162, 18, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4086, 0, 2178, 154, 0, 3, 0, 3249, 1, 555, 3841, 0, 1729, 0, 0, 0, 0, 0, 323, 0, 39, 4087, 7, 500, 3790, 1237, 677, 908, 0, 3265, 0, 251, 4083, 0, 12, 0, 23, 0, 3, 4028, 0, 0, 162, 3514, 0, 0, 0, 185, 0, 1, 0, 221, 0, 84, 4061, 21, 4073, 1, 0, 0, 4096, 4094, 0, 972, 1, 1889, 105, 41, 3891, 0, 1570, 4096, 0, 3978, 816, 0, 2403, 18, 0, 0, 4095, 0, 3294, 3754, 0, 769, 0, 0, 4, 805, 1996, 4096, 3, 262, 0, 9, 1, 102, 0, 0, 4090, 4067, 3820, 1, 5, 3804, 1264, 2, 0, 3286, 13, 0, 14, 0, 64, 0, 4078, 4093, 0, 3435, 0, 0, 0, 515, 6, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 3696, 26, 5, 2295, 629, 0, 0, 4082, 0, 10, 3299, 296, 0, 12, 0, 0, 86, 0, 0, 0, 106, 679, 0, 250, 4086, 56, 4090, 0, 0, 3886, 0, 56, 1942, 0, 5, 39, 228, 16, 2790, 680, 45, 6, 3991, 47, 4096, 3, 0, 0, 1424, 3834, 0, 4065, 889, 2589, 1, 1, 4071, 3, 0, 0, 4096, 3412, 0, 0, 0, 0, 31, 3309, 578, 0, 0, 4078, 1085, 3957, 4096, 1, 4096, 0, 134, 4096, 4096, 0, 1786, 2076, 4096, 0, 0, 898, 0, 0, 0, 60, 0, 0, 30, 0, 0, 0, 0, 264, 0, 0, 4096, 89, 0, 0, 0, 0, 0, 4096, 0, 161, 0, 3, 0, 0, 4096, 3977, 3849, 4072, 0, 1893, 53, 0, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 3180, 3229, 0, 0, 671, 218, 0, 2, 14, 3912, 4094, 2, 0, 0, 0, 0, 102, 0, 543, 386, 153, 62, 0, 653, 4047, 0, 0, 2, 0, 3377, 0, 0, 2, 0, 388, 4050, 0, 0, 4094, 4096, 203, 0, 2143, 2389, 4072, 0, 636, 0, 59, 4046, 0, 0, 3185, 27, 2389, 0, 4084, 0, 11, 0, 1298, 4094, 6, 2722, 3216, 3944, 4061, 97, 189, 4085, 0, 3945, 385, 669, 0, 107, 0, 4066, 0, 0, 5, 46, 80, 7, 4006, 0, 4081, 87, 0, 36, 11, 1205, 65, 0, 0, 0, 0, 13, 153, 35, 3967, 1, 4096, 0, 0, 0, 1839, 0, 0, 1, 11, 0, 13, 0, 3992, 0, 2, 0, 0, 3824, 1030, 4093, 3251, 0, 0, 0, 1717] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1, 3903, 223, 0, 4007, 50, 1, 12, 3764, 63, 3799, 4077, 0, 3568, 1, 0, 11, 0, 0, 0, 0, 0, 40, 0, 65, 81, 377, 3904, 0, 0, 3219, 499, 13, 4, 9, 4096, 0, 4, 9, 1575, 4096, 3506, 0, 26, 3498, 0, 355, 0, 0, 166, 4044, 0, 58, 4096, 4096, 1138, 0, 3951, 19, 0, 0, 1, 94, 4, 986, 0, 0, 0, 0, 4085, 0, 0, 0, 7, 4087, 0, 14, 4094, 73, 0, 4009, 0, 0, 0, 0, 1, 107, 1767, 4074, 0, 112, 0, 4087, 0, 1, 0, 668, 0, 306, 4090, 0, 3858, 82, 4041, 102, 0, 3781, 0, 0, 0, 0, 0, 1, 0, 2, 22, 0, 4084, 0, 151, 4034, 0, 3646, 4094, 0, 4072, 10, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[80, 0, 3853, 0, 0, 0, 1, 18, 4096, 2728, 3, 2, 287, 0, 0, 0, 0, 0, 0, 0, 1851, 0, 4023, 0, 20, 4, 0, 3007, 1, 44, 791, 0, 0, 19, 0, 0, 3009, 0, 663, 670, 3656, 0, 10, 4090, 0, 73, 4094, 0, 0, 1, 3581, 0, 4081, 3940, 0, 0, 27, 4096, 74, 0, 0, 3164, 4049, 0, 2870, 0, 4063, 0, 4036, 2531, 0, 11, 2855, 0, 3, 0, 0, 11, 41, 0, 875, 0, 0, 973, 0, 4096, 0, 2254, 0, 225, 0, 0, 0, 551, 13, 179, 0, 8, 4072, 2900, 0, 3578, 3486, 3896, 0, 0, 3, 3768, 0, 0, 0, 0, 318, 0, 0, 0, 447, 4088, 3794, 407, 4095, 1419, 0, 4092, 3953, 24, 4, 1027] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 2060, 120, 0, 4083, 0, 63, 0, 4068, 2206, 2244, 4096, 0, 4095, 0, 0, 0, 1, 0, 273, 0, 0, 0, 64, 48, 0, 309, 282, 129, 0, 4059, 48, 43, 4089, 0, 3, 3395, 0, 0, 628, 9, 3475, 0, 34, 2, 1831, 4, 0, 35, 0, 4088, 0, 2658, 5, 0, 1, 0, 4077, 3755, 36, 105, 2587, 4069, 0, 0, 0, 170, 0, 4096, 0, 9, 0, 2519, 0, 4096, 3204, 4, 34, 4051, 0, 4040, 1, 0, 0, 862, 4096, 0, 24, 2437, 17, 4095, 0, 4055, 1, 0, 21, 0, 0, 0, 4096, 2532, 3701, 0, 3831, 1, 10, 24, 0, 3, 0, 0, 1753, 1777, 3, 0, 65, 0, 4096, 1438, 0, 3851, 0, 2356, 0, 343, 0, 183, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[543, 0, 0, 1388, 4096, 1, 0, 47, 3868, 250, 27, 4092, 0, 38, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3250, 0, 0, 62, 0, 0, 65, 1, 17, 0, 203, 10, 0, 4089, 0, 3823, 3343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 41, 1873, 0, 20, 0, 4030, 0, 3882, 13, 3619, 0, 2970, 1469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 3, 3, 0, 1944, 0, 252, 4060, 171, 173, 502, 0, 885, 2681, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 928, 3052, 3023, 0, 15, 4095, 0, 1, 0, 0, 41, 0, 3, 7, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3103, 4089, 0, 4064, 237, 332, 3, 0, 0, 0, 111, 990, 2867, 0, 300, 0, 419, 256, 442, 1, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1883, 0, 4096, 0, 0, 4096, 3627, 60, 22, 3925, 0, 0, 0, 2107, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 16, 41, 0, 3524, 3, 1121, 3979, 162, 18, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14352, 14867, 22001, 18608, 15259, 17259, 19856, 8870] avg=16384.0000 max=22001 max_rank=2 min=8870 min_rank=7 imbalance_ratio=1.342834 max_load_reduction=109071 max_load_reduction_pct=0.832146 moved_to_echo=61244 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3180, 3229, 0, 0, 671, 218, 0, 2, 14, 3912, 4094, 2, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 543, 386, 153, 62, 0, 653, 4047, 0, 0, 2, 0, 3377, 0, 3907, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 388, 4050, 0, 0, 4094, 4096, 203, 0, 1910, 2389, 4072, 0, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 4046, 0, 0, 3185, 27, 2389, 0, 177, 0, 11, 0, 1298, 4094, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2722, 3216, 3944, 4061, 97, 189, 4085, 0, 0, 385, 610, 0, 107, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 46, 80, 7, 4006, 0, 4081, 87, 0, 36, 11, 1205, 65, 0, 0, 424, 259, 233, 15, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 153, 35, 3967, 1, 3672, 0, 0, 0, 1839, 0, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 3992, 0, 2, 0, 0, 3565, 1030, 4093, 3251, 0, 0, 0, 1717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15366, 17124, 21840, 15292, 23482, 10613, 9692, 17663] avg=16384.0000 max=23482 max_rank=4 min=9692 min_rank=6 imbalance_ratio=1.433228 max_load_reduction=107590 max_load_reduction_pct=0.820847 moved_to_echo=61450 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3696, 26, 5, 2295, 629, 0, 0, 4082, 0, 10, 3299, 296, 0, 12, 0, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 106, 679, 0, 250, 4086, 56, 4090, 0, 0, 3886, 0, 3895, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 1942, 0, 5, 39, 228, 16, 2790, 680, 45, 6, 3557, 47, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1424, 3834, 0, 4065, 889, 2589, 1, 1, 176, 3, 0, 0, 4096, 3412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3309, 578, 0, 0, 0, 1085, 3600, 4096, 1, 4096, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1786, 2076, 4096, 0, 0, 898, 0, 0, 0, 60, 0, 0, 30, 424, 276, 434, 94, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 3672, 89, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 3, 0, 0, 4096, 3977, 3849, 3796, 0, 1893, 53, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14613, 21158, 13510, 20490, 16930, 18420, 8121, 17830] avg=16384.0000 max=21158 max_rank=1 min=8121 min_rank=6 imbalance_ratio=1.291382 max_load_reduction=109914 max_load_reduction_pct=0.838577 moved_to_echo=61301 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4086, 0, 2178, 154, 0, 3, 0, 3249, 1, 555, 3841, 0, 1729, 0, 0, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 39, 4087, 7, 500, 3790, 1237, 677, 908, 0, 3265, 0, 3897, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 4083, 0, 12, 0, 23, 0, 3, 4028, 0, 0, 145, 3514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 1, 0, 221, 0, 84, 4061, 21, 176, 1, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1, 1889, 105, 41, 3891, 0, 1570, 0, 0, 3619, 816, 0, 2403, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3294, 3754, 0, 769, 0, 0, 4, 805, 1996, 4096, 3, 262, 0, 395, 232, 17, 94, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 102, 0, 0, 4090, 4067, 3425, 1, 5, 3804, 1264, 2, 0, 3286, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 64, 0, 4078, 4093, 0, 3203, 0, 0, 0, 515, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16061, 22772, 12059, 12940, 15325, 19870, 20069, 11976] avg=16384.0000 max=22772 max_rank=1 min=11976 min_rank=7 imbalance_ratio=1.389893 max_load_reduction=108300 max_load_reduction_pct=0.826263 moved_to_echo=67240 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4021, 0, 0, 10, 0, 35, 4096, 0, 0, 4080, 203, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8, 4096, 0, 12, 190, 0, 3653, 41, 0, 0, 0, 3918, 3179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 55, 32, 8, 3, 476, 4033, 4096, 8, 1, 0, 3647, 0, 2752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4048, 0, 0, 1, 9, 3929, 0, 174, 971, 0, 0, 3934, 168, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 2691, 1240, 0, 0, 827, 3730, 0, 16, 1537, 0, 9, 4067, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4094, 0, 4029, 0, 635, 0, 998, 4096, 0, 1, 0, 15, 0, 2, 0, 428, 281, 449, 44, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4025, 4084, 3663, 0, 0, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5, 2, 0, 2959, 2, 3815, 685, 3067, 4093, 127, 0, 2889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12563, 15098, 15124, 13246, 18239, 19194, 19961, 17647] avg=16384.0000 max=19961 max_rank=6 min=12563 min_rank=0 imbalance_ratio=1.218323 max_load_reduction=111111 max_load_reduction_pct=0.847710 moved_to_echo=75041 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[80, 0, 3853, 0, 0, 0, 1, 18, 4096, 2728, 3, 2, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1851, 0, 4023, 0, 20, 4, 0, 3007, 1, 44, 791, 0, 3919, 2818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 3009, 0, 663, 670, 3656, 0, 10, 3645, 0, 73, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3581, 0, 4081, 3940, 0, 0, 27, 177, 74, 0, 0, 3164, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2870, 0, 4063, 0, 4036, 2531, 0, 11, 0, 0, 3, 0, 0, 11, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 875, 0, 0, 973, 0, 4096, 0, 2254, 0, 225, 0, 0, 0, 551, 13, 179, 403, 277, 445, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4072, 2900, 0, 3578, 3486, 3493, 0, 0, 3, 3768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 0, 447, 4088, 3794, 407, 3818, 1419, 0, 4092, 3953, 24, 4, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11068, 16478, 15839, 19094, 13566, 10328, 21308, 23391] avg=16384.0000 max=23391 max_rank=7 min=10328 min_rank=5 imbalance_ratio=1.427673 max_load_reduction=107681 max_load_reduction_pct=0.821541 moved_to_echo=68593 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 3903, 223, 0, 4007, 50, 1, 12, 3764, 63, 3799, 4077, 0, 3568, 1, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 40, 0, 65, 81, 377, 3904, 0, 0, 3219, 499, 3780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4, 9, 4096, 0, 4, 9, 1575, 4096, 3506, 0, 24, 3498, 0, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 4044, 0, 58, 4096, 4096, 1138, 0, 171, 19, 0, 0, 1, 94, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 986, 0, 0, 0, 0, 4085, 0, 0, 0, 7, 3718, 0, 14, 4094, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 0, 1, 107, 1767, 4074, 0, 112, 0, 4087, 0, 1, 0, 418, 273, 2, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 306, 4090, 0, 3858, 82, 3623, 102, 0, 3781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 22, 0, 4084, 0, 151, 3761, 0, 3646, 4094, 0, 4072, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23741, 11976, 17189, 13887, 12977, 14948, 16510, 19844] avg=16384.0000 max=23741 max_rank=0 min=11976 min_rank=1 imbalance_ratio=1.449036 max_load_reduction=107331 max_load_reduction_pct=0.818871 moved_to_echo=64279 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2060, 120, 0, 4083, 0, 63, 0, 4068, 2206, 2244, 4096, 0, 4095, 0, 0, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 273, 0, 0, 0, 64, 48, 0, 309, 282, 129, 0, 4059, 48, 3901, 2485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 4089, 0, 3, 3395, 0, 0, 628, 9, 3475, 0, 31, 2, 1831, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4088, 0, 2658, 5, 0, 1, 0, 176, 3755, 36, 105, 2587, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 4096, 0, 9, 0, 0, 0, 3726, 3204, 4, 34, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 1, 0, 0, 862, 4096, 0, 24, 2437, 17, 4095, 0, 4055, 1, 0, 21, 396, 261, 3, 97, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2532, 3701, 0, 3435, 1, 10, 24, 0, 3, 0, 0, 1753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1777, 3, 0, 65, 0, 4096, 1438, 0, 3590, 0, 2356, 0, 343, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23308, 11599, 13510, 17515, 15294, 20440, 15555, 13851] avg=16384.0000 max=23308 max_rank=0 min=11599 min_rank=1 imbalance_ratio=1.422607 max_load_reduction=107764 max_load_reduction_pct=0.822174 moved_to_echo=65140 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[4096, 1664, 23680, 0, 13056, 12160, 7168, 15616, 256, 1408, 5376, 128, 1024, 21888, 24192, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[20224, 16384, 128, 14336, 7040, 17280, 896, 9216, 11648, 256, 5248, 3072, 16512, 640, 640, 256, 3328, 2176, 2048, 512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[640, 17024, 11520, 3584, 14720, 1408, 384, 128, 27264, 5376, 10624, 27648, 896, 9472, 128, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12544, 6272, 15872, 5504, 11648, 15488, 4992, 5376, 0, 1536, 16896, 8192, 256, 14720, 12544, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[768, 1920, 4608, 19456, 2944, 23296, 15872, 28672, 256, 128, 15744, 6912, 128, 0, 9600, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[384, 14336, 128, 6400, 10496, 768, 4736, 17920, 16640, 11136, 128, 16640, 9472, 15872, 6016, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 3456, 0, 1152, 2304, 384, 13056, 256, 1664, 12288, 2048, 15872, 1152, 128, 22784, 640, 31104, 23808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2304, 128, 128, 4224, 512, 20480, 16384, 4480, 29184, 3200, 16256, 19584, 5120, 4224, 3200, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[2344, 2, 1, 111, 4087, 6, 0, 278, 6, 964, 3470, 4037, 394, 0, 0, 0, 2555, 2, 20, 4095, 174, 712, 0, 0, 0, 0, 0, 37, 0, 0, 6, 1, 0, 0, 0, 0, 1172, 3803, 4096, 13, 0, 0, 3952, 3716, 0, 0, 861, 0, 0, 0, 38, 4091, 0, 3843, 10, 0, 225, 1, 3867, 0, 49, 160, 75, 3944, 0, 0, 0, 5, 0, 0, 0, 4052, 0, 2327, 4096, 4092, 6, 0, 1, 4032, 1112, 1536, 0, 1082, 1, 277, 0, 0, 0, 0, 8, 63, 0, 0, 0, 1, 0, 0, 0, 2941, 0, 3956, 4054, 242, 3487, 4094, 0, 0, 3698, 0, 0, 3864, 140, 4096, 708, 329, 22, 0, 4096, 1321, 3556, 0, 0, 0, 3405, 4092, 0, 3062] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 163, 4092, 42, 0, 98, 288, 1674, 1230, 3785, 0, 0, 0, 119, 4096, 0, 0, 46, 106, 0, 0, 0, 168, 36, 0, 0, 2476, 4096, 202, 0, 0, 8, 0, 0, 855, 1, 234, 0, 0, 0, 270, 0, 545, 0, 226, 1, 48, 4050, 0, 3684, 0, 2518, 0, 17, 31, 568, 31, 0, 0, 0, 37, 4096, 0, 3904, 4096, 0, 8, 0, 569, 4096, 8, 0, 4095, 0, 2781, 0, 0, 3630, 0, 0, 921, 0, 0, 21, 3619, 109, 106, 0, 5, 0, 368, 0, 109, 0, 0, 0, 3971, 0, 0, 3826, 3915, 0, 0, 63, 33, 11, 4065, 0, 4091, 3976, 2532, 3959, 3818, 4032, 443, 0, 4082, 3056, 3495, 1, 0, 54, 3043, 4082, 0, 2046] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4094, 0, 2291, 1999, 0, 359, 110, 4021, 74, 2000, 3848, 0, 57, 0, 23, 9, 4085, 0, 15, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 671, 1086, 4096, 15, 0, 0, 0, 4093, 8, 12, 0, 4096, 1, 0, 0, 0, 1358, 0, 0, 0, 1127, 7, 4096, 3, 2596, 3598, 3993, 0, 0, 0, 0, 0, 3867, 0, 0, 0, 590, 4, 5, 0, 16, 4, 225, 4092, 4096, 1469, 0, 0, 3879, 40, 0, 0, 1527, 3681, 3, 247, 4015, 3, 4, 8, 2086, 1, 0, 106, 0, 9, 0, 0, 3103, 0, 0, 3227, 232, 0, 4092, 85, 1, 3933, 0, 1557, 4084, 0, 3993, 3959, 4069, 224, 0, 3421, 3079, 4096, 0, 0, 2, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4090, 0, 277, 0, 192, 37, 27, 41, 0, 17, 4078, 2764, 0, 0, 0, 0, 5, 0, 0, 1969, 0, 0, 0, 1, 0, 4013, 0, 25, 0, 0, 0, 14, 0, 28, 0, 4086, 0, 0, 9, 4096, 87, 0, 2941, 4092, 17, 255, 0, 0, 4, 0, 91, 4094, 0, 4084, 302, 28, 4096, 4096, 0, 0, 0, 0, 187, 4094, 482, 3690, 0, 1, 15, 0, 0, 4079, 3342, 1, 0, 0, 561, 1, 612, 3455, 0, 4070, 3612, 0, 3943, 0, 853, 1310, 0, 0, 0, 0, 1582, 0, 0, 4096, 1668, 5, 0, 4013, 70, 128, 3773, 4091, 3537, 4095, 441, 0, 4096, 0, 92, 2108, 0, 0, 3748, 126, 0, 0, 2, 4020, 0, 1, 3981, 0, 1133, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[308, 8, 3351, 4057, 2321, 3939, 0, 0, 4061, 0, 54, 3981, 0, 16, 0, 839, 4096, 698, 0, 3441, 0, 0, 78, 2, 4058, 0, 1, 31, 0, 115, 0, 0, 0, 13, 824, 5, 450, 0, 3996, 2, 0, 0, 0, 0, 4, 9, 0, 2, 0, 0, 563, 2340, 8, 4091, 4091, 0, 2, 4089, 0, 0, 0, 210, 2, 4015, 240, 0, 0, 157, 2, 10, 116, 3809, 2, 35, 3948, 2841, 3778, 0, 445, 0, 1, 0, 0, 0, 2, 118, 6, 0, 0, 38, 5, 0, 19, 104, 0, 4096, 0, 0, 20, 4096, 80, 0, 3803, 419, 1616, 3813, 0, 0, 4096, 4095, 4074, 374, 0, 3870, 6, 664, 0, 0, 4047, 3948, 0, 131, 2, 63, 4081, 3745, 0, 4086] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4081, 0, 4082, 184, 0, 3754, 0, 0, 86, 0, 4096, 0, 62, 2, 0, 350, 4011, 0, 0, 2, 0, 0, 2, 0, 0, 35, 0, 1458, 0, 1, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 10, 5, 198, 1734, 2855, 0, 1, 0, 1475, 3109, 844, 0, 2514, 2, 4047, 4092, 0, 4006, 4090, 0, 516, 0, 0, 32, 4096, 68, 3384, 0, 5, 0, 0, 0, 4095, 276, 3270, 361, 1699, 0, 67, 0, 3975, 2688, 3793, 3807, 0, 0, 0, 0, 2001, 0, 0, 0, 0, 0, 1, 23, 4, 5, 1, 0, 3009, 8, 0, 4084, 1, 3669, 1905, 0, 0, 4096, 4096, 0, 4096, 1, 1573, 0, 721, 32, 0, 0, 4096, 1715, 0, 4096, 0, 1579, 0, 0, 2871] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4009, 3, 4092, 0, 4, 0, 0, 0, 214, 2340, 4096, 0, 3027, 2, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 275, 0, 28, 0, 51, 0, 0, 0, 0, 0, 0, 0, 828, 0, 0, 0, 730, 4022, 4, 4059, 1193, 9, 22, 0, 449, 0, 4, 3537, 1, 365, 3960, 0, 0, 2347, 4091, 3659, 0, 0, 0, 58, 1046, 4092, 3754, 0, 91, 0, 0, 0, 4075, 3, 6, 3928, 0, 2625, 311, 0, 70, 4087, 0, 0, 0, 759, 0, 4084, 0, 0, 2479, 0, 0, 0, 1, 0, 0, 350, 84, 0, 4077, 88, 0, 4026, 1, 0, 4077, 0, 3, 4096, 4096, 0, 4096, 10, 4096, 0, 182, 4087, 0, 0, 4096, 4018, 0, 2593, 0, 47, 0, 0, 3965] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4086, 0, 561, 4092, 3926, 0, 0, 0, 409, 0, 3745, 0, 0, 0, 0, 0, 1218, 8, 2644, 96, 2, 6, 0, 4093, 10, 134, 0, 0, 4067, 0, 0, 0, 4077, 1343, 0, 4095, 3, 6, 2245, 0, 709, 16, 1666, 3302, 3, 125, 8, 4090, 0, 0, 0, 4096, 0, 3891, 3, 1, 3885, 2643, 0, 251, 0, 0, 13, 3884, 3791, 3549, 12, 1, 0, 0, 0, 3706, 3951, 432, 0, 0, 2465, 31, 0, 4096, 768, 4058, 0, 0, 1791, 7, 1, 6, 0, 0, 0, 103, 0, 0, 0, 0, 4, 660, 1812, 0, 51, 0, 0, 13, 4089, 1261, 330, 0, 4096, 0, 0, 0, 258, 3507, 4096, 716, 0, 7, 4095, 0, 4095, 0, 0, 8, 3753, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2344, 2, 1, 111, 4087, 6, 0, 278, 6, 964, 2785, 4037, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2555, 2, 20, 4095, 174, 712, 0, 0, 0, 0, 0, 37, 0, 0, 6, 1, 3698, 3168, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1172, 3803, 4096, 13, 0, 0, 3952, 3716, 0, 0, 861, 0, 2164, 443, 685, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4091, 0, 675, 10, 0, 225, 1, 3867, 0, 49, 160, 75, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1888, 0, 2327, 4096, 4092, 6, 0, 1, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1112, 1536, 0, 1082, 1, 277, 0, 0, 0, 0, 8, 63, 0, 0, 0, 1, 3656, 2571, 878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2941, 0, 3956, 398, 242, 3487, 4094, 0, 0, 0, 0, 0, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 708, 329, 22, 0, 4096, 0, 2777, 0, 0, 0, 3405, 4092, 0, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15015, 15993, 21684, 13135, 16447, 11185, 18982, 18631] avg=16384.0000 max=21684 max_rank=2 min=11185 min_rank=5 imbalance_ratio=1.323486 max_load_reduction=109388 max_load_reduction_pct=0.834564 moved_to_echo=65245 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4081, 0, 4082, 184, 0, 3754, 0, 0, 86, 0, 3283, 0, 62, 2, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 2, 0, 0, 2, 0, 0, 35, 0, 1458, 0, 1, 0, 0, 4096, 3340, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 10, 5, 198, 1734, 2855, 0, 1, 0, 1475, 2191, 1373, 813, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3109, 844, 0, 2514, 2, 707, 4092, 0, 4006, 4090, 0, 516, 0, 0, 32, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 3384, 0, 5, 0, 0, 0, 1904, 276, 3270, 361, 1699, 0, 67, 0, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2688, 3793, 3807, 0, 0, 0, 0, 2001, 0, 0, 0, 0, 0, 1, 23, 4, 3687, 987, 2723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 3009, 8, 0, 397, 1, 3669, 1905, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 721, 32, 0, 0, 0, 1336, 0, 4096, 0, 1579, 0, 0, 2871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15884, 13531, 15103, 24008, 15009, 19714, 17187, 10636] avg=16384.0000 max=24008 max_rank=3 min=10636 min_rank=7 imbalance_ratio=1.465332 max_load_reduction=107064 max_load_reduction_pct=0.816833 moved_to_echo=62546 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4009, 3, 4092, 0, 4, 0, 0, 0, 214, 2340, 3288, 0, 3027, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 275, 0, 28, 0, 51, 0, 0, 0, 0, 4096, 3264, 1526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 0, 0, 0, 730, 4022, 4, 4059, 1193, 9, 22, 0, 449, 2176, 1373, 808, 880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3537, 1, 365, 696, 0, 0, 2347, 4091, 3659, 0, 0, 0, 58, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3754, 0, 91, 0, 0, 0, 1899, 3, 6, 3928, 0, 2625, 311, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 759, 0, 4084, 0, 0, 2479, 0, 0, 0, 1, 0, 0, 3631, 2570, 2723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 84, 0, 4077, 88, 0, 395, 1, 0, 4077, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 182, 4087, 0, 0, 0, 3138, 0, 2593, 0, 47, 0, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16979, 13334, 16553, 15804, 16779, 20334, 17267, 14022] avg=16384.0000 max=20334 max_rank=5 min=13334 min_rank=1 imbalance_ratio=1.241089 max_load_reduction=110738 max_load_reduction_pct=0.844864 moved_to_echo=68402 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[308, 8, 3351, 4057, 2321, 3939, 0, 0, 4061, 0, 44, 3981, 0, 16, 0, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 698, 0, 3441, 0, 0, 78, 2, 4058, 0, 1, 31, 0, 115, 0, 0, 4096, 3372, 1441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 824, 5, 450, 0, 3996, 2, 0, 0, 0, 0, 4, 9, 0, 2, 2034, 1322, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 2340, 8, 719, 4091, 0, 2, 4089, 0, 0, 0, 210, 2, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 157, 2, 10, 116, 1775, 2, 35, 3948, 2841, 3778, 0, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 118, 6, 0, 0, 38, 5, 0, 19, 104, 0, 4096, 3429, 2429, 2626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 80, 0, 374, 419, 1616, 3813, 0, 0, 0, 4095, 4074, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 664, 0, 0, 4047, 0, 0, 131, 2, 63, 4081, 3745, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22925, 21429, 8671, 16039, 13349, 12873, 18961, 16825] avg=16384.0000 max=22925 max_rank=0 min=8671 min_rank=2 imbalance_ratio=1.399231 max_load_reduction=108147 max_load_reduction_pct=0.825096 moved_to_echo=62008 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 163, 4092, 42, 0, 98, 288, 1674, 988, 3785, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 46, 106, 0, 0, 0, 168, 36, 0, 0, 2476, 4096, 202, 0, 4065, 2075, 1474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 855, 1, 234, 0, 0, 0, 270, 0, 545, 0, 226, 1, 2187, 1024, 242, 765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 4050, 0, 3684, 0, 443, 0, 17, 31, 568, 31, 0, 0, 0, 37, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 4096, 0, 8, 0, 569, 1909, 8, 0, 4095, 0, 2781, 0, 0, 3630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 921, 0, 0, 21, 3619, 109, 106, 0, 5, 0, 368, 0, 109, 0, 3530, 2485, 2032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 3826, 385, 0, 0, 63, 33, 11, 0, 0, 4091, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2532, 0, 3818, 4032, 443, 0, 4082, 0, 2730, 1, 0, 54, 3043, 4082, 0, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15345, 18840, 6358, 13005, 21000, 13305, 16356, 26863] avg=16384.0000 max=26863 max_rank=7 min=6358 min_rank=2 imbalance_ratio=1.639587 max_load_reduction=104209 max_load_reduction_pct=0.795052 moved_to_echo=77524 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 277, 0, 192, 37, 27, 41, 0, 17, 3273, 2764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1969, 0, 0, 0, 1, 0, 4013, 0, 25, 0, 0, 0, 14, 4096, 3366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 4086, 0, 0, 9, 4096, 87, 0, 2941, 4092, 17, 255, 0, 0, 2178, 1346, 805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 91, 4094, 0, 718, 302, 28, 4096, 4096, 0, 0, 0, 0, 187, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 3690, 0, 1, 15, 0, 0, 1901, 3342, 1, 0, 0, 561, 1, 612, 3455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 3612, 0, 3943, 0, 853, 1310, 0, 0, 0, 0, 1582, 0, 0, 4096, 3402, 0, 2674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1668, 5, 0, 4013, 70, 128, 371, 4091, 3537, 4095, 441, 0, 0, 0, 92, 2108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 126, 0, 0, 2, 0, 0, 1, 3981, 0, 1133, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10718, 13489, 19940, 17710, 14061, 25542, 20619, 8993] avg=16384.0000 max=25542 max_rank=5 min=8993 min_rank=7 imbalance_ratio=1.558960 max_load_reduction=105530 max_load_reduction_pct=0.805130 moved_to_echo=69215 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 2291, 1999, 0, 359, 110, 4021, 74, 2000, 3089, 0, 57, 0, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 15, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 671, 3933, 3376, 1487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 4096, 15, 0, 0, 0, 4093, 8, 12, 0, 4096, 1, 0, 0, 0, 1358, 8, 1031, 759, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1127, 7, 720, 3, 2596, 3598, 3993, 0, 0, 0, 0, 0, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 590, 4, 5, 0, 8, 4, 225, 4092, 4096, 1469, 0, 0, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 1527, 3681, 3, 247, 4015, 3, 4, 8, 2086, 1, 0, 106, 0, 2910, 2506, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3103, 0, 0, 317, 232, 0, 4092, 85, 1, 0, 0, 1557, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 4069, 224, 0, 3421, 0, 3199, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18126, 13568, 17460, 15911, 14372, 19185, 13480, 18970] avg=16384.0000 max=19185 max_rank=5 min=13480 min_rank=6 imbalance_ratio=1.170959 max_load_reduction=111887 max_load_reduction_pct=0.853630 moved_to_echo=66007 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4086, 0, 561, 4092, 3926, 0, 0, 0, 409, 0, 3006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1218, 8, 2644, 96, 2, 6, 0, 4093, 10, 134, 0, 0, 4067, 0, 0, 0, 4096, 3207, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 1343, 0, 4095, 3, 6, 2245, 0, 709, 16, 1666, 3302, 3, 125, 8, 4090, 1979, 0, 739, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 684, 3, 1, 3885, 2643, 0, 251, 0, 0, 13, 3884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3791, 3549, 12, 1, 0, 0, 0, 1727, 3951, 432, 0, 0, 2465, 31, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 768, 4058, 0, 0, 1791, 7, 1, 6, 0, 0, 0, 103, 0, 0, 0, 0, 0, 2200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 660, 1812, 0, 51, 0, 0, 13, 4089, 1261, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 4096, 716, 0, 7, 4095, 0, 3198, 0, 0, 8, 3753, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16080, 20888, 25303, 15460, 20055, 8934, 8220, 16132] avg=16384.0000 max=25303 max_rank=2 min=8220 min_rank=6 imbalance_ratio=1.544373 max_load_reduction=105769 max_load_reduction_pct=0.806953 moved_to_echo=53341 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[3200, 4992, 4352, 22016, 384, 5376, 8576, 2688, 18304, 23680, 7680, 768, 128, 384, 512, 29056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[5248, 5504, 896, 13184, 2560, 3840, 14720, 4864, 4864, 256, 18816, 15232, 640, 512, 1152, 7424, 14976, 7936, 4864, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2944, 0, 16384, 10880, 4864, 128, 19840, 0, 16384, 256, 10752, 128, 17152, 12032, 128, 20224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[24192, 768, 2688, 9728, 384, 768, 128, 4480, 4352, 4352, 128, 1664, 6656, 4224, 256, 768, 32256, 25216, 9472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[23040, 128, 18816, 10624, 14720, 8192, 256, 4480, 5248, 7040, 19840, 14592, 3584, 128, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[8704, 18304, 4224, 896, 128, 128, 768, 13056, 7680, 6400, 20608, 12800, 13696, 512, 1152, 23168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[2048, 768, 5888, 21248, 384, 7936, 2688, 5120, 16512, 23424, 896, 128, 0, 12288, 9856, 22656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[8704, 13568, 8448, 2688, 10240, 512, 8832, 7552, 128, 2560, 128, 2304, 2048, 128, 256, 8320, 24320, 15872, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[62, 3761, 3985, 0, 129, 4083, 0, 10, 3052, 4094, 0, 0, 0, 1, 0, 0, 0, 61, 0, 134, 0, 1741, 678, 0, 0, 0, 4092, 0, 0, 4089, 379, 958, 0, 1, 10, 1514, 0, 17, 0, 279, 0, 3930, 344, 0, 0, 3995, 0, 3594, 4077, 11, 38, 0, 4072, 10, 7, 5, 3858, 0, 1, 1898, 16, 3986, 0, 0, 3681, 4059, 0, 0, 4095, 205, 0, 0, 3879, 0, 0, 21, 2, 0, 18, 0, 2, 7, 0, 40, 0, 0, 3902, 12, 113, 0, 0, 1740, 16, 4096, 68, 0, 537, 0, 4096, 0, 0, 172, 2674, 0, 3805, 0, 0, 47, 3110, 928, 4096, 0, 4096, 0, 0, 0, 29, 13, 3915, 0, 2, 4085, 5, 4096, 2336, 9, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[55, 3652, 4091, 0, 0, 3554, 41, 0, 238, 4000, 0, 0, 18, 77, 0, 725, 0, 0, 74, 617, 1199, 1, 0, 0, 0, 0, 1001, 4092, 0, 377, 0, 0, 0, 2765, 3786, 0, 32, 3, 449, 741, 186, 21, 0, 0, 0, 3613, 4096, 3859, 0, 447, 2514, 0, 4070, 4, 0, 0, 0, 112, 3421, 0, 0, 4036, 2, 0, 3904, 4093, 3548, 130, 3213, 0, 0, 0, 3249, 0, 0, 0, 0, 0, 0, 2029, 523, 0, 4096, 0, 153, 1, 0, 20, 0, 84, 0, 4037, 0, 4093, 4, 0, 0, 4, 3783, 0, 75, 0, 0, 0, 3844, 0, 0, 0, 4096, 4096, 1506, 0, 0, 3984, 3946, 3876, 0, 29, 1535, 0, 4085, 346, 19, 1, 3975, 60, 0, 666] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4096, 3944, 2, 0, 7, 278, 0, 0, 4085, 3678, 2998, 0, 0, 4066, 0, 379, 0, 0, 1983, 3492, 4, 4020, 0, 2541, 0, 0, 4096, 2311, 0, 997, 0, 0, 0, 4087, 4091, 0, 6, 1, 4096, 61, 0, 4091, 0, 0, 0, 4096, 4094, 900, 0, 0, 1, 0, 2085, 1, 0, 0, 4089, 12, 0, 0, 0, 713, 6, 95, 4025, 63, 0, 4, 0, 275, 0, 0, 0, 0, 0, 4017, 2468, 0, 4062, 0, 0, 0, 3, 0, 0, 9, 0, 393, 0, 3486, 0, 4032, 0, 2, 1467, 0, 0, 3123, 0, 0, 1150, 0, 0, 0, 4096, 0, 0, 0, 76, 353, 161, 0, 3762, 0, 3841, 3928, 0, 0, 2941, 0, 2553, 0, 379, 4025, 0, 21, 0, 855] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 1552, 18, 7, 0, 0, 11, 0, 4080, 0, 0, 0, 167, 0, 2, 116, 0, 0, 2951, 1, 0, 248, 0, 0, 14, 3295, 600, 6, 16, 314, 8, 0, 0, 3236, 0, 0, 198, 0, 0, 0, 299, 0, 0, 1874, 0, 564, 1032, 2717, 833, 473, 4004, 3769, 0, 2142, 26, 20, 0, 544, 0, 0, 4096, 24, 1, 3895, 4064, 2, 21, 9, 4096, 0, 383, 362, 78, 4096, 4052, 0, 160, 1044, 3, 126, 0, 4087, 0, 4065, 3745, 0, 1309, 2, 0, 3604, 422, 0, 4096, 0, 29, 1, 0, 0, 3486, 3954, 0, 4095, 0, 1946, 0, 0, 0, 45, 1562, 2717, 54, 1541, 0, 4094, 0, 0, 4063, 0, 0, 2377, 602, 2749, 4093, 2556, 0, 0, 4033] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[3, 3970, 119, 0, 0, 4020, 0, 18, 35, 18, 122, 23, 0, 374, 257, 918, 0, 0, 0, 163, 484, 33, 2871, 4058, 4094, 0, 3957, 0, 0, 1, 2, 0, 0, 41, 4096, 0, 0, 0, 2, 3848, 0, 0, 31, 243, 0, 3795, 16, 0, 2748, 0, 0, 344, 4063, 0, 18, 0, 0, 0, 0, 0, 977, 4093, 1113, 0, 0, 4092, 0, 3855, 4085, 89, 0, 0, 3648, 6, 299, 0, 0, 0, 1811, 76, 0, 0, 2737, 4047, 4057, 9, 3, 0, 0, 4086, 50, 3743, 0, 4032, 0, 0, 0, 1, 0, 459, 0, 0, 1, 0, 0, 648, 3995, 0, 0, 4096, 4067, 1, 4070, 3940, 2994, 4090, 0, 0, 3528, 95, 43, 4095, 0, 1, 0, 0, 0, 3255] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 4085, 0, 0, 4089, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 2313, 4073, 0, 141, 0, 4095, 0, 0, 1, 0, 141, 3271, 4, 0, 0, 4096, 4049, 0, 506, 0, 4096, 0, 0, 4096, 0, 0, 466, 3770, 0, 4096, 34, 2, 1976, 4095, 3855, 0, 0, 224, 0, 37, 0, 0, 1, 3755, 0, 0, 3949, 0, 2654, 0, 0, 0, 0, 0, 4096, 0, 156, 1, 4080, 3610, 339, 3, 0, 8, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4071, 0, 4095, 0, 22, 0, 0, 0, 777, 0, 0, 4091, 3910, 3094, 0, 2604, 250, 3576, 0, 0, 0, 3420, 4052, 958, 0, 611, 2, 0, 8, 2034, 0, 19, 1060, 0, 0, 0, 3953] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 4077, 0, 0, 0, 0, 0, 2019, 20, 102, 0, 0, 35, 3376, 0, 48, 0, 6, 6, 0, 0, 0, 434, 15, 0, 0, 3935, 0, 8, 109, 99, 0, 0, 1682, 3601, 0, 0, 0, 388, 0, 0, 0, 0, 0, 0, 758, 0, 4081, 2041, 119, 0, 0, 4008, 0, 0, 1104, 3231, 38, 18, 3559, 0, 1863, 86, 0, 4050, 3845, 690, 333, 536, 3003, 0, 0, 1013, 276, 0, 0, 0, 4090, 4085, 42, 4010, 0, 3118, 0, 4057, 0, 2, 394, 0, 388, 0, 3377, 0, 4032, 0, 0, 20, 0, 800, 0, 3607, 0, 3647, 50, 2896, 8, 104, 0, 4082, 1164, 3718, 0, 46, 4089, 3384, 4096, 650, 2546, 1061, 2, 1180, 4096, 0, 3475, 0, 48, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1120, 3505, 200, 97, 518, 71, 0, 0, 4092, 3906, 0, 0, 0, 1, 0, 0, 0, 7, 4095, 0, 0, 0, 0, 3327, 0, 0, 3744, 87, 0, 1, 0, 2, 370, 3258, 4052, 0, 0, 0, 3742, 4096, 7, 581, 0, 0, 1788, 4003, 2397, 31, 168, 76, 0, 0, 4022, 6, 0, 109, 35, 2497, 132, 0, 18, 4080, 0, 0, 4090, 4096, 0, 0, 0, 1873, 0, 0, 4081, 0, 2936, 0, 0, 5, 3336, 0, 0, 55, 0, 66, 0, 0, 26, 1825, 0, 0, 0, 4095, 0, 4096, 0, 2, 0, 0, 307, 0, 0, 142, 3900, 852, 2410, 0, 58, 405, 33, 135, 4096, 0, 15, 4096, 4094, 0, 1, 4096, 3673, 0, 4041, 0, 0, 3766, 45, 740, 0, 3344] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[62, 3761, 3985, 0, 129, 4083, 0, 10, 3052, 4094, 0, 0, 0, 1, 0, 0, 1786, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 134, 0, 1741, 678, 0, 0, 0, 4092, 0, 0, 4089, 379, 958, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 1514, 0, 17, 0, 279, 0, 3930, 344, 0, 0, 3995, 0, 3594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 11, 38, 0, 4072, 10, 7, 5, 3858, 0, 1, 1898, 16, 3986, 0, 0, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 819, 4059, 0, 0, 4095, 205, 0, 0, 3879, 0, 0, 21, 2, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7, 0, 40, 0, 0, 3902, 12, 113, 0, 0, 1740, 16, 4096, 68, 0, 1572, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 0, 4096, 0, 0, 172, 2674, 0, 3805, 0, 0, 47, 3110, 928, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 29, 13, 3915, 0, 2, 4085, 5, 1333, 2336, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20964, 16225, 13682, 19269, 13098, 12545, 19466, 15823] avg=16384.0000 max=20964 max_rank=0 min=12545 min_rank=5 imbalance_ratio=1.279541 max_load_reduction=110108 max_load_reduction_pct=0.840057 moved_to_echo=60932 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3, 3970, 119, 0, 0, 4020, 0, 18, 35, 18, 122, 23, 0, 374, 257, 918, 0, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 484, 33, 2871, 4058, 4094, 0, 3957, 0, 0, 1, 2, 0, 3255, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 3036, 0, 0, 0, 2, 3848, 0, 0, 31, 243, 0, 3795, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2748, 0, 0, 344, 4063, 0, 18, 0, 0, 0, 0, 0, 977, 4093, 1113, 0, 0, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3855, 4085, 89, 0, 0, 3648, 6, 299, 0, 0, 0, 1811, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2737, 4047, 4057, 9, 3, 0, 0, 4086, 50, 3743, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 459, 0, 0, 1, 0, 0, 648, 3995, 0, 0, 4096, 4067, 1, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 3940, 0, 4090, 0, 0, 3528, 95, 43, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10423, 20805, 11012, 14463, 17961, 22764, 13782, 19862] avg=16384.0000 max=22764 max_rank=5 min=10423 min_rank=0 imbalance_ratio=1.389404 max_load_reduction=108308 max_load_reduction_pct=0.826324 moved_to_echo=74369 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1552, 18, 7, 0, 0, 11, 0, 4080, 0, 0, 0, 167, 0, 2, 1785, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 2951, 1, 0, 248, 0, 0, 14, 3295, 600, 6, 16, 314, 8, 4033, 2581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2399, 0, 0, 198, 0, 0, 0, 299, 0, 0, 1874, 0, 564, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2717, 833, 473, 4004, 3769, 0, 2142, 26, 20, 0, 544, 0, 0, 4096, 24, 1, 1365, 1513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 867, 4064, 2, 21, 9, 4096, 0, 383, 362, 78, 4096, 4052, 0, 160, 1044, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 4087, 0, 4065, 3745, 0, 1309, 2, 0, 3604, 422, 0, 4096, 0, 29, 1663, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3486, 3954, 0, 4095, 0, 1946, 0, 0, 0, 45, 1562, 2717, 54, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1541, 0, 0, 0, 0, 4063, 0, 0, 2377, 602, 2749, 1332, 2556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12149, 14183, 6366, 21527, 19237, 24124, 18266, 15220] avg=16384.0000 max=24124 max_rank=5 min=6366 min_rank=2 imbalance_ratio=1.472412 max_load_reduction=106948 max_load_reduction_pct=0.815948 moved_to_echo=76847 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 4085, 0, 0, 4089, 0, 4096, 9, 0, 0, 0, 0, 462, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2313, 4073, 0, 141, 0, 4095, 0, 0, 1, 0, 141, 3271, 4, 0, 3953, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3001, 0, 506, 0, 4096, 0, 0, 4096, 0, 0, 466, 3770, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 2, 1976, 4095, 3855, 0, 0, 224, 0, 37, 0, 0, 1, 3755, 0, 0, 1384, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 879, 0, 2654, 0, 0, 0, 0, 0, 4096, 0, 156, 1, 4080, 3610, 339, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4071, 0, 4095, 0, 22, 1686, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 777, 0, 0, 4091, 3910, 3094, 0, 2604, 250, 3576, 0, 0, 0, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3420, 4052, 0, 0, 611, 2, 0, 8, 2034, 0, 19, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17376, 18595, 24127, 15718, 15818, 10136, 18810, 10492] avg=16384.0000 max=24127 max_rank=2 min=10136 min_rank=5 imbalance_ratio=1.472595 max_load_reduction=106945 max_load_reduction_pct=0.815926 moved_to_echo=55256 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[55, 3652, 4091, 0, 0, 3554, 41, 0, 238, 4000, 0, 0, 18, 77, 0, 725, 0, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 617, 1199, 1, 0, 0, 0, 0, 1001, 4092, 0, 377, 0, 0, 666, 2488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2765, 2806, 0, 32, 3, 449, 741, 186, 21, 0, 0, 0, 3613, 4096, 3859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 447, 2514, 0, 4070, 4, 0, 0, 0, 112, 3421, 0, 0, 4036, 2, 0, 1368, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 869, 4093, 3548, 130, 3213, 0, 0, 0, 3249, 0, 0, 0, 0, 0, 0, 2029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 0, 4096, 0, 153, 1, 0, 20, 0, 84, 0, 4037, 0, 4093, 4, 0, 1667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3783, 0, 75, 0, 0, 0, 3844, 0, 0, 0, 4096, 4096, 1506, 0, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 0, 3876, 0, 29, 1535, 0, 4085, 346, 19, 1, 3975, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16956, 10515, 18571, 17432, 17131, 14678, 17879, 17910] avg=16384.0000 max=18571 max_rank=2 min=10515 min_rank=1 imbalance_ratio=1.133484 max_load_reduction=112501 max_load_reduction_pct=0.858315 moved_to_echo=67598 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4077, 0, 0, 0, 0, 0, 2019, 20, 102, 0, 0, 35, 3376, 0, 48, 1515, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 434, 15, 0, 0, 3935, 0, 8, 109, 99, 0, 4096, 2133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1682, 2669, 0, 0, 0, 388, 0, 0, 0, 0, 0, 0, 758, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2041, 119, 0, 0, 4008, 0, 0, 1104, 3231, 38, 18, 3559, 0, 1863, 86, 0, 1419, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 901, 3845, 690, 333, 536, 3003, 0, 0, 1013, 276, 0, 0, 0, 4090, 4085, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 3118, 0, 4057, 0, 2, 394, 0, 388, 0, 3377, 0, 4032, 0, 0, 1730, 829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 800, 0, 3607, 0, 3647, 50, 2896, 8, 104, 0, 4082, 1164, 3718, 0, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 4089, 0, 4096, 650, 2546, 1061, 2, 1180, 4096, 0, 1131, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11672, 10841, 9578, 18737, 18814, 21937, 20548, 18945] avg=16384.0000 max=21937 max_rank=5 min=9578 min_rank=2 imbalance_ratio=1.338928 max_load_reduction=109135 max_load_reduction_pct=0.832634 moved_to_echo=80244 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 3944, 2, 0, 7, 278, 0, 0, 4085, 3678, 2998, 0, 0, 4066, 0, 379, 1759, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1983, 3492, 4, 4020, 0, 2541, 0, 0, 4096, 2311, 0, 997, 0, 0, 855, 2422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 3027, 0, 6, 1, 4096, 61, 0, 4091, 0, 0, 0, 4096, 4094, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2085, 1, 0, 0, 4089, 12, 0, 0, 0, 713, 6, 95, 1415, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 887, 63, 0, 4, 0, 275, 0, 0, 0, 0, 0, 4017, 2468, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 9, 0, 393, 0, 3486, 0, 4032, 0, 2, 1467, 0, 1723, 963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3123, 0, 0, 1150, 0, 0, 0, 4096, 0, 0, 0, 76, 353, 161, 0, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3762, 0, 0, 3928, 0, 0, 2941, 0, 2553, 0, 379, 1303, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[25840, 22721, 24459, 9836, 11776, 12078, 9475, 14887] avg=16384.0000 max=25840 max_rank=0 min=9475 min_rank=6 imbalance_ratio=1.577148 max_load_reduction=105232 max_load_reduction_pct=0.802856 moved_to_echo=48216 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1120, 3505, 200, 97, 518, 71, 0, 0, 4092, 3906, 0, 0, 0, 1, 0, 0, 1642, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4095, 0, 0, 0, 0, 3327, 0, 0, 3744, 87, 0, 1, 0, 2, 3344, 2580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 3258, 3004, 0, 0, 0, 3742, 4096, 7, 581, 0, 0, 1788, 4003, 2397, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 76, 0, 0, 4022, 6, 0, 109, 35, 2497, 132, 0, 18, 4080, 0, 0, 1433, 1514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 910, 4096, 0, 0, 0, 1873, 0, 0, 4081, 0, 2936, 0, 0, 5, 3336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 66, 0, 0, 26, 1825, 0, 0, 0, 4095, 0, 4096, 0, 2, 1747, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 0, 0, 142, 3900, 852, 2410, 0, 58, 405, 33, 135, 4096, 0, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 1, 4096, 3673, 0, 4041, 0, 0, 1226, 45, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15692, 17187, 23277, 14090, 17237, 12810, 12846, 17933] avg=16384.0000 max=23277 max_rank=2 min=12810 min_rank=5 imbalance_ratio=1.420715 max_load_reduction=107795 max_load_reduction_pct=0.822411 moved_to_echo=60826 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[6144, 24320, 6912, 4352, 12032, 9600, 0, 384, 20352, 384, 7552, 8192, 6656, 7936, 14720, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[17024, 20224, 0, 16000, 1408, 10752, 16768, 128, 16384, 13312, 3200, 6784, 8960, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[11904, 1536, 5120, 8448, 29952, 128, 2176, 1536, 11264, 2816, 4224, 5504, 1024, 26624, 1280, 128, 9728, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[5376, 27008, 9984, 128, 4864, 16128, 128, 2176, 15616, 19968, 7296, 128, 128, 8064, 384, 2176, 8960, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[384, 16000, 19968, 1536, 640, 256, 12800, 9088, 256, 13056, 384, 256, 4224, 24064, 11264, 17664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[640, 3200, 9088, 4736, 8832, 384, 18432, 4864, 22144, 768, 6784, 768, 15104, 12416, 20480, 128, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 8576, 11520, 1792, 6016, 4352, 14080, 4096, 128, 24192, 7168, 256, 8960, 896, 1024, 24320, 14720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[4736, 128, 14080, 4224, 12416, 3840, 3968, 3968, 128, 8064, 3712, 25600, 128, 28544, 1664, 128, 11904, 4992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 24, 22, 724, 160, 0, 34, 0, 3099, 0, 0, 4, 1, 0, 0, 132, 4096, 4091, 476, 424, 138, 0, 1666, 6, 0, 163, 301, 0, 0, 0, 32, 0, 0, 0, 4020, 0, 0, 2234, 1, 0, 4096, 3678, 0, 2690, 3783, 0, 0, 4040, 4096, 0, 0, 0, 0, 2, 10, 0, 111, 0, 89, 1, 4079, 3805, 0, 3926, 724, 4096, 0, 1255, 0, 358, 3803, 3108, 0, 4095, 0, 62, 4086, 0, 0, 295, 1, 3975, 0, 0, 0, 936, 0, 1506, 3933, 0, 988, 0, 2859, 1040, 64, 3904, 4082, 0, 4006, 0, 19, 2233, 0, 0, 10, 2982, 4092, 8, 17, 0, 57, 0, 7, 0, 77, 0, 4084, 0, 0, 109, 0, 10, 3745, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 1103, 14, 133, 614, 2889, 2, 4096, 0, 0, 80, 0, 0, 3288, 1064, 8, 4096, 453, 4070, 4074, 0, 4096, 1413, 140, 6, 0, 0, 0, 1808, 4081, 3967, 0, 0, 2963, 4089, 0, 1, 0, 3885, 0, 1394, 0, 2, 0, 2610, 526, 0, 2654, 4057, 6, 31, 0, 3521, 0, 25, 2228, 14, 0, 209, 0, 3825, 0, 15, 2435, 0, 3975, 1953, 1312, 109, 3, 929, 2019, 4016, 4049, 0, 167, 0, 0, 0, 0, 42, 4077, 278, 0, 0, 2, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 3994, 3193, 0, 3941, 0, 0, 3791, 0, 1, 0, 2942, 4045, 17, 1, 71, 187, 0, 384, 157, 1120, 0, 3118, 0, 945, 0, 60, 85, 0, 0, 8, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 4092, 0, 1, 41, 2607, 6, 3, 0, 0, 1, 3311, 2762, 4089, 0, 6, 4095, 0, 184, 0, 0, 190, 0, 0, 14, 0, 57, 0, 0, 1932, 38, 0, 4064, 0, 265, 4090, 0, 0, 1351, 2433, 3269, 2215, 0, 4067, 3666, 226, 4073, 178, 3888, 0, 20, 0, 265, 4059, 0, 0, 0, 1240, 4, 2, 131, 0, 0, 4056, 5, 0, 320, 3908, 0, 3950, 0, 17, 2870, 83, 0, 2, 0, 4096, 1, 2359, 0, 2596, 0, 0, 0, 0, 0, 3692, 3850, 0, 0, 35, 6, 1755, 8, 4089, 4093, 0, 0, 4078, 4045, 26, 0, 0, 348, 26, 4041, 128, 0, 4030, 299, 0, 2, 38, 29, 0, 1311, 0, 34, 3979, 0, 55, 3842, 1, 0, 0, 0, 34] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 3384, 0, 0, 0, 0, 83, 12, 0, 384, 0, 0, 20, 0, 12, 0, 4096, 3, 4053, 0, 0, 0, 0, 0, 0, 0, 761, 0, 2, 4096, 4090, 0, 14, 2199, 4091, 9, 0, 401, 3935, 10, 3, 4095, 1, 1916, 0, 0, 0, 4096, 4057, 7, 1729, 0, 0, 0, 0, 3982, 192, 0, 282, 1508, 1837, 2050, 0, 2362, 0, 1, 3861, 0, 11, 6, 0, 3849, 4013, 15, 0, 0, 48, 702, 0, 48, 944, 1230, 9, 0, 4095, 0, 0, 4095, 4096, 0, 3798, 0, 100, 3996, 120, 4095, 3, 0, 4095, 0, 236, 28, 8, 351, 4096, 598, 4093, 4096, 2, 3943, 0, 1, 0, 101, 3636, 0, 0, 0, 3804, 14, 0, 0, 1, 154, 0, 0, 0, 2908] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 3964, 1, 0, 0, 0, 0, 190, 0, 17, 0, 0, 3719, 809, 0, 0, 4083, 0, 4095, 34, 0, 432, 0, 3965, 54, 0, 3937, 6, 0, 423, 69, 0, 0, 135, 1228, 2548, 0, 0, 4093, 2, 4096, 3617, 0, 0, 2721, 0, 0, 4096, 0, 1014, 3823, 0, 3914, 1188, 0, 0, 6, 0, 0, 0, 3951, 3910, 302, 4096, 109, 4093, 1, 3071, 0, 0, 0, 4048, 0, 0, 0, 2, 0, 0, 2530, 1937, 498, 3962, 167, 0, 4093, 0, 0, 3993, 2020, 1480, 0, 8, 2044, 3871, 12, 4096, 103, 1, 0, 410, 1181, 3694, 54, 0, 161, 475, 0, 0, 0, 3731, 0, 0, 31, 350, 3332, 772, 0, 0, 0, 8, 0, 9, 0, 0, 0, 0, 91, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 0, 24, 116, 0, 79, 0, 5, 0, 3465, 3828, 0, 875, 4094, 3686, 0, 0, 0, 0, 262, 0, 0, 0, 0, 0, 93, 564, 0, 0, 0, 0, 734, 0, 0, 4092, 4096, 0, 4096, 3977, 28, 0, 4087, 0, 3997, 107, 468, 0, 0, 0, 4054, 47, 0, 111, 7, 2773, 0, 44, 3453, 4096, 0, 245, 1634, 0, 0, 0, 0, 4047, 0, 1517, 3024, 0, 0, 0, 2, 3517, 3938, 0, 0, 4095, 1180, 1, 411, 0, 0, 3033, 1665, 0, 0, 1629, 3685, 4096, 1, 4091, 0, 0, 4096, 0, 1, 4085, 424, 3, 0, 4094, 20, 0, 0, 0, 4059, 3, 0, 0, 8, 0, 737, 0, 4095, 3524, 45, 0, 2637, 0, 0, 0, 1, 4064] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 22, 338, 0, 0, 339, 4054, 233, 0, 0, 1525, 0, 2458, 4096, 0, 19, 4083, 0, 0, 0, 0, 633, 0, 0, 0, 0, 3936, 6, 0, 355, 0, 0, 3174, 887, 0, 96, 6, 3349, 4096, 0, 5, 4009, 6, 4092, 0, 54, 356, 4096, 4096, 0, 0, 0, 4073, 4096, 0, 1, 7, 25, 0, 4090, 4031, 102, 0, 2554, 32, 0, 0, 0, 0, 0, 0, 0, 193, 0, 67, 0, 15, 0, 0, 0, 0, 4083, 811, 4, 4096, 0, 0, 3627, 4095, 0, 0, 86, 6, 192, 0, 4080, 0, 0, 1253, 0, 3954, 2859, 3733, 231, 0, 4095, 4096, 0, 0, 753, 4036, 88, 0, 682, 32, 0, 2597, 0, 103, 16, 4096, 0, 0, 3761, 0, 0, 5, 3897] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 1731, 0, 0, 3, 1848, 0, 0, 3977, 0, 0, 1, 9, 2448, 0, 27, 1851, 3046, 3985, 599, 0, 4096, 0, 0, 0, 24, 0, 0, 0, 4062, 40, 0, 0, 0, 4088, 0, 0, 3903, 2574, 0, 4061, 334, 0, 2309, 4083, 0, 0, 4042, 4069, 0, 0, 0, 0, 4095, 8, 143, 50, 56, 48, 915, 2796, 6, 0, 4096, 865, 2690, 610, 24, 0, 0, 531, 29, 3950, 5, 0, 0, 4014, 244, 3284, 9, 828, 2284, 0, 0, 2688, 13, 0, 0, 3290, 0, 225, 36, 4021, 2119, 0, 4095, 0, 0, 4096, 1, 10, 0, 2, 0, 2, 2008, 4096, 0, 0, 4065, 0, 7, 1, 36, 3, 0, 4096, 0, 3129, 0, 0, 0, 182, 0, 10, 0, 0, 4051] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1731, 0, 0, 3, 1848, 0, 0, 3977, 0, 0, 1, 9, 2448, 0, 27, 4095, 2153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1851, 3046, 3985, 599, 0, 4096, 0, 0, 0, 24, 0, 0, 0, 4062, 40, 0, 187, 331, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 3903, 0, 0, 4061, 147, 0, 2309, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2434, 0, 0, 0, 0, 4095, 8, 143, 50, 56, 48, 915, 2796, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 865, 2690, 610, 24, 0, 0, 531, 29, 3950, 5, 0, 0, 4014, 244, 3284, 9, 1458, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 2284, 0, 0, 2688, 13, 0, 0, 3096, 0, 225, 36, 4021, 2119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 10, 0, 2, 0, 2, 2008, 1877, 0, 0, 4065, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 36, 3, 0, 4096, 0, 3129, 0, 0, 0, 182, 0, 10, 0, 0, 4051, 4042, 1888, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16292, 18398, 18591, 14647, 17907, 15310, 12068, 17859] avg=16384.0000 max=18591 max_rank=2 min=12068 min_rank=6 imbalance_ratio=1.134705 max_load_reduction=112481 max_load_reduction_pct=0.858162 moved_to_echo=63144 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 0, 24, 116, 0, 79, 0, 5, 0, 3465, 3828, 0, 875, 4091, 3426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3686, 0, 0, 0, 0, 262, 0, 0, 0, 0, 0, 93, 564, 0, 0, 2233, 4, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 734, 0, 0, 4092, 0, 0, 4096, 1744, 28, 0, 4087, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 0, 4054, 47, 0, 111, 7, 2773, 0, 44, 3453, 4096, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1634, 0, 0, 0, 0, 4047, 0, 1517, 3024, 0, 0, 0, 2, 3517, 3938, 0, 172, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1180, 1, 411, 0, 0, 3033, 1563, 0, 0, 1629, 3685, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4085, 424, 3, 0, 4094, 3, 0, 0, 0, 4059, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 737, 0, 4095, 3524, 45, 0, 2637, 0, 0, 0, 1, 4064, 107, 13, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15916, 10960, 18778, 15102, 17953, 19694, 16768, 15901] avg=16384.0000 max=19694 max_rank=5 min=10960 min_rank=1 imbalance_ratio=1.202026 max_load_reduction=111378 max_load_reduction_pct=0.849747 moved_to_echo=70316 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1103, 14, 133, 614, 2889, 2, 4096, 0, 0, 80, 0, 0, 3288, 1064, 8, 3994, 3248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 453, 4070, 4074, 0, 4096, 1413, 140, 6, 0, 0, 0, 1808, 4081, 3967, 0, 0, 327, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2963, 4089, 0, 1, 0, 0, 0, 1394, 0, 2, 0, 2610, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2427, 6, 31, 0, 3521, 0, 25, 2228, 14, 0, 209, 0, 3825, 0, 15, 2435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 1953, 1312, 109, 3, 929, 2019, 4016, 4049, 0, 167, 0, 0, 0, 0, 1453, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4077, 278, 0, 0, 2, 0, 0, 3854, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3193, 0, 3941, 0, 0, 3791, 0, 1, 0, 2942, 1854, 17, 1, 71, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 157, 1120, 0, 3118, 0, 945, 0, 60, 85, 0, 0, 8, 0, 0, 0, 2654, 1864, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20533, 28708, 11585, 14736, 20226, 8254, 15998, 11032] avg=16384.0000 max=28708 max_rank=1 min=8254 min_rank=5 imbalance_ratio=1.752197 max_load_reduction=102364 max_load_reduction_pct=0.780975 moved_to_echo=55510 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 22, 338, 0, 0, 339, 4054, 233, 0, 0, 1525, 0, 2458, 4096, 0, 19, 4080, 3426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 633, 0, 0, 0, 0, 3936, 6, 0, 355, 0, 0, 2247, 331, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3174, 887, 0, 96, 6, 3349, 0, 0, 5, 1762, 6, 4092, 0, 54, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2451, 0, 0, 0, 4073, 4096, 0, 1, 7, 25, 0, 4090, 4031, 102, 0, 2554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 193, 0, 67, 0, 15, 0, 0, 0, 1467, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 811, 4, 4096, 0, 0, 3627, 3854, 0, 0, 86, 6, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 0, 3954, 2859, 3733, 231, 0, 4095, 1877, 0, 0, 753, 4036, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 682, 32, 0, 2597, 0, 103, 16, 4096, 0, 0, 3761, 0, 0, 5, 3897, 4096, 1888, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20590, 11769, 13787, 21430, 2015, 16759, 22879, 21843] avg=16384.0000 max=22879 max_rank=6 min=2015 min_rank=4 imbalance_ratio=1.396423 max_load_reduction=108193 max_load_reduction_pct=0.825447 moved_to_echo=63496 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 1, 41, 2607, 6, 3, 0, 0, 1, 3311, 2762, 4089, 0, 6, 4089, 1129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 184, 0, 0, 190, 0, 0, 14, 0, 57, 0, 0, 1932, 38, 0, 1241, 326, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 265, 4090, 0, 0, 0, 2433, 3269, 974, 0, 4067, 3666, 226, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2326, 0, 20, 0, 265, 4059, 0, 0, 0, 1240, 4, 2, 131, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 320, 3908, 0, 3950, 0, 17, 2870, 83, 0, 2, 0, 4096, 1, 2359, 1393, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2596, 0, 0, 0, 0, 0, 3692, 3623, 0, 0, 35, 6, 1755, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4078, 4045, 26, 0, 0, 348, 26, 1853, 128, 0, 4030, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 38, 29, 0, 1311, 0, 34, 3979, 0, 55, 3842, 1, 0, 0, 0, 34, 178, 1862, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22137, 8246, 27127, 12103, 19231, 11715, 18926, 11587] avg=16384.0000 max=27127 max_rank=2 min=8246 min_rank=1 imbalance_ratio=1.655701 max_load_reduction=103945 max_load_reduction_pct=0.793037 moved_to_echo=61459 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 24, 22, 724, 160, 0, 34, 0, 3099, 0, 0, 4, 1, 0, 0, 132, 3904, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 476, 424, 138, 0, 1666, 6, 0, 163, 301, 0, 0, 0, 32, 0, 2061, 330, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 2234, 0, 0, 4096, 1617, 0, 2690, 3783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2451, 0, 0, 0, 0, 2, 10, 0, 111, 0, 89, 1, 4079, 3805, 0, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 724, 4096, 0, 1255, 0, 358, 3803, 3108, 0, 4095, 0, 62, 4086, 0, 0, 295, 1467, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3975, 0, 0, 0, 936, 0, 1506, 3701, 0, 988, 0, 2859, 1040, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4006, 0, 19, 2233, 0, 0, 10, 2982, 1876, 8, 17, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 77, 0, 4084, 0, 0, 109, 0, 10, 3745, 0, 4096, 0, 0, 4096, 4040, 1886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 13962, 18440, 14474, 23581, 15070, 15290, 22150] avg=16384.0000 max=23581 max_rank=4 min=8105 min_rank=0 imbalance_ratio=1.439270 max_load_reduction=107491 max_load_reduction_pct=0.820091 moved_to_echo=76091 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 3964, 1, 0, 0, 0, 0, 190, 0, 17, 0, 0, 3719, 809, 0, 0, 4096, 3423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4095, 34, 0, 432, 0, 3965, 54, 0, 3937, 6, 0, 423, 69, 0, 2027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 1228, 2548, 0, 0, 0, 2, 4096, 1590, 0, 0, 2721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 3823, 0, 3914, 1188, 0, 0, 6, 0, 0, 0, 3951, 3910, 302, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 4093, 1, 3071, 0, 0, 0, 4048, 0, 0, 0, 2, 0, 0, 2530, 1937, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 3962, 167, 0, 4093, 0, 0, 3993, 1901, 1480, 0, 8, 2044, 3871, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 1, 0, 410, 1181, 3694, 54, 0, 161, 475, 0, 0, 0, 3731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 350, 3332, 772, 0, 0, 0, 8, 0, 9, 0, 0, 0, 0, 91, 4096, 4096, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16219, 19125, 12320, 22204, 15910, 22029, 9810, 13455] avg=16384.0000 max=22204 max_rank=3 min=9810 min_rank=6 imbalance_ratio=1.355225 max_load_reduction=108868 max_load_reduction_pct=0.830597 moved_to_echo=61204 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 3384, 0, 0, 0, 0, 83, 12, 0, 384, 0, 0, 20, 0, 12, 0, 4095, 3290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4053, 0, 0, 0, 0, 0, 0, 0, 761, 0, 2, 4096, 4090, 0, 2295, 331, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2199, 4091, 9, 0, 401, 0, 10, 3, 1800, 1, 1916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2427, 7, 1729, 0, 0, 0, 0, 3982, 192, 0, 282, 1508, 1837, 2050, 0, 2362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3861, 0, 11, 6, 0, 3849, 4013, 15, 0, 0, 48, 702, 0, 48, 1453, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 944, 1230, 9, 0, 4095, 0, 0, 4095, 3854, 0, 3798, 0, 100, 3996, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 236, 28, 8, 351, 4096, 598, 1876, 4096, 2, 3943, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 3636, 0, 0, 0, 3804, 14, 0, 0, 1, 154, 0, 0, 0, 2908, 4096, 1886, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11280, 19904, 10444, 16376, 14249, 22241, 19333, 17245] avg=16384.0000 max=22241 max_rank=5 min=10444 min_rank=2 imbalance_ratio=1.357483 max_load_reduction=108831 max_load_reduction_pct=0.830315 moved_to_echo=73068 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3456, 14976, 6784, 9600, 128, 8448, 5376, 14592, 18176, 8320, 128, 256, 8192, 8576, 9856, 4736, 8960, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[14848, 1152, 5632, 0, 15872, 13568, 128, 6528, 512, 4096, 640, 6656, 24192, 14080, 384, 23808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2432, 26368, 2560, 128, 15488, 1024, 0, 19968, 25472, 1536, 5120, 1920, 12800, 17152, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[11520, 128, 21504, 4608, 9472, 16768, 4224, 640, 4736, 17280, 11264, 4352, 128, 16640, 8704, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 14336, 384, 896, 896, 7808, 4352, 4608, 7168, 512, 1664, 3328, 12544, 18560, 1152, 1152, 32512, 20096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[512, 1408, 8320, 896, 16000, 0, 12160, 7680, 4224, 256, 10496, 3968, 4224, 0, 128, 23168, 23424, 11392, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[7296, 6272, 18560, 6784, 128, 14080, 0, 2560, 21120, 9728, 128, 15104, 20992, 896, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[30592, 11392, 16896, 5248, 256, 9472, 3456, 4224, 128, 256, 9088, 128, 1920, 15616, 8320, 0, 12416, 2048, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 1203, 12, 0, 0, 0, 2684, 4096, 138, 4090, 0, 0, 329, 0, 5, 0, 1, 1105, 0, 0, 4077, 0, 511, 3519, 32, 3786, 0, 3, 0, 59, 0, 4093, 4096, 65, 3, 0, 46, 0, 41, 2898, 4096, 3960, 645, 3765, 0, 0, 5, 4035, 0, 2498, 0, 4092, 4096, 0, 0, 1212, 82, 0, 0, 8, 583, 4094, 1, 0, 4090, 3524, 0, 164, 4052, 40, 4093, 4053, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 16, 0, 5, 0, 2958, 3, 57, 3588, 494, 0, 186, 15, 0, 4062, 0, 2, 132, 0, 0, 1329, 1884, 468, 4006, 0, 13, 0, 1832, 4096, 0, 0, 19, 4095, 4092, 0, 524, 0, 0, 0, 39, 4068, 0, 0, 2895, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 164, 45, 0, 4071, 4096, 1026, 0, 1168, 0, 0, 0, 3254, 409, 0, 0, 4045, 101, 9, 480, 5, 4000, 0, 1, 0, 0, 547, 4092, 4079, 0, 1597, 0, 141, 0, 0, 4096, 2, 0, 0, 0, 17, 0, 318, 4095, 84, 3843, 0, 4, 3947, 0, 3679, 0, 2761, 5, 0, 867, 42, 361, 0, 378, 0, 2099, 268, 4094, 0, 800, 3751, 3302, 10, 0, 11, 2, 7, 4084, 4, 0, 3660, 806, 0, 0, 0, 0, 0, 3, 129, 0, 0, 0, 738, 0, 2, 4068, 17, 4065, 0, 0, 3542, 317, 0, 0, 0, 2810, 0, 0, 0, 0, 6, 0, 3578, 2, 4093, 3961, 0, 0, 144, 4095, 4096, 2721, 0, 8, 1, 2, 199, 3569, 4082, 0, 4094, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 32, 3, 0, 0, 0, 0, 4096, 3985, 4014, 0, 0, 272, 3779, 533, 6, 0, 0, 0, 3159, 0, 0, 1, 0, 2150, 0, 4080, 2, 0, 83, 1082, 0, 0, 4079, 1018, 0, 173, 0, 73, 4096, 7, 0, 3725, 0, 4082, 0, 0, 1165, 4095, 0, 1831, 0, 3480, 2840, 0, 8, 22, 0, 0, 5, 4096, 4090, 4096, 0, 147, 0, 4094, 0, 0, 4047, 0, 0, 4095, 63, 0, 0, 118, 6, 0, 0, 0, 371, 0, 10, 2, 0, 0, 1422, 0, 2164, 4056, 3, 0, 13, 0, 3063, 4014, 648, 0, 1819, 4096, 0, 23, 0, 0, 2480, 0, 26, 0, 4094, 4040, 11, 0, 0, 3930, 3980, 0, 0, 0, 356, 0, 0, 0, 3715, 233, 3675, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[474, 0, 0, 0, 0, 3346, 2901, 4096, 303, 99, 1080, 4017, 7, 0, 4096, 0, 1373, 4086, 1, 0, 0, 0, 3219, 0, 3166, 4073, 0, 4095, 74, 83, 24, 0, 2380, 686, 0, 262, 29, 0, 0, 14, 0, 57, 4079, 3996, 2, 3928, 4096, 0, 74, 5, 0, 2357, 0, 3611, 4, 0, 0, 2, 5, 4024, 1707, 633, 20, 1088, 0, 0, 104, 2523, 3936, 1576, 2, 0, 4077, 0, 72, 116, 0, 0, 4096, 0, 0, 0, 679, 0, 0, 84, 0, 571, 339, 1, 695, 4096, 0, 0, 0, 3, 3716, 300, 30, 0, 0, 0, 0, 0, 0, 248, 3256, 1, 4094, 0, 0, 3734, 0, 937, 3547, 4085, 4060, 534, 10, 0, 0, 0, 0, 3432, 4, 0, 2381, 61] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 74, 0, 2, 3507, 3928, 0, 73, 1858, 0, 14, 2828, 454, 0, 1592, 77, 0, 243, 0, 0, 48, 213, 0, 0, 4092, 289, 0, 4096, 0, 2, 3, 4096, 0, 4, 1917, 0, 292, 0, 2, 0, 0, 0, 3961, 4070, 56, 4091, 1676, 12, 0, 205, 4, 0, 0, 4096, 0, 139, 0, 65, 10, 0, 4, 1, 8, 4092, 2, 0, 3823, 4065, 0, 0, 4096, 0, 3496, 651, 2477, 0, 34, 0, 1081, 0, 205, 233, 0, 4092, 0, 11, 1079, 12, 23, 36, 0, 0, 0, 3503, 4096, 1476, 0, 0, 2858, 0, 2710, 4096, 4003, 0, 622, 353, 4053, 0, 1421, 0, 3947, 0, 0, 0, 0, 3681, 2934, 14, 3760, 0, 0, 0, 0, 4071, 1673, 0, 4089, 102] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 70, 3962, 68, 4096, 0, 2872, 3169, 2, 220, 0, 0, 0, 0, 0, 4004, 4, 1, 0, 0, 0, 0, 3675, 648, 4092, 4096, 461, 0, 1040, 11, 370, 1, 88, 1, 0, 3860, 3769, 2, 45, 4030, 3985, 3968, 0, 3978, 0, 0, 1038, 3367, 3780, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 507, 1, 941, 105, 4003, 0, 3633, 0, 4074, 3691, 0, 4024, 4033, 13, 0, 346, 0, 0, 0, 0, 6, 0, 141, 0, 0, 0, 233, 23, 0, 285, 2741, 0, 104, 1620, 0, 0, 2546, 111, 3719, 0, 679, 0, 0, 0, 0, 4094, 0, 3998, 8, 10, 4096, 0, 0, 36, 4096, 4087, 136, 0, 0, 0, 23, 0, 0, 4082, 426, 3626, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 3, 3939, 3398, 0, 4092, 0, 4054, 3511, 13, 2529, 0, 0, 10, 0, 286, 136, 0, 3992, 3645, 0, 4096, 0, 4096, 0, 87, 4058, 4072, 1244, 15, 0, 526, 3858, 0, 1131, 0, 4068, 3588, 0, 2801, 3915, 0, 232, 3685, 1, 9, 949, 1778, 4, 1, 0, 2464, 0, 0, 1067, 3625, 2, 1342, 0, 3851, 0, 4088, 4, 0, 4008, 0, 3877, 4094, 424, 0, 1901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 42, 3212, 0, 0, 1378, 134, 0, 859, 0, 1, 0, 157, 4, 0, 2, 3, 4096, 49, 4083, 0, 0, 4080, 0, 0, 0, 4093, 0, 28, 0, 55, 0, 4, 22, 0, 0, 0, 7, 58] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 16, 0, 0, 3970, 0, 0, 0, 4096, 0, 0, 838, 0, 0, 0, 793, 0, 0, 0, 0, 265, 3761, 0, 0, 4095, 0, 0, 4096, 0, 1115, 8, 4091, 0, 4077, 0, 0, 0, 7, 2, 0, 4020, 0, 4006, 4086, 3, 4071, 3390, 5, 0, 106, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 4096, 138, 17, 94, 1, 3921, 4, 4095, 339, 1636, 45, 0, 3143, 0, 10, 0, 1, 91, 0, 0, 0, 2681, 0, 3833, 0, 6, 0, 3, 4096, 0, 167, 2807, 0, 166, 2088, 51, 0, 3375, 2713, 0, 2288, 36, 141, 0, 0, 0, 697, 0, 4042, 0, 4096, 4094, 0, 0, 1855, 4072, 3955, 1336, 230, 0, 0, 2, 0, 3772, 230, 0, 4088, 1506] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1203, 12, 0, 0, 0, 2684, 4096, 138, 4090, 0, 0, 329, 0, 346, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 1105, 0, 0, 4077, 0, 511, 3519, 32, 3786, 0, 3, 0, 59, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 65, 3, 0, 46, 0, 41, 2898, 4096, 0, 645, 3316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4035, 0, 2498, 0, 4092, 4096, 0, 0, 1212, 82, 0, 0, 8, 583, 4094, 1024, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 1873, 0, 164, 4052, 40, 4093, 4053, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 5, 0, 2958, 3, 57, 3588, 494, 0, 186, 15, 3960, 3335, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 2, 132, 0, 0, 1329, 1884, 468, 4006, 0, 13, 0, 1832, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 3488, 4092, 0, 524, 0, 0, 0, 39, 4068, 0, 0, 2895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13236, 13201, 19299, 21998, 18480, 15244, 14489, 15125] avg=16384.0000 max=21998 max_rank=3 min=13201 min_rank=1 imbalance_ratio=1.342651 max_load_reduction=109074 max_load_reduction_pct=0.832169 moved_to_echo=63338 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 32, 3, 0, 0, 0, 0, 4096, 3985, 4014, 0, 0, 272, 3779, 533, 375, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3159, 0, 0, 1, 0, 2150, 0, 4080, 2, 0, 83, 1082, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 1018, 0, 173, 0, 73, 4096, 7, 0, 0, 0, 3595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1165, 4095, 0, 1831, 0, 3480, 2840, 0, 8, 22, 0, 0, 5, 4096, 4090, 4096, 1189, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 2176, 0, 0, 4047, 0, 0, 4095, 63, 0, 0, 118, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 10, 2, 0, 0, 1422, 0, 2164, 4056, 3, 0, 13, 0, 3725, 3289, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3063, 4014, 648, 0, 1819, 4096, 0, 23, 0, 0, 2480, 0, 26, 0, 4094, 751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 3348, 3980, 0, 0, 0, 356, 0, 0, 0, 3715, 233, 3675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17413, 10675, 13041, 27175, 10652, 15784, 21014, 15318] avg=16384.0000 max=27175 max_rank=3 min=10652 min_rank=4 imbalance_ratio=1.658630 max_load_reduction=103897 max_load_reduction_pct=0.792671 moved_to_echo=62768 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 164, 45, 0, 4071, 4096, 1026, 0, 1168, 0, 0, 0, 3254, 409, 0, 0, 353, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 101, 9, 480, 5, 4000, 0, 1, 0, 0, 547, 4092, 4079, 0, 1597, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 4096, 2, 0, 0, 0, 17, 0, 318, 0, 84, 3385, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 0, 3679, 0, 2761, 5, 0, 867, 42, 361, 0, 378, 0, 2099, 268, 4094, 959, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 800, 3751, 1755, 10, 0, 11, 2, 7, 4084, 4, 0, 3660, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 129, 0, 0, 0, 738, 0, 2, 4068, 17, 4065, 0, 0, 4095, 3225, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 317, 0, 0, 0, 2810, 0, 0, 0, 0, 6, 0, 3578, 2, 4093, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 3488, 4096, 2721, 0, 8, 1, 2, 199, 3569, 4082, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14924, 19061, 8047, 19729, 14890, 16930, 15084, 22407] avg=16384.0000 max=22407 max_rank=7 min=8047 min_rank=2 imbalance_ratio=1.367615 max_load_reduction=108665 max_load_reduction_pct=0.829048 moved_to_echo=69311 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[474, 0, 0, 0, 0, 3346, 2901, 4096, 303, 99, 1080, 4017, 7, 0, 4096, 0, 361, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1373, 4086, 1, 0, 0, 0, 3219, 0, 3166, 4073, 0, 4095, 74, 83, 24, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2380, 686, 0, 262, 29, 0, 0, 14, 0, 57, 4079, 0, 2, 3460, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 5, 0, 2357, 0, 3611, 4, 0, 0, 2, 5, 4024, 1707, 633, 20, 1088, 733, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 1341, 3936, 1576, 2, 0, 4077, 0, 72, 116, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 679, 0, 0, 84, 0, 571, 339, 1, 695, 4096, 0, 0, 0, 3, 3996, 3040, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3716, 300, 30, 0, 0, 0, 0, 0, 0, 248, 3256, 1, 4094, 0, 0, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 937, 3547, 3479, 4060, 534, 10, 0, 0, 0, 0, 3432, 4, 0, 2381, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21117, 20301, 15065, 14532, 15320, 13953, 12339, 18445] avg=16384.0000 max=21117 max_rank=0 min=12339 min_rank=6 imbalance_ratio=1.288879 max_load_reduction=109955 max_load_reduction_pct=0.838890 moved_to_echo=60057 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 70, 3962, 68, 4096, 0, 2872, 3169, 2, 220, 0, 0, 0, 0, 366, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 4, 1, 0, 0, 0, 0, 3675, 648, 4092, 4096, 461, 0, 1040, 11, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 1, 88, 1, 0, 3860, 3769, 2, 45, 4030, 3985, 0, 0, 3503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 3367, 3780, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 507, 1, 941, 1055, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 4003, 0, 1931, 0, 4074, 3691, 0, 4024, 4033, 13, 0, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 141, 0, 0, 0, 233, 23, 0, 285, 2741, 0, 104, 1620, 0, 3968, 3335, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2546, 111, 3719, 0, 679, 0, 0, 0, 0, 4094, 0, 3998, 8, 10, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 3489, 4087, 136, 0, 0, 0, 23, 0, 0, 4082, 426, 3626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15163, 18141, 19654, 10960, 22220, 13103, 15926, 15905] avg=16384.0000 max=22220 max_rank=4 min=10960 min_rank=3 imbalance_ratio=1.356201 max_load_reduction=108852 max_load_reduction_pct=0.830475 moved_to_echo=67154 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 74, 0, 2, 3507, 3928, 0, 73, 1858, 0, 14, 2828, 454, 0, 1592, 77, 376, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 48, 213, 0, 0, 4092, 289, 0, 4096, 0, 2, 3, 4096, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1917, 0, 292, 0, 2, 0, 0, 0, 3961, 0, 56, 3603, 1676, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 4, 0, 0, 4096, 0, 139, 0, 65, 10, 0, 4, 1, 8, 4092, 1181, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3823, 2160, 0, 0, 4096, 0, 3496, 651, 2477, 0, 34, 0, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 233, 0, 4092, 0, 11, 1079, 12, 23, 36, 0, 0, 0, 3503, 4096, 1476, 4070, 0, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2858, 0, 2710, 4096, 4003, 0, 622, 353, 4053, 0, 1421, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3135, 2934, 14, 3760, 0, 0, 0, 0, 4071, 1673, 0, 4089, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15087, 13194, 11523, 10047, 17820, 19560, 24063, 19778] avg=16384.0000 max=24063 max_rank=6 min=10047 min_rank=3 imbalance_ratio=1.468689 max_load_reduction=107009 max_load_reduction_pct=0.816414 moved_to_echo=81221 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 3, 3939, 3398, 0, 4092, 0, 4054, 3511, 13, 257, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2529, 0, 0, 10, 0, 286, 136, 0, 3992, 3645, 0, 4096, 0, 4096, 0, 87, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 4072, 1244, 15, 0, 526, 3858, 0, 1131, 0, 4068, 0, 0, 2468, 3915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 3685, 1, 9, 949, 1778, 4, 1, 0, 2464, 0, 0, 1067, 3625, 2, 1342, 1188, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 0, 2172, 4, 0, 4008, 0, 3877, 4094, 424, 0, 1901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 42, 3212, 0, 0, 1378, 134, 3588, 3322, 728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 859, 0, 1, 0, 157, 4, 0, 2, 3, 4096, 49, 4083, 0, 0, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3486, 0, 28, 0, 55, 0, 4, 22, 0, 0, 0, 7, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23701, 18953, 25355, 16616, 20331, 12444, 10012, 3660] avg=16384.0000 max=25355 max_rank=2 min=3660 min_rank=7 imbalance_ratio=1.547546 max_load_reduction=105717 max_load_reduction_pct=0.806557 moved_to_echo=46447 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 16, 0, 0, 3970, 0, 0, 0, 4096, 0, 0, 838, 0, 0, 0, 793, 378, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 3761, 0, 0, 4095, 0, 0, 4096, 0, 1115, 8, 4091, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 7, 2, 0, 4020, 0, 4006, 0, 3, 3578, 3390, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 4096, 138, 17, 94, 1194, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3921, 4, 2168, 339, 1636, 45, 0, 3143, 0, 10, 0, 1, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2681, 0, 3833, 0, 6, 0, 3, 4096, 0, 167, 2807, 0, 166, 2088, 51, 4086, 3337, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3375, 2713, 0, 2288, 36, 141, 0, 0, 0, 697, 0, 4042, 0, 4096, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1855, 3460, 3955, 1336, 230, 0, 0, 2, 0, 3772, 230, 0, 4088, 1506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10431, 17546, 19088, 10015, 11359, 24054, 18145, 20434] avg=16384.0000 max=24054 max_rank=5 min=10015 min_rank=3 imbalance_ratio=1.468140 max_load_reduction=107018 max_load_reduction_pct=0.816483 moved_to_echo=73992 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[10368, 15488, 6400, 3840, 7040, 11904, 4224, 1408, 2560, 1152, 22784, 128, 21376, 128, 18176, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[512, 256, 128, 1280, 19712, 11520, 8064, 4224, 21120, 14848, 5248, 16128, 3840, 4736, 13312, 1536, 2816, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 1024, 5632, 27392, 27264, 4864, 4608, 128, 384, 128, 384, 18944, 13824, 768, 24960, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 12800, 11776, 15616, 4352, 7552, 19968, 128, 22784, 21120, 3072, 128, 6016, 1024, 5248, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[8064, 8448, 128, 1664, 3584, 8320, 7552, 128, 19584, 14336, 4736, 32512, 4736, 5376, 2816, 9472, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[6528, 15616, 7552, 6784, 3712, 21248, 7040, 1024, 128, 4224, 128, 4480, 6912, 11136, 4992, 19968, 8576, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[7040, 13056, 11520, 5504, 384, 4608, 7680, 128, 9472, 7040, 24576, 0, 896, 27008, 13184, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 2944, 1152, 8192, 256, 128, 1152, 896, 9600, 128, 3456, 24576, 640, 7936, 9472, 1792, 31488, 22912, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 3885, 3594, 477, 11, 0, 865, 0, 4094, 0, 4, 1, 10, 0, 5, 463, 3641, 144, 4055, 960, 39, 0, 63, 201, 977, 4089, 3520, 0, 0, 4096, 2205, 0, 20, 0, 638, 0, 3897, 163, 0, 0, 99, 0, 2342, 0, 4046, 174, 1, 50, 0, 0, 3768, 0, 0, 0, 4093, 0, 29, 0, 16, 1316, 278, 4076, 0, 0, 0, 4066, 0, 0, 0, 1405, 3089, 185, 1, 0, 0, 1, 0, 3254, 4019, 0, 10, 0, 29, 1, 0, 1295, 0, 3169, 4096, 0, 1974, 4045, 0, 4039, 3056, 4079, 0, 3966, 0, 86, 0, 0, 0, 0, 0, 4051, 0, 0, 4074, 0, 3850, 3, 1, 13, 4090, 0, 0, 0, 9, 0, 36, 4075, 5, 2593] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 54, 496, 0, 0, 0, 4096, 24, 0, 136, 2804, 1, 4073, 3666, 35, 0, 0, 0, 2210, 0, 70, 4084, 4096, 4, 0, 0, 476, 3346, 4095, 15, 0, 116, 0, 701, 3671, 572, 0, 1, 0, 11, 0, 4002, 0, 12, 0, 4082, 3791, 152, 0, 208, 1, 0, 282, 4, 0, 0, 0, 0, 0, 28, 149, 0, 45, 3029, 0, 0, 0, 672, 4024, 419, 2407, 0, 0, 1, 0, 1, 814, 13, 0, 0, 0, 2046, 0, 0, 60, 4082, 0, 0, 4049, 0, 2947, 3887, 3494, 0, 972, 1902, 2438, 4067, 4074, 4091, 0, 4044, 454, 1181, 68, 0, 2, 393, 0, 1, 5, 0, 4089, 0, 4, 0, 4092, 3228, 4089, 0, 1, 4096, 0, 3956, 0, 0, 271, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[6, 16, 4095, 4025, 0, 0, 4096, 0, 0, 0, 0, 4067, 4082, 4096, 9, 99, 0, 0, 4070, 0, 4038, 0, 10, 0, 23, 0, 3722, 4086, 0, 2, 2, 420, 4096, 0, 3597, 0, 0, 327, 0, 0, 3837, 1482, 5, 0, 0, 24, 3191, 0, 0, 36, 0, 0, 0, 0, 116, 624, 0, 2448, 2698, 0, 2052, 0, 0, 3800, 0, 3896, 5, 4, 0, 40, 170, 0, 16, 0, 385, 0, 0, 432, 0, 132, 0, 0, 0, 0, 0, 818, 2101, 0, 1, 5, 0, 0, 3639, 0, 1767, 0, 0, 3806, 3817, 2893, 4096, 4096, 1261, 1384, 0, 0, 0, 130, 240, 7, 3086, 0, 2522, 482, 0, 2, 4093, 4096, 4060, 0, 1282, 2143, 0, 3926, 0, 0, 942, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[3842, 0, 9, 0, 68, 0, 4071, 0, 0, 0, 0, 0, 4025, 4094, 2, 0, 2, 0, 856, 4093, 2317, 0, 1690, 0, 9, 0, 1276, 75, 4025, 1, 0, 4092, 4096, 109, 4026, 0, 0, 0, 0, 0, 260, 4095, 10, 0, 0, 0, 3760, 0, 486, 4072, 0, 0, 0, 3899, 4095, 0, 0, 3900, 0, 0, 977, 0, 0, 4096, 458, 0, 0, 76, 0, 499, 4084, 0, 12, 0, 1, 34, 0, 321, 0, 3596, 3753, 4078, 0, 0, 8, 0, 0, 0, 4091, 3, 0, 2602, 3935, 0, 12, 4088, 0, 4096, 4096, 0, 4071, 55, 2175, 4, 0, 0, 0, 0, 0, 0, 20, 0, 1, 163, 19, 4085, 4087, 4096, 1, 0, 0, 0, 0, 24, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 1, 24, 4071, 3873, 1547, 0, 2, 2946, 1, 1700, 4096, 4015, 0, 0, 0, 0, 4053, 0, 4096, 198, 0, 83, 0, 0, 935, 0, 1336, 6, 5, 89, 0, 3269, 4024, 0, 45, 0, 0, 0, 38, 4060, 0, 0, 601, 2657, 141, 0, 0, 3355, 0, 0, 4, 3995, 0, 0, 4087, 20, 4090, 2992, 0, 8, 1644, 3988, 63, 0, 5, 0, 3876, 0, 3701, 12, 620, 3, 892, 0, 0, 0, 0, 0, 1, 453, 322, 0, 0, 0, 0, 0, 4096, 0, 0, 3983, 4059, 66, 40, 4060, 22, 427, 4091, 4096, 0, 4096, 4, 0, 0, 2, 0, 0, 137, 8, 10, 3252, 4094, 331, 0, 0, 0, 517, 4094, 0, 0, 3272, 13, 15, 0, 12, 141, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 3787, 4095, 49, 0, 7, 234, 0, 3525, 4072, 0, 554, 0, 0, 135, 0, 4088, 35, 0, 25, 0, 1712, 0, 3, 4096, 29, 4090, 1, 3481, 4028, 416, 488, 0, 2, 0, 21, 284, 2042, 0, 83, 44, 0, 756, 0, 3909, 3700, 9, 0, 0, 4074, 0, 0, 0, 91, 4087, 0, 0, 19, 0, 3112, 1646, 4060, 0, 0, 0, 186, 4096, 637, 0, 5, 4096, 0, 30, 3, 7, 0, 0, 4096, 0, 0, 2939, 0, 7, 17, 2776, 0, 0, 3997, 3601, 1, 24, 4096, 19, 532, 4035, 0, 4002, 4054, 3167, 100, 0, 0, 0, 84, 0, 4076, 0, 0, 449, 2, 0, 1, 0, 0, 4096, 0, 0, 4061, 0, 707, 90, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[76, 0, 2556, 0, 4092, 0, 4032, 12, 0, 1389, 0, 4004, 4096, 2775, 0, 0, 0, 0, 37, 0, 4087, 20, 5, 158, 0, 0, 3842, 0, 3548, 0, 3346, 2506, 0, 4096, 3881, 0, 0, 0, 0, 0, 4096, 4057, 0, 0, 0, 1407, 747, 0, 0, 0, 0, 180, 0, 4096, 0, 0, 0, 0, 3220, 3193, 0, 0, 1, 0, 205, 3922, 23, 0, 110, 0, 0, 2931, 0, 36, 380, 294, 19, 0, 0, 19, 0, 1574, 0, 153, 0, 2265, 0, 1057, 4091, 3730, 0, 0, 4077, 0, 291, 1951, 0, 0, 1276, 3985, 0, 3929, 1653, 0, 0, 0, 4072, 2, 89, 0, 497, 1, 3546, 0, 0, 0, 16, 4096, 4096, 5, 0, 0, 0, 3044, 47, 0, 0, 4035] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 165, 2583, 0, 0, 4, 3973, 4029, 10, 18, 0, 1, 3437, 4096, 672, 11, 0, 0, 4076, 0, 4096, 158, 0, 0, 0, 0, 6, 288, 4096, 0, 0, 0, 3639, 3839, 4088, 0, 2, 0, 35, 0, 506, 3853, 832, 7, 4062, 0, 3229, 0, 3392, 0, 124, 982, 3929, 3630, 0, 17, 0, 0, 1724, 1117, 0, 0, 0, 4045, 275, 2711, 1314, 0, 44, 0, 2825, 0, 28, 64, 4096, 0, 942, 0, 2911, 2, 0, 1093, 0, 372, 0, 4024, 0, 0, 4096, 1368, 0, 4083, 3970, 0, 11, 301, 0, 0, 3990, 682, 4096, 0, 3933, 0, 21, 0, 0, 0, 17, 114, 0, 0, 0, 6, 0, 0, 0, 0, 4093, 0, 447, 4094, 3, 0, 0, 144, 131, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 24, 4071, 3873, 1547, 0, 2, 2946, 1, 1700, 1142, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 4096, 198, 0, 83, 0, 0, 935, 0, 1336, 6, 5, 89, 263, 188, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3269, 3845, 0, 45, 0, 0, 0, 38, 4060, 0, 0, 601, 2657, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3355, 0, 0, 4, 3995, 0, 0, 4087, 20, 4090, 2992, 0, 8, 1644, 3988, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 5, 0, 3876, 0, 3701, 12, 620, 3, 892, 0, 0, 0, 0, 0, 2954, 2932, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 453, 322, 0, 0, 0, 0, 0, 4096, 0, 0, 3983, 1127, 66, 40, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 427, 2233, 4096, 0, 4096, 4, 0, 0, 2, 0, 0, 137, 8, 10, 3252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 331, 0, 0, 0, 517, 3906, 0, 0, 3272, 13, 15, 0, 12, 141, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19322, 11431, 14656, 25290, 15546, 14148, 14287, 16392] avg=16384.0000 max=25290 max_rank=3 min=11431 min_rank=1 imbalance_ratio=1.543579 max_load_reduction=105782 max_load_reduction_pct=0.807053 moved_to_echo=60373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3842, 0, 9, 0, 68, 0, 4071, 0, 0, 0, 0, 0, 1117, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 856, 4093, 2317, 0, 1690, 0, 9, 0, 1276, 75, 4025, 1, 0, 4092, 267, 1, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 109, 3844, 0, 0, 0, 0, 0, 260, 4095, 10, 0, 0, 0, 3760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 4072, 0, 0, 0, 3899, 4095, 0, 0, 3900, 0, 0, 977, 0, 0, 4096, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 0, 0, 76, 0, 499, 4084, 0, 12, 0, 1, 34, 0, 321, 0, 3596, 2908, 2846, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3753, 4078, 0, 0, 8, 0, 0, 0, 4091, 3, 0, 2602, 1089, 0, 12, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2223, 0, 4071, 55, 2175, 4, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 163, 19, 4085, 4087, 4096, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13203, 18886, 16174, 22638, 15328, 19724, 12644, 12475] avg=16384.0000 max=22638 max_rank=3 min=12475 min_rank=7 imbalance_ratio=1.381714 max_load_reduction=108434 max_load_reduction_pct=0.827286 moved_to_echo=60171 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 54, 496, 0, 0, 0, 4096, 24, 0, 136, 2804, 1, 1135, 3666, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2210, 0, 70, 4084, 4096, 4, 0, 0, 476, 3346, 4095, 15, 0, 116, 262, 188, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 701, 3508, 572, 0, 1, 0, 11, 0, 4002, 0, 12, 0, 4082, 3791, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 1, 0, 282, 4, 0, 0, 0, 0, 0, 28, 149, 0, 45, 3029, 1103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 4024, 419, 2407, 0, 0, 1, 0, 1, 814, 13, 0, 0, 2938, 2524, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2046, 0, 0, 60, 4082, 0, 0, 4049, 0, 2947, 3887, 970, 0, 972, 1902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2438, 4067, 2223, 4091, 0, 4044, 454, 1181, 68, 0, 2, 393, 0, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4, 0, 4092, 3228, 3901, 0, 1, 4096, 0, 3956, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12447, 19125, 16832, 4849, 14299, 20915, 18967, 23638] avg=16384.0000 max=23638 max_rank=7 min=4849 min_rank=3 imbalance_ratio=1.442749 max_load_reduction=107434 max_load_reduction_pct=0.819656 moved_to_echo=77819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 3885, 3594, 477, 11, 0, 865, 0, 1141, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 5, 463, 3641, 144, 4055, 960, 39, 0, 63, 201, 977, 4089, 3520, 0, 196, 188, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2107, 0, 20, 0, 638, 0, 3897, 163, 0, 0, 99, 0, 2342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 174, 1, 50, 0, 0, 3768, 0, 0, 0, 4093, 0, 29, 0, 16, 1316, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 4076, 0, 0, 0, 4066, 0, 0, 0, 1405, 3089, 185, 1, 0, 0, 1, 2953, 2958, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3254, 4019, 0, 10, 0, 29, 1, 0, 1295, 0, 3169, 1138, 0, 1974, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 1668, 4079, 0, 3966, 0, 86, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 3850, 3, 1, 13, 3902, 0, 0, 0, 9, 0, 36, 4075, 5, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9980, 18649, 13362, 14320, 19377, 18934, 17889, 18561] avg=16384.0000 max=19377 max_rank=4 min=9980 min_rank=0 imbalance_ratio=1.182678 max_load_reduction=111695 max_load_reduction_pct=0.852165 moved_to_echo=74761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[6, 16, 4095, 4025, 0, 0, 4096, 0, 0, 0, 0, 4067, 1138, 4096, 9, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 4038, 0, 10, 0, 23, 0, 3722, 4086, 0, 2, 2, 420, 245, 187, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3437, 0, 0, 327, 0, 0, 3837, 1482, 5, 0, 0, 24, 3191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 116, 624, 0, 2448, 2698, 0, 2052, 0, 0, 3800, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 5, 4, 0, 40, 170, 0, 16, 0, 385, 0, 0, 432, 0, 132, 2944, 2628, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 2101, 0, 1, 5, 0, 0, 1011, 0, 1767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3806, 2084, 2893, 4096, 4096, 1261, 1384, 0, 0, 0, 130, 240, 7, 3086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2522, 482, 0, 2, 4093, 4096, 3873, 0, 1282, 2143, 0, 3926, 0, 0, 942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21647, 16965, 16399, 12807, 11107, 5703, 23083, 23361] avg=16384.0000 max=23361 max_rank=7 min=5703 min_rank=5 imbalance_ratio=1.425842 max_load_reduction=107711 max_load_reduction_pct=0.821770 moved_to_echo=63254 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[76, 0, 2556, 0, 4092, 0, 4032, 12, 0, 1389, 0, 4004, 1142, 2775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 4087, 20, 5, 158, 0, 0, 3842, 0, 3548, 0, 3346, 2506, 82, 188, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3709, 0, 0, 0, 0, 0, 4096, 4057, 0, 0, 0, 1407, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 4096, 0, 0, 0, 0, 3220, 3193, 0, 0, 1, 0, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 3922, 23, 0, 110, 0, 0, 2931, 0, 36, 380, 294, 19, 0, 0, 19, 2954, 2945, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1574, 0, 153, 0, 2265, 0, 1057, 4091, 3730, 0, 0, 1132, 0, 291, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 697, 3985, 0, 3929, 1653, 0, 0, 0, 4072, 2, 89, 0, 497, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3546, 0, 0, 0, 16, 4096, 3908, 5, 0, 0, 0, 3044, 47, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20078, 17991, 18112, 11035, 13990, 16244, 14925, 18697] avg=16384.0000 max=20078 max_rank=0 min=11035 min_rank=3 imbalance_ratio=1.225464 max_load_reduction=110994 max_load_reduction_pct=0.846817 moved_to_echo=63856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 3787, 4095, 49, 0, 7, 234, 0, 983, 4072, 0, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 4088, 35, 0, 25, 0, 1712, 0, 3, 4096, 29, 4090, 1, 259, 188, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3481, 4028, 398, 488, 0, 2, 0, 21, 284, 2042, 0, 83, 44, 0, 756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 3700, 9, 0, 0, 4074, 0, 0, 0, 91, 4087, 0, 0, 19, 0, 3112, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1646, 4060, 0, 0, 0, 186, 4096, 637, 0, 5, 4096, 0, 30, 3, 7, 0, 2542, 2601, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2939, 0, 7, 17, 2776, 0, 0, 3997, 1000, 1, 24, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 532, 2202, 0, 4002, 4054, 3167, 100, 0, 0, 0, 84, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 2, 0, 1, 0, 0, 3908, 0, 0, 4061, 0, 707, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17875, 14679, 11627, 20093, 20391, 18953, 18236, 9218] avg=16384.0000 max=20391 max_rank=4 min=9218 min_rank=7 imbalance_ratio=1.244568 max_load_reduction=110681 max_load_reduction_pct=0.844429 moved_to_echo=66798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 165, 2583, 0, 0, 4, 3973, 4029, 10, 18, 0, 1, 958, 4096, 672, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 4096, 158, 0, 0, 0, 0, 6, 288, 4096, 0, 0, 0, 256, 188, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3639, 3839, 3906, 0, 2, 0, 35, 0, 506, 3853, 832, 7, 4062, 0, 3229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3392, 0, 124, 982, 3929, 3630, 0, 17, 0, 0, 1724, 1117, 0, 0, 0, 4045, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 2711, 1314, 0, 44, 0, 2825, 0, 28, 64, 4096, 0, 942, 0, 2911, 2, 2479, 2867, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 0, 372, 0, 4024, 0, 0, 4096, 1368, 0, 4083, 1103, 0, 11, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2178, 682, 4096, 0, 3933, 0, 21, 0, 0, 0, 17, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 3905, 0, 447, 4094, 3, 0, 0, 144, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16520, 13346, 23910, 20040, 21034, 16451, 11041, 8730] avg=16384.0000 max=23910 max_rank=2 min=8730 min_rank=7 imbalance_ratio=1.459351 max_load_reduction=107162 max_load_reduction_pct=0.817581 moved_to_echo=57256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[3840, 16640, 4352, 640, 3072, 11264, 2176, 1152, 23296, 6528, 3072, 21760, 8576, 128, 5120, 20480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[15360, 20224, 24832, 1152, 128, 384, 768, 128, 12928, 23808, 896, 128, 4864, 8192, 18048, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2944, 18688, 1408, 768, 8064, 5248, 17408, 3584, 768, 1536, 13056, 640, 1920, 896, 2944, 3840, 22784, 22400, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2560, 17024, 15616, 19840, 16384, 24320, 12672, 2816, 128, 128, 4096, 640, 512, 8320, 3712, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 15488, 4608, 26496, 4736, 9856, 1280, 128, 1792, 10368, 8064, 22272, 4224, 11008, 7296, 1920, 1408, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[11904, 11648, 256, 1280, 4224, 19712, 8064, 768, 4096, 6528, 19968, 7424, 3328, 128, 1792, 23424, 7808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[18816, 1024, 3968, 4096, 12416, 16128, 27392, 128, 1792, 17792, 128, 11776, 256, 4352, 1536, 10752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3968, 256, 9856, 4096, 12416, 11648, 29568, 4608, 128, 4608, 3968, 9856, 8832, 26880, 768, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True + [2026-08-30 05:04:56] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 41325.6 | throughput per GPU (TFLOP/s/GPU): 8.1 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102490E+01 | loss scale: 1.0 | grad norm: 44.116 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 23.22 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 64004.6611328125 | max allocated: 94572.208984375 | reserved: 96956.0 | max reserved: 96956.0 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[84, 2878, 415, 251, 1463, 173, 1107, 1067, 1117, 182, 1463, 143, 673, 677, 98, 325, 97, 791, 883, 683, 819, 95, 2637, 1453, 439, 1460, 800, 992, 1550, 416, 1374, 1889, 2792, 397, 618, 499, 57, 2583, 2104, 681, 2491, 303, 1218, 775, 1478, 1709, 968, 962, 291, 469, 202, 2287, 1178, 513, 575, 88, 280, 163, 959, 2168, 1218, 500, 686, 961, 698, 2056, 541, 1033, 912, 235, 569, 1691, 907, 2042, 156, 2209, 598, 1101, 2465, 1206, 2855, 1008, 2612, 390, 1752, 114, 604, 729, 2323, 119, 780, 1183, 559, 869, 562, 984, 2329, 202, 1904, 586, 1543, 428, 1506, 2737, 428, 1122, 532, 2095, 493, 222, 2405, 886, 227, 1416, 1701, 1250, 279, 445, 826, 1141, 748, 1700, 1478, 815, 1851, 456, 544, 248] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[74, 2791, 585, 257, 1165, 293, 737, 527, 958, 194, 1058, 116, 1179, 554, 114, 323, 106, 388, 632, 567, 1255, 95, 3591, 1788, 170, 1248, 987, 806, 742, 147, 1482, 2299, 3116, 943, 250, 487, 94, 2465, 2000, 714, 2762, 118, 978, 639, 1791, 1487, 1425, 900, 257, 228, 246, 2254, 445, 350, 295, 70, 687, 370, 890, 1669, 1358, 592, 526, 1401, 678, 2164, 328, 858, 1293, 92, 858, 1539, 1212, 2052, 183, 1490, 887, 984, 2838, 1011, 3399, 995, 2903, 380, 2183, 175, 347, 1139, 2154, 202, 876, 1631, 1173, 627, 409, 511, 2269, 431, 1999, 557, 1328, 643, 1647, 2067, 513, 947, 778, 2276, 872, 272, 2268, 823, 656, 1172, 1052, 900, 221, 554, 543, 1389, 850, 1483, 1437, 1067, 1564, 521, 844, 523] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[87, 2600, 678, 196, 2087, 629, 888, 1309, 990, 134, 1032, 386, 964, 709, 143, 426, 147, 427, 1251, 1079, 862, 195, 2271, 1496, 374, 1230, 1122, 1059, 841, 276, 1215, 1592, 3310, 551, 247, 698, 99, 2191, 2336, 560, 2064, 194, 1687, 796, 1193, 1056, 1161, 1383, 387, 169, 332, 2314, 737, 188, 631, 128, 647, 173, 953, 1568, 1528, 580, 602, 1209, 846, 1762, 342, 1188, 2203, 281, 924, 1631, 1332, 2059, 249, 1838, 978, 829, 2527, 930, 2333, 1168, 2854, 240, 1705, 155, 1179, 1091, 2151, 242, 934, 1035, 834, 354, 572, 665, 1983, 337, 1975, 444, 1680, 791, 1607, 1862, 631, 736, 548, 2561, 551, 182, 1970, 1118, 281, 1257, 1128, 1061, 281, 901, 669, 1283, 875, 1501, 1304, 424, 1608, 448, 892, 385] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[99, 2426, 600, 311, 1332, 424, 785, 478, 781, 104, 1297, 188, 827, 779, 257, 333, 173, 452, 658, 916, 825, 396, 2737, 1534, 276, 806, 1273, 818, 994, 137, 1587, 2336, 3263, 565, 147, 422, 543, 2335, 2607, 339, 2525, 653, 1211, 827, 1127, 1481, 1012, 1612, 240, 105, 334, 1806, 628, 377, 659, 161, 1006, 345, 1298, 2277, 1368, 483, 550, 1011, 1016, 1630, 701, 892, 1267, 130, 957, 1710, 1159, 2698, 225, 2099, 967, 788, 2761, 830, 2484, 862, 2458, 536, 1675, 134, 684, 1204, 1873, 146, 483, 1024, 1098, 478, 363, 972, 1812, 221, 2298, 982, 1539, 786, 1169, 1539, 672, 823, 842, 2567, 612, 505, 2020, 1119, 385, 1183, 1400, 1379, 548, 578, 565, 1160, 992, 1443, 1619, 494, 1859, 736, 453, 212] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[134, 2637, 412, 345, 908, 313, 824, 589, 1076, 221, 1403, 349, 966, 521, 226, 298, 105, 451, 612, 802, 1033, 576, 2827, 1855, 386, 1435, 1349, 730, 1369, 233, 1524, 1842, 2979, 932, 359, 463, 148, 1976, 1841, 634, 2648, 484, 1233, 710, 1262, 1187, 1059, 744, 286, 498, 398, 1797, 535, 430, 526, 124, 333, 603, 882, 1734, 1571, 399, 716, 992, 419, 1721, 816, 1733, 1422, 135, 845, 1825, 975, 1810, 192, 1699, 599, 1141, 2709, 958, 2790, 1151, 2743, 417, 1559, 133, 1247, 1249, 2006, 308, 986, 1488, 1084, 619, 625, 406, 2315, 293, 1690, 1116, 1385, 584, 1099, 2632, 696, 768, 544, 2009, 639, 340, 1839, 1076, 292, 1137, 1465, 1623, 280, 448, 1350, 1274, 787, 1483, 1535, 1032, 1061, 541, 820, 275] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[42, 2583, 305, 168, 1499, 251, 938, 757, 1067, 153, 2009, 211, 459, 819, 197, 232, 64, 391, 1169, 437, 1176, 228, 2890, 1954, 499, 1054, 1018, 785, 1618, 387, 2430, 2218, 3041, 427, 517, 345, 65, 2320, 2101, 324, 2697, 520, 1413, 695, 1415, 1854, 763, 506, 204, 248, 205, 1553, 903, 220, 313, 50, 350, 325, 1011, 2117, 1629, 318, 1747, 1126, 697, 1982, 633, 1309, 741, 203, 369, 1616, 747, 2012, 162, 1810, 610, 1331, 2202, 1099, 2814, 1065, 2509, 424, 1969, 376, 911, 1310, 1943, 181, 670, 907, 677, 691, 427, 996, 2651, 477, 1231, 729, 1279, 864, 1215, 2362, 885, 877, 602, 1443, 325, 336, 2473, 637, 329, 1375, 993, 1778, 169, 717, 1290, 1373, 814, 1552, 1434, 688, 1708, 443, 603, 697] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[167, 3514, 693, 277, 1528, 314, 632, 1345, 984, 288, 2131, 255, 651, 823, 187, 337, 140, 362, 748, 913, 1014, 205, 2656, 2092, 650, 1357, 959, 951, 751, 214, 1488, 1571, 2399, 437, 422, 602, 89, 1876, 1783, 255, 2524, 546, 1578, 991, 1120, 1514, 852, 650, 272, 453, 440, 2003, 844, 684, 522, 96, 488, 309, 952, 1739, 1154, 676, 573, 1503, 630, 1703, 763, 941, 1118, 193, 962, 2287, 1221, 2145, 157, 1798, 795, 961, 2142, 754, 2338, 1016, 2379, 821, 1884, 561, 431, 982, 1740, 244, 840, 864, 776, 440, 368, 788, 2188, 204, 1532, 567, 1640, 802, 2100, 2248, 837, 1718, 569, 2038, 668, 427, 2608, 1000, 293, 856, 1106, 2011, 201, 617, 1023, 1333, 853, 1191, 1585, 678, 1491, 443, 316, 344] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[133, 2721, 488, 255, 1414, 551, 912, 1444, 1212, 134, 1537, 122, 694, 804, 226, 415, 166, 415, 535, 696, 1173, 278, 2743, 1437, 306, 1048, 1022, 931, 891, 216, 1378, 1762, 2834, 567, 280, 512, 161, 2369, 2262, 512, 2502, 337, 1115, 517, 1278, 1429, 897, 839, 208, 280, 287, 2503, 868, 595, 720, 78, 681, 366, 1541, 1655, 1189, 593, 698, 1483, 641, 2098, 776, 1655, 1224, 177, 624, 1828, 965, 1889, 263, 1631, 879, 733, 2089, 770, 2079, 1252, 2903, 599, 1776, 269, 837, 1046, 2254, 251, 825, 1268, 1077, 634, 662, 897, 2527, 297, 1876, 379, 1397, 900, 1902, 2594, 568, 721, 638, 1875, 693, 859, 2136, 958, 292, 716, 1055, 1075, 165, 723, 1215, 1542, 864, 1491, 1640, 537, 1387, 397, 318, 249] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[167, 3514, 693, 277, 1528, 314, 632, 1345, 984, 288, 2131, 255, 651, 823, 187, 337, 2288, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 362, 748, 913, 1014, 205, 2656, 2092, 650, 1357, 959, 951, 751, 214, 1488, 1571, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 437, 422, 602, 89, 1876, 1783, 255, 2524, 546, 1578, 991, 1120, 1514, 852, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 453, 440, 2003, 844, 684, 522, 96, 488, 309, 952, 1739, 1154, 676, 573, 1503, 2188, 855, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 1703, 763, 941, 1118, 193, 962, 2287, 1221, 2145, 157, 1798, 795, 961, 92, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2338, 1016, 1128, 821, 1884, 561, 431, 982, 1740, 244, 840, 864, 776, 440, 368, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 1532, 567, 1640, 802, 2100, 1337, 837, 1718, 569, 2038, 668, 427, 2608, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 856, 1106, 2011, 201, 617, 1023, 1333, 853, 1191, 1585, 678, 1491, 443, 316, 344, 847, 634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17609, 16135, 15350, 16368, 16520, 15221, 18047, 15822] avg=16384.0000 max=18047 max_rank=6 min=15221 min_rank=5 imbalance_ratio=1.101501 max_load_reduction=113025 max_load_reduction_pct=0.862312 moved_to_echo=65610 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[84, 2878, 415, 251, 1463, 173, 1107, 1067, 1117, 182, 1463, 143, 673, 677, 98, 325, 2663, 1375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 791, 883, 683, 819, 95, 2637, 1453, 439, 1460, 800, 992, 1550, 416, 1374, 1889, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 397, 618, 499, 57, 2583, 2104, 681, 2491, 303, 1218, 775, 1478, 1709, 968, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 469, 202, 2287, 1178, 513, 575, 88, 280, 163, 959, 2168, 1218, 500, 686, 961, 2329, 984, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 698, 2056, 541, 1033, 912, 235, 569, 1691, 907, 2042, 156, 2209, 598, 1101, 106, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2855, 1008, 1238, 390, 1752, 114, 604, 729, 2323, 119, 780, 1183, 559, 869, 562, 984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 1904, 586, 1543, 428, 1506, 1627, 428, 1122, 532, 2095, 493, 222, 2405, 886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 1416, 1701, 1250, 279, 445, 826, 1141, 748, 1700, 1478, 815, 1851, 456, 544, 248, 1031, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16154, 16457, 16972, 16529, 16060, 16069, 15979, 16852] avg=16384.0000 max=16972 max_rank=2 min=15979 min_rank=6 imbalance_ratio=1.035889 max_load_reduction=114100 max_load_reduction_pct=0.870514 moved_to_echo=64960 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[134, 2637, 412, 345, 908, 313, 824, 589, 1076, 221, 1403, 349, 966, 521, 226, 298, 2841, 1512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 451, 612, 802, 1033, 576, 2827, 1855, 386, 1435, 1349, 730, 1369, 233, 1524, 1842, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 932, 359, 463, 148, 1976, 1841, 634, 2648, 484, 1233, 710, 1262, 1187, 1059, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 498, 398, 1797, 535, 430, 526, 124, 333, 603, 882, 1734, 1571, 399, 716, 992, 2315, 1082, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 419, 1721, 816, 1733, 1422, 135, 845, 1825, 975, 1810, 192, 1699, 599, 1141, 115, 958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2790, 1151, 1300, 417, 1559, 133, 1247, 1249, 2006, 308, 986, 1488, 1084, 619, 625, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 1690, 1116, 1385, 584, 1099, 1564, 696, 768, 544, 2009, 639, 340, 1839, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 1137, 1465, 1623, 280, 448, 1350, 1274, 787, 1483, 1535, 1032, 1061, 541, 820, 275, 992, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15575, 17205, 15818, 15933, 16405, 17368, 15642, 17126] avg=16384.0000 max=17368 max_rank=5 min=15575 min_rank=0 imbalance_ratio=1.060059 max_load_reduction=113704 max_load_reduction_pct=0.867493 moved_to_echo=66541 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[74, 2791, 585, 257, 1165, 293, 737, 527, 958, 194, 1058, 116, 1179, 554, 114, 323, 2974, 1587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 388, 632, 567, 1255, 95, 3591, 1788, 170, 1248, 987, 806, 742, 147, 1482, 2299, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 943, 250, 487, 94, 2465, 2000, 714, 2762, 118, 978, 639, 1791, 1487, 1425, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 228, 246, 2254, 445, 350, 295, 70, 687, 370, 890, 1669, 1358, 592, 526, 1401, 2269, 1138, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 678, 2164, 328, 858, 1293, 92, 858, 1539, 1212, 2052, 183, 1490, 887, 984, 113, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3399, 995, 1368, 380, 2183, 175, 347, 1139, 2154, 202, 876, 1631, 1173, 627, 409, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 1999, 557, 1328, 643, 1647, 1222, 513, 947, 778, 2276, 872, 272, 2268, 823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 656, 1172, 1052, 900, 221, 554, 543, 1389, 850, 1483, 1437, 1067, 1564, 521, 844, 523, 782, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15486, 16366, 17195, 15803, 15742, 17569, 16576, 16335] avg=16384.0000 max=17569 max_rank=5 min=15486 min_rank=0 imbalance_ratio=1.072327 max_load_reduction=113503 max_load_reduction_pct=0.865959 moved_to_echo=66222 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[99, 2426, 600, 311, 1332, 424, 785, 478, 781, 104, 1297, 188, 827, 779, 257, 333, 3112, 1541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 452, 658, 916, 825, 396, 2737, 1534, 276, 806, 1273, 818, 994, 137, 1587, 2336, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 565, 147, 422, 543, 2335, 2607, 339, 2525, 653, 1211, 827, 1127, 1481, 1012, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 105, 334, 1806, 628, 377, 659, 161, 1006, 345, 1298, 2277, 1368, 483, 550, 1011, 1812, 1102, 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1630, 701, 892, 1267, 130, 957, 1710, 1159, 2698, 225, 2099, 967, 788, 118, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2484, 862, 1165, 536, 1675, 134, 684, 1204, 1873, 146, 483, 1024, 1098, 478, 363, 972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 2298, 982, 1539, 786, 1169, 915, 672, 823, 842, 2567, 612, 505, 2020, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 1183, 1400, 1379, 548, 578, 565, 1160, 992, 1443, 1619, 494, 1859, 736, 453, 212, 580, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15674, 15962, 17557, 16200, 17187, 15181, 17070, 16241] avg=16384.0000 max=17557 max_rank=2 min=15181 min_rank=5 imbalance_ratio=1.071594 max_load_reduction=113515 max_load_reduction_pct=0.866051 moved_to_echo=65679 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[87, 2600, 678, 196, 2087, 629, 888, 1309, 990, 134, 1032, 386, 964, 709, 143, 426, 3157, 1410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 427, 1251, 1079, 862, 195, 2271, 1496, 374, 1230, 1122, 1059, 841, 276, 1215, 1592, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 551, 247, 698, 99, 2191, 2336, 560, 2064, 194, 1687, 796, 1193, 1056, 1161, 1383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 169, 332, 2314, 737, 188, 631, 128, 647, 173, 953, 1568, 1528, 580, 602, 1209, 1983, 1009, 741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 846, 1762, 342, 1188, 2203, 281, 924, 1631, 1332, 2059, 249, 1838, 978, 829, 108, 930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2333, 1168, 1353, 240, 1705, 155, 1179, 1091, 2151, 242, 934, 1035, 834, 354, 572, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 1975, 444, 1680, 791, 1607, 1108, 631, 736, 548, 2561, 551, 182, 1970, 1118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 1257, 1128, 1061, 281, 901, 669, 1283, 875, 1501, 1304, 424, 1608, 448, 892, 385, 701, 760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17825, 15490, 16369, 15879, 17500, 16011, 16239, 15759] avg=16384.0000 max=17825 max_rank=0 min=15490 min_rank=1 imbalance_ratio=1.087952 max_load_reduction=113247 max_load_reduction_pct=0.864006 moved_to_echo=65509 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[42, 2583, 305, 168, 1499, 251, 938, 757, 1067, 153, 2009, 211, 459, 819, 197, 232, 2900, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 391, 1169, 437, 1176, 228, 2890, 1954, 499, 1054, 1018, 785, 1618, 387, 2430, 2218, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 427, 517, 345, 65, 2320, 2101, 324, 2697, 520, 1413, 695, 1415, 1854, 763, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 248, 205, 1553, 903, 220, 313, 50, 350, 325, 1011, 2117, 1629, 318, 1747, 1126, 2651, 879, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 697, 1982, 633, 1309, 741, 203, 369, 1616, 747, 2012, 162, 1810, 610, 1331, 94, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2814, 1065, 1190, 424, 1969, 376, 911, 1310, 1943, 181, 670, 907, 677, 691, 427, 996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 1231, 729, 1279, 864, 1215, 1404, 885, 877, 602, 1443, 325, 336, 2473, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 1375, 993, 1778, 169, 717, 1290, 1373, 814, 1552, 1434, 688, 1708, 443, 603, 697, 890, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15819, 18386, 16103, 16500, 15415, 16551, 14777, 17521] avg=16384.0000 max=18386 max_rank=1 min=14777 min_rank=6 imbalance_ratio=1.122192 max_load_reduction=112686 max_load_reduction_pct=0.859726 moved_to_echo=64264 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[133, 2721, 488, 255, 1414, 551, 912, 1444, 1212, 134, 1537, 122, 694, 804, 226, 415, 2703, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 415, 535, 696, 1173, 278, 2743, 1437, 306, 1048, 1022, 931, 891, 216, 1378, 1762, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 567, 280, 512, 161, 2369, 2262, 512, 2502, 337, 1115, 517, 1278, 1429, 897, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 280, 287, 2503, 868, 595, 720, 78, 681, 366, 1541, 1655, 1189, 593, 698, 1483, 2527, 834, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 2098, 776, 1655, 1224, 177, 624, 1828, 965, 1889, 263, 1631, 879, 733, 90, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2079, 1252, 1376, 599, 1776, 269, 837, 1046, 2254, 251, 825, 1268, 1077, 634, 662, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 1876, 379, 1397, 900, 1902, 1543, 568, 721, 638, 1875, 693, 859, 2136, 958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 716, 1055, 1075, 165, 723, 1215, 1542, 864, 1491, 1640, 537, 1387, 397, 318, 249, 977, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16930, 15071, 15708, 17860, 16243, 17102, 16742, 15416] avg=16384.0000 max=17860 max_rank=3 min=15071 min_rank=1 imbalance_ratio=1.090088 max_load_reduction=113212 max_load_reduction_pct=0.863739 moved_to_echo=65503 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[21120, 8576, 10240, 3840, 14592, 1920, 6272, 8832, 16512, 1792, 6400, 9472, 7296, 4736, 4096, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1152, 4864, 2944, 4096, 1280, 18176, 17152, 4096, 20224, 3200, 10496, 6016, 10752, 11776, 8192, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[5632, 15232, 4992, 9728, 10240, 1536, 6144, 14208, 8576, 16768, 1664, 14592, 6400, 7936, 896, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[2176, 2560, 2560, 16640, 6144, 3456, 4352, 896, 4480, 2688, 8576, 14976, 11136, 4224, 6144, 9728, 18176, 7936, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1024, 3712, 6528, 6144, 8192, 2176, 22400, 13696, 3200, 9728, 8576, 7168, 8832, 2048, 12544, 15616, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 2560, 14592, 5376, 11904, 5888, 12288, 10752, 5248, 7808, 5120, 16896, 4864, 3200, 17792, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[896, 22272, 4224, 2176, 11520, 3072, 6912, 7552, 8192, 1536, 12032, 1792, 6528, 5760, 1536, 2816, 22656, 11136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[2816, 9216, 9984, 11136, 2176, 4992, 7552, 10496, 6784, 11904, 12032, 5760, 12544, 4096, 4864, 2944, 6912, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1022, 0, 4095, 163, 1988, 0, 419, 3737, 1942, 357, 27, 129, 759, 70, 0, 2779, 22, 0, 9, 0, 2, 1901, 44, 856, 80, 121, 0, 62, 190, 3284, 1065, 0, 3678, 378, 2974, 0, 0, 0, 695, 0, 1557, 18, 0, 0, 0, 1, 2, 32, 3, 68, 1, 0, 201, 404, 22, 1, 531, 4042, 2279, 1441, 2605, 0, 4, 5, 2, 21, 4080, 1454, 1274, 50, 3835, 6, 4067, 2713, 58, 146, 966, 0, 3135, 0, 4089, 3995, 8, 404, 3941, 9, 1636, 3780, 33, 2195, 751, 4089, 511, 1549, 477, 1502, 25, 2050, 4095, 741, 1304, 0, 4002, 205, 4, 422, 2181, 15, 4094, 345, 6, 4022, 0, 183, 100, 0, 14, 880, 8, 268, 3032, 4, 0, 1214, 496, 74, 357, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[2953, 0, 3080, 622, 1275, 0, 384, 1234, 3443, 737, 2716, 130, 3022, 79, 593, 3532, 94, 0, 3543, 0, 683, 1839, 1883, 10, 0, 2860, 0, 625, 6, 1638, 2267, 9, 2889, 3056, 3550, 0, 0, 0, 4, 1, 1652, 2362, 10, 0, 1, 6, 1023, 91, 0, 14, 134, 0, 9, 2430, 1, 0, 146, 4078, 34, 111, 519, 1063, 34, 0, 0, 24, 1340, 15, 187, 49, 3176, 31, 1052, 407, 19, 69, 86, 0, 3794, 4, 4037, 3772, 731, 297, 4035, 19, 3270, 4042, 3, 5, 134, 4096, 1423, 2343, 7, 2950, 0, 2006, 4096, 1182, 974, 0, 34, 412, 3, 1392, 3912, 345, 1689, 1632, 2, 95, 1, 24, 588, 296, 1, 3272, 1562, 0, 1, 0, 0, 1388, 34, 37, 2, 2200] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2514, 0, 4083, 1013, 607, 0, 2595, 1895, 3804, 50, 44, 5, 3784, 1964, 0, 625, 0, 0, 1438, 0, 1, 369, 226, 256, 114, 0, 0, 157, 4, 3290, 3529, 48, 717, 919, 3550, 0, 0, 8, 570, 0, 166, 1, 0, 0, 0, 18, 66, 186, 0, 22, 5, 0, 0, 2460, 1, 0, 95, 4064, 253, 1075, 3547, 0, 1, 77, 0, 35, 3732, 1276, 2485, 5, 2473, 143, 3354, 197, 517, 0, 902, 0, 48, 0, 3653, 2010, 269, 363, 4090, 1041, 2426, 3550, 56, 1422, 112, 4092, 16, 103, 281, 17, 3, 3660, 3971, 1430, 4067, 9, 3818, 77, 0, 397, 3817, 5, 3645, 3, 19, 2434, 290, 48, 2619, 0, 3, 4048, 263, 19, 588, 0, 0, 3078, 1, 0, 0, 3876] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4067, 0, 152, 738, 3586, 12, 2663, 8, 4065, 198, 234, 4, 965, 2060, 0, 875, 17, 8, 3225, 1, 1, 1066, 41, 8, 22, 1855, 1, 38, 4, 2679, 4091, 1, 2913, 2981, 2690, 0, 0, 0, 3, 14, 21, 182, 0, 0, 0, 0, 2835, 84, 16, 1082, 118, 3, 101, 3014, 0, 1, 11, 3672, 236, 2416, 1765, 0, 203, 0, 11, 2344, 2235, 116, 417, 9, 3955, 4, 948, 763, 0, 15, 36, 7, 3716, 1, 4013, 1287, 41, 396, 3449, 85, 817, 3434, 65, 1695, 574, 4050, 1784, 3575, 89, 90, 0, 0, 4091, 583, 2155, 0, 2753, 0, 555, 79, 2128, 11, 3862, 1497, 246, 2834, 122, 92, 20, 0, 1, 2965, 2215, 59, 233, 0, 5, 3440, 34, 3, 51, 3971] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[3555, 2, 1646, 2906, 2089, 0, 1217, 787, 1144, 341, 251, 9, 469, 104, 0, 2583, 7, 1, 2245, 8, 650, 4041, 1923, 1314, 30, 857, 0, 1388, 5, 733, 4018, 0, 164, 59, 4029, 0, 0, 0, 1517, 0, 292, 59, 0, 0, 1, 2, 195, 0, 0, 355, 110, 0, 134, 68, 0, 9, 17, 4071, 2111, 507, 1186, 261, 13, 42, 5, 206, 4043, 12, 2, 52, 4044, 1, 2492, 378, 8, 13, 2553, 0, 2596, 15, 3069, 2969, 1256, 3129, 1565, 1904, 85, 3998, 1, 3803, 2400, 3425, 950, 6, 0, 209, 0, 121, 4095, 3618, 3698, 0, 1031, 34, 5, 157, 3779, 642, 2625, 1651, 18, 790, 556, 828, 2278, 0, 35, 3553, 276, 961, 834, 6, 0, 1149, 10, 0, 53, 3555] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[3278, 47, 3999, 1514, 3081, 0, 518, 3793, 2711, 1994, 230, 1, 1262, 670, 1, 363, 252, 2, 958, 5, 2, 4068, 0, 26, 0, 293, 7, 11, 260, 4049, 3983, 20, 2310, 198, 4069, 1, 0, 2, 4, 27, 331, 8, 15, 0, 0, 0, 1, 1, 0, 1101, 45, 0, 9, 2643, 9, 17, 6, 3931, 1360, 932, 832, 0, 0, 270, 5, 147, 2613, 13, 655, 0, 4031, 0, 3982, 753, 1, 371, 298, 0, 3505, 38, 3852, 1252, 1465, 20, 3896, 22, 187, 4026, 0, 125, 3207, 4094, 1356, 54, 5, 26, 0, 80, 4041, 1712, 1097, 6, 2978, 1, 1, 188, 41, 17, 4043, 875, 20, 3917, 31, 659, 3674, 1, 129, 916, 1983, 11, 525, 28, 0, 3932, 650, 21, 53, 3923] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[3950, 0, 4010, 415, 2316, 0, 769, 1266, 1957, 294, 4, 0, 302, 2624, 32, 2996, 10, 3, 731, 0, 98, 3068, 246, 2787, 24, 1978, 0, 197, 17, 1890, 3896, 3, 191, 209, 4060, 54, 49, 0, 168, 0, 263, 2230, 0, 0, 18, 326, 348, 245, 46, 30, 162, 0, 1533, 3646, 3, 0, 0, 3905, 3403, 3656, 1830, 2, 0, 334, 118, 53, 3916, 537, 1465, 62, 3990, 117, 492, 366, 20, 207, 281, 4, 1905, 7, 3673, 1456, 40, 841, 1766, 1, 686, 4005, 685, 118, 1142, 4095, 0, 109, 8, 251, 0, 360, 4096, 3630, 3172, 11, 671, 7, 0, 546, 641, 202, 4078, 302, 999, 283, 607, 19, 1075, 6, 4, 3448, 1068, 95, 2787, 0, 5, 2530, 59, 1708, 52, 3601] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[3782, 1, 3766, 406, 2264, 28, 15, 4061, 2910, 2371, 0, 0, 142, 1065, 0, 3622, 8, 4, 2, 40, 4, 1590, 0, 75, 613, 2404, 125, 3, 0, 3996, 2920, 1, 3917, 218, 4068, 0, 0, 830, 0, 13, 56, 19, 12, 0, 3, 596, 298, 0, 1, 251, 2, 36, 3428, 617, 23, 5, 0, 4057, 1612, 1220, 273, 4, 171, 0, 2, 97, 2936, 3264, 542, 218, 4054, 0, 2049, 42, 19, 27, 295, 2, 4081, 64, 3881, 1334, 3834, 16, 3280, 0, 70, 4093, 0, 861, 326, 2566, 47, 50, 0, 0, 2, 2, 3663, 570, 4068, 1901, 2716, 0, 1, 60, 1462, 17, 4093, 276, 0, 1664, 727, 69, 48, 46, 1, 530, 4, 0, 2928, 0, 0, 3653, 764, 1577, 197, 4035] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1833, 1013, 607, 0, 2595, 1895, 3804, 50, 44, 5, 3784, 1964, 0, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 0, 1, 369, 226, 256, 114, 0, 0, 157, 4, 3290, 3529, 48, 2467, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 717, 919, 3550, 0, 0, 8, 570, 0, 166, 1, 0, 0, 0, 18, 66, 186, 3971, 2933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 5, 0, 0, 2460, 1, 0, 95, 4064, 253, 1075, 3547, 0, 1, 77, 4092, 617, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 3732, 1276, 2485, 5, 2473, 143, 3354, 197, 517, 0, 902, 0, 48, 0, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2665, 2010, 269, 363, 4090, 1041, 2426, 0, 56, 1422, 112, 0, 16, 103, 281, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3660, 0, 1430, 4067, 9, 3818, 77, 0, 397, 3817, 5, 2732, 3, 19, 2434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 48, 2619, 0, 3, 4048, 263, 19, 588, 0, 0, 3078, 1, 0, 0, 3876, 1760, 988, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18219, 12389, 13105, 16356, 15845, 14871, 22471, 17816] avg=16384.0000 max=22471 max_rank=6 min=12389 min_rank=1 imbalance_ratio=1.371521 max_load_reduction=108601 max_load_reduction_pct=0.828560 moved_to_echo=71003 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1389, 622, 1275, 0, 384, 1234, 3443, 737, 2716, 130, 3022, 79, 593, 3532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 3543, 0, 683, 1839, 1883, 10, 0, 2860, 0, 625, 6, 1638, 2267, 9, 2897, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2889, 3056, 3550, 0, 0, 0, 4, 1, 1652, 2362, 10, 0, 1, 6, 1023, 91, 4096, 3338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 134, 0, 9, 2430, 1, 0, 146, 4078, 34, 111, 519, 1063, 34, 0, 4096, 704, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 1340, 15, 187, 49, 3176, 31, 1052, 407, 19, 69, 86, 0, 3794, 4, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2949, 3772, 731, 297, 4035, 19, 3270, 0, 3, 5, 134, 0, 1423, 2343, 7, 2950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2006, 0, 1182, 974, 0, 34, 412, 3, 1392, 3912, 345, 1270, 1632, 2, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 24, 588, 296, 1, 3272, 1562, 0, 1, 0, 0, 1388, 34, 37, 2, 2200, 1325, 1088, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19156, 18720, 22079, 13429, 10565, 21938, 13259, 11926] avg=16384.0000 max=22079 max_rank=2 min=10565 min_rank=4 imbalance_ratio=1.347595 max_load_reduction=108993 max_load_reduction_pct=0.831551 moved_to_echo=57688 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1809, 415, 2316, 0, 769, 1266, 1957, 294, 4, 0, 302, 2624, 32, 2996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3, 731, 0, 98, 3068, 246, 2787, 24, 1978, 0, 197, 17, 1890, 3896, 3, 3876, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 209, 4060, 54, 49, 0, 168, 0, 263, 2230, 0, 0, 18, 326, 348, 245, 4096, 3308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 30, 162, 0, 1533, 3646, 3, 0, 0, 3905, 3403, 3656, 1830, 2, 0, 334, 4095, 697, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 53, 3916, 537, 1465, 62, 3990, 117, 492, 366, 20, 207, 281, 4, 1905, 7, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2683, 1456, 40, 841, 1766, 1, 686, 0, 685, 118, 1142, 0, 0, 109, 8, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 3630, 3172, 11, 671, 7, 0, 546, 641, 202, 3066, 302, 999, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 19, 1075, 6, 4, 3448, 1068, 95, 2787, 0, 5, 2530, 59, 1708, 52, 3601, 1725, 990, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14784, 19300, 15565, 23416, 14294, 9786, 13890, 20037] avg=16384.0000 max=23416 max_rank=3 min=9786 min_rank=5 imbalance_ratio=1.429199 max_load_reduction=107656 max_load_reduction_pct=0.821350 moved_to_echo=58007 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 47, 1804, 1514, 3081, 0, 518, 3793, 2711, 1994, 230, 1, 1262, 670, 1, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 2, 958, 5, 2, 4068, 0, 26, 0, 293, 7, 11, 260, 4049, 3983, 20, 3216, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2310, 198, 4069, 1, 0, 2, 4, 27, 331, 8, 15, 0, 0, 0, 1, 1, 4041, 3325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1101, 45, 0, 9, 2643, 9, 17, 6, 3931, 1360, 932, 832, 0, 0, 270, 4094, 701, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 147, 2613, 13, 655, 0, 4031, 0, 3982, 753, 1, 371, 298, 0, 3505, 38, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2814, 1252, 1465, 20, 3896, 22, 187, 0, 0, 125, 3207, 0, 1356, 54, 5, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 1712, 1097, 6, 2978, 1, 1, 188, 41, 17, 3040, 875, 20, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 659, 3674, 1, 129, 916, 1983, 11, 525, 28, 0, 3932, 650, 21, 53, 3923, 1720, 1038, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17989, 17627, 14333, 16012, 17159, 14429, 13973, 19550] avg=16384.0000 max=19550 max_rank=7 min=13973 min_rank=6 imbalance_ratio=1.193237 max_load_reduction=111522 max_load_reduction_pct=0.850845 moved_to_echo=65111 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 69, 738, 3586, 12, 2663, 8, 4065, 198, 234, 4, 965, 2060, 0, 875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 8, 3225, 1, 1, 1066, 41, 8, 22, 1855, 1, 38, 4, 2679, 4091, 1, 3990, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2913, 2981, 2690, 0, 0, 0, 3, 14, 21, 182, 0, 0, 0, 0, 2835, 84, 4091, 2836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1082, 118, 3, 101, 3014, 0, 1, 11, 3672, 236, 2416, 1765, 0, 203, 0, 4050, 598, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2344, 2235, 116, 417, 9, 3955, 4, 948, 763, 0, 15, 36, 7, 3716, 1, 714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2931, 1287, 41, 396, 3449, 85, 817, 0, 65, 1695, 574, 0, 1784, 3575, 89, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 2155, 0, 2753, 0, 555, 79, 2128, 11, 2904, 1497, 246, 2834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 92, 20, 0, 1, 2965, 2215, 59, 233, 0, 5, 3440, 34, 3, 51, 3971, 65, 1082, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15477, 17066, 18650, 17363, 15291, 16878, 15745, 14602] avg=16384.0000 max=18650 max_rank=2 min=14602 min_rank=7 imbalance_ratio=1.138306 max_load_reduction=112422 max_load_reduction_pct=0.857712 moved_to_echo=62516 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 743, 2906, 2089, 0, 1217, 787, 1144, 341, 251, 9, 469, 104, 0, 2583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 2245, 8, 650, 4041, 1923, 1314, 30, 857, 0, 1388, 5, 733, 4018, 0, 3487, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 59, 4029, 0, 0, 0, 1517, 0, 292, 59, 0, 0, 1, 2, 195, 0, 4095, 3302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 110, 0, 134, 68, 0, 9, 17, 4071, 2111, 507, 1186, 261, 13, 42, 3425, 696, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 206, 4043, 12, 2, 52, 4044, 1, 2492, 378, 8, 13, 2553, 0, 2596, 15, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2242, 2969, 1256, 3129, 1565, 1904, 85, 0, 1, 3803, 2400, 0, 950, 6, 0, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 3618, 3698, 0, 1031, 34, 5, 157, 3779, 642, 1974, 1651, 18, 790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 828, 2278, 0, 35, 3553, 276, 961, 834, 6, 0, 1149, 10, 0, 53, 3555, 708, 827, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12645, 20902, 13715, 13073, 16905, 20519, 17518, 15795] avg=16384.0000 max=20902 max_rank=1 min=12645 min_rank=0 imbalance_ratio=1.275757 max_load_reduction=110170 max_load_reduction_pct=0.840530 moved_to_echo=70737 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1848, 163, 1988, 0, 419, 3737, 1942, 357, 27, 129, 759, 70, 0, 2779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 9, 0, 2, 1901, 44, 856, 80, 121, 0, 62, 190, 3284, 1065, 0, 1003, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3678, 378, 2974, 0, 0, 0, 695, 0, 1557, 18, 0, 0, 0, 1, 2, 32, 4095, 3121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 68, 1, 0, 201, 404, 22, 1, 531, 4042, 2279, 1441, 2605, 0, 4, 5, 4089, 659, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 21, 4080, 1454, 1274, 50, 3835, 6, 4067, 2713, 58, 146, 966, 0, 3135, 0, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2987, 3995, 8, 404, 3941, 9, 1636, 0, 33, 2195, 751, 0, 511, 1549, 477, 1502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2050, 0, 741, 1304, 0, 4002, 205, 4, 422, 2181, 15, 3078, 345, 6, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 100, 0, 14, 880, 8, 268, 3032, 4, 0, 1214, 496, 74, 357, 4090, 1761, 1102, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14218, 9125, 16551, 16374, 22564, 19998, 18400, 13842] avg=16384.0000 max=22564 max_rank=4 min=9125 min_rank=1 imbalance_ratio=1.377197 max_load_reduction=108508 max_load_reduction_pct=0.827850 moved_to_echo=74804 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1699, 406, 2264, 28, 15, 4061, 2910, 2371, 0, 0, 142, 1065, 0, 3622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 2, 40, 4, 1590, 0, 75, 613, 2404, 125, 3, 0, 3996, 2920, 1, 3711, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3917, 218, 4068, 0, 0, 830, 0, 13, 56, 19, 12, 0, 3, 596, 298, 0, 3663, 3381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 251, 2, 36, 3428, 617, 23, 5, 0, 4057, 1612, 1220, 273, 4, 171, 0, 2566, 712, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 97, 2936, 3264, 542, 218, 4054, 0, 2049, 42, 19, 27, 295, 2, 4081, 64, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2835, 1334, 3834, 16, 3280, 0, 70, 0, 0, 861, 326, 0, 47, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 570, 4068, 1901, 2716, 0, 1, 60, 1462, 17, 3077, 276, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 727, 69, 48, 46, 1, 530, 4, 0, 2928, 0, 0, 3653, 764, 1577, 197, 4035, 1620, 1046, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18584, 15943, 17074, 15049, 18449, 12653, 15816, 17504] avg=16384.0000 max=18584 max_rank=0 min=12653 min_rank=5 imbalance_ratio=1.134277 max_load_reduction=112488 max_load_reduction_pct=0.858215 moved_to_echo=64422 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[22144, 18176, 7680, 5504, 26112, 3200, 9216, 0, 896, 10240, 8704, 0, 6144, 7808, 896, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2432, 2048, 10496, 384, 256, 19712, 7424, 1536, 11008, 128, 128, 20480, 2048, 3456, 768, 29312, 10752, 8192, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 2944, 24960, 6784, 7040, 512, 29568, 384, 18560, 5632, 768, 896, 5504, 128, 22784, 256, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[16896, 8064, 29056, 128, 128, 896, 3072, 128, 4352, 4992, 128, 0, 128, 1024, 4864, 640, 32256, 25600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 11264, 7808, 17280, 128, 8704, 16896, 22016, 6400, 3584, 384, 10752, 8704, 640, 17408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 2944, 640, 128, 5504, 15360, 128, 128, 896, 31872, 11392, 11392, 12672, 1408, 512, 768, 30592, 5504, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 8320, 0, 13568, 20608, 2048, 18048, 768, 640, 3328, 18048, 1280, 21248, 6656, 1408, 16128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[512, 128, 12160, 128, 1536, 18048, 4480, 5376, 896, 10368, 256, 2560, 512, 21632, 25856, 128, 24704, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[3724, 15, 4077, 841, 0, 2671, 0, 369, 43, 4089, 0, 4092, 3394, 0, 0, 133, 0, 8, 0, 0, 325, 22, 3, 0, 3429, 33, 3320, 129, 946, 4077, 0, 795, 4079, 0, 2, 0, 0, 0, 407, 72, 3805, 0, 44, 2054, 297, 804, 698, 1, 364, 4068, 0, 12, 132, 0, 447, 356, 232, 0, 2028, 3193, 3963, 3257, 2332, 3383, 236, 0, 4014, 0, 0, 890, 3945, 37, 454, 1, 3603, 176, 1, 2, 19, 13, 244, 4096, 177, 754, 66, 4077, 0, 4081, 2, 1592, 1, 0, 9, 1464, 0, 2436, 6, 69, 1146, 0, 60, 219, 7, 13, 0, 0, 3778, 25, 0, 4085, 91, 0, 2, 0, 4096, 2357, 3531, 734, 5, 0, 366, 0, 16, 1, 3, 0, 1642, 3865] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[3737, 0, 3693, 0, 0, 4094, 0, 28, 0, 4074, 0, 4090, 3341, 1763, 0, 26, 2, 528, 0, 0, 155, 64, 0, 85, 173, 1, 1858, 0, 0, 4073, 1258, 619, 4076, 5, 203, 39, 1, 2041, 0, 43, 4096, 0, 8, 779, 3423, 143, 600, 0, 0, 4096, 7, 0, 881, 24, 2141, 572, 569, 25, 0, 3864, 3152, 4017, 14, 0, 2825, 0, 4095, 0, 141, 2283, 4076, 1, 189, 61, 2089, 76, 0, 0, 0, 1, 0, 4096, 2, 4058, 1311, 3278, 1938, 4005, 0, 4052, 0, 6, 0, 10, 23, 114, 0, 2, 4020, 46, 496, 0, 0, 106, 0, 32, 23, 0, 0, 3992, 0, 0, 0, 0, 3972, 3180, 3971, 3, 0, 0, 3928, 60, 3, 0, 11, 0, 11, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[3499, 0, 2023, 0, 2, 3881, 0, 1177, 0, 4096, 1, 3524, 336, 1, 5, 148, 0, 1304, 0, 0, 73, 33, 0, 0, 3673, 1, 4084, 0, 0, 4094, 110, 0, 4096, 3, 0, 1009, 99, 3753, 0, 0, 3584, 58, 4, 2, 3952, 315, 3052, 104, 0, 4096, 28, 3484, 0, 18, 4001, 0, 3, 49, 0, 2246, 4096, 2840, 2769, 1650, 1229, 0, 3488, 0, 0, 265, 4003, 3, 2, 1, 4096, 257, 31, 1569, 0, 16, 0, 58, 7, 25, 0, 3855, 25, 4093, 0, 3283, 4, 0, 0, 0, 1, 112, 0, 2, 2644, 56, 1423, 744, 0, 9, 35, 2, 14, 1955, 0, 44, 6, 37, 3, 0, 4077, 4094, 3722, 1, 1, 0, 3896, 2, 519, 6, 1101, 0, 180, 2700] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4087, 0, 998, 0, 0, 4010, 0, 3450, 0, 4096, 0, 3146, 3770, 47, 0, 75, 2, 2, 0, 0, 0, 104, 0, 0, 1397, 4096, 0, 0, 1942, 3944, 30, 313, 19, 0, 0, 4063, 5, 3610, 0, 0, 576, 0, 363, 400, 38, 93, 200, 0, 0, 4096, 4096, 0, 0, 0, 71, 0, 828, 10, 0, 135, 4095, 155, 153, 4094, 4054, 0, 3828, 0, 0, 0, 3213, 103, 35, 1, 1761, 0, 0, 0, 0, 14, 200, 122, 1517, 22, 3337, 4096, 3851, 3517, 0, 3981, 14, 831, 0, 0, 4030, 1392, 0, 3, 18, 2, 1, 2480, 0, 0, 19, 138, 4089, 2816, 0, 198, 41, 1, 1, 323, 4096, 4093, 732, 0, 0, 572, 32, 4042, 891, 0, 2, 0, 19, 3935] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4096, 0, 3614, 14, 0, 3950, 1, 1192, 0, 4091, 0, 4096, 1, 9, 0, 17, 0, 13, 0, 1, 95, 105, 0, 0, 3962, 37, 8, 1, 39, 4091, 0, 759, 4081, 0, 0, 0, 0, 239, 0, 0, 4077, 3, 0, 22, 3997, 0, 415, 0, 0, 4095, 0, 85, 16, 0, 1224, 0, 2314, 2, 2, 3906, 4045, 4009, 1833, 3350, 16, 0, 3954, 0, 0, 3, 4094, 9, 48, 136, 4082, 0, 0, 0, 1, 0, 0, 4049, 5, 3836, 1, 3656, 1, 4078, 0, 73, 0, 0, 0, 21, 3104, 1866, 0, 0, 4028, 0, 235, 182, 0, 0, 3, 89, 0, 574, 2, 4094, 5, 3, 66, 8, 4096, 4072, 4093, 3, 2, 0, 863, 0, 3905, 0, 3777, 0, 2, 30] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3382, 0, 4096, 16, 187, 4093, 0, 3403, 0, 3433, 0, 4096, 0, 118, 0, 22, 0, 0, 0, 0, 0, 4, 0, 248, 1894, 1028, 3483, 5, 7, 4089, 1, 189, 3845, 0, 0, 1, 0, 2735, 2, 0, 3836, 5, 3, 19, 3604, 0, 21, 0, 490, 4094, 444, 926, 10, 0, 8, 19, 3304, 0, 1758, 1688, 3667, 4027, 43, 989, 2440, 0, 3969, 104, 0, 2047, 4067, 2865, 3, 71, 3966, 17, 0, 38, 3, 36, 1, 2272, 1, 3, 2, 4096, 3384, 4096, 0, 1475, 0, 0, 3, 4093, 332, 506, 8, 3, 3070, 0, 1150, 0, 0, 43, 0, 28, 4, 558, 6, 4043, 21, 2, 2325, 126, 4096, 41, 4034, 1345, 59, 453, 36, 0, 11, 12, 370, 0, 2, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[516, 7, 4058, 13, 28, 186, 5, 2580, 13, 4021, 0, 1778, 4053, 8, 2, 0, 27, 5, 205, 12, 3794, 0, 0, 0, 6, 38, 745, 0, 65, 3255, 76, 3898, 2836, 466, 24, 0, 479, 200, 4, 362, 3432, 0, 2257, 3998, 4046, 3, 91, 1169, 5, 4065, 0, 28, 129, 6, 3, 0, 2536, 0, 16, 4095, 4056, 4010, 3841, 1041, 28, 22, 2416, 0, 0, 686, 3891, 0, 2586, 3, 2667, 3513, 13, 0, 0, 105, 17, 3628, 9, 2, 903, 4093, 44, 3962, 711, 82, 5, 0, 39, 0, 125, 508, 3, 292, 996, 1385, 0, 2, 1, 0, 0, 22, 17, 0, 0, 4096, 4, 1, 57, 992, 4058, 4079, 50, 0, 0, 1, 72, 63, 3846, 198, 0, 56, 4091, 4040] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4093, 0, 4001, 14, 1, 3848, 0, 3545, 8, 4094, 0, 4096, 1766, 266, 0, 347, 33, 0, 0, 31, 75, 23, 0, 7, 3794, 4067, 8, 0, 2150, 4084, 0, 16, 370, 0, 63, 35, 0, 2257, 0, 0, 2027, 19, 0, 4, 3420, 10, 71, 0, 0, 4076, 4065, 7, 5, 2, 0, 996, 63, 1, 0, 1541, 4050, 4083, 394, 1936, 4014, 8, 1872, 223, 0, 1619, 4093, 0, 50, 0, 4021, 18, 0, 0, 6, 4, 0, 1756, 3535, 642, 2477, 4094, 9, 4086, 0, 3313, 0, 8, 2, 4, 220, 193, 15, 0, 3097, 0, 6, 263, 0, 0, 45, 0, 143, 4076, 51, 3380, 0, 0, 3633, 19, 4080, 4091, 164, 654, 12, 0, 15, 168, 472, 1, 0, 5, 118, 435] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3724, 15, 4077, 841, 0, 2671, 0, 369, 43, 0, 0, 1989, 3394, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 325, 22, 3, 0, 3429, 33, 3320, 129, 946, 4077, 0, 795, 3147, 1164, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 2, 0, 0, 0, 407, 72, 3805, 0, 44, 2054, 297, 804, 698, 1, 1625, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 0, 12, 132, 0, 447, 356, 232, 0, 2028, 3193, 2209, 3257, 2332, 3383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 4014, 0, 0, 890, 3945, 37, 454, 1, 3603, 176, 1, 2, 19, 13, 1135, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 4096, 177, 754, 66, 4077, 0, 934, 2, 1592, 1, 0, 9, 1464, 0, 2436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 69, 1146, 0, 60, 219, 7, 13, 0, 0, 3778, 25, 0, 4085, 91, 0, 4068, 3496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2961, 2357, 3531, 734, 5, 0, 366, 0, 16, 1, 3, 0, 1642, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17256, 17991, 14827, 17945, 14655, 15852, 17063, 15483] avg=16384.0000 max=17991 max_rank=1 min=14655 min_rank=4 imbalance_ratio=1.098083 max_load_reduction=113081 max_load_reduction_pct=0.862740 moved_to_echo=63053 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3737, 0, 3693, 0, 0, 4094, 0, 28, 0, 0, 0, 1987, 3341, 1763, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 528, 0, 0, 155, 64, 0, 85, 173, 1, 1858, 0, 0, 4073, 1258, 619, 3088, 1164, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 5, 203, 39, 1, 2041, 0, 43, 4096, 0, 8, 779, 3423, 143, 600, 0, 1292, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 881, 24, 2141, 572, 569, 25, 0, 3864, 1758, 4017, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2825, 0, 4095, 0, 141, 2283, 4076, 1, 189, 61, 2089, 76, 0, 0, 0, 1, 1101, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4058, 1311, 3278, 1938, 917, 0, 4052, 0, 6, 0, 10, 23, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4020, 46, 496, 0, 0, 106, 0, 32, 23, 0, 0, 3992, 0, 0, 4096, 3482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2871, 3180, 3971, 3, 0, 0, 3928, 60, 3, 0, 11, 0, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18669, 13660, 17688, 13872, 17040, 19805, 16295, 14043] avg=16384.0000 max=19805 max_rank=5 min=13660 min_rank=1 imbalance_ratio=1.208801 max_load_reduction=111267 max_load_reduction_pct=0.848900 moved_to_echo=67183 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 3614, 14, 0, 3950, 1, 1192, 0, 0, 0, 1990, 1, 9, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 95, 105, 0, 0, 3962, 37, 8, 1, 39, 4091, 0, 759, 3144, 1166, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 239, 0, 0, 4077, 3, 0, 22, 3997, 0, 415, 0, 1658, 940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 16, 0, 1224, 0, 2314, 2, 2, 3906, 2255, 4009, 1833, 3350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3954, 0, 0, 3, 4094, 9, 48, 136, 4082, 0, 0, 0, 1, 0, 1135, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 5, 3836, 1, 3656, 1, 934, 0, 73, 0, 0, 0, 21, 3104, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 235, 182, 0, 0, 3, 89, 0, 574, 2, 4094, 5, 3, 4095, 3496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 8, 2961, 4072, 4093, 3, 2, 0, 863, 0, 3905, 0, 3777, 0, 2, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14884, 14016, 15432, 18996, 13610, 17546, 16806, 19782] avg=16384.0000 max=19782 max_rank=7 min=13610 min_rank=4 imbalance_ratio=1.207397 max_load_reduction=111290 max_load_reduction_pct=0.849075 moved_to_echo=67744 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4087, 0, 998, 0, 0, 4010, 0, 3450, 0, 0, 0, 1529, 3770, 47, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 104, 0, 0, 1397, 4096, 0, 0, 1942, 3944, 30, 313, 2712, 895, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 4063, 5, 3610, 0, 0, 576, 0, 363, 400, 38, 93, 200, 0, 1679, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 71, 0, 828, 10, 0, 135, 2283, 155, 153, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 3828, 0, 0, 0, 3213, 103, 35, 1, 1761, 0, 0, 0, 0, 14, 1135, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 122, 1517, 22, 3337, 4096, 3851, 805, 0, 3981, 14, 831, 0, 0, 4030, 1392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 18, 2, 1, 2480, 0, 0, 19, 138, 4089, 2816, 0, 198, 41, 1, 4096, 3501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 323, 2961, 4093, 732, 0, 0, 572, 32, 4042, 891, 0, 2, 0, 19, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17966, 16032, 11768, 11825, 14277, 24198, 17403, 17603] avg=16384.0000 max=24198 max_rank=5 min=11768 min_rank=2 imbalance_ratio=1.476929 max_load_reduction=106874 max_load_reduction_pct=0.815384 moved_to_echo=73481 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3499, 0, 2023, 0, 2, 3881, 0, 1177, 0, 0, 1, 1705, 336, 1, 5, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1304, 0, 0, 73, 33, 0, 0, 3673, 1, 4084, 0, 0, 4094, 110, 0, 3160, 1006, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 1009, 99, 3753, 0, 0, 3584, 58, 4, 2, 3952, 315, 3052, 104, 1684, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3484, 0, 18, 4001, 0, 3, 49, 0, 2246, 2274, 2840, 2769, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1229, 0, 3488, 0, 0, 265, 4003, 3, 2, 1, 4096, 257, 31, 1569, 0, 16, 1134, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 7, 25, 0, 3855, 25, 933, 0, 3283, 4, 0, 0, 0, 1, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2644, 56, 1423, 744, 0, 9, 35, 2, 14, 1955, 0, 44, 6, 37, 4096, 3501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2943, 4094, 3722, 1, 1, 0, 3896, 2, 519, 6, 1101, 0, 180, 2700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12778, 18133, 22528, 19362, 16232, 8303, 14568, 19168] avg=16384.0000 max=22528 max_rank=2 min=8303 min_rank=5 imbalance_ratio=1.375000 max_load_reduction=108544 max_load_reduction_pct=0.828125 moved_to_echo=58271 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3382, 0, 4096, 16, 187, 4093, 0, 3403, 0, 0, 0, 1990, 0, 118, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 248, 1894, 1028, 3483, 5, 7, 4089, 1, 189, 3158, 1166, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3845, 0, 0, 1, 0, 2735, 2, 0, 3836, 5, 3, 19, 3604, 0, 21, 0, 1503, 940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 444, 926, 10, 0, 8, 19, 3304, 0, 1758, 1688, 2045, 4027, 43, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2440, 0, 3969, 104, 0, 2047, 4067, 2865, 3, 71, 3966, 17, 0, 38, 3, 36, 1135, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2272, 1, 3, 2, 4096, 3384, 938, 0, 1475, 0, 0, 3, 4093, 332, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 3070, 0, 1150, 0, 0, 43, 0, 28, 4, 558, 6, 4043, 21, 2, 4094, 2935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2325, 126, 2961, 41, 4034, 1345, 59, 453, 36, 0, 11, 12, 370, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17307, 15770, 16514, 15751, 20880, 17106, 15965, 11779] avg=16384.0000 max=20880 max_rank=4 min=11779 min_rank=7 imbalance_ratio=1.274414 max_load_reduction=110192 max_load_reduction_pct=0.840698 moved_to_echo=65730 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[516, 7, 4058, 13, 28, 186, 5, 2580, 13, 0, 0, 864, 4053, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 5, 205, 12, 3794, 0, 0, 0, 6, 38, 745, 0, 65, 3255, 76, 3898, 3055, 506, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2836, 466, 24, 0, 479, 200, 4, 362, 3432, 0, 2257, 3998, 4046, 3, 91, 1169, 1663, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 28, 129, 6, 3, 0, 2536, 0, 16, 4095, 2261, 4010, 3841, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 22, 2416, 0, 0, 686, 3891, 0, 2586, 3, 2667, 3513, 13, 0, 0, 105, 1125, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3628, 9, 2, 903, 4093, 44, 907, 711, 82, 5, 0, 39, 0, 125, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 292, 996, 1385, 0, 2, 1, 0, 0, 22, 17, 0, 0, 4096, 4, 1, 4065, 3436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 992, 2933, 4079, 50, 0, 0, 1, 72, 63, 3846, 198, 0, 56, 4091, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12333, 16272, 21438, 17971, 17187, 11073, 14320, 20478] avg=16384.0000 max=21438 max_rank=2 min=11073 min_rank=5 imbalance_ratio=1.308472 max_load_reduction=109634 max_load_reduction_pct=0.836441 moved_to_echo=63058 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 4001, 14, 1, 3848, 0, 3545, 8, 0, 0, 1990, 1766, 266, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 31, 75, 23, 0, 7, 3794, 4067, 8, 0, 2150, 4084, 0, 16, 3150, 1166, 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 0, 63, 35, 0, 2257, 0, 0, 2027, 19, 0, 4, 3420, 10, 71, 0, 1661, 940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 7, 5, 2, 0, 996, 63, 1, 0, 1541, 2257, 4083, 394, 1936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 8, 1872, 223, 0, 1619, 4093, 0, 50, 0, 4021, 18, 0, 0, 6, 4, 1131, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1756, 3535, 642, 2477, 4094, 9, 936, 0, 3313, 0, 8, 2, 4, 220, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3097, 0, 6, 263, 0, 0, 45, 0, 143, 4076, 51, 3380, 0, 0, 4076, 3500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3633, 19, 2949, 4091, 164, 654, 12, 0, 15, 168, 472, 1, 0, 5, 118, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19879, 19198, 10877, 15350, 17191, 17189, 18652, 12736] avg=16384.0000 max=19879 max_rank=0 min=10877 min_rank=2 imbalance_ratio=1.213318 max_load_reduction=111193 max_load_reduction_pct=0.848335 moved_to_echo=65768 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[27136, 128, 26624, 1024, 256, 26752, 128, 15744, 128, 0, 128, 14080, 16768, 2304, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[14848, 128, 27648, 384, 256, 7808, 31488, 3072, 3456, 384, 26368, 4096, 128, 1664, 128, 256, 9088, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[6144, 1536, 23552, 26112, 20352, 2816, 128, 1152, 9216, 4352, 9728, 256, 5376, 128, 6144, 15104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[512, 20096, 5376, 9344, 8192, 31360, 9344, 7424, 768, 17920, 128, 896, 128, 5632, 7936, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 1920, 256, 128, 4608, 384, 128, 384, 18432, 9344, 13568, 256, 5248, 31744, 1536, 6656, 24704, 8320, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 384, 19072, 1536, 3456, 3968, 128, 256, 128, 384, 8192, 10112, 128, 23936, 256, 128, 32768, 27392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[896, 0, 8704, 4608, 1280, 128, 7936, 2048, 9856, 128, 3840, 20736, 17408, 26496, 11392, 16512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[23424, 512, 384, 5248, 640, 14848, 512, 512, 25472, 128, 2688, 7296, 22784, 1408, 5248, 1280, 12800, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 462, 2385, 103, 2, 0, 1576, 0, 4077, 3756, 188, 3643, 4009, 0, 0, 0, 5, 2069, 0, 551, 287, 1, 0, 243, 0, 3877, 88, 45, 3465, 0, 1, 4094, 5, 0, 1462, 0, 99, 222, 46, 0, 0, 2123, 1, 3425, 329, 3470, 543, 0, 158, 4096, 1, 15, 3715, 3426, 4075, 3962, 1021, 0, 4086, 4088, 0, 0, 225, 0, 3, 4092, 10, 1, 30, 0, 34, 0, 2460, 4, 0, 3982, 2, 8, 24, 7, 1, 0, 0, 2196, 2333, 0, 78, 4089, 102, 0, 51, 1, 0, 3867, 59, 14, 7, 249, 3917, 0, 3804, 1237, 0, 3806, 0, 0, 114, 0, 0, 69, 26, 0, 0, 2921, 4044, 4, 3590, 1, 20, 0, 369, 0, 4096, 0, 3770, 3, 57, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 4086, 1339, 3, 0, 3139, 0, 2225, 3806, 3664, 4020, 4096, 341, 0, 3, 51, 20, 0, 2, 3900, 5, 22, 0, 0, 4086, 4, 63, 10, 6, 3554, 2833, 23, 1, 1, 0, 0, 297, 3111, 473, 0, 123, 132, 3177, 11, 275, 0, 8, 0, 4053, 1410, 14, 3809, 1593, 4056, 4088, 0, 0, 2, 4096, 0, 0, 0, 0, 122, 56, 4029, 0, 2249, 159, 1722, 0, 1376, 0, 0, 4029, 29, 0, 277, 0, 0, 374, 0, 4085, 4092, 0, 0, 4086, 0, 26, 76, 0, 0, 4096, 1, 1774, 0, 261, 4092, 0, 1820, 114, 4, 2027, 0, 0, 440, 0, 0, 2, 2107, 0, 0, 4039, 8, 0, 4093, 2, 0, 0, 3, 0, 3471, 0, 3874, 0, 25, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4, 0, 4096, 838, 499, 2106, 4094, 488, 3238, 1422, 125, 3871, 4096, 193, 472, 0, 3, 4094, 0, 5, 3578, 2, 0, 0, 0, 1752, 22, 0, 2, 10, 1486, 22, 221, 0, 1, 982, 3447, 0, 86, 715, 452, 3, 1679, 2, 9, 15, 113, 0, 0, 10, 3651, 0, 4093, 68, 3785, 4095, 170, 0, 3441, 2614, 21, 0, 0, 0, 0, 958, 1017, 0, 4093, 0, 4, 137, 126, 0, 0, 155, 4, 1, 0, 241, 0, 0, 0, 4029, 3644, 37, 0, 4088, 1, 10, 31, 0, 61, 1026, 0, 3587, 0, 0, 3996, 137, 2871, 4095, 66, 1, 0, 358, 3152, 0, 0, 26, 13, 38, 0, 4013, 3529, 0, 3020, 3330, 0, 0, 0, 1732, 3893, 0, 3279, 0, 4082, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[252, 3920, 4085, 79, 0, 138, 3811, 4089, 4093, 4088, 4016, 1657, 3993, 0, 0, 0, 0, 172, 169, 683, 487, 1770, 0, 0, 0, 4096, 3876, 1, 122, 16, 0, 1869, 484, 34, 40, 0, 266, 0, 1284, 4025, 1, 1546, 3465, 828, 3313, 3, 0, 0, 63, 4095, 0, 0, 1342, 123, 4096, 1391, 2, 0, 4, 4091, 0, 0, 2, 95, 6, 1537, 11, 10, 18, 1884, 9, 0, 68, 0, 0, 638, 3, 3, 0, 38, 22, 2038, 0, 4027, 3740, 3, 0, 4083, 0, 0, 4090, 0, 0, 0, 12, 3045, 0, 2831, 4078, 719, 2867, 5, 0, 3934, 2, 0, 15, 0, 1439, 0, 2, 15, 0, 3537, 0, 0, 29, 181, 0, 0, 799, 3, 4057, 0, 3189, 0, 4, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 170, 145, 4037, 213, 643, 4086, 3402, 1262, 61, 3807, 4096, 4055, 0, 9, 119, 39, 0, 3944, 4045, 18, 0, 0, 806, 4033, 68, 0, 1361, 1, 3163, 117, 34, 0, 8, 49, 377, 0, 26, 432, 7, 1, 2, 0, 0, 1, 1, 0, 0, 23, 1, 923, 1698, 0, 4096, 399, 43, 0, 0, 2718, 27, 0, 6, 3861, 6, 3837, 39, 0, 2849, 0, 0, 11, 3, 0, 413, 4087, 1, 1, 5, 10, 2403, 0, 257, 3931, 3446, 0, 1966, 4072, 0, 0, 40, 0, 0, 1265, 2747, 20, 0, 3758, 4095, 2216, 228, 79, 68, 0, 29, 289, 1, 693, 619, 20, 4092, 0, 0, 4088, 3570, 8, 2804, 2695, 4, 0, 0, 0, 4021, 0, 3999, 0, 3847, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 2358, 3453, 4030, 3201, 167, 4046, 236, 3871, 4096, 0, 1087, 3188, 3665, 4008, 954, 4064, 2172, 0, 11, 1360, 1579, 0, 2, 1, 3887, 0, 12, 66, 2844, 67, 3973, 23, 0, 3, 1, 0, 1897, 1, 2, 1, 4006, 741, 9, 1, 141, 2403, 0, 107, 2029, 0, 0, 4075, 0, 479, 66, 3871, 23, 0, 4076, 731, 0, 72, 3, 21, 147, 5, 1709, 5, 0, 0, 3588, 771, 21, 1, 3995, 0, 123, 0, 520, 1, 370, 1, 294, 3761, 0, 1079, 4050, 963, 3247, 1158, 21, 1, 284, 0, 17, 0, 53, 679, 467, 3, 2008, 360, 4092, 0, 0, 882, 0, 40, 0, 362, 0, 0, 66, 0, 17, 3919, 1, 1, 31, 449, 0, 4096, 0, 3916, 0, 0, 318] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[3, 3728, 0, 10, 6, 0, 0, 0, 4095, 347, 200, 4081, 4096, 3984, 0, 7, 933, 0, 0, 4064, 3768, 0, 12, 0, 0, 0, 3, 0, 4089, 1, 63, 2050, 0, 0, 1903, 88, 697, 12, 3794, 1439, 0, 1307, 296, 1196, 4, 615, 0, 10, 5, 4091, 193, 75, 4028, 16, 4084, 2337, 0, 0, 63, 4090, 0, 0, 0, 0, 2, 865, 32, 24, 2885, 0, 529, 0, 3924, 17, 4033, 2865, 2, 0, 0, 4, 124, 0, 0, 3538, 4096, 0, 0, 3578, 3082, 2, 33, 0, 171, 4096, 3769, 962, 0, 2, 4093, 0, 3566, 23, 33, 1916, 1186, 0, 647, 0, 0, 1, 2837, 0, 232, 4096, 91, 312, 2587, 25, 0, 0, 0, 0, 2920, 0, 60, 0, 1929, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 637, 2, 357, 1659, 0, 3652, 4095, 4094, 3059, 0, 4081, 4026, 0, 6, 124, 5, 4, 7, 31, 2521, 47, 44, 0, 3217, 2936, 3686, 4041, 0, 3958, 441, 0, 950, 0, 0, 2, 0, 3920, 32, 0, 29, 0, 1967, 0, 25, 7, 3, 97, 4061, 1413, 3020, 117, 2, 3687, 4052, 22, 0, 0, 4096, 0, 0, 0, 3, 2260, 2001, 2, 438, 4077, 10, 431, 0, 1856, 0, 0, 3118, 42, 79, 1283, 0, 9, 0, 15, 2012, 4036, 2, 4080, 4070, 4071, 0, 14, 0, 10, 159, 1027, 259, 0, 13, 3944, 0, 40, 17, 0, 13, 21, 2997, 1317, 51, 4, 1, 14, 158, 0, 4096, 0, 0, 4073, 20, 0, 50, 61, 0, 3980, 0, 552, 0, 1, 52] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 462, 2385, 103, 2, 0, 1576, 0, 0, 0, 188, 3222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2069, 0, 551, 287, 1, 0, 243, 0, 3877, 88, 45, 3465, 0, 1, 4094, 421, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1462, 0, 99, 222, 46, 0, 0, 2123, 1, 3425, 329, 3470, 543, 0, 4009, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 4096, 1, 15, 3715, 3426, 4075, 3962, 1021, 0, 4086, 387, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4092, 10, 1, 30, 0, 34, 0, 2460, 4, 0, 3982, 2, 8, 24, 7, 3701, 2446, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2196, 2333, 0, 78, 1987, 102, 0, 51, 1, 0, 3867, 59, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 249, 3917, 0, 3804, 1237, 0, 3806, 0, 0, 114, 0, 0, 69, 26, 0, 2102, 1310, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2921, 4044, 4, 3590, 1, 20, 0, 369, 0, 3427, 0, 3770, 3, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7938, 15306, 19739, 25167, 16876, 10689, 17151, 18206] avg=16384.0000 max=25167 max_rank=3 min=7938 min_rank=0 imbalance_ratio=1.536072 max_load_reduction=105905 max_load_reduction_pct=0.807991 moved_to_echo=62922 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4086, 1339, 3, 0, 3139, 0, 0, 0, 3664, 3555, 0, 341, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 20, 0, 2, 3900, 5, 22, 0, 0, 4086, 4, 63, 10, 6, 3554, 2833, 465, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1, 1, 0, 0, 297, 3111, 473, 0, 123, 132, 3177, 11, 275, 0, 8, 4096, 2186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 1410, 14, 3809, 1593, 4056, 4088, 0, 0, 2, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 56, 4029, 0, 2249, 159, 1722, 0, 1376, 0, 0, 4029, 29, 0, 277, 0, 3709, 2479, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 0, 4085, 4092, 0, 0, 1986, 0, 26, 76, 0, 0, 4096, 1, 1774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 4092, 0, 1820, 114, 4, 2027, 0, 0, 440, 0, 0, 2, 2107, 0, 2100, 1327, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 8, 0, 4093, 2, 0, 0, 3, 0, 2904, 0, 3874, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16131, 15156, 13914, 19412, 20275, 16510, 14726, 14948] avg=16384.0000 max=20275 max_rank=4 min=13914 min_rank=2 imbalance_ratio=1.237488 max_load_reduction=110797 max_load_reduction_pct=0.845314 moved_to_echo=66459 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[252, 3920, 4085, 79, 0, 138, 3811, 4089, 0, 0, 4016, 1466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 169, 683, 487, 1770, 0, 0, 0, 4096, 3876, 1, 122, 16, 0, 1869, 191, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 34, 40, 0, 266, 0, 1284, 4025, 1, 1546, 3465, 828, 3313, 3, 0, 0, 3993, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 4095, 0, 0, 1342, 123, 4096, 1391, 2, 0, 4, 387, 0, 0, 2, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1537, 11, 10, 18, 1884, 9, 0, 68, 0, 0, 638, 3, 3, 0, 38, 3704, 2663, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 2038, 0, 4027, 3740, 3, 0, 1984, 0, 0, 4090, 0, 0, 0, 12, 3045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2831, 4078, 719, 2867, 5, 0, 3934, 2, 0, 15, 0, 1439, 0, 2, 15, 2099, 1425, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3537, 0, 0, 29, 181, 0, 0, 799, 3, 3395, 0, 3189, 0, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21856, 13609, 23303, 11600, 10664, 18961, 19936, 11143] avg=16384.0000 max=23303 max_rank=2 min=10664 min_rank=4 imbalance_ratio=1.422302 max_load_reduction=107769 max_load_reduction_pct=0.822212 moved_to_echo=60704 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4096, 838, 499, 2106, 4094, 488, 0, 0, 125, 3418, 0, 193, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 5, 3578, 2, 0, 0, 0, 1752, 22, 0, 2, 10, 1486, 22, 453, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 1, 982, 3447, 0, 86, 715, 452, 3, 1679, 2, 9, 15, 113, 0, 4096, 3181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3651, 0, 4093, 68, 3785, 4095, 170, 0, 3441, 244, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 958, 1017, 0, 4093, 0, 4, 137, 126, 0, 0, 155, 4, 1, 0, 241, 2370, 927, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 3644, 37, 0, 1983, 1, 10, 31, 0, 61, 1026, 0, 3587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 137, 2871, 4095, 66, 1, 0, 358, 3152, 0, 0, 26, 13, 38, 2105, 495, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 3529, 0, 3020, 3330, 0, 0, 0, 1732, 3248, 0, 3279, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16333, 11585, 15002, 19578, 10090, 14409, 17842, 26233] avg=16384.0000 max=26233 max_rank=7 min=10090 min_rank=4 imbalance_ratio=1.601135 max_load_reduction=104839 max_load_reduction_pct=0.799858 moved_to_echo=68574 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 170, 145, 4037, 213, 643, 4086, 0, 0, 61, 3367, 0, 4055, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 39, 0, 3944, 4045, 18, 0, 0, 806, 4033, 68, 0, 1361, 1, 3163, 117, 440, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 8, 49, 377, 0, 26, 432, 7, 1, 2, 0, 0, 1, 1, 0, 4096, 3343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1, 923, 1698, 0, 4096, 399, 43, 0, 0, 257, 27, 0, 6, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3837, 39, 0, 2849, 0, 0, 11, 3, 0, 413, 4087, 1, 1, 5, 10, 2461, 822, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2403, 0, 257, 3931, 3446, 0, 1966, 1979, 0, 0, 40, 0, 0, 1265, 2747, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3758, 4095, 2216, 228, 79, 68, 0, 29, 289, 1, 693, 619, 20, 4092, 0, 2093, 440, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3570, 8, 2804, 2695, 4, 0, 0, 0, 3365, 0, 3999, 0, 3847, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16788, 18310, 8377, 11334, 14604, 18054, 19220, 24385] avg=16384.0000 max=24385 max_rank=7 min=8377 min_rank=2 imbalance_ratio=1.488342 max_load_reduction=106687 max_load_reduction_pct=0.813957 moved_to_echo=76263 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 3728, 0, 10, 6, 0, 0, 0, 0, 0, 200, 3609, 0, 3984, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 933, 0, 0, 4064, 3768, 0, 12, 0, 0, 0, 3, 0, 4089, 1, 63, 2050, 472, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1903, 88, 697, 12, 3794, 1439, 0, 1307, 296, 1196, 4, 615, 0, 10, 4096, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4091, 193, 75, 4028, 16, 4084, 2337, 0, 0, 63, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 865, 32, 24, 2885, 0, 529, 0, 3924, 17, 4033, 2865, 2, 0, 0, 4, 3703, 226, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 3538, 4096, 0, 0, 1739, 3082, 2, 33, 0, 171, 4096, 3769, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 3566, 23, 33, 1916, 1186, 0, 647, 0, 0, 1, 2837, 0, 1839, 121, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 4096, 91, 312, 2587, 25, 0, 0, 0, 0, 2444, 0, 60, 0, 1929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11547, 15568, 19479, 15279, 19184, 21612, 16627, 11776] avg=16384.0000 max=21612 max_rank=5 min=11547 min_rank=0 imbalance_ratio=1.319092 max_load_reduction=109460 max_load_reduction_pct=0.835114 moved_to_echo=69199 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 2358, 3453, 4030, 3201, 167, 4046, 236, 0, 0, 0, 962, 0, 3665, 4008, 954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 2172, 0, 11, 1360, 1579, 0, 2, 1, 3887, 0, 12, 66, 2844, 67, 3973, 125, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 3, 1, 0, 1897, 1, 2, 1, 4006, 741, 9, 1, 141, 2403, 0, 3188, 3803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 2029, 0, 0, 4075, 0, 479, 66, 3871, 23, 0, 385, 731, 0, 72, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 147, 5, 1709, 5, 0, 0, 3588, 771, 21, 1, 3995, 0, 123, 0, 520, 3691, 2669, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 370, 1, 294, 3761, 0, 1079, 1968, 963, 3247, 1158, 21, 1, 284, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 679, 467, 3, 2008, 360, 4092, 0, 0, 882, 0, 40, 0, 362, 0, 2082, 1427, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 17, 3919, 1, 1, 31, 449, 0, 3427, 0, 3916, 0, 0, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[27080, 20322, 16220, 11841, 17334, 13165, 12965, 12145] avg=16384.0000 max=27080 max_rank=0 min=11841 min_rank=3 imbalance_ratio=1.652832 max_load_reduction=103992 max_load_reduction_pct=0.793396 moved_to_echo=55609 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 637, 2, 357, 1659, 0, 3652, 0, 0, 3059, 0, 0, 4026, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 5, 4, 7, 31, 2521, 47, 44, 0, 3217, 2936, 3686, 4041, 0, 3958, 441, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 950, 0, 0, 2, 0, 3920, 32, 0, 29, 0, 1967, 0, 25, 7, 3, 4081, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 4061, 1413, 3020, 117, 2, 3687, 4052, 22, 0, 0, 387, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2260, 2001, 2, 438, 4077, 10, 431, 0, 1856, 0, 0, 3118, 42, 79, 1283, 0, 3709, 2666, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 15, 2012, 4036, 2, 4080, 1978, 4071, 0, 14, 0, 10, 159, 1027, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3944, 0, 40, 17, 0, 13, 21, 2997, 1317, 51, 4, 1, 14, 158, 2092, 1428, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4073, 20, 0, 50, 61, 0, 3331, 0, 552, 0, 1, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13399, 21216, 15038, 16861, 22045, 17672, 12605, 12236] avg=16384.0000 max=22045 max_rank=4 min=12236 min_rank=7 imbalance_ratio=1.345520 max_load_reduction=109027 max_load_reduction_pct=0.831810 moved_to_echo=64558 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[384, 10496, 18944, 6656, 8192, 4352, 17408, 12672, 0, 0, 11392, 19712, 0, 16384, 4480, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[256, 26880, 11264, 384, 24192, 6272, 128, 128, 1792, 1792, 25600, 0, 22656, 128, 9984, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[5376, 8576, 256, 9344, 17536, 6016, 128, 384, 896, 24960, 7040, 3840, 13184, 2944, 12416, 15488, 2688, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[2432, 13568, 5248, 2304, 16256, 2176, 2816, 3840, 10624, 128, 4480, 22912, 128, 256, 1664, 896, 27136, 14976, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 7168, 28928, 3584, 15232, 7680, 640, 15872, 1280, 3712, 6656, 768, 2176, 128, 9472, 256, 16512, 8064, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[2560, 2816, 384, 24192, 29184, 128, 7296, 15616, 8320, 3328, 5504, 128, 256, 14848, 7680, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[896, 1024, 3456, 1152, 4992, 2432, 12288, 7168, 512, 9216, 6400, 10624, 3712, 4608, 3072, 128, 31744, 28672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[512, 22528, 6784, 4096, 22912, 5248, 28416, 20480, 5248, 128, 7680, 2944, 896, 0, 384, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[3881, 1149, 3198, 0, 0, 4, 115, 0, 4092, 0, 0, 3623, 0, 2, 0, 168, 5, 0, 5, 4089, 306, 0, 4058, 0, 4075, 0, 3305, 2, 65, 4091, 4090, 0, 401, 3753, 28, 0, 1, 0, 3970, 28, 425, 4094, 167, 0, 0, 0, 3669, 136, 1, 4085, 0, 3063, 3996, 901, 3, 126, 874, 0, 2, 0, 1146, 2, 4009, 1800, 20, 3, 0, 1, 4084, 1, 7, 36, 146, 0, 4093, 0, 0, 0, 62, 3063, 0, 3997, 1462, 780, 3965, 0, 0, 4091, 0, 638, 0, 1200, 0, 8, 111, 6, 0, 1178, 0, 144, 0, 173, 4095, 0, 0, 0, 0, 3, 282, 8, 54, 3992, 0, 0, 889, 0, 0, 0, 3093, 0, 0, 1019, 0, 3695, 4042, 0, 3628, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[237, 2, 3961, 0, 0, 1, 2137, 0, 4085, 0, 0, 1, 0, 20, 0, 3145, 1270, 0, 381, 2409, 4095, 0, 3141, 518, 0, 0, 15, 1, 46, 2582, 4094, 0, 2484, 4083, 12, 0, 0, 0, 4095, 1, 4003, 4092, 1731, 0, 0, 0, 4, 57, 1, 3518, 0, 3984, 4096, 38, 7, 3855, 245, 0, 0, 0, 4096, 4, 92, 966, 0, 0, 72, 9, 2152, 1, 4067, 0, 30, 0, 196, 0, 0, 3929, 7, 449, 0, 0, 1452, 4, 2299, 0, 0, 4, 0, 4, 0, 2192, 22, 0, 1, 0, 0, 3400, 3576, 200, 0, 0, 3909, 0, 4077, 0, 0, 4084, 1, 20, 144, 4041, 87, 4092, 0, 0, 26, 0, 4095, 0, 0, 1, 935, 4096, 3655, 0, 0, 138] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1369, 0, 3656, 0, 32, 159, 55, 1, 505, 0, 0, 3822, 0, 1, 1, 3448, 3914, 0, 0, 4076, 117, 6, 4, 486, 0, 0, 4096, 0, 4089, 3633, 4091, 0, 8, 1653, 13, 0, 0, 0, 4096, 3148, 1, 4081, 3943, 0, 0, 0, 2572, 4042, 3, 3941, 0, 773, 9, 7, 48, 3520, 0, 0, 0, 0, 3190, 2, 3329, 3852, 157, 0, 4046, 0, 0, 23, 0, 11, 1, 0, 1921, 3319, 0, 0, 309, 3898, 52, 0, 402, 531, 4095, 0, 0, 1, 20, 12, 0, 0, 4076, 0, 58, 0, 0, 0, 0, 4096, 0, 11, 3992, 7, 3927, 0, 0, 0, 782, 120, 104, 4078, 0, 4093, 44, 0, 164, 0, 3795, 0, 0, 1048, 0, 19, 2068, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[2283, 3028, 4096, 0, 0, 0, 0, 74, 2356, 0, 6, 3993, 0, 47, 199, 1, 3423, 5, 48, 4094, 0, 6, 2726, 4091, 1, 53, 4046, 833, 0, 3853, 501, 0, 28, 0, 1, 157, 0, 0, 3851, 4039, 3210, 3157, 18, 28, 0, 3065, 4076, 4096, 0, 13, 5, 3630, 0, 0, 3769, 777, 150, 130, 224, 32, 3901, 323, 454, 1, 210, 2069, 215, 3844, 8, 121, 1973, 0, 4039, 3, 0, 0, 0, 2, 0, 0, 1, 0, 10, 0, 4094, 8, 0, 0, 9, 1, 0, 0, 0, 0, 0, 27, 1101, 84, 12, 2933, 166, 0, 4074, 11, 0, 907, 31, 3909, 0, 3, 1104, 4088, 0, 264, 4, 0, 1, 356, 4091, 0, 102, 94, 64, 4069, 3429, 0, 585, 4058] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4076, 0, 4096, 1, 614, 0, 0, 0, 2891, 0, 0, 4096, 0, 6, 180, 3786, 3918, 0, 0, 3346, 0, 0, 0, 4007, 0, 120, 3998, 348, 2236, 4072, 4078, 0, 117, 2157, 1111, 0, 2, 0, 4095, 2661, 2, 0, 1481, 3, 14, 2, 131, 918, 0, 627, 0, 3652, 203, 0, 379, 4096, 0, 0, 0, 0, 4029, 0, 4095, 4093, 0, 2, 229, 1, 1, 0, 2, 0, 1061, 0, 675, 0, 0, 3, 0, 1709, 0, 104, 0, 0, 4032, 0, 0, 4086, 0, 0, 0, 3859, 4092, 0, 0, 62, 532, 2310, 0, 4094, 0, 0, 4096, 4, 3, 0, 0, 7, 884, 6, 330, 4087, 3, 3963, 0, 0, 0, 14, 4096, 0, 3, 1, 1, 3577, 2731, 0, 0, 675] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[3959, 3884, 4094, 0, 35, 0, 3, 11, 4047, 0, 0, 3781, 11, 3436, 14, 47, 335, 0, 110, 3229, 4096, 0, 1, 693, 0, 4055, 4041, 1455, 849, 4059, 0, 0, 4084, 3788, 320, 4065, 38, 744, 1659, 0, 410, 4096, 126, 0, 1, 59, 0, 2178, 0, 8, 3, 0, 4012, 3928, 2, 2643, 7, 0, 37, 0, 4048, 2447, 0, 3953, 131, 188, 2586, 4075, 7, 63, 10, 0, 6, 0, 1, 0, 1, 0, 4, 4093, 34, 0, 0, 3969, 237, 431, 0, 0, 116, 41, 0, 1178, 3978, 0, 0, 0, 0, 1952, 199, 84, 44, 0, 3989, 105, 3902, 95, 0, 0, 9, 336, 5, 1582, 0, 3109, 0, 11, 0, 2, 1083, 1, 70, 36, 0, 3972, 0, 430, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[3698, 3579, 4084, 0, 0, 15, 61, 0, 14, 0, 0, 4077, 114, 0, 0, 5, 0, 0, 1, 3536, 35, 0, 3304, 1497, 0, 27, 401, 992, 201, 0, 4096, 0, 0, 1749, 1514, 109, 0, 0, 2241, 4092, 3971, 10, 4077, 0, 332, 1, 1, 477, 0, 2315, 0, 3960, 4024, 0, 0, 4081, 0, 0, 0, 7, 4095, 23, 0, 4095, 3, 0, 4079, 0, 405, 0, 138, 0, 1986, 0, 0, 0, 6, 39, 0, 7, 6, 50, 0, 0, 4019, 6, 2, 0, 619, 0, 19, 0, 3953, 13, 74, 1065, 0, 32, 0, 4096, 0, 0, 4045, 55, 3936, 0, 0, 3544, 0, 29, 4025, 3761, 4, 3886, 0, 0, 1, 0, 4013, 0, 0, 0, 11, 4064, 3918, 158, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[3878, 6, 3977, 0, 595, 4, 0, 0, 0, 0, 4061, 4044, 0, 5, 0, 414, 7, 0, 74, 3518, 3679, 0, 0, 4096, 0, 2030, 1433, 3, 0, 4072, 85, 0, 0, 2497, 20, 0, 19, 0, 4096, 4009, 0, 3507, 4015, 14, 65, 0, 2, 4096, 0, 2293, 2, 3002, 4068, 0, 0, 4096, 0, 0, 2, 0, 596, 0, 2716, 21, 0, 1, 6, 1200, 1, 41, 4089, 0, 4093, 0, 34, 0, 7, 1067, 0, 0, 0, 0, 57, 4031, 4070, 0, 0, 0, 0, 0, 3, 0, 4070, 0, 0, 71, 15, 5, 0, 2883, 264, 0, 4094, 337, 11, 0, 0, 4095, 0, 1, 1456, 3977, 0, 4028, 0, 0, 0, 874, 4058, 0, 566, 0, 0, 4089, 1936, 0, 21, 404] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2283, 3028, 3626, 0, 0, 0, 0, 74, 2356, 0, 6, 3993, 0, 47, 199, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3423, 5, 48, 0, 0, 6, 2726, 4091, 1, 53, 4046, 833, 0, 1866, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 1, 157, 0, 0, 0, 4039, 3210, 2812, 18, 28, 0, 3065, 4076, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 5, 3630, 0, 0, 3769, 777, 150, 130, 224, 32, 237, 323, 454, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 2069, 215, 3844, 8, 121, 1973, 0, 4039, 3, 0, 0, 0, 2, 0, 0, 4094, 3032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 10, 0, 4094, 8, 0, 0, 9, 1, 0, 0, 0, 0, 0, 27, 3664, 1987, 819, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1101, 84, 12, 2933, 166, 0, 3172, 11, 0, 907, 31, 3909, 0, 3, 1104, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 4, 0, 1, 356, 4091, 0, 102, 94, 64, 4069, 3429, 0, 585, 4058, 470, 406, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15613, 17599, 21530, 9745, 19610, 11116, 17521, 18338] avg=16384.0000 max=21530 max_rank=2 min=9745 min_rank=3 imbalance_ratio=1.314087 max_load_reduction=109542 max_load_reduction_pct=0.835739 moved_to_echo=66585 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1369, 0, 3236, 0, 32, 159, 55, 1, 505, 0, 0, 3822, 0, 1, 1, 3448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 0, 0, 0, 117, 6, 4, 486, 0, 0, 4096, 0, 4089, 1759, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1653, 13, 0, 0, 0, 0, 3148, 1, 3635, 3943, 0, 0, 0, 2572, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3941, 0, 773, 9, 7, 48, 3520, 0, 0, 0, 0, 194, 2, 3329, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 4046, 0, 0, 23, 0, 11, 1, 0, 1921, 3319, 0, 0, 309, 3898, 4076, 3226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 402, 531, 4095, 0, 0, 1, 20, 12, 0, 0, 4076, 0, 58, 0, 2996, 1874, 870, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 3108, 7, 3927, 0, 0, 0, 782, 120, 104, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 44, 0, 164, 0, 3795, 0, 0, 1048, 0, 19, 2068, 0, 0, 0, 420, 398, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12629, 18562, 19015, 15678, 20987, 15473, 16233, 12495] avg=16384.0000 max=20987 max_rank=4 min=12495 min_rank=7 imbalance_ratio=1.280945 max_load_reduction=110085 max_load_reduction_pct=0.839882 moved_to_echo=65188 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3881, 1149, 2828, 0, 0, 4, 115, 0, 4092, 0, 0, 3623, 0, 2, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5, 0, 306, 0, 4058, 0, 4075, 0, 3305, 2, 65, 1977, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 3753, 28, 0, 1, 0, 0, 28, 425, 3645, 167, 0, 0, 0, 3669, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 0, 3063, 3996, 901, 3, 126, 874, 0, 2, 0, 65, 2, 4009, 1800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 3, 0, 1, 4084, 1, 7, 36, 146, 0, 4093, 0, 0, 0, 62, 3063, 4089, 3126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 1462, 780, 3965, 0, 0, 4091, 0, 638, 0, 1200, 0, 8, 111, 6, 1081, 2114, 844, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1178, 0, 144, 0, 173, 3178, 0, 0, 0, 0, 3, 282, 8, 54, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 889, 0, 0, 0, 3093, 0, 0, 1019, 0, 3695, 4042, 0, 3628, 0, 370, 413, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15862, 17888, 12253, 18927, 18731, 20801, 9012, 17598] avg=16384.0000 max=20801 max_rank=5 min=9012 min_rank=6 imbalance_ratio=1.269592 max_load_reduction=110271 max_load_reduction_pct=0.841301 moved_to_echo=66142 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[237, 2, 3506, 0, 0, 1, 2137, 0, 4085, 0, 0, 1, 0, 20, 0, 3145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1270, 0, 381, 0, 4095, 0, 3141, 518, 0, 0, 15, 1, 46, 1251, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2484, 4083, 12, 0, 0, 0, 0, 1, 4003, 3645, 1731, 0, 0, 0, 4, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3518, 0, 3984, 4096, 38, 7, 3855, 245, 0, 0, 0, 249, 4, 92, 966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 9, 2152, 1, 4067, 0, 30, 0, 196, 0, 0, 3929, 7, 449, 2409, 3224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1452, 4, 2299, 0, 0, 4, 0, 4, 0, 2192, 22, 0, 1, 0, 3847, 1331, 871, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3400, 3576, 200, 0, 0, 3043, 0, 4077, 0, 0, 4084, 1, 20, 144, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 4092, 0, 0, 26, 0, 4095, 0, 0, 1, 935, 4096, 3655, 0, 0, 138, 455, 390, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13134, 14812, 16020, 17055, 16545, 12503, 22586, 18417] avg=16384.0000 max=22586 max_rank=6 min=12503 min_rank=5 imbalance_ratio=1.378540 max_load_reduction=108486 max_load_reduction_pct=0.827682 moved_to_echo=70051 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4076, 0, 3626, 1, 614, 0, 0, 0, 2891, 0, 0, 4096, 0, 6, 180, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 4007, 0, 120, 3998, 348, 2236, 1972, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 2157, 1111, 0, 2, 0, 0, 2661, 2, 0, 1481, 3, 14, 2, 131, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 627, 0, 3652, 203, 0, 379, 4096, 0, 0, 0, 0, 245, 0, 4095, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 229, 1, 1, 0, 2, 0, 1061, 0, 675, 0, 0, 3, 0, 1709, 3346, 3224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 4032, 0, 0, 4086, 0, 0, 0, 3859, 4092, 0, 0, 62, 3784, 2100, 871, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 2310, 0, 4094, 0, 0, 3188, 4, 3, 0, 0, 7, 884, 6, 330, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3963, 0, 0, 0, 14, 4096, 0, 3, 1, 1, 3577, 2731, 0, 0, 675, 470, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19276, 20677, 8599, 17390, 10253, 23489, 15445, 15943] avg=16384.0000 max=23489 max_rank=5 min=8599 min_rank=2 imbalance_ratio=1.433655 max_load_reduction=107583 max_load_reduction_pct=0.820793 moved_to_echo=65130 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3878, 6, 3521, 0, 595, 4, 0, 0, 0, 0, 4061, 4044, 0, 5, 0, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 74, 0, 3679, 0, 0, 4096, 0, 2030, 1433, 3, 0, 1972, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2497, 20, 0, 19, 0, 0, 4009, 0, 3124, 4015, 14, 65, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2293, 2, 3002, 4068, 0, 0, 4096, 0, 0, 2, 0, 37, 0, 2716, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 1200, 1, 41, 4089, 0, 4093, 0, 34, 0, 7, 1067, 0, 0, 3518, 3225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4031, 4070, 0, 0, 0, 0, 0, 3, 0, 4070, 0, 0, 71, 559, 2100, 871, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 5, 0, 2883, 264, 0, 3187, 337, 11, 0, 0, 4095, 0, 1, 1456, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 874, 4058, 0, 566, 0, 0, 4089, 1936, 0, 21, 404, 456, 408, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16528, 13379, 17861, 16237, 17282, 16331, 16231, 17223] avg=16384.0000 max=17861 max_rank=2 min=13379 min_rank=1 imbalance_ratio=1.090149 max_load_reduction=113211 max_load_reduction_pct=0.863731 moved_to_echo=67067 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3959, 3884, 3624, 0, 35, 0, 3, 11, 4047, 0, 0, 3781, 11, 3436, 14, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 110, 0, 4096, 0, 1, 693, 0, 4055, 4041, 1455, 849, 1966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 3788, 320, 4065, 38, 744, 0, 0, 410, 3649, 126, 0, 1, 59, 0, 2178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 4012, 3928, 2, 2643, 7, 0, 37, 0, 246, 2447, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 188, 2586, 4075, 7, 63, 10, 0, 6, 0, 1, 0, 1, 0, 4, 4093, 3229, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 3969, 237, 431, 0, 0, 116, 41, 0, 1178, 3978, 0, 0, 0, 3802, 2093, 352, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1952, 199, 84, 44, 0, 3105, 105, 3902, 95, 0, 0, 9, 336, 5, 1582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3109, 0, 11, 0, 2, 1083, 1, 70, 36, 0, 3972, 0, 430, 0, 6, 470, 398, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22852, 17601, 19462, 17286, 15701, 16717, 11418, 10035] avg=16384.0000 max=22852 max_rank=0 min=10035 min_rank=7 imbalance_ratio=1.394775 max_load_reduction=108220 max_load_reduction_pct=0.825653 moved_to_echo=53871 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3698, 3579, 3615, 0, 0, 15, 61, 0, 14, 0, 0, 4077, 114, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 35, 0, 3304, 1497, 0, 27, 401, 992, 201, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1749, 1514, 109, 0, 0, 0, 4092, 3971, 9, 4077, 0, 332, 1, 1, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2315, 0, 3960, 4024, 0, 0, 4081, 0, 0, 0, 7, 249, 23, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4079, 0, 405, 0, 138, 0, 1986, 0, 0, 0, 6, 39, 0, 7, 3536, 1764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 50, 0, 0, 4019, 6, 2, 0, 619, 0, 19, 0, 3953, 13, 74, 1065, 3846, 0, 477, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 4096, 0, 0, 3148, 55, 3936, 0, 0, 3544, 0, 29, 4025, 3761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3886, 0, 0, 1, 0, 4013, 0, 0, 0, 11, 4064, 3918, 158, 0, 4094, 469, 404, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15178, 10554, 16332, 18754, 11963, 14642, 22626, 21023] avg=16384.0000 max=22626 max_rank=6 min=10554 min_rank=1 imbalance_ratio=1.380981 max_load_reduction=108446 max_load_reduction_pct=0.827377 moved_to_echo=70254 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[23424, 11648, 27648, 128, 1280, 256, 2432, 128, 18048, 0, 4096, 27520, 128, 3584, 512, 11136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 16896, 128, 22144, 20480, 4992, 4224, 23296, 1280, 256, 384, 128, 1536, 2816, 14720, 18816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1664, 9088, 3840, 18560, 512, 256, 25216, 640, 15872, 1024, 128, 15744, 2048, 640, 7296, 29696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[640, 2304, 11264, 9216, 6784, 256, 10368, 128, 11392, 128, 7040, 3328, 128, 5120, 384, 13312, 28416, 22144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[7168, 19712, 3072, 4352, 128, 768, 0, 18048, 12032, 20608, 15616, 128, 512, 3200, 10496, 16000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 23552, 1024, 128, 256, 1280, 28416, 128, 768, 2304, 1024, 27648, 21888, 640, 4352, 9472, 3584, 3328, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12928, 128, 640, 0, 12416, 128, 13312, 15488, 4096, 6400, 21376, 3712, 7552, 12800, 21120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 4224, 3456, 9344, 26880, 512, 128, 8192, 768, 768, 128, 8448, 20224, 128, 256, 1280, 23680, 13696, 6016, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 2, 27, 6, 0, 118, 0, 4086, 0, 4065, 0, 0, 3068, 0, 0, 220, 213, 3, 0, 1957, 0, 4016, 4040, 455, 0, 4094, 4063, 346, 75, 0, 3965, 0, 4072, 2, 0, 1, 467, 728, 127, 3716, 0, 0, 3786, 4057, 22, 4089, 0, 3, 3, 2515, 0, 1695, 0, 3980, 0, 0, 0, 0, 4096, 832, 17, 0, 4070, 0, 0, 0, 0, 0, 48, 0, 0, 3544, 0, 258, 9, 110, 4091, 6, 4096, 3886, 4096, 0, 3163, 1238, 3976, 212, 0, 0, 23, 208, 0, 0, 564, 0, 316, 4039, 0, 8, 57, 21, 0, 1, 3, 0, 3977, 4094, 2, 3597, 0, 642, 5, 1, 0, 18, 3919, 2, 0, 6, 4092, 26, 0, 0, 0, 3620, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[25, 1, 0, 0, 1, 440, 0, 0, 0, 122, 0, 0, 0, 0, 3, 0, 0, 3, 0, 3761, 9, 3170, 4047, 4046, 17, 4025, 3862, 0, 1155, 33, 4094, 0, 0, 4067, 2157, 2105, 3556, 0, 53, 2429, 86, 0, 3275, 4087, 4031, 0, 0, 0, 0, 0, 0, 709, 0, 1690, 2860, 341, 87, 54, 349, 0, 16, 4096, 4083, 2057, 3, 0, 0, 0, 0, 5, 0, 3547, 1, 2489, 3798, 0, 3638, 1113, 4048, 3509, 2885, 0, 4094, 217, 8, 217, 2, 45, 0, 4079, 0, 104, 0, 0, 103, 4089, 2, 675, 0, 421, 0, 26, 0, 56, 4058, 4096, 0, 4048, 133, 134, 0, 0, 0, 30, 4063, 0, 0, 0, 61, 0, 0, 3, 794, 3470, 4, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 8, 162, 6, 20, 2728, 0, 4095, 12, 4069, 0, 0, 4, 0, 1, 344, 1, 1174, 0, 0, 0, 0, 205, 4096, 0, 4094, 3688, 22, 4096, 52, 0, 0, 392, 0, 0, 92, 0, 26, 401, 4065, 3356, 2989, 357, 4076, 3335, 2, 0, 0, 0, 4, 0, 4022, 0, 0, 2, 0, 0, 0, 4073, 0, 2902, 0, 4095, 0, 0, 0, 0, 530, 1, 0, 229, 1742, 0, 0, 4092, 4088, 3840, 0, 4, 0, 2987, 0, 4091, 0, 3742, 957, 374, 0, 3915, 0, 2, 1, 3, 4041, 3623, 1, 7, 0, 14, 0, 4091, 15, 0, 24, 1076, 2125, 13, 103, 9, 4077, 4094, 0, 448, 0, 4031, 0, 0, 0, 4088, 0, 5, 0, 0, 4071, 595, 0, 857, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4094, 12, 13, 17, 0, 4093, 11, 4093, 2, 0, 942, 0, 7, 0, 18, 0, 0, 1, 0, 2058, 87, 4085, 0, 4048, 3176, 0, 4096, 7, 155, 0, 4094, 0, 3850, 19, 1667, 0, 0, 4085, 248, 0, 3227, 0, 0, 4094, 0, 0, 0, 2687, 0, 3931, 1, 2817, 280, 0, 28, 0, 59, 0, 0, 0, 4096, 78, 0, 722, 454, 3, 2607, 0, 0, 12, 0, 4, 4087, 9, 4095, 0, 11, 0, 2849, 0, 1377, 0, 4080, 996, 4, 0, 3, 2021, 4057, 0, 4, 3302, 0, 322, 3385, 0, 1, 0, 63, 0, 0, 5, 2099, 4096, 0, 2884, 9, 1426, 5, 0, 105, 0, 4096, 4087, 142, 0, 4094, 1450, 1731, 0, 11, 4087, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 3877, 71, 57, 0, 0, 4096, 3, 4095, 0, 0, 7, 0, 4, 64, 2796, 0, 0, 8, 0, 2690, 291, 2, 0, 4, 4096, 58, 9, 2, 4027, 0, 31, 0, 685, 701, 0, 161, 1395, 0, 9, 64, 2046, 3643, 415, 4059, 0, 3746, 0, 31, 0, 1, 61, 4083, 730, 65, 44, 0, 3929, 0, 0, 0, 4096, 1298, 117, 162, 2822, 4, 3574, 0, 648, 3889, 3, 0, 4055, 251, 4091, 0, 4055, 2155, 3803, 0, 1617, 1619, 3863, 7, 1, 2188, 4086, 0, 66, 0, 4, 101, 168, 3509, 401, 0, 11, 0, 3, 23, 0, 1377, 3480, 4096, 39, 569, 32, 39, 3946, 0, 0, 0, 4096, 4028, 0, 0, 0, 0, 3645, 0, 215, 185, 265, 0, 214, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 952, 0, 0, 1162, 23, 4089, 0, 2974, 0, 308, 3, 0, 767, 0, 4007, 0, 10, 0, 266, 95, 4096, 4092, 5, 3831, 3976, 2691, 3734, 0, 0, 6, 314, 0, 0, 14, 4080, 0, 7, 3662, 0, 78, 93, 660, 2048, 3348, 0, 81, 0, 0, 0, 0, 850, 3149, 3601, 64, 4094, 0, 1, 23, 3297, 0, 3600, 9, 48, 570, 0, 54, 4040, 8, 108, 4010, 0, 727, 4075, 0, 4040, 1816, 158, 0, 1026, 157, 4059, 1, 0, 1, 2, 25, 6, 0, 1298, 676, 495, 4065, 0, 4082, 1518, 4, 0, 0, 1376, 194, 0, 12, 3810, 4094, 137, 4, 3961, 2707, 272, 13, 42, 1, 9, 0, 0, 14, 0, 0, 3825, 11, 4, 2847, 440, 70, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[52, 0, 232, 8, 0, 475, 6, 4069, 80, 527, 1, 2, 1820, 7, 202, 2696, 0, 3484, 0, 25, 0, 4095, 3574, 148, 1, 3212, 3852, 1, 0, 0, 4095, 0, 3066, 3742, 436, 0, 9, 117, 2202, 4096, 0, 244, 3448, 3301, 4096, 3383, 0, 0, 0, 31, 0, 0, 30, 356, 0, 45, 4, 0, 1307, 14, 3921, 2, 3980, 0, 0, 0, 0, 0, 6, 0, 21, 4096, 14, 0, 4096, 499, 1478, 424, 19, 22, 4079, 13, 1654, 63, 2961, 0, 0, 3, 119, 4068, 0, 0, 350, 3487, 28, 3889, 103, 0, 355, 0, 0, 3, 0, 780, 2679, 4051, 1409, 2385, 492, 1544, 0, 5, 0, 131, 3881, 3052, 0, 161, 4042, 149, 2, 2, 0, 1825, 2134, 0, 2, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1, 33, 0, 1, 0, 3845, 0, 3218, 67, 2966, 0, 0, 194, 0, 0, 3207, 0, 1223, 0, 0, 0, 4096, 3779, 3204, 0, 3330, 3127, 0, 259, 0, 4096, 21, 0, 76, 55, 1, 3704, 2, 0, 3205, 4, 0, 3141, 4096, 2370, 2348, 1, 12, 1, 0, 0, 3958, 0, 1391, 3120, 0, 113, 4092, 1689, 0, 151, 1944, 0, 418, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 3252, 0, 0, 3966, 4, 2317, 1653, 788, 79, 4044, 0, 1111, 3526, 0, 0, 33, 4088, 0, 0, 306, 3, 523, 4095, 193, 77, 174, 16, 0, 3939, 72, 394, 2, 3893, 0, 3954, 0, 146, 7, 1, 0, 1479, 57, 4095, 0, 0, 3924, 15, 70, 0, 0, 2172, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[25, 1, 0, 0, 1, 440, 0, 0, 0, 122, 0, 0, 0, 0, 3, 0, 3862, 1736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3761, 9, 3170, 4047, 4046, 17, 2197, 0, 0, 1155, 33, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 2157, 2105, 3556, 0, 53, 0, 86, 0, 3275, 2472, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 709, 0, 1690, 2860, 341, 87, 54, 349, 0, 16, 4096, 4083, 2057, 628, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 5, 0, 3547, 1, 2489, 3798, 0, 3010, 1113, 4048, 3509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2885, 0, 426, 217, 8, 217, 2, 45, 0, 4079, 0, 104, 0, 0, 103, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 675, 0, 421, 0, 26, 0, 56, 4058, 4096, 0, 4048, 133, 134, 0, 0, 1654, 965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4063, 0, 0, 0, 61, 0, 0, 3, 794, 3470, 4, 0, 0, 2, 3668, 693, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6190, 22532, 21802, 17620, 21523, 12175, 16268, 12962] avg=16384.0000 max=22532 max_rank=1 min=6190 min_rank=0 imbalance_ratio=1.375244 max_load_reduction=108540 max_load_reduction_pct=0.828094 moved_to_echo=62928 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 27, 6, 0, 118, 0, 4086, 0, 4065, 0, 0, 3068, 0, 0, 220, 4063, 2657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 3, 0, 1957, 0, 4016, 4040, 455, 0, 2234, 0, 346, 75, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 2, 0, 1, 467, 728, 127, 0, 0, 0, 3786, 2454, 22, 4089, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2515, 0, 1695, 0, 3980, 0, 0, 0, 0, 4096, 832, 17, 0, 4070, 0, 706, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 3544, 0, 258, 9, 110, 3385, 6, 4096, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 329, 1238, 3976, 212, 0, 0, 23, 208, 0, 0, 564, 0, 316, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 57, 21, 0, 1, 3, 0, 3977, 4094, 2, 3597, 0, 642, 5, 1, 1683, 958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3919, 2, 0, 6, 4092, 26, 0, 0, 0, 3620, 0, 0, 0, 1, 2834, 1059, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18312, 17304, 15751, 18559, 15342, 15001, 15049, 15754] avg=16384.0000 max=18559 max_rank=3 min=15001 min_rank=5 imbalance_ratio=1.132751 max_load_reduction=112513 max_load_reduction_pct=0.858406 moved_to_echo=61146 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 162, 6, 20, 2728, 0, 4095, 12, 4069, 0, 0, 4, 0, 1, 344, 3688, 2906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1174, 0, 0, 0, 0, 205, 4096, 0, 2234, 0, 22, 4096, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 0, 92, 0, 26, 401, 0, 3356, 2989, 357, 2466, 3335, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4022, 0, 0, 2, 0, 0, 0, 4073, 0, 2902, 0, 4095, 0, 662, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 1, 0, 229, 1742, 0, 0, 4092, 4088, 3178, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2987, 0, 426, 0, 3742, 957, 374, 0, 3915, 0, 2, 1, 3, 4041, 3623, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 14, 0, 4091, 15, 0, 24, 1076, 2125, 13, 103, 9, 4077, 4094, 0, 1683, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 0, 4031, 0, 0, 0, 4088, 0, 5, 0, 0, 4071, 595, 0, 857, 0, 3665, 1159, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18043, 11880, 13416, 16408, 13864, 20072, 18293, 19096] avg=16384.0000 max=20072 max_rank=5 min=11880 min_rank=1 imbalance_ratio=1.225098 max_load_reduction=111000 max_load_reduction_pct=0.846863 moved_to_echo=71325 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 12, 13, 17, 0, 4093, 11, 4093, 2, 0, 942, 0, 7, 0, 3176, 2920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 1, 0, 2058, 87, 4085, 0, 2200, 0, 0, 4096, 7, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3850, 19, 1667, 0, 0, 0, 248, 0, 3227, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2687, 0, 3931, 1, 2817, 280, 0, 28, 0, 59, 0, 0, 0, 4096, 78, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 722, 454, 3, 2607, 0, 0, 12, 0, 4, 4087, 9, 3385, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2849, 0, 140, 0, 4080, 996, 4, 0, 3, 2021, 4057, 0, 4, 3302, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3385, 0, 1, 0, 63, 0, 0, 5, 2099, 4096, 0, 2884, 9, 1426, 5, 0, 1667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 4096, 4087, 142, 0, 4094, 1450, 1731, 0, 11, 4087, 0, 0, 1, 0, 1237, 1165, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19380, 12707, 17199, 14687, 11294, 17778, 15640, 22387] avg=16384.0000 max=22387 max_rank=7 min=11294 min_rank=4 imbalance_ratio=1.366394 max_load_reduction=108685 max_load_reduction_pct=0.829201 moved_to_echo=67099 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3877, 71, 57, 0, 0, 4096, 3, 4095, 0, 0, 7, 0, 4, 64, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2796, 0, 0, 8, 0, 2690, 291, 2, 0, 3, 0, 58, 9, 2, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 685, 701, 0, 161, 1395, 0, 9, 64, 2046, 2204, 415, 4059, 0, 3746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 1, 61, 4083, 730, 65, 44, 0, 3929, 0, 0, 0, 4096, 1298, 706, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 162, 2822, 4, 3574, 0, 648, 3889, 3, 0, 4055, 251, 3385, 0, 4055, 2155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3803, 0, 169, 1619, 3863, 7, 1, 2188, 4086, 0, 66, 0, 4, 101, 168, 3509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 0, 11, 0, 3, 23, 0, 1377, 3480, 4096, 39, 569, 32, 39, 3946, 0, 1, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4028, 0, 0, 0, 0, 3645, 0, 215, 185, 265, 0, 214, 0, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16370, 9886, 15516, 15623, 25120, 19584, 14877, 14096] avg=16384.0000 max=25120 max_rank=4 min=9886 min_rank=1 imbalance_ratio=1.533203 max_load_reduction=105952 max_load_reduction_pct=0.808350 moved_to_echo=73677 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 33, 0, 1, 0, 3845, 0, 3218, 67, 2966, 0, 0, 194, 0, 0, 3207, 3127, 2290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1223, 0, 0, 0, 4096, 3779, 3204, 0, 1817, 0, 0, 259, 0, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 55, 1, 3704, 2, 0, 0, 4, 0, 3141, 2478, 2370, 2348, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3958, 0, 1391, 3120, 0, 113, 4092, 1689, 0, 151, 1944, 0, 418, 684, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 3252, 0, 0, 3282, 4, 2317, 1653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 788, 79, 421, 0, 1111, 3526, 0, 0, 33, 4088, 0, 0, 306, 3, 523, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 77, 174, 16, 0, 3939, 72, 394, 2, 3893, 0, 3954, 0, 146, 7, 1, 1369, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1479, 57, 4095, 0, 0, 3924, 15, 70, 0, 0, 2172, 0, 0, 0, 1, 3623, 915, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18949, 18495, 14192, 18212, 14552, 14973, 15204, 16495] avg=16384.0000 max=18949 max_rank=0 min=14192 min_rank=2 imbalance_ratio=1.156555 max_load_reduction=112123 max_load_reduction_pct=0.855431 moved_to_echo=61224 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[52, 0, 232, 8, 0, 475, 6, 4069, 80, 527, 1, 2, 1820, 7, 202, 2696, 3852, 2927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3484, 0, 25, 0, 4095, 3574, 148, 1, 1753, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3066, 3742, 436, 0, 9, 117, 2202, 0, 0, 244, 3448, 1997, 4096, 3383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 30, 356, 0, 45, 4, 0, 1307, 14, 3921, 2, 3980, 0, 255, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 21, 4096, 14, 0, 4096, 499, 1223, 424, 19, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 13, 172, 63, 2961, 0, 0, 3, 119, 4068, 0, 0, 350, 3487, 28, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 355, 0, 0, 3, 0, 780, 2679, 4051, 1409, 2385, 492, 1544, 0, 5, 1320, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 3881, 3052, 0, 161, 4042, 149, 2, 2, 0, 1825, 2134, 0, 2, 2, 1482, 1169, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16956, 17176, 22740, 10470, 10420, 19232, 15905, 18173] avg=16384.0000 max=22740 max_rank=2 min=10420 min_rank=4 imbalance_ratio=1.387939 max_load_reduction=108332 max_load_reduction_pct=0.826508 moved_to_echo=63730 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 952, 0, 0, 1162, 23, 4089, 0, 2974, 0, 308, 3, 0, 767, 0, 3976, 2618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 10, 0, 266, 95, 4096, 4092, 5, 2090, 0, 2691, 3734, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 14, 4080, 0, 7, 0, 0, 78, 93, 393, 2048, 3348, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 850, 3149, 3601, 64, 4094, 0, 1, 23, 3297, 0, 3600, 9, 697, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 570, 0, 54, 4040, 8, 108, 4010, 0, 727, 4075, 0, 3343, 1816, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 157, 423, 1, 0, 1, 2, 25, 6, 0, 1298, 676, 495, 4065, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1518, 4, 0, 0, 1376, 194, 0, 12, 3810, 4094, 137, 4, 3961, 2707, 272, 13, 1575, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 1, 9, 0, 0, 14, 0, 0, 3825, 11, 4, 2847, 440, 70, 0, 0, 3636, 1044, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16872, 21092, 10456, 19493, 18957, 12257, 19836, 12109] avg=16384.0000 max=21092 max_rank=1 min=10456 min_rank=2 imbalance_ratio=1.287354 max_load_reduction=109980 max_load_reduction_pct=0.839081 moved_to_echo=63159 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 9344, 128, 128, 8832, 128, 27776, 256, 22912, 128, 384, 6144, 128, 1024, 6656, 29952, 18176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[7040, 5888, 128, 5760, 384, 20224, 20224, 20224, 128, 14592, 0, 3200, 13440, 128, 20480, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[22528, 256, 2560, 3200, 19840, 6016, 384, 2304, 8192, 14464, 5504, 896, 1792, 15104, 4864, 24064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 1536, 3328, 640, 10368, 128, 1024, 24960, 128, 6784, 24320, 4992, 24192, 3456, 14720, 11264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12032, 7936, 7296, 2944, 13568, 1152, 4224, 0, 3712, 3456, 19456, 14464, 16384, 21376, 128, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 5376, 0, 14336, 1024, 17536, 10624, 640, 4480, 4224, 15616, 896, 10368, 6144, 28032, 3968, 5120, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[640, 1664, 24192, 15360, 256, 256, 20352, 1664, 9344, 128, 1024, 22400, 3456, 128, 1152, 128, 21632, 7296, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[5632, 768, 640, 512, 5632, 4224, 128, 2688, 21248, 30592, 1664, 17664, 4736, 10752, 8448, 128, 11008, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[3887, 0, 2, 1113, 42, 0, 1760, 0, 35, 186, 744, 0, 0, 3682, 1, 208, 3947, 11, 0, 0, 711, 74, 161, 1, 0, 0, 3647, 744, 45, 0, 144, 3995, 46, 0, 0, 276, 3432, 14, 49, 0, 4096, 333, 4096, 152, 0, 22, 0, 0, 99, 40, 0, 4027, 10, 0, 3900, 3222, 135, 0, 4096, 3745, 0, 734, 2362, 0, 90, 181, 4036, 0, 0, 0, 4096, 5, 40, 0, 0, 4064, 8, 0, 19, 0, 4, 984, 0, 744, 4054, 4094, 0, 4002, 4080, 0, 37, 2166, 0, 0, 0, 1, 1, 4095, 0, 3243, 0, 0, 4057, 0, 1, 0, 0, 0, 2277, 1, 3, 0, 3927, 2892, 358, 35, 3944, 482, 19, 0, 0, 6, 4053, 1459, 4055, 3351, 0, 82] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1, 1030, 3304, 1304, 1788, 182, 353, 0, 3698, 1, 0, 0, 485, 4096, 79, 3698, 975, 122, 0, 0, 0, 25, 0, 0, 0, 282, 4096, 4095, 0, 752, 0, 0, 405, 0, 1, 0, 4068, 2855, 0, 0, 4095, 811, 1, 0, 0, 0, 0, 0, 0, 4052, 0, 3622, 0, 85, 4096, 489, 3686, 4087, 3895, 3865, 322, 0, 0, 0, 3763, 0, 3020, 12, 0, 0, 0, 1, 270, 46, 0, 3292, 2, 0, 0, 746, 1189, 4, 2525, 0, 4096, 3968, 0, 381, 9, 0, 7, 1157, 4095, 0, 0, 0, 3, 4019, 214, 0, 0, 15, 4024, 0, 2271, 0, 6, 0, 1325, 0, 80, 4084, 42, 2, 0, 53, 4096, 10, 4093, 1, 0, 0, 6, 20, 3711, 3612, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 25, 0, 1503, 677, 19, 0, 2, 2538, 0, 0, 5, 4, 4063, 1148, 4029, 4095, 0, 0, 22, 2, 0, 0, 0, 0, 0, 4071, 3885, 0, 0, 0, 2695, 0, 1, 795, 0, 954, 3245, 0, 2115, 3989, 0, 54, 124, 0, 0, 0, 0, 0, 4081, 0, 4096, 5, 0, 3142, 22, 4092, 4096, 3347, 4085, 220, 0, 0, 0, 4096, 0, 4078, 0, 0, 0, 133, 0, 0, 0, 16, 3818, 0, 0, 9, 0, 167, 0, 4095, 0, 0, 4096, 3, 1, 33, 618, 0, 3845, 75, 0, 0, 5, 356, 4004, 29, 1023, 0, 3357, 919, 0, 1, 6, 1425, 0, 4049, 0, 0, 0, 0, 3979, 3, 0, 3983, 1, 4095, 2, 0, 0, 0, 4096, 3326, 4081, 2, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1947, 0, 0, 4095, 0, 111, 371, 0, 4066, 1, 0, 4086, 0, 132, 59, 4068, 1557, 0, 0, 0, 0, 0, 0, 90, 1, 6, 4096, 2313, 0, 0, 0, 3989, 0, 0, 0, 0, 7, 3688, 0, 0, 4096, 179, 1, 0, 0, 125, 61, 0, 0, 0, 0, 4087, 9, 8, 12, 1, 0, 4074, 3823, 1105, 0, 0, 0, 105, 4094, 0, 421, 2285, 3667, 0, 0, 4095, 0, 0, 0, 3679, 5, 0, 4016, 23, 3692, 4, 0, 0, 2236, 257, 3793, 237, 37, 817, 0, 4093, 1, 0, 0, 1, 7, 4096, 3327, 4077, 0, 3785, 37, 0, 1, 0, 0, 0, 2, 0, 0, 19, 3303, 4096, 4088, 291, 41, 0, 16, 17, 0, 33, 671, 663, 4029, 4096, 460, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[103, 0, 4096, 0, 10, 564, 0, 64, 2240, 0, 274, 0, 0, 4068, 0, 4033, 4050, 0, 0, 106, 60, 0, 0, 0, 0, 18, 3555, 3478, 295, 0, 4, 4013, 0, 1, 32, 0, 3930, 1746, 0, 2868, 3935, 99, 1253, 15, 0, 1867, 0, 0, 137, 1, 1, 3873, 639, 0, 3434, 163, 3905, 0, 3235, 918, 167, 5, 0, 3, 4061, 0, 1, 5, 0, 0, 949, 4, 34, 39, 12, 423, 0, 32, 0, 0, 12, 0, 4091, 0, 3831, 4091, 40, 0, 4062, 1, 171, 0, 4079, 0, 4, 0, 1156, 4071, 0, 3730, 128, 3966, 2608, 654, 7, 605, 2, 0, 4043, 336, 0, 211, 79, 0, 4087, 29, 4029, 0, 3854, 0, 160, 42, 0, 4095, 0, 3842, 109, 29] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 19, 0, 143, 4068, 0, 2, 2558, 2, 3428, 0, 4090, 4094, 0, 185, 0, 2, 0, 0, 63, 0, 0, 3721, 4072, 2939, 4063, 0, 4007, 0, 0, 38, 0, 725, 11, 73, 5, 4092, 0, 0, 3, 0, 2, 0, 8, 332, 322, 4076, 4096, 0, 0, 0, 3, 4090, 3168, 3999, 4095, 1, 0, 0, 0, 4096, 0, 4069, 0, 0, 0, 1955, 3708, 0, 0, 21, 4094, 0, 0, 2137, 0, 217, 0, 2494, 0, 1, 4096, 466, 0, 0, 63, 0, 0, 824, 0, 599, 21, 3634, 4088, 16, 3503, 27, 3469, 30, 0, 0, 4096, 12, 1638, 3596, 0, 0, 0, 0, 16, 1664, 0, 3707, 1, 1543, 10, 0, 23, 0, 34, 487, 4019, 2, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[8, 10, 3829, 4092, 0, 87, 473, 0, 0, 281, 0, 197, 0, 4096, 37, 3712, 2216, 3953, 0, 9, 0, 0, 0, 50, 0, 4056, 4096, 4096, 0, 0, 0, 0, 11, 0, 0, 3786, 4041, 108, 0, 3434, 408, 214, 2, 0, 0, 7, 2591, 0, 0, 0, 0, 3985, 1, 1013, 827, 1, 3468, 0, 3998, 769, 0, 0, 0, 0, 2750, 0, 4042, 0, 3935, 0, 23, 1, 0, 0, 0, 4071, 4, 0, 0, 400, 4096, 585, 0, 0, 722, 3473, 350, 317, 85, 3089, 0, 3796, 4032, 0, 0, 0, 864, 4096, 0, 2, 0, 3924, 3769, 1, 1, 0, 14, 0, 697, 0, 0, 22, 1197, 3940, 287, 149, 335, 0, 3795, 1, 0, 0, 0, 6, 4096, 4096, 0, 47] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 3696, 0, 0, 0, 0, 4096, 0, 2, 14, 32, 1909, 1, 3984, 4096, 6, 3317, 0, 0, 0, 16, 2937, 0, 0, 4085, 35, 21, 0, 0, 1228, 0, 0, 7, 1, 2217, 4035, 118, 3821, 4092, 0, 2070, 0, 0, 2, 0, 212, 0, 43, 0, 3777, 191, 92, 77, 2066, 0, 4095, 4069, 3687, 242, 0, 0, 0, 4096, 3, 408, 0, 0, 0, 87, 4067, 0, 10, 92, 4093, 15, 0, 0, 190, 270, 53, 2964, 0, 3646, 4096, 0, 573, 0, 0, 13, 3868, 1249, 0, 15, 0, 4, 1118, 35, 0, 0, 3893, 3609, 5, 0, 1402, 0, 4003, 4035, 0, 11, 90, 4092, 1672, 0, 0, 4061, 0, 101, 25, 0, 6, 1, 3967, 9, 4069, 489, 178] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 1030, 3304, 1304, 1788, 182, 353, 0, 3698, 1, 0, 0, 485, 4096, 79, 3698, 493, 180, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 975, 122, 0, 0, 0, 25, 0, 0, 0, 282, 4096, 4095, 0, 752, 0, 0, 547, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 0, 1, 0, 4068, 2855, 0, 0, 4095, 811, 1, 0, 0, 0, 0, 0, 3622, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 85, 4096, 489, 3686, 4087, 3715, 3865, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3763, 0, 3020, 12, 0, 0, 0, 1, 270, 46, 0, 3292, 2, 0, 0, 746, 2204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1189, 4, 2525, 0, 4096, 3816, 0, 381, 9, 0, 7, 1157, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3152, 214, 0, 0, 15, 4024, 0, 2271, 0, 6, 0, 1325, 0, 80, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 2, 0, 53, 4096, 10, 4093, 1, 0, 0, 6, 20, 3711, 462, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20844, 11268, 16257, 24397, 13356, 17279, 15174, 12497] avg=16384.0000 max=24397 max_rank=3 min=11268 min_rank=1 imbalance_ratio=1.489075 max_load_reduction=106675 max_load_reduction_pct=0.813866 moved_to_echo=58306 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 25, 0, 1503, 677, 19, 0, 2, 2538, 0, 0, 5, 4, 4063, 1148, 4029, 491, 154, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 22, 2, 0, 0, 0, 0, 0, 4071, 3885, 0, 0, 0, 2695, 618, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 795, 0, 954, 3245, 0, 2115, 3989, 0, 54, 124, 0, 0, 0, 0, 4096, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 5, 0, 3142, 22, 4092, 4096, 3193, 4085, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4078, 0, 0, 0, 133, 0, 0, 0, 16, 3818, 0, 0, 9, 0, 2490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 4095, 0, 0, 3939, 3, 1, 33, 618, 0, 3845, 75, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 356, 3141, 29, 1023, 0, 3357, 919, 0, 1, 6, 1425, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 3, 0, 3983, 1, 4095, 2, 0, 0, 0, 4096, 3326, 523, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14815, 15760, 15823, 22936, 14640, 12781, 14306, 20011] avg=16384.0000 max=22936 max_rank=3 min=12781 min_rank=5 imbalance_ratio=1.399902 max_load_reduction=108136 max_load_reduction_pct=0.825012 moved_to_echo=61738 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3887, 0, 2, 1113, 42, 0, 1760, 0, 35, 186, 744, 0, 0, 3682, 1, 208, 502, 189, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 11, 0, 0, 711, 74, 161, 1, 0, 0, 3647, 744, 45, 0, 144, 3995, 508, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 276, 3432, 14, 49, 0, 4096, 333, 4096, 152, 0, 22, 0, 0, 4027, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 40, 0, 0, 10, 0, 3900, 3222, 135, 0, 3907, 3745, 0, 734, 2362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 181, 4036, 0, 0, 0, 4096, 5, 40, 0, 0, 4064, 8, 0, 19, 0, 2044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 984, 0, 744, 4054, 3937, 0, 4002, 4080, 0, 37, 2166, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3212, 0, 3243, 0, 0, 4057, 0, 1, 0, 0, 0, 2277, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 2892, 358, 35, 3944, 482, 19, 0, 0, 6, 4053, 1459, 4055, 429, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12508, 14369, 16913, 18154, 14583, 20009, 12795, 21741] avg=16384.0000 max=21741 max_rank=7 min=12508 min_rank=0 imbalance_ratio=1.326965 max_load_reduction=109331 max_load_reduction_pct=0.834129 moved_to_echo=69128 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1947, 0, 0, 4095, 0, 111, 371, 0, 4066, 1, 0, 4086, 0, 132, 59, 4068, 502, 176, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1557, 0, 0, 0, 0, 0, 0, 90, 1, 6, 4096, 2313, 0, 0, 0, 3989, 620, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3688, 0, 0, 4096, 179, 1, 0, 0, 125, 61, 0, 4087, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 12, 1, 0, 4074, 3647, 1105, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 421, 2285, 3667, 0, 0, 4095, 0, 0, 0, 3679, 5, 0, 4016, 23, 2499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3692, 4, 0, 0, 2236, 248, 3793, 237, 37, 817, 0, 4093, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3213, 3327, 4077, 0, 3785, 37, 0, 1, 0, 0, 0, 2, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3303, 4096, 4088, 291, 41, 0, 16, 17, 0, 33, 671, 663, 4029, 525, 460, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19623, 13053, 12696, 8961, 24784, 15159, 14468, 22328] avg=16384.0000 max=24784 max_rank=4 min=8961 min_rank=3 imbalance_ratio=1.512695 max_load_reduction=106288 max_load_reduction_pct=0.810913 moved_to_echo=76739 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[103, 0, 4096, 0, 10, 564, 0, 64, 2240, 0, 274, 0, 0, 4068, 0, 4033, 499, 149, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 106, 60, 0, 0, 0, 0, 18, 3555, 3478, 295, 0, 4, 4013, 582, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 32, 0, 3930, 1746, 0, 2868, 3935, 99, 1253, 15, 0, 1867, 0, 0, 3873, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 1, 1, 0, 639, 0, 3434, 163, 3905, 0, 3086, 918, 167, 5, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 1, 5, 0, 0, 949, 4, 34, 39, 12, 423, 0, 32, 0, 0, 2344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4091, 0, 3831, 3934, 40, 0, 4062, 1, 171, 0, 4079, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1156, 3193, 0, 3730, 128, 3966, 2608, 654, 7, 605, 2, 0, 4043, 336, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 4087, 29, 4029, 0, 3854, 0, 160, 42, 0, 4095, 0, 492, 109, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16257, 16540, 20043, 12459, 7904, 20225, 20639, 17005] avg=16384.0000 max=20639 max_rank=6 min=7904 min_rank=4 imbalance_ratio=1.259705 max_load_reduction=110433 max_load_reduction_pct=0.842537 moved_to_echo=65773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 19, 0, 143, 4068, 0, 2, 2558, 2, 3428, 0, 4090, 501, 184, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 185, 0, 2, 0, 0, 63, 0, 0, 3721, 4072, 2939, 4063, 0, 4007, 609, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 725, 11, 73, 5, 4092, 0, 0, 3, 0, 2, 0, 8, 4096, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, 322, 4076, 0, 0, 0, 0, 3, 4090, 3168, 3815, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4069, 0, 0, 0, 1955, 3708, 0, 0, 21, 4094, 0, 0, 2137, 0, 2452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 2494, 0, 1, 3939, 466, 0, 0, 63, 0, 0, 824, 0, 599, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3634, 3207, 16, 3503, 27, 3469, 30, 0, 0, 4096, 12, 1638, 3596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1664, 0, 3707, 1, 1543, 10, 0, 23, 0, 34, 487, 515, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15152, 24135, 9496, 19902, 22532, 8624, 23228, 8003] avg=16384.0000 max=24135 max_rank=1 min=8003 min_rank=7 imbalance_ratio=1.473083 max_load_reduction=106937 max_load_reduction_pct=0.815865 moved_to_echo=62387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[8, 10, 3829, 4092, 0, 87, 473, 0, 0, 281, 0, 197, 0, 4096, 37, 3712, 507, 188, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2216, 3953, 0, 9, 0, 0, 0, 50, 0, 4056, 4096, 4096, 0, 0, 0, 0, 625, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 3786, 4041, 108, 0, 3434, 408, 214, 2, 0, 0, 7, 2591, 0, 3985, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1013, 827, 1, 3468, 0, 3810, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2750, 0, 4042, 0, 3935, 0, 23, 1, 0, 0, 0, 4071, 4, 0, 0, 400, 2503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 585, 0, 0, 722, 3333, 350, 317, 85, 3089, 0, 3796, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 864, 3204, 0, 2, 0, 3924, 3769, 1, 1, 0, 14, 0, 697, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1197, 3940, 287, 149, 335, 0, 3795, 1, 0, 0, 0, 6, 4096, 512, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17657, 19486, 19043, 9889, 17729, 20405, 12498, 14365] avg=16384.0000 max=20405 max_rank=5 min=9889 min_rank=3 imbalance_ratio=1.245422 max_load_reduction=110667 max_load_reduction_pct=0.844322 moved_to_echo=64997 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3696, 0, 0, 0, 0, 4096, 0, 2, 14, 32, 1909, 1, 3984, 137, 188, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 3317, 0, 0, 0, 16, 2937, 0, 0, 4085, 35, 21, 0, 0, 1228, 616, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 2217, 4035, 118, 3821, 4092, 0, 2070, 0, 0, 2, 0, 212, 3777, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 191, 92, 77, 2066, 0, 4095, 3881, 3687, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 408, 0, 0, 0, 87, 4067, 0, 10, 92, 4093, 15, 0, 0, 190, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 53, 2964, 0, 3646, 3939, 0, 573, 0, 0, 13, 3868, 1249, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 877, 35, 0, 0, 3893, 3609, 5, 0, 1402, 0, 4003, 4035, 0, 11, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1672, 0, 0, 4061, 0, 101, 25, 0, 6, 1, 3967, 9, 521, 489, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14216, 16461, 20801, 14374, 15544, 16590, 17964, 15122] avg=16384.0000 max=20801 max_rank=2 min=14216 min_rank=0 imbalance_ratio=1.269592 max_load_reduction=110271 max_load_reduction_pct=0.841301 moved_to_echo=65220 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[6144, 23296, 3712, 15616, 256, 22528, 19072, 768, 2304, 6144, 1536, 5760, 20096, 384, 128, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[25088, 4096, 3584, 256, 896, 128, 256, 3200, 128, 4480, 31488, 22784, 3328, 4864, 256, 19968, 4736, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[9728, 1664, 16256, 768, 18688, 27136, 4736, 5632, 8320, 4608, 256, 18944, 14464, 0, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[6016, 1152, 11264, 15872, 2560, 1024, 3072, 256, 20864, 512, 1024, 6912, 640, 25600, 1408, 27904, 3712, 1408, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[27136, 256, 20096, 2304, 7680, 0, 7296, 11904, 384, 128, 256, 27648, 128, 128, 6272, 1408, 19072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[512, 128, 896, 4096, 19456, 15744, 256, 12288, 28928, 1664, 7552, 384, 0, 2048, 2688, 256, 31616, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12672, 16640, 10496, 640, 24320, 512, 17536, 128, 256, 128, 4736, 14464, 19840, 4096, 1152, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[640, 8576, 4096, 0, 896, 1280, 15488, 6016, 19456, 19584, 29056, 22272, 1024, 768, 2432, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[28, 2, 0, 368, 0, 51, 3995, 3577, 0, 4095, 9, 12, 0, 4093, 0, 0, 1, 1426, 0, 0, 3577, 0, 0, 0, 3258, 4025, 0, 5, 0, 4019, 0, 0, 142, 0, 4081, 0, 4039, 95, 2405, 0, 3874, 4096, 15, 1691, 4040, 3723, 4096, 20, 0, 5, 0, 77, 0, 4, 0, 0, 1, 176, 321, 796, 2, 1053, 0, 57, 0, 1, 1738, 0, 0, 0, 1735, 167, 0, 564, 3518, 0, 0, 4094, 0, 2739, 0, 1026, 63, 4010, 3, 0, 3453, 2876, 0, 0, 24, 0, 0, 15, 0, 3740, 0, 419, 2644, 4096, 0, 0, 6, 4081, 0, 285, 0, 2, 0, 3930, 4087, 3984, 0, 0, 0, 269, 4096, 4096, 1641, 3, 0, 186, 0, 0, 0, 0, 93, 38] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[3620, 0, 0, 0, 0, 3999, 4092, 4096, 2, 4096, 3394, 0, 0, 0, 1, 0, 42, 2247, 0, 0, 370, 2925, 26, 0, 0, 4096, 0, 0, 0, 14, 0, 0, 0, 86, 14, 5, 4082, 0, 3786, 3049, 3671, 583, 0, 185, 1068, 3123, 4054, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 6, 3, 0, 0, 0, 0, 14, 181, 4067, 0, 1, 743, 0, 0, 0, 0, 4096, 0, 8, 3719, 0, 3619, 1046, 274, 7, 22, 0, 49, 4084, 4072, 44, 0, 4002, 1, 0, 0, 0, 3767, 0, 339, 0, 1283, 0, 1, 13, 4096, 2, 0, 0, 0, 1909, 0, 4066, 4096, 0, 0, 2882, 0, 4009, 4096, 0, 277, 0, 4094, 0, 2, 3430, 3, 0, 833] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[85, 0, 3418, 0, 1763, 23, 0, 2200, 0, 1, 0, 2, 2307, 0, 4019, 0, 1527, 1, 1480, 37, 12, 3060, 12, 0, 225, 0, 923, 4034, 4070, 4096, 0, 7, 37, 0, 3, 0, 32, 4095, 4093, 0, 2841, 4, 4086, 4037, 2133, 4096, 3796, 12, 349, 137, 1018, 238, 0, 2, 0, 0, 0, 407, 0, 0, 0, 18, 0, 0, 0, 305, 0, 73, 486, 255, 56, 0, 0, 5, 4045, 0, 0, 399, 0, 2331, 0, 4096, 0, 4096, 3514, 406, 546, 4093, 3699, 9, 4056, 0, 0, 141, 0, 0, 232, 4096, 0, 0, 1072, 0, 0, 3914, 0, 0, 1, 551, 7, 26, 26, 3, 0, 3821, 4087, 11, 4045, 2058, 0, 87, 0, 4076, 3985, 6, 0, 0, 0, 1621] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 33, 0, 3, 4001, 803, 4091, 549, 3992, 0, 0, 3366, 3947, 0, 0, 441, 205, 71, 1196, 1, 4096, 0, 0, 2147, 4095, 12, 2, 0, 1919, 0, 40, 126, 0, 11, 0, 3637, 0, 1527, 0, 3092, 4053, 0, 363, 2367, 3844, 2920, 0, 10, 445, 0, 166, 0, 0, 630, 0, 1, 3116, 4066, 0, 0, 9, 0, 0, 0, 1353, 3656, 0, 20, 3992, 0, 0, 0, 0, 3501, 0, 94, 3452, 0, 0, 0, 3329, 0, 23, 4095, 713, 3657, 4095, 0, 0, 4096, 0, 81, 0, 0, 1290, 0, 1131, 0, 486, 2, 1, 156, 4096, 3924, 0, 0, 0, 0, 0, 0, 3078, 0, 0, 2, 0, 3955, 0, 0, 64, 0, 3997, 0, 2394, 4096, 0, 4, 846] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4084, 0, 0, 24, 0, 4096, 4065, 4095, 1916, 3528, 2705, 0, 0, 1701, 0, 321, 308, 3452, 0, 0, 0, 34, 0, 0, 4, 4069, 0, 4002, 0, 4096, 0, 0, 228, 0, 543, 0, 558, 74, 2119, 660, 3971, 2519, 3908, 1344, 39, 2824, 4096, 3816, 0, 6, 4, 0, 0, 0, 0, 129, 1, 239, 0, 0, 5, 0, 0, 0, 3465, 1658, 4096, 0, 54, 1, 0, 8, 0, 4086, 48, 0, 0, 4093, 0, 27, 0, 3701, 150, 4090, 2, 0, 0, 4, 2303, 0, 12, 0, 7, 0, 0, 3656, 1234, 0, 0, 87, 0, 31, 0, 4091, 0, 1610, 155, 256, 3, 812, 4089, 3825, 0, 1132, 0, 0, 3638, 970, 4019, 0, 0, 395, 0, 40, 3341, 0, 40, 260] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4079, 0, 161, 2137, 0, 356, 0, 4096, 0, 4079, 670, 81, 0, 15, 0, 260, 47, 2, 0, 203, 0, 81, 2, 0, 0, 4009, 0, 0, 0, 6, 0, 0, 0, 0, 738, 3991, 101, 2684, 189, 204, 2, 2365, 4096, 4057, 2537, 3864, 3663, 3717, 0, 381, 0, 38, 0, 6, 45, 0, 2, 7, 3656, 0, 1, 0, 0, 30, 0, 65, 11, 0, 0, 4036, 0, 0, 0, 3690, 3586, 0, 165, 4042, 0, 4096, 34, 88, 3625, 3944, 17, 0, 3883, 1697, 0, 0, 2287, 399, 10, 0, 0, 1265, 0, 4038, 0, 4, 0, 0, 0, 4046, 0, 0, 0, 3105, 10, 1699, 2964, 3472, 0, 1556, 2847, 0, 4091, 476, 0, 381, 0, 3354, 0, 0, 1272, 0, 4095, 64] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 58, 4088, 1057, 1230, 3826, 3904, 1, 41, 0, 1, 0, 0, 657, 646, 0, 114, 1022, 0, 0, 0, 4027, 3541, 3296, 0, 3381, 9, 0, 0, 1075, 727, 11, 3540, 0, 1757, 0, 0, 4072, 0, 3788, 270, 3899, 4096, 4084, 1237, 0, 0, 5, 0, 0, 0, 0, 3, 1332, 4096, 0, 0, 0, 0, 0, 422, 2528, 2298, 4, 121, 273, 0, 0, 0, 0, 3460, 8, 0, 4095, 66, 0, 2, 4093, 77, 255, 156, 3918, 4095, 4038, 1411, 154, 238, 1, 0, 0, 0, 4096, 0, 2478, 0, 4094, 0, 55, 16, 4096, 0, 0, 0, 0, 0, 0, 215, 4068, 0, 0, 0, 0, 4015, 16, 0, 574, 0, 4060, 0, 2541, 4091, 44, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[64, 0, 0, 0, 0, 1128, 3726, 2231, 4096, 4096, 0, 4, 37, 1, 0, 4, 0, 1038, 14, 0, 4092, 4047, 2, 0, 0, 1994, 3, 2516, 0, 0, 0, 11, 37, 40, 767, 0, 3033, 0, 4057, 4096, 550, 1258, 4096, 4047, 1, 2159, 4093, 4029, 0, 12, 0, 655, 0, 0, 0, 2, 0, 0, 0, 2305, 0, 5, 0, 2, 0, 4096, 2947, 3, 2, 1317, 0, 375, 0, 0, 2962, 0, 3973, 3756, 78, 2069, 0, 612, 0, 0, 326, 1214, 2, 3511, 2906, 22, 0, 5, 387, 50, 2, 0, 0, 2154, 55, 71, 0, 0, 0, 4044, 0, 0, 0, 2351, 0, 3281, 28, 3965, 1686, 280, 336, 0, 4092, 0, 464, 4088, 0, 4052, 797, 43, 3301, 4, 38, 979] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3620, 0, 0, 0, 0, 3999, 4092, 4096, 2, 1861, 3394, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 2247, 0, 0, 370, 2925, 26, 0, 0, 4096, 0, 0, 0, 14, 0, 0, 1540, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 14, 5, 4082, 0, 0, 3049, 3671, 583, 0, 179, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 6, 3, 0, 0, 0, 0, 4054, 3123, 3786, 4079, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 181, 4067, 0, 1, 743, 0, 0, 0, 0, 4096, 0, 8, 3719, 0, 3619, 485, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 274, 7, 22, 0, 49, 4084, 2356, 44, 0, 4002, 1, 0, 0, 0, 3767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 0, 1283, 0, 1, 13, 4096, 2, 0, 0, 0, 1909, 0, 4066, 4096, 218, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2882, 0, 3306, 4096, 0, 277, 0, 4094, 0, 2, 3430, 3, 0, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21065, 12960, 12737, 16757, 16949, 15652, 16029, 18923] avg=16384.0000 max=21065 max_rank=0 min=12737 min_rank=2 imbalance_ratio=1.285706 max_load_reduction=110007 max_load_reduction_pct=0.839287 moved_to_echo=67553 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 33, 0, 3, 4001, 803, 4091, 549, 1813, 0, 0, 3366, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 441, 205, 71, 1196, 1, 4096, 0, 0, 2147, 4095, 12, 2, 0, 1919, 0, 40, 1501, 1710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 11, 0, 3637, 0, 0, 0, 3092, 4053, 0, 351, 2367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 445, 0, 166, 0, 0, 630, 0, 1, 3116, 4066, 0, 0, 9, 0, 0, 2920, 3844, 1527, 0, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1353, 3656, 0, 20, 3992, 0, 0, 0, 0, 3501, 0, 94, 3452, 0, 0, 478, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3329, 0, 23, 4095, 713, 3657, 2369, 0, 0, 4096, 0, 81, 0, 0, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1131, 0, 486, 2, 1, 156, 4096, 3924, 0, 0, 0, 0, 0, 0, 3078, 215, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3262, 0, 0, 64, 0, 3997, 0, 2394, 4096, 0, 4, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18606, 17436, 13637, 17412, 16562, 19653, 13101, 14665] avg=16384.0000 max=19653 max_rank=5 min=13101 min_rank=6 imbalance_ratio=1.199524 max_load_reduction=111419 max_load_reduction_pct=0.850060 moved_to_echo=63981 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[28, 2, 0, 368, 0, 51, 3995, 3577, 0, 1852, 9, 12, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1426, 0, 0, 3577, 0, 0, 0, 3258, 4025, 0, 5, 0, 4019, 0, 0, 1543, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 4081, 0, 4039, 95, 0, 0, 3874, 4096, 15, 1628, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 77, 0, 4, 0, 0, 1, 176, 321, 796, 2, 1053, 0, 57, 4096, 3723, 2405, 20, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1738, 0, 0, 0, 1735, 167, 0, 564, 3518, 0, 0, 4094, 0, 2739, 498, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 63, 4010, 3, 0, 3453, 1654, 0, 0, 24, 0, 0, 15, 0, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 419, 2644, 4096, 0, 0, 6, 4081, 0, 285, 0, 2, 0, 3930, 4087, 3984, 227, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 3371, 4096, 1641, 3, 0, 186, 0, 0, 0, 0, 93, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13987, 19060, 22010, 13436, 15070, 13988, 23824, 9697] avg=16384.0000 max=23824 max_rank=6 min=9697 min_rank=7 imbalance_ratio=1.454102 max_load_reduction=107248 max_load_reduction_pct=0.818237 moved_to_echo=62579 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4079, 0, 161, 2137, 0, 356, 0, 4096, 0, 1854, 670, 81, 0, 15, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 2, 0, 203, 0, 81, 2, 0, 0, 4009, 0, 0, 0, 6, 0, 0, 1533, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 738, 3991, 101, 2684, 0, 204, 2, 2365, 4096, 3914, 2537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 381, 0, 38, 0, 6, 45, 0, 2, 7, 3656, 0, 1, 0, 0, 30, 3663, 3864, 189, 3717, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 11, 0, 0, 4036, 0, 0, 0, 3690, 3586, 0, 165, 4042, 0, 4096, 495, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 88, 3625, 3944, 17, 0, 3883, 983, 0, 0, 2287, 399, 10, 0, 0, 1265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 4, 0, 0, 0, 4046, 0, 0, 0, 3105, 10, 1699, 2964, 3472, 222, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1556, 2847, 0, 3374, 476, 0, 381, 0, 3354, 0, 0, 1272, 0, 4095, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13709, 6591, 20632, 16291, 20192, 16535, 19703, 17419] avg=16384.0000 max=20632 max_rank=2 min=6591 min_rank=1 imbalance_ratio=1.259277 max_load_reduction=110440 max_load_reduction_pct=0.842590 moved_to_echo=73849 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[85, 0, 3418, 0, 1763, 23, 0, 2200, 0, 1, 0, 2, 2307, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1527, 1, 1480, 37, 12, 3060, 12, 0, 225, 0, 923, 4034, 4070, 4096, 0, 7, 0, 1709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 3, 0, 32, 4095, 0, 0, 2841, 4, 4086, 3895, 2133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 137, 1018, 238, 0, 2, 0, 0, 0, 407, 0, 0, 0, 18, 0, 0, 3796, 4096, 4093, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 0, 73, 486, 255, 56, 0, 0, 5, 4045, 0, 0, 399, 0, 2331, 489, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3514, 406, 546, 2368, 3699, 9, 4056, 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 4096, 0, 0, 1072, 0, 0, 3914, 0, 0, 1, 551, 7, 26, 26, 3, 220, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3821, 4087, 11, 3336, 2058, 0, 87, 0, 4076, 3985, 6, 0, 0, 0, 1621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13818, 21193, 17126, 14166, 8460, 22931, 10290, 23088] avg=16384.0000 max=23088 max_rank=7 min=8460 min_rank=4 imbalance_ratio=1.409180 max_load_reduction=107984 max_load_reduction_pct=0.823853 moved_to_echo=64769 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 0, 24, 0, 4096, 4065, 4095, 1916, 1603, 2705, 0, 0, 1701, 0, 321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 308, 3452, 0, 0, 0, 34, 0, 0, 4, 4069, 0, 4002, 0, 4096, 0, 0, 1326, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 0, 543, 0, 558, 74, 0, 660, 3971, 2519, 3908, 1297, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 129, 1, 239, 0, 0, 5, 0, 0, 0, 4096, 2824, 2119, 3816, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3465, 1658, 4096, 0, 54, 1, 0, 8, 0, 4086, 48, 0, 0, 4093, 0, 27, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3701, 150, 4090, 2, 0, 0, 3, 2303, 0, 12, 0, 7, 0, 0, 3656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 0, 0, 87, 0, 31, 0, 4091, 0, 1610, 155, 256, 3, 812, 4089, 3825, 198, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 0, 0, 3000, 970, 4019, 0, 0, 395, 0, 40, 3341, 0, 40, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[24610, 17292, 13797, 13838, 17976, 13924, 16438, 13197] avg=16384.0000 max=24610 max_rank=0 min=13197 min_rank=7 imbalance_ratio=1.502075 max_load_reduction=106462 max_load_reduction_pct=0.812241 moved_to_echo=61535 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 58, 4088, 1057, 1230, 1739, 3904, 1, 41, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 646, 0, 114, 1022, 0, 0, 0, 4027, 3541, 3296, 0, 3381, 9, 0, 1438, 1686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 727, 11, 3540, 0, 0, 0, 0, 4072, 0, 3655, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1237, 0, 0, 5, 0, 0, 0, 0, 3, 1332, 4096, 0, 0, 0, 0, 0, 4096, 3899, 1757, 4084, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 2528, 2298, 4, 121, 273, 0, 0, 0, 0, 3460, 8, 0, 4095, 66, 0, 485, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 77, 255, 156, 3918, 4095, 2336, 1411, 154, 238, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2478, 0, 4094, 0, 55, 16, 4096, 0, 0, 0, 0, 0, 0, 215, 4068, 218, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3312, 16, 0, 574, 0, 4060, 0, 2541, 4091, 44, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12125, 19817, 13350, 21158, 13776, 20832, 15373, 14641] avg=16384.0000 max=21158 max_rank=3 min=12125 min_rank=0 imbalance_ratio=1.291382 max_load_reduction=109914 max_load_reduction_pct=0.838577 moved_to_echo=64622 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[64, 0, 0, 0, 0, 1128, 3726, 2231, 4096, 1861, 0, 4, 37, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 14, 0, 4092, 4047, 2, 0, 0, 1994, 3, 2516, 0, 0, 0, 11, 1540, 1466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 40, 767, 0, 3033, 0, 0, 4096, 550, 1258, 4096, 3905, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 655, 0, 0, 0, 2, 0, 0, 0, 2305, 0, 5, 0, 2, 4093, 2159, 4057, 4029, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2947, 3, 2, 1317, 0, 375, 0, 0, 2962, 0, 3973, 3756, 78, 2069, 495, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 612, 0, 0, 326, 1214, 2, 2031, 2906, 22, 0, 5, 387, 50, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2154, 55, 71, 0, 0, 0, 4044, 0, 0, 0, 2351, 0, 3281, 28, 3965, 223, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1686, 280, 336, 0, 3374, 0, 464, 4088, 0, 4052, 797, 43, 3301, 4, 38, 979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13152, 16723, 17783, 18014, 22087, 7557, 16314, 19442] avg=16384.0000 max=22087 max_rank=4 min=7557 min_rank=5 imbalance_ratio=1.348083 max_load_reduction=108985 max_load_reduction_pct=0.831490 moved_to_echo=65400 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3968, 10240, 18816, 128, 768, 10624, 1792, 640, 0, 8448, 25216, 128, 4352, 27776, 256, 14976, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12032, 128, 3712, 2560, 1792, 13824, 20864, 25472, 7808, 12672, 10752, 128, 5760, 9856, 4096, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1152, 17280, 3968, 16512, 8192, 6400, 19840, 14208, 10368, 256, 14720, 512, 512, 256, 128, 17920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2432, 9088, 2304, 1536, 8192, 15360, 128, 0, 5760, 26368, 4480, 13952, 4096, 17536, 128, 128, 10496, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1536, 14720, 2816, 10240, 1152, 128, 256, 32512, 3968, 1920, 256, 6272, 2048, 9856, 15488, 26496, 1792, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1792, 6912, 10240, 384, 26368, 11776, 6144, 5504, 0, 24320, 4864, 5120, 19584, 128, 4352, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[1664, 1024, 1024, 1280, 0, 128, 768, 256, 128, 6400, 12160, 3200, 128, 1152, 0, 128, 30848, 27648, 19968, 19840, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[640, 1280, 6912, 4096, 19072, 7040, 0, 8064, 18048, 19072, 16256, 18944, 12544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 14, 0, 0, 0, 363, 4074, 2906, 4018, 1414, 4024, 133, 0, 73, 2, 2, 0, 0, 0, 0, 52, 55, 2879, 4096, 48, 2247, 0, 3562, 1, 19, 16, 4082, 0, 0, 0, 0, 73, 3217, 0, 2847, 0, 0, 4096, 1, 5, 0, 0, 4096, 89, 29, 0, 1, 0, 0, 4084, 0, 0, 745, 493, 0, 0, 4033, 0, 294, 2564, 0, 1, 4057, 0, 1, 8, 0, 568, 44, 33, 0, 2849, 3868, 3907, 0, 0, 4095, 2, 3068, 0, 0, 4096, 0, 1731, 801, 0, 0, 0, 4096, 0, 0, 4028, 0, 4096, 14, 1, 2488, 237, 0, 41, 0, 1641, 0, 0, 0, 2, 2, 2043, 4095, 0, 3582, 1, 152, 666, 3913, 2, 4096, 4018, 1504, 0, 478, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 0, 0, 4096, 4093, 0, 4092, 1, 4095, 39, 0, 4068, 0, 918, 0, 0, 0, 0, 3843, 30, 0, 1911, 2914, 4096, 3, 4096, 0, 0, 1, 172, 0, 0, 3900, 0, 0, 169, 391, 0, 0, 0, 3972, 1, 752, 0, 17, 4037, 0, 7, 0, 0, 0, 156, 4, 0, 0, 0, 236, 0, 0, 0, 0, 1433, 0, 0, 2, 27, 0, 2, 18, 0, 0, 0, 4096, 0, 4022, 0, 3972, 0, 7, 4072, 34, 4087, 4088, 0, 4096, 0, 0, 25, 0, 0, 0, 1378, 2980, 0, 4096, 0, 103, 4, 1006, 325, 0, 0, 217, 0, 87, 3082, 0, 0, 0, 4087, 4089, 4091, 3746, 3922, 1, 3644, 1, 3884, 3, 4096, 9, 0, 39, 0, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[17, 0, 15, 73, 3678, 9, 4096, 4027, 0, 3994, 46, 675, 4096, 26, 2249, 0, 4049, 0, 0, 0, 492, 2120, 0, 0, 7, 343, 3948, 2, 3305, 0, 0, 2, 4037, 0, 6, 1984, 0, 64, 0, 359, 0, 4, 45, 4026, 22, 69, 0, 4, 4067, 0, 150, 1, 27, 2882, 4039, 15, 0, 0, 0, 989, 0, 4, 0, 27, 0, 3686, 55, 0, 0, 28, 0, 0, 0, 1, 0, 1, 90, 3976, 3, 213, 0, 0, 3750, 3363, 2446, 19, 1887, 4091, 2, 0, 4079, 0, 0, 0, 16, 0, 0, 4093, 0, 3989, 0, 1, 4078, 4046, 0, 25, 4067, 801, 0, 0, 14, 0, 0, 4095, 0, 22, 24, 8, 3166, 4094, 2978, 133, 4096, 3347, 0, 198, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[225, 0, 9, 3828, 4010, 0, 44, 0, 0, 4054, 0, 4096, 3367, 4096, 1, 0, 1, 0, 812, 228, 1, 142, 0, 4051, 0, 3891, 1554, 1, 1, 2932, 0, 3548, 115, 0, 145, 31, 27, 32, 2934, 4088, 4081, 0, 318, 0, 9, 2279, 0, 0, 3, 0, 0, 0, 16, 0, 13, 1, 0, 27, 125, 134, 80, 97, 3, 1, 409, 0, 0, 0, 4024, 0, 1213, 0, 0, 0, 0, 3915, 1, 3943, 3030, 633, 3, 0, 4036, 781, 0, 4095, 0, 4091, 157, 0, 0, 0, 0, 0, 0, 0, 77, 1233, 90, 3766, 0, 2132, 4096, 0, 5, 4076, 4068, 0, 56, 59, 1069, 10, 0, 2, 2745, 4090, 0, 0, 2970, 4028, 2875, 0, 3085, 4079, 0, 674, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 53, 0, 1612, 37, 44, 4089, 31, 1188, 4090, 54, 2978, 2520, 4096, 3539, 385, 2, 6, 68, 0, 47, 196, 0, 1, 3388, 0, 4074, 0, 2, 0, 4027, 3377, 0, 0, 0, 0, 2129, 712, 7, 1, 535, 0, 151, 3998, 0, 35, 1, 0, 0, 0, 0, 0, 0, 812, 4096, 1, 0, 0, 0, 0, 0, 0, 24, 169, 0, 0, 0, 0, 2901, 0, 8, 1, 0, 0, 0, 19, 23, 4096, 4090, 61, 274, 15, 26, 1, 2979, 4095, 0, 4096, 0, 0, 0, 0, 1464, 0, 0, 1635, 843, 4096, 3480, 0, 0, 4028, 1, 0, 0, 4096, 4096, 3, 4096, 0, 0, 0, 714, 4091, 3743, 20, 3146, 0, 4074, 8, 3951, 0, 0, 13, 4, 3069, 1079, 4062] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 254, 0, 0, 263, 14, 59, 4096, 0, 3864, 0, 3831, 4096, 5, 23, 0, 4083, 312, 0, 11, 3887, 3719, 0, 16, 3736, 0, 4053, 0, 3916, 0, 18, 0, 0, 0, 0, 0, 253, 2680, 9, 806, 4018, 0, 3972, 286, 0, 0, 0, 4092, 3980, 0, 0, 0, 0, 1700, 146, 4, 0, 0, 0, 191, 0, 0, 8, 0, 0, 17, 0, 4076, 0, 2, 0, 0, 0, 0, 0, 109, 0, 2991, 19, 4087, 0, 2, 4044, 3409, 733, 4095, 0, 4096, 0, 0, 0, 3, 0, 0, 3753, 0, 0, 4096, 3756, 0, 0, 32, 1, 3820, 0, 11, 57, 6, 4064, 0, 0, 0, 1002, 3590, 4045, 2, 268, 0, 913, 0, 4029, 0, 3413, 7, 27, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 356, 3980, 0, 4016, 15, 1235, 4092, 161, 4083, 4092, 522, 0, 0, 2794, 0, 1215, 0, 1271, 37, 35, 0, 69, 0, 3513, 6, 0, 0, 0, 2720, 0, 0, 89, 0, 94, 108, 3695, 837, 267, 0, 0, 3916, 0, 318, 0, 10, 3925, 27, 0, 304, 1, 24, 3794, 0, 1, 0, 3739, 3300, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1032, 0, 0, 221, 0, 592, 0, 3285, 4, 0, 2, 1726, 10, 0, 1166, 3273, 0, 4081, 0, 2, 802, 0, 0, 0, 0, 6, 0, 4062, 0, 0, 3, 3989, 1897, 4093, 0, 0, 3769, 0, 4094, 3, 2, 5, 3868, 2140, 3414, 418, 4095, 0, 4093, 4096, 0, 0, 4026, 4082, 9, 0, 0, 4047] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 32, 0, 10, 42, 172, 2157, 0, 103, 4095, 7, 0, 0, 4061, 6, 0, 0, 4095, 4088, 5, 2656, 3272, 207, 4087, 0, 3183, 0, 569, 4081, 0, 1, 0, 1817, 0, 0, 112, 7, 6, 0, 179, 0, 0, 4090, 23, 0, 4075, 0, 0, 0, 0, 0, 11, 483, 0, 0, 672, 0, 0, 0, 4093, 0, 28, 3, 1, 4047, 4041, 0, 116, 0, 0, 0, 0, 4022, 0, 48, 143, 3846, 0, 152, 4082, 1, 4085, 4095, 0, 3444, 0, 0, 2564, 0, 0, 0, 562, 0, 0, 4096, 0, 0, 0, 4, 1, 3930, 0, 4095, 1913, 24, 4095, 0, 0, 0, 0, 11, 4086, 3338, 1313, 3, 4091, 109, 3996, 0, 4070, 21, 1, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 0, 0, 4096, 4093, 0, 917, 1, 4095, 39, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 918, 0, 0, 0, 0, 3843, 30, 0, 1911, 2914, 2306, 3, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 3900, 0, 0, 169, 391, 0, 0, 0, 3972, 1, 752, 0, 17, 3175, 2114, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 7, 0, 0, 0, 156, 4, 0, 0, 0, 236, 0, 0, 0, 0, 4096, 4091, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1433, 0, 0, 2, 27, 0, 2, 18, 0, 0, 0, 4096, 0, 4022, 0, 3972, 1790, 958, 938, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4072, 34, 4087, 4088, 0, 3436, 0, 0, 25, 0, 0, 0, 1378, 2980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 0, 103, 4, 1006, 325, 0, 0, 217, 0, 87, 3082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4089, 0, 3746, 3922, 1, 2706, 1, 0, 3, 0, 9, 0, 39, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17316, 16022, 15478, 14397, 17918, 20107, 7147, 22687] avg=16384.0000 max=22687 max_rank=7 min=7147 min_rank=6 imbalance_ratio=1.384705 max_load_reduction=108385 max_load_reduction_pct=0.826912 moved_to_echo=67859 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 14, 0, 0, 0, 363, 4074, 2906, 900, 1414, 4024, 133, 0, 73, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 52, 55, 2879, 4096, 48, 1265, 0, 3562, 1, 19, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 73, 3217, 0, 2847, 0, 0, 4096, 1, 5, 0, 0, 3118, 2129, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 89, 29, 0, 1, 0, 0, 4084, 0, 0, 745, 493, 0, 0, 4033, 0, 4096, 4095, 1784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 2564, 0, 1, 4057, 0, 1, 8, 0, 568, 44, 33, 0, 2849, 3868, 3907, 982, 942, 39, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 3068, 0, 0, 3436, 0, 1731, 801, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2284, 0, 4096, 14, 1, 2488, 237, 0, 41, 0, 1641, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2043, 0, 0, 3582, 1, 113, 666, 0, 2, 0, 4018, 1504, 0, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13903, 11995, 20370, 23545, 20817, 17229, 10804, 12409] avg=16384.0000 max=23545 max_rank=3 min=10804 min_rank=6 imbalance_ratio=1.437073 max_load_reduction=107527 max_load_reduction_pct=0.820366 moved_to_echo=61259 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 15, 73, 3678, 9, 4096, 4027, 0, 895, 46, 675, 4096, 26, 2249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 492, 2120, 0, 0, 7, 343, 2222, 2, 3305, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 6, 1984, 0, 64, 0, 359, 0, 4, 45, 4026, 22, 69, 0, 4, 3099, 1620, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 150, 1, 27, 2882, 4039, 15, 0, 0, 0, 989, 0, 4, 0, 27, 4096, 0, 1358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3686, 55, 0, 0, 28, 0, 0, 0, 1, 0, 1, 90, 3976, 3, 213, 1726, 957, 815, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3750, 3363, 2446, 19, 1887, 3432, 2, 0, 4079, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2321, 0, 3989, 0, 1, 4078, 4046, 0, 25, 4067, 801, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 22, 24, 8, 2351, 4094, 0, 133, 0, 3347, 0, 198, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19902, 12542, 16154, 17655, 12210, 18994, 19342, 14273] avg=16384.0000 max=19902 max_rank=0 min=12210 min_rank=4 imbalance_ratio=1.214722 max_load_reduction=111170 max_load_reduction_pct=0.848160 moved_to_echo=64819 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 53, 0, 1612, 37, 44, 4089, 31, 1188, 916, 54, 2978, 2520, 4096, 3539, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 68, 0, 47, 196, 0, 1, 3388, 0, 2293, 0, 2, 0, 4027, 3377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2129, 712, 7, 1, 535, 0, 151, 3998, 0, 35, 1, 0, 3174, 2151, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 812, 4096, 1, 0, 0, 0, 0, 0, 0, 24, 169, 0, 3743, 1800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2901, 0, 8, 1, 0, 0, 0, 19, 23, 4096, 4090, 61, 1781, 958, 1049, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 15, 26, 1, 2979, 4095, 0, 3436, 0, 0, 0, 0, 1464, 0, 0, 1635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 843, 2323, 3480, 0, 0, 4028, 1, 0, 0, 4096, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 714, 4091, 0, 20, 3146, 0, 3025, 8, 0, 0, 0, 13, 4, 3069, 1079, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21542, 13407, 13709, 10645, 15647, 13925, 22966, 19231] avg=16384.0000 max=22966 max_rank=6 min=10645 min_rank=3 imbalance_ratio=1.401733 max_load_reduction=108106 max_load_reduction_pct=0.824783 moved_to_echo=71769 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[225, 0, 9, 3828, 4010, 0, 44, 0, 0, 908, 0, 4096, 3367, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 812, 228, 1, 142, 0, 4051, 0, 3891, 875, 1, 1, 2932, 0, 3548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 145, 31, 27, 32, 2934, 4088, 4081, 0, 318, 0, 9, 2279, 0, 0, 3146, 1565, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 16, 0, 13, 1, 0, 27, 125, 134, 80, 97, 3, 1, 3085, 2745, 1310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 0, 0, 0, 4024, 0, 1213, 0, 0, 0, 0, 3915, 1, 3943, 3030, 633, 679, 288, 765, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4036, 781, 0, 4095, 0, 3432, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 700, 90, 3766, 0, 2132, 4096, 0, 5, 4076, 4068, 0, 56, 59, 1069, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4090, 0, 0, 2205, 4028, 0, 0, 0, 4079, 0, 674, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20584, 16483, 19015, 7640, 19559, 12504, 20204, 15083] avg=16384.0000 max=20584 max_rank=0 min=7640 min_rank=3 imbalance_ratio=1.256348 max_load_reduction=110488 max_load_reduction_pct=0.842957 moved_to_echo=67350 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 254, 0, 0, 263, 14, 59, 4096, 0, 861, 0, 3831, 4096, 5, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 312, 0, 11, 3887, 3719, 0, 16, 3736, 0, 2278, 0, 3916, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 253, 2680, 9, 806, 4018, 0, 3972, 286, 0, 0, 0, 4092, 3003, 2193, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 0, 0, 1700, 146, 4, 0, 0, 0, 191, 0, 0, 8, 0, 3413, 4045, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4076, 0, 2, 0, 0, 0, 0, 0, 109, 0, 2991, 19, 4087, 1775, 959, 239, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4044, 3409, 733, 4095, 0, 3433, 0, 0, 0, 3, 0, 0, 3753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2318, 3756, 0, 0, 32, 1, 3820, 0, 11, 57, 6, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 3590, 0, 2, 268, 0, 674, 0, 0, 0, 0, 7, 27, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13502, 21976, 22131, 15323, 14937, 19472, 14065, 9666] avg=16384.0000 max=22131 max_rank=2 min=9666 min_rank=7 imbalance_ratio=1.350769 max_load_reduction=108941 max_load_reduction_pct=0.831154 moved_to_echo=58140 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 356, 3980, 0, 4016, 15, 1235, 917, 161, 4083, 4092, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2794, 0, 1215, 0, 1271, 37, 35, 0, 69, 0, 1977, 6, 0, 0, 0, 2720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 94, 108, 3695, 837, 267, 0, 0, 3916, 0, 318, 0, 10, 3175, 0, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3925, 27, 0, 304, 1, 24, 3794, 0, 1, 0, 3739, 3300, 0, 0, 0, 0, 4026, 3414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1032, 0, 0, 221, 0, 592, 0, 3285, 4, 0, 1536, 950, 1054, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1726, 10, 0, 1166, 3273, 0, 3424, 0, 2, 802, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2304, 0, 0, 3, 3989, 1897, 4093, 0, 0, 3769, 0, 4094, 3, 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3868, 2140, 0, 418, 4095, 0, 3039, 4096, 0, 0, 0, 4082, 9, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19377, 10124, 13317, 22555, 9335, 10411, 20159, 25794] avg=16384.0000 max=25794 max_rank=7 min=9335 min_rank=4 imbalance_ratio=1.574341 max_load_reduction=105278 max_load_reduction_pct=0.803207 moved_to_echo=65699 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 32, 0, 10, 42, 172, 483, 0, 103, 4095, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 6, 0, 0, 4095, 4088, 5, 2656, 3272, 207, 2300, 0, 3183, 0, 569, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1817, 0, 0, 112, 7, 6, 0, 179, 0, 0, 4090, 23, 0, 1674, 2174, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 11, 483, 0, 0, 672, 0, 0, 0, 4093, 0, 4070, 4086, 1822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3, 1, 4047, 4041, 0, 116, 0, 0, 0, 0, 4022, 0, 48, 143, 3846, 1787, 958, 1054, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 4082, 1, 4085, 4095, 0, 2889, 0, 0, 2564, 0, 0, 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 0, 0, 0, 4, 1, 3930, 0, 4095, 1913, 24, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 3338, 1313, 3, 3037, 109, 0, 0, 0, 21, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4946, 28523, 10898, 19312, 20649, 18430, 16385, 11929] avg=16384.0000 max=28523 max_rank=1 min=4946 min_rank=0 imbalance_ratio=1.740906 max_load_reduction=102549 max_load_reduction_pct=0.782387 moved_to_echo=67393 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[24192, 128, 256, 384, 128, 5504, 12288, 4608, 128, 128, 5376, 5376, 128, 128, 8192, 256, 27008, 26240, 11776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 384, 128, 5888, 12032, 128, 16896, 16384, 5504, 6912, 1792, 23936, 22528, 8832, 9984, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[16000, 384, 2176, 256, 9856, 14208, 128, 9728, 16512, 7424, 15616, 128, 18176, 2944, 4736, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[9728, 20096, 0, 11648, 16384, 128, 17152, 13056, 0, 256, 0, 15616, 1664, 4096, 1664, 20480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 1920, 24192, 7680, 18688, 23808, 1920, 27008, 256, 1792, 8320, 128, 1536, 0, 9856, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1024, 16896, 7424, 12032, 128, 11264, 12928, 16128, 128, 12672, 18048, 2688, 19584, 128, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[8448, 128, 256, 7808, 2560, 3712, 10240, 6528, 11776, 128, 4736, 20352, 128, 7552, 128, 4224, 23680, 13952, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2176, 6272, 128, 8192, 15104, 128, 2432, 128, 0, 896, 128, 12800, 128, 25216, 11264, 16768, 12160, 7040, 6016, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4092, 0, 0, 4096, 0, 0, 0, 0, 0, 234, 0, 2, 0, 0, 3865, 1198, 0, 50, 1173, 0, 4022, 2, 4029, 0, 0, 250, 7, 0, 4090, 64, 0, 0, 293, 1, 0, 0, 0, 3943, 4095, 0, 12, 2998, 0, 1405, 0, 1214, 0, 400, 0, 2284, 304, 640, 0, 1, 0, 4096, 0, 3204, 3, 0, 3804, 20, 0, 0, 3991, 208, 0, 4, 0, 0, 3938, 0, 0, 4092, 0, 0, 1915, 2, 4039, 0, 0, 120, 3847, 0, 4093, 0, 693, 4059, 0, 0, 32, 3941, 0, 2519, 42, 0, 4093, 0, 0, 1, 196, 0, 3722, 0, 0, 321, 2027, 38, 0, 193, 1, 0, 388, 4096, 0, 4096, 207, 0, 2880, 0, 4006, 0, 3747, 7, 4096, 3531, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 0, 4096, 0, 5, 0, 396, 0, 3204, 0, 1, 0, 0, 15, 4092, 0, 1, 102, 0, 4096, 1230, 4016, 4051, 16, 3768, 3335, 4067, 4076, 0, 0, 0, 1347, 1, 0, 663, 0, 4073, 75, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 2, 2, 4, 49, 0, 4096, 0, 4040, 2, 3352, 4036, 0, 0, 0, 3621, 63, 2, 4045, 20, 0, 32, 0, 0, 4087, 0, 131, 5, 3, 4096, 0, 0, 1603, 4082, 0, 4, 0, 90, 4057, 0, 0, 0, 4093, 0, 244, 0, 0, 46, 0, 4006, 67, 14, 0, 4095, 0, 0, 3, 46, 12, 1934, 60, 2, 0, 8, 287, 0, 4096, 34, 0, 1174, 0, 39, 0, 3159, 4094, 4069, 1, 3034, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[3806, 0, 3071, 119, 0, 2, 97, 3988, 0, 300, 0, 1, 4012, 0, 140, 429, 0, 0, 748, 621, 4088, 1, 4071, 0, 1091, 1225, 0, 0, 4092, 3038, 0, 203, 1, 181, 0, 0, 0, 3965, 2529, 0, 0, 0, 0, 3165, 0, 503, 0, 0, 321, 72, 60, 121, 21, 81, 0, 4096, 0, 0, 283, 0, 3434, 0, 0, 0, 4095, 3352, 0, 3, 0, 0, 347, 0, 3952, 4076, 0, 760, 878, 4066, 752, 0, 0, 21, 3940, 0, 51, 916, 4096, 3594, 0, 0, 1574, 3, 0, 4072, 3575, 0, 0, 0, 12, 0, 166, 0, 3514, 0, 0, 10, 2884, 4088, 0, 110, 1, 0, 1729, 4096, 0, 4, 4096, 0, 4085, 0, 4096, 0, 4058, 20, 1, 3, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[3997, 1, 58, 819, 0, 0, 66, 1899, 0, 3808, 0, 11, 2028, 0, 112, 946, 0, 66, 0, 230, 4094, 1, 273, 1128, 367, 4093, 0, 21, 4084, 0, 613, 7, 0, 0, 0, 109, 0, 241, 2965, 0, 0, 3981, 0, 0, 137, 4095, 0, 0, 0, 0, 0, 651, 0, 0, 0, 3915, 4, 43, 0, 538, 4096, 0, 48, 0, 4095, 0, 0, 3046, 0, 0, 0, 1, 0, 4096, 0, 2, 4094, 0, 4094, 0, 0, 3877, 0, 6, 1607, 3077, 4077, 0, 0, 157, 0, 4096, 0, 4043, 467, 0, 4054, 5, 39, 17, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 224, 0, 21, 477, 3474, 34, 4091, 4077, 0, 4089, 0, 4093, 2, 3866, 1, 4096, 1239, 2697, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 2240, 2381, 367, 0, 92, 0, 5, 0, 1660, 0, 0, 0, 484, 4095, 0, 4096, 32, 82, 144, 13, 3945, 0, 1708, 3802, 0, 0, 0, 4090, 0, 357, 5, 2, 1, 225, 195, 31, 3670, 0, 146, 0, 46, 0, 1155, 53, 1262, 55, 1, 0, 0, 0, 0, 3837, 0, 0, 666, 5, 442, 0, 4062, 3993, 311, 0, 806, 3811, 0, 4095, 103, 0, 963, 0, 0, 3967, 39, 3199, 8, 0, 4016, 315, 1, 4019, 0, 4096, 3690, 0, 6, 2922, 3464, 398, 1839, 79, 0, 0, 0, 0, 148, 2, 0, 3880, 0, 210, 0, 342, 3460, 0, 647, 0, 0, 4035, 3, 103, 4063, 512, 0, 3437, 0, 3914, 1, 30, 392, 2632, 3378, 3841, 354] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[447, 1144, 0, 3088, 51, 0, 4095, 114, 0, 2, 0, 3837, 63, 13, 3, 0, 0, 13, 0, 484, 3014, 3, 0, 0, 4094, 4066, 6, 0, 4094, 35, 2329, 242, 0, 140, 34, 133, 2436, 4088, 1563, 1242, 3930, 504, 0, 1837, 3936, 2884, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 46, 4040, 0, 0, 4004, 0, 104, 0, 3952, 4051, 1, 30, 0, 27, 3846, 4, 0, 4048, 0, 0, 0, 2425, 6, 0, 15, 0, 1, 0, 4045, 2724, 0, 0, 0, 0, 3605, 3574, 0, 4065, 4096, 166, 0, 0, 0, 562, 1, 0, 2637, 181, 2, 0, 187, 4096, 0, 0, 0, 50, 4096, 2, 0, 1607, 3010, 1, 0, 0, 4033, 319, 3266, 64, 23, 0, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[3940, 27, 4095, 0, 0, 45, 4062, 28, 0, 4038, 0, 0, 1, 0, 0, 3639, 0, 0, 0, 3147, 3848, 0, 3740, 38, 9, 4091, 0, 257, 3276, 0, 19, 9, 0, 98, 0, 69, 106, 78, 0, 0, 32, 10, 0, 19, 172, 4021, 4095, 0, 0, 1473, 0, 0, 0, 0, 3497, 6, 12, 4091, 0, 4023, 4096, 0, 554, 0, 3880, 0, 0, 564, 0, 0, 2732, 14, 0, 2715, 0, 1, 8, 3616, 4022, 0, 0, 41, 0, 2, 0, 4096, 238, 0, 64, 5, 0, 4096, 0, 1533, 311, 1736, 171, 0, 1508, 13, 0, 0, 4096, 0, 7, 0, 4086, 2621, 1182, 0, 0, 5, 1857, 0, 3, 4004, 3859, 58, 456, 0, 4095, 491, 4021, 0, 9, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[602, 0, 5, 1969, 0, 2735, 1878, 8, 22, 140, 0, 0, 0, 0, 4, 318, 0, 0, 28, 1, 4070, 0, 4096, 4096, 1345, 3693, 0, 4005, 3884, 5, 0, 0, 0, 4091, 0, 0, 58, 4090, 3272, 671, 862, 41, 13, 3068, 0, 114, 191, 22, 3875, 186, 0, 76, 0, 0, 425, 13, 22, 3989, 4077, 494, 3500, 188, 0, 0, 3719, 477, 0, 4095, 0, 96, 0, 0, 0, 3438, 10, 0, 896, 0, 2832, 609, 1, 0, 0, 0, 0, 4073, 1148, 4026, 0, 0, 99, 3207, 0, 1126, 0, 0, 5, 0, 0, 0, 4, 9, 4096, 0, 1, 0, 3699, 3147, 0, 3351, 0, 0, 4096, 0, 0, 3958, 0, 328, 0, 0, 4093, 0, 4093, 59, 4, 1, 4033, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[447, 1144, 0, 3088, 51, 0, 4095, 114, 0, 2, 0, 3837, 63, 13, 3, 0, 1777, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 484, 3014, 3, 0, 0, 4094, 4066, 6, 0, 710, 35, 2329, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 34, 133, 2436, 4088, 1563, 1242, 3930, 504, 0, 1837, 3936, 2884, 0, 0, 2967, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 46, 4040, 0, 0, 4004, 0, 104, 0, 2515, 534, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1641, 4051, 1, 30, 0, 27, 3846, 4, 0, 4048, 0, 0, 0, 2425, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1, 0, 4045, 2724, 0, 0, 0, 0, 3605, 1059, 0, 4065, 4096, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 1, 0, 2637, 181, 2, 0, 187, 4096, 0, 0, 0, 50, 4033, 2263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 1197, 3010, 1, 0, 0, 0, 319, 0, 64, 23, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15044, 14996, 26697, 11662, 16079, 19776, 14012, 12806] avg=16384.0000 max=26697 max_rank=2 min=11662 min_rank=3 imbalance_ratio=1.629456 max_load_reduction=104375 max_load_reduction_pct=0.796318 moved_to_echo=62673 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 4096, 0, 5, 0, 396, 0, 3204, 0, 1, 0, 0, 15, 4092, 1628, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 102, 0, 4096, 1230, 4016, 4051, 16, 3768, 3335, 4067, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1347, 1, 0, 663, 0, 4073, 75, 0, 0, 0, 0, 0, 0, 14, 0, 0, 2954, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 2, 4, 49, 0, 4096, 0, 4040, 2, 3352, 4036, 0, 0, 0, 2880, 489, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1504, 63, 2, 4045, 20, 0, 32, 0, 0, 4087, 0, 131, 5, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1603, 4082, 0, 4, 0, 90, 4057, 0, 0, 0, 1213, 0, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 4006, 67, 14, 0, 4095, 0, 0, 3, 46, 12, 1934, 60, 2, 0, 39, 2189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 287, 0, 3051, 34, 0, 1174, 0, 0, 0, 0, 4094, 4069, 1, 3034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18577, 25389, 10097, 23463, 13988, 11293, 12513, 15752] avg=16384.0000 max=25389 max_rank=1 min=10097 min_rank=2 imbalance_ratio=1.549622 max_load_reduction=105683 max_load_reduction_pct=0.806297 moved_to_echo=53546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 4096, 0, 0, 0, 0, 0, 234, 0, 2, 0, 0, 3865, 1198, 1799, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 1173, 0, 4022, 2, 4029, 0, 0, 250, 7, 0, 698, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 1, 0, 0, 0, 3943, 4095, 0, 12, 2998, 0, 1405, 0, 1214, 0, 400, 2970, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2284, 304, 640, 0, 1, 0, 4096, 0, 3204, 3, 0, 3804, 20, 0, 0, 2776, 545, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1647, 208, 0, 4, 0, 0, 3938, 0, 0, 4092, 0, 0, 1915, 2, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 3847, 0, 4093, 0, 693, 4059, 0, 0, 32, 1165, 0, 2519, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 196, 0, 3722, 0, 0, 321, 2027, 38, 0, 193, 1, 0, 4006, 2595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 4096, 0, 3049, 207, 0, 2880, 0, 0, 0, 0, 7, 4096, 3531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16333, 10295, 18483, 18099, 15845, 16570, 17193, 18254] avg=16384.0000 max=18483 max_rank=2 min=10295 min_rank=1 imbalance_ratio=1.128113 max_load_reduction=112589 max_load_reduction_pct=0.858986 moved_to_echo=67862 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3806, 0, 3071, 119, 0, 2, 97, 3988, 0, 300, 0, 1, 4012, 0, 140, 429, 1842, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 748, 621, 4088, 1, 4071, 0, 1091, 1225, 0, 0, 709, 3038, 0, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 181, 0, 0, 0, 3965, 2529, 0, 0, 0, 0, 3165, 0, 503, 0, 0, 2966, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 321, 72, 60, 121, 21, 81, 0, 4096, 0, 0, 283, 0, 3434, 0, 0, 0, 2, 553, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1700, 3352, 0, 3, 0, 0, 347, 0, 3952, 4076, 0, 760, 878, 4066, 752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 3940, 0, 51, 916, 4096, 3594, 0, 0, 1574, 1, 0, 4072, 3575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 166, 0, 3514, 0, 0, 10, 2884, 4088, 0, 110, 1, 0, 4096, 2811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1729, 4096, 0, 3, 4096, 0, 4085, 0, 0, 0, 0, 20, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17808, 15795, 14557, 9461, 19886, 21840, 17692, 14033] avg=16384.0000 max=21840 max_rank=5 min=9461 min_rank=3 imbalance_ratio=1.333008 max_load_reduction=109232 max_load_reduction_pct=0.833374 moved_to_echo=73451 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3997, 1, 58, 819, 0, 0, 66, 1899, 0, 3808, 0, 11, 2028, 0, 112, 946, 1842, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 230, 4094, 1, 273, 1128, 367, 4093, 0, 21, 708, 0, 613, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 241, 2965, 0, 0, 3981, 0, 0, 137, 4095, 0, 0, 2960, 1187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 0, 0, 0, 3915, 4, 43, 0, 538, 4096, 0, 48, 0, 2883, 553, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1700, 0, 0, 3046, 0, 0, 0, 1, 0, 4096, 0, 2, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 6, 1607, 3077, 4077, 0, 0, 157, 0, 1213, 0, 4043, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 5, 39, 17, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 224, 0, 21, 4093, 2679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 3474, 34, 3047, 4077, 0, 4089, 0, 0, 2, 0, 1, 4096, 1239, 2697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16631, 11601, 15675, 13147, 17033, 18524, 15228, 23233] avg=16384.0000 max=23233 max_rank=7 min=11601 min_rank=1 imbalance_ratio=1.418030 max_load_reduction=107839 max_load_reduction_pct=0.822746 moved_to_echo=74018 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 2240, 2381, 367, 0, 92, 0, 5, 0, 1660, 0, 0, 1796, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 4095, 0, 4096, 32, 82, 144, 13, 3945, 0, 1708, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 357, 5, 2, 1, 225, 195, 31, 3670, 0, 146, 0, 46, 0, 1155, 2756, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 1262, 55, 1, 0, 0, 0, 0, 3837, 0, 0, 666, 5, 442, 0, 4062, 2438, 539, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1658, 311, 0, 806, 3811, 0, 4095, 103, 0, 963, 0, 0, 3967, 39, 3199, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 315, 1, 4019, 0, 4096, 3690, 0, 6, 2922, 1026, 398, 1839, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 2, 0, 3880, 0, 210, 0, 342, 3460, 0, 647, 0, 0, 3914, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 3, 103, 3026, 512, 0, 3437, 0, 0, 1, 0, 392, 2632, 3378, 3841, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13674, 15258, 12689, 13747, 18960, 22407, 12623, 21714] avg=16384.0000 max=22407 max_rank=5 min=12623 min_rank=6 imbalance_ratio=1.367615 max_load_reduction=108665 max_load_reduction_pct=0.829048 moved_to_echo=75704 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[602, 0, 5, 1969, 0, 2735, 1878, 8, 22, 140, 0, 0, 0, 0, 4, 318, 1672, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 1, 4070, 0, 4096, 4096, 1345, 3693, 0, 4005, 673, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 58, 4090, 3272, 671, 862, 41, 13, 3068, 0, 114, 191, 22, 2815, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3875, 186, 0, 76, 0, 0, 425, 13, 22, 3989, 4077, 494, 3500, 188, 0, 0, 2257, 502, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1545, 477, 0, 4095, 0, 96, 0, 0, 0, 3438, 10, 0, 896, 0, 2832, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4073, 1148, 4026, 0, 0, 99, 950, 0, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4, 9, 4096, 0, 1, 0, 3699, 3147, 0, 3351, 0, 0, 4093, 2836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2948, 0, 328, 0, 0, 0, 0, 0, 59, 4, 1, 4033, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10363, 22012, 20565, 20000, 13998, 11423, 21241, 11470] avg=16384.0000 max=22012 max_rank=1 min=10363 min_rank=0 imbalance_ratio=1.343506 max_load_reduction=109060 max_load_reduction_pct=0.832062 moved_to_echo=58132 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3940, 27, 4095, 0, 0, 45, 4062, 28, 0, 4038, 0, 0, 1, 0, 0, 3639, 1745, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3147, 3848, 0, 3740, 38, 9, 4091, 0, 257, 568, 0, 19, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 69, 106, 78, 0, 0, 32, 10, 0, 19, 172, 4021, 4095, 0, 2374, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1473, 0, 0, 0, 0, 3497, 6, 12, 4091, 0, 4023, 4096, 0, 554, 0, 2883, 524, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1611, 0, 0, 564, 0, 0, 2732, 14, 0, 2715, 0, 1, 8, 3616, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 2, 0, 4096, 238, 0, 64, 5, 0, 1213, 0, 1533, 311, 1736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 1508, 13, 0, 0, 4096, 0, 7, 0, 4086, 2621, 1182, 0, 0, 5, 4095, 2786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1857, 0, 3, 2982, 3859, 58, 456, 0, 0, 491, 0, 0, 9, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22642, 15726, 12309, 21493, 15283, 9239, 20570, 13810] avg=16384.0000 max=22642 max_rank=0 min=9239 min_rank=5 imbalance_ratio=1.381958 max_load_reduction=108430 max_load_reduction_pct=0.827255 moved_to_echo=58902 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[25088, 1280, 7296, 14208, 128, 5120, 12672, 6912, 128, 11904, 0, 3968, 6144, 1792, 4224, 10624, 14208, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 9728, 12288, 128, 13824, 14976, 14464, 19456, 128, 256, 8320, 7936, 512, 19456, 8576, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[16768, 12032, 256, 19328, 15872, 512, 16128, 0, 0, 896, 0, 4736, 14976, 8192, 21888, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[8448, 128, 5632, 896, 384, 128, 30208, 256, 256, 384, 13312, 17536, 3200, 4608, 128, 128, 28416, 18304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[13056, 8576, 128, 12672, 3840, 128, 15104, 128, 3968, 27520, 128, 896, 11776, 10240, 23040, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[5760, 4608, 512, 1024, 2688, 20480, 14848, 2176, 4992, 11264, 128, 9728, 4352, 12928, 4352, 1664, 22784, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 640, 6272, 4608, 31360, 1280, 20352, 9472, 7040, 25216, 3456, 10112, 5504, 3200, 3072, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[4352, 9472, 512, 1536, 128, 256, 3968, 16256, 3968, 19456, 4480, 9088, 27008, 768, 768, 4096, 18688, 4352, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 3977, 216, 2450, 3, 0, 0, 1, 4087, 2197, 0, 3997, 3688, 0, 0, 0, 0, 34, 0, 0, 27, 0, 205, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 0, 26, 0, 687, 0, 33, 113, 172, 0, 25, 9, 1954, 33, 4037, 0, 8, 0, 0, 2866, 329, 4096, 0, 2170, 577, 31, 4096, 112, 17, 0, 4088, 4096, 0, 3595, 4024, 4019, 0, 2, 9, 0, 0, 4089, 0, 1, 4078, 1, 0, 0, 0, 4072, 600, 11, 0, 0, 3688, 0, 1183, 79, 0, 2, 4096, 4096, 66, 2, 0, 0, 0, 0, 4027, 238, 4066, 4096, 4057, 0, 927, 1, 0, 130, 0, 0, 46, 746, 307, 0, 380, 0, 3116, 3777, 0, 4096, 328, 3997, 386, 0, 3, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[539, 5, 1819, 4024, 4095, 0, 0, 322, 650, 69, 19, 6, 4071, 8, 0, 0, 0, 1, 0, 0, 3635, 0, 3835, 0, 0, 2785, 1916, 0, 0, 0, 3722, 965, 0, 4, 0, 5, 2521, 0, 6, 4, 3771, 186, 2, 1249, 0, 1, 4084, 363, 4, 0, 3886, 163, 730, 5, 30, 0, 2, 4096, 1, 22, 5, 2200, 4096, 0, 464, 369, 3081, 0, 149, 912, 0, 0, 3978, 0, 3, 4083, 7, 0, 0, 0, 4054, 0, 1, 0, 0, 17, 65, 0, 4095, 0, 9, 2563, 114, 0, 2, 0, 0, 0, 4029, 0, 0, 4046, 4092, 4091, 3867, 112, 2252, 10, 2969, 0, 0, 0, 598, 2, 0, 4033, 0, 4064, 4096, 468, 4096, 82, 831, 426, 0, 2, 0, 3018] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 17, 4077, 0, 0, 0, 0, 0, 0, 0, 4088, 3997, 0, 0, 0, 1, 0, 0, 1, 0, 1, 16, 0, 388, 4001, 0, 4086, 0, 3, 3150, 0, 0, 2, 0, 3943, 3858, 1, 0, 148, 0, 0, 1183, 135, 3432, 3953, 53, 0, 0, 0, 2311, 648, 1, 0, 0, 4076, 0, 4096, 20, 0, 0, 4096, 2549, 0, 0, 1, 4083, 17, 23, 0, 244, 3109, 4096, 4, 0, 0, 0, 3480, 4003, 0, 2944, 4096, 0, 2886, 4096, 4096, 0, 4096, 4, 0, 0, 2, 3989, 13, 0, 0, 0, 89, 0, 0, 1, 128, 4096, 4095, 3184, 3961, 7, 0, 219, 4, 3877, 0, 0, 9, 0, 0, 3, 2981, 0, 0, 1507, 0, 91, 2016, 0, 0, 1191, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4, 0, 30, 245, 0, 0, 0, 0, 3566, 2320, 0, 0, 0, 0, 0, 0, 1, 0, 0, 110, 0, 0, 4096, 0, 161, 4096, 0, 2460, 0, 38, 108, 1531, 0, 1, 0, 0, 3288, 1, 7, 1, 18, 0, 0, 27, 1086, 3010, 3245, 1, 1, 0, 3967, 0, 3980, 0, 0, 44, 0, 2458, 29, 0, 0, 4096, 4095, 1, 4067, 4094, 4072, 1, 4054, 0, 0, 1, 4096, 19, 0, 0, 26, 0, 0, 0, 51, 4096, 3645, 0, 7, 2239, 0, 60, 4, 0, 20, 4088, 4068, 25, 0, 210, 0, 1734, 3900, 538, 4085, 762, 4096, 3438, 26, 3322, 4069, 141, 1, 37, 1842, 604, 941, 144, 0, 0, 1, 3774, 0, 0, 2848, 0, 1, 2272, 4095, 100, 1236, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 42, 61, 3900, 0, 0, 1209, 43, 0, 0, 4092, 0, 2518, 0, 0, 3, 0, 0, 1, 0, 0, 477, 6, 1, 2033, 0, 91, 0, 2, 4062, 4, 0, 4095, 0, 1, 7, 110, 1701, 185, 46, 0, 7, 959, 0, 1669, 124, 0, 1, 0, 1, 0, 619, 0, 4054, 3271, 370, 4093, 1779, 0, 0, 26, 4094, 0, 3080, 1, 37, 6, 0, 3922, 152, 2, 4096, 14, 3968, 2730, 3, 0, 76, 0, 4033, 4096, 0, 0, 4044, 3849, 0, 37, 988, 0, 0, 4066, 3197, 2, 0, 0, 0, 0, 0, 4089, 2710, 4096, 4096, 4095, 2, 3, 4026, 0, 3, 0, 4045, 18, 0, 3, 0, 222, 0, 3801, 4054, 0, 3352, 0, 3982, 0, 104, 212, 2, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 3804, 3913, 2802, 469, 0, 0, 0, 0, 77, 0, 3220, 4081, 0, 0, 0, 67, 3600, 0, 0, 4062, 0, 4089, 16, 78, 4096, 214, 0, 0, 0, 2, 0, 0, 0, 0, 3272, 35, 0, 0, 4, 214, 0, 3944, 1, 156, 355, 0, 2271, 0, 445, 4096, 0, 1, 303, 2537, 36, 878, 4033, 0, 222, 0, 3862, 3720, 84, 0, 120, 0, 2, 2163, 0, 25, 0, 4087, 3503, 0, 991, 0, 0, 0, 0, 3701, 0, 891, 0, 0, 5, 0, 3939, 4011, 0, 18, 2201, 3779, 0, 29, 0, 0, 0, 387, 24, 3974, 2599, 1856, 4076, 1, 2, 4, 3620, 2890, 270, 1, 13, 2610, 43, 0, 0, 54, 288, 4096, 0, 4034, 0, 4096, 4, 1393, 0, 16, 197] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 41, 0, 3778, 0, 0, 48, 0, 4096, 0, 113, 2820, 0, 193, 0, 0, 4096, 348, 0, 165, 0, 398, 4096, 0, 1, 0, 0, 0, 0, 0, 3849, 1698, 0, 3094, 3, 3815, 4006, 0, 0, 634, 0, 51, 0, 88, 573, 0, 4095, 3890, 0, 0, 3618, 3829, 0, 0, 4037, 3313, 201, 3263, 0, 408, 787, 1435, 62, 795, 4002, 3355, 4065, 3749, 39, 69, 0, 0, 4042, 0, 0, 0, 0, 3831, 0, 0, 1041, 4, 219, 3582, 219, 0, 424, 3723, 2044, 0, 0, 3506, 0, 1050, 420, 12, 0, 0, 0, 28, 46, 0, 4072, 49, 2, 241, 0, 11, 1639, 3516, 0, 0, 0, 0, 408, 0, 3797, 0, 590, 0, 4080, 0, 19, 582, 3909, 0, 0, 950] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 3211, 591, 0, 4096, 0, 0, 0, 2150, 0, 3, 3568, 2195, 35, 13, 660, 0, 2, 0, 449, 3, 232, 2496, 10, 0, 4094, 0, 3975, 0, 3, 3713, 53, 36, 68, 11, 21, 10, 0, 0, 0, 0, 0, 1619, 0, 4, 916, 4037, 0, 1182, 200, 24, 138, 1859, 0, 4095, 4, 0, 3814, 270, 0, 3, 4011, 4091, 0, 1091, 2760, 1026, 0, 10, 0, 0, 0, 4063, 0, 0, 0, 2470, 0, 505, 0, 2387, 4064, 0, 248, 4094, 1270, 38, 609, 3815, 0, 400, 4083, 4091, 34, 17, 3, 0, 0, 0, 0, 4095, 4089, 4095, 4014, 286, 0, 592, 0, 29, 0, 3872, 0, 0, 34, 4, 3540, 0, 7, 4096, 5, 2902, 0, 3509, 752, 108, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3977, 216, 2450, 3, 0, 0, 1, 4087, 2197, 0, 3997, 3688, 0, 0, 0, 1502, 1063, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 27, 0, 205, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 2805, 1653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 687, 0, 33, 113, 172, 0, 25, 9, 1954, 33, 4037, 0, 8, 4096, 1291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2866, 329, 4096, 0, 2170, 577, 31, 941, 112, 17, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3595, 4024, 4019, 0, 2, 9, 0, 0, 3420, 0, 1, 4078, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 600, 11, 0, 0, 3688, 0, 1183, 79, 0, 2, 0, 3718, 66, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 238, 4066, 0, 2994, 0, 927, 1, 0, 130, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 746, 307, 0, 380, 0, 3116, 3777, 0, 4096, 328, 3997, 386, 0, 3, 0, 1, 627, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23223, 12901, 12484, 19323, 19149, 13421, 12429, 18142] avg=16384.0000 max=23223 max_rank=0 min=12429 min_rank=6 imbalance_ratio=1.417419 max_load_reduction=107849 max_load_reduction_pct=0.822823 moved_to_echo=63141 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3804, 3913, 2802, 469, 0, 0, 0, 0, 77, 0, 3220, 4081, 0, 0, 0, 1479, 1068, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 3600, 0, 0, 4062, 0, 4089, 16, 78, 4096, 214, 0, 0, 0, 2, 0, 1506, 1628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3272, 35, 0, 0, 4, 214, 0, 3944, 1, 156, 355, 0, 2271, 1856, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 4096, 0, 1, 303, 2537, 36, 878, 926, 0, 222, 0, 3862, 3720, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 2, 2163, 0, 25, 0, 3418, 3503, 0, 991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3701, 0, 891, 0, 0, 5, 0, 3939, 4011, 0, 18, 0, 3430, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 24, 3974, 2599, 0, 3008, 1, 2, 4, 3620, 2890, 270, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2610, 43, 0, 0, 54, 288, 4096, 0, 4034, 0, 4096, 4, 1393, 0, 16, 197, 627, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20955, 19358, 12803, 17110, 10222, 16024, 16793, 17807] avg=16384.0000 max=20955 max_rank=0 min=10222 min_rank=4 imbalance_ratio=1.278992 max_load_reduction=110117 max_load_reduction_pct=0.840126 moved_to_echo=60846 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[539, 5, 1819, 4024, 4095, 0, 0, 322, 650, 69, 19, 6, 4071, 8, 0, 0, 1502, 1072, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3635, 0, 3835, 0, 0, 2785, 1916, 0, 0, 0, 3722, 965, 1754, 1653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 2521, 0, 6, 4, 3771, 186, 2, 1249, 0, 1, 4084, 363, 4092, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3886, 163, 730, 5, 30, 0, 2, 941, 1, 22, 5, 2200, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 369, 3081, 0, 149, 912, 0, 0, 3327, 0, 3, 4083, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 1, 0, 0, 17, 65, 0, 4095, 0, 9, 0, 104, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 4046, 0, 3019, 3867, 112, 2252, 10, 2969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 2, 0, 4033, 0, 4064, 4096, 468, 4096, 82, 831, 426, 0, 2, 0, 3018, 610, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18242, 20266, 17097, 12085, 12395, 8347, 20304, 22336] avg=16384.0000 max=22336 max_rank=7 min=8347 min_rank=5 imbalance_ratio=1.363281 max_load_reduction=108736 max_load_reduction_pct=0.829590 moved_to_echo=63382 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 42, 61, 3900, 0, 0, 1209, 43, 0, 0, 4092, 0, 2518, 0, 0, 1501, 1073, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 477, 6, 1, 2033, 0, 91, 0, 2, 4062, 4, 2783, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 7, 110, 1701, 185, 46, 0, 7, 959, 0, 1669, 124, 0, 4096, 1283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 619, 0, 4054, 3271, 370, 940, 1779, 0, 0, 26, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3080, 1, 37, 6, 0, 3922, 152, 2, 3426, 14, 3968, 2730, 3, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 4096, 0, 0, 4044, 3849, 0, 37, 988, 0, 0, 0, 2902, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 2710, 4096, 0, 3022, 2, 3, 4026, 0, 3, 0, 4045, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 222, 0, 3801, 4054, 0, 3352, 0, 3982, 0, 104, 212, 2, 1, 628, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14481, 11115, 14283, 15155, 17417, 19951, 22014, 16656] avg=16384.0000 max=22014 max_rank=6 min=11115 min_rank=1 imbalance_ratio=1.343628 max_load_reduction=109058 max_load_reduction_pct=0.832047 moved_to_echo=76038 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 17, 4077, 0, 0, 0, 0, 0, 0, 0, 4088, 3997, 0, 0, 0, 1506, 1076, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 16, 0, 388, 4001, 0, 4086, 0, 3, 3150, 0, 2, 1658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3943, 3858, 1, 0, 148, 0, 0, 1183, 135, 3432, 3953, 53, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2311, 648, 1, 0, 0, 4076, 0, 932, 20, 0, 0, 4096, 2549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4083, 17, 23, 0, 244, 3109, 3415, 4, 0, 0, 0, 3480, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2944, 4096, 0, 2886, 4096, 4096, 0, 4096, 4, 0, 0, 0, 3617, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 1, 128, 0, 3019, 3184, 3961, 7, 0, 219, 4, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3, 2981, 0, 0, 1507, 0, 91, 2016, 0, 0, 1191, 0, 634, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14808, 13307, 20804, 14633, 18379, 25848, 14489, 8804] avg=16384.0000 max=25848 max_rank=5 min=8804 min_rank=7 imbalance_ratio=1.577637 max_load_reduction=105224 max_load_reduction_pct=0.802795 moved_to_echo=67520 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 30, 245, 0, 0, 0, 0, 3566, 2320, 0, 0, 0, 0, 0, 0, 901, 901, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 110, 0, 0, 4096, 0, 161, 4096, 0, 2460, 0, 38, 108, 1531, 2799, 992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3288, 1, 7, 1, 18, 0, 0, 27, 1086, 3010, 3245, 1, 4096, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3967, 0, 3980, 0, 0, 44, 0, 565, 29, 0, 0, 4096, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4094, 4072, 1, 4054, 0, 0, 1, 3426, 19, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 4096, 3645, 0, 7, 2239, 0, 60, 4, 0, 20, 0, 3693, 25, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1734, 3900, 538, 4085, 762, 0, 2537, 26, 3322, 4069, 141, 1, 37, 1842, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 941, 144, 0, 0, 1, 3774, 0, 0, 2848, 0, 1, 2272, 4095, 100, 1236, 0, 628, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8009, 16392, 16070, 16778, 19760, 14050, 23598, 16415] avg=16384.0000 max=23598 max_rank=6 min=8009 min_rank=0 imbalance_ratio=1.440308 max_load_reduction=107474 max_load_reduction_pct=0.819962 moved_to_echo=73823 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 41, 0, 3778, 0, 0, 48, 0, 4096, 0, 113, 2820, 0, 193, 0, 0, 1196, 12, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 348, 0, 165, 0, 398, 4096, 0, 1, 0, 0, 0, 0, 0, 3849, 1698, 2401, 1317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3094, 3, 3815, 4006, 0, 0, 634, 0, 51, 0, 88, 573, 0, 4095, 3890, 4072, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3618, 3829, 0, 0, 4037, 3313, 201, 750, 0, 408, 787, 1435, 62, 795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 3355, 4065, 3749, 39, 69, 0, 0, 3380, 0, 0, 0, 0, 3831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 4, 219, 3582, 219, 0, 424, 3723, 2044, 0, 0, 0, 0, 1050, 420, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 46, 0, 0, 37, 2, 241, 0, 11, 1639, 3516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 0, 3797, 0, 590, 0, 4080, 0, 19, 582, 3909, 0, 0, 950, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12339, 18369, 25426, 19235, 22490, 12738, 5520, 14955] avg=16384.0000 max=25426 max_rank=2 min=5520 min_rank=6 imbalance_ratio=1.551880 max_load_reduction=105646 max_load_reduction_pct=0.806015 moved_to_echo=55703 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3211, 591, 0, 4096, 0, 0, 0, 2150, 0, 3, 3568, 2195, 35, 13, 660, 1399, 1052, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 449, 3, 232, 2496, 10, 0, 4094, 0, 3975, 0, 3, 3713, 53, 2795, 1539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 68, 11, 21, 10, 0, 0, 0, 0, 0, 1619, 0, 4, 916, 4037, 0, 4095, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1182, 200, 24, 138, 1859, 0, 4095, 4, 0, 876, 270, 0, 3, 4011, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 2760, 1026, 0, 10, 0, 0, 0, 3398, 0, 0, 0, 2470, 0, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2387, 4064, 0, 248, 4094, 1270, 38, 609, 3815, 0, 400, 0, 3714, 34, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4089, 0, 2962, 286, 0, 592, 0, 29, 0, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4, 3540, 0, 7, 4096, 5, 2902, 0, 3509, 752, 108, 0, 0, 0, 623, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19015, 19364, 12105, 16753, 11260, 20693, 15925, 15957] avg=16384.0000 max=20693 max_rank=5 min=11260 min_rank=4 imbalance_ratio=1.263000 max_load_reduction=110379 max_load_reduction_pct=0.842125 moved_to_echo=63835 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[640, 11136, 6656, 17536, 12672, 0, 128, 1536, 14592, 4736, 256, 21888, 18048, 2816, 128, 768, 11008, 7424, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[22400, 17024, 4864, 6784, 12544, 15232, 640, 13696, 15104, 0, 512, 0, 21248, 1280, 512, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1280, 768, 20864, 5120, 11392, 384, 17024, 11392, 1536, 6912, 2304, 768, 896, 23936, 26880, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[4992, 640, 512, 8192, 3968, 18048, 20736, 512, 27008, 512, 16640, 6528, 9728, 384, 2560, 4224, 5120, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[16384, 14848, 20480, 3840, 6528, 4992, 512, 3200, 27264, 3584, 4096, 11904, 2560, 7424, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 1920, 8320, 8832, 15232, 19840, 0, 20608, 7424, 8576, 11008, 3840, 7936, 3840, 13696, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 7296, 128, 11776, 13824, 256, 1920, 1152, 4096, 384, 6784, 4480, 5376, 13952, 15744, 6656, 30592, 7808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[4224, 4096, 0, 768, 7808, 640, 19328, 128, 640, 25216, 2176, 10624, 0, 128, 22784, 4352, 16896, 12160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 537, 21, 1, 3622, 0, 0, 3690, 0, 2334, 0, 0, 0, 0, 0, 4056, 0, 0, 84, 0, 11, 0, 0, 54, 2374, 27, 493, 0, 0, 589, 0, 0, 305, 17, 2110, 0, 3359, 4030, 0, 0, 0, 4050, 1498, 3, 8, 42, 4096, 217, 187, 1327, 4096, 0, 3429, 130, 0, 129, 4096, 0, 439, 0, 4, 7, 4093, 0, 100, 0, 0, 0, 0, 0, 4096, 4083, 1864, 4042, 4089, 0, 0, 0, 1, 0, 4085, 1, 3237, 11, 2, 3808, 3855, 0, 0, 0, 19, 0, 0, 657, 4095, 0, 0, 0, 3899, 0, 0, 3805, 0, 4096, 2304, 0, 0, 4085, 0, 0, 3851, 6, 561, 4089, 52, 0, 0, 0, 0, 237, 0, 0, 0, 2945, 3436, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3656, 0, 1, 1379, 4096, 73, 0, 1238, 0, 26, 3977, 0, 99, 3, 15, 0, 826, 25, 955, 0, 399, 0, 0, 4096, 0, 24, 4091, 0, 314, 259, 736, 3, 230, 0, 471, 0, 269, 3992, 3686, 4089, 0, 0, 0, 38, 1, 0, 1, 0, 0, 3865, 25, 1012, 3257, 4043, 3975, 0, 14, 3886, 0, 2847, 9, 0, 136, 2218, 270, 3569, 3506, 49, 0, 0, 1541, 265, 10, 171, 0, 0, 3, 0, 0, 3804, 444, 4096, 3135, 4027, 513, 116, 2104, 0, 0, 0, 189, 0, 25, 702, 0, 0, 0, 1, 9, 7, 8, 4050, 0, 670, 0, 3946, 1, 0, 3881, 0, 0, 342, 0, 4076, 3802, 3669, 0, 2581, 0, 4092, 0, 0, 0, 4096, 0, 2947, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[2323, 0, 255, 3950, 2775, 1049, 0, 0, 150, 47, 3458, 3955, 0, 0, 0, 3151, 4095, 0, 0, 148, 0, 0, 4096, 49, 501, 0, 0, 1, 141, 31, 13, 1067, 4, 0, 0, 3, 0, 4088, 0, 0, 0, 2921, 2117, 3499, 0, 0, 0, 0, 0, 29, 8, 4079, 0, 289, 31, 1, 25, 2799, 0, 559, 140, 0, 4061, 4096, 2626, 1828, 4096, 0, 0, 328, 34, 4050, 2, 612, 1, 4080, 4074, 0, 0, 0, 0, 6, 1457, 8, 117, 0, 104, 46, 6, 3, 0, 25, 0, 0, 175, 4025, 43, 0, 0, 1599, 142, 0, 4077, 0, 3234, 4096, 1713, 0, 4096, 0, 0, 511, 1251, 800, 51, 1912, 0, 8, 4015, 1502, 3796, 0, 270, 3479, 3625, 0, 0, 3145] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 4018, 1585, 1546, 0, 31, 54, 134, 0, 4061, 0, 0, 0, 0, 0, 4094, 0, 273, 99, 0, 37, 3216, 1124, 3, 4095, 3875, 0, 1161, 0, 4, 2, 0, 10, 8, 4020, 0, 0, 3288, 81, 0, 4090, 4091, 3724, 5, 3, 2, 1070, 1161, 0, 638, 2957, 0, 4087, 2, 0, 894, 3445, 0, 0, 160, 7, 4, 3733, 2153, 114, 0, 0, 0, 0, 0, 4096, 19, 273, 4096, 1, 4094, 0, 0, 36, 583, 4075, 0, 127, 3667, 1, 0, 1736, 0, 0, 0, 6, 0, 0, 232, 2567, 235, 0, 1368, 2799, 0, 59, 4095, 0, 40, 1584, 0, 0, 3028, 0, 3746, 0, 3, 3617, 3437, 3900, 0, 0, 2, 3548, 0, 0, 0, 0, 0, 0, 78, 669] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3728, 5, 0, 0, 2719, 64, 0, 3567, 0, 39, 4091, 0, 0, 4, 3, 0, 240, 0, 0, 54, 1, 0, 2140, 310, 0, 1992, 0, 381, 0, 0, 0, 102, 0, 0, 363, 279, 0, 0, 0, 1272, 0, 3797, 3632, 1604, 0, 108, 2334, 4075, 1375, 4, 161, 0, 0, 3896, 0, 0, 4065, 4080, 0, 0, 0, 0, 187, 4096, 4052, 3689, 0, 0, 67, 0, 0, 3915, 4096, 4, 4021, 1, 14, 1, 147, 6, 0, 3721, 13, 430, 3893, 0, 2875, 3632, 0, 0, 0, 1, 3211, 0, 0, 897, 2, 92, 0, 3205, 0, 4096, 4016, 69, 3943, 633, 0, 0, 4096, 0, 0, 4073, 0, 1724, 4092, 0, 0, 7, 0, 0, 4092, 91, 0, 9, 3264, 114, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[3824, 0, 3667, 166, 2864, 2, 0, 0, 3984, 4094, 2311, 3119, 2, 1, 0, 410, 4006, 0, 0, 0, 0, 0, 1286, 0, 0, 0, 0, 121, 3870, 0, 4, 112, 103, 194, 0, 719, 0, 0, 4085, 0, 3227, 7, 3959, 1895, 0, 0, 43, 0, 3, 1163, 14, 3764, 0, 4054, 4083, 1, 0, 2, 1, 3, 0, 0, 0, 4096, 32, 4096, 2, 0, 0, 4, 0, 83, 0, 0, 3931, 3805, 2846, 0, 0, 2921, 0, 313, 0, 25, 0, 0, 4032, 4094, 4, 2, 0, 0, 0, 0, 21, 10, 1544, 0, 0, 4095, 119, 0, 4095, 357, 7, 3593, 2, 0, 4096, 1323, 3963, 0, 3485, 4054, 6, 12, 162, 6, 0, 4096, 3636, 0, 0, 0, 941, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[67, 0, 0, 0, 3926, 1, 0, 0, 0, 0, 3986, 0, 8, 0, 0, 682, 4095, 0, 4, 4025, 0, 0, 142, 0, 66, 0, 0, 3, 0, 0, 18, 0, 1, 4077, 0, 4096, 0, 3907, 8, 0, 4093, 4094, 0, 3146, 0, 3020, 14, 3, 78, 3254, 569, 4090, 0, 1519, 4096, 2, 3396, 3878, 0, 424, 0, 0, 0, 4096, 11, 4077, 1127, 537, 0, 0, 0, 1320, 0, 0, 1, 2556, 1, 0, 0, 54, 0, 542, 839, 16, 3261, 0, 106, 4081, 98, 1, 0, 0, 3, 0, 6, 4096, 1, 0, 0, 4094, 649, 0, 4095, 0, 160, 2, 7, 0, 4096, 45, 4074, 104, 2179, 0, 3287, 3979, 678, 3, 0, 4000, 813, 0, 4095, 0, 1, 3092, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 1392, 1306, 120, 0, 39, 218, 0, 0, 3802, 0, 713, 60, 8, 0, 2108, 28, 2, 80, 0, 24, 2561, 53, 641, 3096, 2166, 1626, 31, 632, 71, 0, 205, 0, 0, 23, 31, 651, 4096, 1331, 0, 0, 79, 4093, 0, 4061, 0, 0, 477, 25, 0, 3271, 0, 2743, 25, 174, 166, 4054, 0, 0, 2, 0, 0, 4092, 0, 3777, 0, 0, 0, 0, 3, 4096, 55, 0, 0, 0, 36, 0, 356, 0, 0, 3943, 3508, 4076, 0, 8, 3998, 4096, 0, 85, 6, 0, 0, 38, 98, 2620, 2, 0, 60, 2726, 0, 4096, 0, 0, 3596, 3774, 109, 0, 4066, 0, 12, 84, 4000, 4092, 3938, 2736, 0, 584, 0, 3545, 2786, 7, 0, 0, 2562, 2174, 17, 835] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 537, 21, 1, 3622, 0, 0, 3690, 0, 2334, 0, 0, 0, 0, 0, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 84, 0, 11, 0, 0, 54, 2374, 27, 493, 0, 0, 589, 0, 3068, 4089, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 17, 2110, 0, 3359, 4030, 0, 0, 0, 4050, 963, 3, 8, 42, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 187, 1327, 4096, 0, 3429, 130, 0, 129, 4096, 0, 439, 0, 4, 7, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 4096, 4083, 1864, 4042, 4089, 0, 0, 0, 1, 757, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 3237, 11, 2, 3808, 3855, 0, 0, 0, 19, 0, 0, 657, 4095, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3899, 0, 0, 3805, 0, 4096, 2304, 0, 0, 1017, 0, 0, 3851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 561, 0, 52, 0, 0, 0, 0, 237, 0, 0, 0, 2945, 3436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14713, 14995, 18983, 15332, 19155, 21685, 18972, 7237] avg=16384.0000 max=21685 max_rank=5 min=7237 min_rank=7 imbalance_ratio=1.323547 max_load_reduction=109387 max_load_reduction_pct=0.834557 moved_to_echo=67049 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2323, 0, 255, 3950, 2775, 1049, 0, 0, 150, 47, 3458, 3955, 0, 0, 0, 3151, 962, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 148, 0, 0, 4096, 49, 501, 0, 0, 1, 141, 31, 13, 1067, 3076, 51, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3, 0, 4088, 0, 0, 0, 2921, 2117, 2249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 8, 4079, 0, 289, 31, 1, 25, 2799, 0, 559, 140, 0, 4061, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2626, 1828, 4096, 0, 0, 328, 34, 4050, 2, 612, 1, 4080, 4074, 0, 0, 0, 758, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1457, 8, 117, 0, 104, 46, 6, 3, 0, 25, 0, 0, 175, 4025, 1917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 1599, 142, 0, 4077, 0, 3234, 4096, 1713, 0, 1020, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1251, 800, 0, 1912, 0, 8, 4015, 1422, 3796, 0, 270, 3479, 3625, 0, 0, 3145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22155, 13419, 11382, 13292, 22777, 7889, 16435, 23723] avg=16384.0000 max=23723 max_rank=7 min=7889 min_rank=5 imbalance_ratio=1.447937 max_load_reduction=107349 max_load_reduction_pct=0.819008 moved_to_echo=70824 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3728, 5, 0, 0, 2719, 64, 0, 3567, 0, 39, 4091, 0, 0, 4, 3, 0, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 54, 1, 0, 2140, 310, 0, 1992, 0, 381, 0, 0, 0, 102, 3076, 4092, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 279, 0, 0, 0, 1272, 0, 3797, 3632, 1031, 0, 108, 2334, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1375, 4, 161, 0, 0, 3896, 0, 0, 4065, 4080, 0, 0, 0, 0, 187, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 3689, 0, 0, 67, 0, 0, 3915, 4096, 4, 4021, 1, 14, 1, 147, 6, 758, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3721, 13, 430, 3893, 0, 2875, 3632, 0, 0, 0, 1, 3211, 0, 0, 897, 1917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 92, 0, 3205, 0, 4096, 4016, 69, 3943, 633, 0, 0, 1020, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1724, 0, 0, 0, 7, 0, 0, 4092, 91, 0, 9, 3264, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14661, 12538, 16891, 15039, 20903, 20590, 21149, 9301] avg=16384.0000 max=21149 max_rank=6 min=9301 min_rank=7 imbalance_ratio=1.290833 max_load_reduction=109923 max_load_reduction_pct=0.838646 moved_to_echo=71943 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4018, 1585, 1546, 0, 31, 54, 134, 0, 4061, 0, 0, 0, 0, 0, 1024, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 273, 99, 0, 37, 3216, 1124, 3, 4095, 3875, 0, 1161, 0, 4, 2, 2274, 3437, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8, 4020, 0, 0, 3288, 81, 0, 4090, 4091, 2393, 5, 3, 2, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1161, 0, 638, 2957, 0, 4087, 2, 0, 894, 3445, 0, 0, 160, 7, 4, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2153, 114, 0, 0, 0, 0, 0, 4096, 19, 273, 4096, 1, 4094, 0, 0, 36, 690, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 4075, 0, 127, 3667, 1, 0, 1736, 0, 0, 0, 6, 0, 0, 232, 2567, 1747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 1368, 2799, 0, 59, 4095, 0, 40, 1584, 0, 0, 754, 0, 3746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3617, 0, 3900, 0, 0, 2, 3358, 0, 0, 0, 0, 0, 0, 78, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16739, 23830, 19061, 14515, 15879, 14741, 14680, 11627] avg=16384.0000 max=23830 max_rank=1 min=11627 min_rank=7 imbalance_ratio=1.454468 max_load_reduction=107242 max_load_reduction_pct=0.818192 moved_to_echo=56927 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3656, 0, 1, 1379, 4096, 73, 0, 1238, 0, 26, 3977, 0, 99, 3, 15, 0, 10, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 25, 955, 0, 399, 0, 0, 4096, 0, 24, 4091, 0, 314, 259, 736, 3, 2915, 3802, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 471, 0, 269, 3992, 3686, 4089, 0, 0, 0, 25, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 25, 1012, 3257, 4043, 3975, 0, 14, 3886, 0, 2847, 9, 0, 136, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 3569, 3506, 49, 0, 0, 1541, 265, 10, 171, 0, 0, 3, 0, 0, 3804, 410, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 4096, 3135, 4027, 513, 116, 2104, 0, 0, 0, 189, 0, 25, 702, 0, 0, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 7, 8, 4050, 0, 670, 0, 3946, 1, 0, 966, 0, 0, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 3669, 0, 2581, 0, 3872, 0, 0, 0, 4096, 0, 2947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14793, 18526, 12764, 23758, 13601, 16389, 10000, 21241] avg=16384.0000 max=23758 max_rank=3 min=10000 min_rank=6 imbalance_ratio=1.450073 max_load_reduction=107314 max_load_reduction_pct=0.818741 moved_to_echo=61231 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3824, 0, 3667, 166, 2864, 2, 0, 0, 3984, 4094, 2311, 3119, 2, 1, 0, 410, 521, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 0, 0, 1286, 0, 0, 0, 0, 121, 3870, 0, 4, 112, 3076, 6, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 194, 0, 719, 0, 0, 4085, 0, 3227, 7, 3959, 1218, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1163, 14, 3764, 0, 4054, 4083, 1, 0, 2, 1, 3, 0, 0, 0, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4096, 2, 0, 0, 4, 0, 83, 0, 0, 3931, 3805, 2846, 0, 0, 2921, 758, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 0, 25, 0, 0, 4032, 4094, 4, 2, 0, 0, 0, 0, 21, 10, 1917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1544, 0, 0, 4095, 119, 0, 4095, 357, 7, 3593, 2, 0, 1020, 1323, 3963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3485, 4054, 0, 12, 162, 6, 0, 3876, 3636, 0, 0, 0, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[25185, 12631, 13555, 14359, 18634, 10418, 20118, 16172] avg=16384.0000 max=25185 max_rank=0 min=10418 min_rank=5 imbalance_ratio=1.537170 max_load_reduction=105887 max_load_reduction_pct=0.807854 moved_to_echo=65342 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[67, 0, 0, 0, 3926, 1, 0, 0, 0, 0, 3986, 0, 8, 0, 0, 682, 868, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 4025, 0, 0, 142, 0, 66, 0, 0, 3, 0, 0, 18, 0, 3080, 3287, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4077, 0, 4096, 0, 3907, 8, 0, 4093, 4094, 0, 2016, 0, 3020, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 3254, 569, 4090, 0, 1519, 4096, 2, 3396, 3878, 0, 424, 0, 0, 0, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4077, 1127, 537, 0, 0, 0, 1320, 0, 0, 1, 2556, 1, 0, 0, 54, 760, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 839, 16, 3261, 0, 106, 4081, 98, 1, 0, 0, 3, 0, 6, 4096, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 649, 0, 4095, 0, 160, 2, 7, 0, 1016, 45, 4074, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2179, 0, 0, 3979, 678, 3, 0, 3782, 813, 0, 4095, 0, 1, 3092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9756, 14872, 25329, 22570, 10706, 14969, 14247, 18623] avg=16384.0000 max=25329 max_rank=2 min=9756 min_rank=0 imbalance_ratio=1.545959 max_load_reduction=105743 max_load_reduction_pct=0.806755 moved_to_echo=58545 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 1392, 1306, 120, 0, 39, 218, 0, 0, 3802, 0, 713, 60, 8, 0, 1126, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2108, 28, 2, 80, 0, 24, 2561, 53, 641, 3096, 2166, 1626, 31, 632, 71, 0, 3054, 3938, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 23, 31, 651, 4096, 1331, 0, 0, 79, 2630, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 25, 0, 3271, 0, 2743, 25, 174, 166, 4054, 0, 0, 2, 0, 0, 1270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3777, 0, 0, 0, 0, 3, 4096, 55, 0, 0, 0, 36, 0, 356, 0, 757, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 3508, 4076, 0, 8, 3998, 4096, 0, 85, 6, 0, 0, 38, 98, 2620, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 60, 2726, 0, 4096, 0, 0, 3596, 3774, 109, 0, 1012, 0, 12, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 4092, 0, 2736, 0, 584, 0, 3355, 2786, 7, 0, 0, 2562, 2174, 17, 835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13070, 20261, 13107, 12207, 9417, 24391, 15471, 23148] avg=16384.0000 max=24391 max_rank=5 min=9417 min_rank=4 imbalance_ratio=1.488708 max_load_reduction=106681 max_load_reduction_pct=0.813911 moved_to_echo=72427 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1920, 128, 1536, 22528, 1024, 12416, 24192, 1152, 15104, 19968, 1920, 0, 7936, 1408, 11904, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[9216, 21376, 8832, 640, 128, 384, 1664, 22016, 8320, 2944, 16128, 14592, 11136, 128, 512, 6912, 5760, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[25984, 128, 9984, 8448, 18048, 4864, 128, 5120, 8064, 4224, 28032, 7168, 896, 128, 128, 4352, 5376, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[3328, 8576, 2816, 23296, 3328, 24064, 12416, 256, 8704, 26240, 128, 4352, 384, 128, 4480, 9472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[11008, 18944, 0, 16384, 896, 3200, 4096, 19712, 15360, 128, 4480, 7680, 13440, 11776, 128, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[23552, 128, 1280, 4608, 512, 128, 13568, 5632, 1280, 11648, 10240, 2688, 5632, 1024, 1536, 1408, 23680, 22784, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1152, 20864, 8960, 12032, 11520, 128, 17152, 21632, 128, 128, 256, 128, 3328, 768, 1280, 18432, 14336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[640, 4608, 896, 11264, 384, 16000, 19200, 6784, 7424, 14976, 18048, 12544, 128, 7296, 2560, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1430, 558, 4, 535, 0, 4094, 0, 0, 3982, 2250, 27, 0, 1, 154, 0, 0, 26, 59, 4084, 3414, 0, 0, 0, 4096, 0, 0, 4096, 9, 135, 0, 0, 0, 0, 0, 4096, 0, 689, 0, 0, 39, 0, 3891, 0, 0, 0, 1058, 0, 4088, 0, 0, 237, 0, 4094, 0, 0, 2, 4073, 0, 0, 0, 0, 4095, 0, 0, 2200, 3864, 2384, 21, 2, 4077, 0, 0, 3973, 15, 1, 3890, 2922, 12, 220, 0, 5, 4, 4096, 3269, 0, 19, 0, 5, 0, 9, 51, 4090, 0, 2060, 1, 0, 0, 0, 0, 3922, 4065, 0, 3417, 0, 3912, 0, 0, 0, 1, 0, 4095, 0, 661, 2734, 4086, 182, 0, 1, 0, 0, 3996, 238, 0, 4096, 0, 0, 0, 3160] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1, 334, 1, 0, 140, 3983, 5, 0, 40, 0, 0, 3809, 0, 15, 40, 0, 0, 1, 2341, 4030, 0, 1492, 130, 3996, 26, 8, 4095, 52, 0, 0, 0, 4066, 455, 0, 3001, 0, 1440, 0, 4067, 3951, 0, 4095, 56, 0, 977, 193, 355, 257, 0, 119, 4056, 0, 4040, 0, 0, 1211, 3601, 21, 0, 0, 3202, 2992, 0, 10, 4096, 3976, 0, 15, 81, 4089, 0, 37, 3793, 0, 0, 2807, 128, 0, 1466, 102, 0, 1011, 0, 3759, 2, 1, 0, 1, 232, 0, 1, 4071, 0, 4061, 76, 0, 0, 0, 6, 2796, 181, 0, 0, 1, 1, 0, 384, 0, 0, 50, 4008, 0, 65, 4087, 3750, 0, 0, 0, 76, 0, 3971, 3048, 0, 4082, 0, 0, 1, 4056] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 15, 0, 0, 0, 0, 0, 1, 3754, 3, 424, 1, 1023, 0, 0, 2225, 0, 211, 1, 0, 3883, 308, 4096, 0, 0, 4096, 0, 68, 1154, 0, 4065, 0, 1, 4053, 31, 0, 0, 3085, 1151, 0, 0, 0, 0, 0, 27, 158, 3652, 4, 0, 53, 0, 2611, 0, 0, 169, 563, 0, 0, 98, 0, 4095, 0, 0, 17, 3998, 3789, 3, 111, 4095, 0, 3855, 2968, 10, 3653, 3096, 2, 3990, 10, 1, 0, 15, 2785, 2917, 4074, 3277, 88, 3, 0, 4, 64, 3553, 4, 54, 0, 13, 0, 0, 9, 3997, 4087, 0, 18, 3930, 456, 0, 4007, 0, 0, 0, 4096, 0, 0, 1044, 2194, 13, 0, 2464, 0, 0, 4067, 141, 0, 4096, 0, 0, 830, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[140, 3626, 0, 1, 5, 22, 2104, 0, 2404, 74, 157, 0, 1261, 0, 0, 0, 3512, 12, 180, 247, 1, 6, 11, 4093, 0, 0, 325, 0, 0, 4003, 23, 0, 0, 0, 4096, 3873, 0, 0, 3089, 4, 0, 127, 0, 0, 0, 3530, 0, 4084, 0, 0, 538, 0, 4096, 0, 0, 0, 4096, 2977, 417, 0, 0, 4095, 0, 0, 2337, 4033, 0, 0, 3922, 12, 0, 3619, 57, 4078, 0, 2, 0, 0, 4015, 2783, 0, 0, 3621, 558, 0, 3700, 851, 0, 0, 274, 0, 4096, 0, 3421, 210, 340, 0, 3, 493, 0, 2, 32, 4073, 4073, 2936, 24, 4096, 0, 866, 26, 3953, 14, 21, 0, 1369, 0, 0, 0, 0, 0, 2279, 3793, 0, 27, 3831, 0, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[335, 678, 2531, 2511, 13, 7, 0, 0, 3, 126, 4080, 11, 0, 60, 0, 0, 0, 0, 4078, 4088, 0, 331, 7, 3487, 0, 55, 4075, 1156, 0, 4096, 0, 0, 2, 0, 218, 0, 0, 0, 4072, 5, 719, 3808, 289, 0, 0, 4086, 0, 3953, 3980, 0, 0, 7, 3851, 0, 0, 0, 49, 0, 0, 1, 0, 4094, 0, 26, 0, 2242, 529, 0, 1034, 4040, 44, 0, 4096, 0, 0, 2341, 6, 2, 0, 0, 4025, 0, 1, 3503, 3730, 0, 0, 0, 0, 2357, 3562, 4094, 0, 0, 17, 100, 0, 189, 4068, 3857, 3119, 639, 0, 386, 4096, 12, 4096, 3, 1915, 4092, 0, 0, 2900, 0, 54, 0, 0, 0, 0, 0, 6, 1, 0, 4064, 0, 13, 0, 951] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4095, 4079, 0, 57, 11, 29, 0, 0, 4075, 4092, 844, 770, 315, 0, 0, 0, 0, 682, 3771, 4096, 3970, 19, 1, 4096, 0, 0, 4096, 3976, 0, 3888, 0, 0, 0, 0, 4096, 0, 0, 0, 22, 3393, 0, 4096, 0, 0, 0, 793, 221, 0, 2, 1, 29, 0, 17, 0, 0, 0, 524, 0, 0, 0, 0, 690, 0, 1, 1, 1868, 933, 3931, 0, 19, 602, 0, 4065, 402, 0, 571, 124, 0, 4093, 0, 3864, 0, 0, 4078, 2319, 0, 0, 0, 1, 129, 0, 4096, 0, 0, 100, 0, 0, 239, 1925, 3728, 1, 39, 4004, 133, 4096, 1527, 101, 3, 4021, 3647, 0, 0, 11, 0, 8, 3395, 2176, 0, 2895, 0, 4096, 2, 0, 2981, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 515, 2285, 9, 3441, 0, 3, 785, 4048, 1, 4, 7, 3589, 292, 0, 20, 0, 3782, 3070, 0, 0, 296, 1518, 0, 0, 4081, 0, 84, 0, 32, 4024, 10, 0, 4096, 0, 0, 0, 0, 389, 170, 4096, 332, 10, 3, 4096, 0, 2962, 6, 1, 4, 64, 4096, 2334, 1, 0, 4096, 0, 193, 4045, 15, 4074, 27, 1, 3797, 59, 539, 0, 3, 24, 0, 0, 4059, 3052, 0, 133, 0, 296, 0, 0, 0, 0, 0, 1957, 3583, 320, 0, 0, 0, 0, 1280, 0, 783, 0, 0, 968, 1, 871, 4093, 4084, 0, 0, 0, 2198, 1772, 0, 0, 55, 0, 15, 2334, 6, 3856, 4095, 3927, 0, 0, 0, 254, 6, 3651, 0, 0, 3997, 3920, 8, 0, 4069] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 0, 3667, 11, 3993, 827, 4, 0, 0, 0, 3871, 0, 3881, 4083, 0, 1, 174, 12, 0, 174, 1672, 57, 0, 0, 0, 2720, 9, 901, 0, 118, 1, 932, 52, 850, 0, 55, 0, 3941, 4088, 13, 4096, 35, 4096, 0, 3506, 0, 0, 0, 64, 0, 0, 2566, 12, 4094, 0, 4093, 0, 1422, 1549, 1, 0, 0, 55, 3838, 2356, 134, 0, 43, 0, 145, 438, 23, 0, 1, 0, 3743, 0, 3974, 0, 1, 0, 0, 3928, 0, 3717, 0, 0, 316, 5, 806, 2, 0, 3244, 0, 0, 30, 0, 4044, 0, 1613, 0, 4086, 3960, 4049, 465, 992, 86, 158, 4065, 0, 3293, 3464, 212, 0, 93, 4095, 0, 101, 0, 4002, 133, 6, 6, 0, 0, 0, 3708] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1430, 558, 4, 535, 0, 4094, 0, 0, 3982, 2250, 27, 0, 1, 154, 0, 0, 3973, 1329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 59, 4084, 3414, 0, 0, 0, 4096, 0, 0, 2564, 9, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 689, 0, 0, 39, 0, 3891, 0, 0, 0, 1058, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 4094, 0, 0, 2, 4073, 0, 0, 0, 0, 4095, 0, 0, 2593, 1532, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2200, 3499, 2384, 21, 2, 4077, 0, 0, 0, 15, 1, 3890, 2922, 12, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 4096, 3269, 0, 19, 0, 5, 0, 9, 51, 4090, 0, 2060, 1, 0, 1244, 665, 365, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 3417, 0, 2668, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 661, 2734, 4086, 182, 0, 1, 0, 0, 2672, 238, 0, 4096, 0, 0, 0, 3160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18337, 14387, 13828, 17285, 19243, 15916, 14246, 17830] avg=16384.0000 max=19243 max_rank=4 min=13828 min_rank=2 imbalance_ratio=1.174500 max_load_reduction=111829 max_load_reduction_pct=0.853188 moved_to_echo=67235 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 334, 1, 0, 140, 3983, 5, 0, 40, 0, 0, 3809, 0, 15, 40, 0, 3793, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2341, 4030, 0, 1492, 130, 3996, 26, 8, 2564, 52, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, 2977, 0, 1440, 0, 4067, 3951, 0, 4095, 56, 0, 977, 193, 355, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 4056, 0, 4040, 0, 0, 1211, 3601, 21, 0, 0, 3202, 2992, 0, 10, 1850, 1531, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3600, 0, 15, 81, 4089, 0, 37, 0, 0, 0, 2807, 128, 0, 1466, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 0, 3759, 2, 1, 0, 1, 232, 0, 1, 4071, 0, 4061, 76, 0, 0, 661, 376, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 181, 0, 0, 1, 1, 0, 384, 0, 0, 50, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4087, 3750, 0, 0, 0, 76, 0, 2655, 3048, 0, 4082, 0, 0, 1, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13107, 18706, 18823, 23288, 16421, 14276, 4631, 21820] avg=16384.0000 max=23288 max_rank=3 min=4631 min_rank=6 imbalance_ratio=1.421387 max_load_reduction=107784 max_load_reduction_pct=0.822327 moved_to_echo=57148 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 15, 0, 0, 0, 0, 0, 1, 3754, 3, 424, 1, 1023, 0, 0, 2968, 1353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2225, 0, 211, 1, 0, 3883, 308, 4096, 0, 0, 2564, 0, 68, 1154, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4021, 31, 0, 0, 3085, 1151, 0, 0, 0, 0, 0, 27, 158, 3652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 53, 0, 2611, 0, 0, 169, 563, 0, 0, 98, 0, 4095, 0, 0, 2644, 1532, 671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3620, 3789, 3, 111, 4095, 0, 3855, 0, 10, 3653, 3096, 2, 3990, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2785, 2917, 4074, 3277, 88, 3, 0, 4, 64, 3553, 4, 54, 0, 13, 145, 677, 378, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4087, 0, 18, 3930, 311, 0, 4007, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 2194, 13, 0, 2464, 0, 0, 2719, 141, 0, 4096, 0, 0, 830, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9542, 18575, 12126, 12440, 26252, 18083, 16458, 17596] avg=16384.0000 max=26252 max_rank=4 min=9542 min_rank=0 imbalance_ratio=1.602295 max_load_reduction=104820 max_load_reduction_pct=0.799713 moved_to_echo=78389 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[335, 678, 2531, 2511, 13, 7, 0, 0, 3, 126, 4080, 11, 0, 60, 0, 0, 4096, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4088, 0, 331, 7, 3487, 0, 55, 2551, 1156, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 217, 0, 0, 0, 4072, 5, 719, 3808, 289, 0, 0, 4086, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 7, 3851, 0, 0, 0, 49, 0, 0, 1, 0, 4094, 0, 26, 2550, 1524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2030, 529, 0, 1034, 4040, 44, 0, 0, 0, 0, 2341, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 1, 3503, 3730, 0, 0, 0, 0, 2357, 3562, 4094, 0, 0, 17, 100, 1302, 0, 212, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 4068, 0, 3119, 639, 0, 386, 2794, 12, 4096, 3, 1915, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2900, 0, 54, 0, 0, 0, 0, 0, 6, 1, 0, 4064, 0, 13, 0, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15758, 19849, 17151, 16082, 10026, 22904, 21313, 7989] avg=16384.0000 max=22904 max_rank=5 min=7989 min_rank=7 imbalance_ratio=1.397949 max_load_reduction=108168 max_load_reduction_pct=0.825256 moved_to_echo=62232 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4095, 4079, 0, 57, 11, 29, 0, 0, 4075, 4092, 844, 770, 315, 0, 0, 0, 4065, 1263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 682, 3771, 4096, 3970, 19, 1, 4096, 0, 0, 2561, 3976, 0, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 22, 3393, 0, 4096, 0, 0, 0, 793, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 29, 0, 17, 0, 0, 0, 524, 0, 0, 0, 0, 690, 0, 1, 2465, 1535, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1687, 933, 3931, 0, 19, 602, 0, 0, 402, 0, 571, 124, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 4078, 2319, 0, 0, 0, 1, 129, 0, 4096, 0, 0, 100, 0, 1306, 686, 181, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 1925, 0, 1, 39, 4004, 133, 2790, 1527, 101, 3, 4021, 3647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 8, 3395, 2176, 0, 2895, 0, 2732, 2, 0, 2981, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23695, 27060, 12585, 5942, 12363, 16796, 18430, 14201] avg=16384.0000 max=27060 max_rank=1 min=5942 min_rank=3 imbalance_ratio=1.651611 max_load_reduction=104012 max_load_reduction_pct=0.793549 moved_to_echo=61790 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[140, 3626, 0, 1, 5, 22, 2104, 0, 2404, 74, 157, 0, 1261, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3512, 12, 180, 247, 1, 6, 11, 4093, 0, 0, 204, 0, 0, 4003, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 3873, 0, 0, 3089, 4, 0, 127, 0, 0, 0, 3530, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 0, 4096, 0, 0, 0, 4096, 2977, 417, 0, 0, 4095, 0, 0, 0, 121, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2337, 3652, 0, 0, 3922, 12, 0, 3619, 0, 4078, 0, 2, 0, 0, 4015, 2783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3621, 558, 0, 3700, 851, 0, 0, 274, 0, 4096, 0, 3421, 210, 340, 933, 379, 381, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 493, 0, 2, 32, 4073, 4073, 2003, 24, 4096, 0, 866, 26, 3953, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 1369, 0, 0, 0, 0, 0, 1524, 3793, 0, 27, 3831, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9851, 12292, 18770, 16716, 24420, 18797, 19658, 10568] avg=16384.0000 max=24420 max_rank=4 min=9851 min_rank=0 imbalance_ratio=1.490479 max_load_reduction=106652 max_load_reduction_pct=0.813690 moved_to_echo=73443 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3667, 11, 3993, 827, 4, 0, 0, 0, 3871, 0, 3881, 4083, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 174, 12, 0, 174, 1672, 57, 0, 0, 0, 1703, 9, 901, 0, 118, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 932, 52, 844, 0, 55, 0, 3941, 4088, 13, 4096, 35, 4096, 0, 3506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 2566, 12, 4094, 0, 4093, 0, 1422, 1549, 1, 0, 0, 55, 0, 1017, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 2133, 134, 0, 43, 0, 145, 438, 0, 0, 1, 0, 3743, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3928, 0, 3717, 0, 0, 316, 5, 806, 2, 0, 3244, 0, 0, 1287, 666, 223, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4044, 0, 1613, 0, 4086, 3960, 2762, 465, 992, 86, 158, 4065, 0, 3293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3464, 212, 0, 93, 4095, 0, 101, 0, 2676, 133, 6, 6, 0, 0, 0, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20361, 4822, 21658, 15533, 14449, 14201, 25554, 14494] avg=16384.0000 max=25554 max_rank=6 min=4822 min_rank=1 imbalance_ratio=1.559692 max_load_reduction=105518 max_load_reduction_pct=0.805038 moved_to_echo=68698 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 515, 2285, 9, 3441, 0, 3, 785, 4048, 1, 4, 7, 3589, 292, 0, 4059, 1383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 3782, 3070, 0, 0, 296, 1518, 0, 0, 2555, 0, 84, 0, 32, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4063, 0, 0, 0, 0, 389, 170, 4096, 332, 10, 3, 4096, 0, 2962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 4, 64, 4096, 2334, 1, 0, 4096, 0, 193, 4045, 15, 4074, 27, 1, 2701, 1526, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3797, 54, 539, 0, 3, 24, 0, 0, 0, 3052, 0, 133, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1957, 3583, 320, 0, 0, 0, 0, 1280, 0, 783, 0, 0, 968, 563, 607, 5, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 871, 4093, 0, 0, 0, 0, 2198, 1209, 0, 0, 55, 0, 15, 2334, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3856, 4095, 3927, 0, 0, 0, 254, 6, 2442, 0, 0, 3997, 3920, 8, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20421, 15381, 16131, 23786, 7898, 10099, 10782, 26574] avg=16384.0000 max=26574 max_rank=7 min=7898 min_rank=4 imbalance_ratio=1.621948 max_load_reduction=104498 max_load_reduction_pct=0.797256 moved_to_echo=55353 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[5888, 1024, 18560, 19072, 4224, 7424, 896, 25472, 128, 128, 17280, 5248, 1280, 13184, 256, 12160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[4096, 256, 4992, 128, 25472, 2432, 4096, 1408, 21120, 3072, 2048, 5760, 3328, 24192, 128, 128, 14848, 10368, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[16384, 20352, 8320, 4096, 5248, 16384, 896, 8064, 0, 7680, 3712, 12928, 7040, 4352, 13824, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1408, 128, 24320, 3968, 2304, 0, 18304, 13056, 1024, 24320, 768, 4224, 1024, 17408, 768, 19072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[6016, 9344, 3072, 9088, 256, 15616, 2944, 128, 11392, 14464, 5120, 8960, 1664, 8832, 4480, 0, 23040, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[7936, 1152, 10624, 24064, 13824, 11136, 1024, 128, 640, 2816, 5888, 24064, 896, 12928, 512, 1536, 6784, 4352, 2176, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 1408, 14720, 0, 13184, 768, 15616, 14720, 14592, 2048, 13696, 256, 7040, 11904, 18560, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[11008, 12288, 15488, 3712, 6272, 2560, 3328, 128, 17536, 7424, 128, 23424, 7808, 128, 896, 20096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 968, 0, 81, 0, 2687, 0, 0, 0, 1, 3185, 0, 0, 4096, 4089, 4096, 0, 4, 9, 0, 4039, 19, 0, 3929, 0, 45, 4096, 4087, 32, 4065, 26, 4095, 0, 23, 4065, 843, 0, 4096, 7, 0, 0, 4091, 146, 3990, 0, 4040, 0, 2424, 2, 0, 62, 0, 0, 16, 0, 0, 4, 2533, 13, 0, 4080, 0, 4088, 4088, 0, 0, 4079, 0, 6, 4076, 403, 0, 0, 0, 1580, 4094, 55, 3, 4077, 0, 0, 6, 0, 4, 3113, 3867, 16, 3, 0, 3111, 1, 0, 4096, 0, 0, 0, 341, 0, 3510, 1, 0, 0, 0, 4095, 1446, 0, 2728, 4090, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 25, 0, 0, 3975, 0, 0, 98, 0, 97, 0, 2, 0, 4096, 0, 55, 4017, 4095, 76, 85, 963, 0, 640, 0, 0, 16, 3999, 52, 2, 3213, 3038, 5, 4096, 8, 3180, 48, 503, 1165, 2288, 0, 4084, 0, 1, 0, 3945, 2, 1, 4046, 3960, 73, 4058, 235, 52, 4073, 2100, 0, 0, 0, 0, 18, 0, 1891, 720, 1588, 229, 4096, 0, 0, 0, 0, 0, 180, 4096, 4088, 0, 1, 0, 4001, 4080, 0, 0, 0, 0, 0, 15, 0, 0, 3884, 1455, 4096, 0, 0, 68, 5, 337, 4096, 0, 3, 0, 10, 138, 0, 5, 322, 377, 0, 4096, 0, 7, 0, 3773, 0, 20, 32, 14, 8, 4092, 0, 0, 4075, 0, 12, 4056, 0, 570, 0, 0, 4051] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 2637, 0, 0, 74, 2905, 40, 3, 3706, 0, 0, 0, 20, 3208, 0, 0, 4096, 3838, 481, 0, 4072, 265, 0, 0, 0, 0, 0, 617, 8, 4096, 4073, 0, 4063, 22, 171, 133, 6, 771, 4, 0, 4096, 3707, 0, 0, 12, 4096, 319, 0, 3027, 0, 50, 223, 18, 4096, 0, 0, 65, 0, 0, 3847, 0, 0, 0, 1122, 0, 4084, 622, 3817, 0, 4094, 0, 9, 1161, 4043, 0, 0, 11, 4096, 3075, 0, 0, 5, 0, 0, 3596, 362, 77, 136, 3791, 375, 1, 0, 0, 4, 0, 4096, 0, 0, 0, 105, 10, 4081, 1, 2834, 0, 0, 4072, 0, 0, 1649, 3980, 0, 0, 3886, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 850, 0, 0, 0, 4066] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 358, 0, 0, 4096, 3891, 0, 2, 3770, 0, 0, 0, 0, 3822, 0, 0, 3385, 4096, 4096, 10, 0, 0, 262, 0, 40, 0, 24, 2, 0, 1, 4091, 0, 4095, 0, 2, 0, 0, 1, 3929, 3, 4096, 3694, 9, 0, 4090, 0, 381, 252, 4, 0, 111, 0, 163, 3873, 0, 3, 0, 171, 0, 0, 2287, 282, 0, 1732, 16, 1104, 1, 0, 1, 4047, 59, 0, 3744, 53, 0, 0, 0, 3328, 209, 7, 662, 4096, 0, 1693, 3050, 0, 0, 0, 4090, 0, 359, 0, 4096, 2123, 0, 4096, 3986, 0, 1602, 0, 5, 4, 7, 0, 0, 856, 4002, 0, 0, 2558, 4096, 0, 111, 0, 0, 0, 4050, 0, 3916, 4001, 0, 0, 3462, 0, 0, 20, 0, 438] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 3686, 1995, 0, 0, 0, 980, 0, 0, 0, 0, 0, 0, 3432, 0, 0, 4040, 4090, 542, 36, 1352, 0, 0, 0, 142, 0, 0, 1, 0, 4087, 1178, 30, 2, 4051, 141, 59, 3, 1935, 101, 0, 3946, 3000, 0, 2, 3945, 4051, 0, 0, 4095, 0, 556, 8, 0, 1850, 0, 100, 1, 0, 1693, 0, 3867, 0, 27, 4036, 0, 4039, 0, 0, 0, 3599, 1, 4096, 0, 62, 0, 3, 2061, 2976, 4094, 0, 0, 2736, 0, 3, 0, 0, 1, 60, 3986, 0, 0, 0, 3812, 4096, 1496, 4013, 0, 65, 6, 0, 10, 0, 0, 8, 152, 499, 2194, 4095, 0, 4093, 1358, 0, 0, 0, 3729, 0, 0, 0, 0, 1911, 0, 373, 54, 4080, 0, 0, 0, 155] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[49, 7, 140, 1, 0, 0, 0, 0, 1832, 1, 81, 2731, 4024, 1682, 2919, 43, 1336, 3745, 3981, 269, 1086, 0, 1, 571, 2057, 0, 0, 82, 0, 3664, 3936, 0, 1, 0, 0, 2, 0, 0, 10, 0, 46, 4071, 0, 850, 2502, 0, 4090, 0, 110, 4082, 722, 376, 3354, 0, 270, 4085, 0, 0, 0, 0, 0, 2987, 1095, 0, 4, 0, 0, 0, 0, 10, 1301, 13, 0, 0, 0, 2, 3999, 2972, 3992, 96, 0, 4014, 3678, 0, 2709, 0, 0, 17, 2907, 24, 0, 3951, 4029, 526, 1747, 3837, 0, 0, 1881, 0, 4019, 0, 0, 3, 406, 672, 3235, 0, 2017, 3293, 4091, 0, 4, 6, 0, 171, 146, 3819, 93, 5, 0, 191, 4084, 35, 0, 134, 0, 48] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4070, 4096, 1, 0, 0, 0, 0, 137, 0, 0, 4093, 0, 0, 4096, 1, 2412, 0, 0, 0, 0, 2, 105, 0, 4096, 23, 0, 2, 1, 0, 26, 0, 3606, 0, 0, 0, 4091, 2, 4096, 1, 4096, 3622, 2808, 0, 4095, 3995, 678, 0, 1, 0, 0, 0, 0, 0, 0, 175, 623, 9, 15, 580, 3946, 4092, 2, 45, 0, 1257, 1, 4093, 401, 309, 167, 0, 0, 0, 0, 2, 2319, 4096, 3939, 4096, 0, 292, 3540, 529, 0, 69, 0, 0, 0, 0, 3, 3764, 0, 4096, 4085, 0, 22, 2, 4082, 4, 0, 0, 0, 0, 3805, 0, 371, 57, 3928, 0, 0, 0, 0, 1554, 4071, 3, 7, 26, 0, 0, 63, 0, 4094, 0, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 2195, 0, 0, 92, 1, 0, 0, 790, 0, 0, 0, 0, 4094, 0, 8, 4060, 4096, 518, 239, 0, 1389, 8, 0, 0, 0, 180, 71, 0, 3552, 3703, 0, 4085, 0, 135, 0, 0, 2296, 3490, 29, 4092, 4096, 4, 0, 228, 0, 8, 3829, 43, 0, 2, 0, 1208, 3319, 0, 265, 0, 143, 0, 22, 8, 4016, 0, 4091, 3333, 254, 2081, 0, 7, 19, 650, 3815, 4076, 19, 0, 0, 0, 60, 29, 477, 2193, 4096, 0, 0, 2149, 0, 0, 577, 169, 0, 3986, 0, 835, 837, 0, 4096, 3905, 0, 987, 0, 354, 1195, 0, 0, 0, 4080, 3888, 0, 0, 1607, 3, 0, 0, 0, 1317, 0, 4055, 0, 4096, 3951, 0, 0, 4070, 0, 0, 0, 0, 3401] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 2637, 0, 0, 74, 2905, 40, 3, 3706, 0, 0, 0, 20, 3208, 0, 0, 4096, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 3838, 481, 0, 4072, 265, 0, 0, 0, 0, 0, 617, 8, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 22, 171, 133, 6, 771, 4, 0, 0, 3707, 0, 0, 5, 4096, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3027, 0, 50, 223, 18, 4096, 0, 0, 65, 0, 0, 3847, 0, 0, 0, 1122, 7, 851, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 622, 3817, 0, 4094, 0, 9, 1161, 4043, 0, 0, 11, 4096, 1589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3596, 362, 77, 136, 3791, 375, 1, 0, 0, 4, 0, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 10, 4081, 1, 2834, 0, 0, 4072, 0, 0, 1649, 3980, 0, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3886, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 850, 0, 0, 0, 4066, 2205, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17616, 18734, 13297, 13695, 23526, 9311, 17829, 17064] avg=16384.0000 max=23526 max_rank=4 min=9311 min_rank=5 imbalance_ratio=1.435913 max_load_reduction=107546 max_load_reduction_pct=0.820511 moved_to_echo=67730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 968, 0, 81, 0, 2687, 0, 0, 0, 1, 3185, 0, 0, 4096, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 4089, 4096, 0, 4, 9, 0, 4039, 19, 0, 3929, 0, 45, 4096, 4087, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 26, 4095, 0, 23, 4065, 843, 0, 0, 7, 0, 0, 1692, 146, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 2424, 2, 0, 62, 0, 0, 16, 0, 0, 4, 2533, 13, 0, 4080, 2399, 851, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4088, 0, 0, 4079, 0, 6, 4076, 403, 0, 0, 0, 1580, 2116, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4077, 0, 0, 6, 0, 4, 3113, 3867, 16, 3, 0, 3111, 1, 0, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 0, 3510, 1, 0, 0, 0, 4095, 1446, 0, 2728, 4090, 0, 1460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2205, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11950, 25729, 18952, 16942, 20491, 15165, 17671, 4172] avg=16384.0000 max=25729 max_rank=1 min=4172 min_rank=7 imbalance_ratio=1.570374 max_load_reduction=105343 max_load_reduction_pct=0.803703 moved_to_echo=57499 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 3686, 1995, 0, 0, 0, 980, 0, 0, 0, 0, 0, 0, 3432, 0, 0, 3946, 908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1267, 4090, 542, 36, 1352, 0, 0, 0, 142, 0, 0, 1, 0, 4087, 1178, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4051, 141, 59, 3, 1935, 101, 0, 0, 3000, 0, 2, 1632, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 556, 8, 0, 1850, 0, 100, 1, 0, 1693, 0, 3867, 0, 27, 4036, 2313, 839, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 3599, 1, 4096, 0, 62, 0, 3, 2061, 2976, 2116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2736, 0, 3, 0, 0, 1, 60, 3986, 0, 0, 0, 3812, 4096, 1496, 945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 6, 0, 10, 0, 0, 8, 152, 499, 2194, 4095, 0, 4093, 1358, 0, 1460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3729, 0, 0, 0, 0, 1911, 0, 373, 54, 4080, 0, 0, 0, 155, 2160, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19043, 12725, 14977, 19903, 18953, 17135, 13940, 14396] avg=16384.0000 max=19903 max_rank=3 min=12725 min_rank=1 imbalance_ratio=1.214783 max_load_reduction=111169 max_load_reduction_pct=0.848152 moved_to_echo=64424 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4070, 4096, 1, 0, 0, 0, 0, 137, 0, 0, 4093, 0, 0, 4096, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 1, 2412, 0, 0, 0, 0, 2, 105, 0, 4096, 23, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 3606, 0, 0, 0, 4091, 2, 0, 1, 4096, 3622, 1162, 0, 4095, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 678, 0, 1, 0, 0, 0, 0, 0, 0, 175, 623, 9, 15, 580, 3946, 4092, 1646, 851, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 45, 0, 1257, 1, 4093, 401, 309, 167, 0, 0, 0, 0, 2, 1199, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3939, 4096, 0, 292, 3540, 529, 0, 69, 0, 0, 0, 0, 3, 3764, 0, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 22, 2, 4082, 4, 0, 0, 0, 0, 3805, 0, 371, 57, 3928, 0, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1554, 4071, 3, 7, 26, 0, 0, 63, 0, 4094, 0, 0, 4090, 2205, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21516, 7926, 24696, 12909, 11572, 17196, 17183, 18074] avg=16384.0000 max=24696 max_rank=2 min=7926 min_rank=1 imbalance_ratio=1.507324 max_load_reduction=106376 max_load_reduction_pct=0.811584 moved_to_echo=64025 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 358, 0, 0, 4096, 3891, 0, 2, 3770, 0, 0, 0, 0, 3822, 0, 0, 4096, 930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 4096, 4096, 10, 0, 0, 262, 0, 40, 0, 24, 2, 0, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 1, 3929, 3, 0, 3694, 9, 0, 1688, 0, 381, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 111, 0, 163, 3873, 0, 3, 0, 171, 0, 0, 2287, 282, 0, 1732, 2402, 707, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1104, 1, 0, 1, 4047, 59, 0, 3744, 53, 0, 0, 0, 3328, 103, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 4096, 0, 1693, 3050, 0, 0, 0, 4090, 0, 359, 0, 4096, 2123, 0, 960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 0, 1602, 0, 5, 4, 7, 0, 0, 856, 4002, 0, 0, 2558, 4096, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 4050, 0, 3916, 4001, 0, 0, 3462, 0, 0, 20, 0, 438, 2206, 1625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20965, 13675, 14054, 11764, 12463, 21129, 17193, 19829] avg=16384.0000 max=21129 max_rank=5 min=11764 min_rank=3 imbalance_ratio=1.289612 max_load_reduction=109943 max_load_reduction_pct=0.838799 moved_to_echo=70614 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 25, 0, 0, 3975, 0, 0, 98, 0, 97, 0, 2, 0, 4096, 0, 55, 4084, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1259, 4095, 76, 85, 963, 0, 640, 0, 0, 16, 3999, 52, 2, 3213, 3038, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 3180, 48, 503, 1165, 2288, 0, 0, 0, 1, 0, 1632, 2, 1, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 73, 4058, 235, 52, 4073, 2100, 0, 0, 0, 0, 18, 0, 1891, 720, 1588, 2313, 835, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 4096, 0, 0, 0, 0, 0, 180, 4096, 4088, 0, 1, 0, 4001, 2109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 3884, 1455, 4096, 0, 0, 68, 5, 337, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 10, 138, 0, 5, 322, 377, 0, 4096, 0, 7, 0, 3773, 0, 1455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 32, 14, 8, 4092, 0, 0, 4075, 0, 12, 4056, 0, 570, 0, 0, 4051, 2205, 1923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13359, 17443, 16970, 22432, 18800, 10824, 10186, 21058] avg=16384.0000 max=22432 max_rank=3 min=10186 min_rank=6 imbalance_ratio=1.369141 max_load_reduction=108640 max_load_reduction_pct=0.828857 moved_to_echo=60868 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 2195, 0, 0, 92, 1, 0, 0, 790, 0, 0, 0, 0, 4094, 0, 8, 4092, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1272, 4096, 518, 239, 0, 1389, 8, 0, 0, 0, 180, 71, 0, 3552, 3703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 135, 0, 0, 2296, 3490, 29, 0, 4096, 4, 0, 95, 0, 8, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 2, 0, 1208, 3319, 0, 265, 0, 143, 0, 22, 8, 4016, 0, 4091, 133, 844, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3333, 254, 2081, 0, 7, 19, 650, 3815, 4076, 19, 0, 0, 0, 60, 16, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2193, 4096, 0, 0, 2149, 0, 0, 577, 169, 0, 3986, 0, 835, 837, 0, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 0, 987, 0, 354, 1195, 0, 0, 0, 4080, 3888, 0, 0, 1607, 3, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 0, 4055, 0, 4096, 3951, 0, 0, 4070, 0, 0, 0, 0, 3401, 2205, 1944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12199, 15028, 18067, 14097, 14807, 15806, 16029, 25039] avg=16384.0000 max=25039 max_rank=7 min=12199 min_rank=0 imbalance_ratio=1.528259 max_load_reduction=106033 max_load_reduction_pct=0.808968 moved_to_echo=71681 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[49, 7, 140, 1, 0, 0, 0, 0, 1832, 1, 81, 2731, 4024, 1682, 2919, 43, 46, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 3745, 3981, 269, 1086, 0, 1, 571, 2057, 0, 0, 82, 0, 3664, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 10, 0, 0, 4071, 0, 850, 1035, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 4082, 722, 376, 3354, 0, 270, 4085, 0, 0, 0, 0, 0, 2987, 1095, 0, 1467, 277, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 10, 1301, 13, 0, 0, 0, 2, 3999, 2972, 2063, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 3678, 0, 2709, 0, 0, 17, 2907, 24, 0, 3951, 4029, 526, 1747, 904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1881, 0, 4019, 0, 0, 3, 406, 672, 3235, 0, 2017, 3293, 4091, 0, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 0, 171, 146, 3819, 93, 5, 0, 191, 4084, 35, 0, 134, 0, 48, 2065, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14424, 19812, 10059, 19330, 10460, 24506, 21041, 11440] avg=16384.0000 max=24506 max_rank=5 min=10059 min_rank=2 imbalance_ratio=1.495728 max_load_reduction=106566 max_load_reduction_pct=0.813034 moved_to_echo=67447 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9216, 28160, 16256, 640, 7552, 1664, 1024, 4736, 2432, 128, 12288, 896, 128, 22784, 24192, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[3584, 17792, 6912, 5120, 128, 19968, 2432, 8448, 17408, 8704, 0, 128, 6144, 19072, 11392, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 3968, 5120, 1792, 20608, 3840, 8192, 14080, 0, 640, 15872, 4992, 4736, 256, 0, 16256, 17536, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[4224, 13056, 2176, 4096, 13312, 6912, 1152, 128, 12800, 128, 256, 2816, 4096, 27648, 2944, 128, 28672, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[20480, 4224, 11392, 256, 640, 10240, 14848, 128, 0, 18688, 4224, 4480, 8960, 8320, 12928, 12160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[6912, 23168, 3712, 2048, 15104, 896, 128, 7936, 20352, 4608, 4352, 3968, 16000, 11392, 3584, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12032, 128, 4608, 512, 8704, 8832, 128, 3200, 1024, 6144, 29440, 5632, 2432, 16000, 25344, 0, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[16000, 4224, 7936, 896, 4864, 17280, 2432, 4480, 128, 512, 2432, 3968, 8832, 9856, 5888, 20864, 12800, 6144, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[2, 0, 0, 3356, 4093, 35, 130, 758, 4076, 0, 0, 4, 11, 306, 384, 0, 4073, 20, 0, 0, 0, 0, 3841, 0, 31, 380, 152, 3606, 33, 4081, 0, 4095, 0, 4096, 0, 9, 0, 4, 2, 0, 3823, 1, 3905, 4063, 0, 4090, 4087, 2, 6, 31, 65, 4077, 0, 0, 0, 0, 0, 0, 0, 319, 3946, 5, 3990, 2237, 0, 125, 0, 4091, 0, 4095, 53, 0, 4096, 3845, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 2, 4086, 0, 0, 3820, 0, 38, 682, 0, 1470, 0, 7, 13, 0, 0, 0, 4093, 0, 4096, 0, 934, 4093, 0, 3909, 5, 4096, 3971, 1, 0, 0, 0, 0, 0, 4, 3123, 8, 4081, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[325, 0, 0, 0, 0, 140, 4, 2, 3298, 3994, 12, 2, 1332, 2455, 4096, 0, 3928, 398, 0, 0, 0, 515, 0, 316, 0, 0, 0, 3517, 4096, 542, 9, 281, 17, 4036, 12, 4029, 0, 0, 196, 0, 0, 5, 2607, 3214, 0, 4059, 4, 0, 3940, 1362, 0, 4096, 2696, 70, 4021, 0, 0, 1, 0, 3018, 0, 12, 3238, 4080, 0, 3870, 0, 4071, 0, 0, 6, 0, 919, 4096, 70, 0, 18, 0, 6, 218, 9, 0, 99, 0, 782, 598, 177, 0, 0, 25, 609, 3880, 0, 6, 0, 0, 0, 3466, 22, 4074, 0, 9, 3598, 1, 0, 1, 1770, 8, 4078, 0, 0, 2, 10, 4060, 841, 3679, 4093, 3170, 0, 450, 15, 3, 0, 0, 0, 583, 3705, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 2, 6, 8, 807, 636, 0, 1074, 2089, 0, 50, 0, 0, 68, 3952, 65, 0, 27, 0, 0, 0, 2379, 1891, 0, 0, 2634, 0, 812, 3938, 0, 0, 0, 0, 3043, 101, 1180, 0, 18, 0, 0, 4093, 4089, 766, 3490, 0, 4096, 0, 0, 4096, 150, 9, 0, 0, 0, 0, 0, 109, 24, 41, 645, 3679, 22, 0, 4091, 3, 2027, 3354, 4096, 0, 4069, 3581, 8, 4045, 4035, 450, 0, 297, 156, 0, 0, 5, 0, 24, 0, 0, 0, 0, 0, 3838, 41, 9, 4088, 0, 0, 0, 0, 0, 3812, 0, 0, 5, 727, 59, 3, 379, 53, 0, 0, 4036, 38, 4012, 4069, 0, 1883, 4096, 3825, 4094, 0, 0, 0, 0, 0, 0, 80, 3952, 2, 4033, 3608] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[38, 18, 0, 364, 0, 2681, 0, 0, 955, 955, 3783, 4, 3914, 736, 3390, 0, 0, 810, 0, 241, 0, 0, 4083, 668, 0, 2589, 26, 0, 4096, 3948, 0, 3386, 0, 4096, 386, 4096, 67, 100, 0, 0, 0, 484, 4080, 1683, 0, 3917, 0, 0, 1, 2816, 0, 4075, 0, 60, 372, 0, 2113, 33, 0, 0, 0, 8, 2709, 4096, 0, 2499, 0, 4085, 0, 43, 4091, 0, 4095, 4096, 1, 0, 0, 0, 120, 0, 0, 0, 0, 0, 13, 0, 2096, 0, 0, 0, 2, 4043, 0, 542, 2218, 0, 5, 866, 0, 3, 0, 58, 3256, 0, 0, 0, 4095, 0, 2915, 0, 158, 4074, 0, 0, 5, 3711, 4096, 0, 0, 3774, 0, 0, 0, 142, 0, 3, 4090, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 191, 2698, 4063, 68, 2381, 0, 2034, 3973, 0, 0, 0, 0, 0, 0, 3790, 2417, 0, 0, 382, 0, 217, 0, 1, 0, 12, 4093, 0, 2064, 63, 0, 1551, 0, 3525, 1065, 0, 0, 0, 0, 1, 4073, 3455, 2732, 0, 3748, 254, 8, 3722, 3783, 1236, 0, 0, 1, 0, 0, 0, 0, 3673, 3, 35, 4031, 0, 0, 0, 205, 4096, 4083, 781, 2094, 3436, 0, 0, 0, 15, 0, 0, 0, 37, 15, 0, 928, 0, 35, 0, 1, 0, 0, 272, 0, 0, 4084, 0, 0, 3881, 16, 800, 413, 4079, 759, 0, 337, 1, 0, 903, 1, 4078, 1, 4095, 0, 4096, 4096, 0, 0, 384, 4096, 4092, 2165, 21, 0, 0, 1878, 31, 96, 3, 0, 4096, 1228] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 233, 362, 4018, 0, 17, 4006, 3313, 0, 35, 4087, 0, 0, 0, 0, 2614, 0, 0, 0, 14, 1345, 2, 0, 1, 1587, 0, 3995, 4090, 65, 268, 3271, 187, 2703, 0, 4010, 0, 0, 0, 0, 4095, 4061, 177, 235, 0, 3248, 19, 0, 0, 4096, 0, 1626, 0, 0, 40, 0, 1471, 0, 0, 15, 4096, 60, 1821, 1942, 17, 3955, 21, 4081, 3712, 3, 1921, 49, 0, 0, 4, 0, 0, 0, 0, 210, 0, 19, 0, 0, 0, 1, 2444, 70, 87, 0, 0, 4095, 3, 0, 1325, 78, 0, 4084, 1139, 147, 0, 1, 0, 0, 0, 2, 4093, 6, 0, 2, 4096, 4096, 0, 0, 3947, 4095, 0, 2920, 1291, 0, 0, 50, 0, 2170, 1610, 0, 4096, 3906] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 1, 0, 4060, 0, 4051, 872, 3329, 0, 0, 7, 0, 0, 0, 0, 3798, 0, 0, 0, 242, 4080, 56, 0, 3923, 0, 26, 4001, 0, 5, 4096, 2543, 0, 4095, 0, 1, 0, 0, 0, 0, 4093, 62, 0, 4073, 0, 4096, 2257, 11, 0, 20, 24, 0, 3, 0, 0, 0, 0, 0, 0, 2, 4096, 1138, 0, 2972, 0, 3839, 433, 785, 3728, 1479, 0, 0, 182, 0, 2, 0, 0, 0, 0, 2, 0, 1246, 0, 0, 0, 1, 0, 2669, 4096, 0, 0, 4095, 0, 1, 1164, 15, 4020, 0, 4078, 0, 0, 1, 3614, 0, 0, 1, 0, 66, 1616, 0, 3741, 4084, 0, 0, 4081, 4096, 0, 4061, 33, 0, 0, 1038, 218, 1974, 4096, 0, 4096, 387] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 1726, 0, 0, 1477, 1169, 0, 3522, 7, 3, 0, 3331, 0, 4, 0, 47, 2691, 0, 0, 0, 753, 0, 0, 2371, 4079, 27, 4096, 459, 18, 3991, 17, 32, 0, 71, 0, 0, 0, 48, 0, 0, 4023, 127, 0, 0, 169, 3852, 0, 398, 347, 0, 4022, 0, 2061, 634, 0, 0, 6, 3197, 578, 4, 0, 0, 0, 4096, 4083, 1830, 0, 4070, 0, 29, 0, 4096, 275, 0, 15, 0, 0, 3860, 3971, 0, 839, 18, 90, 0, 43, 38, 0, 2795, 1680, 3900, 3213, 1, 0, 83, 2703, 0, 3678, 3048, 4089, 1412, 20, 4093, 962, 1, 0, 0, 0, 3991, 0, 590, 4096, 187, 0, 3916, 4096, 3949, 162, 0, 0, 38, 0, 0, 5, 91, 85, 1478, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 3356, 4093, 35, 130, 758, 4076, 0, 0, 4, 11, 306, 384, 0, 2390, 905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 20, 0, 0, 0, 0, 3841, 0, 31, 380, 152, 2712, 33, 4081, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 4, 2, 0, 3823, 1, 3905, 4063, 0, 1536, 4087, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 31, 65, 4077, 0, 0, 0, 0, 0, 0, 0, 319, 3946, 5, 3990, 2237, 894, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 2607, 0, 4095, 53, 0, 4096, 3845, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 2, 4086, 0, 0, 3820, 0, 3927, 3223, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 682, 0, 1470, 0, 7, 13, 0, 0, 0, 4093, 0, 4096, 0, 934, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 5, 169, 3971, 1, 0, 0, 0, 0, 0, 4, 3123, 8, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16450, 19418, 21528, 16149, 14829, 15224, 12203, 15271] avg=16384.0000 max=21528 max_rank=2 min=12203 min_rank=6 imbalance_ratio=1.313965 max_load_reduction=109544 max_load_reduction_pct=0.835754 moved_to_echo=57527 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[325, 0, 0, 0, 0, 140, 4, 2, 3298, 3994, 12, 2, 1332, 2455, 4096, 0, 2371, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3928, 398, 0, 0, 0, 515, 0, 316, 0, 0, 0, 2646, 4096, 542, 9, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4036, 12, 4029, 0, 0, 196, 0, 0, 5, 2607, 3214, 0, 1525, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 1362, 0, 4096, 2696, 70, 4021, 0, 0, 1, 0, 3018, 0, 12, 3238, 4080, 871, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3870, 0, 2595, 0, 0, 6, 0, 919, 4096, 70, 0, 18, 0, 6, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 99, 0, 782, 598, 177, 0, 0, 25, 609, 3880, 0, 6, 0, 0, 3527, 1, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3466, 22, 4074, 0, 9, 3598, 1, 0, 1, 1770, 8, 4078, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4060, 841, 152, 4093, 3170, 0, 450, 15, 3, 0, 0, 0, 583, 3705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18931, 12731, 15645, 27981, 11798, 9876, 17028, 17082] avg=16384.0000 max=27981 max_rank=3 min=9876 min_rank=5 imbalance_ratio=1.707825 max_load_reduction=103091 max_load_reduction_pct=0.786522 moved_to_echo=55784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[38, 18, 0, 364, 0, 2681, 0, 0, 955, 955, 3783, 4, 3914, 736, 3390, 0, 2288, 903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 810, 0, 241, 0, 0, 4083, 668, 0, 2589, 26, 0, 4096, 3948, 0, 3386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 386, 4096, 67, 100, 0, 0, 0, 484, 4080, 1683, 0, 1472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2816, 0, 4075, 0, 60, 372, 0, 2113, 33, 0, 0, 0, 8, 2709, 4096, 0, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2499, 0, 2604, 0, 43, 4091, 0, 4095, 4096, 1, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 2096, 0, 0, 0, 2, 4043, 0, 542, 2218, 0, 3558, 3208, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 866, 0, 3, 0, 58, 3256, 0, 0, 0, 4095, 0, 2915, 0, 158, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 153, 4096, 0, 0, 3774, 0, 0, 0, 142, 0, 3, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20029, 19847, 16464, 16861, 17549, 15837, 12222, 12263] avg=16384.0000 max=20029 max_rank=0 min=12222 min_rank=6 imbalance_ratio=1.222473 max_load_reduction=111043 max_load_reduction_pct=0.847191 moved_to_echo=57871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 233, 362, 4018, 0, 17, 4006, 3313, 0, 35, 4087, 0, 0, 0, 0, 1898, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2614, 0, 0, 0, 14, 1345, 2, 0, 1, 1587, 0, 3005, 4090, 65, 268, 3271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 2703, 0, 4010, 0, 0, 0, 0, 4095, 4061, 177, 235, 0, 1220, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1626, 0, 0, 40, 0, 1471, 0, 0, 15, 4096, 60, 1821, 1942, 990, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3955, 21, 2602, 3712, 3, 1921, 49, 0, 0, 4, 0, 0, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 1, 2444, 70, 87, 0, 0, 4095, 3, 0, 1325, 78, 3926, 3225, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1139, 147, 0, 1, 0, 0, 0, 2, 4093, 6, 0, 2, 4096, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 169, 0, 2920, 1291, 0, 0, 50, 0, 2170, 1610, 0, 4096, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18872, 16262, 16707, 16734, 12494, 15403, 14441, 20159] avg=16384.0000 max=20159 max_rank=7 min=12494 min_rank=4 imbalance_ratio=1.230408 max_load_reduction=110913 max_load_reduction_pct=0.846199 moved_to_echo=62497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 191, 2698, 4063, 68, 2381, 0, 2034, 3973, 0, 0, 0, 0, 0, 0, 2190, 903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3790, 2417, 0, 0, 382, 0, 217, 0, 1, 0, 12, 3079, 0, 2064, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1551, 0, 3525, 1065, 0, 0, 0, 0, 1, 4073, 3455, 2732, 0, 1407, 254, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3722, 3783, 1236, 0, 0, 1, 0, 0, 0, 0, 3673, 3, 35, 4031, 0, 0, 1014, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 4096, 2603, 781, 2094, 3436, 0, 0, 0, 15, 0, 0, 0, 37, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 928, 0, 35, 0, 1, 0, 0, 272, 0, 0, 4084, 0, 0, 3881, 16, 3927, 3225, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 800, 413, 4079, 759, 0, 337, 1, 0, 903, 1, 4078, 1, 4095, 0, 4096, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 169, 4092, 2165, 21, 0, 0, 1878, 31, 96, 3, 0, 4096, 1228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18502, 12025, 18071, 18075, 13282, 16520, 20434, 14163] avg=16384.0000 max=20434 max_rank=6 min=12025 min_rank=1 imbalance_ratio=1.247192 max_load_reduction=110638 max_load_reduction_pct=0.844101 moved_to_echo=64399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 6, 8, 807, 636, 0, 1074, 2089, 0, 50, 0, 0, 68, 3952, 65, 2393, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 2379, 1891, 0, 0, 2634, 0, 611, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3043, 101, 1180, 0, 18, 0, 0, 4093, 4089, 766, 3490, 0, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 150, 9, 0, 0, 0, 0, 0, 109, 24, 41, 645, 3679, 22, 0, 4091, 201, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2027, 3354, 2611, 0, 4069, 3581, 8, 4045, 4035, 450, 0, 297, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 24, 0, 0, 0, 0, 0, 3838, 41, 9, 4088, 0, 0, 0, 0, 3667, 3204, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3812, 0, 0, 5, 727, 59, 3, 379, 53, 0, 0, 4036, 38, 4012, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1883, 4096, 158, 4094, 0, 0, 0, 0, 0, 0, 80, 3952, 2, 4033, 3608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12056, 11480, 18318, 13646, 24636, 15041, 13989, 21906] avg=16384.0000 max=24636 max_rank=4 min=11480 min_rank=1 imbalance_ratio=1.503662 max_load_reduction=106436 max_load_reduction_pct=0.812042 moved_to_echo=75572 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 0, 4060, 0, 4051, 872, 3329, 0, 0, 7, 0, 0, 0, 0, 2397, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 0, 0, 242, 4080, 56, 0, 3923, 0, 26, 3006, 0, 5, 4096, 2543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 4093, 62, 0, 4073, 0, 1529, 2257, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 24, 0, 3, 0, 0, 0, 0, 0, 0, 2, 4096, 1138, 0, 2972, 995, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3839, 433, 494, 3728, 1479, 0, 0, 182, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1246, 0, 0, 0, 1, 0, 2669, 4096, 0, 0, 4095, 0, 1, 1164, 15, 3932, 3221, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 4078, 0, 0, 1, 3614, 0, 0, 1, 0, 66, 1616, 0, 3741, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 164, 0, 4061, 33, 0, 0, 1038, 218, 1974, 4096, 0, 4096, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14895, 21775, 16121, 9364, 10159, 20610, 18000, 20148] avg=16384.0000 max=21775 max_rank=1 min=9364 min_rank=3 imbalance_ratio=1.329041 max_load_reduction=109297 max_load_reduction_pct=0.833870 moved_to_echo=68917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1726, 0, 0, 1477, 1169, 0, 3522, 7, 3, 0, 3331, 0, 4, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 2691, 0, 0, 0, 753, 0, 0, 2371, 4079, 27, 3081, 459, 18, 3991, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 71, 0, 0, 0, 48, 0, 0, 4023, 127, 0, 0, 65, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 347, 0, 4022, 0, 2061, 634, 0, 0, 6, 3197, 578, 4, 0, 0, 0, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 1830, 0, 4070, 0, 29, 0, 4096, 275, 0, 15, 0, 0, 3860, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 18, 90, 0, 43, 38, 0, 2795, 1680, 3900, 3213, 1, 0, 83, 2703, 3927, 3225, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3678, 3048, 4089, 1412, 20, 4093, 962, 1, 0, 0, 0, 3991, 0, 590, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 3916, 169, 3949, 162, 0, 0, 38, 0, 0, 5, 91, 85, 1478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11337, 17534, 8218, 12262, 26325, 22561, 22755, 10080] avg=16384.0000 max=26325 max_rank=4 min=8218 min_rank=2 imbalance_ratio=1.606750 max_load_reduction=104747 max_load_reduction_pct=0.799156 moved_to_echo=81721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[12288, 12672, 1408, 17920, 2816, 2304, 5120, 0, 3712, 128, 6912, 4608, 15872, 5376, 11776, 19456, 6016, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[4224, 20608, 9856, 16128, 12416, 11904, 13184, 128, 17536, 16384, 640, 128, 384, 256, 4096, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[1792, 22144, 4096, 14464, 128, 128, 256, 0, 16128, 16896, 15232, 19584, 0, 10368, 10496, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[18304, 6400, 0, 256, 640, 9088, 10112, 1024, 6400, 11392, 256, 18176, 16768, 10752, 8448, 13696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[384, 128, 2176, 6912, 17152, 5120, 7808, 6784, 22656, 8960, 3968, 4224, 8704, 3584, 11904, 128, 16128, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[4864, 17024, 12416, 10624, 1536, 1280, 14720, 1024, 1408, 128, 18176, 128, 24832, 128, 17664, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[256, 9856, 17280, 1408, 24320, 12544, 1408, 4224, 128, 3072, 256, 4480, 12928, 768, 29696, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 3072, 256, 128, 896, 768, 4864, 2816, 11136, 1792, 4608, 31616, 128, 640, 12544, 2816, 30464, 22656, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[3838, 0, 0, 134, 0, 4066, 11, 5, 5, 0, 5, 25, 4088, 41, 0, 0, 0, 0, 434, 1164, 4080, 0, 2, 0, 2, 0, 4077, 1, 21, 44, 44, 2743, 1, 4095, 4096, 0, 0, 0, 0, 0, 484, 4095, 168, 0, 0, 3325, 2940, 0, 27, 2719, 0, 0, 0, 4, 0, 0, 0, 3928, 3999, 0, 1930, 0, 0, 3830, 0, 9, 2610, 199, 0, 1, 565, 0, 4035, 0, 130, 119, 0, 0, 1, 0, 0, 3607, 936, 1, 735, 2006, 3760, 0, 4096, 0, 1301, 69, 4096, 125, 2528, 4096, 0, 121, 4096, 4081, 3426, 3896, 34, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3525, 0, 1, 0, 103, 1849, 4062, 0, 2162, 0, 154, 4059, 0, 0, 148, 3844] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[3587, 0, 0, 0, 0, 4, 3892, 0, 25, 0, 3735, 0, 4084, 4096, 0, 2, 0, 0, 187, 0, 13, 20, 2907, 0, 0, 0, 3626, 0, 4093, 0, 1, 2630, 4059, 4095, 3394, 0, 0, 0, 0, 2, 4038, 3201, 0, 0, 0, 0, 0, 0, 3691, 1112, 0, 12, 861, 4096, 4096, 0, 0, 0, 4036, 108, 0, 159, 388, 2114, 0, 3421, 0, 0, 0, 28, 4043, 172, 1410, 0, 4090, 0, 1609, 0, 0, 1546, 0, 4096, 0, 149, 7, 0, 0, 1453, 3136, 0, 15, 0, 3918, 0, 0, 4074, 644, 0, 4095, 259, 2638, 678, 7, 0, 22, 29, 0, 185, 73, 2, 0, 0, 0, 6, 4064, 4043, 4086, 19, 4011, 0, 0, 423, 0, 0, 146, 0, 106, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 3327, 534, 314, 0, 26, 0, 1, 3821, 4074, 1, 9, 0, 0, 0, 0, 3756, 1012, 0, 149, 528, 0, 4096, 0, 3187, 0, 3, 4096, 0, 3283, 533, 107, 1, 0, 0, 7, 4084, 3682, 0, 19, 0, 43, 0, 0, 0, 4096, 0, 232, 0, 4092, 358, 6, 3800, 0, 4007, 0, 47, 20, 0, 0, 766, 4092, 0, 750, 0, 0, 3928, 10, 3940, 0, 0, 3342, 202, 0, 0, 0, 16, 4045, 2587, 219, 4077, 1, 3459, 0, 4043, 0, 15, 4067, 262, 0, 0, 2975, 0, 0, 0, 4096, 0, 0, 4090, 51, 3, 0, 526, 118, 0, 0, 1, 3, 673, 138, 0, 4096, 0, 1, 4096, 711, 0, 0, 0, 497, 3657, 13, 1, 4058] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[3926, 0, 3989, 0, 281, 9, 0, 0, 0, 0, 590, 51, 3605, 3944, 712, 0, 0, 0, 78, 352, 9, 0, 0, 0, 0, 0, 1325, 0, 2465, 0, 4, 0, 3660, 2784, 4052, 0, 0, 0, 0, 0, 479, 4092, 76, 0, 2891, 79, 3919, 4, 825, 3798, 230, 0, 4093, 0, 5, 0, 149, 4020, 4096, 0, 0, 150, 1293, 0, 1, 3738, 2107, 0, 570, 0, 4079, 0, 4, 614, 4090, 0, 0, 0, 3900, 0, 10, 4092, 0, 47, 0, 1834, 31, 0, 2, 318, 238, 4085, 1576, 0, 0, 4095, 0, 0, 12, 3898, 0, 28, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 4090, 3982, 4096, 0, 3918, 3306, 3598, 0, 23, 4073, 0, 0, 0, 2579, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 51, 14, 4068, 0, 0, 4, 28, 3374, 3084, 0, 0, 0, 1772, 0, 4089, 9, 0, 79, 0, 1, 4, 2, 1757, 4096, 0, 2773, 4096, 50, 4039, 4096, 1, 0, 0, 0, 0, 4061, 4095, 57, 0, 3983, 0, 3610, 0, 4096, 3330, 0, 32, 0, 30, 0, 0, 0, 52, 4019, 0, 439, 0, 0, 4096, 946, 3775, 52, 1, 0, 988, 264, 0, 3886, 0, 0, 38, 0, 19, 3, 3903, 2530, 12, 11, 31, 4096, 0, 4015, 0, 4091, 2626, 0, 525, 4, 561, 0, 4096, 1683, 4096, 195, 0, 61, 2, 49, 0, 0, 243, 0, 0, 0, 0, 67, 348, 3351, 0, 107, 0, 4087, 715, 2163, 0, 0, 0, 44, 2, 0, 0, 0, 3903] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[333, 0, 0, 207, 1077, 0, 2977, 404, 53, 0, 1, 3527, 3781, 3375, 0, 0, 0, 0, 37, 4093, 4095, 103, 1, 0, 15, 3690, 4090, 2, 4015, 0, 0, 2759, 2965, 341, 47, 1, 0, 11, 0, 0, 0, 3594, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 29, 0, 835, 3, 128, 0, 2, 3983, 3081, 0, 0, 0, 49, 3902, 3670, 0, 13, 0, 15, 52, 3, 0, 0, 1358, 11, 2688, 3846, 3132, 0, 0, 115, 0, 1457, 2492, 0, 0, 0, 1074, 0, 0, 4096, 4095, 570, 0, 4096, 255, 4096, 2908, 4021, 4071, 3728, 4096, 0, 1, 1221, 0, 0, 0, 0, 3, 801, 739, 1738, 4057, 770, 74, 1, 4002, 0, 0, 0, 0, 1, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1, 1585, 3741, 0, 298, 136, 0, 108, 3903, 0, 2802, 1236, 191, 3585, 0, 0, 0, 245, 49, 0, 3987, 0, 929, 376, 387, 46, 751, 0, 2600, 0, 11, 4079, 4090, 3040, 0, 0, 8, 53, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 2254, 75, 843, 0, 235, 3738, 1, 0, 0, 0, 20, 26, 2, 1182, 3158, 0, 0, 4090, 0, 120, 0, 0, 2839, 4064, 0, 28, 4088, 2282, 24, 0, 10, 0, 34, 4096, 87, 0, 4086, 10, 9, 890, 5, 0, 2656, 1089, 2205, 0, 0, 3957, 3914, 2401, 4070, 4011, 113, 4029, 0, 0, 0, 17, 0, 0, 25, 4096, 0, 4028, 11, 4095, 0, 3865, 0, 5, 3849, 541, 0, 1465, 48, 0, 0, 1, 0, 35] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1788, 0, 0, 0, 20, 0, 3850, 3, 0, 0, 3233, 0, 4096, 4083, 19, 0, 0, 0, 5, 12, 2805, 117, 4053, 0, 0, 991, 3997, 0, 4077, 0, 4, 3966, 7, 3953, 4089, 0, 0, 0, 0, 0, 485, 4094, 312, 0, 2310, 2, 15, 0, 4089, 3934, 0, 0, 28, 5, 1, 0, 7, 1418, 8, 4, 0, 18, 0, 3694, 3, 1273, 0, 0, 392, 0, 1090, 0, 22, 0, 3807, 968, 0, 0, 0, 0, 73, 4089, 646, 82, 2968, 1383, 0, 0, 1643, 1, 0, 4096, 4063, 0, 0, 4095, 0, 4069, 4095, 3887, 0, 2720, 4, 2, 1, 0, 0, 0, 0, 0, 0, 2, 4030, 0, 258, 0, 3466, 1, 4055, 0, 0, 0, 0, 4094, 0, 11, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3838, 0, 0, 134, 0, 4066, 11, 5, 5, 0, 5, 25, 4088, 41, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 434, 1164, 4080, 0, 2, 0, 2, 0, 4077, 1, 21, 44, 44, 2743, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4096, 0, 0, 0, 0, 0, 484, 4095, 168, 0, 0, 3325, 2940, 0, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2719, 0, 0, 0, 4, 0, 0, 0, 3928, 3999, 0, 1930, 0, 0, 3830, 560, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2610, 199, 0, 1, 565, 0, 4035, 0, 130, 119, 0, 0, 1, 0, 1428, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2623, 936, 1, 735, 2006, 3760, 0, 4096, 0, 1301, 69, 4096, 125, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 4096, 4081, 3426, 3896, 34, 0, 0, 0, 0, 0, 0, 0, 15, 0, 2668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3525, 0, 1, 0, 103, 1849, 2290, 0, 2162, 0, 154, 4059, 0, 0, 148, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12365, 13817, 19481, 17249, 9412, 22276, 18337, 18135] avg=16384.0000 max=22276 max_rank=5 min=9412 min_rank=4 imbalance_ratio=1.359619 max_load_reduction=108796 max_load_reduction_pct=0.830048 moved_to_echo=68160 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3587, 0, 0, 0, 0, 4, 3892, 0, 25, 0, 3735, 0, 4084, 4096, 0, 2, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 13, 20, 2907, 0, 0, 0, 3626, 0, 4093, 0, 1, 2630, 1190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4095, 3394, 0, 0, 0, 0, 2, 4038, 3201, 0, 0, 0, 0, 0, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3691, 1112, 0, 12, 861, 4096, 4096, 0, 0, 0, 4036, 108, 0, 159, 388, 2114, 636, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3421, 0, 0, 0, 28, 4043, 172, 1410, 0, 4090, 0, 1609, 0, 0, 1546, 1419, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2978, 0, 149, 7, 0, 0, 1453, 3136, 0, 15, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 644, 0, 4095, 259, 2638, 678, 7, 0, 22, 29, 0, 185, 73, 2, 0, 0, 2655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4064, 4043, 4086, 19, 2261, 0, 0, 423, 0, 0, 146, 0, 106, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19592, 14667, 19104, 21558, 18049, 11656, 11287, 15159] avg=16384.0000 max=21558 max_rank=3 min=11287 min_rank=6 imbalance_ratio=1.315796 max_load_reduction=109514 max_load_reduction_pct=0.835526 moved_to_echo=56151 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3926, 0, 3989, 0, 281, 9, 0, 0, 0, 0, 590, 51, 3605, 3944, 712, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 352, 9, 0, 0, 0, 0, 0, 1325, 0, 2465, 0, 4, 0, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3660, 2784, 4052, 0, 0, 0, 0, 0, 479, 4092, 76, 0, 2891, 79, 3919, 4, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 825, 3798, 230, 0, 4093, 0, 5, 0, 149, 4020, 4096, 0, 0, 150, 1293, 0, 635, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3738, 2107, 0, 570, 0, 4079, 0, 4, 614, 4090, 0, 0, 0, 3900, 0, 1426, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2976, 0, 47, 0, 1834, 31, 0, 2, 318, 238, 4085, 1576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3898, 0, 28, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 2669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3982, 4096, 0, 3918, 3306, 2028, 0, 23, 4073, 0, 0, 0, 2579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17274, 5301, 22350, 19517, 20808, 11117, 6610, 28095] avg=16384.0000 max=28095 max_rank=7 min=5301 min_rank=1 imbalance_ratio=1.714783 max_load_reduction=102977 max_load_reduction_pct=0.785652 moved_to_echo=66630 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1788, 0, 0, 0, 20, 0, 3850, 3, 0, 0, 3233, 0, 4096, 4083, 19, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 12, 2805, 117, 4053, 0, 0, 991, 3997, 0, 4077, 0, 4, 3966, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3953, 4089, 0, 0, 0, 0, 0, 485, 4094, 312, 0, 2310, 2, 15, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3934, 0, 0, 28, 5, 1, 0, 7, 1418, 8, 4, 0, 18, 0, 3694, 635, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1273, 0, 0, 392, 0, 1090, 0, 22, 0, 3807, 968, 0, 0, 0, 0, 1427, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 2973, 646, 82, 2968, 1383, 0, 0, 1643, 1, 0, 4096, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4095, 3887, 0, 2720, 4, 2, 1, 0, 0, 0, 0, 0, 0, 2, 2668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 258, 0, 3466, 1, 2287, 0, 0, 0, 0, 4094, 0, 11, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17259, 21230, 15581, 14092, 9296, 17928, 17448, 18238] avg=16384.0000 max=21230 max_rank=1 min=9296 min_rank=4 imbalance_ratio=1.295776 max_load_reduction=109842 max_load_reduction_pct=0.838028 moved_to_echo=62910 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[333, 0, 0, 207, 1077, 0, 2977, 404, 53, 0, 1, 3527, 3781, 3375, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4093, 4095, 103, 1, 0, 15, 3690, 4090, 2, 4015, 0, 0, 2759, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2965, 341, 47, 1, 0, 11, 0, 0, 0, 3594, 0, 0, 0, 0, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 29, 0, 835, 3, 128, 0, 2, 3983, 3081, 0, 0, 0, 49, 17, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 3670, 0, 13, 0, 15, 52, 3, 0, 0, 1358, 11, 2688, 3846, 3132, 0, 1428, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 1457, 2492, 0, 0, 0, 1074, 0, 0, 4096, 4095, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 4096, 2908, 4021, 4071, 3728, 4096, 0, 1, 1221, 0, 0, 0, 0, 3, 801, 2668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 739, 1738, 4057, 770, 74, 1, 2256, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15739, 24088, 6970, 12471, 20428, 13870, 27869, 9637] avg=16384.0000 max=27869 max_rank=6 min=6970 min_rank=2 imbalance_ratio=1.700989 max_load_reduction=103203 max_load_reduction_pct=0.787376 moved_to_echo=71804 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 51, 14, 4068, 0, 0, 4, 28, 3374, 3084, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1772, 0, 4089, 9, 0, 79, 0, 1, 4, 2, 1757, 4096, 0, 2773, 4096, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 4039, 4096, 1, 0, 0, 0, 0, 4061, 4095, 57, 0, 3983, 0, 3610, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3330, 0, 32, 0, 30, 0, 0, 0, 52, 4019, 0, 439, 0, 0, 4096, 4, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 946, 3775, 52, 1, 0, 988, 264, 0, 3886, 0, 0, 38, 0, 19, 3, 3903, 1428, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2530, 0, 11, 31, 4096, 0, 4015, 0, 4091, 2626, 0, 525, 4, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1683, 4096, 195, 0, 61, 2, 49, 0, 0, 243, 0, 0, 0, 0, 67, 348, 2668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3351, 0, 107, 0, 4087, 715, 1209, 0, 0, 0, 44, 2, 0, 0, 0, 3903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14722, 19324, 23997, 16235, 15474, 18490, 9412, 13418] avg=16384.0000 max=23997 max_rank=2 min=9412 min_rank=6 imbalance_ratio=1.464661 max_load_reduction=107075 max_load_reduction_pct=0.816917 moved_to_echo=56794 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 3327, 534, 314, 0, 26, 0, 1, 3821, 4074, 1, 9, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3756, 1012, 0, 149, 528, 0, 4096, 0, 3187, 0, 3, 4096, 1216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3283, 533, 107, 1, 0, 0, 7, 4084, 3682, 0, 19, 0, 43, 0, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 232, 0, 4092, 358, 6, 3800, 0, 4007, 0, 47, 20, 0, 0, 628, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 766, 4092, 0, 750, 0, 0, 3928, 10, 3940, 0, 0, 3342, 202, 0, 0, 0, 1036, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2941, 2587, 219, 4077, 1, 3459, 0, 4043, 0, 15, 4067, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 51, 3, 0, 526, 118, 0, 0, 1, 3, 1939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 673, 138, 0, 4096, 0, 1, 2309, 711, 0, 0, 0, 497, 3657, 13, 1, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16368, 18043, 12070, 17540, 18383, 21687, 10827, 16154] avg=16384.0000 max=21687 max_rank=5 min=10827 min_rank=6 imbalance_ratio=1.323669 max_load_reduction=109385 max_load_reduction_pct=0.834541 moved_to_echo=67051 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1585, 3741, 0, 298, 136, 0, 108, 3903, 0, 2802, 1236, 191, 3585, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 49, 0, 3987, 0, 929, 376, 387, 46, 751, 0, 2600, 0, 11, 4079, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3040, 0, 0, 8, 53, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2254, 75, 843, 0, 235, 3738, 1, 0, 0, 0, 20, 26, 2, 1182, 3158, 0, 637, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 120, 0, 0, 2839, 4064, 0, 28, 4088, 2282, 24, 0, 10, 0, 1379, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 2977, 87, 0, 4086, 10, 9, 890, 5, 0, 2656, 1089, 2205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 2401, 4070, 4011, 113, 4029, 0, 0, 0, 17, 0, 0, 25, 4096, 0, 4028, 2578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4095, 0, 3865, 0, 5, 2170, 541, 0, 1465, 48, 0, 0, 1, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17753, 14602, 11519, 12410, 19222, 14048, 29282, 12236] avg=16384.0000 max=29282 max_rank=6 min=11519 min_rank=2 imbalance_ratio=1.787231 max_load_reduction=101790 max_load_reduction_pct=0.776596 moved_to_echo=74788 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[19200, 19072, 1152, 384, 5248, 12800, 4480, 256, 3968, 9472, 24192, 3328, 2432, 1536, 4864, 13824, 3840, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[14848, 25728, 20352, 128, 128, 128, 0, 128, 13696, 30976, 640, 128, 9216, 3456, 10496, 128, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 2048, 896, 9728, 18816, 1280, 8064, 640, 1024, 4736, 22016, 1792, 24576, 128, 2944, 24448, 8960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[17664, 1664, 7808, 384, 5888, 7680, 11392, 4992, 4096, 128, 10496, 4992, 27136, 26368, 768, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2688, 17664, 4352, 2048, 18560, 5248, 11392, 2432, 18176, 3072, 4352, 18048, 20224, 1280, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[5632, 24192, 4864, 1152, 1024, 1152, 16896, 4352, 13312, 768, 17664, 6784, 4608, 3968, 7168, 5504, 11008, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[16512, 9984, 12672, 12800, 15744, 6016, 16896, 1280, 2304, 6016, 256, 8704, 3840, 2688, 256, 16000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[6528, 14848, 19584, 24320, 10368, 15104, 8320, 128, 128, 1536, 640, 384, 128, 4224, 128, 5248, 20608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[61, 2750, 455, 400, 1420, 383, 724, 1127, 1069, 144, 1477, 281, 731, 672, 151, 559, 265, 379, 863, 1104, 732, 146, 3075, 2129, 212, 1061, 820, 1051, 1169, 421, 1440, 1852, 2977, 385, 247, 470, 118, 1882, 2279, 530, 2715, 433, 1396, 970, 1581, 1315, 1085, 669, 269, 271, 293, 2971, 869, 367, 655, 38, 289, 218, 824, 1587, 1372, 458, 1144, 1009, 553, 1738, 509, 836, 823, 211, 689, 1487, 848, 2009, 449, 1783, 555, 963, 2387, 665, 2522, 1103, 2934, 979, 2028, 476, 814, 1178, 1565, 193, 619, 1382, 738, 492, 598, 660, 2318, 179, 1457, 758, 1319, 602, 1464, 2796, 1057, 832, 550, 2199, 687, 313, 2071, 743, 220, 1302, 1358, 1830, 356, 677, 1091, 1211, 792, 1770, 1592, 744, 1880, 340, 440, 529] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[94, 2832, 834, 398, 1051, 344, 747, 965, 1777, 148, 1274, 406, 1066, 633, 145, 408, 87, 393, 1426, 1057, 1011, 165, 2663, 2104, 550, 836, 738, 768, 1976, 433, 2256, 1828, 2857, 366, 594, 938, 90, 1961, 1517, 601, 2816, 759, 1503, 927, 1408, 1845, 815, 730, 249, 431, 233, 2226, 1022, 501, 624, 68, 524, 547, 1013, 1795, 1695, 361, 866, 921, 952, 1509, 414, 1055, 653, 205, 501, 1640, 601, 1581, 308, 2114, 481, 878, 2255, 560, 2650, 931, 2326, 376, 2184, 476, 836, 1138, 1556, 233, 572, 1309, 722, 594, 451, 841, 2441, 414, 1488, 675, 1710, 1046, 1832, 2694, 901, 710, 574, 2087, 507, 140, 1771, 917, 171, 1261, 799, 1441, 277, 560, 1162, 1086, 785, 1359, 1251, 911, 1445, 666, 471, 402] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[62, 2763, 393, 303, 1043, 441, 561, 1349, 1243, 246, 1236, 176, 1283, 1197, 183, 375, 118, 383, 1719, 648, 994, 271, 2442, 1894, 329, 1186, 907, 1176, 1077, 310, 2163, 2449, 2627, 663, 498, 425, 205, 2445, 1982, 603, 2424, 274, 1638, 609, 1525, 1416, 1230, 1591, 306, 321, 415, 1306, 1048, 608, 596, 64, 756, 533, 748, 1769, 1923, 776, 548, 1026, 976, 1594, 498, 1238, 978, 149, 733, 1716, 1066, 2205, 202, 1807, 740, 808, 2691, 722, 2334, 1334, 3027, 368, 2095, 178, 505, 1235, 1170, 246, 646, 1001, 1022, 586, 398, 696, 1579, 402, 1638, 377, 1356, 675, 1819, 2844, 325, 766, 482, 2522, 488, 345, 1787, 1324, 274, 1165, 883, 1294, 197, 489, 590, 1178, 908, 1043, 1551, 400, 2246, 278, 456, 609] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[110, 2608, 860, 163, 1302, 276, 819, 1141, 1098, 167, 971, 231, 907, 472, 130, 319, 101, 483, 741, 903, 1036, 37, 3007, 1715, 509, 765, 1023, 992, 1660, 145, 1400, 1750, 2821, 629, 245, 494, 121, 1986, 2123, 328, 2741, 373, 1379, 950, 1355, 1255, 1009, 708, 370, 381, 329, 2659, 615, 427, 380, 110, 378, 187, 780, 1581, 1120, 375, 1130, 1155, 596, 2315, 893, 1137, 643, 259, 445, 1642, 734, 1801, 332, 2144, 898, 901, 2492, 804, 2839, 890, 2632, 645, 1960, 350, 747, 1254, 2451, 322, 638, 1063, 720, 614, 518, 931, 1764, 272, 2095, 525, 1430, 665, 1990, 2493, 701, 973, 904, 2282, 914, 373, 1941, 719, 351, 1189, 1272, 1490, 533, 588, 1046, 1402, 929, 1590, 1804, 1178, 1122, 528, 670, 394] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[94, 3049, 447, 455, 1064, 323, 960, 731, 980, 118, 1942, 103, 854, 769, 225, 404, 164, 456, 1165, 423, 1010, 294, 3090, 2347, 614, 1567, 806, 1147, 1358, 401, 1756, 1868, 3236, 732, 514, 598, 40, 2361, 1578, 547, 2528, 450, 1574, 943, 1296, 1522, 1034, 1141, 362, 276, 363, 1797, 688, 629, 311, 72, 427, 145, 1157, 2209, 1171, 589, 463, 1167, 594, 1851, 616, 900, 940, 139, 411, 1740, 1068, 2240, 143, 1659, 697, 1798, 2623, 1150, 2825, 1053, 3047, 467, 1658, 218, 705, 1167, 1399, 199, 852, 1164, 772, 920, 342, 926, 2036, 189, 2021, 628, 1195, 255, 1859, 2559, 441, 844, 407, 1207, 386, 429, 2584, 816, 392, 1078, 878, 1043, 198, 637, 1336, 1110, 685, 1543, 1580, 562, 1893, 172, 376, 546] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[117, 3321, 309, 303, 1375, 360, 645, 1332, 1123, 173, 1427, 170, 449, 737, 352, 425, 85, 585, 1132, 619, 919, 235, 2157, 2163, 419, 1076, 651, 1443, 1307, 436, 1714, 1869, 3066, 445, 542, 595, 40, 2303, 2401, 288, 2915, 729, 1339, 844, 1589, 1703, 757, 1140, 245, 475, 385, 2242, 1245, 564, 560, 62, 294, 429, 822, 2024, 1230, 466, 1148, 1334, 822, 2163, 641, 1255, 635, 334, 401, 1711, 794, 1349, 197, 1929, 556, 868, 2880, 542, 2178, 1221, 2921, 329, 1694, 360, 813, 1143, 1479, 185, 783, 1082, 697, 618, 488, 831, 2166, 279, 1723, 592, 1442, 802, 1586, 3036, 810, 961, 417, 2031, 339, 314, 2431, 536, 129, 807, 1239, 1606, 134, 494, 1142, 1348, 1027, 1369, 1816, 738, 1320, 157, 264, 469] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[95, 2582, 342, 297, 1457, 488, 566, 1230, 930, 174, 1175, 153, 998, 1046, 127, 322, 51, 533, 629, 611, 1337, 85, 3263, 2001, 337, 1052, 696, 1389, 751, 467, 1376, 1810, 3161, 435, 177, 416, 192, 2563, 2824, 321, 2465, 297, 1486, 749, 1295, 1846, 1270, 1451, 361, 305, 258, 2520, 949, 311, 401, 94, 966, 201, 963, 2184, 978, 620, 1087, 747, 1075, 1811, 646, 836, 1454, 209, 1054, 1636, 979, 1631, 211, 1582, 986, 1049, 2725, 1200, 2893, 923, 3149, 382, 1606, 92, 464, 875, 1669, 215, 625, 1450, 761, 674, 462, 1021, 1950, 185, 1601, 490, 1172, 462, 1425, 2150, 506, 1032, 582, 2190, 420, 243, 2399, 1218, 206, 1180, 1710, 1010, 279, 636, 409, 1126, 1172, 2060, 1686, 866, 1127, 203, 320, 449] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[186, 2801, 496, 405, 948, 213, 803, 874, 1154, 164, 1707, 517, 623, 772, 483, 391, 48, 530, 922, 555, 807, 341, 2781, 1960, 553, 1044, 907, 1103, 1335, 553, 1856, 1915, 2803, 557, 747, 500, 159, 2025, 1823, 845, 2509, 693, 1755, 927, 1158, 1916, 771, 1009, 207, 412, 327, 2114, 763, 661, 329, 71, 794, 560, 1153, 1779, 1612, 331, 1099, 973, 661, 1738, 630, 1149, 886, 195, 760, 1628, 593, 1639, 232, 1350, 602, 952, 2237, 802, 2557, 779, 2645, 505, 1758, 330, 1075, 1000, 1777, 253, 899, 1016, 891, 617, 492, 1052, 2115, 303, 1512, 750, 1421, 702, 1494, 2124, 704, 1069, 570, 2046, 555, 331, 1942, 1041, 385, 1275, 785, 1507, 304, 582, 1212, 1609, 794, 1454, 1531, 797, 1623, 614, 409, 683] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[61, 2750, 455, 400, 1420, 383, 724, 1127, 1069, 144, 1477, 281, 731, 672, 151, 559, 2977, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 379, 863, 1104, 732, 146, 2098, 2129, 212, 1061, 820, 1051, 1169, 421, 1440, 1852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 247, 470, 118, 1882, 2279, 530, 2103, 433, 1396, 970, 1581, 1315, 1085, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 271, 293, 2971, 869, 367, 655, 38, 289, 218, 824, 1587, 1372, 458, 1144, 1009, 1727, 1198, 837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 1738, 509, 836, 823, 211, 689, 1487, 848, 2009, 449, 1783, 555, 963, 1437, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2522, 1103, 1736, 979, 2028, 476, 814, 1178, 1565, 193, 619, 1382, 738, 492, 598, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2318, 179, 1457, 758, 1319, 602, 1464, 0, 1057, 832, 550, 2199, 687, 313, 2037, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 1302, 1358, 1830, 356, 677, 1091, 1211, 792, 1770, 1592, 744, 1880, 340, 440, 529, 977, 612, 113, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16450, 15742, 15463, 16396, 15555, 17083, 16515, 17868] avg=16384.0000 max=17868 max_rank=7 min=15463 min_rank=2 imbalance_ratio=1.090576 max_load_reduction=113204 max_load_reduction_pct=0.863678 moved_to_echo=67021 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[62, 2763, 393, 303, 1043, 441, 561, 1349, 1243, 246, 1236, 176, 1283, 1197, 183, 375, 2627, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 383, 1719, 648, 994, 271, 1666, 1894, 329, 1186, 907, 1176, 1077, 310, 2163, 2449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 663, 498, 425, 205, 2445, 1982, 603, 1877, 274, 1638, 609, 1525, 1416, 1230, 1591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 321, 415, 1306, 1048, 608, 596, 64, 756, 533, 748, 1769, 1923, 776, 548, 1026, 1757, 1236, 944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1594, 498, 1238, 978, 149, 733, 1716, 1066, 2205, 202, 1807, 740, 808, 1620, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2334, 1334, 1791, 368, 2095, 178, 505, 1235, 1170, 246, 646, 1001, 1022, 586, 398, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1579, 402, 1638, 377, 1356, 675, 1819, 0, 325, 766, 482, 2522, 488, 345, 1758, 1324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 1165, 883, 1294, 197, 489, 590, 1178, 908, 1043, 1551, 400, 2246, 278, 456, 609, 776, 547, 127, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16568, 17290, 16981, 16680, 17052, 15605, 15856, 15040] avg=16384.0000 max=17290 max_rank=1 min=15040 min_rank=7 imbalance_ratio=1.055298 max_load_reduction=113782 max_load_reduction_pct=0.868088 moved_to_echo=63553 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[94, 2832, 834, 398, 1051, 344, 747, 965, 1777, 148, 1274, 406, 1066, 633, 145, 408, 2857, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 393, 1426, 1057, 1011, 165, 1817, 2104, 550, 836, 738, 768, 1976, 433, 2256, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 366, 594, 938, 90, 1961, 1517, 601, 2181, 759, 1503, 927, 1408, 1845, 815, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 431, 233, 2226, 1022, 501, 624, 68, 524, 547, 1013, 1795, 1695, 361, 866, 921, 1664, 949, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 952, 1509, 414, 1055, 653, 205, 501, 1640, 601, 1581, 308, 2114, 481, 878, 1357, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2650, 931, 1377, 376, 2184, 476, 836, 1138, 1556, 233, 572, 1309, 722, 594, 451, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2441, 414, 1488, 675, 1710, 1046, 1832, 0, 901, 710, 574, 2087, 507, 140, 1742, 917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 1261, 799, 1441, 277, 560, 1162, 1086, 785, 1359, 1251, 911, 1445, 666, 471, 402, 846, 635, 107, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17009, 17445, 16235, 16480, 14809, 16246, 17184, 15664] avg=16384.0000 max=17445 max_rank=1 min=14809 min_rank=4 imbalance_ratio=1.064758 max_load_reduction=113627 max_load_reduction_pct=0.866905 moved_to_echo=63903 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[94, 3049, 447, 455, 1064, 323, 960, 731, 980, 118, 1942, 103, 854, 769, 225, 404, 3236, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 456, 1165, 423, 1010, 294, 2108, 2347, 614, 1567, 806, 1147, 1358, 401, 1756, 1868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 732, 514, 598, 40, 2361, 1578, 547, 1958, 450, 1574, 943, 1296, 1522, 1034, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 276, 363, 1797, 688, 629, 311, 72, 427, 145, 1157, 2209, 1171, 589, 463, 1167, 1581, 1244, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 1851, 616, 900, 940, 139, 411, 1740, 1068, 2240, 143, 1659, 697, 1798, 1579, 1150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2825, 1053, 1803, 467, 1658, 218, 705, 1167, 1399, 199, 852, 1164, 772, 920, 342, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2036, 189, 2021, 628, 1195, 255, 1859, 0, 441, 844, 407, 1207, 386, 429, 2542, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 1078, 878, 1043, 198, 637, 1336, 1110, 685, 1543, 1580, 562, 1893, 172, 376, 546, 982, 570, 124, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16732, 17484, 16288, 15571, 17525, 16470, 15255, 15747] avg=16384.0000 max=17525 max_rank=4 min=15255 min_rank=6 imbalance_ratio=1.069641 max_load_reduction=113547 max_load_reduction_pct=0.866295 moved_to_echo=64997 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[117, 3321, 309, 303, 1375, 360, 645, 1332, 1123, 173, 1427, 170, 449, 737, 352, 425, 3066, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 585, 1132, 619, 919, 235, 1472, 2163, 419, 1076, 651, 1443, 1307, 436, 1714, 1869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 542, 595, 40, 2303, 2401, 288, 2257, 729, 1339, 844, 1589, 1703, 757, 1140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 475, 385, 2242, 1245, 564, 560, 62, 294, 429, 822, 2024, 1230, 466, 1148, 1334, 1874, 1192, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 822, 2163, 641, 1255, 635, 334, 401, 1711, 794, 1349, 197, 1929, 556, 868, 1734, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2178, 1221, 1729, 329, 1694, 360, 813, 1143, 1479, 185, 783, 1082, 697, 618, 488, 831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2166, 279, 1723, 592, 1442, 802, 1586, 0, 810, 961, 417, 2031, 339, 314, 2391, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 807, 1239, 1606, 134, 494, 1142, 1348, 1027, 1369, 1816, 738, 1320, 157, 264, 469, 685, 658, 136, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16846, 16125, 16972, 17601, 15931, 15630, 16389, 15578] avg=16384.0000 max=17601 max_rank=3 min=15578 min_rank=7 imbalance_ratio=1.074280 max_load_reduction=113471 max_load_reduction_pct=0.865715 moved_to_echo=63528 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[110, 2608, 860, 163, 1302, 276, 819, 1141, 1098, 167, 971, 231, 907, 472, 130, 319, 2821, 954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 483, 741, 903, 1036, 37, 2048, 1715, 509, 765, 1023, 992, 1660, 145, 1400, 1750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 245, 494, 121, 1986, 2123, 328, 2119, 373, 1379, 950, 1355, 1255, 1009, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 381, 329, 2659, 615, 427, 380, 110, 378, 187, 780, 1581, 1120, 375, 1130, 1155, 1539, 1078, 877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 596, 2315, 893, 1137, 643, 259, 445, 1642, 734, 1801, 332, 2144, 898, 901, 1492, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2839, 890, 1554, 645, 1960, 350, 747, 1254, 2451, 322, 638, 1063, 720, 614, 518, 931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1764, 272, 2095, 525, 1430, 665, 1990, 0, 701, 973, 904, 2282, 914, 373, 1907, 719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 1189, 1272, 1490, 533, 588, 1046, 1402, 929, 1590, 1804, 1178, 1122, 528, 670, 394, 959, 622, 123, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15349, 15308, 15074, 15471, 17036, 17496, 17514, 17824] avg=16384.0000 max=17824 max_rank=7 min=15074 min_rank=2 imbalance_ratio=1.087891 max_load_reduction=113248 max_load_reduction_pct=0.864014 moved_to_echo=69870 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[186, 2801, 496, 405, 948, 213, 803, 874, 1154, 164, 1707, 517, 623, 772, 483, 391, 2803, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 530, 922, 555, 807, 341, 1898, 1960, 553, 1044, 907, 1103, 1335, 553, 1856, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 747, 500, 159, 2025, 1823, 845, 1943, 693, 1755, 927, 1158, 1916, 771, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 412, 327, 2114, 763, 661, 329, 71, 794, 560, 1153, 1779, 1612, 331, 1099, 973, 1311, 1080, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 661, 1738, 630, 1149, 886, 195, 760, 1628, 593, 1639, 232, 1350, 602, 952, 1347, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2557, 779, 1565, 505, 1758, 330, 1075, 1000, 1777, 253, 899, 1016, 891, 617, 492, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2115, 303, 1512, 750, 1421, 702, 1494, 0, 704, 1069, 570, 2046, 555, 331, 1910, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 1275, 785, 1507, 304, 582, 1212, 1609, 794, 1454, 1531, 797, 1623, 614, 409, 683, 883, 566, 106, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16153, 16327, 16828, 16360, 15164, 16566, 16523, 17151] avg=16384.0000 max=17151 max_rank=7 min=15164 min_rank=4 imbalance_ratio=1.046814 max_load_reduction=113921 max_load_reduction_pct=0.869148 moved_to_echo=65404 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[95, 2582, 342, 297, 1457, 488, 566, 1230, 930, 174, 1175, 153, 998, 1046, 127, 322, 3161, 822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 533, 629, 611, 1337, 85, 2226, 2001, 337, 1052, 696, 1389, 751, 467, 1376, 1810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 177, 416, 192, 2563, 2824, 321, 1909, 297, 1486, 749, 1295, 1846, 1270, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 305, 258, 2520, 949, 311, 401, 94, 966, 201, 963, 2184, 978, 620, 1087, 747, 1328, 1285, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1811, 646, 836, 1454, 209, 1054, 1636, 979, 1631, 211, 1582, 986, 1049, 1641, 1200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2893, 923, 1864, 382, 1606, 92, 464, 875, 1669, 215, 625, 1450, 761, 674, 462, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1950, 185, 1601, 490, 1172, 462, 1425, 0, 506, 1032, 582, 2190, 420, 243, 2360, 1218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 1180, 1710, 1010, 279, 636, 409, 1126, 1172, 2060, 1686, 866, 1127, 203, 320, 449, 1037, 556, 129, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15965, 15351, 17231, 16513, 18000, 15976, 15836, 16200] avg=16384.0000 max=18000 max_rank=4 min=15351 min_rank=1 imbalance_ratio=1.098633 max_load_reduction=113072 max_load_reduction_pct=0.862671 moved_to_echo=66012 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1024, 3840, 8704, 6016, 7936, 1664, 15360, 16384, 3584, 8704, 6656, 9088, 10752, 3200, 14080, 15360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[16384, 2304, 13568, 4864, 11136, 5248, 13568, 0, 5504, 7296, 4608, 16640, 4352, 2560, 16768, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[896, 22784, 4224, 2816, 9728, 2944, 5888, 8832, 9472, 1408, 11264, 2048, 6912, 6400, 1920, 3328, 23552, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[2432, 2944, 2688, 17920, 7296, 4096, 3968, 640, 4480, 2944, 7552, 14976, 11136, 4096, 7552, 8448, 12800, 9344, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[6272, 14720, 4864, 8448, 7040, 1792, 5120, 13312, 6784, 14464, 2176, 14464, 5632, 8320, 12288, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[20864, 8320, 13440, 4096, 15104, 2560, 6016, 9088, 13184, 1920, 5760, 9472, 6400, 5120, 3840, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 4224, 3584, 4480, 1024, 17536, 16640, 4096, 16384, 4096, 12160, 7040, 11264, 12928, 8064, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2176, 9344, 8960, 11264, 2304, 4736, 8064, 10112, 7168, 12288, 12928, 6272, 12672, 3072, 3456, 4096, 7168, 4864, 1024, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4095, 0, 3409, 2, 2258, 1, 872, 2707, 1629, 35, 192, 18, 978, 39, 62, 3124, 525, 0, 1733, 299, 9, 1887, 89, 2331, 0, 3, 0, 540, 0, 4057, 3003, 12, 1760, 1234, 3879, 78, 4, 0, 112, 16, 2241, 1, 0, 0, 13, 104, 444, 93, 0, 2671, 136, 0, 141, 1898, 934, 0, 28, 3842, 430, 3826, 419, 7, 1, 0, 0, 0, 4082, 9, 141, 6, 4076, 20, 3583, 127, 2978, 665, 338, 0, 3548, 0, 4080, 2320, 163, 34, 2789, 9, 455, 4039, 486, 152, 3358, 3200, 1, 612, 104, 1, 0, 1970, 4082, 510, 2723, 0, 930, 2, 21, 106, 283, 22, 3208, 2999, 613, 2609, 3589, 122, 311, 12, 1, 2249, 428, 117, 1522, 1, 0, 361, 0, 49, 2, 2633] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[3510, 0, 3090, 38, 3187, 0, 601, 4015, 3046, 956, 0, 10, 2850, 164, 38, 2702, 70, 927, 212, 5, 1, 2611, 0, 46, 1653, 3287, 16, 157, 0, 2039, 3235, 12, 3048, 97, 4091, 0, 8, 1, 48, 4, 22, 113, 9, 268, 561, 1208, 85, 24, 5, 63, 2, 11, 55, 1568, 297, 7, 47, 4090, 1549, 1662, 55, 22, 36, 0, 29, 840, 3504, 0, 303, 98, 4014, 6, 3592, 901, 0, 18, 520, 8, 4062, 1, 3792, 289, 460, 3129, 3817, 4, 216, 4075, 14, 116, 1222, 4073, 1, 9, 0, 2726, 277, 478, 4077, 42, 2098, 1, 1796, 48, 0, 1151, 322, 561, 3564, 11, 22, 2788, 462, 42, 601, 13, 4, 3636, 233, 0, 3002, 9, 0, 1740, 2210, 216, 0, 2295] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[3742, 9, 4032, 801, 4085, 0, 16, 4055, 1851, 183, 1904, 20, 356, 555, 1, 2287, 0, 0, 703, 0, 685, 3687, 4, 2514, 15, 322, 3, 289, 0, 1341, 4014, 0, 2796, 1591, 3779, 0, 0, 0, 104, 8, 106, 0, 228, 0, 1, 2, 4, 165, 2, 520, 91, 0, 457, 3033, 10, 1, 0, 3773, 352, 1351, 3586, 0, 4, 0, 1, 911, 4091, 330, 1001, 429, 3606, 3, 3342, 658, 234, 483, 21, 0, 3001, 277, 4089, 634, 940, 0, 2891, 176, 332, 3955, 2, 548, 686, 2512, 183, 106, 16, 160, 0, 4, 3907, 222, 3933, 0, 3324, 0, 0, 916, 226, 2, 4081, 2181, 3, 3537, 187, 1613, 2040, 9, 0, 2026, 4, 622, 1333, 583, 6, 1139, 20, 2, 2, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4036, 0, 4010, 156, 3718, 0, 76, 3578, 2620, 16, 36, 269, 568, 26, 131, 1302, 2, 0, 395, 14, 342, 2860, 534, 2896, 29, 725, 71, 669, 0, 2209, 3884, 67, 701, 1418, 3198, 0, 0, 6, 0, 38, 3278, 35, 2, 2, 90, 383, 15, 1, 4, 2, 26, 3, 0, 2602, 1, 19, 4, 4094, 126, 2296, 1164, 0, 255, 119, 6, 506, 4024, 2018, 1996, 0, 3996, 30, 673, 1325, 1348, 188, 550, 0, 1751, 0, 3972, 2461, 468, 105, 3965, 0, 1396, 4071, 0, 3785, 6, 4069, 1031, 0, 0, 569, 11, 2149, 4080, 1978, 3760, 0, 3674, 0, 0, 461, 3817, 0, 2709, 789, 95, 292, 64, 7, 801, 137, 0, 2810, 11, 411, 333, 61, 0, 419, 15, 28, 424, 2306] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2357, 0, 3367, 139, 2372, 10, 1, 3682, 3274, 1108, 61, 133, 1136, 587, 0, 460, 7, 0, 3490, 95, 1508, 2862, 3, 91, 0, 1549, 5, 265, 0, 3184, 3311, 0, 2043, 11, 3586, 12, 9, 0, 89, 2, 258, 460, 34, 0, 0, 0, 566, 0, 0, 58, 0, 0, 1925, 3340, 0, 0, 120, 4096, 2, 316, 3755, 1, 2, 30, 0, 0, 3031, 91, 1629, 134, 1520, 0, 799, 598, 0, 0, 543, 0, 3725, 0, 3920, 3487, 1255, 583, 4029, 93, 0, 4089, 1, 8, 987, 4055, 3302, 1252, 0, 3196, 0, 401, 3825, 2718, 2884, 0, 2935, 184, 20, 1739, 604, 314, 3696, 6, 68, 128, 88, 499, 261, 423, 15, 1536, 66, 499, 1721, 50, 2, 3935, 76, 312, 127, 3841] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1252, 0, 2166, 1730, 1411, 0, 15, 3547, 3883, 1337, 62, 5, 2195, 2581, 167, 1278, 0, 1, 658, 87, 304, 3707, 554, 14, 9, 7, 0, 19, 0, 3570, 4096, 0, 2743, 176, 4005, 0, 3, 4, 380, 1, 2412, 0, 113, 2, 1184, 991, 3231, 0, 7, 624, 0, 0, 3859, 3316, 1, 43, 114, 4087, 3390, 148, 2560, 243, 0, 2, 1, 275, 4074, 202, 0, 18, 3929, 0, 202, 750, 23, 3, 178, 0, 4079, 125, 4093, 1906, 2748, 2067, 3723, 7, 558, 4044, 7, 22, 1295, 3382, 1, 12, 2, 228, 0, 214, 1292, 998, 4073, 0, 221, 0, 25, 76, 1658, 19, 2045, 428, 78, 111, 2413, 553, 198, 0, 1, 3053, 6, 98, 1300, 3, 0, 2191, 465, 31, 104, 3170] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2876, 0, 4078, 1441, 1245, 0, 112, 3783, 3718, 2887, 2, 0, 2400, 1151, 0, 3561, 0, 3, 2467, 13, 6, 2848, 1287, 4, 22, 49, 0, 4, 279, 2001, 3358, 10, 3932, 971, 4078, 1, 477, 1, 0, 40, 1, 12, 59, 5, 4, 38, 0, 230, 0, 1, 467, 0, 0, 3720, 1, 1, 0, 3806, 691, 3527, 27, 3, 10, 2, 0, 20, 2496, 901, 629, 0, 3861, 112, 3705, 483, 9, 38, 120, 0, 2076, 0, 4086, 2450, 1485, 12, 3818, 1211, 3736, 3219, 0, 520, 415, 4085, 11, 11, 8, 0, 0, 351, 3823, 44, 2848, 0, 3684, 53, 0, 0, 3510, 0, 1731, 298, 695, 3961, 96, 0, 272, 29, 1, 4039, 109, 76, 506, 359, 6, 1364, 0, 48, 1, 1941] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[3921, 1, 4049, 10, 76, 7, 1076, 4015, 2020, 1558, 44, 8, 2937, 31, 7, 3941, 0, 3, 2075, 11, 106, 131, 662, 1922, 132, 833, 48, 76, 497, 2426, 3340, 0, 4057, 91, 2520, 0, 0, 31, 2187, 0, 43, 8, 32, 6, 40, 1, 0, 0, 24, 1280, 1, 428, 38, 2292, 681, 0, 0, 3843, 6, 3605, 160, 0, 35, 35, 0, 9, 4064, 19, 489, 100, 2214, 0, 4058, 378, 0, 109, 921, 1, 651, 0, 4072, 680, 2038, 29, 4066, 1, 424, 4080, 0, 3096, 959, 3065, 811, 1691, 369, 3213, 0, 2725, 4081, 144, 2256, 0, 3351, 11, 227, 196, 3199, 173, 489, 99, 1, 1006, 1046, 4, 3978, 0, 526, 3628, 1, 0, 84, 0, 11, 679, 84, 31, 0, 4028] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1128, 2, 2258, 1, 872, 0, 1629, 35, 192, 18, 978, 39, 62, 3124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 1733, 299, 9, 1887, 89, 2331, 0, 3, 0, 540, 0, 4057, 3003, 12, 2128, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1760, 1234, 3879, 78, 4, 0, 112, 16, 2241, 1, 0, 0, 13, 104, 444, 93, 4080, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2671, 136, 0, 141, 1898, 934, 0, 28, 3842, 430, 3826, 419, 7, 1, 0, 2755, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 9, 141, 6, 4076, 20, 3583, 127, 2978, 665, 338, 0, 3548, 0, 205, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2320, 163, 34, 2584, 9, 455, 0, 486, 152, 3358, 3200, 1, 612, 104, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1970, 989, 510, 2723, 0, 930, 2, 21, 106, 283, 22, 3208, 2999, 613, 2609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3589, 122, 311, 12, 1, 2249, 428, 117, 1522, 1, 0, 361, 0, 49, 2, 2633, 2066, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14433, 16954, 16652, 17729, 19931, 13479, 16985, 14909] avg=16384.0000 max=19931 max_rank=4 min=13479 min_rank=5 imbalance_ratio=1.216492 max_load_reduction=111141 max_load_reduction_pct=0.847939 moved_to_echo=65304 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3742, 9, 1333, 801, 4085, 0, 16, 0, 1851, 183, 1904, 20, 356, 555, 1, 2287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 703, 0, 685, 3687, 4, 2514, 15, 322, 3, 289, 0, 1341, 4014, 0, 2517, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2796, 1591, 3779, 0, 0, 0, 104, 8, 106, 0, 228, 0, 1, 2, 4, 165, 4089, 2540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 520, 91, 0, 457, 3033, 10, 1, 0, 3773, 352, 1351, 3586, 0, 4, 0, 2637, 961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 911, 4091, 330, 1001, 429, 3606, 3, 3342, 658, 234, 483, 21, 0, 3001, 277, 212, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 634, 940, 0, 2679, 176, 332, 0, 2, 548, 686, 2512, 183, 106, 16, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 946, 222, 3933, 0, 3324, 0, 0, 916, 226, 2, 4081, 2181, 3, 3537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 1613, 2040, 9, 0, 2026, 4, 622, 1333, 583, 6, 1139, 20, 2, 2, 4094, 3094, 1415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17143, 16418, 15413, 16778, 18782, 8974, 19375, 18189] avg=16384.0000 max=19375 max_rank=6 min=8974 min_rank=5 imbalance_ratio=1.182556 max_load_reduction=111697 max_load_reduction_pct=0.852180 moved_to_echo=65320 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3510, 0, 1022, 38, 3187, 0, 601, 0, 3046, 956, 0, 10, 2850, 164, 38, 2702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 927, 212, 5, 1, 2611, 0, 46, 1653, 3287, 16, 157, 0, 2039, 3235, 12, 1929, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3048, 97, 4091, 0, 8, 1, 48, 4, 22, 113, 9, 268, 561, 1208, 85, 24, 3792, 2617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 63, 2, 11, 55, 1568, 297, 7, 47, 4090, 1549, 1662, 55, 22, 36, 0, 2752, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 840, 3504, 0, 303, 98, 4014, 6, 3592, 901, 0, 18, 520, 8, 4062, 1, 280, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 460, 3129, 3537, 4, 216, 0, 14, 116, 1222, 4073, 1, 9, 0, 2726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 478, 987, 42, 2098, 1, 1796, 48, 0, 1151, 322, 561, 3564, 11, 22, 2788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 42, 601, 13, 4, 3636, 233, 0, 3002, 9, 0, 1740, 2210, 216, 0, 2295, 3063, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18124, 16538, 15996, 13173, 18315, 15796, 14146, 18984] avg=16384.0000 max=18984 max_rank=7 min=13173 min_rank=3 imbalance_ratio=1.158691 max_load_reduction=112088 max_load_reduction_pct=0.855164 moved_to_echo=67241 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4036, 0, 1325, 156, 3718, 0, 76, 0, 2620, 16, 36, 269, 568, 26, 131, 1302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 395, 14, 342, 2860, 534, 2896, 29, 725, 71, 669, 0, 2209, 3884, 67, 2504, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 701, 1418, 3198, 0, 0, 6, 0, 38, 3278, 35, 2, 2, 90, 383, 15, 1, 3972, 2613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 26, 3, 0, 2602, 1, 19, 4, 4094, 126, 2296, 1164, 0, 255, 119, 2754, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 506, 4024, 2018, 1996, 0, 3996, 30, 673, 1325, 1348, 188, 550, 0, 1751, 0, 291, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2461, 468, 105, 3674, 0, 1396, 0, 0, 3785, 6, 4069, 1031, 0, 0, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2149, 988, 1978, 3760, 0, 3674, 0, 0, 461, 3817, 0, 2709, 789, 95, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 7, 801, 137, 0, 2810, 11, 411, 333, 61, 0, 419, 15, 28, 424, 2306, 2731, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14279, 17539, 15752, 14316, 18883, 17564, 20723, 12016] avg=16384.0000 max=20723 max_rank=6 min=12016 min_rank=7 imbalance_ratio=1.264832 max_load_reduction=110349 max_load_reduction_pct=0.841896 moved_to_echo=69186 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2357, 0, 1105, 139, 2372, 10, 1, 0, 3274, 1108, 61, 133, 1136, 587, 0, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3490, 95, 1508, 2862, 3, 91, 0, 1549, 5, 265, 0, 3184, 3311, 0, 2107, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2043, 11, 3586, 12, 9, 0, 89, 2, 258, 460, 34, 0, 0, 0, 566, 0, 3920, 2626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 1925, 3340, 0, 0, 120, 4096, 2, 316, 3755, 1, 2, 30, 2586, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3031, 91, 1629, 134, 1520, 0, 799, 598, 0, 0, 543, 0, 3725, 0, 300, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3487, 1255, 583, 3729, 93, 0, 0, 1, 8, 987, 4055, 3302, 1252, 0, 3196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 920, 2718, 2884, 0, 2935, 184, 20, 1739, 604, 314, 3696, 6, 68, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 499, 261, 423, 15, 1536, 66, 499, 1721, 50, 2, 3935, 76, 312, 127, 3841, 2809, 1463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12743, 18796, 13616, 17104, 12525, 21948, 16617, 17723] avg=16384.0000 max=21948 max_rank=5 min=12525 min_rank=4 imbalance_ratio=1.339600 max_load_reduction=109124 max_load_reduction_pct=0.832550 moved_to_echo=68813 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2876, 0, 1348, 1441, 1245, 0, 112, 0, 3718, 2887, 2, 0, 2400, 1151, 0, 3561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2467, 13, 6, 2848, 1287, 4, 22, 49, 0, 4, 279, 2001, 3358, 10, 2546, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 971, 4078, 1, 477, 1, 0, 40, 1, 12, 59, 5, 4, 38, 0, 230, 4086, 2067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 467, 0, 0, 3720, 1, 1, 0, 3806, 691, 3527, 27, 3, 10, 2, 2580, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2496, 901, 629, 0, 3861, 112, 3705, 483, 9, 38, 120, 0, 2076, 0, 280, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2450, 1485, 12, 3538, 1211, 3736, 0, 0, 520, 415, 4085, 11, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 926, 44, 2848, 0, 3684, 53, 0, 0, 3510, 0, 1731, 298, 695, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 272, 29, 1, 4039, 109, 76, 506, 359, 6, 1364, 0, 48, 1, 1941, 2886, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20741, 15214, 16002, 15733, 14914, 17482, 18101, 12885] avg=16384.0000 max=20741 max_rank=0 min=12885 min_rank=7 imbalance_ratio=1.265930 max_load_reduction=110331 max_load_reduction_pct=0.841759 moved_to_echo=63382 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1252, 0, 717, 1730, 1411, 0, 15, 0, 3883, 1337, 62, 5, 2195, 2581, 167, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 658, 87, 304, 3707, 554, 14, 9, 7, 0, 19, 0, 3570, 4096, 0, 1352, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2743, 176, 4005, 0, 3, 4, 380, 1, 2412, 0, 113, 2, 1184, 991, 3231, 0, 4093, 2596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 624, 0, 0, 3859, 3316, 1, 43, 114, 4087, 3390, 148, 2560, 243, 0, 2, 872, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 275, 4074, 202, 0, 18, 3929, 0, 202, 750, 23, 3, 178, 0, 4079, 125, 273, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1906, 2748, 2067, 3450, 7, 558, 0, 7, 22, 1295, 3382, 1, 12, 2, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 313, 998, 4073, 0, 221, 0, 25, 76, 1658, 19, 2045, 428, 78, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2413, 553, 198, 0, 1, 3053, 6, 98, 1300, 3, 0, 2191, 465, 31, 104, 3170, 2707, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16633, 14485, 21934, 20106, 14229, 15685, 10259, 17741] avg=16384.0000 max=21934 max_rank=2 min=10259 min_rank=6 imbalance_ratio=1.338745 max_load_reduction=109138 max_load_reduction_pct=0.832657 moved_to_echo=57914 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3921, 1, 1339, 10, 76, 7, 1076, 0, 2020, 1558, 44, 8, 2937, 31, 7, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2075, 11, 106, 131, 662, 1922, 132, 833, 48, 76, 497, 2426, 3340, 0, 2528, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 91, 2520, 0, 0, 31, 2187, 0, 43, 8, 32, 6, 40, 1, 0, 0, 4072, 2619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 1280, 1, 428, 38, 2292, 681, 0, 0, 3843, 6, 3605, 160, 0, 35, 35, 2754, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4064, 19, 489, 100, 2214, 0, 4058, 378, 0, 109, 921, 1, 651, 0, 298, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 680, 2038, 29, 3768, 1, 424, 0, 0, 3096, 959, 3065, 811, 1691, 369, 3213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2725, 989, 144, 2256, 0, 3351, 11, 227, 196, 3199, 173, 489, 99, 1, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 4, 3978, 0, 526, 3628, 1, 0, 84, 0, 11, 679, 84, 31, 0, 4028, 3064, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16976, 15128, 15707, 16133, 13493, 20144, 14866, 18625] avg=16384.0000 max=20144 max_rank=5 min=13493 min_rank=4 imbalance_ratio=1.229492 max_load_reduction=110928 max_load_reduction_pct=0.846313 moved_to_echo=67128 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[21120, 5632, 29184, 128, 512, 128, 2944, 128, 8448, 640, 512, 384, 1920, 2816, 4352, 640, 32128, 20352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 2688, 29440, 3584, 6272, 896, 27264, 256, 19968, 5248, 4608, 1536, 3200, 128, 22912, 512, 2176, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 14336, 9600, 6016, 27008, 1536, 7168, 0, 512, 8320, 8960, 28544, 5376, 3712, 512, 10112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[25856, 128, 9344, 4352, 18432, 128, 2816, 0, 22144, 8192, 2304, 512, 13440, 5248, 512, 18688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 5248, 768, 512, 6528, 21888, 2048, 128, 384, 31744, 6656, 16768, 11776, 384, 384, 256, 19712, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[640, 1024, 11776, 640, 3072, 20608, 3200, 9856, 1920, 6784, 256, 2048, 896, 20864, 28288, 128, 17664, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 8320, 7168, 6656, 24576, 128, 19968, 384, 384, 4736, 13696, 1152, 21632, 6912, 1664, 14464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[8064, 2944, 8576, 640, 640, 23040, 896, 1920, 9856, 1152, 128, 11904, 2944, 768, 768, 24320, 22528, 11392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[3936, 0, 3760, 0, 1079, 3270, 0, 129, 0, 4095, 0, 2397, 345, 48, 2, 1525, 173, 0, 5, 8, 1, 1, 3, 498, 222, 74, 4096, 7, 923, 4035, 0, 3729, 2423, 0, 0, 0, 0, 2458, 1954, 0, 4064, 56, 115, 75, 4057, 3701, 2015, 11, 3729, 4091, 0, 34, 228, 3, 257, 0, 24, 0, 37, 3984, 2687, 3936, 204, 287, 1439, 0, 4042, 1977, 14, 3867, 4093, 9, 1655, 799, 3056, 34, 0, 1, 4, 3, 18, 4096, 26, 0, 0, 4095, 2, 4094, 0, 3983, 0, 0, 0, 0, 147, 4, 1, 226, 119, 17, 5, 1, 0, 7, 0, 0, 0, 47, 0, 3757, 0, 14, 99, 0, 4058, 2333, 2012, 0, 2, 0, 4073, 317, 893, 0, 0, 8, 0, 834] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[167, 3, 4085, 54, 0, 4062, 148, 1755, 0, 3874, 0, 4096, 31, 42, 0, 0, 12, 103, 0, 180, 3, 0, 5, 0, 3728, 0, 0, 0, 15, 4066, 86, 4088, 3, 0, 399, 559, 0, 0, 902, 0, 4080, 0, 0, 2911, 4071, 161, 5, 1, 0, 3600, 1927, 3526, 0, 189, 0, 456, 4096, 0, 4, 91, 3886, 3878, 45, 3926, 21, 0, 2898, 1, 0, 1616, 4028, 0, 1807, 0, 4071, 0, 0, 0, 0, 0, 7, 4079, 1, 1007, 5, 3912, 63, 4096, 2, 1547, 0, 2, 0, 1, 3638, 1, 0, 0, 1564, 0, 36, 0, 0, 444, 32, 0, 684, 4081, 1, 4035, 197, 0, 0, 3, 4095, 4096, 3749, 0, 134, 0, 711, 759, 144, 8, 205, 0, 0, 3972] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4010, 0, 3475, 1957, 80, 3173, 0, 368, 0, 4056, 0, 2978, 2343, 1, 29, 2889, 0, 401, 1, 8, 4022, 1, 0, 0, 4, 0, 3, 2, 0, 4095, 3, 58, 3688, 0, 10, 1575, 745, 221, 22, 0, 3896, 833, 0, 1, 1931, 4076, 1, 48, 0, 3843, 2, 0, 0, 57, 1473, 1097, 0, 0, 0, 1027, 4080, 2581, 4091, 3663, 464, 1, 3142, 26, 5, 1072, 2501, 370, 29, 0, 2253, 2, 1, 0, 2, 0, 10, 3178, 0, 1307, 202, 3550, 89, 4096, 0, 4034, 0, 3, 1, 0, 2988, 31, 2, 0, 1150, 1108, 625, 152, 0, 251, 21, 0, 4087, 3, 56, 3895, 0, 0, 0, 0, 3992, 3982, 1335, 118, 16, 115, 17, 70, 1730, 12, 1836, 0, 1552, 2672] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1023, 0, 678, 3237, 0, 4096, 0, 2363, 2, 3800, 0, 4026, 3277, 10, 0, 0, 72, 533, 75, 0, 7, 48, 565, 0, 506, 3642, 0, 0, 77, 4018, 1232, 302, 2133, 1, 0, 689, 66, 1, 64, 0, 4085, 3, 0, 3705, 3605, 276, 3662, 6, 0, 3883, 1928, 0, 0, 18, 558, 6, 1, 0, 0, 3948, 4083, 4089, 3069, 1889, 52, 0, 3716, 0, 15, 2, 3945, 0, 1, 0, 4090, 56, 22, 2, 0, 0, 152, 94, 4018, 755, 1, 4096, 3936, 4070, 264, 0, 4, 166, 0, 0, 0, 0, 0, 1, 4078, 0, 110, 23, 0, 149, 0, 31, 1368, 1293, 4, 4094, 0, 0, 197, 100, 4094, 3652, 226, 4, 0, 142, 87, 3, 28, 0, 4, 4, 2702, 3864] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4055, 0, 3984, 74, 25, 4073, 0, 958, 0, 4003, 0, 4076, 3864, 38, 5, 0, 0, 309, 0, 46, 1861, 112, 7, 8, 3779, 8, 5, 0, 4093, 4034, 0, 166, 3990, 0, 0, 35, 0, 945, 448, 2, 3686, 0, 234, 3, 1483, 2999, 0, 21, 2, 4087, 0, 0, 1, 0, 1, 194, 18, 0, 284, 1396, 4017, 3875, 9, 425, 3826, 7, 4094, 19, 31, 2047, 4096, 1, 0, 20, 2206, 107, 0, 427, 0, 1, 2229, 2645, 3396, 0, 2, 4040, 6, 3984, 1, 3893, 0, 3, 1, 794, 1142, 742, 24, 0, 180, 0, 3921, 0, 16, 193, 0, 0, 0, 0, 12, 3645, 0, 33, 2013, 1, 4091, 3337, 290, 2, 0, 1, 1237, 1475, 2555, 0, 1, 0, 2, 545] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[3118, 0, 2477, 0, 3928, 680, 1, 3826, 1, 4092, 0, 4096, 2714, 2529, 0, 382, 21, 17, 0, 5, 293, 13, 264, 4, 1200, 365, 590, 2, 3, 3971, 1124, 4024, 1, 0, 0, 489, 0, 4087, 16, 0, 4095, 26, 3950, 3502, 4093, 0, 7, 0, 0, 4024, 2, 74, 21, 0, 18, 214, 3549, 7, 8, 1379, 4046, 4022, 0, 4095, 1179, 0, 2578, 96, 0, 1119, 4083, 0, 407, 10, 3758, 79, 287, 0, 1, 0, 0, 3108, 23, 0, 3101, 3596, 604, 4096, 0, 507, 0, 171, 0, 7, 360, 0, 0, 0, 159, 0, 6, 2, 0, 1, 1, 175, 1706, 304, 2, 3983, 4, 52, 3150, 23, 4096, 4079, 28, 90, 32, 9, 10, 173, 373, 0, 81, 0, 777, 1121] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[434, 0, 2783, 967, 59, 4078, 0, 115, 0, 4095, 0, 4066, 1140, 2931, 0, 1053, 148, 644, 0, 0, 3, 1614, 1, 21, 1363, 978, 6, 112, 3435, 4092, 3864, 2740, 4078, 0, 106, 0, 1, 250, 0, 38, 3771, 0, 54, 427, 2880, 30, 0, 1, 6, 4076, 349, 0, 0, 0, 0, 37, 2646, 0, 0, 3973, 3886, 4061, 33, 4095, 73, 2, 2540, 2, 48, 1, 3369, 0, 1, 42, 3682, 40, 0, 10, 66, 54, 411, 3883, 1708, 1, 12, 3766, 0, 3976, 0, 2241, 13, 6, 0, 172, 2683, 19, 8, 1, 3708, 1, 143, 4, 0, 1, 0, 815, 2312, 2044, 0, 1371, 66, 3, 240, 2298, 3997, 4089, 164, 129, 0, 0, 1963, 16, 2694, 0, 628, 13, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1559, 4, 1490, 1, 0, 3661, 18, 209, 0, 4066, 0, 3423, 2132, 128, 0, 0, 8, 26, 16, 1, 170, 533, 9, 393, 2, 101, 2604, 1, 4064, 4090, 0, 0, 4095, 21, 0, 1472, 42, 9, 1, 0, 4040, 0, 0, 1, 67, 500, 32, 0, 0, 4064, 211, 13, 15, 0, 3006, 7, 136, 19, 3, 2367, 4052, 1658, 122, 4088, 3566, 0, 4060, 0, 246, 1, 3916, 11, 845, 6, 4054, 0, 16, 18, 0, 0, 0, 4075, 3136, 0, 0, 4021, 0, 4096, 0, 685, 0, 0, 0, 2, 584, 553, 0, 1, 4079, 0, 0, 0, 1, 237, 0, 600, 7, 3932, 15, 3689, 33, 0, 0, 0, 4090, 4078, 3977, 1, 40, 18, 4085, 313, 3531, 98, 2749, 4, 1104, 1749] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3936, 0, 3760, 0, 1079, 3270, 0, 129, 0, 0, 0, 1163, 345, 48, 2, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 5, 8, 1, 1, 3, 498, 222, 74, 4096, 7, 923, 4035, 0, 3729, 1925, 1025, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2423, 0, 0, 0, 0, 2458, 1954, 0, 4064, 56, 115, 75, 4057, 3701, 2015, 11, 705, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3729, 0, 0, 34, 228, 3, 257, 0, 24, 0, 37, 3984, 2123, 3936, 204, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1439, 0, 4042, 1977, 14, 3867, 4093, 9, 1655, 799, 3056, 34, 0, 1, 4, 3, 845, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4096, 26, 0, 0, 4095, 2, 2169, 0, 3983, 0, 0, 0, 0, 147, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 226, 119, 17, 5, 1, 0, 7, 0, 0, 0, 47, 0, 3757, 0, 14, 4095, 3066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 3353, 2333, 2012, 0, 2, 0, 4073, 317, 893, 0, 0, 8, 0, 834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15257, 17114, 21636, 14846, 22400, 14540, 11355, 13924] avg=16384.0000 max=22400 max_rank=4 min=11355 min_rank=6 imbalance_ratio=1.367188 max_load_reduction=108672 max_load_reduction_pct=0.829102 moved_to_echo=62219 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[167, 3, 4085, 54, 0, 4062, 148, 1755, 0, 0, 0, 1977, 31, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 103, 0, 180, 3, 0, 5, 0, 3728, 0, 0, 0, 15, 4066, 86, 4088, 1932, 903, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 399, 559, 0, 0, 902, 0, 4080, 0, 0, 2911, 4071, 161, 5, 1, 716, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1927, 3526, 0, 189, 0, 456, 4096, 0, 4, 91, 3059, 3878, 45, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 2898, 1, 0, 1616, 4028, 0, 1807, 0, 4071, 0, 0, 0, 0, 0, 1449, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4079, 1, 1007, 5, 3912, 63, 2164, 2, 1547, 0, 2, 0, 1, 3638, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1564, 0, 36, 0, 0, 444, 32, 0, 684, 4081, 1, 4035, 197, 0, 3874, 2697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3379, 4096, 3749, 0, 134, 0, 711, 759, 144, 8, 205, 0, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12324, 15791, 13818, 21197, 16708, 16429, 17645, 17160] avg=16384.0000 max=21197 max_rank=3 min=12324 min_rank=0 imbalance_ratio=1.293762 max_load_reduction=109875 max_load_reduction_pct=0.838280 moved_to_echo=67942 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4010, 0, 3475, 1957, 80, 3173, 0, 368, 0, 0, 0, 1444, 2343, 1, 29, 2889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 1, 8, 4022, 1, 0, 0, 4, 0, 3, 2, 0, 4095, 3, 58, 1926, 964, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3688, 0, 10, 1575, 745, 221, 22, 0, 3896, 833, 0, 1, 1931, 4076, 1, 48, 693, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 57, 1473, 1097, 0, 0, 0, 1027, 3224, 2581, 4091, 3663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 1, 3142, 26, 5, 1072, 2501, 370, 29, 0, 2253, 2, 1, 0, 2, 0, 1050, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3178, 0, 1307, 202, 3550, 89, 2170, 0, 4034, 0, 3, 1, 0, 2988, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1150, 1108, 625, 152, 0, 251, 21, 0, 4087, 3, 56, 3895, 0, 0, 4056, 2879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3299, 3982, 1335, 118, 16, 115, 17, 70, 1730, 12, 1836, 0, 1552, 2672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19769, 11972, 17743, 17215, 11771, 17563, 18285, 16754] avg=16384.0000 max=19769 max_rank=0 min=11771 min_rank=4 imbalance_ratio=1.206604 max_load_reduction=111303 max_load_reduction_pct=0.849174 moved_to_echo=64373 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1023, 0, 678, 3237, 0, 4096, 0, 2363, 2, 0, 0, 1951, 3277, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 533, 75, 0, 7, 48, 565, 0, 506, 3642, 0, 0, 77, 4018, 1232, 302, 1914, 973, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2133, 1, 0, 689, 66, 1, 64, 0, 4085, 3, 0, 3705, 3605, 276, 3662, 6, 711, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1928, 0, 0, 18, 558, 6, 1, 0, 0, 3948, 3226, 4089, 3069, 1889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 3716, 0, 15, 2, 3945, 0, 1, 0, 4090, 56, 22, 2, 0, 0, 1420, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 94, 4018, 755, 1, 4096, 3936, 2156, 264, 0, 4, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4078, 0, 110, 23, 0, 149, 0, 31, 1368, 1293, 4, 4094, 0, 0, 3800, 2910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 100, 3383, 3652, 226, 4, 0, 142, 87, 3, 28, 0, 4, 4, 2702, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16637, 14619, 19010, 18732, 14175, 15642, 17861, 14396] avg=16384.0000 max=19010 max_rank=2 min=14175 min_rank=4 imbalance_ratio=1.160278 max_load_reduction=112062 max_load_reduction_pct=0.854965 moved_to_echo=62074 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4055, 0, 3984, 74, 25, 4073, 0, 958, 0, 0, 0, 1975, 3864, 38, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 46, 1861, 112, 7, 8, 3779, 8, 5, 0, 4093, 4034, 0, 166, 1874, 1024, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 0, 0, 35, 0, 945, 448, 2, 3686, 0, 234, 3, 1483, 2999, 0, 21, 710, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1, 194, 18, 0, 284, 1396, 3174, 3875, 9, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3826, 7, 4094, 19, 31, 2047, 4096, 1, 0, 20, 2206, 107, 0, 427, 0, 1, 1438, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2229, 2645, 3396, 0, 2, 4040, 6, 2110, 1, 3893, 0, 3, 1, 794, 1142, 742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 180, 0, 3921, 0, 16, 193, 0, 0, 0, 0, 12, 3645, 0, 33, 4003, 3063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2013, 1, 3381, 3337, 290, 2, 0, 1, 1237, 1475, 2555, 0, 1, 0, 2, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19051, 17989, 14559, 9379, 19160, 21004, 15090, 14840] avg=16384.0000 max=21004 max_rank=5 min=9379 min_rank=3 imbalance_ratio=1.281982 max_load_reduction=110068 max_load_reduction_pct=0.839752 moved_to_echo=70094 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[434, 0, 2783, 967, 59, 4078, 0, 115, 0, 0, 0, 1971, 1140, 2931, 0, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 644, 0, 0, 3, 1614, 1, 21, 1363, 978, 6, 112, 3435, 4092, 3864, 2740, 1870, 1021, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 106, 0, 1, 250, 0, 38, 3771, 0, 54, 427, 2880, 30, 0, 1, 694, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 349, 0, 0, 0, 0, 37, 2646, 0, 0, 3973, 3070, 4061, 33, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 2, 2540, 2, 48, 1, 3369, 0, 1, 42, 3682, 40, 0, 10, 66, 54, 1434, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 411, 3883, 1708, 1, 12, 3766, 0, 2106, 0, 2241, 13, 6, 0, 172, 2683, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 3708, 1, 143, 4, 0, 1, 0, 815, 2312, 2044, 0, 1371, 66, 3, 4095, 3055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 2298, 3303, 4089, 164, 129, 0, 0, 1963, 16, 2694, 0, 628, 13, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15531, 22573, 12333, 18270, 12177, 17021, 17627, 15540] avg=16384.0000 max=22573 max_rank=1 min=12177 min_rank=4 imbalance_ratio=1.377747 max_load_reduction=108499 max_load_reduction_pct=0.827782 moved_to_echo=62365 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3118, 0, 2477, 0, 3928, 680, 1, 3826, 1, 0, 0, 1985, 2714, 2529, 0, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 17, 0, 5, 293, 13, 264, 4, 1200, 365, 590, 2, 3, 3971, 1124, 4024, 1926, 1008, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 489, 0, 4087, 16, 0, 4095, 26, 3950, 3502, 4093, 0, 7, 0, 711, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 74, 21, 0, 18, 214, 3549, 7, 8, 1379, 3197, 4022, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1179, 0, 2578, 96, 0, 1119, 4083, 0, 407, 10, 3758, 79, 287, 0, 1, 0, 1445, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3108, 23, 0, 3101, 3596, 604, 2170, 0, 507, 0, 171, 0, 7, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 6, 2, 0, 1, 1, 175, 1706, 304, 2, 3983, 4, 52, 4092, 3016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3150, 23, 3385, 4079, 28, 90, 32, 9, 10, 173, 373, 0, 81, 0, 777, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21641, 15496, 20980, 16586, 15888, 13647, 13503, 13331] avg=16384.0000 max=21641 max_rank=0 min=13331 min_rank=7 imbalance_ratio=1.320862 max_load_reduction=109431 max_load_reduction_pct=0.834892 moved_to_echo=56369 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1559, 4, 1490, 1, 0, 3661, 18, 209, 0, 0, 0, 1660, 2132, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 26, 16, 1, 170, 533, 9, 393, 2, 101, 2604, 1, 4064, 4090, 0, 0, 1926, 1018, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 21, 0, 1472, 42, 9, 1, 0, 4040, 0, 0, 1, 67, 500, 32, 0, 710, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 13, 15, 0, 3006, 7, 136, 19, 3, 2367, 3202, 1658, 122, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3566, 0, 4060, 0, 246, 1, 3916, 11, 845, 6, 4054, 0, 16, 18, 0, 0, 1207, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 3136, 0, 0, 4021, 0, 2170, 0, 685, 0, 0, 0, 2, 584, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4079, 0, 0, 0, 1, 237, 0, 600, 7, 3932, 15, 3689, 33, 0, 4066, 3046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3380, 4078, 3977, 1, 40, 18, 4085, 313, 3531, 98, 2749, 4, 1104, 1749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10862, 15518, 10993, 14847, 18793, 15226, 19706, 25127] avg=16384.0000 max=25127 max_rank=7 min=10862 min_rank=0 imbalance_ratio=1.533630 max_load_reduction=105945 max_load_reduction_pct=0.808296 moved_to_echo=78852 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 256, 15104, 1152, 4864, 256, 128, 1408, 128, 1664, 10240, 11776, 128, 28544, 384, 128, 32128, 23808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[18304, 128, 22784, 6400, 5248, 27136, 256, 9728, 128, 0, 0, 14208, 15872, 5760, 128, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[5760, 2432, 26880, 29696, 11904, 384, 256, 384, 12288, 3200, 12032, 128, 5504, 128, 6144, 14848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[10624, 128, 27136, 2176, 384, 9728, 30080, 512, 4864, 896, 27264, 384, 384, 512, 128, 128, 10368, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[3840, 0, 4480, 3712, 384, 384, 5376, 2048, 10496, 128, 384, 18176, 24320, 28160, 7680, 22528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2944, 25216, 12416, 3072, 3328, 31104, 4736, 17280, 384, 16896, 128, 384, 128, 1024, 11648, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[512, 2048, 128, 256, 6400, 2432, 896, 1024, 10880, 5248, 7424, 128, 12672, 32512, 6400, 15232, 15360, 7936, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[20480, 128, 640, 4864, 896, 8064, 3456, 128, 31744, 1024, 4480, 10752, 22272, 11776, 5760, 128, 5760, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 3871, 3270, 162, 0, 3178, 0, 2158, 3585, 3525, 4093, 3674, 1713, 515, 0, 295, 4, 0, 4082, 3518, 63, 0, 0, 0, 164, 0, 4096, 1077, 0, 1391, 919, 8, 29, 1484, 307, 0, 4017, 201, 0, 198, 3986, 1916, 3749, 1, 717, 0, 0, 0, 107, 0, 0, 3483, 0, 4080, 4085, 0, 0, 0, 4048, 0, 43, 1891, 70, 534, 5, 1335, 0, 15, 0, 2, 3, 358, 0, 3122, 41, 2, 0, 33, 2, 0, 0, 28, 0, 3382, 0, 4096, 4090, 3457, 237, 4075, 20, 39, 4096, 3, 44, 309, 9, 4086, 1, 4096, 82, 1, 1, 3, 5, 0, 0, 13, 0, 9, 453, 67, 726, 1, 0, 0, 39, 0, 0, 1, 0, 4082, 268, 4095, 0, 3888, 45] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 135, 4090, 0, 0, 56, 8, 3044, 3646, 3389, 4003, 4089, 4048, 733, 0, 0, 4066, 23, 0, 0, 3870, 3, 122, 19, 153, 3964, 0, 1, 874, 0, 705, 4068, 546, 116, 2, 429, 1323, 6, 44, 0, 302, 3994, 3923, 0, 0, 0, 0, 121, 0, 0, 0, 0, 3659, 132, 37, 1880, 4038, 0, 550, 2810, 0, 2, 0, 80, 3, 3460, 2, 1786, 1230, 9, 460, 0, 2, 60, 0, 3658, 5, 0, 0, 3790, 186, 4, 4, 402, 2, 4, 30, 4039, 1, 18, 0, 67, 1069, 2704, 0, 3687, 23, 3453, 4096, 25, 3135, 4078, 1311, 0, 57, 4, 1, 2, 2, 0, 35, 37, 0, 1, 595, 0, 3905, 3724, 0, 0, 0, 22, 2654, 0, 4092, 1, 4034, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[13, 1, 1495, 3545, 338, 274, 52, 5, 4096, 1517, 0, 3832, 3950, 3094, 0, 0, 28, 2725, 0, 3187, 1475, 22, 134, 0, 0, 4091, 0, 0, 5, 0, 229, 1873, 0, 3, 0, 68, 116, 0, 1306, 0, 0, 0, 0, 2799, 3910, 0, 479, 2122, 0, 776, 0, 0, 3963, 0, 4080, 4043, 0, 0, 0, 4040, 1, 0, 23, 13, 597, 230, 0, 8, 4059, 0, 3246, 42, 4, 0, 0, 3964, 0, 1, 70, 0, 0, 164, 586, 3999, 3060, 0, 0, 4096, 561, 58, 0, 0, 0, 3939, 8, 109, 1, 4014, 4096, 0, 4062, 1935, 81, 153, 0, 0, 2023, 0, 1063, 0, 3280, 2, 0, 4034, 1509, 2, 4096, 0, 0, 0, 0, 0, 4089, 5, 4096, 0, 7, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4017, 1866, 0, 0, 1, 573, 3624, 4092, 44, 3822, 4043, 4011, 0, 0, 324, 1933, 0, 4095, 137, 2033, 0, 0, 4, 3820, 119, 5, 6, 1674, 304, 75, 0, 0, 2, 1, 0, 223, 437, 0, 0, 3997, 2278, 31, 0, 0, 168, 1, 0, 0, 0, 0, 4068, 0, 3356, 0, 0, 0, 622, 4035, 0, 0, 1772, 9, 2992, 4, 246, 0, 4095, 0, 3, 23, 3924, 0, 450, 2, 99, 350, 636, 0, 0, 3681, 0, 374, 2, 63, 40, 3999, 2756, 0, 3034, 10, 2, 70, 4075, 3912, 0, 0, 3478, 0, 3422, 3351, 0, 207, 0, 811, 0, 0, 0, 120, 3619, 0, 57, 4086, 3042, 230, 1276, 4096, 0, 0, 0, 0, 4080, 58, 2310, 0, 351, 14] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 3318, 4022, 3393, 0, 288, 0, 536, 4096, 4004, 126, 2810, 4096, 4087, 70, 4, 3517, 0, 0, 3716, 4047, 12, 3, 0, 0, 3943, 36, 310, 70, 0, 4094, 4047, 0, 1, 0, 259, 603, 2756, 391, 129, 0, 4091, 0, 32, 17, 0, 0, 11, 0, 3175, 0, 19, 3174, 0, 2308, 484, 794, 0, 23, 2115, 21, 1, 0, 266, 0, 100, 1252, 25, 363, 85, 2, 0, 4, 20, 0, 3391, 2, 3, 8, 0, 1633, 0, 3, 2636, 3928, 0, 0, 3806, 2929, 968, 0, 0, 121, 4064, 1291, 0, 0, 914, 3740, 0, 13, 3292, 0, 5, 151, 0, 0, 0, 6, 3, 4073, 0, 0, 4096, 5, 0, 3800, 1, 0, 0, 0, 29, 4084, 0, 4095, 0, 816, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 1037, 4047, 6, 8, 23, 0, 1335, 3870, 4072, 4083, 724, 4026, 3660, 13, 0, 521, 0, 492, 4090, 4096, 2, 3, 337, 0, 3781, 2599, 0, 1177, 0, 3, 3546, 228, 0, 158, 0, 25, 9, 67, 2320, 0, 364, 0, 7, 1769, 0, 0, 2, 3, 4088, 2970, 598, 2256, 291, 4040, 3329, 11, 5, 0, 2150, 0, 0, 0, 0, 13, 211, 0, 312, 36, 25, 0, 0, 40, 39, 0, 4079, 18, 0, 4089, 2, 2, 0, 0, 4091, 3936, 0, 15, 4091, 32, 1, 1774, 0, 0, 3897, 65, 2585, 0, 2, 3873, 117, 3306, 116, 0, 41, 1346, 37, 1, 12, 0, 0, 3996, 38, 0, 3814, 0, 0, 4096, 0, 4, 0, 4052, 407, 4096, 2, 122, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[371, 759, 3002, 2, 0, 1, 493, 3961, 1081, 4091, 4096, 3491, 4088, 4095, 2860, 6, 1157, 3034, 82, 4044, 623, 1702, 4, 0, 0, 0, 5, 0, 1116, 1158, 0, 6, 0, 120, 0, 0, 33, 0, 2, 682, 48, 0, 3907, 2227, 0, 0, 4, 16, 1895, 4073, 3930, 0, 3770, 1772, 425, 1648, 0, 2, 380, 3969, 44, 0, 0, 0, 1666, 12, 0, 0, 0, 407, 394, 0, 2143, 0, 3989, 104, 3, 2, 11, 2309, 0, 0, 0, 3094, 3118, 43, 1, 3893, 0, 0, 2901, 0, 0, 1951, 4090, 12, 0, 2095, 3865, 0, 3843, 3647, 0, 1437, 23, 315, 0, 0, 4, 29, 574, 80, 106, 76, 1, 93, 0, 4025, 4, 7, 464, 185, 3961, 7, 187, 24, 1602, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[9, 4014, 3950, 21, 0, 0, 0, 4087, 4063, 4082, 26, 3368, 3901, 51, 6, 24, 0, 449, 0, 0, 3053, 1, 0, 434, 0, 3989, 4, 0, 2, 668, 1, 4038, 16, 0, 3821, 793, 6, 94, 4075, 2, 0, 4082, 108, 15, 81, 0, 479, 0, 459, 3681, 0, 119, 3993, 468, 4013, 1803, 0, 1, 2869, 3826, 0, 0, 0, 49, 322, 2319, 150, 0, 485, 0, 0, 0, 1347, 0, 46, 1, 3, 0, 1893, 769, 0, 2289, 1, 302, 4065, 0, 57, 4096, 2, 29, 0, 0, 0, 10, 2726, 3749, 0, 84, 3933, 552, 4088, 7, 83, 300, 2451, 4091, 1, 3, 769, 0, 4000, 0, 51, 16, 3959, 7, 165, 0, 0, 0, 5, 0, 4096, 2, 2647, 6, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3270, 162, 0, 3178, 0, 1930, 0, 3525, 4093, 0, 1713, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 4, 0, 4082, 3518, 63, 0, 0, 0, 160, 0, 4096, 1077, 0, 1391, 919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 29, 1484, 307, 0, 4017, 201, 0, 198, 3986, 1916, 3749, 1, 717, 0, 0, 3871, 2079, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 2743, 0, 4080, 4085, 0, 0, 0, 4048, 0, 43, 1891, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 5, 1335, 0, 15, 0, 2, 3, 358, 0, 3122, 41, 2, 0, 33, 2, 3674, 2709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 3382, 0, 4096, 1339, 3457, 237, 4075, 20, 39, 4096, 3, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 9, 4086, 1, 4096, 82, 1, 1, 3, 5, 0, 0, 13, 0, 9, 453, 672, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 726, 1, 0, 0, 39, 0, 0, 1, 0, 4082, 268, 4095, 0, 3888, 45, 228, 324, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18386, 15605, 23439, 17067, 11835, 20816, 10156, 13768] avg=16384.0000 max=23439 max_rank=2 min=10156 min_rank=6 imbalance_ratio=1.430603 max_load_reduction=107633 max_load_reduction_pct=0.821175 moved_to_echo=56575 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1037, 0, 6, 8, 23, 0, 1335, 3461, 0, 4083, 724, 0, 3660, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 0, 492, 4090, 4096, 2, 3, 337, 0, 3668, 2599, 0, 1177, 0, 3, 3546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 0, 158, 0, 25, 9, 67, 2320, 0, 364, 0, 7, 1769, 0, 0, 2, 4047, 2080, 994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4088, 2970, 598, 1777, 291, 4040, 3329, 11, 5, 0, 2150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 211, 0, 312, 36, 25, 0, 0, 40, 39, 0, 4079, 18, 0, 4089, 2, 4026, 3078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4091, 3936, 0, 15, 1339, 32, 1, 1774, 0, 0, 3897, 65, 2585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3873, 117, 3306, 116, 0, 41, 1346, 37, 1, 12, 0, 0, 3996, 38, 672, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3814, 0, 0, 4096, 0, 4, 0, 4052, 407, 4096, 2, 122, 0, 0, 0, 409, 210, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14350, 20534, 12070, 19262, 15968, 17737, 13826, 17325] avg=16384.0000 max=20534 max_rank=1 min=12070 min_rank=2 imbalance_ratio=1.253296 max_load_reduction=110538 max_load_reduction_pct=0.843338 moved_to_echo=64856 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1866, 0, 0, 1, 573, 3241, 0, 44, 3822, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 1933, 0, 4095, 137, 2033, 0, 0, 4, 3706, 119, 5, 6, 1674, 304, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 223, 437, 0, 0, 3997, 2278, 31, 0, 0, 168, 1, 4017, 2033, 999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3203, 0, 3356, 0, 0, 0, 622, 4035, 0, 0, 1772, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2992, 4, 246, 0, 4095, 0, 3, 23, 3924, 0, 450, 2, 99, 350, 636, 0, 4043, 3093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3681, 0, 374, 2, 63, 40, 1309, 2756, 0, 3034, 10, 2, 70, 4075, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3478, 0, 3422, 3351, 0, 207, 0, 811, 0, 0, 0, 120, 3619, 0, 657, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4086, 3042, 230, 1276, 4096, 0, 0, 0, 0, 4080, 58, 2310, 0, 351, 14, 383, 379, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13558, 14415, 14187, 12997, 19960, 19328, 16151, 20476] avg=16384.0000 max=20476 max_rank=7 min=12997 min_rank=3 imbalance_ratio=1.249756 max_load_reduction=110596 max_load_reduction_pct=0.843781 moved_to_echo=75915 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 135, 0, 0, 0, 56, 8, 3044, 3261, 0, 4003, 4089, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 23, 0, 0, 3870, 3, 122, 19, 153, 3845, 0, 1, 874, 0, 705, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 116, 2, 429, 1323, 6, 44, 0, 302, 3994, 3923, 0, 0, 0, 0, 121, 4090, 2053, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2881, 132, 37, 1880, 4038, 0, 550, 2810, 0, 2, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3460, 2, 1786, 1230, 9, 460, 0, 2, 60, 0, 3658, 5, 0, 0, 3790, 4048, 2561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 4, 4, 402, 2, 4, 30, 1322, 1, 18, 0, 67, 1069, 2704, 0, 3687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3453, 4096, 25, 3135, 4078, 1311, 0, 57, 4, 1, 2, 2, 0, 35, 37, 664, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 595, 0, 3905, 3724, 0, 0, 0, 22, 2654, 0, 4092, 1, 4034, 0, 385, 341, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15329, 17749, 17777, 12410, 21074, 9500, 17360, 19873] avg=16384.0000 max=21074 max_rank=4 min=9500 min_rank=5 imbalance_ratio=1.286255 max_load_reduction=109998 max_load_reduction_pct=0.839218 moved_to_echo=67807 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[13, 1, 0, 3545, 338, 274, 52, 5, 3659, 0, 0, 3832, 0, 3094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 2725, 0, 3187, 1475, 22, 134, 0, 0, 3964, 0, 0, 5, 0, 229, 1873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 68, 116, 0, 1306, 0, 0, 0, 0, 2799, 3910, 0, 479, 2122, 1495, 2085, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 0, 0, 3115, 0, 4080, 4043, 0, 0, 0, 4040, 1, 0, 23, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 597, 230, 0, 8, 4059, 0, 3246, 42, 4, 0, 0, 3964, 0, 1, 70, 0, 3950, 1146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 586, 3999, 3060, 0, 0, 1333, 561, 58, 0, 0, 0, 3939, 8, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4014, 4096, 0, 4062, 1935, 81, 153, 0, 0, 2023, 0, 1063, 0, 3280, 2, 678, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 1509, 2, 4096, 0, 0, 0, 0, 0, 4089, 5, 4096, 0, 7, 0, 437, 372, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14813, 13642, 14754, 16091, 17317, 13817, 21864, 18774] avg=16384.0000 max=21864 max_rank=6 min=13642 min_rank=1 imbalance_ratio=1.334473 max_load_reduction=109208 max_load_reduction_pct=0.833191 moved_to_echo=71772 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 3318, 0, 3393, 0, 288, 0, 536, 3663, 0, 126, 2810, 0, 4087, 70, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3517, 0, 0, 3716, 4047, 12, 3, 0, 0, 3825, 36, 310, 70, 0, 4094, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 259, 603, 2756, 391, 129, 0, 4091, 0, 32, 17, 0, 0, 11, 4022, 1935, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3175, 0, 19, 2500, 0, 2308, 484, 794, 0, 23, 2115, 21, 1, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 1252, 25, 363, 85, 2, 0, 4, 20, 0, 3391, 2, 3, 8, 0, 4096, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1633, 0, 3, 2636, 3928, 0, 0, 1246, 2929, 968, 0, 0, 121, 4064, 1291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 914, 3740, 0, 13, 3292, 0, 5, 151, 0, 0, 0, 6, 3, 4073, 0, 625, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 3800, 1, 0, 0, 0, 29, 4084, 0, 4095, 0, 816, 0, 433, 295, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18295, 23677, 15225, 11706, 12377, 18819, 13201, 17772] avg=16384.0000 max=23677 max_rank=1 min=11706 min_rank=3 imbalance_ratio=1.445129 max_load_reduction=107395 max_load_reduction_pct=0.819359 moved_to_echo=62169 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[371, 759, 0, 2, 0, 1, 493, 3961, 967, 0, 4096, 3491, 0, 4095, 2860, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1157, 3034, 82, 4044, 623, 1702, 4, 0, 0, 0, 5, 0, 1116, 1158, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 33, 0, 2, 682, 48, 0, 3907, 2227, 0, 0, 4, 16, 3002, 1979, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1895, 4073, 3930, 0, 2969, 1772, 425, 1648, 0, 2, 380, 3969, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1666, 12, 0, 0, 0, 407, 394, 0, 2143, 0, 3989, 104, 3, 2, 11, 2309, 4088, 3091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3094, 3118, 43, 1, 1274, 0, 0, 2901, 0, 0, 1951, 4090, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2095, 3865, 0, 3843, 3647, 0, 1437, 23, 315, 0, 0, 4, 29, 574, 80, 640, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 76, 1, 93, 0, 4025, 4, 7, 464, 185, 3961, 7, 187, 24, 1602, 0, 114, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21102, 12931, 13020, 21107, 18219, 16484, 17002, 11207] avg=16384.0000 max=21107 max_rank=3 min=11207 min_rank=7 imbalance_ratio=1.288269 max_load_reduction=109965 max_load_reduction_pct=0.838966 moved_to_echo=62912 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[9, 4014, 0, 21, 0, 0, 0, 4087, 3633, 0, 26, 3368, 0, 51, 6, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 0, 0, 3053, 1, 0, 434, 0, 3869, 4, 0, 2, 668, 1, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3821, 793, 6, 94, 4075, 2, 0, 4082, 108, 15, 81, 0, 479, 0, 3950, 2082, 996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 3681, 0, 119, 3144, 468, 4013, 1803, 0, 1, 2869, 3826, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 2319, 150, 0, 485, 0, 0, 0, 1347, 0, 46, 1, 3, 0, 1893, 769, 3901, 3086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2289, 1, 302, 4065, 0, 57, 1341, 2, 29, 0, 0, 0, 10, 2726, 3749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 3933, 552, 4088, 7, 83, 300, 2451, 4091, 1, 3, 769, 0, 4000, 0, 673, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 16, 3959, 7, 165, 0, 0, 0, 5, 0, 4096, 2, 2647, 6, 1, 0, 430, 372, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15239, 12519, 20600, 20432, 14322, 14571, 21512, 11877] avg=16384.0000 max=21512 max_rank=6 min=11877 min_rank=7 imbalance_ratio=1.312988 max_load_reduction=109560 max_load_reduction_pct=0.835876 moved_to_echo=62282 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[2432, 16000, 6912, 768, 22400, 2688, 22400, 17280, 4864, 128, 4480, 27008, 128, 128, 3712, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 10624, 31232, 768, 25984, 16512, 1536, 2176, 4096, 5376, 2048, 128, 1920, 256, 19712, 640, 5376, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[6144, 6400, 3072, 2176, 10368, 640, 4224, 128, 7936, 128, 7680, 15360, 256, 384, 6784, 6912, 31872, 21888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[9984, 8192, 640, 23296, 20864, 3840, 384, 896, 256, 23040, 2816, 4480, 4352, 3584, 6784, 18688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[512, 9344, 0, 12160, 512, 768, 3840, 13568, 23936, 0, 16000, 26240, 0, 21504, 3584, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1920, 6144, 640, 14976, 21504, 128, 4352, 10624, 9856, 1408, 11904, 128, 1280, 20736, 12288, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[896, 384, 5504, 1920, 2176, 7168, 6528, 3200, 640, 20608, 12160, 8960, 5888, 768, 1152, 2304, 28544, 16384, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[384, 16896, 9216, 384, 17408, 11904, 128, 128, 4608, 768, 31232, 384, 21760, 128, 10752, 128, 2944, 2688, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[149, 0, 4089, 0, 1908, 0, 27, 5, 3549, 172, 8, 2004, 0, 15, 0, 660, 2, 0, 2, 4095, 2462, 49, 3951, 784, 2129, 82, 4096, 1, 279, 4071, 13, 4, 3009, 53, 2526, 443, 0, 0, 3868, 0, 1465, 0, 4095, 1, 43, 3508, 4088, 23, 251, 3682, 2, 3927, 809, 2322, 0, 0, 0, 4, 4070, 0, 0, 0, 10, 4001, 0, 0, 3980, 4069, 1, 0, 1234, 1, 0, 71, 0, 3167, 15, 7, 3, 3069, 0, 3030, 429, 3455, 1901, 4062, 56, 0, 0, 828, 0, 4004, 3, 15, 0, 10, 0, 27, 10, 2008, 0, 3849, 4091, 0, 2104, 135, 39, 0, 337, 21, 0, 396, 0, 411, 185, 0, 0, 0, 3170, 0, 0, 541, 0, 3498, 0, 25, 2, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4095, 458, 4031, 0, 1672, 197, 851, 695, 98, 0, 0, 4094, 0, 4081, 0, 3880, 4096, 4, 11, 1130, 1479, 0, 175, 4096, 0, 35, 2792, 0, 14, 1, 3317, 0, 286, 0, 0, 6, 0, 0, 4092, 4069, 2380, 3477, 2479, 24, 0, 178, 2756, 57, 14, 4054, 0, 3395, 3491, 0, 1911, 4032, 0, 0, 1614, 0, 0, 0, 3438, 0, 5, 0, 3567, 359, 0, 0, 4096, 0, 0, 9, 0, 839, 1, 1499, 0, 3668, 0, 4064, 0, 562, 392, 2, 83, 1, 0, 0, 0, 1, 3, 0, 6, 0, 283, 3978, 1289, 94, 0, 0, 4053, 0, 332, 0, 0, 233, 26, 16, 0, 4092, 0, 792, 5, 3, 0, 0, 3071, 512, 784, 0, 0, 3825, 6, 1368, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4026, 3648, 4000, 1, 227, 2842, 0, 23, 0, 3, 3, 4093, 2, 0, 1, 2216, 3619, 0, 1, 4072, 68, 0, 0, 0, 12, 7, 4074, 13, 4029, 3622, 0, 104, 32, 0, 3686, 2440, 0, 170, 3937, 0, 3978, 1525, 4054, 6, 88, 393, 2, 4034, 0, 0, 1, 4093, 25, 5, 312, 20, 0, 0, 46, 0, 14, 0, 1393, 3759, 4, 4019, 14, 2326, 226, 0, 132, 0, 649, 2, 15, 0, 1, 428, 3977, 0, 101, 37, 154, 0, 4067, 424, 0, 1830, 0, 0, 390, 479, 3558, 0, 89, 0, 0, 4035, 0, 469, 734, 697, 3692, 0, 0, 22, 0, 15, 0, 2360, 2070, 4067, 7, 3993, 8, 0, 0, 0, 3243, 0, 29, 0, 2, 1442, 2846, 343, 260, 3097] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4067, 1820, 2234, 0, 115, 0, 2, 1015, 3637, 8, 536, 3644, 0, 0, 3241, 4088, 1751, 6, 42, 10, 3633, 0, 0, 4096, 3659, 0, 3814, 2, 0, 3950, 4073, 469, 1, 2935, 20, 1, 591, 1, 4066, 4, 4005, 2695, 1727, 573, 0, 2105, 1, 1554, 98, 3934, 0, 0, 3182, 5, 1, 175, 1260, 0, 0, 78, 4096, 0, 12, 3975, 0, 4, 1, 7, 2459, 0, 111, 0, 0, 0, 2193, 3, 8, 175, 278, 2592, 0, 4, 1301, 6, 3443, 0, 88, 570, 140, 0, 0, 3, 4, 1, 2446, 0, 1017, 1073, 0, 3121, 294, 17, 3339, 0, 0, 11, 44, 4, 26, 13, 536, 4037, 0, 0, 1775, 381, 0, 0, 4095, 0, 0, 0, 0, 4092, 3519, 0, 33, 801] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[3367, 3466, 3919, 1, 135, 0, 0, 4, 3481, 0, 0, 4084, 0, 7, 357, 4084, 0, 0, 0, 762, 3419, 0, 3869, 3, 0, 1, 3230, 3945, 0, 2, 1, 0, 3989, 278, 0, 21, 15, 2, 4073, 3986, 75, 4012, 3646, 767, 46, 10, 250, 3760, 241, 4086, 0, 2732, 4054, 0, 2, 4085, 0, 0, 0, 0, 3927, 0, 188, 10, 0, 315, 10, 924, 432, 214, 1504, 0, 3175, 1, 1, 3048, 0, 4, 0, 3050, 0, 1, 0, 4083, 3936, 2427, 0, 1774, 13, 0, 171, 0, 1949, 0, 0, 41, 17, 28, 906, 257, 10, 0, 4094, 3, 5, 0, 0, 9, 0, 151, 0, 263, 3539, 3805, 0, 0, 0, 47, 973, 0, 12, 0, 0, 2545, 3888, 0, 55, 1000] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4096, 10, 4067, 5, 0, 0, 196, 0, 3682, 31, 30, 140, 0, 1132, 3, 3735, 3933, 0, 2, 3598, 1413, 20, 3860, 1482, 4019, 0, 3989, 0, 0, 3867, 1610, 10, 2, 4014, 885, 257, 0, 12, 3242, 32, 0, 3536, 4095, 57, 0, 3, 1373, 339, 21, 2858, 10, 3221, 4067, 3981, 84, 18, 0, 0, 0, 0, 997, 0, 1118, 3, 0, 53, 2260, 133, 0, 0, 2759, 7, 49, 2, 2221, 12, 46, 6, 0, 12, 27, 0, 2, 0, 4070, 16, 0, 36, 15, 0, 0, 0, 4031, 0, 0, 147, 2, 4096, 578, 0, 1, 0, 3899, 0, 0, 818, 0, 0, 0, 1, 0, 4094, 4, 360, 1, 0, 1, 1593, 4058, 0, 63, 190, 4085, 4067, 4056, 13, 3923, 110] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[611, 0, 4063, 0, 37, 0, 3910, 1, 3337, 0, 0, 3417, 0, 0, 0, 276, 4085, 8, 0, 3800, 2477, 0, 2712, 1400, 2612, 33, 2388, 0, 4087, 4080, 4053, 7, 221, 261, 192, 2310, 0, 0, 4072, 1755, 39, 3851, 2606, 0, 0, 0, 82, 4083, 0, 3700, 0, 3891, 0, 807, 0, 0, 0, 198, 0, 3, 4017, 0, 0, 0, 0, 19, 0, 4022, 0, 817, 1, 0, 0, 65, 0, 3150, 0, 0, 0, 42, 0, 3550, 15, 3655, 2968, 1, 274, 1183, 95, 0, 111, 3505, 15, 0, 604, 192, 1, 171, 94, 4057, 0, 2, 4092, 0, 56, 2, 0, 0, 1, 1491, 1, 4062, 39, 7, 3, 1, 0, 0, 4096, 0, 28, 0, 50, 3924, 1664, 0, 395, 3069] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[85, 335, 2810, 3, 2, 0, 162, 2, 4052, 0, 1, 4076, 0, 0, 287, 1598, 250, 2, 0, 0, 1109, 0, 47, 3605, 0, 0, 3350, 0, 0, 8, 742, 11, 4, 1694, 165, 9, 120, 3, 4087, 752, 0, 3509, 3871, 166, 7, 108, 3938, 4047, 0, 4089, 1, 3171, 3898, 0, 0, 137, 3, 4, 9, 6, 4025, 1, 0, 63, 0, 5, 2766, 12, 0, 1239, 224, 0, 4093, 1250, 4001, 4008, 0, 26, 362, 20, 0, 0, 3924, 3967, 4089, 4009, 0, 0, 3, 21, 0, 1192, 3918, 25, 0, 0, 0, 0, 1351, 3950, 26, 165, 4052, 12, 137, 0, 0, 10, 6, 0, 134, 4094, 3611, 1838, 0, 0, 23, 0, 4090, 0, 59, 6, 0, 4091, 0, 0, 3839, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4026, 3648, 1228, 1, 227, 2842, 0, 23, 0, 3, 3, 4093, 2, 0, 1, 2216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3619, 0, 1, 4072, 68, 0, 0, 0, 12, 7, 0, 13, 4029, 1887, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 3686, 2440, 0, 170, 0, 0, 3978, 1525, 2565, 6, 88, 393, 2, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 25, 5, 312, 20, 0, 0, 46, 0, 14, 0, 1393, 3759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4019, 14, 2326, 226, 0, 132, 0, 649, 2, 15, 0, 1, 428, 3977, 0, 3937, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 37, 154, 0, 4067, 424, 0, 1830, 0, 0, 390, 479, 3558, 0, 89, 0, 3487, 716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 469, 734, 697, 3692, 0, 0, 22, 0, 15, 0, 2360, 2070, 4067, 2056, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3993, 8, 0, 0, 0, 3243, 0, 29, 0, 2, 1442, 2846, 343, 260, 3097, 1735, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18313, 13812, 18919, 9668, 16317, 15332, 21703, 17008] avg=16384.0000 max=21703 max_rank=6 min=9668 min_rank=3 imbalance_ratio=1.324646 max_load_reduction=109369 max_load_reduction_pct=0.834419 moved_to_echo=70360 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[149, 0, 1255, 0, 1908, 0, 27, 5, 3549, 172, 8, 2004, 0, 15, 0, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 4095, 2462, 49, 3951, 784, 2129, 82, 0, 1, 279, 2121, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3009, 53, 2526, 443, 0, 0, 0, 0, 1465, 0, 2591, 1, 43, 3508, 4088, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 3659, 2, 3927, 809, 2322, 0, 0, 0, 4, 4070, 0, 0, 0, 10, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 4069, 1, 0, 1234, 1, 0, 71, 0, 3167, 15, 7, 3, 3069, 3868, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3030, 429, 3455, 1901, 4062, 56, 0, 0, 828, 0, 4004, 3, 15, 0, 10, 3507, 732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 10, 2008, 0, 3849, 4091, 0, 2104, 135, 39, 0, 337, 21, 0, 396, 2102, 1501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 411, 185, 0, 0, 0, 3170, 0, 0, 541, 0, 3498, 0, 25, 2, 7, 1950, 23, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9752, 15974, 17750, 19055, 20074, 22032, 16620, 9815] avg=16384.0000 max=22032 max_rank=5 min=9752 min_rank=0 imbalance_ratio=1.344727 max_load_reduction=109040 max_load_reduction_pct=0.831909 moved_to_echo=68541 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4095, 458, 1229, 0, 1672, 197, 851, 695, 98, 0, 0, 4094, 0, 4081, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 11, 1130, 1479, 0, 175, 4096, 0, 35, 0, 0, 14, 0, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 0, 0, 6, 0, 0, 0, 4069, 2380, 3477, 1559, 24, 0, 178, 2756, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4026, 0, 3395, 3491, 0, 1911, 4032, 0, 0, 1614, 0, 0, 0, 3438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3567, 359, 0, 0, 4096, 0, 0, 9, 0, 839, 1, 1499, 0, 3668, 4092, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 562, 392, 2, 83, 1, 0, 0, 0, 1, 3, 0, 6, 0, 2390, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 3978, 1289, 94, 0, 0, 4053, 0, 332, 0, 0, 233, 26, 16, 0, 4092, 2077, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 792, 5, 3, 0, 0, 3071, 512, 784, 0, 0, 3825, 6, 1368, 2, 4096, 1, 28, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21350, 14357, 14792, 21921, 18537, 8229, 17386, 14500] avg=16384.0000 max=21921 max_rank=3 min=8229 min_rank=5 imbalance_ratio=1.337952 max_load_reduction=109151 max_load_reduction_pct=0.832756 moved_to_echo=58652 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 10, 1248, 5, 0, 0, 196, 0, 3682, 31, 30, 140, 0, 1132, 3, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 0, 2, 3598, 1413, 20, 3860, 1482, 4019, 0, 0, 0, 0, 2015, 1610, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4014, 885, 257, 0, 12, 0, 32, 0, 3536, 2591, 57, 0, 3, 1373, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 2841, 10, 3221, 4067, 3981, 84, 18, 0, 0, 0, 0, 997, 0, 1118, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 2260, 133, 0, 0, 2759, 7, 49, 2, 2221, 12, 46, 6, 0, 12, 3242, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 2, 0, 4070, 16, 0, 36, 15, 0, 0, 0, 4031, 0, 0, 147, 3414, 728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 578, 0, 1, 0, 3899, 0, 0, 818, 0, 0, 0, 1, 0, 4094, 2091, 1501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 360, 1, 0, 1, 1593, 4058, 0, 63, 190, 4085, 4067, 4056, 13, 3923, 110, 1852, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14308, 21962, 13101, 16361, 11377, 12486, 17081, 24396] avg=16384.0000 max=24396 max_rank=7 min=11377 min_rank=4 imbalance_ratio=1.489014 max_load_reduction=106676 max_load_reduction_pct=0.813873 moved_to_echo=65340 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3367, 3466, 1202, 1, 135, 0, 0, 4, 3481, 0, 0, 4084, 0, 7, 357, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 762, 3419, 0, 3869, 3, 0, 1, 0, 3945, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 278, 0, 21, 15, 2, 0, 3986, 75, 4012, 2306, 767, 46, 10, 250, 3760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 4061, 0, 2732, 4054, 0, 2, 4085, 0, 0, 0, 0, 3927, 0, 188, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 10, 924, 432, 214, 1504, 0, 3175, 1, 1, 3048, 0, 4, 0, 3050, 4073, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4083, 3936, 2427, 0, 1774, 13, 0, 171, 0, 1949, 0, 0, 41, 2766, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 28, 906, 257, 10, 0, 4094, 3, 5, 0, 0, 9, 0, 151, 0, 263, 2015, 1337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3539, 3805, 0, 0, 0, 47, 973, 0, 12, 0, 0, 2545, 3888, 0, 55, 1000, 0, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20188, 12002, 19517, 19300, 17215, 17863, 9095, 15892] avg=16384.0000 max=20188 max_rank=0 min=9095 min_rank=6 imbalance_ratio=1.232178 max_load_reduction=110884 max_load_reduction_pct=0.845978 moved_to_echo=60065 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4067, 1820, 686, 0, 115, 0, 2, 1015, 3637, 8, 536, 3644, 0, 0, 3241, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1751, 6, 42, 10, 3633, 0, 0, 4096, 3659, 0, 0, 2, 0, 2053, 4073, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2935, 20, 1, 591, 1, 0, 4, 4005, 2695, 1093, 573, 0, 2105, 1, 1554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 3910, 0, 0, 3182, 5, 1, 175, 1260, 0, 0, 78, 4096, 0, 12, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 7, 2459, 0, 111, 0, 0, 0, 2193, 3, 8, 175, 278, 2592, 4066, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1301, 6, 3443, 0, 88, 570, 140, 0, 0, 3, 4, 1, 2446, 0, 3265, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1073, 0, 3121, 294, 17, 3339, 0, 0, 11, 44, 4, 26, 13, 536, 4037, 1148, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 381, 0, 0, 4095, 0, 0, 0, 0, 4092, 3519, 0, 33, 801, 1897, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22859, 19794, 15579, 16792, 12446, 11671, 15313, 16618] avg=16384.0000 max=22859 max_rank=0 min=11671 min_rank=5 imbalance_ratio=1.395203 max_load_reduction=108213 max_load_reduction_pct=0.825600 moved_to_echo=56048 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[85, 335, 863, 3, 2, 0, 162, 2, 4052, 0, 1, 4076, 0, 0, 287, 1598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 2, 0, 0, 1109, 0, 47, 3605, 0, 0, 0, 0, 0, 5, 742, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1694, 165, 9, 120, 3, 0, 752, 0, 3509, 2449, 166, 7, 108, 3938, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 1, 3171, 3898, 0, 0, 137, 3, 4, 9, 6, 4025, 1, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2766, 12, 0, 1239, 224, 0, 4093, 1250, 4001, 4008, 0, 26, 362, 20, 4087, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3924, 3967, 4089, 4009, 0, 0, 3, 21, 0, 1192, 3918, 25, 0, 0, 2868, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1351, 3950, 26, 165, 4052, 12, 137, 0, 0, 10, 6, 0, 134, 4094, 1444, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3611, 1838, 0, 0, 23, 0, 4090, 0, 59, 6, 0, 4091, 0, 0, 3839, 0, 3, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11466, 5771, 16971, 15382, 22575, 24519, 16800, 17588] avg=16384.0000 max=24519 max_rank=5 min=5771 min_rank=1 imbalance_ratio=1.496521 max_load_reduction=106553 max_load_reduction_pct=0.812935 moved_to_echo=81482 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[611, 0, 1247, 0, 37, 0, 3910, 1, 3337, 0, 0, 3417, 0, 0, 0, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 8, 0, 3800, 2477, 0, 2712, 1400, 2612, 33, 0, 0, 4087, 2126, 4053, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 261, 192, 2310, 0, 0, 0, 1755, 39, 3851, 1649, 0, 0, 0, 82, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3677, 0, 3891, 0, 807, 0, 0, 0, 198, 0, 3, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4022, 0, 817, 1, 0, 0, 65, 0, 3150, 0, 0, 0, 42, 4072, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3550, 15, 3655, 2968, 1, 274, 1183, 95, 0, 111, 3505, 15, 0, 604, 192, 2045, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 171, 94, 4057, 0, 2, 4092, 0, 56, 2, 0, 0, 1, 1491, 1, 4062, 2089, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 7, 3, 1, 0, 0, 4096, 0, 28, 0, 50, 3924, 1664, 0, 395, 3069, 1954, 23, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12836, 27400, 14443, 12593, 12531, 18940, 17074, 15255] avg=16384.0000 max=27400 max_rank=1 min=12531 min_rank=4 imbalance_ratio=1.672363 max_load_reduction=103672 max_load_reduction_pct=0.790955 moved_to_echo=63800 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[20608, 9856, 8960, 128, 4096, 3072, 5248, 1792, 21888, 256, 640, 25600, 128, 5248, 3968, 20608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 10752, 5888, 15744, 24960, 11008, 512, 5504, 384, 896, 768, 9216, 13568, 128, 3200, 512, 23808, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[17792, 128, 128, 17536, 16128, 128, 14720, 15488, 12544, 256, 0, 3968, 8448, 10240, 13824, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[1408, 13440, 4352, 14080, 1152, 4736, 31360, 128, 2688, 1024, 128, 384, 512, 4096, 2816, 25216, 15104, 9856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[7296, 11264, 2048, 512, 128, 1664, 26880, 512, 1024, 768, 4224, 27520, 16000, 1792, 8576, 12288, 9472, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[7552, 9344, 7552, 5504, 768, 256, 0, 10624, 12032, 22656, 16896, 1664, 256, 6400, 12544, 17920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[640, 26240, 128, 24448, 19584, 7168, 2432, 8576, 1280, 256, 5760, 128, 17152, 128, 6272, 11904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 4480, 12672, 11904, 3200, 2304, 10112, 128, 8064, 1408, 8448, 14336, 128, 2176, 4736, 12544, 31488, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 2, 4094, 0, 0, 2831, 1, 1542, 0, 122, 3, 0, 4011, 0, 0, 4079, 581, 3, 1028, 82, 3469, 72, 2380, 1600, 6, 3911, 4036, 0, 0, 5, 4096, 84, 937, 0, 4092, 3, 0, 0, 16, 2477, 0, 0, 182, 3888, 4096, 3808, 0, 27, 0, 503, 0, 0, 1, 3916, 3681, 207, 2, 0, 0, 26, 1430, 1150, 3489, 0, 24, 278, 0, 0, 3668, 4, 0, 3232, 0, 35, 30, 0, 4065, 57, 0, 3597, 629, 0, 4050, 939, 854, 1, 0, 3, 0, 3931, 0, 0, 10, 14, 0, 3997, 287, 0, 136, 1, 23, 424, 226, 0, 4072, 235, 243, 3528, 74, 4089, 5, 711, 119, 0, 4074, 20, 18, 0, 0, 0, 7, 1, 4039, 4085, 45, 4, 3217, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[122, 14, 4055, 2, 0, 0, 0, 4043, 0, 4031, 0, 262, 600, 16, 0, 4088, 0, 3435, 0, 1, 0, 4096, 0, 1680, 0, 4096, 4093, 0, 0, 269, 353, 0, 37, 122, 91, 1, 3715, 0, 0, 3620, 0, 12, 345, 4096, 3885, 90, 9, 49, 2, 805, 0, 27, 0, 2179, 3699, 0, 2917, 78, 4096, 116, 4096, 51, 0, 729, 0, 10, 4, 0, 25, 0, 0, 2757, 0, 1126, 3269, 1499, 3966, 538, 1606, 0, 931, 0, 3937, 0, 4036, 4073, 0, 0, 1, 0, 0, 0, 5, 0, 3934, 4084, 5, 107, 0, 37, 0, 0, 3, 1, 1, 3492, 0, 447, 134, 3944, 1270, 1328, 0, 1423, 4075, 0, 0, 0, 918, 42, 0, 0, 1428, 2920, 3538, 0, 0, 35] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[2, 5, 512, 0, 0, 1, 9, 4002, 0, 3, 0, 0, 1294, 1, 0, 74, 0, 0, 0, 3142, 2677, 4084, 5, 13, 0, 4048, 4067, 0, 1104, 1, 3226, 0, 0, 3965, 7, 11, 3964, 140, 3500, 17, 0, 2201, 3730, 4096, 4005, 3686, 0, 0, 0, 223, 0, 4, 0, 4040, 1889, 30, 3996, 0, 4049, 0, 183, 2, 17, 206, 0, 0, 0, 10, 3592, 31, 101, 1095, 0, 96, 2710, 2012, 3492, 4096, 3968, 0, 4083, 1, 3896, 35, 3751, 0, 7, 83, 113, 4055, 0, 0, 0, 26, 0, 1, 0, 0, 6, 20, 508, 4073, 0, 362, 4084, 4068, 0, 759, 9, 4093, 48, 7, 0, 0, 222, 0, 0, 517, 2098, 0, 1, 0, 0, 0, 655, 0, 56, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[513, 0, 0, 286, 2049, 4025, 0, 4056, 1, 4082, 1557, 2, 0, 0, 9, 0, 0, 10, 0, 787, 1, 557, 1, 4059, 0, 2767, 3819, 0, 4036, 5, 4073, 1, 4083, 0, 2, 0, 3590, 0, 4095, 45, 0, 0, 1447, 506, 1656, 4093, 1, 302, 0, 0, 0, 0, 25, 2895, 3758, 13, 861, 0, 1, 0, 9, 4096, 3780, 1202, 3, 1172, 16, 0, 2920, 1298, 0, 4067, 0, 0, 3195, 60, 3, 39, 0, 0, 45, 28, 703, 1889, 4047, 0, 0, 0, 2, 14, 191, 0, 0, 3989, 0, 4, 0, 0, 55, 0, 4086, 0, 2, 88, 9, 2328, 1994, 1005, 414, 4040, 44, 891, 0, 0, 3624, 2314, 82, 0, 4003, 2428, 0, 0, 0, 4096, 0, 12, 2715, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 148, 3931, 2, 104, 1663, 13, 4091, 3, 0, 0, 0, 1177, 7, 0, 3, 0, 1, 0, 0, 0, 4096, 3229, 0, 0, 0, 4066, 27, 241, 0, 4081, 0, 0, 1, 4096, 21, 4004, 0, 4045, 3342, 0, 6, 1568, 3721, 4096, 4079, 0, 11, 0, 31, 0, 1, 23, 3864, 3756, 196, 0, 0, 0, 0, 855, 2592, 5, 958, 1, 0, 1278, 21, 2747, 0, 0, 694, 92, 638, 4094, 22, 3798, 19, 2518, 4033, 964, 113, 1680, 129, 4037, 0, 5, 396, 1042, 0, 0, 0, 0, 0, 3615, 4096, 100, 12, 2143, 4, 0, 0, 0, 3, 4093, 4047, 7, 38, 5, 4082, 0, 0, 0, 5, 0, 267, 0, 1800, 2168, 0, 15, 0, 0, 4079, 3971, 0, 47, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 2840, 105, 43, 30, 252, 3147, 517, 3760, 111, 0, 2, 0, 0, 0, 203, 0, 0, 2, 0, 0, 4045, 1, 16, 4096, 4096, 222, 4038, 157, 58, 59, 271, 0, 0, 21, 3999, 0, 0, 1, 1987, 17, 3745, 6, 3983, 4024, 0, 332, 4, 36, 0, 3996, 1, 4044, 4096, 0, 0, 1305, 0, 0, 59, 0, 12, 0, 0, 600, 2, 2188, 4095, 0, 0, 4057, 36, 1715, 3457, 742, 3601, 101, 0, 0, 3657, 0, 2637, 0, 4070, 0, 0, 3, 5, 14, 0, 0, 1, 2599, 995, 4034, 0, 0, 170, 0, 3396, 8, 0, 0, 3786, 3820, 1370, 0, 1153, 4003, 138, 0, 0, 22, 3438, 3975, 0, 0, 3626, 3, 123, 0, 0, 3692, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4044, 0, 0, 4009, 0, 3574, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 3846, 18, 0, 261, 4050, 179, 4090, 219, 4093, 2138, 3, 1523, 0, 4080, 2, 0, 277, 0, 3854, 0, 0, 3995, 101, 0, 0, 3339, 1, 3624, 2166, 0, 2825, 0, 664, 0, 182, 0, 2, 2872, 0, 0, 40, 1, 0, 69, 3110, 3017, 743, 15, 23, 0, 18, 1568, 14, 0, 4052, 0, 0, 1074, 0, 3315, 2074, 3850, 3722, 4095, 0, 2989, 3, 4096, 0, 0, 601, 0, 2, 330, 0, 251, 1161, 1312, 3560, 0, 0, 0, 0, 1, 153, 0, 0, 3881, 3784, 694, 3826, 32, 0, 3, 132, 0, 229, 4096, 640, 0, 0, 0, 0, 742, 38, 5, 3524, 0, 0, 0, 56] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[89, 0, 4085, 0, 0, 2411, 0, 1066, 24, 4096, 28, 0, 4064, 4, 14, 1, 6, 0, 75, 1606, 1, 3065, 4096, 4096, 0, 3313, 3313, 0, 3347, 30, 0, 486, 2075, 247, 0, 162, 1062, 1, 0, 3846, 2, 0, 4088, 3584, 50, 4088, 86, 907, 0, 497, 0, 268, 0, 1831, 56, 22, 188, 0, 3957, 370, 0, 411, 3277, 2986, 0, 1299, 0, 0, 1395, 0, 4, 2716, 6, 0, 1, 260, 4026, 61, 2919, 0, 3393, 0, 4096, 0, 2135, 9, 0, 0, 2856, 4083, 0, 0, 28, 104, 0, 113, 132, 7, 0, 0, 35, 1128, 31, 47, 972, 4057, 0, 182, 149, 4096, 0, 0, 38, 1955, 4096, 3212, 0, 0, 1845, 0, 0, 0, 5, 3891, 0, 0, 1562, 751] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4094, 0, 0, 2831, 1, 1542, 0, 122, 3, 0, 4011, 0, 0, 4079, 2312, 723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 3, 1028, 82, 3469, 72, 2380, 1600, 6, 3640, 0, 0, 0, 5, 4096, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 937, 0, 4092, 3, 0, 0, 16, 2477, 0, 0, 182, 3888, 1615, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 0, 1, 3916, 3681, 207, 2, 0, 0, 26, 1430, 1150, 3489, 0, 1303, 875, 271, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 278, 0, 0, 3668, 4, 0, 3232, 0, 35, 30, 0, 3190, 57, 0, 3597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 0, 4050, 939, 835, 1, 0, 3, 0, 3931, 0, 0, 10, 14, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 136, 1, 23, 424, 226, 0, 4072, 235, 243, 3528, 74, 4089, 5, 711, 1496, 1178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 4074, 20, 18, 0, 0, 0, 7, 1, 4039, 4085, 45, 4, 3217, 2, 3313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19720, 17046, 13237, 16873, 14115, 14409, 16728, 18944] avg=16384.0000 max=19720 max_rank=0 min=13237 min_rank=2 imbalance_ratio=1.203613 max_load_reduction=111352 max_load_reduction_pct=0.849548 moved_to_echo=64196 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 5, 512, 0, 0, 1, 9, 4002, 0, 3, 0, 0, 1294, 1, 0, 74, 2238, 728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3142, 2677, 4084, 5, 13, 0, 3767, 0, 0, 1104, 1, 3226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 7, 11, 3964, 140, 3500, 17, 0, 2201, 3730, 4096, 1579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 4, 0, 4040, 1889, 30, 3996, 0, 4049, 0, 183, 2, 17, 206, 1274, 751, 281, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3592, 31, 101, 1095, 0, 96, 2710, 2012, 2741, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 1, 3896, 35, 3664, 0, 7, 83, 113, 4055, 0, 0, 0, 26, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 20, 508, 4073, 0, 362, 4084, 4068, 0, 759, 9, 4093, 48, 7, 1448, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 0, 517, 2098, 0, 1, 0, 0, 0, 655, 0, 56, 1, 3339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8869, 18019, 23210, 17032, 20452, 15964, 20637, 6889] avg=16384.0000 max=23210 max_rank=2 min=6889 min_rank=7 imbalance_ratio=1.416626 max_load_reduction=107862 max_load_reduction_pct=0.822922 moved_to_echo=63942 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[122, 14, 4055, 2, 0, 0, 0, 4043, 0, 4031, 0, 262, 600, 16, 0, 4088, 55, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3435, 0, 1, 0, 4096, 0, 1680, 0, 3812, 0, 0, 0, 269, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 122, 91, 1, 3715, 0, 0, 3620, 0, 12, 345, 4096, 1532, 0, 9, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 805, 0, 27, 0, 2179, 3699, 0, 2917, 78, 4096, 116, 4096, 51, 0, 729, 1236, 853, 284, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4, 0, 25, 0, 0, 2757, 0, 1126, 3269, 1499, 3113, 538, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 931, 0, 3937, 0, 3943, 4073, 0, 0, 1, 0, 0, 0, 5, 0, 3934, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 107, 0, 37, 0, 0, 3, 1, 1, 3492, 0, 447, 134, 3944, 1270, 1328, 35, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1423, 4075, 0, 0, 0, 918, 42, 0, 0, 1428, 2920, 3538, 0, 0, 35, 3360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18021, 13646, 13629, 21261, 13947, 20908, 11921, 17739] avg=16384.0000 max=21261 max_rank=3 min=11921 min_rank=6 imbalance_ratio=1.297668 max_load_reduction=109811 max_load_reduction_pct=0.837791 moved_to_echo=64515 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[513, 0, 0, 286, 2049, 4025, 0, 4056, 1, 4082, 1557, 2, 0, 0, 9, 0, 2485, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 787, 1, 557, 1, 4059, 0, 2575, 0, 0, 4036, 5, 4073, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 2, 0, 3590, 0, 4095, 45, 0, 0, 1447, 506, 653, 0, 1, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2895, 3758, 13, 861, 0, 1, 0, 9, 4096, 3780, 1202, 527, 0, 192, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1172, 16, 0, 2920, 1298, 0, 4067, 0, 0, 3195, 60, 3, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 28, 703, 1889, 3953, 0, 0, 0, 2, 14, 191, 0, 0, 3989, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 4086, 0, 2, 88, 9, 2328, 1994, 1005, 414, 4040, 44, 891, 1608, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3624, 2314, 82, 0, 4003, 2428, 0, 0, 0, 4096, 0, 12, 2715, 1, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19750, 16105, 14724, 17453, 12773, 10818, 17040, 22409] avg=16384.0000 max=22409 max_rank=7 min=10818 min_rank=5 imbalance_ratio=1.367737 max_load_reduction=108663 max_load_reduction_pct=0.829033 moved_to_echo=63040 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2840, 105, 43, 30, 252, 3147, 517, 3760, 111, 0, 2, 0, 0, 0, 2442, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 0, 0, 2, 0, 0, 4045, 1, 16, 3812, 0, 222, 4038, 157, 58, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 0, 0, 21, 3999, 0, 0, 1, 1987, 17, 3745, 6, 1570, 0, 0, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 36, 0, 3996, 1, 4044, 4096, 0, 0, 1305, 0, 0, 59, 0, 12, 0, 1267, 775, 284, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 600, 2, 2188, 4095, 0, 0, 4057, 36, 1715, 3457, 742, 2826, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3657, 0, 2637, 0, 3976, 0, 0, 3, 5, 14, 0, 0, 1, 2599, 995, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 3396, 8, 0, 0, 3786, 3820, 1370, 0, 1153, 4003, 138, 0, 1582, 1146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 3438, 3975, 0, 0, 3626, 3, 123, 0, 0, 3692, 0, 1, 0, 0, 3363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13982, 12613, 11949, 15973, 19819, 17921, 20572, 18243] avg=16384.0000 max=20572 max_rank=6 min=11949 min_rank=2 imbalance_ratio=1.255615 max_load_reduction=110500 max_load_reduction_pct=0.843048 moved_to_echo=76555 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 148, 3931, 2, 104, 1663, 13, 4091, 3, 0, 0, 0, 1177, 7, 0, 3, 2477, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 3229, 0, 0, 0, 0, 27, 241, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 21, 4004, 0, 4045, 3342, 0, 6, 1568, 3721, 1615, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 1, 23, 3864, 3756, 196, 0, 0, 0, 0, 855, 2592, 5, 958, 1303, 817, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1278, 21, 2747, 0, 0, 694, 92, 638, 4094, 22, 2981, 19, 2518, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 964, 113, 1680, 129, 3944, 0, 5, 396, 1042, 0, 0, 0, 0, 0, 3615, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 12, 2143, 4, 0, 0, 0, 3, 4093, 4047, 7, 38, 5, 4082, 0, 0, 1602, 1178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 267, 0, 1800, 2168, 0, 15, 0, 0, 4079, 3971, 0, 47, 0, 3337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14348, 11675, 22430, 14494, 19138, 15984, 17314, 15689] avg=16384.0000 max=22430 max_rank=2 min=11675 min_rank=1 imbalance_ratio=1.369019 max_load_reduction=108642 max_load_reduction_pct=0.828873 moved_to_echo=68125 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4044, 0, 0, 4009, 0, 3574, 0, 4096, 0, 0, 4, 0, 0, 0, 1315, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 18, 0, 261, 4050, 179, 4090, 219, 3806, 0, 3, 1523, 0, 4080, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 3854, 0, 0, 3995, 101, 0, 0, 3339, 1, 1420, 0, 0, 2825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 664, 0, 182, 0, 2, 2872, 0, 0, 40, 1, 0, 69, 3110, 3017, 743, 1158, 718, 287, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 23, 0, 18, 1568, 14, 0, 4052, 0, 0, 1074, 0, 2597, 2074, 3850, 3722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2989, 3, 3996, 0, 0, 601, 0, 2, 330, 0, 251, 1161, 1312, 3560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 153, 0, 0, 3881, 3784, 694, 3826, 32, 0, 3, 132, 851, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 4096, 640, 0, 0, 0, 0, 742, 38, 5, 3524, 0, 0, 0, 56, 1754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17426, 22077, 15812, 12963, 19007, 18300, 14403, 11084] avg=16384.0000 max=22077 max_rank=1 min=11084 min_rank=7 imbalance_ratio=1.347473 max_load_reduction=108995 max_load_reduction_pct=0.831566 moved_to_echo=62794 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[89, 0, 4085, 0, 0, 2411, 0, 1066, 24, 4096, 28, 0, 4064, 4, 14, 1, 2481, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 75, 1606, 1, 3065, 4096, 4096, 0, 3083, 0, 0, 3347, 30, 0, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2075, 247, 0, 162, 1062, 1, 0, 3846, 2, 0, 4088, 3584, 21, 0, 86, 907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 0, 268, 0, 1831, 56, 22, 188, 0, 3957, 370, 0, 411, 3277, 2986, 15, 866, 230, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1299, 0, 0, 1395, 0, 4, 2716, 6, 0, 1, 260, 3160, 61, 2919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3393, 0, 4096, 0, 2086, 9, 0, 0, 2856, 4083, 0, 0, 28, 104, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 7, 0, 0, 35, 1128, 31, 47, 972, 4057, 0, 182, 149, 4096, 0, 0, 1607, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 1955, 4096, 3212, 0, 0, 1845, 0, 0, 0, 5, 3891, 0, 0, 1562, 751, 2720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18956, 19891, 16081, 15023, 11821, 16768, 12457, 20075] avg=16384.0000 max=20075 max_rank=7 min=11821 min_rank=4 imbalance_ratio=1.225281 max_load_reduction=110997 max_load_reduction_pct=0.846840 moved_to_echo=61121 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[896, 7296, 1152, 5632, 6528, 20096, 13952, 15616, 256, 24576, 0, 256, 14336, 512, 19968, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[768, 256, 23680, 512, 2304, 14976, 384, 25600, 640, 20224, 1792, 384, 11264, 128, 128, 8320, 15872, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 3456, 1408, 2304, 20096, 1408, 128, 22784, 256, 3712, 17920, 4608, 20736, 7040, 14976, 11392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[7424, 4736, 8320, 4096, 20352, 256, 15744, 13568, 2048, 2304, 18560, 19968, 10112, 0, 128, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[17920, 256, 24064, 3072, 26496, 4096, 128, 1152, 4096, 12160, 640, 0, 384, 7936, 9856, 19968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[640, 128, 2560, 128, 8064, 5888, 384, 512, 20992, 25856, 4352, 9856, 2048, 28416, 1536, 3072, 10240, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 2816, 0, 4480, 128, 22784, 23808, 512, 8064, 1536, 12160, 512, 6784, 11520, 13696, 6912, 8192, 5760, 1920, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 3712, 23680, 10496, 128, 2432, 14720, 2560, 896, 128, 5504, 26368, 8320, 128, 7680, 896, 24320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 3, 320, 0, 0, 0, 270, 3678, 0, 0, 100, 0, 4066, 1368, 4095, 4095, 1, 0, 0, 366, 0, 0, 0, 0, 36, 2729, 4093, 0, 8, 0, 2828, 0, 0, 0, 4096, 0, 4096, 0, 57, 4027, 1071, 0, 20, 0, 3904, 395, 3308, 17, 5, 0, 4096, 0, 0, 2, 1056, 4066, 0, 4096, 4096, 0, 3500, 15, 1, 4096, 0, 0, 31, 475, 0, 811, 0, 112, 270, 0, 427, 0, 0, 0, 2246, 68, 0, 0, 1, 53, 4056, 4089, 1, 21, 0, 12, 3971, 3988, 0, 201, 0, 0, 3981, 1223, 4096, 0, 3849, 772, 0, 1, 3946, 0, 3, 4096, 0, 0, 0, 0, 137, 0, 0, 0, 1613, 4095, 0, 0, 0, 1549, 39, 2806, 3957, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[6, 0, 4071, 3497, 132, 0, 1, 435, 410, 0, 13, 27, 116, 1879, 0, 4057, 4092, 0, 82, 122, 0, 0, 0, 0, 16, 0, 4094, 4096, 3, 744, 0, 4063, 0, 788, 522, 0, 3837, 1836, 0, 0, 4006, 4021, 2, 0, 5, 3741, 1, 23, 1, 4090, 2, 2803, 0, 65, 603, 65, 3587, 4096, 3763, 3260, 6, 0, 898, 0, 0, 4, 978, 22, 1945, 188, 0, 5, 0, 434, 0, 4046, 89, 0, 2, 929, 0, 0, 134, 19, 3992, 471, 0, 1, 0, 0, 10, 2078, 4096, 2, 0, 0, 4096, 4073, 1, 218, 67, 1, 2458, 46, 4096, 1660, 0, 1303, 3461, 2, 2, 4090, 0, 0, 1, 0, 4096, 0, 44, 3241, 12, 0, 1055, 0, 3627, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4086, 1, 0, 0, 19, 0, 792, 0, 0, 0, 0, 0, 0, 1152, 9, 4056, 0, 16, 0, 872, 50, 2, 103, 0, 1, 4084, 3314, 3723, 17, 0, 0, 0, 1, 0, 37, 0, 720, 3896, 516, 319, 66, 0, 12, 0, 35, 4095, 3984, 18, 2, 0, 0, 3831, 0, 0, 2153, 2416, 0, 2030, 4059, 3346, 0, 0, 0, 0, 3039, 201, 0, 12, 3960, 0, 0, 676, 32, 0, 1, 4095, 0, 0, 2698, 256, 4077, 214, 0, 0, 3953, 0, 36, 4020, 1679, 162, 1, 3227, 4086, 0, 2, 6, 4077, 348, 0, 3903, 0, 4096, 0, 1912, 0, 0, 0, 1696, 14, 123, 0, 0, 3983, 4096, 4011, 0, 3895, 0, 4085, 1, 0, 0, 0, 0, 428, 4096, 0, 42] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 7, 0, 3949, 234, 0, 0, 100, 4088, 0, 1, 2253, 0, 3968, 1, 3808, 4054, 1934, 255, 0, 0, 10, 44, 0, 0, 6, 4096, 4092, 0, 4073, 0, 2215, 0, 1, 2306, 1, 4057, 2709, 0, 450, 4087, 31, 0, 25, 1, 288, 0, 18, 933, 1, 0, 4063, 0, 21, 32, 0, 4095, 4072, 4073, 3349, 829, 42, 2, 0, 4084, 0, 1, 2787, 1, 0, 0, 10, 12, 0, 18, 3055, 0, 0, 0, 0, 10, 36, 0, 0, 442, 4055, 0, 146, 46, 0, 250, 91, 2689, 112, 0, 103, 4056, 1951, 0, 176, 1058, 4096, 252, 0, 0, 698, 78, 3, 4027, 0, 3, 0, 0, 4056, 22, 8, 3295, 0, 4085, 0, 0, 0, 3600, 0, 2891, 4095, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[3773, 0, 437, 3967, 0, 35, 0, 0, 1781, 0, 0, 2263, 1, 1989, 1113, 3641, 3181, 4, 0, 1, 112, 5, 0, 65, 0, 0, 3914, 3836, 7, 351, 0, 4048, 0, 0, 0, 1022, 3850, 8, 0, 4080, 4093, 0, 0, 2, 0, 0, 188, 79, 9, 1802, 0, 3619, 0, 133, 3983, 42, 24, 4096, 3609, 1726, 0, 4083, 0, 0, 3980, 0, 0, 0, 1, 0, 3610, 0, 0, 815, 3, 1815, 0, 0, 297, 419, 52, 0, 239, 0, 3855, 4096, 2566, 3, 103, 47, 0, 4089, 32, 0, 0, 0, 3952, 4026, 4096, 4034, 115, 0, 6, 0, 65, 662, 2540, 0, 2088, 0, 0, 101, 0, 4096, 82, 0, 38, 2, 251, 1913, 1, 3, 178, 74, 1604, 4094, 50, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4095, 159, 491, 2998, 0, 0, 0, 34, 906, 0, 42, 15, 0, 4052, 1517, 4096, 3091, 0, 478, 0, 2833, 0, 4, 0, 8, 0, 1228, 1209, 235, 598, 0, 2476, 0, 0, 0, 0, 14, 0, 0, 0, 3489, 614, 4080, 2, 1, 86, 2441, 0, 0, 0, 0, 3890, 1, 0, 4091, 0, 116, 685, 4053, 3998, 0, 2964, 4062, 11, 0, 19, 113, 0, 0, 4084, 3657, 29, 222, 2969, 0, 3401, 149, 122, 200, 0, 1, 4, 0, 0, 3, 4069, 0, 331, 4077, 0, 0, 17, 3921, 0, 3, 1363, 2975, 4018, 0, 4096, 0, 1739, 351, 0, 3, 0, 40, 0, 599, 71, 3625, 0, 4014, 836, 0, 0, 3937, 0, 743, 2, 50, 0, 2623, 0, 3702, 3729, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[814, 0, 0, 50, 0, 0, 1, 0, 4096, 0, 999, 854, 321, 3941, 0, 4066, 3876, 6, 0, 0, 0, 0, 0, 94, 0, 0, 4081, 2418, 0, 0, 1, 4081, 1, 229, 0, 0, 4058, 3331, 1, 0, 735, 3106, 24, 0, 0, 125, 109, 0, 2, 2, 38, 4096, 0, 11, 472, 0, 3901, 4094, 4059, 1077, 0, 4092, 0, 1, 3657, 0, 2752, 31, 1, 0, 0, 501, 0, 1, 0, 1251, 0, 0, 0, 513, 61, 254, 8, 0, 3673, 4045, 4096, 0, 2497, 4059, 48, 3480, 3891, 0, 30, 327, 180, 4096, 71, 59, 560, 8, 0, 1, 4067, 7, 0, 97, 1, 0, 0, 2, 3926, 0, 3557, 0, 4058, 16, 12, 4090, 0, 0, 385, 0, 3356, 4084, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[35, 0, 1, 3054, 0, 1932, 0, 1, 3735, 0, 0, 2219, 5, 1850, 615, 4028, 3580, 3, 0, 0, 0, 0, 0, 1, 0, 0, 3966, 4096, 0, 186, 0, 2459, 47, 0, 0, 4, 1, 2108, 0, 0, 3727, 126, 0, 1, 0, 0, 1, 12, 151, 0, 3998, 2110, 61, 0, 4096, 0, 3931, 0, 4048, 3895, 19, 1, 2531, 25, 4048, 66, 3407, 0, 38, 510, 0, 287, 1284, 139, 48, 0, 2, 0, 0, 0, 0, 16, 1795, 0, 0, 2566, 4096, 1, 4072, 0, 693, 3281, 0, 0, 0, 865, 0, 3654, 55, 4096, 0, 2295, 7, 1, 1068, 22, 7, 0, 4094, 4, 0, 4092, 93, 184, 4022, 0, 4096, 0, 2665, 0, 0, 36, 0, 3565, 3049, 4094, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 3949, 234, 0, 0, 100, 4088, 0, 1, 2253, 0, 3968, 1, 3808, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 1934, 255, 0, 0, 10, 44, 0, 0, 6, 4096, 4092, 0, 4073, 0, 2215, 1253, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2306, 1, 4057, 2709, 0, 450, 4087, 31, 0, 25, 1, 288, 0, 18, 1153, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 933, 1, 0, 2010, 0, 21, 32, 0, 4095, 4072, 0, 3349, 829, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 1, 2787, 1, 0, 0, 10, 12, 0, 18, 3055, 0, 0, 0, 0, 4073, 1865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 36, 0, 0, 442, 4055, 0, 146, 46, 0, 250, 91, 2689, 112, 0, 103, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 700, 0, 176, 1058, 4096, 252, 0, 0, 698, 78, 3, 4027, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 22, 8, 3295, 0, 4085, 0, 0, 0, 3600, 0, 2891, 1302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18976, 22130, 16055, 15386, 15906, 8213, 15147, 19259] avg=16384.0000 max=22130 max_rank=1 min=8213 min_rank=5 imbalance_ratio=1.350708 max_load_reduction=108942 max_load_reduction_pct=0.831161 moved_to_echo=58525 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 4071, 3497, 132, 0, 1, 435, 410, 0, 13, 27, 116, 1879, 0, 4057, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 82, 122, 0, 0, 0, 0, 16, 0, 4094, 4096, 3, 744, 0, 4063, 864, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 788, 522, 0, 3837, 1836, 0, 0, 4006, 4021, 2, 0, 5, 3741, 1, 23, 2408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 2, 1387, 0, 65, 603, 65, 3587, 4096, 0, 3260, 6, 0, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 978, 22, 1945, 188, 0, 5, 0, 434, 0, 4046, 89, 0, 2, 929, 3763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 19, 3992, 471, 0, 1, 0, 0, 10, 2078, 4096, 2, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1460, 1, 218, 67, 1, 2458, 46, 4096, 1660, 0, 1303, 3461, 2, 2, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 44, 3241, 12, 0, 1055, 0, 3627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15035, 18381, 21190, 18060, 12405, 10964, 22961, 12076] avg=16384.0000 max=22961 max_rank=6 min=10964 min_rank=5 imbalance_ratio=1.401428 max_load_reduction=108111 max_load_reduction_pct=0.824821 moved_to_echo=58406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4095, 159, 491, 2998, 0, 0, 0, 34, 906, 0, 42, 15, 0, 4052, 1517, 4096, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3091, 0, 478, 0, 2833, 0, 4, 0, 8, 0, 1228, 1209, 235, 598, 0, 2476, 1199, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 3489, 614, 4080, 2, 1, 86, 2441, 0, 2376, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1925, 1, 0, 4091, 0, 116, 685, 0, 3998, 0, 2964, 4062, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 113, 0, 0, 4084, 3657, 29, 222, 2969, 0, 3401, 149, 122, 200, 0, 4053, 1698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 3, 4069, 0, 331, 4077, 0, 0, 17, 3921, 0, 3, 1363, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2975, 1440, 0, 4096, 0, 1739, 351, 0, 3, 0, 40, 0, 599, 71, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 836, 0, 0, 3937, 0, 743, 2, 50, 0, 2623, 0, 3702, 1186, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18948, 13561, 13948, 17853, 20716, 14012, 14939, 17095] avg=16384.0000 max=20716 max_rank=4 min=13561 min_rank=1 imbalance_ratio=1.264404 max_load_reduction=110356 max_load_reduction_pct=0.841949 moved_to_echo=66762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 320, 0, 0, 0, 270, 3678, 0, 0, 100, 0, 4066, 1368, 4095, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 366, 0, 0, 0, 0, 36, 2729, 4093, 0, 8, 0, 2828, 1266, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 57, 4027, 1071, 0, 20, 0, 3904, 395, 3308, 2359, 901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 5, 0, 2016, 0, 0, 2, 1056, 4066, 0, 0, 4096, 0, 3500, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 31, 475, 0, 811, 0, 112, 270, 0, 427, 0, 0, 0, 2246, 4096, 1807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 1, 53, 4056, 4089, 1, 21, 0, 12, 3971, 3988, 0, 201, 0, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1418, 1223, 4096, 0, 3849, 772, 0, 1, 3946, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 1613, 4095, 0, 0, 0, 1549, 39, 2806, 1249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14475, 15626, 24234, 14774, 14371, 16700, 19404, 11488] avg=16384.0000 max=24234 max_rank=2 min=11488 min_rank=7 imbalance_ratio=1.479126 max_load_reduction=106838 max_load_reduction_pct=0.815109 moved_to_echo=61963 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[814, 0, 0, 50, 0, 0, 1, 0, 4096, 0, 999, 854, 321, 3941, 0, 4066, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3876, 6, 0, 0, 0, 0, 0, 94, 0, 0, 4081, 2418, 0, 0, 1, 4081, 1263, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 229, 0, 0, 4058, 3331, 1, 0, 735, 3106, 24, 0, 0, 125, 109, 0, 2422, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 38, 2026, 0, 11, 472, 0, 3901, 4094, 0, 1077, 0, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3657, 0, 2752, 31, 1, 0, 0, 501, 0, 1, 0, 1251, 0, 0, 0, 513, 4059, 1860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 254, 8, 0, 3673, 4045, 4096, 0, 2497, 4059, 48, 3480, 3891, 0, 30, 327, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 1468, 71, 59, 560, 8, 0, 1, 4067, 7, 0, 97, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 0, 3557, 0, 4058, 16, 12, 4090, 0, 0, 385, 0, 3356, 1298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15714, 16026, 15067, 15716, 14626, 26704, 6521, 20698] avg=16384.0000 max=26704 max_rank=5 min=6521 min_rank=6 imbalance_ratio=1.629883 max_load_reduction=104368 max_load_reduction_pct=0.796265 moved_to_echo=68549 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3773, 0, 437, 3967, 0, 35, 0, 0, 1781, 0, 0, 2263, 1, 1989, 1113, 3641, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3181, 4, 0, 1, 112, 5, 0, 65, 0, 0, 3914, 3836, 7, 351, 0, 4048, 1116, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 3850, 8, 0, 4080, 4093, 0, 0, 2, 0, 0, 188, 79, 2380, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1802, 0, 1790, 0, 133, 3983, 42, 24, 4096, 0, 1726, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 0, 1, 0, 3610, 0, 0, 815, 3, 1815, 0, 0, 297, 419, 3609, 1864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 239, 0, 3855, 4096, 2566, 3, 103, 47, 0, 4089, 32, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 1443, 4096, 4034, 115, 0, 6, 0, 65, 662, 2540, 0, 2088, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 82, 0, 38, 2, 251, 1913, 1, 3, 178, 74, 1604, 1302, 50, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19505, 16843, 16630, 17688, 16413, 15290, 19102, 9601] avg=16384.0000 max=19505 max_rank=0 min=9601 min_rank=7 imbalance_ratio=1.190491 max_load_reduction=111567 max_load_reduction_pct=0.851189 moved_to_echo=60406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4086, 1, 0, 0, 19, 0, 792, 0, 0, 0, 0, 0, 0, 1152, 9, 4056, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 872, 50, 2, 103, 0, 1, 4084, 3314, 3723, 17, 0, 0, 0, 1181, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 37, 0, 720, 3896, 516, 319, 66, 0, 12, 0, 35, 4095, 3984, 18, 205, 929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1895, 0, 0, 2153, 2416, 0, 2030, 0, 3346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3039, 201, 0, 12, 3960, 0, 0, 676, 32, 0, 1, 4095, 0, 0, 2698, 256, 4059, 1865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 214, 0, 0, 3953, 0, 36, 4020, 1679, 162, 1, 3227, 4086, 0, 2, 6, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 126, 0, 3903, 0, 4096, 0, 1912, 0, 0, 0, 1696, 14, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 4096, 4011, 0, 3895, 0, 4085, 1, 0, 0, 0, 0, 428, 1302, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10650, 13380, 14833, 11842, 20894, 21683, 15947, 21843] avg=16384.0000 max=21843 max_rank=7 min=10650 min_rank=0 imbalance_ratio=1.333191 max_load_reduction=109229 max_load_reduction_pct=0.833351 moved_to_echo=80367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[35, 0, 1, 3054, 0, 1932, 0, 1, 3735, 0, 0, 2219, 5, 1850, 615, 4028, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3580, 3, 0, 0, 0, 0, 0, 1, 0, 0, 3966, 4096, 0, 186, 0, 2459, 650, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 4, 1, 2108, 0, 0, 3727, 126, 0, 1, 0, 0, 1, 12, 2160, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 3998, 1045, 61, 0, 4096, 0, 3931, 0, 0, 3895, 19, 1, 2531, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 66, 3407, 0, 38, 510, 0, 287, 1284, 139, 48, 0, 2, 0, 0, 0, 4048, 1864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1795, 0, 0, 2566, 4096, 1, 4072, 0, 693, 3281, 0, 0, 0, 865, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1310, 55, 4096, 0, 2295, 7, 1, 1068, 22, 7, 0, 4094, 4, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 184, 4022, 0, 4096, 0, 2665, 0, 0, 36, 0, 3565, 3049, 1302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17769, 15125, 9115, 19753, 15741, 17506, 17051, 19012] avg=16384.0000 max=19753 max_rank=3 min=9115 min_rank=2 imbalance_ratio=1.205627 max_load_reduction=111319 max_load_reduction_pct=0.849297 moved_to_echo=69310 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[12032, 13440, 11776, 128, 23424, 1664, 16000, 9344, 128, 128, 9472, 3712, 21504, 8960, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[22912, 384, 7296, 2944, 6528, 4864, 8192, 1536, 1664, 4736, 128, 18176, 256, 128, 3200, 4480, 31872, 12928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[25984, 2048, 896, 1024, 3456, 128, 256, 256, 128, 4224, 27520, 27648, 384, 6016, 128, 22272, 8832, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 1024, 2944, 5248, 16640, 18048, 640, 4992, 24320, 9088, 4224, 128, 128, 12288, 7168, 3584, 15488, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[19456, 9472, 5504, 20736, 1920, 16128, 3968, 2048, 9344, 7040, 2688, 3200, 18432, 256, 3712, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1152, 6016, 4096, 14208, 128, 256, 15488, 3584, 19840, 19200, 0, 24832, 896, 14720, 7552, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[12928, 256, 5120, 17920, 512, 2048, 896, 896, 18816, 0, 1152, 7808, 512, 22912, 4736, 31872, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[4352, 640, 2176, 128, 16000, 23424, 14976, 4608, 12544, 4352, 1024, 20352, 22784, 128, 256, 2688, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 820, 38, 0, 0, 3631, 3965, 2657, 0, 0, 178, 0, 3471, 0, 4089, 1, 404, 103, 7, 0, 0, 0, 0, 0, 4096, 0, 9, 5, 2959, 3585, 0, 0, 0, 38, 0, 4056, 4004, 4037, 1882, 9, 326, 0, 4088, 4096, 48, 4018, 3578, 0, 0, 2, 28, 4, 701, 2, 0, 146, 375, 4062, 9, 0, 0, 0, 0, 0, 2, 0, 4096, 2, 0, 428, 167, 2, 0, 2, 3963, 0, 23, 4094, 15, 296, 11, 3000, 142, 2841, 20, 4084, 314, 2339, 3862, 0, 156, 0, 1, 0, 0, 1142, 0, 36, 0, 3676, 0, 2, 0, 4045, 76, 0, 1003, 1230, 5, 0, 3243, 2015, 0, 0, 3654, 79, 4067, 0, 0, 0, 0, 0, 4093, 0, 4000, 0, 2949, 370] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 475, 0, 0, 0, 4093, 6, 3748, 3954, 0, 0, 3665, 1708, 0, 914, 2, 30, 0, 0, 0, 4036, 0, 0, 130, 3074, 145, 1700, 1384, 27, 435, 0, 3, 23, 0, 1, 0, 0, 4087, 523, 2, 3003, 0, 2, 0, 4095, 4094, 20, 0, 1, 0, 0, 1, 0, 0, 0, 8, 0, 0, 20, 4, 0, 0, 0, 0, 1469, 771, 306, 0, 3945, 48, 0, 0, 855, 4096, 12, 1, 4029, 0, 22, 4003, 3326, 1545, 34, 4075, 4096, 3888, 4041, 4037, 3, 3283, 0, 0, 0, 0, 222, 17, 3152, 1, 0, 77, 1, 3183, 4084, 9, 2, 661, 157, 3, 2, 212, 734, 4073, 3712, 122, 803, 4091, 4096, 0, 255, 44, 4091, 0, 0, 3953, 0, 15, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4093, 145, 3780, 0, 332, 2387, 3393, 4067, 4058, 247, 2, 0, 49, 2, 2, 9, 0, 2698, 0, 0, 0, 71, 4096, 48, 3963, 4067, 884, 5, 0, 7, 0, 0, 0, 46, 0, 4094, 0, 0, 3680, 18, 4096, 2217, 3024, 4004, 3696, 4096, 4041, 1, 1, 12, 0, 0, 0, 0, 418, 0, 0, 145, 4032, 178, 0, 0, 205, 0, 4077, 0, 0, 256, 0, 0, 0, 0, 19, 0, 0, 944, 16, 4096, 0, 0, 133, 0, 0, 0, 0, 1674, 3818, 2751, 0, 0, 2333, 184, 0, 0, 55, 3826, 19, 0, 31, 3053, 0, 436, 0, 4058, 4096, 0, 0, 868, 33, 2037, 0, 1093, 77, 0, 1, 0, 3762, 0, 1, 0, 0, 1605, 4, 1224, 3993, 0, 4090, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4055, 0, 0, 0, 0, 4055, 167, 4036, 916, 4093, 4094, 0, 0, 451, 0, 1021, 0, 3505, 11, 270, 0, 4091, 0, 152, 0, 4027, 0, 14, 0, 3995, 0, 2, 0, 0, 7, 0, 0, 2502, 4091, 3059, 0, 94, 4094, 4093, 0, 3972, 4077, 2596, 0, 0, 0, 12, 0, 0, 0, 0, 7, 956, 3919, 1, 245, 0, 0, 0, 5, 1, 10, 0, 0, 3494, 0, 849, 531, 0, 3025, 0, 0, 3884, 0, 3937, 0, 0, 0, 3996, 58, 162, 1036, 2456, 91, 46, 0, 0, 0, 0, 12, 4089, 0, 0, 0, 1950, 0, 184, 3, 4096, 3311, 0, 0, 0, 694, 16, 3211, 4063, 1554, 1, 0, 0, 3819, 0, 143, 0, 0, 3687, 0, 223, 14, 37, 3703, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[221, 0, 725, 3818, 0, 3932, 2606, 0, 4094, 4021, 15, 731, 3344, 3808, 0, 2, 0, 3813, 857, 7, 0, 0, 0, 0, 4094, 3660, 4, 0, 70, 3864, 35, 0, 237, 0, 1408, 4096, 0, 1303, 2105, 0, 3776, 94, 0, 3122, 0, 4096, 4079, 586, 0, 5, 0, 0, 2, 1, 0, 1, 27, 82, 11, 0, 1, 0, 0, 433, 1989, 2, 23, 0, 0, 3313, 231, 0, 0, 2784, 4096, 24, 1, 4018, 0, 3124, 0, 4096, 152, 2865, 50, 0, 0, 1161, 17, 0, 4028, 0, 0, 3927, 734, 0, 231, 0, 0, 2814, 0, 50, 0, 4096, 0, 1718, 4082, 0, 0, 14, 4093, 6, 0, 0, 64, 3728, 4088, 262, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[58, 0, 1, 0, 0, 489, 1750, 4096, 3593, 4093, 0, 0, 16, 3, 4096, 0, 0, 199, 0, 138, 0, 66, 0, 0, 4, 3206, 0, 3946, 14, 3631, 0, 0, 411, 1, 0, 756, 8, 0, 0, 337, 3581, 0, 4076, 4094, 0, 4081, 3451, 584, 0, 0, 6, 0, 0, 0, 4078, 0, 27, 0, 1, 0, 184, 0, 0, 0, 4000, 3417, 0, 6, 156, 4096, 0, 0, 0, 4029, 4089, 0, 14, 3797, 0, 0, 14, 0, 0, 1, 0, 0, 3959, 0, 3890, 14, 3988, 0, 0, 0, 0, 4096, 49, 0, 1, 1744, 2, 3850, 0, 4096, 2, 0, 1724, 0, 3792, 0, 0, 3827, 0, 922, 2444, 254, 4090, 1453, 4043, 0, 0, 0, 0, 120, 4018, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 91, 131, 4096, 3250, 4096, 140, 0, 0, 0, 35, 0, 0, 0, 3996, 0, 0, 33, 4095, 0, 0, 2248, 4093, 27, 114, 0, 4093, 0, 0, 336, 161, 4, 1808, 0, 2842, 1181, 0, 0, 1, 3459, 3953, 16, 4030, 4069, 3321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3573, 262, 9, 0, 0, 0, 28, 3934, 0, 0, 0, 841, 4071, 0, 0, 48, 3949, 4068, 1, 15, 4058, 426, 4033, 2, 1240, 0, 3875, 3801, 127, 3980, 908, 3716, 66, 2957, 0, 0, 0, 0, 1223, 958, 262, 0, 886, 0, 7, 28, 4096, 0, 0, 949, 0, 0, 0, 34, 29, 0, 3932, 48, 0, 4043, 4096, 1236, 48, 0, 2629, 0, 708, 23, 42, 0, 88] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 82, 3298, 4054, 4096, 4090, 6, 0, 0, 3301, 2, 10, 4050, 2846, 0, 0, 0, 0, 195, 0, 3979, 4079, 0, 2790, 0, 4096, 0, 0, 0, 340, 2, 0, 4079, 46, 4008, 0, 4044, 0, 0, 667, 3813, 1907, 4096, 3798, 20, 0, 0, 22, 0, 0, 1279, 7, 0, 4094, 4092, 0, 0, 0, 33, 0, 41, 558, 0, 0, 106, 3748, 0, 0, 0, 4060, 4050, 0, 1, 3611, 0, 3205, 0, 0, 0, 1491, 2033, 0, 1127, 0, 0, 0, 2162, 2, 488, 0, 0, 5, 0, 3, 0, 4096, 0, 0, 41, 3910, 265, 0, 4075, 0, 968, 0, 0, 1084, 0, 3241, 85, 82, 4094, 0, 1815, 0, 0, 3220, 0, 0, 0, 82, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 820, 38, 0, 0, 3631, 3374, 2657, 0, 0, 178, 0, 3471, 0, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 404, 103, 7, 0, 0, 0, 0, 0, 4096, 0, 9, 5, 2959, 3585, 0, 0, 67, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 4056, 4004, 4037, 1882, 9, 326, 0, 4088, 1874, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 28, 4, 701, 2, 0, 146, 375, 4062, 9, 0, 0, 0, 0, 0, 3578, 4018, 0, 1861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 2, 0, 428, 167, 2, 0, 2, 3963, 0, 23, 2702, 15, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3000, 142, 2841, 20, 4084, 314, 2339, 3862, 0, 88, 0, 1, 0, 0, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 3676, 0, 2, 0, 4045, 76, 0, 1003, 1230, 5, 0, 3243, 2015, 1320, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3654, 79, 4067, 0, 0, 0, 0, 0, 4093, 0, 4000, 0, 2949, 370, 591, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18259, 11307, 20362, 14786, 11698, 17844, 17012, 19804] avg=16384.0000 max=20362 max_rank=2 min=11307 min_rank=1 imbalance_ratio=1.242798 max_load_reduction=110710 max_load_reduction_pct=0.844650 moved_to_echo=66358 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4055, 0, 0, 0, 0, 4055, 143, 4036, 0, 4093, 4094, 0, 0, 451, 0, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3505, 11, 270, 0, 4091, 0, 152, 0, 4027, 0, 14, 0, 3995, 0, 2, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2502, 4091, 3059, 0, 94, 4094, 1873, 0, 0, 0, 2596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 7, 956, 3919, 1, 245, 0, 0, 0, 4077, 3972, 916, 1859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 10, 0, 0, 3494, 0, 849, 531, 0, 3025, 0, 0, 2564, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 58, 162, 1036, 2456, 91, 46, 0, 0, 0, 0, 12, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1950, 0, 184, 3, 4096, 3311, 0, 0, 0, 694, 16, 3211, 4063, 1252, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1554, 1, 0, 0, 3819, 0, 143, 0, 0, 3687, 0, 223, 14, 37, 3703, 1, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21948, 16135, 18316, 15964, 14416, 11946, 19141, 13206] avg=16384.0000 max=21948 max_rank=0 min=11946 min_rank=5 imbalance_ratio=1.339600 max_load_reduction=109124 max_load_reduction_pct=0.832550 moved_to_echo=58709 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4093, 145, 3780, 0, 332, 2387, 2887, 4067, 0, 247, 2, 0, 49, 2, 2, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2698, 0, 0, 0, 71, 4096, 48, 3963, 4067, 884, 5, 0, 7, 0, 0, 1003, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 4094, 0, 0, 3680, 18, 4096, 2217, 3024, 1832, 3696, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0, 0, 0, 0, 418, 0, 0, 145, 4032, 178, 0, 0, 205, 0, 4041, 4096, 4058, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 256, 0, 0, 0, 0, 19, 0, 0, 944, 16, 2703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 1674, 3818, 2751, 0, 0, 1309, 184, 0, 0, 55, 3826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 31, 3053, 0, 436, 0, 4058, 4096, 0, 0, 868, 33, 2037, 0, 1093, 1321, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 1, 0, 3762, 0, 1, 0, 0, 1605, 4, 1224, 3993, 0, 4090, 0, 506, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18002, 16914, 22704, 19005, 8015, 13750, 17398, 15284] avg=16384.0000 max=22704 max_rank=2 min=8015 min_rank=4 imbalance_ratio=1.385742 max_load_reduction=108368 max_load_reduction_pct=0.826782 moved_to_echo=54447 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[221, 0, 725, 3818, 0, 3932, 2218, 0, 0, 4021, 15, 731, 3344, 3808, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3813, 857, 7, 0, 0, 0, 0, 4094, 3660, 4, 0, 70, 3864, 35, 0, 1732, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 1408, 4096, 0, 1303, 2105, 0, 3776, 94, 0, 1429, 0, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2, 1, 0, 1, 27, 82, 11, 0, 1, 0, 0, 433, 4079, 4096, 4094, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1989, 2, 23, 0, 0, 3313, 231, 0, 0, 2784, 4096, 24, 1, 2651, 0, 3124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 152, 2865, 50, 0, 0, 1161, 17, 0, 2259, 0, 0, 3927, 734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 2814, 0, 50, 0, 4096, 0, 1718, 4082, 0, 0, 14, 4093, 6, 1296, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 3728, 4088, 262, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 388, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22835, 18207, 15034, 14250, 18238, 15261, 18675, 8572] avg=16384.0000 max=22835 max_rank=0 min=8572 min_rank=7 imbalance_ratio=1.393738 max_load_reduction=108237 max_load_reduction_pct=0.825783 moved_to_echo=60746 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 475, 0, 0, 0, 3483, 6, 0, 3954, 0, 0, 3665, 1708, 0, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 30, 0, 0, 0, 4036, 0, 0, 130, 3074, 145, 1700, 1384, 27, 435, 0, 1412, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 23, 0, 1, 0, 0, 4087, 523, 2, 3003, 0, 2, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 8, 0, 0, 20, 4, 0, 0, 0, 4094, 4095, 3748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1469, 771, 306, 0, 3945, 48, 0, 0, 855, 4096, 12, 1, 2659, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 3326, 1545, 34, 4075, 4096, 3888, 4041, 4037, 3, 1841, 0, 0, 0, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3152, 1, 0, 77, 1, 3183, 4084, 9, 2, 661, 157, 3, 2, 212, 734, 1299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 3712, 122, 803, 4091, 4096, 0, 255, 44, 4091, 0, 0, 3953, 0, 15, 2, 610, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14205, 12446, 7664, 11971, 14184, 31111, 13594, 25897] avg=16384.0000 max=31111 max_rank=5 min=7664 min_rank=2 imbalance_ratio=1.898865 max_load_reduction=99961 max_load_reduction_pct=0.762642 moved_to_echo=84786 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[58, 0, 1, 0, 0, 489, 1485, 4096, 0, 4093, 0, 0, 16, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 0, 138, 0, 66, 0, 0, 4, 3206, 0, 3946, 14, 3631, 0, 0, 1718, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 411, 1, 0, 756, 8, 0, 0, 337, 3581, 0, 4076, 1865, 0, 0, 0, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4078, 0, 27, 0, 1, 0, 184, 0, 0, 0, 3451, 4081, 3593, 1863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 3417, 0, 6, 156, 4096, 0, 0, 0, 4029, 4089, 0, 14, 2497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1, 0, 0, 3959, 0, 3890, 14, 2231, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 1, 1744, 2, 3850, 0, 4096, 2, 0, 1724, 0, 3792, 0, 0, 3827, 1229, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 922, 2444, 254, 4090, 1453, 4043, 0, 0, 0, 0, 120, 4018, 0, 0, 0, 265, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14337, 12993, 11619, 17284, 22304, 14205, 20682, 17648] avg=16384.0000 max=22304 max_rank=4 min=11619 min_rank=2 imbalance_ratio=1.361328 max_load_reduction=108768 max_load_reduction_pct=0.829834 moved_to_echo=74839 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 91, 131, 3485, 3250, 0, 140, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 33, 4095, 0, 0, 2248, 4093, 27, 114, 0, 4093, 0, 0, 1271, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 161, 4, 1808, 0, 2842, 1181, 0, 0, 1, 3459, 1808, 16, 0, 0, 3321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3573, 262, 9, 0, 0, 0, 28, 4069, 4030, 4096, 1796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 0, 0, 0, 841, 4071, 0, 0, 48, 3949, 4068, 1, 15, 2679, 426, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1240, 0, 3875, 3801, 127, 3980, 908, 3716, 66, 1659, 0, 0, 0, 0, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 958, 262, 0, 886, 0, 7, 28, 4096, 0, 0, 949, 0, 0, 0, 34, 29, 1308, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 48, 0, 4043, 4096, 1236, 48, 0, 2629, 0, 708, 23, 42, 0, 88, 611, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7132, 20041, 14937, 17863, 24065, 20597, 8906, 17531] avg=16384.0000 max=24065 max_rank=4 min=7132 min_rank=0 imbalance_ratio=1.468811 max_load_reduction=107007 max_load_reduction_pct=0.816399 moved_to_echo=71099 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 82, 2807, 4054, 0, 4090, 6, 0, 0, 3301, 2, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 2846, 0, 0, 0, 0, 195, 0, 3979, 4079, 0, 2790, 0, 4096, 0, 0, 930, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 2, 0, 4079, 46, 4008, 0, 4044, 0, 0, 306, 3813, 0, 0, 3798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 22, 0, 0, 1279, 7, 0, 4094, 4092, 0, 0, 0, 33, 0, 4096, 1907, 4096, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 558, 0, 0, 106, 3748, 0, 0, 0, 4060, 4050, 0, 1, 2383, 0, 3205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1491, 2033, 0, 1127, 0, 0, 0, 1212, 2, 488, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 41, 3910, 265, 0, 4075, 0, 968, 0, 0, 1084, 1164, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3241, 85, 82, 4094, 0, 1815, 0, 0, 3220, 0, 0, 0, 82, 0, 0, 491, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14354, 23029, 20436, 19949, 18152, 6358, 15664, 13130] avg=16384.0000 max=23029 max_rank=1 min=6358 min_rank=5 imbalance_ratio=1.405579 max_load_reduction=108043 max_load_reduction_pct=0.824303 moved_to_echo=53304 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[4480, 17280, 896, 512, 128, 12416, 4352, 256, 18560, 26240, 1152, 8576, 4480, 23424, 512, 128, 8192, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[8448, 1024, 5120, 3840, 512, 14720, 19968, 22272, 0, 20736, 4352, 768, 10624, 9344, 8192, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[14080, 5504, 4992, 640, 1152, 23168, 512, 896, 640, 15744, 27392, 1024, 128, 20864, 512, 14720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1280, 3456, 128, 18304, 128, 4608, 3328, 32512, 7808, 1792, 12544, 2304, 5504, 2176, 10880, 12928, 10240, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 768, 128, 5888, 256, 512, 12928, 12416, 256, 512, 0, 256, 512, 31488, 30336, 24704, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[4224, 11776, 1920, 15104, 10112, 10240, 18176, 13696, 15616, 256, 10624, 256, 512, 3968, 896, 14720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1024, 640, 1536, 14848, 8192, 10752, 21120, 3968, 15872, 5504, 18816, 11008, 7680, 0, 0, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[5760, 11904, 6528, 4992, 32128, 9984, 7296, 384, 128, 15232, 4224, 2304, 16128, 256, 10880, 512, 3584, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 87, 0, 185, 644, 4096, 869, 0, 0, 23, 4096, 0, 0, 2, 0, 5, 0, 2815, 38, 106, 0, 1, 0, 3371, 0, 91, 1474, 3025, 251, 1, 12, 5, 0, 4093, 4096, 1, 4092, 4, 0, 48, 4083, 0, 3658, 1, 0, 0, 8, 7, 0, 0, 4096, 1, 29, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 363, 0, 0, 710, 4096, 448, 3, 1317, 0, 4082, 0, 0, 4095, 20, 3989, 15, 0, 3845, 145, 0, 1, 1923, 2796, 1, 3904, 4096, 3, 47, 4096, 1175, 4096, 8, 1831, 0, 0, 3535, 44, 0, 0, 4096, 0, 3361, 749, 2983, 0, 3928, 1569, 2384, 0, 3677, 3999, 0, 0, 21, 4073] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 24, 138, 4074, 0, 6, 0, 0, 16, 368, 3973, 4095, 4, 155, 0, 3715, 0, 0, 458, 3165, 3898, 87, 198, 4096, 4095, 3604, 27, 3274, 0, 832, 3225, 602, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1725, 2987, 352, 1342, 0, 0, 3855, 43, 127, 0, 0, 0, 64, 1078, 0, 0, 0, 1248, 0, 0, 4047, 0, 170, 0, 175, 2, 34, 0, 2, 981, 0, 0, 0, 4096, 0, 3120, 4058, 2384, 0, 233, 445, 4096, 0, 3852, 2275, 4052, 0, 285, 100, 0, 0, 52, 977, 0, 0, 3512, 0, 88, 3, 1, 4096, 3983, 0, 0, 0, 231, 0, 0, 0, 0, 3899, 175, 4095, 558, 0, 1, 3933, 2128, 3832, 2, 4096, 0, 0, 0, 0, 4052] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 22, 3132, 4082, 3493, 4096, 2435, 0, 3996, 3826, 412, 0, 0, 0, 103, 241, 0, 9, 148, 806, 4053, 0, 0, 3296, 0, 2864, 1850, 4096, 2411, 0, 0, 0, 0, 0, 0, 99, 20, 1653, 0, 0, 3623, 428, 0, 0, 23, 0, 1, 0, 3, 2, 302, 3858, 4002, 1600, 4, 0, 0, 0, 0, 4073, 0, 1162, 61, 0, 76, 13, 8, 0, 0, 1, 254, 0, 0, 14, 1177, 0, 0, 0, 175, 4039, 4095, 4054, 0, 259, 4085, 34, 3873, 3538, 101, 0, 0, 548, 13, 1471, 4078, 880, 0, 10, 0, 3343, 4, 27, 28, 0, 0, 43, 0, 0, 0, 4088, 336, 4096, 4087, 0, 0, 4073, 7, 3775, 0, 4096, 39, 3935, 10, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[2, 56, 0, 14, 1825, 2, 0, 3765, 4096, 4086, 0, 3998, 3512, 0, 0, 2, 18, 39, 0, 3978, 1, 0, 0, 1596, 0, 3, 169, 0, 3189, 0, 0, 0, 0, 0, 0, 70, 0, 0, 4075, 0, 4066, 49, 5, 3978, 39, 23, 0, 0, 0, 0, 3162, 0, 0, 0, 6, 107, 0, 0, 0, 0, 218, 77, 3953, 3817, 0, 4049, 0, 31, 966, 0, 428, 0, 0, 55, 0, 0, 60, 4092, 3, 39, 0, 702, 3799, 0, 0, 3381, 2023, 3952, 972, 225, 0, 0, 0, 0, 15, 0, 4090, 3893, 303, 12, 2, 0, 3579, 0, 11, 4096, 0, 0, 3857, 0, 3101, 38, 4096, 3767, 4052, 4086, 1, 0, 4069, 61, 4096, 0, 761, 4025, 200, 0, 88, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[3206, 0, 0, 1, 0, 0, 0, 0, 0, 3825, 0, 9, 3644, 4041, 3, 0, 3206, 0, 0, 0, 118, 0, 0, 0, 4055, 626, 504, 0, 3967, 223, 0, 3987, 4062, 0, 0, 0, 0, 0, 13, 0, 3016, 0, 80, 4094, 0, 0, 1, 0, 3907, 0, 0, 6, 0, 0, 1, 407, 940, 12, 0, 1, 0, 0, 4096, 0, 4096, 0, 1, 4095, 4074, 0, 0, 0, 0, 0, 5, 1, 0, 3910, 0, 4005, 0, 0, 1619, 2, 4095, 3124, 4094, 4096, 0, 1092, 0, 0, 0, 0, 1094, 65, 0, 4095, 22, 4095, 0, 48, 0, 7, 0, 3148, 25, 2572, 1395, 0, 46, 0, 4044, 0, 1619, 1, 2634, 0, 4085, 0, 4096, 0, 4096, 218, 382, 0, 176, 2749] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[480, 0, 0, 0, 0, 238, 92, 909, 1893, 4090, 0, 4073, 2883, 0, 0, 44, 3872, 0, 0, 1, 60, 3955, 10, 0, 2141, 0, 4096, 0, 4053, 0, 965, 4096, 0, 1, 12, 12, 0, 1154, 4093, 0, 441, 0, 0, 665, 57, 2, 1, 6, 4073, 1, 971, 27, 0, 903, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 43, 30, 0, 257, 0, 3, 39, 0, 0, 0, 3, 0, 0, 5, 3843, 0, 1623, 0, 0, 4094, 4096, 0, 3747, 353, 4016, 117, 0, 217, 0, 0, 0, 0, 3944, 0, 8, 0, 1, 0, 115, 4063, 0, 0, 0, 1504, 3, 3776, 0, 22, 0, 4067, 4095, 0, 0, 4015, 0, 3651, 4054, 4016, 376, 4096, 4080, 4092, 49, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[398, 0, 459, 1, 551, 0, 4051, 0, 0, 0, 22, 2968, 3307, 23, 1, 0, 236, 20, 0, 4042, 0, 1556, 124, 3925, 4002, 0, 4096, 0, 6, 1, 0, 907, 21, 0, 0, 4, 0, 0, 3837, 0, 155, 0, 4095, 118, 0, 79, 33, 9, 4088, 0, 0, 5, 9, 0, 3950, 4096, 0, 0, 244, 147, 0, 0, 4040, 3294, 0, 0, 23, 54, 0, 0, 65, 937, 0, 15, 0, 0, 46, 3740, 1600, 0, 0, 3912, 191, 0, 4095, 0, 1, 3850, 12, 0, 85, 0, 0, 0, 0, 33, 0, 3715, 0, 330, 80, 3124, 4095, 1, 0, 984, 1048, 0, 2522, 10, 3, 0, 3987, 3440, 4096, 0, 4096, 0, 4068, 852, 2, 0, 4096, 4093, 4093, 708, 0, 4050] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 1643, 7, 0, 603, 0, 362, 492, 0, 3682, 2649, 4042, 4096, 3801, 250, 0, 0, 3978, 61, 72, 0, 3197, 288, 2911, 4072, 0, 3059, 0, 4078, 0, 0, 1840, 0, 0, 1, 0, 0, 3, 2116, 688, 0, 11, 425, 0, 0, 2795, 1119, 459, 714, 62, 0, 133, 339, 44, 3772, 0, 1, 13, 22, 0, 0, 0, 4066, 76, 2835, 0, 0, 0, 0, 14, 2, 0, 0, 0, 50, 196, 30, 3527, 130, 4096, 1, 0, 1609, 4093, 4036, 3401, 4096, 3911, 1, 0, 0, 0, 0, 4, 755, 10, 0, 4096, 3762, 0, 112, 0, 1465, 0, 0, 1944, 0, 0, 5, 0, 0, 1, 4096, 479, 103, 3455, 3916, 0, 3734, 39, 64, 30, 3960, 393, 556, 0, 0, 4023] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 22, 3132, 4082, 3493, 4096, 2435, 0, 3996, 2987, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 241, 0, 9, 148, 806, 4053, 0, 0, 22, 0, 2864, 1850, 4096, 2411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 20, 1653, 0, 0, 3623, 428, 0, 0, 23, 4096, 2069, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 2, 302, 3858, 4002, 1600, 4, 0, 0, 0, 0, 4073, 0, 2731, 1537, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1162, 61, 0, 76, 13, 8, 0, 0, 1, 254, 0, 0, 14, 1177, 0, 0, 4088, 2868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 4039, 4095, 4054, 0, 259, 1354, 34, 3873, 3538, 101, 0, 0, 548, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1471, 4078, 880, 0, 10, 0, 3343, 4, 27, 28, 0, 0, 43, 0, 0, 0, 957, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 4096, 4087, 0, 0, 0, 7, 1281, 0, 0, 39, 3935, 10, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[24655, 16603, 13216, 19318, 9722, 22083, 11680, 13795] avg=16384.0000 max=24655 max_rank=0 min=9722 min_rank=4 imbalance_ratio=1.504822 max_load_reduction=106417 max_load_reduction_pct=0.811897 moved_to_echo=57280 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 56, 0, 14, 1825, 2, 0, 3765, 4096, 4086, 0, 3998, 2742, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 39, 0, 3978, 1, 0, 0, 1596, 0, 3, 2, 0, 3189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 4075, 0, 4066, 49, 5, 3978, 39, 23, 0, 0, 761, 106, 1202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3162, 0, 0, 0, 6, 107, 0, 0, 0, 0, 218, 77, 3953, 3817, 2642, 1668, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 31, 966, 0, 428, 0, 0, 55, 0, 0, 60, 4092, 3, 39, 4096, 2867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 702, 3799, 0, 0, 3381, 2023, 1310, 972, 225, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3893, 303, 12, 2, 0, 3579, 0, 11, 4096, 0, 0, 3857, 0, 3101, 38, 1038, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3767, 4052, 4086, 1, 0, 0, 61, 1390, 0, 0, 4025, 200, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20588, 8826, 14374, 15711, 16686, 12427, 24790, 17670] avg=16384.0000 max=24790 max_rank=6 min=8826 min_rank=1 imbalance_ratio=1.513062 max_load_reduction=106282 max_load_reduction_pct=0.810867 moved_to_echo=71573 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 24, 138, 4074, 0, 6, 0, 0, 16, 368, 3973, 3197, 4, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3715, 0, 0, 458, 3165, 3898, 87, 198, 4096, 4095, 23, 27, 3274, 0, 832, 3225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1725, 2987, 352, 1342, 0, 0, 4096, 2263, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3855, 43, 127, 0, 0, 0, 64, 1078, 0, 0, 0, 1248, 0, 0, 4047, 0, 2709, 1560, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 175, 2, 34, 0, 2, 981, 0, 0, 0, 4096, 0, 3120, 4058, 2384, 3899, 2770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 445, 4096, 0, 3852, 2275, 1343, 0, 285, 100, 0, 0, 52, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3512, 0, 88, 3, 1, 4096, 3983, 0, 0, 0, 231, 0, 0, 0, 0, 971, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 4095, 558, 0, 1, 0, 2128, 1301, 2, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11955, 27093, 14531, 16049, 21691, 13658, 13783, 12312] avg=16384.0000 max=27093 max_rank=1 min=11955 min_rank=0 imbalance_ratio=1.653625 max_load_reduction=103979 max_load_reduction_pct=0.793297 moved_to_echo=61444 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 87, 0, 185, 644, 4096, 869, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 2815, 38, 106, 0, 1, 0, 19, 0, 91, 1474, 3025, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 5, 0, 4093, 4096, 1, 4092, 4, 0, 48, 4083, 0, 3658, 1, 0, 3677, 2116, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 7, 0, 0, 4096, 1, 29, 0, 0, 0, 0, 0, 36, 0, 0, 2667, 971, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 363, 0, 0, 710, 4096, 448, 3, 4096, 2766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 0, 4082, 0, 0, 4095, 20, 1322, 15, 0, 3845, 145, 0, 1, 1923, 2796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3904, 4096, 3, 47, 4096, 1175, 4096, 8, 1831, 0, 0, 3535, 44, 0, 0, 604, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3361, 749, 2983, 0, 0, 1569, 809, 0, 0, 3999, 0, 0, 21, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14091, 7827, 27049, 9051, 12484, 19561, 23445, 17564] avg=16384.0000 max=27049 max_rank=2 min=7827 min_rank=1 imbalance_ratio=1.650940 max_load_reduction=104023 max_load_reduction_pct=0.793633 moved_to_echo=73054 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3206, 0, 0, 1, 0, 0, 0, 0, 0, 3825, 0, 9, 2843, 4041, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 0, 0, 0, 118, 0, 0, 0, 4055, 626, 0, 0, 3967, 223, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 13, 0, 3016, 0, 80, 4094, 0, 0, 1, 0, 4096, 319, 1208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 0, 0, 6, 0, 0, 1, 407, 940, 12, 0, 1, 0, 0, 4096, 0, 2744, 1672, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4095, 4074, 0, 0, 0, 0, 0, 5, 1, 0, 3910, 0, 4005, 4044, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1619, 2, 4095, 3124, 4094, 1352, 0, 1092, 0, 0, 0, 0, 1094, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 22, 4095, 0, 48, 0, 7, 0, 3148, 25, 2572, 1395, 0, 46, 0, 1042, 801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1619, 1, 2634, 0, 0, 0, 1382, 0, 0, 218, 382, 0, 176, 2749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13928, 16182, 16889, 13971, 27108, 16537, 17296, 9161] avg=16384.0000 max=27108 max_rank=4 min=9161 min_rank=7 imbalance_ratio=1.654541 max_load_reduction=103964 max_load_reduction_pct=0.793182 moved_to_echo=70102 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[398, 0, 459, 1, 551, 0, 4051, 0, 0, 0, 22, 2968, 2582, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 20, 0, 4042, 0, 1556, 124, 3925, 4002, 0, 26, 0, 6, 1, 0, 907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 4, 0, 0, 3837, 0, 155, 0, 4095, 118, 0, 79, 33, 9, 4096, 2572, 1202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 5, 9, 0, 3950, 4096, 0, 0, 244, 147, 0, 0, 4040, 3294, 2574, 0, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 54, 0, 0, 65, 937, 0, 15, 0, 0, 46, 3740, 1600, 0, 3987, 2866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 191, 0, 4095, 0, 1, 1276, 12, 0, 85, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3715, 0, 330, 80, 3124, 4095, 1, 0, 984, 1048, 0, 2522, 10, 3, 0, 0, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3440, 4096, 0, 4096, 0, 0, 852, 2, 0, 0, 4093, 4093, 708, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11056, 14845, 16221, 23945, 13333, 9605, 16637, 25430] avg=16384.0000 max=25430 max_rank=7 min=9605 min_rank=5 imbalance_ratio=1.552124 max_load_reduction=105642 max_load_reduction_pct=0.805984 moved_to_echo=65005 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[480, 0, 0, 0, 0, 238, 92, 909, 1893, 4090, 0, 4073, 2251, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3872, 0, 0, 1, 60, 3955, 10, 0, 2141, 0, 26, 0, 4053, 0, 965, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 12, 0, 1154, 4093, 0, 441, 0, 0, 665, 57, 2, 1, 6, 4096, 2572, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 1, 971, 27, 0, 903, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 43, 2685, 1635, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 257, 0, 3, 39, 0, 0, 0, 3, 0, 0, 5, 3843, 0, 1623, 4067, 2571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 3747, 353, 1331, 117, 0, 217, 0, 0, 0, 0, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 0, 115, 4063, 0, 0, 0, 1504, 3, 3776, 0, 22, 0, 1018, 632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4015, 0, 0, 4054, 1363, 376, 0, 4080, 4092, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14070, 19179, 14192, 20025, 12441, 17899, 11142, 22124] avg=16384.0000 max=22124 max_rank=7 min=11142 min_rank=6 imbalance_ratio=1.350342 max_load_reduction=108948 max_load_reduction_pct=0.831207 moved_to_echo=63606 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1643, 7, 0, 603, 0, 362, 492, 0, 3682, 2649, 4042, 3198, 3801, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 61, 72, 0, 3197, 288, 2911, 4072, 0, 20, 0, 4078, 0, 0, 1840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 2116, 688, 0, 11, 425, 0, 0, 2795, 1119, 459, 3960, 1920, 1103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 714, 62, 0, 133, 339, 44, 3772, 0, 1, 13, 22, 0, 0, 0, 4066, 76, 2615, 26, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2835, 0, 0, 0, 0, 14, 2, 0, 0, 0, 50, 196, 30, 3527, 130, 4096, 4096, 2631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1609, 4093, 4036, 3401, 4096, 1296, 1, 0, 0, 0, 0, 4, 755, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3762, 0, 112, 0, 1465, 0, 0, 1944, 0, 0, 5, 0, 0, 1, 16, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 103, 3455, 3916, 0, 0, 39, 22, 30, 0, 393, 556, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20729, 20517, 14600, 13002, 17607, 19302, 12299, 13016] avg=16384.0000 max=20729 max_rank=0 min=12299 min_rank=6 imbalance_ratio=1.265198 max_load_reduction=110343 max_load_reduction_pct=0.841850 moved_to_echo=62224 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[5632, 27392, 9088, 4608, 256, 7424, 21888, 8192, 128, 12032, 2688, 2816, 15232, 128, 3200, 128, 5760, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[4096, 5888, 512, 256, 7168, 3456, 8832, 9344, 14208, 19072, 3072, 23168, 19840, 12416, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[4736, 128, 128, 128, 4096, 5376, 14336, 4864, 9344, 128, 6400, 19584, 896, 7936, 1280, 512, 28928, 13952, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[1408, 5120, 19968, 16384, 16384, 21632, 13184, 10624, 1152, 5504, 7808, 256, 0, 128, 5376, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 12416, 21504, 13056, 17664, 128, 0, 8832, 7552, 512, 0, 16896, 13312, 768, 384, 19072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[16640, 128, 4352, 256, 384, 5376, 15872, 13824, 2560, 128, 384, 1408, 256, 128, 24320, 7296, 21376, 9088, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[11136, 4224, 384, 8576, 6272, 12800, 1536, 12800, 18432, 4736, 256, 128, 21632, 3584, 8960, 16768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[8320, 4224, 512, 4352, 5120, 128, 512, 1920, 128, 768, 128, 4352, 896, 27520, 6272, 12160, 32384, 22272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3691, 0, 0, 4087, 1, 3725, 0, 0, 0, 3624, 0, 11, 0, 56, 0, 4066, 14, 0, 0, 0, 2656, 2158, 0, 4, 0, 10, 0, 0, 4096, 1060, 0, 2, 0, 4064, 5, 0, 8, 4096, 4093, 0, 0, 0, 0, 0, 0, 4088, 36, 1152, 2, 129, 0, 7, 23, 1490, 120, 4055, 0, 3740, 0, 0, 4096, 0, 0, 2046, 0, 0, 0, 3, 1, 0, 0, 1, 0, 4096, 0, 0, 1224, 0, 3534, 1184, 0, 3268, 0, 0, 4033, 1, 3006, 4079, 756, 0, 74, 4091, 0, 141, 4096, 0, 3917, 0, 0, 0, 0, 0, 4047, 423, 34, 1110, 3312, 4, 0, 0, 0, 0, 4, 422, 2, 4095, 202, 92, 3204, 8, 4050, 5, 3612, 28, 4095, 0, 2301, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 3965, 256, 0, 0, 0, 0, 0, 3109, 1701, 0, 3606, 143, 4, 0, 0, 0, 0, 5, 3704, 2966, 0, 1127, 39, 2, 4096, 0, 292, 4096, 0, 3248, 0, 10, 4, 29, 10, 0, 3459, 4015, 0, 0, 4092, 1172, 1885, 27, 0, 0, 0, 1033, 0, 0, 0, 0, 40, 1, 0, 0, 126, 0, 0, 4083, 0, 0, 899, 3822, 3932, 389, 4053, 14, 38, 3880, 294, 1, 4072, 0, 0, 0, 0, 0, 0, 0, 4096, 3966, 0, 0, 4096, 3942, 4072, 65, 1, 15, 4096, 3688, 1, 0, 15, 335, 5, 0, 92, 0, 0, 2978, 0, 842, 0, 0, 3932, 0, 0, 23, 6, 4061, 3654, 0, 1789, 0, 1, 0, 0, 4089, 0, 3333, 0, 0, 16, 4096, 28] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 217, 3258, 0, 0, 0, 150, 0, 3093, 4, 0, 1004, 3351, 0, 18, 0, 0, 2345, 4074, 0, 1258, 14, 88, 42, 4095, 276, 949, 740, 7, 0, 0, 1213, 4096, 0, 49, 0, 4096, 0, 0, 9, 0, 5, 4016, 3268, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 3078, 4086, 0, 19, 3976, 0, 0, 0, 3681, 0, 22, 0, 0, 0, 0, 0, 36, 4084, 544, 450, 1491, 3, 277, 0, 0, 0, 166, 0, 18, 0, 282, 1286, 0, 2, 1, 3174, 0, 4090, 0, 1, 1, 3215, 0, 2156, 0, 11, 4085, 0, 0, 104, 4096, 3983, 634, 0, 0, 72, 4, 4086, 6, 157, 4084, 0, 3815, 0, 4051, 317, 4092, 4049, 259, 2445, 480, 159] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[3808, 2927, 0, 0, 0, 31, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3413, 4094, 0, 876, 205, 1, 5, 4013, 4096, 0, 14, 625, 1, 4095, 25, 26, 0, 4095, 0, 0, 58, 4096, 0, 4096, 477, 4096, 0, 0, 1024, 0, 0, 0, 0, 0, 636, 405, 2, 4096, 18, 0, 4019, 0, 8, 0, 3993, 12, 0, 1216, 0, 0, 123, 0, 8, 758, 0, 0, 4093, 3790, 4094, 4096, 21, 4048, 0, 0, 83, 2125, 131, 2531, 0, 0, 1029, 4093, 0, 0, 2915, 0, 0, 0, 0, 2100, 0, 0, 3419, 15, 0, 0, 0, 0, 0, 70, 0, 0, 4096, 1997, 0, 4095, 4059, 8, 0, 1370, 4096, 53, 3638, 0, 3195, 49, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4076, 0, 485, 0, 0, 14, 63, 3, 0, 4093, 13, 3534, 2671, 3, 0, 3893, 141, 0, 846, 833, 1866, 4080, 0, 4031, 682, 3313, 0, 134, 0, 3, 11, 2035, 160, 0, 1, 404, 29, 0, 3707, 1, 0, 29, 0, 0, 3858, 4095, 1, 4, 0, 0, 8, 0, 1123, 0, 0, 569, 1671, 4014, 3544, 0, 4084, 0, 0, 0, 4096, 0, 83, 0, 0, 0, 22, 0, 458, 3977, 0, 0, 6, 0, 4096, 0, 0, 0, 4096, 0, 3909, 133, 537, 3562, 0, 1046, 223, 4005, 0, 62, 0, 2, 0, 0, 1061, 4, 0, 0, 1367, 0, 21, 0, 2091, 4096, 0, 0, 33, 0, 4096, 3791, 5, 3144, 3290, 0, 604, 0, 3774, 0, 4096, 0, 0, 3, 4096, 1062] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[3795, 0, 0, 0, 2, 2, 0, 0, 0, 4096, 1, 0, 0, 11, 0, 4047, 622, 0, 0, 0, 4064, 0, 1980, 0, 2, 4061, 0, 9, 1437, 0, 0, 12, 4033, 3328, 0, 4096, 0, 3830, 3, 0, 2875, 4092, 0, 3796, 94, 1, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 12, 1, 11, 0, 4096, 99, 25, 9, 3942, 467, 3254, 0, 160, 0, 4042, 50, 58, 4087, 2, 1, 0, 368, 4087, 0, 0, 4092, 226, 7, 0, 3840, 4094, 0, 0, 1320, 0, 4070, 1, 0, 24, 7, 19, 0, 109, 2140, 0, 0, 3849, 0, 528, 0, 4096, 4094, 891, 0, 0, 0, 1, 0, 0, 2702, 0, 0, 2941, 113, 4095, 0, 4096, 149, 850, 3535, 4095, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[3965, 1, 25, 636, 0, 0, 4, 4053, 0, 452, 1, 223, 59, 0, 0, 4096, 0, 65, 3012, 12, 819, 0, 35, 4048, 0, 3616, 0, 3425, 4094, 8, 882, 235, 1027, 0, 0, 2, 0, 4066, 0, 0, 0, 0, 8, 0, 113, 4066, 3, 0, 6, 3876, 0, 3, 0, 0, 0, 0, 854, 4096, 0, 39, 4096, 0, 187, 0, 4087, 0, 11, 0, 0, 0, 0, 1, 0, 3416, 0, 46, 625, 0, 3491, 1113, 0, 5, 3511, 14, 27, 129, 4058, 4075, 0, 45, 0, 4096, 0, 4090, 262, 0, 744, 0, 776, 4086, 0, 0, 4096, 0, 0, 0, 4080, 1, 0, 324, 2, 3, 1322, 2935, 34, 4046, 0, 0, 4093, 0, 3477, 89, 919, 3934, 4096, 0, 2705, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4090, 2, 4079, 4096, 3, 0, 0, 159, 0, 69, 0, 0, 40, 3, 275, 80, 0, 24, 0, 0, 4028, 0, 103, 683, 1824, 2203, 0, 0, 3792, 0, 15, 169, 4056, 304, 300, 0, 82, 4073, 2, 0, 0, 0, 0, 3936, 0, 4096, 5, 0, 1, 4010, 0, 0, 0, 0, 0, 3805, 0, 16, 0, 0, 2697, 0, 0, 0, 4094, 6, 0, 102, 213, 0, 1, 0, 0, 3978, 66, 92, 1848, 4, 3723, 4061, 0, 3832, 0, 0, 348, 6, 4096, 1, 2183, 0, 0, 4062, 1, 0, 0, 0, 1392, 0, 0, 4092, 0, 0, 3901, 18, 0, 0, 3911, 0, 1351, 0, 98, 5, 3378, 3951, 0, 3833, 6, 1, 2589, 182, 4003, 350, 4006, 4093, 0, 0, 4075, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3965, 256, 0, 0, 0, 0, 0, 3109, 1701, 0, 3606, 143, 4, 0, 0, 1772, 942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3704, 2966, 0, 1127, 39, 2, 4096, 0, 292, 4096, 0, 3248, 0, 512, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4, 29, 10, 0, 2507, 4015, 0, 0, 4092, 1172, 1885, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 0, 0, 0, 0, 40, 1, 0, 0, 126, 0, 0, 4083, 0, 0, 899, 4089, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3822, 3932, 389, 4053, 14, 38, 3880, 294, 1, 4072, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3966, 0, 0, 4096, 3942, 4072, 65, 1, 15, 1949, 3688, 1, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 5, 0, 92, 0, 0, 2978, 0, 842, 0, 0, 3932, 0, 0, 23, 6, 2234, 2232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 3654, 0, 1789, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 922, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15498, 20462, 13751, 11370, 20935, 25906, 12679, 10471] avg=16384.0000 max=25906 max_rank=5 min=10471 min_rank=7 imbalance_ratio=1.581177 max_load_reduction=105166 max_load_reduction_pct=0.802353 moved_to_echo=69991 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 217, 3258, 0, 0, 0, 150, 0, 3093, 4, 0, 1004, 3351, 0, 18, 1373, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2345, 4074, 0, 1258, 14, 88, 42, 4095, 276, 949, 740, 7, 0, 0, 606, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1213, 4096, 0, 49, 0, 2969, 0, 0, 9, 0, 5, 4016, 3268, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 3078, 4086, 0, 19, 3976, 0, 0, 0, 4051, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3681, 0, 22, 0, 0, 0, 0, 0, 36, 4084, 544, 450, 1491, 3, 277, 0, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 18, 0, 282, 1286, 0, 2, 1, 1510, 0, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3215, 0, 2156, 0, 11, 4085, 0, 0, 104, 4096, 3983, 634, 0, 0, 72, 2744, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4086, 6, 157, 4084, 0, 3815, 0, 0, 317, 0, 4049, 259, 2445, 109, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16673, 14785, 19694, 20603, 11109, 7356, 21362, 19490] avg=16384.0000 max=21362 max_rank=6 min=7356 min_rank=5 imbalance_ratio=1.303833 max_load_reduction=109710 max_load_reduction_pct=0.837021 moved_to_echo=59317 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3691, 0, 0, 4087, 1, 3725, 0, 0, 0, 3624, 0, 11, 0, 56, 0, 4066, 1770, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 2656, 2158, 0, 4, 0, 10, 0, 0, 4096, 1060, 0, 2, 606, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 5, 0, 8, 2969, 4093, 0, 0, 0, 0, 0, 0, 4088, 36, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 129, 0, 7, 23, 1490, 120, 4055, 0, 3740, 0, 0, 4096, 0, 0, 2046, 4050, 1191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 1, 0, 4096, 0, 0, 1224, 0, 3534, 1184, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3268, 0, 0, 4033, 1, 3006, 4079, 756, 0, 74, 1946, 0, 141, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3917, 0, 0, 0, 0, 0, 4047, 423, 34, 1110, 3312, 4, 0, 0, 0, 0, 2421, 1254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 422, 2, 4095, 202, 92, 3204, 8, 0, 5, 0, 28, 4095, 0, 518, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21560, 10981, 16415, 20949, 10564, 21400, 16522, 12681] avg=16384.0000 max=21560 max_rank=0 min=10564 min_rank=4 imbalance_ratio=1.315918 max_load_reduction=109512 max_load_reduction_pct=0.835510 moved_to_echo=61167 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3795, 0, 0, 0, 2, 2, 0, 0, 0, 4096, 1, 0, 0, 11, 0, 4047, 1761, 942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 622, 0, 0, 0, 4064, 0, 1980, 0, 2, 4061, 0, 9, 1437, 0, 0, 12, 567, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 3328, 0, 4096, 0, 2776, 3, 0, 2875, 4092, 0, 3796, 94, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 12, 1, 11, 0, 4096, 99, 25, 9, 4095, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3942, 467, 3254, 0, 160, 0, 4042, 50, 58, 4087, 2, 1, 0, 368, 4087, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 226, 7, 0, 3840, 4094, 0, 0, 1320, 0, 1936, 1, 0, 24, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 109, 2140, 0, 0, 3849, 0, 528, 0, 4096, 4094, 891, 0, 0, 0, 2747, 2231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2702, 0, 0, 2941, 113, 0, 0, 0, 149, 850, 3535, 922, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14657, 13127, 25094, 9721, 21005, 15547, 20704, 11217] avg=16384.0000 max=25094 max_rank=2 min=9721 min_rank=3 imbalance_ratio=1.531616 max_load_reduction=105978 max_load_reduction_pct=0.808548 moved_to_echo=68473 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4076, 0, 485, 0, 0, 14, 63, 3, 0, 4093, 13, 3534, 2671, 3, 0, 3893, 1733, 942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 846, 833, 1866, 4080, 0, 4031, 682, 3313, 0, 134, 0, 3, 11, 2035, 0, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 1, 404, 29, 0, 3707, 1, 0, 29, 0, 0, 3858, 4095, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1123, 0, 0, 569, 1671, 4014, 3544, 0, 4084, 0, 0, 0, 3774, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 83, 0, 0, 0, 22, 0, 458, 3977, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3909, 133, 537, 3562, 0, 1046, 223, 1905, 0, 62, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 4, 0, 0, 1367, 0, 21, 0, 2091, 4096, 0, 0, 33, 0, 2746, 2232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3791, 5, 3144, 3290, 0, 604, 0, 0, 0, 0, 0, 0, 3, 922, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21523, 18342, 12289, 20137, 12738, 15475, 13651, 16917] avg=16384.0000 max=21523 max_rank=0 min=12289 min_rank=2 imbalance_ratio=1.313660 max_load_reduction=109549 max_load_reduction_pct=0.835793 moved_to_echo=58781 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3808, 2927, 0, 0, 0, 31, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 1774, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3413, 4094, 0, 876, 205, 1, 5, 4013, 4096, 0, 14, 625, 610, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 25, 26, 0, 2960, 0, 0, 58, 4096, 0, 4096, 477, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 0, 0, 0, 0, 0, 636, 405, 2, 4096, 18, 0, 4019, 0, 8, 0, 4096, 1199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 12, 0, 1216, 0, 0, 123, 0, 8, 758, 0, 0, 4093, 3790, 4094, 4096, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4048, 0, 0, 83, 2125, 131, 2531, 0, 0, 1029, 1940, 0, 0, 2915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2100, 0, 0, 3419, 15, 0, 0, 0, 0, 0, 70, 0, 0, 2439, 2235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1997, 0, 4095, 4059, 8, 0, 1370, 0, 53, 0, 0, 3195, 49, 915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9662, 18331, 19930, 15503, 22708, 14823, 10278, 19837] avg=16384.0000 max=22708 max_rank=4 min=9662 min_rank=0 imbalance_ratio=1.385986 max_load_reduction=108364 max_load_reduction_pct=0.826752 moved_to_echo=67646 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3965, 1, 25, 636, 0, 0, 4, 4053, 0, 452, 1, 223, 59, 0, 0, 4096, 1772, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 3012, 12, 819, 0, 35, 4048, 0, 3616, 0, 3425, 4094, 8, 882, 235, 602, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 0, 0, 2, 0, 2947, 0, 0, 0, 0, 8, 0, 113, 4066, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3876, 0, 3, 0, 0, 0, 0, 854, 4096, 0, 39, 4096, 0, 187, 0, 3477, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 11, 0, 0, 0, 0, 1, 0, 3416, 0, 46, 625, 0, 3491, 1113, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3511, 14, 27, 129, 4058, 4075, 0, 45, 0, 1949, 0, 4090, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 744, 0, 776, 4086, 0, 0, 4096, 0, 0, 0, 4080, 1, 0, 324, 2, 3, 616, 1474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1322, 2935, 34, 4046, 0, 0, 4093, 0, 0, 89, 0, 3934, 4096, 0, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15909, 21228, 8166, 16937, 13307, 18165, 16202, 21158] avg=16384.0000 max=21228 max_rank=1 min=8166 min_rank=2 imbalance_ratio=1.295654 max_load_reduction=109844 max_load_reduction_pct=0.838043 moved_to_echo=68832 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4090, 2, 4079, 4096, 3, 0, 0, 159, 0, 69, 0, 0, 40, 3, 275, 80, 1757, 937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4028, 0, 103, 683, 1824, 2203, 0, 0, 3792, 0, 15, 169, 603, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 304, 300, 0, 82, 2952, 2, 0, 0, 0, 0, 3936, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4010, 0, 0, 0, 0, 0, 3805, 0, 16, 0, 0, 2697, 0, 0, 0, 4003, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 102, 213, 0, 1, 0, 0, 3978, 66, 92, 1848, 4, 3723, 4061, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3832, 0, 0, 348, 6, 4096, 1, 2183, 0, 0, 1933, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1392, 0, 0, 4092, 0, 0, 3901, 18, 0, 0, 3911, 0, 1351, 0, 98, 5, 2686, 2220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3378, 3951, 0, 3833, 6, 1, 2589, 182, 0, 350, 0, 4093, 0, 0, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15590, 13816, 15733, 15852, 18706, 12400, 19674, 19301] avg=16384.0000 max=19674 max_rank=6 min=12400 min_rank=5 imbalance_ratio=1.200806 max_load_reduction=111398 max_load_reduction_pct=0.849899 moved_to_echo=70081 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[6528, 3328, 2048, 14720, 0, 128, 27776, 512, 1536, 1280, 21632, 16128, 2944, 512, 256, 128, 18688, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[17024, 20864, 128, 23936, 11648, 128, 17280, 1792, 0, 896, 0, 12288, 12544, 6144, 5888, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[10624, 16000, 384, 4608, 128, 20096, 11904, 128, 2944, 12416, 1280, 17792, 7936, 24576, 128, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[896, 128, 6272, 8704, 19840, 11648, 3328, 9856, 2816, 21504, 384, 8832, 22400, 1152, 4224, 3200, 4224, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[27520, 6912, 5120, 12160, 128, 3840, 128, 4480, 3200, 17408, 128, 7424, 3968, 3456, 384, 16256, 13824, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 19456, 12032, 128, 8448, 10368, 20224, 19712, 3072, 2432, 1408, 15104, 3712, 8448, 7424, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[27776, 4480, 3840, 5376, 512, 128, 8192, 384, 640, 28544, 640, 640, 9344, 4224, 23424, 10496, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2176, 8064, 128, 128, 1152, 1536, 768, 12928, 5632, 20224, 3584, 128, 31232, 128, 256, 3072, 31744, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[11, 4082, 3824, 21, 218, 164, 0, 0, 0, 3613, 186, 4059, 0, 0, 738, 0, 0, 0, 0, 552, 0, 1195, 4096, 0, 0, 4057, 0, 26, 62, 245, 4066, 0, 10, 0, 0, 4092, 701, 0, 0, 17, 2871, 0, 400, 0, 77, 4074, 3878, 35, 0, 1, 855, 12, 3597, 0, 2, 575, 0, 4092, 1, 171, 0, 4072, 1685, 0, 3195, 0, 0, 4093, 0, 0, 0, 0, 4073, 3, 0, 19, 7, 0, 35, 0, 3002, 3799, 0, 8, 2, 3549, 1, 30, 185, 0, 0, 0, 4096, 3125, 1, 0, 0, 22, 0, 0, 3688, 20, 4043, 4078, 1, 445, 7, 2685, 4013, 32, 12, 67, 0, 0, 0, 0, 0, 4050, 3969, 33, 4023, 4090, 42, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[49, 0, 2810, 105, 46, 0, 37, 0, 4019, 0, 23, 3937, 45, 0, 3, 0, 68, 0, 17, 0, 0, 0, 3965, 1, 0, 3898, 0, 3711, 0, 0, 0, 56, 0, 0, 18, 2574, 73, 862, 127, 583, 707, 597, 1, 110, 0, 4064, 6, 56, 0, 0, 4072, 0, 4, 0, 0, 4038, 0, 4052, 0, 0, 297, 4095, 3964, 327, 3919, 1, 3982, 4094, 0, 3782, 1982, 0, 3455, 4096, 4089, 4041, 125, 4086, 0, 0, 1, 4043, 0, 3391, 2168, 4088, 230, 105, 197, 70, 0, 0, 0, 0, 0, 0, 28, 0, 16, 0, 0, 2513, 1780, 367, 16, 737, 4095, 0, 0, 8, 1131, 144, 0, 0, 5, 4096, 0, 2761, 0, 4, 3895, 0, 22, 4092, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 3810, 2, 3991, 18, 29, 0, 0, 2765, 2, 0, 3879, 1, 2949, 1, 0, 0, 15, 0, 0, 0, 0, 4086, 0, 0, 2654, 2408, 2500, 30, 31, 332, 15, 0, 4025, 0, 3396, 0, 0, 47, 3575, 0, 0, 0, 53, 4075, 286, 0, 0, 1917, 192, 3991, 0, 1760, 5, 264, 0, 859, 4073, 0, 0, 109, 2054, 4093, 0, 9, 4064, 3968, 0, 891, 0, 2, 291, 3906, 0, 4096, 0, 0, 0, 13, 1, 0, 4063, 1, 0, 12, 3742, 92, 353, 1177, 0, 3491, 1612, 4092, 0, 4, 0, 0, 24, 367, 4094, 4095, 3734, 4084, 3791, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3132, 0, 3, 10, 0, 9, 2, 1892, 3259, 81, 63, 2253, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 11, 77, 56, 3, 0, 187, 267, 278, 0, 0, 1974, 0, 4084, 0, 0, 22, 0, 0, 0, 0, 1114, 0, 0, 0, 0, 1, 1, 0, 20, 244, 0, 908, 4052, 0, 3765, 57, 3785, 0, 0, 0, 0, 0, 433, 0, 4, 0, 2, 0, 0, 4096, 0, 0, 7, 0, 420, 3695, 3899, 0, 34, 12, 2536, 4087, 0, 4020, 0, 6, 1, 4096, 2890, 0, 3796, 3358, 6, 0, 0, 132, 0, 4086, 0, 4096, 4020, 0, 1008, 36, 3671, 69, 3122, 4096, 1, 2207, 3666, 3996, 2, 0, 0, 0, 3437, 0, 0, 4013, 4096, 4095, 3964, 0, 0, 4096, 0, 241, 0, 0, 0, 0, 1745, 0, 0, 7, 454, 4023, 0, 3591, 0, 8, 70, 4049, 668, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 3332, 431, 3721, 0, 0, 0, 5, 2538, 0, 0, 4006, 0, 0, 1, 0, 0, 0, 0, 0, 3196, 0, 0, 0, 0, 1487, 12, 0, 0, 0, 4096, 0, 0, 4096, 0, 3769, 2122, 1072, 32, 4, 539, 92, 1, 0, 1384, 2, 0, 0, 4082, 0, 3977, 22, 809, 0, 4073, 1, 119, 4096, 0, 14, 2, 0, 4096, 0, 2852, 4091, 0, 3301, 209, 43, 0, 0, 4066, 0, 3948, 4096, 1, 1179, 0, 0, 3695, 106, 0, 0, 0, 429, 0, 0, 2424, 0, 1881, 4096, 2274, 68, 2, 29, 0, 0, 1937, 3928, 0, 3763, 4096, 4007, 18, 0, 1263, 0, 32, 0, 0, 2769, 23, 5, 0, 10, 0, 3978, 4096, 0, 4096, 2, 1020, 1, 0, 9, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[14, 0, 4, 14, 0, 547, 0, 2334, 4034, 3539, 0, 137, 3723, 3596, 3682, 0, 72, 0, 0, 0, 80, 0, 4021, 0, 0, 0, 116, 3906, 0, 58, 2084, 74, 0, 2, 6, 4094, 4002, 12, 0, 4096, 0, 0, 0, 0, 1482, 1596, 5, 0, 77, 455, 4096, 0, 716, 0, 60, 0, 312, 3993, 3, 0, 145, 3073, 2558, 0, 1134, 3113, 3766, 11, 2833, 0, 1044, 0, 4017, 29, 4031, 0, 207, 0, 0, 56, 3857, 0, 0, 36, 399, 0, 705, 2879, 262, 0, 0, 205, 3980, 0, 1, 0, 17, 0, 0, 3422, 4074, 0, 1462, 2963, 0, 1326, 4091, 0, 0, 21, 2, 0, 0, 3, 0, 4082, 0, 3586, 0, 0, 4031, 0, 2533, 4076, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 3890, 0, 0, 4096, 0, 1, 4087, 3, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 4, 0, 0, 834, 4096, 3115, 0, 156, 3911, 285, 55, 0, 0, 0, 4061, 0, 0, 488, 41, 4094, 0, 0, 4074, 3619, 2, 3972, 4069, 0, 3456, 664, 532, 0, 4096, 4096, 0, 4088, 0, 0, 0, 2022, 4060, 0, 3265, 0, 0, 29, 4044, 1835, 20, 0, 0, 1, 345, 0, 371, 4068, 87, 0, 4096, 0, 0, 0, 6, 31, 0, 0, 0, 2, 0, 98, 4072, 14, 0, 0, 2, 3, 3, 4093, 0, 4091, 4094, 3787, 0, 0, 65, 0, 0, 0, 0, 0, 2052, 63, 0, 33, 0, 4045, 1127, 0, 3942, 1, 0, 2922, 12, 2117, 0, 126] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1783, 446, 0, 4058, 0, 0, 1, 76, 1, 124, 0, 3869, 0, 4096, 0, 0, 0, 15, 0, 2628, 0, 0, 2547, 1, 0, 4092, 34, 0, 0, 0, 2323, 0, 2, 0, 60, 4085, 37, 33, 27, 0, 4096, 4, 23, 13, 589, 0, 2, 0, 876, 1004, 0, 0, 45, 0, 0, 0, 0, 3898, 0, 65, 0, 4035, 4089, 0, 3, 0, 4005, 0, 0, 1637, 3623, 2, 3573, 171, 3983, 0, 244, 4096, 8, 0, 4054, 4086, 0, 197, 3995, 44, 0, 3987, 2, 0, 0, 791, 0, 0, 15, 3, 267, 4096, 561, 3762, 4096, 4092, 3428, 4096, 0, 18, 0, 0, 0, 18, 0, 0, 0, 0, 279, 0, 0, 4055, 105, 0, 4096, 0, 3839, 0, 3740, 1, 2923, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[11, 4082, 3824, 21, 218, 164, 0, 0, 0, 3613, 186, 4059, 0, 0, 738, 0, 0, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 0, 1195, 4096, 0, 0, 4057, 0, 26, 62, 245, 4066, 0, 4073, 3261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4092, 701, 0, 0, 17, 2871, 0, 400, 0, 77, 4074, 3878, 35, 2283, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 855, 12, 3597, 0, 2, 575, 0, 1447, 1, 171, 0, 4072, 1685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3195, 0, 0, 4093, 0, 0, 0, 0, 0, 3, 0, 19, 7, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2677, 3799, 0, 8, 2, 3549, 1, 30, 185, 0, 0, 0, 4096, 3125, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 3688, 20, 644, 4078, 1, 445, 7, 2685, 4013, 32, 12, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 3969, 33, 4023, 4090, 42, 0, 0, 0, 0, 4096, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17278, 21633, 18576, 12418, 7352, 17473, 15714, 20628] avg=16384.0000 max=21633 max_rank=1 min=7352 min_rank=4 imbalance_ratio=1.320374 max_load_reduction=109439 max_load_reduction_pct=0.834953 moved_to_echo=61167 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3810, 2, 3991, 18, 29, 0, 0, 2765, 2, 0, 3879, 1, 2949, 1, 0, 1910, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 4086, 0, 0, 2654, 2408, 2500, 30, 31, 332, 15, 3906, 3294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 3396, 0, 0, 47, 3575, 0, 0, 0, 53, 4075, 286, 0, 0, 2272, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1917, 192, 3991, 0, 1760, 5, 264, 0, 859, 1440, 0, 0, 109, 2054, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4064, 3968, 0, 891, 0, 2, 291, 0, 0, 1973, 0, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 1, 0, 12, 3742, 92, 353, 1177, 0, 3491, 1612, 4092, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 367, 4094, 4095, 3734, 651, 3791, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3132, 0, 3, 10, 0, 9, 2, 1892, 3259, 81, 63, 2253, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19718, 19271, 17868, 16684, 11212, 18639, 16763, 10917] avg=16384.0000 max=19718 max_rank=0 min=10917 min_rank=7 imbalance_ratio=1.203491 max_load_reduction=111354 max_load_reduction_pct=0.849564 moved_to_echo=57531 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[49, 0, 2810, 105, 46, 0, 37, 0, 4019, 0, 23, 3937, 45, 0, 3, 0, 1906, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 17, 0, 0, 0, 3965, 1, 0, 3898, 0, 3711, 0, 0, 0, 56, 3455, 1436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2574, 73, 862, 127, 583, 707, 597, 1, 110, 0, 4064, 6, 56, 2261, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 4, 0, 0, 4038, 0, 1432, 0, 0, 297, 4095, 3964, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3919, 1, 3982, 4094, 0, 3782, 1982, 0, 0, 4096, 1970, 4041, 125, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4043, 0, 3391, 2168, 4088, 230, 105, 197, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 16, 0, 0, 2513, 284, 367, 16, 737, 4095, 0, 0, 8, 1131, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 2761, 0, 4, 3895, 0, 22, 4092, 0, 0, 0, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13339, 16607, 12099, 18229, 32078, 14293, 9339, 15088] avg=16384.0000 max=32078 max_rank=4 min=9339 min_rank=6 imbalance_ratio=1.957886 max_load_reduction=98994 max_load_reduction_pct=0.755264 moved_to_echo=70798 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 4, 14, 0, 547, 0, 2334, 4034, 3539, 0, 137, 3723, 3596, 3682, 0, 1879, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 80, 0, 4021, 0, 0, 0, 116, 3906, 0, 58, 2084, 74, 4017, 1179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 4094, 4002, 12, 0, 4096, 0, 0, 0, 0, 1482, 1596, 5, 0, 2228, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 455, 4096, 0, 716, 0, 60, 0, 312, 1411, 3, 0, 145, 3073, 2558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1134, 3113, 3766, 11, 2833, 0, 1044, 0, 0, 29, 1942, 0, 207, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3440, 0, 0, 36, 399, 0, 705, 2879, 262, 0, 0, 205, 3980, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 3422, 4074, 0, 233, 2963, 0, 1326, 4091, 0, 0, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4082, 0, 3586, 0, 0, 4031, 0, 2533, 4076, 0, 0, 0, 0, 417, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23857, 15607, 17573, 12906, 14135, 11907, 16149, 18938] avg=16384.0000 max=23857 max_rank=0 min=11907 min_rank=5 imbalance_ratio=1.456116 max_load_reduction=107215 max_load_reduction_pct=0.817986 moved_to_echo=61129 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3332, 431, 3721, 0, 0, 0, 5, 2538, 0, 0, 4006, 0, 0, 1, 0, 1841, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3196, 0, 0, 0, 0, 1487, 12, 0, 0, 0, 4096, 0, 4066, 3304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3769, 2122, 1072, 32, 4, 539, 92, 1, 0, 1384, 2, 0, 0, 2285, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 3977, 22, 809, 0, 4073, 1, 119, 1448, 0, 14, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2852, 4091, 0, 3301, 209, 43, 0, 0, 0, 0, 1901, 4096, 1, 1179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3295, 106, 0, 0, 0, 429, 0, 0, 2424, 0, 1881, 4096, 2274, 68, 2, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1937, 3928, 0, 3763, 652, 4007, 18, 0, 1263, 0, 32, 0, 0, 2769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 5, 0, 10, 0, 3978, 4096, 0, 4096, 2, 1020, 1, 0, 9, 0, 0, 400, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16238, 16161, 15538, 18643, 17673, 14604, 18369, 13846] avg=16384.0000 max=18643 max_rank=3 min=13846 min_rank=7 imbalance_ratio=1.137878 max_load_reduction=112429 max_load_reduction_pct=0.857765 moved_to_echo=64492 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 77, 56, 3, 0, 187, 267, 278, 0, 0, 1974, 0, 4084, 0, 0, 0, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 1114, 0, 0, 0, 0, 1, 1, 0, 20, 244, 0, 3358, 3303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 908, 4052, 0, 3765, 57, 3785, 0, 0, 0, 0, 0, 433, 0, 4, 0, 2, 2175, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 420, 3695, 1379, 0, 34, 12, 2536, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 6, 1, 4096, 2890, 0, 3796, 0, 6, 0, 0, 132, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3653, 4020, 0, 1008, 36, 3671, 69, 3122, 4096, 1, 2207, 3666, 3996, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3437, 0, 0, 4013, 4096, 652, 3964, 0, 0, 4096, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1745, 0, 0, 7, 454, 4023, 0, 3591, 0, 8, 70, 4049, 668, 3, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7282, 8063, 15321, 16266, 19033, 29547, 20499, 15061] avg=16384.0000 max=29547 max_rank=5 min=7282 min_rank=0 imbalance_ratio=1.803406 max_load_reduction=101525 max_load_reduction_pct=0.774574 moved_to_echo=84140 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3890, 0, 0, 4096, 0, 1, 4087, 3, 0, 3, 4096, 0, 0, 0, 0, 163, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 4, 0, 0, 834, 4096, 3115, 0, 156, 3911, 285, 0, 3307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 4061, 0, 0, 488, 41, 4094, 0, 0, 4074, 3619, 2, 3972, 2284, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 3456, 664, 532, 0, 4096, 4096, 0, 1439, 0, 0, 0, 2022, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3265, 0, 0, 29, 4044, 1835, 20, 0, 0, 1, 161, 0, 371, 4068, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3650, 0, 0, 0, 6, 31, 0, 0, 0, 2, 0, 98, 4072, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 3, 4093, 0, 4091, 643, 3787, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2052, 63, 0, 33, 0, 4045, 1127, 0, 3942, 1, 0, 2922, 12, 2117, 0, 126, 446, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16704, 15752, 22834, 24434, 13881, 7873, 12687, 16907] avg=16384.0000 max=24434 max_rank=3 min=7873 min_rank=5 imbalance_ratio=1.491333 max_load_reduction=106638 max_load_reduction_pct=0.813583 moved_to_echo=51348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1783, 446, 0, 4058, 0, 0, 1, 76, 1, 124, 0, 3869, 0, 4096, 0, 0, 1857, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 2628, 0, 0, 2547, 1, 0, 4092, 34, 0, 0, 0, 2323, 0, 3573, 2765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 60, 4085, 37, 33, 27, 0, 4096, 4, 23, 13, 589, 0, 2, 0, 2175, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 876, 1004, 0, 0, 45, 0, 0, 0, 0, 1378, 0, 65, 0, 4035, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4005, 0, 0, 1637, 3623, 2, 0, 171, 1919, 0, 244, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3616, 4086, 0, 197, 3995, 44, 0, 3987, 2, 0, 0, 791, 0, 0, 15, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 4096, 561, 3762, 4096, 4092, 546, 4096, 0, 18, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 0, 0, 4055, 105, 0, 4096, 0, 3839, 0, 3740, 1, 2923, 4, 438, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16656, 17978, 11263, 11492, 15708, 16736, 21552, 19687] avg=16384.0000 max=21552 max_rank=6 min=11263 min_rank=2 imbalance_ratio=1.315430 max_load_reduction=109520 max_load_reduction_pct=0.835571 moved_to_echo=73683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 128, 128, 3200, 3328, 2432, 18816, 128, 0, 17024, 6784, 13312, 128, 512, 17152, 512, 26496, 21888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1024, 12288, 128, 25856, 11136, 5888, 256, 8832, 8320, 4864, 512, 640, 11776, 13696, 3968, 4096, 18048, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[20352, 20224, 128, 4736, 6656, 15616, 1152, 10496, 8448, 128, 7680, 10496, 22528, 3328, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1920, 15616, 7168, 12032, 4480, 768, 256, 6784, 13696, 7296, 256, 25984, 3840, 14848, 4480, 0, 9600, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[384, 7680, 2944, 19328, 19968, 22400, 4352, 27136, 128, 2560, 13696, 2688, 4352, 128, 1152, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[18432, 11392, 15744, 11648, 12160, 10240, 6784, 4096, 0, 4352, 9984, 8192, 1152, 13440, 4352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[11136, 1664, 24576, 768, 7552, 128, 8576, 9216, 4992, 11392, 128, 384, 640, 21888, 28672, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[2176, 1920, 384, 11392, 128, 23040, 13440, 128, 27776, 4096, 9472, 14464, 7936, 2944, 5248, 4352, 2560, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4002, 0, 0, 6, 0, 0, 1161, 4089, 0, 0, 3072, 153, 4, 0, 0, 0, 4096, 241, 0, 8, 0, 6, 0, 0, 55, 4095, 57, 0, 0, 0, 0, 328, 4, 0, 29, 4096, 668, 4060, 0, 499, 1464, 0, 4096, 4094, 0, 8, 0, 0, 0, 0, 892, 0, 0, 4096, 19, 3821, 1, 532, 0, 65, 0, 201, 0, 226, 0, 3516, 3420, 1, 0, 0, 0, 4091, 0, 0, 663, 3974, 2300, 0, 0, 0, 35, 4096, 0, 123, 4092, 0, 3803, 4081, 4, 5, 35, 392, 0, 0, 4096, 4053, 132, 2634, 0, 4096, 884, 0, 3295, 0, 0, 3862, 0, 0, 3682, 0, 3754, 0, 0, 0, 0, 4095, 0, 0, 0, 3352, 4029, 18, 53, 0, 0, 64, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[38, 0, 4096, 914, 0, 889, 1298, 872, 3811, 31, 1505, 4026, 43, 0, 1, 1821, 4081, 0, 0, 54, 0, 17, 581, 0, 9, 1239, 0, 11, 40, 1107, 38, 199, 0, 3, 0, 3411, 59, 0, 3825, 0, 0, 0, 4096, 3404, 10, 94, 0, 0, 0, 0, 4094, 3715, 2905, 3753, 0, 1093, 3770, 0, 0, 7, 0, 350, 0, 4039, 0, 3964, 0, 0, 3, 0, 483, 4092, 5, 0, 938, 13, 4096, 31, 1, 300, 0, 2345, 0, 5, 0, 11, 4076, 131, 0, 1372, 107, 2097, 0, 0, 201, 1522, 226, 0, 0, 4096, 0, 0, 44, 0, 0, 3322, 0, 0, 3538, 0, 3291, 713, 27, 2981, 3793, 3937, 0, 0, 3906, 254, 4069, 0, 0, 651, 687, 294, 5, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[876, 65, 2789, 4094, 567, 3111, 0, 39, 0, 0, 4096, 5, 200, 0, 498, 0, 4029, 0, 0, 3880, 182, 291, 357, 66, 38, 8, 0, 0, 3678, 0, 0, 1785, 0, 1549, 0, 3454, 139, 0, 389, 86, 4096, 3434, 0, 0, 25, 0, 0, 5, 0, 2, 81, 0, 19, 184, 3592, 18, 3334, 2763, 0, 1, 528, 0, 4051, 4023, 6, 4096, 3883, 0, 0, 15, 0, 3291, 138, 0, 0, 75, 4068, 2359, 11, 0, 1092, 3838, 85, 2, 0, 3, 2751, 4081, 3643, 1, 80, 8, 30, 1259, 0, 0, 0, 2, 0, 1327, 0, 1, 4068, 0, 0, 4059, 0, 0, 4078, 0, 3863, 774, 1, 0, 3194, 0, 4095, 0, 230, 4040, 0, 0, 2, 23, 0, 4010, 1, 62] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[3605, 10, 4084, 4, 4006, 3036, 0, 18, 0, 0, 4074, 4044, 0, 0, 5, 0, 7, 48, 20, 4091, 0, 0, 0, 191, 3983, 0, 3942, 0, 703, 0, 0, 15, 171, 0, 0, 3285, 2, 1, 0, 28, 116, 26, 471, 4031, 292, 6, 9, 3638, 4091, 0, 3892, 1545, 91, 3795, 17, 1, 3937, 4096, 1409, 0, 0, 0, 0, 4096, 1, 4027, 590, 0, 0, 0, 3927, 4096, 2975, 0, 4044, 0, 0, 0, 58, 0, 277, 3950, 0, 1, 0, 22, 1139, 4088, 0, 0, 67, 14, 22, 196, 0, 0, 0, 0, 4, 1642, 0, 517, 0, 881, 113, 8, 0, 0, 4080, 24, 4096, 0, 0, 4092, 2949, 3405, 0, 0, 0, 0, 6, 5, 210, 7, 4060, 543, 1, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[320, 0, 4036, 3966, 54, 2968, 3995, 0, 4096, 0, 2752, 174, 0, 0, 0, 0, 2866, 0, 345, 831, 0, 418, 0, 0, 205, 4091, 3501, 0, 0, 81, 0, 0, 0, 4, 1, 4095, 0, 4014, 200, 0, 3737, 775, 4003, 4096, 4, 147, 10, 982, 4057, 0, 9, 3296, 0, 4092, 1339, 227, 347, 1438, 4003, 0, 0, 3, 7, 4079, 0, 9, 0, 0, 0, 0, 0, 238, 92, 313, 1871, 61, 4073, 0, 2, 4089, 91, 3983, 0, 104, 11, 778, 0, 0, 0, 0, 0, 246, 0, 0, 737, 1, 0, 0, 0, 4054, 0, 0, 3780, 0, 7, 3943, 0, 0, 3848, 0, 0, 582, 2, 3477, 4089, 2, 0, 0, 0, 3464, 1552, 0, 0, 0, 749, 3046, 0, 2114] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[3952, 5, 1387, 0, 0, 908, 0, 2, 0, 0, 4056, 0, 0, 3827, 312, 99, 3642, 10, 0, 82, 0, 0, 0, 0, 0, 0, 1744, 0, 28, 0, 4, 382, 4096, 0, 1437, 0, 0, 0, 0, 2, 0, 0, 4081, 2592, 2, 1, 0, 0, 0, 4085, 148, 3744, 0, 2304, 3940, 0, 0, 3104, 55, 0, 0, 5, 4, 3246, 0, 4096, 48, 1894, 0, 0, 46, 4040, 0, 0, 4085, 0, 3, 1, 31, 4096, 0, 0, 3, 4094, 2752, 558, 1437, 105, 0, 0, 145, 31, 4096, 0, 0, 1170, 0, 3086, 0, 790, 12, 3, 3568, 0, 4095, 1201, 15, 0, 4066, 3578, 229, 6, 0, 3869, 4034, 4089, 0, 42, 0, 4094, 3973, 6, 142, 0, 0, 91, 37, 4029] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[3592, 0, 0, 2004, 0, 0, 59, 0, 3, 14, 4096, 144, 0, 0, 0, 71, 0, 0, 0, 48, 7, 0, 4090, 0, 0, 0, 0, 0, 0, 42, 435, 933, 3774, 0, 17, 4047, 1, 0, 1757, 0, 0, 0, 3980, 0, 2, 0, 1984, 36, 514, 658, 4096, 2350, 9, 4077, 21, 881, 0, 4062, 3240, 3530, 0, 0, 0, 4096, 0, 0, 39, 0, 0, 0, 0, 3969, 116, 0, 1831, 36, 4016, 0, 5, 848, 0, 4077, 1001, 1310, 0, 1757, 2994, 0, 47, 728, 2, 16, 3714, 87, 0, 3602, 106, 0, 0, 2678, 0, 12, 3199, 0, 281, 0, 0, 0, 4094, 0, 3238, 4096, 1, 4025, 3940, 51, 12, 65, 166, 4096, 4079, 0, 0, 0, 4091, 3977, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[2246, 0, 4014, 0, 7, 20, 1, 0, 0, 789, 3860, 3260, 0, 0, 0, 0, 4090, 0, 0, 140, 0, 0, 0, 54, 0, 1522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 1, 4072, 3708, 4096, 3421, 4010, 4093, 1, 718, 518, 4094, 0, 0, 4096, 0, 4096, 4048, 0, 2, 4050, 4096, 5, 1, 14, 0, 2, 1045, 4015, 7, 0, 70, 0, 79, 3, 4077, 0, 3432, 158, 0, 3144, 0, 0, 0, 739, 0, 4075, 65, 3652, 1150, 1582, 0, 0, 4001, 26, 0, 1, 43, 0, 0, 836, 21, 15, 4096, 3, 130, 0, 0, 0, 3599, 0, 0, 532, 2, 5, 1745, 0, 0, 4056, 4086, 0, 0, 0, 3984, 699, 0, 0, 0, 2, 4095, 50, 2559] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4002, 0, 0, 6, 0, 0, 1161, 4089, 0, 0, 2952, 153, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 241, 0, 8, 0, 6, 0, 0, 55, 4095, 57, 0, 0, 0, 0, 328, 4096, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 29, 4096, 668, 4060, 0, 499, 1464, 0, 2978, 4094, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 892, 0, 0, 0, 19, 3821, 1, 532, 0, 65, 0, 201, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3516, 3420, 1, 0, 0, 0, 4091, 0, 0, 663, 3974, 2300, 0, 0, 0, 458, 209, 166, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4096, 0, 123, 4092, 0, 3803, 4081, 4, 5, 35, 392, 0, 0, 4096, 4053, 79, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 2634, 0, 4096, 884, 0, 3295, 0, 0, 3862, 0, 0, 3473, 0, 3754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3186, 4029, 18, 53, 0, 0, 64, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12367, 13032, 17900, 5628, 18918, 25554, 22130, 15543] avg=16384.0000 max=25554 max_rank=5 min=5628 min_rank=3 imbalance_ratio=1.559692 max_load_reduction=105518 max_load_reduction_pct=0.805038 moved_to_echo=82145 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[38, 0, 4096, 914, 0, 889, 1298, 872, 3811, 31, 1447, 4026, 43, 0, 1, 1821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 54, 0, 17, 581, 0, 9, 1239, 0, 11, 40, 1107, 38, 199, 3753, 3793, 894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3411, 59, 0, 3825, 0, 0, 0, 2978, 3404, 10, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3715, 2905, 0, 0, 1093, 3770, 0, 0, 7, 0, 350, 0, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 0, 0, 3, 0, 483, 4092, 5, 0, 938, 13, 4096, 31, 1, 300, 458, 200, 12, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2345, 0, 5, 0, 11, 4076, 131, 0, 1372, 107, 2097, 0, 0, 201, 1522, 1421, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 4096, 0, 0, 44, 0, 0, 3322, 0, 0, 3338, 0, 3291, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2981, 0, 3937, 0, 0, 3906, 242, 4069, 0, 0, 651, 687, 294, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19287, 15816, 13784, 17658, 14654, 13948, 15030, 20895] avg=16384.0000 max=20895 max_rank=7 min=13784 min_rank=2 imbalance_ratio=1.275330 max_load_reduction=110177 max_load_reduction_pct=0.840584 moved_to_echo=64527 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3605, 10, 4084, 4, 4006, 3036, 0, 18, 0, 0, 3915, 4044, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 48, 20, 4091, 0, 0, 0, 191, 3983, 0, 3942, 0, 703, 0, 0, 15, 3795, 2949, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 3285, 2, 1, 0, 28, 116, 26, 344, 4031, 292, 6, 9, 3638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3892, 1545, 91, 0, 17, 1, 3937, 4096, 1409, 0, 0, 0, 0, 1749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4027, 590, 0, 0, 0, 3927, 4096, 2975, 0, 4044, 0, 0, 0, 58, 0, 52, 231, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 3950, 0, 1, 0, 22, 1139, 4088, 0, 0, 67, 14, 22, 196, 0, 0, 1441, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1642, 0, 517, 0, 881, 113, 8, 0, 0, 3849, 24, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3405, 0, 0, 0, 0, 6, 5, 210, 7, 4060, 543, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22727, 20650, 11949, 20828, 20160, 11292, 11134, 12332] avg=16384.0000 max=22727 max_rank=0 min=11134 min_rank=6 imbalance_ratio=1.387146 max_load_reduction=108345 max_load_reduction_pct=0.826607 moved_to_echo=54918 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3952, 5, 1387, 0, 0, 908, 0, 2, 0, 0, 3898, 0, 0, 3827, 312, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3642, 10, 0, 82, 0, 0, 0, 0, 0, 0, 1744, 0, 28, 0, 4, 382, 2304, 4034, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1437, 0, 0, 0, 0, 2, 0, 0, 2967, 2592, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 148, 3744, 0, 0, 3940, 0, 0, 3104, 55, 0, 0, 5, 4, 1386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 48, 1894, 0, 0, 46, 4040, 0, 0, 4085, 0, 3, 1, 31, 4096, 457, 230, 203, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 2752, 558, 1437, 105, 0, 0, 145, 31, 4096, 0, 0, 1170, 1142, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3086, 0, 790, 12, 3, 3568, 0, 4095, 1201, 15, 0, 3836, 3578, 229, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3869, 0, 4089, 0, 42, 0, 3891, 3973, 6, 142, 0, 0, 91, 37, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14390, 12948, 11097, 16471, 19388, 16190, 20419, 20169] avg=16384.0000 max=20419 max_rank=6 min=11097 min_rank=2 imbalance_ratio=1.246277 max_load_reduction=110653 max_load_reduction_pct=0.844215 moved_to_echo=76166 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[876, 65, 2789, 4094, 567, 3111, 0, 39, 0, 0, 3936, 5, 200, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 3880, 182, 291, 357, 66, 38, 8, 0, 0, 3678, 0, 0, 1785, 184, 3194, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1549, 0, 3454, 139, 0, 389, 86, 4096, 3434, 0, 0, 25, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 81, 0, 19, 0, 3592, 18, 3334, 2763, 0, 1, 528, 0, 4051, 1718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 3883, 0, 0, 15, 0, 3291, 138, 0, 0, 75, 4068, 2359, 11, 0, 0, 231, 200, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 3838, 85, 2, 0, 3, 2751, 4081, 3643, 1, 80, 8, 30, 1259, 0, 0, 1415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1327, 0, 1, 4068, 0, 0, 4059, 0, 0, 3847, 0, 3863, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 230, 3840, 0, 0, 2, 23, 0, 4010, 1, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16180, 18582, 13177, 16107, 18533, 18288, 17941, 12264] avg=16384.0000 max=18582 max_rank=1 min=12264 min_rank=7 imbalance_ratio=1.134155 max_load_reduction=112490 max_load_reduction_pct=0.858231 moved_to_echo=67026 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[320, 0, 4036, 3966, 54, 2968, 3995, 0, 4096, 0, 2645, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2866, 0, 345, 831, 0, 418, 0, 0, 205, 4091, 3501, 0, 0, 81, 0, 0, 4092, 4089, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 4095, 0, 4014, 200, 0, 3737, 775, 2910, 4096, 4, 147, 10, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 9, 3296, 0, 0, 1339, 227, 347, 1438, 4003, 0, 0, 3, 7, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 238, 92, 313, 1871, 61, 4073, 0, 2, 4089, 448, 218, 171, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 3983, 0, 104, 11, 778, 0, 0, 0, 0, 0, 246, 0, 0, 737, 1, 1435, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 3780, 0, 7, 3943, 0, 0, 3630, 0, 0, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3477, 0, 2, 0, 0, 0, 3293, 1552, 0, 0, 0, 749, 3046, 0, 2114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22254, 21421, 20975, 16468, 11692, 8031, 15996, 14235] avg=16384.0000 max=22254 max_rank=0 min=8031 min_rank=5 imbalance_ratio=1.358276 max_load_reduction=108818 max_load_reduction_pct=0.830215 moved_to_echo=49954 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2246, 0, 4014, 0, 7, 20, 1, 0, 0, 789, 3707, 3260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 140, 0, 0, 0, 54, 0, 1522, 0, 0, 0, 0, 0, 0, 4048, 4056, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 1, 4072, 3708, 4096, 3421, 2908, 4093, 1, 718, 518, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 4050, 4096, 5, 1, 14, 0, 2, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 7, 0, 70, 0, 79, 3, 4077, 0, 3432, 158, 0, 3144, 0, 0, 0, 454, 34, 201, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 739, 0, 4075, 65, 3652, 1150, 1582, 0, 0, 4001, 26, 0, 1, 43, 0, 0, 370, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 836, 21, 15, 4096, 3, 130, 0, 0, 0, 3599, 0, 0, 498, 2, 5, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 3783, 699, 0, 0, 0, 2, 4095, 50, 2559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14044, 14145, 27678, 16802, 15827, 16352, 10950, 15274] avg=16384.0000 max=27678 max_rank=2 min=10950 min_rank=6 imbalance_ratio=1.689331 max_load_reduction=103394 max_load_reduction_pct=0.788834 moved_to_echo=58403 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3592, 0, 0, 2004, 0, 0, 59, 0, 3, 14, 3936, 144, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 7, 0, 4090, 0, 0, 0, 0, 0, 0, 42, 435, 933, 4077, 3940, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3774, 0, 17, 4047, 1, 0, 1757, 0, 0, 0, 2894, 0, 2, 0, 1984, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 658, 4096, 2350, 9, 0, 21, 881, 0, 4062, 3240, 3530, 0, 0, 0, 1749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 3969, 116, 0, 1831, 36, 4016, 0, 5, 848, 445, 232, 203, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 1001, 1310, 0, 1757, 2994, 0, 47, 728, 2, 16, 3714, 87, 0, 3602, 1441, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 2678, 0, 12, 3199, 0, 281, 0, 0, 0, 3862, 0, 3238, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4025, 0, 51, 12, 65, 166, 3893, 4079, 0, 0, 0, 4091, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9823, 14478, 14512, 21110, 11900, 21417, 17472, 20360] avg=16384.0000 max=21417 max_rank=5 min=9823 min_rank=0 imbalance_ratio=1.307190 max_load_reduction=109655 max_load_reduction_pct=0.836601 moved_to_echo=71149 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 18560, 0, 19712, 4224, 128, 4352, 22144, 18432, 128, 512, 768, 9600, 16128, 128, 17024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1408, 5760, 128, 22784, 1024, 768, 18048, 896, 4608, 20096, 128, 0, 26368, 3712, 18560, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[8064, 1664, 1536, 22528, 896, 8192, 10368, 4352, 13568, 7680, 18048, 22400, 384, 1024, 2560, 8832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[8704, 4864, 17408, 14720, 7168, 0, 8960, 6144, 15488, 20096, 8832, 3712, 640, 640, 4096, 10624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[18688, 128, 20480, 11008, 4736, 11008, 6528, 5120, 7936, 896, 26496, 11904, 256, 3840, 896, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[4096, 19840, 8064, 2048, 128, 128, 4480, 27904, 3328, 3840, 13696, 4224, 21760, 2432, 128, 9344, 2816, 1664, 1280, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[22912, 384, 384, 9216, 256, 768, 5120, 384, 4352, 11008, 9344, 128, 4480, 1280, 512, 3712, 26368, 26112, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2304, 22400, 5248, 5760, 10624, 4352, 17792, 12544, 3712, 6144, 512, 2816, 7936, 1664, 5120, 10368, 8832, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[3233, 3791, 26, 2936, 0, 0, 243, 0, 0, 3968, 0, 4096, 0, 1999, 2280, 0, 314, 0, 2537, 4057, 0, 294, 0, 4081, 1, 0, 4059, 4088, 0, 0, 0, 3552, 0, 6, 119, 0, 2799, 1925, 47, 3991, 3062, 80, 0, 17, 0, 0, 812, 0, 0, 0, 13, 0, 4091, 0, 0, 2, 178, 0, 0, 0, 0, 3, 38, 1, 4079, 4095, 0, 0, 72, 0, 0, 1, 4095, 0, 4091, 0, 0, 0, 666, 76, 0, 5, 0, 9, 2030, 345, 0, 0, 0, 0, 0, 2036, 1099, 4096, 0, 0, 0, 3990, 4087, 3136, 4096, 21, 12, 0, 1909, 125, 1301, 0, 1604, 243, 0, 5, 0, 2930, 0, 14, 5, 19, 3982, 0, 4077, 0, 0, 4080, 1836, 0, 0, 4066] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[5, 1969, 45, 1381, 0, 4094, 2707, 0, 1126, 0, 3630, 0, 77, 3277, 704, 85, 8, 13, 4076, 0, 34, 0, 716, 0, 4, 0, 0, 59, 0, 3476, 0, 0, 3, 0, 4096, 62, 0, 0, 4095, 2869, 0, 3957, 0, 7, 0, 520, 449, 815, 27, 26, 0, 0, 0, 0, 133, 0, 4032, 4096, 0, 21, 0, 1465, 0, 0, 3461, 4002, 1299, 21, 16, 0, 0, 2480, 4096, 0, 1, 3, 5, 1623, 4075, 27, 0, 0, 0, 25, 1, 3674, 0, 0, 166, 20, 0, 4096, 20, 2332, 20, 0, 0, 0, 1666, 1506, 1444, 0, 0, 0, 3949, 0, 4096, 4, 3409, 3, 0, 0, 4083, 2, 4096, 2, 0, 0, 825, 0, 4066, 4025, 0, 4096, 4089, 0, 0, 4089] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[15, 0, 0, 0, 3088, 773, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 142, 103, 105, 0, 4079, 0, 15, 4096, 1729, 0, 4042, 2294, 0, 4044, 39, 6, 21, 15, 3731, 1991, 7, 103, 0, 6, 0, 412, 0, 2, 60, 389, 0, 4019, 4051, 0, 109, 0, 3121, 0, 0, 0, 845, 309, 4067, 24, 78, 0, 151, 272, 1866, 3956, 0, 3, 4095, 3529, 2, 0, 3966, 0, 168, 78, 2769, 46, 195, 0, 0, 0, 2, 1280, 1938, 995, 250, 260, 3, 0, 0, 3907, 0, 4077, 0, 4095, 5, 0, 4080, 941, 0, 2020, 12, 156, 3897, 0, 0, 0, 0, 2679, 4096, 0, 4082, 4096, 693, 49, 0, 3796, 0, 46, 1545, 667, 0, 4072, 1, 103, 4, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[31, 4021, 2, 149, 37, 0, 0, 0, 546, 2562, 0, 0, 20, 22, 0, 4, 268, 0, 36, 157, 602, 0, 71, 4096, 0, 0, 2228, 1293, 1651, 1, 0, 4081, 116, 3401, 3994, 0, 0, 18, 2132, 3832, 0, 4086, 0, 0, 0, 3244, 0, 4095, 1200, 0, 159, 0, 4080, 0, 0, 21, 0, 0, 4, 0, 15, 872, 0, 0, 3884, 4093, 34, 67, 6, 1530, 63, 12, 2516, 13, 2, 4055, 7, 129, 4036, 0, 0, 4, 3148, 4056, 722, 28, 0, 14, 0, 6, 112, 3968, 26, 4096, 0, 0, 0, 0, 4077, 177, 0, 0, 4090, 1381, 262, 0, 3985, 0, 4057, 1603, 245, 0, 0, 4096, 208, 1210, 0, 14, 0, 0, 4092, 4, 0, 4096, 3684, 0, 0, 4017] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[3908, 17, 7, 29, 1986, 203, 6, 0, 3088, 143, 0, 0, 1633, 130, 0, 0, 0, 0, 2942, 3723, 4093, 35, 39, 4066, 1, 30, 4096, 825, 0, 0, 6, 0, 1169, 0, 4096, 0, 94, 0, 0, 3320, 4, 4011, 0, 0, 429, 0, 0, 1858, 0, 47, 4096, 51, 4057, 0, 0, 1994, 4096, 0, 1299, 3, 0, 849, 6, 0, 0, 4076, 0, 0, 14, 0, 0, 0, 4022, 0, 12, 1, 4, 2, 530, 0, 91, 0, 0, 0, 5, 427, 0, 1, 0, 0, 0, 4035, 0, 4096, 2, 0, 0, 0, 4094, 3981, 1325, 256, 440, 3681, 4091, 0, 4094, 0, 3957, 2245, 1, 0, 3722, 35, 3966, 0, 0, 0, 4092, 0, 4096, 1585, 0, 3027, 834, 1, 0, 1746] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[2346, 3344, 0, 1643, 0, 167, 0, 0, 0, 0, 104, 83, 0, 86, 8, 0, 115, 0, 3999, 3843, 0, 189, 0, 4095, 3, 0, 3787, 4062, 0, 0, 11, 633, 0, 0, 4074, 847, 0, 62, 4096, 4096, 272, 0, 0, 15, 5, 3740, 1, 3680, 0, 0, 0, 57, 0, 0, 0, 899, 3009, 0, 2065, 0, 0, 0, 1, 0, 418, 3664, 48, 406, 7, 2294, 19, 0, 4096, 0, 0, 4, 0, 4089, 0, 34, 226, 0, 0, 13, 4049, 4044, 944, 0, 43, 432, 0, 4095, 0, 7, 0, 0, 0, 35, 3913, 1014, 164, 0, 25, 0, 4033, 0, 0, 370, 0, 3943, 3967, 18, 3634, 473, 4096, 0, 0, 3009, 0, 24, 3589, 0, 2358, 3868, 4078, 0, 194, 3898] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 5, 1962, 0, 146, 0, 0, 0, 4, 708, 0, 0, 1102, 1, 4, 0, 0, 0, 128, 4096, 46, 33, 0, 3776, 1161, 4, 4094, 3645, 0, 589, 0, 6, 1, 0, 1267, 0, 0, 0, 0, 0, 0, 4096, 50, 671, 5, 4, 437, 4065, 0, 0, 1052, 395, 3852, 0, 0, 0, 4096, 0, 0, 4089, 0, 699, 0, 0, 3875, 4078, 1748, 0, 0, 225, 0, 4074, 93, 0, 4087, 0, 0, 2933, 4039, 0, 0, 0, 1, 3258, 0, 3949, 0, 0, 0, 3383, 0, 3977, 340, 4096, 0, 0, 0, 0, 4031, 2091, 0, 0, 4073, 4096, 3974, 3277, 1, 278, 4096, 4039, 0, 0, 26, 461, 225, 0, 83, 0, 268, 0, 362, 328, 0, 1330, 3915, 0, 0, 3673] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[115, 2260, 0, 0, 0, 3605, 0, 448, 4052, 4094, 461, 354, 0, 0, 1, 0, 0, 1013, 5, 10, 8, 0, 0, 4090, 0, 0, 3788, 12, 0, 4002, 0, 0, 0, 0, 4002, 109, 0, 0, 0, 4072, 4, 3675, 0, 205, 0, 0, 258, 106, 242, 0, 10, 2032, 0, 0, 0, 0, 1987, 0, 0, 0, 0, 4095, 279, 135, 6, 3, 3690, 0, 4096, 2501, 28, 0, 4005, 1, 4066, 3897, 0, 83, 4089, 24, 43, 0, 0, 4096, 0, 0, 801, 0, 11, 4, 4093, 2464, 0, 13, 53, 0, 0, 3, 4096, 3941, 0, 181, 464, 4041, 4096, 0, 0, 74, 0, 0, 1928, 0, 87, 3694, 0, 4047, 3895, 21, 3229, 35, 75, 10, 0, 3196, 1604, 4096, 66, 597] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3233, 3791, 26, 2936, 0, 0, 243, 0, 0, 3968, 0, 4096, 0, 1999, 2280, 0, 3705, 904, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 2537, 4057, 0, 294, 0, 3133, 1, 0, 4059, 4088, 0, 0, 0, 3552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 119, 0, 2799, 1925, 47, 3991, 3062, 80, 0, 17, 0, 0, 812, 0, 948, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4091, 0, 0, 2, 178, 0, 0, 0, 0, 3, 38, 1, 4087, 3237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 624, 0, 0, 72, 0, 0, 1, 4095, 0, 4091, 0, 0, 0, 666, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 9, 2030, 345, 0, 0, 0, 0, 0, 2036, 1099, 4096, 0, 0, 2567, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 0, 3136, 4096, 21, 12, 0, 1350, 125, 1301, 0, 1604, 243, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2930, 0, 14, 5, 19, 3982, 0, 4077, 0, 0, 0, 1836, 0, 0, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[28024, 22035, 13820, 11650, 13704, 12732, 15883, 13224] avg=16384.0000 max=28024 max_rank=0 min=11650 min_rank=3 imbalance_ratio=1.710449 max_load_reduction=103048 max_load_reduction_pct=0.786194 moved_to_echo=55543 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[15, 0, 0, 0, 3088, 773, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 3728, 873, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 103, 105, 0, 4079, 0, 15, 3145, 1729, 0, 4042, 2294, 0, 4044, 39, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 15, 3731, 1991, 7, 103, 0, 6, 0, 412, 0, 2, 60, 389, 0, 4019, 951, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 109, 0, 3121, 0, 0, 0, 845, 309, 4067, 24, 78, 0, 151, 272, 4080, 3232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1866, 603, 0, 3, 4095, 3529, 2, 0, 3966, 0, 168, 78, 2769, 46, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1280, 1938, 995, 250, 260, 3, 0, 0, 3907, 0, 4077, 0, 4095, 2480, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 941, 0, 2020, 12, 156, 2756, 0, 0, 0, 0, 2679, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 693, 49, 0, 3796, 0, 46, 1545, 667, 0, 0, 1, 103, 4, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13425, 19743, 11736, 20339, 17320, 20399, 12665, 15445] avg=16384.0000 max=20399 max_rank=5 min=11736 min_rank=2 imbalance_ratio=1.245056 max_load_reduction=110673 max_load_reduction_pct=0.844368 moved_to_echo=65829 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[5, 1969, 45, 1381, 0, 4094, 2707, 0, 1126, 0, 3630, 0, 77, 3277, 704, 85, 3726, 883, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 13, 4076, 0, 34, 0, 716, 0, 4, 0, 0, 59, 0, 3476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 62, 0, 0, 4095, 2869, 0, 3957, 0, 7, 0, 520, 449, 815, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 26, 0, 0, 0, 0, 133, 0, 4032, 4096, 0, 21, 0, 1465, 0, 0, 1666, 3251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3461, 610, 1299, 21, 16, 0, 0, 2480, 4096, 0, 1, 3, 5, 1623, 4075, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 1, 3674, 0, 0, 166, 20, 0, 4096, 20, 2332, 20, 0, 2509, 1127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1506, 1444, 0, 0, 0, 2793, 0, 4096, 4, 3409, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 2, 4096, 2, 0, 0, 825, 0, 4066, 4025, 0, 0, 4089, 0, 0, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[24554, 8386, 16902, 14717, 17717, 13990, 13255, 21551] avg=16384.0000 max=24554 max_rank=0 min=8386 min_rank=1 imbalance_ratio=1.498657 max_load_reduction=106518 max_load_reduction_pct=0.812668 moved_to_echo=66513 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[31, 4021, 2, 149, 37, 0, 0, 0, 546, 2562, 0, 0, 20, 22, 0, 4, 3661, 904, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 268, 0, 36, 157, 602, 0, 71, 3145, 0, 0, 2228, 1293, 1651, 1, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 3401, 3994, 0, 0, 18, 2132, 3832, 0, 4086, 0, 0, 0, 3244, 0, 4095, 951, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1200, 0, 159, 0, 4080, 0, 0, 21, 0, 0, 4, 0, 15, 872, 0, 0, 4077, 3250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 623, 34, 67, 6, 1530, 63, 12, 2516, 13, 2, 4055, 7, 129, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3148, 4056, 722, 28, 0, 14, 0, 6, 112, 3968, 26, 4096, 0, 0, 2566, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 4090, 1381, 187, 0, 3985, 0, 4057, 1603, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 208, 1210, 0, 14, 0, 0, 4092, 4, 0, 0, 3684, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12805, 13533, 25870, 13678, 16977, 18820, 15725, 13664] avg=16384.0000 max=25870 max_rank=2 min=12805 min_rank=0 imbalance_ratio=1.578979 max_load_reduction=105202 max_load_reduction_pct=0.802628 moved_to_echo=65186 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2346, 3344, 0, 1643, 0, 167, 0, 0, 0, 0, 104, 83, 0, 86, 8, 0, 3552, 809, 799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 3999, 3843, 0, 189, 0, 3144, 3, 0, 3787, 4062, 0, 0, 11, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 847, 0, 62, 4096, 4096, 272, 0, 0, 15, 5, 3740, 1, 3680, 951, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 899, 3009, 0, 2065, 0, 0, 0, 1, 0, 3913, 3069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 558, 48, 406, 7, 2294, 19, 0, 4096, 0, 0, 4, 0, 4089, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 13, 4049, 4044, 944, 0, 43, 432, 0, 4095, 0, 7, 0, 0, 2297, 1151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 1014, 164, 0, 25, 0, 2852, 0, 0, 370, 0, 3943, 3967, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3634, 473, 4096, 0, 0, 3009, 0, 24, 3589, 0, 2358, 0, 4078, 0, 194, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12941, 19786, 21869, 13013, 11973, 17301, 12388, 21801] avg=16384.0000 max=21869 max_rank=2 min=11973 min_rank=4 imbalance_ratio=1.334778 max_load_reduction=109203 max_load_reduction_pct=0.833153 moved_to_echo=63463 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 1962, 0, 146, 0, 0, 0, 4, 708, 0, 0, 1102, 1, 4, 0, 3352, 904, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 4096, 46, 33, 0, 2897, 1161, 4, 4094, 3645, 0, 589, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1267, 0, 0, 0, 0, 0, 0, 4096, 50, 671, 5, 4, 437, 4065, 879, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 395, 3852, 0, 0, 0, 4096, 0, 0, 4089, 0, 699, 0, 0, 4031, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3875, 613, 1748, 0, 0, 225, 0, 4074, 93, 0, 4087, 0, 0, 2933, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3258, 0, 3949, 0, 0, 0, 3383, 0, 3977, 340, 4096, 0, 0, 2561, 1138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2091, 0, 0, 4073, 4096, 2802, 3277, 1, 278, 4096, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 461, 225, 0, 83, 0, 268, 0, 362, 328, 0, 0, 3915, 0, 0, 321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8463, 16699, 11509, 19269, 21687, 22703, 24753, 5989] avg=16384.0000 max=24753 max_rank=6 min=5989 min_rank=7 imbalance_ratio=1.510803 max_load_reduction=106319 max_load_reduction_pct=0.811150 moved_to_echo=75132 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[115, 2260, 0, 0, 0, 3605, 0, 448, 4052, 4094, 461, 354, 0, 0, 1, 0, 544, 0, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 5, 10, 8, 0, 0, 3140, 0, 0, 3788, 12, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 109, 0, 0, 0, 4072, 4, 3675, 0, 205, 0, 0, 258, 106, 950, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 10, 2032, 0, 0, 0, 0, 1987, 0, 0, 0, 0, 4095, 279, 135, 4096, 2536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 3690, 0, 4096, 2501, 28, 0, 4005, 1, 4066, 3897, 0, 83, 4089, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 4096, 0, 0, 801, 0, 11, 4, 4093, 2464, 0, 13, 53, 0, 1, 1169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3941, 0, 181, 464, 4041, 2897, 0, 0, 74, 0, 0, 1928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 3694, 0, 4047, 3895, 21, 3229, 35, 75, 10, 0, 0, 1604, 4096, 66, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16594, 11978, 13411, 15412, 26488, 12748, 13529, 20912] avg=16384.0000 max=26488 max_rank=4 min=11978 min_rank=1 imbalance_ratio=1.616699 max_load_reduction=104584 max_load_reduction_pct=0.797913 moved_to_echo=73677 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3908, 17, 7, 29, 1986, 203, 6, 0, 3088, 143, 0, 0, 1633, 130, 0, 0, 1591, 900, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2942, 3723, 4093, 35, 39, 3122, 1, 30, 4096, 825, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1169, 0, 4096, 0, 94, 0, 0, 3320, 4, 4011, 0, 0, 429, 0, 0, 1858, 944, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4096, 51, 4057, 0, 0, 1994, 4096, 0, 1299, 3, 0, 849, 6, 0, 4094, 2402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 14, 0, 0, 0, 4022, 0, 12, 1, 4, 2, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 5, 427, 0, 1, 0, 0, 0, 4035, 0, 4096, 2, 0, 2555, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 1325, 256, 440, 3681, 2894, 0, 4094, 0, 3957, 2245, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3722, 35, 3966, 0, 0, 0, 4092, 0, 4096, 1585, 0, 0, 834, 1, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14266, 18912, 15955, 22994, 5206, 12379, 22874, 18486] avg=16384.0000 max=22994 max_rank=3 min=5206 min_rank=4 imbalance_ratio=1.403442 max_load_reduction=108078 max_load_reduction_pct=0.824570 moved_to_echo=58945 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[896, 1152, 13952, 16000, 8960, 640, 896, 21760, 2944, 128, 26112, 16384, 1664, 12160, 128, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[5632, 128, 5504, 2560, 19328, 0, 256, 2944, 18304, 4480, 7552, 4224, 128, 8064, 512, 512, 30080, 22144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[17664, 4352, 6912, 512, 8320, 10112, 128, 6656, 27008, 128, 12544, 8064, 2816, 8960, 17664, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 4096, 0, 16896, 7040, 2560, 9216, 13440, 18560, 3456, 13568, 768, 17152, 14848, 10240, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[9728, 15488, 2048, 6144, 5376, 8960, 3072, 512, 12928, 11520, 4224, 4608, 2944, 5632, 3072, 128, 23936, 6272, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 128, 3200, 12800, 8832, 13568, 2048, 384, 256, 3968, 4224, 28672, 1536, 22912, 128, 4096, 17536, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1408, 3456, 25472, 3072, 2944, 2176, 10496, 22272, 3456, 20352, 128, 1024, 512, 7936, 2048, 18688, 6656, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[15744, 15872, 13312, 5376, 4096, 6912, 12416, 128, 22016, 6656, 2432, 0, 20096, 4224, 384, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4, 3, 0, 0, 3928, 0, 110, 0, 0, 0, 0, 10, 0, 4060, 0, 2748, 4066, 3989, 561, 247, 0, 0, 115, 6, 0, 0, 4096, 0, 100, 0, 49, 0, 0, 0, 4087, 0, 0, 0, 0, 71, 4013, 4040, 0, 0, 0, 7, 0, 3651, 4083, 3, 0, 1, 0, 72, 0, 204, 80, 0, 0, 4019, 2221, 3743, 13, 3984, 122, 4080, 3638, 0, 0, 1, 0, 2, 120, 276, 0, 0, 3478, 1, 0, 4070, 2918, 4001, 110, 0, 1318, 3669, 0, 3993, 3837, 597, 0, 2, 0, 0, 22, 4047, 4096, 16, 7, 409, 50, 4096, 30, 0, 18, 461, 2399, 0, 4048, 173, 2951, 0, 0, 18, 4096, 0, 298, 0, 93, 1, 0, 0, 4096, 0, 941, 0, 0, 4089] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[3, 4080, 391, 0, 1740, 0, 0, 725, 3906, 22, 201, 8, 0, 160, 0, 0, 4077, 90, 4084, 4085, 0, 51, 752, 0, 0, 0, 3845, 0, 0, 0, 4096, 0, 979, 0, 0, 0, 187, 11, 45, 0, 3767, 243, 0, 0, 3468, 4096, 366, 4080, 4096, 17, 4082, 0, 0, 0, 2825, 0, 3, 277, 0, 0, 4095, 4096, 0, 0, 0, 4054, 3536, 111, 0, 4096, 0, 2972, 3467, 3377, 0, 0, 0, 69, 3, 30, 16, 4095, 154, 0, 0, 0, 0, 2529, 4025, 0, 1, 0, 0, 4096, 78, 3208, 621, 0, 70, 0, 0, 1635, 0, 291, 0, 0, 3934, 493, 0, 1, 4055, 37, 187, 565, 326, 2179, 4096, 5, 36, 2711, 2, 844, 0, 0, 0, 17, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[7, 0, 2, 0, 563, 0, 0, 3832, 0, 1, 1393, 0, 0, 1691, 38, 159, 2448, 24, 0, 3794, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 50, 129, 1, 8, 0, 0, 4055, 4096, 0, 4069, 0, 2895, 1665, 0, 127, 3288, 4078, 4076, 0, 25, 50, 543, 3947, 0, 0, 0, 3919, 762, 0, 4083, 4096, 76, 0, 0, 286, 4096, 2856, 0, 4042, 0, 563, 2785, 3396, 0, 0, 858, 3697, 0, 2348, 3675, 4096, 2660, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 370, 0, 2043, 793, 14, 3, 24, 0, 0, 0, 0, 5, 4094, 2453, 60, 0, 0, 3669, 0, 480, 4096, 2694, 0, 367, 0, 28, 18, 0, 0, 4066, 1543, 4, 0, 666, 29] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 4069, 0, 0, 2, 0, 16, 0, 0, 78, 0, 0, 0, 2, 3, 0, 4096, 4096, 3434, 1558, 0, 17, 0, 0, 0, 0, 0, 0, 48, 402, 4045, 0, 4093, 1395, 643, 0, 2005, 0, 4094, 0, 2557, 4096, 0, 0, 0, 1, 0, 3996, 4056, 0, 5, 28, 0, 4096, 0, 0, 55, 0, 0, 16, 1481, 3478, 0, 4095, 0, 35, 4, 0, 0, 2, 0, 2977, 4093, 0, 0, 0, 1327, 7, 0, 0, 2, 3389, 0, 0, 4096, 6, 12, 3968, 4096, 5, 26, 0, 0, 4093, 0, 4096, 710, 0, 3349, 113, 4085, 3839, 260, 2, 4026, 3774, 4095, 0, 0, 136, 86, 2461, 0, 1420, 0, 834, 3710, 2, 3700, 16, 98, 0, 0, 0, 0, 0, 33, 33] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[21, 4096, 3, 28, 0, 3311, 41, 0, 0, 0, 121, 0, 0, 4096, 4, 0, 4093, 137, 0, 85, 0, 0, 4038, 0, 0, 4, 3965, 1053, 0, 0, 2426, 0, 4096, 0, 44, 0, 0, 4096, 3662, 0, 159, 0, 16, 9, 2030, 0, 4026, 3392, 4064, 0, 14, 0, 4087, 0, 2, 186, 0, 0, 0, 0, 738, 3949, 0, 4079, 85, 98, 0, 3758, 16, 1714, 2, 0, 3941, 3910, 0, 0, 24, 0, 61, 3823, 0, 3674, 0, 0, 3573, 0, 0, 1952, 4066, 0, 0, 75, 3372, 0, 0, 1974, 0, 0, 4096, 0, 4089, 280, 0, 1504, 0, 391, 3870, 0, 0, 4092, 3798, 0, 0, 0, 58, 0, 0, 334, 4096, 0, 539, 0, 155, 0, 2, 186, 18, 1275] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[248, 90, 0, 0, 0, 0, 0, 2775, 2147, 0, 37, 0, 224, 4075, 0, 0, 1046, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 484, 0, 0, 0, 4031, 0, 2, 2889, 454, 0, 0, 219, 3115, 1, 57, 9, 40, 4086, 4095, 1032, 3, 3381, 4096, 1227, 2, 0, 3703, 4054, 556, 0, 0, 6, 774, 59, 21, 4096, 3440, 87, 0, 1, 0, 12, 0, 3986, 0, 0, 4074, 0, 0, 0, 3835, 3186, 0, 0, 0, 3960, 0, 0, 3999, 1444, 0, 4032, 4096, 0, 0, 9, 0, 0, 0, 3968, 0, 0, 1037, 0, 4096, 86, 4077, 523, 4096, 3880, 4096, 0, 0, 4096, 4096, 0, 0, 56, 0, 0, 161, 0, 0, 0, 2699, 0, 553, 90, 0, 69, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[162, 4089, 241, 407, 4040, 37, 0, 1, 0, 224, 0, 4096, 3481, 3980, 0, 0, 4093, 4090, 0, 0, 1, 18, 1136, 0, 0, 1148, 2, 1, 0, 5, 3988, 0, 3909, 34, 3, 32, 1, 0, 3095, 0, 0, 10, 31, 12, 3931, 0, 4053, 4041, 0, 0, 4, 0, 4091, 3764, 35, 0, 0, 297, 14, 0, 865, 563, 0, 4090, 0, 0, 30, 51, 203, 0, 0, 0, 869, 0, 2, 6, 10, 0, 3128, 0, 951, 4077, 0, 0, 3978, 4094, 103, 0, 4047, 0, 1, 0, 32, 4094, 0, 4096, 1351, 27, 4082, 3848, 7, 1360, 149, 113, 0, 2, 3298, 18, 18, 0, 4041, 3768, 0, 40, 3934, 0, 201, 2, 3862, 2046, 0, 0, 613, 0, 0, 399, 4, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[380, 0, 1100, 0, 0, 0, 62, 3696, 0, 0, 0, 35, 0, 4096, 0, 380, 3942, 1836, 4015, 0, 0, 139, 7, 0, 0, 0, 1302, 99, 8, 14, 3163, 281, 233, 4, 0, 0, 0, 3, 4021, 4064, 3899, 3901, 0, 20, 0, 4079, 4096, 3843, 2756, 0, 0, 0, 3411, 0, 14, 30, 0, 0, 0, 142, 0, 1066, 5, 3743, 8, 4081, 4096, 0, 128, 4096, 0, 2958, 314, 4094, 16, 12, 0, 0, 939, 4012, 0, 3651, 0, 0, 3189, 0, 3867, 4, 34, 0, 1708, 13, 0, 3910, 2, 4074, 2258, 207, 0, 0, 2216, 104, 415, 117, 3667, 21, 0, 2182, 1006, 0, 3667, 0, 3, 2, 3850, 0, 58, 1, 1, 45, 0, 0, 2, 0, 123, 7, 2029, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4, 3, 0, 0, 3928, 0, 110, 0, 0, 0, 0, 10, 0, 4060, 0, 2748, 2951, 1478, 753, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 3989, 561, 247, 0, 0, 115, 6, 0, 0, 4096, 0, 100, 0, 49, 0, 0, 645, 8, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 71, 4013, 4040, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2605, 3, 0, 1, 0, 72, 0, 204, 80, 0, 0, 4019, 2221, 3743, 13, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 4080, 3638, 0, 0, 1, 0, 2, 112, 276, 0, 0, 3478, 1, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2918, 0, 110, 0, 1318, 3669, 0, 3993, 2815, 597, 0, 2, 0, 0, 22, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 7, 409, 50, 4096, 30, 0, 18, 461, 2335, 0, 4048, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4096, 0, 298, 0, 93, 1, 0, 0, 4096, 0, 941, 0, 0, 4089, 4001, 2898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16422, 13946, 12218, 16945, 15780, 19491, 15739, 20531] avg=16384.0000 max=20531 max_rank=7 min=12218 min_rank=2 imbalance_ratio=1.253113 max_load_reduction=110541 max_load_reduction_pct=0.843361 moved_to_echo=71541 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 4080, 391, 0, 1740, 0, 0, 725, 3906, 22, 201, 8, 0, 160, 0, 0, 4055, 1483, 842, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 90, 4084, 4085, 0, 51, 752, 0, 0, 0, 3845, 0, 0, 0, 4096, 0, 14, 677, 243, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 979, 0, 0, 0, 187, 11, 31, 0, 3767, 243, 0, 0, 3468, 4096, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2613, 17, 4082, 0, 0, 0, 2825, 0, 3, 277, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 3536, 111, 0, 4096, 0, 2972, 3224, 3377, 0, 0, 0, 69, 3, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 154, 0, 0, 0, 0, 2529, 2952, 0, 1, 0, 0, 4096, 78, 3208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 70, 0, 0, 1635, 0, 291, 0, 0, 3828, 493, 0, 1, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 565, 326, 2179, 4096, 5, 36, 2711, 2, 844, 0, 0, 0, 17, 0, 1, 4095, 3238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18012, 22120, 13148, 18008, 21472, 13034, 6976, 18302] avg=16384.0000 max=22120 max_rank=1 min=6976 min_rank=6 imbalance_ratio=1.350098 max_load_reduction=108952 max_load_reduction_pct=0.831238 moved_to_echo=59784 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 2, 0, 563, 0, 0, 3832, 0, 1, 1393, 0, 0, 1691, 38, 159, 3669, 1475, 841, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2448, 24, 0, 3794, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 50, 1313, 689, 195, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 1, 8, 0, 0, 4055, 2783, 0, 4069, 0, 2895, 1665, 0, 127, 3288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2601, 0, 25, 50, 543, 3947, 0, 0, 0, 3919, 762, 0, 4083, 4096, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 4096, 2856, 0, 4042, 0, 563, 2590, 3396, 0, 0, 858, 3697, 0, 2348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3675, 0, 2660, 0, 0, 0, 0, 0, 3003, 0, 0, 0, 0, 370, 0, 2043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 793, 14, 3, 24, 0, 0, 0, 0, 5, 4094, 2387, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 4096, 2694, 0, 367, 0, 28, 18, 0, 0, 4066, 1543, 4, 0, 666, 29, 4096, 3237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14074, 12689, 19020, 20102, 24732, 11751, 7380, 21324] avg=16384.0000 max=24732 max_rank=4 min=7380 min_rank=6 imbalance_ratio=1.509521 max_load_reduction=106340 max_load_reduction_pct=0.811310 moved_to_echo=65187 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4069, 0, 0, 2, 0, 16, 0, 0, 78, 0, 0, 0, 2, 3, 0, 86, 1468, 825, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3434, 1558, 0, 17, 0, 0, 0, 0, 0, 0, 48, 402, 4045, 0, 1312, 689, 287, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1395, 643, 0, 2005, 0, 2782, 0, 2557, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2588, 0, 5, 28, 0, 4096, 0, 0, 55, 0, 0, 16, 1481, 3478, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4, 0, 0, 2, 0, 2977, 3806, 0, 0, 0, 1327, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 6, 12, 3968, 3004, 5, 26, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 710, 0, 3349, 113, 4085, 3839, 260, 2, 4026, 3774, 3985, 0, 0, 136, 0, 2461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 0, 834, 3710, 2, 3700, 16, 98, 0, 0, 0, 0, 0, 33, 33, 3389, 3171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6952, 20094, 17572, 15842, 8158, 19308, 26740, 16406] avg=16384.0000 max=26740 max_rank=6 min=6952 min_rank=0 imbalance_ratio=1.632080 max_load_reduction=104332 max_load_reduction_pct=0.795990 moved_to_echo=70612 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[380, 0, 1100, 0, 0, 0, 62, 3696, 0, 0, 0, 35, 0, 4096, 0, 380, 3667, 997, 794, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3942, 1836, 4015, 0, 0, 139, 7, 0, 0, 0, 1302, 99, 8, 14, 3163, 281, 1289, 5, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 4, 0, 0, 0, 3, 2732, 4064, 3899, 3901, 0, 20, 0, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 0, 0, 0, 3411, 0, 14, 30, 0, 0, 0, 142, 0, 1066, 5, 3743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4081, 4096, 0, 128, 4096, 0, 2958, 292, 4094, 16, 12, 0, 0, 939, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3189, 0, 3867, 4, 26, 0, 1708, 13, 0, 3910, 2, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2258, 207, 0, 0, 2216, 104, 415, 117, 3667, 21, 0, 2182, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 3850, 0, 58, 1, 1, 45, 0, 0, 2, 0, 123, 7, 2029, 0, 3651, 3049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15210, 16122, 23031, 10170, 24732, 16793, 12193, 12821] avg=16384.0000 max=24732 max_rank=4 min=10170 min_rank=3 imbalance_ratio=1.509521 max_load_reduction=106340 max_load_reduction_pct=0.811310 moved_to_echo=66539 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[162, 4089, 241, 407, 4040, 37, 0, 1, 0, 224, 0, 4096, 3481, 3980, 0, 0, 4041, 0, 834, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4090, 0, 0, 1, 18, 1136, 0, 0, 1148, 2, 1, 0, 5, 3988, 0, 992, 681, 61, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 34, 3, 32, 1, 0, 2103, 0, 0, 10, 31, 12, 3931, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4091, 3764, 35, 0, 0, 297, 14, 0, 865, 563, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 51, 203, 0, 0, 0, 808, 0, 2, 6, 10, 0, 3128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 0, 0, 0, 3978, 4094, 103, 0, 2968, 0, 1, 0, 32, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1351, 27, 4082, 3848, 7, 1360, 149, 113, 0, 2, 3209, 18, 18, 0, 0, 3768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 3934, 0, 201, 2, 3862, 2046, 0, 0, 613, 0, 0, 399, 4, 2, 4077, 3207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[26031, 16305, 14119, 13723, 4238, 20317, 17952, 18387] avg=16384.0000 max=26031 max_rank=0 min=4238 min_rank=4 imbalance_ratio=1.588806 max_load_reduction=105041 max_load_reduction_pct=0.801399 moved_to_echo=60894 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[248, 90, 0, 0, 0, 0, 0, 2775, 2147, 0, 37, 0, 224, 4075, 0, 0, 4096, 1487, 698, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 484, 0, 0, 0, 4031, 0, 1002, 693, 290, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2889, 454, 0, 0, 219, 2113, 1, 57, 9, 40, 4086, 4095, 1032, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2609, 1227, 2, 0, 3703, 4054, 556, 0, 0, 6, 774, 59, 21, 4096, 3440, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 3986, 0, 0, 3784, 0, 0, 0, 3835, 3186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 1444, 0, 4032, 2998, 0, 0, 9, 0, 0, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 0, 4096, 86, 4077, 523, 4096, 3880, 3983, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 161, 0, 0, 0, 2699, 0, 553, 90, 0, 69, 0, 0, 3960, 2683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16282, 11757, 15000, 20634, 14804, 16450, 25874, 10271] avg=16384.0000 max=25874 max_rank=6 min=10271 min_rank=7 imbalance_ratio=1.579224 max_load_reduction=105198 max_load_reduction_pct=0.802597 moved_to_echo=67399 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[21, 4096, 3, 28, 0, 3311, 41, 0, 0, 0, 121, 0, 0, 4096, 4, 0, 3798, 1471, 699, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 137, 0, 85, 0, 0, 4038, 0, 0, 4, 3965, 1053, 0, 0, 2426, 0, 1174, 684, 276, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 44, 0, 0, 4096, 2488, 0, 159, 0, 16, 9, 2030, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2593, 0, 14, 0, 4087, 0, 2, 186, 0, 0, 0, 0, 738, 3949, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 98, 0, 3758, 16, 1714, 2, 0, 3665, 3910, 0, 0, 24, 0, 61, 3823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3573, 0, 0, 1952, 2982, 0, 0, 75, 3372, 0, 0, 1974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 280, 0, 1504, 0, 391, 3766, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 334, 4096, 0, 539, 0, 155, 0, 2, 186, 18, 1275, 3674, 2693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18089, 18039, 16964, 15648, 17156, 13928, 18218, 13030] avg=16384.0000 max=18218 max_rank=6 min=13030 min_rank=7 imbalance_ratio=1.111938 max_load_reduction=112854 max_load_reduction_pct=0.861008 moved_to_echo=62332 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 12672, 15488, 6912, 384, 18048, 128, 9472, 18304, 15104, 128, 128, 9600, 7040, 4224, 14336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[27904, 14336, 12160, 13952, 128, 256, 6144, 128, 0, 1152, 13696, 1280, 256, 512, 25984, 384, 7168, 4864, 1408, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[7680, 0, 2944, 0, 20224, 9216, 4096, 16512, 20864, 640, 1792, 128, 3456, 16640, 128, 27520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[17408, 1280, 4224, 128, 15872, 16000, 3456, 512, 256, 4608, 1664, 4352, 13568, 25088, 3584, 20096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[896, 16512, 1792, 512, 10368, 3456, 256, 11136, 6144, 384, 1792, 4224, 3712, 22272, 128, 3328, 26368, 9984, 6400, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[13568, 4352, 5248, 128, 2304, 8448, 15104, 4224, 18560, 12416, 3072, 5888, 13568, 9344, 15872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[9856, 384, 12672, 4480, 14592, 11520, 4992, 2560, 11904, 12672, 23552, 2816, 5120, 8576, 0, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[768, 6272, 14976, 3072, 8960, 384, 11904, 4864, 3456, 896, 9600, 1664, 1152, 768, 2816, 5504, 30976, 24192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1678, 0, 0, 4080, 0, 35, 0, 54, 3336, 3975, 4090, 0, 3, 0, 1, 0, 4016, 46, 72, 0, 0, 0, 0, 0, 3363, 4096, 3698, 4096, 0, 3, 7, 4096, 0, 2, 889, 0, 0, 0, 0, 0, 0, 10, 4096, 4095, 0, 3310, 13, 0, 1, 1188, 1, 4011, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2373, 4096, 3857, 12, 3667, 0, 2332, 432, 4093, 18, 4096, 3337, 47, 0, 1, 0, 10, 0, 0, 481, 48, 89, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 2790, 4090, 0, 16, 2976, 121, 0, 10, 3139, 4096, 0, 5, 1325, 4089, 17, 4052, 0, 0, 0, 0, 0, 0, 4017, 3189, 0, 0, 0, 0, 0, 0, 488, 292, 0, 4096, 913] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 24, 4095, 0, 0, 2853, 4065, 4092, 0, 0, 0, 0, 0, 432, 1393, 0, 0, 27, 0, 75, 0, 0, 1, 0, 0, 2012, 0, 0, 0, 4018, 36, 4023, 0, 0, 0, 45, 8, 0, 138, 0, 3986, 3912, 0, 4091, 0, 0, 183, 4096, 2, 2202, 6, 146, 3402, 0, 0, 2, 1, 3, 4095, 4096, 713, 4096, 983, 2713, 1272, 3625, 0, 0, 66, 0, 0, 0, 0, 0, 0, 2152, 0, 200, 1, 0, 921, 0, 0, 0, 15, 61, 870, 0, 694, 3421, 0, 520, 171, 0, 0, 1335, 3267, 0, 290, 803, 1, 0, 0, 0, 4087, 0, 4068, 0, 4046, 4094, 0, 1, 4093, 3957, 4079, 4083, 0, 6, 0, 53, 1, 2964, 3515, 0, 4008, 2267] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[27, 19, 118, 0, 3874, 4087, 3969, 4092, 1364, 135, 231, 1156, 1, 0, 0, 0, 4096, 10, 498, 0, 17, 3898, 0, 0, 0, 4096, 0, 4060, 0, 0, 658, 0, 0, 4073, 4076, 0, 0, 3540, 0, 0, 3702, 139, 579, 38, 657, 4095, 1679, 0, 0, 249, 0, 0, 0, 428, 1437, 0, 14, 82, 0, 54, 4096, 4007, 0, 4038, 0, 5, 0, 2840, 0, 0, 0, 0, 204, 0, 0, 0, 3931, 0, 0, 135, 0, 4096, 0, 2677, 2, 0, 0, 745, 4070, 0, 89, 0, 824, 3967, 4096, 0, 953, 41, 3749, 0, 4066, 0, 4, 0, 8, 1, 4, 10, 3371, 5, 4011, 2953, 0, 0, 1, 762, 863, 0, 0, 0, 1014, 2, 0, 0, 4062, 0, 4058, 64] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[2, 39, 0, 0, 379, 0, 115, 0, 2598, 0, 4089, 4093, 276, 654, 4025, 0, 0, 28, 0, 11, 9, 97, 756, 16, 1, 15, 57, 1874, 1, 355, 0, 695, 0, 4096, 3133, 4096, 0, 156, 0, 11, 4084, 0, 0, 4091, 1156, 4088, 0, 0, 3865, 15, 1, 2399, 96, 3398, 3618, 141, 3, 0, 0, 587, 0, 0, 4026, 4023, 0, 4089, 856, 4042, 4083, 0, 1493, 0, 3587, 4094, 2226, 1, 3, 0, 0, 0, 0, 3, 4, 0, 39, 0, 26, 0, 19, 0, 0, 3671, 5, 20, 0, 120, 4090, 3815, 1, 3905, 0, 1011, 12, 2591, 0, 0, 4, 123, 3, 0, 4063, 0, 0, 4092, 0, 3572, 4070, 0, 0, 5, 0, 0, 259, 7, 0, 0, 3800, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 3, 1611, 0, 0, 0, 22, 0, 4061, 39, 0, 0, 3995, 0, 4076, 3, 81, 24, 0, 0, 4091, 3180, 0, 0, 9, 3811, 0, 4084, 0, 48, 1348, 4080, 0, 884, 0, 0, 1, 128, 824, 43, 3836, 0, 2580, 4055, 749, 0, 4056, 438, 955, 1, 0, 2981, 0, 3188, 61, 205, 12, 0, 16, 0, 0, 0, 246, 151, 637, 4079, 0, 0, 2487, 0, 0, 0, 3797, 3821, 0, 492, 0, 0, 0, 34, 404, 4, 1, 4050, 4, 0, 0, 0, 0, 5, 44, 3880, 0, 0, 0, 1283, 3562, 0, 3023, 0, 43, 4096, 4088, 62, 2294, 91, 4094, 0, 2, 0, 4085, 3, 9, 1, 4096, 4077, 4095, 25, 4056, 0, 105, 0, 0, 0, 71, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[2, 19, 0, 268, 4080, 0, 0, 0, 4070, 2292, 0, 21, 19, 92, 0, 0, 1898, 3980, 0, 0, 0, 0, 0, 0, 5, 3207, 31, 3888, 3896, 449, 0, 3844, 0, 4091, 0, 214, 0, 99, 223, 0, 5, 4058, 4021, 4092, 0, 4094, 0, 0, 196, 1783, 1, 3274, 0, 2005, 3433, 0, 0, 0, 3797, 0, 0, 1, 597, 3225, 0, 210, 3984, 3488, 0, 259, 1527, 0, 4072, 947, 2412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 4039, 0, 0, 3660, 0, 1113, 2610, 0, 1937, 506, 0, 290, 0, 0, 607, 77, 0, 4096, 0, 12, 3996, 0, 0, 3754, 4095, 4096, 0, 0, 1, 0, 0, 0, 3912, 0, 2, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[46, 0, 0, 18, 4078, 1, 5, 0, 0, 0, 4093, 3781, 51, 38, 4096, 8, 1, 2010, 0, 0, 53, 0, 162, 733, 0, 0, 23, 2557, 1399, 43, 5, 4096, 0, 0, 4067, 0, 0, 210, 0, 0, 2665, 0, 3865, 4096, 0, 4087, 3852, 73, 0, 323, 0, 1, 0, 1, 0, 0, 1, 0, 632, 2709, 3816, 1739, 0, 1370, 492, 127, 4095, 3576, 0, 0, 0, 0, 10, 3, 0, 0, 0, 594, 991, 38, 0, 0, 232, 0, 4, 1078, 0, 0, 4023, 6, 0, 3415, 0, 0, 0, 0, 0, 0, 4096, 2078, 0, 16, 0, 0, 134, 4059, 3907, 479, 0, 0, 4089, 4096, 0, 12, 3049, 3769, 3403, 4096, 0, 0, 0, 4041, 0, 4049, 1896, 29, 4094, 192] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1, 4091, 0, 666, 4053, 0, 1605, 4091, 4089, 0, 0, 859, 0, 212, 0, 0, 3591, 0, 0, 39, 0, 4068, 0, 0, 3851, 1, 0, 3543, 3875, 0, 18, 3313, 2, 3854, 0, 4, 163, 30, 0, 0, 0, 47, 0, 4096, 0, 4047, 0, 129, 586, 4096, 0, 3110, 0, 120, 420, 0, 0, 0, 297, 0, 0, 23, 0, 4089, 0, 978, 0, 4069, 4096, 2957, 0, 47, 3464, 5, 37, 0, 0, 0, 0, 1695, 4096, 0, 10, 0, 20, 544, 0, 0, 0, 130, 1, 3580, 0, 0, 2408, 0, 4096, 0, 0, 0, 1484, 2273, 1, 0, 0, 0, 0, 0, 3862, 21, 308, 4048, 0, 126, 4070, 3529, 4096, 1642, 30, 1951, 0, 0, 0, 3, 1, 0, 4094, 221] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 24, 4095, 0, 0, 2853, 4065, 4092, 0, 0, 0, 0, 0, 432, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1393, 0, 0, 27, 0, 75, 0, 0, 1, 0, 0, 2012, 0, 0, 0, 4018, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 4023, 0, 0, 0, 45, 8, 0, 138, 0, 3986, 1324, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 4096, 2, 2202, 6, 146, 3402, 0, 0, 2, 1, 3, 4095, 4096, 713, 4096, 1406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 983, 2713, 1272, 3625, 0, 0, 66, 0, 0, 0, 0, 0, 0, 2152, 0, 200, 577, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 921, 0, 0, 0, 15, 61, 870, 0, 694, 3421, 0, 520, 171, 0, 3291, 4046, 605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1335, 3267, 0, 290, 803, 1, 0, 0, 0, 4087, 0, 2255, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 3957, 4079, 4083, 0, 6, 0, 53, 1, 2964, 3515, 0, 717, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16054, 8251, 13651, 24449, 12183, 14616, 16132, 25736] avg=16384.0000 max=25736 max_rank=7 min=8251 min_rank=1 imbalance_ratio=1.570801 max_load_reduction=105336 max_load_reduction_pct=0.803650 moved_to_echo=68667 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 39, 0, 0, 379, 0, 115, 0, 2598, 0, 4089, 4093, 276, 654, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 11, 9, 97, 756, 16, 1, 15, 57, 1874, 1, 355, 0, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3133, 4096, 0, 156, 0, 11, 4084, 0, 0, 1384, 1156, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 15, 1, 2399, 96, 3398, 3618, 141, 3, 0, 0, 587, 0, 0, 4026, 4023, 1471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 856, 4042, 4083, 0, 1493, 0, 3587, 4094, 2226, 1, 3, 0, 0, 0, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 39, 0, 26, 0, 19, 0, 0, 3671, 5, 20, 0, 120, 3120, 4063, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3815, 1, 3905, 0, 1011, 12, 2591, 0, 0, 4, 123, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3572, 4070, 0, 0, 5, 0, 0, 259, 7, 0, 0, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16270, 3915, 22204, 23643, 25077, 11723, 15555, 12685] avg=16384.0000 max=25077 max_rank=4 min=3915 min_rank=1 imbalance_ratio=1.530579 max_load_reduction=105995 max_load_reduction_pct=0.808678 moved_to_echo=65040 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 1611, 0, 0, 0, 22, 0, 4061, 39, 0, 0, 3995, 0, 4076, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 24, 0, 0, 4091, 3180, 0, 0, 9, 3811, 0, 4084, 0, 48, 1348, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 884, 0, 0, 1, 128, 824, 43, 3836, 0, 2580, 1372, 749, 0, 4056, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 955, 1, 0, 2981, 0, 3188, 61, 205, 12, 0, 16, 0, 0, 0, 246, 151, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 637, 4079, 0, 0, 2487, 0, 0, 0, 3797, 3821, 0, 492, 0, 0, 0, 34, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 404, 4, 1, 4050, 4, 0, 0, 0, 0, 5, 44, 3880, 0, 0, 0, 1283, 3363, 4085, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3562, 0, 3023, 0, 43, 4096, 4088, 62, 2294, 91, 4094, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 4096, 4077, 4095, 25, 4056, 0, 105, 0, 0, 0, 71, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13810, 20756, 14911, 9274, 15945, 17750, 21358, 17268] avg=16384.0000 max=21358 max_rank=6 min=9274 min_rank=3 imbalance_ratio=1.303589 max_load_reduction=109714 max_load_reduction_pct=0.837051 moved_to_echo=72321 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1678, 0, 0, 4080, 0, 35, 0, 54, 3336, 3975, 4090, 0, 3, 0, 1, 0, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 46, 72, 0, 0, 0, 0, 0, 3363, 4096, 3698, 4096, 0, 3, 7, 4096, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 889, 0, 0, 0, 0, 0, 0, 10, 4096, 1374, 0, 3310, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1188, 1, 4011, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2373, 4096, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3857, 12, 3667, 0, 2332, 432, 4093, 18, 4096, 3337, 47, 0, 1, 0, 10, 0, 608, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 48, 89, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 2790, 4090, 0, 3368, 0, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2976, 121, 0, 10, 3139, 4096, 0, 5, 1325, 4089, 17, 2236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 3189, 0, 0, 0, 0, 0, 0, 488, 292, 0, 728, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17746, 24219, 9694, 13226, 23106, 15424, 18030, 9627] avg=16384.0000 max=24219 max_rank=1 min=9627 min_rank=7 imbalance_ratio=1.478210 max_load_reduction=106853 max_load_reduction_pct=0.815224 moved_to_echo=66187 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[27, 19, 118, 0, 3874, 4087, 3969, 4092, 1364, 135, 231, 1156, 1, 0, 0, 0, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 498, 0, 17, 3898, 0, 0, 0, 4096, 0, 4060, 0, 0, 658, 0, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4076, 0, 0, 3540, 0, 0, 3702, 139, 579, 15, 657, 4095, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 0, 0, 428, 1437, 0, 14, 82, 0, 54, 4096, 4007, 0, 4038, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2840, 0, 0, 0, 0, 204, 0, 0, 0, 3931, 0, 0, 135, 5, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2677, 2, 0, 0, 745, 4070, 0, 89, 0, 824, 3967, 4096, 0, 3332, 4011, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 41, 3749, 0, 4066, 0, 4, 0, 8, 1, 4, 10, 1868, 5, 0, 2953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 762, 863, 0, 0, 0, 1014, 2, 0, 0, 4062, 0, 726, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19482, 17934, 22555, 14418, 7613, 27914, 13662, 7494] avg=16384.0000 max=27914 max_rank=5 min=7494 min_rank=7 imbalance_ratio=1.703735 max_load_reduction=103158 max_load_reduction_pct=0.787033 moved_to_echo=56683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 4091, 0, 666, 4053, 0, 1605, 4091, 4089, 0, 0, 859, 0, 212, 0, 0, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 0, 0, 39, 0, 4068, 0, 0, 3851, 1, 0, 3543, 3875, 0, 18, 3313, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3854, 0, 4, 163, 30, 0, 0, 0, 47, 0, 1386, 0, 4047, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 586, 4096, 0, 3110, 0, 120, 420, 0, 0, 0, 297, 0, 0, 23, 0, 4089, 1473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 978, 0, 4069, 4096, 2957, 0, 47, 3464, 5, 37, 0, 0, 0, 0, 1695, 604, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 20, 544, 0, 0, 0, 130, 1, 3580, 0, 0, 2408, 0, 3362, 308, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1484, 2273, 1, 0, 0, 0, 0, 0, 2141, 21, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 4070, 3529, 4096, 1642, 30, 1951, 0, 0, 0, 3, 1, 0, 732, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20135, 22987, 9662, 14214, 18517, 15092, 14064, 16401] avg=16384.0000 max=22987 max_rank=1 min=9662 min_rank=2 imbalance_ratio=1.403015 max_load_reduction=108085 max_load_reduction_pct=0.824623 moved_to_echo=64074 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 19, 0, 268, 4080, 0, 0, 0, 4070, 2292, 0, 21, 19, 92, 0, 0, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1898, 3980, 0, 0, 0, 0, 0, 0, 5, 3207, 31, 3888, 3896, 449, 0, 3844, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 214, 0, 99, 223, 0, 5, 4058, 4021, 1385, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 1783, 1, 3274, 0, 2005, 3433, 0, 0, 0, 3797, 0, 0, 1, 597, 3225, 1471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 3984, 3488, 0, 259, 1527, 0, 4072, 947, 2412, 0, 0, 0, 0, 0, 603, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 4039, 0, 0, 3660, 0, 3363, 12, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1113, 2610, 0, 1937, 506, 0, 290, 0, 0, 607, 77, 0, 2270, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3754, 4095, 4096, 0, 0, 1, 0, 0, 0, 3912, 0, 2, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11360, 21928, 18190, 19783, 18101, 11711, 13406, 16593] avg=16384.0000 max=21928 max_rank=1 min=11360 min_rank=0 imbalance_ratio=1.338379 max_load_reduction=109144 max_load_reduction_pct=0.832703 moved_to_echo=59811 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[46, 0, 0, 18, 4078, 1, 5, 0, 0, 0, 4093, 3781, 51, 38, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2010, 0, 0, 53, 0, 162, 733, 0, 0, 23, 2557, 1399, 43, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 210, 0, 0, 2665, 0, 3865, 1386, 0, 4087, 3852, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 1, 0, 1, 0, 0, 1, 0, 632, 2709, 3816, 1739, 0, 1370, 1473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 127, 4095, 3576, 0, 0, 0, 0, 10, 3, 0, 0, 0, 594, 991, 38, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 0, 4, 1078, 0, 0, 4023, 6, 0, 3415, 0, 0, 0, 0, 3362, 4089, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2078, 0, 16, 0, 0, 134, 4059, 3907, 479, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3049, 3769, 3403, 4096, 0, 0, 0, 4041, 0, 4049, 1896, 29, 732, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16215, 11082, 20205, 12065, 10530, 16842, 18865, 25268] avg=16384.0000 max=25268 max_rank=7 min=10530 min_rank=4 imbalance_ratio=1.542236 max_load_reduction=105804 max_load_reduction_pct=0.807220 moved_to_echo=71505 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1792, 4224, 1792, 5120, 20608, 4224, 5760, 11136, 23680, 10624, 12544, 9984, 4352, 1024, 12288, 512, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[5888, 11776, 128, 18048, 256, 9344, 12416, 384, 128, 128, 4864, 3456, 12032, 9984, 8064, 25088, 10368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 21120, 12288, 4352, 256, 4224, 1152, 128, 14464, 4352, 19200, 9728, 2688, 27904, 9600, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[13952, 10880, 14336, 7936, 6400, 11392, 8576, 2688, 2560, 6144, 16384, 640, 10880, 128, 0, 19200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[15104, 6144, 640, 128, 4224, 11392, 1024, 768, 7296, 15232, 3840, 26240, 9216, 1024, 2048, 24192, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[6016, 12288, 13952, 21760, 13056, 3712, 7296, 128, 19328, 12288, 4736, 512, 3968, 2816, 1024, 2176, 4224, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[4608, 4608, 1280, 6912, 128, 1664, 128, 896, 9088, 256, 896, 25984, 896, 7424, 14464, 1408, 26624, 20736, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 4352, 19072, 27904, 27904, 9856, 4096, 2048, 1152, 4096, 384, 11520, 9856, 128, 5888, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[8, 0, 1409, 0, 1961, 255, 4065, 0, 0, 0, 0, 0, 4094, 2746, 0, 0, 0, 0, 3214, 0, 4094, 34, 319, 0, 0, 0, 3789, 0, 4096, 0, 4095, 0, 1595, 180, 3450, 0, 279, 97, 0, 43, 0, 4088, 3, 24, 0, 174, 0, 7, 3194, 0, 1288, 0, 0, 4056, 1027, 1, 0, 0, 0, 937, 0, 0, 13, 3382, 3984, 4, 349, 183, 2, 0, 3804, 0, 0, 0, 2600, 0, 4096, 4018, 0, 0, 1, 4086, 594, 0, 4040, 58, 0, 0, 157, 0, 14, 3910, 4096, 0, 2857, 3714, 0, 3625, 0, 4096, 0, 393, 447, 0, 10, 0, 3531, 901, 0, 0, 562, 385, 1158, 709, 176, 0, 3944, 0, 4073, 0, 10, 0, 0, 2532, 67, 23, 19, 3827] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 2121, 1, 15, 78, 4019, 0, 0, 66, 0, 0, 3, 4083, 1, 0, 0, 0, 4094, 34, 4004, 0, 21, 0, 0, 0, 1183, 0, 4060, 0, 8, 1832, 2, 3965, 4052, 0, 0, 0, 0, 0, 28, 4095, 36, 0, 4094, 3, 12, 27, 0, 218, 2, 0, 1169, 0, 3114, 0, 588, 0, 71, 0, 0, 3, 2417, 0, 119, 4095, 1634, 0, 2670, 0, 3181, 0, 3441, 0, 3821, 0, 0, 0, 4074, 232, 3, 3367, 0, 0, 1159, 3628, 12, 0, 126, 3, 0, 4096, 3928, 0, 0, 4095, 1, 0, 4095, 3274, 0, 4096, 0, 0, 1, 0, 410, 0, 0, 0, 0, 0, 3931, 3484, 618, 0, 400, 2592, 3838, 0, 2, 2655, 0, 4032, 0, 282, 68, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[11, 0, 76, 411, 383, 0, 77, 0, 0, 0, 2371, 3790, 4096, 3748, 3695, 0, 0, 58, 0, 0, 4065, 1, 0, 0, 0, 0, 4084, 0, 4096, 0, 0, 375, 3608, 1792, 406, 1, 0, 23, 0, 1, 898, 2, 0, 0, 0, 4092, 232, 7, 255, 3962, 0, 0, 99, 2098, 0, 4, 0, 0, 4082, 0, 0, 0, 0, 3465, 0, 4096, 37, 0, 3779, 0, 4096, 0, 3936, 284, 0, 0, 0, 0, 0, 1861, 0, 101, 93, 0, 0, 71, 3890, 0, 0, 0, 0, 4096, 2575, 0, 0, 3542, 0, 4009, 4094, 3222, 37, 4096, 4085, 2262, 0, 0, 0, 0, 0, 140, 1692, 0, 4024, 965, 3, 0, 4044, 398, 4027, 74, 0, 0, 0, 4043, 0, 0, 416, 620] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 3258, 4074, 0, 3595, 0, 0, 138, 3384, 2588, 3712, 0, 271, 452, 2, 647, 0, 0, 3619, 176, 4048, 845, 63, 0, 747, 0, 4021, 0, 40, 3059, 4081, 3616, 136, 268, 0, 3273, 6, 7, 4, 68, 4, 168, 0, 0, 0, 356, 4096, 935, 0, 0, 0, 0, 0, 3, 9, 3441, 4065, 46, 0, 41, 0, 3951, 0, 3734, 0, 0, 0, 33, 3853, 3991, 0, 0, 455, 103, 3303, 0, 0, 0, 0, 3814, 0, 0, 0, 444, 0, 19, 2, 3942, 15, 4093, 380, 0, 739, 4094, 0, 4061, 3121, 4096, 88, 4090, 1288, 606, 6, 0, 0, 0, 2, 3973, 0, 0, 2919, 0, 0, 743, 1071, 36, 0, 0, 298, 4096, 227, 8, 0, 0, 15, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 21, 4095, 1207, 4019, 22, 3528, 161, 23, 0, 1994, 3814, 4039, 4096, 0, 0, 0, 0, 0, 157, 3698, 3, 342, 0, 9, 504, 34, 0, 4084, 0, 0, 2386, 2, 46, 3530, 1540, 91, 0, 0, 0, 0, 24, 176, 0, 675, 4, 0, 0, 3957, 4089, 0, 0, 607, 4093, 1756, 0, 0, 0, 2533, 0, 22, 0, 0, 2807, 0, 4010, 0, 1, 0, 0, 4085, 0, 0, 0, 0, 0, 3888, 0, 9, 4055, 0, 4096, 0, 223, 1385, 0, 0, 0, 917, 0, 0, 4096, 0, 0, 370, 4096, 30, 4010, 4096, 0, 3892, 4063, 4096, 0, 6, 0, 0, 22, 2, 0, 0, 206, 3183, 1313, 147, 0, 1884, 0, 149, 562, 0, 0, 0, 4067, 0, 0, 0, 3925] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 753, 0, 1079, 31, 0, 4096, 16, 131, 0, 0, 1, 1232, 4044, 0, 2, 0, 0, 0, 2029, 4091, 0, 1, 0, 229, 4091, 4096, 12, 3927, 388, 4094, 3635, 3985, 0, 3895, 0, 0, 0, 2793, 0, 2992, 946, 0, 66, 0, 4096, 0, 2, 3816, 91, 0, 0, 0, 94, 0, 0, 0, 513, 2603, 0, 1546, 0, 0, 4022, 3203, 3057, 0, 5, 16, 39, 684, 2402, 0, 5, 3931, 13, 0, 0, 164, 1336, 0, 0, 0, 4005, 0, 1, 29, 3263, 0, 125, 0, 3222, 0, 0, 3, 4022, 0, 3420, 134, 26, 0, 33, 186, 74, 59, 0, 0, 249, 43, 0, 0, 4093, 4076, 692, 1016, 18, 1957, 0, 3910, 0, 0, 4096, 0, 4096, 0, 3931, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[211, 70, 34, 0, 0, 0, 0, 36, 0, 0, 0, 4, 3556, 4096, 571, 0, 2, 17, 0, 4084, 4096, 830, 0, 0, 0, 0, 4094, 7, 3809, 0, 1, 3786, 8, 3963, 4096, 0, 0, 8, 0, 0, 0, 3011, 1909, 11, 0, 2119, 4070, 0, 0, 2522, 4072, 2, 0, 150, 0, 0, 0, 0, 3972, 5, 18, 0, 1336, 2059, 2168, 2073, 0, 0, 3, 1, 1514, 0, 4058, 0, 0, 2969, 0, 15, 27, 2916, 0, 0, 0, 135, 4096, 4005, 2030, 2730, 3536, 0, 0, 4020, 0, 328, 4093, 2660, 0, 4022, 62, 0, 3818, 4095, 49, 648, 0, 0, 9, 0, 0, 1, 1388, 0, 697, 0, 74, 520, 3623, 10, 4072, 113, 0, 210, 0, 0, 863, 786, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 3916, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 4096, 3771, 3481, 135, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 1, 3989, 0, 0, 4064, 4096, 4078, 0, 0, 0, 0, 0, 0, 2208, 4085, 27, 5, 4024, 3, 0, 6, 3085, 672, 0, 4082, 11, 4070, 0, 0, 4, 0, 4096, 33, 0, 123, 3879, 2710, 0, 4077, 0, 1887, 209, 4086, 0, 0, 0, 0, 4096, 1401, 0, 0, 0, 429, 4085, 0, 0, 17, 1, 5, 42, 4, 3976, 129, 4, 31, 1315, 0, 0, 4075, 0, 1, 9, 0, 0, 137, 0, 0, 0, 0, 4093, 0, 0, 260, 117, 0, 3449, 0, 2818, 288, 3846, 80, 1859, 0, 0, 0, 6, 3367, 16, 40, 8, 2094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 1409, 0, 1961, 255, 4065, 0, 0, 0, 0, 0, 4094, 2289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3214, 0, 4094, 34, 319, 0, 0, 0, 3789, 0, 3299, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1595, 180, 3450, 0, 279, 97, 0, 43, 0, 4088, 3, 24, 0, 174, 0, 7, 1955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3194, 0, 1288, 0, 0, 4056, 1027, 1, 0, 0, 0, 937, 0, 0, 13, 3382, 659, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 4, 349, 183, 2, 0, 3804, 0, 0, 0, 2600, 0, 4096, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 594, 0, 4040, 58, 0, 0, 157, 0, 14, 3910, 4096, 0, 2857, 1759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3625, 0, 4096, 0, 393, 447, 0, 10, 0, 3531, 901, 0, 0, 562, 385, 362, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 796, 709, 176, 0, 3944, 0, 4073, 0, 10, 0, 0, 2532, 67, 23, 19, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14081, 18844, 11895, 15014, 19040, 21572, 14450, 16176] avg=16384.0000 max=21572 max_rank=5 min=11895 min_rank=2 imbalance_ratio=1.316650 max_load_reduction=109500 max_load_reduction_pct=0.835419 moved_to_echo=71238 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 3258, 4074, 0, 3595, 0, 0, 138, 3384, 2588, 3712, 0, 271, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 647, 0, 0, 3619, 176, 4048, 845, 63, 0, 747, 0, 3239, 0, 40, 3059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 3616, 136, 268, 0, 3273, 6, 7, 4, 68, 4, 168, 0, 0, 0, 356, 2155, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 935, 0, 0, 0, 0, 0, 3, 9, 3441, 4065, 46, 0, 41, 0, 3951, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2602, 0, 0, 0, 33, 3853, 3991, 0, 0, 455, 103, 3303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3814, 0, 0, 0, 444, 0, 19, 2, 3942, 15, 4093, 380, 0, 739, 1939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 3121, 4096, 88, 4090, 1288, 606, 6, 0, 0, 0, 2, 3973, 0, 0, 914, 653, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2005, 0, 0, 743, 1071, 36, 0, 0, 298, 4096, 227, 8, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21473, 16485, 14621, 17234, 14340, 15387, 23033, 8499] avg=16384.0000 max=23033 max_rank=6 min=8499 min_rank=7 imbalance_ratio=1.405823 max_load_reduction=108039 max_load_reduction_pct=0.824272 moved_to_echo=61259 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2121, 1, 15, 78, 4019, 0, 0, 66, 0, 0, 3, 3404, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 34, 4004, 0, 21, 0, 0, 0, 1183, 0, 3271, 0, 8, 1832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3965, 4052, 0, 0, 0, 0, 0, 28, 4095, 36, 0, 4094, 3, 12, 27, 2156, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 2, 0, 1169, 0, 3114, 0, 588, 0, 71, 0, 0, 3, 2417, 0, 653, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 2854, 1634, 0, 2670, 0, 3181, 0, 3441, 0, 3821, 0, 0, 0, 4074, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3367, 0, 0, 1159, 3628, 12, 0, 126, 3, 0, 4096, 3928, 0, 0, 1939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 3274, 0, 4096, 0, 0, 1, 0, 410, 0, 0, 0, 0, 0, 1232, 716, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2699, 3484, 618, 0, 400, 2592, 3838, 0, 2, 2655, 0, 4032, 0, 282, 68, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9708, 14447, 18995, 8914, 22026, 18261, 13961, 24760] avg=16384.0000 max=24760 max_rank=7 min=8914 min_rank=3 imbalance_ratio=1.511230 max_load_reduction=106312 max_load_reduction_pct=0.811096 moved_to_echo=79008 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 3916, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 4096, 3144, 3481, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 1, 3213, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4078, 0, 0, 0, 0, 0, 0, 2208, 4085, 27, 5, 4024, 3, 0, 6, 2145, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3085, 672, 0, 4082, 11, 4070, 0, 0, 4, 0, 4096, 33, 0, 123, 3879, 2710, 642, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2841, 0, 1887, 209, 4086, 0, 0, 0, 0, 4096, 1401, 0, 0, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 17, 1, 5, 42, 4, 3976, 129, 4, 31, 1315, 0, 0, 1930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 0, 0, 137, 0, 0, 0, 0, 4093, 0, 0, 260, 117, 0, 1081, 713, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2368, 0, 2818, 288, 3846, 80, 1859, 0, 0, 0, 6, 3367, 16, 40, 8, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22833, 13182, 21200, 24034, 14949, 11539, 6545, 16790] avg=16384.0000 max=24034 max_rank=3 min=6545 min_rank=6 imbalance_ratio=1.466919 max_load_reduction=107038 max_load_reduction_pct=0.816635 moved_to_echo=49823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 76, 411, 383, 0, 77, 0, 0, 0, 2371, 3790, 4096, 3120, 3695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 4065, 1, 0, 0, 0, 0, 4084, 0, 3294, 0, 0, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3608, 1792, 406, 1, 0, 23, 0, 1, 898, 2, 0, 0, 0, 4092, 232, 7, 1868, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 3962, 0, 0, 99, 2098, 0, 4, 0, 0, 4082, 0, 0, 0, 0, 3465, 661, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2847, 37, 0, 3779, 0, 4096, 0, 3936, 284, 0, 0, 0, 0, 0, 1861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 93, 0, 0, 71, 3890, 0, 0, 0, 0, 4096, 2575, 0, 0, 1674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 4094, 3222, 37, 4096, 4085, 2262, 0, 0, 0, 0, 0, 140, 1692, 0, 1265, 721, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2759, 965, 3, 0, 4044, 398, 4027, 74, 0, 0, 0, 4043, 0, 0, 416, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18030, 11877, 13458, 15254, 16840, 12500, 25764, 17349] avg=16384.0000 max=25764 max_rank=6 min=11877 min_rank=1 imbalance_ratio=1.572510 max_load_reduction=105308 max_load_reduction_pct=0.803436 moved_to_echo=72453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 753, 0, 1079, 31, 0, 4096, 16, 131, 0, 0, 1, 1232, 3371, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2029, 4091, 0, 1, 0, 229, 4091, 4096, 12, 3163, 388, 4094, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 3895, 0, 0, 0, 2793, 0, 2992, 946, 0, 66, 0, 4096, 0, 2, 2117, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3816, 91, 0, 0, 0, 94, 0, 0, 0, 513, 2603, 0, 1546, 0, 0, 4022, 632, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3203, 2130, 0, 5, 16, 39, 684, 2402, 0, 5, 3931, 13, 0, 0, 164, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 1, 29, 3263, 0, 125, 0, 3222, 0, 0, 3, 1905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3420, 134, 26, 0, 33, 186, 74, 59, 0, 0, 249, 43, 0, 0, 4093, 1277, 535, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2799, 692, 1016, 18, 1957, 0, 3910, 0, 0, 4096, 0, 4096, 0, 3931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10712, 25829, 21284, 13990, 13928, 12553, 10261, 22515] avg=16384.0000 max=25829 max_rank=1 min=10261 min_rank=6 imbalance_ratio=1.576477 max_load_reduction=105243 max_load_reduction_pct=0.802940 moved_to_echo=59257 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[211, 70, 34, 0, 0, 0, 0, 36, 0, 0, 0, 4, 3556, 3415, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 17, 0, 4084, 4096, 830, 0, 0, 0, 0, 4094, 7, 3068, 0, 1, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3963, 4096, 0, 0, 8, 0, 0, 0, 3011, 1909, 11, 0, 2119, 4070, 0, 1400, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2522, 4072, 2, 0, 150, 0, 0, 0, 0, 3972, 5, 18, 0, 1336, 2059, 613, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2168, 1445, 0, 0, 3, 1, 1514, 0, 4058, 0, 0, 2969, 0, 15, 27, 2916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 4096, 4005, 2030, 2730, 3536, 0, 0, 4020, 0, 328, 4093, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 62, 0, 3818, 4095, 49, 648, 0, 0, 9, 0, 0, 1, 1388, 0, 218, 362, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 0, 74, 520, 3623, 10, 4072, 113, 0, 210, 0, 0, 863, 786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7897, 19985, 20861, 15430, 15116, 26233, 14800, 10750] avg=16384.0000 max=26233 max_rank=5 min=7897 min_rank=0 imbalance_ratio=1.601135 max_load_reduction=104839 max_load_reduction_pct=0.799858 moved_to_echo=66899 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 21, 4095, 1207, 4019, 22, 3528, 161, 23, 0, 1994, 3814, 4039, 3415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 3698, 3, 342, 0, 9, 504, 34, 0, 3290, 0, 0, 2386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 46, 3530, 1540, 91, 0, 0, 0, 0, 24, 176, 0, 675, 4, 0, 0, 2156, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3957, 4089, 0, 0, 607, 4093, 1756, 0, 0, 0, 2533, 0, 22, 0, 0, 2807, 657, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2795, 0, 1, 0, 0, 4085, 0, 0, 0, 0, 0, 3888, 0, 9, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 223, 1385, 0, 0, 0, 917, 0, 0, 4096, 0, 0, 370, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4010, 4096, 0, 3892, 4063, 4096, 0, 6, 0, 0, 22, 2, 0, 0, 206, 997, 701, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2186, 1313, 147, 0, 1884, 0, 149, 562, 0, 0, 0, 4067, 0, 0, 0, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[26338, 10423, 8758, 21202, 14833, 13027, 22258, 14233] avg=16384.0000 max=26338 max_rank=0 min=8758 min_rank=2 imbalance_ratio=1.607544 max_load_reduction=104734 max_load_reduction_pct=0.799057 moved_to_echo=64351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 4864, 7424, 6400, 27776, 1152, 4736, 896, 384, 4608, 19968, 128, 25856, 512, 8320, 19200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 23168, 15616, 14720, 7936, 21120, 10240, 3712, 128, 0, 8064, 1280, 128, 4480, 3840, 4736, 7424, 4480, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[18432, 12544, 5376, 4096, 1920, 14592, 6016, 128, 640, 3968, 21504, 1024, 1664, 256, 7680, 22400, 5248, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[4096, 15488, 768, 4480, 10752, 8320, 6016, 6016, 8832, 4224, 128, 27648, 12416, 384, 8064, 14464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9600, 17536, 2048, 2176, 6784, 4224, 21248, 6400, 11520, 384, 14976, 4608, 11392, 4096, 4352, 10880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[4352, 896, 11776, 6016, 14464, 384, 19456, 256, 256, 256, 7808, 10240, 24832, 22272, 8064, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[17408, 17664, 19584, 1920, 384, 3456, 2816, 128, 6144, 16384, 2176, 384, 8832, 10496, 4352, 512, 16000, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[16128, 7168, 4864, 1664, 20864, 3200, 22016, 768, 384, 11136, 256, 22272, 1024, 5120, 640, 14592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True + [2026-08-30 05:04:57] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 989.2 | throughput per GPU (TFLOP/s/GPU): 337.0 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101839E+01 | loss scale: 1.0 | grad norm: 45.880 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[113, 2340, 660, 76, 2333, 86, 758, 279, 780, 69, 1461, 241, 857, 639, 48, 404, 133, 746, 674, 889, 1533, 329, 2647, 1184, 45, 1647, 391, 546, 866, 193, 1392, 1894, 3582, 290, 117, 613, 161, 898, 1821, 916, 2338, 1455, 1093, 393, 552, 1864, 1380, 843, 314, 181, 915, 1153, 474, 398, 1667, 188, 208, 841, 2515, 2614, 737, 261, 321, 365, 736, 3081, 296, 1243, 928, 369, 1165, 2026, 781, 1920, 172, 1128, 853, 1589, 2727, 1291, 2450, 1227, 2536, 1641, 1227, 336, 653, 3045, 1094, 231, 891, 1513, 1008, 495, 449, 479, 1622, 258, 2866, 784, 845, 790, 1657, 2161, 1396, 1297, 582, 1121, 626, 543, 1209, 1497, 325, 1036, 515, 885, 139, 746, 1140, 1715, 993, 1532, 1813, 1143, 1401, 369, 525, 240] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[48, 2392, 861, 141, 2337, 34, 486, 294, 1027, 90, 1398, 580, 843, 757, 88, 424, 335, 732, 1605, 1200, 1446, 110, 2958, 1083, 94, 1461, 306, 654, 1176, 312, 1352, 1919, 3041, 272, 350, 217, 407, 1035, 1299, 1116, 1927, 1309, 1752, 751, 899, 1175, 884, 1125, 250, 300, 306, 1040, 187, 221, 1911, 136, 117, 1045, 1503, 2620, 646, 378, 647, 487, 837, 2784, 204, 1346, 1142, 360, 1068, 1908, 947, 2094, 132, 1126, 964, 1364, 2586, 776, 2012, 1236, 2126, 1161, 1623, 791, 833, 2354, 725, 344, 673, 1482, 970, 605, 465, 185, 2119, 250, 1473, 970, 849, 1275, 1840, 2609, 943, 1499, 704, 1944, 230, 486, 1193, 1686, 209, 1775, 1098, 1285, 212, 884, 875, 1243, 925, 1666, 1954, 976, 1886, 249, 374, 272] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[57, 2548, 1478, 172, 2233, 34, 726, 341, 1112, 163, 1454, 271, 757, 650, 45, 377, 137, 565, 726, 1333, 1471, 456, 2341, 1280, 136, 1731, 546, 746, 1526, 249, 1831, 2014, 3122, 160, 534, 560, 160, 911, 1623, 657, 2350, 1841, 1673, 683, 430, 1467, 1009, 679, 300, 299, 441, 1379, 312, 420, 1312, 65, 70, 822, 2409, 2115, 615, 191, 1064, 498, 662, 3506, 323, 1651, 726, 209, 970, 2117, 442, 1770, 170, 526, 736, 2120, 1717, 1130, 2217, 489, 2222, 996, 1870, 397, 1049, 2097, 1228, 179, 772, 998, 906, 575, 525, 358, 2065, 384, 2184, 754, 818, 979, 1950, 2603, 1019, 1369, 612, 1649, 410, 359, 1178, 1594, 146, 1288, 1147, 1467, 167, 783, 1224, 1545, 962, 1005, 1880, 1218, 1335, 657, 699, 292] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[46, 2712, 1484, 113, 1842, 77, 661, 459, 765, 144, 978, 232, 1336, 604, 28, 413, 192, 596, 1054, 899, 1892, 125, 2608, 996, 81, 1496, 827, 858, 1527, 372, 1173, 2012, 3364, 245, 136, 408, 100, 425, 1817, 1067, 3281, 1637, 961, 386, 737, 1639, 1090, 611, 329, 372, 247, 1397, 574, 459, 2128, 78, 98, 621, 2009, 1735, 686, 234, 799, 465, 838, 2643, 200, 1529, 813, 162, 700, 1980, 606, 2363, 548, 1532, 697, 1524, 2150, 899, 1671, 1084, 2062, 1590, 1603, 502, 911, 2237, 1011, 213, 523, 1356, 1482, 516, 486, 594, 1696, 529, 2494, 514, 758, 1146, 2045, 2789, 1143, 1768, 636, 1975, 365, 582, 1181, 1316, 138, 1343, 794, 1360, 132, 818, 1138, 1547, 546, 1639, 1718, 690, 1492, 211, 717, 90] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[120, 2312, 1560, 136, 1874, 35, 727, 422, 959, 92, 1678, 613, 1104, 662, 57, 407, 179, 712, 1162, 830, 1906, 255, 2501, 1072, 106, 1672, 588, 933, 1218, 394, 1540, 1869, 3079, 277, 348, 473, 153, 771, 1726, 631, 2654, 1098, 1197, 870, 577, 1710, 1072, 522, 490, 307, 342, 1239, 294, 245, 1420, 70, 63, 1006, 2011, 2131, 963, 419, 717, 449, 710, 2992, 186, 1592, 721, 176, 1200, 1707, 445, 1647, 211, 878, 819, 1833, 2810, 829, 2211, 1144, 2021, 1277, 1950, 551, 1070, 1803, 920, 350, 938, 1427, 852, 914, 526, 560, 2070, 695, 2134, 498, 499, 860, 1432, 2750, 1001, 1465, 616, 1602, 237, 286, 1178, 1701, 255, 1598, 659, 825, 208, 920, 1432, 1170, 922, 1560, 1921, 1121, 1914, 233, 791, 230] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[78, 2714, 1108, 98, 1596, 19, 746, 317, 1013, 246, 1508, 299, 1290, 823, 63, 407, 167, 503, 1099, 549, 1902, 343, 1978, 1256, 109, 1521, 521, 901, 1357, 378, 1541, 1852, 3104, 307, 411, 751, 223, 1110, 1382, 625, 2410, 1261, 1224, 353, 519, 1490, 1128, 566, 262, 199, 340, 1327, 673, 383, 1901, 82, 55, 1285, 2055, 1950, 1038, 255, 856, 608, 865, 2917, 160, 1868, 917, 123, 544, 1753, 168, 1956, 310, 801, 747, 1460, 2193, 1057, 2066, 1337, 2337, 1079, 1684, 533, 863, 1892, 1136, 227, 872, 1215, 1279, 608, 401, 366, 2020, 930, 2470, 558, 752, 1150, 2186, 2896, 1128, 1621, 506, 1837, 254, 488, 2115, 1049, 203, 1212, 892, 1202, 78, 834, 1689, 1133, 774, 1325, 1537, 876, 1852, 217, 958, 161] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[21, 2188, 1167, 66, 2028, 43, 825, 249, 624, 85, 1226, 258, 979, 806, 38, 404, 110, 984, 557, 482, 1460, 391, 2538, 1036, 92, 1941, 770, 678, 952, 284, 888, 2067, 3388, 272, 188, 684, 192, 880, 1441, 1154, 2745, 1020, 934, 349, 734, 1741, 1579, 833, 335, 143, 253, 1190, 349, 229, 1789, 55, 281, 429, 1846, 2531, 620, 336, 551, 476, 875, 2921, 38, 1896, 1019, 120, 953, 2423, 593, 2660, 220, 628, 659, 1813, 2589, 1120, 2376, 605, 2503, 1342, 1587, 333, 541, 1824, 956, 188, 712, 1978, 1161, 623, 359, 448, 2079, 443, 2593, 478, 561, 1023, 1481, 2812, 1038, 1388, 785, 1589, 390, 789, 1754, 1222, 297, 1493, 1166, 578, 123, 959, 1256, 1367, 773, 2390, 2225, 982, 1815, 233, 1027, 116] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[45, 2276, 1417, 140, 1929, 62, 714, 440, 904, 116, 1927, 428, 652, 865, 65, 568, 166, 1021, 980, 1066, 1590, 257, 2454, 1021, 52, 1587, 276, 888, 1265, 183, 1617, 2040, 3418, 248, 390, 594, 400, 998, 1395, 725, 2514, 1419, 1448, 926, 712, 1686, 1076, 687, 339, 372, 334, 1381, 576, 248, 2107, 88, 228, 576, 2027, 2561, 516, 298, 756, 459, 875, 2652, 160, 1546, 1126, 207, 1139, 1840, 474, 2038, 208, 621, 702, 1821, 2762, 865, 2000, 1077, 2159, 1120, 1730, 671, 972, 2567, 837, 234, 619, 1094, 844, 572, 512, 208, 1910, 212, 2242, 722, 656, 1064, 1795, 2653, 828, 1212, 903, 1624, 333, 408, 2114, 1529, 449, 1216, 585, 1568, 88, 911, 1101, 867, 765, 1002, 1740, 900, 1771, 172, 507, 460] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[113, 2340, 660, 76, 2333, 86, 758, 279, 780, 69, 1461, 241, 857, 639, 48, 404, 3014, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 746, 674, 889, 1533, 329, 2647, 1184, 45, 1647, 391, 546, 866, 193, 1392, 1894, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2025, 290, 117, 613, 161, 898, 1821, 916, 2338, 1455, 1093, 393, 552, 1864, 1380, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 181, 915, 1153, 474, 398, 1667, 188, 208, 841, 2515, 2614, 737, 261, 321, 365, 1647, 1525, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 736, 67, 296, 1243, 928, 369, 1165, 2026, 781, 1920, 172, 1128, 853, 1589, 2727, 1291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2450, 1227, 889, 1641, 1227, 336, 653, 3045, 1094, 231, 891, 1513, 1008, 495, 449, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1622, 258, 1399, 784, 845, 790, 1657, 0, 1396, 1297, 582, 1121, 626, 543, 1209, 1497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 1036, 515, 885, 139, 746, 1140, 1715, 993, 1532, 1813, 1143, 1401, 369, 525, 240, 1341, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15396, 15235, 16759, 17247, 17291, 17628, 15626, 15890] avg=16384.0000 max=17628 max_rank=5 min=15235 min_rank=1 imbalance_ratio=1.075928 max_load_reduction=113444 max_load_reduction_pct=0.865509 moved_to_echo=66435 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[46, 2712, 1484, 113, 1842, 77, 661, 459, 765, 144, 978, 232, 1336, 604, 28, 413, 2585, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 596, 1054, 899, 1892, 125, 2608, 996, 81, 1496, 827, 858, 1527, 372, 1173, 2012, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1901, 245, 136, 408, 100, 425, 1817, 1067, 3281, 1637, 961, 386, 737, 1639, 1090, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 372, 247, 1397, 574, 459, 2128, 78, 98, 621, 2009, 1735, 686, 234, 799, 465, 1339, 1433, 1190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 838, 58, 200, 1529, 813, 162, 700, 1980, 606, 2363, 548, 1532, 697, 1524, 2150, 899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1671, 1084, 723, 1590, 1603, 502, 911, 2237, 1011, 213, 523, 1356, 1482, 516, 486, 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1696, 529, 1217, 514, 758, 1146, 2045, 0, 1143, 1768, 636, 1975, 365, 582, 1181, 1316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 1343, 794, 1360, 132, 818, 1138, 1547, 546, 1639, 1718, 690, 1492, 211, 717, 90, 1167, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16078, 16818, 16441, 16193, 16599, 16502, 16871, 15570] avg=16384.0000 max=16871 max_rank=6 min=15570 min_rank=7 imbalance_ratio=1.029724 max_load_reduction=114201 max_load_reduction_pct=0.871284 moved_to_echo=65542 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[57, 2548, 1478, 172, 2233, 34, 726, 341, 1112, 163, 1454, 271, 757, 650, 45, 377, 3430, 1491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 565, 726, 1333, 1471, 456, 2341, 1280, 136, 1731, 546, 746, 1526, 249, 1831, 2014, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1764, 160, 534, 560, 160, 911, 1623, 657, 2350, 1841, 1673, 683, 430, 1467, 1009, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 299, 441, 1379, 312, 420, 1312, 65, 70, 822, 2409, 2115, 615, 191, 1064, 498, 1443, 1330, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 76, 323, 1651, 726, 209, 970, 2117, 442, 1770, 170, 526, 736, 2120, 1717, 1130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2217, 489, 779, 996, 1870, 397, 1049, 2097, 1228, 179, 772, 998, 906, 575, 525, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2065, 384, 1066, 754, 818, 979, 1950, 0, 1019, 1369, 612, 1649, 410, 359, 1178, 1594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 1288, 1147, 1467, 167, 783, 1224, 1545, 962, 1005, 1880, 1218, 1335, 657, 699, 292, 1022, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17339, 17184, 16501, 16197, 15345, 15435, 16206, 16865] avg=16384.0000 max=17339 max_rank=0 min=15345 min_rank=4 imbalance_ratio=1.058289 max_load_reduction=113733 max_load_reduction_pct=0.867714 moved_to_echo=63851 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[48, 2392, 861, 141, 2337, 34, 486, 294, 1027, 90, 1398, 580, 843, 757, 88, 424, 2723, 1495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 732, 1605, 1200, 1446, 110, 2958, 1083, 94, 1461, 306, 654, 1176, 312, 1352, 1919, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1719, 272, 350, 217, 407, 1035, 1299, 1116, 1927, 1309, 1752, 751, 899, 1175, 884, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 300, 306, 1040, 187, 221, 1911, 136, 117, 1045, 1503, 2620, 646, 378, 647, 487, 1380, 1295, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 837, 61, 204, 1346, 1142, 360, 1068, 1908, 947, 2094, 132, 1126, 964, 1364, 2586, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2012, 1236, 746, 1161, 1623, 791, 833, 2354, 725, 344, 673, 1482, 970, 605, 465, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2119, 250, 719, 970, 849, 1275, 1840, 0, 943, 1499, 704, 1944, 230, 486, 1193, 1686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 1775, 1098, 1285, 212, 884, 875, 1243, 925, 1666, 1954, 976, 1886, 249, 374, 272, 689, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16018, 16808, 16237, 15583, 16915, 16205, 16707, 16599] avg=16384.0000 max=16915 max_rank=4 min=15583 min_rank=3 imbalance_ratio=1.032410 max_load_reduction=114157 max_load_reduction_pct=0.870949 moved_to_echo=66426 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[120, 2312, 1560, 136, 1874, 35, 727, 422, 959, 92, 1678, 613, 1104, 662, 57, 407, 2932, 1576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 712, 1162, 830, 1906, 255, 2501, 1072, 106, 1672, 588, 933, 1218, 394, 1540, 1869, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1732, 277, 348, 473, 153, 771, 1726, 631, 2654, 1098, 1197, 870, 577, 1710, 1072, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 307, 342, 1239, 294, 245, 1420, 70, 63, 1006, 2011, 2131, 963, 419, 717, 449, 1315, 1314, 1174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 710, 60, 186, 1592, 721, 176, 1200, 1707, 445, 1647, 211, 878, 819, 1833, 2810, 829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2211, 1144, 706, 1277, 1950, 551, 1070, 1803, 920, 350, 938, 1427, 852, 914, 526, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2070, 695, 1033, 498, 499, 860, 1432, 0, 1001, 1465, 616, 1602, 237, 286, 1178, 1701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 1598, 659, 825, 208, 920, 1432, 1170, 922, 1560, 1921, 1121, 1914, 233, 791, 230, 1003, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17266, 17035, 15811, 15969, 15824, 17199, 15173, 16795] avg=16384.0000 max=17266 max_rank=0 min=15173 min_rank=6 imbalance_ratio=1.053833 max_load_reduction=113806 max_load_reduction_pct=0.868271 moved_to_echo=64991 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[21, 2188, 1167, 66, 2028, 43, 825, 249, 624, 85, 1226, 258, 979, 806, 38, 404, 2857, 1611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 984, 557, 482, 1460, 391, 2538, 1036, 92, 1941, 770, 678, 952, 284, 888, 2067, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1914, 272, 188, 684, 192, 880, 1441, 1154, 2745, 1020, 934, 349, 734, 1741, 1579, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 143, 253, 1190, 349, 229, 1789, 55, 281, 429, 1846, 2531, 620, 336, 551, 476, 1625, 1443, 1201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 875, 64, 38, 1896, 1019, 120, 953, 2423, 593, 2660, 220, 628, 659, 1813, 2589, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2376, 605, 878, 1342, 1587, 333, 541, 1824, 956, 188, 712, 1978, 1161, 623, 359, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2079, 443, 1266, 478, 561, 1023, 1481, 0, 1038, 1388, 785, 1589, 390, 789, 1754, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 1493, 1166, 578, 123, 959, 1256, 1367, 773, 2390, 2225, 982, 1815, 233, 1027, 116, 1213, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15475, 15344, 16660, 15682, 17670, 15911, 16286, 18044] avg=16384.0000 max=18044 max_rank=7 min=15344 min_rank=1 imbalance_ratio=1.101318 max_load_reduction=113028 max_load_reduction_pct=0.862335 moved_to_echo=67911 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[78, 2714, 1108, 98, 1596, 19, 746, 317, 1013, 246, 1508, 299, 1290, 823, 63, 407, 2853, 1660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 503, 1099, 549, 1902, 343, 1978, 1256, 109, 1521, 521, 901, 1357, 378, 1541, 1852, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1754, 307, 411, 751, 223, 1110, 1382, 625, 2410, 1261, 1224, 353, 519, 1490, 1128, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 199, 340, 1327, 673, 383, 1901, 82, 55, 1285, 2055, 1950, 1038, 255, 856, 608, 1517, 1322, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 865, 64, 160, 1868, 917, 123, 544, 1753, 168, 1956, 310, 801, 747, 1460, 2193, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2066, 1337, 820, 1079, 1684, 533, 863, 1892, 1136, 227, 872, 1215, 1279, 608, 401, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2020, 930, 1205, 558, 752, 1150, 2186, 0, 1128, 1621, 506, 1837, 254, 488, 2115, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 1212, 892, 1202, 78, 834, 1689, 1133, 774, 1325, 1537, 876, 1852, 217, 958, 161, 1156, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16838, 16086, 15514, 17344, 14986, 16378, 17799, 16127] avg=16384.0000 max=17799 max_rank=6 min=14986 min_rank=4 imbalance_ratio=1.086365 max_load_reduction=113273 max_load_reduction_pct=0.864204 moved_to_echo=65290 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[45, 2276, 1417, 140, 1929, 62, 714, 440, 904, 116, 1927, 428, 652, 865, 65, 568, 2594, 1520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 1021, 980, 1066, 1590, 257, 2454, 1021, 52, 1587, 276, 888, 1265, 183, 1617, 2040, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1931, 248, 390, 594, 400, 998, 1395, 725, 2514, 1419, 1448, 926, 712, 1686, 1076, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 372, 334, 1381, 576, 248, 2107, 88, 228, 576, 2027, 2561, 516, 298, 756, 459, 1402, 1456, 1133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 875, 58, 160, 1546, 1126, 207, 1139, 1840, 474, 2038, 208, 621, 702, 1821, 2762, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2000, 1077, 757, 1120, 1730, 671, 972, 2567, 837, 234, 619, 1094, 844, 572, 512, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1910, 212, 1094, 722, 656, 1064, 1795, 0, 828, 1212, 903, 1624, 333, 408, 2114, 1529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 1216, 585, 1568, 88, 911, 1101, 867, 765, 1002, 1740, 900, 1771, 172, 507, 460, 1049, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16662, 16562, 17149, 16857, 16442, 15814, 16404, 15182] avg=16384.0000 max=17149 max_rank=2 min=15182 min_rank=7 imbalance_ratio=1.046692 max_load_reduction=113923 max_load_reduction_pct=0.869164 moved_to_echo=63842 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[15616, 3712, 9088, 5376, 5760, 8320, 14464, 0, 8576, 11648, 5376, 13440, 2944, 3968, 12032, 11648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[2048, 11008, 6912, 9216, 1152, 6912, 9856, 10624, 6784, 12160, 14848, 7936, 13568, 2432, 5632, 1920, 8704, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1536, 5888, 7936, 7296, 13312, 2304, 20096, 8960, 768, 13056, 4352, 6272, 9984, 2432, 11392, 15744, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[640, 19584, 9856, 1024, 16256, 512, 5760, 2816, 7296, 1024, 11648, 2944, 7936, 5888, 512, 3456, 23040, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[17024, 8320, 6400, 10240, 13312, 4224, 6912, 17920, 7936, 2048, 6016, 11136, 8576, 4992, 3840, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[2688, 2176, 3200, 10112, 3456, 2688, 14336, 768, 1152, 6656, 16384, 18304, 5888, 2432, 5760, 3840, 11776, 11136, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[14848, 2176, 2560, 4352, 1920, 7040, 12544, 6912, 20224, 11136, 10368, 4736, 5248, 12800, 9344, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[6400, 512, 1664, 12672, 7424, 1792, 7808, 15872, 4480, 16512, 2048, 7296, 6272, 13568, 19584, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3145, 0, 3961, 0, 0, 1353, 97, 6, 0, 292, 4034, 3581, 0, 44, 5, 0, 0, 6, 58, 4072, 1, 0, 556, 0, 5, 2, 5, 3816, 1037, 194, 3764, 0, 978, 0, 2, 12, 4086, 2, 124, 3, 94, 0, 0, 1, 0, 4, 0, 4093, 3403, 0, 0, 0, 0, 0, 4092, 4096, 0, 13, 3780, 0, 2, 1, 0, 2998, 0, 3855, 3, 27, 395, 0, 302, 4093, 5, 0, 165, 0, 3215, 0, 306, 1894, 27, 3697, 4096, 4092, 2044, 4084, 3968, 0, 0, 0, 9, 672, 1956, 33, 68, 3799, 4096, 1, 3494, 0, 22, 1, 0, 13, 1, 1, 2571, 0, 1, 0, 2012, 0, 9, 4, 1, 3377, 2, 3756, 797, 236, 0, 2716, 1015, 2265, 0, 4058] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[2, 44, 4070, 0, 3840, 114, 15, 2, 62, 10, 1, 2946, 4091, 3965, 0, 3, 733, 3, 11, 15, 67, 4091, 4, 4, 103, 0, 1, 34, 0, 3918, 4063, 1, 3647, 2, 1750, 0, 62, 0, 3985, 4, 172, 31, 14, 0, 7, 9, 160, 1, 0, 3986, 3965, 1, 115, 51, 124, 17, 4052, 4088, 0, 1, 4092, 3, 12, 0, 1, 462, 42, 3989, 430, 0, 0, 48, 3456, 3089, 36, 2, 0, 8, 1142, 0, 1793, 2301, 57, 2428, 4080, 4041, 66, 3985, 2427, 0, 9, 24, 0, 0, 2144, 0, 34, 4086, 3967, 431, 2874, 0, 58, 3, 0, 25, 0, 0, 460, 18, 0, 0, 4070, 453, 0, 142, 0, 3021, 1, 2684, 565, 987, 0, 2873, 0, 1748, 1, 3947] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4095, 4, 3945, 0, 95, 3902, 211, 1059, 0, 420, 2948, 4096, 0, 45, 2, 0, 2, 2, 0, 3986, 0, 0, 1591, 0, 38, 0, 0, 4096, 3665, 14, 4000, 0, 1106, 0, 77, 122, 3943, 1, 4, 2, 0, 0, 0, 1, 1, 0, 18, 3072, 4065, 0, 0, 0, 0, 1, 4096, 4078, 0, 1, 2109, 1, 0, 47, 0, 111, 5, 4093, 230, 55, 179, 19, 2911, 3939, 0, 0, 2, 0, 3700, 0, 3734, 1189, 587, 2643, 4081, 4088, 2335, 4094, 3600, 0, 4, 42, 27, 230, 49, 1, 4, 4092, 4096, 183, 3743, 0, 0, 0, 0, 1, 0, 8, 157, 0, 1, 0, 2095, 1, 0, 0, 0, 438, 251, 1640, 288, 150, 0, 3758, 2, 3160, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3919, 0, 3997, 3, 2, 411, 12, 9, 0, 147, 2874, 4000, 0, 1, 236, 1, 0, 2, 7, 4089, 1, 4, 2670, 6, 0, 40, 0, 4076, 4090, 0, 4079, 0, 1905, 0, 0, 118, 4096, 0, 53, 18, 0, 1, 0, 104, 0, 42, 0, 4092, 3195, 0, 49, 0, 3, 3, 3998, 3963, 7, 3, 1935, 0, 45, 0, 0, 217, 48, 4055, 52, 27, 1703, 0, 3921, 3599, 10, 1, 0, 1, 3891, 26, 3381, 738, 168, 3237, 3757, 4044, 241, 3873, 3044, 0, 71, 0, 8, 0, 1914, 4, 55, 3698, 4087, 216, 4092, 61, 11, 4, 0, 0, 0, 1, 3436, 0, 0, 20, 667, 1, 0, 0, 0, 29, 2, 1859, 891, 554, 0, 3833, 17, 3111, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 8, 3884, 49, 4096, 6, 0, 105, 15, 37, 83, 60, 4055, 4023, 0, 172, 1684, 61, 65, 373, 140, 4079, 0, 0, 4, 20, 14, 0, 1, 3998, 2944, 0, 3622, 0, 1729, 5, 812, 0, 4093, 0, 0, 3597, 0, 0, 13, 0, 26, 0, 4, 3671, 3108, 0, 29, 12, 189, 0, 4092, 3338, 2, 0, 3687, 0, 0, 62, 0, 566, 37, 3588, 590, 214, 1274, 0, 3708, 4061, 0, 44, 4, 185, 3831, 0, 1218, 1405, 262, 523, 3903, 4093, 3, 4046, 3633, 8, 4, 1, 31, 0, 1025, 162, 791, 4076, 4020, 32, 2336, 268, 75, 1, 0, 8, 0, 0, 318, 0, 40, 0, 3954, 144, 0, 0, 3, 651, 17, 3133, 285, 1894, 0, 3903, 48, 702, 0, 3887] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 2, 2495, 0, 3730, 3, 34, 1670, 6, 0, 0, 703, 3327, 3994, 2, 5, 944, 0, 78, 43, 558, 3997, 3, 52, 1676, 0, 2, 0, 1, 4054, 3776, 1, 1292, 33, 3, 0, 1, 2, 3753, 1, 5, 68, 402, 0, 16, 0, 21, 1, 0, 4015, 3099, 0, 44, 75, 15, 17, 4061, 4073, 0, 0, 2853, 17, 1, 1, 11, 380, 3, 4075, 46, 0, 1501, 0, 3531, 4045, 460, 20, 0, 5, 3133, 34, 236, 509, 15, 2285, 4089, 4077, 63, 3781, 3992, 8, 10, 0, 14, 4, 1756, 80, 0, 3883, 4040, 48, 4062, 0, 101, 0, 1, 37, 21, 2, 3685, 0, 6, 0, 4008, 164, 0, 18, 15, 2099, 180, 1067, 1586, 726, 15, 4053, 63, 3899, 9, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 5, 4076, 17, 4081, 34, 12, 2076, 30, 6, 0, 54, 4068, 3960, 9, 125, 381, 0, 0, 1, 40, 4052, 0, 95, 158, 0, 83, 1, 27, 4058, 2738, 1, 1173, 0, 91, 0, 0, 18, 4096, 1, 1392, 10, 0, 0, 1, 13, 0, 2, 0, 3898, 3958, 0, 0, 1, 126, 2, 4094, 3810, 14, 2, 4004, 0, 16, 3, 2, 65, 613, 4095, 62, 30, 68, 169, 478, 3122, 46, 2, 0, 0, 3979, 17, 1110, 2445, 80, 4009, 4070, 4096, 3296, 4010, 3996, 0, 1, 2, 0, 2, 3780, 750, 20, 4095, 4096, 4, 3468, 0, 6, 0, 1, 0, 0, 1, 1352, 0, 0, 0, 3592, 1, 0, 0, 17, 512, 563, 1081, 1416, 1107, 3, 310, 0, 4052, 0, 4067] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3712, 49, 4068, 0, 0, 1, 221, 0, 3, 924, 4020, 4052, 0, 371, 1030, 0, 1, 0, 330, 4094, 0, 0, 667, 0, 3, 0, 21, 3898, 4080, 58, 4094, 52, 3433, 0, 0, 509, 4025, 3, 60, 12, 69, 14, 0, 29, 4, 86, 20, 4086, 4074, 7, 0, 0, 0, 1, 4096, 3825, 97, 162, 3477, 2, 4, 0, 0, 755, 1, 3519, 12, 15, 94, 14, 2605, 4015, 2, 0, 47, 0, 3893, 5, 1134, 2181, 8, 1338, 4037, 4077, 1466, 4062, 3895, 0, 2, 16, 189, 4, 2862, 59, 2, 3911, 4039, 1, 4087, 2, 0, 4, 0, 0, 552, 0, 1059, 0, 35, 32, 2849, 1, 5, 13, 65, 881, 826, 713, 178, 1701, 9, 102, 134, 1639, 0, 4081] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3145, 0, 2796, 0, 0, 1353, 97, 6, 0, 292, 4034, 3581, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 6, 58, 4072, 1, 0, 556, 0, 5, 2, 5, 3816, 1037, 194, 1801, 1393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3764, 0, 978, 0, 2, 12, 4086, 2, 124, 3, 94, 0, 0, 1, 0, 4, 4092, 2703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3403, 0, 0, 0, 0, 0, 1589, 4096, 0, 13, 3780, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2998, 0, 3855, 3, 27, 395, 0, 302, 4093, 5, 0, 165, 0, 3215, 0, 490, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 1894, 27, 3697, 0, 0, 2044, 3594, 3968, 0, 0, 0, 9, 672, 1956, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 3799, 4096, 1, 3494, 0, 22, 1, 0, 13, 1, 1, 2571, 0, 1, 0, 1398, 715, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2012, 0, 9, 4, 1, 3377, 2, 3756, 797, 236, 0, 2716, 1015, 2265, 0, 2660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15348, 12951, 15865, 16977, 15998, 18200, 16883, 18850] avg=16384.0000 max=18850 max_rank=7 min=12951 min_rank=1 imbalance_ratio=1.150513 max_load_reduction=112222 max_load_reduction_pct=0.856186 moved_to_echo=69931 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 3884, 49, 2891, 6, 0, 105, 15, 37, 83, 60, 4055, 4023, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1684, 61, 65, 373, 140, 4079, 0, 0, 4, 20, 14, 0, 1, 3998, 2944, 0, 1801, 1327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3622, 0, 1729, 5, 812, 0, 4093, 0, 0, 3597, 0, 0, 13, 0, 26, 0, 4093, 2576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3671, 3108, 0, 29, 12, 189, 0, 1589, 3338, 2, 0, 3687, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 37, 3588, 590, 214, 1274, 0, 3708, 4061, 0, 44, 4, 185, 3831, 0, 486, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1218, 1405, 262, 523, 0, 0, 3, 3560, 3633, 8, 4, 1, 31, 0, 1025, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 791, 4076, 4020, 32, 2336, 268, 75, 1, 0, 8, 0, 0, 318, 0, 40, 0, 1339, 740, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 144, 0, 0, 3, 651, 17, 3133, 285, 1894, 0, 3903, 48, 702, 0, 2548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15388, 16511, 20566, 15691, 19053, 11835, 14746, 17282] avg=16384.0000 max=20566 max_rank=2 min=11835 min_rank=5 imbalance_ratio=1.255249 max_load_reduction=110506 max_load_reduction_pct=0.843094 moved_to_echo=62916 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 4, 2784, 0, 95, 3902, 211, 1059, 0, 420, 2948, 4096, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 2, 0, 3986, 0, 0, 1591, 0, 38, 0, 0, 4096, 3665, 14, 1803, 1387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 1106, 0, 77, 122, 3943, 1, 4, 2, 0, 0, 0, 1, 1, 0, 4088, 2694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3072, 4065, 0, 0, 0, 0, 1, 1590, 4078, 0, 1, 2109, 1, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 5, 4093, 230, 55, 179, 19, 2911, 3939, 0, 0, 2, 0, 3700, 0, 491, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3734, 1189, 587, 2643, 0, 0, 2335, 3603, 3600, 0, 4, 42, 27, 230, 49, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 4096, 183, 3743, 0, 0, 0, 0, 1, 0, 8, 157, 0, 1, 0, 1411, 713, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2095, 1, 0, 0, 0, 438, 251, 1640, 288, 150, 0, 3758, 2, 3160, 0, 2684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19659, 16586, 16039, 14982, 16183, 18044, 15112, 14467] avg=16384.0000 max=19659 max_rank=0 min=14467 min_rank=7 imbalance_ratio=1.199890 max_load_reduction=111413 max_load_reduction_pct=0.850014 moved_to_echo=63806 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 44, 4070, 0, 2710, 114, 15, 2, 62, 10, 1, 2946, 4091, 3965, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 733, 3, 11, 15, 67, 4091, 4, 4, 103, 0, 1, 34, 0, 3918, 4063, 1, 1784, 1387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3647, 2, 1750, 0, 62, 0, 3985, 4, 172, 31, 14, 0, 7, 9, 160, 1, 4041, 2693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 3965, 1, 115, 51, 124, 17, 1573, 4088, 0, 1, 4092, 3, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 462, 42, 3989, 430, 0, 0, 48, 3456, 3089, 36, 2, 0, 8, 1142, 0, 478, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1793, 2301, 57, 2428, 0, 0, 66, 3507, 2427, 0, 9, 24, 0, 0, 2144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4086, 3967, 431, 2874, 0, 58, 3, 0, 25, 0, 0, 460, 18, 0, 0, 1360, 694, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 453, 0, 142, 0, 3021, 1, 2684, 565, 987, 0, 2873, 0, 1748, 1, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18035, 16219, 16578, 18028, 13619, 14756, 14705, 19132] avg=16384.0000 max=19132 max_rank=7 min=13619 min_rank=4 imbalance_ratio=1.167725 max_load_reduction=111940 max_load_reduction_pct=0.854034 moved_to_echo=62212 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 2495, 0, 2632, 3, 34, 1670, 6, 0, 0, 703, 3327, 3994, 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 944, 0, 78, 43, 558, 3997, 3, 52, 1676, 0, 2, 0, 1, 4054, 3776, 1, 1788, 1391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1292, 33, 3, 0, 1, 2, 3753, 1, 5, 68, 402, 0, 16, 0, 21, 1, 4077, 2698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 3099, 0, 44, 75, 15, 17, 1576, 4073, 0, 0, 2853, 17, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 380, 3, 4075, 46, 0, 1501, 0, 3531, 4045, 460, 20, 0, 5, 3133, 34, 454, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 509, 15, 2285, 0, 0, 63, 3327, 3992, 8, 10, 0, 14, 4, 1756, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3883, 4040, 48, 4062, 0, 101, 0, 1, 37, 21, 2, 3685, 0, 6, 0, 1411, 674, 697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 164, 0, 18, 15, 2099, 180, 1067, 1586, 726, 15, 4053, 63, 3899, 9, 2685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14873, 18364, 12373, 15786, 18122, 12299, 18668, 20587] avg=16384.0000 max=20587 max_rank=7 min=12299 min_rank=5 imbalance_ratio=1.256531 max_load_reduction=110485 max_load_reduction_pct=0.842934 moved_to_echo=69676 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4076, 17, 2874, 34, 12, 2076, 30, 6, 0, 54, 4068, 3960, 9, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 381, 0, 0, 1, 40, 4052, 0, 95, 158, 0, 83, 1, 27, 4058, 2738, 1, 1806, 1384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1173, 0, 91, 0, 0, 18, 4096, 1, 1392, 10, 0, 0, 1, 13, 0, 2, 4096, 2686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3898, 3958, 0, 0, 1, 126, 2, 1582, 3810, 14, 2, 4004, 0, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 65, 613, 4095, 62, 30, 68, 169, 478, 3122, 46, 2, 0, 0, 3979, 17, 485, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1110, 2445, 80, 4009, 0, 0, 3296, 3525, 3996, 0, 1, 2, 0, 2, 3780, 750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4095, 4096, 4, 3468, 0, 6, 0, 1, 0, 0, 1, 1352, 0, 0, 0, 1403, 740, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3592, 1, 0, 0, 17, 512, 563, 1081, 1416, 1107, 3, 310, 0, 4052, 0, 2664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17346, 14825, 13579, 17416, 13700, 22996, 15892, 15318] avg=16384.0000 max=22996 max_rank=5 min=13579 min_rank=2 imbalance_ratio=1.403564 max_load_reduction=108076 max_load_reduction_pct=0.824554 moved_to_echo=67906 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3919, 0, 2821, 3, 2, 411, 12, 9, 0, 147, 2874, 4000, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 1, 0, 2, 7, 4089, 1, 4, 2670, 6, 0, 40, 0, 4076, 4090, 0, 1760, 1277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 1905, 0, 0, 118, 4096, 0, 53, 18, 0, 1, 0, 104, 0, 42, 4044, 2480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3195, 0, 49, 0, 3, 3, 1552, 3963, 7, 3, 1935, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 48, 4055, 52, 27, 1703, 0, 3921, 3599, 10, 1, 0, 1, 3891, 26, 465, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3381, 738, 168, 3237, 0, 0, 241, 3408, 3044, 0, 71, 0, 8, 0, 1914, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 3698, 4087, 216, 4092, 61, 11, 4, 0, 0, 0, 1, 3436, 0, 0, 20, 1409, 722, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, 1, 0, 0, 0, 29, 2, 1859, 891, 554, 0, 3833, 17, 3111, 0, 2681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14199, 18259, 16940, 14847, 18470, 16214, 18498, 13645] avg=16384.0000 max=18498 max_rank=6 min=13645 min_rank=7 imbalance_ratio=1.129028 max_load_reduction=112574 max_load_reduction_pct=0.858871 moved_to_echo=66827 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3712, 49, 2871, 0, 0, 1, 221, 0, 3, 924, 4020, 4052, 0, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 0, 1, 0, 330, 4094, 0, 0, 667, 0, 3, 0, 21, 3898, 4080, 58, 1803, 1372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 52, 3433, 0, 0, 509, 4025, 3, 60, 12, 69, 14, 0, 29, 4, 86, 4077, 2665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4086, 4074, 7, 0, 0, 0, 1, 1590, 3825, 97, 162, 3477, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 755, 1, 3519, 12, 15, 94, 14, 2605, 4015, 2, 0, 47, 0, 3893, 5, 488, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1134, 2181, 8, 1338, 0, 0, 1466, 3574, 3895, 0, 2, 16, 189, 4, 2862, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3911, 4039, 1, 4087, 2, 0, 4, 0, 0, 552, 0, 1059, 0, 35, 32, 1406, 735, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2849, 1, 5, 13, 65, 881, 826, 713, 178, 1701, 9, 102, 134, 1639, 0, 2675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16224, 17357, 19132, 17345, 15927, 16728, 16568, 11791] avg=16384.0000 max=19132 max_rank=2 min=11791 min_rank=7 imbalance_ratio=1.167725 max_load_reduction=111940 max_load_reduction_pct=0.854034 moved_to_echo=61014 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[23296, 768, 128, 256, 128, 11008, 1920, 16000, 6016, 7424, 128, 21632, 1280, 20608, 128, 21248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[1024, 31744, 32512, 1024, 28160, 384, 384, 128, 128, 128, 640, 128, 13056, 128, 128, 128, 11264, 5760, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[25728, 128, 11008, 128, 1024, 896, 32128, 128, 1920, 3840, 640, 128, 128, 256, 256, 256, 32640, 21248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[12928, 12672, 1280, 20224, 0, 0, 9600, 28160, 28672, 128, 128, 128, 384, 1024, 15488, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 29440, 128, 22400, 256, 256, 9600, 768, 1152, 128, 5632, 29440, 31744, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 5632, 768, 31360, 1536, 384, 5248, 256, 20992, 30080, 640, 128, 256, 256, 26880, 128, 3840, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[5120, 128, 256, 512, 1280, 32512, 128, 256, 7552, 128, 256, 128, 128, 32000, 26496, 384, 14464, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 30976, 28928, 128, 256, 256, 512, 128, 12672, 31360, 128, 256, 25984, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4015, 0, 14, 2, 432, 0, 324, 0, 0, 3345, 3, 4095, 1705, 0, 0, 3936, 3005, 2389, 0, 0, 1865, 1524, 1600, 0, 0, 3658, 0, 0, 3, 4094, 196, 2344, 12, 4054, 180, 4079, 0, 0, 4096, 4065, 0, 0, 0, 29, 0, 0, 0, 0, 105, 3129, 0, 61, 3839, 0, 0, 0, 94, 0, 280, 3634, 3849, 4, 282, 0, 317, 3300, 0, 14, 28, 0, 20, 1795, 3239, 3915, 12, 1611, 428, 4094, 0, 197, 2, 28, 0, 2961, 1002, 4084, 301, 3948, 5, 0, 0, 0, 0, 0, 4091, 2, 0, 408, 4068, 0, 2, 0, 1198, 62, 295, 0, 4052, 388, 0, 4095, 15, 0, 0, 0, 3830, 2753, 6, 0, 30, 0, 4092, 0, 20, 19, 0, 0, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[3652, 0, 0, 0, 4093, 0, 0, 0, 0, 4090, 27, 4073, 1457, 0, 0, 4092, 1979, 3840, 0, 0, 4095, 523, 1981, 0, 0, 4054, 0, 0, 0, 4096, 7, 2244, 243, 4096, 0, 2675, 0, 9, 4091, 4070, 0, 0, 0, 1, 0, 0, 0, 0, 68, 2043, 0, 47, 3824, 0, 0, 0, 5, 6, 0, 2355, 4091, 0, 20, 0, 0, 3720, 0, 0, 0, 0, 0, 2850, 4062, 2081, 1, 0, 101, 4096, 0, 1618, 0, 65, 266, 223, 0, 4083, 2, 3997, 0, 7, 0, 0, 614, 0, 4096, 131, 0, 93, 4096, 1, 0, 0, 0, 595, 0, 0, 4096, 30, 0, 4085, 0, 0, 3108, 0, 4095, 2799, 0, 0, 35, 0, 4041, 0, 0, 1, 0, 0, 37, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[3983, 0, 0, 0, 3808, 9, 0, 12, 19, 3414, 0, 4028, 132, 23, 0, 3941, 686, 3866, 2, 0, 4072, 3074, 4036, 0, 1, 4095, 0, 0, 0, 4092, 706, 82, 0, 3438, 123, 698, 2, 0, 4076, 4095, 1, 0, 29, 0, 167, 0, 6, 0, 0, 1465, 0, 0, 4088, 3, 16, 0, 0, 0, 0, 4005, 3955, 0, 0, 0, 73, 139, 0, 3, 0, 7, 0, 2425, 3995, 1283, 0, 6, 39, 4094, 13, 2603, 0, 2027, 0, 2483, 15, 4093, 91, 4020, 0, 58, 59, 1, 2066, 0, 4091, 72, 0, 13, 4090, 0, 0, 0, 0, 1114, 0, 0, 4093, 0, 0, 4074, 50, 0, 2394, 0, 4096, 3034, 8, 9, 0, 0, 4060, 0, 0, 0, 4, 0, 0, 29] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4096, 181, 0, 0, 4092, 0, 1, 0, 0, 3985, 0, 4096, 2234, 1, 0, 4096, 2193, 4093, 0, 0, 4042, 40, 6, 0, 0, 4095, 0, 0, 0, 4095, 2, 3846, 11, 4096, 0, 4095, 0, 0, 4095, 3967, 0, 1, 0, 9, 1, 0, 0, 0, 0, 3882, 0, 0, 4095, 0, 0, 3, 0, 7, 1, 3389, 4094, 0, 408, 0, 0, 3963, 0, 0, 0, 1, 0, 574, 4096, 3783, 0, 77, 61, 4096, 0, 121, 0, 8, 0, 3692, 50, 4090, 0, 67, 0, 4, 0, 0, 0, 3, 4096, 0, 0, 73, 4096, 0, 0, 1, 0, 302, 0, 0, 4095, 9, 1, 4095, 0, 1, 218, 0, 4096, 3843, 0, 0, 47, 0, 4065, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4063, 5, 1, 0, 1743, 0, 576, 1, 0, 4096, 1, 4040, 0, 0, 0, 3830, 2064, 4093, 0, 0, 4095, 1931, 1706, 1, 0, 3835, 2, 1, 1, 4094, 1997, 711, 98, 4054, 0, 270, 15, 0, 4080, 2431, 0, 0, 0, 6, 0, 0, 0, 4, 3, 1923, 0, 125, 3960, 0, 0, 2, 13, 31, 1, 59, 4091, 0, 181, 0, 39, 3118, 0, 0, 0, 0, 0, 4075, 4095, 4093, 14, 1, 0, 4091, 0, 3954, 14, 4057, 3, 4095, 0, 4087, 4, 1, 0, 0, 0, 1, 259, 1, 4096, 417, 0, 1553, 4060, 23, 0, 2, 176, 80, 0, 28, 4083, 0, 0, 4092, 0, 2, 21, 0, 4071, 4044, 0, 1, 0, 0, 4007, 0, 1, 6, 0, 1, 72, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4095, 0, 35, 0, 2376, 15, 0, 41, 0, 3944, 0, 4096, 3916, 0, 4, 4096, 111, 4048, 0, 0, 651, 202, 1463, 0, 0, 4063, 101, 0, 12, 3866, 4010, 1026, 666, 4091, 0, 4095, 0, 0, 4003, 4044, 7, 3, 34, 1, 55, 2, 0, 6, 0, 3084, 0, 0, 4045, 19, 7, 235, 85, 0, 0, 3989, 4042, 0, 3786, 2, 444, 3946, 257, 8, 28, 0, 10, 927, 4063, 571, 0, 0, 17, 4069, 29, 65, 0, 3693, 9, 157, 1, 3112, 254, 0, 89, 0, 3, 3, 31, 0, 3784, 0, 17, 7, 4096, 0, 0, 1, 0, 1245, 0, 0, 4058, 1, 8, 3873, 0, 40, 1567, 0, 3931, 3985, 0, 31, 0, 0, 4095, 1, 4, 0, 50, 1, 19, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[3750, 0, 0, 0, 2277, 0, 0, 4, 5, 4096, 13, 4055, 1886, 0, 0, 4096, 1983, 3557, 0, 0, 2278, 35, 41, 0, 4, 3733, 0, 0, 0, 4096, 913, 460, 3, 4087, 0, 4022, 0, 5, 4091, 4074, 0, 13, 0, 0, 35, 0, 29, 0, 179, 4084, 0, 1, 4096, 0, 1, 0, 221, 4, 2, 3984, 4091, 0, 1281, 0, 0, 4013, 0, 3, 0, 0, 9, 101, 4082, 2297, 202, 1, 0, 4094, 0, 2184, 4, 4089, 0, 4096, 0, 3703, 38, 253, 231, 2, 0, 0, 348, 4, 4086, 48, 0, 97, 3914, 5, 0, 9, 986, 5, 1, 0, 4090, 15, 0, 4096, 0, 0, 22, 2, 4093, 4088, 0, 0, 0, 0, 4075, 0, 0, 0, 38, 0, 52, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4054, 94, 0, 52, 3784, 75, 0, 86, 103, 2683, 0, 4049, 2950, 18, 11, 4094, 1405, 4095, 0, 31, 95, 1, 3959, 2, 0, 3881, 3, 25, 9, 4081, 2486, 4080, 0, 3796, 0, 1489, 0, 0, 4071, 3565, 0, 0, 1, 0, 0, 0, 0, 0, 638, 3865, 0, 0, 4068, 2, 0, 3, 1, 0, 3, 1921, 4092, 36, 189, 4, 0, 4093, 0, 1, 0, 0, 0, 1631, 4093, 12, 609, 1, 0, 4059, 0, 3085, 30, 115, 100, 439, 28, 4096, 0, 308, 17, 101, 1, 0, 360, 223, 4014, 78, 182, 0, 4080, 0, 30, 46, 493, 1, 23, 2, 3888, 1375, 0, 4094, 5, 0, 4069, 0, 4094, 3065, 63, 3, 1, 11, 4082, 0, 0, 0, 2, 0, 0, 19] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4015, 0, 14, 2, 432, 0, 324, 0, 0, 3345, 3, 0, 1705, 0, 0, 3801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3005, 2389, 0, 0, 1865, 1524, 1600, 0, 0, 2218, 0, 0, 3, 0, 196, 2344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4054, 180, 4079, 0, 0, 4096, 4065, 0, 0, 0, 29, 0, 0, 0, 0, 1440, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 3129, 0, 61, 3839, 0, 0, 0, 94, 0, 280, 3634, 3849, 4, 282, 0, 1297, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 3300, 0, 14, 28, 0, 20, 1795, 3239, 3915, 12, 1611, 428, 2797, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 28, 0, 2961, 1002, 4084, 301, 3948, 5, 0, 0, 0, 0, 0, 4091, 2, 1902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 4068, 0, 2, 0, 1198, 62, 295, 0, 4052, 388, 0, 4095, 15, 0, 2086, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3830, 2753, 6, 0, 30, 0, 4092, 0, 20, 19, 0, 0, 0, 4, 3348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13641, 15144, 18061, 16709, 17673, 18326, 17416, 14102] avg=16384.0000 max=18326 max_rank=5 min=13641 min_rank=0 imbalance_ratio=1.118530 max_load_reduction=112746 max_load_reduction_pct=0.860184 moved_to_echo=67517 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3983, 0, 0, 0, 3808, 9, 0, 12, 19, 3414, 0, 0, 132, 23, 0, 3805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 686, 3866, 2, 0, 4072, 3074, 4036, 0, 1, 2483, 0, 0, 0, 0, 706, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3438, 123, 698, 2, 0, 4076, 4095, 1, 0, 29, 0, 167, 0, 6, 0, 1612, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1465, 0, 0, 4088, 3, 16, 0, 0, 0, 0, 4005, 3955, 0, 0, 0, 1297, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 139, 0, 3, 0, 7, 0, 2425, 3995, 1283, 0, 6, 39, 2797, 13, 2603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2027, 0, 2483, 15, 4093, 91, 4020, 0, 58, 59, 1, 2066, 0, 4091, 72, 1899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4090, 0, 0, 0, 0, 1114, 0, 0, 4093, 0, 0, 4074, 50, 0, 2087, 734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2394, 0, 4096, 3034, 8, 9, 0, 0, 4060, 0, 0, 0, 4, 0, 0, 29, 3294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15205, 19008, 14353, 14965, 13383, 20975, 16255, 16928] avg=16384.0000 max=20975 max_rank=5 min=13383 min_rank=4 imbalance_ratio=1.280212 max_load_reduction=110097 max_load_reduction_pct=0.839973 moved_to_echo=67541 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3652, 0, 0, 0, 4093, 0, 0, 0, 0, 4090, 27, 0, 1457, 0, 0, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1979, 3840, 0, 0, 4095, 523, 1981, 0, 0, 2458, 0, 0, 0, 0, 7, 2244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 4096, 0, 2675, 0, 9, 4091, 4070, 0, 0, 0, 1, 0, 0, 0, 0, 1596, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 2043, 0, 47, 3824, 0, 0, 0, 5, 6, 0, 2355, 4091, 0, 20, 0, 1298, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 0, 0, 0, 0, 0, 2850, 4062, 2081, 1, 0, 101, 2798, 0, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 266, 223, 0, 4083, 2, 3997, 0, 7, 0, 0, 614, 0, 4096, 131, 1902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 4096, 1, 0, 0, 0, 595, 0, 0, 4096, 30, 0, 4085, 0, 0, 2087, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3108, 0, 4095, 2799, 0, 0, 35, 0, 4041, 0, 0, 1, 0, 0, 37, 0, 3330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17270, 17127, 16888, 13898, 17231, 15386, 15826, 17446] avg=16384.0000 max=17446 max_rank=7 min=13898 min_rank=3 imbalance_ratio=1.064819 max_load_reduction=113626 max_load_reduction_pct=0.866898 moved_to_echo=65889 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 181, 0, 0, 4092, 0, 1, 0, 0, 3985, 0, 0, 2234, 1, 0, 3955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2193, 4093, 0, 0, 4042, 40, 6, 0, 0, 2483, 0, 0, 0, 0, 2, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 4095, 0, 0, 4095, 3967, 0, 1, 0, 9, 1, 0, 0, 0, 1612, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3882, 0, 0, 4095, 0, 0, 3, 0, 7, 1, 3389, 4094, 0, 408, 0, 1298, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 1, 0, 574, 4096, 3783, 0, 77, 61, 2798, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3692, 50, 4090, 0, 67, 0, 4, 0, 0, 0, 3, 4096, 0, 1901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 4096, 0, 0, 1, 0, 302, 0, 0, 4095, 9, 1, 4095, 0, 1, 2088, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 4096, 3843, 0, 0, 47, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 3350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18545, 16705, 17993, 17318, 15474, 13911, 15507, 15619] avg=16384.0000 max=18545 max_rank=0 min=13911 min_rank=5 imbalance_ratio=1.131897 max_load_reduction=112527 max_load_reduction_pct=0.858513 moved_to_echo=60511 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 35, 0, 2376, 15, 0, 41, 0, 3944, 0, 0, 3916, 0, 4, 3955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 4048, 0, 0, 651, 202, 1463, 0, 0, 2464, 101, 0, 12, 0, 4010, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 666, 4091, 0, 4095, 0, 0, 4003, 4044, 7, 3, 34, 1, 55, 2, 0, 6, 1599, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3084, 0, 0, 4045, 19, 7, 235, 85, 0, 0, 3989, 4042, 0, 3786, 2, 1289, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 3946, 257, 8, 28, 0, 10, 927, 4063, 571, 0, 0, 17, 2780, 29, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3693, 9, 157, 1, 3112, 254, 0, 89, 0, 3, 3, 31, 0, 3784, 0, 1796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7, 4096, 0, 0, 1, 0, 1245, 0, 0, 4058, 1, 8, 3873, 0, 40, 1970, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1567, 0, 3931, 3985, 0, 31, 0, 0, 4095, 1, 4, 0, 50, 1, 19, 0, 3350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18381, 14088, 18706, 20724, 13145, 12932, 16062, 17034] avg=16384.0000 max=20724 max_rank=3 min=12932 min_rank=5 imbalance_ratio=1.264893 max_load_reduction=110348 max_load_reduction_pct=0.841888 moved_to_echo=59173 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4063, 5, 1, 0, 1743, 0, 576, 1, 0, 4096, 1, 0, 0, 0, 0, 3695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2064, 4093, 0, 0, 4095, 1931, 1706, 1, 0, 2319, 2, 1, 1, 0, 1997, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 4054, 0, 270, 15, 0, 4080, 2431, 0, 0, 0, 6, 0, 0, 0, 4, 1514, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1923, 0, 125, 3960, 0, 0, 2, 13, 31, 1, 59, 4091, 0, 181, 0, 1301, 135, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 3118, 0, 0, 0, 0, 0, 4075, 4095, 4093, 14, 1, 0, 2790, 0, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4057, 3, 4095, 0, 4087, 4, 1, 0, 0, 0, 1, 259, 1, 4096, 417, 1900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1553, 4060, 23, 0, 2, 176, 80, 0, 28, 4083, 0, 0, 4092, 0, 2, 2086, 737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4071, 4044, 0, 1, 0, 0, 4007, 0, 1, 6, 0, 1, 72, 0, 3303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14181, 18921, 12580, 11827, 22179, 18935, 16922, 15527] avg=16384.0000 max=22179 max_rank=4 min=11827 min_rank=3 imbalance_ratio=1.353699 max_load_reduction=108893 max_load_reduction_pct=0.830788 moved_to_echo=73563 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4054, 94, 0, 52, 3784, 75, 0, 86, 103, 2683, 0, 0, 2950, 18, 11, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1405, 4095, 0, 31, 95, 1, 3959, 2, 0, 2353, 3, 25, 9, 0, 2486, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 0, 1489, 0, 0, 4071, 3565, 0, 0, 1, 0, 0, 0, 0, 0, 1528, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 638, 3865, 0, 0, 4068, 2, 0, 3, 1, 0, 3, 1921, 4092, 36, 189, 4, 1286, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 0, 0, 1631, 4093, 12, 609, 1, 0, 2773, 0, 3085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 115, 100, 439, 28, 4096, 0, 308, 17, 101, 1, 0, 360, 223, 4014, 78, 1894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 4080, 0, 30, 46, 493, 1, 23, 2, 3888, 1375, 0, 4094, 5, 0, 2080, 739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 4094, 3065, 63, 3, 1, 11, 4082, 0, 0, 0, 2, 0, 0, 19, 3310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17863, 18544, 14557, 16249, 16298, 11804, 17038, 18719] avg=16384.0000 max=18719 max_rank=7 min=11804 min_rank=5 imbalance_ratio=1.142517 max_load_reduction=112353 max_load_reduction_pct=0.857185 moved_to_echo=63859 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3750, 0, 0, 0, 2277, 0, 0, 4, 5, 4096, 13, 0, 1886, 0, 0, 3955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1983, 3557, 0, 0, 2278, 35, 41, 0, 4, 2264, 0, 0, 0, 0, 913, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4087, 0, 4022, 0, 5, 4091, 4074, 0, 13, 0, 0, 35, 0, 29, 0, 1469, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 4084, 0, 1, 4096, 0, 1, 0, 221, 4, 2, 3984, 4091, 0, 1281, 0, 1297, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 3, 0, 0, 9, 101, 4082, 2297, 202, 1, 0, 2797, 0, 2184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4089, 0, 4096, 0, 3703, 38, 253, 231, 2, 0, 0, 348, 4, 4086, 48, 1901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 3914, 5, 0, 9, 986, 5, 1, 0, 4090, 15, 0, 4096, 0, 0, 2089, 739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 2, 4093, 4088, 0, 0, 0, 0, 4075, 0, 0, 0, 38, 0, 52, 11, 3316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15986, 11535, 17934, 19382, 15689, 18803, 16046, 15697] avg=16384.0000 max=19382 max_rank=3 min=11535 min_rank=1 imbalance_ratio=1.182983 max_load_reduction=111690 max_load_reduction_pct=0.852127 moved_to_echo=66235 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[13440, 30080, 128, 128, 21248, 7424, 14848, 128, 128, 19072, 128, 128, 128, 0, 10368, 14848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[31744, 384, 128, 128, 22656, 128, 1024, 256, 128, 29696, 128, 0, 14336, 128, 128, 31104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[1024, 23552, 0, 256, 32128, 128, 128, 256, 512, 128, 384, 23424, 32384, 128, 6272, 128, 10368, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 2304, 32512, 128, 128, 128, 2944, 3456, 384, 128, 32512, 1920, 128, 32512, 128, 128, 16640, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 14208, 384, 18176, 1152, 31360, 768, 12672, 384, 256, 128, 128, 3712, 256, 32384, 768, 15104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[11520, 128, 32384, 27648, 128, 128, 128, 128, 32640, 128, 128, 128, 128, 128, 256, 128, 26624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[1152, 31744, 384, 21504, 128, 128, 32640, 30336, 128, 128, 128, 128, 384, 128, 128, 128, 12416, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[896, 26368, 384, 128, 128, 128, 128, 14464, 31744, 18048, 896, 1792, 768, 22400, 128, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4096, 1136, 0, 0, 4083, 55, 4013, 4096, 83, 1, 4018, 302, 3830, 230, 0, 0, 0, 638, 0, 30, 0, 1, 0, 0, 0, 4, 4073, 3721, 4096, 0, 3, 62, 0, 3, 0, 0, 3, 3655, 4094, 0, 0, 0, 0, 3611, 11, 0, 5, 4096, 610, 4083, 4095, 0, 2, 0, 58, 4087, 0, 0, 0, 3390, 5, 158, 1, 0, 0, 2088, 0, 1055, 4037, 0, 4095, 0, 0, 4093, 0, 0, 0, 0, 243, 0, 0, 91, 327, 13, 1884, 15, 0, 4093, 14, 3866, 0, 0, 0, 68, 114, 0, 6, 3, 4035, 0, 4058, 0, 0, 4096, 4091, 3568, 1639, 0, 61, 0, 0, 193, 0, 6, 0, 0, 71, 0, 0, 59, 3927, 32, 4073, 0, 39, 0, 4095, 386] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4065, 471, 0, 0, 3801, 132, 4070, 3976, 0, 0, 3389, 30, 4096, 48, 16, 0, 432, 3954, 17, 52, 0, 189, 1, 0, 0, 4069, 4089, 3778, 4096, 21, 0, 3691, 57, 5, 2, 0, 0, 3084, 4075, 0, 47, 29, 217, 786, 0, 9, 22, 3176, 0, 3417, 3964, 0, 0, 1, 0, 4068, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 450, 12, 0, 753, 8, 197, 5, 1, 4041, 0, 0, 11, 91, 10, 1, 2, 13, 42, 0, 4088, 0, 113, 4043, 0, 4042, 62, 643, 0, 0, 1510, 0, 15, 3870, 4033, 0, 1298, 87, 0, 3904, 4092, 4094, 3824, 21, 1, 0, 7, 44, 96, 1, 0, 30, 2, 0, 0, 3, 3339, 0, 2482, 2, 45, 9, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4095, 47, 0, 0, 3969, 521, 3462, 4092, 0, 0, 2967, 2029, 4096, 0, 2, 0, 0, 1929, 14, 0, 0, 5, 8, 1, 0, 3970, 4075, 1045, 4096, 1, 0, 3161, 6, 0, 0, 0, 0, 4067, 4096, 191, 1, 0, 11, 393, 0, 1, 0, 4019, 0, 897, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2971, 0, 82, 4007, 0, 3837, 0, 0, 4086, 0, 2, 2, 1, 57, 0, 0, 2, 383, 0, 3298, 2, 0, 4066, 20, 4096, 2, 2, 0, 0, 9, 6, 0, 3475, 4074, 0, 2438, 1, 0, 4096, 4096, 4096, 113, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 4091, 1, 4001, 0, 2, 3, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4094, 0, 30, 8, 3800, 552, 386, 3813, 2, 2, 947, 8, 4085, 3, 0, 0, 0, 4059, 0, 1, 0, 169, 0, 0, 0, 4091, 4085, 1046, 4010, 4, 0, 4091, 199, 32, 43, 2, 0, 4079, 4081, 118, 4, 19, 5, 3639, 1, 75, 0, 3310, 0, 3551, 4094, 0, 0, 1, 0, 4072, 1, 0, 13, 4096, 1, 0, 2, 0, 5, 3651, 4, 857, 2305, 0, 3509, 0, 0, 4095, 0, 0, 0, 0, 1422, 0, 1, 0, 120, 21, 23, 23, 2, 4089, 0, 4012, 10, 0, 0, 1, 0, 7, 0, 3206, 4087, 2, 3525, 6, 0, 4093, 4086, 3398, 2164, 0, 0, 2, 2, 0, 0, 2, 0, 0, 2, 0, 0, 0, 4090, 10, 2285, 0, 1, 8, 3754, 1468] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[2839, 0, 0, 0, 3722, 0, 4083, 4024, 0, 184, 0, 484, 4085, 3, 0, 257, 48, 3981, 3, 13, 0, 0, 2, 11, 0, 3829, 4038, 3252, 3919, 0, 0, 4059, 0, 1, 174, 47, 141, 3065, 4068, 1234, 0, 0, 0, 2365, 207, 1, 14, 4096, 705, 4052, 4050, 0, 0, 0, 0, 4043, 124, 0, 0, 3771, 0, 0, 0, 0, 0, 1608, 8, 1443, 2611, 30, 3194, 0, 60, 3595, 11, 0, 0, 0, 9, 3, 2, 9, 3402, 0, 930, 1, 68, 4065, 0, 3268, 0, 1, 0, 423, 0, 37, 31, 0, 4078, 34, 3992, 0, 0, 3963, 4069, 3489, 0, 0, 12, 0, 76, 0, 614, 0, 0, 65, 565, 60, 2, 0, 3720, 1, 1869, 15, 2, 635, 4008, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4085, 490, 1, 0, 3201, 24, 3022, 4096, 2, 0, 306, 220, 3733, 0, 0, 8, 80, 4041, 9, 0, 35, 2, 0, 7, 0, 4062, 4014, 671, 4088, 67, 10, 4085, 0, 0, 0, 0, 22, 4093, 4078, 1, 0, 0, 84, 3668, 0, 0, 2, 3937, 2, 3999, 4069, 0, 0, 0, 33, 4084, 0, 18, 0, 3733, 0, 9, 1, 0, 0, 3971, 0, 3826, 16, 0, 49, 8, 0, 3970, 0, 0, 0, 32, 2, 0, 1, 16, 3843, 143, 3937, 0, 0, 4046, 460, 4092, 22, 344, 98, 0, 0, 0, 1, 42, 3790, 9, 764, 103, 3, 4072, 4070, 3970, 0, 9, 0, 2, 51, 140, 0, 0, 0, 4, 0, 9, 338, 173, 4095, 1, 4089, 0, 33, 0, 4082, 154] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4092, 41, 0, 0, 4094, 3, 4081, 4076, 4, 0, 3845, 258, 4096, 0, 0, 3, 2, 1851, 0, 0, 0, 1, 2, 0, 0, 4092, 4096, 1626, 4096, 0, 0, 3923, 3, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 28, 64, 4, 0, 0, 4096, 777, 2500, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 404, 0, 1, 107, 0, 3854, 0, 0, 3518, 0, 0, 255, 0, 0, 0, 0, 0, 4037, 16, 3532, 0, 0, 3869, 117, 4096, 113, 0, 3, 3, 2560, 0, 0, 381, 4062, 0, 2469, 414, 0, 4095, 4095, 4091, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 27, 4096, 0, 4096, 0, 13, 0, 4094, 384] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4096, 6, 0, 0, 4076, 370, 3976, 4088, 14, 0, 645, 369, 4096, 0, 0, 49, 0, 951, 3, 0, 0, 0, 5, 0, 0, 4084, 4063, 4096, 4079, 0, 2, 3784, 30, 1, 0, 0, 0, 4068, 4096, 10, 0, 5, 228, 4073, 4, 326, 13, 3722, 0, 4081, 4096, 2, 0, 0, 0, 4084, 0, 0, 0, 4094, 0, 0, 17, 0, 26, 3515, 176, 0, 2661, 0, 4078, 0, 4, 79, 1, 0, 0, 1, 0, 1, 0, 0, 1798, 51, 3176, 60, 0, 4093, 4, 4020, 13, 29, 27, 0, 20, 11, 0, 250, 4025, 0, 837, 48, 0, 4096, 3363, 2826, 2579, 0, 2, 0, 5, 0, 1, 0, 0, 0, 303, 1, 0, 0, 4074, 0, 4044, 0, 0, 232, 4095, 645] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 1136, 0, 0, 4083, 55, 4013, 0, 83, 1, 4018, 302, 2046, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 638, 0, 30, 0, 1, 0, 0, 0, 4, 207, 3721, 4096, 0, 3, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 3655, 4094, 0, 0, 0, 0, 3611, 11, 0, 5, 4096, 1731, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 4083, 4095, 0, 2, 0, 58, 4087, 0, 0, 0, 3390, 5, 158, 1, 0, 619, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2088, 0, 1055, 4037, 0, 4095, 0, 0, 4093, 0, 0, 0, 0, 243, 0, 4096, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 327, 13, 1884, 15, 0, 4093, 14, 3866, 0, 0, 0, 68, 114, 0, 3127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 3416, 0, 4058, 0, 0, 0, 4091, 3568, 1639, 0, 61, 0, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 71, 0, 0, 59, 3927, 32, 4073, 0, 39, 0, 4095, 386, 3658, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20063, 8762, 17323, 17161, 20145, 13612, 17035, 16971] avg=16384.0000 max=20145 max_rank=4 min=8762 min_rank=1 imbalance_ratio=1.229553 max_load_reduction=110927 max_load_reduction_pct=0.846306 moved_to_echo=67763 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4065, 471, 0, 0, 3801, 132, 4070, 0, 0, 0, 3389, 30, 2189, 48, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 3954, 17, 52, 0, 189, 1, 0, 0, 4069, 208, 3778, 4096, 21, 0, 3691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 5, 2, 0, 0, 3084, 4075, 0, 47, 29, 217, 786, 0, 9, 22, 3176, 1851, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3417, 3964, 0, 0, 1, 0, 4068, 0, 0, 0, 4095, 0, 0, 0, 2, 619, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 12, 0, 753, 8, 197, 5, 1, 4041, 0, 0, 11, 91, 10, 1, 3904, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 13, 42, 0, 4088, 0, 113, 4043, 0, 4042, 62, 643, 0, 0, 1510, 0, 3139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3870, 3414, 0, 1298, 87, 0, 0, 4092, 4094, 3824, 21, 1, 0, 7, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 1, 0, 30, 2, 0, 0, 3, 3339, 0, 2482, 2, 45, 9, 4096, 0, 3552, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18211, 20508, 13475, 16222, 9908, 17697, 20767, 14284] avg=16384.0000 max=20767 max_rank=6 min=9908 min_rank=4 imbalance_ratio=1.267517 max_load_reduction=110305 max_load_reduction_pct=0.841560 moved_to_echo=62656 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4085, 490, 1, 0, 3201, 24, 3022, 0, 2, 0, 306, 220, 1995, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 4041, 9, 0, 35, 2, 0, 7, 0, 4062, 204, 671, 4088, 67, 10, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4093, 4078, 1, 0, 0, 84, 3668, 0, 0, 2, 3937, 1687, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3999, 4069, 0, 0, 0, 33, 4084, 0, 18, 0, 3733, 0, 9, 1, 0, 582, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 3826, 16, 0, 49, 8, 0, 3970, 0, 0, 0, 32, 2, 0, 4072, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 16, 3843, 143, 3937, 0, 0, 4046, 460, 4092, 22, 344, 98, 0, 0, 0, 3081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 42, 3208, 9, 764, 103, 3, 0, 4070, 3970, 0, 9, 0, 2, 51, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 9, 338, 173, 4095, 1, 4089, 0, 33, 0, 4082, 154, 3659, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13354, 17361, 17685, 16581, 16383, 20083, 12372, 17253] avg=16384.0000 max=20083 max_rank=5 min=12372 min_rank=6 imbalance_ratio=1.225769 max_load_reduction=110989 max_load_reduction_pct=0.846779 moved_to_echo=66091 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4095, 47, 0, 0, 3969, 521, 3462, 0, 0, 0, 2967, 2029, 2189, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1929, 14, 0, 0, 5, 8, 1, 0, 3970, 208, 1045, 4096, 1, 0, 3161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4067, 4096, 191, 1, 0, 11, 393, 0, 1, 0, 4019, 1851, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 897, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 625, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2971, 0, 82, 4007, 0, 3837, 0, 0, 4086, 0, 2, 2, 1, 57, 0, 4096, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 383, 0, 3298, 2, 0, 4066, 20, 4096, 2, 2, 0, 0, 9, 6, 3128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3475, 3449, 0, 2438, 1, 0, 0, 4096, 4096, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 4091, 1, 4001, 0, 2, 3, 4096, 0, 3654, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19281, 14438, 14750, 13866, 19579, 15014, 17668, 16476] avg=16384.0000 max=19579 max_rank=4 min=13866 min_rank=3 imbalance_ratio=1.195007 max_load_reduction=111493 max_load_reduction_pct=0.850624 moved_to_echo=68737 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4092, 41, 0, 0, 4094, 3, 4081, 0, 4, 0, 3845, 258, 2189, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1851, 0, 0, 0, 1, 2, 0, 0, 4092, 209, 1626, 4096, 0, 0, 3923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 28, 64, 4, 0, 0, 4096, 1851, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 777, 2500, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 623, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 404, 0, 1, 107, 0, 3854, 0, 0, 3518, 0, 0, 255, 0, 0, 0, 4095, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 16, 3532, 0, 0, 3869, 117, 4096, 113, 0, 3, 3, 2560, 0, 3144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 381, 3439, 0, 2469, 414, 0, 0, 4095, 4091, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 27, 4096, 0, 4096, 0, 13, 0, 4094, 384, 3641, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18610, 15802, 14353, 16243, 12669, 21490, 14912, 16993] avg=16384.0000 max=21490 max_rank=5 min=12669 min_rank=4 imbalance_ratio=1.311646 max_load_reduction=109582 max_load_reduction_pct=0.836044 moved_to_echo=66064 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 30, 8, 3800, 552, 386, 0, 2, 2, 947, 8, 2176, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 1, 0, 169, 0, 0, 0, 4091, 196, 1046, 4010, 4, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 32, 43, 2, 0, 4079, 4081, 118, 4, 19, 5, 3639, 1, 75, 0, 3310, 1848, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3551, 4094, 0, 0, 1, 0, 4072, 1, 0, 13, 4096, 1, 0, 2, 0, 632, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3651, 4, 857, 2305, 0, 3509, 0, 0, 4095, 0, 0, 0, 0, 1422, 0, 4093, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 120, 21, 23, 23, 2, 4089, 0, 4012, 10, 0, 0, 1, 0, 7, 3140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 3455, 2, 3525, 6, 0, 0, 4086, 3398, 2164, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 4090, 10, 2285, 0, 1, 8, 3754, 1468, 3405, 631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12008, 17667, 17573, 16524, 20349, 11449, 19846, 15656] avg=16384.0000 max=20349 max_rank=4 min=11449 min_rank=5 imbalance_ratio=1.242004 max_load_reduction=110723 max_load_reduction_pct=0.844749 moved_to_echo=67300 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2839, 0, 0, 0, 3722, 0, 4083, 0, 0, 184, 0, 484, 2183, 3, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 3981, 3, 13, 0, 0, 2, 11, 0, 3829, 206, 3252, 3919, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 174, 47, 141, 3065, 4068, 1234, 0, 0, 0, 2365, 207, 1, 14, 4096, 1846, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 705, 4052, 4050, 0, 0, 0, 0, 4043, 124, 0, 0, 3771, 0, 0, 0, 0, 626, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1608, 8, 1443, 2611, 30, 3194, 0, 60, 3595, 11, 0, 0, 0, 9, 3, 3963, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 3402, 0, 930, 1, 68, 4065, 0, 3268, 0, 1, 0, 423, 0, 37, 3100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 3452, 34, 3992, 0, 0, 0, 4069, 3489, 0, 0, 12, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 614, 0, 0, 65, 565, 60, 2, 0, 3720, 1, 1869, 15, 2, 635, 4008, 0, 3594, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13755, 19323, 17372, 17427, 16965, 15306, 15155, 15769] avg=16384.0000 max=19323 max_rank=1 min=13755 min_rank=0 imbalance_ratio=1.179382 max_load_reduction=111749 max_load_reduction_pct=0.852577 moved_to_echo=63195 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 6, 0, 0, 4076, 370, 3976, 0, 14, 0, 645, 369, 2189, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 3, 0, 0, 0, 5, 0, 0, 4084, 207, 4096, 4079, 0, 2, 3784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 1, 0, 0, 0, 4068, 4096, 10, 0, 5, 228, 4073, 4, 326, 13, 3722, 1851, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4096, 2, 0, 0, 0, 4084, 0, 0, 0, 4094, 0, 0, 17, 0, 618, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 3515, 176, 0, 2661, 0, 4078, 0, 4, 79, 1, 0, 0, 1, 0, 1, 4096, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1798, 51, 3176, 60, 0, 4093, 4, 4020, 13, 29, 27, 0, 20, 11, 3119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 3407, 0, 837, 48, 0, 0, 3363, 2826, 2579, 0, 2, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 303, 1, 0, 0, 4074, 0, 4044, 0, 0, 232, 4095, 645, 3652, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15790, 17211, 18541, 17048, 15074, 16421, 13317, 17670] avg=16384.0000 max=18541 max_rank=2 min=13317 min_rank=6 imbalance_ratio=1.131653 max_load_reduction=112531 max_load_reduction_pct=0.858543 moved_to_echo=62482 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[640, 21504, 128, 128, 128, 384, 128, 128, 0, 28288, 1664, 19328, 32512, 128, 128, 26880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[31488, 2304, 128, 128, 30848, 1664, 27136, 0, 128, 256, 16128, 3712, 17280, 384, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[384, 128, 256, 128, 256, 30208, 32768, 1664, 128, 128, 640, 18688, 256, 512, 128, 30464, 14592, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 18688, 256, 7296, 16512, 128, 22912, 128, 128, 27520, 128, 128, 384, 128, 1792, 128, 32512, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 256, 13952, 256, 20992, 128, 256, 32384, 640, 31616, 256, 1024, 128, 512, 4224, 128, 25088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[2176, 26624, 32640, 128, 128, 128, 128, 32640, 128, 128, 128, 31488, 128, 256, 128, 128, 4992, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 11264, 27264, 128, 19456, 768, 128, 0, 32000, 29568, 10368, 128, 128, 128, 256, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[768, 128, 0, 128, 1024, 128, 384, 384, 31488, 128, 27008, 128, 256, 896, 32384, 3072, 28928, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 719, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 30, 0, 0, 0, 4096, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 3339, 0, 0, 4096, 0, 4096, 0, 3971, 0, 1181, 1, 5, 0, 4096, 0, 0, 0, 0, 109, 0, 0, 4096, 4096, 0, 0, 0, 0, 4085, 0, 0, 21, 4092, 0, 3688, 0, 3747, 4095, 0, 2, 243, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 107, 4079, 0, 4, 0, 0, 0, 0, 0, 0, 3954, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 3462, 1, 1, 5, 0, 0, 0, 1, 4096, 0, 4, 4094, 0, 4091, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[31, 0, 4090, 0, 0, 147, 0, 10, 0, 85, 0, 4089, 66, 29, 6, 1681, 0, 0, 40, 3815, 36, 4037, 4019, 37, 4070, 60, 0, 0, 4071, 0, 4096, 0, 0, 2867, 0, 2704, 3899, 0, 4054, 0, 3948, 0, 3682, 0, 1207, 0, 3985, 106, 1, 0, 244, 1993, 0, 0, 4095, 4096, 0, 0, 610, 0, 4053, 0, 7, 1, 3733, 0, 3913, 119, 561, 3815, 0, 5, 63, 0, 0, 0, 5, 10, 843, 0, 1888, 2, 337, 0, 0, 3793, 0, 10, 0, 0, 0, 0, 1, 26, 15, 4087, 3955, 3, 2, 4096, 194, 0, 3899, 3278, 4085, 7, 0, 0, 3, 23, 234, 0, 293, 31, 0, 1, 0, 0, 4087, 0, 0, 4095, 3, 3473, 3, 14, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4, 0, 2, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 4096, 22, 4095, 4096, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 2, 2132, 0, 111, 4091, 0, 4027, 0, 3317, 0, 3094, 0, 73, 141, 4038, 0, 0, 0, 0, 0, 1, 301, 4096, 4066, 0, 0, 0, 0, 3999, 1, 0, 173, 3753, 2, 27, 0, 4089, 4096, 0, 7, 3771, 0, 0, 0, 0, 0, 149, 0, 308, 0, 83, 0, 0, 4093, 0, 0, 0, 111, 0, 0, 0, 0, 3989, 4096, 4038, 0, 0, 4096, 0, 0, 4096, 4026, 4096, 0, 0, 0, 0, 1, 3390, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[6, 0, 4090, 0, 0, 0, 129, 334, 0, 0, 0, 4096, 0, 0, 388, 30, 5, 0, 2, 4095, 0, 4085, 4083, 100, 4095, 0, 0, 0, 4053, 0, 4096, 0, 0, 0, 0, 1, 4092, 0, 4096, 0, 4046, 0, 3280, 0, 360, 2536, 4042, 441, 3, 0, 0, 2376, 0, 0, 4026, 4037, 0, 0, 849, 0, 2457, 0, 0, 0, 3792, 40, 3834, 4, 3942, 3665, 2, 177, 0, 1, 0, 0, 4, 93, 0, 0, 484, 0, 34, 12, 0, 4096, 1, 84, 23, 0, 0, 270, 0, 0, 282, 4096, 3947, 0, 1, 4096, 19, 0, 4096, 3800, 4017, 0, 0, 0, 0, 2, 4018, 0, 0, 22, 1, 0, 0, 0, 3729, 340, 6, 3969, 0, 3744, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 4096, 0, 1, 119, 30, 3, 964, 27, 14, 4085, 22, 0, 409, 0, 0, 0, 0, 4096, 0, 4072, 4094, 10, 4095, 4, 0, 0, 4094, 0, 4084, 0, 2, 4095, 0, 46, 3749, 1, 3625, 0, 3257, 0, 481, 2, 42, 5, 3989, 0, 0, 0, 0, 202, 0, 388, 4095, 4079, 35, 0, 0, 11, 4094, 0, 17, 0, 4048, 0, 0, 0, 4090, 4090, 0, 3, 274, 0, 0, 1, 0, 0, 35, 23, 1064, 0, 0, 0, 1, 4062, 5, 1467, 397, 0, 0, 0, 0, 14, 4034, 4083, 4041, 0, 0, 4095, 4, 0, 4092, 4036, 4056, 1, 2, 7, 2, 83, 4095, 0, 0, 0, 0, 0, 83, 0, 4088, 0, 0, 4064, 0, 4091, 2, 0, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2359, 18, 4, 2, 198, 367, 0, 0, 4081, 0, 0, 146, 3384, 0, 0, 19, 4084, 3, 4010, 4095, 0, 4043, 0, 0, 0, 4096, 0, 4087, 0, 1, 2778, 0, 51, 3929, 0, 4088, 0, 4069, 0, 4096, 1, 4, 1, 4084, 58, 5, 0, 103, 0, 48, 0, 4007, 4009, 16, 0, 99, 0, 4070, 0, 0, 0, 3048, 0, 3369, 7, 2292, 4096, 26, 1, 1248, 150, 1, 1, 0, 30, 33, 0, 788, 93, 45, 6, 3, 4087, 1, 47, 2, 108, 2, 0, 0, 2, 3098, 3973, 4096, 0, 0, 3922, 0, 0, 4095, 4069, 4096, 0, 0, 0, 1, 45, 271, 3, 0, 4, 0, 5, 0, 6, 4067, 9, 4, 4023, 0, 1190, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 128, 4093, 46, 0, 0, 529, 0, 1, 0, 0, 4017, 0, 2, 0, 0, 0, 0, 0, 4074, 1, 4053, 4031, 2, 4000, 12, 0, 0, 4091, 0, 4091, 1, 10, 4096, 1, 1165, 4061, 40, 4081, 0, 3123, 0, 4096, 0, 0, 60, 3963, 0, 0, 0, 0, 114, 4, 0, 4056, 4061, 34, 0, 0, 80, 3905, 21, 0, 27, 3545, 78, 0, 580, 3914, 4094, 4, 713, 2929, 0, 7, 0, 0, 1, 4, 45, 772, 0, 2, 27, 0, 4041, 0, 182, 470, 0, 0, 0, 0, 0, 0, 4092, 4090, 0, 0, 4085, 0, 16, 4075, 3821, 4094, 0, 1, 42, 8, 0, 2929, 12, 3, 0, 0, 2, 0, 0, 4096, 2, 60, 4080, 0, 3981, 2, 3, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1, 6, 4096, 0, 0, 0, 0, 11, 512, 0, 0, 4095, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 4096, 4091, 0, 4088, 2, 0, 1, 4096, 0, 4095, 0, 49, 3558, 1, 0, 3737, 1, 3235, 0, 3922, 0, 638, 0, 21, 0, 3513, 6, 0, 0, 0, 0, 19, 0, 4083, 4094, 0, 0, 0, 2, 4096, 0, 0, 979, 4096, 0, 220, 0, 4064, 4079, 3, 3157, 3829, 0, 0, 0, 0, 0, 11, 0, 191, 0, 31, 0, 0, 4095, 0, 118, 0, 111, 0, 0, 2, 0, 1152, 4095, 4096, 0, 3, 4090, 0, 0, 4087, 4096, 4084, 0, 1, 0, 0, 0, 3839, 0, 0, 38, 0, 0, 0, 0, 4094, 2, 0, 4089, 0, 4054, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 719, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 30, 4096, 2497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 4096, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3339, 0, 0, 2259, 0, 4096, 0, 3971, 0, 1181, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 4096, 4096, 0, 0, 0, 0, 4085, 0, 0, 21, 2558, 745, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3688, 0, 3747, 4095, 0, 2, 243, 0, 0, 0, 0, 4, 2, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 4079, 0, 4, 0, 0, 0, 0, 0, 0, 3954, 4096, 3397, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1538, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3462, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 1, 4096, 0, 4, 4094, 0, 4091, 0, 0, 0, 0, 4068, 632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15550, 16450, 14852, 15738, 16965, 17236, 17289, 16992] avg=16384.0000 max=17289 max_rank=6 min=14852 min_rank=2 imbalance_ratio=1.055237 max_load_reduction=113783 max_load_reduction_pct=0.868095 moved_to_echo=68482 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4, 0, 2, 0, 0, 4096, 0, 0, 10, 0, 4096, 2498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 22, 4095, 4096, 0, 4095, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2132, 0, 111, 2256, 0, 4027, 0, 3317, 0, 3094, 0, 73, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 301, 4096, 4066, 0, 0, 0, 0, 3999, 1, 0, 173, 2558, 744, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3753, 2, 27, 0, 4089, 4096, 0, 7, 3771, 0, 0, 0, 0, 0, 149, 0, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 308, 0, 83, 0, 0, 4093, 0, 0, 0, 111, 0, 0, 0, 0, 3989, 4096, 3397, 1598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 1538, 0, 0, 4096, 4026, 0, 0, 0, 0, 0, 1, 3390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4009, 632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14802, 16471, 15153, 15968, 16985, 17675, 17089, 16929] avg=16384.0000 max=17675 max_rank=5 min=14802 min_rank=0 imbalance_ratio=1.078796 max_load_reduction=113397 max_load_reduction_pct=0.865150 moved_to_echo=68678 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 4090, 0, 0, 0, 129, 334, 0, 0, 0, 4096, 0, 0, 388, 30, 4096, 2449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 4095, 0, 4085, 4083, 100, 4095, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2257, 0, 4096, 0, 4046, 0, 3280, 0, 360, 2536, 0, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2376, 0, 0, 4026, 4037, 0, 0, 849, 0, 2457, 0, 0, 0, 2558, 744, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3792, 40, 3834, 4, 3942, 3665, 2, 177, 0, 1, 0, 0, 4, 93, 0, 0, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 0, 34, 12, 0, 4096, 1, 84, 23, 0, 0, 270, 0, 0, 282, 4096, 3361, 1568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 0, 1, 1538, 19, 0, 4096, 3800, 0, 0, 0, 0, 0, 2, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 1, 0, 0, 0, 3729, 340, 6, 3969, 0, 3744, 0, 0, 0, 0, 4014, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15618, 16532, 17017, 17078, 16645, 14311, 17421, 16450] avg=16384.0000 max=17421 max_rank=6 min=14311 min_rank=5 imbalance_ratio=1.063293 max_load_reduction=113651 max_load_reduction_pct=0.867088 moved_to_echo=64827 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 2359, 18, 4, 2, 198, 367, 0, 0, 4081, 0, 0, 146, 3384, 4087, 2497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4084, 3, 4010, 4095, 0, 4043, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2778, 0, 51, 2166, 0, 4088, 0, 4069, 0, 4096, 1, 4, 1, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 103, 0, 48, 0, 4007, 4009, 16, 0, 99, 0, 4070, 0, 0, 0, 2449, 715, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3048, 0, 3369, 7, 2292, 4096, 26, 1, 1248, 150, 1, 1, 0, 30, 33, 0, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 788, 93, 45, 6, 3, 4087, 1, 47, 2, 108, 2, 0, 0, 2, 3098, 3973, 3397, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1473, 0, 0, 4095, 4069, 0, 0, 0, 0, 1, 45, 271, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 0, 6, 4067, 9, 4, 4023, 0, 1190, 0, 0, 0, 0, 4056, 632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21239, 16321, 17313, 15549, 15350, 17251, 14053, 13996] avg=16384.0000 max=21239 max_rank=0 min=13996 min_rank=7 imbalance_ratio=1.296326 max_load_reduction=109833 max_load_reduction_pct=0.837959 moved_to_echo=60650 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 128, 4093, 46, 0, 0, 529, 0, 1, 0, 0, 4017, 0, 2, 0, 0, 4091, 2496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 1, 4053, 4031, 2, 4000, 12, 0, 0, 67, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 1, 1165, 2239, 40, 4081, 0, 3123, 0, 4096, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 4, 0, 4056, 4061, 34, 0, 0, 80, 3905, 21, 0, 27, 2551, 739, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3545, 78, 0, 580, 3914, 4094, 4, 713, 2929, 0, 7, 0, 0, 1, 4, 45, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 772, 0, 2, 27, 0, 4041, 0, 182, 470, 0, 0, 0, 0, 0, 0, 4092, 3393, 1598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 1534, 0, 16, 4075, 3821, 0, 0, 1, 42, 8, 0, 2929, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 0, 4096, 2, 60, 4080, 0, 3981, 2, 3, 0, 0, 3936, 631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15403, 16241, 18911, 15619, 16997, 14577, 16528, 16796] avg=16384.0000 max=18911 max_rank=2 min=14577 min_rank=5 imbalance_ratio=1.154236 max_load_reduction=112161 max_load_reduction_pct=0.855721 moved_to_echo=64898 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[31, 0, 4090, 0, 0, 147, 0, 10, 0, 85, 0, 4089, 66, 29, 6, 1681, 4096, 2491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 3815, 36, 4037, 4019, 37, 4070, 60, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2867, 0, 2704, 2150, 0, 4054, 0, 3948, 0, 3682, 0, 1207, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 244, 1993, 0, 0, 4095, 4096, 0, 0, 610, 0, 4053, 0, 7, 1, 2558, 709, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3733, 0, 3913, 119, 561, 3815, 0, 5, 63, 0, 0, 0, 5, 10, 843, 0, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1888, 2, 337, 0, 0, 3793, 0, 10, 0, 0, 0, 0, 1, 26, 15, 4087, 3376, 1594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 3, 2, 1538, 194, 0, 3899, 3278, 0, 7, 0, 0, 3, 23, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 31, 0, 1, 0, 0, 4087, 0, 0, 4095, 3, 3473, 3, 14, 0, 0, 3957, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16821, 16181, 20718, 18395, 14107, 15129, 13136, 16585] avg=16384.0000 max=20718 max_rank=2 min=13136 min_rank=6 imbalance_ratio=1.264526 max_load_reduction=110354 max_load_reduction_pct=0.841934 moved_to_echo=58957 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4096, 0, 1, 119, 30, 3, 964, 27, 14, 4085, 22, 0, 409, 0, 4084, 2473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4072, 4094, 10, 4095, 4, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 46, 2060, 1, 3625, 0, 3257, 0, 481, 2, 42, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 388, 4095, 4079, 35, 0, 0, 11, 4094, 0, 17, 0, 2560, 685, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 4090, 4090, 0, 3, 274, 0, 0, 1, 0, 0, 35, 23, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 0, 0, 0, 1, 4062, 5, 1467, 397, 0, 0, 0, 0, 14, 4034, 4083, 3397, 1583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 1535, 4, 0, 4092, 4036, 0, 1, 2, 7, 2, 83, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 4088, 0, 0, 4064, 0, 4091, 2, 0, 0, 4, 3961, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16328, 16435, 13616, 16194, 13568, 20107, 17898, 16926] avg=16384.0000 max=20107 max_rank=5 min=13568 min_rank=4 imbalance_ratio=1.227234 max_load_reduction=110965 max_load_reduction_pct=0.846596 moved_to_echo=68499 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 6, 4096, 0, 0, 0, 0, 11, 512, 0, 0, 4095, 0, 5, 0, 0, 4095, 2490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4091, 0, 4088, 2, 0, 1, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 3558, 1, 0, 2061, 1, 3235, 0, 3922, 0, 638, 0, 21, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4083, 4094, 0, 0, 0, 2, 4096, 0, 0, 979, 2554, 680, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 220, 0, 4064, 4079, 3, 3157, 3829, 0, 0, 0, 0, 0, 11, 0, 996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 31, 0, 0, 4095, 0, 118, 0, 111, 0, 0, 2, 0, 1152, 4095, 3397, 1594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 1536, 0, 0, 4087, 4096, 0, 0, 1, 0, 0, 0, 3839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 4094, 2, 0, 4089, 0, 4054, 0, 0, 0, 0, 3489, 632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15311, 16441, 13492, 16531, 20455, 14786, 17658, 16398] avg=16384.0000 max=20455 max_rank=4 min=13492 min_rank=2 imbalance_ratio=1.248474 max_load_reduction=110617 max_load_reduction_pct=0.843941 moved_to_echo=69297 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32384, 128, 128, 12288, 256, 128, 32640, 31232, 0, 128, 128, 128, 128, 256, 22272, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[30208, 128, 15104, 768, 26752, 32128, 128, 4096, 12416, 256, 128, 128, 128, 256, 1152, 128, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 22912, 128, 4096, 17536, 128, 31360, 0, 29696, 0, 20608, 128, 1792, 2816, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 256, 32768, 3200, 128, 384, 768, 640, 1920, 128, 128, 32768, 128, 128, 1024, 5248, 32768, 19968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[5504, 128, 640, 128, 128, 32384, 128, 1920, 896, 384, 128, 384, 128, 128, 16640, 32640, 27136, 12800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 32512, 128, 32640, 32640, 256, 32640, 128, 0, 128, 640, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 384, 4864, 128, 768, 32640, 32640, 128, 0, 1664, 128, 30848, 128, 128, 1280, 20352, 5888, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[384, 128, 128, 128, 128, 128, 32384, 384, 128, 32512, 128, 28800, 128, 128, 0, 128, 31616, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[156, 4096, 3963, 0, 0, 0, 13, 0, 2670, 3985, 0, 0, 0, 7, 0, 1485, 0, 0, 1, 130, 307, 4085, 136, 3811, 0, 17, 0, 34, 0, 3, 167, 3891, 832, 71, 0, 2, 12, 3882, 4001, 3, 0, 3047, 2232, 0, 417, 2, 346, 9, 0, 0, 0, 0, 0, 3109, 13, 4096, 4081, 0, 0, 3950, 0, 4072, 25, 1141, 3415, 62, 322, 0, 0, 2, 4091, 13, 0, 269, 0, 0, 0, 0, 0, 963, 4092, 4086, 9, 338, 0, 2672, 0, 0, 3839, 4094, 54, 0, 2, 3905, 2113, 3776, 591, 3, 3996, 2, 48, 4033, 2, 0, 0, 243, 0, 171, 522, 2501, 0, 4096, 3, 3, 0, 1106, 9, 3, 1067, 0, 0, 0, 0, 2452, 3909, 67, 3828, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[2, 3902, 4084, 9, 1, 1, 0, 285, 1, 4090, 313, 37, 86, 1, 93, 129, 287, 0, 3483, 108, 0, 1770, 143, 4081, 0, 1, 0, 0, 1, 87, 3969, 4094, 0, 0, 0, 0, 574, 2659, 3788, 0, 0, 4093, 0, 0, 0, 7, 0, 0, 0, 0, 378, 0, 585, 2735, 0, 4066, 4081, 0, 10, 3624, 4, 3007, 3575, 0, 4048, 0, 3071, 3, 312, 0, 4096, 0, 0, 2, 379, 0, 0, 6, 0, 0, 3812, 3957, 0, 4, 3841, 1842, 0, 23, 3953, 4005, 31, 0, 0, 4086, 0, 0, 4086, 7, 4090, 0, 15, 4023, 0, 16, 0, 822, 20, 119, 244, 3399, 0, 4096, 0, 0, 47, 76, 0, 12, 0, 0, 0, 0, 97, 0, 4070, 27, 4091, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 3940, 0, 0, 0, 0, 0, 116, 4096, 1, 0, 40, 0, 0, 2, 0, 0, 7, 8, 0, 2082, 2857, 4096, 0, 0, 0, 0, 0, 0, 365, 3258, 2715, 0, 0, 0, 3, 4096, 4092, 0, 0, 4095, 610, 0, 0, 0, 0, 0, 3, 0, 0, 11, 0, 3994, 0, 4096, 4096, 0, 59, 4095, 2, 3060, 2534, 0, 8, 0, 3207, 1, 0, 195, 4093, 0, 0, 2, 0, 0, 0, 1, 0, 697, 3791, 4096, 3565, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 1552, 3666, 4096, 0, 4094, 16, 0, 4095, 1, 0, 0, 1, 0, 0, 0, 3977, 0, 4096, 0, 3, 0, 1, 0, 0, 1, 0, 0, 0, 1, 2829, 4092, 0, 4082, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 3401, 0, 0, 0, 0, 0, 1842, 4096, 0, 0, 2911, 0, 8, 2, 0, 0, 4096, 56, 4, 4096, 3228, 4096, 3, 0, 0, 47, 0, 0, 1914, 0, 53, 0, 0, 0, 0, 3741, 3217, 0, 0, 2862, 1860, 376, 0, 2, 57, 10, 0, 0, 0, 0, 1238, 3944, 4, 3992, 4039, 0, 0, 4096, 0, 311, 12, 11, 834, 0, 4092, 0, 69, 3954, 4093, 83, 0, 0, 3, 0, 0, 32, 0, 1055, 4068, 4095, 2662, 113, 299, 0, 0, 699, 3938, 4085, 1, 0, 0, 3940, 1, 15, 711, 0, 3712, 53, 1, 4096, 0, 0, 0, 0, 17, 36, 10, 4039, 0, 4096, 0, 358, 0, 3713, 0, 0, 0, 0, 0, 0, 0, 188, 4096, 0, 4093, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 1621, 0, 2, 0, 0, 0, 94, 4096, 1, 0, 253, 0, 14, 4079, 4, 0, 3845, 3909, 0, 771, 3557, 4096, 0, 0, 2, 12, 1, 19, 3, 18, 3153, 0, 0, 0, 413, 4096, 3600, 0, 0, 4083, 37, 0, 0, 0, 0, 0, 4, 0, 0, 2126, 56, 3684, 0, 4096, 4096, 0, 0, 4096, 0, 4062, 2309, 0, 99, 0, 4095, 0, 0, 0, 4093, 0, 0, 2, 0, 0, 0, 1, 0, 1977, 4056, 4094, 0, 0, 110, 161, 0, 0, 4093, 3970, 0, 0, 0, 4096, 903, 3952, 302, 0, 4096, 1, 0, 4096, 0, 0, 0, 55, 0, 0, 28, 4089, 0, 4093, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 3977, 4095, 22, 20, 0, 35, 127, 29, 4067, 4, 0, 0, 9, 520, 794, 0, 15, 207, 2349, 1, 0, 156, 4083, 11, 2, 6, 0, 72, 4, 53, 0, 4072, 1, 26, 0, 90, 4002, 3504, 0, 1, 4066, 315, 0, 0, 0, 1, 30, 0, 1, 0, 3991, 2, 834, 0, 4089, 4094, 1, 2, 4060, 4, 4023, 543, 27, 0, 0, 4066, 177, 314, 2, 3964, 0, 0, 59, 274, 7, 0, 312, 0, 0, 3964, 4096, 4020, 0, 3684, 112, 5, 7, 4034, 3798, 15, 0, 0, 4081, 3895, 3526, 20, 10, 2740, 0, 2, 4033, 0, 0, 0, 53, 0, 0, 7, 4089, 0, 3945, 91, 3737, 0, 14, 40, 0, 31, 38, 0, 0, 0, 1309, 3993, 0, 4071, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[7, 4051, 3989, 0, 0, 1, 0, 1, 4, 4093, 36, 0, 7, 0, 0, 0, 379, 0, 2, 3657, 0, 1768, 3743, 4091, 0, 0, 0, 4, 0, 0, 0, 270, 597, 0, 0, 0, 445, 4095, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 341, 0, 3549, 1597, 1, 4096, 4067, 0, 0, 4059, 0, 1979, 3102, 0, 946, 0, 4094, 0, 0, 0, 4093, 0, 0, 39, 0, 0, 0, 0, 4, 4089, 3958, 4096, 1432, 4, 132, 247, 22, 0, 4092, 2158, 0, 0, 2, 4089, 16, 3404, 4096, 0, 3471, 52, 0, 4096, 0, 5, 0, 390, 2, 0, 57, 4051, 0, 4027, 0, 36, 0, 0, 3, 0, 178, 1, 0, 0, 0, 3358, 4084, 0, 4024, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[18, 4022, 4093, 0, 0, 0, 0, 79, 155, 4085, 0, 0, 9, 0, 4, 100, 0, 0, 0, 4039, 0, 3524, 4095, 4092, 0, 7, 5, 0, 24, 27, 0, 484, 618, 12, 0, 0, 0, 4094, 4071, 0, 1, 3830, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1239, 2253, 4, 4090, 4067, 9, 341, 4095, 0, 5, 3011, 6, 339, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 16, 2, 0, 89, 3282, 4096, 4072, 4041, 0, 1, 1062, 0, 0, 4080, 4096, 150, 0, 20, 4041, 989, 3659, 3355, 0, 4039, 32, 2, 4092, 0, 27, 3, 438, 0, 0, 0, 3988, 3, 4081, 18, 13, 0, 0, 0, 0, 19, 0, 0, 0, 2, 3, 4057, 13, 4051, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[156, 4096, 3963, 0, 0, 0, 13, 0, 2670, 3985, 0, 0, 0, 7, 0, 1485, 1325, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 130, 307, 4085, 136, 3811, 0, 17, 0, 34, 0, 3, 167, 3891, 1273, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 832, 71, 0, 2, 12, 3882, 4001, 3, 0, 3047, 2232, 0, 417, 2, 346, 9, 1786, 607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3109, 13, 0, 3260, 0, 0, 3950, 0, 4072, 25, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3415, 62, 322, 0, 0, 2, 4091, 13, 0, 269, 0, 0, 0, 0, 0, 963, 2791, 2490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 9, 338, 0, 2672, 0, 0, 728, 4094, 54, 0, 2, 0, 2113, 3776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 3, 3996, 2, 48, 2153, 2, 0, 0, 243, 0, 171, 522, 2501, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 1106, 9, 3, 1067, 0, 0, 0, 0, 2452, 3909, 67, 3828, 0, 4096, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18814, 14676, 17249, 15570, 14418, 17878, 14328, 18139] avg=16384.0000 max=18814 max_rank=0 min=14328 min_rank=6 imbalance_ratio=1.148315 max_load_reduction=112258 max_load_reduction_pct=0.856461 moved_to_echo=64763 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 3902, 4084, 9, 1, 1, 0, 285, 1, 4090, 313, 37, 86, 1, 93, 129, 1364, 1164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 3483, 108, 0, 1770, 143, 4081, 0, 1, 0, 0, 1, 87, 3969, 4094, 1269, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 2659, 3788, 0, 0, 4093, 0, 0, 0, 7, 0, 0, 1839, 606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 585, 2735, 0, 0, 3260, 0, 10, 3624, 4, 3007, 3575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 3071, 3, 312, 0, 4096, 0, 0, 2, 379, 0, 0, 6, 0, 0, 2922, 2410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3812, 0, 0, 4, 3841, 1842, 0, 23, 750, 4005, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 7, 4090, 0, 15, 2148, 0, 16, 0, 822, 20, 119, 244, 3399, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 76, 0, 12, 0, 0, 0, 0, 97, 0, 4070, 27, 4091, 0, 4066, 1547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15562, 20114, 13566, 17178, 17249, 14308, 19062, 14033] avg=16384.0000 max=20114 max_rank=1 min=13566 min_rank=2 imbalance_ratio=1.227661 max_load_reduction=110958 max_load_reduction_pct=0.846542 moved_to_echo=64652 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 3401, 0, 0, 0, 0, 0, 1842, 4096, 0, 0, 2911, 0, 8, 2, 1359, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 56, 4, 4096, 3228, 4096, 3, 0, 0, 47, 0, 0, 1914, 0, 1292, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 3741, 3217, 0, 0, 2862, 1860, 376, 0, 2, 57, 10, 1832, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1238, 3944, 4, 0, 3226, 0, 0, 4096, 0, 311, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 834, 0, 4092, 0, 69, 3954, 4093, 83, 0, 0, 3, 0, 0, 32, 0, 1055, 2816, 2495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 2662, 113, 299, 0, 0, 699, 747, 4085, 1, 0, 0, 0, 1, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 711, 0, 3712, 53, 1, 2187, 0, 0, 0, 0, 17, 36, 10, 4039, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 0, 3713, 0, 0, 0, 0, 0, 0, 0, 188, 4096, 0, 4093, 1, 3992, 1600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18839, 19645, 14627, 12842, 19526, 12690, 14862, 18041] avg=16384.0000 max=19645 max_rank=1 min=12690 min_rank=5 imbalance_ratio=1.199036 max_load_reduction=111427 max_load_reduction_pct=0.850121 moved_to_echo=65119 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 3940, 0, 0, 0, 0, 0, 116, 4096, 1, 0, 40, 0, 0, 2, 1413, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 0, 2082, 2857, 4096, 0, 0, 0, 0, 0, 0, 365, 3258, 1292, 824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2715, 0, 0, 0, 3, 4096, 4092, 0, 0, 4095, 610, 0, 0, 0, 0, 0, 1906, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 11, 0, 3994, 0, 0, 3272, 0, 59, 4095, 2, 3060, 2534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3207, 1, 0, 195, 4093, 0, 0, 2, 0, 0, 0, 1, 0, 697, 2929, 2495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3791, 0, 3565, 0, 0, 0, 0, 0, 777, 4096, 0, 0, 0, 0, 1552, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 16, 0, 2186, 1, 0, 0, 1, 0, 0, 0, 3977, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 1, 0, 0, 0, 1, 2829, 4092, 0, 4082, 0, 4096, 1601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14871, 14789, 18134, 17030, 13628, 17447, 18467, 16706] avg=16384.0000 max=18467 max_rank=6 min=13628 min_rank=4 imbalance_ratio=1.127136 max_load_reduction=112605 max_load_reduction_pct=0.859108 moved_to_echo=66248 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1621, 0, 2, 0, 0, 0, 94, 4096, 1, 0, 253, 0, 14, 4079, 1412, 1168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3845, 3909, 0, 771, 3557, 4096, 0, 0, 2, 12, 1, 19, 3, 18, 1292, 824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3153, 0, 0, 0, 413, 4096, 3600, 0, 0, 4083, 37, 0, 0, 0, 0, 0, 1905, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2126, 56, 3684, 0, 0, 3272, 0, 0, 4096, 0, 4062, 2309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 4095, 0, 0, 0, 4093, 0, 0, 2, 0, 0, 0, 1, 0, 1977, 2928, 2495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 110, 161, 0, 0, 776, 3970, 0, 0, 0, 0, 903, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 302, 0, 4096, 1, 0, 2187, 0, 0, 0, 55, 0, 0, 28, 4089, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16836, 18353, 17904, 19609, 15690, 13928, 14851, 13901] avg=16384.0000 max=19609 max_rank=3 min=13901 min_rank=7 imbalance_ratio=1.196838 max_load_reduction=111463 max_load_reduction_pct=0.850395 moved_to_echo=58370 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3977, 4095, 22, 20, 0, 35, 127, 29, 4067, 4, 0, 0, 9, 520, 794, 1392, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 207, 2349, 1, 0, 156, 4083, 11, 2, 6, 0, 72, 4, 53, 0, 1273, 824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 1, 26, 0, 90, 4002, 3504, 0, 1, 4066, 315, 0, 0, 0, 1, 30, 1877, 607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3991, 2, 834, 0, 0, 3270, 1, 2, 4060, 4, 4023, 543, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 177, 314, 2, 3964, 0, 0, 59, 274, 7, 0, 312, 0, 0, 2918, 2495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 0, 4020, 0, 3684, 112, 5, 7, 765, 3798, 15, 0, 0, 0, 3895, 3526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 10, 2740, 0, 2, 2153, 0, 0, 0, 53, 0, 0, 7, 4089, 0, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 3737, 0, 14, 40, 0, 31, 38, 0, 0, 0, 1309, 3993, 0, 4071, 0, 4089, 1601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16254, 9056, 18592, 16758, 14588, 23791, 13019, 19014] avg=16384.0000 max=23791 max_rank=5 min=9056 min_rank=1 imbalance_ratio=1.452087 max_load_reduction=107281 max_load_reduction_pct=0.818489 moved_to_echo=70412 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[18, 4022, 4093, 0, 0, 0, 0, 79, 155, 4085, 0, 0, 9, 0, 4, 100, 1412, 1153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 3524, 4095, 4092, 0, 7, 5, 0, 24, 27, 0, 484, 1296, 822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 618, 12, 0, 0, 0, 4094, 4071, 0, 1, 3830, 0, 0, 0, 0, 0, 1, 1903, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1239, 2253, 4, 0, 3245, 9, 341, 4095, 0, 5, 3011, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 16, 2, 0, 89, 3282, 2888, 2481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4041, 0, 1, 1062, 0, 0, 765, 4096, 150, 0, 20, 0, 989, 3659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3355, 0, 4039, 32, 2, 2177, 0, 27, 3, 438, 0, 0, 0, 3988, 3, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 13, 0, 0, 0, 0, 19, 0, 0, 0, 2, 3, 4057, 13, 4051, 0, 4090, 1591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15130, 18415, 15149, 14208, 17289, 18879, 18145, 13857] avg=16384.0000 max=18879 max_rank=5 min=13857 min_rank=7 imbalance_ratio=1.152283 max_load_reduction=112193 max_load_reduction_pct=0.855965 moved_to_echo=68170 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[7, 4051, 3989, 0, 0, 1, 0, 1, 4, 4093, 36, 0, 7, 0, 0, 0, 1412, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 2, 3657, 0, 1768, 3743, 4091, 0, 0, 0, 4, 0, 0, 0, 270, 1292, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 597, 0, 0, 0, 445, 4095, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 1904, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 0, 3549, 1597, 1, 0, 3249, 0, 0, 4059, 0, 1979, 3102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 946, 0, 4094, 0, 0, 0, 4093, 0, 0, 39, 0, 0, 0, 0, 4, 4089, 2924, 2495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 0, 1432, 4, 132, 247, 22, 0, 776, 2158, 0, 0, 2, 0, 16, 3404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3471, 52, 0, 2187, 0, 5, 0, 390, 2, 0, 57, 4051, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 3, 0, 178, 1, 0, 0, 0, 3358, 4084, 0, 4024, 0, 4096, 1601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14766, 16024, 15851, 17877, 18684, 12151, 18338, 17381] avg=16384.0000 max=18684 max_rank=4 min=12151 min_rank=5 imbalance_ratio=1.140381 max_load_reduction=112388 max_load_reduction_pct=0.857452 moved_to_echo=66554 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[17280, 128, 30336, 256, 128, 17408, 128, 128, 128, 2048, 128, 384, 896, 30208, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 32384, 29312, 128, 128, 128, 128, 512, 4992, 32640, 384, 128, 3328, 128, 640, 6656, 11136, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[31872, 0, 15744, 512, 8192, 6144, 128, 768, 6144, 30336, 256, 0, 128, 0, 9472, 22016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9728, 128, 27136, 256, 768, 4224, 32640, 128, 0, 384, 768, 128, 128, 384, 128, 12160, 23168, 19968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[768, 128, 11648, 14336, 384, 18176, 17920, 32512, 128, 128, 128, 128, 128, 256, 6528, 12032, 10368, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[12160, 128, 128, 128, 1664, 30720, 30464, 128, 128, 30208, 5120, 384, 512, 128, 512, 128, 14976, 4992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 768, 6144, 6784, 22272, 128, 0, 26112, 128, 512, 32128, 128, 20608, 15232, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 4224, 128, 4992, 128, 128, 1408, 128, 0, 0, 128, 10240, 32512, 128, 32384, 128, 32640, 12800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 4067, 0, 0, 7, 4018, 0, 6, 265, 26, 0, 4046, 4079, 3838, 4069, 4096, 497, 0, 11, 0, 11, 0, 0, 63, 7, 0, 3650, 0, 0, 0, 0, 120, 3939, 56, 4096, 0, 3872, 0, 18, 0, 4067, 314, 512, 0, 520, 8, 4, 91, 2083, 0, 0, 3647, 3962, 0, 4082, 0, 4027, 1778, 9, 4090, 0, 3861, 4096, 0, 8, 1, 575, 0, 3904, 1052, 4030, 0, 4095, 0, 70, 0, 0, 29, 0, 0, 85, 0, 3835, 21, 0, 4094, 4096, 0, 0, 3810, 1, 0, 75, 0, 132, 22, 0, 4009, 90, 662, 0, 0, 0, 0, 0, 0, 0, 3331, 2, 21, 7, 1, 265, 2, 272, 0, 4, 2176, 69, 4073, 0, 4096, 0, 0, 0, 0, 0, 48] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 4094, 11, 0, 0, 1, 0, 0, 1511, 49, 0, 4094, 4005, 4095, 2, 2802, 4070, 0, 0, 0, 0, 0, 74, 0, 0, 1, 4059, 14, 0, 3, 0, 25, 3848, 0, 4095, 234, 0, 0, 0, 0, 3906, 3778, 3588, 1625, 0, 298, 19, 33, 3300, 0, 0, 4096, 4095, 0, 4096, 0, 4096, 57, 3515, 72, 0, 669, 4096, 0, 20, 0, 0, 32, 3733, 911, 4095, 0, 4095, 0, 0, 0, 0, 0, 12, 0, 0, 0, 3963, 3, 1, 4075, 4096, 0, 0, 3974, 2, 6, 0, 0, 203, 0, 0, 3806, 0, 4096, 0, 0, 0, 0, 0, 729, 0, 6, 202, 0, 2, 314, 3, 396, 1061, 0, 0, 4096, 141, 4096, 422, 4000, 1, 0, 54, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4, 0, 0, 1, 0, 0, 4096, 4091, 4096, 3799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 4070, 0, 4096, 0, 0, 0, 0, 0, 4096, 513, 37, 4096, 0, 0, 0, 0, 3858, 0, 0, 4096, 4096, 0, 4091, 0, 4096, 2700, 0, 1757, 0, 1895, 4030, 0, 472, 0, 0, 0, 4089, 4070, 4095, 211, 4096, 0, 0, 1, 0, 0, 38, 0, 2, 0, 4096, 172, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4087, 0, 0, 0, 0, 0, 4, 0, 84, 3, 0, 0, 0, 4000, 3, 1145, 0, 0, 4019, 0, 4096, 1, 4096, 0, 0, 8, 0, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1, 4093, 0, 0, 29, 375, 6, 0, 3, 0, 15, 4069, 4054, 4090, 3983, 1317, 0, 2, 0, 13, 9, 0, 0, 3, 52, 0, 4096, 0, 2, 0, 26, 4, 4094, 13, 4049, 2, 2881, 1, 0, 0, 3997, 3882, 2205, 5, 7, 2, 0, 0, 3625, 0, 0, 4026, 4087, 1, 4095, 0, 4090, 0, 1, 4090, 6, 4087, 1629, 1, 3824, 0, 0, 0, 2470, 334, 4078, 2961, 4081, 0, 0, 1, 2, 0, 0, 210, 0, 76, 3916, 0, 0, 3871, 4074, 0, 5, 4096, 1, 3, 0, 0, 0, 0, 0, 4092, 64, 4096, 1, 5, 46, 0, 0, 3, 16, 0, 0, 0, 0, 977, 1382, 235, 547, 4, 0, 293, 0, 4094, 0, 4091, 0, 2, 1, 1, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 4003, 74, 0, 0, 355, 1, 0, 0, 1, 0, 4089, 3935, 4093, 3989, 3333, 1419, 14, 0, 0, 0, 2, 791, 0, 2, 0, 3559, 0, 0, 0, 0, 1233, 3641, 0, 3112, 118, 1299, 0, 0, 0, 3269, 1392, 4092, 491, 0, 0, 2, 0, 3998, 0, 58, 4023, 4091, 0, 3835, 0, 4062, 11, 568, 867, 0, 4039, 4089, 0, 4014, 34, 0, 0, 4091, 2518, 4096, 0, 4096, 11, 0, 0, 0, 0, 1, 0, 8, 0, 3982, 0, 68, 4021, 3780, 0, 12, 3726, 119, 0, 0, 0, 4, 0, 0, 4096, 0, 79, 0, 0, 167, 0, 2, 23, 0, 94, 9, 0, 0, 2968, 16, 6, 1, 0, 0, 0, 218, 3970, 356, 4095, 7, 0, 1876, 534, 0, 24] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 1, 0, 0, 0, 14, 0, 4096, 3985, 4096, 3573, 3802, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4096, 0, 0, 0, 0, 14, 4096, 0, 4031, 0, 991, 0, 0, 0, 2516, 4071, 4096, 0, 0, 0, 0, 0, 3966, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 3535, 0, 4067, 2545, 0, 722, 0, 0, 0, 3998, 3056, 4096, 910, 4089, 0, 0, 30, 0, 0, 4, 0, 0, 1, 4095, 6, 0, 3839, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 3695, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 184, 1347, 0, 1235, 280, 128, 621, 152, 4096, 0, 4096, 0, 0, 27, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 3, 0, 0, 0, 0, 162, 8, 39, 3980, 4014, 4089, 0, 318, 45, 2, 0, 5, 0, 0, 153, 415, 6, 0, 4074, 113, 1, 11, 0, 3562, 4093, 0, 4068, 1, 72, 0, 0, 0, 4096, 2841, 1450, 0, 427, 0, 0, 1, 4034, 0, 0, 3797, 4092, 10, 4094, 0, 4026, 4010, 2, 2131, 3, 3763, 1509, 4, 6, 60, 3, 0, 4092, 1, 3697, 4096, 4093, 1, 0, 15, 23, 0, 11, 0, 0, 179, 3939, 0, 5, 2687, 2860, 0, 28, 4004, 69, 0, 0, 25, 2, 0, 58, 3990, 7, 4081, 0, 0, 0, 0, 0, 0, 0, 1683, 63, 0, 0, 801, 3328, 6, 3772, 259, 1152, 0, 0, 4085, 0, 4071, 0, 0, 0, 9, 191, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 4095, 5, 10, 0, 403, 0, 0, 18, 19, 0, 4089, 4089, 4089, 352, 80, 330, 0, 0, 0, 0, 0, 144, 7, 8, 0, 4062, 0, 1, 0, 0, 4, 4094, 0, 4092, 0, 3180, 2, 4, 0, 3957, 4085, 4082, 3215, 14, 0, 0, 0, 393, 3, 0, 4015, 4096, 0, 4045, 0, 4090, 2388, 2, 2236, 2, 4007, 3972, 0, 320, 0, 0, 0, 2699, 3900, 4096, 0, 4055, 0, 1, 6, 0, 1, 0, 8, 0, 0, 4087, 0, 17, 4093, 3739, 0, 12, 4096, 3, 20, 3, 2, 0, 0, 0, 4096, 0, 3233, 0, 1, 0, 0, 19, 0, 0, 0, 0, 0, 0, 1825, 79, 2, 0, 153, 5, 4088, 0, 4091, 1, 4096, 0, 0, 347, 99, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 0, 0, 1, 0, 0, 1511, 49, 0, 2940, 4005, 4095, 2, 2802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 74, 0, 0, 1, 4059, 14, 0, 3, 0, 25, 4094, 4095, 2617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 604, 0, 4095, 234, 0, 0, 0, 0, 3906, 3778, 3588, 1625, 0, 298, 19, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3300, 0, 0, 2519, 0, 0, 0, 0, 0, 57, 3515, 72, 0, 669, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 32, 3733, 911, 4095, 0, 4095, 0, 0, 0, 0, 0, 12, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 3, 1, 4075, 4096, 0, 0, 3974, 2, 6, 0, 0, 203, 0, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3806, 0, 4096, 0, 0, 0, 0, 0, 729, 0, 6, 202, 0, 2, 314, 4096, 2024, 1479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 396, 1061, 0, 0, 4096, 141, 4096, 422, 4000, 1, 0, 54, 0, 0, 0, 1577, 1220, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15416, 19052, 18180, 14228, 13203, 16858, 16754, 17381] avg=16384.0000 max=19052 max_rank=1 min=13203 min_rank=4 imbalance_ratio=1.162842 max_load_reduction=112020 max_load_reduction_pct=0.854645 moved_to_echo=64196 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 2942, 4091, 4096, 3799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 4096, 4096, 2616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 638, 0, 4096, 0, 0, 0, 0, 0, 4096, 513, 37, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3858, 0, 0, 2519, 0, 0, 0, 0, 0, 2700, 0, 1757, 0, 1895, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 0, 0, 0, 4089, 4070, 4095, 211, 4096, 0, 0, 1, 0, 0, 38, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 172, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4087, 0, 0, 0, 0, 0, 4, 0, 84, 3, 0, 0, 0, 4091, 2141, 1480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 3, 1145, 0, 0, 4019, 0, 4096, 1, 4096, 0, 0, 8, 0, 0, 3, 1577, 1291, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14933, 14907, 13476, 16759, 17377, 17093, 15974, 20553] avg=16384.0000 max=20553 max_rank=7 min=13476 min_rank=2 imbalance_ratio=1.254456 max_load_reduction=110519 max_load_reduction_pct=0.843193 moved_to_echo=70997 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 74, 0, 0, 355, 1, 0, 0, 1, 0, 2938, 3935, 4093, 3989, 3333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1419, 14, 0, 0, 0, 2, 791, 0, 2, 0, 3559, 0, 0, 0, 0, 1233, 4003, 4091, 2595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 0, 3112, 118, 1299, 0, 0, 0, 3269, 1392, 4092, 491, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 58, 2475, 0, 0, 0, 0, 0, 11, 568, 867, 0, 4039, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 34, 0, 0, 4091, 2518, 4096, 0, 4096, 11, 0, 0, 0, 0, 1, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3982, 0, 68, 4021, 3780, 0, 12, 3726, 119, 0, 0, 0, 4, 0, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 79, 0, 0, 167, 0, 2, 23, 0, 94, 9, 0, 0, 2968, 3835, 1915, 1467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 6, 1, 0, 0, 0, 218, 3970, 356, 4095, 7, 0, 1876, 534, 0, 24, 1548, 1155, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18719, 17709, 14346, 16105, 19165, 16254, 14655, 14119] avg=16384.0000 max=19165 max_rank=4 min=14119 min_rank=7 imbalance_ratio=1.169739 max_load_reduction=111907 max_load_reduction_pct=0.853783 moved_to_echo=64193 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 7, 4018, 0, 6, 265, 26, 0, 2907, 4079, 3838, 4069, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 0, 11, 0, 11, 0, 0, 63, 7, 0, 3650, 0, 0, 0, 0, 120, 4067, 3962, 2573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 618, 56, 4096, 0, 3872, 0, 18, 0, 4067, 314, 512, 0, 520, 8, 4, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2083, 0, 0, 2243, 0, 0, 0, 0, 0, 1778, 9, 4090, 0, 3861, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 575, 0, 3904, 1052, 4030, 0, 4095, 0, 70, 0, 0, 29, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 3835, 21, 0, 4094, 4096, 0, 0, 3810, 1, 0, 75, 0, 132, 22, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 90, 662, 0, 0, 0, 0, 0, 0, 0, 3331, 2, 21, 7, 1, 4082, 2072, 1454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 2, 272, 0, 4, 2176, 69, 4073, 0, 4096, 0, 0, 0, 0, 0, 48, 1404, 1249, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23312, 14961, 14176, 18160, 14065, 16699, 15731, 13968] avg=16384.0000 max=23312 max_rank=0 min=13968 min_rank=7 imbalance_ratio=1.422852 max_load_reduction=107760 max_load_reduction_pct=0.822144 moved_to_echo=60463 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 1, 0, 0, 0, 14, 0, 2942, 3985, 4096, 3573, 3802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4096, 0, 0, 0, 0, 14, 4096, 4096, 2617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 642, 0, 4031, 0, 991, 0, 0, 0, 2516, 4071, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 2519, 0, 0, 0, 0, 0, 0, 0, 3535, 0, 4067, 2545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 722, 0, 0, 0, 3998, 3056, 4096, 910, 4089, 0, 0, 30, 0, 0, 4, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 6, 0, 3839, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 3695, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 184, 4096, 2155, 1479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1347, 0, 1235, 280, 128, 621, 152, 4096, 0, 4096, 0, 0, 27, 0, 0, 0, 1577, 1299, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18413, 14937, 16347, 16632, 17210, 16668, 15693, 15172] avg=16384.0000 max=18413 max_rank=0 min=14937 min_rank=1 imbalance_ratio=1.123840 max_load_reduction=112659 max_load_reduction_pct=0.859520 moved_to_echo=64743 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 29, 375, 6, 0, 3, 0, 15, 2911, 4054, 4090, 3983, 1317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 13, 9, 0, 0, 3, 52, 0, 4096, 0, 2, 0, 26, 4, 4093, 4087, 2613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 633, 13, 4049, 2, 2881, 1, 0, 0, 3997, 3882, 2205, 5, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3625, 0, 0, 2473, 0, 1, 0, 0, 0, 0, 1, 4090, 6, 4087, 1629, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 0, 0, 0, 2470, 334, 4078, 2961, 4081, 0, 0, 1, 2, 0, 0, 210, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 3916, 0, 0, 3871, 4074, 0, 5, 4096, 1, 3, 0, 0, 0, 0, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 64, 4096, 1, 5, 46, 0, 0, 3, 16, 0, 0, 0, 0, 977, 4095, 2158, 1477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1382, 235, 547, 4, 0, 293, 0, 4094, 0, 4091, 0, 2, 1, 1, 0, 1, 1553, 1303, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16784, 15000, 17677, 15913, 18268, 16576, 17030, 13824] avg=16384.0000 max=18268 max_rank=4 min=13824 min_rank=7 imbalance_ratio=1.114990 max_load_reduction=112804 max_load_reduction_pct=0.860626 moved_to_echo=65698 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 0, 162, 8, 39, 2859, 4014, 4089, 0, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 2, 0, 5, 0, 0, 153, 415, 6, 0, 4074, 113, 1, 11, 0, 3562, 4096, 4092, 2572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 642, 0, 4068, 1, 72, 0, 0, 0, 4096, 2841, 1450, 0, 427, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 2336, 0, 10, 0, 0, 0, 4010, 2, 2131, 3, 3763, 1509, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 60, 3, 0, 4092, 1, 3697, 4096, 4093, 1, 0, 15, 23, 0, 11, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 3939, 0, 5, 2687, 2860, 0, 28, 4004, 69, 0, 0, 25, 2, 0, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 3990, 7, 4081, 0, 0, 0, 0, 0, 0, 0, 1683, 63, 0, 0, 801, 4094, 2153, 1454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3328, 6, 3772, 259, 1152, 0, 0, 4085, 0, 4071, 0, 0, 0, 9, 191, 0, 1461, 1298, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11492, 19147, 13598, 17802, 16394, 14318, 18384, 19937] avg=16384.0000 max=19937 max_rank=7 min=11492 min_rank=0 imbalance_ratio=1.216858 max_load_reduction=111135 max_load_reduction_pct=0.847893 moved_to_echo=69033 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 10, 0, 403, 0, 0, 18, 19, 0, 2938, 4089, 4089, 352, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 144, 7, 8, 0, 4062, 0, 1, 0, 0, 4, 4095, 4096, 2612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 0, 4092, 0, 3180, 2, 4, 0, 3957, 4085, 4082, 3215, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 3, 0, 2470, 0, 0, 0, 0, 0, 2388, 2, 2236, 2, 4007, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 0, 0, 0, 2699, 3900, 4096, 0, 4055, 0, 1, 6, 0, 1, 0, 8, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 17, 4093, 3739, 0, 12, 4096, 3, 20, 3, 2, 0, 0, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3233, 0, 1, 0, 0, 19, 0, 0, 0, 0, 0, 0, 1825, 4045, 2154, 1478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 2, 0, 153, 5, 4088, 0, 4091, 1, 4096, 0, 0, 347, 99, 0, 0, 1545, 1299, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12003, 15359, 23272, 15473, 15390, 16606, 16851, 16118] avg=16384.0000 max=23272 max_rank=2 min=12003 min_rank=0 imbalance_ratio=1.420410 max_load_reduction=107800 max_load_reduction_pct=0.822449 moved_to_echo=64965 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[25344, 128, 128, 19584, 0, 128, 0, 0, 0, 11008, 4224, 18816, 128, 26496, 25984, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32256, 256, 24064, 128, 128, 256, 0, 128, 768, 128, 5248, 384, 128, 128, 7168, 32512, 16896, 11776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[6400, 128, 128, 128, 128, 128, 1280, 512, 128, 128, 31744, 128, 128, 128, 128, 4992, 32640, 32640, 20864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 5248, 128, 128, 2048, 128, 128, 23424, 32256, 32512, 19840, 15872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 256, 32000, 256, 128, 30848, 30848, 0, 128, 32000, 256, 128, 128, 128, 384, 128, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[4992, 128, 31744, 384, 12416, 128, 128, 0, 29952, 20992, 20096, 9472, 1024, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9472, 128, 640, 128, 29184, 15872, 32384, 8192, 32768, 128, 128, 128, 128, 128, 128, 256, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[10496, 768, 8064, 768, 1408, 15360, 640, 32640, 896, 32768, 128, 128, 2432, 768, 256, 128, 12288, 10240, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3397, 4096, 4096, 4049, 0, 0, 0, 0, 0, 4011, 0, 0, 0, 0, 4096, 5, 0, 3988, 0, 0, 3, 0, 0, 3284, 11, 4096, 0, 1, 0, 19, 0, 0, 0, 742, 0, 4095, 0, 0, 4096, 0, 0, 3440, 0, 4018, 0, 0, 0, 17, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 0, 2299, 0, 0, 4096, 2, 2539, 4096, 0, 4096, 2865, 294, 0, 0, 0, 0, 0, 1508, 0, 4096, 3, 4096, 0, 4063, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 264, 128, 0, 4088, 4096, 0, 4096, 4096, 0, 3, 17, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 6, 27, 0, 57, 0, 4018, 3716, 4096, 0, 0, 0, 1, 0, 4033, 41, 0, 0, 6, 4096, 42, 0, 2585, 8, 0, 0, 0, 63, 2842, 0, 4088, 0, 851, 8, 0, 104, 0, 0, 4042, 0, 4096, 0, 1, 4092, 1, 0, 3062, 0, 4095, 0, 89, 0, 2973, 2, 0, 4073, 3, 0, 0, 4, 97, 1, 11, 2, 85, 1, 201, 8, 4096, 0, 17, 0, 1, 4095, 0, 0, 0, 0, 4094, 0, 229, 14, 4094, 0, 0, 4095, 0, 3604, 4036, 23, 4025, 3853, 2, 0, 0, 0, 0, 0, 68, 0, 4048, 0, 4070, 0, 4086, 0, 0, 26, 0, 4038, 5, 2, 4095, 84, 191, 0, 0, 4079, 4094, 12, 4085, 4091, 0, 69, 0, 0, 24] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 23, 306, 4060, 4092, 3583, 4, 219, 0, 0, 0, 4092, 33, 21, 11, 4096, 0, 0, 3, 3754, 4, 0, 0, 3678, 14, 0, 71, 4077, 0, 0, 0, 197, 6, 0, 0, 4086, 0, 3253, 0, 46, 4056, 0, 0, 30, 123, 2441, 149, 0, 42, 3993, 45, 0, 4096, 8, 0, 0, 0, 3, 0, 0, 27, 0, 20, 0, 0, 4051, 0, 0, 0, 3, 4076, 0, 173, 0, 0, 4009, 108, 734, 1273, 251, 160, 0, 4087, 765, 2866, 3997, 120, 3933, 3781, 16, 0, 0, 1, 0, 9, 3, 0, 4056, 23, 4096, 5, 3669, 0, 0, 5, 0, 2443, 0, 0, 4096, 2031, 4027, 878, 0, 4038, 3852, 0, 3808, 4058, 0, 674, 135, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 1, 0, 4096, 3916, 3513, 0, 0, 487, 480, 154, 4039, 0, 0, 0, 4096, 4001, 0, 0, 4092, 0, 0, 62, 2, 0, 3974, 0, 3594, 0, 12, 0, 107, 0, 0, 0, 1478, 2, 3627, 9, 0, 4092, 0, 4, 83, 64, 687, 1191, 0, 0, 320, 23, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 922, 0, 4096, 68, 7, 0, 1, 3302, 0, 0, 0, 16, 4095, 16, 3881, 0, 4, 0, 0, 4096, 0, 33, 4096, 2, 3442, 4057, 4017, 0, 205, 0, 0, 0, 14, 0, 4096, 452, 4096, 0, 3703, 1, 0, 1, 0, 2790, 0, 0, 4095, 711, 591, 4067, 0, 4096, 1504, 0, 4078, 4094, 19, 53, 3951, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 28, 0, 0, 656, 4096, 3789, 4096, 0, 404, 0, 11, 0, 3931, 0, 0, 1, 3078, 3287, 0, 0, 4090, 1, 0, 0, 191, 0, 0, 96, 4096, 0, 1197, 0, 0, 0, 0, 0, 2409, 0, 3492, 0, 0, 4031, 58, 1, 3978, 346, 28, 0, 10, 0, 2660, 0, 0, 3733, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 3587, 454, 0, 0, 3729, 4084, 1, 820, 4, 3379, 10, 0, 4070, 0, 334, 4096, 0, 4094, 4096, 3439, 0, 18, 0, 0, 0, 116, 262, 4096, 0, 4096, 0, 3793, 0, 0, 124, 0, 3736, 14, 0, 3441, 0, 12, 0, 0, 4087, 4081, 0, 4068, 4056, 45, 9, 920, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[15, 11, 0, 0, 0, 62, 0, 3890, 4087, 4058, 0, 0, 0, 314, 33, 3166, 18, 55, 1, 77, 4091, 9, 0, 0, 1, 7, 6, 0, 0, 233, 294, 4072, 184, 4001, 0, 20, 193, 2972, 34, 3956, 47, 3828, 0, 0, 4035, 0, 8, 67, 24, 4094, 0, 27, 0, 3964, 33, 14, 4064, 18, 0, 13, 11, 0, 3, 6, 0, 0, 0, 0, 0, 4071, 0, 1, 316, 12, 4096, 0, 54, 3, 0, 4096, 0, 4096, 0, 4046, 293, 0, 4068, 422, 831, 3925, 0, 4025, 3864, 0, 38, 0, 0, 0, 6, 3594, 1, 3995, 0, 4096, 107, 4011, 0, 0, 0, 0, 123, 0, 119, 3856, 1004, 47, 2, 18, 4015, 3422, 0, 4090, 4080, 0, 39, 0, 74, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 6, 2, 0, 5, 4, 4096, 3890, 3485, 0, 1819, 0, 0, 0, 18, 1, 0, 0, 1135, 3968, 0, 0, 4031, 0, 0, 2328, 7, 0, 2034, 317, 4096, 0, 329, 0, 0, 13, 0, 0, 4079, 0, 345, 11, 0, 4096, 0, 0, 3847, 0, 4096, 0, 2, 0, 3911, 0, 0, 4096, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3995, 0, 2250, 0, 0, 4096, 60, 9, 4089, 0, 4096, 4094, 3303, 0, 0, 0, 0, 0, 114, 0, 4046, 0, 4095, 0, 4096, 0, 0, 0, 0, 3700, 0, 9, 4041, 159, 0, 59, 0, 4094, 4093, 0, 4096, 4096, 0, 0, 10, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 6, 0, 0, 4096, 4041, 3365, 0, 305, 4, 0, 0, 3930, 8, 0, 1, 1833, 4074, 251, 0, 3819, 0, 0, 0, 8, 0, 0, 0, 3454, 0, 179, 0, 3, 0, 0, 1, 4071, 0, 4074, 1, 0, 4069, 0, 0, 3869, 31, 3730, 0, 61, 2, 4088, 0, 0, 4039, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 31, 0, 4096, 363, 1, 0, 0, 4026, 0, 0, 0, 20, 4094, 0, 661, 50, 840, 0, 14, 4096, 0, 1349, 4048, 0, 4094, 4064, 4037, 0, 0, 0, 1, 0, 649, 1, 4075, 2, 4096, 0, 4096, 0, 0, 32, 0, 177, 1, 0, 4091, 579, 0, 3746, 0, 3841, 4095, 0, 4094, 4092, 54, 32, 7, 2, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3397, 4096, 4096, 4049, 0, 0, 0, 0, 0, 4011, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 3988, 0, 0, 3, 0, 0, 3284, 11, 4096, 1410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 19, 0, 0, 0, 742, 0, 4095, 0, 0, 4096, 0, 0, 3440, 2039, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 0, 17, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 2314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1782, 1721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2299, 0, 0, 0, 2, 2539, 336, 0, 4096, 2865, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1508, 0, 4096, 3, 4096, 0, 4063, 0, 0, 0, 0, 4096, 0, 765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 264, 128, 0, 4088, 4096, 0, 1620, 0, 0, 3, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19734, 16893, 14648, 14542, 15790, 16525, 18627, 14313] avg=16384.0000 max=19734 max_rank=0 min=14313 min_rank=7 imbalance_ratio=1.204468 max_load_reduction=111338 max_load_reduction_pct=0.849442 moved_to_echo=65255 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 1, 0, 4096, 3916, 3513, 0, 0, 487, 480, 154, 4039, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4001, 0, 0, 4092, 0, 0, 62, 2, 0, 3974, 0, 3594, 1404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 107, 0, 0, 0, 1478, 2, 3627, 9, 0, 4092, 0, 4, 83, 2039, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 687, 1191, 0, 0, 320, 23, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 922, 0, 4096, 68, 7, 0, 1, 3302, 0, 0, 0, 16, 4095, 1780, 1721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 3881, 0, 4, 0, 0, 0, 0, 33, 336, 2, 3442, 4057, 4017, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4096, 452, 4096, 0, 3703, 1, 0, 1, 0, 2790, 0, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 711, 591, 4067, 0, 4096, 1504, 0, 1613, 0, 19, 53, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16773, 21225, 11668, 12791, 16008, 15993, 15914, 20700] avg=16384.0000 max=21225 max_rank=1 min=11668 min_rank=2 imbalance_ratio=1.295471 max_load_reduction=109847 max_load_reduction_pct=0.838066 moved_to_echo=68615 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 28, 0, 0, 656, 4096, 3789, 4096, 0, 404, 0, 11, 0, 3931, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3078, 3287, 0, 0, 4090, 1, 0, 0, 191, 0, 0, 96, 4096, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1197, 0, 0, 0, 0, 0, 2409, 0, 3492, 0, 0, 4031, 58, 1, 3978, 2039, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 346, 28, 0, 10, 0, 2660, 0, 0, 3733, 0, 0, 0, 0, 3, 0, 2, 4070, 2292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 3587, 454, 0, 0, 3729, 4084, 1764, 1721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 820, 4, 3379, 10, 0, 0, 0, 334, 336, 0, 4094, 4096, 3439, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 262, 4096, 0, 4096, 0, 3793, 0, 0, 124, 0, 3736, 14, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3441, 0, 12, 0, 0, 4087, 4081, 0, 1609, 0, 45, 9, 920, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17096, 16241, 17419, 13144, 19440, 16531, 16996, 14205] avg=16384.0000 max=19440 max_rank=4 min=13144 min_rank=3 imbalance_ratio=1.186523 max_load_reduction=111632 max_load_reduction_pct=0.851685 moved_to_echo=67172 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 27, 0, 57, 0, 4018, 3716, 4096, 0, 0, 0, 1, 0, 4033, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 6, 4096, 42, 0, 2585, 8, 0, 0, 0, 63, 2842, 0, 4088, 1412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 851, 8, 0, 104, 0, 0, 4042, 0, 4096, 0, 1, 4092, 1, 0, 3062, 2012, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 89, 0, 2973, 2, 0, 4073, 3, 0, 0, 4, 97, 1, 11, 4095, 2312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 85, 1, 201, 8, 4096, 0, 17, 0, 1, 4095, 0, 0, 0, 0, 4094, 1779, 1701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 14, 4094, 0, 0, 0, 0, 3604, 323, 23, 4025, 3853, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 4048, 0, 4070, 0, 4086, 0, 0, 26, 0, 4038, 5, 767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 84, 191, 0, 0, 4079, 4094, 12, 1598, 0, 0, 69, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16043, 15183, 18488, 17755, 16080, 16167, 17108, 14248] avg=16384.0000 max=18488 max_rank=2 min=14248 min_rank=7 imbalance_ratio=1.128418 max_load_reduction=112584 max_load_reduction_pct=0.858948 moved_to_echo=63603 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 23, 306, 4060, 4092, 3583, 4, 219, 0, 0, 0, 4092, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 21, 11, 4096, 0, 0, 3, 3754, 4, 0, 0, 3678, 14, 0, 71, 4077, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 6, 0, 0, 4086, 0, 3253, 0, 46, 4056, 0, 0, 30, 1990, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 2441, 149, 0, 42, 3993, 45, 0, 4096, 8, 0, 0, 0, 3, 0, 0, 4087, 2294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 20, 0, 0, 4051, 0, 0, 0, 3, 4076, 0, 173, 0, 0, 4009, 1764, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 734, 1273, 251, 160, 0, 0, 765, 2866, 328, 120, 3933, 3781, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 3, 0, 4056, 23, 4096, 5, 3669, 0, 0, 5, 0, 2443, 0, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2031, 4027, 878, 0, 4038, 3852, 0, 1506, 0, 0, 674, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16458, 17073, 13865, 17281, 15802, 14335, 15021, 21237] avg=16384.0000 max=21237 max_rank=7 min=13865 min_rank=2 imbalance_ratio=1.296204 max_load_reduction=109835 max_load_reduction_pct=0.837975 moved_to_echo=66395 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 2, 0, 5, 4, 4096, 3890, 3485, 0, 1819, 0, 0, 0, 18, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1135, 3968, 0, 0, 4031, 0, 0, 2328, 7, 0, 2034, 317, 4096, 1410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 0, 13, 0, 0, 4079, 0, 345, 11, 0, 4096, 0, 0, 3847, 2035, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 3911, 0, 0, 4096, 0, 0, 0, 0, 39, 0, 0, 4096, 2315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1781, 1718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 2250, 0, 0, 0, 60, 9, 336, 0, 4096, 4094, 3303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 4046, 0, 4095, 0, 4096, 0, 0, 0, 0, 3700, 0, 765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4041, 159, 0, 59, 0, 4094, 4093, 0, 1620, 0, 0, 0, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13410, 19327, 14971, 18555, 15762, 18143, 16816, 14088] avg=16384.0000 max=19327 max_rank=1 min=13410 min_rank=0 imbalance_ratio=1.179626 max_load_reduction=111745 max_load_reduction_pct=0.852547 moved_to_echo=64809 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[15, 11, 0, 0, 0, 62, 0, 3890, 4087, 4058, 0, 0, 0, 314, 33, 3166, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 55, 1, 77, 4091, 9, 0, 0, 1, 7, 6, 0, 0, 233, 294, 4072, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 4001, 0, 20, 193, 2972, 34, 3956, 47, 3828, 0, 0, 4035, 0, 8, 67, 1954, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4094, 0, 27, 0, 3964, 33, 14, 4064, 18, 0, 13, 11, 0, 3, 6, 4068, 2305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 1, 316, 12, 4096, 0, 54, 3, 0, 4096, 1775, 1649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4046, 293, 0, 0, 422, 831, 322, 0, 4025, 3864, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3594, 1, 3995, 0, 4096, 107, 4011, 0, 0, 0, 0, 123, 0, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 3856, 1004, 47, 2, 18, 4015, 3422, 0, 1618, 0, 0, 39, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15721, 10272, 21515, 18644, 16073, 17937, 16696, 14214] avg=16384.0000 max=21515 max_rank=2 min=10272 min_rank=1 imbalance_ratio=1.313171 max_load_reduction=109557 max_load_reduction_pct=0.835854 moved_to_echo=64920 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 6, 0, 0, 4096, 4041, 3365, 0, 305, 4, 0, 0, 3930, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 1833, 4074, 251, 0, 3819, 0, 0, 0, 8, 0, 0, 0, 3454, 1410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 3, 0, 0, 1, 4071, 0, 4074, 1, 0, 4069, 0, 0, 3869, 2015, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3730, 0, 61, 2, 4088, 0, 0, 4039, 0, 0, 1, 0, 0, 0, 0, 4096, 2312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 31, 0, 4096, 363, 1, 0, 0, 4026, 0, 0, 0, 20, 4094, 1780, 1701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 661, 50, 840, 0, 14, 0, 0, 1349, 332, 0, 4094, 4064, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 649, 1, 4075, 2, 4096, 0, 4096, 0, 0, 32, 0, 177, 1, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 579, 0, 3746, 0, 3841, 4095, 0, 1619, 0, 54, 32, 7, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15837, 14858, 18498, 18360, 16117, 15441, 13894, 18067] avg=16384.0000 max=18498 max_rank=2 min=13894 min_rank=6 imbalance_ratio=1.129028 max_load_reduction=112574 max_load_reduction_pct=0.858871 moved_to_echo=63519 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 14336, 27648, 384, 128, 26368, 128, 128, 2432, 3968, 128, 12416, 896, 31616, 11264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 6144, 384, 32512, 512, 32768, 128, 31616, 128, 0, 256, 0, 21120, 128, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 256, 4480, 32512, 31744, 30336, 128, 2816, 512, 896, 256, 27264, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[640, 23296, 1408, 256, 128, 22016, 128, 128, 32384, 128, 128, 128, 128, 256, 128, 128, 32768, 18560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 1280, 128, 32768, 512, 128, 384, 128, 31488, 512, 256, 128, 3840, 32768, 14208, 13696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 18560, 1408, 17280, 512, 128, 0, 1280, 11648, 2688, 256, 31872, 30720, 15232, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 31872, 4608, 5248, 8960, 128, 32384, 29312, 128, 12928, 0, 128, 896, 5120, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[256, 6656, 128, 384, 384, 3072, 128, 24960, 128, 26880, 128, 128, 32640, 128, 128, 18432, 16128, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4077, 0, 0, 4096, 4096, 172, 0, 4096, 1, 4096, 399, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3871, 0, 1937, 0, 0, 4096, 4096, 0, 323, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4053, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 9, 7, 0, 0, 0, 16, 0, 0, 0, 168, 3952, 0, 4096, 4096, 0, 0, 4032, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 23, 0, 4082, 0, 0, 3, 4096, 0, 4096, 0, 1, 4091, 0, 0, 0, 0, 1464, 0, 0, 0, 4096, 4095, 4096, 1, 4096, 210, 0, 0, 0, 0, 0, 0, 0, 2851, 0, 0, 0, 4096, 0, 0, 0, 68, 0, 2806, 2791, 1, 0, 0, 3645, 0, 4089] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[3088, 0, 0, 3628, 3430, 0, 0, 4020, 48, 4035, 1, 0, 0, 204, 0, 1, 4051, 2, 0, 7, 4068, 0, 3642, 23, 0, 4085, 4081, 74, 0, 188, 1, 0, 1, 0, 538, 4043, 0, 0, 0, 2696, 31, 3902, 2027, 826, 57, 12, 0, 0, 0, 0, 61, 0, 200, 0, 0, 0, 0, 1, 15, 23, 0, 4027, 15, 607, 4052, 26, 89, 360, 0, 0, 0, 1, 4022, 389, 61, 0, 2363, 0, 2794, 4000, 66, 0, 0, 4039, 141, 3722, 8, 0, 3988, 103, 0, 0, 0, 0, 1, 25, 0, 3790, 4070, 4096, 0, 3948, 762, 1104, 1, 77, 1062, 0, 0, 0, 4063, 1, 0, 34, 4070, 49, 0, 0, 26, 36, 4055, 4095, 0, 0, 8, 3822, 23, 3871] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4088, 2, 0, 4096, 4096, 2567, 24, 4096, 2, 4096, 22, 0, 0, 0, 39, 0, 4073, 0, 0, 0, 4095, 0, 83, 2, 34, 4081, 4096, 0, 46, 0, 0, 0, 0, 0, 0, 1114, 0, 0, 0, 394, 0, 4030, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 387, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2154, 0, 3997, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3379, 413, 4096, 0, 0, 74, 4096, 1135, 4096, 0, 3, 4096, 0, 0, 8, 0, 285, 0, 0, 0, 4096, 4096, 4092, 0, 4096, 4077, 58, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 0, 4096, 122, 3, 0, 0, 0, 2972, 1827, 0, 0, 45, 3864, 0, 3752] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4092, 0, 0, 4096, 104, 0, 0, 3997, 25, 4096, 18, 0, 0, 0, 231, 0, 3882, 0, 0, 0, 4068, 0, 0, 0, 0, 4096, 4096, 0, 3963, 7, 10, 31, 0, 0, 0, 39, 0, 0, 0, 100, 0, 2946, 3974, 19, 437, 0, 1, 0, 0, 75, 0, 4096, 1344, 0, 3, 0, 222, 1, 0, 0, 183, 0, 7, 3657, 4096, 0, 0, 3665, 0, 0, 0, 0, 4096, 87, 449, 0, 3869, 0, 0, 4093, 0, 0, 3180, 4092, 48, 4075, 0, 1, 4086, 0, 0, 0, 147, 0, 0, 0, 0, 4095, 289, 4096, 0, 3985, 3810, 558, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 13, 0, 12, 0, 4087, 3930, 4095, 4, 4, 1758, 0, 2244] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[3840, 0, 0, 4056, 3897, 1692, 3, 4054, 155, 4096, 0, 3, 0, 4, 6, 0, 4094, 335, 0, 0, 4095, 15, 37, 253, 64, 3873, 4096, 5, 3, 0, 1, 64, 13, 119, 0, 0, 67, 0, 0, 4084, 4, 4056, 4031, 10, 632, 81, 150, 0, 0, 151, 37, 3945, 23, 0, 0, 23, 0, 0, 0, 0, 0, 1124, 0, 4023, 3998, 1, 162, 1026, 0, 83, 13, 2, 4054, 0, 0, 0, 4080, 2517, 247, 4060, 65, 0, 1, 3858, 1409, 4069, 50, 69, 3998, 0, 2, 1, 709, 0, 0, 0, 0, 3943, 3947, 3666, 0, 3967, 862, 0, 0, 13, 0, 19, 0, 27, 3132, 0, 0, 0, 3988, 160, 0, 0, 46, 0, 1419, 3320, 0, 0, 0, 2922, 7, 3846] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4028, 0, 2, 4094, 4085, 3640, 0, 4095, 0, 4074, 229, 0, 0, 327, 0, 1, 4096, 2, 36, 0, 3927, 1, 3983, 0, 3, 4077, 4080, 12, 84, 19, 151, 23, 0, 15, 1, 3724, 27, 1, 1, 641, 0, 3930, 4092, 0, 0, 0, 700, 23, 0, 31, 0, 132, 0, 2, 12, 0, 388, 3, 0, 3, 1748, 4078, 0, 2923, 4062, 16, 0, 132, 0, 101, 108, 20, 4065, 0, 2, 0, 2938, 0, 0, 4096, 0, 0, 21, 4041, 119, 4092, 0, 0, 4093, 13, 0, 3, 0, 602, 16, 0, 1, 4096, 4092, 4055, 0, 4092, 0, 17, 2, 0, 0, 1, 0, 2, 3843, 0, 0, 0, 4092, 1276, 0, 4, 465, 0, 555, 243, 0, 0, 0, 4081, 0, 4071] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[3702, 77, 0, 4096, 3832, 6, 0, 4085, 0, 3810, 68, 0, 0, 43, 0, 0, 2455, 0, 0, 0, 3435, 0, 4, 0, 0, 4090, 3972, 28, 0, 256, 74, 0, 0, 290, 9, 568, 386, 0, 0, 4052, 0, 4087, 4023, 0, 323, 2, 0, 0, 0, 0, 5, 3857, 0, 0, 0, 1, 0, 0, 0, 0, 0, 182, 0, 4089, 4095, 1, 0, 1773, 147, 0, 0, 0, 3955, 90, 32, 2, 4088, 16, 668, 4025, 44, 0, 162, 4093, 2718, 4096, 0, 71, 4049, 49, 0, 0, 903, 3901, 0, 0, 0, 4056, 3432, 3819, 0, 4096, 25, 1179, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 4096, 0, 33, 1, 5, 0, 0, 3708, 0, 12, 0, 4001, 10, 3671] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4096, 3, 3, 4096, 4017, 638, 7, 4096, 21, 4096, 41, 0, 0, 10, 330, 0, 4068, 0, 4, 2, 4051, 0, 3286, 0, 0, 4096, 4094, 0, 61, 0, 0, 0, 0, 69, 0, 11, 0, 0, 0, 1909, 0, 4018, 4096, 0, 3, 1, 0, 0, 0, 9, 7, 175, 0, 0, 0, 0, 13, 0, 0, 41, 0, 3201, 0, 3624, 4092, 0, 1, 106, 8, 40, 0, 0, 4072, 102, 26, 0, 4074, 2155, 0, 4092, 0, 75, 3228, 4083, 0, 4083, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 2, 0, 3974, 4096, 3939, 0, 4094, 176, 76, 0, 0, 137, 0, 0, 9, 2975, 0, 0, 5, 4096, 0, 0, 1, 0, 0, 2632, 4046, 0, 0, 1, 4094, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4077, 0, 0, 4096, 4096, 172, 0, 0, 1, 3690, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3871, 0, 1937, 0, 0, 4096, 2131, 0, 323, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4053, 4096, 0, 1, 0, 0, 0, 2325, 1965, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 7, 0, 0, 0, 16, 0, 0, 0, 168, 3952, 0, 4096, 4096, 4096, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1396, 0, 0, 4032, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 23, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 4096, 0, 1, 4091, 0, 0, 0, 0, 1464, 0, 0, 664, 562, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3534, 4095, 4096, 1, 0, 210, 0, 0, 0, 0, 0, 0, 0, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3037, 0, 0, 0, 68, 0, 2806, 2791, 1, 0, 0, 3645, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16531, 16466, 16929, 16815, 17724, 15383, 14787, 16437] avg=16384.0000 max=17724 max_rank=4 min=14787 min_rank=6 imbalance_ratio=1.081787 max_load_reduction=113348 max_load_reduction_pct=0.864777 moved_to_echo=64331 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 4096, 104, 0, 0, 0, 25, 3690, 18, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3882, 0, 0, 0, 4068, 0, 0, 0, 0, 4096, 2131, 0, 3963, 7, 10, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 100, 0, 2946, 3974, 19, 437, 0, 1, 0, 2325, 1965, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 4096, 1344, 0, 3, 0, 222, 1, 0, 0, 183, 0, 7, 3657, 3997, 3985, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1396, 0, 0, 3665, 0, 0, 0, 0, 4096, 87, 449, 0, 3869, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3180, 4092, 48, 4075, 0, 1, 4086, 0, 0, 0, 147, 0, 0, 0, 664, 562, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3533, 289, 4096, 0, 0, 3810, 558, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3037, 0, 13, 0, 12, 0, 4087, 3930, 4095, 4, 4, 1758, 0, 2244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12256, 18188, 12201, 17945, 17655, 17261, 16382, 19184] avg=16384.0000 max=19184 max_rank=7 min=12201 min_rank=2 imbalance_ratio=1.170898 max_load_reduction=111888 max_load_reduction_pct=0.853638 moved_to_echo=70482 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3702, 77, 0, 4096, 3832, 6, 0, 0, 0, 3432, 68, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2455, 0, 0, 0, 3435, 0, 4, 0, 0, 4090, 2066, 28, 0, 256, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 9, 568, 386, 0, 0, 4052, 0, 4087, 4023, 0, 323, 2, 0, 0, 2324, 1906, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3857, 0, 0, 0, 1, 0, 0, 0, 0, 0, 182, 0, 4089, 4085, 4096, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1397, 1, 0, 1773, 147, 0, 0, 0, 3955, 90, 32, 2, 4088, 16, 668, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 162, 4093, 2718, 4096, 0, 71, 4049, 49, 0, 0, 903, 3901, 0, 0, 664, 556, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3500, 3432, 3819, 0, 0, 25, 1179, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3037, 0, 33, 1, 5, 0, 0, 3708, 0, 12, 0, 4001, 10, 3671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15256, 12408, 18365, 16689, 16194, 21684, 15998, 14478] avg=16384.0000 max=21684 max_rank=5 min=12408 min_rank=1 imbalance_ratio=1.323486 max_load_reduction=109388 max_load_reduction_pct=0.834564 moved_to_echo=68354 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3088, 0, 0, 3628, 3430, 0, 0, 0, 48, 3635, 1, 0, 0, 204, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 2, 0, 7, 4068, 0, 3642, 23, 0, 4085, 2123, 74, 0, 188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 538, 4043, 0, 0, 0, 2696, 31, 3902, 2027, 826, 57, 12, 0, 0, 2300, 1958, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 200, 0, 0, 0, 0, 1, 15, 23, 0, 4027, 15, 607, 4020, 3948, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1381, 26, 89, 360, 0, 0, 0, 1, 4022, 389, 61, 0, 2363, 0, 2794, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 4039, 141, 3722, 8, 0, 3988, 103, 0, 0, 0, 0, 1, 25, 660, 520, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3270, 4070, 4096, 0, 0, 762, 1104, 1, 77, 1062, 0, 0, 0, 4063, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 3018, 49, 0, 0, 26, 36, 4055, 4095, 0, 0, 8, 3822, 23, 3871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14035, 18264, 18783, 13288, 15486, 13673, 18506, 19037] avg=16384.0000 max=19037 max_rank=7 min=13288 min_rank=3 imbalance_ratio=1.161926 max_load_reduction=112035 max_load_reduction_pct=0.854759 moved_to_echo=66702 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3840, 0, 0, 4056, 3897, 1692, 3, 0, 155, 3690, 0, 3, 0, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 335, 0, 0, 4095, 15, 37, 253, 64, 3873, 2131, 5, 3, 0, 1, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 119, 0, 0, 67, 0, 0, 4084, 4, 4056, 4031, 10, 632, 81, 150, 0, 2269, 1965, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 37, 3945, 23, 0, 0, 23, 0, 0, 0, 0, 0, 1124, 0, 4023, 4054, 3967, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1363, 1, 162, 1026, 0, 83, 13, 2, 4054, 0, 0, 0, 4080, 2517, 247, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 1, 3858, 1409, 4069, 50, 69, 3998, 0, 2, 1, 709, 0, 0, 0, 646, 541, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3402, 3947, 3666, 0, 0, 862, 0, 0, 13, 0, 19, 0, 27, 3132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2958, 160, 0, 0, 46, 0, 1419, 3320, 0, 0, 0, 2922, 7, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17346, 14970, 17865, 17713, 17608, 15824, 15068, 14678] avg=16384.0000 max=17865 max_rank=2 min=14678 min_rank=7 imbalance_ratio=1.090393 max_load_reduction=113207 max_load_reduction_pct=0.863701 moved_to_echo=63178 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4088, 2, 0, 4096, 4096, 2567, 24, 0, 2, 3690, 22, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 4095, 0, 83, 2, 34, 4081, 2131, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 0, 0, 0, 394, 0, 4030, 4096, 0, 0, 0, 0, 0, 2325, 1965, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2154, 0, 3997, 4096, 4096, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1396, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3379, 413, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 4096, 1135, 4096, 0, 3, 4096, 0, 0, 8, 0, 285, 0, 0, 664, 562, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3534, 4096, 4092, 0, 0, 4077, 58, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3037, 122, 3, 0, 0, 0, 2972, 1827, 0, 0, 45, 3864, 0, 3752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18626, 14545, 14319, 15107, 17477, 15425, 19951, 15622] avg=16384.0000 max=19951 max_rank=6 min=14319 min_rank=2 imbalance_ratio=1.217712 max_load_reduction=111121 max_load_reduction_pct=0.847786 moved_to_echo=68475 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4028, 0, 2, 4094, 4085, 3640, 0, 0, 0, 3664, 229, 0, 0, 327, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 36, 0, 3927, 1, 3983, 0, 3, 4077, 2119, 12, 84, 19, 151, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 3724, 27, 1, 1, 641, 0, 3930, 4092, 0, 0, 0, 700, 23, 2309, 1961, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 132, 0, 2, 12, 0, 388, 3, 0, 3, 1748, 4078, 0, 2923, 4095, 4092, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1379, 16, 0, 132, 0, 101, 108, 20, 4065, 0, 2, 0, 2938, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4041, 119, 4092, 0, 0, 4093, 13, 0, 3, 0, 602, 16, 0, 667, 566, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3530, 4092, 4055, 0, 0, 0, 17, 2, 0, 0, 1, 0, 2, 3843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3029, 1276, 0, 4, 465, 0, 555, 243, 0, 0, 0, 4081, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20070, 18533, 17821, 17881, 12857, 14643, 15543, 13724] avg=16384.0000 max=20070 max_rank=0 min=12857 min_rank=4 imbalance_ratio=1.224976 max_load_reduction=111002 max_load_reduction_pct=0.846878 moved_to_echo=56767 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 3, 3, 4096, 4017, 638, 7, 0, 21, 3690, 41, 0, 0, 10, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 4, 2, 4051, 0, 3286, 0, 0, 4096, 2130, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 11, 0, 0, 0, 1909, 0, 4018, 4096, 0, 3, 1, 0, 0, 2323, 1964, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 7, 175, 0, 0, 0, 0, 13, 0, 0, 41, 0, 3201, 0, 3624, 4096, 4094, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1395, 0, 1, 106, 8, 40, 0, 0, 4072, 102, 26, 0, 4074, 2155, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 3228, 4083, 0, 4083, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 2, 664, 545, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3429, 4096, 3939, 0, 0, 176, 76, 0, 0, 137, 0, 0, 9, 2975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3037, 0, 0, 1, 0, 0, 2632, 4046, 0, 0, 1, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16952, 17698, 14789, 15634, 16071, 17179, 14837, 17912] avg=16384.0000 max=17912 max_rank=7 min=14789 min_rank=2 imbalance_ratio=1.093262 max_load_reduction=113160 max_load_reduction_pct=0.863342 moved_to_echo=65999 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 27776, 28160, 31872, 128, 0, 9984, 3072, 128, 128, 1280, 128, 0, 128, 29184, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[31104, 128, 128, 32384, 27648, 8832, 128, 0, 256, 29184, 896, 128, 0, 640, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[30848, 384, 128, 128, 31616, 128, 13056, 384, 128, 32512, 17024, 128, 4480, 512, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 24192, 1664, 128, 128, 640, 128, 18560, 24064, 4096, 128, 128, 28288, 128, 29696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[11136, 128, 256, 11136, 256, 256, 128, 128, 32512, 768, 640, 128, 29696, 8192, 4224, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[256, 128, 6784, 32512, 5632, 32384, 128, 256, 32512, 256, 128, 128, 1792, 6272, 128, 128, 5376, 4480, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 384, 128, 12672, 1664, 128, 128, 128, 640, 128, 128, 128, 2176, 18816, 128, 27136, 32640, 32384, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 512, 640, 9600, 512, 128, 128, 18048, 128, 31104, 30464, 896, 1536, 128, 896, 128, 18560, 15744, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4019, 2, 3888, 0, 44, 0, 4052, 0, 0, 0, 261, 0, 0, 303, 3729, 0, 4095, 5, 4076, 130, 2, 0, 4094, 4000, 4077, 0, 76, 0, 0, 1468, 905, 0, 6, 4078, 3406, 0, 7, 0, 4096, 4096, 4096, 0, 420, 0, 27, 0, 0, 0, 562, 0, 2778, 3898, 0, 13, 8, 4096, 0, 212, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 2534, 61, 740, 4096, 359, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 4096, 3855, 2, 1, 5, 306, 0, 1, 12, 10, 3569, 0, 0, 0, 5, 0, 0, 0, 161, 0, 0, 0, 0, 0, 938, 0, 1, 0, 3216, 0, 4094, 4086, 0, 0, 0, 0, 2, 0, 4, 15, 3377, 0, 4023, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 102, 263, 93, 3913, 0, 0, 0, 6, 0, 0, 31, 4093, 0, 4093, 4015, 3966, 320, 0, 0, 4081, 0, 0, 3, 3732, 33, 3, 0, 0, 1, 0, 3475, 379, 8, 0, 0, 4051, 4095, 4046, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4089, 4084, 7, 0, 3840, 4080, 1, 0, 0, 0, 0, 0, 0, 0, 85, 4039, 1, 1, 0, 4096, 0, 0, 4095, 11, 0, 4096, 4071, 4095, 0, 0, 4095, 0, 3991, 4095, 26, 0, 0, 17, 1, 2, 18, 0, 0, 0, 0, 0, 1, 1, 28, 46, 3999, 0, 0, 0, 1, 2, 0, 4, 121, 24, 4061, 0, 3831, 4060, 602, 0, 2, 3464, 41, 28, 0, 0, 3967, 14, 842, 13, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 18, 0, 4094, 0, 0, 3, 2, 0, 0, 1, 4082, 0, 4087, 3785, 235, 1137, 0, 0, 4093, 440, 2785, 0, 4051, 0, 0, 0, 0, 0, 0, 2152, 81, 0, 0, 0, 4089, 4096, 4096, 0, 0, 0, 4, 0, 0, 0, 4074, 0, 4096, 4096, 0, 0, 29, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 4085, 3261, 0, 0, 0, 0, 0, 3, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 6, 4091, 0, 0, 0, 0, 3381, 1779, 0, 60, 0, 4096, 0, 2, 4096, 2129, 0, 0, 943, 0, 0, 0, 0, 4093, 0, 2519, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 2, 4095, 0, 2150, 3508, 4093, 0, 0, 659, 0, 0, 0, 0, 4096, 0, 4096, 2, 4096, 699, 0, 0, 4096, 12, 4021, 0, 4096, 0, 0, 2, 12, 0, 0, 4078, 87, 0, 0, 0, 4090, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 2831, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 2, 0, 1, 0, 457, 0, 6, 0, 3769, 0, 0, 0, 2, 0, 0, 0, 2245, 0, 0, 0, 0, 0, 1, 0, 17, 0, 4096, 0, 0, 4096, 5, 0, 0, 4092, 0, 0, 0, 0, 4064, 0, 68, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[154, 3, 3, 3777, 87, 0, 218, 3899, 0, 414, 35, 0, 0, 24, 0, 4096, 0, 4069, 0, 3990, 4094, 0, 8, 4059, 11, 3596, 14, 4069, 0, 0, 0, 0, 292, 59, 4096, 181, 0, 0, 16, 4086, 4070, 4095, 0, 0, 0, 0, 0, 0, 0, 7, 1, 4094, 4095, 0, 0, 0, 4008, 41, 5, 19, 53, 0, 118, 0, 3, 0, 3919, 0, 0, 56, 4096, 0, 9, 4095, 0, 24, 4096, 4089, 4070, 0, 0, 4091, 0, 3911, 4069, 4005, 165, 0, 127, 66, 0, 1900, 4, 0, 11, 0, 10, 0, 0, 2, 0, 3313, 0, 0, 0, 1, 4073, 0, 0, 342, 0, 3880, 0, 0, 4033, 4051, 0, 24, 17, 0, 0, 0, 0, 2564, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[707, 140, 1, 4088, 0, 1990, 3525, 3978, 0, 11, 0, 0, 59, 0, 2, 4070, 2, 4084, 0, 125, 0, 0, 0, 3715, 0, 0, 0, 4088, 137, 32, 45, 0, 0, 0, 2398, 0, 56, 0, 3739, 3962, 4096, 4096, 0, 0, 0, 3576, 0, 34, 0, 4053, 0, 3849, 4093, 0, 0, 3, 4059, 9, 0, 7, 10, 7, 0, 0, 0, 0, 4087, 172, 0, 0, 4096, 0, 3, 4066, 0, 55, 4024, 4032, 4067, 5, 0, 4047, 1, 3946, 4083, 133, 0, 0, 1327, 0, 404, 0, 7, 2, 0, 0, 0, 0, 0, 34, 0, 1207, 0, 0, 0, 0, 377, 3910, 0, 0, 0, 4096, 6, 0, 4095, 2670, 28, 0, 4091, 10, 0, 0, 0, 3145, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 90, 41, 4083, 11, 0, 0, 4094, 0, 2, 26, 0, 34, 11, 0, 4041, 0, 4095, 3539, 3656, 5, 0, 729, 4083, 99, 660, 0, 4052, 0, 0, 0, 1, 0, 4, 2515, 4096, 0, 74, 21, 4096, 4076, 4077, 0, 0, 0, 0, 0, 0, 0, 108, 17, 3583, 4096, 0, 24, 2, 4096, 0, 0, 0, 4, 16, 0, 0, 0, 328, 4085, 1, 0, 0, 1539, 0, 0, 4096, 0, 0, 3930, 4096, 4091, 0, 0, 4076, 0, 4074, 4068, 489, 0, 1, 93, 0, 285, 0, 0, 4017, 2, 0, 0, 0, 0, 0, 12, 2310, 0, 0, 0, 0, 1094, 3906, 17, 87, 0, 4089, 0, 0, 4090, 4038, 0, 0, 3662, 0, 0, 0, 18, 4093, 0, 128, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[11, 0, 0, 4093, 0, 746, 379, 4003, 0, 0, 0, 1, 4, 0, 10, 4077, 0, 4096, 142, 3775, 0, 0, 81, 4066, 0, 2869, 0, 4069, 19, 0, 474, 0, 2, 0, 520, 1352, 0, 0, 1080, 4096, 4096, 4095, 0, 0, 3, 4022, 0, 0, 0, 165, 6, 4094, 4096, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 1, 14, 4069, 0, 0, 2, 3996, 0, 2, 4096, 0, 1, 4084, 4096, 4090, 0, 1, 4091, 1, 4085, 4000, 1, 0, 779, 0, 0, 15, 0, 0, 3935, 0, 0, 1, 0, 0, 1, 0, 3300, 0, 0, 4, 0, 64, 1, 0, 3234, 0, 4087, 0, 97, 4096, 3599, 3, 0, 2534, 0, 2, 0, 29, 3601, 0, 421, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4019, 2, 3888, 0, 44, 0, 4052, 0, 0, 0, 261, 0, 0, 303, 3729, 1090, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 5, 4076, 130, 2, 0, 4094, 4000, 4077, 0, 76, 0, 0, 1468, 905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4078, 3406, 0, 7, 0, 4096, 2437, 4096, 0, 420, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 0, 2778, 3898, 0, 13, 8, 4096, 0, 212, 0, 0, 0, 0, 0, 1864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 2534, 61, 740, 0, 359, 0, 4096, 4096, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3855, 2, 1, 5, 306, 0, 1, 12, 10, 3569, 0, 0, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 161, 0, 0, 0, 0, 0, 938, 0, 1, 0, 3216, 4096, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4086, 0, 0, 0, 0, 2, 0, 4, 15, 3377, 0, 4023, 1, 0, 1171, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18081, 19112, 18573, 13431, 16347, 16522, 11233, 17773] avg=16384.0000 max=19112 max_rank=1 min=11233 min_rank=6 imbalance_ratio=1.166504 max_load_reduction=111960 max_load_reduction_pct=0.854187 moved_to_echo=61875 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 102, 263, 93, 3913, 0, 0, 0, 6, 0, 0, 31, 4093, 1092, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 4015, 3966, 320, 0, 0, 4081, 0, 0, 3, 3732, 33, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3475, 379, 8, 0, 0, 4051, 2431, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4089, 4084, 7, 0, 3840, 4080, 1, 0, 0, 0, 0, 0, 0, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 4039, 1, 1, 0, 4096, 0, 0, 0, 11, 0, 4096, 4071, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3991, 4095, 26, 0, 0, 17, 1, 2, 18, 0, 0, 0, 0, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 28, 46, 3999, 0, 0, 0, 1, 2, 0, 4, 121, 24, 4061, 4095, 2819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3831, 4060, 602, 0, 2, 3464, 41, 28, 0, 0, 3967, 14, 842, 13, 4, 1176, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14380, 16422, 14391, 22053, 16758, 12817, 15202, 19049] avg=16384.0000 max=22053 max_rank=3 min=12817 min_rank=5 imbalance_ratio=1.346008 max_load_reduction=109019 max_load_reduction_pct=0.831749 moved_to_echo=63826 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 4095, 0, 2150, 3508, 4093, 0, 0, 659, 0, 0, 0, 0, 4096, 1090, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 2, 4096, 699, 0, 0, 4096, 12, 4021, 0, 4096, 0, 0, 2, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 87, 0, 0, 0, 4090, 2437, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2831, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 4096, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 2, 0, 1, 0, 457, 0, 6, 0, 3769, 0, 0, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2245, 0, 0, 0, 0, 0, 1, 0, 17, 0, 4096, 4096, 2817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 4092, 0, 0, 0, 0, 4064, 0, 68, 0, 0, 1171, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20386, 17315, 14788, 16981, 16740, 17092, 13274, 14496] avg=16384.0000 max=20386 max_rank=0 min=13274 min_rank=6 imbalance_ratio=1.244263 max_load_reduction=110686 max_load_reduction_pct=0.844467 moved_to_echo=61602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 90, 41, 4083, 11, 0, 0, 4094, 0, 2, 26, 0, 34, 11, 0, 4041, 1084, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 3539, 3656, 5, 0, 729, 4083, 99, 660, 0, 4052, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2515, 4096, 0, 74, 21, 4096, 2426, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 17, 3583, 4096, 0, 24, 2, 4096, 0, 0, 0, 4, 16, 0, 0, 1862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 4085, 1, 0, 0, 1539, 0, 0, 0, 0, 0, 3930, 4096, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 4074, 4068, 489, 0, 1, 93, 0, 285, 0, 0, 4017, 2, 0, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2310, 0, 0, 0, 0, 1094, 3906, 17, 87, 0, 4089, 4096, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4038, 0, 0, 3662, 0, 0, 0, 18, 4093, 0, 128, 0, 0, 1170, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14209, 17103, 17309, 13808, 14346, 17671, 18427, 18199] avg=16384.0000 max=18427 max_rank=6 min=13808 min_rank=3 imbalance_ratio=1.124695 max_load_reduction=112645 max_load_reduction_pct=0.859413 moved_to_echo=68643 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 18, 0, 4094, 0, 0, 3, 2, 0, 0, 1, 4082, 1090, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 3785, 235, 1137, 0, 0, 4093, 440, 2785, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2152, 81, 0, 0, 0, 4089, 2437, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 4096, 4096, 0, 0, 29, 4087, 0, 0, 0, 0, 0, 0, 0, 1864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4085, 3261, 0, 0, 0, 0, 0, 3, 0, 0, 4046, 0, 0, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4091, 0, 0, 0, 0, 3381, 1779, 0, 60, 0, 4096, 4096, 2810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 2129, 0, 0, 943, 0, 0, 0, 0, 4093, 0, 2519, 0, 0, 1171, 998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14080, 16805, 12859, 18246, 16752, 16060, 20319, 15951] avg=16384.0000 max=20319 max_rank=6 min=12859 min_rank=2 imbalance_ratio=1.240173 max_load_reduction=110753 max_load_reduction_pct=0.844978 moved_to_echo=69082 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[154, 3, 3, 3777, 87, 0, 218, 3899, 0, 414, 35, 0, 0, 24, 0, 4096, 1083, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 3990, 4094, 0, 8, 4059, 11, 3596, 14, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 59, 4096, 181, 0, 0, 16, 4086, 2422, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 4094, 4095, 0, 0, 0, 4008, 41, 5, 19, 53, 0, 118, 0, 1852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3919, 0, 0, 56, 4096, 0, 9, 0, 0, 24, 4096, 4089, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3911, 4069, 4005, 165, 0, 127, 66, 0, 1900, 4, 0, 11, 0, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 0, 3313, 0, 0, 0, 1, 4073, 0, 0, 342, 0, 3880, 4095, 2798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 4051, 0, 24, 17, 0, 0, 0, 0, 2564, 0, 0, 0, 0, 1164, 994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14481, 20118, 15247, 14293, 16658, 18914, 18514, 12847] avg=16384.0000 max=20118 max_rank=1 min=12847 min_rank=7 imbalance_ratio=1.227905 max_load_reduction=110954 max_load_reduction_pct=0.846512 moved_to_echo=66933 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[707, 140, 1, 4088, 0, 1990, 3525, 3978, 0, 11, 0, 0, 59, 0, 2, 4070, 1090, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 279, 0, 125, 0, 0, 0, 3715, 0, 0, 0, 4088, 137, 32, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2398, 0, 56, 0, 3739, 3962, 2437, 4096, 0, 0, 0, 3576, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 3849, 4093, 0, 0, 3, 4059, 9, 0, 7, 10, 7, 0, 0, 1851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 172, 0, 0, 4096, 0, 3, 0, 0, 55, 4024, 4032, 365, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 1, 3946, 4083, 133, 0, 0, 1327, 0, 404, 0, 7, 2, 0, 0, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 1207, 0, 0, 0, 0, 377, 3910, 0, 0, 0, 4096, 4066, 2808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4095, 2670, 28, 0, 4091, 10, 0, 0, 0, 3145, 0, 0, 0, 0, 1163, 997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20349, 8423, 20298, 17941, 16839, 14519, 16498, 16205] avg=16384.0000 max=20349 max_rank=0 min=8423 min_rank=1 imbalance_ratio=1.242004 max_load_reduction=110723 max_load_reduction_pct=0.844749 moved_to_echo=64061 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 0, 4093, 0, 746, 379, 4003, 0, 0, 0, 1, 4, 0, 10, 4077, 1090, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 142, 3775, 0, 0, 81, 4066, 0, 2869, 0, 4069, 19, 0, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 520, 1352, 0, 0, 1080, 4096, 2437, 4095, 0, 0, 3, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 6, 4094, 4096, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 1861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 4069, 0, 0, 2, 3996, 0, 2, 0, 0, 1, 4084, 4096, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 1, 4085, 4000, 1, 0, 779, 0, 0, 15, 0, 0, 3935, 0, 0, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 3300, 0, 0, 4, 0, 64, 1, 0, 3234, 0, 4087, 4096, 2817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 4096, 3599, 3, 0, 2534, 0, 2, 0, 29, 3601, 0, 421, 0, 0, 1170, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15106, 15774, 17607, 14319, 16632, 17477, 17605, 16552] avg=16384.0000 max=17607 max_rank=2 min=14319 min_rank=3 imbalance_ratio=1.074646 max_load_reduction=113465 max_load_reduction_pct=0.865669 moved_to_echo=68266 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[896, 4352, 128, 32256, 256, 5248, 7808, 32128, 0, 512, 768, 384, 128, 128, 384, 32384, 8832, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 512, 32512, 256, 128, 128, 28672, 128, 768, 0, 384, 128, 32640, 32768, 2944, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 16000, 128, 30720, 32640, 128, 128, 3968, 32640, 128, 256, 128, 128, 128, 128, 0, 14976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 2304, 11520, 23936, 6400, 128, 896, 32384, 4608, 18048, 128, 28288, 256, 128, 2048, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[384, 128, 23424, 9600, 128, 128, 4864, 32640, 19584, 32768, 0, 512, 128, 7680, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 8064, 32768, 17152, 128, 128, 18816, 128, 128, 128, 128, 28928, 128, 8064, 128, 128, 9472, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 128, 32384, 31616, 4736, 256, 896, 1920, 640, 768, 2048, 128, 19456, 128, 0, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 20736, 0, 0, 128, 128, 9088, 10624, 128, 3968, 128, 31744, 32768, 22528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 15, 0, 0, 0, 15, 0, 4093, 838, 0, 0, 0, 0, 0, 432, 0, 0, 4096, 0, 4096, 0, 0, 0, 3, 0, 43, 490, 4067, 8, 0, 2325, 4096, 0, 0, 3761, 4, 4096, 59, 0, 0, 2820, 59, 0, 3712, 0, 4096, 0, 4027, 0, 80, 1079, 0, 4, 0, 3857, 0, 0, 361, 4070, 1, 0, 4096, 0, 1814, 3893, 0, 527, 0, 3525, 0, 9, 0, 0, 0, 0, 0, 0, 0, 6, 0, 21, 4078, 0, 4048, 0, 0, 9, 0, 1, 4066, 0, 424, 0, 0, 2, 1, 15, 4096, 2586, 0, 0, 7, 4080, 0, 0, 0, 4096, 2, 0, 319, 0, 428, 127, 0, 4096, 2730, 4096, 4036, 4096, 747, 0, 4096, 11, 4089, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2329, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4096, 0, 0, 459, 0, 0, 38, 0, 4096, 0, 0, 4096, 4095, 0, 0, 4096, 50, 4096, 0, 0, 0, 4093, 0, 0, 132, 233, 4096, 0, 4096, 0, 1, 0, 0, 4, 0, 1068, 0, 0, 4091, 4096, 2, 0, 4092, 0, 746, 4095, 0, 0, 0, 4092, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3400, 0, 3341, 4037, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 5, 0, 0, 0, 1, 0, 4096, 18, 0, 0, 0, 4096, 0, 0, 0, 4047, 0, 832, 3, 0, 0, 2657, 0, 1833, 4079, 4096, 145, 4096, 0, 0, 4096, 0, 3331, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 0, 0, 89, 0, 4094, 0, 0, 0, 0, 0, 0, 1700, 140, 3, 4096, 0, 4096, 0, 0, 1781, 0, 0, 9, 0, 4096, 0, 0, 4096, 4096, 0, 0, 3817, 4093, 4096, 0, 0, 0, 4096, 0, 0, 3861, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3448, 4096, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2603, 0, 75, 4096, 0, 0, 1, 0, 0, 0, 1913, 2799, 1, 0, 0, 0, 0, 4, 42, 4096, 536, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2699, 0, 292, 4007, 4053, 3732, 4096, 0, 0, 4096, 0, 3361, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[7, 293, 5, 0, 0, 1, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 2, 0, 3569, 92, 209, 4085, 87, 4085, 1, 33, 4096, 0, 59, 13, 0, 4095, 0, 1, 4069, 4073, 0, 0, 4096, 1688, 4064, 0, 0, 4, 4094, 2, 1, 1391, 402, 4068, 19, 4058, 16, 0, 0, 0, 19, 0, 0, 0, 1, 19, 4096, 0, 17, 3842, 0, 1060, 4008, 0, 0, 0, 4095, 25, 0, 0, 0, 8, 0, 0, 3, 0, 4096, 1, 0, 4096, 1, 0, 94, 0, 0, 4, 3658, 4094, 73, 1, 35, 0, 5, 0, 0, 4074, 3956, 2, 0, 25, 4075, 0, 37, 0, 4021, 0, 0, 0, 1, 0, 158, 8, 850, 574, 4037, 4003, 3863, 0, 0, 4085, 0, 3156, 3927] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 104, 0, 26, 0, 4, 57, 0, 3997, 15, 0, 0, 0, 0, 8, 2362, 0, 331, 4088, 8, 3821, 0, 354, 0, 0, 0, 108, 1, 3956, 0, 2, 4096, 4094, 74, 0, 4094, 701, 4096, 0, 0, 0, 4078, 0, 110, 4, 0, 4034, 0, 4095, 0, 64, 0, 18, 0, 0, 0, 6, 0, 4030, 4090, 0, 0, 4023, 0, 0, 4095, 0, 4, 60, 3771, 0, 319, 0, 5, 12, 0, 19, 0, 0, 4089, 0, 4086, 4087, 25, 1, 13, 0, 0, 0, 736, 3821, 59, 0, 0, 66, 0, 22, 0, 4012, 2212, 0, 1, 0, 3886, 76, 1, 232, 4021, 3, 0, 0, 69, 0, 4053, 0, 857, 4024, 2509, 2815, 4096, 7, 1, 4096, 0, 4049, 3812] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 26, 8, 0, 281, 0, 0, 115, 0, 27, 3873, 0, 0, 2, 0, 0, 96, 0, 0, 0, 3986, 0, 3994, 0, 244, 3827, 401, 0, 1255, 0, 4073, 14, 83, 3764, 4079, 0, 0, 387, 42, 4019, 0, 0, 0, 4045, 0, 0, 43, 0, 3554, 2, 3859, 26, 94, 0, 28, 0, 0, 2612, 0, 165, 491, 4092, 2209, 0, 4095, 30, 9, 4075, 0, 480, 231, 4096, 0, 0, 0, 34, 77, 188, 0, 0, 0, 4058, 0, 3937, 3477, 0, 137, 0, 0, 0, 0, 4096, 4029, 0, 0, 0, 0, 0, 0, 465, 3993, 3927, 0, 1, 0, 4080, 93, 0, 81, 2248, 0, 39, 0, 36, 0, 3, 3, 4068, 3934, 4096, 2586, 3985, 0, 0, 3839, 13, 621, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 42, 0, 24, 0, 5, 0, 788, 0, 4033, 157, 4, 0, 0, 0, 1641, 0, 121, 1, 4032, 0, 4096, 0, 0, 4095, 0, 0, 3255, 16, 3906, 0, 0, 4063, 4095, 0, 221, 4093, 185, 4001, 1, 0, 0, 0, 0, 75, 80, 0, 4042, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3598, 44, 0, 3983, 0, 0, 4096, 0, 153, 0, 4069, 0, 0, 1, 0, 0, 0, 1, 0, 0, 2557, 0, 3, 4040, 51, 357, 0, 16, 0, 3, 1726, 4043, 0, 0, 0, 0, 41, 2119, 0, 4095, 0, 0, 0, 0, 4096, 0, 7, 0, 4096, 17, 23, 3, 70, 1730, 0, 0, 3952, 3057, 3660, 4096, 4095, 0, 1, 4095, 0, 4089, 3759] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 43, 2, 4089, 0, 0, 0, 0, 0, 0, 1142, 22, 0, 4095, 166, 4092, 0, 0, 16, 2, 0, 0, 94, 4094, 0, 0, 4071, 4082, 0, 0, 1787, 3382, 4076, 0, 0, 0, 4085, 0, 60, 4096, 19, 4027, 0, 4056, 2, 47, 23, 0, 85, 0, 0, 0, 0, 1661, 4017, 0, 0, 4056, 0, 3679, 4095, 0, 2, 0, 4050, 28, 0, 0, 0, 3, 0, 10, 0, 0, 4015, 0, 214, 4063, 7, 0, 0, 0, 0, 0, 3532, 3304, 7, 5, 0, 0, 0, 650, 0, 4072, 55, 2, 0, 0, 4093, 1, 0, 36, 4085, 0, 0, 0, 0, 48, 0, 0, 1656, 3603, 4096, 3801, 4096, 0, 2, 4081, 0, 4095, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 15, 0, 0, 0, 15, 0, 4093, 838, 0, 0, 0, 0, 4096, 4096, 3560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 0, 0, 4096, 0, 4096, 0, 0, 0, 3, 0, 43, 490, 4067, 8, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2325, 0, 0, 0, 3761, 4, 2792, 59, 0, 0, 2820, 59, 0, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4027, 0, 80, 1079, 0, 4, 0, 3857, 0, 0, 361, 4070, 1, 0, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1814, 3893, 0, 527, 0, 3525, 0, 9, 0, 0, 0, 0, 0, 0, 2659, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 21, 4078, 0, 4048, 0, 0, 9, 0, 1, 4066, 0, 424, 0, 566, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 15, 4096, 2586, 0, 0, 7, 4080, 0, 0, 0, 4096, 2, 0, 1437, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 319, 0, 428, 127, 0, 4096, 2730, 2977, 4036, 0, 747, 0, 0, 11, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16713, 13354, 15532, 18212, 17059, 13582, 17060, 19560] avg=16384.0000 max=19560 max_rank=7 min=13354 min_rank=1 imbalance_ratio=1.193848 max_load_reduction=111512 max_load_reduction_pct=0.850769 moved_to_echo=67261 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2329, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 3559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4096, 0, 0, 459, 0, 0, 38, 0, 4096, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 50, 2792, 0, 0, 0, 4093, 0, 0, 132, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 4, 0, 1068, 0, 0, 4091, 4096, 2, 0, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 746, 4095, 0, 0, 0, 4092, 0, 0, 0, 2, 0, 0, 0, 0, 2658, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3400, 0, 3341, 4037, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 5, 0, 566, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 18, 0, 0, 0, 4096, 0, 0, 0, 4047, 0, 832, 1438, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2657, 0, 1833, 4079, 2977, 145, 0, 0, 0, 0, 0, 3331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18175, 12906, 15492, 18091, 16222, 19895, 15266, 15025] avg=16384.0000 max=19895 max_rank=5 min=12906 min_rank=1 imbalance_ratio=1.214294 max_load_reduction=111177 max_load_reduction_pct=0.848213 moved_to_echo=66408 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 42, 0, 24, 0, 5, 0, 788, 0, 4033, 157, 4, 0, 0, 0, 4095, 4095, 3558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1641, 0, 121, 1, 4032, 0, 4096, 0, 0, 4095, 0, 0, 3255, 16, 3906, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 221, 4093, 185, 2728, 1, 0, 0, 0, 0, 75, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3598, 44, 0, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 4096, 0, 153, 0, 4069, 0, 0, 1, 0, 0, 0, 1, 0, 2440, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2557, 0, 3, 4040, 51, 357, 0, 16, 0, 3, 1726, 4043, 0, 0, 0, 552, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 2119, 0, 4095, 0, 0, 0, 0, 4096, 0, 7, 0, 4096, 17, 23, 1319, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 70, 1730, 0, 0, 3952, 3057, 2660, 4096, 0, 0, 1, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16801, 21269, 11446, 16412, 15280, 13672, 16534, 19658] avg=16384.0000 max=21269 max_rank=1 min=11446 min_rank=2 imbalance_ratio=1.298157 max_load_reduction=109803 max_load_reduction_pct=0.837730 moved_to_echo=65144 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 0, 0, 89, 0, 4094, 0, 0, 0, 0, 0, 4096, 4096, 3560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1700, 140, 3, 4096, 0, 4096, 0, 0, 1781, 0, 0, 9, 0, 4096, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3817, 4093, 2792, 0, 0, 0, 4096, 0, 0, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3448, 4096, 0, 0, 631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 2657, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2603, 0, 75, 4096, 0, 0, 1, 0, 0, 0, 1913, 2799, 1, 0, 0, 566, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 42, 4096, 536, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1435, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2699, 0, 292, 4007, 2946, 3732, 0, 0, 0, 0, 0, 3361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15943, 16038, 22755, 16359, 15484, 12413, 15043, 17037] avg=16384.0000 max=22755 max_rank=2 min=12413 min_rank=5 imbalance_ratio=1.388855 max_load_reduction=108317 max_load_reduction_pct=0.826393 moved_to_echo=59977 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 104, 0, 26, 0, 4, 57, 0, 3997, 15, 0, 0, 0, 0, 4094, 4096, 3560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2362, 0, 331, 4088, 8, 3821, 0, 354, 0, 0, 0, 108, 1, 3956, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 74, 0, 4094, 701, 2792, 0, 0, 0, 4078, 0, 110, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 4095, 0, 64, 0, 18, 0, 0, 0, 6, 0, 4030, 4090, 0, 0, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 4095, 0, 4, 60, 3771, 0, 319, 0, 5, 12, 0, 19, 0, 2474, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4086, 4087, 25, 1, 13, 0, 0, 0, 736, 3821, 59, 0, 0, 566, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 22, 0, 4012, 2212, 0, 1, 0, 3886, 76, 1, 232, 4021, 3, 0, 1338, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 4053, 0, 857, 4024, 1824, 2815, 0, 7, 1, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15954, 15110, 15951, 16727, 15318, 17705, 16608, 17699] avg=16384.0000 max=17705 max_rank=5 min=15110 min_rank=1 imbalance_ratio=1.080627 max_load_reduction=113367 max_load_reduction_pct=0.864922 moved_to_echo=67330 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[7, 293, 5, 0, 0, 1, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 2, 4073, 4085, 3358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3569, 92, 209, 4085, 87, 4085, 1, 33, 4096, 0, 59, 13, 0, 4095, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4069, 0, 0, 0, 4096, 1688, 2765, 0, 0, 4, 4094, 2, 1, 1391, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 19, 4058, 16, 0, 0, 0, 19, 0, 0, 0, 1, 19, 4096, 0, 17, 634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 0, 1060, 4008, 0, 0, 0, 4095, 25, 0, 0, 0, 8, 0, 0, 3, 2550, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 1, 0, 94, 0, 0, 4, 3658, 4094, 73, 1, 35, 564, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4074, 3956, 2, 0, 25, 4075, 0, 37, 0, 4021, 0, 0, 1377, 735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 158, 8, 850, 574, 2920, 4003, 0, 0, 0, 0, 0, 3156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15915, 20544, 18513, 12947, 16096, 17080, 18307, 11670] avg=16384.0000 max=20544 max_rank=1 min=11670 min_rank=7 imbalance_ratio=1.253906 max_load_reduction=110528 max_load_reduction_pct=0.843262 moved_to_echo=63153 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 26, 8, 0, 281, 0, 0, 115, 0, 27, 3873, 0, 0, 2, 0, 0, 4079, 3839, 3464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 3986, 0, 3994, 0, 244, 3827, 401, 0, 1255, 0, 4073, 14, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 3764, 0, 0, 0, 387, 42, 2740, 0, 0, 0, 4045, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3554, 2, 3859, 26, 94, 0, 28, 0, 0, 2612, 0, 165, 491, 4092, 2209, 0, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 30, 9, 4075, 0, 480, 231, 4096, 0, 0, 0, 34, 77, 188, 0, 0, 2659, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 3937, 3477, 0, 137, 0, 0, 0, 0, 4096, 4029, 0, 0, 0, 555, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 3993, 3927, 0, 1, 0, 4080, 93, 0, 81, 2248, 0, 39, 1437, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 3, 3, 4068, 3934, 2977, 2586, 0, 0, 0, 0, 13, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15714, 18009, 11104, 17769, 16495, 20652, 17088, 14241] avg=16384.0000 max=20652 max_rank=5 min=11104 min_rank=2 imbalance_ratio=1.260498 max_load_reduction=110420 max_load_reduction_pct=0.842438 moved_to_echo=68476 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 0, 0, 0, 43, 2, 4089, 0, 0, 0, 0, 0, 4082, 4081, 3559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1142, 22, 0, 4095, 166, 4092, 0, 0, 16, 2, 0, 0, 94, 4094, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 1787, 3382, 2779, 0, 0, 0, 4085, 0, 60, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 4056, 2, 47, 23, 0, 85, 0, 0, 0, 0, 1661, 4017, 0, 0, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 3679, 4095, 0, 2, 0, 4050, 28, 0, 0, 0, 3, 0, 10, 0, 2658, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 214, 4063, 7, 0, 0, 0, 0, 0, 3532, 3304, 7, 5, 0, 563, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 650, 0, 4072, 55, 2, 0, 0, 4093, 1, 0, 36, 4085, 0, 0, 1438, 734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 1656, 3603, 2977, 3801, 0, 0, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15857, 13842, 20279, 14555, 19118, 16073, 15166, 16182] avg=16384.0000 max=20279 max_rank=2 min=13842 min_rank=1 imbalance_ratio=1.237732 max_load_reduction=110793 max_load_reduction_pct=0.845284 moved_to_echo=66539 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 24832, 128, 11776, 32000, 128, 4608, 128, 128, 128, 128, 19840, 30336, 256, 512, 128, 4608, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32128, 128, 32384, 128, 384, 1152, 128, 128, 0, 7552, 128, 256, 18304, 32256, 2304, 128, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1792, 9216, 384, 640, 32640, 384, 32384, 128, 640, 14336, 512, 128, 4736, 640, 32384, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32384, 128, 7424, 32512, 0, 1280, 384, 31872, 128, 384, 128, 128, 128, 256, 128, 128, 20864, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 384, 128, 128, 384, 128, 128, 128, 3328, 128, 32384, 1024, 128, 128, 0, 128, 32768, 32512, 28288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 2816, 640, 32640, 13312, 128, 128, 128, 32512, 256, 128, 384, 30720, 128, 896, 11264, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 30592, 0, 128, 256, 26240, 10240, 22272, 128, 0, 128, 27392, 128, 256, 13440, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[384, 256, 2304, 9728, 128, 17664, 26112, 22272, 25216, 0, 768, 128, 0, 128, 26880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 82, 3306, 0, 129, 4096, 0, 0, 1473, 710, 14, 4094, 0, 0, 0, 0, 0, 0, 4080, 4070, 0, 585, 0, 85, 4019, 0, 175, 4091, 0, 5, 4096, 0, 0, 4096, 0, 0, 0, 0, 132, 0, 1407, 0, 0, 0, 4096, 2745, 917, 4096, 4096, 0, 1, 0, 0, 914, 0, 0, 6, 441, 0, 0, 4, 4079, 0, 0, 3972, 0, 0, 0, 0, 0, 0, 1246, 4069, 0, 0, 0, 0, 3428, 0, 0, 0, 4096, 0, 0, 0, 4058, 0, 4096, 0, 115, 0, 4096, 3631, 0, 31, 0, 0, 759, 7, 4096, 3974, 0, 0, 0, 80, 893, 0, 0, 3678, 4089, 3950, 4096, 0, 0, 3, 0, 3514, 0, 2846, 0, 15, 4091, 0, 3, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 2, 1, 0, 4075, 0, 0, 3780, 0, 0, 40, 4, 42, 4082, 0, 29, 0, 0, 0, 3, 4096, 4093, 34, 0, 0, 0, 4067, 47, 0, 4093, 4090, 121, 4091, 40, 0, 4084, 0, 0, 13, 0, 1131, 5, 2282, 0, 6, 0, 4077, 85, 4093, 4057, 348, 6, 0, 2, 271, 19, 0, 0, 0, 3, 0, 0, 0, 4096, 3, 0, 4095, 9, 0, 0, 0, 0, 2, 8, 4083, 1043, 9, 0, 0, 4053, 0, 0, 10, 4071, 0, 0, 1, 3974, 4, 4006, 1, 4096, 21, 90, 4018, 3786, 0, 0, 0, 0, 0, 4087, 3971, 0, 0, 0, 0, 1, 1, 15, 3340, 4039, 4076, 4094, 1, 0, 111, 12, 1345, 17, 3096, 4, 0, 4070, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 48, 0, 0, 4088, 0, 0, 2735, 80, 0, 1007, 0, 0, 4096, 1, 21, 2850, 0, 0, 0, 576, 3835, 2, 539, 0, 911, 4024, 50, 617, 3845, 4094, 3507, 4007, 0, 0, 4060, 182, 0, 149, 34, 0, 0, 0, 0, 0, 0, 4092, 0, 3351, 4072, 255, 26, 0, 0, 0, 0, 0, 0, 11, 27, 0, 2, 0, 4075, 0, 0, 3653, 148, 0, 0, 32, 2, 3, 0, 3539, 1, 88, 0, 0, 4066, 2, 0, 0, 4096, 302, 7, 4, 3774, 0, 3820, 0, 4048, 29, 4035, 3896, 2586, 0, 318, 0, 0, 2, 4055, 4075, 0, 195, 0, 454, 9, 35, 0, 3560, 3982, 4096, 4096, 0, 0, 476, 1, 1253, 1, 2288, 0, 6, 2668, 0, 35, 0, 67] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4085, 0, 4, 4091, 0, 0, 79, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 2525, 4096, 0, 0, 0, 0, 4096, 0, 0, 4094, 4096, 4, 4096, 0, 387, 4096, 0, 0, 0, 1, 78, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 26, 2, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 42, 3971, 0, 4075, 0, 3797, 3982, 2334, 0, 0, 0, 0, 0, 3926, 3835, 0, 0, 0, 248, 0, 0, 0, 4095, 4096, 4096, 4096, 0, 0, 1, 0, 3662, 0, 1, 0, 3897, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 0, 0, 3940, 43, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3981, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 835, 4076, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 2, 4095, 4096, 7, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 7, 4095, 82, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 4094, 0, 4096, 0, 1905, 4096, 7, 1, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 2486, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 2992, 0, 15, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 38, 4074, 431, 284, 4096, 0, 12, 264, 0, 1, 4034, 0, 0, 0, 0, 0, 0, 615, 4080, 482, 46, 0, 106, 4088, 0, 1063, 4095, 2983, 35, 4096, 0, 0, 4023, 0, 0, 2, 0, 1785, 0, 0, 0, 0, 0, 4096, 77, 2145, 4062, 16, 0, 0, 0, 1517, 62, 33, 0, 0, 6, 0, 0, 1, 4096, 0, 55, 2308, 0, 156, 0, 0, 0, 0, 21, 3911, 2987, 0, 265, 32, 3857, 1, 0, 0, 4096, 0, 4, 0, 4081, 0, 4086, 0, 3591, 0, 3466, 3586, 0, 1, 0, 3, 0, 0, 4027, 4096, 0, 0, 253, 0, 2475, 0, 106, 4006, 4088, 4096, 4038, 0, 0, 0, 0, 3883, 5, 1966, 0, 580, 4090, 12, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[419, 0, 1, 10, 3986, 0, 348, 4069, 0, 3, 0, 32, 91, 3740, 92, 0, 4060, 0, 0, 13, 3557, 4064, 0, 0, 19, 103, 4085, 20, 0, 4096, 4006, 0, 3758, 0, 1, 4061, 129, 6, 1, 0, 2379, 1, 548, 0, 215, 0, 4084, 0, 4039, 4041, 0, 0, 4, 0, 11, 8, 0, 0, 99, 0, 1, 0, 0, 4071, 0, 0, 3562, 0, 29, 0, 0, 0, 55, 0, 4072, 831, 0, 0, 2, 4093, 2, 152, 0, 4054, 4, 0, 0, 4096, 0, 3694, 0, 3974, 28, 0, 4086, 15, 0, 0, 0, 0, 0, 3481, 3248, 0, 79, 54, 7, 0, 0, 0, 4089, 4096, 4030, 3998, 25, 0, 935, 0, 3048, 0, 2373, 0, 35, 4091, 0, 36, 0, 427] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 52, 12, 4070, 38, 0, 939, 544, 0, 4062, 0, 0, 1349, 0, 5, 0, 3603, 4076, 19, 265, 0, 3, 4096, 5, 0, 4090, 4093, 0, 4084, 231, 17, 4094, 4, 0, 2, 0, 0, 2, 0, 0, 0, 2, 4095, 0, 3524, 4094, 1, 0, 0, 0, 2787, 0, 19, 0, 0, 0, 0, 0, 2, 4091, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 12, 3880, 67, 0, 0, 0, 3938, 0, 0, 0, 4096, 0, 0, 2, 4096, 0, 4096, 0, 4003, 0, 4055, 3735, 461, 16, 20, 0, 122, 0, 4088, 2389, 43, 1, 0, 0, 0, 0, 0, 4029, 4094, 4096, 4096, 0, 6, 0, 0, 4046, 0, 39, 0, 890, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 82, 3306, 0, 129, 4096, 0, 0, 1473, 710, 14, 4094, 0, 0, 3285, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4070, 0, 585, 0, 85, 0, 0, 175, 3801, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 132, 0, 1407, 0, 0, 0, 4096, 2745, 1687, 333, 290, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 917, 4096, 4096, 0, 1, 0, 0, 914, 0, 0, 6, 441, 0, 0, 4, 4079, 2815, 734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 0, 0, 0, 0, 0, 0, 1246, 4069, 0, 0, 0, 0, 3428, 3839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1125, 0, 4096, 0, 115, 0, 4096, 3631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 759, 7, 4096, 3974, 0, 0, 0, 80, 893, 0, 0, 3678, 3756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3950, 2409, 0, 0, 3, 0, 3514, 0, 2846, 0, 15, 4091, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17446, 12801, 19000, 18103, 16554, 13063, 17274, 16831] avg=16384.0000 max=19000 max_rank=2 min=12801 min_rank=1 imbalance_ratio=1.159668 max_load_reduction=112072 max_load_reduction_pct=0.855042 moved_to_echo=63722 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 0, 0, 3940, 43, 0, 0, 0, 0, 4095, 0, 0, 3347, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 4096, 0, 0, 0, 0, 0, 0, 0, 3806, 4096, 835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 2, 1687, 334, 290, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 7, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2842, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 7, 4095, 82, 0, 0, 0, 4096, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1134, 0, 4094, 0, 4096, 0, 1905, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 2486, 0, 0, 4096, 3762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2409, 0, 0, 0, 0, 4096, 0, 2992, 0, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15778, 16814, 14702, 15989, 16214, 15333, 18538, 17704] avg=16384.0000 max=18538 max_rank=6 min=14702 min_rank=2 imbalance_ratio=1.131470 max_load_reduction=112534 max_load_reduction_pct=0.858566 moved_to_echo=67789 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 38, 4074, 431, 284, 4096, 0, 12, 264, 0, 1, 4034, 0, 0, 3341, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 615, 4080, 482, 46, 0, 106, 0, 0, 1063, 3805, 2983, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4023, 0, 0, 2, 0, 1785, 0, 0, 0, 0, 0, 4096, 77, 1663, 333, 290, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2145, 4062, 16, 0, 0, 0, 1517, 62, 33, 0, 0, 6, 0, 0, 1, 4096, 2831, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 2308, 0, 156, 0, 0, 0, 0, 21, 3911, 2987, 0, 265, 32, 3857, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 1131, 0, 4086, 0, 3591, 0, 3466, 3586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 4027, 4096, 0, 0, 253, 0, 2475, 0, 106, 4006, 3755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2375, 0, 0, 0, 0, 3883, 5, 1966, 0, 580, 4090, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16831, 13215, 16484, 15516, 17432, 15865, 18722, 17007] avg=16384.0000 max=18722 max_rank=6 min=13215 min_rank=1 imbalance_ratio=1.142700 max_load_reduction=112350 max_load_reduction_pct=0.857162 moved_to_echo=69026 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 1, 0, 4075, 0, 0, 3780, 0, 0, 40, 4, 42, 4082, 0, 29, 3324, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4093, 34, 0, 0, 0, 0, 47, 0, 3800, 4090, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 40, 0, 4084, 0, 0, 13, 0, 1131, 5, 2282, 0, 6, 0, 4077, 85, 1692, 332, 293, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4057, 348, 6, 0, 2, 271, 19, 0, 0, 0, 3, 0, 0, 0, 4096, 2762, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 9, 0, 0, 0, 0, 2, 8, 4083, 1043, 9, 0, 0, 4053, 3815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 1, 1093, 4, 4006, 1, 4096, 21, 90, 4018, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 3971, 0, 0, 0, 0, 1, 1, 15, 3340, 3707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 2402, 1, 0, 111, 12, 1345, 17, 3096, 4, 0, 4070, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15635, 16284, 18250, 16400, 17120, 17126, 15122, 15135] avg=16384.0000 max=18250 max_rank=2 min=15122 min_rank=6 imbalance_ratio=1.113892 max_load_reduction=112822 max_load_reduction_pct=0.860764 moved_to_echo=64503 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4085, 0, 4, 4091, 0, 0, 79, 0, 0, 4096, 0, 0, 3348, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 2525, 4096, 0, 0, 0, 0, 0, 0, 0, 3804, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 387, 4096, 0, 0, 0, 1, 78, 0, 0, 0, 0, 0, 4096, 0, 1687, 334, 290, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 26, 2, 0, 0, 0, 16, 0, 0, 0, 4096, 2842, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 4096, 3839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1134, 42, 3971, 0, 4075, 0, 3797, 3982, 2334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 3835, 0, 0, 0, 248, 0, 0, 0, 4095, 3762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2409, 0, 0, 1, 0, 3662, 0, 1, 0, 3897, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15960, 14537, 15185, 15921, 16106, 19335, 15866, 18162] avg=16384.0000 max=19335 max_rank=5 min=14537 min_rank=1 imbalance_ratio=1.180115 max_load_reduction=111737 max_load_reduction_pct=0.852486 moved_to_echo=69469 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 48, 0, 0, 4088, 0, 0, 2735, 80, 0, 1007, 0, 0, 4096, 1, 21, 3289, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2850, 0, 0, 0, 576, 3835, 2, 539, 0, 911, 0, 50, 617, 3573, 4094, 3507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 4060, 182, 0, 149, 34, 0, 0, 0, 0, 0, 0, 4092, 0, 1687, 324, 272, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3351, 4072, 255, 26, 0, 0, 0, 0, 0, 0, 11, 27, 0, 2, 0, 4075, 2618, 735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3653, 148, 0, 0, 32, 2, 3, 0, 3539, 1, 88, 0, 0, 4066, 3839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 302, 7, 4, 1046, 0, 3820, 0, 4048, 29, 4035, 3896, 2586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 2, 4055, 4075, 0, 195, 0, 454, 9, 35, 0, 3560, 3658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2409, 0, 0, 476, 1, 1253, 1, 2288, 0, 6, 2668, 0, 35, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15622, 20554, 14917, 15172, 15371, 19775, 16361, 13300] avg=16384.0000 max=20554 max_rank=1 min=13300 min_rank=7 imbalance_ratio=1.254517 max_load_reduction=110518 max_load_reduction_pct=0.843185 moved_to_echo=64807 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[419, 0, 1, 10, 3986, 0, 348, 4069, 0, 3, 0, 32, 91, 3740, 92, 0, 3339, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 13, 3557, 4064, 0, 0, 19, 103, 0, 20, 0, 3806, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3758, 0, 1, 4061, 129, 6, 1, 0, 2379, 1, 548, 0, 215, 0, 4084, 0, 1647, 334, 290, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 4041, 0, 0, 4, 0, 11, 8, 0, 0, 99, 0, 1, 0, 0, 4071, 2842, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3562, 0, 29, 0, 0, 0, 55, 0, 4072, 831, 0, 0, 2, 4093, 3800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 152, 0, 0, 4, 0, 0, 1134, 0, 3694, 0, 3974, 28, 0, 4086, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3481, 3248, 0, 79, 54, 7, 0, 0, 0, 4089, 3762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 2351, 25, 0, 935, 0, 3048, 0, 2373, 0, 35, 4091, 0, 36, 0, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 19648, 17574, 15862, 16444, 13089, 14720, 17351] avg=16384.0000 max=19648 max_rank=1 min=13089 min_rank=5 imbalance_ratio=1.199219 max_load_reduction=111424 max_load_reduction_pct=0.850098 moved_to_echo=61604 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 52, 12, 4070, 38, 0, 939, 544, 0, 4062, 0, 0, 3347, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1349, 0, 5, 0, 3603, 4076, 19, 265, 0, 3, 0, 5, 0, 3801, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 231, 17, 4094, 4, 0, 2, 0, 0, 2, 0, 0, 0, 2, 4095, 0, 1687, 333, 289, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3524, 4094, 1, 0, 0, 0, 2787, 0, 19, 0, 0, 0, 0, 0, 2, 4091, 2842, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 12, 3880, 67, 0, 0, 0, 3938, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1134, 0, 4096, 0, 4003, 0, 4055, 3735, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 20, 0, 122, 0, 4088, 2389, 43, 1, 0, 0, 0, 0, 0, 4029, 3761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2409, 0, 6, 0, 0, 4046, 0, 39, 0, 890, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17416, 17219, 14960, 18109, 15831, 17486, 14469, 15582] avg=16384.0000 max=18109 max_rank=3 min=14469 min_rank=6 imbalance_ratio=1.105286 max_load_reduction=112963 max_load_reduction_pct=0.861839 moved_to_echo=63368 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[26368, 32640, 4736, 128, 128, 128, 4736, 1024, 128, 0, 128, 512, 128, 128, 128, 32768, 22400, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 384, 128, 896, 128, 31872, 29696, 128, 384, 384, 896, 5888, 128, 128, 30976, 29952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[8320, 0, 128, 128, 23040, 32512, 640, 1536, 128, 1280, 0, 128, 1920, 30208, 27520, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32640, 19200, 128, 128, 1536, 128, 24960, 128, 15616, 128, 5504, 31360, 128, 128, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 29952, 256, 256, 128, 128, 128, 128, 1408, 31744, 5120, 128, 384, 128, 31744, 30720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[512, 128, 128, 256, 31872, 512, 896, 30976, 256, 128, 3840, 1408, 256, 32384, 128, 128, 26624, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32384, 384, 512, 32640, 384, 128, 256, 128, 5632, 128, 4352, 0, 256, 128, 32768, 2944, 13440, 2688, 2304, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 256, 128, 0, 384, 128, 128, 8960, 128, 31872, 128, 28032, 128, 21504, 31104, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4093, 0, 0, 1073, 0, 2, 0, 4096, 4096, 0, 1, 0, 913, 1, 0, 0, 0, 5, 0, 0, 58, 4096, 0, 4008, 4041, 4096, 0, 65, 0, 0, 0, 290, 0, 0, 0, 4091, 0, 1, 0, 622, 4094, 3527, 0, 0, 0, 0, 4096, 4096, 0, 159, 4091, 0, 0, 0, 7, 53, 0, 4096, 4096, 0, 4022, 0, 1, 4096, 0, 0, 0, 0, 3755, 4091, 0, 857, 0, 0, 0, 0, 0, 0, 655, 7, 0, 643, 0, 14, 0, 4092, 6, 0, 4096, 0, 16, 110, 4096, 4096, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 4096, 1192, 0, 4096, 0, 4096, 0, 0, 0, 2623] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 6, 4096, 19, 0, 4096, 0, 0, 93, 80, 75, 0, 4096, 4096, 0, 0, 0, 4096, 3217, 196, 0, 0, 0, 0, 0, 4063, 4096, 0, 4096, 4090, 4071, 0, 705, 0, 0, 0, 479, 0, 0, 0, 161, 0, 1010, 0, 6, 4096, 3909, 0, 0, 183, 0, 4096, 4000, 413, 4094, 154, 2, 0, 0, 24, 21, 0, 3996, 3964, 0, 0, 0, 0, 4096, 0, 77, 1840, 0, 3509, 2113, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 1, 0, 3, 4096, 0, 3063, 0, 0, 0, 4096, 4096, 0, 4019, 0, 3655, 279, 0, 1, 3, 12, 92, 0, 0, 0, 0, 6, 0, 4096, 0, 4096, 4064, 13, 4096, 0, 3745, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[68, 0, 0, 4062, 0, 0, 4073, 1, 3, 0, 0, 425, 0, 4095, 3769, 0, 86, 0, 3, 284, 0, 240, 34, 0, 0, 0, 0, 3987, 10, 3162, 4096, 4096, 22, 500, 27, 0, 0, 3835, 1048, 48, 0, 1, 0, 0, 0, 4077, 4027, 3918, 0, 0, 0, 112, 4094, 3981, 56, 68, 136, 0, 1, 0, 0, 74, 0, 4047, 2704, 3, 3826, 9, 9, 4096, 0, 0, 0, 0, 4045, 4096, 0, 3973, 0, 13, 1, 0, 1, 30, 300, 0, 2407, 0, 0, 0, 0, 4096, 0, 0, 3724, 0, 15, 3378, 3957, 3972, 0, 3964, 0, 4028, 3, 0, 80, 0, 10, 0, 14, 0, 8, 0, 2, 36, 2, 2, 4096, 4095, 48, 3606, 75, 3665, 117, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 4096, 0, 0, 2928, 33, 0, 0, 4096, 4096, 0, 0, 0, 3975, 825, 0, 0, 0, 154, 0, 0, 106, 4096, 0, 4096, 4089, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4076, 4096, 4003, 0, 0, 0, 0, 4096, 4096, 0, 315, 4052, 0, 0, 0, 0, 35, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 3413, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 0, 0, 122, 4096, 4096, 0, 4096, 0, 3979, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 0, 4096, 4094, 0, 4071, 0, 4093, 0, 81, 0, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[2, 12, 0, 4075, 53, 8, 4088, 0, 26, 128, 0, 0, 0, 3756, 4076, 18, 0, 1, 3258, 1835, 0, 0, 92, 1040, 0, 0, 1392, 4080, 0, 4088, 2441, 3591, 0, 0, 17, 1, 1, 2978, 0, 0, 2, 4095, 3, 5, 37, 133, 4071, 4083, 0, 1, 0, 7, 4092, 4078, 385, 442, 740, 0, 0, 0, 446, 291, 6, 4050, 3994, 0, 0, 0, 0, 3837, 0, 0, 0, 0, 3545, 3048, 12, 0, 27, 3, 0, 0, 0, 15, 0, 0, 0, 4096, 0, 189, 0, 2824, 3953, 0, 4090, 7, 0, 0, 4093, 4095, 0, 4096, 10, 3575, 0, 460, 0, 0, 43, 0, 0, 6, 0, 0, 0, 0, 1193, 1, 4084, 3463, 0, 4096, 0, 4096, 0, 0, 0, 97] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 2, 0, 4010, 0, 0, 4040, 0, 0, 0, 0, 2, 0, 4096, 4057, 5, 172, 0, 2862, 3790, 0, 2, 43, 9, 0, 0, 520, 4032, 0, 4074, 4040, 3524, 31, 3912, 29, 0, 0, 0, 0, 0, 0, 3680, 2, 0, 0, 0, 3602, 4096, 0, 22, 168, 0, 4053, 3999, 34, 2064, 4096, 2, 62, 0, 0, 0, 0, 4090, 3931, 0, 2315, 0, 85, 4096, 0, 313, 289, 0, 1382, 4096, 0, 1400, 87, 14, 31, 0, 2, 16, 0, 606, 464, 0, 0, 0, 0, 4085, 219, 0, 2677, 0, 6, 45, 4072, 4086, 0, 4095, 0, 2745, 0, 0, 0, 1, 2, 0, 0, 0, 125, 0, 0, 15, 588, 67, 4091, 3920, 7, 4088, 0, 3785, 0, 0, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[698, 1, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 106, 0, 4076, 4091, 0, 2, 13, 33, 1992, 0, 11, 37, 225, 0, 0, 0, 4089, 3, 3231, 4066, 4087, 0, 0, 3, 0, 0, 3707, 31, 0, 0, 4051, 0, 100, 9, 2, 4058, 3239, 3, 0, 0, 1, 4089, 4096, 2, 2839, 0, 0, 0, 209, 2, 32, 0, 4093, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3645, 4089, 0, 149, 230, 1, 0, 0, 22, 1, 814, 0, 0, 0, 561, 0, 70, 3407, 2562, 0, 3796, 0, 41, 0, 4093, 4094, 0, 4096, 1, 4096, 0, 5, 0, 0, 5, 0, 0, 0, 0, 1, 1, 5, 1160, 0, 4091, 4075, 0, 4096, 0, 4059, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4083, 0, 228, 4095, 0, 0, 271, 115, 48, 0, 3757, 4073, 0, 0, 0, 0, 615, 0, 0, 256, 1, 0, 0, 238, 4096, 0, 4096, 3547, 3764, 3, 518, 0, 0, 0, 0, 0, 33, 32, 4042, 0, 3054, 0, 3413, 4096, 4046, 0, 12, 35, 0, 4034, 4081, 0, 39, 5, 0, 0, 0, 0, 85, 0, 4096, 3970, 0, 2131, 0, 0, 4087, 0, 16, 0, 41, 3442, 1007, 0, 4048, 0, 0, 0, 0, 1, 0, 3285, 0, 0, 0, 520, 25, 75, 2118, 0, 0, 4096, 0, 0, 0, 4053, 4079, 11, 4050, 0, 4070, 0, 102, 524, 0, 0, 95, 0, 0, 0, 15, 0, 3, 4051, 0, 4096, 3962, 0, 4096, 0, 3832, 0, 249, 15, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 4096, 19, 0, 2936, 0, 0, 93, 80, 75, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3217, 196, 0, 0, 0, 0, 0, 4063, 228, 0, 4096, 4090, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 705, 0, 0, 0, 479, 0, 0, 0, 161, 0, 1010, 0, 6, 4096, 3909, 594, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 4096, 4000, 413, 4094, 154, 2, 0, 0, 24, 21, 0, 3996, 620, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 0, 0, 0, 0, 2654, 0, 77, 1840, 0, 3509, 2113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 1, 0, 3, 4096, 0, 3063, 0, 3868, 1680, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3795, 0, 4019, 0, 3655, 279, 0, 1, 3, 12, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 2416, 4064, 13, 4096, 0, 3745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15497, 24057, 11500, 17904, 14157, 13569, 15952, 18436] avg=16384.0000 max=24057 max_rank=1 min=11500 min_rank=2 imbalance_ratio=1.468323 max_load_reduction=107015 max_load_reduction_pct=0.816460 moved_to_echo=62114 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 2933, 0, 0, 1073, 0, 2, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 913, 1, 0, 0, 0, 5, 0, 0, 58, 228, 0, 4008, 4041, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 290, 0, 0, 0, 4091, 0, 1, 0, 622, 4094, 3527, 594, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 159, 4091, 0, 0, 0, 7, 53, 0, 4096, 620, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4022, 0, 1, 2654, 0, 0, 0, 0, 3755, 4091, 0, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 655, 7, 0, 643, 0, 14, 0, 4092, 6, 0, 4096, 0, 3868, 1680, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 110, 4096, 3795, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 2416, 1192, 0, 4096, 0, 4096, 0, 0, 0, 2623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16296, 13351, 13824, 17519, 19476, 15909, 16208, 18489] avg=16384.0000 max=19476 max_rank=4 min=13351 min_rank=1 imbalance_ratio=1.188721 max_load_reduction=111596 max_load_reduction_pct=0.851410 moved_to_echo=70082 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[68, 0, 0, 4062, 0, 0, 2919, 1, 3, 0, 0, 425, 0, 4095, 3769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 3, 284, 0, 240, 34, 0, 0, 0, 0, 222, 10, 3162, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 500, 27, 0, 0, 3835, 1048, 48, 0, 1, 0, 0, 0, 4077, 4027, 3918, 594, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 4094, 3981, 56, 68, 136, 0, 1, 0, 0, 74, 0, 4047, 617, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2704, 3, 3826, 9, 9, 2654, 0, 0, 0, 0, 4045, 4096, 0, 3973, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 30, 300, 0, 2407, 0, 0, 0, 0, 4096, 0, 0, 3724, 0, 3765, 1680, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3378, 3957, 3680, 0, 3964, 0, 4028, 3, 0, 80, 0, 10, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 36, 2, 2, 2416, 4095, 48, 3606, 75, 3665, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15342, 12233, 18634, 13478, 21332, 16852, 19129, 14072] avg=16384.0000 max=21332 max_rank=4 min=12233 min_rank=1 imbalance_ratio=1.302002 max_load_reduction=109740 max_load_reduction_pct=0.837250 moved_to_echo=71385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4083, 0, 228, 2935, 0, 0, 271, 115, 48, 0, 3757, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 615, 0, 0, 256, 1, 0, 0, 238, 228, 0, 4096, 3547, 3764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 518, 0, 0, 0, 0, 0, 33, 32, 4042, 0, 3054, 0, 3413, 4096, 4046, 592, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 35, 0, 4034, 4081, 0, 39, 5, 0, 0, 0, 0, 85, 0, 4096, 620, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 0, 2131, 0, 0, 2649, 0, 16, 0, 41, 3442, 1007, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3285, 0, 0, 0, 520, 25, 75, 2118, 0, 0, 4096, 0, 3868, 1680, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 3780, 11, 4050, 0, 4070, 0, 102, 524, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3, 4051, 0, 2416, 3962, 0, 4096, 0, 3832, 0, 249, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15510, 12745, 20369, 13306, 17304, 16514, 16685, 18639] avg=16384.0000 max=20369 max_rank=2 min=12745 min_rank=1 imbalance_ratio=1.243225 max_load_reduction=110703 max_load_reduction_pct=0.844597 moved_to_echo=69142 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 0, 2936, 0, 0, 2928, 33, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 825, 0, 0, 0, 154, 0, 0, 106, 228, 0, 4096, 4089, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4076, 4096, 4003, 594, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 315, 4052, 0, 0, 0, 0, 35, 0, 4096, 620, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2654, 0, 0, 2, 0, 3413, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 0, 3868, 1680, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 4096, 3795, 0, 4096, 0, 3979, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 0, 2416, 4094, 0, 4071, 0, 4093, 0, 81, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18186, 17536, 17405, 17611, 14261, 14671, 16141, 15261] avg=16384.0000 max=18186 max_rank=0 min=14261 min_rank=4 imbalance_ratio=1.109985 max_load_reduction=112886 max_load_reduction_pct=0.861252 moved_to_echo=60334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 12, 0, 4075, 53, 8, 2921, 0, 26, 128, 0, 0, 0, 3756, 4076, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3258, 1835, 0, 0, 92, 1040, 0, 0, 1392, 223, 0, 4088, 2441, 3591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 1, 2978, 0, 0, 2, 4095, 3, 5, 37, 133, 4071, 4083, 558, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 4092, 4078, 385, 442, 740, 0, 0, 0, 446, 291, 6, 4050, 624, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 2483, 0, 0, 0, 0, 3545, 3048, 12, 0, 27, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4096, 0, 189, 0, 2824, 3953, 0, 4090, 7, 3857, 1677, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3792, 0, 4096, 10, 3575, 0, 460, 0, 0, 43, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1193, 1, 2407, 3463, 0, 4096, 0, 4096, 0, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15075, 17961, 16527, 15465, 13112, 21504, 16075, 15353] avg=16384.0000 max=21504 max_rank=5 min=13112 min_rank=4 imbalance_ratio=1.312500 max_load_reduction=109568 max_load_reduction_pct=0.835938 moved_to_echo=66044 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 4010, 0, 0, 2895, 0, 0, 0, 0, 2, 0, 4096, 4057, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 2862, 3790, 0, 2, 43, 9, 0, 0, 520, 225, 0, 4074, 4040, 3524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3912, 29, 0, 0, 0, 0, 0, 0, 3680, 2, 0, 0, 0, 3602, 4096, 594, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 168, 0, 4053, 3999, 34, 2064, 4096, 2, 62, 0, 0, 0, 0, 4090, 612, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3931, 0, 2315, 0, 85, 2654, 0, 313, 289, 0, 1382, 4096, 0, 1400, 87, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 2, 16, 0, 606, 464, 0, 0, 0, 0, 4085, 219, 0, 2677, 0, 3807, 1678, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 45, 4072, 3786, 0, 4095, 0, 2745, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 15, 588, 67, 2413, 3920, 7, 4088, 0, 3785, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15067, 19261, 16479, 19502, 16566, 14433, 14752, 15012] avg=16384.0000 max=19502 max_rank=3 min=14433 min_rank=5 imbalance_ratio=1.190308 max_load_reduction=111570 max_load_reduction_pct=0.851212 moved_to_echo=60763 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[698, 1, 0, 4096, 0, 0, 2935, 0, 0, 4096, 0, 106, 0, 4076, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 13, 33, 1992, 0, 11, 37, 225, 0, 0, 0, 228, 3, 3231, 4066, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3707, 31, 0, 0, 4051, 0, 100, 9, 2, 4058, 3239, 594, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 4089, 4096, 2, 2839, 0, 0, 0, 209, 2, 32, 0, 4093, 620, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2654, 0, 0, 0, 0, 3645, 4089, 0, 149, 230, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 1, 814, 0, 0, 0, 561, 0, 70, 3407, 2562, 0, 3796, 0, 3861, 1678, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 4093, 3793, 0, 4096, 1, 4096, 0, 5, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5, 1160, 0, 2413, 4075, 0, 4096, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20099, 13928, 16334, 16287, 14864, 17620, 16130, 15810] avg=16384.0000 max=20099 max_rank=0 min=13928 min_rank=1 imbalance_ratio=1.226746 max_load_reduction=110973 max_load_reduction_pct=0.846657 moved_to_echo=64424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 512, 128, 32768, 32512, 896, 10112, 13312, 128, 128, 256, 512, 640, 128, 32640, 4992, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 128, 15232, 12672, 256, 256, 512, 1536, 0, 0, 6400, 1920, 128, 30976, 30464, 31360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[768, 128, 128, 32640, 128, 256, 23424, 128, 128, 8704, 256, 768, 0, 32128, 32384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 3712, 32640, 30336, 128, 32512, 128, 30336, 384, 640, 640, 128, 128, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[30976, 128, 12416, 128, 128, 21120, 0, 512, 2176, 128, 26752, 26752, 128, 10496, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 5248, 640, 2944, 4864, 1152, 256, 256, 24832, 10880, 0, 29696, 128, 30848, 13440, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 128, 128, 128, 15744, 128, 19328, 28928, 128, 32256, 128, 31488, 128, 384, 128, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 5760, 128, 128, 128, 11392, 1152, 128, 128, 24320, 128, 4224, 128, 12416, 32256, 30848, 4736, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[624, 0, 612, 0, 0, 0, 0, 39, 4096, 0, 0, 1, 2, 0, 4096, 0, 4, 0, 0, 3525, 85, 0, 0, 0, 3753, 0, 0, 0, 0, 0, 2, 0, 0, 3444, 4096, 3963, 4095, 0, 0, 0, 230, 4096, 0, 3, 1119, 0, 0, 97, 4096, 39, 0, 0, 0, 4096, 0, 4096, 0, 90, 0, 0, 3682, 4096, 0, 1, 4096, 4096, 0, 2, 0, 0, 0, 4084, 0, 0, 0, 4021, 0, 106, 0, 0, 0, 4096, 3455, 4069, 0, 0, 0, 1, 19, 649, 0, 0, 0, 4096, 4080, 0, 3133, 0, 4080, 4070, 0, 0, 0, 0, 101, 4092, 16, 0, 0, 0, 5, 0, 4094, 0, 0, 0, 0, 0, 0, 247, 3904, 0, 0, 0, 4090, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1794, 39, 0, 0, 0, 65, 0, 441, 4088, 4, 0, 4096, 13, 0, 4095, 0, 0, 19, 0, 80, 0, 0, 683, 15, 4095, 7, 0, 0, 58, 91, 0, 5, 3, 1, 4063, 4075, 4052, 0, 1, 33, 3040, 4038, 0, 0, 179, 0, 41, 0, 4086, 0, 0, 8, 4, 4089, 6, 4086, 2, 0, 0, 13, 0, 4016, 7, 0, 4090, 4095, 1, 0, 0, 0, 51, 3602, 7, 0, 0, 3307, 0, 515, 4087, 0, 19, 4084, 4065, 4096, 0, 0, 0, 0, 0, 2850, 1, 0, 0, 4081, 4068, 0, 49, 209, 4061, 4093, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 2, 39, 0, 3789, 0, 0, 6, 15, 28, 712, 0, 4093, 0, 12, 0, 3301, 4005, 2, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[108, 0, 1364, 94, 0, 0, 0, 456, 4096, 0, 0, 4093, 0, 0, 3745, 0, 0, 0, 0, 2371, 0, 52, 0, 0, 3975, 0, 0, 6, 0, 0, 11, 0, 0, 1105, 3085, 4095, 3998, 12, 0, 0, 598, 3989, 0, 15, 2803, 70, 138, 373, 4026, 0, 0, 0, 141, 4089, 13, 3795, 11, 0, 16, 0, 948, 4096, 15, 20, 4094, 4096, 2, 24, 0, 205, 1, 4070, 0, 4, 0, 1, 0, 1730, 3566, 0, 0, 3933, 4048, 1817, 0, 0, 0, 0, 0, 2532, 0, 51, 0, 4082, 4017, 2, 401, 0, 3764, 3770, 1, 1, 0, 0, 0, 3603, 710, 24, 0, 0, 7, 0, 3953, 0, 103, 0, 3161, 3251, 0, 0, 4069, 0, 0, 12, 1, 4062, 0, 4082] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 40, 0, 0, 0, 0, 0, 4096, 0, 0, 1486, 0, 0, 4096, 0, 1542, 0, 0, 2448, 0, 0, 1, 0, 4073, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 3975, 4096, 0, 0, 0, 4059, 4096, 0, 0, 12, 0, 39, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 11, 3914, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 51, 0, 3634, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 4096, 4, 3892, 129, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 30, 0, 21, 0, 447, 0, 2975, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 1, 0, 0, 0, 14, 4096, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4067, 0, 1, 9, 4096, 243, 0, 0, 0, 4096, 0, 4096, 14, 0, 0, 0, 147, 4096, 164, 9, 4096, 2194, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 67, 0, 29, 4095, 0, 0, 4095, 4096, 4094, 2, 0, 1, 0, 3, 4096, 0, 0, 0, 4096, 4096, 0, 1759, 0, 4096, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 4004, 2930, 0, 0, 472, 0, 3569, 0, 804, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[34, 0, 3923, 2, 0, 0, 0, 0, 4096, 0, 88, 3870, 0, 0, 4009, 301, 53, 0, 0, 4096, 0, 1, 300, 0, 3598, 0, 0, 0, 0, 0, 41, 0, 0, 4095, 4096, 757, 4096, 0, 0, 1, 3563, 4064, 0, 0, 344, 2, 0, 205, 4095, 542, 0, 0, 0, 4074, 0, 4093, 0, 0, 28, 0, 0, 3932, 399, 0, 4096, 1254, 0, 3, 0, 5, 113, 4096, 0, 607, 0, 223, 0, 801, 3838, 0, 0, 4096, 4095, 4089, 48, 0, 0, 0, 95, 4050, 0, 0, 0, 4096, 4031, 0, 3176, 7, 4096, 3564, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 7, 21, 2, 0, 0, 1, 9, 0, 0, 1393, 0, 0, 4096, 0, 4081] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 81, 0, 0, 120, 0, 0, 4077, 7, 0, 3914, 0, 2, 3923, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4069, 0, 79, 0, 5, 80, 6, 0, 0, 297, 4035, 2214, 4081, 415, 90, 0, 3417, 4001, 0, 27, 123, 0, 4, 21, 4096, 3, 0, 0, 0, 4057, 0, 4095, 0, 5, 0, 0, 3275, 3756, 228, 1, 4078, 206, 0, 0, 0, 6, 0, 3976, 36, 0, 0, 3939, 9, 267, 3970, 0, 0, 4052, 4096, 3047, 0, 7, 0, 0, 669, 3439, 0, 0, 4, 4085, 4096, 0, 2803, 125, 4064, 3458, 0, 11, 51, 4, 0, 4093, 0, 0, 29, 1, 0, 1690, 4052, 586, 4, 0, 3788, 2, 2084, 0, 0, 0, 1426, 42, 83, 3989, 17, 4083] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[114, 68, 26, 0, 4, 0, 0, 8, 4096, 0, 0, 4072, 0, 0, 4080, 0, 100, 0, 0, 122, 100, 0, 0, 0, 4091, 0, 0, 0, 0, 6, 4, 0, 0, 256, 4096, 4096, 4017, 0, 4, 0, 4037, 4087, 0, 0, 23, 0, 21, 34, 3967, 0, 7, 0, 0, 4096, 0, 4045, 0, 0, 0, 0, 185, 3999, 0, 5, 4092, 3758, 0, 0, 0, 0, 1, 4085, 0, 0, 0, 7, 0, 38, 4095, 0, 0, 4096, 4096, 4096, 1, 1, 0, 1, 0, 4095, 0, 0, 0, 4090, 4095, 0, 3697, 0, 4024, 4096, 0, 0, 0, 0, 1, 4052, 90, 5, 0, 2, 367, 29, 3892, 0, 0, 19, 3445, 0, 0, 597, 34, 0, 0, 0, 4022, 4091, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[624, 0, 612, 0, 0, 0, 0, 39, 4096, 0, 0, 1, 2, 0, 4096, 0, 2784, 1000, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3525, 85, 0, 0, 0, 3753, 0, 0, 0, 0, 0, 2, 0, 4096, 4095, 2261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3444, 4096, 3963, 0, 0, 0, 0, 230, 3729, 0, 3, 1119, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 39, 0, 0, 0, 2999, 0, 4096, 0, 90, 0, 0, 3682, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3457, 4096, 0, 2, 0, 0, 0, 4084, 0, 0, 0, 4021, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3455, 4069, 0, 0, 0, 1, 19, 649, 0, 0, 0, 0, 1296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3133, 0, 4080, 4070, 0, 0, 0, 0, 101, 4092, 16, 0, 0, 0, 5, 0, 639, 603, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 247, 3904, 0, 0, 0, 4090, 4096, 0, 835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13748, 17821, 16681, 19099, 15766, 13585, 17106, 17266] avg=16384.0000 max=19099 max_rank=3 min=13585 min_rank=5 imbalance_ratio=1.165710 max_load_reduction=111973 max_load_reduction_pct=0.854286 moved_to_echo=63723 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 7, 0, 0, 4096, 0, 2795, 1000, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 1, 0, 0, 4096, 4096, 2261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 4096, 0, 0, 0, 0, 4096, 3729, 0, 0, 4067, 0, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 243, 0, 0, 0, 2999, 0, 4096, 14, 0, 0, 0, 147, 4096, 164, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3457, 2194, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 67, 0, 29, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4094, 2, 0, 1, 0, 3, 4096, 0, 0, 0, 0, 1301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 0, 4096, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 639, 603, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 4004, 2930, 0, 0, 472, 0, 3569, 0, 804, 4096, 0, 835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12489, 14557, 20108, 15864, 13938, 17688, 15654, 20774] avg=16384.0000 max=20774 max_rank=7 min=12489 min_rank=0 imbalance_ratio=1.267944 max_load_reduction=110298 max_load_reduction_pct=0.841507 moved_to_echo=68054 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[34, 0, 3923, 2, 0, 0, 0, 0, 4096, 0, 88, 3870, 0, 0, 4009, 301, 2751, 996, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 4096, 0, 1, 300, 0, 3598, 0, 0, 0, 0, 0, 41, 0, 4096, 4096, 2253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 757, 0, 0, 0, 1, 3563, 3700, 0, 0, 344, 2, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 542, 0, 0, 0, 2982, 0, 4093, 0, 0, 28, 0, 0, 3932, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3457, 1254, 0, 3, 0, 5, 113, 4096, 0, 607, 0, 223, 0, 801, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4089, 48, 0, 0, 0, 95, 4050, 0, 0, 0, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3176, 7, 4096, 3564, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 639, 600, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 21, 2, 0, 0, 1, 9, 0, 0, 1393, 0, 0, 4096, 0, 832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20562, 18534, 16763, 16071, 14397, 17753, 16535, 10457] avg=16384.0000 max=20562 max_rank=0 min=10457 min_rank=7 imbalance_ratio=1.255005 max_load_reduction=110510 max_load_reduction_pct=0.843124 moved_to_echo=59142 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 40, 0, 0, 0, 0, 0, 4096, 0, 0, 1486, 0, 0, 4096, 0, 2795, 1000, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1542, 0, 0, 2448, 0, 0, 1, 0, 4073, 0, 1, 0, 0, 0, 0, 0, 4096, 4096, 2261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3975, 0, 0, 0, 0, 4059, 3729, 0, 0, 12, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2999, 0, 4096, 0, 0, 0, 0, 11, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3457, 4086, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 51, 0, 3634, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 1301, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 129, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 639, 603, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 30, 0, 21, 0, 447, 0, 2975, 4096, 0, 835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14007, 18518, 15910, 15116, 19417, 17683, 17918, 12503] avg=16384.0000 max=19417 max_rank=4 min=12503 min_rank=7 imbalance_ratio=1.185120 max_load_reduction=111655 max_load_reduction_pct=0.851860 moved_to_echo=67521 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[108, 0, 1364, 94, 0, 0, 0, 456, 4096, 0, 0, 4093, 0, 0, 3745, 0, 2741, 997, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2371, 0, 52, 0, 0, 3975, 0, 0, 6, 0, 0, 11, 0, 4082, 3998, 2253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 3085, 4095, 0, 12, 0, 0, 598, 3632, 0, 15, 2803, 70, 138, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 141, 2993, 13, 3795, 11, 0, 16, 0, 948, 4096, 15, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3455, 4096, 2, 24, 0, 205, 1, 4070, 0, 4, 0, 1, 0, 1730, 3566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 4048, 1817, 0, 0, 0, 0, 0, 2532, 0, 51, 0, 0, 1276, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 0, 3764, 3770, 1, 1, 0, 0, 0, 3603, 710, 24, 0, 0, 7, 0, 639, 602, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3953, 0, 103, 0, 3161, 3251, 0, 0, 4069, 0, 0, 12, 1, 4062, 0, 832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18188, 16748, 15926, 16074, 17154, 13659, 13879, 19444] avg=16384.0000 max=19444 max_rank=7 min=13659 min_rank=5 imbalance_ratio=1.186768 max_load_reduction=111628 max_load_reduction_pct=0.851654 moved_to_echo=64136 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 81, 0, 0, 120, 0, 0, 4077, 7, 0, 3914, 0, 2, 3923, 0, 2795, 997, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4069, 0, 79, 0, 5, 80, 6, 0, 4085, 4081, 2254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 4035, 2214, 0, 415, 90, 0, 3417, 3643, 0, 27, 123, 0, 4, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 2970, 0, 4095, 0, 5, 0, 0, 3275, 3756, 228, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3442, 206, 0, 0, 0, 6, 0, 3976, 36, 0, 0, 3939, 9, 267, 3970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 4096, 3047, 0, 7, 0, 0, 669, 3439, 0, 0, 4, 0, 1301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2803, 125, 4064, 3458, 0, 11, 51, 4, 0, 4093, 0, 0, 29, 1, 0, 1690, 636, 597, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 586, 4, 0, 3788, 2, 2084, 0, 0, 0, 1426, 42, 83, 3989, 17, 832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16406, 14660, 14286, 18429, 15851, 16615, 17920, 16905] avg=16384.0000 max=18429 max_rank=3 min=14286 min_rank=2 imbalance_ratio=1.124817 max_load_reduction=112643 max_load_reduction_pct=0.859398 moved_to_echo=67291 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1794, 39, 0, 0, 0, 65, 0, 441, 4088, 4, 0, 4096, 13, 0, 4095, 0, 2781, 1002, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 80, 0, 0, 683, 15, 4095, 7, 0, 0, 58, 91, 0, 5, 4081, 4052, 2257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 4063, 4075, 0, 0, 1, 33, 3040, 3674, 0, 0, 179, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 8, 4, 2984, 6, 4086, 2, 0, 0, 13, 0, 4016, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3447, 4095, 1, 0, 0, 0, 51, 3602, 7, 0, 0, 3307, 0, 515, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4084, 4065, 4096, 0, 0, 0, 0, 0, 2850, 1, 0, 0, 0, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 209, 4061, 4093, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 2, 39, 0, 643, 607, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3789, 0, 0, 6, 15, 28, 712, 0, 4093, 0, 12, 0, 3301, 4005, 2, 825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18916, 15443, 15110, 15212, 19112, 16402, 14089, 16788] avg=16384.0000 max=19112 max_rank=4 min=14089 min_rank=6 imbalance_ratio=1.166504 max_load_reduction=111960 max_load_reduction_pct=0.854187 moved_to_echo=66391 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[114, 68, 26, 0, 4, 0, 0, 8, 4096, 0, 0, 4072, 0, 0, 4080, 0, 2794, 1000, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 122, 100, 0, 0, 0, 4091, 0, 0, 0, 0, 6, 4, 0, 4090, 4017, 2261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 4096, 4096, 0, 0, 4, 0, 4037, 3721, 0, 0, 23, 0, 21, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 7, 0, 0, 2999, 0, 4045, 0, 0, 0, 0, 185, 3999, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3453, 3758, 0, 0, 0, 0, 1, 4085, 0, 0, 0, 7, 0, 38, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 1, 1, 0, 1, 0, 4095, 0, 0, 0, 0, 1301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3697, 0, 4024, 4096, 0, 0, 0, 0, 1, 4052, 90, 5, 0, 2, 367, 29, 639, 603, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 0, 0, 19, 3445, 0, 0, 597, 34, 0, 0, 0, 4022, 4091, 0, 835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16756, 14791, 16288, 15207, 15437, 17687, 17971, 16935] avg=16384.0000 max=17971 max_rank=6 min=14791 min_rank=1 imbalance_ratio=1.096863 max_load_reduction=113101 max_load_reduction_pct=0.862892 moved_to_echo=68030 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[2688, 128, 6144, 128, 128, 256, 128, 1024, 32768, 128, 128, 21632, 128, 128, 32256, 384, 22272, 8064, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 9216, 31744, 27392, 0, 512, 128, 128, 23040, 29568, 0, 128, 8704, 128, 256, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32128, 640, 128, 128, 14464, 6272, 2944, 896, 12672, 0, 6912, 128, 15360, 32640, 128, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[27648, 23808, 128, 128, 0, 256, 256, 32128, 128, 640, 0, 11648, 128, 7168, 27776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32640, 32128, 29440, 128, 128, 128, 128, 896, 25856, 128, 128, 128, 0, 10368, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[1792, 128, 0, 12672, 256, 128, 1024, 128, 31872, 128, 128, 128, 128, 256, 128, 128, 32768, 32640, 18176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32640, 896, 128, 128, 256, 23936, 128, 32512, 128, 128, 128, 128, 8320, 32000, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[18944, 512, 32384, 31360, 128, 128, 128, 128, 128, 32256, 896, 128, 128, 128, 512, 1792, 5120, 4864, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 296, 0, 4096, 0, 3976, 5, 103, 4096, 0, 4096, 4096, 0, 0, 48, 0, 4096, 0, 4096, 0, 0, 6, 0, 0, 0, 81, 0, 0, 0, 2455, 11, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 5, 0, 0, 0, 0, 14, 35, 0, 0, 4096, 0, 0, 4093, 0, 0, 4, 0, 0, 39, 2738, 64, 14, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 4096, 3473, 0, 0, 0, 4096, 4096, 0, 0, 4053, 4094, 0, 0, 0, 4096, 0, 0, 9, 2082, 0, 0, 0, 4085, 4096, 0, 9, 4096, 0, 0, 4096, 0, 0, 0, 4089, 4096, 1, 0, 0, 0, 0, 977, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 3454, 0, 765, 4085, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4, 3770, 0, 4087, 0, 4096, 3661, 0, 0, 31, 0, 3587, 0, 2522, 0, 0, 0, 523, 0, 0, 1722, 0, 0, 0, 0, 3908, 0, 0, 176, 0, 0, 67, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 470, 4096, 0, 0, 433, 0, 0, 67, 410, 248, 14, 0, 0, 4007, 0, 0, 0, 2, 4096, 0, 0, 3481, 477, 0, 0, 0, 4094, 4096, 0, 0, 4096, 3613, 0, 0, 4, 4095, 0, 0, 86, 3944, 8, 0, 0, 4078, 4023, 0, 0, 3969, 0, 0, 4066, 0, 9, 0, 59, 4096, 0, 0, 0, 0, 0, 3812, 3, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4044, 0, 4096, 0, 20, 380, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 642, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 3383, 0, 0, 3817, 0, 0, 179, 1, 59, 5, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 3810, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4024, 3836, 0, 0, 0, 4096, 0, 0, 1, 4077, 0, 0, 0, 213, 4096, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 3979, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 3901, 5, 0, 4068, 0, 4093, 4009, 83, 13, 27, 0, 4062, 0, 4096, 54, 0, 4004, 0, 4096, 3964, 0, 0, 0, 0, 3925, 0, 4002, 0, 14, 717, 4, 10, 1, 0, 0, 0, 7, 0, 3966, 0, 0, 267, 2, 11, 0, 5, 0, 263, 3918, 0, 3641, 6, 33, 0, 95, 0, 0, 0, 59, 0, 0, 4070, 0, 0, 4096, 0, 105, 0, 18, 2, 1134, 0, 4093, 275, 36, 0, 4077, 0, 0, 0, 0, 4055, 0, 56, 4095, 3813, 0, 0, 0, 4096, 3790, 0, 0, 4096, 1006, 0, 0, 129, 3968, 0, 0, 23, 16, 0, 0, 0, 2630, 4000, 18, 0, 4096, 1, 35, 4086, 3, 0, 1304, 1415, 4068, 1, 0, 0, 0, 0, 755, 114, 4076] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4060, 3, 0, 4084, 0, 4094, 0, 0, 0, 3004, 4, 4032, 0, 4046, 2851, 0, 4096, 25, 4064, 4030, 0, 11, 0, 4, 4096, 29, 4086, 4, 1, 84, 0, 0, 0, 146, 0, 0, 0, 0, 10, 0, 12, 0, 1, 0, 8, 14, 0, 0, 4069, 80, 4082, 0, 42, 0, 0, 0, 1, 0, 91, 0, 0, 4053, 0, 0, 4005, 65, 0, 2, 0, 2, 694, 0, 106, 23, 2, 0, 4095, 0, 0, 0, 52, 3905, 0, 5, 4096, 3963, 16, 0, 0, 4061, 4073, 17, 55, 53, 4002, 0, 4, 0, 4058, 0, 0, 0, 1595, 0, 4, 39, 4096, 4090, 0, 4, 4087, 1, 0, 4092, 0, 0, 753, 4080, 4080, 77, 0, 42, 13, 4, 3252, 0, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4085, 10, 4059, 0, 0, 0, 6, 1, 4096, 0, 3086, 0, 3, 3927, 0, 4095, 4089, 0, 0, 635, 0, 4096, 0, 4081, 0, 0, 1462, 9, 0, 0, 2, 0, 0, 0, 531, 2421, 0, 0, 482, 2, 0, 16, 4, 173, 1581, 3825, 0, 4096, 0, 0, 204, 0, 6, 38, 0, 91, 0, 0, 4091, 0, 150, 4065, 0, 0, 4096, 5, 0, 389, 0, 1, 66, 0, 0, 3976, 0, 0, 0, 0, 2379, 0, 0, 4096, 4063, 0, 0, 0, 4029, 4096, 0, 0, 3314, 493, 0, 0, 0, 4096, 0, 0, 0, 649, 0, 0, 0, 2788, 3828, 47, 0, 3922, 0, 0, 4017, 0, 0, 930, 4068, 4096, 0, 0, 0, 0, 0, 3916, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 1, 2, 4093, 8, 4095, 0, 0, 2, 1458, 0, 4096, 0, 3807, 7, 126, 4096, 0, 4096, 4085, 0, 0, 0, 0, 4019, 0, 4068, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 829, 4095, 0, 0, 24, 0, 0, 77, 2, 2, 0, 4062, 1, 4093, 0, 0, 0, 0, 0, 12, 11, 244, 0, 0, 4095, 0, 19, 4053, 0, 68, 1601, 2, 0, 2, 4059, 84, 0, 0, 0, 4085, 0, 0, 0, 0, 3739, 0, 0, 3915, 370, 0, 0, 0, 4090, 4096, 0, 0, 720, 3082, 34, 0, 0, 4096, 0, 0, 812, 4096, 0, 0, 4, 3963, 4096, 0, 0, 4092, 0, 11, 4096, 0, 2, 2, 3812, 4096, 0, 0, 0, 3, 2, 45, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 3001, 37, 0, 4035, 0, 4093, 0, 0, 0, 5, 0, 4042, 0, 5, 16, 117, 4088, 1, 3920, 4071, 0, 13, 1062, 0, 3655, 0, 4095, 1, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 434, 47, 0, 0, 0, 30, 2, 3990, 0, 3979, 133, 0, 220, 229, 0, 0, 0, 0, 0, 15, 4008, 0, 0, 3976, 0, 19, 3928, 0, 0, 0, 149, 0, 0, 1, 111, 3925, 26, 0, 0, 27, 4096, 0, 0, 3948, 3770, 3, 1, 0, 3830, 3910, 0, 0, 4096, 4095, 3, 5, 0, 4096, 0, 0, 4096, 4095, 64, 29, 157, 4072, 4096, 0, 0, 4077, 47, 0, 4092, 28, 157, 0, 4096, 4039, 0, 0, 74, 0, 36, 56, 252, 4082] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2969, 0, 0, 0, 0, 0, 4044, 0, 4096, 0, 20, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3938, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 642, 297, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 1321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 1674, 1127, 466, 395, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3383, 0, 0, 3817, 0, 0, 179, 1, 59, 5, 0, 0, 4096, 0, 0, 2775, 1910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3344, 4096, 0, 0, 0, 0, 0, 0, 0, 4024, 3836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1791, 0, 0, 1, 4077, 0, 0, 0, 213, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 3979, 4096, 0, 0, 0, 0, 0, 4096, 0, 2422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15605, 16225, 14548, 16108, 16225, 19396, 14274, 18691] avg=16384.0000 max=19396 max_rank=5 min=14274 min_rank=6 imbalance_ratio=1.183838 max_load_reduction=111676 max_load_reduction_pct=0.852020 moved_to_echo=68586 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3454, 0, 765, 2961, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4, 3770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 0, 0, 3661, 0, 0, 31, 0, 3587, 0, 2522, 0, 0, 0, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1722, 0, 0, 0, 0, 3908, 0, 0, 176, 0, 0, 67, 0, 0, 4096, 4096, 4094, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1674, 1124, 426, 395, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 4096, 0, 0, 433, 0, 0, 67, 410, 248, 14, 0, 0, 4007, 0, 0, 2774, 1910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 3055, 477, 0, 0, 0, 0, 0, 0, 0, 4096, 3613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1790, 0, 0, 86, 3944, 8, 0, 0, 4078, 4023, 0, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 9, 0, 59, 4096, 0, 0, 0, 0, 0, 3812, 3, 2422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19146, 14254, 19481, 16054, 14429, 15339, 17902, 14467] avg=16384.0000 max=19481 max_rank=2 min=14254 min_rank=1 imbalance_ratio=1.189026 max_load_reduction=111591 max_load_reduction_pct=0.851372 moved_to_echo=62137 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2969, 0, 0, 0, 0, 0, 296, 0, 4096, 0, 3976, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 3938, 0, 0, 4096, 0, 0, 48, 0, 4096, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 2455, 11, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 4096, 4096, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 5, 0, 0, 0, 0, 14, 35, 0, 0, 4096, 0, 1674, 1127, 501, 395, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4, 0, 0, 39, 2738, 64, 14, 0, 0, 4096, 0, 0, 2774, 1910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 3595, 3473, 0, 0, 0, 0, 0, 0, 0, 4053, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1791, 0, 0, 9, 2082, 0, 0, 0, 4085, 4096, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 4096, 1, 0, 0, 0, 0, 977, 0, 2422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15438, 16383, 16160, 16197, 15732, 19313, 16168, 15681] avg=16384.0000 max=19313 max_rank=5 min=15438 min_rank=0 imbalance_ratio=1.178772 max_load_reduction=111759 max_load_reduction_pct=0.852654 moved_to_echo=66894 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2961, 10, 0, 0, 0, 0, 6, 1, 4096, 0, 3086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3776, 0, 0, 4089, 0, 0, 635, 0, 4096, 0, 4081, 0, 0, 1462, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 531, 2421, 0, 0, 482, 2, 0, 16, 4, 173, 4059, 4095, 4029, 1321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1581, 3825, 0, 4096, 0, 0, 204, 0, 6, 38, 0, 91, 0, 0, 4091, 0, 1674, 1124, 501, 395, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 4065, 0, 0, 4096, 5, 0, 389, 0, 1, 66, 0, 0, 3976, 0, 0, 2775, 1910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2379, 0, 0, 3595, 4063, 0, 0, 0, 0, 0, 0, 0, 3314, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1791, 0, 0, 0, 649, 0, 0, 0, 2788, 3828, 47, 0, 3922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 0, 930, 4068, 4096, 0, 0, 0, 0, 0, 3916, 2, 2422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14256, 18151, 17135, 17777, 17433, 13844, 13025, 19451] avg=16384.0000 max=19451 max_rank=7 min=13025 min_rank=6 imbalance_ratio=1.187195 max_load_reduction=111621 max_load_reduction_pct=0.851601 moved_to_echo=63753 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3901, 5, 0, 2948, 0, 0, 4009, 83, 13, 27, 0, 4062, 0, 4096, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3850, 0, 0, 3964, 0, 0, 0, 0, 3925, 0, 4002, 0, 14, 717, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 7, 0, 3966, 0, 0, 267, 2, 11, 0, 5, 0, 4093, 4096, 4096, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 3918, 0, 3641, 6, 33, 0, 95, 0, 0, 0, 59, 0, 0, 4070, 0, 1666, 1120, 501, 383, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 105, 0, 18, 2, 1134, 0, 4093, 275, 36, 0, 4077, 0, 0, 2568, 1850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 56, 3594, 3813, 0, 0, 0, 0, 0, 0, 0, 4096, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 1735, 0, 0, 23, 16, 0, 0, 0, 2630, 4000, 18, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 35, 4086, 3, 0, 1304, 1415, 4068, 1, 0, 0, 0, 0, 755, 114, 2410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19198, 16476, 17776, 15909, 18254, 16620, 12647, 14192] avg=16384.0000 max=19198 max_rank=0 min=12647 min_rank=6 imbalance_ratio=1.171753 max_load_reduction=111874 max_load_reduction_pct=0.853531 moved_to_echo=61713 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3001, 37, 0, 2924, 0, 0, 0, 0, 0, 5, 0, 4042, 0, 5, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 3930, 1, 0, 4071, 0, 13, 1062, 0, 3655, 0, 4095, 1, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 434, 47, 0, 0, 0, 30, 4093, 3920, 3830, 1262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3990, 0, 3979, 133, 0, 220, 229, 0, 0, 0, 0, 0, 15, 4008, 0, 1668, 1111, 483, 395, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 0, 19, 3928, 0, 0, 0, 149, 0, 0, 1, 111, 3925, 26, 0, 2648, 1910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4096, 0, 0, 3465, 3770, 3, 1, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 0, 1791, 0, 0, 4096, 4095, 64, 29, 157, 4072, 4096, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 4092, 28, 157, 0, 4096, 4039, 0, 0, 74, 0, 36, 56, 252, 2414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10030, 17013, 13616, 16391, 16693, 19553, 22485, 15291] avg=16384.0000 max=22485 max_rank=6 min=10030 min_rank=0 imbalance_ratio=1.372375 max_load_reduction=108587 max_load_reduction_pct=0.828453 moved_to_echo=74022 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4060, 3, 0, 2956, 0, 0, 0, 0, 0, 3004, 4, 4032, 0, 4046, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 25, 0, 4030, 0, 11, 0, 4, 4096, 29, 4086, 4, 1, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 10, 0, 12, 0, 1, 0, 8, 14, 0, 4094, 4064, 4061, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 80, 4082, 0, 42, 0, 0, 0, 1, 0, 91, 0, 0, 4053, 0, 1676, 1128, 504, 398, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 65, 0, 2, 0, 2, 694, 0, 106, 23, 2, 0, 4095, 0, 0, 2758, 1896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 3905, 0, 5, 3592, 3963, 16, 0, 0, 0, 0, 17, 55, 53, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1764, 0, 0, 0, 1595, 0, 4, 39, 4096, 4090, 0, 4, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 0, 0, 753, 4080, 4080, 77, 0, 42, 13, 4, 3252, 0, 2416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20956, 16304, 13725, 16286, 13648, 15660, 15683, 18810] avg=16384.0000 max=20956 max_rank=0 min=13648 min_rank=4 imbalance_ratio=1.279053 max_load_reduction=110116 max_load_reduction_pct=0.840118 moved_to_echo=63801 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 2, 2966, 8, 0, 0, 0, 2, 1458, 0, 4096, 0, 3807, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 3938, 0, 0, 4085, 0, 0, 0, 0, 4019, 0, 4068, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 829, 4095, 0, 0, 24, 0, 0, 77, 2, 2, 4095, 4096, 4090, 1321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 1, 4093, 0, 0, 0, 0, 0, 12, 11, 244, 0, 0, 4095, 0, 1673, 1127, 479, 395, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4053, 0, 68, 1601, 2, 0, 2, 4059, 84, 0, 0, 0, 4085, 0, 0, 2775, 1910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3739, 0, 0, 3436, 370, 0, 0, 0, 0, 0, 0, 0, 720, 3082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 1791, 0, 0, 812, 4096, 0, 0, 4, 3963, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 2, 2, 3812, 4096, 0, 0, 0, 3, 2, 45, 0, 2420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16443, 16266, 18631, 16350, 18658, 11347, 18888, 14489] avg=16384.0000 max=18888 max_rank=6 min=11347 min_rank=5 imbalance_ratio=1.152832 max_load_reduction=112184 max_load_reduction_pct=0.855896 moved_to_echo=63382 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 30848, 128, 768, 23680, 128, 0, 4096, 128, 128, 13056, 128, 32640, 0, 19072, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 30464, 0, 128, 27776, 24064, 128, 128, 0, 0, 0, 128, 128, 24576, 24320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[640, 31872, 128, 256, 13952, 128, 128, 2560, 7424, 4736, 512, 128, 128, 32384, 128, 0, 21888, 15232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[384, 31360, 128, 0, 32128, 0, 128, 1792, 128, 31616, 128, 31104, 128, 128, 2304, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 256, 14336, 0, 0, 5120, 20608, 128, 128, 256, 25984, 32384, 128, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[1920, 32256, 128, 32256, 256, 128, 512, 384, 128, 128, 128, 640, 0, 128, 32640, 0, 13440, 9088, 3968, 3200, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 128, 256, 3072, 25600, 32768, 128, 0, 128, 128, 128, 17024, 384, 19456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 2048, 0, 0, 128, 4480, 14720, 0, 128, 1408, 128, 128, 256, 128, 256, 32768, 32640, 32512, 10496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[2290, 0, 4096, 56, 0, 4052, 0, 0, 0, 3196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4063, 4096, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 3983, 0, 0, 686, 0, 30, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3223, 4096, 0, 0, 0, 0, 4095, 0, 0, 127, 0, 0, 0, 0, 4096, 0, 0, 4096, 655, 510, 0, 12, 4011, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 3717, 0, 0, 4096, 4069, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 168, 0, 0, 4096, 0, 0, 3936, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 2215, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4030, 0, 4095, 0, 0, 4096, 0, 0, 0, 4036, 0, 0, 0, 0, 31, 0, 0, 0, 1, 11, 0, 0, 0, 4074, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 388, 4096, 0, 0, 4096, 0, 0, 0, 2406, 0, 0, 0, 0, 2589, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 156, 0, 0, 4096, 0, 5, 0, 3302, 4096, 0, 0, 0, 2, 4096, 0, 0, 1, 0, 1, 0, 1, 4071, 0, 0, 4096, 41, 4, 0, 0, 4061, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 3933, 0, 0, 4096, 4096, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 100, 0, 0, 4088, 0, 0, 3642, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[29, 0, 4096, 22, 0, 4090, 0, 0, 0, 376, 0, 0, 0, 0, 0, 0, 2, 0, 4077, 0, 0, 6, 0, 4089, 0, 0, 0, 0, 2705, 0, 0, 4095, 0, 4096, 4087, 398, 7, 4070, 0, 0, 156, 4096, 0, 0, 64, 0, 4096, 0, 0, 0, 7, 0, 0, 3, 4096, 0, 23, 792, 1, 4096, 3, 8, 0, 2991, 3936, 989, 0, 0, 0, 3617, 1, 37, 4, 110, 3, 0, 0, 3379, 4, 0, 3809, 33, 0, 0, 0, 4096, 3, 0, 4093, 96, 4083, 3490, 0, 15, 0, 720, 0, 0, 3993, 3932, 2, 5, 4028, 4025, 45, 0, 71, 0, 0, 0, 854, 12, 0, 4085, 0, 0, 4051, 4069, 30, 77, 0, 18, 0, 1, 292, 4096, 4095, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 1263, 12, 4095, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2, 4096, 0, 0, 4096, 1, 721, 0, 0, 0, 0, 5, 0, 4096, 54, 0, 1856, 0, 0, 439, 0, 4046, 8, 0, 0, 0, 4096, 0, 0, 0, 4092, 3796, 289, 193, 0, 0, 4094, 7, 0, 0, 1, 0, 0, 0, 3583, 0, 0, 4025, 794, 3634, 0, 0, 4096, 0, 0, 4010, 0, 4085, 4096, 0, 0, 0, 1197, 0, 0, 4093, 4086, 0, 0, 4096, 959, 0, 0, 1, 0, 2, 0, 4093, 0, 0, 2993, 0, 0, 2749, 4096, 0, 73, 0, 0, 0, 164, 0, 4030, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1902, 1, 3980, 0, 0, 4085, 40, 27, 0, 971, 0, 264, 0, 24, 0, 0, 0, 1, 0, 705, 0, 0, 129, 4096, 89, 34, 43, 0, 3785, 0, 24, 4095, 0, 4094, 4061, 0, 2, 4068, 1, 0, 0, 333, 53, 0, 0, 3, 2978, 0, 2, 3960, 0, 0, 7, 1, 4049, 8, 0, 0, 18, 4044, 1, 0, 3, 4041, 4045, 0, 0, 3, 0, 4069, 8, 0, 553, 0, 45, 0, 159, 4061, 14, 0, 4094, 164, 3470, 2, 0, 1385, 0, 0, 4076, 29, 4038, 4024, 2, 76, 0, 3805, 0, 0, 4096, 3971, 4, 0, 4067, 3898, 0, 0, 289, 0, 0, 0, 1789, 0, 3, 4069, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 16, 0, 4091, 2557, 4082] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 3979, 166, 0, 3616, 102, 0, 0, 304, 0, 0, 0, 95, 184, 6, 129, 10, 43, 1745, 0, 0, 0, 3786, 38, 0, 0, 46, 4064, 0, 456, 4073, 0, 4094, 4091, 0, 29, 4096, 0, 0, 3, 1, 0, 0, 0, 0, 3492, 0, 0, 0, 3, 0, 0, 0, 4070, 0, 17, 42, 0, 3880, 0, 12, 1, 3990, 4086, 9, 0, 13, 39, 4007, 0, 0, 543, 0, 0, 107, 2, 3888, 0, 0, 4096, 4088, 0, 222, 0, 0, 0, 4047, 4096, 22, 3847, 4085, 166, 0, 73, 0, 0, 0, 4058, 4042, 0, 0, 4096, 3950, 7, 0, 0, 0, 0, 24, 3836, 0, 2, 4056, 0, 97, 4011, 4086, 0, 0, 0, 418, 0, 183, 0, 3970, 2169, 3967] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 9, 3959, 0, 0, 0, 45, 3807, 0, 0, 22, 36, 0, 0, 0, 0, 3565, 0, 0, 0, 0, 4069, 0, 0, 4096, 33, 4041, 4053, 0, 0, 4094, 31, 0, 0, 4, 0, 0, 0, 0, 117, 220, 0, 4095, 4, 2, 0, 0, 4096, 0, 0, 1, 2, 4096, 10, 0, 0, 4042, 4040, 280, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 134, 4043, 0, 0, 4096, 0, 324, 0, 78, 0, 11, 57, 3934, 0, 4066, 3749, 0, 0, 0, 1, 0, 28, 4087, 4086, 1, 0, 3926, 4096, 0, 1, 0, 0, 132, 0, 233, 8, 0, 4096, 0, 0, 3953, 4020, 0, 187, 0, 174, 0, 35, 0, 4074, 4096, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[3790, 0, 3969, 2, 0, 4090, 15, 0, 0, 1559, 0, 168, 0, 24, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4045, 10, 0, 0, 0, 4089, 0, 0, 4095, 0, 4032, 4092, 69, 0, 4081, 45, 0, 4, 2070, 21, 0, 452, 0, 3725, 0, 0, 11, 2, 4, 53, 0, 4037, 0, 65, 30, 0, 4096, 0, 0, 0, 4094, 4088, 0, 0, 0, 0, 4072, 0, 0, 0, 2, 4, 5, 0, 3997, 0, 0, 4088, 9, 4087, 0, 0, 4096, 0, 68, 4096, 0, 4087, 4095, 0, 0, 0, 1365, 0, 0, 4096, 4093, 0, 0, 4096, 2019, 0, 0, 0, 0, 1, 1, 143, 0, 0, 4095, 0, 6, 1073, 4086, 0, 652, 0, 0, 1, 9, 43, 4092, 3468, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4030, 0, 4095, 0, 0, 4096, 0, 0, 0, 4036, 0, 0, 0, 0, 31, 0, 2533, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 0, 0, 4074, 0, 0, 0, 0, 4096, 0, 0, 4096, 3415, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 4096, 0, 0, 4096, 0, 0, 0, 2406, 0, 0, 0, 0, 2589, 0, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 156, 0, 0, 4096, 0, 5, 0, 3302, 2301, 593, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 1, 0, 1, 0, 1, 4071, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 970, 41, 4, 0, 0, 4061, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3421, 4096, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 3642, 4096, 0, 0, 0, 0, 0, 0, 0, 1795, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19502, 15779, 14142, 14657, 16278, 17201, 15796, 17717] avg=16384.0000 max=19502 max_rank=0 min=14142 min_rank=2 imbalance_ratio=1.190308 max_load_reduction=111570 max_load_reduction_pct=0.851212 moved_to_echo=66992 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2290, 0, 4096, 56, 0, 4052, 0, 0, 0, 3196, 0, 0, 0, 0, 0, 0, 2533, 682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 4096, 0, 0, 4096, 3414, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4096, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 3983, 0, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 686, 0, 30, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3223, 2301, 593, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 127, 0, 0, 0, 0, 4096, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 970, 655, 510, 0, 12, 4011, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 3717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3421, 4069, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3936, 4096, 0, 0, 0, 0, 0, 1, 0, 1795, 2215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16905, 15747, 16818, 15134, 16423, 18066, 15840, 16139] avg=16384.0000 max=18066 max_rank=5 min=15134 min_rank=3 imbalance_ratio=1.102661 max_load_reduction=113006 max_load_reduction_pct=0.862167 moved_to_echo=66468 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 1263, 12, 4095, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 2489, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4096, 0, 0, 4096, 3344, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 1, 721, 0, 0, 0, 0, 5, 0, 4096, 54, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1856, 0, 0, 439, 0, 4046, 8, 0, 0, 0, 4096, 0, 0, 0, 4092, 2264, 582, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 289, 193, 0, 0, 4094, 7, 0, 0, 1, 0, 0, 0, 3583, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 954, 794, 3634, 0, 0, 4096, 0, 0, 0, 0, 4085, 4096, 0, 0, 0, 1197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3418, 4086, 0, 0, 4096, 959, 0, 0, 1, 0, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2993, 0, 0, 2749, 4096, 0, 73, 0, 0, 0, 164, 0, 1766, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16720, 15802, 13638, 17491, 15973, 18856, 16655, 15937] avg=16384.0000 max=18856 max_rank=5 min=13638 min_rank=2 imbalance_ratio=1.150879 max_load_reduction=112216 max_load_reduction_pct=0.856140 moved_to_echo=67421 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 9, 3959, 0, 0, 0, 45, 3807, 0, 2533, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 36, 0, 0, 0, 0, 3565, 0, 0, 0, 0, 4069, 0, 0, 4096, 3280, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4041, 4053, 0, 0, 4094, 31, 0, 0, 4, 0, 0, 0, 0, 117, 220, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 2, 0, 0, 4096, 0, 0, 1, 2, 4096, 10, 0, 0, 4042, 2288, 593, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 280, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 134, 4043, 0, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 970, 0, 324, 0, 78, 0, 11, 57, 0, 0, 4066, 3749, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3413, 4086, 1, 0, 3926, 4096, 0, 1, 0, 0, 132, 0, 233, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3953, 4020, 0, 187, 0, 174, 0, 35, 0, 1786, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23295, 15154, 13159, 19337, 16600, 9256, 15924, 18347] avg=16384.0000 max=23295 max_rank=0 min=9256 min_rank=5 imbalance_ratio=1.421814 max_load_reduction=107777 max_load_reduction_pct=0.822273 moved_to_echo=60127 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 4096, 22, 0, 4090, 0, 0, 0, 376, 0, 0, 0, 0, 0, 0, 2355, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4077, 0, 0, 6, 0, 4089, 0, 0, 0, 0, 2705, 0, 0, 4095, 3412, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 398, 7, 4070, 0, 0, 156, 4096, 0, 0, 64, 0, 4096, 0, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 4096, 0, 23, 792, 1, 4096, 3, 8, 0, 2991, 2301, 551, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 989, 0, 0, 0, 3617, 1, 37, 4, 110, 3, 0, 0, 3379, 4, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 903, 33, 0, 0, 0, 4096, 3, 0, 0, 96, 4083, 3490, 0, 15, 0, 720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3335, 3932, 2, 5, 4028, 4025, 45, 0, 71, 0, 0, 0, 854, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4051, 4069, 30, 77, 0, 18, 0, 1, 292, 1795, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11649, 18473, 21623, 14977, 16089, 13439, 16309, 18513] avg=16384.0000 max=21623 max_rank=2 min=11649 min_rank=0 imbalance_ratio=1.319763 max_load_reduction=109449 max_load_reduction_pct=0.835030 moved_to_echo=64350 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3979, 166, 0, 3616, 102, 0, 0, 304, 0, 0, 0, 95, 184, 6, 2533, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 10, 43, 1745, 0, 0, 0, 3786, 38, 0, 0, 46, 4064, 0, 456, 4073, 3415, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4091, 0, 29, 4096, 0, 0, 3, 1, 0, 0, 0, 0, 3492, 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4070, 0, 17, 42, 0, 3880, 0, 12, 1, 3990, 2230, 593, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 9, 0, 13, 39, 4007, 0, 0, 543, 0, 0, 107, 2, 3888, 0, 0, 3884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 970, 4088, 0, 222, 0, 0, 0, 4047, 0, 22, 3847, 4085, 166, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3389, 4042, 0, 0, 4096, 3950, 7, 0, 0, 0, 0, 24, 3836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4056, 0, 97, 4011, 4086, 0, 0, 0, 418, 0, 183, 0, 1740, 2169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11667, 17888, 16368, 14945, 16578, 17520, 19344, 16762] avg=16384.0000 max=19344 max_rank=6 min=11667 min_rank=0 imbalance_ratio=1.180664 max_load_reduction=111728 max_load_reduction_pct=0.852417 moved_to_echo=70204 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1902, 1, 3980, 0, 0, 4085, 40, 27, 0, 971, 0, 264, 0, 24, 0, 0, 2536, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 705, 0, 0, 129, 4096, 89, 34, 43, 0, 3785, 0, 24, 4095, 3398, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4061, 0, 2, 4068, 1, 0, 0, 333, 53, 0, 0, 3, 2978, 0, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3960, 0, 0, 7, 1, 4049, 8, 0, 0, 18, 4044, 1, 0, 3, 4041, 2301, 595, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 3, 0, 4069, 8, 0, 553, 0, 45, 0, 159, 4061, 14, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 963, 164, 3470, 2, 0, 1385, 0, 0, 0, 29, 4038, 4024, 2, 76, 0, 3805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3412, 3971, 4, 0, 4067, 3898, 0, 0, 289, 0, 0, 0, 1789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4069, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 16, 0, 1790, 2557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14508, 16485, 16166, 19141, 16953, 17958, 17430, 12431] avg=16384.0000 max=19141 max_rank=3 min=12431 min_rank=7 imbalance_ratio=1.168274 max_load_reduction=111931 max_load_reduction_pct=0.853966 moved_to_echo=64772 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3790, 0, 3969, 2, 0, 4090, 15, 0, 0, 1559, 0, 168, 0, 24, 0, 0, 2528, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4045, 10, 0, 0, 0, 4089, 0, 0, 4095, 3415, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 4092, 69, 0, 4081, 45, 0, 4, 2070, 21, 0, 452, 0, 3725, 0, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2, 4, 53, 0, 4037, 0, 65, 30, 0, 4096, 0, 0, 0, 4094, 2299, 591, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4072, 0, 0, 0, 2, 4, 5, 0, 3997, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 969, 9, 4087, 0, 0, 4096, 0, 68, 0, 0, 4087, 4095, 0, 0, 0, 1365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3421, 4093, 0, 0, 4096, 2019, 0, 0, 0, 0, 1, 1, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 6, 1073, 4086, 0, 652, 0, 0, 1, 9, 43, 1793, 3468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16826, 15744, 19158, 15390, 16178, 18776, 13774, 15226] avg=16384.0000 max=19158 max_rank=2 min=13774 min_rank=6 imbalance_ratio=1.169312 max_load_reduction=111914 max_load_reduction_pct=0.853836 moved_to_echo=63954 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32256, 1664, 256, 128, 128, 32256, 128, 128, 1280, 128, 128, 128, 384, 31232, 128, 0, 32000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[256, 256, 4224, 2560, 0, 128, 256, 31872, 256, 128, 128, 128, 31104, 0, 512, 32768, 27136, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[20352, 128, 32512, 1536, 128, 32256, 256, 128, 128, 14464, 128, 512, 0, 256, 4096, 128, 20096, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 10624, 128, 128, 512, 128, 32640, 128, 384, 896, 128, 32512, 128, 128, 128, 29824, 18304, 4736, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 24832, 32768, 512, 128, 32768, 128, 768, 256, 8960, 128, 0, 640, 128, 25088, 384, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 31616, 0, 128, 23424, 32640, 128, 1024, 0, 640, 128, 512, 384, 14336, 26880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[7680, 5888, 12032, 256, 128, 21760, 128, 4224, 0, 256, 32512, 31744, 256, 128, 128, 14848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 27264, 32384, 128, 128, 32512, 27136, 128, 128, 384, 0, 256, 128, 11264, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[82, 2324, 1074, 89, 2495, 52, 655, 440, 826, 122, 1898, 435, 1090, 758, 89, 303, 161, 576, 1029, 976, 1500, 476, 2434, 1332, 158, 1547, 409, 872, 1333, 458, 1908, 1846, 2760, 159, 275, 686, 246, 880, 1619, 439, 2545, 1761, 1340, 881, 877, 1355, 636, 1274, 534, 110, 339, 1338, 950, 266, 1504, 74, 63, 932, 2320, 2330, 758, 159, 1151, 439, 914, 3018, 297, 937, 741, 318, 816, 1970, 305, 1792, 209, 1148, 570, 1911, 3118, 846, 1835, 1376, 1451, 1047, 1613, 520, 989, 2364, 777, 282, 668, 1317, 933, 390, 346, 284, 1850, 476, 1654, 1215, 692, 996, 2173, 2773, 1520, 1679, 686, 1580, 275, 275, 1040, 913, 190, 1503, 715, 1359, 215, 884, 1651, 1434, 1008, 957, 1847, 1043, 1640, 207, 530, 243] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[37, 2777, 1243, 66, 1768, 45, 732, 408, 1029, 151, 1593, 191, 766, 751, 33, 326, 140, 536, 1697, 779, 1766, 481, 2185, 1243, 68, 1994, 447, 660, 1704, 429, 1655, 2109, 2880, 169, 580, 1047, 128, 872, 1679, 656, 2766, 1263, 955, 576, 407, 1318, 1168, 596, 414, 179, 462, 1761, 738, 360, 1197, 67, 145, 612, 2085, 1750, 1112, 271, 1236, 455, 861, 3238, 190, 1460, 687, 408, 962, 2266, 580, 1473, 178, 1539, 600, 1524, 1760, 854, 2007, 1011, 1829, 1199, 1857, 737, 756, 1338, 1017, 193, 703, 1227, 884, 846, 443, 302, 1951, 293, 2080, 985, 422, 1243, 2223, 3396, 1136, 1438, 796, 1615, 455, 353, 1509, 863, 222, 1354, 1084, 813, 92, 955, 1694, 1169, 784, 1263, 1992, 1000, 1956, 328, 805, 161] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[52, 2647, 1717, 16, 2032, 78, 802, 486, 616, 119, 1096, 378, 822, 376, 82, 290, 153, 573, 1112, 1486, 1534, 185, 2389, 1173, 134, 1779, 427, 741, 1318, 233, 1217, 1826, 3514, 267, 239, 862, 242, 659, 1513, 777, 2738, 1466, 1215, 692, 534, 1790, 1173, 546, 395, 250, 370, 1354, 496, 267, 1688, 64, 106, 470, 1830, 2628, 852, 211, 839, 553, 729, 2972, 147, 1167, 659, 309, 1026, 2414, 866, 1556, 284, 1106, 600, 1386, 2346, 819, 2739, 883, 2345, 1491, 1932, 498, 1113, 2572, 1020, 172, 509, 1341, 820, 555, 431, 376, 1690, 351, 2284, 525, 883, 1180, 1832, 3139, 1249, 1443, 684, 1528, 504, 267, 1226, 2049, 259, 1311, 528, 1210, 215, 960, 1106, 1245, 879, 1434, 1678, 1112, 1531, 399, 582, 117] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[43, 1892, 936, 54, 2193, 60, 934, 203, 1011, 128, 1549, 715, 989, 1082, 18, 413, 134, 1194, 811, 1175, 1694, 253, 2421, 1439, 64, 1734, 303, 900, 1317, 319, 1672, 2422, 3445, 282, 195, 672, 93, 737, 944, 725, 2238, 1330, 1126, 360, 583, 1897, 1121, 657, 276, 237, 452, 992, 453, 240, 1794, 55, 296, 824, 2043, 2742, 706, 263, 588, 540, 949, 2907, 295, 1387, 751, 118, 1148, 2253, 508, 1769, 142, 573, 877, 2025, 2449, 1278, 2015, 914, 2174, 1226, 1475, 526, 665, 2469, 703, 498, 862, 1742, 1334, 756, 438, 297, 1832, 580, 2483, 717, 985, 1232, 1231, 2823, 1204, 1286, 731, 1510, 622, 449, 1374, 1453, 350, 1115, 688, 745, 138, 516, 799, 1276, 801, 1609, 1616, 1178, 1670, 470, 865, 223] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[26, 2852, 774, 118, 1270, 133, 671, 173, 525, 279, 1162, 537, 790, 696, 40, 522, 332, 636, 1319, 774, 2054, 103, 2288, 1160, 43, 2047, 392, 657, 1221, 286, 955, 2063, 3192, 627, 180, 848, 248, 1036, 1662, 668, 2382, 838, 1078, 519, 505, 1596, 1628, 791, 469, 164, 443, 1371, 682, 902, 1528, 69, 624, 534, 2408, 1480, 505, 487, 704, 548, 1269, 2410, 270, 1240, 760, 223, 927, 1850, 527, 2288, 769, 1333, 853, 1245, 3129, 1091, 1690, 1199, 2034, 1292, 1205, 591, 512, 1990, 619, 344, 1111, 1954, 1029, 550, 503, 643, 1693, 597, 2104, 304, 939, 1109, 1882, 2475, 1143, 2395, 1013, 2010, 541, 1194, 1666, 1259, 264, 1046, 971, 890, 203, 691, 813, 1074, 882, 1154, 1776, 977, 1571, 334, 909, 129] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[66, 2523, 984, 218, 2158, 113, 999, 455, 937, 102, 1826, 146, 819, 612, 123, 500, 141, 601, 1311, 1075, 1468, 431, 1864, 1143, 208, 1561, 715, 390, 1271, 297, 1554, 2014, 2569, 161, 381, 350, 143, 767, 1865, 521, 2790, 1457, 1469, 916, 487, 1554, 1130, 575, 413, 246, 848, 1737, 670, 414, 1568, 99, 84, 884, 2428, 1648, 665, 401, 432, 617, 1106, 3232, 286, 1592, 874, 540, 1093, 2133, 617, 1562, 388, 1081, 913, 1386, 1694, 902, 2176, 1188, 1895, 1009, 1315, 577, 1879, 1825, 1533, 161, 884, 1040, 1039, 435, 562, 544, 1523, 415, 1833, 922, 715, 651, 2591, 2454, 1534, 1512, 692, 1544, 647, 575, 1226, 1264, 168, 1077, 697, 1209, 159, 1141, 1319, 1289, 883, 1525, 1602, 1037, 1358, 555, 457, 123] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[156, 2944, 1324, 89, 1928, 35, 615, 267, 1121, 107, 1787, 178, 947, 823, 39, 425, 286, 1298, 754, 1067, 1486, 186, 2284, 995, 77, 1500, 447, 985, 1419, 145, 1175, 2219, 3222, 268, 262, 317, 100, 1040, 1940, 1178, 2293, 1387, 1306, 701, 602, 1737, 1363, 614, 384, 467, 359, 1327, 225, 455, 1655, 118, 166, 423, 2646, 2621, 1021, 292, 712, 462, 678, 2734, 215, 1096, 654, 225, 1123, 2307, 806, 1868, 359, 622, 582, 1780, 1964, 641, 2365, 1235, 2062, 1185, 1742, 616, 926, 2631, 1336, 272, 780, 1229, 737, 749, 330, 413, 2156, 289, 2687, 549, 829, 644, 1811, 2876, 987, 1623, 406, 1349, 461, 426, 1109, 1660, 339, 1003, 549, 1021, 196, 601, 1294, 1253, 631, 1426, 1959, 689, 2061, 269, 720, 166] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[94, 2122, 764, 68, 2225, 91, 658, 597, 793, 112, 1584, 355, 657, 526, 17, 389, 217, 954, 811, 1280, 1516, 242, 2681, 916, 60, 1424, 431, 475, 1087, 232, 1201, 1891, 3611, 200, 241, 244, 449, 731, 2077, 575, 2491, 1020, 1229, 612, 604, 1612, 1241, 740, 579, 219, 425, 1272, 412, 143, 2244, 198, 182, 519, 2098, 2263, 468, 459, 598, 284, 948, 3321, 195, 1250, 1237, 260, 1450, 2018, 569, 1669, 220, 759, 665, 1358, 2489, 690, 2322, 1425, 2102, 1483, 1764, 489, 893, 2590, 1136, 152, 801, 1209, 1134, 320, 572, 247, 1498, 159, 2602, 554, 813, 938, 2067, 2435, 1660, 1506, 901, 2001, 720, 539, 1223, 1754, 230, 1206, 1419, 976, 102, 737, 782, 1379, 905, 1849, 1764, 978, 1633, 727, 481, 287] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[82, 2324, 1074, 89, 2495, 52, 655, 440, 826, 122, 1898, 435, 1090, 758, 89, 303, 2773, 1676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 576, 1029, 976, 1500, 476, 2161, 1332, 158, 1547, 409, 872, 1333, 458, 1908, 1846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 159, 275, 686, 246, 880, 1619, 439, 2545, 1761, 1340, 881, 877, 1355, 636, 1274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 110, 339, 1338, 950, 266, 1504, 74, 63, 932, 2320, 2330, 758, 159, 1151, 439, 1130, 1180, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 914, 414, 297, 937, 741, 318, 816, 1970, 305, 1792, 209, 1148, 570, 1911, 3118, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1835, 1376, 1451, 1047, 1613, 520, 989, 911, 777, 282, 668, 1317, 933, 390, 346, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1850, 476, 524, 1215, 692, 996, 2173, 0, 1520, 1679, 686, 1580, 275, 275, 1040, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 1503, 715, 1359, 215, 884, 1651, 1434, 1008, 957, 1847, 1043, 1640, 207, 530, 243, 1018, 273, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17181, 16742, 16715, 16505, 16306, 14739, 15894, 16990] avg=16384.0000 max=17181 max_rank=0 min=14739 min_rank=5 imbalance_ratio=1.048645 max_load_reduction=113891 max_load_reduction_pct=0.868919 moved_to_echo=63929 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[37, 2777, 1243, 66, 1768, 45, 732, 408, 1029, 151, 1593, 191, 766, 751, 33, 326, 3396, 1798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 536, 1697, 779, 1766, 481, 1940, 1243, 68, 1994, 447, 660, 1704, 429, 1655, 2109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1818, 169, 580, 1047, 128, 872, 1679, 656, 2766, 1263, 955, 576, 407, 1318, 1168, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 414, 179, 462, 1761, 738, 360, 1197, 67, 145, 612, 2085, 1750, 1112, 271, 1236, 455, 1421, 668, 995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 861, 445, 190, 1460, 687, 408, 962, 2266, 580, 1473, 178, 1539, 600, 1524, 1760, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2007, 1011, 1829, 1199, 1857, 737, 756, 516, 1017, 193, 703, 1227, 884, 846, 443, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1951, 293, 659, 985, 422, 1243, 2223, 0, 1136, 1438, 796, 1615, 455, 353, 1509, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 1354, 1084, 813, 92, 955, 1694, 1169, 784, 1263, 1992, 1000, 1956, 328, 805, 161, 1062, 245, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17110, 17648, 15998, 15928, 15787, 15527, 15941, 17133] avg=16384.0000 max=17648 max_rank=1 min=15527 min_rank=5 imbalance_ratio=1.077148 max_load_reduction=113424 max_load_reduction_pct=0.865356 moved_to_echo=64388 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[52, 2647, 1717, 16, 2032, 78, 802, 486, 616, 119, 1096, 378, 822, 376, 82, 290, 3139, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 573, 1112, 1486, 1534, 185, 2121, 1173, 134, 1779, 427, 741, 1318, 233, 1217, 1826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2218, 267, 239, 862, 242, 659, 1513, 777, 2738, 1466, 1215, 692, 534, 1790, 1173, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 395, 250, 370, 1354, 496, 267, 1688, 64, 106, 470, 1830, 2628, 852, 211, 839, 553, 1560, 1284, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 729, 408, 147, 1167, 659, 309, 1026, 2414, 866, 1556, 284, 1106, 600, 1386, 2346, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2739, 883, 2345, 1491, 1932, 498, 1113, 991, 1020, 172, 509, 1341, 820, 555, 431, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1690, 351, 724, 525, 883, 1180, 1832, 0, 1249, 1443, 684, 1528, 504, 267, 1226, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 1311, 528, 1210, 215, 960, 1106, 1245, 879, 1434, 1678, 1112, 1531, 399, 582, 117, 1296, 268, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16398, 16012, 16931, 16131, 15822, 17216, 16135, 16427] avg=16384.0000 max=17216 max_rank=5 min=15822 min_rank=4 imbalance_ratio=1.050781 max_load_reduction=113856 max_load_reduction_pct=0.868652 moved_to_echo=65600 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[43, 1892, 936, 54, 2193, 60, 934, 203, 1011, 128, 1549, 715, 989, 1082, 18, 413, 2823, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 1194, 811, 1175, 1694, 253, 2150, 1439, 64, 1734, 303, 900, 1317, 319, 1672, 2422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2175, 282, 195, 672, 93, 737, 944, 725, 2238, 1330, 1126, 360, 583, 1897, 1121, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 237, 452, 992, 453, 240, 1794, 55, 296, 824, 2043, 2742, 706, 263, 588, 540, 1696, 1232, 894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 949, 399, 295, 1387, 751, 118, 1148, 2253, 508, 1769, 142, 573, 877, 2025, 2449, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2015, 914, 2174, 1226, 1475, 526, 665, 952, 703, 498, 862, 1742, 1334, 756, 438, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1832, 580, 787, 717, 985, 1232, 1231, 0, 1204, 1286, 731, 1510, 622, 449, 1374, 1453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 1115, 688, 745, 138, 516, 799, 1276, 801, 1609, 1616, 1178, 1670, 470, 865, 223, 1270, 271, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16657, 17581, 15135, 16323, 16921, 16577, 15993, 15885] avg=16384.0000 max=17581 max_rank=1 min=15135 min_rank=2 imbalance_ratio=1.073059 max_load_reduction=113491 max_load_reduction_pct=0.865868 moved_to_echo=65376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[156, 2944, 1324, 89, 1928, 35, 615, 267, 1121, 107, 1787, 178, 947, 823, 39, 425, 2876, 1518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 1298, 754, 1067, 1486, 186, 2028, 995, 77, 1500, 447, 985, 1419, 145, 1175, 2219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2034, 268, 262, 317, 100, 1040, 1940, 1178, 2293, 1387, 1306, 701, 602, 1737, 1363, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 467, 359, 1327, 225, 455, 1655, 118, 166, 423, 2646, 2621, 1021, 292, 712, 462, 1835, 1313, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 678, 376, 215, 1096, 654, 225, 1123, 2307, 806, 1868, 359, 622, 582, 1780, 1964, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2365, 1235, 2062, 1185, 1742, 616, 926, 1014, 1336, 272, 780, 1229, 737, 749, 330, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2156, 289, 852, 549, 829, 644, 1811, 0, 987, 1623, 406, 1349, 461, 426, 1109, 1660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 1003, 549, 1021, 196, 601, 1294, 1253, 631, 1426, 1959, 689, 2061, 269, 720, 166, 1188, 256, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17179, 16067, 17142, 17321, 15296, 16991, 15151, 15925] avg=16384.0000 max=17321 max_rank=3 min=15151 min_rank=6 imbalance_ratio=1.057190 max_load_reduction=113751 max_load_reduction_pct=0.867851 moved_to_echo=63363 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[66, 2523, 984, 218, 2158, 113, 999, 455, 937, 102, 1826, 146, 819, 612, 123, 500, 2454, 1795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 601, 1311, 1075, 1468, 431, 1655, 1143, 208, 1561, 715, 390, 1271, 297, 1554, 2014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1622, 161, 381, 350, 143, 767, 1865, 521, 2790, 1457, 1469, 916, 487, 1554, 1130, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 246, 848, 1737, 670, 414, 1568, 99, 84, 884, 2428, 1648, 665, 401, 432, 617, 1252, 911, 994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1106, 443, 286, 1592, 874, 540, 1093, 2133, 617, 1562, 388, 1081, 913, 1386, 1694, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2176, 1188, 1895, 1009, 1315, 577, 1879, 703, 1533, 161, 884, 1040, 1039, 435, 562, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1523, 415, 581, 922, 715, 651, 2591, 0, 1534, 1512, 692, 1544, 647, 575, 1226, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 1077, 697, 1209, 159, 1141, 1319, 1289, 883, 1525, 1602, 1037, 1358, 555, 457, 123, 947, 209, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16830, 15835, 16188, 16311, 16610, 16940, 16392, 15966] avg=16384.0000 max=16940 max_rank=5 min=15835 min_rank=1 imbalance_ratio=1.033936 max_load_reduction=114132 max_load_reduction_pct=0.870758 moved_to_echo=65908 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[26, 2852, 774, 118, 1270, 133, 671, 173, 525, 279, 1162, 537, 790, 696, 40, 522, 2475, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, 636, 1319, 774, 2054, 103, 2028, 1160, 43, 2047, 392, 657, 1221, 286, 955, 2063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2011, 627, 180, 848, 248, 1036, 1662, 668, 2382, 838, 1078, 519, 505, 1596, 1628, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 164, 443, 1371, 682, 902, 1528, 69, 624, 534, 2408, 1480, 505, 487, 704, 548, 1441, 996, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 323, 270, 1240, 760, 223, 927, 1850, 527, 2288, 769, 1333, 853, 1245, 3129, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1690, 1199, 2034, 1292, 1205, 591, 512, 759, 619, 344, 1111, 1954, 1029, 550, 503, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1693, 597, 663, 304, 939, 1109, 1882, 0, 1143, 2395, 1013, 2010, 541, 1194, 1666, 1259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 1046, 971, 890, 203, 691, 813, 1074, 882, 1154, 1776, 977, 1571, 334, 909, 129, 1181, 260, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14386, 16070, 16617, 16099, 18097, 16035, 18408, 15360] avg=16384.0000 max=18408 max_rank=6 min=14386 min_rank=0 imbalance_ratio=1.123535 max_load_reduction=112664 max_load_reduction_pct=0.859558 moved_to_echo=67900 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[94, 2122, 764, 68, 2225, 91, 658, 597, 793, 112, 1584, 355, 657, 526, 17, 389, 2435, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 954, 811, 1280, 1516, 242, 2380, 916, 60, 1424, 431, 475, 1087, 232, 1201, 1891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2280, 200, 241, 244, 449, 731, 2077, 575, 2491, 1020, 1229, 612, 604, 1612, 1241, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 579, 219, 425, 1272, 412, 143, 2244, 198, 182, 519, 2098, 2263, 468, 459, 598, 284, 1777, 1293, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 948, 456, 195, 1250, 1237, 260, 1450, 2018, 569, 1669, 220, 759, 665, 1358, 2489, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2322, 1425, 2102, 1483, 1764, 489, 893, 998, 1136, 152, 801, 1209, 1134, 320, 572, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1498, 159, 825, 554, 813, 938, 2067, 0, 1660, 1506, 901, 2001, 720, 539, 1223, 1754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 1206, 1419, 976, 102, 737, 782, 1379, 905, 1849, 1764, 978, 1633, 727, 481, 287, 1331, 301, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15331, 15117, 16346, 16454, 16233, 17047, 17158, 17386] avg=16384.0000 max=17386 max_rank=7 min=15117 min_rank=1 imbalance_ratio=1.061157 max_load_reduction=113686 max_load_reduction_pct=0.867355 moved_to_echo=67824 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3584, 1920, 3712, 11264, 4736, 3072, 13184, 768, 1792, 5248, 17920, 17536, 6144, 2560, 6272, 3968, 12160, 8960, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1664, 6400, 8960, 8704, 13056, 2432, 16512, 9472, 896, 13696, 3584, 5760, 10752, 2432, 11392, 16512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[14208, 3200, 5632, 5888, 6400, 8064, 15872, 0, 10496, 12928, 6016, 13184, 4352, 4096, 10496, 11264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[640, 20096, 8832, 768, 16128, 640, 6144, 3072, 6912, 1152, 12544, 2944, 6912, 5632, 512, 3200, 22400, 13312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[17152, 9344, 16000, 9984, 12928, 4608, 7808, 6912, 8192, 2176, 6400, 11136, 7936, 4608, 3712, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[16000, 2176, 2432, 5120, 1664, 6784, 13312, 5632, 20352, 10624, 9728, 5376, 4608, 12928, 9472, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2048, 9728, 6656, 8320, 1408, 6528, 9472, 10240, 6784, 11264, 14336, 8064, 13440, 3328, 5376, 1536, 9344, 2176, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[7552, 3328, 1920, 10240, 6400, 2432, 8576, 17280, 4864, 14080, 2560, 8192, 5760, 12672, 19072, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[2, 102, 4092, 0, 4013, 0, 361, 2462, 1, 13, 0, 117, 4060, 4034, 9, 91, 851, 15, 13, 19, 14, 4080, 4, 267, 23, 16, 0, 0, 26, 4016, 4045, 75, 1594, 0, 423, 0, 2, 7, 4057, 1, 232, 1, 0, 1, 1, 1, 0, 6, 3, 4080, 3600, 0, 7, 1, 1, 40, 4093, 3807, 10, 0, 3996, 2, 0, 53, 6, 218, 115, 3860, 70, 394, 302, 239, 1062, 3989, 15, 0, 0, 80, 4008, 0, 1600, 3492, 7, 65, 3770, 4081, 1261, 4095, 3990, 0, 9, 2, 0, 91, 3959, 21, 2, 4035, 4096, 1, 3291, 6, 0, 9, 0, 0, 27, 0, 401, 0, 1, 18, 3215, 339, 0, 4, 26, 87, 11, 3204, 46, 3245, 0, 1323, 5, 3942, 2, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3498, 0, 4058, 0, 2, 3411, 14, 42, 0, 71, 4052, 1136, 52, 87, 2, 0, 0, 0, 62, 4030, 0, 2, 1295, 0, 0, 0, 2, 4094, 1767, 1, 2154, 59, 92, 0, 11, 1474, 3870, 2, 501, 31, 4, 0, 0, 193, 0, 1, 0, 4057, 2074, 0, 0, 16, 1, 1, 4072, 4065, 0, 1, 1571, 1, 2, 0, 0, 828, 67, 3244, 185, 1298, 29, 6, 3803, 3741, 12, 5, 0, 0, 3972, 10, 2589, 1066, 124, 4081, 3930, 3900, 37, 4094, 4062, 0, 0, 0, 0, 1166, 194, 0, 0, 4074, 4095, 0, 4096, 183, 0, 1, 0, 35, 2, 96, 4086, 226, 1, 0, 3549, 0, 20, 0, 1, 25, 32, 3159, 1965, 75, 0, 3490, 698, 2693, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[3, 44, 3581, 0, 3513, 335, 13, 226, 258, 9, 0, 3243, 4023, 2442, 0, 91, 507, 8, 1, 8, 3, 4003, 325, 692, 237, 913, 0, 1, 0, 4012, 4047, 47, 107, 119, 821, 0, 41, 233, 3273, 8, 68, 242, 0, 128, 2, 1128, 338, 0, 826, 2541, 3814, 11, 721, 15, 93, 66, 3988, 4038, 29, 11, 3696, 429, 0, 11, 0, 764, 22, 2043, 2, 536, 308, 34, 1410, 4019, 291, 510, 483, 1, 3493, 20, 1, 3602, 543, 3540, 3715, 3533, 18, 4004, 4008, 0, 753, 5, 0, 13, 1739, 1705, 10, 4077, 3592, 3, 890, 0, 0, 0, 126, 0, 1, 0, 2218, 111, 1, 0, 4020, 1, 0, 7, 0, 3703, 0, 2644, 25, 79, 0, 2781, 31, 2902, 14, 3339] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 30, 4010, 34, 4077, 0, 0, 4000, 22, 2593, 0, 44, 4036, 3569, 0, 62, 29, 0, 86, 494, 35, 4080, 27, 0, 503, 0, 0, 82, 0, 3875, 2572, 3, 2164, 1, 970, 1, 0, 548, 3754, 0, 0, 3, 1, 1, 0, 0, 62, 3, 75, 3821, 4085, 0, 7, 53, 0, 0, 3399, 3965, 5, 0, 2639, 2, 0, 10, 0, 16, 1, 2687, 12, 0, 174, 6, 2763, 3928, 0, 84, 0, 0, 3986, 0, 4023, 1580, 843, 4039, 4012, 4096, 957, 4096, 2865, 3, 480, 6, 104, 97, 803, 12, 0, 3636, 4076, 68, 2351, 1, 237, 0, 15, 281, 2, 4, 245, 0, 1, 0, 2083, 1, 0, 0, 0, 2059, 667, 3372, 25, 970, 0, 3978, 2, 1383, 33, 4077] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3429, 7, 4090, 0, 0, 2032, 52, 401, 0, 9, 2655, 3174, 45, 3, 29, 0, 0, 2, 97, 4096, 0, 4, 192, 3, 0, 89, 29, 4058, 3627, 83, 2829, 3, 271, 0, 0, 1391, 4031, 8, 234, 47, 1, 0, 2, 0, 2, 63, 0, 3907, 3988, 6, 31, 7, 0, 0, 4070, 4093, 7, 0, 2539, 0, 180, 0, 3, 1145, 0, 3971, 381, 559, 1064, 0, 2304, 3801, 158, 0, 21, 0, 3995, 0, 930, 3387, 1, 3071, 3881, 4006, 429, 3698, 2034, 14, 13, 7, 0, 7, 642, 0, 0, 2229, 4073, 75, 3577, 173, 13, 0, 1, 166, 9, 0, 3945, 1, 0, 0, 3162, 1, 0, 37, 0, 2649, 470, 3484, 521, 144, 1, 2693, 231, 3891, 9, 4079] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3237, 0, 3436, 0, 1, 1211, 136, 248, 0, 12, 3487, 4096, 6, 23, 0, 13, 0, 0, 0, 4068, 0, 0, 81, 2, 839, 0, 3, 4096, 3146, 44, 4072, 114, 3857, 18, 0, 547, 4053, 0, 0, 5, 0, 0, 39, 31, 1078, 0, 0, 3763, 4002, 34, 50, 0, 0, 0, 3645, 4071, 60, 0, 3097, 8, 2, 0, 5, 620, 23, 4093, 271, 38, 33, 0, 1695, 4026, 0, 0, 0, 0, 3808, 2, 2287, 2039, 272, 3620, 3986, 4065, 569, 4062, 3296, 0, 0, 0, 0, 13, 734, 0, 0, 2537, 4031, 6, 3725, 81, 1, 0, 4, 0, 0, 11, 980, 0, 11, 0, 461, 0, 0, 0, 7, 1138, 536, 3333, 2323, 359, 0, 4085, 80, 3947, 0, 3128] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 4057, 0, 4093, 0, 20, 3669, 141, 11, 1, 1066, 4089, 4033, 4, 19, 9, 0, 0, 2, 1070, 4070, 1, 645, 1633, 12, 28, 0, 0, 4093, 2532, 41, 484, 5, 851, 10, 128, 3, 2784, 8, 257, 342, 88, 24, 4, 0, 47, 85, 2, 4063, 3996, 0, 0, 0, 69, 0, 4082, 4087, 6, 0, 3855, 0, 7, 13, 0, 451, 29, 3972, 460, 7, 267, 23, 3237, 4095, 178, 322, 30, 1, 4025, 5, 664, 1650, 2, 1707, 4074, 4096, 1294, 4095, 4080, 0, 0, 0, 21, 6, 3417, 4, 165, 4095, 4090, 50, 2111, 1, 321, 10, 0, 1, 19, 0, 356, 0, 0, 0, 1362, 9, 0, 0, 0, 1717, 493, 3729, 578, 51, 0, 3032, 0, 1949, 4, 4077] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8, 147, 923, 1, 3998, 1, 48, 33, 3, 0, 1, 771, 3942, 3993, 18, 5, 0, 4, 25, 1, 0, 4077, 8, 9, 95, 0, 19, 272, 0, 4095, 3524, 0, 4087, 48, 1404, 4, 154, 0, 4011, 0, 34, 106, 138, 0, 0, 63, 17, 6, 0, 3050, 3906, 24, 0, 1, 10, 10, 4017, 4051, 5, 0, 4078, 55, 5, 1, 174, 3618, 1, 3603, 28, 3, 110, 0, 435, 4030, 3, 0, 0, 0, 3979, 3, 2766, 1766, 8, 3862, 4038, 4068, 230, 4091, 4058, 0, 0, 439, 0, 3213, 3167, 23, 1, 4092, 4033, 11, 3206, 0, 1, 7, 35, 4, 0, 68, 938, 71, 8, 9, 1476, 0, 15, 0, 0, 1421, 456, 3472, 1, 260, 0, 3021, 31, 1539, 0, 3874] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3498, 0, 2521, 0, 2, 3411, 14, 42, 0, 71, 4052, 1136, 52, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 62, 4030, 0, 2, 1295, 0, 0, 0, 2, 4094, 1767, 1, 1955, 1039, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2154, 59, 92, 0, 11, 1474, 3870, 2, 501, 31, 4, 0, 0, 193, 0, 1, 4094, 3900, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 2074, 0, 0, 16, 1, 1, 4072, 1567, 0, 1, 1571, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 67, 3244, 185, 1298, 29, 6, 3803, 3741, 12, 5, 0, 0, 3972, 10, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2589, 1066, 124, 4081, 1975, 0, 37, 0, 4062, 0, 0, 0, 0, 1166, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4095, 0, 4096, 183, 0, 1, 0, 35, 2, 96, 4086, 226, 1, 0, 2587, 1459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3549, 0, 20, 0, 1, 25, 32, 3159, 1965, 75, 0, 3490, 698, 2693, 0, 1460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14886, 14902, 16435, 13363, 18084, 15294, 20941, 17167] avg=16384.0000 max=20941 max_rank=6 min=13363 min_rank=3 imbalance_ratio=1.278137 max_load_reduction=110131 max_load_reduction_pct=0.840233 moved_to_echo=71486 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 102, 4092, 0, 2493, 0, 361, 2462, 1, 13, 0, 117, 4060, 4034, 9, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 851, 15, 13, 19, 14, 4080, 4, 267, 23, 16, 0, 0, 26, 4016, 4045, 75, 1876, 973, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1594, 0, 423, 0, 2, 7, 4057, 1, 232, 1, 0, 1, 1, 1, 0, 6, 4095, 4081, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4080, 3600, 0, 7, 1, 1, 40, 4093, 1467, 10, 0, 3996, 2, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 218, 115, 3860, 70, 394, 302, 239, 1062, 3989, 15, 0, 0, 80, 4008, 0, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1600, 3492, 7, 65, 1894, 0, 1261, 0, 3990, 0, 9, 2, 0, 91, 3959, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4035, 4096, 1, 3291, 6, 0, 9, 0, 0, 27, 0, 401, 0, 1, 18, 2587, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3215, 339, 0, 4, 26, 87, 11, 3204, 46, 3245, 0, 1323, 5, 3942, 2, 1459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17837, 16959, 14551, 17353, 15232, 16391, 15841, 16908] avg=16384.0000 max=17837 max_rank=0 min=14551 min_rank=2 imbalance_ratio=1.088684 max_load_reduction=113235 max_load_reduction_pct=0.863914 moved_to_echo=64372 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3237, 0, 2135, 0, 1, 1211, 136, 248, 0, 12, 3487, 4096, 6, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4068, 0, 0, 81, 2, 839, 0, 3, 4096, 3146, 44, 1983, 1041, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 114, 3857, 18, 0, 547, 4053, 0, 0, 5, 0, 0, 39, 31, 1078, 0, 4062, 4065, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3763, 4002, 34, 50, 0, 0, 0, 3645, 1569, 60, 0, 3097, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 620, 23, 4093, 271, 38, 33, 0, 1695, 4026, 0, 0, 0, 0, 3808, 2, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2287, 2039, 272, 3620, 2003, 0, 569, 0, 3296, 0, 0, 0, 0, 13, 734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2537, 4031, 6, 3725, 81, 1, 0, 4, 0, 0, 11, 980, 0, 11, 0, 1976, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 0, 0, 0, 7, 1138, 536, 3333, 2323, 359, 0, 4085, 80, 3947, 0, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14592, 15869, 21978, 16230, 15362, 14833, 14824, 17384] avg=16384.0000 max=21978 max_rank=2 min=14592 min_rank=0 imbalance_ratio=1.341431 max_load_reduction=109094 max_load_reduction_pct=0.832321 moved_to_echo=62403 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3429, 7, 2541, 0, 0, 2032, 52, 401, 0, 9, 2655, 3174, 45, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 2, 97, 4096, 0, 4, 192, 3, 0, 89, 29, 4058, 3627, 83, 1931, 1046, 658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2829, 3, 271, 0, 0, 1391, 4031, 8, 234, 47, 1, 0, 2, 0, 2, 63, 3698, 4006, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 3988, 6, 31, 7, 0, 0, 4070, 1578, 7, 0, 2539, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1145, 0, 3971, 381, 559, 1064, 0, 2304, 3801, 158, 0, 21, 0, 3995, 0, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 930, 3387, 1, 3071, 1950, 0, 429, 0, 2034, 14, 13, 7, 0, 7, 642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2229, 4073, 75, 3577, 173, 13, 0, 1, 166, 9, 0, 3945, 1, 0, 0, 2577, 1469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3162, 1, 0, 37, 0, 2649, 470, 3484, 521, 144, 1, 2693, 231, 3891, 9, 1453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14348, 15944, 16635, 16313, 18293, 12485, 18308, 18746] avg=16384.0000 max=18746 max_rank=7 min=12485 min_rank=5 imbalance_ratio=1.144165 max_load_reduction=112326 max_load_reduction_pct=0.856979 moved_to_echo=67832 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 44, 3581, 0, 2176, 335, 13, 226, 258, 9, 0, 3243, 4023, 2442, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 8, 1, 8, 3, 4003, 325, 692, 237, 913, 0, 1, 0, 4012, 4047, 47, 1854, 1036, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 119, 821, 0, 41, 233, 3273, 8, 68, 242, 0, 128, 2, 1128, 338, 0, 4004, 3533, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 2541, 3814, 11, 721, 15, 93, 66, 3988, 1547, 29, 11, 3696, 429, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 764, 22, 2043, 2, 536, 308, 34, 1410, 4019, 291, 510, 483, 1, 3493, 20, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3602, 543, 3540, 1861, 0, 18, 0, 4008, 0, 753, 5, 0, 13, 1739, 1705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4077, 3592, 3, 890, 0, 0, 0, 126, 0, 1, 0, 2218, 111, 1, 0, 2113, 1455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 1, 0, 7, 0, 3703, 0, 2644, 25, 79, 0, 2781, 31, 2902, 14, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16444, 18262, 14087, 17798, 14705, 17788, 14597, 17391] avg=16384.0000 max=18262 max_rank=1 min=14087 min_rank=2 imbalance_ratio=1.114624 max_load_reduction=112810 max_load_reduction_pct=0.860672 moved_to_echo=64481 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 30, 4010, 34, 2533, 0, 0, 4000, 22, 2593, 0, 44, 4036, 3569, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 86, 494, 35, 4080, 27, 0, 503, 0, 0, 82, 0, 3875, 2572, 3, 1996, 1014, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2164, 1, 970, 1, 0, 548, 3754, 0, 0, 3, 1, 1, 0, 0, 62, 3, 4096, 4096, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 3821, 4085, 0, 7, 53, 0, 0, 3399, 1528, 5, 0, 2639, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 2687, 12, 0, 174, 6, 2763, 3928, 0, 84, 0, 0, 3986, 0, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 1580, 843, 4039, 2016, 0, 957, 0, 2865, 3, 480, 6, 104, 97, 803, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3636, 4076, 68, 2351, 1, 237, 0, 15, 281, 2, 4, 245, 0, 1, 0, 2575, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2083, 1, 0, 0, 0, 2059, 667, 3372, 25, 970, 0, 3978, 2, 1383, 33, 1453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20933, 15452, 15749, 15624, 14545, 17828, 14915, 16026] avg=16384.0000 max=20933 max_rank=0 min=14545 min_rank=4 imbalance_ratio=1.277649 max_load_reduction=110139 max_load_reduction_pct=0.840294 moved_to_echo=63314 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4057, 0, 2543, 0, 20, 3669, 141, 11, 1, 1066, 4089, 4033, 4, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2, 1070, 4070, 1, 645, 1633, 12, 28, 0, 0, 4093, 2532, 41, 2027, 1045, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 5, 851, 10, 128, 3, 2784, 8, 257, 342, 88, 24, 4, 0, 47, 85, 4095, 4096, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4063, 3996, 0, 0, 0, 69, 0, 4082, 1575, 6, 0, 3855, 0, 7, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 29, 3972, 460, 7, 267, 23, 3237, 4095, 178, 322, 30, 1, 4025, 5, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 664, 1650, 2, 1707, 2047, 0, 1294, 0, 4080, 0, 0, 0, 21, 6, 3417, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 4095, 4090, 50, 2111, 1, 321, 10, 0, 1, 19, 0, 356, 0, 0, 0, 2575, 1467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1362, 9, 0, 0, 0, 1717, 493, 3729, 578, 51, 0, 3032, 0, 1949, 4, 1453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19654, 17867, 13360, 17668, 17993, 14892, 15261, 14377] avg=16384.0000 max=19654 max_rank=0 min=13360 min_rank=2 imbalance_ratio=1.199585 max_load_reduction=111418 max_load_reduction_pct=0.850052 moved_to_echo=62523 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8, 147, 923, 1, 2484, 1, 48, 33, 3, 0, 1, 771, 3942, 3993, 18, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 25, 1, 0, 4077, 8, 9, 95, 0, 19, 272, 0, 4095, 3524, 0, 2009, 1036, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 48, 1404, 4, 154, 0, 4011, 0, 34, 106, 138, 0, 0, 63, 17, 6, 4091, 4068, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3050, 3906, 24, 0, 1, 10, 10, 4017, 1561, 5, 0, 4078, 55, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 3618, 1, 3603, 28, 3, 110, 0, 435, 4030, 3, 0, 0, 0, 3979, 3, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2766, 1766, 8, 3862, 2029, 0, 230, 0, 4058, 0, 0, 439, 0, 3213, 3167, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 4033, 11, 3206, 0, 1, 7, 35, 4, 0, 68, 938, 71, 8, 9, 2447, 1454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1476, 0, 15, 0, 0, 1421, 456, 3472, 1, 260, 0, 3021, 31, 1539, 0, 1381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12378, 15817, 18277, 16723, 16858, 21561, 16385, 13073] avg=16384.0000 max=21561 max_rank=5 min=12378 min_rank=0 imbalance_ratio=1.315979 max_load_reduction=109511 max_load_reduction_pct=0.835503 moved_to_echo=67877 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[19328, 384, 128, 128, 128, 12800, 2688, 26496, 5504, 5248, 128, 24448, 1152, 22272, 128, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 384, 26880, 128, 19456, 384, 512, 17152, 640, 3328, 128, 5376, 30464, 26496, 256, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[17536, 384, 8704, 128, 384, 4224, 29952, 128, 1408, 896, 256, 256, 128, 1536, 1664, 256, 32256, 31872, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[14976, 18688, 1920, 24064, 15872, 0, 4864, 0, 28416, 128, 1280, 512, 128, 4608, 14720, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 28800, 32128, 256, 23296, 512, 640, 128, 256, 512, 128, 256, 13184, 512, 128, 128, 19456, 11648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1024, 29312, 29568, 128, 896, 128, 256, 128, 31488, 12416, 128, 128, 25472, 512, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1536, 128, 128, 640, 1408, 32512, 384, 1664, 4096, 1024, 896, 512, 128, 32384, 25344, 384, 15744, 8320, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[256, 7680, 384, 27520, 1536, 2944, 2304, 384, 16768, 31744, 768, 1024, 640, 128, 31360, 128, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[2454, 541, 0, 3, 4094, 10, 1573, 535, 111, 801, 0, 2660, 0, 34, 0, 2050, 1721, 2447, 45, 0, 3989, 1043, 290, 86, 0, 4035, 56, 327, 0, 3914, 4093, 54, 0, 3485, 231, 0, 3, 310, 3751, 4085, 0, 0, 117, 177, 0, 0, 5, 0, 0, 4064, 0, 360, 3583, 2, 0, 0, 409, 0, 675, 767, 3949, 0, 1836, 0, 1178, 1730, 40, 57, 5, 371, 0, 3668, 4093, 113, 0, 13, 2008, 3614, 346, 4030, 369, 4032, 0, 1623, 1410, 4034, 69, 27, 0, 1, 728, 0, 1, 1178, 3540, 998, 99, 2940, 3987, 0, 0, 17, 1529, 33, 0, 0, 4060, 0, 0, 4015, 307, 0, 2548, 6, 3467, 587, 0, 179, 2, 0, 3154, 0, 0, 64, 1, 1, 0, 25] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4024, 0, 0, 0, 110, 0, 3, 0, 0, 3994, 0, 2947, 2785, 0, 0, 4096, 1771, 4065, 0, 0, 3605, 310, 124, 0, 0, 4096, 1, 0, 0, 4062, 61, 3931, 279, 4096, 0, 4050, 0, 0, 4095, 1970, 0, 0, 25, 9, 0, 0, 0, 0, 0, 4092, 0, 14, 4077, 0, 0, 417, 0, 4, 0, 1244, 4076, 0, 2429, 0, 0, 4077, 1, 0, 0, 0, 0, 2, 4096, 177, 70, 0, 0, 3827, 0, 3820, 0, 4091, 12, 785, 0, 4087, 0, 2137, 161, 2, 22, 0, 55, 0, 4095, 213, 0, 336, 3781, 27, 0, 1, 3712, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 72, 0, 4046, 4076, 0, 0, 0, 0, 4047, 171, 0, 0, 0, 0, 23, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4094, 36, 86, 0, 553, 0, 282, 0, 0, 3996, 2, 4095, 38, 0, 8, 4090, 1775, 1567, 0, 0, 4091, 52, 2807, 1, 2, 2245, 0, 0, 0, 4008, 3989, 3189, 143, 4083, 0, 4, 8, 1, 4088, 4089, 0, 0, 0, 0, 0, 0, 33, 0, 0, 3341, 0, 0, 4096, 54, 0, 1, 0, 103, 1, 1690, 4056, 1, 0, 10, 7, 3946, 90, 0, 17, 0, 0, 3621, 3937, 1464, 0, 0, 7, 4095, 1, 3969, 1, 62, 0, 4088, 8, 4046, 0, 1874, 0, 0, 0, 0, 8, 1, 4079, 2507, 0, 1282, 4075, 3, 0, 0, 0, 687, 5, 0, 4096, 0, 0, 4091, 1, 0, 4, 0, 4096, 3968, 0, 0, 0, 0, 4092, 0, 0, 0, 36, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4095, 0, 0, 15, 3853, 1, 12, 24, 1, 2706, 0, 3937, 1, 54, 2, 3843, 3082, 4059, 10, 0, 3623, 147, 1395, 0, 0, 3949, 0, 0, 6, 3998, 3891, 992, 2457, 4096, 891, 1060, 5, 10, 4065, 4031, 7, 0, 0, 0, 0, 0, 1, 0, 0, 1909, 5, 0, 3814, 0, 0, 560, 11, 19, 1, 2907, 4096, 7, 0, 3, 84, 1547, 3, 13, 15, 1, 9, 4061, 3670, 1423, 0, 0, 0, 4078, 0, 1539, 1, 4042, 0, 2211, 0, 4086, 410, 14, 0, 0, 0, 3, 128, 0, 4087, 402, 20, 863, 4091, 0, 6, 0, 0, 889, 0, 0, 4096, 0, 0, 4042, 0, 2, 33, 0, 3375, 4087, 0, 23, 0, 2, 4058, 0, 0, 0, 2, 5, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3996, 0, 2, 0, 3489, 12, 0, 53, 5, 3677, 0, 3248, 2553, 1, 1, 4078, 1285, 2576, 32, 1, 4093, 201, 4004, 0, 0, 4080, 0, 0, 0, 3899, 533, 3819, 75, 4094, 11, 3980, 0, 0, 4088, 3465, 16, 5, 0, 0, 14, 1, 2, 8, 0, 4096, 1, 1, 4092, 11, 0, 155, 23, 0, 3, 887, 4085, 0, 416, 34, 0, 4039, 38, 1, 0, 0, 7, 2860, 4065, 54, 139, 0, 0, 3912, 4, 3753, 2, 1958, 7, 1405, 0, 4051, 0, 1108, 0, 15, 0, 0, 508, 0, 4095, 0, 19, 862, 4077, 0, 0, 0, 214, 92, 3, 0, 4093, 147, 0, 4096, 0, 2, 44, 0, 4054, 2647, 7, 0, 16, 5, 3326, 0, 28, 0, 1, 25, 91, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4066, 0, 0, 0, 3930, 0, 0, 33, 0, 4085, 0, 4058, 3404, 0, 0, 3922, 106, 1998, 20, 17, 4067, 21, 3396, 0, 0, 821, 327, 0, 0, 3280, 0, 3992, 8, 4095, 0, 3831, 0, 0, 4093, 4079, 2, 111, 0, 5, 42, 0, 2, 1, 0, 430, 0, 1, 4095, 12, 0, 163, 0, 0, 12, 3418, 4092, 0, 104, 0, 0, 4083, 3, 0, 3, 2, 0, 2456, 4081, 982, 1, 600, 0, 4016, 43, 62, 3, 4006, 1, 2028, 460, 4096, 702, 2667, 0, 1, 0, 0, 3, 2, 4079, 120, 0, 375, 4096, 5, 0, 139, 1, 2878, 0, 0, 4090, 268, 6, 3977, 0, 0, 0, 0, 4094, 4031, 60, 22, 360, 0, 2024, 0, 0, 5, 1, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[2667, 31, 18, 0, 3712, 0, 0, 36, 0, 3628, 0, 3758, 2352, 5, 0, 3879, 3262, 371, 2, 34, 4061, 0, 695, 0, 0, 2723, 0, 0, 0, 4085, 10, 4048, 0, 4096, 0, 3681, 44, 6, 4070, 3118, 0, 26, 216, 38, 163, 0, 0, 40, 25, 4095, 0, 0, 4042, 0, 0, 0, 478, 41, 0, 185, 4093, 92, 1202, 0, 0, 4077, 4, 0, 1, 0, 0, 2650, 4096, 2, 8, 8, 0, 4009, 0, 3831, 0, 2435, 1, 0, 819, 4065, 0, 3955, 389, 13, 0, 0, 0, 0, 4067, 0, 0, 602, 3978, 114, 0, 0, 444, 1980, 2, 0, 4024, 0, 48, 4091, 7, 0, 0, 0, 4046, 4096, 13, 0, 0, 0, 3871, 5, 122, 69, 0, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4028, 0, 0, 0, 2111, 52, 87, 140, 0, 4089, 0, 4016, 3302, 0, 0, 4084, 3163, 4048, 6, 18, 0, 304, 2814, 6, 7, 4096, 3, 0, 22, 3969, 3768, 525, 61, 4070, 31, 2527, 2, 118, 4086, 4095, 0, 38, 56, 0, 15, 0, 3, 1, 4, 876, 0, 0, 4069, 17, 1, 18, 0, 21, 128, 1652, 3957, 2, 2488, 0, 92, 172, 0, 0, 22, 18, 0, 3309, 4095, 3615, 0, 107, 0, 4024, 0, 3508, 44, 1532, 0, 3248, 0, 4070, 0, 581, 0, 0, 1, 0, 117, 92, 4040, 154, 0, 168, 4048, 0, 9, 0, 7, 503, 11, 27, 4065, 0, 0, 4086, 25, 23, 316, 0, 4021, 3885, 0, 0, 0, 0, 4018, 4, 10, 0, 0, 11, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4024, 0, 0, 0, 110, 0, 3, 0, 0, 3994, 0, 2947, 2785, 0, 0, 799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1771, 4065, 0, 0, 1599, 310, 124, 0, 0, 4096, 1, 0, 0, 0, 61, 3931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 4096, 0, 4050, 0, 0, 4095, 1970, 0, 0, 25, 9, 0, 0, 0, 0, 1616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 14, 4077, 0, 0, 417, 0, 4, 0, 1244, 4076, 0, 2429, 0, 2169, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 1, 0, 0, 0, 0, 2, 1968, 177, 70, 0, 0, 3827, 0, 3820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 12, 785, 0, 4087, 0, 2137, 161, 2, 22, 0, 55, 0, 4095, 213, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 3781, 27, 0, 1, 3712, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 1430, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 4046, 4076, 0, 0, 0, 0, 4047, 171, 0, 0, 0, 0, 23, 0, 4062, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14662, 15958, 16140, 18886, 13942, 16236, 17623, 17625] avg=16384.0000 max=18886 max_rank=3 min=13942 min_rank=4 imbalance_ratio=1.152710 max_load_reduction=112186 max_load_reduction_pct=0.855911 moved_to_echo=65426 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3996, 0, 2, 0, 3489, 12, 0, 53, 5, 3677, 0, 3248, 2553, 1, 1, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1285, 2576, 32, 1, 1816, 201, 4004, 0, 0, 4080, 0, 0, 0, 0, 533, 3819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 4094, 11, 3980, 0, 0, 4088, 3465, 16, 5, 0, 0, 14, 1, 2, 8, 1604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 1, 4092, 11, 0, 155, 23, 0, 3, 887, 4085, 0, 416, 34, 2159, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 38, 1, 0, 0, 7, 2860, 1952, 54, 139, 0, 0, 3912, 4, 3753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1958, 7, 1405, 0, 4051, 0, 1108, 0, 15, 0, 0, 508, 0, 4095, 0, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 862, 4077, 0, 0, 0, 214, 92, 3, 0, 4093, 147, 0, 4096, 0, 2, 1623, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 4054, 2647, 7, 0, 16, 5, 3326, 0, 28, 0, 1, 25, 91, 1, 3899, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17833, 18347, 17363, 16325, 16759, 13803, 15375, 15267] avg=16384.0000 max=18347 max_rank=1 min=13803 min_rank=5 imbalance_ratio=1.119812 max_load_reduction=112725 max_load_reduction_pct=0.860023 moved_to_echo=61204 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2667, 31, 18, 0, 3712, 0, 0, 36, 0, 3628, 0, 3758, 2352, 5, 0, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3262, 371, 2, 34, 1801, 0, 695, 0, 0, 2723, 0, 0, 0, 0, 10, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3681, 44, 6, 4070, 3118, 0, 26, 216, 38, 163, 0, 0, 40, 1616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4095, 0, 0, 4042, 0, 0, 0, 478, 41, 0, 185, 4093, 92, 1202, 0, 2054, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4, 0, 1, 0, 0, 2650, 1968, 2, 8, 8, 0, 4009, 0, 3831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2435, 1, 0, 819, 4065, 0, 3955, 389, 13, 0, 0, 0, 0, 4067, 0, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 3978, 114, 0, 0, 444, 1980, 2, 0, 4024, 0, 48, 4091, 7, 0, 1611, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 4096, 13, 0, 0, 0, 3871, 5, 122, 69, 0, 0, 2, 0, 4085, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16964, 12946, 17114, 16671, 16558, 16393, 17049, 17377] avg=16384.0000 max=17377 max_rank=7 min=12946 min_rank=1 imbalance_ratio=1.060608 max_load_reduction=113695 max_load_reduction_pct=0.867424 moved_to_echo=67377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2454, 541, 0, 3, 4094, 10, 1573, 535, 111, 801, 0, 2660, 0, 34, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1721, 2447, 45, 0, 1762, 1043, 290, 86, 0, 4035, 56, 327, 0, 0, 4093, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3485, 231, 0, 3, 310, 3751, 4085, 0, 0, 117, 177, 0, 0, 5, 0, 1620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 360, 3583, 2, 0, 0, 409, 0, 675, 767, 3949, 0, 1836, 0, 1088, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1178, 1730, 40, 57, 5, 371, 0, 3668, 1951, 113, 0, 13, 2008, 3614, 346, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 4032, 0, 1623, 1410, 4034, 69, 27, 0, 1, 728, 0, 1, 1178, 3540, 998, 642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 2940, 3987, 0, 0, 17, 1529, 33, 0, 0, 4060, 0, 0, 4015, 307, 0, 1585, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2548, 6, 3467, 587, 0, 179, 2, 0, 3154, 0, 0, 64, 1, 1, 0, 25, 3914, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13210, 15959, 13784, 17102, 19124, 18652, 18725, 14516] avg=16384.0000 max=19124 max_rank=4 min=13210 min_rank=0 imbalance_ratio=1.167236 max_load_reduction=111948 max_load_reduction_pct=0.854095 moved_to_echo=71017 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 15, 3853, 1, 12, 24, 1, 2706, 0, 3937, 1, 54, 2, 750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3082, 4059, 10, 0, 1607, 147, 1395, 0, 0, 3949, 0, 0, 6, 0, 3891, 992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2457, 4096, 891, 1060, 5, 10, 4065, 4031, 7, 0, 0, 0, 0, 0, 1, 0, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1909, 5, 0, 3814, 0, 0, 560, 11, 19, 1, 2907, 4096, 7, 0, 3, 2035, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 1547, 3, 13, 15, 1, 9, 4061, 1763, 1423, 0, 0, 0, 4078, 0, 1539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4042, 0, 2211, 0, 4086, 410, 14, 0, 0, 0, 3, 128, 0, 4087, 402, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 863, 4091, 0, 6, 0, 0, 889, 0, 0, 4096, 0, 0, 4042, 0, 2, 1437, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 3375, 4087, 0, 23, 0, 2, 4058, 0, 0, 0, 2, 5, 0, 0, 3998, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15451, 19138, 18071, 15693, 14536, 15963, 15579, 16641] avg=16384.0000 max=19138 max_rank=1 min=14536 min_rank=4 imbalance_ratio=1.168091 max_load_reduction=111934 max_load_reduction_pct=0.853989 moved_to_echo=62719 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4094, 36, 86, 0, 553, 0, 282, 0, 0, 3996, 2, 4095, 38, 0, 8, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 1567, 0, 0, 1815, 52, 2807, 1, 2, 2245, 0, 0, 0, 0, 3989, 3189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 4083, 0, 4, 8, 1, 4088, 4089, 0, 0, 0, 0, 0, 0, 33, 0, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3341, 0, 0, 4096, 54, 0, 1, 0, 103, 1, 1690, 4056, 1, 0, 10, 2166, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3946, 90, 0, 17, 0, 0, 3621, 1891, 1464, 0, 0, 7, 4095, 1, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 62, 0, 4088, 8, 4046, 0, 1874, 0, 0, 0, 0, 8, 1, 4079, 2507, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1282, 4075, 3, 0, 0, 0, 687, 5, 0, 4096, 0, 0, 4091, 1, 0, 1622, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 3968, 0, 0, 0, 0, 4092, 0, 0, 0, 36, 0, 0, 0, 4008, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13988, 17442, 14002, 15869, 19108, 17328, 16005, 17330] avg=16384.0000 max=19108 max_rank=4 min=13988 min_rank=0 imbalance_ratio=1.166260 max_load_reduction=111964 max_load_reduction_pct=0.854218 moved_to_echo=69771 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4066, 0, 0, 0, 3930, 0, 0, 33, 0, 4085, 0, 4058, 3404, 0, 0, 765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 1998, 20, 17, 1804, 21, 3396, 0, 0, 821, 327, 0, 0, 0, 0, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4095, 0, 3831, 0, 0, 4093, 4079, 2, 111, 0, 5, 42, 0, 2, 1, 1610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 430, 0, 1, 4095, 12, 0, 163, 0, 0, 12, 3418, 4092, 0, 104, 0, 2077, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 3, 0, 3, 2, 0, 2456, 1960, 982, 1, 600, 0, 4016, 43, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4006, 1, 2028, 460, 4096, 702, 2667, 0, 1, 0, 0, 3, 2, 4079, 120, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 4096, 5, 0, 139, 1, 2878, 0, 0, 4090, 268, 6, 3977, 0, 0, 1613, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4031, 60, 22, 360, 0, 2024, 0, 0, 5, 1, 0, 0, 1, 3280, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20341, 12502, 17879, 14767, 14211, 18818, 17596, 14958] avg=16384.0000 max=20341 max_rank=0 min=12502 min_rank=1 imbalance_ratio=1.241516 max_load_reduction=110731 max_load_reduction_pct=0.844810 moved_to_echo=65583 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4028, 0, 0, 0, 2111, 52, 87, 140, 0, 4089, 0, 4016, 3302, 0, 0, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3163, 4048, 6, 18, 0, 304, 2814, 6, 7, 4096, 3, 0, 22, 0, 3768, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 4070, 31, 2527, 2, 118, 4086, 4095, 0, 38, 56, 0, 15, 0, 3, 1, 1616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 876, 0, 0, 4069, 17, 1, 18, 0, 21, 128, 1652, 3957, 2, 2488, 0, 2162, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 172, 0, 0, 22, 18, 0, 3309, 1967, 3615, 0, 107, 0, 4024, 0, 3508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 1532, 0, 3248, 0, 4070, 0, 581, 0, 0, 1, 0, 117, 92, 4040, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 4048, 0, 9, 0, 7, 503, 11, 27, 4065, 0, 0, 4086, 25, 23, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 0, 4021, 3885, 0, 0, 0, 0, 4018, 4, 10, 0, 0, 11, 0, 0, 3969, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18623, 18780, 16719, 15759, 16834, 13879, 13120, 17358] avg=16384.0000 max=18780 max_rank=1 min=13120 min_rank=6 imbalance_ratio=1.146240 max_load_reduction=112292 max_load_reduction_pct=0.856720 moved_to_echo=61191 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3072, 128, 31232, 27392, 128, 256, 384, 128, 28672, 256, 256, 256, 128, 128, 128, 128, 31232, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[29440, 640, 128, 128, 21888, 128, 2048, 896, 128, 27008, 128, 28800, 14464, 128, 128, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[3072, 32128, 1280, 19200, 128, 512, 32384, 29056, 128, 256, 512, 256, 256, 128, 128, 128, 12800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[256, 7552, 32256, 256, 128, 256, 6016, 7168, 128, 128, 32640, 512, 128, 32512, 384, 128, 11008, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[16256, 21248, 128, 128, 12288, 2176, 15616, 128, 128, 26112, 512, 384, 128, 0, 16384, 20608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1408, 23680, 256, 128, 128, 512, 128, 22656, 15488, 7936, 256, 768, 2048, 31616, 512, 24576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 22912, 128, 384, 31872, 128, 128, 1408, 1024, 256, 896, 12800, 32512, 128, 8576, 128, 16000, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[512, 22272, 128, 15488, 2816, 32640, 1280, 12416, 640, 128, 768, 128, 896, 1280, 32128, 4480, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4069, 0, 0, 0, 4096, 14, 4060, 3283, 6, 0, 2080, 452, 3937, 1450, 0, 419, 0, 1679, 12, 2, 11, 0, 21, 0, 0, 4086, 4096, 3414, 4096, 29, 0, 3712, 23, 0, 0, 0, 0, 3965, 4050, 382, 8, 10, 35, 3868, 0, 2243, 1321, 1544, 3, 3958, 4024, 0, 254, 0, 7, 4096, 0, 1, 0, 4088, 0, 0, 13, 0, 69, 44, 179, 120, 49, 0, 3952, 0, 0, 24, 0, 61, 20, 17, 0, 0, 26, 0, 2265, 159, 1813, 5, 0, 4050, 0, 3218, 82, 60, 0, 2, 200, 0, 0, 3081, 4070, 0, 2965, 0, 0, 4084, 4086, 4084, 63, 0, 4, 11, 11, 11, 4, 0, 1, 32, 0, 0, 3, 0, 3993, 0, 4090, 6, 0, 0, 3982, 1159] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4096, 7, 88, 0, 4095, 50, 4092, 4038, 0, 0, 3984, 6, 4096, 3944, 0, 0, 77, 1064, 0, 0, 0, 1, 0, 0, 0, 4096, 1059, 3309, 4096, 1, 0, 4032, 211, 0, 0, 0, 0, 4096, 4096, 7, 14, 0, 28, 1924, 0, 291, 172, 4091, 0, 3784, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 48, 0, 2, 0, 0, 3114, 23, 745, 6, 0, 3888, 0, 0, 0, 0, 3, 1, 0, 13, 0, 0, 0, 0, 0, 0, 2, 0, 3972, 0, 4096, 172, 0, 0, 0, 623, 4, 0, 3677, 4096, 1, 4081, 1, 0, 4082, 4096, 4096, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4066, 0, 4092, 0, 1, 27, 3933, 59] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4031, 150, 0, 0, 3222, 311, 4066, 3984, 0, 1, 1445, 0, 3726, 0, 0, 0, 35, 3372, 0, 97, 0, 0, 8, 0, 0, 4096, 4093, 3327, 4091, 0, 4, 201, 188, 0, 0, 0, 1, 2960, 4079, 84, 12, 0, 51, 381, 0, 1, 0, 2318, 0, 4076, 4078, 0, 2, 5, 0, 4090, 0, 0, 0, 4000, 16, 435, 52, 3, 11, 2479, 8, 0, 4095, 0, 2065, 7, 0, 4068, 0, 1, 0, 13, 1737, 0, 1, 0, 3947, 0, 834, 254, 0, 3853, 242, 3981, 0, 0, 1, 0, 1979, 98, 1, 541, 4093, 0, 4059, 0, 0, 4064, 4090, 3587, 2611, 0, 0, 7, 9, 0, 0, 4, 0, 27, 21, 2, 0, 4, 3986, 3, 3966, 4, 2, 0, 3190, 35] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4092, 9, 87, 0, 3415, 51, 4015, 4096, 51, 10, 2806, 1, 4084, 0, 0, 0, 3, 995, 0, 0, 9, 675, 40, 0, 0, 4092, 4082, 700, 4094, 0, 3, 4076, 462, 1, 0, 0, 10, 4092, 4093, 0, 0, 2, 0, 1080, 917, 1, 0, 4093, 193, 4090, 4092, 0, 0, 0, 0, 3936, 0, 0, 0, 4094, 110, 0, 0, 5, 0, 1198, 0, 174, 3573, 0, 4060, 6, 8, 4079, 0, 0, 25, 18, 1195, 5, 0, 0, 180, 0, 3076, 0, 0, 4086, 0, 4092, 0, 6, 0, 0, 224, 19, 4, 0, 4095, 0, 534, 17, 0, 4085, 4095, 4092, 123, 0, 1, 0, 0, 1, 0, 0, 0, 4, 157, 0, 0, 4, 4095, 1, 4071, 0, 0, 0, 3886, 3026] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4088, 4078, 0, 0, 96, 1, 3806, 4096, 1405, 29, 68, 3903, 3959, 19, 0, 559, 12, 3922, 14, 343, 9, 41, 135, 0, 505, 112, 2539, 1480, 3605, 30, 33, 0, 3, 0, 0, 0, 0, 1318, 4068, 3750, 85, 0, 12, 3709, 1, 5, 0, 1346, 16, 4096, 4078, 0, 574, 59, 2443, 3020, 464, 2, 353, 3895, 0, 20, 0, 278, 0, 16, 1, 2994, 0, 0, 0, 0, 60, 4081, 36, 892, 0, 0, 1607, 0, 12, 281, 4093, 48, 3452, 0, 0, 3484, 22, 4040, 67, 6, 2, 0, 3007, 1, 4, 625, 3772, 20, 4058, 3, 23, 3920, 4040, 3607, 1, 229, 0, 7, 0, 53, 0, 0, 0, 1, 130, 19, 6, 3, 4096, 0, 4037, 0, 0, 0, 1520, 214] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4023, 1, 0, 354, 4096, 8, 129, 3468, 0, 90, 3040, 0, 3891, 2265, 99, 672, 0, 3861, 8, 0, 0, 408, 0, 5, 0, 3696, 3872, 517, 3802, 62, 0, 4037, 0, 2, 0, 0, 0, 4092, 3968, 0, 0, 79, 6, 4006, 0, 0, 0, 3895, 1, 151, 4081, 0, 254, 0, 0, 4081, 208, 0, 0, 4096, 96, 20, 0, 0, 0, 23, 7, 3989, 1310, 0, 4065, 0, 34, 754, 0, 383, 0, 0, 0, 0, 0, 223, 708, 0, 4090, 0, 27, 4093, 0, 3965, 0, 0, 31, 90, 3855, 0, 0, 3186, 3326, 0, 1952, 209, 1, 4091, 3632, 4055, 537, 0, 0, 0, 8, 0, 0, 0, 0, 0, 7, 0, 28, 56, 4096, 0, 4095, 0, 0, 138, 2069, 499] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 21, 4096, 0, 1383, 4096, 0, 0, 836, 0, 4070, 70, 1, 5, 7, 4076, 0, 15, 0, 0, 0, 0, 2, 1349, 3848, 4056, 4094, 0, 140, 7, 0, 0, 0, 0, 63, 3996, 4092, 747, 0, 0, 81, 3944, 0, 1146, 0, 3864, 0, 4020, 4087, 0, 2, 0, 0, 3768, 0, 0, 0, 4010, 80, 14, 76, 51, 20, 622, 0, 0, 1321, 20, 4085, 58, 5, 4057, 13, 5, 2, 0, 298, 0, 11, 0, 0, 0, 4075, 0, 0, 4012, 5, 4072, 1, 2, 5, 0, 185, 0, 11, 3955, 4007, 0, 3148, 1, 3, 4043, 4080, 4096, 1444, 0, 0, 17, 102, 0, 4, 0, 0, 4, 0, 2, 0, 8, 3721, 0, 4090, 0, 28, 116, 3994, 1015] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4057, 419, 0, 1, 4091, 0, 3595, 750, 0, 1, 4068, 4095, 4089, 0, 0, 17, 0, 48, 0, 0, 0, 62, 22, 0, 0, 3944, 2640, 4033, 4046, 6, 91, 2348, 279, 0, 0, 0, 75, 2245, 4043, 0, 106, 2, 68, 0, 859, 1706, 0, 4096, 0, 4089, 4072, 0, 1, 1, 0, 3457, 0, 0, 4, 3859, 55, 0, 0, 2, 8, 8, 2, 3744, 3947, 0, 4040, 0, 0, 3808, 0, 7, 0, 0, 0, 4, 0, 0, 667, 0, 1910, 53, 0, 4005, 0, 4093, 18, 0, 586, 72, 50, 0, 0, 0, 4093, 0, 4094, 186, 0, 4023, 4086, 99, 9, 0, 0, 27, 0, 377, 0, 0, 6, 0, 0, 16, 0, 262, 4096, 0, 4032, 1, 47, 0, 3938, 3216] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 88, 0, 4095, 50, 4092, 4038, 0, 0, 3984, 6, 948, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 1064, 0, 0, 0, 1, 0, 0, 0, 4096, 1059, 3309, 2327, 1, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 0, 0, 0, 0, 4096, 4096, 7, 14, 0, 28, 1924, 0, 291, 172, 4091, 780, 766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3784, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 48, 0, 2, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3114, 23, 745, 6, 0, 3888, 0, 0, 0, 0, 3, 1, 0, 13, 0, 4096, 1391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3972, 0, 4096, 172, 0, 0, 0, 623, 4, 2098, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3677, 4096, 1, 4081, 1, 0, 0, 3166, 4096, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4066, 0, 4092, 0, 1, 27, 3933, 59, 2691, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21252, 15966, 16476, 16284, 13280, 11956, 19936, 15922] avg=16384.0000 max=21252 max_rank=0 min=11956 min_rank=5 imbalance_ratio=1.297119 max_load_reduction=109820 max_load_reduction_pct=0.837860 moved_to_echo=61094 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 14, 4060, 3283, 6, 0, 2080, 452, 910, 1450, 0, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1679, 12, 2, 11, 0, 21, 0, 0, 4086, 4096, 3414, 2327, 29, 0, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 3965, 4050, 382, 8, 10, 35, 3868, 0, 2243, 1321, 1544, 780, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3958, 4024, 0, 254, 0, 7, 4096, 0, 1, 0, 4088, 0, 0, 13, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 44, 179, 120, 49, 0, 3952, 0, 0, 24, 0, 61, 20, 17, 0, 0, 4069, 1392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 2265, 159, 1813, 5, 0, 4050, 0, 3218, 82, 60, 0, 2, 200, 0, 2017, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3081, 4070, 0, 2965, 0, 0, 0, 3158, 4084, 63, 0, 4, 11, 11, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 32, 0, 0, 3, 0, 3993, 0, 4090, 6, 0, 0, 3982, 1159, 2692, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16770, 19389, 18993, 16608, 9996, 14886, 17458, 16972] avg=16384.0000 max=19389 max_rank=1 min=9996 min_rank=4 imbalance_ratio=1.183411 max_load_reduction=111683 max_load_reduction_pct=0.852074 moved_to_echo=59312 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 150, 0, 0, 3222, 311, 4066, 3984, 0, 1, 1445, 0, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 3372, 0, 97, 0, 0, 8, 0, 0, 4096, 4093, 3327, 2324, 0, 4, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, 1, 2960, 4079, 84, 12, 0, 51, 381, 0, 1, 0, 2318, 779, 765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4078, 0, 2, 5, 0, 4090, 0, 0, 0, 4000, 16, 435, 52, 3, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2479, 8, 0, 4095, 0, 2065, 7, 0, 4068, 0, 1, 0, 13, 1737, 0, 4031, 1385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3947, 0, 834, 254, 0, 3853, 242, 3981, 0, 0, 1, 0, 1979, 98, 1908, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 541, 4093, 0, 4059, 0, 0, 0, 3161, 3587, 2611, 0, 0, 7, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 27, 21, 2, 0, 4, 3986, 3, 3966, 4, 2, 0, 3190, 35, 2679, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14042, 17557, 11619, 16921, 19900, 18086, 18069, 14878] avg=16384.0000 max=19900 max_rank=4 min=11619 min_rank=2 imbalance_ratio=1.214600 max_load_reduction=111172 max_load_reduction_pct=0.848175 moved_to_echo=70933 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 87, 0, 3415, 51, 4015, 4096, 51, 10, 2806, 1, 945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 995, 0, 0, 9, 675, 40, 0, 0, 4092, 4082, 700, 2325, 0, 3, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 1, 0, 0, 10, 4092, 4093, 0, 0, 2, 0, 1080, 917, 1, 0, 4093, 780, 766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 4090, 4092, 0, 0, 0, 0, 3936, 0, 0, 0, 4094, 110, 0, 0, 5, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1198, 0, 174, 3573, 0, 4060, 6, 8, 4079, 0, 0, 25, 18, 1195, 5, 4092, 1392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 3076, 0, 0, 4086, 0, 4092, 0, 6, 0, 0, 224, 19, 2092, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4095, 0, 534, 17, 0, 0, 3165, 4092, 123, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 157, 0, 0, 4, 4095, 1, 4071, 0, 0, 0, 3886, 3026, 2693, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15486, 17000, 16297, 16684, 19825, 14764, 12032, 18984] avg=16384.0000 max=19825 max_rank=4 min=12032 min_rank=6 imbalance_ratio=1.210022 max_load_reduction=111247 max_load_reduction_pct=0.848747 moved_to_echo=65605 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 354, 4096, 8, 129, 3468, 0, 90, 3040, 0, 900, 2265, 99, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 8, 0, 0, 408, 0, 5, 0, 3696, 3872, 517, 2160, 62, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4092, 3968, 0, 0, 79, 6, 4006, 0, 0, 0, 3895, 724, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 151, 4081, 0, 254, 0, 0, 4081, 208, 0, 0, 4096, 96, 20, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 7, 3989, 1310, 0, 4065, 0, 34, 754, 0, 383, 0, 0, 0, 0, 4023, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 708, 0, 4090, 0, 27, 4093, 0, 3965, 0, 0, 31, 90, 3855, 0, 1993, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3186, 3326, 0, 1952, 209, 1, 0, 2807, 4055, 537, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 28, 56, 4096, 0, 4095, 0, 0, 138, 2069, 499, 2697, 998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15122, 18626, 17451, 13134, 15982, 19993, 16081, 14683] avg=16384.0000 max=19993 max_rank=5 min=13134 min_rank=3 imbalance_ratio=1.220276 max_load_reduction=111079 max_load_reduction_pct=0.847466 moved_to_echo=66739 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4078, 0, 0, 96, 1, 3806, 4096, 1405, 29, 68, 3903, 908, 19, 0, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3922, 14, 343, 9, 41, 135, 0, 505, 112, 2539, 1480, 2042, 30, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1318, 4068, 3750, 85, 0, 12, 3709, 1, 5, 0, 1346, 690, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 4078, 0, 574, 59, 2443, 3020, 464, 2, 353, 3895, 0, 20, 0, 278, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 2994, 0, 0, 0, 0, 60, 4081, 36, 892, 0, 0, 1607, 0, 4088, 1337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 281, 4093, 48, 3452, 0, 0, 3484, 22, 4040, 67, 6, 2, 0, 3007, 1, 2032, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 625, 3772, 20, 4058, 3, 23, 0, 3114, 3607, 1, 229, 0, 7, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 130, 19, 6, 3, 4096, 0, 4037, 0, 0, 0, 1520, 214, 2583, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18968, 11217, 15746, 19465, 15112, 21420, 15516, 13628] avg=16384.0000 max=21420 max_rank=5 min=11217 min_rank=1 imbalance_ratio=1.307373 max_load_reduction=109652 max_load_reduction_pct=0.836578 moved_to_echo=65676 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 419, 0, 1, 4091, 0, 3595, 750, 0, 1, 4068, 4095, 946, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 62, 22, 0, 0, 3944, 2640, 4033, 2298, 6, 91, 2348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 0, 0, 0, 75, 2245, 4043, 0, 106, 2, 68, 0, 859, 1706, 0, 4096, 771, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4072, 0, 1, 1, 0, 3457, 0, 0, 4, 3859, 55, 0, 0, 2, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 2, 3744, 3947, 0, 4040, 0, 0, 3808, 0, 7, 0, 0, 0, 4, 4057, 1372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, 0, 1910, 53, 0, 4005, 0, 4093, 18, 0, 586, 72, 50, 0, 2094, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4094, 186, 0, 0, 3158, 99, 9, 0, 0, 27, 0, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 16, 0, 262, 4096, 0, 4032, 1, 47, 0, 3938, 3216, 2651, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17983, 15492, 15014, 15704, 20997, 14525, 12043, 19314] avg=16384.0000 max=20997 max_rank=4 min=12043 min_rank=6 imbalance_ratio=1.281555 max_load_reduction=110075 max_load_reduction_pct=0.839806 moved_to_echo=66879 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 21, 4096, 0, 1383, 4096, 0, 0, 836, 0, 941, 70, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4076, 0, 15, 0, 0, 0, 0, 2, 1349, 3848, 4056, 2325, 0, 140, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 3996, 4092, 747, 0, 0, 81, 3944, 0, 1146, 0, 3864, 780, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 4087, 0, 2, 0, 0, 3768, 0, 0, 0, 4010, 80, 14, 76, 51, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 622, 0, 0, 1321, 20, 4085, 58, 5, 4057, 13, 5, 2, 0, 298, 0, 4096, 1378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4075, 0, 0, 4012, 5, 4072, 1, 2, 5, 0, 185, 0, 2085, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3955, 4007, 0, 3148, 1, 3, 0, 3153, 4096, 1444, 0, 0, 17, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4, 0, 2, 0, 8, 3721, 0, 4090, 0, 28, 116, 3994, 1015, 2665, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11449, 15825, 19476, 16272, 15980, 15442, 19937, 16691] avg=16384.0000 max=19937 max_rank=6 min=11449 min_rank=0 imbalance_ratio=1.216858 max_load_reduction=111135 max_load_reduction_pct=0.847893 moved_to_echo=68050 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 4736, 256, 384, 27264, 512, 25216, 27904, 1536, 256, 18432, 8576, 7424, 7808, 128, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 28288, 32640, 0, 1152, 128, 2560, 30592, 768, 128, 384, 32256, 512, 512, 256, 384, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 7552, 256, 11776, 14336, 128, 26240, 128, 128, 20992, 128, 1408, 128, 128, 4864, 128, 32640, 11136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 15104, 31616, 128, 24960, 512, 128, 0, 24960, 27776, 5632, 256, 128, 128, 256, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[256, 19072, 128, 512, 128, 1280, 256, 128, 512, 25472, 26240, 20864, 18176, 128, 384, 18432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 512, 11904, 256, 19328, 384, 128, 31616, 384, 31616, 384, 128, 640, 256, 10240, 128, 16384, 7808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 384, 128, 128, 384, 32256, 128, 32512, 128, 128, 384, 26624, 9344, 21376, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1280, 128, 0, 0, 256, 26880, 32512, 4992, 256, 128, 384, 18944, 1792, 5504, 1536, 25344, 6144, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 18, 3995, 0, 0, 2, 0, 393, 0, 0, 0, 2701, 0, 0, 0, 4093, 68, 0, 0, 4096, 0, 4049, 3853, 77, 3373, 0, 72, 101, 3708, 0, 3347, 46, 978, 149, 0, 1628, 3956, 56, 4096, 388, 178, 0, 4078, 0, 0, 0, 4086, 22, 0, 1, 272, 3201, 0, 0, 4067, 3298, 0, 349, 73, 114, 3612, 2, 26, 0, 4093, 2, 4096, 0, 4000, 3877, 0, 0, 1033, 3, 0, 0, 431, 701, 4, 0, 0, 80, 1, 307, 11, 4095, 0, 0, 16, 155, 0, 1, 336, 0, 121, 3895, 3571, 4, 0, 3834, 5, 0, 3981, 4086, 3950, 0, 0, 0, 0, 3, 0, 0, 0, 0, 148, 1, 34, 521, 3824, 3250, 95, 4096, 12, 3734, 0, 1, 42, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 98, 5, 5, 1, 0, 0, 4096, 0, 0, 1, 1, 0, 1, 0, 4096, 121, 4071, 4061, 0, 3996, 0, 0, 1, 4094, 0, 4095, 1, 7, 2056, 0, 16, 4095, 4, 3633, 0, 3373, 0, 818, 18, 1775, 34, 1886, 0, 0, 14, 0, 4067, 0, 0, 4089, 3834, 0, 0, 178, 0, 4096, 0, 0, 0, 3807, 19, 2370, 0, 4086, 4096, 1, 3, 11, 0, 1, 8, 0, 50, 2, 16, 391, 0, 71, 0, 0, 4077, 7, 0, 788, 123, 0, 0, 14, 5, 4095, 3992, 3738, 0, 0, 4019, 0, 5, 4083, 4091, 4023, 70, 0, 1, 6, 0, 3862, 0, 0, 0, 1, 0, 0, 13, 4074, 17, 11, 4090, 0, 4096, 0, 7, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4091, 20, 0, 74, 47, 1, 15, 4, 2, 4045, 0, 0, 91, 0, 1, 33, 1, 4036, 0, 4096, 4094, 0, 4092, 0, 0, 0, 4095, 4, 4096, 0, 0, 3951, 10, 0, 3949, 0, 4081, 12, 3234, 0, 2178, 3, 0, 1013, 4096, 0, 0, 0, 0, 155, 0, 0, 4094, 3894, 0, 0, 14, 0, 4081, 0, 0, 0, 4096, 0, 2234, 0, 3460, 4061, 0, 0, 457, 1, 0, 0, 0, 0, 0, 0, 4063, 0, 21, 0, 0, 4078, 0, 702, 192, 0, 0, 0, 5, 0, 149, 4095, 3716, 0, 1, 4078, 0, 0, 4029, 4063, 4092, 0, 0, 0, 6, 75, 4004, 1, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 4, 3496, 0, 0, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 58, 0, 1061, 0, 0, 4096, 0, 0, 3, 3236, 0, 0, 0, 4091, 9, 4095, 4096, 0, 4096, 0, 0, 0, 4096, 2897, 4096, 0, 32, 1073, 0, 1499, 4096, 0, 2957, 0, 4047, 0, 139, 0, 0, 0, 872, 0, 0, 0, 0, 2603, 0, 2, 4096, 4092, 0, 0, 0, 0, 4096, 0, 0, 56, 4096, 47, 2968, 0, 3674, 3605, 0, 0, 1, 0, 0, 0, 0, 349, 204, 0, 0, 0, 16, 0, 2, 4095, 0, 0, 0, 58, 0, 0, 0, 0, 4064, 4096, 4095, 0, 0, 4096, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 14, 1252, 0, 0, 79, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 4096, 0, 1, 0, 0, 0, 1188, 14, 1, 4093, 0, 0, 0, 2392, 1, 0, 0, 4067, 612, 4020, 4077, 28, 4096, 0, 12, 0, 4053, 99, 3946, 0, 657, 3056, 5, 1560, 3868, 2, 1803, 0, 3001, 0, 4096, 0, 547, 0, 3918, 0, 0, 0, 0, 0, 0, 47, 4096, 4060, 25, 13, 3, 7, 3688, 0, 2, 2, 4087, 0, 17, 0, 4096, 4004, 0, 0, 2, 0, 9, 0, 7, 89, 3, 0, 183, 0, 356, 46, 115, 4062, 0, 0, 713, 17, 0, 372, 4, 2, 4092, 4076, 3924, 23, 33, 3809, 300, 0, 4093, 4068, 4096, 31, 0, 0, 1, 0, 766, 0, 1, 116, 0, 2, 0, 5, 4096, 0, 0, 4071, 0, 4020, 0, 4, 0, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 4092, 4, 0, 258, 0, 0, 75, 0, 0, 4021, 0, 0, 1037, 413, 1, 0, 0, 4096, 0, 4094, 4092, 0, 4070, 0, 2, 0, 3505, 0, 4050, 1, 0, 0, 0, 0, 4095, 5, 4091, 0, 3957, 3, 599, 2, 0, 10, 4096, 0, 0, 1, 129, 4062, 3, 0, 4094, 4039, 511, 2, 0, 0, 2178, 0, 10, 0, 4096, 53, 0, 7, 4039, 3218, 0, 7, 3964, 0, 1, 0, 24, 15, 0, 0, 360, 0, 12, 63, 0, 4005, 0, 4057, 1, 4, 0, 2, 0, 0, 4016, 4019, 3969, 3, 60, 4093, 0, 0, 4096, 4088, 4094, 0, 0, 0, 5, 7, 0, 0, 0, 0, 0, 0, 2, 235, 4096, 39, 7, 4096, 7, 4042, 471, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 19, 4092, 20, 0, 0, 0, 0, 40, 0, 0, 3957, 0, 0, 1, 0, 69, 1, 0, 3997, 158, 4086, 4096, 9, 4003, 423, 0, 0, 3859, 411, 3785, 73, 922, 1934, 60, 500, 4051, 191, 2960, 0, 3931, 193, 3168, 7, 0, 0, 2490, 0, 5, 0, 694, 0, 127, 1490, 4096, 4075, 0, 0, 0, 5, 4096, 95, 0, 20, 3937, 118, 4049, 146, 4061, 3741, 0, 2, 1032, 0, 0, 4, 8, 0, 0, 69, 0, 0, 0, 0, 19, 4053, 40, 8, 1965, 0, 0, 1, 0, 7, 3339, 3966, 4095, 60, 3, 4062, 0, 8, 4073, 3592, 3853, 17, 152, 0, 0, 0, 318, 0, 0, 0, 0, 0, 17, 0, 3919, 0, 0, 3980, 89, 4029, 31, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3933, 3028, 0, 7, 17, 1, 0, 0, 0, 4070, 2, 0, 0, 0, 0, 0, 0, 4092, 0, 4078, 3873, 31, 4091, 0, 0, 3, 4096, 0, 4096, 37, 0, 1461, 0, 2, 4094, 3, 3515, 87, 4060, 0, 4096, 120, 23, 0, 4044, 1, 1, 0, 0, 2750, 0, 805, 4085, 3989, 288, 0, 0, 0, 2338, 38, 0, 1, 3225, 0, 2473, 0, 4037, 3738, 334, 11, 3, 135, 152, 0, 0, 0, 0, 0, 7, 0, 366, 0, 0, 3979, 0, 1415, 88, 0, 0, 0, 15, 0, 1, 4055, 3961, 0, 20, 4032, 1, 0, 3989, 3999, 4084, 0, 0, 32, 0, 1, 3160, 180, 46, 0, 27, 0, 0, 0, 4013, 22, 2, 4058, 0, 4095, 46, 0, 11, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 58, 0, 1061, 0, 0, 4096, 0, 0, 3, 3236, 4095, 2156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3918, 0, 4096, 0, 0, 0, 4096, 2897, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1073, 0, 1499, 945, 0, 2957, 0, 4047, 0, 139, 0, 0, 0, 872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2603, 0, 2, 4096, 4092, 0, 0, 0, 0, 4096, 0, 0, 56, 1835, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 47, 2968, 0, 3674, 3605, 0, 0, 1, 0, 0, 0, 0, 349, 204, 0, 1102, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 2, 4095, 0, 0, 0, 58, 0, 0, 0, 0, 4064, 4096, 2349, 1316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4095, 2716, 0, 0, 0, 0, 14, 1252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1935, 1747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18801, 19112, 11564, 17058, 16224, 15996, 16268, 16049] avg=16384.0000 max=19112 max_rank=1 min=11564 min_rank=2 imbalance_ratio=1.166504 max_load_reduction=111960 max_load_reduction_pct=0.854187 moved_to_echo=64537 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 19, 4092, 20, 0, 0, 0, 0, 40, 0, 0, 3957, 0, 0, 1, 0, 4086, 2106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 1, 0, 0, 158, 0, 3918, 9, 4003, 423, 0, 0, 3859, 411, 3785, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 922, 1934, 60, 500, 935, 191, 2960, 0, 3931, 193, 3168, 7, 0, 0, 2490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 694, 0, 127, 1490, 4096, 4075, 0, 0, 0, 5, 4096, 95, 0, 20, 1815, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3937, 118, 4049, 146, 4061, 3741, 0, 2, 1032, 0, 0, 4, 8, 0, 0, 69, 1036, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4053, 40, 8, 1965, 0, 0, 1, 0, 7, 3339, 3966, 2335, 1301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 60, 3, 4062, 0, 8, 0, 3592, 2556, 17, 152, 0, 0, 0, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 3919, 0, 0, 3980, 89, 4029, 31, 0, 0, 0, 1891, 1738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14321, 16709, 17291, 16779, 18381, 17034, 14863, 15694] avg=16384.0000 max=18381 max_rank=4 min=14321 min_rank=0 imbalance_ratio=1.121887 max_load_reduction=112691 max_load_reduction_pct=0.859764 moved_to_echo=65972 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4096, 0, 1, 0, 0, 0, 1188, 14, 1, 4093, 0, 0, 0, 2392, 4020, 2143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 612, 0, 3899, 28, 4096, 0, 12, 0, 4053, 99, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 3056, 5, 1560, 892, 2, 1803, 0, 3001, 0, 4096, 0, 547, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4096, 4060, 25, 13, 3, 7, 3688, 0, 2, 2, 1733, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 17, 0, 4096, 4004, 0, 0, 2, 0, 9, 0, 7, 89, 3, 0, 1102, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 356, 46, 115, 4062, 0, 0, 713, 17, 0, 372, 4, 2, 4092, 4076, 2347, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3924, 23, 33, 3809, 300, 0, 0, 4068, 2716, 31, 0, 0, 1, 0, 766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 116, 0, 2, 0, 5, 4096, 0, 0, 4071, 0, 4020, 0, 4, 0, 8, 1924, 1746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17949, 16746, 19537, 13954, 13594, 17628, 15671, 15993] avg=16384.0000 max=19537 max_rank=2 min=13594 min_rank=4 imbalance_ratio=1.192444 max_load_reduction=111535 max_load_reduction_pct=0.850945 moved_to_echo=62886 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 98, 5, 5, 1, 0, 0, 4096, 0, 0, 1, 1, 4071, 2159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 121, 0, 3884, 0, 3996, 0, 0, 1, 4094, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2056, 0, 16, 944, 4, 3633, 0, 3373, 0, 818, 18, 1775, 34, 1886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4067, 0, 0, 4089, 3834, 0, 0, 178, 0, 4096, 0, 0, 0, 1835, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3807, 19, 2370, 0, 4086, 4096, 1, 3, 11, 0, 1, 8, 0, 50, 2, 16, 1082, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 0, 71, 0, 0, 4077, 7, 0, 788, 123, 0, 0, 14, 5, 4095, 3992, 2342, 1316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3738, 0, 0, 4019, 0, 5, 0, 4091, 2668, 70, 0, 1, 6, 0, 3862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 13, 4074, 17, 11, 4090, 0, 4096, 0, 7, 0, 0, 1937, 1741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14532, 16193, 14564, 18386, 15729, 17221, 18460, 15987] avg=16384.0000 max=18460 max_rank=6 min=14532 min_rank=0 imbalance_ratio=1.126709 max_load_reduction=112612 max_load_reduction_pct=0.859161 moved_to_echo=67397 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 20, 0, 74, 47, 1, 15, 4, 2, 4045, 0, 0, 91, 0, 4096, 2127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 33, 1, 0, 0, 0, 3916, 0, 4092, 0, 0, 0, 4095, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 10, 0, 911, 0, 4081, 12, 3234, 0, 2178, 3, 0, 1013, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 4094, 3894, 0, 0, 14, 0, 4081, 0, 0, 0, 1769, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2234, 0, 3460, 4061, 0, 0, 457, 1, 0, 0, 0, 0, 0, 0, 1101, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 21, 0, 0, 4078, 0, 702, 192, 0, 0, 0, 5, 0, 149, 4095, 2310, 1269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3716, 0, 1, 4078, 0, 0, 0, 4063, 2714, 0, 0, 0, 6, 75, 4004, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 4, 3496, 0, 0, 3, 0, 1909, 1719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14613, 16238, 19489, 14284, 15588, 16884, 18658, 15318] avg=16384.0000 max=19489 max_rank=2 min=14284 min_rank=3 imbalance_ratio=1.189514 max_load_reduction=111583 max_load_reduction_pct=0.851311 moved_to_echo=66448 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 3995, 0, 0, 2, 0, 393, 0, 0, 0, 2701, 0, 0, 0, 4093, 4049, 2159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 3681, 77, 3373, 0, 72, 101, 3708, 0, 3347, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 978, 149, 0, 1628, 904, 56, 4096, 388, 178, 0, 4078, 0, 0, 0, 4086, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 272, 3201, 0, 0, 4067, 3298, 0, 349, 73, 114, 3612, 2, 26, 0, 1778, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 4096, 0, 4000, 3877, 0, 0, 1033, 3, 0, 0, 431, 701, 4, 0, 1066, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 1, 307, 11, 4095, 0, 0, 16, 155, 0, 1, 336, 0, 121, 3895, 2284, 1274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3571, 4, 0, 3834, 5, 0, 0, 4086, 2613, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 1, 34, 521, 3824, 3250, 95, 4096, 12, 3734, 0, 1, 42, 0, 1937, 1697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17410, 14473, 16563, 17064, 19478, 12576, 14116, 19392] avg=16384.0000 max=19478 max_rank=4 min=12576 min_rank=5 imbalance_ratio=1.188843 max_load_reduction=111594 max_load_reduction_pct=0.851395 moved_to_echo=65562 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4092, 4, 0, 258, 0, 0, 75, 0, 0, 4021, 0, 0, 1037, 413, 4094, 2159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3914, 0, 4070, 0, 2, 0, 3505, 0, 4050, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 944, 5, 4091, 0, 3957, 3, 599, 2, 0, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 129, 4062, 3, 0, 4094, 4039, 511, 2, 0, 0, 2178, 0, 10, 0, 1835, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 53, 0, 7, 4039, 3218, 0, 7, 3964, 0, 1, 0, 24, 15, 0, 0, 1101, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 12, 63, 0, 4005, 0, 4057, 1, 4, 0, 2, 0, 0, 4016, 4019, 2350, 1316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 3, 60, 4093, 0, 0, 0, 4088, 2716, 0, 0, 0, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 235, 4096, 39, 7, 4096, 7, 4042, 471, 0, 0, 0, 1937, 1746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16154, 15543, 13707, 17141, 16703, 20205, 14941, 16678] avg=16384.0000 max=20205 max_rank=5 min=13707 min_rank=2 imbalance_ratio=1.233215 max_load_reduction=110867 max_load_reduction_pct=0.845848 moved_to_echo=68527 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3933, 3028, 0, 7, 17, 1, 0, 0, 0, 4070, 2, 0, 0, 0, 4078, 2156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3704, 31, 4091, 0, 0, 3, 4096, 0, 4096, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1461, 0, 2, 945, 3, 3515, 87, 4060, 0, 4096, 120, 23, 0, 4044, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2750, 0, 805, 4085, 3989, 288, 0, 0, 0, 2338, 38, 0, 1, 1834, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3225, 0, 2473, 0, 4037, 3738, 334, 11, 3, 135, 152, 0, 0, 0, 0, 0, 1098, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 366, 0, 0, 3979, 0, 1415, 88, 0, 0, 0, 15, 0, 1, 4055, 2287, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 0, 20, 4032, 1, 0, 0, 3999, 2709, 0, 0, 32, 0, 1, 3160, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 27, 0, 0, 0, 4013, 22, 2, 4058, 0, 4095, 46, 0, 11, 3, 1936, 1702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17292, 16058, 18357, 16406, 15375, 13528, 18095, 15961] avg=16384.0000 max=18357 max_rank=2 min=13528 min_rank=5 imbalance_ratio=1.120422 max_load_reduction=112715 max_load_reduction_pct=0.859947 moved_to_echo=62959 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 128, 128, 0, 1024, 0, 30848, 256, 31872, 512, 128, 128, 31616, 3456, 31616, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 32512, 3072, 128, 512, 128, 512, 2432, 128, 128, 31104, 128, 0, 1152, 10240, 32640, 17280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2688, 13696, 128, 5248, 7424, 384, 27136, 512, 25856, 256, 19200, 256, 2432, 1152, 25600, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[31488, 256, 18304, 256, 31488, 30464, 384, 128, 6528, 256, 256, 128, 512, 1280, 256, 128, 8704, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 1152, 16896, 256, 2432, 32768, 31360, 896, 384, 384, 128, 28288, 256, 128, 128, 14464, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 256, 256, 128, 128, 896, 32256, 3328, 128, 32640, 128, 31616, 640, 128, 128, 128, 15488, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[31104, 128, 128, 32128, 384, 128, 0, 32128, 21504, 128, 256, 128, 128, 128, 13440, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[5120, 128, 896, 512, 256, 32512, 128, 6272, 3840, 384, 0, 384, 384, 128, 19968, 32256, 18688, 10368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 317, 0, 2, 0, 7, 0, 0, 3998, 1699, 0, 2716, 0, 0, 4022, 2, 0, 0, 1488, 4, 4081, 2200, 4094, 0, 0, 11, 0, 0, 0, 615, 0, 388, 100, 16, 0, 45, 3660, 3506, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 29, 0, 85, 0, 0, 4092, 11, 4095, 4096, 29, 0, 3155, 247, 4089, 1099, 1, 0, 0, 997, 0, 0, 1724, 2591, 0, 1, 0, 0, 0, 0, 82, 0, 3013, 3971, 4045, 253, 5, 4083, 173, 0, 0, 3819, 3997, 43, 0, 0, 4095, 1, 160, 1015, 0, 4085, 116, 29, 4020, 273, 0, 0, 3, 0, 0, 0, 4096, 0, 4059, 0, 1442, 0, 3313, 0, 2, 826, 0, 0, 5, 391, 4042, 4045, 0, 4065, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4017, 5, 25, 0, 12, 9, 9, 560, 4094, 14, 54, 0, 0, 0, 4033, 266, 40, 1213, 3290, 1, 314, 4096, 4074, 4, 0, 61, 0, 0, 39, 115, 536, 2562, 0, 0, 0, 48, 3530, 4051, 16, 0, 4095, 322, 0, 0, 0, 1, 2, 0, 0, 0, 84, 0, 4096, 0, 4065, 3963, 1, 9, 4063, 1, 3, 3963, 0, 0, 8, 1530, 4, 22, 0, 4096, 55, 0, 44, 0, 28, 0, 0, 34, 3350, 3646, 4085, 0, 0, 25, 344, 0, 0, 3692, 4074, 0, 19, 0, 4091, 3058, 3749, 4027, 0, 4096, 0, 0, 4075, 22, 0, 24, 0, 0, 0, 255, 4052, 0, 4026, 0, 0, 1, 374, 0, 0, 293, 0, 0, 0, 9, 3946, 4096, 32, 4034, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[24, 4096, 1587, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 76, 0, 85, 235, 65, 0, 0, 1848, 0, 4054, 0, 4076, 0, 0, 3, 0, 0, 28, 2850, 4089, 96, 5, 0, 1, 4, 4056, 2944, 0, 0, 3155, 438, 3, 0, 0, 1, 0, 0, 0, 48, 6, 2941, 3836, 0, 4096, 4039, 0, 0, 4095, 0, 9, 4087, 0, 4091, 0, 3328, 1, 0, 1017, 4080, 0, 4, 86, 0, 0, 24, 23, 0, 108, 3836, 3943, 4095, 0, 3, 46, 0, 0, 4086, 4029, 47, 3, 8, 4063, 183, 821, 4074, 0, 4096, 389, 0, 4093, 2, 0, 0, 1958, 0, 0, 0, 2886, 0, 4000, 0, 189, 21, 0, 0, 0, 0, 3, 3, 0, 0, 2171, 4089, 11, 4092, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 3758, 0, 5, 0, 0, 0, 389, 4084, 0, 13, 11, 0, 175, 3, 11, 0, 246, 2605, 0, 0, 4096, 4096, 0, 0, 1, 1, 1, 16, 1, 20, 4096, 0, 0, 0, 211, 4096, 2396, 0, 0, 4025, 201, 0, 2230, 0, 0, 0, 3, 0, 0, 0, 3, 4050, 528, 4096, 4096, 0, 0, 4096, 0, 578, 4095, 0, 2, 0, 1436, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 69, 0, 1234, 4079, 4094, 2936, 0, 126, 68, 0, 0, 4096, 4029, 0, 0, 0, 4096, 3808, 4095, 2745, 0, 4057, 1, 0, 4096, 0, 0, 0, 208, 0, 0, 107, 4070, 0, 4073, 0, 6, 0, 73, 0, 0, 2, 0, 0, 0, 1791, 967, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 3373, 8, 0, 523, 0, 2, 188, 0, 4095, 21, 255, 24, 0, 0, 4084, 10, 3, 20, 0, 0, 3954, 3, 3840, 0, 1, 0, 0, 72, 1663, 0, 3024, 2209, 0, 0, 0, 1667, 3976, 3143, 0, 2, 2213, 0, 1440, 0, 17, 4083, 0, 71, 11, 4078, 0, 0, 2885, 0, 3522, 3491, 0, 28, 2197, 365, 0, 159, 0, 1488, 3, 26, 9, 7, 698, 3097, 0, 855, 0, 779, 0, 3374, 0, 1737, 3696, 4096, 3998, 152, 3, 0, 0, 39, 15, 4068, 1647, 4, 0, 0, 4019, 3450, 4042, 3830, 0, 3499, 8, 3, 3166, 320, 274, 4, 10, 0, 3, 1220, 3516, 0, 3965, 0, 3717, 0, 98, 0, 409, 8, 0, 834, 0, 118, 0, 2705, 12, 1331, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 3978, 1578, 28, 0, 0, 0, 0, 208, 4096, 0, 20, 7, 4, 0, 0, 0, 0, 3538, 245, 0, 3735, 16, 4095, 46, 0, 0, 0, 84, 53, 3441, 133, 2704, 6, 0, 0, 0, 4086, 4096, 0, 0, 3470, 2606, 61, 0, 2, 1, 3, 0, 16, 7, 1, 825, 4082, 0, 4010, 4079, 0, 0, 4092, 28, 3798, 490, 0, 4096, 0, 4041, 0, 0, 5, 4041, 0, 1, 28, 0, 0, 0, 112, 53, 0, 4096, 4044, 8, 246, 0, 386, 0, 261, 4096, 4058, 0, 2, 12, 4096, 2201, 2973, 0, 0, 3771, 0, 0, 4094, 0, 8, 0, 0, 0, 0, 836, 3289, 0, 4069, 0, 443, 0, 3643, 0, 0, 105, 14, 0, 0, 0, 31, 4066, 2, 4063, 14] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 723, 0, 339, 0, 0, 101, 0, 4039, 59, 1, 0, 0, 22, 2, 58, 0, 1250, 3375, 247, 3512, 3905, 4096, 0, 0, 0, 148, 0, 0, 22, 0, 0, 0, 0, 0, 1420, 3694, 4096, 140, 0, 3775, 2, 21, 699, 0, 0, 0, 990, 0, 0, 0, 3209, 4096, 223, 4086, 4096, 5, 0, 4096, 400, 139, 0, 8, 0, 37, 3425, 0, 812, 0, 3698, 0, 0, 0, 0, 0, 33, 0, 0, 3821, 3915, 4029, 491, 0, 0, 13, 0, 0, 3509, 4015, 14, 0, 0, 3984, 2505, 4096, 4096, 0, 3196, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 263, 3549, 0, 3925, 0, 0, 0, 1, 0, 1, 0, 0, 3, 0, 121, 4095, 4096, 0, 4063, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[8, 4096, 2400, 0, 10, 0, 0, 0, 3529, 4053, 0, 0, 2, 2, 1425, 54, 3, 0, 1940, 2932, 0, 596, 542, 3992, 0, 0, 1, 2, 1, 49, 793, 539, 1418, 2, 7, 0, 159, 4095, 4077, 2, 25, 3289, 23, 132, 0, 75, 0, 4, 0, 1, 5, 3, 0, 3994, 50, 4077, 3935, 0, 0, 3515, 2, 3825, 493, 0, 4089, 0, 4093, 2, 16, 0, 3919, 0, 27, 0, 30, 0, 67, 0, 0, 753, 4023, 4015, 0, 0, 4087, 0, 0, 62, 4095, 4092, 0, 0, 7, 4056, 0, 4005, 1410, 18, 3165, 117, 0, 4093, 0, 17, 0, 0, 3, 0, 0, 4046, 47, 3999, 17, 267, 3, 3603, 0, 36, 229, 0, 8, 1, 254, 5, 4040, 0, 4079, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 3758, 0, 5, 0, 0, 0, 389, 4084, 0, 13, 11, 0, 175, 3, 3050, 930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 246, 2605, 0, 0, 4096, 4096, 0, 0, 1, 1, 1, 16, 1, 20, 1600, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 211, 4096, 2396, 0, 0, 4025, 201, 0, 2230, 0, 0, 0, 1467, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 4050, 528, 0, 3795, 0, 0, 4096, 0, 578, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1436, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 69, 0, 1234, 4096, 1071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1604, 0, 2936, 0, 126, 68, 0, 0, 4096, 4029, 0, 0, 0, 0, 3808, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2745, 0, 4057, 1, 0, 4096, 0, 0, 0, 208, 0, 0, 107, 4070, 0, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 73, 0, 0, 2, 0, 0, 0, 1791, 967, 4096, 0, 4096, 0, 3023, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16514, 13702, 19023, 17148, 11996, 20762, 16827, 15100] avg=16384.0000 max=20762 max_rank=5 min=11996 min_rank=4 imbalance_ratio=1.267212 max_load_reduction=110310 max_load_reduction_pct=0.841599 moved_to_echo=64685 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4017, 5, 25, 0, 12, 9, 9, 560, 4094, 14, 54, 0, 0, 0, 4033, 3026, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 40, 1213, 3290, 1, 314, 4096, 4074, 4, 0, 61, 0, 0, 39, 115, 536, 1581, 901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2562, 0, 0, 0, 48, 3530, 4051, 16, 0, 4095, 322, 0, 0, 0, 1, 2, 1311, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 4096, 0, 0, 3672, 1, 9, 4063, 1, 3, 3963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1530, 4, 22, 0, 4096, 55, 0, 44, 0, 28, 0, 0, 34, 3350, 4091, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1434, 0, 0, 0, 25, 344, 0, 0, 3692, 4074, 0, 19, 0, 0, 3058, 3749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 4096, 0, 0, 4075, 22, 0, 24, 0, 0, 0, 255, 4052, 0, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 374, 0, 0, 293, 0, 0, 0, 9, 3946, 4096, 32, 4034, 0, 3017, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16778, 16531, 16229, 15892, 14330, 16395, 18076, 16841] avg=16384.0000 max=18076 max_rank=6 min=14330 min_rank=4 imbalance_ratio=1.103271 max_load_reduction=112996 max_load_reduction_pct=0.862091 moved_to_echo=65642 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 317, 0, 2, 0, 7, 0, 0, 3998, 1699, 0, 2716, 0, 0, 4022, 3050, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1488, 4, 4081, 2200, 4094, 0, 0, 11, 0, 0, 0, 615, 0, 1594, 981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 100, 16, 0, 45, 3660, 3506, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 1428, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 85, 0, 0, 4092, 11, 0, 3795, 29, 0, 3155, 247, 4089, 1099, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 997, 0, 0, 1724, 2591, 0, 1, 0, 0, 0, 0, 82, 0, 3013, 4095, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1562, 0, 253, 5, 4083, 173, 0, 0, 3819, 3997, 43, 0, 0, 0, 1, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 0, 4085, 116, 29, 4020, 273, 0, 0, 3, 0, 0, 0, 4096, 0, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1442, 0, 3313, 0, 2, 826, 0, 0, 5, 391, 4042, 4045, 0, 4065, 0, 2987, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20834, 15070, 13541, 16632, 13561, 14096, 15175, 22163] avg=16384.0000 max=22163 max_rank=7 min=13541 min_rank=2 imbalance_ratio=1.352722 max_load_reduction=108909 max_load_reduction_pct=0.830910 moved_to_echo=64995 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3978, 1578, 28, 0, 0, 0, 0, 208, 4096, 0, 20, 7, 4, 0, 0, 2986, 930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3538, 245, 0, 3735, 16, 4095, 46, 0, 0, 0, 84, 53, 3441, 133, 1598, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2704, 6, 0, 0, 0, 4086, 4096, 0, 0, 3470, 2606, 61, 0, 2, 1, 3, 1473, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 7, 1, 825, 4082, 0, 0, 3779, 0, 0, 4092, 28, 3798, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4041, 0, 0, 5, 4041, 0, 1, 28, 0, 0, 0, 112, 53, 0, 4096, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1611, 0, 8, 246, 0, 386, 0, 261, 4096, 4058, 0, 2, 12, 0, 2201, 2973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3771, 0, 0, 4094, 0, 8, 0, 0, 0, 0, 836, 3289, 0, 1541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 0, 3643, 0, 0, 105, 14, 0, 0, 0, 31, 4066, 2, 4063, 14, 2986, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13835, 17996, 18808, 17118, 17531, 15854, 13539, 16391] avg=16384.0000 max=18808 max_rank=2 min=13539 min_rank=6 imbalance_ratio=1.147949 max_load_reduction=112264 max_load_reduction_pct=0.856506 moved_to_echo=63315 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[24, 4096, 1587, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 76, 0, 85, 235, 3050, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 1848, 0, 4054, 0, 4076, 0, 0, 3, 0, 0, 28, 2850, 4089, 1571, 948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 5, 0, 1, 4, 4056, 2944, 0, 0, 3155, 438, 3, 0, 0, 1, 0, 1380, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 6, 2941, 3836, 0, 0, 3742, 0, 0, 4095, 0, 9, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3328, 1, 0, 1017, 4080, 0, 4, 86, 0, 0, 24, 23, 0, 108, 4063, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1508, 0, 4095, 0, 3, 46, 0, 0, 4086, 4029, 47, 3, 8, 0, 183, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 4096, 389, 0, 4093, 2, 0, 0, 1958, 0, 0, 0, 2886, 0, 1515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 21, 0, 0, 0, 0, 3, 3, 0, 0, 2171, 4089, 11, 4092, 4, 2912, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14157, 19532, 12380, 18764, 17856, 14829, 19013, 14541] avg=16384.0000 max=19532 max_rank=1 min=12380 min_rank=2 imbalance_ratio=1.192139 max_load_reduction=111540 max_load_reduction_pct=0.850983 moved_to_echo=66239 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 723, 0, 339, 0, 0, 101, 0, 4039, 59, 1, 0, 0, 22, 2, 3042, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 1250, 3375, 247, 3512, 3905, 4096, 0, 0, 0, 148, 0, 0, 22, 0, 1541, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 3694, 4096, 140, 0, 3775, 2, 21, 699, 0, 0, 0, 1408, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 990, 0, 0, 0, 3209, 4096, 223, 0, 3795, 5, 0, 4096, 400, 139, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 3425, 0, 812, 0, 3698, 0, 0, 0, 0, 0, 33, 0, 0, 3821, 3984, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1540, 0, 491, 0, 0, 13, 0, 0, 3509, 4015, 14, 0, 0, 0, 2505, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3196, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 263, 3549, 0, 1487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 3, 0, 121, 4095, 4096, 0, 4063, 3, 2976, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13321, 19121, 15556, 16961, 16863, 16183, 16664, 16403] avg=16384.0000 max=19121 max_rank=1 min=13321 min_rank=0 imbalance_ratio=1.167053 max_load_reduction=111951 max_load_reduction_pct=0.854118 moved_to_echo=66113 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3373, 8, 0, 523, 0, 2, 188, 0, 4095, 21, 255, 24, 0, 0, 4084, 2623, 909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3, 20, 0, 0, 3954, 3, 3840, 0, 1, 0, 0, 72, 1663, 0, 3024, 1561, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2209, 0, 0, 0, 1667, 3976, 3143, 0, 2, 2213, 0, 1440, 0, 17, 4083, 0, 1479, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 11, 4078, 0, 0, 2885, 0, 0, 3229, 0, 28, 2197, 365, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1488, 3, 26, 9, 7, 698, 3097, 0, 855, 0, 779, 0, 3374, 0, 1737, 3696, 4019, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1603, 0, 152, 3, 0, 0, 39, 15, 4068, 1647, 4, 0, 0, 0, 3450, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3830, 0, 3499, 8, 3, 3166, 320, 274, 4, 10, 0, 3, 1220, 3516, 0, 1495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3717, 0, 98, 0, 409, 8, 0, 834, 0, 118, 0, 2705, 12, 1331, 0, 2952, 899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16105, 15165, 20491, 13023, 20834, 15023, 17348, 13083] avg=16384.0000 max=20834 max_rank=4 min=13023 min_rank=3 imbalance_ratio=1.271606 max_load_reduction=110238 max_load_reduction_pct=0.841049 moved_to_echo=66288 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[8, 4096, 2400, 0, 10, 0, 0, 0, 3529, 4053, 0, 0, 2, 2, 1425, 54, 3035, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1940, 2932, 0, 596, 542, 3992, 0, 0, 1, 2, 1, 49, 793, 539, 1571, 994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1418, 2, 7, 0, 159, 4095, 4077, 2, 25, 3289, 23, 132, 0, 75, 0, 4, 1447, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 3, 0, 3994, 50, 0, 3646, 0, 0, 3515, 2, 3825, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4093, 2, 16, 0, 3919, 0, 27, 0, 30, 0, 67, 0, 0, 753, 4056, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1582, 0, 0, 0, 4087, 0, 0, 62, 4095, 4092, 0, 0, 7, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1410, 18, 3165, 117, 0, 4093, 0, 17, 0, 0, 3, 0, 0, 4046, 47, 1514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 267, 3, 3603, 0, 36, 229, 0, 8, 1, 254, 5, 4040, 0, 4079, 0, 2966, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19528, 13955, 15044, 15534, 18101, 17930, 14430, 16550] avg=16384.0000 max=19528 max_rank=0 min=13955 min_rank=1 imbalance_ratio=1.191895 max_load_reduction=111544 max_load_reduction_pct=0.851013 moved_to_echo=67011 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[21248, 128, 30080, 640, 128, 31744, 640, 384, 128, 2304, 128, 128, 2688, 29568, 128, 12160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[512, 128, 8320, 15872, 256, 20352, 14976, 32384, 128, 128, 128, 256, 256, 1920, 7936, 8448, 12672, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 31872, 10496, 128, 896, 128, 128, 384, 4736, 32640, 1920, 384, 2944, 128, 1792, 12544, 23936, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[13824, 128, 18944, 128, 896, 3456, 29696, 128, 896, 256, 896, 128, 3584, 384, 1920, 16000, 32512, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1152, 128, 4224, 128, 7040, 31232, 896, 0, 29568, 128, 128, 29312, 1152, 12544, 14464, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[13568, 128, 128, 128, 3584, 31232, 28416, 256, 128, 28160, 3712, 1664, 2944, 128, 4096, 128, 11520, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[12544, 0, 7936, 256, 8448, 1152, 128, 384, 31488, 29952, 128, 128, 128, 0, 15232, 24064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 6144, 128, 11136, 0, 512, 1536, 128, 896, 128, 2688, 15360, 31360, 128, 29824, 128, 23936, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[348, 3207, 85, 0, 4, 4095, 0, 0, 2588, 0, 1, 2418, 4096, 3899, 36, 4080, 12, 0, 24, 11, 0, 0, 531, 1, 0, 0, 3756, 0, 0, 0, 0, 0, 4096, 1, 1788, 81, 11, 0, 0, 5, 4078, 0, 0, 0, 1260, 3, 0, 2, 3381, 0, 19, 39, 3700, 0, 3391, 337, 3738, 0, 3480, 2169, 0, 4076, 845, 0, 14, 697, 0, 0, 2783, 4005, 3237, 3875, 4096, 0, 1, 0, 0, 0, 0, 2, 0, 0, 3733, 0, 2050, 4096, 4093, 0, 249, 1307, 21, 255, 0, 0, 1193, 23, 0, 4087, 0, 3441, 0, 0, 194, 26, 19, 0, 0, 0, 2, 0, 0, 0, 0, 116, 0, 4056, 28, 4033, 224, 4053, 5, 3849, 4, 790, 1089, 3634, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4, 4095, 4, 0, 0, 3377, 1, 0, 15, 1, 0, 4045, 4093, 4095, 1660, 109, 1674, 9, 90, 0, 0, 0, 22, 17, 0, 0, 3882, 0, 0, 0, 0, 0, 3997, 0, 4094, 0, 68, 0, 0, 0, 3783, 2701, 0, 96, 0, 1, 0, 1, 1, 0, 0, 3803, 4091, 0, 4082, 19, 4079, 3758, 4015, 0, 5, 3993, 1477, 0, 10, 0, 0, 0, 2941, 3588, 4095, 36, 4002, 0, 0, 107, 0, 0, 993, 17, 16, 0, 4096, 1, 0, 4093, 4028, 0, 0, 4018, 0, 0, 0, 3, 0, 0, 0, 4090, 5, 4010, 0, 0, 0, 2, 18, 95, 0, 843, 2517, 9, 48, 1, 90, 0, 4087, 16, 0, 2040, 8, 4094, 4, 4088, 0, 0, 993, 258, 455, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 4086, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 4088, 3657, 4096, 41, 287, 983, 16, 120, 0, 0, 1, 151, 0, 0, 1, 4005, 162, 0, 13, 3, 672, 4068, 1, 3978, 0, 13, 0, 0, 26, 4096, 2642, 3587, 0, 47, 4, 0, 10, 4070, 92, 0, 2724, 4050, 5, 4065, 2, 4096, 84, 1, 1270, 2, 4021, 3611, 21, 192, 0, 0, 0, 4049, 4021, 4088, 3, 4039, 2, 13, 0, 3, 12, 81, 0, 0, 0, 4093, 2750, 0, 3993, 806, 0, 0, 3937, 0, 0, 0, 8, 22, 1, 0, 4073, 0, 4061, 0, 2, 0, 13, 0, 0, 0, 0, 4, 1, 0, 0, 3143, 0, 106, 443, 123, 4096, 0, 4091, 964, 4096, 0, 0, 37, 718, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 4087, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 4057, 4096, 70, 2933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2805, 3870, 0, 4076, 0, 852, 0, 0, 0, 4096, 4096, 351, 1, 0, 0, 0, 0, 3099, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4025, 0, 153, 0, 3059, 4069, 0, 21, 29, 0, 0, 3194, 4087, 4095, 2027, 4096, 0, 0, 0, 0, 0, 923, 0, 68, 1, 4096, 4088, 24, 3473, 4095, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 7, 1606, 0, 43, 2, 28, 156, 0, 4096, 0, 4096, 0, 0, 7, 44, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[2, 4095, 0, 0, 2, 521, 0, 1, 66, 3, 0, 3809, 4085, 4096, 4096, 77, 18, 0, 0, 8, 0, 0, 710, 0, 0, 0, 4089, 0, 6, 0, 0, 0, 4096, 1, 4056, 0, 2659, 0, 0, 0, 113, 1465, 4052, 4096, 0, 2, 0, 0, 4048, 3, 5, 4091, 4051, 6, 4095, 2, 4096, 1, 3837, 3590, 0, 2581, 3965, 0, 1509, 4, 51, 0, 169, 812, 4085, 0, 4096, 0, 42, 13, 0, 0, 0, 0, 0, 10, 4087, 8, 0, 3537, 3831, 8, 0, 4093, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 3768, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 262, 3979, 0, 7, 20, 64, 2002, 9, 4092, 0, 4079, 31, 0, 464, 1246, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 0, 17, 0, 0, 0, 0, 0, 4059, 4094, 4089, 0, 495, 2, 0, 0, 0, 0, 0, 661, 0, 454, 0, 4084, 13, 0, 0, 0, 0, 3962, 0, 3876, 0, 2, 1, 0, 32, 4082, 4013, 2801, 34, 2, 0, 1, 0, 2688, 11, 0, 4096, 3984, 0, 4096, 0, 4096, 26, 1171, 1515, 0, 4096, 4072, 0, 3944, 0, 0, 101, 3968, 3, 4066, 0, 4094, 0, 0, 10, 0, 0, 0, 0, 0, 155, 4074, 487, 7, 4075, 4087, 0, 23, 4092, 0, 59, 30, 0, 0, 0, 4, 3932, 29, 3891, 1, 0, 0, 0, 0, 0, 0, 53, 3949, 0, 6, 456, 699, 14, 3931, 26, 21, 806, 88, 4073, 0, 4011, 8, 0, 23, 1014, 41, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[14, 4002, 0, 0, 1, 0, 12, 0, 0, 0, 45, 4000, 4079, 4084, 558, 0, 3000, 0, 71, 0, 0, 0, 0, 18, 0, 0, 3506, 0, 0, 0, 48, 0, 3999, 62, 3900, 106, 19, 0, 0, 0, 4096, 4096, 2167, 0, 0, 0, 0, 0, 11, 0, 0, 648, 4096, 53, 4096, 122, 4081, 4096, 134, 3419, 30, 4053, 4083, 0, 0, 31, 6, 0, 2148, 4096, 4096, 1136, 3998, 0, 40, 0, 0, 0, 0, 0, 0, 393, 3975, 927, 0, 4096, 1341, 0, 122, 3909, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 4061, 0, 0, 235, 14, 0, 223, 20, 0, 4087, 0, 1111, 40, 3899, 140, 7, 0, 0, 3832, 0, 4064, 1, 4027, 0, 81, 0, 0, 2, 78] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 4090, 0, 47, 3, 0, 8, 1, 6, 0, 0, 4033, 4091, 4041, 3475, 2650, 116, 0, 50, 0, 0, 0, 73, 24, 7, 0, 4017, 0, 0, 158, 0, 4, 3841, 0, 4085, 0, 0, 1, 0, 3, 3993, 3463, 4036, 3169, 11, 0, 0, 2, 4075, 0, 0, 3313, 4084, 1, 4004, 0, 4036, 911, 0, 114, 0, 4092, 814, 0, 0, 25, 0, 3, 4053, 1555, 3888, 12, 4076, 34, 0, 0, 1, 0, 4002, 30, 0, 0, 4096, 190, 99, 4057, 3861, 7, 0, 4024, 11, 0, 0, 0, 0, 0, 48, 4095, 0, 3938, 3, 51, 0, 0, 271, 6, 0, 2027, 0, 14, 207, 65, 0, 0, 22, 0, 0, 2456, 333, 4090, 357, 4088, 1, 0, 0, 0, 12, 22] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4087, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 4057, 82, 70, 2933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2805, 4096, 4096, 2875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2513, 0, 4076, 0, 852, 0, 0, 0, 4096, 4096, 351, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3099, 0, 0, 4096, 0, 0, 0, 0, 0, 4025, 0, 153, 0, 2598, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 29, 0, 0, 3194, 4087, 4095, 2027, 4096, 0, 0, 0, 0, 0, 923, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 1, 3904, 4088, 24, 3473, 4095, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 7, 4014, 1221, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1606, 0, 43, 2, 28, 156, 0, 4096, 0, 4096, 0, 0, 7, 44, 0, 0, 461, 273, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15332, 17968, 15985, 18040, 18649, 19750, 14521, 10827] avg=16384.0000 max=19750 max_rank=5 min=10827 min_rank=7 imbalance_ratio=1.205444 max_load_reduction=111322 max_load_reduction_pct=0.849319 moved_to_echo=63747 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4086, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 4088, 3657, 82, 41, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 983, 16, 120, 0, 0, 1, 151, 0, 0, 1, 4005, 162, 0, 13, 3, 672, 4096, 4050, 2853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2642, 1, 3978, 0, 13, 0, 0, 26, 4096, 2642, 3587, 0, 47, 4, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 92, 0, 2724, 0, 5, 0, 2, 0, 84, 1, 1270, 2, 3415, 3611, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 4049, 4021, 4088, 3, 4039, 2, 13, 0, 3, 12, 81, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 2750, 0, 3993, 806, 0, 0, 3937, 0, 0, 0, 8, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 4061, 0, 2, 0, 13, 0, 0, 0, 0, 4, 1, 0, 0, 4014, 1212, 1139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3143, 0, 106, 443, 123, 4096, 0, 4091, 964, 4096, 0, 0, 37, 718, 0, 0, 606, 287, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16261, 17126, 17046, 15297, 16680, 15418, 14519, 18725] avg=16384.0000 max=18725 max_rank=7 min=14519 min_rank=6 imbalance_ratio=1.142883 max_load_reduction=112347 max_load_reduction_pct=0.857140 moved_to_echo=65342 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[348, 3207, 85, 0, 4, 4095, 0, 0, 2588, 0, 1, 2418, 4096, 75, 36, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 24, 11, 0, 0, 531, 1, 0, 0, 3756, 0, 0, 0, 0, 0, 3738, 3700, 2381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2652, 1, 1788, 81, 11, 0, 0, 5, 4078, 0, 0, 0, 1260, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3381, 0, 19, 39, 0, 0, 0, 337, 0, 0, 3480, 2169, 0, 3458, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 697, 0, 0, 2783, 4005, 3237, 3875, 4096, 0, 1, 0, 0, 0, 0, 2, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3545, 0, 2050, 4096, 4093, 0, 249, 1307, 21, 255, 0, 0, 1193, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 3441, 0, 0, 194, 26, 19, 0, 0, 0, 2, 0, 0, 0, 3824, 1010, 1151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 4056, 28, 4033, 224, 4053, 5, 3849, 4, 790, 1089, 3634, 0, 0, 618, 293, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21033, 14154, 9881, 13728, 18878, 16832, 13754, 22812] avg=16384.0000 max=22812 max_rank=7 min=9881 min_rank=2 imbalance_ratio=1.392334 max_load_reduction=108260 max_load_reduction_pct=0.825958 moved_to_echo=72276 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 0, 17, 0, 0, 0, 0, 0, 4059, 4094, 82, 0, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 661, 0, 454, 0, 4084, 13, 0, 0, 0, 0, 4096, 3984, 2875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2573, 0, 3876, 0, 2, 1, 0, 32, 4082, 4013, 2801, 34, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2688, 11, 0, 4096, 0, 0, 0, 0, 0, 26, 1171, 1515, 0, 3478, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3944, 0, 0, 101, 3968, 3, 4066, 0, 4094, 0, 0, 10, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 3883, 487, 7, 4075, 4087, 0, 23, 4092, 0, 59, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3932, 29, 3891, 1, 0, 0, 0, 0, 0, 0, 53, 3949, 0, 6, 456, 4007, 1221, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 699, 14, 3931, 26, 21, 806, 88, 4073, 0, 4011, 8, 0, 23, 1014, 41, 0, 618, 279, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12843, 16169, 17417, 17057, 16362, 16898, 18659, 15667] avg=16384.0000 max=18659 max_rank=6 min=12843 min_rank=0 imbalance_ratio=1.138855 max_load_reduction=112413 max_load_reduction_pct=0.857643 moved_to_echo=67586 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[14, 4002, 0, 0, 1, 0, 12, 0, 0, 0, 45, 4000, 4079, 82, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3000, 0, 71, 0, 0, 0, 0, 18, 0, 0, 3506, 0, 0, 0, 48, 0, 4081, 4096, 2874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2597, 62, 3900, 106, 19, 0, 0, 0, 4096, 4096, 2167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 648, 0, 53, 0, 122, 0, 4096, 134, 3419, 30, 3442, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 6, 0, 2148, 4096, 4096, 1136, 3998, 0, 40, 0, 0, 0, 0, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 3788, 927, 0, 4096, 1341, 0, 122, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 4061, 0, 0, 235, 14, 0, 223, 20, 0, 4087, 0, 1111, 40, 4002, 1222, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3899, 140, 7, 0, 0, 3832, 0, 4064, 1, 4027, 0, 81, 0, 0, 2, 78, 611, 282, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12793, 17694, 17043, 16038, 15723, 14576, 20166, 17039] avg=16384.0000 max=20166 max_rank=6 min=12793 min_rank=0 imbalance_ratio=1.230835 max_load_reduction=110906 max_load_reduction_pct=0.846146 moved_to_echo=67504 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 4095, 4, 0, 0, 3377, 1, 0, 15, 1, 0, 4045, 4093, 82, 1660, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1674, 9, 90, 0, 0, 0, 22, 17, 0, 0, 3882, 0, 0, 0, 0, 0, 4079, 4091, 2866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2595, 0, 4094, 0, 68, 0, 0, 0, 3783, 2701, 0, 96, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3803, 0, 0, 0, 19, 0, 3758, 4015, 0, 5, 3391, 1477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 2941, 3588, 4095, 36, 4002, 0, 0, 107, 0, 0, 993, 17, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3904, 1, 0, 4093, 4028, 0, 0, 4018, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 5, 4010, 0, 0, 0, 2, 18, 95, 0, 843, 2517, 9, 48, 1, 4013, 1216, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 4087, 16, 0, 2040, 8, 4094, 4, 4088, 0, 0, 993, 258, 455, 0, 602, 282, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17486, 16730, 13339, 16469, 15966, 16063, 17987, 17032] avg=16384.0000 max=17987 max_rank=6 min=13339 min_rank=2 imbalance_ratio=1.097839 max_load_reduction=113085 max_load_reduction_pct=0.862770 moved_to_echo=67048 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 4095, 0, 0, 2, 521, 0, 1, 66, 3, 0, 3809, 4085, 82, 4096, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 8, 0, 0, 710, 0, 0, 0, 4089, 0, 6, 0, 0, 0, 4096, 4051, 2874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2660, 1, 4056, 0, 2659, 0, 0, 0, 113, 1465, 4052, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 3, 5, 4091, 0, 6, 0, 2, 0, 1, 3837, 3590, 0, 2192, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1509, 4, 51, 0, 169, 812, 4085, 0, 4096, 0, 42, 13, 0, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3895, 8, 0, 3537, 3831, 8, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 3768, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 262, 4014, 1221, 1147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 7, 20, 64, 2002, 9, 4092, 0, 4079, 31, 0, 464, 1246, 0, 0, 389, 289, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16839, 15852, 19104, 21740, 10958, 15382, 14511, 16686] avg=16384.0000 max=21740 max_rank=3 min=10958 min_rank=4 imbalance_ratio=1.326904 max_load_reduction=109332 max_load_reduction_pct=0.834137 moved_to_echo=57537 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 0, 47, 3, 0, 8, 1, 6, 0, 0, 4033, 4091, 81, 3475, 2650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 50, 0, 0, 0, 73, 24, 7, 0, 4017, 0, 0, 158, 0, 4, 4036, 4084, 2810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2494, 0, 4085, 0, 0, 1, 0, 3, 3993, 3463, 4036, 3169, 11, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 3313, 0, 1, 0, 0, 0, 911, 0, 114, 0, 3475, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 3, 4053, 1555, 3888, 12, 4076, 34, 0, 0, 1, 0, 4002, 30, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 190, 99, 4057, 3861, 7, 0, 4024, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 4095, 0, 3938, 3, 51, 0, 0, 271, 6, 0, 2027, 0, 14, 207, 65, 3960, 1194, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 2456, 333, 4090, 357, 4088, 1, 0, 0, 0, 12, 22, 617, 271, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18485, 15379, 21257, 12703, 17856, 16153, 16955, 12284] avg=16384.0000 max=21257 max_rank=2 min=12284 min_rank=7 imbalance_ratio=1.297424 max_load_reduction=109815 max_load_reduction_pct=0.837822 moved_to_echo=63248 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32512, 128, 31360, 128, 128, 512, 128, 384, 384, 128, 2944, 10624, 128, 1408, 896, 31872, 9600, 8960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[384, 31872, 128, 128, 128, 12160, 128, 128, 2688, 128, 128, 30592, 32256, 768, 9984, 10752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[5888, 128, 384, 128, 0, 128, 2176, 128, 512, 128, 31488, 256, 128, 256, 128, 3584, 32384, 32256, 22528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[21376, 128, 128, 22912, 0, 128, 0, 512, 0, 12928, 12672, 12288, 128, 25472, 23040, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[20736, 128, 29952, 256, 3712, 128, 0, 128, 28416, 22528, 17024, 7424, 1408, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[13440, 384, 8320, 4608, 384, 19456, 768, 32768, 1408, 32384, 128, 896, 2688, 7040, 512, 128, 4608, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 640, 30848, 8576, 2304, 31488, 26240, 128, 512, 29568, 128, 384, 128, 128, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[5760, 896, 128, 128, 23424, 22272, 31744, 7168, 32512, 128, 128, 256, 128, 128, 6016, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 10, 1142, 3965, 4096, 2120, 0, 19, 8, 0, 502, 0, 0, 0, 73, 3420, 2858, 292, 0, 4077, 1, 0, 1530, 3999, 25, 117, 1286, 3689, 0, 0, 0, 0, 0, 113, 0, 3775, 93, 2840, 64, 282, 3994, 0, 337, 0, 0, 4014, 1748, 0, 0, 566, 0, 0, 4014, 1, 0, 8, 0, 0, 0, 0, 0, 325, 2, 0, 50, 3311, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 136, 0, 4096, 0, 0, 4040, 0, 4087, 3817, 0, 4096, 543, 729, 0, 1958, 0, 156, 0, 778, 0, 3777, 0, 2524, 0, 4000, 0, 34, 0, 0, 4096, 0, 0, 4096, 716, 0, 4096, 0, 3863, 3527, 0, 4096, 4063, 0, 796, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 0, 0, 17, 3935, 4096, 3764, 0, 474, 1, 84, 0, 2531, 0, 0, 0, 2659, 3899, 7, 41, 3884, 0, 1, 1804, 0, 33, 48, 0, 3780, 0, 1118, 10, 2, 2, 0, 0, 2480, 9, 3926, 0, 0, 4084, 0, 0, 3958, 0, 4090, 211, 0, 0, 104, 0, 18, 4067, 22, 0, 22, 0, 0, 6, 0, 0, 1, 0, 62, 1, 4092, 60, 3, 1, 17, 4095, 11, 0, 0, 0, 4089, 0, 4090, 0, 5, 0, 171, 4083, 0, 0, 4008, 8, 4096, 4089, 148, 0, 0, 0, 0, 0, 3089, 0, 4096, 33, 4096, 0, 3543, 5, 0, 0, 0, 4062, 0, 5, 4096, 21, 0, 4065, 0, 1973, 4034, 4, 4095, 4077, 0, 0, 1420, 36, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[19, 4, 27, 0, 2, 0, 1683, 4096, 4074, 4072, 0, 4, 2, 733, 0, 4078, 75, 3, 0, 3524, 48, 48, 0, 3105, 0, 7, 0, 0, 0, 1065, 0, 4078, 0, 3718, 25, 88, 1, 0, 0, 4080, 0, 283, 322, 1, 4072, 0, 42, 1009, 848, 4091, 88, 0, 0, 4055, 0, 0, 4086, 0, 1, 0, 0, 15, 0, 0, 1, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4086, 0, 3548, 613, 1297, 0, 27, 4068, 1, 627, 4087, 26, 4076, 4031, 4017, 0, 69, 0, 0, 0, 1627, 42, 4046, 0, 4008, 0, 4063, 0, 0, 0, 33, 0, 0, 58, 3904, 6, 0, 174, 1, 4034, 4004, 0, 3968, 4061, 0, 0, 692, 10, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4, 24, 0, 3, 0, 4096, 4089, 8, 0, 9, 0, 754, 0, 2038, 11, 0, 0, 4094, 3771, 0, 0, 3612, 0, 0, 3948, 2091, 0, 2463, 2, 4096, 0, 1, 0, 0, 4, 0, 0, 4095, 0, 4096, 113, 0, 4096, 0, 0, 1, 0, 3810, 302, 0, 0, 15, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4091, 0, 0, 0, 66, 4096, 0, 0, 0, 0, 4096, 0, 5, 12, 4086, 0, 0, 4096, 0, 2779, 3911, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 350, 0, 4096, 0, 4095, 0, 4096, 0, 192, 0, 0, 4094, 0, 0, 4081, 0, 0, 1, 0, 4075, 4095, 0, 4096, 4087, 0, 2535, 0, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[86, 40, 290, 0, 0, 0, 557, 4096, 4089, 2867, 0, 2680, 45, 0, 2, 4090, 0, 0, 0, 1523, 3506, 0, 117, 3887, 0, 0, 455, 0, 78, 1011, 66, 4068, 3, 2260, 0, 0, 0, 0, 0, 73, 10, 4071, 693, 10, 4086, 0, 21, 1099, 0, 557, 0, 49, 26, 4032, 0, 0, 4075, 0, 0, 2, 0, 7, 1, 0, 0, 255, 0, 444, 0, 3902, 8, 25, 2, 0, 4087, 0, 11, 69, 0, 4096, 31, 3346, 95, 3796, 0, 0, 4074, 0, 109, 4014, 0, 4045, 2417, 5, 0, 0, 133, 0, 0, 4053, 0, 4072, 3, 4080, 0, 4094, 2, 0, 1, 24, 4092, 0, 0, 3458, 280, 0, 38, 0, 3898, 4079, 24, 4051, 3921, 20, 726, 559, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 391, 0, 3, 4035, 3772, 2841, 0, 968, 0, 1690, 37, 0, 935, 3, 1, 19, 10, 0, 0, 4096, 0, 1153, 4037, 348, 1, 8, 1, 4059, 0, 4096, 0, 5, 0, 145, 1, 0, 0, 4090, 83, 0, 4092, 239, 892, 3, 4017, 1249, 2015, 32, 0, 0, 0, 0, 3896, 0, 0, 3, 761, 0, 0, 2, 0, 0, 0, 0, 4, 2728, 0, 4, 3474, 3533, 8, 2363, 39, 8, 2206, 4091, 0, 18, 0, 4095, 202, 2778, 4096, 0, 1635, 4096, 0, 4065, 4096, 4095, 0, 150, 0, 1, 0, 3428, 0, 3934, 0, 4096, 0, 531, 0, 0, 0, 0, 92, 4, 0, 1280, 0, 0, 516, 0, 3651, 3213, 0, 3980, 4033, 2, 0, 487, 8, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 1, 2, 5, 0, 3600, 4095, 4091, 798, 0, 328, 8, 1, 0, 1034, 0, 0, 153, 0, 4096, 37, 0, 4096, 0, 1, 4, 15, 0, 2812, 611, 2389, 0, 931, 3, 688, 0, 8, 0, 0, 2, 24, 0, 0, 4094, 0, 38, 726, 0, 4072, 1307, 0, 0, 18, 4, 0, 4088, 11, 7, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3854, 0, 0, 0, 3961, 4082, 37, 0, 0, 0, 4095, 0, 4096, 0, 1493, 0, 5, 4092, 4, 3737, 4095, 0, 4060, 4055, 117, 0, 0, 1, 0, 0, 4066, 0, 4095, 69, 4088, 0, 4096, 0, 0, 8, 0, 4096, 0, 0, 2961, 0, 0, 2622, 0, 2814, 3995, 0, 4085, 3988, 0, 4089, 17, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1, 77, 1, 0, 23, 0, 23, 4060, 4094, 2782, 0, 1, 0, 0, 0, 1868, 0, 0, 7, 4085, 0, 0, 0, 4085, 0, 5, 2162, 4064, 0, 0, 0, 4075, 0, 0, 9, 0, 0, 2, 6, 4083, 0, 3976, 4, 16, 4082, 10, 0, 19, 1, 3529, 1997, 7, 90, 2865, 0, 43, 4061, 0, 0, 0, 0, 0, 103, 0, 0, 0, 9, 113, 0, 3422, 117, 21, 0, 888, 3883, 0, 0, 0, 99, 4096, 95, 3, 177, 4037, 2, 0, 4096, 0, 3996, 4090, 4, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 12, 4087, 0, 3341, 5, 4086, 0, 49, 214, 0, 1381, 0, 5, 3796, 0, 1, 4047, 0, 4072, 4096, 16, 3999, 4096, 6, 34, 84, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 10, 1142, 3965, 4096, 2120, 0, 19, 8, 0, 502, 0, 1275, 604, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 3420, 2858, 292, 0, 3473, 1, 0, 1530, 3999, 25, 117, 1286, 3689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 3775, 93, 2840, 64, 282, 3994, 0, 337, 0, 2172, 2017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 1748, 0, 0, 566, 0, 0, 4014, 1, 0, 8, 0, 0, 0, 0, 4040, 1924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 2, 0, 50, 3311, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 1966, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 4096, 0, 0, 0, 0, 4087, 3817, 0, 113, 543, 729, 0, 1958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 778, 0, 3743, 0, 2524, 0, 4000, 0, 34, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 716, 0, 4096, 0, 3863, 3527, 0, 0, 1980, 0, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13775, 20763, 15687, 16315, 14648, 15479, 15331, 19074] avg=16384.0000 max=20763 max_rank=1 min=13775 min_rank=0 imbalance_ratio=1.267273 max_load_reduction=110309 max_load_reduction_pct=0.841591 moved_to_echo=64532 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 24, 0, 3, 0, 4096, 4089, 8, 0, 9, 0, 754, 0, 2038, 1283, 535, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4094, 3771, 0, 0, 3077, 0, 0, 3948, 2091, 0, 2463, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 4095, 0, 4096, 113, 0, 4096, 0, 0, 1, 2173, 2017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 302, 0, 0, 15, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4091, 0, 0, 0, 66, 4096, 0, 0, 0, 0, 4096, 1966, 812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 12, 4086, 0, 0, 0, 0, 2779, 3911, 0, 113, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 4059, 0, 4095, 0, 4096, 0, 192, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 1, 0, 4075, 4095, 0, 0, 1992, 0, 2535, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12880, 23553, 16596, 14242, 15133, 15000, 16886, 16782] avg=16384.0000 max=23553 max_rank=1 min=12880 min_rank=0 imbalance_ratio=1.437561 max_load_reduction=107519 max_load_reduction_pct=0.820305 moved_to_echo=63801 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[19, 4, 27, 0, 2, 0, 1683, 4096, 4074, 4072, 0, 4, 2, 733, 0, 4078, 1274, 460, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 3, 0, 3524, 48, 48, 0, 2645, 0, 7, 0, 0, 0, 1065, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3718, 25, 88, 1, 0, 0, 4080, 0, 283, 322, 1, 4072, 0, 42, 1009, 2105, 2008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 848, 4091, 88, 0, 0, 4055, 0, 0, 4086, 0, 1, 0, 0, 15, 0, 0, 4068, 1863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4086, 1956, 807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3548, 613, 1297, 0, 27, 0, 1, 627, 4087, 26, 112, 4031, 4017, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1627, 42, 4010, 0, 4008, 0, 4063, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 3904, 6, 0, 174, 1, 4034, 4004, 0, 0, 1980, 0, 0, 692, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20564, 11493, 17754, 19115, 15045, 18455, 13783, 14863] avg=16384.0000 max=20564 max_rank=0 min=11493 min_rank=1 imbalance_ratio=1.255127 max_load_reduction=110508 max_load_reduction_pct=0.843109 moved_to_echo=62146 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 0, 0, 17, 3935, 4096, 3764, 0, 474, 1, 84, 0, 2531, 1279, 575, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2659, 3899, 7, 41, 3309, 0, 1, 1804, 0, 33, 48, 0, 3780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1118, 10, 2, 2, 0, 0, 2480, 9, 3926, 0, 0, 4084, 0, 0, 3958, 2172, 2017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 211, 0, 0, 104, 0, 18, 4067, 22, 0, 22, 0, 0, 6, 0, 4083, 1923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 62, 1, 4092, 60, 3, 1, 17, 4095, 11, 0, 0, 0, 4089, 1966, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 5, 0, 171, 0, 0, 0, 4008, 8, 113, 4089, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3089, 0, 4059, 33, 4096, 0, 3543, 5, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 21, 0, 4065, 0, 1973, 4034, 4, 0, 1988, 0, 0, 1420, 36, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16795, 15581, 19778, 14546, 15208, 12632, 18887, 17645] avg=16384.0000 max=19778 max_rank=2 min=12632 min_rank=5 imbalance_ratio=1.207153 max_load_reduction=111294 max_load_reduction_pct=0.849106 moved_to_echo=64372 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 391, 0, 3, 4035, 3772, 2841, 0, 968, 0, 1690, 37, 0, 1271, 609, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 935, 3, 1, 19, 10, 0, 0, 3487, 0, 1153, 4037, 348, 1, 8, 1, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 145, 1, 0, 0, 4090, 83, 0, 4092, 239, 892, 3, 2111, 2008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 1249, 2015, 32, 0, 0, 0, 0, 3896, 0, 0, 3, 761, 0, 0, 2, 4096, 1869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2728, 0, 4, 3474, 3533, 8, 2363, 39, 8, 2206, 4091, 1955, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4095, 202, 2778, 0, 0, 1635, 4096, 0, 102, 4096, 4095, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3428, 0, 3896, 0, 4096, 0, 531, 0, 0, 0, 0, 92, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1280, 0, 0, 516, 0, 3651, 3213, 0, 0, 1956, 2, 0, 487, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15658, 14062, 17765, 17940, 21219, 21267, 12048, 11113] avg=16384.0000 max=21267 max_rank=5 min=11113 min_rank=7 imbalance_ratio=1.298035 max_load_reduction=109805 max_load_reduction_pct=0.837746 moved_to_echo=65647 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 2, 5, 0, 3600, 4095, 4091, 798, 0, 328, 8, 1, 0, 1034, 1251, 607, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 4096, 37, 0, 3489, 0, 1, 4, 15, 0, 2812, 611, 2389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 931, 3, 688, 0, 8, 0, 0, 2, 24, 0, 0, 4094, 0, 38, 726, 2167, 2000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 1307, 0, 0, 18, 4, 0, 4088, 11, 7, 0, 0, 0, 1, 0, 4092, 1918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3854, 0, 0, 0, 3961, 4082, 37, 0, 0, 0, 4095, 1949, 793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1493, 0, 5, 0, 4, 3737, 4095, 0, 111, 4055, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4066, 0, 4058, 69, 4088, 0, 4096, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2961, 0, 0, 2622, 0, 2814, 3995, 0, 0, 1944, 0, 4089, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15858, 13607, 10681, 15518, 18771, 17713, 20482, 18442] avg=16384.0000 max=20482 max_rank=6 min=10681 min_rank=2 imbalance_ratio=1.250122 max_load_reduction=110590 max_load_reduction_pct=0.843735 moved_to_echo=75408 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[86, 40, 290, 0, 0, 0, 557, 4096, 4089, 2867, 0, 2680, 45, 0, 2, 4090, 1230, 576, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1523, 3506, 0, 117, 3311, 0, 0, 455, 0, 78, 1011, 66, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2260, 0, 0, 0, 0, 0, 73, 10, 4071, 693, 10, 4086, 0, 21, 1099, 2148, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 0, 49, 26, 4032, 0, 0, 4075, 0, 0, 2, 0, 7, 1, 0, 4074, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 444, 0, 3902, 8, 25, 2, 0, 4087, 0, 11, 69, 0, 4096, 1942, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3346, 95, 3796, 0, 0, 0, 0, 109, 4014, 0, 111, 2417, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 4053, 0, 4035, 3, 4080, 0, 4094, 2, 0, 1, 24, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3458, 280, 0, 38, 0, 3898, 4079, 24, 0, 1912, 20, 726, 559, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20685, 14135, 16466, 14726, 15620, 13924, 20517, 14999] avg=16384.0000 max=20685 max_rank=0 min=13924 min_rank=5 imbalance_ratio=1.262512 max_load_reduction=110387 max_load_reduction_pct=0.842186 moved_to_echo=65060 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 77, 1, 0, 23, 0, 23, 4060, 4094, 2782, 0, 1, 0, 0, 0, 1868, 1285, 605, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4085, 0, 0, 0, 3480, 0, 5, 2162, 4064, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2, 6, 4083, 0, 3976, 4, 16, 4082, 10, 0, 19, 2121, 2017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3529, 1997, 7, 90, 2865, 0, 43, 4061, 0, 0, 0, 0, 0, 103, 0, 4096, 1878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 113, 0, 3422, 117, 21, 0, 888, 3883, 0, 0, 0, 99, 4096, 1966, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 3, 177, 4037, 2, 0, 0, 0, 3996, 4090, 4, 113, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4050, 0, 3341, 5, 4086, 0, 49, 214, 0, 1381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3796, 0, 1, 4047, 0, 4072, 4096, 16, 0, 1997, 6, 34, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14857, 17878, 16345, 18670, 15428, 16602, 13138, 18154] avg=16384.0000 max=18670 max_rank=3 min=13138 min_rank=6 imbalance_ratio=1.139526 max_load_reduction=112402 max_load_reduction_pct=0.857559 moved_to_echo=63322 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[4992, 25472, 7680, 128, 128, 11776, 128, 128, 32384, 128, 128, 128, 768, 128, 128, 128, 32768, 15232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 640, 128, 640, 128, 29568, 256, 128, 3584, 8576, 28544, 2432, 128, 128, 2432, 32768, 15744, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1024, 128, 256, 19328, 18304, 384, 256, 26368, 128, 1280, 13952, 10624, 256, 7552, 2048, 30336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 15360, 1024, 22912, 256, 3072, 0, 128, 17024, 32128, 128, 896, 27520, 9216, 0, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 12160, 128, 896, 128, 384, 128, 18688, 128, 23424, 1280, 384, 32640, 256, 1408, 6912, 17280, 16128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 384, 128, 512, 128, 7040, 32384, 32512, 19328, 0, 4608, 128, 3328, 640, 15744, 10240, 4608, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 256, 0, 17408, 128, 32000, 128, 30336, 128, 28544, 128, 384, 256, 128, 22016, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 27776, 1024, 128, 15616, 128, 28416, 31104, 128, 0, 15872, 128, 8192, 3328, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4001, 0, 6, 4096, 3745, 3922, 0, 4094, 0, 4096, 5, 269, 0, 0, 0, 0, 2258, 0, 0, 0, 3968, 186, 0, 14, 2, 4096, 4095, 0, 4, 0, 81, 0, 0, 0, 0, 3941, 7, 0, 13, 309, 0, 3691, 4096, 0, 4, 0, 0, 0, 0, 25, 0, 37, 0, 0, 0, 4, 0, 0, 0, 1, 2126, 2145, 0, 3332, 4088, 0, 0, 4055, 0, 0, 70, 0, 4096, 29, 62, 4, 3918, 4093, 899, 4064, 0, 22, 38, 4083, 876, 4096, 3, 0, 4079, 0, 0, 0, 31, 29, 0, 0, 0, 4096, 3960, 4096, 5, 4095, 2, 0, 0, 0, 724, 0, 0, 22, 4069, 0, 0, 0, 4096, 188, 0, 0, 1, 0, 2401, 195, 1407, 93, 0, 250, 0, 4068] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4095, 111, 0, 3502, 4070, 0, 170, 3029, 0, 3666, 3910, 756, 0, 79, 0, 2, 3380, 0, 27, 0, 4078, 111, 12, 13, 20, 3954, 4086, 176, 0, 10, 0, 0, 702, 19, 4, 4096, 154, 0, 0, 0, 0, 4089, 2724, 0, 10, 367, 0, 0, 0, 0, 0, 804, 0, 0, 0, 0, 3, 0, 2, 285, 3350, 4040, 0, 2601, 4096, 250, 0, 3571, 772, 22, 0, 600, 3642, 385, 921, 0, 3358, 104, 173, 3547, 3, 0, 0, 4063, 2963, 919, 0, 0, 227, 0, 6, 480, 1488, 3467, 0, 0, 56, 3957, 192, 4063, 0, 4090, 4046, 0, 0, 0, 0, 0, 2, 82, 1770, 0, 0, 0, 3760, 55, 114, 0, 0, 647, 0, 4091, 0, 1, 2481, 3301, 321, 479] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[3798, 0, 0, 4017, 4096, 19, 186, 3953, 329, 4084, 20, 446, 26, 14, 0, 26, 4076, 0, 0, 22, 3081, 0, 0, 0, 0, 3859, 4093, 73, 11, 0, 96, 124, 0, 0, 0, 49, 0, 20, 0, 3037, 0, 3960, 3962, 0, 0, 11, 0, 6, 65, 0, 0, 3889, 692, 0, 0, 0, 21, 31, 1, 0, 887, 0, 16, 4088, 4074, 0, 323, 129, 7, 1, 0, 17, 4062, 0, 59, 0, 4088, 0, 0, 3859, 0, 10, 2, 4022, 3844, 4096, 0, 168, 4069, 0, 0, 1107, 0, 0, 0, 0, 13, 4081, 3786, 4055, 25, 4073, 1107, 3, 25, 0, 0, 0, 22, 1, 804, 0, 0, 4, 4078, 0, 0, 27, 0, 110, 4076, 4071, 3586, 0, 188, 2110, 0, 3706] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[3972, 0, 0, 4087, 4096, 3910, 1, 1636, 0, 4096, 157, 623, 260, 122, 0, 0, 4096, 0, 0, 0, 4055, 6, 0, 3, 0, 4095, 3857, 0, 0, 117, 51, 4095, 0, 0, 0, 3, 0, 1, 0, 2392, 41, 4079, 3144, 79, 0, 0, 77, 0, 0, 44, 0, 41, 0, 0, 0, 0, 569, 0, 0, 9, 90, 11, 0, 3966, 4096, 0, 0, 3996, 128, 0, 1, 0, 3659, 0, 940, 123, 4080, 38, 0, 3690, 0, 1, 686, 4026, 0, 3932, 1, 18, 2565, 0, 0, 8, 0, 2477, 0, 2, 0, 3952, 3628, 4096, 0, 4093, 4023, 0, 0, 0, 317, 3, 4, 0, 1867, 0, 0, 0, 3971, 364, 0, 25, 561, 141, 4038, 4094, 1855, 0, 0, 56, 0, 1636] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4007, 3, 0, 2486, 4089, 23, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 6, 318, 0, 4062, 4096, 0, 174, 0, 0, 2488, 0, 0, 0, 3995, 0, 0, 0, 3904, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4086, 0, 4092, 4051, 0, 0, 4034, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 138, 363, 4096, 0, 0, 1860, 4096, 125, 4096, 0, 4, 4096, 0, 0, 0, 17, 2252, 0, 0, 0, 4096, 4096, 2890, 0, 4096, 8, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 19, 0, 0, 0, 3660, 1480, 0, 0, 0, 3970, 0, 2807] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4025, 4, 0, 4041, 4041, 0, 0, 4090, 0, 4090, 3447, 0, 14, 2, 46, 0, 1408, 77, 0, 0, 4071, 0, 61, 0, 1, 4050, 4083, 0, 49, 0, 1, 0, 3, 203, 46, 0, 8, 0, 0, 1236, 0, 3768, 4071, 0, 0, 0, 0, 1, 0, 0, 11, 0, 0, 0, 4, 0, 129, 3, 0, 6, 4096, 108, 0, 3416, 4094, 0, 4, 0, 0, 19, 0, 0, 4026, 311, 0, 0, 4092, 16, 14, 4090, 7, 0, 1709, 4094, 4, 4092, 0, 0, 4082, 0, 0, 78, 843, 4095, 0, 1, 0, 4086, 4060, 3856, 0, 4090, 9, 47, 0, 0, 0, 0, 2, 53, 4091, 0, 0, 1, 4096, 1, 2, 0, 0, 0, 4079, 4047, 2, 1, 0, 4073, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[503, 2, 0, 3705, 4094, 1, 0, 4086, 0, 4088, 28, 1, 82, 0, 0, 0, 4025, 0, 0, 0, 4045, 0, 523, 3, 2, 4086, 4096, 62, 371, 0, 0, 557, 0, 14, 7, 1899, 522, 0, 0, 3928, 0, 3073, 3900, 0, 35, 0, 16, 0, 0, 0, 9, 0, 0, 0, 1, 0, 6, 0, 10, 0, 0, 2767, 1, 4089, 4091, 0, 0, 2201, 261, 2, 0, 0, 4094, 0, 0, 0, 4096, 2046, 2, 3171, 0, 0, 58, 4086, 0, 4083, 0, 0, 6, 0, 0, 2788, 0, 2, 0, 0, 1, 4085, 4096, 4077, 34, 4096, 4096, 4070, 0, 0, 0, 0, 0, 0, 6, 0, 0, 5, 4070, 0, 0, 0, 2, 0, 2172, 4093, 2428, 0, 0, 4095, 0, 4022] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4014, 0, 0, 3557, 3748, 2424, 0, 4096, 0, 4073, 3680, 1, 1, 0, 0, 0, 4008, 0, 0, 0, 3921, 2, 29, 0, 0, 4086, 4096, 1, 0, 0, 0, 59, 19, 0, 0, 259, 3, 0, 0, 2305, 0, 3851, 4056, 0, 45, 22, 0, 0, 0, 34, 0, 6, 0, 7, 0, 0, 2, 0, 0, 0, 99, 4034, 15, 4079, 4069, 0, 1, 7, 47, 4, 5, 3, 4082, 17, 0, 1, 4091, 4096, 0, 4058, 35, 1, 0, 3974, 1287, 4068, 0, 0, 4091, 46, 10, 0, 556, 1244, 11, 0, 0, 4043, 4090, 3921, 0, 4053, 150, 4, 0, 3, 333, 0, 0, 57, 0, 0, 0, 0, 4090, 97, 0, 0, 373, 0, 3863, 4034, 0, 8, 0, 4096, 79, 1342] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4001, 0, 6, 4096, 2220, 3922, 0, 4094, 0, 0, 5, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2258, 0, 0, 0, 3968, 186, 0, 14, 2, 4096, 3791, 0, 4, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 7, 0, 13, 309, 0, 3691, 4096, 0, 4, 0, 0, 0, 2700, 1525, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 37, 0, 0, 0, 4, 0, 0, 0, 1, 2126, 2145, 0, 3332, 4096, 3893, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 4055, 0, 0, 70, 0, 4096, 29, 62, 4, 3918, 4093, 899, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 38, 4083, 876, 4096, 3, 0, 4079, 0, 0, 0, 31, 29, 0, 0, 1123, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3960, 4096, 5, 970, 2, 0, 0, 0, 724, 0, 0, 22, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2717, 188, 0, 0, 1, 0, 2401, 195, 1407, 93, 0, 250, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18613, 14400, 16542, 16084, 21485, 14684, 17944, 11320] avg=16384.0000 max=21485 max_rank=4 min=11320 min_rank=7 imbalance_ratio=1.311340 max_load_reduction=109587 max_load_reduction_pct=0.836082 moved_to_echo=65433 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4007, 3, 0, 2486, 2424, 23, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 6, 318, 0, 4062, 3792, 0, 174, 0, 0, 2488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 0, 0, 3904, 0, 4096, 4096, 0, 0, 0, 0, 0, 2701, 1665, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4086, 0, 4092, 4096, 3857, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 4034, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 138, 363, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1860, 4096, 125, 4096, 0, 4, 4096, 0, 0, 0, 17, 2252, 0, 0, 1123, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2890, 0, 970, 8, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2717, 1, 19, 0, 0, 0, 3660, 1480, 0, 0, 0, 3970, 0, 2807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13045, 19032, 20713, 16559, 17017, 17973, 12079, 14654] avg=16384.0000 max=20713 max_rank=2 min=12079 min_rank=6 imbalance_ratio=1.264221 max_load_reduction=110359 max_load_reduction_pct=0.841972 moved_to_echo=61723 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3972, 0, 0, 4087, 2428, 3910, 1, 1636, 0, 0, 157, 623, 260, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4055, 6, 0, 3, 0, 4095, 3571, 0, 0, 117, 51, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 2392, 41, 4079, 3144, 79, 0, 0, 77, 0, 2699, 1668, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 41, 0, 0, 0, 0, 569, 0, 0, 9, 90, 11, 0, 3966, 4096, 3900, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 3996, 128, 0, 1, 0, 3659, 0, 940, 123, 4080, 38, 0, 3690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 686, 4026, 0, 3932, 1, 18, 2565, 0, 0, 8, 0, 2477, 0, 2, 1089, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 3628, 4096, 0, 970, 4023, 0, 0, 0, 317, 3, 4, 0, 1867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2633, 364, 0, 25, 561, 141, 4038, 4094, 1855, 0, 0, 56, 0, 1636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17196, 20089, 14432, 13150, 16851, 15091, 18860, 15403] avg=16384.0000 max=20089 max_rank=1 min=13150 min_rank=3 imbalance_ratio=1.226135 max_load_reduction=110983 max_load_reduction_pct=0.846733 moved_to_echo=66205 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4095, 111, 0, 3502, 2408, 0, 170, 3029, 0, 0, 3910, 756, 0, 79, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3380, 0, 27, 0, 4078, 111, 12, 13, 20, 3954, 3777, 176, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 702, 19, 4, 4096, 154, 0, 0, 0, 0, 4089, 2724, 0, 10, 367, 0, 0, 2699, 1662, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 0, 0, 0, 0, 3, 0, 2, 285, 3350, 4040, 0, 2601, 3666, 3905, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 250, 0, 3571, 772, 22, 0, 600, 3642, 385, 921, 0, 3358, 104, 173, 3547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4063, 2963, 919, 0, 0, 227, 0, 6, 480, 1488, 3467, 0, 0, 1036, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 3957, 192, 4063, 0, 963, 4046, 0, 0, 0, 0, 0, 2, 82, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2484, 55, 114, 0, 0, 647, 0, 4091, 0, 1, 2481, 3301, 321, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18062, 15558, 16766, 19084, 17536, 14961, 15131, 13974] avg=16384.0000 max=19084 max_rank=3 min=13974 min_rank=7 imbalance_ratio=1.164795 max_load_reduction=111988 max_load_reduction_pct=0.854401 moved_to_echo=61602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3798, 0, 0, 4017, 2428, 19, 186, 3953, 329, 0, 20, 446, 26, 14, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 22, 3081, 0, 0, 0, 0, 3859, 3789, 73, 11, 0, 96, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 20, 0, 3037, 0, 3960, 3962, 0, 0, 11, 0, 6, 2685, 1668, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 3889, 692, 0, 0, 0, 21, 31, 1, 0, 887, 0, 16, 4088, 4084, 3879, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 323, 129, 7, 1, 0, 17, 4062, 0, 59, 0, 4088, 0, 0, 3859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2, 4022, 3844, 4096, 0, 168, 4069, 0, 0, 1107, 0, 0, 0, 0, 1118, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4081, 3786, 4055, 25, 966, 1107, 3, 25, 0, 0, 0, 22, 1, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2705, 0, 0, 27, 0, 110, 4076, 4071, 3586, 0, 188, 2110, 0, 3706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15262, 15131, 15653, 18075, 12740, 18740, 14888, 20583] avg=16384.0000 max=20583 max_rank=7 min=12740 min_rank=4 imbalance_ratio=1.256287 max_load_reduction=110489 max_load_reduction_pct=0.842964 moved_to_echo=66951 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4025, 4, 0, 4041, 2395, 0, 0, 4090, 0, 0, 3447, 0, 14, 2, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1408, 77, 0, 0, 4071, 0, 61, 0, 1, 4050, 3780, 0, 49, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 203, 46, 0, 8, 0, 0, 1236, 0, 3768, 4071, 0, 0, 0, 0, 1, 2697, 1646, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4, 0, 129, 3, 0, 6, 4096, 108, 0, 3416, 4090, 3898, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 4, 0, 0, 19, 0, 0, 4026, 311, 0, 0, 4092, 16, 14, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1709, 4094, 4, 4092, 0, 0, 4082, 0, 0, 78, 843, 4095, 0, 1, 1123, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4060, 3856, 0, 969, 9, 47, 0, 0, 0, 0, 2, 53, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2717, 1, 2, 0, 0, 0, 4079, 4047, 2, 1, 0, 4073, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18064, 13498, 13935, 16185, 12768, 20431, 17173, 19018] avg=16384.0000 max=20431 max_rank=5 min=12768 min_rank=4 imbalance_ratio=1.247009 max_load_reduction=110641 max_load_reduction_pct=0.844124 moved_to_echo=69390 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4014, 0, 0, 3557, 2222, 2424, 0, 4096, 0, 0, 3680, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 0, 3921, 2, 29, 0, 0, 4086, 3792, 1, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 259, 3, 0, 0, 2305, 0, 3851, 4056, 0, 45, 22, 0, 0, 2672, 1526, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 6, 0, 7, 0, 0, 2, 0, 0, 0, 99, 4034, 15, 4079, 4073, 3875, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 1, 7, 47, 4, 5, 3, 4082, 17, 0, 1, 4091, 4096, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 1, 0, 3974, 1287, 4068, 0, 0, 4091, 46, 10, 0, 556, 1244, 11, 0, 1122, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 4090, 3921, 0, 961, 150, 4, 0, 3, 333, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2712, 97, 0, 0, 373, 0, 3863, 4034, 0, 8, 0, 4096, 79, 1342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19995, 15898, 15014, 16644, 16606, 16749, 13562, 16604] avg=16384.0000 max=19995 max_rank=0 min=13562 min_rank=6 imbalance_ratio=1.220398 max_load_reduction=111077 max_load_reduction_pct=0.847450 moved_to_echo=63521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[503, 2, 0, 3705, 2427, 1, 0, 4086, 0, 0, 28, 1, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 0, 4045, 0, 523, 3, 2, 4086, 3792, 62, 371, 0, 0, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 7, 1899, 522, 0, 0, 3928, 0, 3073, 3900, 0, 35, 0, 16, 0, 2701, 1667, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1, 0, 6, 0, 10, 0, 0, 2767, 1, 4089, 4088, 3895, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 2201, 261, 2, 0, 0, 4094, 0, 0, 0, 4096, 2046, 2, 3171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 4086, 0, 4083, 0, 0, 6, 0, 0, 2788, 0, 2, 0, 0, 1116, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 4096, 4077, 34, 970, 4096, 4070, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2699, 0, 0, 0, 2, 0, 2172, 4093, 2428, 0, 0, 4095, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10835, 17466, 18017, 15291, 16069, 12443, 21435, 19516] avg=16384.0000 max=21435 max_rank=6 min=10835 min_rank=0 imbalance_ratio=1.308289 max_load_reduction=109637 max_load_reduction_pct=0.836464 moved_to_echo=69463 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[27392, 128, 128, 128, 31360, 384, 640, 384, 128, 32384, 30208, 384, 640, 128, 256, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1664, 256, 384, 18048, 1280, 128, 128, 640, 31872, 768, 2048, 128, 31872, 10624, 1536, 30592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 4480, 32512, 9216, 29440, 128, 256, 23296, 128, 128, 4480, 2944, 13568, 128, 128, 8960, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[768, 256, 128, 14336, 768, 128, 128, 17152, 128, 30720, 30080, 128, 128, 512, 128, 128, 21632, 13056, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 21504, 768, 256, 128, 1024, 1024, 24320, 26112, 9344, 128, 2688, 22016, 512, 22272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[28416, 128, 128, 29568, 19072, 10368, 384, 29184, 384, 0, 11264, 2176, 384, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32512, 28032, 31104, 128, 7808, 13568, 4224, 128, 128, 1408, 128, 128, 256, 12672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 4864, 768, 128, 128, 128, 768, 128, 128, 384, 10752, 17280, 128, 29696, 32384, 31104, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 2171, 0, 4081, 0, 0, 1292, 1, 0, 0, 1, 4096, 0, 4089, 909, 590, 180, 0, 0, 4087, 312, 3902, 0, 4094, 0, 0, 0, 1382, 162, 1, 2926, 2, 0, 0, 24, 4095, 2462, 4096, 0, 0, 0, 0, 0, 0, 0, 438, 0, 4096, 3749, 0, 0, 2126, 3787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 1, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 3488, 4096, 0, 0, 3331, 0, 3, 0, 3929, 0, 25, 0, 0, 0, 0, 0, 0, 34, 4096, 0, 0, 0, 0, 15, 0, 0, 12, 0, 4088, 0, 1, 4096, 4088, 0, 0, 37, 0, 0, 0, 0, 4096, 0, 3719, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 385, 209, 4085, 0, 1493, 1121, 4007, 0, 1, 1363, 3, 0, 7, 0, 3984, 0, 4038, 64, 3874, 0, 0, 0, 4062, 382, 38, 4, 192, 48, 0, 10, 1899, 1, 0, 4067, 4028, 0, 0, 0, 4075, 4096, 4087, 0, 1, 0, 3, 11, 537, 0, 0, 1, 4050, 4095, 0, 0, 286, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 15, 0, 0, 109, 0, 1, 4095, 0, 0, 4090, 4095, 4096, 0, 25, 4066, 0, 4056, 3276, 40, 0, 4045, 3484, 4094, 402, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 17, 1, 0, 4079, 26, 0, 0, 4096, 0, 3695, 4096, 412, 0, 0, 1872, 0, 0, 424, 0, 3560, 0, 5, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 68, 0, 3967, 1, 3602, 4044, 4075, 0, 18, 1238, 1, 0, 1, 0, 4085, 0, 3939, 127, 2360, 100, 0, 0, 4039, 0, 3682, 0, 3931, 0, 0, 0, 22, 0, 0, 4088, 0, 0, 0, 0, 3896, 3485, 4096, 81, 0, 1, 3722, 0, 2, 0, 0, 0, 4063, 1903, 1, 0, 0, 4029, 0, 0, 0, 0, 6, 0, 244, 12, 0, 2, 1, 369, 30, 4082, 0, 1, 3846, 408, 0, 4095, 4056, 4078, 8, 0, 4066, 0, 3336, 4037, 0, 0, 2140, 0, 0, 0, 0, 169, 0, 0, 0, 0, 3, 6, 0, 386, 4, 37, 50, 45, 0, 0, 2219, 1, 710, 11, 3757, 238, 1910, 4095, 3061, 0, 69, 4072, 0, 0, 547, 0, 2161, 0, 3937, 0, 100] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[40, 0, 301, 4095, 962, 3410, 0, 4006, 0, 0, 0, 65, 0, 0, 0, 3900, 0, 3866, 0, 4092, 671, 0, 0, 4096, 1799, 0, 0, 4055, 0, 11, 519, 0, 0, 0, 4089, 2882, 0, 6, 0, 3968, 4023, 4077, 1, 7, 0, 0, 8, 0, 0, 46, 0, 4038, 4094, 9, 13, 627, 4092, 18, 102, 42, 0, 5, 0, 27, 0, 0, 4048, 0, 0, 0, 2664, 14, 1412, 4083, 93, 0, 4064, 4089, 4088, 1, 0, 4073, 0, 3594, 4096, 43, 0, 0, 14, 1, 1331, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 3662, 0, 0, 30, 0, 4096, 0, 258, 4096, 0, 4, 1, 4089, 5, 7, 19, 18, 2659, 0, 2226, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[24, 0, 0, 4095, 0, 17, 3654, 4092, 0, 0, 82, 0, 6, 0, 1673, 3798, 0, 4078, 6, 4063, 553, 1, 33, 4096, 37, 3536, 0, 3505, 0, 0, 2350, 848, 0, 2, 2761, 2, 2, 0, 0, 4050, 3773, 3677, 0, 20, 0, 0, 0, 0, 0, 111, 23, 4089, 4096, 0, 0, 37, 4096, 0, 0, 0, 2, 0, 0, 20, 0, 548, 4093, 0, 0, 0, 4033, 0, 131, 4096, 0, 6, 4091, 4079, 4090, 4, 0, 4096, 0, 4082, 631, 224, 1, 0, 0, 971, 121, 0, 0, 4054, 3, 0, 0, 0, 0, 5, 127, 3728, 0, 0, 0, 0, 2719, 131, 14, 8, 0, 4094, 0, 1087, 4096, 2, 0, 0, 4090, 28, 0, 2, 0, 771, 0, 3494, 0, 14] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 202, 1431, 3670, 0, 2556, 0, 4095, 0, 5, 298, 0, 408, 0, 42, 3884, 0, 4040, 1, 3540, 0, 1, 0, 2818, 173, 3530, 1948, 4094, 0, 0, 2735, 0, 0, 0, 3876, 40, 0, 0, 19, 3743, 3293, 1772, 1, 0, 0, 0, 683, 776, 0, 926, 9, 3944, 4096, 0, 1240, 0, 3601, 0, 0, 1928, 0, 16, 335, 0, 0, 5, 2616, 0, 0, 0, 0, 0, 4029, 3589, 205, 0, 1878, 2817, 4096, 329, 0, 3518, 0, 4096, 58, 4038, 0, 0, 2654, 502, 2291, 0, 0, 3539, 2, 0, 12, 35, 0, 479, 630, 3997, 0, 0, 2, 99, 5, 22, 153, 2801, 0, 3255, 0, 74, 2017, 175, 255, 0, 163, 1967, 9, 688, 0, 3774, 0, 429, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1, 0, 0, 3987, 11, 2637, 19, 3869, 0, 0, 0, 0, 0, 9, 1, 4081, 0, 3736, 4069, 4084, 0, 0, 0, 4090, 0, 2325, 0, 3912, 18, 58, 0, 0, 0, 10, 34, 900, 80, 0, 13, 4079, 3918, 4096, 0, 121, 0, 0, 365, 132, 0, 0, 2, 4068, 4051, 248, 0, 0, 3727, 0, 0, 25, 22, 19, 2, 0, 0, 0, 4075, 155, 0, 0, 4094, 0, 0, 4064, 0, 0, 4065, 4069, 4096, 0, 0, 4093, 3, 4093, 2971, 0, 0, 61, 0, 1110, 0, 2528, 0, 2045, 0, 0, 45, 0, 0, 12, 0, 3986, 0, 0, 12, 50, 2, 4094, 929, 227, 27, 3959, 0, 3956, 3488, 0, 0, 7, 469, 29, 2, 2, 28, 4096, 0, 1307, 4, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[57, 0, 1, 4079, 302, 2733, 0, 2664, 0, 1, 20, 1008, 1, 0, 0, 4095, 0, 4032, 82, 3978, 0, 331, 0, 3876, 107, 426, 14, 3745, 0, 0, 0, 4071, 52, 0, 4078, 4002, 262, 96, 0, 4078, 4053, 4094, 15, 0, 0, 7, 0, 1, 172, 0, 0, 3993, 3811, 0, 2, 0, 4045, 6, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 22, 0, 2, 3426, 0, 0, 4091, 56, 56, 3939, 4077, 4089, 0, 0, 4038, 24, 3992, 3354, 0, 22, 1515, 0, 0, 103, 798, 0, 4035, 23, 0, 0, 0, 29, 23, 0, 4000, 1, 0, 0, 13, 1476, 298, 163, 0, 2, 3218, 6, 0, 3918, 100, 0, 1, 0, 90, 0, 0, 28, 4073, 0, 3484, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3738, 0, 2171, 0, 4081, 0, 0, 1292, 1, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 909, 590, 180, 0, 0, 4087, 312, 3902, 0, 4094, 0, 0, 0, 1382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 1, 2926, 2, 0, 0, 24, 3454, 2462, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 4096, 3749, 0, 0, 2126, 3787, 0, 0, 0, 0, 0, 0, 0, 2578, 950, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 1, 1518, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2554, 0, 3488, 4096, 0, 0, 3331, 0, 3, 0, 3929, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4096, 0, 0, 0, 0, 15, 0, 0, 12, 0, 4088, 4096, 2819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4088, 0, 0, 37, 0, 0, 0, 0, 4096, 0, 3719, 0, 0, 1451, 641, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15380, 15776, 13127, 17815, 17901, 17426, 15160, 18487] avg=16384.0000 max=18487 max_rank=7 min=13127 min_rank=2 imbalance_ratio=1.128357 max_load_reduction=112585 max_load_reduction_pct=0.858955 moved_to_echo=68974 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 68, 0, 3620, 1, 3602, 4044, 4075, 0, 18, 1238, 1, 0, 1, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 308, 127, 2360, 100, 0, 0, 4039, 0, 3682, 0, 3931, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 3286, 3485, 4096, 81, 0, 1, 3722, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 1903, 1, 0, 0, 4029, 0, 0, 0, 0, 6, 0, 244, 2420, 915, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 2, 1, 369, 30, 4082, 0, 1, 1426, 408, 0, 4095, 4056, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2536, 0, 3336, 4037, 0, 0, 2140, 0, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6, 0, 386, 4, 37, 50, 45, 0, 0, 2219, 1, 710, 11, 3757, 4078, 2716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 1910, 4095, 3061, 0, 69, 4072, 0, 0, 547, 0, 2161, 0, 3937, 0, 100, 1440, 610, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20753, 14569, 18761, 13671, 14490, 12218, 14023, 22587] avg=16384.0000 max=22587 max_rank=7 min=12218 min_rank=5 imbalance_ratio=1.378601 max_load_reduction=108485 max_load_reduction_pct=0.827675 moved_to_echo=63318 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[40, 0, 301, 3737, 962, 3410, 0, 4006, 0, 0, 0, 65, 0, 0, 0, 3900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 4092, 671, 0, 0, 4096, 1799, 0, 0, 4055, 0, 11, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 2882, 0, 6, 0, 3347, 4023, 4077, 1, 7, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 4038, 4094, 9, 13, 627, 4092, 18, 102, 42, 0, 5, 0, 27, 2570, 898, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 2664, 14, 1412, 1513, 93, 0, 4064, 4089, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2539, 0, 3594, 4096, 43, 0, 0, 14, 1, 1331, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 3662, 0, 0, 30, 0, 4096, 4088, 2665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 4096, 0, 4, 1, 4089, 5, 7, 19, 18, 2659, 0, 2226, 0, 0, 1443, 621, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16421, 15546, 18440, 16672, 17898, 15714, 14577, 15804] avg=16384.0000 max=18440 max_rank=2 min=14577 min_rank=6 imbalance_ratio=1.125488 max_load_reduction=112632 max_load_reduction_pct=0.859314 moved_to_echo=63993 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 385, 209, 3727, 0, 1493, 1121, 4007, 0, 1, 1363, 3, 0, 7, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 64, 3874, 0, 0, 0, 4062, 382, 38, 4, 192, 48, 0, 10, 1899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4067, 4028, 0, 0, 0, 3437, 4096, 4087, 0, 1, 0, 3, 11, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4050, 4095, 0, 0, 286, 4095, 0, 0, 0, 0, 0, 0, 0, 2577, 938, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 15, 0, 0, 109, 0, 1, 1518, 0, 0, 4090, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2536, 0, 4056, 3276, 40, 0, 4045, 3484, 4094, 402, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 17, 1, 0, 4079, 26, 0, 0, 4096, 4096, 2784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3695, 4096, 412, 0, 0, 1872, 0, 0, 424, 0, 3560, 0, 5, 0, 0, 1440, 638, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16300, 10889, 20268, 16132, 13919, 21961, 15103, 16500] avg=16384.0000 max=21961 max_rank=5 min=10889 min_rank=1 imbalance_ratio=1.340393 max_load_reduction=109111 max_load_reduction_pct=0.832451 moved_to_echo=67483 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[24, 0, 0, 3737, 0, 17, 3654, 4092, 0, 0, 82, 0, 6, 0, 1673, 3798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 6, 4063, 553, 1, 33, 4096, 37, 3536, 0, 3505, 0, 0, 2350, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2761, 2, 2, 0, 0, 3416, 3773, 3677, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 23, 4089, 4096, 0, 0, 37, 4096, 0, 0, 0, 2, 0, 0, 20, 2578, 947, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 4093, 0, 0, 0, 4033, 0, 131, 1518, 0, 6, 4091, 4079, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2554, 0, 4082, 631, 224, 1, 0, 0, 971, 121, 0, 0, 4054, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 127, 3728, 0, 0, 0, 0, 2719, 131, 14, 8, 0, 4094, 4090, 2811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1087, 4096, 2, 0, 0, 4090, 28, 0, 2, 0, 771, 0, 3494, 0, 14, 1451, 634, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17083, 19348, 13653, 16090, 18503, 12641, 17727, 16027] avg=16384.0000 max=19348 max_rank=1 min=12641 min_rank=5 imbalance_ratio=1.180908 max_load_reduction=111724 max_load_reduction_pct=0.852386 moved_to_echo=64898 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 202, 1431, 3344, 0, 2556, 0, 4095, 0, 5, 298, 0, 408, 0, 42, 3884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 1, 3540, 0, 1, 0, 2818, 173, 3530, 1948, 4094, 0, 0, 2735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3876, 40, 0, 0, 19, 3153, 3293, 1772, 1, 0, 0, 0, 683, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 926, 9, 3944, 4096, 0, 1240, 0, 3601, 0, 0, 1928, 0, 16, 335, 0, 2262, 941, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2616, 0, 0, 0, 0, 0, 4029, 1327, 205, 0, 1878, 2817, 0, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2185, 0, 4096, 58, 4038, 0, 0, 2654, 502, 2291, 0, 0, 3539, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 35, 0, 479, 630, 3997, 0, 0, 2, 99, 5, 22, 153, 2801, 0, 3255, 4096, 2789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 2017, 175, 255, 0, 163, 1967, 9, 688, 0, 3774, 0, 429, 0, 0, 1250, 590, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16265, 19150, 13613, 19381, 13206, 19365, 18375, 11717] avg=16384.0000 max=19381 max_rank=3 min=11717 min_rank=7 imbalance_ratio=1.182922 max_load_reduction=111691 max_load_reduction_pct=0.852135 moved_to_echo=62663 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3638, 11, 2637, 19, 3869, 0, 0, 0, 0, 0, 9, 1, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 4069, 4084, 0, 0, 0, 4090, 0, 2325, 0, 3912, 18, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 34, 900, 80, 0, 13, 3441, 3918, 4096, 0, 121, 0, 0, 365, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4068, 4051, 248, 0, 0, 3727, 0, 0, 25, 22, 19, 2, 0, 2558, 868, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 155, 0, 0, 4094, 0, 0, 1506, 0, 0, 4065, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2552, 3, 4093, 2971, 0, 0, 61, 0, 1110, 0, 2528, 0, 2045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 12, 0, 3986, 0, 0, 12, 50, 2, 4094, 929, 227, 27, 3959, 4096, 2576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 3488, 0, 0, 7, 469, 29, 2, 2, 28, 4096, 0, 1307, 4, 0, 1450, 638, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14266, 18848, 13110, 15681, 17964, 15363, 20015, 15825] avg=16384.0000 max=20015 max_rank=6 min=13110 min_rank=2 imbalance_ratio=1.221619 max_load_reduction=111057 max_load_reduction_pct=0.847298 moved_to_echo=69167 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[57, 0, 1, 3722, 302, 2733, 0, 2664, 0, 1, 20, 1008, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 82, 3978, 0, 331, 0, 3876, 107, 426, 14, 3745, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4078, 4002, 262, 96, 0, 3440, 4053, 4094, 15, 0, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 3993, 3811, 0, 2, 0, 4045, 6, 0, 0, 0, 0, 0, 0, 2575, 936, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 22, 0, 2, 3426, 0, 0, 1516, 56, 56, 3939, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2518, 24, 3992, 3354, 0, 22, 1515, 0, 0, 103, 798, 0, 4035, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 23, 0, 4000, 1, 0, 0, 13, 1476, 298, 163, 0, 2, 3218, 4089, 2780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3918, 100, 0, 1, 0, 90, 0, 0, 28, 4073, 0, 3484, 0, 0, 1430, 638, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14604, 16946, 20100, 15630, 17191, 16384, 16092, 14125] avg=16384.0000 max=20100 max_rank=2 min=14125 min_rank=7 imbalance_ratio=1.226807 max_load_reduction=110972 max_load_reduction_pct=0.846649 moved_to_echo=63792 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 20096, 128, 30848, 22528, 4352, 128, 11136, 6272, 6784, 4352, 7296, 256, 17920, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 768, 2048, 29312, 1280, 18688, 8960, 30976, 0, 128, 4352, 1152, 512, 128, 1792, 32000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 2560, 5376, 26624, 1536, 384, 128, 31232, 2816, 17536, 2048, 27648, 128, 128, 5632, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 640, 27136, 256, 384, 128, 22528, 128, 5632, 11904, 768, 128, 30336, 31488, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 640, 1280, 19840, 128, 128, 128, 256, 7936, 10880, 1408, 3840, 128, 30592, 32768, 22016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 1536, 128, 32384, 29952, 384, 1280, 3200, 31488, 128, 128, 2048, 128, 128, 384, 384, 20224, 7424, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 128, 25984, 11904, 384, 128, 128, 27008, 29184, 30080, 128, 256, 128, 3840, 1152, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 11008, 29952, 7936, 384, 128, 14848, 2176, 128, 1792, 128, 25216, 0, 18688, 128, 128, 11392, 5120, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4095, 60, 0, 0, 0, 0, 1340, 2532, 0, 0, 4096, 0, 4096, 0, 0, 1282, 0, 0, 49, 0, 4093, 0, 0, 4096, 4096, 0, 0, 962, 19, 4096, 0, 0, 0, 2883, 0, 0, 276, 0, 4096, 0, 4093, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4083, 4096, 0, 0, 4090, 0, 14, 4096, 0, 0, 0, 4096, 19, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4018, 0, 32, 1, 0, 0, 0, 3755, 4093, 1, 152, 0, 0, 0, 68, 0, 4096, 3934, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4078, 2, 0, 3239, 4096, 4081, 4096, 0, 0, 4096, 0, 4087, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 32, 0, 0, 0, 0, 0, 0, 3896, 0, 0, 93, 5, 0, 7, 4075, 32, 39, 4096, 384, 4086, 0, 2, 170, 0, 65, 583, 198, 4043, 0, 0, 3187, 4096, 2, 0, 3638, 832, 4096, 1, 3, 0, 1365, 0, 0, 3368, 31, 4096, 11, 4094, 0, 73, 1, 0, 0, 4, 1791, 19, 1, 2221, 4096, 104, 0, 4090, 0, 78, 4078, 0, 8, 0, 4075, 0, 28, 0, 161, 0, 9, 0, 0, 0, 8, 0, 4091, 4013, 0, 0, 0, 0, 0, 0, 3984, 3775, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 4091, 2, 927, 0, 0, 139, 4, 199, 4066, 3827, 4096, 2110, 4020, 0, 1, 4090, 5, 4039, 3929] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 61, 476, 0, 0, 1, 0, 100, 11, 4095, 34, 1, 0, 0, 0, 4, 148, 0, 79, 4095, 0, 3522, 0, 20, 0, 0, 0, 374, 0, 4095, 0, 0, 4013, 4088, 1, 45, 3380, 6, 4096, 0, 0, 0, 239, 0, 0, 1, 0, 3551, 3, 4095, 0, 0, 1, 0, 0, 0, 4037, 0, 5, 4096, 4096, 633, 0, 4048, 0, 0, 4085, 0, 1, 0, 2887, 0, 0, 5, 1433, 0, 1, 0, 0, 2, 115, 0, 1379, 4032, 92, 4047, 0, 0, 0, 0, 4092, 4095, 38, 0, 0, 3, 0, 4096, 0, 4096, 3960, 0, 0, 0, 4096, 1, 0, 0, 4096, 2, 1911, 0, 0, 7, 3985, 0, 4096, 4086, 493, 16, 4053, 0, 96, 4095, 0, 50, 3980] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 955, 0, 889, 0, 0, 0, 0, 251, 0, 4096, 0, 11, 0, 0, 0, 1215, 3963, 0, 0, 4029, 278, 4055, 0, 0, 0, 12, 0, 0, 89, 4052, 570, 0, 4083, 3785, 86, 404, 4, 3342, 3933, 0, 0, 4, 467, 0, 0, 3722, 4096, 3734, 1, 4082, 360, 0, 0, 73, 0, 2, 0, 0, 0, 3919, 4082, 0, 0, 4014, 51, 173, 3949, 0, 0, 0, 2954, 706, 0, 5, 541, 0, 0, 0, 0, 13, 47, 2, 51, 295, 0, 3779, 8, 0, 1, 0, 4096, 3782, 101, 1, 0, 27, 0, 0, 0, 3978, 0, 1, 0, 0, 3612, 0, 1, 0, 4052, 36, 0, 9, 0, 243, 1479, 881, 3818, 4096, 3607, 4095, 3220, 0, 0, 4096, 0, 1070, 3538] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 1015, 3686, 0, 0, 385, 31, 85, 0, 3025, 95, 0, 838, 9, 9, 394, 3345, 33, 0, 3524, 0, 3897, 0, 0, 4034, 0, 0, 1650, 0, 3508, 1861, 0, 0, 4020, 0, 0, 1, 323, 4096, 0, 0, 0, 4096, 0, 240, 76, 3749, 3902, 0, 3238, 0, 0, 639, 0, 11, 0, 53, 0, 0, 3882, 2647, 4044, 0, 3656, 0, 3796, 2029, 425, 0, 0, 881, 0, 14, 0, 293, 0, 864, 0, 0, 0, 4092, 0, 4085, 55, 0, 833, 0, 0, 476, 30, 3711, 265, 15, 3, 68, 1, 180, 4057, 172, 3876, 0, 0, 0, 820, 3779, 55, 0, 12, 4094, 56, 4047, 4, 5, 1109, 0, 183, 0, 0, 3954, 1144, 3305, 0, 0, 4094, 0, 2148, 1945] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 60, 0, 0, 0, 64, 45, 4051, 0, 0, 0, 12, 344, 10, 2814, 24, 0, 4096, 0, 4044, 0, 0, 0, 70, 78, 194, 67, 4075, 0, 0, 3783, 4095, 1, 0, 4012, 0, 4008, 184, 23, 0, 3812, 0, 0, 4094, 0, 4050, 10, 4007, 0, 99, 0, 0, 4, 96, 0, 8, 11, 2756, 3979, 0, 1, 3423, 0, 986, 3688, 0, 0, 0, 4078, 2, 12, 101, 335, 0, 0, 3, 17, 0, 586, 298, 1, 3934, 217, 714, 0, 8, 17, 5, 4096, 4096, 51, 122, 0, 45, 0, 21, 0, 4090, 4085, 0, 28, 0, 3956, 0, 0, 0, 1380, 0, 1, 1362, 7, 0, 4091, 54, 62, 2463, 3804, 3360, 4012, 0, 238, 4019, 0, 4021, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[7, 71, 109, 0, 53, 0, 8, 2, 0, 0, 4093, 0, 0, 0, 42, 24, 1, 4061, 24, 131, 3979, 543, 4071, 21, 0, 3516, 0, 0, 939, 0, 4096, 36, 0, 3516, 4041, 0, 4, 1, 19, 4058, 1323, 0, 0, 3090, 54, 271, 242, 87, 3890, 0, 4008, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4087, 4054, 0, 8, 4096, 0, 0, 4096, 0, 0, 1, 4063, 0, 0, 0, 3697, 0, 0, 0, 0, 1, 3102, 56, 0, 4090, 0, 723, 7, 0, 0, 0, 4042, 4082, 0, 0, 0, 0, 0, 2, 0, 4091, 0, 0, 31, 78, 4069, 0, 0, 0, 4092, 0, 2355, 0, 0, 14, 97, 81, 4076, 3908, 490, 3368, 4096, 9, 39, 4013, 0, 1423, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[5, 354, 0, 0, 0, 36, 27, 2, 0, 0, 4096, 0, 48, 0, 72, 1, 0, 3, 72, 33, 4096, 2, 4094, 0, 0, 0, 0, 0, 891, 260, 4093, 0, 0, 4096, 4075, 0, 3, 4015, 165, 4096, 80, 0, 0, 4088, 0, 0, 2912, 37, 4084, 0, 4096, 0, 0, 0, 7, 0, 0, 1036, 15, 0, 3948, 3808, 0, 0, 4024, 0, 1207, 4047, 0, 0, 0, 3918, 0, 115, 0, 0, 1, 0, 7, 0, 0, 3729, 0, 1243, 4084, 0, 3, 0, 0, 5, 1, 3095, 4095, 5, 185, 0, 17, 7, 614, 0, 4093, 4076, 23, 0, 0, 4013, 16, 0, 0, 3773, 54, 0, 57, 0, 0, 115, 0, 2573, 2697, 4095, 0, 4096, 0, 34, 4096, 0, 4096, 4017] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4095, 60, 0, 0, 0, 0, 4096, 4096, 3228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1340, 2532, 0, 0, 4096, 0, 4096, 0, 0, 1282, 0, 0, 49, 0, 3140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 962, 19, 735, 0, 0, 0, 2883, 0, 0, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3440, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 14, 4096, 0, 0, 0, 4096, 19, 1, 0, 0, 0, 0, 0, 0, 3310, 311, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4018, 0, 32, 1, 0, 0, 0, 3755, 4093, 1, 152, 0, 642, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 4096, 3934, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 2, 0, 3239, 4096, 4081, 0, 0, 0, 0, 0, 4087, 786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15578, 16535, 13067, 15719, 16070, 17258, 16476, 20369] avg=16384.0000 max=20369 max_rank=7 min=13067 min_rank=2 imbalance_ratio=1.243225 max_load_reduction=110703 max_load_reduction_pct=0.844597 moved_to_echo=70173 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 955, 0, 889, 0, 0, 0, 0, 251, 0, 4096, 0, 11, 0, 0, 0, 4096, 3220, 3099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1215, 3963, 0, 0, 4029, 278, 4055, 0, 0, 0, 12, 0, 0, 89, 3108, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 3785, 86, 404, 4, 3342, 707, 0, 0, 4, 467, 0, 0, 3722, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3734, 1, 3431, 360, 0, 0, 73, 0, 2, 0, 0, 0, 3919, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 51, 173, 3949, 0, 0, 0, 2954, 706, 0, 5, 541, 0, 0, 0, 0, 2859, 308, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 47, 2, 51, 295, 0, 3779, 8, 0, 1, 0, 4096, 3782, 101, 1, 0, 636, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 3978, 0, 1, 0, 0, 3612, 0, 1, 0, 4052, 36, 0, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 243, 1479, 881, 3818, 4096, 3607, 4095, 0, 0, 0, 0, 0, 1070, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16617, 17319, 20700, 15602, 15687, 13278, 11892, 19977] avg=16384.0000 max=20700 max_rank=2 min=11892 min_rank=6 imbalance_ratio=1.263428 max_load_reduction=110372 max_load_reduction_pct=0.842072 moved_to_echo=60834 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 32, 0, 0, 0, 0, 0, 0, 3896, 0, 0, 93, 5, 0, 4090, 4020, 3228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4075, 32, 39, 4096, 384, 4086, 0, 2, 170, 0, 65, 583, 198, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3187, 4096, 2, 0, 3638, 832, 735, 1, 3, 0, 1365, 0, 0, 3368, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 3441, 0, 73, 1, 0, 0, 4, 1791, 19, 1, 2221, 4096, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 78, 4078, 0, 8, 0, 4075, 0, 28, 0, 161, 0, 9, 0, 0, 3175, 308, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4091, 4013, 0, 0, 0, 0, 0, 0, 3984, 3775, 0, 0, 0, 634, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 4091, 2, 927, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 4, 199, 4066, 3827, 4096, 2110, 0, 0, 1, 0, 5, 4039, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15365, 16838, 17258, 15858, 16143, 16972, 13398, 19240] avg=16384.0000 max=19240 max_rank=7 min=13398 min_rank=6 imbalance_ratio=1.174316 max_load_reduction=111832 max_load_reduction_pct=0.853210 moved_to_echo=65753 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1015, 3686, 0, 0, 385, 31, 85, 0, 3025, 95, 0, 838, 9, 9, 4094, 3305, 3232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 394, 3345, 33, 0, 3524, 0, 3897, 0, 0, 4034, 0, 0, 1650, 0, 2682, 1861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 1, 323, 728, 0, 0, 0, 4096, 0, 240, 76, 3749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 2714, 0, 0, 639, 0, 11, 0, 53, 0, 0, 3882, 2647, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3656, 0, 3796, 2029, 425, 0, 0, 881, 0, 14, 0, 293, 0, 864, 0, 0, 1576, 271, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4085, 55, 0, 833, 0, 0, 476, 30, 3711, 265, 15, 3, 68, 555, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 180, 4057, 172, 3876, 0, 0, 0, 820, 3779, 55, 0, 12, 4094, 56, 4047, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 1109, 0, 183, 0, 0, 3954, 1144, 0, 0, 0, 0, 0, 2148, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19809, 21420, 13233, 17892, 13941, 14562, 21299, 8916] avg=16384.0000 max=21420 max_rank=1 min=8916 min_rank=7 imbalance_ratio=1.307373 max_load_reduction=109652 max_load_reduction_pct=0.836578 moved_to_echo=58718 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 61, 476, 0, 0, 1, 0, 100, 11, 4095, 34, 1, 0, 0, 0, 4095, 4053, 3228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 148, 0, 79, 4095, 0, 3522, 0, 20, 0, 0, 0, 374, 0, 3141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 4088, 1, 45, 3380, 6, 735, 0, 0, 0, 239, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3551, 3, 3442, 0, 0, 1, 0, 0, 0, 4037, 0, 5, 4096, 4096, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 4085, 0, 1, 0, 2887, 0, 0, 5, 1433, 0, 1, 0, 0, 3216, 311, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 115, 0, 1379, 4032, 92, 4047, 0, 0, 0, 0, 4092, 4095, 38, 0, 0, 643, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 3960, 0, 0, 0, 4096, 1, 0, 0, 4096, 2, 1911, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3985, 0, 4096, 4086, 493, 16, 0, 0, 96, 0, 0, 50, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16155, 11383, 12508, 19864, 16120, 19002, 22447, 13593] avg=16384.0000 max=22447 max_rank=6 min=11383 min_rank=1 imbalance_ratio=1.370056 max_load_reduction=108625 max_load_reduction_pct=0.828743 moved_to_echo=71162 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[5, 354, 0, 0, 0, 36, 27, 2, 0, 0, 4096, 0, 48, 0, 72, 1, 4096, 4096, 3228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 72, 33, 4096, 2, 4094, 0, 0, 0, 0, 0, 891, 260, 3140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4075, 0, 3, 4015, 165, 735, 80, 0, 0, 4088, 0, 0, 2912, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 3442, 0, 0, 0, 7, 0, 0, 1036, 15, 0, 3948, 3808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 1207, 4047, 0, 0, 0, 3918, 0, 115, 0, 0, 1, 0, 7, 0, 3246, 311, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3729, 0, 1243, 4084, 0, 3, 0, 0, 5, 1, 3095, 4095, 5, 185, 0, 642, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7, 614, 0, 4093, 4076, 23, 0, 0, 4013, 16, 0, 0, 3773, 54, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 115, 0, 2573, 2697, 4095, 0, 0, 0, 34, 0, 0, 4096, 771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16061, 12591, 20206, 16340, 17009, 17555, 16872, 14438] avg=16384.0000 max=20206 max_rank=2 min=12591 min_rank=1 imbalance_ratio=1.233276 max_load_reduction=110866 max_load_reduction_pct=0.845840 moved_to_echo=65874 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[7, 71, 109, 0, 53, 0, 8, 2, 0, 0, 4093, 0, 0, 0, 42, 24, 4013, 4096, 3198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4061, 24, 131, 3979, 543, 4071, 21, 0, 3516, 0, 0, 939, 0, 3141, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3516, 4041, 0, 4, 1, 19, 728, 1323, 0, 0, 3090, 54, 271, 242, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3890, 0, 3368, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4087, 4054, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 4063, 0, 0, 0, 3697, 0, 0, 0, 0, 3310, 312, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3102, 56, 0, 4090, 0, 723, 7, 0, 0, 0, 4042, 4082, 0, 0, 0, 643, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4091, 0, 0, 31, 78, 4069, 0, 0, 0, 4092, 0, 2355, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 97, 81, 4076, 3908, 490, 3368, 0, 9, 39, 0, 0, 1423, 786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15716, 20463, 13376, 15415, 19707, 17204, 14900, 14291] avg=16384.0000 max=20463 max_rank=1 min=13376 min_rank=2 imbalance_ratio=1.248962 max_load_reduction=110609 max_load_reduction_pct=0.843880 moved_to_echo=66102 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 60, 0, 0, 0, 64, 45, 4051, 0, 0, 0, 12, 344, 4019, 4012, 3158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2814, 24, 0, 4096, 0, 4044, 0, 0, 0, 70, 78, 194, 67, 3126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3783, 4095, 1, 0, 4012, 0, 720, 184, 23, 0, 3812, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 10, 3368, 0, 99, 0, 0, 4, 96, 0, 8, 11, 2756, 3979, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3423, 0, 986, 3688, 0, 0, 0, 4078, 2, 12, 101, 335, 0, 0, 3, 17, 3310, 310, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 586, 298, 1, 3934, 217, 714, 0, 8, 17, 5, 4096, 4096, 51, 122, 0, 639, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 21, 0, 4090, 4085, 0, 28, 0, 3956, 0, 0, 0, 1380, 0, 1, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1362, 7, 0, 4091, 54, 62, 2463, 3804, 3360, 0, 0, 238, 0, 0, 4021, 786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15771, 14523, 20724, 14382, 16395, 15241, 13788, 20248] avg=16384.0000 max=20724 max_rank=2 min=13788 min_rank=6 imbalance_ratio=1.264893 max_load_reduction=110348 max_load_reduction_pct=0.841888 moved_to_echo=65672 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[31488, 128, 26752, 384, 256, 768, 128, 128, 128, 7040, 128, 128, 29056, 30976, 4864, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[31488, 128, 6272, 30080, 512, 128, 128, 27008, 768, 256, 128, 6528, 128, 896, 128, 128, 24064, 2560, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 26880, 32384, 128, 512, 16128, 4736, 5888, 1664, 128, 128, 20096, 128, 512, 14720, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[3072, 20992, 256, 384, 32128, 1280, 31872, 128, 128, 9088, 128, 256, 4736, 640, 24704, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 1408, 1280, 5120, 128, 128, 512, 128, 512, 128, 31488, 256, 128, 1024, 256, 384, 32640, 30976, 25600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 15872, 384, 10880, 24576, 384, 10240, 128, 128, 512, 128, 30976, 28288, 256, 512, 128, 5120, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1536, 128, 1536, 13952, 1408, 18816, 24320, 24704, 18176, 0, 128, 512, 0, 128, 20992, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 256, 8960, 256, 32512, 16128, 128, 128, 1024, 31744, 128, 128, 128, 29696, 256, 9344, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 2826, 0, 0, 0, 1669, 345, 4096, 0, 0, 704, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 859, 4096, 4091, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 25, 1, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 3531, 1979, 279, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 4096, 0, 4096, 0, 2261, 4019, 3784, 271, 0, 0, 46, 22, 3907, 4096, 0, 0, 0, 1, 0, 0, 18, 4047, 4096, 4096, 4096, 0, 0, 22, 1, 3245, 0, 1, 0, 3016, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[675, 0, 0, 5, 4096, 307, 5, 4081, 4066, 0, 66, 0, 3981, 4024, 801, 0, 1751, 230, 0, 17, 4017, 3623, 14, 141, 0, 221, 4096, 0, 0, 3197, 2362, 2600, 3647, 0, 0, 3812, 0, 0, 0, 29, 1, 0, 3534, 1, 0, 0, 4096, 0, 2998, 2573, 466, 0, 0, 321, 0, 4, 0, 1, 2479, 622, 1, 0, 1321, 3250, 0, 2, 0, 1, 58, 0, 136, 0, 0, 113, 58, 0, 0, 104, 0, 3930, 29, 0, 0, 4092, 0, 1987, 0, 1339, 1, 4085, 0, 3990, 0, 1232, 3494, 0, 621, 0, 0, 0, 15, 4096, 4091, 26, 0, 47, 0, 4096, 0, 0, 4012, 3252, 4088, 4094, 0, 16, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 199, 138] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3959, 0, 0, 3597, 0, 0, 562, 0, 43, 4042, 0, 0, 0, 0, 0, 7, 4096, 4096, 41, 0, 0, 0, 4096, 0, 22, 4094, 4091, 0, 4093, 0, 0, 4094, 0, 5, 0, 0, 34, 3, 6, 5, 262, 116, 4088, 296, 4066, 4091, 1, 0, 0, 0, 0, 15, 3, 0, 0, 0, 0, 1, 0, 4052, 0, 0, 4095, 0, 0, 12, 0, 0, 0, 33, 3728, 1271, 0, 7, 0, 4071, 35, 0, 0, 4084, 0, 0, 3, 4096, 0, 4094, 2, 2458, 0, 4078, 4096, 0, 3, 0, 0, 53, 85, 4051, 4054, 0, 0, 0, 1, 0, 1, 0, 4089, 4094, 4091, 4096, 0, 0, 31, 0, 4095, 0, 4095, 0, 0, 4076, 16, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[36, 1, 0, 72, 4028, 0, 0, 3968, 62, 0, 0, 0, 1, 4074, 1, 0, 0, 4, 0, 0, 686, 4055, 0, 0, 0, 0, 4061, 0, 529, 3694, 4095, 130, 3327, 0, 0, 4090, 0, 91, 0, 0, 1, 16, 0, 0, 49, 0, 4096, 0, 4096, 4096, 0, 3, 0, 0, 9, 0, 0, 3, 0, 0, 0, 0, 5, 4083, 1, 4, 3858, 0, 17, 0, 0, 0, 1, 3782, 4081, 198, 0, 204, 0, 4096, 8, 0, 0, 4094, 0, 0, 0, 4096, 0, 3941, 0, 3840, 0, 3951, 4095, 5, 0, 0, 0, 787, 305, 3843, 4093, 0, 31, 0, 0, 0, 0, 2, 3985, 4075, 4095, 4089, 0, 1, 0, 0, 4096, 0, 3825, 5, 3, 4092, 2, 9, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[13, 0, 0, 28, 4096, 0, 5, 3775, 0, 0, 0, 108, 44, 4069, 0, 0, 1082, 0, 0, 0, 1438, 3291, 15, 0, 10, 0, 4096, 0, 52, 4080, 4091, 1511, 4037, 0, 0, 4083, 130, 3, 0, 0, 386, 0, 13, 0, 16, 0, 4069, 0, 4051, 4082, 0, 9, 7, 0, 93, 9, 0, 7, 46, 2315, 0, 0, 1, 4038, 0, 0, 4080, 5, 0, 0, 0, 22, 0, 264, 4086, 78, 0, 143, 0, 3952, 4, 0, 0, 3977, 0, 2, 10, 4035, 0, 4096, 0, 4091, 0, 3030, 4024, 0, 0, 0, 20, 1, 0, 3870, 2239, 55, 0, 0, 0, 4087, 0, 0, 3908, 4007, 4062, 4093, 0, 2, 931, 33, 4092, 94, 68, 0, 245, 4057, 0, 0, 0, 110] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 426, 4068, 0, 905, 4000, 0, 0, 0, 336, 1657, 3995, 0, 0, 0, 40, 24, 0, 17, 4050, 0, 23, 0, 0, 4051, 0, 0, 4023, 3969, 0, 4096, 0, 89, 4051, 0, 1182, 0, 4094, 0, 85, 0, 0, 0, 3, 3866, 1787, 4021, 4096, 0, 0, 0, 0, 3806, 11, 121, 0, 0, 716, 237, 5, 0, 3398, 0, 0, 3864, 0, 0, 0, 88, 0, 0, 1755, 0, 2, 0, 0, 0, 4011, 0, 0, 70, 3958, 0, 0, 19, 3753, 428, 2570, 0, 4028, 0, 4096, 4096, 25, 0, 0, 56, 0, 14, 4074, 1985, 54, 0, 0, 41, 515, 0, 0, 389, 3633, 3932, 4096, 74, 0, 0, 187, 3766, 88, 4034, 0, 0, 3957, 0, 0, 166, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 133, 1, 64, 3602, 55, 133, 695, 0, 0, 406, 0, 0, 3909, 3, 2, 0, 0, 20, 4, 2219, 4036, 28, 1, 6, 0, 4079, 0, 2, 4006, 2977, 1997, 4025, 0, 0, 4068, 0, 220, 0, 1923, 97, 0, 1017, 0, 0, 0, 4096, 1, 2449, 3146, 3881, 14, 0, 41, 4, 254, 0, 1, 0, 0, 0, 73, 21, 4091, 38, 0, 4096, 0, 0, 0, 0, 5, 0, 21, 3956, 0, 140, 56, 0, 3947, 8, 49, 0, 4092, 0, 0, 12, 4060, 8, 4070, 29, 4038, 49, 4068, 4096, 0, 77, 259, 0, 0, 25, 3656, 3913, 0, 1, 0, 821, 0, 1, 0, 3511, 4078, 3854, 4095, 0, 8, 0, 0, 4082, 170, 2547, 0, 1343, 3993, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[45, 38, 17, 0, 4054, 72, 0, 4096, 128, 0, 67, 206, 2, 3938, 0, 2, 8, 0, 15, 33, 0, 3967, 0, 0, 0, 0, 4093, 0, 2990, 4061, 4096, 0, 4095, 0, 0, 4080, 70, 0, 0, 392, 0, 0, 927, 0, 26, 0, 4073, 0, 3959, 4092, 0, 0, 0, 0, 2362, 0, 0, 11, 0, 0, 0, 0, 0, 4019, 0, 0, 4017, 0, 0, 0, 1, 91, 0, 0, 1313, 0, 0, 7, 1, 3058, 0, 0, 0, 4050, 0, 0, 6, 4064, 9, 4028, 0, 1978, 0, 4042, 4082, 3731, 68, 0, 0, 0, 16, 4057, 3742, 109, 0, 282, 3, 21, 0, 1319, 4093, 4070, 4023, 4064, 0, 34, 0, 0, 2680, 53, 35, 0, 3793, 4096, 0, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 2826, 0, 0, 0, 1669, 345, 4096, 0, 0, 2428, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 704, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1410, 0, 859, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1950, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 25, 1, 0, 0, 0, 7, 0, 0, 0, 4096, 2316, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 3531, 1979, 279, 0, 0, 0, 4096, 4096, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 4096, 0, 2261, 624, 3784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 0, 0, 46, 22, 3621, 4096, 0, 0, 0, 1, 0, 0, 18, 4047, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1868, 0, 0, 22, 1, 3245, 0, 1, 0, 3016, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15828, 15254, 14524, 14914, 19031, 18958, 16218, 16345] avg=16384.0000 max=19031 max_rank=4 min=14524 min_rank=2 imbalance_ratio=1.161560 max_load_reduction=112041 max_load_reduction_pct=0.854805 moved_to_echo=70552 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 426, 4068, 0, 905, 4000, 0, 0, 0, 336, 1657, 3995, 0, 0, 2474, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 24, 0, 17, 4050, 0, 23, 0, 0, 1394, 0, 0, 4023, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 89, 4051, 0, 1182, 0, 4094, 0, 85, 0, 0, 0, 3, 3866, 1787, 1950, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 4096, 0, 0, 0, 0, 3806, 11, 121, 0, 0, 716, 237, 5, 0, 3398, 2291, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 0, 88, 0, 0, 1755, 0, 2, 0, 0, 0, 4011, 3958, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 19, 3753, 428, 2570, 0, 4028, 0, 4096, 637, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 14, 3776, 1985, 54, 0, 0, 41, 515, 0, 0, 389, 3633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 1868, 74, 0, 0, 187, 3766, 88, 4034, 0, 0, 3957, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18227, 13540, 21501, 18980, 14663, 15626, 10463, 18072] avg=16384.0000 max=21501 max_rank=2 min=10463 min_rank=6 imbalance_ratio=1.312317 max_load_reduction=109571 max_load_reduction_pct=0.835960 moved_to_echo=58824 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[675, 0, 0, 5, 4096, 307, 5, 4081, 4066, 0, 66, 0, 3981, 4024, 801, 0, 2116, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1751, 230, 0, 17, 4017, 3623, 14, 141, 0, 221, 1399, 0, 0, 3197, 2362, 2600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3647, 0, 0, 3812, 0, 0, 0, 29, 1, 0, 3534, 1, 0, 0, 4096, 0, 1953, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2998, 2573, 466, 0, 0, 321, 0, 4, 0, 1, 2479, 622, 1, 0, 1321, 3250, 2321, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 58, 0, 136, 0, 0, 113, 58, 0, 0, 104, 0, 3930, 4092, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 1987, 0, 1339, 1, 4085, 0, 3990, 0, 1232, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 15, 3794, 4091, 26, 0, 47, 0, 4096, 0, 0, 4012, 3252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1857, 0, 16, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 199, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[24599, 19572, 17375, 16641, 9340, 13195, 19954, 10396] avg=16384.0000 max=24599 max_rank=0 min=9340 min_rank=4 imbalance_ratio=1.501404 max_load_reduction=106473 max_load_reduction_pct=0.812325 moved_to_echo=52885 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3959, 0, 0, 3597, 0, 0, 562, 0, 43, 4042, 0, 0, 2474, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 4096, 41, 0, 0, 0, 1410, 0, 22, 4094, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4094, 0, 5, 0, 0, 34, 3, 6, 5, 262, 116, 4088, 296, 1950, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4091, 1, 0, 0, 0, 0, 15, 3, 0, 0, 0, 0, 1, 0, 4052, 2316, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 12, 0, 0, 0, 33, 3728, 1271, 0, 7, 0, 4071, 4084, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 3, 4096, 0, 4094, 2, 2458, 0, 4078, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 53, 85, 3755, 4054, 0, 0, 0, 1, 0, 1, 0, 4089, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1868, 0, 0, 31, 0, 4095, 0, 4095, 0, 0, 4076, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15048, 17857, 15248, 14823, 18286, 15403, 16135, 18272] avg=16384.0000 max=18286 max_rank=4 min=14823 min_rank=3 imbalance_ratio=1.116089 max_load_reduction=112786 max_load_reduction_pct=0.860489 moved_to_echo=68096 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 0, 28, 4096, 0, 5, 3775, 0, 0, 0, 108, 44, 4069, 0, 0, 2431, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 0, 0, 0, 1438, 3291, 15, 0, 10, 0, 1410, 0, 52, 4080, 4091, 1511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 4083, 130, 3, 0, 0, 386, 0, 13, 0, 16, 0, 4069, 0, 1948, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4082, 0, 9, 7, 0, 93, 9, 0, 7, 46, 2315, 0, 0, 1, 4038, 2316, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 5, 0, 0, 0, 22, 0, 264, 4086, 78, 0, 143, 0, 3952, 3977, 968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 2, 10, 4035, 0, 4096, 0, 4091, 0, 3030, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1, 0, 3587, 2239, 55, 0, 0, 0, 4087, 0, 0, 3908, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 1867, 0, 2, 931, 33, 4092, 94, 68, 0, 245, 4057, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14939, 16980, 14968, 17252, 17575, 15893, 17904, 15561] avg=16384.0000 max=17904 max_rank=6 min=14939 min_rank=0 imbalance_ratio=1.092773 max_load_reduction=113168 max_load_reduction_pct=0.863403 moved_to_echo=66933 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[36, 1, 0, 72, 4028, 0, 0, 3968, 62, 0, 0, 0, 1, 4074, 1, 0, 2474, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 686, 4055, 0, 0, 0, 0, 1398, 0, 529, 3694, 4095, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3327, 0, 0, 4090, 0, 91, 0, 0, 1, 16, 0, 0, 49, 0, 4096, 0, 1947, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 9, 0, 0, 3, 0, 0, 0, 0, 5, 4083, 2296, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 3858, 0, 17, 0, 0, 0, 1, 3782, 4081, 198, 0, 204, 0, 4096, 4094, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 3941, 0, 3840, 0, 3951, 636, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 305, 3562, 4093, 0, 31, 0, 0, 0, 0, 2, 3985, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1864, 0, 1, 0, 0, 4096, 0, 3825, 5, 3, 4092, 2, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15084, 14591, 13898, 14869, 21321, 16477, 16840, 17992] avg=16384.0000 max=21321 max_rank=4 min=13898 min_rank=2 imbalance_ratio=1.301331 max_load_reduction=109751 max_load_reduction_pct=0.837334 moved_to_echo=72630 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[45, 38, 17, 0, 4054, 72, 0, 4096, 128, 0, 67, 206, 2, 3938, 0, 2, 2466, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 15, 33, 0, 3967, 0, 0, 0, 0, 1409, 0, 2990, 4061, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4080, 70, 0, 0, 392, 0, 0, 927, 0, 26, 0, 4073, 0, 1935, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 4092, 0, 0, 0, 0, 2362, 0, 0, 11, 0, 0, 0, 0, 0, 4019, 2314, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 0, 0, 1, 91, 0, 0, 1313, 0, 0, 7, 1, 3058, 4050, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4064, 9, 4028, 0, 1978, 0, 4042, 634, 3731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 16, 3760, 3742, 109, 0, 282, 3, 21, 0, 1319, 4093, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 1853, 0, 34, 0, 0, 2680, 53, 35, 0, 3793, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15501, 16579, 15895, 17033, 13520, 18492, 17483, 16569] avg=16384.0000 max=18492 max_rank=5 min=13520 min_rank=4 imbalance_ratio=1.128662 max_load_reduction=112580 max_load_reduction_pct=0.858917 moved_to_echo=66064 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 133, 1, 64, 3602, 55, 133, 695, 0, 0, 406, 0, 0, 3909, 3, 2, 2474, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4, 2219, 4036, 28, 1, 6, 0, 1403, 0, 2, 4006, 2977, 1997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 4068, 0, 220, 0, 1923, 97, 0, 1017, 0, 0, 0, 4096, 1, 1949, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2449, 3146, 3881, 14, 0, 41, 4, 254, 0, 1, 0, 0, 0, 73, 21, 4091, 2307, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 4096, 0, 0, 0, 0, 5, 0, 21, 3956, 0, 140, 56, 0, 3947, 4092, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 49, 0, 0, 0, 0, 12, 4060, 8, 4070, 29, 4038, 49, 4068, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 259, 0, 0, 25, 3389, 3913, 0, 1, 0, 821, 0, 1, 0, 3511, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3854, 1868, 0, 8, 0, 0, 4082, 170, 2547, 0, 1343, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11846, 16699, 17663, 16560, 17336, 17028, 16075, 17865] avg=16384.0000 max=17865 max_rank=7 min=11846 min_rank=0 imbalance_ratio=1.090393 max_load_reduction=113207 max_load_reduction_pct=0.863701 moved_to_echo=68304 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1152, 384, 128, 896, 512, 29312, 28288, 256, 128, 384, 896, 8832, 128, 1408, 28160, 31360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[896, 256, 128, 640, 32000, 512, 1152, 27136, 4352, 0, 1152, 2432, 6144, 32256, 896, 128, 19456, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[3584, 384, 128, 128, 12544, 31232, 128, 256, 128, 256, 11264, 0, 4480, 31360, 29824, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32256, 14976, 128, 128, 1024, 256, 26112, 512, 14720, 128, 8448, 32512, 128, 128, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 0, 2048, 128, 29568, 512, 31104, 128, 28544, 128, 26880, 4992, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[29824, 30336, 4352, 128, 128, 384, 6400, 384, 128, 128, 2560, 3712, 256, 128, 1408, 31104, 18560, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[31488, 0, 128, 32384, 256, 1536, 0, 6528, 640, 128, 5504, 128, 384, 128, 32512, 2176, 15616, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 28160, 128, 128, 128, 256, 128, 128, 9600, 19328, 1920, 256, 640, 128, 31232, 32512, 7808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[47, 0, 0, 4096, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 4093, 2719, 0, 0, 0, 18, 0, 0, 12, 4096, 0, 4096, 4095, 4096, 0, 1797, 0, 0, 0, 4077, 0, 0, 0, 4096, 0, 0, 0, 279, 4096, 4088, 0, 0, 0, 0, 4096, 4096, 0, 1, 4042, 720, 0, 0, 0, 2, 0, 4096, 4094, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 3996, 4095, 0, 1291, 0, 0, 0, 0, 0, 0, 0, 0, 59, 226, 0, 71, 0, 4028, 0, 0, 1007, 0, 0, 746, 4096, 4096, 0, 4096, 0, 3847, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 3794, 0, 4084, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 29, 0, 4094, 36, 0, 4092, 1, 0, 3119, 111, 0, 0, 4093, 4092, 1, 0, 0, 9, 9, 6, 18, 0, 3122, 0, 0, 0, 4096, 3, 4090, 847, 4056, 0, 0, 0, 2, 0, 3790, 23, 98, 0, 4008, 0, 0, 0, 568, 4068, 4013, 0, 91, 0, 0, 4094, 4089, 0, 4095, 4083, 0, 0, 0, 13, 0, 0, 4096, 4095, 0, 4096, 0, 6, 4085, 6, 0, 0, 0, 1058, 2053, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4, 1790, 0, 0, 4096, 0, 0, 0, 4077, 3947, 0, 4091, 0, 4034, 0, 1454, 0, 0, 0, 1, 2, 0, 46, 0, 16, 4, 4096, 0, 4091, 1039, 0, 4084, 0, 4096, 262, 0, 4, 1373] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 27, 0, 4096, 9, 0, 4096, 0, 11, 1221, 269, 0, 0, 4089, 4096, 56, 9, 0, 4075, 0, 0, 0, 3, 0, 30, 0, 9, 4000, 0, 3946, 4096, 4074, 2, 3512, 0, 0, 0, 1742, 6, 0, 0, 4075, 0, 175, 106, 2890, 4096, 2349, 0, 0, 0, 0, 4074, 4085, 2, 920, 0, 2211, 0, 670, 0, 4, 0, 4091, 4019, 0, 64, 0, 0, 4041, 2, 0, 0, 0, 3442, 3940, 31, 413, 179, 0, 0, 72, 0, 0, 3307, 0, 0, 0, 0, 0, 5, 2835, 2622, 12, 0, 0, 94, 0, 4045, 4089, 0, 4094, 193, 4072, 0, 0, 0, 0, 8, 262, 0, 0, 0, 0, 0, 0, 0, 63, 4056, 4041, 8, 3721, 13, 4096, 0, 7, 1, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 5, 0, 4096, 0, 47, 4096, 46, 0, 4005, 35, 0, 0, 4095, 4095, 0, 0, 0, 3454, 0, 0, 1, 1, 3615, 0, 0, 996, 4087, 0, 4094, 941, 3769, 0, 132, 0, 0, 0, 60, 0, 7, 1, 4075, 0, 87, 0, 4002, 4071, 4052, 0, 0, 0, 0, 4091, 4037, 0, 427, 4096, 30, 0, 0, 1300, 2, 0, 4096, 3892, 0, 0, 0, 0, 4086, 0, 1, 0, 256, 4094, 4095, 0, 1, 388, 0, 0, 0, 28, 0, 0, 156, 206, 3999, 0, 0, 8, 0, 5, 0, 391, 9, 0, 4, 4082, 4000, 0, 4089, 0, 4049, 0, 0, 0, 0, 0, 0, 113, 0, 15, 0, 36, 11, 3747, 0, 4087, 985, 5, 4081, 2, 4096, 6, 0, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 3755, 0, 0, 0, 2, 1, 0, 4090, 4068, 0, 0, 26, 0, 4094, 0, 464, 210, 19, 44, 0, 2654, 4090, 38, 4039, 1736, 4056, 0, 73, 0, 0, 5, 0, 0, 0, 502, 3660, 0, 4059, 7, 4093, 4096, 4000, 440, 0, 0, 0, 3884, 4040, 0, 2871, 197, 124, 0, 97, 0, 1, 0, 4096, 4022, 0, 3598, 0, 1, 4096, 0, 0, 0, 0, 0, 3156, 0, 3677, 323, 0, 0, 1418, 0, 0, 0, 0, 0, 0, 7, 922, 711, 481, 0, 0, 3842, 0, 0, 0, 4086, 2843, 3, 4094, 0, 4096, 194, 58, 36, 0, 0, 0, 0, 12, 5, 92, 0, 0, 0, 0, 4096, 3028, 0, 4088, 0, 3791, 1, 20, 481, 67] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[12, 4, 0, 4078, 12, 29, 4003, 141, 0, 4, 133, 0, 0, 4073, 4056, 0, 4, 0, 0, 2800, 0, 0, 0, 1, 16, 0, 0, 4096, 11, 3740, 4095, 4021, 0, 0, 1, 0, 0, 41, 0, 0, 9, 4096, 0, 0, 0, 1983, 3812, 3819, 0, 0, 0, 0, 4093, 4082, 0, 0, 2096, 0, 19, 212, 3, 260, 0, 4066, 4039, 0, 4096, 3, 0, 4093, 0, 0, 0, 17, 4096, 3741, 0, 1739, 178, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 260, 3, 0, 4064, 4096, 0, 4080, 0, 11, 0, 4049, 4096, 0, 2412, 0, 3907, 138, 0, 1, 0, 2, 548, 0, 0, 0, 2, 7, 1, 1335, 0, 4096, 402, 0, 4092, 0, 3390, 7, 54, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 40, 0, 4096, 359, 426, 3868, 0, 0, 343, 0, 0, 34, 4094, 4096, 0, 0, 0, 9, 4093, 0, 0, 0, 0, 2, 94, 0, 4074, 0, 3980, 1518, 4096, 0, 0, 0, 7, 0, 32, 0, 20, 0, 4096, 32, 3616, 0, 1803, 3967, 15, 0, 16, 0, 140, 4096, 3222, 1183, 12, 18, 274, 0, 9, 0, 0, 0, 4095, 4072, 0, 0, 0, 0, 4063, 120, 0, 0, 0, 4072, 4094, 15, 0, 90, 0, 0, 0, 0, 0, 4079, 0, 0, 3544, 0, 0, 18, 945, 3632, 12, 3, 0, 0, 13, 4075, 4048, 11, 4020, 15, 3934, 6, 0, 0, 0, 0, 146, 0, 0, 1, 0, 0, 0, 3908, 0, 4008, 3958, 63, 3960, 89, 4096, 2, 0, 0, 4085] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[6, 91, 54, 4071, 1, 0, 3898, 0, 19, 86, 0, 0, 73, 3487, 4096, 0, 0, 21, 4079, 886, 0, 0, 0, 9, 3, 35, 2633, 4071, 0, 2242, 4077, 3957, 3, 15, 4, 0, 12, 274, 70, 0, 45, 1, 14, 0, 0, 4096, 4092, 4026, 47, 0, 309, 4, 3958, 4022, 0, 4071, 1930, 0, 111, 0, 88, 0, 0, 4093, 4079, 0, 0, 16, 0, 3805, 0, 0, 344, 66, 4096, 3706, 0, 3538, 0, 1, 240, 0, 0, 0, 0, 2, 696, 27, 0, 0, 0, 4038, 650, 1, 3617, 25, 0, 77, 4096, 4083, 2, 3917, 0, 3859, 0, 7, 0, 2, 171, 0, 0, 0, 0, 0, 115, 21, 10, 3, 4083, 2848, 0, 3995, 8, 4043, 30, 338, 0, 1237] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[47, 0, 0, 2931, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2719, 0, 0, 0, 18, 0, 0, 12, 361, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1797, 0, 0, 0, 4077, 0, 0, 0, 4096, 0, 0, 0, 279, 4096, 4088, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 4042, 720, 0, 0, 0, 2, 0, 3727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2590, 0, 0, 1, 0, 3996, 4095, 0, 1291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 226, 0, 71, 0, 4028, 0, 0, 1007, 0, 3735, 1506, 1165, 897, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 746, 4096, 4096, 0, 4096, 0, 3847, 0, 24, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3199, 4091, 0, 3794, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15262, 19490, 18449, 16684, 16067, 13010, 16942, 15168] avg=16384.0000 max=19490 max_rank=1 min=13010 min_rank=5 imbalance_ratio=1.189575 max_load_reduction=111582 max_load_reduction_pct=0.851303 moved_to_echo=61187 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 29, 0, 2930, 36, 0, 4092, 1, 0, 3119, 111, 0, 0, 4093, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 6, 18, 0, 3122, 0, 0, 0, 361, 3, 4090, 847, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3790, 23, 98, 0, 4008, 0, 0, 0, 568, 4068, 4013, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 4094, 4089, 0, 4095, 4083, 0, 0, 0, 13, 0, 0, 3727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 6, 2583, 6, 0, 0, 0, 1058, 2053, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4, 1790, 0, 0, 4096, 0, 3735, 1502, 1164, 896, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3947, 0, 4091, 0, 4034, 0, 1454, 0, 0, 0, 1, 2, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 16, 4, 4096, 0, 3195, 1039, 0, 4084, 0, 4096, 262, 0, 4, 1373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18504, 12521, 16586, 20192, 13905, 13506, 17643, 18215] avg=16384.0000 max=20192 max_rank=3 min=12521 min_rank=1 imbalance_ratio=1.232422 max_load_reduction=110880 max_load_reduction_pct=0.845947 moved_to_echo=63269 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[12, 4, 0, 2919, 12, 29, 4003, 141, 0, 4, 133, 0, 0, 4073, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2800, 0, 0, 0, 1, 16, 0, 0, 361, 11, 3740, 4095, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 41, 0, 0, 9, 4096, 0, 0, 0, 1983, 3812, 3819, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4082, 0, 0, 2096, 0, 19, 212, 3, 260, 0, 3700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 4096, 3, 0, 2588, 0, 0, 0, 17, 4096, 3741, 0, 1739, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 260, 3, 0, 4064, 4096, 0, 4080, 0, 3735, 1505, 1159, 897, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4049, 4096, 0, 2412, 0, 3907, 138, 0, 1, 0, 2, 548, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7, 1, 1335, 0, 3199, 402, 0, 4092, 0, 3390, 7, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15386, 15049, 13777, 14465, 20497, 24208, 15201, 12489] avg=16384.0000 max=24208 max_rank=5 min=12489 min_rank=7 imbalance_ratio=1.477539 max_load_reduction=106864 max_load_reduction_pct=0.815308 moved_to_echo=72395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 40, 0, 2931, 359, 426, 3868, 0, 0, 343, 0, 0, 34, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4093, 0, 0, 0, 0, 2, 94, 0, 359, 0, 3980, 1518, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 32, 0, 20, 0, 4096, 32, 3616, 0, 1803, 3967, 15, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 140, 4096, 3222, 1183, 12, 18, 274, 0, 9, 0, 0, 0, 3727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 2569, 120, 0, 0, 0, 4072, 4094, 15, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 3544, 0, 0, 18, 945, 3632, 12, 3, 0, 3715, 1494, 1165, 878, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4075, 4048, 11, 4020, 15, 3934, 6, 0, 0, 0, 0, 146, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3908, 0, 3130, 3958, 63, 3960, 89, 4096, 2, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16191, 14151, 13604, 12697, 15032, 19800, 16305, 23292] avg=16384.0000 max=23292 max_rank=7 min=12697 min_rank=3 imbalance_ratio=1.421631 max_load_reduction=107780 max_load_reduction_pct=0.822296 moved_to_echo=74429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 27, 0, 2931, 9, 0, 4096, 0, 11, 1221, 269, 0, 0, 4089, 4096, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4075, 0, 0, 0, 3, 0, 30, 0, 9, 352, 0, 3946, 4096, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3512, 0, 0, 0, 1742, 6, 0, 0, 4075, 0, 175, 106, 2890, 4096, 2349, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4085, 2, 920, 0, 2211, 0, 670, 0, 4, 0, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 64, 0, 0, 2555, 2, 0, 0, 0, 3442, 3940, 31, 413, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 3307, 0, 0, 0, 0, 0, 5, 2835, 2622, 12, 0, 0, 3648, 1486, 1165, 888, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 4045, 4089, 0, 4094, 193, 4072, 0, 0, 0, 0, 8, 262, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 3168, 4041, 8, 3721, 13, 4096, 0, 7, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16805, 16594, 18969, 15689, 14645, 16355, 16894, 15121] avg=16384.0000 max=18969 max_rank=2 min=14645 min_rank=4 imbalance_ratio=1.157776 max_load_reduction=112103 max_load_reduction_pct=0.855278 moved_to_echo=63015 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 2931, 0, 47, 4096, 46, 0, 4005, 35, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3454, 0, 0, 1, 1, 3615, 0, 0, 996, 360, 0, 4094, 941, 3769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 60, 0, 7, 1, 4075, 0, 87, 0, 4002, 4071, 4052, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4037, 0, 427, 4096, 30, 0, 0, 1300, 2, 0, 3727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 0, 0, 0, 0, 2584, 0, 1, 0, 256, 4094, 4095, 0, 1, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 156, 206, 3999, 0, 0, 8, 0, 5, 0, 391, 9, 3727, 1502, 1165, 895, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4082, 4000, 0, 4089, 0, 4049, 0, 0, 0, 0, 0, 0, 113, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 36, 11, 3747, 0, 3192, 985, 5, 4081, 2, 4096, 6, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19355, 17231, 16503, 17710, 15311, 12407, 16374, 16181] avg=16384.0000 max=19355 max_rank=0 min=12407 min_rank=5 imbalance_ratio=1.181335 max_load_reduction=111717 max_load_reduction_pct=0.852333 moved_to_echo=60273 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2928, 0, 0, 3755, 0, 0, 0, 2, 1, 0, 4090, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 4094, 0, 464, 210, 19, 44, 0, 2654, 355, 38, 4039, 1736, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 5, 0, 0, 0, 502, 3660, 0, 4059, 7, 4093, 4096, 4000, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 3884, 4040, 0, 2871, 197, 124, 0, 97, 0, 1, 0, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 3598, 0, 1, 2586, 0, 0, 0, 0, 0, 3156, 0, 3677, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1418, 0, 0, 0, 0, 0, 0, 7, 922, 711, 481, 0, 0, 3842, 0, 3735, 1510, 1168, 900, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 2843, 3, 4094, 0, 4096, 194, 58, 36, 0, 0, 0, 0, 12, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 92, 0, 0, 0, 0, 3196, 3028, 0, 4088, 0, 3791, 1, 20, 481, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14844, 17735, 20516, 15370, 17363, 15013, 15462, 14769] avg=16384.0000 max=20516 max_rank=2 min=14769 min_rank=7 imbalance_ratio=1.252197 max_load_reduction=110556 max_load_reduction_pct=0.843475 moved_to_echo=62607 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[6, 91, 54, 2914, 1, 0, 3898, 0, 19, 86, 0, 0, 73, 3487, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4079, 886, 0, 0, 0, 9, 3, 35, 2633, 359, 0, 2242, 4077, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 15, 4, 0, 12, 274, 70, 0, 45, 1, 14, 0, 0, 4096, 4092, 4026, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 309, 4, 3958, 4022, 0, 4071, 1930, 0, 111, 0, 88, 0, 0, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 16, 0, 2406, 0, 0, 344, 66, 4096, 3706, 0, 3538, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, 0, 2, 696, 27, 0, 0, 0, 4038, 650, 1, 3617, 25, 3712, 1399, 1157, 894, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 4096, 4083, 2, 3917, 0, 3859, 0, 7, 0, 2, 171, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 21, 10, 3, 3189, 2848, 0, 3995, 8, 4043, 30, 338, 0, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14725, 18301, 12668, 18265, 18252, 16773, 16251, 15837] avg=16384.0000 max=18301 max_rank=1 min=12668 min_rank=2 imbalance_ratio=1.117004 max_load_reduction=112771 max_load_reduction_pct=0.860374 moved_to_echo=67113 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 15744, 14720, 128, 512, 256, 6784, 128, 256, 6400, 2944, 128, 30336, 21504, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 256, 128, 23424, 512, 512, 32000, 256, 128, 8832, 640, 128, 128, 32128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[512, 128, 384, 256, 32512, 31744, 1280, 12416, 16512, 3456, 256, 1024, 1408, 384, 0, 29824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32384, 0, 11904, 128, 128, 20480, 128, 128, 384, 384, 24960, 28928, 128, 10752, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 896, 32640, 31232, 128, 30848, 256, 31872, 384, 1664, 128, 128, 256, 1024, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 5632, 128, 128, 128, 10112, 128, 128, 640, 28160, 128, 8064, 128, 19840, 32384, 26368, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 256, 128, 13184, 128, 25472, 20480, 128, 31872, 128, 31744, 384, 512, 512, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 1536, 128, 0, 11520, 256, 1024, 7808, 384, 1024, 768, 18304, 11008, 128, 17152, 128, 29824, 11904, 9344, 7168, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[147, 0, 3389, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 4096, 342, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 619, 4096, 0, 0, 0, 4069, 4096, 0, 0, 37, 0, 2, 0, 4096, 2851, 0, 0, 0, 4096, 0, 4096, 0, 484, 0, 0, 9, 4093, 0, 0, 4096, 696, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 42, 0, 0, 1176, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 4096, 1, 4087, 2, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4076, 2426, 3840, 387, 1162, 0, 11, 0, 2909, 4095, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 68, 0, 0, 0, 10, 0, 46, 4093, 0, 4, 0, 0, 0, 4096, 1, 685, 2, 4, 2534, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 546, 0, 7, 0, 4096, 3623, 4095, 0, 0, 0, 4093, 4076, 0, 1, 4071, 3, 0, 1, 4080, 0, 0, 0, 0, 4083, 8, 4085, 0, 0, 1, 0, 0, 3546, 0, 9, 4096, 4046, 0, 56, 0, 5, 11, 4096, 0, 703, 13, 1135, 0, 0, 3995, 1, 13, 4083, 4070, 4091, 41, 0, 0, 0, 0, 4087, 30, 4, 0, 4087, 4074, 69, 1525, 1739, 4095, 3190, 0, 0, 21, 0, 26, 4095, 0, 0, 0, 0, 17, 16, 4096, 0, 1, 0, 2, 18, 13, 936, 231, 12, 3, 0, 4065, 4076, 1, 4013] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 33, 4077, 0, 0, 0, 4, 3, 4096, 25, 178, 24, 0, 1050, 0, 0, 0, 544, 4096, 45, 3071, 0, 0, 0, 0, 62, 0, 0, 4096, 1598, 4090, 40, 0, 21, 2739, 4072, 0, 0, 4064, 0, 0, 408, 4068, 0, 1, 28, 0, 4076, 1, 4092, 0, 12, 0, 1, 4095, 3541, 0, 0, 4047, 4048, 120, 0, 0, 0, 0, 3571, 0, 2, 0, 3802, 0, 0, 1287, 51, 0, 4065, 4081, 4096, 2, 0, 0, 2, 0, 4063, 0, 10, 11, 4075, 3769, 1, 3975, 579, 4076, 4069, 0, 0, 0, 0, 0, 4065, 21, 0, 0, 0, 18, 0, 4036, 0, 0, 0, 122, 0, 0, 0, 23, 21, 0, 0, 2547, 3949, 25, 4088] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4, 28, 250, 0, 0, 0, 0, 0, 4041, 0, 0, 7, 0, 0, 3650, 4096, 29, 0, 270, 9, 202, 0, 1, 0, 3753, 0, 0, 22, 0, 1945, 54, 0, 0, 2279, 4096, 7, 0, 0, 0, 0, 4031, 4054, 46, 0, 0, 0, 0, 0, 65, 0, 2, 230, 0, 3931, 32, 4096, 0, 3411, 0, 69, 0, 4067, 428, 0, 4095, 0, 1, 17, 0, 0, 251, 4090, 272, 4091, 0, 1, 0, 2901, 548, 1283, 3, 4096, 3968, 2115, 252, 2, 0, 157, 0, 4096, 113, 0, 0, 3978, 2141, 0, 4059, 3659, 4096, 2412, 2, 0, 3783, 0, 0, 4096, 21, 0, 4, 0, 0, 0, 4076, 0, 8, 0, 0, 3948, 1469, 16, 0, 0, 320, 0, 3697, 3774, 0, 3956] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[47, 2, 28, 3, 0, 0, 12, 4, 4019, 0, 2, 98, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4060, 0, 0, 0, 0, 7, 38, 0, 0, 28, 4095, 4078, 4091, 0, 0, 0, 4094, 4095, 0, 0, 3176, 0, 0, 170, 3237, 0, 0, 0, 0, 4056, 0, 4092, 0, 0, 0, 0, 115, 4088, 1, 4, 4091, 491, 0, 2, 0, 32, 0, 4094, 0, 33, 3, 3870, 153, 2753, 286, 0, 0, 4069, 4073, 4095, 17, 0, 201, 2, 737, 4096, 0, 0, 1, 4095, 4096, 0, 3532, 66, 4096, 3984, 0, 0, 0, 0, 0, 4090, 0, 4, 0, 0, 0, 0, 4091, 4, 0, 1, 2578, 1133, 0, 0, 3730, 0, 2, 0, 2439, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4030, 39, 1, 0, 30, 0, 4096, 0, 13, 1649, 115, 0, 4096, 15, 0, 0, 314, 0, 1, 0, 0, 119, 4096, 0, 0, 31, 1, 987, 251, 0, 20, 0, 3245, 2418, 4096, 24, 36, 0, 2302, 4096, 0, 0, 47, 505, 2196, 0, 1363, 282, 0, 25, 0, 4088, 0, 4055, 0, 6, 0, 0, 0, 3993, 4022, 0, 4092, 794, 0, 0, 0, 0, 0, 4060, 0, 0, 718, 0, 1709, 3702, 3373, 0, 0, 4067, 3951, 859, 371, 0, 0, 0, 0, 3620, 0, 0, 9, 3720, 3898, 0, 3808, 0, 3977, 3500, 0, 69, 1906, 1, 209, 4010, 0, 0, 132, 0, 1, 450, 3963, 0, 31, 7, 4096, 1154, 0, 0, 0, 0, 0, 2, 0, 4078, 7, 4025] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 1, 18, 0, 0, 4094, 2, 36, 4096, 0, 0, 4057, 2, 3, 0, 1, 0, 21, 0, 81, 0, 4087, 0, 0, 14, 0, 16, 0, 0, 0, 0, 4084, 3798, 4017, 0, 352, 0, 1656, 4096, 266, 9, 0, 39, 530, 0, 4096, 0, 0, 0, 118, 4084, 0, 4095, 7, 157, 0, 0, 4, 4096, 0, 0, 4095, 4092, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 701, 0, 3186, 3821, 0, 0, 4096, 4053, 3967, 16, 1, 0, 0, 0, 4095, 0, 0, 0, 4085, 4054, 0, 166, 1, 4008, 3943, 0, 0, 0, 2, 29, 4096, 3, 4, 0, 0, 0, 1, 3727, 157, 0, 5, 3778, 0, 2033, 0, 354, 87, 0, 0, 4096, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 53, 4, 0, 0, 3968, 3, 147, 4047, 0, 57, 3862, 908, 1, 4, 1, 49, 0, 0, 0, 0, 4095, 0, 435, 130, 0, 0, 0, 225, 0, 0, 3685, 4086, 3722, 94, 10, 0, 4001, 4023, 0, 0, 0, 44, 228, 127, 3761, 0, 0, 52, 0, 4078, 0, 3925, 0, 0, 24, 73, 0, 4088, 417, 0, 4096, 3316, 1, 0, 0, 0, 0, 3841, 8, 0, 0, 873, 0, 0, 4094, 278, 167, 4096, 4032, 3967, 0, 5, 703, 13, 0, 2907, 94, 0, 0, 3995, 3726, 0, 4085, 0, 3973, 4096, 6, 0, 173, 14, 6, 4067, 0, 9, 122, 16, 1, 0, 4012, 0, 0, 0, 4089, 1, 107, 0, 0, 0, 0, 0, 3650, 3918, 0, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[147, 0, 3389, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 4096, 342, 3120, 908, 844, 302, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 3961, 2588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 619, 4096, 0, 0, 0, 4069, 664, 0, 0, 37, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2851, 0, 0, 0, 4096, 0, 3188, 0, 484, 0, 0, 9, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3794, 696, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 42, 0, 0, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 976, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 2, 4096, 4096, 0, 0, 0, 0, 0, 3893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4076, 2426, 3840, 387, 1162, 0, 11, 0, 2909, 4095, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17456, 14746, 13583, 18817, 9800, 17359, 16174, 23137] avg=16384.0000 max=23137 max_rank=7 min=9800 min_rank=4 imbalance_ratio=1.412170 max_load_reduction=107935 max_load_reduction_pct=0.823479 moved_to_echo=66470 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 33, 4077, 0, 0, 0, 4, 3, 4096, 25, 3109, 907, 839, 298, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 24, 0, 1050, 0, 0, 0, 544, 4096, 45, 3071, 0, 0, 0, 0, 62, 4065, 3954, 2573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1598, 4090, 40, 0, 21, 2739, 660, 0, 0, 4064, 0, 0, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 1, 28, 0, 4076, 1, 3185, 0, 12, 0, 1, 4095, 3541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3749, 4048, 120, 0, 0, 0, 0, 3571, 0, 2, 0, 3802, 0, 0, 1287, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 972, 4096, 2, 0, 0, 2, 0, 4063, 0, 10, 11, 4075, 3769, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 579, 4076, 4069, 0, 0, 0, 0, 0, 3864, 21, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 122, 0, 0, 0, 23, 21, 0, 0, 2547, 3949, 25, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13596, 19662, 17716, 19008, 16630, 17001, 16602, 10857] avg=16384.0000 max=19662 max_rank=1 min=10857 min_rank=7 imbalance_ratio=1.200073 max_load_reduction=111410 max_load_reduction_pct=0.849991 moved_to_echo=61090 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 68, 0, 0, 0, 10, 0, 46, 4093, 0, 4, 0, 0, 0, 4096, 1, 3101, 905, 840, 302, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 685, 2, 4, 2534, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 546, 0, 4083, 3881, 2575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 3623, 4095, 0, 0, 0, 4093, 661, 0, 1, 4071, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4083, 8, 3180, 0, 0, 1, 0, 0, 3546, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3794, 4046, 0, 56, 0, 5, 11, 4096, 0, 703, 13, 1135, 0, 0, 3995, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 969, 4091, 41, 0, 0, 0, 0, 4087, 30, 4, 0, 4087, 4074, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1525, 1739, 4095, 3190, 0, 0, 21, 0, 26, 3892, 0, 0, 0, 0, 17, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 2, 18, 13, 936, 231, 12, 3, 0, 4065, 4076, 1, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13669, 18418, 20651, 14907, 17855, 17465, 14521, 13586] avg=16384.0000 max=20651 max_rank=2 min=13586 min_rank=7 imbalance_ratio=1.260437 max_load_reduction=110421 max_load_reduction_pct=0.842445 moved_to_echo=63427 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4030, 39, 1, 0, 30, 0, 4096, 0, 13, 1649, 115, 0, 4096, 15, 3010, 898, 844, 302, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 1, 0, 0, 119, 4096, 0, 0, 31, 1, 987, 251, 0, 4067, 3893, 2588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 3245, 2418, 4096, 24, 36, 0, 2302, 664, 0, 0, 47, 505, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1363, 282, 0, 25, 0, 4088, 0, 3157, 0, 6, 0, 0, 0, 3993, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3790, 794, 0, 0, 0, 0, 0, 4060, 0, 0, 718, 0, 1709, 3702, 3373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 941, 859, 371, 0, 0, 0, 0, 3620, 0, 0, 9, 3720, 3898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3808, 0, 3977, 3500, 0, 69, 1906, 1, 209, 3811, 0, 0, 132, 0, 1, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 31, 7, 4096, 1154, 0, 0, 0, 0, 0, 2, 0, 4078, 7, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19337, 16348, 15553, 16936, 18146, 13418, 17864, 13470] avg=16384.0000 max=19337 max_rank=0 min=13418 min_rank=5 imbalance_ratio=1.180237 max_load_reduction=111735 max_load_reduction_pct=0.852470 moved_to_echo=62898 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[47, 2, 28, 3, 0, 0, 12, 4, 4019, 0, 2, 98, 1, 0, 4096, 0, 3103, 907, 844, 302, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4060, 0, 0, 0, 0, 7, 38, 0, 4069, 3961, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4095, 4078, 4091, 0, 0, 0, 4094, 664, 0, 0, 3176, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3237, 0, 0, 0, 0, 4056, 0, 3185, 0, 0, 0, 0, 115, 4088, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3789, 491, 0, 2, 0, 32, 0, 4094, 0, 33, 3, 3870, 153, 2753, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 970, 4095, 17, 0, 201, 2, 737, 4096, 0, 0, 1, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3532, 66, 4096, 3984, 0, 0, 0, 0, 0, 3887, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 1, 2578, 1133, 0, 0, 3730, 0, 2, 0, 2439, 4096, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13671, 14725, 20396, 14686, 15506, 18310, 15569, 18209] avg=16384.0000 max=20396 max_rank=2 min=13671 min_rank=0 imbalance_ratio=1.244873 max_load_reduction=110676 max_load_reduction_pct=0.844391 moved_to_echo=67594 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4, 28, 250, 0, 0, 0, 0, 0, 4041, 0, 0, 7, 0, 0, 3650, 4096, 3026, 911, 838, 305, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 270, 9, 202, 0, 1, 0, 3753, 0, 0, 22, 0, 1945, 54, 0, 4096, 3830, 2566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2279, 4096, 7, 0, 0, 0, 0, 4031, 650, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 2, 230, 0, 3931, 32, 3185, 0, 3411, 0, 69, 0, 4067, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3790, 0, 1, 17, 0, 0, 251, 4090, 272, 4091, 0, 1, 0, 2901, 548, 1283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 942, 2115, 252, 2, 0, 157, 0, 4096, 113, 0, 0, 3978, 2141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 3659, 4096, 2412, 2, 0, 3783, 0, 0, 3890, 21, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 8, 0, 0, 3948, 1469, 16, 0, 0, 320, 0, 3697, 3774, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17362, 16777, 11109, 15420, 17245, 13799, 21926, 17434] avg=16384.0000 max=21926 max_rank=6 min=11109 min_rank=2 imbalance_ratio=1.338257 max_load_reduction=109146 max_load_reduction_pct=0.832718 moved_to_echo=70404 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 1, 18, 0, 0, 4094, 2, 36, 4096, 0, 0, 4057, 2, 3088, 907, 844, 302, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 21, 0, 81, 0, 4087, 0, 0, 14, 0, 16, 0, 0, 4096, 3961, 2588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 3798, 4017, 0, 352, 0, 1656, 664, 266, 9, 0, 39, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 118, 4084, 0, 3188, 7, 157, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3793, 4092, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 701, 0, 3186, 3821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 965, 3967, 16, 1, 0, 0, 0, 4095, 0, 0, 0, 4085, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 1, 4008, 3943, 0, 0, 0, 2, 29, 3893, 3, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3727, 157, 0, 5, 3778, 0, 2033, 0, 354, 87, 0, 0, 4096, 4096, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17653, 14868, 15415, 15750, 19685, 17183, 12050, 18468] avg=16384.0000 max=19685 max_rank=4 min=12050 min_rank=6 imbalance_ratio=1.201477 max_load_reduction=111387 max_load_reduction_pct=0.849815 moved_to_echo=67386 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 53, 4, 0, 0, 3968, 3, 147, 4047, 0, 57, 3862, 908, 3072, 870, 829, 302, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 1, 49, 0, 0, 0, 0, 4095, 0, 435, 130, 0, 0, 0, 225, 4096, 3950, 2542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3685, 4086, 3722, 94, 10, 0, 4001, 652, 0, 0, 0, 44, 228, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 0, 0, 52, 0, 4078, 0, 3055, 0, 0, 24, 73, 0, 4088, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3794, 3316, 1, 0, 0, 0, 0, 3841, 8, 0, 0, 873, 0, 0, 4094, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 960, 3967, 0, 5, 703, 13, 0, 2907, 94, 0, 0, 3995, 3726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 3973, 4096, 6, 0, 173, 14, 6, 3865, 0, 9, 122, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 4089, 1, 107, 0, 0, 0, 0, 0, 3650, 3918, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18328, 15528, 16649, 15548, 16205, 16537, 16366, 15911] avg=16384.0000 max=18328 max_rank=0 min=15528 min_rank=1 imbalance_ratio=1.118652 max_load_reduction=112744 max_load_reduction_pct=0.860168 moved_to_echo=65019 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32128, 256, 128, 128, 18816, 8704, 7552, 1408, 5504, 128, 384, 128, 23424, 32128, 128, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[25344, 6144, 32512, 29312, 128, 128, 5888, 128, 384, 31104, 128, 128, 384, 128, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 0, 7808, 27392, 768, 128, 1024, 256, 768, 31104, 256, 128, 128, 32256, 29952, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 2432, 31616, 20352, 28288, 256, 512, 128, 27008, 5376, 384, 128, 11520, 640, 3072, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[896, 128, 640, 3712, 256, 128, 128, 768, 32384, 128, 3584, 256, 128, 3072, 896, 384, 32768, 31488, 20608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[24832, 3200, 128, 384, 128, 32512, 128, 25344, 128, 4096, 128, 256, 4224, 31616, 4992, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[30336, 17536, 128, 128, 0, 128, 384, 32000, 384, 4864, 768, 10496, 1920, 12544, 18688, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[256, 128, 7808, 128, 128, 128, 128, 128, 32512, 128, 256, 9984, 128, 128, 32128, 5504, 24704, 7296, 6784, 2432, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4070, 1, 130, 4091, 0, 4095, 0, 0, 0, 3, 5, 4095, 0, 3, 91, 6, 4019, 0, 4089, 4090, 0, 0, 330, 15, 4068, 0, 2974, 2, 0, 43, 6, 0, 0, 0, 7, 9, 0, 0, 13, 0, 0, 2, 0, 0, 0, 0, 68, 0, 4073, 0, 4086, 0, 0, 0, 0, 0, 0, 1092, 28, 0, 0, 4087, 0, 0, 4096, 0, 0, 254, 7, 0, 308, 0, 3332, 0, 0, 49, 4096, 0, 0, 1, 14, 4093, 28, 0, 4096, 4096, 0, 10, 0, 4061, 4080, 0, 12, 3627, 4058, 0, 6, 5, 4096, 0, 0, 425, 4033, 1, 0, 0, 4089, 4096, 16, 0, 4086, 0, 0, 4087, 0, 0, 6, 4096, 3979, 0, 0, 0, 0, 13, 3940, 0, 4089] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 3798, 314, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 15, 23, 3240, 53, 0, 3306, 4096, 1, 0, 2807, 0, 3728, 825, 3221, 0, 0, 79, 0, 0, 0, 81, 5, 0, 0, 813, 4012, 12, 0, 1, 0, 0, 0, 0, 41, 1, 3825, 0, 3208, 545, 66, 69, 4, 5, 12, 0, 3841, 0, 0, 4095, 0, 0, 4062, 90, 0, 1348, 0, 0, 73, 4086, 4032, 0, 0, 2, 4089, 1749, 0, 0, 697, 0, 0, 0, 1161, 2149, 3, 116, 29, 3763, 3505, 0, 0, 2726, 4000, 0, 16, 0, 4091, 0, 0, 0, 17, 375, 75, 0, 383, 4096, 0, 0, 3936, 623, 0, 4030, 0, 992, 136, 4096, 3913, 0, 0, 0, 0, 0, 4087, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 4096, 0, 4074, 0, 0, 0, 1182, 86, 4080, 0, 4064, 388, 1, 3748, 0, 4033, 4081, 0, 0, 20, 0, 4079, 13, 3901, 0, 0, 437, 0, 1, 0, 0, 8, 0, 0, 1682, 38, 0, 0, 0, 1, 0, 898, 2, 0, 0, 2928, 0, 4096, 0, 31, 0, 0, 0, 1, 0, 255, 0, 15, 4093, 0, 4, 3782, 0, 0, 3709, 0, 0, 3648, 4073, 306, 0, 0, 3, 4095, 1, 0, 0, 0, 882, 0, 94, 3916, 3983, 5, 3, 0, 3972, 4096, 0, 0, 4096, 0, 0, 207, 0, 4093, 3, 0, 19, 526, 0, 0, 0, 4094, 4096, 0, 8, 4096, 0, 0, 4095, 49, 0, 0, 450, 4053, 0, 0, 0, 0, 0, 4095, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 6, 0, 4096, 0, 0, 5, 0, 4062, 0, 4096, 4096, 0, 0, 0, 0, 3819, 0, 4096, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 3915, 14, 0, 0, 0, 0, 0, 0, 0, 60, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 73, 4091, 0, 2044, 3692, 0, 0, 327, 0, 1, 2, 0, 0, 4096, 0, 0, 0, 40, 45, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4085, 0, 0, 4091, 3815, 0, 0, 0, 4096, 0, 0, 0, 2425, 0, 0, 0, 4033, 4096, 0, 170, 4096, 0, 0, 4096, 0, 0, 197, 4053, 4096, 0, 0, 0, 0, 0, 4073, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[27, 3636, 0, 0, 4096, 0, 4080, 0, 0, 0, 4059, 20, 4096, 0, 3822, 71, 0, 4073, 0, 4096, 4085, 0, 0, 822, 31, 4082, 0, 4096, 0, 0, 239, 0, 5, 0, 1, 0, 56, 0, 2, 114, 6, 0, 598, 42, 7, 6, 0, 0, 218, 4063, 0, 3947, 0, 0, 18, 0, 0, 0, 3584, 0, 3, 0, 4033, 0, 0, 4096, 0, 0, 314, 0, 0, 0, 5, 2, 0, 0, 98, 4095, 0, 0, 0, 0, 4096, 11, 33, 1844, 3979, 2, 0, 0, 4096, 4095, 0, 88, 4015, 4084, 0, 9, 0, 4092, 11, 0, 59, 7, 0, 0, 0, 2829, 4069, 1, 0, 4090, 0, 0, 4095, 0, 10, 0, 3527, 4054, 35, 0, 0, 0, 0, 3017, 0, 4080] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4065, 10, 0, 3979, 0, 4085, 0, 839, 226, 4096, 0, 4096, 0, 3766, 72, 1, 2702, 64, 3300, 3815, 0, 41, 0, 0, 4096, 0, 4033, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 1, 0, 45, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4077, 0, 53, 0, 0, 674, 0, 864, 0, 0, 0, 0, 0, 1008, 1851, 79, 0, 1, 0, 0, 4081, 3026, 0, 0, 0, 0, 3292, 0, 323, 634, 0, 3519, 109, 0, 4096, 4004, 50, 0, 59, 4094, 4021, 0, 0, 2993, 4090, 0, 954, 27, 3957, 0, 0, 1934, 3941, 0, 4, 0, 1330, 4096, 0, 0, 4082, 0, 23, 4043, 0, 0, 0, 3890, 4042, 69, 0, 0, 11, 0, 142, 61, 3996] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4069, 17, 0, 4067, 5, 4096, 0, 0, 0, 3916, 0, 3879, 0, 4096, 1985, 74, 4090, 0, 4061, 4032, 0, 34, 0, 0, 3940, 0, 1818, 4, 0, 188, 0, 28, 0, 0, 0, 0, 0, 3986, 4096, 14, 0, 0, 0, 0, 14, 0, 18, 0, 4071, 62, 4072, 92, 7, 11, 0, 0, 0, 222, 172, 0, 32, 4034, 0, 417, 4079, 5, 1110, 4050, 1, 0, 532, 3044, 213, 0, 0, 14, 4096, 0, 0, 0, 0, 70, 0, 0, 4085, 3524, 0, 9, 0, 4092, 4095, 0, 0, 589, 0, 0, 0, 43, 4070, 31, 0, 2, 3141, 0, 0, 9, 311, 4092, 0, 0, 4054, 0, 0, 3990, 0, 0, 0, 4000, 3735, 37, 124, 0, 0, 1, 14, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4053, 4, 158, 4086, 0, 3993, 0, 5, 130, 3741, 199, 4047, 0, 4084, 0, 0, 4070, 0, 3562, 4007, 0, 21, 0, 0, 3999, 0, 3951, 0, 63, 0, 0, 13, 3, 78, 75, 550, 110, 23, 3975, 0, 0, 104, 0, 0, 0, 0, 83, 0, 4069, 0, 3879, 0, 1, 0, 19, 119, 0, 0, 0, 45, 0, 3812, 0, 100, 4027, 0, 0, 0, 0, 0, 8, 0, 2176, 37, 0, 273, 4090, 0, 0, 11, 1, 2302, 0, 0, 4095, 3260, 0, 12, 1, 3985, 4006, 0, 0, 3968, 0, 0, 0, 0, 4013, 1, 0, 0, 4096, 0, 3, 0, 4096, 4094, 9, 7, 4082, 190, 0, 4096, 38, 0, 910, 4, 4044, 0, 0, 0, 328, 2, 3437, 45, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 3146, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 4096, 557, 0, 0, 0, 0, 3819, 0, 4096, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3915, 14, 0, 0, 0, 0, 0, 0, 0, 60, 4096, 4096, 3539, 1610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 2112, 1023, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 4091, 0, 2044, 3692, 0, 0, 327, 0, 1, 2, 0, 0, 4096, 0, 0, 950, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 45, 0, 0, 4096, 4096, 0, 0, 0, 0, 1452, 0, 0, 4091, 3815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2425, 0, 0, 0, 4033, 2786, 0, 170, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 197, 4053, 4096, 0, 0, 0, 0, 0, 4073, 0, 1984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11349, 16643, 17330, 15816, 16194, 17635, 17606, 18499] avg=16384.0000 max=18499 max_rank=7 min=11349 min_rank=0 imbalance_ratio=1.129089 max_load_reduction=112573 max_load_reduction_pct=0.858864 moved_to_echo=69934 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4070, 1, 130, 3142, 0, 0, 0, 0, 0, 3, 5, 4095, 0, 3, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4019, 0, 4089, 556, 0, 0, 330, 15, 4068, 0, 2974, 2, 0, 43, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 0, 0, 13, 0, 0, 2, 0, 0, 0, 0, 68, 4095, 4061, 3534, 1608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 4086, 0, 0, 0, 0, 0, 0, 1092, 28, 0, 0, 4087, 0, 2108, 1022, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 254, 7, 0, 308, 0, 3332, 0, 0, 49, 4096, 0, 0, 949, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 4093, 28, 0, 4096, 4096, 0, 10, 0, 0, 1450, 0, 12, 3627, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 4096, 0, 0, 425, 4033, 1, 0, 0, 4089, 2786, 16, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 6, 4096, 3979, 0, 0, 0, 0, 13, 3940, 0, 1981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11540, 16108, 13397, 16888, 14009, 21485, 19543, 18102] avg=16384.0000 max=21485 max_rank=5 min=11540 min_rank=0 imbalance_ratio=1.311340 max_load_reduction=109587 max_load_reduction_pct=0.836082 moved_to_echo=73139 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3798, 314, 0, 3140, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 15, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3240, 53, 0, 3306, 554, 1, 0, 2807, 0, 3728, 825, 3221, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 5, 0, 0, 813, 4012, 12, 0, 1, 0, 0, 0, 0, 41, 4096, 3763, 3542, 1384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3825, 0, 3208, 545, 66, 69, 4, 5, 12, 0, 3841, 0, 0, 4095, 0, 2117, 882, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 90, 0, 1348, 0, 0, 73, 4086, 4032, 0, 0, 2, 4089, 1749, 0, 955, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 697, 0, 0, 0, 1161, 2149, 3, 116, 29, 0, 1239, 0, 0, 2726, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4091, 0, 0, 0, 17, 375, 75, 0, 383, 2780, 0, 0, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 623, 0, 4030, 0, 992, 136, 4096, 3913, 0, 0, 0, 0, 0, 4087, 0, 1979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11387, 17814, 17750, 19064, 21408, 12120, 11673, 19856] avg=16384.0000 max=21408 max_rank=4 min=11387 min_rank=0 imbalance_ratio=1.306641 max_load_reduction=109664 max_load_reduction_pct=0.836670 moved_to_echo=65057 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[27, 3636, 0, 0, 3146, 0, 0, 0, 0, 0, 4059, 20, 4096, 0, 3822, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 4096, 556, 0, 0, 822, 31, 4082, 0, 4096, 0, 0, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 56, 0, 2, 114, 6, 0, 598, 42, 7, 6, 0, 0, 4080, 4096, 3529, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 4063, 0, 3947, 0, 0, 18, 0, 0, 0, 3584, 0, 3, 0, 4033, 0, 2104, 1026, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 314, 0, 0, 0, 5, 2, 0, 0, 98, 4095, 0, 0, 950, 912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 33, 1844, 3979, 2, 0, 0, 0, 1455, 0, 88, 4015, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4092, 11, 0, 59, 7, 0, 0, 0, 2829, 2768, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 10, 0, 3527, 4054, 35, 0, 0, 0, 0, 3017, 0, 1976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18877, 17995, 14156, 19385, 10472, 19607, 13866, 16714] avg=16384.0000 max=19607 max_rank=5 min=10472 min_rank=4 imbalance_ratio=1.196716 max_load_reduction=111465 max_load_reduction_pct=0.850410 moved_to_echo=60659 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4065, 10, 0, 3056, 0, 0, 0, 839, 226, 4096, 0, 4096, 0, 3766, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2702, 64, 3300, 519, 0, 41, 0, 0, 4096, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 1, 0, 45, 0, 0, 0, 0, 0, 0, 4085, 4094, 3296, 1584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4077, 0, 53, 0, 0, 674, 0, 864, 0, 0, 0, 0, 0, 2060, 1007, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1851, 79, 0, 1, 0, 0, 4081, 3026, 0, 0, 0, 0, 3292, 0, 323, 923, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 634, 0, 3519, 109, 0, 4096, 4004, 50, 0, 59, 0, 1430, 0, 0, 2993, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 954, 27, 3957, 0, 0, 1934, 3941, 0, 4, 0, 1330, 2786, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4043, 0, 0, 0, 3890, 4042, 69, 0, 0, 11, 0, 142, 61, 1936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20226, 14756, 13149, 13223, 15502, 20984, 19015, 14217] avg=16384.0000 max=20984 max_rank=5 min=13149 min_rank=2 imbalance_ratio=1.280762 max_load_reduction=110088 max_load_reduction_pct=0.839905 moved_to_echo=69718 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 3146, 0, 0, 0, 0, 0, 1182, 86, 4080, 0, 4064, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3748, 0, 4033, 555, 0, 0, 20, 0, 4079, 13, 3901, 0, 0, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 1682, 38, 0, 0, 0, 1, 0, 898, 2, 0, 4074, 3972, 3526, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2928, 0, 4096, 0, 31, 0, 0, 0, 1, 0, 255, 0, 15, 4093, 0, 2109, 1026, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3782, 0, 0, 3709, 0, 0, 3648, 4073, 306, 0, 0, 3, 4095, 1, 0, 950, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 882, 0, 94, 3916, 3983, 5, 3, 0, 0, 1456, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 4093, 3, 0, 19, 526, 0, 0, 0, 4094, 2786, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 49, 0, 0, 450, 4053, 0, 0, 0, 0, 0, 4095, 0, 1982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17043, 16787, 15816, 14946, 21489, 14435, 15832, 14724] avg=16384.0000 max=21489 max_rank=4 min=14435 min_rank=5 imbalance_ratio=1.311584 max_load_reduction=109583 max_load_reduction_pct=0.836052 moved_to_echo=66480 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4069, 17, 0, 3124, 5, 0, 0, 0, 0, 3916, 0, 3879, 0, 4096, 1985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 4090, 0, 4061, 549, 0, 34, 0, 0, 3940, 0, 1818, 4, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 3986, 4096, 14, 0, 0, 0, 0, 14, 0, 18, 4096, 4092, 3483, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 62, 4072, 92, 7, 11, 0, 0, 0, 222, 172, 0, 32, 4034, 0, 2109, 1026, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 4079, 5, 1110, 4050, 1, 0, 532, 3044, 213, 0, 0, 14, 4096, 0, 0, 943, 917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 4085, 3524, 0, 9, 0, 0, 1455, 0, 0, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 4070, 31, 0, 2, 3141, 0, 0, 9, 311, 2784, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 0, 0, 0, 4000, 3735, 37, 124, 0, 0, 1, 14, 0, 1982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21091, 14758, 21441, 16301, 19421, 9732, 14445, 13883] avg=16384.0000 max=21441 max_rank=2 min=9732 min_rank=5 imbalance_ratio=1.308655 max_load_reduction=109631 max_load_reduction_pct=0.836418 moved_to_echo=57481 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4053, 4, 158, 3138, 0, 0, 0, 5, 130, 3741, 199, 4047, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 3562, 545, 0, 21, 0, 0, 3999, 0, 3951, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3, 78, 75, 550, 110, 23, 3975, 0, 0, 104, 0, 0, 0, 0, 83, 3993, 3985, 3462, 1579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 3879, 0, 1, 0, 19, 119, 0, 0, 0, 45, 0, 3812, 0, 2111, 1003, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 4027, 0, 0, 0, 0, 0, 8, 0, 2176, 37, 0, 273, 4090, 0, 0, 948, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 2302, 0, 0, 4095, 3260, 0, 12, 1, 0, 1424, 0, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 1, 0, 0, 4096, 0, 3, 0, 4096, 2785, 9, 7, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 4096, 38, 0, 910, 4, 4044, 0, 0, 0, 328, 2, 3437, 45, 1983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19559, 16211, 18033, 15449, 12577, 15074, 19092, 15077] avg=16384.0000 max=19559 max_rank=0 min=12577 min_rank=4 imbalance_ratio=1.193787 max_load_reduction=111513 max_load_reduction_pct=0.850777 moved_to_echo=61820 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[896, 128, 32640, 128, 1024, 1280, 24192, 32000, 256, 128, 0, 384, 128, 22912, 128, 15872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32000, 384, 384, 25088, 128, 0, 0, 896, 384, 17024, 384, 32512, 128, 19968, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3328, 26880, 128, 30592, 4480, 128, 128, 4096, 128, 31872, 896, 28160, 128, 128, 1024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 1280, 128, 32512, 128, 0, 2560, 18304, 384, 128, 128, 21248, 22272, 128, 256, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 256, 128, 640, 128, 10496, 12288, 128, 128, 768, 128, 128, 1024, 128, 384, 32640, 32256, 28032, 12672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[768, 768, 15104, 256, 128, 27392, 29184, 128, 256, 128, 0, 11392, 0, 128, 26112, 20096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1664, 30208, 256, 3200, 13440, 128, 0, 9088, 14336, 10112, 128, 0, 512, 32000, 1792, 384, 7680, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 31232, 128, 31488, 640, 256, 128, 128, 896, 128, 5888, 4352, 128, 128, 28288, 0, 16896, 8064, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[3660, 0, 3949, 4, 21, 4073, 0, 0, 4, 0, 0, 0, 0, 0, 118, 141, 0, 0, 0, 0, 14, 0, 12, 3835, 52, 0, 0, 0, 4046, 0, 0, 4096, 0, 0, 4096, 0, 37, 4085, 13, 266, 1, 4025, 0, 0, 0, 0, 3827, 0, 4, 3945, 0, 0, 332, 20, 3992, 0, 82, 0, 0, 4096, 2, 0, 0, 1507, 4031, 230, 0, 26, 0, 4096, 197, 0, 0, 464, 0, 14, 125, 4072, 0, 0, 4095, 33, 0, 146, 0, 2449, 360, 102, 4091, 0, 4065, 4085, 0, 0, 240, 3802, 0, 31, 4096, 4096, 0, 0, 4079, 3734, 2, 0, 66, 0, 8, 13, 3821, 0, 0, 4023, 11, 23, 5, 4048, 2, 0, 12, 0, 0, 0, 63, 4057, 3776, 4028] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4088, 0, 4096, 92, 0, 4095, 0, 0, 0, 2927, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 25, 0, 0, 0, 4022, 0, 0, 0, 0, 3680, 0, 0, 4096, 0, 3758, 4096, 0, 0, 4096, 0, 0, 0, 2306, 0, 0, 0, 0, 569, 0, 0, 8, 0, 0, 0, 1, 4096, 0, 440, 0, 0, 4096, 0, 0, 0, 3269, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4, 10, 0, 4, 3857, 0, 0, 4096, 0, 4081, 4096, 0, 0, 0, 2540, 0, 0, 4096, 4094, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 1255, 0, 0, 4092, 0, 0, 3034, 4096, 0, 1, 0, 0, 0, 3, 0, 4096, 4080, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4046, 0, 4082, 0, 582, 3447, 0, 0, 0, 4092, 11, 0, 0, 16, 37, 1, 4, 64, 2, 472, 0, 1, 5, 3597, 0, 0, 0, 21, 4096, 5, 0, 4094, 1, 3587, 4096, 0, 0, 4091, 0, 0, 0, 41, 14, 0, 98, 0, 227, 0, 0, 0, 0, 0, 0, 470, 4055, 48, 0, 0, 0, 4096, 0, 0, 0, 4092, 3891, 0, 0, 0, 0, 4050, 0, 0, 1, 2, 0, 0, 258, 3005, 0, 132, 4096, 0, 3571, 616, 0, 927, 0, 0, 4096, 0, 4060, 4070, 0, 0, 0, 3192, 0, 0, 4094, 4096, 4, 0, 4096, 4089, 2, 0, 0, 12, 5, 0, 3868, 0, 0, 4019, 0, 0, 1365, 4093, 0, 0, 0, 0, 0, 0, 0, 4010, 3679, 4082] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4080, 0, 1513, 0, 12, 4096, 0, 0, 0, 3809, 39, 938, 530, 0, 3705, 0, 0, 0, 0, 0, 0, 0, 1951, 3584, 0, 0, 0, 0, 1684, 0, 0, 4096, 225, 323, 4094, 66, 0, 4095, 4, 1048, 34, 0, 0, 0, 109, 0, 0, 5, 0, 145, 0, 4057, 12, 17, 3606, 0, 3406, 0, 0, 4096, 648, 104, 0, 0, 3424, 0, 0, 0, 0, 4096, 0, 0, 0, 124, 0, 0, 4096, 573, 0, 1, 4096, 2489, 0, 0, 6, 0, 0, 500, 4006, 0, 0, 1873, 200, 0, 0, 1, 519, 3965, 3824, 4096, 0, 0, 4094, 4096, 0, 0, 0, 3656, 282, 0, 1, 54, 0, 2602, 3719, 0, 15, 3861, 125, 16, 0, 1874, 410, 0, 0, 4093, 4096, 4058] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4068, 0, 3995, 0, 27, 4079, 103, 0, 0, 2978, 4, 0, 0, 41, 130, 86, 0, 0, 0, 0, 0, 56, 186, 4013, 256, 0, 78, 0, 2837, 0, 0, 4044, 0, 3692, 4096, 408, 0, 4051, 0, 25, 0, 0, 0, 0, 76, 0, 4031, 30, 0, 1016, 0, 0, 5, 0, 4030, 0, 0, 2, 21, 4082, 0, 0, 0, 3910, 4010, 0, 0, 0, 30, 4065, 0, 7, 0, 0, 0, 0, 0, 4093, 0, 0, 3842, 4, 4027, 0, 0, 4095, 0, 5, 3972, 0, 4096, 4096, 1369, 0, 0, 4000, 0, 20, 4096, 4085, 0, 0, 3997, 4096, 0, 0, 69, 0, 7, 66, 0, 30, 7, 3817, 0, 1, 1, 4096, 63, 231, 0, 10, 21, 0, 26, 4095, 0, 4071] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[3904, 4, 4085, 0, 0, 3673, 0, 0, 0, 4096, 0, 0, 19, 0, 0, 0, 0, 0, 153, 130, 0, 0, 0, 3915, 0, 0, 0, 1, 4068, 0, 5, 4096, 13, 3805, 4082, 5, 0, 4095, 168, 0, 0, 2430, 0, 0, 1, 0, 4026, 107, 1, 0, 0, 0, 0, 18, 4089, 1, 0, 1, 0, 4079, 0, 0, 0, 4062, 4082, 0, 0, 0, 1, 3945, 0, 0, 344, 0, 383, 0, 553, 3770, 1, 0, 4096, 0, 0, 12, 0, 3904, 0, 3700, 4043, 74, 4087, 4096, 57, 0, 0, 7, 23, 0, 4094, 3360, 0, 0, 4077, 3708, 27, 0, 0, 0, 0, 0, 1, 0, 0, 1077, 0, 3, 1475, 4096, 0, 1, 0, 556, 2, 0, 2, 4095, 4092, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[2341, 17, 4053, 8, 0, 4083, 76, 0, 0, 31, 4, 0, 0, 1, 109, 0, 142, 10, 0, 661, 0, 0, 0, 3429, 44, 0, 14, 85, 4054, 104, 0, 4051, 0, 4095, 4044, 0, 0, 4036, 0, 0, 0, 0, 190, 30, 0, 36, 4082, 0, 0, 3918, 22, 0, 1976, 0, 4075, 34, 848, 0, 0, 4067, 0, 83, 0, 7, 4009, 0, 0, 42, 0, 4038, 0, 0, 12, 0, 55, 11, 0, 4056, 0, 0, 4091, 0, 3919, 382, 0, 4028, 4, 3, 3944, 21, 3717, 4091, 0, 0, 0, 465, 0, 0, 3969, 3997, 72, 0, 4078, 2603, 64, 0, 0, 40, 0, 0, 0, 0, 0, 4090, 528, 0, 4014, 4023, 0, 0, 0, 0, 0, 93, 0, 4081, 3621, 4051] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[2723, 65, 4048, 0, 0, 668, 0, 40, 289, 3757, 1, 0, 0, 25, 0, 0, 337, 0, 0, 0, 0, 0, 10, 4096, 0, 29, 0, 0, 3847, 0, 0, 4096, 0, 4094, 3766, 0, 0, 3967, 0, 0, 0, 739, 0, 0, 0, 0, 89, 0, 0, 0, 1, 1521, 0, 0, 3984, 2, 0, 0, 0, 3706, 0, 196, 1, 2373, 3608, 302, 34, 82, 0, 3441, 481, 0, 3956, 0, 126, 32, 0, 3904, 58, 16, 3168, 0, 0, 1904, 0, 4096, 0, 4096, 4096, 0, 4070, 4000, 0, 58, 0, 3304, 21, 0, 4056, 4089, 0, 0, 4023, 3245, 0, 292, 0, 0, 0, 1194, 0, 99, 0, 4042, 801, 0, 699, 4073, 262, 117, 0, 0, 0, 1114, 0, 4096, 1666, 3881] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4088, 0, 4096, 92, 0, 4095, 0, 0, 0, 2927, 0, 0, 0, 0, 0, 0, 1092, 837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 25, 0, 0, 0, 4022, 0, 0, 0, 0, 3680, 0, 0, 4096, 4096, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3758, 4096, 0, 0, 4096, 0, 0, 0, 2306, 0, 0, 0, 0, 569, 0, 904, 789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1, 4096, 0, 440, 0, 0, 4096, 0, 0, 0, 3269, 2751, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 4, 10, 0, 4, 3857, 0, 0, 0, 0, 4081, 4096, 0, 0, 0, 2540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 2215, 4089, 0, 0, 0, 0, 0, 0, 1255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 3034, 3192, 0, 1, 0, 0, 0, 3, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17227, 17066, 16518, 15039, 15875, 15100, 15749, 18498] avg=16384.0000 max=18498 max_rank=7 min=15039 min_rank=3 imbalance_ratio=1.129028 max_load_reduction=112574 max_load_reduction_pct=0.858871 moved_to_echo=65222 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3660, 0, 3949, 4, 21, 4073, 0, 0, 4, 0, 0, 0, 0, 0, 118, 141, 1087, 837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 12, 3835, 52, 0, 0, 0, 4046, 0, 0, 4096, 4057, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 37, 4085, 13, 266, 1, 4025, 0, 0, 0, 0, 3827, 0, 894, 786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3945, 0, 0, 332, 20, 3992, 0, 82, 0, 0, 4096, 2, 0, 0, 1507, 2751, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 230, 0, 26, 0, 4096, 197, 0, 0, 464, 0, 14, 125, 4072, 0, 0, 3583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 33, 0, 146, 0, 2449, 360, 102, 0, 0, 4065, 4085, 0, 0, 240, 3802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4096, 4096, 0, 0, 2206, 3734, 2, 0, 66, 0, 8, 13, 3821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 11, 23, 5, 3154, 2, 0, 12, 0, 0, 0, 63, 0, 3776, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13894, 16243, 18030, 17108, 16838, 15789, 18073, 15097] avg=16384.0000 max=18073 max_rank=6 min=13894 min_rank=0 imbalance_ratio=1.103088 max_load_reduction=112999 max_load_reduction_pct=0.862114 moved_to_echo=65797 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4068, 0, 3995, 0, 27, 4079, 103, 0, 0, 2978, 4, 0, 0, 41, 130, 86, 1066, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 186, 4013, 256, 0, 78, 0, 2837, 0, 0, 4044, 4095, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3692, 4096, 408, 0, 4051, 0, 25, 0, 0, 0, 0, 76, 0, 4031, 30, 904, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 0, 0, 5, 0, 4030, 0, 0, 2, 21, 4082, 0, 0, 0, 3910, 2581, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 0, 30, 4065, 0, 7, 0, 0, 0, 0, 0, 4093, 0, 0, 3480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 4, 4027, 0, 0, 4095, 0, 5, 0, 0, 4096, 4096, 1369, 0, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 4085, 0, 0, 2161, 4096, 0, 0, 69, 0, 7, 66, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3817, 0, 1, 1, 3192, 63, 231, 0, 10, 21, 0, 26, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17362, 15691, 18083, 16013, 15685, 22168, 14630, 11440] avg=16384.0000 max=22168 max_rank=5 min=11440 min_rank=7 imbalance_ratio=1.353027 max_load_reduction=108904 max_load_reduction_pct=0.830872 moved_to_echo=63923 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2723, 65, 4048, 0, 0, 668, 0, 40, 289, 3757, 1, 0, 0, 25, 0, 0, 1073, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 0, 0, 0, 0, 0, 10, 4096, 0, 29, 0, 0, 3847, 0, 0, 4096, 4096, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3766, 0, 0, 3967, 0, 0, 0, 739, 0, 0, 0, 0, 89, 0, 899, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1521, 0, 0, 3984, 2, 0, 0, 0, 3706, 0, 196, 1, 2373, 2128, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3608, 302, 34, 82, 0, 3441, 481, 0, 3956, 0, 126, 32, 0, 3904, 58, 16, 3587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 0, 1904, 0, 4096, 0, 4096, 0, 0, 4070, 4000, 0, 58, 0, 3304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4056, 4089, 0, 0, 2175, 3245, 0, 292, 0, 0, 0, 1194, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 801, 0, 699, 3174, 262, 117, 0, 0, 0, 1114, 0, 0, 1666, 3881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13336, 16641, 14329, 14291, 19627, 21921, 15171, 15756] avg=16384.0000 max=21921 max_rank=5 min=13336 min_rank=0 imbalance_ratio=1.337952 max_load_reduction=109151 max_load_reduction_pct=0.832756 moved_to_echo=72475 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4046, 0, 4082, 0, 582, 3447, 0, 0, 0, 4092, 11, 0, 0, 16, 37, 1, 1092, 837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 64, 2, 472, 0, 1, 5, 3597, 0, 0, 0, 21, 4096, 5, 0, 4094, 4010, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3587, 4096, 0, 0, 4091, 0, 0, 0, 41, 14, 0, 98, 0, 227, 0, 904, 789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 4055, 48, 0, 0, 0, 4096, 0, 0, 0, 4092, 2751, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3891, 0, 0, 0, 0, 4050, 0, 0, 1, 2, 0, 0, 258, 3005, 0, 132, 3587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 0, 3571, 616, 0, 927, 0, 0, 0, 0, 4060, 4070, 0, 0, 0, 3192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 4, 0, 2215, 4089, 2, 0, 0, 12, 5, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 1365, 3189, 0, 0, 0, 0, 0, 0, 0, 0, 3679, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18243, 16501, 13848, 15891, 14926, 16944, 18385, 16334] avg=16384.0000 max=18385 max_rank=6 min=13848 min_rank=2 imbalance_ratio=1.122131 max_load_reduction=112687 max_load_reduction_pct=0.859734 moved_to_echo=66589 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4080, 0, 1513, 0, 12, 4096, 0, 0, 0, 3809, 39, 938, 530, 0, 3705, 0, 1095, 842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1951, 3584, 0, 0, 0, 0, 1684, 0, 0, 4096, 4093, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 323, 4094, 66, 0, 4095, 4, 1048, 34, 0, 0, 0, 109, 0, 0, 5, 857, 795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 4057, 12, 17, 3606, 0, 3406, 0, 0, 4096, 648, 104, 0, 0, 2755, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3424, 0, 0, 0, 0, 4096, 0, 0, 0, 124, 0, 0, 4096, 573, 0, 1, 3508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 499, 2489, 0, 0, 6, 0, 0, 500, 0, 0, 0, 1873, 200, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 3965, 3824, 4096, 0, 0, 2204, 4096, 0, 0, 0, 3656, 282, 0, 1, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2602, 3719, 0, 15, 3004, 125, 16, 0, 1874, 410, 0, 0, 0, 4096, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20659, 15537, 11655, 19215, 15822, 5568, 22697, 19919] avg=16384.0000 max=22697 max_rank=6 min=5568 min_rank=5 imbalance_ratio=1.385315 max_load_reduction=108375 max_load_reduction_pct=0.826836 moved_to_echo=64006 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3904, 4, 4085, 0, 0, 3673, 0, 0, 0, 4096, 0, 0, 19, 0, 0, 0, 1087, 837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 130, 0, 0, 0, 3915, 0, 0, 0, 1, 4068, 0, 5, 4096, 4095, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3805, 4082, 5, 0, 4095, 168, 0, 0, 2430, 0, 0, 1, 0, 4026, 107, 904, 786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 18, 4089, 1, 0, 1, 0, 4079, 0, 0, 0, 4062, 2751, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 1, 3945, 0, 0, 344, 0, 383, 0, 553, 3770, 1, 0, 3541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 0, 0, 12, 0, 3904, 0, 3700, 0, 74, 4087, 4096, 57, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4094, 3360, 0, 0, 2204, 3708, 27, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 0, 3, 1475, 3192, 0, 1, 0, 556, 2, 0, 2, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17705, 16593, 20422, 15374, 16620, 16445, 13417, 14496] avg=16384.0000 max=20422 max_rank=2 min=13417 min_rank=6 imbalance_ratio=1.246460 max_load_reduction=110650 max_load_reduction_pct=0.844193 moved_to_echo=60978 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2341, 17, 4053, 8, 0, 4083, 76, 0, 0, 31, 4, 0, 0, 1, 109, 0, 1087, 836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 10, 0, 661, 0, 0, 0, 3429, 44, 0, 14, 85, 4054, 104, 0, 4051, 4081, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4044, 0, 0, 4036, 0, 0, 0, 0, 190, 30, 0, 36, 4082, 0, 888, 786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 22, 0, 1976, 0, 4075, 34, 848, 0, 0, 4067, 0, 83, 0, 7, 2748, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 42, 0, 4038, 0, 0, 12, 0, 55, 11, 0, 4056, 0, 0, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 0, 3919, 382, 0, 4028, 4, 3, 0, 21, 3717, 4091, 0, 0, 0, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 3997, 72, 0, 2205, 2603, 64, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 528, 0, 4014, 3135, 0, 0, 0, 0, 0, 93, 0, 0, 3621, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12646, 16800, 18187, 18141, 15679, 17137, 12950, 19532] avg=16384.0000 max=19532 max_rank=7 min=12646 min_rank=0 imbalance_ratio=1.192139 max_load_reduction=111540 max_load_reduction_pct=0.850983 moved_to_echo=65298 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[3968, 2560, 11648, 3072, 128, 23424, 384, 8448, 0, 128, 28288, 30464, 1664, 128, 256, 17408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[28928, 128, 29824, 128, 768, 28288, 256, 128, 384, 21760, 128, 1024, 640, 128, 4224, 256, 8704, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 27776, 5120, 128, 10624, 25344, 512, 384, 128, 2560, 512, 1280, 128, 0, 25088, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[640, 4096, 32384, 32000, 128, 0, 17664, 29696, 128, 384, 256, 3712, 384, 1280, 8960, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 23424, 32384, 512, 128, 32640, 256, 1408, 128, 9600, 256, 128, 384, 128, 16896, 256, 7168, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[31232, 640, 128, 256, 128, 31872, 768, 128, 4352, 640, 640, 128, 5120, 27648, 128, 256, 28416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 9088, 128, 5632, 2432, 640, 32000, 128, 4864, 128, 128, 32384, 768, 384, 128, 19328, 21248, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[512, 128, 1280, 1408, 128, 128, 2176, 30592, 384, 128, 128, 128, 28416, 128, 128, 32768, 32640, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True + [2026-08-30 05:04:57] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 745.0 | throughput per GPU (TFLOP/s/GPU): 447.5 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.061052E+01 | loss scale: 1.0 | grad norm: 216.847 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[21, 1010, 2063, 51, 2417, 11, 671, 59, 796, 104, 2184, 410, 781, 397, 14, 329, 149, 1896, 1813, 1172, 2770, 529, 970, 301, 13, 2416, 178, 570, 1065, 96, 943, 1966, 3474, 26, 77, 1194, 894, 44, 782, 878, 2278, 3038, 529, 382, 183, 1498, 1117, 656, 552, 102, 1079, 397, 232, 449, 3645, 133, 8, 1276, 3081, 2054, 224, 231, 365, 41, 785, 3617, 83, 2318, 505, 223, 1747, 2687, 157, 1112, 447, 53, 999, 2465, 2874, 1277, 1445, 647, 1149, 2962, 538, 679, 1600, 3355, 67, 439, 677, 637, 1694, 273, 676, 47, 1126, 292, 2848, 605, 67, 1095, 1533, 2128, 2608, 2466, 531, 1257, 330, 627, 569, 1980, 192, 1684, 370, 659, 30, 895, 658, 1230, 873, 1227, 2307, 1083, 1573, 21, 748, 142] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[39, 1699, 1763, 11, 2318, 7, 696, 121, 967, 75, 2116, 364, 843, 650, 8, 421, 264, 1634, 1738, 1517, 2768, 373, 708, 291, 1, 2717, 222, 480, 986, 162, 990, 2064, 3437, 57, 60, 807, 1227, 39, 1076, 924, 2441, 3160, 655, 387, 200, 967, 2002, 866, 590, 330, 835, 228, 81, 249, 3075, 267, 26, 1080, 3324, 2449, 469, 264, 479, 33, 1065, 3258, 55, 1328, 459, 239, 1927, 2675, 280, 934, 388, 60, 866, 2376, 2104, 1505, 1352, 417, 1380, 3184, 569, 744, 1210, 3519, 122, 280, 702, 885, 1553, 350, 515, 53, 1330, 231, 3051, 948, 59, 1346, 1069, 1899, 2682, 2076, 328, 1465, 800, 652, 342, 1852, 182, 1353, 654, 730, 60, 1141, 708, 1637, 649, 874, 1992, 1034, 1443, 62, 911, 61] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[20, 1484, 2416, 61, 1935, 7, 710, 47, 875, 143, 2341, 276, 885, 483, 27, 303, 211, 1743, 1391, 1268, 2923, 449, 651, 173, 4, 2241, 176, 648, 1290, 226, 1120, 2349, 3323, 35, 44, 1364, 844, 45, 864, 1572, 2631, 3220, 791, 459, 341, 1284, 1157, 888, 433, 178, 576, 151, 121, 263, 3140, 125, 32, 1097, 3154, 1792, 250, 184, 489, 16, 920, 3129, 81, 1744, 308, 250, 1525, 2392, 174, 844, 737, 124, 896, 2096, 2418, 779, 1876, 617, 892, 3379, 615, 1313, 1174, 3051, 82, 359, 576, 662, 1712, 379, 581, 64, 1094, 412, 2966, 606, 29, 1358, 1728, 2164, 2549, 2543, 445, 1506, 440, 557, 606, 2019, 205, 1513, 419, 869, 30, 1344, 397, 1470, 953, 1032, 2627, 1088, 1754, 25, 675, 161] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[23, 773, 1568, 10, 2323, 6, 754, 52, 869, 68, 2183, 411, 732, 546, 16, 336, 162, 1826, 1674, 1771, 3000, 467, 679, 141, 6, 2482, 179, 342, 904, 87, 985, 2224, 3548, 54, 74, 716, 524, 88, 827, 1259, 2506, 3191, 733, 237, 238, 1611, 1726, 972, 480, 231, 849, 379, 213, 270, 3387, 124, 10, 1131, 3108, 2031, 120, 430, 307, 76, 683, 3407, 65, 1272, 585, 243, 1986, 2828, 481, 1102, 509, 58, 759, 2154, 1921, 1511, 1785, 655, 1176, 3102, 255, 663, 1870, 3447, 111, 246, 714, 793, 1649, 220, 1123, 83, 948, 227, 3373, 530, 10, 1171, 1575, 1628, 3176, 2026, 326, 1324, 628, 664, 483, 2163, 163, 1197, 465, 667, 49, 1996, 578, 1986, 891, 1034, 2325, 1053, 1304, 37, 520, 50] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[52, 1381, 2014, 19, 2185, 12, 817, 32, 594, 114, 2483, 258, 585, 635, 10, 349, 185, 1820, 1305, 1785, 2957, 525, 880, 228, 9, 2657, 167, 703, 962, 297, 816, 1919, 3490, 11, 173, 678, 662, 137, 921, 1233, 2297, 3164, 897, 675, 165, 1747, 1538, 1339, 378, 211, 1003, 401, 254, 181, 2827, 114, 32, 1002, 2344, 2389, 113, 303, 533, 60, 911, 3112, 81, 1406, 337, 189, 1113, 1938, 372, 746, 521, 106, 776, 2299, 2670, 1573, 1433, 643, 987, 3488, 399, 976, 1368, 3328, 89, 283, 1165, 889, 1637, 557, 902, 69, 1551, 316, 2757, 441, 81, 874, 1480, 1568, 2965, 2088, 301, 1035, 275, 643, 529, 2103, 210, 1196, 900, 820, 50, 1812, 740, 1658, 940, 1503, 2200, 1418, 1515, 25, 659, 29] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[16, 1036, 2463, 6, 2782, 5, 785, 22, 622, 108, 1552, 136, 1083, 289, 1, 434, 117, 2439, 1298, 1956, 2767, 271, 991, 161, 1, 2065, 160, 455, 1294, 46, 833, 2249, 3674, 192, 26, 946, 1123, 20, 1138, 1214, 2725, 2940, 637, 243, 148, 1279, 1612, 1551, 319, 133, 1172, 167, 137, 90, 3223, 192, 63, 1039, 2771, 2430, 130, 230, 271, 15, 1261, 3368, 34, 1860, 428, 288, 2008, 2654, 498, 1016, 541, 75, 1247, 2430, 1894, 875, 1628, 386, 874, 3360, 640, 511, 1284, 3743, 55, 291, 558, 632, 1709, 249, 551, 54, 841, 265, 3067, 581, 25, 1339, 1133, 2103, 2866, 1468, 369, 1556, 426, 686, 415, 2360, 192, 1491, 386, 972, 37, 1445, 481, 1694, 1027, 1139, 2188, 775, 1706, 12, 705, 27] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[9, 1893, 2089, 34, 2684, 3, 734, 53, 717, 93, 2035, 286, 679, 411, 9, 262, 238, 1785, 1422, 1259, 3235, 802, 675, 214, 27, 2442, 209, 602, 1121, 347, 897, 2106, 3451, 20, 140, 1264, 361, 45, 1084, 1153, 2602, 2940, 1108, 433, 256, 1434, 1826, 1764, 303, 185, 1302, 176, 130, 260, 2887, 328, 12, 847, 2364, 3023, 179, 250, 439, 18, 886, 3478, 57, 1083, 457, 248, 1634, 2387, 158, 861, 330, 244, 907, 2747, 2112, 1394, 2197, 798, 1103, 3251, 584, 553, 1467, 3321, 57, 239, 1182, 836, 1479, 473, 551, 53, 1561, 321, 3446, 428, 56, 859, 1122, 1912, 2860, 1524, 373, 965, 270, 657, 455, 1727, 199, 1056, 412, 832, 67, 1560, 883, 1640, 785, 1136, 2043, 525, 1517, 15, 731, 52] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[64, 1584, 1782, 11, 2596, 11, 1078, 49, 1063, 60, 2539, 585, 693, 484, 28, 315, 278, 1387, 885, 1441, 3110, 811, 663, 368, 49, 2509, 231, 382, 869, 126, 883, 2120, 3019, 41, 51, 1028, 966, 144, 414, 1189, 2527, 2442, 725, 292, 247, 1335, 1839, 455, 507, 200, 796, 487, 310, 432, 3212, 137, 31, 1002, 3241, 2131, 364, 383, 256, 75, 665, 2962, 97, 1535, 497, 280, 1490, 2865, 429, 1302, 788, 37, 746, 1966, 1598, 1288, 1599, 701, 976, 2954, 530, 730, 1606, 3404, 532, 323, 702, 548, 1712, 440, 571, 176, 1341, 425, 2556, 494, 46, 676, 1965, 2379, 2767, 2646, 303, 818, 718, 991, 723, 1837, 218, 1339, 440, 685, 54, 1533, 1111, 1886, 647, 1137, 2038, 1452, 1545, 57, 813, 51] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[21, 1010, 2063, 51, 2417, 11, 671, 59, 796, 104, 2184, 410, 781, 397, 14, 329, 2848, 1798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 1896, 1813, 1172, 2265, 529, 970, 301, 13, 2416, 178, 570, 1065, 96, 943, 1966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1366, 26, 77, 1194, 894, 44, 782, 878, 2278, 3038, 529, 382, 183, 1498, 1117, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 102, 1079, 397, 232, 449, 3645, 133, 8, 1276, 3081, 2054, 224, 231, 365, 41, 2108, 1488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 785, 331, 83, 2318, 505, 223, 1747, 2687, 157, 1112, 447, 53, 999, 2465, 2874, 1277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1445, 647, 1149, 2962, 538, 679, 1600, 2337, 67, 439, 677, 637, 1694, 273, 676, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1126, 292, 0, 605, 67, 1095, 1533, 2128, 2138, 2466, 531, 1257, 330, 627, 569, 1980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 1684, 370, 659, 30, 895, 658, 1230, 873, 1227, 2307, 1083, 1573, 21, 748, 142, 1018, 505, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15964, 16342, 14942, 17465, 18063, 15867, 16744, 15685] avg=16384.0000 max=18063 max_rank=4 min=14942 min_rank=2 imbalance_ratio=1.102478 max_load_reduction=113009 max_load_reduction_pct=0.862190 moved_to_echo=66359 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[39, 1699, 1763, 11, 2318, 7, 696, 121, 967, 75, 2116, 364, 843, 650, 8, 421, 3051, 1620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 1634, 1738, 1517, 2263, 373, 708, 291, 1, 2717, 222, 480, 986, 162, 990, 2064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1352, 57, 60, 807, 1227, 39, 1076, 924, 2441, 3160, 655, 387, 200, 967, 2002, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 590, 330, 835, 228, 81, 249, 3075, 267, 26, 1080, 3324, 2449, 469, 264, 479, 33, 2085, 1340, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 298, 55, 1328, 459, 239, 1927, 2675, 280, 934, 388, 60, 866, 2376, 2104, 1505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1352, 417, 1380, 3184, 569, 744, 1210, 2450, 122, 280, 702, 885, 1553, 350, 515, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1330, 231, 0, 948, 59, 1346, 1069, 1899, 2198, 2076, 328, 1465, 800, 652, 342, 1852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 1353, 654, 730, 60, 1141, 708, 1637, 649, 874, 1992, 1034, 1443, 62, 911, 61, 1068, 505, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16769, 16410, 16220, 17205, 16559, 15766, 16595, 15548] avg=16384.0000 max=17205 max_rank=3 min=15548 min_rank=7 imbalance_ratio=1.050110 max_load_reduction=113867 max_load_reduction_pct=0.868736 moved_to_echo=64468 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[20, 1484, 2416, 61, 1935, 7, 710, 47, 875, 143, 2341, 276, 885, 483, 27, 303, 2966, 1560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 1743, 1391, 1268, 2387, 449, 651, 173, 4, 2241, 176, 648, 1290, 226, 1120, 2349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1303, 35, 44, 1364, 844, 45, 864, 1572, 2631, 3220, 791, 459, 341, 1284, 1157, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 178, 576, 151, 121, 263, 3140, 125, 32, 1097, 3154, 1792, 250, 184, 489, 16, 2020, 1291, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 920, 278, 81, 1744, 308, 250, 1525, 2392, 174, 844, 737, 124, 896, 2096, 2418, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1876, 617, 892, 3379, 615, 1313, 1174, 2116, 82, 359, 576, 662, 1712, 379, 581, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 412, 0, 606, 29, 1358, 1728, 2164, 2085, 2543, 445, 1506, 440, 557, 606, 2019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 1513, 419, 869, 30, 1344, 397, 1470, 953, 1032, 2627, 1088, 1754, 25, 675, 161, 931, 536, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16539, 16327, 16842, 15316, 15566, 16397, 17592, 16493] avg=16384.0000 max=17592 max_rank=6 min=15316 min_rank=3 imbalance_ratio=1.073730 max_load_reduction=113480 max_load_reduction_pct=0.865784 moved_to_echo=66048 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[16, 1036, 2463, 6, 2782, 5, 785, 22, 622, 108, 1552, 136, 1083, 289, 1, 434, 3067, 1675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 2439, 1298, 1956, 2263, 271, 991, 161, 1, 2065, 160, 455, 1294, 46, 833, 2249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1445, 192, 26, 946, 1123, 20, 1138, 1214, 2725, 2940, 637, 243, 148, 1279, 1612, 1551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 319, 133, 1172, 167, 137, 90, 3223, 192, 63, 1039, 2771, 2430, 130, 230, 271, 15, 2229, 1386, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 307, 34, 1860, 428, 288, 2008, 2654, 498, 1016, 541, 75, 1247, 2430, 1894, 875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1628, 386, 874, 3360, 640, 511, 1284, 2606, 55, 291, 558, 632, 1709, 249, 551, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 841, 265, 0, 581, 25, 1339, 1133, 2103, 2349, 1468, 369, 1556, 426, 686, 415, 2360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 1491, 386, 972, 37, 1445, 481, 1694, 1027, 1139, 2188, 775, 1706, 12, 705, 27, 1136, 504, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16082, 16599, 17239, 15998, 17416, 15388, 15916, 16434] avg=16384.0000 max=17416 max_rank=4 min=15388 min_rank=5 imbalance_ratio=1.062988 max_load_reduction=113656 max_load_reduction_pct=0.867126 moved_to_echo=65154 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[23, 773, 1568, 10, 2323, 6, 754, 52, 869, 68, 2183, 411, 732, 546, 16, 336, 3373, 1694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 1826, 1674, 1771, 2453, 467, 679, 141, 6, 2482, 179, 342, 904, 87, 985, 2224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1395, 54, 74, 716, 524, 88, 827, 1259, 2506, 3191, 733, 237, 238, 1611, 1726, 972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 231, 849, 379, 213, 270, 3387, 124, 10, 1131, 3108, 2031, 120, 430, 307, 76, 2153, 1402, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 311, 65, 1272, 585, 243, 1986, 2828, 481, 1102, 509, 58, 759, 2154, 1921, 1511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1785, 655, 1176, 3102, 255, 663, 1870, 2400, 111, 246, 714, 793, 1649, 220, 1123, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 948, 227, 0, 530, 10, 1171, 1575, 1628, 2603, 2026, 326, 1324, 628, 664, 483, 2163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 1197, 465, 667, 49, 1996, 578, 1986, 891, 1034, 2325, 1053, 1304, 37, 520, 50, 1046, 547, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15737, 16382, 16151, 16702, 16468, 16845, 16306, 16481] avg=16384.0000 max=16845 max_rank=5 min=15737 min_rank=0 imbalance_ratio=1.028137 max_load_reduction=114227 max_load_reduction_pct=0.871483 moved_to_echo=66100 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[64, 1584, 1782, 11, 2596, 11, 1078, 49, 1063, 60, 2539, 585, 693, 484, 28, 315, 2556, 1473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 1387, 885, 1441, 2543, 811, 663, 368, 49, 2509, 231, 382, 869, 126, 883, 2120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1187, 41, 51, 1028, 966, 144, 414, 1189, 2527, 2442, 725, 292, 247, 1335, 1839, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 200, 796, 487, 310, 432, 3212, 137, 31, 1002, 3241, 2131, 364, 383, 256, 75, 1832, 1218, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 665, 271, 97, 1535, 497, 280, 1490, 2865, 429, 1302, 788, 37, 746, 1966, 1598, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1599, 701, 976, 2954, 530, 730, 1606, 2370, 532, 323, 702, 548, 1712, 440, 571, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1341, 425, 0, 494, 46, 676, 1965, 2379, 2268, 2646, 303, 818, 718, 991, 723, 1837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 1339, 440, 685, 54, 1533, 1111, 1886, 647, 1137, 2038, 1452, 1545, 57, 813, 51, 1033, 567, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16971, 15545, 14882, 16615, 15854, 16470, 17630, 17105] avg=16384.0000 max=17630 max_rank=6 min=14882 min_rank=2 imbalance_ratio=1.076050 max_load_reduction=113442 max_load_reduction_pct=0.865494 moved_to_echo=67059 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[52, 1381, 2014, 19, 2185, 12, 817, 32, 594, 114, 2483, 258, 585, 635, 10, 349, 2757, 1547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 1820, 1305, 1785, 2418, 525, 880, 228, 9, 2657, 167, 703, 962, 297, 816, 1919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1372, 11, 173, 678, 662, 137, 921, 1233, 2297, 3164, 897, 675, 165, 1747, 1538, 1339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 211, 1003, 401, 254, 181, 2827, 114, 32, 1002, 2344, 2389, 113, 303, 533, 60, 2118, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 911, 285, 81, 1406, 337, 189, 1113, 1938, 372, 746, 521, 106, 776, 2299, 2670, 1573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1433, 643, 987, 3488, 399, 976, 1368, 2318, 89, 283, 1165, 889, 1637, 557, 902, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1551, 316, 0, 441, 81, 874, 1480, 1568, 2430, 2088, 301, 1035, 275, 643, 529, 2103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 1196, 900, 820, 50, 1812, 740, 1658, 940, 1503, 2200, 1418, 1515, 25, 659, 29, 1010, 539, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15844, 16676, 17009, 15543, 15323, 17203, 15715, 17759] avg=16384.0000 max=17759 max_rank=7 min=15323 min_rank=4 imbalance_ratio=1.083923 max_load_reduction=113313 max_load_reduction_pct=0.864510 moved_to_echo=66000 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[9, 1893, 2089, 34, 2684, 3, 734, 53, 717, 93, 2035, 286, 679, 411, 9, 262, 3446, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 1785, 1422, 1259, 2645, 802, 675, 214, 27, 2442, 209, 602, 1121, 347, 897, 2106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1357, 20, 140, 1264, 361, 45, 1084, 1153, 2602, 2940, 1108, 433, 256, 1434, 1826, 1764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 185, 1302, 176, 130, 260, 2887, 328, 12, 847, 2364, 3023, 179, 250, 439, 18, 2094, 1431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 886, 318, 57, 1083, 457, 248, 1634, 2387, 158, 861, 330, 244, 907, 2747, 2112, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2197, 798, 1103, 3251, 584, 553, 1467, 2313, 57, 239, 1182, 836, 1479, 473, 551, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1561, 321, 0, 428, 56, 859, 1122, 1912, 2344, 1524, 373, 965, 270, 657, 455, 1727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 1056, 412, 832, 67, 1560, 883, 1640, 785, 1136, 2043, 525, 1517, 15, 731, 52, 1008, 590, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17166, 16791, 17787, 16228, 15823, 17136, 14574, 15567] avg=16384.0000 max=17787 max_rank=2 min=14574 min_rank=6 imbalance_ratio=1.085632 max_load_reduction=113285 max_load_reduction_pct=0.864296 moved_to_echo=63100 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 10880, 16256, 256, 19328, 128, 6272, 512, 6528, 768, 17536, 2816, 6400, 3968, 128, 2816, 24064, 13184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1664, 10880, 4096, 6272, 384, 11776, 5632, 13312, 6784, 9088, 17792, 8448, 12416, 256, 5888, 640, 8320, 4352, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1664, 14592, 11648, 12288, 19328, 4352, 6272, 1920, 128, 19584, 1536, 4224, 8576, 1408, 7552, 17024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3584, 1664, 7680, 2432, 1536, 2304, 25472, 1536, 256, 8576, 23424, 18304, 1920, 2304, 3200, 384, 16640, 10880, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[13440, 4864, 8576, 25728, 4224, 6272, 11648, 18944, 1152, 2560, 6400, 5888, 13184, 2944, 5504, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[10880, 512, 768, 8064, 6656, 640, 7168, 9472, 20096, 24192, 6144, 3200, 1792, 11264, 12928, 8576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[9856, 2560, 0, 4736, 384, 8832, 11648, 15872, 18432, 16896, 3072, 9984, 3968, 5504, 4224, 16128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[7296, 2432, 640, 12672, 3584, 2048, 13440, 20480, 2560, 7936, 4352, 768, 7296, 18560, 17664, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1613, 6, 3, 0, 2394, 0, 0, 7, 0, 19, 0, 4096, 3541, 0, 1665, 10, 1, 2807, 3522, 3908, 2, 3, 8, 4, 4074, 1, 0, 4000, 2, 2, 1, 0, 3, 0, 2949, 0, 13, 0, 0, 3635, 3958, 1, 0, 0, 0, 0, 40, 3438, 4082, 0, 1, 1, 0, 0, 4096, 0, 18, 0, 2696, 0, 0, 0, 3901, 3409, 0, 4079, 0, 3970, 7, 3459, 60, 4087, 0, 0, 1, 3986, 4096, 0, 0, 0, 9, 9, 4091, 4093, 2460, 11, 4089, 0, 1, 1, 8, 51, 4087, 25, 2, 4096, 4092, 2, 12, 7, 0, 0, 0, 2, 1, 13, 0, 14, 0, 0, 3548, 0, 0, 0, 11, 26, 303, 1, 0, 4092, 64, 4085, 1, 4088, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4004, 67, 32, 0, 2389, 411, 0, 179, 0, 0, 17, 4092, 196, 2, 3735, 0, 0, 1167, 3187, 3169, 4, 0, 7, 3, 4064, 0, 1, 3875, 8, 1, 6, 0, 70, 0, 3670, 0, 24, 8, 2, 1294, 3739, 9, 12, 0, 2, 2, 1, 4068, 4096, 0, 0, 0, 0, 119, 4092, 0, 21, 2, 4062, 0, 0, 68, 3626, 2694, 0, 4089, 0, 3939, 3, 490, 8, 4096, 3, 158, 0, 3900, 4046, 1, 0, 13, 1, 46, 4089, 4093, 4001, 1, 4007, 1, 0, 5, 0, 34, 4043, 0, 0, 4084, 4092, 13, 2708, 67, 0, 0, 4, 33, 4, 0, 0, 68, 0, 0, 3112, 7, 0, 0, 3, 0, 445, 0, 13, 4095, 673, 4094, 9, 4089, 0, 95] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4, 0, 1170, 9, 954, 2, 48, 57, 30, 1802, 11, 102, 0, 4072, 2903, 0, 694, 1, 18, 3656, 3605, 3407, 0, 0, 0, 0, 4083, 0, 1, 4083, 31, 80, 2, 2, 0, 0, 3975, 23, 0, 2, 73, 3196, 4002, 20, 0, 33, 10, 144, 0, 3896, 3598, 0, 0, 12, 15, 35, 4089, 8, 0, 0, 3518, 0, 0, 288, 1801, 3836, 0, 4041, 3, 4081, 0, 262, 21, 3760, 1, 2018, 13, 4071, 4014, 10, 18, 987, 36, 28, 4082, 4087, 598, 0, 3244, 6, 6, 41, 0, 23, 4086, 123, 0, 4088, 4087, 0, 1232, 0, 0, 0, 0, 3, 13, 7, 0, 1, 0, 0, 3610, 31, 0, 0, 0, 162, 2505, 3, 0, 4059, 40, 4059, 7, 4073, 0, 31] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 8, 746, 1, 35, 0, 998, 1, 1, 22, 0, 12, 135, 3982, 2351, 0, 3279, 4, 19, 2205, 2818, 2930, 0, 0, 111, 0, 4094, 0, 0, 4096, 15, 197, 5, 12, 0, 0, 3440, 595, 91, 0, 2, 3308, 4048, 0, 11, 0, 0, 79, 0, 4038, 3898, 0, 0, 0, 0, 4, 4083, 0, 7, 0, 3475, 104, 57, 7, 3181, 3255, 0, 4030, 1, 4050, 0, 3269, 0, 4054, 2, 67, 0, 3996, 4068, 0, 0, 17, 1, 0, 4096, 4077, 346, 84, 3991, 0, 0, 0, 45, 12, 4091, 8, 647, 4095, 4096, 0, 1019, 3, 2, 0, 0, 128, 0, 1, 178, 22, 0, 0, 4043, 0, 19, 0, 50, 6, 1470, 0, 274, 3967, 2, 4088, 0, 4076, 0, 821] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2, 9, 2415, 46, 44, 0, 290, 3, 0, 447, 0, 2, 46, 4096, 3176, 0, 3649, 2, 1, 2632, 4086, 3910, 0, 0, 0, 9, 4001, 0, 1, 4087, 0, 15, 181, 0, 10, 0, 3280, 0, 0, 0, 8, 2028, 3861, 12, 0, 0, 0, 13, 0, 3977, 4078, 0, 0, 0, 2, 38, 4088, 13, 0, 0, 3868, 0, 37, 0, 4062, 1797, 0, 4069, 0, 4050, 6, 787, 13, 4096, 11, 6, 1, 3893, 3984, 0, 6, 170, 0, 20, 4096, 4083, 1300, 64, 3901, 4, 0, 0, 12, 3, 4096, 7, 39, 4088, 4084, 0, 1336, 32, 0, 0, 1, 23, 0, 0, 10, 0, 0, 18, 3956, 0, 0, 0, 0, 7, 2112, 1, 79, 4026, 0, 4089, 1, 4095, 0, 75] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 605, 52, 160, 10, 948, 4, 116, 416, 0, 8, 1, 4051, 3311, 1, 1990, 1, 3, 1117, 3991, 3923, 0, 1, 391, 0, 4011, 0, 0, 4093, 35, 0, 695, 0, 0, 0, 3524, 164, 96, 0, 0, 2599, 3980, 27, 0, 3, 102, 0, 0, 4092, 4096, 0, 35, 10, 0, 25, 4048, 0, 72, 0, 3137, 44, 0, 44, 3073, 3069, 0, 4087, 0, 4089, 0, 2062, 22, 4079, 5, 438, 1, 2918, 4040, 11, 0, 0, 0, 55, 4034, 4087, 2995, 5, 4090, 2, 18, 18, 56, 2, 3821, 0, 0, 3972, 4083, 3, 426, 53, 0, 0, 1, 2, 0, 0, 5, 8, 3, 0, 3992, 9, 5, 0, 0, 16, 3151, 2, 163, 3969, 0, 4004, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 516, 8, 2, 0, 2206, 0, 0, 154, 0, 142, 0, 4092, 3844, 2, 3991, 0, 13, 1176, 3346, 1483, 1, 3, 69, 0, 4078, 0, 0, 4093, 1, 0, 3, 0, 0, 0, 4063, 0, 13, 7, 0, 1569, 3460, 3, 1, 1, 0, 0, 0, 3669, 4022, 0, 0, 2, 0, 0, 4095, 0, 1, 0, 3879, 3, 1, 230, 2391, 1756, 2, 4092, 0, 3405, 1, 3543, 3, 3993, 55, 67, 0, 3910, 4056, 1, 0, 373, 0, 222, 4093, 4096, 3541, 4, 4094, 0, 2, 0, 54, 0, 4094, 0, 81, 4095, 4092, 0, 1245, 58, 2, 0, 0, 2, 0, 0, 4, 1, 0, 0, 4072, 1, 2, 1, 5, 0, 3148, 0, 2, 4091, 4, 3979, 1, 4094, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1125, 15, 423, 1, 2133, 0, 2, 19, 5, 0, 0, 4082, 2322, 0, 1764, 0, 0, 2478, 4065, 3997, 37, 0, 0, 0, 4089, 1, 0, 4084, 24, 4, 756, 21, 1, 0, 4095, 56, 1, 8, 5, 1980, 4090, 2, 28, 0, 3, 0, 0, 4075, 4092, 5, 0, 1, 2, 0, 4091, 0, 3, 0, 3585, 32, 9, 0, 2624, 2763, 0, 4089, 7, 4093, 0, 4041, 1, 3905, 7, 3, 0, 2284, 4006, 0, 0, 6, 5, 14, 4096, 4082, 4075, 16, 4096, 0, 0, 4, 0, 0, 4070, 2, 3, 4092, 4096, 3, 325, 4, 0, 0, 3, 18, 2, 0, 5, 0, 0, 2, 3272, 2, 1, 0, 0, 0, 980, 1, 11, 4093, 0, 4083, 7, 4094, 2, 73] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 516, 8, 2, 0, 2206, 0, 0, 154, 0, 142, 0, 4092, 3844, 2, 2544, 2431, 940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 13, 1176, 3346, 1483, 1, 3, 69, 0, 523, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4063, 0, 13, 7, 0, 1569, 3460, 3, 1, 1, 0, 0, 3713, 1139, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3669, 4022, 0, 0, 2, 0, 0, 4095, 0, 1, 0, 3879, 3, 1, 230, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2391, 1756, 2, 0, 0, 3405, 1, 3543, 3, 280, 55, 67, 0, 3910, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 0, 222, 4093, 1552, 3541, 4, 4094, 0, 2, 0, 54, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 4095, 4092, 0, 1245, 58, 2, 0, 0, 2, 0, 0, 4, 1, 0, 0, 4092, 2917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 1, 2, 1, 5, 0, 3148, 0, 2, 4091, 4, 3979, 1, 2461, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16882, 14699, 15096, 16595, 15414, 18029, 16589, 17768] avg=16384.0000 max=18029 max_rank=5 min=14699 min_rank=1 imbalance_ratio=1.100403 max_load_reduction=113043 max_load_reduction_pct=0.862450 moved_to_echo=67800 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 605, 52, 160, 10, 948, 4, 116, 416, 0, 8, 1, 4051, 3311, 1, 2539, 2391, 940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1990, 1, 3, 1117, 3991, 3923, 0, 1, 391, 0, 514, 0, 0, 4093, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 695, 0, 0, 0, 3524, 164, 96, 0, 0, 2599, 3980, 27, 0, 3, 102, 0, 3793, 1135, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 35, 10, 0, 25, 4048, 0, 72, 0, 3137, 44, 0, 44, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3073, 3069, 0, 0, 0, 4089, 0, 2062, 22, 286, 5, 438, 1, 2918, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 4034, 1548, 2995, 5, 4090, 2, 18, 18, 56, 2, 3821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 4083, 3, 426, 53, 0, 0, 1, 2, 0, 0, 5, 8, 3, 0, 4087, 2905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 9, 5, 0, 0, 16, 3151, 2, 163, 3969, 0, 4004, 0, 2463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15553, 16059, 17224, 16296, 15974, 16644, 15548, 17774] avg=16384.0000 max=17774 max_rank=7 min=15548 min_rank=6 imbalance_ratio=1.084839 max_load_reduction=113298 max_load_reduction_pct=0.864395 moved_to_echo=65940 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 1170, 9, 954, 2, 48, 57, 30, 1802, 11, 102, 0, 4072, 2903, 0, 2539, 2434, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 694, 1, 18, 3656, 3605, 3407, 0, 0, 0, 0, 524, 0, 1, 4083, 31, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 3975, 23, 0, 2, 73, 3196, 4002, 20, 0, 33, 10, 144, 3496, 1126, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 3598, 0, 0, 12, 15, 35, 4089, 8, 0, 0, 3518, 0, 0, 288, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1801, 3836, 0, 0, 3, 4081, 0, 262, 21, 264, 1, 2018, 13, 4071, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 987, 36, 28, 4082, 1548, 598, 0, 3244, 6, 6, 41, 0, 23, 4086, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4087, 0, 1232, 0, 0, 0, 0, 3, 13, 7, 0, 1, 0, 0, 4041, 2888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3610, 31, 0, 0, 0, 162, 2505, 3, 0, 4059, 40, 4059, 7, 2449, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17072, 16100, 17229, 16148, 16381, 14826, 16360, 16956] avg=16384.0000 max=17229 max_rank=2 min=14826 min_rank=5 imbalance_ratio=1.051575 max_load_reduction=113843 max_load_reduction_pct=0.868553 moved_to_echo=64523 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1613, 6, 3, 0, 2394, 0, 0, 7, 0, 19, 0, 4096, 3541, 0, 2542, 2428, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1665, 10, 1, 2807, 3522, 3908, 2, 3, 8, 4, 523, 1, 0, 4000, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 2949, 0, 13, 0, 0, 3635, 3958, 1, 0, 0, 0, 0, 3800, 1149, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 3438, 4082, 0, 1, 1, 0, 0, 4096, 0, 18, 0, 2696, 0, 0, 0, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 3409, 0, 0, 0, 3970, 7, 3459, 60, 287, 0, 0, 1, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 4091, 1551, 2460, 11, 4089, 0, 1, 1, 8, 51, 4087, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4092, 2, 12, 7, 0, 0, 0, 2, 1, 13, 0, 14, 0, 0, 4079, 2947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3548, 0, 0, 0, 11, 26, 303, 1, 0, 4092, 64, 4085, 1, 2458, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17587, 16458, 16632, 15064, 19080, 16393, 15267, 14591] avg=16384.0000 max=19080 max_rank=4 min=14591 min_rank=7 imbalance_ratio=1.164551 max_load_reduction=111992 max_load_reduction_pct=0.854431 moved_to_echo=65331 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 9, 2415, 46, 44, 0, 290, 3, 0, 447, 0, 2, 46, 4096, 3176, 0, 2541, 2388, 944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3649, 2, 1, 2632, 4086, 3910, 0, 0, 0, 9, 506, 0, 1, 4087, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 10, 0, 3280, 0, 0, 0, 8, 2028, 3861, 12, 0, 0, 0, 13, 3812, 1119, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 4078, 0, 0, 0, 2, 38, 4088, 13, 0, 0, 3868, 0, 37, 0, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 1797, 0, 0, 0, 4050, 6, 787, 13, 284, 11, 6, 1, 3893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 170, 0, 20, 4096, 1542, 1300, 64, 3901, 4, 0, 0, 12, 3, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4088, 4084, 0, 1336, 32, 0, 0, 1, 23, 0, 0, 10, 0, 0, 18, 4069, 2865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 7, 2112, 1, 79, 4026, 0, 4089, 1, 2455, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16449, 18898, 15431, 16797, 14910, 15221, 16565, 16801] avg=16384.0000 max=18898 max_rank=1 min=14910 min_rank=4 imbalance_ratio=1.153442 max_load_reduction=112174 max_load_reduction_pct=0.855820 moved_to_echo=63497 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1125, 15, 423, 1, 2133, 0, 2, 19, 5, 0, 0, 4082, 2322, 0, 2535, 2437, 940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1764, 0, 0, 2478, 4065, 3997, 37, 0, 0, 0, 525, 1, 0, 4084, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 756, 21, 1, 0, 4095, 56, 1, 8, 5, 1980, 4090, 2, 28, 0, 3, 0, 3631, 1125, 1127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4092, 5, 0, 1, 2, 0, 4091, 0, 3, 0, 3585, 32, 9, 0, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2624, 2763, 0, 0, 7, 4093, 0, 4041, 1, 274, 7, 3, 0, 2284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 14, 4096, 1547, 4075, 16, 4096, 0, 0, 4, 0, 0, 4070, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4092, 4096, 3, 325, 4, 0, 0, 3, 18, 2, 0, 5, 0, 0, 2, 4089, 2881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3272, 2, 1, 0, 0, 0, 980, 1, 11, 4093, 0, 4083, 7, 2461, 2, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16039, 16979, 16929, 16588, 16097, 17931, 15523, 14986] avg=16384.0000 max=17931 max_rank=5 min=14986 min_rank=7 imbalance_ratio=1.094421 max_load_reduction=113141 max_load_reduction_pct=0.863197 moved_to_echo=64537 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4004, 67, 32, 0, 2389, 411, 0, 179, 0, 0, 17, 4092, 196, 2, 2542, 2422, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3735, 0, 0, 1167, 3187, 3169, 4, 0, 7, 3, 522, 0, 1, 3875, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 70, 0, 3670, 0, 24, 8, 2, 1294, 3739, 9, 12, 0, 2, 2, 3808, 1135, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4068, 4096, 0, 0, 0, 0, 119, 4092, 0, 21, 2, 4062, 0, 0, 68, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3626, 2694, 0, 0, 0, 3939, 3, 490, 8, 288, 3, 158, 0, 3900, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 46, 4089, 1551, 4001, 1, 4007, 1, 0, 5, 0, 34, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4092, 13, 2708, 67, 0, 0, 4, 33, 4, 0, 0, 68, 0, 0, 4089, 2911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3112, 7, 0, 0, 3, 0, 445, 0, 13, 4095, 673, 4094, 9, 2459, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17291, 15679, 14901, 17221, 15110, 17792, 18073, 15005] avg=16384.0000 max=18073 max_rank=6 min=14901 min_rank=2 imbalance_ratio=1.103088 max_load_reduction=112999 max_load_reduction_pct=0.862114 moved_to_echo=65980 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 746, 1, 35, 0, 998, 1, 1, 22, 0, 12, 135, 3982, 2351, 0, 2532, 2440, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3279, 4, 19, 2205, 2818, 2930, 0, 0, 111, 0, 526, 0, 0, 4096, 15, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 12, 0, 0, 3440, 595, 91, 0, 2, 3308, 4048, 0, 11, 0, 0, 79, 3769, 1142, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 3898, 0, 0, 0, 0, 4, 4083, 0, 7, 0, 3475, 104, 57, 7, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3181, 3255, 0, 0, 1, 4050, 0, 3269, 0, 285, 2, 67, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 0, 4096, 1545, 346, 84, 3991, 0, 0, 0, 45, 12, 4091, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 647, 4095, 4096, 0, 1019, 3, 2, 0, 0, 128, 0, 1, 178, 22, 0, 0, 4030, 2926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 19, 0, 50, 6, 1470, 0, 274, 3967, 2, 4088, 0, 2451, 0, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14199, 16200, 17630, 16363, 18106, 14236, 17147, 17191] avg=16384.0000 max=18106 max_rank=4 min=14199 min_rank=0 imbalance_ratio=1.105103 max_load_reduction=112966 max_load_reduction_pct=0.861862 moved_to_echo=66680 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[896, 32640, 32768, 128, 8320, 256, 128, 0, 128, 256, 128, 128, 256, 128, 128, 128, 32640, 23296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 31360, 32000, 128, 128, 128, 128, 256, 32768, 128, 128, 128, 28288, 256, 128, 640, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[20864, 128, 128, 17280, 28672, 26752, 128, 128, 640, 128, 4224, 128, 128, 32512, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[29696, 128, 128, 128, 128, 256, 14208, 128, 640, 32512, 896, 32512, 128, 19712, 128, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1664, 128, 128, 0, 29056, 896, 256, 128, 128, 19712, 31232, 128, 128, 128, 128, 256, 29824, 9088, 8960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[24704, 22656, 128, 0, 128, 31744, 128, 17920, 128, 2304, 128, 2816, 128, 29056, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 12288, 256, 1664, 128, 11520, 512, 256, 3072, 128, 384, 256, 32640, 21760, 128, 20352, 19456, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 1664, 128, 512, 32768, 12416, 19328, 256, 31616, 128, 128, 128, 256, 128, 32512, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 3, 1042, 14, 0, 2515, 3, 0, 0, 42, 4094, 4096, 7, 0, 4079, 4089, 4096, 6, 352, 4096, 4044, 0, 3, 0, 4005, 10, 0, 23, 1194, 4083, 0, 0, 4096, 12, 4093, 0, 0, 4083, 4058, 0, 202, 9, 0, 0, 0, 0, 4, 0, 1, 1, 4, 1974, 0, 0, 0, 0, 4095, 0, 15, 3976, 0, 4087, 0, 2, 4041, 0, 0, 11, 0, 2, 102, 4084, 1052, 0, 4096, 1482, 4089, 9, 4, 152, 0, 0, 286, 4077, 0, 0, 2, 12, 3, 9, 0, 4082, 0, 4090, 0, 0, 4095, 1181, 0, 19, 0, 4053, 4082, 0, 0, 4093, 0, 0, 4096, 0, 0, 17, 0, 1056, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[50, 0, 0, 2162, 0, 0, 8, 1, 5, 9, 221, 3103, 4015, 0, 0, 4094, 4095, 4095, 57, 1426, 4079, 4090, 204, 0, 2, 3700, 6, 1, 0, 3179, 4095, 0, 0, 4096, 1247, 4071, 14, 0, 1348, 4020, 0, 396, 3638, 3, 0, 1, 42, 0, 3, 0, 3, 8, 4050, 0, 2, 0, 0, 4094, 2, 317, 4001, 0, 4088, 3, 118, 3946, 0, 0, 4, 0, 53, 1229, 4077, 0, 10, 4093, 2454, 4096, 10, 23, 5, 4, 0, 58, 3684, 0, 2, 3, 289, 0, 145, 0, 4090, 0, 4068, 1438, 0, 331, 8, 0, 10, 0, 90, 4078, 0, 0, 4092, 0, 0, 4086, 2, 1, 0, 0, 2085, 4094, 0, 31, 0, 1, 20, 0, 0, 100, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[87, 1, 5, 609, 2, 0, 35, 1, 15, 1, 0, 4094, 3993, 0, 0, 4031, 4017, 4095, 1, 1031, 4079, 3328, 7, 0, 0, 2468, 0, 0, 0, 2338, 4095, 0, 0, 4090, 35, 4081, 1, 0, 2510, 4022, 4, 990, 3502, 0, 1629, 0, 11, 6, 0, 74, 0, 10, 3877, 0, 0, 0, 7, 4092, 1, 552, 3609, 0, 4087, 0, 4, 4087, 9, 0, 0, 3, 2, 195, 4088, 16, 0, 4089, 4094, 4095, 1, 0, 2, 0, 3, 24, 4096, 20, 5, 0, 0, 2, 0, 0, 4096, 0, 4091, 948, 1, 3876, 0, 0, 6, 2, 288, 3726, 1, 0, 4092, 0, 0, 4096, 0, 6, 0, 0, 1378, 4096, 0, 0, 1, 0, 0, 0, 3, 0, 3, 4, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[373, 1354, 1, 3294, 0, 0, 742, 0, 42, 16, 2, 2805, 4038, 0, 0, 4074, 4028, 4082, 0, 191, 3937, 2178, 2, 0, 0, 4089, 0, 0, 3, 2583, 4088, 0, 0, 4083, 2432, 4095, 0, 0, 2989, 4072, 1, 52, 1168, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 17, 9, 14, 4090, 0, 632, 3282, 0, 4092, 13, 6, 4057, 0, 0, 10, 0, 0, 19, 4080, 1, 0, 4060, 403, 4079, 1, 0, 7, 9, 9, 576, 4096, 0, 3, 0, 54, 0, 15, 18, 4096, 0, 4093, 994, 0, 2608, 237, 9, 0, 0, 3016, 3767, 41, 60, 4094, 0, 0, 4083, 3, 0, 0, 6, 1298, 4095, 0, 16, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[115, 991, 0, 2650, 0, 0, 1414, 0, 0, 0, 18, 2682, 4095, 0, 0, 4081, 4096, 3942, 1, 1785, 4052, 3956, 0, 0, 0, 4080, 0, 0, 1, 453, 4085, 0, 0, 4095, 1794, 4090, 0, 13, 1208, 4096, 0, 0, 1181, 0, 14, 0, 0, 0, 0, 52, 0, 355, 2247, 0, 0, 10, 0, 4076, 0, 443, 3061, 4, 4095, 0, 2, 3529, 0, 0, 0, 0, 0, 807, 3996, 166, 0, 4096, 1198, 4096, 0, 2, 199, 9, 0, 2544, 4095, 56, 0, 0, 216, 0, 0, 0, 4096, 0, 4090, 3, 0, 3991, 18, 0, 0, 10, 3984, 4096, 0, 29, 4089, 0, 0, 4023, 7, 0, 0, 0, 186, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[57, 268, 0, 3535, 0, 0, 102, 6, 1, 0, 0, 1872, 4038, 0, 0, 4095, 4096, 4091, 0, 1443, 4066, 2176, 0, 0, 2, 4096, 0, 0, 0, 1948, 3981, 0, 0, 4078, 3296, 4011, 4, 0, 4076, 4095, 0, 1, 2243, 0, 94, 0, 6, 0, 0, 105, 0, 25, 4095, 0, 0, 4, 0, 4084, 13, 218, 4064, 0, 4073, 0, 0, 4096, 1, 0, 0, 13, 0, 1851, 4095, 1, 0, 4094, 1242, 4031, 0, 0, 63, 1, 0, 939, 4096, 0, 1, 0, 623, 0, 0, 2, 4069, 0, 4003, 0, 15, 296, 375, 0, 0, 2, 1760, 3203, 0, 0, 4094, 0, 0, 4087, 2, 29, 344, 0, 1175, 3984, 0, 1, 0, 0, 1, 0, 0, 0, 25, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[46, 60, 0, 2919, 18, 0, 3855, 3, 5, 0, 16, 1863, 4060, 0, 0, 4095, 4064, 4095, 0, 112, 4066, 3557, 10, 3, 0, 4065, 0, 1, 0, 2820, 4040, 0, 0, 4089, 3, 4096, 0, 0, 4070, 4069, 0, 1, 2048, 0, 0, 0, 0, 4, 10, 51, 0, 70, 3727, 1, 0, 27, 30, 4096, 11, 567, 297, 0, 4088, 23, 29, 4076, 0, 4, 0, 9, 0, 1, 4083, 103, 40, 4065, 563, 4074, 0, 0, 400, 1, 1, 38, 4079, 15, 0, 0, 5, 2, 1053, 11, 4078, 0, 4085, 303, 0, 3904, 11, 0, 0, 0, 3787, 4048, 1, 19, 4066, 2, 2, 3933, 0, 22, 3, 22, 831, 4069, 0, 9, 0, 10, 0, 0, 0, 0, 2, 3, 48, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[6, 369, 0, 2401, 1, 13, 306, 6, 0, 0, 9, 2658, 4082, 0, 0, 4011, 4095, 4041, 10, 3126, 3933, 3324, 0, 0, 0, 4083, 0, 2, 1, 4078, 4091, 0, 0, 4087, 1624, 4080, 0, 0, 1799, 4092, 2, 89, 217, 0, 1145, 1, 0, 0, 0, 11, 0, 0, 3375, 0, 8, 0, 0, 4089, 0, 11, 4091, 0, 4096, 2, 339, 4068, 2, 0, 0, 0, 12, 436, 4091, 30, 0, 4096, 2743, 4093, 1, 0, 267, 0, 0, 10, 3950, 0, 0, 0, 10, 0, 57, 1, 3789, 0, 4087, 202, 0, 2836, 1, 5, 0, 12, 1848, 4075, 0, 5, 4096, 0, 0, 4081, 0, 0, 0, 6, 252, 4094, 0, 0, 0, 0, 0, 7, 2, 3, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[50, 0, 0, 2162, 0, 0, 8, 1, 5, 9, 221, 3103, 4015, 0, 0, 4094, 1383, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1651, 57, 1426, 4079, 4090, 204, 0, 2, 3700, 6, 1, 0, 3179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1837, 1247, 4071, 14, 0, 1348, 4020, 0, 396, 3638, 3, 0, 1, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 8, 4050, 0, 2, 0, 0, 4094, 2, 317, 4001, 0, 4088, 3, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 3946, 0, 0, 4, 0, 53, 1229, 4077, 0, 10, 1628, 2454, 4096, 10, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 58, 3684, 0, 2, 3, 289, 0, 145, 0, 4090, 0, 4068, 1438, 1858, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 331, 8, 0, 10, 0, 90, 4078, 0, 0, 3077, 0, 0, 4086, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2085, 4094, 0, 31, 0, 1, 20, 0, 0, 100, 0, 0, 0, 0, 4095, 4095, 2465, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15053, 18395, 16617, 17584, 17648, 16520, 11683, 17572] avg=16384.0000 max=18395 max_rank=1 min=11683 min_rank=6 imbalance_ratio=1.122742 max_load_reduction=112677 max_load_reduction_pct=0.859657 moved_to_echo=63423 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3, 1042, 14, 0, 2515, 3, 0, 0, 42, 4094, 4096, 7, 0, 4079, 1383, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1651, 6, 352, 4096, 4044, 0, 3, 0, 4005, 10, 0, 23, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1837, 12, 4093, 0, 0, 4083, 4058, 0, 202, 9, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4, 1974, 0, 0, 0, 0, 4095, 0, 15, 3976, 0, 4087, 0, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4041, 0, 0, 11, 0, 2, 102, 4084, 1052, 0, 1630, 1482, 4089, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 286, 4077, 0, 0, 2, 12, 3, 9, 0, 4082, 0, 4090, 0, 1859, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1181, 0, 19, 0, 4053, 4082, 0, 0, 3078, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 1056, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4083, 2466, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17281, 15384, 14298, 15166, 16508, 15448, 20604, 16383] avg=16384.0000 max=20604 max_rank=6 min=14298 min_rank=2 imbalance_ratio=1.257568 max_load_reduction=110468 max_load_reduction_pct=0.842804 moved_to_echo=68943 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[87, 1, 5, 609, 2, 0, 35, 1, 15, 1, 0, 4094, 3993, 0, 0, 4031, 1381, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1651, 1, 1031, 4079, 3328, 7, 0, 0, 2468, 0, 0, 0, 2338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1835, 35, 4081, 1, 0, 2510, 4022, 4, 990, 3502, 0, 1629, 0, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 10, 3877, 0, 0, 0, 7, 4092, 1, 552, 3609, 0, 4087, 0, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4087, 9, 0, 0, 3, 2, 195, 4088, 16, 0, 1627, 4094, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 24, 4096, 20, 5, 0, 0, 2, 0, 0, 4096, 0, 4091, 948, 1858, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3876, 0, 0, 6, 2, 288, 3726, 1, 0, 3077, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1378, 4096, 0, 0, 1, 0, 0, 0, 3, 0, 3, 4, 0, 0, 4017, 4095, 2462, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14257, 14903, 18626, 17322, 18221, 16019, 15079, 16645] avg=16384.0000 max=18626 max_rank=2 min=14257 min_rank=0 imbalance_ratio=1.136841 max_load_reduction=112446 max_load_reduction_pct=0.857895 moved_to_echo=65964 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[57, 268, 0, 3535, 0, 0, 102, 6, 1, 0, 0, 1872, 4038, 0, 0, 4095, 1377, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 0, 1443, 4066, 2176, 0, 0, 2, 4096, 0, 0, 0, 1948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1829, 3296, 4011, 4, 0, 4076, 4095, 0, 1, 2243, 0, 94, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 25, 4095, 0, 0, 4, 0, 4084, 13, 218, 4064, 0, 4073, 0, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 13, 0, 1851, 4095, 1, 0, 1629, 1242, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 1, 0, 939, 4096, 0, 1, 0, 623, 0, 0, 2, 4069, 0, 4003, 0, 1856, 872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 296, 375, 0, 0, 2, 1760, 3203, 0, 0, 3079, 0, 0, 4087, 2, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 1175, 3984, 0, 1, 0, 0, 1, 0, 0, 0, 25, 0, 0, 0, 4096, 3981, 2465, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15353, 15381, 19655, 17694, 16959, 16525, 12848, 16657] avg=16384.0000 max=19655 max_rank=2 min=12848 min_rank=6 imbalance_ratio=1.199646 max_load_reduction=111417 max_load_reduction_pct=0.850044 moved_to_echo=62989 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[373, 1354, 1, 3294, 0, 0, 742, 0, 42, 16, 2, 2805, 4038, 0, 0, 4074, 1382, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1638, 0, 191, 3937, 2178, 2, 0, 0, 4089, 0, 0, 3, 2583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1824, 2432, 4095, 0, 0, 2989, 4072, 1, 52, 1168, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 17, 9, 14, 4090, 0, 632, 3282, 0, 4092, 13, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4057, 0, 0, 10, 0, 0, 19, 4080, 1, 0, 1610, 403, 4079, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 9, 576, 4096, 0, 3, 0, 54, 0, 15, 18, 4096, 0, 4093, 994, 1857, 877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2608, 237, 9, 0, 0, 3016, 3767, 41, 60, 3071, 0, 0, 4083, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1298, 4095, 0, 16, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4028, 4088, 2450, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18130, 14621, 16636, 17250, 14266, 16704, 16895, 16570] avg=16384.0000 max=18130 max_rank=0 min=14266 min_rank=4 imbalance_ratio=1.106567 max_load_reduction=112942 max_load_reduction_pct=0.861679 moved_to_echo=64435 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[115, 991, 0, 2650, 0, 0, 1414, 0, 0, 0, 18, 2682, 4095, 0, 0, 4081, 1383, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1589, 1, 1785, 4052, 3956, 0, 0, 0, 4080, 0, 0, 1, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1837, 1794, 4090, 0, 13, 1208, 4096, 0, 0, 1181, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 355, 2247, 0, 0, 10, 0, 4076, 0, 443, 3061, 4, 4095, 0, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3529, 0, 0, 0, 0, 0, 807, 3996, 166, 0, 1630, 1198, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 9, 0, 2544, 4095, 56, 0, 0, 216, 0, 0, 0, 4096, 0, 4090, 3, 1789, 875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 18, 0, 0, 10, 3984, 4096, 0, 29, 3075, 0, 0, 4023, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 4085, 2466, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17431, 15917, 14233, 15355, 15426, 17972, 19233, 15505] avg=16384.0000 max=19233 max_rank=6 min=14233 min_rank=2 imbalance_ratio=1.173889 max_load_reduction=111839 max_load_reduction_pct=0.853264 moved_to_echo=68136 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[46, 60, 0, 2919, 18, 0, 3855, 3, 5, 0, 16, 1863, 4060, 0, 0, 4095, 1381, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1651, 0, 112, 4066, 3557, 10, 3, 0, 4065, 0, 1, 0, 2820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1834, 3, 4096, 0, 0, 4070, 4069, 0, 1, 2048, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 51, 0, 70, 3727, 1, 0, 27, 30, 4096, 11, 567, 297, 0, 4088, 23, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4076, 0, 4, 0, 9, 0, 1, 4083, 103, 40, 1617, 563, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 1, 1, 38, 4079, 15, 0, 0, 5, 2, 1053, 11, 4078, 0, 4085, 303, 1858, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 11, 0, 0, 0, 3787, 4048, 1, 19, 3058, 2, 2, 3933, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 22, 831, 4069, 0, 9, 0, 10, 0, 0, 0, 0, 2, 3, 48, 11, 4064, 4040, 2448, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18323, 16285, 16125, 14004, 14599, 16803, 18787, 16146] avg=16384.0000 max=18787 max_rank=6 min=14004 min_rank=3 imbalance_ratio=1.146667 max_load_reduction=112285 max_load_reduction_pct=0.856667 moved_to_echo=66335 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[6, 369, 0, 2401, 1, 13, 306, 6, 0, 0, 9, 2658, 4082, 0, 0, 4011, 1380, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1629, 10, 3126, 3933, 3324, 0, 0, 0, 4083, 0, 2, 1, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1833, 1624, 4080, 0, 0, 1799, 4092, 2, 89, 217, 0, 1145, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 3375, 0, 8, 0, 0, 4089, 0, 11, 4091, 0, 4096, 2, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 4068, 2, 0, 0, 0, 12, 436, 4091, 30, 0, 1630, 2743, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 0, 0, 10, 3950, 0, 0, 0, 10, 0, 57, 1, 3789, 0, 4087, 202, 1834, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2836, 1, 5, 0, 12, 1848, 4075, 0, 5, 3080, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 252, 4094, 0, 0, 0, 0, 0, 7, 2, 3, 0, 0, 0, 0, 4095, 4091, 2466, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15244, 20186, 14882, 16697, 17445, 15081, 15943, 15594] avg=16384.0000 max=20186 max_rank=1 min=14882 min_rank=2 imbalance_ratio=1.232056 max_load_reduction=110886 max_load_reduction_pct=0.845993 moved_to_echo=64063 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 13184, 128, 9472, 32384, 26752, 256, 128, 128, 30592, 128, 128, 128, 18688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[768, 3072, 128, 18688, 128, 128, 9088, 128, 128, 128, 384, 23296, 32512, 128, 0, 32640, 11136, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[512, 32000, 128, 128, 128, 128, 128, 4736, 32640, 1408, 128, 13056, 14208, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 22016, 1920, 128, 128, 128, 18944, 31104, 128, 128, 24704, 128, 128, 32512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1152, 128, 128, 4480, 32256, 128, 128, 128, 1280, 128, 1280, 128, 32512, 0, 32640, 3968, 14848, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 14720, 10496, 32640, 128, 128, 22144, 32640, 128, 1792, 14080, 128, 2944, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 384, 128, 512, 27520, 128, 128, 128, 128, 32768, 128, 2816, 26496, 128, 32768, 128, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[384, 128, 8320, 32640, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32640, 32640, 19712, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 4071, 2, 4, 4094, 0, 2, 6, 0, 0, 1, 0, 2, 2671, 1683, 0, 2, 1, 4, 0, 0, 1, 2029, 66, 15, 8, 0, 0, 4095, 8, 0, 32, 0, 0, 4095, 4089, 0, 0, 2, 3306, 4096, 4096, 0, 4094, 3973, 33, 0, 0, 2, 2, 0, 0, 4039, 8, 0, 5, 0, 0, 853, 127, 0, 4093, 0, 0, 4092, 1, 0, 4064, 1, 0, 182, 0, 0, 4086, 168, 29, 1, 0, 4091, 4094, 0, 0, 3952, 0, 0, 0, 0, 0, 3690, 4091, 0, 0, 0, 694, 0, 0, 0, 4092, 12, 0, 4094, 772, 4096, 0, 3, 4094, 167, 0, 0, 4096, 3, 2, 0, 87, 1, 4087, 3615, 0, 1001, 0, 0, 3646, 68, 1, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4091, 8, 0, 0, 4096, 0, 244, 4040, 0, 24, 1, 0, 0, 0, 0, 0, 3826, 4090, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 57, 0, 1, 4096, 0, 0, 157, 0, 3, 109, 4096, 20, 42, 0, 3974, 4088, 4095, 0, 4094, 4065, 1, 0, 0, 0, 0, 35, 0, 4080, 0, 0, 0, 0, 0, 1965, 0, 0, 4066, 0, 0, 3731, 0, 0, 4070, 21, 0, 4074, 0, 0, 3938, 3695, 16, 0, 0, 4096, 4096, 0, 0, 4057, 29, 0, 0, 0, 0, 0, 4096, 6, 10, 0, 59, 0, 218, 0, 4095, 0, 0, 4096, 4064, 4096, 2, 0, 3620, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 10, 18, 6, 0, 0, 2238, 928, 2, 4030] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4042, 44, 0, 0, 4055, 0, 1, 4093, 7, 90, 3, 0, 0, 0, 0, 0, 3850, 4095, 0, 2, 4, 0, 0, 1, 0, 1826, 0, 13, 2963, 5, 15, 4096, 9, 0, 540, 0, 262, 3413, 4091, 2, 4, 2, 4096, 4073, 4093, 0, 4096, 3729, 1, 0, 0, 0, 0, 37, 3, 4014, 80, 14, 0, 4, 0, 36, 0, 6, 4095, 10, 5, 4096, 2, 0, 4018, 0, 0, 234, 0, 0, 3659, 90, 0, 0, 1, 4091, 4096, 0, 0, 2664, 7, 0, 0, 8, 0, 1725, 4087, 0, 46, 0, 2, 2, 14, 0, 1812, 3, 0, 3853, 4093, 4096, 1, 0, 3545, 119, 2, 0, 4090, 0, 2, 4, 0, 2, 4086, 1744, 69, 7, 27, 0, 4091, 171, 443, 4050] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4074, 0, 8, 0, 4096, 0, 0, 3666, 0, 20, 1, 0, 0, 0, 0, 0, 551, 4081, 4, 0, 5, 0, 1, 2, 0, 2532, 0, 156, 222, 0, 0, 4082, 8, 12, 0, 0, 40, 2719, 4094, 0, 14, 1, 4096, 4068, 4045, 0, 4096, 4088, 7, 0, 0, 12, 0, 15, 0, 4020, 0, 0, 1, 23, 0, 653, 24, 1, 4081, 13, 3, 3970, 0, 0, 4095, 0, 0, 1103, 0, 0, 2726, 259, 10, 1, 0, 4095, 4094, 12, 0, 3803, 0, 0, 0, 0, 0, 3607, 4077, 0, 0, 0, 55, 52, 14, 9, 3648, 0, 0, 4080, 4069, 4085, 324, 7, 4096, 63, 12, 0, 4071, 0, 0, 0, 1, 20, 4051, 62, 1, 1, 0, 0, 4092, 2297, 380, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4092, 4, 0, 0, 4096, 0, 0, 1887, 1, 0, 4, 0, 0, 0, 0, 23, 3017, 3630, 0, 8, 0, 3, 187, 0, 0, 9, 40, 1, 36, 5, 0, 4088, 1, 0, 1, 0, 0, 3612, 4093, 0, 120, 0, 4088, 4048, 4095, 0, 4096, 3899, 0, 19, 2, 0, 0, 86, 0, 4092, 1, 0, 0, 6, 0, 3843, 0, 4, 4096, 8, 2, 4069, 7, 0, 4095, 0, 7, 1467, 0, 1, 4066, 4, 0, 4, 1, 4094, 4093, 0, 2, 3795, 0, 0, 2, 2, 0, 8, 4089, 20, 97, 2, 776, 1, 150, 0, 4078, 9, 0, 4096, 4092, 4092, 17, 1, 4065, 0, 3, 0, 4086, 0, 0, 1, 18, 0, 4087, 1417, 0, 263, 0, 0, 4095, 460, 0, 3997] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4090, 156, 0, 0, 4041, 4, 0, 4095, 1, 0, 4, 0, 2, 0, 0, 0, 2047, 4070, 0, 0, 0, 0, 11, 1, 0, 702, 2, 121, 643, 0, 0, 4096, 0, 1, 8, 0, 6, 2750, 4092, 0, 35, 0, 4066, 4096, 4091, 0, 4089, 4090, 0, 7, 0, 0, 0, 3, 0, 4096, 0, 0, 3, 1, 20, 848, 0, 0, 4076, 0, 0, 4040, 2, 0, 3960, 0, 4, 3664, 0, 0, 4078, 1038, 4, 0, 5, 4085, 4094, 2, 0, 4082, 0, 0, 0, 18, 0, 0, 4089, 0, 41, 0, 108, 0, 0, 1, 4082, 0, 0, 4092, 3981, 4096, 0, 0, 3960, 0, 0, 5, 4087, 0, 1, 0, 0, 0, 4079, 941, 59, 9, 0, 0, 3775, 13, 21, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4081, 298, 6, 0, 4019, 0, 149, 3680, 0, 22, 2, 10, 0, 33, 0, 0, 2463, 4090, 1, 0, 0, 0, 19, 0, 0, 420, 23, 0, 193, 9, 1, 4078, 0, 11, 60, 0, 0, 4075, 4083, 62, 0, 10, 4051, 3713, 4035, 0, 4083, 3330, 0, 1, 0, 4, 0, 0, 0, 3830, 0, 0, 0, 53, 170, 13, 0, 0, 4096, 0, 0, 4096, 8, 1, 3870, 92, 5, 3504, 6, 0, 3600, 3354, 50, 7, 0, 4060, 4075, 7, 0, 4051, 15, 0, 0, 0, 16, 0, 4079, 12, 14, 3, 0, 2, 44, 1, 4087, 0, 9, 4004, 4085, 4096, 50, 0, 4073, 15, 0, 37, 4081, 19, 6, 9, 0, 0, 4003, 0, 384, 240, 0, 0, 3627, 33, 0, 3965] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4096, 27, 4, 8, 4063, 0, 550, 3936, 4, 0, 0, 0, 3, 0, 0, 3, 3759, 3632, 4, 0, 0, 12, 0, 6, 2, 8, 0, 37, 2293, 0, 1, 4096, 0, 10, 0, 0, 0, 790, 4089, 0, 0, 0, 4019, 4095, 4087, 0, 4096, 3977, 23, 0, 0, 0, 0, 0, 3, 4078, 3, 0, 0, 14, 0, 1073, 23, 0, 4037, 34, 8, 3606, 0, 0, 3559, 6, 2, 4090, 3, 0, 3687, 760, 426, 6, 0, 4085, 4086, 0, 13, 3910, 77, 0, 7, 0, 0, 184, 4087, 1, 16, 1, 1025, 17, 581, 0, 3920, 4, 1, 3032, 4085, 4069, 0, 0, 4093, 132, 0, 0, 4086, 1, 0, 6, 0, 0, 4091, 100, 2, 1, 16, 1, 3969, 165, 0, 4060] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 4071, 2, 4, 4094, 0, 2, 6, 0, 0, 1, 0, 2, 2925, 755, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2671, 1683, 0, 2, 1, 4, 0, 0, 1, 2029, 66, 15, 8, 0, 0, 4095, 1721, 1411, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 32, 0, 0, 4095, 0, 0, 0, 2, 3306, 4096, 1620, 0, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 2, 2, 0, 0, 4039, 8, 0, 5, 0, 0, 853, 127, 0, 4094, 4089, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1168, 0, 0, 4092, 1, 0, 4064, 1, 0, 182, 0, 0, 4086, 168, 29, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3210, 0, 0, 3952, 0, 0, 0, 0, 0, 3690, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 694, 0, 0, 0, 4092, 12, 0, 4094, 772, 538, 0, 3, 4094, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2685, 3, 2, 0, 87, 1, 4087, 3615, 0, 1001, 0, 0, 3646, 68, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16613, 14591, 17132, 16153, 13792, 19034, 14466, 19291] avg=16384.0000 max=19291 max_rank=7 min=13792 min_rank=4 imbalance_ratio=1.177429 max_load_reduction=111781 max_load_reduction_pct=0.852821 moved_to_echo=66583 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4042, 44, 0, 0, 4055, 0, 1, 4093, 7, 90, 3, 0, 0, 0, 0, 0, 2927, 754, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3850, 4095, 0, 2, 4, 0, 0, 1, 0, 1826, 0, 13, 2963, 5, 15, 4096, 1720, 1409, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 540, 0, 262, 3413, 0, 2, 4, 2, 4096, 4073, 1619, 0, 0, 3729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 37, 3, 4014, 80, 14, 0, 4, 0, 36, 0, 6, 4096, 4091, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1168, 10, 5, 4096, 2, 0, 4018, 0, 0, 234, 0, 0, 3659, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 3212, 0, 0, 2664, 7, 0, 0, 8, 0, 1725, 4087, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 14, 0, 1812, 3, 0, 3853, 4093, 538, 1, 0, 3545, 119, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2681, 0, 2, 4, 0, 2, 4086, 1744, 69, 7, 27, 0, 4091, 171, 443, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16673, 20883, 17749, 15283, 13282, 15841, 13984, 17377] avg=16384.0000 max=20883 max_rank=1 min=13282 min_rank=4 imbalance_ratio=1.274597 max_load_reduction=110189 max_load_reduction_pct=0.840675 moved_to_echo=60484 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4092, 4, 0, 0, 4096, 0, 0, 1887, 1, 0, 4, 0, 0, 0, 0, 23, 2927, 754, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3017, 3630, 0, 8, 0, 3, 187, 0, 0, 9, 40, 1, 36, 5, 0, 4088, 1721, 1408, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 3612, 0, 0, 120, 0, 4088, 4048, 1620, 0, 0, 3899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 86, 0, 4092, 1, 0, 0, 6, 0, 3843, 0, 4, 4096, 4093, 2898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1169, 8, 2, 4069, 7, 0, 4095, 0, 7, 1467, 0, 1, 4066, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 3209, 0, 2, 3795, 0, 0, 2, 2, 0, 8, 4089, 20, 97, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 1, 150, 0, 4078, 9, 0, 4096, 4092, 538, 17, 1, 4065, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2678, 0, 0, 1, 18, 0, 4087, 1417, 0, 263, 0, 0, 4095, 460, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14444, 15037, 17389, 19140, 14899, 15321, 17826, 17016] avg=16384.0000 max=19140 max_rank=3 min=14444 min_rank=0 imbalance_ratio=1.168213 max_load_reduction=111932 max_load_reduction_pct=0.853973 moved_to_echo=65062 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4091, 8, 0, 0, 4096, 0, 244, 4040, 0, 24, 1, 0, 0, 0, 0, 0, 2906, 754, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3826, 4090, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 57, 0, 1, 4096, 1721, 1411, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 3, 109, 0, 20, 42, 0, 3974, 4088, 1620, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 35, 0, 4080, 0, 0, 0, 0, 0, 1965, 0, 0, 4094, 4096, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 0, 0, 3731, 0, 0, 4070, 21, 0, 4074, 0, 0, 3938, 3695, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3212, 0, 0, 4057, 29, 0, 0, 0, 0, 0, 4096, 6, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 218, 0, 4095, 0, 0, 4096, 4064, 538, 2, 0, 3620, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2685, 0, 0, 0, 0, 0, 4088, 10, 18, 6, 0, 0, 2238, 928, 2, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16821, 16087, 14078, 17172, 20705, 15506, 16698, 14005] avg=16384.0000 max=20705 max_rank=4 min=14005 min_rank=7 imbalance_ratio=1.263733 max_load_reduction=110367 max_load_reduction_pct=0.842033 moved_to_echo=66914 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4074, 0, 8, 0, 4096, 0, 0, 3666, 0, 20, 1, 0, 0, 0, 0, 0, 2921, 750, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 4081, 4, 0, 5, 0, 1, 2, 0, 2532, 0, 156, 222, 0, 0, 4082, 1704, 1408, 887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 0, 0, 40, 2719, 0, 0, 14, 1, 4096, 4068, 1591, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 12, 0, 15, 0, 4020, 0, 0, 1, 23, 0, 653, 24, 1, 4096, 4094, 2896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 13, 3, 3970, 0, 0, 4095, 0, 0, 1103, 0, 0, 2726, 259, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3207, 12, 0, 3803, 0, 0, 0, 0, 0, 3607, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 52, 14, 9, 3648, 0, 0, 4080, 4069, 532, 324, 7, 4096, 63, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2663, 0, 0, 0, 1, 20, 4051, 62, 1, 1, 0, 0, 4092, 2297, 380, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16193, 15635, 16637, 15842, 13340, 18801, 16961, 17663] avg=16384.0000 max=18801 max_rank=5 min=13340 min_rank=4 imbalance_ratio=1.147522 max_load_reduction=112271 max_load_reduction_pct=0.856560 moved_to_echo=66765 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4090, 156, 0, 0, 4041, 4, 0, 4095, 1, 0, 4, 0, 2, 0, 0, 0, 2913, 754, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2047, 4070, 0, 0, 0, 0, 11, 1, 0, 702, 2, 121, 643, 0, 0, 4096, 1719, 1408, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 6, 2750, 0, 0, 35, 0, 4066, 4096, 1618, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3, 0, 4096, 0, 0, 3, 1, 20, 848, 0, 0, 4089, 4092, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1163, 0, 0, 4040, 2, 0, 3960, 0, 4, 3664, 0, 0, 4078, 1038, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4085, 3210, 2, 0, 4082, 0, 0, 0, 18, 0, 0, 4089, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 1, 4082, 0, 0, 4092, 3981, 538, 0, 0, 3960, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2679, 0, 1, 0, 0, 0, 4079, 941, 59, 9, 0, 0, 3775, 13, 21, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16717, 15704, 16670, 16060, 17953, 15532, 16767, 15669] avg=16384.0000 max=17953 max_rank=4 min=15532 min_rank=5 imbalance_ratio=1.095764 max_load_reduction=113119 max_load_reduction_pct=0.863029 moved_to_echo=65921 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 27, 4, 8, 4063, 0, 550, 3936, 4, 0, 0, 0, 3, 0, 0, 3, 2885, 753, 652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3759, 3632, 4, 0, 0, 12, 0, 6, 2, 8, 0, 37, 2293, 0, 1, 4096, 1717, 1408, 882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 790, 0, 0, 0, 0, 4019, 4095, 1617, 0, 0, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 3, 4078, 3, 0, 0, 14, 0, 1073, 23, 0, 4096, 4089, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1152, 34, 8, 3606, 0, 0, 3559, 6, 2, 4090, 3, 0, 3687, 760, 426, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 3204, 0, 13, 3910, 77, 0, 7, 0, 0, 184, 4087, 1, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 17, 581, 0, 3920, 4, 1, 3032, 4085, 535, 0, 0, 4093, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2678, 1, 0, 6, 0, 0, 4091, 100, 2, 1, 16, 1, 3969, 165, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16984, 17857, 14508, 16284, 17339, 15585, 17425, 15090] avg=16384.0000 max=17857 max_rank=1 min=14508 min_rank=2 imbalance_ratio=1.089905 max_load_reduction=113215 max_load_reduction_pct=0.863762 moved_to_echo=65439 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4081, 298, 6, 0, 4019, 0, 149, 3680, 0, 22, 2, 10, 0, 33, 0, 0, 2927, 743, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2463, 4090, 1, 0, 0, 0, 19, 0, 0, 420, 23, 0, 193, 9, 1, 4078, 1695, 1406, 880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 60, 0, 0, 4075, 0, 62, 0, 10, 4051, 3713, 1597, 0, 0, 3330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 3830, 0, 0, 0, 53, 170, 13, 0, 0, 4083, 4083, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1169, 0, 0, 4096, 8, 1, 3870, 92, 5, 3504, 6, 0, 3600, 3354, 50, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 3195, 7, 0, 4051, 15, 0, 0, 0, 16, 0, 4079, 12, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 44, 1, 4087, 0, 9, 4004, 4085, 538, 50, 0, 4073, 15, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2675, 19, 6, 9, 0, 0, 4003, 0, 384, 240, 0, 0, 3627, 33, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16627, 15278, 16909, 15138, 19762, 15452, 16945, 14961] avg=16384.0000 max=19762 max_rank=4 min=14961 min_rank=7 imbalance_ratio=1.206177 max_load_reduction=111310 max_load_reduction_pct=0.849228 moved_to_echo=67120 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132992 observed_loss=-1920 tokens_per_expert=[32768, 640, 128, 128, 32640, 128, 1024, 29568, 128, 256, 128, 128, 128, 128, 0, 128, 23424, 6144, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 25728, 128, 128, 30336, 128, 0, 128, 128, 128, 9216, 32768, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 896, 0, 384, 21632, 0, 128, 256, 128, 31744, 32384, 12928, 0, 0, 31232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 256, 128, 32256, 128, 128, 128, 128, 256, 9344, 256, 128, 32768, 32768, 23296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[9344, 128, 128, 31744, 128, 128, 31744, 128, 128, 18432, 128, 128, 29952, 9472, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[22272, 29440, 128, 128, 128, 128, 256, 128, 128, 7552, 256, 384, 6528, 128, 128, 32768, 13824, 11392, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2816, 128, 1024, 128, 29824, 128, 128, 31360, 29312, 4352, 512, 128, 31616, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[21504, 128, 128, 128, 128, 128, 32640, 7936, 640, 1536, 128, 128, 29568, 4224, 896, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132992 slack_slots=7040 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 83, 4057, 0, 0, 0, 4095, 4001, 3819, 0, 0, 335, 1, 23, 4096, 4096, 3488, 0, 3994, 1, 4093, 12, 1708, 2, 0, 0, 1, 4094, 0, 2687, 0, 7, 0, 4094, 0, 0, 1, 0, 0, 4067, 0, 4094, 4096, 0, 14, 14, 1, 2, 0, 0, 4030, 3, 0, 4091, 4095, 4090, 4, 2839, 3, 0, 3924, 573, 0, 0, 0, 4, 0, 4092, 4095, 0, 4094, 4032, 7, 0, 2, 0, 0, 176, 3, 51, 177, 0, 0, 1, 0, 1, 0, 33, 154, 3, 298, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 2713, 0, 0, 3896, 4088, 0, 101, 0, 10, 4095, 0, 2, 1, 4095, 465, 4089, 12, 4, 4051, 2, 1494, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[9, 0, 16, 3999, 0, 20, 53, 4056, 4054, 0, 7, 0, 0, 0, 0, 4089, 4046, 835, 9, 4090, 0, 4081, 0, 0, 1520, 1, 0, 1, 4095, 14, 3905, 0, 0, 2, 4088, 0, 4, 10, 0, 0, 4019, 0, 4052, 4090, 13, 1, 2, 1, 0, 0, 0, 4096, 6, 0, 4095, 4079, 4086, 5, 2691, 26, 3, 4096, 501, 180, 0, 0, 0, 0, 4090, 4096, 0, 4065, 3881, 0, 0, 5, 0, 39, 3985, 0, 144, 0, 4, 0, 1, 26, 0, 2, 0, 1, 0, 816, 0, 4, 0, 0, 4, 5, 0, 6, 0, 0, 17, 4093, 74, 0, 1474, 4095, 14, 617, 71, 0, 4082, 13, 2, 13, 4087, 0, 4076, 0, 0, 4005, 23, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 22, 4073, 0, 0, 2, 4081, 4043, 2086, 0, 0, 0, 0, 0, 4094, 4089, 1441, 1, 4076, 0, 4096, 0, 0, 2362, 2974, 0, 0, 3989, 0, 3670, 1, 0, 8, 4089, 14, 67, 1, 0, 0, 4096, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 3929, 0, 0, 4067, 4096, 4096, 8, 2323, 0, 151, 3887, 0, 0, 326, 47, 0, 0, 3880, 4096, 0, 4092, 4056, 0, 1, 0, 0, 0, 9, 0, 6, 4, 1, 22, 1, 6, 0, 7, 24, 1, 0, 12, 0, 0, 0, 0, 0, 0, 1, 73, 10, 0, 3, 4087, 58, 0, 1488, 4096, 0, 0, 145, 2, 4096, 0, 0, 1, 4025, 5, 4096, 32, 23, 4057, 0, 4057, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 35, 4094, 3198, 162, 0, 0, 2, 0, 0, 4070, 4067, 888, 0, 4096, 1, 4096, 123, 0, 2474, 3185, 2, 0, 4096, 0, 2902, 0, 0, 0, 4096, 1, 3, 0, 0, 0, 3922, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 4, 4068, 4095, 4096, 0, 3997, 0, 0, 3888, 0, 0, 32, 0, 0, 0, 4080, 4096, 0, 4096, 4007, 317, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 1, 0, 32, 25, 0, 150, 0, 5, 0, 25, 0, 0, 9, 0, 0, 0, 0, 2714, 1, 0, 4093, 4096, 0, 5, 6, 2, 4096, 0, 0, 0, 4096, 3637, 4095, 1, 0, 4096, 0, 1244, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 111, 4095, 0, 0, 0, 4082, 1867, 1376, 5, 0, 0, 0, 197, 4091, 4084, 3338, 2, 4096, 0, 4096, 3, 1, 1563, 466, 0, 0, 4096, 0, 4042, 0, 0, 3, 4081, 7, 1880, 0, 0, 4, 4096, 0, 4095, 4083, 0, 0, 8, 0, 1, 0, 1, 4096, 0, 1, 4096, 4096, 4094, 0, 3639, 0, 0, 2853, 8, 0, 84, 0, 0, 4, 4083, 4096, 0, 4093, 4047, 2, 0, 0, 0, 0, 535, 0, 0, 0, 0, 2, 17, 0, 0, 1, 5, 0, 0, 1, 0, 1, 0, 13, 0, 0, 0, 0, 4, 0, 2, 4069, 18, 0, 2842, 4093, 0, 11, 0, 0, 4090, 4, 0, 0, 4019, 2715, 4089, 6, 0, 4049, 0, 1318, 4, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[107, 2, 0, 4092, 1, 0, 6, 4067, 3092, 309, 12, 1, 0, 0, 0, 4096, 4088, 3407, 0, 4089, 2, 4091, 29, 0, 711, 2057, 2, 4, 4063, 0, 3617, 0, 0, 1, 4086, 0, 18, 5, 8, 0, 4055, 0, 4096, 4095, 1, 0, 3, 0, 5, 0, 0, 3809, 0, 0, 4092, 4091, 4091, 0, 3265, 0, 11, 3798, 0, 0, 2, 1, 2, 0, 4056, 4086, 5, 4087, 4096, 12, 0, 4, 0, 0, 1863, 1, 0, 0, 0, 0, 23, 4, 2, 0, 0, 0, 0, 327, 5, 6, 0, 6, 9, 0, 0, 1, 0, 3, 2, 3199, 32, 0, 3547, 4032, 1, 2, 93, 0, 4072, 0, 0, 0, 4068, 729, 4083, 1, 3, 4090, 0, 3033, 9, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 0, 4092, 0, 0, 1, 4094, 4080, 2589, 8, 0, 1, 0, 12, 4075, 4096, 2114, 6, 4093, 0, 4078, 0, 0, 3897, 6, 27, 0, 4062, 3, 4084, 12, 0, 37, 4070, 0, 2, 0, 0, 6, 4055, 0, 4094, 4092, 0, 0, 0, 0, 6, 0, 0, 4094, 16, 0, 4054, 4096, 4082, 10, 3597, 9, 0, 1569, 20, 0, 0, 16, 14, 18, 4065, 4087, 0, 4095, 4087, 0, 1, 1, 0, 0, 1998, 0, 0, 0, 0, 0, 0, 0, 2, 0, 43, 1, 0, 42, 2, 0, 3, 0, 0, 22, 0, 1, 0, 3, 4, 4096, 0, 0, 4087, 4096, 3, 2, 0, 42, 4090, 28, 0, 0, 4085, 776, 4089, 0, 0, 3610, 5, 215, 0, 0, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 28, 4044, 5, 0, 0, 4094, 1371, 3481, 0, 0, 20, 9, 567, 4050, 4039, 3537, 0, 4070, 6, 4082, 1, 11, 2393, 1205, 25, 0, 4095, 0, 2441, 6, 1, 80, 3991, 4, 206, 0, 25, 3, 4013, 0, 4089, 4092, 51, 1, 0, 0, 10, 8, 0, 4093, 0, 0, 3755, 4092, 4038, 20, 4080, 0, 46, 3540, 11, 0, 882, 40, 0, 0, 4087, 4085, 0, 4080, 4094, 9, 2, 0, 0, 0, 40, 0, 0, 0, 8, 0, 6, 2, 0, 0, 408, 0, 0, 891, 0, 0, 6, 0, 1, 0, 4, 0, 12, 0, 0, 3803, 0, 0, 778, 4054, 0, 483, 35, 33, 4057, 0, 0, 35, 4060, 3077, 4095, 16, 12, 4006, 0, 21, 0, 14, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 83, 4057, 0, 0, 0, 3089, 4001, 3819, 0, 0, 335, 1, 23, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3488, 0, 3094, 1, 0, 12, 1708, 2, 0, 0, 1, 4094, 0, 2687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4094, 0, 0, 1, 0, 0, 4067, 0, 4094, 3869, 0, 14, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4030, 3, 0, 4091, 0, 1382, 4, 2839, 3, 0, 3924, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4092, 2879, 0, 4094, 4032, 7, 0, 2, 0, 0, 176, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 177, 0, 0, 1, 0, 1, 0, 33, 154, 3, 298, 0, 0, 0, 0, 4095, 4093, 4096, 3246, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 2713, 0, 0, 3896, 4088, 0, 101, 0, 10, 2311, 1216, 1006, 900, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 1, 4095, 465, 843, 12, 4, 4051, 2, 1494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19504, 15087, 16161, 16851, 15289, 16645, 16471, 15064] avg=16384.0000 max=19504 max_rank=0 min=15064 min_rank=7 imbalance_ratio=1.190430 max_load_reduction=111568 max_load_reduction_pct=0.851196 moved_to_echo=63469 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 22, 4073, 0, 0, 2, 3078, 4043, 2086, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1441, 1, 3158, 0, 0, 0, 0, 2362, 2974, 0, 0, 3989, 0, 3670, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4089, 14, 67, 1, 0, 0, 4096, 0, 4096, 3869, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3929, 0, 0, 4067, 0, 1385, 8, 2323, 0, 151, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 47, 0, 0, 3880, 2880, 0, 4092, 4056, 0, 1, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 1, 22, 1, 6, 0, 7, 24, 1, 0, 12, 0, 0, 0, 0, 4096, 4096, 4089, 3252, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 73, 10, 0, 3, 4087, 58, 0, 1488, 4096, 0, 0, 145, 2, 2314, 1216, 1003, 918, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4025, 5, 844, 32, 23, 4057, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17399, 17596, 16241, 15750, 15291, 16014, 15641, 17140] avg=16384.0000 max=17596 max_rank=1 min=15291 min_rank=4 imbalance_ratio=1.073975 max_load_reduction=113476 max_load_reduction_pct=0.865753 moved_to_echo=64086 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[107, 2, 0, 4092, 1, 0, 6, 3068, 3092, 309, 12, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3407, 0, 3168, 2, 0, 29, 0, 711, 2057, 2, 4, 4063, 0, 3617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 0, 18, 5, 8, 0, 4055, 0, 4096, 3868, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3809, 0, 0, 4092, 0, 1382, 0, 3265, 0, 11, 3798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 0, 4056, 2873, 5, 4087, 4096, 12, 0, 4, 0, 0, 1863, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4, 2, 0, 0, 0, 0, 327, 5, 6, 0, 6, 4091, 4091, 4088, 3242, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 0, 3, 2, 3199, 32, 0, 3547, 4032, 1, 2, 93, 0, 2312, 1213, 999, 921, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 4068, 729, 841, 1, 3, 4090, 0, 3033, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14786, 17060, 16141, 16362, 17002, 16282, 16593, 16846] avg=16384.0000 max=17060 max_rank=1 min=14786 min_rank=0 imbalance_ratio=1.041260 max_load_reduction=114012 max_load_reduction_pct=0.869843 moved_to_echo=66723 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 111, 4095, 0, 0, 0, 3079, 1867, 1376, 5, 0, 0, 0, 197, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 2, 3173, 0, 0, 3, 1, 1563, 466, 0, 0, 4096, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4081, 7, 1880, 0, 0, 4, 4096, 0, 4095, 3857, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4096, 0, 1, 4096, 0, 1384, 0, 3639, 0, 0, 2853, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 4, 4083, 2880, 0, 4093, 4047, 2, 0, 0, 0, 0, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 17, 0, 0, 1, 5, 0, 0, 1, 0, 1, 0, 13, 4096, 4096, 4084, 3246, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 4069, 18, 0, 2842, 4093, 0, 11, 0, 0, 2313, 1216, 1003, 923, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4, 0, 0, 4019, 2715, 843, 6, 0, 4049, 0, 1318, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14822, 16684, 18031, 16079, 15728, 15959, 16720, 17049] avg=16384.0000 max=18031 max_rank=2 min=14822 min_rank=0 imbalance_ratio=1.100525 max_load_reduction=113041 max_load_reduction_pct=0.862434 moved_to_echo=65456 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 16, 3999, 0, 20, 53, 3055, 4054, 0, 7, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 835, 9, 3166, 0, 0, 0, 0, 1520, 1, 0, 1, 4095, 14, 3905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4088, 0, 4, 10, 0, 0, 4019, 0, 4052, 3861, 13, 1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 4095, 0, 1372, 5, 2691, 26, 3, 4096, 501, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2877, 0, 4065, 3881, 0, 0, 5, 0, 39, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 4, 0, 1, 26, 0, 2, 0, 1, 0, 816, 0, 4, 0, 0, 4079, 4081, 4046, 3241, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 6, 0, 0, 17, 4093, 74, 0, 1474, 4095, 14, 617, 71, 0, 2313, 1219, 1001, 924, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 13, 2, 13, 4087, 0, 835, 0, 0, 4005, 23, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15302, 13546, 16053, 17071, 18942, 16846, 16156, 17156] avg=16384.0000 max=18942 max_rank=4 min=13546 min_rank=1 imbalance_ratio=1.156128 max_load_reduction=112130 max_load_reduction_pct=0.855484 moved_to_echo=69100 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 35, 3088, 3198, 162, 0, 0, 2, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 888, 0, 3173, 1, 0, 123, 0, 2474, 3185, 2, 0, 4096, 0, 2902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 3, 0, 0, 0, 3922, 0, 4096, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 4, 4068, 0, 1385, 0, 3997, 0, 0, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 4080, 2880, 0, 4096, 4007, 317, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 32, 25, 0, 150, 0, 5, 0, 25, 4095, 4096, 4067, 3251, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 2714, 1, 0, 4093, 4096, 0, 5, 6, 2, 2314, 1216, 1006, 923, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3637, 844, 1, 0, 4096, 0, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14648, 16844, 15987, 17278, 15545, 16144, 16612, 18014] avg=16384.0000 max=18014 max_rank=7 min=14648 min_rank=0 imbalance_ratio=1.099487 max_load_reduction=113058 max_load_reduction_pct=0.862564 moved_to_echo=66315 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4092, 0, 0, 1, 3088, 4080, 2589, 8, 0, 1, 0, 12, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2114, 6, 3171, 0, 0, 0, 0, 3897, 6, 27, 0, 4062, 3, 4084, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4070, 0, 2, 0, 0, 6, 4055, 0, 4094, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4094, 16, 0, 4054, 0, 1379, 10, 3597, 9, 0, 1569, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 14, 18, 4065, 2874, 0, 4095, 4087, 0, 1, 1, 0, 0, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 43, 1, 0, 42, 2, 0, 3, 0, 4096, 4078, 4096, 3246, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 1, 0, 3, 4, 4096, 0, 0, 4087, 4096, 3, 2, 0, 42, 2307, 1213, 1006, 922, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 28, 0, 0, 4085, 776, 843, 0, 0, 3610, 5, 215, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17947, 17382, 16129, 14754, 17169, 16005, 18031, 13655] avg=16384.0000 max=18031 max_rank=6 min=13655 min_rank=7 imbalance_ratio=1.100525 max_load_reduction=113041 max_load_reduction_pct=0.862434 moved_to_echo=64860 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 28, 4044, 5, 0, 0, 3088, 1371, 3481, 0, 0, 20, 9, 567, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3537, 0, 3153, 6, 0, 1, 11, 2393, 1205, 25, 0, 4095, 0, 2441, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 80, 3991, 4, 206, 0, 25, 3, 4013, 0, 4089, 3865, 51, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8, 0, 4093, 0, 0, 3755, 0, 1364, 20, 4080, 0, 46, 3540, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 882, 40, 0, 0, 4087, 2872, 0, 4080, 4094, 9, 2, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 6, 2, 0, 0, 408, 0, 0, 891, 0, 0, 6, 0, 4092, 4082, 4039, 3251, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 12, 0, 0, 3803, 0, 0, 778, 4054, 0, 483, 35, 33, 2282, 1213, 1006, 917, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 35, 4060, 3077, 844, 16, 12, 4006, 0, 21, 0, 14, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16664, 16873, 16329, 16927, 16106, 17177, 14848, 16148] avg=16384.0000 max=17177 max_rank=5 min=14848 min_rank=6 imbalance_ratio=1.048401 max_load_reduction=113895 max_load_reduction_pct=0.868950 moved_to_echo=64279 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 32128, 128, 128, 32384, 0, 11136, 128, 26496, 128, 256, 27648, 1152, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 128, 32640, 11520, 6784, 128, 128, 32000, 128, 15488, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 28800, 256, 0, 22272, 32768, 128, 1280, 384, 128, 18560, 9728, 8064, 7424, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 19072, 128, 25344, 128, 0, 256, 1792, 14976, 9984, 128, 128, 32640, 128, 27392, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 384, 32640, 128, 128, 128, 24704, 25728, 13824, 128, 128, 384, 128, 896, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[256, 256, 128, 128, 128, 128, 128, 128, 640, 256, 128, 2560, 128, 128, 128, 128, 32768, 32768, 32640, 25984, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1408, 128, 128, 128, 32512, 23040, 128, 32768, 32384, 384, 128, 128, 0, 128, 8832, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 256, 32640, 128, 2304, 128, 128, 128, 32384, 0, 32768, 30976, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[95, 1938, 347, 0, 0, 0, 5, 4041, 0, 1, 0, 19, 2485, 2735, 4096, 10, 4095, 48, 4088, 4096, 0, 19, 2746, 0, 104, 7, 0, 0, 0, 40, 0, 4096, 0, 0, 0, 3929, 0, 127, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 1, 0, 3922, 9, 4083, 0, 0, 4096, 12, 0, 0, 18, 0, 0, 4020, 1, 0, 3, 4096, 9, 4082, 4062, 0, 0, 22, 0, 4096, 0, 0, 0, 0, 0, 4, 1, 190, 0, 0, 4095, 0, 3896, 4096, 0, 0, 0, 0, 3924, 0, 4096, 0, 0, 4095, 4084, 0, 4096, 4095, 181, 58, 38, 0, 0, 266, 2163, 0, 4094, 0, 0, 3995, 4096, 0, 0, 36, 0, 0, 0, 3717, 4053, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 3128, 1254, 0, 1, 0, 0, 4067, 4, 1, 3, 5, 4096, 4017, 4095, 0, 4004, 19, 4096, 4089, 6, 0, 2592, 0, 1, 0, 0, 0, 0, 0, 56, 4096, 0, 0, 0, 4095, 0, 905, 0, 0, 27, 1, 0, 0, 0, 0, 28, 0, 0, 4, 353, 131, 4093, 3, 0, 4096, 18, 2, 3, 0, 0, 0, 4095, 2, 3, 0, 4093, 0, 4074, 4065, 0, 0, 956, 21, 4096, 0, 0, 5, 0, 0, 1, 0, 0, 1, 0, 4054, 0, 4095, 4096, 0, 0, 0, 0, 4086, 0, 4095, 0, 0, 4094, 4096, 0, 4096, 4066, 58, 0, 1, 0, 0, 4, 2, 0, 4094, 62, 0, 4091, 4079, 0, 0, 4, 1, 0, 0, 3004, 4092, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 108, 1, 4, 0, 6, 4094, 0, 20, 0, 0, 4096, 4088, 4090, 0, 4089, 233, 4048, 4093, 0, 0, 0, 0, 0, 5, 0, 0, 0, 15, 0, 3477, 0, 60, 0, 4096, 1, 2, 1, 2, 0, 0, 0, 0, 0, 0, 0, 1, 58, 0, 4080, 2, 4090, 0, 0, 4094, 0, 1, 3, 0, 1, 0, 4069, 0, 7, 4, 4084, 4, 4092, 4059, 0, 0, 4042, 0, 4093, 1, 11, 0, 6, 0, 0, 0, 23, 1, 0, 4094, 0, 4096, 4076, 0, 1, 3, 0, 4024, 0, 4094, 2, 1, 4088, 4082, 0, 4089, 4091, 345, 0, 39, 6, 1, 0, 2, 0, 4092, 0, 0, 4093, 4057, 0, 1, 1, 0, 48, 0, 4095, 4090, 0, 6, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[2, 2269, 2972, 0, 0, 0, 9, 3785, 0, 0, 0, 0, 4092, 283, 4052, 10, 4026, 460, 4077, 4092, 0, 0, 3676, 61, 12, 0, 0, 10, 0, 9, 3, 3866, 4, 0, 0, 4038, 0, 318, 0, 1, 5, 209, 0, 0, 3, 4, 0, 0, 2, 1, 4094, 101, 4096, 0, 0, 4083, 0, 0, 0, 10, 0, 5, 4084, 0, 0, 0, 4047, 0, 4096, 3997, 10, 10, 1, 0, 4080, 35, 0, 0, 0, 0, 0, 7, 0, 0, 210, 4096, 0, 4096, 4071, 0, 0, 0, 4, 4083, 0, 4060, 0, 0, 4094, 3970, 0, 4094, 4096, 0, 0, 0, 13, 0, 114, 44, 0, 4096, 4, 1, 4042, 3905, 0, 0, 5, 0, 0, 0, 2417, 4096, 218, 69, 58, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1, 3970, 3796, 7, 18, 7, 0, 4096, 0, 79, 0, 0, 4093, 1819, 4094, 0, 4085, 14, 4096, 4090, 0, 0, 13, 3, 0, 0, 0, 0, 4, 61, 147, 4084, 0, 0, 0, 4084, 6, 1702, 1, 0, 59, 168, 0, 0, 0, 0, 0, 0, 274, 0, 7, 156, 4072, 0, 0, 4089, 0, 0, 1, 34, 3, 0, 4017, 13, 0, 6, 4096, 2, 4096, 3955, 21, 1, 2760, 0, 4090, 0, 0, 0, 8, 0, 3, 0, 0, 0, 0, 4096, 1, 4086, 4083, 0, 0, 0, 0, 4091, 0, 4096, 0, 1, 4092, 3942, 8, 4085, 3972, 0, 0, 0, 8, 0, 0, 0, 0, 4094, 0, 1, 2785, 4080, 11, 0, 0, 3, 10, 12, 3122, 4092, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 60, 1374, 0, 0, 0, 0, 3114, 1, 236, 0, 0, 3812, 3973, 4095, 0, 4075, 0, 4095, 4096, 70, 0, 2005, 0, 0, 0, 0, 0, 0, 4, 927, 4093, 0, 0, 0, 4090, 45, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3719, 43, 4083, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 4048, 0, 0, 0, 4096, 0, 4096, 4012, 29, 0, 1000, 0, 4096, 0, 27, 0, 0, 1, 0, 0, 0, 0, 1, 4087, 0, 4096, 4096, 0, 0, 0, 0, 4037, 0, 4096, 0, 0, 4091, 4043, 0, 4096, 4095, 243, 27, 0, 0, 0, 11, 5, 0, 4096, 4, 0, 4079, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 3841, 1761, 29, 15, 0, 0, 4075, 0, 3, 0, 0, 4063, 2844, 4044, 0, 4008, 0, 4080, 4081, 13, 67, 3241, 0, 0, 0, 0, 1, 16, 0, 0, 3826, 37, 8, 10, 4080, 20, 504, 0, 0, 2, 0, 2, 8, 66, 0, 20, 0, 0, 10, 314, 5, 4070, 1, 0, 4068, 29, 0, 55, 1162, 4, 220, 4072, 17, 19, 5, 4095, 37, 4078, 979, 0, 1, 3931, 46, 4087, 10, 1167, 0, 40, 0, 0, 2, 0, 0, 1, 4042, 2, 3921, 4056, 0, 0, 3, 0, 3076, 3, 4059, 0, 0, 4053, 4096, 0, 4081, 4081, 111, 0, 2, 2, 1, 255, 0, 42, 4090, 0, 22, 3969, 4011, 0, 0, 32, 9, 0, 1, 1740, 3911, 0, 12, 73, 26] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1, 833, 1488, 7, 9, 0, 0, 3819, 0, 110, 2, 0, 4087, 3368, 4096, 108, 4041, 16, 4093, 4096, 0, 43, 162, 16, 2, 0, 7, 4, 0, 0, 2, 4093, 1, 0, 0, 4089, 0, 940, 0, 0, 0, 39, 15, 11, 4, 49, 0, 0, 43, 2, 4033, 0, 4094, 0, 0, 4062, 0, 0, 0, 5, 0, 2, 4091, 0, 0, 0, 4090, 0, 4072, 3995, 0, 0, 819, 671, 4072, 0, 0, 0, 0, 42, 0, 10, 85, 0, 0, 4096, 0, 4096, 3923, 0, 0, 0, 8, 3977, 5, 3581, 4, 0, 4096, 4073, 0, 4096, 4095, 181, 0, 0, 0, 0, 483, 7, 0, 4096, 0, 0, 4077, 4095, 2, 0, 54, 0, 0, 4, 4089, 4086, 0, 0, 7, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 108, 1, 4, 0, 6, 4094, 0, 20, 0, 0, 4096, 4088, 2012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 233, 4048, 2175, 0, 0, 0, 0, 0, 5, 0, 0, 0, 15, 0, 3477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 4096, 1, 2, 1, 2, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 3824, 2078, 1653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 4080, 2, 4090, 0, 0, 4094, 0, 1, 3, 0, 1, 0, 4069, 0, 1012, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 4084, 4, 4092, 4059, 0, 0, 4042, 0, 2600, 1, 11, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1, 0, 270, 0, 4096, 4076, 0, 1, 3, 0, 4024, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4088, 4082, 0, 3581, 4091, 345, 0, 39, 6, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4057, 0, 1, 1, 0, 48, 0, 4095, 4090, 0, 6, 0, 0, 508, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14429, 14042, 15810, 17675, 18910, 16588, 16238, 17380] avg=16384.0000 max=18910 max_rank=4 min=14042 min_rank=1 imbalance_ratio=1.154175 max_load_reduction=112162 max_load_reduction_pct=0.855728 moved_to_echo=69116 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[95, 1938, 347, 0, 0, 0, 5, 4041, 0, 1, 0, 19, 2485, 2735, 2015, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 48, 4088, 2176, 0, 19, 2746, 0, 104, 7, 0, 0, 0, 40, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3929, 0, 127, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 4094, 3825, 2081, 1655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3922, 9, 4083, 0, 0, 4096, 12, 0, 0, 18, 0, 0, 4020, 1, 1013, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 9, 4082, 4062, 0, 0, 22, 0, 2602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 190, 0, 0, 270, 0, 3896, 4096, 0, 0, 0, 0, 3924, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4084, 0, 3587, 4095, 181, 58, 38, 0, 0, 266, 2163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 4096, 0, 0, 36, 0, 0, 0, 3717, 4053, 0, 0, 0, 0, 509, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13691, 17419, 15715, 17440, 14876, 16477, 18567, 16887] avg=16384.0000 max=18567 max_rank=6 min=13691 min_rank=0 imbalance_ratio=1.133240 max_load_reduction=112505 max_load_reduction_pct=0.858345 moved_to_echo=66807 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3128, 1254, 0, 1, 0, 0, 4067, 4, 1, 3, 5, 4096, 4017, 2014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 19, 4096, 2173, 6, 0, 2592, 0, 1, 0, 0, 0, 0, 0, 56, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 905, 0, 0, 27, 1, 0, 0, 0, 0, 28, 0, 4094, 3787, 2081, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 353, 131, 4093, 3, 0, 4096, 18, 2, 3, 0, 0, 0, 4095, 2, 1013, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 4074, 4065, 0, 0, 956, 21, 2602, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 267, 0, 4095, 4096, 0, 0, 0, 0, 4086, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 3587, 4066, 58, 0, 1, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 4091, 4079, 0, 0, 4, 1, 0, 0, 3004, 4092, 0, 1, 0, 0, 509, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18590, 17043, 16670, 14077, 15819, 16641, 15908, 16324] avg=16384.0000 max=18590 max_rank=0 min=14077 min_rank=3 imbalance_ratio=1.134644 max_load_reduction=112482 max_load_reduction_pct=0.858170 moved_to_echo=64692 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 833, 1488, 7, 9, 0, 0, 3819, 0, 110, 2, 0, 4087, 3368, 2015, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 16, 4093, 2176, 0, 43, 162, 16, 2, 0, 7, 4, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 0, 940, 0, 0, 0, 39, 15, 11, 4, 49, 0, 0, 4096, 3826, 2081, 1655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 2, 4033, 0, 4094, 0, 0, 4062, 0, 0, 0, 5, 0, 2, 4091, 0, 1007, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4072, 3995, 0, 0, 819, 671, 2587, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 85, 0, 0, 270, 0, 4096, 3923, 0, 0, 0, 8, 3977, 5, 3581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 4073, 0, 3587, 4095, 181, 0, 0, 0, 0, 483, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4095, 2, 0, 54, 0, 0, 4, 4089, 4086, 0, 0, 7, 2, 509, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15847, 14655, 16806, 17604, 16276, 15955, 16526, 17403] avg=16384.0000 max=17604 max_rank=3 min=14655 min_rank=1 imbalance_ratio=1.074463 max_load_reduction=113468 max_load_reduction_pct=0.865692 moved_to_echo=66160 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 2269, 2972, 0, 0, 0, 9, 3785, 0, 0, 0, 0, 4092, 283, 1989, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 460, 4077, 2168, 0, 0, 3676, 61, 12, 0, 0, 10, 0, 9, 3, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4038, 0, 318, 0, 1, 5, 209, 0, 0, 3, 4, 0, 0, 4096, 3830, 2063, 1656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4094, 101, 4096, 0, 0, 4083, 0, 0, 0, 10, 0, 5, 4084, 0, 1012, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 4096, 3997, 10, 10, 1, 0, 2584, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 210, 266, 0, 4096, 4071, 0, 0, 0, 4, 4083, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3970, 0, 3582, 4096, 0, 0, 0, 13, 0, 114, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 4042, 3905, 0, 0, 5, 0, 0, 0, 2417, 4096, 218, 69, 58, 9, 512, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15411, 18368, 16227, 17756, 14780, 16797, 15913, 15820] avg=16384.0000 max=18368 max_rank=1 min=14780 min_rank=4 imbalance_ratio=1.121094 max_load_reduction=112704 max_load_reduction_pct=0.859863 moved_to_echo=63310 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 3970, 3796, 7, 18, 7, 0, 4096, 0, 79, 0, 0, 4093, 1819, 2014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 14, 4096, 2173, 0, 0, 13, 3, 0, 0, 0, 0, 4, 61, 147, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 6, 1702, 1, 0, 59, 168, 0, 0, 0, 0, 0, 0, 4094, 3826, 2080, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 0, 7, 156, 4072, 0, 0, 4089, 0, 0, 1, 34, 3, 0, 4017, 13, 1011, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 2, 4096, 3955, 21, 1, 2760, 0, 2598, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 270, 1, 4086, 4083, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 3942, 8, 3578, 3972, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2785, 4080, 11, 0, 0, 3, 10, 12, 3122, 4092, 0, 0, 0, 0, 507, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19900, 14680, 17672, 13942, 17543, 16630, 15601, 15104] avg=16384.0000 max=19900 max_rank=0 min=13942 min_rank=3 imbalance_ratio=1.214600 max_load_reduction=111172 max_load_reduction_pct=0.848175 moved_to_echo=64878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 60, 1374, 0, 0, 0, 0, 3114, 1, 236, 0, 0, 3812, 3973, 2014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4095, 2176, 70, 0, 2005, 0, 0, 0, 0, 0, 0, 4, 927, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 45, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3818, 2081, 1655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3719, 43, 4083, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 4048, 0, 1013, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4012, 29, 0, 1000, 0, 2602, 0, 27, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 269, 0, 4096, 4096, 0, 0, 0, 0, 4037, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4043, 0, 3587, 4095, 243, 27, 0, 0, 0, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4079, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 509, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14584, 17445, 15845, 17277, 15863, 16595, 16102, 17361] avg=16384.0000 max=17445 max_rank=1 min=14584 min_rank=0 imbalance_ratio=1.064758 max_load_reduction=113627 max_load_reduction_pct=0.866905 moved_to_echo=65921 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3841, 1761, 29, 15, 0, 0, 4075, 0, 3, 0, 0, 4063, 2844, 1989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 4080, 2168, 13, 67, 3241, 0, 0, 0, 0, 1, 16, 0, 0, 3826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 8, 10, 4080, 20, 504, 0, 0, 2, 0, 2, 8, 66, 0, 20, 0, 4090, 3776, 2055, 1649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 314, 5, 4070, 1, 0, 4068, 29, 0, 55, 1162, 4, 220, 4072, 17, 1010, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 5, 4095, 37, 4078, 979, 0, 1, 3931, 46, 2597, 10, 1167, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 266, 2, 3921, 4056, 0, 0, 3, 0, 3076, 3, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 4096, 0, 3574, 4081, 111, 0, 2, 2, 1, 255, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 3969, 4011, 0, 0, 32, 9, 0, 1, 1740, 3911, 0, 12, 73, 26, 507, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18620, 17420, 16327, 15301, 17005, 15389, 16217, 14793] avg=16384.0000 max=18620 max_rank=0 min=14793 min_rank=7 imbalance_ratio=1.136475 max_load_reduction=112452 max_load_reduction_pct=0.857941 moved_to_echo=63404 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 16128, 13184, 128, 128, 128, 128, 31104, 128, 512, 128, 128, 30848, 23168, 16128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32512, 896, 32768, 17408, 128, 256, 14464, 128, 128, 128, 128, 128, 128, 256, 1152, 31744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 32512, 128, 28672, 32640, 1152, 128, 128, 128, 128, 1152, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 384, 128, 256, 2176, 128, 32384, 32512, 0, 128, 128, 128, 31360, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[384, 128, 20608, 512, 32768, 128, 0, 32768, 128, 128, 128, 1280, 128, 256, 32512, 128, 8192, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 31232, 32512, 128, 128, 256, 128, 128, 128, 26368, 32640, 256, 128, 256, 128, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 32768, 128, 32768, 29184, 128, 128, 13568, 768, 20864, 128, 1280, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 32512, 128, 4608, 128, 128, 128, 512, 128, 128, 128, 128, 128, 128, 32768, 30592, 16640, 13312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1649, 0, 3011, 0, 4082, 6, 0, 5, 0, 0, 0, 0, 608, 748, 1, 0, 4, 0, 0, 4082, 0, 4096, 0, 170, 4093, 0, 1, 0, 0, 20, 0, 4096, 4051, 4095, 4096, 0, 2819, 0, 4079, 0, 2, 0, 1, 4096, 0, 1, 0, 0, 0, 3, 716, 36, 4088, 3988, 0, 0, 12, 30, 20, 0, 0, 3, 0, 0, 1, 4093, 0, 7, 0, 0, 6, 3629, 4096, 0, 4089, 2, 4057, 0, 4055, 3839, 0, 0, 0, 56, 0, 0, 4096, 1, 0, 0, 274, 0, 0, 0, 4090, 0, 0, 0, 3870, 4, 0, 0, 0, 0, 4094, 12, 0, 0, 0, 0, 0, 3846, 17, 0, 0, 0, 0, 0, 4093, 3557, 4096, 4096, 0, 0, 0, 0, 4092, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4072, 4, 1246, 0, 4094, 60, 8, 306, 0, 0, 6, 0, 20, 2, 0, 0, 0, 3, 0, 4088, 3, 4094, 9, 3141, 4081, 0, 0, 1, 10, 0, 0, 4094, 4093, 4092, 4085, 88, 3971, 0, 4022, 0, 0, 12, 2, 4092, 0, 0, 0, 0, 0, 1, 4032, 0, 4084, 4084, 0, 0, 0, 0, 260, 0, 2, 0, 3, 2, 0, 3922, 0, 0, 0, 0, 11, 3971, 4055, 321, 4083, 10, 4093, 0, 110, 3201, 2, 0, 0, 0, 0, 20, 4055, 0, 0, 0, 0, 0, 0, 0, 4081, 1, 1, 0, 4086, 0, 2, 0, 4, 2, 4093, 0, 3, 4, 2, 0, 0, 3888, 0, 317, 0, 5, 0, 16, 4092, 19, 4096, 4057, 0, 0, 2, 0, 4093, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[3378, 9, 4096, 0, 4073, 0, 0, 289, 3, 4, 0, 1, 1495, 1, 7, 0, 0, 0, 179, 4096, 0, 4083, 19, 764, 4074, 0, 1, 0, 0, 8, 0, 4096, 3674, 3711, 4096, 0, 4095, 0, 3250, 0, 49, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3235, 0, 4052, 4095, 3, 0, 0, 13, 551, 0, 0, 0, 18, 0, 0, 4096, 0, 0, 227, 0, 0, 2621, 4091, 0, 3919, 6, 4038, 0, 30, 472, 590, 0, 0, 221, 478, 3, 4008, 0, 2, 0, 2243, 0, 55, 18, 4094, 0, 2, 0, 4080, 0, 1, 0, 0, 0, 4096, 0, 0, 108, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4036, 1453, 4096, 4094, 0, 0, 2, 5, 4087, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[905, 0, 4085, 0, 4096, 27, 0, 0, 0, 0, 7, 7, 2633, 17, 33, 0, 11, 0, 0, 4018, 4, 4041, 0, 0, 4092, 4, 0, 13, 0, 17, 6, 4082, 4091, 3556, 4081, 0, 3073, 1, 4094, 11, 4, 9, 0, 4080, 9, 2, 0, 0, 20, 0, 613, 0, 4096, 4096, 762, 0, 0, 128, 0, 20, 5, 7, 0, 0, 0, 4075, 0, 144, 732, 28, 0, 2287, 4077, 0, 4090, 2, 2677, 0, 3184, 3, 53, 0, 13, 55, 0, 211, 4018, 0, 0, 0, 3608, 0, 0, 0, 4057, 21, 6, 0, 3970, 15, 4, 0, 0, 7, 4075, 0, 0, 14, 0, 34, 0, 4078, 143, 6, 1, 5, 0, 0, 4090, 4091, 4090, 4081, 0, 0, 0, 0, 4087, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1445, 0, 3810, 0, 4096, 0, 0, 71, 0, 3, 0, 1, 1992, 1468, 0, 0, 0, 0, 94, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 4096, 4008, 4091, 4096, 0, 4095, 0, 3007, 0, 0, 0, 0, 3672, 0, 0, 0, 0, 0, 0, 4071, 21, 4047, 4007, 55, 0, 0, 0, 595, 0, 0, 0, 18, 0, 0, 4095, 0, 0, 1003, 0, 0, 4061, 4096, 0, 4095, 0, 4096, 0, 0, 59, 0, 0, 1, 0, 11, 68, 4023, 0, 0, 0, 1779, 0, 0, 0, 4096, 0, 0, 0, 4096, 11, 0, 8, 0, 42, 4096, 0, 0, 36, 64, 0, 0, 4055, 1, 0, 51, 0, 0, 0, 4096, 1501, 4096, 4096, 0, 0, 0, 0, 4096, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4078, 0, 3989, 0, 4020, 0, 13, 8, 0, 0, 17, 49, 11, 0, 0, 9, 0, 8, 400, 4095, 5, 4030, 0, 33, 4070, 9, 0, 0, 0, 0, 7, 4062, 4096, 4088, 3993, 0, 3676, 0, 3752, 1, 0, 25, 7, 4086, 2, 10, 12, 11, 0, 0, 936, 10, 4029, 3777, 81, 35, 0, 0, 894, 4, 0, 6, 0, 6, 0, 4046, 0, 0, 717, 0, 21, 54, 4060, 0, 4052, 0, 4091, 0, 1275, 2503, 95, 0, 0, 0, 24, 791, 4047, 0, 61, 8, 2499, 38, 0, 62, 3963, 30, 22, 0, 4094, 0, 0, 28, 0, 0, 4078, 1, 0, 0, 3047, 1, 0, 4017, 136, 40, 0, 23, 0, 0, 4088, 563, 4076, 4041, 1, 0, 1, 0, 3886, 4042] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3017, 0, 3931, 0, 4035, 13, 0, 246, 4, 10, 0, 0, 2988, 63, 30, 0, 0, 0, 0, 4093, 5, 4062, 0, 0, 4090, 0, 1, 0, 0, 137, 0, 4096, 4096, 4027, 4093, 0, 4051, 0, 3928, 0, 42, 0, 5, 3833, 0, 0, 0, 0, 150, 0, 60, 62, 4092, 4036, 105, 0, 1, 0, 1, 0, 0, 0, 0, 0, 18, 4096, 0, 0, 2837, 0, 47, 1446, 4096, 11, 4096, 0, 3449, 0, 528, 1827, 32, 0, 0, 60, 0, 789, 4092, 6, 0, 0, 1770, 0, 3, 0, 4087, 4, 0, 0, 4093, 1, 0, 0, 0, 526, 3949, 0, 0, 0, 21, 0, 0, 3564, 0, 0, 0, 0, 1, 13, 4093, 1742, 4070, 4096, 21, 0, 0, 31, 4085, 4069] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[245, 0, 3670, 0, 4096, 0, 0, 0, 0, 1, 11, 873, 3991, 0, 0, 0, 0, 0, 21, 4089, 0, 4095, 0, 27, 4002, 0, 1, 7, 22, 4, 18, 4057, 3779, 2865, 4071, 1, 3805, 6, 3848, 0, 0, 0, 0, 4084, 0, 0, 6, 0, 1, 0, 3265, 0, 4022, 3859, 0, 0, 0, 790, 0, 12, 2, 2, 6, 7, 0, 4087, 14, 44, 68, 2, 0, 2490, 4070, 71, 4088, 4, 1120, 0, 3734, 2211, 31, 7, 0, 339, 53, 0, 4077, 2, 22, 15, 0, 1, 6, 0, 4086, 5, 2, 0, 3917, 34, 0, 0, 0, 0, 4096, 0, 1, 147, 0, 0, 0, 4089, 0, 65, 0, 0, 0, 0, 4070, 4079, 4047, 4078, 0, 0, 0, 0, 4057, 4082] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4072, 4, 1246, 0, 4094, 60, 8, 306, 0, 0, 6, 0, 20, 2, 0, 0, 2971, 875, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4088, 3, 4094, 9, 3141, 4081, 0, 0, 1, 10, 0, 0, 3512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4092, 0, 88, 3971, 0, 4022, 0, 0, 12, 2, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4032, 0, 4084, 4084, 0, 0, 0, 0, 260, 0, 2, 0, 3, 2, 3656, 1956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3922, 0, 0, 0, 0, 11, 3971, 0, 321, 2135, 10, 4093, 0, 110, 3201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 20, 4055, 0, 0, 0, 0, 0, 0, 0, 4081, 1, 4096, 2099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4086, 0, 2, 0, 4, 2, 4093, 0, 3, 4, 2, 0, 0, 3888, 1948, 1287, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 5, 0, 16, 1930, 19, 0, 4057, 0, 0, 2, 0, 4093, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14093, 18942, 17401, 18080, 17774, 14354, 15902, 14526] avg=16384.0000 max=18942 max_rank=1 min=14093 min_rank=0 imbalance_ratio=1.156128 max_load_reduction=112130 max_load_reduction_pct=0.855484 moved_to_echo=62556 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3378, 9, 4096, 0, 4073, 0, 0, 289, 3, 4, 0, 1, 1495, 1, 7, 0, 2974, 863, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 4096, 0, 4083, 19, 764, 4074, 0, 1, 0, 0, 8, 0, 3514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3674, 3711, 0, 0, 4095, 0, 3250, 0, 49, 0, 0, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3235, 0, 4052, 4095, 3, 0, 0, 13, 551, 0, 0, 0, 18, 0, 3665, 1974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 227, 0, 0, 2621, 0, 0, 2049, 6, 4038, 0, 30, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 590, 0, 0, 221, 478, 3, 4008, 0, 2, 0, 2243, 0, 55, 18, 4094, 0, 4096, 2117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4080, 0, 1, 0, 0, 0, 4096, 0, 0, 108, 0, 0, 0, 4096, 1870, 1269, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1904, 1453, 0, 4094, 0, 0, 2, 5, 4087, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17624, 16738, 15901, 17606, 13539, 17925, 16104, 15635] avg=16384.0000 max=17925 max_rank=5 min=13539 min_rank=4 imbalance_ratio=1.094055 max_load_reduction=113147 max_load_reduction_pct=0.863243 moved_to_echo=63203 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3017, 0, 3931, 0, 4035, 13, 0, 246, 4, 10, 0, 0, 2988, 63, 30, 0, 2783, 875, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 4062, 0, 0, 4090, 0, 1, 0, 0, 137, 0, 3514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4027, 0, 0, 4051, 0, 3928, 0, 42, 0, 5, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 60, 62, 4092, 4036, 105, 0, 1, 0, 1, 0, 0, 0, 0, 0, 3664, 1975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4096, 0, 0, 2837, 0, 47, 1446, 0, 11, 2142, 0, 3449, 0, 528, 1827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 60, 0, 789, 4092, 6, 0, 0, 1770, 0, 3, 0, 4087, 4, 4070, 2121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 526, 3949, 0, 0, 0, 21, 0, 0, 3564, 1954, 1287, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 13, 1931, 1742, 0, 4096, 21, 0, 0, 31, 4085, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18424, 15902, 17199, 14146, 16401, 17034, 15977, 15989] avg=16384.0000 max=18424 max_rank=0 min=14146 min_rank=3 imbalance_ratio=1.124512 max_load_reduction=112648 max_load_reduction_pct=0.859436 moved_to_echo=65401 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1445, 0, 3810, 0, 4096, 0, 0, 71, 0, 3, 0, 1, 1992, 1468, 0, 0, 2666, 876, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 3514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 4091, 0, 0, 4095, 0, 3007, 0, 0, 0, 0, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 21, 4047, 4007, 55, 0, 0, 0, 595, 0, 0, 0, 18, 0, 3665, 1976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1003, 0, 0, 4061, 0, 0, 2141, 0, 4096, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 68, 4023, 0, 0, 0, 1779, 0, 0, 0, 4096, 0, 4096, 2120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 8, 0, 42, 4096, 0, 0, 36, 64, 0, 0, 4055, 1954, 1288, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 51, 0, 0, 0, 1932, 1501, 0, 4096, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16859, 15901, 16207, 18455, 15455, 16194, 16232, 15769] avg=16384.0000 max=18455 max_rank=3 min=15455 min_rank=4 imbalance_ratio=1.126404 max_load_reduction=112617 max_load_reduction_pct=0.859200 moved_to_echo=63650 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[905, 0, 4085, 0, 4096, 27, 0, 0, 0, 0, 7, 7, 2633, 17, 33, 0, 2965, 877, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4018, 4, 4041, 0, 0, 4092, 4, 0, 13, 0, 17, 6, 3496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3556, 0, 0, 3073, 1, 4094, 11, 4, 9, 0, 1115, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 613, 0, 4096, 4096, 762, 0, 0, 128, 0, 20, 5, 7, 0, 0, 3652, 1967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 144, 732, 28, 0, 2287, 0, 0, 2135, 2, 2677, 0, 3184, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 13, 55, 0, 211, 4018, 0, 0, 0, 3608, 0, 0, 0, 4057, 21, 4090, 2110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3970, 15, 4, 0, 0, 7, 4075, 0, 0, 14, 0, 34, 0, 4078, 1955, 1290, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 6, 1, 5, 0, 0, 1923, 4091, 0, 4081, 0, 0, 0, 0, 4087, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16081, 15702, 15965, 15366, 15267, 18236, 16034, 18421] avg=16384.0000 max=18421 max_rank=7 min=15267 min_rank=4 imbalance_ratio=1.124329 max_load_reduction=112651 max_load_reduction_pct=0.859459 moved_to_echo=67958 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[245, 0, 3670, 0, 4096, 0, 0, 0, 0, 1, 11, 873, 3991, 0, 0, 0, 2965, 870, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4089, 0, 4095, 0, 27, 4002, 0, 1, 7, 22, 4, 18, 3480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3779, 2865, 0, 1, 3805, 6, 3848, 0, 0, 0, 0, 1119, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3265, 0, 4022, 3859, 0, 0, 0, 790, 0, 12, 2, 2, 6, 7, 3644, 1962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 14, 44, 68, 2, 0, 2490, 0, 71, 2138, 4, 1120, 0, 3734, 2211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 7, 0, 339, 53, 0, 4077, 2, 22, 15, 0, 1, 6, 0, 4086, 5, 4047, 2108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3917, 34, 0, 0, 0, 0, 4096, 0, 1, 147, 0, 0, 0, 4089, 1950, 1280, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 1920, 4079, 0, 4078, 0, 0, 0, 0, 4057, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17149, 15766, 15429, 17572, 15983, 14799, 16093, 18281] avg=16384.0000 max=18281 max_rank=7 min=14799 min_rank=5 imbalance_ratio=1.115784 max_load_reduction=112791 max_load_reduction_pct=0.860527 moved_to_echo=65156 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4078, 0, 3989, 0, 4020, 0, 13, 8, 0, 0, 17, 49, 11, 0, 0, 9, 2966, 874, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 400, 4095, 5, 4030, 0, 33, 4070, 9, 0, 0, 0, 0, 7, 3484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 3676, 0, 3752, 1, 0, 25, 7, 1120, 2, 10, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 936, 10, 4029, 3777, 81, 35, 0, 0, 894, 4, 0, 6, 0, 6, 3574, 1958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 717, 0, 21, 54, 0, 0, 2119, 0, 4091, 0, 1275, 2503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 24, 791, 4047, 0, 61, 8, 2499, 38, 0, 62, 3963, 30, 4076, 2102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4094, 0, 0, 28, 0, 0, 4078, 1, 0, 0, 3047, 1, 0, 4017, 1933, 1285, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 40, 0, 23, 0, 0, 1929, 563, 0, 4041, 1, 0, 1, 0, 3886, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16453, 16141, 16800, 15310, 14826, 17796, 19084, 14662] avg=16384.0000 max=19084 max_rank=6 min=14662 min_rank=7 imbalance_ratio=1.164795 max_load_reduction=111988 max_load_reduction_pct=0.854401 moved_to_echo=66368 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1649, 0, 3011, 0, 4082, 6, 0, 5, 0, 0, 0, 0, 608, 748, 1, 0, 2974, 875, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4082, 0, 4096, 0, 170, 4093, 0, 1, 0, 0, 20, 0, 3514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4095, 0, 0, 2819, 0, 4079, 0, 2, 0, 1, 1122, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 716, 36, 4088, 3988, 0, 0, 12, 30, 20, 0, 0, 3, 0, 0, 3666, 1975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 7, 0, 0, 6, 3629, 0, 0, 2138, 2, 4057, 0, 4055, 3839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 4096, 1, 0, 0, 274, 0, 0, 0, 4090, 0, 4096, 2121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3870, 4, 0, 0, 0, 0, 4094, 12, 0, 0, 0, 0, 0, 3846, 1951, 1287, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 1931, 3557, 0, 4096, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14389, 15980, 16170, 14537, 21827, 14734, 15646, 17789] avg=16384.0000 max=21827 max_rank=4 min=14389 min_rank=0 imbalance_ratio=1.332214 max_load_reduction=109245 max_load_reduction_pct=0.833473 moved_to_echo=69996 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 768, 32768, 128, 32640, 128, 4224, 32640, 128, 128, 128, 128, 256, 128, 28032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[256, 128, 17024, 256, 32512, 32000, 1024, 128, 128, 1024, 2432, 128, 128, 128, 128, 128, 29312, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32512, 128, 256, 5632, 128, 128, 20608, 0, 512, 17024, 128, 27648, 0, 12928, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 32256, 128, 128, 128, 128, 640, 32640, 128, 128, 384, 3200, 128, 0, 31744, 15616, 10368, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32000, 30592, 0, 128, 29696, 128, 30080, 128, 128, 128, 128, 8832, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 512, 128, 128, 128, 128, 15488, 17024, 0, 32640, 128, 0, 128, 128, 32512, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[18816, 128, 27904, 0, 32640, 128, 128, 1024, 128, 128, 128, 1024, 13824, 2304, 128, 128, 23296, 7040, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[896, 128, 128, 768, 640, 1920, 32512, 128, 128, 128, 12288, 128, 128, 128, 32640, 128, 32768, 17024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[3, 2, 0, 4088, 2, 0, 10, 4077, 5, 4095, 0, 0, 4094, 1, 9, 0, 4, 0, 307, 0, 0, 1, 8, 4089, 0, 0, 0, 16, 4093, 0, 4032, 4084, 0, 4091, 0, 0, 4082, 4012, 0, 0, 1, 19, 6, 103, 0, 3938, 0, 0, 2, 4091, 0, 0, 0, 3259, 13, 4094, 3825, 3, 0, 3803, 2, 4077, 4081, 1270, 6, 0, 0, 0, 4, 3886, 0, 3, 4094, 4, 0, 0, 21, 1, 350, 4090, 0, 2, 0, 161, 3, 419, 5, 1, 9, 3876, 24, 4084, 0, 0, 4082, 0, 4, 42, 0, 4058, 0, 4058, 0, 4087, 1, 4089, 13, 0, 3194, 0, 0, 0, 0, 0, 0, 0, 2, 129, 57, 3730, 110, 0, 0, 4004, 7, 393, 157, 20] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 9, 4078, 0, 0, 0, 4072, 8, 4032, 220, 0, 4095, 1, 0, 788, 13, 8, 4095, 0, 0, 0, 0, 4096, 1, 0, 11, 0, 4084, 0, 4089, 4095, 7, 4085, 6, 10, 4084, 2976, 49, 4, 0, 26, 19, 1, 2455, 351, 9, 0, 0, 3999, 13, 6, 3, 4035, 0, 4044, 4094, 0, 0, 264, 8, 3683, 3316, 15, 0, 0, 3, 0, 0, 3789, 0, 0, 3981, 0, 49, 0, 4, 0, 1727, 4057, 0, 0, 0, 50, 0, 27, 0, 0, 285, 4091, 8, 4082, 1174, 0, 4051, 0, 0, 4, 0, 4094, 5, 4084, 0, 4079, 0, 4096, 0, 0, 3611, 0, 114, 0, 3, 3, 0, 0, 0, 0, 36, 4095, 0, 0, 0, 3932, 10, 171, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 3, 0, 4096, 0, 0, 1, 0, 22, 4096, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 4096, 5, 4096, 4096, 0, 4064, 0, 0, 3820, 3, 0, 0, 0, 0, 15, 249, 3883, 632, 0, 0, 0, 4084, 0, 0, 0, 4088, 81, 4096, 4086, 0, 0, 3, 0, 4014, 3204, 0, 0, 0, 0, 396, 19, 1039, 0, 1, 3161, 0, 0, 0, 5, 0, 4020, 4096, 5, 0, 0, 0, 0, 209, 0, 5, 0, 4096, 0, 4096, 0, 0, 4096, 62, 0, 0, 0, 4084, 0, 4096, 0, 4084, 0, 4095, 0, 2, 4096, 1, 9, 0, 0, 0, 1415, 0, 0, 0, 0, 2482, 723, 0, 0, 4091, 0, 3362, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4091, 0, 3, 0, 4095, 0, 4096, 0, 0, 4091, 0, 4, 0, 0, 8, 2556, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4096, 135, 4093, 4096, 0, 4095, 0, 0, 4088, 814, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 4096, 0, 2, 0, 4088, 0, 4079, 4096, 0, 0, 3190, 0, 3081, 5, 0, 0, 0, 0, 2, 0, 4084, 0, 0, 4051, 0, 0, 0, 0, 1, 4093, 4094, 4, 0, 1, 0, 1, 1194, 0, 56, 0, 4093, 0, 4095, 1322, 0, 4096, 0, 0, 0, 0, 4088, 0, 4096, 18, 4096, 0, 4061, 0, 0, 4096, 0, 19, 0, 0, 33, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 4096, 1, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2, 4091, 0, 6, 5, 4034, 0, 4088, 0, 0, 4089, 0, 0, 0, 0, 383, 4094, 0, 0, 7, 0, 3967, 0, 0, 0, 0, 4096, 579, 4079, 4096, 0, 4088, 0, 43, 4063, 3740, 0, 1, 0, 0, 0, 0, 1212, 1619, 0, 0, 1, 4096, 0, 0, 21, 815, 211, 3736, 3759, 0, 0, 756, 0, 1573, 3681, 0, 0, 0, 0, 38, 2, 1320, 0, 0, 4057, 9, 0, 0, 0, 0, 4096, 4096, 0, 0, 3, 0, 2, 74, 0, 5, 6, 4079, 0, 4093, 3499, 2, 4096, 3, 0, 8, 0, 4096, 0, 4094, 125, 4096, 0, 4077, 0, 0, 4096, 0, 0, 0, 0, 252, 2, 0, 0, 0, 0, 4096, 3, 0, 0, 4096, 0, 1438, 82, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[7, 0, 0, 4062, 2, 0, 37, 4090, 0, 4096, 49, 0, 4089, 4, 2, 59, 0, 0, 4011, 0, 0, 0, 22, 4096, 5, 0, 0, 3, 4082, 1, 4081, 4090, 0, 4086, 26, 0, 4032, 351, 8, 0, 1, 0, 0, 0, 4089, 438, 2, 0, 0, 3810, 2, 13, 0, 3741, 0, 4083, 4065, 0, 0, 9, 0, 3983, 2862, 26, 23, 17, 0, 0, 53, 3434, 0, 6, 4090, 0, 110, 11, 0, 5, 4023, 4055, 0, 0, 22, 74, 0, 131, 0, 0, 0, 4071, 0, 4096, 113, 0, 4087, 44, 0, 4, 7, 4096, 0, 4088, 0, 4074, 0, 4048, 9, 0, 4087, 0, 142, 0, 3, 0, 1, 8, 0, 0, 0, 3957, 4, 9, 0, 4066, 0, 1479, 0, 10] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4073, 0, 2, 0, 4093, 0, 4077, 0, 0, 3840, 2, 0, 14, 5, 0, 3888, 0, 0, 0, 3, 4094, 0, 0, 0, 0, 4096, 10, 4093, 4092, 4, 4090, 0, 0, 4093, 751, 0, 0, 0, 0, 3, 0, 369, 81, 1, 20, 0, 4082, 0, 0, 0, 3930, 0, 4061, 4093, 0, 9, 0, 0, 3901, 3896, 0, 0, 0, 1, 5, 0, 481, 0, 0, 4018, 0, 6, 3, 0, 0, 3780, 4093, 2, 0, 4, 0, 0, 23, 0, 19, 25, 3933, 3, 4096, 3454, 473, 4089, 1, 0, 0, 0, 4096, 3, 4090, 52, 4074, 0, 4078, 0, 5, 4096, 0, 24, 0, 0, 274, 224, 0, 0, 0, 35, 4093, 0, 2, 0, 4096, 1, 3554, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1, 12, 0, 4054, 38, 1, 0, 4090, 21, 4080, 1, 0, 4070, 0, 12, 0, 67, 96, 3628, 0, 0, 0, 0, 3791, 28, 9, 0, 1, 4076, 0, 4050, 4083, 0, 4087, 15, 0, 3983, 1666, 18, 34, 0, 68, 0, 0, 3897, 2758, 0, 0, 44, 4070, 0, 6, 48, 3316, 3, 3933, 4078, 70, 19, 3125, 8, 3126, 17, 3, 20, 20, 1, 0, 0, 934, 1, 0, 3513, 0, 0, 7, 0, 0, 2761, 4082, 0, 2, 6, 0, 2, 2, 0, 299, 0, 4078, 0, 4087, 3792, 2, 4043, 0, 0, 4, 0, 3668, 0, 4095, 1, 4088, 0, 4057, 6, 0, 4096, 11, 1, 0, 13, 2, 553, 0, 0, 12, 0, 4096, 0, 0, 0, 4018, 0, 0, 98, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 2, 0, 4088, 2, 0, 10, 4077, 5, 4010, 0, 0, 4094, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 307, 0, 0, 1, 8, 4089, 0, 0, 0, 16, 4093, 0, 4032, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4082, 4012, 0, 0, 1, 19, 6, 103, 0, 3938, 0, 0, 1853, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3259, 13, 1294, 3825, 3, 0, 3803, 2, 4077, 4081, 1270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4, 3886, 0, 3, 4094, 4, 0, 0, 21, 1, 350, 4090, 1572, 1261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 161, 3, 419, 5, 1, 9, 3876, 24, 4084, 0, 0, 4082, 0, 1987, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 42, 0, 4058, 0, 188, 0, 4087, 1, 4089, 13, 0, 3194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 129, 57, 3730, 110, 0, 0, 4004, 7, 393, 157, 20, 4091, 2609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16301, 13209, 18918, 21629, 15292, 14738, 15676, 15309] avg=16384.0000 max=21629 max_rank=3 min=13209 min_rank=1 imbalance_ratio=1.320129 max_load_reduction=109443 max_load_reduction_pct=0.834984 moved_to_echo=61015 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4011, 3, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4096, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 4096, 5, 4096, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 3820, 3, 0, 0, 0, 0, 15, 249, 3883, 632, 0, 0, 1858, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 81, 1295, 4086, 0, 0, 3, 0, 4014, 3204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 19, 1039, 0, 1, 3161, 0, 0, 0, 5, 0, 4020, 4096, 1576, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 209, 0, 5, 0, 4096, 0, 4096, 0, 0, 4096, 62, 1988, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 190, 0, 4084, 0, 4095, 0, 2, 4096, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1415, 0, 0, 0, 0, 2482, 723, 0, 0, 4091, 0, 3362, 2, 0, 4084, 2633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16303, 17080, 15337, 16771, 15586, 14642, 16561, 18792] avg=16384.0000 max=18792 max_rank=7 min=14642 min_rank=5 imbalance_ratio=1.146973 max_load_reduction=112280 max_load_reduction_pct=0.856628 moved_to_echo=65581 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4091, 0, 3, 0, 4095, 0, 4011, 0, 0, 4091, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2556, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4096, 135, 4093, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4088, 814, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 1858, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4088, 0, 1290, 4096, 0, 0, 3190, 0, 3081, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4084, 0, 0, 4051, 0, 0, 0, 0, 1, 4093, 4094, 1576, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 1, 1194, 0, 56, 0, 4093, 0, 4095, 1322, 0, 4096, 0, 1979, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 190, 18, 4096, 0, 4061, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 4096, 1, 0, 1, 0, 4096, 2633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16295, 15653, 15768, 15752, 19174, 16926, 16568, 14936] avg=16384.0000 max=19174 max_rank=4 min=14936 min_rank=7 imbalance_ratio=1.170288 max_load_reduction=111898 max_load_reduction_pct=0.853714 moved_to_echo=67604 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 4091, 0, 6, 5, 4034, 0, 4004, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 4094, 0, 0, 7, 0, 3967, 0, 0, 0, 0, 4096, 579, 4079, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 43, 4063, 3740, 0, 1, 0, 0, 0, 0, 1212, 1619, 0, 0, 1858, 741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 21, 815, 211, 1182, 3759, 0, 0, 756, 0, 1573, 3681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 2, 1320, 0, 0, 4057, 9, 0, 0, 0, 0, 4096, 4096, 1576, 1272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 74, 0, 5, 6, 4079, 0, 4093, 3499, 2, 4096, 3, 1813, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 190, 125, 4096, 0, 4077, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 2, 0, 0, 0, 0, 4096, 3, 0, 0, 4096, 0, 1438, 82, 0, 4096, 2632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16231, 17867, 17365, 11999, 16466, 17759, 16688, 16697] avg=16384.0000 max=17867 max_rank=1 min=11999 min_rank=3 imbalance_ratio=1.090515 max_load_reduction=113205 max_load_reduction_pct=0.863686 moved_to_echo=67610 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 4078, 0, 0, 0, 4072, 8, 3946, 220, 0, 4095, 1, 0, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8, 4095, 0, 0, 0, 0, 4096, 1, 0, 11, 0, 4084, 0, 4089, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4085, 6, 10, 4084, 2976, 49, 4, 0, 26, 19, 1, 2455, 351, 9, 0, 1862, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 6, 3, 4035, 0, 1272, 4094, 0, 0, 264, 8, 3683, 3316, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3789, 0, 0, 3981, 0, 49, 0, 4, 0, 1727, 4057, 1582, 1274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 27, 0, 0, 285, 4091, 8, 4082, 1174, 0, 4051, 0, 1966, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4094, 5, 180, 0, 4079, 0, 4096, 0, 0, 3611, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 36, 4095, 0, 0, 0, 3932, 10, 171, 0, 0, 3999, 2630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17217, 17048, 16750, 16709, 16466, 15820, 16183, 14879] avg=16384.0000 max=17217 max_rank=0 min=14879 min_rank=7 imbalance_ratio=1.050842 max_load_reduction=113855 max_load_reduction_pct=0.868645 moved_to_echo=63348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4073, 0, 2, 0, 4093, 0, 3993, 0, 0, 3840, 2, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3888, 0, 0, 0, 3, 4094, 0, 0, 0, 0, 4096, 10, 4093, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4090, 0, 0, 4093, 751, 0, 0, 0, 0, 3, 0, 369, 81, 1, 20, 1857, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 0, 1284, 4093, 0, 9, 0, 0, 3901, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 481, 0, 0, 4018, 0, 6, 3, 0, 0, 3780, 4093, 1575, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 23, 0, 19, 25, 3933, 3, 4096, 3454, 473, 4089, 1, 1971, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 190, 52, 4074, 0, 4078, 0, 5, 4096, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 224, 0, 0, 0, 35, 4093, 0, 2, 0, 4096, 1, 3554, 0, 0, 4082, 2629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16017, 16849, 12075, 17113, 15233, 18177, 16618, 18990] avg=16384.0000 max=18990 max_rank=7 min=12075 min_rank=2 imbalance_ratio=1.159058 max_load_reduction=112082 max_load_reduction_pct=0.855118 moved_to_echo=69018 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 4062, 2, 0, 37, 4090, 0, 4011, 49, 0, 4089, 4, 2, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 0, 22, 4096, 5, 0, 0, 3, 4082, 1, 4081, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 26, 0, 4032, 351, 8, 0, 1, 0, 0, 0, 4089, 438, 2, 0, 1856, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 13, 0, 3741, 0, 1292, 4065, 0, 0, 9, 0, 3983, 2862, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 17, 0, 0, 53, 3434, 0, 6, 4090, 0, 110, 11, 0, 5, 4023, 4055, 1574, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 74, 0, 131, 0, 0, 0, 4071, 0, 4096, 113, 0, 4087, 44, 1981, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 4096, 0, 189, 0, 4074, 0, 4048, 9, 0, 4087, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 8, 0, 0, 0, 3957, 4, 9, 0, 4066, 0, 1479, 0, 10, 3810, 2628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 16961, 15699, 15993, 18672, 14704, 16656, 15975] avg=16384.0000 max=18672 max_rank=4 min=14704 min_rank=5 imbalance_ratio=1.139648 max_load_reduction=112400 max_load_reduction_pct=0.857544 moved_to_echo=66007 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 12, 0, 4054, 38, 1, 0, 4090, 21, 3996, 1, 0, 4070, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 96, 3628, 0, 0, 0, 0, 3791, 28, 9, 0, 1, 4076, 0, 4050, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 15, 0, 3983, 1666, 18, 34, 0, 68, 0, 0, 3897, 2758, 0, 0, 1853, 781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 6, 48, 3316, 3, 1243, 4078, 70, 19, 3125, 8, 3126, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 20, 1, 0, 0, 934, 1, 0, 3513, 0, 0, 7, 0, 0, 2761, 4082, 1571, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 2, 2, 0, 299, 0, 4078, 0, 4087, 3792, 2, 4043, 0, 1909, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3668, 0, 190, 1, 4088, 0, 4057, 6, 0, 4096, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2, 553, 0, 0, 12, 0, 4096, 0, 0, 0, 4018, 0, 0, 98, 0, 4070, 2632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16296, 16405, 19160, 15106, 14183, 18306, 16122, 15494] avg=16384.0000 max=19160 max_rank=2 min=14183 min_rank=4 imbalance_ratio=1.169434 max_load_reduction=111912 max_load_reduction_pct=0.853821 moved_to_echo=64105 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 640, 26752, 0, 0, 128, 128, 32384, 128, 128, 128, 128, 32768, 768, 32640, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 32640, 128, 128, 128, 32768, 128, 32000, 384, 0, 32512, 128, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 128, 27392, 384, 10240, 32128, 128, 128, 11264, 128, 27520, 21120, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 128, 32256, 14336, 128, 128, 128, 128, 128, 384, 20096, 9856, 128, 128, 14976, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 32384, 128, 1536, 256, 32768, 128, 32640, 128, 128, 31488, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 384, 128, 2176, 128, 512, 384, 32384, 128, 32768, 13440, 512, 32640, 128, 15616, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 512, 128, 19072, 128, 128, 30976, 128, 256, 128, 128, 128, 24960, 32768, 12672, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 640, 2304, 128, 128, 256, 128, 30720, 896, 128, 0, 32512, 128, 10496, 384, 128, 32384, 21120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4088, 0, 5, 1859, 0, 0, 0, 805, 0, 0, 0, 1, 0, 0, 0, 356, 0, 4090, 307, 1495, 0, 0, 0, 14, 1, 4089, 0, 0, 4096, 2, 0, 4091, 829, 61, 12, 4083, 0, 6, 1, 0, 5, 0, 11, 4064, 0, 0, 4092, 14, 4095, 14, 32, 3071, 0, 4091, 4084, 0, 4091, 4089, 4073, 2, 4094, 3849, 0, 2, 4094, 0, 0, 0, 5, 22, 2, 0, 87, 0, 0, 0, 861, 0, 4, 0, 0, 0, 1, 2644, 19, 0, 0, 0, 4037, 4, 3272, 4087, 5, 0, 4090, 0, 4081, 4059, 0, 3730, 0, 3, 70, 0, 4085, 0, 4094, 0, 30, 2656, 0, 0, 1, 1, 2, 0, 0, 4089, 2, 4082, 0, 0, 2, 2749, 0, 28, 2, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 187, 0, 4096, 0, 0, 0, 1291, 0, 0, 0, 0, 0, 0, 0, 29, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4017, 0, 113, 4026, 0, 0, 4096, 231, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 4096, 0, 4096, 0, 1, 4095, 0, 3562, 4096, 0, 3990, 4096, 4096, 0, 4094, 3996, 0, 0, 4096, 0, 21, 0, 0, 0, 0, 0, 2094, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 4096, 0, 2706, 4096, 0, 0, 4096, 0, 5, 4094, 0, 3967, 0, 0, 41, 0, 4096, 90, 4096, 0, 192, 0, 0, 22, 0, 0, 1329, 0, 0, 4096, 0, 4096, 0, 0, 1, 882, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4051, 0, 0, 4095, 0, 0, 0, 4096, 9, 0, 3898, 0, 0, 2, 42, 0, 0, 4066, 2, 0, 0, 0, 0, 968, 4, 4086, 17, 0, 4096, 0, 3, 4091, 38, 80, 3, 4090, 0, 1, 0, 10, 0, 265, 0, 4041, 0, 11, 4071, 0, 4013, 24, 39, 4076, 3, 0, 3558, 0, 4092, 4088, 4061, 0, 4096, 3163, 0, 6, 4071, 2, 0, 1, 5, 0, 0, 0, 4075, 0, 0, 0, 522, 25, 0, 5, 0, 0, 6, 4050, 81, 2, 13, 4, 4061, 5, 1, 4088, 0, 0, 4047, 0, 1199, 4096, 31, 3984, 21, 49, 15, 0, 3974, 0, 4084, 0, 0, 0, 0, 0, 1, 0, 2293, 0, 0, 4096, 1, 4093, 13, 0, 0, 703, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 64, 0, 4084, 0, 0, 0, 2476, 3, 0, 975, 0, 2, 3, 0, 0, 2, 4096, 14, 485, 0, 0, 0, 121, 0, 3997, 0, 0, 4023, 0, 0, 4096, 169, 0, 0, 4096, 7, 0, 1, 0, 0, 4033, 4, 4077, 0, 9, 4096, 0, 4093, 0, 2, 3408, 8, 1744, 3828, 0, 3855, 4096, 4096, 0, 4096, 4038, 0, 0, 4091, 1, 2, 0, 0, 0, 0, 1, 4057, 1, 0, 0, 3524, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 3407, 0, 24, 4094, 0, 0, 4086, 1, 0, 4090, 0, 3448, 5, 0, 8, 0, 4091, 0, 4087, 0, 0, 13, 0, 0, 0, 0, 1187, 0, 0, 4094, 0, 4096, 0, 0, 272, 0, 0, 1, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 1, 4089, 2, 0, 0, 4091, 0, 0, 200, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 1, 2, 4090, 7, 0, 4094, 0, 0, 4096, 0, 15, 0, 4074, 3, 1, 0, 23, 0, 395, 9, 4090, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 1, 3896, 0, 4088, 4096, 4096, 0, 4094, 3178, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3, 3980, 5, 0, 0, 3865, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4073, 0, 1709, 4094, 0, 0, 4084, 0, 295, 4084, 0, 4091, 0, 0, 100, 0, 4092, 1, 4095, 0, 2, 0, 0, 0, 0, 0, 2862, 0, 0, 4090, 4, 4091, 0, 0, 13, 30, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4090, 0, 0, 4078, 0, 10, 0, 4057, 0, 0, 37, 8, 0, 50, 15, 24, 0, 4050, 7, 0, 0, 0, 0, 10, 0, 4083, 0, 0, 4057, 0, 0, 4079, 0, 0, 0, 4059, 0, 6, 10, 0, 13, 3663, 0, 4057, 0, 0, 4094, 87, 4084, 0, 87, 4077, 0, 31, 3364, 23, 4071, 4076, 4046, 5, 4089, 417, 1, 0, 4081, 16, 40, 0, 0, 40, 7, 0, 4063, 0, 6, 14, 4079, 0, 1, 1, 12, 5, 0, 4010, 16, 0, 22, 0, 4005, 17, 210, 4083, 0, 0, 4086, 1, 0, 4046, 21, 4022, 0, 38, 0, 0, 4093, 0, 4090, 10, 0, 0, 13, 34, 4, 0, 3811, 0, 63, 4096, 0, 4087, 0, 0, 0, 762, 0, 50, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4093, 12, 0, 4096, 0, 0, 0, 3746, 0, 8, 363, 0, 0, 9, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 27, 0, 4084, 0, 123, 3367, 0, 0, 4096, 1727, 0, 0, 4093, 0, 0, 3, 0, 0, 178, 27, 4067, 3, 0, 4089, 0, 4092, 0, 0, 4096, 0, 1890, 4074, 1, 3849, 4096, 4049, 9, 4096, 3716, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 4094, 0, 6, 0, 1741, 0, 0, 0, 0, 0, 0, 4096, 643, 0, 0, 0, 4096, 0, 0, 4093, 0, 1, 4096, 0, 0, 4096, 5, 4096, 0, 0, 0, 0, 4085, 42, 4090, 0, 0, 602, 0, 21, 3, 0, 2608, 0, 1, 4079, 0, 4091, 6, 4, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4095, 0, 0, 3809, 0, 1, 0, 3180, 52, 46, 1735, 13, 0, 97, 26, 0, 0, 4037, 15, 0, 24, 0, 0, 197, 0, 4036, 55, 0, 3368, 0, 0, 4084, 0, 0, 5, 4013, 0, 11, 0, 75, 12, 1922, 1, 3914, 0, 0, 4080, 1, 4037, 16, 36, 4011, 60, 1439, 4047, 24, 2376, 4050, 3963, 0, 4083, 4087, 1, 15, 4079, 23, 0, 1, 6, 68, 17, 0, 1284, 0, 19, 3, 4034, 0, 0, 0, 0, 0, 0, 4083, 566, 6, 1, 0, 4077, 0, 87, 4076, 0, 1, 4018, 0, 0, 4093, 0, 4088, 0, 0, 0, 0, 4023, 63, 4053, 1, 0, 129, 8, 0, 49, 16, 3716, 0, 0, 4045, 0, 4070, 0, 2, 0, 1097, 0, 21, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4088, 0, 5, 1859, 0, 0, 0, 805, 0, 0, 0, 1, 0, 0, 0, 356, 3852, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 307, 1495, 0, 0, 0, 14, 1, 4089, 0, 0, 4096, 2, 0, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 829, 61, 12, 4083, 0, 6, 1, 0, 5, 0, 11, 4064, 0, 0, 4092, 14, 1021, 835, 239, 221, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 32, 3071, 0, 4091, 3975, 0, 4091, 0, 0, 2, 0, 3849, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 5, 22, 2, 0, 87, 0, 0, 0, 861, 0, 4, 0, 4094, 2352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2644, 19, 0, 0, 0, 4037, 4, 3272, 3252, 5, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4059, 0, 3730, 0, 3, 70, 0, 4085, 0, 3073, 0, 30, 2656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 0, 0, 4089, 2, 4082, 0, 0, 2, 2749, 0, 28, 2, 8, 3414, 1737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11647, 17946, 15603, 19127, 11521, 17324, 21787, 16117] avg=16384.0000 max=21787 max_rank=6 min=11521 min_rank=4 imbalance_ratio=1.329773 max_load_reduction=109285 max_load_reduction_pct=0.833778 moved_to_echo=66749 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 64, 0, 4084, 0, 0, 0, 2476, 3, 0, 975, 0, 2, 3, 0, 0, 3874, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 14, 485, 0, 0, 0, 121, 0, 3997, 0, 0, 4023, 0, 0, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 4096, 7, 0, 1, 0, 0, 4033, 4, 4077, 0, 9, 4096, 0, 1019, 836, 240, 222, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3408, 8, 1744, 3726, 0, 3855, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 2, 0, 0, 0, 0, 1, 4057, 1, 0, 0, 3524, 0, 0, 0, 4096, 2356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 3407, 0, 24, 3258, 0, 0, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3448, 5, 0, 8, 0, 4091, 0, 3068, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1187, 0, 0, 4094, 0, 4096, 0, 0, 272, 0, 0, 1, 0, 1, 3413, 1740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16257, 16594, 18911, 16781, 18129, 14873, 14723, 14804] avg=16384.0000 max=18911 max_rank=2 min=14723 min_rank=6 imbalance_ratio=1.154236 max_load_reduction=112161 max_load_reduction_pct=0.855721 moved_to_echo=62529 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 1, 4089, 2, 0, 0, 4091, 0, 0, 200, 0, 0, 2, 0, 0, 3873, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 1, 2, 4090, 7, 0, 4094, 0, 0, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4074, 3, 1, 0, 23, 0, 395, 9, 4090, 0, 0, 4096, 0, 1021, 836, 240, 223, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 3792, 0, 4088, 0, 0, 0, 0, 3178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3, 3980, 5, 0, 0, 3865, 0, 0, 0, 4094, 2355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4073, 0, 1709, 3258, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 4084, 0, 4091, 0, 0, 100, 0, 4092, 1, 3074, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2862, 0, 0, 4090, 4, 4091, 0, 0, 13, 30, 0, 1, 0, 0, 3416, 1741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17034, 16151, 15130, 15155, 18395, 17220, 15739, 16248] avg=16384.0000 max=18395 max_rank=4 min=15130 min_rank=2 imbalance_ratio=1.122742 max_load_reduction=112677 max_load_reduction_pct=0.859657 moved_to_echo=67602 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 187, 0, 4096, 0, 0, 0, 1291, 0, 0, 0, 0, 0, 0, 0, 29, 3874, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4017, 0, 113, 4026, 0, 0, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 4096, 0, 1021, 837, 240, 222, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 3562, 3987, 0, 3990, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 0, 0, 0, 2094, 0, 0, 0, 4093, 0, 0, 0, 4094, 2356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 4096, 0, 2706, 3259, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4094, 0, 3967, 0, 0, 41, 0, 4096, 90, 3075, 0, 192, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1329, 0, 0, 4096, 0, 4096, 0, 0, 1, 882, 0, 0, 0, 0, 3415, 1740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14254, 16109, 14951, 19631, 16754, 18232, 15582, 15559] avg=16384.0000 max=19631 max_rank=3 min=14254 min_rank=0 imbalance_ratio=1.198181 max_load_reduction=111441 max_load_reduction_pct=0.850227 moved_to_echo=66127 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4093, 12, 0, 4096, 0, 0, 0, 3746, 0, 8, 363, 0, 0, 9, 0, 0, 3829, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 27, 0, 4084, 0, 123, 3367, 0, 0, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1727, 0, 0, 4093, 0, 0, 3, 0, 0, 178, 27, 4067, 3, 0, 4089, 0, 1020, 836, 240, 220, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1890, 3965, 1, 3849, 0, 0, 9, 0, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 4094, 0, 6, 0, 1741, 0, 0, 0, 4096, 2355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 643, 0, 0, 0, 4096, 0, 0, 3257, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 4096, 0, 0, 0, 0, 4085, 42, 3070, 0, 0, 602, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2608, 0, 1, 4079, 0, 4091, 6, 4, 0, 0, 0, 0, 0, 0, 3413, 1741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16835, 15556, 16612, 17526, 16391, 16189, 16017, 15946] avg=16384.0000 max=17526 max_rank=3 min=15556 min_rank=1 imbalance_ratio=1.069702 max_load_reduction=113546 max_load_reduction_pct=0.866287 moved_to_echo=64543 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4051, 0, 0, 4095, 0, 0, 0, 4096, 9, 0, 3898, 0, 0, 2, 42, 0, 3841, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 2, 0, 0, 0, 0, 968, 4, 4086, 17, 0, 4096, 0, 3, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 80, 3, 4090, 0, 1, 0, 10, 0, 265, 0, 4041, 0, 11, 4071, 0, 1023, 839, 241, 220, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 39, 4076, 3, 0, 3459, 0, 4092, 0, 0, 0, 0, 3163, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 2, 0, 1, 5, 0, 0, 0, 4075, 0, 0, 0, 522, 25, 0, 5, 4096, 2352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4050, 81, 2, 13, 4, 4061, 5, 1, 3249, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1199, 4096, 31, 3984, 21, 49, 15, 0, 3974, 0, 3061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2293, 0, 0, 4096, 1, 4093, 13, 0, 0, 703, 0, 0, 0, 0, 3346, 1736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20701, 17092, 15032, 14862, 15154, 15519, 16430, 16282] avg=16384.0000 max=20701 max_rank=0 min=14862 min_rank=3 imbalance_ratio=1.263489 max_load_reduction=110371 max_load_reduction_pct=0.842064 moved_to_echo=63385 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 4078, 0, 10, 0, 4057, 0, 0, 37, 8, 0, 50, 15, 24, 3826, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 7, 0, 0, 0, 0, 10, 0, 4083, 0, 0, 4057, 0, 0, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 6, 10, 0, 13, 3663, 0, 4057, 0, 0, 4094, 87, 1020, 834, 239, 220, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 4077, 0, 31, 3274, 23, 4071, 0, 0, 5, 0, 417, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 16, 40, 0, 0, 40, 7, 0, 4063, 0, 6, 14, 4079, 0, 1, 1, 4089, 2345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 5, 0, 4010, 16, 0, 22, 0, 4005, 17, 210, 3249, 0, 0, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 21, 4022, 0, 38, 0, 0, 4093, 0, 3070, 10, 0, 0, 13, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3811, 0, 63, 4096, 0, 4087, 0, 0, 0, 762, 0, 50, 0, 2, 3406, 1731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16873, 16047, 18392, 11986, 18782, 15633, 15347, 18012] avg=16384.0000 max=18782 max_rank=4 min=11986 min_rank=3 imbalance_ratio=1.146362 max_load_reduction=112290 max_load_reduction_pct=0.856705 moved_to_echo=67774 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 3809, 0, 1, 0, 3180, 52, 46, 1735, 13, 0, 97, 26, 0, 3747, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 15, 0, 24, 0, 0, 197, 0, 4036, 55, 0, 3368, 0, 0, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4013, 0, 11, 0, 75, 12, 1922, 1, 3914, 0, 0, 4080, 1, 1011, 833, 239, 216, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 36, 4011, 60, 1439, 3939, 24, 2376, 0, 0, 0, 0, 4087, 1, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 23, 0, 1, 6, 68, 17, 0, 1284, 0, 19, 3, 4034, 0, 0, 0, 4083, 2329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 566, 6, 1, 0, 4077, 0, 87, 3243, 0, 1, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4088, 0, 0, 0, 0, 4023, 63, 3042, 1, 0, 129, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 16, 3716, 0, 0, 4045, 0, 4070, 0, 2, 0, 1097, 0, 21, 0, 0, 3367, 1721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17471, 15577, 16441, 16004, 15946, 16082, 15447, 18104] avg=16384.0000 max=18104 max_rank=7 min=15447 min_rank=6 imbalance_ratio=1.104980 max_load_reduction=112968 max_load_reduction_pct=0.861877 moved_to_echo=65579 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 384, 128, 30208, 128, 128, 0, 23808, 128, 128, 7296, 128, 128, 256, 128, 512, 30720, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 31232, 1408, 128, 128, 128, 31872, 128, 8064, 26112, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32640, 384, 2048, 128, 128, 0, 1408, 128, 32512, 128, 256, 31232, 128, 128, 30848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 17920, 0, 128, 32768, 128, 32768, 128, 128, 384, 6272, 0, 128, 128, 128, 27264, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 256, 30976, 128, 12800, 30208, 128, 30464, 0, 0, 128, 0, 26496, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[5632, 32768, 128, 31488, 128, 128, 256, 0, 32640, 256, 24576, 128, 256, 3456, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 128, 128, 128, 128, 256, 128, 128, 23808, 128, 128, 128, 22784, 128, 128, 128, 32768, 18816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[3072, 256, 128, 32640, 128, 128, 128, 128, 128, 10496, 128, 32512, 128, 128, 32768, 128, 8192, 6784, 1920, 1792, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[2, 0, 1, 4082, 0, 4046, 2436, 0, 4092, 0, 4092, 0, 4, 2, 0, 11, 0, 0, 4052, 0, 3, 14, 0, 1, 4091, 0, 3676, 0, 0, 1, 0, 1, 0, 4047, 4038, 0, 1, 0, 4082, 3824, 4094, 0, 1363, 3, 10, 0, 0, 23, 631, 3448, 4094, 4093, 0, 0, 0, 0, 0, 5, 0, 0, 0, 470, 1, 4071, 37, 0, 0, 4093, 4094, 4083, 4093, 16, 2, 0, 0, 281, 3969, 0, 0, 342, 4092, 0, 4, 4094, 7, 0, 0, 0, 0, 0, 4038, 0, 4087, 0, 12, 8, 4094, 0, 108, 2, 0, 4094, 3, 0, 2, 1, 6, 3995, 4093, 10, 4088, 4091, 56, 4, 0, 4, 0, 61, 0, 9, 0, 4, 2, 0, 3, 0, 0, 85] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 15, 0, 4087, 0, 836, 748, 0, 4082, 0, 4095, 339, 0, 0, 1, 1701, 0, 233, 4096, 0, 0, 0, 0, 0, 4095, 0, 3061, 0, 0, 0, 3581, 1164, 0, 4096, 4079, 0, 0, 0, 3492, 4095, 4096, 0, 1517, 0, 0, 0, 0, 2, 0, 4096, 4047, 4096, 0, 0, 0, 7, 0, 0, 0, 2, 0, 0, 0, 4081, 1, 0, 0, 4096, 4094, 4093, 4084, 0, 0, 0, 69, 14, 4092, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 1, 13, 0, 1, 3821, 0, 4096, 16, 0, 0, 4085, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4061, 4096, 4, 0, 4, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[3, 0, 0, 4090, 0, 4096, 65, 0, 4093, 0, 4095, 0, 0, 0, 0, 943, 0, 0, 4094, 0, 0, 25, 0, 3, 4084, 0, 3506, 0, 9, 4, 0, 448, 0, 4096, 4096, 0, 0, 0, 4096, 4089, 4096, 0, 3789, 0, 0, 2, 0, 1, 0, 3547, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 4096, 4081, 4096, 4096, 167, 1, 0, 0, 385, 4083, 0, 2, 367, 4096, 0, 11, 3853, 0, 0, 0, 0, 0, 0, 3032, 0, 4091, 35, 0, 0, 4078, 1, 190, 0, 0, 4090, 0, 3, 0, 0, 66, 4034, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 7, 17, 0, 0, 0, 10, 137, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4076, 0, 1854, 1143, 0, 4068, 0, 4096, 1014, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 26, 0, 259, 4096, 0, 3237, 5, 3, 0, 2411, 699, 0, 4086, 4084, 5, 0, 0, 2928, 4096, 4094, 18, 4094, 0, 0, 2, 0, 291, 0, 4096, 3982, 4092, 12, 0, 4, 0, 0, 0, 13, 8, 4, 0, 0, 4080, 0, 0, 4, 4069, 4089, 4065, 4096, 0, 0, 48, 0, 0, 4096, 14, 31, 0, 4093, 4, 0, 3982, 0, 1, 0, 0, 3, 54, 2736, 0, 4083, 0, 0, 4, 4043, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4069, 4088, 0, 4096, 4095, 0, 0, 33, 0, 3, 0, 0, 0, 0, 11, 1, 1, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[5, 1, 0, 4091, 7, 1731, 18, 0, 4070, 71, 4073, 2981, 1, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 108, 4096, 0, 13, 0, 0, 24, 1212, 2581, 4, 4096, 4040, 0, 0, 14, 3684, 4090, 4096, 14, 1709, 0, 0, 2, 0, 157, 0, 4096, 3954, 4083, 1, 31, 0, 12, 246, 0, 79, 0, 0, 59, 0, 4083, 1, 15, 0, 4079, 4073, 3943, 4093, 0, 0, 0, 0, 0, 4096, 3933, 156, 0, 4096, 0, 0, 3991, 4, 11, 0, 127, 0, 0, 1829, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 4086, 0, 0, 1, 0, 28, 3977, 4083, 4, 4077, 4079, 4, 1, 95, 0, 235, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1, 0, 0, 4085, 0, 4051, 1132, 0, 4076, 1, 4060, 0, 9, 26, 0, 527, 0, 0, 4038, 1, 9, 0, 0, 0, 4051, 146, 1849, 43, 0, 0, 32, 0, 0, 4071, 4077, 0, 0, 11, 4083, 4089, 4070, 0, 3940, 7, 0, 0, 0, 1925, 27, 4015, 4096, 4096, 47, 0, 0, 0, 0, 33, 0, 1, 0, 2, 6, 4083, 9, 0, 0, 4095, 4095, 4096, 4090, 0, 0, 0, 0, 16, 4042, 3039, 48, 2, 4075, 0, 0, 3994, 0, 0, 0, 29, 0, 25, 1245, 0, 4074, 0, 20, 31, 3904, 1, 8, 12, 0, 4082, 0, 0, 0, 0, 124, 2668, 4094, 0, 4091, 4078, 0, 0, 0, 0, 0, 3, 0, 19, 1, 0, 16, 8, 17, 0, 20, 85] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 2074, 2491, 0, 4096, 96, 4096, 607, 0, 0, 0, 21, 0, 0, 3858, 0, 0, 759, 0, 2, 3987, 0, 3664, 0, 0, 0, 99, 2036, 0, 4034, 4094, 0, 0, 0, 4086, 4095, 4051, 0, 2262, 0, 0, 0, 0, 58, 88, 3479, 4095, 4096, 0, 0, 0, 0, 0, 80, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 4035, 4096, 0, 0, 0, 0, 0, 4063, 0, 8, 0, 4096, 0, 0, 3945, 0, 0, 0, 0, 0, 0, 2924, 0, 4096, 1, 0, 0, 4095, 0, 111, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 289, 0, 0, 0, 39] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[11, 0, 0, 4037, 16, 2, 0, 0, 4069, 0, 4090, 0, 22, 0, 0, 38, 2, 0, 3986, 8, 23, 0, 2, 87, 4052, 0, 4019, 7, 0, 0, 4089, 5, 12, 4080, 4079, 0, 0, 0, 3975, 4084, 4095, 1, 331, 38, 0, 0, 0, 3768, 0, 4062, 4050, 4078, 0, 69, 31, 0, 100, 5, 0, 10, 1, 0, 0, 4086, 0, 0, 1, 4056, 4089, 4035, 4081, 32, 4, 160, 91, 0, 4075, 0, 0, 3, 4059, 14, 10, 3713, 2, 14, 47, 0, 1, 795, 4063, 1, 4008, 0, 0, 0, 4016, 8, 0, 26, 0, 4082, 10, 0, 6, 2, 2, 3296, 4040, 4, 4082, 4074, 1, 0, 0, 1, 431, 0, 3, 2, 0, 0, 28, 3, 0, 31, 49, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 1, 4082, 0, 4046, 2436, 0, 4092, 0, 3827, 0, 4, 2, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 3, 14, 0, 1, 4091, 0, 3676, 0, 0, 1, 0, 1, 2692, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1270, 4038, 0, 1, 0, 4082, 3824, 0, 0, 1363, 3, 10, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 631, 3448, 4094, 4093, 0, 0, 0, 0, 0, 5, 0, 0, 0, 470, 1, 4071, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 4093, 3021, 4083, 0, 16, 2, 0, 0, 281, 3969, 0, 0, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4, 4094, 7, 0, 0, 0, 0, 0, 4038, 0, 4087, 0, 12, 8, 911, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 108, 2, 0, 0, 3, 0, 2, 1, 6, 3995, 4093, 10, 4088, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 4, 0, 4, 0, 61, 0, 9, 0, 4, 2, 0, 3, 0, 0, 85, 4093, 4094, 4094, 3800, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18503, 14693, 14614, 16854, 15844, 17477, 16693, 16394] avg=16384.0000 max=18503 max_rank=0 min=14614 min_rank=2 imbalance_ratio=1.129333 max_load_reduction=112569 max_load_reduction_pct=0.858833 moved_to_echo=66408 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 4090, 0, 4096, 65, 0, 4093, 0, 3830, 0, 0, 0, 0, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 25, 0, 3, 4084, 0, 3506, 0, 9, 4, 0, 448, 2725, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1285, 4096, 0, 0, 0, 4096, 4089, 0, 0, 3789, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3547, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 3011, 4096, 0, 167, 1, 0, 0, 385, 4083, 0, 2, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 3853, 0, 0, 0, 0, 0, 0, 3032, 0, 4091, 35, 0, 0, 908, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 1, 190, 0, 0, 0, 0, 3, 0, 0, 66, 4034, 4096, 0, 4096, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 17, 0, 0, 0, 10, 137, 0, 0, 0, 0, 4096, 4090, 4096, 3805, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17120, 15060, 17358, 15874, 16211, 16250, 16855, 16344] avg=16384.0000 max=17358 max_rank=2 min=15060 min_rank=1 imbalance_ratio=1.059448 max_load_reduction=113714 max_load_reduction_pct=0.867569 moved_to_echo=65660 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4076, 0, 1854, 1143, 0, 4068, 0, 3831, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 26, 0, 259, 4096, 0, 3237, 5, 3, 0, 2411, 699, 2718, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1282, 4084, 5, 0, 0, 2928, 4096, 0, 18, 4094, 0, 0, 2, 0, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3982, 4092, 12, 0, 4, 0, 0, 0, 13, 8, 4, 0, 0, 4080, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4069, 3017, 4065, 0, 0, 0, 48, 0, 0, 4096, 14, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 0, 3982, 0, 1, 0, 0, 3, 54, 2736, 0, 4083, 0, 0, 4, 910, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4088, 0, 4096, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 3, 0, 0, 0, 0, 11, 1, 1, 0, 0, 0, 0, 4096, 4090, 4094, 3804, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15986, 17710, 16800, 16332, 15344, 16094, 16587, 16219] avg=16384.0000 max=17710 max_rank=1 min=15344 min_rank=4 imbalance_ratio=1.080933 max_load_reduction=113362 max_load_reduction_pct=0.864883 moved_to_echo=64244 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 0, 4087, 0, 836, 748, 0, 4082, 0, 3830, 339, 0, 0, 1, 1701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 4096, 0, 0, 0, 0, 0, 4095, 0, 3061, 0, 0, 0, 3581, 1164, 2725, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1285, 4079, 0, 0, 0, 3492, 4095, 0, 0, 1517, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4047, 4096, 0, 0, 0, 7, 0, 0, 0, 2, 0, 0, 0, 4081, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 3021, 4093, 0, 0, 0, 0, 69, 14, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 1, 13, 0, 1, 3821, 0, 4096, 16, 0, 0, 911, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4061, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 4096, 3805, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15639, 19117, 14470, 16370, 15386, 17278, 16629, 16183] avg=16384.0000 max=19117 max_rank=1 min=14470 min_rank=2 imbalance_ratio=1.166809 max_load_reduction=111955 max_load_reduction_pct=0.854149 moved_to_echo=65476 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4085, 0, 4051, 1132, 0, 4076, 1, 3798, 0, 9, 26, 0, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 1, 9, 0, 0, 0, 4051, 146, 1849, 43, 0, 0, 32, 0, 2708, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1278, 4077, 0, 0, 11, 4083, 4089, 0, 0, 3940, 7, 0, 0, 0, 1925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4015, 4096, 4096, 47, 0, 0, 0, 0, 33, 0, 1, 0, 2, 6, 4083, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4095, 3022, 4096, 0, 0, 0, 0, 0, 16, 4042, 3039, 48, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 3994, 0, 0, 0, 29, 0, 25, 1245, 0, 4074, 0, 20, 31, 911, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 1, 8, 12, 0, 0, 0, 0, 0, 0, 124, 2668, 4094, 0, 4091, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 19, 1, 0, 16, 8, 17, 0, 20, 85, 4090, 4082, 4070, 3788, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17706, 13039, 19410, 16446, 18369, 14626, 15192, 16284] avg=16384.0000 max=19410 max_rank=2 min=13039 min_rank=1 imbalance_ratio=1.184692 max_load_reduction=111662 max_load_reduction_pct=0.851913 moved_to_echo=64471 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 2074, 2491, 0, 4096, 96, 3831, 607, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3858, 0, 0, 759, 0, 2, 3987, 0, 3664, 0, 0, 0, 99, 2036, 2684, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1266, 4094, 0, 0, 0, 4086, 4095, 0, 0, 2262, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 3479, 4095, 4096, 0, 0, 0, 0, 0, 80, 1, 0, 0, 0, 0, 4096, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3022, 4035, 0, 0, 0, 0, 0, 0, 4063, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3945, 0, 0, 0, 0, 0, 0, 2924, 0, 4096, 1, 0, 0, 911, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 289, 0, 0, 0, 39, 4096, 4096, 4051, 3805, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17312, 17251, 15861, 15976, 15224, 16197, 16784, 16467] avg=16384.0000 max=17312 max_rank=0 min=15224 min_rank=4 imbalance_ratio=1.056641 max_load_reduction=113760 max_load_reduction_pct=0.867920 moved_to_echo=64672 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[5, 1, 0, 4091, 7, 1731, 18, 0, 4070, 71, 3805, 2981, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 108, 4096, 0, 13, 0, 0, 24, 1212, 2581, 2730, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1276, 4040, 0, 0, 14, 3684, 4090, 0, 14, 1709, 0, 0, 2, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3954, 4083, 1, 31, 0, 12, 246, 0, 79, 0, 0, 59, 0, 4083, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 15, 0, 4079, 2997, 3943, 0, 0, 0, 0, 0, 0, 4096, 3933, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3991, 4, 11, 0, 127, 0, 0, 1829, 0, 4096, 0, 0, 0, 911, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 1, 0, 28, 3977, 4083, 4, 4077, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 95, 0, 235, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4086, 4096, 3791, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16785, 15046, 14990, 16687, 19220, 15290, 16550, 16504] avg=16384.0000 max=19220 max_rank=4 min=14990 min_rank=2 imbalance_ratio=1.173096 max_load_reduction=111852 max_load_reduction_pct=0.853363 moved_to_echo=67564 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 0, 4037, 16, 2, 0, 0, 4069, 0, 3826, 0, 22, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3986, 8, 23, 0, 2, 87, 4052, 0, 4019, 7, 0, 0, 4089, 5, 2714, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1281, 4079, 0, 0, 0, 3975, 4084, 0, 1, 331, 38, 0, 0, 0, 3768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4050, 4078, 0, 69, 31, 0, 100, 5, 0, 10, 1, 0, 0, 4086, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4056, 3017, 4035, 0, 32, 4, 160, 91, 0, 4075, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 14, 10, 3713, 2, 14, 47, 0, 1, 795, 4063, 1, 4008, 0, 0, 0, 910, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 8, 0, 26, 0, 0, 10, 0, 6, 2, 2, 3296, 4040, 4, 4082, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 431, 0, 3, 2, 0, 0, 28, 3, 0, 31, 49, 1, 4081, 4082, 4095, 3784, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12021, 19156, 17569, 16533, 15474, 17860, 15782, 16677] avg=16384.0000 max=19156 max_rank=1 min=12021 min_rank=0 imbalance_ratio=1.169189 max_load_reduction=111916 max_load_reduction_pct=0.853851 moved_to_echo=65793 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 31744, 128, 128, 128, 256, 128, 896, 23808, 128, 32640, 128, 128, 128, 7296, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[768, 30848, 32512, 32768, 128, 128, 128, 128, 384, 128, 128, 128, 128, 640, 128, 32768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32512, 128, 512, 128, 0, 0, 128, 128, 128, 128, 256, 30336, 32768, 128, 32768, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 32768, 24192, 32512, 0, 256, 128, 256, 256, 768, 32640, 7040, 256, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 256, 32384, 128, 128, 896, 128, 512, 32640, 256, 23040, 128, 128, 128, 11520, 7040, 21760, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 10240, 32640, 128, 128, 128, 30464, 32512, 0, 128, 19072, 128, 128, 128, 0, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 32768, 128, 18816, 8064, 0, 32768, 256, 30592, 4992, 128, 128, 128, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 256, 128, 768, 128, 128, 128, 128, 128, 128, 512, 128, 128, 128, 256, 32768, 32768, 32768, 30464, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 529, 4083, 0, 3953, 34, 4071, 4095, 3869, 0, 0, 17, 1, 0, 0, 39, 4096, 0, 4011, 4066, 0, 0, 4058, 3843, 4073, 69, 0, 0, 4096, 4077, 0, 1000, 0, 0, 12, 0, 3, 4090, 249, 4084, 54, 2, 0, 0, 0, 4087, 0, 29, 0, 4096, 4095, 0, 0, 24, 0, 131, 1, 16, 43, 901, 0, 0, 0, 0, 3, 7, 0, 92, 0, 5, 4093, 0, 2, 15, 56, 0, 0, 0, 4092, 49, 4050, 4072, 0, 0, 9, 38, 0, 2, 11, 14, 0, 0, 459, 3765, 1114, 1, 6, 9, 27, 0, 0, 10, 52, 270, 3999, 0, 0, 8, 4046, 0, 4005, 0, 4067, 3977, 0, 0, 4085, 4067, 0, 0, 12, 4096, 0, 390, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[6, 0, 21, 0, 0, 4091, 2, 4083, 0, 4094, 4093, 4081, 0, 0, 6, 47, 1, 4, 0, 4094, 3, 4092, 4089, 0, 14, 4092, 4095, 4094, 18, 0, 1, 4000, 4086, 0, 4, 0, 0, 3, 0, 2, 4093, 532, 4096, 0, 2, 1, 2, 4, 4093, 1, 0, 0, 4091, 2886, 0, 0, 31, 10, 31, 22, 0, 0, 30, 5, 0, 869, 0, 0, 0, 8, 0, 6, 3, 4051, 1, 3, 0, 0, 0, 3968, 3, 4094, 1, 4083, 4091, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 856, 4051, 0, 0, 0, 0, 0, 338, 3, 4, 0, 3430, 4088, 0, 12, 13, 4051, 0, 4088, 206, 0, 3909, 0, 0, 3536, 4094, 0, 110, 10, 3944, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096, 48, 4096, 4095, 4018, 0, 6, 0, 758, 0, 0, 0, 4096, 1, 4029, 4096, 0, 62, 4095, 4094, 4096, 100, 39, 0, 3998, 4096, 5, 41, 0, 0, 0, 0, 0, 4093, 905, 3928, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4060, 1643, 0, 0, 83, 0, 4, 0, 0, 0, 1154, 0, 19, 15, 2, 5, 0, 0, 0, 0, 0, 3701, 0, 0, 0, 4087, 0, 230, 0, 4096, 0, 4094, 4090, 4, 0, 0, 14, 2244, 0, 2, 0, 0, 0, 54, 4083, 306, 0, 0, 1, 0, 0, 0, 0, 1, 4094, 4087, 0, 40, 0, 4096, 0, 108, 0, 1288, 4083, 0, 0, 4096, 4005, 9, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4095, 196, 4088, 4080, 4042, 0, 3, 0, 1, 0, 0, 0, 4096, 0, 3583, 4096, 0, 0, 4096, 4094, 4092, 0, 0, 0, 3758, 4094, 20, 1934, 3, 0, 0, 1, 0, 4094, 2273, 4096, 0, 0, 2, 0, 0, 4090, 2, 0, 0, 4066, 707, 0, 0, 0, 0, 0, 0, 0, 0, 3367, 0, 0, 0, 3, 31, 0, 7, 0, 0, 0, 4094, 0, 0, 0, 2460, 0, 0, 0, 4096, 0, 4096, 4093, 0, 15, 0, 0, 1, 0, 0, 0, 0, 0, 14, 4088, 3437, 0, 3, 38, 0, 1, 0, 0, 0, 2, 4095, 0, 18, 0, 4086, 0, 4079, 0, 78, 4086, 4, 13, 4084, 2799, 0, 0, 0, 4085, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[2, 0, 10, 9, 0, 4077, 1, 4066, 31, 4085, 4074, 4095, 15, 0, 0, 1, 17, 17, 0, 4087, 0, 2737, 4073, 2, 0, 4091, 3995, 4093, 0, 11, 0, 4096, 4096, 0, 4086, 15, 0, 5, 0, 0, 4092, 1258, 4026, 20, 3, 2, 0, 11, 4077, 37, 0, 0, 4072, 3582, 0, 0, 6, 10, 9, 4, 0, 43, 4072, 4, 0, 0, 0, 0, 0, 0, 2, 0, 5, 4013, 3, 0, 0, 1714, 10, 42, 0, 4084, 0, 4056, 4010, 0, 77, 0, 0, 0, 19, 0, 25, 53, 0, 43, 4076, 8, 0, 0, 1, 0, 0, 19, 1, 0, 0, 4075, 0, 0, 0, 4089, 4, 2433, 0, 674, 4033, 0, 0, 4074, 4096, 0, 0, 27, 4057, 32, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 88, 4096, 0, 4096, 0, 4096, 4096, 4094, 0, 0, 0, 502, 0, 0, 0, 4096, 0, 4019, 4096, 0, 0, 4096, 4096, 4096, 0, 453, 0, 4095, 4096, 0, 64, 0, 0, 0, 0, 0, 4020, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3531, 3040, 0, 0, 0, 0, 1, 0, 0, 0, 3516, 0, 0, 5, 0, 0, 0, 0, 39, 0, 0, 4096, 0, 0, 3, 42, 0, 79, 0, 4096, 0, 4095, 4092, 0, 102, 0, 0, 335, 0, 0, 0, 0, 0, 2, 4076, 1325, 0, 0, 48, 0, 93, 0, 0, 0, 3491, 4096, 0, 2, 0, 4096, 0, 3953, 0, 46, 4096, 0, 0, 4096, 4095, 0, 4, 0, 3995, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 0, 4095, 0, 4092, 0, 4093, 4085, 3830, 0, 17, 0, 28, 0, 0, 1, 4095, 30, 4050, 4096, 3, 0, 4089, 4039, 4096, 0, 193, 2, 3984, 4096, 35, 26, 0, 15, 0, 37, 0, 3478, 72, 4095, 4, 0, 0, 0, 0, 4096, 1, 12, 0, 4089, 3981, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 31, 1, 0, 0, 0, 54, 0, 2, 4096, 0, 0, 29, 2168, 0, 0, 0, 4061, 0, 4096, 4090, 0, 0, 0, 0, 4059, 0, 193, 0, 0, 1, 1733, 3967, 1480, 5, 19, 9, 0, 0, 0, 0, 0, 7, 4068, 0, 0, 0, 4082, 0, 4083, 0, 641, 4096, 0, 18, 4093, 2780, 6, 48, 0, 4093, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4090, 38, 4069, 0, 4075, 4039, 4057, 60, 0, 0, 200, 30, 48, 38, 4066, 323, 4091, 4048, 59, 2, 4076, 4049, 4070, 82, 0, 38, 3951, 4031, 2, 2, 0, 0, 43, 0, 54, 4091, 15, 3920, 1, 20, 0, 0, 0, 4094, 0, 116, 0, 4080, 5, 0, 0, 0, 35, 0, 3, 17, 0, 2413, 10, 113, 0, 6, 172, 13, 0, 0, 0, 0, 4081, 3, 25, 0, 3782, 0, 0, 0, 4027, 0, 4060, 3999, 0, 0, 11, 1, 1738, 37, 0, 1, 0, 55, 9, 3900, 1736, 0, 16, 172, 12, 1, 7, 11, 51, 2026, 4060, 0, 0, 0, 3895, 11, 4027, 0, 166, 4042, 0, 2, 4085, 4012, 0, 0, 0, 4085, 67, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 21, 0, 0, 0, 2, 4083, 0, 3894, 4093, 4081, 0, 0, 6, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 3, 4092, 4089, 0, 14, 228, 4095, 0, 18, 0, 1, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4, 0, 0, 3, 0, 2, 4093, 532, 4096, 0, 2, 1, 2, 4, 2541, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 4091, 2886, 0, 0, 31, 10, 31, 22, 0, 0, 30, 5, 2448, 1323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 869, 0, 0, 0, 8, 0, 6, 3, 4051, 1, 3, 0, 0, 0, 3968, 4094, 4091, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 1, 4083, 4091, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 856, 2549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 338, 3, 4, 0, 3430, 4088, 0, 12, 13, 4051, 1054, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 206, 0, 3909, 0, 0, 3536, 4094, 0, 110, 10, 237, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16233, 16545, 15470, 14971, 18740, 15678, 17244, 16191] avg=16384.0000 max=18740 max_rank=4 min=14971 min_rank=3 imbalance_ratio=1.143799 max_load_reduction=112332 max_load_reduction_pct=0.857025 moved_to_echo=67853 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4095, 196, 3889, 4080, 4042, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3583, 4096, 0, 0, 229, 4094, 0, 0, 0, 0, 3758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 20, 1934, 3, 0, 0, 1, 0, 4094, 2273, 4096, 0, 0, 2, 0, 0, 2543, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 0, 4066, 707, 0, 0, 0, 0, 0, 0, 0, 0, 3367, 0, 2446, 1324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 31, 0, 7, 0, 0, 0, 4094, 0, 0, 0, 2460, 0, 0, 4096, 4096, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4093, 0, 15, 0, 0, 1, 0, 0, 0, 0, 0, 14, 2640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3437, 0, 3, 38, 0, 1, 0, 0, 0, 2, 4095, 0, 18, 0, 4086, 1092, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 78, 4086, 4, 13, 4084, 2799, 0, 0, 0, 245, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16306, 15760, 19168, 16002, 16433, 14955, 17059, 15389] avg=16384.0000 max=19168 max_rank=2 min=14955 min_rank=5 imbalance_ratio=1.169922 max_load_reduction=111904 max_load_reduction_pct=0.853760 moved_to_echo=63836 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 0, 0, 0, 4092, 0, 3893, 4085, 3830, 0, 17, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 30, 4050, 4096, 3, 0, 228, 4039, 0, 0, 193, 2, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 35, 26, 0, 15, 0, 37, 0, 3478, 72, 4095, 4, 0, 0, 0, 0, 2539, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 12, 0, 4089, 3981, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2448, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 31, 1, 0, 0, 0, 54, 0, 2, 4096, 0, 0, 29, 2168, 0, 0, 4095, 4095, 1648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 4096, 4090, 0, 0, 0, 0, 4059, 0, 193, 0, 0, 1, 1733, 2645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 1480, 5, 19, 9, 0, 0, 0, 0, 0, 7, 4068, 0, 0, 0, 4082, 1094, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 641, 4096, 0, 18, 4093, 2780, 6, 48, 0, 246, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15948, 16626, 14505, 15952, 16220, 20878, 14931, 16012] avg=16384.0000 max=20878 max_rank=5 min=14505 min_rank=2 imbalance_ratio=1.274292 max_load_reduction=110194 max_load_reduction_pct=0.840714 moved_to_echo=68041 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 48, 3896, 4095, 4018, 0, 6, 0, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4029, 4096, 0, 62, 228, 4094, 0, 100, 39, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 41, 0, 0, 0, 0, 0, 4093, 905, 3928, 0, 0, 0, 0, 0, 2543, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4060, 1643, 0, 0, 83, 0, 4, 0, 0, 0, 1154, 0, 2449, 1324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 15, 2, 5, 0, 0, 0, 0, 0, 3701, 0, 0, 0, 4087, 0, 230, 4096, 4096, 1647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 4090, 4, 0, 0, 14, 2244, 0, 2, 0, 0, 0, 54, 2647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 306, 0, 0, 1, 0, 0, 0, 0, 1, 4094, 4087, 0, 40, 0, 4096, 1095, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 1288, 4083, 0, 0, 4096, 4005, 9, 0, 0, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16917, 16647, 15719, 14808, 17898, 17245, 18003, 13835] avg=16384.0000 max=18003 max_rank=6 min=13835 min_rank=7 imbalance_ratio=1.098816 max_load_reduction=113069 max_load_reduction_pct=0.862648 moved_to_echo=66981 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 10, 9, 0, 0, 1, 4066, 31, 3886, 4074, 4095, 15, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 17, 0, 0, 0, 2737, 4073, 2, 0, 228, 3995, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 15, 0, 5, 0, 0, 4092, 1258, 4026, 20, 3, 2, 0, 11, 2540, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 37, 0, 0, 4072, 3582, 0, 0, 6, 10, 9, 4, 0, 43, 4072, 4, 2446, 1323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 4013, 3, 0, 0, 1714, 10, 42, 4087, 4077, 1647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4056, 4010, 0, 77, 0, 0, 0, 19, 0, 25, 53, 0, 43, 2622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 8, 0, 0, 1, 0, 0, 19, 1, 0, 0, 4075, 0, 0, 0, 4089, 1085, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2433, 0, 674, 4033, 0, 0, 4074, 4096, 0, 0, 27, 243, 32, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16190, 15176, 20261, 19685, 15600, 14989, 13553, 15618] avg=16384.0000 max=20261 max_rank=2 min=13553 min_rank=6 imbalance_ratio=1.236633 max_load_reduction=110811 max_load_reduction_pct=0.845421 moved_to_echo=59760 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 88, 0, 0, 4096, 0, 3896, 4096, 4094, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 4096, 0, 0, 229, 4096, 0, 0, 453, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 64, 0, 0, 0, 0, 0, 4020, 0, 4095, 0, 0, 0, 0, 0, 2543, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3531, 3040, 0, 0, 0, 0, 1, 0, 0, 0, 3516, 0, 2449, 1324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 39, 0, 0, 4096, 0, 0, 3, 42, 0, 79, 4096, 4096, 1647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4092, 0, 102, 0, 0, 335, 0, 0, 0, 0, 0, 2, 2582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 1325, 0, 0, 48, 0, 93, 0, 0, 0, 3491, 4096, 0, 2, 0, 4096, 1068, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3953, 0, 46, 4096, 0, 0, 4096, 4095, 0, 4, 0, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16772, 16988, 14924, 17957, 14103, 15304, 18495, 16529] avg=16384.0000 max=18495 max_rank=6 min=14103 min_rank=4 imbalance_ratio=1.128845 max_load_reduction=112577 max_load_reduction_pct=0.858894 moved_to_echo=64431 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 529, 0, 0, 3953, 34, 3869, 4095, 3869, 0, 0, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 4011, 4066, 0, 0, 220, 3843, 0, 69, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 1000, 0, 0, 12, 0, 3, 4090, 249, 4084, 54, 2, 0, 0, 0, 2523, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 29, 0, 4096, 4095, 0, 0, 24, 0, 131, 1, 16, 43, 901, 0, 2435, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 0, 92, 0, 5, 4093, 0, 2, 15, 56, 0, 0, 4096, 4083, 1638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 49, 4050, 4072, 0, 0, 9, 38, 0, 2, 11, 14, 0, 0, 459, 2649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3765, 1114, 1, 6, 9, 27, 0, 0, 10, 52, 270, 3999, 0, 0, 8, 4046, 1100, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 4067, 3977, 0, 0, 4085, 4067, 0, 0, 12, 234, 0, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16367, 16344, 16207, 17173, 14090, 15445, 14609, 20837] avg=16384.0000 max=20837 max_rank=7 min=14090 min_rank=4 imbalance_ratio=1.271790 max_load_reduction=110235 max_load_reduction_pct=0.841026 moved_to_echo=64981 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 38, 4069, 0, 3876, 4039, 4057, 60, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 48, 38, 0, 323, 4091, 4048, 59, 2, 227, 4049, 0, 82, 0, 38, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 2, 2, 0, 0, 43, 0, 54, 4091, 15, 3920, 1, 20, 0, 0, 0, 2531, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 116, 0, 4080, 5, 0, 0, 0, 35, 0, 3, 17, 0, 2413, 10, 2433, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 6, 172, 13, 0, 0, 0, 0, 4081, 3, 25, 0, 3782, 0, 0, 4066, 4090, 1637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 4060, 3999, 0, 0, 11, 1, 1738, 37, 0, 1, 0, 55, 9, 2640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3900, 1736, 0, 16, 172, 12, 1, 7, 11, 51, 2026, 4060, 0, 0, 0, 3895, 1092, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4027, 0, 166, 4042, 0, 2, 4085, 4012, 0, 0, 0, 245, 67, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16339, 16986, 14818, 14524, 17988, 16578, 17178, 16661] avg=16384.0000 max=17988 max_rank=4 min=14524 min_rank=3 imbalance_ratio=1.097900 max_load_reduction=113084 max_load_reduction_pct=0.862762 moved_to_echo=68405 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 384, 30720, 32768, 128, 128, 1920, 32384, 0, 384, 768, 128, 32000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 640, 0, 128, 32640, 384, 31104, 32768, 32128, 128, 128, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 7168, 128, 128, 128, 128, 128, 32128, 5376, 32384, 128, 128, 128, 128, 128, 20352, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 256, 0, 32128, 19968, 128, 0, 256, 128, 256, 128, 128, 128, 15488, 128, 19584, 10624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32128, 9472, 128, 128, 384, 128, 512, 128, 128, 128, 13440, 32640, 0, 128, 128, 32512, 8704, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 26880, 256, 7040, 32384, 128, 128, 32256, 29952, 128, 256, 128, 2048, 128, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[256, 1024, 128, 256, 128, 128, 256, 128, 128, 32256, 128, 128, 128, 14336, 128, 4352, 32768, 32768, 13184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 32640, 32640, 128, 256, 128, 128, 8448, 128, 256, 128, 128, 128, 3200, 20992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[353, 4096, 0, 16, 0, 0, 0, 4081, 4095, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 32, 4096, 4096, 4092, 0, 0, 0, 4089, 0, 0, 1, 11, 0, 0, 0, 0, 3, 0, 0, 0, 4087, 4052, 4096, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3821, 4092, 4072, 0, 0, 0, 0, 0, 0, 4092, 4093, 0, 0, 0, 4084, 0, 0, 0, 0, 4091, 381, 4094, 0, 8, 0, 0, 6, 0, 0, 4096, 0, 4069, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4, 1, 0, 0, 0, 53, 0, 0, 0, 4096, 0, 21, 4027, 4085, 0, 0, 8, 5, 4096, 68, 0, 0, 0, 4096, 0, 0, 4071, 4047, 2, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[216, 4096, 0, 7, 0, 2, 0, 4090, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2260, 4095, 4076, 3891, 0, 0, 8, 4087, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4079, 4028, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3678, 4074, 4056, 0, 41, 0, 0, 0, 6, 4096, 4074, 0, 0, 0, 4028, 20, 0, 0, 0, 4068, 75, 4093, 10, 0, 0, 0, 544, 0, 0, 2721, 0, 4082, 0, 0, 0, 3, 0, 0, 4096, 0, 4096, 0, 20, 0, 4, 0, 0, 3, 0, 0, 4096, 0, 0, 427, 4096, 0, 9, 1, 1, 4096, 2722, 0, 5, 1, 4096, 0, 6, 4089, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[599, 4096, 0, 0, 0, 0, 0, 4077, 4096, 3217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 3869, 4096, 3832, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4079, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 4095, 4059, 0, 0, 0, 0, 19, 0, 4090, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 4051, 0, 37, 0, 0, 0, 0, 0, 3548, 523, 4043, 0, 0, 0, 0, 216, 0, 4096, 0, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 0, 4096, 0, 0, 962, 4096, 0, 0, 0, 0, 4034, 3797, 0, 1, 0, 4096, 18, 0, 4096, 4096, 0, 4092, 41, 4, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[2492, 4073, 0, 0, 234, 0, 0, 3493, 4061, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 4080, 4090, 4086, 18, 1, 90, 4087, 0, 138, 1, 2, 9, 0, 25, 0, 0, 3, 0, 0, 4021, 3893, 4067, 4077, 0, 0, 8, 0, 14, 0, 25, 10, 2, 6, 4093, 3956, 2372, 0, 0, 8, 85, 55, 37, 4075, 4069, 46, 0, 12, 4065, 41, 1, 13, 13, 3935, 0, 4053, 2, 37, 0, 0, 1, 0, 8, 3999, 23, 4086, 0, 0, 0, 0, 39, 1, 4053, 16, 4040, 13, 0, 0, 23, 99, 0, 0, 0, 0, 4071, 0, 0, 3888, 4086, 0, 0, 0, 0, 4048, 15, 1, 0, 0, 4082, 158, 0, 3977, 4085, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[329, 4095, 25, 0, 17, 0, 0, 4066, 4096, 4022, 2, 0, 0, 0, 0, 0, 3, 0, 14, 0, 700, 3288, 4094, 4040, 0, 5, 0, 4093, 0, 69, 0, 6, 3, 0, 149, 11, 3, 5, 2, 42, 4030, 4090, 3912, 4090, 0, 4, 0, 0, 0, 12, 0, 2, 0, 0, 4092, 4091, 3902, 0, 0, 0, 146, 134, 0, 3982, 4080, 0, 0, 6, 4093, 15, 0, 1, 0, 4038, 0, 4091, 17, 0, 0, 0, 0, 2, 4, 0, 3049, 4094, 0, 0, 101, 0, 827, 2, 4095, 1, 3946, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4091, 0, 0, 332, 4084, 6, 0, 7, 0, 3991, 4060, 6, 0, 0, 4089, 0, 9, 4086, 4085, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1034, 4079, 0, 101, 3, 17, 3, 3811, 4094, 3873, 0, 0, 0, 10, 4, 0, 9, 0, 14, 0, 0, 4088, 4095, 4096, 11, 0, 98, 4083, 1, 0, 51, 0, 1, 333, 0, 9, 0, 0, 1, 0, 4089, 4094, 4096, 4094, 0, 13, 0, 0, 12, 0, 32, 0, 0, 0, 3394, 4093, 3760, 0, 0, 0, 0, 0, 2, 4095, 4050, 3, 29, 1, 3982, 5, 0, 0, 0, 4090, 0, 4062, 0, 0, 16, 0, 66, 0, 0, 4078, 243, 4070, 0, 0, 0, 0, 0, 0, 4065, 0, 4054, 23, 0, 0, 9, 250, 13, 0, 8, 24, 4076, 0, 159, 3173, 4030, 13, 5, 7, 0, 4071, 3, 0, 0, 0, 4082, 0, 460, 4073, 4092, 0, 3951, 0, 43, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[7, 4083, 3, 0, 112, 6, 0, 3290, 4043, 3504, 0, 0, 36, 13, 0, 0, 22, 9, 24, 0, 1059, 4093, 4065, 4072, 0, 83, 117, 4042, 10, 0, 0, 0, 15, 0, 28, 32, 33, 10, 2, 701, 4052, 4064, 4078, 4067, 1, 40, 88, 0, 6, 0, 38, 5, 0, 0, 602, 4074, 2959, 0, 8, 17, 2, 0, 70, 4091, 4010, 7, 0, 8, 4047, 69, 4, 0, 1, 4078, 308, 4081, 49, 0, 10, 0, 0, 0, 3, 4086, 2050, 4089, 0, 4, 0, 0, 0, 18, 4092, 0, 4090, 0, 0, 11, 0, 0, 9, 0, 31, 0, 4036, 0, 353, 1075, 4028, 0, 0, 0, 0, 4092, 3796, 0, 0, 20, 4072, 0, 0, 4060, 4002, 17, 3719, 0, 871, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1298, 4087, 0, 0, 101, 0, 1, 3900, 4083, 2793, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 59, 4094, 4092, 4096, 0, 0, 0, 4077, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4096, 3984, 4095, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 3764, 4095, 4021, 0, 4, 1, 0, 0, 1, 4060, 4073, 5, 0, 0, 4087, 1, 0, 2, 0, 4096, 72, 4007, 0, 0, 0, 3, 0, 0, 0, 4087, 717, 4091, 0, 0, 0, 9, 0, 1, 3986, 0, 4096, 15, 19, 0, 0, 22, 11, 6, 0, 0, 4096, 0, 135, 3931, 4095, 1, 0, 9, 0, 4096, 0, 0, 1, 0, 4092, 1, 0, 4096, 4096, 10, 3867, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[329, 3917, 25, 0, 17, 0, 0, 4066, 4096, 4022, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 14, 0, 700, 3288, 3669, 4040, 0, 5, 0, 4093, 0, 69, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 149, 11, 3, 5, 2, 42, 4030, 4090, 3912, 4008, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 2, 0, 0, 4092, 4091, 3902, 0, 0, 0, 146, 134, 0, 3982, 425, 224, 178, 104, 82, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 6, 4093, 15, 0, 1, 0, 4038, 0, 4091, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 3049, 3870, 0, 0, 101, 0, 827, 2, 4095, 1, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4091, 0, 0, 332, 4084, 6, 0, 7, 4089, 1744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 4060, 6, 0, 0, 0, 0, 9, 4086, 2237, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16474, 15887, 16259, 17412, 16303, 15897, 14355, 18485] avg=16384.0000 max=18485 max_rank=7 min=14355 min_rank=6 imbalance_ratio=1.128235 max_load_reduction=112587 max_load_reduction_pct=0.858971 moved_to_echo=65040 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[353, 3918, 0, 16, 0, 0, 0, 4081, 4095, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 32, 4096, 3671, 4092, 0, 0, 0, 4089, 0, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4087, 4052, 4096, 4012, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3821, 4092, 4072, 0, 0, 0, 0, 0, 0, 4092, 425, 223, 178, 103, 82, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 4084, 0, 0, 0, 0, 4091, 381, 4094, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 3846, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 53, 0, 0, 0, 4096, 0, 21, 4027, 4085, 0, 0, 8, 4096, 1727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 68, 0, 0, 0, 0, 0, 0, 4071, 2217, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16162, 16006, 16253, 17126, 16713, 16143, 18114, 14555] avg=16384.0000 max=18114 max_rank=6 min=14555 min_rank=7 imbalance_ratio=1.105591 max_load_reduction=112958 max_load_reduction_pct=0.861801 moved_to_echo=65525 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[216, 3918, 0, 7, 0, 2, 0, 4090, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2260, 4095, 3653, 3891, 0, 0, 8, 4087, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4079, 4028, 4096, 4014, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3678, 4074, 4056, 0, 41, 0, 0, 0, 6, 4096, 423, 224, 178, 104, 82, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 4028, 20, 0, 0, 0, 4068, 75, 4093, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 544, 0, 0, 2721, 0, 3858, 0, 0, 0, 3, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 4, 0, 0, 3, 0, 0, 4096, 0, 0, 427, 4096, 0, 9, 1, 4096, 1748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 2722, 0, 5, 1, 0, 0, 6, 4089, 2244, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16422, 18001, 16241, 17000, 16330, 15318, 14500, 17260] avg=16384.0000 max=18001 max_rank=1 min=14500 min_rank=6 imbalance_ratio=1.098694 max_load_reduction=113071 max_load_reduction_pct=0.862663 moved_to_echo=63408 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[599, 3918, 0, 0, 0, 0, 0, 4077, 4096, 3217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 3869, 3671, 3832, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4079, 4096, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 4095, 4059, 0, 0, 0, 0, 19, 0, 4090, 425, 221, 178, 104, 82, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 4051, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3548, 523, 3822, 0, 0, 0, 0, 216, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 4096, 0, 0, 962, 4096, 0, 0, 0, 4096, 1748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 3797, 0, 1, 0, 0, 18, 0, 4096, 2244, 0, 4092, 41, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15907, 15519, 16291, 17366, 16340, 16301, 15021, 18327] avg=16384.0000 max=18327 max_rank=7 min=15021 min_rank=6 imbalance_ratio=1.118591 max_load_reduction=112745 max_load_reduction_pct=0.860176 moved_to_echo=65989 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1298, 3909, 0, 0, 101, 0, 1, 3900, 4083, 2793, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 4094, 3667, 4096, 0, 0, 0, 4077, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4096, 3984, 4095, 4014, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3764, 4095, 4021, 0, 4, 1, 0, 0, 1, 4060, 425, 224, 178, 104, 82, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 5, 0, 0, 4087, 1, 0, 2, 0, 4096, 72, 4007, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 717, 3867, 0, 0, 0, 9, 0, 1, 3986, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 22, 11, 6, 0, 0, 4096, 0, 135, 3931, 4095, 1, 0, 9, 4092, 1748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 1, 0, 4096, 2244, 10, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16087, 16191, 16231, 16997, 16308, 16778, 18165, 14315] avg=16384.0000 max=18165 max_rank=6 min=14315 min_rank=7 imbalance_ratio=1.108704 max_load_reduction=112907 max_load_reduction_pct=0.861412 moved_to_echo=65566 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2492, 3896, 0, 0, 234, 0, 0, 3493, 4061, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 4080, 3665, 4086, 18, 1, 90, 4087, 0, 138, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 25, 0, 0, 3, 0, 0, 4021, 3893, 4067, 3996, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 25, 10, 2, 6, 4093, 3956, 2372, 0, 0, 8, 85, 55, 37, 4075, 425, 224, 177, 104, 81, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 46, 0, 12, 4065, 41, 1, 13, 13, 3935, 0, 4053, 2, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 3999, 23, 3862, 0, 0, 0, 0, 39, 1, 4053, 16, 4040, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 99, 0, 0, 0, 0, 4071, 0, 0, 3888, 4086, 0, 0, 0, 4082, 1744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 15, 1, 0, 0, 0, 158, 0, 3977, 2237, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18256, 16178, 16022, 15787, 16249, 16055, 17993, 14532] avg=16384.0000 max=18256 max_rank=0 min=14532 min_rank=7 imbalance_ratio=1.114258 max_load_reduction=112816 max_load_reduction_pct=0.860718 moved_to_echo=64829 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1034, 3895, 0, 101, 3, 17, 3, 3811, 4094, 3873, 0, 0, 0, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 14, 0, 0, 4088, 3665, 4096, 11, 0, 98, 4083, 1, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 333, 0, 9, 0, 0, 1, 0, 4089, 4094, 4096, 4009, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 32, 0, 0, 0, 3394, 4093, 3760, 0, 0, 0, 0, 0, 2, 4095, 430, 227, 184, 110, 85, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 3, 29, 1, 3982, 5, 0, 0, 0, 4090, 0, 4062, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 4078, 243, 3843, 0, 0, 0, 0, 0, 0, 4065, 0, 4054, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 250, 13, 0, 8, 24, 4076, 0, 159, 3173, 4030, 13, 5, 7, 4082, 1752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 3, 0, 0, 0, 0, 0, 460, 4073, 2230, 0, 3951, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16845, 16116, 16645, 16467, 16195, 16372, 17601, 14831] avg=16384.0000 max=17601 max_rank=6 min=14831 min_rank=7 imbalance_ratio=1.074280 max_load_reduction=113471 max_load_reduction_pct=0.865715 moved_to_echo=64999 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[7, 3905, 3, 0, 112, 6, 0, 3290, 4043, 3504, 0, 0, 36, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 9, 24, 0, 1059, 4093, 3643, 4072, 0, 83, 117, 4042, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 28, 32, 33, 10, 2, 701, 4052, 4064, 4078, 3986, 1, 40, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 38, 5, 0, 0, 602, 4074, 2959, 0, 8, 17, 2, 0, 70, 4091, 422, 224, 178, 102, 81, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 7, 0, 8, 4047, 69, 4, 0, 1, 4078, 308, 4081, 49, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4086, 2050, 3865, 0, 4, 0, 0, 0, 18, 4092, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 9, 0, 31, 0, 4036, 0, 353, 1075, 4028, 0, 0, 0, 4072, 1708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3796, 0, 0, 20, 0, 0, 0, 4060, 2192, 17, 3719, 0, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14919, 17174, 17130, 12917, 16634, 18208, 15323, 18767] avg=16384.0000 max=18767 max_rank=7 min=12917 min_rank=3 imbalance_ratio=1.145447 max_load_reduction=112305 max_load_reduction_pct=0.856819 moved_to_echo=68932 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[640, 128, 128, 26624, 6656, 30848, 0, 128, 128, 128, 1152, 128, 32640, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32256, 128, 128, 128, 32512, 256, 128, 128, 128, 32512, 896, 32640, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 4224, 31744, 29312, 32256, 128, 128, 384, 32768, 128, 256, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[6400, 31360, 128, 128, 512, 128, 128, 30848, 32768, 29312, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32640, 14464, 128, 128, 128, 0, 256, 512, 32640, 17920, 128, 32128, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 384, 256, 128, 128, 128, 128, 896, 32640, 32384, 32640, 32128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 27520, 32640, 29312, 0, 128, 128, 256, 256, 128, 32640, 3456, 1792, 1536, 896, 768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 384, 128, 128, 128, 128, 32768, 0, 768, 17920, 32640, 128, 128, 128, 32768, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 1, 0, 4096, 0, 0, 4, 0, 8, 3843, 4081, 1, 0, 2, 0, 4096, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 4067, 0, 2642, 4096, 3, 3797, 16, 0, 0, 0, 0, 4085, 4094, 0, 0, 0, 0, 4092, 34, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 3893, 0, 0, 0, 4094, 4096, 2, 0, 0, 1766, 0, 0, 4058, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 3649, 0, 4090, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4092, 4095, 0, 0, 12, 0, 4096, 413, 0, 4096, 0, 4096, 0, 0, 6, 492] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[2, 0, 22, 4074, 0, 0, 0, 1, 13, 3848, 4079, 2, 0, 5, 0, 4094, 0, 4083, 1, 3583, 15, 1, 0, 31, 1, 3950, 0, 5, 4079, 0, 4091, 0, 3, 0, 0, 0, 4055, 4092, 0, 0, 0, 13, 2841, 0, 0, 0, 26, 0, 1, 0, 0, 0, 281, 4079, 1, 87, 0, 10, 0, 4062, 4095, 0, 2, 0, 4074, 0, 2, 4096, 7, 0, 0, 0, 56, 14, 4, 1, 4074, 4085, 1, 4067, 4082, 0, 39, 0, 1358, 0, 3, 4096, 0, 21, 0, 1, 4066, 4, 0, 0, 0, 0, 19, 30, 0, 0, 0, 14, 0, 4079, 0, 4089, 0, 4096, 0, 1, 4095, 3928, 21, 2, 0, 15, 4086, 4075, 12, 4091, 0, 4081, 0, 545, 0, 14] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[33, 0, 13, 6, 0, 0, 5, 0, 74, 3834, 4090, 0, 0, 1, 0, 4077, 0, 4093, 0, 4094, 5, 0, 0, 0, 0, 3993, 6, 0, 4092, 0, 4074, 13, 0, 9, 656, 197, 4090, 4052, 0, 3, 2719, 354, 4093, 0, 4, 0, 0, 0, 3, 59, 0, 0, 0, 4080, 0, 4096, 0, 0, 3, 4094, 4093, 0, 69, 10, 1781, 0, 5, 4092, 0, 0, 17, 7, 0, 19, 1, 0, 4093, 2900, 0, 4096, 4091, 0, 0, 4, 0, 0, 3, 4092, 0, 0, 13, 0, 3741, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 4089, 4, 4080, 0, 4066, 2, 106, 4093, 3912, 0, 8, 3957, 3, 4034, 0, 0, 4094, 3, 4086, 1, 0, 4, 382] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 2327, 0, 0, 0, 0, 16, 2180, 4096, 0, 0, 0, 0, 4094, 0, 4096, 0, 4025, 0, 0, 0, 0, 1, 4095, 0, 70, 4096, 0, 4085, 0, 0, 0, 60, 19, 4096, 3988, 0, 0, 127, 62, 4092, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 4096, 4, 0, 0, 1635, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3940, 0, 4096, 4096, 0, 0, 0, 3491, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 1911, 0, 4096, 0, 0, 4012, 0, 4096, 0, 0, 302, 442] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 79, 0, 2696, 0, 0, 0, 2, 2, 3056, 4096, 0, 0, 0, 0, 4061, 0, 4093, 0, 3029, 8, 0, 0, 0, 0, 4089, 0, 32, 4096, 0, 4069, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 711, 0, 3991, 0, 0, 0, 0, 2, 0, 49, 0, 0, 0, 4096, 0, 3601, 0, 0, 0, 4008, 4084, 22, 12, 0, 999, 0, 0, 3919, 22, 0, 0, 0, 0, 1, 0, 0, 4094, 4096, 0, 4090, 4092, 0, 0, 0, 48, 0, 0, 4094, 2, 8, 102, 0, 4070, 19, 26, 0, 1, 0, 6, 0, 0, 0, 0, 4, 0, 4092, 0, 4096, 0, 4096, 0, 0, 4090, 4096, 0, 0, 2879, 0, 4096, 0, 0, 3725, 0, 4096, 7, 0, 225, 3703] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 67, 0, 4095, 0, 0, 0, 0, 0, 4094, 4092, 0, 0, 5, 18, 4094, 3, 4089, 0, 3251, 0, 14, 0, 0, 0, 4082, 0, 121, 4090, 1, 3773, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 3189, 2928, 0, 0, 0, 0, 0, 0, 6, 0, 7, 4095, 0, 3433, 6, 0, 0, 4081, 4096, 3, 0, 0, 3107, 0, 0, 4079, 0, 0, 0, 0, 14, 0, 0, 0, 4095, 4042, 8, 4081, 4095, 49, 0, 0, 0, 0, 4, 4093, 1, 0, 0, 0, 4096, 3, 0, 0, 21, 3, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 66, 4063, 4090, 0, 0, 12, 0, 4092, 310, 4, 4085, 0, 4096, 0, 0, 0, 245] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[15, 0, 45, 4001, 0, 0, 0, 0, 41, 4058, 4067, 0, 24, 11, 0, 4076, 62, 4074, 3, 2301, 0, 0, 0, 0, 0, 4053, 0, 290, 4087, 2, 3861, 48, 4, 0, 30, 8, 4081, 4045, 0, 0, 0, 0, 3060, 0, 14, 35, 0, 0, 0, 15, 25, 0, 0, 4063, 8, 2901, 0, 0, 0, 4095, 4077, 0, 0, 0, 3520, 70, 6, 4088, 0, 0, 51, 2, 0, 1, 0, 0, 4078, 4088, 0, 4096, 4059, 0, 3, 0, 1257, 0, 8, 4084, 0, 33, 0, 0, 4023, 11, 0, 0, 15, 13, 0, 0, 0, 0, 0, 0, 0, 4078, 17, 4051, 0, 4065, 5, 385, 4090, 4071, 0, 10, 0, 10, 4085, 44, 0, 4088, 4, 4084, 31, 0, 0, 2868] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 822, 0, 0, 82, 23, 1, 3743, 4066, 22, 0, 6, 0, 4034, 14, 4065, 0, 4096, 3, 0, 29, 82, 0, 4055, 13, 3982, 4069, 1, 3995, 0, 0, 0, 23, 0, 4030, 4081, 0, 0, 0, 0, 3118, 0, 22, 0, 0, 105, 0, 0, 47, 2, 0, 4095, 33, 3476, 204, 90, 0, 3992, 4075, 0, 0, 35, 1470, 0, 16, 3642, 6, 0, 17, 13, 17, 2, 11, 36, 4061, 4084, 0, 4067, 3968, 35, 0, 0, 2273, 19, 1, 4076, 0, 44, 0, 0, 4059, 5, 0, 0, 1, 0, 0, 13, 0, 0, 21, 26, 0, 4089, 14, 4055, 0, 4070, 0, 635, 4052, 4073, 9, 19, 42, 29, 4079, 715, 49, 4025, 0, 4072, 0, 4, 0, 427] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[33, 0, 13, 6, 0, 0, 5, 0, 74, 3834, 4090, 0, 0, 1, 0, 4077, 1405, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3876, 0, 4094, 5, 0, 0, 0, 0, 3993, 6, 0, 0, 0, 4074, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 656, 197, 4090, 4052, 0, 3, 2719, 354, 4093, 0, 4, 0, 0, 0, 3508, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 59, 0, 0, 0, 4080, 0, 4096, 0, 0, 3, 4094, 4093, 0, 69, 10, 600, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1781, 0, 5, 4092, 0, 0, 17, 7, 0, 19, 1, 0, 2088, 2900, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4, 0, 0, 3, 4092, 0, 0, 13, 0, 3741, 0, 0, 0, 2501, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 4089, 4, 4080, 0, 4066, 2, 106, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1199, 3912, 0, 8, 3957, 3, 4034, 0, 0, 4094, 3, 0, 1, 0, 4, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13931, 16061, 19834, 17324, 15006, 15029, 16290, 17597] avg=16384.0000 max=19834 max_rank=2 min=13931 min_rank=0 imbalance_ratio=1.210571 max_load_reduction=111238 max_load_reduction_pct=0.848679 moved_to_echo=63922 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4096, 0, 0, 4, 0, 8, 3843, 4081, 1, 0, 2, 0, 4096, 1405, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3879, 0, 4094, 0, 0, 0, 0, 0, 4067, 0, 2642, 0, 3, 3797, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4094, 0, 0, 0, 0, 4092, 34, 0, 0, 0, 0, 3510, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 3893, 0, 0, 0, 4094, 4096, 2, 0, 0, 600, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1766, 0, 0, 4058, 0, 1, 0, 0, 0, 0, 0, 0, 2090, 3649, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4096, 0, 4, 0, 2500, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 4096, 0, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1199, 4095, 0, 0, 12, 0, 4096, 413, 0, 4096, 0, 0, 0, 0, 6, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17930, 18498, 15963, 17011, 15654, 15370, 16237, 14409] avg=16384.0000 max=18498 max_rank=1 min=14409 min_rank=7 imbalance_ratio=1.129028 max_load_reduction=112574 max_load_reduction_pct=0.858871 moved_to_echo=61670 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 22, 4074, 0, 0, 0, 1, 13, 3848, 4079, 2, 0, 5, 0, 4094, 1402, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3863, 1, 3583, 15, 1, 0, 31, 1, 3950, 0, 5, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4055, 4092, 0, 0, 0, 13, 2841, 0, 0, 0, 26, 0, 3497, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 281, 4079, 1, 87, 0, 10, 0, 4062, 4095, 0, 2, 0, 603, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 2, 4096, 7, 0, 0, 0, 56, 14, 4, 1, 2069, 4085, 1, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 39, 0, 1358, 0, 3, 4096, 0, 21, 0, 1, 4066, 4, 0, 0, 2506, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 30, 0, 0, 0, 14, 0, 4079, 0, 4089, 0, 4096, 0, 1, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 3928, 21, 2, 0, 15, 4086, 4075, 12, 4091, 0, 0, 0, 545, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17939, 15541, 14675, 13441, 18476, 16758, 16261, 17981] avg=16384.0000 max=18476 max_rank=4 min=13441 min_rank=3 imbalance_ratio=1.127686 max_load_reduction=112596 max_load_reduction_pct=0.859039 moved_to_echo=69476 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 67, 0, 4095, 0, 0, 0, 0, 0, 4094, 4092, 0, 0, 5, 18, 4094, 1405, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3872, 0, 3251, 0, 14, 0, 0, 0, 4082, 0, 121, 0, 1, 3773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 3189, 2928, 0, 0, 0, 0, 3505, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7, 4095, 0, 3433, 6, 0, 0, 4081, 4096, 3, 0, 0, 600, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3107, 0, 0, 4079, 0, 0, 0, 0, 14, 0, 0, 0, 2090, 4042, 8, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 49, 0, 0, 0, 0, 4, 4093, 1, 0, 0, 0, 4096, 3, 0, 0, 2483, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 3, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 66, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1190, 4090, 0, 0, 12, 0, 4092, 310, 4, 4085, 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18260, 15117, 17965, 16544, 17421, 15409, 16328, 14028] avg=16384.0000 max=18260 max_rank=0 min=14028 min_rank=7 imbalance_ratio=1.114502 max_load_reduction=112812 max_load_reduction_pct=0.860687 moved_to_echo=63186 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 79, 0, 2696, 0, 0, 0, 2, 2, 3056, 4096, 0, 0, 0, 0, 4061, 1405, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3876, 0, 3029, 8, 0, 0, 0, 0, 4089, 0, 32, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 711, 0, 3991, 0, 0, 0, 0, 2, 3511, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 4096, 0, 3601, 0, 0, 0, 4008, 4084, 22, 12, 0, 600, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 999, 0, 0, 3919, 22, 0, 0, 0, 0, 1, 0, 0, 2089, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 48, 0, 0, 4094, 2, 8, 102, 0, 4070, 19, 26, 0, 2499, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 4, 0, 4092, 0, 4096, 0, 4096, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1199, 4096, 0, 0, 2879, 0, 4096, 0, 0, 3725, 0, 0, 7, 0, 225, 3703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15789, 15103, 16558, 16689, 15216, 15545, 16242, 19930] avg=16384.0000 max=19930 max_rank=7 min=15103 min_rank=1 imbalance_ratio=1.216431 max_load_reduction=111142 max_load_reduction_pct=0.847946 moved_to_echo=66933 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2327, 0, 0, 0, 0, 16, 2180, 4096, 0, 0, 0, 0, 4094, 1406, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3879, 0, 4025, 0, 0, 0, 0, 1, 4095, 0, 70, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 19, 4096, 3988, 0, 0, 127, 62, 4092, 32, 0, 0, 0, 0, 3511, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 4096, 4, 0, 0, 601, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1635, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 2089, 3940, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3491, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 8, 0, 2503, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1200, 4096, 0, 0, 1911, 0, 4096, 0, 0, 4012, 0, 0, 0, 0, 302, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14512, 16155, 16136, 17205, 15843, 18863, 16299, 16059] avg=16384.0000 max=18863 max_rank=5 min=14512 min_rank=0 imbalance_ratio=1.151306 max_load_reduction=112209 max_load_reduction_pct=0.856087 moved_to_echo=67064 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[15, 0, 45, 4001, 0, 0, 0, 0, 41, 4058, 4067, 0, 24, 11, 0, 4076, 1399, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 3858, 3, 2301, 0, 0, 0, 0, 0, 4053, 0, 290, 0, 2, 3861, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 30, 8, 4081, 4045, 0, 0, 0, 0, 3060, 0, 14, 35, 0, 0, 3504, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 25, 0, 0, 4063, 8, 2901, 0, 0, 0, 4095, 4077, 0, 0, 0, 598, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3520, 70, 6, 4088, 0, 0, 51, 2, 0, 1, 0, 0, 2081, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 3, 0, 1257, 0, 8, 4084, 0, 33, 0, 0, 4023, 11, 0, 0, 2499, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 13, 0, 0, 0, 0, 0, 0, 0, 4078, 17, 4051, 0, 4065, 5, 385, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1199, 4071, 0, 10, 0, 10, 4085, 44, 0, 4088, 4, 0, 31, 0, 0, 2868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18129, 14478, 14928, 15998, 18003, 16560, 16566, 16410] avg=16384.0000 max=18129 max_rank=0 min=14478 min_rank=1 imbalance_ratio=1.106506 max_load_reduction=112943 max_load_reduction_pct=0.861687 moved_to_echo=67539 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 822, 0, 0, 82, 23, 1, 3743, 4066, 22, 0, 6, 0, 4034, 1394, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3849, 0, 4096, 3, 0, 29, 82, 0, 4055, 13, 3982, 0, 1, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4030, 4081, 0, 0, 0, 0, 3118, 0, 22, 0, 0, 105, 3487, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 2, 0, 4095, 33, 3476, 204, 90, 0, 3992, 4075, 0, 0, 35, 595, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1470, 0, 16, 3642, 6, 0, 17, 13, 17, 2, 11, 36, 2072, 4084, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 35, 0, 0, 2273, 19, 1, 4076, 0, 44, 0, 0, 4059, 5, 0, 0, 2476, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 13, 0, 0, 21, 26, 0, 4089, 14, 4055, 0, 4070, 0, 635, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1187, 4073, 9, 19, 42, 29, 4079, 715, 49, 4025, 0, 0, 0, 4, 0, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14582, 20119, 15013, 16860, 15453, 17538, 16849, 14658] avg=16384.0000 max=20119 max_rank=1 min=14582 min_rank=0 imbalance_ratio=1.227966 max_load_reduction=110953 max_load_reduction_pct=0.846504 moved_to_echo=64498 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32640, 128, 128, 128, 8448, 128, 128, 32768, 128, 128, 128, 128, 32256, 128, 128, 0, 19968, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 256, 128, 22144, 0, 0, 128, 128, 256, 28672, 32768, 128, 128, 128, 128, 32640, 11264, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 30976, 128, 28544, 128, 128, 128, 128, 128, 32384, 128, 7168, 0, 128, 31872, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[18432, 128, 128, 32128, 128, 128, 128, 128, 128, 128, 128, 128, 16768, 30976, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 128, 0, 32768, 128, 32768, 0, 32768, 128, 1280, 31616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[9600, 32384, 128, 128, 8832, 128, 32768, 5632, 128, 32256, 128, 0, 128, 640, 640, 8576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 896, 256, 32640, 32640, 128, 128, 3584, 512, 28544, 3072, 128, 128, 128, 128, 28160, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 256, 128, 128, 384, 32768, 128, 25600, 256, 128, 128, 32640, 32768, 128, 128, 128, 4864, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4086, 10, 0, 1388, 0, 32, 0, 0, 0, 2, 4049, 4093, 3981, 4, 0, 1924, 0, 4085, 748, 1, 0, 0, 4075, 9, 0, 0, 5, 21, 54, 3809, 4094, 4090, 4094, 13, 10, 3753, 0, 0, 0, 1, 0, 3, 4083, 0, 0, 1016, 0, 0, 1, 4016, 0, 2, 0, 0, 143, 0, 3891, 3131, 25, 0, 0, 0, 15, 0, 4068, 0, 0, 3968, 5, 46, 3379, 2, 0, 4094, 180, 0, 41, 4088, 4082, 1, 4096, 0, 3, 4068, 0, 3, 1152, 0, 0, 8, 327, 2, 0, 0, 3421, 0, 0, 4077, 0, 15, 0, 45, 3910, 0, 1, 0, 0, 0, 4087, 0, 4094, 0, 6, 0, 0, 119, 0, 4069, 4094, 0, 1, 2, 0, 4093, 0, 0, 668, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3407, 132, 0, 67, 62, 0, 0, 4096, 4096, 4059, 0, 0, 0, 0, 1214, 11, 0, 0, 4, 4096, 0, 0, 180, 0, 0, 1, 133, 4084, 4002, 4096, 16, 0, 4095, 0, 0, 0, 0, 0, 0, 4093, 1554, 192, 0, 658, 0, 0, 4096, 0, 0, 0, 120, 0, 0, 4085, 4096, 0, 0, 4096, 0, 0, 5, 4096, 0, 0, 4096, 0, 0, 1, 0, 3, 4096, 0, 0, 0, 4092, 4093, 0, 4079, 0, 0, 4096, 876, 0, 3813, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 5, 4086, 6, 0, 1, 0, 338, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 1, 0, 0, 2, 0, 4032, 4093, 0, 0, 0, 0, 3848, 0, 0, 4058, 30] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4084, 0, 0, 139, 88, 4, 5, 4026, 4096, 3807, 0, 0, 271, 0, 4044, 1689, 0, 0, 3, 4096, 0, 2, 74, 0, 31, 9, 39, 4096, 2183, 4088, 0, 0, 4096, 0, 0, 0, 0, 13, 0, 4085, 601, 0, 0, 3700, 2, 0, 3949, 0, 0, 0, 282, 0, 6, 4096, 3599, 0, 11, 89, 0, 0, 872, 3681, 0, 0, 4092, 14, 0, 1394, 0, 0, 4096, 0, 1, 0, 4025, 4093, 0, 3994, 0, 0, 4096, 4062, 0, 3772, 0, 0, 0, 0, 0, 0, 28, 4088, 0, 37, 4075, 5, 0, 27, 0, 0, 0, 0, 7, 0, 0, 4095, 0, 4093, 0, 0, 0, 0, 821, 0, 4056, 4083, 0, 0, 0, 0, 1915, 0, 0, 48, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4071, 30, 1, 382, 837, 0, 12, 4013, 4077, 4002, 1, 12, 79, 9, 0, 3649, 0, 0, 44, 4096, 0, 0, 0, 0, 0, 1, 41, 3913, 1461, 4085, 0, 0, 4093, 35, 0, 1, 0, 0, 0, 4090, 4096, 573, 0, 2648, 0, 0, 4096, 1, 2, 6, 0, 0, 0, 3945, 4096, 8, 4, 4035, 25, 10, 438, 4089, 11, 20, 4093, 2, 0, 0, 21, 0, 4085, 0, 19, 0, 4027, 4087, 19, 4096, 0, 0, 4078, 0, 0, 2875, 0, 1, 0, 0, 0, 0, 0, 4096, 17, 9, 4074, 0, 39, 701, 0, 1, 0, 0, 16, 0, 6, 3998, 16, 4080, 0, 33, 0, 27, 0, 0, 2889, 4053, 0, 0, 4, 1, 169, 0, 0, 4031, 175] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4017, 0, 0, 1713, 232, 63, 0, 4082, 0, 0, 4061, 4069, 4085, 23, 0, 726, 0, 633, 3002, 0, 14, 0, 4069, 10, 0, 0, 0, 0, 260, 314, 3993, 4089, 4073, 0, 0, 4079, 0, 0, 25, 0, 0, 0, 4041, 1888, 3799, 135, 2142, 18, 5, 3937, 0, 12, 0, 0, 18, 0, 4069, 4077, 6, 38, 3557, 0, 37, 20, 4017, 0, 0, 4069, 16, 21, 94, 0, 0, 4085, 50, 0, 26, 4082, 4032, 28, 4096, 24, 11, 4069, 1757, 0, 3282, 0, 0, 9, 0, 17, 0, 0, 4073, 0, 0, 4078, 26, 0, 0, 3, 80, 0, 0, 0, 3, 0, 4038, 2, 3326, 1, 0, 0, 0, 197, 0, 779, 4096, 0, 19, 0, 8, 596, 0, 0, 531, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 27, 3830, 754, 0, 0, 0, 0, 0, 4095, 4045, 4096, 0, 0, 429, 0, 2927, 3950, 0, 0, 0, 4096, 0, 0, 0, 0, 13, 71, 15, 3988, 3679, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 4, 2576, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 3984, 4072, 0, 0, 1671, 0, 0, 67, 3653, 0, 0, 4090, 1, 0, 224, 0, 0, 4096, 54, 0, 0, 4086, 4096, 0, 4096, 91, 0, 4096, 40, 0, 3949, 0, 0, 0, 9, 0, 9, 0, 4055, 0, 0, 4096, 0, 0, 0, 84, 161, 0, 0, 0, 0, 0, 4096, 0, 4042, 0, 0, 0, 0, 115, 0, 4096, 4096, 0, 0, 0, 0, 789, 0, 0, 4065, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4067, 0, 0, 3166, 0, 29, 14, 1645, 0, 20, 3807, 4010, 3814, 19, 0, 19, 0, 3244, 0, 18, 5, 0, 4084, 56, 0, 601, 0, 0, 0, 0, 4058, 1821, 4044, 0, 34, 4039, 0, 0, 5, 80, 0, 44, 4039, 1, 0, 15, 1821, 0, 0, 4094, 0, 3, 0, 342, 3, 51, 3996, 4079, 0, 0, 4091, 0, 0, 437, 1795, 70, 0, 3889, 19, 2, 0, 27, 0, 4075, 0, 0, 0, 4096, 4078, 0, 3613, 0, 0, 4082, 2338, 0, 3582, 0, 0, 1, 0, 0, 0, 0, 4086, 0, 49, 3987, 5, 42, 0, 0, 47, 0, 5, 99, 0, 0, 3969, 27, 4062, 6, 103, 0, 265, 0, 13, 3741, 4081, 165, 0, 4, 2, 4087, 9, 36, 4080, 720] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 7, 4095, 433, 0, 78, 954, 9, 0, 4096, 4096, 4096, 0, 17, 0, 0, 3403, 2403, 0, 0, 20, 4096, 0, 2, 152, 0, 0, 10, 0, 4096, 1905, 4071, 0, 1, 4089, 0, 2, 0, 0, 1, 0, 4096, 2249, 1675, 0, 360, 0, 0, 4063, 4, 0, 0, 1236, 6, 0, 4081, 4096, 0, 0, 3489, 0, 0, 383, 3789, 0, 0, 4072, 4, 0, 0, 0, 0, 4086, 0, 4, 0, 4086, 4091, 0, 3927, 0, 0, 4093, 2, 0, 2673, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 3, 4094, 0, 0, 0, 0, 0, 4, 0, 29, 2, 0, 4068, 4, 4096, 0, 0, 0, 0, 0, 0, 3738, 4020, 3, 0, 0, 0, 0, 4, 0, 4092, 26] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 0, 1388, 0, 32, 0, 0, 0, 2, 4049, 3450, 3981, 4, 0, 1924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 748, 1, 0, 0, 4075, 9, 0, 0, 5, 21, 54, 3809, 4094, 4090, 181, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 13, 10, 3753, 0, 0, 0, 1, 0, 3, 4083, 0, 0, 1016, 0, 0, 147, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4016, 0, 2, 0, 0, 143, 0, 3891, 3131, 25, 0, 0, 0, 15, 0, 748, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 3968, 5, 46, 3379, 2, 0, 0, 180, 0, 41, 4088, 3893, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 4024, 0, 3, 1152, 0, 0, 8, 327, 2, 0, 0, 3421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 15, 0, 45, 3910, 0, 1, 0, 0, 0, 4087, 0, 4094, 0, 3338, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 119, 0, 4069, 4094, 0, 1, 2, 0, 4093, 0, 0, 668, 0, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14840, 21214, 13164, 12434, 19671, 13036, 19772, 16941] avg=16384.0000 max=21214 max_rank=1 min=12434 min_rank=3 imbalance_ratio=1.294800 max_load_reduction=109858 max_load_reduction_pct=0.838150 moved_to_echo=69420 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3407, 132, 0, 67, 62, 0, 0, 4096, 3453, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1214, 11, 0, 0, 4, 4096, 0, 0, 180, 0, 0, 1, 133, 4084, 4002, 181, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 4095, 0, 0, 0, 0, 0, 0, 4093, 1554, 192, 0, 658, 0, 148, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 120, 0, 0, 4085, 4096, 0, 0, 4096, 0, 0, 5, 750, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 3, 0, 0, 0, 0, 4092, 3903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 4052, 876, 0, 3813, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4086, 6, 0, 1, 0, 338, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 3346, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 4032, 4093, 0, 0, 0, 0, 3848, 0, 0, 4058, 30, 3891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15276, 13948, 14896, 17710, 16191, 16925, 16171, 19955] avg=16384.0000 max=19955 max_rank=7 min=13948 min_rank=1 imbalance_ratio=1.217957 max_load_reduction=111117 max_load_reduction_pct=0.847755 moved_to_echo=69242 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4084, 0, 0, 139, 88, 4, 5, 4026, 3453, 3807, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 1689, 0, 0, 3, 4096, 0, 2, 74, 0, 31, 9, 39, 4096, 2183, 181, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 0, 0, 13, 0, 4085, 601, 0, 0, 3700, 2, 148, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 0, 0, 282, 0, 6, 4096, 3599, 0, 11, 89, 0, 0, 872, 751, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3681, 0, 0, 4092, 14, 0, 1394, 0, 0, 0, 0, 1, 0, 4025, 3903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 4052, 4062, 0, 3772, 0, 0, 0, 0, 0, 0, 28, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4075, 5, 0, 27, 0, 0, 0, 0, 7, 0, 0, 4095, 0, 4093, 0, 3345, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 821, 0, 4056, 4083, 0, 0, 0, 0, 1915, 0, 0, 48, 3, 3890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15877, 16489, 16777, 14117, 17110, 19996, 15890, 14816] avg=16384.0000 max=19996 max_rank=5 min=14117 min_rank=3 imbalance_ratio=1.220459 max_load_reduction=111076 max_load_reduction_pct=0.847443 moved_to_echo=67812 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 4095, 433, 0, 78, 954, 9, 0, 4096, 3453, 4096, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3403, 2403, 0, 0, 20, 4096, 0, 2, 152, 0, 0, 10, 0, 4096, 1905, 181, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 1, 4089, 0, 2, 0, 0, 1, 0, 4096, 2249, 1675, 0, 360, 0, 147, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4, 0, 0, 1236, 6, 0, 4081, 4096, 0, 0, 3489, 0, 0, 383, 750, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3789, 0, 0, 4072, 4, 0, 0, 0, 0, 0, 0, 4, 0, 4086, 3902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 0, 0, 4049, 2, 0, 2673, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 0, 0, 0, 0, 4, 0, 29, 2, 0, 4068, 4, 4096, 0, 3345, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3738, 4020, 3, 0, 0, 0, 0, 4, 0, 4092, 26, 3881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17238, 16310, 16735, 18570, 15857, 14748, 15850, 15764] avg=16384.0000 max=18570 max_rank=3 min=14748 min_rank=5 imbalance_ratio=1.133423 max_load_reduction=112502 max_load_reduction_pct=0.858322 moved_to_echo=62219 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1713, 232, 63, 0, 4082, 0, 0, 4061, 3430, 4085, 23, 0, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 633, 3002, 0, 14, 0, 4069, 10, 0, 0, 0, 0, 260, 314, 3993, 4089, 180, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 4079, 0, 0, 25, 0, 0, 0, 4041, 1888, 3799, 135, 2142, 18, 145, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3937, 0, 12, 0, 0, 18, 0, 4069, 4077, 6, 38, 3557, 0, 37, 20, 735, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 0, 4069, 16, 21, 94, 0, 0, 0, 50, 0, 26, 4082, 3846, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 24, 11, 4025, 1757, 0, 3282, 0, 0, 9, 0, 17, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 26, 0, 0, 3, 80, 0, 0, 0, 3, 0, 4038, 2, 3326, 1, 3282, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 0, 779, 4096, 0, 19, 0, 8, 596, 0, 0, 531, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18415, 16605, 20389, 16970, 16249, 17294, 15044, 10106] avg=16384.0000 max=20389 max_rank=2 min=10106 min_rank=7 imbalance_ratio=1.244446 max_load_reduction=110683 max_load_reduction_pct=0.844444 moved_to_echo=58693 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 27, 3830, 754, 0, 0, 0, 0, 0, 4095, 3410, 4096, 0, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2927, 3950, 0, 0, 0, 4096, 0, 0, 0, 0, 13, 71, 15, 3988, 3679, 179, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 4, 2576, 7, 148, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3984, 4072, 0, 0, 1671, 0, 0, 67, 751, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3653, 0, 0, 4090, 1, 0, 224, 0, 0, 0, 54, 0, 0, 4086, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 91, 0, 4052, 40, 0, 3949, 0, 0, 0, 9, 0, 9, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 84, 161, 0, 0, 0, 0, 0, 4096, 0, 4042, 0, 3345, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 4096, 4096, 0, 0, 0, 0, 789, 0, 0, 4065, 1, 3890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16641, 18960, 14977, 15097, 16014, 16301, 16030, 17052] avg=16384.0000 max=18960 max_rank=1 min=14977 min_rank=2 imbalance_ratio=1.157227 max_load_reduction=112112 max_load_reduction_pct=0.855347 moved_to_echo=65397 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4071, 30, 1, 382, 837, 0, 12, 4013, 3431, 4002, 1, 12, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3649, 0, 0, 44, 4096, 0, 0, 0, 0, 0, 1, 41, 3913, 1461, 182, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4093, 35, 0, 1, 0, 0, 0, 4090, 4096, 573, 0, 2648, 0, 151, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 2, 6, 0, 0, 0, 3945, 4096, 8, 4, 4035, 25, 10, 438, 751, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 11, 20, 4093, 2, 0, 0, 21, 0, 0, 0, 19, 0, 4027, 3892, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4029, 0, 0, 2875, 0, 1, 0, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4074, 0, 39, 701, 0, 1, 0, 0, 16, 0, 6, 3998, 16, 4080, 0, 3345, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 27, 0, 0, 2889, 4053, 0, 0, 4, 1, 169, 0, 0, 4031, 175, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16871, 13440, 19821, 17881, 16193, 15114, 16491, 15261] avg=16384.0000 max=19821 max_rank=2 min=13440 min_rank=1 imbalance_ratio=1.209778 max_load_reduction=111251 max_load_reduction_pct=0.848778 moved_to_echo=63059 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3166, 0, 29, 14, 1645, 0, 20, 3807, 3381, 3814, 19, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3244, 0, 18, 5, 0, 4084, 56, 0, 601, 0, 0, 0, 0, 4058, 1821, 177, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 34, 4039, 0, 0, 5, 80, 0, 44, 4039, 1, 0, 15, 1821, 0, 147, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3, 0, 342, 3, 51, 3996, 4079, 0, 0, 4091, 0, 0, 437, 745, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1795, 70, 0, 3889, 19, 2, 0, 27, 0, 0, 0, 0, 0, 4096, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3613, 0, 0, 4038, 2338, 0, 3582, 0, 0, 1, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 3987, 5, 42, 0, 0, 47, 0, 5, 99, 0, 0, 3969, 27, 4062, 6, 3322, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 265, 0, 13, 3741, 4081, 165, 0, 4, 2, 4087, 9, 36, 4080, 720, 3871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15914, 14106, 14313, 18293, 13787, 17658, 15824, 21177] avg=16384.0000 max=21177 max_rank=7 min=13787 min_rank=4 imbalance_ratio=1.292542 max_load_reduction=109895 max_load_reduction_pct=0.838432 moved_to_echo=68446 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[29312, 128, 128, 32384, 128, 128, 5120, 128, 128, 0, 384, 128, 128, 32640, 31232, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32640, 128, 128, 768, 256, 4608, 128, 128, 256, 128, 128, 32512, 128, 32000, 128, 26752, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[256, 0, 384, 1280, 128, 27520, 32640, 256, 128, 128, 128, 15616, 128, 128, 21632, 1024, 31104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32000, 128, 128, 32384, 9088, 128, 25216, 0, 128, 128, 384, 128, 128, 128, 32128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32384, 128, 128, 128, 2048, 256, 128, 32256, 31360, 128, 128, 21120, 128, 128, 2304, 6016, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 25856, 1664, 128, 768, 7680, 128, 128, 32256, 27520, 32000, 128, 128, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 19584, 15488, 128, 128, 128, 32768, 128, 128, 1024, 128, 128, 512, 4352, 32384, 23296, 1536, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 128, 32384, 128, 128, 128, 128, 128, 128, 32640, 10496, 6272, 1280, 13952, 128, 1280, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 105, 23, 0, 0, 0, 0, 0, 0, 5, 0, 1, 4096, 2752, 4096, 0, 0, 3942, 0, 15, 0, 0, 102, 1, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4055, 4096, 0, 0, 442, 4082, 0, 0, 4096, 4096, 3935, 0, 0, 4096, 56, 0, 4096, 5, 0, 4096, 3449, 4087, 46, 0, 0, 0, 1, 4029, 108, 5, 4057, 33, 1, 0, 690, 4096, 4083, 0, 6, 4096, 4044, 0, 0, 0, 4055, 0, 4078, 0, 0, 5, 0, 0, 0, 0, 2, 0, 1562, 4090, 0, 0, 67, 3513, 4096, 0, 0, 0, 0, 3981, 0, 0, 73, 0, 0, 0, 0, 8, 0, 8, 19, 0, 3999, 4044, 0, 1, 68, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[16, 2982, 610, 0, 0, 4, 0, 200, 0, 0, 0, 0, 0, 4087, 0, 4086, 0, 0, 4094, 0, 0, 2, 0, 0, 2948, 0, 3, 0, 0, 4065, 0, 13, 9, 4, 0, 631, 4092, 0, 0, 3776, 4024, 0, 2, 0, 3260, 0, 2741, 4077, 4070, 4074, 0, 58, 4087, 0, 5, 3747, 0, 7, 3361, 4094, 4072, 0, 0, 0, 0, 2, 528, 0, 0, 3503, 4093, 0, 0, 1223, 3879, 4096, 10, 0, 4088, 4096, 0, 9, 0, 4095, 0, 4095, 33, 28, 0, 0, 3, 0, 0, 0, 0, 0, 4092, 1, 1, 0, 2762, 4091, 0, 5, 8, 34, 4083, 0, 3, 0, 0, 18, 0, 0, 4, 736, 1, 38, 2, 4092, 3968, 0, 0, 31, 13, 4, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 19, 4055, 0, 0, 1710, 0, 0, 0, 0, 0, 4096, 4088, 4096, 0, 0, 1049, 773, 2, 0, 0, 38, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 12, 0, 4096, 0, 79, 4078, 4056, 0, 0, 0, 2360, 0, 2783, 4096, 4096, 4082, 0, 4, 4091, 3, 0, 4096, 0, 0, 4033, 4064, 4084, 0, 5, 6, 0, 0, 730, 0, 0, 4096, 60, 213, 0, 4, 4095, 4048, 0, 0, 4096, 551, 29, 0, 0, 4076, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4046, 4096, 0, 0, 248, 4096, 4096, 0, 0, 0, 0, 960, 7, 0, 14, 0, 0, 13, 0, 0, 985, 0, 0, 0, 4078, 4096, 0, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4068, 8, 70, 391, 0, 0, 88, 25, 1, 0, 3, 0, 3928, 4016, 4051, 0, 0, 104, 3437, 33, 12, 0, 0, 14, 435, 0, 9, 0, 4096, 114, 3, 7, 1, 13, 0, 4054, 13, 1006, 4082, 4059, 0, 16, 6, 3137, 0, 4057, 3971, 4096, 4096, 0, 0, 4075, 0, 0, 4091, 0, 0, 4087, 4096, 4032, 5, 0, 0, 1, 38, 183, 0, 0, 3363, 0, 1522, 10, 1, 4093, 2432, 6, 42, 4080, 1, 6, 0, 0, 3910, 0, 4078, 0, 0, 0, 0, 0, 0, 6, 2, 0, 4013, 4090, 0, 9, 4086, 4065, 4074, 0, 0, 0, 0, 649, 45, 0, 2, 0, 0, 0, 277, 0, 2436, 0, 0, 0, 4030, 1501, 0, 0, 19, 26, 0, 0, 70] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[29, 4047, 1, 0, 4049, 8, 0, 135, 85, 3, 0, 0, 0, 4075, 1565, 4075, 0, 0, 918, 3767, 0, 8, 0, 0, 415, 2772, 16, 0, 22, 4095, 0, 0, 0, 13, 0, 5, 4080, 0, 0, 4073, 4071, 0, 5, 0, 0, 0, 1040, 4066, 4086, 4090, 24, 46, 4062, 0, 5, 4048, 0, 24, 3971, 4086, 4072, 0, 0, 0, 0, 0, 3387, 11, 0, 2543, 51, 0, 0, 0, 4072, 3732, 14, 0, 4068, 1038, 133, 36, 14, 4096, 22, 4086, 44, 1, 8, 11, 2, 0, 0, 0, 17, 3950, 4070, 4, 0, 1120, 4019, 4061, 9, 0, 0, 0, 2165, 0, 0, 0, 13, 3, 4, 0, 0, 0, 27, 14, 24, 4049, 4072, 10, 0, 117, 0, 0, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 36, 640, 0, 14, 3976, 0, 0, 3, 0, 8, 4076, 1170, 4087, 38, 0, 3588, 336, 28, 0, 2, 81, 81, 0, 0, 0, 8, 4094, 0, 6, 0, 1, 0, 0, 4090, 29, 17, 4093, 4096, 0, 0, 0, 4073, 200, 545, 4096, 4096, 4095, 0, 0, 4088, 0, 0, 4096, 0, 0, 4092, 4096, 4070, 0, 0, 7, 0, 1, 3293, 0, 0, 3622, 0, 233, 0, 1, 4096, 3210, 0, 0, 4086, 136, 0, 8, 0, 4096, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 58, 3, 4082, 4079, 0, 0, 389, 4071, 4096, 0, 0, 0, 0, 3751, 0, 0, 64, 0, 0, 0, 12, 0, 7, 3, 0, 0, 4038, 3321, 1, 12, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 11, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4073, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4013, 4096, 0, 0, 7, 3274, 0, 3563, 4096, 4096, 4094, 0, 50, 4096, 16, 0, 4096, 0, 0, 4058, 2695, 4061, 0, 0, 0, 0, 0, 3323, 42, 0, 3988, 3218, 0, 0, 7, 4085, 4096, 0, 0, 4096, 3949, 132, 0, 0, 4096, 0, 4096, 60, 0, 0, 0, 0, 0, 0, 0, 0, 126, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 307, 0, 0, 0, 4096, 4032, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4078, 0, 213, 2719, 12, 67, 2233, 0, 10, 0, 19, 32, 4016, 4089, 4071, 34, 26, 0, 194, 8, 0, 0, 76, 0, 0, 4, 0, 0, 3910, 0, 0, 0, 0, 0, 0, 4060, 0, 358, 4095, 4000, 0, 16, 0, 2005, 156, 3427, 4050, 4075, 3927, 0, 48, 4064, 0, 13, 4091, 0, 0, 4046, 3866, 4032, 0, 12, 4, 5, 51, 2471, 9, 36, 3844, 111, 69, 2, 40, 4061, 3969, 0, 0, 4082, 2054, 780, 0, 14, 4062, 0, 4079, 32, 0, 0, 0, 11, 0, 24, 0, 0, 4055, 4079, 1, 16, 601, 4060, 4080, 16, 0, 0, 0, 1, 0, 9, 0, 6, 0, 35, 287, 85, 3043, 0, 0, 33, 4087, 611, 0, 0, 0, 0, 0, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[16, 2982, 610, 0, 0, 4, 0, 200, 0, 0, 0, 0, 0, 4087, 0, 2957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 2948, 0, 3, 0, 0, 4065, 0, 13, 4070, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4, 0, 631, 0, 0, 0, 3776, 4024, 0, 2, 0, 3260, 0, 2741, 2871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 58, 0, 0, 5, 3747, 0, 7, 3361, 4094, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 528, 0, 0, 3503, 4093, 0, 0, 1223, 3178, 4096, 10, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4095, 0, 4095, 33, 28, 0, 0, 3, 0, 0, 0, 0, 0, 3611, 1206, 302, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1, 0, 2762, 4091, 0, 5, 8, 34, 4083, 0, 3, 0, 0, 18, 827, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 736, 1, 38, 2, 4092, 3968, 0, 0, 31, 13, 4, 3, 0, 4092, 3878, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10856, 19219, 17318, 15807, 20729, 13591, 16626, 16926] avg=16384.0000 max=20729 max_rank=4 min=10856 min_rank=0 imbalance_ratio=1.265198 max_load_reduction=110343 max_load_reduction_pct=0.841850 moved_to_echo=67872 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 105, 23, 0, 0, 0, 0, 0, 0, 5, 0, 1, 4096, 2752, 2964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3942, 0, 15, 0, 0, 102, 1, 0, 0, 0, 0, 4091, 0, 0, 4096, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 4096, 0, 0, 442, 4082, 0, 0, 2884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 56, 0, 4096, 5, 0, 4096, 3449, 463, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4029, 108, 5, 4057, 33, 1, 0, 690, 3356, 4083, 0, 6, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 4078, 0, 0, 5, 0, 0, 0, 0, 2, 0, 1562, 3624, 1212, 303, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 67, 3513, 4096, 0, 0, 0, 0, 3981, 0, 0, 73, 0, 0, 829, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8, 19, 0, 3999, 4044, 0, 1, 68, 0, 0, 0, 0, 4096, 3886, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14042, 16280, 15559, 16146, 20413, 15051, 17389, 16192] avg=16384.0000 max=20413 max_rank=4 min=14042 min_rank=0 imbalance_ratio=1.245911 max_load_reduction=110659 max_load_reduction_pct=0.844261 moved_to_echo=69045 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 19, 4055, 0, 0, 1710, 0, 0, 0, 0, 0, 4096, 4088, 2964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 773, 2, 0, 0, 38, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 79, 4078, 4056, 0, 0, 0, 2360, 0, 2783, 2884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4, 0, 3, 0, 4096, 0, 0, 4033, 4064, 463, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 0, 0, 4096, 60, 213, 0, 4, 3355, 4048, 0, 0, 0, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 4076, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4046, 3621, 1212, 303, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 248, 4096, 4096, 0, 0, 0, 0, 960, 7, 0, 14, 0, 0, 829, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 985, 0, 0, 0, 4078, 4096, 0, 2, 0, 0, 0, 0, 0, 4096, 3883, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21028, 14080, 16252, 16756, 13057, 17596, 15086, 17217] avg=16384.0000 max=21028 max_rank=0 min=13057 min_rank=4 imbalance_ratio=1.283447 max_load_reduction=110044 max_load_reduction_pct=0.839569 moved_to_echo=62956 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 36, 640, 0, 14, 3976, 0, 0, 3, 0, 8, 4076, 1170, 2958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 3588, 336, 28, 0, 2, 81, 81, 0, 0, 0, 8, 4094, 0, 6, 4096, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 29, 17, 4093, 4096, 0, 0, 0, 4073, 200, 545, 2884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 4096, 461, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3293, 0, 0, 3622, 0, 233, 0, 1, 3356, 3210, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 58, 3, 4082, 3609, 1212, 302, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 389, 4071, 4096, 0, 0, 0, 0, 3751, 0, 0, 64, 0, 0, 827, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 7, 3, 0, 0, 4038, 3321, 1, 12, 0, 0, 0, 0, 0, 4090, 3880, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16977, 16380, 15938, 16847, 13852, 17633, 18017, 15428] avg=16384.0000 max=18017 max_rank=6 min=13852 min_rank=4 imbalance_ratio=1.099670 max_load_reduction=113055 max_load_reduction_pct=0.862541 moved_to_echo=64930 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4068, 8, 70, 391, 0, 0, 88, 25, 1, 0, 3, 0, 3928, 4016, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 3437, 33, 12, 0, 0, 14, 435, 0, 9, 0, 4096, 114, 3, 4096, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 13, 0, 0, 13, 1006, 4082, 4059, 0, 16, 6, 3137, 0, 4057, 2792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 0, 0, 4087, 4096, 453, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 38, 183, 0, 0, 3363, 0, 1522, 10, 1, 3350, 2432, 6, 42, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3910, 0, 4078, 0, 0, 0, 0, 0, 0, 6, 2, 0, 4013, 3579, 1179, 304, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 9, 4086, 4065, 4074, 0, 0, 0, 0, 649, 45, 0, 2, 0, 0, 822, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 2436, 0, 0, 0, 4030, 1501, 0, 0, 19, 26, 0, 0, 70, 4054, 3867, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15523, 16370, 19189, 16828, 10949, 17285, 18585, 16343] avg=16384.0000 max=19189 max_rank=2 min=10949 min_rank=4 imbalance_ratio=1.171204 max_load_reduction=111883 max_load_reduction_pct=0.853600 moved_to_echo=63162 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[29, 4047, 1, 0, 4049, 8, 0, 135, 85, 3, 0, 0, 0, 4075, 1565, 2949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 918, 3767, 0, 8, 0, 0, 415, 2772, 16, 0, 22, 4095, 0, 0, 4086, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 5, 0, 0, 0, 4073, 4071, 0, 5, 0, 0, 0, 1040, 2863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 24, 46, 0, 0, 5, 4048, 0, 24, 3971, 4086, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3387, 11, 0, 2543, 51, 0, 0, 0, 3337, 3732, 14, 0, 0, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 36, 14, 4096, 22, 4086, 44, 1, 8, 11, 2, 0, 0, 0, 17, 3950, 3611, 1203, 301, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 4, 0, 1120, 4019, 4061, 9, 0, 0, 0, 2165, 0, 0, 0, 13, 3, 825, 735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 27, 14, 24, 4049, 4072, 10, 0, 117, 0, 0, 3, 0, 4080, 3855, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16946, 20103, 12070, 16755, 14113, 17742, 17024, 16319] avg=16384.0000 max=20103 max_rank=1 min=12070 min_rank=2 imbalance_ratio=1.226990 max_load_reduction=110969 max_load_reduction_pct=0.846626 moved_to_echo=65198 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 11, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4088, 0, 0, 4096, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 4096, 0, 0, 7, 3274, 0, 3563, 2884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 50, 0, 16, 0, 4096, 0, 0, 4058, 2695, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3323, 42, 0, 3988, 3218, 0, 0, 7, 3347, 4096, 0, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 4096, 0, 4096, 60, 0, 0, 0, 0, 0, 0, 0, 0, 126, 3601, 1212, 303, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 829, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 0, 0, 0, 4096, 4032, 0, 0, 0, 0, 0, 0, 0, 4096, 3886, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15266, 16295, 17837, 15469, 21970, 13836, 13919, 16480] avg=16384.0000 max=21970 max_rank=4 min=13836 min_rank=5 imbalance_ratio=1.340942 max_load_reduction=109102 max_load_reduction_pct=0.832382 moved_to_echo=66205 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4078, 0, 213, 2719, 12, 67, 2233, 0, 10, 0, 19, 32, 4016, 4089, 2946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 26, 0, 194, 8, 0, 0, 76, 0, 0, 4, 0, 0, 3910, 0, 0, 4075, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 4095, 4000, 0, 16, 0, 2005, 156, 3427, 2852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 0, 48, 0, 0, 13, 4091, 0, 0, 4046, 3866, 457, 0, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 51, 2471, 9, 36, 3844, 111, 69, 2, 40, 3328, 3969, 0, 0, 0, 2054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 780, 0, 14, 4062, 0, 4079, 32, 0, 0, 0, 11, 0, 24, 0, 0, 4055, 3575, 1198, 301, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 1, 16, 601, 4060, 4080, 16, 0, 0, 0, 1, 0, 9, 0, 6, 0, 824, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 287, 85, 3043, 0, 0, 33, 4087, 611, 0, 0, 0, 0, 0, 5, 0, 4060, 3857, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20434, 12345, 16909, 16464, 15989, 18338, 14426, 16167] avg=16384.0000 max=20434 max_rank=0 min=12345 min_rank=1 imbalance_ratio=1.247192 max_load_reduction=110638 max_load_reduction_pct=0.844101 moved_to_echo=64920 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 640, 128, 7552, 128, 128, 128, 32512, 25728, 128, 128, 256, 128, 128, 128, 128, 32768, 31104, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 18048, 256, 128, 29056, 7680, 2048, 128, 2048, 26624, 29696, 128, 128, 0, 15872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32512, 128, 256, 0, 128, 128, 32384, 128, 128, 31744, 30464, 3712, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 128, 6528, 30720, 32768, 128, 128, 128, 128, 15744, 128, 128, 256, 128, 128, 6656, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 31616, 768, 384, 11904, 128, 128, 8448, 128, 128, 128, 128, 128, 32512, 21888, 23680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 17792, 8576, 128, 128, 128, 384, 3584, 3328, 128, 128, 128, 32640, 128, 128, 32768, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 640, 0, 128, 1536, 32384, 32512, 0, 128, 512, 22272, 384, 18176, 23040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[1152, 128, 128, 32512, 128, 32768, 256, 128, 128, 128, 128, 0, 128, 128, 128, 21888, 28928, 9728, 2432, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 16, 5, 0, 0, 4056, 4078, 0, 0, 0, 4072, 4073, 4070, 0, 2, 4095, 4, 294, 0, 0, 0, 0, 3, 1, 15, 1, 4063, 0, 0, 4089, 0, 3865, 1, 0, 0, 29, 24, 1, 0, 0, 3993, 0, 4087, 0, 2, 261, 3830, 0, 76, 1271, 4094, 12, 4090, 0, 0, 6, 0, 40, 320, 4, 9, 0, 0, 4092, 4080, 4091, 2, 4, 3992, 4093, 0, 0, 1202, 5, 0, 2, 2738, 3, 0, 0, 0, 288, 0, 0, 4095, 2, 47, 0, 8, 4087, 4083, 0, 0, 47, 6, 3, 0, 4085, 0, 4, 4, 0, 40, 0, 3, 22, 4, 0, 4093, 14, 0, 30, 4057, 0, 8, 3907, 4093, 2495, 0, 4093, 4079, 4096, 18, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4045, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 2818, 0, 0, 4096, 0, 4096, 0, 0, 80, 52, 4075, 0, 0, 0, 0, 3, 4095, 0, 0, 0, 4093, 1, 4096, 3764, 4096, 0, 4095, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 729, 4092, 0, 0, 885, 0, 1, 0, 4091, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 1, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 4087, 4092, 4096, 0, 4095, 4096, 4018, 0, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 16, 58, 0, 0, 5, 0, 1, 4079, 4067, 0, 0, 11, 2654, 4082, 4066, 43, 0, 4090, 18, 32, 0, 0, 0, 9, 18, 0, 0, 0, 3985, 20, 0, 4070, 0, 4058, 0, 0, 0, 109, 3260, 0, 0, 10, 89, 8, 4086, 0, 0, 15, 4038, 9, 3955, 4063, 4032, 13, 4079, 0, 0, 0, 0, 1, 113, 0, 0, 19, 0, 2336, 4090, 4073, 0, 0, 3891, 4080, 0, 21, 0, 0, 0, 0, 4013, 0, 0, 3, 0, 19, 0, 142, 4087, 8, 44, 0, 0, 4055, 4077, 0, 7, 0, 3, 6, 26, 3968, 13, 9, 19, 37, 47, 1, 15, 0, 11, 16, 4096, 3, 0, 0, 4063, 0, 0, 4061, 4059, 94, 0, 4096, 4082, 4096, 13, 0, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 347, 0, 51, 0, 4096, 4046, 0, 134, 0, 4039, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 12, 0, 0, 5, 0, 0, 3901, 0, 0, 4096, 0, 4096, 1, 0, 4, 47, 16, 3, 21, 0, 349, 63, 4090, 2, 0, 73, 3777, 0, 4007, 4086, 4096, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 4, 0, 80, 4081, 4047, 0, 0, 4077, 4090, 0, 2, 1618, 0, 0, 0, 4096, 0, 0, 0, 4, 68, 0, 0, 4088, 2, 11, 0, 0, 4090, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 4033, 0, 0, 0, 4096, 0, 0, 4095, 4096, 2212, 0, 4018, 4096, 4000, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 16, 0, 0, 613, 0, 4086, 4093, 83, 0, 0, 154, 4086, 4085, 0, 0, 3681, 1, 11, 12, 0, 20, 60, 0, 0, 0, 7, 3509, 23, 0, 4062, 0, 4083, 0, 0, 0, 249, 4096, 11, 0, 7, 0, 0, 4078, 0, 0, 0, 4092, 1, 4062, 4086, 4047, 0, 4089, 12, 5, 7, 33, 0, 0, 2, 21, 0, 0, 3470, 4094, 4082, 571, 10, 4006, 4084, 0, 11, 3, 6, 0, 0, 4042, 0, 0, 0, 0, 283, 35, 0, 4074, 0, 0, 0, 37, 4089, 4096, 0, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 48, 1, 4, 3, 0, 0, 4071, 0, 10, 9, 3846, 1, 0, 4090, 4075, 377, 0, 4096, 4093, 3664, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[3, 0, 21, 10, 0, 26, 483, 0, 4096, 4076, 0, 0, 0, 129, 4077, 4083, 0, 0, 4083, 8, 0, 0, 18, 0, 17, 0, 0, 23, 6, 0, 3, 51, 4060, 0, 4066, 0, 20, 30, 1299, 3826, 0, 0, 0, 13, 116, 4063, 0, 0, 0, 4061, 0, 3928, 4078, 4048, 177, 4087, 10, 0, 30, 40, 0, 0, 0, 50, 98, 15, 4089, 4009, 4083, 252, 0, 2749, 4072, 0, 0, 61, 35, 15, 0, 4067, 19, 44, 56, 42, 79, 16, 0, 3911, 14, 2, 1, 0, 4037, 4029, 0, 0, 0, 0, 11, 0, 3993, 0, 17, 16, 0, 31, 0, 0, 0, 10, 3, 4065, 0, 36, 0, 4094, 5, 1, 4045, 4045, 4021, 6, 4072, 4080, 3511, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 38, 0, 34, 0, 0, 0, 4096, 4096, 0, 17, 0, 582, 4096, 4096, 0, 5, 3972, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 2, 108, 0, 4004, 0, 2, 0, 155, 0, 4088, 112, 0, 0, 4080, 0, 4096, 4095, 4084, 0, 4096, 0, 10, 0, 1, 0, 67, 0, 0, 27, 0, 572, 4055, 4094, 0, 0, 3858, 4090, 0, 11, 2995, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 6, 0, 1, 4096, 4096, 5, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 4010, 4092, 4096, 0, 4088, 4093, 4038, 0, 64, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 0, 4096, 4042, 0, 48, 0, 3604, 4082, 4096, 0, 0, 4096, 0, 16, 0, 0, 17, 0, 0, 7, 0, 0, 4096, 0, 0, 4096, 0, 4062, 0, 0, 0, 0, 506, 0, 0, 0, 879, 0, 4096, 0, 0, 5, 4069, 0, 3271, 17, 4058, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 4070, 0, 0, 4087, 4096, 0, 13, 33, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 41, 4096, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4060, 0, 4091, 4096, 4081, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 16, 5, 0, 0, 1964, 4078, 0, 0, 0, 4072, 4073, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 4, 294, 0, 0, 0, 0, 3, 1, 15, 1, 4063, 0, 0, 4079, 4093, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 3865, 1, 0, 0, 29, 24, 1, 0, 0, 3993, 0, 4087, 0, 2, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 3830, 0, 76, 1271, 4094, 12, 1522, 0, 0, 6, 0, 40, 320, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4080, 1142, 2, 4, 3992, 0, 0, 0, 1202, 5, 0, 2, 2738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 288, 0, 0, 4095, 2, 47, 0, 8, 4087, 4083, 0, 0, 2457, 1394, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 6, 3, 0, 4085, 0, 4, 4, 0, 40, 0, 3, 22, 4, 0, 4093, 2949, 2807, 2440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 30, 4057, 0, 8, 3907, 1636, 2495, 0, 0, 0, 4096, 18, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18279, 17936, 16789, 11445, 17259, 16592, 16507, 16265] avg=16384.0000 max=18279 max_rank=0 min=11445 min_rank=3 imbalance_ratio=1.115662 max_load_reduction=112793 max_load_reduction_pct=0.860542 moved_to_echo=66623 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 1984, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 2818, 0, 0, 4096, 4092, 1285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 80, 52, 4075, 0, 0, 0, 0, 3, 4095, 0, 0, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 4096, 3764, 4096, 0, 1524, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 1143, 0, 0, 729, 0, 0, 0, 885, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 1, 1, 0, 4096, 4096, 0, 0, 2456, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4094, 2953, 2810, 2443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 1636, 4096, 0, 0, 0, 4018, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14273, 16358, 17201, 17578, 15040, 16283, 16403, 17936] avg=16384.0000 max=17936 max_rank=7 min=14273 min_rank=0 imbalance_ratio=1.094727 max_load_reduction=113136 max_load_reduction_pct=0.863159 moved_to_echo=65662 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 347, 0, 51, 0, 1984, 4046, 0, 134, 0, 4039, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 12, 0, 0, 5, 0, 0, 3901, 0, 0, 4096, 4090, 1262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 4, 47, 16, 3, 21, 0, 349, 63, 4090, 2, 0, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 3777, 0, 4007, 4086, 4096, 0, 1524, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 80, 4081, 1129, 0, 0, 4077, 0, 0, 2, 1618, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 68, 0, 0, 4088, 2, 11, 0, 0, 4090, 4095, 0, 0, 2459, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 4033, 2918, 2756, 2442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 1637, 2212, 0, 0, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18793, 17462, 13492, 17565, 15087, 16353, 16280, 16040] avg=16384.0000 max=18793 max_rank=0 min=13492 min_rank=2 imbalance_ratio=1.147034 max_load_reduction=112279 max_load_reduction_pct=0.856621 moved_to_echo=63760 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 16, 58, 0, 0, 5, 0, 1, 1975, 4067, 0, 0, 11, 2654, 4082, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 4090, 18, 32, 0, 0, 0, 9, 18, 0, 0, 0, 3985, 20, 0, 4082, 4080, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 4058, 0, 0, 0, 109, 3260, 0, 0, 10, 89, 8, 4086, 0, 0, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4038, 9, 3955, 4063, 4032, 13, 1518, 0, 0, 0, 0, 1, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 2336, 4090, 1137, 0, 0, 3891, 0, 0, 21, 0, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 19, 0, 142, 4087, 8, 44, 0, 0, 4055, 4077, 0, 7, 2437, 1402, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6, 26, 3968, 13, 9, 19, 37, 47, 1, 15, 0, 11, 16, 4096, 2936, 2810, 2433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4063, 0, 0, 4061, 1622, 94, 0, 0, 0, 4096, 13, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16935, 17663, 16392, 17757, 15507, 16409, 16446, 13963] avg=16384.0000 max=17757 max_rank=3 min=13963 min_rank=7 imbalance_ratio=1.083801 max_load_reduction=113315 max_load_reduction_pct=0.864525 moved_to_echo=62325 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 38, 0, 34, 0, 0, 0, 1984, 4096, 0, 17, 0, 582, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3972, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4090, 1283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 2, 108, 0, 4004, 0, 2, 0, 155, 0, 4088, 112, 0, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 4096, 4095, 4084, 0, 1524, 0, 10, 0, 1, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 572, 4055, 1142, 0, 0, 3858, 0, 0, 11, 2995, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 6, 0, 1, 4096, 4096, 5, 0, 2456, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 4094, 2952, 2805, 2444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4010, 1636, 4096, 0, 0, 0, 4038, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14943, 13517, 17364, 17957, 16759, 16281, 16311, 17940] avg=16384.0000 max=17957 max_rank=3 min=13517 min_rank=1 imbalance_ratio=1.096008 max_load_reduction=113115 max_load_reduction_pct=0.862999 moved_to_echo=67291 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 16, 0, 0, 613, 0, 1968, 4093, 83, 0, 0, 154, 4086, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3681, 1, 11, 12, 0, 20, 60, 0, 0, 0, 7, 3509, 23, 0, 4093, 4084, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 4083, 0, 0, 0, 249, 4096, 11, 0, 7, 0, 0, 4078, 0, 0, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 4062, 4086, 4047, 0, 1515, 12, 5, 7, 33, 0, 0, 2, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3470, 4094, 1135, 571, 10, 4006, 0, 0, 11, 3, 6, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 35, 0, 4074, 0, 0, 0, 37, 4089, 4096, 0, 6, 2451, 1410, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 48, 1, 4, 3, 0, 0, 4071, 2947, 2809, 2443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 9, 3846, 1, 0, 4090, 1624, 377, 0, 0, 0, 3664, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15098, 16788, 17294, 17883, 17348, 16612, 16427, 13622] avg=16384.0000 max=17883 max_rank=3 min=13622 min_rank=7 imbalance_ratio=1.091492 max_load_reduction=113189 max_load_reduction_pct=0.863564 moved_to_echo=64009 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 0, 1984, 4042, 0, 48, 0, 3604, 4082, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 17, 0, 0, 7, 0, 0, 4096, 0, 0, 4096, 4096, 1285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4062, 0, 0, 0, 0, 506, 0, 0, 0, 879, 0, 4096, 0, 0, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4069, 0, 3271, 17, 4058, 0, 1524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 1136, 0, 0, 4087, 0, 0, 13, 33, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 41, 4096, 0, 0, 0, 0, 4096, 4096, 2, 0, 2459, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2934, 2806, 2444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1637, 4060, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17859, 17709, 14343, 12944, 17543, 16328, 16376, 17970] avg=16384.0000 max=17970 max_rank=7 min=12944 min_rank=3 imbalance_ratio=1.096802 max_load_reduction=113102 max_load_reduction_pct=0.862900 moved_to_echo=68217 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 21, 10, 0, 26, 483, 0, 1984, 4076, 0, 0, 0, 129, 4077, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 8, 0, 0, 18, 0, 17, 0, 0, 23, 6, 0, 3, 51, 4080, 4072, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 4066, 0, 20, 30, 1299, 3826, 0, 0, 0, 13, 116, 4063, 0, 0, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 3928, 4078, 4048, 177, 1521, 10, 0, 30, 40, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 15, 4089, 4009, 1139, 252, 0, 2749, 0, 0, 0, 61, 35, 15, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 44, 56, 42, 79, 16, 0, 3911, 14, 2, 1, 0, 4037, 4029, 0, 0, 2428, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 3993, 0, 17, 16, 0, 31, 0, 0, 0, 10, 3, 4065, 2944, 2794, 2438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 4094, 5, 1, 4045, 1617, 4021, 6, 0, 0, 3511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14892, 13639, 18197, 17943, 16529, 16214, 16322, 17336] avg=16384.0000 max=18197 max_rank=2 min=13639 min_rank=1 imbalance_ratio=1.110657 max_load_reduction=112875 max_load_reduction_pct=0.861168 moved_to_echo=66401 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 512, 128, 1152, 128, 15872, 32640, 128, 256, 128, 15360, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 128, 22912, 32640, 9216, 896, 128, 27392, 0, 0, 128, 6912, 128, 128, 128, 31360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 0, 32512, 128, 128, 256, 1792, 19840, 128, 128, 128, 5504, 256, 32768, 128, 128, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 32256, 128, 384, 128, 128, 256, 128, 128, 128, 128, 128, 22400, 128, 128, 32768, 32768, 10368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 32512, 128, 128, 128, 128, 256, 128, 128, 128, 128, 128, 32768, 23552, 22400, 19584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 32128, 128, 27520, 25472, 32640, 256, 12288, 128, 128, 128, 128, 128, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 32512, 128, 128, 32512, 13056, 21504, 128, 0, 0, 31616, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 768, 128, 256, 32640, 128, 128, 128, 128, 32768, 32768, 128, 128, 19712, 11264, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[15, 1648, 1740, 19, 2436, 0, 749, 29, 926, 58, 2224, 405, 1340, 353, 6, 350, 292, 1321, 1180, 1312, 2911, 818, 716, 253, 21, 2643, 170, 337, 795, 62, 888, 1524, 3328, 9, 77, 705, 708, 62, 712, 1053, 2629, 2858, 563, 286, 238, 1288, 1230, 786, 303, 289, 1266, 156, 244, 249, 3283, 114, 39, 1090, 3261, 2257, 182, 240, 190, 43, 943, 3485, 56, 1843, 572, 203, 1694, 2648, 330, 1048, 186, 45, 827, 2571, 2158, 1578, 2056, 796, 609, 3059, 566, 658, 1385, 3695, 188, 404, 1001, 546, 2214, 349, 960, 46, 1264, 386, 3560, 711, 20, 942, 1457, 2208, 2968, 2079, 477, 834, 668, 824, 488, 2102, 297, 1155, 236, 996, 12, 1052, 413, 2194, 718, 1095, 2045, 1448, 1476, 4, 844, 101] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[24, 1559, 2284, 5, 2136, 0, 719, 14, 757, 67, 2285, 304, 791, 591, 7, 269, 113, 1719, 1105, 1505, 3136, 408, 962, 227, 11, 2966, 170, 574, 872, 171, 779, 2264, 3451, 10, 68, 405, 816, 66, 570, 920, 2703, 3108, 535, 333, 226, 788, 1537, 1565, 337, 234, 995, 429, 190, 406, 2912, 119, 30, 1062, 3083, 2763, 156, 336, 243, 54, 1102, 3208, 20, 968, 361, 228, 1984, 2163, 425, 787, 443, 66, 833, 2617, 2317, 1077, 1428, 598, 1046, 3441, 514, 1072, 1196, 3712, 79, 264, 897, 1060, 1702, 398, 655, 54, 1510, 238, 2855, 562, 10, 1257, 1664, 1996, 2850, 2443, 395, 1093, 484, 719, 455, 2023, 132, 1165, 516, 855, 70, 1943, 894, 1670, 764, 1269, 2126, 1085, 1245, 19, 783, 23] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[19, 1485, 1997, 18, 2490, 0, 1044, 15, 982, 143, 2315, 244, 639, 484, 5, 228, 139, 1967, 758, 1117, 3168, 608, 869, 190, 12, 2736, 146, 462, 448, 66, 948, 2349, 3373, 50, 48, 1290, 496, 78, 691, 941, 2695, 3036, 492, 253, 225, 1238, 1987, 587, 283, 282, 1268, 292, 343, 490, 2913, 84, 58, 1090, 3408, 2853, 150, 297, 165, 68, 609, 3479, 37, 1243, 718, 182, 1515, 2611, 513, 1143, 428, 42, 899, 2645, 2423, 1723, 1858, 531, 1069, 2950, 497, 586, 1180, 3629, 127, 254, 737, 654, 2391, 678, 682, 48, 1221, 226, 3306, 748, 11, 954, 1492, 2076, 2659, 2047, 368, 1013, 602, 561, 609, 2593, 332, 973, 512, 1052, 21, 792, 537, 1483, 897, 1126, 2130, 784, 1388, 43, 811, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[11, 1290, 1810, 63, 2468, 2, 914, 21, 854, 41, 1903, 166, 677, 412, 11, 261, 190, 1624, 980, 1310, 2987, 572, 707, 409, 7, 2498, 214, 356, 267, 56, 883, 2223, 3502, 126, 110, 982, 353, 137, 750, 1078, 2812, 2314, 437, 88, 267, 1268, 1875, 1503, 393, 158, 1068, 263, 97, 251, 3191, 151, 38, 819, 3450, 2766, 202, 224, 357, 41, 632, 3385, 91, 1937, 811, 113, 2288, 2444, 390, 1221, 358, 41, 1020, 2432, 2120, 1423, 1121, 625, 1433, 2652, 434, 470, 1404, 3515, 122, 426, 992, 978, 1897, 520, 760, 37, 1316, 349, 3290, 589, 35, 1321, 2013, 1970, 2942, 1914, 276, 1157, 554, 776, 404, 2840, 373, 1234, 702, 627, 26, 1153, 664, 1458, 558, 1173, 2635, 745, 1358, 32, 1207, 31] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[24, 1260, 1961, 13, 2169, 21, 854, 32, 483, 178, 2178, 429, 670, 411, 28, 271, 156, 1614, 1210, 1425, 2906, 505, 644, 298, 23, 2175, 181, 499, 1189, 171, 920, 2764, 3408, 92, 185, 1189, 1012, 106, 460, 1415, 2701, 2765, 590, 281, 316, 1521, 1829, 1075, 343, 378, 757, 337, 151, 328, 2961, 86, 26, 1075, 2721, 2307, 240, 373, 623, 83, 1074, 3034, 74, 1314, 513, 174, 965, 2442, 197, 921, 608, 108, 1005, 2217, 2468, 808, 1955, 578, 1051, 3028, 545, 762, 1742, 3323, 123, 412, 815, 843, 1641, 585, 532, 37, 1590, 416, 3137, 427, 35, 1335, 1283, 1980, 2833, 1620, 459, 947, 782, 992, 587, 2839, 312, 1483, 337, 750, 49, 1426, 557, 1125, 965, 1331, 2352, 1142, 1421, 12, 1219, 39] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[17, 1226, 1799, 17, 2183, 1, 847, 27, 1290, 145, 2537, 379, 826, 705, 13, 305, 128, 1096, 1635, 1047, 3063, 892, 661, 520, 3, 2463, 174, 510, 992, 90, 1230, 1919, 3249, 30, 64, 1172, 966, 53, 443, 1156, 2502, 3426, 976, 507, 209, 1357, 1095, 425, 341, 157, 1139, 550, 509, 274, 3088, 197, 6, 1622, 3302, 2053, 540, 117, 454, 34, 663, 3247, 122, 1361, 441, 248, 1232, 1860, 308, 1331, 475, 85, 700, 2714, 2009, 1134, 1460, 816, 874, 2815, 410, 932, 1335, 3628, 181, 266, 509, 665, 1732, 426, 400, 114, 1753, 642, 2799, 655, 35, 876, 2062, 2351, 3106, 1898, 333, 1221, 566, 1274, 374, 1814, 161, 1624, 366, 1057, 42, 1199, 1172, 1719, 519, 936, 1931, 890, 1414, 27, 974, 36] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[5, 994, 1825, 5, 2284, 0, 820, 27, 462, 147, 1871, 502, 925, 370, 16, 288, 196, 1450, 1069, 1504, 2994, 689, 656, 216, 2, 2552, 212, 385, 1102, 112, 1009, 2226, 3597, 24, 72, 770, 602, 89, 832, 885, 2698, 2903, 651, 240, 250, 1312, 1739, 1334, 291, 132, 907, 408, 147, 127, 3547, 179, 3, 1231, 2842, 2245, 181, 327, 508, 19, 953, 3331, 54, 1489, 590, 90, 1645, 2278, 259, 666, 316, 106, 723, 2309, 2775, 1344, 1156, 827, 1050, 3624, 559, 669, 1401, 3483, 94, 194, 600, 1176, 1976, 484, 1388, 50, 1742, 427, 3293, 588, 11, 1068, 1234, 1631, 2896, 1771, 362, 964, 266, 743, 471, 2438, 176, 1780, 477, 855, 28, 2013, 559, 1536, 1010, 946, 2326, 1071, 1657, 21, 1043, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[17, 1307, 2382, 9, 2446, 5, 744, 44, 443, 157, 1912, 473, 901, 399, 35, 295, 177, 1635, 1093, 1486, 2889, 414, 655, 136, 47, 2347, 299, 457, 857, 201, 618, 2385, 3578, 24, 70, 876, 916, 92, 857, 1188, 2750, 2622, 848, 206, 178, 1360, 2273, 989, 380, 304, 787, 226, 175, 532, 3312, 155, 13, 1093, 2728, 2522, 202, 501, 172, 53, 616, 3569, 52, 1017, 553, 255, 1257, 2346, 231, 845, 760, 110, 888, 1672, 2323, 1217, 1575, 482, 1104, 3033, 426, 667, 1944, 3709, 303, 263, 756, 890, 1926, 391, 656, 92, 882, 393, 3560, 706, 77, 1462, 1599, 1752, 3148, 1615, 395, 1335, 860, 624, 522, 2513, 268, 1328, 617, 763, 90, 1455, 483, 1834, 1058, 1366, 2372, 687, 1131, 19, 975, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[15, 1648, 1740, 19, 2436, 0, 749, 29, 926, 58, 2224, 405, 1340, 353, 6, 350, 3560, 1660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 1321, 1180, 1312, 2911, 818, 716, 253, 21, 2643, 170, 337, 795, 62, 888, 1524, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1992, 9, 77, 705, 708, 62, 712, 1053, 2629, 2858, 563, 286, 238, 1288, 1230, 786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 289, 1266, 156, 244, 249, 3283, 114, 39, 1090, 3261, 2257, 182, 240, 190, 43, 1585, 1092, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 943, 733, 56, 1843, 572, 203, 1694, 2648, 330, 1048, 186, 45, 827, 2571, 2158, 1578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2056, 796, 609, 3059, 566, 658, 1385, 2110, 188, 404, 1001, 546, 2214, 349, 960, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1264, 386, 0, 711, 20, 942, 1457, 2208, 1539, 2079, 477, 834, 668, 824, 488, 2102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 1155, 236, 996, 12, 1052, 413, 2194, 718, 1095, 2045, 1448, 1476, 4, 844, 101, 1045, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17518, 15534, 15196, 16249, 17435, 16947, 15999, 16194] avg=16384.0000 max=17518 max_rank=0 min=15196 min_rank=2 imbalance_ratio=1.069214 max_load_reduction=113554 max_load_reduction_pct=0.866348 moved_to_echo=66575 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[24, 1559, 2284, 5, 2136, 0, 719, 14, 757, 67, 2285, 304, 791, 591, 7, 269, 2855, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 1719, 1105, 1505, 3136, 408, 962, 227, 11, 2966, 170, 574, 872, 171, 779, 2264, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2065, 10, 68, 405, 816, 66, 570, 920, 2703, 3108, 535, 333, 226, 788, 1537, 1565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 234, 995, 429, 190, 406, 2912, 119, 30, 1062, 3083, 2763, 156, 336, 243, 54, 1593, 1005, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1102, 675, 20, 968, 361, 228, 1984, 2163, 425, 787, 443, 66, 833, 2617, 2317, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1428, 598, 1046, 3441, 514, 1072, 1196, 2119, 79, 264, 897, 1060, 1702, 398, 655, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1510, 238, 0, 562, 10, 1257, 1664, 1996, 1478, 2443, 395, 1093, 484, 719, 455, 2023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 1165, 516, 855, 70, 1943, 894, 1670, 764, 1269, 2126, 1085, 1245, 19, 783, 23, 1084, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16195, 17284, 15715, 16298, 16066, 16523, 16327, 16664] avg=16384.0000 max=17284 max_rank=1 min=15715 min_rank=2 imbalance_ratio=1.054932 max_load_reduction=113788 max_load_reduction_pct=0.868134 moved_to_echo=65580 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[19, 1485, 1997, 18, 2490, 0, 1044, 15, 982, 143, 2315, 244, 639, 484, 5, 228, 3306, 1657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 1967, 758, 1117, 3168, 608, 869, 190, 12, 2736, 146, 462, 448, 66, 948, 2349, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2019, 50, 48, 1290, 496, 78, 691, 941, 2695, 3036, 492, 253, 225, 1238, 1987, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 282, 1268, 292, 343, 490, 2913, 84, 58, 1090, 3408, 2853, 150, 297, 165, 68, 1557, 1090, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 609, 732, 37, 1243, 718, 182, 1515, 2611, 513, 1143, 428, 42, 899, 2645, 2423, 1723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1858, 531, 1069, 2950, 497, 586, 1180, 2072, 127, 254, 737, 654, 2391, 678, 682, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1221, 226, 0, 748, 11, 954, 1492, 2076, 1379, 2047, 368, 1013, 602, 561, 609, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, 973, 512, 1052, 21, 792, 537, 1483, 897, 1126, 2130, 784, 1388, 43, 811, 9, 1059, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17071, 16278, 16126, 17019, 17463, 16314, 15900, 14901] avg=16384.0000 max=17463 max_rank=4 min=14901 min_rank=7 imbalance_ratio=1.065857 max_load_reduction=113609 max_load_reduction_pct=0.866768 moved_to_echo=64578 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[17, 1226, 1799, 17, 2183, 1, 847, 27, 1290, 145, 2537, 379, 826, 705, 13, 305, 2799, 1547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 1096, 1635, 1047, 3063, 892, 661, 520, 3, 2463, 174, 510, 992, 90, 1230, 1919, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1945, 30, 64, 1172, 966, 53, 443, 1156, 2502, 3426, 976, 507, 209, 1357, 1095, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 157, 1139, 550, 509, 274, 3088, 197, 6, 1622, 3302, 2053, 540, 117, 454, 34, 1557, 1017, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 663, 683, 122, 1361, 441, 248, 1232, 1860, 308, 1331, 475, 85, 700, 2714, 2009, 1134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1460, 816, 874, 2815, 410, 932, 1335, 2071, 181, 266, 509, 665, 1732, 426, 400, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1753, 642, 0, 655, 35, 876, 2062, 2351, 1611, 1898, 333, 1221, 566, 1274, 374, 1814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 1624, 366, 1057, 42, 1199, 1172, 1719, 519, 936, 1931, 890, 1414, 27, 974, 36, 1020, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16663, 16707, 16326, 17340, 15366, 15006, 17465, 16199] avg=16384.0000 max=17465 max_rank=6 min=15006 min_rank=5 imbalance_ratio=1.065979 max_load_reduction=113607 max_load_reduction_pct=0.866753 moved_to_echo=64036 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 1290, 1810, 63, 2468, 2, 914, 21, 854, 41, 1903, 166, 677, 412, 11, 261, 3290, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 1624, 980, 1310, 2987, 572, 707, 409, 7, 2498, 214, 356, 267, 56, 883, 2223, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2096, 126, 110, 982, 353, 137, 750, 1078, 2812, 2314, 437, 88, 267, 1268, 1875, 1503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 158, 1068, 263, 97, 251, 3191, 151, 38, 819, 3450, 2766, 202, 224, 357, 41, 1508, 1060, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 632, 713, 91, 1937, 811, 113, 2288, 2444, 390, 1221, 358, 41, 1020, 2432, 2120, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1121, 625, 1433, 2652, 434, 470, 1404, 2007, 122, 426, 992, 978, 1897, 520, 760, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1316, 349, 0, 589, 35, 1321, 2013, 1970, 1526, 1914, 276, 1157, 554, 776, 404, 2840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 1234, 702, 627, 26, 1153, 664, 1458, 558, 1173, 2635, 745, 1358, 32, 1207, 31, 1100, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15806, 15589, 16196, 16399, 18034, 15878, 17040, 16130] avg=16384.0000 max=18034 max_rank=4 min=15589 min_rank=1 imbalance_ratio=1.100708 max_load_reduction=113038 max_load_reduction_pct=0.862411 moved_to_echo=67082 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[17, 1307, 2382, 9, 2446, 5, 744, 44, 443, 157, 1912, 473, 901, 399, 35, 295, 3560, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 1635, 1093, 1486, 2889, 414, 655, 136, 47, 2347, 299, 457, 857, 201, 618, 2385, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2141, 24, 70, 876, 916, 92, 857, 1188, 2750, 2622, 848, 206, 178, 1360, 2273, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 304, 787, 226, 175, 532, 3312, 155, 13, 1093, 2728, 2522, 202, 501, 172, 53, 1591, 1118, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 751, 52, 1017, 553, 255, 1257, 2346, 231, 845, 760, 110, 888, 1672, 2323, 1217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1575, 482, 1104, 3033, 426, 667, 1944, 2118, 303, 263, 756, 890, 1926, 391, 656, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 882, 393, 0, 706, 77, 1462, 1599, 1752, 1633, 1615, 395, 1335, 860, 624, 522, 2513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 268, 1328, 617, 763, 90, 1455, 483, 1834, 1058, 1366, 2372, 687, 1131, 19, 975, 8, 1124, 1127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16829, 16009, 17390, 16252, 14893, 16626, 16368, 16705] avg=16384.0000 max=17390 max_rank=2 min=14893 min_rank=4 imbalance_ratio=1.061401 max_load_reduction=113682 max_load_reduction_pct=0.867325 moved_to_echo=64592 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[24, 1260, 1961, 13, 2169, 21, 854, 32, 483, 178, 2178, 429, 670, 411, 28, 271, 3137, 1450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 1614, 1210, 1425, 2906, 505, 644, 298, 23, 2175, 181, 499, 1189, 171, 920, 2764, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2033, 92, 185, 1189, 1012, 106, 460, 1415, 2701, 2765, 590, 281, 316, 1521, 1829, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 378, 757, 337, 151, 328, 2961, 86, 26, 1075, 2721, 2307, 240, 373, 623, 83, 1431, 955, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 629, 74, 1314, 513, 174, 965, 2442, 197, 921, 608, 108, 1005, 2217, 2468, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1955, 578, 1051, 3028, 545, 762, 1742, 1892, 123, 412, 815, 843, 1641, 585, 532, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1590, 416, 0, 427, 35, 1335, 1283, 1980, 1463, 1620, 459, 947, 782, 992, 587, 2839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 1483, 337, 750, 49, 1426, 557, 1125, 965, 1331, 2352, 1142, 1421, 12, 1219, 39, 1073, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15569, 16982, 17570, 15527, 15517, 16541, 16755, 16611] avg=16384.0000 max=17570 max_rank=2 min=15517 min_rank=4 imbalance_ratio=1.072388 max_load_reduction=113502 max_load_reduction_pct=0.865952 moved_to_echo=65424 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 994, 1825, 5, 2284, 0, 820, 27, 462, 147, 1871, 502, 925, 370, 16, 288, 3293, 1587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 1450, 1069, 1504, 2994, 689, 656, 216, 2, 2552, 212, 385, 1102, 112, 1009, 2226, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2152, 24, 72, 770, 602, 89, 832, 885, 2698, 2903, 651, 240, 250, 1312, 1739, 1334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 132, 907, 408, 147, 127, 3547, 179, 3, 1231, 2842, 2245, 181, 327, 508, 19, 1494, 1043, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 701, 54, 1489, 590, 90, 1645, 2278, 259, 666, 316, 106, 723, 2309, 2775, 1344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1156, 827, 1050, 3624, 559, 669, 1401, 1989, 94, 194, 600, 1176, 1976, 484, 1388, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 427, 0, 588, 11, 1068, 1234, 1631, 1502, 1771, 362, 964, 266, 743, 471, 2438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 1780, 477, 855, 28, 2013, 559, 1536, 1010, 946, 2326, 1071, 1657, 21, 1043, 3, 1130, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15421, 16689, 16553, 15988, 16298, 17237, 15218, 17668] avg=16384.0000 max=17668 max_rank=7 min=15218 min_rank=6 imbalance_ratio=1.078369 max_load_reduction=113404 max_load_reduction_pct=0.865204 moved_to_echo=66421 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12672, 5376, 8320, 24704, 3968, 5888, 11648, 16384, 1280, 2560, 6400, 6912, 15488, 3840, 6144, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[11392, 3200, 0, 4992, 256, 9216, 12928, 16000, 12160, 15488, 3072, 8576, 4864, 6528, 3968, 19200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2176, 10752, 3840, 7040, 384, 11136, 5376, 13056, 6528, 9344, 17920, 7936, 11136, 256, 7936, 256, 8704, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[16512, 384, 768, 7424, 5888, 768, 5376, 8704, 21504, 23040, 5120, 2304, 1920, 10240, 13568, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2688, 2048, 8192, 2688, 1920, 2688, 25216, 1152, 256, 9088, 24832, 19840, 1920, 2432, 2816, 512, 12416, 8448, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 10880, 15872, 256, 18688, 128, 6784, 256, 6272, 1024, 17280, 2944, 6784, 3840, 128, 2304, 25856, 12800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[6656, 5632, 512, 11264, 4608, 1536, 12672, 18816, 2688, 8064, 3584, 640, 6912, 19200, 18688, 10368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[1408, 12544, 9088, 10752, 24064, 4992, 5888, 2304, 128, 20480, 1664, 3584, 6528, 1024, 7296, 17664, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 2, 3683, 6, 9, 1, 1575, 38, 55, 156, 0, 9, 0, 4064, 409, 2, 228, 2, 6, 2715, 3498, 3213, 16, 0, 192, 0, 4083, 0, 0, 4085, 5, 0, 1, 0, 2, 0, 4088, 0, 0, 0, 8, 3914, 3567, 0, 0, 13, 0, 16, 1, 3449, 4088, 1, 0, 0, 2, 0, 4091, 23, 0, 7, 4093, 0, 0, 0, 2931, 2486, 0, 4094, 5, 4071, 0, 3012, 28, 3986, 7, 1032, 0, 4066, 4068, 0, 3, 0, 2, 11, 4096, 4091, 2201, 0, 4054, 0, 3, 6, 20, 0, 4069, 2, 3, 4095, 4091, 0, 968, 119, 0, 0, 0, 4, 0, 26, 0, 0, 0, 0, 4092, 0, 0, 7, 0, 1, 1663, 0, 0, 4036, 15, 4080, 0, 4096, 0, 16] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 421, 4, 84, 0, 3565, 52, 2, 0, 0, 58, 0, 3940, 3750, 0, 1081, 0, 8, 778, 2898, 838, 6, 1, 736, 0, 3957, 2, 0, 4076, 0, 0, 44, 0, 2, 0, 3800, 13, 37, 1, 0, 2733, 3482, 0, 0, 0, 0, 1, 0, 3956, 4032, 0, 2, 1, 0, 0, 4088, 28, 190, 0, 3843, 7, 11, 0, 2275, 3782, 0, 4094, 0, 3950, 0, 3859, 30, 3806, 102, 26, 0, 4084, 4034, 0, 0, 184, 1, 6, 4095, 4089, 3790, 0, 4093, 0, 0, 0, 0, 0, 4089, 2, 2, 4096, 4094, 0, 882, 1, 0, 0, 0, 76, 0, 0, 230, 0, 0, 23, 3432, 0, 0, 1, 0, 7, 3019, 0, 0, 4011, 85, 4096, 1, 4096, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3258, 0, 72, 1, 810, 246, 161, 674, 0, 108, 0, 3921, 512, 6, 93, 0, 0, 1174, 3941, 3361, 0, 0, 10, 0, 4089, 0, 0, 4093, 0, 211, 520, 0, 0, 0, 3958, 51, 353, 0, 82, 2988, 4022, 0, 0, 7, 0, 148, 1, 4095, 4094, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 3764, 0, 1, 48, 2926, 2980, 0, 4096, 0, 4068, 0, 3735, 3, 4076, 38, 2, 0, 3522, 4090, 0, 0, 135, 0, 0, 4096, 4096, 3616, 0, 3925, 0, 0, 0, 0, 0, 4096, 0, 29, 4096, 4096, 0, 1103, 6, 0, 0, 0, 361, 0, 0, 8, 0, 0, 16, 3770, 0, 0, 0, 0, 0, 346, 0, 27, 4092, 2, 4096, 0, 4096, 0, 450] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2, 59, 794, 2, 2, 7, 39, 42, 0, 1074, 0, 6, 0, 4062, 1139, 24, 952, 32, 16, 304, 4084, 4053, 82, 6, 17, 0, 4086, 0, 4, 3666, 3, 1, 278, 18, 95, 0, 3712, 11, 0, 4, 39, 3129, 3761, 0, 9, 33, 2, 35, 0, 3902, 3992, 13, 0, 1, 50, 0, 4086, 0, 0, 12, 4032, 0, 13, 13, 3940, 2184, 0, 4058, 0, 4052, 28, 3955, 18, 4076, 396, 702, 6, 4078, 3831, 26, 0, 50, 0, 209, 4074, 4069, 2640, 21, 4045, 2, 0, 5, 0, 8, 4077, 1, 47, 4074, 4086, 0, 2180, 372, 0, 0, 0, 158, 0, 57, 157, 0, 0, 0, 4087, 15, 3, 0, 19, 263, 882, 26, 0, 4034, 11, 4076, 0, 4095, 9, 72] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1443, 1, 981, 2, 3283, 790, 26, 837, 0, 1245, 42, 4096, 420, 0, 672, 0, 2, 1829, 4011, 1329, 0, 0, 198, 0, 4078, 0, 0, 4086, 10, 3, 30, 2, 0, 0, 4079, 102, 0, 0, 0, 641, 3684, 0, 9, 0, 3, 9, 0, 4061, 4030, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 2751, 3, 1, 277, 3030, 3698, 0, 4096, 0, 3290, 0, 2514, 0, 4078, 53, 1439, 0, 3770, 4096, 0, 0, 5, 2, 0, 4095, 4094, 3731, 19, 4052, 0, 0, 1, 26, 0, 3975, 0, 0, 4083, 4074, 1, 2788, 91, 0, 1, 0, 144, 10, 0, 0, 0, 0, 0, 3763, 0, 0, 3, 0, 32, 572, 0, 1, 4080, 1, 4084, 0, 4093, 0, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1654, 0, 23, 0, 2601, 1, 1, 127, 0, 90, 0, 3512, 2481, 36, 3278, 0, 0, 144, 3175, 4036, 0, 0, 55, 0, 4046, 0, 0, 4014, 0, 1, 9, 4, 0, 0, 4087, 192, 0, 0, 27, 4053, 4093, 0, 0, 23, 0, 3, 2, 3966, 4092, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 2303, 0, 1, 638, 3809, 3899, 0, 4096, 0, 3242, 0, 3881, 13, 4073, 1004, 137, 0, 4045, 4095, 0, 0, 2, 0, 11, 4096, 4096, 784, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 4096, 0, 119, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 122, 0, 0, 4050, 0, 4096, 0, 4096, 0, 76] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 368, 0, 6, 0, 2877, 5, 16, 409, 0, 182, 19, 4063, 186, 0, 3146, 4, 1, 3360, 3596, 4078, 12, 9, 31, 0, 4081, 14, 0, 4063, 0, 0, 241, 0, 4, 3, 4075, 8, 222, 0, 4, 4017, 3984, 58, 0, 32, 0, 6, 0, 3933, 4089, 3, 0, 21, 13, 75, 4063, 0, 4, 0, 3722, 0, 3, 23, 2129, 3095, 0, 3998, 1, 3567, 0, 1218, 118, 4088, 60, 79, 3, 3503, 4083, 0, 10, 11, 228, 18, 4092, 4084, 1303, 0, 4082, 0, 44, 1, 2, 2, 4045, 4, 2, 4082, 4072, 1, 1075, 9, 0, 0, 5, 489, 0, 114, 7, 7, 0, 1, 2833, 35, 0, 0, 0, 8, 3095, 1, 2, 4056, 0, 4084, 1, 4095, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 2, 80, 0, 0, 0, 637, 0, 0, 79, 0, 94, 0, 4085, 258, 5, 3846, 1, 0, 3450, 3794, 3699, 0, 1, 88, 0, 4092, 0, 0, 3680, 1, 1, 205, 0, 6, 0, 3956, 1, 1, 1, 1, 3280, 4053, 1, 0, 0, 0, 0, 4, 4091, 4087, 1, 0, 0, 0, 2, 4093, 1, 0, 0, 3034, 0, 0, 1, 3992, 3573, 0, 4091, 0, 4091, 0, 3109, 19, 4096, 0, 6, 0, 4007, 4088, 0, 6, 1, 20, 138, 4096, 4092, 2642, 0, 4096, 0, 0, 2, 0, 1, 4095, 0, 0, 4093, 4094, 0, 487, 70, 0, 1, 1, 20, 3, 7, 112, 0, 3, 2, 4085, 0, 0, 0, 1, 79, 55, 0, 82, 4095, 481, 4096, 0, 4093, 0, 139] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 3683, 6, 9, 1, 1575, 38, 55, 156, 0, 9, 0, 4064, 409, 2, 4094, 2823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 2, 6, 2715, 3498, 3213, 16, 0, 192, 0, 2119, 0, 0, 4085, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 4088, 0, 0, 0, 8, 3914, 3567, 0, 0, 13, 0, 16, 1964, 1534, 1279, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3449, 4088, 1, 0, 0, 2, 0, 4091, 23, 0, 7, 4093, 0, 0, 0, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2931, 2486, 0, 0, 5, 4071, 0, 3012, 28, 0, 7, 1032, 0, 2532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 11, 1433, 4091, 2201, 0, 4054, 0, 3, 6, 20, 0, 4069, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 4091, 0, 968, 119, 0, 0, 0, 4, 0, 26, 0, 0, 0, 0, 3986, 1384, 1245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 7, 0, 1, 1663, 0, 0, 4036, 15, 4080, 0, 3019, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16926, 16079, 16796, 16422, 16104, 15895, 15921, 16929] avg=16384.0000 max=16929 max_rank=7 min=15895 min_rank=5 imbalance_ratio=1.033264 max_load_reduction=114143 max_load_reduction_pct=0.870842 moved_to_echo=64849 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3258, 0, 72, 1, 810, 246, 161, 674, 0, 108, 0, 3921, 512, 6, 4096, 2838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 1174, 3941, 3361, 0, 0, 10, 0, 2122, 0, 0, 4093, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 0, 0, 0, 3958, 51, 353, 0, 82, 2988, 4022, 0, 0, 7, 0, 148, 1967, 1329, 1279, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4094, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 3764, 0, 1, 48, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2926, 2980, 0, 0, 0, 4068, 0, 3735, 3, 0, 38, 2, 0, 2193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 1433, 4096, 3616, 0, 3925, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4096, 4096, 0, 1103, 6, 0, 0, 0, 361, 0, 0, 8, 0, 0, 16, 4076, 1384, 1252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3770, 0, 0, 0, 0, 0, 346, 0, 27, 4092, 2, 4096, 0, 3019, 0, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16703, 15005, 17114, 16775, 15945, 17301, 16427, 15802] avg=16384.0000 max=17301 max_rank=5 min=15005 min_rank=1 imbalance_ratio=1.055969 max_load_reduction=113771 max_load_reduction_pct=0.868004 moved_to_echo=65475 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1443, 1, 981, 2, 3283, 790, 26, 837, 0, 1245, 42, 4096, 420, 0, 4096, 2843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 0, 2, 1829, 4011, 1329, 0, 0, 198, 0, 2116, 0, 0, 4086, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 2, 0, 0, 4079, 102, 0, 0, 0, 641, 3684, 0, 9, 0, 3, 9, 1962, 1422, 1279, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 4030, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 2751, 3, 1, 277, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3030, 3698, 0, 0, 0, 3290, 0, 2514, 0, 0, 53, 1439, 0, 2348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 1432, 4094, 3731, 19, 4052, 0, 0, 1, 26, 0, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4074, 1, 2788, 91, 0, 1, 0, 144, 10, 0, 0, 0, 0, 0, 4078, 1384, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3763, 0, 0, 3, 0, 32, 572, 0, 1, 4080, 1, 4084, 0, 3016, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20105, 14256, 13632, 15900, 16372, 17337, 17907, 15563] avg=16384.0000 max=20105 max_rank=0 min=13632 min_rank=2 imbalance_ratio=1.227112 max_load_reduction=110967 max_load_reduction_pct=0.846611 moved_to_echo=67179 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 421, 4, 84, 0, 3565, 52, 2, 0, 0, 58, 0, 3940, 3750, 0, 4094, 2801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 0, 8, 778, 2898, 838, 6, 1, 736, 0, 2053, 2, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 2, 0, 3800, 13, 37, 1, 0, 2733, 3482, 0, 0, 0, 0, 1, 1904, 1541, 1279, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 4032, 0, 2, 1, 0, 0, 4088, 28, 190, 0, 3843, 7, 11, 0, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2275, 3782, 0, 0, 0, 3950, 0, 3859, 30, 0, 102, 26, 0, 2543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 1, 6, 1432, 4089, 3790, 0, 4093, 0, 0, 0, 0, 0, 4089, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4094, 0, 882, 1, 0, 0, 0, 76, 0, 0, 230, 0, 0, 23, 3806, 1384, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3432, 0, 0, 1, 0, 7, 3019, 0, 0, 4011, 85, 4096, 1, 3019, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18771, 12477, 15247, 16825, 16567, 17686, 15827, 17672] avg=16384.0000 max=18771 max_rank=0 min=12477 min_rank=1 imbalance_ratio=1.145691 max_load_reduction=112301 max_load_reduction_pct=0.856789 moved_to_echo=67752 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1654, 0, 23, 0, 2601, 1, 1, 127, 0, 90, 0, 3512, 2481, 36, 4096, 2842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3278, 0, 0, 144, 3175, 4036, 0, 0, 55, 0, 2100, 0, 0, 4014, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4, 0, 0, 4087, 192, 0, 0, 27, 4053, 4093, 0, 0, 23, 0, 3, 1946, 1526, 1279, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3966, 4092, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 2303, 0, 1, 638, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3809, 3899, 0, 0, 0, 3242, 0, 3881, 13, 0, 1004, 137, 0, 2519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 11, 1433, 4096, 784, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 119, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4073, 1384, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 122, 0, 0, 4050, 0, 4096, 0, 3019, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17464, 16803, 17652, 15766, 18504, 14518, 15023, 15342] avg=16384.0000 max=18504 max_rank=4 min=14518 min_rank=5 imbalance_ratio=1.129395 max_load_reduction=112568 max_load_reduction_pct=0.858826 moved_to_echo=63387 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 59, 794, 2, 2, 7, 39, 42, 0, 1074, 0, 6, 0, 4062, 1139, 24, 4058, 2660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 952, 32, 16, 304, 4084, 4053, 82, 6, 17, 0, 2116, 0, 4, 3666, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 18, 95, 0, 3712, 11, 0, 4, 39, 3129, 3761, 0, 9, 33, 2, 35, 1970, 1542, 1275, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 3992, 13, 0, 1, 50, 0, 4086, 0, 0, 12, 4032, 0, 13, 13, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 2184, 0, 0, 0, 4052, 28, 3955, 18, 0, 396, 702, 6, 2536, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 209, 1417, 4069, 2640, 21, 4045, 2, 0, 5, 0, 8, 4077, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4074, 4086, 0, 2180, 372, 0, 0, 0, 158, 0, 57, 157, 0, 0, 0, 4076, 1382, 1171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 15, 3, 0, 19, 263, 882, 26, 0, 4034, 11, 4076, 0, 3005, 9, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13970, 15336, 16330, 16787, 17843, 16544, 17760, 16502] avg=16384.0000 max=17843 max_rank=4 min=13970 min_rank=0 imbalance_ratio=1.089050 max_load_reduction=113229 max_load_reduction_pct=0.863869 moved_to_echo=68649 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 368, 0, 6, 0, 2877, 5, 16, 409, 0, 182, 19, 4063, 186, 0, 3998, 2835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3146, 4, 1, 3360, 3596, 4078, 12, 9, 31, 0, 2118, 14, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 4, 3, 4075, 8, 222, 0, 4, 4017, 3984, 58, 0, 32, 0, 6, 1963, 1321, 1278, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 4089, 3, 0, 21, 13, 75, 4063, 0, 4, 0, 3722, 0, 3, 23, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2129, 3095, 0, 0, 1, 3567, 0, 1218, 118, 0, 60, 79, 3, 2182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 11, 228, 18, 1431, 4084, 1303, 0, 4082, 0, 44, 1, 2, 2, 4045, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4082, 4072, 1, 1075, 9, 0, 0, 5, 489, 0, 114, 7, 7, 0, 1, 4088, 1383, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2833, 35, 0, 0, 0, 8, 3095, 1, 2, 4056, 0, 4084, 1, 3018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14965, 20432, 17626, 16616, 12452, 15265, 16583, 17133] avg=16384.0000 max=20432 max_rank=1 min=12452 min_rank=4 imbalance_ratio=1.247070 max_load_reduction=110640 max_load_reduction_pct=0.844116 moved_to_echo=61433 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 80, 0, 0, 0, 637, 0, 0, 79, 0, 94, 0, 4085, 258, 5, 4091, 2837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 1, 0, 3450, 3794, 3699, 0, 1, 88, 0, 2123, 0, 0, 3680, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 6, 0, 3956, 1, 1, 1, 1, 3280, 4053, 1, 0, 0, 0, 0, 1969, 1512, 1279, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4091, 4087, 1, 0, 0, 0, 2, 4093, 1, 0, 0, 3034, 0, 0, 1, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 3573, 0, 0, 0, 4091, 0, 3109, 19, 0, 0, 6, 0, 2495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 20, 138, 1433, 4092, 2642, 0, 4096, 0, 0, 2, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4094, 0, 487, 70, 0, 1, 1, 20, 3, 7, 112, 0, 3, 2, 4096, 1384, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 1, 79, 55, 0, 82, 4095, 481, 4096, 0, 3016, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12168, 20684, 16675, 15981, 17285, 16526, 15624, 16129] avg=16384.0000 max=20684 max_rank=1 min=12168 min_rank=0 imbalance_ratio=1.262451 max_load_reduction=110388 max_load_reduction_pct=0.842194 moved_to_echo=65564 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[25088, 25728, 0, 0, 128, 30336, 128, 25344, 256, 0, 1664, 3456, 128, 19456, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 11776, 128, 1280, 128, 15488, 1280, 384, 3456, 0, 1792, 128, 31744, 9216, 128, 32640, 22528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 32768, 32768, 128, 9728, 768, 0, 128, 128, 1280, 128, 256, 640, 128, 128, 128, 32384, 11136, 9984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[30080, 128, 128, 128, 128, 512, 9856, 128, 128, 32512, 640, 32768, 128, 24192, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[13312, 128, 128, 13824, 29056, 24704, 128, 128, 1408, 0, 16896, 128, 128, 31872, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 31488, 32512, 128, 128, 128, 128, 128, 32768, 128, 256, 128, 27648, 128, 128, 1024, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 512, 256, 512, 11520, 32768, 20736, 128, 32512, 128, 128, 128, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[1408, 128, 128, 128, 31872, 384, 640, 128, 256, 24832, 30720, 128, 128, 128, 128, 256, 15744, 11776, 10240, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[5, 0, 0, 2494, 37, 0, 3849, 0, 0, 0, 96, 3343, 4094, 0, 0, 4094, 4088, 4096, 28, 288, 4092, 3820, 5, 0, 0, 3991, 18, 0, 1, 3083, 4093, 0, 0, 4088, 1146, 4096, 1, 1, 3451, 3903, 0, 23, 766, 0, 105, 0, 1, 1, 4, 0, 0, 0, 3216, 0, 0, 0, 0, 4090, 0, 1543, 451, 0, 4095, 0, 11, 4096, 1, 0, 2, 0, 0, 382, 4095, 0, 11, 4096, 3164, 4096, 4, 0, 393, 0, 0, 258, 4052, 0, 0, 0, 1027, 0, 19, 0, 4083, 11, 4081, 29, 0, 3486, 26, 0, 0, 0, 511, 4090, 0, 0, 4095, 0, 0, 4096, 0, 15, 0, 0, 145, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[2, 1, 21, 1798, 173, 0, 452, 0, 35, 6, 518, 4007, 4079, 17, 12, 4070, 4094, 4069, 14, 0, 4091, 624, 11, 9, 0, 4081, 0, 1, 0, 2319, 4074, 57, 19, 4069, 2376, 4028, 16, 11, 4096, 4094, 34, 0, 1074, 0, 130, 12, 0, 1, 0, 59, 7, 397, 3786, 0, 0, 0, 134, 4081, 24, 0, 3116, 0, 3950, 32, 19, 4083, 10, 0, 0, 4, 0, 930, 4076, 0, 0, 4081, 3055, 4082, 0, 0, 55, 1, 13, 283, 4081, 3, 0, 1, 1978, 1, 0, 0, 4085, 0, 4063, 0, 2, 3572, 26, 1, 0, 8, 195, 3908, 0, 0, 4084, 5, 3, 4049, 0, 27, 0, 19, 1969, 4082, 0, 0, 0, 10, 0, 2, 0, 0, 0, 2, 23, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 1351, 0, 521, 42, 0, 3, 0, 0, 9, 249, 2727, 3997, 0, 0, 4096, 4094, 4036, 0, 2774, 4095, 2391, 335, 1, 1, 3863, 0, 0, 0, 1534, 4096, 46, 0, 4095, 86, 4032, 3, 0, 848, 3999, 2, 137, 196, 0, 4, 0, 0, 0, 0, 4, 0, 11, 1167, 2, 0, 1, 0, 4093, 0, 0, 3965, 0, 4068, 0, 0, 4094, 0, 0, 1, 0, 0, 3735, 4096, 5, 0, 4092, 3932, 4095, 0, 56, 3, 15, 0, 992, 3767, 0, 0, 5, 9, 0, 10, 0, 4096, 0, 4096, 414, 0, 4077, 1549, 1, 0, 0, 2696, 4051, 0, 0, 4096, 0, 0, 4090, 0, 26, 0, 0, 1898, 4094, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 449, 0, 0, 873, 0, 0, 0, 183, 4095, 4096, 0, 0, 4096, 4096, 4096, 11, 3629, 4096, 3691, 0, 0, 0, 4096, 0, 0, 0, 1877, 4096, 0, 0, 4096, 2, 4051, 0, 0, 2571, 4054, 0, 60, 26, 0, 455, 0, 0, 0, 0, 1, 0, 9, 4089, 0, 0, 0, 1, 4096, 0, 5, 2077, 0, 4096, 0, 0, 4080, 0, 0, 94, 0, 0, 11, 4096, 22, 1, 4095, 3772, 4096, 0, 0, 23, 0, 0, 137, 3988, 0, 0, 0, 0, 0, 368, 0, 4095, 0, 4096, 41, 0, 4069, 1625, 0, 0, 0, 1088, 3987, 0, 31, 4096, 0, 0, 4096, 0, 0, 16, 0, 1782, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 3489, 0, 0, 9, 0, 0, 0, 1606, 4083, 4096, 0, 0, 4096, 4096, 4096, 0, 1, 4096, 4096, 0, 0, 0, 4083, 0, 0, 0, 117, 4096, 0, 0, 4096, 2403, 3689, 0, 0, 3475, 3978, 0, 0, 189, 0, 3, 0, 0, 0, 0, 1, 0, 6, 4068, 0, 0, 0, 1, 4096, 0, 367, 561, 0, 4096, 0, 0, 4074, 0, 0, 0, 0, 0, 1083, 4096, 6, 1, 4096, 3697, 4096, 0, 0, 0, 0, 0, 914, 4082, 0, 0, 0, 1, 0, 1717, 0, 4077, 0, 4096, 176, 0, 2661, 2200, 0, 0, 0, 641, 3881, 0, 21, 4096, 0, 0, 4095, 0, 0, 20, 0, 22, 4096, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 27, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 508, 351, 0, 3565, 0, 0, 0, 101, 4096, 4017, 0, 0, 4085, 4096, 4095, 0, 4, 4095, 1843, 0, 1, 0, 4041, 0, 0, 0, 238, 4089, 0, 2, 4087, 722, 4078, 0, 0, 3419, 4020, 0, 102, 0, 0, 539, 0, 0, 0, 1, 0, 0, 573, 683, 0, 4, 0, 1, 4094, 0, 6, 3605, 60, 4089, 0, 0, 4024, 0, 6, 127, 0, 2, 1583, 4094, 15, 32, 4093, 2595, 4093, 0, 1, 92, 8, 1, 340, 4068, 0, 27, 2, 0, 0, 0, 0, 3966, 2, 4074, 6, 9, 3583, 2470, 0, 2, 0, 3451, 3912, 3, 96, 4088, 2, 1, 4095, 0, 0, 12, 0, 2667, 4092, 0, 0, 0, 0, 0, 0, 0, 1, 23, 0, 2, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[17, 1393, 0, 1133, 0, 0, 4, 0, 0, 2, 4, 2541, 4086, 0, 0, 4096, 4091, 4096, 0, 2209, 4084, 1422, 0, 0, 0, 3999, 77, 0, 0, 1358, 4096, 162, 0, 4096, 2091, 4096, 0, 0, 4033, 4096, 0, 14, 2165, 0, 145, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 4089, 0, 1232, 2692, 0, 4096, 0, 2, 4096, 0, 0, 0, 0, 0, 266, 4096, 9, 0, 4096, 3213, 4063, 2, 0, 15, 0, 0, 59, 4094, 4, 0, 0, 9, 0, 864, 0, 4053, 0, 4096, 695, 0, 2926, 1049, 0, 0, 0, 1322, 3726, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 239, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 1, 3434, 4, 4, 439, 1, 9, 3, 176, 4081, 4090, 0, 0, 4095, 4094, 4092, 0, 51, 4027, 4048, 6, 0, 0, 4074, 0, 0, 0, 2721, 4093, 4, 2, 4094, 25, 4088, 0, 4, 2215, 3979, 0, 0, 248, 0, 17, 1, 3, 0, 2, 0, 0, 843, 3053, 2, 0, 0, 0, 4094, 0, 11, 484, 1, 4093, 3, 0, 4091, 3, 0, 0, 0, 3, 2720, 4096, 4, 7, 4094, 4080, 4095, 0, 0, 22, 0, 2, 126, 4092, 2, 0, 0, 14, 2, 29, 0, 4088, 0, 4077, 59, 0, 3789, 38, 2, 2, 0, 3937, 4075, 1, 0, 4093, 0, 0, 4089, 0, 0, 2, 0, 405, 4092, 0, 1, 0, 2, 0, 3, 2, 21, 0, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 2494, 37, 0, 3849, 0, 0, 0, 96, 3343, 4094, 0, 0, 4094, 674, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2663, 28, 288, 4092, 3820, 5, 0, 0, 3991, 18, 0, 1, 3083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3414, 1146, 4096, 1, 1, 3451, 3903, 0, 23, 766, 0, 105, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3216, 0, 0, 0, 0, 4090, 0, 1543, 451, 0, 4095, 0, 1433, 519, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 1, 0, 2, 0, 0, 382, 0, 0, 11, 3384, 3164, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 0, 258, 4052, 0, 0, 0, 1027, 0, 19, 0, 4083, 11, 4081, 29, 1638, 1157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3486, 26, 0, 0, 0, 511, 4090, 0, 0, 1938, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, 4088, 4095, 2936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18931, 17989, 16908, 15818, 15151, 16748, 14162, 15365] avg=16384.0000 max=18931 max_rank=0 min=14162 min_rank=6 imbalance_ratio=1.155457 max_load_reduction=112141 max_load_reduction_pct=0.855568 moved_to_echo=61426 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1351, 0, 521, 42, 0, 3, 0, 0, 9, 249, 2727, 3997, 0, 0, 4096, 675, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2624, 0, 2774, 4095, 2391, 335, 1, 1, 3863, 0, 0, 0, 1534, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3420, 86, 4032, 3, 0, 848, 3999, 2, 137, 196, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 11, 1167, 2, 0, 1, 0, 4093, 0, 0, 3965, 0, 4068, 0, 1412, 519, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 3735, 0, 5, 0, 3380, 3932, 4095, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 15, 0, 992, 3767, 0, 0, 5, 9, 0, 10, 0, 4096, 0, 4096, 414, 1639, 1157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 1549, 1, 0, 0, 2696, 4051, 0, 0, 1938, 0, 0, 4090, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1898, 4094, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 2939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13915, 17664, 12727, 15709, 19298, 16203, 18428, 17128] avg=16384.0000 max=19298 max_rank=4 min=12727 min_rank=2 imbalance_ratio=1.177856 max_load_reduction=111774 max_load_reduction_pct=0.852768 moved_to_echo=71057 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 449, 0, 0, 873, 0, 0, 0, 183, 4095, 4096, 0, 0, 4096, 676, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2663, 11, 3629, 4096, 3691, 0, 0, 0, 4096, 0, 0, 0, 1877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3420, 2, 4051, 0, 0, 2571, 4054, 0, 60, 26, 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 4089, 0, 0, 0, 1, 4096, 0, 5, 2077, 0, 4096, 0, 1433, 519, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 94, 0, 0, 11, 0, 22, 1, 3383, 3772, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 137, 3988, 0, 0, 0, 0, 0, 368, 0, 4095, 0, 4096, 41, 1639, 1156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 1625, 0, 0, 0, 1088, 3987, 0, 31, 1938, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 1782, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4096, 4096, 2940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14713, 20063, 14639, 16793, 15459, 15543, 16834, 17028] avg=16384.0000 max=20063 max_rank=1 min=14639 min_rank=2 imbalance_ratio=1.224548 max_load_reduction=111009 max_load_reduction_pct=0.846931 moved_to_echo=64864 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 21, 1798, 173, 0, 452, 0, 35, 6, 518, 4007, 4079, 17, 12, 4070, 675, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2642, 14, 0, 4091, 624, 11, 9, 0, 4081, 0, 1, 0, 2319, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 3394, 2376, 4028, 16, 11, 4096, 4094, 34, 0, 1074, 0, 130, 12, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 7, 397, 3786, 0, 0, 0, 134, 4081, 24, 0, 3116, 0, 3950, 32, 1427, 519, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4083, 10, 0, 0, 4, 0, 930, 0, 0, 0, 3361, 3055, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 1, 13, 283, 4081, 3, 0, 1, 1978, 1, 0, 0, 4085, 0, 4063, 0, 1637, 1151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3572, 26, 1, 0, 8, 195, 3908, 0, 0, 1928, 5, 3, 4049, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1969, 4082, 0, 0, 0, 10, 0, 2, 0, 0, 0, 2, 23, 0, 4094, 4076, 2923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16116, 13849, 19285, 18002, 15544, 17352, 13724, 17200] avg=16384.0000 max=19285 max_rank=2 min=13724 min_rank=6 imbalance_ratio=1.177063 max_load_reduction=111787 max_load_reduction_pct=0.852867 moved_to_echo=63820 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[17, 1393, 0, 1133, 0, 0, 4, 0, 0, 2, 4, 2541, 4086, 0, 0, 4096, 676, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2663, 0, 2209, 4084, 1422, 0, 0, 0, 3999, 77, 0, 0, 1358, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3420, 2091, 4096, 0, 0, 4033, 4096, 0, 14, 2165, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 4089, 0, 1232, 2692, 0, 4096, 0, 1433, 519, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 266, 0, 9, 0, 3384, 3213, 4063, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 59, 4094, 4, 0, 0, 9, 0, 864, 0, 4053, 0, 4096, 695, 1639, 1156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2926, 1049, 0, 0, 0, 1322, 3726, 1, 0, 1938, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4091, 4096, 2940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14197, 15974, 20060, 18600, 15035, 16684, 15058, 15464] avg=16384.0000 max=20060 max_rank=2 min=14197 min_rank=0 imbalance_ratio=1.224365 max_load_reduction=111012 max_load_reduction_pct=0.846954 moved_to_echo=62241 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 508, 351, 0, 3565, 0, 0, 0, 101, 4096, 4017, 0, 0, 4085, 674, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2663, 0, 4, 4095, 1843, 0, 1, 0, 4041, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3413, 722, 4078, 0, 0, 3419, 4020, 0, 102, 0, 0, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 573, 683, 0, 4, 0, 1, 4094, 0, 6, 3605, 60, 4089, 0, 1432, 518, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 6, 127, 0, 2, 1583, 0, 15, 32, 3381, 2595, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 8, 1, 340, 4068, 0, 27, 2, 0, 0, 0, 0, 3966, 2, 4074, 6, 1635, 1155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3583, 2470, 0, 2, 0, 3451, 3912, 3, 96, 1935, 2, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 2667, 4092, 0, 0, 0, 0, 0, 0, 0, 1, 23, 0, 2, 2, 4096, 4094, 2934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17642, 12885, 16295, 15533, 15859, 15376, 19559, 17923] avg=16384.0000 max=19559 max_rank=6 min=12885 min_rank=1 imbalance_ratio=1.193787 max_load_reduction=111513 max_load_reduction_pct=0.850777 moved_to_echo=68717 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3489, 0, 0, 9, 0, 0, 0, 1606, 4083, 4096, 0, 0, 4096, 676, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2663, 0, 1, 4096, 4096, 0, 0, 0, 4083, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3420, 2403, 3689, 0, 0, 3475, 3978, 0, 0, 189, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 4068, 0, 0, 0, 1, 4096, 0, 367, 561, 0, 4096, 0, 1433, 519, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 1083, 0, 6, 1, 3384, 3697, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 914, 4082, 0, 0, 0, 1, 0, 1717, 0, 4077, 0, 4096, 176, 1639, 1157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2661, 2200, 0, 0, 0, 641, 3881, 0, 21, 1938, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 22, 4096, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 27, 0, 4096, 4096, 2939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18300, 15056, 17157, 15615, 16341, 17859, 15437, 15307] avg=16384.0000 max=18300 max_rank=0 min=15056 min_rank=1 imbalance_ratio=1.116943 max_load_reduction=112772 max_load_reduction_pct=0.860382 moved_to_echo=64944 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 3434, 4, 4, 439, 1, 9, 3, 176, 4081, 4090, 0, 0, 4095, 675, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2661, 0, 51, 4027, 4048, 6, 0, 0, 4074, 0, 0, 0, 2721, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3419, 25, 4088, 0, 4, 2215, 3979, 0, 0, 248, 0, 17, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 843, 3053, 2, 0, 0, 0, 4094, 0, 11, 484, 1, 4093, 3, 1431, 518, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 3, 2720, 0, 4, 7, 3382, 4080, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 2, 126, 4092, 2, 0, 0, 14, 2, 29, 0, 4088, 0, 4077, 59, 1638, 1156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3789, 38, 2, 2, 0, 3937, 4075, 1, 0, 1937, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 405, 4092, 0, 1, 0, 2, 0, 3, 2, 21, 0, 2, 0, 0, 4094, 4096, 2937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17258, 17592, 14001, 15002, 18385, 15307, 17870, 15657] avg=16384.0000 max=18385 max_rank=4 min=14001 min_rank=2 imbalance_ratio=1.122131 max_load_reduction=112687 max_load_reduction_pct=0.859734 moved_to_echo=67219 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 9216, 32768, 0, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 23552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32640, 128, 128, 256, 128, 128, 10752, 0, 128, 128, 27136, 27520, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 28288, 9088, 128, 128, 128, 13952, 31744, 128, 256, 15616, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 2816, 128, 13952, 640, 128, 9216, 128, 128, 128, 2944, 29056, 32640, 128, 128, 32768, 5504, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[640, 128, 128, 3200, 32256, 128, 128, 128, 3072, 128, 3072, 0, 32640, 128, 32768, 1536, 13184, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 21248, 128, 8960, 32768, 22016, 384, 128, 128, 32256, 128, 128, 128, 13312, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 27392, 8960, 32256, 128, 128, 24192, 32128, 128, 384, 4736, 0, 1408, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 1920, 24192, 128, 128, 128, 256, 32768, 128, 3200, 17024, 128, 32640, 128, 11520, 4224, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 4096, 1, 14, 3293, 0, 1, 0, 0, 0, 0, 0, 0, 790, 4090, 0, 0, 0, 0, 227, 0, 0, 377, 1, 84, 2519, 0, 0, 4089, 0, 0, 18, 0, 0, 2980, 4095, 106, 19, 0, 3929, 4041, 4096, 0, 4095, 4094, 0, 0, 1, 0, 17, 1, 0, 4073, 0, 0, 0, 1, 0, 207, 0, 0, 4094, 0, 3, 4096, 2, 0, 4096, 0, 2, 3803, 0, 0, 4087, 3926, 0, 0, 0, 4096, 4096, 1, 0, 4082, 0, 7, 0, 0, 0, 380, 4096, 1, 0, 0, 79, 0, 252, 0, 4083, 5, 0, 4096, 4093, 4096, 0, 0, 3947, 0, 1, 25, 4096, 0, 1, 0, 22, 0, 4090, 468, 1, 0, 0, 0, 406, 731, 170, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4086, 0, 0, 0, 3789, 0, 15, 4037, 10, 13, 1, 0, 0, 0, 0, 0, 3, 3637, 27, 5, 2, 0, 0, 0, 0, 32, 6, 458, 3, 5, 19, 4039, 39, 7, 0, 11, 0, 3730, 4066, 81, 132, 48, 4086, 4077, 4004, 0, 4063, 4045, 0, 63, 0, 0, 144, 1, 8, 4090, 0, 0, 0, 31, 0, 1932, 0, 25, 4060, 46, 21, 4023, 3, 0, 4096, 0, 12, 3949, 15, 1, 3913, 0, 4, 12, 0, 4078, 4080, 34, 3, 4088, 4, 0, 0, 1, 15, 33, 4047, 3, 0, 0, 23, 488, 0, 0, 4087, 0, 18, 4090, 4063, 3994, 12, 5, 3535, 5, 0, 0, 4055, 5, 5, 5, 0, 22, 4046, 3151, 20, 244, 0, 37, 3706, 4, 93, 3948] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4081, 2, 0, 0, 4096, 0, 4, 3942, 0, 0, 7, 0, 0, 0, 0, 0, 2590, 4096, 0, 0, 0, 0, 9, 0, 0, 54, 0, 421, 243, 0, 0, 4089, 0, 2, 1, 8, 41, 3301, 4096, 0, 0, 0, 4095, 4065, 4094, 0, 4096, 3869, 0, 0, 0, 0, 0, 4, 0, 4029, 1, 0, 0, 1, 1, 3050, 1, 1, 4086, 0, 0, 3509, 0, 0, 3831, 0, 0, 1246, 0, 0, 4073, 38, 129, 0, 11, 4096, 4089, 0, 2, 4051, 0, 1, 0, 22, 0, 664, 4096, 1, 2, 0, 1712, 0, 34, 0, 4069, 0, 0, 4087, 4096, 4079, 7, 0, 4068, 8, 0, 10, 4096, 0, 0, 0, 0, 0, 4096, 29, 0, 94, 0, 0, 4072, 3, 88, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 53, 1621, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 4009, 0, 0, 0, 0, 0, 0, 0, 297, 1, 324, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 3625, 4096, 8, 27, 0, 4096, 4096, 4096, 0, 4096, 4051, 0, 0, 0, 0, 0, 0, 0, 4055, 224, 0, 0, 0, 4, 1381, 49, 0, 4096, 1, 0, 4091, 0, 0, 4044, 0, 0, 2805, 0, 0, 3971, 50, 98, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 9, 1, 3751, 4096, 0, 0, 0, 843, 0, 37, 0, 3966, 1, 0, 4061, 4069, 4096, 212, 0, 1494, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4095, 307, 0, 49, 0, 0, 3819, 19, 18, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4092, 0, 3, 0, 4096, 0, 13, 3941, 5, 0, 0, 0, 7, 0, 0, 0, 277, 3605, 0, 2, 0, 2, 0, 0, 0, 490, 0, 1409, 0, 0, 0, 4093, 0, 0, 0, 5, 0, 4051, 4094, 0, 0, 19, 4095, 4095, 4096, 8, 4096, 2746, 2, 5, 5, 0, 0, 1, 0, 4081, 0, 5, 0, 0, 10, 1362, 20, 0, 4074, 0, 6, 3993, 3, 3, 4049, 0, 0, 3804, 1, 9, 4081, 2700, 0, 1, 0, 4090, 4094, 4, 0, 4038, 0, 0, 0, 3, 0, 20, 4091, 0, 13, 1, 2214, 1, 0, 2, 4096, 8, 0, 4087, 3354, 4092, 2, 0, 4073, 0, 0, 0, 4090, 0, 0, 0, 1, 0, 4085, 0, 4, 5, 0, 0, 2895, 61, 2, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1505, 2095, 0, 0, 0, 0, 0, 0, 0, 3143, 1, 420, 7, 0, 0, 4096, 0, 0, 4, 0, 0, 3300, 4096, 2, 1, 0, 4095, 4096, 4096, 0, 4096, 3988, 0, 0, 0, 0, 0, 3, 0, 4092, 0, 0, 0, 0, 6, 177, 0, 0, 4096, 1, 0, 4085, 0, 0, 3795, 0, 0, 2948, 0, 0, 4096, 42, 58, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 33, 4096, 0, 0, 0, 3360, 0, 1, 0, 4086, 3, 0, 4096, 4095, 4096, 7, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 423, 0, 113, 0, 0, 3137, 20, 116, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 2760, 0, 0, 1, 0, 1, 0, 0, 1, 3748, 3399, 0, 4, 0, 0, 0, 3, 0, 1757, 0, 2, 266, 0, 0, 4096, 0, 1, 0, 0, 1, 1640, 4096, 0, 0, 0, 4090, 3797, 4096, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 3566, 0, 0, 0, 0, 39, 10, 1, 0, 4094, 0, 0, 4047, 0, 0, 4080, 0, 0, 1656, 0, 0, 3868, 388, 0, 0, 1, 4096, 4096, 0, 0, 4075, 77, 0, 0, 0, 0, 3523, 4096, 0, 2, 0, 11, 0, 129, 0, 4075, 0, 0, 4074, 4030, 4096, 17, 0, 3667, 15, 0, 0, 4091, 0, 0, 0, 45, 2, 4091, 2736, 0, 0, 0, 0, 3931, 104, 9, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4092, 55, 1, 0, 4096, 1, 0, 1320, 0, 0, 44, 0, 0, 5, 0, 1, 2427, 2988, 6, 0, 0, 0, 1, 0, 0, 0, 0, 3463, 128, 18, 2, 4087, 0, 1, 134, 0, 124, 3774, 4083, 74, 0, 0, 4078, 4095, 4083, 1, 4041, 4086, 0, 21, 0, 0, 0, 110, 11, 3728, 11, 2, 0, 0, 0, 27, 2, 0, 4096, 1, 2, 4093, 39, 1, 4080, 0, 0, 4066, 0, 0, 3716, 0, 0, 0, 0, 4070, 4095, 7, 0, 4056, 0, 0, 0, 1, 6, 0, 4095, 0, 0, 1, 3421, 1, 0, 0, 4039, 0, 0, 3653, 3368, 4079, 41, 0, 3948, 0, 0, 0, 4081, 1, 0, 0, 0, 1, 4095, 2, 243, 441, 8, 0, 3900, 33, 0, 4071] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 4096, 1, 14, 3293, 0, 1, 0, 0, 0, 0, 0, 0, 3672, 960, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 790, 4090, 0, 0, 0, 0, 227, 0, 0, 377, 1, 84, 2519, 0, 0, 4089, 2655, 1041, 838, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 2980, 0, 106, 19, 0, 3929, 4041, 985, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 17, 1, 0, 4073, 0, 0, 0, 1, 0, 207, 0, 0, 4095, 4095, 3136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 0, 3, 4096, 2, 0, 4096, 0, 2, 3803, 0, 0, 4087, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3055, 1, 0, 4082, 0, 7, 0, 0, 0, 380, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 252, 0, 3245, 5, 0, 4096, 4093, 0, 0, 0, 3947, 0, 1, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 0, 1, 0, 22, 0, 4090, 468, 1, 0, 0, 0, 406, 731, 170, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16584, 17052, 16172, 15626, 20437, 15718, 15743, 13740] avg=16384.0000 max=20437 max_rank=4 min=13740 min_rank=7 imbalance_ratio=1.247375 max_load_reduction=110635 max_load_reduction_pct=0.844078 moved_to_echo=65638 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4081, 2, 0, 0, 4096, 0, 4, 3942, 0, 0, 7, 0, 0, 0, 0, 0, 3665, 956, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2590, 4096, 0, 0, 0, 0, 9, 0, 0, 54, 0, 421, 243, 0, 0, 4089, 2654, 1039, 835, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 8, 41, 3301, 0, 0, 0, 0, 4095, 4065, 984, 0, 0, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4029, 1, 0, 0, 1, 1, 3050, 1, 1, 4096, 4096, 3123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 3509, 0, 0, 3831, 0, 0, 1246, 0, 0, 4073, 38, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 3050, 0, 2, 4051, 0, 1, 0, 22, 0, 664, 4096, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1712, 0, 34, 0, 3234, 0, 0, 4087, 4096, 0, 7, 0, 4068, 8, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 0, 0, 0, 4096, 29, 0, 94, 0, 0, 4072, 3, 88, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17209, 16371, 16366, 18403, 13247, 15996, 17256, 16224] avg=16384.0000 max=18403 max_rank=3 min=13247 min_rank=4 imbalance_ratio=1.123230 max_load_reduction=112669 max_load_reduction_pct=0.859596 moved_to_echo=62723 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 2760, 0, 0, 1, 0, 1, 0, 0, 1, 3672, 960, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 3399, 0, 4, 0, 0, 0, 3, 0, 1757, 0, 2, 266, 0, 0, 4096, 2655, 1041, 836, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1640, 0, 0, 0, 0, 4090, 3797, 985, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3566, 0, 0, 0, 0, 39, 10, 1, 0, 4096, 4092, 3136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 0, 0, 4047, 0, 0, 4080, 0, 0, 1656, 0, 0, 3868, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 3055, 0, 0, 4075, 77, 0, 0, 0, 0, 3523, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 129, 0, 3239, 0, 0, 4074, 4030, 0, 17, 0, 3667, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3750, 0, 0, 0, 45, 2, 4091, 2736, 0, 0, 0, 0, 3931, 104, 9, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16043, 18148, 14610, 14940, 14461, 18925, 15182, 18763] avg=16384.0000 max=18925 max_rank=5 min=14461 min_rank=4 imbalance_ratio=1.155090 max_load_reduction=112147 max_load_reduction_pct=0.855614 moved_to_echo=67331 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4086, 0, 0, 0, 3789, 0, 15, 4037, 10, 13, 1, 0, 0, 0, 0, 0, 3647, 936, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3637, 27, 5, 2, 0, 0, 0, 0, 32, 6, 458, 3, 5, 19, 4039, 2599, 1039, 843, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 7, 0, 11, 0, 3730, 0, 81, 132, 48, 4086, 4077, 956, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 144, 1, 8, 4090, 0, 0, 0, 31, 0, 1932, 0, 25, 4066, 4063, 3058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 46, 21, 4023, 3, 0, 4096, 0, 12, 3949, 15, 1, 3913, 0, 4, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 3041, 34, 3, 4088, 4, 0, 0, 1, 15, 33, 4047, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 488, 0, 0, 3244, 0, 18, 4090, 4063, 0, 12, 5, 3535, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3714, 5, 5, 5, 0, 22, 4046, 3151, 20, 244, 0, 37, 3706, 4, 93, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16983, 13058, 17212, 17481, 16508, 15347, 15483, 19000] avg=16384.0000 max=19000 max_rank=7 min=13058 min_rank=1 imbalance_ratio=1.159668 max_load_reduction=112072 max_load_reduction_pct=0.855042 moved_to_echo=66338 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 53, 1621, 0, 0, 0, 0, 0, 0, 0, 0, 3674, 959, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 4009, 0, 0, 0, 0, 0, 0, 0, 297, 1, 324, 1, 0, 0, 4096, 2655, 1041, 814, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3625, 0, 8, 27, 0, 4096, 4096, 985, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 224, 0, 0, 0, 4, 1381, 49, 0, 4096, 4096, 3137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 1, 0, 4091, 0, 0, 4044, 0, 0, 2805, 0, 0, 3971, 50, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3055, 0, 0, 4096, 0, 0, 0, 9, 1, 3751, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 843, 0, 37, 0, 3152, 1, 0, 4061, 4069, 0, 212, 0, 1494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 0, 2, 0, 4095, 307, 0, 49, 0, 0, 3819, 19, 18, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14955, 17572, 16888, 17042, 15482, 19104, 13869, 16160] avg=16384.0000 max=19104 max_rank=5 min=13869 min_rank=6 imbalance_ratio=1.166016 max_load_reduction=111968 max_load_reduction_pct=0.854248 moved_to_echo=64615 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3674, 960, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1505, 2095, 0, 0, 0, 0, 0, 0, 0, 3143, 1, 420, 7, 0, 0, 4096, 2655, 1041, 838, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3300, 0, 2, 1, 0, 4095, 4096, 985, 0, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4092, 0, 0, 0, 0, 6, 177, 0, 0, 4096, 4096, 3136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 1, 0, 4085, 0, 0, 3795, 0, 0, 2948, 0, 0, 4096, 42, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3055, 0, 0, 4096, 0, 0, 0, 0, 1, 33, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3360, 0, 1, 0, 3248, 3, 0, 4096, 4095, 0, 7, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 0, 0, 0, 4095, 423, 0, 113, 0, 0, 3137, 20, 116, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17374, 16142, 16471, 15606, 15447, 15377, 18900, 15755] avg=16384.0000 max=18900 max_rank=6 min=15377 min_rank=5 imbalance_ratio=1.153564 max_load_reduction=112172 max_load_reduction_pct=0.855804 moved_to_echo=65479 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 3, 0, 4096, 0, 13, 3941, 5, 0, 0, 0, 7, 0, 0, 0, 3654, 958, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 3605, 0, 2, 0, 2, 0, 0, 0, 490, 0, 1409, 0, 0, 0, 4093, 2655, 1040, 840, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4051, 0, 0, 0, 19, 4095, 4095, 985, 8, 0, 2746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 5, 0, 0, 1, 0, 4081, 0, 5, 0, 0, 10, 1362, 20, 0, 4094, 4096, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 0, 6, 3993, 3, 3, 4049, 0, 0, 3804, 1, 9, 4081, 2700, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3054, 4, 0, 4038, 0, 0, 0, 3, 0, 20, 4091, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2214, 1, 0, 2, 3256, 8, 0, 4087, 3354, 0, 2, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3750, 0, 0, 0, 1, 0, 4085, 0, 4, 5, 0, 0, 2895, 61, 2, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17225, 14753, 16004, 16815, 19070, 15314, 16997, 14894] avg=16384.0000 max=19070 max_rank=4 min=14753 min_rank=1 imbalance_ratio=1.163940 max_load_reduction=112002 max_load_reduction_pct=0.854507 moved_to_echo=66275 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4092, 55, 1, 0, 4096, 1, 0, 1320, 0, 0, 44, 0, 0, 5, 0, 1, 3674, 956, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2427, 2988, 6, 0, 0, 0, 1, 0, 0, 0, 0, 3463, 128, 18, 2, 4087, 2647, 1040, 829, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 134, 0, 124, 3774, 0, 74, 0, 0, 4078, 4095, 982, 1, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 110, 11, 3728, 11, 2, 0, 0, 0, 27, 2, 0, 4083, 4041, 3123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 1, 2, 4093, 39, 1, 4080, 0, 0, 4066, 0, 0, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 3055, 7, 0, 4056, 0, 0, 0, 1, 6, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3421, 1, 0, 0, 3210, 0, 0, 3653, 3368, 0, 41, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3741, 1, 0, 0, 0, 1, 4095, 2, 243, 441, 8, 0, 3900, 33, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14699, 17976, 17349, 15159, 16420, 15291, 17642, 16536] avg=16384.0000 max=17976 max_rank=1 min=14699 min_rank=0 imbalance_ratio=1.097168 max_load_reduction=113096 max_load_reduction_pct=0.862854 moved_to_echo=65889 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 0, 32512, 128, 128, 25088, 128, 128, 128, 0, 128, 128, 0, 128, 29440, 7680, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[11776, 512, 512, 128, 25856, 128, 128, 32256, 31232, 0, 384, 128, 28928, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 24448, 128, 128, 32640, 128, 128, 0, 128, 128, 8448, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[30080, 128, 128, 128, 128, 128, 32768, 7168, 384, 1024, 128, 128, 25984, 1024, 512, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[3456, 128, 128, 32000, 128, 128, 32128, 0, 128, 24320, 128, 128, 31872, 7168, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[15360, 28032, 128, 128, 128, 128, 256, 128, 0, 6272, 128, 6656, 3200, 128, 128, 32768, 21248, 8448, 6784, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 256, 128, 256, 26496, 0, 384, 256, 128, 32640, 32384, 7936, 128, 0, 30976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 256, 128, 128, 31744, 256, 128, 0, 128, 128, 8192, 128, 128, 32768, 32768, 25088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 1, 4038, 0, 0, 4, 4096, 4094, 3461, 0, 0, 0, 0, 208, 4094, 4094, 1172, 0, 4096, 5, 4096, 15, 0, 53, 18, 0, 5, 4096, 0, 3589, 0, 0, 0, 4056, 0, 51, 3, 2, 1, 4096, 0, 4096, 4078, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4092, 4084, 4096, 0, 4077, 36, 14, 4075, 9, 0, 0, 1, 0, 0, 4093, 4072, 0, 4095, 4095, 0, 0, 0, 0, 0, 2594, 0, 0, 0, 1, 0, 0, 0, 0, 17, 10, 0, 2, 0, 0, 6, 0, 0, 0, 0, 25, 0, 0, 2, 0, 4094, 9, 0, 1903, 4085, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 1139, 4088, 1, 1, 4062, 1, 2296, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 11, 4096, 0, 1, 0, 4096, 2152, 4038, 0, 0, 1, 0, 361, 4091, 4093, 2101, 0, 4096, 0, 4096, 1, 0, 50, 6, 0, 0, 4096, 0, 3640, 0, 1, 1, 4096, 0, 8, 0, 0, 0, 4085, 0, 4096, 4095, 0, 1, 0, 0, 0, 0, 0, 4076, 0, 0, 4076, 4096, 4096, 0, 3782, 1, 127, 3973, 0, 0, 42, 0, 0, 0, 4096, 4096, 0, 4096, 4073, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 0, 0, 4087, 1, 0, 2994, 4096, 0, 21, 7, 0, 4096, 0, 0, 0, 4095, 1267, 4096, 3, 0, 4096, 0, 201, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4090, 0, 0, 6, 4095, 4083, 1172, 0, 0, 0, 0, 0, 4096, 4084, 257, 0, 4091, 15, 4093, 0, 6, 198, 3205, 0, 0, 4096, 0, 2287, 0, 0, 0, 4096, 0, 247, 0, 0, 0, 4078, 11, 4094, 4087, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 4093, 4093, 0, 3984, 55, 0, 1777, 0, 0, 1338, 0, 0, 0, 4096, 4096, 0, 4096, 4055, 12, 0, 0, 0, 3, 2065, 0, 13, 1, 0, 0, 0, 1, 0, 7, 0, 3, 0, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 4096, 14, 0, 3895, 4096, 0, 0, 37, 3, 4084, 0, 0, 0, 4093, 4, 4096, 16, 0, 4096, 6, 4056, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[6, 10, 463, 4077, 0, 0, 0, 4074, 3915, 33, 78, 1, 13, 2, 65, 4077, 4039, 2705, 0, 4067, 46, 3997, 11, 0, 1505, 335, 0, 0, 4020, 15, 973, 0, 7, 4, 4050, 0, 129, 0, 17, 0, 4062, 0, 4039, 4095, 0, 87, 0, 0, 23, 11, 0, 3983, 52, 2, 4019, 4032, 4055, 41, 3092, 0, 82, 2982, 0, 0, 435, 8, 28, 0, 4081, 4089, 12, 4085, 4058, 0, 0, 2, 36, 12, 3684, 5, 14, 0, 1, 25, 0, 5, 3, 0, 0, 0, 0, 47, 6, 0, 0, 2, 161, 34, 32, 0, 69, 3, 0, 4009, 0, 0, 2524, 4079, 18, 0, 0, 27, 4033, 50, 0, 35, 4039, 1674, 4051, 30, 48, 4080, 7, 4010, 0, 18, 7, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 245, 4096, 0, 6, 0, 4096, 4095, 1869, 0, 0, 0, 0, 136, 4049, 4095, 3994, 0, 4096, 0, 4096, 1, 0, 59, 201, 0, 0, 4096, 0, 4070, 0, 1, 576, 4096, 0, 86, 0, 0, 0, 4093, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4095, 4096, 4096, 0, 2261, 7, 39, 4096, 2, 0, 65, 0, 0, 0, 4096, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 3862, 0, 0, 3827, 4096, 0, 95, 61, 0, 4095, 0, 0, 0, 4096, 1042, 4096, 0, 0, 4029, 0, 1959, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 3, 4092, 2847, 2512, 0, 0, 28, 7, 3873, 4096, 3935, 3642, 0, 4085, 0, 4092, 10, 0, 457, 90, 6, 1, 4094, 0, 3816, 0, 87, 0, 4095, 0, 1056, 0, 0, 0, 3845, 0, 4092, 4090, 19, 0, 4, 0, 0, 2, 0, 4068, 0, 0, 4096, 4090, 4093, 0, 4077, 20, 16, 4094, 4, 3, 0, 0, 0, 2, 4088, 4094, 0, 4094, 4096, 0, 0, 0, 2, 0, 951, 3, 6, 6, 1, 2, 0, 0, 0, 0, 0, 1, 0, 4, 0, 2, 0, 0, 0, 2, 0, 2, 0, 0, 1, 4096, 0, 0, 64, 4086, 4, 3, 24, 6, 4090, 0, 4, 0, 4086, 1305, 4092, 0, 0, 4076, 0, 144, 2, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 8, 4094, 4094, 1223, 0, 0, 0, 0, 104, 4096, 4077, 3939, 0, 4096, 0, 4096, 0, 79, 38, 10, 0, 0, 4096, 0, 1758, 0, 0, 0, 4088, 1, 10, 0, 0, 0, 4096, 0, 4096, 4092, 2, 0, 0, 1, 0, 0, 0, 4089, 0, 0, 4091, 4096, 4096, 0, 4089, 0, 264, 3941, 0, 26, 1749, 0, 0, 0, 4094, 4095, 0, 4095, 4096, 0, 0, 0, 0, 1, 682, 0, 0, 0, 2, 0, 0, 1, 0, 0, 108, 0, 0, 2, 0, 8, 0, 0, 0, 0, 48, 4, 0, 0, 0, 4095, 8, 0, 2755, 4096, 0, 0, 3, 0, 4096, 0, 0, 0, 4096, 145, 4096, 0, 0, 4096, 0, 3619, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[70, 6, 0, 4095, 1, 0, 8, 4073, 4057, 241, 2, 0, 0, 0, 20, 4095, 4050, 488, 0, 4095, 0, 4091, 6, 0, 3605, 4032, 0, 1, 4096, 5, 4055, 0, 0, 30, 4077, 0, 0, 2, 53, 44, 4046, 0, 4091, 4086, 24, 0, 1, 0, 1, 0, 11, 4093, 0, 0, 3984, 4095, 4093, 0, 1488, 0, 0, 2126, 13, 0, 0, 0, 0, 0, 4091, 4095, 0, 4095, 4088, 21, 4, 0, 25, 0, 1866, 1, 39, 91, 0, 0, 1, 13, 0, 49, 0, 84, 3, 1123, 8, 0, 28, 0, 0, 0, 272, 0, 0, 0, 0, 4096, 0, 0, 1014, 3969, 5, 0, 0, 0, 4072, 1, 0, 2, 4065, 3981, 4043, 2, 10, 4091, 0, 174, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 4038, 0, 0, 4, 2011, 4094, 3461, 0, 0, 0, 0, 208, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1172, 0, 0, 5, 4096, 15, 0, 53, 18, 0, 5, 1175, 0, 3589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 51, 3, 2, 1, 4096, 0, 4096, 3833, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 1396, 0, 0, 4077, 36, 14, 4075, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 4072, 0, 1651, 4095, 0, 0, 0, 0, 0, 2594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 17, 10, 0, 2, 0, 0, 6, 0, 0, 4096, 4096, 3360, 2921, 1708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 2, 0, 4094, 9, 0, 1903, 4085, 0, 0, 0, 0, 2444, 2085, 980, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 736, 1139, 4088, 1, 1, 4062, 1, 2296, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17911, 14222, 16139, 17795, 16506, 16217, 15872, 16410] avg=16384.0000 max=17911 max_rank=0 min=14222 min_rank=1 imbalance_ratio=1.093201 max_load_reduction=113161 max_load_reduction_pct=0.863350 moved_to_echo=65005 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 11, 4096, 0, 1, 0, 2011, 2152, 4038, 0, 0, 1, 0, 361, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2101, 0, 0, 0, 4096, 1, 0, 50, 6, 0, 0, 1175, 0, 3640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4096, 0, 8, 0, 0, 0, 4085, 0, 4096, 3849, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4076, 1400, 0, 0, 3782, 1, 127, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 4096, 4096, 0, 1652, 4073, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3359, 2921, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 0, 0, 4087, 1, 0, 2994, 4096, 0, 21, 7, 0, 2444, 2085, 983, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 736, 1267, 4096, 3, 0, 4096, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16762, 15162, 16137, 17435, 17915, 16193, 16973, 14495] avg=16384.0000 max=17915 max_rank=4 min=14495 min_rank=7 imbalance_ratio=1.093445 max_load_reduction=113157 max_load_reduction_pct=0.863319 moved_to_echo=65576 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 8, 2010, 4094, 1223, 0, 0, 0, 0, 104, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3939, 0, 0, 0, 4096, 0, 79, 38, 10, 0, 0, 1175, 0, 1758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 10, 0, 0, 0, 4096, 0, 4096, 3846, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4091, 1400, 0, 0, 4089, 0, 264, 3941, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1749, 0, 0, 0, 4094, 4095, 0, 1651, 4096, 0, 0, 0, 0, 1, 682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 108, 0, 0, 2, 0, 8, 0, 0, 4096, 4096, 3360, 2921, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 4, 0, 0, 0, 4095, 8, 0, 2755, 4096, 0, 0, 3, 0, 2444, 2084, 983, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 736, 145, 4096, 0, 0, 4096, 0, 3619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15631, 15172, 16140, 17900, 16368, 16307, 16766, 16788] avg=16384.0000 max=17900 max_rank=3 min=15172 min_rank=1 imbalance_ratio=1.092529 max_load_reduction=113172 max_load_reduction_pct=0.863434 moved_to_echo=66229 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4090, 0, 0, 6, 2011, 4083, 1172, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 257, 0, 0, 15, 4093, 0, 6, 198, 3205, 0, 0, 1175, 0, 2287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 247, 0, 0, 0, 4078, 11, 4094, 3842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 1398, 0, 0, 3984, 55, 0, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1338, 0, 0, 0, 4096, 4096, 0, 1652, 4055, 12, 0, 0, 0, 3, 2065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 0, 0, 0, 1, 0, 7, 0, 3, 0, 7, 0, 0, 0, 0, 4091, 4093, 3357, 2921, 1712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 4096, 14, 0, 3895, 4096, 0, 0, 37, 3, 2444, 2084, 983, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 736, 4, 4096, 16, 0, 4096, 6, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15458, 15320, 16368, 15408, 17317, 16206, 17901, 17094] avg=16384.0000 max=17901 max_rank=6 min=15320 min_rank=1 imbalance_ratio=1.092590 max_load_reduction=113171 max_load_reduction_pct=0.863426 moved_to_echo=68518 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 245, 4096, 0, 6, 0, 2011, 4095, 1869, 0, 0, 0, 0, 136, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3994, 0, 0, 0, 4096, 1, 0, 59, 201, 0, 0, 1175, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 576, 4096, 0, 86, 0, 0, 0, 4093, 0, 4096, 3849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4095, 1400, 0, 0, 2261, 7, 39, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 4096, 4096, 0, 1652, 4095, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 4096, 4096, 3360, 2921, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 3862, 0, 0, 3827, 4096, 0, 95, 61, 0, 2444, 2085, 983, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 736, 1042, 4096, 0, 0, 4029, 0, 1959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16507, 17691, 16797, 15988, 14144, 16267, 17721, 15957] avg=16384.0000 max=17721 max_rank=6 min=14144 min_rank=4 imbalance_ratio=1.081604 max_load_reduction=113351 max_load_reduction_pct=0.864799 moved_to_echo=64089 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[6, 10, 463, 4077, 0, 0, 0, 1999, 3915, 33, 78, 1, 13, 2, 65, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 2705, 0, 0, 46, 3997, 11, 0, 1505, 335, 0, 0, 1148, 15, 973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 4050, 0, 129, 0, 17, 0, 4062, 0, 4039, 3845, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 11, 0, 3983, 52, 2, 4019, 1367, 0, 41, 3092, 0, 82, 2982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 8, 28, 0, 4081, 4089, 12, 1643, 4058, 0, 0, 2, 36, 12, 3684, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 1, 25, 0, 5, 3, 0, 0, 0, 0, 47, 6, 0, 0, 2, 4067, 4055, 3316, 2872, 1692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 34, 32, 0, 69, 3, 0, 4009, 0, 0, 2524, 4079, 18, 0, 0, 27, 2442, 2075, 973, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 50, 0, 35, 723, 1674, 4051, 30, 48, 4080, 7, 4010, 0, 18, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14739, 14774, 16240, 15654, 18093, 16105, 16696, 18771] avg=16384.0000 max=18771 max_rank=7 min=14739 min_rank=0 imbalance_ratio=1.145691 max_load_reduction=112301 max_load_reduction_pct=0.856789 moved_to_echo=69665 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[70, 6, 0, 4095, 1, 0, 8, 2000, 4057, 241, 2, 0, 0, 0, 20, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 488, 0, 0, 0, 4091, 6, 0, 3605, 4032, 0, 1, 1175, 5, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4077, 0, 0, 2, 53, 44, 4046, 0, 4091, 3841, 24, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 4093, 0, 0, 3984, 1399, 0, 0, 1488, 0, 0, 2126, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 0, 1651, 4088, 21, 4, 0, 25, 0, 1866, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 91, 0, 0, 1, 13, 0, 49, 0, 84, 3, 1123, 8, 0, 28, 0, 4095, 4093, 3334, 2921, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 0, 0, 4096, 0, 0, 1014, 3969, 5, 0, 0, 0, 2444, 2073, 983, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 1, 0, 2, 731, 3981, 4043, 2, 10, 4091, 0, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14595, 21508, 16209, 13115, 15842, 17595, 15101, 17107] avg=16384.0000 max=21508 max_rank=1 min=13115 min_rank=3 imbalance_ratio=1.312744 max_load_reduction=109564 max_load_reduction_pct=0.835907 moved_to_echo=65645 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 3, 2009, 2847, 2512, 0, 0, 28, 7, 3873, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 3642, 0, 0, 0, 4092, 10, 0, 457, 90, 6, 1, 1174, 0, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 4095, 0, 1056, 0, 0, 0, 3845, 0, 4092, 3844, 19, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4068, 0, 0, 4096, 1397, 0, 0, 4077, 20, 16, 4094, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4088, 4094, 0, 1651, 4096, 0, 0, 0, 2, 0, 951, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 1, 2, 0, 0, 0, 0, 0, 1, 0, 4, 0, 2, 0, 0, 4085, 4093, 3351, 2920, 1711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 1, 4096, 0, 0, 64, 4086, 4, 3, 24, 6, 2443, 2083, 982, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4, 0, 735, 1305, 4092, 0, 0, 4076, 0, 144, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19469, 17223, 17042, 17777, 14887, 16182, 14042, 14450] avg=16384.0000 max=19469 max_rank=0 min=14042 min_rank=6 imbalance_ratio=1.188293 max_load_reduction=111603 max_load_reduction_pct=0.851463 moved_to_echo=59561 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 768, 32768, 128, 128, 128, 16128, 29440, 14592, 128, 128, 128, 128, 4864, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32512, 18304, 0, 0, 128, 32768, 128, 128, 6016, 7936, 128, 128, 9472, 128, 24192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[256, 128, 512, 128, 128, 128, 128, 32512, 128, 0, 19072, 32640, 128, 128, 256, 128, 19584, 16768, 7936, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 640, 32768, 128, 1664, 128, 128, 128, 32512, 128, 32768, 30848, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 32640, 128, 128, 32640, 11264, 0, 128, 26880, 128, 640, 27136, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 128, 5888, 10624, 32768, 128, 128, 32640, 128, 16512, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[3712, 128, 128, 128, 32768, 32768, 128, 13312, 32768, 128, 128, 128, 128, 128, 16000, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 1280, 128, 128, 128, 128, 32768, 32768, 26880, 23424, 13696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 3993, 0, 0, 1, 0, 4086, 0, 4, 3, 0, 4088, 2259, 4096, 0, 4081, 48, 4096, 4096, 40, 0, 287, 1, 109, 0, 0, 0, 0, 0, 0, 3564, 0, 0, 9, 4095, 7, 3304, 7, 2, 19, 55, 2, 0, 0, 2, 0, 0, 0, 1, 4027, 0, 4095, 0, 1, 4094, 0, 0, 0, 9, 0, 11, 4091, 0, 0, 0, 4096, 2, 4095, 3768, 0, 0, 1832, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4060, 4092, 0, 0, 0, 0, 4093, 0, 3502, 0, 0, 4091, 4056, 0, 4094, 4066, 338, 2, 25, 0, 0, 0, 0, 0, 4090, 1, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 1595, 4096, 2, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 53, 169, 0, 1, 0, 0, 4084, 1, 2, 0, 2, 4096, 3316, 4096, 0, 4087, 0, 4096, 4096, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4091, 0, 619, 0, 0, 0, 2971, 0, 0, 0, 0, 0, 0, 0, 0, 3920, 1, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 1026, 0, 4096, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4067, 0, 4093, 0, 0, 4096, 4079, 0, 4096, 4093, 1172, 32, 11, 0, 0, 298, 17, 0, 4096, 1, 0, 4096, 4031, 0, 0, 0, 0, 0, 0, 2873, 4089, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 326, 4033, 0, 1, 0, 0, 4096, 6, 27, 0, 0, 4096, 2645, 4096, 0, 4094, 6, 4096, 4096, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 3920, 4, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 3966, 0, 0, 1805, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4080, 0, 4096, 4086, 404, 0, 1, 0, 0, 68, 20, 13, 4096, 2, 0, 4094, 4035, 0, 0, 0, 0, 0, 0, 3229, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4079, 0, 0, 0, 0, 3945, 0, 110, 0, 431, 4096, 3930, 4096, 15, 4087, 1, 4092, 4096, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 510, 0, 0, 0, 1, 0, 0, 0, 0, 4082, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4085, 0, 4095, 3319, 1, 0, 45, 1, 4096, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 4093, 4096, 11, 0, 0, 0, 4001, 0, 4090, 0, 0, 4096, 4087, 0, 4095, 4096, 0, 0, 3, 0, 0, 0, 102, 0, 4088, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4085, 4096, 0, 18, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 38, 1250, 20, 1, 0, 28, 4053, 1, 193, 0, 0, 4026, 4068, 4051, 0, 4074, 63, 4037, 4059, 0, 8, 143, 96, 0, 0, 0, 0, 20, 2, 0, 4072, 35, 58, 13, 4050, 0, 1569, 0, 12, 136, 289, 0, 0, 18, 39, 52, 0, 0, 7, 4076, 20, 4078, 28, 18, 4085, 0, 10, 1, 160, 0, 0, 4079, 0, 0, 0, 4095, 0, 3916, 4026, 13, 0, 978, 90, 4072, 13, 0, 0, 0, 7, 5, 0, 2, 48, 8, 4039, 0, 4082, 3846, 0, 0, 17, 7, 3999, 0, 3893, 26, 10, 4078, 4042, 0, 4084, 4042, 889, 12, 0, 21, 4, 45, 396, 3, 4038, 12, 12, 3865, 3885, 1, 0, 0, 24, 0, 18, 2909, 3998, 1, 157, 206, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[8, 1068, 2249, 0, 0, 0, 2, 4020, 0, 1, 0, 0, 4034, 1182, 4089, 0, 4093, 0, 4092, 4094, 1, 0, 194, 9, 0, 2, 3, 0, 3, 0, 13, 4096, 8, 0, 0, 4096, 0, 3624, 0, 0, 0, 11, 4, 0, 0, 2, 0, 0, 126, 0, 3947, 278, 4056, 0, 0, 4095, 3, 2, 0, 42, 0, 0, 4096, 4, 0, 0, 4096, 3, 4091, 4078, 1, 0, 1156, 0, 4091, 0, 23, 0, 0, 10, 0, 0, 29, 0, 6, 4087, 0, 4093, 4093, 6, 5, 0, 0, 4094, 1, 4090, 1, 0, 4095, 4093, 0, 4095, 4090, 58, 0, 0, 0, 0, 77, 0, 10, 4095, 0, 0, 4058, 4053, 0, 4, 4, 0, 0, 0, 2522, 4075, 1, 0, 40, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 2921, 177, 0, 10, 0, 0, 4095, 1, 87, 0, 0, 4059, 1643, 4095, 0, 4083, 46, 4091, 4090, 2, 0, 2189, 0, 62, 0, 5, 0, 1, 19, 842, 4078, 6, 0, 0, 3821, 0, 1125, 0, 1, 0, 680, 0, 3, 0, 0, 0, 0, 4, 0, 3887, 0, 4071, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 3972, 5, 37, 0, 4055, 0, 4095, 4095, 0, 0, 68, 0, 4094, 0, 1, 0, 2, 24, 0, 0, 0, 0, 127, 4092, 0, 4095, 4087, 3, 2, 0, 1, 4093, 0, 4084, 4, 0, 4042, 4090, 0, 4091, 4087, 358, 0, 2, 3, 0, 0, 0, 314, 4083, 1, 0, 4096, 4096, 1, 3, 0, 0, 0, 2, 2563, 3914, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 145, 4059, 0, 0, 0, 0, 4087, 0, 304, 0, 32, 4064, 3007, 4096, 0, 4093, 3, 4096, 4096, 146, 0, 360, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4095, 0, 213, 1, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 1, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 3325, 0, 4096, 4084, 0, 0, 87, 83, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 0, 4061, 4096, 0, 0, 0, 0, 4096, 0, 3786, 0, 0, 4096, 3821, 1, 4096, 4096, 0, 0, 0, 0, 0, 7, 1181, 0, 4096, 0, 0, 4054, 4096, 0, 0, 16, 0, 0, 0, 4087, 4094, 0, 0, 14, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3993, 0, 0, 1, 0, 4086, 0, 4, 3, 0, 4088, 2259, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 48, 4096, 3590, 40, 0, 287, 1, 109, 0, 0, 0, 0, 0, 0, 3564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4095, 7, 3304, 7, 2, 19, 55, 2, 0, 0, 2, 0, 0, 4094, 3915, 1824, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4027, 0, 4095, 0, 1, 4094, 0, 0, 0, 9, 0, 11, 4091, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4095, 3768, 0, 0, 1832, 11, 3476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 4060, 4092, 0, 0, 0, 0, 4093, 0, 3502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3437, 4056, 0, 0, 4066, 338, 2, 25, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 1595, 4096, 2, 0, 0, 0, 620, 458, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16707, 15816, 17896, 16377, 17280, 15928, 16014, 15054] avg=16384.0000 max=17896 max_rank=2 min=15054 min_rank=7 imbalance_ratio=1.092285 max_load_reduction=113176 max_load_reduction_pct=0.863464 moved_to_echo=64276 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 53, 169, 0, 1, 0, 0, 4084, 1, 2, 0, 2, 4096, 3316, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 3590, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 619, 0, 0, 0, 2971, 0, 0, 0, 0, 0, 0, 4096, 3915, 1824, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3920, 1, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 4096, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 1026, 0, 3476, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 4095, 4096, 0, 0, 0, 0, 4067, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3441, 4079, 0, 0, 4093, 1172, 32, 11, 0, 0, 298, 17, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4031, 0, 0, 0, 0, 0, 0, 2873, 4089, 0, 0, 0, 0, 620, 458, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13996, 15905, 18078, 16261, 16800, 16532, 17239, 16261] avg=16384.0000 max=18078 max_rank=2 min=13996 min_rank=0 imbalance_ratio=1.103394 max_load_reduction=112994 max_load_reduction_pct=0.862076 moved_to_echo=66832 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4079, 0, 0, 0, 0, 3945, 0, 110, 0, 431, 4096, 3930, 2272, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 4092, 3590, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 510, 0, 0, 0, 1, 0, 0, 4095, 3915, 1824, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4095, 3319, 1, 0, 45, 1, 3476, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 4093, 4096, 11, 0, 0, 0, 4001, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3441, 4087, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 102, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4085, 4096, 0, 18, 1, 0, 620, 458, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18879, 15887, 15004, 16419, 15033, 16472, 15817, 17561] avg=16384.0000 max=18879 max_rank=0 min=15004 min_rank=2 imbalance_ratio=1.152283 max_load_reduction=112193 max_load_reduction_pct=0.855965 moved_to_echo=64883 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 326, 4033, 0, 1, 0, 0, 4096, 6, 27, 0, 0, 4096, 2645, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 4096, 3590, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 4096, 3915, 1824, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3920, 4, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 4096, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3966, 0, 0, 1805, 0, 3476, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3441, 4080, 0, 0, 4086, 404, 0, 1, 0, 0, 68, 20, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 4035, 0, 0, 0, 0, 0, 0, 3229, 4095, 0, 0, 0, 0, 620, 458, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17502, 15916, 14550, 16263, 17441, 16565, 16209, 16626] avg=16384.0000 max=17502 max_rank=0 min=14550 min_rank=2 imbalance_ratio=1.068237 max_load_reduction=113570 max_load_reduction_pct=0.866470 moved_to_echo=66841 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 145, 4059, 0, 0, 0, 0, 4087, 0, 304, 0, 32, 4064, 3007, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 4096, 3590, 146, 0, 360, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 213, 1, 0, 0, 106, 0, 0, 0, 0, 0, 0, 4096, 3914, 1824, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 1, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4078, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3325, 0, 4096, 4084, 0, 0, 87, 83, 3476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 181, 0, 4061, 4096, 0, 0, 0, 0, 4096, 0, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3441, 3821, 1, 0, 4096, 0, 0, 0, 0, 0, 7, 1181, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 4096, 0, 0, 16, 0, 0, 0, 4087, 4094, 0, 0, 14, 0, 620, 458, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17970, 16381, 14811, 16362, 15151, 16222, 16643, 17532] avg=16384.0000 max=17970 max_rank=0 min=14811 min_rank=2 imbalance_ratio=1.096802 max_load_reduction=113102 max_load_reduction_pct=0.862900 moved_to_echo=65548 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2921, 177, 0, 10, 0, 0, 4095, 1, 87, 0, 0, 4059, 1643, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 46, 4091, 3586, 2, 0, 2189, 0, 62, 0, 5, 0, 1, 19, 842, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3821, 0, 1125, 0, 1, 0, 680, 0, 3, 0, 0, 0, 0, 4091, 3911, 1824, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3887, 0, 4071, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 3972, 5, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 4055, 0, 4095, 4095, 0, 0, 68, 0, 3474, 0, 1, 0, 2, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 181, 0, 4095, 4087, 3, 2, 0, 1, 4093, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3396, 4090, 0, 0, 4087, 358, 0, 2, 3, 0, 0, 0, 314, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 1, 3, 0, 0, 0, 2, 2563, 3914, 0, 0, 0, 0, 620, 457, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15264, 19004, 16016, 16082, 15851, 16673, 16337, 15845] avg=16384.0000 max=19004 max_rank=1 min=15264 min_rank=0 imbalance_ratio=1.159912 max_load_reduction=112068 max_load_reduction_pct=0.855011 moved_to_echo=64706 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 38, 1250, 20, 1, 0, 28, 4053, 1, 193, 0, 0, 4026, 4068, 2244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 63, 4037, 3554, 0, 8, 143, 96, 0, 0, 0, 0, 20, 2, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 58, 13, 4050, 0, 1569, 0, 12, 136, 289, 0, 0, 18, 39, 52, 0, 4084, 3864, 1807, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4076, 20, 4078, 28, 18, 4085, 0, 10, 1, 160, 0, 0, 4079, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3916, 4026, 13, 0, 978, 90, 3449, 13, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 48, 8, 175, 0, 4082, 3846, 0, 0, 17, 7, 3999, 0, 3893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 10, 3421, 4042, 0, 0, 4042, 889, 12, 0, 21, 4, 45, 396, 3, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 3865, 3885, 1, 0, 0, 24, 0, 18, 2909, 3998, 1, 157, 206, 2, 623, 456, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15922, 16069, 16588, 16611, 16587, 16082, 16949, 16264] avg=16384.0000 max=16949 max_rank=6 min=15922 min_rank=0 imbalance_ratio=1.034485 max_load_reduction=114123 max_load_reduction_pct=0.870689 moved_to_echo=65882 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[8, 1068, 2249, 0, 0, 0, 2, 4020, 0, 1, 0, 0, 4034, 1182, 2268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4092, 3588, 1, 0, 194, 9, 0, 2, 3, 0, 3, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 3624, 0, 0, 0, 11, 4, 0, 0, 2, 0, 0, 4095, 3906, 1821, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 3947, 278, 4056, 0, 0, 4095, 3, 2, 0, 42, 0, 0, 4096, 4, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4091, 4078, 1, 0, 1156, 0, 3471, 0, 23, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 6, 181, 0, 4093, 4093, 6, 5, 0, 0, 4094, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3440, 4093, 0, 0, 4090, 58, 0, 0, 0, 0, 77, 0, 10, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 4053, 0, 4, 4, 0, 0, 0, 2522, 4075, 1, 0, 40, 1, 620, 458, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14832, 16094, 18129, 16697, 16929, 16598, 15864, 15929] avg=16384.0000 max=18129 max_rank=2 min=14832 min_rank=0 imbalance_ratio=1.106506 max_load_reduction=112943 max_load_reduction_pct=0.861687 moved_to_echo=65320 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 256, 32768, 28800, 256, 128, 3328, 128, 256, 128, 128, 0, 128, 128, 896, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 256, 1536, 0, 32768, 32512, 128, 128, 128, 128, 32640, 128, 31744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 27520, 32384, 128, 0, 32768, 3328, 128, 128, 128, 128, 512, 1792, 384, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 32000, 128, 32640, 31488, 128, 0, 7040, 256, 27776, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 4608, 20096, 128, 128, 128, 128, 32512, 128, 768, 128, 512, 32640, 22144, 18176, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[256, 128, 32000, 384, 32768, 128, 128, 32768, 128, 128, 128, 256, 128, 128, 32640, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 32512, 32512, 128, 128, 128, 128, 0, 128, 23936, 32512, 128, 256, 384, 128, 4992, 3712, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 32512, 128, 10496, 128, 128, 256, 4736, 128, 128, 128, 128, 128, 0, 32768, 31360, 14592, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[2522, 0, 3193, 0, 4096, 0, 0, 6, 0, 293, 0, 0, 3610, 58, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4091, 4096, 0, 4076, 0, 4072, 0, 0, 0, 0, 3820, 0, 0, 0, 0, 0, 0, 725, 0, 4096, 4095, 10, 0, 0, 0, 505, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1766, 0, 0, 82, 4096, 1, 4069, 0, 4071, 0, 16, 4094, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 457, 0, 0, 0, 4096, 0, 0, 0, 4084, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 9, 0, 0, 4073, 78, 0, 0, 0, 0, 1, 4096, 3460, 4096, 4095, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[3093, 0, 4087, 0, 4083, 1, 0, 79, 0, 117, 0, 81, 1645, 12, 0, 0, 0, 0, 5, 4096, 0, 4096, 5, 10, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 4090, 6, 4092, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 3481, 0, 4085, 4078, 6, 0, 0, 0, 2109, 0, 0, 0, 0, 5, 0, 4091, 0, 2, 701, 0, 0, 96, 4094, 3, 4055, 0, 3712, 0, 310, 4093, 0, 34, 0, 95, 47, 0, 4093, 0, 0, 0, 406, 0, 0, 0, 4074, 0, 0, 0, 4096, 0, 1, 0, 0, 8, 4070, 0, 0, 0, 1, 36, 1, 3964, 0, 4, 0, 0, 0, 0, 4086, 404, 4096, 4076, 0, 0, 0, 199, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1335, 21, 3957, 0, 3948, 146, 8, 0, 24, 0, 29, 787, 545, 0, 0, 5, 0, 20, 1, 4067, 6, 3885, 0, 58, 4007, 0, 7, 14, 0, 0, 0, 3919, 4088, 3783, 4029, 0, 4092, 18, 4077, 0, 28, 20, 23, 4051, 29, 26, 13, 47, 113, 0, 3708, 0, 4065, 4029, 0, 12, 36, 0, 0, 45, 0, 0, 0, 0, 0, 4083, 7, 0, 1937, 0, 10, 665, 4065, 54, 4002, 33, 2204, 0, 2341, 2817, 0, 0, 147, 590, 0, 0, 3945, 50, 3, 5, 1417, 0, 0, 80, 4038, 2, 37, 31, 3050, 0, 0, 0, 2, 68, 4083, 0, 12, 0, 11, 0, 0, 3745, 189, 0, 0, 0, 12, 39, 4054, 3790, 4082, 4041, 1, 7, 57, 0, 4079, 4066] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1212, 3, 3630, 0, 4095, 13, 0, 400, 0, 1326, 0, 94, 319, 1941, 0, 0, 0, 0, 0, 4088, 0, 4077, 7, 87, 4095, 0, 0, 0, 0, 72, 0, 4096, 4094, 4024, 4096, 2, 4096, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 2591, 0, 4096, 4096, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 34, 1, 0, 1784, 4096, 0, 4089, 0, 4001, 0, 1699, 4022, 0, 1, 0, 189, 48, 24, 4096, 0, 0, 0, 82, 0, 0, 0, 4096, 0, 0, 0, 4040, 0, 0, 0, 0, 121, 4060, 0, 0, 1, 0, 0, 0, 3595, 0, 0, 0, 0, 0, 85, 4096, 1443, 4094, 4087, 0, 0, 2, 39, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[2951, 0, 4084, 0, 4096, 0, 0, 3, 0, 19, 0, 0, 39, 269, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 4089, 0, 4063, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3820, 0, 4096, 4095, 9, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 3678, 4096, 1, 4094, 0, 2677, 0, 1, 3790, 0, 0, 0, 4, 3, 0, 4096, 0, 0, 0, 1063, 0, 0, 0, 4095, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 69, 0, 0, 4094, 0, 0, 0, 0, 0, 3, 4096, 2140, 4076, 4095, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3825, 0, 2682, 0, 4091, 12, 0, 526, 0, 543, 0, 3, 1582, 20, 0, 0, 0, 0, 2, 4087, 0, 4087, 5, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 3481, 4019, 4096, 0, 4090, 0, 3815, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2135, 0, 4096, 4029, 41, 0, 0, 5, 3892, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 917, 0, 0, 20, 4095, 0, 4092, 0, 3555, 0, 221, 4083, 0, 0, 0, 3, 72, 0, 4096, 0, 0, 0, 1180, 0, 0, 0, 4096, 0, 0, 0, 4091, 5, 0, 0, 0, 39, 4066, 0, 0, 111, 110, 0, 0, 4063, 5, 1, 6, 0, 0, 0, 4096, 111, 4096, 4096, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4037, 0, 4066, 0, 4096, 22, 2, 1, 2, 125, 0, 0, 3626, 0, 0, 0, 5, 1, 0, 3777, 4, 4095, 1, 0, 4095, 4, 4, 0, 0, 0, 0, 4086, 4096, 4085, 4091, 0, 4013, 1, 4096, 89, 33, 8, 1, 4094, 8, 0, 0, 0, 0, 1, 3894, 0, 4081, 4094, 19, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 4092, 0, 0, 0, 0, 2, 1201, 4055, 163, 4096, 0, 3959, 0, 216, 4086, 8, 0, 0, 0, 0, 298, 4083, 0, 0, 0, 6, 0, 0, 0, 4091, 0, 1, 2, 4037, 1, 0, 0, 3, 246, 4086, 10, 0, 33, 2, 0, 0, 2719, 0, 0, 0, 0, 0, 29, 4096, 371, 4038, 4095, 13, 0, 0, 0, 4076, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[728, 0, 4096, 0, 4096, 16, 0, 1, 3, 11, 0, 15, 497, 0, 0, 1, 3, 0, 0, 4065, 0, 4028, 0, 0, 4095, 0, 11, 0, 0, 19, 1, 4091, 4079, 4095, 4096, 0, 4096, 0, 4094, 0, 1, 0, 0, 4096, 3, 0, 0, 0, 29, 0, 2490, 4, 4096, 4092, 277, 0, 0, 0, 0, 4, 2, 0, 2, 0, 0, 4086, 2, 0, 28, 0, 0, 3443, 4093, 11, 4085, 0, 3937, 0, 1321, 3286, 3, 1, 2, 33, 0, 0, 4089, 1, 4, 1, 146, 0, 0, 0, 4075, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 726, 0, 0, 4096, 0, 34, 3, 0, 0, 0, 4094, 3617, 4093, 4093, 8, 0, 0, 0, 4074, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2522, 0, 3193, 0, 4096, 0, 0, 6, 0, 293, 0, 0, 3610, 58, 0, 0, 1920, 1846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 183, 0, 4072, 0, 0, 0, 0, 3820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 725, 0, 4096, 4095, 10, 0, 0, 0, 505, 0, 0, 0, 0, 0, 3705, 771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1766, 0, 0, 82, 2171, 1, 4069, 0, 4071, 0, 16, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 457, 0, 0, 0, 4096, 0, 4096, 3325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 9, 0, 0, 4073, 3893, 391, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 1, 2112, 3460, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17544, 16463, 16262, 13907, 16270, 16072, 16616, 17938] avg=16384.0000 max=17938 max_rank=7 min=13907 min_rank=3 imbalance_ratio=1.094849 max_load_reduction=113134 max_load_reduction_pct=0.863144 moved_to_echo=66896 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3093, 0, 4087, 0, 4083, 1, 0, 79, 0, 117, 0, 81, 1645, 12, 0, 0, 1915, 1845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 4096, 5, 10, 4096, 0, 0, 0, 0, 0, 0, 4096, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 183, 6, 4092, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3481, 0, 4085, 4078, 6, 0, 0, 0, 2109, 0, 0, 0, 0, 5, 3706, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 701, 0, 0, 96, 2170, 3, 4055, 0, 3712, 0, 310, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 95, 47, 0, 4093, 0, 0, 0, 406, 0, 0, 0, 4074, 0, 4096, 3321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 8, 4070, 0, 0, 0, 1, 36, 1, 3964, 3907, 390, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 2107, 404, 4096, 4076, 0, 0, 0, 199, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16958, 16483, 16563, 18240, 15142, 16166, 16538, 14982] avg=16384.0000 max=18240 max_rank=3 min=14982 min_rank=7 imbalance_ratio=1.113281 max_load_reduction=112832 max_load_reduction_pct=0.860840 moved_to_echo=62828 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1212, 3, 3630, 0, 4095, 13, 0, 400, 0, 1326, 0, 94, 319, 1941, 0, 0, 1920, 1846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4077, 7, 87, 4095, 0, 0, 0, 0, 72, 0, 4096, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4024, 0, 2, 183, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2591, 0, 4096, 4096, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 3706, 771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 34, 1, 0, 1784, 2171, 0, 4089, 0, 4001, 0, 1699, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 189, 48, 24, 4096, 0, 0, 0, 82, 0, 0, 0, 4096, 0, 4096, 3325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 0, 0, 121, 4060, 0, 0, 1, 0, 0, 0, 3595, 3913, 390, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 2112, 1443, 4094, 4087, 0, 0, 2, 39, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16799, 16601, 16490, 15278, 17805, 15957, 16184, 15958] avg=16384.0000 max=17805 max_rank=4 min=15278 min_rank=3 imbalance_ratio=1.086731 max_load_reduction=113267 max_load_reduction_pct=0.864159 moved_to_echo=65904 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2951, 0, 4084, 0, 4096, 0, 0, 3, 0, 19, 0, 0, 39, 269, 0, 0, 1920, 1846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 183, 0, 4063, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3820, 0, 4096, 4095, 9, 0, 0, 0, 17, 0, 0, 0, 0, 0, 3706, 771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3678, 2171, 1, 4094, 0, 2677, 0, 1, 3790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 4096, 0, 0, 0, 1063, 0, 0, 0, 4095, 0, 4096, 3325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 69, 0, 0, 4094, 3906, 390, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2112, 2140, 4076, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15227, 16463, 16534, 16514, 16417, 16682, 16713, 16522] avg=16384.0000 max=16713 max_rank=6 min=15227 min_rank=0 imbalance_ratio=1.020081 max_load_reduction=114359 max_load_reduction_pct=0.872490 moved_to_echo=66334 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3825, 0, 2682, 0, 4091, 12, 0, 526, 0, 543, 0, 3, 1582, 20, 0, 0, 1920, 1845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4087, 0, 4087, 5, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3481, 4019, 0, 0, 183, 0, 3815, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2135, 0, 4096, 4029, 41, 0, 0, 5, 3892, 0, 0, 0, 0, 0, 3706, 771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 917, 0, 0, 20, 2171, 0, 4092, 0, 3555, 0, 221, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 72, 0, 4096, 0, 0, 0, 1180, 0, 0, 0, 4096, 0, 4096, 3325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 5, 0, 0, 0, 39, 4066, 0, 0, 111, 110, 0, 0, 4063, 3907, 390, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 6, 0, 0, 0, 2112, 111, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17049, 16454, 15598, 18675, 15059, 16868, 16846, 14523] avg=16384.0000 max=18675 max_rank=3 min=14523 min_rank=7 imbalance_ratio=1.139832 max_load_reduction=112397 max_load_reduction_pct=0.857521 moved_to_echo=63296 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1335, 21, 3957, 0, 3948, 146, 8, 0, 24, 0, 29, 787, 545, 0, 0, 5, 1903, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1, 4067, 6, 3885, 0, 58, 4007, 0, 7, 14, 0, 0, 0, 3919, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3783, 0, 0, 180, 18, 4077, 0, 28, 20, 23, 4051, 29, 26, 13, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 3708, 0, 4065, 4029, 0, 12, 36, 0, 0, 45, 0, 0, 0, 0, 3646, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1937, 0, 10, 665, 2147, 54, 4002, 33, 2204, 0, 2341, 2817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 590, 0, 0, 3945, 50, 3, 5, 1417, 0, 0, 80, 4038, 2, 4066, 3314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 31, 3050, 0, 0, 0, 2, 68, 4083, 0, 12, 0, 11, 0, 0, 3745, 3912, 383, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 0, 12, 39, 2085, 3790, 4082, 4041, 1, 7, 57, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14544, 16066, 16383, 16423, 16217, 17657, 15400, 18382] avg=16384.0000 max=18382 max_rank=7 min=14544 min_rank=0 imbalance_ratio=1.121948 max_load_reduction=112690 max_load_reduction_pct=0.859756 moved_to_echo=67656 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4037, 0, 4066, 0, 4096, 22, 2, 1, 2, 125, 0, 0, 3626, 0, 0, 0, 1920, 1827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 3777, 4, 4095, 1, 0, 4095, 4, 4, 0, 0, 0, 0, 4086, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 180, 1, 4096, 89, 33, 8, 1, 4094, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3894, 0, 4081, 4094, 19, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3702, 771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 2, 1201, 2150, 163, 4096, 0, 3959, 0, 216, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 298, 4083, 0, 0, 0, 6, 0, 0, 0, 4091, 0, 4096, 3321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4037, 1, 0, 0, 3, 246, 4086, 10, 0, 33, 2, 0, 0, 2719, 3833, 389, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 2112, 371, 4038, 4095, 13, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19724, 16150, 16691, 16565, 15879, 15903, 15426, 14734] avg=16384.0000 max=19724 max_rank=0 min=14734 min_rank=7 imbalance_ratio=1.203857 max_load_reduction=111348 max_load_reduction_pct=0.849518 moved_to_echo=61942 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[728, 0, 4096, 0, 4096, 16, 0, 1, 3, 11, 0, 15, 497, 0, 0, 1, 1919, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4065, 0, 4028, 0, 0, 4095, 0, 11, 0, 0, 19, 1, 4091, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4095, 0, 0, 183, 0, 4094, 0, 1, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 2490, 4, 4096, 4092, 277, 0, 0, 0, 0, 4, 2, 0, 2, 0, 3705, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 28, 0, 0, 3443, 2170, 11, 4085, 0, 3937, 0, 1321, 3286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 2, 33, 0, 0, 4089, 1, 4, 1, 146, 0, 0, 0, 4075, 0, 4095, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 726, 0, 0, 4096, 3913, 391, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 3, 0, 0, 0, 2111, 3617, 4093, 4093, 8, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13227, 16392, 16551, 15470, 18283, 15767, 17349, 18033] avg=16384.0000 max=18283 max_rank=4 min=13227 min_rank=0 imbalance_ratio=1.115906 max_load_reduction=112789 max_load_reduction_pct=0.860512 moved_to_echo=69432 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[19712, 128, 29824, 0, 32640, 256, 128, 1024, 128, 2560, 128, 1024, 11904, 2304, 0, 128, 15360, 14848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 5504, 128, 128, 11008, 17408, 256, 32640, 128, 28160, 0, 6144, 30336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 32384, 128, 32512, 128, 256, 32768, 128, 128, 128, 0, 128, 128, 32640, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[256, 128, 22912, 128, 32768, 32640, 384, 128, 128, 128, 6528, 128, 128, 0, 128, 128, 29696, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 256, 1024, 256, 384, 32640, 128, 128, 128, 4864, 0, 0, 128, 32768, 128, 32768, 26624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 31616, 128, 128, 0, 128, 512, 32768, 128, 128, 256, 1024, 128, 128, 30464, 31232, 3200, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 128, 128, 0, 128, 256, 16896, 15360, 32768, 32768, 128, 128, 128, 256, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32128, 32384, 0, 128, 1536, 128, 32512, 128, 128, 128, 128, 32512, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 26, 0, 0, 2279, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 7, 4096, 4096, 0, 4096, 0, 0, 4096, 3905, 0, 0, 0, 0, 0, 90, 1443, 468, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4011, 0, 0, 2, 0, 4061, 3933, 0, 0, 0, 0, 0, 0, 2343, 0, 0, 4080, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 5, 4096, 0, 4096, 2022, 1, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4092, 0, 4096, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 44, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 227, 0, 0, 4006, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 157, 4096, 4096, 0, 4096, 0, 0, 4096, 2479, 0, 0, 0, 0, 0, 5, 3900, 466, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 1799, 0, 2036, 3874, 0, 0, 0, 0, 0, 0, 2923, 0, 0, 4096, 0, 1, 0, 0, 0, 2078, 4096, 0, 0, 0, 0, 0, 66, 0, 0, 1, 4096, 0, 4096, 1083, 0, 4085, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4069, 0, 19, 22, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 10, 40, 3986, 12, 16, 51, 3984, 29, 4023, 58, 0, 3965, 38, 1, 0, 32, 0, 3897, 0, 0, 1, 29, 4057, 6, 0, 0, 162, 4021, 0, 4075, 4073, 23, 4071, 8, 1, 4035, 2046, 0, 0, 0, 0, 0, 0, 3461, 0, 3, 108, 5, 4031, 0, 60, 19, 4004, 0, 3911, 3989, 0, 0, 3872, 10, 1217, 207, 0, 9, 69, 13, 39, 126, 555, 1, 1, 3906, 1, 0, 12, 0, 0, 2677, 4068, 0, 1, 14, 0, 33, 76, 35, 0, 0, 4082, 0, 4071, 4021, 0, 3957, 0, 1, 126, 130, 4065, 0, 4075, 0, 4059, 4, 4053, 26, 0, 3954, 0, 0, 0, 37, 68, 0, 30, 8, 0, 242, 4086, 0, 58, 76, 3578, 1, 2890, 90, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 11, 4096, 0, 4096, 0, 0, 4034, 0, 0, 91, 0, 7, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4096, 34, 4079, 4093, 0, 4089, 0, 20, 4096, 9, 0, 0, 0, 20, 0, 0, 2748, 160, 0, 0, 0, 4075, 0, 0, 0, 4096, 1, 4096, 4094, 0, 0, 3, 0, 3985, 2753, 0, 0, 0, 0, 0, 13, 4015, 0, 53, 4058, 0, 0, 0, 0, 0, 4092, 4095, 0, 0, 3, 0, 0, 0, 0, 0, 3, 4096, 0, 4096, 1074, 0, 4095, 0, 0, 2, 0, 4096, 0, 4096, 27, 4067, 0, 4096, 0, 0, 4021, 0, 13, 0, 0, 5, 3, 0, 0, 0, 0, 4083, 0, 0, 0, 4095, 0, 1613, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4084, 1, 4095, 0, 1, 4096, 0, 0, 0, 0, 0, 2602, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 173, 4096, 4096, 0, 4092, 0, 1, 4096, 40, 0, 0, 0, 0, 0, 0, 3825, 1499, 0, 0, 0, 4093, 11, 0, 0, 4081, 0, 4095, 4096, 0, 0, 3475, 0, 4043, 1333, 2, 0, 7, 0, 11, 0, 1, 0, 0, 3013, 0, 0, 0, 0, 0, 3814, 4093, 5, 0, 0, 0, 0, 16, 0, 2, 0, 4095, 0, 4096, 3002, 943, 4096, 0, 0, 1, 0, 4093, 0, 4096, 4, 3973, 0, 4084, 0, 0, 4096, 1, 72, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 168, 0, 0, 4096, 0, 749, 35, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4, 4096, 0, 0, 0, 4093, 3, 4094, 61, 0, 4063, 0, 0, 0, 0, 0, 838, 0, 0, 0, 9, 4095, 0, 0, 0, 1, 4095, 3, 4096, 4094, 0, 4092, 0, 3, 4096, 41, 0, 0, 0, 1, 0, 0, 3677, 0, 0, 0, 0, 4092, 0, 0, 12, 4095, 0, 4093, 4090, 0, 4, 3202, 4, 4036, 993, 0, 0, 1, 0, 0, 2, 4092, 0, 0, 4093, 0, 19, 0, 0, 0, 3969, 4093, 0, 0, 13, 0, 0, 0, 0, 0, 49, 4086, 0, 4088, 0, 38, 4078, 0, 4, 11, 7, 4095, 0, 4095, 4, 4093, 0, 4086, 2, 0, 3772, 0, 4, 0, 0, 5, 0, 1, 3, 2, 0, 4095, 12, 0, 1, 3960, 0, 4023, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 0, 4086, 0, 0, 0, 4039, 4, 4085, 0, 1, 4095, 0, 0, 1, 0, 0, 4083, 2, 1, 1, 0, 4096, 0, 0, 91, 0, 4086, 40, 4084, 4096, 3, 4095, 8, 0, 4026, 503, 0, 29, 0, 19, 0, 24, 3897, 0, 0, 26, 1, 4096, 23, 0, 0, 2978, 0, 4030, 3969, 0, 0, 1455, 0, 4096, 154, 57, 2, 6, 0, 0, 56, 3918, 6, 0, 3575, 1, 0, 19, 0, 15, 4085, 4092, 6, 4, 0, 2, 0, 0, 5, 0, 306, 4094, 0, 4083, 1946, 2, 4090, 37, 1, 6, 16, 4093, 0, 4061, 2, 4096, 0, 4075, 0, 0, 4031, 0, 139, 0, 1, 253, 0, 0, 0, 14, 13, 4094, 1, 1, 0, 4092, 2, 1334, 16, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4094, 7, 0, 4077, 0, 0, 3, 0, 11, 4096, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 4096, 20, 4096, 4096, 0, 4096, 0, 0, 4091, 1456, 0, 1, 0, 0, 0, 4, 6, 3995, 0, 1, 0, 4096, 0, 0, 0, 4080, 0, 4093, 4096, 0, 0, 5, 0, 3707, 3974, 0, 0, 0, 0, 0, 0, 287, 0, 0, 4083, 2, 0, 0, 1, 0, 4017, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 8, 4096, 0, 4096, 868, 6, 4096, 0, 0, 0, 0, 4040, 0, 4096, 0, 4096, 0, 4096, 0, 0, 2685, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 3579, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 0, 0, 4096, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2279, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 7, 4096, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3905, 0, 0, 0, 0, 0, 90, 1443, 468, 0, 0, 1631, 1534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1095, 4011, 0, 0, 2, 0, 4061, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2343, 0, 0, 4080, 0, 0, 0, 0, 0, 4095, 4096, 1677, 829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 5, 4096, 0, 4096, 2022, 1, 4096, 0, 2172, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 297, 0, 4092, 0, 4096, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 44, 0, 4096, 2265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16403, 15362, 17263, 17198, 17120, 16502, 16627, 14597] avg=16384.0000 max=17263 max_rank=2 min=14597 min_rank=7 imbalance_ratio=1.053650 max_load_reduction=113809 max_load_reduction_pct=0.868294 moved_to_echo=64846 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 11, 4096, 0, 4089, 0, 0, 4034, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4096, 34, 4079, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 20, 4096, 9, 0, 0, 0, 20, 0, 0, 2748, 160, 0, 0, 1629, 1534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 1095, 4094, 0, 0, 3, 0, 3985, 2753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4015, 0, 53, 4058, 0, 0, 0, 0, 0, 4092, 4095, 1676, 829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 4096, 0, 4096, 1074, 0, 4095, 0, 2172, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 297, 27, 4067, 0, 4096, 0, 0, 4021, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 0, 0, 0, 0, 4083, 0, 0, 0, 4095, 0, 1613, 3, 0, 4075, 2265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16416, 17186, 14305, 16027, 18831, 15546, 16619, 16142] avg=16384.0000 max=18831 max_rank=4 min=14305 min_rank=2 imbalance_ratio=1.149353 max_load_reduction=112241 max_load_reduction_pct=0.856331 moved_to_echo=67138 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 0, 0, 4096, 0, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 157, 4096, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2479, 0, 0, 0, 0, 0, 5, 3900, 466, 0, 0, 1631, 1534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1095, 4096, 0, 0, 1799, 0, 2036, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2923, 0, 0, 4096, 0, 1, 0, 0, 0, 2078, 4096, 1677, 829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 1, 4096, 0, 4096, 1083, 0, 4085, 0, 2172, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 297, 0, 4096, 0, 4096, 0, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4069, 0, 19, 22, 0, 4096, 2265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16604, 17238, 18207, 16995, 15700, 15606, 16155, 14567] avg=16384.0000 max=18207 max_rank=2 min=14567 min_rank=7 imbalance_ratio=1.111267 max_load_reduction=112865 max_load_reduction_pct=0.861092 moved_to_echo=62028 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4084, 1, 4088, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2602, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 173, 4096, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 4096, 40, 0, 0, 0, 0, 0, 0, 3825, 1499, 0, 0, 1631, 1534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4081, 0, 1094, 4096, 0, 0, 3475, 0, 4043, 1333, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 11, 0, 1, 0, 0, 3013, 0, 0, 0, 0, 0, 3814, 4093, 1677, 829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 16, 0, 2, 0, 4095, 0, 4096, 3002, 943, 4096, 0, 2172, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 297, 4, 3973, 0, 4084, 0, 0, 4096, 1, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 168, 0, 0, 4096, 0, 749, 35, 0, 4093, 2265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16366, 15851, 16718, 18135, 13445, 18434, 16621, 15502] avg=16384.0000 max=18434 max_rank=5 min=13445 min_rank=4 imbalance_ratio=1.125122 max_load_reduction=112638 max_load_reduction_pct=0.859360 moved_to_echo=64002 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4087, 7, 0, 4077, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 4096, 20, 4096, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 1456, 0, 1, 0, 0, 0, 4, 6, 3995, 0, 1, 1631, 1534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 1094, 4096, 0, 0, 5, 0, 3707, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 0, 4083, 2, 0, 0, 1, 0, 4017, 4096, 1677, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8, 4096, 0, 4096, 868, 6, 4096, 0, 2171, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 297, 0, 4096, 0, 4096, 0, 0, 2685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 3579, 2, 0, 4096, 2265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16366, 17181, 16815, 16956, 14991, 15359, 15214, 18190] avg=16384.0000 max=18190 max_rank=7 min=14991 min_rank=4 imbalance_ratio=1.110229 max_load_reduction=112882 max_load_reduction_pct=0.861221 moved_to_echo=63754 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 40, 3986, 12, 16, 51, 3984, 29, 4015, 58, 0, 3965, 38, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 3897, 0, 0, 1, 29, 4057, 6, 0, 0, 162, 4021, 0, 4075, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4071, 8, 1, 4035, 2046, 0, 0, 0, 0, 0, 0, 3461, 0, 3, 108, 1624, 1529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 60, 19, 4004, 0, 1038, 3989, 0, 0, 3872, 10, 1217, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 69, 13, 39, 126, 555, 1, 1, 3906, 1, 0, 12, 0, 0, 2677, 4068, 1672, 795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 33, 76, 35, 0, 0, 4082, 0, 4071, 4021, 0, 3957, 0, 2078, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 126, 130, 4065, 0, 289, 0, 4059, 4, 4053, 26, 0, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 68, 0, 30, 8, 0, 242, 4086, 0, 58, 76, 3578, 1, 2890, 90, 1, 4031, 2257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16205, 17057, 16909, 14421, 13944, 18376, 16707, 17453] avg=16384.0000 max=18376 max_rank=5 min=13944 min_rank=4 imbalance_ratio=1.121582 max_load_reduction=112696 max_load_reduction_pct=0.859802 moved_to_echo=66480 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4086, 0, 0, 0, 4039, 4, 4078, 0, 1, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 2, 1, 1, 0, 4096, 0, 0, 91, 0, 4086, 40, 4084, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 8, 0, 4026, 503, 0, 29, 0, 19, 0, 24, 3897, 0, 0, 26, 1631, 1521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 23, 0, 0, 2978, 0, 1077, 3969, 0, 0, 1455, 0, 4096, 154, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 56, 3918, 6, 0, 3575, 1, 0, 19, 0, 15, 4085, 4092, 1677, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 2, 0, 0, 5, 0, 306, 4094, 0, 4083, 1946, 2, 4090, 37, 2137, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 16, 4093, 0, 294, 2, 4096, 0, 4075, 0, 0, 4031, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 253, 0, 0, 0, 14, 13, 4094, 1, 1, 0, 4092, 2, 1334, 16, 0, 4096, 2246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16305, 17272, 15782, 13810, 18268, 16719, 16753, 16163] avg=16384.0000 max=18268 max_rank=4 min=13810 min_rank=3 imbalance_ratio=1.114990 max_load_reduction=112804 max_load_reduction_pct=0.860626 moved_to_echo=67903 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 4096, 0, 0, 0, 4093, 3, 4087, 61, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 838, 0, 0, 0, 9, 4095, 0, 0, 0, 1, 4095, 3, 4096, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3, 4096, 41, 0, 0, 0, 1, 0, 0, 3677, 0, 0, 0, 1630, 1533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4095, 0, 1094, 4090, 0, 4, 3202, 4, 4036, 993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 4092, 0, 0, 4093, 0, 19, 0, 0, 0, 3969, 4093, 1676, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 49, 4086, 0, 4088, 0, 38, 4078, 0, 2171, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 11, 7, 4095, 0, 298, 4, 4093, 0, 4086, 2, 0, 3772, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 3, 2, 0, 4095, 12, 0, 1, 3960, 0, 4023, 0, 0, 4092, 2264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 13925, 15073, 17530, 18773, 14530, 16376, 18458] avg=16384.0000 max=18773 max_rank=4 min=13925 min_rank=1 imbalance_ratio=1.145813 max_load_reduction=112299 max_load_reduction_pct=0.856773 moved_to_echo=68137 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 128, 32640, 128, 32640, 128, 128, 32640, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 31616, 128, 8704, 32512, 0, 128, 13824, 128, 27264, 17280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 256, 256, 32768, 0, 2432, 128, 32640, 128, 32768, 128, 0, 30208, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 25984, 128, 128, 128, 128, 32768, 128, 0, 128, 256, 32768, 512, 32768, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 128, 32640, 10496, 0, 128, 0, 128, 0, 128, 23040, 6656, 128, 256, 13056, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 256, 18176, 128, 128, 30976, 128, 128, 128, 128, 128, 28928, 32768, 13440, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 512, 128, 128, 128, 128, 256, 32768, 256, 128, 128, 32128, 128, 14208, 256, 128, 32768, 18176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 256, 128, 128, 384, 32768, 0, 32768, 14080, 1024, 32640, 128, 17280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 17, 0, 4085, 0, 0, 0, 2683, 0, 0, 1, 0, 0, 0, 0, 126, 0, 4096, 0, 723, 0, 0, 0, 4, 0, 4095, 0, 44, 1950, 0, 0, 4096, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 2215, 0, 4093, 0, 0, 4096, 0, 4096, 0, 0, 3881, 0, 2862, 4094, 0, 4095, 4096, 4090, 0, 4096, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2341, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 4096, 39, 0, 0, 0, 4087, 0, 0, 4096, 0, 0, 4096, 0, 2, 4096, 0, 4067, 0, 0, 1, 0, 4096, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 3525, 0, 0, 4096, 0, 4096, 0, 0, 0, 121, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4095, 155, 0, 4096, 0, 0, 0, 3973, 0, 0, 7, 0, 0, 0, 0, 0, 1, 4095, 0, 40, 0, 0, 5, 49, 0, 4092, 1, 4, 4078, 0, 0, 4095, 0, 12, 0, 4096, 0, 0, 0, 0, 0, 528, 0, 4093, 0, 1, 4096, 38, 4094, 1, 0, 4096, 0, 3670, 4032, 0, 4080, 4096, 4096, 0, 4095, 4083, 0, 0, 4096, 6, 1, 0, 0, 0, 0, 0, 47, 0, 0, 0, 1283, 0, 0, 0, 0, 2, 0, 4088, 1, 0, 4, 0, 4037, 0, 2080, 4096, 9, 0, 4096, 0, 390, 4095, 0, 4096, 0, 0, 0, 4, 4096, 0, 4095, 0, 0, 7, 0, 0, 0, 0, 3566, 0, 0, 4096, 0, 4095, 0, 0, 19, 674, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4076, 0, 0, 3531, 0, 0, 0, 2938, 0, 0, 1152, 0, 0, 0, 0, 0, 18, 4094, 0, 228, 0, 0, 0, 0, 0, 4096, 0, 0, 3815, 0, 0, 4096, 2808, 0, 0, 4093, 0, 0, 0, 0, 0, 1200, 0, 4090, 0, 0, 4096, 3, 4096, 0, 0, 4096, 0, 19, 4094, 0, 4095, 4095, 4096, 0, 4095, 4083, 1, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 1732, 1, 0, 0, 3746, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 4046, 0, 1, 4096, 3, 5, 4095, 0, 447, 4096, 0, 4087, 0, 0, 17, 0, 4095, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 2997, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4059, 240, 6, 4014, 2, 26, 0, 4025, 0, 63, 0, 0, 0, 98, 154, 0, 0, 4036, 0, 0, 0, 2, 39, 958, 0, 4068, 108, 9, 4078, 17, 42, 4064, 0, 149, 0, 4056, 51, 0, 19, 0, 29, 0, 0, 4033, 1, 0, 4045, 0, 3968, 16, 0, 3629, 0, 4071, 4010, 2, 3935, 4071, 3884, 0, 4076, 2394, 2, 62, 4080, 0, 24, 19, 72, 58, 22, 0, 4087, 0, 0, 4, 3859, 4, 0, 0, 0, 13, 17, 4071, 352, 48, 90, 10, 3968, 50, 0, 4084, 0, 0, 4019, 2, 483, 4038, 53, 3790, 16, 1, 0, 0, 3844, 0, 4048, 0, 0, 10, 0, 0, 1, 0, 1038, 9, 27, 4031, 0, 3933, 29, 65, 0, 0, 0, 0, 44, 48] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4084, 100, 5, 4073, 7, 5, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 55, 380, 0, 25, 0, 0, 1, 4094, 0, 69, 4039, 0, 6, 4096, 995, 35, 0, 4091, 1, 11, 0, 5, 3, 4013, 84, 4094, 48, 0, 4054, 1, 4072, 0, 0, 3944, 0, 77, 4096, 0, 3910, 4083, 4010, 27, 4096, 4074, 0, 3, 4095, 0, 1, 0, 0, 14, 0, 1, 3728, 0, 0, 0, 4033, 2, 0, 4, 7, 0, 0, 3539, 3, 2, 0, 6, 3300, 0, 201, 4092, 0, 0, 4090, 0, 649, 4033, 0, 4088, 0, 4, 0, 12, 4087, 0, 4091, 5, 0, 88, 0, 0, 2, 1, 3710, 0, 0, 4082, 0, 4093, 0, 0, 0, 0, 0, 0, 14, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 11, 0, 4032, 0, 0, 0, 3979, 0, 0, 119, 0, 0, 0, 0, 0, 0, 4096, 0, 119, 0, 0, 0, 11, 0, 4095, 0, 56, 4092, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 3901, 0, 3998, 4096, 0, 4095, 4096, 4068, 0, 4096, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4089, 38, 0, 0, 0, 4096, 0, 125, 4096, 0, 0, 4096, 0, 6, 4096, 0, 4069, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3092, 1, 0, 0, 1166, 0, 0, 392, 0, 0, 0, 0, 0, 0, 4096, 0, 67, 0, 0, 0, 72, 0, 4096, 0, 0, 3828, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 2034, 18, 4086, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 3700, 4050, 0, 4035, 4096, 4096, 0, 4096, 4093, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 1973, 0, 0, 0, 2978, 0, 0, 0, 0, 0, 0, 4094, 4, 0, 0, 0, 4090, 0, 2065, 4096, 0, 0, 4095, 0, 966, 4096, 0, 4096, 0, 0, 12, 0, 4096, 0, 4095, 0, 2, 1531, 0, 0, 0, 0, 270, 0, 0, 4096, 0, 4095, 0, 0, 0, 534, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4094, 143, 0, 4088, 0, 6, 0, 4093, 0, 0, 0, 0, 0, 2, 0, 64, 13, 4081, 0, 61, 1, 1, 0, 210, 0, 4095, 0, 0, 4017, 0, 2, 4093, 23, 0, 0, 4090, 2, 0, 0, 0, 2, 29, 17, 4079, 0, 0, 4089, 0, 4094, 0, 0, 3956, 0, 19, 4085, 0, 4095, 4095, 4035, 0, 4096, 3893, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 3929, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4008, 3, 0, 7, 0, 4088, 0, 29, 4096, 0, 1, 4095, 0, 185, 4095, 5, 4096, 0, 19, 8, 8, 4077, 0, 4092, 0, 0, 0, 0, 0, 0, 2, 4086, 0, 0, 4073, 0, 4084, 2, 9, 0, 9, 0, 11, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4095, 155, 0, 4096, 0, 0, 0, 3973, 0, 0, 7, 0, 0, 0, 0, 0, 3524, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 40, 0, 0, 5, 49, 0, 4092, 1, 4, 4078, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4096, 0, 0, 0, 0, 0, 528, 0, 4093, 0, 1, 4096, 38, 1358, 514, 423, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 3670, 0, 0, 4080, 0, 2253, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 1, 0, 0, 0, 0, 0, 47, 0, 0, 0, 1283, 0, 0, 0, 4095, 2407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4088, 1, 0, 4, 0, 4037, 0, 2080, 3673, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 390, 4095, 0, 4096, 0, 0, 0, 4, 4096, 0, 3581, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3566, 0, 0, 4096, 0, 4095, 0, 0, 19, 674, 0, 0, 0, 0, 4032, 570, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17539, 16411, 15208, 18183, 11935, 17990, 16269, 17537] avg=16384.0000 max=18183 max_rank=3 min=11935 min_rank=4 imbalance_ratio=1.109802 max_load_reduction=112889 max_load_reduction_pct=0.861275 moved_to_echo=63731 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 17, 0, 4085, 0, 0, 0, 2683, 0, 0, 1, 0, 0, 0, 0, 126, 3524, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 723, 0, 0, 0, 4, 0, 4095, 0, 44, 1950, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 2215, 0, 4093, 0, 0, 4096, 0, 1356, 514, 423, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 2862, 0, 0, 4095, 0, 2250, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2341, 0, 0, 0, 4080, 0, 0, 0, 4096, 2408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 39, 0, 0, 0, 4087, 0, 0, 3673, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 4067, 0, 0, 1, 0, 4096, 0, 3582, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3525, 0, 0, 4096, 0, 4096, 0, 0, 0, 121, 0, 0, 0, 0, 4094, 572, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16220, 14959, 16858, 17180, 17021, 15991, 15855, 16988] avg=16384.0000 max=17180 max_rank=3 min=14959 min_rank=1 imbalance_ratio=1.048584 max_load_reduction=113892 max_load_reduction_pct=0.868927 moved_to_echo=65855 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4076, 0, 0, 3531, 0, 0, 0, 2938, 0, 0, 1152, 0, 0, 0, 0, 0, 3525, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4094, 0, 228, 0, 0, 0, 0, 0, 4096, 0, 0, 3815, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2808, 0, 0, 4093, 0, 0, 0, 0, 0, 1200, 0, 4090, 0, 0, 4096, 3, 1358, 514, 423, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 4095, 0, 2253, 0, 0, 4083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 1732, 1, 0, 0, 3746, 0, 0, 0, 4095, 2407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 4046, 0, 1, 3673, 3, 5, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 447, 4096, 0, 4087, 0, 0, 17, 0, 4095, 0, 3582, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2997, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 571, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16910, 16298, 18634, 14547, 16078, 15935, 16332, 16338] avg=16384.0000 max=18634 max_rank=2 min=14547 min_rank=3 imbalance_ratio=1.137329 max_load_reduction=112438 max_load_reduction_pct=0.857834 moved_to_echo=64683 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4059, 240, 6, 4014, 2, 26, 0, 4025, 0, 63, 0, 0, 0, 98, 154, 0, 3415, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 2, 39, 958, 0, 4068, 108, 9, 4078, 17, 42, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 4056, 51, 0, 19, 0, 29, 0, 0, 4033, 1, 0, 4045, 0, 1292, 512, 427, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3629, 0, 4071, 0, 2, 3935, 0, 2129, 0, 0, 2394, 2, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 24, 19, 72, 58, 22, 0, 4087, 0, 0, 4, 3859, 4, 0, 0, 4076, 2392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 17, 4071, 352, 48, 90, 10, 3968, 50, 0, 3657, 0, 0, 4019, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 4038, 53, 3790, 16, 1, 0, 0, 3844, 0, 3536, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1038, 9, 27, 4031, 0, 3933, 29, 65, 0, 0, 0, 0, 44, 48, 4010, 553, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17781, 17365, 14670, 16240, 18697, 16297, 15771, 14251] avg=16384.0000 max=18697 max_rank=4 min=14251 min_rank=7 imbalance_ratio=1.141174 max_load_reduction=112375 max_load_reduction_pct=0.857353 moved_to_echo=65016 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 3092, 1, 0, 0, 1166, 0, 0, 392, 0, 0, 0, 0, 0, 3524, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 67, 0, 0, 0, 72, 0, 4096, 0, 0, 3828, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 2034, 18, 4086, 0, 0, 4096, 0, 1358, 514, 423, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3700, 0, 0, 4035, 0, 2253, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 1973, 0, 0, 0, 2978, 0, 0, 0, 4096, 2408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4, 0, 0, 0, 4090, 0, 2065, 3673, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 966, 4096, 0, 4096, 0, 0, 12, 0, 4096, 0, 3581, 0, 2, 1531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 0, 0, 4096, 0, 4095, 0, 0, 0, 534, 0, 0, 0, 0, 4050, 572, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13959, 16206, 16675, 18177, 15552, 18021, 18380, 14102] avg=16384.0000 max=18380 max_rank=6 min=13959 min_rank=0 imbalance_ratio=1.121826 max_load_reduction=112692 max_load_reduction_pct=0.859772 moved_to_echo=66055 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 11, 0, 4032, 0, 0, 0, 3979, 0, 0, 119, 0, 0, 0, 0, 0, 3524, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 119, 0, 0, 0, 11, 0, 4095, 0, 56, 4092, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 4096, 0, 1349, 514, 423, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 0, 3998, 0, 0, 4095, 0, 2238, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 4096, 0, 0, 0, 4096, 2408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 38, 0, 0, 0, 4096, 0, 125, 3673, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 4069, 0, 0, 0, 0, 4096, 0, 3582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 572, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17449, 16516, 14623, 18322, 14796, 16117, 15849, 17400] avg=16384.0000 max=18322 max_rank=3 min=14623 min_rank=2 imbalance_ratio=1.118286 max_load_reduction=112750 max_load_reduction_pct=0.860214 moved_to_echo=64162 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4084, 100, 5, 4073, 7, 5, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 3505, 1683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 55, 380, 0, 25, 0, 0, 1, 4094, 0, 69, 4039, 0, 6, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 995, 35, 0, 4091, 1, 11, 0, 5, 3, 4013, 84, 4094, 48, 0, 4054, 1, 1330, 513, 423, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3944, 0, 77, 0, 0, 3910, 0, 2205, 27, 0, 4074, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 14, 0, 1, 3728, 0, 0, 0, 4033, 2, 0, 4, 4096, 2400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3539, 3, 2, 0, 6, 3300, 0, 201, 3669, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 4033, 0, 4088, 0, 4, 0, 12, 4087, 0, 3578, 5, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 3710, 0, 0, 4082, 0, 4093, 0, 0, 0, 0, 0, 0, 14, 1, 4096, 567, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13514, 16792, 19750, 14240, 18374, 14817, 16544, 17041] avg=16384.0000 max=19750 max_rank=2 min=13514 min_rank=0 imbalance_ratio=1.205444 max_load_reduction=111322 max_load_reduction_pct=0.849319 moved_to_echo=66776 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4094, 143, 0, 4088, 0, 6, 0, 4093, 0, 0, 0, 0, 0, 2, 0, 64, 3523, 1687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4081, 0, 61, 1, 1, 0, 210, 0, 4095, 0, 0, 4017, 0, 2, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 4090, 2, 0, 0, 0, 2, 29, 17, 4079, 0, 0, 4089, 0, 1338, 513, 423, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 19, 0, 0, 4095, 0, 2220, 0, 0, 3893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 3929, 0, 0, 0, 4089, 0, 0, 0, 4096, 2408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 3, 0, 7, 0, 4088, 0, 29, 3673, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 4095, 5, 4096, 0, 19, 8, 8, 4077, 0, 3579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4086, 0, 0, 4073, 0, 4084, 2, 9, 0, 9, 0, 11, 0, 6, 4085, 571, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17700, 16525, 14654, 14183, 18619, 15904, 16072, 17415] avg=16384.0000 max=18619 max_rank=4 min=14183 min_rank=3 imbalance_ratio=1.136414 max_load_reduction=112453 max_load_reduction_pct=0.857948 moved_to_echo=68010 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 768, 128, 31104, 128, 128, 0, 22912, 0, 128, 1792, 0, 0, 128, 256, 256, 28160, 13568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 32128, 512, 128, 128, 128, 31744, 128, 4608, 29440, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 0, 31616, 0, 18432, 0, 128, 32384, 0, 17920, 128, 0, 30848, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3200, 32768, 128, 32512, 128, 128, 128, 128, 32512, 0, 28672, 128, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 128, 18048, 128, 0, 128, 28288, 128, 0, 128, 32768, 19328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 23296, 128, 128, 32768, 0, 32640, 128, 128, 128, 1408, 0, 128, 128, 128, 32640, 4608, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 1664, 128, 128, 128, 1408, 128, 32768, 128, 256, 29952, 128, 128, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[3968, 256, 0, 32768, 128, 128, 128, 128, 128, 10112, 128, 32768, 128, 128, 32768, 128, 10752, 4224, 3456, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 144, 0, 4096, 0, 1, 253, 0, 4096, 68, 4096, 0, 0, 0, 0, 0, 0, 8, 4095, 0, 0, 16, 0, 0, 4096, 13, 3902, 0, 4, 0, 4046, 0, 0, 4095, 4096, 0, 0, 0, 3823, 4015, 4082, 0, 4063, 0, 0, 3, 0, 113, 0, 4096, 4096, 4096, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4072, 0, 0, 0, 0, 0, 4088, 44, 0, 2, 4095, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 24, 0, 0, 0, 4053, 4096, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 3043, 609, 0, 4096, 74, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3108, 0, 0, 0, 2915, 1, 0, 4094, 4096, 0, 0, 0, 4096, 4074, 4096, 0, 2063, 0, 0, 0, 0, 1, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4084, 4096, 0, 0, 0, 75, 73, 4096, 0, 0, 0, 4096, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 3526, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 944, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 55, 22, 4078, 42, 2165, 0, 0, 3975, 0, 3984, 0, 16, 2, 23, 87, 0, 157, 4058, 0, 11, 0, 0, 77, 4027, 0, 0, 12, 0, 48, 2975, 3124, 34, 4067, 3793, 67, 0, 1, 3327, 4051, 4075, 8, 3407, 13, 0, 0, 14, 1169, 0, 4078, 3918, 4073, 0, 0, 17, 0, 158, 3, 114, 30, 0, 0, 0, 4078, 0, 71, 18, 4037, 4069, 3703, 4082, 0, 3, 48, 26, 0, 4059, 299, 284, 0, 4045, 40, 0, 4090, 12, 0, 2, 0, 12, 27, 2955, 27, 3960, 0, 4, 7, 4080, 1, 0, 20, 0, 4037, 12, 0, 0, 0, 136, 4085, 4071, 0, 3970, 4055, 37, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 1235, 14, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 3, 0, 4096, 0, 3879, 0, 0, 4094, 0, 4096, 1105, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 31, 3, 0, 4091, 12, 3972, 0, 0, 0, 184, 136, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 4089, 0, 3762, 0, 0, 0, 0, 126, 0, 4061, 4092, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 4096, 0, 0, 0, 4095, 4093, 4096, 4096, 0, 0, 0, 8, 47, 4096, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 3165, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[6, 0, 2, 4095, 0, 1751, 2617, 0, 4073, 0, 4079, 1, 0, 0, 1, 1383, 0, 0, 4095, 0, 0, 9, 1, 20, 4008, 0, 4073, 1, 104, 4, 16, 1673, 0, 4052, 4071, 0, 0, 0, 3939, 4064, 4096, 5, 3126, 0, 0, 0, 0, 0, 0, 4096, 4096, 4087, 0, 0, 2, 0, 169, 0, 16, 3, 1, 82, 2, 4081, 1, 39, 21, 4095, 4086, 2166, 4095, 0, 0, 0, 456, 2, 4091, 12, 1, 124, 4092, 0, 0, 4096, 0, 1, 0, 61, 0, 0, 3072, 0, 4076, 0, 0, 1, 4093, 0, 0, 0, 0, 4095, 0, 0, 3, 3, 0, 3999, 4090, 0, 4090, 4095, 0, 0, 0, 2, 0, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 84, 84, 0, 4096, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3876, 0, 0, 0, 884, 3882, 0, 4096, 4096, 0, 0, 0, 4091, 4086, 4096, 0, 2916, 0, 0, 0, 0, 452, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 10, 0, 4096, 79, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 2589, 2484, 0, 4096, 451, 4096, 0, 0, 0, 0, 0, 0, 7, 4093, 0, 0, 0, 0, 2, 4095, 2, 2060, 0, 0, 0, 63, 8, 0, 4096, 4096, 1, 0, 0, 4096, 4094, 4096, 0, 4062, 0, 0, 0, 0, 670, 0, 4096, 4032, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 4095, 4096, 4092, 4096, 0, 0, 32, 89, 0, 4096, 30, 7, 0, 4096, 0, 0, 4060, 0, 0, 0, 1, 0, 0, 3972, 0, 4096, 8, 0, 0, 4078, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4083, 4096, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 5, 24, 1628, 5, 4085, 46, 4096, 88, 0, 0, 3, 18, 0, 0, 4092, 0, 3, 0, 3, 1, 4090, 0, 3468, 0, 0, 0, 3861, 340, 4, 4096, 4094, 0, 0, 0, 4095, 3899, 4091, 0, 2676, 0, 1, 0, 0, 25, 2, 4093, 4087, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3, 19, 0, 4096, 3, 0, 0, 4093, 4095, 4073, 4095, 0, 0, 0, 0, 0, 4095, 309, 50, 0, 4095, 0, 0, 4096, 0, 7, 0, 0, 2, 0, 4057, 4, 4080, 0, 10, 0, 4093, 6, 0, 3, 3, 4096, 0, 0, 0, 0, 0, 4095, 4095, 2, 4091, 4094, 0, 1, 0, 2, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 144, 0, 4096, 0, 1, 253, 0, 4096, 68, 4096, 0, 0, 0, 0, 0, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4095, 0, 0, 16, 0, 0, 4096, 13, 3902, 0, 4, 0, 4046, 0, 1874, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 741, 4096, 0, 0, 0, 3823, 4015, 0, 0, 4063, 0, 0, 3, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4062, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 33, 0, 0, 0, 0, 0, 4088, 44, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 453, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 4096, 0, 24, 0, 0, 0, 4053, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 4069, 3831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13781, 18262, 16854, 16380, 16455, 16867, 16393, 16080] avg=16384.0000 max=18262 max_rank=1 min=13781 min_rank=0 imbalance_ratio=1.114624 max_load_reduction=112810 max_load_reduction_pct=0.860672 moved_to_echo=65795 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 3043, 609, 0, 4096, 74, 4096, 3, 0, 0, 0, 0, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3108, 0, 0, 0, 2915, 1, 1873, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 742, 4096, 0, 0, 0, 4096, 4074, 0, 0, 2063, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4084, 33, 0, 0, 0, 75, 73, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 3526, 0, 4096, 0, 0, 0, 452, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 944, 0, 0, 0, 0, 4096, 4096, 4070, 3854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17044, 16298, 15072, 16350, 16553, 16285, 16410, 17060] avg=16384.0000 max=17060 max_rank=7 min=15072 min_rank=2 imbalance_ratio=1.041260 max_load_reduction=114012 max_load_reduction_pct=0.869843 moved_to_echo=66308 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 4096, 0, 3879, 0, 0, 4094, 0, 4096, 1105, 0, 0, 0, 0, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 31, 3, 0, 4091, 12, 3972, 0, 0, 0, 184, 136, 1874, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 742, 4096, 0, 0, 0, 4096, 4096, 0, 0, 3762, 0, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 4092, 4061, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4093, 4096, 33, 0, 0, 0, 8, 47, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 3165, 0, 4096, 1, 0, 0, 453, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4089, 4070, 3854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18300, 14608, 16918, 16315, 16468, 15937, 16410, 16116] avg=16384.0000 max=18300 max_rank=0 min=14608 min_rank=1 imbalance_ratio=1.116943 max_load_reduction=112772 max_load_reduction_pct=0.860382 moved_to_echo=64931 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 55, 22, 4078, 42, 2165, 0, 0, 3975, 0, 3984, 0, 16, 2, 23, 87, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 4058, 0, 11, 0, 0, 77, 4027, 0, 0, 12, 0, 48, 2975, 3124, 1864, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 725, 3793, 67, 0, 1, 3327, 4051, 0, 8, 3407, 13, 0, 0, 14, 1169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 3918, 4033, 0, 0, 17, 0, 158, 3, 114, 30, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 18, 4037, 4069, 3703, 25, 0, 3, 48, 26, 0, 4059, 299, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 40, 0, 4090, 12, 0, 2, 0, 12, 27, 2955, 27, 3960, 0, 4, 7, 452, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 1, 0, 20, 0, 4037, 12, 0, 0, 0, 136, 4085, 0, 0, 3970, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 1235, 14, 0, 2, 0, 4071, 4075, 4058, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15475, 16566, 16609, 16429, 16642, 15673, 16338, 17340] avg=16384.0000 max=17340 max_rank=7 min=15475 min_rank=0 imbalance_ratio=1.058350 max_load_reduction=113732 max_load_reduction_pct=0.867706 moved_to_echo=65993 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 2, 4095, 0, 1751, 2617, 0, 4073, 0, 4079, 1, 0, 0, 1, 1383, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 9, 1, 20, 4008, 0, 4073, 1, 104, 4, 16, 1673, 1854, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 734, 4071, 0, 0, 0, 3939, 4064, 0, 5, 3126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4053, 0, 0, 2, 0, 169, 0, 16, 3, 1, 82, 2, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 39, 21, 4095, 4086, 2166, 33, 0, 0, 0, 456, 2, 4091, 12, 1, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 1, 0, 61, 0, 0, 3072, 0, 4076, 0, 0, 1, 448, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 4095, 0, 0, 3, 3, 0, 3999, 0, 0, 4090, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 4067, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19024, 16067, 15939, 16601, 15127, 15881, 16311, 16122] avg=16384.0000 max=19024 max_rank=0 min=15127 min_rank=4 imbalance_ratio=1.161133 max_load_reduction=112048 max_load_reduction_pct=0.854858 moved_to_echo=63441 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 2589, 2484, 0, 4096, 451, 4096, 0, 0, 0, 0, 0, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4093, 0, 0, 0, 0, 2, 4095, 2, 2060, 0, 0, 0, 63, 8, 1874, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 742, 4096, 1, 0, 0, 4096, 4094, 0, 0, 4062, 0, 0, 0, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4032, 4062, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4092, 33, 0, 0, 32, 89, 0, 4096, 30, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4060, 0, 0, 0, 1, 0, 0, 3972, 0, 4096, 8, 0, 0, 453, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4083, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4052, 3854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18839, 12413, 17761, 16287, 16570, 16720, 16384, 16098] avg=16384.0000 max=18839 max_rank=0 min=12413 min_rank=1 imbalance_ratio=1.149841 max_load_reduction=112233 max_load_reduction_pct=0.856270 moved_to_echo=65772 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 84, 84, 0, 4096, 0, 4096, 8, 0, 0, 0, 0, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3876, 0, 0, 0, 884, 3882, 1874, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 742, 4096, 0, 0, 0, 4091, 4086, 0, 0, 2916, 0, 0, 0, 0, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 33, 0, 0, 0, 10, 0, 4096, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 453, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 4096, 4096, 4070, 3854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13491, 18917, 16383, 16350, 16506, 16871, 16410, 16144] avg=16384.0000 max=18917 max_rank=1 min=13491 min_rank=0 imbalance_ratio=1.154602 max_load_reduction=112155 max_load_reduction_pct=0.855675 moved_to_echo=65931 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 5, 24, 1628, 5, 4085, 46, 4096, 88, 0, 0, 3, 18, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3, 0, 3, 1, 4090, 0, 3468, 0, 0, 0, 3861, 340, 1874, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 742, 4094, 0, 0, 0, 4095, 3899, 0, 0, 2676, 0, 1, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 4087, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 3, 19, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 4095, 4073, 33, 0, 0, 0, 0, 0, 4095, 309, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 7, 0, 0, 2, 0, 4057, 4, 4080, 0, 10, 0, 453, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 6, 0, 3, 3, 4096, 0, 0, 0, 0, 0, 4095, 0, 2, 4091, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 4095, 4091, 4067, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15118, 17941, 15536, 16360, 16751, 16838, 16416, 16112] avg=16384.0000 max=17941 max_rank=1 min=15118 min_rank=0 imbalance_ratio=1.095032 max_load_reduction=113131 max_load_reduction_pct=0.863121 moved_to_echo=66117 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 256, 128, 32768, 128, 13568, 7680, 128, 32640, 640, 32640, 1280, 128, 128, 128, 1536, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 256, 32768, 0, 128, 128, 128, 128, 32640, 128, 24576, 128, 128, 128, 14976, 9216, 14976, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 6016, 32512, 128, 0, 128, 31616, 32384, 0, 128, 26112, 128, 128, 128, 128, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 128, 0, 32768, 128, 128, 128, 128, 128, 128, 29056, 128, 32640, 128, 128, 128, 3712, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 32640, 32512, 0, 0, 128, 128, 384, 128, 256, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 256, 32640, 0, 128, 32640, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 0, 128, 0, 0, 128, 128, 128, 128, 128, 2304, 128, 0, 128, 128, 32768, 32768, 32640, 30848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32768, 32768, 30464, 256, 0, 128, 128, 768, 128, 32768, 896, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 35, 4096, 0, 4096, 580, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4081, 4096, 0, 0, 4096, 3391, 4096, 9, 11, 0, 4061, 4096, 20, 3890, 0, 0, 0, 0, 0, 4085, 0, 4091, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 245, 0, 0, 0, 4096, 0, 4096, 4096, 0, 5, 0, 0, 43, 0, 0, 0, 0, 0, 2, 4022, 1334, 0, 0, 0, 0, 0, 0, 0, 0, 2903, 4092, 0, 76, 0, 4096, 0, 4006, 0, 30, 4093, 0, 0, 4096, 3976, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 78, 4096, 0, 4096, 0, 4094, 4096, 4089, 0, 4, 0, 505, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4089, 4095, 4096, 0, 118, 0, 3830, 4096, 1, 0, 0, 0, 0, 0, 0, 3941, 16, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4071, 3223, 1, 0, 2, 0, 0, 0, 0, 0, 2112, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 660, 0, 188, 0, 4096, 0, 4096, 4093, 0, 0, 0, 0, 2256, 0, 0, 0, 0, 2, 319, 4091, 136, 0, 0, 1, 0, 10, 0, 0, 0, 2689, 4096, 0, 100, 0, 4096, 0, 3885, 0, 64, 4096, 0, 0, 4096, 4067, 0, 5, 0, 4086, 0, 497, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[23, 0, 31, 2, 0, 4050, 14, 4004, 2230, 4032, 4074, 3825, 31, 6, 82, 0, 46, 55, 69, 4058, 0, 4043, 4074, 120, 0, 4061, 2971, 3998, 0, 0, 23, 4062, 3956, 0, 625, 0, 0, 63, 3, 67, 4065, 96, 4019, 0, 23, 3, 62, 53, 4059, 4, 0, 0, 3790, 4, 0, 37, 0, 10, 0, 0, 97, 171, 83, 30, 53, 0, 0, 273, 1, 0, 0, 58, 7, 4038, 1, 7, 13, 1874, 2, 0, 0, 4074, 0, 4062, 4049, 2, 78, 7, 0, 0, 10, 0, 28, 76, 0, 3928, 3985, 0, 0, 36, 0, 3, 0, 0, 21, 282, 3961, 4033, 0, 732, 4, 4011, 0, 4002, 0, 14, 3900, 6, 0, 3905, 4089, 40, 0, 0, 4066, 0, 0, 37] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 4096, 0, 4096, 0, 4084, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4075, 4089, 0, 0, 4075, 3725, 4095, 0, 17, 0, 3947, 4096, 0, 121, 2, 9, 0, 0, 0, 3863, 0, 4079, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4041, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 229, 0, 0, 0, 0, 12, 0, 0, 4060, 2, 0, 60, 1104, 0, 466, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 172, 0, 0, 0, 0, 0, 3383, 4096, 198, 0, 0, 0, 0, 9, 0, 0, 10, 3630, 4092, 2, 13, 1, 4096, 0, 4096, 0, 2, 4088, 0, 0, 4089, 3707, 0, 34, 0, 4096, 13, 13, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4088, 1, 4083, 4, 4094, 4096, 4029, 0, 0, 5, 66, 0, 0, 15, 4093, 0, 4095, 4089, 0, 1, 4096, 4092, 4083, 0, 283, 0, 4094, 4091, 0, 162, 0, 0, 0, 34, 12, 4081, 29, 4096, 185, 1, 3, 1, 0, 4095, 1, 0, 7, 4092, 3860, 0, 0, 0, 0, 54, 6, 0, 2, 2054, 1, 0, 374, 0, 0, 0, 0, 0, 0, 0, 3969, 0, 0, 0, 89, 1, 0, 0, 4095, 5, 4095, 4063, 0, 0, 0, 0, 0, 30, 5, 1, 0, 2, 3248, 4014, 3794, 0, 0, 0, 0, 99, 0, 3, 9, 1, 4093, 0, 0, 12, 4090, 0, 2343, 53, 214, 4094, 1, 0, 4091, 3812, 0, 0, 0, 4094, 0, 1, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 17, 4093, 1, 4096, 0, 4094, 4089, 4068, 0, 2, 0, 0, 0, 0, 1, 4094, 0, 4052, 4095, 0, 0, 4096, 3057, 4090, 0, 14, 0, 3988, 4093, 0, 6, 0, 1, 0, 10, 13, 4079, 1358, 4096, 6, 0, 0, 0, 7, 4093, 0, 7, 0, 4084, 944, 0, 2, 0, 0, 1, 5, 3, 78, 3928, 0, 4, 113, 1, 0, 0, 4, 13, 5, 0, 4096, 0, 1, 0, 154, 0, 0, 0, 4095, 0, 4096, 4089, 0, 0, 0, 1, 0, 0, 24, 0, 0, 0, 1420, 3757, 1779, 2, 0, 0, 0, 0, 0, 0, 41, 3913, 4080, 0, 0, 0, 4096, 1, 4090, 0, 20, 4094, 0, 5, 4091, 4096, 0, 0, 0, 4091, 1, 36, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096, 17, 4096, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4085, 4096, 0, 0, 4096, 4046, 4096, 8, 316, 0, 4094, 4096, 0, 285, 0, 0, 0, 0, 0, 4095, 6, 4096, 20, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 3657, 0, 0, 0, 0, 0, 0, 0, 1, 536, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 151, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3461, 4096, 3843, 0, 0, 0, 0, 0, 0, 0, 0, 208, 4095, 0, 83, 0, 4096, 0, 4096, 0, 31, 4096, 0, 0, 4096, 3839, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096, 2, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 4096, 0, 0, 4096, 3087, 4096, 3, 10, 0, 4092, 4096, 0, 2, 0, 0, 0, 0, 0, 4091, 0, 4096, 26, 0, 0, 0, 0, 4096, 0, 0, 0, 4076, 4012, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 13, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 195, 4086, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 4096, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 1158, 4095, 0, 1, 0, 4096, 0, 4087, 0, 0, 4096, 0, 0, 4096, 3747, 0, 0, 0, 4096, 0, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 35, 3707, 0, 4096, 580, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 508, 3391, 4096, 9, 11, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 20, 3890, 0, 0, 0, 0, 0, 4085, 0, 4091, 2, 0, 0, 0, 0, 2387, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2833, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 245, 0, 0, 4096, 4096, 2808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 5, 0, 0, 43, 0, 0, 0, 0, 0, 2, 1201, 389, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 1334, 0, 0, 0, 0, 0, 0, 0, 0, 2903, 4092, 0, 76, 0, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 30, 4093, 0, 0, 4096, 3976, 0, 0, 0, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16609, 16157, 19471, 16343, 15477, 14118, 16333, 16564] avg=16384.0000 max=19471 max_rank=2 min=14118 min_rank=5 imbalance_ratio=1.188416 max_load_reduction=111601 max_load_reduction_pct=0.851448 moved_to_echo=62492 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 3707, 0, 4096, 0, 4084, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4075, 0, 0, 0, 505, 3725, 4095, 0, 17, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 121, 2, 9, 0, 0, 0, 3863, 0, 4079, 0, 0, 0, 0, 0, 2375, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4041, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 2833, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 0, 0, 12, 0, 0, 4060, 2, 0, 60, 1104, 0, 466, 4095, 4089, 2808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 172, 0, 0, 0, 0, 0, 3383, 1195, 389, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 198, 0, 0, 0, 0, 9, 0, 0, 10, 3630, 4092, 2, 13, 1, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4088, 0, 0, 4089, 3707, 0, 34, 0, 363, 13, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15986, 16368, 15445, 16369, 16925, 17617, 15957, 16405] avg=16384.0000 max=17617 max_rank=5 min=15445 min_rank=2 imbalance_ratio=1.075256 max_load_reduction=113455 max_load_reduction_pct=0.865593 moved_to_echo=66904 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 78, 3707, 0, 4096, 0, 4094, 0, 4089, 0, 4, 0, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 507, 4095, 4096, 0, 118, 0, 3830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 3941, 16, 4096, 0, 0, 0, 0, 0, 2383, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4071, 3223, 1, 0, 2, 0, 0, 0, 0, 0, 2112, 0, 2826, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 660, 0, 188, 4096, 4096, 2808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4093, 0, 0, 0, 0, 2256, 0, 0, 0, 0, 2, 319, 1199, 389, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 136, 0, 0, 1, 0, 10, 0, 0, 0, 2689, 4096, 0, 100, 0, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3885, 0, 64, 4096, 0, 0, 4096, 4067, 0, 5, 0, 362, 0, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16573, 16742, 15431, 17619, 15966, 16640, 15029, 17072] avg=16384.0000 max=17619 max_rank=3 min=15029 min_rank=6 imbalance_ratio=1.075378 max_load_reduction=113453 max_load_reduction_pct=0.865578 moved_to_echo=64707 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[23, 0, 31, 2, 0, 3661, 14, 4004, 2230, 4032, 0, 3825, 31, 6, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 55, 69, 0, 0, 4043, 0, 120, 0, 496, 2971, 3998, 0, 0, 23, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 625, 0, 0, 63, 3, 67, 4065, 96, 4019, 0, 23, 3, 62, 53, 2373, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4, 0, 0, 3790, 4, 0, 37, 0, 10, 0, 0, 97, 171, 83, 30, 2816, 1277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 273, 1, 0, 0, 58, 7, 4038, 1, 7, 13, 1874, 2, 0, 4074, 4074, 2781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 4062, 4049, 2, 78, 7, 0, 0, 10, 0, 28, 76, 0, 3928, 1192, 389, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 36, 0, 3, 0, 0, 21, 282, 3961, 4033, 0, 732, 4, 3822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 14, 3900, 6, 0, 3905, 4089, 40, 0, 0, 353, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17941, 15883, 16305, 12378, 17256, 18084, 16879, 16346] avg=16384.0000 max=18084 max_rank=5 min=12378 min_rank=3 imbalance_ratio=1.103760 max_load_reduction=112988 max_load_reduction_pct=0.862030 moved_to_echo=68565 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3700, 1, 4083, 4, 4094, 0, 4029, 0, 0, 5, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4095, 0, 0, 1, 508, 4092, 4083, 0, 283, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 162, 0, 0, 0, 34, 12, 4081, 29, 4096, 185, 1, 3, 1, 0, 2387, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 7, 4092, 3860, 0, 0, 0, 0, 54, 6, 0, 2, 2054, 1, 2832, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 0, 0, 0, 0, 0, 0, 0, 3969, 0, 0, 0, 89, 1, 0, 4096, 4089, 2806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 4095, 4063, 0, 0, 0, 0, 0, 30, 5, 1, 0, 2, 3248, 1201, 388, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 3794, 0, 0, 0, 0, 99, 0, 3, 9, 1, 4093, 0, 0, 12, 3901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2343, 53, 214, 4094, 1, 0, 4091, 3812, 0, 0, 0, 362, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15982, 17171, 15982, 18291, 15424, 17322, 15926, 14974] avg=16384.0000 max=18291 max_rank=3 min=14974 min_rank=7 imbalance_ratio=1.116394 max_load_reduction=112781 max_load_reduction_pct=0.860451 moved_to_echo=63646 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3707, 0, 4096, 2, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 508, 3087, 4096, 3, 10, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4091, 0, 4096, 26, 0, 0, 0, 0, 2387, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4076, 4012, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 2833, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 195, 4086, 0, 0, 4096, 4096, 2808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 1201, 389, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 1158, 4095, 0, 1, 0, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 0, 4096, 3747, 0, 0, 0, 363, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15995, 15888, 15598, 16312, 19391, 14257, 17239, 16392] avg=16384.0000 max=19391 max_rank=4 min=14257 min_rank=5 imbalance_ratio=1.183533 max_load_reduction=111681 max_load_reduction_pct=0.852058 moved_to_echo=67279 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3707, 0, 4096, 17, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 508, 4046, 4096, 8, 316, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 285, 0, 0, 0, 0, 0, 4095, 6, 4096, 20, 0, 0, 0, 0, 2387, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 3657, 0, 0, 0, 0, 0, 0, 0, 1, 536, 0, 2833, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 151, 0, 0, 4096, 4096, 2808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3461, 1201, 389, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3843, 0, 0, 0, 0, 0, 0, 0, 0, 208, 4095, 0, 83, 0, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 31, 4096, 0, 0, 4096, 3839, 0, 0, 0, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16004, 17153, 15885, 16498, 15251, 17529, 16231, 16521] avg=16384.0000 max=17529 max_rank=5 min=15251 min_rank=4 imbalance_ratio=1.069885 max_load_reduction=113543 max_load_reduction_pct=0.866264 moved_to_echo=65532 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 17, 3704, 1, 4096, 0, 4094, 0, 4068, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4052, 0, 0, 0, 508, 3057, 4090, 0, 14, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 6, 0, 1, 0, 10, 13, 4079, 1358, 4096, 6, 0, 0, 0, 7, 2387, 899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 7, 0, 4084, 944, 0, 2, 0, 0, 1, 5, 3, 78, 3928, 0, 2830, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 113, 1, 0, 0, 4, 13, 5, 0, 4096, 0, 1, 0, 154, 0, 0, 4089, 4095, 2807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4089, 0, 0, 0, 1, 0, 0, 24, 0, 0, 0, 1420, 1201, 389, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3757, 1779, 2, 0, 0, 0, 0, 0, 0, 41, 3913, 4080, 0, 0, 0, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 20, 4094, 0, 5, 4091, 4096, 0, 0, 0, 362, 1, 36, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15982, 15710, 16955, 17262, 15382, 15505, 17478, 16798] avg=16384.0000 max=17478 max_rank=6 min=15382 min_rank=4 imbalance_ratio=1.066772 max_load_reduction=113594 max_load_reduction_pct=0.866653 moved_to_echo=65163 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32256, 15104, 128, 128, 128, 128, 128, 0, 128, 384, 18560, 32768, 128, 1024, 128, 31232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 0, 128, 32640, 0, 128, 128, 4096, 28544, 32768, 128, 896, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 768, 128, 512, 128, 128, 128, 128, 128, 32640, 128, 128, 384, 8448, 128, 768, 32768, 32768, 22528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 256, 29696, 128, 32768, 2944, 32768, 0, 32384, 128, 128, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 32768, 128, 32768, 32768, 128, 128, 128, 128, 2560, 128, 128, 128, 128, 128, 16000, 9600, 3200, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 128, 128, 32384, 23936, 128, 128, 128, 128, 128, 128, 128, 256, 8832, 128, 22656, 10368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32640, 128, 5120, 128, 128, 128, 128, 128, 32384, 1536, 32768, 256, 128, 128, 128, 128, 19072, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 30720, 128, 384, 32640, 128, 128, 32640, 31360, 128, 128, 0, 2944, 128, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1341, 4096, 0, 0, 1, 0, 0, 4095, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 4057, 4096, 3992, 0, 0, 1, 4096, 0, 0, 13, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3920, 4096, 4083, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 3192, 505, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 3238, 4096, 0, 0, 0, 0, 4096, 30, 0, 0, 0, 4096, 0, 0, 4096, 4093, 0, 4096, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[727, 4057, 0, 0, 718, 11, 0, 4077, 4068, 4071, 0, 21, 37, 22, 2, 0, 52, 0, 54, 0, 12, 4070, 4056, 4068, 0, 60, 0, 4046, 25, 1, 342, 0, 27, 0, 0, 118, 16, 0, 30, 292, 4044, 3407, 4075, 4034, 4, 32, 47, 0, 0, 0, 98, 8, 0, 0, 3681, 4021, 3941, 0, 5, 23, 0, 0, 43, 4069, 3407, 0, 70, 10, 3937, 0, 0, 4, 5, 4061, 10, 4058, 0, 0, 39, 4, 145, 25, 12, 4056, 0, 4037, 0, 36, 0, 41, 0, 18, 3925, 38, 4051, 3, 0, 0, 0, 0, 0, 38, 26, 1, 4026, 0, 29, 4083, 3974, 0, 0, 47, 0, 4047, 0, 35, 0, 0, 4056, 0, 189, 4036, 3992, 71, 3818, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[925, 4096, 0, 1, 0, 2, 0, 3518, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3337, 4091, 4096, 4089, 3, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 4096, 4095, 0, 0, 0, 0, 5, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3907, 287, 4094, 0, 0, 0, 0, 5, 0, 4082, 0, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 15, 13, 4096, 0, 0, 0, 0, 4096, 313, 0, 17, 3, 4096, 0, 0, 4095, 4096, 0, 4090, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 38, 0, 0, 4070, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 130, 4094, 4096, 3760, 0, 0, 0, 4094, 0, 1, 8, 12, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 4096, 4070, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4055, 4095, 4096, 0, 0, 0, 0, 6, 0, 4096, 4096, 0, 0, 0, 4095, 143, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 119, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 4087, 0, 0, 0, 4, 5, 0, 0, 0, 0, 4096, 0, 0, 555, 4096, 0, 0, 0, 57, 4084, 3422, 0, 5, 0, 4096, 8, 0, 4096, 4096, 0, 4093, 1, 24, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4094, 19, 260, 0, 1, 18, 4095, 4094, 4091, 0, 0, 0, 0, 0, 0, 1, 31, 6, 0, 3793, 4093, 4088, 4095, 0, 2, 0, 4078, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 1, 0, 4095, 3940, 4094, 4093, 10, 0, 0, 0, 0, 0, 0, 0, 2, 12, 4095, 4055, 4091, 0, 0, 0, 67, 0, 0, 3969, 4089, 0, 0, 1, 4038, 162, 1, 0, 2, 4090, 0, 4085, 0, 0, 0, 0, 138, 7, 0, 3598, 426, 3971, 4, 0, 0, 1, 127, 0, 4096, 1, 4088, 2, 0, 3, 1, 0, 1, 5, 0, 0, 4084, 4, 0, 124, 4081, 1, 0, 9, 0, 4061, 18, 7, 0, 0, 4091, 4, 0, 4096, 4070, 6, 4040, 41, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 0, 0, 4096, 4096, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 4096, 4067, 0, 0, 8, 4096, 0, 0, 6, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4087, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4040, 4096, 0, 0, 0, 0, 4096, 107, 0, 0, 0, 4096, 18, 0, 4096, 4095, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[175, 4096, 0, 0, 32, 0, 0, 4081, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2882, 4096, 4096, 4089, 0, 0, 8, 4096, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3305, 4096, 4023, 0, 0, 0, 0, 0, 0, 4086, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 3, 4096, 0, 0, 0, 0, 31, 0, 0, 3573, 0, 4096, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 0, 0, 0, 41, 0, 0, 0, 1, 4096, 0, 0, 2364, 4096, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 4096, 0, 2, 4053, 4095, 0, 4072, 0, 45, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 132, 8, 0, 4095, 4096, 4095, 0, 0, 0, 2, 2, 0, 0, 0, 0, 2, 0, 4092, 4092, 4096, 0, 0, 3, 4096, 0, 0, 64, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4087, 4096, 4093, 0, 0, 0, 16, 0, 6, 2, 0, 0, 0, 4071, 4096, 4061, 6, 0, 0, 2, 0, 0, 4084, 4096, 0, 10, 0, 4063, 0, 0, 15, 0, 4083, 0, 4096, 0, 0, 1, 12, 1, 0, 0, 4096, 2, 4096, 3, 1, 0, 0, 0, 0, 4084, 0, 4092, 0, 0, 0, 0, 0, 4, 0, 3, 0, 4093, 12, 0, 4093, 4082, 0, 0, 0, 54, 4094, 0, 0, 0, 0, 4094, 3, 18, 4064, 4041, 0, 3975, 2, 9, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 38, 0, 0, 4070, 3638, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 130, 4094, 2796, 3760, 0, 0, 0, 4094, 0, 1, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 4096, 4070, 4096, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4055, 4095, 4096, 0, 0, 0, 0, 6, 0, 4096, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 143, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 119, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 4087, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 0, 0, 0, 4096, 0, 0, 555, 4096, 0, 0, 0, 4096, 1300, 458, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 3635, 3422, 0, 5, 0, 0, 8, 0, 4096, 4096, 0, 4093, 1, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15922, 14918, 16420, 16571, 16595, 16571, 14638, 19437] avg=16384.0000 max=19437 max_rank=7 min=14638 min_rank=6 imbalance_ratio=1.186340 max_load_reduction=111635 max_load_reduction_pct=0.851707 moved_to_echo=67241 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1341, 4096, 0, 0, 1, 0, 0, 4095, 3638, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 4057, 2796, 3992, 0, 0, 1, 4096, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3920, 4096, 4083, 0, 0, 0, 0, 0, 0, 4096, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 11, 4096, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3192, 505, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 3238, 4096, 0, 0, 0, 4096, 1300, 458, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3647, 30, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17259, 15147, 16444, 16416, 16464, 16064, 17314, 15964] avg=16384.0000 max=17314 max_rank=6 min=15147 min_rank=1 imbalance_ratio=1.056763 max_load_reduction=113758 max_load_reduction_pct=0.867905 moved_to_echo=65806 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[925, 4096, 0, 1, 0, 2, 0, 3518, 3634, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3337, 4091, 2796, 4089, 3, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 4095, 4096, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 4096, 4095, 0, 0, 0, 0, 5, 0, 4096, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 287, 4094, 0, 0, 0, 0, 5, 0, 4082, 0, 4096, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 15, 13, 4096, 0, 0, 0, 4096, 1300, 458, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3647, 313, 0, 17, 3, 0, 0, 0, 4095, 4096, 0, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16272, 18412, 16427, 16585, 16453, 16551, 14109, 16263] avg=16384.0000 max=18412 max_rank=1 min=14109 min_rank=6 imbalance_ratio=1.123779 max_load_reduction=112660 max_load_reduction_pct=0.859528 moved_to_echo=63376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[727, 4057, 0, 0, 718, 11, 0, 4077, 3610, 4071, 0, 21, 37, 22, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 54, 0, 12, 4070, 2763, 4068, 0, 60, 0, 4046, 25, 1, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 118, 16, 0, 30, 292, 4044, 3407, 4075, 4034, 4, 32, 47, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 8, 0, 0, 3681, 4021, 3941, 0, 5, 23, 0, 0, 43, 4069, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3407, 0, 70, 10, 3937, 0, 0, 4, 5, 4061, 10, 4058, 0, 0, 39, 4, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 25, 12, 4056, 0, 4037, 0, 36, 0, 41, 0, 18, 3925, 38, 4051, 3, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 26, 1, 4026, 0, 29, 4083, 3974, 0, 0, 47, 4056, 1293, 458, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3582, 0, 35, 0, 0, 0, 0, 189, 4036, 3992, 71, 3818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17353, 15493, 16181, 16113, 15675, 16471, 18063, 15723] avg=16384.0000 max=18063 max_rank=6 min=15493 min_rank=1 imbalance_ratio=1.102478 max_load_reduction=113009 max_load_reduction_pct=0.862190 moved_to_echo=65932 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[175, 4096, 0, 0, 32, 0, 0, 4081, 3638, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2882, 4096, 2796, 4089, 0, 0, 8, 4096, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 4096, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3305, 4096, 4023, 0, 0, 0, 0, 0, 0, 4086, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 3, 4096, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 3573, 0, 4096, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 1, 4096, 0, 0, 2364, 4096, 0, 0, 0, 4096, 1300, 458, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3647, 0, 0, 4, 0, 0, 0, 2, 4053, 4095, 0, 4072, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16118, 17981, 16434, 15731, 16450, 15960, 16480, 15918] avg=16384.0000 max=17981 max_rank=1 min=15731 min_rank=3 imbalance_ratio=1.097473 max_load_reduction=113091 max_load_reduction_pct=0.862816 moved_to_echo=64808 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 0, 0, 4096, 3638, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 2796, 4067, 0, 0, 8, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4087, 0, 0, 0, 0, 0, 0, 4096, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4040, 4096, 0, 0, 0, 4096, 1300, 458, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3647, 107, 0, 0, 0, 0, 18, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15868, 15082, 16437, 16596, 16454, 16462, 18114, 16059] avg=16384.0000 max=18114 max_rank=6 min=15082 min_rank=1 imbalance_ratio=1.105591 max_load_reduction=112958 max_load_reduction_pct=0.861801 moved_to_echo=67089 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 19, 260, 0, 1, 18, 4095, 3636, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 31, 6, 0, 3793, 4093, 2790, 4095, 0, 2, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 1, 0, 4095, 3940, 4094, 4093, 10, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 12, 4095, 4055, 4091, 0, 0, 0, 67, 0, 0, 3969, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 1, 4038, 162, 1, 0, 2, 4090, 0, 4085, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 7, 0, 3598, 426, 3971, 4, 0, 0, 1, 127, 0, 4096, 1, 4088, 2, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 1, 5, 0, 0, 4084, 4, 0, 124, 4081, 1, 0, 9, 4091, 1298, 458, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3615, 18, 7, 0, 0, 0, 4, 0, 4096, 4070, 6, 4040, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16214, 18889, 16299, 16511, 16536, 16538, 14188, 15897] avg=16384.0000 max=18889 max_rank=1 min=14188 min_rank=6 imbalance_ratio=1.152893 max_load_reduction=112183 max_load_reduction_pct=0.855888 moved_to_echo=63159 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 132, 8, 0, 4095, 3638, 4095, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 2793, 4096, 0, 0, 3, 4096, 0, 0, 64, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4087, 4096, 4093, 0, 0, 0, 16, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 4071, 4096, 4061, 6, 0, 0, 2, 0, 0, 4084, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 4063, 0, 0, 15, 0, 4083, 0, 4096, 0, 0, 1, 12, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 2, 4096, 3, 1, 0, 0, 0, 0, 4084, 0, 4092, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 0, 4093, 12, 0, 4093, 4082, 0, 0, 0, 4094, 1299, 458, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 3645, 0, 0, 0, 0, 0, 3, 18, 4064, 4041, 0, 3975, 2, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16066, 15150, 16430, 16549, 16445, 16455, 18166, 15811] avg=16384.0000 max=18166 max_rank=6 min=15150 min_rank=1 imbalance_ratio=1.108765 max_load_reduction=112906 max_load_reduction_pct=0.861404 moved_to_echo=66877 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[384, 128, 128, 30720, 1408, 32640, 128, 128, 0, 128, 256, 128, 32640, 128, 32768, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32128, 0, 128, 128, 32640, 384, 128, 128, 128, 32768, 128, 32768, 0, 0, 128, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 29184, 3968, 128, 128, 128, 0, 128, 256, 32640, 32640, 128, 32384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[3200, 32768, 128, 384, 1024, 128, 128, 32128, 29184, 32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 128, 128, 31360, 32768, 32512, 128, 128, 128, 128, 128, 128, 32640, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 18560, 32640, 128, 0, 128, 32768, 10496, 3712, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 10368, 32768, 22400, 32256, 128, 128, 128, 32768, 128, 128, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 384, 32768, 32000, 32768, 32768, 128, 128, 128, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 85, 0, 4094, 0, 0, 0, 12, 0, 4095, 4060, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4082, 0, 1, 4096, 0, 3407, 0, 0, 3, 31, 0, 4096, 4059, 0, 0, 15, 0, 3976, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4022, 0, 0, 0, 4096, 4096, 0, 1, 0, 292, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 32, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 555, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4096, 0, 4096, 0, 4096, 0, 0, 4090, 3702, 0, 0, 4095, 0, 4096, 15, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 32, 0, 2532, 0, 0, 0, 1, 0, 3420, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 3915, 0, 0, 0, 0, 0, 3986, 0, 155, 4096, 0, 3986, 0, 0, 1, 0, 0, 4096, 4082, 0, 0, 0, 0, 3691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4096, 0, 4047, 0, 0, 0, 4095, 4096, 3, 1, 0, 2646, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 5, 4096, 4096, 0, 0, 41, 0, 4096, 114, 0, 4096, 0, 4096, 0, 174, 0, 4027] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 0, 2198, 4096, 0, 15, 0, 0, 4096, 0, 4096, 0, 4096, 0, 1, 0, 0, 0, 4094, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 4094, 4096, 0, 2, 991, 0, 3519, 299, 4, 1, 0, 0, 0, 0, 0, 0, 32, 4096, 0, 1149, 0, 0, 0, 4096, 4095, 1, 16, 1, 3571, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3803, 7, 4096, 4096, 0, 0, 0, 3930, 0, 0, 4088, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 126, 4096, 4090, 0, 0, 406, 0, 4095, 109, 0, 4059, 0, 4096, 0, 2, 16, 342] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 72, 4076, 65, 0, 191, 0, 2, 4075, 4027, 237, 0, 45, 0, 4067, 19, 4071, 35, 3118, 129, 0, 0, 1, 0, 4004, 27, 202, 4044, 6, 3888, 0, 1, 0, 0, 0, 4056, 4073, 0, 8, 0, 1, 3768, 145, 63, 0, 95, 0, 33, 0, 21, 3, 1031, 4059, 25, 96, 0, 0, 0, 4026, 4061, 0, 0, 0, 3892, 0, 30, 3721, 41, 0, 63, 30, 0, 5, 34, 28, 4040, 4063, 33, 4037, 3911, 0, 0, 0, 2, 0, 25, 4042, 18, 94, 0, 0, 4068, 1, 0, 0, 3, 37, 1, 0, 0, 58, 10, 124, 21, 4055, 40, 4007, 22, 4034, 1, 473, 4055, 3770, 10, 32, 3170, 23, 4074, 713, 56, 4058, 0, 4076, 0, 5, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 1024, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 2742, 0, 0, 0, 0, 0, 4095, 0, 8, 4096, 0, 1987, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 404, 37, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3681, 0, 0, 0, 4096, 4096, 0, 0, 0, 3553, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3921, 0, 4096, 4096, 0, 0, 0, 1709, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 67, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 1, 4096, 4095, 0, 0, 4065, 0, 4096, 14, 0, 4089, 0, 4096, 0, 4, 0, 1356] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 14, 0, 4096, 0, 0, 0, 0, 0, 3853, 4092, 0, 0, 4, 8, 4081, 0, 4088, 0, 4096, 0, 4, 0, 0, 0, 4086, 1, 23, 4094, 0, 4083, 0, 0, 0, 0, 0, 4085, 4093, 0, 0, 0, 0, 4073, 11, 7, 0, 0, 0, 1, 0, 0, 4, 0, 4095, 0, 3737, 0, 0, 0, 4072, 4096, 0, 1, 18, 1269, 0, 0, 4077, 2, 1, 10, 0, 0, 1, 0, 0, 4093, 4087, 14, 4089, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 3, 0, 0, 4094, 0, 190, 0, 6, 0, 28, 6, 0, 0, 0, 0, 8, 4092, 2, 4096, 0, 4096, 0, 0, 4094, 4060, 0, 0, 46, 3, 4094, 2, 0, 4090, 0, 4096, 0, 1, 0, 3213] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4062, 0, 0, 0, 0, 0, 3332, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 4095, 0, 18, 4096, 0, 4087, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4074, 0, 0, 0, 4096, 4096, 0, 0, 0, 2996, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 38, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 10, 4096, 4053, 0, 0, 1327, 0, 4096, 0, 0, 4096, 0, 4096, 0, 86, 0, 307] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 3381, 0, 0, 0, 0, 49, 4092, 4096, 2, 0, 0, 0, 4088, 0, 4090, 0, 4048, 12, 0, 0, 0, 13, 3810, 0, 2, 4091, 0, 3507, 0, 25, 1, 0, 0, 4083, 4096, 0, 0, 21, 181, 4096, 500, 8, 0, 0, 0, 0, 0, 0, 14, 0, 4085, 0, 3753, 0, 0, 7, 4093, 4094, 0, 0, 0, 4059, 0, 2, 3796, 0, 9, 1, 1, 4, 9, 0, 0, 4095, 4095, 0, 4096, 4096, 234, 0, 0, 4, 0, 0, 4092, 0, 0, 0, 0, 4058, 0, 119, 0, 0, 0, 0, 11, 0, 5, 5, 0, 0, 4089, 3, 4078, 3, 4094, 2, 33, 4073, 4082, 3, 5, 472, 0, 4091, 6, 0, 4076, 0, 4084, 0, 0, 1, 748] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 85, 0, 4094, 0, 0, 0, 12, 0, 4095, 4060, 0, 0, 0, 0, 4096, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 778, 0, 4096, 0, 0, 0, 0, 0, 4082, 0, 1, 4096, 0, 3407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 31, 0, 4096, 4059, 0, 0, 15, 0, 3976, 333, 0, 0, 0, 0, 3318, 457, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4022, 0, 0, 0, 4096, 4096, 0, 1, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1429, 4096, 32, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 555, 0, 2542, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4096, 0, 4096, 0, 4096, 0, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3702, 0, 0, 4095, 0, 0, 15, 0, 4096, 0, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17486, 16460, 16389, 17206, 14041, 16109, 16286, 17095] avg=16384.0000 max=17486 max_rank=0 min=14041 min_rank=4 imbalance_ratio=1.067261 max_load_reduction=113586 max_load_reduction_pct=0.866592 moved_to_echo=63531 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 32, 0, 2532, 0, 0, 0, 1, 0, 3420, 4096, 0, 0, 0, 0, 4096, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 778, 0, 3915, 0, 0, 0, 0, 0, 3986, 0, 155, 4096, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4082, 0, 0, 0, 0, 3691, 0, 0, 0, 0, 0, 3318, 457, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4096, 0, 4047, 0, 0, 0, 4095, 4096, 3, 1, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2646, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1430, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 2542, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 5, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 41, 0, 0, 114, 0, 4096, 0, 1097, 0, 174, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15221, 16916, 15746, 17256, 16362, 15542, 16288, 17741] avg=16384.0000 max=17741 max_rank=7 min=15221 min_rank=0 imbalance_ratio=1.082825 max_load_reduction=113331 max_load_reduction_pct=0.864647 moved_to_echo=65933 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 0, 2198, 4096, 0, 15, 0, 0, 4096, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 778, 0, 4096, 0, 1, 0, 0, 0, 4094, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 2, 991, 0, 3519, 299, 4, 1, 0, 0, 3318, 457, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4096, 0, 1149, 0, 0, 0, 4096, 4095, 1, 16, 1, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3571, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1430, 3803, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3930, 0, 0, 4088, 0, 0, 0, 0, 4090, 0, 0, 0, 2542, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 126, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 406, 0, 0, 109, 0, 4059, 0, 1097, 0, 2, 16, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15544, 17160, 16882, 14382, 17003, 19472, 16412, 14217] avg=16384.0000 max=19472 max_rank=5 min=14217 min_rank=7 imbalance_ratio=1.188477 max_load_reduction=111600 max_load_reduction_pct=0.851440 moved_to_echo=67104 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 72, 4076, 65, 0, 191, 0, 2, 4075, 4027, 237, 0, 45, 0, 4067, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 768, 35, 3118, 129, 0, 0, 1, 0, 4004, 27, 202, 4044, 6, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4056, 4073, 0, 8, 0, 1, 3768, 145, 63, 0, 95, 0, 3303, 460, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 21, 3, 1031, 4059, 25, 96, 0, 0, 0, 4026, 4061, 0, 0, 0, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 0, 30, 3721, 41, 0, 63, 30, 0, 5, 34, 28, 1399, 4063, 33, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3911, 0, 0, 0, 2, 0, 25, 4042, 18, 94, 0, 0, 4068, 1, 0, 0, 2533, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 37, 1, 0, 0, 58, 10, 124, 21, 4055, 40, 4007, 22, 4034, 1, 473, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 3770, 10, 32, 3170, 23, 0, 713, 56, 4058, 0, 1083, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17892, 16241, 16074, 14243, 17376, 15412, 16859, 16975] avg=16384.0000 max=17892 max_rank=0 min=14243 min_rank=3 imbalance_ratio=1.092041 max_load_reduction=113180 max_load_reduction_pct=0.863495 moved_to_echo=66622 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 1024, 4096, 0, 0, 0, 0, 4096, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 778, 0, 2742, 0, 0, 0, 0, 0, 4095, 0, 8, 4096, 0, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 404, 37, 4096, 1, 0, 0, 0, 0, 3318, 457, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3681, 0, 0, 0, 4096, 4096, 0, 0, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3553, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1430, 3921, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1709, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 67, 0, 2542, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 1, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4065, 0, 0, 14, 0, 4089, 0, 1097, 0, 4, 0, 1356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14356, 13706, 16606, 16865, 17096, 17332, 16295, 18816] avg=16384.0000 max=18816 max_rank=7 min=13706 min_rank=1 imbalance_ratio=1.148438 max_load_reduction=112256 max_load_reduction_pct=0.856445 moved_to_echo=69539 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4062, 0, 0, 0, 0, 0, 3332, 4096, 0, 0, 0, 0, 4096, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 778, 0, 4092, 0, 0, 0, 0, 0, 4095, 0, 18, 4096, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 191, 0, 0, 0, 0, 3318, 457, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4074, 0, 0, 0, 4096, 4096, 0, 0, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2996, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1430, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 38, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 2542, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 10, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4053, 0, 0, 1327, 0, 0, 0, 0, 4096, 0, 1097, 0, 86, 0, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16630, 17166, 16354, 17258, 16714, 15595, 16293, 15062] avg=16384.0000 max=17258 max_rank=3 min=15062 min_rank=7 imbalance_ratio=1.053345 max_load_reduction=113814 max_load_reduction_pct=0.868332 moved_to_echo=63664 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 14, 0, 4096, 0, 0, 0, 0, 0, 3853, 4092, 0, 0, 4, 8, 4081, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 0, 4096, 0, 4, 0, 0, 0, 4086, 1, 23, 4094, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4093, 0, 0, 0, 0, 4073, 11, 7, 0, 0, 0, 3312, 457, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 4095, 0, 3737, 0, 0, 0, 4072, 4096, 0, 1, 18, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 0, 0, 4077, 2, 1, 10, 0, 0, 1, 0, 0, 1429, 4087, 14, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 3, 0, 0, 4094, 0, 190, 0, 2542, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 28, 6, 0, 0, 0, 0, 8, 4092, 2, 4096, 0, 4096, 0, 0, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4060, 0, 0, 46, 3, 0, 2, 0, 4090, 0, 1097, 0, 1, 0, 3213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17191, 17163, 16139, 16920, 14979, 15747, 16327, 16606] avg=16384.0000 max=17191 max_rank=0 min=14979 min_rank=4 imbalance_ratio=1.049255 max_load_reduction=113881 max_load_reduction_pct=0.868843 moved_to_echo=63659 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3381, 0, 0, 0, 0, 49, 4092, 4096, 2, 0, 0, 0, 4088, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 777, 0, 4048, 12, 0, 0, 0, 13, 3810, 0, 2, 4091, 0, 3507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 1, 0, 0, 4083, 4096, 0, 0, 21, 181, 4096, 500, 8, 0, 0, 0, 3313, 456, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4085, 0, 3753, 0, 0, 7, 4093, 4094, 0, 0, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 2, 3796, 0, 9, 1, 1, 4, 9, 0, 0, 1429, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 234, 0, 0, 4, 0, 0, 4092, 0, 0, 0, 0, 4058, 0, 119, 0, 2534, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 5, 5, 0, 0, 4089, 3, 4078, 3, 4094, 2, 33, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4082, 3, 5, 472, 0, 0, 6, 0, 4076, 0, 1094, 0, 0, 1, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16752, 16260, 16882, 16942, 17501, 15863, 16312, 14560] avg=16384.0000 max=17501 max_rank=4 min=14560 min_rank=7 imbalance_ratio=1.068176 max_load_reduction=113571 max_load_reduction_pct=0.866478 moved_to_echo=64236 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32640, 256, 0, 128, 5760, 0, 128, 32768, 128, 128, 0, 0, 32768, 128, 1024, 0, 20352, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 6272, 128, 30208, 256, 128, 0, 128, 128, 32256, 128, 512, 32768, 128, 29056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 1152, 32768, 128, 24576, 0, 0, 128, 32768, 32768, 128, 128, 128, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 256, 128, 30464, 128, 0, 256, 128, 128, 26112, 32768, 256, 128, 128, 128, 32768, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 32000, 128, 128, 13696, 128, 0, 1024, 128, 32768, 0, 8832, 0, 384, 128, 10112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 32768, 128, 32768, 128, 768, 32000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 32768, 32768, 0, 128, 1536, 256, 31360, 1536, 128, 128, 128, 0, 26624, 3712, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[22400, 0, 128, 32128, 128, 128, 128, 128, 128, 128, 128, 128, 11520, 32384, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 2645, 91, 0, 0, 2, 0, 0, 4095, 4096, 4094, 0, 0, 0, 0, 4081, 3778, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 2, 76, 4096, 3955, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 204, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4055, 4096, 0, 0, 4096, 0, 0, 1, 3300, 0, 0, 4096, 17, 0, 431, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4088, 1, 0, 4096, 4024, 0, 3482, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 37, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 2579, 0, 0, 8, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 2710, 208, 0, 121, 28, 0, 0, 4095, 4096, 4096, 0, 0, 12, 0, 3900, 3916, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 24, 4096, 952, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 1, 25, 0, 3154, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 1, 3626, 0, 0, 4096, 0, 0, 2003, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4092, 2, 0, 4096, 3955, 0, 310, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3701, 0, 0, 29, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 1615, 380, 0, 1, 0, 0, 0, 4073, 4096, 4095, 0, 0, 5, 0, 4093, 3367, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 47, 4096, 3154, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4048, 79, 111, 0, 0, 38, 0, 4082, 0, 0, 0, 9, 2, 0, 4093, 4096, 0, 0, 4096, 0, 0, 116, 3876, 0, 0, 3917, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4067, 4083, 0, 4094, 0, 0, 4096, 3981, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 4096, 18, 0, 4096, 0, 0, 20, 0, 27, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 1, 35, 0, 4096, 4093, 0, 0, 0, 12, 95, 0, 0, 3814, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3898, 52, 0, 78, 63, 0, 0, 4082, 4096, 4047, 0, 0, 0, 0, 3972, 2893, 0, 0, 0, 4096, 0, 0, 35, 0, 0, 5, 0, 4096, 3523, 4095, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 3943, 34, 0, 631, 22, 0, 4096, 0, 0, 0, 52, 0, 0, 4096, 4096, 0, 0, 3878, 1, 0, 158, 3731, 0, 0, 4096, 0, 0, 1041, 0, 0, 4096, 0, 0, 0, 4089, 4096, 0, 3929, 0, 0, 4096, 4, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 6, 0, 333, 0, 0, 0, 0, 9, 0, 0, 4067, 0, 4044, 1, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 565, 0, 0, 99, 10] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 1588, 122, 0, 0, 239, 0, 0, 4096, 4096, 4093, 0, 0, 1, 0, 4000, 2388, 0, 0, 0, 4096, 0, 7, 0, 0, 11, 116, 2345, 4093, 4028, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 996, 78, 0, 196, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3881, 4089, 0, 7, 1630, 0, 0, 0, 3970, 0, 0, 4096, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 9, 0, 3981, 0, 0, 0, 63, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 98, 41, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 3980, 4096, 0, 0, 0, 0, 3783, 0, 0, 3420, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 7, 0, 4026, 417, 0, 0, 603, 0, 0, 4000, 4096, 4071, 0, 15, 0, 0, 4086, 116, 0, 0, 0, 4093, 0, 60, 1, 4, 0, 0, 32, 4096, 3398, 4094, 0, 3, 4092, 0, 0, 0, 4, 0, 5, 4094, 522, 3238, 0, 116, 0, 0, 4085, 5, 2, 0, 0, 0, 0, 4085, 4094, 0, 0, 3179, 0, 0, 0, 4096, 0, 0, 4093, 2, 0, 1215, 0, 0, 4092, 0, 0, 0, 4080, 4094, 0, 4050, 0, 0, 4095, 71, 2, 3526, 3, 0, 0, 0, 1, 2, 0, 4095, 0, 16, 4048, 0, 11, 0, 7, 16, 0, 2, 0, 5, 1, 4096, 0, 4094, 0, 0, 0, 0, 0, 1, 4080, 4084, 26, 2, 0, 0, 93, 1, 0, 4081, 56] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4057, 19, 0, 4080, 0, 68, 1547, 2796, 0, 3, 3952, 3988, 4028, 12, 60, 0, 18, 471, 2280, 14, 0, 105, 4069, 0, 0, 39, 6, 100, 0, 1, 4056, 2110, 4010, 8, 39, 4053, 5, 0, 0, 0, 0, 41, 4035, 3862, 2521, 0, 4049, 0, 21, 3986, 27, 35, 0, 219, 0, 0, 3976, 4074, 4, 0, 3630, 0, 0, 0, 4078, 34, 0, 4044, 0, 0, 0, 48, 56, 4048, 0, 89, 0, 4003, 4067, 0, 4088, 138, 22, 4062, 666, 28, 1412, 0, 26, 30, 0, 0, 1, 44, 4069, 2, 0, 3974, 77, 27, 0, 0, 0, 48, 1, 0, 41, 6, 3960, 0, 4067, 0, 120, 0, 0, 2, 26, 349, 4082, 95, 0, 7, 0, 0, 36, 78, 2036, 511] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 0, 4094, 1020, 6, 0, 2162, 3, 0, 4095, 4096, 4096, 0, 19, 0, 2, 2757, 0, 4, 8, 0, 4090, 9, 0, 2631, 7, 0, 0, 86, 4094, 1685, 4090, 0, 0, 4041, 0, 0, 0, 6, 0, 0, 4094, 0, 0, 4, 3978, 137, 0, 4096, 1, 4, 0, 837, 0, 0, 3838, 4085, 0, 0, 3727, 1, 41, 505, 3468, 0, 0, 4090, 0, 2, 2, 0, 0, 4092, 97, 11, 0, 4089, 4091, 0, 4096, 0, 0, 4095, 1706, 0, 4090, 0, 0, 1, 0, 10, 0, 0, 3926, 0, 0, 3982, 0, 12, 291, 0, 0, 0, 0, 0, 2, 5, 4086, 6, 4096, 0, 0, 0, 0, 75, 0, 3927, 4037, 0, 0, 7, 0, 1, 0, 0, 135, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2645, 91, 0, 0, 2, 0, 0, 4095, 3907, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 3778, 0, 0, 0, 2960, 0, 0, 1, 0, 0, 2, 76, 4096, 3955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 204, 0, 0, 0, 0, 595, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4055, 4096, 0, 0, 4096, 0, 0, 1, 212, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3300, 0, 0, 4096, 17, 0, 431, 0, 0, 3850, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 3282, 4024, 0, 3482, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 37, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 2668, 1136, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 2579, 0, 0, 8, 0, 4096, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14834, 18949, 13120, 16745, 15790, 18973, 16358, 16303] avg=16384.0000 max=18973 max_rank=5 min=13120 min_rank=2 imbalance_ratio=1.158020 max_load_reduction=112099 max_load_reduction_pct=0.855247 moved_to_echo=67424 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3898, 52, 0, 78, 63, 0, 0, 4082, 3907, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 2893, 0, 0, 0, 2960, 0, 0, 35, 0, 0, 5, 0, 4096, 3523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 3943, 34, 0, 631, 22, 595, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 52, 0, 0, 4096, 4096, 0, 0, 3878, 1, 0, 158, 212, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3731, 0, 0, 4096, 0, 0, 1041, 0, 0, 3850, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3929, 0, 0, 3282, 4, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 0, 333, 0, 0, 0, 0, 9, 0, 0, 4067, 0, 4044, 1, 2668, 1136, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 565, 0, 0, 99, 10, 4096, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16127, 17484, 17544, 16778, 16807, 15363, 16578, 14391] avg=16384.0000 max=17544 max_rank=2 min=14391 min_rank=7 imbalance_ratio=1.070801 max_load_reduction=113528 max_load_reduction_pct=0.866150 moved_to_echo=63139 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1615, 380, 0, 1, 0, 0, 0, 4073, 3907, 4095, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3367, 0, 0, 0, 2960, 0, 4, 0, 0, 0, 0, 47, 4096, 3154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4048, 79, 111, 0, 0, 38, 595, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 9, 2, 0, 4093, 4096, 0, 0, 4096, 0, 0, 116, 212, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3876, 0, 0, 3917, 0, 0, 0, 0, 0, 3850, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3282, 3981, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 0, 27, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 2661, 1136, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 35, 0, 4096, 4093, 0, 0, 0, 12, 95, 0, 0, 3814, 0, 4096, 1422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14076, 17721, 13088, 16895, 15710, 19571, 16347, 17664] avg=16384.0000 max=19571 max_rank=5 min=13088 min_rank=2 imbalance_ratio=1.194519 max_load_reduction=111501 max_load_reduction_pct=0.850685 moved_to_echo=69292 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2710, 208, 0, 121, 28, 0, 0, 4095, 3907, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3900, 3916, 0, 0, 0, 2960, 0, 0, 0, 0, 0, 0, 24, 4096, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 1, 25, 0, 3154, 0, 595, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 1, 212, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3626, 0, 0, 4096, 0, 0, 2003, 0, 0, 3850, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 3282, 3955, 0, 310, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2669, 1136, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3701, 0, 0, 29, 0, 4096, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15177, 15848, 16096, 16786, 17671, 15737, 16312, 17445] avg=16384.0000 max=17671 max_rank=4 min=15177 min_rank=0 imbalance_ratio=1.078552 max_load_reduction=113401 max_load_reduction_pct=0.865181 moved_to_echo=67165 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1588, 122, 0, 0, 239, 0, 0, 4096, 3907, 4093, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 2388, 0, 0, 0, 2960, 0, 7, 0, 0, 11, 116, 2345, 4093, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 996, 78, 0, 196, 0, 595, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3881, 4089, 0, 7, 1630, 0, 0, 0, 212, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 0, 0, 4096, 0, 0, 2, 0, 0, 3850, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3282, 9, 0, 3981, 0, 0, 0, 63, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 98, 41, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 2669, 1136, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 4096, 0, 0, 0, 0, 3783, 0, 0, 3420, 2, 4096, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14046, 19948, 14187, 14104, 16014, 15521, 16448, 20804] avg=16384.0000 max=20804 max_rank=7 min=14046 min_rank=0 imbalance_ratio=1.269775 max_load_reduction=110268 max_load_reduction_pct=0.841278 moved_to_echo=68787 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 19, 0, 4080, 0, 68, 1547, 2796, 0, 3, 3952, 3799, 4028, 12, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 471, 2280, 14, 0, 105, 2934, 0, 0, 39, 6, 100, 0, 1, 4056, 2110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 8, 39, 4053, 5, 0, 0, 0, 0, 41, 4035, 3862, 2521, 0, 4049, 0, 597, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 3986, 27, 35, 0, 219, 0, 0, 3976, 4074, 4, 0, 3630, 0, 0, 0, 216, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 34, 0, 4044, 0, 0, 0, 48, 56, 3791, 0, 89, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 138, 22, 3240, 666, 28, 1412, 0, 26, 30, 0, 0, 1, 44, 4069, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 77, 27, 0, 0, 0, 48, 1, 0, 41, 6, 3960, 0, 4067, 0, 2650, 1135, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 2, 26, 349, 4082, 95, 0, 7, 0, 0, 36, 78, 2036, 511, 4057, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20364, 12134, 23261, 16377, 16143, 13766, 16211, 12816] avg=16384.0000 max=23261 max_rank=2 min=12134 min_rank=1 imbalance_ratio=1.419739 max_load_reduction=107811 max_load_reduction_pct=0.822533 moved_to_echo=58936 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 1020, 6, 0, 2162, 3, 0, 4095, 3907, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2757, 0, 4, 8, 0, 2955, 9, 0, 2631, 7, 0, 0, 86, 4094, 1685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4041, 0, 0, 0, 6, 0, 0, 4094, 0, 0, 4, 3978, 137, 595, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4, 0, 837, 0, 0, 3838, 4085, 0, 0, 3727, 1, 41, 505, 212, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 0, 4090, 0, 2, 2, 0, 0, 3846, 97, 11, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3281, 1706, 0, 4090, 0, 0, 1, 0, 10, 0, 0, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 0, 12, 291, 0, 0, 0, 0, 0, 2, 5, 4086, 6, 4096, 0, 2666, 1135, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 3927, 4037, 0, 0, 7, 0, 1, 0, 0, 135, 0, 4095, 1425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19402, 14238, 16979, 17536, 15605, 17110, 16500, 13702] avg=16384.0000 max=19402 max_rank=0 min=13702 min_rank=7 imbalance_ratio=1.184204 max_load_reduction=111670 max_load_reduction_pct=0.851974 moved_to_echo=62917 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 4026, 417, 0, 0, 603, 0, 0, 4000, 3907, 4071, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 116, 0, 0, 0, 2957, 0, 60, 1, 4, 0, 0, 32, 4096, 3398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3, 4092, 0, 0, 0, 4, 0, 5, 4094, 522, 3238, 0, 116, 0, 595, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 5, 2, 0, 0, 0, 0, 4085, 4094, 0, 0, 3179, 0, 0, 0, 212, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 2, 0, 1215, 0, 0, 3846, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 3281, 71, 2, 3526, 3, 0, 0, 0, 1, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4048, 0, 11, 0, 7, 16, 0, 2, 0, 5, 1, 4096, 0, 4094, 0, 2667, 1136, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4080, 4084, 26, 2, 0, 0, 93, 1, 0, 4081, 56, 4096, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17046, 14750, 16797, 15851, 17332, 15031, 16318, 17947] avg=16384.0000 max=17947 max_rank=7 min=14750 min_rank=1 imbalance_ratio=1.095398 max_load_reduction=113125 max_load_reduction_pct=0.863075 moved_to_echo=66628 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 32768, 128, 0, 0, 128, 0, 128, 32768, 9728, 6016, 128, 12160, 256, 4864, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[30208, 128, 0, 32640, 128, 128, 4736, 128, 128, 30848, 128, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32640, 256, 128, 26240, 14464, 128, 24960, 128, 128, 128, 128, 128, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 27392, 18816, 128, 128, 128, 23680, 128, 128, 2816, 128, 128, 128, 2688, 32768, 22912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 24704, 2304, 128, 1792, 6016, 128, 128, 32512, 31232, 32640, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 128, 28800, 32768, 128, 128, 128, 128, 10880, 128, 128, 13696, 640, 32768, 11520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32640, 128, 128, 0, 1152, 128, 0, 32128, 32768, 128, 128, 28416, 128, 128, 896, 1792, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32512, 128, 128, 896, 128, 128, 128, 128, 128, 128, 128, 32640, 128, 32768, 128, 21376, 9088, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4072, 0, 0, 1593, 0, 2, 204, 0, 0, 0, 0, 0, 4096, 1128, 4096, 0, 0, 4096, 326, 0, 0, 0, 0, 308, 63, 0, 44, 0, 4062, 36, 0, 1, 0, 0, 6, 4096, 4, 0, 4078, 4096, 0, 0, 22, 4095, 16, 4091, 4096, 4096, 4065, 0, 0, 4095, 0, 0, 4094, 0, 0, 4096, 4069, 4096, 11, 3, 0, 0, 0, 1245, 0, 0, 4048, 45, 0, 0, 3, 4096, 4029, 0, 0, 4096, 0, 0, 0, 0, 3883, 0, 4017, 0, 11, 0, 0, 0, 0, 0, 0, 0, 3494, 4096, 0, 0, 0, 4093, 4081, 0, 0, 0, 0, 803, 0, 0, 57, 0, 0, 0, 0, 0, 3500, 0, 2, 0, 4093, 3965, 0, 63, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 148, 2, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3515, 4090, 0, 0, 3479, 0, 0, 0, 0, 0, 262, 90, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 4086, 4096, 0, 0, 5, 1511, 17, 2707, 4096, 4096, 4096, 0, 0, 4096, 2, 0, 4095, 0, 0, 4090, 4093, 4068, 15, 0, 0, 0, 0, 3449, 8, 0, 316, 3210, 0, 0, 0, 3790, 4095, 0, 0, 4096, 690, 4, 0, 0, 4005, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 1, 0, 628, 4092, 0, 0, 14, 4088, 4075, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 600, 0, 697, 0, 12, 0, 4096, 3913, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 10, 0, 4042, 0, 0, 1397, 0, 0, 0, 0, 0, 4096, 3768, 4096, 0, 0, 4040, 1, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 234, 0, 3605, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4086, 4096, 0, 0, 0, 0, 0, 3284, 0, 0, 3180, 3, 6, 0, 1, 4096, 4096, 0, 0, 4096, 3950, 0, 0, 0, 3551, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4096, 0, 0, 4, 4096, 4096, 0, 0, 0, 0, 2006, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 3714, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4095, 129, 2, 33, 0, 0, 3386, 0, 0, 0, 1, 0, 4090, 553, 4096, 0, 0, 3955, 302, 0, 0, 0, 0, 0, 1, 0, 3, 0, 4095, 3, 0, 0, 0, 1, 0, 4096, 0, 8, 4096, 4096, 0, 0, 0, 4033, 2, 3472, 4096, 4096, 4079, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4052, 4096, 138, 0, 0, 0, 0, 3196, 0, 0, 3994, 1206, 184, 0, 12, 4096, 3911, 0, 0, 4096, 2074, 0, 0, 0, 4096, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1878, 4092, 0, 0, 107, 4096, 4096, 0, 0, 0, 0, 1, 0, 0, 91, 0, 2, 0, 0, 0, 157, 0, 0, 0, 4096, 4079, 0, 38, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 2759, 0, 0, 558, 0, 0, 0, 0, 0, 4096, 1905, 4096, 0, 0, 3081, 2, 0, 0, 0, 0, 162, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 2291, 0, 4096, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 3347, 0, 0, 4087, 200, 11, 0, 2, 4096, 4096, 0, 0, 4096, 541, 0, 0, 0, 4003, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1946, 0, 0, 2, 0, 0, 0, 0, 0, 312, 0, 0, 0, 4096, 3556, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4076, 244, 0, 4058, 23, 529, 952, 9, 17, 205, 0, 23, 4021, 4090, 4050, 0, 42, 2130, 2050, 80, 0, 6, 0, 0, 0, 21, 0, 75, 4013, 3, 21, 4, 20, 0, 0, 3981, 0, 157, 4083, 4061, 0, 39, 0, 1024, 155, 3960, 4022, 4063, 3874, 11, 56, 4069, 0, 0, 4075, 0, 19, 4053, 4081, 3821, 0, 0, 0, 4, 152, 2851, 128, 0, 3382, 0, 1423, 65, 0, 4066, 3344, 12, 38, 4059, 0, 139, 2, 45, 2883, 1, 4086, 0, 78, 0, 6, 33, 0, 7, 0, 0, 4048, 3940, 0, 0, 1340, 3866, 3962, 16, 24, 15, 0, 1461, 0, 41, 0, 8, 2, 36, 366, 0, 0, 0, 0, 29, 4072, 544, 0, 0, 47, 0, 1, 7, 77] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4090, 3, 0, 4094, 0, 8, 0, 15, 1, 2, 0, 4, 4090, 4096, 4087, 0, 5, 0, 264, 0, 2, 0, 44, 0, 782, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4084, 17, 61, 4093, 4096, 0, 1, 2, 1422, 4, 4091, 4094, 4051, 4094, 0, 1, 4093, 0, 0, 4081, 0, 0, 4096, 4095, 4096, 0, 0, 0, 2, 0, 3167, 0, 0, 4096, 53, 4, 0, 57, 4057, 4092, 2, 0, 4085, 1247, 0, 0, 0, 4093, 1, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4025, 4078, 11, 0, 917, 4038, 4082, 0, 11, 0, 0, 107, 0, 0, 17, 0, 56, 0, 0, 0, 567, 0, 0, 2, 4071, 3783, 0, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 3942, 0, 0, 4086, 2, 0, 249, 0, 1, 0, 1, 4, 4088, 1842, 4092, 0, 1, 2156, 25, 0, 0, 0, 98, 6, 3570, 5, 8, 0, 3953, 0, 0, 8, 0, 5, 0, 4084, 0, 0, 4095, 4056, 37, 10, 0, 28, 0, 12, 4093, 4094, 4087, 76, 1, 4086, 0, 3, 4092, 47, 8, 3952, 4096, 4068, 205, 0, 2, 0, 4, 3098, 0, 0, 4096, 322, 0, 0, 5, 4042, 3958, 0, 0, 4077, 3690, 0, 0, 0, 4084, 1, 4031, 0, 0, 0, 0, 0, 0, 0, 40, 17, 1740, 4071, 0, 3, 0, 3760, 4095, 0, 4, 0, 61, 3997, 0, 2, 146, 0, 86, 0, 0, 0, 23, 0, 48, 6, 4091, 4096, 0, 0, 0, 4, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4072, 0, 0, 1593, 0, 2, 204, 0, 0, 0, 0, 0, 4096, 1128, 1533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 326, 0, 0, 0, 0, 308, 63, 0, 44, 0, 4062, 36, 0, 4095, 4094, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 4096, 4, 0, 0, 3066, 0, 0, 22, 4095, 16, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4069, 4096, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1245, 0, 0, 4048, 45, 0, 0, 3, 3707, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3883, 0, 4017, 0, 11, 0, 0, 0, 0, 0, 0, 0, 3494, 4039, 1925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 4081, 0, 0, 0, 0, 803, 0, 0, 57, 0, 0, 1030, 638, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3500, 0, 2, 0, 4093, 3965, 0, 63, 0, 0, 0, 0, 0, 4096, 3896, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12628, 17306, 19493, 16379, 13077, 17369, 15187, 19633] avg=16384.0000 max=19633 max_rank=7 min=12628 min_rank=0 imbalance_ratio=1.198303 max_load_reduction=111439 max_load_reduction_pct=0.850212 moved_to_echo=65266 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 148, 2, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3515, 1531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3479, 0, 0, 0, 0, 0, 262, 90, 0, 0, 0, 4096, 0, 0, 4096, 4095, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 3066, 0, 0, 5, 1511, 17, 2707, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4090, 4093, 4068, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3449, 8, 0, 316, 3210, 0, 0, 0, 3430, 4095, 0, 0, 0, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4005, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 1, 0, 628, 4039, 1922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 14, 4088, 4075, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 1030, 637, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 600, 0, 697, 0, 12, 0, 4096, 3913, 0, 0, 0, 0, 0, 0, 0, 4096, 3904, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17299, 16300, 15508, 16403, 15198, 14695, 18333, 17336] avg=16384.0000 max=18333 max_rank=6 min=14695 min_rank=5 imbalance_ratio=1.118958 max_load_reduction=112739 max_load_reduction_pct=0.860130 moved_to_echo=65562 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 10, 0, 4042, 0, 0, 1397, 0, 0, 0, 0, 0, 4096, 3768, 1533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 1, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3066, 0, 0, 0, 234, 0, 3605, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3284, 0, 0, 3180, 3, 6, 0, 1, 3707, 4096, 0, 0, 0, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3551, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4039, 1925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 4096, 4096, 0, 0, 0, 0, 2006, 0, 0, 0, 0, 0, 1030, 638, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 3714, 0, 0, 0, 0, 0, 0, 0, 4096, 3915, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18942, 16523, 15097, 16413, 18227, 13676, 16355, 15839] avg=16384.0000 max=18942 max_rank=0 min=13676 min_rank=5 imbalance_ratio=1.156128 max_load_reduction=112130 max_load_reduction_pct=0.855484 moved_to_echo=64097 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 129, 2, 33, 0, 0, 3386, 0, 0, 0, 1, 0, 4090, 553, 1533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 302, 0, 0, 0, 0, 0, 1, 0, 3, 0, 4095, 3, 0, 4096, 4096, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 8, 0, 3066, 0, 0, 0, 4033, 2, 3472, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4052, 4096, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3196, 0, 0, 3994, 1206, 184, 0, 12, 3707, 3911, 0, 0, 0, 2074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1878, 4039, 1925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 107, 4096, 4096, 0, 0, 0, 0, 1, 0, 0, 91, 0, 2, 1030, 638, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 4096, 4079, 0, 38, 0, 0, 0, 0, 0, 4096, 3914, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13822, 16733, 18774, 16500, 18284, 16019, 14542, 16398] avg=16384.0000 max=18774 max_rank=2 min=13822 min_rank=0 imbalance_ratio=1.145874 max_load_reduction=112298 max_load_reduction_pct=0.856766 moved_to_echo=65243 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 2759, 0, 0, 558, 0, 0, 0, 0, 0, 4096, 1905, 1533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3081, 2, 0, 0, 0, 0, 162, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3066, 0, 0, 0, 2291, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3347, 0, 0, 4087, 200, 11, 0, 2, 3707, 4096, 0, 0, 0, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 4039, 1925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1946, 0, 0, 2, 0, 0, 1030, 638, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 0, 0, 0, 4096, 3556, 0, 0, 0, 0, 0, 0, 0, 4096, 3914, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14948, 15715, 17645, 16423, 15991, 18065, 16293, 15992] avg=16384.0000 max=18065 max_rank=5 min=14948 min_rank=0 imbalance_ratio=1.102600 max_load_reduction=113007 max_load_reduction_pct=0.862175 moved_to_echo=66341 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4076, 244, 0, 4058, 23, 529, 952, 9, 17, 205, 0, 23, 4021, 4090, 1508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 2130, 2050, 80, 0, 6, 0, 0, 0, 21, 0, 75, 4013, 3, 21, 4069, 4075, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 20, 0, 0, 3981, 0, 157, 0, 3038, 0, 39, 0, 1024, 155, 3960, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 3874, 11, 56, 0, 0, 0, 0, 0, 19, 4053, 4081, 3821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 152, 2851, 128, 0, 3382, 0, 1423, 65, 0, 3674, 3344, 12, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 2, 45, 2883, 1, 4086, 0, 78, 0, 6, 33, 0, 7, 0, 0, 4048, 4010, 1907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 0, 1340, 3866, 3962, 16, 24, 15, 0, 1461, 0, 41, 0, 8, 2, 1023, 635, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 366, 0, 0, 0, 0, 29, 4072, 544, 0, 0, 47, 0, 1, 7, 77, 4059, 3901, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19755, 16767, 16400, 15948, 15073, 17245, 16725, 13159] avg=16384.0000 max=19755 max_rank=0 min=13159 min_rank=7 imbalance_ratio=1.205750 max_load_reduction=111317 max_load_reduction_pct=0.849281 moved_to_echo=62202 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 3, 0, 4094, 0, 8, 0, 15, 1, 2, 0, 4, 4090, 4096, 1529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 264, 0, 2, 0, 44, 0, 782, 0, 0, 0, 4094, 0, 0, 4093, 4081, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 17, 61, 0, 3066, 0, 1, 2, 1422, 4, 4091, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3167, 0, 0, 4096, 53, 4, 0, 57, 3671, 4092, 2, 0, 0, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4025, 3994, 1921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 11, 0, 917, 4038, 4082, 0, 11, 0, 0, 107, 0, 0, 17, 0, 56, 1030, 637, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 0, 0, 2, 4071, 3783, 0, 2, 0, 0, 0, 0, 0, 4085, 3912, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17932, 13546, 16842, 16422, 16391, 18130, 15370, 16439] avg=16384.0000 max=18130 max_rank=5 min=13546 min_rank=1 imbalance_ratio=1.106567 max_load_reduction=112942 max_load_reduction_pct=0.861679 moved_to_echo=66330 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3942, 0, 0, 4086, 2, 0, 249, 0, 1, 0, 1, 4, 4088, 1842, 1531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2156, 25, 0, 0, 0, 98, 6, 3570, 5, 8, 0, 3953, 0, 0, 4086, 4092, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 5, 0, 4084, 0, 0, 0, 3036, 37, 10, 0, 28, 0, 12, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4087, 76, 1, 0, 0, 3, 0, 47, 8, 3952, 4096, 4068, 205, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3098, 0, 0, 4096, 322, 0, 0, 5, 3658, 3958, 0, 0, 0, 3690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1, 4031, 0, 0, 0, 0, 0, 0, 0, 40, 17, 1740, 4037, 1923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 3, 0, 3760, 4095, 0, 4, 0, 61, 3997, 0, 2, 146, 0, 86, 1020, 638, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 48, 6, 4091, 4096, 0, 0, 0, 4, 0, 0, 0, 4077, 3913, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15746, 18182, 11313, 16584, 18831, 15873, 18267, 16276] avg=16384.0000 max=18831 max_rank=4 min=11313 min_rank=2 imbalance_ratio=1.149353 max_load_reduction=112241 max_load_reduction_pct=0.856331 moved_to_echo=69247 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32640, 640, 128, 24576, 128, 640, 6784, 128, 128, 256, 128, 128, 32768, 20992, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 23040, 3072, 128, 128, 128, 256, 768, 4608, 128, 128, 128, 32512, 128, 128, 32768, 32768, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 256, 0, 24576, 128, 128, 128, 14720, 256, 26112, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 1024, 0, 5376, 0, 128, 128, 32768, 27776, 0, 128, 128, 128, 128, 128, 128, 32768, 31360, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 256, 23680, 256, 0, 27264, 5120, 1664, 128, 128, 29312, 31744, 128, 128, 0, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32640, 128, 128, 2432, 32256, 32640, 128, 128, 128, 128, 14464, 0, 128, 384, 128, 256, 8320, 5120, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[256, 128, 128, 30720, 128, 32640, 0, 128, 128, 128, 128, 0, 128, 128, 128, 19968, 32256, 15488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 32512, 128, 128, 0, 128, 128, 0, 128, 128, 32640, 32768, 32512, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[26, 0, 0, 0, 0, 0, 0, 0, 4047, 4096, 0, 8, 0, 3336, 4096, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3017, 0, 0, 4096, 0, 4095, 0, 0, 0, 48, 4075, 3, 0, 0, 2613, 3, 4096, 0, 0, 1, 4088, 0, 4096, 1256, 4096, 0, 4095, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1496, 4096, 4096, 0, 0, 1989, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 93, 0, 0, 4096, 0, 6, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 4084, 0, 0, 4093, 4096, 2567, 0, 4096, 4096, 4096, 0, 13, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[12, 0, 3, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 4096, 4096, 0, 0, 3758, 0, 2, 0, 0, 7, 0, 0, 3, 0, 0, 4094, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4047, 0, 0, 0, 11, 1, 4096, 0, 0, 1, 4095, 0, 4096, 334, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 49, 0, 0, 0, 4090, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4070, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 4094, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 0, 4096, 4096, 0, 0, 0, 78, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2630, 0, 0, 4096, 0, 4096, 0, 0, 2, 0, 2449, 0, 0, 0, 287, 0, 4096, 0, 0, 0, 4096, 0, 4093, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2849, 4096, 4096, 0, 0, 3874, 4096, 0, 0, 557, 0, 0, 0, 3753, 0, 0, 0, 0, 73, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 4096, 4096, 3932, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 6, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3757, 0, 0, 4096, 0, 4096, 0, 0, 7, 0, 2104, 0, 0, 0, 3761, 0, 4096, 0, 0, 0, 4096, 0, 3296, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 4096, 4096, 0, 0, 4025, 4096, 0, 0, 392, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 4096, 4096, 3021, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 4060, 4096, 4096, 0, 0, 4075, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4076, 0, 0, 4096, 0, 4096, 0, 0, 8, 118, 3617, 0, 0, 0, 4, 23, 4096, 3, 0, 0, 4096, 0, 4096, 48, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 445, 4061, 4095, 0, 0, 4096, 4096, 0, 0, 31, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4091, 0, 0, 4096, 4096, 4091, 0, 4096, 4096, 3990, 0, 13, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 0, 0, 4093, 4096, 0, 0, 0, 3759, 4096, 4090, 0, 0, 4027, 0, 0, 0, 10, 0, 0, 2, 0, 0, 0, 2145, 0, 3, 4092, 14, 4096, 0, 0, 0, 1, 4095, 0, 0, 0, 132, 8, 4093, 0, 1, 0, 4096, 0, 4096, 4050, 4093, 0, 4095, 0, 0, 2, 1, 0, 12, 0, 3, 0, 1, 4046, 4083, 4096, 0, 0, 944, 4094, 0, 0, 10, 5, 3, 0, 4096, 0, 1, 2, 1, 4, 0, 5, 4086, 0, 0, 0, 18, 4090, 4093, 0, 0, 0, 2, 1, 3, 4085, 1, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4089, 0, 0, 60, 4003, 0, 0, 4095, 4096, 1357, 0, 4090, 4094, 4095, 4, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[192, 0, 102, 20, 207, 69, 640, 0, 4057, 4068, 0, 0, 0, 324, 4062, 4066, 0, 0, 4076, 26, 0, 10, 2, 0, 15, 5, 0, 0, 25, 393, 5, 81, 3975, 0, 4085, 2, 31, 354, 303, 4084, 17, 0, 21, 0, 784, 3960, 24, 0, 0, 4073, 2, 3873, 3990, 4013, 80, 4049, 0, 0, 21, 0, 0, 0, 50, 366, 0, 0, 2867, 3733, 4042, 802, 5, 2164, 4039, 0, 0, 872, 22, 40, 4, 4078, 26, 12, 1, 5, 0, 11, 51, 4010, 28, 0, 0, 115, 4064, 4019, 0, 13, 0, 4, 25, 25, 3709, 0, 37, 19, 0, 97, 18, 14, 0, 0, 35, 4065, 12, 0, 530, 3550, 0, 50, 4056, 4059, 3637, 3, 3998, 4064, 3392, 38, 0, 10] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 18, 0, 4095, 4094, 0, 15, 2, 912, 4077, 4078, 1, 0, 4085, 1, 0, 0, 2, 55, 45, 8, 0, 2, 0, 18, 3, 0, 4094, 0, 4082, 0, 0, 0, 0, 4094, 0, 0, 7, 3488, 0, 4091, 0, 0, 94, 4090, 3, 3481, 4056, 4089, 0, 4092, 7, 0, 0, 0, 57, 0, 0, 0, 2, 0, 4074, 4015, 4091, 0, 1, 6, 4045, 1, 19, 86, 0, 9, 5, 4043, 6, 0, 20, 16, 233, 0, 0, 4085, 0, 0, 0, 11, 4092, 4096, 0, 0, 2, 5, 2, 2, 4058, 0, 3, 0, 0, 4, 0, 0, 15, 3, 0, 4095, 1, 1, 0, 4095, 0, 12, 4094, 4090, 4088, 0, 4038, 4071, 4096, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[26, 0, 0, 0, 0, 0, 0, 0, 4047, 2398, 0, 8, 0, 3336, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3017, 0, 0, 4096, 4093, 1613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3148, 0, 0, 0, 48, 4075, 3, 0, 0, 2613, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1491, 0, 4096, 1256, 4096, 0, 4095, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1496, 4096, 0, 0, 0, 1989, 2803, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 4096, 0, 6, 0, 0, 4096, 4096, 0, 0, 1698, 1293, 947, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3234, 2502, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4084, 0, 0, 0, 0, 2567, 0, 4096, 862, 4096, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18007, 16908, 18082, 15042, 14480, 16420, 16411, 15722] avg=16384.0000 max=18082 max_rank=2 min=14480 min_rank=4 imbalance_ratio=1.103638 max_load_reduction=112990 max_load_reduction_pct=0.862045 moved_to_echo=63033 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 0, 4096, 2398, 0, 0, 0, 78, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2630, 0, 0, 4096, 4096, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3148, 0, 0, 2, 0, 2449, 0, 0, 0, 287, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1493, 0, 4093, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2849, 4096, 0, 0, 0, 3874, 2803, 0, 0, 557, 0, 0, 0, 3753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 1698, 1293, 948, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3234, 2507, 2484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3932, 0, 4096, 862, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14767, 16530, 14078, 17873, 17932, 16396, 16417, 17079] avg=16384.0000 max=17932 max_rank=4 min=14078 min_rank=2 imbalance_ratio=1.094482 max_load_reduction=113140 max_load_reduction_pct=0.863190 moved_to_echo=67824 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 3, 0, 0, 0, 0, 0, 4096, 2398, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3758, 0, 2, 0, 0, 7, 0, 0, 3, 0, 0, 4094, 0, 0, 4096, 4096, 1613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3148, 0, 0, 0, 0, 4047, 0, 0, 0, 11, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1493, 0, 4096, 334, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2803, 0, 0, 49, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 1698, 1293, 948, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3234, 2506, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 4096, 0, 4096, 862, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14703, 17669, 15399, 14112, 19230, 16327, 16414, 17218] avg=16384.0000 max=19230 max_rank=4 min=14112 min_rank=3 imbalance_ratio=1.173706 max_load_reduction=111842 max_load_reduction_pct=0.853287 moved_to_echo=69189 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 0, 0, 0, 0, 4096, 2398, 1, 0, 0, 4060, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4076, 0, 0, 4096, 4096, 1613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3148, 0, 0, 8, 118, 3617, 0, 0, 0, 4, 23, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1493, 0, 4096, 48, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 445, 4061, 0, 0, 0, 4096, 2803, 0, 0, 31, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 6, 0, 1698, 1293, 948, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3234, 2507, 2482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4091, 0, 4096, 862, 3990, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18750, 17957, 15113, 13830, 15540, 16329, 16410, 17143] avg=16384.0000 max=18750 max_rank=0 min=13830 min_rank=3 imbalance_ratio=1.144409 max_load_reduction=112322 max_load_reduction_pct=0.856949 moved_to_echo=65422 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 2398, 0, 0, 0, 6, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3757, 0, 0, 4096, 4096, 1613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3148, 0, 0, 7, 0, 2104, 0, 0, 0, 3761, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1493, 0, 3296, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 4096, 0, 0, 0, 4025, 2803, 0, 0, 392, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 1698, 1293, 948, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3234, 2507, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 3021, 0, 4096, 862, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14692, 17658, 17212, 17076, 15522, 16323, 16416, 16173] avg=16384.0000 max=17658 max_rank=1 min=14692 min_rank=0 imbalance_ratio=1.077759 max_load_reduction=113414 max_load_reduction_pct=0.865280 moved_to_echo=64434 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[192, 0, 102, 20, 207, 69, 640, 0, 4057, 2375, 0, 0, 0, 324, 4062, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 26, 0, 10, 2, 0, 15, 5, 0, 0, 25, 393, 5, 81, 4059, 4056, 1592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 3137, 2, 31, 354, 303, 4084, 17, 0, 21, 0, 784, 3960, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1479, 2, 3873, 3990, 4013, 80, 4049, 0, 0, 21, 0, 0, 0, 50, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2867, 3733, 0, 802, 5, 2164, 2762, 0, 0, 872, 22, 40, 4, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 12, 1, 5, 0, 11, 51, 4010, 28, 0, 0, 115, 4064, 4019, 0, 13, 1693, 1277, 948, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 25, 25, 3709, 0, 37, 19, 0, 97, 18, 14, 0, 0, 35, 4065, 3210, 2496, 2450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 530, 3550, 0, 50, 0, 0, 3637, 3, 3998, 854, 3392, 38, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16114, 14345, 16692, 17923, 17349, 16371, 16204, 16074] avg=16384.0000 max=17923 max_rank=3 min=14345 min_rank=1 imbalance_ratio=1.093933 max_load_reduction=113149 max_load_reduction_pct=0.863258 moved_to_echo=65998 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 18, 0, 4095, 2397, 0, 15, 2, 912, 4077, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4085, 1, 0, 0, 2, 55, 45, 8, 0, 2, 0, 18, 3, 0, 4090, 4094, 1611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3138, 0, 0, 0, 0, 4094, 0, 0, 7, 3488, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 1492, 3, 3481, 4056, 4089, 0, 4092, 7, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4074, 4015, 0, 0, 1, 6, 2768, 1, 19, 86, 0, 9, 5, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 20, 16, 233, 0, 0, 4085, 0, 0, 0, 11, 4092, 4096, 0, 0, 1697, 1277, 944, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 2, 2, 4058, 0, 3, 0, 0, 4, 0, 0, 15, 3, 0, 4095, 3214, 2503, 2480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 12, 0, 0, 4088, 0, 4038, 857, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15597, 14015, 18912, 17371, 15029, 16572, 16386, 17190] avg=16384.0000 max=18912 max_rank=2 min=14015 min_rank=1 imbalance_ratio=1.154297 max_load_reduction=112160 max_load_reduction_pct=0.855713 moved_to_echo=65177 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 0, 0, 4093, 2398, 0, 0, 0, 3759, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 10, 0, 0, 2, 0, 0, 0, 2145, 0, 3, 4096, 4095, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 14, 3148, 0, 0, 0, 1, 4095, 0, 0, 0, 132, 8, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1493, 0, 4096, 4050, 4093, 0, 4095, 0, 0, 2, 1, 0, 12, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4046, 4083, 0, 0, 0, 944, 2802, 0, 0, 10, 5, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 4, 0, 5, 4086, 0, 0, 0, 18, 4090, 4093, 0, 0, 1698, 1292, 948, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 3, 4085, 1, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4089, 3232, 2507, 2484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 4003, 0, 0, 0, 0, 1357, 0, 4090, 862, 4095, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18442, 15990, 15584, 17845, 15990, 16334, 16414, 14473] avg=16384.0000 max=18442 max_rank=0 min=14473 min_rank=7 imbalance_ratio=1.125610 max_load_reduction=112630 max_load_reduction_pct=0.859299 moved_to_echo=63211 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 25216, 128, 128, 384, 512, 28672, 128, 0, 128, 10368, 896, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 0, 128, 128, 256, 128, 768, 0, 32768, 19200, 128, 128, 128, 12544, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 640, 32128, 0, 128, 0, 0, 26880, 128, 32640, 6912, 32000, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 32384, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 32768, 25856, 20096, 19840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 20096, 32384, 0, 896, 128, 21248, 22400, 128, 128, 2048, 128, 128, 128, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 0, 32384, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 20224, 128, 128, 32768, 32768, 12928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 12032, 128, 31232, 22016, 32768, 128, 32768, 128, 0, 128, 128, 128, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 512, 128, 128, 32768, 128, 128, 0, 256, 32768, 32768, 128, 128, 13696, 10368, 7680, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True + [2026-08-30 05:04:58] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 743.7 | throughput per GPU (TFLOP/s/GPU): 448.3 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.300478E+01 | loss scale: 1.0 | grad norm: 46.600 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[16, 477, 1919, 1, 1517, 4, 768, 9, 1618, 117, 1544, 591, 1117, 298, 0, 334, 304, 1675, 781, 1107, 3822, 994, 282, 78, 1, 2840, 188, 124, 880, 27, 826, 1745, 4008, 1, 13, 943, 2041, 39, 708, 1109, 2835, 3035, 600, 207, 1, 1047, 792, 1016, 542, 372, 637, 97, 40, 13, 3904, 185, 0, 2051, 3679, 2730, 218, 378, 451, 2, 849, 4040, 0, 1150, 177, 70, 2898, 2372, 717, 573, 202, 2, 214, 1626, 1272, 837, 2107, 521, 49, 3933, 127, 1082, 2275, 3936, 37, 30, 628, 1004, 2666, 265, 1400, 2, 1169, 368, 3946, 216, 3, 1882, 891, 1660, 3955, 2189, 39, 1378, 582, 1304, 179, 1410, 29, 1305, 1759, 436, 7, 1237, 323, 1659, 710, 797, 2357, 1102, 361, 10, 1041, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[11, 369, 2266, 0, 2013, 0, 664, 9, 904, 123, 1019, 676, 1262, 278, 0, 305, 208, 1860, 707, 1643, 3514, 934, 333, 89, 1, 2555, 173, 247, 566, 24, 618, 1973, 4013, 6, 12, 1007, 2432, 19, 635, 1531, 2493, 1994, 331, 155, 2, 878, 1086, 739, 594, 617, 720, 152, 24, 29, 3973, 287, 0, 2317, 3651, 2146, 132, 276, 453, 5, 686, 4034, 0, 1720, 256, 34, 3060, 2808, 505, 822, 315, 5, 304, 1934, 983, 726, 2253, 261, 84, 3892, 142, 586, 2513, 4048, 88, 58, 1026, 876, 2647, 225, 1248, 1, 867, 398, 3989, 264, 2, 1996, 814, 1756, 3799, 1636, 17, 1384, 812, 2273, 218, 2103, 150, 911, 1286, 511, 23, 979, 232, 1647, 811, 735, 2312, 919, 447, 8, 1579, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[22, 466, 1725, 3, 1415, 2, 769, 41, 1551, 193, 1630, 637, 951, 346, 10, 459, 306, 1189, 1019, 1352, 3806, 1153, 333, 135, 1, 2433, 190, 151, 1611, 90, 1061, 1618, 3812, 4, 35, 1232, 1644, 30, 921, 1228, 3206, 2617, 643, 181, 0, 802, 504, 570, 549, 334, 558, 196, 102, 34, 3864, 168, 2, 2044, 3209, 2348, 329, 248, 848, 0, 980, 4003, 12, 1303, 195, 95, 2592, 2028, 572, 695, 326, 8, 88, 1723, 943, 724, 1357, 968, 96, 3711, 342, 855, 2929, 3771, 63, 102, 566, 497, 2569, 315, 1664, 6, 990, 902, 3751, 309, 2, 1750, 1022, 2158, 3723, 2148, 67, 1604, 857, 1763, 169, 1446, 39, 1026, 1384, 624, 37, 1523, 933, 1707, 545, 636, 1935, 737, 676, 13, 1561, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[23, 791, 2416, 2, 1916, 7, 1044, 27, 1400, 76, 992, 709, 1302, 282, 2, 413, 156, 1995, 526, 1511, 3679, 955, 334, 35, 2, 2520, 158, 276, 887, 19, 761, 1944, 4035, 2, 19, 809, 1731, 24, 780, 1403, 2647, 2726, 283, 199, 6, 1259, 1054, 971, 543, 420, 482, 100, 27, 32, 3843, 139, 2, 2772, 3829, 2734, 201, 341, 271, 1, 755, 3987, 7, 1139, 175, 36, 2762, 2610, 862, 588, 185, 22, 195, 1805, 1465, 736, 2021, 619, 77, 3905, 129, 1044, 1850, 4043, 38, 54, 721, 806, 3252, 364, 1042, 8, 632, 431, 3916, 288, 37, 1723, 761, 1957, 3469, 2112, 4, 1502, 1100, 1655, 256, 1617, 21, 717, 999, 225, 5, 567, 154, 1787, 788, 884, 2421, 1148, 558, 10, 1168, 13] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[14, 300, 2098, 0, 1463, 0, 913, 5, 1363, 90, 1506, 645, 663, 349, 1, 386, 299, 1789, 607, 1527, 3877, 780, 519, 40, 0, 2847, 205, 127, 732, 15, 592, 2056, 3947, 14, 18, 990, 2367, 19, 405, 1602, 3227, 2594, 162, 228, 8, 1189, 1046, 792, 505, 399, 859, 128, 12, 4, 3885, 90, 0, 2560, 3549, 2611, 219, 305, 282, 1, 642, 4033, 1, 1445, 170, 47, 3121, 2365, 417, 816, 217, 18, 131, 1807, 1373, 824, 2324, 588, 54, 3809, 144, 780, 2541, 4057, 99, 64, 673, 516, 3346, 264, 1146, 4, 717, 542, 4016, 221, 0, 1943, 832, 2501, 3786, 1776, 5, 1181, 714, 1825, 108, 2057, 135, 686, 791, 302, 7, 969, 286, 1249, 569, 957, 1917, 1481, 496, 6, 1334, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[21, 486, 1980, 4, 1338, 0, 658, 10, 1513, 275, 1482, 842, 1304, 530, 0, 413, 239, 1883, 1321, 1335, 3726, 1269, 374, 124, 2, 2217, 185, 276, 1110, 24, 880, 2080, 3883, 1, 25, 1599, 2158, 24, 375, 1596, 2935, 2595, 243, 131, 15, 960, 961, 698, 520, 547, 689, 142, 51, 21, 3770, 204, 1, 2595, 3605, 2540, 353, 327, 390, 7, 971, 3913, 8, 1277, 175, 161, 2010, 2456, 289, 984, 254, 4, 114, 1772, 1188, 511, 1241, 863, 73, 3809, 192, 918, 2208, 4004, 24, 69, 361, 914, 2952, 424, 841, 0, 609, 479, 3695, 437, 21, 2122, 1295, 2654, 3640, 2169, 30, 1352, 653, 2193, 109, 1767, 191, 891, 1119, 680, 8, 529, 434, 1559, 751, 589, 1996, 659, 750, 4, 839, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[13, 552, 2102, 0, 1484, 0, 722, 14, 1383, 95, 872, 951, 1772, 86, 5, 336, 281, 1408, 996, 1044, 3806, 1155, 470, 141, 6, 2183, 281, 255, 563, 67, 686, 2040, 3785, 0, 19, 971, 1607, 27, 463, 1172, 3251, 2817, 567, 377, 25, 568, 743, 1015, 569, 475, 586, 105, 16, 39, 3715, 104, 0, 2753, 3704, 1928, 148, 269, 442, 23, 516, 3813, 3, 1347, 874, 82, 2267, 2891, 421, 1042, 495, 9, 161, 1551, 1164, 456, 1632, 611, 63, 3695, 147, 720, 2972, 4031, 53, 121, 623, 1053, 2538, 453, 1031, 31, 936, 689, 3681, 280, 5, 1407, 1479, 1550, 3760, 2226, 19, 905, 1418, 1720, 109, 2504, 175, 1090, 427, 860, 19, 905, 796, 1806, 533, 577, 2628, 1866, 630, 7, 1141, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[10, 608, 1816, 4, 1870, 0, 626, 15, 1373, 110, 1781, 884, 1243, 241, 5, 425, 248, 1162, 882, 1384, 3797, 998, 404, 82, 3, 2549, 179, 404, 688, 22, 860, 1568, 3933, 2, 10, 949, 1734, 19, 552, 1328, 3102, 2673, 404, 180, 28, 1130, 943, 1314, 450, 567, 803, 107, 31, 23, 3951, 127, 4, 2527, 3611, 2539, 323, 462, 290, 3, 1066, 3984, 0, 1725, 387, 117, 2869, 2397, 705, 661, 296, 5, 76, 1063, 1193, 794, 2003, 1179, 61, 3764, 125, 833, 2812, 3985, 51, 27, 722, 1052, 2366, 217, 1741, 9, 752, 466, 3697, 219, 3, 1284, 1233, 2484, 3866, 1772, 47, 1176, 1047, 1174, 220, 1896, 39, 783, 833, 349, 17, 573, 263, 1645, 753, 832, 2103, 1145, 404, 0, 1349, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 369, 2266, 0, 2013, 0, 664, 9, 904, 123, 1019, 676, 1262, 278, 0, 305, 3989, 2092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 1860, 707, 1643, 3514, 934, 333, 89, 1, 2555, 173, 247, 566, 24, 618, 1973, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2411, 6, 12, 1007, 2432, 19, 635, 1531, 2493, 1994, 331, 155, 2, 878, 1086, 739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 617, 720, 152, 24, 29, 3973, 287, 0, 2317, 3651, 2146, 132, 276, 453, 5, 812, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 686, 3314, 0, 1720, 256, 34, 3060, 2808, 505, 822, 315, 5, 304, 1934, 983, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2253, 261, 84, 3892, 142, 586, 2513, 399, 88, 58, 1026, 876, 2647, 225, 1248, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 867, 398, 0, 264, 2, 1996, 814, 1756, 1858, 1636, 17, 1384, 812, 2273, 218, 2103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 911, 1286, 511, 23, 979, 232, 1647, 811, 735, 2312, 919, 447, 8, 1579, 1, 1941, 1557, 790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15980, 15962, 15731, 16391, 17472, 16299, 16398, 16839] avg=16384.0000 max=17472 max_rank=4 min=15731 min_rank=2 imbalance_ratio=1.066406 max_load_reduction=113600 max_load_reduction_pct=0.866699 moved_to_echo=67008 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[23, 791, 2416, 2, 1916, 7, 1044, 27, 1400, 76, 992, 709, 1302, 282, 2, 413, 3916, 2090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 1995, 526, 1511, 3679, 955, 334, 35, 2, 2520, 158, 276, 887, 19, 761, 1944, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2425, 2, 19, 809, 1731, 24, 780, 1403, 2647, 2726, 283, 199, 6, 1259, 1054, 971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 420, 482, 100, 27, 32, 3843, 139, 2, 2772, 3829, 2734, 201, 341, 271, 1, 816, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 755, 3276, 7, 1139, 175, 36, 2762, 2610, 862, 588, 185, 22, 195, 1805, 1465, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2021, 619, 77, 3905, 129, 1044, 1850, 398, 38, 54, 721, 806, 3252, 364, 1042, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 632, 431, 0, 288, 37, 1723, 761, 1957, 1697, 2112, 4, 1502, 1100, 1655, 256, 1617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 717, 999, 225, 5, 567, 154, 1787, 788, 884, 2421, 1148, 558, 10, 1168, 13, 1772, 1555, 794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17408, 16269, 16338, 16753, 16618, 16328, 15772, 15586] avg=16384.0000 max=17408 max_rank=0 min=15586 min_rank=7 imbalance_ratio=1.062500 max_load_reduction=113664 max_load_reduction_pct=0.867188 moved_to_echo=64304 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[22, 466, 1725, 3, 1415, 2, 769, 41, 1551, 193, 1630, 637, 951, 346, 10, 459, 3751, 1949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 1189, 1019, 1352, 3806, 1153, 333, 135, 1, 2433, 190, 151, 1611, 90, 1061, 1618, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2291, 4, 35, 1232, 1644, 30, 921, 1228, 3206, 2617, 643, 181, 0, 802, 504, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 334, 558, 196, 102, 34, 3864, 168, 2, 2044, 3209, 2348, 329, 248, 848, 0, 771, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 980, 3289, 12, 1303, 195, 95, 2592, 2028, 572, 695, 326, 8, 88, 1723, 943, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1357, 968, 96, 3711, 342, 855, 2929, 371, 63, 102, 566, 497, 2569, 315, 1664, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 990, 902, 0, 309, 2, 1750, 1022, 2158, 1821, 2148, 67, 1604, 857, 1763, 169, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1026, 1384, 624, 37, 1523, 933, 1707, 545, 636, 1935, 737, 676, 13, 1561, 7, 1902, 1451, 750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15920, 16961, 15908, 15805, 15573, 16411, 17008, 17486] avg=16384.0000 max=17486 max_rank=7 min=15573 min_rank=4 imbalance_ratio=1.067261 max_load_reduction=113586 max_load_reduction_pct=0.866592 moved_to_echo=66478 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[14, 300, 2098, 0, 1463, 0, 913, 5, 1363, 90, 1506, 645, 663, 349, 1, 386, 4016, 2097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 1789, 607, 1527, 3877, 780, 519, 40, 0, 2847, 205, 127, 732, 15, 592, 2056, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2371, 14, 18, 990, 2367, 19, 405, 1602, 3227, 2594, 162, 228, 8, 1189, 1046, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 399, 859, 128, 12, 4, 3885, 90, 0, 2560, 3549, 2611, 219, 305, 282, 1, 799, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 642, 3313, 1, 1445, 170, 47, 3121, 2365, 417, 816, 217, 18, 131, 1807, 1373, 824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2324, 588, 54, 3809, 144, 780, 2541, 399, 99, 64, 673, 516, 3346, 264, 1146, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 717, 542, 0, 221, 0, 1943, 832, 2501, 1852, 1776, 5, 1181, 714, 1825, 108, 2057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 686, 791, 302, 7, 969, 286, 1249, 569, 957, 1917, 1481, 496, 6, 1334, 2, 1934, 1561, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15909, 16529, 17032, 16411, 16707, 16751, 16274, 15459] avg=16384.0000 max=17032 max_rank=2 min=15459 min_rank=7 imbalance_ratio=1.039551 max_load_reduction=114040 max_load_reduction_pct=0.870056 moved_to_echo=65191 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[21, 486, 1980, 4, 1338, 0, 658, 10, 1513, 275, 1482, 842, 1304, 530, 0, 413, 3695, 2074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 1883, 1321, 1335, 3726, 1269, 374, 124, 2, 2217, 185, 276, 1110, 24, 880, 2080, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2325, 1, 25, 1599, 2158, 24, 375, 1596, 2935, 2595, 243, 131, 15, 960, 961, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 547, 689, 142, 51, 21, 3770, 204, 1, 2595, 3605, 2540, 353, 327, 390, 7, 790, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 971, 3208, 8, 1277, 175, 161, 2010, 2456, 289, 984, 254, 4, 114, 1772, 1188, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1241, 863, 73, 3809, 192, 918, 2208, 387, 24, 69, 361, 914, 2952, 424, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 609, 479, 0, 437, 21, 2122, 1295, 2654, 1776, 2169, 30, 1352, 653, 2193, 109, 1767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 891, 1119, 680, 8, 529, 434, 1559, 751, 589, 1996, 659, 750, 4, 839, 6, 1864, 1543, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16625, 17550, 16641, 16752, 15382, 15276, 17666, 15180] avg=16384.0000 max=17666 max_rank=6 min=15180 min_rank=7 imbalance_ratio=1.078247 max_load_reduction=113406 max_load_reduction_pct=0.865219 moved_to_echo=63504 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[13, 552, 2102, 0, 1484, 0, 722, 14, 1383, 95, 872, 951, 1772, 86, 5, 336, 3681, 2084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 1408, 996, 1044, 3806, 1155, 470, 141, 6, 2183, 281, 255, 563, 67, 686, 2040, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2274, 0, 19, 971, 1607, 27, 463, 1172, 3251, 2817, 567, 377, 25, 568, 743, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 569, 475, 586, 105, 16, 39, 3715, 104, 0, 2753, 3704, 1928, 148, 269, 442, 23, 766, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 3134, 3, 1347, 874, 82, 2267, 2891, 421, 1042, 495, 9, 161, 1551, 1164, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1632, 611, 63, 3695, 147, 720, 2972, 396, 53, 121, 623, 1053, 2538, 453, 1031, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 936, 689, 0, 280, 5, 1407, 1479, 1550, 1839, 2226, 19, 905, 1418, 1720, 109, 2504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 1090, 427, 860, 19, 905, 796, 1806, 533, 577, 2628, 1866, 630, 7, 1141, 6, 1921, 1551, 745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16152, 15870, 15896, 15833, 16413, 16139, 17086, 17683] avg=16384.0000 max=17683 max_rank=7 min=15833 min_rank=3 imbalance_ratio=1.079285 max_load_reduction=113389 max_load_reduction_pct=0.865089 moved_to_echo=67321 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[10, 608, 1816, 4, 1870, 0, 626, 15, 1373, 110, 1781, 884, 1243, 241, 5, 425, 3697, 2060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 1162, 882, 1384, 3797, 998, 404, 82, 3, 2549, 179, 404, 688, 22, 860, 1568, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2363, 2, 10, 949, 1734, 19, 552, 1328, 3102, 2673, 404, 180, 28, 1130, 943, 1314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 567, 803, 107, 31, 23, 3951, 127, 4, 2527, 3611, 2539, 323, 462, 290, 3, 796, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 3274, 0, 1725, 387, 117, 2869, 2397, 705, 661, 296, 5, 76, 1063, 1193, 794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2003, 1179, 61, 3764, 125, 833, 2812, 392, 51, 27, 722, 1052, 2366, 217, 1741, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 752, 466, 0, 219, 3, 1284, 1233, 2484, 1891, 1772, 47, 1176, 1047, 1174, 220, 1896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 783, 833, 349, 17, 573, 263, 1645, 753, 832, 2103, 1145, 404, 0, 1349, 3, 1975, 1533, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16768, 15740, 16731, 16814, 16628, 17354, 15664, 15373] avg=16384.0000 max=17354 max_rank=5 min=15373 min_rank=7 imbalance_ratio=1.059204 max_load_reduction=113718 max_load_reduction_pct=0.867599 moved_to_echo=65019 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[16, 477, 1919, 1, 1517, 4, 768, 9, 1618, 117, 1544, 591, 1117, 298, 0, 334, 3946, 2034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 1675, 781, 1107, 3822, 994, 282, 78, 1, 2840, 188, 124, 880, 27, 826, 1745, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2408, 1, 13, 943, 2041, 39, 708, 1109, 2835, 3035, 600, 207, 1, 1047, 792, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 372, 637, 97, 40, 13, 3904, 185, 0, 2051, 3679, 2730, 218, 378, 451, 2, 811, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 3320, 0, 1150, 177, 70, 2898, 2372, 717, 573, 202, 2, 214, 1626, 1272, 837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2107, 521, 49, 3933, 127, 1082, 2275, 388, 37, 30, 628, 1004, 2666, 265, 1400, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1169, 368, 0, 216, 3, 1882, 891, 1660, 1934, 2189, 39, 1378, 582, 1304, 179, 1410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 1305, 1759, 436, 7, 1237, 323, 1659, 710, 797, 2357, 1102, 361, 10, 1041, 9, 2021, 1514, 789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16310, 16191, 16795, 16313, 16279, 16514, 15204, 17466] avg=16384.0000 max=17466 max_rank=7 min=15204 min_rank=6 imbalance_ratio=1.066040 max_load_reduction=113606 max_load_reduction_pct=0.866745 moved_to_echo=65463 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[896, 7424, 8704, 4096, 128, 7296, 3456, 13184, 5504, 6016, 17792, 9088, 4352, 128, 10112, 128, 15360, 12288, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[6784, 4352, 0, 2304, 128, 14208, 8448, 16768, 14720, 16128, 256, 10496, 7296, 13952, 1408, 14848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 4096, 16384, 128, 13056, 128, 6272, 256, 11136, 1152, 10880, 6016, 9728, 2432, 128, 3072, 30720, 16512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[18944, 128, 256, 8576, 15744, 256, 4864, 11008, 23808, 21120, 3328, 1664, 128, 7936, 7168, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[14976, 5632, 640, 30592, 1408, 6912, 20224, 3200, 512, 640, 5376, 6784, 22400, 2560, 10240, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2048, 13056, 6912, 11008, 30080, 8320, 3072, 768, 128, 20224, 1664, 1920, 7040, 384, 6400, 15104, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[6528, 26240, 128, 11136, 2432, 768, 21632, 19968, 4608, 6272, 2304, 128, 1408, 13312, 9600, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[4352, 3840, 5376, 1152, 384, 256, 30976, 1408, 128, 19712, 28928, 19584, 2048, 2688, 3456, 128, 6400, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[2, 32, 4086, 67, 4071, 2, 0, 4, 230, 4043, 19, 1380, 9, 0, 4012, 0, 0, 11, 39, 1, 4085, 4037, 2, 3, 8, 9, 0, 0, 4, 517, 3, 0, 4049, 0, 3159, 0, 58, 0, 15, 0, 0, 125, 3374, 705, 45, 0, 2, 7, 1829, 1689, 4013, 3, 0, 0, 62, 4, 4086, 2, 0, 0, 4078, 1, 5, 0, 1747, 0, 0, 3920, 0, 2393, 0, 4083, 3541, 4052, 0, 0, 2, 0, 10, 0, 0, 102, 66, 21, 4091, 4095, 4042, 76, 3447, 6, 0, 4076, 9, 12, 150, 15, 0, 4073, 4014, 0, 57, 6, 0, 0, 0, 33, 0, 106, 5, 2, 4042, 5, 49, 0, 11, 4, 7, 4089, 4089, 4082, 0, 3720, 2, 4062, 2, 4054, 10, 605] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2827, 4096, 0, 0, 1, 1231, 4095, 24, 1132, 0, 0, 4073, 0, 0, 0, 0, 0, 4096, 4076, 0, 0, 166, 0, 0, 0, 0, 993, 0, 0, 4096, 0, 1405, 0, 0, 0, 2, 0, 12, 0, 1351, 7, 0, 0, 0, 187, 3695, 1445, 4096, 0, 0, 0, 5, 0, 4095, 0, 7, 0, 4096, 0, 0, 0, 1575, 0, 0, 4020, 0, 842, 0, 4096, 4071, 4093, 91, 0, 0, 0, 2, 0, 0, 90, 0, 17, 4096, 4096, 4096, 0, 4037, 0, 0, 4077, 0, 0, 199, 0, 0, 4095, 4096, 1, 1, 0, 0, 0, 0, 33, 0, 0, 23, 0, 4078, 0, 11, 1, 0, 0, 0, 3452, 4093, 3961, 0, 4069, 0, 4096, 0, 4096, 0, 63] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4078, 242, 4088, 0, 1, 17, 893, 4042, 0, 2056, 0, 0, 4030, 0, 0, 1, 0, 0, 4095, 4082, 0, 7, 110, 0, 7, 27, 0, 106, 0, 0, 4096, 2, 2071, 0, 0, 0, 4, 0, 0, 3, 3363, 56, 0, 0, 0, 0, 3349, 2455, 4096, 0, 0, 0, 100, 0, 4096, 0, 0, 0, 4094, 0, 1, 0, 2196, 9, 0, 4034, 0, 596, 0, 3990, 4086, 4029, 6, 0, 0, 0, 20, 0, 0, 0, 0, 0, 4096, 4096, 4072, 0, 3961, 0, 0, 4017, 0, 0, 1, 0, 0, 4096, 4093, 0, 212, 0, 0, 0, 0, 16, 0, 63, 86, 0, 3937, 1, 284, 0, 0, 0, 0, 3897, 4091, 3151, 7, 4022, 1, 4061, 0, 4094, 0, 83] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 4095, 1277, 4096, 0, 0, 10, 489, 4095, 2, 1059, 0, 0, 4044, 0, 0, 0, 0, 0, 4095, 4084, 0, 2, 285, 0, 3, 2, 0, 57, 0, 0, 4096, 0, 1897, 0, 0, 0, 0, 0, 7, 0, 698, 89, 0, 0, 0, 26, 3549, 2589, 4079, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 4088, 0, 3, 0, 3945, 0, 0, 4093, 3, 26, 0, 4091, 4083, 4094, 21, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4094, 4095, 0, 3973, 0, 0, 4076, 1, 0, 77, 0, 2, 4096, 4096, 0, 0, 1, 0, 0, 0, 5, 0, 1, 23, 0, 4065, 0, 52, 7, 0, 0, 0, 4049, 4092, 3955, 0, 4095, 25, 4096, 1, 4091, 0, 638] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4096, 746, 4093, 0, 0, 4, 3379, 4096, 0, 217, 0, 0, 4081, 0, 0, 0, 0, 0, 4094, 4091, 0, 0, 189, 0, 0, 0, 0, 261, 0, 0, 4096, 0, 1824, 0, 0, 5, 0, 0, 0, 12, 2806, 1, 0, 0, 0, 6, 916, 477, 4094, 0, 0, 0, 57, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 1688, 0, 0, 4094, 1, 738, 0, 4086, 3917, 4095, 215, 0, 0, 0, 0, 0, 0, 57, 0, 0, 4096, 4096, 4096, 0, 4042, 0, 0, 4016, 0, 1, 560, 0, 0, 4096, 4096, 0, 87, 0, 0, 0, 0, 1, 0, 0, 108, 0, 4096, 0, 2, 0, 0, 0, 0, 4054, 4096, 3754, 1, 4092, 0, 4096, 0, 4096, 0, 2767] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 4, 4095, 1855, 4093, 1, 0, 0, 371, 4095, 2, 562, 0, 20, 4072, 6, 0, 13, 6, 0, 4092, 4085, 2, 2, 1652, 1, 0, 0, 3, 264, 4, 1, 4091, 0, 1180, 0, 19, 12, 0, 2, 0, 6, 2857, 0, 0, 0, 0, 416, 2857, 1457, 4091, 1, 2, 0, 76, 2, 4093, 3, 0, 0, 4085, 0, 0, 0, 2873, 10, 0, 4017, 0, 790, 0, 4034, 3335, 4050, 2, 7, 2, 3, 1, 2, 0, 0, 2, 0, 4093, 4094, 4054, 0, 3712, 2, 2, 4072, 0, 0, 89, 1, 0, 4094, 4094, 0, 1, 0, 0, 2, 0, 1, 0, 0, 60, 0, 4091, 0, 108, 0, 0, 0, 3, 3966, 4092, 4057, 2, 4091, 1, 4044, 0, 4094, 0, 473] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4091, 1199, 4094, 1, 0, 0, 1295, 4096, 0, 433, 0, 0, 3453, 0, 0, 12, 0, 0, 4094, 4095, 2, 1, 65, 0, 0, 0, 0, 1317, 0, 0, 4095, 0, 1336, 0, 7, 7, 0, 0, 0, 1, 3199, 0, 0, 0, 0, 0, 2067, 821, 4093, 0, 0, 1, 1, 1, 4095, 0, 0, 0, 4096, 2, 0, 0, 3108, 0, 0, 4088, 0, 1425, 0, 4096, 3895, 4096, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 4095, 4095, 4095, 0, 4091, 1, 0, 4094, 0, 1, 106, 0, 0, 4096, 4096, 0, 1, 1, 1, 0, 0, 0, 0, 5, 3, 0, 4092, 0, 6, 0, 0, 0, 0, 4094, 4095, 3856, 2, 4068, 0, 4085, 0, 4093, 0, 1116] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 5, 4091, 722, 4090, 1, 0, 0, 391, 4094, 0, 157, 0, 0, 3342, 0, 0, 18, 0, 0, 4095, 4090, 0, 1, 48, 0, 13, 2, 0, 166, 3, 0, 4094, 1, 1666, 0, 29, 0, 2, 1, 0, 0, 2343, 0, 4, 0, 0, 0, 4071, 2796, 4090, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 4085, 0, 0, 1, 3204, 0, 0, 4093, 0, 1702, 0, 4095, 3819, 4090, 0, 0, 1, 2, 13, 0, 5, 3, 10, 20, 4094, 4096, 4085, 0, 3712, 0, 0, 4094, 0, 2, 365, 7, 0, 4094, 4096, 0, 13, 7, 0, 0, 2, 0, 3, 24, 6, 0, 4081, 0, 10, 0, 0, 3, 0, 4039, 4095, 4049, 0, 4095, 2, 4057, 0, 4026, 2, 248] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1277, 1199, 4094, 1, 0, 0, 1295, 4096, 0, 433, 0, 0, 3453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4094, 4095, 2, 1, 65, 0, 0, 0, 0, 1317, 0, 0, 4093, 2912, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1336, 0, 7, 7, 0, 0, 0, 1, 3199, 0, 0, 0, 0, 0, 4095, 3650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2067, 821, 4093, 0, 0, 1, 1, 1, 3664, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3108, 0, 0, 4088, 0, 1425, 0, 4096, 3895, 770, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 4095, 13, 4095, 0, 4091, 1, 0, 4094, 0, 1, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 1, 1, 0, 0, 0, 0, 5, 3, 0, 4092, 0, 2814, 431, 414, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4094, 0, 3856, 2, 4068, 0, 3723, 0, 0, 0, 1116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15848, 17023, 16390, 14746, 17384, 16500, 16316, 16865] avg=16384.0000 max=17384 max_rank=4 min=14746 min_rank=3 imbalance_ratio=1.061035 max_load_reduction=113688 max_load_reduction_pct=0.867371 moved_to_echo=67065 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1278, 1277, 4096, 0, 0, 10, 489, 4095, 2, 1059, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4084, 0, 2, 285, 0, 3, 2, 0, 57, 0, 0, 4091, 2910, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1897, 0, 0, 0, 0, 0, 7, 0, 698, 89, 0, 0, 0, 26, 4092, 3649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3549, 2589, 4079, 0, 0, 0, 1, 0, 3663, 0, 0, 0, 4088, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 0, 0, 4093, 3, 26, 0, 4091, 4083, 770, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 14, 4095, 0, 3973, 0, 0, 4076, 1, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 1, 0, 0, 0, 5, 0, 1, 23, 0, 4065, 0, 2817, 431, 414, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 7, 0, 0, 0, 4049, 0, 3955, 0, 4095, 25, 3733, 1, 0, 0, 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16351, 15960, 14554, 17972, 17032, 16334, 16314, 16555] avg=16384.0000 max=17972 max_rank=3 min=14554 min_rank=2 imbalance_ratio=1.096924 max_load_reduction=113100 max_load_reduction_pct=0.862885 moved_to_echo=66235 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1278, 2827, 4096, 0, 0, 1, 1231, 4095, 24, 1132, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4076, 0, 0, 166, 0, 0, 0, 0, 993, 0, 0, 4096, 2909, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1405, 0, 0, 0, 2, 0, 12, 0, 1351, 7, 0, 0, 0, 187, 4093, 3651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3695, 1445, 4096, 0, 0, 0, 5, 0, 3664, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1575, 0, 0, 4020, 0, 842, 0, 4096, 4071, 770, 91, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 17, 4096, 13, 4096, 0, 4037, 0, 0, 4077, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 1, 1, 0, 0, 0, 0, 33, 0, 0, 23, 0, 4078, 0, 2818, 431, 414, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 0, 3452, 0, 3961, 0, 4069, 0, 3733, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18757, 16768, 14804, 17008, 15467, 16625, 16353, 15290] avg=16384.0000 max=18757 max_rank=0 min=14804 min_rank=2 imbalance_ratio=1.144836 max_load_reduction=112315 max_load_reduction_pct=0.856895 moved_to_echo=63735 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 32, 1275, 67, 4071, 2, 0, 4, 230, 4043, 19, 1380, 9, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 39, 1, 4085, 4037, 2, 3, 8, 9, 0, 0, 4, 517, 3, 0, 4054, 2880, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 3159, 0, 58, 0, 15, 0, 0, 125, 3374, 705, 45, 0, 2, 7, 4089, 3650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1829, 1689, 4013, 3, 0, 0, 62, 4, 3656, 2, 0, 0, 4078, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1747, 0, 0, 3920, 0, 2393, 0, 4083, 3541, 762, 0, 0, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 66, 21, 4091, 13, 4042, 76, 3447, 6, 0, 4076, 9, 12, 150, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4014, 0, 57, 6, 0, 0, 0, 33, 0, 106, 5, 2, 4042, 5, 2811, 430, 410, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 11, 4, 7, 4089, 0, 4082, 0, 3720, 2, 3702, 2, 0, 10, 605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15146, 16085, 19278, 15342, 16458, 16126, 16354, 16283] avg=16384.0000 max=19278 max_rank=2 min=15146 min_rank=0 imbalance_ratio=1.176636 max_load_reduction=111794 max_load_reduction_pct=0.852921 moved_to_echo=65221 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1273, 242, 4088, 0, 1, 17, 893, 4042, 0, 2056, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 4082, 0, 7, 110, 0, 7, 27, 0, 106, 0, 0, 4094, 2864, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 2071, 0, 0, 0, 4, 0, 0, 3, 3363, 56, 0, 0, 0, 0, 4091, 3651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3349, 2455, 4096, 0, 0, 0, 100, 0, 3665, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2196, 9, 0, 4034, 0, 596, 0, 3990, 4086, 758, 6, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 4072, 0, 3961, 0, 0, 4017, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 212, 0, 0, 0, 0, 16, 0, 63, 86, 0, 3937, 1, 2805, 431, 407, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284, 0, 0, 0, 0, 3897, 0, 3151, 7, 4022, 1, 3701, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16642, 15825, 17337, 17760, 15695, 16160, 16507, 15146] avg=16384.0000 max=17760 max_rank=3 min=15146 min_rank=7 imbalance_ratio=1.083984 max_load_reduction=113312 max_load_reduction_pct=0.864502 moved_to_echo=63508 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 1274, 1855, 4093, 1, 0, 0, 371, 4095, 2, 562, 0, 20, 4072, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 6, 0, 4092, 4085, 2, 2, 1652, 1, 0, 0, 3, 264, 4, 1, 4094, 2883, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1180, 0, 19, 12, 0, 2, 0, 6, 2857, 0, 0, 0, 0, 416, 4092, 3651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2857, 1457, 4091, 1, 2, 0, 76, 2, 3658, 3, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2873, 10, 0, 4017, 0, 790, 0, 4034, 3335, 753, 2, 7, 2, 3, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 9, 4054, 0, 3712, 2, 2, 4072, 0, 0, 89, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 1, 0, 0, 2, 0, 1, 0, 0, 60, 0, 4091, 0, 2821, 435, 414, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 3, 3966, 0, 4057, 2, 4091, 1, 3683, 0, 0, 0, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16355, 17536, 16326, 16232, 15829, 16036, 16374, 16384] avg=16384.0000 max=17536 max_rank=1 min=15829 min_rank=4 imbalance_ratio=1.070312 max_load_reduction=113536 max_load_reduction_pct=0.866211 moved_to_echo=64623 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1278, 746, 4093, 0, 0, 4, 3379, 4096, 0, 217, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4091, 0, 0, 189, 0, 0, 0, 0, 261, 0, 0, 4096, 2911, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1824, 0, 0, 5, 0, 0, 0, 12, 2806, 1, 0, 0, 0, 6, 4096, 3651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 916, 477, 4094, 0, 0, 0, 57, 0, 3665, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1688, 0, 0, 4094, 1, 738, 0, 4086, 3917, 770, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 4096, 13, 4096, 0, 4042, 0, 0, 4016, 0, 1, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 87, 0, 0, 0, 0, 1, 0, 0, 108, 0, 4096, 0, 2818, 431, 414, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4054, 0, 3754, 1, 4092, 0, 3733, 0, 0, 0, 2767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17894, 16074, 16497, 13304, 15509, 16881, 16510, 18403] avg=16384.0000 max=18403 max_rank=7 min=13304 min_rank=3 imbalance_ratio=1.123230 max_load_reduction=112669 max_load_reduction_pct=0.859596 moved_to_echo=67303 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 1277, 722, 4090, 1, 0, 0, 391, 4094, 0, 157, 0, 0, 3342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4095, 4090, 0, 1, 48, 0, 13, 2, 0, 166, 3, 0, 4026, 2907, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1666, 0, 29, 0, 2, 1, 0, 0, 2343, 0, 4, 0, 0, 0, 4095, 3651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 2796, 4090, 1, 0, 0, 0, 0, 3664, 0, 0, 0, 4085, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3204, 0, 0, 4093, 0, 1702, 0, 4095, 3819, 769, 0, 0, 1, 2, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 10, 20, 4094, 13, 4085, 0, 3712, 0, 0, 4094, 0, 2, 365, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 13, 7, 0, 0, 2, 0, 3, 24, 6, 0, 4081, 0, 2814, 431, 414, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 3, 0, 4039, 0, 4049, 0, 4095, 2, 3698, 0, 0, 2, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14079, 15801, 15886, 18708, 17698, 16410, 16344, 16146] avg=16384.0000 max=18708 max_rank=3 min=14079 min_rank=0 imbalance_ratio=1.141846 max_load_reduction=112364 max_load_reduction_pct=0.857269 moved_to_echo=66598 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[0, 128, 128, 128, 32768, 32640, 128, 128, 2560, 128, 128, 128, 128, 3712, 128, 128, 32768, 23296, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 14592, 0, 128, 128, 128, 128, 128, 256, 20096, 896, 128, 0, 128, 768, 32768, 29312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 32768, 128, 384, 128, 128, 128, 128, 128, 128, 256, 384, 128, 32512, 128, 22528, 3456, 3328, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[22400, 13824, 32768, 128, 128, 128, 384, 128, 29312, 128, 128, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[20352, 128, 0, 32384, 128, 8576, 0, 32640, 30848, 6144, 384, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[640, 128, 128, 128, 128, 31744, 0, 30976, 128, 32256, 128, 29824, 128, 0, 128, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 256, 128, 128, 32768, 128, 32640, 128, 30976, 128, 128, 32640, 128, 128, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 10240, 8960, 32768, 128, 128, 128, 8320, 32768, 128, 7040, 128, 128, 31232, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4087, 0, 0, 4094, 25, 0, 0, 0, 4092, 0, 1, 4094, 4012, 1, 0, 87, 549, 4093, 1, 4022, 324, 3918, 0, 0, 0, 4094, 3, 2, 3963, 389, 0, 4096, 0, 4096, 9, 12, 2, 0, 0, 4094, 0, 4, 0, 0, 4094, 0, 0, 1, 0, 2645, 2, 0, 0, 0, 0, 2, 4094, 1, 0, 0, 4092, 0, 4094, 0, 0, 0, 0, 0, 0, 3, 0, 17, 4096, 0, 4090, 0, 1, 4094, 0, 2, 0, 2, 0, 4096, 0, 0, 1, 0, 9, 0, 2, 3, 4093, 0, 4095, 3379, 0, 1005, 4092, 2, 0, 0, 4077, 0, 527, 2, 4095, 3593, 0, 4096, 0, 39, 4067, 0, 205, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[4033, 0, 11, 4081, 59, 14, 9, 0, 4059, 0, 1, 4085, 4055, 0, 0, 406, 3275, 4084, 7, 3975, 15, 2122, 0, 14, 0, 4063, 0, 14, 4093, 2142, 0, 4079, 0, 4030, 12, 0, 1, 19, 15, 4083, 9, 11, 0, 3, 4084, 3, 1, 0, 33, 342, 2, 33, 0, 2, 11, 33, 4071, 26, 15, 13, 4070, 8, 4071, 0, 0, 0, 20, 11, 3, 14, 0, 1, 4070, 20, 4081, 16, 0, 4079, 0, 12, 0, 0, 2, 4064, 9, 9, 0, 6, 891, 9, 2, 0, 4092, 20, 4082, 3632, 8, 552, 4085, 17, 0, 0, 4073, 0, 410, 0, 4077, 2669, 0, 4088, 3, 46, 4055, 7, 1, 4078, 8, 1, 14, 10, 23, 4, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4093, 0, 0, 4094, 0, 1, 2, 2, 4088, 0, 7, 4089, 4094, 0, 0, 220, 1117, 4075, 0, 3862, 0, 4053, 0, 1, 1, 4091, 1, 0, 3963, 1317, 1, 4088, 1, 4092, 16, 21, 0, 0, 0, 4094, 0, 1, 1, 2, 4092, 2, 1, 0, 0, 2165, 0, 0, 3, 0, 0, 0, 4094, 0, 7, 2, 4094, 2, 4092, 3, 3, 0, 0, 2, 2, 1, 5, 1, 4093, 1, 4070, 0, 0, 4090, 0, 1, 0, 3, 0, 4093, 4, 1, 0, 1, 196, 14, 4, 0, 4094, 0, 4084, 3708, 0, 1556, 4094, 2, 0, 2, 3833, 0, 868, 4, 4088, 1578, 0, 4094, 2, 8, 4086, 2, 182, 4090, 0, 2, 0, 2, 0, 1, 5, 0, 41, 0, 2, 14] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 93, 0, 0, 0, 4096, 0, 0, 4093, 4049, 0, 0, 625, 1251, 4095, 0, 4063, 2, 3926, 0, 0, 0, 4096, 0, 0, 3450, 936, 0, 4091, 0, 4096, 1, 152, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 1565, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 30, 4096, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 3, 4096, 3, 4096, 0, 0, 4096, 0, 2, 0, 2, 0, 4096, 0, 0, 0, 0, 46, 28, 0, 0, 4096, 0, 4096, 3464, 0, 1361, 4061, 0, 0, 0, 4025, 0, 125, 0, 4096, 3385, 0, 4096, 0, 2, 4092, 0, 209, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4072, 0, 0, 4096, 86, 0, 0, 0, 4091, 0, 0, 4096, 4095, 0, 0, 2308, 1316, 4095, 0, 3982, 1060, 3279, 0, 0, 0, 4095, 0, 0, 2556, 859, 3, 4068, 9, 4095, 0, 0, 1, 0, 0, 4091, 0, 1, 3, 0, 4096, 0, 0, 0, 0, 3960, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 4091, 0, 4096, 0, 0, 5, 1, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 1, 4096, 0, 3, 0, 0, 0, 4096, 0, 0, 1, 0, 127, 0, 2, 1, 4096, 0, 4092, 279, 20, 911, 3981, 0, 1, 0, 4053, 3, 2, 0, 4095, 3453, 0, 4094, 0, 24, 4088, 0, 553, 4096, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4080, 0, 0, 4096, 5, 0, 0, 0, 4095, 0, 0, 4092, 4092, 0, 0, 33, 2488, 4095, 0, 4095, 0, 3893, 1, 0, 0, 4095, 0, 0, 3411, 1958, 0, 3849, 0, 4096, 1, 0, 0, 1, 0, 4095, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 2566, 0, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 4096, 0, 4095, 0, 0, 3, 0, 0, 0, 0, 0, 1, 4095, 15, 4094, 0, 0, 4096, 0, 3, 0, 1, 0, 4096, 0, 0, 1, 0, 0, 41, 0, 0, 4084, 0, 4093, 3296, 0, 19, 4095, 0, 0, 0, 4096, 0, 91, 0, 4096, 2826, 0, 4096, 0, 13, 4090, 0, 105, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 1, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4095, 0, 0, 4096, 13, 0, 0, 0, 4096, 0, 0, 4096, 4057, 0, 0, 16, 736, 4095, 0, 3674, 2, 3099, 0, 0, 0, 4096, 0, 0, 4091, 1048, 0, 4092, 0, 4096, 49, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2706, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 7, 4094, 2, 0, 4096, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 175, 2, 0, 0, 4093, 0, 4096, 3112, 0, 1613, 4095, 0, 0, 0, 4096, 0, 41, 0, 4095, 3584, 0, 4096, 0, 30, 4072, 0, 644, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4077, 0, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 5, 1751, 4096, 0, 3031, 474, 1122, 0, 0, 0, 4096, 0, 0, 4046, 1872, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3235, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 4096, 0, 0, 4096, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 48, 10, 0, 0, 4096, 0, 4095, 3263, 0, 428, 4070, 0, 0, 0, 4096, 0, 293, 0, 4096, 3275, 0, 4096, 0, 16, 4096, 0, 1742, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4087, 0, 0, 0, 25, 0, 0, 0, 4092, 0, 1, 3644, 4012, 1, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 1, 4022, 324, 3918, 0, 0, 0, 0, 3, 2, 3963, 389, 0, 2583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 12, 2, 0, 0, 4094, 0, 4, 0, 0, 4094, 0, 0, 1, 3607, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2645, 2, 0, 0, 0, 0, 2, 4094, 1, 0, 0, 4092, 0, 4094, 0, 1205, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 17, 4096, 0, 4090, 0, 1, 4094, 0, 2, 3633, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 1, 0, 9, 0, 2, 3, 4093, 0, 4095, 3379, 450, 308, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1005, 4092, 2, 0, 0, 4077, 0, 527, 2, 3971, 3593, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 205, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4, 4096, 3641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15949, 15754, 16379, 16622, 16389, 16562, 17308, 16109] avg=16384.0000 max=17308 max_rank=6 min=15754 min_rank=1 imbalance_ratio=1.056396 max_load_reduction=113764 max_load_reduction_pct=0.867950 moved_to_echo=66368 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4033, 0, 11, 0, 59, 14, 9, 0, 4059, 0, 1, 3636, 4055, 0, 0, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3275, 0, 7, 3975, 15, 2122, 0, 14, 0, 0, 0, 14, 4093, 2142, 0, 2572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 12, 0, 1, 19, 15, 4083, 9, 11, 0, 3, 4084, 3, 1, 0, 3580, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 342, 2, 33, 0, 2, 11, 33, 4071, 26, 15, 13, 4070, 8, 4071, 0, 1200, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 11, 3, 14, 0, 1, 4070, 20, 4081, 16, 0, 4079, 0, 12, 3625, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4064, 9, 9, 0, 6, 891, 9, 2, 0, 4092, 20, 4082, 3632, 449, 307, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 552, 4085, 17, 0, 0, 4073, 0, 410, 0, 3953, 2669, 0, 0, 3, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 7, 1, 4078, 8, 1, 14, 10, 23, 4, 0, 0, 0, 1, 0, 0, 4088, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16283, 18229, 16310, 14413, 16404, 17698, 15816, 15919] avg=16384.0000 max=18229 max_rank=1 min=14413 min_rank=3 imbalance_ratio=1.112610 max_load_reduction=112843 max_load_reduction_pct=0.860924 moved_to_echo=65837 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 93, 0, 0, 0, 4096, 0, 0, 3644, 4049, 0, 0, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1251, 0, 0, 4063, 2, 3926, 0, 0, 0, 0, 0, 0, 3450, 936, 0, 2579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 152, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 3609, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1565, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 30, 4096, 0, 4096, 0, 1204, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 3, 4096, 3, 4096, 0, 0, 4096, 0, 2, 3636, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 46, 28, 0, 0, 4096, 0, 4096, 3464, 449, 308, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1361, 4061, 0, 0, 0, 4025, 0, 125, 0, 3972, 3385, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 209, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 9, 4096, 3642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16603, 16207, 16510, 15574, 16393, 16709, 16931, 16145] avg=16384.0000 max=16931 max_rank=6 min=15574 min_rank=3 imbalance_ratio=1.033386 max_load_reduction=114141 max_load_reduction_pct=0.870827 moved_to_echo=66178 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4072, 0, 0, 0, 86, 0, 0, 0, 4091, 0, 0, 3646, 4095, 0, 0, 2308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1316, 0, 0, 3982, 1060, 3279, 0, 0, 0, 0, 0, 0, 2556, 859, 3, 2565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4095, 0, 0, 1, 0, 0, 4091, 0, 1, 3, 0, 4096, 0, 0, 0, 3609, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 4091, 0, 4096, 0, 1197, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 1, 4096, 0, 3, 3635, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 127, 0, 2, 1, 4096, 0, 4092, 279, 450, 306, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 911, 3981, 0, 1, 0, 4053, 3, 2, 0, 3971, 3453, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 553, 4096, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 3642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18298, 15620, 16365, 17933, 16383, 13574, 16419, 16480] avg=16384.0000 max=18298 max_rank=0 min=13574 min_rank=5 imbalance_ratio=1.116821 max_load_reduction=112774 max_load_reduction_pct=0.860397 moved_to_echo=62856 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 1, 2, 2, 4088, 0, 7, 3637, 4094, 0, 0, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1117, 0, 0, 3862, 0, 4053, 0, 1, 1, 0, 1, 0, 3963, 1317, 1, 2569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 16, 21, 0, 0, 0, 4094, 0, 1, 1, 2, 4092, 2, 1, 0, 3605, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2165, 0, 0, 3, 0, 0, 0, 4094, 0, 7, 2, 4094, 2, 4092, 3, 1208, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 2, 1, 5, 1, 4093, 1, 4070, 0, 0, 4090, 0, 1, 3617, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 4, 1, 0, 1, 196, 14, 4, 0, 4094, 0, 4084, 3708, 452, 311, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1556, 4094, 2, 0, 2, 3833, 0, 868, 4, 3959, 1578, 0, 0, 2, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 2, 182, 4090, 0, 2, 0, 2, 0, 1, 5, 0, 41, 0, 2, 14, 4094, 3640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16144, 16885, 16386, 16156, 16340, 17094, 15906, 16161] avg=16384.0000 max=17094 max_rank=5 min=15906 min_rank=6 imbalance_ratio=1.043335 max_load_reduction=113978 max_load_reduction_pct=0.869583 moved_to_echo=65501 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4080, 0, 0, 0, 5, 0, 0, 0, 4095, 0, 0, 3643, 4092, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2488, 0, 0, 4095, 0, 3893, 1, 0, 0, 0, 0, 0, 3411, 1958, 0, 2426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 1, 0, 4095, 0, 1, 0, 0, 4096, 0, 0, 0, 3608, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2566, 0, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 4096, 0, 4095, 0, 1133, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 4095, 15, 4094, 0, 0, 4096, 0, 3, 3635, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 1, 0, 0, 41, 0, 0, 4084, 0, 4093, 3296, 449, 290, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4095, 0, 0, 0, 4096, 0, 91, 0, 3972, 2826, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 105, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 1, 2, 4096, 3642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15948, 18272, 16358, 16475, 16396, 16475, 15112, 16036] avg=16384.0000 max=18272 max_rank=1 min=15112 min_rank=6 imbalance_ratio=1.115234 max_load_reduction=112800 max_load_reduction_pct=0.860596 moved_to_echo=64019 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 3646, 4057, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 736, 0, 0, 3674, 2, 3099, 0, 0, 0, 0, 0, 0, 4091, 1048, 0, 2580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 49, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3610, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2706, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 1204, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 7, 4094, 2, 0, 4096, 0, 0, 3635, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 175, 2, 0, 0, 4093, 0, 4096, 3112, 450, 308, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1613, 4095, 0, 0, 0, 4096, 0, 41, 0, 3971, 3584, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 644, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15923, 15230, 16407, 16685, 16386, 16462, 17430, 16549] avg=16384.0000 max=17430 max_rank=6 min=15230 min_rank=1 imbalance_ratio=1.063843 max_load_reduction=113642 max_load_reduction_pct=0.867020 moved_to_echo=66827 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4077, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 3646, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1751, 0, 0, 3031, 474, 1122, 0, 0, 0, 0, 0, 0, 4046, 1872, 0, 2579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3609, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3235, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 1204, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 4096, 0, 0, 4096, 0, 0, 3636, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 48, 10, 0, 0, 4096, 0, 4095, 3263, 450, 308, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 4070, 0, 0, 0, 4096, 0, 293, 0, 3972, 3275, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1742, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 3642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15924, 14875, 16357, 17214, 16381, 16498, 16150, 17673] avg=16384.0000 max=17673 max_rank=7 min=14875 min_rank=1 imbalance_ratio=1.078674 max_load_reduction=113399 max_load_reduction_pct=0.865166 moved_to_echo=66702 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[32768, 128, 3712, 32768, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 29184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 32768, 128, 128, 32768, 0, 128, 29056, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 7552, 32640, 128, 128, 128, 32384, 128, 2432, 128, 31744, 24448, 0, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[12544, 0, 128, 30720, 1920, 25472, 128, 128, 128, 0, 128, 128, 29696, 10624, 128, 20480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32640, 0, 128, 0, 384, 128, 128, 128, 32768, 0, 128, 29184, 32640, 128, 0, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 19200, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 32768, 128, 32768, 128, 9600, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 32768, 128, 256, 128, 128, 128, 32768, 128, 128, 128, 128, 32768, 128, 128, 128, 28928, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 128, 1536, 128, 128, 128, 32768, 128, 32768, 24192, 3712, 2560, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 1408, 0, 4096, 418, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 4094, 1, 0, 4086, 73, 0, 1, 0, 0, 3, 4096, 0, 1174, 0, 4095, 0, 4096, 0, 6, 4083, 1, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 1, 4096, 1, 11, 0, 284, 4096, 0, 0, 0, 0, 2728, 1, 0, 3348, 0, 0, 3975, 4096, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 3234, 0, 2, 0, 239, 4096, 0, 4057, 0, 3855, 3916, 4096, 0, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 80, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4061, 0, 0, 0, 4096, 4008] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4082, 27, 19, 20, 3073, 5, 4074, 223, 0, 0, 0, 0, 61, 1, 21, 12, 0, 27, 8, 5, 3, 15, 0, 0, 9, 4068, 0, 4075, 4087, 9, 16, 4035, 0, 19, 0, 0, 12, 10, 4080, 0, 1938, 0, 4017, 38, 4057, 0, 50, 4082, 2, 0, 39, 18, 0, 0, 7, 4080, 27, 5, 17, 0, 4082, 0, 82, 0, 230, 4075, 26, 0, 0, 3, 391, 20, 8, 3727, 5, 10, 4092, 4051, 0, 7, 16, 4082, 4086, 1, 0, 31, 0, 4075, 2, 31, 4, 69, 3224, 2, 4056, 1, 4043, 4054, 4052, 1, 4084, 0, 7, 4072, 0, 4050, 0, 0, 5, 20, 12, 0, 4063, 14, 4, 6, 0, 0, 0, 8, 47, 15, 3359, 3, 0, 31, 4084, 4076] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1626, 0, 4096, 147, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 4096, 0, 0, 3990, 0, 0, 0, 0, 0, 0, 4096, 2, 2448, 0, 4090, 0, 4093, 0, 16, 4096, 0, 0, 0, 0, 3, 0, 0, 4096, 6, 0, 0, 61, 4096, 20, 0, 0, 572, 4096, 0, 0, 0, 0, 393, 0, 0, 3746, 0, 0, 3968, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4061, 0, 0, 0, 215, 4095, 0, 4070, 0, 3827, 4087, 4096, 0, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3622, 0, 0, 2, 4096, 4085] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4089, 0, 1, 3, 1697, 3, 4088, 236, 6, 0, 18, 0, 0, 1, 0, 3, 0, 0, 0, 0, 3, 6, 0, 1, 0, 4093, 2, 4093, 4094, 0, 0, 4033, 22, 4, 0, 0, 5, 2, 4093, 0, 3525, 2, 4056, 0, 4094, 1, 5, 4094, 0, 0, 0, 1, 1, 0, 5, 4089, 0, 3, 4, 115, 4092, 0, 0, 0, 333, 4087, 0, 0, 0, 2, 104, 2, 2, 3568, 2, 2, 4094, 4091, 0, 1, 0, 4091, 4092, 1, 0, 0, 6, 3874, 1, 0, 0, 153, 4069, 4, 4083, 0, 3897, 3723, 4094, 0, 4092, 0, 5, 4093, 0, 4093, 0, 0, 3, 3, 0, 43, 4091, 3, 1, 2, 0, 0, 1, 4, 0, 6, 3629, 2, 0, 0, 4092, 3850] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 957, 0, 4096, 2562, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4096, 0, 568, 0, 4082, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 19, 4096, 0, 0, 0, 840, 4082, 1, 1, 0, 0, 160, 0, 0, 3991, 0, 0, 4095, 4095, 0, 0, 0, 4096, 4096, 0, 2, 1, 0, 4095, 0, 0, 0, 2, 4096, 0, 4091, 0, 4094, 4095, 4095, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3217, 1, 2, 0, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 693, 0, 4093, 109, 0, 0, 3, 3, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 4094, 1, 4092, 4093, 0, 1, 4091, 0, 0, 0, 0, 0, 3, 4092, 0, 1109, 0, 3812, 2, 4093, 1, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 5, 4094, 0, 0, 0, 2964, 4086, 0, 0, 2, 0, 52, 1, 0, 3929, 2, 2, 4092, 4096, 0, 0, 1, 4096, 4095, 0, 5, 0, 1, 4020, 2, 0, 1, 114, 4093, 0, 4096, 0, 4051, 4039, 4094, 0, 4096, 0, 1, 4095, 0, 4096, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 15, 2, 0, 0, 9, 0, 3763, 0, 0, 0, 4094, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 242, 0, 4096, 472, 0, 0, 47, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 6, 668, 0, 4091, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 51, 4096, 25, 0, 0, 2127, 4095, 0, 0, 0, 0, 1087, 0, 0, 4087, 0, 0, 4029, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4042, 0, 0, 0, 531, 4096, 0, 4096, 0, 3899, 3994, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3314, 0, 0, 2, 4096, 4074] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1205, 0, 4096, 1197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 444, 0, 4096, 1, 4096, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 4096, 129, 0, 0, 11, 4096, 0, 1, 0, 393, 4064, 0, 0, 0, 0, 3285, 0, 0, 3443, 0, 0, 4094, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3599, 0, 0, 0, 3, 4095, 0, 4096, 0, 3557, 3782, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3624, 0, 0, 0, 4096, 4076] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4082, 27, 19, 20, 3073, 5, 4074, 223, 0, 0, 0, 0, 61, 1, 21, 12, 3661, 1565, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 8, 5, 3, 15, 0, 0, 9, 4068, 0, 4075, 4087, 9, 16, 4035, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 12, 10, 2638, 0, 1938, 0, 4017, 38, 4057, 0, 50, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 39, 18, 0, 0, 7, 4080, 27, 5, 17, 0, 4082, 0, 82, 0, 4084, 3914, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 4075, 26, 0, 0, 3, 391, 20, 8, 3727, 5, 10, 4092, 2486, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4082, 172, 1, 0, 31, 0, 4075, 2, 31, 4, 69, 3224, 2, 4056, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 4054, 4052, 1, 0, 0, 7, 317, 0, 4050, 0, 0, 5, 20, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 14, 4, 6, 0, 0, 0, 8, 47, 15, 3359, 3, 0, 31, 4084, 4076, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17704, 16384, 16861, 16451, 15080, 15766, 16561, 16265] avg=16384.0000 max=17704 max_rank=0 min=15080 min_rank=4 imbalance_ratio=1.080566 max_load_reduction=113368 max_load_reduction_pct=0.864929 moved_to_echo=63672 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 1408, 0, 4096, 418, 0, 0, 3, 0, 0, 0, 1, 0, 3683, 1583, 864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 4094, 1, 0, 4086, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 1, 0, 0, 3, 2648, 0, 1174, 0, 4095, 0, 4096, 0, 6, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 1, 4096, 1, 11, 0, 4096, 3924, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284, 4096, 0, 0, 0, 0, 2728, 1, 0, 3348, 0, 0, 3975, 2513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 172, 0, 0, 2, 0, 3234, 0, 2, 0, 239, 4096, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3855, 3916, 4096, 0, 0, 0, 0, 319, 0, 4095, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4061, 0, 0, 0, 4096, 4008, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16150, 16395, 16179, 16331, 16945, 15898, 16361, 16813] avg=16384.0000 max=16945 max_rank=4 min=15898 min_rank=5 imbalance_ratio=1.034241 max_load_reduction=114127 max_load_reduction_pct=0.870720 moved_to_echo=66017 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1626, 0, 4096, 147, 0, 0, 6, 0, 0, 0, 0, 0, 3683, 1583, 864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 4096, 0, 0, 3990, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2648, 2, 2448, 0, 4090, 0, 4093, 0, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 6, 0, 0, 61, 4096, 20, 0, 0, 4096, 3924, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 572, 4096, 0, 0, 0, 0, 393, 0, 0, 3746, 0, 0, 3968, 2513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 172, 0, 0, 0, 0, 4061, 0, 0, 0, 215, 4095, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3827, 4087, 4096, 0, 0, 0, 0, 319, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3622, 0, 0, 2, 4096, 4085, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16101, 16303, 17393, 16396, 15288, 16709, 16424, 16458] avg=16384.0000 max=17393 max_rank=2 min=15288 min_rank=4 imbalance_ratio=1.061584 max_load_reduction=113679 max_load_reduction_pct=0.867302 moved_to_echo=64879 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 693, 0, 4093, 109, 0, 0, 3, 3, 0, 0, 1, 0, 3682, 1583, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 4094, 1, 4092, 4093, 0, 1, 4091, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2646, 0, 1109, 0, 3812, 2, 4093, 1, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 5, 4094, 0, 0, 0, 4096, 3923, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2964, 4086, 0, 0, 2, 0, 52, 1, 0, 3929, 2, 2, 4092, 2513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 172, 0, 5, 0, 1, 4020, 2, 0, 1, 114, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4039, 4094, 0, 0, 0, 1, 319, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 15, 2, 0, 0, 9, 0, 3763, 0, 0, 0, 4094, 4096, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15123, 16401, 15762, 16306, 17643, 16601, 16603, 16633] avg=16384.0000 max=17643 max_rank=4 min=15123 min_rank=0 imbalance_ratio=1.076843 max_load_reduction=113429 max_load_reduction_pct=0.865395 moved_to_echo=67480 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 957, 0, 4096, 2562, 0, 0, 0, 0, 7, 0, 0, 0, 3683, 1582, 864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 4086, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2648, 0, 568, 0, 4082, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 19, 4096, 0, 0, 0, 4096, 3924, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 840, 4082, 1, 1, 0, 0, 160, 0, 0, 3991, 0, 0, 4095, 2513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 172, 0, 2, 1, 0, 4095, 0, 0, 0, 2, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 4095, 0, 0, 0, 0, 319, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3217, 1, 2, 0, 4096, 4096, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17845, 16410, 15489, 16325, 15683, 16555, 16699, 16066] avg=16384.0000 max=17845 max_rank=0 min=15489 min_rank=2 imbalance_ratio=1.089172 max_load_reduction=113227 max_load_reduction_pct=0.863853 moved_to_echo=65003 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4089, 0, 1, 3, 1697, 3, 4088, 236, 6, 0, 18, 0, 0, 1, 0, 3, 3683, 1585, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6, 0, 1, 0, 4093, 2, 4093, 4094, 0, 0, 4033, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4, 0, 0, 5, 2, 2634, 0, 3525, 2, 4056, 0, 4094, 1, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 5, 4089, 0, 3, 4, 115, 4092, 0, 0, 0, 4092, 3925, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 4087, 0, 0, 0, 2, 104, 2, 2, 3568, 2, 2, 4094, 2506, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 167, 1, 0, 0, 6, 3874, 1, 0, 0, 153, 4069, 4, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3897, 3723, 4094, 0, 0, 0, 5, 309, 0, 4093, 0, 0, 3, 3, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 1, 2, 0, 0, 1, 4, 0, 6, 3629, 2, 0, 0, 4092, 3850, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16278, 16358, 18444, 16428, 14703, 16449, 16170, 16242] avg=16384.0000 max=18444 max_rank=2 min=14703 min_rank=4 imbalance_ratio=1.125732 max_load_reduction=112628 max_load_reduction_pct=0.859283 moved_to_echo=63564 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 242, 0, 4096, 472, 0, 0, 47, 0, 65, 0, 0, 0, 3683, 1583, 864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2648, 6, 668, 0, 4091, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 51, 4096, 25, 0, 0, 4096, 3924, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2127, 4095, 0, 0, 0, 0, 1087, 0, 0, 4087, 0, 0, 4029, 2513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 172, 0, 0, 0, 0, 4042, 0, 0, 0, 531, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3899, 3994, 4096, 0, 0, 0, 0, 319, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3314, 0, 0, 2, 4096, 4074, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15147, 16411, 15605, 16388, 17938, 17033, 16404, 16146] avg=16384.0000 max=17938 max_rank=4 min=15147 min_rank=0 imbalance_ratio=1.094849 max_load_reduction=113134 max_load_reduction_pct=0.863144 moved_to_echo=67521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1205, 0, 4096, 1197, 0, 0, 0, 0, 0, 0, 0, 0, 3683, 1583, 864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4095, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2648, 0, 444, 0, 4096, 1, 4096, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 129, 0, 0, 11, 4096, 0, 1, 0, 4096, 3924, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 4064, 0, 0, 0, 0, 3285, 0, 0, 3443, 0, 0, 4094, 2513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 172, 0, 0, 0, 0, 3599, 0, 0, 0, 3, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3557, 3782, 4096, 0, 0, 0, 0, 319, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3624, 0, 0, 0, 4096, 4076, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16724, 16410, 15339, 16447, 17792, 16061, 15850, 16449] avg=16384.0000 max=17792 max_rank=4 min=15339 min_rank=2 imbalance_ratio=1.085938 max_load_reduction=113280 max_load_reduction_pct=0.864258 moved_to_echo=66152 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 128, 128, 11008, 128, 32768, 5376, 128, 0, 128, 128, 256, 128, 128, 128, 29568, 12672, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[7808, 32768, 128, 128, 128, 128, 8320, 128, 128, 29952, 128, 128, 32512, 20096, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[31232, 31744, 32768, 128, 0, 0, 128, 2560, 0, 32768, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 128, 32768, 128, 32768, 32768, 128, 128, 32512, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 128, 0, 128, 32768, 256, 128, 128, 384, 32768, 128, 128, 0, 32768, 31488, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 1408, 128, 128, 128, 128, 31104, 128, 128, 128, 1408, 31872, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 128, 128, 128, 28672, 128, 128, 128, 32768, 32384, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 128, 128, 21248, 128, 11904, 128, 32384, 128, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4085, 2, 4094, 4093, 0, 8, 0, 4088, 0, 0, 1, 0, 1, 3, 4, 4093, 1, 4, 0, 4091, 0, 4092, 3140, 0, 4089, 8, 0, 1, 4092, 9, 4092, 5, 0, 111, 4, 0, 0, 0, 4093, 2, 4091, 0, 4093, 0, 6, 0, 0, 25, 5, 3, 0, 606, 0, 2525, 4087, 4075, 4091, 3, 0, 0, 53, 0, 1, 0, 0, 4091, 20, 2, 0, 4092, 0, 0, 4072, 2, 0, 3, 3, 4089, 4090, 0, 6, 2, 4085, 3, 0, 0, 5, 4027, 4090, 0, 4, 0, 3, 0, 0, 0, 1826, 3, 5, 4093, 0, 2, 0, 4088, 7, 3, 6, 0, 3, 0, 0, 6, 4085, 0, 3, 5, 0, 0, 4091, 0, 1, 4089, 4, 4089, 0, 1, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4085, 0, 4092, 3908, 0, 0, 0, 4085, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4093, 2, 4093, 2487, 0, 4095, 0, 0, 2, 4093, 2, 4093, 1, 0, 2, 1, 4, 8, 0, 4093, 0, 4094, 3, 4095, 5, 0, 0, 3, 0, 0, 2, 0, 150, 0, 3128, 4091, 4091, 4095, 0, 3, 0, 0, 0, 0, 0, 0, 4092, 9, 3, 9, 4094, 0, 0, 3866, 0, 0, 0, 2, 4089, 4092, 0, 3, 0, 4043, 0, 0, 43, 0, 4088, 3930, 2, 0, 0, 0, 0, 2, 0, 2687, 0, 2, 4094, 0, 0, 0, 4094, 15, 2, 0, 5, 0, 1, 0, 0, 4096, 324, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 4095, 1, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4054, 2, 4077, 3454, 7, 39, 0, 4067, 0, 17, 0, 0, 0, 18, 0, 4075, 0, 19, 0, 4075, 0, 4078, 662, 22, 4084, 0, 2, 2, 4068, 0, 4076, 28, 0, 69, 0, 1, 4, 0, 4052, 32, 4078, 8, 4078, 0, 67, 30, 0, 0, 18, 12, 0, 3176, 13, 3734, 4067, 4025, 4057, 20, 0, 17, 13, 0, 0, 0, 0, 4076, 0, 31, 0, 4000, 0, 2, 3773, 0, 19, 13, 3, 4085, 4080, 0, 0, 9, 4058, 7, 0, 1, 14, 4037, 4071, 0, 18, 0, 11, 0, 0, 42, 718, 16, 23, 4073, 22, 20, 15, 4072, 0, 18, 13, 0, 14, 0, 0, 29, 4066, 906, 5, 9, 34, 0, 4072, 0, 9, 3976, 20, 4076, 0, 0, 19, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4011, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 3681, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 771, 4096, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 3585, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 29, 0, 4088, 3887, 0, 0, 0, 0, 0, 18, 0, 4055, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 31, 0, 4096, 297, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4090, 0, 4095, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 3147, 0, 4096, 4, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 255, 0, 3155, 4096, 4095, 4096, 0, 1, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4039, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 6, 0, 4095, 4095, 0, 0, 0, 0, 1, 3, 0, 1457, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 226, 0, 0, 1, 0, 4096, 0, 0, 4095, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 3540, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 2397, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 284, 0, 2525, 4096, 4096, 4096, 0, 143, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 5, 0, 4093, 4087, 0, 0, 0, 0, 0, 0, 0, 2677, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 22, 0, 4096, 703, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4090, 0, 4095, 4060, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 4096, 2056, 0, 4096, 0, 0, 0, 4096, 2, 4096, 0, 0, 125, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 927, 0, 934, 4095, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4094, 3, 41, 3884, 0, 0, 0, 0, 4093, 4093, 4, 0, 0, 4093, 0, 0, 0, 0, 4093, 3802, 0, 0, 0, 0, 0, 4, 0, 3825, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 3, 0, 0, 17, 0, 4096, 845, 0, 0, 0, 0, 4096, 0, 0, 4062, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 2638, 0, 4096, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 359, 0, 3429, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 0, 931, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 11, 0, 4096, 838, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4085, 0, 4092, 3908, 0, 0, 0, 4085, 0, 1, 0, 0, 0, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 1568, 2487, 0, 0, 0, 0, 2, 4093, 2, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4, 8, 0, 4093, 0, 4094, 3, 4095, 5, 0, 0, 3, 0, 2205, 1840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 150, 0, 3128, 4091, 4091, 4095, 0, 3, 0, 0, 0, 0, 0, 499, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 9, 3, 9, 4094, 0, 0, 3866, 0, 0, 0, 2, 169, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4043, 0, 0, 43, 0, 4088, 3930, 2, 0, 0, 0, 0, 2, 0, 2128, 2057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2687, 0, 2, 4094, 0, 0, 0, 4094, 15, 2, 0, 5, 0, 1, 0, 0, 4095, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 324, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 3596, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16368, 16341, 16353, 16379, 16336, 16296, 16787, 16212] avg=16384.0000 max=16787 max_rank=6 min=16212 min_rank=7 imbalance_ratio=1.024597 max_load_reduction=114285 max_load_reduction_pct=0.871925 moved_to_echo=65631 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4054, 2, 4077, 3454, 7, 39, 0, 4067, 0, 17, 0, 0, 0, 18, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4075, 0, 1562, 662, 22, 0, 0, 2, 2, 4068, 0, 4076, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 1, 4, 0, 4052, 32, 4078, 8, 4078, 0, 67, 30, 0, 0, 2197, 1831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 12, 0, 3176, 13, 3734, 4067, 4025, 4057, 20, 0, 17, 13, 0, 0, 0, 497, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 31, 0, 4000, 0, 2, 3773, 0, 19, 13, 3, 169, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4058, 7, 0, 1, 14, 4037, 4071, 0, 18, 0, 11, 0, 0, 42, 2126, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 718, 16, 23, 4073, 22, 20, 15, 4072, 0, 18, 13, 0, 14, 0, 0, 29, 4084, 1790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 906, 5, 9, 34, 0, 4072, 0, 9, 3976, 20, 3579, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15932, 14516, 16447, 19968, 16166, 16441, 14907, 16695] avg=16384.0000 max=19968 max_rank=3 min=14516 min_rank=1 imbalance_ratio=1.218750 max_load_reduction=111104 max_load_reduction_pct=0.847656 moved_to_echo=64209 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4011, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1570, 3681, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4086, 0, 0, 0, 0, 0, 2206, 1841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 771, 4096, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 499, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 3585, 0, 0, 0, 0, 170, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 29, 0, 4088, 3887, 0, 0, 0, 0, 0, 18, 0, 2131, 2058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 31, 0, 4096, 1795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 297, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 3597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16496, 17539, 16325, 14006, 16044, 16307, 18174, 16181] avg=16384.0000 max=18174 max_rank=6 min=14006 min_rank=3 imbalance_ratio=1.109253 max_load_reduction=112898 max_load_reduction_pct=0.861343 moved_to_echo=66706 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 4095, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1570, 3147, 0, 0, 4, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 4095, 0, 2, 0, 0, 0, 2206, 1841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 3155, 4096, 4095, 4096, 0, 1, 0, 0, 1, 0, 0, 499, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4039, 0, 0, 0, 0, 170, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4095, 4095, 0, 0, 0, 0, 1, 3, 0, 2131, 2058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1457, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 1795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 226, 0, 0, 1, 0, 4096, 0, 0, 4095, 0, 3597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16574, 17009, 16338, 16518, 16495, 16485, 15542, 16111] avg=16384.0000 max=17009 max_rank=1 min=15542 min_rank=6 imbalance_ratio=1.038147 max_load_reduction=114063 max_load_reduction_pct=0.870232 moved_to_echo=64633 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 4095, 4060, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 1, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1570, 2056, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4, 0, 0, 0, 2206, 1841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 927, 0, 934, 4095, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 499, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4094, 3, 41, 3884, 0, 0, 0, 0, 170, 4093, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4093, 3802, 0, 0, 0, 0, 0, 4, 0, 2130, 2058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 3, 0, 0, 17, 0, 4096, 1793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 845, 0, 0, 0, 0, 4096, 0, 0, 4062, 0, 3597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16535, 15916, 16464, 14966, 16387, 16180, 17928, 16696] avg=16384.0000 max=17928 max_rank=6 min=14966 min_rank=3 imbalance_ratio=1.094238 max_load_reduction=113144 max_load_reduction_pct=0.863220 moved_to_echo=67191 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4085, 2, 4094, 4093, 0, 8, 0, 4088, 0, 0, 1, 0, 1, 3, 4, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4091, 0, 1556, 3140, 0, 0, 8, 0, 1, 4092, 9, 4092, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 4, 0, 0, 0, 4093, 2, 4091, 0, 4093, 0, 6, 0, 0, 25, 2209, 1841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 0, 606, 0, 2525, 4087, 4075, 4091, 3, 0, 0, 53, 0, 1, 0, 505, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 20, 2, 0, 4092, 0, 0, 4072, 2, 0, 3, 3, 162, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 4085, 3, 0, 0, 5, 4027, 4090, 0, 4, 0, 3, 0, 0, 0, 2132, 2061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1826, 3, 5, 4093, 0, 2, 0, 4088, 7, 3, 6, 0, 3, 0, 0, 6, 4089, 1795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 3, 5, 0, 0, 4091, 0, 1, 4089, 4, 3584, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16570, 16999, 16475, 16281, 16537, 16418, 15926, 15866] avg=16384.0000 max=16999 max_rank=1 min=15866 min_rank=7 imbalance_ratio=1.037537 max_load_reduction=114073 max_load_reduction_pct=0.870308 moved_to_echo=64747 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 3540, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1570, 2397, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2206, 1841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284, 0, 2525, 4096, 4096, 4096, 0, 143, 0, 0, 0, 0, 0, 499, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 170, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 4093, 4087, 0, 0, 0, 0, 0, 0, 0, 2131, 2058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2677, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 22, 0, 4096, 1795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 703, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16024, 16255, 16335, 16059, 16559, 16470, 16782, 16588] avg=16384.0000 max=16782 max_rank=6 min=16024 min_rank=0 imbalance_ratio=1.024292 max_load_reduction=114290 max_load_reduction_pct=0.871964 moved_to_echo=66399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1570, 2638, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2206, 1841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 359, 0, 3429, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 499, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 170, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 0, 2131, 2058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 931, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 11, 0, 4096, 1795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 838, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16573, 16497, 16335, 16895, 16548, 16475, 15026, 16723] avg=16384.0000 max=16895 max_rank=3 min=15026 min_rank=6 imbalance_ratio=1.031189 max_load_reduction=114177 max_load_reduction_pct=0.871101 moved_to_echo=64772 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 32768, 31360, 128, 128, 0, 32768, 0, 128, 128, 0, 128, 128, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[18176, 128, 128, 32768, 128, 128, 128, 32768, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 14464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 32768, 128, 0, 128, 128, 32640, 32128, 128, 128, 0, 128, 128, 128, 128, 17152, 16512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 128, 128, 128, 32768, 128, 128, 31488, 128, 128, 128, 128, 1408, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 4224, 128, 128, 128, 0, 32768, 0, 128, 32640, 128, 28800, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 32768, 128, 12544, 20224, 128, 0, 128, 128, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 5888, 128, 20224, 32768, 32768, 32768, 128, 256, 128, 128, 128, 128, 0, 4096, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 384, 128, 128, 128, 0, 32768, 128, 32768, 128, 32768, 128, 128, 128, 128, 128, 17664, 14720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 4095, 0, 347, 2838, 4096, 0, 4096, 4096, 4094, 0, 4096, 4050, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 3613, 0, 74, 0, 0, 22, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3242, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 803, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1369, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4083, 4093, 5, 5, 2, 7, 4087, 0, 4089, 0, 0, 4091, 3, 30, 3776, 4092, 0, 4089, 4091, 4089, 0, 4086, 4093, 0, 2, 2, 2, 5, 2, 4087, 0, 8, 8, 0, 6, 0, 4077, 4078, 4, 0, 11, 0, 3, 0, 0, 4089, 10, 8, 0, 0, 12, 4080, 0, 0, 0, 0, 5, 4, 3972, 0, 2, 1012, 4, 65, 3, 10, 73, 3, 2, 0, 2, 8, 4, 4088, 0, 2, 0, 6, 0, 0, 837, 89, 0, 0, 4072, 54, 0, 4090, 0, 0, 6, 1, 4084, 2, 6, 9, 5, 4091, 4091, 3, 4091, 0, 0, 0, 47, 5, 0, 4024, 3408, 5, 4091, 0, 6, 2, 0, 0, 1, 4081, 3, 0, 5, 0, 3111, 4084, 0, 4093, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4093, 4094, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 4096, 0, 111, 3865, 4094, 1, 4096, 4096, 4095, 0, 4096, 4095, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 49, 4096, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 1782, 0, 1, 0, 0, 28, 1, 185, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2239, 3, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 4096, 0, 4096, 2, 0, 30, 0, 0, 0, 4096, 489, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 3544, 4096, 0, 4090, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4068, 4082, 22, 25, 2, 31, 4075, 0, 4072, 0, 0, 4036, 19, 0, 1633, 4058, 0, 4084, 4049, 4057, 1, 4071, 4084, 0, 14, 0, 1, 20, 0, 4046, 7, 26, 30, 0, 0, 1, 4002, 4071, 19, 30, 0, 0, 88, 0, 56, 4075, 0, 27, 0, 0, 0, 3988, 0, 20, 632, 0, 18, 25, 4076, 14, 4, 3831, 17, 211, 11, 59, 168, 0, 14, 0, 4, 13, 23, 4066, 0, 12, 0, 20, 21, 16, 2591, 47, 19, 13, 3976, 36, 0, 4066, 0, 0, 23, 7, 4043, 0, 0, 27, 20, 4072, 4070, 6, 4063, 0, 0, 0, 60, 16, 0, 4001, 34, 0, 4081, 0, 40, 0, 0, 6, 16, 4052, 13, 0, 21, 0, 3248, 4062, 0, 4080, 18] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4095, 0, 0, 0, 0, 4095, 0, 4096, 0, 2, 4094, 0, 1, 1913, 4091, 1, 4096, 4096, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 104, 4096, 0, 0, 9, 0, 0, 0, 3972, 0, 0, 3320, 0, 0, 0, 0, 106, 0, 10, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3287, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 4096, 1, 0, 0, 1, 0, 0, 4096, 782, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2889, 4096, 0, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4091, 4091, 2, 0, 0, 0, 4093, 0, 4096, 0, 0, 4085, 0, 3, 3464, 4086, 23, 4094, 4094, 4096, 0, 4094, 4094, 0, 2, 0, 0, 2, 1, 4093, 0, 0, 1, 0, 0, 0, 4094, 4079, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 1, 0, 0, 7, 4095, 4, 0, 68, 11, 2, 0, 4054, 2, 1, 1410, 0, 0, 0, 0, 549, 0, 47, 0, 1, 0, 4, 4096, 0, 0, 0, 2, 0, 0, 3954, 0, 0, 0, 4088, 6, 0, 4095, 0, 0, 1, 0, 4094, 2, 0, 0, 1, 4094, 4093, 0, 4093, 0, 1, 0, 6, 0, 0, 4060, 206, 0, 4093, 1, 3, 3, 0, 0, 0, 4096, 0, 0, 0, 3, 2691, 4094, 0, 4072, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 4042, 0, 54, 1285, 4096, 0, 4096, 4096, 4093, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 0, 0, 0, 4059, 0, 0, 3464, 0, 109, 0, 0, 1, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2712, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4013, 754, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4065, 4096, 0, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 11, 4071, 4096, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3400, 0, 0, 1478, 0, 2, 0, 0, 731, 0, 482, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3247, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 8, 0, 0, 4096, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2951, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 0, 4093, 0, 1185, 0, 0, 4096, 0, 111, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 4096, 18, 0, 0, 4095, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2824, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 4096, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 1782, 0, 4096, 1253, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 28, 1, 185, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2239, 3, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 1325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3303, 4096, 0, 4096, 2, 0, 30, 0, 0, 0, 4096, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 3544, 4096, 0, 4090, 0, 793, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17443, 16402, 16376, 15728, 16223, 15848, 16112, 16940] avg=16384.0000 max=17443 max_rank=0 min=15728 min_rank=3 imbalance_ratio=1.064636 max_load_reduction=113629 max_load_reduction_pct=0.866920 moved_to_echo=65123 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 0, 2, 0, 0, 0, 4093, 0, 1185, 0, 0, 4085, 0, 3, 3464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 23, 0, 4094, 17, 0, 0, 4094, 0, 2, 0, 0, 2, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 4079, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 2825, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 4095, 4, 0, 68, 11, 2, 0, 4054, 2, 1, 1410, 0, 4094, 1254, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 47, 0, 1, 0, 4, 4096, 0, 0, 0, 2, 0, 4094, 4093, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 0, 0, 4088, 6, 0, 4095, 0, 0, 1, 0, 4094, 2, 0, 1325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3302, 4093, 0, 4093, 0, 1, 0, 6, 0, 0, 4060, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 3, 0, 0, 0, 4096, 0, 0, 0, 3, 2691, 4094, 0, 4072, 0, 792, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16923, 16412, 16353, 15376, 16604, 17565, 15762, 16077] avg=16384.0000 max=17565 max_rank=5 min=15376 min_rank=3 imbalance_ratio=1.072083 max_load_reduction=113507 max_load_reduction_pct=0.865990 moved_to_echo=66008 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4068, 0, 22, 25, 2, 31, 4075, 0, 1178, 0, 0, 4036, 19, 0, 1633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 4049, 17, 1, 0, 4084, 0, 14, 0, 1, 20, 0, 4046, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 30, 0, 0, 1, 4002, 4071, 19, 30, 0, 0, 88, 0, 56, 4075, 0, 2798, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 3988, 0, 20, 632, 0, 18, 25, 4076, 14, 4, 3831, 17, 4071, 1242, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 11, 59, 168, 0, 14, 0, 4, 13, 23, 4066, 0, 12, 0, 20, 21, 4084, 4081, 3710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2591, 47, 19, 13, 3976, 36, 0, 4066, 0, 0, 23, 7, 4043, 0, 0, 1317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 20, 3284, 4070, 6, 4063, 0, 0, 0, 60, 16, 0, 4001, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 6, 16, 4052, 13, 0, 21, 0, 3248, 4062, 0, 4080, 18, 788, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15089, 16297, 16453, 18337, 16497, 16154, 15581, 16664] avg=16384.0000 max=18337 max_rank=3 min=15089 min_rank=0 imbalance_ratio=1.119202 max_load_reduction=112735 max_load_reduction_pct=0.860100 moved_to_echo=64896 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4095, 0, 1185, 0, 2, 4094, 0, 1, 1913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 4096, 17, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 2825, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 4096, 0, 0, 9, 0, 0, 0, 3972, 0, 0, 3320, 0, 4096, 1254, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 10, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3287, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 1325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3303, 4096, 0, 4096, 1, 0, 0, 1, 0, 0, 4096, 782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2889, 4096, 0, 4094, 0, 793, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15386, 16395, 16378, 17224, 16126, 16900, 16375, 16288] avg=16384.0000 max=17224 max_rank=3 min=15386 min_rank=0 imbalance_ratio=1.051270 max_load_reduction=113848 max_load_reduction_pct=0.868591 moved_to_echo=65689 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4095, 0, 1185, 0, 0, 4042, 0, 54, 1285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 17, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2823, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 0, 0, 0, 4059, 0, 0, 3464, 0, 4096, 1253, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 1, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2712, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3303, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4013, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4065, 4096, 0, 4093, 0, 793, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14757, 16405, 16375, 17360, 16123, 16325, 16262, 17465] avg=16384.0000 max=17465 max_rank=7 min=14757 min_rank=0 imbalance_ratio=1.065979 max_load_reduction=113607 max_load_reduction_pct=0.866753 moved_to_echo=66175 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1185, 0, 1, 4095, 0, 347, 2838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 18, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2823, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 3613, 0, 4096, 1253, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 22, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3242, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3303, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1369, 4096, 0, 4096, 0, 793, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16658, 16372, 16375, 17537, 16108, 16855, 16395, 14772] avg=16384.0000 max=17537 max_rank=3 min=14772 min_rank=7 imbalance_ratio=1.070374 max_load_reduction=113535 max_load_reduction_pct=0.866203 moved_to_echo=64130 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4083, 0, 5, 5, 2, 7, 4087, 0, 1172, 0, 0, 4091, 3, 30, 3776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4091, 12, 0, 0, 4093, 0, 2, 2, 2, 5, 2, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 0, 6, 0, 4077, 4078, 4, 0, 11, 0, 3, 0, 0, 4089, 10, 2823, 1268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 12, 4080, 0, 0, 0, 0, 5, 4, 3972, 0, 2, 1012, 4, 4086, 1254, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 3, 10, 73, 3, 2, 0, 2, 8, 4, 4088, 0, 2, 0, 6, 0, 4089, 4091, 3719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 837, 89, 0, 0, 4072, 54, 0, 4090, 0, 0, 6, 1, 4084, 2, 6, 1324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 5, 3295, 4091, 3, 4091, 0, 0, 0, 47, 5, 0, 4024, 3408, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 1, 4081, 3, 0, 5, 0, 3111, 4084, 0, 4093, 5, 796, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17261, 16388, 16385, 14813, 16165, 14565, 18983, 16512] avg=16384.0000 max=18983 max_rank=6 min=14565 min_rank=5 imbalance_ratio=1.158630 max_load_reduction=112089 max_load_reduction_pct=0.855171 moved_to_echo=66225 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1185, 0, 0, 4096, 0, 11, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 17, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2825, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3400, 0, 0, 1478, 0, 4096, 1254, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 731, 0, 482, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3247, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3303, 4096, 0, 4096, 0, 0, 0, 8, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2951, 4096, 0, 4096, 0, 793, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17555, 16401, 16377, 14697, 17226, 16860, 15602, 16354] avg=16384.0000 max=17555 max_rank=0 min=14697 min_rank=3 imbalance_ratio=1.071472 max_load_reduction=113517 max_load_reduction_pct=0.866066 moved_to_echo=66042 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 0, 128, 128, 128, 128, 32768, 0, 9472, 0, 128, 32640, 128, 640, 22912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 22144, 256, 128, 128, 32640, 128, 0, 32768, 0, 0, 128, 128, 32768, 128, 128, 10624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 0, 32768, 256, 128, 0, 32768, 0, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 128, 32768, 32768, 128, 128, 128, 0, 128, 0, 128, 32768, 128, 22656, 10112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[512, 128, 128, 1792, 128, 768, 0, 128, 128, 128, 32768, 0, 128, 0, 128, 128, 32768, 32768, 29824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 26496, 32768, 128, 32768, 128, 128, 128, 128, 128, 0, 32512, 6528, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 128, 0, 128, 128, 23936, 32768, 0, 32768, 128, 6400, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 0, 256, 32768, 128, 128, 768, 128, 128, 128, 31744, 128, 128, 19968, 128, 32768, 10112, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[8, 1, 4072, 7, 19, 0, 28, 4, 13, 0, 30, 4076, 4074, 4013, 4075, 0, 27, 10, 6, 0, 24, 1, 0, 27, 0, 23, 9, 35, 2, 37, 24, 4048, 0, 41, 4061, 17, 2, 22, 0, 0, 4049, 21, 25, 4074, 20, 0, 48, 43, 35, 0, 70, 0, 4050, 21, 4070, 0, 3922, 4049, 0, 6, 18, 4060, 0, 32, 0, 0, 22, 0, 4073, 0, 395, 0, 1143, 44, 4, 4093, 46, 14, 4067, 4038, 4059, 0, 0, 0, 0, 690, 4067, 0, 64, 0, 0, 1, 0, 4, 4070, 23, 0, 22, 0, 0, 30, 3, 0, 4018, 4073, 4063, 14, 0, 92, 0, 31, 4067, 4063, 0, 0, 0, 2, 4064, 4071, 4074, 0, 4065, 1, 25, 8, 2002, 0, 3918] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[2, 0, 4092, 0, 0, 1, 0, 1, 0, 0, 2, 4088, 4095, 4095, 4093, 0, 0, 1, 0, 0, 2, 0, 0, 2, 0, 1, 0, 0, 0, 0, 2, 4096, 2, 0, 4094, 0, 16, 0, 0, 3, 4075, 0, 0, 4095, 0, 0, 2, 0, 0, 2, 5, 0, 4094, 0, 4093, 2, 1896, 4094, 0, 0, 2, 4090, 0, 0, 2, 0, 0, 1, 4094, 5, 676, 84, 822, 0, 0, 4074, 0, 0, 4094, 4094, 4095, 0, 0, 0, 0, 1604, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 2, 2, 1, 0, 0, 0, 1, 4093, 4094, 4093, 0, 0, 0, 1, 0, 4086, 4094, 0, 7, 3, 2, 4094, 4094, 4096, 2, 4094, 0, 1, 3, 3135, 3, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2, 0, 0, 0, 4095, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3758, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 311, 3, 23, 0, 0, 4087, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1904, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 2206, 1, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 13, 0, 4094, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 109, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 4096, 0, 140, 62, 14, 0, 0, 4056, 0, 0, 4095, 4096, 4094, 6, 0, 4, 0, 3859, 4096, 1, 7, 0, 2, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 4092, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 4087, 3, 4079] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1206, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 4096, 0, 1380, 0, 2, 0, 0, 3991, 0, 0, 4096, 4092, 4096, 0, 0, 3, 0, 2736, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 4060, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 3008, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[2, 3, 4090, 11, 4, 2, 10, 0, 8, 0, 6, 4089, 4091, 4088, 4091, 0, 5, 0, 2, 0, 5, 2, 0, 2, 0, 5, 0, 5, 3, 6, 8, 4090, 0, 0, 4087, 4, 0, 4, 16, 0, 4086, 3, 9, 4089, 6, 0, 5, 7, 0, 0, 13, 0, 4089, 0, 4089, 0, 1265, 4085, 21, 2, 2, 4087, 0, 7, 0, 0, 5, 0, 4088, 0, 3509, 0, 2, 11, 1, 4094, 10, 5, 4089, 4081, 4084, 0, 0, 0, 0, 1593, 4086, 26, 0, 1, 0, 0, 0, 3, 4059, 2, 0, 7, 0, 0, 10, 4, 0, 3993, 4092, 4087, 4, 0, 4, 0, 0, 4094, 4090, 0, 3, 2, 1, 4091, 4091, 4091, 0, 4091, 1, 6, 1, 1942, 0, 4014] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 169, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 318, 0, 7, 0, 0, 4068, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 3670, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 4056, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 861, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 273, 12, 46, 0, 0, 4038, 0, 0, 4096, 4096, 4096, 0, 0, 12, 0, 3054, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 4056, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 4086, 0, 4003] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[8, 1, 4072, 7, 19, 0, 28, 4, 13, 0, 30, 4076, 4072, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 10, 6, 0, 24, 1, 0, 27, 0, 23, 9, 35, 2, 37, 24, 4048, 4074, 4075, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 4061, 17, 2, 22, 0, 0, 4049, 21, 25, 4074, 20, 0, 48, 43, 2993, 1040, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 70, 0, 4050, 21, 2428, 0, 3922, 4049, 0, 6, 18, 4060, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 2957, 0, 395, 0, 1143, 44, 4, 4093, 46, 14, 4067, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 690, 4067, 0, 64, 0, 0, 1, 0, 4, 4070, 23, 1098, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 30, 3, 0, 4018, 4073, 4063, 14, 0, 92, 0, 31, 4067, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 2, 4064, 0, 0, 0, 1072, 1, 25, 8, 2002, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16343, 16476, 16473, 18691, 16823, 14678, 16433, 15155] avg=16384.0000 max=18691 max_rank=3 min=14678 min_rank=5 imbalance_ratio=1.140808 max_load_reduction=112381 max_load_reduction_pct=0.857399 moved_to_echo=63089 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4092, 0, 0, 1, 0, 1, 0, 0, 2, 4088, 4093, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 2, 0, 1, 0, 0, 0, 0, 2, 4096, 4096, 4093, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 16, 0, 0, 3, 4075, 0, 0, 4095, 0, 0, 2, 0, 3015, 1046, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 4094, 0, 2442, 2, 1896, 4094, 0, 0, 2, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 2971, 5, 676, 84, 822, 0, 0, 4074, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1604, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 1104, 605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 1, 0, 0, 0, 1, 4093, 4094, 4093, 0, 0, 0, 1, 0, 4086, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 7, 3, 2, 4094, 0, 0, 2, 1079, 0, 1, 3, 3135, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16374, 16370, 16365, 16627, 16823, 15602, 16394, 16517] avg=16384.0000 max=16823 max_rank=4 min=15602 min_rank=5 imbalance_ratio=1.026794 max_load_reduction=114249 max_load_reduction_pct=0.871651 moved_to_echo=65336 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 13, 0, 4094, 0, 0, 4091, 0, 0, 0, 0, 3016, 1047, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2443, 0, 109, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2972, 0, 140, 62, 14, 0, 0, 4056, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 4, 0, 3859, 4096, 1, 7, 0, 2, 0, 0, 0, 4073, 0, 1105, 606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 4092, 0, 0, 0, 0, 0, 4088, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1080, 0, 0, 0, 4087, 3, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 16367, 16375, 14830, 15439, 17853, 16389, 17441] avg=16384.0000 max=17853 max_rank=5 min=14830 min_rank=3 imbalance_ratio=1.089661 max_load_reduction=113219 max_load_reduction_pct=0.863792 moved_to_echo=67122 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4095, 0, 0, 4093, 0, 0, 0, 0, 3016, 1047, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2443, 0, 3758, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2972, 0, 311, 3, 23, 0, 0, 4087, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1904, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 1105, 606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 4094, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1080, 0, 0, 0, 2206, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16367, 16366, 18489, 15588, 15904, 16402, 15574] avg=16384.0000 max=18489 max_rank=3 min=15574 min_rank=7 imbalance_ratio=1.128479 max_load_reduction=112583 max_load_reduction_pct=0.858940 moved_to_echo=63468 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 3016, 1047, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2443, 0, 1206, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2972, 0, 1380, 0, 2, 0, 0, 3991, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 2736, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1105, 606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 4060, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1080, 0, 0, 0, 3008, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16370, 16367, 15937, 16539, 16733, 16368, 16376] avg=16384.0000 max=16733 max_rank=5 min=15937 min_rank=3 imbalance_ratio=1.021301 max_load_reduction=114339 max_load_reduction_pct=0.872337 moved_to_echo=66016 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 3, 4090, 11, 4, 2, 10, 0, 8, 0, 6, 4089, 4088, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 0, 5, 2, 0, 2, 0, 5, 0, 5, 3, 6, 8, 4090, 4091, 4091, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4, 0, 4, 16, 0, 4086, 3, 9, 4089, 6, 0, 5, 7, 3018, 1047, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4089, 0, 2434, 0, 1265, 4085, 21, 2, 2, 4087, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2963, 0, 3509, 0, 2, 11, 1, 4094, 10, 5, 4089, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 1593, 4086, 26, 0, 1, 0, 0, 0, 3, 4059, 2, 1105, 608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 10, 4, 0, 3993, 4092, 4087, 4, 0, 4, 0, 0, 4094, 20, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3, 2, 1, 4091, 0, 0, 0, 1073, 1, 6, 1, 1942, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16401, 16389, 16398, 16005, 18770, 15567, 16318, 15224] avg=16384.0000 max=18770 max_rank=4 min=15224 min_rank=7 imbalance_ratio=1.145630 max_load_reduction=112302 max_load_reduction_pct=0.856796 moved_to_echo=65879 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 4096, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 0, 3016, 1047, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2443, 0, 861, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2972, 0, 273, 12, 46, 0, 0, 4038, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 0, 3054, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1105, 606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 4056, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1080, 0, 0, 0, 4086, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16430, 16367, 16359, 15593, 15533, 17065, 16364, 17361] avg=16384.0000 max=17361 max_rank=7 min=15533 min_rank=4 imbalance_ratio=1.059631 max_load_reduction=113711 max_load_reduction_pct=0.867546 moved_to_echo=66323 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3016, 1047, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2443, 0, 169, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2972, 0, 318, 0, 7, 0, 0, 4068, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 3670, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1105, 606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 4096, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1080, 0, 0, 0, 4056, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16366, 16369, 14900, 15557, 17670, 16404, 17424] avg=16384.0000 max=17670 max_rank=5 min=14900 min_rank=3 imbalance_ratio=1.078491 max_load_reduction=113402 max_load_reduction_pct=0.865189 moved_to_echo=67055 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 0, 128, 128, 128, 32768, 0, 0, 128, 8704, 128, 128, 128, 24576, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 32768, 32768, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 32640, 32768, 32768, 128, 0, 128, 128, 128, 32768, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 128, 128, 128, 0, 128, 32768, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 0, 128, 0, 19200, 32768, 128, 128, 128, 128, 128, 0, 128, 32768, 128, 8832, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 23808, 128, 7040, 256, 2176, 128, 128, 32512, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 0, 128, 128, 24192, 8448, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 19584, 128, 13312, 32768, 128, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4077, 0, 4096, 4084, 0, 4096, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4060, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 3719, 0, 3936, 4096, 4096, 4096, 4096, 0, 48, 405, 0, 85, 0, 0, 0, 0, 4095, 0, 4084, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 101, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 0, 4096, 4082, 0, 4096, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 3538, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4091, 0, 2799, 4096, 4096, 4096, 4096, 0, 334, 523, 9, 48, 0, 0, 0, 0, 4096, 0, 4070, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4093, 0, 0, 3, 0, 4096, 0, 0, 1033, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[6, 5, 2, 0, 0, 0, 0, 4061, 4, 3, 4089, 0, 4, 3, 0, 4092, 0, 0, 4087, 0, 8, 2, 7, 4089, 4, 3, 8, 2, 4084, 0, 4088, 3981, 0, 4091, 9, 0, 4090, 0, 11, 0, 5, 0, 0, 0, 4090, 11, 0, 4087, 7, 25, 0, 5, 7, 4055, 0, 0, 4076, 6, 0, 0, 4, 4, 0, 7, 3, 5, 3, 0, 3, 4093, 2, 0, 22, 0, 0, 0, 0, 4, 0, 4090, 0, 4090, 0, 4049, 3, 3803, 4092, 4089, 4090, 4090, 19, 27, 141, 0, 56, 0, 3, 7, 2, 4089, 0, 4092, 0, 4089, 2, 1, 4, 0, 4058, 5, 4084, 2, 8, 4087, 4078, 6, 8, 6, 4092, 0, 3, 0, 0, 4088, 0, 0, 261, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3, 4096, 4081, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 20, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4078, 0, 3828, 4096, 4096, 4096, 4096, 0, 107, 160, 3, 10, 0, 0, 0, 0, 4095, 0, 4084, 4, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4093, 0, 0, 0, 4096, 0, 0, 17, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 2, 2, 0, 4094, 1, 0, 4094, 0, 0, 1, 6, 4094, 1, 0, 4096, 2, 80, 0, 0, 4090, 2, 2, 0, 4, 4095, 0, 4095, 4094, 0, 4092, 0, 0, 4095, 15, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 4094, 0, 0, 0, 0, 1, 3726, 0, 0, 4095, 1, 0, 0, 2, 1, 4, 0, 0, 3, 1, 0, 2, 4072, 0, 0, 0, 1, 0, 0, 0, 0, 2, 4095, 5, 4056, 0, 4096, 0, 3555, 4091, 4094, 4095, 4094, 4, 1, 878, 0, 0, 1, 0, 0, 1, 4094, 0, 4094, 0, 4094, 0, 0, 2, 0, 4094, 0, 4094, 0, 0, 4094, 4090, 2, 0, 0, 4095, 0, 2, 0, 3, 4077, 2, 0, 11, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 4071, 0, 4096, 4090, 0, 4096, 0, 0, 4080, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 2741, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4094, 0, 3972, 0, 1634, 4096, 4096, 4096, 4096, 0, 77, 3758, 1, 3, 0, 0, 0, 0, 4096, 0, 4095, 3, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 4090, 1, 0, 0, 0, 4096, 0, 0, 152, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[22, 26, 0, 0, 7, 0, 0, 3964, 27, 26, 4071, 9, 33, 18, 0, 4076, 0, 0, 4063, 0, 230, 14, 0, 4066, 31, 0, 2, 32, 4064, 0, 4056, 4058, 0, 4066, 29, 31, 3943, 0, 17, 0, 36, 0, 46, 0, 4069, 27, 13, 4062, 30, 0, 0, 0, 33, 4053, 0, 49, 4019, 23, 0, 0, 30, 7, 0, 31, 36, 0, 15, 0, 31, 4081, 0, 1, 107, 0, 0, 16, 0, 21, 0, 4067, 0, 4011, 43, 4059, 0, 1023, 4070, 4061, 4063, 4068, 4, 80, 2845, 0, 0, 0, 26, 6, 3, 4073, 0, 4080, 0, 4066, 11, 0, 22, 0, 3996, 28, 4060, 8, 30, 4047, 4069, 38, 34, 25, 4068, 0, 21, 0, 0, 4033, 0, 0, 107, 10] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 3976, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4093, 0, 614, 4096, 4096, 4096, 4096, 0, 2, 3597, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 11, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 3795, 206, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3538, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 3811, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 2799, 0, 3984, 0, 0, 0, 334, 523, 9, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4070, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3887, 0, 0, 3, 0, 4096, 0, 0, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16368, 16360, 16111, 16369, 15884, 16379, 17211] avg=16384.0000 max=17211 max_rank=7 min=15884 min_rank=5 imbalance_ratio=1.050476 max_load_reduction=113861 max_load_reduction_pct=0.868690 moved_to_echo=65843 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4091, 3794, 206, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 3810, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4078, 0, 3828, 0, 3984, 0, 0, 0, 107, 160, 3, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4084, 4, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 3890, 0, 0, 17, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16382, 16395, 16690, 16365, 16266, 16397, 16192] avg=16384.0000 max=16690 max_rank=3 min=16192 min_rank=7 imbalance_ratio=1.018677 max_load_reduction=114382 max_load_reduction_pct=0.872665 moved_to_echo=65220 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 3794, 206, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4077, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 3810, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3719, 0, 3936, 0, 3984, 0, 0, 0, 48, 405, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4084, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3890, 0, 0, 0, 0, 4096, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16363, 16378, 16634, 16368, 16268, 16393, 16279] avg=16384.0000 max=16634 max_rank=3 min=16268 min_rank=5 imbalance_ratio=1.015259 max_load_reduction=114438 max_load_reduction_pct=0.873093 moved_to_echo=65308 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 2, 2, 0, 4094, 1, 0, 4094, 0, 0, 1, 6, 4094, 3793, 206, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 10, 2, 80, 0, 0, 4090, 2, 2, 0, 4, 4095, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4095, 15, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 4094, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3726, 0, 0, 4095, 1, 0, 0, 2, 1, 4, 0, 4091, 4094, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 2, 4072, 0, 0, 0, 1, 0, 0, 0, 0, 2, 4095, 4086, 3810, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4056, 0, 4096, 0, 3555, 0, 3982, 0, 0, 4, 1, 878, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 21, 0, 4094, 0, 4094, 0, 0, 2, 0, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4090, 2, 0, 0, 3889, 0, 2, 0, 3, 4077, 2, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16396, 16475, 16401, 16300, 16352, 16578, 16400, 16170] avg=16384.0000 max=16578 max_rank=5 min=16170 min_rank=7 imbalance_ratio=1.011841 max_load_reduction=114494 max_load_reduction_pct=0.873520 moved_to_echo=65500 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[22, 26, 0, 0, 7, 0, 0, 3964, 27, 26, 4071, 9, 33, 18, 0, 4076, 3774, 204, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 230, 14, 0, 4066, 31, 0, 2, 32, 4064, 0, 4056, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 29, 31, 3943, 0, 17, 0, 36, 0, 46, 0, 4069, 27, 13, 4062, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 33, 4053, 0, 49, 4019, 23, 0, 0, 30, 7, 0, 31, 4070, 4068, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 15, 0, 31, 4081, 0, 1, 107, 0, 0, 16, 0, 21, 0, 4067, 4053, 3780, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 43, 4059, 0, 1023, 0, 3950, 0, 0, 4, 80, 2845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 6, 3, 21, 0, 4080, 0, 4066, 11, 0, 22, 0, 3996, 28, 4060, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4047, 4069, 38, 34, 25, 3864, 0, 21, 0, 0, 4033, 0, 0, 107, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16357, 16563, 16350, 16696, 16486, 16015, 16327, 16278] avg=16384.0000 max=16696 max_rank=3 min=16015 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=114376 max_load_reduction_pct=0.872620 moved_to_echo=65106 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[6, 5, 2, 0, 0, 0, 0, 4061, 4, 3, 4089, 0, 4, 3, 0, 4092, 3792, 209, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8, 2, 7, 4089, 4, 3, 8, 2, 4084, 0, 4088, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 9, 0, 4090, 0, 11, 0, 5, 0, 0, 0, 4090, 11, 0, 4087, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 25, 0, 5, 7, 4055, 0, 0, 4076, 6, 0, 0, 4, 4, 0, 7, 4092, 4090, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 3, 0, 3, 4093, 2, 0, 22, 0, 0, 0, 0, 4, 0, 4090, 4082, 3805, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4049, 3, 3803, 0, 3965, 0, 0, 19, 27, 141, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 2, 15, 0, 4092, 0, 4089, 2, 1, 4, 0, 4058, 5, 4084, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4087, 4078, 6, 8, 6, 3883, 0, 3, 0, 0, 4088, 0, 0, 261, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16376, 16281, 16412, 16663, 16394, 16153, 16364, 16429] avg=16384.0000 max=16663 max_rank=3 min=16153 min_rank=5 imbalance_ratio=1.017029 max_load_reduction=114409 max_load_reduction_pct=0.872871 moved_to_echo=65340 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 3795, 206, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 4071, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2741, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4086, 3811, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3972, 0, 1634, 0, 3984, 0, 0, 0, 77, 3758, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4095, 3, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 3884, 1, 0, 0, 0, 4096, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16365, 16381, 15314, 16369, 17523, 16407, 16324] avg=16384.0000 max=17523 max_rank=5 min=15314 min_rank=3 imbalance_ratio=1.069519 max_load_reduction=113549 max_load_reduction_pct=0.866310 moved_to_echo=66623 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 3795, 206, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 3811, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4093, 0, 614, 0, 3984, 0, 0, 0, 2, 3597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3890, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16275, 16395, 16664, 16369, 16385, 16405, 16189] avg=16384.0000 max=16664 max_rank=3 min=16189 min_rank=7 imbalance_ratio=1.017090 max_load_reduction=114408 max_load_reduction_pct=0.872864 moved_to_echo=65348 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 32640, 128, 128, 32768, 128, 128, 128, 128, 32768, 30336, 1664, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 256, 0, 32768, 128, 32768, 128, 128, 128, 0, 32640, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 31104, 128, 128, 128, 128, 32768, 128, 0, 1792, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 128, 384, 128, 128, 32768, 128, 128, 128, 128, 32768, 128, 32768, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32640, 128, 32256, 128, 21248, 0, 31872, 0, 0, 128, 768, 12416, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32768, 128, 128, 32640, 128, 128, 0, 128, 0, 128, 0, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 128, 30464, 0, 128, 32768, 128, 0, 0, 128, 128, 128, 128, 32768, 32768, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 0, 128, 32768, 128, 128, 256, 128, 0, 128, 0, 128, 128, 32768, 32768, 30464, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[436, 4092, 1, 4096, 4, 0, 2, 4094, 0, 4095, 0, 2, 2, 0, 4093, 0, 4096, 0, 0, 0, 0, 1, 4094, 4095, 2, 4090, 4091, 0, 0, 0, 4093, 4094, 2, 4094, 0, 4076, 0, 0, 1, 100, 0, 4096, 177, 0, 0, 0, 4065, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 4, 3, 0, 5, 4094, 4096, 2, 3, 4096, 0, 0, 0, 0, 0, 1, 0, 4093, 4084, 0, 0, 10, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3907, 0, 0, 4094, 2, 2, 0, 2, 1676, 4094, 0, 4096, 1, 0, 1922, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 2, 3, 24, 0, 0, 0, 4094, 4090, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[239, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4092, 6, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 3, 4096, 4096, 0, 4095, 4093, 0, 0, 0, 4096, 4096, 0, 4096, 0, 3958, 4, 0, 0, 193, 0, 4096, 1, 1, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 163, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 1241, 4096, 0, 4096, 0, 0, 2405, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[156, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 2, 4096, 4096, 0, 4096, 4095, 0, 0, 0, 4094, 4096, 0, 4096, 0, 4090, 0, 0, 0, 1, 0, 4096, 58, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3958, 0, 0, 4096, 0, 0, 0, 0, 36, 4096, 0, 4096, 0, 0, 3988, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[64, 4063, 7, 4075, 0, 0, 0, 4087, 0, 4089, 0, 0, 0, 30, 4090, 4, 4091, 0, 2, 4, 5, 0, 4079, 4090, 3, 4079, 4088, 0, 20, 4, 4054, 4087, 3, 4090, 10, 3902, 0, 8, 7, 209, 6, 4082, 4, 0, 0, 0, 4069, 9, 0, 7, 13, 4086, 0, 5, 0, 0, 12, 0, 0, 2, 4091, 4085, 0, 0, 4092, 6, 3, 1, 8, 6, 9, 0, 4087, 4089, 6, 6, 6, 0, 5, 0, 0, 0, 4, 4088, 0, 7, 0, 5, 3661, 7, 6, 4087, 4, 8, 0, 2, 830, 4089, 9, 4080, 5, 0, 3642, 0, 5, 0, 4063, 3, 0, 0, 0, 5, 0, 0, 4090, 0, 0, 0, 0, 16, 5, 26, 0, 4090, 4085, 9, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[765, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4081, 4, 0, 0, 19, 0, 4096, 81, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 81, 4096, 0, 4096, 0, 0, 3153, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[17, 4096, 0, 4082, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4080, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 35, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3976, 0, 0, 4096, 0, 0, 0, 0, 1484, 4096, 0, 4096, 0, 0, 2706, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[2073, 3935, 44, 3986, 0, 0, 27, 4048, 0, 4041, 0, 0, 0, 21, 4073, 25, 4073, 0, 0, 0, 15, 0, 4041, 4065, 25, 4037, 4061, 19, 33, 0, 3996, 4059, 0, 4065, 45, 3441, 0, 30, 32, 411, 27, 4032, 2, 0, 0, 0, 3988, 35, 0, 37, 62, 4059, 0, 32, 0, 0, 72, 3, 0, 0, 4069, 4037, 0, 0, 4056, 23, 36, 0, 0, 20, 35, 0, 4041, 4063, 13, 27, 70, 0, 8, 0, 5, 1, 22, 4045, 0, 12, 60, 8, 2791, 29, 124, 4060, 16, 31, 5, 26, 2767, 4065, 44, 4050, 31, 0, 738, 48, 23, 0, 4021, 18, 0, 0, 0, 17, 0, 0, 4067, 132, 0, 15, 0, 55, 27, 0, 0, 4064, 4061, 23, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1255, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 1, 0, 4096, 4096, 0, 4096, 0, 3694, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 14, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 164, 4096, 0, 4096, 0, 0, 3065, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[239, 4096, 0, 4096, 0, 0, 0, 3507, 0, 4092, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 4096, 0, 0, 4095, 4093, 0, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3958, 4, 0, 0, 193, 0, 4096, 1, 1, 0, 0, 4093, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 3854, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 163, 0, 0, 3858, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 4096, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1241, 4096, 0, 4096, 0, 0, 2405, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3863, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16036, 16413, 16472, 16374, 16500, 16623, 16293, 16361] avg=16384.0000 max=16623 max_rank=5 min=16036 min_rank=0 imbalance_ratio=1.014587 max_load_reduction=114449 max_load_reduction_pct=0.873177 moved_to_echo=65777 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[436, 4092, 1, 4096, 4, 0, 2, 3505, 0, 4095, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 2, 4090, 4091, 0, 0, 0, 4093, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 4076, 0, 0, 1, 100, 0, 4096, 177, 0, 0, 0, 4065, 1, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4, 3, 0, 5, 4094, 4096, 2, 3, 3851, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4093, 4084, 0, 0, 10, 0, 2, 0, 3856, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3907, 0, 0, 4094, 2, 2, 0, 2, 4096, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1676, 4094, 0, 4096, 1, 0, 1922, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 2, 3, 24, 0, 0, 0, 4094, 4090, 0, 0, 0, 3862, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16235, 16399, 16642, 16385, 16342, 16439, 16246, 16384] avg=16384.0000 max=16642 max_rank=2 min=16235 min_rank=0 imbalance_ratio=1.015747 max_load_reduction=114430 max_load_reduction_pct=0.873032 moved_to_echo=65411 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[156, 4096, 0, 4096, 0, 0, 0, 3507, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 4096, 0, 0, 4096, 4095, 0, 0, 0, 4094, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 1, 0, 4096, 58, 0, 0, 0, 4096, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3853, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3858, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3958, 0, 0, 4096, 0, 0, 0, 0, 4096, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 4096, 0, 4096, 0, 0, 3988, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3863, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15952, 16412, 16467, 16374, 16346, 16489, 16671, 16361] avg=16384.0000 max=16671 max_rank=6 min=15952 min_rank=0 imbalance_ratio=1.017517 max_load_reduction=114401 max_load_reduction_pct=0.872810 moved_to_echo=65867 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2073, 3935, 44, 3986, 0, 0, 27, 3466, 0, 4041, 0, 0, 0, 21, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4041, 0, 25, 4037, 4061, 19, 33, 0, 3996, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 45, 3441, 0, 30, 32, 411, 27, 4032, 2, 0, 0, 0, 3988, 35, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 62, 4059, 0, 32, 0, 0, 72, 3, 0, 0, 4069, 4037, 0, 0, 3831, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 23, 36, 0, 0, 20, 35, 0, 4041, 4063, 13, 27, 70, 0, 8, 0, 3823, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 22, 4045, 0, 12, 60, 8, 2791, 29, 124, 4060, 16, 31, 5, 26, 4073, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2767, 4065, 44, 4050, 31, 0, 738, 48, 23, 0, 4021, 18, 0, 0, 0, 17, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 132, 0, 15, 0, 55, 27, 0, 0, 4064, 4061, 23, 3, 0, 3834, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17618, 16255, 16138, 16433, 16412, 15550, 16177, 16489] avg=16384.0000 max=17618 max_rank=0 min=15550 min_rank=5 imbalance_ratio=1.075317 max_load_reduction=113454 max_load_reduction_pct=0.865585 moved_to_echo=64628 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[765, 4096, 0, 4096, 0, 0, 0, 3507, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4081, 4, 0, 0, 19, 0, 4096, 81, 0, 0, 0, 4096, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3854, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 9, 0, 0, 3858, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 4096, 0, 4096, 0, 0, 3153, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3862, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16560, 16412, 16503, 16376, 16354, 16626, 15881, 16360] avg=16384.0000 max=16626 max_rank=5 min=15881 min_rank=6 imbalance_ratio=1.014771 max_load_reduction=114446 max_load_reduction_pct=0.873154 moved_to_echo=65221 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[64, 4063, 7, 4075, 0, 0, 0, 3492, 0, 4089, 0, 0, 0, 30, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 5, 0, 4079, 0, 3, 4079, 4088, 0, 20, 4, 4054, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4090, 10, 3902, 0, 8, 7, 209, 6, 4082, 4, 0, 0, 0, 4069, 9, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 13, 4086, 0, 5, 0, 0, 12, 0, 0, 2, 4091, 4085, 0, 0, 3848, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 6, 3, 1, 8, 6, 9, 0, 4087, 4089, 6, 6, 6, 0, 5, 0, 3855, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4088, 0, 7, 0, 5, 3661, 7, 6, 4087, 4, 8, 0, 2, 4091, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 830, 4089, 9, 4080, 5, 0, 3642, 0, 5, 0, 4063, 3, 0, 0, 0, 5, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 16, 5, 26, 0, 4090, 4085, 9, 2, 0, 3857, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15824, 16355, 16433, 16382, 16380, 16212, 17091, 16395] avg=16384.0000 max=17091 max_rank=6 min=15824 min_rank=0 imbalance_ratio=1.043152 max_load_reduction=113981 max_load_reduction_pct=0.869606 moved_to_echo=66078 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[17, 4096, 0, 4082, 0, 0, 0, 3507, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4080, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 4095, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3853, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 35, 0, 0, 3858, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3976, 0, 0, 4096, 0, 0, 0, 0, 4096, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1484, 4096, 0, 4096, 0, 0, 2706, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3863, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15797, 16413, 16405, 16374, 16378, 16507, 16837, 16361] avg=16384.0000 max=16837 max_rank=6 min=15797 min_rank=0 imbalance_ratio=1.027649 max_load_reduction=114235 max_load_reduction_pct=0.871544 moved_to_echo=66083 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1255, 4096, 0, 4096, 0, 0, 0, 3507, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 1, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3694, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3853, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 14, 0, 0, 3858, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 4096, 0, 4096, 0, 0, 3065, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3863, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17050, 16413, 16012, 16374, 16360, 16626, 15876, 16361] avg=16384.0000 max=17050 max_rank=0 min=15876 min_rank=6 imbalance_ratio=1.040649 max_load_reduction=114022 max_load_reduction_pct=0.869919 moved_to_echo=65223 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[5120, 32640, 128, 32640, 128, 0, 128, 28032, 0, 32768, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 0, 32768, 32768, 128, 128, 128, 256, 128, 0, 30848, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 32768, 256, 0, 128, 128, 128, 128, 128, 0, 32768, 32768, 128, 128, 0, 30976, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 31360, 128, 128, 128, 1024, 128, 32768, 384, 128, 0, 0, 32640, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 32768, 0, 128, 0, 0, 128, 128, 0, 128, 32768, 32768, 128, 128, 30848, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 128, 32768, 0, 128, 32768, 32768, 128, 128, 128, 32640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 32768, 0, 128, 128, 128, 30592, 128, 256, 32768, 128, 128, 128, 128, 32768, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[8320, 32768, 128, 32768, 128, 0, 21632, 128, 128, 128, 32768, 128, 0, 0, 0, 128, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 3322, 21, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 19, 0, 0, 4093, 4091, 4073, 0, 4096, 0, 4095, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 44, 0, 4093, 0, 0, 0, 0, 93, 3, 0, 0, 0, 102, 0, 0, 1730, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2881, 4094, 4096, 0, 7, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4095, 2, 4094, 2, 0, 3989, 0, 0, 4, 0, 4094, 0, 0, 0, 4094, 0, 4092, 4095, 0, 1, 6, 0, 4086, 4093, 1, 0, 2, 0, 5, 0, 0, 0, 4096, 4093, 4075, 0, 4091, 2, 4094, 4094, 0, 0, 0, 10, 0, 0, 0, 1, 4094, 1, 4, 4, 0, 0, 4091, 4093, 2, 0, 0, 0, 1, 2, 2, 0, 0, 0, 0, 4093, 1, 4094, 0, 0, 4095, 0, 1, 4095, 4095, 4094, 0, 4095, 0, 0, 4093, 1, 0, 0, 0, 31, 0, 3, 0, 0, 81, 1, 1, 199, 2, 1, 1, 2, 4094, 0, 0, 0, 0, 3902, 4095, 4095, 2, 0, 4095, 1, 0, 4093, 0, 0, 0, 3, 0, 1, 1, 0, 0, 2, 3, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 3765, 6, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 61, 0, 0, 4025, 4074, 4093, 0, 4096, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 21, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 11, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4053, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 4049, 15, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 22, 0, 0, 4096, 4096, 4094, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4084, 0, 4089, 8, 0, 4091, 4, 0, 0, 0, 4083, 0, 7, 5, 4084, 0, 4078, 4086, 0, 5, 0, 0, 4088, 4087, 5, 0, 2, 0, 0, 0, 0, 0, 4053, 4081, 4089, 9, 4088, 2, 4091, 4090, 0, 0, 6, 0, 0, 0, 0, 7, 4087, 7, 0, 32, 0, 5, 4087, 4090, 7, 0, 0, 0, 1, 5, 0, 5, 27, 3, 0, 4087, 11, 4090, 0, 1, 4088, 0, 7, 4086, 4086, 4088, 3, 4079, 0, 3, 4090, 10, 9, 0, 0, 96, 0, 17, 0, 0, 0, 1, 7, 1, 0, 7, 9, 0, 4090, 6, 0, 6, 4, 3995, 4057, 4082, 9, 0, 4089, 12, 13, 4089, 13, 0, 0, 0, 10, 2, 11, 3, 0, 7, 6, 8, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4062, 43, 4060, 48, 0, 4027, 0, 0, 0, 0, 4038, 10, 31, 23, 4056, 2, 4047, 4051, 0, 37, 0, 1, 4072, 4053, 6, 0, 15, 0, 0, 0, 0, 0, 4062, 3940, 4054, 43, 4068, 0, 4073, 4066, 0, 0, 18, 131, 0, 0, 0, 27, 4015, 34, 0, 0, 0, 0, 4025, 4069, 38, 0, 0, 0, 9, 25, 39, 19, 5, 16, 0, 4059, 0, 4063, 4, 19, 4056, 0, 45, 4058, 4048, 4065, 21, 4012, 0, 29, 4066, 40, 57, 0, 0, 11, 0, 72, 0, 0, 0, 0, 41, 393, 0, 34, 41, 0, 4071, 23, 0, 28, 22, 3869, 3814, 4037, 118, 0, 4050, 56, 0, 4066, 0, 0, 0, 0, 56, 0, 40, 14, 0, 37, 2, 46, 31] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 2849, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 164, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 50, 0, 0, 0, 0, 2, 0, 0, 1034, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4027, 4096, 4096, 1, 60, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 3385, 14, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 56, 0, 0, 4028, 4092, 4078, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 705, 0, 0, 12, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 2, 404, 2, 0, 3989, 0, 0, 4, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4095, 0, 1, 6, 0, 4084, 4093, 1, 0, 2, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 4075, 0, 4091, 2, 0, 45, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 4, 4, 0, 0, 4091, 4093, 2, 0, 0, 0, 1, 2, 2, 3690, 229, 147, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 4095, 0, 1, 4095, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4093, 1, 0, 0, 0, 31, 0, 3, 0, 0, 81, 1, 1, 4092, 3902, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 2, 1, 1, 2, 0, 0, 0, 0, 0, 3902, 4095, 4095, 2, 0, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 0, 3, 0, 1, 1, 0, 0, 2, 3, 0, 1, 4094, 4094, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16684, 16379, 16412, 16363, 16382, 16330, 16165, 16357] avg=16384.0000 max=16684 max_rank=0 min=16165 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=114388 max_load_reduction_pct=0.872711 moved_to_echo=65234 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 404, 0, 0, 3322, 21, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4091, 4073, 0, 4096, 0, 0, 45, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 3692, 229, 147, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 44, 0, 4093, 0, 0, 0, 0, 93, 3, 0, 0, 0, 102, 0, 0, 4094, 3904, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2881, 4094, 4096, 0, 7, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16036, 16400, 16402, 16361, 16387, 16459, 16675, 16352] avg=16384.0000 max=16675 max_rank=6 min=16036 min_rank=0 imbalance_ratio=1.017761 max_load_reduction=114397 max_load_reduction_pct=0.872780 moved_to_echo=65873 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 404, 0, 0, 3765, 6, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 4074, 4093, 0, 4096, 0, 0, 45, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 21, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3692, 229, 147, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3904, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 4096, 4096, 0, 0, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16463, 16443, 16334, 16379, 16386, 16228, 16484, 16355] avg=16384.0000 max=16484 max_rank=6 min=16228 min_rank=5 imbalance_ratio=1.006104 max_load_reduction=114588 max_load_reduction_pct=0.874237 moved_to_echo=65453 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4062, 43, 400, 48, 0, 4027, 0, 0, 0, 0, 4038, 10, 31, 23, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4047, 4051, 0, 37, 0, 1, 4070, 4053, 6, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 3940, 4054, 43, 4068, 0, 0, 44, 0, 0, 18, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4015, 34, 0, 0, 0, 0, 4025, 4069, 38, 0, 0, 0, 9, 25, 39, 3660, 226, 146, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 5, 16, 0, 4059, 0, 0, 4, 19, 4056, 0, 45, 4058, 4048, 2, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 29, 4066, 40, 57, 0, 0, 11, 0, 72, 0, 0, 0, 0, 41, 4063, 3876, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 34, 41, 0, 0, 23, 0, 28, 22, 3869, 3814, 4037, 118, 0, 3824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 4066, 0, 0, 0, 0, 56, 0, 40, 14, 0, 37, 2, 46, 31, 4073, 4071, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16738, 16282, 16360, 16315, 16352, 16297, 16203, 16525] avg=16384.0000 max=16738 max_rank=0 min=16203 min_rank=6 imbalance_ratio=1.021606 max_load_reduction=114334 max_load_reduction_pct=0.872299 moved_to_echo=65377 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 404, 0, 0, 4049, 15, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4094, 0, 4096, 0, 0, 45, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3692, 229, 147, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 3904, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 4096, 0, 0, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16754, 16404, 16431, 16358, 16386, 16221, 16164, 16354] avg=16384.0000 max=16754 max_rank=0 min=16164 min_rank=6 imbalance_ratio=1.022583 max_load_reduction=114318 max_load_reduction_pct=0.872177 moved_to_echo=65125 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 404, 0, 0, 3385, 14, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 4092, 4078, 0, 4096, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3692, 229, 147, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 705, 0, 0, 4096, 3904, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16091, 16438, 16339, 16368, 16384, 16931, 16167, 16354] avg=16384.0000 max=16931 max_rank=5 min=16091 min_rank=0 imbalance_ratio=1.033386 max_load_reduction=114141 max_load_reduction_pct=0.870827 moved_to_echo=65836 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4084, 0, 399, 8, 0, 4091, 4, 0, 0, 0, 4083, 0, 7, 5, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4086, 0, 5, 0, 0, 4081, 4087, 5, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 4081, 4089, 9, 4088, 2, 0, 37, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4087, 7, 0, 32, 0, 5, 4087, 4090, 7, 0, 0, 0, 1, 5, 0, 3690, 232, 149, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 27, 3, 0, 4087, 11, 0, 0, 1, 4088, 0, 7, 4086, 4086, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 3, 4090, 10, 9, 0, 0, 96, 0, 17, 0, 0, 0, 1, 7, 4088, 3904, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 9, 0, 0, 6, 0, 6, 4, 3995, 4057, 4082, 9, 0, 3857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 4089, 13, 0, 0, 0, 10, 2, 11, 3, 0, 7, 6, 8, 6, 4091, 4090, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16765, 16344, 16365, 16406, 16404, 16334, 16033, 16421] avg=16384.0000 max=16765 max_rank=0 min=16033 min_rank=6 imbalance_ratio=1.023254 max_load_reduction=114307 max_load_reduction_pct=0.872093 moved_to_echo=65192 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 404, 0, 0, 2849, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 164, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3692, 229, 147, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 50, 0, 0, 0, 0, 2, 0, 0, 4094, 3904, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 4096, 4096, 1, 60, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15541, 16382, 16429, 16522, 16391, 16272, 17181, 16354] avg=16384.0000 max=17181 max_rank=6 min=15541 min_rank=0 imbalance_ratio=1.048645 max_load_reduction=113891 max_load_reduction_pct=0.868919 moved_to_echo=66198 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 32768, 128, 128, 0, 0, 384, 128, 128, 0, 0, 896, 128, 128, 32768, 31232, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32768, 128, 3328, 128, 0, 29568, 128, 0, 128, 0, 32768, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 128, 256, 0, 128, 32768, 32768, 128, 0, 0, 0, 128, 128, 128, 29568, 1920, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 128, 128, 32768, 0, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32640, 32640, 32768, 128, 32768, 128, 0, 384, 128, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 32768, 0, 128, 128, 128, 32768, 32768, 128, 0, 128, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[3840, 128, 128, 128, 128, 0, 128, 0, 128, 128, 30976, 32512, 32768, 256, 128, 30976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 128, 128, 128, 128, 128, 0, 128, 128, 128, 128, 32768, 32768, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[8, 8, 5, 11, 0, 4079, 1, 0, 4088, 4084, 730, 4090, 0, 0, 7, 0, 0, 0, 0, 4088, 4090, 18, 4085, 5, 6, 0, 0, 0, 4086, 0, 3064, 0, 0, 4090, 4089, 4066, 0, 3, 0, 4088, 48, 0, 0, 7, 0, 4, 0, 12, 4091, 6, 4056, 5, 12, 4084, 1, 8, 3, 11, 9, 6, 0, 4087, 4089, 6, 0, 0, 0, 4082, 4, 0, 0, 13, 0, 0, 0, 6, 0, 3999, 9, 5, 0, 4084, 0, 4089, 63, 5, 4089, 10, 0, 0, 0, 10, 9, 2, 0, 0, 0, 0, 4092, 0, 6, 24, 0, 4, 4, 4090, 0, 4, 0, 4076, 5, 4081, 0, 7, 0, 0, 0, 4060, 4086, 4083, 4089, 3, 331, 11, 4088, 3, 0, 12] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[2, 0, 1, 0, 0, 4094, 1, 0, 4094, 4096, 233, 4093, 8, 1, 0, 0, 0, 0, 0, 4094, 4095, 2, 4094, 0, 1, 0, 1, 0, 4095, 0, 1543, 0, 2, 4094, 4094, 4078, 0, 0, 29, 4094, 652, 0, 0, 2, 0, 0, 1, 2, 4094, 0, 4093, 1, 0, 4094, 1, 0, 0, 0, 1, 1, 0, 4092, 4095, 1, 70, 0, 0, 4094, 0, 0, 2, 2, 0, 0, 0, 3, 0, 4080, 1, 0, 2, 4094, 1, 4070, 1446, 0, 4093, 2, 0, 0, 2, 0, 0, 4, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 4095, 0, 4095, 0, 0, 1, 22, 0, 4085, 4095, 4094, 4096, 3, 161, 0, 4095, 1, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[38, 39, 33, 0, 53, 4056, 1, 3, 4050, 4046, 743, 4074, 0, 0, 36, 0, 0, 15, 0, 3999, 4060, 62, 4048, 0, 23, 0, 3, 0, 4042, 0, 2625, 22, 0, 4061, 4045, 4038, 0, 29, 0, 3993, 116, 0, 10, 38, 27, 25, 1, 58, 4069, 20, 3937, 29, 0, 4049, 5, 0, 14, 23, 36, 17, 0, 4034, 4062, 1, 0, 0, 2, 4037, 16, 0, 19, 48, 0, 0, 12, 2, 13, 4058, 34, 35, 3, 4028, 0, 4008, 40, 17, 4062, 51, 0, 0, 48, 3, 60, 76, 0, 0, 0, 21, 4046, 0, 57, 138, 0, 26, 31, 4061, 27, 5, 0, 3939, 0, 3944, 21, 31, 34, 97, 0, 3584, 4018, 4054, 4066, 0, 1249, 36, 4047, 35, 0, 25] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 151, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 3614, 0, 0, 4096, 4096, 4096, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 4096, 297, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 1, 4092, 4096, 4096, 4096, 0, 34, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 1546, 4096, 0, 0, 0, 0, 0, 4, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4, 1591, 0, 0, 4096, 4096, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 0, 4072, 966, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 4056, 4096, 4096, 4096, 0, 57, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 93, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 87, 2266, 0, 0, 4096, 4096, 4096, 0, 0, 27, 4096, 42, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4047, 0, 0, 0, 4096, 0, 4095, 1268, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1, 4096, 4096, 4096, 4096, 0, 360, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 23, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 3, 2306, 0, 0, 4096, 4096, 4096, 0, 0, 13, 4095, 16, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 4096, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 37, 0, 4096, 4096, 4096, 4096, 0, 1734, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 449, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 4096, 135, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 4096, 4096, 0, 3519, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 1, 0, 0, 4094, 1, 0, 4094, 4096, 233, 3648, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1914, 2, 4094, 0, 1, 0, 1, 0, 4095, 0, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3989, 4094, 4078, 0, 0, 29, 4094, 652, 0, 0, 2, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4093, 1, 0, 4094, 1, 0, 0, 0, 1, 1, 0, 4092, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 4094, 0, 0, 2, 2, 0, 0, 0, 3, 0, 4080, 1, 0, 4096, 4073, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 1, 4070, 1446, 0, 4093, 2, 0, 0, 2, 0, 0, 4, 0, 0, 2150, 878, 445, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 4095, 0, 4095, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 22, 0, 4085, 4095, 3216, 0, 3, 161, 0, 4095, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16178, 15744, 16943, 16400, 16452, 17280, 16395, 15680] avg=16384.0000 max=17280 max_rank=5 min=15680 min_rank=7 imbalance_ratio=1.054688 max_load_reduction=113792 max_load_reduction_pct=0.868164 moved_to_echo=65807 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[38, 39, 33, 0, 53, 4056, 1, 3, 4050, 4046, 743, 3631, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3999, 1897, 62, 4048, 0, 23, 0, 3, 0, 4042, 0, 2625, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 4045, 4038, 0, 29, 0, 3993, 116, 0, 10, 38, 27, 25, 1, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 20, 3937, 29, 0, 4049, 5, 0, 14, 23, 36, 17, 0, 4034, 4062, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4037, 16, 0, 19, 48, 0, 0, 12, 2, 13, 4058, 34, 35, 4066, 4048, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4028, 0, 4008, 40, 17, 4062, 51, 0, 0, 48, 3, 60, 76, 0, 0, 2132, 870, 443, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4046, 0, 57, 138, 0, 26, 31, 4061, 27, 5, 0, 3939, 0, 3944, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 31, 34, 97, 0, 3584, 4018, 3184, 0, 0, 1249, 36, 4047, 35, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16729, 16736, 16336, 16248, 16421, 15934, 16307, 16361] avg=16384.0000 max=16736 max_rank=1 min=15934 min_rank=5 imbalance_ratio=1.021484 max_load_reduction=114336 max_load_reduction_pct=0.872314 moved_to_echo=65023 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 1546, 3651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 1914, 0, 4096, 0, 0, 0, 0, 0, 4096, 4, 1591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 4096, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4096, 4075, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4072, 966, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2151, 879, 445, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4056, 4096, 3217, 0, 0, 57, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17485, 15801, 16281, 16405, 16383, 16798, 16396, 15523] avg=16384.0000 max=17485 max_rank=0 min=15523 min_rank=7 imbalance_ratio=1.067200 max_load_reduction=113587 max_load_reduction_pct=0.866600 moved_to_echo=65100 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[8, 8, 5, 11, 0, 4079, 1, 0, 4088, 4084, 730, 3642, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1905, 18, 4085, 5, 6, 0, 0, 0, 4086, 0, 3064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 4089, 4066, 0, 3, 0, 4088, 48, 0, 0, 7, 0, 4, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 6, 4056, 5, 12, 4084, 1, 8, 3, 11, 9, 6, 0, 4087, 4089, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4, 0, 0, 13, 0, 0, 0, 6, 0, 3999, 9, 5, 4089, 4076, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4089, 63, 5, 4089, 10, 0, 0, 0, 10, 9, 2, 0, 0, 2152, 878, 448, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 6, 24, 0, 4, 4, 4090, 0, 4, 0, 4076, 5, 4081, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4060, 4086, 3205, 0, 3, 331, 11, 4088, 3, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16663, 17257, 16293, 16398, 16316, 15939, 16400, 15806] avg=16384.0000 max=17257 max_rank=1 min=15806 min_rank=7 imbalance_ratio=1.053284 max_load_reduction=113815 max_load_reduction_pct=0.868340 moved_to_echo=64461 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 151, 3650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1914, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 3614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 4096, 4096, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 4075, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 297, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2151, 879, 445, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4092, 4096, 3217, 0, 0, 34, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16089, 17817, 16285, 16406, 16390, 16154, 16394, 15537] avg=16384.0000 max=17817 max_rank=1 min=15537 min_rank=7 imbalance_ratio=1.087463 max_load_reduction=113255 max_load_reduction_pct=0.864067 moved_to_echo=64475 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 93, 3651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1914, 0, 4096, 0, 0, 0, 0, 0, 4096, 87, 2266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 4096, 4096, 0, 0, 27, 4096, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4047, 0, 0, 4096, 4075, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 1268, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2151, 879, 445, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 3217, 0, 0, 360, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16031, 16555, 16348, 16405, 16355, 17124, 16388, 15866] avg=16384.0000 max=17124 max_rank=5 min=15866 min_rank=7 imbalance_ratio=1.045166 max_load_reduction=113948 max_load_reduction_pct=0.869354 moved_to_echo=65733 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 23, 3651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1914, 0, 4096, 0, 0, 0, 0, 0, 4096, 3, 2306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 4096, 4096, 0, 0, 13, 4095, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 4096, 4075, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2151, 879, 445, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 4096, 4096, 3217, 0, 0, 1734, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15962, 16511, 16307, 16405, 16363, 15852, 16396, 17276] avg=16384.0000 max=17276 max_rank=7 min=15852 min_rank=5 imbalance_ratio=1.054443 max_load_reduction=113796 max_load_reduction_pct=0.868195 moved_to_echo=65887 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4092, 0, 3651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1914, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 4075, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 135, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2151, 879, 445, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 3217, 0, 0, 3519, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15935, 14651, 16279, 16405, 16392, 15991, 16396, 19023] avg=16384.0000 max=19023 max_rank=7 min=14651 min_rank=1 imbalance_ratio=1.161072 max_load_reduction=112049 max_load_reduction_pct=0.854866 moved_to_echo=67802 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 32768, 15360, 128, 32768, 128, 128, 0, 128, 0, 32768, 128, 17536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 128, 128, 0, 128, 128, 128, 128, 32640, 128, 128, 32768, 32640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 128, 128, 32768, 32768, 3584, 29184, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 32640, 4224, 128, 32768, 128, 128, 0, 128, 128, 128, 128, 0, 0, 17280, 7040, 3584, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32000, 32768, 32768, 0, 128, 128, 32768, 896, 0, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[256, 128, 32640, 128, 128, 32768, 128, 128, 128, 128, 128, 128, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 0, 128, 256, 0, 128, 128, 32768, 128, 128, 0, 32640, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 256, 128, 32256, 32768, 25728, 0, 128, 7552, 128, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4095, 0, 2, 4094, 0, 0, 0, 14, 4058, 2, 1, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 4094, 4093, 53, 0, 4094, 3986, 75, 1, 0, 4094, 4047, 0, 4088, 0, 4093, 0, 1, 1, 3, 4094, 0, 4094, 4095, 1, 0, 4094, 0, 0, 0, 0, 2, 1, 2, 0, 4094, 0, 0, 0, 0, 2, 0, 3, 4094, 0, 60, 4094, 1, 0, 0, 0, 0, 2, 0, 4092, 4094, 1, 0, 2, 4088, 8, 0, 2, 4095, 2, 0, 2, 0, 2, 0, 1, 0, 0, 1, 4095, 0, 0, 1, 1, 4094, 4094, 0, 0, 1, 9, 0, 31, 0, 3, 2, 0, 4094, 4095, 4095, 2, 0, 0, 0, 4094, 0, 4095, 0, 0, 4060, 0, 0, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 644, 0, 4096, 3440, 4, 0, 0, 4096, 4094, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 4078, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4081, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4088, 0, 0, 4089, 0, 0, 0, 0, 3921, 2, 0, 0, 0, 0, 0, 6, 8, 0, 4087, 3, 4087, 4086, 392, 4, 4083, 4091, 0, 0, 0, 4086, 4088, 7, 4089, 10, 4088, 9, 10, 7, 0, 4063, 0, 4084, 4079, 7, 0, 4089, 0, 0, 0, 6, 7, 8, 8, 15, 4086, 0, 0, 0, 0, 4, 6, 0, 4084, 3, 135, 4088, 8, 0, 7, 6, 0, 5, 0, 3973, 4090, 10, 0, 0, 4082, 34, 9, 0, 4088, 0, 0, 5, 0, 9, 0, 7, 6, 9, 8, 4089, 0, 0, 0, 6, 4087, 4080, 4, 0, 6, 0, 0, 0, 9, 0, 12, 9, 3824, 4085, 4085, 5, 9, 0, 2, 4087, 8, 4089, 4, 0, 4090, 13, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 421, 0, 4096, 3686, 15, 0, 0, 4096, 4092, 0, 4081, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 0, 4077, 4095, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4093, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 49, 0, 4095, 4048, 7, 0, 0, 4096, 4086, 0, 4094, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 44, 4096, 0, 0, 0, 0, 0, 0, 0, 4077, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 4083, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4057, 2, 0, 4059, 0, 0, 4, 0, 4017, 45, 45, 0, 0, 0, 0, 38, 45, 0, 4053, 0, 4052, 4051, 178, 27, 4029, 4072, 0, 0, 6, 4025, 4065, 37, 4062, 46, 4058, 15, 53, 34, 0, 3964, 0, 4041, 4023, 30, 0, 4059, 0, 0, 0, 30, 45, 48, 43, 41, 4048, 0, 0, 0, 0, 37, 15, 0, 4043, 11, 824, 4052, 36, 0, 54, 26, 0, 26, 0, 3120, 4065, 48, 23, 0, 4051, 0, 12, 7, 4051, 0, 0, 30, 0, 41, 0, 43, 46, 5, 45, 4060, 0, 0, 9, 35, 4059, 4044, 48, 97, 21, 0, 0, 0, 11, 0, 44, 26, 3956, 4041, 4048, 21, 43, 0, 0, 4051, 9, 4062, 26, 0, 4046, 57, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 558, 0, 4096, 4040, 0, 0, 0, 4096, 4056, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 28, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3633, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4083, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 1614, 0, 4096, 2584, 10, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3992, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 2, 4094, 0, 0, 0, 14, 4058, 2, 1, 0, 0, 0, 0, 3995, 128, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 53, 0, 4094, 3986, 75, 1, 0, 3966, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 16, 0, 1, 1, 3, 4094, 0, 4094, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 0, 4094, 0, 0, 0, 0, 2, 0, 3, 4094, 4094, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 4094, 1, 0, 0, 0, 0, 2, 0, 4092, 4083, 1, 0, 2, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 4095, 2, 0, 2, 0, 2, 0, 1, 0, 0, 1, 4095, 0, 4090, 4066, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4094, 4094, 0, 0, 1, 9, 0, 31, 0, 3, 2, 0, 4094, 4082, 82, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 2, 0, 0, 0, 4094, 0, 13, 0, 0, 4060, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16400, 16223, 16393, 16374, 16423, 16376, 16521, 16362] avg=16384.0000 max=16521 max_rank=6 min=16223 min_rank=1 imbalance_ratio=1.008362 max_load_reduction=114551 max_load_reduction_pct=0.873955 moved_to_echo=65682 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3998, 128, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 644, 0, 4096, 3440, 4, 0, 0, 3968, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4068, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 4078, 4083, 82, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16425, 16246, 16400, 16371, 16399, 16369, 16480, 16382] avg=16384.0000 max=16480 max_rank=6 min=16246 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=114592 max_load_reduction_pct=0.874268 moved_to_echo=65630 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4057, 2, 0, 4059, 0, 0, 4, 0, 4017, 45, 45, 0, 0, 0, 0, 3961, 126, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 45, 0, 0, 0, 0, 0, 178, 27, 4029, 4072, 0, 0, 6, 3899, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4062, 46, 16, 15, 53, 34, 0, 3964, 0, 4041, 4023, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 45, 48, 43, 41, 4048, 0, 0, 0, 0, 37, 15, 0, 4043, 4059, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 824, 4052, 36, 0, 54, 26, 0, 26, 0, 3120, 4055, 48, 23, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 7, 4051, 0, 0, 30, 0, 41, 0, 43, 46, 5, 45, 4060, 0, 4053, 4025, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 35, 4059, 4044, 48, 97, 21, 0, 0, 0, 11, 0, 44, 26, 3956, 4049, 81, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 4048, 21, 43, 0, 0, 4051, 9, 13, 26, 0, 4046, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16326, 16359, 16321, 16450, 16326, 16429, 16506, 16355] avg=16384.0000 max=16506 max_rank=6 min=16321 min_rank=2 imbalance_ratio=1.007446 max_load_reduction=114566 max_load_reduction_pct=0.874069 moved_to_echo=65616 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3998, 128, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 4096, 3686, 15, 0, 0, 3968, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 16, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 0, 4077, 4084, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4069, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4093, 4083, 82, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16425, 16278, 16388, 16372, 16370, 16369, 16473, 16397] avg=16384.0000 max=16473 max_rank=6 min=16278 min_rank=1 imbalance_ratio=1.005432 max_load_reduction=114599 max_load_reduction_pct=0.874321 moved_to_echo=65609 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4088, 0, 0, 4089, 0, 0, 0, 0, 3921, 2, 0, 0, 0, 0, 0, 3995, 134, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8, 0, 0, 3, 0, 0, 392, 4, 4083, 4091, 0, 0, 0, 3952, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 10, 7, 9, 10, 7, 0, 4063, 0, 4084, 4079, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 8, 15, 4086, 0, 0, 0, 0, 4, 6, 0, 4084, 4089, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 135, 4088, 8, 0, 7, 6, 0, 5, 0, 3973, 4078, 10, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 9, 0, 4088, 0, 0, 5, 0, 9, 0, 7, 6, 9, 8, 4089, 0, 4087, 4060, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4087, 4080, 4, 0, 6, 0, 0, 0, 9, 0, 12, 9, 3824, 4078, 86, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4085, 5, 9, 0, 2, 4087, 8, 11, 4, 0, 4090, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16241, 16627, 16372, 16389, 16395, 16422, 16227, 16399] avg=16384.0000 max=16627 max_rank=1 min=16227 min_rank=6 imbalance_ratio=1.014832 max_load_reduction=114445 max_load_reduction_pct=0.873146 moved_to_echo=65443 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 3998, 128, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 4095, 4048, 7, 0, 0, 3968, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 16, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 4096, 0, 0, 0, 0, 0, 0, 0, 4077, 4084, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4069, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 4083, 4083, 82, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16416, 16253, 16398, 16373, 16397, 16369, 16470, 16396] avg=16384.0000 max=16470 max_rank=6 min=16253 min_rank=1 imbalance_ratio=1.005249 max_load_reduction=114602 max_load_reduction_pct=0.874344 moved_to_echo=65632 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3998, 128, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 0, 4096, 4040, 0, 0, 0, 3968, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4085, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4069, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3633, 4083, 82, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16425, 16718, 16400, 16372, 16387, 16369, 16017, 16384] avg=16384.0000 max=16718 max_rank=1 min=16017 min_rank=6 imbalance_ratio=1.020386 max_load_reduction=114354 max_load_reduction_pct=0.872452 moved_to_echo=65157 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 3998, 128, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1614, 0, 4096, 2584, 10, 0, 0, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 4085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4068, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3992, 4083, 82, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16414, 16368, 16400, 16371, 16375, 16369, 16378, 16397] avg=16384.0000 max=16414 max_rank=0 min=16368 min_rank=1 imbalance_ratio=1.001831 max_load_reduction=114658 max_load_reduction_pct=0.874771 moved_to_echo=65519 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32768, 128, 128, 32768, 0, 0, 128, 128, 32512, 128, 128, 0, 0, 0, 0, 32000, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 32768, 32768, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 31872, 32640, 768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 128, 0, 128, 0, 128, 128, 128, 128, 32768, 0, 32768, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 1152, 32768, 128, 0, 128, 128, 0, 128, 0, 31744, 32640, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 128, 32640, 0, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 128, 0, 128, 32768, 128, 128, 128, 0, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 0, 0, 128, 0, 0, 3968, 128, 32768, 29952, 128, 128, 128, 31744, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 128, 128, 128, 32768, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[11, 4083, 4089, 4086, 0, 0, 4080, 0, 7, 4072, 4066, 0, 3, 0, 0, 0, 0, 4088, 1, 0, 0, 8, 0, 0, 7, 12, 10, 4087, 10, 0, 0, 0, 7, 0, 10, 6, 0, 0, 4078, 7, 5, 10, 8, 7, 2, 4076, 0, 0, 0, 11, 0, 8, 4085, 4086, 0, 77, 4086, 0, 5, 16, 4088, 0, 2, 4084, 4083, 7, 0, 6, 12, 13, 0, 9, 4089, 0, 2, 0, 14, 4086, 4089, 4, 14, 2, 3, 6, 4088, 15, 2, 4083, 4081, 4087, 0, 6, 4073, 8, 0, 5, 1, 0, 4029, 4088, 0, 0, 0, 0, 0, 4077, 16, 0, 9, 4088, 2, 0, 0, 4, 0, 10, 0, 9, 4085, 4084, 0, 11, 4, 4079, 4089, 11, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0, 0, 4096, 0, 8, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4063, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1, 4095, 4094, 4095, 0, 7, 4094, 0, 2, 4094, 4094, 0, 2, 1, 17, 5, 0, 4095, 0, 0, 0, 1, 4, 1, 2, 12, 0, 4093, 0, 8, 0, 0, 0, 0, 2, 0, 0, 4, 4094, 0, 1, 1, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 1, 4094, 4092, 0, 0, 4088, 1, 0, 2, 4094, 0, 0, 4093, 4094, 2, 0, 0, 1, 0, 1, 0, 4094, 0, 3, 3, 2, 4094, 4095, 0, 3, 1, 0, 0, 4092, 0, 0, 4091, 4094, 4093, 0, 2, 4094, 1, 28, 2, 0, 0, 4095, 4059, 0, 0, 0, 0, 0, 4082, 2, 0, 0, 4094, 2, 0, 0, 0, 0, 1, 0, 0, 4094, 4091, 1, 0, 2, 4094, 4084, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4087, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4096, 4093, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4080, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[46, 4042, 4059, 4049, 0, 78, 4003, 25, 28, 4031, 3969, 0, 15, 0, 0, 0, 0, 4032, 11, 4, 0, 30, 0, 7, 3, 2, 50, 4055, 59, 0, 0, 0, 0, 0, 43, 33, 0, 0, 4037, 30, 34, 15, 1, 26, 55, 4051, 0, 0, 0, 36, 36, 37, 4043, 4036, 0, 210, 4045, 23, 32, 52, 4049, 0, 36, 3999, 4008, 40, 0, 32, 44, 0, 34, 48, 4043, 0, 64, 0, 0, 4046, 4056, 2, 82, 24, 26, 39, 4059, 64, 41, 4032, 4022, 4063, 0, 9, 3993, 56, 0, 25, 0, 33, 3990, 4055, 0, 0, 0, 0, 0, 3893, 65, 0, 69, 4049, 6, 0, 0, 22, 0, 45, 41, 0, 4047, 4028, 0, 77, 36, 4029, 4061, 0, 17, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 10, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 58, 0, 0, 0, 0, 4096, 4096, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 10, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 4096, 0, 0, 0, 4096, 4010, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 4091, 0, 0, 0, 2, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 4096, 0, 0, 4096, 0, 3, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4094, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 6, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 4096, 0, 0, 0, 4096, 4075, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 4095, 0, 23, 0, 7, 4094, 0, 2, 4094, 4094, 0, 2, 1, 17, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 4, 1, 2, 12, 0, 4093, 0, 8, 0, 0, 4092, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 4094, 0, 1, 1, 0, 0, 0, 4096, 0, 0, 4094, 4071, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4061, 4092, 0, 0, 4088, 1, 0, 2, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 1, 0, 1, 0, 4094, 0, 3, 3, 2, 4094, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 4091, 4094, 4076, 0, 2, 4094, 1, 28, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4059, 0, 0, 0, 0, 0, 4082, 2, 0, 0, 4094, 2, 0, 33, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 4091, 1, 0, 2, 4094, 4084, 0, 0, 0, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16435, 16395, 16370, 16339, 16381, 16392, 16377, 16383] avg=16384.0000 max=16435 max_rank=0 min=16339 min_rank=3 imbalance_ratio=1.003113 max_load_reduction=114637 max_load_reduction_pct=0.874611 moved_to_echo=65533 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 23, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 1, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 4096, 4072, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4078, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 33, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4063, 0, 0, 0, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16381, 16389, 16351, 16376, 16374, 16427, 16367] avg=16384.0000 max=16427 max_rank=6 min=16351 min_rank=3 imbalance_ratio=1.002625 max_load_reduction=114645 max_load_reduction_pct=0.874672 moved_to_echo=65544 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 23, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 3, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4072, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4076, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 33, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4080, 0, 0, 0, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16408, 16395, 16367, 16351, 16376, 16364, 16427, 16384] avg=16384.0000 max=16427 max_rank=6 min=16351 min_rank=3 imbalance_ratio=1.002625 max_load_reduction=114645 max_load_reduction_pct=0.874672 moved_to_echo=65551 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[46, 4042, 0, 22, 0, 78, 4003, 25, 28, 4031, 3969, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 11, 4, 0, 30, 0, 7, 3, 2, 50, 4055, 59, 0, 0, 0, 4059, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 33, 0, 0, 4037, 30, 34, 15, 1, 26, 55, 4051, 0, 0, 4049, 4026, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 36, 37, 4010, 4036, 0, 210, 4045, 23, 32, 52, 0, 0, 36, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 40, 0, 32, 44, 0, 34, 48, 4043, 0, 64, 0, 0, 4046, 4048, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 24, 26, 39, 0, 64, 41, 4032, 4022, 4047, 0, 9, 3993, 56, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 3990, 4055, 0, 0, 0, 0, 0, 3893, 65, 0, 69, 4049, 6, 0, 33, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 45, 41, 0, 4047, 4028, 0, 77, 36, 4029, 4061, 0, 17, 0, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16259, 16364, 16407, 16552, 16409, 16460, 16202, 16419] avg=16384.0000 max=16552 max_rank=3 min=16202 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=114520 max_load_reduction_pct=0.873718 moved_to_echo=65490 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[11, 4083, 0, 13, 0, 0, 4080, 0, 7, 4072, 4066, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 8, 0, 0, 7, 12, 10, 4087, 10, 0, 0, 0, 4088, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 10, 6, 0, 0, 4078, 7, 5, 10, 8, 7, 2, 4076, 0, 0, 4088, 4068, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 8, 4046, 4086, 0, 77, 4086, 0, 5, 16, 0, 0, 2, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 7, 0, 6, 12, 13, 0, 9, 4089, 0, 2, 0, 14, 4086, 4077, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2, 3, 6, 0, 15, 2, 4083, 4081, 4065, 0, 6, 4073, 8, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4029, 4088, 0, 0, 0, 0, 0, 4077, 16, 0, 9, 4088, 2, 0, 39, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 10, 0, 9, 4085, 4084, 0, 11, 4, 4079, 4089, 11, 5, 0, 12, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16335, 16393, 16379, 16421, 16402, 16363, 16364, 16415] avg=16384.0000 max=16421 max_rank=3 min=16335 min_rank=0 imbalance_ratio=1.002258 max_load_reduction=114651 max_load_reduction_pct=0.874718 moved_to_echo=65544 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 23, 0, 0, 4096, 0, 0, 4096, 4091, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4072, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4079, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 33, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4094, 0, 0, 0, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16404, 16384, 16367, 16351, 16376, 16369, 16423, 16398] avg=16384.0000 max=16423 max_rank=6 min=16351 min_rank=3 imbalance_ratio=1.002380 max_load_reduction=114649 max_load_reduction_pct=0.874702 moved_to_echo=65566 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 23, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 58, 4096, 4072, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4096, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4079, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 33, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 4096, 0, 0, 0, 4096, 4010, 0, 0, 0, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16417, 16377, 16425, 16356, 16376, 16377, 16425, 16319] avg=16384.0000 max=16425 max_rank=2 min=16319 min_rank=7 imbalance_ratio=1.002502 max_load_reduction=114647 max_load_reduction_pct=0.874687 moved_to_echo=65497 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 23, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4072, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4079, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 33, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 4096, 0, 0, 0, 4096, 4075, 0, 0, 0, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16383, 16368, 16351, 16376, 16373, 16427, 16387] avg=16384.0000 max=16427 max_rank=6 min=16351 min_rank=3 imbalance_ratio=1.002625 max_load_reduction=114645 max_load_reduction_pct=0.874672 moved_to_echo=65563 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 32640, 32768, 0, 0, 0, 0, 0, 32640, 128, 0, 128, 32768, 128, 0, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 0, 256, 0, 128, 32768, 128, 128, 32768, 32640, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 32512, 32768, 0, 384, 32768, 128, 128, 128, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 32768, 32768, 32640, 0, 128, 32640, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 0, 128, 128, 128, 128, 128, 32768, 0, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 128, 0, 128, 32768, 128, 128, 128, 128, 128, 128, 32768, 0, 128, 32768, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 128, 128, 32768, 32768, 128, 128, 128, 32768, 32640, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4093, 0, 2, 1, 4094, 1, 87, 4094, 0, 0, 0, 4095, 4, 1, 4, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 4095, 0, 2, 0, 1, 0, 4095, 1, 0, 0, 0, 4094, 0, 2, 1, 0, 0, 4078, 2, 0, 4094, 1, 4093, 0, 2, 0, 0, 1, 0, 4094, 0, 2, 4088, 4094, 1, 0, 4094, 0, 3, 4093, 4094, 4081, 2, 1, 0, 1, 0, 0, 0, 4094, 1, 4060, 1, 4096, 0, 0, 2, 4094, 0, 4094, 4087, 0, 0, 3, 0, 0, 0, 1, 2, 0, 4094, 0, 0, 0, 4092, 2, 21, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 52, 4010, 0, 0, 2, 4094, 0, 4094, 0, 2, 1, 1, 2, 2, 4096, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 810, 4096, 0, 0, 0, 3615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3987, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 119, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 241, 4086, 0, 0, 0, 4096, 0, 3552, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 84, 4096, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4047, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 96, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 99, 4005, 0, 0, 0, 4096, 0, 4008, 0, 0, 0, 0, 0, 0, 4090, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 344, 4096, 0, 0, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4078, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 9, 0, 0, 4096, 0, 195, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 258, 3737, 0, 0, 0, 4096, 0, 3787, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 32, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 3, 0, 0, 4096, 0, 47, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 0, 4096, 0, 4031, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4021, 2, 21, 53, 4025, 72, 0, 4041, 0, 43, 0, 4062, 153, 31, 195, 9, 0, 37, 0, 15, 0, 105, 4056, 0, 4030, 0, 3, 5, 0, 48, 4040, 29, 34, 55, 0, 4053, 0, 0, 54, 75, 0, 3342, 0, 0, 4040, 10, 4026, 0, 0, 90, 0, 42, 6, 3902, 33, 18, 4019, 4025, 36, 28, 4042, 33, 67, 4008, 4047, 3959, 0, 0, 0, 47, 13, 0, 17, 4036, 42, 4051, 40, 4058, 0, 52, 60, 4049, 82, 4044, 3285, 0, 0, 0, 0, 38, 1, 30, 0, 0, 4037, 0, 0, 2, 4032, 42, 111, 0, 40, 0, 16, 4047, 48, 0, 0, 30, 116, 1292, 3922, 51, 0, 0, 3910, 0, 4063, 0, 23, 39, 0, 60, 0, 4060, 0, 4046] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4083, 0, 1, 9, 4083, 12, 0, 4086, 0, 9, 0, 3924, 40, 8, 29, 8, 0, 8, 0, 3, 0, 23, 4087, 0, 4079, 0, 0, 0, 0, 5, 4086, 6, 6, 14, 0, 4050, 0, 0, 5, 15, 0, 4052, 0, 0, 4086, 0, 4084, 0, 0, 7, 7, 10, 0, 4056, 5, 0, 3844, 4082, 7, 1, 4087, 0, 14, 4081, 4083, 3865, 2, 0, 10, 10, 6, 0, 4, 4080, 9, 4088, 7, 4088, 0, 12, 14, 4087, 11, 4084, 4022, 0, 4, 0, 0, 8, 0, 5, 0, 0, 4085, 23, 0, 0, 4078, 7, 191, 0, 0, 0, 4, 4086, 16, 3, 0, 8, 16, 500, 4081, 0, 0, 3, 4075, 4, 4089, 0, 8, 9, 0, 11, 0, 4089, 0, 4085] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 937, 4096, 0, 0, 0, 3322, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 4096, 0, 0, 0, 4096, 0, 4096, 3558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 4096, 0, 414, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1034, 3056, 0, 0, 0, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 2, 1, 4094, 1, 87, 3953, 0, 0, 0, 4095, 4, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 4095, 0, 2, 0, 1, 0, 4095, 1, 3605, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 1, 0, 0, 4078, 2, 0, 4094, 1, 4093, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 4094, 0, 2, 4088, 4094, 1, 0, 69, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4081, 2, 1, 0, 1, 0, 0, 0, 4094, 1, 4060, 1, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 4094, 4087, 0, 0, 3, 0, 0, 0, 1, 2, 0, 4094, 0, 127, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 21, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 52, 4052, 3540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 2, 4094, 0, 4094, 0, 2, 1, 1, 2, 2, 364, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16335, 16378, 16463, 16447, 16379, 16547, 15856, 16667] avg=16384.0000 max=16667 max_rank=7 min=15856 min_rank=6 imbalance_ratio=1.017273 max_load_reduction=114405 max_load_reduction_pct=0.872841 moved_to_echo=65449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 84, 3955, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3599, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4047, 4096, 0, 0, 69, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 127, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 96, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 99, 4052, 3542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 0, 4096, 0, 4008, 0, 0, 0, 0, 0, 0, 364, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16298, 16372, 16482, 16404, 16428, 16538, 15981, 16569] avg=16384.0000 max=16569 max_rank=7 min=15981 min_rank=6 imbalance_ratio=1.011292 max_load_reduction=114503 max_load_reduction_pct=0.873589 moved_to_echo=65516 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 344, 3955, 0, 0, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 3605, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4078, 4096, 0, 0, 69, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 9, 127, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 195, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 258, 4052, 3542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3737, 0, 0, 0, 4096, 0, 3787, 0, 0, 0, 0, 0, 0, 364, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16462, 16381, 16482, 16435, 16428, 16565, 16239, 16080] avg=16384.0000 max=16565 max_rank=5 min=16080 min_rank=7 imbalance_ratio=1.011047 max_load_reduction=114507 max_load_reduction_pct=0.873619 moved_to_echo=65312 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 810, 3955, 0, 0, 0, 3615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3605, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3987, 4096, 0, 0, 69, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 127, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 119, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 241, 4052, 3542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 3552, 0, 0, 0, 0, 0, 0, 364, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16572, 16378, 16482, 16344, 16428, 16528, 16146, 16194] avg=16384.0000 max=16572 max_rank=0 min=16146 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=114500 max_load_reduction_pct=0.873566 moved_to_echo=65296 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 32, 3955, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3605, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 4096, 0, 0, 69, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 3, 127, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 47, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4052, 3542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 4031, 0, 0, 0, 0, 0, 0, 364, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16270, 16378, 16481, 16449, 16426, 16558, 15834, 16676] avg=16384.0000 max=16676 max_rank=7 min=15834 min_rank=6 imbalance_ratio=1.017822 max_load_reduction=114396 max_load_reduction_pct=0.872772 moved_to_echo=65494 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4021, 2, 21, 53, 4025, 72, 0, 3901, 0, 43, 0, 4062, 153, 31, 195, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 15, 0, 105, 4056, 0, 4030, 0, 3, 5, 0, 48, 4040, 29, 3573, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 55, 0, 4053, 0, 0, 54, 75, 0, 3342, 0, 0, 4040, 10, 4026, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 42, 6, 3902, 33, 18, 4019, 4025, 36, 28, 68, 33, 67, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 3959, 0, 0, 0, 47, 13, 0, 17, 4036, 42, 4051, 40, 43, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 4049, 82, 4044, 3285, 0, 0, 0, 0, 38, 1, 30, 0, 0, 4037, 0, 126, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4032, 42, 111, 0, 40, 0, 16, 4047, 48, 0, 0, 30, 116, 1292, 4015, 3495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3922, 51, 0, 0, 3910, 0, 4063, 0, 23, 39, 0, 60, 0, 361, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16588, 16420, 15786, 16375, 16347, 15795, 17286, 16475] avg=16384.0000 max=17286 max_rank=6 min=15786 min_rank=2 imbalance_ratio=1.055054 max_load_reduction=113786 max_load_reduction_pct=0.868118 moved_to_echo=65903 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4083, 0, 1, 9, 4083, 12, 0, 3934, 0, 9, 0, 3924, 40, 8, 29, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3, 0, 23, 4087, 0, 4079, 0, 0, 0, 0, 5, 4086, 6, 3602, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 14, 0, 4050, 0, 0, 5, 15, 0, 4052, 0, 0, 4086, 0, 4084, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 10, 0, 4056, 5, 0, 3844, 4082, 7, 1, 61, 0, 14, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 3865, 2, 0, 10, 10, 6, 0, 4, 4080, 9, 4088, 7, 39, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4087, 11, 4084, 4022, 0, 4, 0, 0, 8, 0, 5, 0, 0, 4085, 23, 128, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 7, 191, 0, 0, 0, 4, 4086, 16, 3, 0, 8, 16, 500, 4049, 3538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 3, 4075, 4, 4089, 0, 8, 9, 0, 11, 0, 359, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16140, 16387, 16415, 16175, 16215, 16520, 16496, 16724] avg=16384.0000 max=16724 max_rank=7 min=16140 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=114348 max_load_reduction_pct=0.872406 moved_to_echo=65955 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 937, 3955, 0, 0, 0, 3322, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3605, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 4096, 0, 0, 69, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 127, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 414, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1034, 4052, 3542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3056, 0, 0, 0, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 364, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16378, 16481, 16443, 16421, 16021, 17234, 15687] avg=16384.0000 max=17234 max_rank=6 min=15687 min_rank=7 imbalance_ratio=1.051880 max_load_reduction=113838 max_load_reduction_pct=0.868515 moved_to_echo=65363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 128, 32768, 128, 2304, 31616, 0, 128, 0, 31232, 256, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 32384, 128, 128, 128, 128, 128, 0, 128, 32768, 128, 32768, 128, 384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 32768, 31360, 0, 128, 128, 0, 128, 128, 128, 128, 128, 32768, 128, 1024, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 32768, 0, 128, 128, 128, 0, 32000, 128, 0, 32768, 128, 32768, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[31104, 128, 0, 128, 32640, 128, 31744, 0, 128, 128, 128, 128, 128, 2944, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 32640, 128, 128, 32256, 32768, 128, 128, 640, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 32768, 128, 1280, 128, 128, 0, 128, 32768, 128, 128, 0, 128, 256, 3584, 32384, 28288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 0, 128, 32768, 0, 32768, 128, 128, 128, 128, 128, 32768, 128, 28800, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4094, 1, 0, 4094, 1, 4094, 4092, 0, 2, 12, 3, 0, 0, 0, 4094, 41, 0, 0, 4094, 4093, 1, 0, 4091, 13, 6, 2, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 0, 2, 4094, 0, 0, 4093, 4094, 1, 0, 4, 0, 4, 4085, 5, 1, 4094, 4090, 0, 0, 0, 0, 0, 4093, 0, 1, 4093, 0, 1, 4085, 0, 1, 0, 2, 0, 4091, 4095, 0, 10, 0, 0, 2, 2, 3, 0, 3, 1, 1, 2, 4040, 0, 0, 0, 4093, 4095, 2, 2, 4094, 4094, 0, 1, 2, 2, 1, 3, 4080, 0, 0, 4094, 0, 1, 0, 17, 0, 1, 0, 4095, 4094, 1, 0, 3, 0, 4090, 0, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4069, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 4096, 0, 0, 0, 49, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4058, 0, 0, 13, 0, 0, 0, 0, 4096, 10, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 4094, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4077, 0, 2, 4096, 0, 0, 0, 28, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4084, 7, 0, 4081, 12, 4083, 4080, 18, 0, 0, 3, 0, 0, 0, 4079, 16, 0, 9, 4084, 4073, 8, 3, 4078, 254, 0, 10, 28, 4, 0, 12, 0, 0, 9, 0, 0, 0, 0, 4083, 4088, 0, 10, 0, 2, 6, 4081, 12, 0, 4087, 4080, 7, 13, 0, 0, 20, 4067, 0, 11, 4085, 4088, 0, 0, 0, 0, 0, 4087, 0, 0, 4089, 5, 10, 4081, 0, 10, 0, 12, 0, 4087, 4082, 14, 7, 3, 4, 0, 15, 16, 0, 10, 2, 8, 11, 3701, 0, 0, 0, 4083, 4073, 5, 11, 4084, 4082, 0, 14, 12, 7, 9, 0, 4041, 0, 2, 4075, 15, 0, 0, 205, 0, 8, 10, 4080, 4035, 11, 8, 0, 3, 4083, 0, 3, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4036, 37, 0, 4052, 45, 4042, 3626, 17, 2, 2, 0, 0, 0, 31, 4034, 188, 0, 51, 4032, 3958, 47, 17, 3984, 969, 19, 50, 99, 27, 14, 425, 0, 0, 64, 0, 20, 0, 0, 4034, 4052, 0, 41, 0, 12, 33, 4032, 65, 0, 4058, 4039, 82, 25, 0, 0, 203, 3712, 0, 52, 4045, 4017, 0, 0, 0, 0, 0, 4055, 47, 19, 4049, 21, 44, 3903, 0, 37, 0, 49, 0, 4052, 4032, 61, 20, 0, 4, 2, 54, 64, 0, 53, 24, 26, 47, 3549, 0, 0, 0, 4017, 4058, 19, 42, 4022, 4027, 0, 67, 68, 44, 43, 27, 3972, 47, 1, 4038, 0, 0, 1, 193, 0, 5, 57, 3512, 4048, 19, 27, 0, 28, 4023, 0, 21, 4044] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 4096, 0, 0, 0, 18, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 1, 0, 73, 1, 4094, 4092, 0, 2, 12, 3, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 4094, 4093, 1, 0, 4091, 13, 6, 2, 0, 1, 1, 0, 0, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 0, 2, 4094, 0, 0, 3942, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4094, 1, 0, 4, 0, 4, 4085, 5, 1, 4094, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 1, 4093, 0, 1, 4085, 0, 1, 0, 2, 0, 4091, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 2, 3, 0, 3, 1, 1, 2, 4040, 0, 0, 0, 4093, 4087, 4081, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2, 2, 4094, 4094, 0, 1, 2, 2, 1, 3, 4080, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 17, 0, 1, 0, 4095, 4094, 1, 0, 3, 0, 4090, 0, 0, 4094, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16466, 16196, 16377, 16407, 16381, 16447, 16383, 16415] avg=16384.0000 max=16466 max_rank=0 min=16196 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=114606 max_load_reduction_pct=0.874374 moved_to_echo=65626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 73, 0, 4096, 4058, 0, 0, 13, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 26, 0, 0, 0, 0, 3855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 3944, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 4086, 4084, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4077, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16432, 16179, 16380, 16415, 16382, 16479, 16375, 16430] avg=16384.0000 max=16479 max_rank=5 min=16179 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=114593 max_load_reduction_pct=0.874275 moved_to_echo=65666 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 73, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3944, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 4088, 4084, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16457, 16145, 16381, 16415, 16396, 16483, 16390, 16405] avg=16384.0000 max=16483 max_rank=5 min=16145 min_rank=1 imbalance_ratio=1.006042 max_load_reduction=114589 max_load_reduction_pct=0.874245 moved_to_echo=65674 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 73, 0, 4096, 4069, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3944, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4087, 4084, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16431, 16143, 16381, 16414, 16396, 16484, 16371, 16452] avg=16384.0000 max=16484 max_rank=5 min=16143 min_rank=1 imbalance_ratio=1.006104 max_load_reduction=114588 max_load_reduction_pct=0.874237 moved_to_echo=65703 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4036, 37, 0, 73, 45, 4042, 3626, 17, 2, 2, 0, 0, 0, 31, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 51, 4032, 3958, 47, 17, 3984, 969, 19, 50, 99, 27, 14, 425, 0, 3814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 20, 0, 0, 4034, 4052, 0, 41, 0, 12, 33, 4032, 65, 0, 3908, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4039, 82, 25, 0, 0, 203, 3712, 0, 52, 4045, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 47, 19, 4049, 21, 44, 3903, 0, 37, 0, 49, 0, 4052, 4032, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 4, 2, 54, 64, 0, 53, 24, 26, 47, 3549, 0, 0, 0, 4017, 4050, 4043, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 19, 42, 4022, 4027, 0, 67, 68, 44, 43, 27, 3972, 47, 1, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 193, 0, 5, 57, 3512, 4048, 19, 27, 0, 28, 4023, 0, 21, 4044, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15945, 17694, 16408, 16204, 16326, 16074, 16425, 15996] avg=16384.0000 max=17694 max_rank=1 min=15945 min_rank=0 imbalance_ratio=1.079956 max_load_reduction=113378 max_load_reduction_pct=0.865005 moved_to_echo=64821 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4084, 7, 0, 63, 12, 4083, 4080, 18, 0, 0, 3, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 9, 4084, 4073, 8, 3, 4078, 254, 0, 10, 28, 4, 0, 12, 0, 3847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4083, 4088, 0, 10, 0, 2, 6, 4081, 12, 0, 3941, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4080, 7, 13, 0, 0, 20, 4067, 0, 11, 4085, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4089, 5, 10, 4081, 0, 10, 0, 12, 0, 4087, 4082, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 4, 0, 15, 16, 0, 10, 2, 8, 11, 3701, 0, 0, 0, 4083, 4069, 4078, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 11, 4084, 4082, 0, 14, 12, 7, 9, 0, 4041, 0, 2, 4075, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 8, 10, 4080, 4035, 11, 8, 0, 3, 4083, 0, 3, 4084, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16429, 16426, 16383, 16389, 16399, 16135, 16361, 16550] avg=16384.0000 max=16550 max_rank=7 min=16135 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=114522 max_load_reduction_pct=0.873734 moved_to_echo=65445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 73, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 3855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3944, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 4088, 4084, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16455, 16145, 16381, 16414, 16396, 16484, 16376, 16421] avg=16384.0000 max=16484 max_rank=5 min=16145 min_rank=1 imbalance_ratio=1.006104 max_load_reduction=114588 max_load_reduction_pct=0.874237 moved_to_echo=65677 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 73, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 3855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3944, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4088, 4084, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16457, 16144, 16381, 16414, 16396, 16486, 16391, 16403] avg=16384.0000 max=16486 max_rank=5 min=16144 min_rank=1 imbalance_ratio=1.006226 max_load_reduction=114586 max_load_reduction_pct=0.874222 moved_to_echo=65676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 32768, 32768, 0, 128, 128, 128, 128, 128, 32640, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 128, 0, 0, 32768, 32768, 0, 128, 0, 128, 128, 32768, 128, 0, 31616, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32768, 128, 0, 640, 128, 32768, 32256, 128, 128, 128, 128, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[256, 32768, 128, 128, 128, 0, 256, 32384, 128, 128, 32768, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 128, 32640, 0, 128, 0, 128, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 0, 128, 32768, 32640, 128, 128, 32640, 1280, 128, 128, 256, 128, 128, 512, 0, 30848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 128, 31872, 0, 0, 0, 32768, 32768, 32640, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 640, 0, 128, 128, 32256, 32768, 128, 128, 128, 128, 32768, 0, 128, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 3893, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3968, 205, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3948, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 10, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4093, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[3977, 8, 33, 1, 4012, 0, 4083, 0, 4085, 4085, 0, 0, 9, 5, 4087, 4081, 0, 1, 4088, 11, 7, 5, 0, 0, 0, 11, 10, 5, 4086, 3, 4075, 176, 0, 0, 4086, 0, 6, 4080, 0, 4082, 0, 3, 0, 10, 0, 687, 4072, 4, 8, 4079, 11, 3212, 0, 0, 14, 4086, 0, 4084, 0, 4078, 0, 13, 3, 4084, 0, 0, 0, 11, 0, 1, 7, 0, 4084, 0, 0, 2, 0, 0, 0, 4060, 0, 0, 4041, 4087, 0, 6, 27, 17, 1, 0, 0, 4089, 4048, 13, 0, 4, 2, 9, 11, 10, 0, 12, 15, 12, 0, 11, 0, 10, 10, 8, 9, 230, 12, 0, 4084, 0, 7, 4087, 4079, 4087, 3, 4089, 0, 4086, 12, 13, 8, 12] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[3058, 29, 0, 1, 4035, 12, 4028, 0, 4038, 4042, 0, 0, 41, 0, 4044, 4013, 0, 0, 4060, 51, 40, 32, 0, 0, 1, 52, 52, 44, 4039, 0, 4048, 870, 0, 0, 4044, 0, 38, 4019, 0, 4022, 0, 22, 0, 59, 0, 3887, 3979, 45, 35, 4008, 38, 109, 0, 0, 74, 4041, 3, 4045, 1, 4036, 0, 72, 0, 4034, 0, 0, 0, 57, 0, 1, 41, 0, 4052, 4, 0, 5, 0, 0, 0, 3913, 0, 0, 3789, 4058, 0, 0, 93, 0, 7, 0, 0, 4054, 3991, 54, 0, 1, 13, 37, 76, 86, 0, 61, 65, 63, 3, 40, 0, 58, 48, 48, 17, 524, 54, 0, 4036, 0, 29, 4048, 4045, 4033, 21, 4058, 0, 4042, 54, 58, 39, 55] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4, 0, 4090, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3669, 964, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 30, 4096, 0, 0, 4096, 0, 3374, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3989, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 356, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 0, 3762, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 3775, 0, 4094, 0, 4095, 4094, 4, 0, 0, 0, 4094, 4094, 0, 0, 4096, 2, 1, 2, 0, 3, 0, 1, 1, 2, 4093, 0, 4094, 32, 0, 3, 4093, 0, 0, 4093, 0, 4093, 0, 1, 0, 2, 0, 602, 4092, 0, 2, 4093, 0, 3831, 0, 1, 4, 4094, 0, 4095, 0, 4092, 1, 1, 0, 4094, 1, 0, 0, 2, 0, 0, 2, 0, 4094, 0, 0, 2, 1, 0, 3, 4091, 3, 0, 4093, 4091, 0, 1, 13, 2, 1, 0, 0, 4094, 4094, 2, 0, 0, 0, 2, 0, 5, 0, 1, 3, 0, 0, 0, 2, 2, 1, 2, 0, 0, 2, 1, 4093, 0, 0, 4094, 4094, 4094, 0, 4031, 0, 4094, 1, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 3, 0, 4001, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 34, 1, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 283, 4096, 0, 0, 4096, 0, 3868, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4094, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 21, 0, 4091, 0, 4096, 0, 4096, 4096, 2, 0, 0, 0, 4096, 4096, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 109, 4, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3875, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 4073, 2, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 3893, 0, 4096, 0, 257, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3968, 205, 3197, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3434, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3948, 0, 0, 0, 850, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3839, 239, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 45, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 4096, 4096, 4096, 3857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16439, 16177, 15722, 17086, 16425, 16426, 16422, 16375] avg=16384.0000 max=17086 max_rank=3 min=15722 min_rank=2 imbalance_ratio=1.042847 max_load_reduction=113986 max_load_reduction_pct=0.869644 moved_to_echo=65648 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 3775, 0, 4094, 0, 257, 0, 4, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 1, 2, 0, 3, 0, 1, 1, 2, 4093, 0, 4094, 32, 3196, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3432, 0, 0, 4093, 0, 4093, 0, 1, 0, 2, 0, 602, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 3831, 0, 1, 4, 849, 0, 0, 0, 4092, 1, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 2, 0, 4094, 0, 0, 2, 1, 0, 3, 4091, 4094, 3838, 240, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 4091, 0, 1, 13, 2, 1, 0, 0, 4094, 4094, 2, 0, 0, 45, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 1, 3, 0, 0, 0, 2, 2, 1, 2, 0, 0, 4094, 4094, 4095, 3854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4093, 0, 0, 0, 4088, 4094, 0, 4031, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16318, 16139, 16318, 16968, 16417, 16445, 16155, 16312] avg=16384.0000 max=16968 max_rank=3 min=16139 min_rank=1 imbalance_ratio=1.035645 max_load_reduction=114104 max_load_reduction_pct=0.870544 moved_to_echo=65329 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3058, 29, 0, 1, 4035, 12, 4028, 0, 254, 0, 0, 0, 41, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 51, 40, 32, 0, 0, 1, 52, 52, 44, 4039, 0, 4048, 870, 3154, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3390, 0, 38, 4019, 0, 4022, 0, 22, 0, 59, 0, 3887, 3979, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4008, 38, 109, 0, 0, 74, 838, 3, 0, 1, 4036, 0, 72, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 1, 41, 0, 4052, 4, 0, 5, 0, 0, 0, 3913, 4042, 3784, 237, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3789, 4058, 0, 0, 93, 0, 7, 0, 0, 4054, 3991, 54, 0, 1, 45, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 37, 76, 86, 0, 61, 65, 63, 3, 40, 0, 58, 48, 48, 17, 524, 4048, 4044, 4045, 3805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 4036, 0, 29, 0, 4039, 4033, 21, 4058, 0, 0, 54, 58, 39, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15471, 17052, 19461, 13248, 16185, 16098, 17081, 16476] avg=16384.0000 max=19461 max_rank=2 min=13248 min_rank=3 imbalance_ratio=1.187805 max_load_reduction=111611 max_load_reduction_pct=0.851524 moved_to_echo=65840 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 3, 0, 4001, 0, 4096, 0, 257, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 34, 3197, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3434, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 283, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3868, 0, 0, 0, 850, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3839, 240, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 45, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 4096, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16549, 16135, 16006, 17006, 16421, 16433, 16146, 16376] avg=16384.0000 max=17006 max_rank=3 min=16006 min_rank=2 imbalance_ratio=1.037964 max_load_reduction=114066 max_load_reduction_pct=0.870255 moved_to_echo=65376 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 21, 0, 4091, 0, 4096, 0, 257, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 109, 3197, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3434, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3875, 0, 0, 0, 850, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3839, 240, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4073, 2, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 45, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 4096, 4096, 4096, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16659, 16181, 15726, 17013, 16416, 16410, 16289, 16378] avg=16384.0000 max=17013 max_rank=3 min=15726 min_rank=2 imbalance_ratio=1.038391 max_load_reduction=114059 max_load_reduction_pct=0.870201 moved_to_echo=65493 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4, 0, 4090, 0, 4096, 0, 257, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3669, 964, 3197, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3434, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 30, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3374, 0, 0, 0, 850, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3839, 240, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 45, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 4096, 4096, 4096, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16639, 16639, 15752, 16512, 16416, 16436, 16300, 16378] avg=16384.0000 max=16639 max_rank=0 min=15752 min_rank=2 imbalance_ratio=1.015564 max_load_reduction=114433 max_load_reduction_pct=0.873055 moved_to_echo=65530 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3989, 0, 4096, 0, 257, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 356, 3197, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3434, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3762, 0, 0, 0, 850, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3839, 239, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 45, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 4096, 4096, 4096, 3857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16534, 16457, 15725, 16900, 16417, 16435, 16226, 16378] avg=16384.0000 max=16900 max_rank=3 min=15725 min_rank=2 imbalance_ratio=1.031494 max_load_reduction=114172 max_load_reduction_pct=0.871063 moved_to_echo=65456 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3977, 8, 33, 1, 4012, 0, 4083, 0, 254, 0, 0, 0, 9, 5, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 11, 7, 5, 0, 0, 0, 11, 10, 5, 4086, 3, 4075, 176, 3195, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3418, 0, 6, 4080, 0, 4082, 0, 3, 0, 10, 0, 687, 4072, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4079, 11, 3212, 0, 0, 14, 837, 0, 0, 0, 4078, 0, 13, 3, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1, 7, 0, 4084, 0, 0, 2, 0, 0, 0, 4060, 4085, 3831, 240, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 4087, 0, 6, 27, 17, 1, 0, 0, 4089, 4048, 13, 0, 4, 49, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 11, 10, 0, 12, 15, 12, 0, 11, 0, 10, 10, 8, 9, 230, 4087, 4087, 4084, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4084, 0, 7, 0, 4072, 4087, 3, 4089, 0, 0, 12, 13, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16463, 16292, 16362, 16339, 16375, 16389, 16453, 16399] avg=16384.0000 max=16463 max_rank=0 min=16292 min_rank=1 imbalance_ratio=1.004822 max_load_reduction=114609 max_load_reduction_pct=0.874397 moved_to_echo=65616 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[31616, 128, 128, 128, 32000, 128, 32768, 0, 2176, 0, 128, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 25984, 0, 128, 128, 6784, 128, 0, 128, 32768, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 32512, 32768, 128, 128, 256, 128, 128, 0, 0, 32768, 32640, 128, 0, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 27520, 0, 128, 32768, 0, 32768, 0, 128, 0, 128, 0, 5504, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 0, 128, 128, 128, 128, 128, 32768, 128, 32256, 2816, 25600, 4992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 0, 128, 0, 128, 128, 0, 32768, 128, 0, 128, 128, 0, 128, 32640, 32768, 30720, 1920, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 32768, 32768, 128, 32768, 0, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 1536, 32768, 32768, 32768, 30848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2, 832, 1702, 1, 1309, 0, 912, 8, 2137, 66, 2039, 1217, 1510, 429, 1, 436, 179, 1819, 460, 682, 3900, 1105, 198, 55, 2, 3048, 133, 89, 374, 11, 704, 1945, 3896, 6, 9, 1474, 1690, 19, 306, 1074, 3399, 2580, 164, 98, 6, 781, 968, 686, 337, 569, 883, 72, 48, 16, 3816, 108, 2, 3059, 3763, 2486, 271, 147, 224, 0, 407, 4028, 1, 1514, 208, 22, 3002, 2816, 443, 831, 180, 0, 156, 1966, 917, 1400, 2147, 344, 39, 3958, 122, 1150, 2121, 4034, 38, 50, 861, 451, 3097, 153, 806, 3, 809, 401, 4020, 221, 0, 1625, 1168, 2711, 3848, 1808, 0, 988, 795, 2539, 310, 1508, 111, 491, 867, 219, 4, 690, 345, 1848, 477, 556, 2118, 1133, 445, 2, 1488, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[3, 224, 1841, 0, 1837, 2, 936, 2, 1477, 115, 1059, 796, 865, 268, 1, 280, 142, 2216, 570, 1192, 3726, 1132, 204, 82, 0, 2447, 170, 65, 888, 15, 987, 2486, 4037, 8, 7, 847, 2090, 34, 532, 2086, 2907, 2936, 511, 171, 7, 650, 1005, 1007, 402, 441, 820, 78, 5, 11, 3661, 129, 6, 2713, 3493, 2309, 201, 279, 389, 0, 696, 4058, 1, 1414, 220, 50, 3095, 2537, 337, 541, 182, 8, 474, 2063, 1245, 695, 1915, 484, 42, 3960, 116, 659, 2851, 4033, 116, 28, 785, 934, 3241, 146, 1084, 0, 1364, 410, 4031, 301, 0, 1887, 847, 1951, 3638, 1488, 17, 834, 497, 1330, 142, 2380, 56, 1027, 1099, 254, 4, 1170, 321, 1463, 751, 695, 2112, 1041, 647, 0, 1504, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1, 393, 2452, 4, 1034, 0, 827, 3, 1716, 79, 1697, 1022, 1229, 235, 0, 317, 140, 1918, 426, 1437, 3863, 591, 290, 57, 0, 2562, 157, 167, 551, 16, 522, 1856, 4021, 16, 15, 687, 2015, 26, 170, 1308, 2732, 2467, 158, 173, 8, 1026, 862, 736, 356, 372, 610, 113, 65, 35, 3678, 137, 4, 2460, 3547, 2619, 184, 425, 220, 1, 779, 4015, 1, 1560, 214, 39, 3363, 2800, 612, 596, 214, 5, 103, 1239, 1547, 977, 2249, 476, 48, 3852, 90, 913, 2380, 4042, 79, 12, 856, 917, 3235, 261, 898, 3, 645, 466, 3989, 216, 2, 1913, 832, 1815, 3785, 2293, 22, 1347, 765, 1685, 283, 2266, 205, 1205, 1068, 175, 9, 1041, 389, 1997, 813, 1085, 2515, 1278, 377, 5, 1400, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2, 568, 2276, 10, 1785, 2, 830, 4, 1746, 90, 1423, 683, 1415, 184, 0, 650, 255, 1962, 699, 1073, 3741, 739, 318, 143, 0, 2680, 193, 396, 418, 24, 610, 1820, 3848, 1, 11, 1072, 1757, 21, 574, 1626, 2762, 2677, 354, 229, 7, 803, 933, 592, 409, 472, 781, 78, 26, 56, 3830, 141, 0, 2767, 3643, 2977, 129, 326, 207, 2, 617, 4065, 1, 996, 186, 56, 2933, 2748, 868, 649, 147, 21, 282, 1834, 1122, 930, 1967, 931, 49, 3981, 101, 632, 2571, 4025, 43, 41, 612, 794, 2877, 219, 1197, 0, 960, 322, 3977, 271, 0, 2042, 1171, 1892, 3992, 1865, 2, 1017, 839, 1161, 141, 2342, 93, 762, 932, 571, 4, 611, 247, 2063, 594, 691, 2303, 1114, 578, 8, 1162, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[4, 367, 2237, 0, 1092, 0, 707, 2, 1599, 97, 1099, 654, 1333, 258, 0, 586, 203, 1840, 832, 1579, 3580, 847, 419, 43, 0, 2681, 242, 248, 799, 36, 584, 1997, 4057, 0, 22, 1097, 2680, 27, 511, 982, 3322, 2596, 174, 149, 2, 739, 1527, 764, 468, 246, 1022, 123, 36, 12, 4015, 278, 1, 2471, 3683, 2154, 178, 303, 299, 0, 804, 4039, 0, 1239, 208, 89, 3206, 2585, 713, 996, 292, 4, 412, 1287, 1139, 745, 1628, 757, 37, 3893, 132, 998, 2355, 3999, 49, 34, 448, 799, 2697, 80, 1183, 0, 515, 590, 4043, 199, 2, 1885, 831, 2445, 3911, 1992, 7, 1825, 687, 1964, 155, 1855, 54, 995, 986, 341, 11, 914, 280, 1686, 753, 529, 1858, 1044, 486, 5, 1473, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[23, 321, 1532, 56, 1482, 0, 628, 42, 1456, 522, 1357, 337, 892, 343, 5, 590, 261, 1523, 673, 1189, 3790, 1674, 397, 133, 28, 2381, 178, 455, 592, 22, 758, 1937, 3674, 34, 66, 1084, 1827, 30, 630, 1065, 2554, 2577, 388, 156, 58, 1039, 1275, 1446, 452, 203, 1146, 292, 127, 111, 3621, 186, 1, 2627, 3805, 2830, 294, 239, 387, 40, 808, 3812, 35, 1393, 494, 92, 3365, 2422, 754, 583, 212, 17, 453, 1394, 1354, 699, 1466, 1216, 249, 3837, 184, 547, 1914, 3957, 98, 43, 927, 960, 2446, 232, 1650, 22, 1111, 530, 3835, 224, 7, 1356, 1156, 2248, 3370, 1842, 28, 1549, 823, 1486, 184, 2017, 79, 634, 1082, 325, 17, 657, 291, 1902, 912, 616, 2333, 1234, 796, 18, 921, 13] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[6, 759, 2342, 0, 974, 1, 435, 39, 1523, 183, 1151, 725, 1139, 157, 0, 408, 217, 1434, 1241, 1228, 3577, 793, 365, 88, 0, 2338, 183, 320, 905, 25, 756, 2075, 3957, 1, 56, 879, 2456, 16, 650, 1488, 2682, 3201, 444, 195, 7, 764, 1032, 889, 665, 691, 668, 156, 24, 21, 3664, 224, 0, 2413, 3358, 2605, 457, 211, 426, 9, 1472, 4035, 0, 1028, 145, 121, 3107, 2455, 401, 596, 368, 21, 223, 1639, 989, 806, 1875, 813, 57, 3672, 228, 1050, 2347, 3967, 39, 41, 516, 512, 2659, 88, 1072, 6, 750, 541, 3891, 240, 5, 2107, 1245, 2238, 3796, 1908, 19, 1406, 1132, 1380, 89, 2015, 123, 1367, 1097, 577, 17, 887, 394, 1792, 676, 596, 1941, 1438, 343, 8, 1007, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[11, 606, 2425, 2, 979, 0, 887, 8, 1510, 248, 1734, 741, 1506, 311, 0, 447, 217, 1762, 951, 1270, 3850, 1110, 239, 35, 0, 2817, 166, 229, 591, 29, 703, 1979, 3873, 1, 12, 1072, 1686, 13, 434, 1584, 3138, 2548, 345, 358, 4, 859, 998, 1044, 370, 382, 898, 120, 98, 26, 3842, 119, 4, 2321, 3789, 2279, 197, 126, 201, 0, 780, 4007, 2, 1292, 104, 131, 2975, 2286, 590, 703, 245, 0, 90, 1764, 1189, 1258, 2230, 470, 49, 3888, 256, 971, 2693, 3921, 35, 47, 697, 832, 2610, 475, 1295, 1, 992, 472, 3897, 279, 1, 1714, 1231, 2190, 3899, 1965, 50, 1116, 657, 1472, 136, 1347, 207, 644, 767, 443, 8, 967, 460, 1570, 581, 940, 2570, 1133, 404, 5, 960, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 832, 1702, 1, 1309, 0, 912, 8, 2137, 66, 2039, 1217, 1510, 429, 1, 436, 4020, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 1819, 460, 682, 3900, 1105, 198, 55, 2, 3048, 133, 89, 374, 11, 704, 1945, 880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2426, 6, 9, 1474, 1690, 19, 306, 1074, 3399, 2580, 164, 98, 6, 781, 968, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 569, 883, 72, 48, 16, 3816, 108, 2, 3059, 3763, 2486, 271, 147, 224, 0, 526, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 2720, 1, 1514, 208, 22, 3002, 2816, 443, 831, 180, 0, 156, 1966, 917, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2147, 344, 39, 3958, 122, 1150, 2121, 592, 38, 50, 861, 451, 3097, 153, 806, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 809, 401, 0, 221, 0, 1625, 1168, 2711, 2031, 1808, 0, 988, 795, 2539, 310, 1508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 491, 867, 219, 4, 690, 345, 1848, 477, 556, 2118, 1133, 445, 2, 1488, 0, 1817, 1736, 944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18327, 15584, 15686, 16755, 16583, 15932, 16914, 15291] avg=16384.0000 max=18327 max_rank=0 min=15291 min_rank=7 imbalance_ratio=1.118591 max_load_reduction=112745 max_load_reduction_pct=0.860176 moved_to_echo=64720 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 393, 2452, 4, 1034, 0, 827, 3, 1716, 79, 1697, 1022, 1229, 235, 0, 317, 3989, 1709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 1918, 426, 1437, 3863, 591, 290, 57, 0, 2562, 157, 167, 551, 16, 522, 1856, 877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2503, 16, 15, 687, 2015, 26, 170, 1308, 2732, 2467, 158, 173, 8, 1026, 862, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 356, 372, 610, 113, 65, 35, 3678, 137, 4, 2460, 3547, 2619, 184, 425, 220, 1, 543, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 779, 2711, 1, 1560, 214, 39, 3363, 2800, 612, 596, 214, 5, 103, 1239, 1547, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2249, 476, 48, 3852, 90, 913, 2380, 593, 79, 12, 856, 917, 3235, 261, 898, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 645, 466, 0, 216, 2, 1913, 832, 1815, 1998, 2293, 22, 1347, 765, 1685, 283, 2266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 1205, 1068, 175, 9, 1041, 389, 1997, 813, 1085, 2515, 1278, 377, 5, 1400, 3, 1787, 1740, 975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16707, 15430, 14902, 15796, 16760, 16862, 16548, 18067] avg=16384.0000 max=18067 max_rank=7 min=14902 min_rank=2 imbalance_ratio=1.102722 max_load_reduction=113005 max_load_reduction_pct=0.862160 moved_to_echo=68237 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 568, 2276, 10, 1785, 2, 830, 4, 1746, 90, 1423, 683, 1415, 184, 0, 650, 3977, 1702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 1962, 699, 1073, 3741, 739, 318, 143, 0, 2680, 193, 396, 418, 24, 610, 1820, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2396, 1, 11, 1072, 1757, 21, 574, 1626, 2762, 2677, 354, 229, 7, 803, 933, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 472, 781, 78, 26, 56, 3830, 141, 0, 2767, 3643, 2977, 129, 326, 207, 2, 519, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 617, 2745, 1, 996, 186, 56, 2933, 2748, 868, 649, 147, 21, 282, 1834, 1122, 930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1967, 931, 49, 3981, 101, 632, 2571, 590, 43, 41, 612, 794, 2877, 219, 1197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 960, 322, 0, 271, 0, 2042, 1171, 1892, 2107, 1865, 2, 1017, 839, 1161, 141, 2342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 762, 932, 571, 4, 611, 247, 2063, 594, 691, 2303, 1114, 578, 8, 1162, 0, 1885, 1733, 933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17347, 15959, 15815, 16795, 16135, 16605, 16132, 16284] avg=16384.0000 max=17347 max_rank=0 min=15815 min_rank=2 imbalance_ratio=1.058777 max_load_reduction=113725 max_load_reduction_pct=0.867653 moved_to_echo=65156 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 224, 1841, 0, 1837, 2, 936, 2, 1477, 115, 1059, 796, 865, 268, 1, 280, 4031, 1705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 2216, 570, 1192, 3726, 1132, 204, 82, 0, 2447, 170, 65, 888, 15, 987, 2486, 887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2513, 8, 7, 847, 2090, 34, 532, 2086, 2907, 2936, 511, 171, 7, 650, 1005, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 441, 820, 78, 5, 11, 3661, 129, 6, 2713, 3493, 2309, 201, 279, 389, 0, 545, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 2739, 1, 1414, 220, 50, 3095, 2537, 337, 541, 182, 8, 474, 2063, 1245, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1915, 484, 42, 3960, 116, 659, 2851, 592, 116, 28, 785, 934, 3241, 146, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1364, 410, 0, 301, 0, 1887, 847, 1951, 1920, 1488, 17, 834, 497, 1330, 142, 2380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 1027, 1099, 254, 4, 1170, 321, 1463, 751, 695, 2112, 1041, 647, 0, 1504, 1, 1718, 1736, 979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15442, 17209, 17311, 15914, 16297, 16953, 15368, 16578] avg=16384.0000 max=17311 max_rank=2 min=15368 min_rank=6 imbalance_ratio=1.056580 max_load_reduction=113761 max_load_reduction_pct=0.867928 moved_to_echo=65196 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 367, 2237, 0, 1092, 0, 707, 2, 1599, 97, 1099, 654, 1333, 258, 0, 586, 4043, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 1840, 832, 1579, 3580, 847, 419, 43, 0, 2681, 242, 248, 799, 36, 584, 1997, 886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2519, 0, 22, 1097, 2680, 27, 511, 982, 3322, 2596, 174, 149, 2, 739, 1527, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 246, 1022, 123, 36, 12, 4015, 278, 1, 2471, 3683, 2154, 178, 303, 299, 0, 551, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 2719, 0, 1239, 208, 89, 3206, 2585, 713, 996, 292, 4, 412, 1287, 1139, 745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1628, 757, 37, 3893, 132, 998, 2355, 578, 49, 34, 448, 799, 2697, 80, 1183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 590, 0, 199, 2, 1885, 831, 2445, 2059, 1992, 7, 1825, 687, 1964, 155, 1855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 995, 986, 341, 11, 914, 280, 1686, 753, 529, 1858, 1044, 486, 5, 1473, 1, 1852, 1725, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15774, 16816, 17111, 16274, 16438, 15668, 17011, 15980] avg=16384.0000 max=17111 max_rank=2 min=15668 min_rank=5 imbalance_ratio=1.044373 max_load_reduction=113961 max_load_reduction_pct=0.869453 moved_to_echo=65097 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[23, 321, 1532, 56, 1482, 0, 628, 42, 1456, 522, 1357, 337, 892, 343, 5, 590, 3835, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 1523, 673, 1189, 3790, 1674, 397, 133, 28, 2381, 178, 455, 592, 22, 758, 1937, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2287, 34, 66, 1084, 1827, 30, 630, 1065, 2554, 2577, 388, 156, 58, 1039, 1275, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 203, 1146, 292, 127, 111, 3621, 186, 1, 2627, 3805, 2830, 294, 239, 387, 40, 496, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 808, 2574, 35, 1393, 494, 92, 3365, 2422, 754, 583, 212, 17, 453, 1394, 1354, 699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1466, 1216, 249, 3837, 184, 547, 1914, 581, 98, 43, 927, 960, 2446, 232, 1650, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1111, 530, 0, 224, 7, 1356, 1156, 2248, 1779, 1842, 28, 1549, 823, 1486, 184, 2017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 634, 1082, 325, 17, 657, 291, 1902, 912, 616, 2333, 1234, 796, 18, 921, 13, 1591, 1703, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15094, 16824, 16516, 17262, 16649, 16372, 16340, 16015] avg=16384.0000 max=17262 max_rank=3 min=15094 min_rank=0 imbalance_ratio=1.053589 max_load_reduction=113810 max_load_reduction_pct=0.868301 moved_to_echo=65376 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[6, 759, 2342, 0, 974, 1, 435, 39, 1523, 183, 1151, 725, 1139, 157, 0, 408, 3891, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 1434, 1241, 1228, 3577, 793, 365, 88, 0, 2338, 183, 320, 905, 25, 756, 2075, 882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2464, 1, 56, 879, 2456, 16, 650, 1488, 2682, 3201, 444, 195, 7, 764, 1032, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 665, 691, 668, 156, 24, 21, 3664, 224, 0, 2413, 3358, 2605, 457, 211, 426, 9, 534, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1472, 2724, 0, 1028, 145, 121, 3107, 2455, 401, 596, 368, 21, 223, 1639, 989, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1875, 813, 57, 3672, 228, 1050, 2347, 581, 39, 41, 516, 512, 2659, 88, 1072, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 750, 541, 0, 240, 5, 2107, 1245, 2238, 2004, 1908, 19, 1406, 1132, 1380, 89, 2015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 1367, 1097, 577, 17, 887, 394, 1792, 676, 596, 1941, 1438, 343, 8, 1007, 3, 1792, 1708, 959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15411, 16427, 17224, 16555, 16095, 15556, 17079, 16725] avg=16384.0000 max=17224 max_rank=2 min=15411 min_rank=0 imbalance_ratio=1.051270 max_load_reduction=113848 max_load_reduction_pct=0.868591 moved_to_echo=65455 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 606, 2425, 2, 979, 0, 887, 8, 1510, 248, 1734, 741, 1506, 311, 0, 447, 3897, 1658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 1762, 951, 1270, 3850, 1110, 239, 35, 0, 2817, 166, 229, 591, 29, 703, 1979, 875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2411, 1, 12, 1072, 1686, 13, 434, 1584, 3138, 2548, 345, 358, 4, 859, 998, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 382, 898, 120, 98, 26, 3842, 119, 4, 2321, 3789, 2279, 197, 126, 201, 0, 523, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 780, 2706, 2, 1292, 104, 131, 2975, 2286, 590, 703, 245, 0, 90, 1764, 1189, 1258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2230, 470, 49, 3888, 256, 971, 2693, 575, 35, 47, 697, 832, 2610, 475, 1295, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 992, 472, 0, 279, 1, 1714, 1231, 2190, 2058, 1965, 50, 1116, 657, 1472, 136, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 644, 767, 443, 8, 967, 460, 1570, 581, 940, 2570, 1133, 404, 5, 960, 5, 1841, 1688, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16970, 16823, 16507, 15721, 16115, 17124, 15680, 16132] avg=16384.0000 max=17124 max_rank=5 min=15680 min_rank=6 imbalance_ratio=1.045166 max_load_reduction=113948 max_load_reduction_pct=0.869354 moved_to_echo=65051 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[15488, 5504, 640, 31104, 1280, 7040, 19328, 4736, 512, 384, 5760, 6272, 22912, 1664, 9216, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 4096, 16896, 128, 10496, 128, 6272, 128, 13184, 1408, 11648, 6272, 9984, 2304, 128, 3840, 31744, 13568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[19584, 128, 256, 8320, 16256, 256, 3840, 11264, 23552, 21632, 2560, 1536, 128, 6784, 8704, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1024, 7168, 7936, 2944, 128, 7040, 2816, 14336, 5632, 5760, 17792, 9472, 4096, 128, 9984, 128, 14336, 13824, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[3584, 3456, 6912, 1152, 512, 384, 30208, 1408, 128, 20864, 29184, 20352, 1920, 2176, 2432, 128, 4352, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[6400, 21760, 128, 10496, 1792, 640, 25088, 20736, 4736, 5504, 1920, 128, 2304, 13312, 9600, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1664, 14592, 5888, 9728, 30080, 8064, 2432, 640, 128, 20992, 1536, 2048, 5120, 256, 5632, 16128, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[7168, 3840, 0, 2048, 128, 14592, 8576, 17536, 16000, 15232, 256, 10112, 6272, 13056, 1536, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4095, 1840, 4095, 0, 0, 6, 622, 4093, 0, 2331, 1, 0, 4003, 0, 0, 74, 0, 0, 4092, 4094, 0, 0, 94, 1, 0, 0, 1, 734, 0, 0, 4091, 0, 1944, 1, 0, 11, 0, 0, 2, 0, 561, 5, 0, 0, 2, 20, 3688, 1813, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 4, 0, 879, 0, 0, 4089, 4, 2110, 0, 4071, 4073, 4095, 1, 0, 0, 0, 2, 1, 1, 6, 0, 0, 4096, 4094, 4095, 0, 3990, 0, 0, 4086, 0, 0, 36, 0, 2, 4093, 4096, 0, 0, 0, 0, 0, 3, 6, 0, 2, 6, 0, 4095, 0, 8, 0, 1, 0, 0, 3942, 4095, 4027, 0, 4095, 0, 4096, 0, 4094, 0, 82] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1, 1, 4095, 2688, 4071, 23, 0, 0, 65, 3972, 0, 750, 0, 13, 3843, 5, 0, 22, 58, 5, 4055, 4091, 0, 16, 115, 1, 4, 6, 0, 525, 0, 0, 4088, 16, 711, 0, 82, 11, 0, 0, 40, 2, 2400, 0, 7, 4, 0, 0, 3223, 2334, 4060, 12, 5, 0, 2, 55, 4090, 0, 0, 0, 4095, 0, 0, 8, 3325, 162, 1, 3815, 0, 762, 0, 4051, 3734, 4069, 0, 0, 0, 31, 137, 0, 0, 41, 2, 24, 4048, 4076, 4095, 35, 4046, 0, 2, 3887, 1, 0, 158, 32, 0, 4090, 4093, 0, 15, 0, 0, 0, 1, 0, 0, 0, 0, 6, 4072, 0, 89, 6, 1, 15, 0, 4008, 4070, 3994, 5, 4038, 18, 4096, 0, 4094, 6, 151] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1104, 4079, 0, 50, 9, 122, 4096, 0, 155, 0, 0, 4037, 0, 0, 0, 0, 0, 4096, 4060, 0, 0, 266, 0, 0, 22, 0, 180, 0, 0, 4096, 0, 2038, 0, 0, 0, 19, 0, 10, 0, 3221, 0, 0, 2, 0, 60, 3731, 985, 4073, 0, 0, 0, 12, 0, 4095, 0, 2, 0, 4096, 0, 3, 0, 3074, 1, 0, 4058, 0, 1425, 0, 4090, 4095, 4089, 46, 0, 0, 0, 0, 0, 0, 2, 0, 10, 4096, 4096, 4096, 0, 4094, 0, 0, 4093, 0, 3, 16, 0, 0, 4094, 4096, 0, 1, 0, 0, 0, 1, 5, 0, 0, 16, 0, 4069, 0, 278, 15, 0, 0, 0, 4044, 4095, 4043, 0, 3923, 0, 4095, 0, 4096, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 8, 4090, 907, 4074, 1, 0, 1, 1276, 4094, 0, 234, 0, 7, 3709, 0, 1, 0, 8, 0, 4086, 4090, 20, 7, 76, 0, 0, 0, 0, 1121, 0, 0, 4090, 0, 2014, 0, 73, 4, 2, 0, 1, 0, 3570, 7, 0, 0, 0, 0, 2665, 2725, 4082, 0, 0, 27, 20, 15, 4096, 0, 0, 2, 4075, 0, 0, 0, 1823, 0, 0, 4061, 6, 1310, 0, 4060, 3761, 4056, 1, 1, 1, 0, 0, 0, 4, 0, 22, 8, 4096, 4090, 4093, 6, 4072, 0, 0, 3925, 0, 0, 0, 0, 4, 4096, 4080, 18, 2, 0, 3, 0, 3, 8, 6, 10, 21, 0, 3972, 0, 17, 10, 0, 0, 0, 4059, 4086, 3634, 0, 4025, 0, 4084, 0, 4080, 0, 180] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4095, 950, 4095, 0, 0, 0, 94, 4095, 0, 562, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 307, 0, 0, 4096, 0, 957, 0, 0, 2, 0, 0, 0, 0, 3196, 47, 0, 0, 0, 3, 3922, 257, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3863, 4, 0, 4059, 0, 1712, 0, 4096, 3955, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4096, 4096, 4096, 0, 4075, 0, 0, 4068, 0, 3, 76, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 4089, 0, 18, 0, 0, 0, 0, 4094, 4096, 3991, 0, 4095, 0, 4096, 0, 4096, 0, 746] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4075, 43, 4091, 40, 6, 1, 3767, 4063, 0, 378, 0, 59, 2329, 0, 0, 36, 2, 0, 4010, 4066, 95, 16, 0, 0, 0, 1, 0, 84, 0, 0, 3963, 11, 2494, 2, 11, 4, 1, 0, 0, 0, 3422, 148, 28, 25, 0, 9, 1271, 47, 4082, 0, 0, 0, 7, 0, 4074, 0, 3, 3, 4092, 0, 18, 0, 2383, 1, 1, 4048, 0, 2386, 2, 4096, 3932, 4058, 136, 0, 0, 132, 0, 0, 0, 0, 0, 23, 4072, 4096, 4072, 1, 3665, 0, 0, 4068, 0, 0, 1367, 61, 10, 4061, 4096, 0, 45, 0, 0, 1, 0, 45, 0, 1, 0, 0, 4092, 0, 65, 1, 1, 0, 0, 4090, 4087, 3973, 2, 3630, 0, 4074, 5, 4090, 0, 1226] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 4092, 153, 4093, 0, 0, 32, 1649, 4093, 0, 19, 0, 0, 4028, 0, 0, 0, 0, 0, 4090, 4075, 1, 0, 12, 0, 2, 0, 0, 3057, 0, 0, 4095, 0, 1715, 1, 0, 104, 0, 0, 0, 57, 1898, 10, 1, 4, 0, 24, 3465, 486, 4093, 0, 1, 0, 19, 2, 4095, 0, 0, 0, 4087, 0, 0, 0, 3214, 0, 0, 4068, 0, 2242, 0, 4026, 3681, 4095, 2, 0, 0, 2, 2, 0, 2, 0, 0, 4, 4096, 4094, 4096, 0, 3922, 0, 2, 4052, 0, 2, 2, 0, 2, 4096, 4095, 0, 7, 0, 0, 0, 0, 3, 0, 0, 1, 0, 4095, 0, 0, 1, 0, 0, 1, 2952, 4095, 4029, 0, 4067, 1, 4095, 1, 4091, 0, 282] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4093, 64, 4096, 2, 7, 2, 78, 4088, 0, 2473, 0, 1, 3767, 0, 0, 0, 1, 0, 4096, 4093, 0, 0, 18, 0, 0, 0, 0, 2231, 9, 0, 4080, 0, 1449, 0, 2, 0, 0, 1, 0, 87, 3413, 0, 0, 1, 0, 0, 1393, 1908, 4083, 0, 3, 1, 128, 11, 4096, 0, 0, 23, 4095, 0, 0, 8, 3484, 1, 0, 4062, 1, 30, 0, 4096, 3956, 4096, 3, 0, 0, 1, 3, 0, 0, 64, 6, 19, 4096, 4095, 4066, 5, 4049, 0, 0, 3978, 0, 0, 0, 14, 0, 4095, 4089, 0, 39, 2, 1, 0, 0, 13, 0, 0, 42, 0, 4080, 1, 246, 41, 1, 0, 0, 3970, 4089, 4090, 4, 4066, 9, 4084, 0, 4069, 0, 15] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1673, 1840, 4095, 0, 0, 6, 622, 4093, 0, 2331, 1, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 4092, 4094, 0, 0, 94, 1, 0, 0, 1, 734, 0, 0, 4095, 2619, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1944, 1, 0, 11, 0, 0, 2, 0, 561, 5, 0, 0, 2, 20, 4094, 3727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3688, 1813, 4092, 0, 0, 0, 0, 0, 3926, 0, 0, 0, 4094, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 879, 0, 0, 4089, 4, 2110, 0, 4071, 4073, 43, 1, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 4096, 0, 4037, 0, 3990, 0, 0, 4086, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 4096, 0, 0, 0, 0, 0, 3, 6, 0, 2, 6, 0, 4095, 0, 2422, 1433, 170, 58, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 0, 0, 3942, 0, 4027, 0, 4095, 0, 0, 0, 4086, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18664, 16173, 14458, 17617, 15273, 16252, 16394, 16241] avg=16384.0000 max=18664 max_rank=0 min=14458 min_rank=2 imbalance_ratio=1.139160 max_load_reduction=112408 max_load_reduction_pct=0.857605 moved_to_echo=64160 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 1671, 907, 4074, 1, 0, 1, 1276, 4094, 0, 234, 0, 7, 3709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 0, 4086, 4090, 20, 7, 76, 0, 0, 0, 0, 1121, 0, 0, 4086, 2595, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 2014, 0, 73, 4, 2, 0, 1, 0, 3570, 7, 0, 0, 0, 0, 4090, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2665, 2725, 4082, 0, 0, 27, 20, 15, 3926, 0, 0, 2, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1823, 0, 0, 4061, 6, 1310, 0, 4060, 3761, 42, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 22, 8, 4096, 0, 4035, 6, 4072, 0, 0, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4080, 18, 2, 0, 3, 0, 3, 8, 6, 10, 21, 0, 3972, 0, 2419, 1419, 170, 58, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 10, 0, 0, 0, 4059, 0, 3634, 0, 4025, 0, 0, 0, 4072, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15982, 16458, 17567, 17537, 15066, 16168, 16297, 15997] avg=16384.0000 max=17567 max_rank=2 min=15066 min_rank=4 imbalance_ratio=1.072205 max_load_reduction=113505 max_load_reduction_pct=0.865974 moved_to_echo=63528 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1674, 1104, 4079, 0, 50, 9, 122, 4096, 0, 155, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4060, 0, 0, 266, 0, 0, 22, 0, 180, 0, 0, 4095, 2616, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2038, 0, 0, 0, 19, 0, 10, 0, 3221, 0, 0, 2, 0, 60, 4096, 3726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3731, 985, 4073, 0, 0, 0, 12, 0, 3925, 0, 2, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3074, 1, 0, 4058, 0, 1425, 0, 4090, 4095, 42, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 10, 4096, 0, 4038, 0, 4094, 0, 0, 4093, 0, 3, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 1, 0, 0, 0, 1, 5, 0, 0, 16, 0, 4069, 0, 2422, 1431, 170, 58, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 15, 0, 0, 0, 4044, 0, 4043, 0, 3923, 0, 0, 0, 4088, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15326, 15704, 17268, 16827, 16831, 16352, 16371, 16393] avg=16384.0000 max=17268 max_rank=2 min=15326 min_rank=0 imbalance_ratio=1.053955 max_load_reduction=113804 max_load_reduction_pct=0.868256 moved_to_echo=65947 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1671, 950, 4095, 0, 0, 0, 94, 4095, 0, 562, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 307, 0, 0, 4096, 2624, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 957, 0, 0, 2, 0, 0, 0, 0, 3196, 47, 0, 0, 0, 3, 4096, 3726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3922, 257, 4096, 0, 0, 0, 0, 0, 3921, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3863, 4, 0, 4059, 0, 1712, 0, 4096, 3955, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4096, 0, 4035, 0, 4075, 0, 0, 4068, 0, 3, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 4089, 0, 2424, 1438, 175, 61, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 4094, 0, 3991, 0, 4095, 0, 0, 0, 4085, 0, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15562, 15590, 16123, 16292, 17723, 16356, 16397, 17029] avg=16384.0000 max=17723 max_rank=4 min=15562 min_rank=0 imbalance_ratio=1.081726 max_load_reduction=113349 max_load_reduction_pct=0.864784 moved_to_echo=67505 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1673, 64, 4096, 2, 7, 2, 78, 4088, 0, 2473, 0, 1, 3767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4093, 0, 0, 18, 0, 0, 0, 0, 2231, 9, 0, 4089, 2620, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 1449, 0, 2, 0, 0, 1, 0, 87, 3413, 0, 0, 1, 0, 0, 4095, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1393, 1908, 4083, 0, 3, 1, 128, 11, 3926, 0, 0, 23, 4095, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3484, 1, 0, 4062, 1, 30, 0, 4096, 3956, 43, 3, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 6, 19, 4096, 0, 4009, 5, 4049, 0, 0, 3978, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4089, 0, 39, 2, 1, 0, 0, 13, 0, 0, 42, 0, 4080, 1, 2420, 1433, 170, 57, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 41, 1, 0, 0, 3970, 0, 4090, 4, 4066, 9, 0, 0, 4061, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16251, 17525, 16844, 15579, 15680, 16240, 16450, 16503] avg=16384.0000 max=17525 max_rank=1 min=15579 min_rank=3 imbalance_ratio=1.069641 max_load_reduction=113547 max_load_reduction_pct=0.866295 moved_to_echo=64873 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1665, 43, 4091, 40, 6, 1, 3767, 4063, 0, 378, 0, 59, 2329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 2, 0, 4010, 4066, 95, 16, 0, 0, 0, 1, 0, 84, 0, 0, 4087, 2596, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 11, 2494, 2, 11, 4, 1, 0, 0, 0, 3422, 148, 28, 25, 0, 9, 4096, 3706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1271, 47, 4082, 0, 0, 0, 7, 0, 3905, 0, 3, 3, 4092, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2383, 1, 1, 4048, 0, 2386, 2, 4096, 3932, 42, 136, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4072, 0, 4014, 1, 3665, 0, 0, 4068, 0, 0, 1367, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4061, 4096, 0, 45, 0, 0, 1, 0, 45, 0, 1, 0, 0, 4092, 0, 2410, 1420, 169, 58, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 1, 1, 0, 0, 4090, 0, 3973, 2, 3630, 0, 0, 5, 4082, 0, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16442, 15361, 17920, 13428, 17159, 17271, 16416, 17075] avg=16384.0000 max=17920 max_rank=2 min=13428 min_rank=3 imbalance_ratio=1.093750 max_load_reduction=113152 max_load_reduction_pct=0.863281 moved_to_echo=67921 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1672, 153, 4093, 0, 0, 32, 1649, 4093, 0, 19, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4075, 1, 0, 12, 0, 2, 0, 0, 3057, 0, 0, 4095, 2619, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1715, 1, 0, 104, 0, 0, 0, 57, 1898, 10, 1, 4, 0, 24, 4094, 3726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3465, 486, 4093, 0, 1, 0, 19, 2, 3925, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3214, 0, 0, 4068, 0, 2242, 0, 4026, 3681, 43, 2, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 4096, 0, 4038, 0, 3922, 0, 2, 4052, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4095, 0, 7, 0, 0, 0, 0, 3, 0, 0, 1, 0, 4095, 0, 2420, 1433, 170, 58, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 2952, 0, 4029, 0, 4067, 1, 0, 1, 4083, 0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15740, 18320, 15729, 16078, 17280, 16120, 16388, 15417] avg=16384.0000 max=18320 max_rank=1 min=15417 min_rank=7 imbalance_ratio=1.118164 max_load_reduction=112752 max_load_reduction_pct=0.860229 moved_to_echo=65205 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1673, 2688, 4071, 23, 0, 0, 65, 3972, 0, 750, 0, 13, 3843, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 58, 5, 4055, 4091, 0, 16, 115, 1, 4, 6, 0, 525, 0, 0, 4070, 2603, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 16, 711, 0, 82, 11, 0, 0, 40, 2, 2400, 0, 7, 4, 0, 0, 4076, 3726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3223, 2334, 4060, 12, 5, 0, 2, 55, 3920, 0, 0, 0, 4095, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3325, 162, 1, 3815, 0, 762, 0, 4051, 3734, 42, 0, 0, 0, 31, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 2, 24, 4048, 0, 4037, 35, 4046, 0, 2, 3887, 1, 0, 158, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4093, 0, 15, 0, 0, 0, 1, 0, 0, 0, 0, 6, 4072, 0, 2422, 1424, 170, 58, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 6, 1, 15, 0, 4008, 0, 3994, 5, 4038, 18, 0, 0, 4086, 6, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17105, 15941, 15163, 17714, 16060, 16313, 16359, 16417] avg=16384.0000 max=17714 max_rank=3 min=15163 min_rank=2 imbalance_ratio=1.081177 max_load_reduction=113358 max_load_reduction_pct=0.864853 moved_to_echo=65149 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32640, 128, 13440, 128, 256, 256, 128, 128, 128, 256, 21760, 256, 128, 128, 128, 128, 32768, 29824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[768, 128, 128, 128, 128, 31232, 0, 31872, 128, 32000, 128, 0, 128, 32768, 128, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[22144, 256, 128, 32384, 128, 12032, 128, 32640, 31232, 384, 256, 128, 128, 256, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[23424, 10624, 32768, 128, 128, 128, 256, 128, 31488, 0, 128, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 13440, 7808, 32768, 128, 128, 128, 7680, 32640, 0, 6912, 128, 128, 29824, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 256, 128, 128, 32640, 32768, 128, 128, 640, 128, 128, 128, 128, 8320, 128, 0, 32768, 20992, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32640, 128, 19456, 11520, 1408, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 32256, 128, 32000, 0, 128, 32256, 128, 128, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3631, 0, 0, 4096, 0, 0, 1, 0, 4092, 0, 1, 4094, 4092, 0, 0, 221, 1857, 4095, 0, 4000, 489, 3392, 0, 1, 0, 4095, 0, 0, 3896, 1852, 0, 4096, 1, 4095, 3, 1, 1, 0, 3, 4095, 1, 0, 0, 0, 4095, 0, 2, 0, 2, 2598, 0, 5, 0, 1, 0, 0, 4096, 0, 0, 5, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 4092, 0, 1, 4093, 0, 4, 77, 4, 0, 4096, 0, 2, 1, 0, 26, 213, 0, 0, 4096, 1, 4092, 2973, 0, 174, 4095, 1, 0, 0, 4096, 1, 0, 0, 4095, 1993, 0, 4096, 1, 4, 4095, 0, 1256, 4093, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4080, 0, 0, 4096, 13, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 24, 2443, 4096, 0, 4096, 103, 3690, 0, 0, 0, 4096, 0, 0, 3984, 986, 0, 4091, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 1669, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 4096, 0, 0, 445, 0, 0, 4096, 0, 0, 0, 0, 26, 55, 0, 0, 4096, 0, 4082, 2348, 0, 775, 4087, 0, 0, 0, 4088, 0, 270, 0, 4096, 3393, 0, 4096, 0, 10, 4095, 0, 294, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[3845, 0, 0, 4096, 20, 0, 0, 0, 4096, 0, 0, 4088, 4094, 0, 0, 63, 2932, 4084, 0, 4071, 3, 4018, 0, 0, 0, 4096, 0, 0, 2738, 648, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 2399, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 4096, 0, 1, 0, 1, 0, 4096, 0, 0, 0, 0, 13, 15, 0, 0, 4096, 0, 4096, 4058, 0, 72, 4096, 0, 0, 0, 4096, 0, 50, 0, 4096, 3642, 0, 4096, 0, 9, 4078, 0, 97, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 8, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[3878, 0, 0, 4096, 0, 0, 0, 2, 4096, 3, 0, 4075, 4072, 9, 0, 1, 1650, 4088, 0, 3898, 68, 4048, 0, 0, 0, 4096, 0, 0, 4056, 2135, 16, 4094, 3, 4071, 14, 0, 19, 33, 0, 4095, 0, 0, 1, 0, 4081, 0, 15, 0, 0, 1332, 0, 23, 5, 0, 0, 0, 4086, 0, 26, 0, 4066, 4, 4080, 0, 1, 0, 1, 0, 0, 8, 9, 2, 4096, 11, 4028, 61, 0, 4093, 0, 0, 30, 0, 0, 4086, 0, 28, 24, 0, 75, 109, 0, 0, 4086, 0, 4038, 2854, 9, 1158, 4095, 9, 0, 8, 3968, 2, 160, 4, 4096, 3140, 0, 4086, 0, 83, 4024, 7, 159, 4083, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4090, 0, 0, 4096, 117, 0, 0, 0, 4091, 0, 0, 4096, 4051, 0, 0, 1339, 2377, 4095, 0, 4084, 113, 2359, 0, 0, 0, 4092, 0, 0, 3685, 827, 1, 4079, 0, 4096, 35, 3, 0, 0, 0, 4096, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 3727, 0, 1, 1, 1, 0, 0, 4096, 0, 0, 5, 4095, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 8, 4096, 0, 1, 4096, 0, 0, 8, 0, 0, 4096, 1, 4, 0, 0, 184, 0, 0, 0, 4093, 0, 4085, 3091, 0, 153, 4093, 0, 2, 0, 4087, 0, 87, 0, 4092, 2235, 0, 4096, 0, 0, 4073, 0, 242, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4096, 1, 2, 4095, 199, 0, 0, 1, 4093, 1, 0, 4095, 4090, 0, 0, 1570, 1744, 4094, 3, 3962, 10, 1651, 0, 0, 0, 4096, 0, 0, 4003, 154, 0, 4086, 0, 4096, 2, 6, 0, 0, 0, 4093, 0, 3, 2, 0, 4096, 0, 0, 0, 0, 2919, 0, 0, 0, 0, 0, 3, 4094, 1, 0, 0, 4095, 0, 4092, 0, 3, 2, 2, 1, 2, 0, 0, 0, 4093, 3, 4091, 0, 4, 4096, 1, 3, 55, 0, 0, 4093, 0, 0, 0, 0, 32, 3, 0, 0, 4087, 0, 4091, 2131, 0, 65, 4090, 0, 1, 0, 4074, 0, 1981, 0, 4096, 2552, 0, 4096, 2, 8, 4091, 0, 1548, 4095, 0, 1, 0, 0, 0, 2, 1, 0, 28, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4064, 0, 1, 4096, 493, 0, 0, 2, 4067, 0, 0, 4095, 4031, 0, 0, 0, 591, 4095, 0, 3202, 124, 3865, 0, 0, 8, 4096, 0, 0, 3631, 1954, 1, 3644, 0, 4096, 416, 4, 0, 1, 0, 4095, 0, 0, 17, 0, 4070, 0, 0, 0, 0, 896, 0, 3, 0, 0, 0, 0, 4096, 16, 0, 0, 4070, 0, 4074, 0, 4, 0, 0, 1, 0, 1, 0, 15, 4096, 22, 4079, 1, 0, 4094, 2, 0, 0, 0, 0, 4094, 2, 0, 32, 2, 319, 8, 0, 20, 4013, 0, 4084, 1775, 0, 2615, 4083, 0, 0, 0, 4096, 0, 0, 0, 4096, 3883, 0, 4093, 8, 0, 4039, 0, 1430, 4096, 0, 0, 0, 0, 0, 0, 0, 13, 1, 0, 41, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[3582, 0, 2, 4094, 1, 0, 0, 0, 4096, 0, 0, 4093, 4095, 0, 0, 10, 2897, 4096, 1, 3994, 160, 3568, 0, 0, 1, 4086, 0, 0, 3807, 355, 2, 4066, 4, 4094, 6, 7, 2, 0, 0, 4093, 0, 0, 8, 1, 4096, 0, 0, 0, 0, 371, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 3, 0, 4096, 6, 4092, 0, 0, 4096, 7, 0, 91, 4, 0, 4095, 1, 3, 5, 0, 1, 211, 0, 0, 4095, 0, 4096, 3623, 0, 2486, 4096, 0, 0, 0, 4006, 0, 150, 0, 4096, 1483, 0, 4095, 0, 53, 4096, 0, 1924, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3631, 0, 0, 0, 0, 0, 1, 0, 4092, 0, 1, 3798, 4092, 0, 0, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1857, 0, 0, 4000, 489, 3392, 0, 1, 0, 0, 0, 0, 3896, 1852, 0, 2143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 3, 1, 1, 0, 3, 4095, 1, 0, 0, 0, 4095, 0, 2, 0, 3160, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2598, 0, 5, 0, 1, 0, 0, 4096, 0, 0, 5, 4096, 0, 4096, 1, 1237, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 4092, 0, 1, 4093, 0, 4, 3228, 859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 4, 0, 4096, 0, 2, 1, 0, 26, 213, 0, 0, 4096, 1, 4092, 2973, 716, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 4095, 1, 0, 0, 4096, 1, 0, 0, 44, 1993, 0, 4096, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1256, 4093, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0, 4096, 3236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15836, 17630, 16324, 17028, 16373, 16593, 14505, 16783] avg=16384.0000 max=17630 max_rank=1 min=14505 min_rank=6 imbalance_ratio=1.076050 max_load_reduction=113442 max_load_reduction_pct=0.865494 moved_to_echo=64254 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3878, 0, 0, 0, 0, 0, 0, 2, 4096, 3, 0, 3780, 4072, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 0, 0, 3898, 68, 4048, 0, 0, 0, 0, 0, 0, 4056, 2135, 16, 2143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4071, 14, 0, 19, 33, 0, 4095, 0, 0, 1, 0, 4081, 0, 15, 0, 3161, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1332, 0, 23, 5, 0, 0, 0, 4086, 0, 26, 0, 4066, 4, 4080, 0, 1236, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8, 9, 2, 4096, 11, 4028, 61, 0, 4093, 0, 0, 3221, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 4086, 0, 28, 24, 0, 75, 109, 0, 0, 4086, 0, 4038, 2854, 715, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1158, 4095, 9, 0, 8, 3968, 2, 160, 4, 44, 3140, 0, 4086, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 7, 159, 4083, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15841, 18014, 16360, 15749, 16391, 16340, 16766, 15611] avg=16384.0000 max=18014 max_rank=1 min=15611 min_rank=7 imbalance_ratio=1.099487 max_load_reduction=113058 max_load_reduction_pct=0.862564 moved_to_echo=65108 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4080, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 3796, 4096, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2443, 0, 0, 4096, 103, 3690, 0, 0, 0, 0, 0, 0, 3984, 986, 0, 2132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 3167, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1669, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 1240, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 4096, 0, 0, 3228, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, 0, 4096, 0, 0, 0, 0, 26, 55, 0, 0, 4096, 0, 4082, 2348, 719, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 775, 4087, 0, 0, 0, 4088, 0, 270, 0, 32, 3393, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 294, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 3236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16105, 17434, 16325, 16094, 16378, 16167, 16751, 15818] avg=16384.0000 max=17434 max_rank=1 min=15818 min_rank=7 imbalance_ratio=1.064087 max_load_reduction=113638 max_load_reduction_pct=0.866989 moved_to_echo=65114 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 117, 0, 0, 0, 4091, 0, 0, 3800, 4051, 0, 0, 1339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2377, 0, 0, 4084, 113, 2359, 0, 0, 0, 0, 0, 0, 3685, 827, 1, 2134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 35, 3, 0, 0, 0, 4096, 0, 0, 3, 0, 4096, 0, 0, 0, 3158, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3727, 0, 1, 1, 1, 0, 0, 4096, 0, 0, 5, 4095, 0, 4091, 0, 1232, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 8, 4096, 0, 1, 4096, 0, 0, 3227, 859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 1, 4, 0, 0, 184, 0, 0, 0, 4093, 0, 4085, 3091, 713, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 4093, 0, 2, 0, 4087, 0, 87, 0, 44, 2235, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 242, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 9, 4096, 3233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17488, 15580, 16355, 18139, 16384, 16571, 14797, 15758] avg=16384.0000 max=18139 max_rank=3 min=14797 min_rank=6 imbalance_ratio=1.107117 max_load_reduction=112933 max_load_reduction_pct=0.861610 moved_to_echo=63510 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3845, 0, 0, 0, 20, 0, 0, 0, 4096, 0, 0, 3792, 4094, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2932, 0, 0, 4071, 3, 4018, 0, 0, 0, 0, 0, 0, 2738, 648, 0, 2143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3161, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2399, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 0, 4096, 0, 1236, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 4096, 0, 1, 3218, 859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 13, 15, 0, 0, 4096, 0, 4096, 4058, 715, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 4096, 0, 0, 0, 4096, 0, 50, 0, 44, 3642, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 97, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 8, 0, 0, 1, 4096, 3237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15910, 16553, 16315, 16817, 16370, 17386, 16105, 15616] avg=16384.0000 max=17386 max_rank=5 min=15616 min_rank=7 imbalance_ratio=1.061157 max_load_reduction=113686 max_load_reduction_pct=0.867355 moved_to_echo=65477 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3582, 0, 2, 0, 1, 0, 0, 0, 4096, 0, 0, 3797, 4095, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2897, 0, 1, 3994, 160, 3568, 0, 0, 1, 0, 0, 0, 3807, 355, 2, 2128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 6, 7, 2, 0, 0, 4093, 0, 0, 8, 1, 4096, 0, 0, 0, 3161, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4089, 0, 4096, 0, 1228, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 4096, 6, 4092, 0, 0, 4096, 7, 0, 3229, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 4, 0, 4095, 1, 3, 5, 0, 1, 211, 0, 0, 4095, 0, 4096, 3623, 710, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2486, 4096, 0, 0, 0, 4006, 0, 150, 0, 44, 1483, 0, 4095, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1924, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15583, 16913, 16339, 14770, 16387, 17231, 16413, 17436] avg=16384.0000 max=17436 max_rank=7 min=14770 min_rank=3 imbalance_ratio=1.064209 max_load_reduction=113636 max_load_reduction_pct=0.866974 moved_to_echo=67467 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4064, 0, 1, 0, 493, 0, 0, 2, 4067, 0, 0, 3799, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 591, 0, 0, 3202, 124, 3865, 0, 0, 8, 0, 0, 0, 3631, 1954, 1, 1907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 416, 4, 0, 1, 0, 4095, 0, 0, 17, 0, 4070, 0, 0, 0, 3161, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 896, 0, 3, 0, 0, 0, 0, 4096, 16, 0, 0, 4070, 0, 4074, 0, 1100, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 1, 0, 15, 4096, 22, 4079, 1, 0, 4094, 2, 0, 3227, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 32, 2, 319, 8, 0, 20, 4013, 0, 4084, 1775, 637, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2615, 4083, 0, 0, 0, 4096, 0, 0, 0, 44, 3883, 0, 4093, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 1430, 4096, 0, 0, 0, 0, 0, 0, 0, 13, 1, 0, 41, 0, 4096, 3236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16457, 15283, 16728, 15146, 16402, 15282, 18822, 16952] avg=16384.0000 max=18822 max_rank=6 min=15146 min_rank=3 imbalance_ratio=1.148804 max_load_reduction=112250 max_load_reduction_pct=0.856400 moved_to_echo=67458 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 1, 2, 0, 199, 0, 0, 1, 4093, 1, 0, 3799, 4090, 0, 0, 1570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1744, 0, 3, 3962, 10, 1651, 0, 0, 0, 0, 0, 0, 4003, 154, 0, 2138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 6, 0, 0, 0, 4093, 0, 3, 2, 0, 4096, 0, 0, 0, 3161, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2919, 0, 0, 0, 0, 0, 3, 4094, 1, 0, 0, 4095, 0, 4092, 0, 1234, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 2, 1, 2, 0, 0, 0, 4093, 3, 4091, 0, 4, 4096, 1, 3, 3227, 859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 4093, 0, 0, 0, 0, 32, 3, 0, 0, 4087, 0, 4091, 2131, 714, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4090, 0, 1, 0, 4074, 0, 1981, 0, 44, 2552, 0, 4096, 2, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1548, 4095, 0, 1, 0, 0, 0, 2, 1, 0, 28, 0, 0, 0, 4095, 3237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17852, 13665, 16326, 17329, 16387, 15502, 16913, 17098] avg=16384.0000 max=17852 max_rank=0 min=13665 min_rank=1 imbalance_ratio=1.089600 max_load_reduction=113220 max_load_reduction_pct=0.863800 moved_to_echo=65900 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[16512, 0, 128, 31360, 1152, 26624, 0, 128, 128, 0, 0, 0, 29824, 8960, 128, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32640, 128, 7040, 32768, 0, 128, 128, 128, 0, 128, 128, 128, 128, 0, 128, 128, 32768, 25984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 7552, 32768, 128, 128, 128, 32512, 128, 2816, 128, 384, 22400, 0, 32768, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[31360, 128, 128, 0, 896, 0, 128, 128, 32768, 128, 128, 30464, 32640, 128, 0, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 512, 128, 128, 128, 128, 32768, 128, 128, 128, 128, 32768, 0, 128, 0, 25344, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 32768, 128, 128, 32768, 128, 128, 25856, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 16000, 0, 128, 128, 128, 0, 128, 32768, 128, 128, 128, 32768, 128, 32768, 128, 9856, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[768, 128, 0, 32768, 128, 128, 128, 128, 768, 640, 0, 128, 32768, 128, 32768, 22912, 5760, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4092, 2, 0, 0, 444, 1, 4093, 1786, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4, 1, 0, 0, 0, 0, 0, 1, 4092, 0, 4093, 4093, 2, 0, 4095, 0, 1, 0, 0, 0, 0, 4095, 2, 2305, 0, 4087, 1, 4095, 0, 1, 4096, 1, 0, 3, 0, 15, 0, 0, 4096, 0, 0, 0, 1, 4094, 5, 2, 0, 407, 4093, 3, 2, 0, 1, 334, 0, 0, 3075, 0, 0, 4053, 4095, 2, 0, 0, 4095, 4095, 1, 0, 0, 0, 4095, 1, 0, 1, 332, 4079, 0, 4063, 1, 3827, 3965, 4095, 0, 4095, 0, 0, 4095, 0, 4093, 0, 0, 1, 1, 0, 0, 4096, 0, 0, 1, 5, 0, 0, 0, 0, 1, 4071, 0, 0, 8, 4094, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 27, 0, 4096, 1291, 0, 4, 50, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 4096, 0, 4075, 4092, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 1161, 3, 4090, 0, 4092, 0, 22, 4074, 1, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 8, 4096, 22, 8, 0, 3428, 4096, 0, 0, 0, 0, 174, 0, 0, 2860, 0, 0, 4026, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3998, 0, 1, 0, 494, 4095, 0, 4087, 0, 4079, 4055, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 36, 0, 7, 0, 0, 0, 2943, 0, 0, 1, 4096, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 1962, 0, 4092, 30, 4, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 4094, 4095, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 4096, 0, 1376, 2, 4011, 4, 4094, 0, 0, 4088, 0, 0, 0, 1, 0, 1, 0, 4094, 0, 0, 0, 0, 4096, 2, 0, 0, 695, 4063, 0, 0, 0, 0, 1651, 0, 0, 3528, 0, 0, 3906, 4092, 0, 0, 4, 4096, 4096, 0, 4, 5, 0, 3858, 0, 0, 0, 169, 4096, 0, 4062, 0, 3952, 3629, 4094, 0, 4096, 0, 0, 4096, 2, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4090, 0, 0, 2, 4095, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4056, 43, 2, 0, 347, 0, 4069, 191, 10, 0, 94, 0, 12, 0, 0, 1, 0, 10, 0, 107, 0, 0, 0, 0, 0, 4079, 0, 4084, 4087, 0, 0, 3948, 0, 0, 9, 0, 7, 0, 4093, 35, 2150, 0, 3591, 0, 4096, 0, 0, 4047, 0, 0, 0, 0, 0, 5, 0, 4081, 0, 0, 0, 148, 4081, 0, 0, 21, 2754, 4096, 15, 15, 2, 9, 35, 0, 1, 3999, 0, 5, 4091, 4096, 0, 0, 6, 4095, 4087, 1, 1, 0, 0, 3636, 0, 1, 0, 71, 4095, 2, 4015, 0, 4080, 4086, 4083, 2, 4084, 15, 0, 4093, 12, 4096, 12, 14, 0, 0, 0, 0, 4090, 0, 4, 0, 0, 6, 0, 0, 0, 0, 3639, 0, 0, 0, 4096, 4028] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 123, 0, 4096, 1911, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 4096, 0, 1052, 0, 4009, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 5, 0, 0, 634, 4096, 0, 0, 0, 0, 636, 0, 0, 4067, 0, 0, 4094, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 32, 4095, 0, 4061, 0, 4089, 4095, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4070, 5, 0, 0, 617, 0, 4087, 3842, 0, 0, 383, 0, 20, 0, 0, 0, 0, 1, 0, 15, 0, 1, 2, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4095, 391, 0, 0, 0, 0, 0, 4096, 0, 2051, 0, 4095, 1, 4078, 0, 1, 4096, 0, 0, 13, 0, 0, 0, 0, 4095, 9, 0, 0, 126, 4096, 31, 0, 0, 102, 3391, 0, 1, 0, 0, 220, 0, 1, 3938, 0, 0, 3979, 4084, 0, 0, 10, 4074, 4096, 0, 0, 0, 0, 3997, 0, 0, 0, 1136, 4062, 0, 4096, 0, 3250, 2593, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 12, 0, 0, 0, 0, 4, 4093, 0, 0, 0, 1, 0, 0, 0, 151, 0, 2785, 0, 0, 0, 4096, 3912] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 194, 0, 4096, 591, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4055, 6, 0, 0, 0, 0, 0, 4096, 3, 672, 0, 4092, 0, 4089, 0, 4, 4096, 0, 0, 0, 0, 25, 0, 0, 4096, 2, 0, 0, 71, 4096, 13, 0, 0, 3375, 4096, 0, 0, 0, 0, 163, 0, 0, 3304, 0, 0, 4075, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4068, 0, 0, 0, 40, 4096, 0, 4078, 0, 4058, 4081, 4096, 0, 4096, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4096, 2, 0, 0, 510, 0, 4096, 161, 1, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 4096, 1, 4096, 4088, 0, 0, 4068, 24, 0, 0, 0, 0, 0, 4096, 0, 2565, 0, 3931, 0, 4096, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 4096, 29, 0, 0, 21, 4096, 5, 5, 0, 958, 4077, 0, 0, 0, 0, 81, 0, 0, 4078, 0, 0, 4088, 4095, 2, 0, 13, 4096, 4096, 1, 17, 0, 0, 4002, 0, 20, 0, 141, 4094, 0, 4091, 0, 4062, 4035, 4095, 0, 4095, 1, 0, 4096, 0, 4093, 1, 0, 0, 0, 0, 44, 4094, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4018, 0, 0, 1, 4096, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4092, 2, 0, 0, 444, 1, 4093, 1786, 0, 0, 1, 0, 0, 0, 0, 1, 3654, 1605, 1039, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 1, 4092, 0, 4093, 4093, 2, 0, 4095, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2490, 2, 2305, 0, 4087, 1, 4095, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 15, 0, 0, 4096, 0, 0, 0, 1, 4094, 5, 2, 0, 4095, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 4093, 3, 2, 0, 1, 334, 0, 0, 3075, 0, 0, 4053, 2777, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 0, 1, 0, 0, 0, 4095, 1, 0, 1, 332, 4079, 0, 4063, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3827, 3965, 4095, 0, 4095, 0, 0, 0, 0, 439, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 5, 0, 0, 0, 0, 1, 4071, 0, 0, 8, 4094, 4095, 279, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16787, 16395, 17078, 16338, 14747, 16521, 16423, 16783] avg=16384.0000 max=17078 max_rank=2 min=14747 min_rank=4 imbalance_ratio=1.042358 max_load_reduction=113994 max_load_reduction_pct=0.869705 moved_to_echo=64474 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 27, 0, 4096, 1291, 0, 4, 50, 0, 0, 0, 0, 0, 3657, 1605, 1039, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 4096, 0, 4075, 4092, 0, 0, 4095, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2491, 0, 1161, 3, 4090, 0, 4092, 0, 22, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 8, 4096, 22, 8, 0, 4096, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3428, 4096, 0, 0, 0, 0, 174, 0, 0, 2860, 0, 0, 4026, 2777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 0, 0, 0, 0, 3998, 0, 1, 0, 494, 4095, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4055, 4096, 0, 4096, 0, 0, 0, 0, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 36, 0, 7, 0, 0, 0, 2943, 0, 0, 1, 4096, 4095, 280, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15934, 16411, 15933, 16357, 17361, 16624, 16765, 15687] avg=16384.0000 max=17361 max_rank=4 min=15687 min_rank=7 imbalance_ratio=1.059631 max_load_reduction=113711 max_load_reduction_pct=0.867546 moved_to_echo=66437 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4056, 43, 2, 0, 347, 0, 4069, 191, 10, 0, 94, 0, 12, 0, 0, 1, 3657, 1604, 1039, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 107, 0, 0, 0, 0, 0, 4079, 0, 4084, 4087, 0, 0, 3948, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 7, 0, 2489, 35, 2150, 0, 3591, 0, 4096, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4081, 0, 0, 0, 148, 4081, 0, 0, 21, 4093, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2754, 4096, 15, 15, 2, 9, 35, 0, 1, 3999, 0, 5, 4091, 2777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3948, 0, 1, 1, 0, 0, 3636, 0, 1, 0, 71, 4095, 2, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4086, 4083, 2, 4084, 15, 0, 0, 12, 439, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4, 0, 0, 6, 0, 0, 0, 0, 3639, 0, 0, 0, 4096, 4028, 280, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15194, 16329, 16424, 16447, 17799, 15776, 16827, 16276] avg=16384.0000 max=17799 max_rank=4 min=15194 min_rank=0 imbalance_ratio=1.086365 max_load_reduction=113273 max_load_reduction_pct=0.864204 moved_to_echo=66678 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 2, 0, 0, 510, 0, 4096, 161, 1, 3, 0, 0, 0, 0, 1, 0, 3654, 1605, 1039, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 4096, 1, 4096, 4088, 0, 0, 4068, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 2491, 0, 2565, 0, 3931, 0, 4096, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 29, 0, 0, 21, 4096, 5, 5, 0, 4096, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 958, 4077, 0, 0, 0, 0, 81, 0, 0, 4078, 0, 0, 4088, 2777, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3949, 0, 1, 17, 0, 0, 4002, 0, 20, 0, 141, 4094, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4035, 4095, 0, 4095, 1, 0, 0, 0, 439, 1, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4018, 0, 0, 1, 4096, 4087, 279, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15237, 16410, 17173, 16375, 16061, 16328, 16772, 16716] avg=16384.0000 max=17173 max_rank=2 min=15237 min_rank=0 imbalance_ratio=1.048157 max_load_reduction=113899 max_load_reduction_pct=0.868980 moved_to_echo=65877 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 123, 0, 4096, 1911, 0, 0, 0, 0, 1, 0, 0, 0, 3660, 1608, 1045, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4078, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2488, 0, 1052, 0, 4009, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 5, 0, 0, 4096, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 634, 4096, 0, 0, 0, 0, 636, 0, 0, 4067, 0, 0, 4094, 2768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 32, 4095, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4095, 4096, 0, 4096, 0, 0, 0, 0, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 0, 0, 0, 4096, 4096, 283, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16610, 16386, 15741, 16320, 16295, 16217, 16812, 16691] avg=16384.0000 max=16812 max_rank=6 min=15741 min_rank=2 imbalance_ratio=1.026123 max_load_reduction=114260 max_load_reduction_pct=0.871735 moved_to_echo=66015 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 194, 0, 4096, 591, 0, 1, 11, 0, 0, 0, 0, 0, 3655, 1605, 1039, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4055, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 2491, 3, 672, 0, 4092, 0, 4089, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 4096, 2, 0, 0, 71, 4096, 13, 0, 0, 4096, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3375, 4096, 0, 0, 0, 0, 163, 0, 0, 3304, 0, 0, 4075, 2777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 40, 4096, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 4081, 4096, 0, 4096, 0, 0, 0, 0, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 0, 0, 0, 4096, 4096, 280, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15357, 16356, 15453, 16426, 17790, 16231, 16770, 16689] avg=16384.0000 max=17790 max_rank=4 min=15357 min_rank=0 imbalance_ratio=1.085815 max_load_reduction=113282 max_load_reduction_pct=0.864273 moved_to_echo=67480 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 1962, 0, 4092, 30, 4, 0, 6, 0, 2, 0, 0, 0, 3657, 1605, 1038, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 4094, 4095, 0, 0, 4063, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2491, 0, 1376, 2, 4011, 4, 4094, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4094, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 695, 4063, 0, 0, 0, 0, 1651, 0, 0, 3528, 0, 0, 3906, 2775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3949, 0, 0, 4, 5, 0, 3858, 0, 0, 0, 169, 4096, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 3629, 4094, 0, 4096, 0, 0, 0, 2, 439, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4090, 0, 0, 2, 4095, 4084, 279, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16561, 16370, 16066, 16317, 16618, 16147, 16213, 16780] avg=16384.0000 max=16780 max_rank=7 min=16066 min_rank=2 imbalance_ratio=1.024170 max_load_reduction=114292 max_load_reduction_pct=0.871979 moved_to_echo=65758 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4070, 5, 0, 0, 617, 0, 4087, 3842, 0, 0, 383, 0, 20, 0, 0, 0, 3657, 1605, 1036, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 15, 0, 1, 2, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4095, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 0, 0, 0, 0, 0, 2491, 0, 2051, 0, 4095, 1, 4078, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4095, 9, 0, 0, 126, 4096, 31, 0, 0, 4096, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 3391, 0, 1, 0, 0, 220, 0, 1, 3938, 0, 0, 3979, 2769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3927, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 1136, 4062, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3250, 2593, 4096, 0, 4096, 0, 0, 0, 0, 439, 12, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 0, 151, 0, 2785, 0, 0, 0, 4096, 3912, 279, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19392, 16415, 17204, 16492, 14401, 17228, 14490, 15450] avg=16384.0000 max=19392 max_rank=0 min=14401 min_rank=4 imbalance_ratio=1.183594 max_load_reduction=111680 max_load_reduction_pct=0.852051 moved_to_echo=61569 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 128, 128, 4224, 128, 32768, 9856, 128, 128, 640, 0, 128, 0, 128, 128, 29312, 12928, 8320, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[512, 128, 128, 0, 128, 0, 19968, 128, 13440, 128, 32000, 128, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[12416, 32128, 128, 128, 128, 128, 3328, 0, 128, 28928, 0, 128, 32384, 22272, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 31616, 0, 128, 128, 128, 0, 31872, 128, 128, 128, 2432, 32768, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 128, 128, 256, 128, 29568, 0, 0, 128, 32768, 32512, 2304, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[31488, 30592, 32768, 128, 32768, 128, 0, 0, 128, 3584, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 256, 0, 128, 128, 0, 128, 32768, 128, 32768, 32768, 128, 0, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 128, 128, 0, 32768, 128, 0, 0, 384, 32768, 128, 128, 128, 32768, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4095, 1, 4093, 4092, 0, 0, 1, 4095, 0, 0, 0, 2, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 4095, 3234, 1, 4096, 0, 0, 0, 4095, 1, 4095, 1, 0, 0, 1, 0, 0, 0, 4095, 1, 4095, 0, 4093, 1, 2, 2, 0, 1, 1, 0, 0, 239, 0, 1157, 4096, 4095, 4095, 1, 4, 1, 0, 0, 0, 3, 0, 4095, 0, 1, 0, 4094, 0, 0, 4014, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 4095, 1, 0, 13, 0, 4083, 4061, 0, 0, 1, 0, 0, 5, 0, 3627, 1, 0, 4094, 0, 0, 1, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 47, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 4094, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4094, 0, 4096, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 2906, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 949, 0, 3166, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3369, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 4096, 0, 0, 3, 0, 4093, 4029, 0, 0, 0, 0, 0, 0, 0, 1955, 0, 0, 4096, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 1, 0, 4096, 8, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4077, 1, 4083, 4095, 0, 0, 4, 4058, 13, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 9, 4094, 0, 4085, 1183, 2, 4096, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 7, 0, 4095, 2, 4096, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 11, 160, 0, 3502, 4096, 4087, 4096, 0, 1, 4, 9, 1, 0, 0, 0, 4088, 1, 0, 15, 4081, 3, 0, 3665, 27, 7, 0, 0, 4081, 4091, 3, 0, 0, 4064, 0, 0, 15, 0, 4096, 4094, 0, 0, 5, 0, 0, 53, 25, 3618, 0, 0, 4082, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 38, 0, 4096, 0, 0, 0, 13, 0, 4096, 58, 0, 4086, 0, 4096, 21, 0, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4039, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 3607, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 983, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 5, 3806, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 8, 0, 4054, 4008, 0, 0, 0, 0, 0, 5, 0, 3789, 0, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 1, 0, 0, 48, 0, 4096, 94, 0, 0, 0, 0, 4096, 0, 0, 4073, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 3800, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 4096, 0, 4096, 1830, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 4, 318, 0, 3356, 4096, 4093, 4096, 0, 50, 0, 0, 0, 0, 0, 2, 4096, 48, 0, 0, 4096, 0, 0, 3993, 0, 0, 0, 0, 4088, 4095, 0, 0, 0, 4096, 0, 0, 41, 0, 4096, 3691, 0, 0, 0, 0, 0, 0, 0, 2771, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 42, 0, 4096, 564, 0, 0, 0, 0, 4096, 0, 0, 4076, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4094, 4057, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4094, 0, 4095, 559, 0, 4095, 3, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 4095, 0, 0, 5, 0, 0, 0, 0, 0, 1141, 0, 2794, 4095, 4094, 4095, 0, 3, 0, 1, 1, 0, 0, 0, 4096, 0, 1, 1, 4096, 0, 0, 3929, 2, 0, 0, 0, 4094, 4094, 2, 0, 0, 4083, 0, 2, 0, 0, 4091, 4093, 0, 0, 2, 0, 2, 0, 0, 3661, 0, 2, 4094, 0, 0, 0, 4094, 19, 0, 0, 0, 0, 0, 13, 0, 4094, 221, 0, 0, 0, 1, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4088, 4091, 0, 0, 0, 4096, 3, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 5, 4096, 0, 4096, 625, 0, 4094, 0, 0, 0, 4096, 0, 4096, 0, 1, 2, 0, 0, 0, 0, 4096, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 0, 3888, 4096, 4095, 4096, 0, 1, 6, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 4096, 6, 0, 3812, 0, 0, 0, 0, 4090, 4093, 0, 0, 0, 4061, 0, 0, 61, 0, 4069, 3913, 0, 0, 0, 0, 5, 12, 0, 2774, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 34, 0, 4096, 186, 0, 0, 0, 0, 4096, 0, 0, 4080, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4078, 3974, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 18, 4096, 0, 4096, 3835, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 98, 12, 0, 87, 4096, 4095, 4096, 0, 7, 0, 1, 5, 0, 0, 7, 4096, 0, 0, 1, 4096, 0, 0, 3967, 0, 0, 0, 0, 4080, 4029, 0, 0, 0, 4072, 0, 0, 123, 0, 4094, 4092, 0, 0, 19, 0, 0, 8, 0, 1376, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 196, 0, 4096, 2800, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 2, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4095, 1, 4093, 4092, 0, 0, 1, 4095, 0, 0, 0, 2, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 3234, 1, 4096, 0, 0, 0, 1871, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 1, 4095, 0, 4093, 1, 2, 2, 0, 1, 2565, 1529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 239, 0, 1157, 4096, 4095, 4095, 1, 4, 1, 0, 0, 0, 3, 695, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 0, 1, 0, 4094, 0, 0, 4014, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 13, 0, 4083, 4061, 0, 0, 1, 0, 0, 5, 0, 2916, 1252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3627, 1, 0, 4094, 0, 0, 1, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 47, 1, 0, 0, 0, 3603, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16459, 17395, 16385, 14880, 16576, 16427, 17014, 15936] avg=16384.0000 max=17395 max_rank=1 min=14880 min_rank=3 imbalance_ratio=1.061707 max_load_reduction=113677 max_load_reduction_pct=0.867287 moved_to_echo=65953 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4077, 1, 4083, 4095, 0, 0, 4, 4058, 13, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4094, 0, 4085, 1183, 2, 4096, 0, 0, 0, 1871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4095, 2, 4096, 0, 4096, 0, 0, 0, 0, 2, 2560, 1529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 160, 0, 3502, 4096, 4087, 4096, 0, 1, 4, 9, 1, 0, 0, 695, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 1, 0, 15, 4081, 3, 0, 3665, 27, 7, 0, 0, 4081, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 15, 0, 4096, 4094, 0, 0, 5, 0, 0, 53, 25, 2910, 1250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3618, 0, 0, 4082, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 38, 0, 4096, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 3603, 58, 0, 4086, 0, 4096, 21, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 15340, 16387, 17155, 16252, 16512, 17037, 15978] avg=16384.0000 max=17155 max_rank=3 min=15340 min_rank=1 imbalance_ratio=1.047058 max_load_reduction=113917 max_load_reduction_pct=0.869118 moved_to_echo=65779 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4088, 4091, 0, 0, 0, 4096, 3, 0, 0, 1, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 4096, 625, 0, 4094, 0, 0, 0, 1871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 4096, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 2565, 1530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 0, 3888, 4096, 4095, 4096, 0, 1, 6, 0, 0, 0, 0, 695, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 279, 0, 0, 0, 4096, 6, 0, 3812, 0, 0, 0, 0, 4090, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 61, 0, 4069, 3913, 0, 0, 0, 0, 5, 12, 0, 2914, 1252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2774, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 34, 0, 4096, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 186, 0, 0, 0, 0, 3603, 0, 0, 4080, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16455, 14787, 16381, 18525, 16379, 16287, 16197, 16061] avg=16384.0000 max=18525 max_rank=3 min=14787 min_rank=1 imbalance_ratio=1.130676 max_load_reduction=112547 max_load_reduction_pct=0.858665 moved_to_echo=64924 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4039, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3607, 0, 4096, 0, 0, 0, 1871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4085, 0, 0, 0, 0, 0, 2565, 1530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 983, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 695, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 279, 0, 0, 0, 4096, 0, 5, 3806, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 4054, 4008, 0, 0, 0, 0, 0, 5, 0, 2916, 1252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3789, 0, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 1, 0, 0, 48, 0, 4096, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 94, 0, 0, 0, 0, 3603, 0, 0, 4073, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16408, 17766, 16372, 14616, 16380, 16339, 17229, 15962] avg=16384.0000 max=17766 max_rank=1 min=14616 min_rank=3 imbalance_ratio=1.084351 max_load_reduction=113306 max_load_reduction_pct=0.864456 moved_to_echo=65910 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 3800, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1830, 0, 4096, 0, 0, 0, 1871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 0, 4094, 0, 0, 1, 0, 0, 2565, 1530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 318, 0, 3356, 4096, 4093, 4096, 0, 50, 0, 0, 0, 0, 0, 695, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 279, 48, 0, 0, 4096, 0, 0, 3993, 0, 0, 0, 0, 4088, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 41, 0, 4096, 3691, 0, 0, 0, 0, 0, 0, 0, 2915, 1252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2771, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 42, 0, 4096, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 564, 0, 0, 0, 0, 3603, 0, 0, 4076, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16169, 15989, 16385, 17201, 16601, 16091, 16201, 16435] avg=16384.0000 max=17201 max_rank=3 min=15989 min_rank=1 imbalance_ratio=1.049866 max_load_reduction=113871 max_load_reduction_pct=0.868767 moved_to_echo=65328 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 4096, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2906, 0, 4096, 0, 0, 0, 1863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2571, 1533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 949, 0, 3166, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 700, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 0, 0, 0, 4096, 0, 0, 3369, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 3, 0, 4093, 4029, 0, 0, 0, 0, 0, 0, 0, 2922, 1255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1955, 0, 0, 4096, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 1, 0, 4096, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 3601, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16447, 17057, 16392, 17598, 15927, 16399, 15355, 15897] avg=16384.0000 max=17598 max_rank=3 min=15355 min_rank=6 imbalance_ratio=1.074097 max_load_reduction=113474 max_load_reduction_pct=0.865738 moved_to_echo=63578 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4078, 3974, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 18, 4096, 0, 4096, 3835, 0, 4095, 0, 0, 0, 1871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 2565, 1530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 12, 0, 87, 4096, 4095, 4096, 0, 7, 0, 1, 5, 0, 0, 695, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 279, 0, 0, 1, 4096, 0, 0, 3967, 0, 0, 0, 0, 4080, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 123, 0, 4094, 4092, 0, 0, 19, 0, 0, 8, 0, 2916, 1252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1376, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 196, 0, 4096, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2800, 0, 0, 0, 0, 3603, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16299, 18019, 16381, 13685, 16459, 16576, 14960, 18693] avg=16384.0000 max=18693 max_rank=7 min=13685 min_rank=3 imbalance_ratio=1.140930 max_load_reduction=112379 max_load_reduction_pct=0.857384 moved_to_echo=66688 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4094, 4057, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 4095, 559, 0, 4095, 3, 0, 0, 1871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 4095, 0, 0, 5, 0, 0, 2565, 1530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1141, 0, 2794, 4095, 4094, 4095, 0, 3, 0, 1, 1, 0, 0, 695, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 0, 1, 1, 4096, 0, 0, 3929, 2, 0, 0, 0, 4094, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 2, 0, 0, 4091, 4093, 0, 0, 2, 0, 2, 0, 0, 2916, 1252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3661, 0, 2, 4094, 0, 0, 0, 4094, 19, 0, 0, 0, 0, 0, 13, 0, 4096, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 221, 0, 0, 0, 1, 3602, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16424, 14719, 16389, 17412, 16498, 16441, 17079, 16110] avg=16384.0000 max=17412 max_rank=3 min=14719 min_rank=1 imbalance_ratio=1.062744 max_load_reduction=113660 max_load_reduction_pct=0.867157 moved_to_echo=66128 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[23680, 128, 128, 32768, 0, 0, 128, 32768, 128, 0, 128, 128, 0, 0, 384, 0, 32768, 8832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 3968, 128, 0, 128, 128, 28928, 128, 0, 32768, 0, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 2304, 128, 128, 128, 32768, 128, 128, 30592, 128, 128, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 128, 0, 32768, 128, 32768, 0, 32768, 128, 128, 128, 0, 128, 20608, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 32768, 128, 128, 384, 0, 32768, 32000, 0, 0, 128, 0, 128, 128, 128, 23424, 10112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 32768, 32256, 0, 0, 128, 32768, 128, 0, 0, 128, 128, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 4224, 0, 18944, 32768, 32768, 32768, 128, 128, 128, 128, 128, 0, 128, 5632, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 128, 32768, 0, 32768, 17792, 128, 32768, 128, 0, 0, 14976, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[13, 4096, 4089, 0, 0, 0, 0, 4085, 0, 4095, 0, 0, 4096, 0, 238, 2994, 4084, 0, 4096, 4096, 4092, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 4087, 94, 0, 0, 1, 0, 0, 4096, 4008, 0, 0, 0, 20, 0, 0, 0, 4096, 0, 0, 0, 13, 59, 4076, 17, 0, 73, 0, 0, 0, 3883, 0, 0, 4093, 0, 332, 0, 0, 1771, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 198, 0, 0, 0, 4096, 0, 8, 4086, 0, 0, 0, 0, 4076, 0, 0, 1, 0, 4096, 4096, 0, 4082, 0, 13, 9, 0, 0, 0, 4069, 1147, 0, 4095, 0, 0, 6, 0, 0, 0, 4085, 0, 0, 0, 12, 1637, 4086, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 5, 4051, 0, 32, 1146, 4089, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 3, 0, 0, 0, 4082, 0, 0, 0, 0, 6, 4096, 0, 0, 9, 0, 0, 0, 4038, 0, 0, 3214, 0, 59, 0, 0, 151, 0, 414, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3004, 37, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 4076, 406, 0, 4096, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 3991, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4094, 0, 4096, 0, 1, 4093, 0, 4, 2043, 4096, 0, 4091, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 2278, 0, 69, 0, 0, 866, 0, 65, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3806, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 219, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2951, 4096, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 4094, 0, 1, 0, 0, 4092, 0, 4095, 0, 1, 4091, 1, 73, 1113, 4084, 0, 4096, 4095, 4095, 0, 4095, 4093, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 4091, 4091, 1, 1, 0, 5, 0, 1, 0, 4076, 2, 0, 1, 3, 0, 4090, 0, 1, 20, 0, 0, 0, 4066, 1, 0, 3714, 0, 3, 1, 0, 70, 5, 0, 1, 0, 0, 2, 4092, 0, 0, 3, 0, 2, 0, 467, 12, 0, 1, 4095, 1, 0, 4095, 0, 0, 1, 1, 4090, 0, 0, 0, 0, 4093, 4095, 1, 4095, 0, 0, 6, 1, 1, 0, 4095, 2953, 1, 4095, 0, 0, 0, 0, 1, 0, 4095, 0, 1, 0, 2, 3929, 4095, 1, 4095, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 1, 1773, 4096, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 0, 5, 4096, 0, 0, 80, 0, 0, 0, 4078, 0, 0, 3796, 0, 8, 0, 0, 7, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3355, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 624, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2664, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4092, 0, 0, 0, 0, 4093, 0, 4096, 0, 18, 4084, 0, 52, 3349, 4096, 0, 4086, 4096, 4096, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 10, 0, 0, 0, 0, 3974, 0, 0, 0, 1, 3, 4096, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 3555, 0, 0, 0, 0, 861, 0, 180, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1330, 25, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 4038, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 9, 1, 0, 0, 0, 4092, 17, 0, 4096, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 3114, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4094, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4087, 0, 143, 3123, 4087, 0, 4096, 4096, 4092, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 10, 4094, 0, 0, 8, 0, 0, 0, 3920, 0, 0, 1366, 0, 1, 0, 0, 0, 3, 78, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 3361, 0, 0, 0, 4096, 0, 3, 4096, 2, 2, 0, 0, 4093, 0, 0, 0, 0, 4096, 4096, 0, 4096, 2, 4, 1, 0, 0, 0, 4096, 688, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3702, 4096, 0, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4062, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 4095, 0, 260, 4051, 4096, 27, 4096, 4096, 4096, 0, 4096, 3983, 0, 0, 0, 0, 0, 0, 4082, 10, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4, 27, 4089, 0, 0, 0, 0, 5, 4096, 0, 0, 5, 0, 0, 0, 4036, 0, 0, 721, 0, 879, 0, 75, 213, 0, 468, 0, 0, 0, 1, 4094, 2, 0, 0, 0, 0, 0, 2753, 60, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 4035, 0, 6, 0, 0, 4096, 4096, 0, 4096, 0, 0, 12, 0, 0, 0, 3949, 166, 0, 4096, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 3123, 4096, 0, 4058, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[13, 1561, 4089, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 4096, 0, 238, 2994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 10, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4087, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4008, 0, 0, 0, 20, 0, 0, 0, 4096, 0, 3620, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 59, 4076, 17, 0, 73, 0, 0, 0, 3883, 0, 0, 4093, 0, 4095, 824, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, 0, 0, 1771, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 4096, 4096, 3272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 0, 0, 0, 4096, 0, 8, 4086, 0, 0, 0, 0, 4076, 0, 0, 1816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4082, 0, 13, 9, 0, 0, 0, 4069, 1147, 0, 3339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4085, 0, 0, 0, 12, 1637, 4086, 0, 4096, 0, 756, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17076, 16457, 16338, 17595, 17670, 14280, 16756, 14900] avg=16384.0000 max=17670 max_rank=4 min=14280 min_rank=5 imbalance_ratio=1.078491 max_load_reduction=113402 max_load_reduction_pct=0.865189 moved_to_echo=63606 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1561, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 4051, 0, 32, 1146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 3, 0, 0, 0, 4082, 0, 3623, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 9, 0, 0, 0, 4038, 0, 0, 3214, 0, 4096, 824, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 151, 0, 414, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3004, 37, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4065, 0, 0, 1816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4076, 406, 0, 3340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 3991, 4096, 0, 4096, 0, 756, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14987, 16388, 16395, 16745, 16184, 17114, 16015, 17244] avg=16384.0000 max=17244 max_rank=7 min=14987 min_rank=0 imbalance_ratio=1.052490 max_load_reduction=113828 max_load_reduction_pct=0.868439 moved_to_echo=66557 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1561, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 4093, 0, 4, 2043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3623, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 2278, 0, 4096, 823, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 866, 0, 65, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3806, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 1816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 219, 0, 3340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2951, 4096, 0, 4095, 0, 756, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15892, 16393, 16408, 15859, 16561, 17902, 15841, 16216] avg=16384.0000 max=17902 max_rank=5 min=15841 min_rank=6 imbalance_ratio=1.092651 max_load_reduction=113170 max_load_reduction_pct=0.863419 moved_to_echo=66520 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1561, 4092, 0, 0, 0, 0, 4093, 0, 0, 0, 18, 4084, 0, 52, 3349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 11, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 10, 0, 0, 0, 0, 3974, 0, 3623, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 4096, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 3555, 0, 4096, 823, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 861, 0, 180, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1330, 25, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 4038, 0, 0, 1816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 9, 1, 0, 0, 0, 4092, 17, 0, 3340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 3114, 4096, 0, 4096, 0, 756, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17249, 16376, 16296, 17132, 16602, 15402, 15651, 16364] avg=16384.0000 max=17249 max_rank=0 min=15402 min_rank=5 imbalance_ratio=1.052795 max_load_reduction=113823 max_load_reduction_pct=0.868401 moved_to_echo=64019 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1560, 4094, 0, 1, 0, 0, 4092, 0, 0, 0, 1, 4091, 1, 73, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 11, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4091, 4091, 1, 1, 0, 5, 0, 1, 0, 4076, 2, 3622, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 4090, 0, 1, 20, 0, 0, 0, 4066, 1, 0, 3714, 0, 4095, 823, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 70, 5, 0, 1, 0, 0, 2, 4092, 0, 0, 3, 0, 2, 4095, 4095, 3272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 12, 0, 1, 4095, 1, 0, 4095, 0, 0, 1, 1, 4090, 0, 0, 1816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 4095, 0, 0, 6, 1, 1, 0, 4095, 2953, 1, 3339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 1, 0, 2, 3929, 4095, 1, 4095, 1, 756, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15026, 16379, 16388, 17276, 15641, 14579, 18585, 17198] avg=16384.0000 max=18585 max_rank=6 min=14579 min_rank=5 imbalance_ratio=1.134338 max_load_reduction=112487 max_load_reduction_pct=0.858208 moved_to_echo=66003 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1548, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 1773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3628, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 4096, 0, 0, 80, 0, 0, 0, 4078, 0, 0, 3796, 0, 4096, 824, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 7, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3355, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 624, 0, 3337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2664, 4096, 0, 4096, 0, 759, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15610, 16387, 16412, 17441, 15576, 17464, 16243, 15939] avg=16384.0000 max=17464 max_rank=5 min=15576 min_rank=4 imbalance_ratio=1.065918 max_load_reduction=113608 max_load_reduction_pct=0.866760 moved_to_echo=65222 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1561, 4062, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4095, 0, 260, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 27, 4096, 0, 11, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 4082, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4, 27, 4089, 0, 3623, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 5, 0, 0, 0, 4036, 0, 0, 721, 0, 4096, 824, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 879, 0, 75, 213, 0, 468, 0, 0, 0, 1, 4094, 2, 0, 0, 0, 0, 4096, 4096, 3272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2753, 60, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 4035, 0, 6, 1816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 3949, 166, 0, 3340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 3123, 4096, 0, 4058, 0, 756, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18126, 16305, 16432, 14245, 17196, 16863, 15659, 16246] avg=16384.0000 max=18126 max_rank=0 min=14245 min_rank=3 imbalance_ratio=1.106323 max_load_reduction=112946 max_load_reduction_pct=0.861710 moved_to_echo=65964 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1561, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 143, 3123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 0, 10, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3620, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4094, 0, 0, 8, 0, 0, 0, 3920, 0, 0, 1366, 0, 4096, 823, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 78, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 4096, 4096, 3273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3361, 0, 0, 0, 4096, 0, 3, 4096, 2, 2, 0, 0, 4093, 0, 0, 1815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 4, 1, 0, 0, 0, 4096, 688, 0, 3339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3702, 4096, 0, 4093, 0, 756, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17106, 16387, 16403, 14779, 15642, 17468, 16322, 16965] avg=16384.0000 max=17468 max_rank=5 min=14779 min_rank=3 imbalance_ratio=1.066162 max_load_reduction=113604 max_load_reduction_pct=0.866730 moved_to_echo=66397 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 32768, 0, 128, 32768, 128, 128, 128, 128, 128, 0, 32640, 6272, 128, 26752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 18304, 256, 0, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 32640, 0, 128, 14592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 12544, 32768, 0, 128, 0, 0, 32768, 0, 0, 0, 128, 32768, 128, 896, 19712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 0, 0, 32768, 32768, 128, 128, 128, 128, 0, 128, 128, 32640, 128, 29056, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1408, 128, 128, 3968, 128, 1280, 128, 0, 0, 128, 32768, 128, 0, 128, 0, 128, 32768, 32768, 26240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[32768, 128, 32768, 0, 128, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 256, 0, 0, 0, 32256, 128, 0, 22784, 0, 32768, 6656, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 0, 128, 0, 32640, 0, 128, 0, 128, 25216, 32768, 128, 32768, 128, 6144, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4095, 1, 0, 0, 1, 0, 0, 1, 0, 4094, 4096, 4095, 4095, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 2, 0, 0, 1, 1, 4093, 0, 1, 4095, 0, 10, 0, 2, 0, 4094, 1, 1, 4094, 0, 1, 2, 1, 0, 1, 0, 0, 4095, 0, 4096, 0, 2641, 4095, 0, 0, 1, 4094, 0, 1, 0, 0, 1, 0, 4095, 0, 1576, 17, 21, 0, 0, 3957, 0, 1, 4094, 4091, 4094, 0, 0, 0, 3, 1110, 4095, 9, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 4093, 4077, 0, 0, 2, 0, 1, 4095, 4095, 0, 0, 0, 1, 4094, 4095, 4095, 0, 4096, 0, 0, 0, 2972, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2389, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 838, 2, 19, 0, 0, 4096, 0, 0, 4096, 4090, 4096, 0, 0, 0, 0, 1677, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 3278, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 11, 0, 5, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2770, 4094, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 865, 10, 240, 0, 0, 3878, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 524, 4096, 3, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 4093, 0, 0, 0, 0, 0, 4063, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 4028, 7, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1667, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 9, 4096, 0, 1801, 0, 427, 0, 0, 4059, 0, 0, 4096, 4088, 4096, 0, 0, 15, 0, 1444, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4070, 0, 0, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 2914, 10, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 4080, 4096, 4096, 4096, 2, 15, 0, 0, 10, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 7, 4096, 0, 32, 0, 0, 0, 4047, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 16, 0, 4095, 0, 4096, 0, 340, 4096, 0, 4, 0, 4082, 2, 0, 3, 0, 0, 5, 4095, 0, 2346, 417, 0, 0, 0, 3928, 0, 0, 4080, 4096, 4075, 11, 0, 1, 12, 2267, 4096, 17, 4, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4052, 4086, 0, 0, 0, 0, 0, 3991, 4096, 0, 0, 0, 0, 4096, 4096, 4086, 0, 4096, 0, 0, 0, 3100, 33, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 4096, 4096, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 6, 0, 0, 0, 4095, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 2, 4092, 0, 4096, 0, 2486, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 45, 1, 34, 0, 0, 4041, 0, 0, 4096, 4086, 4096, 0, 0, 3, 0, 2119, 4096, 0, 0, 0, 3, 0, 1, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 1, 4093, 4095, 4096, 0, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 0, 4096, 4096, 4095, 0, 4096, 0, 0, 0, 3572, 2, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 4096, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 1, 0, 4096, 0, 0, 4087, 0, 2, 0, 0, 0, 0, 0, 2, 4096, 0, 4096, 0, 2919, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 846, 0, 174, 0, 0, 3895, 0, 0, 4096, 4096, 4096, 0, 0, 5, 0, 2310, 4096, 0, 0, 0, 1, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 4085, 0, 0, 0, 0, 0, 4072, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 2178, 1, 4088] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4086, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 4096, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 21, 0, 0, 0, 4078, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3, 7, 4096, 0, 4096, 0, 538, 4088, 49, 0, 0, 4089, 0, 0, 0, 0, 0, 59, 4095, 27, 221, 128, 438, 0, 0, 3706, 0, 0, 4089, 4070, 4096, 1, 0, 0, 0, 3312, 4096, 0, 1, 0, 4, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4077, 4095, 0, 1, 0, 0, 0, 3808, 4095, 49, 0, 0, 0, 4093, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 16, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 5, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 3278, 813, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2122, 0, 2770, 4094, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2937, 0, 865, 10, 240, 0, 0, 3878, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 524, 4096, 3, 0, 0, 0, 0, 0, 0, 4086, 0, 1161, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 4093, 0, 0, 0, 0, 0, 4063, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4028, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16380, 16397, 17185, 16122, 15050, 16416, 17141] avg=16384.0000 max=17185 max_rank=3 min=15050 min_rank=5 imbalance_ratio=1.048889 max_load_reduction=113887 max_load_reduction_pct=0.868889 moved_to_echo=64729 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3281, 820, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2115, 0, 2389, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2928, 0, 838, 2, 19, 0, 0, 4096, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1677, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1161, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 0, 4096, 79, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 815, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3278, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16380, 16393, 16792, 16069, 16211, 16467, 16381] avg=16384.0000 max=16792 max_rank=3 min=16069 min_rank=4 imbalance_ratio=1.024902 max_load_reduction=114280 max_load_reduction_pct=0.871887 moved_to_echo=65128 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 1, 0, 0, 1, 0, 0, 1, 0, 4094, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 2, 0, 0, 1, 1, 4093, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 10, 0, 2, 0, 4094, 1, 1, 4094, 0, 1, 2, 1, 3277, 813, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 2122, 0, 2641, 4095, 0, 0, 1, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2936, 0, 1576, 17, 21, 0, 0, 3957, 0, 1, 4094, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 3, 1110, 4095, 9, 0, 0, 0, 0, 0, 1, 4095, 0, 1161, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 4093, 4077, 0, 0, 2, 0, 1, 4095, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 0, 0, 0, 1, 4094, 0, 4095, 0, 0, 0, 0, 0, 2972, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16382, 16397, 17050, 16694, 15652, 16440, 16075] avg=16384.0000 max=17050 max_rank=3 min=15652 min_rank=5 imbalance_ratio=1.040649 max_load_reduction=114022 max_load_reduction_pct=0.869919 moved_to_echo=64861 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 1, 0, 4096, 0, 0, 4087, 0, 2, 0, 0, 3278, 813, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 2122, 0, 2919, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2937, 0, 846, 0, 174, 0, 0, 3895, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 2310, 4096, 0, 0, 0, 1, 0, 0, 0, 4091, 0, 1161, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 4085, 0, 0, 0, 0, 0, 4072, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2178, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 16385, 16378, 17325, 16044, 16844, 16422, 15277] avg=16384.0000 max=17325 max_rank=3 min=15277 min_rank=7 imbalance_ratio=1.057434 max_load_reduction=113747 max_load_reduction_pct=0.867821 moved_to_echo=64587 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3278, 813, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2122, 0, 1667, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2937, 0, 1801, 0, 427, 0, 0, 4059, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 1444, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 1161, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4070, 0, 0, 0, 0, 0, 4084, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2914, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16380, 16383, 16077, 17417, 15983, 16421, 16030] avg=16384.0000 max=17417 max_rank=4 min=15983 min_rank=5 imbalance_ratio=1.063049 max_load_reduction=113655 max_load_reduction_pct=0.867119 moved_to_echo=65851 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 4080, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 10, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 32, 0, 0, 0, 4047, 0, 0, 4083, 0, 0, 0, 0, 3278, 813, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4095, 0, 2122, 0, 340, 4096, 0, 4, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5, 2936, 0, 2346, 417, 0, 0, 0, 3928, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 11, 0, 1, 12, 2267, 4096, 17, 4, 0, 7, 0, 0, 0, 4096, 0, 1161, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4052, 4086, 0, 0, 0, 0, 0, 3991, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 0, 0, 3100, 33, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16405, 16360, 14757, 17811, 16831, 16300, 16229] avg=16384.0000 max=17811 max_rank=4 min=14757 min_rank=3 imbalance_ratio=1.087097 max_load_reduction=113261 max_load_reduction_pct=0.864113 moved_to_echo=67171 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 4095, 0, 0, 4088, 0, 0, 0, 0, 3277, 813, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4092, 0, 2122, 0, 2486, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2937, 0, 45, 1, 34, 0, 0, 4041, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 2119, 4096, 0, 0, 0, 3, 0, 1, 0, 4094, 0, 1161, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 4093, 4095, 4096, 0, 0, 0, 0, 0, 4094, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 3572, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16380, 16380, 16894, 15241, 16657, 16455, 16676] avg=16384.0000 max=16894 max_rank=3 min=15241 min_rank=4 imbalance_ratio=1.031128 max_load_reduction=114178 max_load_reduction_pct=0.871109 moved_to_echo=65029 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4086, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 0, 4078, 0, 0, 4095, 0, 0, 0, 0, 3277, 813, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 4096, 0, 2122, 0, 538, 4088, 49, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 2936, 27, 221, 128, 438, 0, 0, 3706, 0, 0, 4089, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3312, 4096, 0, 1, 0, 4, 0, 0, 0, 4089, 0, 1161, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4077, 4095, 0, 1, 0, 0, 0, 3808, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 49, 0, 0, 0, 4093, 0, 4095, 0, 0, 0, 0, 0, 4096, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16380, 16384, 14992, 15674, 17844, 16151, 17263] avg=16384.0000 max=17844 max_rank=5 min=14992 min_rank=3 imbalance_ratio=1.089111 max_load_reduction=113228 max_load_reduction_pct=0.863861 moved_to_echo=66932 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 0, 128, 128, 14848, 32768, 128, 128, 0, 128, 0, 128, 128, 32768, 0, 9344, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[6656, 128, 0, 0, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 26240, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 128, 128, 0, 128, 0, 32768, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 0, 0, 128, 32768, 32768, 32768, 0, 128, 128, 0, 128, 32384, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 32768, 0, 128, 0, 128, 0, 32768, 128, 128, 32768, 0, 128, 128, 128, 26240, 6528, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 23552, 128, 8576, 640, 1408, 0, 0, 31616, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 128, 128, 128, 0, 0, 0, 128, 0, 0, 128, 128, 32768, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 17024, 0, 15872, 32768, 128, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4081, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4093, 0, 0, 4034, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 2167, 4096, 4096, 4096, 4096, 0, 50, 1154, 10, 7, 0, 0, 0, 0, 4095, 0, 4070, 38, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 791, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 0, 0, 0, 6, 4071, 0, 0, 4096, 0, 82, 0, 0, 4096, 0, 0, 0, 0, 4086, 22, 4096, 4095, 0, 4095, 0, 0, 3932, 1, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 4096, 0, 0, 1, 5, 0, 2025, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 0, 4008, 0, 4096, 0, 2363, 4090, 4096, 4096, 4096, 0, 129, 3526, 0, 23, 0, 0, 0, 0, 4096, 0, 4058, 44, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4093, 4079, 0, 0, 0, 4073, 9, 0, 25, 0, 4096, 0, 0, 301, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 4096, 4095, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 2391, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 4053, 0, 4096, 0, 3910, 4096, 4096, 4096, 4096, 0, 113, 1752, 0, 52, 0, 0, 0, 0, 4096, 0, 4076, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4093, 0, 0, 3, 0, 4096, 0, 0, 45, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 4096, 4094, 0, 4096, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4022, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 4096, 0, 2373, 4096, 4096, 4096, 4096, 0, 301, 1416, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 107, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 1, 0, 1, 2, 4095, 0, 1, 4094, 0, 5, 0, 1, 4095, 0, 0, 0, 0, 4088, 3, 4095, 4075, 0, 4094, 1, 0, 4095, 0, 0, 0, 1, 0, 2, 3, 4095, 0, 1, 4095, 1, 0, 1, 1, 0, 3972, 0, 0, 4093, 0, 1, 0, 0, 0, 2, 1, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 4096, 0, 4083, 0, 3853, 4095, 4095, 4095, 4093, 0, 6, 213, 0, 10, 0, 1, 0, 1, 4095, 0, 4095, 0, 4095, 1, 0, 1, 2, 4095, 1, 4094, 0, 0, 4094, 4095, 1, 1, 0, 4078, 0, 1, 17, 0, 4094, 0, 0, 170, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 9, 4096, 4066, 0, 4089, 0, 0, 4096, 1, 0, 0, 0, 4, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1641, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 4096, 0, 4048, 0, 2659, 4096, 4096, 4096, 4096, 0, 52, 3914, 0, 19, 0, 0, 0, 0, 4095, 0, 4080, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4078, 3, 0, 0, 0, 4096, 0, 0, 8, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 1, 0, 0, 4092, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3, 4096, 4080, 0, 4096, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4005, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4085, 1, 4084, 0, 1434, 4096, 4096, 4096, 4096, 2, 5, 2529, 6, 10, 0, 0, 0, 0, 4088, 0, 4093, 13, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4095, 6, 0, 4, 0, 4091, 0, 0, 240, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3992, 0, 0, 4095, 0, 0, 0, 0, 4071, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4070, 14, 4096, 4013, 0, 4096, 0, 0, 4069, 15, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 3860, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4096, 0, 4088, 0, 4067, 0, 715, 4096, 4096, 4096, 4096, 0, 11, 3194, 1, 92, 0, 0, 0, 0, 4090, 0, 3968, 110, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 4096, 4013, 0, 0, 0, 4096, 0, 0, 12, 0, 4091, 0, 0, 744, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 0, 0, 0, 6, 4071, 3233, 629, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 82, 0, 0, 9, 0, 0, 0, 0, 4086, 22, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3932, 1, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 2025, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 4090, 3252, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 4096, 0, 2363, 3461, 0, 0, 0, 0, 129, 3526, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4058, 44, 4096, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3807, 4079, 0, 0, 0, 4073, 9, 0, 25, 0, 4096, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16370, 16486, 16260, 15163, 16401, 17606, 16396, 16390] avg=16384.0000 max=17606 max_rank=5 min=15163 min_rank=3 imbalance_ratio=1.074585 max_load_reduction=113466 max_load_reduction_pct=0.865677 moved_to_echo=66793 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4081, 3233, 630, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 9, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4034, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 3252, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2167, 3466, 0, 0, 0, 0, 50, 1154, 10, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4070, 38, 4096, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 4096, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16391, 16394, 16352, 17225, 16386, 15046, 16401, 16877] avg=16384.0000 max=17225 max_rank=3 min=15046 min_rank=5 imbalance_ratio=1.051331 max_load_reduction=113847 max_load_reduction_pct=0.868584 moved_to_echo=64710 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 1, 0, 1, 2, 4095, 3232, 630, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 5, 0, 1, 9, 0, 0, 0, 0, 4088, 3, 4095, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 4095, 0, 0, 0, 1, 0, 2, 3, 4095, 0, 1, 4095, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 3972, 0, 0, 4093, 0, 1, 0, 0, 0, 2, 1, 4095, 4095, 842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 4090, 3251, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 0, 3853, 3465, 0, 0, 0, 0, 6, 213, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 4095, 0, 4095, 1, 0, 1, 2, 5, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3808, 4095, 1, 1, 0, 4078, 0, 1, 17, 0, 4094, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16409, 16371, 16418, 17104, 16388, 15726, 16391, 16265] avg=16384.0000 max=17104 max_rank=3 min=15726 min_rank=5 imbalance_ratio=1.043945 max_load_reduction=113968 max_load_reduction_pct=0.869507 moved_to_echo=64770 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 3233, 630, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4095, 9, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 1, 0, 0, 0, 4, 1, 0, 4096, 0, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1641, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 4090, 3253, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4048, 0, 2659, 3466, 0, 0, 0, 0, 52, 3914, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4080, 0, 4096, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 4096, 0, 0, 0, 4078, 3, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16405, 16371, 16414, 14772, 16392, 18254, 16373, 16091] avg=16384.0000 max=18254 max_rank=5 min=14772 min_rank=3 imbalance_ratio=1.114136 max_load_reduction=112818 max_load_reduction_pct=0.860733 moved_to_echo=67110 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 3233, 630, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4091, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2391, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4090, 3252, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 4096, 0, 3910, 3466, 0, 0, 0, 0, 113, 1752, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 0, 4096, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 4096, 0, 0, 0, 4093, 0, 0, 3, 0, 4096, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16406, 16387, 16412, 15523, 16389, 17442, 16370, 16143] avg=16384.0000 max=17442 max_rank=5 min=15523 min_rank=3 imbalance_ratio=1.064575 max_load_reduction=113630 max_load_reduction_pct=0.866928 moved_to_echo=66344 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 3234, 634, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4, 0, 0, 0, 0, 4094, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3252, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 2373, 3462, 0, 0, 0, 0, 301, 1416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3803, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16413, 16386, 16396, 17154, 16398, 15742, 16385, 16198] avg=16384.0000 max=17154 max_rank=3 min=15742 min_rank=5 imbalance_ratio=1.046997 max_load_reduction=113918 max_load_reduction_pct=0.869125 moved_to_echo=64723 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3992, 0, 0, 4095, 0, 0, 0, 0, 4071, 3233, 630, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4070, 14, 4096, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4069, 15, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3860, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4096, 4090, 3252, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4067, 0, 715, 3466, 0, 0, 0, 0, 11, 3194, 1, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3968, 110, 4096, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 4013, 0, 0, 0, 4096, 0, 0, 12, 0, 4091, 0, 0, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16276, 16298, 16404, 16995, 16334, 15634, 16365, 16766] avg=16384.0000 max=16995 max_rank=3 min=15634 min_rank=5 imbalance_ratio=1.037292 max_load_reduction=114077 max_load_reduction_pct=0.870338 moved_to_echo=65099 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 1, 0, 0, 4092, 3233, 630, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4096, 3, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 3253, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 4084, 0, 1434, 3466, 0, 0, 0, 2, 5, 2529, 6, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4093, 13, 4095, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 4096, 0, 0, 0, 4095, 6, 0, 4, 0, 4091, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16402, 16379, 16416, 17136, 16384, 15622, 16391, 16342] avg=16384.0000 max=17136 max_rank=3 min=15622 min_rank=5 imbalance_ratio=1.045898 max_load_reduction=113936 max_load_reduction_pct=0.869263 moved_to_echo=64739 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 128, 0, 32512, 128, 0, 0, 128, 128, 128, 128, 32768, 0, 128, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 0, 26112, 0, 0, 32768, 0, 128, 0, 0, 0, 128, 128, 32768, 32768, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 0, 128, 0, 128, 128, 0, 0, 0, 0, 32768, 128, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 32768, 0, 32640, 256, 32768, 128, 0, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 30592, 32768, 128, 128, 0, 32768, 128, 128, 128, 0, 32768, 0, 0, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 0, 0, 0, 0, 32768, 0, 0, 32768, 128, 128, 128, 128, 32768, 25984, 5120, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32640, 128, 32768, 0, 19584, 27776, 0, 0, 0, 128, 768, 17792, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 128, 128, 32768, 32768, 26112, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[149, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 3995, 0, 0, 0, 3, 0, 4096, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4065, 0, 0, 4096, 0, 0, 0, 0, 273, 4096, 0, 4096, 0, 0, 3800, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[2, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 11, 4096, 0, 4094, 0, 0, 0, 0, 28, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4071, 6, 0, 0, 284, 0, 4096, 0, 2, 0, 0, 4056, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 261, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 482, 4096, 0, 4096, 0, 0, 3090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[3, 4095, 1, 4087, 0, 0, 1, 4095, 0, 4096, 0, 0, 0, 0, 4095, 1, 4095, 0, 1, 0, 1, 0, 4095, 4095, 0, 4093, 4095, 2, 1, 0, 4079, 4094, 0, 4095, 1, 3829, 0, 0, 1, 10, 1, 4093, 15, 0, 0, 0, 4095, 1, 0, 1, 2, 4095, 0, 0, 1, 0, 1, 0, 0, 0, 4095, 4095, 1, 1, 4090, 1, 0, 1, 2, 0, 0, 1, 4095, 4095, 0, 1, 2, 0, 1, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 4041, 0, 0, 4095, 1, 0, 0, 1, 347, 4095, 0, 4093, 1, 0, 4059, 0, 0, 1, 4094, 1, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 0, 2, 7, 1, 0, 0, 4095, 4095, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[7, 4096, 0, 4094, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 6, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 179, 0, 4096, 44, 0, 0, 0, 4075, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 43, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4067, 0, 0, 4096, 0, 0, 0, 0, 2138, 4096, 0, 4096, 0, 0, 1746, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[208, 4096, 0, 4096, 18, 0, 0, 4096, 2, 4096, 17, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 0, 61, 4096, 4096, 0, 4056, 4040, 0, 0, 0, 4096, 4096, 0, 4095, 0, 3805, 48, 0, 0, 18, 0, 4096, 1, 32, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 3828, 0, 38, 4096, 0, 0, 0, 0, 1984, 4096, 0, 4096, 0, 0, 2347, 0, 0, 0, 4082, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[111, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4086, 11, 0, 0, 0, 4096, 0, 4096, 8, 0, 0, 0, 7, 4096, 4096, 0, 4085, 4094, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4087, 11, 0, 0, 54, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4032, 0, 0, 4096, 0, 0, 0, 0, 594, 4096, 0, 4096, 0, 0, 3390, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[2477, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 60, 0, 0, 0, 4085, 0, 4096, 18, 0, 0, 0, 110, 4096, 4096, 0, 4077, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4090, 84, 0, 0, 771, 0, 4096, 0, 0, 0, 0, 4020, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 838, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2625, 0, 0, 4096, 0, 0, 0, 0, 446, 4096, 0, 4096, 0, 0, 908, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[375, 4096, 0, 4094, 0, 0, 0, 4096, 0, 4095, 5, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 16, 4095, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 1, 4096, 0, 4049, 0, 0, 0, 790, 0, 4096, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 4095, 4093, 0, 0, 0, 163, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3820, 0, 1, 4096, 0, 0, 0, 0, 29, 4096, 0, 4096, 0, 0, 3056, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[149, 3660, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3911, 0, 3995, 0, 0, 0, 3, 0, 4096, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3828, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3831, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4065, 0, 0, 4096, 0, 0, 0, 0, 4096, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 4096, 0, 4096, 0, 0, 3800, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 320, 185, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3828, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16097, 16367, 16104, 16384, 16235, 16621, 16883, 16381] avg=16384.0000 max=16883 max_rank=6 min=16097 min_rank=0 imbalance_ratio=1.030457 max_load_reduction=114189 max_load_reduction_pct=0.871193 moved_to_echo=66120 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3, 3670, 1, 4087, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 1, 0, 1, 0, 0, 0, 0, 4093, 4095, 2, 1, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 1, 3829, 0, 0, 1, 10, 1, 4093, 15, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4095, 0, 0, 1, 0, 1, 0, 0, 0, 4095, 4095, 1, 1, 3828, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 1, 2, 0, 0, 1, 4095, 4095, 0, 1, 2, 0, 1, 0, 3828, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 4041, 0, 0, 4095, 1, 0, 0, 1, 4095, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 347, 4095, 0, 4093, 1, 0, 4059, 0, 0, 1, 4094, 1, 0, 0, 0, 0, 316, 181, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 0, 2, 7, 1, 0, 0, 4095, 4095, 0, 0, 0, 3829, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15954, 16354, 15960, 16386, 16226, 16596, 17201, 16395] avg=16384.0000 max=17201 max_rank=6 min=15954 min_rank=0 imbalance_ratio=1.049866 max_load_reduction=113871 max_load_reduction_pct=0.868767 moved_to_echo=66418 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 3671, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 28, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3915, 0, 4071, 6, 0, 0, 284, 0, 4096, 0, 2, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3828, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 261, 0, 0, 3830, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 4096, 0, 4096, 0, 0, 3090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 316, 181, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3829, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15972, 16397, 16430, 16383, 16485, 16652, 16370, 16383] avg=16384.0000 max=16652 max_rank=5 min=15972 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=114420 max_load_reduction_pct=0.872955 moved_to_echo=65890 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[208, 3671, 0, 4096, 18, 0, 0, 0, 2, 4096, 17, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 61, 0, 0, 0, 4056, 4040, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 0, 3805, 48, 0, 0, 18, 0, 4096, 1, 32, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3828, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 3831, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 3828, 0, 38, 4096, 0, 0, 0, 0, 4096, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1984, 4096, 0, 4096, 0, 0, 2347, 0, 0, 0, 4082, 0, 5, 0, 0, 0, 316, 181, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 3829, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16194, 16336, 16005, 16386, 16217, 16433, 17120, 16381] avg=16384.0000 max=17120 max_rank=6 min=16005 min_rank=2 imbalance_ratio=1.044922 max_load_reduction=113952 max_load_reduction_pct=0.869385 moved_to_echo=66151 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[7, 3671, 0, 4094, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3915, 0, 4096, 0, 0, 0, 179, 0, 4096, 44, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3828, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 43, 0, 0, 3830, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4067, 0, 0, 4096, 0, 0, 0, 0, 4096, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2138, 4096, 0, 4096, 0, 0, 1746, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 316, 181, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3829, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15963, 16377, 16405, 16383, 16256, 16623, 16682, 16383] avg=16384.0000 max=16682 max_rank=6 min=15963 min_rank=0 imbalance_ratio=1.018188 max_load_reduction=114390 max_load_reduction_pct=0.872726 moved_to_echo=65944 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[111, 3671, 0, 4096, 0, 0, 0, 0, 0, 4086, 11, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 8, 0, 0, 0, 7, 0, 0, 0, 4085, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3915, 0, 4087, 11, 0, 0, 54, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3828, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 11, 0, 0, 3830, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4032, 0, 0, 4096, 0, 0, 0, 0, 4096, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 4096, 0, 4096, 0, 0, 3390, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 316, 181, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3829, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16071, 16373, 16259, 16383, 16233, 16588, 16782, 16383] avg=16384.0000 max=16782 max_rank=6 min=16071 min_rank=0 imbalance_ratio=1.024292 max_load_reduction=114290 max_load_reduction_pct=0.871964 moved_to_echo=65986 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2477, 3671, 0, 4096, 0, 0, 0, 0, 0, 4096, 60, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 18, 0, 0, 0, 110, 0, 0, 0, 4077, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3915, 0, 4090, 84, 0, 0, 771, 0, 4096, 0, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3828, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 838, 0, 0, 3831, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2625, 0, 0, 4096, 0, 0, 0, 0, 4096, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 446, 4096, 0, 4096, 0, 0, 908, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 316, 181, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3828, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18485, 16480, 16976, 16384, 17034, 15181, 14151, 16381] avg=16384.0000 max=18485 max_rank=0 min=14151 min_rank=6 imbalance_ratio=1.128235 max_load_reduction=112587 max_load_reduction_pct=0.858971 moved_to_echo=62747 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[375, 3671, 0, 4094, 0, 0, 0, 0, 0, 4095, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 1, 0, 0, 0, 16, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3915, 0, 4049, 0, 0, 0, 790, 0, 4096, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3827, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 4095, 4093, 0, 0, 0, 163, 0, 0, 3830, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3820, 0, 1, 4096, 0, 0, 0, 0, 4096, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4096, 0, 4096, 0, 0, 3056, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 316, 181, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3829, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16336, 16388, 16933, 16383, 16386, 16378, 15883, 16385] avg=16384.0000 max=16933 max_rank=2 min=15883 min_rank=6 imbalance_ratio=1.033508 max_load_reduction=114139 max_load_reduction_pct=0.870811 moved_to_echo=65032 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 0, 128, 32768, 32768, 0, 128, 128, 1408, 128, 0, 30720, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 31360, 128, 32128, 256, 0, 128, 2176, 128, 32768, 128, 128, 0, 0, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[3456, 29440, 128, 32768, 128, 0, 128, 0, 128, 32768, 128, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 0, 32768, 128, 128, 0, 0, 30592, 0, 128, 32768, 128, 0, 0, 128, 32768, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[6400, 32768, 0, 32768, 128, 0, 22400, 0, 0, 128, 32768, 128, 128, 0, 0, 0, 2560, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 128, 128, 128, 0, 0, 32768, 32768, 0, 0, 0, 30720, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 0, 128, 256, 0, 0, 0, 32768, 32768, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 32768, 128, 0, 128, 0, 128, 0, 0, 0, 32768, 32768, 128, 128, 30720, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4095, 1, 0, 3626, 0, 2, 2, 0, 4095, 0, 1, 1, 4095, 0, 4095, 4095, 0, 1, 0, 0, 4094, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 4091, 4093, 1, 4095, 0, 4095, 4095, 1, 0, 1, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 4089, 4096, 1, 0, 0, 0, 0, 1, 2, 0, 0, 1, 0, 4094, 3, 4095, 0, 1, 4095, 0, 0, 4095, 4095, 4095, 0, 4094, 0, 1, 4095, 0, 0, 0, 4, 0, 2, 0, 1, 0, 40, 0, 0, 2000, 0, 1, 1, 0, 4095, 1, 0, 1, 1, 2522, 4095, 4095, 6, 0, 4095, 1, 0, 4095, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 2, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 3921, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4, 0, 0, 177, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4082, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 3487, 9, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 15, 0, 0, 4066, 4082, 4089, 0, 4096, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 9, 0, 4093, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 624, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4052, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 2712, 630, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 2, 8, 0, 0, 4095, 4095, 4083, 0, 4096, 0, 4093, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 73, 0, 0, 0, 0, 5, 0, 0, 728, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 4060, 4096, 4096, 0, 1, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 2469, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 4088, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 26, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 12, 0, 4085, 0, 0, 0, 0, 15, 6, 0, 0, 0, 73, 0, 0, 1590, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4074, 4094, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 3746, 10, 0, 2, 0, 4096, 0, 0, 0, 4072, 0, 4096, 4094, 0, 0, 12, 0, 4066, 4096, 0, 0, 0, 0, 8, 6, 0, 0, 4055, 4096, 4089, 0, 4096, 0, 4085, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 36, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 2, 0, 4096, 4096, 4096, 0, 4096, 31, 0, 4081, 0, 0, 0, 0, 87, 2, 0, 0, 0, 0, 0, 0, 374, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 3939, 4096, 4096, 0, 10, 4096, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 3329, 39, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 202, 0, 0, 3820, 4090, 4083, 0, 4096, 0, 4094, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 523, 0, 0, 4058, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 44, 0, 4040, 0, 0, 0, 0, 1382, 121, 0, 0, 3, 127, 0, 0, 449, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2412, 4096, 4096, 0, 7, 4096, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 3330, 20, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 0, 0, 18, 0, 0, 4090, 4091, 4090, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 9, 0, 4077, 0, 0, 0, 0, 13, 1, 0, 0, 0, 96, 0, 0, 763, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3995, 4096, 4096, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 683, 0, 4095, 1, 0, 3626, 0, 2, 2, 0, 4095, 0, 1, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4095, 0, 1, 0, 0, 4094, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4091, 4093, 1, 0, 0, 4095, 64, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 4089, 4096, 1, 0, 0, 0, 0, 1, 2, 3412, 387, 198, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 3, 4095, 0, 1, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 4095, 0, 0, 0, 4, 0, 2, 0, 1, 0, 40, 0, 0, 4095, 3833, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2000, 0, 1, 1, 0, 4095, 1, 0, 1, 1, 2522, 4095, 0, 6, 0, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 2, 1, 1, 4095, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16601, 16355, 16440, 16307, 16384, 16166, 16431, 16388] avg=16384.0000 max=16601 max_rank=0 min=16166 min_rank=5 imbalance_ratio=1.013245 max_load_reduction=114471 max_load_reduction_pct=0.873344 moved_to_echo=65369 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 683, 0, 4096, 0, 0, 2469, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4096, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 4088, 4096, 0, 0, 0, 4096, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 26, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 3413, 388, 199, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 4085, 0, 0, 0, 0, 15, 6, 0, 0, 0, 73, 0, 0, 4096, 3834, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1590, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4074, 4094, 0, 0, 0, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15440, 16353, 16393, 16332, 16384, 16218, 17562, 16390] avg=16384.0000 max=17562 max_rank=6 min=15440 min_rank=0 imbalance_ratio=1.071899 max_load_reduction=113510 max_load_reduction_pct=0.866013 moved_to_echo=66554 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 683, 0, 4096, 0, 0, 3487, 9, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4082, 4089, 0, 0, 0, 4096, 63, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3413, 388, 199, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 4093, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 4096, 3834, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 624, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4052, 4096, 0, 0, 0, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16467, 16372, 16397, 16332, 16384, 16158, 16576, 16386] avg=16384.0000 max=16576 max_rank=6 min=16158 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=114496 max_load_reduction_pct=0.873535 moved_to_echo=65504 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 678, 0, 4096, 0, 0, 3921, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3418, 389, 201, 28, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4, 0, 0, 4096, 3837, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4082, 4096, 0, 0, 0, 3707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16887, 16356, 16442, 16326, 16382, 16138, 16158, 16383] avg=16384.0000 max=16887 max_rank=0 min=16138 min_rank=5 imbalance_ratio=1.030701 max_load_reduction=114185 max_load_reduction_pct=0.871162 moved_to_echo=65061 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 683, 0, 4096, 0, 0, 3746, 10, 0, 2, 0, 4096, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4094, 0, 0, 12, 0, 4066, 4096, 0, 0, 0, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 4096, 4089, 0, 0, 0, 4085, 63, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 36, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3413, 388, 199, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 31, 0, 4081, 0, 0, 0, 0, 87, 2, 0, 0, 0, 0, 0, 0, 4096, 3834, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 3939, 4096, 0, 0, 10, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16705, 16351, 16389, 16351, 16386, 16228, 16228, 16434] avg=16384.0000 max=16705 max_rank=0 min=16228 min_rank=5 imbalance_ratio=1.019592 max_load_reduction=114367 max_load_reduction_pct=0.872551 moved_to_echo=65276 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 683, 0, 4096, 0, 0, 3330, 20, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4096, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4091, 4090, 0, 0, 0, 4096, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3413, 388, 199, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 4077, 0, 0, 0, 0, 13, 1, 0, 0, 0, 96, 0, 0, 4096, 3834, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 763, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3995, 4096, 0, 0, 2, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16319, 16357, 16430, 16315, 16384, 16223, 16660, 16384] avg=16384.0000 max=16660 max_rank=6 min=16223 min_rank=5 imbalance_ratio=1.016846 max_load_reduction=114412 max_load_reduction_pct=0.872894 moved_to_echo=65651 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 683, 0, 4096, 0, 0, 2712, 630, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 2, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 4083, 0, 0, 0, 4093, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 3413, 388, 198, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 73, 0, 0, 0, 0, 5, 0, 0, 4096, 3832, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 728, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 4060, 4096, 0, 0, 1, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16314, 16368, 16430, 16309, 16384, 16197, 16689, 16381] avg=16384.0000 max=16689 max_rank=6 min=16197 min_rank=5 imbalance_ratio=1.018616 max_load_reduction=114383 max_load_reduction_pct=0.872673 moved_to_echo=65651 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 683, 0, 4096, 0, 0, 3329, 39, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3820, 4090, 4083, 0, 0, 0, 4094, 63, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 523, 0, 0, 4058, 4096, 0, 0, 0, 0, 0, 0, 0, 3413, 388, 199, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 44, 0, 4040, 0, 0, 0, 0, 1382, 121, 0, 0, 3, 127, 0, 0, 4096, 3834, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2412, 4096, 0, 0, 7, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16339, 16560, 16151, 16800, 16384, 17744, 14768, 16326] avg=16384.0000 max=17744 max_rank=5 min=14768 min_rank=6 imbalance_ratio=1.083008 max_load_reduction=113328 max_load_reduction_pct=0.864624 moved_to_echo=65222 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 5504, 0, 32768, 128, 0, 26624, 768, 128, 128, 128, 32768, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 32640, 32768, 0, 128, 128, 0, 32768, 32768, 128, 0, 0, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 32768, 0, 0, 0, 128, 1664, 256, 0, 128, 128, 384, 0, 0, 32768, 30720, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32384, 32768, 32768, 128, 0, 0, 32768, 512, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 32768, 128, 32768, 0, 128, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 32768, 0, 0, 0, 0, 128, 0, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 0, 0, 640, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 128, 128, 27392, 3200, 1664, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[6784, 128, 128, 128, 0, 32768, 128, 128, 128, 128, 29184, 32768, 0, 128, 128, 29696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4087, 41, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 8, 1389, 1, 0, 4096, 4096, 4088, 0, 0, 96, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4062, 0, 0, 0, 4096, 0, 4096, 4, 0, 4096, 0, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 162, 0, 4092, 4096, 4096, 4096, 0, 2439, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 27, 4096, 0, 0, 0, 0, 0, 3, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3430, 0, 0, 4096, 4096, 4096, 0, 0, 19, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 4096, 22, 4066, 25, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 568, 0, 4095, 4096, 4096, 4096, 0, 49, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4085, 3, 4083, 2, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3775, 0, 0, 4096, 4096, 4055, 0, 0, 86, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4089, 0, 9, 0, 0, 4096, 0, 1, 0, 0, 0, 71, 0, 0, 0, 4096, 0, 0, 0, 4096, 53, 4060, 1, 0, 4096, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 31, 4034, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 31, 4096, 0, 4096, 0, 0, 0, 0, 54, 3840, 4096, 4096, 4080, 0, 412, 0, 4096, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 194, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3463, 0, 0, 4096, 4096, 4093, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4088, 0, 0, 0, 4096, 0, 4096, 63, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4091, 4096, 4096, 4096, 0, 390, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 2, 4094, 0, 0, 4094, 4096, 124, 4095, 4, 0, 1, 1, 0, 2, 0, 4092, 4095, 0, 4095, 0, 1, 0, 1, 0, 4094, 0, 2445, 0, 0, 4095, 4095, 4090, 0, 0, 0, 4093, 4, 0, 1, 0, 0, 1, 0, 0, 4093, 0, 4094, 1, 20, 4095, 0, 0, 0, 1, 1, 1, 0, 4095, 4092, 1, 0, 1, 0, 4094, 1, 0, 0, 2, 0, 5, 1, 0, 0, 4095, 0, 1, 1, 4095, 18, 4088, 1023, 0, 4095, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 4074, 0, 1, 2, 0, 1, 0, 4095, 1, 1, 0, 4091, 0, 4094, 2, 0, 0, 12, 0, 4067, 4091, 4095, 4095, 0, 518, 1, 4094, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 2, 4092, 7, 0, 0, 0, 0, 5, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 33, 431, 0, 0, 4096, 4096, 4096, 0, 0, 198, 4096, 52, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 85, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 4096, 0, 4073, 8, 0, 4096, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 36, 3967, 1, 0, 34, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 179, 4, 4088, 4096, 4096, 4096, 0, 3251, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 11, 3867, 0, 0, 4096, 4096, 4096, 0, 0, 47, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4083, 3, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 65, 1, 4088, 4096, 4096, 4096, 0, 113, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 145, 4094, 2, 0, 0, 0, 0, 1, 2, 4094, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 670, 0, 0, 4096, 4096, 4095, 0, 0, 20, 4096, 264, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 4076, 6, 0, 4096, 0, 0, 4, 0, 8, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 85, 3, 4095, 4096, 4096, 4095, 0, 2918, 0, 4096, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4032, 0, 0, 4096, 4087, 41, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1656, 0, 0, 0, 0, 0, 4096, 8, 1389, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 4096, 4088, 0, 0, 96, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4062, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4, 0, 4096, 0, 7, 4, 0, 0, 0, 0, 0, 0, 2440, 1361, 86, 56, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 4092, 4096, 0, 4096, 0, 2439, 0, 2735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16352, 15342, 16374, 16385, 16362, 16251, 16386, 17620] avg=16384.0000 max=17620 max_rank=7 min=15342 min_rank=1 imbalance_ratio=1.075439 max_load_reduction=113452 max_load_reduction_pct=0.865570 moved_to_echo=66619 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4032, 0, 0, 4096, 4096, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 4096, 0, 1656, 0, 0, 0, 0, 0, 4096, 0, 3430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 4096, 4096, 0, 0, 19, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 22, 4066, 25, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2440, 1361, 86, 56, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 0, 4095, 4096, 0, 4096, 0, 49, 0, 2735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16347, 17377, 16322, 16384, 16355, 16253, 16395, 15639] avg=16384.0000 max=17377 max_rank=1 min=15639 min_rank=7 imbalance_ratio=1.060608 max_load_reduction=113695 max_load_reduction_pct=0.867424 moved_to_echo=64642 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4032, 0, 0, 4096, 4085, 3, 4083, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 1656, 0, 0, 0, 0, 0, 4096, 0, 3775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 4096, 4055, 0, 0, 86, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 1, 0, 0, 0, 71, 0, 0, 0, 4096, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 53, 4060, 1, 0, 4096, 0, 0, 14, 0, 0, 0, 0, 0, 0, 2440, 1361, 86, 56, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4034, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 31, 4096, 0, 4096, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 3840, 4096, 0, 4080, 0, 412, 0, 2735, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16301, 17717, 16322, 16376, 16460, 16268, 16409, 15219] avg=16384.0000 max=17717 max_rank=1 min=15219 min_rank=7 imbalance_ratio=1.081360 max_load_reduction=113355 max_load_reduction_pct=0.864830 moved_to_echo=64356 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 2, 4030, 0, 0, 4094, 4096, 124, 4095, 4, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 4095, 0, 1655, 0, 1, 0, 1, 0, 4094, 0, 2445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 4095, 4090, 0, 0, 0, 4093, 4, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4093, 1, 20, 0, 0, 0, 0, 1, 1, 1, 0, 4095, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 1, 0, 0, 2, 0, 5, 1, 0, 0, 4095, 0, 1, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 18, 4088, 1023, 0, 4095, 1, 0, 0, 0, 0, 2, 0, 0, 0, 2440, 1361, 86, 56, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4074, 0, 1, 2, 0, 1, 0, 4095, 1, 1, 0, 4091, 0, 4094, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 12, 0, 4067, 4091, 0, 4095, 0, 518, 1, 2733, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16449, 16385, 16293, 16398, 16386, 17270, 16371, 15520] avg=16384.0000 max=17270 max_rank=5 min=15520 min_rank=7 imbalance_ratio=1.054077 max_load_reduction=113802 max_load_reduction_pct=0.868240 moved_to_echo=65547 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4032, 0, 0, 4096, 4094, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1656, 0, 0, 0, 0, 0, 4096, 11, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 4096, 4096, 0, 0, 47, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 3, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2440, 1361, 86, 56, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 1, 4088, 4096, 0, 4096, 0, 113, 0, 2735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16326, 17822, 16341, 16383, 16383, 16231, 16392, 15194] avg=16384.0000 max=17822 max_rank=1 min=15194 min_rank=7 imbalance_ratio=1.087769 max_load_reduction=113250 max_load_reduction_pct=0.864029 moved_to_echo=64200 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4025, 0, 0, 4096, 4096, 194, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1649, 0, 0, 0, 0, 0, 4096, 0, 3463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 4096, 4093, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4088, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 63, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2447, 1367, 92, 62, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 4096, 0, 390, 0, 2729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16507, 17400, 16290, 16382, 16372, 16324, 16395, 15402] avg=16384.0000 max=17400 max_rank=1 min=15402 min_rank=7 imbalance_ratio=1.062012 max_load_reduction=113672 max_load_reduction_pct=0.867249 moved_to_echo=64493 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4032, 0, 0, 4096, 4096, 2, 4092, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 4096, 0, 1656, 0, 0, 0, 0, 0, 4096, 33, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 4096, 4096, 0, 0, 198, 4096, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4073, 8, 0, 4096, 0, 0, 19, 0, 0, 0, 0, 0, 0, 2440, 1361, 86, 56, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 3967, 1, 0, 34, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 4, 4088, 4096, 0, 4096, 0, 3251, 0, 2735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16325, 14413, 16548, 16383, 16380, 16240, 16334, 18449] avg=16384.0000 max=18449 max_rank=7 min=14413 min_rank=1 imbalance_ratio=1.126038 max_load_reduction=112623 max_load_reduction_pct=0.859245 moved_to_echo=67403 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4032, 0, 0, 4096, 4096, 145, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4094, 4096, 0, 1656, 0, 0, 0, 0, 0, 4096, 1, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 4096, 4095, 0, 0, 20, 4096, 264, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 6, 0, 4096, 0, 0, 4, 0, 8, 0, 0, 0, 0, 2440, 1361, 86, 56, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 3, 4095, 4096, 0, 4095, 0, 2918, 0, 2735, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16465, 14616, 16582, 16381, 16374, 16235, 16390, 18029] avg=16384.0000 max=18029 max_rank=7 min=14616 min_rank=1 imbalance_ratio=1.100403 max_load_reduction=113043 max_load_reduction_pct=0.862450 moved_to_echo=67028 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 32768, 128, 128, 0, 0, 0, 0, 128, 128, 128, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 32640, 1152, 0, 32768, 128, 128, 128, 0, 128, 128, 0, 0, 0, 19584, 11008, 768, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 32640, 128, 128, 128, 0, 128, 0, 32768, 128, 128, 128, 32768, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 0, 128, 0, 128, 128, 0, 0, 32640, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 32256, 0, 0, 32768, 32768, 640, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 128, 32768, 32768, 0, 13312, 0, 128, 0, 128, 0, 32768, 128, 19584, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 0, 1152, 128, 32512, 32768, 0, 32768, 0, 10112, 128, 21888, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 32128, 32768, 32768, 0, 0, 512, 32768, 384, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 4095, 0, 0, 0, 1, 4094, 0, 1, 0, 0, 0, 0, 0, 1, 0, 4089, 1, 4095, 4095, 492, 1, 4092, 3748, 88, 0, 0, 4094, 4095, 1, 4095, 1, 4095, 1, 1, 1, 0, 4095, 0, 4095, 4094, 1, 0, 4095, 1, 0, 0, 0, 1, 1, 1, 1, 4093, 0, 0, 0, 0, 0, 1, 0, 4094, 1, 267, 4095, 1, 0, 0, 1, 0, 1, 3, 4038, 4095, 1, 1, 0, 4089, 1, 1, 0, 4095, 0, 0, 1, 0, 1, 2, 1, 0, 1, 1, 4095, 4, 2, 0, 1, 4095, 4094, 1, 0, 0, 8, 0, 0, 0, 0, 0, 1, 3651, 4095, 4095, 0, 1, 0, 0, 4095, 0, 4095, 0, 0, 4092, 4, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 6, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 256, 0, 4096, 3847, 21, 0, 0, 4096, 4062, 0, 4075, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 21, 4096, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 13, 4096, 4090, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4082, 0, 10, 0, 59, 0, 11, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 4035, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 4096, 919, 0, 4096, 3439, 44, 0, 0, 4096, 4096, 0, 4087, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 3834, 4096, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 8, 0, 14, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4051, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 189, 0, 4096, 3957, 12, 0, 0, 4096, 4095, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 4035, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 426, 0, 4096, 3795, 1, 0, 0, 4096, 4094, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4, 0, 0, 0, 8, 0, 0, 0, 0, 3971, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4087, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 2599, 0, 4096, 1743, 66, 0, 0, 4096, 3990, 0, 3969, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 1, 4096, 0, 176, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 45, 0, 83, 0, 0, 0, 0, 3847, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4060, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 597, 0, 4096, 3695, 9, 0, 0, 4096, 4090, 0, 4094, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 107, 4096, 0, 0, 0, 0, 0, 0, 5, 3820, 4095, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 2, 2, 0, 0, 4096, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4092, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 390, 0, 4092, 4029, 25, 0, 0, 4096, 4095, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 75, 4096, 0, 0, 0, 0, 0, 0, 6, 4088, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3716, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4070, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 4095, 0, 0, 0, 1, 4094, 0, 1, 0, 0, 0, 0, 3930, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4089, 1, 0, 0, 492, 1, 3913, 3748, 88, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 21, 1, 1, 1, 0, 4095, 0, 0, 4094, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 4093, 0, 0, 0, 0, 0, 1, 0, 4094, 4095, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 267, 4095, 1, 0, 0, 1, 0, 1, 3, 4038, 4095, 1, 1, 0, 4089, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 1, 0, 1, 2, 1, 0, 1, 1, 4095, 4, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4095, 4094, 1, 0, 0, 8, 0, 0, 0, 0, 0, 1, 3651, 4074, 141, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4095, 0, 1, 0, 0, 4095, 0, 4095, 0, 0, 4092, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16393, 16428, 16407, 16378, 16595, 16392, 16072, 16407] avg=16384.0000 max=16595 max_rank=4 min=16072 min_rank=6 imbalance_ratio=1.012878 max_load_reduction=114477 max_load_reduction_pct=0.873390 moved_to_echo=65466 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3931, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 919, 0, 3916, 3439, 44, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 21, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 3834, 4096, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 8, 0, 14, 0, 0, 0, 0, 4096, 4075, 141, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 16502, 16396, 16382, 16122, 16382, 16528, 16363] avg=16384.0000 max=16528 max_rank=6 min=16122 min_rank=4 imbalance_ratio=1.008789 max_load_reduction=114544 max_load_reduction_pct=0.873901 moved_to_echo=65395 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 6, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 3931, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 256, 0, 3916, 3847, 21, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 21, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 21, 4096, 4096, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 13, 4096, 4090, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 10, 0, 59, 0, 11, 0, 0, 0, 0, 4096, 4075, 141, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4096, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16404, 16196, 16375, 16403, 16395, 16382, 16574, 16343] avg=16384.0000 max=16574 max_rank=6 min=16196 min_rank=1 imbalance_ratio=1.011597 max_load_reduction=114498 max_load_reduction_pct=0.873550 moved_to_echo=65694 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3931, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 426, 0, 3916, 3795, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4, 0, 0, 0, 8, 0, 0, 0, 0, 3971, 4075, 141, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 16326, 16405, 16381, 16386, 16383, 16395, 16399] avg=16384.0000 max=16405 max_rank=2 min=16326 min_rank=1 imbalance_ratio=1.001282 max_load_reduction=114667 max_load_reduction_pct=0.874840 moved_to_echo=65563 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3934, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 189, 0, 3910, 3957, 12, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 4035, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4076, 144, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16403, 16259, 16404, 16383, 16341, 16382, 16498, 16402] avg=16384.0000 max=16498 max_rank=6 min=16259 min_rank=1 imbalance_ratio=1.006958 max_load_reduction=114574 max_load_reduction_pct=0.874130 moved_to_echo=65623 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 3931, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 390, 0, 3913, 4029, 25, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 4096, 0, 0, 0, 0, 0, 0, 6, 4088, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3716, 4075, 141, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16548, 16405, 16381, 16459, 16383, 16134, 16382] avg=16384.0000 max=16548 max_rank=1 min=16134 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=114524 max_load_reduction_pct=0.873749 moved_to_echo=65358 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 3931, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2599, 0, 3916, 1743, 66, 0, 0, 0, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 0, 21, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 1, 4096, 4096, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 45, 0, 83, 0, 0, 0, 0, 3847, 4075, 141, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16300, 16410, 16277, 16385, 16560, 16382, 16386, 16372] avg=16384.0000 max=16560 max_rank=4 min=16277 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=114512 max_load_reduction_pct=0.873657 moved_to_echo=65700 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 3931, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 597, 0, 3916, 3695, 9, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 21, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4094, 4096, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 4096, 0, 0, 0, 0, 0, 0, 5, 3820, 4095, 0, 0, 0, 4089, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4075, 141, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16398, 16403, 16403, 16379, 16214, 16386, 16485, 16404] avg=16384.0000 max=16485 max_rank=6 min=16214 min_rank=4 imbalance_ratio=1.006165 max_load_reduction=114587 max_load_reduction_pct=0.874229 moved_to_echo=65489 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 0, 128, 32768, 0, 0, 0, 128, 32768, 0, 128, 0, 128, 0, 0, 31488, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 32768, 32768, 128, 128, 0, 128, 0, 128, 0, 0, 0, 128, 31616, 32640, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32640, 128, 256, 128, 128, 128, 0, 32768, 0, 0, 32768, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 128, 0, 32768, 128, 0, 0, 5888, 128, 31360, 28288, 384, 0, 0, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[256, 32768, 0, 128, 0, 0, 32768, 0, 32768, 0, 0, 32640, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 768, 32768, 128, 0, 0, 128, 0, 128, 128, 32128, 32768, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 128, 128, 128, 128, 128, 0, 128, 128, 32768, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 0, 128, 128, 128, 128, 32768, 128, 0, 128, 0, 0, 128, 128, 32768, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 4095, 4095, 4094, 0, 0, 4094, 0, 1, 4094, 4095, 0, 0, 0, 0, 4, 0, 4093, 0, 0, 0, 1, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 4095, 0, 1, 1, 0, 18, 0, 4095, 0, 0, 0, 0, 3, 1, 4095, 4093, 0, 3, 4091, 0, 1, 2, 4095, 0, 1, 4094, 4095, 1, 0, 2, 1, 0, 0, 1, 4095, 1, 0, 1, 0, 4095, 4095, 0, 2, 1, 1, 0, 4095, 1, 0, 4095, 4095, 4095, 0, 0, 4094, 1, 0, 1, 0, 0, 4094, 4090, 0, 0, 0, 0, 0, 4079, 1, 0, 0, 4095, 0, 1, 4, 0, 0, 1, 0, 0, 4095, 4094, 0, 2, 1, 4094, 4095, 0, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 3987, 4096, 0, 0, 4096, 0, 0, 4096, 3982, 0, 0, 0, 98, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 394, 0, 0, 0, 0, 4096, 4096, 14, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4065, 0, 0, 4096, 0, 139, 0, 0, 0, 4095, 3693, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 4060, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4070, 4096, 0, 0, 4096, 0, 0, 4096, 4073, 3, 0, 0, 107, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 26, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4089, 0, 27, 0, 0, 0, 0, 4096, 4096, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 2, 0, 4096, 0, 14, 0, 0, 67, 4096, 4027, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3976, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4090, 4096, 0, 0, 4096, 0, 0, 4096, 4066, 0, 0, 0, 103, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4083, 0, 0, 4096, 0, 32, 0, 0, 1, 4096, 4086, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 4096, 0, 0, 0, 4096, 3991, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 15, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4078, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 37, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4094, 0, 0, 4096, 0, 32, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4022, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 4093, 0, 0, 0, 2, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 5, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 4090, 0, 0, 4096, 0, 93, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4003, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4084, 4096, 0, 0, 4096, 0, 0, 4096, 4091, 0, 0, 0, 65, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 29, 0, 4096, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 4066, 0, 0, 4096, 0, 9, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4028, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 4094, 0, 1, 4088, 4095, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 1, 4095, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4095, 0, 1, 1, 0, 18, 0, 4095, 0, 0, 4095, 4015, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 4093, 0, 3, 4091, 0, 1, 2, 4094, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 2, 1, 0, 0, 1, 4095, 1, 0, 1, 0, 4095, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 4064, 1, 0, 0, 4095, 4095, 0, 0, 4094, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4090, 0, 0, 0, 0, 0, 4079, 1, 0, 0, 4095, 0, 1, 31, 16, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 4095, 4094, 0, 2, 1, 4094, 4095, 0, 1, 1, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16368, 16337, 16383, 16388, 16355, 16413, 16451] avg=16384.0000 max=16451 max_rank=7 min=16337 min_rank=2 imbalance_ratio=1.004089 max_load_reduction=114621 max_load_reduction_pct=0.874489 moved_to_echo=65607 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 4096, 0, 0, 4090, 4066, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 4096, 4017, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 4096, 4083, 0, 0, 4096, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4086, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 31, 16, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 4096, 0, 0, 0, 4096, 3991, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16445, 16382, 16329, 16383, 16385, 16372, 16428, 16348] avg=16384.0000 max=16445 max_rank=0 min=16329 min_rank=2 imbalance_ratio=1.003723 max_load_reduction=114627 max_load_reduction_pct=0.874535 moved_to_echo=65533 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4070, 0, 0, 0, 4096, 0, 0, 4090, 4073, 3, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 26, 4, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4089, 0, 27, 4096, 4017, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 4096, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 4096, 4096, 2, 0, 4096, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 4096, 4027, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 31, 17, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3976, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16439, 16403, 16339, 16388, 16370, 16369, 16436, 16328] avg=16384.0000 max=16439 max_rank=0 min=16328 min_rank=7 imbalance_ratio=1.003357 max_load_reduction=114633 max_load_reduction_pct=0.874580 moved_to_echo=65503 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3987, 0, 0, 0, 4096, 0, 0, 4090, 3982, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 46, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 394, 4096, 4017, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 4096, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 4096, 4065, 0, 0, 4096, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3693, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 31, 17, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 4060, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16253, 16418, 16712, 16397, 16385, 16461, 16035, 16411] avg=16384.0000 max=16712 max_rank=2 min=16035 min_rank=6 imbalance_ratio=1.020020 max_load_reduction=114360 max_load_reduction_pct=0.872498 moved_to_echo=65292 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4084, 0, 0, 0, 4096, 0, 0, 4090, 4091, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 29, 0, 4096, 0, 6, 4096, 4017, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 4094, 4066, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 31, 16, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4028, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16426, 16377, 16355, 16383, 16385, 16330, 16437, 16379] avg=16384.0000 max=16437 max_rank=6 min=16330 min_rank=5 imbalance_ratio=1.003235 max_load_reduction=114635 max_load_reduction_pct=0.874596 moved_to_echo=65531 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 1, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 37, 4096, 4017, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 4096, 4094, 0, 0, 4096, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 31, 16, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4022, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 16376, 16357, 16383, 16385, 16383, 16437, 16373] avg=16384.0000 max=16437 max_rank=6 min=16357 min_rank=2 imbalance_ratio=1.003235 max_load_reduction=114635 max_load_reduction_pct=0.874596 moved_to_echo=65578 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4088, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 4018, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 36, 16, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4078, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16370, 16322, 16373, 16390, 16363, 16449, 16428] avg=16384.0000 max=16449 max_rank=6 min=16322 min_rank=2 imbalance_ratio=1.003967 max_load_reduction=114623 max_load_reduction_pct=0.874504 moved_to_echo=65630 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4090, 4093, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 5, 4096, 4017, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 4095, 4090, 0, 0, 4096, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 31, 16, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4003, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16378, 16321, 16382, 16384, 16439, 16437, 16354] avg=16384.0000 max=16439 max_rank=5 min=16321 min_rank=2 imbalance_ratio=1.003357 max_load_reduction=114633 max_load_reduction_pct=0.874580 moved_to_echo=65614 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 0, 128, 32768, 128, 0, 128, 0, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 0, 0, 0, 128, 0, 0, 128, 128, 128, 32768, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 32384, 0, 0, 0, 128, 0, 32768, 128, 0, 0, 32768, 0, 128, 256, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 32768, 128, 128, 32768, 0, 128, 128, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 32640, 0, 0, 0, 32768, 0, 128, 32768, 32640, 128, 0, 0, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 0, 32640, 128, 0, 0, 32768, 32768, 128, 0, 32768, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 128, 0, 32768, 32768, 0, 128, 128, 32768, 32256, 0, 128, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 0, 0, 32768, 0, 128, 128, 0, 128, 0, 32768, 0, 512, 32768, 32256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 274, 4096, 0, 0, 0, 3987, 0, 0, 0, 0, 2, 0, 23, 0, 0, 0, 4096, 0, 4096, 18, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 47, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4026, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4086, 0, 0, 0, 4096, 0, 4096, 4090, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 132, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4084, 0, 0, 0, 4096, 0, 3833, 0, 0, 0, 0, 0, 0, 4071, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 173, 4096, 0, 0, 0, 3785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4047, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4052, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 0, 0, 4096, 0, 333, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 122, 3924, 0, 0, 0, 4096, 0, 4039, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 20, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4094, 0, 0, 1, 4095, 1, 81, 4095, 0, 1, 0, 4089, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 1, 4095, 0, 0, 1, 1, 1, 4093, 1, 0, 0, 0, 4086, 0, 0, 0, 1, 0, 4083, 3, 1, 4095, 0, 4095, 0, 0, 0, 0, 1, 0, 4094, 0, 2, 4095, 4095, 1, 1, 4094, 4, 1, 4095, 4094, 4091, 1, 0, 0, 1, 1, 3, 0, 4093, 1, 4095, 1, 4095, 0, 1, 1, 4095, 1, 4095, 3700, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 5, 0, 1, 4095, 0, 333, 0, 0, 0, 0, 4094, 0, 1, 2, 1, 1, 65, 4032, 0, 0, 0, 4094, 0, 4093, 0, 2, 1, 0, 1, 2, 4095, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 109, 4096, 0, 0, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4065, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 116, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 50, 4094, 0, 0, 0, 4096, 0, 4048, 0, 0, 0, 0, 0, 0, 4095, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 1111, 4096, 0, 0, 0, 3252, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3245, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4038, 0, 4096, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 4096, 0, 716, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 338, 3721, 0, 0, 0, 4096, 0, 4069, 0, 0, 0, 0, 0, 0, 4079, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 427, 4096, 0, 0, 0, 3668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4046, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4095, 1, 0, 0, 4096, 0, 4096, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4095, 0, 26, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 103, 4085, 0, 0, 0, 4096, 0, 4062, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 230, 4096, 0, 0, 0, 3973, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 139, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 60, 3873, 0, 0, 0, 4096, 0, 4034, 0, 0, 0, 0, 0, 0, 4093, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 274, 4040, 0, 0, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 23, 0, 0, 0, 4096, 0, 4096, 18, 0, 0, 0, 0, 4096, 0, 3913, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 47, 4092, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4026, 134, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4090, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 53, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 132, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4085, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4096, 0, 3833, 0, 0, 0, 0, 0, 0, 4071, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16493, 16420, 16431, 16448, 16385, 16438, 16196, 16261] avg=16384.0000 max=16493 max_rank=0 min=16196 min_rank=6 imbalance_ratio=1.006653 max_load_reduction=114579 max_load_reduction_pct=0.874168 moved_to_echo=65280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 173, 4040, 0, 0, 0, 3785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 3913, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4047, 0, 0, 4096, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4052, 134, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 53, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 333, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 122, 4085, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3924, 0, 0, 0, 4096, 0, 4039, 0, 0, 0, 0, 0, 0, 4096, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16190, 16379, 16338, 16474, 16385, 16456, 16518, 16332] avg=16384.0000 max=16518 max_rank=6 min=16190 min_rank=0 imbalance_ratio=1.008179 max_load_reduction=114554 max_load_reduction_pct=0.873978 moved_to_echo=65691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 1, 4095, 1, 81, 4039, 0, 1, 0, 4089, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 1, 4095, 0, 0, 1, 1, 1, 4093, 1, 3912, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 1, 0, 4083, 3, 1, 4095, 0, 4095, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 2, 4095, 134, 1, 1, 4094, 4, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4091, 1, 0, 0, 1, 1, 3, 0, 4093, 1, 4095, 1, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 4095, 3700, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 5, 53, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 333, 0, 0, 0, 0, 4094, 0, 1, 2, 1, 1, 65, 4084, 3785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 4094, 0, 4093, 0, 2, 1, 0, 1, 2, 4095, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16402, 16376, 16367, 16522, 16393, 16053, 16462, 16497] avg=16384.0000 max=16522 max_rank=3 min=16053 min_rank=5 imbalance_ratio=1.008423 max_load_reduction=114550 max_load_reduction_pct=0.873947 moved_to_echo=65405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 230, 4040, 0, 0, 0, 3973, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 3913, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 134, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 53, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 139, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 60, 4085, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3873, 0, 0, 0, 4096, 0, 4034, 0, 0, 0, 0, 0, 0, 4093, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16438, 16378, 16386, 16498, 16395, 16442, 16262, 16273] avg=16384.0000 max=16498 max_rank=3 min=16262 min_rank=6 imbalance_ratio=1.006958 max_load_reduction=114574 max_load_reduction_pct=0.874130 moved_to_echo=65372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 109, 4040, 0, 0, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3913, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4065, 134, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 53, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 116, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 50, 4085, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 4048, 0, 0, 0, 0, 0, 0, 4095, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16306, 16377, 16387, 16487, 16395, 16381, 16229, 16510] avg=16384.0000 max=16510 max_rank=7 min=16229 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=114562 max_load_reduction_pct=0.874039 moved_to_echo=65515 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 2, 4029, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3920, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 125, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 58, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4092, 3792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16319, 16387, 16390, 16508, 16387, 16443, 16097, 16541] avg=16384.0000 max=16541 max_rank=7 min=16097 min_rank=6 imbalance_ratio=1.009583 max_load_reduction=114531 max_load_reduction_pct=0.873802 moved_to_echo=65468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 1111, 4040, 0, 0, 0, 3252, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 3913, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3245, 134, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4038, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 53, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 716, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 338, 4085, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3721, 0, 0, 0, 4096, 0, 4069, 0, 0, 0, 0, 0, 0, 4079, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16597, 16378, 16387, 15667, 16337, 16447, 17117, 16142] avg=16384.0000 max=17117 max_rank=6 min=15667 min_rank=3 imbalance_ratio=1.044739 max_load_reduction=113955 max_load_reduction_pct=0.869408 moved_to_echo=66043 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 427, 4040, 0, 0, 0, 3668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3913, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4046, 134, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 53, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 26, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 103, 4085, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 0, 4062, 0, 0, 0, 0, 0, 0, 4096, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16327, 16377, 16386, 16468, 16395, 16412, 16191, 16516] avg=16384.0000 max=16516 max_rank=7 min=16191 min_rank=6 imbalance_ratio=1.008057 max_load_reduction=114556 max_load_reduction_pct=0.873993 moved_to_echo=65514 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 0, 128, 32768, 128, 2432, 32384, 0, 128, 0, 30848, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 0, 0, 32768, 128, 32768, 128, 0, 128, 128, 128, 32768, 128, 31360, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 0, 1920, 0, 0, 0, 0, 32768, 0, 128, 128, 128, 128, 768, 32768, 30336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 0, 128, 0, 32768, 128, 128, 32768, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 32768, 32384, 0, 0, 0, 128, 0, 0, 0, 0, 128, 32768, 128, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 0, 32768, 0, 128, 31744, 1152, 128, 128, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32000, 0, 0, 0, 32768, 0, 32384, 0, 128, 128, 0, 128, 128, 32768, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 0, 0, 128, 128, 128, 0, 32768, 128, 32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4094, 1, 0, 4095, 1, 4095, 4001, 0, 1, 0, 0, 0, 0, 1, 4095, 0, 0, 1, 4094, 4094, 1, 0, 4093, 14, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 4095, 4095, 0, 1, 0, 0, 1, 4095, 0, 0, 4094, 4095, 2, 1, 0, 0, 3, 4082, 0, 1, 4095, 4093, 0, 1, 0, 0, 0, 4095, 0, 2, 4092, 0, 1, 4092, 0, 1, 0, 1, 1, 4093, 4094, 2, 2, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 4039, 0, 1, 0, 4094, 4095, 0, 1, 4095, 4094, 0, 0, 1, 1, 1, 1, 4050, 0, 0, 4094, 4, 0, 2, 195, 0, 1, 2, 4094, 4095, 1, 1, 0, 1, 4095, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 4096, 4096, 0, 0, 4096, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4081, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 4061, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 56, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 3876, 0, 0, 206, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4093, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 23, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 2, 0, 34, 0, 0, 4096, 3, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4080, 0, 0, 6, 0, 0, 4096, 0, 0, 4083, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 21, 0, 0, 0, 4060, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4048, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 4096, 0, 0, 0, 120, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 2, 4094, 0, 4096, 4074, 0, 0, 38, 0, 2, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 4096, 0, 0, 0, 48, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 1, 0, 4095, 1, 16, 4001, 0, 1, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 4094, 1, 0, 4093, 14, 0, 1, 0, 1, 0, 0, 0, 4040, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 4095, 4095, 0, 1, 0, 0, 1, 4095, 0, 0, 4059, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 5, 2, 1, 0, 0, 3, 4082, 0, 1, 4095, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 4092, 0, 1, 4092, 0, 1, 0, 1, 1, 4093, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 4039, 0, 1, 0, 4094, 4092, 4090, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 20, 4094, 0, 0, 1, 1, 1, 1, 4050, 0, 0, 4094, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 195, 0, 1, 2, 4094, 4095, 1, 1, 0, 1, 4046, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16305, 16388, 16384, 16377, 16382, 16341, 16362, 16533] avg=16384.0000 max=16533 max_rank=7 min=16305 min_rank=0 imbalance_ratio=1.009094 max_load_reduction=114539 max_load_reduction_pct=0.873863 moved_to_echo=65618 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 16, 3876, 0, 0, 206, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4060, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4093, 4090, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 20, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 4047, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16379, 16383, 16388, 16387, 16395, 16400, 16354] avg=16384.0000 max=16400 max_rank=6 min=16354 min_rank=7 imbalance_ratio=1.000977 max_load_reduction=114672 max_load_reduction_pct=0.874878 moved_to_echo=65536 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 16, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4060, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4093, 4090, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 4096, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 4047, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16396, 16383, 16383, 16390, 16387, 16396, 16367, 16370] avg=16384.0000 max=16396 max_rank=0 min=16367 min_rank=6 imbalance_ratio=1.000732 max_load_reduction=114676 max_load_reduction_pct=0.874908 moved_to_echo=65520 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 16, 4081, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 4041, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4060, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 4095, 4091, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 20, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 4047, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16381, 16383, 16389, 16385, 16395, 16366, 16385] avg=16384.0000 max=16395 max_rank=5 min=16366 min_rank=6 imbalance_ratio=1.000671 max_load_reduction=114677 max_load_reduction_pct=0.874916 moved_to_echo=65531 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 16, 4096, 0, 0, 2, 0, 34, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12, 4041, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4060, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4080, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4083, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4093, 4090, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 4060, 4096, 0, 0, 0, 0, 4047, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16436, 16393, 16383, 16370, 16374, 16394, 16402, 16320] avg=16384.0000 max=16436 max_rank=0 min=16320 min_rank=7 imbalance_ratio=1.003174 max_load_reduction=114636 max_load_reduction_pct=0.874603 moved_to_echo=65490 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4096, 0, 0, 4096, 1, 0, 0, 1, 0, 0, 0, 0, 4041, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4060, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4093, 4090, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 4047, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16400, 16383, 16383, 16390, 16387, 16395, 16404, 16330] avg=16384.0000 max=16404 max_rank=6 min=16330 min_rank=7 imbalance_ratio=1.001221 max_load_reduction=114668 max_load_reduction_pct=0.874847 moved_to_echo=65516 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 7, 4048, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4062, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 4096, 4096, 4096, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 4045, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16343, 16387, 16390, 16383, 16384, 16375, 16367, 16443] avg=16384.0000 max=16443 max_rank=7 min=16343 min_rank=0 imbalance_ratio=1.003601 max_load_reduction=114629 max_load_reduction_pct=0.874550 moved_to_echo=65569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 2, 4094, 0, 16, 4074, 0, 0, 38, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4060, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4096, 4093, 4090, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4047, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16418, 16378, 16383, 16385, 16386, 16381, 16404, 16337] avg=16384.0000 max=16418 max_rank=0 min=16337 min_rank=7 imbalance_ratio=1.002075 max_load_reduction=114654 max_load_reduction_pct=0.874741 moved_to_echo=65508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 128, 256, 32768, 0, 0, 128, 128, 128, 128, 32768, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 32768, 32768, 128, 0, 32768, 128, 0, 128, 128, 128, 0, 0, 128, 32384, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 128, 128, 32768, 128, 128, 32384, 0, 128, 256, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 32768, 0, 128, 0, 32768, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 128, 32768, 0, 128, 32768, 0, 128, 0, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 0, 32768, 32768, 0, 128, 0, 0, 128, 32768, 0, 0, 32512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 128, 32768, 0, 128, 32768, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 512, 0, 128, 128, 32768, 32768, 128, 128, 0, 128, 32384, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4077, 1, 9, 0, 3947, 0, 4095, 0, 4094, 4095, 0, 0, 1, 2, 4095, 4095, 0, 0, 4088, 1, 1, 0, 0, 0, 3, 1, 1, 0, 4095, 2, 4092, 265, 0, 0, 4095, 0, 1, 4094, 0, 4094, 0, 1, 0, 1, 0, 42, 4093, 0, 1, 4094, 0, 3866, 2, 4, 0, 4095, 0, 4093, 0, 4093, 1, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 1, 0, 4095, 0, 6, 0, 0, 0, 0, 4094, 0, 0, 4087, 4095, 0, 0, 0, 2, 0, 0, 0, 4093, 4092, 1, 0, 0, 1, 1, 2, 1, 0, 1, 1, 1, 0, 2, 0, 1, 1, 1, 0, 88, 1, 2, 4095, 0, 0, 4095, 4095, 4094, 1, 4095, 1, 4094, 1, 3, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 2, 0, 4005, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 19, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 4096, 0, 4068, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4094, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 82, 0, 4058, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3969, 15, 39, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 8, 58, 4096, 0, 0, 4096, 0, 4066, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 51, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4038, 4096, 0, 0, 51, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4060, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3932, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 134, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 56, 4096, 0, 0, 4096, 0, 4068, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3947, 0, 30, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 4096, 0, 4077, 0, 23, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 103, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 4096, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4073, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 4096, 0, 0, 4096, 4092, 4096, 0, 4068, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 13, 0, 4079, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 30, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 74, 4096, 0, 0, 4096, 0, 4039, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 4095, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 12, 0, 4078, 0, 4096, 0, 4096, 4096, 1, 0, 0, 0, 4096, 4096, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 19, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 1, 0, 4096, 4096, 1, 0, 14, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 4095, 4096, 0, 4087, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4077, 1, 9, 0, 3947, 0, 0, 0, 4094, 0, 0, 0, 1, 2, 4095, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 1, 0, 0, 0, 3, 1, 1, 0, 4095, 2, 4092, 265, 4043, 23, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 1, 4094, 0, 4094, 0, 1, 0, 1, 0, 42, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 3866, 2, 4, 0, 0, 0, 4093, 0, 4093, 1, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4095, 0, 6, 0, 0, 0, 0, 4094, 4084, 4030, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4095, 0, 0, 0, 2, 0, 0, 0, 4093, 4092, 1, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 1, 0, 1, 1, 1, 0, 2, 0, 1, 1, 1, 0, 88, 4095, 4095, 4095, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 4095, 4095, 4094, 1, 4095, 1, 10, 1, 3, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16278, 16633, 16378, 16201, 16364, 16390, 16428, 16400] avg=16384.0000 max=16633 max_rank=1 min=16201 min_rank=3 imbalance_ratio=1.015198 max_load_reduction=114439 max_load_reduction_pct=0.873100 moved_to_echo=65582 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 2, 0, 4005, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 19, 4044, 23, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4068, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4086, 4031, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 4096, 4096, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4094, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16347, 16371, 16356, 16403, 16373, 16404, 16426, 16392] avg=16384.0000 max=16426 max_rank=6 min=16347 min_rank=0 imbalance_ratio=1.002563 max_load_reduction=114646 max_load_reduction_pct=0.874680 moved_to_echo=65595 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 13, 0, 4079, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 4044, 23, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4053, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 74, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4039, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4086, 4031, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4095, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16432, 16333, 16445, 16374, 16362, 16400, 16332, 16394] avg=16384.0000 max=16445 max_rank=2 min=16332 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=114627 max_load_reduction_pct=0.874535 moved_to_echo=65488 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3947, 0, 4044, 23, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4053, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4077, 0, 23, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 4096, 4086, 4031, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4073, 4096, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 4096, 4096, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 4096, 0, 4068, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16436, 16100, 16375, 16435, 16468, 16382, 16514, 16362] avg=16384.0000 max=16514 max_rank=6 min=16100 min_rank=1 imbalance_ratio=1.007935 max_load_reduction=114558 max_load_reduction_pct=0.874008 moved_to_echo=65726 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 82, 0, 4058, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3969, 15, 4044, 23, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 4053, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 8, 58, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4066, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4086, 4031, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 4096, 0, 0, 51, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4060, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16480, 16245, 16446, 16401, 16413, 16397, 16332, 16358] avg=16384.0000 max=16480 max_rank=0 min=16245 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=114592 max_load_reduction_pct=0.874268 moved_to_echo=65500 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3932, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 134, 4049, 29, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 56, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4068, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4088, 4032, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 4096, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16267, 16520, 16384, 16399, 16365, 16399, 16346, 16392] avg=16384.0000 max=16520 max_rank=1 min=16267 min_rank=0 imbalance_ratio=1.008301 max_load_reduction=114552 max_load_reduction_pct=0.873962 moved_to_echo=65502 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 12, 0, 4078, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 19, 4044, 23, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 4086, 4031, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 1, 0, 14, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 4096, 0, 4087, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16431, 16373, 16342, 16430, 16362, 16420, 16332, 16382] avg=16384.0000 max=16431 max_rank=0 min=16332 min_rank=6 imbalance_ratio=1.002869 max_load_reduction=114641 max_load_reduction_pct=0.874641 moved_to_echo=65496 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4, 0, 4057, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4069, 153, 5, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4094, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3979, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4094, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4, 0, 4057, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4069, 153, 4044, 23, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4053, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4086, 4031, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3979, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4096, 4096, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4094, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16401, 16497, 16346, 16429, 16365, 16280, 16362, 16392] avg=16384.0000 max=16497 max_rank=1 min=16280 min_rank=5 imbalance_ratio=1.006897 max_load_reduction=114575 max_load_reduction_pct=0.874138 moved_to_echo=65399 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 0, 32256, 0, 0, 0, 32768, 0, 128, 0, 128, 128, 32768, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 32512, 0, 128, 32768, 0, 32768, 0, 128, 0, 128, 128, 256, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 32768, 32640, 128, 0, 128, 128, 0, 0, 0, 32768, 32768, 128, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 32640, 128, 128, 0, 0, 0, 128, 128, 128, 0, 32768, 128, 32512, 640, 32384, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 32768, 0, 32384, 128, 128, 0, 0, 0, 32768, 0, 32768, 128, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 256, 128, 0, 32768, 0, 128, 0, 0, 0, 0, 32768, 32768, 32256, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 128, 0, 128, 0, 128, 128, 128, 0, 512, 32768, 32768, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 0, 0, 32768, 32768, 32768, 128, 32768, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True + [2026-08-30 05:04:59] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 742.6 | throughput per GPU (TFLOP/s/GPU): 448.9 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.033592E+01 | loss scale: 1.0 | grad norm: 17.797 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[8, 408, 1162, 0, 342, 0, 1002, 6, 2170, 151, 451, 2565, 1241, 533, 2, 1034, 446, 1046, 701, 728, 3854, 1943, 158, 45, 6, 1508, 238, 349, 1284, 7, 913, 1288, 4087, 0, 23, 1385, 2982, 17, 608, 2876, 2534, 1423, 298, 418, 0, 440, 456, 666, 1026, 1476, 1115, 52, 5, 6, 3408, 462, 2, 3012, 3169, 2571, 637, 361, 744, 0, 1174, 4060, 0, 1242, 105, 22, 1834, 1724, 1026, 550, 134, 2, 62, 620, 289, 494, 1896, 2062, 7, 3721, 29, 2064, 2310, 3860, 44, 15, 707, 1301, 2853, 353, 2647, 1, 668, 2046, 3273, 176, 3, 2158, 684, 1818, 3541, 1232, 5, 805, 949, 1269, 101, 750, 55, 862, 2098, 806, 5, 503, 295, 1253, 857, 671, 1476, 1396, 97, 12, 2151, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[4, 441, 1030, 1, 432, 4, 699, 17, 2310, 38, 269, 2038, 1196, 524, 0, 1600, 807, 756, 889, 1258, 3701, 2170, 218, 69, 11, 1691, 334, 173, 746, 14, 894, 1470, 4037, 0, 24, 768, 2564, 18, 1232, 1920, 3118, 1492, 267, 172, 1, 757, 513, 863, 811, 1164, 775, 127, 8, 4, 3246, 243, 0, 2480, 3236, 1968, 546, 366, 804, 8, 1101, 4010, 1, 544, 299, 42, 1584, 2326, 1539, 699, 228, 0, 184, 461, 420, 703, 1381, 1274, 15, 3759, 62, 1981, 2988, 3887, 133, 11, 715, 1266, 2782, 273, 2299, 5, 1056, 1520, 3808, 102, 0, 1955, 763, 1685, 3416, 1630, 1, 1405, 1599, 1540, 183, 611, 18, 441, 2635, 575, 39, 922, 124, 1912, 922, 991, 1479, 1557, 58, 1, 1816, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[5, 260, 690, 0, 372, 0, 875, 1, 2534, 105, 412, 2210, 1358, 561, 0, 1243, 423, 483, 456, 920, 3982, 2776, 158, 35, 1, 1642, 234, 65, 1080, 1, 1145, 1219, 4085, 0, 13, 1456, 2905, 16, 579, 2301, 2612, 1034, 287, 154, 0, 634, 253, 383, 950, 1404, 1547, 3, 2, 1, 3678, 271, 0, 2876, 3730, 2352, 567, 222, 681, 0, 719, 4089, 0, 1347, 115, 14, 1659, 1356, 1302, 652, 133, 1, 38, 503, 214, 1244, 2019, 1674, 6, 3871, 42, 1772, 2708, 3969, 50, 9, 895, 800, 3275, 582, 2209, 0, 877, 1758, 3905, 183, 0, 2265, 760, 2108, 3844, 1224, 0, 593, 1215, 2351, 75, 533, 92, 476, 2767, 593, 0, 429, 171, 1499, 464, 848, 842, 1515, 80, 0, 2096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[12, 240, 561, 0, 268, 0, 856, 1, 2384, 90, 456, 2521, 892, 298, 1, 853, 723, 534, 455, 576, 3934, 2171, 150, 13, 0, 1726, 310, 283, 1328, 10, 999, 1363, 4082, 1, 7, 1722, 2869, 13, 477, 2486, 3260, 1090, 227, 140, 3, 565, 380, 1037, 1006, 1324, 1229, 5, 6, 1, 3175, 203, 0, 2925, 3319, 2127, 508, 263, 875, 1, 540, 4003, 1, 956, 178, 45, 1740, 2321, 1534, 586, 177, 8, 48, 543, 357, 512, 1844, 997, 8, 3742, 38, 2802, 2290, 3993, 46, 29, 704, 1104, 2991, 429, 2089, 0, 952, 1599, 3723, 196, 0, 2525, 608, 2354, 3638, 1592, 3, 1127, 1261, 1412, 151, 717, 110, 590, 2942, 656, 21, 748, 272, 870, 1110, 1036, 1513, 1402, 57, 0, 1897, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[14, 294, 789, 0, 317, 0, 570, 4, 2485, 68, 200, 2533, 1584, 199, 4, 1002, 574, 354, 516, 662, 4011, 2765, 131, 24, 1, 1687, 241, 76, 666, 7, 1069, 1353, 4094, 5, 14, 952, 3258, 19, 514, 2062, 2878, 1195, 154, 183, 6, 608, 412, 951, 819, 1397, 1178, 58, 0, 3, 3564, 259, 0, 3170, 3568, 2494, 286, 555, 436, 0, 934, 4060, 0, 965, 149, 26, 2884, 2142, 1774, 563, 70, 1, 42, 320, 463, 318, 1937, 1475, 3, 3992, 37, 2602, 2014, 3996, 47, 7, 585, 1450, 3193, 366, 2559, 0, 781, 1545, 3870, 99, 1, 1730, 442, 1820, 3262, 1789, 1, 843, 1545, 1674, 175, 807, 49, 718, 2213, 341, 6, 525, 188, 1175, 973, 1069, 1489, 1765, 53, 0, 1856, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[18, 259, 1054, 0, 339, 2, 674, 1, 2449, 47, 261, 2074, 1123, 325, 2, 1257, 820, 456, 900, 620, 3888, 1773, 181, 35, 1, 1526, 197, 316, 1421, 17, 740, 1351, 4009, 1, 27, 1076, 3449, 20, 631, 2656, 3060, 1340, 213, 223, 6, 671, 453, 1011, 1010, 1734, 983, 66, 8, 8, 3564, 196, 0, 2943, 3433, 2276, 701, 275, 792, 0, 1246, 4037, 0, 863, 174, 141, 1750, 1531, 1463, 505, 196, 13, 52, 369, 261, 360, 1344, 1930, 0, 3670, 103, 2298, 2350, 4001, 57, 5, 585, 1147, 2814, 133, 2246, 0, 1121, 1523, 3526, 180, 1, 2242, 747, 2228, 3883, 1458, 1, 1023, 1036, 1808, 48, 952, 71, 589, 2453, 642, 9, 546, 217, 1133, 836, 1187, 1324, 1583, 107, 1, 1989, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2, 182, 819, 0, 392, 0, 911, 0, 1900, 84, 189, 2262, 794, 797, 0, 1405, 787, 544, 341, 577, 3968, 2868, 148, 49, 1, 1848, 196, 222, 870, 0, 997, 1285, 4096, 5, 13, 732, 2930, 14, 638, 2533, 2688, 972, 223, 120, 0, 635, 301, 1029, 1070, 1581, 1388, 3, 0, 0, 3667, 165, 0, 3228, 3342, 2394, 177, 308, 406, 0, 848, 4048, 0, 936, 109, 11, 2157, 1678, 1477, 456, 70, 0, 84, 720, 754, 405, 1922, 1068, 1, 3901, 111, 2697, 2421, 3984, 47, 2, 1006, 975, 3147, 315, 2791, 0, 1426, 1327, 3063, 203, 0, 2014, 608, 2345, 3973, 1445, 4, 754, 1171, 1389, 99, 831, 64, 453, 2753, 826, 0, 823, 200, 1138, 800, 1091, 1740, 1580, 96, 0, 1618, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 166, 838, 2, 293, 0, 640, 0, 2376, 169, 234, 2508, 912, 577, 0, 1661, 788, 958, 620, 808, 3932, 2619, 148, 13, 0, 1641, 241, 132, 973, 7, 788, 1436, 4074, 5, 7, 1039, 3169, 22, 360, 2456, 2277, 1264, 112, 132, 0, 781, 490, 513, 868, 1524, 1046, 24, 5, 5, 3835, 293, 0, 2996, 3465, 2603, 500, 267, 647, 5, 1068, 4078, 0, 870, 214, 14, 1556, 2398, 1601, 717, 162, 1, 60, 420, 375, 368, 1853, 989, 6, 3878, 29, 2199, 1971, 4031, 30, 7, 458, 1996, 3242, 308, 2966, 4, 843, 1230, 3736, 195, 0, 2370, 516, 2195, 3599, 1203, 4, 810, 1267, 1847, 172, 836, 74, 421, 2193, 758, 13, 593, 284, 1346, 748, 911, 1563, 1190, 37, 3, 1975, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[8, 408, 1162, 0, 342, 0, 1002, 6, 2170, 151, 451, 2565, 1241, 533, 2, 1034, 3860, 2132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 446, 1046, 701, 728, 3854, 1943, 158, 45, 6, 1508, 238, 349, 1284, 7, 913, 1288, 872, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2865, 0, 23, 1385, 2982, 17, 608, 2876, 2534, 1423, 298, 418, 0, 440, 456, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 1476, 1115, 52, 5, 6, 2246, 462, 2, 3012, 3169, 2571, 637, 361, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1174, 4060, 0, 1242, 105, 22, 1834, 1724, 1026, 550, 134, 2, 62, 620, 289, 494, 1222, 1006, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1896, 2062, 7, 530, 29, 2064, 2310, 0, 44, 15, 707, 1301, 2853, 353, 2647, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 2046, 2401, 176, 3, 2158, 684, 1818, 0, 1232, 5, 805, 949, 1269, 101, 750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 862, 2098, 806, 5, 503, 295, 1253, 857, 671, 1476, 1396, 97, 12, 2151, 1, 2185, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17067, 16145, 16991, 16884, 15969, 16819, 15065, 16132] avg=16384.0000 max=17067 max_rank=0 min=15065 min_rank=6 imbalance_ratio=1.041687 max_load_reduction=114005 max_load_reduction_pct=0.869789 moved_to_echo=63985 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 441, 1030, 1, 432, 4, 699, 17, 2310, 38, 269, 2038, 1196, 524, 0, 1600, 3887, 2057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 807, 756, 889, 1258, 3701, 2170, 218, 69, 11, 1691, 334, 173, 746, 14, 894, 1470, 1015, 723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2830, 0, 24, 768, 2564, 18, 1232, 1920, 3118, 1492, 267, 172, 1, 757, 513, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 811, 1164, 775, 127, 8, 4, 2139, 243, 0, 2480, 3236, 1968, 546, 366, 804, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1101, 4010, 1, 544, 299, 42, 1584, 2326, 1539, 699, 228, 0, 184, 461, 420, 703, 1207, 1017, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1381, 1274, 15, 535, 62, 1981, 2988, 0, 133, 11, 715, 1266, 2782, 273, 2299, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 1520, 2793, 102, 0, 1955, 763, 1685, 0, 1630, 1, 1405, 1599, 1540, 183, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 441, 2635, 575, 39, 922, 124, 1912, 922, 991, 1479, 1557, 58, 1, 1816, 0, 2207, 1359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16547, 16939, 16539, 14679, 16749, 15720, 16843, 17056] avg=16384.0000 max=17056 max_rank=7 min=14679 min_rank=3 imbalance_ratio=1.041016 max_load_reduction=114016 max_load_reduction_pct=0.869873 moved_to_echo=66368 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 260, 690, 0, 372, 0, 875, 1, 2534, 105, 412, 2210, 1358, 561, 0, 1243, 3969, 2314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 483, 456, 920, 3982, 2776, 158, 35, 1, 1642, 234, 65, 1080, 1, 1145, 1219, 1041, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2864, 0, 13, 1456, 2905, 16, 579, 2301, 2612, 1034, 287, 154, 0, 634, 253, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 950, 1404, 1547, 3, 2, 1, 2424, 271, 0, 2876, 3730, 2352, 567, 222, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 719, 4089, 0, 1347, 115, 14, 1659, 1356, 1302, 652, 133, 1, 38, 503, 214, 1244, 1221, 1047, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2019, 1674, 6, 551, 42, 1772, 2708, 0, 50, 9, 895, 800, 3275, 582, 2209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 877, 1758, 2864, 183, 0, 2265, 760, 2108, 0, 1224, 0, 593, 1215, 2351, 75, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 476, 2767, 593, 0, 429, 171, 1499, 464, 848, 842, 1515, 80, 0, 2096, 0, 2273, 1530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16909, 16480, 15491, 17030, 16089, 16592, 16806, 15675] avg=16384.0000 max=17030 max_rank=3 min=15491 min_rank=2 imbalance_ratio=1.039429 max_load_reduction=114042 max_load_reduction_pct=0.870071 moved_to_echo=65162 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[12, 240, 561, 0, 268, 0, 856, 1, 2384, 90, 456, 2521, 892, 298, 1, 853, 3993, 2191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 723, 534, 455, 576, 3934, 2171, 150, 13, 0, 1726, 310, 283, 1328, 10, 999, 1363, 992, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2862, 1, 7, 1722, 2869, 13, 477, 2486, 3260, 1090, 227, 140, 3, 565, 380, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 1324, 1229, 5, 6, 1, 2093, 203, 0, 2925, 3319, 2127, 508, 263, 875, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 4003, 1, 956, 178, 45, 1740, 2321, 1534, 586, 177, 8, 48, 543, 357, 512, 1220, 1012, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1844, 997, 8, 533, 38, 2802, 2290, 0, 46, 29, 704, 1104, 2991, 429, 2089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 952, 1599, 2731, 196, 0, 2525, 608, 2354, 0, 1592, 3, 1127, 1261, 1412, 151, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 590, 2942, 656, 21, 748, 272, 870, 1110, 1036, 1513, 1402, 57, 0, 1897, 1, 2197, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15617, 16274, 17139, 15885, 16156, 15904, 17228, 16869] avg=16384.0000 max=17228 max_rank=6 min=15617 min_rank=0 imbalance_ratio=1.051514 max_load_reduction=113844 max_load_reduction_pct=0.868561 moved_to_echo=66157 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[14, 294, 789, 0, 317, 0, 570, 4, 2485, 68, 200, 2533, 1584, 199, 4, 1002, 3996, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 354, 516, 662, 4011, 2765, 131, 24, 1, 1687, 241, 76, 666, 7, 1069, 1353, 1032, 794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2870, 5, 14, 952, 3258, 19, 514, 2062, 2878, 1195, 154, 183, 6, 608, 412, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 819, 1397, 1178, 58, 0, 3, 2349, 259, 0, 3170, 3568, 2494, 286, 555, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 934, 4060, 0, 965, 149, 26, 2884, 2142, 1774, 563, 70, 1, 42, 320, 463, 318, 1224, 1080, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1937, 1475, 3, 568, 37, 2602, 2014, 0, 47, 7, 585, 1450, 3193, 366, 2559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 781, 1545, 2838, 99, 1, 1730, 442, 1820, 0, 1789, 1, 843, 1545, 1674, 175, 807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 718, 2213, 341, 6, 525, 188, 1175, 973, 1069, 1489, 1765, 53, 0, 1856, 2, 2344, 1299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16022, 15963, 16081, 16572, 17436, 16843, 16090, 16065] avg=16384.0000 max=17436 max_rank=4 min=15963 min_rank=1 imbalance_ratio=1.064209 max_load_reduction=113636 max_load_reduction_pct=0.866974 moved_to_echo=66434 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 182, 819, 0, 392, 0, 911, 0, 1900, 84, 189, 2262, 794, 797, 0, 1405, 3984, 2391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 544, 341, 577, 3968, 2868, 148, 49, 1, 1848, 196, 222, 870, 0, 997, 1285, 816, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2871, 5, 13, 732, 2930, 14, 638, 2533, 2688, 972, 223, 120, 0, 635, 301, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1581, 1388, 3, 0, 0, 2417, 165, 0, 3228, 3342, 2394, 177, 308, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 848, 4048, 0, 936, 109, 11, 2157, 1678, 1477, 456, 70, 0, 84, 720, 754, 405, 1225, 1055, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1922, 1068, 1, 555, 111, 2697, 2421, 0, 47, 2, 1006, 975, 3147, 315, 2791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1426, 1327, 2247, 203, 0, 2014, 608, 2345, 0, 1445, 4, 754, 1171, 1389, 99, 831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 453, 2753, 826, 0, 823, 200, 1138, 800, 1091, 1740, 1580, 96, 0, 1618, 1, 2291, 1582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16112, 16334, 15704, 16479, 16466, 17058, 15863, 17056] avg=16384.0000 max=17058 max_rank=5 min=15704 min_rank=2 imbalance_ratio=1.041138 max_load_reduction=114014 max_load_reduction_pct=0.869858 moved_to_echo=66443 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[18, 259, 1054, 0, 339, 2, 674, 1, 2449, 47, 261, 2074, 1123, 325, 2, 1257, 4001, 2338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 820, 456, 900, 620, 3888, 1773, 181, 35, 1, 1526, 197, 316, 1421, 17, 740, 1351, 940, 794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2811, 1, 27, 1076, 3449, 20, 631, 2656, 3060, 1340, 213, 223, 6, 671, 453, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 1734, 983, 66, 8, 8, 2349, 196, 0, 2943, 3433, 2276, 701, 275, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1246, 4037, 0, 863, 174, 141, 1750, 1531, 1463, 505, 196, 13, 52, 369, 261, 360, 1198, 993, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1344, 1930, 0, 522, 103, 2298, 2350, 0, 57, 5, 585, 1147, 2814, 133, 2246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1121, 1523, 2586, 180, 1, 2242, 747, 2228, 0, 1458, 1, 1023, 1036, 1808, 48, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 589, 2453, 642, 9, 546, 217, 1133, 836, 1187, 1324, 1583, 107, 1, 1989, 2, 2155, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16224, 15976, 17648, 16774, 15573, 15534, 16954, 16389] avg=16384.0000 max=17648 max_rank=2 min=15534 min_rank=5 imbalance_ratio=1.077148 max_load_reduction=113424 max_load_reduction_pct=0.865356 moved_to_echo=64450 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 166, 838, 2, 293, 0, 640, 0, 2376, 169, 234, 2508, 912, 577, 0, 1661, 4031, 2167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 788, 958, 620, 808, 3932, 2619, 148, 13, 0, 1641, 241, 132, 973, 7, 788, 1436, 1001, 856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2852, 5, 7, 1039, 3169, 22, 360, 2456, 2277, 1264, 112, 132, 0, 781, 490, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 868, 1524, 1046, 24, 5, 5, 2521, 293, 0, 2996, 3465, 2603, 500, 267, 647, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1068, 4078, 0, 870, 214, 14, 1556, 2398, 1601, 717, 162, 1, 60, 420, 375, 368, 1222, 1052, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 989, 6, 544, 29, 2199, 1971, 0, 30, 7, 458, 1996, 3242, 308, 2966, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 843, 1230, 2735, 195, 0, 2370, 516, 2195, 0, 1203, 4, 810, 1267, 1847, 172, 836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 421, 2193, 758, 13, 593, 284, 1346, 748, 911, 1563, 1190, 37, 3, 1975, 7, 2282, 1432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16574, 16961, 15479, 16769, 16634, 16602, 16223, 15830] avg=16384.0000 max=16961 max_rank=1 min=15479 min_rank=2 imbalance_ratio=1.035217 max_load_reduction=114111 max_load_reduction_pct=0.870598 moved_to_echo=65289 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 2304, 7040, 128, 2816, 128, 6272, 128, 18688, 768, 2560, 18816, 9216, 3840, 128, 10112, 31744, 17664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[7808, 12672, 21248, 1408, 128, 17280, 5248, 16640, 0, 11648, 128, 7424, 10112, 13312, 1024, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[14208, 11520, 128, 4352, 512, 18432, 19072, 0, 512, 128, 5760, 10112, 24320, 2816, 19840, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[7680, 32512, 128, 7808, 1408, 384, 15232, 15488, 11776, 4736, 1280, 128, 640, 3968, 3200, 4480, 9856, 8320, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[640, 4608, 20096, 5248, 128, 5120, 1792, 10368, 6784, 7808, 11520, 12032, 640, 128, 15488, 128, 18048, 11648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[7680, 11648, 9344, 384, 128, 128, 18560, 2176, 128, 23680, 27264, 18816, 3968, 2688, 5504, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[22912, 128, 128, 9216, 24192, 256, 5120, 19328, 22528, 9856, 1792, 1664, 128, 5120, 3328, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[5376, 5248, 4992, 6272, 31360, 19200, 1408, 384, 128, 13312, 2048, 1664, 8448, 128, 7552, 10880, 7808, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 1337, 0, 0, 189, 3914, 4096, 3544, 4096, 0, 0, 69, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4090, 0, 0, 0, 0, 4093, 0, 4096, 0, 2, 4094, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4093, 0, 0, 3847, 0, 0, 2, 4049, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2807, 6, 0, 0, 0, 0, 0, 0, 0, 3567, 4095, 0, 2736, 4096, 4096, 0, 0, 0, 0, 1184, 0, 0, 239, 3, 0, 4096, 154, 0, 4055, 0, 0, 0, 0, 0, 0, 3920, 3969, 0, 4096, 0, 282, 9, 0, 0, 0, 4081, 4096, 3, 441, 46, 8, 0, 0, 607, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 4080, 4093, 302, 0, 0, 48, 4089, 4094, 4019, 4089, 0, 0, 230, 0, 0, 4084, 0, 1, 31, 0, 0, 0, 4094, 1, 1, 4089, 0, 1, 4, 1, 4092, 0, 4092, 3, 0, 4083, 0, 0, 2, 0, 4078, 4091, 0, 0, 0, 4014, 0, 23, 3983, 0, 2, 5, 4088, 4, 4095, 1, 1, 0, 4096, 1, 0, 0, 0, 2, 1, 0, 0, 3, 1, 0, 2129, 0, 0, 1, 0, 0, 0, 0, 0, 4081, 4096, 0, 263, 4095, 4093, 1, 0, 3, 2, 155, 4, 4, 218, 1, 1, 4049, 1432, 0, 4040, 11, 0, 0, 0, 0, 1, 3854, 3998, 4, 4095, 1, 134, 0, 4, 7, 0, 4080, 4095, 0, 2607, 42, 38, 3, 0, 1316, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 1066, 0, 0, 311, 3788, 4096, 2716, 4093, 0, 0, 77, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4092, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4001, 0, 0, 3863, 0, 0, 0, 4070, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 591, 11, 0, 0, 0, 0, 0, 0, 0, 3922, 4093, 0, 1287, 4096, 4096, 0, 0, 0, 0, 1596, 0, 0, 1, 48, 0, 4075, 314, 0, 4056, 0, 0, 0, 0, 0, 0, 4016, 4095, 0, 4096, 0, 2005, 1, 0, 0, 0, 4095, 4096, 0, 1763, 6, 3, 0, 0, 1477, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 3, 4092, 4091, 1624, 0, 19, 1, 4059, 4093, 3775, 4069, 0, 0, 0, 2, 0, 4094, 0, 0, 1, 0, 0, 0, 4093, 0, 5, 4089, 1, 5, 0, 0, 4096, 0, 4092, 0, 0, 4076, 30, 0, 0, 0, 4091, 4079, 0, 0, 0, 4023, 4, 0, 4087, 0, 0, 1, 4088, 0, 4094, 0, 0, 1, 4090, 4, 4, 0, 0, 0, 0, 0, 1, 0, 0, 38, 247, 0, 0, 0, 4, 0, 7, 3, 0, 3875, 4085, 7, 2628, 4092, 4096, 6, 0, 1, 0, 1006, 0, 0, 708, 0, 0, 4096, 206, 12, 3657, 4, 4, 0, 0, 25, 4, 3971, 4052, 0, 4085, 1, 163, 9, 0, 0, 0, 4084, 4092, 9, 2223, 11, 1, 23, 2, 556, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 3, 4096, 4088, 785, 0, 0, 438, 3911, 4096, 3303, 4091, 0, 0, 520, 0, 0, 4096, 0, 1, 9, 0, 0, 3, 4096, 0, 0, 4094, 0, 0, 0, 0, 4094, 3, 4094, 0, 4, 4096, 0, 2, 6, 4, 4096, 4075, 0, 0, 0, 4083, 0, 0, 4067, 0, 0, 0, 4088, 1, 4093, 0, 0, 1, 4095, 2, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 3285, 2, 0, 0, 0, 0, 3, 0, 0, 3348, 3966, 0, 527, 4096, 4095, 7, 9, 0, 0, 250, 0, 0, 186, 0, 0, 3922, 264, 0, 4052, 1, 0, 0, 1, 28, 0, 4071, 4080, 3, 4094, 0, 58, 5, 0, 13, 0, 3887, 4095, 0, 1730, 0, 5, 40, 0, 2362, 0, 44] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4093, 193, 0, 0, 28, 3724, 4096, 3594, 4096, 0, 0, 277, 0, 0, 4082, 0, 0, 19, 30, 0, 0, 4079, 0, 2, 4083, 0, 2, 0, 0, 4096, 0, 4095, 0, 1, 4085, 0, 0, 2, 27, 4088, 4096, 0, 5, 0, 3617, 2, 12, 4061, 0, 0, 0, 4091, 1, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1773, 0, 2, 0, 0, 0, 0, 0, 0, 4060, 4088, 0, 1744, 4096, 4095, 1, 0, 0, 0, 277, 0, 2, 13, 8, 0, 4085, 109, 0, 4048, 0, 0, 0, 0, 3, 0, 4095, 4046, 0, 4096, 0, 166, 1, 0, 12, 0, 3927, 4096, 0, 2989, 107, 0, 0, 0, 2164, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4095, 4077, 1334, 4, 25, 1, 3962, 4065, 3150, 4092, 0, 6, 147, 3, 0, 4079, 2, 0, 2, 6, 0, 0, 4092, 2, 0, 4089, 1, 0, 1, 0, 4095, 0, 4085, 0, 0, 4019, 0, 2, 0, 49, 4067, 4089, 0, 0, 3, 3897, 0, 28, 3919, 0, 0, 0, 4094, 0, 4092, 0, 2, 0, 4095, 3, 13, 2, 0, 0, 0, 0, 1, 0, 1, 0, 1710, 50, 1, 4, 3, 1, 0, 0, 0, 3844, 4077, 0, 3258, 4091, 4086, 1, 0, 4, 3, 251, 0, 0, 832, 0, 0, 4085, 390, 1, 3986, 9, 4, 3, 0, 30, 7, 3965, 4045, 1, 4093, 0, 15, 1, 0, 0, 0, 4089, 4096, 0, 1523, 146, 0, 17, 14, 295, 3, 247] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 4095, 4093, 142, 1, 13, 31, 3617, 4094, 3917, 4095, 0, 0, 158, 0, 1, 4084, 0, 0, 0, 0, 0, 6, 4095, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 4094, 0, 0, 4055, 1, 0, 8, 0, 4075, 4090, 0, 2, 0, 4039, 3, 0, 4023, 1, 0, 0, 4083, 0, 4095, 1, 3, 0, 4096, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 3, 3771, 11, 0, 0, 0, 1, 0, 3, 1, 3983, 4088, 3, 701, 4094, 4092, 0, 0, 0, 0, 580, 0, 0, 916, 0, 1, 4062, 93, 0, 3607, 1, 0, 1, 0, 1, 0, 4080, 4076, 0, 4095, 0, 77, 8, 0, 1, 3, 4074, 4095, 0, 1762, 177, 24, 0, 0, 1274, 0, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3741, 1337, 0, 0, 189, 3914, 0, 3544, 4096, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4090, 0, 0, 0, 0, 2422, 1388, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 2, 4094, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3847, 0, 0, 2, 4049, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 70, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2807, 6, 0, 0, 0, 0, 0, 0, 4093, 4096, 4096, 2043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3567, 4095, 0, 2736, 1674, 4096, 0, 0, 0, 0, 1184, 0, 0, 239, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 154, 0, 4055, 0, 0, 0, 0, 0, 0, 3920, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 9, 0, 0, 0, 4081, 4096, 3, 441, 46, 8, 0, 0, 607, 0, 0, 2564, 2053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16890, 16378, 16524, 16205, 17142, 17594, 16149, 14190] avg=16384.0000 max=17594 max_rank=5 min=14190 min_rank=7 imbalance_ratio=1.073853 max_load_reduction=113478 max_load_reduction_pct=0.865768 moved_to_echo=65075 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3741, 1066, 0, 0, 311, 3788, 0, 2716, 4093, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4092, 0, 0, 1, 0, 2422, 1388, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3863, 0, 0, 0, 4070, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 70, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 591, 11, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 2043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3922, 4093, 0, 1287, 1674, 4096, 0, 0, 0, 0, 1596, 0, 0, 1, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 314, 0, 4056, 0, 0, 0, 0, 0, 0, 4016, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2005, 1, 0, 0, 0, 4095, 4096, 0, 1763, 6, 3, 0, 0, 1477, 0, 0, 2564, 2053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15792, 16382, 16433, 16239, 14934, 16717, 16512, 18063] avg=16384.0000 max=18063 max_rank=7 min=14934 min_rank=4 imbalance_ratio=1.102478 max_load_reduction=113009 max_load_reduction_pct=0.862190 moved_to_echo=66226 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3738, 302, 0, 0, 48, 4089, 0, 4019, 4089, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 1, 31, 0, 0, 0, 4094, 1, 1, 4089, 0, 1, 4, 1, 2422, 1387, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 3, 0, 4083, 0, 0, 2, 0, 4078, 4091, 0, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3983, 0, 2, 5, 4088, 4, 4095, 1, 1, 0, 4096, 1, 0, 0, 70, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 3, 1, 0, 2129, 0, 0, 1, 0, 0, 0, 0, 4092, 4095, 4080, 2043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4096, 0, 263, 1673, 4093, 1, 0, 3, 2, 155, 4, 4, 218, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4005, 1432, 0, 4040, 11, 0, 0, 0, 0, 1, 3854, 3998, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 4, 7, 0, 4080, 4095, 0, 2607, 42, 38, 3, 0, 1316, 1, 0, 2561, 2051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16516, 16401, 16415, 16413, 16447, 14594, 17347, 16939] avg=16384.0000 max=17347 max_rank=6 min=14594 min_rank=5 imbalance_ratio=1.058777 max_load_reduction=113725 max_load_reduction_pct=0.867653 moved_to_echo=65327 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3738, 193, 0, 0, 28, 3724, 0, 3594, 4096, 0, 0, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 19, 30, 0, 0, 4079, 0, 2, 4083, 0, 2, 0, 0, 2422, 1388, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 1, 4085, 0, 0, 2, 27, 4088, 4096, 0, 5, 0, 3617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 12, 4061, 0, 0, 0, 4091, 1, 4096, 0, 0, 2, 4096, 0, 0, 0, 70, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1773, 0, 2, 0, 0, 0, 0, 0, 4096, 4096, 4096, 2044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 4088, 0, 1744, 1674, 4095, 1, 0, 0, 0, 277, 0, 2, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 109, 0, 4048, 0, 0, 0, 0, 3, 0, 4095, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 1, 0, 12, 0, 3927, 4096, 0, 2989, 107, 0, 0, 0, 2164, 0, 0, 2563, 2052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15650, 16392, 16065, 16475, 16109, 15962, 16342, 18077] avg=16384.0000 max=18077 max_rank=7 min=15650 min_rank=0 imbalance_ratio=1.103333 max_load_reduction=112995 max_load_reduction_pct=0.862083 moved_to_echo=66490 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 3734, 785, 0, 0, 438, 3911, 0, 3303, 4091, 0, 0, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 9, 0, 0, 3, 4096, 0, 0, 4094, 0, 0, 0, 0, 2422, 1387, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 145, 0, 4, 4096, 0, 2, 6, 4, 4096, 4075, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 4088, 1, 4093, 0, 0, 1, 4095, 2, 0, 0, 70, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 3285, 2, 0, 0, 0, 0, 3, 0, 4094, 4094, 4096, 2044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3348, 3966, 0, 527, 1674, 4095, 7, 9, 0, 0, 250, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3879, 264, 0, 4052, 1, 0, 0, 1, 28, 0, 4071, 4080, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 5, 0, 13, 0, 3887, 4095, 0, 1730, 0, 5, 40, 0, 2362, 0, 44, 2562, 2052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16785, 16392, 16514, 16460, 17627, 14062, 16379, 16853] avg=16384.0000 max=17627 max_rank=4 min=14062 min_rank=5 imbalance_ratio=1.075867 max_load_reduction=113445 max_load_reduction_pct=0.865517 moved_to_echo=64921 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 3736, 1624, 0, 19, 1, 4059, 0, 3775, 4069, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 0, 4093, 0, 5, 4089, 1, 5, 0, 0, 2420, 1387, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 4076, 30, 0, 0, 0, 4091, 4079, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4087, 0, 0, 1, 4088, 0, 4094, 0, 0, 1, 4090, 4, 4, 0, 70, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 38, 247, 0, 0, 0, 4, 0, 7, 3, 4096, 4085, 4092, 2042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3875, 4085, 7, 2628, 1672, 4096, 6, 0, 1, 0, 1006, 0, 0, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 206, 12, 3657, 4, 4, 0, 0, 25, 4, 3971, 4052, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 9, 0, 0, 0, 4084, 4092, 9, 2223, 11, 1, 23, 2, 556, 0, 2, 2561, 2051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17288, 16380, 16443, 16488, 14615, 18084, 15987, 15787] avg=16384.0000 max=18084 max_rank=5 min=14615 min_rank=4 imbalance_ratio=1.103760 max_load_reduction=112988 max_load_reduction_pct=0.862030 moved_to_echo=64473 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3714, 1334, 4, 25, 1, 3962, 0, 3150, 4092, 0, 6, 147, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 2, 0, 2, 6, 0, 0, 4092, 2, 0, 4089, 1, 0, 1, 0, 2425, 1387, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 4019, 0, 2, 0, 49, 4067, 4089, 0, 0, 3, 3897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3919, 0, 0, 0, 4094, 0, 4092, 0, 2, 0, 4095, 3, 13, 2, 77, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1710, 50, 1, 4, 3, 1, 0, 0, 4095, 4093, 4095, 2029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3844, 4077, 0, 3258, 1666, 4086, 1, 0, 4, 3, 251, 0, 0, 832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 390, 1, 3986, 9, 4, 3, 0, 30, 7, 3965, 4045, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 0, 0, 0, 4089, 4096, 0, 1523, 146, 0, 17, 14, 295, 3, 247, 2561, 2036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16438, 16372, 16263, 16373, 16083, 18022, 16478, 15043] avg=16384.0000 max=18022 max_rank=5 min=15043 min_rank=7 imbalance_ratio=1.099976 max_load_reduction=113050 max_load_reduction_pct=0.862503 moved_to_echo=65626 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3738, 142, 1, 13, 31, 3617, 0, 3917, 4095, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4084, 0, 0, 0, 0, 0, 6, 4095, 0, 0, 4096, 0, 0, 0, 0, 2421, 1387, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 4055, 1, 0, 8, 0, 4075, 4090, 0, 2, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4023, 1, 0, 0, 4083, 0, 4095, 1, 3, 0, 4096, 0, 0, 0, 70, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 3, 3771, 11, 0, 0, 0, 1, 0, 3, 4091, 4095, 4095, 2042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3983, 4088, 3, 701, 1673, 4092, 0, 0, 0, 0, 580, 0, 0, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4018, 93, 0, 3607, 1, 0, 1, 0, 1, 0, 4080, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 8, 0, 1, 3, 4074, 4095, 0, 1762, 177, 24, 0, 0, 1274, 0, 11, 2562, 2052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15713, 16375, 16415, 16419, 18115, 16037, 15878, 16120] avg=16384.0000 max=18115 max_rank=4 min=15713 min_rank=0 imbalance_ratio=1.105652 max_load_reduction=112957 max_load_reduction_pct=0.861794 moved_to_echo=66150 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32128, 3072, 128, 31616, 128, 128, 128, 128, 128, 128, 32000, 32384, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 29952, 6784, 128, 128, 1152, 31104, 0, 28032, 32768, 0, 128, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 1152, 128, 128, 32640, 128, 128, 128, 128, 32768, 32768, 0, 128, 128, 31872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 16384, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 32768, 16384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[2944, 128, 128, 128, 128, 32384, 32768, 128, 15104, 640, 128, 128, 128, 10112, 128, 384, 20608, 16512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 31872, 128, 128, 128, 32768, 128, 32768, 128, 128, 128, 32768, 128, 128, 128, 640, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 30720, 32640, 128, 13184, 13440, 32768, 128, 128, 128, 128, 5376, 128, 128, 3200, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 19456, 11136, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[21, 3, 0, 4095, 4089, 0, 1, 1, 4095, 0, 1, 2585, 0, 1, 2, 3, 0, 4092, 0, 161, 30, 0, 37, 1, 0, 4095, 0, 3, 4093, 0, 6, 4092, 641, 4095, 1, 0, 3901, 0, 2, 4093, 1, 3618, 0, 1, 4095, 1, 0, 7, 0, 702, 0, 0, 5, 1, 0, 0, 4090, 0, 4083, 1757, 4055, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 4093, 4094, 3, 4081, 0, 2, 6, 0, 0, 49, 1, 0, 4094, 1, 0, 0, 0, 4094, 4094, 2, 0, 4089, 0, 0, 4074, 4058, 4094, 0, 1, 0, 1, 7, 0, 4092, 1, 2, 4094, 0, 4090, 0, 4090, 0, 1, 5, 2938, 4, 1, 9, 1, 0, 3, 0, 0, 4095, 0, 0, 48] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4095, 0, 0, 1, 4095, 0, 0, 1651, 0, 0, 0, 0, 1, 4095, 1, 34, 2, 2, 0, 0, 28, 4095, 0, 1, 4094, 0, 7, 4095, 151, 4093, 3, 0, 3957, 3, 2, 3673, 1, 2807, 0, 0, 4095, 1, 0, 0, 0, 1408, 0, 1, 0, 1, 1, 1, 4096, 0, 4094, 1909, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4094, 3, 4096, 0, 0, 0, 0, 4, 839, 0, 0, 4095, 0, 0, 0, 0, 4095, 4095, 0, 4, 4095, 0, 0, 4089, 4090, 4080, 0, 0, 0, 0, 2, 0, 4094, 1, 0, 4093, 0, 4091, 0, 4094, 0, 1, 0, 4025, 1, 1, 0, 1, 0, 1, 0, 1, 4095, 1, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 4092, 4079, 3, 3, 4, 4094, 5, 1, 664, 0, 0, 0, 2, 0, 4089, 3, 439, 0, 4, 1, 0, 4, 4093, 0, 1, 4093, 0, 0, 4084, 1540, 4092, 3, 0, 3532, 0, 0, 3810, 4, 2454, 0, 0, 4093, 3, 0, 0, 0, 2575, 4, 0, 0, 0, 1, 0, 4090, 0, 4073, 2844, 4093, 1, 2, 6, 1, 5, 4, 0, 0, 0, 0, 4088, 4091, 4, 4075, 4, 0, 6, 6, 9, 1196, 0, 1, 4089, 5, 0, 0, 1, 4084, 4092, 0, 0, 4093, 1, 1, 4094, 4086, 4093, 18, 3, 2, 2, 2, 0, 4096, 0, 0, 4092, 3, 4094, 4, 4091, 0, 4, 0, 1455, 0, 1, 4, 3, 2, 0, 0, 0, 4088, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[24, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2564, 0, 0, 0, 1, 0, 4096, 0, 14, 11, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 248, 4096, 3, 10, 4047, 0, 0, 3999, 0, 3675, 0, 0, 4096, 0, 0, 0, 0, 771, 0, 0, 0, 0, 0, 0, 4095, 0, 4085, 1372, 4095, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 6, 4084, 0, 0, 0, 0, 0, 38, 0, 0, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 0, 2, 4074, 4095, 4096, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 4096, 0, 4088, 0, 4096, 0, 0, 0, 3659, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 61] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 420, 0, 0, 0, 0, 0, 4096, 0, 236, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 65, 4096, 0, 0, 4055, 0, 0, 3151, 0, 3816, 0, 0, 4096, 0, 0, 0, 0, 381, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3382, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4096, 0, 0, 3, 0, 0, 903, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4094, 4090, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 441, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 116, 4096, 0, 0, 3829, 0, 0, 3752, 0, 3567, 0, 0, 4096, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 3818, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 4096, 0, 0, 0, 0, 0, 883, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4095, 3954, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[21, 1, 0, 4095, 4083, 0, 0, 0, 4096, 0, 0, 1747, 0, 0, 0, 1, 10, 4096, 0, 28, 0, 0, 7, 1, 0, 4095, 0, 0, 4094, 2, 1, 4092, 114, 4096, 1, 0, 3841, 2, 4, 3983, 0, 3904, 0, 0, 4095, 0, 0, 0, 0, 202, 0, 0, 0, 0, 1, 6, 4090, 0, 4092, 3217, 4096, 0, 0, 0, 0, 4, 3, 1, 0, 0, 1, 4095, 4095, 0, 4095, 2, 4, 0, 0, 0, 691, 0, 0, 4096, 0, 0, 0, 0, 4092, 4096, 0, 0, 4096, 0, 1, 4084, 4080, 4096, 7, 0, 0, 0, 5, 0, 4096, 0, 0, 4095, 0, 4090, 7, 4087, 0, 0, 0, 2736, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[10, 0, 0, 4096, 4096, 0, 0, 0, 4091, 0, 0, 536, 0, 0, 0, 2, 1, 4096, 0, 32, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 4087, 290, 4096, 3, 0, 4050, 0, 0, 3689, 0, 3423, 0, 0, 4096, 0, 0, 0, 0, 1664, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 2446, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 1, 4074, 0, 0, 18, 0, 0, 124, 0, 0, 4096, 0, 0, 5, 0, 4096, 4096, 2, 0, 4096, 0, 5, 4095, 4095, 4092, 0, 0, 0, 0, 5, 0, 4090, 0, 1, 4096, 0, 4096, 3, 4096, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 139] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[21, 3, 0, 4095, 4089, 0, 1, 1, 4095, 0, 1, 2585, 0, 1, 2, 3, 2563, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 161, 30, 0, 37, 1, 0, 3961, 0, 3, 4093, 0, 6, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 4095, 1, 0, 3901, 0, 2, 4093, 1, 3618, 0, 1, 810, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 702, 0, 0, 5, 1, 0, 0, 4090, 0, 4083, 1757, 4055, 0, 0, 1, 382, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 4093, 4094, 3, 4081, 0, 2, 6, 0, 0, 3371, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 1, 0, 3514, 1, 0, 0, 0, 4094, 0, 2, 0, 4089, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 4094, 0, 1, 0, 1, 7, 0, 28, 1, 2, 0, 0, 4090, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 2938, 4, 1, 9, 1, 0, 3, 0, 0, 4095, 0, 0, 48, 4094, 4094, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17658, 16476, 17171, 15210, 16375, 15824, 16372, 15986] avg=16384.0000 max=17658 max_rank=0 min=15210 min_rank=3 imbalance_ratio=1.077759 max_load_reduction=113414 max_load_reduction_pct=0.865280 moved_to_echo=64557 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[24, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2564, 0, 0, 0, 1, 2564, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 14, 11, 0, 0, 0, 0, 3962, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 4096, 3, 10, 4047, 0, 0, 3999, 0, 3675, 0, 0, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 771, 0, 0, 0, 0, 0, 0, 4095, 0, 4085, 1372, 4095, 0, 28, 0, 382, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 6, 4084, 0, 0, 0, 0, 0, 3374, 723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 3516, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 2, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 2, 0, 28, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3659, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 61, 4096, 4096, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17639, 16275, 16887, 14962, 16378, 15823, 16405, 16703] avg=16384.0000 max=17639 max_rank=0 min=14962 min_rank=3 imbalance_ratio=1.076599 max_load_reduction=113433 max_load_reduction_pct=0.865425 moved_to_echo=65309 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 441, 0, 0, 0, 0, 2564, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 1, 0, 0, 3962, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 4096, 0, 0, 3829, 0, 0, 3752, 0, 3567, 0, 0, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 3818, 4093, 0, 1, 0, 382, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 4096, 0, 0, 0, 0, 0, 3374, 723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 883, 0, 0, 3516, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 4096, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 4096, 4096, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15491, 16272, 16169, 16866, 16382, 16686, 16270, 16936] avg=16384.0000 max=16936 max_rank=7 min=15491 min_rank=0 imbalance_ratio=1.033691 max_load_reduction=114136 max_load_reduction_pct=0.870789 moved_to_echo=66274 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 420, 0, 0, 0, 0, 2564, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 236, 0, 0, 0, 0, 0, 3962, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4096, 0, 0, 4055, 0, 0, 3151, 0, 3816, 0, 0, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 381, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3382, 4087, 0, 0, 0, 382, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4096, 0, 0, 3, 0, 0, 3374, 723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 903, 0, 0, 3516, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 4096, 4096, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15473, 16486, 15992, 16558, 16386, 16705, 16406, 17066] avg=16384.0000 max=17066 max_rank=7 min=15473 min_rank=0 imbalance_ratio=1.041626 max_load_reduction=114006 max_load_reduction_pct=0.869797 moved_to_echo=66563 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4095, 0, 0, 1, 4095, 0, 0, 1651, 0, 0, 0, 0, 2563, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 34, 2, 2, 0, 0, 28, 3961, 0, 1, 4094, 0, 7, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 4093, 3, 0, 3957, 3, 2, 3673, 1, 2807, 0, 0, 810, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1408, 0, 1, 0, 1, 1, 1, 4096, 0, 4094, 1909, 4096, 0, 11, 0, 382, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4094, 3, 4096, 0, 0, 0, 0, 4, 3373, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 0, 0, 3515, 0, 0, 0, 0, 4095, 0, 0, 4, 4095, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4080, 0, 0, 0, 0, 2, 0, 28, 1, 0, 0, 0, 4091, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4025, 1, 1, 0, 1, 0, 1, 0, 1, 4095, 1, 0, 5, 4095, 4093, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16698, 16321, 15501, 16134, 16382, 16637, 16386, 17013] avg=16384.0000 max=17013 max_rank=7 min=15501 min_rank=2 imbalance_ratio=1.038391 max_load_reduction=114059 max_load_reduction_pct=0.870201 moved_to_echo=66418 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4092, 4079, 3, 3, 4, 4094, 5, 1, 664, 0, 0, 0, 2, 2568, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 439, 0, 4, 1, 0, 4, 3957, 0, 1, 4093, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1540, 4092, 3, 0, 3532, 0, 0, 3810, 4, 2454, 0, 0, 799, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2575, 4, 0, 0, 0, 1, 0, 4090, 0, 4073, 2844, 4093, 1, 2, 6, 388, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 4, 0, 0, 0, 0, 4088, 4091, 4, 4075, 4, 0, 6, 6, 9, 3379, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1196, 0, 1, 3502, 5, 0, 0, 1, 4084, 0, 0, 0, 4093, 1, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4093, 18, 3, 2, 2, 2, 0, 20, 0, 0, 0, 3, 4094, 4, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1455, 0, 1, 4, 3, 2, 0, 0, 0, 4088, 0, 0, 0, 4092, 4092, 697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15715, 16675, 16237, 18213, 16398, 16978, 16418, 14438] avg=16384.0000 max=18213 max_rank=3 min=14438 min_rank=7 imbalance_ratio=1.111633 max_load_reduction=112859 max_load_reduction_pct=0.861046 moved_to_echo=64232 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 4096, 4096, 0, 0, 0, 4091, 0, 0, 536, 0, 0, 0, 2, 2564, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 32, 0, 0, 0, 0, 1, 3962, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 4096, 3, 0, 4050, 0, 0, 3689, 0, 3423, 0, 0, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1664, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 2446, 4096, 0, 3, 0, 382, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 1, 4074, 0, 0, 18, 0, 0, 3370, 723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 3516, 0, 0, 5, 0, 4096, 0, 2, 0, 4096, 0, 5, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4092, 0, 0, 0, 0, 5, 0, 27, 0, 1, 0, 0, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 139, 4096, 4096, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15593, 16275, 16360, 16909, 16375, 15939, 16415, 17206] avg=16384.0000 max=17206 max_rank=7 min=15593 min_rank=0 imbalance_ratio=1.050171 max_load_reduction=113866 max_load_reduction_pct=0.868729 moved_to_echo=65935 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[21, 1, 0, 4095, 4083, 0, 0, 0, 4096, 0, 0, 1747, 0, 0, 0, 1, 2563, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 28, 0, 0, 7, 1, 0, 3961, 0, 0, 4094, 2, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 4096, 1, 0, 3841, 2, 4, 3983, 0, 3904, 0, 0, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 0, 0, 1, 6, 4090, 0, 4092, 3217, 4096, 0, 0, 0, 382, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 1, 0, 0, 1, 4095, 4095, 0, 4095, 2, 4, 0, 0, 0, 3374, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, 0, 3516, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 1, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 7, 0, 0, 0, 5, 0, 28, 0, 0, 0, 0, 4090, 7, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2736, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 4096, 4095, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16805, 16292, 16755, 16220, 16396, 16480, 16400, 15724] avg=16384.0000 max=16805 max_rank=0 min=15724 min_rank=7 imbalance_ratio=1.025696 max_load_reduction=114267 max_load_reduction_pct=0.871788 moved_to_echo=65000 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32640, 32768, 128, 128, 128, 128, 128, 0, 256, 128, 128, 0, 128, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 32768, 32768, 128, 128, 128, 32768, 128, 128, 10624, 0, 128, 128, 128, 20608, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3200, 32768, 128, 128, 31232, 128, 128, 30208, 128, 27264, 0, 128, 6528, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[4736, 128, 128, 28160, 128, 0, 128, 128, 32768, 0, 128, 128, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 1024, 128, 128, 128, 128, 128, 31744, 0, 128, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 32768, 32768, 128, 32768, 128, 128, 128, 128, 128, 27008, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 8064, 128, 128, 128, 128, 128, 128, 32768, 0, 32768, 20864, 32768, 128, 128, 128, 3072, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[0, 128, 128, 27008, 128, 128, 128, 128, 128, 128, 0, 128, 32768, 128, 128, 384, 32768, 32768, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 6, 8, 4, 2509, 7, 4090, 4087, 0, 13, 12, 0, 2, 0, 0, 3, 2, 55, 4, 4090, 5, 0, 0, 6, 2, 4087, 2, 4089, 0, 0, 8, 0, 0, 0, 0, 0, 4, 0, 4092, 0, 4092, 0, 0, 16, 5, 0, 0, 0, 7, 0, 0, 2, 35, 0, 2, 4092, 4092, 4, 3, 4084, 4089, 4090, 4092, 10, 1281, 0, 0, 3, 0, 4, 4089, 0, 7, 4091, 4, 4, 240, 4092, 0, 2, 0, 4092, 4092, 0, 0, 4082, 2, 4061, 0, 4076, 0, 4091, 4, 1, 0, 3, 6, 4094, 4084, 4, 4086, 2, 4, 4091, 0, 3810, 0, 4, 2, 0, 3, 254, 23, 4, 0, 3, 4091, 0, 0, 5, 0, 4, 4084, 3, 0, 4091, 4089, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3294, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 264, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 368, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 218, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4093, 0, 4035, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4076, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4077, 0, 4096, 4087, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 19, 120, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 14, 4096, 4096, 0, 0, 4079, 0, 3957, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4006, 0, 0, 0, 0, 0, 68, 2, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2454, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 283, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1373, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 4001, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4074, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1, 4, 0, 1, 3893, 1, 4094, 4083, 2, 0, 18, 0, 0, 1, 0, 1, 0, 0, 2, 4093, 1, 0, 0, 0, 0, 4085, 0, 4094, 0, 0, 1, 15, 5, 0, 0, 0, 1, 0, 4094, 0, 4089, 0, 0, 12, 10, 0, 3, 0, 1, 0, 0, 2, 13, 10, 0, 4094, 4095, 1, 1, 4092, 4095, 4092, 4092, 0, 160, 5, 0, 10, 0, 2, 4095, 0, 1, 4091, 2, 2, 6, 4094, 0, 1, 3, 4092, 4092, 1, 0, 4080, 2, 3983, 0, 4088, 0, 4094, 2, 0, 0, 1, 0, 4095, 4090, 1, 4093, 1, 2, 4093, 2, 4071, 0, 4, 1, 1, 1, 117, 0, 1, 0, 1, 4095, 0, 0, 4, 1, 3, 4091, 0, 3, 4092, 4094, 16] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4078, 0, 4096, 4087, 0, 0, 7, 0, 0, 0, 1, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 164, 0, 0, 5, 0, 0, 0, 4096, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 36, 6, 0, 4096, 4096, 0, 0, 4095, 4096, 4092, 4096, 4, 17, 1, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 63, 4096, 4096, 0, 0, 4096, 0, 4054, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4091, 4096, 0, 4093, 0, 0, 4096, 0, 3763, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 4096, 52] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 1, 3885, 2, 4095, 4091, 0, 8, 1, 1, 1, 1, 0, 0, 1, 2, 0, 4094, 1, 0, 0, 0, 0, 4094, 0, 4065, 0, 0, 0, 86, 2, 1, 2, 0, 0, 1, 4095, 0, 4095, 0, 0, 52, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 4095, 1, 1, 4093, 4094, 4095, 4095, 3, 87, 128, 2, 0, 0, 1, 4095, 1, 1, 4095, 0, 1, 4, 4088, 0, 0, 0, 4095, 4095, 0, 0, 4089, 0, 4014, 0, 4092, 2, 4095, 0, 0, 0, 1, 2, 4094, 4095, 0, 4095, 0, 1, 4095, 1, 3971, 0, 1, 1, 0, 0, 81, 0, 1, 0, 1, 4096, 0, 0, 0, 0, 0, 4095, 2, 0, 4095, 4095, 12] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4004, 0, 4095, 4096, 0, 0, 23, 0, 0, 0, 4, 0, 0, 133, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 19, 3, 0, 1, 0, 0, 0, 4096, 0, 4086, 2, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4, 0, 4096, 4096, 0, 0, 4095, 4096, 4094, 4093, 10, 84, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 3, 4085, 0, 3984, 0, 4096, 0, 4094, 0, 0, 5, 0, 0, 4094, 4096, 0, 4091, 0, 0, 4096, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3294, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 546, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 12, 3509, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 25, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 4076, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 218, 4096, 0, 0, 3518, 20, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 4093, 0, 4035, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 41, 0, 4076, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16145, 16564, 16392, 16376, 16420, 16407, 16447, 16321] avg=16384.0000 max=16564 max_rank=1 min=16145 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=114508 max_load_reduction_pct=0.873627 moved_to_echo=65595 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2454, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 546, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 9, 3509, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 4096, 4076, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1373, 4096, 0, 0, 3518, 20, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 4095, 0, 4001, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 41, 0, 4074, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15302, 16321, 16372, 16382, 17488, 16375, 16452, 16380] avg=16384.0000 max=17488 max_rank=4 min=15302 min_rank=0 imbalance_ratio=1.067383 max_load_reduction=113584 max_load_reduction_pct=0.866577 moved_to_echo=66695 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4077, 0, 4096, 4087, 0, 0, 16, 0, 0, 0, 0, 0, 4096, 546, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 29, 3509, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4076, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 120, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3518, 20, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 87, 0, 0, 4079, 0, 3957, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 41, 0, 4006, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16933, 16316, 16367, 16365, 15971, 16329, 16403, 16388] avg=16384.0000 max=16933 max_rank=0 min=15971 min_rank=4 imbalance_ratio=1.033508 max_load_reduction=114139 max_load_reduction_pct=0.870811 moved_to_echo=65091 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 8, 4, 2509, 7, 4090, 4087, 0, 13, 12, 0, 2, 0, 0, 3, 4092, 547, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 55, 4, 4090, 5, 0, 0, 6, 2, 4087, 2, 4089, 0, 0, 8, 0, 3509, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4092, 0, 4092, 0, 0, 16, 5, 0, 0, 0, 4092, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2, 35, 0, 2, 0, 0, 4, 3, 4084, 4064, 4090, 4092, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1281, 0, 0, 3, 0, 4, 4089, 0, 7, 4091, 4, 4, 240, 4092, 0, 2, 3520, 25, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 4082, 2, 4061, 0, 4076, 0, 4091, 4, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4094, 4084, 4, 4086, 2, 4, 35, 0, 3810, 0, 4, 2, 0, 3, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4, 0, 3, 4091, 0, 0, 5, 0, 4, 4084, 3, 0, 4079, 4089, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15395, 16356, 16378, 16393, 17374, 16395, 16388, 16393] avg=16384.0000 max=17374 max_rank=4 min=15395 min_rank=0 imbalance_ratio=1.060425 max_load_reduction=113698 max_load_reduction_pct=0.867447 moved_to_echo=66550 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 4, 0, 1, 3893, 1, 4094, 4083, 2, 0, 18, 0, 0, 1, 0, 1, 4092, 545, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 1, 0, 0, 0, 0, 4085, 0, 4094, 0, 0, 1, 15, 3506, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 4094, 0, 4089, 0, 0, 12, 10, 0, 3, 0, 4095, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 13, 10, 0, 0, 0, 1, 1, 4092, 4075, 4092, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 5, 0, 10, 0, 2, 4095, 0, 1, 4091, 2, 2, 6, 4094, 0, 1, 3515, 20, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 86, 1, 0, 4080, 2, 3983, 0, 4088, 0, 4094, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4090, 1, 4093, 1, 2, 42, 2, 4071, 0, 4, 1, 1, 1, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 0, 4, 1, 3, 4091, 0, 3, 4086, 4094, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16751, 16288, 16388, 16379, 16010, 16340, 16521, 16395] avg=16384.0000 max=16751 max_rank=0 min=16010 min_rank=4 imbalance_ratio=1.022400 max_load_reduction=114321 max_load_reduction_pct=0.872200 moved_to_echo=65266 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 1, 3885, 2, 4095, 4091, 0, 8, 1, 1, 1, 1, 0, 0, 4095, 545, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 4094, 1, 0, 0, 0, 0, 4094, 0, 4065, 0, 0, 0, 86, 3508, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 0, 0, 1, 4095, 0, 4095, 0, 0, 52, 0, 1, 0, 0, 4095, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4093, 4074, 4095, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 128, 2, 0, 0, 1, 4095, 1, 1, 4095, 0, 1, 4, 4088, 0, 0, 3517, 20, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 4089, 0, 4014, 0, 4092, 2, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 4095, 0, 4095, 0, 1, 42, 1, 3971, 0, 1, 1, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4096, 0, 0, 0, 0, 0, 4095, 2, 0, 4089, 4095, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16742, 16342, 16424, 16363, 16046, 16380, 16384, 16391] avg=16384.0000 max=16742 max_rank=0 min=16046 min_rank=4 imbalance_ratio=1.021851 max_load_reduction=114330 max_load_reduction_pct=0.872269 moved_to_echo=65201 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4078, 0, 4096, 4087, 0, 0, 7, 0, 0, 0, 1, 0, 4096, 546, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 164, 3509, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 6, 0, 0, 0, 0, 0, 4095, 4076, 4092, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3518, 20, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 87, 0, 0, 4096, 0, 4054, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 4093, 0, 0, 41, 0, 3763, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4093, 0, 0, 4090, 4096, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16926, 16448, 16378, 16405, 15851, 16492, 16151, 16421] avg=16384.0000 max=16926 max_rank=0 min=15851 min_rank=4 imbalance_ratio=1.033081 max_load_reduction=114146 max_load_reduction_pct=0.870865 moved_to_echo=64915 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4004, 0, 4095, 4096, 0, 0, 23, 0, 0, 0, 4, 0, 4096, 546, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 19, 3509, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 4096, 0, 4086, 2, 7, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4, 0, 0, 0, 0, 0, 4095, 4076, 4094, 4093, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 3518, 20, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 3, 4085, 0, 3984, 0, 4096, 0, 4094, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4091, 0, 0, 41, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16878, 16437, 16373, 16409, 15912, 16354, 16326, 16383] avg=16384.0000 max=16878 max_rank=0 min=15912 min_rank=4 imbalance_ratio=1.030151 max_load_reduction=114194 max_load_reduction_pct=0.871231 moved_to_echo=64975 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 512, 128, 32768, 128, 0, 0, 128, 128, 32768, 128, 32768, 0, 0, 128, 384, 28160, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 0, 32768, 128, 128, 128, 128, 128, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 28288, 128, 32768, 32768, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 4480, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2304, 256, 128, 128, 0, 128, 32768, 128, 128, 32768, 128, 128, 1920, 32768, 0, 128, 28160, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 128, 128, 128, 32768, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 768, 128, 128, 32768, 128, 32128, 0, 32768, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 32768, 128, 32768, 128, 128, 384, 128, 31872, 0, 128, 128, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 256, 128, 128, 0, 0, 128, 128, 32768, 32640, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[2, 6, 4091, 0, 4, 3, 0, 0, 0, 5, 0, 0, 0, 5, 0, 4090, 0, 5, 0, 3574, 0, 4090, 1, 6, 4086, 4038, 5, 2, 4084, 4092, 4090, 7, 0, 8, 3, 0, 4010, 2, 4091, 0, 4091, 0, 60, 2, 4092, 0, 0, 0, 6, 5, 4090, 4091, 6, 4089, 4, 1375, 4090, 4, 0, 0, 650, 0, 0, 0, 0, 4090, 4094, 0, 0, 18, 2786, 0, 4078, 1642, 0, 5, 0, 4092, 4091, 0, 4092, 3, 3, 4, 4, 3961, 4, 0, 2256, 0, 5, 2, 5, 0, 0, 4089, 3, 4, 7, 4091, 0, 5, 0, 4089, 130, 6, 5, 0, 0, 3, 0, 8, 4087, 25, 1, 0, 5, 37, 4090, 0, 5, 0, 4, 4088, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4095, 2, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 4095, 0, 1, 0, 3986, 0, 4096, 0, 0, 4095, 4000, 0, 0, 4095, 4084, 4095, 1, 0, 52, 0, 0, 4076, 0, 4094, 1, 4095, 0, 21, 1, 4095, 1, 0, 0, 1, 1, 4094, 4095, 1, 4095, 198, 260, 4095, 1, 0, 1, 3137, 0, 2, 0, 0, 4095, 4056, 0, 0, 0, 3270, 0, 4050, 127, 1, 1, 1, 4091, 4095, 1, 4095, 0, 0, 0, 0, 4094, 1, 0, 1034, 0, 1, 3, 1, 0, 3, 4092, 0, 1, 1, 4095, 1, 1, 0, 4095, 409, 0, 1, 0, 1, 0, 0, 1, 4094, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 1, 4094, 1, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3074, 0, 4096, 0, 0, 4096, 3920, 0, 0, 4096, 4080, 4096, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 4096, 0, 18, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 366, 581, 4095, 0, 0, 0, 2253, 0, 0, 0, 1, 4096, 3964, 0, 0, 0, 3142, 0, 4081, 194, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3921, 0, 0, 3022, 0, 0, 2, 0, 0, 3, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 115, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 15, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 4090, 0, 1, 4, 0, 3, 4, 1, 0, 0, 0, 0, 0, 4089, 0, 4, 0, 3404, 0, 4085, 0, 2, 4094, 3958, 0, 0, 4092, 4093, 4093, 3, 0, 33, 3, 27, 4095, 0, 4094, 2, 4093, 0, 38, 2, 4091, 0, 0, 0, 0, 2, 4093, 4094, 3, 4093, 32, 838, 4092, 1, 0, 0, 1543, 0, 0, 0, 0, 4093, 3872, 0, 0, 0, 2895, 0, 4075, 299, 0, 2, 2, 4095, 4091, 0, 4094, 3, 3, 3, 1, 4086, 2, 12, 3452, 0, 2, 0, 2, 0, 1, 4087, 1, 1, 4, 4093, 0, 2, 0, 4093, 91, 2, 2, 0, 2, 0, 0, 2, 4094, 0, 0, 2, 0, 13, 4094, 0, 1, 5, 3, 4093, 12, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2169, 0, 4096, 0, 0, 4096, 4095, 0, 0, 4096, 4090, 4096, 0, 0, 18, 0, 0, 4096, 0, 4096, 0, 4096, 0, 155, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 28, 1015, 4096, 0, 0, 0, 2863, 0, 0, 0, 0, 4096, 4069, 0, 0, 0, 3375, 0, 4095, 528, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4077, 0, 0, 1936, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 252, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3775, 0, 4096, 0, 0, 4096, 4046, 0, 0, 4096, 4083, 4096, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 323, 609, 4096, 0, 0, 0, 1272, 0, 0, 0, 15, 4096, 3843, 0, 0, 0, 3403, 0, 4096, 73, 0, 0, 0, 4091, 4096, 0, 4095, 0, 0, 0, 0, 4094, 0, 5, 3089, 0, 0, 0, 0, 5, 9, 4085, 0, 0, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 1, 0, 4096, 33, 0, 0, 0, 9, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4083, 0, 4096, 0, 0, 4096, 4091, 0, 0, 4096, 4090, 4096, 0, 0, 0, 0, 1, 4091, 0, 4096, 0, 4096, 0, 644, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 16, 182, 4096, 0, 0, 0, 1439, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 3657, 0, 4096, 183, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 923, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 1185, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[3, 0, 4088, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 4092, 0, 0, 0, 3464, 0, 4095, 0, 0, 4096, 4062, 0, 0, 4096, 4076, 4096, 0, 0, 1, 0, 0, 4092, 0, 4096, 0, 4096, 4, 3, 0, 4089, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 64, 86, 4092, 0, 0, 0, 1132, 2, 0, 0, 1, 4096, 3725, 0, 0, 0, 3809, 0, 4096, 32, 0, 0, 0, 4092, 4094, 4, 4091, 0, 0, 0, 0, 4094, 0, 7, 3830, 0, 0, 3, 0, 1, 8, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 283, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 1, 4096, 1, 0, 0, 0, 4096, 6, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 2, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 4095, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3986, 0, 4096, 0, 0, 0, 4000, 0, 0, 734, 4084, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 4076, 0, 3976, 1, 4095, 0, 21, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4094, 1564, 1, 4095, 198, 260, 4095, 1, 0, 1, 3137, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 3270, 0, 4050, 127, 1, 1, 1, 4091, 561, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4094, 1, 0, 1034, 0, 1, 3, 1, 0, 3, 4092, 1569, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 1, 1, 0, 4095, 409, 0, 1, 0, 1, 0, 0, 1, 4095, 3534, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 1, 4094, 1, 0, 1, 0, 3113, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16902, 16318, 17450, 16159, 15011, 16483, 16362] avg=16384.0000 max=17450 max_rank=3 min=15011 min_rank=5 imbalance_ratio=1.065063 max_load_reduction=113622 max_load_reduction_pct=0.866867 moved_to_echo=64015 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3074, 0, 4096, 0, 0, 0, 3920, 0, 0, 733, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 3978, 0, 4096, 0, 18, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1564, 0, 4096, 366, 581, 4095, 0, 0, 0, 2253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3964, 0, 0, 0, 3142, 0, 4081, 194, 0, 0, 0, 4096, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3921, 0, 0, 3022, 0, 0, 2, 0, 0, 3, 4096, 1570, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 115, 0, 0, 0, 0, 0, 0, 0, 4096, 3535, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 15, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3114, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 15903, 16285, 17051, 16039, 16828, 16187, 16395] avg=16384.0000 max=17051 max_rank=3 min=15903 min_rank=1 imbalance_ratio=1.040710 max_load_reduction=114021 max_load_reduction_pct=0.869911 moved_to_echo=65449 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4090, 0, 1, 4, 0, 3, 4, 1, 0, 0, 0, 0, 0, 4089, 4093, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3404, 0, 4085, 0, 2, 0, 3958, 0, 0, 733, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 3, 27, 4095, 0, 3976, 2, 4093, 0, 38, 2, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 1563, 3, 4093, 32, 838, 4092, 1, 0, 0, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3872, 0, 0, 0, 2895, 0, 4075, 299, 0, 2, 2, 4095, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 3, 3, 1, 4086, 2, 12, 3452, 0, 2, 0, 2, 0, 1, 4087, 1569, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4, 4093, 0, 2, 0, 4093, 91, 2, 2, 0, 2, 0, 0, 2, 4093, 3531, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 13, 4094, 0, 1, 5, 3, 4093, 12, 0, 0, 0, 3111, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16282, 16360, 16260, 15800, 17435, 16165, 16390] avg=16384.0000 max=17435 max_rank=5 min=15800 min_rank=4 imbalance_ratio=1.064148 max_load_reduction=113637 max_load_reduction_pct=0.866982 moved_to_echo=65790 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 6, 4091, 0, 4, 3, 0, 0, 0, 5, 0, 0, 0, 5, 0, 4090, 4090, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3574, 0, 4090, 1, 6, 0, 4038, 5, 2, 726, 4092, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 4010, 2, 3972, 0, 4091, 0, 60, 2, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 4090, 1557, 6, 4089, 4, 1375, 4090, 4, 0, 0, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 18, 2786, 0, 4078, 1642, 0, 5, 0, 4092, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 3, 4, 4, 3961, 4, 0, 2256, 0, 5, 2, 5, 0, 0, 4089, 1570, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 7, 4091, 0, 5, 0, 4089, 130, 6, 5, 0, 0, 3, 0, 8, 4085, 3535, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 25, 1, 0, 5, 37, 4090, 0, 5, 0, 4, 4088, 0, 0, 0, 0, 3108, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16546, 16240, 15876, 17271, 16117, 16221, 16414] avg=16384.0000 max=17271 max_rank=4 min=15876 min_rank=3 imbalance_ratio=1.054138 max_load_reduction=113801 max_load_reduction_pct=0.868233 moved_to_echo=66023 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 0, 0, 0, 4091, 0, 0, 733, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 3978, 0, 4096, 0, 644, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1564, 0, 4096, 16, 182, 4096, 0, 0, 0, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 3657, 0, 4096, 183, 0, 0, 0, 4096, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 923, 0, 0, 0, 0, 0, 2, 4096, 1570, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1185, 0, 0, 0, 0, 0, 0, 0, 4096, 3535, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 3114, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 17093, 16906, 15489, 16677, 14901, 17257, 16365] avg=16384.0000 max=17257 max_rank=6 min=14901 min_rank=5 imbalance_ratio=1.053284 max_load_reduction=113815 max_load_reduction_pct=0.868340 moved_to_echo=65200 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2169, 0, 4096, 0, 0, 0, 4095, 0, 0, 733, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4096, 0, 3978, 0, 4096, 0, 155, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1564, 0, 4096, 28, 1015, 4096, 0, 0, 0, 2863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 3375, 0, 4095, 528, 0, 0, 0, 4096, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4077, 0, 0, 1936, 0, 0, 0, 0, 0, 0, 4096, 1570, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 252, 0, 0, 0, 0, 0, 0, 0, 4096, 3535, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3114, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 15183, 16439, 17758, 16724, 15893, 16324, 16367] avg=16384.0000 max=17758 max_rank=3 min=15183 min_rank=1 imbalance_ratio=1.083862 max_load_reduction=113314 max_load_reduction_pct=0.864517 moved_to_echo=65308 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3775, 0, 4096, 0, 0, 0, 4046, 0, 0, 733, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 3978, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1564, 0, 4096, 323, 609, 4096, 0, 0, 0, 1272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3843, 0, 0, 0, 3403, 0, 4096, 73, 0, 0, 0, 4091, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4094, 0, 5, 3089, 0, 0, 0, 0, 5, 9, 4085, 1569, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 1, 0, 4095, 3535, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 33, 0, 0, 0, 9, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3114, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16733, 16261, 16056, 16082, 17069, 16077, 16407] avg=16384.0000 max=17069 max_rank=5 min=16056 min_rank=3 imbalance_ratio=1.041809 max_load_reduction=114003 max_load_reduction_pct=0.869774 moved_to_echo=65635 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4088, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 4092, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3464, 0, 4095, 0, 0, 0, 4062, 0, 0, 733, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4092, 0, 3978, 0, 4096, 4, 3, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1564, 0, 4096, 64, 86, 4092, 0, 0, 0, 1132, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3725, 0, 0, 0, 3809, 0, 4096, 32, 0, 0, 0, 4092, 561, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4094, 0, 7, 3830, 0, 0, 3, 0, 1, 8, 4096, 1570, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 283, 0, 0, 0, 0, 0, 5, 0, 4096, 3533, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 1, 0, 0, 0, 4096, 6, 0, 0, 0, 3114, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16430, 16263, 15132, 16320, 17818, 16358, 16372] avg=16384.0000 max=17818 max_rank=5 min=15132 min_rank=3 imbalance_ratio=1.087524 max_load_reduction=113254 max_load_reduction_pct=0.864059 moved_to_echo=66868 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 128, 128, 128, 128, 32512, 128, 128, 19584, 0, 128, 128, 128, 128, 128, 32768, 12672, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 0, 128, 128, 128, 0, 128, 128, 128, 128, 32768, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 31872, 128, 32768, 128, 1024, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 27648, 0, 32768, 128, 128, 0, 32256, 128, 128, 5888, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 31744, 0, 0, 128, 26368, 0, 32768, 3200, 128, 128, 128, 32768, 4608, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 128, 32768, 128, 0, 128, 0, 24960, 7808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 12544, 128, 32768, 1152, 4992, 32768, 128, 0, 128, 14336, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 0, 32768, 2560, 128, 128, 0, 128, 128, 128, 128, 32768, 28288, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 4096, 4092, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 0, 0, 0, 0, 21, 20, 0, 0, 0, 4096, 4096, 0, 4096, 4090, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 387, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 35, 0, 4095, 0, 0, 0, 529, 3855, 0, 0, 0, 0, 0, 0, 4096, 2335, 0, 0, 0, 4096, 0, 0, 1066, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 291, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4092, 4092, 3, 3, 0, 5, 4094, 0, 4089, 0, 4094, 1, 0, 0, 0, 0, 4090, 4092, 4091, 4092, 3, 4091, 4093, 0, 2, 0, 0, 2, 5, 0, 0, 3394, 4, 0, 2, 0, 0, 15, 2, 4, 0, 4090, 4093, 0, 4094, 3930, 4, 4094, 3, 0, 4091, 0, 0, 2, 0, 1113, 3, 3, 0, 4, 3, 0, 0, 4075, 0, 0, 0, 3, 3695, 0, 2, 1, 772, 4052, 1, 3, 1, 2, 0, 0, 4090, 2039, 0, 4, 2, 4094, 0, 0, 1461, 0, 3, 2, 5, 0, 4094, 7, 2, 4087, 4091, 1, 4093, 0, 0, 4093, 11, 1, 1, 0, 4094, 3, 0, 0, 4, 0, 3, 0, 2, 0, 3, 0, 2, 5, 0, 4092, 0, 4092, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4090, 4088, 6, 6, 3, 9, 4091, 0, 4090, 0, 4088, 0, 7, 0, 0, 0, 4074, 4078, 4087, 4086, 4, 4088, 4090, 0, 5, 0, 0, 4, 17, 0, 10, 3339, 7, 0, 0, 0, 0, 0, 5, 0, 0, 4087, 4091, 0, 4090, 4089, 7, 4092, 2, 0, 4090, 0, 0, 5, 0, 605, 6, 6, 0, 5, 1, 8, 0, 4087, 4, 0, 0, 5, 4064, 0, 4, 4, 323, 3810, 4, 5, 0, 6, 4, 8, 4085, 1393, 5, 0, 0, 4092, 3, 0, 2813, 0, 6, 4, 6, 0, 4090, 6, 5, 4052, 4089, 3, 4089, 0, 0, 4076, 17, 5, 1, 0, 4088, 8, 37, 0, 1, 0, 7, 2, 6, 8, 5, 0, 5, 0, 0, 4089, 1, 4086, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 26, 0, 0, 2892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4076, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1377, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 173, 0, 4093, 0, 0, 0, 1033, 4076, 0, 0, 0, 0, 0, 0, 4096, 756, 0, 0, 0, 4095, 0, 0, 1430, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 468, 0, 0, 0, 4096, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 4095, 1, 1, 0, 0, 4095, 1, 4095, 0, 4094, 1, 1, 0, 0, 0, 4094, 4095, 4094, 4081, 0, 4095, 4095, 0, 1, 0, 0, 0, 0, 0, 2, 3753, 0, 0, 0, 0, 28, 16, 1, 0, 0, 4095, 4095, 0, 4095, 4095, 1, 4095, 0, 0, 4094, 0, 0, 1, 0, 15, 1, 1, 0, 1, 0, 2, 0, 4095, 1, 0, 1, 0, 4084, 1, 1, 0, 295, 3951, 1, 0, 0, 1, 0, 0, 4093, 1457, 1, 0, 0, 4092, 0, 3, 1496, 1, 1, 1, 1, 1, 4069, 2, 1, 4093, 4094, 1, 4095, 0, 0, 4095, 1321, 1, 1, 0, 4095, 1, 2, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 4095, 0, 4095, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 17, 0, 0, 2780, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4090, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 717, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 4096, 0, 0, 0, 477, 3969, 0, 0, 0, 0, 0, 0, 4096, 1502, 0, 0, 0, 4096, 0, 0, 2012, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 792, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4095, 0, 0, 0, 0, 4093, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 4096, 4090, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3160, 0, 0, 0, 0, 24, 15, 0, 0, 0, 4096, 4096, 0, 4096, 4095, 0, 4096, 0, 0, 4091, 0, 5, 0, 0, 1283, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 4068, 0, 0, 0, 514, 4037, 0, 0, 0, 0, 0, 0, 4096, 2261, 0, 0, 0, 4096, 0, 7, 873, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 146, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4091, 0, 0, 0, 0, 4091, 0, 4096, 0, 4096, 0, 0, 4, 2, 0, 4096, 4084, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 12, 0, 5, 3439, 0, 4, 0, 0, 34, 31, 0, 0, 0, 4096, 4090, 0, 4096, 4073, 0, 4091, 0, 0, 4096, 7, 2, 0, 0, 697, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 5, 0, 4095, 0, 0, 0, 193, 4037, 0, 0, 3, 0, 0, 0, 4096, 423, 0, 0, 9, 4070, 0, 1, 3427, 1, 0, 0, 0, 0, 4092, 0, 0, 4096, 4096, 0, 4096, 0, 7, 4096, 37, 0, 0, 0, 4092, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 0, 0, 0, 0, 21, 20, 0, 0, 0, 4096, 4096, 0, 784, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 387, 0, 0, 0, 0, 0, 0, 0, 4096, 3317, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 35, 0, 4095, 0, 0, 0, 529, 3855, 0, 0, 0, 0, 0, 2952, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2335, 0, 0, 0, 4096, 0, 0, 1066, 0, 0, 0, 0, 0, 4096, 395, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3701, 0, 0, 0, 0, 4096, 291, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16379, 16868, 15995, 16341, 16444, 16281, 16381] avg=16384.0000 max=16868 max_rank=2 min=15995 min_rank=3 imbalance_ratio=1.029541 max_load_reduction=114204 max_load_reduction_pct=0.871307 moved_to_echo=65447 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4092, 3, 3, 0, 5, 4094, 0, 4089, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4092, 4091, 4092, 3, 0, 0, 0, 2, 0, 0, 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3394, 4, 0, 2, 0, 0, 15, 2, 4, 0, 4090, 4093, 0, 784, 3930, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 4091, 0, 0, 2, 0, 1113, 3, 3, 0, 4, 3, 0, 0, 4092, 3315, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 3, 3695, 0, 2, 1, 772, 4052, 1, 3, 1, 2, 0, 2951, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2039, 0, 4, 2, 4094, 0, 0, 1461, 0, 3, 2, 5, 0, 4094, 395, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 4087, 3696, 1, 0, 0, 0, 4093, 11, 1, 1, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 0, 2, 0, 3, 0, 2, 5, 0, 4092, 0, 4092, 2, 4093, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16377, 16322, 16726, 16334, 16548, 15996, 16388] avg=16384.0000 max=16726 max_rank=3 min=15996 min_rank=6 imbalance_ratio=1.020874 max_load_reduction=114346 max_load_reduction_pct=0.872391 moved_to_echo=65266 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4088, 6, 6, 3, 9, 4091, 0, 4090, 0, 4088, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4078, 4087, 4086, 4, 0, 0, 0, 5, 0, 0, 4, 17, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3339, 7, 0, 0, 0, 0, 0, 5, 0, 0, 4087, 4091, 0, 775, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 4090, 0, 0, 5, 0, 605, 6, 6, 0, 5, 1, 8, 0, 4090, 3313, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4, 0, 0, 5, 4064, 0, 4, 4, 323, 3810, 4, 5, 0, 6, 4, 2954, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4085, 1393, 5, 0, 0, 4092, 3, 0, 2813, 0, 6, 4, 6, 0, 4090, 398, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 4052, 3691, 3, 0, 0, 0, 4076, 17, 5, 1, 0, 4088, 8, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 2, 6, 8, 5, 0, 5, 0, 0, 4089, 1, 4086, 5, 4090, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16365, 16400, 16226, 16049, 17264, 15989, 16391] avg=16384.0000 max=17264 max_rank=5 min=15989 min_rank=6 imbalance_ratio=1.053711 max_load_reduction=113808 max_load_reduction_pct=0.868286 moved_to_echo=65693 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 784, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1377, 0, 0, 0, 0, 0, 0, 0, 4094, 3317, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 173, 0, 4093, 0, 0, 0, 1033, 4076, 0, 0, 0, 0, 0, 2952, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 756, 0, 0, 0, 4095, 0, 0, 1430, 0, 0, 0, 0, 0, 4096, 395, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3701, 0, 0, 0, 0, 4096, 468, 0, 0, 0, 4096, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16412, 15944, 16982, 17198, 15228, 16540, 16382] avg=16384.0000 max=17198 max_rank=4 min=15228 min_rank=5 imbalance_ratio=1.049683 max_load_reduction=113874 max_load_reduction_pct=0.868790 moved_to_echo=65348 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2780, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 4096, 0, 784, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 717, 0, 0, 0, 0, 0, 0, 0, 4094, 3317, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 4096, 0, 0, 0, 477, 3969, 0, 0, 0, 0, 0, 2952, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1502, 0, 0, 0, 4096, 0, 0, 2012, 2, 0, 0, 0, 0, 4096, 395, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3701, 0, 0, 0, 0, 4096, 792, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16403, 15850, 16322, 16385, 16559, 16786, 16381] avg=16384.0000 max=16786 max_rank=6 min=15850 min_rank=2 imbalance_ratio=1.024536 max_load_reduction=114286 max_load_reduction_pct=0.871933 moved_to_echo=66111 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4095, 1, 1, 0, 0, 4095, 1, 4095, 0, 4094, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 4094, 4081, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3753, 0, 0, 0, 0, 28, 16, 1, 0, 0, 4095, 4095, 0, 783, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 0, 0, 1, 0, 15, 1, 1, 0, 1, 0, 2, 0, 4093, 3316, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 4084, 1, 1, 0, 295, 3951, 1, 0, 0, 1, 0, 2952, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1457, 1, 0, 0, 4092, 0, 3, 1496, 1, 1, 1, 1, 1, 4069, 395, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4093, 3699, 1, 0, 0, 0, 4095, 1321, 1, 1, 0, 4095, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 4095, 0, 4095, 1, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16368, 16867, 15622, 16161, 15971, 17312, 16385] avg=16384.0000 max=17312 max_rank=6 min=15622 min_rank=3 imbalance_ratio=1.056641 max_load_reduction=113760 max_load_reduction_pct=0.867920 moved_to_echo=65829 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 4093, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3160, 0, 0, 0, 0, 24, 15, 0, 0, 0, 4096, 4096, 0, 784, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 5, 0, 0, 1283, 0, 0, 0, 0, 0, 0, 0, 4096, 3319, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 4068, 0, 0, 0, 514, 4037, 0, 0, 0, 0, 0, 2952, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2261, 0, 0, 0, 4096, 0, 7, 873, 0, 0, 0, 0, 0, 4096, 395, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3701, 0, 0, 0, 0, 4096, 146, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16377, 16270, 16893, 16448, 16184, 16136, 16384] avg=16384.0000 max=16893 max_rank=3 min=16136 min_rank=6 imbalance_ratio=1.031067 max_load_reduction=114179 max_load_reduction_pct=0.871117 moved_to_echo=65152 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4091, 0, 0, 0, 0, 4091, 0, 4096, 0, 4096, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 12, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 0, 4, 0, 0, 34, 31, 0, 0, 0, 4096, 4090, 0, 784, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 7, 2, 0, 0, 697, 0, 0, 0, 0, 0, 0, 0, 4094, 3317, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 5, 0, 4095, 0, 0, 0, 193, 4037, 0, 0, 3, 0, 0, 2952, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 423, 0, 0, 9, 4070, 0, 1, 3427, 1, 0, 0, 0, 0, 4092, 395, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3701, 0, 0, 0, 7, 4096, 37, 0, 0, 0, 4092, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16391, 16551, 16306, 16156, 16874, 16032, 16380] avg=16384.0000 max=16874 max_rank=5 min=16032 min_rank=6 imbalance_ratio=1.029907 max_load_reduction=114198 max_load_reduction_pct=0.871262 moved_to_echo=65442 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 128, 32768, 128, 32768, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 0, 32768, 128, 128, 128, 0, 6272, 128, 128, 0, 128, 128, 128, 0, 32768, 26624, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 32768, 29696, 128, 0, 0, 128, 32768, 3200, 128, 128, 0, 32768, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 32768, 32768, 32768, 128, 128, 0, 0, 128, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 128, 0, 256, 128, 32384, 128, 128, 128, 4224, 31872, 128, 128, 128, 128, 128, 23680, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 32768, 12288, 128, 128, 128, 32768, 128, 128, 14592, 128, 128, 128, 128, 128, 32768, 3200, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[26624, 128, 128, 128, 0, 128, 128, 128, 128, 0, 32768, 32768, 0, 6272, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 32768, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 627, 4096, 0, 53, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 2, 1206, 4096, 4090, 0, 4096, 1, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 4096, 0, 11, 0, 14, 4096, 560, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1999, 0, 0, 0, 0, 0, 3, 3746, 4096, 4094, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4079] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[2, 0, 4093, 4, 3, 1, 3, 0, 0, 4084, 3, 6, 4093, 0, 4092, 4090, 0, 0, 2, 0, 2, 4, 10, 4, 0, 3, 0, 4, 1, 5, 2, 4088, 1, 709, 4093, 0, 842, 3, 2, 0, 4092, 2, 0, 0, 2, 0, 4, 4, 3, 0, 0, 0, 4092, 3, 4092, 0, 0, 709, 4093, 4083, 0, 4092, 0, 3, 0, 0, 3, 4086, 4092, 4093, 4087, 4093, 20, 4, 6, 4094, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 5, 4093, 1084, 4089, 0, 0, 0, 0, 2, 0, 3, 4, 2, 3, 34, 4, 0, 0, 0, 891, 0, 2, 3, 4, 0, 0, 4069, 4094, 4057, 0, 1, 2, 0, 4092, 4088, 3, 4093, 1, 8, 2, 4091, 0, 3961] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4093, 0, 1783, 4096, 0, 746, 0, 0, 0, 4095, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 696, 4096, 4096, 0, 4095, 4, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 4086, 0, 0, 2, 0, 4094, 4096, 0, 11, 0, 23, 4096, 326, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1545, 0, 0, 0, 0, 0, 2, 3470, 4096, 4095, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 3676] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4, 0, 4089, 6, 7, 21, 7, 0, 0, 4062, 6, 0, 4091, 0, 4088, 4085, 4, 3, 5, 0, 6, 5, 1, 8, 4, 7, 0, 7, 3, 0, 7, 4064, 1, 1072, 4089, 0, 93, 5, 0, 0, 4089, 9, 17, 0, 8, 0, 10, 9, 0, 2, 0, 0, 4087, 6, 4084, 0, 37, 323, 4090, 4071, 0, 4091, 0, 6, 0, 0, 7, 4084, 4084, 4089, 4088, 4071, 0, 23, 7, 4093, 1, 3, 0, 0, 4087, 4087, 0, 0, 0, 14, 4088, 61, 4071, 0, 0, 0, 0, 5, 0, 6, 0, 3, 4, 29, 13, 0, 0, 0, 3474, 0, 5, 7, 0, 7, 1, 3316, 4087, 4034, 0, 0, 4, 0, 4088, 4091, 8, 4089, 0, 8, 2, 4085, 0, 3959] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 634, 4096, 0, 388, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4094, 0, 0, 250, 4096, 4096, 0, 4003, 0, 0, 0, 0, 0, 4096, 4094, 4096, 4095, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 4096, 124, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3240, 0, 0, 0, 0, 0, 0, 3683, 4096, 4096, 0, 0, 0, 1, 4096, 4096, 0, 4096, 0, 0, 0, 4094, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 147, 4096, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 4096, 0, 0, 350, 4096, 4096, 0, 3987, 0, 0, 0, 0, 0, 4096, 4083, 4096, 4096, 4095, 1, 0, 0, 4090, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 4096, 7, 4096, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3937, 0, 0, 0, 0, 0, 0, 3964, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4006] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 4095, 1, 1, 4, 1, 0, 0, 4086, 1, 10, 4095, 0, 4095, 4094, 0, 0, 1, 1, 1, 1, 12, 1, 0, 1, 1, 1, 0, 0, 1, 4084, 0, 1093, 4094, 0, 82, 1, 1, 0, 4088, 1, 1, 3, 1, 0, 0, 1, 0, 0, 0, 0, 4095, 1, 4094, 0, 0, 451, 4095, 4095, 0, 4096, 0, 2, 0, 0, 1, 4094, 4094, 4095, 4094, 4095, 2, 1, 0, 4095, 0, 1, 7, 0, 4092, 4093, 0, 0, 0, 4, 4095, 110, 4092, 1, 0, 2, 0, 1, 0, 1, 0, 0, 1, 0, 2, 0, 0, 0, 3196, 0, 1, 1, 0, 0, 0, 3292, 4095, 4087, 1, 0, 1, 0, 4095, 4095, 1, 4095, 0, 0, 2, 4094, 0, 4059] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4000, 0, 6, 4092, 1, 4096, 4096, 0, 0, 0, 5, 0, 0, 9, 0, 0, 0, 9, 0, 0, 0, 0, 4088, 0, 178, 4096, 0, 727, 0, 0, 0, 4087, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4066, 0, 4096, 0, 0, 1776, 4096, 4091, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4091, 4096, 10, 0, 0, 4070, 0, 0, 8, 1, 4090, 4096, 0, 2, 0, 15, 4096, 394, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1702, 2, 0, 0, 0, 0, 0, 3604, 4096, 4089, 0, 0, 0, 0, 4096, 4093, 0, 4092, 0, 0, 0, 4096, 0, 4037] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 627, 4096, 0, 53, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3969, 2042, 688, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3408, 0, 2, 1206, 0, 4090, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 4096, 4096, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 11, 0, 14, 3756, 560, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1999, 0, 0, 0, 0, 0, 3, 3746, 4096, 4090, 2036, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 127, 0, 0, 0, 4096, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16402, 16377, 15911, 16898, 16377, 16638, 15977, 16492] avg=16384.0000 max=16898 max_rank=3 min=15911 min_rank=2 imbalance_ratio=1.031372 max_load_reduction=114174 max_load_reduction_pct=0.871078 moved_to_echo=65484 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 18, 4, 3, 1, 3, 0, 0, 4084, 3, 6, 4093, 0, 4092, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 4, 10, 4, 0, 3, 0, 4, 1, 5, 2, 4088, 4093, 4093, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 709, 4093, 0, 842, 3, 2, 0, 4092, 2, 0, 0, 2, 0, 4, 4, 3966, 2040, 687, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4092, 3, 3405, 0, 0, 709, 0, 4083, 0, 4092, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 4092, 0, 4087, 4093, 20, 4, 6, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 5, 3753, 1084, 4089, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 3, 34, 4, 0, 0, 0, 891, 0, 2, 3, 4, 0, 0, 4069, 4092, 4081, 2035, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 1, 2, 0, 0, 4088, 3, 127, 1, 8, 2, 4091, 0, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16399, 16398, 16787, 16390, 16404, 17122, 15231, 16341] avg=16384.0000 max=17122 max_rank=5 min=15231 min_rank=6 imbalance_ratio=1.045044 max_load_reduction=113950 max_load_reduction_pct=0.869370 moved_to_echo=65098 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 18, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 634, 4096, 0, 388, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3969, 2042, 687, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3407, 0, 0, 250, 0, 4096, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4094, 0, 4095, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 3756, 124, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3240, 0, 0, 0, 0, 0, 0, 3683, 4096, 4090, 2036, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 127, 0, 0, 0, 4094, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16376, 16252, 15851, 16388, 16168, 17153, 16504] avg=16384.0000 max=17153 max_rank=6 min=15851 min_rank=3 imbalance_ratio=1.046936 max_load_reduction=113919 max_load_reduction_pct=0.869133 moved_to_echo=66213 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 18, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 4096, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3969, 2042, 688, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 3408, 0, 0, 350, 0, 4096, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4083, 0, 4096, 4095, 1, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 3756, 7, 4096, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3937, 0, 0, 0, 0, 0, 0, 3964, 4096, 4090, 2036, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 127, 0, 0, 0, 4096, 0, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16395, 16377, 15383, 15937, 16371, 16058, 18130, 16421] avg=16384.0000 max=18130 max_rank=6 min=15383 min_rank=2 imbalance_ratio=1.106567 max_load_reduction=112942 max_load_reduction_pct=0.861679 moved_to_echo=66980 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 18, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4093, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1783, 4096, 0, 746, 0, 0, 0, 4095, 0, 0, 0, 0, 11, 0, 0, 3969, 2042, 688, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3408, 0, 0, 696, 0, 4096, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 4096, 4096, 0, 0, 0, 4086, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 11, 0, 23, 3756, 326, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1545, 0, 0, 0, 0, 0, 2, 3470, 4096, 4091, 2036, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 127, 0, 0, 0, 4096, 0, 3676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16393, 16387, 17770, 16394, 16381, 16410, 15247, 16090] avg=16384.0000 max=17770 max_rank=2 min=15247 min_rank=6 imbalance_ratio=1.084595 max_load_reduction=113302 max_load_reduction_pct=0.864426 moved_to_echo=64128 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 18, 1, 1, 4, 1, 0, 0, 4086, 1, 10, 4095, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 12, 1, 0, 1, 1, 1, 0, 0, 1, 4084, 4095, 4095, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 4094, 0, 82, 1, 1, 0, 4088, 1, 1, 3, 1, 0, 0, 1, 3968, 2041, 687, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 3407, 0, 0, 451, 0, 4095, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 4094, 0, 4094, 4095, 2, 1, 0, 4095, 0, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4093, 0, 0, 0, 4, 3755, 110, 4092, 1, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 3196, 0, 1, 1, 0, 0, 0, 3292, 4095, 4088, 2036, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 0, 1, 0, 0, 4095, 1, 127, 0, 0, 2, 4094, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16383, 16402, 16147, 16396, 16151, 16719, 16467] avg=16384.0000 max=16719 max_rank=6 min=16147 min_rank=3 imbalance_ratio=1.020447 max_load_reduction=114353 max_load_reduction_pct=0.872444 moved_to_echo=65733 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 10, 6, 7, 21, 7, 0, 0, 4062, 6, 0, 4091, 0, 4088, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 5, 0, 6, 5, 1, 8, 4, 7, 0, 7, 3, 0, 7, 4064, 4090, 4089, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1072, 4089, 0, 93, 5, 0, 0, 4089, 9, 17, 0, 8, 0, 10, 9, 3965, 2041, 688, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4087, 6, 3396, 0, 37, 323, 0, 4071, 0, 4091, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4084, 0, 4088, 4071, 0, 23, 7, 4093, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4087, 0, 0, 0, 14, 3744, 61, 4071, 0, 0, 0, 0, 5, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 29, 13, 0, 0, 0, 3474, 0, 5, 7, 0, 7, 1, 3316, 4088, 4084, 2038, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 4, 0, 0, 4091, 8, 124, 0, 8, 2, 4085, 0, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16383, 16440, 16019, 16377, 16075, 17076, 16315] avg=16384.0000 max=17076 max_rank=6 min=16019 min_rank=3 imbalance_ratio=1.042236 max_load_reduction=113996 max_load_reduction_pct=0.869720 moved_to_echo=65843 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 18, 0, 0, 0, 0, 0, 0, 4000, 0, 6, 4092, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 9, 0, 0, 0, 9, 0, 0, 0, 0, 4088, 4096, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 4096, 0, 727, 0, 0, 0, 4087, 0, 0, 0, 0, 4, 0, 0, 3965, 2042, 688, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 3408, 0, 0, 1776, 0, 4091, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 4091, 4096, 10, 0, 0, 4070, 0, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 2, 0, 15, 3756, 394, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1702, 2, 0, 0, 0, 0, 0, 3604, 4096, 4090, 2036, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4093, 0, 127, 0, 0, 0, 4096, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16309, 16391, 16127, 17436, 16378, 16450, 15539, 16442] avg=16384.0000 max=17436 max_rank=3 min=15539 min_rank=6 imbalance_ratio=1.064209 max_load_reduction=113636 max_load_reduction_pct=0.866974 moved_to_echo=64809 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 256, 128, 128, 128, 128, 0, 0, 32640, 128, 128, 32768, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 27264, 0, 128, 5888, 0, 32768, 0, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 0, 128, 0, 128, 30080, 2688, 32768, 128, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 0, 20096, 128, 128, 128, 128, 128, 128, 29184, 32768, 32768, 16384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 0, 32768, 128, 1024, 128, 128, 128, 32768, 0, 31872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 6272, 32768, 0, 2944, 128, 128, 0, 32768, 128, 128, 128, 128, 128, 128, 128, 31744, 16384, 5504, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 32768, 0, 32768, 32768, 128, 128, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132992 observed_loss=-1920 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132992 slack_slots=7040 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 129, 0, 4096, 4088, 0, 4096, 0, 0, 4092, 0, 4, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 1665, 0, 0, 0, 8, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3857, 0, 0, 2355, 0, 0, 0, 0, 0, 31, 4096, 0, 4096, 56, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 18, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[3, 2, 4093, 0, 0, 1, 0, 215, 3, 1, 0, 2, 2, 3, 1, 4092, 0, 1, 1, 0, 4092, 4093, 0, 4091, 1, 4, 4092, 3, 33, 0, 4090, 0, 4093, 4, 0, 4093, 0, 0, 3340, 3, 0, 0, 5, 0, 4093, 4, 0, 4088, 4, 4092, 4093, 4068, 4, 0, 0, 2, 0, 4, 0, 0, 2, 0, 0, 3, 3, 0, 3, 0, 3, 3200, 0, 0, 1094, 0, 0, 0, 0, 3, 3, 4092, 0, 4087, 304, 4092, 1, 4091, 4092, 4093, 4092, 2, 4092, 1, 0, 0, 4091, 4078, 3, 2, 4, 45, 0, 4, 0, 4092, 4, 1, 3, 0, 4092, 2, 4089, 1, 4, 4086, 2, 1, 2, 3, 4091, 0, 3, 4092, 0, 9, 0, 0, 6, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 3859, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 4095, 0, 0, 0, 0, 4095, 4092, 1, 4095, 0, 0, 4090, 1, 51, 0, 4094, 0, 4093, 0, 0, 4095, 0, 0, 2497, 0, 1, 0, 27, 0, 4095, 1, 0, 4094, 1, 4095, 4095, 4072, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 3693, 2, 0, 2059, 1, 1, 0, 0, 1, 23, 4095, 0, 4045, 153, 4095, 1, 4095, 4095, 4094, 4095, 0, 4095, 0, 0, 0, 4094, 4088, 1, 1, 1, 13, 0, 0, 0, 4094, 0, 0, 1, 0, 4095, 1, 4094, 0, 1, 4094, 0, 1, 1, 1, 4095, 1, 1, 4095, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 56, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 116, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 2841, 0, 0, 0, 5, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3293, 0, 0, 1806, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 16, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[8, 7, 4085, 0, 0, 5, 0, 187, 7, 5, 3, 6, 6, 6, 0, 4090, 0, 0, 0, 0, 4088, 4090, 3, 4076, 3, 3, 4088, 7, 0, 0, 4087, 0, 4087, 9, 0, 4089, 9, 0, 2246, 0, 5, 0, 0, 0, 4089, 9, 0, 4075, 8, 4090, 4089, 4033, 3, 0, 0, 0, 0, 8, 0, 0, 5, 0, 0, 8, 7, 0, 5, 0, 6, 3290, 9, 0, 2611, 0, 0, 3, 0, 7, 0, 4089, 0, 3970, 0, 4088, 5, 4085, 4090, 4090, 4086, 7, 4087, 1, 0, 0, 4084, 4089, 7, 6, 7, 38, 0, 7, 0, 4086, 3, 0, 6, 0, 4087, 6, 4083, 0, 7, 4081, 3, 0, 7, 7, 4089, 0, 5, 4087, 0, 2, 0, 0, 17, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4066, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4093, 0, 0, 4096, 0, 0, 3201, 0, 0, 0, 4, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2942, 0, 0, 2020, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 2, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4069, 0, 0, 0, 4, 102, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 282, 0, 4096, 4094, 0, 4096, 0, 0, 4076, 0, 0, 0, 4096, 3, 4069, 0, 0, 4089, 0, 5, 3087, 0, 0, 0, 6, 0, 4096, 0, 0, 4096, 0, 4091, 4096, 4090, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2464, 0, 0, 2214, 0, 0, 0, 0, 0, 28, 4096, 0, 4096, 60, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 6, 0, 25, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 5, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 76, 0, 4096, 4089, 0, 4096, 0, 0, 4092, 0, 372, 0, 4096, 2, 4096, 0, 0, 4096, 0, 0, 2299, 0, 0, 0, 21, 0, 4096, 0, 0, 4096, 0, 4096, 4095, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 0, 0, 3062, 0, 0, 0, 0, 0, 47, 4096, 0, 4096, 40, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 6, 4096, 4086, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 8, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 4088, 0, 4096, 0, 0, 4092, 0, 4, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1665, 0, 0, 0, 8, 0, 1446, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2849, 1258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3857, 0, 0, 2355, 0, 0, 0, 0, 0, 31, 4096, 4096, 2838, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 56, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 18, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 2650, 1251, 130, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2845, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16418, 16375, 15407, 16401, 17335, 16387, 16369, 16380] avg=16384.0000 max=17335 max_rank=4 min=15407 min_rank=2 imbalance_ratio=1.058044 max_load_reduction=113737 max_load_reduction_pct=0.867744 moved_to_echo=66471 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 2, 4093, 0, 0, 1, 0, 215, 3, 1, 0, 2, 2, 3, 1, 4092, 4092, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4093, 0, 4091, 1, 4, 4092, 3, 33, 0, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 0, 4093, 0, 0, 3340, 3, 0, 0, 5, 0, 1445, 4, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 4093, 4068, 4, 0, 0, 2, 0, 4, 0, 0, 2, 0, 0, 3, 2846, 1258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 3, 3200, 0, 0, 1094, 0, 0, 0, 0, 3, 3, 4092, 4092, 2834, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 304, 0, 1, 4041, 0, 0, 0, 2, 0, 1, 0, 0, 4091, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 4, 45, 0, 4, 0, 4092, 4, 1, 3, 0, 4092, 2, 4089, 1, 2648, 1250, 130, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4086, 2, 1, 2, 3, 4091, 0, 3, 4092, 0, 9, 0, 0, 6, 0, 2842, 1246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16541, 16279, 17075, 16376, 15389, 16605, 16420, 16387] avg=16384.0000 max=17075 max_rank=2 min=15389 min_rank=4 imbalance_ratio=1.042175 max_load_reduction=113997 max_load_reduction_pct=0.869728 moved_to_echo=64801 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 3859, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 4095, 4095, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 4095, 0, 0, 4090, 1, 51, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4095, 0, 0, 2497, 0, 1, 0, 27, 0, 1446, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4095, 4072, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 2847, 1258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 3693, 2, 0, 2059, 1, 1, 0, 0, 1, 23, 4095, 4095, 2837, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 153, 0, 1, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 13, 0, 0, 0, 4094, 0, 0, 1, 0, 4095, 1, 4094, 0, 2649, 1250, 130, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 1, 1, 1, 4095, 1, 1, 4095, 0, 1, 0, 0, 0, 0, 2845, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16089, 16294, 16254, 16373, 16872, 16426, 16380, 16384] avg=16384.0000 max=16872 max_rank=4 min=16089 min_rank=0 imbalance_ratio=1.029785 max_load_reduction=114200 max_load_reduction_pct=0.871277 moved_to_echo=66062 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 116, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2841, 0, 0, 0, 5, 0, 1446, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2848, 1258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3293, 0, 0, 1806, 0, 0, 0, 0, 0, 0, 4096, 4096, 2838, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 16, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 2650, 1251, 130, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2845, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16426, 16580, 16394, 16191, 16347, 16369, 16381] avg=16384.0000 max=16580 max_rank=2 min=16191 min_rank=4 imbalance_ratio=1.011963 max_load_reduction=114492 max_load_reduction_pct=0.873505 moved_to_echo=65288 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4066, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 3201, 0, 0, 0, 4, 0, 1446, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2848, 1258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2942, 0, 0, 2020, 0, 0, 0, 0, 0, 0, 4096, 4096, 2838, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 2650, 1251, 130, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2845, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16340, 16266, 16936, 16394, 16054, 16332, 16369, 16381] avg=16384.0000 max=16936 max_rank=2 min=16054 min_rank=4 imbalance_ratio=1.033691 max_load_reduction=114136 max_load_reduction_pct=0.870789 moved_to_echo=65136 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[8, 7, 4085, 0, 0, 5, 0, 187, 7, 5, 3, 6, 6, 6, 0, 4090, 4090, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 4076, 3, 3, 4088, 7, 0, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 9, 0, 4089, 9, 0, 2246, 0, 5, 0, 0, 0, 1439, 9, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4090, 4089, 4033, 3, 0, 0, 0, 0, 8, 0, 0, 5, 0, 0, 8, 2843, 1256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 5, 0, 6, 3290, 9, 0, 2611, 0, 0, 3, 0, 7, 0, 4089, 4088, 2832, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 0, 0, 5, 4030, 0, 0, 0, 7, 0, 1, 0, 0, 4084, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 7, 38, 0, 7, 0, 4086, 3, 0, 6, 0, 4087, 6, 4083, 0, 2650, 1249, 135, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4081, 3, 0, 7, 7, 4089, 0, 5, 4087, 0, 2, 0, 0, 17, 0, 2837, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16533, 16222, 15968, 16343, 17009, 16186, 16425, 16386] avg=16384.0000 max=17009 max_rank=4 min=15968 min_rank=2 imbalance_ratio=1.038147 max_load_reduction=114063 max_load_reduction_pct=0.870232 moved_to_echo=66006 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 4089, 0, 4096, 0, 0, 4092, 0, 372, 0, 3966, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2299, 0, 0, 0, 21, 0, 1446, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2849, 1259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 0, 0, 3062, 0, 0, 0, 0, 0, 47, 4096, 4096, 2837, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 40, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 2650, 1252, 130, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 2844, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 16693, 16054, 16402, 16423, 16370, 16371, 16387] avg=16384.0000 max=16693 max_rank=1 min=16054 min_rank=2 imbalance_ratio=1.018860 max_load_reduction=114379 max_load_reduction_pct=0.872643 moved_to_echo=65551 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4069, 0, 0, 0, 4, 102, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 0, 4094, 0, 4096, 0, 0, 4076, 0, 0, 0, 3966, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 4089, 0, 5, 3087, 0, 0, 0, 6, 0, 1446, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 4090, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2848, 1258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2464, 0, 0, 2214, 0, 0, 0, 0, 0, 28, 4096, 4096, 2838, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 60, 0, 0, 4046, 0, 0, 0, 0, 0, 6, 0, 25, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 2650, 1251, 130, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 2845, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16395, 16517, 16798, 16389, 15799, 16419, 16369, 16386] avg=16384.0000 max=16798 max_rank=2 min=15799 min_rank=4 imbalance_ratio=1.025269 max_load_reduction=114274 max_load_reduction_pct=0.871841 moved_to_echo=64973 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 0, 32768, 128, 128, 21248, 128, 128, 0, 128, 0, 11648, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 32768, 32768, 128, 128, 0, 128, 0, 128, 0, 0, 128, 128, 0, 128, 22784, 10112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 32512, 0, 0, 128, 128, 896, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 640, 0, 0, 32768, 128, 32768, 128, 128, 32768, 128, 640, 0, 31744, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 32640, 640, 0, 128, 32384, 0, 0, 0, 128, 0, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 0, 128, 25088, 128, 0, 17280, 128, 128, 128, 0, 128, 256, 32768, 32768, 22784, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 0, 128, 0, 32768, 128, 128, 128, 0, 32768, 128, 32768, 128, 21248, 10112, 1152, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 32768, 128, 128, 32768, 0, 128, 128, 0, 128, 0, 22784, 9984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 2774, 0, 4096, 0, 0, 0, 110, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4065, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 31, 0, 0, 0, 0, 37, 4096, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 759, 4096, 4096, 0, 0, 0, 4033, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 459, 0, 4096, 0, 4096, 41, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4092, 0, 1, 0, 4093, 0, 1, 3607, 0, 4095, 0, 0, 0, 5, 4095, 1, 4095, 0, 1, 0, 1, 4094, 0, 4095, 1, 4095, 2, 1, 4094, 1, 4094, 4095, 0, 4094, 1, 0, 0, 1, 1, 4095, 1, 4091, 4046, 0, 0, 0, 0, 1, 0, 0, 2, 4094, 0, 0, 0, 0, 2, 0, 0, 1, 6, 4095, 0, 0, 4, 1, 1, 14, 0, 0, 1, 1, 4095, 4095, 1, 1, 4, 4093, 0, 2, 0, 0, 1, 4094, 2, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 1, 1, 4095, 497, 4095, 4094, 1, 1, 3, 4059, 1, 0, 4090, 1, 0, 0, 0, 0, 0, 1, 4095, 4095, 0, 1, 0, 6, 0, 4093, 0, 4095, 49, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4060, 0, 4, 0, 4092, 0, 3, 3825, 0, 4092, 0, 0, 5, 28, 4093, 3, 4092, 0, 3, 0, 2, 4087, 1, 4092, 2, 4070, 1, 1, 4091, 0, 4084, 4092, 1, 4093, 4, 0, 0, 4, 3, 4093, 3, 4088, 4063, 0, 1, 0, 0, 4, 0, 4, 4, 4090, 0, 0, 0, 6, 0, 2, 0, 0, 0, 4092, 0, 0, 0, 3, 3, 8, 0, 0, 3, 1, 4092, 4091, 2, 4, 9, 4092, 1, 5, 0, 0, 3, 4090, 0, 3, 0, 2, 0, 0, 0, 4089, 3, 5, 0, 2, 4093, 356, 4094, 4091, 0, 0, 6, 3871, 4, 0, 4083, 0, 0, 5, 0, 1, 3, 0, 4092, 4094, 0, 4, 4, 165, 0, 4092, 0, 4092, 26, 4, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 767, 0, 4096, 0, 0, 0, 37, 4096, 0, 4096, 0, 0, 0, 0, 4091, 0, 4096, 0, 4087, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 2330, 4096, 4096, 0, 0, 0, 4083, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 989, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 3252, 0, 4096, 0, 0, 0, 7, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4085, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 22, 0, 0, 0, 0, 21, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 182, 4093, 4096, 0, 0, 0, 4044, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 794, 0, 4096, 0, 4096, 158, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4089, 0, 9, 0, 4087, 11, 7, 516, 0, 4088, 0, 0, 2, 153, 4089, 7, 4088, 0, 6, 0, 7, 3704, 0, 4089, 6, 4055, 7, 0, 4088, 1, 4065, 4087, 0, 4084, 9, 0, 0, 8, 8, 4090, 7, 4084, 3966, 0, 0, 0, 0, 9, 0, 8, 14, 4086, 0, 0, 0, 2, 8, 3, 0, 10, 143, 4081, 0, 0, 0, 4, 6, 0, 0, 0, 7, 6, 4083, 4082, 6, 8, 32, 4086, 1, 0, 0, 0, 7, 4075, 0, 6, 0, 5, 0, 0, 5, 4087, 4, 11, 0, 7, 4089, 125, 4090, 4081, 0, 0, 19, 3938, 7, 0, 4074, 0, 0, 6, 0, 3, 0, 0, 4089, 4090, 0, 9, 0, 3810, 0, 4087, 0, 4089, 42, 8, 7, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 3114, 0, 4096, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 4094, 0, 4096, 0, 4082, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 622, 4094, 4096, 0, 0, 0, 4079, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 387, 0, 4096, 0, 4096, 4, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4091, 5, 0, 1, 4096, 0, 0, 1948, 0, 4096, 4, 3, 0, 3, 4096, 0, 4096, 9, 0, 0, 0, 3942, 1, 4096, 0, 4075, 0, 0, 4095, 0, 4096, 4096, 0, 4096, 0, 0, 4, 0, 0, 4092, 0, 4096, 4078, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 29, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 425, 4084, 4096, 0, 0, 0, 3954, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 1986, 0, 4096, 0, 4096, 6, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 2774, 0, 4096, 0, 0, 0, 110, 1581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4065, 0, 0, 4096, 0, 4096, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4076, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 31, 0, 0, 0, 0, 37, 4096, 0, 0, 4096, 4005, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 2464, 1114, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 759, 4096, 3528, 0, 0, 0, 4033, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2982, 4096, 0, 0, 0, 459, 0, 4096, 0, 4096, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16753, 16444, 16428, 16412, 16377, 16384, 16504, 15770] avg=16384.0000 max=16753 max_rank=0 min=15770 min_rank=7 imbalance_ratio=1.022522 max_load_reduction=114319 max_load_reduction_pct=0.872185 moved_to_echo=65035 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 767, 0, 4096, 0, 0, 0, 37, 1581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4087, 0, 0, 4096, 0, 4096, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4005, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2464, 1114, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2330, 4096, 3528, 0, 0, 0, 4083, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2982, 4096, 0, 0, 0, 989, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14673, 16461, 16448, 16344, 16370, 16384, 18133, 16259] avg=16384.0000 max=18133 max_rank=6 min=14673 min_rank=0 imbalance_ratio=1.106750 max_load_reduction=112939 max_load_reduction_pct=0.861656 moved_to_echo=67146 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4060, 0, 4, 0, 4092, 0, 3, 3825, 0, 4092, 0, 0, 5, 28, 1580, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 4087, 1, 0, 2, 4070, 1, 1, 4091, 0, 4084, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 4, 0, 0, 4, 3, 4093, 3, 4088, 4063, 0, 1, 0, 0, 4, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4090, 0, 0, 0, 6, 0, 2, 0, 0, 0, 4092, 0, 0, 4092, 4001, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 8, 0, 0, 3, 1, 4092, 4091, 2, 4, 9, 4092, 1, 5, 2462, 1113, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4090, 0, 3, 0, 2, 0, 0, 0, 4089, 3, 5, 0, 2, 4093, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 356, 4094, 3525, 0, 0, 6, 3871, 4, 0, 4083, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2979, 4094, 0, 4, 4, 165, 0, 4092, 0, 4092, 26, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17692, 16433, 16420, 16342, 16392, 16382, 15945, 15466] avg=16384.0000 max=17692 max_rank=0 min=15466 min_rank=7 imbalance_ratio=1.079834 max_load_reduction=113380 max_load_reduction_pct=0.865021 moved_to_echo=64185 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 1, 0, 4093, 0, 1, 3607, 0, 4095, 0, 0, 0, 5, 1581, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4094, 0, 0, 1, 4095, 2, 1, 4094, 1, 4094, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 1, 1, 4095, 1, 4091, 4046, 0, 0, 0, 0, 1, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 2, 0, 0, 1, 6, 4095, 0, 0, 4095, 4004, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 1, 14, 0, 0, 1, 1, 4095, 4095, 1, 1, 4, 4093, 0, 2, 2463, 1114, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 2, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 1, 1, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 4095, 3527, 1, 1, 3, 4059, 1, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2981, 4095, 0, 1, 0, 6, 0, 4093, 0, 4095, 49, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17476, 16475, 16395, 16350, 16393, 16386, 16275, 15322] avg=16384.0000 max=17476 max_rank=0 min=15322 min_rank=7 imbalance_ratio=1.066650 max_load_reduction=113596 max_load_reduction_pct=0.866669 moved_to_echo=64376 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 3114, 0, 4096, 0, 0, 0, 1, 1581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4082, 0, 0, 4096, 0, 4096, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4005, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2464, 1114, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 622, 4094, 3528, 0, 0, 0, 4079, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2982, 4096, 0, 0, 0, 387, 0, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16984, 16459, 16448, 16347, 16370, 16384, 16419, 15661] avg=16384.0000 max=16984 max_rank=0 min=15661 min_rank=7 imbalance_ratio=1.036621 max_load_reduction=114088 max_load_reduction_pct=0.870422 moved_to_echo=64834 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 3252, 0, 4096, 0, 0, 0, 7, 1581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4085, 0, 0, 4096, 0, 4096, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3816, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 22, 0, 0, 0, 0, 21, 4096, 0, 0, 4096, 4005, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 4096, 0, 0, 2464, 1114, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 4093, 3528, 0, 0, 0, 4044, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2982, 4096, 0, 0, 0, 794, 0, 4096, 0, 4096, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17128, 16464, 16168, 16387, 16378, 16384, 15941, 16222] avg=16384.0000 max=17128 max_rank=0 min=15941 min_rank=6 imbalance_ratio=1.045410 max_load_reduction=113944 max_load_reduction_pct=0.869324 moved_to_echo=64925 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4089, 0, 9, 0, 4087, 11, 7, 516, 0, 4088, 0, 0, 2, 153, 1569, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7, 3704, 0, 0, 6, 4055, 7, 0, 4088, 1, 4065, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 9, 0, 0, 8, 8, 4090, 7, 4084, 3966, 0, 0, 0, 0, 9, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 14, 4086, 0, 0, 0, 2, 8, 3, 0, 10, 143, 4081, 0, 0, 4088, 3999, 58, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 0, 0, 0, 7, 6, 4083, 4082, 6, 8, 32, 4086, 1, 0, 2462, 1117, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4075, 0, 6, 0, 5, 0, 0, 5, 4087, 4, 11, 0, 7, 4089, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 4090, 3512, 0, 0, 19, 3938, 7, 0, 4074, 0, 0, 6, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2972, 4090, 0, 9, 0, 3810, 0, 4087, 0, 4089, 42, 8, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14538, 16027, 16330, 16501, 16404, 16384, 15774, 19114] avg=16384.0000 max=19114 max_rank=7 min=14538 min_rank=0 imbalance_ratio=1.166626 max_load_reduction=111958 max_load_reduction_pct=0.854172 moved_to_echo=67676 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4091, 5, 0, 1, 4096, 0, 0, 1948, 0, 4096, 4, 3, 0, 3, 1581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3942, 1, 0, 0, 4075, 0, 0, 4095, 0, 4096, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4092, 0, 4096, 4078, 5, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 29, 0, 0, 0, 0, 12, 4096, 0, 0, 4096, 4005, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 4096, 0, 0, 2464, 1114, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 425, 4084, 3528, 0, 0, 0, 3954, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2982, 4092, 0, 0, 0, 1986, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15828, 16309, 16435, 16389, 16388, 16384, 16081, 17258] avg=16384.0000 max=17258 max_rank=7 min=15828 min_rank=0 imbalance_ratio=1.053345 max_load_reduction=113814 max_load_reduction_pct=0.868332 moved_to_echo=66111 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 0, 128, 32256, 128, 0, 128, 32640, 128, 128, 32768, 128, 32768, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 5376, 32768, 28288, 128, 128, 128, 32128, 128, 0, 32768, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 128, 32768, 128, 128, 19840, 0, 32768, 128, 128, 128, 384, 12672, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 23936, 32768, 0, 128, 128, 8704, 0, 32768, 0, 32768, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 32768, 128, 128, 0, 128, 0, 0, 128, 32768, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 32768, 128, 0, 0, 128, 128, 128, 0, 128, 256, 32768, 0, 0, 32768, 32128, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 128, 32768, 128, 32768, 32256, 128, 128, 0, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 128, 32768, 32768, 128, 128, 128, 32768, 128, 128, 19712, 8960, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4060, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4094, 25, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4095, 0, 4096, 4096, 0, 4091, 0, 0, 0, 0, 0, 4096, 19, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 3996, 2, 4087, 7, 0, 22, 4081, 5, 0, 0, 0, 0, 6, 7, 0, 1, 0, 4087, 9, 7, 0, 1, 4088, 4086, 8, 7, 10, 4084, 53, 4080, 0, 0, 0, 8, 0, 10, 4088, 10, 4089, 4088, 0, 0, 7, 0, 0, 4, 0, 7, 0, 8, 0, 4072, 0, 1, 4087, 4089, 8, 0, 0, 0, 6, 7, 1, 6, 4088, 4, 0, 4086, 4086, 4088, 0, 7, 8, 4085, 10, 4086, 4086, 4087, 5, 0, 4040, 7, 12, 10, 4089, 0, 4084, 4083, 1, 4089, 0, 0, 0, 0, 9, 4070, 10, 7, 10, 0, 4087, 5, 0, 7, 3, 0, 0, 4083, 4090, 0, 4087, 0, 2, 24, 2, 0, 3, 0, 12, 0, 58, 0, 0, 9, 8, 8, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4059, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 0, 4096, 38, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4005, 3, 4092, 2, 0, 3, 4087, 2, 0, 0, 0, 0, 3, 3, 0, 0, 0, 4091, 5, 3, 4, 7, 4093, 4092, 4, 4, 3, 4090, 77, 4088, 0, 0, 0, 3, 0, 4, 4092, 2, 4093, 4092, 0, 1, 4, 0, 0, 3, 6, 3, 5, 3, 0, 4072, 0, 0, 4092, 4093, 4, 0, 0, 0, 2, 3, 1, 0, 4090, 1, 0, 4091, 4092, 4092, 3, 4, 7, 4092, 5, 4091, 4091, 4092, 3, 0, 4082, 2, 0, 4, 4091, 1, 4092, 4085, 14, 4093, 0, 0, 0, 0, 4, 4092, 0, 4, 4, 1, 4090, 2, 0, 4, 1, 0, 0, 4089, 4092, 0, 4092, 0, 0, 7, 2, 0, 2, 0, 6, 0, 12, 0, 0, 4, 0, 3, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4045, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4085, 47, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 3, 4070, 0, 0, 0, 0, 0, 4096, 38, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4080, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4058, 0, 4095, 1, 0, 1, 4093, 5, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4095, 0, 1, 0, 8, 4094, 4095, 1, 1, 0, 4090, 21, 4094, 0, 0, 0, 0, 0, 1, 4095, 0, 4095, 4095, 0, 0, 1, 2, 0, 0, 0, 1, 5, 1, 0, 4094, 0, 1, 4095, 4095, 1, 0, 0, 0, 1, 1, 0, 0, 4095, 1, 0, 4095, 4095, 4095, 0, 1, 3, 4093, 1, 4095, 4095, 4094, 1, 0, 4085, 1, 0, 1, 4095, 0, 4094, 4088, 0, 4095, 0, 0, 0, 0, 1, 4093, 18, 1, 1, 1, 4095, 0, 0, 1, 1, 0, 1, 4095, 4095, 0, 4095, 2, 0, 0, 0, 0, 0, 2, 2, 0, 1, 0, 0, 1, 2, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4020, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4095, 32, 4096, 0, 5, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4091, 0, 4096, 4096, 4, 4063, 0, 0, 1, 0, 0, 4096, 77, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4060, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 37, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 4094, 25, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4039, 31, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4059, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 17, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 19, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16401, 16376, 16380, 16387, 16384, 16395, 16381, 16368] avg=16384.0000 max=16401 max_rank=0 min=16368 min_rank=7 imbalance_ratio=1.001038 max_load_reduction=114671 max_load_reduction_pct=0.874870 moved_to_echo=65528 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4080, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 36, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4039, 31, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4060, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 17, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16420, 16356, 16380, 16385, 16384, 16401, 16378, 16368] avg=16384.0000 max=16420 max_rank=0 min=16356 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=114652 max_load_reduction_pct=0.874725 moved_to_echo=65531 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4059, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 36, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4039, 31, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4060, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 17, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 38, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16399, 16357, 16380, 16384, 16384, 16400, 16400, 16368] avg=16384.0000 max=16400 max_rank=5 min=16357 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=114672 max_load_reduction_pct=0.874878 moved_to_echo=65552 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4005, 3, 4092, 2, 0, 3, 4087, 2, 0, 0, 0, 0, 3, 3, 0, 4092, 37, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 5, 3, 4, 7, 0, 4061, 4, 4, 3, 4090, 77, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 4092, 2, 4093, 4092, 0, 1, 4, 0, 0, 3, 6, 4035, 31, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 3, 0, 4072, 0, 0, 4092, 4093, 4, 0, 0, 0, 2, 3, 1, 4055, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 3, 4, 7, 4092, 5, 4091, 4091, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 2, 0, 4, 4091, 1, 18, 4085, 14, 4093, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4, 4, 1, 4090, 2, 0, 4, 1, 0, 0, 4089, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 2, 0, 6, 0, 12, 0, 0, 4, 0, 3, 4, 4092, 4093, 4092, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16345, 16437, 16388, 16372, 16388, 16394, 16357, 16391] avg=16384.0000 max=16437 max_rank=1 min=16345 min_rank=0 imbalance_ratio=1.003235 max_load_reduction=114635 max_load_reduction_pct=0.874596 moved_to_echo=65530 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3996, 2, 4087, 7, 0, 22, 4081, 5, 0, 0, 0, 0, 6, 7, 0, 4086, 37, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4087, 9, 7, 0, 1, 0, 4051, 8, 7, 10, 4084, 53, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 10, 4088, 10, 4089, 4088, 0, 0, 7, 0, 0, 4, 0, 4030, 35, 27, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8, 0, 4072, 0, 1, 4087, 4089, 8, 0, 0, 0, 6, 7, 1, 4050, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4, 0, 4086, 0, 0, 0, 7, 8, 4085, 10, 4082, 4086, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 7, 12, 10, 4089, 0, 13, 4083, 1, 4089, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 10, 7, 10, 0, 4087, 5, 0, 7, 3, 0, 0, 4083, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 24, 2, 0, 3, 0, 12, 0, 58, 0, 0, 9, 8, 8, 7, 4088, 4088, 4087, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16353, 16398, 16400, 16377, 16379, 16353, 16345, 16467] avg=16384.0000 max=16467 max_rank=7 min=16345 min_rank=6 imbalance_ratio=1.005066 max_load_reduction=114605 max_load_reduction_pct=0.874367 moved_to_echo=65544 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4045, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 37, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 4085, 47, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4039, 31, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4059, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 17, 4096, 3, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 38, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16389, 16380, 16387, 16384, 16378, 16400, 16367] avg=16384.0000 max=16400 max_rank=6 min=16367 min_rank=7 imbalance_ratio=1.000977 max_load_reduction=114672 max_load_reduction_pct=0.874878 moved_to_echo=65529 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4058, 0, 4095, 1, 0, 1, 4093, 5, 0, 0, 0, 0, 1, 1, 0, 4095, 37, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 8, 0, 4064, 1, 1, 0, 4090, 21, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 4095, 4095, 0, 0, 1, 2, 0, 0, 0, 4037, 31, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 1, 0, 4094, 0, 1, 4095, 4095, 1, 0, 0, 0, 1, 1, 0, 4057, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 1, 3, 4093, 1, 4095, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 1, 4095, 0, 17, 4088, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 18, 1, 1, 1, 4095, 0, 0, 1, 1, 0, 1, 4095, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 2, 0, 1, 0, 0, 1, 2, 1, 1, 4095, 4094, 4095, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16403, 16375, 16379, 16392, 16385, 16383, 16380, 16375] avg=16384.0000 max=16403 max_rank=0 min=16375 min_rank=1 imbalance_ratio=1.001160 max_load_reduction=114669 max_load_reduction_pct=0.874855 moved_to_echo=65523 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4020, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 36, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 4095, 32, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4039, 31, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4060, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 17, 4096, 4, 4063, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 77, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16364, 16384, 16385, 16388, 16384, 16368, 16431, 16368] avg=16384.0000 max=16431 max_rank=6 min=16364 min_rank=0 imbalance_ratio=1.002869 max_load_reduction=114641 max_load_reduction_pct=0.874641 moved_to_echo=65551 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32384, 128, 32768, 128, 0, 128, 32768, 128, 0, 0, 128, 0, 128, 128, 0, 32768, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 256, 128, 128, 128, 32768, 128, 0, 128, 128, 0, 128, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 128, 128, 128, 32768, 128, 256, 32768, 128, 32768, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 32768, 128, 128, 128, 128, 0, 32512, 128, 128, 128, 32768, 384, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 128, 32768, 32768, 128, 0, 0, 0, 128, 128, 128, 32512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 128, 0, 128, 0, 0, 128, 128, 128, 128, 32768, 32768, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 0, 128, 32768, 128, 32768, 32768, 0, 128, 128, 128, 0, 128, 128, 32384, 256, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 32768, 0, 0, 128, 128, 128, 32768, 128, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[9, 0, 7, 9, 0, 0, 7, 1, 4087, 4085, 6, 4091, 0, 0, 9, 1, 0, 4083, 4063, 1, 4088, 15, 4085, 4084, 9, 0, 5, 0, 4086, 0, 4089, 8, 9, 4089, 4085, 0, 0, 5, 0, 0, 32, 0, 3, 6, 0, 0, 0, 3876, 4088, 7, 0, 7, 4, 4084, 6, 0, 4, 219, 10, 7, 2, 2, 4082, 0, 4084, 0, 0, 0, 6, 4044, 0, 4089, 0, 0, 0, 0, 0, 14, 9, 7, 5, 4081, 4085, 6, 2, 7, 4087, 11, 4080, 4, 0, 4087, 9, 8, 0, 5, 3, 0, 10, 6, 4087, 24, 0, 6, 7, 4086, 0, 7, 0, 0, 9, 4075, 0, 9, 0, 4083, 0, 8, 0, 4087, 4088, 0, 4088, 12, 4085, 6, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 63, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 55, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3616, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 321, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 142, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 5, 0, 25, 4096, 4096, 0, 4091, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 1, 0, 2, 1, 1, 0, 4095, 4095, 0, 4095, 0, 0, 1, 0, 0, 4095, 4092, 0, 4095, 2, 4094, 4095, 1, 0, 0, 0, 4094, 0, 4095, 0, 1, 4095, 4094, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 3649, 4095, 1, 0, 1, 2, 4095, 1, 0, 0, 363, 1, 1, 0, 0, 4095, 0, 4094, 2, 0, 0, 1, 4074, 1, 4095, 0, 0, 0, 1, 0, 0, 1, 1, 0, 4094, 4095, 1, 1, 1, 4094, 1, 4095, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 103, 4094, 0, 0, 1, 1, 4093, 0, 1, 0, 0, 1, 4095, 0, 1, 0, 4095, 0, 0, 2, 4095, 4095, 0, 4095, 2, 4095, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3908, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 132, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4091, 46, 0, 23, 4096, 4096, 0, 4093, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[5, 0, 3, 0, 0, 4, 5, 0, 4092, 4091, 0, 4094, 0, 0, 4, 2, 0, 4090, 4078, 0, 4092, 6, 4091, 4091, 4, 0, 1, 0, 4091, 0, 4092, 0, 5, 4093, 4092, 3, 0, 1, 0, 0, 12, 0, 5, 0, 5, 0, 0, 3456, 4092, 4, 0, 3, 0, 4091, 4, 0, 1, 649, 4, 3, 0, 16, 4089, 0, 4092, 0, 0, 0, 3, 4044, 0, 4092, 0, 0, 0, 0, 0, 0, 4, 4, 2, 4083, 4091, 4, 7, 3, 4092, 5, 4086, 2, 4, 4092, 4, 4, 0, 0, 4, 0, 7, 29, 4091, 15, 0, 2, 4, 4091, 0, 3, 0, 0, 4, 4069, 0, 4, 0, 4092, 0, 4, 0, 4092, 4092, 4, 4090, 0, 4091, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3937, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 116, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 29, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 14, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 4093, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3872, 4096, 0, 5, 0, 0, 4096, 0, 0, 0, 103, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 70, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 18, 0, 38, 4096, 4090, 0, 4096, 0, 4096, 0, 4, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3779, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 4096, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 63, 0, 0, 0, 0, 4096, 0, 3837, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3783, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 55, 0, 4096, 0, 4096, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16393, 16562, 16212, 16401, 16384, 16325, 16415] avg=16384.0000 max=16562 max_rank=2 min=16212 min_rank=3 imbalance_ratio=1.010864 max_load_reduction=114510 max_load_reduction_pct=0.873642 moved_to_echo=65525 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 7, 9, 0, 0, 7, 1, 4087, 4085, 6, 4091, 0, 0, 9, 1, 3772, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4063, 1, 0, 15, 4085, 4084, 9, 0, 5, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4089, 4085, 0, 0, 5, 0, 0, 32, 0, 3, 6, 0, 0, 0, 3876, 4089, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 7, 4, 4084, 6, 0, 4, 219, 10, 7, 2, 2, 4082, 0, 3834, 29, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 6, 4044, 0, 4089, 0, 0, 0, 0, 0, 14, 9, 7, 3787, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4081, 4085, 6, 2, 7, 0, 11, 4080, 4, 0, 4083, 9, 8, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 10, 6, 4087, 24, 0, 6, 7, 4086, 0, 7, 0, 0, 9, 4075, 3773, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4083, 0, 8, 0, 0, 4088, 0, 4088, 12, 4056, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16398, 16353, 16495, 16389, 16355, 16386, 16345, 16351] avg=16384.0000 max=16495 max_rank=2 min=16345 min_rank=6 imbalance_ratio=1.006775 max_load_reduction=114577 max_load_reduction_pct=0.874153 moved_to_echo=65437 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3780, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3616, 4096, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 321, 0, 0, 0, 0, 4096, 0, 3837, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3795, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 3782, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 25, 0, 4096, 0, 4091, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16393, 16205, 16470, 16399, 16382, 16456, 16385] avg=16384.0000 max=16470 max_rank=3 min=16205 min_rank=2 imbalance_ratio=1.005249 max_load_reduction=114602 max_load_reduction_pct=0.874344 moved_to_echo=65622 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3779, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3937, 4096, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 116, 0, 0, 0, 0, 4096, 0, 3837, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3795, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3782, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 4096, 0, 4096, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16393, 16526, 16265, 16400, 16384, 16349, 16374] avg=16384.0000 max=16526 max_rank=2 min=16265 min_rank=3 imbalance_ratio=1.008667 max_load_reduction=114546 max_load_reduction_pct=0.873917 moved_to_echo=65507 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 0, 2, 1, 1, 0, 4095, 4095, 0, 4095, 0, 0, 1, 0, 3778, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4092, 0, 0, 2, 4094, 4095, 1, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4094, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 3649, 4095, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 2, 4095, 1, 0, 0, 363, 1, 1, 0, 0, 4095, 0, 3836, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 1, 4074, 1, 4095, 0, 0, 0, 1, 0, 0, 1, 1, 3794, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 1, 1, 1, 0, 1, 4095, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 4094, 0, 0, 1, 1, 4093, 0, 1, 0, 0, 1, 4095, 3780, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 2, 0, 4095, 0, 4095, 2, 4071, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16387, 16241, 16515, 16381, 16383, 16419, 16362] avg=16384.0000 max=16515 max_rank=3 min=16241 min_rank=2 imbalance_ratio=1.007996 max_load_reduction=114557 max_load_reduction_pct=0.874001 moved_to_echo=65545 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3780, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3908, 4096, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 3837, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3795, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 3782, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 46, 0, 23, 0, 4096, 0, 4093, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16391, 16503, 16151, 16397, 16384, 16443, 16421] avg=16384.0000 max=16503 max_rank=2 min=16151 min_rank=3 imbalance_ratio=1.007263 max_load_reduction=114569 max_load_reduction_pct=0.874092 moved_to_echo=65645 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 3, 0, 0, 4, 5, 0, 4092, 4091, 0, 4094, 0, 0, 4, 2, 3776, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4078, 0, 0, 6, 4091, 4091, 4, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4093, 4092, 3, 0, 1, 0, 0, 12, 0, 5, 0, 5, 0, 0, 3456, 4092, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 3, 0, 4091, 4, 0, 1, 649, 4, 3, 0, 16, 4089, 0, 3833, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 3, 4044, 0, 4092, 0, 0, 0, 0, 0, 0, 4, 4, 3791, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4083, 4091, 4, 7, 3, 0, 5, 4086, 2, 4, 4092, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7, 29, 4091, 15, 0, 2, 4, 4091, 0, 3, 0, 0, 4, 4069, 3778, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4092, 0, 4, 0, 0, 4092, 4, 4090, 0, 4067, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16369, 16065, 16813, 16346, 16387, 16347, 16355] avg=16384.0000 max=16813 max_rank=3 min=16065 min_rank=2 imbalance_ratio=1.026184 max_load_reduction=114259 max_load_reduction_pct=0.871727 moved_to_echo=65435 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 4093, 0, 0, 0, 0, 3779, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3872, 4096, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 4096, 0, 0, 0, 103, 0, 0, 0, 0, 4096, 0, 3837, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 1, 0, 0, 0, 0, 3796, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3783, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 18, 0, 38, 0, 4090, 0, 4096, 0, 4072, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16393, 16475, 16257, 16393, 16382, 16388, 16409] avg=16384.0000 max=16475 max_rank=2 min=16257 min_rank=3 imbalance_ratio=1.005554 max_load_reduction=114597 max_load_reduction_pct=0.874306 moved_to_echo=65572 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 128, 128, 128, 32768, 128, 0, 128, 1920, 128, 128, 128, 128, 32768, 0, 30720, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 0, 0, 128, 32768, 128, 32768, 0, 0, 0, 128, 0, 128, 128, 128, 30464, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 32768, 128, 0, 128, 0, 128, 128, 128, 128, 128, 128, 0, 128, 30336, 32768, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 0, 128, 32768, 128, 128, 32768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 32768, 128, 0, 128, 32768, 32768, 128, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 640, 32768, 128, 0, 128, 128, 32768, 0, 128, 0, 0, 128, 32768, 30336, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 32768, 128, 128, 256, 0, 32768, 128, 32768, 128, 32640, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 128, 32768, 32768, 128, 32768, 0, 0, 128, 128, 30336, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 4095, 4091, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 1, 0, 4084, 0, 4095, 4095, 0, 0, 4094, 4095, 660, 0, 0, 4094, 4095, 1, 1, 1, 4095, 0, 1, 1, 0, 0, 0, 3996, 0, 1, 4095, 4095, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 4094, 0, 1, 1, 0, 4095, 1, 4095, 4095, 1, 0, 0, 1, 0, 1, 0, 579, 4095, 1, 0, 0, 4095, 4095, 1, 0, 4095, 1, 1, 1, 0, 1, 8, 1, 4095, 1, 2, 4094, 0, 0, 0, 1, 68, 0, 1, 4095, 0, 4095, 0, 4093, 0, 1, 2880, 1, 0, 4094, 14, 1, 1, 0, 1, 4095, 1, 4095, 1, 1, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 4096, 1864, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 565, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 49, 0, 0, 4096, 0, 4096, 0, 4094, 0, 0, 1962, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 8, 0, 4096, 4096, 2084, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 383, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 43, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 1711, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4092, 1, 0, 4092, 4070, 0, 0, 0, 0, 5, 4, 4090, 0, 3, 0, 2, 5, 0, 3998, 0, 4092, 4091, 0, 0, 4090, 4092, 284, 0, 0, 4091, 4091, 3, 3, 4, 4091, 4, 5, 4, 0, 0, 0, 4080, 0, 4, 4090, 4092, 0, 0, 0, 5, 4, 4, 4, 0, 0, 1, 0, 4090, 0, 4, 4, 0, 4091, 3, 4092, 4091, 4, 0, 4, 4, 2, 0, 0, 0, 4092, 4, 0, 0, 4092, 4091, 0, 0, 4091, 3, 0, 4, 0, 4, 0, 4, 4092, 5, 3, 4092, 0, 0, 1, 4, 127, 7, 6, 4085, 0, 4090, 2, 4077, 2, 0, 3793, 0, 0, 4089, 23, 3, 5, 0, 0, 4091, 5, 4092, 4, 2, 2, 5, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 3, 4089, 0, 4096, 4096, 35, 0, 4096, 4091, 2295, 0, 0, 4096, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 11, 522, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 4053, 0, 0, 4096, 0, 4, 0, 0, 92, 0, 0, 4096, 0, 4095, 0, 4056, 0, 0, 1276, 0, 0, 4096, 4, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4087, 3, 0, 4088, 4058, 0, 0, 2, 0, 7, 2, 4082, 0, 0, 0, 10, 9, 0, 4087, 0, 4085, 4085, 0, 0, 4078, 4088, 1041, 0, 0, 4085, 4086, 8, 9, 10, 4085, 3, 11, 8, 0, 0, 0, 4058, 0, 8, 4084, 4088, 0, 0, 0, 10, 9, 9, 9, 10, 0, 5, 0, 4085, 0, 8, 8, 0, 4085, 6, 4086, 4086, 9, 0, 0, 8, 2, 0, 0, 72, 4088, 10, 0, 0, 4087, 4086, 6, 0, 4085, 6, 0, 9, 0, 10, 22, 9, 4087, 11, 5, 4088, 0, 0, 5, 8, 219, 0, 8, 4080, 0, 4082, 0, 4077, 2, 14, 2771, 0, 0, 4080, 15, 8, 8, 0, 3, 4087, 11, 4088, 8, 3, 11, 13, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 24, 0, 4096, 4092, 548, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 232, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 4096, 0, 2, 0, 0, 9, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 3373, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 4096, 2, 0, 4096, 4096, 312, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 5, 186, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 4096, 0, 0, 0, 0, 47, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 3628, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 4096, 4096, 2084, 0, 0, 2973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 4096, 4096, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 383, 3778, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 1711, 0, 0, 1123, 210, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16400, 17353, 16339, 16380, 16449, 16388, 15389, 16374] avg=16384.0000 max=17353 max_rank=1 min=15389 min_rank=6 imbalance_ratio=1.059143 max_load_reduction=113719 max_load_reduction_pct=0.867607 moved_to_echo=64600 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 4095, 4091, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4084, 0, 0, 0, 0, 0, 4094, 4095, 660, 0, 0, 2972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4095, 0, 1, 1, 0, 0, 0, 3996, 0, 1, 4095, 4095, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 4094, 0, 1, 1, 0, 4095, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4095, 1, 0, 0, 1, 0, 1, 0, 579, 3777, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 4095, 1, 1, 1, 0, 1, 8, 1, 4095, 1, 2, 4094, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 68, 0, 1, 4095, 0, 4095, 0, 4093, 0, 1, 2880, 1, 0, 1122, 210, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 14, 1, 1, 0, 1, 4095, 1, 4095, 1, 1, 0, 1, 0, 0, 0, 4081, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16393, 15906, 16370, 16381, 16646, 16405, 16581, 16390] avg=16384.0000 max=16646 max_rank=4 min=15906 min_rank=1 imbalance_ratio=1.015991 max_load_reduction=114426 max_load_reduction_pct=0.873001 moved_to_echo=66022 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 1864, 0, 0, 2973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 4096, 4096, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 565, 3778, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 4096, 0, 4096, 0, 4094, 0, 0, 1962, 0, 0, 1123, 210, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16399, 17125, 16126, 16380, 16631, 16393, 15644, 16374] avg=16384.0000 max=17125 max_rank=1 min=15644 min_rank=6 imbalance_ratio=1.045227 max_load_reduction=113947 max_load_reduction_pct=0.869347 moved_to_echo=65042 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 1, 0, 4092, 4070, 0, 0, 0, 0, 5, 4, 4090, 0, 3, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 3998, 0, 0, 0, 0, 0, 4090, 4092, 284, 0, 0, 2969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 4, 4091, 4, 5, 4, 0, 0, 0, 4080, 0, 4, 4090, 4092, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 4, 4, 0, 0, 1, 0, 4090, 0, 4, 4, 0, 4091, 4092, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4092, 4091, 4, 0, 4, 4, 2, 0, 0, 0, 3774, 4, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4091, 3, 0, 4, 0, 4, 0, 4, 4092, 5, 3, 4092, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 127, 7, 6, 4085, 0, 4090, 2, 4077, 2, 0, 3793, 0, 0, 1122, 210, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 23, 3, 5, 0, 0, 4091, 5, 4092, 4, 2, 2, 5, 0, 0, 0, 4076, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 15440, 16463, 16386, 16070, 16398, 17541, 16401] avg=16384.0000 max=17541 max_rank=6 min=15440 min_rank=1 imbalance_ratio=1.070618 max_load_reduction=113531 max_load_reduction_pct=0.866173 moved_to_echo=66410 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 24, 0, 4096, 4092, 548, 0, 0, 2973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 4096, 4096, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 232, 3778, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 9, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 3373, 0, 0, 1123, 210, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4081, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 15827, 16441, 16380, 16300, 16334, 17020, 16373] avg=16384.0000 max=17020 max_rank=6 min=15827 min_rank=1 imbalance_ratio=1.038818 max_load_reduction=114052 max_load_reduction_pct=0.870148 moved_to_echo=66027 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4087, 3, 0, 4088, 4058, 0, 0, 2, 0, 7, 2, 4082, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 9, 0, 4087, 0, 0, 0, 0, 0, 4078, 4088, 1041, 0, 0, 2961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 10, 4085, 3, 11, 8, 0, 0, 0, 4058, 0, 8, 4084, 4088, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 9, 9, 9, 10, 0, 5, 0, 4085, 0, 8, 8, 0, 4085, 4085, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4086, 4086, 9, 0, 0, 8, 2, 0, 0, 72, 3754, 10, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 6, 0, 4085, 6, 0, 9, 0, 10, 22, 9, 4087, 11, 5, 4088, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8, 219, 0, 8, 4080, 0, 4082, 0, 4077, 2, 14, 2771, 0, 0, 1124, 215, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 15, 8, 8, 0, 3, 4087, 11, 4088, 8, 3, 11, 13, 0, 0, 0, 4070, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16348, 16274, 16458, 16405, 16120, 16438, 16620, 16409] avg=16384.0000 max=16620 max_rank=6 min=16120 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=114452 max_load_reduction_pct=0.873199 moved_to_echo=65587 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 2, 0, 4096, 4096, 312, 0, 0, 2973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 4096, 4096, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 5, 186, 3778, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 3628, 0, 0, 1123, 210, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4081, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16400, 15565, 16425, 16380, 16257, 16361, 17311, 16373] avg=16384.0000 max=17311 max_rank=6 min=15565 min_rank=1 imbalance_ratio=1.056580 max_load_reduction=113761 max_load_reduction_pct=0.867928 moved_to_echo=66302 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4089, 0, 0, 0, 35, 0, 4096, 4091, 2295, 0, 0, 2973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 4090, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 11, 522, 3778, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 4053, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 92, 0, 0, 4096, 0, 4095, 0, 4056, 0, 0, 1276, 0, 0, 1123, 210, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4082, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16362, 17582, 16450, 16380, 16599, 16355, 14966, 16378] avg=16384.0000 max=17582 max_rank=1 min=14966 min_rank=6 imbalance_ratio=1.073120 max_load_reduction=113490 max_load_reduction_pct=0.865860 moved_to_echo=64298 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 0, 32768, 128, 128, 128, 0, 128, 128, 128, 32640, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 128, 0, 32768, 32768, 0, 0, 128, 0, 128, 128, 32768, 0, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32640, 0, 0, 0, 128, 0, 32768, 32768, 9088, 0, 0, 23808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 128, 0, 128, 32768, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 32768, 128, 0, 128, 128, 128, 128, 128, 2560, 30208, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 32128, 0, 128, 32768, 32768, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 0, 128, 0, 32768, 0, 128, 128, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 768, 128, 128, 32768, 0, 32768, 128, 32768, 128, 128, 21504, 128, 0, 9088, 1792, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 2, 4096, 0, 0, 0, 4096, 0, 0, 4094, 4096, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4046, 0, 0, 4096, 0, 8, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4092, 3, 4096, 0, 0, 0, 4096, 0, 0, 4093, 4096, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 4081, 0, 0, 4096, 0, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 21, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4040, 0, 0, 4096, 0, 13, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 4082, 1, 4087, 0, 4088, 4020, 2, 8, 0, 4, 0, 0, 0, 4074, 0, 0, 4086, 4, 0, 0, 10, 0, 6, 10, 3, 4, 4086, 12, 0, 0, 0, 0, 0, 8, 8, 9, 0, 0, 9, 8, 13, 8, 4085, 2, 11, 7, 4082, 0, 10, 0, 8, 4084, 0, 3923, 356, 4083, 0, 0, 52, 4086, 0, 8, 3854, 4084, 10, 0, 9, 12, 66, 0, 11, 4087, 0, 4087, 3, 0, 4086, 4087, 7, 35, 5, 7, 8, 4087, 16, 9, 4085, 3, 4084, 0, 6, 4081, 14, 0, 6, 4080, 0, 0, 4, 0, 4084, 0, 4079, 3, 0, 14, 4, 4090, 4085, 0, 0, 0, 9, 0, 12, 4057, 0, 4083, 0, 0, 0, 0, 4080, 4086, 0, 6, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 4094, 1, 4094, 0, 4095, 4083, 0, 1, 0, 0, 0, 2, 0, 4094, 0, 0, 4095, 0, 0, 0, 1, 0, 1, 1, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 2, 0, 4095, 0, 0, 1, 4093, 0, 1, 1, 1, 4094, 0, 4084, 72, 4094, 0, 0, 3, 4095, 0, 1, 4094, 4094, 1, 0, 0, 1, 10, 0, 1, 4095, 0, 4095, 0, 0, 4095, 4095, 1, 2, 1, 1, 1, 4094, 2, 1, 4095, 2, 4037, 0, 1, 4095, 2, 0, 1, 4094, 0, 1, 1, 0, 4093, 0, 4094, 0, 0, 1, 1, 4095, 4095, 0, 2, 0, 1, 0, 1, 4090, 0, 4094, 0, 0, 1, 0, 4094, 4095, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 4089, 0, 4091, 0, 4092, 4014, 0, 5, 0, 6, 0, 0, 0, 4088, 0, 0, 4092, 1, 0, 0, 5, 0, 3, 5, 0, 0, 4092, 5, 26, 0, 0, 0, 0, 4, 3, 5, 0, 2, 4, 4, 4, 3, 4091, 3, 8, 4, 4076, 0, 5, 0, 4, 4090, 0, 3834, 243, 4091, 0, 0, 9, 4091, 0, 4, 4082, 4090, 6, 0, 2, 5, 20, 0, 4, 4092, 0, 4092, 4, 0, 4091, 4092, 3, 13, 4, 4, 4, 4092, 6, 4, 4091, 53, 4084, 0, 3, 4088, 7, 0, 3, 4091, 0, 0, 0, 0, 4091, 0, 4090, 1, 0, 6, 4, 4092, 4090, 0, 0, 1, 4, 0, 4, 4087, 0, 4091, 0, 0, 0, 1, 4090, 4092, 0, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4092, 1, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4092, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 108, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1, 4096, 0, 4, 0, 4096, 0, 0, 4063, 4096, 0, 0, 0, 0, 17, 0, 0, 4092, 0, 4091, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 4001, 0, 0, 4096, 0, 23, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 4075, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 4085, 4069, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 3, 4096, 0, 0, 0, 2, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 4096, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 4081, 0, 0, 4096, 0, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 16363, 16377, 16382, 16370, 16382, 16395, 16392] avg=16384.0000 max=16411 max_rank=0 min=16363 min_rank=1 imbalance_ratio=1.001648 max_load_reduction=114661 max_load_reduction_pct=0.874794 moved_to_echo=65539 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 50, 0, 0, 4085, 4069, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 4096, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4046, 0, 0, 4096, 0, 8, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4084, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 16402, 16378, 16384, 16370, 16352, 16383, 16392] avg=16384.0000 max=16411 max_rank=0 min=16352 min_rank=5 imbalance_ratio=1.001648 max_load_reduction=114661 max_load_reduction_pct=0.874794 moved_to_echo=65497 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4085, 4069, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 4096, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4092, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 16352, 16379, 16381, 16372, 16390, 16395, 16392] avg=16384.0000 max=16411 max_rank=0 min=16352 min_rank=1 imbalance_ratio=1.001648 max_load_reduction=114661 max_load_reduction_pct=0.874794 moved_to_echo=65549 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 1, 4094, 0, 27, 4083, 0, 1, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 1, 1, 1, 0, 4095, 1, 0, 0, 0, 4084, 4068, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 2, 0, 4095, 0, 0, 1, 4093, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4094, 0, 4084, 72, 4094, 0, 0, 3, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 1, 10, 0, 1, 4095, 0, 4095, 0, 0, 4077, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 1, 4094, 2, 1, 4095, 2, 4037, 0, 1, 4095, 2, 0, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 1, 0, 4093, 0, 4094, 0, 0, 1, 1, 4095, 11, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4090, 0, 4094, 0, 0, 1, 0, 4094, 4095, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16396, 16352, 16381, 16445, 16375, 16345, 16393, 16385] avg=16384.0000 max=16445 max_rank=3 min=16345 min_rank=5 imbalance_ratio=1.003723 max_load_reduction=114627 max_load_reduction_pct=0.874535 moved_to_echo=65498 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 38, 0, 0, 4085, 4069, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 21, 0, 0, 4096, 0, 4096, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4040, 0, 0, 4096, 0, 13, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4083, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 16391, 16377, 16382, 16387, 16351, 16382, 16391] avg=16384.0000 max=16411 max_rank=0 min=16351 min_rank=5 imbalance_ratio=1.001648 max_load_reduction=114661 max_load_reduction_pct=0.874794 moved_to_echo=65511 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4082, 1, 4087, 0, 23, 4020, 2, 8, 0, 4, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4, 0, 0, 10, 0, 6, 10, 3, 4, 4086, 12, 0, 0, 0, 4080, 4063, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 9, 0, 0, 9, 8, 13, 8, 4085, 2, 11, 7, 4082, 4087, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8, 4084, 0, 3923, 356, 4083, 0, 0, 52, 0, 0, 8, 3854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 10, 0, 9, 12, 66, 0, 11, 4087, 0, 4087, 3, 0, 4057, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 5, 7, 8, 4087, 16, 9, 4085, 3, 4084, 0, 6, 4081, 14, 0, 6, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 4, 0, 4084, 0, 4079, 3, 0, 14, 4, 4090, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 12, 4057, 0, 4083, 0, 0, 0, 0, 4080, 4086, 0, 6, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16301, 16371, 16416, 16378, 16433, 16462, 16363, 16348] avg=16384.0000 max=16462 max_rank=5 min=16301 min_rank=0 imbalance_ratio=1.004761 max_load_reduction=114610 max_load_reduction_pct=0.874405 moved_to_echo=65606 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 4091, 0, 27, 4014, 0, 5, 0, 6, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 5, 0, 3, 5, 0, 0, 4092, 5, 26, 0, 0, 4079, 4065, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 5, 0, 2, 4, 4, 4, 3, 4091, 3, 8, 4, 4076, 4092, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4, 4090, 0, 3834, 243, 4091, 0, 0, 9, 0, 0, 4, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 6, 0, 2, 5, 20, 0, 4, 4092, 0, 4092, 4, 0, 4073, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4, 4, 4, 4092, 6, 4, 4091, 53, 4084, 0, 3, 4088, 7, 0, 3, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4091, 0, 4090, 1, 0, 6, 4, 4092, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4, 4087, 0, 4091, 0, 0, 0, 1, 4090, 4092, 0, 1, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16320, 16380, 16387, 16362, 16391, 16466, 16386, 16380] avg=16384.0000 max=16466 max_rank=5 min=16320 min_rank=0 imbalance_ratio=1.005005 max_load_reduction=114606 max_load_reduction_pct=0.874374 moved_to_echo=65623 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 108, 2, 0, 4085, 4069, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 4096, 0, 4, 0, 2, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 17, 0, 0, 4092, 0, 4091, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 4001, 0, 0, 4096, 0, 23, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 4075, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 16461, 16377, 16358, 16374, 16324, 16375, 16392] avg=16384.0000 max=16461 max_rank=1 min=16324 min_rank=5 imbalance_ratio=1.004700 max_load_reduction=114611 max_load_reduction_pct=0.874413 moved_to_echo=65465 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 128, 32768, 0, 256, 32640, 128, 128, 0, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 128, 128, 0, 32768, 0, 32768, 128, 128, 128, 128, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32768, 128, 0, 0, 128, 0, 128, 128, 128, 128, 32768, 128, 256, 128, 0, 32768, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 32384, 768, 32768, 0, 128, 128, 128, 0, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 128, 128, 256, 0, 128, 32768, 0, 32768, 128, 0, 32640, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 32768, 0, 0, 128, 128, 32768, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 32768, 128, 32512, 0, 128, 32768, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 0, 128, 128, 128, 0, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[146, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1785, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 1985, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 180, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[373, 0, 0, 0, 4096, 0, 5, 4096, 0, 0, 0, 2581, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 1079, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 83, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[121, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1291, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 1851, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 873, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[440, 2, 0, 1, 4095, 2, 0, 4095, 0, 1, 0, 2937, 0, 1, 4095, 1, 0, 2, 0, 0, 0, 4095, 3, 1, 0, 4094, 0, 1, 1, 0, 4094, 1, 1, 2, 0, 4092, 4095, 0, 1, 1, 4090, 4094, 2, 4093, 4095, 1, 4094, 0, 0, 0, 1, 1, 1, 622, 0, 0, 4094, 4093, 1, 1, 4095, 4, 1, 4066, 0, 0, 0, 4094, 0, 1, 0, 0, 1, 0, 1, 4094, 1, 4095, 2, 1, 1, 1, 5, 4095, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 4094, 4095, 0, 1, 4089, 0, 0, 1, 0, 0, 1, 4094, 2, 1, 1, 1, 1, 3, 0, 0, 0, 0, 4092, 0, 128, 4093, 1, 1, 0, 1, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[305, 4, 0, 5, 4090, 7, 0, 4090, 0, 5, 0, 2478, 11, 5, 4092, 1, 0, 5, 2, 0, 0, 4088, 6, 0, 0, 4068, 0, 4, 4, 0, 4089, 3, 2, 6, 0, 4090, 4090, 0, 10, 2, 4077, 4085, 5, 4087, 4090, 6, 4088, 2, 0, 4, 4, 4, 5, 408, 1, 3, 4091, 4090, 3, 3, 4091, 0, 4, 3959, 5, 0, 1, 4086, 0, 5, 0, 0, 3, 0, 4, 4090, 5, 4092, 0, 0, 5, 7, 16, 4091, 4091, 0, 0, 4090, 0, 0, 0, 1, 0, 4089, 4091, 4091, 0, 0, 4089, 0, 0, 0, 0, 0, 3, 4090, 3, 2, 2, 4, 4, 26, 0, 4, 0, 0, 4082, 2, 1034, 4084, 6, 5, 0, 6, 0, 0, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[402, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3120, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 596, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 63, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[458, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2665, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 943, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4073, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 52, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[3, 10, 0, 9, 4085, 13, 0, 4086, 0, 9, 0, 3356, 23, 10, 4086, 6, 0, 9, 9, 0, 7, 4085, 13, 5, 7, 4051, 6, 10, 12, 0, 4078, 6, 7, 13, 0, 4083, 4083, 0, 6, 10, 4038, 4074, 9, 4079, 4084, 11, 4082, 0, 0, 0, 9, 8, 8, 1, 0, 0, 4086, 4081, 7, 4, 4086, 14, 8, 4006, 0, 0, 5, 4077, 0, 11, 0, 0, 6, 0, 9, 4084, 9, 4087, 3, 6, 14, 31, 46, 4085, 4082, 0, 0, 4083, 0, 0, 0, 2, 0, 4077, 4083, 4080, 0, 0, 4071, 0, 0, 0, 0, 0, 5, 4084, 5, 8, 8, 8, 7, 64, 0, 7, 0, 0, 4069, 0, 769, 4073, 11, 10, 0, 14, 0, 3, 0, 4085] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[146, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1785, 0, 0, 4096, 0, 909, 365, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4082, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1985, 0, 0, 4096, 4096, 0, 0, 3187, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 14, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 180, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15497, 16387, 16394, 17460, 16406, 16380, 16376, 16172] avg=16384.0000 max=17460 max_rank=3 min=15497 min_rank=0 imbalance_ratio=1.065674 max_load_reduction=113612 max_load_reduction_pct=0.866791 moved_to_echo=65334 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[305, 4, 0, 5, 4090, 7, 0, 4090, 0, 5, 0, 2478, 11, 5, 4092, 1, 907, 364, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 4088, 6, 0, 0, 4068, 0, 4, 4, 0, 4089, 3, 4077, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 11, 0, 0, 10, 2, 4077, 4085, 5, 4087, 0, 6, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 5, 408, 1, 3, 4091, 4090, 3, 3, 3184, 0, 4, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 4086, 0, 5, 0, 0, 3, 0, 4, 4090, 5, 4092, 0, 0, 4090, 14, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7, 16, 0, 4087, 0, 0, 4090, 0, 0, 0, 1, 0, 4089, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 3, 4090, 3, 2, 2, 4, 4, 26, 4091, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4082, 2, 1034, 4084, 6, 5, 0, 6, 0, 0, 0, 4091, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16368, 16362, 16381, 15763, 16403, 16386, 16396, 17013] avg=16384.0000 max=17013 max_rank=7 min=15763 min_rank=3 imbalance_ratio=1.038391 max_load_reduction=114059 max_load_reduction_pct=0.870201 moved_to_echo=66198 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[440, 2, 0, 1, 4095, 2, 0, 4095, 0, 1, 0, 2937, 0, 1, 4095, 1, 908, 364, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 3, 1, 0, 4094, 0, 1, 1, 0, 4094, 1, 4080, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 12, 0, 0, 1, 1, 4090, 4094, 2, 4093, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 622, 0, 0, 4094, 4093, 1, 1, 3187, 4, 1, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 1, 0, 1, 4094, 1, 4095, 2, 1, 4095, 14, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5, 0, 4090, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 1, 0, 0, 1, 4094, 2, 1, 1, 1, 1, 3, 4095, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 128, 4093, 1, 1, 0, 1, 0, 0, 0, 4095, 3700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16946, 16388, 16391, 16072, 16407, 16380, 16377, 16111] avg=16384.0000 max=16946 max_rank=0 min=16072 min_rank=3 imbalance_ratio=1.034302 max_load_reduction=114126 max_load_reduction_pct=0.870712 moved_to_echo=65275 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[402, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3120, 0, 0, 4096, 0, 909, 365, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4082, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 596, 0, 0, 4096, 4096, 0, 0, 3187, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 4096, 0, 0, 4096, 14, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 63, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17088, 16387, 16394, 16071, 16321, 16380, 16376, 16055] avg=16384.0000 max=17088 max_rank=0 min=16055 min_rank=7 imbalance_ratio=1.042969 max_load_reduction=113984 max_load_reduction_pct=0.869629 moved_to_echo=65132 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[121, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1291, 0, 0, 4096, 0, 909, 365, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4082, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1851, 0, 0, 4096, 4096, 0, 0, 3187, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 4096, 0, 0, 4096, 14, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 873, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14978, 16387, 16394, 17325, 16367, 16380, 16376, 16865] avg=16384.0000 max=17325 max_rank=3 min=14978 min_rank=0 imbalance_ratio=1.057434 max_load_reduction=113747 max_load_reduction_pct=0.867821 moved_to_echo=65988 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[458, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2665, 0, 0, 4096, 0, 909, 365, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4082, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 943, 0, 0, 4096, 4096, 0, 0, 3187, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4073, 0, 4096, 0, 0, 4096, 14, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 52, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16689, 16387, 16394, 16418, 16383, 16380, 16377, 16044] avg=16384.0000 max=16689 max_rank=0 min=16044 min_rank=7 imbalance_ratio=1.018616 max_load_reduction=114383 max_load_reduction_pct=0.872673 moved_to_echo=65184 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[373, 0, 0, 0, 4096, 0, 5, 4096, 0, 0, 0, 2581, 0, 0, 4096, 0, 909, 365, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4082, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1079, 0, 0, 4096, 4096, 0, 0, 3187, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4096, 0, 0, 4096, 14, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 83, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16525, 16387, 16394, 16554, 16384, 16380, 16373, 16075] avg=16384.0000 max=16554 max_rank=3 min=16075 min_rank=7 imbalance_ratio=1.010376 max_load_reduction=114518 max_load_reduction_pct=0.873703 moved_to_echo=65212 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 10, 0, 9, 4085, 13, 0, 4086, 0, 9, 0, 3356, 23, 10, 4086, 6, 909, 367, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 7, 4085, 13, 5, 7, 4051, 6, 10, 12, 0, 4078, 6, 4070, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 13, 0, 1, 0, 0, 6, 10, 4038, 4074, 9, 4079, 0, 11, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 8, 1, 0, 0, 4086, 4081, 7, 4, 3177, 14, 8, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4077, 0, 11, 0, 0, 6, 0, 9, 4084, 9, 4087, 3, 6, 4083, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 31, 46, 0, 4073, 0, 0, 4083, 0, 0, 0, 2, 0, 4077, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 5, 4084, 5, 8, 8, 8, 7, 64, 4085, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4069, 0, 769, 4073, 11, 10, 0, 14, 0, 3, 0, 4085, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16981, 16387, 16330, 15409, 16401, 16406, 16421, 16737] avg=16384.0000 max=16981 max_rank=0 min=15409 min_rank=3 imbalance_ratio=1.036438 max_load_reduction=114091 max_load_reduction_pct=0.870445 moved_to_echo=65965 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 32768, 0, 0, 128, 0, 0, 128, 32768, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 0, 32768, 0, 0, 0, 128, 0, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2304, 128, 0, 128, 32768, 128, 128, 32768, 0, 128, 0, 20224, 128, 128, 32768, 128, 7296, 2944, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 128, 7552, 128, 128, 32768, 32768, 128, 128, 25600, 128, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 32768, 128, 3200, 32768, 128, 128, 0, 128, 0, 128, 0, 32768, 29696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 0, 128, 32768, 128, 128, 128, 32768, 128, 128, 128, 0, 32768, 128, 32640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 32768, 0, 128, 0, 0, 128, 128, 128, 32640, 128, 32768, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 0, 0, 128, 128, 32768, 32768, 128, 32768, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4, 4096, 0, 4096, 0, 0, 0, 244, 4096, 1, 0, 4096, 0, 0, 603, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 659, 0, 0, 0, 4096, 0, 0, 0, 2621, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4060, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4090, 4, 349, 4092, 0, 4089, 4049, 4092, 1, 0, 4087, 0, 0, 2, 4091, 290, 4084, 4, 4089, 0, 5, 0, 2086, 4092, 21, 4, 4091, 5, 5, 157, 0, 0, 6, 0, 5, 0, 4, 4089, 3, 4, 0, 0, 2, 5, 0, 6, 0, 4092, 0, 8, 0, 0, 3, 2, 0, 1, 5, 4091, 2, 0, 7, 0, 5, 0, 4091, 5, 7, 14, 4, 5, 0, 0, 6, 0, 5, 0, 1, 0, 40, 0, 4089, 0, 4091, 6, 1, 4090, 117, 2, 1, 2, 4087, 0, 0, 0, 1120, 0, 0, 5, 4088, 4090, 0, 10, 4, 0, 4, 4069, 4083, 4092, 4091, 4086, 6, 0, 0, 0, 4, 4, 3, 4090, 4090, 0, 3, 4082, 0, 0, 4071, 3, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4083, 9, 72, 4087, 12, 4079, 4043, 4086, 0, 0, 4078, 0, 0, 2, 4084, 809, 4052, 10, 4082, 0, 11, 0, 504, 4087, 71, 9, 4084, 10, 8, 752, 0, 0, 12, 0, 7, 0, 9, 4072, 18, 8, 0, 0, 4, 10, 0, 16, 0, 4087, 0, 16, 1, 0, 7, 10, 0, 0, 12, 4083, 12, 0, 1, 0, 10, 0, 4084, 13, 15, 31, 9, 11, 0, 0, 16, 0, 13, 0, 9, 0, 53, 0, 4081, 0, 4085, 14, 0, 4083, 790, 0, 3, 7, 4081, 0, 0, 0, 1149, 0, 0, 10, 4081, 4084, 0, 30, 12, 0, 11, 4082, 4073, 4083, 4080, 4080, 4, 0, 0, 0, 9, 12, 10, 4066, 4086, 0, 6, 4065, 0, 16, 4016, 5, 4085] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 13, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 75, 4096, 0, 4096, 0, 0, 0, 927, 4096, 0, 0, 4096, 0, 0, 650, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 126, 0, 0, 0, 4096, 0, 0, 0, 2346, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4094, 4096, 4096, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 4096, 0, 0, 4080, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4095, 1, 1, 4095, 0, 4094, 4059, 4095, 0, 0, 4092, 0, 0, 1, 4095, 164, 4094, 1, 4094, 0, 1, 0, 1032, 4095, 1, 2, 4094, 1, 1, 1604, 0, 0, 1, 0, 0, 0, 1, 4093, 0, 1, 0, 0, 1, 1, 0, 2, 0, 4095, 0, 2, 1, 0, 1, 1, 0, 12, 1, 4095, 1, 0, 1, 0, 1, 0, 4094, 1, 1, 0, 1, 1, 0, 0, 2, 0, 2, 0, 0, 0, 2, 0, 4094, 0, 4095, 2, 0, 4094, 15, 1, 0, 1, 4094, 0, 9, 0, 1328, 0, 0, 0, 4094, 4095, 0, 0, 2, 0, 1, 4077, 4094, 4095, 4095, 4092, 1, 0, 0, 3, 1, 1, 1, 4091, 4094, 0, 1, 4094, 0, 4, 4080, 1, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 8, 4096, 0, 4096, 4096, 4096, 0, 1, 4096, 0, 0, 0, 4096, 16, 4096, 0, 4096, 0, 0, 0, 376, 4096, 0, 0, 4096, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 40, 0, 0, 0, 4096, 0, 0, 0, 3629, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 4096, 4096, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 4046, 0, 0, 4096, 0, 0, 4095, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 102, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 5, 0, 0, 4096, 132, 4096, 0, 4096, 0, 0, 0, 907, 4096, 0, 0, 4096, 0, 0, 1024, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 1325, 0, 0, 0, 4096, 0, 0, 0, 681, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4081, 4096, 4092, 4091, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 4052, 0, 0, 4096, 0, 0, 4082, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 1, 4096, 0, 0, 0, 4096, 1, 4096, 0, 4096, 0, 0, 0, 158, 4096, 0, 0, 4096, 0, 0, 2735, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 664, 0, 0, 0, 4096, 0, 0, 0, 586, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4049, 4096, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4050, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 4096, 0, 0, 0, 244, 2171, 1, 0, 4096, 0, 0, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4061, 2145, 1925, 46, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4066, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1951, 0, 0, 4096, 659, 0, 0, 0, 4096, 0, 0, 0, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 4060, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16338, 15311, 16378, 16373, 16360, 17519, 16374, 16419] avg=16384.0000 max=17519 max_rank=5 min=15311 min_rank=1 imbalance_ratio=1.069275 max_load_reduction=113553 max_load_reduction_pct=0.866341 moved_to_echo=66672 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4044, 4, 349, 0, 0, 4089, 4049, 0, 1, 0, 4087, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 4084, 4, 4089, 0, 5, 0, 2086, 2169, 21, 4, 4091, 5, 5, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 5, 0, 4, 4089, 3, 4, 0, 0, 2, 5, 0, 6, 0, 4092, 4092, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 8, 0, 0, 3, 2, 0, 1, 5, 4091, 2, 0, 7, 0, 5, 4056, 2143, 1923, 46, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 5, 7, 14, 4, 5, 0, 0, 6, 0, 5, 0, 1, 0, 40, 4090, 4092, 4058, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 1948, 6, 1, 4090, 117, 2, 1, 2, 4087, 0, 0, 0, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 26, 0, 0, 10, 4, 0, 4, 4069, 4083, 0, 4091, 4086, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 3, 4090, 4090, 0, 3, 4082, 0, 0, 4071, 3, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16625, 17010, 16393, 16388, 16424, 15463, 16384, 16385] avg=16384.0000 max=17010 max_rank=1 min=15463 min_rank=5 imbalance_ratio=1.038208 max_load_reduction=114062 max_load_reduction_pct=0.870224 moved_to_echo=64656 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4050, 0, 13, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 4096, 0, 4096, 0, 0, 0, 927, 2171, 0, 0, 4096, 0, 0, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4061, 2145, 1925, 46, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4066, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1951, 0, 0, 4096, 126, 0, 0, 0, 4096, 0, 0, 0, 2346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 4096, 0, 0, 4080, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16351, 16111, 16378, 16373, 16360, 16711, 16394, 16394] avg=16384.0000 max=16711 max_rank=5 min=16111 min_rank=1 imbalance_ratio=1.019958 max_load_reduction=114361 max_load_reduction_pct=0.872505 moved_to_echo=65859 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4035, 9, 72, 0, 12, 4079, 4043, 0, 0, 0, 4078, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 809, 4052, 10, 4082, 0, 11, 0, 504, 2161, 71, 9, 4084, 10, 8, 752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 7, 0, 9, 4072, 18, 8, 0, 0, 4, 10, 0, 16, 0, 4087, 4086, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 16, 1, 0, 7, 10, 0, 0, 12, 4083, 12, 0, 1, 0, 10, 4056, 2144, 1926, 48, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 13, 15, 31, 9, 11, 0, 0, 16, 0, 13, 0, 9, 0, 53, 4084, 4083, 4057, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 1941, 14, 0, 4083, 790, 0, 3, 7, 4081, 0, 0, 0, 1149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 13, 0, 0, 30, 12, 0, 11, 4082, 4073, 0, 4080, 4080, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 12, 10, 4066, 4086, 0, 6, 4065, 0, 16, 4016, 5, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16330, 16563, 16408, 16424, 16483, 16149, 16395, 16320] avg=16384.0000 max=16563 max_rank=1 min=16149 min_rank=5 imbalance_ratio=1.010925 max_load_reduction=114509 max_load_reduction_pct=0.873634 moved_to_echo=65347 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4049, 1, 1, 0, 0, 4094, 4059, 0, 0, 0, 4092, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 4094, 1, 4094, 0, 1, 0, 1032, 2170, 1, 2, 4094, 1, 1, 1604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4093, 0, 1, 0, 0, 1, 1, 0, 2, 0, 4095, 4095, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 1, 0, 1, 1, 0, 12, 1, 4095, 1, 0, 1, 0, 1, 4060, 2145, 1925, 46, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 1, 1, 0, 0, 2, 0, 2, 0, 0, 0, 2, 4095, 4095, 4064, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1950, 2, 0, 4094, 15, 1, 0, 1, 4094, 0, 9, 0, 1328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 2, 0, 1, 4077, 4094, 0, 4095, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 1, 1, 4091, 4094, 0, 1, 4094, 0, 4, 4080, 1, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16297, 17259, 16379, 16391, 16364, 15588, 16388, 16406] avg=16384.0000 max=17259 max_rank=1 min=15588 min_rank=5 imbalance_ratio=1.053406 max_load_reduction=113813 max_load_reduction_pct=0.868324 moved_to_echo=64746 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4050, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 158, 2171, 0, 0, 4096, 0, 0, 2735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4061, 2145, 1925, 46, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4066, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1951, 0, 0, 4096, 664, 0, 0, 0, 4096, 0, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 4049, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16339, 17353, 16378, 16373, 16360, 15489, 16361, 16419] avg=16384.0000 max=17353 max_rank=1 min=15489 min_rank=5 imbalance_ratio=1.059143 max_load_reduction=113719 max_load_reduction_pct=0.867607 moved_to_echo=64629 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4050, 0, 8, 0, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 0, 4096, 0, 0, 0, 376, 2171, 0, 0, 4096, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4061, 2145, 1925, 46, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4066, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1951, 0, 0, 4096, 40, 0, 0, 0, 4096, 0, 0, 0, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4046, 0, 0, 4096, 0, 0, 4095, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16347, 14943, 16379, 16373, 16359, 17908, 16395, 16368] avg=16384.0000 max=17908 max_rank=5 min=14943 min_rank=1 imbalance_ratio=1.093018 max_load_reduction=113164 max_load_reduction_pct=0.863373 moved_to_echo=67030 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4050, 0, 102, 0, 0, 4096, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 4096, 0, 4096, 0, 0, 0, 907, 2171, 0, 0, 4096, 0, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 4061, 2145, 1925, 46, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 4092, 4066, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1951, 0, 0, 4096, 1325, 0, 0, 0, 4096, 0, 0, 0, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 4081, 4096, 0, 4091, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4052, 0, 0, 4096, 0, 0, 4082, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16445, 16522, 16379, 16377, 16362, 16245, 16381, 16361] avg=16384.0000 max=16522 max_rank=1 min=16245 min_rank=5 imbalance_ratio=1.008423 max_load_reduction=114550 max_load_reduction_pct=0.873947 moved_to_echo=65349 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32384, 128, 640, 0, 128, 32768, 32640, 0, 128, 128, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 128, 32768, 128, 128, 0, 0, 128, 128, 0, 128, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32768, 0, 15616, 128, 128, 32768, 3840, 128, 128, 128, 32768, 0, 128, 0, 13568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1536, 32768, 128, 32768, 0, 128, 0, 6272, 17408, 128, 128, 32768, 128, 128, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 128, 0, 0, 128, 0, 128, 128, 128, 0, 128, 32768, 32768, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 128, 256, 0, 0, 128, 128, 0, 128, 32640, 32768, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 0, 128, 128, 0, 128, 128, 128, 128, 128, 32768, 128, 0, 128, 0, 128, 32512, 17280, 15488, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 128, 128, 32768, 32768, 0, 128, 32768, 0, 128, 32640, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4096, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 68, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 4096, 0, 0, 0, 0, 10, 0, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4088, 5, 4091, 0, 1, 0, 6, 1, 3, 4091, 0, 0, 2, 0, 4091, 4083, 0, 1, 4091, 4, 5, 5, 0, 2, 245, 5, 5, 0, 4090, 3, 4091, 4091, 0, 4, 10, 1, 0, 0, 0, 4084, 0, 0, 5, 6, 0, 4092, 4065, 4, 5, 4, 0, 0, 0, 0, 5, 4090, 0, 4091, 0, 4088, 1, 4, 3, 4090, 0, 0, 2, 6, 0, 0, 4, 0, 3920, 0, 4090, 5, 3, 0, 5, 0, 0, 0, 4084, 6, 4014, 4086, 4089, 4092, 0, 0, 0, 4091, 0, 4, 0, 0, 1, 5, 4092, 0, 0, 5, 4091, 4, 0, 4, 0, 5, 6, 6, 0, 6, 6, 4091, 4089, 0, 0, 4091, 4091, 4090, 3, 32, 0, 4090, 4, 2, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4071, 11, 4081, 0, 0, 0, 4, 0, 18, 4086, 0, 0, 12, 0, 4086, 4063, 0, 0, 4087, 3, 10, 10, 0, 9, 315, 11, 12, 0, 4083, 7, 4083, 4086, 0, 6, 32, 0, 1, 0, 0, 4064, 5, 0, 10, 12, 0, 4088, 4040, 11, 9, 9, 4, 0, 0, 0, 8, 4082, 0, 4085, 0, 4080, 0, 0, 5, 4082, 0, 0, 5, 13, 0, 0, 9, 0, 3909, 0, 4084, 10, 0, 0, 13, 0, 0, 1, 4070, 9, 3935, 4077, 4083, 4084, 0, 0, 0, 4085, 0, 13, 0, 0, 4, 11, 4088, 7, 0, 11, 4085, 6, 0, 8, 0, 12, 13, 8, 9, 0, 12, 4084, 4084, 0, 0, 4084, 4084, 4082, 8, 57, 0, 4085, 14, 19, 4, 12] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 9, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 0, 0, 0, 0, 7, 0, 2, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4093, 1, 4094, 0, 0, 0, 4, 0, 0, 4095, 0, 0, 0, 0, 4095, 4091, 0, 0, 4095, 2, 1, 1, 0, 0, 26, 1, 1, 0, 4094, 1, 4095, 4095, 0, 1, 1, 0, 0, 0, 0, 4094, 1, 0, 2, 1, 0, 4095, 4086, 1, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 1, 4094, 0, 0, 1, 4094, 0, 0, 1, 2, 0, 0, 1, 0, 4088, 0, 4095, 1, 0, 0, 1, 0, 0, 0, 4092, 1, 4091, 4094, 4095, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 1, 4082, 1, 0, 2, 4094, 2, 0, 1, 0, 1, 1, 1, 2, 4, 2, 4094, 4095, 0, 0, 4095, 4094, 4095, 1, 4, 0, 4094, 1, 2, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 10, 0, 0, 0, 4096, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 24, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 0, 4096, 0, 0, 0, 0, 15, 0, 109, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 4096, 0, 0, 0, 0, 1, 0, 26, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 3999, 89, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4026, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 16295, 16389, 16391, 16436, 16416, 16382, 16371] avg=16384.0000 max=16436 max_rank=4 min=16295 min_rank=1 imbalance_ratio=1.003174 max_load_reduction=114636 max_load_reduction_pct=0.874603 moved_to_echo=65605 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4088, 5, 4091, 0, 1, 0, 6, 1, 3, 8, 0, 0, 2, 0, 4091, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4002, 4, 5, 5, 0, 2, 245, 5, 5, 0, 4090, 3, 4091, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 10, 1, 0, 0, 0, 4084, 0, 0, 5, 6, 0, 4092, 4065, 4, 3994, 89, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 0, 0, 0, 5, 4090, 0, 4091, 0, 4088, 1, 4, 3, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 4, 0, 3920, 0, 4090, 5, 3, 0, 5, 0, 4092, 4091, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 6, 4014, 4086, 4089, 0, 0, 0, 0, 21, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 4092, 0, 0, 5, 4091, 4, 0, 4, 0, 5, 6, 6, 0, 6, 4083, 4020, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4089, 0, 0, 0, 4091, 4090, 3, 32, 0, 4077, 4, 2, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16549, 16367, 16381, 16289, 16304, 16404, 16399] avg=16384.0000 max=16549 max_rank=1 min=16289 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=114523 max_load_reduction_pct=0.873741 moved_to_echo=65396 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 2, 0, 9, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 3999, 89, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 0, 0, 0, 0, 7, 4096, 4096, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4025, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 16306, 16389, 16384, 16453, 16407, 16370, 16371] avg=16384.0000 max=16453 max_rank=4 min=16306 min_rank=1 imbalance_ratio=1.004211 max_load_reduction=114619 max_load_reduction_pct=0.874474 moved_to_echo=65601 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 68, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 3999, 89, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 4096, 0, 0, 0, 0, 10, 4096, 4096, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4025, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 16363, 16389, 16384, 16411, 16405, 16357, 16371] avg=16384.0000 max=16411 max_rank=4 min=16357 min_rank=6 imbalance_ratio=1.001648 max_load_reduction=114661 max_load_reduction_pct=0.874794 moved_to_echo=65544 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4071, 11, 4081, 0, 0, 0, 4, 0, 18, 1, 0, 0, 12, 0, 4086, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 3, 10, 10, 0, 9, 315, 11, 12, 0, 4083, 7, 4083, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 32, 0, 1, 0, 0, 4064, 5, 0, 10, 12, 0, 4088, 4040, 11, 3989, 94, 20, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 4, 0, 0, 0, 8, 4082, 0, 4080, 0, 4080, 0, 0, 5, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 13, 0, 0, 9, 0, 3909, 0, 4084, 10, 0, 0, 13, 0, 4084, 4084, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4070, 9, 3935, 4077, 4083, 0, 0, 0, 0, 13, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 11, 4088, 7, 0, 11, 4085, 6, 0, 8, 0, 12, 13, 8, 9, 0, 4085, 4013, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4084, 0, 0, 0, 4084, 4082, 8, 57, 0, 4065, 14, 19, 4, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16347, 16622, 16377, 16359, 16282, 16201, 16443, 16441] avg=16384.0000 max=16622 max_rank=1 min=16201 min_rank=5 imbalance_ratio=1.014526 max_load_reduction=114450 max_load_reduction_pct=0.873184 moved_to_echo=65367 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 3999, 89, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 4096, 0, 0, 0, 0, 1, 4096, 4096, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4026, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 16306, 16389, 16384, 16426, 16431, 16373, 16371] avg=16384.0000 max=16431 max_rank=5 min=16306 min_rank=1 imbalance_ratio=1.002869 max_load_reduction=114641 max_load_reduction_pct=0.874641 moved_to_echo=65601 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4093, 1, 4094, 0, 0, 0, 4, 0, 0, 8, 0, 0, 0, 0, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 2, 1, 1, 0, 0, 26, 1, 1, 0, 4094, 1, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4094, 1, 0, 2, 1, 0, 4095, 4086, 1, 3998, 89, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 1, 4094, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 1, 0, 4088, 0, 4095, 1, 0, 0, 1, 0, 4095, 4095, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 4091, 4094, 4095, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4082, 1, 0, 2, 4094, 2, 0, 1, 0, 1, 1, 1, 2, 4, 4087, 4023, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 4094, 4095, 1, 4, 0, 4081, 1, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16323, 16382, 16381, 16450, 16394, 16379, 16377] avg=16384.0000 max=16450 max_rank=4 min=16323 min_rank=1 imbalance_ratio=1.004028 max_load_reduction=114622 max_load_reduction_pct=0.874496 moved_to_echo=65600 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 3, 0, 10, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 4096, 0, 3999, 89, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 0, 4096, 0, 0, 0, 0, 15, 4096, 4096, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4026, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 16308, 16390, 16408, 16325, 16514, 16364, 16371] avg=16384.0000 max=16514 max_rank=5 min=16308 min_rank=1 imbalance_ratio=1.007935 max_load_reduction=114558 max_load_reduction_pct=0.874008 moved_to_echo=65574 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 32768, 0, 128, 0, 128, 128, 128, 128, 0, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 32128, 128, 128, 128, 128, 128, 768, 128, 128, 0, 32768, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 32768, 128, 0, 128, 32768, 128, 0, 128, 0, 128, 128, 128, 128, 128, 32768, 32256, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 32768, 0, 0, 0, 32768, 32768, 128, 128, 0, 32640, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 0, 0, 128, 0, 32256, 0, 32768, 128, 128, 0, 128, 128, 32768, 32768, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 32768, 128, 0, 128, 128, 0, 32768, 32768, 128, 32000, 768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 256, 32768, 128, 32640, 32768, 32768, 0, 0, 0, 0, 256, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 32768, 0, 32768, 128, 32768, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3, 315, 1250, 1, 353, 0, 831, 2, 2164, 49, 232, 2232, 836, 471, 0, 890, 831, 993, 417, 538, 3916, 2046, 173, 45, 2, 1747, 178, 210, 1413, 7, 997, 1377, 4078, 0, 10, 1294, 2569, 15, 487, 2646, 2385, 1419, 151, 328, 4, 604, 352, 915, 1037, 1681, 1069, 11, 0, 3, 3529, 270, 2, 3218, 3621, 2427, 316, 304, 852, 0, 867, 4076, 0, 1180, 131, 8, 1702, 1671, 991, 657, 46, 0, 66, 841, 518, 502, 2107, 1373, 3, 3783, 35, 2414, 2398, 4050, 73, 17, 940, 980, 3292, 448, 2055, 0, 1009, 1307, 3584, 132, 0, 1888, 437, 2201, 3681, 1610, 0, 1055, 890, 1801, 79, 579, 29, 816, 2430, 502, 5, 582, 207, 1172, 977, 1132, 1442, 1993, 57, 0, 2135, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[5, 306, 1092, 0, 393, 0, 696, 3, 2238, 77, 289, 2000, 1219, 463, 0, 1183, 760, 645, 364, 942, 3944, 1972, 180, 32, 0, 1590, 315, 245, 802, 21, 1084, 1425, 4085, 0, 12, 1057, 3064, 11, 1583, 2298, 2932, 1553, 131, 152, 4, 1047, 460, 717, 696, 1356, 1019, 84, 2, 9, 3311, 229, 0, 3133, 3012, 2478, 284, 516, 406, 0, 744, 4057, 0, 778, 181, 26, 2456, 1981, 1584, 652, 141, 0, 84, 498, 360, 261, 1965, 1308, 10, 3653, 38, 2037, 2361, 4059, 77, 31, 1035, 1369, 3175, 475, 2720, 0, 1001, 1664, 3718, 115, 0, 2310, 586, 1759, 3510, 1597, 2, 933, 1128, 1599, 87, 964, 122, 216, 1849, 452, 14, 559, 210, 1329, 1334, 1049, 1519, 1430, 75, 2, 1897, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[0, 181, 1079, 3, 325, 2, 867, 2, 2137, 85, 177, 2046, 904, 381, 0, 1595, 804, 397, 1118, 810, 3780, 2057, 164, 37, 0, 1265, 199, 205, 952, 3, 973, 1574, 4093, 0, 10, 1143, 3484, 23, 689, 2425, 2417, 1495, 254, 130, 1, 644, 694, 1019, 1180, 1580, 1057, 35, 4, 5, 3318, 246, 2, 2897, 3536, 2370, 607, 243, 590, 0, 1025, 4061, 0, 535, 75, 38, 2705, 2091, 1609, 769, 222, 2, 202, 453, 275, 478, 1929, 786, 5, 3649, 23, 2046, 2041, 4018, 31, 9, 684, 1124, 2996, 319, 1920, 0, 707, 1241, 3794, 213, 0, 2202, 583, 2232, 3925, 1486, 0, 1123, 1528, 1311, 37, 960, 87, 680, 2359, 626, 10, 444, 178, 1298, 982, 1052, 1653, 1598, 61, 1, 2261, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[7, 212, 1176, 0, 391, 0, 651, 4, 1722, 88, 297, 2599, 1037, 367, 0, 1018, 1008, 675, 935, 970, 3910, 2413, 168, 51, 1, 1612, 245, 179, 876, 3, 580, 1215, 4065, 0, 12, 1287, 2321, 17, 854, 2304, 2799, 1170, 159, 104, 0, 575, 543, 1250, 946, 1556, 994, 9, 1, 0, 3675, 569, 0, 2525, 3654, 2508, 445, 409, 795, 0, 983, 4062, 0, 875, 201, 65, 2413, 1654, 1279, 747, 158, 1, 123, 479, 521, 507, 1709, 1146, 5, 3833, 43, 2273, 2204, 4016, 74, 4, 991, 1895, 3137, 341, 2302, 0, 701, 1223, 3847, 165, 1, 2048, 473, 2056, 3616, 1498, 1, 1052, 1039, 1352, 150, 883, 65, 688, 2311, 520, 40, 604, 150, 1040, 1043, 1400, 1709, 1478, 54, 0, 1868, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[6, 132, 1377, 1, 351, 0, 909, 3, 2546, 92, 263, 2653, 1012, 309, 0, 1322, 796, 490, 649, 950, 3881, 2126, 196, 46, 3, 1737, 206, 178, 1140, 12, 1104, 1588, 4050, 1, 11, 1326, 2750, 36, 670, 1808, 2469, 1250, 454, 326, 0, 436, 542, 651, 1575, 1574, 897, 102, 1, 6, 3006, 194, 0, 3131, 3419, 1857, 291, 248, 841, 1, 776, 4006, 0, 893, 176, 58, 1546, 1869, 1150, 653, 136, 3, 64, 588, 415, 347, 1698, 1450, 12, 3710, 52, 2855, 2824, 3833, 54, 5, 553, 1277, 2960, 431, 2405, 2, 1691, 1280, 3164, 208, 3, 2205, 899, 2165, 3907, 1432, 6, 806, 1158, 1216, 96, 363, 14, 808, 2579, 939, 25, 785, 269, 1488, 1009, 896, 1176, 1763, 58, 1, 1857, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2, 261, 1083, 3, 364, 0, 580, 11, 2110, 81, 128, 1881, 1030, 374, 0, 1331, 948, 902, 935, 542, 3837, 2334, 164, 35, 0, 820, 216, 163, 1293, 12, 978, 1432, 4082, 0, 7, 1233, 3492, 22, 779, 2392, 2724, 1162, 179, 173, 0, 552, 467, 980, 823, 1077, 751, 62, 3, 2, 3652, 320, 1, 3103, 3597, 2312, 427, 245, 970, 18, 1306, 4064, 0, 1327, 199, 32, 2608, 1457, 1413, 989, 213, 4, 404, 394, 399, 342, 1384, 1269, 13, 3864, 46, 2265, 2302, 4042, 55, 10, 459, 1257, 2980, 357, 1951, 1, 552, 1407, 3752, 242, 0, 2328, 714, 2281, 3316, 1303, 13, 1469, 1453, 2249, 99, 679, 73, 863, 2185, 774, 9, 514, 179, 1121, 848, 754, 1548, 1339, 110, 15, 2019, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2, 140, 1031, 0, 291, 0, 971, 0, 2422, 148, 311, 2266, 1218, 337, 0, 1104, 656, 889, 539, 605, 3934, 2058, 128, 26, 0, 1950, 214, 49, 1305, 2, 1164, 1353, 4085, 1, 6, 1454, 2995, 19, 357, 2535, 2837, 804, 143, 230, 6, 718, 441, 938, 664, 1407, 924, 16, 1, 0, 3676, 259, 0, 3229, 3635, 2615, 340, 474, 536, 0, 783, 4046, 0, 835, 147, 29, 1631, 2199, 1386, 780, 93, 9, 52, 444, 503, 268, 1737, 1468, 4, 3904, 43, 2371, 2236, 4041, 54, 8, 822, 1044, 3389, 219, 2692, 7, 1051, 1466, 3794, 134, 0, 2235, 544, 2195, 3675, 1638, 1, 701, 1404, 1613, 80, 591, 46, 797, 2254, 398, 3, 613, 201, 1027, 765, 995, 1082, 1830, 91, 1, 2145, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[5, 345, 1301, 2, 361, 1, 723, 33, 2394, 108, 227, 2631, 1119, 622, 3, 1343, 1221, 639, 738, 909, 3946, 2231, 135, 26, 5, 1475, 306, 199, 1389, 3, 1020, 1442, 4089, 1, 9, 1035, 3426, 21, 768, 2327, 2497, 1313, 223, 283, 0, 651, 417, 701, 913, 1099, 1176, 20, 32, 5, 3698, 379, 0, 3234, 3080, 1647, 363, 397, 467, 0, 1001, 4045, 0, 1564, 178, 15, 2099, 1871, 1742, 751, 99, 4, 120, 454, 417, 304, 1249, 1508, 7, 3795, 66, 2501, 2153, 3968, 50, 17, 588, 813, 3276, 369, 2354, 1, 939, 1613, 3651, 83, 0, 2107, 798, 1994, 3725, 1912, 0, 1113, 837, 2243, 111, 572, 145, 530, 1653, 734, 6, 501, 143, 1145, 1079, 825, 934, 1323, 72, 1, 1730, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 306, 1092, 0, 393, 0, 696, 3, 2238, 77, 289, 2000, 1219, 463, 0, 1183, 4059, 2203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 760, 645, 364, 942, 3944, 1972, 180, 32, 0, 1590, 315, 245, 802, 21, 1084, 1425, 954, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2630, 0, 12, 1057, 3064, 11, 1583, 2298, 2932, 1553, 131, 152, 4, 1047, 460, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 1356, 1019, 84, 2, 9, 2319, 229, 0, 3133, 3012, 2478, 284, 516, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 744, 4057, 0, 778, 181, 26, 2456, 1981, 1584, 652, 141, 0, 84, 498, 360, 261, 1455, 558, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1965, 1308, 10, 767, 38, 2037, 2361, 0, 77, 31, 1035, 1369, 3175, 475, 2720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 1664, 2764, 115, 0, 2310, 586, 1759, 0, 1597, 2, 933, 1128, 1599, 87, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 216, 1849, 452, 14, 559, 210, 1329, 1334, 1049, 1519, 1430, 75, 2, 1897, 0, 2328, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16226, 15856, 17651, 15543, 16227, 17368, 16509, 15692] avg=16384.0000 max=17651 max_rank=2 min=15543 min_rank=3 imbalance_ratio=1.077332 max_load_reduction=113421 max_load_reduction_pct=0.865334 moved_to_echo=65796 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 315, 1250, 1, 353, 0, 831, 2, 2164, 49, 232, 2232, 836, 471, 0, 890, 4050, 2310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 831, 993, 417, 538, 3916, 2046, 173, 45, 2, 1747, 178, 210, 1413, 7, 997, 1377, 919, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2625, 0, 10, 1294, 2569, 15, 487, 2646, 2385, 1419, 151, 328, 4, 604, 352, 915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1681, 1069, 11, 0, 3, 2472, 270, 2, 3218, 3621, 2427, 316, 304, 852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 867, 4076, 0, 1180, 131, 8, 1702, 1671, 991, 657, 46, 0, 66, 841, 518, 502, 1453, 578, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2107, 1373, 3, 794, 35, 2414, 2398, 0, 73, 17, 940, 980, 3292, 448, 2055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1307, 2665, 132, 0, 1888, 437, 2201, 0, 1610, 0, 1055, 890, 1801, 79, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 816, 2430, 502, 5, 582, 207, 1172, 977, 1132, 1442, 1993, 57, 0, 2135, 0, 2411, 1371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15989, 16428, 15804, 17283, 15725, 16929, 15653, 17261] avg=16384.0000 max=17283 max_rank=3 min=15653 min_rank=6 imbalance_ratio=1.054871 max_load_reduction=113789 max_load_reduction_pct=0.868141 moved_to_echo=65568 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 181, 1079, 3, 325, 2, 867, 2, 2137, 85, 177, 2046, 904, 381, 0, 1595, 4018, 2463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 397, 1118, 810, 3780, 2057, 164, 37, 0, 1265, 199, 205, 952, 3, 973, 1574, 973, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2635, 0, 10, 1143, 3484, 23, 689, 2425, 2417, 1495, 254, 130, 1, 644, 694, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1180, 1580, 1057, 35, 4, 5, 2325, 246, 2, 2897, 3536, 2370, 607, 243, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 4061, 0, 535, 75, 38, 2705, 2091, 1609, 769, 222, 2, 202, 453, 275, 478, 1458, 558, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1929, 786, 5, 765, 23, 2046, 2041, 0, 31, 9, 684, 1124, 2996, 319, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 707, 1241, 2821, 213, 0, 2202, 583, 2232, 0, 1486, 0, 1123, 1528, 1311, 37, 960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 680, 2359, 626, 10, 444, 178, 1298, 982, 1052, 1653, 1598, 61, 1, 2261, 7, 2326, 1462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16265, 15893, 17063, 16677, 16967, 14678, 16444, 17085] avg=16384.0000 max=17085 max_rank=7 min=14678 min_rank=5 imbalance_ratio=1.042786 max_load_reduction=113987 max_load_reduction_pct=0.869652 moved_to_echo=65174 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[6, 132, 1377, 1, 351, 0, 909, 3, 2546, 92, 263, 2653, 1012, 309, 0, 1322, 3833, 2453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 796, 490, 649, 950, 3881, 2126, 196, 46, 3, 1737, 206, 178, 1140, 12, 1104, 1588, 812, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2607, 1, 11, 1326, 2750, 36, 670, 1808, 2469, 1250, 454, 326, 0, 436, 542, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1575, 1574, 897, 102, 1, 6, 2106, 194, 0, 3131, 3419, 1857, 291, 248, 841, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 4006, 0, 893, 176, 58, 1546, 1869, 1150, 653, 136, 3, 64, 588, 415, 347, 1443, 567, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1698, 1450, 12, 778, 52, 2855, 2824, 0, 54, 5, 553, 1277, 2960, 431, 2405, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1691, 1280, 2352, 208, 3, 2205, 899, 2165, 0, 1432, 6, 806, 1158, 1216, 96, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 808, 2579, 939, 25, 785, 269, 1488, 1009, 896, 1176, 1763, 58, 1, 1857, 4, 2365, 1454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17262, 16441, 15337, 16243, 15063, 17356, 15880, 17490] avg=16384.0000 max=17490 max_rank=7 min=15063 min_rank=4 imbalance_ratio=1.067505 max_load_reduction=113582 max_load_reduction_pct=0.866562 moved_to_echo=65789 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[7, 212, 1176, 0, 391, 0, 651, 4, 1722, 88, 297, 2599, 1037, 367, 0, 1018, 4016, 2269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 675, 935, 970, 3910, 2413, 168, 51, 1, 1612, 245, 179, 876, 3, 580, 1215, 987, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2617, 0, 12, 1287, 2321, 17, 854, 2304, 2799, 1170, 159, 104, 0, 575, 543, 1250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 946, 1556, 994, 9, 1, 0, 2574, 569, 0, 2525, 3654, 2508, 445, 409, 795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 983, 4062, 0, 875, 201, 65, 2413, 1654, 1279, 747, 158, 1, 123, 479, 521, 507, 1448, 586, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1709, 1146, 5, 804, 43, 2273, 2204, 0, 74, 4, 991, 1895, 3137, 341, 2302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 701, 1223, 2860, 165, 1, 2048, 473, 2056, 0, 1498, 1, 1052, 1039, 1352, 150, 883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 688, 2311, 520, 40, 604, 150, 1040, 1043, 1400, 1709, 1478, 54, 0, 1868, 0, 2443, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15854, 16473, 16012, 16985, 16558, 16928, 15502, 16760] avg=16384.0000 max=16985 max_rank=3 min=15502 min_rank=6 imbalance_ratio=1.036682 max_load_reduction=114087 max_load_reduction_pct=0.870415 moved_to_echo=65748 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 261, 1083, 3, 364, 0, 580, 11, 2110, 81, 128, 1881, 1030, 374, 0, 1331, 4042, 2081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 948, 902, 935, 542, 3837, 2334, 164, 35, 0, 820, 216, 163, 1293, 12, 978, 1432, 963, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2628, 0, 7, 1233, 3492, 22, 779, 2392, 2724, 1162, 179, 173, 0, 552, 467, 980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 823, 1077, 751, 62, 3, 2, 2558, 320, 1, 3103, 3597, 2312, 427, 245, 970, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1306, 4064, 0, 1327, 199, 32, 2608, 1457, 1413, 989, 213, 4, 404, 394, 399, 342, 1454, 591, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1384, 1269, 13, 810, 46, 2265, 2302, 0, 55, 10, 459, 1257, 2980, 357, 1951, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 1407, 2789, 242, 0, 2328, 714, 2281, 0, 1303, 13, 1469, 1453, 2249, 99, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 863, 2185, 774, 9, 514, 179, 1121, 848, 754, 1548, 1339, 110, 15, 2019, 1, 2463, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15362, 16215, 16790, 16269, 17649, 15159, 17578, 16050] avg=16384.0000 max=17649 max_rank=4 min=15159 min_rank=5 imbalance_ratio=1.077209 max_load_reduction=113423 max_load_reduction_pct=0.865349 moved_to_echo=66436 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 140, 1031, 0, 291, 0, 971, 0, 2422, 148, 311, 2266, 1218, 337, 0, 1104, 4041, 2307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 656, 889, 539, 605, 3934, 2058, 128, 26, 0, 1950, 214, 49, 1305, 2, 1164, 1353, 977, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2627, 1, 6, 1454, 2995, 19, 357, 2535, 2837, 804, 143, 230, 6, 718, 441, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 664, 1407, 924, 16, 1, 0, 2570, 259, 0, 3229, 3635, 2615, 340, 474, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 783, 4046, 0, 835, 147, 29, 1631, 2199, 1386, 780, 93, 9, 52, 444, 503, 268, 1458, 601, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1737, 1468, 4, 812, 43, 2371, 2236, 0, 54, 8, 822, 1044, 3389, 219, 2692, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 1466, 2817, 134, 0, 2235, 544, 2195, 0, 1638, 1, 701, 1404, 1613, 80, 591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 797, 2254, 398, 3, 613, 201, 1027, 765, 995, 1082, 1830, 91, 1, 2145, 0, 2491, 1368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16589, 16496, 16111, 16670, 15723, 16906, 16470, 16107] avg=16384.0000 max=16906 max_rank=5 min=15723 min_rank=4 imbalance_ratio=1.031860 max_load_reduction=114166 max_load_reduction_pct=0.871017 moved_to_echo=65206 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 345, 1301, 2, 361, 1, 723, 33, 2394, 108, 227, 2631, 1119, 622, 3, 1343, 3968, 2339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1221, 639, 738, 909, 3946, 2231, 135, 26, 5, 1475, 306, 199, 1389, 3, 1020, 1442, 937, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2632, 1, 9, 1035, 3426, 21, 768, 2327, 2497, 1313, 223, 283, 0, 651, 417, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 913, 1099, 1176, 20, 32, 5, 2590, 379, 0, 3234, 3080, 1647, 363, 397, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 4045, 0, 1564, 178, 15, 2099, 1871, 1742, 751, 99, 4, 120, 454, 417, 304, 1457, 580, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1249, 1508, 7, 796, 66, 2501, 2153, 0, 50, 17, 588, 813, 3276, 369, 2354, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 939, 1613, 2714, 83, 0, 2107, 798, 1994, 0, 1912, 0, 1113, 837, 2243, 111, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 530, 1653, 734, 6, 501, 143, 1145, 1079, 825, 934, 1323, 72, 1, 1730, 1, 2419, 1386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17525, 17270, 16304, 15402, 17160, 15748, 17036, 14627] avg=16384.0000 max=17525 max_rank=0 min=14627 min_rank=7 imbalance_ratio=1.069641 max_load_reduction=113547 max_load_reduction_pct=0.866295 moved_to_echo=64571 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[7680, 11264, 21888, 1408, 128, 17408, 5120, 16896, 0, 12544, 128, 8320, 9472, 13440, 768, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[13824, 10368, 128, 6400, 384, 18816, 18560, 0, 512, 128, 6144, 9856, 25216, 3072, 18432, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[640, 5504, 17664, 4992, 128, 4608, 1664, 9728, 8064, 8192, 11136, 12800, 640, 128, 16000, 128, 19328, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 1920, 9472, 128, 2944, 128, 6272, 128, 17792, 768, 2048, 18432, 8448, 3328, 128, 9856, 32128, 18432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[7552, 32512, 0, 8064, 1408, 384, 17280, 14848, 11264, 6016, 1152, 128, 1152, 4224, 3456, 3072, 11648, 4736, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[7936, 11392, 7936, 384, 128, 128, 19584, 2560, 128, 24576, 27648, 18304, 3200, 2944, 5504, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[21120, 128, 128, 9856, 24192, 256, 6272, 18816, 21120, 10240, 1792, 1792, 128, 5248, 3968, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[7040, 5632, 5760, 6272, 31232, 17280, 1408, 384, 128, 12288, 1920, 1536, 9216, 128, 7936, 11520, 7552, 4992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4095, 4096, 1765, 0, 8, 35, 3746, 4090, 3994, 4095, 0, 0, 214, 0, 0, 4093, 0, 0, 0, 0, 7, 5, 4096, 0, 0, 4095, 0, 0, 8, 1, 4090, 1, 4096, 0, 0, 4089, 2, 0, 0, 18, 4091, 4088, 0, 5, 0, 3919, 0, 0, 3943, 0, 0, 3, 4090, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 6, 3, 0, 0, 0, 0, 0, 6, 1708, 0, 0, 0, 0, 0, 2, 0, 0, 4009, 4073, 3, 1256, 4092, 4091, 0, 6, 1, 0, 241, 1, 0, 44, 97, 1, 4096, 248, 0, 4043, 0, 0, 0, 0, 3, 0, 4080, 4084, 0, 4091, 3, 42, 8, 0, 1, 0, 4036, 4090, 8, 2291, 165, 32, 0, 3, 1034, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 271, 0, 0, 17, 3945, 4092, 3976, 4095, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 4095, 0, 0, 5, 0, 4096, 0, 4096, 0, 0, 4093, 0, 0, 0, 3, 4096, 4080, 0, 0, 0, 4087, 0, 1, 4094, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2148, 0, 0, 0, 0, 0, 0, 0, 0, 3786, 3999, 0, 221, 4096, 4096, 0, 0, 0, 0, 251, 0, 0, 2247, 6, 0, 4095, 20, 0, 4095, 0, 0, 0, 0, 1, 0, 3991, 4058, 0, 4096, 0, 151, 0, 0, 0, 0, 4067, 4096, 0, 2506, 106, 5, 1, 0, 965, 0, 142] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 1267, 0, 3, 28, 3849, 4067, 4062, 4096, 0, 0, 78, 0, 0, 4069, 0, 0, 2, 0, 0, 1, 4094, 0, 0, 4096, 0, 0, 0, 0, 4081, 0, 4096, 0, 8, 4065, 21, 0, 2, 0, 4091, 4065, 0, 5, 0, 4083, 0, 5, 3955, 0, 0, 0, 4095, 24, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 908, 4, 37, 0, 0, 0, 0, 0, 0, 3963, 4032, 0, 2244, 4096, 4086, 0, 0, 0, 0, 72, 0, 0, 1230, 4, 0, 4083, 0, 0, 3958, 0, 0, 0, 0, 0, 0, 4071, 4088, 0, 4096, 0, 239, 11, 0, 0, 0, 3927, 4096, 0, 1719, 600, 21, 0, 0, 786, 0, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4095, 4096, 2130, 0, 7, 4, 3810, 4096, 3601, 4096, 0, 3, 147, 0, 1, 4088, 3, 0, 0, 37, 0, 0, 4095, 1, 2, 4092, 0, 0, 1, 0, 4093, 0, 4093, 5, 13, 4096, 0, 2, 8, 0, 4095, 4096, 0, 4, 1, 4060, 0, 1, 3778, 0, 0, 1, 4087, 20, 4094, 0, 0, 0, 4089, 0, 0, 1, 10, 0, 1, 1, 1, 27, 0, 0, 546, 41, 0, 3, 0, 0, 4, 0, 0, 3898, 4092, 0, 2160, 4089, 4090, 0, 6, 1, 1, 351, 0, 0, 170, 16, 0, 4087, 611, 3, 3911, 0, 0, 0, 0, 0, 0, 4077, 3938, 0, 4096, 0, 13, 17, 1, 0, 0, 4086, 4091, 0, 2679, 15, 0, 0, 0, 897, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 4084, 4092, 617, 4, 0, 12, 4019, 4093, 3280, 4094, 4, 5, 5, 1, 0, 4093, 9, 1, 1, 28, 0, 0, 4096, 0, 0, 4073, 0, 8, 0, 0, 4084, 2, 4095, 0, 1, 4088, 0, 0, 0, 2, 4089, 4093, 1, 0, 4, 3847, 0, 19, 4089, 2, 0, 5, 3986, 1, 4093, 0, 0, 0, 4091, 0, 11, 0, 0, 2, 0, 0, 0, 0, 0, 6, 1581, 132, 0, 2, 2, 2, 28, 0, 1, 4092, 4093, 4, 2006, 4094, 4090, 7, 0, 0, 0, 902, 2, 0, 267, 0, 3, 4095, 108, 0, 4011, 1, 0, 2, 0, 0, 1, 4073, 3941, 0, 4088, 8, 319, 1, 1, 1, 0, 4054, 4090, 1, 2791, 2, 0, 1, 0, 938, 4, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4095, 4096, 534, 0, 3, 196, 4085, 4096, 3945, 4096, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 4070, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 4088, 0, 0, 4066, 0, 0, 0, 4094, 0, 4096, 0, 0, 8, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2489, 3, 0, 0, 0, 0, 0, 0, 0, 3986, 4075, 0, 1085, 4096, 4096, 0, 0, 0, 0, 114, 0, 0, 463, 0, 0, 4096, 153, 0, 4078, 0, 0, 0, 0, 5, 0, 4093, 3520, 0, 4096, 0, 69, 0, 0, 0, 0, 4061, 4096, 0, 2399, 0, 3, 0, 0, 1657, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 1751, 0, 0, 28, 3807, 4094, 3674, 4095, 0, 0, 2, 0, 0, 4096, 0, 0, 4, 4, 0, 3, 4096, 0, 0, 4090, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 4082, 0, 0, 0, 0, 4079, 4096, 0, 0, 0, 4072, 0, 8, 4013, 0, 0, 0, 4092, 0, 4096, 0, 2, 0, 4096, 0, 0, 0, 1, 4, 0, 0, 0, 4, 0, 0, 1353, 14, 0, 0, 0, 0, 0, 0, 0, 3956, 4092, 0, 2204, 4096, 4096, 0, 0, 0, 0, 324, 0, 0, 858, 40, 0, 4087, 11, 0, 4077, 0, 0, 0, 0, 25, 0, 3857, 4078, 0, 4085, 0, 9, 4, 0, 0, 0, 4096, 4096, 0, 1095, 19, 0, 2, 0, 1718, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4087, 4095, 699, 0, 0, 23, 3943, 4094, 3945, 4096, 1, 0, 30, 2, 0, 4095, 6, 0, 0, 12, 1, 0, 4095, 0, 3, 4092, 7, 0, 6, 0, 4090, 0, 4093, 3, 0, 4076, 0, 3, 0, 14, 4091, 4076, 0, 0, 0, 4096, 16, 13, 4078, 0, 1, 0, 3931, 0, 4092, 0, 0, 0, 4087, 2, 10, 0, 1, 0, 0, 4, 1, 0, 0, 9, 3572, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 4093, 0, 607, 4096, 4092, 0, 0, 0, 0, 142, 1, 0, 2584, 128, 0, 3893, 92, 1, 4067, 4, 0, 0, 0, 0, 0, 4073, 3983, 0, 4091, 0, 9, 0, 0, 0, 0, 4066, 4094, 0, 548, 103, 12, 1, 0, 523, 0, 166] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3348, 0, 1765, 0, 8, 35, 3746, 4090, 3994, 0, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 7, 5, 4096, 0, 0, 4095, 0, 0, 8, 1, 2615, 835, 631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 4089, 2, 0, 0, 18, 79, 4088, 0, 5, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 0, 0, 3, 4090, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 0, 0, 0, 6, 1708, 0, 0, 0, 0, 0, 2, 0, 4095, 4096, 4096, 2264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 4073, 3, 1256, 1477, 4091, 0, 6, 1, 0, 241, 1, 0, 44, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 248, 0, 4043, 0, 0, 0, 0, 3, 0, 4080, 4084, 0, 125, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 8, 0, 1, 0, 4036, 4090, 8, 2291, 165, 32, 0, 3, 1034, 0, 5, 3131, 1748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17200, 16386, 16291, 16343, 16276, 15299, 16683, 16594] avg=16384.0000 max=17200 max_rank=0 min=15299 min_rank=5 imbalance_ratio=1.049805 max_load_reduction=113872 max_load_reduction_pct=0.868774 moved_to_echo=64852 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3349, 0, 271, 0, 0, 17, 3945, 4092, 3976, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 4095, 0, 0, 5, 0, 2618, 836, 631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 3, 78, 4080, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2148, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3786, 3999, 0, 221, 1478, 4096, 0, 0, 0, 0, 251, 0, 0, 2247, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 20, 0, 4095, 0, 0, 0, 0, 1, 0, 3991, 4058, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 4067, 4096, 0, 2506, 106, 5, 1, 0, 965, 0, 142, 3135, 1751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15657, 16384, 16437, 16497, 16703, 16084, 16385, 16925] avg=16384.0000 max=16925 max_rank=7 min=15657 min_rank=0 imbalance_ratio=1.033020 max_load_reduction=114147 max_load_reduction_pct=0.870872 moved_to_echo=66097 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3349, 0, 1267, 0, 3, 28, 3849, 4067, 4062, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 2, 0, 0, 1, 4094, 0, 0, 4096, 0, 0, 0, 0, 2618, 836, 631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 8, 4065, 21, 0, 2, 0, 79, 4065, 0, 5, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3955, 0, 0, 0, 4095, 24, 4096, 0, 0, 0, 4096, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 908, 4, 37, 0, 0, 0, 0, 0, 4096, 4096, 4096, 2264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 4032, 0, 2244, 1478, 4086, 0, 0, 0, 0, 72, 0, 0, 1230, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 3958, 0, 0, 0, 0, 0, 0, 4071, 4088, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 11, 0, 0, 0, 3927, 4096, 0, 1719, 600, 21, 0, 0, 786, 0, 9, 3135, 1748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16703, 16347, 16409, 16387, 15501, 17109, 16325, 16291] avg=16384.0000 max=17109 max_rank=5 min=15501 min_rank=4 imbalance_ratio=1.044250 max_load_reduction=113963 max_load_reduction_pct=0.869469 moved_to_echo=65226 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3348, 0, 534, 0, 3, 196, 4085, 4096, 3945, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 1, 0, 0, 2618, 836, 631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 78, 4095, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 4094, 0, 4096, 0, 0, 8, 4095, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2489, 3, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 4075, 0, 1085, 1478, 4096, 0, 0, 0, 0, 114, 0, 0, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 153, 0, 4078, 0, 0, 0, 0, 5, 0, 4093, 3520, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 4061, 4096, 0, 2399, 0, 3, 0, 0, 1657, 0, 3, 3135, 1750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16210, 16373, 16427, 16475, 17047, 15297, 16070, 17173] avg=16384.0000 max=17173 max_rank=7 min=15297 min_rank=5 imbalance_ratio=1.048157 max_load_reduction=113899 max_load_reduction_pct=0.868980 moved_to_echo=65587 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3348, 0, 2130, 0, 7, 4, 3810, 4096, 3601, 0, 0, 3, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 3, 0, 0, 37, 0, 0, 4095, 1, 2, 4092, 0, 0, 1, 0, 2613, 836, 631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 5, 13, 4096, 0, 2, 8, 0, 78, 4096, 0, 4, 1, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3778, 0, 0, 1, 4087, 20, 4094, 0, 0, 0, 4089, 0, 0, 1, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 1, 1, 1, 27, 0, 0, 546, 41, 0, 3, 0, 0, 4, 0, 4096, 4096, 4093, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3898, 4092, 0, 2160, 1476, 4090, 0, 6, 1, 1, 351, 0, 0, 170, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 611, 3, 3911, 0, 0, 0, 0, 0, 0, 4077, 3938, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 17, 1, 0, 0, 4086, 4091, 0, 2679, 15, 0, 0, 0, 897, 0, 0, 3135, 1750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17146, 16400, 16456, 16187, 15186, 16261, 16752, 16684] avg=16384.0000 max=17146 max_rank=0 min=15186 min_rank=4 imbalance_ratio=1.046509 max_load_reduction=113926 max_load_reduction_pct=0.869186 moved_to_echo=64883 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3340, 0, 1751, 0, 0, 28, 3807, 4094, 3674, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 4, 0, 3, 4096, 0, 0, 4090, 0, 1, 0, 0, 2622, 839, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 69, 4096, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4013, 0, 0, 0, 4092, 0, 4096, 0, 2, 0, 4096, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 4, 0, 0, 1353, 14, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 2262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 4092, 0, 2204, 1474, 4096, 0, 0, 0, 0, 324, 0, 0, 858, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 11, 0, 4077, 0, 0, 0, 0, 25, 0, 3857, 4078, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4, 0, 0, 0, 4096, 4096, 0, 1095, 19, 0, 2, 0, 1718, 0, 6, 3129, 1748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16696, 16392, 16415, 16426, 15925, 17044, 16252, 15922] avg=16384.0000 max=17044 max_rank=5 min=15922 min_rank=7 imbalance_ratio=1.040283 max_load_reduction=114028 max_load_reduction_pct=0.869965 moved_to_echo=65143 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 3340, 0, 617, 4, 0, 12, 4019, 4093, 3280, 0, 4, 5, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 9, 1, 1, 28, 0, 0, 4096, 0, 0, 4073, 0, 8, 0, 0, 2617, 835, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 2, 0, 0, 1, 4088, 0, 0, 0, 2, 78, 4093, 1, 0, 4, 3847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4089, 2, 0, 5, 3986, 1, 4093, 0, 0, 0, 4091, 0, 11, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6, 1581, 132, 0, 2, 2, 2, 28, 0, 4094, 4092, 4095, 2263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 4093, 4, 2006, 1477, 4090, 7, 0, 0, 0, 902, 2, 0, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 108, 0, 4011, 1, 0, 2, 0, 0, 1, 4073, 3941, 0, 124, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 319, 1, 1, 1, 0, 4054, 4090, 1, 2791, 2, 0, 1, 0, 938, 4, 2, 3129, 1748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15381, 16390, 16200, 16412, 16299, 16941, 16367, 17082] avg=16384.0000 max=17082 max_rank=7 min=15381 min_rank=0 imbalance_ratio=1.042603 max_load_reduction=113990 max_load_reduction_pct=0.869675 moved_to_echo=66689 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3342, 0, 699, 0, 0, 23, 3943, 4094, 3945, 0, 1, 0, 30, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 0, 0, 12, 1, 0, 4095, 0, 3, 4092, 7, 0, 6, 0, 2618, 835, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 3, 0, 4076, 0, 3, 0, 14, 79, 4076, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 13, 4078, 0, 1, 0, 3931, 0, 4092, 0, 0, 0, 4087, 2, 10, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 1, 0, 0, 9, 3572, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4093, 2264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 4093, 0, 607, 1478, 4092, 0, 0, 0, 0, 142, 1, 0, 2584, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3893, 92, 1, 4067, 4, 0, 0, 0, 0, 0, 4073, 3983, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4066, 4094, 0, 548, 103, 12, 1, 0, 523, 0, 166, 3131, 1748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16079, 16400, 16437, 16345, 18135, 17037, 16238, 14401] avg=16384.0000 max=18135 max_rank=4 min=14401 min_rank=7 imbalance_ratio=1.106873 max_load_reduction=112937 max_load_reduction_pct=0.861641 moved_to_echo=65811 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 640, 32768, 128, 128, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32640, 1280, 128, 32256, 128, 0, 128, 0, 128, 128, 32384, 31744, 0, 1024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 26880, 0, 9088, 128, 128, 384, 31232, 32768, 30592, 0, 128, 128, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 32128, 128, 128, 128, 32512, 128, 32768, 0, 128, 128, 32768, 128, 128, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 31616, 32640, 128, 11904, 11904, 32768, 128, 128, 128, 128, 2432, 128, 0, 7936, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 128, 14336, 256, 128, 128, 128, 128, 128, 0, 32768, 32768, 32768, 18176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[896, 128, 128, 128, 0, 32512, 32768, 128, 16128, 1024, 128, 128, 128, 8576, 128, 384, 25088, 14080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 20992, 6784, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2425, 0, 0, 0, 0, 0, 4096, 0, 112, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 982, 4096, 0, 0, 3902, 0, 0, 4077, 0, 3192, 0, 0, 4096, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 2018, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 18, 0, 1, 435, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4095, 3980, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 3229, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[33, 0, 0, 4096, 4096, 0, 0, 0, 4091, 0, 0, 583, 0, 0, 0, 1, 0, 4096, 0, 92, 33, 0, 1, 0, 0, 4096, 0, 0, 4089, 0, 0, 4095, 139, 4096, 5, 0, 4071, 1, 0, 3222, 0, 4005, 0, 0, 4096, 0, 0, 0, 0, 1154, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2110, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4088, 0, 0, 5, 0, 0, 1237, 0, 0, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4074, 4071, 4094, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 4096, 0, 4092, 0, 4096, 0, 0, 5, 3837, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 19] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1578, 0, 0, 0, 0, 0, 4096, 0, 82, 3, 0, 0, 0, 5, 4096, 0, 0, 4096, 1, 0, 4096, 623, 4096, 0, 0, 3084, 0, 0, 3728, 0, 3312, 0, 0, 4096, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3263, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4096, 0, 4096, 0, 0, 140, 0, 1, 882, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4093, 4096, 4092, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 4096, 0, 4095, 0, 4095, 0, 0, 0, 3683, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 39] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[7, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 3188, 0, 0, 0, 0, 0, 4096, 0, 32, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 0, 4095, 490, 4096, 18, 3, 4006, 0, 0, 3983, 0, 3988, 0, 0, 4096, 0, 0, 0, 0, 376, 0, 0, 0, 0, 0, 0, 4095, 0, 4086, 1247, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 0, 0, 34, 0, 0, 759, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4090, 4087, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 1, 2354, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 16] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 4081, 0, 0, 0, 4092, 0, 0, 1060, 3, 0, 0, 0, 2, 4096, 0, 120, 2, 0, 4, 0, 0, 4093, 1, 0, 4092, 1, 0, 4096, 62, 4096, 9, 6, 3861, 0, 0, 4031, 0, 3254, 0, 0, 4093, 0, 1, 2, 0, 969, 0, 1, 1, 0, 1, 0, 4095, 0, 4096, 2793, 4096, 1, 4, 0, 1, 0, 0, 0, 0, 0, 0, 4079, 4096, 0, 4090, 0, 0, 0, 2, 0, 1164, 4, 0, 4095, 0, 0, 1, 0, 4092, 4094, 1, 4, 4090, 0, 0, 4096, 4096, 4096, 0, 0, 4, 0, 0, 0, 4091, 0, 0, 4096, 0, 4094, 0, 4070, 0, 0, 0, 3200, 1, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 3, 12] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[64, 1, 0, 4092, 4092, 1, 3, 0, 4092, 2, 0, 2453, 0, 0, 4, 0, 0, 4092, 1, 133, 0, 0, 0, 1, 0, 4094, 0, 3, 4092, 1, 5, 4092, 1069, 4089, 31, 13, 4003, 6, 0, 4093, 0, 3094, 2, 4, 4094, 3, 4, 0, 2, 606, 1, 0, 0, 0, 0, 2, 4093, 3, 4091, 2319, 4094, 4, 14, 0, 4, 0, 0, 0, 5, 3, 0, 4093, 4091, 0, 4052, 1, 0, 0, 0, 3, 130, 0, 2, 4093, 0, 7, 7, 0, 4092, 4093, 0, 21, 4092, 0, 1, 4075, 4074, 4090, 6, 1, 5, 0, 1, 0, 4094, 1, 0, 4092, 0, 4091, 9, 4086, 0, 4, 0, 2500, 0, 2, 0, 4, 0, 0, 3, 0, 4090, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[16, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 784, 0, 0, 0, 0, 0, 4096, 0, 27, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 197, 4096, 1, 3, 3890, 0, 0, 3907, 0, 3576, 0, 0, 4096, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 4096, 0, 4079, 2730, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 4075, 0, 0, 1, 0, 0, 1484, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4093, 4079, 4096, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4094, 0, 0, 0, 3753, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 33] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[40, 0, 1, 4096, 4095, 0, 0, 0, 4096, 0, 5, 1263, 1, 0, 1, 0, 0, 4094, 0, 8, 24, 1, 1, 0, 0, 4095, 0, 0, 4094, 0, 0, 4096, 89, 4096, 21, 0, 4023, 0, 0, 3783, 0, 4082, 0, 0, 4096, 0, 0, 0, 0, 149, 0, 0, 0, 0, 1, 1, 4096, 0, 4083, 3460, 4094, 0, 12, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4095, 3, 4096, 0, 0, 0, 0, 0, 710, 0, 0, 4095, 0, 0, 5, 0, 4088, 4096, 0, 0, 4096, 0, 1, 4082, 4068, 4096, 0, 0, 0, 0, 7, 0, 4092, 0, 0, 4095, 0, 4096, 6, 4096, 0, 0, 2, 2852, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 2, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[33, 0, 0, 4096, 4096, 0, 0, 0, 4091, 0, 0, 583, 0, 0, 0, 1, 2411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 92, 33, 0, 1, 0, 0, 4013, 0, 0, 4089, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 4096, 5, 0, 4071, 1, 0, 3222, 0, 4005, 0, 0, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1154, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2110, 4096, 0, 0, 0, 844, 215, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4088, 0, 0, 5, 0, 0, 3159, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1237, 0, 0, 0, 0, 0, 1, 0, 4096, 3252, 0, 0, 4096, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 4094, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 0, 33, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3837, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 4096, 4096, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15311, 16419, 16085, 16694, 16372, 16756, 16386, 17049] avg=16384.0000 max=17049 max_rank=7 min=15311 min_rank=0 imbalance_ratio=1.040588 max_load_reduction=114023 max_load_reduction_pct=0.869926 moved_to_echo=66563 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2425, 0, 0, 0, 0, 2411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 112, 0, 0, 1, 0, 0, 4013, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 982, 4096, 0, 0, 3902, 0, 0, 4077, 0, 3192, 0, 0, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 2018, 4096, 0, 0, 0, 844, 215, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 18, 0, 1, 3162, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 0, 0, 0, 0, 0, 0, 0, 4096, 3252, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 33, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3229, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 4096, 4096, 901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17124, 16414, 16795, 15646, 16392, 15974, 16301, 16426] avg=16384.0000 max=17124 max_rank=0 min=15646 min_rank=3 imbalance_ratio=1.045166 max_load_reduction=113948 max_load_reduction_pct=0.869354 moved_to_echo=65093 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 3188, 0, 0, 0, 0, 2411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 32, 0, 0, 0, 0, 2, 4013, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 4096, 18, 3, 4006, 0, 0, 3983, 0, 3988, 0, 0, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 0, 0, 0, 0, 0, 0, 4095, 0, 4086, 1247, 4096, 0, 2, 0, 844, 215, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 0, 0, 34, 0, 0, 3162, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 0, 4096, 3252, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 33, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2354, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 16, 4096, 4096, 901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17894, 16334, 17130, 15044, 16407, 16293, 16408, 15562] avg=16384.0000 max=17894 max_rank=0 min=15044 min_rank=3 imbalance_ratio=1.092163 max_load_reduction=113178 max_load_reduction_pct=0.863480 moved_to_echo=64670 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1578, 0, 0, 0, 0, 2411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 82, 3, 0, 0, 0, 5, 4013, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 623, 4096, 0, 0, 3084, 0, 0, 3728, 0, 3312, 0, 0, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3263, 4082, 0, 0, 0, 844, 215, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4096, 0, 4096, 0, 0, 140, 0, 1, 3161, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 882, 0, 0, 0, 0, 0, 0, 0, 4096, 3252, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 33, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3683, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 39, 4096, 4096, 901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16277, 16392, 15389, 16801, 16477, 16419, 16395, 16922] avg=16384.0000 max=16922 max_rank=7 min=15389 min_rank=2 imbalance_ratio=1.032837 max_load_reduction=114150 max_load_reduction_pct=0.870895 moved_to_echo=66213 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 4081, 0, 0, 0, 4092, 0, 0, 1060, 3, 0, 0, 0, 2409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 120, 2, 0, 4, 0, 0, 4011, 1, 0, 4092, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 4096, 9, 6, 3861, 0, 0, 4031, 0, 3254, 0, 0, 546, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 969, 0, 1, 1, 0, 1, 0, 4095, 0, 4096, 2793, 4096, 1, 4, 0, 844, 215, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4079, 4096, 0, 4090, 0, 0, 0, 2, 0, 3161, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1164, 4, 0, 0, 0, 0, 1, 0, 4092, 3250, 1, 4, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 4091, 0, 0, 0, 0, 33, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3200, 1, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 3, 12, 4096, 4095, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15738, 16425, 15868, 17198, 16352, 16702, 16390, 16399] avg=16384.0000 max=17198 max_rank=3 min=15738 min_rank=0 imbalance_ratio=1.049683 max_load_reduction=113874 max_load_reduction_pct=0.868790 moved_to_echo=65843 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 784, 0, 0, 0, 0, 2417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 27, 0, 0, 0, 0, 0, 4010, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 4096, 1, 3, 3890, 0, 0, 3907, 0, 3576, 0, 0, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 4096, 0, 4079, 2730, 4092, 0, 0, 0, 848, 216, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 4075, 0, 0, 1, 0, 0, 3167, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1484, 0, 0, 0, 0, 0, 0, 0, 4096, 3248, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4096, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 25, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3753, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 33, 4096, 4096, 904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15505, 16325, 16208, 16283, 16362, 17017, 16394, 16978] avg=16384.0000 max=17017 max_rank=5 min=15505 min_rank=0 imbalance_ratio=1.038635 max_load_reduction=114055 max_load_reduction_pct=0.870171 moved_to_echo=66751 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[40, 0, 1, 4096, 4095, 0, 0, 0, 4096, 0, 5, 1263, 1, 0, 1, 0, 2411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 8, 24, 1, 1, 0, 0, 4013, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 4096, 21, 0, 4023, 0, 0, 3783, 0, 4082, 0, 0, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 0, 0, 1, 1, 4096, 0, 4083, 3460, 4094, 0, 12, 0, 844, 215, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4095, 3, 4096, 0, 0, 0, 0, 0, 3162, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 710, 0, 0, 0, 0, 0, 5, 0, 4088, 3252, 0, 0, 4096, 0, 1, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 4096, 0, 0, 0, 0, 7, 0, 4092, 0, 0, 0, 0, 33, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2852, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 2, 2, 4095, 4095, 901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16009, 16331, 16640, 17037, 16377, 16234, 16398, 16046] avg=16384.0000 max=17037 max_rank=3 min=16009 min_rank=0 imbalance_ratio=1.039856 max_load_reduction=114035 max_load_reduction_pct=0.870018 moved_to_echo=65055 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[64, 1, 0, 4092, 4092, 1, 3, 0, 4092, 2, 0, 2453, 0, 0, 4, 0, 2410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 133, 0, 0, 0, 1, 0, 4012, 0, 3, 4092, 1, 5, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 4089, 31, 13, 4003, 6, 0, 4093, 0, 3094, 2, 4, 546, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 606, 1, 0, 0, 0, 0, 2, 4093, 3, 4091, 2319, 4094, 4, 14, 0, 843, 215, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 5, 3, 0, 4093, 4091, 0, 4052, 1, 0, 0, 0, 3, 3158, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 2, 0, 0, 7, 7, 0, 4092, 3250, 0, 21, 4092, 0, 1, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4090, 6, 1, 5, 0, 1, 0, 4094, 1, 0, 0, 0, 33, 9, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2500, 0, 2, 0, 4, 0, 0, 3, 0, 4090, 0, 0, 2, 4092, 4093, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17214, 16432, 16957, 16369, 16333, 15677, 16400, 15690] avg=16384.0000 max=17214 max_rank=0 min=15677 min_rank=5 imbalance_ratio=1.050659 max_load_reduction=113858 max_load_reduction_pct=0.868668 moved_to_echo=64100 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32640, 32768, 128, 128, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 256, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[256, 128, 128, 32768, 32768, 128, 128, 0, 32768, 128, 128, 13440, 128, 0, 128, 128, 19328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3712, 32768, 128, 128, 30848, 128, 0, 30848, 0, 28544, 128, 128, 4480, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[6912, 128, 128, 0, 0, 128, 128, 0, 32768, 26112, 128, 128, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 25472, 128, 128, 0, 128, 128, 0, 128, 128, 32768, 128, 128, 256, 32768, 32768, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 3712, 128, 128, 128, 0, 128, 128, 32768, 128, 32768, 19968, 32768, 128, 128, 0, 6784, 1792, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 32768, 32768, 128, 32768, 128, 0, 256, 128, 128, 25344, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 640, 128, 128, 128, 128, 128, 32128, 128, 128, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3900, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 1, 7, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 173, 5, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 16, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 4088, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 3997, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3650, 0, 4096, 4096, 0, 0, 21, 0, 0, 0, 2, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4050, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 34, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 40, 55, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 284, 4096, 0, 0, 3, 4096, 4096, 0, 0, 4095, 0, 4061, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4046, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 98] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3836, 0, 4096, 4085, 0, 0, 6, 0, 0, 0, 0, 0, 0, 20, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4095, 4096, 0, 197, 46, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 33, 4096, 0, 0, 5, 4096, 4096, 0, 0, 4084, 0, 4076, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4056, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 20] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4065, 0, 4096, 4087, 0, 0, 6, 0, 0, 0, 1, 0, 0, 39, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 1, 12, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 23, 4096, 4096, 0, 0, 4093, 0, 4052, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 4095, 0, 0, 4096, 0, 3981, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4025, 0, 4096, 4095, 0, 0, 8, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 4096, 4094, 4094, 4, 60, 25, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 7, 4096, 0, 0, 25, 4096, 4096, 0, 0, 4095, 0, 4042, 0, 4096, 0, 4096, 0, 0, 3, 0, 0, 4091, 4096, 0, 4091, 0, 0, 4096, 0, 4029, 0, 0, 0, 0, 0, 29, 6, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4089, 0, 0, 4094, 4096, 22] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[6, 2, 3, 4, 3108, 0, 4083, 4083, 0, 8, 25, 4, 0, 7, 0, 2, 0, 12, 4, 4090, 5, 6, 0, 6, 0, 4081, 0, 4085, 0, 0, 1, 0, 0, 4, 0, 5, 5, 0, 4092, 0, 4086, 0, 0, 36, 0, 0, 0, 5, 6, 0, 4, 4, 3, 0, 5, 4091, 4092, 4, 3, 4090, 4091, 4091, 4092, 0, 1118, 7, 2, 0, 4, 4, 4089, 0, 5, 4093, 4, 4, 1, 4087, 1, 0, 0, 4089, 4092, 0, 0, 4073, 0, 3881, 0, 4090, 0, 4091, 11, 2, 0, 5, 1, 4092, 4092, 0, 4092, 0, 4, 4088, 0, 4091, 3, 0, 4, 2, 0, 41, 6, 4, 2, 0, 4074, 0, 0, 4, 1, 5, 4089, 0, 7, 4092, 4091, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4022, 0, 4092, 4092, 0, 0, 19, 0, 0, 0, 0, 0, 5, 81, 0, 4095, 0, 0, 0, 0, 0, 4090, 0, 4090, 0, 2, 0, 7, 5, 0, 3, 0, 0, 0, 4095, 0, 4086, 0, 0, 78, 0, 0, 0, 0, 0, 6, 0, 0, 32, 0, 0, 4096, 4094, 0, 0, 4093, 4093, 4093, 4096, 1, 41, 0, 3, 1, 0, 0, 4095, 0, 0, 4089, 0, 0, 33, 4090, 0, 1, 5, 4093, 4096, 0, 0, 4086, 0, 4015, 3, 4096, 0, 4096, 0, 0, 3, 0, 1, 4094, 4094, 0, 4091, 2, 0, 4096, 0, 4096, 0, 0, 0, 0, 3, 1, 3, 0, 0, 0, 3985, 0, 0, 0, 6, 0, 4096, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3914, 0, 4096, 4094, 0, 0, 18, 0, 0, 0, 3, 0, 0, 64, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 12, 1, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 13, 6, 0, 0, 1, 0, 0, 0, 5, 0, 1, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4095, 0, 372, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4, 4096, 4096, 0, 4, 4075, 0, 3720, 0, 4096, 0, 4096, 0, 0, 1, 0, 0, 4088, 4093, 0, 4096, 0, 0, 4096, 0, 4013, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 4096, 158] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3900, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 235, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 3832, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 98, 0, 0, 0, 0, 4096, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 5, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 16, 4096, 0, 0, 3753, 20, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 4088, 0, 4096, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 29, 0, 3997, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4076, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16459, 16366, 16447, 16378, 16259, 16479, 16335, 16349] avg=16384.0000 max=16479 max_rank=5 min=16259 min_rank=4 imbalance_ratio=1.005798 max_load_reduction=114593 max_load_reduction_pct=0.874275 moved_to_echo=65422 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3650, 0, 4096, 4096, 0, 0, 21, 0, 0, 0, 2, 0, 4096, 235, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4050, 0, 0, 0, 6, 3832, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 34, 1, 0, 0, 0, 0, 4096, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 55, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 284, 4096, 0, 0, 3753, 20, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 4095, 0, 4061, 0, 4096, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 29, 0, 4046, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4076, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16230, 16326, 16379, 16378, 16446, 16455, 16397, 16461] avg=16384.0000 max=16461 max_rank=7 min=16230 min_rank=0 imbalance_ratio=1.004700 max_load_reduction=114611 max_load_reduction_pct=0.874413 moved_to_echo=65759 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4025, 0, 4096, 4095, 0, 0, 8, 0, 0, 0, 5, 0, 4096, 235, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 3832, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 32, 0, 0, 0, 0, 0, 4096, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4096, 4094, 4088, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 25, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 7, 4096, 0, 0, 3753, 20, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 0, 0, 0, 4095, 0, 4042, 0, 4096, 0, 104, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 4091, 0, 0, 29, 0, 4029, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4089, 0, 0, 4094, 4076, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16595, 16360, 16376, 16379, 16159, 16461, 16365, 16377] avg=16384.0000 max=16595 max_rank=0 min=16159 min_rank=4 imbalance_ratio=1.012878 max_load_reduction=114477 max_load_reduction_pct=0.873390 moved_to_echo=65362 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3836, 0, 4096, 4085, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 238, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 2, 3833, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 3, 0, 0, 0, 0, 4096, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 46, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 33, 4096, 0, 0, 3759, 27, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 4084, 0, 4076, 0, 4096, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 25, 0, 4056, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4069, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 16375, 16353, 16380, 16361, 16453, 16381, 16377] avg=16384.0000 max=16453 max_rank=5 min=16353 min_rank=2 imbalance_ratio=1.004211 max_load_reduction=114619 max_load_reduction_pct=0.874474 moved_to_echo=65572 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4065, 0, 4096, 4087, 0, 0, 6, 0, 0, 0, 1, 0, 4096, 235, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 21, 3832, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 1, 0, 0, 1, 0, 4096, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 3753, 20, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4096, 0, 0, 0, 4093, 0, 4052, 0, 4096, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4095, 0, 0, 29, 0, 3981, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4076, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16621, 16418, 16349, 16379, 16080, 16464, 16386, 16375] avg=16384.0000 max=16621 max_rank=0 min=16080 min_rank=4 imbalance_ratio=1.014465 max_load_reduction=114451 max_load_reduction_pct=0.873192 moved_to_echo=65305 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4022, 0, 4092, 4092, 0, 0, 19, 0, 0, 0, 0, 0, 4094, 235, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 81, 0, 4095, 0, 0, 0, 0, 0, 4090, 0, 4090, 0, 2, 0, 7, 3832, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3, 0, 0, 0, 4095, 0, 4086, 0, 0, 78, 0, 0, 0, 0, 4096, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 32, 0, 0, 0, 0, 0, 0, 4093, 4093, 4093, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 3, 1, 0, 0, 4095, 0, 0, 4089, 0, 0, 33, 4090, 0, 1, 3753, 20, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4093, 0, 0, 0, 4086, 0, 4015, 3, 4096, 0, 104, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 4094, 0, 4091, 2, 0, 29, 0, 4096, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3985, 0, 0, 0, 6, 0, 4096, 0, 0, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16589, 16441, 16424, 16408, 16132, 16405, 16411, 16262] avg=16384.0000 max=16589 max_rank=0 min=16132 min_rank=4 imbalance_ratio=1.012512 max_load_reduction=114483 max_load_reduction_pct=0.873436 moved_to_echo=65210 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[6, 2, 3, 4, 3108, 0, 4083, 4083, 0, 8, 25, 4, 0, 7, 0, 2, 4092, 234, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4, 4090, 5, 6, 0, 6, 0, 4081, 0, 4085, 0, 0, 1, 0, 3824, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 5, 0, 4092, 0, 4086, 0, 0, 36, 0, 0, 0, 5, 4092, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4, 4, 3, 0, 5, 0, 0, 4, 3, 4090, 4091, 4091, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1118, 7, 2, 0, 4, 4, 4089, 0, 5, 4093, 4, 4, 1, 4087, 1, 0, 3749, 20, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4073, 0, 3881, 0, 4090, 0, 104, 11, 2, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 4092, 0, 4092, 0, 4, 30, 0, 4091, 3, 0, 4, 2, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 2, 0, 4074, 0, 0, 4, 1, 5, 4089, 0, 7, 4092, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15695, 16352, 16382, 16387, 17194, 16255, 16452, 16355] avg=16384.0000 max=17194 max_rank=4 min=15695 min_rank=0 imbalance_ratio=1.049438 max_load_reduction=113878 max_load_reduction_pct=0.868820 moved_to_echo=66256 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3914, 0, 4096, 4094, 0, 0, 18, 0, 0, 0, 3, 0, 4096, 235, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 12, 3832, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 13, 6, 0, 0, 1, 0, 4096, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3753, 20, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4, 4075, 0, 3720, 0, 4096, 0, 104, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4093, 0, 4096, 0, 0, 29, 0, 4013, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 4076, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16491, 16434, 16362, 16383, 16441, 16100, 16345, 16516] avg=16384.0000 max=16516 max_rank=7 min=16100 min_rank=5 imbalance_ratio=1.008057 max_load_reduction=114556 max_load_reduction_pct=0.873993 moved_to_echo=65402 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 30592, 0, 32768, 32768, 0, 128, 128, 128, 0, 128, 128, 128, 32768, 1920, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 32768, 0, 32768, 128, 128, 256, 0, 32384, 128, 0, 128, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 256, 128, 32768, 128, 128, 0, 128, 0, 32768, 0, 32768, 0, 128, 128, 128, 30720, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 0, 32768, 128, 128, 256, 0, 0, 128, 128, 32768, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 0, 0, 128, 32768, 0, 32000, 128, 32768, 0, 896, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 32640, 0, 0, 128, 128, 128, 32768, 0, 128, 32768, 32640, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[2048, 256, 128, 128, 128, 128, 32768, 0, 128, 32768, 128, 128, 384, 32768, 128, 128, 30080, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 0, 0, 128, 128, 32768, 32768, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4031, 0, 4096, 0, 0, 4096, 4086, 0, 0, 4096, 4084, 4096, 0, 0, 0, 0, 0, 4089, 0, 4096, 0, 4096, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 107, 280, 4096, 0, 0, 0, 2649, 0, 0, 0, 0, 4096, 4069, 0, 0, 0, 3912, 0, 4093, 419, 0, 0, 0, 4093, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 647, 0, 0, 6, 0, 5, 8, 4093, 0, 0, 0, 4096, 0, 0, 0, 4096, 262, 0, 0, 0, 0, 0, 3, 0, 4096, 2, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4096, 7, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3809, 0, 4096, 0, 0, 4096, 4010, 0, 0, 4096, 4073, 4096, 0, 0, 0, 0, 6, 4095, 0, 4096, 0, 4096, 0, 32, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 51, 190, 4096, 0, 0, 0, 3185, 0, 0, 0, 0, 4096, 4057, 0, 0, 0, 869, 0, 4096, 2, 0, 0, 0, 4091, 4096, 0, 4096, 0, 0, 0, 0, 4071, 0, 5, 4068, 0, 0, 0, 0, 0, 4, 4056, 0, 0, 0, 4096, 0, 0, 0, 4096, 280, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3635, 0, 4096, 0, 0, 4096, 4088, 0, 0, 4096, 4090, 4096, 0, 0, 3, 0, 5, 4014, 0, 4096, 0, 4096, 0, 35, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 87, 531, 4096, 0, 0, 0, 362, 0, 0, 0, 0, 4096, 4076, 0, 0, 2, 3945, 0, 4096, 706, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 4084, 0, 0, 891, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 4096, 2211, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4035, 0, 4096, 0, 0, 4096, 4040, 0, 0, 4096, 4070, 4096, 0, 0, 39, 0, 0, 4089, 0, 4096, 0, 4096, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 51, 15, 4096, 0, 0, 0, 2187, 0, 0, 0, 0, 4096, 4095, 0, 0, 11, 3796, 0, 4087, 355, 0, 0, 0, 4093, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1401, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 498, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[3, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4018, 0, 4096, 0, 0, 4096, 4060, 0, 0, 4092, 4058, 4096, 0, 0, 3, 0, 0, 4087, 0, 4096, 0, 4096, 0, 9, 0, 4093, 2, 0, 1, 0, 0, 4096, 4089, 0, 4095, 49, 99, 4092, 0, 0, 1, 2121, 0, 0, 0, 8, 4096, 3905, 0, 0, 0, 3184, 0, 4093, 57, 0, 0, 0, 4088, 4091, 0, 4094, 0, 0, 0, 0, 4096, 0, 6, 2604, 0, 0, 5, 0, 2, 13, 4096, 1, 0, 0, 4096, 0, 0, 3, 4096, 390, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 1, 4096, 7, 1, 0, 0, 4096, 5, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3859, 0, 4096, 0, 0, 4096, 3979, 0, 0, 4096, 4096, 4096, 0, 0, 3, 0, 0, 4095, 0, 4096, 0, 4096, 0, 18, 0, 4096, 0, 0, 0, 0, 0, 4096, 4094, 0, 4096, 0, 221, 4096, 0, 0, 0, 374, 0, 0, 0, 0, 4096, 4083, 0, 0, 1, 3596, 0, 4094, 35, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3572, 0, 0, 3475, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 1360, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 4091, 0, 5, 6, 0, 0, 0, 5, 0, 0, 0, 4, 0, 4091, 0, 7, 0, 3735, 0, 4076, 0, 3, 4091, 3883, 1, 5, 4049, 4071, 4089, 5, 0, 0, 4, 3, 3951, 1, 4090, 6, 4090, 0, 254, 7, 4091, 0, 0, 0, 7, 6, 4078, 4091, 0, 4088, 119, 143, 4091, 6, 0, 0, 455, 0, 0, 0, 0, 4090, 4075, 0, 2, 13, 3929, 0, 4092, 0, 4, 5, 3, 4092, 4091, 0, 4091, 1, 4, 5, 0, 4080, 5, 0, 4019, 1, 7, 1, 5, 0, 0, 4066, 2, 5, 6, 4091, 0, 4, 0, 4090, 69, 5, 8, 0, 5, 0, 0, 2, 4090, 17, 5, 1, 0, 0, 4090, 3, 2, 0, 6, 4089, 0, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 3880, 0, 4086, 0, 0, 4096, 4030, 0, 0, 4096, 4061, 4096, 0, 0, 36, 0, 0, 4095, 0, 4096, 0, 4096, 1, 21, 0, 4096, 0, 0, 0, 0, 0, 4096, 4092, 0, 4096, 105, 174, 4096, 0, 0, 1, 1875, 0, 0, 0, 1, 4096, 3695, 0, 0, 3, 2277, 0, 4096, 0, 0, 0, 0, 4085, 4096, 6, 4094, 0, 0, 0, 0, 4096, 0, 10, 4013, 0, 0, 2, 0, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 1, 4096, 399, 0, 0, 0, 0, 0, 1, 0, 4096, 4, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 320, 0, 0, 0, 4086, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4068, 0, 4096, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2172, 107, 280, 4096, 0, 0, 0, 2649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 3912, 0, 4093, 419, 0, 0, 0, 4093, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 647, 0, 0, 6, 0, 5, 8, 4093, 1499, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 262, 0, 0, 0, 0, 0, 3, 0, 4096, 3410, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4096, 7, 0, 0, 0, 3296, 425, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16617, 16358, 17496, 17386, 14479, 15964, 16386] avg=16384.0000 max=17496 max_rank=3 min=14479 min_rank=5 imbalance_ratio=1.067871 max_load_reduction=113576 max_load_reduction_pct=0.866516 moved_to_echo=64215 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3809, 0, 320, 0, 0, 0, 4010, 0, 0, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4095, 0, 4068, 0, 4096, 0, 32, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2172, 51, 190, 4096, 0, 0, 0, 3185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 869, 0, 4096, 2, 0, 0, 0, 4091, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4071, 0, 5, 4068, 0, 0, 0, 0, 0, 4, 4056, 1499, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 280, 0, 0, 0, 0, 0, 0, 0, 4096, 3410, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3296, 425, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16308, 16393, 17886, 13915, 17827, 15979, 16381] avg=16384.0000 max=17886 max_rank=3 min=13915 min_rank=4 imbalance_ratio=1.091675 max_load_reduction=113186 max_load_reduction_pct=0.863541 moved_to_echo=64102 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 320, 0, 0, 0, 4040, 0, 0, 4096, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 4089, 0, 4068, 0, 4096, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2172, 51, 15, 4096, 0, 0, 0, 2187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 11, 3796, 0, 4087, 355, 0, 0, 0, 4093, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1401, 0, 0, 0, 0, 0, 0, 4096, 1499, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 498, 0, 0, 0, 0, 0, 0, 0, 4096, 3410, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3296, 425, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16561, 16390, 16713, 17237, 15216, 16198, 16375] avg=16384.0000 max=17237 max_rank=4 min=15216 min_rank=5 imbalance_ratio=1.052063 max_load_reduction=113835 max_load_reduction_pct=0.868492 moved_to_echo=65026 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3859, 0, 320, 0, 0, 0, 3979, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4095, 0, 4068, 0, 4096, 0, 18, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 2172, 0, 221, 4096, 0, 0, 0, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 1, 3596, 0, 4094, 35, 0, 0, 0, 4096, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3572, 0, 0, 3475, 0, 0, 0, 0, 0, 0, 4096, 1499, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1360, 0, 0, 0, 0, 0, 0, 0, 4096, 3410, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3296, 425, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16350, 16376, 15053, 16705, 16766, 17060, 16376] avg=16384.0000 max=17060 max_rank=6 min=15053 min_rank=3 imbalance_ratio=1.041260 max_load_reduction=114012 max_load_reduction_pct=0.869843 moved_to_echo=66907 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3635, 0, 312, 0, 0, 0, 4088, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 5, 4014, 0, 4065, 0, 4096, 0, 35, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2163, 87, 531, 4096, 0, 0, 0, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 2, 3945, 0, 4096, 706, 0, 0, 0, 4095, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 0, 0, 891, 0, 0, 0, 0, 0, 0, 4086, 1502, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2211, 0, 0, 0, 0, 0, 0, 0, 4096, 3412, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3298, 431, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16393, 16221, 16314, 15431, 17718, 14690, 17912, 16393] avg=16384.0000 max=17912 max_rank=6 min=14690 min_rank=5 imbalance_ratio=1.093262 max_load_reduction=113160 max_load_reduction_pct=0.863342 moved_to_echo=66713 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4018, 0, 320, 0, 0, 0, 4060, 0, 0, 4092, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4087, 0, 4068, 0, 4096, 0, 9, 0, 4093, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 2171, 49, 99, 4092, 0, 0, 1, 2121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3905, 0, 0, 0, 3184, 0, 4093, 57, 0, 0, 0, 4088, 799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 6, 2604, 0, 0, 5, 0, 2, 13, 4096, 1499, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 3, 4096, 390, 0, 0, 0, 0, 0, 3, 0, 4096, 3410, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 7, 1, 0, 0, 4096, 5, 0, 0, 0, 3292, 425, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16550, 16359, 16718, 16134, 16443, 16097, 16385] avg=16384.0000 max=16718 max_rank=3 min=16097 min_rank=6 imbalance_ratio=1.020386 max_load_reduction=114354 max_load_reduction_pct=0.872452 moved_to_echo=65059 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3880, 0, 320, 0, 0, 0, 4030, 0, 0, 4096, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 4095, 0, 4068, 0, 4096, 1, 21, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 2172, 105, 174, 4096, 0, 0, 1, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3695, 0, 0, 3, 2277, 0, 4096, 0, 0, 0, 0, 4085, 800, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 10, 4013, 0, 0, 2, 0, 0, 4, 4096, 1499, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 399, 0, 0, 0, 0, 0, 1, 0, 4096, 3401, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3296, 425, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16374, 16387, 16413, 16611, 14963, 17842, 16091, 16391] avg=16384.0000 max=17842 max_rank=5 min=14963 min_rank=4 imbalance_ratio=1.088989 max_load_reduction=113230 max_load_reduction_pct=0.863876 moved_to_echo=65287 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4091, 0, 5, 6, 0, 0, 0, 5, 0, 0, 0, 4, 0, 4091, 4091, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3735, 0, 319, 0, 3, 0, 3883, 1, 5, 4049, 4071, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 3951, 1, 4062, 6, 4090, 0, 254, 7, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 4078, 4091, 0, 2168, 119, 143, 4091, 6, 0, 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 2, 13, 3929, 0, 4092, 0, 4, 5, 3, 4092, 799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 4, 5, 0, 4080, 5, 0, 4019, 1, 7, 1, 5, 0, 0, 4066, 1496, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 6, 4091, 0, 4, 0, 4090, 69, 5, 8, 0, 5, 0, 0, 2, 4089, 3393, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 17, 5, 1, 0, 0, 4090, 3, 2, 0, 6, 4089, 0, 0, 2, 0, 3292, 424, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16078, 16469, 15164, 17014, 17809, 15771, 16385] avg=16384.0000 max=17809 max_rank=5 min=15164 min_rank=3 imbalance_ratio=1.086975 max_load_reduction=113263 max_load_reduction_pct=0.864128 moved_to_echo=66979 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 0, 0, 128, 128, 0, 0, 0, 128, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 32768, 0, 17408, 640, 1664, 32768, 128, 0, 128, 13312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 31104, 0, 2560, 0, 128, 0, 32256, 128, 128, 32768, 32640, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 128, 128, 0, 32256, 128, 128, 21120, 128, 128, 128, 128, 128, 128, 32768, 12032, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 128, 0, 128, 32768, 128, 128, 0, 128, 32768, 128, 0, 128, 0, 26368, 3456, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 32128, 0, 128, 128, 25600, 0, 32768, 1664, 128, 128, 128, 32768, 6400, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 32768, 0, 128, 128, 32768, 5504, 128, 128, 0, 128, 0, 128, 128, 32768, 27264, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 32640, 128, 32640, 128, 32768, 128, 384, 128, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4094, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 3027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4095, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4075, 0, 0, 0, 76, 3876, 0, 0, 0, 0, 0, 0, 4095, 1756, 0, 0, 0, 4096, 0, 0, 3472, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 13, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[2, 4096, 4094, 0, 0, 0, 0, 4095, 0, 4096, 0, 4096, 0, 0, 3, 0, 5, 4096, 4088, 4096, 4092, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 6, 2779, 0, 0, 0, 0, 32, 0, 0, 0, 2, 4096, 4090, 0, 4096, 4071, 1, 4094, 0, 0, 4096, 1, 30, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 14, 4003, 0, 0, 0, 0, 0, 0, 4096, 1936, 0, 0, 10, 4096, 0, 0, 2927, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 7, 0, 0, 0, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4093, 0, 0, 0, 0, 4093, 0, 4096, 0, 4096, 0, 0, 0, 0, 1, 4096, 4096, 4096, 4091, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1524, 0, 0, 0, 0, 34, 10, 0, 0, 0, 4096, 4093, 0, 4096, 4079, 0, 4093, 0, 0, 4096, 0, 2, 0, 0, 1627, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 296, 0, 4096, 0, 0, 0, 837, 3409, 0, 0, 6, 0, 0, 0, 4096, 1487, 0, 0, 41, 4083, 0, 0, 2204, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 857, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 3, 4096, 4077, 4096, 4085, 0, 4096, 4096, 0, 0, 0, 0, 0, 30, 0, 0, 3857, 0, 0, 0, 0, 26, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4094, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 1360, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4085, 0, 0, 0, 38, 3935, 0, 0, 0, 0, 0, 0, 4096, 971, 0, 0, 11, 4096, 0, 0, 2102, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2719, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 798, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 4093, 0, 0, 0, 7, 4081, 0, 0, 2, 0, 0, 0, 4096, 367, 0, 0, 6, 4096, 0, 0, 3277, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 1015, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4089, 4088, 6, 7, 4, 8, 4091, 0, 4090, 0, 4086, 0, 4, 0, 0, 0, 4085, 4091, 4081, 4090, 18, 4087, 4089, 0, 5, 0, 0, 6, 0, 0, 9, 2775, 4, 0, 0, 0, 0, 0, 6, 9, 0, 4079, 4091, 0, 4091, 4090, 0, 4092, 0, 0, 4033, 0, 0, 6, 0, 875, 6, 7, 0, 3, 8, 0, 1, 4079, 3, 3, 77, 7, 4080, 1, 5, 4, 219, 4089, 5, 5, 0, 7, 1, 1, 4089, 2679, 1, 0, 0, 4092, 0, 0, 1677, 3, 7, 4, 0, 0, 4072, 7, 7, 4022, 4088, 1, 4089, 0, 0, 4079, 18, 1, 3, 0, 4090, 0, 30, 0, 1, 0, 7, 0, 4, 8, 6, 0, 6, 3, 0, 4088, 0, 4088, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4093, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3618, 0, 0, 0, 0, 6, 7, 0, 0, 1, 4096, 4096, 0, 4096, 4089, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 460, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 0, 4049, 0, 0, 0, 1542, 3935, 0, 0, 0, 0, 0, 0, 4096, 787, 0, 0, 0, 4094, 0, 0, 1722, 0, 0, 0, 0, 0, 4086, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 259, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4094, 0, 4096, 0, 4096, 0, 0, 0, 0, 3, 4096, 4092, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 16, 2, 0, 0, 1, 4096, 4095, 0, 4096, 4090, 0, 4096, 0, 0, 4091, 1, 1, 0, 0, 975, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 5, 0, 4083, 0, 0, 0, 35, 3958, 0, 0, 1, 0, 0, 0, 4096, 570, 0, 0, 0, 4096, 0, 2, 1900, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 764, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1040, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 4095, 3251, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4075, 0, 0, 0, 76, 3876, 0, 0, 0, 0, 0, 3053, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1756, 0, 0, 0, 4096, 0, 0, 3472, 0, 0, 0, 0, 0, 4096, 360, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 13, 0, 0, 0, 3736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16385, 16354, 15636, 16015, 17878, 16037, 16382] avg=16384.0000 max=17878 max_rank=5 min=15636 min_rank=3 imbalance_ratio=1.091187 max_load_reduction=113194 max_load_reduction_pct=0.863602 moved_to_echo=66312 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 4094, 0, 0, 0, 0, 4095, 0, 1, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4088, 4096, 4092, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2779, 0, 0, 0, 0, 32, 0, 0, 0, 2, 4096, 4090, 0, 1040, 4071, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 1, 30, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 4095, 3251, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 14, 4003, 0, 0, 0, 0, 0, 3053, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1936, 0, 0, 10, 4096, 0, 0, 2927, 0, 0, 0, 0, 0, 4095, 360, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 3734, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16384, 16111, 16155, 16099, 17523, 16032, 16381] avg=16384.0000 max=17523 max_rank=5 min=16032 min_rank=6 imbalance_ratio=1.069519 max_load_reduction=113549 max_load_reduction_pct=0.866310 moved_to_echo=66035 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4077, 4096, 4085, 0, 2, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3857, 0, 0, 0, 0, 26, 0, 0, 0, 0, 4096, 4096, 0, 1040, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 1360, 0, 0, 0, 0, 0, 0, 0, 4095, 3251, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4085, 0, 0, 0, 38, 3935, 0, 0, 0, 0, 0, 3053, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 971, 0, 0, 11, 4096, 0, 0, 2102, 0, 0, 0, 0, 0, 4093, 360, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 3736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16389, 17209, 16897, 16051, 15732, 16026, 16383] avg=16384.0000 max=17209 max_rank=2 min=15732 min_rank=5 imbalance_ratio=1.050354 max_load_reduction=113863 max_load_reduction_pct=0.868706 moved_to_echo=64192 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3618, 0, 0, 0, 0, 6, 7, 0, 0, 1, 4096, 4096, 0, 1040, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 460, 0, 0, 0, 0, 0, 0, 0, 4095, 3253, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 0, 4049, 0, 0, 0, 1542, 3935, 0, 0, 0, 0, 0, 3053, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 787, 0, 0, 0, 4094, 0, 0, 1722, 0, 0, 0, 0, 0, 4086, 360, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 259, 0, 0, 0, 3736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16381, 16953, 16002, 17538, 15148, 16283, 16382] avg=16384.0000 max=17538 max_rank=4 min=15148 min_rank=5 imbalance_ratio=1.070435 max_load_reduction=113534 max_load_reduction_pct=0.866196 moved_to_echo=65351 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4093, 0, 0, 0, 0, 4093, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1524, 0, 0, 0, 0, 34, 10, 0, 0, 0, 4096, 4093, 0, 1040, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 2, 0, 0, 1627, 0, 0, 0, 0, 0, 0, 0, 4095, 3253, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 296, 0, 4096, 0, 0, 0, 837, 3409, 0, 0, 6, 0, 0, 3053, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1487, 0, 0, 41, 4083, 0, 0, 2204, 0, 0, 0, 0, 0, 4095, 360, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 857, 0, 0, 0, 3736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16380, 14876, 17168, 16633, 16369, 16881, 16386] avg=16384.0000 max=17168 max_rank=3 min=14876 min_rank=2 imbalance_ratio=1.047852 max_load_reduction=113904 max_load_reduction_pct=0.869019 moved_to_echo=66269 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2719, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4096, 4096, 0, 1033, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 798, 0, 0, 0, 0, 0, 0, 0, 4096, 3253, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 4093, 0, 0, 0, 7, 4081, 0, 0, 2, 0, 0, 3054, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 367, 0, 0, 6, 4096, 0, 0, 3277, 2, 0, 0, 0, 0, 4096, 363, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 1015, 0, 0, 0, 3733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16383, 16052, 16337, 16186, 16312, 17036, 16382] avg=16384.0000 max=17036 max_rank=6 min=16052 min_rank=2 imbalance_ratio=1.039795 max_load_reduction=114036 max_load_reduction_pct=0.870026 moved_to_echo=65916 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4094, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4092, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 16, 2, 0, 0, 1, 4096, 4095, 0, 1040, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 1, 1, 0, 0, 975, 0, 0, 0, 0, 0, 0, 0, 4095, 3251, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 5, 0, 4083, 0, 0, 0, 35, 3958, 0, 0, 1, 0, 0, 3053, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 570, 0, 0, 0, 4096, 0, 2, 1900, 0, 0, 0, 0, 0, 4096, 360, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 764, 0, 0, 0, 3736, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16383, 17424, 16512, 16075, 15123, 16790, 16382] avg=16384.0000 max=17424 max_rank=2 min=15123 min_rank=5 imbalance_ratio=1.063477 max_load_reduction=113648 max_load_reduction_pct=0.867065 moved_to_echo=64370 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 4088, 6, 7, 4, 8, 4091, 0, 1, 0, 4086, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4091, 4081, 4090, 18, 2, 0, 0, 5, 0, 0, 6, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2775, 4, 0, 0, 0, 0, 0, 6, 9, 0, 4079, 4091, 0, 1039, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4033, 0, 0, 6, 0, 875, 6, 7, 0, 3, 8, 0, 1, 4089, 3244, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 3, 3, 77, 7, 4080, 1, 5, 4, 219, 4089, 5, 5, 0, 7, 1, 3049, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 2679, 1, 0, 0, 4092, 0, 0, 1677, 3, 7, 4, 0, 0, 4072, 359, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 4022, 4088, 1, 0, 0, 0, 4079, 18, 1, 3, 0, 3731, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 4, 8, 6, 0, 6, 3, 0, 4088, 0, 4088, 6, 4089, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16387, 16093, 16365, 16475, 16987, 15987, 16394] avg=16384.0000 max=16987 max_rank=5 min=15987 min_rank=6 imbalance_ratio=1.036804 max_load_reduction=114085 max_load_reduction_pct=0.870399 moved_to_echo=65843 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 32768, 128, 0, 0, 0, 32768, 2944, 128, 128, 0, 29952, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 128, 32768, 0, 128, 0, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 32768, 32768, 32768, 128, 128, 0, 128, 128, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[32768, 128, 128, 512, 128, 32768, 128, 128, 128, 2816, 31360, 128, 128, 128, 128, 128, 24448, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 0, 0, 32768, 128, 128, 128, 0, 6784, 128, 128, 0, 128, 128, 0, 128, 32768, 26112, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 10624, 128, 0, 128, 32768, 0, 128, 19328, 128, 128, 128, 0, 0, 32768, 2944, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 0, 128, 128, 128, 0, 128, 128, 0, 32768, 0, 32768, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[24448, 128, 0, 0, 0, 128, 128, 128, 128, 128, 32768, 32768, 0, 8320, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 430, 4096, 0, 569, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 0, 368, 4096, 4093, 0, 4095, 21, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 4096, 0, 2, 0, 2, 4096, 688, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2598, 0, 0, 0, 0, 0, 5, 3591, 4096, 4092, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4019] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 206, 4096, 0, 228, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4095, 0, 0, 662, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4090, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 1998, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3064, 0, 0, 0, 0, 0, 0, 3787, 4096, 4096, 0, 0, 0, 1, 4096, 4096, 0, 4096, 0, 0, 0, 4095, 0, 2363] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 349, 4096, 0, 1036, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 459, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 18, 4096, 608, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2569, 0, 0, 0, 0, 0, 0, 3967, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 3284] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4082, 0, 1, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4092, 0, 398, 4096, 0, 669, 0, 0, 0, 4092, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4080, 0, 4096, 0, 3, 980, 4096, 4095, 0, 4094, 5, 0, 0, 0, 0, 4096, 4096, 4096, 4094, 4093, 0, 0, 0, 4088, 0, 0, 1, 0, 4096, 4096, 0, 6, 0, 26, 4096, 3, 4095, 0, 10, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3726, 0, 0, 0, 0, 0, 0, 2444, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4062] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 55, 4096, 0, 1311, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 4096, 0, 9, 357, 4096, 4096, 0, 4093, 14, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 4085, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 4096, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3182, 0, 0, 0, 0, 0, 0, 3966, 4096, 4095, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 3389] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 4096, 0, 0, 0, 0, 0, 0, 4026, 0, 6, 4096, 2, 4094, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 92, 4096, 0, 44, 0, 0, 0, 4091, 0, 0, 6, 0, 7, 0, 0, 0, 0, 16, 1, 4096, 0, 4096, 0, 0, 987, 4096, 4091, 0, 3871, 1, 0, 0, 0, 0, 4096, 4096, 4095, 4096, 4096, 0, 0, 0, 4076, 0, 0, 0, 0, 4093, 4096, 0, 6, 0, 11, 4096, 5, 4096, 0, 14, 0, 1, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3298, 0, 0, 0, 0, 0, 11, 4039, 4096, 4088, 1, 0, 0, 0, 4095, 4088, 0, 4094, 0, 0, 0, 4096, 0, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[5, 0, 4086, 0, 6, 10, 8, 0, 6, 4041, 6, 0, 4090, 0, 4088, 4085, 5, 0, 6, 0, 7, 7, 0, 9, 5, 7, 0, 5, 4, 4, 7, 4034, 0, 246, 4089, 0, 1326, 7, 0, 0, 4090, 4, 20, 0, 7, 0, 10, 10, 0, 0, 0, 0, 4088, 5, 4087, 0, 0, 343, 4089, 4086, 0, 4089, 0, 8, 0, 0, 6, 4087, 4083, 4086, 4084, 4083, 0, 35, 0, 4092, 1, 3, 0, 0, 4089, 4077, 0, 0, 0, 65, 4089, 659, 4071, 0, 0, 0, 0, 3, 22, 5, 5, 9, 4, 18, 12, 4, 0, 0, 2282, 0, 5, 8, 9, 1, 0, 3735, 4090, 4074, 0, 8, 4, 3, 4090, 4091, 7, 4089, 1, 13, 0, 4079, 0, 3656] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 4096, 0, 4096, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4088, 0, 486, 4096, 0, 195, 0, 0, 0, 4069, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1861, 4096, 4083, 0, 4083, 0, 0, 0, 0, 0, 4096, 4096, 4093, 4096, 4096, 0, 0, 0, 4081, 0, 0, 7, 0, 4096, 4096, 0, 0, 0, 24, 4096, 758, 4096, 0, 9, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2545, 0, 0, 0, 0, 0, 17, 3791, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4094, 0, 0, 0, 4096, 0, 2639] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 16, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 430, 4096, 0, 569, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3433, 2466, 720, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 3376, 0, 0, 368, 0, 4093, 0, 4095, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4096, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2, 0, 2, 3482, 688, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2598, 0, 0, 0, 0, 0, 5, 3591, 4096, 4088, 1614, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 663, 0, 0, 0, 4096, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 16390, 16424, 16037, 16385, 16466, 15993, 16966] avg=16384.0000 max=16966 max_rank=7 min=15993 min_rank=6 imbalance_ratio=1.035522 max_load_reduction=114106 max_load_reduction_pct=0.870560 moved_to_echo=65810 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 16, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 4096, 0, 228, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3433, 2466, 720, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 3375, 0, 0, 662, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4096, 4090, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3482, 1998, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3064, 0, 0, 0, 0, 0, 0, 3787, 4096, 4088, 1614, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 663, 0, 0, 0, 4095, 0, 2363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16395, 16382, 15859, 16316, 16387, 17768, 16651, 15314] avg=16384.0000 max=17768 max_rank=5 min=15314 min_rank=7 imbalance_ratio=1.084473 max_load_reduction=113304 max_load_reduction_pct=0.864441 moved_to_echo=66120 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 16, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 4096, 0, 1036, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 3433, 2466, 720, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3376, 0, 0, 459, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4096, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 18, 3482, 608, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2569, 0, 0, 0, 0, 0, 0, 3967, 4096, 4088, 1614, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 663, 0, 0, 0, 4096, 0, 3284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16400, 16382, 16808, 16123, 16391, 16397, 16336, 16235] avg=16384.0000 max=16808 max_rank=2 min=16123 min_rank=3 imbalance_ratio=1.025879 max_load_reduction=114264 max_load_reduction_pct=0.871765 moved_to_echo=65359 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4082, 0, 1, 16, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4092, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 4096, 0, 669, 0, 0, 0, 4092, 0, 0, 0, 0, 1, 0, 0, 3433, 2466, 720, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 3376, 0, 3, 980, 0, 4095, 0, 4094, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4094, 4093, 0, 0, 0, 4088, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 6, 0, 26, 3482, 3, 4095, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3726, 0, 0, 0, 0, 0, 0, 2444, 4096, 4088, 1614, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 663, 0, 0, 0, 4096, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16385, 16489, 16633, 16380, 15814, 15971, 17013] avg=16384.0000 max=17013 max_rank=7 min=15814 min_rank=5 imbalance_ratio=1.038391 max_load_reduction=114059 max_load_reduction_pct=0.870201 moved_to_echo=65178 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 5, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 4096, 0, 1311, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3436, 2473, 724, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 3372, 0, 9, 357, 0, 4096, 0, 4093, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4096, 4096, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 3477, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3182, 0, 0, 0, 0, 0, 0, 3966, 4096, 4090, 1618, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 660, 0, 0, 0, 4096, 0, 3389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16391, 16810, 16037, 16379, 15778, 16953, 16336] avg=16384.0000 max=16953 max_rank=6 min=15778 min_rank=5 imbalance_ratio=1.034729 max_load_reduction=114119 max_load_reduction_pct=0.870659 moved_to_echo=65446 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4096, 0, 0, 0, 0, 0, 0, 4026, 0, 6, 16, 2, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 4096, 0, 44, 0, 0, 0, 4091, 0, 0, 6, 0, 7, 0, 0, 3431, 2466, 720, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 4096, 0, 3376, 0, 0, 987, 0, 4091, 0, 3871, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4095, 4096, 4096, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 6, 0, 11, 3482, 5, 4096, 0, 14, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3298, 0, 0, 0, 0, 0, 11, 4039, 4096, 4088, 1614, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 4088, 0, 663, 0, 0, 0, 4096, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16337, 16380, 15567, 16439, 16371, 15810, 17148, 17020] avg=16384.0000 max=17148 max_rank=6 min=15567 min_rank=2 imbalance_ratio=1.046631 max_load_reduction=113924 max_load_reduction_pct=0.869171 moved_to_echo=66349 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 16, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4088, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 4096, 0, 195, 0, 0, 0, 4069, 0, 0, 0, 0, 8, 0, 0, 3431, 2466, 720, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3376, 0, 0, 1861, 0, 4083, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4093, 4096, 4096, 0, 0, 0, 4081, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 24, 3482, 758, 4096, 0, 9, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2545, 0, 0, 0, 0, 0, 17, 3791, 4096, 4088, 1614, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 663, 0, 0, 0, 4096, 0, 2639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16398, 16395, 16085, 17499, 16381, 16572, 16152, 15590] avg=16384.0000 max=17499 max_rank=3 min=15590 min_rank=7 imbalance_ratio=1.068054 max_load_reduction=113573 max_load_reduction_pct=0.866493 moved_to_echo=64695 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 4086, 0, 6, 10, 8, 0, 6, 4041, 6, 0, 15, 0, 4088, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 0, 7, 7, 0, 9, 5, 7, 0, 5, 4, 4, 7, 4034, 4090, 4089, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 4089, 0, 1326, 7, 0, 0, 4090, 4, 20, 0, 7, 0, 10, 10, 3427, 2463, 718, 613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 5, 3369, 0, 0, 343, 0, 4086, 0, 4089, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8, 4086, 4084, 4083, 0, 35, 0, 4092, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4077, 0, 0, 0, 65, 3476, 659, 4071, 0, 0, 0, 0, 3, 22, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 9, 4, 18, 12, 4, 0, 0, 2282, 0, 5, 8, 9, 1, 0, 3735, 4087, 4075, 1612, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 8, 4, 3, 0, 4091, 7, 662, 1, 13, 0, 4079, 0, 3656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16356, 16367, 17030, 15988, 16398, 16467, 15868, 16598] avg=16384.0000 max=17030 max_rank=2 min=15868 min_rank=6 imbalance_ratio=1.039429 max_load_reduction=114042 max_load_reduction_pct=0.870071 moved_to_echo=65331 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 128, 0, 128, 32640, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 0, 32768, 128, 5376, 128, 128, 0, 32768, 0, 27520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 27008, 0, 128, 6144, 0, 32768, 0, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 32768, 0, 128, 0, 256, 27904, 4736, 32768, 0, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 0, 23296, 0, 128, 128, 128, 128, 128, 29440, 32768, 32768, 12928, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 2304, 32768, 0, 5504, 128, 0, 0, 32768, 128, 128, 128, 128, 128, 128, 128, 27520, 19840, 5888, 4992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 128, 32768, 32768, 32768, 128, 128, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 0, 269, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4, 0, 4096, 4092, 0, 4096, 0, 0, 4096, 0, 46, 0, 4096, 0, 4095, 0, 0, 4096, 0, 0, 2520, 0, 0, 0, 14, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2916, 0, 0, 2409, 0, 0, 0, 0, 0, 7, 4096, 0, 4084, 14, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 3, 0, 16, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 43, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 3984, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 0, 3525, 0, 0, 0, 0, 0, 17, 4096, 0, 4096, 247, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 7, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4034, 0, 0, 0, 0, 45, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 24, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 3, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 1352, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 0, 2836, 0, 0, 0, 0, 0, 29, 4096, 0, 4096, 11, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 25, 4096, 4096, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4072, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 4085, 5, 0, 21, 0, 4096, 4089, 0, 4096, 0, 0, 4090, 0, 12, 0, 4096, 2, 4096, 0, 0, 4092, 0, 1, 1718, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4095, 4096, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3075, 0, 0, 3106, 0, 0, 0, 0, 0, 13, 4096, 1, 4090, 97, 4096, 0, 4096, 4096, 4094, 4096, 0, 4094, 0, 0, 16, 4096, 4079, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 2, 0, 4096, 0, 0, 7, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2817, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 39, 0, 4096, 4096, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 3125, 0, 0, 0, 26, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3501, 0, 0, 2626, 0, 0, 0, 0, 0, 4, 4096, 0, 4095, 83, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 12, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[9, 6, 4005, 0, 0, 5, 0, 0, 7, 8, 4, 6, 7, 5, 0, 4090, 0, 0, 1, 0, 4088, 4089, 0, 4078, 4, 1, 4088, 8, 0, 0, 4087, 0, 4087, 12, 0, 4090, 5, 0, 3789, 0, 8, 0, 84, 0, 4088, 8, 0, 4083, 8, 4088, 4089, 4049, 6, 0, 0, 2, 0, 8, 0, 0, 5, 0, 0, 8, 9, 0, 7, 0, 6, 1005, 0, 0, 3450, 0, 0, 0, 0, 7, 1, 4089, 0, 4012, 0, 4086, 3, 4086, 4088, 4088, 4089, 7, 4089, 0, 0, 0, 4073, 4089, 5, 8, 7, 68, 0, 14, 0, 4087, 8, 1, 6, 0, 4080, 6, 4082, 0, 8, 4084, 15, 0, 4, 8, 4088, 0, 6, 4078, 0, 1, 0, 0, 11, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4081, 0, 0, 0, 0, 4, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 1, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 3474, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3259, 0, 0, 1459, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 5, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 4094, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4039, 0, 0, 0, 0, 500, 0, 0, 0, 0, 0, 0, 0, 4084, 2, 0, 0, 0, 4096, 4091, 0, 4096, 0, 0, 4088, 0, 0, 0, 4096, 4, 4095, 0, 0, 4090, 0, 0, 4009, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 905, 0, 0, 2662, 0, 0, 0, 0, 0, 14, 4096, 0, 4095, 154, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 1, 0, 25, 4096, 4081, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 7, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 0, 269, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 4096, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4092, 0, 4096, 0, 0, 4096, 0, 46, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 2520, 0, 0, 0, 14, 0, 1086, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3110, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2916, 0, 0, 2409, 0, 0, 0, 0, 0, 7, 4096, 4040, 3107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 14, 4031, 0, 0, 0, 0, 0, 0, 0, 3, 0, 16, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 3010, 989, 65, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 3107, 986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16698, 16412, 15907, 16390, 16575, 16337, 16370, 16383] avg=16384.0000 max=16698 max_rank=0 min=15907 min_rank=2 imbalance_ratio=1.019165 max_load_reduction=114374 max_load_reduction_pct=0.872604 moved_to_echo=65665 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4034, 0, 0, 0, 0, 45, 0, 0, 2, 0, 0, 0, 0, 4096, 4096, 4096, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 3, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1352, 0, 0, 0, 0, 0, 1086, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3109, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 0, 2836, 0, 0, 0, 0, 0, 29, 4096, 4041, 3108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 3010, 989, 65, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3107, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16424, 16393, 14726, 16385, 18031, 16355, 16376, 16382] avg=16384.0000 max=18031 max_rank=4 min=14726 min_rank=2 imbalance_ratio=1.100525 max_load_reduction=113041 max_load_reduction_pct=0.862434 moved_to_echo=67144 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4081, 0, 0, 0, 0, 4, 0, 0, 2, 0, 0, 0, 0, 4096, 4096, 4096, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 1, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3474, 0, 0, 0, 0, 0, 1086, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3109, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3259, 0, 0, 1459, 0, 0, 0, 0, 0, 0, 4096, 4041, 3107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 3010, 989, 65, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 3107, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16430, 16367, 16848, 16386, 15962, 16321, 16374, 16384] avg=16384.0000 max=16848 max_rank=2 min=15962 min_rank=4 imbalance_ratio=1.028320 max_load_reduction=114224 max_load_reduction_pct=0.871460 moved_to_echo=65041 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3984, 0, 0, 0, 1, 0, 1086, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3110, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 0, 3525, 0, 0, 0, 0, 0, 17, 4096, 4040, 3107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 247, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 3010, 989, 65, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3107, 986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16452, 16409, 17359, 16387, 15160, 16554, 16370, 16381] avg=16384.0000 max=17359 max_rank=2 min=15160 min_rank=4 imbalance_ratio=1.059509 max_load_reduction=113713 max_load_reduction_pct=0.867561 moved_to_echo=64465 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4072, 0, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 4096, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 21, 0, 0, 4089, 0, 4096, 0, 0, 4090, 0, 12, 0, 4078, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 1, 1718, 0, 0, 0, 0, 0, 1086, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3109, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3075, 0, 0, 3106, 0, 0, 0, 0, 0, 13, 4096, 4041, 3105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 97, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 3010, 988, 65, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 2, 0, 4096, 0, 0, 7, 0, 0, 0, 3106, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16592, 16393, 15089, 16390, 17437, 16410, 16372, 16389] avg=16384.0000 max=17437 max_rank=4 min=15089 min_rank=2 imbalance_ratio=1.064270 max_load_reduction=113635 max_load_reduction_pct=0.866966 moved_to_echo=66608 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2817, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 4096, 0, 4096, 0, 0, 4095, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3125, 0, 0, 0, 26, 0, 1079, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3110, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3501, 0, 0, 2626, 0, 0, 0, 0, 0, 4, 4096, 4040, 3107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 83, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 3017, 990, 68, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3106, 986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15221, 16401, 16518, 16387, 17374, 16407, 16384, 16380] avg=16384.0000 max=17374 max_rank=4 min=15221 min_rank=0 imbalance_ratio=1.060425 max_load_reduction=113698 max_load_reduction_pct=0.867447 moved_to_echo=66545 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[9, 6, 4005, 0, 0, 5, 0, 0, 7, 8, 4, 6, 7, 5, 0, 4090, 4089, 4088, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 0, 4078, 4, 1, 4088, 8, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 12, 0, 4090, 5, 0, 3789, 0, 8, 0, 84, 0, 1083, 8, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4088, 4089, 4049, 6, 0, 0, 2, 0, 8, 0, 0, 5, 0, 0, 8, 3101, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 7, 0, 6, 1005, 0, 0, 3450, 0, 0, 0, 0, 7, 1, 4089, 4033, 3102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 4021, 3, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4073, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8, 7, 68, 0, 14, 0, 4087, 8, 1, 6, 0, 4080, 6, 4082, 0, 3005, 987, 65, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4084, 15, 0, 4, 8, 4088, 0, 6, 4078, 0, 1, 0, 0, 11, 0, 3101, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16338, 17249, 16351, 15709, 16205, 16447, 16389] avg=16384.0000 max=17249 max_rank=2 min=15709 min_rank=4 imbalance_ratio=1.052795 max_load_reduction=113823 max_load_reduction_pct=0.868401 moved_to_echo=64750 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4039, 0, 0, 0, 0, 500, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 4096, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 4088, 0, 0, 0, 4078, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4090, 0, 0, 4009, 0, 0, 0, 0, 0, 1086, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3110, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 905, 0, 0, 2662, 0, 0, 0, 0, 0, 14, 4096, 4040, 3107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 154, 4031, 0, 0, 0, 0, 0, 0, 0, 1, 0, 25, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 7, 0, 0, 4096, 0, 4096, 0, 3010, 989, 65, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 3107, 986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16871, 16359, 17376, 16396, 14824, 16483, 16379, 16384] avg=16384.0000 max=17376 max_rank=2 min=14824 min_rank=4 imbalance_ratio=1.060547 max_load_reduction=113696 max_load_reduction_pct=0.867432 moved_to_echo=64070 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 32768, 128, 128, 32768, 0, 128, 128, 0, 128, 0, 24960, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 256, 0, 0, 32768, 0, 32768, 128, 128, 32768, 128, 128, 0, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 32768, 640, 32256, 128, 0, 0, 0, 0, 128, 0, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 31232, 0, 0, 128, 0, 1152, 128, 128, 128, 128, 128, 128, 0, 32768, 32768, 32768, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 32768, 32768, 128, 128, 0, 128, 128, 128, 0, 0, 128, 0, 0, 128, 24960, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 0, 32768, 128, 128, 24064, 0, 128, 0, 128, 0, 8704, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 128, 19072, 0, 0, 22144, 0, 0, 0, 0, 128, 128, 32768, 32384, 24960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 32768, 128, 128, 128, 0, 32768, 128, 32768, 0, 24192, 7936, 640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 2889, 0, 4096, 0, 0, 0, 24, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4087, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 753, 4084, 4096, 0, 0, 0, 3999, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 534, 5, 4096, 0, 4096, 7, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 4096, 0, 0, 3225, 0, 4096, 0, 0, 0, 23, 4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 4093, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 14, 4096, 4096, 0, 0, 0, 3927, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1023, 0, 4096, 0, 4096, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 3200, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4084, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 851, 4094, 4096, 0, 0, 0, 4067, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 75, 0, 4096, 0, 4096, 14, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 3342, 0, 4096, 0, 0, 0, 5, 4096, 0, 4096, 0, 0, 0, 0, 4093, 0, 4096, 0, 4093, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 474, 4096, 4096, 0, 0, 0, 4077, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 300, 0, 4096, 0, 4096, 3, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4085, 0, 8, 0, 4084, 0, 8, 2356, 0, 4088, 0, 0, 1, 209, 4088, 7, 4089, 0, 4, 0, 7, 3659, 0, 4088, 6, 4076, 5, 6, 4088, 2, 4070, 4087, 10, 4081, 10, 0, 0, 7, 8, 4089, 7, 4082, 4079, 0, 0, 0, 0, 9, 0, 9, 14, 4085, 0, 0, 0, 0, 3, 4, 0, 0, 118, 4084, 0, 0, 9, 6, 8, 28, 0, 0, 8, 4, 4083, 4087, 6, 8, 19, 4081, 0, 0, 0, 0, 7, 4082, 5, 1, 0, 8, 0, 0, 5, 4087, 6, 11, 0, 8, 4087, 645, 4090, 4083, 0, 0, 10, 3157, 8, 0, 4058, 6, 0, 6, 0, 2, 0, 0, 4089, 4090, 0, 7, 6, 2148, 4, 4087, 0, 4088, 0, 8, 6, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 3119, 0, 4096, 0, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 4032, 0, 4096, 0, 4075, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 18, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 843, 4095, 4096, 0, 0, 0, 4079, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 201, 0, 4096, 0, 4096, 31, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 3, 0, 0, 4096, 0, 0, 1825, 0, 4096, 2, 0, 0, 1, 4095, 0, 4096, 5, 0, 0, 0, 4093, 0, 4096, 0, 4093, 0, 0, 4093, 0, 4096, 4096, 0, 4096, 0, 0, 3, 0, 0, 4093, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 19, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 22, 4085, 4096, 0, 0, 0, 3321, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 2, 3014, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4096, 0, 0, 2850, 0, 4096, 0, 1, 0, 5, 4096, 0, 4096, 8, 0, 0, 0, 4092, 0, 4096, 0, 4050, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 29, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1083, 4073, 4096, 0, 0, 0, 3993, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 268, 0, 4096, 0, 4096, 26, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 2889, 0, 1213, 0, 0, 0, 24, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 4096, 0, 4096, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 4096, 4072, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4018, 48, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2835, 956, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 753, 4084, 3789, 0, 0, 0, 3999, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3140, 4095, 0, 0, 0, 534, 5, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16414, 16453, 16375, 16378, 16376, 16382, 16721, 15973] avg=16384.0000 max=16721 max_rank=6 min=15973 min_rank=7 imbalance_ratio=1.020569 max_load_reduction=114351 max_load_reduction_pct=0.872429 moved_to_echo=65452 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 3342, 0, 1213, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 4096, 0, 4096, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4092, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4018, 48, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2835, 956, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 4096, 3789, 0, 0, 0, 4077, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3140, 4096, 0, 0, 0, 300, 0, 4096, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16848, 16456, 16396, 16357, 16370, 16382, 16532, 15731] avg=16384.0000 max=16848 max_rank=0 min=15731 min_rank=7 imbalance_ratio=1.028320 max_load_reduction=114224 max_load_reduction_pct=0.871460 moved_to_echo=65015 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 4096, 0, 0, 3225, 0, 1213, 0, 0, 0, 23, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4093, 0, 0, 4096, 0, 4096, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4062, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4018, 48, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2835, 956, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 3789, 0, 0, 0, 3927, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3140, 4096, 0, 0, 0, 1023, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16747, 16458, 16366, 16374, 16370, 16383, 15922, 16452] avg=16384.0000 max=16747 max_rank=0 min=15922 min_rank=6 imbalance_ratio=1.022156 max_load_reduction=114325 max_load_reduction_pct=0.872231 moved_to_echo=65127 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 3200, 0, 1213, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4084, 0, 0, 4096, 0, 4096, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4083, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4018, 48, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2835, 956, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 851, 4094, 3789, 0, 0, 0, 4067, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3140, 4096, 0, 0, 0, 75, 0, 4096, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16701, 16450, 16387, 16358, 16380, 16382, 16897, 15517] avg=16384.0000 max=16897 max_rank=6 min=15517 min_rank=7 imbalance_ratio=1.031311 max_load_reduction=114175 max_load_reduction_pct=0.871086 moved_to_echo=65176 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 3, 0, 0, 4096, 0, 0, 1825, 0, 1213, 2, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4093, 0, 0, 0, 4093, 0, 0, 4093, 0, 4096, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 4093, 0, 4096, 4085, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 19, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4018, 48, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2835, 956, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4085, 3789, 0, 0, 0, 3321, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3140, 4094, 0, 0, 2, 3014, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15331, 16458, 16389, 16376, 16381, 16383, 15312, 18442] avg=16384.0000 max=18442 max_rank=7 min=15312 min_rank=6 imbalance_ratio=1.125610 max_load_reduction=112630 max_load_reduction_pct=0.859299 moved_to_echo=66518 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 3119, 0, 1209, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 4075, 0, 0, 4096, 0, 4096, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4071, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 18, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4019, 49, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2838, 959, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 843, 4095, 3782, 0, 0, 0, 4079, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3137, 4096, 0, 0, 0, 201, 0, 4096, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16618, 16376, 16376, 16376, 16392, 16382, 16895, 15657] avg=16384.0000 max=16895 max_rank=6 min=15657 min_rank=7 imbalance_ratio=1.031189 max_load_reduction=114177 max_load_reduction_pct=0.871101 moved_to_echo=65326 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4085, 0, 8, 0, 4084, 0, 8, 2356, 0, 1211, 0, 0, 1, 209, 4088, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7, 3659, 0, 0, 6, 4076, 5, 6, 4088, 2, 4070, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4081, 10, 0, 0, 7, 8, 4089, 7, 4082, 4079, 0, 0, 0, 0, 9, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 14, 4085, 0, 0, 0, 0, 3, 4, 0, 0, 118, 4084, 0, 0, 4087, 4009, 48, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 6, 8, 28, 0, 0, 8, 4, 4083, 4087, 6, 8, 19, 4081, 0, 0, 2829, 954, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4082, 5, 1, 0, 8, 0, 0, 5, 4087, 6, 11, 0, 8, 4089, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 645, 4090, 3777, 0, 0, 10, 3157, 8, 0, 4058, 6, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3135, 4090, 0, 7, 6, 2148, 4, 4087, 0, 4088, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16057, 16001, 16398, 16462, 16420, 16396, 15759, 17579] avg=16384.0000 max=17579 max_rank=7 min=15759 min_rank=6 imbalance_ratio=1.072937 max_load_reduction=113493 max_load_reduction_pct=0.865883 moved_to_echo=66154 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4096, 0, 0, 2850, 0, 1213, 0, 1, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4092, 0, 0, 0, 4050, 0, 0, 4096, 0, 4096, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 4096, 4083, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 29, 0, 0, 0, 0, 5, 4096, 0, 0, 4096, 4018, 48, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2835, 956, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 4073, 3789, 0, 0, 0, 3993, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3140, 4095, 0, 0, 0, 268, 0, 4096, 0, 4096, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16356, 16420, 16385, 16391, 16383, 16382, 17034, 15721] avg=16384.0000 max=17034 max_rank=6 min=15721 min_rank=7 imbalance_ratio=1.039673 max_load_reduction=114038 max_load_reduction_pct=0.870041 moved_to_echo=65520 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 0, 128, 0, 0, 128, 32768, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 32768, 128, 0, 0, 128, 128, 128, 0, 0, 128, 32768, 0, 0, 32768, 32256, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 0, 32768, 0, 128, 22912, 0, 9728, 128, 128, 128, 384, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 4736, 32768, 30336, 0, 0, 128, 30720, 128, 0, 32768, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 128, 32256, 0, 0, 128, 32768, 128, 128, 32768, 128, 32768, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 25216, 32768, 0, 128, 128, 7680, 128, 32768, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 128, 32768, 128, 32768, 32640, 0, 0, 0, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 128, 32768, 32768, 128, 128, 128, 32768, 0, 0, 22784, 7680, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4093, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4071, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4094, 5, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4095, 0, 4096, 4096, 0, 4094, 0, 0, 0, 0, 0, 4096, 25, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4023, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 73, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4061, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 35, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4033, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 4075, 22, 4096, 0, 4, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4091, 0, 4096, 4096, 4, 4095, 0, 0, 2, 0, 0, 4096, 62, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 4096, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4077, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 5, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4082, 0, 0, 0, 0, 0, 4096, 28, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1, 4082, 9, 4086, 8, 0, 0, 4083, 10, 0, 0, 0, 0, 6, 7, 0, 0, 0, 4087, 5, 7, 0, 0, 4088, 4085, 9, 8, 0, 4089, 1, 4080, 0, 0, 4, 6, 4, 11, 4087, 0, 4089, 4088, 0, 0, 7, 0, 0, 6, 9, 8, 11, 8, 0, 4053, 0, 4, 4087, 4089, 9, 0, 0, 0, 4, 7, 0, 0, 4086, 1, 6, 4086, 4084, 4088, 0, 8, 29, 4085, 11, 4087, 4085, 4087, 7, 0, 4038, 6, 2, 11, 4089, 0, 4085, 4071, 0, 4088, 0, 0, 0, 0, 9, 4083, 0, 8, 10, 4, 4088, 5, 0, 8, 5, 0, 0, 4081, 4090, 0, 4084, 7, 0, 0, 3, 0, 0, 0, 15, 0, 60, 0, 0, 9, 5, 8, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4031, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4092, 24, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 1, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 4096, 0, 4075, 0, 0, 0, 0, 0, 4096, 62, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 4096, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4054, 25, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4071, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 15, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4071, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16416, 16373, 16380, 16386, 16382, 16399, 16362, 16374] avg=16384.0000 max=16416 max_rank=0 min=16362 min_rank=6 imbalance_ratio=1.001953 max_load_reduction=114656 max_load_reduction_pct=0.874756 moved_to_echo=65517 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4071, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 0, 0, 0, 4094, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4054, 25, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4071, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 15, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4071, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16394, 16376, 16380, 16386, 16382, 16396, 16384, 16374] avg=16384.0000 max=16396 max_rank=5 min=16374 min_rank=7 imbalance_ratio=1.000732 max_load_reduction=114676 max_load_reduction_pct=0.874908 moved_to_echo=65536 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4023, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 0, 0, 0, 4096, 73, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4054, 25, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4071, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 15, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4071, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16346, 16446, 16380, 16386, 16382, 16399, 16359, 16374] avg=16384.0000 max=16446 max_rank=1 min=16346 min_rank=0 imbalance_ratio=1.003784 max_load_reduction=114626 max_load_reduction_pct=0.874527 moved_to_echo=65514 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4061, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4054, 25, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4071, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 15, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4071, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16373, 16380, 16386, 16382, 16399, 16394, 16374] avg=16384.0000 max=16399 max_rank=5 min=16373 min_rank=1 imbalance_ratio=1.000916 max_load_reduction=114673 max_load_reduction_pct=0.874886 moved_to_echo=65549 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4033, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 25, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 0, 0, 0, 4075, 22, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 4054, 25, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4071, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 15, 4096, 4, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4068, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16355, 16373, 16382, 16387, 16382, 16399, 16419, 16375] avg=16384.0000 max=16419 max_rank=6 min=16355 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=114653 max_load_reduction_pct=0.874733 moved_to_echo=65575 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4031, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 26, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 0, 0, 0, 4092, 24, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4054, 25, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4070, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 15, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4070, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16360, 16393, 16380, 16385, 16383, 16376, 16421, 16374] avg=16384.0000 max=16421 max_rank=6 min=16360 min_rank=0 imbalance_ratio=1.002258 max_load_reduction=114651 max_load_reduction_pct=0.874718 moved_to_echo=65554 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4077, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 26, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 4096, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4061, 32, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4070, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 5, 4096, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4064, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16402, 16376, 16399, 16388, 16379, 16375, 16380, 16373] avg=16384.0000 max=16402 max_rank=0 min=16373 min_rank=7 imbalance_ratio=1.001099 max_load_reduction=114670 max_load_reduction_pct=0.874863 moved_to_echo=65507 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 4082, 9, 4086, 8, 0, 0, 4083, 10, 0, 0, 0, 0, 6, 7, 0, 4088, 25, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7, 0, 0, 4078, 4085, 9, 8, 0, 4089, 1, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 4, 11, 4087, 0, 4089, 4088, 0, 0, 7, 0, 0, 6, 9, 4043, 25, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 11, 8, 0, 4053, 0, 4, 4087, 4089, 9, 0, 0, 0, 4, 7, 0, 4061, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1, 6, 0, 4084, 0, 0, 8, 29, 4085, 11, 0, 4085, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 6, 2, 11, 4089, 0, 15, 4071, 0, 4088, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 8, 10, 4, 0, 5, 0, 8, 5, 0, 0, 4081, 4065, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 0, 0, 0, 15, 0, 60, 0, 0, 9, 5, 8, 8, 4088, 4087, 4087, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16415, 16362, 16391, 16368, 16400, 16329, 16353, 16454] avg=16384.0000 max=16454 max_rank=7 min=16329 min_rank=5 imbalance_ratio=1.004272 max_load_reduction=114618 max_load_reduction_pct=0.874466 moved_to_echo=65536 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 256, 128, 128, 128, 0, 128, 128, 128, 128, 128, 0, 32768, 32640, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 128, 128, 0, 32768, 0, 0, 128, 128, 32768, 128, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 128, 128, 128, 32768, 0, 128, 32768, 128, 32768, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32512, 128, 32768, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 128, 128, 128, 32768, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 128, 32768, 32768, 128, 0, 0, 0, 128, 128, 0, 32640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 0, 32768, 32768, 0, 0, 128, 0, 0, 128, 128, 32512, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 32768, 32768, 128, 128, 0, 32768, 256, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 0, 0, 0, 128, 0, 128, 0, 128, 128, 128, 128, 128, 32768, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4010, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 65, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 6, 0, 4, 4096, 4091, 0, 4093, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3401, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 417, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 278, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3317, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 686, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 91, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 1, 0, 0, 4096, 4095, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4089, 0, 0, 0, 0, 3, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3930, 4096, 0, 4, 0, 0, 4096, 0, 0, 0, 77, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4091, 2, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4091, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 80, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 21, 0, 9, 4096, 4093, 0, 4087, 0, 4096, 0, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3629, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 427, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 38, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[10, 0, 7, 4, 0, 0, 5, 0, 4086, 4087, 0, 4090, 0, 0, 9, 0, 0, 4084, 4064, 1, 4088, 10, 4085, 4085, 8, 0, 5, 0, 4084, 1, 4087, 0, 9, 4090, 4086, 0, 0, 8, 0, 0, 28, 0, 9, 7, 3, 0, 0, 3924, 4088, 8, 0, 7, 0, 4086, 7, 0, 5, 176, 10, 8, 0, 27, 4082, 0, 4086, 0, 0, 0, 7, 4035, 3, 4088, 0, 0, 0, 0, 0, 9, 10, 8, 7, 4082, 4083, 8, 16, 6, 4083, 11, 4073, 4, 3, 4087, 12, 7, 0, 12, 0, 0, 11, 0, 4086, 69, 0, 6, 6, 4082, 0, 7, 0, 0, 8, 4029, 0, 9, 0, 4088, 0, 0, 3, 4086, 4088, 0, 4088, 7, 4086, 5, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3479, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 612, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4091, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3765, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 34, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 297, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 12, 4, 8, 4096, 4091, 0, 4083, 0, 4096, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3681, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3401, 4096, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 417, 0, 0, 0, 0, 4096, 0, 3780, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3689, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3680, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16390, 16096, 16489, 16392, 16382, 16556, 16382] avg=16384.0000 max=16556 max_rank=6 min=16096 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=114516 max_load_reduction_pct=0.873688 moved_to_echo=65712 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3681, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4010, 4096, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 0, 0, 0, 0, 4096, 0, 3780, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3689, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3680, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 4, 0, 4091, 0, 4093, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16390, 16707, 16093, 16388, 16382, 16343, 16384] avg=16384.0000 max=16707 max_rank=2 min=16093 min_rank=3 imbalance_ratio=1.019714 max_load_reduction=114365 max_load_reduction_pct=0.872536 moved_to_echo=65497 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3681, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3317, 4096, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 686, 0, 0, 0, 0, 4096, 0, 3780, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3689, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4094, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 3680, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4095, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16390, 16016, 16758, 16392, 16381, 16368, 16382] avg=16384.0000 max=16758 max_rank=3 min=16016 min_rank=2 imbalance_ratio=1.022827 max_load_reduction=114314 max_load_reduction_pct=0.872147 moved_to_echo=65523 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4089, 0, 0, 0, 0, 3681, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3930, 4096, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 4096, 0, 0, 0, 77, 0, 0, 0, 0, 4096, 0, 3780, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4091, 2, 0, 0, 1, 0, 0, 0, 0, 3688, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4094, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4085, 3678, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 9, 0, 4093, 0, 4087, 0, 4094, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 16393, 16632, 16153, 16389, 16377, 16345, 16405] avg=16384.0000 max=16632 max_rank=2 min=16153 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=114440 max_load_reduction_pct=0.873108 moved_to_echo=65516 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3681, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3479, 4096, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 612, 0, 0, 0, 0, 4096, 0, 3780, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3689, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3680, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16390, 16174, 16684, 16391, 16382, 16283, 16383] avg=16384.0000 max=16684 max_rank=3 min=16174 min_rank=2 imbalance_ratio=1.018311 max_load_reduction=114388 max_load_reduction_pct=0.872711 moved_to_echo=65439 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3682, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3629, 4096, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 427, 0, 0, 0, 0, 4096, 0, 3785, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3689, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3679, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 4096, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16384, 16324, 16511, 16390, 16381, 16316, 16379] avg=16384.0000 max=16511 max_rank=3 min=16316 min_rank=6 imbalance_ratio=1.007751 max_load_reduction=114561 max_load_reduction_pct=0.874031 moved_to_echo=65466 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4091, 0, 0, 0, 0, 3682, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3765, 4096, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 34, 0, 0, 0, 0, 4096, 0, 3780, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3689, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3678, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 12, 4, 8, 0, 4091, 0, 4083, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16387, 16466, 16106, 16388, 16382, 16576, 16385] avg=16384.0000 max=16576 max_rank=6 min=16106 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=114496 max_load_reduction_pct=0.873535 moved_to_echo=65731 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 7, 4, 0, 0, 5, 0, 4086, 4087, 0, 4090, 0, 0, 9, 0, 3672, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4064, 1, 0, 10, 4085, 6, 8, 0, 5, 0, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4090, 4086, 0, 0, 8, 0, 0, 28, 0, 9, 7, 3, 0, 0, 3924, 4088, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 8, 0, 7, 0, 4086, 7, 0, 5, 176, 10, 8, 0, 27, 4082, 0, 3770, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 7, 4035, 3, 4088, 0, 0, 0, 0, 0, 9, 10, 8, 3682, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4082, 4083, 8, 16, 6, 4081, 11, 4073, 4, 3, 0, 12, 7, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4086, 69, 0, 6, 6, 4082, 0, 7, 0, 0, 8, 4029, 3672, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4088, 0, 0, 3, 0, 4088, 0, 4088, 7, 4084, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16348, 16657, 16278, 16342, 16405, 16285, 16372] avg=16384.0000 max=16657 max_rank=2 min=16278 min_rank=3 imbalance_ratio=1.016663 max_load_reduction=114415 max_load_reduction_pct=0.872917 moved_to_echo=65404 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 128, 0, 32768, 128, 0, 128, 2560, 128, 128, 0, 128, 32768, 0, 30336, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 0, 0, 128, 0, 32768, 32768, 0, 32768, 0, 0, 128, 0, 29568, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 896, 32768, 128, 0, 128, 128, 32768, 0, 128, 0, 0, 128, 32768, 29440, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 0, 0, 128, 32768, 128, 32768, 128, 0, 0, 128, 0, 128, 128, 128, 29568, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 32768, 0, 0, 128, 0, 0, 128, 0, 128, 128, 128, 0, 128, 29568, 32768, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 32768, 128, 32768, 128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 32768, 128, 128, 128, 0, 32768, 0, 32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32768, 32768, 128, 0, 128, 32768, 128, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 4058, 0, 4096, 4096, 22, 0, 4096, 4096, 2185, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 88, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 1937, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4048, 0, 4096, 4096, 24, 0, 4096, 4096, 1155, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 7, 67, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 4096, 0, 0, 0, 0, 63, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 2889, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 4096, 1475, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 359, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 54, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 2279, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4086, 4, 0, 4088, 4067, 0, 0, 7, 0, 3, 0, 4081, 0, 0, 0, 9, 10, 0, 4087, 0, 4086, 4084, 0, 0, 4071, 4088, 536, 0, 0, 4085, 4084, 8, 7, 9, 4084, 2, 12, 8, 0, 0, 0, 4054, 0, 8, 4082, 4088, 0, 0, 0, 11, 9, 9, 9, 12, 0, 4, 0, 4084, 0, 9, 10, 0, 4084, 6, 4086, 4085, 9, 0, 17, 8, 0, 2, 0, 156, 4088, 11, 0, 0, 4087, 4085, 1, 0, 4085, 5, 0, 9, 0, 10, 10, 9, 4087, 11, 9, 4088, 0, 0, 7, 7, 184, 0, 14, 4079, 0, 4083, 0, 4068, 4, 5, 3242, 7, 0, 4078, 0, 8, 9, 0, 0, 4086, 12, 4087, 8, 5, 24, 12, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 4096, 12, 0, 4096, 4094, 538, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3, 50, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 4057, 0, 0, 4096, 0, 6, 0, 0, 91, 0, 0, 4096, 0, 4093, 0, 4096, 0, 0, 3460, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 5, 4094, 0, 4096, 4096, 16, 0, 4096, 4090, 1164, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 7, 472, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 4068, 0, 0, 4096, 0, 0, 0, 0, 33, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 2470, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 8, 0, 4096, 4096, 1116, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 176, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 18, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 2801, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 3069, 24, 0, 4096, 4096, 1155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 4096, 4096, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 7, 67, 3865, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 2889, 0, 0, 1027, 161, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16488, 16392, 16381, 16227, 16379, 16436, 16379] avg=16384.0000 max=16488 max_rank=1 min=16227 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=114584 max_load_reduction_pct=0.874207 moved_to_echo=65421 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3069, 0, 0, 4096, 4096, 1475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 4096, 4096, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 359, 3865, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 2279, 0, 0, 1027, 161, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 16832, 16359, 16380, 16512, 16396, 15816, 16380] avg=16384.0000 max=16832 max_rank=1 min=15816 min_rank=6 imbalance_ratio=1.027344 max_load_reduction=114240 max_load_reduction_pct=0.871582 moved_to_echo=65104 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 3068, 22, 0, 4096, 4096, 2185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 4096, 4096, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 88, 3848, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 4096, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 1937, 0, 0, 1028, 164, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 17525, 16391, 16381, 16224, 16363, 15426, 16379] avg=16384.0000 max=17525 max_rank=1 min=15426 min_rank=6 imbalance_ratio=1.069641 max_load_reduction=113547 max_load_reduction_pct=0.866295 moved_to_echo=64392 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3069, 12, 0, 4096, 4094, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 4096, 4093, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3, 50, 3865, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 4057, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 91, 0, 0, 4096, 0, 4093, 0, 4096, 0, 0, 3460, 0, 0, 1027, 161, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 15902, 16427, 16380, 16206, 16362, 17038, 16379] avg=16384.0000 max=17038 max_rank=6 min=15902 min_rank=1 imbalance_ratio=1.039917 max_load_reduction=114034 max_load_reduction_pct=0.870010 moved_to_echo=65985 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4094, 0, 0, 3069, 16, 0, 4096, 4090, 1164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 4096, 4092, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 7, 472, 3865, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 4068, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 2470, 0, 0, 1027, 161, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16534, 16402, 16381, 16632, 16375, 15987, 16379] avg=16384.0000 max=16632 max_rank=4 min=15987 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=114440 max_load_reduction_pct=0.873108 moved_to_echo=65373 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4086, 4, 0, 4088, 4067, 0, 0, 7, 0, 3, 0, 4081, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 10, 0, 4087, 0, 0, 3061, 0, 0, 4071, 4088, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 7, 9, 4084, 2, 12, 8, 0, 0, 0, 4054, 0, 8, 4082, 4088, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 9, 9, 9, 12, 0, 4, 0, 4084, 0, 9, 10, 0, 4084, 4086, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4086, 4085, 9, 0, 17, 8, 0, 2, 0, 156, 3857, 11, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 4085, 5, 0, 9, 0, 10, 10, 9, 4087, 11, 9, 4088, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 184, 0, 14, 4079, 0, 4083, 0, 4068, 4, 5, 3242, 7, 0, 1023, 161, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 8, 9, 0, 0, 4086, 12, 4087, 8, 5, 24, 12, 0, 0, 0, 4077, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16349, 15862, 16407, 16408, 16324, 16421, 16891, 16410] avg=16384.0000 max=16891 max_rank=6 min=15862 min_rank=1 imbalance_ratio=1.030945 max_load_reduction=114181 max_load_reduction_pct=0.871132 moved_to_echo=66046 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3069, 8, 0, 4096, 4096, 1116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 4096, 4096, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 176, 3865, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 2801, 0, 0, 1027, 161, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 16481, 16413, 16381, 16329, 16389, 16303, 16379] avg=16384.0000 max=16481 max_rank=1 min=16303 min_rank=6 imbalance_ratio=1.005920 max_load_reduction=114591 max_load_reduction_pct=0.874260 moved_to_echo=65400 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 4035, 0, 4096, 4096, 20, 0, 4096, 4096, 131, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3948, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 465, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 0, 0, 0, 92, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 3601, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4035, 0, 0, 3069, 20, 0, 4096, 4096, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3948, 0, 0, 4096, 4096, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 465, 3865, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 3601, 0, 0, 1027, 161, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16396, 15448, 16281, 16380, 16618, 16387, 17175, 16387] avg=16384.0000 max=17175 max_rank=6 min=15448 min_rank=1 imbalance_ratio=1.048279 max_load_reduction=113897 max_load_reduction_pct=0.868965 moved_to_echo=66567 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 32768, 128, 0, 128, 128, 0, 128, 128, 1920, 30976, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 0, 32768, 128, 128, 128, 0, 128, 128, 128, 32640, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 0, 32768, 32768, 0, 0, 128, 0, 128, 0, 32768, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 32768, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 32512, 128, 128, 32768, 32768, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 640, 0, 128, 32768, 0, 32768, 0, 32768, 128, 128, 22784, 128, 0, 8320, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32640, 0, 0, 24576, 128, 0, 32768, 32768, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 0, 128, 0, 32768, 0, 128, 128, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 2, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 8, 0, 0, 4095, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4071, 0, 0, 4096, 0, 25, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 17, 4096, 0, 0, 0, 4096, 0, 0, 4090, 4096, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4031, 0, 0, 4096, 0, 48, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1, 4096, 0, 0, 0, 4096, 0, 0, 4094, 4096, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4086, 0, 0, 4096, 0, 8, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 8, 4096, 0, 0, 0, 4096, 0, 0, 4090, 4096, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4057, 0, 0, 4096, 0, 27, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 66, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 0, 4096, 0, 0, 0, 4096, 0, 0, 4089, 4096, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4039, 0, 0, 4096, 0, 12, 0, 4096, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4082, 6, 4085, 0, 4085, 3944, 0, 8, 0, 9, 0, 0, 0, 4073, 0, 0, 4085, 3, 0, 0, 11, 0, 5, 10, 7, 0, 4086, 12, 0, 0, 0, 0, 0, 10, 7, 9, 3, 0, 9, 9, 8, 8, 4086, 1, 10, 7, 4077, 0, 11, 0, 9, 4083, 2, 3646, 498, 4080, 0, 0, 40, 4086, 0, 8, 3984, 4081, 10, 0, 11, 11, 139, 0, 10, 4087, 0, 4087, 5, 0, 4086, 4086, 7, 25, 16, 9, 7, 4087, 16, 8, 4085, 21, 4071, 0, 10, 4078, 14, 0, 6, 4082, 0, 0, 0, 0, 4077, 0, 4079, 4, 0, 16, 5, 4090, 4084, 0, 0, 0, 9, 0, 11, 4086, 0, 4084, 0, 0, 2, 0, 4077, 4086, 1, 4, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 13, 4096, 0, 0, 0, 4096, 0, 0, 4093, 4096, 0, 0, 0, 0, 7, 0, 0, 4089, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3937, 0, 0, 4096, 0, 60, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4077, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 23, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 4073, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 4095, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4071, 0, 0, 4096, 0, 25, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4087, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16358, 16382, 16386, 16370, 16397, 16385, 16387] avg=16384.0000 max=16407 max_rank=0 min=16358 min_rank=1 imbalance_ratio=1.001404 max_load_reduction=114665 max_load_reduction_pct=0.874825 moved_to_echo=65539 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 23, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 4073, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 4096, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16406, 16358, 16379, 16386, 16364, 16398, 16394, 16387] avg=16384.0000 max=16406 max_rank=0 min=16358 min_rank=1 imbalance_ratio=1.001343 max_load_reduction=114666 max_load_reduction_pct=0.874832 moved_to_echo=65543 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 23, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 4086, 4073, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 17, 4096, 0, 0, 0, 3, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4031, 0, 0, 4096, 0, 48, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4094, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16364, 16379, 16398, 16365, 16380, 16392, 16387] avg=16384.0000 max=16407 max_rank=0 min=16364 min_rank=1 imbalance_ratio=1.001404 max_load_reduction=114665 max_load_reduction_pct=0.874825 moved_to_echo=65524 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 23, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 4073, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4086, 0, 0, 4096, 0, 8, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4092, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16359, 16381, 16383, 16370, 16395, 16390, 16387] avg=16384.0000 max=16407 max_rank=0 min=16359 min_rank=1 imbalance_ratio=1.001404 max_load_reduction=114665 max_load_reduction_pct=0.874825 moved_to_echo=65542 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 4080, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 8, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4057, 0, 0, 4096, 0, 27, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16400, 16376, 16379, 16385, 16365, 16386, 16391, 16390] avg=16384.0000 max=16400 max_rank=0 min=16365 min_rank=4 imbalance_ratio=1.000977 max_load_reduction=114672 max_load_reduction_pct=0.874878 moved_to_echo=65532 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 23, 4096, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 66, 1, 0, 4086, 4073, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 4096, 0, 0, 0, 1, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4039, 0, 0, 4096, 0, 12, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4083, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16409, 16425, 16381, 16376, 16363, 16352, 16382, 16384] avg=16384.0000 max=16425 max_rank=1 min=16352 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=114647 max_load_reduction_pct=0.874687 moved_to_echo=65481 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 23, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 108, 0, 0, 4086, 4073, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 13, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 4089, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3937, 0, 0, 4096, 0, 60, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4077, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16469, 16379, 16394, 16363, 16298, 16375, 16387] avg=16384.0000 max=16469 max_rank=1 min=16298 min_rank=5 imbalance_ratio=1.005188 max_load_reduction=114603 max_load_reduction_pct=0.874352 moved_to_echo=65423 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4082, 6, 4085, 0, 22, 3944, 0, 8, 0, 9, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 3, 0, 0, 11, 0, 5, 10, 7, 0, 4086, 12, 0, 0, 0, 4074, 4063, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 7, 9, 3, 0, 9, 9, 8, 8, 4086, 1, 10, 7, 4077, 4087, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 9, 4083, 2, 3646, 498, 4080, 0, 0, 40, 3, 0, 8, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 10, 0, 11, 11, 139, 0, 10, 4087, 0, 0, 5, 0, 4086, 4065, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 16, 9, 7, 4087, 16, 8, 4085, 21, 4071, 0, 10, 4078, 14, 0, 6, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4077, 0, 4079, 4, 0, 16, 5, 4090, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 11, 4086, 0, 4084, 0, 0, 2, 0, 4077, 4086, 1, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16229, 16363, 16412, 16364, 16512, 16466, 16363, 16363] avg=16384.0000 max=16512 max_rank=4 min=16229 min_rank=0 imbalance_ratio=1.007812 max_load_reduction=114560 max_load_reduction_pct=0.874023 moved_to_echo=65704 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 32768, 0, 256, 32640, 0, 128, 128, 128, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 0, 0, 32768, 0, 32768, 128, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 128, 128, 256, 0, 128, 32768, 0, 0, 128, 0, 32768, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 32384, 640, 32768, 0, 0, 128, 128, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32768, 128, 0, 0, 128, 0, 128, 128, 128, 0, 32768, 128, 256, 128, 0, 32768, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 32768, 128, 32512, 0, 128, 32768, 128, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 32768, 0, 32768, 0, 0, 128, 0, 32768, 32768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 0, 128, 128, 128, 128, 0, 128, 128, 128, 128, 32768, 128, 128, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[39, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3327, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 648, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 106, 4096, 0, 0, 0, 0, 1, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[29, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3258, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4095, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 689, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 133, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[268, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 1766, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 1588, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 540, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[277, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2956, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 916, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4077, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 30, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[147, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 3893, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 60, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 4086, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[32, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 3569, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 140, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 396, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[278, 11, 0, 10, 4085, 16, 0, 4083, 0, 9, 0, 2545, 31, 10, 4086, 9, 0, 10, 2, 0, 5, 4079, 15, 0, 0, 4059, 10, 8, 11, 0, 4080, 7, 9, 13, 0, 4085, 4082, 0, 14, 12, 3994, 4074, 9, 4079, 4085, 10, 4074, 0, 0, 5, 10, 10, 6, 32, 5, 6, 4082, 4085, 7, 6, 4084, 12, 9, 4033, 0, 0, 5, 4078, 0, 11, 0, 1, 6, 0, 9, 4077, 10, 4087, 0, 1, 16, 14, 31, 4085, 4081, 0, 0, 4083, 0, 0, 0, 0, 0, 4080, 4082, 4084, 0, 0, 4076, 0, 0, 0, 0, 0, 5, 4084, 10, 0, 6, 9, 9, 40, 0, 2, 0, 0, 4069, 0, 1348, 4059, 12, 10, 0, 13, 0, 4, 0, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[300, 0, 0, 0, 4096, 0, 9, 4096, 0, 0, 0, 3154, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 688, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 32, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[39, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3327, 0, 0, 4096, 0, 589, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4060, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 648, 0, 0, 4096, 3507, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 4096, 0, 0, 4096, 36, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 106, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 3778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16510, 16386, 16397, 16442, 16399, 16384, 16381, 16173] avg=16384.0000 max=16510 max_rank=0 min=16173 min_rank=7 imbalance_ratio=1.007690 max_load_reduction=114562 max_load_reduction_pct=0.874039 moved_to_echo=65337 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3258, 0, 0, 4096, 0, 589, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4060, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, 0, 0, 4096, 3507, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 0, 4096, 36, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 133, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16431, 16386, 16396, 16484, 16411, 16384, 16381, 16199] avg=16384.0000 max=16484 max_rank=3 min=16199 min_rank=7 imbalance_ratio=1.006104 max_load_reduction=114588 max_load_reduction_pct=0.874237 moved_to_echo=65375 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[268, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 1766, 0, 0, 4096, 0, 589, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4060, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1588, 0, 0, 4096, 3507, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 4096, 0, 0, 4096, 36, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 540, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15179, 16386, 16397, 17383, 16357, 16384, 16381, 16605] avg=16384.0000 max=17383 max_rank=3 min=15179 min_rank=0 imbalance_ratio=1.060974 max_load_reduction=113689 max_load_reduction_pct=0.867378 moved_to_echo=65727 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[277, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2956, 0, 0, 4096, 0, 589, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4060, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 916, 0, 0, 4096, 3507, 0, 0, 4096, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 4096, 0, 0, 4096, 36, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 30, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16386, 16397, 16692, 16359, 16384, 16381, 16096] avg=16384.0000 max=16692 max_rank=3 min=16096 min_rank=7 imbalance_ratio=1.018799 max_load_reduction=114380 max_load_reduction_pct=0.872650 moved_to_echo=65220 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 3569, 0, 0, 4096, 0, 591, 270, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4060, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 4096, 3505, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 4096, 0, 0, 4096, 36, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 396, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16755, 16388, 16388, 15928, 16385, 16381, 16381, 16466] avg=16384.0000 max=16755 max_rank=0 min=15928 min_rank=3 imbalance_ratio=1.022644 max_load_reduction=114317 max_load_reduction_pct=0.872169 moved_to_echo=65613 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[147, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 3893, 0, 0, 4096, 0, 589, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4060, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 4096, 3507, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 4086, 0, 4096, 0, 0, 4096, 36, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17187, 16385, 16398, 15855, 16417, 16384, 16380, 16066] avg=16384.0000 max=17187 max_rank=0 min=15855 min_rank=3 imbalance_ratio=1.049011 max_load_reduction=113885 max_load_reduction_pct=0.868874 moved_to_echo=65247 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[300, 0, 0, 0, 4096, 0, 9, 4096, 0, 0, 0, 3154, 0, 0, 4096, 0, 589, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4060, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 688, 0, 0, 4096, 3507, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 4096, 0, 0, 4096, 36, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 32, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16607, 16386, 16397, 16483, 16337, 16384, 16380, 16098] avg=16384.0000 max=16607 max_rank=0 min=16098 min_rank=7 imbalance_ratio=1.013611 max_load_reduction=114465 max_load_reduction_pct=0.873299 moved_to_echo=65199 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[278, 11, 0, 10, 4085, 16, 0, 4083, 0, 9, 0, 2545, 31, 10, 4086, 9, 587, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2, 0, 5, 4079, 15, 0, 0, 4059, 10, 8, 11, 0, 4080, 7, 4046, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 13, 0, 14, 0, 0, 14, 12, 3994, 4074, 9, 4079, 0, 10, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 10, 10, 6, 32, 5, 6, 4082, 3498, 7, 6, 4084, 12, 9, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4078, 0, 11, 0, 1, 6, 0, 9, 4077, 10, 4087, 0, 1, 4084, 36, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 14, 31, 0, 4081, 0, 0, 4083, 0, 0, 0, 0, 0, 4080, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 5, 4084, 10, 0, 6, 9, 9, 40, 4085, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4069, 0, 1348, 4059, 12, 10, 0, 13, 0, 4, 0, 4084, 3768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16026, 16369, 16302, 15805, 16407, 16387, 16407, 17369] avg=16384.0000 max=17369 max_rank=7 min=15805 min_rank=3 imbalance_ratio=1.060120 max_load_reduction=113703 max_load_reduction_pct=0.867485 moved_to_echo=66570 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 0, 128, 32768, 128, 0, 0, 32768, 128, 128, 128, 0, 32768, 128, 32512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1408, 128, 0, 128, 32768, 128, 128, 32768, 0, 128, 0, 24576, 128, 128, 32768, 128, 4736, 2176, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 0, 32768, 0, 2688, 32768, 128, 128, 0, 128, 128, 128, 0, 32768, 30336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 0, 0, 128, 32768, 128, 0, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 0, 0, 128, 128, 32768, 32768, 128, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 32768, 0, 128, 0, 128, 128, 128, 128, 32512, 128, 32768, 0, 128, 32768, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 4864, 128, 128, 32768, 28160, 128, 128, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 11, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 16, 4096, 0, 4096, 0, 0, 0, 288, 4096, 0, 0, 4096, 0, 0, 926, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 346, 0, 0, 0, 4096, 0, 0, 0, 2545, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 4096, 4072, 0, 0, 4096, 0, 0, 4095, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 339, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 82, 4096, 0, 4096, 0, 0, 0, 2568, 4096, 0, 0, 4096, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 209, 0, 0, 0, 4096, 0, 0, 0, 855, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4025, 4096, 4096, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 156, 4096, 0, 4096, 4096, 4096, 0, 5, 4096, 0, 0, 0, 4096, 24, 4096, 0, 4096, 0, 0, 0, 221, 4096, 0, 0, 4096, 0, 0, 608, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 1632, 0, 0, 0, 4096, 0, 0, 0, 1508, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4088, 4096, 4092, 4095, 4083, 0, 0, 0, 0, 0, 0, 0, 4096, 4072, 0, 0, 4096, 0, 0, 4079, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4092, 4096, 0, 0, 4096, 0, 0, 0, 4096, 116, 4096, 0, 4096, 0, 0, 0, 2363, 4096, 0, 0, 4096, 0, 0, 221, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 231, 0, 0, 0, 4096, 0, 0, 0, 1242, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4077, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4054, 0, 0, 4096, 0, 0, 4084, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 13, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 84, 4096, 0, 4096, 0, 0, 0, 1279, 4096, 0, 0, 4096, 0, 0, 704, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 315, 0, 0, 0, 4096, 0, 0, 0, 1744, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4080, 4096, 4096, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4083, 9, 6, 4085, 4, 4082, 4032, 4085, 2, 0, 4082, 0, 0, 10, 4084, 823, 4065, 9, 4081, 0, 11, 0, 2, 4086, 56, 8, 4084, 10, 12, 635, 0, 0, 11, 0, 0, 0, 10, 4068, 13, 8, 0, 0, 5, 10, 0, 16, 0, 4087, 0, 15, 0, 0, 8, 12, 0, 12, 12, 4084, 9, 0, 5, 0, 10, 0, 4083, 12, 15, 35, 8, 12, 0, 0, 26, 0, 14, 0, 2, 0, 29, 0, 4080, 0, 4084, 14, 0, 4080, 2495, 3, 0, 5, 4077, 0, 0, 0, 156, 7, 0, 9, 4079, 4082, 0, 26, 13, 0, 11, 4081, 4075, 4082, 4064, 4081, 5, 0, 0, 0, 7, 12, 10, 4065, 4086, 0, 6, 4053, 0, 0, 4059, 4, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 752, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 90, 4096, 0, 4096, 0, 0, 0, 593, 4096, 0, 0, 4096, 0, 0, 595, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 1496, 0, 0, 0, 4096, 0, 0, 0, 590, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4076, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 590, 4096, 0, 4096, 4096, 4092, 0, 0, 4094, 1, 0, 0, 4096, 28, 4096, 0, 4096, 0, 0, 0, 377, 4096, 0, 0, 4096, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 2474, 0, 0, 0, 4096, 0, 0, 0, 561, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4094, 4096, 4096, 4096, 4033, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 0, 0, 3873, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 0, 4096, 0, 0, 0, 288, 2476, 0, 0, 4096, 0, 0, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4047, 2295, 1620, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4065, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1801, 0, 0, 4096, 346, 0, 0, 0, 4096, 0, 0, 0, 2545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 33, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4072, 0, 0, 4096, 0, 0, 4095, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16172, 15994, 16383, 16377, 16370, 16980, 16406, 16390] avg=16384.0000 max=16980 max_rank=5 min=15994 min_rank=1 imbalance_ratio=1.036377 max_load_reduction=114092 max_load_reduction_pct=0.870453 moved_to_echo=66146 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 339, 0, 0, 3873, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 4096, 0, 4096, 0, 0, 0, 2568, 2476, 0, 0, 4096, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4047, 2295, 1620, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4065, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1801, 0, 0, 4096, 209, 0, 0, 0, 4096, 0, 0, 0, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 4096, 33, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 4096, 0, 0, 4096, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16500, 17545, 16383, 16377, 16370, 15153, 16336, 16408] avg=16384.0000 max=17545 max_rank=1 min=15153 min_rank=5 imbalance_ratio=1.070862 max_load_reduction=113527 max_load_reduction_pct=0.866142 moved_to_echo=64267 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 752, 0, 0, 3873, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 4096, 0, 4096, 0, 0, 0, 593, 2476, 0, 0, 4096, 0, 0, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4047, 2295, 1620, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4065, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1801, 0, 0, 4096, 1496, 0, 0, 0, 4096, 0, 0, 0, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 33, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16913, 16042, 16383, 16377, 16370, 16175, 16397, 16415] avg=16384.0000 max=16913 max_rank=0 min=16042 min_rank=1 imbalance_ratio=1.032288 max_load_reduction=114159 max_load_reduction_pct=0.870964 moved_to_echo=65357 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3873, 4092, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 4096, 0, 4096, 0, 0, 0, 2363, 2476, 0, 0, 4096, 0, 0, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4047, 2295, 1620, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4065, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1801, 0, 0, 4096, 231, 0, 0, 0, 4096, 0, 0, 0, 1242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4096, 33, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4054, 0, 0, 4096, 0, 0, 4084, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16157, 17464, 16383, 16377, 16370, 15562, 16398, 16361] avg=16384.0000 max=17464 max_rank=1 min=15562 min_rank=5 imbalance_ratio=1.065918 max_load_reduction=113608 max_load_reduction_pct=0.866760 moved_to_echo=64691 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 156, 0, 0, 3873, 4096, 4096, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4096, 0, 4096, 0, 0, 0, 221, 2476, 0, 0, 4096, 0, 0, 608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 1, 0, 4043, 2295, 1620, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 4096, 4096, 4065, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1801, 0, 0, 4096, 1632, 0, 0, 0, 4096, 0, 0, 0, 1508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 33, 4095, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4072, 0, 0, 4096, 0, 0, 4079, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16322, 15617, 16383, 16378, 16374, 17229, 16395, 16374] avg=16384.0000 max=17229 max_rank=5 min=15617 min_rank=1 imbalance_ratio=1.051575 max_load_reduction=113843 max_load_reduction_pct=0.868553 moved_to_echo=66372 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 13, 0, 0, 3871, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 4096, 0, 4096, 0, 0, 0, 1279, 2472, 0, 0, 4096, 0, 0, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4050, 2300, 1624, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4072, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1796, 0, 0, 4096, 315, 0, 0, 0, 4096, 0, 0, 0, 1744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 27, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 4096, 0, 0, 4096, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16172, 16827, 16383, 16391, 16380, 16143, 16383, 16393] avg=16384.0000 max=16827 max_rank=1 min=16143 min_rank=5 imbalance_ratio=1.027039 max_load_reduction=114245 max_load_reduction_pct=0.871620 moved_to_echo=65299 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 590, 0, 0, 3873, 4096, 4092, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4096, 0, 4096, 0, 0, 0, 377, 2476, 0, 0, 4096, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 2, 0, 4096, 0, 0, 0, 0, 0, 4047, 2295, 1620, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4065, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1801, 0, 0, 4096, 2474, 0, 0, 0, 4096, 0, 0, 0, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 33, 4096, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 4096, 0, 0, 4096, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16746, 15317, 16383, 16382, 16370, 17124, 16352, 16398] avg=16384.0000 max=17124 max_rank=5 min=15317 min_rank=1 imbalance_ratio=1.045166 max_load_reduction=113948 max_load_reduction_pct=0.869354 moved_to_echo=66244 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 6, 0, 4, 3860, 4032, 4085, 2, 0, 4082, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 823, 4065, 9, 4081, 0, 11, 0, 2, 2470, 56, 8, 4084, 10, 12, 635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 10, 4068, 13, 8, 0, 0, 5, 10, 0, 16, 0, 4085, 4084, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 15, 0, 0, 8, 12, 0, 12, 12, 4084, 9, 0, 5, 0, 10, 4033, 2288, 1616, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 12, 15, 35, 8, 12, 0, 0, 26, 0, 14, 0, 2, 0, 29, 4082, 4079, 4053, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 1796, 14, 0, 4080, 2495, 3, 0, 5, 4077, 0, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 0, 0, 0, 26, 13, 0, 11, 4081, 4075, 33, 4064, 4081, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 12, 10, 4065, 4086, 0, 6, 4053, 0, 0, 4059, 4, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16090, 16266, 16391, 16413, 16468, 16706, 16405, 16333] avg=16384.0000 max=16706 max_rank=5 min=16090 min_rank=0 imbalance_ratio=1.019653 max_load_reduction=114366 max_load_reduction_pct=0.872543 moved_to_echo=65912 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1280, 32768, 128, 32768, 0, 128, 0, 7808, 19840, 128, 128, 32768, 128, 128, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 0, 128, 0, 0, 128, 128, 128, 128, 128, 32768, 128, 0, 128, 128, 128, 32384, 18432, 13056, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 128, 0, 128, 128, 0, 128, 128, 128, 0, 128, 32768, 32768, 32640, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 0, 128, 32768, 128, 128, 0, 0, 128, 128, 0, 128, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 0, 0, 0, 128, 128, 0, 128, 32640, 32768, 384, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 32768, 0, 14464, 128, 0, 32768, 9216, 128, 0, 128, 32768, 0, 0, 0, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 128, 32768, 32640, 0, 128, 32768, 0, 0, 32768, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 1920, 0, 128, 30976, 32768, 32768, 128, 128, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 3, 0, 14, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 19, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4074, 11, 4081, 0, 0, 0, 14, 0, 0, 4086, 0, 0, 10, 0, 4086, 4069, 0, 0, 4087, 9, 10, 11, 0, 0, 231, 11, 7, 0, 4084, 7, 4083, 4085, 0, 1, 26, 8, 5, 0, 0, 4029, 6, 0, 12, 14, 0, 4087, 4068, 10, 9, 10, 6, 0, 0, 0, 8, 4082, 0, 4083, 0, 4079, 0, 9, 5, 4080, 0, 0, 4, 15, 0, 0, 9, 0, 4076, 0, 4081, 10, 0, 0, 13, 0, 0, 0, 4063, 8, 3863, 4081, 4082, 4084, 0, 0, 0, 4085, 0, 9, 0, 0, 2, 11, 4089, 1, 0, 15, 4086, 8, 0, 10, 0, 12, 14, 6, 8, 0, 13, 4083, 4085, 0, 4, 4084, 4085, 4083, 8, 61, 0, 4084, 13, 19, 1, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 21, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 0, 0, 0, 4, 0, 3, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 104, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 4096, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4096, 0, 0, 0, 0, 15, 0, 20, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 11, 0, 1, 0, 0, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 3, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 4096, 0, 0, 0, 0, 13, 0, 57, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4095, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4039, 49, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 3, 4096, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 4096, 4096, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4075, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16337, 16387, 16383, 16408, 16419, 16375, 16374] avg=16384.0000 max=16419 max_rank=5 min=16337 min_rank=1 imbalance_ratio=1.002136 max_load_reduction=114653 max_load_reduction_pct=0.874733 moved_to_echo=65576 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 1, 0, 19, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4039, 49, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 4096, 4096, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4075, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16355, 16387, 16383, 16403, 16411, 16370, 16374] avg=16384.0000 max=16411 max_rank=5 min=16355 min_rank=1 imbalance_ratio=1.001648 max_load_reduction=114661 max_load_reduction_pct=0.874794 moved_to_echo=65558 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 21, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4040, 55, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 0, 0, 0, 4, 4096, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 4096, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4074, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4085, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16350, 16396, 16382, 16398, 16403, 16382, 16373] avg=16384.0000 max=16403 max_rank=5 min=16350 min_rank=1 imbalance_ratio=1.001160 max_load_reduction=114669 max_load_reduction_pct=0.874855 moved_to_echo=65556 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 11, 0, 1, 0, 0, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 4096, 0, 4039, 49, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4095, 3, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 4096, 0, 0, 0, 0, 13, 4096, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4096, 0, 4096, 4096, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4075, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16344, 16382, 16404, 16344, 16462, 16373, 16374] avg=16384.0000 max=16462 max_rank=5 min=16344 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=114610 max_load_reduction_pct=0.874405 moved_to_echo=65553 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 1, 0, 104, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4039, 49, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 4096, 4096, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4074, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16440, 16387, 16383, 16309, 16420, 16370, 16374] avg=16384.0000 max=16440 max_rank=1 min=16309 min_rank=4 imbalance_ratio=1.003418 max_load_reduction=114632 max_load_reduction_pct=0.874573 moved_to_echo=65473 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4039, 49, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4096, 0, 0, 0, 0, 15, 4096, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 0, 4096, 4096, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4074, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16334, 16395, 16383, 16407, 16425, 16365, 16374] avg=16384.0000 max=16425 max_rank=5 min=16334 min_rank=1 imbalance_ratio=1.002502 max_load_reduction=114647 max_load_reduction_pct=0.874687 moved_to_echo=65571 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4074, 11, 4081, 0, 0, 0, 14, 0, 0, 5, 0, 0, 10, 0, 4086, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 9, 10, 11, 0, 0, 231, 11, 7, 0, 4084, 7, 4083, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 26, 8, 5, 0, 0, 4029, 6, 0, 12, 14, 0, 4087, 4068, 10, 4027, 49, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 10, 6, 0, 0, 0, 8, 4082, 0, 4083, 0, 4079, 0, 9, 5, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 15, 0, 0, 9, 0, 4076, 0, 4081, 10, 0, 0, 13, 0, 4085, 4085, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 8, 3863, 4081, 4082, 21, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 11, 4089, 1, 0, 15, 4086, 8, 0, 10, 0, 12, 14, 6, 8, 0, 4081, 4064, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4083, 0, 0, 4, 4075, 0, 4083, 8, 61, 0, 4084, 13, 19, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16350, 16576, 16351, 16371, 16399, 16127, 16443, 16455] avg=16384.0000 max=16576 max_rank=1 min=16127 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=114496 max_load_reduction_pct=0.873535 moved_to_echo=65424 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4039, 49, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4095, 4096, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4075, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16336, 16387, 16383, 16404, 16405, 16394, 16374] avg=16384.0000 max=16405 max_rank=5 min=16336 min_rank=1 imbalance_ratio=1.001282 max_load_reduction=114667 max_load_reduction_pct=0.874840 moved_to_echo=65577 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 128, 32768, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 32384, 128, 128, 128, 128, 0, 384, 128, 128, 0, 32768, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 32768, 128, 0, 128, 128, 0, 32768, 32768, 128, 32384, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 128, 32640, 32768, 32768, 256, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 0, 128, 0, 32640, 0, 32768, 128, 0, 0, 128, 128, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 0, 0, 128, 32768, 0, 32768, 128, 128, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 32640, 128, 0, 128, 32768, 128, 0, 128, 0, 128, 128, 128, 128, 0, 32768, 32640, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 32768, 128, 32768, 0, 32768, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True + [2026-08-30 05:05:00] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 753.3 | throughput per GPU (TFLOP/s/GPU): 442.6 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.016028E+01 | loss scale: 1.0 | grad norm: 16.550 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[94, 585, 442, 0, 206, 0, 1167, 33, 1461, 139, 62, 1199, 1439, 997, 0, 3873, 3160, 498, 2321, 843, 2107, 1957, 145, 77, 6, 110, 1531, 487, 1986, 4, 1831, 457, 3965, 1, 113, 2363, 1291, 75, 2756, 1279, 1416, 28, 882, 1525, 0, 1078, 338, 2275, 2750, 3701, 2397, 192, 3, 2, 392, 1294, 2, 1101, 889, 1436, 2438, 1154, 2456, 18, 2024, 3264, 0, 490, 240, 293, 620, 673, 3668, 1204, 333, 1, 54, 7, 50, 1112, 1629, 2354, 4, 707, 57, 1665, 1369, 1820, 753, 8, 2048, 2389, 352, 292, 1224, 3, 1975, 2592, 330, 254, 6, 1829, 832, 1514, 1286, 1046, 5, 1665, 2229, 362, 197, 309, 265, 958, 1946, 2561, 91, 207, 398, 621, 1940, 1122, 702, 2417, 19, 4, 1853, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[26, 261, 383, 0, 27, 1, 1074, 7, 1146, 181, 102, 2071, 1171, 1102, 3, 3933, 3120, 239, 1574, 842, 2685, 2052, 59, 90, 11, 35, 1247, 641, 2068, 0, 1544, 383, 4014, 1, 11, 2151, 1851, 34, 2274, 1371, 998, 17, 481, 887, 3, 1704, 114, 1870, 3231, 3149, 2268, 39, 2, 1, 857, 1114, 1, 1322, 742, 1393, 2047, 1582, 2421, 0, 2985, 3071, 4, 663, 494, 85, 166, 645, 3612, 1673, 245, 7, 111, 8, 118, 572, 1938, 2492, 6, 1344, 68, 2031, 278, 2480, 233, 8, 1873, 2720, 1144, 779, 1441, 1, 1374, 3090, 450, 168, 0, 1735, 553, 1631, 1338, 773, 1, 1466, 2425, 1011, 256, 346, 323, 1048, 2077, 1882, 15, 318, 235, 798, 1968, 1513, 405, 2786, 21, 0, 1742, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[17, 258, 675, 0, 11, 0, 1301, 3, 1297, 238, 141, 2483, 1122, 1422, 0, 3757, 1950, 249, 1664, 985, 3116, 2255, 14, 74, 4, 27, 1289, 438, 2712, 1, 1808, 436, 4049, 0, 19, 2850, 2676, 24, 1914, 892, 997, 15, 402, 874, 3, 721, 135, 1455, 3046, 3519, 2149, 1, 1, 2, 527, 1416, 0, 2069, 602, 1648, 2104, 1449, 2454, 1, 2976, 3305, 0, 885, 268, 115, 250, 1022, 3730, 1148, 247, 0, 99, 28, 30, 382, 1844, 2675, 3, 959, 93, 1949, 396, 2515, 137, 5, 1759, 2473, 934, 526, 1465, 1, 1789, 3280, 373, 221, 3, 2302, 572, 1339, 1292, 300, 4, 780, 2034, 802, 170, 418, 362, 764, 1800, 2454, 28, 190, 449, 638, 2603, 1091, 330, 2893, 17, 3, 1778, 13] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[7, 157, 442, 0, 15, 0, 1254, 4, 941, 227, 91, 1798, 1023, 894, 1, 3899, 3299, 499, 1651, 1193, 2958, 2254, 11, 37, 0, 35, 1364, 424, 2301, 0, 1752, 720, 4073, 0, 19, 2442, 1709, 18, 2066, 810, 824, 4, 316, 630, 0, 1104, 245, 2157, 2630, 3709, 1902, 27, 0, 1, 566, 1303, 0, 1873, 726, 1086, 1976, 1005, 2335, 3, 2737, 2997, 0, 604, 383, 71, 288, 772, 3895, 1593, 265, 1, 107, 2, 32, 348, 1649, 2508, 0, 1243, 54, 1926, 607, 2413, 322, 1, 1751, 2483, 661, 588, 1623, 0, 1647, 3077, 703, 175, 2, 2054, 507, 1430, 1527, 537, 0, 1301, 2404, 1252, 216, 362, 397, 776, 2045, 2665, 63, 180, 213, 785, 2560, 1534, 588, 2888, 36, 5, 2409, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[24, 275, 264, 2, 15, 2, 1136, 8, 990, 120, 133, 1963, 1453, 1038, 0, 3927, 2134, 437, 1378, 822, 2095, 2244, 25, 191, 39, 10, 859, 558, 2554, 3, 1377, 489, 4039, 2, 10, 1717, 1513, 26, 2529, 1915, 982, 35, 529, 699, 9, 1265, 216, 1925, 2812, 3575, 2437, 45, 14, 11, 923, 1452, 1, 2058, 1043, 1098, 1960, 1023, 2057, 1, 2551, 3301, 0, 965, 537, 39, 281, 446, 3720, 2037, 318, 55, 81, 5, 78, 537, 1503, 2998, 7, 1224, 38, 1880, 762, 2624, 713, 14, 1886, 2841, 919, 484, 1612, 16, 1748, 3147, 708, 220, 5, 1741, 774, 1749, 1396, 561, 6, 901, 2365, 649, 146, 469, 259, 771, 1688, 2199, 19, 198, 358, 589, 2101, 1267, 750, 2962, 24, 7, 2337, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[30, 382, 668, 2, 22, 3, 1283, 2, 763, 119, 79, 1772, 1729, 1128, 2, 3909, 2709, 201, 1598, 1892, 2459, 1881, 109, 44, 1, 19, 1292, 698, 2198, 0, 1565, 419, 4078, 6, 9, 2314, 1504, 56, 2725, 1654, 974, 7, 548, 655, 5, 1113, 356, 2105, 2893, 3409, 2107, 53, 1, 0, 765, 1019, 0, 1655, 726, 776, 2193, 1186, 2303, 2, 2210, 3041, 6, 587, 254, 43, 357, 688, 3737, 1441, 402, 5, 94, 2, 68, 437, 1601, 2969, 2, 1284, 42, 1995, 1085, 2765, 252, 8, 1969, 2957, 628, 462, 1687, 4, 1156, 3000, 1183, 231, 21, 2060, 934, 1094, 969, 595, 10, 975, 2439, 908, 176, 273, 184, 825, 1744, 2231, 33, 566, 272, 1148, 2311, 1123, 573, 2750, 24, 0, 1996, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[32, 272, 572, 0, 47, 0, 944, 1, 623, 229, 52, 1975, 1670, 1366, 0, 3971, 3416, 317, 1637, 1131, 2648, 2299, 109, 99, 1, 13, 1371, 550, 2577, 3, 1650, 391, 4073, 1, 18, 2231, 2055, 29, 2724, 1991, 2185, 0, 361, 823, 0, 802, 178, 2321, 2325, 3581, 1828, 51, 1, 1, 756, 1705, 0, 1573, 668, 793, 2068, 1119, 2560, 1, 2712, 2820, 0, 737, 582, 36, 307, 790, 3753, 1673, 231, 5, 71, 1, 82, 460, 1710, 2906, 0, 1366, 90, 1862, 522, 1417, 471, 7, 1447, 3208, 1046, 457, 1024, 0, 1643, 3202, 421, 213, 1, 1692, 469, 1394, 1042, 520, 3, 2001, 2100, 615, 236, 70, 221, 916, 1984, 2103, 54, 393, 177, 618, 1796, 1374, 409, 2887, 45, 0, 1892, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[34, 202, 327, 2, 37, 0, 1117, 7, 1060, 210, 42, 2209, 1155, 1220, 2, 3856, 2878, 401, 1335, 1345, 2810, 2820, 4, 75, 3, 14, 1318, 461, 2169, 2, 1618, 574, 4062, 0, 9, 2603, 1780, 33, 2478, 1241, 911, 4, 332, 705, 1, 1199, 218, 1244, 2843, 3496, 1707, 9, 3, 3, 730, 1731, 0, 1397, 915, 1530, 2318, 1233, 1850, 3, 1787, 2984, 0, 580, 396, 105, 257, 722, 3860, 1356, 466, 0, 172, 1, 38, 540, 1549, 2986, 0, 1106, 33, 2040, 483, 2526, 479, 5, 1589, 2825, 949, 301, 1815, 2, 1758, 3192, 831, 146, 1, 2047, 810, 2142, 1098, 523, 1, 1076, 2404, 798, 295, 176, 298, 641, 2074, 2397, 33, 328, 302, 838, 1806, 1332, 415, 3143, 39, 3, 2278, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[26, 261, 383, 0, 27, 1, 1074, 7, 1146, 181, 102, 2071, 1171, 1102, 3, 3933, 3116, 1252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2172, 239, 1574, 842, 2685, 2052, 59, 90, 11, 35, 1247, 641, 2068, 0, 1544, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2567, 1, 11, 2151, 1851, 34, 2274, 1371, 998, 17, 481, 887, 3, 1704, 114, 1870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3231, 33, 2268, 39, 2, 1, 857, 1114, 1, 1322, 742, 1393, 2047, 1582, 2421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2985, 3071, 4, 663, 494, 85, 166, 645, 3612, 1673, 245, 7, 111, 8, 118, 572, 1447, 666, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1938, 2492, 6, 1344, 68, 2031, 278, 2480, 233, 8, 1873, 802, 1144, 779, 1441, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1374, 3043, 450, 168, 0, 1735, 553, 1631, 1338, 773, 1, 1466, 2425, 1011, 256, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 1048, 2077, 1882, 15, 318, 235, 798, 1968, 1513, 405, 2786, 21, 0, 1742, 1, 795, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15856, 15642, 16334, 17053, 16725, 16918, 16570, 15974] avg=16384.0000 max=17053 max_rank=3 min=15642 min_rank=1 imbalance_ratio=1.040833 max_load_reduction=114019 max_load_reduction_pct=0.869896 moved_to_echo=66187 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[17, 258, 675, 0, 11, 0, 1301, 3, 1297, 238, 141, 2483, 1122, 1422, 0, 3757, 3482, 1138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1358, 249, 1664, 985, 3116, 2255, 14, 74, 4, 27, 1289, 438, 2712, 1, 1808, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2589, 0, 19, 2850, 2676, 24, 1914, 892, 997, 15, 402, 874, 3, 721, 135, 1455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3046, 37, 2149, 1, 1, 2, 527, 1416, 0, 2069, 602, 1648, 2104, 1449, 2454, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2976, 3305, 0, 885, 268, 115, 250, 1022, 3730, 1148, 247, 0, 99, 28, 30, 382, 1460, 606, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1844, 2675, 3, 959, 93, 1949, 396, 2515, 137, 5, 1759, 729, 934, 526, 1465, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1789, 3230, 373, 221, 3, 2302, 572, 1339, 1292, 300, 4, 780, 2034, 802, 170, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 764, 1800, 2454, 28, 190, 449, 638, 2603, 1091, 330, 2893, 17, 3, 1778, 13, 497, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17345, 16430, 15566, 17506, 16646, 15990, 15629, 15960] avg=16384.0000 max=17506 max_rank=3 min=15566 min_rank=2 imbalance_ratio=1.068481 max_load_reduction=113566 max_load_reduction_pct=0.866440 moved_to_echo=64225 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[94, 585, 442, 0, 206, 0, 1167, 33, 1461, 139, 62, 1199, 1439, 997, 0, 3873, 3667, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2192, 498, 2321, 843, 2107, 1957, 145, 77, 6, 110, 1531, 487, 1986, 4, 1831, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2532, 1, 113, 2363, 1291, 75, 2756, 1279, 1416, 28, 882, 1525, 0, 1078, 338, 2275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2750, 34, 2397, 192, 3, 2, 392, 1294, 2, 1101, 889, 1436, 2438, 1154, 2456, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2024, 3264, 0, 490, 240, 293, 620, 673, 3668, 1204, 333, 1, 54, 7, 50, 1112, 1433, 590, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1629, 2354, 4, 707, 57, 1665, 1369, 1820, 753, 8, 2048, 695, 352, 292, 1224, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1975, 2550, 330, 254, 6, 1829, 832, 1514, 1286, 1046, 5, 1665, 2229, 362, 197, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 958, 1946, 2561, 91, 207, 398, 621, 1940, 1122, 702, 2417, 19, 4, 1853, 3, 809, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16468, 16552, 17952, 16558, 16215, 14980, 16389, 15958] avg=16384.0000 max=17952 max_rank=2 min=14980 min_rank=5 imbalance_ratio=1.095703 max_load_reduction=113120 max_load_reduction_pct=0.863037 moved_to_echo=63542 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[7, 157, 442, 0, 15, 0, 1254, 4, 941, 227, 91, 1798, 1023, 894, 1, 3899, 3670, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2296, 499, 1651, 1193, 2958, 2254, 11, 37, 0, 35, 1364, 424, 2301, 0, 1752, 720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2604, 0, 19, 2442, 1709, 18, 2066, 810, 824, 4, 316, 630, 0, 1104, 245, 2157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2630, 39, 1902, 27, 0, 1, 566, 1303, 0, 1873, 726, 1086, 1976, 1005, 2335, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2737, 2997, 0, 604, 383, 71, 288, 772, 3895, 1593, 265, 1, 107, 2, 32, 348, 1469, 608, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1649, 2508, 0, 1243, 54, 1926, 607, 2413, 322, 1, 1751, 732, 661, 588, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1647, 3030, 703, 175, 2, 2054, 507, 1430, 1527, 537, 0, 1301, 2404, 1252, 216, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 776, 2045, 2665, 63, 180, 213, 785, 2560, 1534, 588, 2888, 36, 5, 2409, 0, 841, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15566, 17495, 14948, 15472, 16334, 16078, 17147, 18032] avg=16384.0000 max=18032 max_rank=7 min=14948 min_rank=2 imbalance_ratio=1.100586 max_load_reduction=113040 max_load_reduction_pct=0.862427 moved_to_echo=67591 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[24, 275, 264, 2, 15, 2, 1136, 8, 990, 120, 133, 1963, 1453, 1038, 0, 3927, 3538, 1308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1486, 437, 1378, 822, 2095, 2244, 25, 191, 39, 10, 859, 558, 2554, 3, 1377, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2583, 2, 10, 1717, 1513, 26, 2529, 1915, 982, 35, 529, 699, 9, 1265, 216, 1925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2812, 37, 2437, 45, 14, 11, 923, 1452, 1, 2058, 1043, 1098, 1960, 1023, 2057, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2551, 3301, 0, 965, 537, 39, 281, 446, 3720, 2037, 318, 55, 81, 5, 78, 537, 1456, 696, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1503, 2998, 7, 1224, 38, 1880, 762, 2624, 713, 14, 1886, 837, 919, 484, 1612, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1748, 3099, 708, 220, 5, 1741, 774, 1749, 1396, 561, 6, 901, 2365, 649, 146, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 771, 1688, 2199, 19, 198, 358, 589, 2101, 1267, 750, 2962, 24, 7, 2337, 0, 544, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16196, 14567, 15955, 16972, 17207, 17517, 16537, 16121] avg=16384.0000 max=17517 max_rank=5 min=14567 min_rank=1 imbalance_ratio=1.069153 max_load_reduction=113555 max_load_reduction_pct=0.866356 moved_to_echo=67382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[30, 382, 668, 2, 22, 3, 1283, 2, 763, 119, 79, 1772, 1729, 1128, 2, 3909, 3373, 1361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1885, 201, 1598, 1892, 2459, 1881, 109, 44, 1, 19, 1292, 698, 2198, 0, 1565, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2608, 6, 9, 2314, 1504, 56, 2725, 1654, 974, 7, 548, 655, 5, 1113, 356, 2105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2893, 36, 2107, 53, 1, 0, 765, 1019, 0, 1655, 726, 776, 2193, 1186, 2303, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2210, 3041, 6, 587, 254, 43, 357, 688, 3737, 1441, 402, 5, 94, 2, 68, 437, 1470, 725, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1601, 2969, 2, 1284, 42, 1995, 1085, 2765, 252, 8, 1969, 871, 628, 462, 1687, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1156, 2954, 1183, 231, 21, 2060, 934, 1094, 969, 595, 10, 975, 2439, 908, 176, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 825, 1744, 2231, 33, 566, 272, 1148, 2311, 1123, 573, 2750, 24, 0, 1996, 11, 691, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16627, 16261, 16639, 15715, 15700, 17624, 15978, 16528] avg=16384.0000 max=17624 max_rank=5 min=15700 min_rank=4 imbalance_ratio=1.075684 max_load_reduction=113448 max_load_reduction_pct=0.865540 moved_to_echo=65830 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[34, 202, 327, 2, 37, 0, 1117, 7, 1060, 210, 42, 2209, 1155, 1220, 2, 3856, 3459, 1301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2003, 401, 1335, 1345, 2810, 2820, 4, 75, 3, 14, 1318, 461, 2169, 2, 1618, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2597, 0, 9, 2603, 1780, 33, 2478, 1241, 911, 4, 332, 705, 1, 1199, 218, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2843, 37, 1707, 9, 3, 3, 730, 1731, 0, 1397, 915, 1530, 2318, 1233, 1850, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1787, 2984, 0, 580, 396, 105, 257, 722, 3860, 1356, 466, 0, 172, 1, 38, 540, 1465, 692, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1549, 2986, 0, 1106, 33, 2040, 483, 2526, 479, 5, 1589, 832, 949, 301, 1815, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1758, 3143, 831, 146, 1, 2047, 810, 2142, 1098, 523, 1, 1076, 2404, 798, 295, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 641, 2074, 2397, 33, 328, 302, 838, 1806, 1332, 415, 3143, 39, 3, 2278, 0, 734, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16240, 16952, 15355, 16309, 15562, 16695, 17249, 16710] avg=16384.0000 max=17249 max_rank=6 min=15355 min_rank=2 imbalance_ratio=1.052795 max_load_reduction=113823 max_load_reduction_pct=0.868401 moved_to_echo=66216 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[32, 272, 572, 0, 47, 0, 944, 1, 623, 229, 52, 1975, 1670, 1366, 0, 3971, 3543, 1477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2377, 317, 1637, 1131, 2648, 2299, 109, 99, 1, 13, 1371, 550, 2577, 3, 1650, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2604, 1, 18, 2231, 2055, 29, 2724, 1991, 2185, 0, 361, 823, 0, 802, 178, 2321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2325, 38, 1828, 51, 1, 1, 756, 1705, 0, 1573, 668, 793, 2068, 1119, 2560, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2712, 2820, 0, 737, 582, 36, 307, 790, 3753, 1673, 231, 5, 71, 1, 82, 460, 1469, 786, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1710, 2906, 0, 1366, 90, 1862, 522, 1417, 471, 7, 1447, 945, 1046, 457, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1643, 3153, 421, 213, 1, 1692, 469, 1394, 1042, 520, 3, 2001, 2100, 615, 236, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 916, 1984, 2103, 54, 393, 177, 618, 1796, 1374, 409, 2887, 45, 0, 1892, 0, 871, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16774, 17173, 18323, 15487, 16683, 15270, 15573, 15789] avg=16384.0000 max=18323 max_rank=2 min=15270 min_rank=5 imbalance_ratio=1.118347 max_load_reduction=112749 max_load_reduction_pct=0.860207 moved_to_echo=63315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[20096, 24832, 128, 5632, 3200, 896, 2560, 5760, 30080, 12160, 2560, 128, 896, 128, 512, 4480, 11776, 5376, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[15872, 2944, 13184, 9088, 20992, 17792, 512, 768, 128, 384, 10368, 4352, 18688, 128, 13184, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[13184, 24320, 5120, 1664, 128, 15488, 5504, 12416, 9984, 4864, 128, 10240, 18432, 6400, 1792, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[384, 2432, 3840, 128, 384, 128, 9344, 128, 8320, 1536, 768, 15488, 10880, 9216, 128, 31232, 27904, 10112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[13440, 21888, 128, 9344, 512, 15360, 5504, 18560, 3456, 128, 14336, 6528, 6656, 3968, 11904, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2432, 6784, 15360, 18560, 384, 2432, 2432, 6144, 17152, 10368, 4224, 22784, 256, 128, 16384, 128, 5888, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[22656, 384, 16896, 512, 128, 128, 5632, 11136, 128, 13056, 6400, 9856, 17152, 9856, 18560, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[20736, 128, 256, 18688, 14464, 384, 19584, 11264, 9344, 128, 3968, 6912, 128, 9088, 1920, 15360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 8, 4, 4089, 0, 0, 4094, 4094, 0, 6, 54, 0, 0, 0, 3693, 5, 1, 4091, 9, 3, 1, 1, 4086, 3899, 4090, 0, 1, 4096, 6, 10, 4095, 0, 4, 259, 579, 0, 4, 8, 4088, 0, 0, 4073, 4074, 39, 0, 4076, 2, 15, 9, 9, 4, 1, 0, 4096, 1, 4088, 4084, 0, 0, 4082, 2699, 0, 124, 0, 3, 0, 0, 0, 1, 15, 5, 0, 11, 1, 1, 1, 3, 2, 0, 0, 5, 0, 4093, 0, 0, 4065, 25, 0, 0, 8, 4, 3868, 0, 4, 1252, 4095, 7, 3, 0, 4096, 5, 4092, 0, 3, 4055, 14, 0, 10, 3, 0, 4075, 0, 4093, 4083, 0, 4087, 0, 15, 4091, 0, 9, 0, 4001, 0, 0, 2, 3, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 4, 4090, 2, 3, 4095, 4094, 1, 36, 383, 5, 0, 3, 3836, 0, 5, 4091, 2, 0, 5, 1, 4083, 2116, 4086, 1, 0, 4089, 0, 0, 4090, 0, 0, 45, 302, 7, 25, 2, 4093, 3, 2, 4094, 4085, 31, 0, 4088, 2, 0, 0, 0, 0, 1, 3, 4091, 4, 4092, 4092, 0, 7, 4096, 2310, 1, 151, 0, 0, 1, 0, 0, 3, 0, 0, 2, 0, 3, 0, 0, 0, 0, 5, 0, 0, 4, 4089, 4, 1, 4086, 0, 0, 0, 2, 6, 3541, 0, 0, 3560, 4095, 0, 0, 0, 4096, 3, 4086, 2, 1, 4078, 15, 0, 0, 0, 0, 4092, 2, 4092, 4096, 0, 4096, 0, 53, 4095, 0, 0, 12, 4096, 6, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 4095, 4095, 0, 16, 159, 0, 1, 0, 3109, 1, 0, 4096, 15, 1, 0, 1, 4095, 3159, 4095, 1, 1, 4096, 0, 0, 4095, 0, 0, 208, 494, 0, 0, 6, 4096, 0, 0, 4096, 4054, 1, 0, 4084, 1, 0, 0, 1, 0, 0, 0, 4091, 7, 4095, 4096, 0, 0, 4094, 2519, 0, 10, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 3, 4094, 1, 0, 4093, 2, 0, 1, 0, 1, 3755, 0, 0, 3050, 4096, 0, 0, 2, 4095, 0, 4095, 0, 0, 4089, 23, 0, 2, 1, 0, 4090, 0, 4096, 4096, 0, 4095, 0, 10, 4095, 0, 0, 1, 4000, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 2, 1, 4092, 4096, 3, 1, 177, 7, 0, 0, 4049, 2, 0, 4096, 1, 0, 0, 4, 4093, 2724, 4092, 0, 0, 4089, 5, 0, 4092, 1, 0, 210, 1537, 0, 7, 0, 4093, 1, 2, 4092, 3982, 31, 1, 4090, 0, 4, 0, 3, 0, 0, 0, 4096, 18, 4095, 4095, 0, 2, 4091, 795, 1, 503, 4, 1, 0, 0, 0, 2, 51, 2, 0, 0, 1, 0, 2, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4095, 16, 0, 0, 0, 0, 3572, 0, 0, 2691, 4093, 0, 0, 0, 4095, 5, 4096, 0, 0, 4059, 127, 0, 0, 1, 0, 4091, 1, 4095, 4096, 1, 4095, 0, 7, 4093, 1, 5, 6, 4095, 2, 0, 0, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 2, 4081, 0, 0, 4050, 4096, 1, 26, 7, 0, 0, 0, 1514, 0, 0, 4089, 0, 0, 6, 86, 4096, 3490, 4070, 0, 3, 4096, 51, 1, 4073, 0, 0, 1231, 630, 0, 0, 0, 4082, 0, 5, 4095, 3979, 13, 0, 4074, 0, 0, 45, 0, 0, 0, 0, 4054, 4, 4090, 4070, 0, 5, 4077, 3431, 0, 130, 0, 0, 2, 1, 0, 0, 58, 0, 5, 12, 0, 8, 0, 0, 0, 0, 0, 0, 16, 4096, 0, 0, 3995, 132, 0, 0, 0, 0, 3228, 0, 5, 2636, 4096, 1, 0, 0, 4092, 16, 4096, 0, 0, 4094, 87, 0, 63, 0, 1, 4028, 0, 4092, 4096, 0, 4093, 0, 0, 4082, 7, 101, 0, 3952, 0, 0, 0, 0, 28] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 2, 4093, 0, 1, 4094, 4096, 2, 1, 154, 1, 0, 0, 3246, 0, 0, 4095, 9, 0, 1, 1, 4095, 3371, 4094, 0, 0, 4096, 2, 1, 4096, 0, 0, 263, 421, 1, 0, 0, 4093, 0, 1, 4095, 4077, 44, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 4094, 95, 4095, 4093, 0, 0, 4093, 1610, 2, 13, 0, 1, 2, 0, 1, 1, 2, 1, 0, 0, 0, 1, 0, 1, 2, 0, 0, 1, 0, 4096, 0, 1, 4092, 0, 3, 0, 0, 0, 3587, 0, 0, 3615, 4094, 0, 0, 0, 4094, 24, 4095, 0, 2, 4035, 17, 0, 7, 0, 1, 4091, 2, 4094, 4092, 0, 4096, 0, 0, 4094, 0, 4, 0, 4087, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 7, 0, 4094, 4091, 1, 4, 322, 0, 0, 4, 3300, 7, 0, 4092, 0, 0, 3, 0, 4090, 3272, 4090, 0, 0, 4089, 6, 0, 4091, 4, 1, 416, 174, 0, 2, 3, 4091, 0, 0, 4091, 4087, 0, 0, 4087, 0, 0, 0, 1, 0, 1, 0, 4093, 1, 4095, 4096, 0, 0, 4085, 1822, 0, 25, 4, 0, 0, 1, 3, 3, 1, 0, 6, 0, 0, 0, 0, 4, 3, 0, 2, 1, 6, 4086, 0, 5, 4095, 12, 0, 0, 5, 0, 3348, 1, 2, 3605, 4089, 0, 0, 0, 4094, 19, 4096, 3, 2, 4087, 13, 0, 5, 8, 2, 4085, 2, 4090, 4090, 1, 4092, 1, 10, 4094, 3, 2, 0, 4085, 71, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 2, 4096, 12, 0, 4096, 4095, 4, 0, 872, 0, 0, 0, 4037, 0, 0, 4091, 8, 0, 2, 0, 4089, 1813, 4092, 0, 0, 4093, 0, 6, 4095, 0, 0, 47, 2039, 0, 1, 0, 4090, 0, 1, 4094, 4092, 4, 0, 4091, 0, 0, 1, 0, 0, 0, 0, 4096, 22, 4096, 4096, 0, 0, 4086, 886, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4, 4092, 0, 0, 4086, 21, 0, 0, 0, 0, 3522, 0, 7, 3116, 4096, 0, 0, 6, 4095, 4, 4092, 0, 0, 4083, 24, 0, 5, 3, 0, 4095, 0, 4095, 4083, 0, 4091, 0, 21, 4094, 0, 48, 0, 3924, 2, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4, 4090, 2, 3, 4095, 4094, 1, 36, 383, 5, 0, 3, 3836, 0, 445, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1100, 2, 0, 5, 1, 4083, 2116, 4086, 1, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 302, 7, 25, 2, 4093, 3, 2, 3016, 4085, 31, 0, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 4091, 4, 1942, 4092, 0, 7, 4096, 2310, 1, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 2, 0, 3, 0, 0, 0, 0, 5, 0, 4089, 4070, 2991, 2411, 2150, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4089, 4, 1, 4086, 0, 0, 0, 2, 6, 3541, 0, 0, 3560, 1684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 3, 4086, 2, 1, 4078, 15, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4096, 0, 4096, 0, 53, 4095, 0, 0, 12, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17023, 15489, 15701, 16698, 16358, 16977, 16350, 16476] avg=16384.0000 max=17023 max_rank=0 min=15489 min_rank=1 imbalance_ratio=1.039001 max_load_reduction=114049 max_load_reduction_pct=0.870125 moved_to_echo=66161 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 4, 4089, 0, 0, 4094, 4094, 0, 6, 54, 0, 0, 0, 3693, 5, 443, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1100, 9, 3, 1, 1, 4086, 3899, 4090, 0, 1, 0, 6, 10, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 259, 579, 0, 4, 8, 4088, 0, 0, 3000, 4074, 39, 0, 4076, 2, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 4, 1, 0, 4096, 1, 1940, 4084, 0, 0, 4082, 2699, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 15, 5, 0, 11, 1, 1, 1, 3, 2, 0, 0, 4096, 4071, 2991, 2411, 2148, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4093, 0, 0, 4065, 25, 0, 0, 8, 4, 3868, 0, 4, 1252, 1684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 0, 4071, 5, 4092, 0, 3, 4055, 14, 0, 10, 3, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4083, 0, 4087, 0, 15, 4091, 0, 9, 0, 4001, 0, 0, 2, 3, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16515, 17302, 16148, 17049, 16390, 15008, 16338, 16322] avg=16384.0000 max=17302 max_rank=1 min=15008 min_rank=5 imbalance_ratio=1.056030 max_load_reduction=113770 max_load_reduction_pct=0.867996 moved_to_echo=64058 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 7, 0, 4094, 4091, 1, 4, 322, 0, 0, 4, 3300, 7, 445, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1100, 0, 0, 3, 0, 4090, 3272, 4090, 0, 0, 0, 6, 0, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 416, 174, 0, 2, 3, 4091, 0, 0, 3013, 4087, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4093, 1, 1944, 4096, 0, 0, 4085, 1822, 0, 25, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 3, 1, 0, 6, 0, 0, 0, 0, 4, 3, 0, 2, 4089, 4068, 2992, 2407, 2151, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 4086, 0, 5, 4095, 12, 0, 0, 5, 0, 3348, 1, 2, 3605, 1682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 19, 4096, 3, 2, 4087, 13, 0, 5, 8, 2, 4085, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4090, 1, 4092, 1, 10, 4094, 3, 2, 0, 4085, 71, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16396, 16656, 15874, 16072, 16363, 16848, 16391, 16472] avg=16384.0000 max=16848 max_rank=5 min=15874 min_rank=2 imbalance_ratio=1.028320 max_load_reduction=114224 max_load_reduction_pct=0.871460 moved_to_echo=66074 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 2, 1, 4092, 4096, 3, 1, 177, 7, 0, 0, 4049, 2, 445, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1101, 1, 0, 0, 4, 4093, 2724, 4092, 0, 0, 0, 5, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 1537, 0, 7, 0, 4093, 1, 2, 3014, 3982, 31, 1, 4090, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 18, 1944, 4095, 0, 2, 4091, 795, 1, 503, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 51, 2, 0, 0, 1, 0, 2, 0, 0, 0, 1, 4089, 4073, 2995, 2410, 2151, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 16, 0, 0, 0, 0, 3572, 0, 0, 2691, 1683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 5, 4096, 0, 0, 4059, 127, 0, 0, 1, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4096, 1, 4095, 0, 7, 4093, 1, 5, 6, 4095, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16995, 16113, 16972, 15552, 16411, 16153, 16450, 16426] avg=16384.0000 max=16995 max_rank=0 min=15552 min_rank=3 imbalance_ratio=1.037292 max_load_reduction=114077 max_load_reduction_pct=0.870338 moved_to_echo=65440 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 4095, 4095, 0, 16, 159, 0, 1, 0, 3109, 1, 445, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1101, 15, 1, 0, 1, 4095, 3159, 4095, 1, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 494, 0, 0, 6, 4096, 0, 0, 3018, 4054, 1, 0, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4091, 7, 1944, 4096, 0, 0, 4094, 2519, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 4096, 4074, 2995, 2411, 2151, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 1, 0, 4093, 2, 0, 1, 0, 1, 3755, 0, 0, 3050, 1685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4070, 0, 4095, 0, 0, 4089, 23, 0, 2, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4096, 0, 4095, 0, 10, 4095, 0, 0, 1, 4000, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16041, 16564, 15962, 16762, 16366, 16685, 16372, 16320] avg=16384.0000 max=16762 max_rank=3 min=15962 min_rank=2 imbalance_ratio=1.023071 max_load_reduction=114310 max_load_reduction_pct=0.872116 moved_to_echo=65743 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 4081, 0, 0, 4050, 4096, 1, 26, 7, 0, 0, 0, 1514, 0, 445, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1099, 0, 0, 6, 86, 4096, 3490, 4070, 0, 3, 0, 51, 1, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1231, 630, 0, 0, 0, 4082, 0, 5, 3017, 3979, 13, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 4054, 4, 1941, 4070, 0, 5, 4077, 3431, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 58, 0, 5, 12, 0, 8, 0, 0, 0, 0, 0, 4096, 4070, 2990, 2411, 2149, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 0, 0, 3995, 132, 0, 0, 0, 0, 3228, 0, 5, 2636, 1685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4067, 16, 4096, 0, 0, 4094, 87, 0, 63, 0, 1, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4096, 0, 4093, 0, 0, 4082, 7, 101, 0, 3952, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14247, 16975, 17031, 17757, 16435, 15793, 16453, 16381] avg=16384.0000 max=17757 max_rank=3 min=14247 min_rank=0 imbalance_ratio=1.083801 max_load_reduction=113315 max_load_reduction_pct=0.864525 moved_to_echo=65062 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2, 4093, 0, 1, 4094, 4096, 2, 1, 154, 1, 0, 0, 3246, 0, 445, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1101, 9, 0, 1, 1, 4095, 3371, 4094, 0, 0, 0, 2, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 421, 1, 0, 0, 4093, 0, 1, 3017, 4077, 44, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 95, 1944, 4093, 0, 0, 4093, 1610, 2, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1, 1, 2, 1, 0, 0, 0, 1, 0, 1, 2, 0, 0, 4096, 4072, 2994, 2410, 2151, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 1, 4092, 0, 3, 0, 0, 0, 3587, 0, 0, 3615, 1684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 24, 4095, 0, 2, 4035, 17, 0, 7, 0, 1, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4092, 0, 4096, 0, 0, 4094, 0, 4, 0, 4087, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16161, 16771, 16010, 15944, 16368, 17079, 16343, 16396] avg=16384.0000 max=17079 max_rank=5 min=15944 min_rank=3 imbalance_ratio=1.042419 max_load_reduction=113993 max_load_reduction_pct=0.869698 moved_to_echo=66186 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 4096, 12, 0, 4096, 4095, 4, 0, 872, 0, 0, 0, 4037, 0, 448, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 8, 0, 2, 0, 4089, 1813, 4092, 0, 0, 0, 0, 6, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 2039, 0, 1, 0, 4090, 0, 1, 3009, 4092, 4, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 22, 1939, 4096, 0, 0, 4086, 886, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 4080, 2994, 2416, 2157, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 4092, 0, 0, 4086, 21, 0, 0, 0, 0, 3522, 0, 7, 3116, 1680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4063, 4, 4092, 0, 0, 4083, 24, 0, 5, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4083, 0, 4091, 0, 21, 4094, 0, 48, 0, 3924, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17694, 15202, 17374, 15238, 16381, 16529, 16375, 16279] avg=16384.0000 max=17694 max_rank=0 min=15202 min_rank=1 imbalance_ratio=1.079956 max_load_reduction=113378 max_load_reduction_pct=0.865005 moved_to_echo=65564 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 32768, 128, 128, 32640, 256, 128, 128, 128, 128, 28544, 128, 128, 23552, 13568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 32768, 128, 32768, 128, 128, 32768, 128, 256, 128, 32256, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 8832, 128, 128, 128, 128, 32768, 23936, 32768, 128, 128, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 32768, 32768, 128, 128, 2176, 128, 128, 128, 26880, 128, 3584, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 32640, 128, 32768, 128, 128, 32640, 384, 0, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=133248 observed_loss=-2176 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 32640, 24064, 19328, 17280, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 256, 15616, 32768, 0, 128, 32768, 16128, 128, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 2688, 6272, 128, 128, 128, 32768, 128, 128, 24192, 32512, 256, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=133248 slack_slots=6784 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4090, 4090, 3873, 4091, 0, 1, 3, 0, 4052, 5, 4082, 8, 1, 0, 0, 4091, 0, 4094, 0, 5, 3112, 0, 1, 1, 32, 4088, 0, 0, 4093, 1, 0, 0, 4093, 4092, 2466, 13, 4065, 0, 54, 1, 5, 4082, 0, 0, 323, 3, 6, 0, 0, 2235, 4, 0, 0, 6, 0, 14, 4036, 2, 4083, 0, 8, 0, 4, 0, 0, 0, 0, 6, 0, 0, 0, 8, 4089, 0, 0, 0, 1, 0, 3, 4064, 4077, 0, 0, 4093, 1, 0, 4092, 0, 4093, 4095, 3, 0, 4095, 8, 0, 0, 4075, 10, 12, 0, 0, 0, 24, 7, 4090, 0, 0, 14, 2, 4, 4082, 0, 0, 4, 0, 2, 0, 0, 2627, 3, 1789, 0, 4, 1, 4091, 0, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4081, 4094, 4080, 4095, 0, 2, 1, 0, 4093, 1, 4095, 0, 0, 0, 2, 4095, 0, 4094, 0, 0, 2038, 1, 5, 0, 23, 4096, 2, 0, 4094, 0, 0, 0, 4094, 4095, 120, 1, 4091, 0, 3, 2, 0, 4096, 1, 0, 732, 1, 0, 0, 0, 3699, 0, 2, 0, 1, 0, 0, 4093, 0, 4093, 0, 0, 1, 0, 0, 2, 0, 1, 0, 0, 0, 0, 2, 4094, 26, 0, 0, 0, 0, 0, 4095, 4094, 0, 1, 4094, 0, 0, 3813, 0, 4096, 4096, 0, 2, 4094, 0, 2, 0, 4088, 0, 2, 0, 3, 1, 0, 0, 4095, 0, 1, 0, 0, 2, 4090, 2, 0, 1, 0, 0, 0, 0, 2718, 0, 3248, 1, 0, 0, 4095, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4029, 4090, 4068, 4090, 0, 1, 3, 0, 4094, 1, 4091, 3, 2, 15, 0, 4093, 0, 4096, 1, 0, 3169, 1, 24, 3, 6, 4092, 0, 3, 4091, 6, 5, 6, 4092, 4090, 46, 0, 4077, 0, 0, 0, 1, 4092, 0, 5, 251, 3, 0, 0, 1, 2988, 4, 3, 0, 0, 0, 0, 4089, 1, 4095, 0, 0, 4, 3, 4, 0, 0, 0, 0, 3, 1, 3, 0, 4089, 5, 0, 0, 5, 0, 0, 4087, 4092, 1, 4, 4090, 1, 0, 4067, 4, 4092, 4092, 0, 0, 4095, 5, 0, 0, 4060, 3, 0, 4, 1, 0, 0, 1, 4095, 4, 0, 0, 4, 0, 4070, 4, 4, 4, 0, 0, 3, 2, 2272, 4, 3686, 0, 2, 0, 4090, 0, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4047, 4093, 3920, 4095, 1, 0, 1, 1, 4089, 0, 4088, 2, 0, 0, 0, 4096, 1, 4092, 0, 0, 2777, 1, 35, 0, 16, 4095, 3, 0, 4096, 0, 1, 0, 4095, 4096, 136, 3, 4078, 2, 1, 2, 1, 4094, 21, 1, 729, 1, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 2, 4094, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4095, 3, 0, 1, 2, 0, 0, 4092, 4094, 0, 1, 4095, 0, 0, 3542, 1, 4095, 4095, 0, 1, 4095, 0, 0, 0, 4095, 2, 0, 0, 0, 1, 9, 0, 4087, 0, 0, 0, 0, 0, 4090, 1, 1, 0, 0, 0, 0, 0, 1314, 1, 4026, 0, 0, 0, 4092, 2, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4091, 4096, 3765, 4096, 9, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 4091, 0, 4095, 0, 0, 1258, 0, 0, 0, 16, 4096, 0, 0, 4092, 0, 0, 0, 4096, 4096, 1636, 0, 4096, 0, 0, 0, 0, 4096, 3, 0, 329, 0, 0, 4, 0, 3610, 0, 0, 0, 0, 0, 0, 4073, 0, 4087, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 54, 0, 0, 0, 0, 0, 4039, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4088, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 9, 0, 4088, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2318, 0, 3488, 0, 0, 4, 4093, 4, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4096, 4094, 3913, 4096, 30, 0, 0, 0, 4093, 0, 4095, 4, 0, 0, 0, 4003, 0, 4059, 0, 0, 3776, 0, 1, 0, 2, 4096, 0, 0, 4095, 0, 0, 0, 4096, 4096, 1725, 7, 4004, 0, 107, 0, 0, 4090, 0, 0, 1149, 0, 0, 0, 0, 2884, 0, 0, 0, 0, 0, 0, 4093, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4096, 136, 0, 0, 0, 0, 0, 4081, 4048, 0, 0, 4094, 0, 0, 4093, 0, 4096, 4096, 0, 0, 4084, 0, 0, 0, 3927, 144, 0, 0, 0, 0, 16, 0, 4068, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 416, 0, 2597, 0, 0, 10, 4096, 3, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4076, 4095, 3377, 4096, 2, 0, 0, 0, 4096, 0, 4070, 1, 0, 0, 0, 4086, 0, 4094, 0, 0, 2506, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 4096, 4096, 168, 0, 4079, 0, 0, 0, 0, 4095, 0, 0, 794, 0, 0, 0, 0, 2956, 0, 0, 1, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4096, 12, 0, 0, 0, 1, 0, 4093, 4095, 0, 0, 4096, 0, 0, 4082, 0, 4095, 4096, 0, 0, 4095, 1, 0, 0, 4084, 2, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 1, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 2649, 0, 4040, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[4090, 4090, 4073, 4093, 6, 1, 0, 0, 4095, 0, 4095, 4, 1, 1, 0, 4090, 2, 4089, 0, 0, 1891, 0, 0, 1, 2, 4094, 0, 0, 4093, 2, 0, 0, 4091, 4096, 12, 2, 4093, 1, 0, 0, 0, 4091, 18, 0, 240, 1, 2, 2, 0, 3840, 0, 0, 2, 4, 0, 2, 4091, 0, 4088, 13, 1, 1, 6, 2, 0, 0, 7, 2, 0, 0, 0, 0, 4087, 0, 0, 0, 4, 2, 3, 4092, 4092, 0, 1, 4096, 0, 2, 4082, 0, 4086, 4095, 0, 0, 4091, 0, 0, 0, 4075, 0, 0, 0, 0, 1, 11, 0, 4092, 1, 0, 7, 0, 0, 4082, 0, 0, 1, 5, 0, 1, 0, 2336, 1, 4004, 0, 8, 2, 4095, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4029, 0, 4068, 0, 0, 1, 3, 0, 4094, 1, 245, 3, 2, 15, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 3169, 1, 24, 3, 6, 4092, 0, 3, 4091, 6, 5, 6, 1359, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2731, 46, 0, 4077, 0, 0, 0, 1, 4092, 0, 5, 251, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2988, 4, 3, 0, 0, 0, 0, 4089, 1, 4095, 0, 0, 4, 3, 4, 4090, 810, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 0, 4089, 5, 0, 0, 5, 0, 0, 4087, 4092, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 4, 125, 1, 0, 4067, 4, 4092, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 3, 0, 4, 1, 0, 0, 1, 4095, 4, 0, 0, 4, 0, 4070, 4, 3155, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 3, 2, 2272, 4, 3686, 0, 2, 0, 4090, 0, 0, 4091, 2744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16554, 17015, 15298, 16110, 16357, 16486, 16350, 16902] avg=16384.0000 max=17015 max_rank=1 min=15298 min_rank=2 imbalance_ratio=1.038513 max_load_reduction=114057 max_load_reduction_pct=0.870186 moved_to_echo=66095 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 3873, 0, 0, 1, 3, 0, 4052, 5, 244, 8, 1, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 5, 3112, 0, 1, 1, 32, 4088, 0, 0, 4093, 1, 0, 0, 1360, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2732, 2466, 13, 4065, 0, 54, 1, 5, 4082, 0, 0, 323, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2235, 4, 0, 0, 6, 0, 14, 4036, 2, 4083, 0, 8, 0, 4, 0, 4090, 811, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8, 4089, 0, 0, 0, 1, 0, 3, 4064, 4095, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 125, 1, 0, 4092, 0, 4093, 0, 3, 0, 4095, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 10, 12, 0, 0, 0, 24, 7, 4090, 0, 0, 14, 2, 4, 4082, 0, 3157, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 2627, 3, 1789, 0, 4, 1, 4091, 0, 0, 4094, 2738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16368, 16940, 17843, 15310, 16340, 16494, 16424, 15353] avg=16384.0000 max=17843 max_rank=2 min=15310 min_rank=3 imbalance_ratio=1.089050 max_load_reduction=113229 max_load_reduction_pct=0.863869 moved_to_echo=64611 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4047, 0, 3920, 0, 1, 0, 1, 1, 4089, 0, 244, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 0, 2777, 1, 35, 0, 16, 4095, 3, 0, 4096, 0, 1, 0, 1361, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2735, 136, 3, 4078, 2, 1, 2, 1, 4094, 21, 1, 729, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 2, 4094, 0, 4091, 0, 0, 0, 0, 0, 4093, 811, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4095, 3, 0, 1, 2, 0, 0, 4092, 4095, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 125, 0, 0, 3542, 1, 4095, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 1, 9, 0, 4087, 0, 0, 0, 0, 0, 4090, 1, 3159, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1314, 1, 4026, 0, 0, 0, 4092, 2, 0, 4095, 2742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16401, 16631, 15899, 17151, 16370, 15954, 16393, 16273] avg=16384.0000 max=17151 max_rank=3 min=15899 min_rank=2 imbalance_ratio=1.046814 max_load_reduction=113921 max_load_reduction_pct=0.869148 moved_to_echo=64990 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 4073, 0, 6, 1, 0, 0, 4095, 0, 245, 4, 1, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4089, 0, 0, 1891, 0, 0, 1, 2, 4094, 0, 0, 4093, 2, 0, 0, 1361, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2735, 12, 2, 4093, 1, 0, 0, 0, 4091, 18, 0, 240, 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3840, 0, 0, 2, 4, 0, 2, 4091, 0, 4088, 13, 1, 1, 6, 2, 4090, 811, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 0, 0, 0, 4087, 0, 0, 0, 4, 2, 3, 4092, 4095, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 125, 0, 2, 4082, 0, 4086, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 1, 11, 0, 4092, 1, 0, 7, 0, 0, 4082, 0, 3160, 950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 1, 0, 2336, 1, 4004, 0, 8, 2, 4095, 0, 0, 4096, 2747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16606, 15688, 15288, 16968, 16368, 16479, 16379, 17296] avg=16384.0000 max=17296 max_rank=7 min=15288 min_rank=2 imbalance_ratio=1.055664 max_load_reduction=113776 max_load_reduction_pct=0.868042 moved_to_echo=66522 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 3765, 0, 9, 0, 0, 0, 4090, 0, 231, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1258, 0, 0, 0, 16, 4096, 0, 0, 4092, 0, 0, 0, 1367, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2729, 1636, 0, 4096, 0, 0, 0, 0, 4096, 3, 0, 329, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3610, 0, 0, 0, 0, 0, 0, 4073, 0, 4087, 0, 0, 0, 3, 0, 4096, 815, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 54, 0, 0, 0, 0, 0, 4039, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 117, 0, 0, 4096, 0, 4096, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 9, 0, 4088, 0, 0, 3, 0, 0, 4096, 0, 3164, 956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2318, 0, 3488, 0, 0, 4, 4093, 4, 0, 4096, 2751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16277, 15082, 16989, 16702, 16365, 16493, 16410, 16754] avg=16384.0000 max=16989 max_rank=2 min=15082 min_rank=1 imbalance_ratio=1.036926 max_load_reduction=114083 max_load_reduction_pct=0.870384 moved_to_echo=66022 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4081, 0, 4080, 0, 0, 2, 1, 0, 4093, 1, 245, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2038, 1, 5, 0, 23, 4096, 2, 0, 4094, 0, 0, 0, 1361, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2734, 120, 1, 4091, 0, 3, 2, 0, 4096, 1, 0, 732, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3699, 0, 2, 0, 1, 0, 0, 4093, 0, 4093, 0, 0, 1, 0, 0, 4094, 811, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 2, 4094, 26, 0, 0, 0, 0, 0, 4095, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 125, 0, 0, 3813, 0, 4096, 0, 0, 2, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 2, 0, 3, 1, 0, 0, 4095, 0, 1, 0, 0, 2, 4090, 2, 3158, 950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2718, 0, 3248, 1, 0, 0, 4095, 0, 0, 4096, 2747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16600, 15867, 15875, 16811, 16394, 16227, 16392, 16906] avg=16384.0000 max=16906 max_rank=7 min=15867 min_rank=1 imbalance_ratio=1.031860 max_load_reduction=114166 max_load_reduction_pct=0.871017 moved_to_echo=65919 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 3913, 0, 30, 0, 0, 0, 4093, 0, 245, 4, 0, 0, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 3776, 0, 1, 0, 2, 4096, 0, 0, 4095, 0, 0, 0, 1361, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2735, 1725, 7, 4004, 0, 107, 0, 0, 4090, 0, 0, 1149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2884, 0, 0, 0, 0, 0, 0, 4093, 0, 4050, 0, 0, 0, 0, 0, 4094, 811, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4096, 136, 0, 0, 0, 0, 0, 4081, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 125, 0, 0, 4093, 0, 4096, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 144, 0, 0, 0, 0, 16, 0, 4068, 0, 0, 1, 0, 0, 4096, 0, 3158, 950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 416, 0, 2597, 0, 0, 10, 4096, 3, 0, 4096, 2747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 17543, 17913, 15949, 16500, 16446, 16360, 13977] avg=16384.0000 max=17913 max_rank=2 min=13977 min_rank=7 imbalance_ratio=1.093323 max_load_reduction=113159 max_load_reduction_pct=0.863335 moved_to_echo=63283 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4076, 0, 3377, 0, 2, 0, 0, 0, 4096, 0, 244, 1, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2506, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 1361, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2735, 168, 0, 4079, 0, 0, 0, 0, 4095, 0, 0, 794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2956, 0, 0, 1, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 4095, 811, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4096, 12, 0, 0, 0, 1, 0, 4093, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 125, 0, 0, 4082, 0, 4095, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 2, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 1, 0, 0, 4077, 0, 3160, 944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2649, 0, 4040, 0, 0, 0, 4096, 0, 0, 4096, 2730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15882, 16306, 15967, 16071, 16378, 16493, 16364, 17611] avg=16384.0000 max=17611 max_rank=7 min=15882 min_rank=0 imbalance_ratio=1.074890 max_load_reduction=113461 max_load_reduction_pct=0.865639 moved_to_echo=66846 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32640, 0, 31104, 0, 128, 128, 128, 128, 32768, 128, 2048, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[32512, 256, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 128, 128, 32768, 128, 25344, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 32768, 128, 128, 20608, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 128, 128, 11008, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 128, 32768, 256, 0, 128, 128, 128, 128, 32768, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 21888, 6400, 128, 32640, 128, 256, 128, 128, 32768, 128, 128, 4608, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 1024, 128, 128, 31872, 128, 32768, 0, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 16768, 128, 26880, 128, 128, 128, 32768, 128, 0, 32768, 22016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132992 observed_loss=-1920 tokens_per_expert=[128, 26368, 128, 128, 128, 128, 0, 128, 32768, 128, 32768, 128, 128, 128, 128, 128, 32768, 6528, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132992 slack_slots=7040 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 6, 5, 2, 4094, 0, 4092, 1, 4094, 0, 0, 691, 0, 4079, 4079, 0, 4092, 0, 3, 1, 0, 0, 0, 1, 0, 4, 0, 0, 0, 1, 1, 0, 5, 3, 4, 4092, 3, 18, 4092, 1, 4089, 0, 0, 2, 3, 7, 41, 1, 0, 4093, 18, 6, 4090, 4090, 3, 4, 1622, 0, 4093, 4087, 59, 4024, 0, 0, 4092, 1, 0, 3779, 4091, 0, 4067, 1, 1, 0, 3, 0, 2, 4095, 4084, 4092, 1, 0, 3650, 0, 3, 10, 4093, 0, 4092, 2, 0, 9, 0, 2, 0, 0, 2, 0, 0, 3, 4090, 0, 1, 0, 4, 1, 3, 0, 2506, 2, 0, 0, 3, 213, 0, 0, 3, 4089, 2, 4090, 0, 0, 4084, 4096, 3912] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 1, 3816, 0, 0, 2758, 0, 4090, 4095, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 0, 14, 0, 0, 4093, 5, 0, 4096, 4095, 0, 0, 1546, 1, 4096, 4096, 20, 4042, 0, 0, 4096, 0, 0, 4074, 4095, 0, 4091, 0, 0, 0, 1, 0, 0, 4095, 4096, 4095, 0, 0, 3583, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 740, 0, 0, 0, 0, 83, 0, 0, 0, 4096, 0, 4096, 0, 0, 4084, 4096, 3919] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 1, 1, 0, 1, 0, 4090, 0, 4095, 0, 1895, 0, 0, 1313, 0, 4096, 4095, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 5, 4095, 0, 8, 8, 0, 0, 0, 0, 1, 4091, 25, 0, 4095, 4088, 1, 1, 3307, 3, 4095, 4095, 24, 3855, 0, 1, 4095, 0, 1, 4092, 4093, 0, 4069, 1, 1, 0, 0, 0, 0, 4095, 4095, 4095, 0, 0, 3332, 1, 1, 1, 4095, 0, 4095, 0, 0, 0, 1, 1, 2, 0, 0, 1, 0, 0, 4095, 1, 1, 5, 0, 1, 0, 0, 2746, 0, 0, 0, 0, 2, 0, 0, 1, 4095, 1, 4095, 1, 0, 4096, 4095, 3994] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 20, 4063, 0, 4096, 27, 4062, 0, 0, 1468, 0, 4020, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 48, 4096, 0, 0, 51, 0, 0, 21, 0, 0, 4037, 156, 0, 4096, 4096, 0, 0, 1137, 23, 4096, 4096, 42, 3948, 0, 0, 4096, 0, 0, 4095, 4071, 0, 4057, 0, 0, 0, 31, 0, 0, 4096, 4095, 4096, 0, 0, 3997, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1810, 0, 0, 0, 0, 254, 0, 0, 0, 4095, 0, 4094, 0, 0, 4068, 4096, 3638] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4, 0, 5, 5, 0, 3, 0, 4092, 0, 4092, 0, 3750, 0, 6, 626, 4, 4092, 4091, 4, 4089, 5, 7, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 6, 4, 0, 4092, 0, 0, 4086, 0, 4091, 0, 19, 0, 2, 0, 95, 0, 4, 4091, 0, 0, 4091, 4091, 5, 5, 2471, 0, 4091, 4092, 35, 4065, 0, 0, 4089, 3, 4, 4090, 4090, 2, 4065, 4, 5, 0, 0, 5, 0, 4085, 4091, 4091, 2, 0, 3983, 7, 0, 2, 4090, 7, 4090, 0, 4, 0, 3, 7, 12, 0, 0, 7, 0, 5, 4091, 1, 0, 0, 0, 4, 5, 0, 1407, 0, 5, 0, 2, 35, 0, 2, 4, 4091, 3, 4091, 2, 0, 4092, 4091, 3987] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 1, 1, 2, 0, 0, 4093, 0, 4095, 1, 2041, 0, 1, 2718, 1, 4094, 4094, 0, 4094, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4, 0, 0, 0, 0, 0, 4095, 0, 4, 4095, 5, 4095, 1, 7, 11, 1, 1, 0, 0, 0, 4095, 9, 0, 4094, 4091, 1, 1, 2514, 0, 4094, 4095, 12, 4085, 0, 0, 4095, 0, 0, 4053, 4093, 0, 4095, 1, 1, 0, 0, 0, 0, 4090, 4095, 4096, 2, 0, 3612, 1, 0, 0, 4095, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 2, 0, 2, 1, 1801, 2, 1, 2, 0, 31, 0, 1, 1, 4094, 0, 4095, 0, 0, 4080, 4095, 3690] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 4, 4002, 0, 0, 1104, 0, 4083, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 10, 0, 4080, 18, 0, 4096, 4096, 0, 0, 1619, 4, 4096, 4096, 21, 3944, 0, 0, 4096, 0, 0, 4049, 4093, 0, 4086, 0, 0, 0, 7, 0, 0, 4096, 4095, 4095, 0, 0, 4008, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2377, 2, 0, 0, 0, 37, 0, 0, 0, 4096, 0, 4096, 0, 0, 4084, 4096, 3427] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4087, 0, 4092, 6, 3736, 0, 0, 1366, 0, 4092, 4092, 0, 4096, 0, 0, 3, 0, 0, 1, 1, 0, 0, 0, 0, 1, 14, 0, 4, 0, 0, 0, 4095, 3, 0, 4094, 1, 4091, 0, 0, 0, 0, 0, 10, 4, 0, 4096, 3, 0, 4096, 4096, 0, 0, 2058, 0, 4093, 4094, 23, 4076, 0, 0, 4096, 0, 0, 4087, 4092, 0, 4076, 0, 0, 0, 3, 0, 0, 4096, 4092, 4093, 0, 1, 3985, 0, 0, 6, 4093, 0, 4096, 0, 0, 5, 0, 0, 0, 4, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 1656, 3, 0, 0, 1, 79, 0, 0, 0, 4096, 0, 4095, 0, 0, 4067, 4096, 3532] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 6, 5, 2, 4094, 0, 4092, 1, 4094, 0, 0, 691, 0, 2107, 1148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4079, 0, 4092, 0, 3, 1, 0, 0, 0, 1, 0, 4, 0, 0, 0, 2391, 1719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 5, 3, 4, 4092, 3, 18, 4092, 1, 4089, 0, 0, 2, 3, 2618, 1449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 41, 1, 0, 4093, 18, 6, 0, 4090, 3, 4, 1622, 0, 1986, 4087, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 164, 1, 0, 3779, 4091, 0, 4067, 1, 1, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4084, 4092, 1, 0, 3650, 0, 3, 10, 327, 0, 0, 2, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 3, 4090, 0, 1, 0, 4, 1, 3, 0, 2506, 4092, 4090, 2209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 213, 0, 0, 3, 4089, 2, 4090, 0, 0, 4084, 256, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16242, 16369, 16381, 16017, 16133, 16273, 17003, 16654] avg=16384.0000 max=17003 max_rank=6 min=16017 min_rank=3 imbalance_ratio=1.037781 max_load_reduction=114069 max_load_reduction_pct=0.870277 moved_to_echo=66063 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 20, 4063, 0, 4096, 27, 4062, 0, 0, 1468, 0, 2108, 1149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2391, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 48, 4096, 0, 0, 51, 0, 2620, 1449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 4037, 156, 0, 0, 4096, 0, 0, 1137, 23, 1988, 4096, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 0, 0, 165, 0, 0, 4095, 4071, 0, 4057, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 3997, 0, 0, 2, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1810, 4096, 4096, 2211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 0, 4095, 0, 4094, 0, 0, 4068, 256, 3638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16993, 16323, 16466, 15596, 16367, 16613, 16309, 16405] avg=16384.0000 max=16993 max_rank=0 min=15596 min_rank=3 imbalance_ratio=1.037170 max_load_reduction=114079 max_load_reduction_pct=0.870354 moved_to_echo=65694 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 1, 0, 1, 0, 4090, 0, 4095, 0, 1895, 0, 0, 1313, 0, 2108, 1149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2391, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 5, 4095, 0, 8, 8, 0, 2620, 1449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 25, 0, 0, 4088, 1, 1, 3307, 3, 1987, 4095, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3855, 0, 1, 165, 0, 1, 4092, 4093, 0, 4069, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 4095, 0, 0, 3332, 1, 1, 1, 326, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 1, 0, 0, 4095, 1, 1, 5, 0, 1, 0, 0, 2746, 4095, 4095, 2210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 4095, 1, 4095, 1, 0, 4096, 255, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14655, 16401, 16375, 17623, 16278, 15947, 17253, 16540] avg=16384.0000 max=17623 max_rank=3 min=14655 min_rank=0 imbalance_ratio=1.075623 max_load_reduction=113449 max_load_reduction_pct=0.865547 moved_to_echo=66018 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 1, 3816, 0, 0, 2758, 0, 2108, 1149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4095, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2391, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 2620, 1449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4093, 5, 0, 0, 4095, 0, 0, 1546, 1, 1988, 4096, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 165, 0, 0, 4074, 4095, 0, 4091, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4095, 0, 0, 3583, 0, 0, 2, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 740, 4096, 4096, 2211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 4096, 0, 4096, 0, 0, 4084, 256, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18022, 16393, 16360, 15858, 16468, 16198, 15239, 16534] avg=16384.0000 max=18022 max_rank=0 min=15239 min_rank=6 imbalance_ratio=1.099976 max_load_reduction=113050 max_load_reduction_pct=0.862503 moved_to_echo=64439 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 5, 5, 0, 3, 0, 4092, 0, 4092, 0, 3750, 0, 6, 626, 4, 2106, 1148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4091, 4, 4089, 5, 7, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 2388, 1717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 4092, 0, 0, 4086, 0, 4091, 0, 19, 0, 2, 2616, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 4, 4091, 0, 0, 0, 4091, 5, 5, 2471, 0, 1985, 4092, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 165, 3, 4, 4090, 4090, 2, 4065, 4, 5, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4091, 4091, 2, 0, 3983, 7, 0, 2, 326, 7, 0, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 12, 0, 0, 7, 0, 5, 4091, 1, 0, 0, 0, 4, 5, 0, 1407, 4090, 4091, 2207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 35, 0, 2, 4, 4091, 3, 4091, 2, 0, 4092, 256, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15841, 16398, 16363, 16874, 16498, 16601, 15927, 16570] avg=16384.0000 max=16874 max_rank=3 min=15841 min_rank=0 imbalance_ratio=1.029907 max_load_reduction=114198 max_load_reduction_pct=0.871262 moved_to_echo=65596 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 4, 4002, 0, 0, 1104, 0, 2112, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2396, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 2625, 1454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4080, 18, 0, 0, 4096, 0, 0, 1619, 4, 1984, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3944, 0, 0, 157, 0, 0, 4049, 4093, 0, 4086, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4095, 0, 0, 4008, 0, 0, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2377, 4096, 4096, 2215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 37, 0, 0, 0, 4096, 0, 4096, 0, 0, 4084, 246, 3427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16564, 16394, 16371, 15928, 16336, 16611, 16880, 15988] avg=16384.0000 max=16880 max_rank=6 min=15928 min_rank=3 imbalance_ratio=1.030273 max_load_reduction=114192 max_load_reduction_pct=0.871216 moved_to_echo=65815 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4087, 0, 4092, 6, 3736, 0, 0, 1366, 0, 2107, 1148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4092, 0, 4096, 0, 0, 3, 0, 0, 1, 1, 0, 0, 0, 0, 1, 2391, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4, 0, 0, 0, 4095, 3, 0, 4094, 1, 4091, 0, 0, 0, 0, 2618, 1449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4, 0, 4096, 3, 0, 0, 4096, 0, 0, 2058, 0, 1986, 4094, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 165, 0, 0, 4087, 4092, 0, 4076, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 4093, 0, 1, 3985, 0, 0, 6, 327, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 1656, 4096, 4096, 2211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 79, 0, 0, 0, 4096, 0, 4095, 0, 0, 4067, 256, 3532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16543, 16397, 16369, 16370, 16499, 16605, 16160, 16129] avg=16384.0000 max=16605 max_rank=5 min=16129 min_rank=7 imbalance_ratio=1.013489 max_load_reduction=114467 max_load_reduction_pct=0.873314 moved_to_echo=65393 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 1, 2, 0, 0, 4093, 0, 4095, 1, 2041, 0, 1, 2718, 1, 2107, 1149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 4094, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2391, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4095, 0, 4, 4095, 5, 4095, 1, 7, 11, 1, 2620, 1449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 9, 0, 0, 4091, 1, 1, 2514, 0, 1987, 4095, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 165, 0, 0, 4053, 4093, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4095, 4096, 2, 0, 3612, 1, 0, 0, 326, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 2, 0, 2, 1, 1801, 4095, 4094, 2210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 0, 31, 0, 1, 1, 4094, 0, 4095, 0, 0, 4080, 255, 3690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16212, 16397, 16387, 16806, 16493, 16224, 16301, 16252] avg=16384.0000 max=16806 max_rank=3 min=16212 min_rank=0 imbalance_ratio=1.025757 max_load_reduction=114266 max_load_reduction_pct=0.871780 moved_to_echo=65270 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 32768, 128, 32768, 128, 128, 128, 0, 0, 128, 128, 0, 128, 0, 128, 128, 19200, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 256, 128, 128, 32768, 256, 128, 0, 32768, 128, 128, 16384, 128, 16000, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 768, 0, 128, 128, 32768, 128, 32768, 128, 0, 32768, 2048, 30208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 32768, 0, 32768, 128, 27520, 0, 128, 12160, 128, 16896, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32128, 0, 128, 1408, 128, 128, 32384, 32768, 128, 32640, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 32768, 128, 128, 128, 128, 20992, 11648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 32768, 32768, 128, 128, 30208, 128, 128, 128, 2688, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 32768, 128, 128, 128, 128, 128, 128, 128, 15104, 32768, 32768, 17792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 3, 8, 0, 4095, 3, 3, 4096, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 4095, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 4089, 5, 4089, 0, 4095, 4096, 0, 0, 4095, 0, 2, 0, 1, 0, 4093, 4087, 2, 0, 183, 0, 4095, 0, 4095, 4, 4094, 0, 0, 4076, 5, 4095, 0, 0, 0, 0, 0, 0, 4079, 4090, 0, 1, 4, 4094, 4093, 0, 4090, 0, 0, 2, 0, 1, 0, 0, 0, 0, 1, 1, 0, 4092, 0, 3, 4093, 2, 0, 4093, 0, 3, 0, 4096, 4087, 0, 0, 0, 0, 1, 26, 0, 0, 0, 0, 3910, 1, 4096, 4093, 16, 2, 0, 1, 4095, 4095, 4, 2, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 4, 10, 6, 5, 0, 0, 0, 4087, 5, 0, 4091, 2, 6, 0, 24, 0, 5, 0, 7, 0, 8, 5, 4, 4090, 4, 0, 3, 0, 5, 4090, 4, 0, 0, 1, 4091, 4073, 0, 4091, 5, 4089, 4091, 0, 0, 4091, 1, 0, 0, 3, 5, 4090, 4091, 5, 0, 3, 4, 4090, 5, 4087, 0, 4090, 0, 19, 4062, 3, 4090, 33, 0, 5, 0, 0, 2, 4040, 4090, 0, 5, 0, 4091, 4091, 0, 4090, 0, 4, 5, 4, 0, 5, 0, 0, 0, 6, 0, 5, 4084, 0, 0, 4091, 5, 3, 4091, 0, 3, 0, 4085, 4091, 3, 5, 0, 6, 1, 8, 5, 0, 15, 4, 4084, 0, 4086, 4088, 8, 0, 0, 5, 4088, 4091, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1, 0, 0, 0, 4091, 1, 0, 4092, 0, 0, 0, 1, 1, 1, 0, 0, 2, 0, 1, 1, 4094, 1, 0, 1, 0, 0, 4095, 0, 1, 0, 0, 4095, 4088, 0, 4095, 1, 4095, 4095, 0, 1, 4095, 0, 0, 1, 0, 1, 4095, 4095, 1, 0, 136, 0, 4035, 1, 4092, 0, 4095, 0, 41, 4095, 5, 4094, 0, 1, 0, 0, 0, 0, 4013, 4095, 0, 1, 0, 4090, 4094, 0, 4095, 0, 0, 1, 1, 0, 1, 0, 3, 0, 1, 4, 1, 4095, 0, 0, 4092, 1, 2, 4095, 0, 0, 0, 4094, 4095, 0, 1, 0, 1, 1, 0, 1, 8, 3, 1, 3952, 0, 4095, 4095, 110, 0, 1, 1, 4095, 4095, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4076, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 462, 0, 3950, 0, 4096, 163, 4096, 0, 5, 4096, 43, 4096, 0, 0, 0, 0, 0, 0, 4040, 4096, 0, 0, 0, 4091, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 4096, 2, 0, 4064, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 67, 0, 47, 0, 0, 3633, 0, 4096, 4096, 41, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 4090, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 629, 0, 4067, 0, 4096, 8, 4096, 0, 4, 4096, 13, 4096, 0, 0, 0, 0, 0, 0, 4076, 4096, 0, 0, 0, 4090, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 25, 0, 6, 0, 0, 3466, 0, 4096, 4096, 17, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 15, 0, 4089, 0, 4096, 1, 4096, 0, 0, 4096, 20, 4096, 0, 0, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 4095, 4096, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 1, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 3939, 0, 4096, 4096, 145, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 0, 0, 0, 4095, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 0, 2, 0, 0, 0, 1, 4094, 1, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 4094, 4087, 0, 4095, 1, 4095, 4094, 1, 0, 4093, 0, 0, 2, 0, 1, 4095, 4092, 1, 0, 252, 3, 4093, 1, 4094, 5, 4095, 0, 0, 4092, 9, 4095, 0, 0, 0, 0, 0, 0, 4043, 4094, 1, 1, 0, 4095, 4095, 0, 4095, 0, 2, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 4091, 0, 0, 4095, 1, 0, 4095, 0, 0, 1, 4095, 4095, 1, 1, 0, 0, 0, 1, 1, 1, 4, 1, 3819, 0, 4095, 4095, 77, 0, 0, 1, 4095, 4094, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 4087, 0, 4095, 0, 4096, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 49, 0, 4057, 0, 4096, 5, 4096, 0, 0, 4096, 50, 4096, 0, 0, 0, 0, 1, 0, 4010, 4096, 0, 0, 0, 4090, 4088, 7, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 4, 10, 4093, 0, 0, 4096, 0, 0, 1, 4096, 4093, 0, 0, 0, 0, 0, 30, 0, 6, 0, 0, 3981, 0, 4096, 4096, 118, 0, 0, 0, 4096, 4096, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 10, 6, 5, 0, 0, 0, 4087, 5, 0, 4091, 2, 6, 0, 24, 4085, 4078, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7, 0, 8, 5, 4, 4090, 4, 0, 3, 0, 5, 4090, 4, 4091, 4090, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4073, 0, 4091, 5, 4089, 13, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 0, 4091, 5, 0, 3, 4, 4090, 5, 3885, 0, 0, 0, 19, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 32, 33, 0, 5, 0, 0, 2, 4040, 4090, 0, 5, 0, 4091, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4, 5, 4, 0, 5, 0, 0, 0, 6, 0, 5, 4084, 0, 0, 4036, 3920, 202, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 5, 3, 4078, 0, 3, 0, 4085, 4091, 3, 5, 0, 6, 1, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4, 4084, 0, 4086, 4088, 8, 0, 0, 5, 4088, 171, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16425, 16411, 16364, 16172, 16392, 16374, 16384, 16550] avg=16384.0000 max=16550 max_rank=7 min=16172 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=114522 max_load_reduction_pct=0.873734 moved_to_echo=65700 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 3, 8, 0, 4095, 3, 3, 4096, 0, 1, 0, 0, 4088, 4083, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 4095, 2, 0, 0, 0, 0, 4095, 0, 4095, 4094, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 4089, 0, 4095, 13, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4087, 2, 0, 183, 0, 4095, 0, 3892, 4, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 32, 0, 0, 0, 0, 0, 0, 4079, 4090, 0, 1, 4, 4094, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 2, 0, 1, 0, 0, 0, 0, 1, 1, 0, 4092, 0, 3, 4041, 3924, 203, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 4080, 0, 3, 0, 4096, 4087, 0, 0, 0, 0, 1, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3910, 1, 4096, 4093, 16, 2, 0, 1, 4095, 171, 4, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16404, 16389, 16388, 16340, 16398, 16371, 16388, 16394] avg=16384.0000 max=16404 max_rank=0 min=16340 min_rank=3 imbalance_ratio=1.001221 max_load_reduction=114668 max_load_reduction_pct=0.874847 moved_to_echo=65551 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1, 0, 0, 0, 4091, 1, 0, 4092, 0, 0, 0, 1, 4090, 4082, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 2, 0, 1, 1, 4094, 1, 0, 1, 0, 0, 4095, 0, 4095, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4088, 0, 4095, 1, 4095, 13, 0, 1, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 136, 0, 4035, 1, 3890, 0, 0, 0, 41, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 32, 0, 1, 0, 0, 0, 0, 4013, 4095, 0, 1, 0, 4090, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 1, 0, 1, 0, 3, 0, 1, 4, 1, 4095, 0, 0, 4040, 3924, 202, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 2, 4081, 0, 0, 0, 4094, 4095, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 1, 3952, 0, 4095, 4095, 110, 0, 1, 1, 4095, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16392, 16390, 16295, 16331, 16382, 16369, 16532] avg=16384.0000 max=16532 max_rank=7 min=16295 min_rank=3 imbalance_ratio=1.009033 max_load_reduction=114540 max_load_reduction_pct=0.873871 moved_to_echo=65614 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 4096, 0, 0, 0, 0, 4091, 4083, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 4096, 0, 4096, 13, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 462, 0, 3950, 0, 3893, 163, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 32, 0, 0, 0, 0, 0, 0, 4040, 4096, 0, 0, 0, 4091, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 4096, 2, 0, 4042, 3925, 203, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 4082, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 3633, 0, 4096, 4096, 41, 0, 0, 0, 4096, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16318, 16332, 16377, 16665, 16397, 16398, 16405, 16180] avg=16384.0000 max=16665 max_rank=3 min=16180 min_rank=7 imbalance_ratio=1.017151 max_load_reduction=114407 max_load_reduction_pct=0.872856 moved_to_echo=65380 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 4092, 4087, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4090, 0, 4096, 0, 4096, 9, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 629, 0, 4067, 0, 3891, 8, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 25, 0, 0, 0, 0, 0, 0, 4076, 4096, 0, 0, 0, 4090, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 4047, 3926, 205, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4080, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3466, 0, 4096, 4096, 17, 0, 0, 0, 4096, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16391, 16385, 16388, 16791, 16392, 16392, 16386, 15947] avg=16384.0000 max=16791 max_rank=3 min=15947 min_rank=7 imbalance_ratio=1.024841 max_load_reduction=114281 max_load_reduction_pct=0.871895 moved_to_echo=65117 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 0, 0, 4091, 4083, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 1, 0, 4096, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4095, 0, 4096, 13, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 49, 0, 4057, 0, 3893, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 32, 0, 0, 0, 0, 1, 0, 4010, 4096, 0, 0, 0, 4090, 4088, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 4, 10, 4042, 3925, 203, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4082, 0, 0, 1, 4096, 4093, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3981, 0, 4096, 4096, 118, 0, 0, 0, 4096, 171, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16388, 16381, 16189, 16374, 16399, 16395, 16566] avg=16384.0000 max=16566 max_rank=7 min=16189 min_rank=3 imbalance_ratio=1.011108 max_load_reduction=114506 max_load_reduction_pct=0.873611 moved_to_echo=65734 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 1, 4096, 0, 0, 0, 0, 4091, 4082, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 13, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 15, 0, 4089, 0, 3893, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 32, 0, 0, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 4095, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 1, 0, 4042, 3925, 203, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4082, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3939, 0, 4096, 4096, 145, 0, 0, 0, 4096, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16385, 16397, 16189, 16418, 16380, 16373, 16544] avg=16384.0000 max=16544 max_rank=7 min=16189 min_rank=3 imbalance_ratio=1.009766 max_load_reduction=114528 max_load_reduction_pct=0.873779 moved_to_echo=65715 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 0, 0, 0, 4095, 1, 0, 4095, 0, 1, 0, 0, 4090, 4081, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 1, 4094, 1, 1, 1, 0, 0, 4095, 0, 4094, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4095, 1, 4095, 13, 1, 0, 4093, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 1, 0, 252, 3, 4093, 1, 3891, 5, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 32, 0, 0, 0, 0, 0, 0, 4043, 4094, 1, 1, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 4091, 0, 0, 4041, 3923, 203, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 4081, 0, 0, 1, 4095, 4095, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 1, 3819, 0, 4095, 4095, 77, 0, 0, 1, 4095, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16390, 16387, 16431, 16370, 16376, 16372, 16359] avg=16384.0000 max=16431 max_rank=3 min=16359 min_rank=7 imbalance_ratio=1.002869 max_load_reduction=114641 max_load_reduction_pct=0.874641 moved_to_echo=65477 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 0, 32768, 128, 128, 32768, 128, 128, 0, 128, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 1792, 128, 32512, 128, 31232, 256, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 32768, 128, 128, 32384, 31488, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132992 observed_loss=-1920 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 128, 128, 32768, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 32768, 0, 128, 128, 32768, 32768, 128, 128, 0, 128, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 256, 128, 128, 128, 0, 128, 128, 32384, 32768, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 32768, 128, 32768, 128, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 30848, 128, 32768, 32768, 640, 128, 128, 128, 32768, 1408, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132992 slack_slots=7040 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[2203, 4095, 4092, 1, 1, 0, 0, 3, 21, 0, 0, 273, 0, 0, 0, 0, 0, 2158, 1, 0, 2, 860, 0, 4095, 2, 1, 0, 0, 1, 4095, 0, 4095, 4, 3283, 4095, 0, 4095, 1, 0, 1, 0, 4095, 0, 4095, 0, 4095, 0, 4095, 4095, 4092, 1, 0, 0, 4093, 1, 0, 4044, 1, 1, 0, 1, 1, 0, 4091, 0, 0, 0, 4093, 1, 10, 1, 1, 0, 4095, 1, 1, 1, 1, 1, 4095, 1, 1, 4095, 0, 0, 0, 5, 0, 0, 4095, 4095, 1, 0, 1, 0, 4095, 4095, 1, 0, 4071, 1, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 4095, 3533, 0, 1, 0, 1, 12, 1, 0, 0, 4095, 0, 4, 0, 4092, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[537, 4096, 4084, 0, 0, 0, 0, 0, 288, 0, 5, 109, 0, 0, 1, 0, 0, 1872, 0, 0, 0, 2333, 0, 4091, 3, 0, 23, 0, 0, 4088, 0, 4094, 0, 3639, 4096, 0, 4095, 2, 8, 0, 0, 4096, 0, 4090, 0, 4096, 0, 4092, 4091, 4095, 1, 0, 0, 4096, 0, 4, 4069, 1, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4085, 0, 14, 0, 0, 1, 4092, 0, 0, 0, 0, 0, 4086, 0, 11, 4094, 0, 0, 2, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 4094, 4094, 1, 10, 3757, 1, 4095, 3, 0, 0, 4, 3, 2, 0, 0, 0, 0, 4096, 3854, 12, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 5, 0, 4084, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1336, 4096, 4048, 0, 0, 0, 0, 0, 63, 0, 0, 265, 0, 0, 0, 0, 0, 2002, 0, 36, 0, 2719, 0, 4096, 0, 0, 91, 0, 0, 4096, 0, 4096, 0, 3424, 4096, 0, 4096, 33, 1, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 101, 0, 0, 4096, 0, 0, 3382, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 52, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 1, 3981, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3126, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4064, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[99, 4089, 4091, 6, 7, 4, 15, 4, 40, 5, 0, 0, 0, 0, 0, 0, 1, 2232, 5, 0, 0, 2178, 0, 4089, 0, 5, 0, 0, 5, 4090, 6, 4090, 1, 3698, 4086, 4, 4089, 0, 0, 6, 7, 4090, 0, 4090, 1, 4088, 0, 4090, 4090, 4090, 0, 0, 0, 4080, 5, 0, 4088, 6, 6, 0, 5, 5, 0, 4088, 0, 0, 0, 4092, 6, 0, 4, 5, 2, 4088, 0, 4, 3, 0, 7, 4091, 4, 0, 4088, 3, 0, 0, 5, 0, 0, 4090, 4090, 6, 0, 6, 0, 4090, 4090, 6, 0, 4049, 3, 4089, 0, 0, 0, 8, 0, 0, 6, 1, 7, 3, 4089, 4056, 0, 7, 0, 5, 1, 5, 0, 4, 4089, 0, 0, 0, 4091, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[478, 4095, 4094, 1, 1, 0, 0, 0, 34, 0, 0, 30, 0, 0, 0, 0, 0, 2586, 0, 0, 0, 1775, 0, 4094, 0, 1, 3, 0, 1, 4093, 1, 4095, 0, 3825, 4095, 0, 4095, 1, 0, 1, 1, 4094, 0, 4094, 0, 4095, 0, 4094, 4095, 4094, 0, 1, 0, 4095, 1, 0, 4083, 1, 1, 0, 1, 1, 2, 4094, 0, 1, 0, 4079, 1, 0, 1, 2, 2, 4095, 1, 1, 1, 4, 1, 4095, 1, 1, 4095, 0, 1, 0, 2, 0, 1, 4095, 4095, 1, 0, 1, 0, 4094, 4093, 1, 0, 4080, 0, 4095, 1, 0, 2, 6, 0, 0, 0, 0, 2, 0, 4095, 3585, 0, 1, 1, 1, 1, 1, 0, 0, 4095, 0, 0, 1, 4093, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[588, 4096, 4089, 0, 0, 0, 0, 0, 15, 0, 0, 69, 0, 0, 0, 0, 0, 1354, 0, 12, 0, 3278, 0, 4096, 0, 0, 7, 0, 0, 4096, 0, 4096, 0, 3244, 4096, 0, 4096, 4, 4, 0, 0, 4096, 0, 4094, 0, 4096, 0, 4096, 4096, 4096, 4, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4081, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 19, 0, 4095, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4081, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3739, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[802, 4096, 4088, 0, 0, 0, 0, 0, 68, 0, 0, 48, 0, 0, 0, 1, 0, 2266, 0, 5, 0, 1415, 0, 4096, 4, 0, 9, 0, 0, 4095, 0, 4096, 0, 3773, 4096, 0, 4096, 4, 4, 0, 0, 4096, 21, 4091, 0, 4096, 0, 4094, 4095, 4096, 6, 0, 0, 4096, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 20, 0, 4086, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3, 4057, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3991, 9, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[522, 4096, 4095, 0, 0, 0, 0, 0, 53, 0, 0, 19, 0, 0, 0, 0, 0, 3093, 0, 0, 0, 1663, 0, 4096, 0, 0, 4, 0, 0, 4096, 0, 4096, 0, 3860, 4096, 0, 4096, 1, 1, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 3, 0, 0, 4096, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4085, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3105, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2203, 4095, 4092, 1, 1, 0, 0, 3, 21, 0, 0, 273, 0, 0, 0, 0, 4095, 3107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2158, 1, 0, 2, 860, 0, 4095, 2, 1, 0, 0, 1, 4095, 0, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3283, 4095, 0, 496, 1, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 103, 1, 0, 0, 4093, 1, 0, 4044, 1, 1, 0, 1, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 10, 1, 1, 0, 4095, 1, 1, 1, 1, 1, 4095, 3286, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 5, 0, 0, 4087, 4095, 1, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 4071, 1, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 3175, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3533, 0, 1, 0, 1, 12, 1, 0, 0, 4095, 0, 4, 0, 4092, 1, 313, 150, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17891, 15160, 16070, 16432, 16401, 16381, 16431, 16306] avg=16384.0000 max=17891 max_rank=0 min=15160 min_rank=1 imbalance_ratio=1.091980 max_load_reduction=113181 max_load_reduction_pct=0.863503 moved_to_echo=65519 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[537, 4096, 4084, 0, 0, 0, 0, 0, 288, 0, 5, 109, 0, 0, 1, 0, 4096, 3108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1872, 0, 0, 0, 2333, 0, 4091, 3, 0, 23, 0, 0, 4088, 0, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3639, 4096, 0, 496, 2, 8, 0, 0, 0, 0, 4090, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 104, 1, 0, 0, 4096, 0, 4, 4069, 1, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 14, 0, 0, 1, 4092, 0, 0, 0, 0, 0, 4086, 3286, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4094, 0, 0, 2, 0, 0, 0, 4088, 4095, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 10, 3757, 1, 4095, 3, 0, 0, 4, 3, 2, 0, 0, 0, 0, 3177, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3854, 12, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 5, 0, 4084, 1, 313, 150, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16324, 16354, 16423, 16456, 16378, 16384, 16135, 16618] avg=16384.0000 max=16618 max_rank=7 min=16135 min_rank=6 imbalance_ratio=1.014282 max_load_reduction=114454 max_load_reduction_pct=0.873215 moved_to_echo=65515 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[99, 4089, 4091, 6, 7, 4, 15, 4, 40, 5, 0, 0, 0, 0, 0, 0, 4090, 3102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2232, 5, 0, 0, 2178, 0, 4089, 0, 5, 0, 0, 5, 4090, 6, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3698, 4086, 4, 496, 0, 0, 6, 7, 0, 0, 4090, 1, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 104, 0, 0, 0, 4080, 5, 0, 4088, 6, 6, 0, 5, 5, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 6, 0, 4, 5, 2, 4088, 0, 4, 3, 0, 7, 4091, 3281, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4088, 3, 0, 0, 5, 0, 0, 4082, 4090, 6, 0, 6, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 6, 0, 4049, 3, 4089, 0, 0, 0, 8, 0, 0, 6, 1, 7, 3, 3173, 986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4056, 0, 7, 0, 5, 1, 5, 0, 4, 4089, 0, 0, 0, 4091, 5, 312, 149, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15552, 16552, 16479, 16477, 16396, 16374, 16421, 16821] avg=16384.0000 max=16821 max_rank=7 min=15552 min_rank=0 imbalance_ratio=1.026672 max_load_reduction=114251 max_load_reduction_pct=0.871666 moved_to_echo=66012 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1336, 4096, 4048, 0, 0, 0, 0, 0, 63, 0, 0, 265, 0, 0, 0, 0, 4096, 3108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2002, 0, 36, 0, 2719, 0, 4096, 0, 0, 91, 0, 0, 4096, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3424, 4096, 0, 496, 33, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 104, 101, 0, 0, 4096, 0, 0, 3382, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3287, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 4096, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 3981, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3178, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3126, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4064, 0, 313, 150, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17012, 16986, 16242, 15864, 16343, 16428, 16340, 15857] avg=16384.0000 max=17012 max_rank=0 min=15857 min_rank=7 imbalance_ratio=1.038330 max_load_reduction=114060 max_load_reduction_pct=0.870209 moved_to_echo=64968 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[802, 4096, 4088, 0, 0, 0, 0, 0, 68, 0, 0, 48, 0, 0, 0, 1, 4096, 3108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2266, 0, 5, 0, 1415, 0, 4096, 4, 0, 9, 0, 0, 4095, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3773, 4096, 0, 496, 4, 4, 0, 0, 0, 21, 4091, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 104, 6, 0, 0, 4096, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 20, 0, 4086, 3287, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 4057, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3178, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3991, 9, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 313, 150, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16307, 15836, 16579, 16418, 16379, 16379, 16419, 16755] avg=16384.0000 max=16755 max_rank=7 min=15836 min_rank=1 imbalance_ratio=1.022644 max_load_reduction=114317 max_load_reduction_pct=0.872169 moved_to_echo=65932 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[588, 4096, 4089, 0, 0, 0, 0, 0, 15, 0, 0, 69, 0, 0, 0, 0, 4096, 3109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1354, 0, 12, 0, 3278, 0, 4096, 0, 0, 7, 0, 0, 4096, 0, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3244, 4096, 0, 488, 4, 4, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 97, 4, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 19, 0, 4095, 3292, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4081, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4081, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3180, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3739, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 316, 152, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16062, 16787, 16026, 16478, 16403, 16371, 16440, 16505] avg=16384.0000 max=16787 max_rank=1 min=16026 min_rank=2 imbalance_ratio=1.024597 max_load_reduction=114285 max_load_reduction_pct=0.871925 moved_to_echo=65719 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[522, 4096, 4095, 0, 0, 0, 0, 0, 53, 0, 0, 19, 0, 0, 0, 0, 4096, 3109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3093, 0, 0, 0, 1663, 0, 4096, 0, 0, 4, 0, 0, 4096, 0, 3946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3860, 4096, 0, 496, 1, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 104, 3, 0, 0, 4096, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 4094, 3287, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3178, 987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3105, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 313, 150, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15990, 16898, 16646, 16468, 16388, 16376, 16442, 15864] avg=16384.0000 max=16898 max_rank=1 min=15864 min_rank=7 imbalance_ratio=1.031372 max_load_reduction=114174 max_load_reduction_pct=0.871078 moved_to_echo=65070 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[478, 4095, 4094, 1, 1, 0, 0, 0, 34, 0, 0, 30, 0, 0, 0, 0, 4094, 3107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2586, 0, 0, 0, 1775, 0, 4094, 0, 1, 3, 0, 1, 4093, 1, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 4095, 0, 496, 1, 0, 1, 1, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 104, 0, 1, 0, 4095, 1, 0, 4083, 1, 1, 0, 1, 1, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4079, 1, 0, 1, 2, 2, 4095, 1, 1, 1, 4, 1, 4095, 3286, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 1, 0, 2, 0, 1, 4087, 4095, 1, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 4080, 0, 4095, 1, 0, 2, 6, 0, 0, 0, 0, 2, 0, 3176, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3585, 0, 1, 1, 1, 1, 1, 0, 0, 4095, 0, 0, 1, 4093, 1, 313, 150, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15934, 16499, 16607, 16479, 16384, 16379, 16444, 16346] avg=16384.0000 max=16607 max_rank=2 min=15934 min_rank=0 imbalance_ratio=1.013611 max_load_reduction=114465 max_load_reduction_pct=0.873299 moved_to_echo=65553 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 17664, 128, 128, 128, 16256, 0, 32768, 128, 128, 256, 0, 128, 32768, 128, 31616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 128, 0, 128, 32768, 32768, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 896, 128, 128, 0, 32768, 128, 128, 31872, 128, 128, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 29056, 128, 128, 128, 128, 128, 128, 0, 128, 32768, 128, 128, 128, 32768, 128, 2560, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[6656, 32768, 32768, 128, 128, 128, 128, 128, 640, 128, 128, 896, 0, 0, 128, 128, 32768, 24960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 28800, 32768, 128, 3968, 128, 128, 128, 128, 0, 128, 32768, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[32768, 128, 128, 32256, 128, 32768, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 25472, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 0, 32768, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 128, 128, 32768, 26368, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4082, 441, 0, 4092, 0, 0, 0, 1, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 2, 48, 0, 0, 0, 4067, 4095, 5, 4092, 0, 2475, 0, 0, 1, 4083, 0, 0, 4094, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 4090, 0, 27, 4096, 0, 4096, 0, 4093, 1057, 3, 4048, 4086, 10, 0, 0, 5, 15, 0, 1, 34, 4095, 14, 8, 293, 4096, 4096, 0, 0, 4096, 1, 5, 0, 0, 0, 0, 0, 0, 4092, 4092, 6, 3, 0, 0, 0, 0, 0, 0, 0, 1, 3980, 0, 0, 4093, 0, 0, 0, 0, 4082, 0, 4069, 0, 4034, 0, 12, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[5, 0, 4089, 1410, 6, 4090, 7, 0, 14, 0, 6, 0, 4090, 0, 0, 4067, 0, 0, 5, 0, 6, 6, 4088, 6, 6, 6, 4084, 0, 2, 0, 7, 4, 0, 4060, 4084, 0, 4088, 5, 2344, 0, 7, 5, 4090, 0, 6, 4079, 8, 8, 0, 0, 4058, 0, 0, 5, 0, 0, 6, 0, 4089, 4090, 4, 0, 4087, 7, 0, 4090, 4, 4084, 0, 4090, 408, 0, 4088, 4091, 5, 13, 7, 10, 0, 0, 0, 0, 4071, 0, 0, 0, 4090, 4088, 0, 4, 4086, 0, 0, 4, 0, 6, 0, 2, 6, 4090, 4092, 0, 1, 2, 1, 0, 5, 6, 0, 0, 0, 4092, 0, 0, 4089, 5, 4, 7, 14, 4091, 5, 4091, 4, 4092, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 4095, 395, 1, 4095, 1, 0, 0, 0, 1, 0, 4095, 0, 0, 4094, 0, 0, 1, 0, 1, 1, 4095, 1, 0, 1, 4095, 1, 0, 11, 1, 0, 0, 4094, 4095, 0, 4094, 1, 2249, 0, 1, 0, 4095, 0, 1, 4095, 1, 1, 8, 1, 4088, 0, 0, 1, 0, 0, 0, 0, 4095, 4095, 1, 0, 4094, 1, 8, 4095, 1, 4095, 1, 4095, 1488, 0, 4023, 4095, 5, 2, 1, 0, 0, 0, 0, 0, 4089, 0, 0, 126, 4095, 4095, 0, 0, 4094, 0, 0, 1, 0, 1, 0, 0, 1, 4095, 4095, 4, 0, 1, 0, 0, 1, 1, 0, 1, 0, 4086, 0, 0, 4095, 1, 1, 1, 1, 4095, 1, 4095, 1, 3989, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 729, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4085, 4096, 0, 4096, 0, 2856, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 6, 4096, 0, 4096, 0, 4096, 573, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 42, 4096, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4092, 0, 4003, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4087, 1983, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 4095, 4096, 0, 4096, 0, 950, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 71, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 64, 4096, 0, 4096, 0, 4096, 612, 0, 4018, 4096, 66, 0, 0, 0, 2, 0, 0, 38, 4096, 6, 5, 679, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4084, 0, 3874, 0, 12, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4095, 1923, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 4095, 4096, 0, 4096, 0, 443, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 4096, 0, 38, 4096, 0, 4096, 0, 4096, 1460, 0, 4088, 4096, 5, 0, 0, 0, 3, 0, 0, 14, 4096, 6, 0, 340, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4078, 0, 4005, 0, 3, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 123, 0, 4092, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 0, 1980, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 3, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 4095, 0, 26, 4096, 0, 4096, 0, 4096, 1977, 3, 4093, 4091, 4, 0, 0, 0, 0, 0, 0, 20, 4094, 9, 0, 150, 4096, 4093, 7, 0, 4096, 0, 2, 0, 0, 0, 4, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 4086, 0, 3951, 0, 7, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 4091, 777, 1, 4095, 2, 0, 0, 1, 1, 0, 4095, 0, 0, 4093, 0, 0, 1, 0, 1, 1, 4095, 1, 0, 1, 4094, 1, 0, 0, 2, 0, 0, 4090, 4095, 0, 4094, 1, 3045, 0, 0, 2, 4095, 0, 1, 4094, 1, 1, 0, 0, 4045, 1, 0, 1, 0, 1, 0, 0, 4095, 4094, 0, 0, 4093, 2, 0, 4095, 1, 4095, 0, 4094, 305, 0, 4093, 4095, 1, 1, 2, 0, 3, 0, 0, 0, 4095, 0, 0, 29, 4094, 4095, 0, 0, 4094, 1, 0, 2, 0, 1, 0, 0, 1, 4095, 4094, 0, 0, 1, 2, 0, 1, 1, 0, 0, 4, 4094, 0, 0, 4095, 1, 1, 1, 3, 4095, 1, 4095, 1, 4084, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 4089, 1410, 6, 4090, 7, 0, 14, 0, 6, 0, 3126, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 6, 4088, 6, 6, 6, 4084, 0, 2, 0, 7, 4, 4090, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 2061, 5, 2344, 0, 7, 5, 4090, 0, 6, 4079, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 5, 0, 0, 6, 0, 4089, 4090, 4, 0, 4087, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4084, 0, 3098, 408, 0, 4088, 4091, 5, 13, 7, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 3870, 4088, 0, 4, 4086, 0, 0, 4, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 4090, 4092, 0, 1, 2, 1, 0, 5, 6, 0, 0, 0, 4092, 2027, 992, 964, 124, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 4, 7, 14, 4091, 5, 4091, 4, 4092, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16820, 16382, 16673, 16347, 15808, 16129, 16416, 16497] avg=16384.0000 max=16820 max_rank=0 min=15808 min_rank=4 imbalance_ratio=1.026611 max_load_reduction=114252 max_load_reduction_pct=0.871674 moved_to_echo=64850 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4082, 441, 0, 4092, 0, 0, 0, 1, 0, 0, 3127, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 2, 48, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 5, 2063, 0, 2475, 0, 0, 1, 4083, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 4090, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 4096, 0, 3100, 1057, 3, 4048, 4086, 10, 0, 0, 5, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 34, 4095, 14, 8, 293, 3874, 4096, 0, 0, 4096, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4092, 6, 3, 0, 0, 0, 0, 0, 0, 0, 1, 3980, 2029, 993, 965, 125, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4082, 0, 4069, 0, 4034, 0, 12, 2, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15839, 16418, 16788, 16378, 16447, 16517, 16298, 16387] avg=16384.0000 max=16788 max_rank=2 min=15839 min_rank=0 imbalance_ratio=1.024658 max_load_reduction=114284 max_load_reduction_pct=0.871918 moved_to_echo=65649 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4095, 395, 1, 4095, 1, 0, 0, 0, 1, 0, 3130, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 4095, 1, 0, 1, 4095, 1, 0, 11, 1, 0, 4095, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2064, 1, 2249, 0, 1, 0, 4095, 0, 1, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 4088, 0, 0, 1, 0, 0, 0, 0, 4095, 4095, 1, 0, 4094, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 4095, 1, 3101, 1488, 0, 4023, 4095, 5, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 126, 3873, 4095, 0, 0, 4094, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4095, 4, 0, 1, 0, 0, 1, 1, 0, 1, 0, 4086, 2030, 994, 965, 125, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 1, 1, 4095, 1, 4095, 1, 3989, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15813, 16386, 16602, 16386, 16820, 16279, 16411, 16375] avg=16384.0000 max=16820 max_rank=4 min=15813 min_rank=0 imbalance_ratio=1.026611 max_load_reduction=114252 max_load_reduction_pct=0.871674 moved_to_echo=65885 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 123, 0, 4092, 0, 0, 0, 0, 0, 0, 3127, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 2066, 0, 1980, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 4096, 0, 3102, 1977, 3, 4093, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4094, 9, 0, 150, 3874, 4093, 7, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 2030, 994, 965, 125, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 4086, 0, 3951, 0, 7, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15534, 16376, 16329, 16386, 17392, 16345, 16385, 16325] avg=16384.0000 max=17392 max_rank=4 min=15534 min_rank=0 imbalance_ratio=1.061523 max_load_reduction=113680 max_load_reduction_pct=0.867310 moved_to_echo=66447 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4087, 1983, 0, 4096, 0, 0, 0, 0, 0, 0, 3130, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2066, 0, 950, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 4096, 0, 3102, 612, 0, 4018, 4096, 66, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4096, 6, 5, 679, 3874, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 2030, 994, 966, 125, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4084, 0, 3874, 0, 12, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17392, 16380, 15298, 16455, 16056, 16890, 16344, 16257] avg=16384.0000 max=17392 max_rank=0 min=15298 min_rank=2 imbalance_ratio=1.061523 max_load_reduction=113680 max_load_reduction_pct=0.867310 moved_to_echo=65547 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 1923, 0, 4094, 0, 0, 0, 0, 0, 0, 3127, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2062, 0, 443, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 4096, 0, 3098, 1460, 0, 4088, 4096, 5, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 6, 0, 340, 3868, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 2034, 998, 969, 127, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4078, 0, 4005, 0, 3, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17335, 16381, 14790, 16401, 16884, 16515, 16391, 16375] avg=16384.0000 max=17335 max_rank=0 min=14790 min_rank=2 imbalance_ratio=1.058044 max_load_reduction=113737 max_load_reduction_pct=0.867744 moved_to_echo=66165 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4091, 777, 1, 4095, 2, 0, 0, 1, 1, 0, 3130, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 4095, 1, 0, 1, 4094, 1, 0, 0, 2, 0, 4095, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 2064, 1, 3045, 0, 0, 2, 4095, 0, 1, 4094, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 1, 0, 1, 0, 1, 0, 0, 4095, 4094, 0, 0, 4093, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 3101, 305, 0, 4093, 4095, 1, 1, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 29, 3874, 4095, 0, 0, 4094, 1, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4094, 0, 0, 1, 2, 0, 1, 1, 0, 0, 4, 4094, 2030, 993, 965, 124, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 1, 3, 4095, 1, 4095, 1, 4084, 1, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16192, 16375, 17394, 16333, 15697, 16191, 16417, 16473] avg=16384.0000 max=17394 max_rank=2 min=15697 min_rank=4 imbalance_ratio=1.061646 max_load_reduction=113678 max_load_reduction_pct=0.867294 moved_to_echo=64778 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 729, 0, 4096, 0, 0, 0, 0, 0, 0, 3130, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 2066, 0, 2856, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 3102, 573, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 42, 3874, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2030, 994, 966, 125, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4092, 0, 4003, 0, 1, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16147, 16374, 17198, 16386, 15968, 16206, 16410, 16383] avg=16384.0000 max=17198 max_rank=2 min=15968 min_rank=4 imbalance_ratio=1.049683 max_load_reduction=113874 max_load_reduction_pct=0.868790 moved_to_echo=64967 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 32768, 7808, 128, 32768, 128, 0, 128, 128, 128, 0, 25088, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32768, 0, 128, 16512, 128, 16384, 0, 128, 128, 32768, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 128, 0, 128, 0, 128, 128, 0, 32768, 32768, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 32768, 128, 128, 1664, 31104, 32768, 128, 128, 32768, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 0, 128, 128, 128, 128, 32768, 128, 128, 128, 32768, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 32768, 32768, 128, 128, 128, 128, 0, 128, 128, 0, 128, 128, 32512, 16256, 8064, 7808, 1024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 32768, 128, 128, 128, 128, 32768, 128, 32768, 128, 32128, 128, 128, 128, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 0, 128, 32768, 128, 24832, 7936, 128, 32640, 32768, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2332, 0, 4096, 0, 0, 0, 4092, 7, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4082, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 2630, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 72, 4096, 3161, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4093, 0, 4096, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2073, 0, 4095, 0, 0, 0, 4087, 1599, 4096, 0, 1, 0, 4092, 0, 4096, 0, 4096, 3, 4090, 0, 5, 4090, 0, 0, 4071, 0, 0, 4090, 0, 1, 4096, 0, 0, 2, 0, 4094, 4092, 47, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 4090, 0, 4094, 0, 0, 3081, 0, 0, 0, 1, 4, 0, 0, 4095, 4093, 381, 0, 4096, 4096, 0, 4096, 0, 0, 4094, 0, 4093, 1, 37, 117, 4096, 880, 0, 0, 0, 0, 8, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 9, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 1979, 0, 4096, 0, 0, 0, 4096, 1043, 4095, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4037, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 3677, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 48, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 166, 295, 4096, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 4096, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 1, 0, 4094, 1, 1, 0, 1, 0, 1, 3478, 0, 4095, 0, 0, 0, 4094, 15, 4094, 0, 0, 1, 4095, 1, 4095, 0, 4094, 0, 4095, 1, 0, 4095, 0, 0, 4091, 0, 1, 4095, 0, 0, 4095, 2, 0, 0, 1, 4095, 4095, 5, 1, 2, 4094, 0, 0, 1, 0, 0, 1, 4092, 1, 1, 1, 4095, 1, 4093, 1, 1, 3921, 0, 1, 1, 0, 0, 1, 1, 4095, 4095, 18, 0, 4095, 4095, 1, 4095, 0, 0, 4094, 1, 4095, 0, 0, 26, 4095, 751, 1, 1, 1, 2, 1, 0, 0, 0, 1, 1, 1, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4076, 1, 4094, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[6, 7, 0, 0, 0, 5, 0, 4087, 6, 8, 3, 5, 7, 5, 1630, 19, 4010, 0, 0, 0, 4084, 162, 4089, 0, 4, 3, 4090, 7, 4089, 0, 4084, 0, 4088, 10, 0, 4090, 0, 0, 4083, 0, 6, 4089, 6, 0, 4090, 4, 0, 0, 7, 4090, 4090, 0, 6, 0, 4084, 0, 4, 4, 0, 0, 6, 4089, 0, 7, 7, 4081, 6, 4090, 5, 11, 3773, 0, 69, 0, 0, 0, 0, 6, 4087, 4089, 28, 0, 4089, 4088, 5, 4088, 6, 3, 4088, 7, 4089, 0, 0, 0, 4073, 2638, 6, 7, 4, 0, 0, 13, 0, 0, 6, 0, 6, 0, 4085, 0, 0, 2, 6, 0, 0, 1, 6, 5, 4089, 4054, 3, 4085, 0, 1, 0, 0, 4, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 2685, 0, 4096, 0, 0, 0, 4076, 619, 4095, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4095, 0, 0, 4091, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4095, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 3360, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 241, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 8, 40, 4096, 1251, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2578, 0, 4096, 0, 0, 0, 4095, 161, 4090, 0, 0, 0, 4094, 0, 4096, 0, 4096, 2, 4096, 0, 0, 4090, 0, 0, 4071, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 4095, 2, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4091, 0, 0, 3566, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 22, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 43, 220, 4096, 1630, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 1, 0, 4095, 1, 1, 0, 1, 2, 1, 2444, 0, 4095, 0, 1, 0, 4094, 33, 4095, 0, 0, 1, 4094, 1, 4095, 0, 4095, 1, 4095, 0, 1, 4095, 1, 0, 4095, 1, 1, 4094, 0, 0, 4095, 2, 0, 1, 1, 4094, 4092, 17, 1, 0, 4093, 0, 0, 1, 0, 0, 1, 4094, 2, 1, 1, 4095, 1, 4094, 0, 1, 3687, 0, 6, 0, 1, 0, 0, 1, 4094, 4095, 6, 0, 4094, 4095, 1, 4094, 1, 0, 4095, 1, 4094, 0, 1, 1, 4091, 2001, 1, 1, 2, 0, 0, 1, 0, 1, 2, 1, 1, 0, 4094, 0, 0, 0, 1, 0, 1, 0, 1, 1, 4094, 4090, 0, 4094, 0, 0, 0, 0, 4, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2073, 0, 4096, 4076, 1701, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4087, 1599, 4096, 0, 1, 0, 3656, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 5, 4090, 0, 0, 4071, 0, 0, 4090, 0, 1, 20, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4092, 47, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4094, 0, 0, 3081, 0, 0, 0, 1, 4, 0, 0, 4095, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 381, 0, 0, 0, 0, 2209, 0, 0, 4094, 0, 4093, 1, 37, 117, 4096, 880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 9, 0, 0, 0, 1887, 1767, 436, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16049, 17537, 16369, 16424, 15990, 15908, 16398, 16397] avg=16384.0000 max=17537 max_rank=1 min=15908 min_rank=5 imbalance_ratio=1.070374 max_load_reduction=113535 max_load_reduction_pct=0.866203 moved_to_echo=64693 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 1, 0, 4094, 1, 1, 0, 1, 0, 1, 3478, 0, 4094, 4075, 1701, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4094, 15, 4094, 0, 0, 1, 3659, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 4095, 0, 0, 4091, 0, 1, 4095, 0, 0, 20, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 4095, 5, 1, 2, 4094, 0, 0, 1, 0, 0, 1, 4092, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 4093, 1, 1, 3921, 0, 1, 1, 0, 0, 1, 1, 4095, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 1, 2209, 0, 0, 4094, 1, 4095, 0, 0, 26, 4095, 751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 1, 0, 0, 0, 1, 1, 1, 1, 4094, 0, 0, 0, 4095, 4095, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4076, 1, 4094, 0, 0, 0, 0, 1, 0, 1886, 1768, 436, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17454, 15959, 16400, 16379, 16838, 15290, 16383, 16369] avg=16384.0000 max=17454 max_rank=0 min=15290 min_rank=5 imbalance_ratio=1.065308 max_load_reduction=113618 max_load_reduction_pct=0.866837 moved_to_echo=64880 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 1979, 0, 4096, 4076, 1702, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1043, 4095, 0, 0, 0, 3660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4037, 0, 0, 4096, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3677, 0, 0, 0, 0, 0, 0, 0, 4096, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 2209, 0, 0, 4096, 0, 4096, 0, 166, 295, 4096, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 4096, 0, 0, 1, 0, 0, 0, 1887, 1768, 436, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15963, 16990, 16345, 16374, 16591, 16048, 16378, 16383] avg=16384.0000 max=16990 max_rank=1 min=15963 min_rank=0 imbalance_ratio=1.036987 max_load_reduction=114082 max_load_reduction_pct=0.870377 moved_to_echo=65400 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2578, 0, 4096, 4076, 1702, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 161, 4090, 0, 0, 0, 3658, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 4071, 0, 0, 4096, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4095, 2, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 3566, 0, 0, 0, 0, 0, 0, 0, 4096, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 2209, 0, 0, 4096, 0, 4096, 0, 43, 220, 4096, 1630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 1, 0, 0, 0, 1887, 1768, 436, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16554, 16102, 16373, 16376, 16475, 16412, 16384, 16396] avg=16384.0000 max=16554 max_rank=0 min=16102 min_rank=1 imbalance_ratio=1.010376 max_load_reduction=114518 max_load_reduction_pct=0.873703 moved_to_echo=65667 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 2685, 0, 4096, 4076, 1702, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4076, 619, 4095, 0, 0, 0, 3660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 4091, 0, 0, 4096, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4095, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 3360, 0, 0, 0, 0, 0, 0, 0, 4096, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 2209, 0, 0, 4096, 0, 4096, 0, 8, 40, 4096, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 1, 0, 0, 0, 1887, 1768, 436, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16663, 16546, 16398, 16387, 16273, 16037, 16378, 16390] avg=16384.0000 max=16663 max_rank=0 min=16037 min_rank=5 imbalance_ratio=1.017029 max_load_reduction=114409 max_load_reduction_pct=0.872871 moved_to_echo=65078 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[6, 7, 0, 0, 0, 5, 0, 4087, 6, 8, 3, 5, 7, 5, 1630, 19, 4084, 4070, 1699, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 0, 4084, 162, 4089, 0, 4, 3, 3654, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 10, 0, 4090, 0, 0, 4083, 0, 6, 4089, 6, 0, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4080, 4090, 0, 6, 0, 4084, 0, 4, 4, 0, 0, 6, 4089, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4081, 6, 4090, 5, 11, 3773, 0, 69, 0, 0, 0, 0, 6, 4087, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 5, 2205, 6, 3, 4088, 7, 4089, 0, 0, 0, 4073, 2638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 4, 0, 0, 13, 0, 0, 6, 0, 6, 0, 4085, 0, 0, 2, 4089, 4089, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 6, 5, 4089, 4054, 3, 4085, 0, 1, 0, 0, 4, 0, 1883, 1765, 436, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15647, 16013, 16396, 16377, 16760, 17142, 16389, 16348] avg=16384.0000 max=17142 max_rank=5 min=15647 min_rank=0 imbalance_ratio=1.046265 max_load_reduction=113930 max_load_reduction_pct=0.869217 moved_to_echo=66639 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2332, 0, 4096, 4081, 1705, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 7, 4096, 0, 0, 0, 3655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4082, 0, 0, 4096, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2630, 0, 0, 0, 0, 0, 0, 0, 4096, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 2208, 0, 0, 4096, 0, 4096, 0, 0, 72, 4096, 3161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4093, 0, 4096, 0, 0, 1, 0, 0, 0, 1888, 1771, 441, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16317, 15946, 16385, 16368, 15538, 17735, 16380, 16403] avg=16384.0000 max=17735 max_rank=5 min=15538 min_rank=4 imbalance_ratio=1.082458 max_load_reduction=113337 max_load_reduction_pct=0.864693 moved_to_echo=66056 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 1, 0, 4095, 1, 1, 0, 1, 2, 1, 2444, 0, 4095, 4075, 1701, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 4094, 33, 4095, 0, 0, 1, 3658, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 4095, 1, 0, 4095, 1, 1, 4094, 0, 0, 20, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4084, 4092, 17, 1, 0, 4093, 0, 0, 1, 0, 0, 1, 4094, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 4094, 0, 1, 3687, 0, 6, 0, 1, 0, 0, 1, 4094, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1, 2208, 1, 0, 4095, 1, 4094, 0, 1, 1, 4091, 2001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 0, 0, 1, 0, 1, 2, 1, 1, 0, 4094, 0, 0, 0, 4095, 4094, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 4094, 4090, 0, 4094, 0, 0, 0, 0, 4, 0, 1886, 1768, 436, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16425, 15979, 16406, 16387, 16607, 16500, 16382, 16386] avg=16384.0000 max=16607 max_rank=4 min=15979 min_rank=1 imbalance_ratio=1.013611 max_load_reduction=114465 max_load_reduction_pct=0.873299 moved_to_echo=65875 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 32768, 128, 128, 27776, 0, 128, 128, 128, 128, 128, 128, 32768, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 0, 128, 0, 32768, 3712, 32768, 0, 128, 128, 29312, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 32768, 128, 0, 32640, 128, 128, 32768, 128, 128, 256, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 0, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 128, 128, 19200, 128, 32768, 32640, 13696, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 32768, 0, 128, 128, 0, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[768, 0, 0, 0, 128, 17792, 128, 128, 32768, 128, 32768, 128, 256, 896, 32768, 13440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 0, 0, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32768, 32768, 128, 32768, 0, 128, 128, 0, 128, 0, 15104, 14208, 3584, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4093, 5, 0, 3, 1, 0, 0, 0, 1, 0, 3, 5, 0, 4093, 4091, 0, 4095, 456, 0, 0, 0, 6, 0, 4092, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 4089, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 0, 0, 0, 0, 1, 4096, 3, 0, 4096, 4084, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3639, 0, 0, 0, 4096, 0, 0, 5, 8, 0, 3, 0, 0, 0, 0, 0, 0, 4083, 1, 0, 0, 0, 0, 4096, 0, 4090, 4096, 0, 4096, 4089, 4096, 0, 4096, 4093, 1, 0, 0, 3, 3, 0, 0, 4096, 0, 4, 0, 4096, 4093, 4096, 4094, 8, 4092, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 108, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3988, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 4096, 4095, 4096, 0, 4096, 4093, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 2, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 3, 1, 0, 0, 4, 0, 1, 0, 4094, 4095, 1, 4094, 431, 1, 2, 1, 0, 2, 4095, 1, 1, 0, 1, 4095, 0, 4058, 4094, 1, 4093, 1, 0, 0, 2, 1, 8, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 4089, 2, 0, 4095, 4094, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 1, 4094, 3676, 1, 1, 19, 4095, 0, 1, 0, 0, 1, 0, 0, 0, 1, 2, 0, 0, 4093, 0, 0, 2, 0, 1, 4095, 0, 4095, 4094, 0, 4094, 4095, 4092, 1, 4095, 4091, 0, 0, 2, 0, 0, 0, 1, 4094, 5, 0, 1, 4095, 4095, 4095, 4095, 0, 4095, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4089, 0, 6, 0, 0, 6, 7, 0, 0, 3, 0, 0, 3, 4088, 4089, 7, 4089, 0, 6, 0, 6, 0, 2, 4090, 6, 7, 0, 6, 4089, 4, 3833, 4089, 2, 4087, 10, 4, 0, 8, 7, 58, 6, 0, 9, 0, 0, 0, 0, 9, 0, 7, 44, 0, 4033, 0, 0, 4083, 4090, 0, 4087, 5, 0, 0, 0, 0, 2, 1, 6, 4089, 6, 1, 0, 4, 4086, 4090, 6, 7, 10, 4086, 0, 2, 0, 0, 7, 0, 0, 0, 7, 6, 5, 0, 4089, 0, 0, 8, 1, 6, 4089, 0, 4090, 4080, 0, 4078, 4091, 4077, 7, 4081, 4075, 0, 0, 4, 0, 0, 0, 0, 4089, 12, 0, 6, 4088, 4090, 4089, 4087, 0, 4089, 5, 241, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 2, 1, 4095, 4095, 1, 4095, 15, 1, 0, 1, 0, 0, 4095, 1, 1, 0, 1, 4095, 0, 4067, 4095, 1, 4094, 1, 1, 0, 1, 1, 6, 1, 0, 2, 0, 0, 1, 0, 1, 0, 0, 1, 0, 4094, 0, 0, 4094, 4093, 0, 4094, 1, 1, 1, 0, 0, 2, 0, 1, 4095, 0, 0, 0, 1, 4094, 4095, 1, 1, 13, 4094, 0, 3, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 4093, 0, 0, 1, 0, 1, 4095, 0, 4095, 4094, 0, 4093, 4095, 4092, 1, 4095, 4089, 0, 0, 2, 0, 0, 0, 0, 4095, 1, 0, 1, 4095, 4095, 4095, 4095, 0, 4095, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 266, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3831, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 4092, 4096, 0, 4096, 4091, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 779, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3317, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 4095, 4096, 0, 4096, 4067, 0, 0, 0, 27, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 2, 0, 4096, 4096, 0, 4096, 253, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 4096, 0, 4094, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 4096, 4087, 0, 4094, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3845, 0, 0, 1, 4096, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 4096, 0, 4093, 4096, 0, 4096, 4090, 4096, 0, 4096, 4093, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4095, 4096, 10, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4093, 5, 0, 3, 1, 0, 0, 0, 1, 0, 3, 5, 0, 4093, 4091, 0, 4081, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 456, 0, 0, 0, 6, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 3847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 0, 4096, 4096, 4079, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 3, 0, 4096, 4084, 0, 4091, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3639, 0, 0, 0, 4096, 0, 0, 249, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8, 0, 3, 0, 0, 0, 0, 0, 0, 4083, 1, 0, 0, 0, 0, 4096, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4096, 4089, 4096, 0, 0, 4093, 1, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4093, 4065, 4094, 8, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16592, 16377, 16377, 16204, 16386, 16393, 16356] avg=16384.0000 max=16592 max_rank=1 min=16204 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=114480 max_load_reduction_pct=0.873413 moved_to_echo=65339 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4089, 0, 6, 0, 0, 6, 7, 0, 0, 3, 0, 0, 3, 4088, 4089, 7, 4065, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 6, 0, 6, 0, 2, 4090, 6, 7, 0, 6, 0, 4, 3833, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4087, 10, 4, 0, 8, 7, 58, 6, 0, 9, 0, 0, 0, 0, 9, 4088, 4081, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 44, 0, 4033, 0, 0, 4083, 4090, 0, 4087, 5, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 6, 4089, 6, 1, 0, 4, 4086, 4090, 6, 7, 10, 4086, 0, 2, 249, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 7, 6, 5, 0, 4089, 0, 0, 8, 1, 6, 4089, 4089, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4078, 4091, 4077, 7, 0, 4075, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 6, 0, 4090, 4058, 4087, 0, 4089, 5, 241, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16374, 15889, 16448, 16355, 16673, 16396, 16344, 16593] avg=16384.0000 max=16673 max_rank=4 min=15889 min_rank=1 imbalance_ratio=1.017639 max_load_reduction=114399 max_load_reduction_pct=0.872795 moved_to_echo=66006 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 2, 1, 4095, 4095, 1, 4079, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 15, 1, 0, 1, 0, 0, 4095, 1, 1, 0, 1, 0, 0, 4067, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 1, 0, 1, 1, 6, 1, 0, 2, 0, 0, 1, 0, 1, 4095, 4095, 4082, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 4094, 4093, 0, 4094, 1, 1, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4095, 0, 0, 0, 1, 4094, 4095, 1, 1, 13, 4094, 0, 3, 249, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 4093, 0, 0, 1, 0, 1, 4095, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4093, 4095, 4092, 1, 0, 4089, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4095, 4064, 4095, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16123, 16383, 16385, 16677, 16384, 16384, 16352] avg=16384.0000 max=16677 max_rank=4 min=16123 min_rank=1 imbalance_ratio=1.017883 max_load_reduction=114395 max_load_reduction_pct=0.872765 moved_to_echo=65797 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4081, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 108, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4082, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 4095, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3988, 0, 0, 1, 4096, 0, 0, 249, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4096, 4095, 4096, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4065, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16243, 16372, 16391, 16554, 16381, 16393, 16355] avg=16384.0000 max=16554 max_rank=4 min=16243 min_rank=1 imbalance_ratio=1.010376 max_load_reduction=114518 max_load_reduction_pct=0.873703 moved_to_echo=65683 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 2, 0, 4096, 4096, 0, 4081, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 253, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 4094, 3847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 4082, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 4096, 4087, 0, 4094, 0, 0, 3, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3845, 0, 0, 1, 4096, 0, 0, 249, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4096, 4090, 4096, 0, 0, 4093, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4064, 4096, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16386, 16364, 16391, 16411, 16380, 16391, 16362] avg=16384.0000 max=16411 max_rank=4 min=16362 min_rank=7 imbalance_ratio=1.001648 max_load_reduction=114661 max_load_reduction_pct=0.874794 moved_to_echo=65544 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4081, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 779, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3317, 0, 0, 1, 4096, 0, 0, 249, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4096, 4095, 4096, 0, 0, 4067, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4065, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16914, 16373, 16392, 15883, 16383, 16393, 16353] avg=16384.0000 max=16914 max_rank=1 min=15883 min_rank=4 imbalance_ratio=1.032349 max_load_reduction=114158 max_load_reduction_pct=0.870956 moved_to_echo=65012 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4085, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 266, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4083, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 2, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3831, 0, 0, 2, 4096, 0, 0, 256, 29, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 4092, 4096, 0, 0, 4091, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4056, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16393, 16373, 16397, 16410, 16378, 16387, 16344] avg=16384.0000 max=16410 max_rank=4 min=16344 min_rank=7 imbalance_ratio=1.001587 max_load_reduction=114662 max_load_reduction_pct=0.874802 moved_to_echo=65519 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 3, 1, 0, 0, 4, 0, 1, 0, 4094, 4095, 1, 4079, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 431, 1, 2, 1, 0, 2, 4095, 1, 1, 0, 1, 0, 0, 4058, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 1, 0, 0, 2, 1, 8, 1, 0, 0, 0, 0, 1, 0, 1, 4095, 4095, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4089, 2, 0, 4095, 4094, 0, 4094, 2, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 1, 4094, 3676, 1, 1, 19, 4095, 0, 1, 249, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 2, 0, 0, 4093, 0, 0, 2, 0, 1, 4095, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4094, 4095, 4092, 1, 0, 4091, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 1, 0, 4095, 4064, 4095, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16532, 16382, 16384, 16260, 16384, 16387, 16357] avg=16384.0000 max=16532 max_rank=1 min=16260 min_rank=4 imbalance_ratio=1.009033 max_load_reduction=114540 max_load_reduction_pct=0.873871 moved_to_echo=65388 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 2432, 128, 128, 128, 128, 128, 32768, 128, 128, 0, 128, 0, 128, 32512, 30848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 0, 128, 32768, 30592, 128, 128, 128, 32768, 0, 128, 2048, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 128, 128, 0, 128, 0, 0, 128, 128, 128, 32768, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 128, 128, 0, 32768, 128, 0, 128, 128, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 32768, 32768, 0, 32768, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 32768, 32768, 32768, 128, 0, 32768, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 128, 128, 128, 0, 32768, 32512, 32768, 128, 32768, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4094, 4096, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 447, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 4095, 4096, 3648, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 4094, 0, 8, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 2, 4089, 4087, 9, 0, 12, 4070, 4088, 0, 0, 0, 0, 6, 7, 0, 0, 0, 4089, 14, 7, 0, 4088, 4088, 0, 7, 4, 0, 928, 4084, 4078, 0, 0, 2, 5, 2, 9, 4088, 4088, 4090, 4090, 1, 0, 7, 8, 0, 6, 8, 7, 0, 7, 0, 0, 4, 4, 4089, 31, 7, 0, 0, 4048, 6, 6, 0, 0, 4086, 1, 0, 0, 0, 4089, 0, 6, 3, 4086, 10, 4082, 4084, 4084, 6, 5, 0, 6, 5, 8, 4089, 0, 4085, 4081, 3556, 4088, 3, 6, 0, 0, 2, 4072, 4084, 7, 8, 0, 37, 0, 0, 6, 2, 0, 0, 0, 4090, 0, 6, 0, 4088, 1, 9, 0, 0, 11, 11, 8, 4089, 0, 3670, 8, 9, 9, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4083, 4096, 0, 0, 0, 4096, 4096, 0, 0, 11, 0, 0, 0, 0, 2, 0, 4093, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 413, 4096, 4096, 8, 7, 0, 0, 0, 0, 4096, 4088, 4084, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 4, 8, 4043, 0, 0, 4, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 4094, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 4088, 0, 4094, 4091, 3709, 4090, 0, 0, 0, 9, 0, 4096, 4096, 0, 0, 0, 0, 2, 11, 0, 0, 20, 0, 0, 4078, 0, 33, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 2, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4095, 4094, 1, 0, 2, 4093, 4095, 0, 0, 0, 0, 1, 1, 0, 2, 0, 4095, 0, 1, 0, 4095, 4095, 0, 1, 1, 0, 57, 4095, 4095, 0, 0, 0, 0, 1, 1, 4095, 4095, 4095, 4095, 0, 1, 1, 1, 0, 1, 1, 1, 4, 1, 0, 0, 0, 1, 4095, 1, 1, 0, 0, 4053, 1, 1, 0, 0, 4095, 1, 0, 1, 0, 4095, 0, 1, 2, 4094, 1, 4094, 4095, 4093, 1, 1, 0, 1, 0, 2, 4095, 0, 4092, 4093, 4083, 4094, 0, 1, 0, 0, 0, 4094, 4095, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4095, 0, 18, 0, 4092, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 4073, 1, 2, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 156, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4096, 0, 0, 0, 0, 3997, 0, 0, 0, 0, 4031, 0, 0, 0, 4, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 4096, 3940, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 4079, 0, 85, 0, 4084, 0, 0, 15, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 6, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 43, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4056, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4089, 0, 19, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4094, 4094, 1, 0, 2, 4090, 4094, 0, 0, 1, 1, 1, 1, 0, 0, 0, 4094, 2, 1, 0, 4095, 4094, 0, 1, 1, 0, 99, 4094, 4093, 1, 0, 0, 0, 0, 1, 4095, 4095, 4095, 4095, 0, 0, 1, 0, 1, 1, 2, 1, 0, 1, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 4032, 1, 1, 0, 0, 4094, 1, 1, 0, 3, 4095, 1, 1, 0, 4095, 2, 4094, 4094, 4095, 1, 2, 0, 1, 0, 0, 4094, 0, 4093, 4095, 4060, 4095, 0, 1, 0, 1, 0, 4094, 4094, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 4095, 0, 2, 0, 4094, 1, 1, 0, 0, 5, 0, 1, 4095, 0, 4089, 1, 1, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4093, 4096, 0, 0, 0, 4096, 4096, 0, 0, 12, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 31, 4096, 4096, 2, 1, 0, 0, 0, 0, 4096, 4093, 4087, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 1, 4090, 0, 0, 0, 0, 4094, 0, 0, 2, 5, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4084, 0, 4095, 4095, 4065, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 9, 0, 0, 4082, 0, 14, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4089, 4087, 9, 0, 12, 4070, 4088, 0, 0, 0, 0, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3820, 14, 7, 0, 0, 4088, 0, 7, 4, 0, 928, 4084, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 2, 9, 4088, 4088, 4090, 4090, 1, 0, 7, 8, 0, 6, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 0, 0, 4, 4, 4089, 31, 7, 0, 0, 4048, 6, 6, 0, 4089, 4088, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 0, 0, 0, 0, 0, 6, 3, 4086, 10, 4082, 11, 4084, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 5, 8, 4089, 0, 4085, 4081, 3556, 212, 3, 6, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 4084, 7, 8, 0, 37, 0, 0, 6, 2, 0, 0, 0, 4090, 0, 6, 3787, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 9, 0, 0, 11, 11, 8, 4089, 0, 3670, 8, 9, 9, 6, 4044, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16370, 17030, 16406, 16415, 16375, 16058, 16366, 16052] avg=16384.0000 max=17030 max_rank=1 min=16052 min_rank=7 imbalance_ratio=1.039429 max_load_reduction=114042 max_load_reduction_pct=0.870071 moved_to_echo=64851 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4083, 4096, 0, 0, 0, 4096, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3823, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 413, 4096, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 4088, 4084, 4086, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 4, 8, 4043, 0, 0, 4, 4096, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 4096, 11, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4094, 4091, 3709, 212, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 2, 11, 0, 0, 20, 0, 0, 4078, 0, 33, 3789, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 2, 4096, 0, 0, 0, 0, 4056, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16528, 16363, 16380, 16387, 16203, 16393, 16436] avg=16384.0000 max=16528 max_rank=1 min=16203 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=114544 max_load_reduction_pct=0.873901 moved_to_echo=65419 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 4094, 1, 0, 2, 4093, 4095, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3825, 0, 1, 0, 0, 4095, 0, 1, 1, 0, 57, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4095, 4095, 4095, 0, 1, 1, 1, 0, 1, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 1, 0, 0, 0, 1, 4095, 1, 1, 0, 0, 4053, 1, 1, 0, 4095, 4095, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 0, 0, 1, 2, 4094, 1, 4094, 11, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 4095, 0, 4092, 4093, 4083, 211, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4095, 0, 18, 3793, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 4073, 1, 2, 0, 1, 4055, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16172, 16389, 16378, 16394, 16579, 16367, 16411] avg=16384.0000 max=16579 max_rank=5 min=16172 min_rank=1 imbalance_ratio=1.011902 max_load_reduction=114493 max_load_reduction_pct=0.873512 moved_to_echo=65751 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 4096, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3826, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 447, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4095, 4096, 3648, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 4094, 0, 8, 3794, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4056, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16561, 16385, 16410, 16396, 16143, 16359, 16434] avg=16384.0000 max=16561 max_rank=1 min=16143 min_rank=5 imbalance_ratio=1.010803 max_load_reduction=114511 max_load_reduction_pct=0.873650 moved_to_echo=65332 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 4096, 0, 0, 0, 4096, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3826, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 31, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4093, 4087, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 1, 4090, 0, 0, 0, 4096, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 5, 0, 0, 0, 0, 4096, 0, 4096, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4095, 4095, 4065, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 9, 0, 0, 4082, 0, 14, 3794, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4056, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16393, 16151, 16373, 16408, 16400, 16551, 16362, 16434] avg=16384.0000 max=16551 max_rank=5 min=16151 min_rank=1 imbalance_ratio=1.010193 max_load_reduction=114521 max_load_reduction_pct=0.873726 moved_to_echo=65747 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3826, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 156, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4096, 0, 0, 0, 0, 3997, 0, 0, 0, 4096, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4096, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 4096, 3940, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 4079, 0, 85, 3794, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 15, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4056, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16270, 16381, 16332, 16334, 16431, 16497, 16437] avg=16384.0000 max=16497 max_rank=6 min=16270 min_rank=1 imbalance_ratio=1.006897 max_load_reduction=114575 max_load_reduction_pct=0.874138 moved_to_echo=65699 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 6, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3818, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 43, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 4075, 0, 0, 0, 4096, 4096, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4056, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4089, 0, 19, 3799, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4058, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 16150, 16387, 16398, 16387, 16549, 16375, 16434] avg=16384.0000 max=16549 max_rank=5 min=16150 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=114523 max_load_reduction_pct=0.873741 moved_to_echo=65745 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 4094, 1, 0, 2, 4090, 4094, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 2, 1, 0, 0, 4094, 0, 1, 1, 0, 99, 4094, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4095, 4095, 4095, 0, 0, 1, 0, 1, 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 4032, 1, 1, 0, 4095, 4095, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 3, 0, 1, 1, 0, 4095, 2, 4094, 11, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 4094, 0, 4093, 4095, 4060, 211, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 4095, 0, 2, 3794, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 0, 5, 0, 1, 4095, 0, 4089, 1, 1, 1, 1, 4054, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16210, 16388, 16351, 16399, 16558, 16353, 16434] avg=16384.0000 max=16558 max_rank=5 min=16210 min_rank=1 imbalance_ratio=1.010620 max_load_reduction=114514 max_load_reduction_pct=0.873672 moved_to_echo=65744 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 32768, 128, 128, 128, 0, 128, 128, 128, 32768, 128, 32384, 128, 128, 128, 128, 32512, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 30592, 128, 128, 0, 0, 32768, 0, 128, 128, 0, 2176, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 32768, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 0, 256, 30464, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 32768, 128, 128, 128, 128, 32512, 128, 128, 128, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 32768, 32768, 128, 0, 128, 32768, 32768, 0, 0, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 0, 128, 128, 128, 32768, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 128, 32768, 0, 32768, 32768, 31232, 1792, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 32768, 32768, 32768, 128, 128, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[8, 4084, 6, 8, 8, 0, 4, 7, 4086, 4088, 7, 36, 0, 5, 8, 0, 0, 4082, 3908, 18, 4089, 5, 4085, 4086, 7, 0, 5, 0, 4085, 0, 8, 21, 7, 176, 0, 8, 4078, 4, 0, 0, 123, 0, 8, 4087, 8, 0, 0, 128, 4090, 6, 4054, 7, 1, 4080, 6, 4083, 0, 4088, 7, 7, 4087, 0, 391, 0, 4086, 0, 0, 0, 6, 3830, 0, 4089, 4076, 0, 1568, 4066, 4085, 3, 10, 6, 4, 3990, 4087, 7, 237, 0, 0, 9, 0, 5, 0, 8, 4089, 3, 0, 0, 3, 6, 7, 0, 4088, 1, 0, 6, 5, 2982, 515, 6, 2, 0, 7, 0, 0, 8, 0, 4088, 2601, 4001, 0, 73, 13, 17, 4, 4087, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 3963, 0, 5, 20, 0, 4090, 0, 4096, 0, 0, 4096, 0, 4091, 0, 4096, 0, 0, 4087, 0, 1, 0, 4091, 1, 0, 0, 0, 4096, 0, 4092, 4096, 0, 668, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 11, 0, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 2741, 879, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4069, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1, 4093, 1, 0, 1, 0, 1, 1, 4095, 4094, 0, 2, 0, 1, 1, 0, 0, 4094, 4075, 2, 4095, 2, 4094, 4095, 1, 0, 1, 0, 4095, 0, 3, 7, 1, 21, 0, 1, 4093, 1, 0, 0, 20, 0, 2, 4095, 1, 0, 0, 1, 4095, 1, 4092, 1, 0, 4094, 1, 4091, 0, 4095, 1, 1, 4094, 3, 0, 0, 4095, 0, 0, 1, 1, 3990, 0, 4095, 4094, 0, 112, 4094, 4094, 0, 0, 1, 1, 4089, 4095, 1, 128, 0, 1, 1, 0, 1, 0, 0, 4049, 3, 7, 0, 0, 1, 1, 0, 4094, 1, 0, 1, 1, 3066, 1345, 1, 1, 0, 1, 0, 0, 1, 0, 4095, 3658, 4090, 0, 0, 1, 6, 0, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 90, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 5, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 20, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3632, 0, 1, 0, 0, 0, 0, 22, 4096, 0, 0, 0, 0, 3865, 746, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4005, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 24, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 4096, 0, 10, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 570, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4089, 0, 4, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 3423, 133, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4080, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 6, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 58, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 4, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 3872, 266, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4091, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1, 4094, 1, 2, 0, 0, 0, 0, 4094, 4094, 1, 3, 0, 0, 1, 0, 0, 4094, 4077, 0, 4095, 2, 4093, 4094, 2, 0, 1, 0, 4094, 0, 4, 4, 1, 24, 0, 2, 4093, 1, 0, 0, 12, 0, 2, 4095, 2, 0, 0, 4, 4095, 1, 4092, 1, 1, 4095, 1, 4094, 0, 4094, 1, 1, 4093, 0, 3, 0, 4095, 0, 0, 1, 1, 4075, 1, 4095, 4094, 0, 174, 4092, 4094, 1, 2, 1, 1, 4085, 4094, 1, 32, 0, 2, 1, 0, 1, 2, 0, 4095, 2, 14, 1, 0, 2, 1, 1, 4094, 1, 0, 1, 1, 3989, 274, 1, 0, 0, 1, 0, 0, 1, 0, 4095, 3708, 4088, 0, 2, 1, 0, 0, 4094, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4048, 0, 0, 10, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4090, 0, 4092, 0, 0, 4092, 0, 254, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 367, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 39, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 22, 0, 0, 0, 0, 9, 4095, 0, 0, 0, 0, 2265, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4087, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[8, 4084, 6, 8, 8, 0, 4, 7, 4086, 4088, 7, 36, 0, 5, 8, 0, 3931, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 3908, 18, 0, 5, 4085, 20, 7, 0, 5, 0, 4085, 0, 8, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 176, 0, 8, 4078, 4, 0, 0, 123, 0, 8, 4087, 8, 0, 0, 128, 4088, 3962, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4054, 7, 1, 4008, 6, 4083, 0, 0, 7, 7, 4087, 0, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 0, 0, 6, 3830, 0, 0, 4076, 0, 1568, 4066, 0, 3, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3990, 4087, 7, 237, 0, 0, 9, 0, 5, 0, 8, 4089, 3, 0, 0, 3989, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6, 7, 0, 4088, 1, 0, 6, 5, 2982, 515, 6, 2, 0, 7, 0, 4089, 4089, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4088, 2601, 4001, 0, 73, 13, 17, 4, 4087, 0, 0, 0, 0, 236, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16436, 16244, 16773, 16657, 17265, 16563, 15934, 15200] avg=16384.0000 max=17265 max_rank=4 min=15200 min_rank=7 imbalance_ratio=1.053772 max_load_reduction=113807 max_load_reduction_pct=0.868279 moved_to_echo=64962 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 3940, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 4096, 20, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 3963, 0, 5, 20, 0, 4096, 3961, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4024, 0, 4091, 0, 0, 0, 0, 4087, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3704, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 668, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 11, 0, 3999, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 2741, 879, 0, 0, 0, 0, 0, 4096, 4092, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4069, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 0, 237, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16363, 16421, 16243, 16299, 16661, 16371, 16046, 16668] avg=16384.0000 max=16668 max_rank=7 min=16046 min_rank=6 imbalance_ratio=1.017334 max_load_reduction=114404 max_load_reduction_pct=0.872833 moved_to_echo=65746 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 4093, 1, 0, 1, 0, 1, 1, 4095, 4094, 0, 2, 0, 1, 1, 0, 3939, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4075, 2, 0, 2, 4094, 20, 1, 0, 1, 0, 4095, 0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 21, 0, 1, 4093, 1, 0, 0, 20, 0, 2, 4095, 1, 0, 0, 1, 4095, 3967, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 1, 0, 4022, 1, 4091, 0, 0, 1, 1, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3708, 0, 0, 1, 1, 3990, 0, 0, 4094, 0, 112, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 4095, 1, 128, 0, 1, 1, 0, 1, 0, 0, 4049, 3, 7, 0, 3997, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4094, 1, 0, 1, 1, 3066, 1345, 1, 1, 0, 1, 0, 4095, 4095, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 3658, 4090, 0, 0, 1, 6, 0, 4095, 0, 0, 0, 0, 237, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16394, 16395, 16307, 16001, 16509, 16831, 16255] avg=16384.0000 max=16831 max_rank=6 min=16001 min_rank=4 imbalance_ratio=1.027283 max_load_reduction=114241 max_load_reduction_pct=0.871590 moved_to_echo=65596 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3940, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 4096, 20, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 90, 0, 4096, 3967, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4024, 0, 4096, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3709, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 20, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3632, 0, 1, 0, 3999, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4096, 0, 0, 0, 0, 3865, 746, 0, 0, 0, 0, 0, 4096, 4096, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4005, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 24, 0, 237, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 16405, 16346, 16317, 16017, 15960, 17050, 16599] avg=16384.0000 max=17050 max_rank=6 min=15960 min_rank=5 imbalance_ratio=1.040649 max_load_reduction=114022 max_load_reduction_pct=0.869919 moved_to_echo=65626 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 3940, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 0, 0, 0, 4096, 20, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4048, 0, 0, 10, 0, 4092, 3967, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4024, 0, 4090, 0, 0, 0, 0, 4092, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3709, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 367, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 39, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 22, 0, 3999, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4095, 0, 0, 0, 0, 2265, 1307, 0, 0, 0, 0, 0, 4096, 4096, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4087, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 237, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 16413, 16311, 16556, 16364, 16379, 15997, 16679] avg=16384.0000 max=16679 max_rank=7 min=15997 min_rank=6 imbalance_ratio=1.018005 max_load_reduction=114393 max_load_reduction_pct=0.872749 moved_to_echo=65419 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3940, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 20, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 2, 0, 4096, 3968, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4024, 0, 4095, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3709, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 570, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4089, 0, 4, 0, 3998, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 3423, 133, 0, 0, 0, 0, 0, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4080, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 237, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 16405, 16329, 16321, 16567, 16424, 15973, 16675] avg=16384.0000 max=16675 max_rank=7 min=15973 min_rank=6 imbalance_ratio=1.017761 max_load_reduction=114397 max_load_reduction_pct=0.872780 moved_to_echo=65639 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3945, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 4096, 13, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 6, 0, 4096, 3968, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4017, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3702, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 58, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 4, 0, 3998, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 3872, 266, 0, 0, 0, 0, 0, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4091, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 240, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16399, 16282, 16304, 16047, 16402, 16556, 16697] avg=16384.0000 max=16697 max_rank=7 min=16047 min_rank=4 imbalance_ratio=1.019104 max_load_reduction=114375 max_load_reduction_pct=0.872612 moved_to_echo=65702 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 4094, 1, 2, 0, 0, 0, 0, 4094, 4094, 1, 3, 0, 0, 1, 0, 3938, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4077, 0, 0, 2, 4093, 20, 2, 0, 1, 0, 4094, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 24, 0, 2, 4093, 1, 0, 0, 12, 0, 2, 4095, 2, 0, 0, 4, 4094, 3966, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 1, 1, 4023, 1, 4094, 0, 0, 1, 1, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3708, 0, 0, 1, 1, 4075, 1, 0, 4094, 0, 174, 4092, 0, 1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 4094, 1, 32, 0, 2, 1, 0, 1, 2, 0, 4095, 2, 14, 1, 3997, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 4094, 1, 0, 1, 1, 3989, 274, 1, 0, 0, 1, 0, 4095, 4095, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 3708, 4088, 0, 2, 1, 0, 0, 4094, 1, 0, 0, 0, 237, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16391, 16393, 16311, 16150, 16464, 16685, 16299] avg=16384.0000 max=16685 max_rank=6 min=16150 min_rank=4 imbalance_ratio=1.018372 max_load_reduction=114387 max_load_reduction_pct=0.872704 moved_to_echo=65598 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[29696, 128, 0, 128, 128, 32384, 128, 0, 32768, 0, 3584, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 32768, 32768, 128, 512, 0, 128, 128, 128, 128, 128, 128, 32128, 128, 128, 128, 32000, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 128, 128, 26240, 5504, 128, 128, 0, 128, 0, 32768, 32768, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 128, 128, 32768, 32768, 128, 128, 0, 128, 128, 0, 31616, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 32640, 128, 0, 128, 32768, 256, 128, 0, 128, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 32768, 128, 128, 32256, 128, 32768, 0, 0, 128, 128, 32768, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 256, 0, 128, 32768, 128, 128, 0, 256, 0, 128, 32384, 128, 128, 128, 256, 32768, 31744, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 32768, 30336, 32768, 0, 128, 128, 128, 128, 32768, 128, 0, 128, 0, 1920, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 4092, 0, 1, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4090, 0, 0, 4096, 4096, 7, 61, 1, 4093, 4092, 0, 15, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 2017, 0, 0, 0, 0, 0, 0, 919, 4, 0, 0, 0, 0, 0, 4095, 0, 3, 4096, 0, 0, 0, 0, 5, 0, 4096, 4083, 0, 9, 4077, 0, 0, 0, 4096, 33, 4087, 0, 4096, 0, 4092, 4096, 0, 0, 4096, 0, 4096, 0, 11, 0, 4086, 0, 0, 0, 7, 4092, 2, 16, 0, 0, 1, 4096, 0, 1296, 0, 4067, 1, 0, 0, 3955, 4095, 4046, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 7, 3, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 1767, 0, 0, 0, 0, 0, 0, 1219, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 4086, 0, 0, 0, 4096, 13, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 17, 4096, 0, 10, 0, 0, 15, 4096, 0, 1150, 0, 4082, 32, 0, 0, 4061, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4086, 0, 8, 63, 3694, 0, 0, 1, 2, 2, 0, 0, 3909, 8, 0, 10, 8, 4079, 7, 0, 4086, 4082, 0, 0, 0, 4089, 4034, 0, 0, 4085, 4086, 7, 10, 7, 0, 386, 10, 7, 4058, 0, 1626, 6, 0, 7, 0, 220, 0, 738, 0, 10, 6, 7, 7, 4, 4049, 6, 0, 4087, 10, 7, 7, 0, 1, 8, 4088, 4089, 8, 0, 4088, 8, 0, 0, 4001, 0, 4089, 10, 4068, 0, 4089, 4086, 0, 0, 4087, 0, 4081, 8, 0, 0, 4086, 7, 9, 11, 0, 4088, 0, 0, 0, 3, 107, 4083, 41, 1579, 103, 4087, 11, 0, 8, 4088, 4087, 4080, 0, 0, 0, 0, 4, 0, 5, 4087, 9, 10, 8, 5, 22, 12, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4095, 0, 1, 6, 3912, 0, 0, 0, 0, 1, 0, 5, 4071, 1, 2, 0, 1, 4095, 2, 0, 4095, 4095, 0, 0, 0, 4095, 4089, 0, 0, 4095, 4094, 1, 1, 1, 0, 11, 1, 1, 4083, 0, 1680, 1, 0, 1, 0, 14, 3, 1379, 0, 1, 1, 1, 1, 0, 4094, 1, 0, 4095, 0, 1, 1, 0, 0, 1, 4095, 4095, 1, 0, 4089, 1, 1, 0, 4094, 21, 4095, 1, 4093, 0, 4095, 4094, 1, 0, 4095, 0, 4095, 1, 0, 0, 4095, 1, 0, 1, 6, 4095, 0, 0, 0, 1, 30, 4094, 1, 1181, 0, 4094, 64, 1, 0, 4087, 4095, 4030, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 2, 1, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 3954, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 31, 8, 0, 4096, 4093, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 1814, 0, 0, 0, 0, 0, 0, 1715, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4057, 0, 0, 4079, 0, 0, 0, 4096, 134, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 61, 4096, 0, 56, 0, 0, 77, 4096, 0, 683, 0, 4052, 144, 0, 0, 3918, 4096, 3953, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4094, 0, 1, 6, 3909, 0, 0, 0, 0, 2, 0, 0, 4041, 2, 3, 2, 2, 4094, 2, 0, 4094, 4094, 0, 0, 0, 4095, 4088, 0, 0, 4095, 4094, 1, 2, 2, 0, 24, 2, 1, 4087, 0, 662, 1, 0, 1, 0, 33, 2, 2115, 0, 2, 2, 1, 1, 1, 4091, 2, 0, 4094, 0, 2, 1, 0, 1, 1, 4094, 4093, 2, 0, 4092, 2, 0, 0, 4094, 4, 4094, 2, 4090, 0, 4094, 4094, 1, 0, 4095, 0, 4094, 1, 0, 0, 4095, 1, 0, 2, 2, 4095, 0, 0, 0, 0, 24, 4093, 5, 1473, 0, 4094, 27, 0, 0, 4095, 4094, 4071, 0, 0, 0, 0, 1, 0, 0, 4095, 1, 2, 1, 1, 2, 0, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 6, 46, 0, 4093, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 851, 0, 0, 0, 1, 0, 0, 3145, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 4096, 4089, 0, 6, 4082, 0, 0, 0, 4096, 40, 4093, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 4, 0, 4095, 0, 0, 0, 4, 4096, 0, 10, 0, 0, 5, 4096, 0, 276, 0, 4082, 2, 0, 0, 3987, 4096, 4057, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 1, 4, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2536, 0, 0, 0, 0, 0, 0, 1374, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 4096, 4094, 0, 0, 4093, 0, 0, 0, 4096, 5, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4, 4096, 0, 4, 0, 0, 1, 4096, 0, 233, 0, 4093, 1, 0, 0, 4060, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 4092, 0, 1, 0, 0, 0, 0, 0, 4090, 0, 0, 3267, 956, 11, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 7, 61, 1, 4084, 4092, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 2017, 0, 0, 0, 0, 0, 0, 4096, 4095, 2398, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 919, 4, 0, 0, 0, 0, 0, 4095, 0, 3, 4096, 0, 0, 0, 0, 5, 4092, 1698, 820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 9, 4077, 0, 0, 0, 4096, 33, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 11, 0, 4086, 0, 0, 0, 7, 4081, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 1, 3140, 0, 1296, 0, 4067, 1, 0, 0, 3955, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16521, 16446, 16697, 15732, 16393, 16379, 16522, 16382] avg=16384.0000 max=16697 max_rank=2 min=15732 min_rank=3 imbalance_ratio=1.019104 max_load_reduction=114375 max_load_reduction_pct=0.872612 moved_to_echo=65676 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4086, 0, 8, 63, 3694, 0, 0, 1, 2, 2, 0, 0, 3909, 8, 0, 3269, 953, 11, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8, 4079, 7, 0, 0, 4082, 0, 0, 0, 4080, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 10, 7, 0, 386, 10, 7, 4058, 0, 1626, 6, 0, 7, 0, 220, 0, 4085, 4087, 2392, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 738, 0, 10, 6, 7, 7, 4, 4049, 6, 0, 4087, 10, 7, 7, 0, 1, 4089, 1694, 820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4089, 8, 0, 4088, 8, 0, 0, 4001, 0, 0, 10, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4081, 8, 0, 0, 4086, 7, 9, 11, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 107, 3130, 41, 1579, 103, 4087, 11, 0, 8, 4088, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 4087, 9, 10, 8, 5, 22, 12, 0, 2, 0, 4087, 4088, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16016, 16300, 16914, 15542, 16279, 16365, 17237, 16419] avg=16384.0000 max=17237 max_rank=6 min=15542 min_rank=3 imbalance_ratio=1.052063 max_load_reduction=113835 max_load_reduction_pct=0.868492 moved_to_echo=66300 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 3276, 961, 16, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 7, 3, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 1767, 0, 0, 0, 0, 0, 0, 4096, 4096, 2399, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1219, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1697, 820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4086, 0, 0, 0, 4096, 13, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 17, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 15, 3135, 0, 1150, 0, 4082, 32, 0, 0, 4061, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16528, 16381, 16451, 16024, 16373, 16385, 16553, 16377] avg=16384.0000 max=16553 max_rank=6 min=16024 min_rank=3 imbalance_ratio=1.010315 max_load_reduction=114519 max_load_reduction_pct=0.873711 moved_to_echo=65688 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 1, 6, 3912, 0, 0, 0, 0, 1, 0, 5, 4071, 1, 2, 3275, 956, 11, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 2, 0, 0, 4095, 0, 0, 0, 4085, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 11, 1, 1, 4083, 0, 1680, 1, 0, 1, 0, 14, 3, 4095, 4095, 2398, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1379, 0, 1, 1, 1, 1, 0, 4094, 1, 0, 4095, 0, 1, 1, 0, 0, 4095, 1696, 820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 4089, 1, 1, 0, 4094, 21, 0, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 4095, 1, 0, 0, 4095, 1, 0, 1, 6, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 30, 3138, 1, 1181, 0, 4094, 64, 1, 0, 4087, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 2, 1, 1, 0, 0, 0, 0, 0, 4095, 4095, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16347, 16367, 16392, 16186, 16396, 16378, 16627, 16379] avg=16384.0000 max=16627 max_rank=6 min=16186 min_rank=3 imbalance_ratio=1.014832 max_load_reduction=114445 max_load_reduction_pct=0.873146 moved_to_echo=65780 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 3954, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3275, 956, 11, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 31, 8, 0, 4086, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 1814, 0, 0, 0, 0, 0, 0, 4096, 4096, 2398, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1715, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1698, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 4079, 0, 0, 0, 4096, 134, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 61, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 77, 3140, 0, 683, 0, 4052, 144, 0, 0, 3918, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16398, 16410, 16447, 16521, 16461, 16434, 16023, 16378] avg=16384.0000 max=16521 max_rank=3 min=16023 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=114551 max_load_reduction_pct=0.873955 moved_to_echo=65296 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 0, 3272, 956, 11, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 6, 46, 0, 4084, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 851, 0, 0, 0, 1, 0, 0, 4096, 4096, 2398, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3145, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 8, 4096, 1698, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 6, 4082, 0, 0, 0, 4096, 40, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4, 0, 4095, 0, 0, 0, 4, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 5, 3140, 0, 276, 0, 4082, 2, 0, 0, 3987, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16434, 16422, 15533, 17958, 16408, 16380, 15559, 16378] avg=16384.0000 max=17958 max_rank=3 min=15533 min_rank=2 imbalance_ratio=1.096069 max_load_reduction=113114 max_load_reduction_pct=0.862991 moved_to_echo=64725 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 3276, 956, 11, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 4, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2536, 0, 0, 0, 0, 0, 0, 4096, 4096, 2399, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1374, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 5, 4096, 1697, 820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4093, 0, 0, 0, 4096, 5, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 3140, 0, 233, 0, 4093, 1, 0, 0, 4060, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16520, 16379, 17228, 16184, 16383, 16377, 15624, 16377] avg=16384.0000 max=17228 max_rank=2 min=15624 min_rank=6 imbalance_ratio=1.051514 max_load_reduction=113844 max_load_reduction_pct=0.868561 moved_to_echo=64761 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 1, 6, 3909, 0, 0, 0, 0, 2, 0, 0, 4041, 2, 3, 3273, 955, 11, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4094, 2, 0, 0, 4094, 0, 0, 0, 4085, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 24, 2, 1, 4087, 0, 662, 1, 0, 1, 0, 33, 2, 4095, 4094, 2397, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2115, 0, 2, 2, 1, 1, 1, 4091, 2, 0, 4094, 0, 2, 1, 0, 1, 4094, 1697, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 2, 0, 4092, 2, 0, 0, 4094, 4, 0, 2, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 4094, 1, 0, 0, 4095, 1, 0, 2, 2, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 3138, 5, 1473, 0, 4094, 27, 0, 0, 4095, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 1, 2, 1, 1, 2, 0, 0, 2, 0, 4095, 4094, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16308, 16367, 15410, 16925, 16379, 16374, 16927, 16382] avg=16384.0000 max=16927 max_rank=6 min=15410 min_rank=2 imbalance_ratio=1.033142 max_load_reduction=114145 max_load_reduction_pct=0.870857 moved_to_echo=66062 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 0, 0, 0, 32768, 128, 128, 0, 32768, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 32768, 0, 128, 128, 31744, 0, 128, 128, 128, 128, 0, 128, 32512, 128, 128, 26240, 7680, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 32768, 128, 128, 128, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[12672, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 128, 128, 0, 128, 32768, 13696, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 384, 25216, 128, 7936, 128, 32768, 384, 128, 128, 32256, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 0, 512, 128, 128, 32640, 0, 13056, 128, 0, 128, 128, 384, 128, 32768, 32768, 19200, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 0, 128, 0, 128, 32768, 128, 128, 128, 128, 128, 128, 0, 128, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 32768, 128, 128, 32768, 128, 128, 0, 32768, 256, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 3970, 0, 10, 0, 4088, 0, 4086, 0, 0, 11, 0, 4085, 6, 0, 0, 0, 4086, 5, 4082, 0, 4, 0, 8, 10, 8, 4088, 4088, 10, 4080, 0, 0, 2, 0, 10, 5, 9, 5, 0, 8, 9, 88, 4087, 4086, 3, 4, 5, 4075, 0, 4, 0, 7, 4083, 10, 0, 4089, 0, 0, 0, 4087, 0, 0, 6, 0, 4062, 11, 0, 5, 9, 4086, 4082, 0, 10, 0, 4087, 6, 8, 4088, 4088, 7, 4085, 0, 3, 6, 7, 20, 7, 4084, 1456, 24, 6, 2, 0, 4089, 0, 6, 4083, 0, 7, 7, 3933, 4068, 0, 0, 0, 7, 21, 6, 2822, 4085, 5, 0, 7, 8, 0, 9, 5, 1, 4080, 0, 0, 0, 0, 4080, 4087, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 4094, 0, 2, 0, 4095, 0, 4095, 0, 0, 1, 0, 4095, 1, 0, 0, 0, 4095, 0, 4091, 0, 1, 0, 1, 1, 3, 4095, 4095, 1, 4094, 0, 0, 0, 1, 2, 1, 1, 0, 0, 1, 1, 0, 4095, 4094, 0, 1, 1, 4093, 0, 0, 2, 1, 4094, 2, 0, 4095, 0, 0, 0, 4095, 1, 1, 1, 0, 4093, 2, 0, 0, 1, 4094, 4094, 1, 2, 0, 4095, 1, 1, 4092, 4095, 1, 4045, 2, 1, 1, 4, 1, 1, 4095, 3686, 0, 0, 0, 0, 4095, 0, 1, 4093, 0, 1, 0, 4088, 4094, 0, 0, 0, 6, 2, 1, 457, 4094, 1, 0, 1, 1, 0, 1, 1, 1, 4094, 0, 0, 0, 0, 4094, 4095, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 137, 4095, 5, 4096, 0, 4, 0, 0, 4096, 0, 32, 0, 42, 4090, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 4096, 4, 0, 1, 0, 4096, 0, 0, 4092, 34, 0, 10, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4095, 4096, 0, 0, 0, 4091, 0, 0, 4069, 4096, 0, 3852, 0, 0, 0, 0, 0, 0, 4096, 3514, 0, 0, 0, 0, 4059, 1, 0, 4091, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 644, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 35, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 138, 0, 36, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 113, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4060, 4096, 0, 3888, 0, 0, 0, 0, 0, 0, 4096, 3816, 0, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 19, 0, 0, 202, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 10, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 37, 0, 21, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4, 0, 4096, 0, 0, 4096, 6, 0, 4, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4091, 4096, 0, 3689, 0, 0, 0, 0, 0, 0, 4096, 2677, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1758, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4082, 0, 0, 0, 4094, 0, 4094, 0, 0, 1, 1, 4094, 2, 0, 1, 0, 4094, 0, 4094, 0, 1, 0, 2, 2, 3, 4094, 4094, 2, 4094, 0, 0, 0, 2, 2, 1, 2, 0, 1, 1, 2, 6, 4095, 4094, 1, 2, 1, 4093, 0, 1, 0, 2, 4094, 2, 0, 4094, 0, 0, 0, 4094, 0, 0, 1, 0, 4089, 1, 0, 0, 2, 4094, 4093, 0, 2, 0, 4095, 1, 0, 4094, 4094, 1, 4042, 0, 2, 1, 0, 3, 2, 4094, 3738, 0, 1, 0, 0, 4095, 0, 1, 4092, 0, 2, 0, 4091, 4094, 0, 0, 0, 3, 0, 1, 420, 4094, 1, 0, 1, 2, 0, 2, 0, 0, 4094, 0, 0, 0, 1, 4093, 4095, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 31, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 26, 0, 19, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 4095, 10, 0, 4, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4095, 0, 0, 4082, 4096, 0, 4036, 0, 0, 0, 0, 0, 0, 4096, 3532, 0, 0, 0, 0, 4080, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 565, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 9, 0, 10, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4094, 4096, 0, 4057, 0, 0, 0, 0, 0, 0, 4096, 1966, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 2150, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3970, 0, 10, 0, 4048, 0, 4086, 0, 0, 11, 0, 4085, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 5, 4082, 0, 4, 0, 8, 10, 8, 0, 4075, 10, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 10, 5, 9, 5, 0, 8, 9, 88, 4087, 4086, 3, 4, 5, 4075, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7, 4083, 10, 0, 4089, 0, 0, 0, 4087, 0, 0, 6, 0, 4052, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 11, 0, 5, 9, 14, 4082, 0, 10, 0, 4087, 6, 8, 4088, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 3, 6, 7, 20, 7, 4084, 1456, 24, 6, 2, 0, 4089, 0, 6, 1105, 40, 20, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 7, 7, 3933, 4068, 0, 0, 0, 7, 21, 6, 2822, 2980, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 0, 9, 5, 1, 4080, 0, 0, 0, 0, 4080, 4087, 0, 0, 0, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16216, 16368, 16470, 16349, 16389, 14973, 17939, 16368] avg=16384.0000 max=17939 max_rank=6 min=14973 min_rank=5 imbalance_ratio=1.094910 max_load_reduction=113133 max_load_reduction_pct=0.863136 moved_to_echo=65669 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 2, 0, 4055, 0, 4095, 0, 0, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4091, 0, 1, 0, 1, 1, 3, 0, 4082, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 1, 0, 0, 1, 1, 0, 4095, 4094, 0, 1, 1, 4093, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4094, 2, 0, 4095, 0, 0, 0, 4095, 1, 1, 1, 0, 4060, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 1, 14, 4094, 1, 2, 0, 4095, 1, 1, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 2, 1, 1, 4, 1, 1, 4095, 3686, 0, 0, 0, 0, 4095, 0, 1, 1107, 40, 20, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 4088, 4094, 0, 0, 0, 6, 2, 1, 457, 2987, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 4094, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16343, 16369, 16372, 16364, 16397, 17112, 15730, 16385] avg=16384.0000 max=17112 max_rank=5 min=15730 min_rank=6 imbalance_ratio=1.044434 max_load_reduction=113960 max_load_reduction_pct=0.869446 moved_to_echo=65624 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 137, 4055, 5, 4096, 0, 4, 0, 0, 4096, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4090, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 4096, 0, 0, 4092, 34, 0, 10, 4096, 0, 0, 0, 0, 4061, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 14, 4096, 0, 0, 0, 4091, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3852, 0, 0, 0, 0, 0, 0, 4096, 3514, 0, 0, 0, 0, 4059, 1, 0, 1108, 40, 20, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 644, 2988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16523, 16405, 16366, 16405, 16367, 16703, 15916, 16387] avg=16384.0000 max=16703 max_rank=5 min=15916 min_rank=6 imbalance_ratio=1.019470 max_load_reduction=114369 max_load_reduction_pct=0.872566 moved_to_echo=65373 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 31, 4056, 0, 4096, 0, 0, 0, 0, 4096, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4095, 10, 0, 4, 4096, 0, 0, 0, 0, 4062, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 4095, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 4096, 3532, 0, 0, 0, 0, 4080, 0, 0, 1108, 40, 20, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 565, 2988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16401, 16391, 16370, 16375, 16383, 16925, 15840, 16387] avg=16384.0000 max=16925 max_rank=5 min=15840 min_rank=6 imbalance_ratio=1.033020 max_load_reduction=114147 max_load_reduction_pct=0.870872 moved_to_echo=65535 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 35, 4056, 0, 4096, 0, 0, 0, 0, 4096, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 113, 0, 0, 4096, 0, 0, 0, 0, 4062, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 4096, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3888, 0, 0, 0, 0, 0, 0, 4096, 3816, 0, 0, 0, 0, 4085, 0, 0, 1108, 40, 20, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 19, 0, 0, 202, 2988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16517, 16399, 16370, 16474, 16362, 17066, 15497, 16387] avg=16384.0000 max=17066 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.041626 max_load_reduction=114006 max_load_reduction_pct=0.869797 moved_to_echo=65312 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 10, 4054, 0, 4096, 0, 0, 0, 0, 4096, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 4096, 6, 0, 4, 4096, 0, 0, 0, 0, 4067, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3689, 0, 0, 0, 0, 0, 0, 4096, 2677, 0, 0, 0, 0, 4091, 0, 0, 1110, 42, 22, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1758, 2986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16386, 16371, 16380, 16386, 15742, 17032, 16386] avg=16384.0000 max=17032 max_rank=6 min=15742 min_rank=5 imbalance_ratio=1.039551 max_load_reduction=114040 max_load_reduction_pct=0.870056 moved_to_echo=65546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4082, 0, 0, 0, 4054, 0, 4094, 0, 0, 1, 1, 4094, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 0, 1, 0, 2, 2, 3, 0, 4081, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 1, 2, 0, 1, 1, 2, 6, 4095, 4094, 1, 2, 1, 4093, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 4094, 2, 0, 4094, 0, 0, 0, 4094, 0, 0, 1, 0, 4060, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 2, 14, 4093, 0, 2, 0, 4095, 1, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 2, 1, 0, 3, 2, 4094, 3738, 0, 1, 0, 0, 4095, 0, 1, 1107, 40, 20, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 0, 4091, 4094, 0, 0, 0, 3, 0, 1, 420, 2987, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 2, 0, 0, 4094, 0, 0, 0, 1, 4093, 4095, 1, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16329, 16373, 16383, 16359, 16392, 17159, 15691, 16386] avg=16384.0000 max=17159 max_rank=5 min=15691 min_rank=6 imbalance_ratio=1.047302 max_load_reduction=113913 max_load_reduction_pct=0.869087 moved_to_echo=65628 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 4056, 0, 4096, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4, 0, 0, 4096, 0, 0, 0, 0, 4062, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 4096, 1966, 0, 0, 0, 0, 4092, 0, 0, 1108, 40, 20, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 2150, 2988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16354, 16381, 16370, 16366, 16396, 15392, 17427, 16386] avg=16384.0000 max=17427 max_rank=6 min=15392 min_rank=5 imbalance_ratio=1.063660 max_load_reduction=113645 max_load_reduction_pct=0.867043 moved_to_echo=65601 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 32768, 0, 128, 0, 128, 128, 128, 0, 32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 32768, 256, 0, 128, 32768, 128, 128, 128, 0, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 32768, 128, 128, 0, 32768, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[31744, 128, 128, 128, 128, 128, 128, 32768, 24448, 128, 128, 128, 0, 32768, 128, 128, 8960, 384, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32640, 0, 128, 256, 32512, 128, 32768, 0, 128, 128, 128, 32768, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 0, 128, 128, 32640, 32768, 0, 0, 0, 128, 128, 128, 9088, 23936, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 128, 128, 32768, 0, 0, 0, 128, 32768, 32768, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4089, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 4096, 4096, 4088, 0, 0, 204, 0, 0, 4096, 0, 0, 0, 0, 3873, 0, 0, 0, 4096, 0, 0, 4096, 4092, 0, 0, 4096, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 4096, 0, 4071, 1, 0, 24, 0, 0, 0, 4096, 4096, 7, 1, 0, 0, 4087, 4, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 8, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 144, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 4096, 4096, 4096, 0, 0, 416, 0, 0, 4096, 0, 0, 0, 0, 3828, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 4096, 0, 4093, 0, 0, 28, 0, 0, 0, 4096, 4096, 6, 68, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 10, 4071, 9, 3956, 5, 0, 0, 0, 8, 5, 10, 18, 8, 4088, 13, 0, 9, 217, 0, 0, 4081, 0, 0, 0, 0, 10, 7, 10, 4083, 4025, 6, 8, 13, 0, 4019, 4083, 4060, 4088, 9, 0, 0, 12, 0, 4084, 7, 0, 109, 0, 4076, 7, 8, 0, 4037, 5, 2, 4085, 4085, 8, 0, 4086, 4086, 10, 0, 4078, 0, 0, 4079, 0, 9, 0, 10, 7, 0, 8, 3, 8, 84, 1, 23, 15, 21, 815, 4087, 4084, 4, 3932, 191, 7, 0, 0, 0, 0, 4055, 4081, 20, 0, 0, 0, 4068, 0, 0, 3, 0, 0, 4085, 5, 0, 4, 7, 14, 73, 3725, 8, 0, 0, 0, 9, 4087, 4070, 11, 9, 4078, 11, 3462, 0, 0, 4085] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3829, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 39, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 80, 0, 0, 4096, 0, 0, 0, 0, 4035, 0, 0, 0, 4096, 0, 0, 4096, 4091, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 4096, 3, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 1, 4093, 1, 4080, 1, 2, 2, 0, 1, 1, 0, 2, 1, 4095, 0, 0, 1, 70, 0, 1, 4094, 1, 1, 1, 0, 0, 1, 2, 4094, 4087, 1, 1, 2, 0, 4078, 4094, 4093, 4095, 1, 0, 2, 1, 0, 4095, 2, 0, 0, 0, 4019, 1, 1, 1, 4089, 1, 0, 4094, 4091, 1, 0, 4095, 4095, 1, 0, 4093, 0, 0, 4092, 0, 1, 0, 1, 1, 0, 1, 1, 1, 5, 0, 0, 1, 3, 170, 4095, 4094, 1, 4035, 15, 0, 0, 0, 0, 0, 4092, 4094, 8, 3, 0, 0, 4088, 1, 0, 0, 0, 0, 4094, 2, 0, 0, 1, 1, 16, 3996, 1, 0, 0, 0, 1, 4095, 4094, 2, 1, 4093, 2, 4095, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4058, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 4040, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 4096, 0, 4095, 0, 0, 69, 0, 0, 0, 4095, 4096, 2, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1, 2, 4093, 2, 4085, 1, 2, 0, 0, 1, 2, 2, 3, 1, 4095, 0, 0, 2, 102, 0, 0, 4094, 1, 0, 0, 0, 0, 1, 2, 4094, 4087, 1, 2, 2, 0, 4094, 4093, 4092, 4094, 0, 0, 0, 3, 0, 4094, 2, 0, 7, 0, 3877, 2, 1, 1, 4086, 1, 0, 4094, 4092, 2, 0, 4094, 4094, 3, 0, 4092, 0, 0, 4093, 0, 2, 0, 2, 1, 0, 1, 2, 2, 18, 0, 4, 1, 3, 296, 4094, 4094, 1, 4047, 11, 0, 0, 0, 0, 0, 4078, 4093, 13, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 1, 13, 3967, 1, 0, 0, 0, 1, 4094, 4092, 1, 2, 4093, 3, 4075, 1, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3910, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 4094, 0, 0, 61, 0, 0, 4096, 0, 0, 0, 0, 4061, 0, 0, 0, 4096, 0, 0, 4096, 4090, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 4096, 4096, 0, 4077, 0, 0, 88, 0, 0, 0, 4090, 4096, 21, 3, 0, 0, 4090, 4, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3967, 212, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3720, 217, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4000, 4096, 4096, 0, 0, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3828, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 3879, 0, 4093, 0, 0, 28, 0, 0, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4063, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16165, 16465, 16702, 16244, 16368, 16212, 16435, 16481] avg=16384.0000 max=16702 max_rank=2 min=16165 min_rank=0 imbalance_ratio=1.019409 max_load_reduction=114370 max_load_reduction_pct=0.872574 moved_to_echo=65496 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 4071, 9, 3956, 5, 0, 0, 0, 8, 5, 10, 18, 8, 4088, 13, 3957, 211, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 217, 0, 0, 4081, 0, 0, 0, 0, 10, 7, 10, 4083, 4025, 6, 3710, 216, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 13, 0, 4019, 3987, 4060, 4088, 9, 0, 0, 12, 0, 0, 7, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 7, 8, 0, 4037, 5, 2, 4085, 4085, 8, 0, 0, 129, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 4079, 0, 9, 0, 10, 7, 0, 8, 3, 8, 84, 1, 23, 4087, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 21, 815, 0, 3868, 4, 3932, 191, 7, 0, 0, 0, 0, 4055, 4081, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 3, 0, 0, 4085, 5, 0, 4, 7, 14, 73, 4086, 4051, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3725, 8, 0, 0, 0, 9, 0, 4070, 11, 9, 4078, 11, 3462, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16402, 16470, 16312, 16452, 16469, 17009, 16411, 15547] avg=16384.0000 max=17009 max_rank=5 min=15547 min_rank=7 imbalance_ratio=1.038147 max_load_reduction=114063 max_load_reduction_pct=0.870232 moved_to_echo=65436 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4089, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4090, 0, 3967, 212, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3720, 217, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4000, 4096, 4088, 0, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3873, 0, 0, 0, 4096, 0, 0, 4096, 4092, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 3879, 0, 4071, 1, 0, 24, 0, 0, 0, 4096, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4087, 4, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4063, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 8, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16491, 16321, 16483, 16286, 16369, 16193, 16372, 16557] avg=16384.0000 max=16557 max_rank=7 min=16193 min_rank=5 imbalance_ratio=1.010559 max_load_reduction=114515 max_load_reduction_pct=0.873680 moved_to_echo=65491 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4093, 1, 4080, 1, 2, 2, 0, 1, 1, 0, 2, 1, 4095, 0, 3966, 212, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 70, 0, 1, 4094, 1, 1, 1, 0, 0, 1, 2, 4094, 4087, 1, 3719, 217, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 4078, 3998, 4093, 4095, 1, 0, 2, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 1, 1, 1, 4089, 1, 0, 4094, 4091, 1, 0, 0, 129, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4092, 0, 1, 0, 1, 1, 0, 1, 1, 1, 5, 0, 0, 4095, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 170, 0, 3877, 1, 4035, 15, 0, 0, 0, 0, 0, 4092, 4094, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4088, 1, 0, 0, 0, 0, 4094, 2, 0, 0, 1, 1, 16, 4095, 4062, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 1, 0, 0, 0, 1, 0, 4094, 2, 1, 4093, 2, 4095, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16491, 16386, 16273, 16428, 16371, 16296, 16378, 16449] avg=16384.0000 max=16491 max_rank=0 min=16273 min_rank=2 imbalance_ratio=1.006531 max_load_reduction=114581 max_load_reduction_pct=0.874184 moved_to_echo=65494 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3910, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3967, 212, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3720, 217, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4000, 4096, 4094, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 4096, 0, 0, 4096, 4090, 0, 0, 0, 129, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 3879, 0, 4077, 0, 0, 88, 0, 0, 0, 4090, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4090, 4, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4063, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16318, 16321, 16347, 16473, 16368, 16322, 16373, 16550] avg=16384.0000 max=16550 max_rank=7 min=16318 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=114522 max_load_reduction_pct=0.873734 moved_to_echo=65613 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3829, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3970, 214, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3726, 221, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3997, 4096, 4096, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 4096, 0, 0, 4096, 4091, 0, 0, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 3875, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4062, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16243, 16373, 16365, 16444, 16369, 16369, 16369, 16540] avg=16384.0000 max=16540 max_rank=7 min=16243 min_rank=0 imbalance_ratio=1.009521 max_load_reduction=114532 max_load_reduction_pct=0.873810 moved_to_echo=65647 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4058, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3967, 212, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3720, 217, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4000, 4096, 4096, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 129, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3879, 0, 4095, 0, 0, 69, 0, 0, 0, 4095, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4063, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16463, 16321, 16297, 16457, 16369, 16251, 16367, 16547] avg=16384.0000 max=16547 max_rank=7 min=16251 min_rank=5 imbalance_ratio=1.009949 max_load_reduction=114525 max_load_reduction_pct=0.873756 moved_to_echo=65534 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 4093, 2, 4085, 1, 2, 0, 0, 1, 2, 2, 3, 1, 4095, 0, 3965, 211, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 102, 0, 0, 4094, 1, 0, 0, 0, 0, 1, 2, 4094, 4087, 1, 3718, 217, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 4094, 3997, 4092, 4094, 0, 0, 0, 3, 0, 0, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 2, 1, 1, 4086, 1, 0, 4094, 4092, 2, 0, 0, 129, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4093, 0, 2, 0, 2, 1, 0, 1, 2, 2, 18, 0, 4, 4094, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 296, 0, 3877, 1, 4047, 11, 0, 0, 0, 0, 0, 4078, 4093, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 1, 13, 4094, 4061, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 1, 0, 0, 0, 1, 0, 4092, 1, 2, 4093, 3, 4075, 1, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16499, 16415, 16293, 16288, 16389, 16420, 16367, 16401] avg=16384.0000 max=16499 max_rank=0 min=16288 min_rank=3 imbalance_ratio=1.007019 max_load_reduction=114573 max_load_reduction_pct=0.874123 moved_to_echo=65577 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 128, 128, 0, 32768, 128, 0, 128, 128, 128, 128, 32768, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 0, 32768, 0, 128, 0, 128, 128, 0, 128, 128, 128, 128, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 640, 0, 128, 32768, 128, 128, 128, 0, 128, 128, 128, 32768, 32768, 128, 29824, 1792, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 31872, 128, 128, 128, 32768, 128, 128, 32768, 32768, 128, 0, 0, 1152, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 1664, 0, 31104, 128, 32512, 256, 128, 256, 0, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132992 observed_loss=-1920 tokens_per_expert=[128, 128, 32768, 128, 31872, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 31744, 1792, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32128, 128, 128, 0, 128, 128, 0, 32768, 128, 128, 32768, 128, 32128, 128, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 0, 32768, 32000, 32768, 32768, 128, 0, 896, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132992 slack_slots=7040 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 4096, 0, 0, 0, 4084, 0, 4074, 4096, 2, 0, 0, 4096, 5, 4096, 0, 4096, 0, 0, 0, 0, 4095, 4091, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 13, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 4, 0, 0, 4094, 0, 4, 0, 4096, 35, 4093, 4082, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4094, 1, 4094, 4094, 0, 0, 0, 4095, 1, 4091, 4094, 0, 0, 1, 4095, 3, 4093, 1, 4094, 0, 1, 0, 0, 4095, 4095, 1, 4095, 1, 1, 4095, 0, 0, 1, 0, 2, 0, 2, 0, 1, 1, 0, 0, 1, 1, 1, 2, 0, 4095, 0, 2, 1, 0, 1, 2, 4090, 0, 1, 4095, 1, 4094, 5, 0, 1, 0, 4095, 1, 2, 2, 1, 1, 0, 0, 2, 0, 2, 4088, 0, 1, 4095, 4, 4093, 0, 11, 2, 0, 4093, 0, 0, 0, 0, 4094, 0, 4093, 0, 0, 2, 0, 1, 0, 1, 0, 4095, 6, 0, 1, 4094, 0, 4094, 4087, 1, 2, 0, 0, 0, 1, 0, 0, 0, 1, 4090, 1, 4094, 0, 0, 4090, 1, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4084, 8, 4081, 4086, 0, 0, 0, 4088, 1, 4081, 4085, 0, 0, 16, 4084, 37, 4073, 5, 4082, 0, 10, 2, 0, 4088, 4088, 8, 4080, 9, 8, 4087, 0, 0, 10, 0, 0, 0, 10, 0, 9, 7, 0, 0, 5, 8, 0, 17, 0, 4088, 0, 15, 0, 5, 7, 16, 4032, 0, 11, 4086, 8, 4066, 29, 0, 9, 0, 4086, 10, 16, 20, 8, 7, 0, 0, 25, 0, 9, 3978, 0, 4, 4088, 0, 4077, 0, 17, 7, 0, 4079, 140, 0, 3, 4, 4079, 0, 4069, 0, 0, 11, 0, 9, 0, 103, 0, 4087, 34, 0, 9, 4085, 0, 4086, 3951, 26, 3, 0, 4, 0, 6, 2, 0, 0, 1, 4065, 7, 4069, 0, 6, 4059, 7, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4094, 2, 4093, 4094, 0, 0, 0, 4094, 0, 4089, 4094, 0, 0, 2, 4094, 12, 4091, 0, 4093, 0, 2, 0, 0, 4094, 4095, 1, 4094, 2, 2, 4094, 0, 0, 2, 0, 2, 0, 2, 0, 0, 1, 0, 0, 1, 2, 1, 2, 0, 4095, 0, 2, 0, 0, 1, 2, 4086, 5, 2, 4094, 2, 4093, 9, 0, 2, 0, 4094, 2, 3, 1, 1, 2, 0, 0, 5, 0, 3, 4083, 0, 1, 4094, 0, 4093, 0, 7, 2, 0, 4093, 0, 0, 0, 0, 4093, 0, 4090, 0, 0, 2, 0, 2, 0, 15, 0, 4094, 4, 0, 2, 4094, 0, 4094, 4076, 4, 1, 0, 2, 0, 2, 0, 0, 0, 0, 4090, 1, 4094, 0, 1, 4090, 1, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4094, 0, 4096, 4096, 0, 0, 0, 4080, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 17, 4096, 4096, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4094, 0, 4096, 4096, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 4096, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4092, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4084, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 1, 0, 4096, 10, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4084, 0, 4074, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 4096, 4083, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4080, 0, 4, 0, 4096, 35, 4093, 4082, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4095, 8, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16365, 16381, 16375, 16380, 16397, 16385, 16395, 16394] avg=16384.0000 max=16397 max_rank=4 min=16365 min_rank=0 imbalance_ratio=1.000793 max_load_reduction=114675 max_load_reduction_pct=0.874901 moved_to_echo=65571 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 1, 4094, 0, 0, 0, 0, 4095, 1, 4091, 13, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 1, 4092, 0, 1, 0, 0, 0, 0, 1, 4095, 1, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 2, 0, 1, 1, 0, 0, 1, 1, 1, 2, 0, 4095, 4094, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 1, 0, 1, 2, 4090, 0, 1, 4095, 1, 4094, 5, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 2, 2, 1, 1, 0, 0, 2, 0, 2, 4088, 0, 1, 4095, 4081, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4093, 0, 11, 2, 0, 4093, 0, 0, 0, 0, 4086, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 1, 0, 4081, 6, 0, 1, 4094, 0, 4094, 4087, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4090, 1, 4094, 0, 0, 4090, 1, 4095, 8, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16383, 16387, 16389, 16387, 16382, 16370, 16384] avg=16384.0000 max=16390 max_rank=0 min=16370 min_rank=6 imbalance_ratio=1.000366 max_load_reduction=114682 max_load_reduction_pct=0.874954 moved_to_echo=65523 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4084, 8, 4081, 0, 0, 0, 0, 4088, 1, 4081, 13, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4073, 5, 4081, 0, 10, 2, 0, 0, 0, 8, 4080, 9, 8, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 10, 0, 9, 7, 0, 0, 5, 8, 0, 17, 0, 4088, 4086, 4084, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 15, 0, 5, 7, 16, 4032, 0, 11, 4086, 8, 4066, 29, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 10, 16, 20, 8, 7, 0, 0, 25, 0, 9, 3978, 0, 4, 4088, 4072, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 17, 7, 0, 4079, 140, 0, 3, 4, 4071, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 9, 0, 103, 0, 4073, 34, 0, 9, 4085, 0, 4086, 3951, 26, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6, 2, 0, 0, 1, 4065, 7, 4069, 0, 6, 4059, 7, 4087, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 16400, 16407, 16373, 16340, 16467, 16390, 16323] avg=16384.0000 max=16467 max_rank=5 min=16323 min_rank=7 imbalance_ratio=1.005066 max_load_reduction=114605 max_load_reduction_pct=0.874367 moved_to_echo=65520 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4083, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4096, 2, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 8, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16395, 16381, 16380, 16385, 16387, 16376, 16373, 16395] avg=16384.0000 max=16395 max_rank=0 min=16373 min_rank=6 imbalance_ratio=1.000671 max_load_reduction=114677 max_load_reduction_pct=0.874916 moved_to_echo=65531 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 4096, 0, 0, 0, 0, 4080, 0, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4083, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4096, 17, 4096, 4096, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 8, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16382, 16379, 16381, 16388, 16347, 16421, 16395] avg=16384.0000 max=16421 max_rank=6 min=16347 min_rank=5 imbalance_ratio=1.002258 max_load_reduction=114651 max_load_reduction_pct=0.874718 moved_to_echo=65551 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 2, 4093, 0, 0, 0, 0, 4094, 0, 4089, 13, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4091, 0, 4092, 0, 2, 0, 0, 0, 0, 1, 4094, 2, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 2, 0, 0, 1, 0, 0, 1, 2, 1, 2, 0, 4094, 4094, 4094, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 1, 2, 4086, 5, 2, 4094, 2, 4093, 9, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 3, 1, 1, 2, 0, 0, 5, 0, 3, 4083, 0, 1, 4094, 4081, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 7, 2, 0, 4093, 0, 0, 0, 0, 4085, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 15, 0, 4080, 4, 0, 2, 4094, 0, 4094, 4076, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 4090, 1, 4094, 0, 1, 4090, 1, 4094, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16390, 16385, 16394, 16387, 16370, 16374, 16385] avg=16384.0000 max=16394 max_rank=3 min=16370 min_rank=5 imbalance_ratio=1.000610 max_load_reduction=114678 max_load_reduction_pct=0.874924 moved_to_echo=65516 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 4096, 0, 0, 0, 0, 4095, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4086, 15, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 4096, 2, 4096, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 12, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16391, 16375, 16380, 16391, 16393, 16372, 16363, 16407] avg=16384.0000 max=16407 max_rank=7 min=16363 min_rank=6 imbalance_ratio=1.001404 max_load_reduction=114665 max_load_reduction_pct=0.874825 moved_to_echo=65535 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 4096, 4083, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4082, 0, 1, 0, 4096, 10, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4090, 8, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16393, 16380, 16379, 16379, 16393, 16373, 16386, 16389] avg=16384.0000 max=16393 max_rank=0 min=16373 min_rank=5 imbalance_ratio=1.000549 max_load_reduction=114679 max_load_reduction_pct=0.874931 moved_to_echo=65541 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 32768, 0, 128, 128, 0, 0, 0, 128, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 32768, 0, 0, 0, 0, 32768, 128, 32768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 128, 0, 32640, 128, 128, 128, 32768, 128, 32768, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 128, 0, 128, 128, 0, 128, 32640, 0, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 128, 128, 0, 128, 128, 0, 0, 128, 128, 128, 128, 0, 32768, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 128, 32768, 128, 32768, 0, 128, 32768, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 32768, 256, 0, 128, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4095, 0, 11, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 4067, 4096, 4096, 0, 0, 0, 0, 4090, 0, 4096, 46, 4090, 0, 0, 0, 1, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 16, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4027, 4091, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 220, 4038, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 3912, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4058, 2, 4094, 0, 1, 4090, 1, 0, 1, 4094, 0, 0, 0, 4093, 4094, 0, 0, 0, 4094, 0, 2, 2, 4092, 4094, 4094, 2, 1, 0, 4094, 1, 4, 4094, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 2, 2, 0, 4094, 0, 2, 1, 2, 2, 0, 2, 0, 4093, 4094, 4092, 5, 0, 0, 0, 4094, 1, 4088, 0, 4078, 0, 0, 2, 0, 1, 0, 1, 0, 2, 1, 0, 0, 2, 0, 0, 4091, 0, 2, 0, 4092, 2, 4094, 1, 0, 4094, 4094, 1, 2, 0, 0, 0, 2, 4, 0, 4094, 0, 4094, 1, 0, 1, 0, 2, 2, 2, 0, 0, 2, 4094, 4093, 1, 0, 4094, 4094, 4094, 1, 17, 1, 5, 2, 4094, 36, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 66, 4085, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4030, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[3903, 10, 4078, 4, 0, 4074, 8, 0, 9, 4087, 0, 0, 0, 4082, 4088, 0, 4, 0, 4087, 0, 9, 8, 4049, 4083, 4085, 1, 11, 0, 4086, 9, 37, 4087, 0, 0, 28, 9, 1, 0, 0, 5, 0, 0, 11, 12, 0, 4087, 0, 11, 8, 10, 10, 0, 7, 0, 4085, 4085, 4073, 13, 0, 23, 0, 4085, 5, 4061, 0, 3987, 0, 9, 7, 0, 9, 0, 1, 0, 0, 9, 0, 0, 9, 0, 0, 4069, 0, 2, 0, 4078, 0, 4084, 5, 0, 4080, 4086, 0, 0, 0, 0, 0, 9, 19, 10, 4075, 9, 4087, 7, 0, 8, 0, 10, 11, 11, 0, 0, 12, 4083, 4084, 0, 0, 4086, 4085, 4087, 6, 129, 6, 15, 11, 4087, 161, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 4096, 0, 0, 0, 0, 4095, 0, 4096, 12, 4092, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4084, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4068, 2, 4095, 0, 1, 4090, 0, 0, 1, 4094, 0, 0, 0, 4093, 4095, 0, 0, 0, 4095, 0, 1, 1, 4092, 4095, 4095, 0, 1, 0, 4094, 1, 0, 4095, 0, 0, 3, 0, 1, 1, 0, 0, 0, 0, 1, 2, 0, 4094, 0, 1, 1, 2, 2, 0, 1, 0, 4094, 4094, 4093, 2, 1, 0, 4, 4094, 1, 4087, 1, 4086, 0, 1, 1, 0, 1, 0, 2, 0, 1, 1, 0, 0, 2, 0, 0, 4092, 0, 2, 0, 4093, 6, 4094, 1, 0, 4094, 4095, 0, 1, 0, 0, 0, 1, 1, 0, 4094, 1, 4095, 1, 0, 1, 0, 1, 2, 2, 0, 0, 2, 4094, 4094, 0, 0, 4094, 4094, 4093, 1, 14, 1, 5, 2, 4094, 23, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 49, 4092, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4033, 4094, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 36, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4067, 10, 4096, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 4090, 0, 0, 0, 1, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 4095, 4094, 4052, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 16, 4096, 0, 4096, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4059, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 4027, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16431, 16385, 16377, 16362, 16424, 16406, 16375, 16312] avg=16384.0000 max=16431 max_rank=0 min=16312 min_rank=7 imbalance_ratio=1.002869 max_load_reduction=114641 max_load_reduction_pct=0.874641 moved_to_echo=65517 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3903, 10, 4078, 4, 0, 4074, 8, 0, 9, 36, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 9, 8, 4049, 4083, 4085, 1, 11, 0, 4086, 9, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 9, 1, 0, 0, 5, 0, 0, 11, 12, 0, 4087, 0, 11, 4088, 4087, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 10, 10, 0, 7, 0, 4085, 10, 4073, 13, 0, 23, 0, 4085, 5, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 9, 7, 0, 9, 0, 1, 0, 0, 9, 0, 0, 9, 0, 4087, 4085, 4042, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 2, 0, 4078, 0, 4084, 5, 0, 4080, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 19, 10, 4075, 9, 4087, 7, 0, 8, 0, 10, 11, 11, 0, 0, 4075, 4051, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4083, 4084, 0, 0, 1, 4085, 4087, 6, 129, 6, 15, 11, 0, 161, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16204, 16382, 16421, 16390, 16250, 16325, 16419, 16681] avg=16384.0000 max=16681 max_rank=7 min=16204 min_rank=0 imbalance_ratio=1.018127 max_load_reduction=114391 max_load_reduction_pct=0.872734 moved_to_echo=65675 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 36, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 10, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 4038, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 4095, 4094, 4052, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 4096, 0, 4096, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4060, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 3912, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16420, 16385, 16378, 16375, 16543, 16393, 16376, 16202] avg=16384.0000 max=16543 max_rank=4 min=16202 min_rank=7 imbalance_ratio=1.009705 max_load_reduction=114529 max_load_reduction_pct=0.873787 moved_to_echo=65514 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4068, 2, 4095, 0, 1, 4090, 0, 0, 1, 36, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4092, 4095, 4095, 0, 1, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 1, 0, 0, 0, 0, 1, 2, 0, 4094, 0, 1, 4095, 4095, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 1, 0, 4094, 10, 4093, 2, 1, 0, 4, 4094, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 0, 1, 1, 0, 1, 0, 2, 0, 1, 1, 0, 0, 2, 0, 4095, 4094, 4051, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 0, 4093, 6, 4094, 1, 0, 4094, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4094, 1, 4095, 1, 0, 1, 0, 1, 2, 2, 0, 0, 4084, 4058, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 4094, 0, 0, 0, 4094, 4093, 1, 14, 1, 5, 2, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16380, 16382, 16392, 16341, 16389, 16379, 16423] avg=16384.0000 max=16423 max_rank=7 min=16341 min_rank=4 imbalance_ratio=1.002380 max_load_reduction=114649 max_load_reduction_pct=0.874702 moved_to_echo=65532 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 30, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 4085, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 4096, 4053, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4066, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4030, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16414, 16384, 16378, 16388, 16413, 16387, 16390, 16318] avg=16384.0000 max=16414 max_rank=0 min=16318 min_rank=7 imbalance_ratio=1.001831 max_load_reduction=114658 max_load_reduction_pct=0.874771 moved_to_echo=65508 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4058, 2, 4094, 0, 1, 4090, 1, 0, 1, 36, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4092, 4094, 4094, 2, 1, 0, 4094, 1, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 2, 2, 0, 4094, 0, 2, 4094, 4094, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 2, 0, 4093, 10, 4092, 5, 0, 0, 0, 4094, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 2, 0, 1, 0, 1, 0, 2, 1, 0, 0, 2, 0, 4093, 4092, 4050, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2, 0, 4092, 2, 4094, 1, 0, 4094, 7, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 4094, 0, 4094, 1, 0, 1, 0, 2, 2, 2, 0, 0, 4084, 4058, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 4093, 1, 0, 2, 4094, 4094, 1, 17, 1, 5, 2, 0, 36, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16376, 16387, 16381, 16390, 16328, 16386, 16381, 16443] avg=16384.0000 max=16443 max_rank=7 min=16328 min_rank=4 imbalance_ratio=1.003601 max_load_reduction=114629 max_load_reduction_pct=0.874550 moved_to_echo=65538 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 36, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 10, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 4092, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 4095, 4094, 4052, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 4096, 0, 4096, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4060, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 4033, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16421, 16385, 16377, 16384, 16413, 16395, 16376, 16321] avg=16384.0000 max=16421 max_rank=0 min=16321 min_rank=7 imbalance_ratio=1.002258 max_load_reduction=114651 max_load_reduction_pct=0.874718 moved_to_echo=65505 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 36, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 10, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4092, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 4096, 4052, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 4096, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4060, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16420, 16384, 16378, 16391, 16360, 16391, 16376, 16372] avg=16384.0000 max=16420 max_rank=0 min=16360 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=114652 max_load_reduction_pct=0.874725 moved_to_echo=65499 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 32768, 128, 0, 128, 0, 128, 128, 128, 0, 0, 32768, 32512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 128, 32768, 128, 128, 128, 128, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32512, 128, 32768, 128, 128, 32768, 128, 0, 128, 384, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 32768, 128, 0, 128, 32384, 32768, 128, 256, 128, 128, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 0, 128, 128, 32768, 128, 32768, 128, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[512, 32640, 0, 128, 128, 128, 128, 0, 128, 128, 128, 128, 0, 0, 128, 0, 32768, 32768, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 0, 128, 128, 32768, 32768, 32768, 128, 128, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 128, 128, 0, 128, 0, 0, 128, 128, 0, 32768, 0, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[27, 200, 466, 0, 87, 0, 1299, 6, 941, 151, 61, 1721, 925, 1059, 0, 3955, 3420, 350, 1368, 1053, 3098, 2167, 12, 54, 4, 41, 1033, 430, 2127, 0, 1481, 498, 4083, 0, 11, 2144, 2250, 37, 2301, 1975, 1130, 5, 316, 512, 0, 1050, 228, 2009, 2545, 3830, 2195, 13, 5, 0, 556, 1593, 0, 1525, 688, 1585, 1920, 1134, 2226, 1, 2044, 3083, 0, 956, 400, 36, 338, 468, 3956, 1448, 212, 11, 116, 0, 31, 653, 2231, 2422, 0, 1153, 40, 2194, 777, 2164, 460, 4, 1681, 2746, 933, 588, 1377, 8, 1986, 2505, 771, 119, 6, 2368, 645, 1146, 911, 550, 4, 1778, 2367, 837, 114, 493, 250, 1012, 2542, 2131, 15, 256, 153, 631, 2029, 1534, 507, 2793, 27, 5, 2155, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[32, 347, 478, 0, 52, 0, 1283, 4, 883, 223, 47, 1941, 1457, 1938, 0, 3778, 2537, 275, 1304, 1439, 2778, 2229, 37, 78, 0, 68, 1331, 825, 2615, 0, 1884, 413, 4041, 0, 16, 2410, 1978, 42, 2314, 1618, 1484, 5, 535, 488, 0, 1441, 146, 1748, 2899, 3675, 2080, 15, 1, 2, 417, 1060, 0, 1474, 576, 1244, 2225, 912, 2211, 9, 3242, 3203, 0, 806, 239, 233, 303, 811, 3857, 1790, 331, 4, 115, 6, 95, 315, 1809, 2738, 1, 1221, 107, 2428, 906, 2111, 182, 12, 1594, 2565, 851, 666, 1061, 3, 1533, 3268, 545, 138, 0, 2026, 612, 1551, 888, 396, 0, 1614, 2405, 599, 203, 341, 219, 785, 2151, 1522, 68, 173, 210, 783, 2479, 1013, 372, 2336, 41, 1, 1852, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[55, 396, 588, 0, 86, 0, 855, 7, 868, 154, 27, 1623, 1802, 824, 0, 3949, 3379, 264, 1664, 1855, 2468, 1879, 7, 63, 0, 9, 1493, 513, 2298, 0, 1410, 307, 4045, 1, 9, 1908, 1964, 27, 2971, 1347, 945, 9, 444, 686, 0, 1018, 190, 2119, 3084, 3511, 2129, 28, 0, 18, 855, 1735, 0, 1562, 593, 909, 2180, 825, 2268, 0, 2784, 3282, 0, 597, 335, 115, 473, 912, 4013, 1607, 286, 1, 180, 2, 85, 374, 1752, 2988, 1, 1540, 59, 2269, 470, 2760, 188, 5, 1498, 2772, 1040, 437, 1814, 0, 780, 2979, 721, 188, 0, 1751, 688, 1813, 1043, 536, 1, 1513, 2276, 741, 128, 248, 223, 865, 1493, 2482, 46, 117, 151, 882, 2452, 924, 640, 2358, 47, 0, 2124, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[106, 509, 431, 0, 44, 1, 1153, 4, 1803, 194, 158, 2222, 1392, 1727, 0, 3948, 3108, 304, 1181, 695, 3017, 2212, 81, 97, 2, 61, 964, 519, 1979, 0, 1906, 633, 3792, 1, 19, 2058, 2015, 83, 1808, 1201, 828, 14, 415, 798, 0, 1664, 172, 1042, 2788, 3296, 2004, 85, 4, 6, 477, 1132, 0, 2002, 1214, 1544, 2091, 764, 2200, 12, 1963, 3255, 0, 801, 276, 101, 215, 616, 3717, 1488, 282, 9, 123, 5, 41, 1302, 1413, 2429, 0, 1094, 52, 2189, 836, 1929, 501, 0, 1435, 2853, 581, 979, 1338, 4, 1750, 3366, 625, 285, 6, 2289, 974, 1061, 980, 615, 3, 1309, 2291, 806, 204, 484, 271, 785, 2416, 2837, 59, 430, 278, 794, 2042, 1373, 410, 2986, 36, 0, 1493, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[6, 222, 399, 0, 15, 1, 1327, 6, 667, 136, 84, 1598, 946, 995, 1, 3813, 3193, 372, 820, 1338, 2423, 2605, 4, 58, 4, 60, 1571, 396, 2337, 1, 1468, 556, 4083, 0, 8, 2263, 1493, 36, 2517, 1640, 1518, 0, 230, 782, 0, 1078, 383, 1956, 2459, 3368, 2015, 37, 0, 0, 612, 1673, 0, 1455, 720, 1412, 1516, 1345, 2448, 1, 2641, 3397, 1, 755, 388, 66, 298, 668, 3932, 1209, 310, 2, 127, 3, 51, 966, 1828, 2399, 0, 1267, 70, 1973, 441, 2288, 431, 2, 1580, 3173, 818, 657, 2030, 1, 1446, 3187, 671, 199, 2, 1806, 666, 1466, 1211, 481, 0, 867, 2329, 1116, 213, 341, 374, 876, 2432, 1778, 25, 288, 293, 767, 2168, 1858, 838, 2669, 27, 0, 2436, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[10, 211, 683, 0, 17, 0, 793, 0, 1110, 152, 70, 1988, 1365, 1327, 0, 3886, 3007, 262, 2256, 1270, 2627, 1936, 6, 58, 0, 14, 1539, 684, 2267, 1, 1826, 1013, 4062, 0, 13, 2328, 2043, 20, 2420, 1488, 874, 5, 413, 897, 0, 1435, 167, 1993, 2929, 3308, 2282, 30, 0, 0, 825, 1468, 0, 1393, 884, 1241, 2009, 1738, 2416, 0, 2860, 3241, 0, 479, 298, 92, 510, 907, 3776, 2225, 321, 3, 82, 0, 52, 424, 2010, 2576, 0, 1481, 31, 1528, 532, 2272, 252, 4, 1645, 3042, 956, 358, 1162, 0, 840, 2634, 599, 119, 0, 2068, 682, 1060, 1128, 380, 1, 1611, 1860, 569, 89, 215, 305, 843, 1939, 2061, 27, 222, 233, 675, 2405, 1234, 462, 2746, 22, 0, 1865, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[36, 141, 492, 1, 16, 0, 1191, 4, 1479, 186, 85, 2132, 1286, 1135, 0, 3747, 2711, 537, 2294, 1930, 2364, 3021, 16, 96, 2, 46, 1063, 543, 3100, 0, 2214, 512, 4075, 1, 47, 2483, 2462, 71, 2282, 1461, 1098, 15, 520, 1017, 1, 1020, 105, 1622, 3018, 3612, 2363, 31, 0, 0, 810, 1474, 0, 1638, 718, 1219, 2522, 995, 2339, 0, 2365, 3284, 2, 545, 249, 110, 229, 662, 3641, 1166, 314, 4, 117, 10, 74, 291, 1547, 2977, 0, 824, 64, 2125, 1293, 1920, 143, 8, 1198, 2109, 424, 528, 1482, 1, 1260, 3088, 210, 173, 1, 2163, 401, 1688, 832, 349, 1, 1102, 2296, 585, 174, 142, 178, 1635, 2129, 2493, 43, 192, 326, 667, 2395, 1014, 484, 2199, 13, 1, 1731, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[29, 241, 336, 2, 14, 2, 1028, 3, 1041, 165, 99, 1846, 1087, 1462, 0, 4021, 3332, 456, 1399, 1029, 2644, 2188, 75, 150, 0, 45, 1410, 493, 1824, 4, 1745, 502, 4021, 1, 8, 2068, 1474, 36, 2287, 1630, 729, 5, 538, 600, 0, 1306, 320, 1761, 3234, 3654, 2367, 26, 0, 7, 839, 1631, 0, 1371, 751, 1334, 2216, 1381, 2001, 2, 1859, 3105, 0, 907, 318, 98, 263, 625, 3801, 1430, 452, 2, 177, 12, 55, 733, 1551, 2586, 0, 1073, 24, 2008, 869, 2192, 431, 9, 1771, 2956, 1030, 762, 1468, 0, 1629, 2724, 986, 221, 0, 2125, 515, 1070, 922, 468, 0, 1029, 2447, 894, 415, 376, 357, 783, 2381, 2512, 22, 463, 640, 669, 2002, 1231, 882, 2716, 10, 1, 1745, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[27, 200, 466, 0, 87, 0, 1299, 6, 941, 151, 61, 1721, 925, 1059, 0, 3955, 3795, 1193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1764, 350, 1368, 1053, 3098, 2167, 12, 54, 4, 41, 1033, 430, 2127, 0, 1481, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2811, 0, 11, 2144, 2250, 37, 2301, 1975, 1130, 5, 316, 512, 0, 1050, 228, 2009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2545, 35, 2195, 13, 5, 0, 556, 1593, 0, 1525, 688, 1585, 1920, 1134, 2226, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2044, 3083, 0, 956, 400, 36, 338, 468, 3956, 1448, 212, 11, 116, 0, 31, 653, 1389, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2231, 2422, 0, 1153, 40, 2194, 777, 2164, 460, 4, 1681, 847, 933, 588, 1377, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1986, 2505, 771, 119, 6, 2368, 645, 1146, 911, 550, 4, 1778, 2367, 837, 114, 493, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 1012, 2542, 2131, 15, 256, 153, 631, 2029, 1534, 507, 2793, 27, 5, 2155, 1, 543, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15886, 15480, 16779, 16021, 15847, 16879, 17329, 16851] avg=16384.0000 max=17329 max_rank=6 min=15480 min_rank=1 imbalance_ratio=1.057678 max_load_reduction=113743 max_load_reduction_pct=0.867790 moved_to_echo=66906 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[106, 509, 431, 0, 44, 1, 1153, 4, 1803, 194, 158, 2222, 1392, 1727, 0, 3948, 3266, 1239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1604, 304, 1181, 695, 3017, 2212, 81, 97, 2, 61, 964, 519, 1979, 0, 1906, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2610, 1, 19, 2058, 2015, 83, 1808, 1201, 828, 14, 415, 798, 0, 1664, 172, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2788, 30, 2004, 85, 4, 6, 477, 1132, 0, 2002, 1214, 1544, 2091, 764, 2200, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1963, 3255, 0, 801, 276, 101, 215, 616, 3717, 1488, 282, 9, 123, 5, 41, 1302, 1262, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1413, 2429, 0, 1094, 52, 2189, 836, 1929, 501, 0, 1435, 881, 581, 979, 1338, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1750, 3366, 625, 285, 6, 2289, 974, 1061, 980, 615, 3, 1309, 2291, 806, 204, 484, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 785, 2416, 2837, 59, 430, 278, 794, 2042, 1373, 410, 2986, 36, 0, 1493, 7, 505, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18197, 15255, 14728, 16353, 16189, 15661, 17725, 16964] avg=16384.0000 max=18197 max_rank=0 min=14728 min_rank=2 imbalance_ratio=1.110657 max_load_reduction=112875 max_load_reduction_pct=0.861168 moved_to_echo=66539 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[55, 396, 588, 0, 86, 0, 855, 7, 868, 154, 27, 1623, 1802, 824, 0, 3949, 3479, 1204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 264, 1664, 1855, 2468, 1879, 7, 63, 0, 9, 1493, 513, 2298, 0, 1410, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2784, 1, 9, 1908, 1964, 27, 2971, 1347, 945, 9, 444, 686, 0, 1018, 190, 2119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3084, 32, 2129, 28, 0, 18, 855, 1735, 0, 1562, 593, 909, 2180, 825, 2268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2784, 3282, 0, 597, 335, 115, 473, 912, 4013, 1607, 286, 1, 180, 2, 85, 374, 1373, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1752, 2988, 1, 1540, 59, 2269, 470, 2760, 188, 5, 1498, 856, 1040, 437, 1814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 780, 2979, 721, 188, 0, 1751, 688, 1813, 1043, 536, 1, 1513, 2276, 741, 128, 248, 723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 865, 1493, 2482, 46, 117, 151, 882, 2452, 924, 640, 2358, 47, 0, 2124, 0, 538, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15917, 15972, 16422, 16218, 17131, 17677, 16129, 15606] avg=16384.0000 max=17677 max_rank=5 min=15606 min_rank=7 imbalance_ratio=1.078918 max_load_reduction=113395 max_load_reduction_pct=0.865135 moved_to_echo=66543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[6, 222, 399, 0, 15, 1, 1327, 6, 667, 136, 84, 1598, 946, 995, 1, 3813, 3337, 1378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1647, 372, 820, 1338, 2423, 2605, 4, 58, 4, 60, 1571, 396, 2337, 1, 1468, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2811, 0, 8, 2263, 1493, 36, 2517, 1640, 1518, 0, 230, 782, 0, 1078, 383, 1956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2459, 31, 2015, 37, 0, 0, 612, 1673, 0, 1455, 720, 1412, 1516, 1345, 2448, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2641, 3397, 1, 755, 388, 66, 298, 668, 3932, 1209, 310, 2, 127, 3, 51, 966, 1297, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1828, 2399, 0, 1267, 70, 1973, 441, 2288, 431, 2, 1580, 979, 818, 657, 2030, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1446, 3187, 671, 199, 2, 1806, 666, 1466, 1211, 481, 0, 867, 2329, 1116, 213, 341, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 876, 2432, 1778, 25, 288, 293, 767, 2168, 1858, 838, 2669, 27, 0, 2436, 0, 543, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14931, 15660, 16715, 15724, 16927, 16764, 16730, 17621] avg=16384.0000 max=17621 max_rank=7 min=14931 min_rank=0 imbalance_ratio=1.075500 max_load_reduction=113451 max_load_reduction_pct=0.865562 moved_to_echo=68042 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[32, 347, 478, 0, 52, 0, 1283, 4, 883, 223, 47, 1941, 1457, 1938, 0, 3778, 3641, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1309, 275, 1304, 1439, 2778, 2229, 37, 78, 0, 68, 1331, 825, 2615, 0, 1884, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2781, 0, 16, 2410, 1978, 42, 2314, 1618, 1484, 5, 535, 488, 0, 1441, 146, 1748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2899, 34, 2080, 15, 1, 2, 417, 1060, 0, 1474, 576, 1244, 2225, 912, 2211, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3242, 3203, 0, 806, 239, 233, 303, 811, 3857, 1790, 331, 4, 115, 6, 95, 315, 1030, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1809, 2738, 1, 1221, 107, 2428, 906, 2111, 182, 12, 1594, 792, 851, 666, 1061, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1533, 3268, 545, 138, 0, 2026, 612, 1551, 888, 396, 0, 1614, 2405, 599, 203, 341, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 785, 2151, 1522, 68, 173, 210, 783, 2479, 1013, 372, 2336, 41, 1, 1852, 1, 538, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17218, 16585, 17006, 15159, 17039, 16482, 16841, 14742] avg=16384.0000 max=17218 max_rank=0 min=14742 min_rank=7 imbalance_ratio=1.050903 max_load_reduction=113854 max_load_reduction_pct=0.868637 moved_to_echo=65104 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[10, 211, 683, 0, 17, 0, 793, 0, 1110, 152, 70, 1988, 1365, 1327, 0, 3886, 3277, 1321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1551, 262, 2256, 1270, 2627, 1936, 6, 58, 0, 14, 1539, 684, 2267, 1, 1826, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2795, 0, 13, 2328, 2043, 20, 2420, 1488, 874, 5, 413, 897, 0, 1435, 167, 1993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2929, 31, 2282, 30, 0, 0, 825, 1468, 0, 1393, 884, 1241, 2009, 1738, 2416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2860, 3241, 0, 479, 298, 92, 510, 907, 3776, 2225, 321, 3, 82, 0, 52, 424, 1221, 782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2010, 2576, 0, 1481, 31, 1528, 532, 2272, 252, 4, 1645, 939, 956, 358, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 840, 2634, 599, 119, 0, 2068, 682, 1060, 1128, 380, 1, 1611, 1860, 569, 89, 215, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 843, 1939, 2061, 27, 222, 233, 675, 2405, 1234, 462, 2746, 22, 0, 1865, 0, 541, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16210, 17310, 16891, 17246, 17273, 15746, 14581, 15815] avg=16384.0000 max=17310 max_rank=1 min=14581 min_rank=6 imbalance_ratio=1.056519 max_load_reduction=113762 max_load_reduction_pct=0.867935 moved_to_echo=63415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[36, 141, 492, 1, 16, 0, 1191, 4, 1479, 186, 85, 2132, 1286, 1135, 0, 3747, 3582, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1390, 537, 2294, 1930, 2364, 3021, 16, 96, 2, 46, 1063, 543, 3100, 0, 2214, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2795, 1, 47, 2483, 2462, 71, 2282, 1461, 1098, 15, 520, 1017, 1, 1020, 105, 1622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3018, 30, 2363, 31, 0, 0, 810, 1474, 0, 1638, 718, 1219, 2522, 995, 2339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2365, 3284, 2, 545, 249, 110, 229, 662, 3641, 1166, 314, 4, 117, 10, 74, 291, 1106, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1547, 2977, 0, 824, 64, 2125, 1293, 1920, 143, 8, 1198, 643, 424, 528, 1482, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1260, 3088, 210, 173, 1, 2163, 401, 1688, 832, 349, 1, 1102, 2296, 585, 174, 142, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 1635, 2129, 2493, 43, 192, 326, 667, 2395, 1014, 484, 2199, 13, 1, 1731, 2, 547, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16433, 19128, 17000, 17157, 14715, 15177, 15198, 16264] avg=16384.0000 max=19128 max_rank=1 min=14715 min_rank=4 imbalance_ratio=1.167480 max_load_reduction=111944 max_load_reduction_pct=0.854065 moved_to_echo=61354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[29, 241, 336, 2, 14, 2, 1028, 3, 1041, 165, 99, 1846, 1087, 1462, 0, 4021, 3620, 1284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1718, 456, 1399, 1029, 2644, 2188, 75, 150, 0, 45, 1410, 493, 1824, 4, 1745, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2768, 1, 8, 2068, 1474, 36, 2287, 1630, 729, 5, 538, 600, 0, 1306, 320, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3234, 34, 2367, 26, 0, 7, 839, 1631, 0, 1371, 751, 1334, 2216, 1381, 2001, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1859, 3105, 0, 907, 318, 98, 263, 625, 3801, 1430, 452, 2, 177, 12, 55, 733, 1354, 760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1551, 2586, 0, 1073, 24, 2008, 869, 2192, 431, 9, 1771, 912, 1030, 762, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1629, 2724, 986, 221, 0, 2125, 515, 1070, 922, 468, 0, 1029, 2447, 894, 415, 376, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 783, 2381, 2512, 22, 463, 640, 669, 2002, 1231, 882, 2716, 10, 1, 1745, 0, 535, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16280, 15682, 15531, 17194, 15951, 16686, 16539, 17209] avg=16384.0000 max=17209 max_rank=7 min=15531 min_rank=2 imbalance_ratio=1.050354 max_load_reduction=113863 max_load_reduction_pct=0.868706 moved_to_echo=66385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2304, 7680, 17536, 17920, 384, 2176, 2304, 5888, 18048, 10240, 4608, 20864, 256, 128, 15488, 128, 4352, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[11264, 23808, 5248, 1536, 128, 16640, 5248, 10880, 7936, 3840, 128, 10880, 18304, 6272, 1664, 2688, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[14208, 21120, 128, 9728, 512, 16768, 6144, 17664, 2688, 128, 12416, 6912, 6656, 4992, 11776, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 2304, 3968, 128, 384, 128, 8960, 128, 8832, 1408, 640, 15104, 10368, 10496, 128, 31104, 28032, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[22272, 128, 256, 17664, 15744, 384, 18944, 12416, 8704, 128, 3456, 5888, 128, 10112, 1792, 14336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12800, 2944, 12288, 10624, 21504, 18304, 256, 768, 128, 384, 10496, 4480, 18560, 128, 13952, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[23040, 384, 17536, 384, 128, 128, 5504, 11776, 0, 12544, 6144, 10496, 16768, 9216, 18176, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[19840, 25856, 128, 5888, 2560, 896, 2688, 5760, 30720, 12416, 2560, 128, 1152, 128, 512, 5120, 10112, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 6, 4096, 5, 0, 4096, 4096, 0, 26, 17, 0, 0, 0, 2134, 0, 0, 4096, 4, 0, 0, 0, 4096, 3769, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 419, 378, 0, 4, 0, 4096, 0, 0, 4096, 4066, 17, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4091, 3485, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 4090, 1, 0, 0, 0, 0, 3871, 0, 0, 2239, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 4074, 12, 0, 0, 0, 0, 4074, 0, 4096, 4096, 0, 4096, 0, 11, 4096, 0, 14, 0, 4096, 4, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 4095, 2, 2, 35, 0, 0, 0, 3525, 0, 0, 4096, 0, 0, 0, 0, 4095, 2595, 4096, 0, 0, 4092, 0, 0, 4096, 0, 2, 254, 1272, 0, 0, 0, 4096, 0, 0, 4095, 3998, 61, 0, 4094, 0, 1, 0, 1, 5, 0, 0, 4096, 16, 4096, 4096, 0, 0, 4096, 2553, 0, 62, 0, 0, 4, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 45, 0, 0, 0, 0, 2905, 0, 0, 3136, 4096, 0, 0, 0, 4083, 41, 4086, 0, 0, 4055, 8, 0, 56, 0, 0, 4092, 0, 4095, 4094, 0, 4095, 0, 25, 4096, 0, 1, 0, 4033, 23, 0, 0, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 4081, 4090, 6, 9, 222, 1, 1, 0, 3032, 0, 5, 4096, 4, 0, 0, 0, 4082, 2986, 4096, 1, 0, 4096, 7, 0, 4091, 0, 116, 211, 1509, 0, 0, 0, 4090, 0, 0, 4078, 3956, 35, 0, 4089, 0, 0, 14, 0, 0, 0, 1, 4071, 14, 4090, 4093, 3, 0, 4096, 1500, 0, 415, 3, 0, 62, 0, 1, 1, 2, 2, 7, 10, 2, 0, 0, 0, 0, 0, 0, 3, 5, 4096, 0, 0, 4084, 27, 0, 0, 0, 0, 3352, 0, 2, 3190, 4095, 0, 0, 0, 4096, 5, 4096, 0, 0, 4094, 0, 1, 9, 1, 1, 4033, 0, 4093, 4096, 2, 4089, 0, 26, 4091, 0, 11, 0, 3987, 16, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 18, 90, 0, 0, 0, 3746, 0, 0, 4096, 3, 0, 0, 1, 4096, 3042, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 92, 784, 0, 0, 0, 4096, 0, 0, 4096, 4093, 1, 0, 4093, 0, 0, 1, 0, 1, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4095, 1808, 0, 332, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 4066, 18, 0, 0, 0, 0, 3216, 0, 0, 3183, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4095, 16, 0, 0, 0, 0, 4081, 0, 4096, 4096, 0, 4096, 0, 2, 4096, 0, 80, 0, 4096, 1, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 5, 188, 0, 0, 0, 4063, 0, 0, 4096, 6, 0, 0, 0, 4096, 3392, 4095, 0, 0, 4090, 0, 1, 4096, 0, 0, 120, 366, 0, 7, 0, 4096, 0, 0, 4096, 4088, 3, 0, 4095, 0, 0, 1, 0, 0, 0, 1, 4095, 16, 4096, 4096, 0, 2, 4094, 975, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4091, 13, 0, 1, 0, 1, 3941, 0, 0, 3336, 4092, 0, 0, 3, 4094, 1, 4094, 0, 0, 4007, 16, 0, 0, 0, 0, 4096, 1, 4096, 4096, 0, 4091, 0, 0, 4095, 0, 4, 11, 4094, 1, 1, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 4095, 0, 0, 4096, 4096, 0, 0, 152, 0, 0, 0, 3879, 1, 0, 4095, 6, 0, 0, 1, 4096, 3511, 4096, 0, 1, 4096, 0, 0, 4096, 0, 1, 344, 272, 0, 4, 0, 4095, 0, 0, 4095, 4087, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4093, 1453, 0, 22, 0, 1, 0, 0, 0, 0, 8, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4093, 2, 0, 0, 0, 1, 3901, 0, 0, 2856, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 4046, 31, 0, 6, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 1, 4095, 0, 0, 0, 4094, 3, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 4093, 3, 1, 4094, 4093, 0, 3, 489, 0, 0, 0, 3071, 0, 1, 4094, 3, 1, 0, 0, 4095, 3564, 4094, 2, 2, 4095, 0, 2, 4094, 2, 2, 652, 432, 0, 0, 0, 4090, 0, 1, 4095, 4084, 0, 2, 4090, 1, 1, 2, 0, 4, 1, 0, 4092, 17, 4094, 4094, 1, 2, 4091, 1631, 2, 37, 0, 0, 4, 1, 0, 0, 2, 2, 2, 0, 1, 0, 0, 1, 0, 2, 0, 1, 0, 4095, 0, 0, 4091, 1, 0, 2, 0, 0, 3241, 2, 0, 3192, 4094, 2, 0, 1, 4092, 0, 4094, 0, 2, 4065, 92, 1, 1, 0, 2, 4093, 0, 4092, 4090, 1, 4094, 1, 7, 4092, 0, 4, 0, 4089, 5, 0, 1, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 2, 1, 4093, 4093, 0, 0, 195, 1, 1, 0, 3256, 0, 1, 4094, 10, 1, 0, 2, 4091, 3710, 4095, 0, 0, 4092, 4, 2, 4095, 0, 0, 542, 652, 0, 0, 2, 4093, 1, 0, 4095, 4081, 3, 1, 4088, 2, 1, 0, 0, 0, 1, 0, 4092, 17, 4095, 4094, 1, 2, 4093, 2185, 1, 41, 1, 0, 0, 1, 1, 0, 1, 1, 1, 4, 1, 0, 0, 1, 0, 2, 0, 0, 2, 4095, 0, 1, 4093, 7, 0, 1, 1, 1, 2548, 1, 0, 3070, 4094, 0, 1, 0, 4095, 0, 4095, 0, 1, 4027, 219, 1, 4, 0, 1, 4093, 0, 4093, 4095, 2, 4093, 1, 5, 4095, 1, 0, 0, 4095, 8, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 4081, 4090, 6, 9, 222, 1, 1, 0, 3032, 0, 559, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 772, 4, 0, 0, 0, 4082, 2986, 0, 1, 0, 4096, 7, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 211, 1509, 0, 0, 0, 3069, 0, 0, 4078, 3956, 35, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 1, 4071, 14, 4090, 2012, 3, 0, 4096, 1500, 0, 415, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 1, 1, 2, 2, 7, 10, 2, 0, 0, 0, 0, 0, 0, 4096, 4096, 3324, 2305, 2081, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 4096, 0, 0, 4084, 27, 0, 0, 0, 0, 3352, 0, 2, 3190, 1790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 4086, 0, 0, 4094, 0, 1, 9, 1, 1, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 2, 4089, 0, 26, 4091, 0, 11, 0, 3987, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16111, 16044, 17063, 16219, 16451, 16549, 16326, 16309] avg=16384.0000 max=17063 max_rank=2 min=16044 min_rank=1 imbalance_ratio=1.041443 max_load_reduction=114009 max_load_reduction_pct=0.869820 moved_to_echo=65635 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 4096, 5, 0, 4096, 4096, 0, 26, 17, 0, 0, 0, 2134, 0, 559, 10, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 772, 4, 0, 0, 0, 4096, 3769, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 419, 378, 0, 4, 0, 3074, 0, 0, 4096, 4066, 17, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2014, 0, 0, 4091, 3485, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 3324, 2306, 2082, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 4090, 1, 0, 0, 0, 0, 3871, 0, 0, 2239, 1790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4085, 0, 0, 4074, 12, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 11, 4096, 0, 14, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15052, 16833, 16148, 17843, 16368, 16081, 16341, 16406] avg=16384.0000 max=17843 max_rank=3 min=15052 min_rank=0 imbalance_ratio=1.089050 max_load_reduction=113229 max_load_reduction_pct=0.863869 moved_to_echo=65196 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 4095, 2, 2, 35, 0, 0, 0, 3525, 0, 559, 10, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 772, 0, 0, 0, 0, 4095, 2595, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 254, 1272, 0, 0, 0, 3074, 0, 0, 4095, 3998, 61, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 4096, 16, 4096, 2014, 0, 0, 4096, 2553, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 3324, 2306, 2082, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 45, 0, 0, 0, 0, 2905, 0, 0, 3136, 1790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 41, 4076, 0, 0, 4055, 8, 0, 56, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4095, 0, 25, 4096, 0, 1, 0, 4033, 23, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16427, 15650, 16851, 16939, 16373, 16056, 16411, 16365] avg=16384.0000 max=16939 max_rank=3 min=15650 min_rank=1 imbalance_ratio=1.033875 max_load_reduction=114133 max_load_reduction_pct=0.870766 moved_to_echo=65205 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 18, 90, 0, 0, 0, 3746, 0, 559, 10, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 772, 3, 0, 0, 1, 4096, 3042, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 784, 0, 0, 0, 3074, 0, 0, 4096, 4093, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4096, 0, 4096, 2014, 0, 0, 4095, 1808, 0, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 3324, 2306, 2082, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 4066, 18, 0, 0, 0, 0, 3216, 0, 0, 3183, 1790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 4095, 16, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 2, 4096, 0, 80, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16717, 16107, 16233, 16443, 16371, 16367, 16374, 16460] avg=16384.0000 max=16717 max_rank=0 min=16107 min_rank=1 imbalance_ratio=1.020325 max_load_reduction=114355 max_load_reduction_pct=0.872459 moved_to_echo=65572 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 2, 1, 4093, 4093, 0, 0, 195, 1, 1, 0, 3256, 0, 559, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 771, 10, 1, 0, 2, 4091, 3710, 0, 0, 0, 4092, 4, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 652, 0, 0, 2, 3071, 1, 0, 4095, 4081, 3, 1, 4088, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 17, 4095, 2013, 1, 2, 4093, 2185, 1, 41, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 4, 1, 0, 0, 1, 0, 2, 0, 4095, 4095, 3323, 2305, 2081, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 1, 4093, 7, 0, 1, 1, 1, 2548, 1, 0, 3070, 1789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 4085, 0, 1, 4027, 219, 1, 4, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 2, 4093, 1, 5, 4095, 1, 0, 0, 4095, 8, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16313, 16779, 16539, 16542, 16375, 15609, 16527, 16388] avg=16384.0000 max=16779 max_rank=1 min=15609 min_rank=5 imbalance_ratio=1.024109 max_load_reduction=114293 max_load_reduction_pct=0.871986 moved_to_echo=64899 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 5, 188, 0, 0, 0, 4063, 0, 564, 14, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 767, 6, 0, 0, 0, 4096, 3392, 0, 0, 0, 4090, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 366, 0, 7, 0, 3064, 0, 0, 4096, 4088, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4095, 16, 4096, 2007, 0, 2, 4094, 975, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 3329, 2306, 2089, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4091, 13, 0, 1, 0, 1, 3941, 0, 0, 3336, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 1, 4080, 0, 0, 4007, 16, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4091, 0, 0, 4095, 0, 4, 11, 4094, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17133, 16448, 15839, 15325, 16386, 17259, 16298, 16384] avg=16384.0000 max=17259 max_rank=5 min=15325 min_rank=3 imbalance_ratio=1.053406 max_load_reduction=113813 max_load_reduction_pct=0.868324 moved_to_echo=66327 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 4095, 0, 0, 4096, 4096, 0, 0, 152, 0, 0, 0, 3879, 1, 559, 10, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 772, 6, 0, 0, 1, 4096, 3511, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 344, 272, 0, 4, 0, 3073, 0, 0, 4095, 4087, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2014, 0, 0, 4093, 1453, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3323, 2306, 2082, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4093, 2, 0, 0, 0, 1, 3901, 0, 0, 2856, 1790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4085, 0, 0, 4046, 31, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 1, 4095, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16896, 16579, 15970, 15774, 16378, 16737, 16360, 16378] avg=16384.0000 max=16896 max_rank=0 min=15774 min_rank=3 imbalance_ratio=1.031250 max_load_reduction=114176 max_load_reduction_pct=0.871094 moved_to_echo=65853 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4093, 3, 1, 4094, 4093, 0, 3, 489, 0, 0, 0, 3071, 0, 559, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 771, 3, 1, 0, 0, 4095, 3564, 0, 2, 2, 4095, 0, 2, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 652, 432, 0, 0, 0, 3069, 0, 1, 4095, 4084, 0, 2, 4090, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 1, 0, 4092, 17, 4094, 2013, 1, 2, 4091, 1631, 2, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 2, 2, 2, 0, 1, 0, 0, 1, 0, 2, 0, 4094, 4090, 3323, 2305, 2081, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 4091, 1, 0, 2, 0, 0, 3241, 2, 0, 3192, 1789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4092, 0, 4084, 0, 2, 4065, 92, 1, 1, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 1, 4094, 1, 7, 4092, 0, 4, 0, 4089, 5, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16423, 16632, 16429, 15987, 16370, 16414, 16435, 16382] avg=16384.0000 max=16632 max_rank=1 min=15987 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=114440 max_load_reduction_pct=0.873108 moved_to_echo=65601 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 32768, 128, 32768, 0, 128, 32512, 512, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 32768, 0, 128, 32768, 128, 0, 128, 128, 128, 27008, 128, 128, 24320, 14336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 32768, 32768, 128, 128, 1408, 128, 128, 0, 26752, 128, 4480, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 0, 128, 32768, 128, 128, 32768, 128, 128, 128, 32640, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 2688, 5760, 0, 128, 128, 24576, 128, 128, 32768, 32512, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 6272, 128, 128, 0, 128, 32768, 26624, 0, 128, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 128, 32768, 16128, 128, 128, 32768, 15616, 128, 1024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 0, 0, 128, 0, 128, 0, 32768, 32768, 26624, 18560, 16768, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4093, 4096, 3969, 4096, 1, 0, 0, 0, 4096, 0, 4055, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 3048, 0, 5, 0, 10, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1077, 0, 4090, 0, 0, 0, 0, 4096, 3, 0, 473, 0, 0, 0, 0, 2519, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4064, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2578, 0, 2784, 0, 1, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4092, 4091, 3843, 4096, 6, 0, 0, 0, 4096, 0, 4096, 3, 0, 0, 0, 4090, 0, 4081, 0, 0, 1616, 0, 14, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 245, 12, 4083, 0, 0, 0, 0, 4094, 0, 0, 1315, 0, 0, 0, 0, 3910, 0, 0, 0, 2, 0, 3, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 6, 4010, 4058, 0, 0, 4092, 0, 0, 3986, 0, 4096, 4096, 0, 0, 4091, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 23, 0, 4088, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 2729, 0, 2942, 0, 5, 0, 4096, 3, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 3992, 4096, 10, 0, 0, 0, 4085, 0, 4096, 0, 0, 0, 0, 4075, 0, 4096, 0, 0, 3429, 0, 0, 0, 51, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 125, 14, 4087, 0, 1, 0, 0, 4095, 1, 0, 224, 0, 0, 0, 0, 3884, 0, 0, 0, 0, 0, 0, 4091, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4044, 4096, 0, 0, 4096, 0, 0, 4082, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4091, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 915, 0, 3855, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 3908, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 3042, 0, 0, 0, 24, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 207, 0, 4063, 0, 2, 0, 0, 4096, 0, 0, 556, 0, 0, 0, 0, 3103, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 42, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 4089, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2550, 0, 3102, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4081, 4094, 4001, 4093, 0, 2, 2, 0, 4093, 0, 4093, 0, 3, 0, 1, 4093, 0, 4094, 2, 0, 3332, 0, 28, 2, 0, 4094, 0, 2, 4094, 0, 0, 3, 4093, 4094, 412, 0, 4081, 1, 0, 5, 2, 4092, 1, 3, 454, 2, 0, 0, 4, 3140, 2, 2, 0, 0, 0, 2, 4094, 2, 4095, 6, 1, 3, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 4094, 22, 2, 0, 0, 0, 0, 4093, 4092, 2, 2, 4093, 3, 0, 4059, 3, 4093, 4092, 1, 2, 4093, 3, 1, 0, 4083, 0, 0, 2, 0, 0, 0, 0, 4094, 1, 1, 0, 0, 0, 4091, 7, 2, 2, 0, 0, 1, 2, 1119, 2, 3921, 2, 3, 0, 4094, 0, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 4088, 4096, 2, 0, 0, 0, 4096, 0, 3972, 0, 0, 0, 0, 4093, 0, 4095, 0, 0, 2336, 0, 0, 0, 16, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 247, 0, 4094, 0, 0, 0, 0, 4095, 0, 0, 251, 0, 0, 0, 0, 3789, 0, 0, 0, 0, 0, 0, 4089, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 4096, 0, 0, 4077, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 1, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 2828, 0, 2979, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 3995, 4096, 9, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 4074, 0, 4095, 0, 0, 1837, 0, 0, 0, 11, 4096, 0, 0, 4096, 0, 0, 0, 4095, 4096, 405, 2, 4095, 0, 0, 0, 0, 4096, 5, 0, 414, 0, 0, 0, 0, 3789, 0, 0, 0, 0, 0, 0, 4082, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4093, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2302, 0, 3662, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[4082, 4095, 4082, 4094, 0, 2, 1, 0, 4088, 0, 4093, 0, 1, 0, 0, 4093, 1, 4094, 1, 0, 2320, 0, 24, 1, 0, 4095, 0, 0, 4094, 0, 6, 1, 4094, 4092, 5, 0, 4093, 0, 4, 3, 1, 4095, 0, 2, 503, 1, 0, 0, 2, 4025, 2, 1, 0, 0, 0, 0, 4095, 1, 4093, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 2, 0, 4094, 19, 0, 2, 0, 0, 0, 4095, 4089, 2, 1, 4094, 1, 0, 3392, 2, 4094, 4093, 1, 2, 4095, 1, 0, 0, 4060, 0, 0, 2, 1, 1, 0, 0, 4095, 1, 0, 0, 1, 0, 4095, 0, 2, 2, 1, 0, 0, 2, 2736, 1, 3420, 2, 2, 0, 4091, 0, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 3969, 0, 1, 0, 0, 0, 141, 0, 4055, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3048, 0, 5, 0, 10, 4096, 0, 0, 4096, 0, 0, 0, 859, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3237, 4096, 1077, 0, 4090, 0, 0, 0, 0, 4096, 3, 0, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2519, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4094, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 118, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 3396, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2578, 0, 2784, 0, 1, 0, 4096, 0, 0, 4096, 2638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16351, 16805, 17072, 15387, 16388, 16502, 16374, 16193] avg=16384.0000 max=17072 max_rank=2 min=15387 min_rank=3 imbalance_ratio=1.041992 max_load_reduction=114000 max_load_reduction_pct=0.869751 moved_to_echo=65457 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 3843, 0, 6, 0, 0, 0, 141, 0, 4096, 3, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 1616, 0, 14, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 859, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3237, 4096, 245, 12, 4083, 0, 0, 0, 0, 4094, 0, 0, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3910, 0, 0, 0, 2, 0, 3, 4096, 0, 4075, 0, 0, 0, 0, 0, 4089, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 6, 4010, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 4092, 0, 0, 3986, 0, 0, 118, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 23, 0, 4088, 0, 0, 8, 0, 0, 4096, 0, 3396, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2729, 0, 2942, 0, 5, 0, 4096, 3, 0, 4096, 2638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16271, 15357, 17082, 16757, 16331, 16345, 16418, 16511] avg=16384.0000 max=17082 max_rank=2 min=15357 min_rank=1 imbalance_ratio=1.042603 max_load_reduction=113990 max_load_reduction_pct=0.869675 moved_to_echo=65605 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 3992, 0, 10, 0, 0, 0, 141, 0, 4096, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3429, 0, 0, 0, 51, 4096, 0, 0, 4096, 0, 0, 0, 859, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3237, 4096, 125, 14, 4087, 0, 1, 0, 0, 4095, 1, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 0, 0, 0, 0, 0, 0, 4091, 0, 4093, 0, 0, 0, 0, 0, 4095, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4044, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4082, 0, 0, 118, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 3396, 720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 915, 0, 3855, 0, 0, 0, 4096, 0, 0, 4096, 2631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16410, 17220, 15880, 16745, 16334, 16488, 16402, 15593] avg=16384.0000 max=17220 max_rank=1 min=15593 min_rank=7 imbalance_ratio=1.051025 max_load_reduction=113852 max_load_reduction_pct=0.868622 moved_to_echo=64817 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 3908, 0, 0, 0, 0, 0, 141, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3042, 0, 0, 0, 24, 4096, 0, 0, 4096, 0, 0, 0, 859, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3237, 4096, 207, 0, 4063, 0, 2, 0, 0, 4096, 0, 0, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3103, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 4094, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 42, 0, 0, 0, 0, 0, 4095, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4089, 0, 0, 118, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3396, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2550, 0, 3102, 0, 0, 0, 4096, 0, 0, 4096, 2638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16334, 16808, 16257, 15970, 16427, 16495, 16299, 16482] avg=16384.0000 max=16808 max_rank=1 min=15970 min_rank=3 imbalance_ratio=1.025879 max_load_reduction=114264 max_load_reduction_pct=0.871765 moved_to_echo=65703 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4082, 0, 4082, 0, 0, 2, 1, 0, 141, 0, 4093, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 0, 2320, 0, 24, 1, 0, 4095, 0, 0, 4094, 0, 6, 1, 859, 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3235, 4092, 5, 0, 4093, 0, 4, 3, 1, 4095, 0, 2, 503, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4025, 2, 1, 0, 0, 0, 0, 4095, 1, 4093, 0, 0, 1, 0, 0, 4093, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 2, 0, 4094, 19, 0, 2, 0, 0, 0, 4095, 4094, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 2, 1, 4094, 1, 0, 3392, 2, 0, 118, 1, 2, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 2, 1, 1, 0, 0, 4095, 1, 0, 0, 1, 0, 4095, 0, 3394, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 1, 0, 0, 2, 2736, 1, 3420, 2, 2, 0, 4091, 0, 0, 4094, 2632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16495, 16090, 16034, 16894, 16405, 15798, 16371, 16985] avg=16384.0000 max=16985 max_rank=7 min=15798 min_rank=5 imbalance_ratio=1.036682 max_load_reduction=114087 max_load_reduction_pct=0.870415 moved_to_echo=65559 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 3995, 0, 9, 0, 0, 0, 132, 0, 4096, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1837, 0, 0, 0, 11, 4096, 0, 0, 4096, 0, 0, 0, 863, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3232, 4096, 405, 2, 4095, 0, 0, 0, 0, 4096, 5, 0, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3789, 0, 0, 0, 0, 0, 0, 4082, 0, 4095, 0, 0, 0, 0, 0, 4095, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 4096, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4093, 0, 0, 111, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 1, 0, 0, 4096, 0, 3401, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2302, 0, 3662, 0, 0, 0, 4096, 0, 0, 4096, 2638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16402, 15596, 16345, 16645, 16390, 16490, 16410, 16794] avg=16384.0000 max=16794 max_rank=7 min=15596 min_rank=1 imbalance_ratio=1.025024 max_load_reduction=114278 max_load_reduction_pct=0.871872 moved_to_echo=66084 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4081, 0, 4001, 0, 0, 2, 2, 0, 141, 0, 4093, 0, 3, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 3332, 0, 28, 2, 0, 4094, 0, 2, 4094, 0, 0, 3, 858, 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3235, 4094, 412, 0, 4081, 1, 0, 5, 2, 4092, 1, 3, 454, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3140, 2, 2, 0, 0, 0, 2, 4094, 2, 4095, 6, 1, 3, 0, 0, 4092, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 4094, 22, 2, 0, 0, 0, 0, 4093, 4093, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 2, 4093, 3, 0, 4059, 3, 0, 118, 1, 2, 4093, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 2, 0, 0, 0, 0, 4094, 1, 1, 0, 0, 0, 4091, 7, 3393, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 1, 2, 1119, 2, 3921, 2, 3, 0, 4094, 0, 0, 4093, 2636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16417, 17103, 16382, 16024, 16403, 16472, 16394, 15877] avg=16384.0000 max=17103 max_rank=1 min=15877 min_rank=7 imbalance_ratio=1.043884 max_load_reduction=113969 max_load_reduction_pct=0.869514 moved_to_echo=65146 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4088, 0, 2, 0, 0, 0, 141, 0, 3972, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2336, 0, 0, 0, 16, 4096, 0, 0, 4096, 0, 0, 0, 859, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3237, 4096, 247, 0, 4094, 0, 0, 0, 0, 4095, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3789, 0, 0, 0, 0, 0, 0, 4089, 0, 4095, 0, 0, 0, 0, 0, 4095, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 4093, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4077, 0, 0, 118, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 1, 0, 0, 4091, 1, 3396, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2828, 0, 2979, 0, 0, 0, 4096, 0, 0, 4096, 2638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 16093, 16020, 16650, 16394, 16482, 16404, 16637] avg=16384.0000 max=16650 max_rank=3 min=16020 min_rank=2 imbalance_ratio=1.016235 max_load_reduction=114422 max_load_reduction_pct=0.872971 moved_to_echo=65917 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 32000, 0, 128, 128, 128, 0, 1152, 0, 32640, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 28160, 128, 128, 0, 128, 0, 128, 32768, 128, 32768, 128, 128, 128, 0, 0, 32768, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 32768, 128, 0, 32000, 128, 0, 1024, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 0, 128, 128, 0, 32768, 128, 128, 128, 0, 0, 128, 32640, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32640, 128, 0, 128, 128, 128, 128, 0, 32768, 128, 128, 128, 128, 0, 32768, 128, 27264, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 0, 20992, 0, 128, 128, 128, 32768, 0, 128, 32768, 0, 128, 128, 6912, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[25984, 32768, 2816, 128, 32768, 128, 128, 128, 128, 32768, 128, 128, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 128, 128, 17792, 128, 26752, 128, 128, 0, 32768, 128, 0, 32768, 21120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4094, 0, 0, 457, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 5, 0, 0, 98, 0, 0, 4096, 1, 0, 4096, 4096, 0, 0, 2025, 0, 4096, 4096, 27, 4035, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4044, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1593, 0, 0, 0, 0, 83, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4027] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 5, 4081, 0, 4093, 10, 4070, 0, 0, 439, 0, 4078, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 9, 4095, 0, 0, 4, 0, 0, 70, 19, 0, 4049, 19, 0, 4096, 4090, 0, 0, 2235, 8, 4096, 4096, 24, 3881, 0, 0, 4096, 0, 0, 4096, 4089, 0, 4066, 0, 0, 0, 12, 0, 0, 4096, 4096, 4096, 0, 0, 3531, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 2569, 0, 0, 0, 0, 263, 0, 0, 0, 4096, 0, 4089, 0, 0, 4054, 4096, 3490] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 0, 2, 1, 2, 0, 4092, 0, 4094, 0, 3723, 0, 2, 1485, 2, 4094, 4094, 2, 4094, 2, 0, 0, 0, 1, 0, 1, 0, 1, 0, 2, 0, 0, 0, 0, 3, 2, 2, 4094, 0, 12, 4094, 0, 4094, 0, 8, 12, 2, 0, 6, 0, 1, 4094, 6, 0, 4093, 4091, 2, 2, 1604, 0, 4094, 4095, 10, 4094, 0, 0, 4091, 0, 2, 3973, 4094, 0, 4089, 2, 2, 0, 1, 1, 1, 4087, 4094, 4094, 0, 0, 3532, 3, 0, 0, 4093, 0, 4093, 2, 0, 0, 2, 1, 1, 0, 0, 1, 0, 2, 4091, 1, 0, 0, 1, 2, 2, 1, 2923, 0, 2, 1, 1, 12, 0, 0, 2, 4094, 2, 4094, 1, 0, 4093, 4094, 3183] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 3772, 0, 0, 1561, 0, 4094, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 11, 9, 0, 0, 1, 0, 0, 4092, 14, 0, 4096, 4096, 0, 0, 2976, 2, 4096, 4096, 35, 4063, 0, 0, 4096, 0, 0, 4083, 4095, 0, 4081, 0, 0, 0, 3, 0, 0, 4096, 4096, 4096, 0, 0, 3927, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 838, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 4096, 0, 0, 4078, 4096, 3311] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 4089, 0, 0, 919, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 0, 2, 2, 0, 4090, 1, 0, 4096, 4096, 0, 0, 957, 0, 4096, 4096, 6, 4038, 0, 0, 4096, 0, 0, 4089, 4096, 0, 4090, 0, 0, 0, 2, 0, 0, 4096, 4096, 4096, 0, 0, 4065, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2424, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 4096, 0, 0, 4087, 4096, 3997] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[3, 0, 3, 3, 0, 2, 0, 4093, 0, 4094, 1, 3548, 0, 3, 2133, 3, 4093, 4092, 2, 4093, 3, 0, 0, 1, 0, 1, 0, 0, 0, 0, 3, 0, 3, 0, 0, 1, 3, 1, 4094, 0, 4, 4091, 0, 4093, 0, 5, 12, 1, 0, 5, 1, 2, 4093, 0, 0, 4093, 4080, 3, 2, 480, 0, 4092, 4094, 35, 4094, 0, 3, 4092, 0, 2, 4078, 4093, 3, 4092, 3, 3, 1, 0, 2, 0, 4090, 4093, 4094, 3, 0, 3717, 4, 3, 0, 4092, 0, 4091, 0, 0, 1, 3, 2, 1, 0, 0, 0, 0, 3, 4087, 2, 1, 0, 0, 3, 2, 0, 2908, 0, 3, 2, 2, 82, 0, 0, 2, 4094, 2, 4093, 0, 0, 4093, 4093, 3478] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 4095, 0, 4096, 0, 4046, 0, 0, 662, 0, 4072, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 4088, 29, 0, 4096, 4096, 0, 0, 2454, 0, 4096, 4096, 40, 3557, 0, 0, 4096, 0, 0, 4057, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 3640, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2433, 0, 0, 0, 0, 20, 0, 0, 0, 4096, 0, 4096, 0, 0, 4052, 4096, 3706] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 3948, 0, 0, 1394, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 4096, 0, 0, 10, 0, 0, 1, 0, 0, 4093, 5, 0, 4096, 4096, 0, 0, 1618, 0, 4096, 4096, 37, 4082, 0, 0, 4096, 0, 0, 3984, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 3985, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1616, 0, 0, 0, 0, 29, 0, 0, 0, 4096, 0, 4096, 0, 0, 4090, 4096, 3878] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4094, 0, 0, 457, 0, 1844, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2100, 2002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 5, 0, 2377, 1704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 4096, 1, 0, 4096, 4096, 0, 0, 2025, 0, 2252, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 162, 0, 0, 4096, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 292, 0, 0, 0, 4044, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1593, 4096, 4096, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 416, 0, 4096, 0, 0, 4096, 4096, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15890, 16390, 16374, 16691, 16475, 16624, 15814, 16814] avg=16384.0000 max=16814 max_rank=7 min=15814 min_rank=6 imbalance_ratio=1.026245 max_load_reduction=114258 max_load_reduction_pct=0.871719 moved_to_echo=65727 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 5, 4081, 0, 4093, 10, 4070, 0, 0, 439, 0, 1844, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2100, 2002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 9, 4095, 0, 0, 4, 0, 2377, 1704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 19, 0, 4049, 19, 0, 4096, 4090, 0, 0, 2235, 8, 2252, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 0, 162, 0, 0, 4096, 4089, 0, 4066, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 292, 0, 0, 0, 3531, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 2569, 4096, 4096, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 0, 0, 0, 416, 0, 4089, 0, 0, 4054, 4096, 3490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15845, 16368, 16381, 16862, 16306, 16112, 16790, 16408] avg=16384.0000 max=16862 max_rank=3 min=15845 min_rank=0 imbalance_ratio=1.029175 max_load_reduction=114210 max_load_reduction_pct=0.871353 moved_to_echo=65616 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 2, 1, 2, 0, 4092, 0, 4094, 0, 3723, 0, 2, 1485, 2, 1843, 1302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 2, 4094, 2, 0, 0, 0, 1, 0, 1, 0, 1, 0, 2, 0, 2099, 2000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 2, 4094, 0, 12, 4094, 0, 4094, 0, 8, 12, 2, 2376, 1703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 4094, 6, 0, 4093, 4091, 2, 2, 1604, 0, 2251, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 161, 0, 2, 3973, 4094, 0, 4089, 2, 2, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 292, 0, 0, 0, 3532, 3, 0, 0, 4093, 0, 4093, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 2, 4091, 1, 0, 0, 1, 2, 2, 1, 2923, 4095, 4094, 1930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 12, 0, 0, 2, 416, 2, 4094, 1, 0, 4093, 4094, 3183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16550, 16390, 16402, 16160, 16420, 16104, 17145, 15901] avg=16384.0000 max=17145 max_rank=6 min=15901 min_rank=7 imbalance_ratio=1.046448 max_load_reduction=113927 max_load_reduction_pct=0.869194 moved_to_echo=65570 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 4089, 0, 0, 919, 0, 1844, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2100, 2002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 2377, 1704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 4090, 1, 0, 4096, 4096, 0, 0, 957, 0, 2252, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 162, 0, 0, 4089, 4096, 0, 4090, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 292, 0, 0, 0, 4065, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2424, 4096, 4096, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 416, 0, 4096, 0, 0, 4087, 4096, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16348, 16390, 16370, 15502, 16477, 16645, 16644, 16696] avg=16384.0000 max=16696 max_rank=7 min=15502 min_rank=3 imbalance_ratio=1.019043 max_load_reduction=114376 max_load_reduction_pct=0.872620 moved_to_echo=66462 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 3772, 0, 0, 1561, 0, 1848, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2101, 2006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 11, 9, 0, 2383, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4092, 14, 0, 4096, 4096, 0, 0, 2976, 2, 2248, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 154, 0, 0, 4083, 4095, 0, 4081, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 289, 0, 0, 0, 3927, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 838, 4096, 4096, 1936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 409, 0, 4096, 0, 0, 4078, 4096, 3311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16678, 16393, 16398, 17560, 16479, 16505, 15063, 15996] avg=16384.0000 max=17560 max_rank=3 min=15063 min_rank=6 imbalance_ratio=1.071777 max_load_reduction=113512 max_load_reduction_pct=0.866028 moved_to_echo=64043 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3, 3, 0, 2, 0, 4093, 0, 4094, 1, 3548, 0, 3, 2133, 3, 1842, 1302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4092, 2, 4093, 3, 0, 0, 1, 0, 1, 0, 0, 0, 0, 3, 0, 2098, 2000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 3, 1, 4094, 0, 4, 4091, 0, 4093, 0, 5, 12, 1, 2376, 1702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 2, 4093, 0, 0, 4093, 4080, 3, 2, 480, 0, 2250, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3, 162, 0, 2, 4078, 4093, 3, 4092, 3, 3, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 293, 0, 3, 0, 3717, 4, 3, 0, 4092, 0, 4091, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 3, 4087, 2, 1, 0, 0, 3, 2, 0, 2908, 4094, 4094, 1930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 2, 82, 0, 0, 2, 416, 2, 4093, 0, 0, 4093, 4093, 3478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17030, 16386, 16386, 15044, 16536, 16297, 17127, 16266] avg=16384.0000 max=17127 max_rank=6 min=15044 min_rank=3 imbalance_ratio=1.045349 max_load_reduction=113945 max_load_reduction_pct=0.869331 moved_to_echo=66226 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 4095, 0, 4096, 0, 4046, 0, 0, 662, 0, 1844, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2100, 2002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 12, 0, 2377, 1704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 29, 0, 4096, 4096, 0, 0, 2454, 0, 2252, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3557, 0, 0, 162, 0, 0, 4057, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 292, 0, 0, 0, 3640, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2433, 4096, 4096, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 416, 0, 4096, 0, 0, 4052, 4096, 3706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16049, 16366, 16381, 17055, 15962, 16220, 16653, 16386] avg=16384.0000 max=17055 max_rank=3 min=15962 min_rank=4 imbalance_ratio=1.040955 max_load_reduction=114017 max_load_reduction_pct=0.869881 moved_to_echo=65221 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 3948, 0, 0, 1394, 0, 1844, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2100, 2002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 4096, 0, 0, 10, 0, 2377, 1704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 5, 0, 4096, 4096, 0, 0, 1618, 0, 2252, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 162, 0, 0, 3984, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 292, 0, 0, 0, 3985, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1616, 4096, 4096, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 416, 0, 4096, 0, 0, 4090, 4096, 3878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16682, 16389, 16380, 16198, 16417, 16565, 15836, 16605] avg=16384.0000 max=16682 max_rank=0 min=15836 min_rank=6 imbalance_ratio=1.018188 max_load_reduction=114390 max_load_reduction_pct=0.872726 moved_to_echo=65423 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 32768, 128, 32768, 128, 0, 0, 128, 128, 128, 128, 0, 128, 0, 128, 0, 16896, 16128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 32768, 0, 32768, 128, 31360, 0, 128, 9088, 128, 14848, 10496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 512, 0, 0, 128, 3328, 128, 32768, 128, 0, 32768, 32768, 29184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 2432, 0, 128, 0, 30464, 128, 128, 128, 32768, 0, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[31872, 0, 128, 1408, 0, 128, 32512, 32768, 128, 32768, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 128, 128, 32768, 0, 128, 32768, 128, 32768, 0, 128, 128, 128, 19072, 13696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 32768, 128, 128, 128, 128, 128, 128, 128, 17408, 32768, 32768, 15488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 256, 128, 128, 32768, 128, 0, 32768, 32768, 128, 128, 14464, 128, 18048, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 69, 0, 4080, 0, 4096, 0, 4096, 0, 0, 4096, 23, 4096, 0, 0, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4019, 0, 4096, 4096, 10, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 253, 0, 4067, 0, 4096, 0, 4096, 0, 1, 4096, 20, 4096, 0, 0, 0, 0, 0, 0, 4084, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3839, 0, 4096, 4096, 23, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4092, 0, 4096, 0, 4096, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 443, 0, 4052, 0, 4096, 24, 4096, 0, 10, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4062, 4096, 0, 0, 0, 4085, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4096, 3, 0, 4093, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 31, 0, 41, 0, 0, 3651, 0, 4096, 4096, 11, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 4094, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 238, 0, 4074, 0, 4096, 1, 4096, 0, 1, 4096, 19, 4096, 0, 0, 0, 0, 0, 0, 4050, 4096, 0, 0, 0, 4093, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 17, 0, 3, 0, 0, 3878, 0, 4096, 4096, 11, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 277, 0, 3988, 0, 4096, 31, 4096, 0, 0, 4096, 22, 4096, 0, 0, 0, 0, 0, 0, 4081, 4095, 0, 0, 0, 4094, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 33, 0, 7, 0, 0, 3824, 0, 4096, 4096, 46, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 1, 2, 2, 2, 0, 3, 4091, 3, 0, 4091, 1, 4, 0, 0, 0, 3, 0, 0, 0, 0, 3, 3, 4092, 0, 1, 2, 0, 14, 4093, 1, 2, 0, 2, 4093, 4080, 0, 4093, 2, 4092, 4092, 6, 2, 4093, 0, 0, 0, 3, 3, 4092, 4093, 3, 0, 390, 0, 4094, 2, 4092, 3, 4093, 0, 1, 4089, 0, 4091, 0, 0, 0, 0, 0, 2, 4013, 4093, 0, 3, 0, 4094, 4093, 0, 4093, 0, 3, 3, 3, 0, 3, 0, 0, 1, 3, 0, 3, 4087, 0, 3, 4093, 3, 2, 4093, 0, 2, 2, 4093, 4093, 1, 4, 0, 3, 0, 0, 5, 0, 0, 2, 3777, 0, 4092, 4093, 7, 0, 1, 3, 4090, 4092, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 11, 0, 4087, 0, 4096, 0, 4096, 0, 3, 4096, 19, 4096, 0, 0, 0, 0, 0, 0, 4069, 4096, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4058, 0, 4096, 4096, 39, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 2, 0, 0, 3, 4094, 2, 0, 4093, 2, 2, 0, 10, 0, 2, 0, 0, 0, 0, 2, 2, 4092, 0, 1, 1, 0, 18, 4094, 2, 0, 0, 1, 4094, 4075, 0, 4094, 2, 4093, 4094, 1, 2, 4094, 0, 0, 2, 1, 2, 4093, 4094, 2, 0, 7, 3, 4094, 2, 4094, 0, 4094, 0, 0, 4086, 24, 4092, 0, 0, 0, 0, 0, 1, 4036, 4092, 0, 2, 0, 4094, 4094, 0, 4094, 0, 3, 2, 1, 0, 2, 0, 2, 1, 2, 0, 2, 4088, 0, 0, 4094, 2, 2, 4093, 0, 3, 0, 4092, 4094, 2, 3, 0, 1, 0, 0, 3, 0, 0, 2, 4093, 0, 4094, 4093, 31, 0, 0, 3, 4092, 4094, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4095, 4090, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 69, 0, 4080, 0, 3906, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4096, 0, 0, 0, 0, 0, 0, 4083, 25, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4066, 3920, 190, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 4096, 176, 10, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16382, 16390, 16247, 16419, 16380, 16376, 16493] avg=16384.0000 max=16493 max_rank=7 min=16247 min_rank=3 imbalance_ratio=1.006653 max_load_reduction=114579 max_load_reduction_pct=0.874168 moved_to_echo=65668 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 4, 4096, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 4096, 0, 4094, 4090, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4092, 0, 0, 0, 4096, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 443, 0, 4052, 0, 3906, 24, 0, 0, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4062, 25, 0, 0, 0, 4085, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4096, 3, 0, 4066, 3920, 190, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 3651, 0, 4096, 176, 11, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16370, 16367, 16382, 16627, 16357, 16402, 16400, 16167] avg=16384.0000 max=16627 max_rank=3 min=16167 min_rank=7 imbalance_ratio=1.014832 max_load_reduction=114445 max_load_reduction_pct=0.873146 moved_to_echo=65326 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 4094, 4090, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 277, 0, 3988, 0, 3906, 31, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4096, 0, 0, 0, 0, 0, 0, 4081, 25, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4065, 3920, 190, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3824, 0, 4096, 176, 46, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 16377, 16390, 16394, 16414, 16379, 16405, 16341] avg=16384.0000 max=16414 max_rank=4 min=16341 min_rank=7 imbalance_ratio=1.001831 max_load_reduction=114658 max_load_reduction_pct=0.874771 moved_to_echo=65539 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 4094, 4094, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 4094, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 238, 0, 4074, 0, 3904, 1, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 0, 0, 4050, 22, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 4067, 3921, 192, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3878, 0, 4096, 175, 11, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16387, 16385, 16410, 16376, 16388, 16387, 16355] avg=16384.0000 max=16410 max_rank=3 min=16355 min_rank=7 imbalance_ratio=1.001587 max_load_reduction=114662 max_load_reduction_pct=0.874802 moved_to_echo=65506 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 2, 0, 0, 3, 4094, 2, 0, 4093, 2, 2, 0, 10, 4094, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 2, 4092, 0, 1, 1, 0, 18, 4094, 2, 4093, 4088, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 4075, 0, 0, 2, 4093, 4094, 1, 2, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4093, 0, 2, 0, 7, 3, 4094, 2, 3904, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4092, 0, 0, 0, 0, 0, 1, 4036, 25, 0, 2, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3, 2, 1, 0, 2, 0, 2, 1, 2, 0, 2, 4088, 0, 0, 4062, 3917, 190, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 2, 4093, 0, 3, 0, 4092, 4082, 2, 3, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 4094, 176, 31, 0, 0, 3, 4092, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16399, 16400, 16370, 16194, 16368, 16378, 16377, 16586] avg=16384.0000 max=16586 max_rank=7 min=16194 min_rank=3 imbalance_ratio=1.012329 max_load_reduction=114486 max_load_reduction_pct=0.873459 moved_to_echo=65709 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 1, 2, 2, 2, 0, 3, 4091, 3, 0, 4091, 1, 4, 0, 0, 4093, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 3, 4092, 0, 1, 2, 0, 14, 4093, 1, 4092, 4087, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 6, 4080, 0, 0, 2, 4092, 4092, 6, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 4092, 0, 3, 0, 390, 0, 4094, 2, 3902, 3, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 2, 4013, 25, 0, 3, 0, 4094, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 3, 3, 0, 3, 0, 0, 1, 3, 0, 3, 4087, 0, 3, 4063, 3917, 190, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 2, 4093, 0, 2, 2, 4093, 4081, 1, 4, 0, 3, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3777, 0, 4092, 176, 7, 0, 1, 3, 4090, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16396, 16377, 16582, 16321, 16384, 16382, 16241] avg=16384.0000 max=16582 max_rank=3 min=16241 min_rank=7 imbalance_ratio=1.012085 max_load_reduction=114490 max_load_reduction_pct=0.873489 moved_to_echo=65328 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 2, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 4094, 4090, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4094, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 4087, 0, 3906, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 0, 0, 4069, 25, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4066, 3920, 190, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4058, 0, 4096, 176, 39, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16382, 16388, 16199, 16400, 16381, 16373, 16562] avg=16384.0000 max=16562 max_rank=7 min=16199 min_rank=3 imbalance_ratio=1.010864 max_load_reduction=114510 max_load_reduction_pct=0.873642 moved_to_echo=65716 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4094, 4090, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 253, 0, 4067, 0, 3906, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 0, 0, 0, 0, 0, 0, 4084, 25, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4066, 3920, 190, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3839, 0, 4096, 176, 23, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16381, 16390, 16419, 16417, 16380, 16372, 16327] avg=16384.0000 max=16419 max_rank=3 min=16327 min_rank=7 imbalance_ratio=1.002136 max_load_reduction=114653 max_load_reduction_pct=0.874733 moved_to_echo=65496 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 32768, 0, 128, 128, 32768, 32768, 128, 128, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 31232, 0, 32768, 1408, 256, 0, 128, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 128, 0, 128, 128, 128, 128, 128, 32768, 128, 128, 32640, 31360, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 0, 0, 0, 0, 128, 32512, 256, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 32768, 128, 128, 32768, 128, 128, 0, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 1792, 128, 32640, 128, 31360, 128, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 128, 32768, 32768, 128, 128, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 128, 32768, 0, 128, 128, 0, 128, 32768, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[263, 4096, 4096, 0, 0, 0, 0, 0, 19, 0, 0, 18, 0, 0, 0, 0, 0, 2931, 0, 0, 0, 1485, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3681, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4078, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3907, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[979, 4096, 4078, 0, 0, 0, 0, 0, 447, 0, 0, 48, 0, 0, 0, 0, 0, 2453, 0, 16, 0, 1260, 0, 4096, 0, 0, 10, 0, 0, 4096, 0, 4096, 0, 3680, 4096, 0, 4096, 10, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 33, 0, 0, 4096, 0, 0, 3908, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 2, 4084, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 16, 0, 4093, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4071, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 3572, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[498, 4096, 4096, 0, 0, 0, 0, 0, 79, 0, 0, 28, 0, 0, 0, 0, 0, 1988, 0, 0, 0, 2299, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3344, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 1, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[934, 4096, 4096, 0, 0, 0, 0, 0, 377, 0, 0, 169, 0, 0, 0, 0, 0, 2944, 0, 9, 0, 1028, 0, 4096, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 3816, 4096, 0, 4096, 0, 1, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 20, 0, 0, 4095, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4068, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3055, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[333, 4096, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 47, 0, 0, 0, 0, 0, 2470, 0, 8, 0, 2772, 0, 4096, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 2828, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 3, 0, 0, 4096, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 17, 0, 4096, 0, 2, 4096, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 3945, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[509, 4092, 4093, 4, 4, 2, 2, 4, 24, 3, 0, 0, 0, 0, 0, 0, 0, 1873, 5, 0, 4, 1920, 0, 4093, 0, 3, 0, 0, 3, 4091, 3, 4092, 1, 3975, 4091, 0, 4091, 0, 0, 3, 2, 4092, 0, 4093, 0, 4092, 0, 4093, 4093, 4092, 0, 0, 0, 4090, 3, 0, 4014, 3, 4, 0, 3, 3, 0, 4093, 0, 1, 0, 4093, 3, 0, 2, 3, 1, 4093, 1, 3, 2, 0, 4, 4094, 3, 0, 4088, 1, 0, 0, 4, 0, 0, 4092, 4092, 3, 0, 3, 0, 4088, 4093, 3, 0, 4084, 2, 4093, 0, 0, 0, 0, 0, 0, 4, 6, 0, 0, 4092, 4060, 0, 4, 0, 3, 5, 3, 0, 3, 4092, 0, 0, 0, 4093, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[913, 4093, 4094, 2, 2, 2, 5, 1, 185, 1, 0, 0, 0, 0, 0, 1, 0, 1816, 3, 0, 4, 1993, 0, 4094, 0, 2, 0, 0, 2, 4092, 0, 4093, 2, 4080, 4092, 1, 4093, 0, 0, 3, 0, 4093, 0, 4094, 0, 4094, 0, 4094, 4093, 4094, 1, 0, 0, 4092, 2, 1, 4023, 3, 2, 0, 2, 2, 0, 4094, 0, 1, 0, 4094, 2, 0, 2, 2, 1, 4094, 0, 2, 2, 1, 3, 4094, 3, 0, 4094, 2, 0, 0, 1, 0, 0, 4093, 4094, 2, 0, 2, 0, 4091, 4093, 2, 0, 4078, 1, 4093, 0, 0, 0, 14, 0, 1, 1, 0, 0, 0, 4093, 3366, 0, 3, 1, 2, 1, 2, 0, 2, 4093, 0, 0, 0, 4091, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[496, 4096, 4095, 0, 0, 0, 0, 0, 7, 0, 0, 22, 0, 0, 0, 0, 0, 3370, 0, 1, 0, 1333, 0, 4096, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 3203, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 1, 0, 0, 4096, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4077, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3885, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[263, 4096, 4096, 0, 0, 0, 0, 0, 19, 0, 0, 18, 0, 0, 0, 0, 4096, 3297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2931, 0, 0, 0, 1485, 0, 3944, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3681, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 4096, 0, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 4096, 3378, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4078, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3329, 799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3907, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 197, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15885, 16552, 16490, 16434, 16385, 16384, 16398, 16544] avg=16384.0000 max=16552 max_rank=1 min=15885 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=114520 max_load_reduction_pct=0.873718 moved_to_echo=65711 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[979, 4096, 4078, 0, 0, 0, 0, 0, 447, 0, 0, 48, 0, 0, 0, 0, 4096, 3298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2453, 0, 16, 0, 1260, 0, 3944, 0, 0, 10, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3680, 4096, 0, 4096, 10, 0, 0, 0, 4096, 0, 0, 0, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 4096, 33, 0, 0, 4096, 0, 0, 3908, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4084, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 16, 0, 4093, 3378, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4071, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3329, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3572, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 197, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17042, 15875, 16499, 16284, 16377, 16396, 16391, 16208] avg=16384.0000 max=17042 max_rank=0 min=15875 min_rank=1 imbalance_ratio=1.040161 max_load_reduction=114030 max_load_reduction_pct=0.869980 moved_to_echo=65372 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[934, 4096, 4096, 0, 0, 0, 0, 0, 377, 0, 0, 169, 0, 0, 0, 0, 4096, 3298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2944, 0, 9, 0, 1028, 0, 3944, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3816, 4096, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 4096, 20, 0, 0, 4095, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 4096, 3378, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4068, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3329, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3055, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 197, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17066, 16118, 16626, 16423, 16376, 16384, 16387, 15692] avg=16384.0000 max=17066 max_rank=0 min=15692 min_rank=7 imbalance_ratio=1.041626 max_load_reduction=114006 max_load_reduction_pct=0.869797 moved_to_echo=64839 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[498, 4096, 4096, 0, 0, 0, 0, 0, 79, 0, 0, 28, 0, 0, 0, 0, 4096, 3297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1988, 0, 0, 0, 2299, 0, 3944, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3344, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 4096, 1, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 4096, 3378, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3329, 799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 197, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16190, 16423, 16153, 16435, 16386, 16385, 16415, 16685] avg=16384.0000 max=16685 max_rank=7 min=16153 min_rank=2 imbalance_ratio=1.018372 max_load_reduction=114387 max_load_reduction_pct=0.872704 moved_to_echo=65871 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[913, 4093, 4094, 2, 2, 2, 5, 1, 185, 1, 0, 0, 0, 0, 0, 1, 4094, 3296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1816, 3, 0, 4, 1993, 0, 3942, 0, 2, 0, 0, 2, 4092, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4080, 4092, 1, 4093, 0, 0, 3, 0, 4093, 0, 0, 0, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 4094, 1, 0, 0, 4092, 2, 1, 4023, 3, 2, 0, 2, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 2, 0, 2, 2, 1, 4094, 0, 2, 2, 1, 3, 4094, 3376, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4094, 2, 0, 0, 1, 0, 0, 4093, 4094, 2, 0, 2, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 4078, 1, 4093, 0, 0, 0, 14, 0, 1, 1, 0, 0, 0, 3327, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3366, 0, 3, 1, 2, 1, 2, 0, 2, 4093, 0, 0, 0, 4091, 2, 197, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16689, 15947, 16885, 16374, 16382, 16382, 16408, 16005] avg=16384.0000 max=16885 max_rank=2 min=15947 min_rank=1 imbalance_ratio=1.030579 max_load_reduction=114187 max_load_reduction_pct=0.871178 moved_to_echo=65177 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[333, 4096, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 47, 0, 0, 0, 0, 4096, 3298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2470, 0, 8, 0, 2772, 0, 3938, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2828, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 4096, 3, 0, 0, 4096, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 17, 0, 4096, 3382, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 1, 0, 0, 4096, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3945, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3332, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 199, 158, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15965, 17381, 15631, 16340, 16396, 16382, 16267, 16710] avg=16384.0000 max=17381 max_rank=1 min=15631 min_rank=2 imbalance_ratio=1.060852 max_load_reduction=113691 max_load_reduction_pct=0.867393 moved_to_echo=65755 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[496, 4096, 4095, 0, 0, 0, 0, 0, 7, 0, 0, 22, 0, 0, 0, 0, 4096, 3298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3370, 0, 1, 0, 1333, 0, 3944, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3203, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 4096, 1, 0, 0, 4096, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 0, 4096, 3378, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3329, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3885, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 197, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16110, 16841, 16012, 16419, 16387, 16385, 16396, 16522] avg=16384.0000 max=16841 max_rank=1 min=16012 min_rank=2 imbalance_ratio=1.027893 max_load_reduction=114231 max_load_reduction_pct=0.871513 moved_to_echo=65690 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[509, 4092, 4093, 4, 4, 2, 2, 4, 24, 3, 0, 0, 0, 0, 0, 0, 4093, 3295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1873, 5, 0, 4, 1920, 0, 3941, 0, 3, 0, 0, 3, 4091, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3975, 4091, 0, 4091, 0, 0, 3, 2, 4092, 0, 0, 0, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 4092, 0, 0, 0, 4090, 3, 0, 4014, 3, 4, 0, 3, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 3, 0, 2, 3, 1, 4093, 1, 3, 2, 0, 4, 4094, 3375, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4088, 1, 0, 0, 4, 0, 0, 4092, 4092, 3, 0, 3, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 4084, 2, 4093, 0, 0, 0, 0, 0, 0, 4, 6, 0, 0, 3327, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4060, 0, 4, 0, 3, 5, 3, 0, 3, 4092, 0, 0, 0, 4093, 3, 196, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16125, 15935, 16776, 16363, 16383, 16374, 16410, 16706] avg=16384.0000 max=16776 max_rank=2 min=15935 min_rank=1 imbalance_ratio=1.023926 max_load_reduction=114296 max_load_reduction_pct=0.872009 moved_to_echo=65873 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[4992, 32768, 32768, 128, 128, 128, 128, 128, 1152, 128, 0, 384, 0, 0, 0, 128, 32768, 26496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 19968, 128, 128, 128, 14208, 0, 31616, 0, 128, 128, 0, 128, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 32512, 128, 32768, 0, 0, 0, 128, 0, 128, 128, 128, 0, 0, 26752, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 128, 0, 0, 32768, 32768, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 30080, 128, 128, 128, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 32768, 128, 1664, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[512, 32768, 128, 0, 0, 32768, 128, 128, 32256, 128, 128, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 28672, 32768, 128, 32768, 128, 128, 128, 128, 32768, 0, 0, 0, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 128, 128, 32768, 27136, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1471, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 4096, 4096, 0, 4096, 0, 1639, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4, 4096, 0, 4096, 0, 4096, 927, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 83, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 4066, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 604, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 29, 0, 0, 0, 4096, 4096, 0, 4096, 0, 2815, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 19, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 4096, 0, 72, 4096, 0, 4096, 0, 4096, 616, 0, 4030, 4094, 18, 0, 0, 0, 6, 0, 0, 38, 4096, 6, 0, 222, 4096, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4077, 0, 3878, 0, 7, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 797, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 4096, 4096, 0, 4096, 0, 1499, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 32, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 51, 4096, 0, 4096, 0, 4096, 2034, 0, 4055, 4096, 5, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 162, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4039, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4087, 0, 3702, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1091, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 1508, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 17, 4096, 0, 4096, 0, 4096, 1823, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 3743, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 74, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 2744, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 4096, 0, 1, 4096, 0, 4096, 0, 4096, 1288, 0, 4094, 4096, 6, 0, 0, 0, 0, 0, 0, 8, 4096, 2, 0, 49, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4065, 0, 4056, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 882, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4095, 4096, 0, 4096, 0, 2397, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 2, 4096, 0, 4096, 0, 4096, 803, 0, 4095, 4096, 2, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 18, 4096, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 4088, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[2, 0, 4088, 1532, 2, 4094, 3, 0, 0, 0, 2, 0, 4094, 0, 0, 4080, 1, 0, 2, 0, 2, 2, 4093, 3, 3, 2, 4093, 0, 1, 0, 3, 0, 1, 4042, 4090, 0, 4093, 2, 2146, 0, 2, 1, 4094, 2, 1, 4091, 5, 3, 0, 2, 4090, 0, 1, 2, 1, 0, 0, 0, 4093, 4094, 0, 3, 4091, 2, 0, 4093, 1, 4089, 0, 4093, 469, 0, 4093, 4094, 0, 4, 2, 1, 0, 1, 0, 13, 4093, 0, 0, 0, 4093, 4094, 0, 1, 4092, 2, 0, 2, 1, 2, 0, 1, 2, 4094, 4094, 0, 1, 2, 2, 0, 2, 2, 0, 0, 0, 4094, 0, 0, 4092, 1, 1, 3, 7, 4093, 3, 4094, 2, 4094, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[2, 0, 4092, 1178, 3, 4093, 4, 0, 0, 0, 4, 0, 4092, 0, 0, 4082, 0, 1, 3, 0, 3, 2, 4091, 5, 4, 3, 4090, 0, 1, 0, 4, 0, 1, 4051, 4086, 0, 4092, 3, 2537, 0, 2, 1, 4093, 1, 4, 4090, 6, 5, 0, 1, 4049, 0, 2, 3, 0, 0, 1, 0, 4092, 4092, 0, 3, 4088, 5, 0, 4092, 3, 4090, 0, 4092, 455, 0, 4091, 4093, 0, 5, 3, 6, 0, 0, 0, 7, 4084, 0, 0, 2, 4092, 4092, 0, 3, 4091, 2, 0, 3, 0, 3, 6, 2, 3, 4093, 4093, 0, 1, 2, 0, 0, 3, 4, 0, 0, 0, 4094, 0, 0, 4092, 4, 2, 4, 9, 4092, 4, 4094, 3, 4093, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 604, 0, 3147, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 29, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2815, 0, 0, 0, 4090, 0, 0, 1948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 3037, 0, 4096, 0, 0, 616, 0, 4030, 4094, 18, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4096, 6, 0, 222, 4020, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 2148, 1059, 891, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4077, 0, 3878, 0, 7, 0, 0, 76, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16034, 16406, 17045, 16396, 15969, 16571, 16363, 16288] avg=16384.0000 max=17045 max_rank=2 min=15969 min_rank=4 imbalance_ratio=1.040344 max_load_reduction=114027 max_load_reduction_pct=0.869957 moved_to_echo=65191 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 797, 0, 3147, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1499, 0, 0, 0, 4096, 0, 0, 1948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 3037, 0, 4096, 0, 0, 2034, 0, 4055, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 162, 4020, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4039, 2148, 1059, 891, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4087, 0, 3702, 0, 0, 0, 0, 76, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16232, 16384, 15735, 16415, 17374, 16480, 16337, 16115] avg=16384.0000 max=17374 max_rank=4 min=15735 min_rank=2 imbalance_ratio=1.060425 max_load_reduction=113698 max_load_reduction_pct=0.867447 moved_to_echo=66306 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 1471, 0, 3147, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1639, 0, 0, 0, 4096, 0, 0, 1948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3037, 0, 4096, 0, 0, 927, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 83, 4020, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2148, 1059, 891, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 4066, 0, 0, 0, 0, 76, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16906, 16380, 15875, 16384, 16256, 16391, 16393, 16487] avg=16384.0000 max=16906 max_rank=0 min=15875 min_rank=2 imbalance_ratio=1.031860 max_load_reduction=114166 max_load_reduction_pct=0.871017 moved_to_echo=65527 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 1091, 0, 3147, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 1508, 0, 0, 0, 4096, 0, 0, 1948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3037, 0, 4096, 0, 0, 1823, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 4020, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2148, 1059, 891, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 3743, 0, 0, 0, 0, 76, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16526, 16377, 15743, 16390, 17161, 16322, 16388, 16165] avg=16384.0000 max=17161 max_rank=4 min=15743 min_rank=2 imbalance_ratio=1.047424 max_load_reduction=113911 max_load_reduction_pct=0.869072 moved_to_echo=66036 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 74, 0, 3142, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2744, 0, 0, 0, 4096, 0, 0, 1941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3031, 0, 4096, 0, 0, 1288, 0, 4094, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 2, 0, 49, 4012, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2155, 1065, 894, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4065, 0, 4056, 0, 0, 0, 0, 80, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15504, 16378, 16973, 16387, 16612, 16359, 16406, 16453] avg=16384.0000 max=16973 max_rank=2 min=15504 min_rank=0 imbalance_ratio=1.035950 max_load_reduction=114099 max_load_reduction_pct=0.870506 moved_to_echo=65830 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 882, 0, 3147, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 2397, 0, 0, 0, 4096, 0, 0, 1948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3037, 0, 4096, 0, 0, 803, 0, 4095, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 18, 4020, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2148, 1059, 891, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 4088, 0, 1, 0, 0, 76, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16317, 16379, 16632, 16385, 16131, 16329, 16389, 16510] avg=16384.0000 max=16632 max_rank=2 min=16131 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=114440 max_load_reduction_pct=0.873108 moved_to_echo=65359 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4092, 1178, 3, 3146, 4, 0, 0, 0, 4, 0, 4092, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 3, 2, 4091, 5, 4, 3, 4090, 0, 1, 0, 4, 0, 4092, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4051, 4086, 0, 0, 3, 2537, 0, 2, 1, 4093, 1, 4, 1945, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4049, 0, 2, 3, 0, 0, 1, 0, 4092, 4092, 0, 3, 4088, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3034, 3, 4090, 0, 0, 455, 0, 4091, 4093, 0, 5, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4084, 0, 0, 2, 4016, 4092, 0, 3, 4091, 2, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 3, 4093, 4093, 0, 1, 2, 0, 0, 3, 4, 0, 0, 0, 4094, 2145, 1058, 890, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 2, 4, 9, 4092, 4, 4094, 3, 4093, 0, 0, 0, 0, 76, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16603, 16384, 16735, 16336, 15780, 16303, 16401, 16530] avg=16384.0000 max=16735 max_rank=2 min=15780 min_rank=4 imbalance_ratio=1.021423 max_load_reduction=114337 max_load_reduction_pct=0.872322 moved_to_echo=65014 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4088, 1532, 2, 3147, 3, 0, 0, 0, 2, 0, 4094, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 2, 2, 4093, 3, 3, 2, 4093, 0, 1, 0, 3, 0, 4093, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4042, 4090, 0, 0, 2, 2146, 0, 2, 1, 4094, 2, 1, 1945, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4090, 0, 1, 2, 1, 0, 0, 0, 4093, 4094, 0, 3, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3035, 1, 4089, 0, 0, 469, 0, 4093, 4094, 0, 4, 2, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4093, 0, 0, 0, 4017, 4094, 0, 1, 4092, 2, 0, 2, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4094, 4094, 0, 1, 2, 2, 0, 2, 2, 0, 0, 0, 4094, 2146, 1058, 890, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1, 3, 7, 4093, 3, 4094, 2, 4094, 0, 0, 1, 0, 76, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16950, 16384, 16334, 16379, 15789, 16317, 16395, 16524] avg=16384.0000 max=16950 max_rank=0 min=15789 min_rank=4 imbalance_ratio=1.034546 max_load_reduction=114122 max_load_reduction_pct=0.870682 moved_to_echo=65025 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 128, 128, 128, 32768, 0, 128, 128, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 32768, 7680, 128, 25216, 128, 0, 0, 0, 128, 0, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 32768, 32768, 128, 128, 128, 128, 0, 128, 128, 0, 0, 128, 32768, 17280, 8576, 7168, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 32768, 0, 0, 128, 17408, 0, 128, 128, 32768, 128, 128, 15616, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 24320, 128, 32768, 0, 0, 8448, 0, 32768, 32768, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 32768, 128, 128, 128, 128, 32768, 128, 32768, 128, 31744, 0, 128, 128, 0, 640, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 128, 0, 128, 0, 32768, 32768, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 32768, 128, 0, 640, 32256, 32768, 0, 128, 32768, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3026, 0, 4096, 0, 0, 0, 4096, 361, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4076, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 2950, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 124, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 76, 109, 4096, 1571, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 4096, 0, 0, 5, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2627, 0, 4096, 0, 0, 0, 4096, 337, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4077, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 3874, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 10, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 7, 38, 4096, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3473, 0, 4096, 0, 0, 0, 4015, 1, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4009, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 3604, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4, 169, 4096, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4087, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3464, 0, 4096, 0, 0, 0, 4093, 84, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 3928, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 45, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 9, 91, 4096, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3739, 0, 4096, 0, 0, 0, 4087, 137, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4080, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 3853, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 58, 4096, 427, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 517, 0, 4096, 0, 0, 0, 4096, 1882, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4093, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 3453, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 162, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 13, 14, 4096, 2155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4, 4, 0, 0, 0, 3, 0, 4090, 4, 4, 2, 3, 4, 3, 969, 5, 4092, 0, 0, 0, 4088, 125, 4092, 0, 2, 2, 4092, 4, 4089, 0, 4090, 0, 4092, 5, 0, 4093, 0, 0, 4092, 0, 4, 4092, 8, 0, 4092, 2, 0, 0, 4, 4092, 4092, 1, 3, 0, 4089, 0, 2, 0, 0, 0, 3, 4092, 0, 4, 4, 4088, 3, 4092, 3, 4, 3454, 0, 53, 1, 0, 0, 3, 4, 4089, 4091, 0, 0, 4092, 4091, 3, 4091, 3, 3, 4092, 4, 4092, 0, 9, 0, 4084, 3584, 3, 4, 3, 2, 0, 6, 0, 0, 4, 0, 4, 0, 4089, 1, 0, 2, 2, 0, 0, 0, 4, 0, 4092, 4086, 2, 4091, 0, 3, 0, 0, 4, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1, 3, 0, 0, 0, 2, 0, 4092, 2, 1, 0, 2, 2, 2, 2358, 4, 4093, 1, 1, 0, 4087, 125, 4093, 0, 1, 1, 4094, 3, 4092, 0, 4093, 0, 4093, 3, 0, 4094, 1, 0, 4093, 0, 1, 4093, 3, 1, 4094, 2, 0, 0, 3, 4093, 4094, 0, 2, 0, 4091, 0, 1, 2, 0, 0, 2, 4093, 1, 3, 2, 4090, 2, 4092, 2, 4, 3222, 1, 28, 1, 1, 0, 2, 3, 4091, 4093, 81, 0, 4093, 4093, 2, 4092, 0, 2, 4094, 2, 4093, 0, 1, 0, 4087, 2387, 2, 3, 3, 0, 0, 4, 0, 0, 3, 0, 2, 0, 4091, 0, 0, 1, 1, 0, 0, 0, 2, 1, 4093, 4081, 3, 4093, 0, 4, 0, 0, 8, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3026, 0, 4096, 4078, 1586, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3730, 0, 0, 0, 4096, 361, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 4076, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 544, 0, 0, 2950, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 2330, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 76, 109, 4096, 1571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 4096, 0, 0, 5, 0, 0, 0, 1966, 1766, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16884, 16379, 16383, 16381, 15780, 16498, 16384, 16383] avg=16384.0000 max=16884 max_rank=0 min=15780 min_rank=4 imbalance_ratio=1.030518 max_load_reduction=114188 max_load_reduction_pct=0.871185 moved_to_echo=65045 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3473, 0, 4096, 4078, 1586, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3730, 0, 0, 0, 4015, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 4009, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 544, 0, 0, 3604, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2330, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 169, 4096, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4087, 0, 4096, 0, 0, 0, 0, 0, 0, 1966, 1766, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17332, 15938, 16315, 16386, 16436, 15906, 16381, 16378] avg=16384.0000 max=17332 max_rank=0 min=15906 min_rank=5 imbalance_ratio=1.057861 max_load_reduction=113740 max_load_reduction_pct=0.867767 moved_to_echo=65101 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3464, 0, 4096, 4078, 1586, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3730, 0, 0, 0, 4093, 84, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 4094, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 544, 0, 0, 3928, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 2330, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 9, 91, 4096, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 1966, 1766, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17322, 16099, 16400, 16384, 16760, 15339, 16382, 16386] avg=16384.0000 max=17322 max_rank=0 min=15339 min_rank=5 imbalance_ratio=1.057251 max_load_reduction=113750 max_load_reduction_pct=0.867844 moved_to_echo=64867 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2627, 0, 4096, 4078, 1586, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3730, 0, 0, 0, 4096, 337, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 4077, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 544, 0, 0, 3874, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 2330, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 38, 4096, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 1966, 1766, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16485, 16355, 16383, 16386, 16706, 15991, 16382, 16384] avg=16384.0000 max=16706 max_rank=4 min=15991 min_rank=5 imbalance_ratio=1.019653 max_load_reduction=114366 max_load_reduction_pct=0.872543 moved_to_echo=65463 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3739, 0, 4096, 4079, 1587, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3727, 0, 0, 0, 4087, 137, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 4080, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 537, 0, 0, 3853, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2326, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 58, 4096, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 1972, 1770, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17600, 16143, 16385, 16385, 16678, 15101, 16382, 16398] avg=16384.0000 max=17600 max_rank=0 min=15101 min_rank=5 imbalance_ratio=1.074219 max_load_reduction=113472 max_load_reduction_pct=0.865723 moved_to_echo=64559 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 517, 0, 4096, 4078, 1586, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3730, 0, 0, 0, 4096, 1882, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 4093, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 544, 0, 0, 3453, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 2330, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 13, 14, 4096, 2155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 1966, 1766, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14377, 17900, 16399, 16383, 16285, 16962, 16381, 16385] avg=16384.0000 max=17900 max_rank=1 min=14377 min_rank=0 imbalance_ratio=1.092529 max_load_reduction=113172 max_load_reduction_pct=0.863434 moved_to_echo=66013 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 3, 0, 0, 0, 2, 0, 4092, 2, 1, 0, 2, 2, 2, 2358, 4, 4093, 4076, 1584, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3727, 1, 1, 0, 4087, 125, 0, 0, 1, 1, 0, 3, 4092, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 18, 1, 0, 4093, 0, 1, 4093, 3, 1, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 4094, 0, 2, 0, 4091, 0, 1, 2, 0, 0, 2, 4093, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4090, 2, 544, 2, 4, 3222, 1, 28, 1, 1, 0, 2, 3, 4091, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 2328, 4093, 2, 4092, 0, 2, 0, 2, 0, 0, 1, 0, 4087, 2387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 3, 0, 0, 4, 0, 0, 3, 0, 2, 0, 4091, 0, 0, 1, 4094, 4094, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 1, 4093, 4081, 3, 4093, 0, 4, 0, 0, 8, 0, 1964, 1765, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16224, 16131, 16402, 16385, 16086, 17075, 16388, 16381] avg=16384.0000 max=17075 max_rank=5 min=16086 min_rank=4 imbalance_ratio=1.042175 max_load_reduction=113997 max_load_reduction_pct=0.869728 moved_to_echo=65930 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4, 4, 0, 0, 0, 3, 0, 4090, 4, 4, 2, 3, 4, 3, 969, 5, 4092, 4075, 1584, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3727, 0, 0, 0, 4088, 125, 0, 0, 2, 2, 0, 4, 4089, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 5, 0, 18, 0, 0, 4092, 0, 4, 4092, 8, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 4092, 1, 3, 0, 4089, 0, 2, 0, 0, 0, 3, 4092, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4088, 3, 544, 3, 4, 3454, 0, 53, 1, 0, 0, 3, 4, 4089, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2328, 4091, 3, 4091, 3, 3, 0, 4, 0, 0, 9, 0, 4084, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 3, 2, 0, 6, 0, 0, 4, 0, 4, 0, 4089, 1, 0, 2, 4092, 4092, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 4092, 4086, 2, 4091, 0, 3, 0, 0, 4, 0, 1964, 1764, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14848, 16127, 16405, 16382, 16341, 18200, 16392, 16377] avg=16384.0000 max=18200 max_rank=5 min=14848 min_rank=0 imbalance_ratio=1.110840 max_load_reduction=112872 max_load_reduction_pct=0.861145 moved_to_echo=67310 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[29952, 128, 128, 0, 32768, 3072, 0, 0, 128, 128, 0, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 4352, 128, 128, 28416, 128, 128, 128, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 32768, 128, 128, 0, 32768, 0, 128, 128, 0, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 256, 128, 0, 32640, 0, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 0, 128, 0, 128, 0, 32768, 128, 0, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 0, 0, 0, 128, 128, 32768, 128, 128, 128, 128, 128, 128, 20224, 128, 32768, 32640, 12800, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[512, 0, 18688, 32768, 128, 32768, 128, 128, 0, 128, 0, 0, 128, 512, 32768, 13184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 0, 128, 128, 32768, 32768, 128, 32768, 0, 128, 128, 0, 128, 0, 15744, 14208, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 180, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3912, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 7, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 250, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4075, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3867, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 4096, 4094, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 384, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4081, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3727, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 133, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4080, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3979, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 4096, 4094, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 4096, 660, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 4096, 4090, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3438, 0, 0, 8, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 4096, 4092, 4096, 0, 4096, 4089, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 31, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4065, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 2, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4091, 0, 4, 0, 0, 1, 4, 0, 0, 3, 0, 0, 0, 4092, 4092, 4, 4092, 5, 4, 2, 4, 0, 2, 4093, 3, 4, 0, 4, 4092, 1, 3977, 4092, 1, 4089, 5, 3, 0, 4, 4, 49, 4, 0, 0, 0, 0, 0, 0, 6, 0, 2, 32, 0, 4062, 0, 0, 4090, 4092, 0, 4091, 1, 0, 0, 0, 0, 0, 2, 3, 4092, 3, 0, 0, 2, 4089, 4088, 4, 4, 22, 4091, 0, 0, 0, 0, 4, 4, 0, 0, 5, 3, 4, 0, 4092, 0, 0, 5, 0, 4, 4092, 4, 4092, 4087, 0, 4081, 4093, 4072, 4, 4088, 4086, 0, 0, 0, 0, 2, 0, 0, 4092, 9, 0, 4, 4091, 4092, 4092, 4090, 0, 4092, 4, 79, 4, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4093, 0, 3, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 4093, 4093, 3, 4094, 4, 2, 1, 2, 0, 2, 4094, 2, 3, 0, 3, 4093, 1, 4030, 4093, 3, 4085, 4, 2, 0, 3, 3, 23, 3, 0, 1, 0, 0, 0, 0, 4, 0, 2, 18, 0, 4070, 0, 0, 4092, 4093, 1, 4091, 3, 0, 0, 0, 0, 0, 1, 2, 4093, 1, 1, 0, 2, 4092, 4089, 2, 3, 18, 4092, 0, 0, 0, 0, 2, 2, 0, 0, 3, 3, 2, 0, 4093, 0, 0, 3, 0, 3, 4093, 3, 4093, 4091, 0, 4092, 4094, 4087, 3, 4090, 4086, 0, 0, 0, 0, 0, 1, 0, 4094, 5, 0, 2, 4093, 4093, 4093, 4092, 0, 4093, 2, 49, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4085, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4085, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3912, 0, 0, 0, 4096, 0, 0, 178, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4080, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16383, 16375, 16387, 16391, 16382, 16395, 16375] avg=16384.0000 max=16395 max_rank=6 min=16375 min_rank=2 imbalance_ratio=1.000671 max_load_reduction=114677 max_load_reduction_pct=0.874916 moved_to_echo=65543 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4085, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4075, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4084, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3867, 0, 0, 0, 4096, 0, 0, 178, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 11, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16435, 16375, 16387, 16346, 16382, 16394, 16368] avg=16384.0000 max=16435 max_rank=1 min=16346 min_rank=4 imbalance_ratio=1.003113 max_load_reduction=114637 max_load_reduction_pct=0.874611 moved_to_echo=65490 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 4085, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4081, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 4096, 4090, 0, 4096, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3438, 0, 0, 8, 4096, 0, 8, 178, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 11, 4089, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16864, 16363, 16388, 15933, 16379, 16391, 16368] avg=16384.0000 max=16864 max_rank=1 min=15933 min_rank=4 imbalance_ratio=1.029297 max_load_reduction=114208 max_load_reduction_pct=0.871338 moved_to_echo=65071 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4085, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4081, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4085, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3727, 0, 0, 0, 4096, 0, 0, 178, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16575, 16376, 16387, 16206, 16381, 16395, 16368] avg=16384.0000 max=16575 max_rank=1 min=16206 min_rank=4 imbalance_ratio=1.011658 max_load_reduction=114497 max_load_reduction_pct=0.873543 moved_to_echo=65350 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 3, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 4093, 4093, 3, 4079, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3916, 4, 2, 1, 2, 0, 2, 0, 2, 3, 0, 3, 4093, 1, 4030, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4085, 4, 2, 0, 3, 3, 23, 3, 0, 1, 0, 0, 0, 0, 4, 4093, 4093, 4083, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 18, 0, 4070, 0, 0, 4092, 4093, 1, 4091, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4093, 1, 1, 0, 2, 4092, 4089, 2, 3, 18, 4092, 0, 0, 178, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 3, 3, 2, 0, 4093, 0, 0, 3, 0, 3, 4094, 4094, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4091, 0, 4092, 0, 4087, 3, 11, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 0, 2, 4093, 4093, 4077, 4092, 0, 0, 2, 49, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16152, 16402, 16373, 16587, 16390, 16373, 16416] avg=16384.0000 max=16587 max_rank=4 min=16152 min_rank=1 imbalance_ratio=1.012390 max_load_reduction=114485 max_load_reduction_pct=0.873451 moved_to_echo=65766 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4090, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3916, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4084, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3979, 0, 0, 0, 4096, 0, 0, 180, 20, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4070, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16321, 16375, 16390, 16473, 16381, 16384, 16358] avg=16384.0000 max=16473 max_rank=4 min=16321 min_rank=1 imbalance_ratio=1.005432 max_load_reduction=114599 max_load_reduction_pct=0.874321 moved_to_echo=65596 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4085, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4084, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4065, 0, 0, 0, 4096, 0, 1, 178, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 10, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4080, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16237, 16373, 16387, 16546, 16381, 16393, 16370] avg=16384.0000 max=16546 max_rank=4 min=16237 min_rank=1 imbalance_ratio=1.009888 max_load_reduction=114526 max_load_reduction_pct=0.873764 moved_to_echo=65690 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 4, 0, 0, 1, 4, 0, 0, 3, 0, 0, 0, 4092, 4092, 4, 4077, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3915, 5, 4, 2, 4, 0, 2, 0, 3, 4, 0, 4, 4092, 1, 3977, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 5, 3, 0, 4, 4, 49, 4, 0, 0, 0, 0, 0, 0, 6, 4092, 4092, 4082, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 32, 0, 4062, 0, 0, 4090, 4092, 0, 4091, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4092, 3, 0, 0, 2, 4089, 4088, 4, 4, 22, 4091, 0, 0, 177, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 5, 3, 4, 0, 4092, 0, 0, 5, 0, 4, 4093, 4092, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4087, 0, 4081, 0, 4072, 4, 11, 4086, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4, 4091, 4092, 4076, 4090, 0, 0, 4, 79, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16105, 16433, 16373, 16590, 16396, 16347, 16449] avg=16384.0000 max=16590 max_rank=4 min=16105 min_rank=1 imbalance_ratio=1.012573 max_load_reduction=114482 max_load_reduction_pct=0.873428 moved_to_echo=65782 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 0, 128, 0, 0, 128, 128, 0, 0, 128, 0, 128, 0, 32768, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 128, 32768, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[31360, 1664, 128, 128, 128, 0, 128, 0, 128, 128, 0, 128, 32768, 128, 32640, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 0, 128, 128, 128, 0, 32768, 0, 0, 128, 0, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 32768, 32768, 128, 32768, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 0, 128, 32768, 31232, 128, 128, 128, 32768, 0, 128, 1536, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 128, 0, 32768, 0, 32768, 0, 32768, 128, 128, 32768, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 128, 32768, 32768, 32640, 32768, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 113, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 3983, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 36, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 4031, 0, 0, 0, 0, 4090, 0, 0, 0, 4, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 4096, 4060, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4077, 0, 60, 0, 4092, 0, 0, 8, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 17, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 4079, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4095, 0, 24, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 13, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 4083, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4095, 0, 48, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 36, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4054, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 25, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4094, 4096, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 80, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4016, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 6, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 2, 4092, 4091, 5, 0, 5, 4083, 4091, 0, 0, 0, 0, 4, 4, 0, 0, 0, 4091, 4, 4, 0, 4092, 4091, 0, 4, 3, 0, 870, 4091, 4088, 0, 0, 0, 1, 5, 2, 4092, 4092, 4093, 4092, 0, 0, 4, 1, 0, 3, 5, 4, 0, 4, 2, 0, 1, 2, 4092, 18, 4, 0, 0, 3981, 3, 4, 0, 0, 4091, 3, 0, 0, 0, 4092, 0, 3, 2, 4089, 5, 4089, 4091, 4088, 3, 3, 0, 3, 0, 5, 4091, 0, 4090, 4090, 3497, 4091, 0, 4, 0, 0, 3, 4077, 4090, 4, 5, 0, 15, 0, 0, 4, 2, 0, 2, 0, 4093, 0, 22, 5, 4092, 0, 2, 0, 0, 2, 6, 4, 4092, 0, 3885, 5, 3, 5, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 4093, 4092, 2, 1, 4, 4087, 4093, 0, 1, 0, 0, 3, 3, 0, 0, 0, 4093, 3, 3, 0, 4093, 4092, 0, 3, 2, 0, 601, 4091, 4092, 0, 0, 0, 1, 2, 2, 4093, 4093, 4094, 4093, 0, 0, 3, 3, 0, 2, 3, 3, 0, 3, 0, 0, 1, 2, 4093, 8, 3, 0, 0, 4062, 2, 2, 0, 0, 4093, 2, 0, 0, 1, 4093, 0, 2, 3, 4092, 4, 4091, 4093, 4082, 2, 2, 0, 3, 0, 3, 4093, 0, 4090, 4088, 3730, 4093, 1, 2, 0, 0, 0, 4090, 4092, 3, 3, 0, 7, 0, 0, 2, 1, 0, 2, 0, 4094, 0, 25, 2, 4093, 0, 2, 0, 0, 7, 4, 3, 4093, 0, 3856, 3, 2, 2, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 36, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 4031, 0, 0, 0, 4096, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4096, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 4096, 4060, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4077, 0, 60, 3847, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 8, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4058, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16201, 16377, 16355, 16390, 16504, 16419, 16438] avg=16384.0000 max=16504 max_rank=5 min=16201 min_rank=1 imbalance_ratio=1.007324 max_load_reduction=114568 max_load_reduction_pct=0.874084 moved_to_echo=65751 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 113, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 3983, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 3847, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4058, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16278, 16384, 16414, 16389, 16432, 16357, 16434] avg=16384.0000 max=16434 max_rank=7 min=16278 min_rank=1 imbalance_ratio=1.003052 max_load_reduction=114638 max_load_reduction_pct=0.874619 moved_to_echo=65612 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 17, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 4079, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4095, 0, 24, 3847, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4058, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16182, 16384, 16402, 16373, 16527, 16386, 16434] avg=16384.0000 max=16527 max_rank=5 min=16182 min_rank=1 imbalance_ratio=1.008728 max_load_reduction=114545 max_load_reduction_pct=0.873909 moved_to_echo=65720 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3875, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 13, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4048, 0, 0, 0, 4096, 4096, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 4083, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4095, 0, 48, 3853, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4061, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 16176, 16380, 16372, 16380, 16523, 16411, 16438] avg=16384.0000 max=16523 max_rank=5 min=16176 min_rank=1 imbalance_ratio=1.008484 max_load_reduction=114549 max_load_reduction_pct=0.873940 moved_to_echo=65752 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 36, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4077, 0, 0, 0, 4096, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4054, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 25, 3847, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4058, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16201, 16384, 16395, 16390, 16503, 16381, 16434] avg=16384.0000 max=16503 max_rank=5 min=16201 min_rank=1 imbalance_ratio=1.007263 max_load_reduction=114569 max_load_reduction_pct=0.874092 moved_to_echo=65708 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 4096, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 80, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4016, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 6, 3847, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4061, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16245, 16384, 16410, 16389, 16465, 16360, 16436] avg=16384.0000 max=16465 max_rank=5 min=16245 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=114607 max_load_reduction_pct=0.874382 moved_to_echo=65650 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4093, 4092, 2, 1, 4, 4087, 4093, 0, 1, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3874, 3, 3, 0, 0, 4092, 0, 3, 2, 0, 601, 4091, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 4093, 4093, 4094, 4093, 0, 0, 3, 3, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 1, 2, 4093, 8, 3, 0, 0, 4062, 2, 2, 0, 4093, 4093, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 1, 0, 0, 2, 3, 4092, 4, 4091, 8, 4082, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 3, 4093, 0, 4090, 4088, 3730, 160, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4092, 3, 3, 0, 7, 0, 0, 2, 1, 0, 2, 0, 4094, 0, 25, 3845, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 2, 0, 0, 7, 4, 3, 4093, 0, 3856, 3, 2, 2, 2, 4055, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16761, 16389, 16395, 16380, 16172, 16383, 16212] avg=16384.0000 max=16761 max_rank=1 min=16172 min_rank=5 imbalance_ratio=1.023010 max_load_reduction=114311 max_load_reduction_pct=0.872124 moved_to_echo=65147 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4092, 4091, 5, 0, 5, 4083, 4091, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3873, 4, 4, 0, 0, 4091, 0, 4, 3, 0, 870, 4091, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 2, 4092, 4092, 4093, 4092, 0, 0, 4, 1, 0, 3, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 2, 0, 1, 2, 4092, 18, 4, 0, 0, 3981, 3, 4, 0, 4092, 4092, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 3, 2, 4089, 5, 4089, 8, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 5, 4091, 0, 4090, 4090, 3497, 160, 0, 4, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4090, 4, 5, 0, 15, 0, 0, 4, 2, 0, 2, 0, 4093, 0, 22, 3843, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4092, 0, 2, 0, 0, 2, 6, 4, 4092, 0, 3885, 5, 3, 5, 4, 4053, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 17028, 16390, 16329, 16381, 15946, 16375, 16246] avg=16384.0000 max=17028 max_rank=1 min=15946 min_rank=5 imbalance_ratio=1.039307 max_load_reduction=114044 max_load_reduction_pct=0.870087 moved_to_echo=64948 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 128, 0, 128, 0, 0, 128, 128, 128, 128, 0, 32768, 0, 256, 30848, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 31104, 128, 128, 0, 0, 32768, 0, 128, 128, 0, 1792, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 32768, 32768, 128, 128, 128, 32768, 32768, 0, 128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 128, 0, 0, 128, 0, 0, 128, 128, 32768, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 128, 32768, 0, 32768, 32768, 31616, 1280, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 128, 128, 128, 32768, 0, 32384, 128, 128, 128, 128, 32512, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 32768, 128, 128, 0, 0, 32512, 128, 128, 0, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 32768, 32768, 32768, 32768, 0, 0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 14, 0, 4096, 0, 4096, 0, 0, 4094, 0, 4096, 0, 4096, 0, 0, 4096, 0, 11, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 117, 4096, 4096, 0, 0, 0, 0, 4095, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3973, 0, 13, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 4014, 111, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4071, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 4096, 4096, 0, 524, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3520, 234, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3961, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2, 0, 4096, 0, 0, 0, 0, 4090, 0, 4096, 4096, 0, 238, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3589, 495, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3990, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 4096, 0, 4096, 4096, 0, 148, 4096, 4096, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 3, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 3399, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3498, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 15, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 13, 0, 4096, 0, 0, 0, 0, 3962, 0, 4096, 4096, 0, 271, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3008, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3734, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4, 4091, 3, 0, 4, 5, 2, 3, 4091, 4091, 2, 18, 0, 3, 5, 0, 0, 4089, 3999, 5, 4092, 4, 4090, 4090, 4, 0, 4, 0, 4090, 0, 8, 23, 4, 98, 0, 5, 4088, 2, 0, 0, 63, 0, 6, 4092, 4, 0, 0, 49, 4092, 4, 4075, 4, 1, 4086, 4, 4090, 0, 4092, 4, 4, 4091, 0, 172, 0, 4091, 0, 0, 4, 3, 3965, 0, 4092, 4086, 0, 728, 4083, 4088, 8, 5, 4, 2, 4068, 4091, 4, 126, 2, 0, 5, 1, 3, 0, 0, 4091, 1, 0, 1, 1, 1, 4, 0, 4090, 3, 0, 3, 3, 3593, 375, 4, 1, 0, 4, 0, 0, 4, 0, 4092, 3241, 4044, 0, 2, 7, 1, 6, 4091, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 751, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3743, 232, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3472, 4096, 0, 2, 0, 0, 0, 4096, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[3, 4092, 2, 2, 7, 2, 1, 2, 4093, 4093, 2, 13, 0, 3, 3, 0, 0, 4091, 4028, 5, 4093, 4, 4092, 4092, 3, 0, 2, 0, 4092, 1, 0, 4, 3, 74, 0, 3, 4091, 1, 0, 0, 46, 0, 4, 4093, 3, 0, 0, 27, 4093, 3, 4081, 3, 1, 4089, 3, 4092, 0, 4093, 3, 3, 4093, 0, 113, 0, 4092, 0, 0, 2, 2, 3982, 0, 4093, 4089, 1, 490, 4084, 4091, 4, 4, 2, 1, 4085, 4092, 3, 175, 1, 0, 4, 0, 2, 0, 1, 4092, 2, 5, 2, 1, 1, 1, 0, 4092, 0, 0, 2, 2, 3835, 209, 2, 1, 0, 3, 0, 0, 3, 0, 4093, 3415, 4070, 0, 1, 5, 0, 1, 4093, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4040, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 14, 0, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 4062, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 117, 4096, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3973, 0, 13, 0, 4047, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 4014, 111, 0, 0, 0, 0, 0, 4096, 4024, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4071, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 357, 39, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16405, 16332, 16359, 16144, 16297, 16375, 16785] avg=16384.0000 max=16785 max_rank=7 min=16144 min_rank=4 imbalance_ratio=1.024475 max_load_reduction=114287 max_load_reduction_pct=0.871941 moved_to_echo=65601 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4040, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 524, 4096, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4048, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3520, 234, 0, 0, 0, 0, 0, 4096, 4025, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3961, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 357, 39, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16376, 16401, 16328, 16350, 16542, 16404, 15991, 16680] avg=16384.0000 max=16680 max_rank=7 min=15991 min_rank=6 imbalance_ratio=1.018066 max_load_reduction=114392 max_load_reduction_pct=0.872742 moved_to_echo=65617 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4040, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 15, 0, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4062, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 0, 0, 4096, 0, 271, 4096, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 0, 0, 4047, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3008, 1612, 0, 0, 0, 0, 0, 4096, 4024, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3734, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 357, 39, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16401, 16311, 16363, 16164, 16148, 16856, 16452] avg=16384.0000 max=16856 max_rank=6 min=16148 min_rank=5 imbalance_ratio=1.028809 max_load_reduction=114216 max_load_reduction_pct=0.871399 moved_to_echo=65620 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4040, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4062, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 238, 4096, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 1, 0, 4047, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3589, 495, 0, 0, 0, 0, 0, 4096, 4024, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3990, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 357, 39, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16401, 16328, 16352, 16259, 16327, 16320, 16708] avg=16384.0000 max=16708 max_rank=7 min=16259 min_rank=4 imbalance_ratio=1.019775 max_load_reduction=114364 max_load_reduction_pct=0.872528 moved_to_echo=65614 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4047, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 1, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 148, 4096, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 3, 0, 4047, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 3399, 1185, 0, 0, 0, 0, 0, 4096, 4024, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3498, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 361, 43, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16391, 16331, 16345, 16173, 16401, 16821, 16226] avg=16384.0000 max=16821 max_rank=6 min=16173 min_rank=4 imbalance_ratio=1.026672 max_load_reduction=114251 max_load_reduction_pct=0.871666 moved_to_echo=65621 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 4092, 2, 2, 7, 2, 1, 2, 4093, 4093, 2, 13, 0, 3, 3, 0, 4036, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4028, 5, 0, 4, 4092, 17, 3, 0, 2, 0, 4092, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 74, 0, 3, 4091, 1, 0, 0, 46, 0, 4, 4093, 3, 0, 0, 27, 4093, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4081, 3, 1, 4089, 3, 4092, 0, 0, 3, 3, 4059, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 3982, 0, 0, 4089, 1, 490, 4084, 3734, 4, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 4092, 3, 175, 1, 0, 4, 0, 2, 0, 1, 4092, 2, 5, 2, 4044, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 4092, 0, 0, 2, 2, 3835, 209, 2, 1, 0, 3, 0, 4093, 4022, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 3415, 4070, 0, 1, 5, 0, 1, 4093, 1, 0, 0, 0, 357, 39, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16402, 16339, 16511, 16450, 16394, 16580, 16284, 16112] avg=16384.0000 max=16580 max_rank=5 min=16112 min_rank=7 imbalance_ratio=1.011963 max_load_reduction=114492 max_load_reduction_pct=0.873505 moved_to_echo=65370 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4040, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 1, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4062, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 751, 4096, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 2, 0, 4047, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3743, 232, 0, 0, 0, 0, 0, 4096, 4024, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3472, 4096, 0, 2, 0, 0, 0, 4096, 0, 0, 1, 0, 357, 39, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16401, 16356, 16352, 16778, 16406, 16210, 16192] avg=16384.0000 max=16778 max_rank=4 min=16192 min_rank=7 imbalance_ratio=1.024048 max_load_reduction=114294 max_load_reduction_pct=0.871994 moved_to_echo=65586 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4, 4091, 3, 0, 4, 5, 2, 3, 4091, 4091, 2, 18, 0, 3, 5, 0, 4034, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3999, 5, 0, 4, 4090, 17, 4, 0, 4, 0, 4090, 0, 8, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 98, 0, 5, 4088, 2, 0, 0, 63, 0, 6, 4092, 4, 0, 0, 49, 4092, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4075, 4, 1, 4086, 4, 4090, 0, 0, 4, 4, 4057, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 3965, 0, 0, 4086, 0, 728, 4083, 3732, 8, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4068, 4091, 4, 126, 2, 0, 5, 1, 3, 0, 0, 4091, 1, 0, 1, 4043, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4, 0, 4090, 3, 0, 3, 3, 3593, 375, 4, 1, 0, 4, 0, 4092, 4021, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4092, 3241, 4044, 0, 2, 7, 1, 6, 4091, 0, 0, 0, 0, 356, 39, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16404, 16333, 16575, 16501, 16618, 16509, 16215, 15917] avg=16384.0000 max=16618 max_rank=4 min=15917 min_rank=7 imbalance_ratio=1.014282 max_load_reduction=114454 max_load_reduction_pct=0.873215 moved_to_echo=65259 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 128, 32768, 32768, 128, 128, 0, 128, 128, 0, 32384, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 32768, 128, 384, 128, 0, 128, 128, 128, 0, 128, 32384, 128, 128, 128, 32384, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 128, 128, 32768, 128, 32768, 0, 0, 128, 128, 32512, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 32640, 128, 0, 128, 32768, 128, 128, 0, 128, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 128, 128, 28800, 4480, 128, 128, 0, 128, 0, 32768, 32256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 128, 32384, 0, 0, 32768, 128, 3328, 32768, 29952, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 256, 0, 128, 32768, 128, 0, 0, 128, 0, 128, 32640, 128, 0, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 32768, 29440, 32768, 0, 128, 128, 128, 128, 32768, 128, 0, 128, 0, 2944, 384, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 61, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 1500, 0, 0, 0, 0, 0, 0, 2218, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 10, 4096, 0, 343, 0, 4095, 4, 0, 0, 4082, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 1, 0, 4096, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1889, 0, 0, 0, 0, 0, 0, 1595, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 4096, 0, 0, 0, 4096, 98, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 31, 4096, 0, 627, 0, 4083, 64, 0, 0, 4023, 4096, 4053, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 7, 0, 0, 4096, 4096, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 1214, 0, 0, 0, 0, 0, 0, 2145, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 22, 0, 4096, 4078, 0, 1, 4083, 0, 0, 0, 4096, 40, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 39, 4096, 0, 19, 0, 0, 15, 4096, 0, 759, 0, 4090, 62, 0, 0, 4031, 4096, 4045, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 1, 0, 0, 4096, 4095, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 404, 0, 0, 0, 0, 0, 0, 3392, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 22, 0, 4096, 4095, 0, 0, 4094, 0, 0, 0, 4096, 1, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 2, 0, 0, 6, 4096, 0, 314, 0, 4096, 1, 0, 0, 4085, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 1, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 2382, 0, 0, 0, 0, 0, 0, 1462, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 9, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 5, 4096, 0, 1, 0, 0, 1, 4096, 0, 277, 0, 4092, 17, 0, 0, 4073, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 2, 3, 0, 4096, 4094, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2362, 0, 0, 0, 0, 0, 0, 1270, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 4092, 0, 0, 0, 4096, 48, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4, 4096, 0, 3, 0, 0, 6, 4096, 0, 510, 0, 4079, 26, 0, 0, 4063, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1, 4091, 2, 4, 29, 3780, 0, 0, 8, 2, 3, 0, 11, 4005, 5, 0, 7, 4, 4086, 6, 0, 4090, 4089, 0, 0, 0, 4092, 4063, 0, 0, 4091, 4090, 4, 5, 4, 0, 231, 5, 4, 4072, 0, 1791, 4, 0, 4, 0, 108, 0, 1394, 0, 6, 4, 4, 5, 2, 4085, 3, 0, 4090, 3, 4, 5, 0, 0, 4, 4091, 4092, 4, 0, 4091, 4, 0, 0, 4063, 2, 4092, 6, 4085, 0, 4092, 4090, 2, 0, 4091, 0, 4089, 4, 0, 0, 4092, 4, 4, 6, 0, 4092, 0, 0, 0, 0, 73, 4088, 20, 913, 7, 4091, 0, 0, 0, 4092, 4090, 4072, 0, 0, 0, 0, 4, 0, 3, 4091, 4, 6, 4, 3, 11, 4, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4092, 1, 3, 18, 3833, 0, 0, 0, 2, 2, 0, 1, 4015, 3, 0, 3, 3, 4090, 3, 0, 4092, 4091, 0, 0, 0, 4093, 4074, 0, 0, 4092, 4092, 2, 4, 3, 1, 158, 4, 3, 4083, 0, 2869, 3, 0, 3, 0, 107, 1, 911, 0, 4, 3, 3, 3, 3, 4091, 2, 0, 4092, 2, 3, 3, 0, 0, 3, 4092, 4093, 3, 0, 4093, 3, 0, 0, 4068, 0, 4093, 4, 4087, 0, 4093, 4092, 2, 0, 4092, 0, 4091, 3, 0, 0, 4092, 3, 3, 4, 0, 4093, 0, 0, 0, 2, 53, 4089, 20, 345, 5, 4093, 1, 0, 0, 4093, 4092, 4093, 0, 0, 0, 0, 2, 1, 0, 4093, 2, 4, 3, 2, 5, 3, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 3683, 512, 10, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1889, 0, 0, 0, 0, 0, 0, 4096, 4096, 2219, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1595, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1877, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 4096, 98, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3584, 0, 627, 0, 4083, 64, 0, 0, 4023, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16417, 16383, 16399, 16173, 16465, 16390, 16465, 16380] avg=16384.0000 max=16465 max_rank=4 min=16173 min_rank=3 imbalance_ratio=1.004944 max_load_reduction=114607 max_load_reduction_pct=0.874382 moved_to_echo=65700 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3682, 512, 10, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 61, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 1500, 0, 0, 0, 0, 0, 0, 4096, 4096, 2218, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2218, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 0, 0, 4, 4096, 1878, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3584, 0, 343, 0, 4095, 4, 0, 0, 4082, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16494, 16444, 16011, 16792, 16374, 16377, 16200, 16380] avg=16384.0000 max=16792 max_rank=3 min=16011 min_rank=2 imbalance_ratio=1.024902 max_load_reduction=114280 max_load_reduction_pct=0.871887 moved_to_echo=65331 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 3683, 512, 10, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 7, 0, 0, 0, 4096, 0, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 1214, 0, 0, 0, 0, 0, 0, 4096, 4096, 2219, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2145, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 22, 4096, 1877, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 1, 4083, 0, 0, 0, 4096, 40, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 15, 3584, 0, 759, 0, 4090, 62, 0, 0, 4031, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16446, 16392, 15704, 16745, 16384, 16416, 16605, 16380] avg=16384.0000 max=16745 max_rank=3 min=15704 min_rank=2 imbalance_ratio=1.022034 max_load_reduction=114327 max_load_reduction_pct=0.872246 moved_to_echo=65785 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 3682, 512, 10, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 2382, 0, 0, 0, 0, 0, 0, 4096, 4096, 2219, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1462, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1877, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 9, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 3584, 0, 277, 0, 4092, 17, 0, 0, 4073, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16468, 16384, 16894, 16041, 16383, 16382, 16140, 16380] avg=16384.0000 max=16894 max_rank=2 min=16041 min_rank=3 imbalance_ratio=1.031128 max_load_reduction=114178 max_load_reduction_pct=0.871109 moved_to_echo=65285 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 1, 3, 18, 3833, 0, 0, 0, 2, 2, 0, 1, 4015, 3, 0, 3678, 511, 10, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 4090, 3, 0, 0, 4091, 0, 0, 0, 0, 4074, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 3, 1, 158, 4, 3, 4083, 0, 2869, 3, 0, 3, 0, 107, 1, 4093, 4092, 2216, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 911, 0, 4, 3, 3, 3, 3, 4091, 2, 0, 4092, 2, 3, 3, 0, 0, 4092, 1876, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 3, 0, 4093, 3, 0, 0, 4068, 0, 0, 4, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 4092, 0, 4091, 3, 0, 0, 4085, 3, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 53, 3578, 20, 345, 5, 4093, 1, 0, 0, 4093, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 4093, 2, 4, 3, 2, 5, 3, 0, 0, 0, 4093, 4093, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16177, 16355, 17645, 15502, 16344, 16375, 16283, 16391] avg=16384.0000 max=17645 max_rank=2 min=15502 min_rank=3 imbalance_ratio=1.076965 max_load_reduction=113427 max_load_reduction_pct=0.865379 moved_to_echo=65393 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 3683, 515, 17, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 3, 0, 0, 4094, 0, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2362, 0, 0, 0, 0, 0, 0, 4096, 4096, 2219, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1270, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1877, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4092, 0, 0, 0, 4096, 48, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4087, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 6, 3581, 0, 510, 0, 4079, 26, 0, 0, 4063, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16450, 16386, 16870, 15847, 16409, 16379, 16352, 16379] avg=16384.0000 max=16870 max_rank=2 min=15847 min_rank=3 imbalance_ratio=1.029663 max_load_reduction=114202 max_load_reduction_pct=0.871292 moved_to_echo=65519 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 3683, 512, 10, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 4095, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 404, 0, 0, 0, 0, 0, 0, 4096, 4096, 2219, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3392, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 22, 4096, 1877, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 0, 0, 0, 4096, 1, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 3584, 0, 314, 0, 4096, 1, 0, 0, 4085, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16473, 16384, 14915, 17989, 16372, 16378, 16182, 16379] avg=16384.0000 max=17989 max_rank=3 min=14915 min_rank=2 imbalance_ratio=1.097961 max_load_reduction=113083 max_load_reduction_pct=0.862755 moved_to_echo=65311 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 4091, 2, 4, 29, 3780, 0, 0, 8, 2, 3, 0, 11, 4005, 5, 0, 3677, 511, 10, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 4086, 6, 0, 0, 4089, 0, 0, 0, 0, 4063, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 4, 0, 231, 5, 4, 4072, 0, 1791, 4, 0, 4, 0, 108, 0, 4092, 4091, 2216, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1394, 0, 6, 4, 4, 5, 2, 4085, 3, 0, 4090, 3, 4, 5, 0, 0, 4090, 1875, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4092, 4, 0, 4091, 4, 0, 0, 4063, 2, 0, 6, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 4091, 0, 4089, 4, 0, 0, 4085, 4, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 3577, 20, 913, 7, 4091, 0, 0, 0, 4092, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 4091, 4, 6, 4, 3, 11, 4, 0, 0, 0, 4092, 4092, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16147, 16344, 16634, 15983, 16341, 16375, 16845, 16403] avg=16384.0000 max=16845 max_rank=6 min=15983 min_rank=3 imbalance_ratio=1.028137 max_load_reduction=114227 max_load_reduction_pct=0.871483 moved_to_echo=65964 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 32768, 128, 128, 0, 0, 32768, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 0, 32768, 0, 32768, 128, 0, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 32768, 128, 128, 32768, 128, 0, 0, 32768, 256, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 32768, 128, 128, 128, 32128, 0, 0, 128, 128, 128, 0, 128, 32512, 128, 0, 29568, 4224, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[14464, 0, 128, 128, 128, 128, 128, 32768, 128, 0, 32768, 128, 128, 128, 0, 128, 32768, 15104, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 0, 128, 128, 128, 32768, 128, 128, 128, 128, 128, 128, 0, 0, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 128, 256, 28672, 128, 4096, 128, 32768, 256, 0, 0, 32640, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 512, 128, 128, 32768, 0, 14464, 128, 0, 128, 0, 256, 128, 32768, 32768, 17792, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 2, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 3902, 0, 0, 0, 0, 0, 0, 4096, 2578, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1707, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 3, 0, 6, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 3456, 0, 0, 0, 0, 0, 0, 4096, 2213, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 4095, 4096, 0, 0, 0, 1, 0, 0, 2515, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 79, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 47, 0, 19, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 4096, 27, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4077, 4096, 0, 4037, 0, 0, 0, 0, 0, 0, 4096, 3788, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 6, 0, 0, 215, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 63, 0, 20, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4066, 0, 0, 0, 0, 0, 0, 4096, 3373, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 669, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 20, 0, 15, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 3705, 0, 0, 0, 0, 0, 0, 4096, 2234, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2218, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 11, 0, 5, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 3978, 0, 0, 0, 0, 0, 0, 4096, 3037, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1161, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4037, 5, 5, 0, 4091, 0, 4091, 0, 0, 6, 0, 4090, 5, 0, 1, 0, 4090, 2, 4086, 0, 7, 0, 4, 5, 8, 4091, 4091, 6, 4087, 0, 0, 0, 0, 6, 3, 5, 0, 0, 4, 4, 29, 4091, 4091, 3, 3, 3, 4086, 0, 0, 0, 4, 4089, 6, 0, 4092, 0, 0, 0, 4091, 3, 0, 4, 0, 4081, 5, 0, 4, 5, 4091, 4087, 0, 10, 0, 4090, 4, 3, 4092, 4092, 4, 4091, 0, 3, 4, 0, 11, 4, 4089, 2373, 8, 1, 1, 0, 4092, 0, 3, 4090, 0, 1, 0, 4019, 4079, 2, 0, 2, 6, 8, 3, 1818, 4089, 3, 0, 5, 4, 0, 5, 0, 0, 4088, 0, 0, 0, 0, 4086, 4091, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 4044, 3, 3, 0, 4092, 0, 4092, 0, 0, 4, 0, 4092, 3, 0, 0, 0, 4092, 2, 4089, 0, 5, 0, 3, 4, 6, 4093, 4093, 4, 4090, 0, 0, 0, 0, 4, 2, 3, 0, 0, 3, 3, 28, 4092, 4092, 1, 2, 2, 4089, 0, 0, 0, 3, 4091, 4, 0, 4093, 0, 0, 0, 4093, 2, 0, 3, 0, 4084, 4, 0, 2, 3, 4092, 4091, 0, 5, 0, 4092, 3, 1, 4092, 4093, 3, 4062, 0, 3, 2, 0, 5, 3, 4091, 3533, 14, 1, 0, 0, 4093, 0, 2, 4090, 0, 3, 2, 4028, 4080, 2, 0, 1, 7, 5, 2, 677, 4092, 2, 0, 3, 3, 0, 4, 0, 1, 4091, 0, 0, 0, 0, 4089, 4092, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4081, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 0, 0, 0, 0, 0, 4096, 2578, 0, 0, 0, 0, 4096, 0, 0, 1354, 15, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1707, 2742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16371, 16378, 16376, 16381, 16388, 16054, 16737, 16387] avg=16384.0000 max=16737 max_rank=6 min=16054 min_rank=5 imbalance_ratio=1.021545 max_load_reduction=114335 max_load_reduction_pct=0.872307 moved_to_echo=65566 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 79, 4096, 0, 4081, 0, 0, 0, 0, 4096, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 4096, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 27, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 4096, 3788, 0, 0, 0, 0, 4092, 0, 0, 1354, 15, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 6, 0, 0, 215, 2742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16495, 16392, 16374, 16410, 16369, 17395, 15251, 16386] avg=16384.0000 max=17395 max_rank=5 min=15251 min_rank=6 imbalance_ratio=1.061707 max_load_reduction=113677 max_load_reduction_pct=0.867287 moved_to_echo=65401 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4081, 0, 0, 0, 0, 4096, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 4096, 3373, 0, 0, 0, 0, 4096, 0, 0, 1354, 15, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 669, 2742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16432, 16396, 16376, 16382, 16388, 17013, 15699, 16386] avg=16384.0000 max=17013 max_rank=5 min=15699 min_rank=6 imbalance_ratio=1.038391 max_load_reduction=114059 max_load_reduction_pct=0.870201 moved_to_echo=65486 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4081, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4087, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3456, 0, 0, 0, 0, 0, 0, 4096, 2213, 0, 0, 0, 0, 4094, 0, 0, 1354, 15, 8, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 0, 0, 0, 1, 0, 0, 2515, 2742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 16382, 16377, 16382, 16383, 15245, 17545, 16386] avg=16384.0000 max=17545 max_rank=6 min=15245 min_rank=5 imbalance_ratio=1.070862 max_load_reduction=113527 max_load_reduction_pct=0.866142 moved_to_echo=65559 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4081, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 0, 0, 0, 0, 0, 0, 4096, 3037, 0, 0, 0, 0, 4096, 0, 0, 1354, 15, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1161, 2742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16381, 16376, 16381, 16388, 16589, 16191, 16386] avg=16384.0000 max=16589 max_rank=5 min=16191 min_rank=6 imbalance_ratio=1.012512 max_load_reduction=114483 max_load_reduction_pct=0.873436 moved_to_echo=65554 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4080, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4089, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 1, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3705, 0, 0, 0, 0, 0, 0, 4096, 2234, 0, 0, 0, 0, 4096, 0, 0, 1359, 16, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2218, 2737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16385, 16379, 16382, 16383, 15526, 17243, 16386] avg=16384.0000 max=17243 max_rank=6 min=15526 min_rank=5 imbalance_ratio=1.052429 max_load_reduction=113829 max_load_reduction_pct=0.868446 moved_to_echo=65538 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4044, 3, 3, 0, 4092, 0, 4077, 0, 0, 4, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 4089, 0, 5, 0, 3, 4, 6, 0, 4085, 4, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 3, 0, 0, 3, 3, 28, 4092, 4092, 1, 2, 2, 4089, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4091, 4, 0, 4093, 0, 0, 0, 4093, 2, 0, 3, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4, 0, 2, 3, 4, 4091, 0, 5, 0, 4092, 3, 1, 4092, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 3, 2, 0, 5, 3, 4091, 3533, 14, 1, 0, 0, 4093, 0, 2, 1353, 15, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3, 2, 4028, 4080, 2, 0, 1, 7, 5, 2, 677, 2739, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 4, 0, 1, 4091, 0, 0, 0, 0, 4089, 4092, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16318, 16380, 16404, 16378, 16384, 17190, 15638, 16380] avg=16384.0000 max=17190 max_rank=5 min=15638 min_rank=6 imbalance_ratio=1.049194 max_load_reduction=113882 max_load_reduction_pct=0.868851 moved_to_echo=65592 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4037, 5, 5, 0, 4091, 0, 4076, 0, 0, 6, 0, 4090, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 4086, 0, 7, 0, 4, 5, 8, 0, 4083, 6, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 5, 0, 0, 4, 4, 29, 4091, 4091, 3, 3, 3, 4086, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4089, 6, 0, 4092, 0, 0, 0, 4091, 3, 0, 4, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 5, 0, 4, 5, 4, 4087, 0, 10, 0, 4090, 4, 3, 4092, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3, 4, 0, 11, 4, 4089, 2373, 8, 1, 1, 0, 4092, 0, 3, 1352, 15, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 1, 0, 4019, 4079, 2, 0, 2, 6, 8, 3, 1818, 2737, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 5, 0, 0, 4088, 0, 0, 0, 0, 4086, 4091, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16316, 16378, 16410, 16376, 16389, 16060, 16768, 16375] avg=16384.0000 max=16768 max_rank=6 min=16060 min_rank=5 imbalance_ratio=1.023438 max_load_reduction=114304 max_load_reduction_pct=0.872070 moved_to_echo=65592 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 0, 32768, 128, 0, 128, 32768, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 32768, 0, 128, 0, 128, 128, 128, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 128, 128, 128, 32768, 0, 32640, 0, 0, 128, 0, 32768, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 0, 128, 128, 128, 32768, 0, 128, 0, 32768, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 0, 128, 128, 32640, 32768, 128, 0, 128, 128, 128, 128, 11008, 22016, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 128, 128, 0, 0, 128, 128, 128, 32768, 32768, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[31360, 0, 128, 128, 0, 128, 128, 32768, 23168, 128, 128, 128, 0, 32768, 0, 128, 10880, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 32768, 32768, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 24, 0, 0, 4096, 0, 0, 0, 0, 4032, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4094, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3998, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 115, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4088, 4096, 4096, 4096, 0, 0, 127, 0, 0, 4096, 0, 0, 0, 0, 3953, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 4077, 0, 0, 21, 0, 0, 0, 4095, 4096, 6, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4094, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3972, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 6, 0, 0, 4096, 0, 0, 9, 0, 4074, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 4096, 4096, 0, 4096, 0, 0, 15, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4090, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 6, 4083, 5, 4061, 6, 0, 0, 0, 5, 4, 6, 11, 4, 4091, 5, 0, 5, 259, 1, 0, 4086, 0, 0, 0, 0, 4, 4, 5, 4088, 4058, 3, 4, 7, 0, 4025, 4089, 4076, 4091, 6, 0, 0, 9, 0, 4090, 6, 0, 23, 0, 3914, 4, 5, 0, 4065, 2, 0, 4089, 4084, 4, 0, 4090, 4090, 5, 0, 4087, 0, 0, 4083, 0, 5, 0, 6, 4, 0, 4, 4, 5, 37, 2, 18, 6, 10, 551, 4091, 4089, 2, 3958, 127, 8, 0, 0, 0, 0, 4054, 4089, 20, 0, 2, 0, 4086, 0, 0, 7, 0, 1, 4090, 0, 0, 0, 4, 11, 55, 3767, 4, 0, 0, 0, 4, 4091, 4073, 6, 5, 4085, 4, 3849, 0, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 52, 0, 0, 4096, 0, 0, 0, 0, 4071, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 4096, 0, 0, 11, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 54, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 41, 0, 0, 4096, 0, 0, 0, 0, 4061, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 4096, 0, 4096, 0, 0, 16, 0, 0, 0, 4096, 4096, 4, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4008, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 98, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 4096, 4096, 4096, 0, 0, 80, 0, 0, 4096, 0, 0, 0, 0, 3991, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 4096, 0, 4096, 0, 0, 4, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 5, 4088, 4, 4035, 4, 0, 0, 0, 3, 3, 1, 6, 3, 4093, 3, 0, 4, 87, 1, 0, 4089, 2, 1, 0, 0, 4, 3, 4, 4091, 4073, 2, 3, 5, 0, 4082, 4091, 4079, 4093, 4, 0, 0, 5, 0, 4091, 4, 0, 20, 0, 3999, 3, 3, 0, 4069, 2, 0, 4091, 4080, 3, 0, 4092, 4092, 5, 0, 4089, 0, 0, 4087, 0, 4, 0, 3, 3, 0, 3, 3, 3, 31, 0, 9, 3, 8, 376, 4092, 4092, 2, 3980, 90, 8, 1, 0, 0, 0, 4067, 4091, 17, 0, 1, 0, 4085, 0, 0, 0, 0, 1, 4092, 2, 0, 0, 3, 7, 29, 3913, 3, 0, 0, 0, 3, 4093, 4082, 4, 4, 4088, 3, 3981, 0, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3998, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4003, 139, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3841, 147, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 4096, 0, 0, 0, 127, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3953, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3949, 0, 4077, 0, 0, 21, 0, 0, 0, 4095, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4073, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4094, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16358, 16426, 16468, 16333, 16373, 16245, 16384, 16485] avg=16384.0000 max=16485 max_rank=7 min=16245 min_rank=5 imbalance_ratio=1.006165 max_load_reduction=114587 max_load_reduction_pct=0.874229 moved_to_echo=65487 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4003, 139, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3841, 147, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 24, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 3949, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4074, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4094, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16316, 16373, 16413, 16372, 16319, 16374, 16498] avg=16384.0000 max=16498 max_rank=7 min=16316 min_rank=1 imbalance_ratio=1.006958 max_load_reduction=114574 max_load_reduction_pct=0.874130 moved_to_echo=65563 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4003, 139, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3841, 147, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 52, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3949, 0, 4096, 0, 0, 11, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4074, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16400, 16321, 16401, 16452, 16373, 16253, 16373, 16499] avg=16384.0000 max=16499 max_rank=7 min=16253 min_rank=5 imbalance_ratio=1.007019 max_load_reduction=114573 max_load_reduction_pct=0.874123 moved_to_echo=65498 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3972, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4003, 139, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3841, 147, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 6, 0, 0, 4061, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 3949, 0, 4096, 0, 0, 15, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4073, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4090, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16330, 16311, 16364, 16455, 16373, 16374, 16372, 16493] avg=16384.0000 max=16493 max_rank=7 min=16311 min_rank=1 imbalance_ratio=1.006653 max_load_reduction=114579 max_load_reduction_pct=0.874168 moved_to_echo=65612 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4008, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4003, 139, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3841, 147, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 80, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3949, 0, 4096, 0, 0, 4, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4073, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16366, 16409, 16428, 16372, 16373, 16250, 16375, 16499] avg=16384.0000 max=16499 max_rank=7 min=16250 min_rank=5 imbalance_ratio=1.007019 max_load_reduction=114573 max_load_reduction_pct=0.874123 moved_to_echo=65497 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4009, 142, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3846, 154, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 41, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 3942, 0, 4096, 0, 0, 16, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4073, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16356, 16378, 16389, 16436, 16373, 16276, 16374, 16490] avg=16384.0000 max=16490 max_rank=7 min=16276 min_rank=5 imbalance_ratio=1.006470 max_load_reduction=114582 max_load_reduction_pct=0.874191 moved_to_echo=65513 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4088, 4, 4035, 4, 0, 0, 0, 3, 3, 1, 6, 3, 4093, 3, 4000, 139, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 87, 1, 0, 4089, 2, 1, 0, 0, 4, 3, 4, 4091, 4073, 2, 3837, 147, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 0, 4082, 4091, 4079, 0, 4, 0, 0, 5, 0, 4056, 4, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 3, 3, 0, 4069, 2, 0, 4091, 4080, 3, 0, 0, 92, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4087, 0, 4, 0, 3, 3, 0, 3, 3, 3, 31, 0, 9, 4093, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 376, 0, 3945, 2, 3980, 90, 8, 1, 0, 0, 0, 4067, 4091, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4085, 0, 0, 0, 0, 1, 4092, 2, 0, 0, 3, 7, 29, 4092, 4070, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 3, 0, 0, 0, 3, 0, 4082, 4, 4, 4088, 3, 3981, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16409, 16380, 16349, 16347, 16410, 16588, 16393, 16196] avg=16384.0000 max=16588 max_rank=5 min=16196 min_rank=7 imbalance_ratio=1.012451 max_load_reduction=114484 max_load_reduction_pct=0.873444 moved_to_echo=65587 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 4083, 5, 4061, 6, 0, 0, 0, 5, 4, 6, 11, 4, 4091, 5, 3998, 139, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 259, 1, 0, 4086, 0, 0, 0, 0, 4, 4, 5, 4088, 4058, 3, 3836, 147, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 0, 4025, 4089, 4076, 0, 6, 0, 0, 9, 0, 4055, 6, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 4, 5, 0, 4065, 2, 0, 4089, 4084, 4, 0, 0, 92, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4083, 0, 5, 0, 6, 4, 0, 4, 4, 5, 37, 2, 18, 4091, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 10, 551, 0, 3942, 2, 3958, 127, 8, 0, 0, 0, 0, 4054, 4089, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4086, 0, 0, 7, 0, 1, 4090, 0, 0, 0, 4, 11, 55, 4091, 4068, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3767, 4, 0, 0, 0, 4, 0, 4073, 6, 5, 4085, 4, 3849, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16446, 16531, 16300, 16264, 16425, 16767, 16427, 15912] avg=16384.0000 max=16767 max_rank=5 min=15912 min_rank=7 imbalance_ratio=1.023376 max_load_reduction=114305 max_load_reduction_pct=0.872078 moved_to_echo=65531 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 0, 0, 32768, 0, 128, 0, 128, 128, 0, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 32768, 128, 32256, 128, 128, 0, 0, 128, 128, 128, 128, 128, 32768, 128, 32128, 1152, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 1280, 0, 31616, 128, 32512, 256, 128, 128, 0, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32256, 128, 0, 0, 0, 128, 0, 32768, 128, 128, 32768, 128, 32512, 0, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32128, 128, 128, 0, 32768, 128, 0, 32768, 32768, 128, 0, 0, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 128, 0, 128, 32768, 128, 0, 0, 128, 128, 128, 32768, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 640, 128, 0, 32768, 128, 128, 0, 0, 128, 128, 128, 32768, 32768, 128, 30848, 1280, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 0, 32768, 32768, 32768, 0, 128, 0, 384, 128, 0, 32512, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 4096, 0, 0, 0, 4093, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4089, 5, 4088, 4090, 0, 0, 0, 4091, 2, 4081, 4089, 0, 0, 11, 4089, 2, 4082, 3, 4088, 1, 6, 1, 0, 4092, 4091, 3, 4085, 5, 5, 4091, 0, 0, 6, 0, 0, 0, 6, 0, 6, 4, 0, 0, 3, 5, 3, 10, 2, 4092, 0, 8, 0, 0, 3, 6, 4067, 0, 6, 4091, 6, 4078, 23, 0, 4, 0, 4090, 7, 7, 8, 5, 5, 1, 0, 19, 0, 6, 4024, 0, 3, 4092, 0, 4084, 0, 19, 4, 0, 4088, 83, 0, 2, 1, 4085, 0, 4087, 0, 0, 5, 0, 3, 0, 67, 0, 4091, 13, 0, 5, 4090, 0, 4090, 4010, 10, 0, 0, 1, 0, 4, 0, 8, 0, 2, 4075, 4, 4086, 0, 3, 4072, 4, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4090, 3, 4090, 4092, 0, 0, 0, 4092, 1, 4087, 4091, 0, 0, 5, 4091, 57, 4086, 2, 4090, 1, 4, 1, 0, 4093, 4093, 2, 4087, 4, 3, 4093, 0, 0, 4, 0, 0, 0, 4, 0, 3, 3, 0, 0, 2, 3, 2, 6, 1, 4093, 0, 6, 0, 0, 2, 5, 4074, 0, 4, 4092, 4, 4078, 12, 0, 2, 0, 4092, 4, 5, 5, 3, 3, 1, 0, 15, 0, 3, 4057, 0, 1, 4093, 0, 4089, 0, 15, 1, 0, 4089, 50, 0, 1, 2, 4088, 0, 4081, 0, 0, 4, 1, 4, 0, 19, 0, 4092, 15, 0, 4, 4092, 0, 4092, 4023, 7, 0, 0, 2, 0, 3, 2, 6, 0, 7, 4077, 3, 4087, 0, 3, 4079, 3, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4094, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4091, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4, 4096, 4096, 0, 3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16378, 16380, 16382, 16388, 16373, 16394, 16391] avg=16384.0000 max=16394 max_rank=6 min=16373 min_rank=5 imbalance_ratio=1.000610 max_load_reduction=114678 max_load_reduction_pct=0.874924 moved_to_echo=65546 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4091, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16381, 16380, 16385, 16388, 16371, 16387, 16391] avg=16384.0000 max=16391 max_rank=7 min=16371 min_rank=5 imbalance_ratio=1.000427 max_load_reduction=114681 max_load_reduction_pct=0.874947 moved_to_echo=65537 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4091, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16381, 16379, 16385, 16389, 16371, 16387, 16391] avg=16384.0000 max=16391 max_rank=7 min=16371 min_rank=5 imbalance_ratio=1.000427 max_load_reduction=114681 max_load_reduction_pct=0.874947 moved_to_echo=65538 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 3, 4090, 0, 0, 0, 0, 4092, 1, 4087, 0, 0, 0, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4086, 2, 4090, 1, 4, 1, 0, 0, 0, 2, 4087, 4, 3, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 0, 3, 3, 0, 0, 2, 3, 2, 6, 1, 4093, 4093, 4092, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 6, 0, 0, 2, 5, 4074, 0, 4, 4092, 4, 4078, 12, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 5, 5, 3, 3, 1, 0, 15, 0, 3, 4057, 0, 1, 4093, 4086, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 15, 1, 0, 4078, 50, 0, 1, 2, 4088, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 4, 0, 19, 0, 4092, 15, 0, 4, 4092, 0, 4092, 4023, 7, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 2, 6, 0, 7, 4077, 3, 4087, 0, 3, 4079, 3, 4091, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 16427, 16392, 16373, 16378, 16405, 16355, 16369] avg=16384.0000 max=16427 max_rank=1 min=16355 min_rank=6 imbalance_ratio=1.002625 max_load_reduction=114645 max_load_reduction_pct=0.874672 moved_to_echo=65507 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4091, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4091, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16381, 16379, 16386, 16389, 16375, 16382, 16391] avg=16384.0000 max=16391 max_rank=7 min=16375 min_rank=5 imbalance_ratio=1.000427 max_load_reduction=114681 max_load_reduction_pct=0.874947 moved_to_echo=65537 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 5, 4088, 0, 0, 0, 0, 4091, 2, 4081, 0, 0, 0, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4082, 3, 4088, 1, 6, 1, 0, 0, 0, 3, 4085, 5, 5, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 6, 0, 6, 4, 0, 0, 3, 5, 3, 10, 2, 4092, 4091, 4090, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 8, 0, 0, 3, 6, 4067, 0, 6, 4091, 6, 4078, 23, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 7, 7, 8, 5, 5, 1, 0, 19, 0, 6, 4024, 0, 3, 4092, 4084, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 19, 4, 0, 4077, 83, 0, 2, 1, 4085, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3, 0, 67, 0, 4091, 13, 0, 5, 4090, 0, 4090, 4010, 10, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 8, 0, 2, 4075, 4, 4086, 0, 3, 4072, 4, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 16369, 16402, 16385, 16361, 16442, 16386, 16355] avg=16384.0000 max=16442 max_rank=5 min=16355 min_rank=7 imbalance_ratio=1.003540 max_load_reduction=114630 max_load_reduction_pct=0.874557 moved_to_echo=65544 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4092, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 4096, 4096, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16374, 16380, 16391, 16391, 16365, 16392, 16393] avg=16384.0000 max=16393 max_rank=7 min=16365 min_rank=5 imbalance_ratio=1.000549 max_load_reduction=114679 max_load_reduction_pct=0.874931 moved_to_echo=65541 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4091, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16381, 16380, 16385, 16388, 16370, 16389, 16391] avg=16384.0000 max=16391 max_rank=7 min=16370 min_rank=5 imbalance_ratio=1.000427 max_load_reduction=114681 max_load_reduction_pct=0.874947 moved_to_echo=65538 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 32768, 128, 128, 128, 0, 0, 0, 128, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 32768, 0, 0, 0, 0, 32768, 128, 32768, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 128, 128, 128, 0, 32768, 256, 0, 128, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 128, 0, 32768, 128, 0, 128, 32768, 128, 32768, 32640, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 128, 128, 0, 0, 128, 128, 128, 128, 128, 32768, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 0, 128, 0, 0, 128, 128, 32768, 0, 128, 32768, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 128, 0, 128, 32768, 128, 32768, 0, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 128, 128, 0, 128, 0, 128, 32768, 0, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 6, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4091, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 38, 4081, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4056, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 50, 4094, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4047, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[3989, 6, 4089, 2, 0, 4081, 5, 0, 4, 4090, 0, 0, 0, 4088, 4091, 0, 0, 0, 4091, 0, 5, 5, 4075, 4089, 4090, 2, 7, 0, 4090, 8, 23, 4091, 0, 0, 16, 7, 2, 1, 0, 4, 0, 0, 6, 7, 0, 4091, 0, 6, 4, 6, 2, 0, 4, 0, 4087, 4089, 4084, 9, 0, 11, 0, 4089, 3, 4065, 0, 4043, 0, 6, 5, 0, 5, 0, 1, 0, 2, 5, 0, 2, 6, 0, 0, 4082, 0, 0, 0, 4085, 0, 4089, 3, 0, 4086, 4090, 0, 0, 0, 0, 0, 5, 13, 0, 4087, 3, 4091, 2, 0, 5, 0, 6, 7, 6, 2, 0, 7, 4089, 4090, 1, 0, 4091, 4087, 4089, 2, 60, 3, 9, 6, 4091, 96, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 4096, 0, 0, 0, 0, 4095, 0, 4096, 17, 4084, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4083, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4094, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4008, 4, 4090, 1, 0, 4086, 3, 0, 2, 4092, 0, 0, 0, 4091, 4093, 0, 0, 0, 4092, 0, 4, 3, 4081, 4091, 4092, 3, 3, 0, 4092, 6, 16, 4092, 0, 0, 11, 0, 1, 1, 6, 0, 0, 0, 5, 4, 0, 4093, 0, 4, 3, 4, 0, 0, 3, 0, 4091, 4091, 4086, 2, 3, 0, 5, 4090, 2, 4077, 0, 4059, 0, 2, 3, 0, 4, 0, 2, 0, 0, 3, 0, 1, 3, 0, 0, 4085, 0, 2, 0, 4086, 3, 4092, 2, 0, 4090, 4092, 0, 0, 0, 1, 1, 4, 8, 0, 4089, 3, 4093, 2, 0, 3, 0, 4, 5, 4, 0, 0, 5, 4092, 4092, 0, 0, 4093, 4092, 4091, 2, 59, 3, 12, 5, 4092, 61, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4081, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4068, 21, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 16382, 16377, 16393, 16377, 16392, 16377, 16362] avg=16384.0000 max=16412 max_rank=0 min=16362 min_rank=7 imbalance_ratio=1.001709 max_load_reduction=114660 max_load_reduction_pct=0.874786 moved_to_echo=65508 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 9, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4084, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 4096, 4096, 4081, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 4096, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4068, 21, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 16382, 16378, 16378, 16402, 16393, 16377, 16350] avg=16384.0000 max=16412 max_rank=0 min=16350 min_rank=7 imbalance_ratio=1.001709 max_load_reduction=114660 max_load_reduction_pct=0.874786 moved_to_echo=65522 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 9, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4081, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 4096, 4096, 4081, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4068, 21, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 4056, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 16382, 16378, 16392, 16416, 16392, 16377, 16323] avg=16384.0000 max=16416 max_rank=4 min=16323 min_rank=7 imbalance_ratio=1.001953 max_load_reduction=114656 max_load_reduction_pct=0.874756 moved_to_echo=65508 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 23, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 4094, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4096, 4096, 4083, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4073, 28, 8, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4068, 0, 0, 0, 4047, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16381, 16378, 16376, 16438, 16389, 16396, 16307] avg=16384.0000 max=16438 max_rank=4 min=16307 min_rank=7 imbalance_ratio=1.003296 max_load_reduction=114634 max_load_reduction_pct=0.874588 moved_to_echo=65530 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4081, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4068, 21, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 16382, 16378, 16393, 16380, 16392, 16377, 16358] avg=16384.0000 max=16412 max_rank=0 min=16358 min_rank=7 imbalance_ratio=1.001709 max_load_reduction=114660 max_load_reduction_pct=0.874786 moved_to_echo=65507 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4008, 4, 4090, 1, 0, 4086, 3, 0, 2, 28, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 4081, 4091, 4092, 3, 3, 0, 4090, 6, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1, 1, 6, 0, 0, 0, 5, 4, 0, 4093, 0, 4, 4093, 4093, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 3, 0, 4091, 9, 4086, 2, 3, 0, 5, 4090, 2, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 2, 3, 0, 4, 0, 2, 0, 0, 3, 0, 1, 3, 0, 4092, 4092, 4077, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 2, 0, 4086, 3, 4092, 2, 0, 4090, 8, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 8, 0, 4089, 3, 4093, 2, 0, 3, 0, 4, 5, 4, 0, 0, 4082, 4064, 21, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4092, 4071, 0, 0, 0, 4092, 4091, 2, 59, 3, 12, 5, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16313, 16389, 16398, 16375, 16343, 16369, 16392, 16493] avg=16384.0000 max=16493 max_rank=7 min=16313 min_rank=0 imbalance_ratio=1.006653 max_load_reduction=114579 max_load_reduction_pct=0.874168 moved_to_echo=65597 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 9, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 4081, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4068, 21, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 16382, 16378, 16392, 16378, 16392, 16377, 16361] avg=16384.0000 max=16412 max_rank=0 min=16361 min_rank=7 imbalance_ratio=1.001709 max_load_reduction=114660 max_load_reduction_pct=0.874786 moved_to_echo=65508 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3989, 6, 4089, 2, 0, 4081, 5, 0, 4, 28, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 4075, 4089, 4090, 2, 7, 0, 4088, 8, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 7, 2, 1, 0, 4, 0, 0, 6, 7, 0, 4091, 0, 6, 4091, 4091, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 2, 0, 4, 0, 4087, 9, 4084, 9, 0, 11, 0, 4089, 3, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 6, 5, 0, 5, 0, 1, 0, 2, 5, 0, 2, 6, 0, 4091, 4091, 4075, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4085, 0, 4089, 3, 0, 4086, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 13, 0, 4087, 3, 4091, 2, 0, 5, 0, 6, 7, 6, 2, 0, 4080, 4062, 21, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 4069, 1, 0, 0, 4087, 4089, 2, 60, 3, 9, 6, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16292, 16392, 16407, 16373, 16338, 16353, 16399, 16518] avg=16384.0000 max=16518 max_rank=7 min=16292 min_rank=0 imbalance_ratio=1.008179 max_load_reduction=114554 max_load_reduction_pct=0.873978 moved_to_echo=65608 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32640, 128, 32768, 128, 0, 32768, 128, 0, 128, 256, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 128, 0, 32768, 128, 32768, 128, 128, 128, 128, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 32768, 128, 0, 128, 0, 128, 128, 128, 128, 0, 32768, 32640, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 128, 0, 128, 128, 128, 128, 0, 128, 128, 0, 32768, 32768, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 128, 32768, 32768, 32768, 128, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 0, 128, 128, 32768, 128, 32768, 128, 0, 32768, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 32640, 128, 0, 0, 32768, 32768, 128, 128, 128, 128, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 128, 0, 0, 128, 128, 0, 32768, 0, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True + [2026-08-30 05:05:00] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 748.7 | throughput per GPU (TFLOP/s/GPU): 445.3 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.890746E+00 | loss scale: 1.0 | grad norm: 6.158 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[226, 1294, 993, 0, 376, 0, 2640, 42, 767, 775, 196, 310, 2488, 2348, 0, 2820, 2080, 1073, 2073, 2379, 70, 962, 151, 1136, 62, 13, 3559, 2010, 1089, 6, 1754, 964, 3396, 0, 429, 742, 491, 579, 1774, 157, 176, 11, 1799, 2861, 0, 3184, 690, 1380, 1548, 1874, 1003, 275, 0, 0, 66, 2611, 0, 324, 347, 313, 1661, 3260, 1873, 48, 1640, 245, 0, 1187, 2788, 498, 910, 850, 2583, 2799, 1956, 22, 873, 0, 190, 1465, 688, 812, 0, 78, 899, 770, 935, 104, 2458, 18, 1542, 1444, 316, 1656, 475, 0, 1172, 1004, 18, 882, 7, 541, 1623, 1022, 308, 1142, 38, 1658, 645, 2482, 1022, 1240, 2084, 1866, 538, 2437, 767, 535, 1578, 1698, 1557, 2218, 1004, 1115, 285, 34, 821, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[455, 1247, 1529, 0, 93, 0, 2212, 86, 551, 1189, 422, 424, 2504, 3568, 0, 2792, 2223, 1410, 2301, 2582, 91, 1241, 7, 910, 17, 18, 3351, 2250, 1452, 8, 2338, 706, 2930, 1, 365, 545, 923, 568, 1817, 111, 129, 1, 1368, 3286, 0, 2868, 503, 1115, 1976, 2177, 912, 104, 1, 7, 44, 2696, 0, 134, 88, 156, 1827, 3874, 1577, 9, 1196, 235, 0, 1275, 1815, 232, 343, 642, 3205, 2891, 1832, 21, 1069, 5, 304, 2098, 278, 1150, 0, 61, 942, 1055, 333, 85, 1686, 11, 1349, 1384, 142, 2190, 465, 0, 909, 1134, 10, 1086, 0, 538, 1257, 1316, 133, 1254, 27, 1548, 495, 2933, 1669, 1217, 1737, 2677, 517, 2152, 607, 446, 1129, 1329, 1382, 2472, 1020, 800, 422, 0, 462, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[389, 1125, 1784, 2, 238, 0, 2582, 9, 533, 1302, 91, 687, 2795, 2982, 0, 2749, 2039, 1346, 2388, 2113, 76, 871, 123, 1154, 3, 4, 3366, 2214, 1574, 7, 2014, 935, 3188, 2, 412, 586, 626, 522, 1785, 119, 165, 3, 1234, 2954, 0, 3107, 455, 1541, 1968, 1518, 656, 186, 0, 6, 199, 2941, 1, 316, 331, 144, 2074, 3392, 1839, 19, 2379, 323, 0, 1539, 2084, 398, 365, 876, 2777, 2846, 1651, 78, 870, 6, 423, 1173, 350, 1380, 0, 74, 869, 1417, 355, 195, 1452, 24, 1044, 1609, 237, 2251, 352, 3, 770, 1231, 12, 1467, 0, 876, 1520, 919, 216, 976, 26, 1663, 1017, 2694, 753, 881, 2007, 2470, 313, 1882, 899, 357, 987, 1209, 1593, 2124, 967, 1122, 406, 9, 522, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[378, 1106, 1164, 0, 76, 0, 2878, 24, 204, 1345, 181, 612, 2266, 3502, 0, 2904, 2165, 1200, 2138, 2075, 172, 1168, 91, 1068, 10, 7, 3745, 2116, 1382, 32, 1860, 904, 2659, 0, 314, 429, 456, 427, 1978, 250, 150, 8, 1980, 3079, 0, 3118, 322, 1016, 1510, 2397, 1205, 169, 0, 5, 45, 3093, 0, 150, 85, 277, 1955, 3695, 1810, 26, 937, 242, 0, 1563, 1874, 410, 320, 542, 2819, 2657, 1697, 7, 1162, 11, 218, 2075, 512, 756, 0, 64, 667, 980, 500, 40, 2030, 12, 1414, 1334, 212, 2609, 469, 1, 1535, 1532, 36, 1217, 1, 803, 1277, 833, 281, 1036, 43, 1087, 433, 2350, 1446, 945, 1465, 2806, 591, 2232, 836, 693, 1149, 1440, 1373, 3014, 1066, 913, 278, 0, 846, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[372, 1049, 1262, 4, 58, 0, 2853, 94, 539, 1579, 268, 311, 2033, 3389, 0, 2962, 1041, 1804, 2136, 2797, 97, 943, 70, 1115, 2, 16, 3630, 2265, 1453, 17, 2483, 797, 2596, 1, 358, 584, 452, 630, 1675, 284, 101, 3, 2013, 3207, 0, 2876, 333, 1281, 1858, 3205, 1334, 125, 0, 80, 63, 3052, 1, 172, 23, 270, 1937, 3403, 1313, 10, 1347, 339, 0, 1275, 1458, 812, 363, 720, 2660, 2710, 1667, 8, 987, 8, 314, 2045, 593, 761, 0, 10, 517, 629, 633, 75, 1703, 46, 1657, 1163, 135, 2982, 234, 0, 1812, 1310, 35, 878, 1, 656, 1102, 693, 232, 898, 3, 847, 553, 2097, 951, 1439, 1761, 2665, 612, 2745, 1042, 416, 1148, 1282, 1705, 3035, 789, 1036, 208, 0, 591, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[620, 1526, 1429, 5, 48, 1, 3132, 66, 513, 1747, 225, 313, 2405, 3551, 0, 2512, 1498, 1279, 2039, 2346, 357, 1015, 154, 1328, 10, 16, 3628, 2216, 1439, 24, 2274, 712, 2351, 0, 295, 414, 379, 425, 2197, 193, 274, 1, 2043, 3202, 0, 2965, 343, 1384, 1494, 2274, 1343, 174, 1, 17, 8, 2510, 1, 213, 100, 446, 2048, 2945, 1763, 13, 1058, 266, 0, 1341, 1365, 821, 306, 962, 2624, 2423, 1525, 69, 769, 8, 268, 1882, 910, 1094, 0, 61, 732, 459, 654, 81, 1717, 15, 1971, 1441, 212, 2987, 258, 0, 2215, 1627, 29, 846, 3, 433, 1357, 978, 393, 1173, 3, 1136, 385, 2383, 1485, 1097, 1693, 2177, 338, 2166, 642, 690, 1119, 1134, 1556, 2961, 673, 974, 178, 6, 699, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[291, 1102, 1345, 0, 66, 4, 2866, 35, 908, 1504, 353, 837, 2396, 3066, 0, 2923, 2046, 1752, 2129, 2204, 62, 1134, 82, 1246, 17, 16, 3722, 1878, 1308, 9, 2063, 453, 2229, 0, 300, 510, 432, 435, 1736, 146, 165, 6, 1830, 3018, 0, 3034, 452, 840, 1887, 2624, 1261, 141, 0, 3, 60, 2614, 0, 370, 107, 346, 2014, 3467, 1439, 22, 1209, 226, 0, 1978, 2382, 264, 317, 783, 2521, 2392, 1451, 37, 633, 7, 234, 1866, 957, 1097, 1, 65, 462, 486, 627, 110, 2324, 13, 1592, 1830, 207, 2406, 292, 0, 1831, 1475, 23, 1271, 0, 432, 1703, 918, 230, 935, 11, 657, 373, 2149, 1043, 1711, 1893, 2391, 393, 2221, 698, 386, 1400, 1641, 1606, 2801, 479, 1441, 262, 7, 613, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[247, 1001, 1783, 0, 54, 0, 2987, 47, 397, 1459, 207, 755, 2670, 3295, 0, 2534, 1557, 1346, 2418, 2637, 126, 1019, 119, 1087, 1, 13, 3745, 2205, 1968, 10, 2357, 990, 3349, 0, 269, 558, 1003, 592, 1637, 183, 142, 0, 1482, 3408, 0, 3247, 561, 1082, 1571, 2496, 837, 98, 0, 1, 37, 3183, 2, 147, 39, 162, 1948, 3519, 1831, 2, 1749, 227, 0, 1786, 1403, 464, 432, 811, 2304, 2662, 1583, 11, 885, 0, 194, 1263, 666, 871, 0, 34, 882, 634, 418, 65, 1735, 10, 1604, 1515, 177, 2523, 358, 0, 1447, 1232, 25, 923, 0, 789, 1458, 770, 273, 768, 6, 1084, 622, 2599, 1198, 1164, 1882, 2307, 351, 2496, 683, 482, 1457, 1041, 1507, 2592, 685, 872, 105, 5, 558, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[226, 1294, 993, 0, 376, 0, 2640, 42, 767, 775, 196, 310, 2488, 1875, 0, 2820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2080, 1073, 2073, 1938, 70, 962, 151, 1136, 62, 13, 0, 2010, 1089, 6, 1754, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3396, 0, 429, 742, 491, 579, 1774, 157, 176, 11, 1799, 2688, 0, 3184, 690, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1548, 1874, 1003, 275, 0, 0, 66, 2611, 0, 324, 347, 313, 1661, 3260, 1873, 48, 280, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1640, 245, 0, 1187, 2788, 498, 910, 850, 2236, 2799, 1956, 22, 873, 0, 190, 1465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 688, 812, 0, 78, 899, 770, 935, 104, 2458, 18, 1542, 1444, 316, 1656, 475, 0, 3559, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1172, 1004, 18, 882, 7, 541, 1623, 1022, 308, 1142, 38, 1658, 645, 2482, 1022, 1240, 473, 441, 356, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2084, 1866, 538, 2437, 767, 535, 1578, 1698, 1557, 1016, 1004, 1115, 285, 34, 821, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14802, 15381, 17496, 15656, 17659, 16600, 16141, 17337] avg=16384.0000 max=17659 max_rank=4 min=14802 min_rank=0 imbalance_ratio=1.077820 max_load_reduction=113413 max_load_reduction_pct=0.865273 moved_to_echo=67737 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[389, 1125, 1784, 2, 238, 0, 2582, 9, 533, 1302, 91, 687, 2795, 2381, 0, 2749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2039, 1346, 2388, 1722, 76, 871, 123, 1154, 3, 4, 0, 2214, 1574, 7, 2014, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3188, 2, 412, 586, 626, 522, 1785, 119, 165, 3, 1234, 2775, 0, 3107, 455, 1541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1968, 1518, 656, 186, 0, 6, 199, 2941, 1, 316, 331, 144, 2074, 3392, 1839, 19, 301, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2379, 323, 0, 1539, 2084, 398, 365, 876, 2404, 2846, 1651, 78, 870, 6, 423, 1173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 1380, 0, 74, 869, 1417, 355, 195, 1452, 24, 1044, 1609, 237, 2251, 352, 3, 3366, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 770, 1231, 12, 1467, 0, 876, 1520, 919, 216, 976, 26, 1663, 1017, 2694, 753, 881, 601, 391, 341, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2007, 2470, 313, 1882, 899, 357, 987, 1209, 1593, 973, 967, 1122, 406, 9, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16667, 16470, 16520, 16070, 17415, 15788, 16426, 15716] avg=16384.0000 max=17415 max_rank=4 min=15716 min_rank=7 imbalance_ratio=1.062927 max_load_reduction=113657 max_load_reduction_pct=0.867134 moved_to_echo=65345 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[372, 1049, 1262, 4, 58, 0, 2853, 94, 539, 1579, 268, 311, 2033, 2706, 0, 2962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 1804, 2136, 2279, 97, 943, 70, 1115, 2, 16, 0, 2265, 1453, 17, 2483, 797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2596, 1, 358, 584, 452, 630, 1675, 284, 101, 3, 2013, 3013, 0, 2876, 333, 1281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1858, 3205, 1334, 125, 0, 80, 63, 3052, 1, 172, 23, 270, 1937, 3403, 1313, 10, 288, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1347, 339, 0, 1275, 1458, 812, 363, 720, 2303, 2710, 1667, 8, 987, 8, 314, 2045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 593, 761, 0, 10, 517, 629, 633, 75, 1703, 46, 1657, 1163, 135, 2982, 234, 0, 3630, 1158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1812, 1310, 35, 878, 1, 656, 1102, 693, 232, 898, 3, 847, 553, 2097, 951, 1439, 683, 518, 487, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1761, 2665, 612, 2745, 1042, 416, 1148, 1282, 1705, 1390, 789, 1036, 208, 0, 591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16090, 16518, 16200, 17328, 16356, 15926, 15264, 17390] avg=16384.0000 max=17390 max_rank=7 min=15264 min_rank=6 imbalance_ratio=1.061401 max_load_reduction=113682 max_load_reduction_pct=0.867325 moved_to_echo=64936 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[378, 1106, 1164, 0, 76, 0, 2878, 24, 204, 1345, 181, 612, 2266, 2794, 0, 2904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2165, 1200, 2138, 1687, 172, 1168, 91, 1068, 10, 7, 0, 2116, 1382, 32, 1860, 904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2659, 0, 314, 429, 456, 427, 1978, 250, 150, 8, 1980, 2889, 0, 3118, 322, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1510, 2397, 1205, 169, 0, 5, 45, 3093, 0, 150, 85, 277, 1955, 3695, 1810, 26, 310, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 937, 242, 0, 1563, 1874, 410, 320, 542, 2433, 2657, 1697, 7, 1162, 11, 218, 2075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 756, 0, 64, 667, 980, 500, 40, 2030, 12, 1414, 1334, 212, 2609, 469, 1, 3745, 1153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1535, 1532, 36, 1217, 1, 803, 1277, 833, 281, 1036, 43, 1087, 433, 2350, 1446, 945, 708, 388, 487, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1465, 2806, 591, 2232, 836, 693, 1149, 1440, 1373, 1374, 1066, 913, 278, 0, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15932, 16000, 15996, 16922, 16148, 16498, 16514, 17062] avg=16384.0000 max=17062 max_rank=7 min=15932 min_rank=0 imbalance_ratio=1.041382 max_load_reduction=114010 max_load_reduction_pct=0.869827 moved_to_echo=66222 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[620, 1526, 1429, 5, 48, 1, 3132, 66, 513, 1747, 225, 313, 2405, 2836, 0, 2512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1498, 1279, 2039, 1911, 357, 1015, 154, 1328, 10, 16, 0, 2216, 1439, 24, 2274, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2351, 0, 295, 414, 379, 425, 2197, 193, 274, 1, 2043, 3008, 0, 2965, 343, 1384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 2274, 1343, 174, 1, 17, 8, 2510, 1, 213, 100, 446, 2048, 2945, 1763, 13, 284, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 266, 0, 1341, 1365, 821, 306, 962, 2272, 2423, 1525, 69, 769, 8, 268, 1882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 910, 1094, 0, 61, 732, 459, 654, 81, 1717, 15, 1971, 1441, 212, 2987, 258, 0, 3628, 1130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2215, 1627, 29, 846, 3, 433, 1357, 978, 393, 1173, 3, 1136, 385, 2383, 1485, 1097, 715, 435, 475, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1693, 2177, 338, 2166, 642, 690, 1119, 1134, 1556, 1356, 673, 974, 178, 6, 699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17378, 16272, 16272, 15828, 15335, 17350, 17236, 15401] avg=16384.0000 max=17378 max_rank=0 min=15335 min_rank=4 imbalance_ratio=1.060669 max_load_reduction=113694 max_load_reduction_pct=0.867416 moved_to_echo=65322 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[455, 1247, 1529, 0, 93, 0, 2212, 86, 551, 1189, 422, 424, 2504, 2849, 0, 2792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 1410, 2301, 2104, 91, 1241, 7, 910, 17, 18, 0, 2250, 1452, 8, 2338, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2930, 1, 365, 545, 923, 568, 1817, 111, 129, 1, 1368, 3087, 0, 2868, 503, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1976, 2177, 912, 104, 1, 7, 44, 2696, 0, 134, 88, 156, 1827, 3874, 1577, 9, 347, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1196, 235, 0, 1275, 1815, 232, 343, 642, 2775, 2891, 1832, 21, 1069, 5, 304, 2098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 1150, 0, 61, 942, 1055, 333, 85, 1686, 11, 1349, 1384, 142, 2190, 465, 0, 3351, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 909, 1134, 10, 1086, 0, 538, 1257, 1316, 133, 1254, 27, 1548, 495, 2933, 1669, 1217, 719, 478, 396, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1737, 2677, 517, 2152, 607, 446, 1129, 1329, 1382, 1133, 1020, 800, 422, 0, 462, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16353, 17076, 16331, 16128, 16733, 15425, 17202, 15824] avg=16384.0000 max=17202 max_rank=6 min=15425 min_rank=5 imbalance_ratio=1.049927 max_load_reduction=113870 max_load_reduction_pct=0.868759 moved_to_echo=65184 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[291, 1102, 1345, 0, 66, 4, 2866, 35, 908, 1504, 353, 837, 2396, 2448, 0, 2923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2046, 1752, 2129, 1796, 62, 1134, 82, 1246, 17, 16, 0, 1878, 1308, 9, 2063, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2229, 0, 300, 510, 432, 435, 1736, 146, 165, 6, 1830, 2835, 0, 3034, 452, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1887, 2624, 1261, 141, 0, 3, 60, 2614, 0, 370, 107, 346, 2014, 3467, 1439, 22, 273, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1209, 226, 0, 1978, 2382, 264, 317, 783, 2183, 2392, 1451, 37, 633, 7, 234, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 957, 1097, 1, 65, 462, 486, 627, 110, 2324, 13, 1592, 1830, 207, 2406, 292, 0, 3722, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1831, 1475, 23, 1271, 0, 432, 1703, 918, 230, 935, 11, 657, 373, 2149, 1043, 1711, 618, 408, 449, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1893, 2391, 393, 2221, 698, 386, 1400, 1641, 1606, 1284, 479, 1441, 262, 7, 613, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17078, 15991, 14950, 16811, 15962, 17259, 16302, 16719] avg=16384.0000 max=17259 max_rank=5 min=14950 min_rank=2 imbalance_ratio=1.053406 max_load_reduction=113813 max_load_reduction_pct=0.868324 moved_to_echo=66242 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[247, 1001, 1783, 0, 54, 0, 2987, 47, 397, 1459, 207, 755, 2670, 2631, 0, 2534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1557, 1346, 2418, 2148, 126, 1019, 119, 1087, 1, 13, 0, 2205, 1968, 10, 2357, 990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3349, 0, 269, 558, 1003, 592, 1637, 183, 142, 0, 1482, 3202, 0, 3247, 561, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1571, 2496, 837, 98, 0, 1, 37, 3183, 2, 147, 39, 162, 1948, 3519, 1831, 2, 250, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1749, 227, 0, 1786, 1403, 464, 432, 811, 1994, 2662, 1583, 11, 885, 0, 194, 1263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 666, 871, 0, 34, 882, 634, 418, 65, 1735, 10, 1604, 1515, 177, 2523, 358, 0, 3745, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1447, 1232, 25, 923, 0, 789, 1458, 770, 273, 768, 6, 1084, 622, 2599, 1198, 1164, 664, 489, 416, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1882, 2307, 351, 2496, 683, 482, 1457, 1041, 1507, 1187, 685, 872, 105, 5, 558, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16772, 17364, 17307, 16329, 15464, 16226, 15987, 15623] avg=16384.0000 max=17364 max_rank=1 min=15464 min_rank=4 imbalance_ratio=1.059814 max_load_reduction=113708 max_load_reduction_pct=0.867523 moved_to_echo=63300 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[14720, 11264, 17664, 15616, 1152, 8448, 896, 9088, 128, 128, 0, 17280, 11776, 128, 17152, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[11776, 10624, 256, 8576, 128, 5120, 11392, 7552, 2176, 8192, 256, 9728, 4608, 19712, 9600, 9728, 5248, 3584, 3456, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[3072, 9472, 11392, 128, 1024, 128, 22272, 512, 4480, 11008, 2048, 4352, 19584, 20608, 0, 22272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[4992, 7936, 128, 512, 6016, 6528, 4480, 768, 15232, 256, 12288, 11776, 1664, 19712, 2944, 128, 28800, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[11520, 2176, 0, 12032, 15232, 3968, 3456, 6272, 18688, 21504, 13440, 256, 7296, 128, 2176, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[13824, 18688, 8576, 1280, 128, 128, 640, 22784, 128, 1920, 1152, 2176, 15488, 27648, 13568, 256, 2432, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[22784, 128, 2816, 4480, 4864, 4224, 14720, 1536, 1408, 128, 13824, 23552, 0, 24448, 3712, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[14592, 19456, 3712, 18432, 6272, 4096, 9984, 10880, 12288, 9728, 6784, 8320, 2176, 128, 5120, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3, 1371, 0, 0, 4096, 4094, 0, 0, 4095, 2, 0, 0, 1, 2, 0, 0, 4095, 0, 0, 4092, 4095, 0, 55, 0, 4095, 0, 4096, 36, 4091, 1, 0, 4096, 2, 0, 4092, 1, 4095, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 1879, 4096, 0, 0, 716, 8, 0, 4096, 1756, 0, 0, 1546, 0, 0, 4095, 0, 0, 4091, 1, 0, 3742, 3630, 0, 102, 0, 2, 6, 0, 1, 0, 1, 0, 1, 3730, 0, 0, 1, 1, 0, 3, 0, 0, 1, 0, 0, 4095, 0, 1562, 0, 0, 0, 4092, 4093, 4074, 0, 0, 5, 4095, 0, 9, 0, 0, 0, 358, 4096, 0, 1, 4095, 1, 5, 1467, 1, 4091, 2, 1, 2646, 0, 0, 0, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 852, 0, 0, 4073, 4028, 0, 10, 4094, 0, 0, 0, 1, 6, 0, 0, 4096, 0, 0, 4096, 4092, 0, 14, 0, 4083, 0, 4096, 126, 4091, 0, 3, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4015, 0, 15, 1025, 4084, 0, 0, 514, 0, 0, 4096, 3583, 0, 0, 348, 0, 0, 4096, 0, 0, 4094, 0, 0, 3604, 3453, 0, 25, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3385, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 2732, 0, 0, 0, 4096, 4055, 4079, 0, 0, 2, 4096, 0, 68, 0, 0, 0, 1297, 4096, 0, 0, 4096, 0, 0, 1949, 0, 4083, 0, 0, 1861, 0, 0, 0, 4075] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1, 2, 1, 197, 0, 2, 4086, 3912, 4, 4, 4088, 0, 0, 1, 6, 10, 2, 1, 4094, 0, 0, 4093, 4092, 7, 123, 0, 4087, 0, 4094, 60, 4096, 0, 0, 4093, 0, 0, 4092, 0, 4095, 0, 0, 0, 1, 3, 0, 4047, 3, 1, 1735, 4089, 3, 1, 176, 10, 0, 4093, 4050, 1, 0, 727, 0, 0, 4089, 0, 1, 4089, 0, 0, 4047, 3288, 0, 150, 1, 7, 14, 1, 1, 0, 1, 1, 3, 3621, 6, 1, 0, 3, 0, 0, 3, 0, 2, 0, 1, 4094, 0, 3458, 0, 0, 0, 4093, 4063, 4085, 0, 2, 0, 4085, 3, 36, 1, 1, 3, 437, 4093, 6, 2, 4093, 0, 16, 773, 4, 4096, 1, 1, 1927, 0, 0, 0, 4086] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 923, 15, 0, 4096, 4084, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 241, 0, 4094, 1, 4096, 1, 4087, 0, 0, 4096, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 1074, 4096, 0, 0, 1424, 0, 0, 4096, 3728, 0, 3, 1077, 0, 0, 4096, 0, 0, 4094, 0, 0, 3722, 2079, 0, 48, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3892, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 2499, 0, 0, 0, 4096, 4079, 3805, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 762, 4096, 0, 0, 4096, 0, 1, 1906, 0, 4095, 0, 0, 1570, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 295, 0, 0, 4090, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4096, 1, 182, 0, 4095, 5, 4096, 761, 4043, 0, 0, 4096, 0, 1, 4091, 0, 4086, 0, 0, 1, 0, 23, 0, 4094, 0, 3, 223, 4096, 0, 0, 667, 0, 2, 4096, 2158, 0, 0, 1389, 0, 0, 4096, 0, 0, 4091, 0, 0, 3998, 2708, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 2045, 0, 0, 0, 4096, 4095, 3879, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 549, 4096, 0, 0, 4096, 0, 9, 3275, 0, 4095, 0, 0, 2434, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 2575, 24, 0, 4092, 4066, 0, 0, 4095, 0, 0, 0, 0, 51, 0, 0, 4096, 0, 0, 4096, 4086, 6, 92, 0, 4090, 0, 4096, 58, 4080, 0, 0, 4096, 0, 0, 4096, 0, 4093, 0, 0, 2, 0, 0, 0, 4076, 0, 2, 297, 4096, 1, 0, 1190, 11, 0, 4096, 3382, 0, 1, 255, 0, 0, 4092, 0, 0, 4070, 0, 0, 3582, 2164, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 4096, 0, 2213, 0, 0, 0, 4096, 4091, 3737, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 1680, 4096, 0, 0, 4096, 0, 11, 3352, 0, 4091, 0, 0, 804, 0, 0, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1016, 1, 0, 4096, 4042, 0, 0, 4092, 1, 0, 0, 1, 6, 0, 0, 4096, 0, 0, 4076, 4092, 0, 107, 0, 4093, 0, 4096, 4, 4079, 0, 0, 4096, 1, 0, 4096, 1, 4094, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 962, 4093, 0, 0, 208, 24, 2, 4096, 4015, 0, 2, 855, 0, 0, 4096, 0, 0, 4092, 0, 0, 3952, 2903, 0, 119, 0, 1, 6, 0, 0, 0, 0, 0, 0, 3733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2240, 0, 0, 0, 4096, 4096, 4042, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 806, 4096, 0, 0, 4093, 0, 1, 3068, 0, 4088, 7, 1, 711, 0, 0, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 561, 0, 0, 4096, 3298, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4090, 2, 12, 0, 4095, 0, 4093, 3, 4071, 0, 0, 4096, 0, 0, 4096, 1, 4095, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 2621, 4095, 0, 0, 816, 0, 0, 4096, 3986, 0, 6, 1226, 0, 0, 4095, 0, 0, 4093, 0, 0, 3916, 3583, 0, 58, 0, 1, 1, 0, 0, 0, 0, 0, 0, 3889, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 2643, 0, 0, 0, 4096, 4026, 4092, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 192, 4096, 0, 0, 4096, 0, 2, 312, 0, 4095, 1, 1, 1713, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 1371, 0, 0, 4096, 4094, 0, 0, 4095, 2, 0, 0, 1, 2, 2109, 631, 333, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 4095, 0, 55, 0, 4095, 0, 0, 36, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4092, 1, 4095, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1879, 4096, 0, 0, 716, 8, 0, 1987, 1756, 0, 0, 1546, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 3742, 3630, 0, 102, 0, 2, 6, 0, 1, 0, 1, 0, 3385, 1991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3730, 0, 0, 1, 1, 0, 3, 0, 0, 1, 0, 0, 4095, 0, 1562, 4095, 2105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3461, 4093, 4074, 0, 0, 5, 4095, 0, 9, 0, 0, 0, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 1, 4095, 1, 5, 1467, 1, 4091, 2, 1, 2646, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16915, 16257, 16401, 16083, 16952, 15594, 16095, 16775] avg=16384.0000 max=16952 max_rank=4 min=15594 min_rank=5 imbalance_ratio=1.034668 max_load_reduction=114120 max_load_reduction_pct=0.870667 moved_to_echo=65416 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 1, 197, 0, 2, 4086, 3912, 4, 4, 4088, 0, 0, 1, 6, 10, 2108, 631, 333, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 3884, 4092, 7, 123, 0, 4087, 0, 0, 60, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4092, 0, 4095, 0, 0, 0, 1, 3, 0, 4047, 3, 1, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1735, 4089, 3, 1, 176, 10, 0, 1985, 4050, 1, 0, 727, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 4047, 3288, 0, 150, 1, 7, 14, 1, 1, 0, 1, 1, 3382, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3621, 6, 1, 0, 3, 0, 0, 3, 0, 2, 0, 1, 4094, 0, 3458, 4094, 2104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3462, 4063, 4085, 0, 2, 0, 4085, 3, 36, 1, 1, 3, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 6, 2, 4093, 0, 16, 773, 4, 4096, 1, 1, 1927, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15564, 16352, 16366, 16866, 16973, 17390, 16178, 15383] avg=16384.0000 max=17390 max_rank=5 min=15383 min_rank=7 imbalance_ratio=1.061401 max_load_reduction=113682 max_load_reduction_pct=0.867325 moved_to_echo=65924 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 852, 0, 0, 4073, 4028, 0, 10, 4094, 0, 0, 0, 1, 6, 2109, 632, 333, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 4092, 0, 14, 0, 4083, 0, 0, 126, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4015, 0, 15, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1025, 4084, 0, 0, 514, 0, 0, 1987, 3583, 0, 0, 348, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3604, 3453, 0, 25, 0, 0, 1, 0, 0, 0, 0, 0, 3385, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3385, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 2732, 4096, 2104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3464, 4055, 4079, 0, 0, 2, 4096, 0, 68, 0, 0, 0, 1297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 4096, 0, 0, 1949, 0, 4083, 0, 0, 1861, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16316, 16293, 16351, 15637, 16554, 16418, 17061, 16442] avg=16384.0000 max=17061 max_rank=6 min=15637 min_rank=3 imbalance_ratio=1.041321 max_load_reduction=114011 max_load_reduction_pct=0.869835 moved_to_echo=66475 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 561, 0, 0, 4096, 3298, 0, 0, 4095, 0, 0, 0, 0, 1, 2109, 632, 333, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 4090, 2, 12, 0, 4095, 0, 0, 3, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 4095, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2621, 4095, 0, 0, 816, 0, 0, 1987, 3986, 0, 6, 1226, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3916, 3583, 0, 58, 0, 1, 1, 0, 0, 0, 0, 0, 3385, 1989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 2643, 4096, 2104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3464, 4026, 4092, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 4096, 0, 2, 312, 0, 4095, 1, 1, 1713, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15303, 16160, 16355, 18832, 17026, 16829, 15874, 14693] avg=16384.0000 max=18832 max_rank=3 min=14693 min_rank=7 imbalance_ratio=1.149414 max_load_reduction=112240 max_load_reduction_pct=0.856323 moved_to_echo=64422 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 295, 0, 0, 4090, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 2109, 632, 333, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 4096, 1, 182, 0, 4095, 5, 0, 761, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4091, 0, 4086, 0, 0, 1, 0, 23, 0, 4094, 0, 3, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 4096, 0, 0, 667, 0, 2, 1987, 2158, 0, 0, 1389, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 3998, 2708, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 3385, 1991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 2045, 4096, 2105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3464, 4095, 3879, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 4096, 0, 9, 3275, 0, 4095, 0, 0, 2434, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15830, 17070, 16426, 14618, 16498, 16165, 16083, 18382] avg=16384.0000 max=18382 max_rank=7 min=14618 min_rank=3 imbalance_ratio=1.121948 max_load_reduction=112690 max_load_reduction_pct=0.859756 moved_to_echo=67128 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 923, 15, 0, 4096, 4084, 0, 0, 4095, 0, 0, 0, 0, 0, 2114, 633, 338, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3879, 4095, 0, 241, 0, 4094, 1, 0, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 4096, 0, 0, 1424, 0, 0, 1982, 3728, 0, 3, 1077, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3722, 2079, 0, 48, 0, 1, 0, 0, 0, 0, 0, 0, 3387, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 2499, 4096, 2104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3463, 4079, 3805, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 0, 4096, 0, 1, 1906, 0, 4095, 0, 0, 1570, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16482, 16398, 16362, 17480, 15323, 16688, 16205, 16134] avg=16384.0000 max=17480 max_rank=3 min=15323 min_rank=4 imbalance_ratio=1.066895 max_load_reduction=113592 max_load_reduction_pct=0.866638 moved_to_echo=64350 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1016, 1, 0, 4096, 4042, 0, 0, 4092, 1, 0, 0, 1, 6, 2109, 632, 333, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3868, 4092, 0, 107, 0, 4093, 0, 0, 4, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 1, 4094, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 962, 4093, 0, 0, 208, 24, 2, 1987, 4015, 0, 2, 855, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 3952, 2903, 0, 119, 0, 1, 6, 0, 0, 0, 0, 0, 3385, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2240, 4096, 2104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3464, 4096, 4042, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 4093, 0, 1, 3068, 0, 4088, 7, 1, 711, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16507, 16243, 16415, 16244, 16450, 16269, 16507, 16437] avg=16384.0000 max=16507 max_rank=0 min=16243 min_rank=1 imbalance_ratio=1.007507 max_load_reduction=114565 max_load_reduction_pct=0.874062 moved_to_echo=65663 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2575, 24, 0, 4092, 4066, 0, 0, 4095, 0, 0, 0, 0, 51, 2109, 632, 333, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 4086, 6, 92, 0, 4090, 0, 0, 58, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4093, 0, 0, 2, 0, 0, 0, 4076, 0, 2, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 4096, 1, 0, 1190, 11, 0, 1987, 3382, 0, 1, 255, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 3582, 2164, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 3385, 1991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 4096, 0, 2213, 4096, 2105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3464, 4091, 3737, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 1680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 4096, 0, 11, 3352, 0, 4091, 0, 0, 804, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18155, 16299, 16396, 15312, 15296, 15719, 17069, 16826] avg=16384.0000 max=18155 max_rank=0 min=15296 min_rank=4 imbalance_ratio=1.108093 max_load_reduction=112917 max_load_reduction_pct=0.861488 moved_to_echo=64910 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 128, 32768, 128, 32768, 0, 128, 128, 128, 128, 0, 32512, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 27776, 32640, 31872, 0, 128, 128, 32768, 128, 128, 128, 128, 128, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 7808, 128, 128, 32768, 31744, 128, 128, 32768, 128, 0, 128, 128, 128, 16896, 5120, 2688, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[9856, 32768, 128, 128, 5760, 128, 128, 16000, 26752, 128, 128, 7424, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 29312, 128, 128, 128, 128, 128, 128, 128, 0, 128, 0, 128, 32768, 0, 19456, 32768, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 0, 0, 0, 31104, 32768, 128, 896, 0, 32768, 128, 0, 1152, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 0, 30592, 23808, 0, 1024, 128, 128, 128, 128, 128, 0, 128, 128, 27136, 16000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[3072, 128, 128, 32768, 128, 128, 16128, 128, 32768, 128, 128, 13696, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 4069, 6, 0, 4078, 0, 0, 2947, 0, 0, 10, 0, 0, 3987, 0, 4096, 0, 0, 0, 4095, 886, 0, 4096, 0, 4096, 4096, 0, 0, 4095, 4096, 1994, 0, 3494, 0, 4096, 0, 0, 10, 4093, 0, 0, 4088, 4096, 0, 1, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 2466, 1906, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 4, 4096, 0, 0, 149, 0, 0, 45, 0, 4096, 2, 0, 0, 0, 3999, 0, 4090, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 39, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 90, 0, 3570, 0, 0, 3135] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1, 4093, 6, 0, 4096, 0, 0, 3489, 0, 0, 26, 0, 0, 3922, 0, 4096, 0, 0, 0, 4078, 523, 0, 4096, 0, 4096, 4096, 0, 0, 4093, 4093, 2614, 0, 3975, 8, 4096, 1, 0, 1, 4096, 0, 0, 4064, 4096, 0, 0, 0, 0, 4096, 1, 471, 0, 0, 0, 0, 2257, 592, 0, 0, 4088, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 54, 4095, 0, 0, 138, 0, 0, 82, 0, 4096, 20, 0, 0, 0, 4006, 0, 4076, 0, 13, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 102, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 166, 5, 3013, 0, 0, 3282] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[2, 4033, 0, 4, 4058, 0, 0, 3523, 0, 19, 12, 0, 0, 4041, 0, 4096, 2, 0, 0, 4075, 868, 0, 4094, 0, 4096, 4095, 2, 2, 4096, 4090, 1437, 1, 2658, 1, 4096, 0, 0, 3, 4093, 0, 0, 4072, 4095, 0, 7, 0, 0, 4089, 0, 24, 0, 2, 0, 1, 2526, 1801, 3, 0, 4094, 4093, 0, 0, 3, 2, 0, 1, 4091, 1, 1, 0, 0, 0, 4, 4094, 0, 0, 4096, 1, 0, 44, 4094, 0, 0, 868, 0, 1, 257, 0, 4093, 5, 0, 0, 0, 3964, 0, 4090, 0, 15, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 3, 0, 1, 4093, 25, 0, 0, 3, 1, 0, 1, 4094, 0, 0, 0, 236, 0, 2961, 0, 0, 3532] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 4094, 13, 0, 4095, 1, 1, 3792, 0, 0, 291, 0, 0, 3893, 0, 4096, 0, 0, 0, 4083, 230, 0, 4095, 1, 4095, 4095, 0, 1, 4095, 4096, 829, 0, 3611, 0, 4095, 2, 1, 9, 4092, 0, 0, 4045, 4095, 1, 0, 1, 0, 4094, 0, 446, 1, 0, 0, 0, 2659, 213, 0, 0, 4091, 4092, 0, 1, 1, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4095, 1, 0, 40, 4091, 0, 1, 651, 0, 0, 14, 0, 4095, 9, 0, 0, 0, 3982, 1, 4096, 0, 3, 0, 0, 1, 1, 4095, 1, 0, 0, 0, 1, 0, 1, 4095, 13, 1, 0, 0, 0, 0, 1, 4095, 1, 0, 1, 451, 0, 4007, 0, 0, 3589] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 4067, 1, 0, 4096, 0, 0, 3281, 0, 24, 74, 0, 0, 4040, 0, 4096, 0, 0, 0, 4085, 242, 0, 4096, 0, 4096, 4096, 0, 0, 4093, 4096, 1429, 0, 3562, 0, 4096, 0, 0, 1, 4096, 0, 0, 3887, 4096, 0, 5, 0, 0, 4096, 0, 1215, 0, 0, 0, 0, 2103, 289, 0, 0, 4095, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 4096, 0, 0, 135, 4092, 0, 0, 536, 0, 0, 182, 0, 4095, 12, 0, 0, 0, 3981, 0, 4096, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 272, 0, 4002, 0, 0, 3532] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 3950, 1, 0, 4096, 0, 0, 3630, 0, 4, 178, 0, 0, 3834, 0, 4096, 0, 0, 0, 4088, 350, 0, 4096, 0, 4096, 4096, 0, 0, 4094, 4096, 1989, 0, 3366, 0, 4096, 0, 0, 0, 4096, 0, 0, 3516, 4096, 0, 0, 0, 0, 4096, 0, 590, 0, 0, 0, 0, 2084, 785, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 200, 4096, 0, 0, 274, 0, 0, 86, 0, 4096, 18, 0, 0, 0, 3792, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 718, 0, 3981, 0, 0, 3522] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 4074, 1, 0, 4096, 0, 0, 3067, 0, 17, 51, 0, 0, 4002, 0, 4096, 0, 1, 0, 4094, 849, 0, 4096, 0, 4096, 4096, 0, 0, 4095, 4096, 3038, 0, 3448, 0, 4096, 0, 3, 25, 4087, 0, 0, 3985, 4096, 0, 0, 0, 0, 4095, 0, 233, 0, 0, 0, 0, 698, 858, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 71, 4096, 0, 0, 530, 0, 0, 2, 0, 4088, 3, 0, 0, 0, 3663, 0, 4092, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 105, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 375, 0, 3888, 0, 0, 3904] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[3, 4084, 0, 0, 4096, 0, 0, 3749, 0, 0, 0, 0, 0, 3930, 0, 4096, 0, 0, 0, 4093, 204, 0, 4096, 0, 4096, 4096, 0, 0, 4094, 4096, 2729, 0, 3419, 0, 4096, 0, 0, 18, 4078, 0, 0, 3796, 4096, 0, 2, 0, 0, 4096, 0, 1059, 0, 0, 0, 0, 481, 861, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 213, 4096, 0, 0, 530, 0, 0, 6, 0, 4096, 10, 0, 0, 0, 3812, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 90, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 432, 0, 3890, 0, 0, 3576] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4069, 6, 0, 4078, 0, 0, 2947, 0, 0, 10, 0, 0, 3987, 0, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 886, 0, 1581, 0, 0, 0, 0, 0, 4095, 4096, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3494, 0, 0, 0, 0, 10, 4093, 0, 0, 4088, 813, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2466, 1906, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 719, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 4, 2515, 1135, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3747, 0, 0, 149, 0, 0, 45, 0, 4096, 2, 0, 0, 0, 3999, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 39, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 90, 0, 3570, 0, 0, 3135, 2623, 2148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15851, 16747, 16594, 17433, 16289, 16128, 16368, 15662] avg=16384.0000 max=17433 max_rank=3 min=15662 min_rank=7 imbalance_ratio=1.064026 max_load_reduction=113639 max_load_reduction_pct=0.866997 moved_to_echo=64447 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 4033, 0, 4, 4058, 0, 0, 3523, 0, 19, 12, 0, 0, 4041, 0, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4075, 868, 0, 1581, 0, 0, 0, 2, 2, 4096, 4090, 1437, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2658, 1, 0, 0, 0, 3, 4093, 0, 0, 4072, 813, 0, 7, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 2, 0, 1, 2526, 1801, 3, 0, 4094, 4093, 0, 0, 3, 2, 4095, 719, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 1, 1, 0, 0, 0, 4, 4094, 0, 0, 4096, 1, 0, 44, 2513, 1135, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3745, 0, 0, 868, 0, 1, 257, 0, 4093, 5, 0, 0, 0, 3964, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 3, 0, 1, 4093, 25, 4096, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 1, 4094, 0, 0, 0, 236, 0, 2961, 0, 0, 3532, 2623, 2147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16446, 16154, 15736, 17415, 16330, 17023, 16370, 15598] avg=16384.0000 max=17415 max_rank=3 min=15598 min_rank=7 imbalance_ratio=1.062927 max_load_reduction=113657 max_load_reduction_pct=0.867134 moved_to_echo=65321 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 13, 0, 4095, 1, 1, 3792, 0, 0, 291, 0, 0, 3893, 0, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 230, 0, 1581, 1, 0, 0, 0, 1, 4095, 4096, 829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3611, 0, 0, 2, 1, 9, 4092, 0, 0, 4045, 813, 1, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 446, 1, 0, 0, 0, 2659, 213, 0, 0, 4091, 4092, 0, 1, 1, 0, 4095, 719, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4095, 1, 0, 40, 2514, 1135, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3742, 0, 1, 651, 0, 0, 14, 0, 4095, 9, 0, 0, 0, 3982, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 1, 4095, 1, 0, 0, 0, 1, 0, 1, 4095, 13, 4095, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 4095, 1, 0, 1, 451, 0, 4007, 0, 0, 3589, 2623, 2147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16934, 14916, 16669, 16371, 16327, 16591, 16348, 16916] avg=16384.0000 max=16934 max_rank=0 min=14916 min_rank=1 imbalance_ratio=1.033569 max_load_reduction=114138 max_load_reduction_pct=0.870804 moved_to_echo=66182 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 4093, 6, 0, 4096, 0, 0, 3489, 0, 0, 26, 0, 0, 3922, 0, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 523, 0, 1581, 0, 0, 0, 0, 0, 4093, 4093, 2614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 8, 0, 1, 0, 1, 4096, 0, 0, 4064, 813, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 471, 0, 0, 0, 0, 2257, 592, 0, 0, 4088, 4087, 0, 0, 0, 0, 4096, 719, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 54, 2515, 1135, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3746, 0, 0, 138, 0, 0, 82, 0, 4096, 20, 0, 0, 0, 4006, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 102, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 166, 5, 3013, 0, 0, 3282, 2623, 2148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16982, 17054, 16364, 16342, 16164, 16446, 15333] avg=16384.0000 max=17054 max_rank=2 min=15333 min_rank=7 imbalance_ratio=1.040894 max_load_reduction=114018 max_load_reduction_pct=0.869888 moved_to_echo=64285 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4067, 1, 0, 4096, 0, 0, 3281, 0, 24, 74, 0, 0, 4040, 0, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 242, 0, 1581, 0, 0, 0, 0, 0, 4093, 4096, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3562, 0, 0, 0, 0, 1, 4096, 0, 0, 3887, 813, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1215, 0, 0, 0, 0, 2103, 289, 0, 0, 4095, 4089, 0, 0, 0, 0, 4096, 719, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 4096, 0, 0, 135, 2515, 1135, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3743, 0, 0, 536, 0, 0, 182, 0, 4095, 12, 0, 0, 0, 3981, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 272, 0, 4002, 0, 0, 3532, 2623, 2148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16337, 15526, 16460, 16659, 16427, 16645, 16345, 16673] avg=16384.0000 max=16673 max_rank=7 min=15526 min_rank=1 imbalance_ratio=1.017639 max_load_reduction=114399 max_load_reduction_pct=0.872795 moved_to_echo=66090 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3950, 1, 0, 4096, 0, 0, 3630, 0, 4, 178, 0, 0, 3834, 0, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 350, 0, 1579, 0, 0, 0, 0, 0, 4094, 4096, 1989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3366, 0, 0, 0, 0, 0, 4096, 0, 0, 3516, 807, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 590, 0, 0, 0, 0, 2084, 785, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 726, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 200, 2517, 1139, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3742, 0, 0, 274, 0, 0, 86, 0, 4096, 18, 0, 0, 0, 3792, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 718, 0, 3981, 0, 0, 3522, 2624, 2150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16439, 16196, 15881, 16526, 16498, 16103, 16338, 17091] avg=16384.0000 max=17091 max_rank=7 min=15881 min_rank=2 imbalance_ratio=1.043152 max_load_reduction=113981 max_load_reduction_pct=0.869606 moved_to_echo=66030 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4074, 1, 0, 4096, 0, 0, 3067, 0, 17, 51, 0, 0, 4002, 0, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 849, 0, 1581, 0, 0, 0, 0, 0, 4095, 4096, 3038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3448, 0, 0, 0, 3, 25, 4087, 0, 0, 3985, 813, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 0, 698, 858, 0, 0, 4095, 4096, 0, 0, 0, 0, 4096, 719, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 71, 2515, 1135, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3747, 0, 0, 530, 0, 0, 2, 0, 4088, 3, 0, 0, 0, 3663, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 105, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 375, 0, 3888, 0, 0, 3904, 2623, 2148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16062, 17754, 16456, 14848, 16358, 16125, 16435, 17034] avg=16384.0000 max=17754 max_rank=1 min=14848 min_rank=3 imbalance_ratio=1.083618 max_load_reduction=113318 max_load_reduction_pct=0.864548 moved_to_echo=65952 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3, 4084, 0, 0, 4096, 0, 0, 3749, 0, 0, 0, 0, 0, 3930, 0, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 204, 0, 1581, 0, 0, 0, 0, 0, 4094, 4096, 2729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3419, 0, 0, 0, 0, 18, 4078, 0, 0, 3796, 813, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 0, 0, 0, 0, 481, 861, 0, 0, 4092, 4096, 0, 0, 0, 0, 4096, 719, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 213, 2515, 1135, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3747, 0, 0, 530, 0, 0, 6, 0, 4096, 10, 0, 0, 0, 3812, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 90, 4096, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 432, 0, 3890, 0, 0, 3576, 2623, 2148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16616, 16797, 16222, 15456, 16501, 16293, 16422, 16765] avg=16384.0000 max=16797 max_rank=1 min=15456 min_rank=3 imbalance_ratio=1.025208 max_load_reduction=114275 max_load_reduction_pct=0.871849 moved_to_echo=65981 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32512, 128, 128, 32768, 128, 128, 27520, 0, 128, 768, 0, 0, 31744, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[30080, 0, 128, 3712, 0, 128, 768, 0, 32768, 128, 0, 0, 0, 31232, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[27648, 128, 0, 128, 128, 128, 32768, 0, 0, 31488, 6528, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 4096, 128, 128, 0, 128, 15360, 7424, 128, 0, 32768, 32768, 0, 128, 128, 128, 32768, 5760, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 32768, 4224, 0, 12672, 128, 0, 0, 128, 128, 32768, 32768, 16128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 128, 128, 32768, 128, 0, 0, 0, 128, 0, 128, 32768, 512, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 0, 0, 128, 32768, 128, 0, 32768, 128, 128, 768, 20224, 9088, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 0, 128, 32768, 128, 0, 128, 2816, 128, 29312, 0, 0, 28160, 20992, 17280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 0, 0, 0, 2979, 1622, 4095, 1344, 0, 0, 0, 4095, 1, 4092, 0, 1, 4095, 0, 1, 0, 92, 0, 0, 0, 1, 0, 1544, 0, 0, 4095, 0, 0, 0, 1, 0, 4095, 0, 0, 4095, 4095, 4095, 2, 4095, 4095, 1, 4096, 4095, 4095, 1, 5, 4095, 0, 1993, 0, 1, 1, 0, 0, 0, 0, 4095, 1, 0, 1, 4095, 103, 1, 0, 4094, 4076, 1, 1, 1, 0, 0, 1, 1, 4095, 0, 4095, 4095, 1, 0, 0, 0, 4095, 2819, 0, 2605, 0, 2, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 1, 17, 0, 1, 1, 0, 2824, 0, 0, 1, 1, 2621, 0, 1, 0, 0, 1, 4095, 4083, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3620, 3376, 4096, 2348, 0, 0, 0, 4096, 0, 4096, 2, 0, 4096, 0, 0, 0, 192, 0, 0, 0, 0, 0, 374, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4093, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 1226, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 40, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3504, 0, 2148, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2882, 12, 0, 0, 0, 834, 0, 0, 0, 0, 0, 4096, 4059, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2945, 3245, 4096, 2516, 0, 0, 0, 4096, 0, 4096, 1, 0, 4096, 0, 0, 0, 25, 0, 0, 0, 0, 0, 165, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4093, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 1396, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 4096, 11, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 2391, 0, 3564, 0, 0, 0, 0, 0, 0, 0, 3779, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 1914, 0, 0, 0, 0, 2577, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2437, 1628, 4094, 3508, 0, 0, 0, 4096, 0, 4082, 1, 0, 4096, 0, 0, 4, 936, 0, 2, 6, 0, 0, 62, 0, 0, 4096, 0, 1, 0, 0, 0, 4094, 0, 0, 4096, 4093, 4095, 0, 3979, 4096, 0, 4096, 4096, 4095, 0, 2, 4096, 0, 2162, 0, 0, 0, 2, 3, 4, 0, 4096, 0, 0, 0, 4096, 111, 0, 0, 4095, 4067, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4093, 4092, 1, 0, 0, 0, 4096, 3259, 0, 1906, 0, 0, 0, 0, 0, 0, 3, 3984, 0, 0, 0, 3, 0, 0, 12, 0, 0, 0, 0, 1427, 8, 0, 0, 0, 3291, 0, 0, 0, 0, 0, 4094, 4090, 0, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2436, 3861, 4096, 3361, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 213, 0, 0, 0, 0, 0, 188, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4080, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 833, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 25, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3134, 0, 3926, 0, 0, 0, 0, 0, 0, 0, 3733, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 1850, 21, 0, 0, 0, 1035, 0, 0, 0, 0, 0, 4096, 4072, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1048, 3474, 4096, 1804, 0, 0, 0, 4096, 0, 4084, 0, 0, 4096, 0, 0, 0, 46, 0, 0, 0, 0, 0, 158, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4083, 4096, 0, 4079, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 2632, 0, 0, 0, 1, 3, 0, 0, 4096, 0, 0, 0, 4096, 21, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4091, 0, 0, 0, 0, 4096, 2953, 0, 3933, 0, 0, 0, 0, 0, 0, 0, 3865, 0, 0, 0, 3, 0, 0, 20, 0, 0, 0, 0, 3355, 22, 0, 0, 0, 1302, 0, 0, 0, 0, 0, 4096, 4094, 1, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2107, 1743, 4096, 2700, 0, 0, 0, 4096, 0, 4096, 3, 0, 4095, 0, 0, 0, 57, 0, 0, 0, 0, 0, 133, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 1678, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 402, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3984, 0, 2962, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 1, 0, 0, 18, 0, 0, 0, 0, 2076, 0, 0, 0, 0, 2668, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1662, 3573, 4096, 1177, 0, 0, 0, 4096, 0, 4096, 5, 0, 4096, 0, 0, 0, 547, 0, 0, 0, 0, 0, 288, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4075, 4096, 0, 4051, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 2398, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 188, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4004, 0, 2406, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1640, 0, 0, 0, 0, 2666, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 0, 0, 0, 2979, 1622, 4095, 1344, 0, 0, 0, 4095, 1, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 4095, 0, 1, 0, 92, 0, 0, 0, 1, 0, 1544, 0, 0, 4095, 2121, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 4095, 4095, 2, 4095, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2304, 4095, 1, 5, 4095, 0, 1993, 0, 1, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 103, 1, 0, 4094, 4076, 1, 1, 1, 0, 0, 1, 1, 2716, 1277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2007, 4095, 1, 0, 0, 0, 0, 2819, 0, 2605, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 1, 17, 0, 1, 1, 0, 2824, 4096, 4095, 1974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2621, 0, 1, 0, 0, 1, 4095, 4083, 0, 0, 4096, 0, 1164, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14766, 17390, 16415, 16590, 16369, 15624, 17044, 16874] avg=16384.0000 max=17390 max_rank=1 min=14766 min_rank=0 imbalance_ratio=1.061401 max_load_reduction=113682 max_load_reduction_pct=0.867325 moved_to_echo=65911 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2437, 1628, 4094, 3508, 0, 0, 0, 4096, 0, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 1, 0, 4096, 0, 0, 4, 936, 0, 2, 6, 0, 0, 62, 0, 0, 4096, 2121, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 4093, 4095, 0, 3979, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2305, 4095, 0, 2, 4096, 0, 2162, 0, 0, 0, 2, 3, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 111, 0, 0, 4095, 4067, 0, 0, 0, 0, 0, 0, 0, 2717, 1277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2006, 4092, 1, 0, 0, 0, 0, 3259, 0, 1906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3984, 0, 0, 0, 3, 0, 0, 12, 0, 0, 0, 0, 1427, 4096, 4096, 1975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 3291, 0, 0, 0, 0, 0, 4094, 4090, 0, 0, 4095, 0, 1164, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16754, 16293, 16765, 16363, 15359, 15596, 17552] avg=16384.0000 max=17552 max_rank=7 min=15359 min_rank=5 imbalance_ratio=1.071289 max_load_reduction=113520 max_load_reduction_pct=0.866089 moved_to_echo=64870 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3620, 3376, 4096, 2348, 0, 0, 0, 4096, 0, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 192, 0, 0, 0, 0, 0, 374, 0, 0, 4096, 2121, 1351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4093, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2295, 4096, 0, 0, 4096, 0, 1226, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 40, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 2722, 1281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2002, 4096, 0, 0, 0, 0, 0, 3504, 0, 2148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2882, 4096, 4096, 1975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 834, 0, 0, 0, 0, 0, 4096, 4059, 0, 0, 4096, 0, 1168, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18169, 16328, 16404, 15809, 16331, 15846, 17107, 15078] avg=16384.0000 max=18169 max_rank=0 min=15078 min_rank=7 imbalance_ratio=1.108948 max_load_reduction=112903 max_load_reduction_pct=0.861382 moved_to_echo=64362 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1048, 3474, 4096, 1804, 0, 0, 0, 4096, 0, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 4096, 0, 0, 0, 46, 0, 0, 0, 0, 0, 158, 0, 0, 4096, 2121, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4083, 4096, 0, 4079, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2305, 4096, 0, 0, 4096, 0, 2632, 0, 0, 0, 1, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 2717, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2007, 4091, 0, 0, 0, 0, 0, 2953, 0, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 0, 0, 0, 3, 0, 0, 20, 0, 0, 0, 0, 3355, 4096, 4096, 1975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 1302, 0, 0, 0, 0, 0, 4096, 4094, 1, 0, 4096, 0, 1164, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15145, 15949, 16385, 17229, 16288, 17080, 17410, 15586] avg=16384.0000 max=17410 max_rank=6 min=15145 min_rank=0 imbalance_ratio=1.062622 max_load_reduction=113662 max_load_reduction_pct=0.867172 moved_to_echo=66364 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2436, 3861, 4096, 3361, 0, 0, 0, 4096, 0, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 213, 0, 0, 0, 0, 0, 188, 0, 0, 4096, 2121, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4080, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2305, 4096, 0, 0, 4096, 0, 833, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 2717, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2007, 4096, 0, 0, 0, 0, 0, 3134, 0, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3733, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 1850, 4096, 4096, 1975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 1035, 0, 0, 0, 0, 0, 4096, 4072, 0, 0, 4096, 0, 1164, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18477, 16158, 16399, 15426, 16293, 17259, 15765, 15295] avg=16384.0000 max=18477 max_rank=0 min=15295 min_rank=7 imbalance_ratio=1.127747 max_load_reduction=112595 max_load_reduction_pct=0.859032 moved_to_echo=64612 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1662, 3573, 4096, 1177, 0, 0, 0, 4096, 0, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 4096, 0, 0, 0, 547, 0, 0, 0, 0, 0, 288, 0, 0, 4096, 2121, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4075, 4096, 0, 4051, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2305, 4096, 0, 0, 4096, 0, 2398, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 188, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 2717, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2007, 4096, 0, 0, 0, 0, 0, 4004, 0, 2406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1640, 4096, 4096, 1975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2666, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 1164, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15231, 16597, 16349, 16991, 16466, 16606, 15904, 16928] avg=16384.0000 max=16991 max_rank=3 min=15231 min_rank=0 imbalance_ratio=1.037048 max_load_reduction=114081 max_load_reduction_pct=0.870369 moved_to_echo=65904 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2107, 1743, 4096, 2700, 0, 0, 0, 4096, 0, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 4095, 0, 0, 0, 57, 0, 0, 0, 0, 0, 133, 0, 0, 4096, 2121, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2305, 4096, 0, 0, 4096, 0, 1678, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 402, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 2717, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2007, 4096, 0, 0, 0, 0, 0, 3984, 0, 2962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 1, 0, 0, 18, 0, 0, 0, 0, 2076, 4096, 4096, 1975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2668, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 4096, 0, 1164, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15369, 15949, 16415, 16271, 16682, 17144, 16314, 16928] avg=16384.0000 max=17144 max_rank=5 min=15369 min_rank=0 imbalance_ratio=1.046387 max_load_reduction=113928 max_load_reduction_pct=0.869202 moved_to_echo=67068 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 2945, 3245, 4096, 2516, 0, 0, 0, 4096, 0, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 25, 0, 0, 0, 0, 0, 165, 0, 0, 4096, 2120, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4093, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2305, 4096, 0, 0, 4096, 0, 1396, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 2717, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2007, 4096, 0, 0, 0, 0, 0, 2391, 0, 3564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3779, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 1914, 4096, 4096, 1976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2577, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 4096, 0, 1164, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17525, 15947, 16412, 15991, 16280, 16154, 15932, 16831] avg=16384.0000 max=17525 max_rank=0 min=15932 min_rank=6 imbalance_ratio=1.069641 max_load_reduction=113547 max_load_reduction_pct=0.866295 moved_to_echo=65197 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 32768, 0, 128, 128, 2176, 0, 128, 128, 128, 0, 2944, 0, 0, 32768, 17024, 10880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 0, 16128, 32768, 128, 0, 0, 0, 0, 26112, 0, 23552, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 31616, 0, 0, 0, 128, 0, 128, 256, 0, 128, 128, 0, 18048, 32768, 32768, 15872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 0, 0, 0, 19328, 22528, 32768, 18816, 0, 0, 0, 32768, 128, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 17024, 0, 128, 0, 0, 128, 32768, 32768, 128, 0, 32768, 0, 9344, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 18432, 32768, 128, 128, 32768, 0, 14336, 0, 128, 128, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 32768, 1024, 128, 0, 32768, 32768, 128, 128, 128, 0, 0, 128, 128, 21760, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 128, 0, 32768, 0, 0, 0, 32768, 32768, 128, 32640, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 145, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 3905, 0, 4085, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 887, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 8, 0, 4096, 0, 0, 0, 0, 3057, 4096, 1, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 31, 0, 0, 4096, 4096, 4096, 0, 4095, 0, 4096, 4095, 4096, 4094, 0, 0, 0, 0, 0, 3921, 0, 0, 0, 2927, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 4093, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1980, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 322, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 4095, 4095, 4092, 0, 0, 0, 4095, 0, 1, 0, 1, 3113, 4095, 1, 1, 1, 0, 5, 0, 0, 1, 4095, 1, 0, 0, 4095, 8, 0, 1, 4093, 4095, 4095, 0, 4096, 3, 4095, 4095, 4093, 4094, 0, 1, 0, 1, 0, 3956, 0, 1, 0, 3154, 1, 4094, 4095, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 4095, 1, 0, 0, 4092, 4067, 1, 1, 4095, 0, 0, 2, 0, 4095, 0, 1, 0, 1, 1, 4095, 1, 9, 1757, 0, 1, 0, 4095, 0, 4095, 0, 0, 1, 4095, 4095, 0, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 4091, 0, 11, 0, 4096, 0, 0, 0, 0, 3476, 4096, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 0, 0, 0, 4096, 7, 0, 0, 4096, 4096, 4096, 0, 4096, 2, 4096, 4096, 4066, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 1288, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 39, 4096, 4089, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 10, 2075, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 3, 0, 0, 0, 6, 104, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 4090, 0, 41, 0, 4096, 0, 0, 0, 0, 3095, 4092, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 4096, 31, 0, 0, 4096, 4096, 4096, 0, 4094, 5, 4096, 4096, 4068, 4096, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 2911, 0, 4096, 4077, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 29, 4095, 4069, 0, 0, 4096, 0, 0, 2, 2, 4093, 0, 0, 0, 0, 0, 4096, 0, 18, 2179, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2902, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 83, 0, 0, 4096, 4096, 4096, 0, 4095, 0, 4096, 4096, 4095, 4096, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 3564, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 24, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 967, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 4095, 0, 11, 0, 4096, 0, 0, 0, 0, 3594, 4081, 0, 0, 0, 0, 25, 0, 0, 0, 4096, 0, 0, 0, 4096, 40, 0, 0, 4096, 4096, 4096, 0, 4095, 0, 4096, 4096, 4087, 4096, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 2614, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 56, 4096, 4081, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 9, 792, 0, 0, 0, 4096, 2, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 3391, 4096, 0, 0, 3, 0, 12, 0, 0, 0, 4096, 0, 0, 0, 4096, 90, 0, 0, 4096, 4096, 4096, 0, 4096, 6, 4096, 4096, 4088, 4096, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 2004, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 21, 4091, 4075, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3, 1785, 0, 0, 0, 4096, 6, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 0, 1, 4096, 4096, 3460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 3865, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3086, 4096, 0, 0, 0, 0, 0, 0, 4096, 636, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 145, 0, 0, 4096, 0, 4042, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1382, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 3905, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2475, 1559, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 887, 0, 0, 0, 4096, 0, 2537, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15943, 16391, 16241, 16474, 17549, 16366, 16396, 15712] avg=16384.0000 max=17549 max_rank=4 min=15712 min_rank=7 imbalance_ratio=1.071106 max_load_reduction=113523 max_load_reduction_pct=0.866112 moved_to_echo=66023 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 322, 0, 1, 1, 1, 4095, 4095, 3458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 4095, 4095, 4092, 3864, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 1, 3113, 4095, 1, 1, 1, 0, 5, 0, 4095, 637, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4095, 8, 0, 1, 4093, 0, 4041, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1382, 0, 1, 0, 1, 0, 3956, 0, 1, 0, 3154, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 4095, 1, 0, 0, 4092, 2473, 1558, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 1, 1, 4095, 0, 0, 2, 0, 4095, 0, 1, 0, 1, 1, 4095, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1757, 0, 1, 0, 4095, 0, 2537, 0, 0, 1, 0, 4095, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16068, 16389, 16276, 16339, 16683, 16360, 16371, 16586] avg=16384.0000 max=16683 max_rank=4 min=16068 min_rank=0 imbalance_ratio=1.018250 max_load_reduction=114389 max_load_reduction_pct=0.872719 moved_to_echo=66000 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1351, 0, 0, 0, 0, 4096, 4096, 3460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 4091, 3865, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 3476, 4096, 0, 0, 0, 0, 10, 0, 4096, 636, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 4096, 0, 4042, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 1382, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 1288, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 39, 4096, 2475, 1559, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2075, 0, 0, 0, 4096, 0, 2537, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17096, 16367, 16652, 16339, 14916, 16404, 16388, 16910] avg=16384.0000 max=17096 max_rank=0 min=14916 min_rank=4 imbalance_ratio=1.043457 max_load_reduction=113976 max_load_reduction_pct=0.869568 moved_to_echo=64618 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 4096, 4096, 3460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 3864, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 3057, 4096, 1, 0, 1, 0, 0, 0, 4096, 636, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 31, 0, 0, 4096, 0, 4042, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1382, 0, 0, 0, 0, 0, 3921, 0, 0, 0, 2927, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 2473, 1559, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1980, 0, 0, 0, 4095, 0, 2537, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16116, 16391, 16222, 16360, 16422, 16365, 16392, 16804] avg=16384.0000 max=16804 max_rank=7 min=16116 min_rank=0 imbalance_ratio=1.025635 max_load_reduction=114268 max_load_reduction_pct=0.871796 moved_to_echo=65983 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 3, 0, 0, 0, 6, 104, 0, 0, 0, 0, 4096, 4096, 3459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 4090, 3864, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 4096, 0, 0, 0, 0, 3095, 4092, 0, 0, 0, 0, 2, 0, 4096, 637, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 31, 0, 0, 4096, 0, 4042, 0, 4094, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 1382, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 2911, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 29, 4095, 2475, 1559, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 4096, 0, 0, 2, 2, 4093, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2179, 0, 0, 0, 4096, 0, 2537, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15859, 16377, 16291, 16364, 16412, 16378, 16369, 17022] avg=16384.0000 max=17022 max_rank=7 min=15859 min_rank=0 imbalance_ratio=1.038940 max_load_reduction=114050 max_load_reduction_pct=0.870132 moved_to_echo=66181 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 0, 0, 0, 0, 4096, 4096, 3460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 4095, 3865, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 3594, 4081, 0, 0, 0, 0, 25, 0, 4096, 636, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 40, 0, 0, 4096, 0, 4042, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1382, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 2614, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 56, 4096, 2475, 1559, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 792, 0, 0, 0, 4096, 2, 2537, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16899, 16379, 16770, 16369, 16226, 16421, 16380, 15628] avg=16384.0000 max=16899 max_rank=0 min=15628 min_rank=7 imbalance_ratio=1.031433 max_load_reduction=114173 max_load_reduction_pct=0.871071 moved_to_echo=64655 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 675, 0, 0, 0, 0, 4096, 4096, 3460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 3865, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2902, 4096, 0, 0, 0, 0, 0, 0, 4096, 636, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 83, 0, 0, 4096, 0, 4031, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1376, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 3564, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 24, 4096, 2478, 1560, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 967, 0, 0, 0, 4096, 0, 2536, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16422, 16395, 16057, 16401, 17207, 16397, 16402, 15791] avg=16384.0000 max=17207 max_rank=4 min=15791 min_rank=7 imbalance_ratio=1.050232 max_load_reduction=113865 max_load_reduction_pct=0.868721 moved_to_echo=65797 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 927, 0, 0, 0, 0, 4096, 4096, 3459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 4093, 3864, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3391, 4096, 0, 0, 3, 0, 12, 0, 4096, 637, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 90, 0, 0, 4096, 0, 4042, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1382, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 2004, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 21, 4091, 2475, 1559, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1785, 0, 0, 0, 4096, 6, 2537, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16669, 16383, 16563, 16426, 15657, 16381, 16374, 16619] avg=16384.0000 max=16669 max_rank=0 min=15657 min_rank=4 imbalance_ratio=1.017395 max_load_reduction=114403 max_load_reduction_pct=0.872826 moved_to_echo=65031 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 32768, 0, 0, 128, 128, 32768, 0, 128, 0, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 32768, 0, 128, 0, 128, 25728, 32768, 128, 128, 128, 0, 128, 0, 32768, 5120, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 0, 32768, 512, 0, 128, 32768, 0, 32384, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 0, 0, 128, 0, 128, 128, 0, 0, 32768, 32768, 32768, 30976, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 0, 128, 0, 128, 128, 0, 0, 0, 128, 32768, 128, 128, 256, 32768, 19840, 12544, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 0, 0, 0, 0, 0, 128, 0, 0, 128, 128, 5120, 0, 128, 128, 128, 32768, 32768, 27776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 12544, 0, 128, 0, 32768, 128, 20352, 0, 0, 128, 0, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 32768, 11136, 0, 128, 0, 128, 0, 32256, 0, 128, 0, 22400, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 76, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3, 0, 0, 0, 4096, 4, 0, 4096, 11, 0, 4084, 0, 4096, 4081, 4095, 4096, 0, 3408, 6, 1, 0, 0, 4096, 2168, 209, 0, 7, 0, 4095, 4096, 4096, 6, 3228, 0, 3541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 501, 0, 0, 3, 0, 0, 0, 4090, 0, 0, 0, 4096, 3112, 0, 0, 3, 190, 0, 0, 0, 4089, 0, 0, 4096, 4096, 0, 0, 0, 36, 4096, 4084, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 1, 0, 4093, 0, 0, 4096, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4096, 6, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4064, 0, 4096, 4096, 4096, 4096, 0, 4014, 0, 0, 0, 0, 4096, 3813, 203, 0, 0, 0, 4096, 4096, 4096, 10, 1981, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 163, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2069, 0, 0, 15, 65, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4093, 0, 0, 4096, 0, 4078, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 51, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4089, 0, 4096, 4096, 4096, 4096, 0, 3758, 0, 0, 0, 0, 4096, 3473, 128, 0, 1, 0, 4096, 4096, 4096, 15, 2047, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 335, 0, 0, 5, 0, 0, 0, 4092, 0, 0, 0, 4096, 2484, 0, 0, 10, 56, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 1, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 56, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4095, 4096, 4096, 4096, 0, 4081, 0, 0, 0, 0, 4096, 1058, 185, 0, 0, 0, 4096, 4096, 4096, 0, 3642, 0, 3561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 612, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 3227, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4057, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4096, 17, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4095, 4096, 4096, 4096, 0, 3845, 0, 0, 0, 0, 4096, 3117, 18, 0, 0, 0, 4096, 4096, 4096, 0, 2338, 0, 3849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 60, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3165, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4064, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 60, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 53, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 4062, 0, 4095, 4096, 4096, 4096, 0, 4047, 0, 0, 0, 0, 4096, 3699, 60, 0, 0, 0, 4096, 4096, 4096, 2, 956, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 410, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 2916, 0, 0, 0, 163, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4062, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4095, 8, 1, 1, 1, 0, 4095, 1, 4095, 1, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 1, 4095, 0, 1, 4081, 1, 4095, 4095, 4095, 4095, 0, 3857, 0, 0, 1, 0, 4095, 3958, 310, 0, 2, 0, 4095, 4095, 4093, 0, 2072, 0, 3961, 1, 0, 0, 1, 1, 0, 0, 1, 0, 4095, 0, 0, 311, 1, 1, 33, 1, 1, 0, 4095, 1, 1, 1, 4095, 1854, 3, 0, 0, 12, 0, 0, 0, 4095, 0, 1, 4095, 4095, 1, 0, 1, 0, 4089, 4090, 1, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 4095, 0, 4089, 1, 0, 1, 7, 0, 0, 0, 4095, 0, 0, 4095, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 195, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 4096, 4096, 4096, 4096, 0, 2910, 0, 0, 0, 0, 4096, 3940, 432, 0, 0, 0, 4096, 4096, 4096, 4, 2240, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 25, 0, 0, 15, 0, 0, 0, 4093, 0, 0, 0, 4096, 2518, 0, 0, 4, 28, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4096, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 76, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2121, 1864, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3, 0, 0, 0, 4096, 4, 0, 4096, 11, 0, 4084, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4095, 0, 0, 3408, 6, 1, 0, 0, 0, 2168, 209, 0, 7, 0, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1975, 4096, 6, 3228, 0, 3541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 0, 0, 3, 0, 0, 0, 4090, 0, 0, 0, 4096, 3112, 0, 4096, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 190, 0, 0, 0, 4089, 0, 0, 4096, 4026, 0, 0, 0, 36, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 1, 0, 4093, 0, 0, 2977, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16405, 15081, 16939, 17017, 16536, 16376, 16338] avg=16384.0000 max=17017 max_rank=4 min=15081 min_rank=2 imbalance_ratio=1.038635 max_load_reduction=114055 max_load_reduction_pct=0.870171 moved_to_echo=66267 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 56, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 2121, 1864, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4095, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4081, 0, 0, 0, 0, 0, 1058, 185, 0, 0, 0, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1975, 4096, 0, 3642, 0, 3561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 612, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 3227, 0, 4096, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4026, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 2976, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4057, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16365, 16392, 14623, 17370, 17246, 16313, 16390, 16373] avg=16384.0000 max=17370 max_rank=3 min=14623 min_rank=2 imbalance_ratio=1.060181 max_load_reduction=113702 max_load_reduction_pct=0.867477 moved_to_echo=66322 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 51, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 18, 0, 0, 0, 0, 2121, 1864, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4089, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3758, 0, 0, 0, 0, 0, 3473, 128, 0, 1, 0, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1975, 4096, 15, 2047, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 0, 5, 0, 0, 0, 4092, 0, 0, 0, 4096, 2484, 0, 4096, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 56, 0, 0, 0, 4096, 0, 0, 4096, 4026, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 2976, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16376, 16386, 16659, 16283, 16228, 16381, 16374, 16385] avg=16384.0000 max=16659 max_rank=2 min=16228 min_rank=4 imbalance_ratio=1.016785 max_load_reduction=114413 max_load_reduction_pct=0.872902 moved_to_echo=65368 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 6, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 18, 0, 0, 0, 0, 2125, 1867, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4064, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4014, 0, 0, 0, 0, 0, 3813, 203, 0, 0, 0, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1971, 4096, 10, 1981, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2069, 0, 4096, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 65, 0, 0, 0, 4095, 0, 0, 4096, 4018, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4093, 0, 0, 2976, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4078, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16339, 16366, 17320, 16233, 15640, 16385, 16393, 16396] avg=16384.0000 max=17320 max_rank=2 min=15640 min_rank=4 imbalance_ratio=1.057129 max_load_reduction=113752 max_load_reduction_pct=0.867859 moved_to_echo=64814 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4095, 8, 1, 1, 1, 0, 4095, 1, 4095, 1, 0, 27, 0, 0, 0, 0, 2121, 1864, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 1, 4095, 0, 1, 4081, 1, 4095, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 3857, 0, 0, 1, 0, 0, 3958, 310, 0, 2, 0, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1974, 4093, 0, 2072, 0, 3961, 1, 0, 0, 1, 1, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 1, 1, 33, 1, 1, 0, 4095, 1, 1, 1, 4095, 1854, 3, 4095, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4095, 0, 1, 4095, 4025, 1, 0, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 2976, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4089, 1, 0, 1, 7, 0, 0, 0, 4095, 0, 0, 4095, 1, 1, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16344, 16379, 17424, 16199, 15612, 16319, 16383, 16412] avg=16384.0000 max=17424 max_rank=2 min=15612 min_rank=4 imbalance_ratio=1.063477 max_load_reduction=113648 max_load_reduction_pct=0.867065 moved_to_echo=64726 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 60, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 53, 0, 4, 0, 0, 2121, 1864, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 4062, 0, 4095, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4047, 0, 0, 0, 0, 0, 3699, 60, 0, 0, 0, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1975, 4096, 2, 956, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 2916, 0, 4096, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 0, 4096, 0, 0, 4096, 4026, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 2977, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4062, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16424, 16360, 17105, 15209, 16734, 16477, 16385, 16378] avg=16384.0000 max=17105 max_rank=2 min=15209 min_rank=3 imbalance_ratio=1.044006 max_load_reduction=113967 max_load_reduction_pct=0.869499 moved_to_echo=65974 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 17, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 2121, 1864, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4095, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3845, 0, 0, 0, 0, 0, 3117, 18, 0, 0, 0, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1975, 4096, 0, 2338, 0, 3849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3165, 0, 4096, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 4026, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 4096, 0, 0, 2976, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4064, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16326, 16392, 16279, 16354, 16633, 16315, 16393, 16380] avg=16384.0000 max=16633 max_rank=4 min=16279 min_rank=2 imbalance_ratio=1.015198 max_load_reduction=114439 max_load_reduction_pct=0.873100 moved_to_echo=65721 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 195, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 16, 0, 0, 0, 0, 2121, 1864, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2910, 0, 0, 0, 0, 0, 3940, 432, 0, 0, 0, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1975, 4096, 4, 2240, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 15, 0, 0, 0, 4093, 0, 0, 0, 4096, 2518, 0, 4096, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 28, 0, 0, 0, 4096, 0, 0, 4096, 4026, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 2977, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16518, 16392, 16581, 16485, 15962, 16346, 16378, 16410] avg=16384.0000 max=16581 max_rank=2 min=15962 min_rank=4 imbalance_ratio=1.012024 max_load_reduction=114491 max_load_reduction_pct=0.873497 moved_to_echo=65096 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 32640, 128, 0, 128, 128, 0, 0, 0, 32768, 0, 0, 32768, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[15872, 32768, 128, 18560, 0, 31232, 128, 0, 0, 128, 128, 0, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 0, 0, 128, 0, 0, 128, 0, 32768, 0, 128, 0, 32768, 0, 0, 23936, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 512, 128, 128, 128, 0, 32768, 128, 32768, 128, 0, 256, 0, 128, 0, 0, 17024, 14976, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 32768, 128, 128, 32768, 128, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 32768, 0, 0, 29952, 128, 128, 128, 0, 0, 25344, 1664, 0, 128, 0, 8960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 2432, 128, 128, 128, 128, 128, 0, 32768, 128, 128, 128, 32768, 21376, 128, 32768, 8960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 640, 0, 0, 0, 32768, 0, 128, 32768, 32256, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3256, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 4091, 0, 0, 4056, 4096, 0, 4096, 4076, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 2791, 622, 0, 0, 4094, 20, 0, 30, 4096, 0, 4, 2, 4096, 4096, 4096, 1741, 0, 4096, 0, 0, 0, 4096, 0, 52, 0, 0, 4044, 3880, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3263, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 4055, 4096, 0, 4096, 4093, 4096, 0, 0, 0, 0, 0, 0, 4091, 4096, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 2835, 180, 0, 0, 4096, 4, 0, 33, 4096, 0, 0, 0, 4096, 4096, 4096, 1712, 0, 4096, 0, 0, 0, 4096, 0, 276, 0, 0, 4094, 4020, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 1, 1, 1, 1, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 4095, 1, 0, 1, 1971, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 1, 4025, 0, 1, 0, 4095, 1, 0, 3988, 4095, 1, 4095, 4076, 4094, 0, 0, 1, 0, 0, 0, 4095, 4095, 58, 4093, 0, 1, 1, 4095, 0, 1, 0, 0, 4095, 4094, 0, 0, 3471, 1446, 2, 1, 4095, 0, 0, 6, 4095, 0, 0, 0, 4095, 4095, 4095, 1308, 1, 4095, 0, 0, 1, 4095, 1, 142, 0, 1, 4094, 4084, 4095, 0, 1, 0, 0, 1, 1, 0, 0, 3, 0, 1, 4089, 4095, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3007, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 2916, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4093, 0, 0, 3840, 788, 0, 0, 4096, 6, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 2113, 0, 4096, 0, 0, 0, 4096, 0, 20, 0, 0, 4094, 3695, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2282, 0, 0, 4095, 0, 0, 0, 0, 0, 5, 0, 0, 4058, 0, 0, 0, 4087, 1, 0, 3748, 4093, 0, 4096, 4072, 4096, 3, 0, 0, 0, 1, 0, 4081, 4096, 1, 4095, 0, 0, 0, 4096, 0, 0, 3, 0, 4093, 4095, 2, 0, 3015, 479, 0, 0, 4093, 57, 0, 44, 4096, 0, 4, 0, 4096, 4096, 4095, 3009, 0, 4096, 0, 3, 0, 4093, 0, 16, 0, 0, 4037, 3831, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 14, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1267, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 3909, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 27, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4095, 0, 0, 3738, 293, 0, 0, 4095, 2, 0, 28, 4096, 0, 0, 0, 4096, 4096, 4096, 2856, 0, 4096, 0, 0, 0, 4096, 0, 199, 0, 0, 4095, 4069, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2777, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 4096, 0, 0, 4078, 4096, 0, 4096, 4093, 4096, 0, 0, 0, 0, 0, 0, 4091, 4096, 3, 4096, 0, 4, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 3404, 471, 0, 0, 4096, 3, 0, 4, 4096, 0, 0, 0, 4096, 4096, 4096, 1207, 0, 4096, 0, 0, 0, 4096, 0, 624, 0, 0, 4075, 3956, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2958, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 4009, 4096, 0, 4096, 4094, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 8, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4088, 0, 0, 3024, 118, 0, 0, 4096, 6, 0, 30, 4096, 0, 0, 0, 4096, 4096, 4096, 2141, 0, 4096, 0, 0, 0, 4096, 0, 25, 0, 0, 4096, 4078, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3007, 0, 0, 4096, 0, 0, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 2916, 4096, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 0, 0, 4096, 4093, 0, 0, 3840, 788, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2063, 0, 0, 0, 0, 0, 4096, 2113, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 4094, 3695, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 246, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16369, 16793, 15493, 16388, 17191, 16464, 15989, 16385] avg=16384.0000 max=17191 max_rank=4 min=15493 min_rank=2 imbalance_ratio=1.049255 max_load_reduction=113881 max_load_reduction_pct=0.868843 moved_to_echo=66029 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 45, 4095, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3256, 0, 0, 4096, 0, 0, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 4091, 0, 0, 4056, 4096, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 0, 0, 4096, 4096, 0, 0, 2791, 622, 0, 0, 4094, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 2063, 0, 4, 2, 0, 0, 4096, 1741, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 4044, 3880, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 246, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16415, 17042, 16592, 16366, 15991, 16128, 16155, 16383] avg=16384.0000 max=17042 max_rank=1 min=15991 min_rank=4 imbalance_ratio=1.040161 max_load_reduction=114030 max_load_reduction_pct=0.869980 moved_to_echo=64657 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 1, 1, 1, 1, 4095, 0, 1, 0, 0, 0, 0, 0, 4093, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 4095, 1, 0, 1, 1971, 0, 0, 4095, 1, 1, 1497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4025, 0, 1, 0, 4095, 1, 0, 3988, 4095, 1, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4094, 0, 0, 1, 0, 0, 0, 4095, 0, 58, 4093, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 1, 0, 0, 4095, 4094, 0, 0, 3471, 1446, 2, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2063, 0, 0, 0, 0, 0, 4095, 1308, 1, 4095, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 1, 4094, 4084, 4095, 0, 1, 0, 0, 1, 1, 0, 0, 3, 0, 3823, 246, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 4095, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16370, 15760, 16496, 16419, 17477, 15665, 16505, 16380] avg=16384.0000 max=17477 max_rank=4 min=15665 min_rank=5 imbalance_ratio=1.066711 max_load_reduction=113595 max_load_reduction_pct=0.866661 moved_to_echo=66027 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 36, 4094, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2282, 0, 0, 4095, 0, 0, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4058, 0, 0, 0, 4087, 1, 0, 3748, 4093, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 4096, 3, 0, 0, 0, 1, 0, 4081, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 3, 0, 4093, 4095, 2, 0, 3015, 479, 0, 0, 4093, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 2063, 0, 4, 0, 0, 0, 4095, 3009, 0, 4096, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4037, 3831, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 14, 0, 3824, 246, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16393, 16067, 16281, 16349, 16109, 17407, 16081, 16385] avg=16384.0000 max=17407 max_rank=5 min=16067 min_rank=1 imbalance_ratio=1.062439 max_load_reduction=113665 max_load_reduction_pct=0.867195 moved_to_echo=65982 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 20, 4095, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3263, 0, 0, 4096, 0, 0, 1499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 4055, 4096, 0, 292, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4091, 0, 0, 0, 0, 0, 0, 4091, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 268, 0, 0, 0, 0, 4096, 4096, 0, 0, 2835, 180, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 2056, 0, 0, 0, 0, 0, 4096, 1712, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 0, 0, 4094, 4020, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3828, 249, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 17050, 16633, 16372, 15575, 16089, 16579, 16385] avg=16384.0000 max=17050 max_rank=1 min=15575 min_rank=4 imbalance_ratio=1.040649 max_load_reduction=114022 max_load_reduction_pct=0.869919 moved_to_echo=64628 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1267, 0, 0, 4096, 0, 0, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 3909, 4096, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 0, 0, 4096, 4095, 0, 0, 3738, 293, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 2063, 0, 0, 0, 0, 0, 4096, 2856, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 0, 0, 4095, 4069, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 246, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16370, 15053, 16485, 16411, 16591, 17235, 16542, 16385] avg=16384.0000 max=17235 max_rank=5 min=15053 min_rank=1 imbalance_ratio=1.051941 max_load_reduction=113837 max_load_reduction_pct=0.868507 moved_to_echo=66753 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 24, 4094, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2777, 0, 0, 4096, 0, 0, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 4096, 0, 0, 4078, 4096, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 3, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 0, 0, 4096, 4096, 0, 0, 3404, 471, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2063, 0, 0, 0, 0, 0, 4096, 1207, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 624, 0, 0, 4075, 3956, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3824, 246, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 16563, 16515, 16383, 16438, 15562, 16836, 16383] avg=16384.0000 max=16836 max_rank=6 min=15562 min_rank=5 imbalance_ratio=1.027588 max_load_reduction=114236 max_load_reduction_pct=0.871552 moved_to_echo=65219 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 4094, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2958, 0, 0, 4096, 0, 0, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 4009, 4096, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 0, 0, 4096, 4088, 0, 0, 3024, 118, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 2063, 0, 0, 0, 0, 0, 4096, 2141, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 4096, 4078, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3824, 246, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16374, 16744, 16577, 16384, 15700, 16522, 16385, 16386] avg=16384.0000 max=16744 max_rank=1 min=15700 min_rank=4 imbalance_ratio=1.021973 max_load_reduction=114328 max_load_reduction_pct=0.872253 moved_to_echo=64993 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 32768, 32768, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 32768, 128, 128, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 256, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1408, 0, 128, 32640, 31616, 32768, 0, 128, 0, 128, 128, 128, 0, 0, 128, 0, 30720, 2048, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 32768, 128, 0, 128, 0, 128, 0, 32768, 0, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[2176, 0, 128, 128, 0, 32768, 32768, 128, 0, 26240, 4480, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[256, 16512, 0, 128, 128, 0, 0, 32768, 16128, 128, 32768, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 128, 0, 128, 32512, 0, 128, 0, 32768, 128, 0, 30848, 32768, 128, 2432, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 32768, 128, 128, 32768, 128, 0, 128, 20864, 0, 0, 32768, 128, 128, 12032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4056, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4093, 0, 4, 0, 0, 1, 0, 4088, 3864, 0, 0, 0, 0, 4096, 0, 4093, 7, 0, 4093, 0, 0, 1, 0, 4092, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4077, 0, 4089, 0, 0, 0, 0, 4093, 0, 4096, 2, 0, 0, 0, 319, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 7, 0, 35, 0, 4096, 4096, 6, 3986, 0, 0, 0, 0, 3, 0, 29, 0, 0, 1, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 4065, 4096, 34, 4092, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4070, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2757, 0, 0, 0, 0, 4096, 0, 4096, 64, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4071, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1126, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 6, 0, 252, 0, 4096, 4096, 0, 4088, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4026, 4096, 20, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3960, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3042, 0, 0, 0, 0, 4096, 0, 4096, 58, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3615, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1538, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 35, 0, 10, 0, 4096, 4096, 0, 4074, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4052, 4096, 105, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[3633, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3805, 0, 0, 0, 0, 4096, 0, 4096, 209, 0, 4096, 0, 1, 0, 0, 4062, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3983, 0, 4093, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1196, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 9, 0, 8, 0, 4096, 4096, 2, 4038, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3617, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4026, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2813, 0, 0, 0, 0, 4096, 0, 4096, 71, 0, 4095, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4006, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1260, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 24, 0, 228, 0, 4096, 4096, 1, 3972, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4055, 4096, 4, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4059, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3818, 0, 0, 0, 0, 4096, 0, 4096, 41, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3968, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 653, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 10, 0, 5, 0, 4095, 4096, 0, 3852, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4069, 4096, 2, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[3991, 1, 0, 0, 0, 1, 4094, 0, 1, 1, 0, 1, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 4095, 22, 0, 0, 0, 4095, 1, 3, 0, 0, 0, 1, 4095, 3026, 3, 0, 0, 0, 4095, 0, 4095, 64, 0, 4095, 1, 2, 0, 1, 4095, 0, 0, 1, 0, 4095, 0, 0, 1, 0, 0, 1, 4095, 4095, 1, 1, 4088, 1, 4072, 1, 1, 0, 0, 4095, 1, 4095, 0, 1, 1, 0, 983, 4095, 0, 4095, 4095, 1, 4095, 1, 1, 0, 1, 181, 0, 4092, 4095, 0, 4061, 1, 1, 1, 1, 0, 2, 0, 0, 1, 0, 1, 4088, 0, 4095, 1, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 4094, 0, 4084, 4094, 1, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[3859, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3817, 0, 0, 0, 0, 4096, 0, 4096, 32, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3983, 0, 4090, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 953, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 164, 0, 15, 0, 4096, 4096, 0, 3933, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3708, 4096, 3, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4070, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3335, 876, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2757, 0, 0, 0, 0, 662, 0, 4096, 64, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 4095, 0, 0, 0, 0, 4096, 0, 3220, 0, 0, 0, 0, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 6, 0, 252, 0, 4096, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 3979, 99, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 4096, 0, 4026, 4096, 20, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16495, 16378, 15770, 16389, 16608, 16606, 16375, 16451] avg=16384.0000 max=16608 max_rank=4 min=15770 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=114464 max_load_reduction_pct=0.873291 moved_to_echo=66040 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4056, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 3335, 876, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4093, 0, 4, 0, 0, 1, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3864, 0, 0, 0, 0, 662, 0, 4093, 7, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4089, 0, 0, 0, 0, 4093, 0, 3220, 2, 0, 0, 0, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 7, 0, 35, 0, 4096, 4096, 6, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 29, 0, 0, 1, 0, 4096, 0, 4095, 0, 0, 4096, 3979, 99, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 117, 0, 0, 4096, 0, 4065, 4096, 34, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16485, 16388, 16808, 16390, 15800, 16294, 16406, 16501] avg=16384.0000 max=16808 max_rank=2 min=15800 min_rank=4 imbalance_ratio=1.025879 max_load_reduction=114264 max_load_reduction_pct=0.871765 moved_to_echo=65001 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3633, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3335, 876, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3805, 0, 0, 0, 0, 662, 0, 4096, 209, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 0, 4093, 0, 0, 0, 0, 4096, 0, 3220, 0, 0, 0, 0, 1196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 9, 0, 8, 0, 4096, 4096, 2, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 3979, 99, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 4096, 0, 3617, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16058, 16376, 16965, 16356, 16588, 16317, 16390, 16022] avg=16384.0000 max=16965 max_rank=2 min=16022 min_rank=7 imbalance_ratio=1.035461 max_load_reduction=114107 max_load_reduction_pct=0.870567 moved_to_echo=65317 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3960, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3337, 883, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3042, 0, 0, 0, 0, 659, 0, 4096, 58, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3615, 0, 4096, 0, 0, 0, 0, 4096, 0, 3213, 0, 0, 0, 0, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 35, 0, 10, 0, 4096, 4096, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 3985, 100, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 4096, 0, 4052, 4096, 105, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 16377, 16047, 16384, 16558, 16372, 16381, 16556] avg=16384.0000 max=16558 max_rank=4 min=16047 min_rank=2 imbalance_ratio=1.010620 max_load_reduction=114514 max_load_reduction_pct=0.873672 moved_to_echo=65867 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4026, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3335, 876, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2813, 0, 0, 0, 0, 662, 0, 4096, 71, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 4095, 0, 0, 0, 0, 4096, 0, 3220, 0, 0, 0, 0, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 24, 0, 228, 0, 4096, 4096, 1, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 3979, 99, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 4096, 0, 4055, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16451, 16385, 15833, 16383, 16677, 16485, 16394, 16464] avg=16384.0000 max=16677 max_rank=4 min=15833 min_rank=2 imbalance_ratio=1.017883 max_load_reduction=114395 max_load_reduction_pct=0.872765 moved_to_echo=66020 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3991, 1, 0, 0, 0, 1, 4094, 0, 1, 1, 0, 1, 1, 0, 4094, 0, 3334, 876, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 22, 0, 0, 0, 4095, 1, 3, 0, 0, 0, 4095, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 3026, 3, 0, 0, 0, 662, 0, 4095, 64, 0, 4095, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 1, 0, 4095, 0, 0, 1, 0, 0, 1, 4095, 4095, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 1, 4072, 1, 1, 0, 0, 4095, 1, 3219, 0, 1, 1, 0, 983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 1, 0, 1, 1, 0, 1, 181, 0, 4092, 4095, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 2, 0, 0, 1, 0, 1, 4088, 0, 4095, 1, 0, 4095, 3978, 99, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 117, 0, 0, 4094, 0, 4084, 4094, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16417, 16398, 16044, 16391, 16464, 16500, 16372, 16486] avg=16384.0000 max=16500 max_rank=5 min=16044 min_rank=2 imbalance_ratio=1.007080 max_load_reduction=114572 max_load_reduction_pct=0.874115 moved_to_echo=65822 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4059, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 3335, 876, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3818, 0, 0, 0, 0, 662, 0, 4096, 41, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 4095, 0, 0, 0, 0, 4096, 0, 3220, 1, 0, 0, 0, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 10, 0, 5, 0, 4095, 4096, 0, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 3979, 99, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 4096, 0, 4069, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16485, 16382, 16806, 16390, 16033, 16126, 16374, 16476] avg=16384.0000 max=16806 max_rank=2 min=16033 min_rank=4 imbalance_ratio=1.025757 max_load_reduction=114266 max_load_reduction_pct=0.871780 moved_to_echo=65009 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3859, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3335, 876, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3817, 0, 0, 0, 0, 662, 0, 4096, 32, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 0, 4090, 0, 0, 0, 0, 4096, 0, 3220, 2, 0, 0, 0, 953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 164, 0, 15, 0, 4096, 4096, 0, 3933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 3979, 99, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 4096, 0, 3708, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16284, 16388, 16799, 16389, 16344, 16372, 16380, 16116] avg=16384.0000 max=16799 max_rank=2 min=16116 min_rank=7 imbalance_ratio=1.025330 max_load_reduction=114273 max_load_reduction_pct=0.871834 moved_to_echo=65212 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32768, 0, 0, 128, 128, 32768, 0, 0, 128, 0, 0, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[31744, 128, 0, 0, 0, 128, 32768, 128, 128, 128, 128, 128, 128, 0, 32768, 0, 26752, 7040, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 32768, 128, 0, 0, 0, 32768, 128, 128, 0, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32768, 27008, 128, 0, 0, 0, 5376, 0, 32768, 640, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 32640, 0, 128, 0, 128, 128, 256, 128, 768, 0, 32768, 32768, 128, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 31872, 128, 32768, 128, 128, 0, 0, 32768, 128, 25856, 128, 128, 128, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 32768, 0, 32768, 128, 0, 32768, 31872, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 0, 1024, 0, 0, 32768, 0, 31744, 32768, 256, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 675, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 74, 0, 3961, 0, 0, 0, 0, 0, 4058, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3900, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 3711, 0, 4096, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 635, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 843, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 3290, 0, 4096, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 3440, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 8, 0, 1583, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 970, 0, 4033, 0, 0, 0, 0, 0, 3986, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 2900, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4079, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 2923, 0, 4094, 0, 14] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1197, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 923, 0, 4074, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2605, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 1, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 3493, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3, 22, 0, 2297, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 4096, 0, 0, 123, 0, 3606, 0, 0, 2, 0, 0, 4041, 0, 0, 4092, 0, 1, 5, 0, 0, 0, 0, 4067, 0, 0, 3252, 0, 4096, 4093, 0, 0, 4084, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 4095, 4096, 2, 4095, 0, 6, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 4096, 0, 0, 0, 4096, 0, 4091, 4096, 9, 4, 4076, 0, 0, 4096, 0, 5, 4096, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 4096, 4093, 4096, 6, 4096, 3052, 0, 4087, 0, 13] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 21, 0, 1857, 2, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 553, 0, 4071, 0, 0, 0, 0, 0, 4076, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4062, 0, 0, 3048, 0, 4096, 4096, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4075, 0, 0, 4096, 0, 12, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 1, 4096, 2800, 0, 4093, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 1723, 5, 1, 4095, 4095, 1, 0, 1, 0, 0, 4095, 1, 1, 0, 1, 4095, 0, 0, 918, 1, 4084, 1, 0, 0, 1, 1, 4094, 1, 0, 4095, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 0, 2654, 0, 4095, 4095, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 1, 0, 0, 1, 1, 4095, 4094, 0, 4095, 0, 0, 1, 0, 0, 0, 1, 1, 0, 6, 0, 0, 4091, 1, 0, 1, 4095, 0, 4095, 3936, 0, 0, 4096, 0, 1, 4095, 0, 0, 4093, 1, 4095, 0, 0, 0, 4095, 1, 0, 1, 4095, 4095, 4095, 0, 4095, 3063, 0, 4095, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 4, 0, 2622, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 489, 0, 3982, 0, 0, 0, 0, 0, 4051, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 2032, 0, 4096, 4096, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4092, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 3197, 0, 4093, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1197, 0, 0, 4096, 4071, 2531, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3485, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 2945, 680, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2605, 0, 1151, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 885, 3493, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15998, 16696, 16497, 16044, 16384, 16377, 16410, 16666] avg=16384.0000 max=16696 max_rank=1 min=15998 min_rank=0 imbalance_ratio=1.019043 max_load_reduction=114376 max_load_reduction_pct=0.872620 moved_to_echo=65837 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 675, 0, 0, 4096, 4071, 2531, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3485, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 0, 0, 0, 0, 0, 4058, 0, 0, 4096, 0, 0, 0, 0, 0, 2945, 680, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3900, 0, 1151, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 885, 3711, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15476, 15847, 16351, 17339, 16384, 16380, 16410, 16885] avg=16384.0000 max=17339 max_rank=3 min=15476 min_rank=0 imbalance_ratio=1.058289 max_load_reduction=113733 max_load_reduction_pct=0.867714 moved_to_echo=66059 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 1723, 5, 1, 4095, 4070, 2531, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3484, 4095, 1, 0, 1, 0, 0, 4095, 1, 1, 0, 1, 4095, 0, 0, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4084, 1, 0, 0, 1, 1, 4094, 1, 0, 4095, 0, 0, 0, 0, 1, 2944, 679, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 2654, 0, 1151, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 1, 0, 0, 1, 1, 4095, 4094, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 6, 0, 0, 4091, 1, 0, 1, 4095, 4095, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3936, 0, 0, 4096, 0, 1, 0, 0, 0, 4093, 1, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4095, 4095, 0, 885, 3063, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16528, 16692, 16513, 16091, 16385, 16381, 16247, 16235] avg=16384.0000 max=16692 max_rank=1 min=16091 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=114380 max_load_reduction_pct=0.872650 moved_to_echo=65248 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3, 22, 0, 2297, 0, 0, 4096, 4071, 2531, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3485, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 4096, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3606, 0, 0, 2, 0, 0, 4041, 0, 0, 4092, 0, 1, 5, 0, 0, 2945, 680, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 3252, 0, 1151, 4093, 0, 0, 4084, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 4095, 4096, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 9, 4, 4076, 0, 0, 0, 0, 5, 4096, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4093, 4096, 6, 885, 3052, 0, 4087, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17120, 15893, 15983, 16648, 16389, 16403, 16402, 16234] avg=16384.0000 max=17120 max_rank=0 min=15893 min_rank=1 imbalance_ratio=1.044922 max_load_reduction=113952 max_load_reduction_pct=0.869385 moved_to_echo=65428 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 21, 0, 1857, 2, 0, 4096, 4071, 2531, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3485, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 4076, 0, 0, 4096, 0, 2, 0, 0, 0, 2945, 680, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 3048, 0, 1151, 4096, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4075, 0, 0, 0, 0, 12, 4096, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 885, 2800, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16681, 16326, 16481, 16455, 16381, 16377, 16400, 15971] avg=16384.0000 max=16681 max_rank=0 min=15971 min_rank=7 imbalance_ratio=1.018127 max_load_reduction=114391 max_load_reduction_pct=0.872734 moved_to_echo=65129 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 8, 0, 1583, 0, 0, 4096, 4071, 2531, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3485, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 0, 0, 3986, 0, 0, 4096, 0, 0, 0, 0, 0, 2945, 680, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 2900, 0, 1151, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4079, 0, 0, 0, 0, 0, 4096, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 885, 2923, 0, 4094, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 16743, 16351, 16318, 16384, 16384, 16392, 16108] avg=16384.0000 max=16743 max_rank=1 min=16108 min_rank=7 imbalance_ratio=1.021912 max_load_reduction=114329 max_load_reduction_pct=0.872261 moved_to_echo=65268 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 635, 0, 0, 4096, 4074, 2537, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3482, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 2949, 681, 614, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 3290, 0, 1147, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 878, 3440, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15446, 16613, 16531, 16705, 16382, 16383, 16406, 16606] avg=16384.0000 max=16705 max_rank=3 min=15446 min_rank=0 imbalance_ratio=1.019592 max_load_reduction=114367 max_load_reduction_pct=0.872551 moved_to_echo=65777 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 4, 0, 2622, 0, 0, 4096, 4071, 2531, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3485, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 0, 0, 0, 0, 0, 4051, 0, 0, 4096, 0, 0, 0, 0, 0, 2945, 680, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 2032, 0, 1151, 4096, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 885, 3197, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17431, 16262, 16365, 15472, 16383, 16387, 16405, 16367] avg=16384.0000 max=17431 max_rank=0 min=15472 min_rank=3 imbalance_ratio=1.063904 max_load_reduction=113641 max_load_reduction_pct=0.867012 moved_to_echo=65542 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[27904, 32768, 128, 0, 128, 0, 0, 32768, 128, 128, 128, 128, 32768, 0, 0, 4992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 32768, 32640, 128, 128, 32768, 128, 128, 0, 0, 128, 32768, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 32000, 128, 0, 128, 128, 128, 32512, 128, 0, 32768, 0, 128, 128, 0, 128, 23680, 5504, 4992, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 0, 128, 0, 32768, 32768, 128, 7168, 25728, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 0, 128, 32768, 128, 0, 0, 0, 128, 128, 0, 12672, 128, 128, 32768, 32640, 20352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 0, 0, 23808, 0, 9216, 32768, 0, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 32768, 128, 0, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 0, 128, 128, 0, 128, 128, 32768, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[3973, 0, 0, 4096, 0, 0, 0, 4095, 3717, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4072, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 90, 0, 4066, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[3763, 0, 0, 4096, 0, 0, 0, 4096, 3117, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 941, 4096, 0, 0, 0, 0, 0, 4093, 4096, 2, 4, 4096, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4084, 4096, 0, 6, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 4089, 4095, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 387, 0, 4094, 0, 0, 0, 5, 4096, 0, 10, 4096, 0, 0, 0, 0, 4096, 4, 0, 0, 4096, 0, 0, 0, 4096, 0, 4083, 4096, 0, 0, 4096, 0, 4093, 0, 0, 4094, 1, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[2569, 0, 0, 4096, 0, 0, 0, 4096, 2581, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 724, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2338, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4075, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[3214, 0, 0, 4096, 0, 0, 0, 4096, 3257, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1203, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4092, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 531, 0, 4095, 0, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4085, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3472, 0, 0, 4096, 0, 0, 0, 4094, 1863, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 824, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2035, 0, 4095, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1999, 0, 0, 4096, 0, 0, 0, 4096, 3343, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2115, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 752, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[3768, 1, 8, 4095, 1, 0, 1, 4055, 3721, 0, 0, 4094, 1, 0, 1, 2, 0, 0, 3, 13, 1, 214, 4095, 2, 0, 1, 0, 0, 4095, 4089, 0, 0, 4094, 0, 0, 0, 4095, 4095, 0, 1, 1, 1, 0, 1, 4, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 4095, 4095, 1, 0, 4084, 0, 1, 1, 4095, 0, 0, 0, 0, 0, 4091, 4095, 0, 1, 0, 0, 1, 4093, 0, 0, 1, 1, 0, 1, 4095, 4095, 0, 0, 828, 0, 3767, 0, 0, 1, 0, 4094, 0, 0, 4095, 1, 1, 0, 1, 4092, 0, 1, 0, 4095, 1, 0, 2, 4095, 0, 4095, 4095, 5, 20, 4095, 0, 4095, 0, 1, 4095, 0, 4089, 1, 1, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[3716, 0, 0, 4096, 0, 0, 0, 4096, 1822, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1637, 4096, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1020, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3973, 0, 0, 1960, 0, 0, 0, 4095, 3717, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 2125, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4084, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 1, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3144, 4096, 0, 0, 90, 0, 4066, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17841, 15831, 16384, 16386, 16365, 15492, 16383, 16390] avg=16384.0000 max=17841 max_rank=0 min=15492 min_rank=5 imbalance_ratio=1.088928 max_load_reduction=113231 max_load_reduction_pct=0.863884 moved_to_echo=64630 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3763, 0, 0, 1960, 0, 0, 0, 4096, 3117, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 941, 4096, 0, 0, 0, 0, 0, 4093, 4096, 2, 4, 2125, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4084, 4096, 0, 6, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4089, 4095, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3144, 4090, 0, 0, 387, 0, 4094, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 0, 4096, 4, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 6, 0, 0, 0, 0, 4093, 0, 0, 4094, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17032, 16309, 16377, 16382, 16386, 15816, 16397, 16373] avg=16384.0000 max=17032 max_rank=0 min=15816 min_rank=5 imbalance_ratio=1.039551 max_load_reduction=114040 max_load_reduction_pct=0.870056 moved_to_echo=64972 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3214, 0, 0, 1960, 0, 0, 0, 4096, 3257, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1203, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2125, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4092, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3144, 4096, 0, 0, 531, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 6, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16624, 16568, 16382, 16382, 16388, 15962, 16387, 16379] avg=16384.0000 max=16624 max_rank=0 min=15962 min_rank=5 imbalance_ratio=1.014648 max_load_reduction=114448 max_load_reduction_pct=0.873169 moved_to_echo=65116 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2569, 0, 0, 1960, 0, 0, 0, 4096, 2581, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 724, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2125, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3144, 4096, 0, 0, 2338, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 6, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15303, 16089, 16383, 16385, 16390, 17770, 16383, 16369] avg=16384.0000 max=17770 max_rank=5 min=15303 min_rank=0 imbalance_ratio=1.084595 max_load_reduction=113302 max_load_reduction_pct=0.864426 moved_to_echo=66912 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3768, 1, 8, 1959, 1, 0, 1, 4055, 3721, 0, 0, 4094, 1, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 13, 1, 214, 4095, 2, 0, 1, 0, 0, 4095, 4089, 0, 0, 2125, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4095, 4095, 0, 1, 1, 1, 0, 1, 4, 0, 1, 1, 4083, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 4095, 4095, 1, 0, 4084, 0, 1, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 0, 1, 0, 0, 1, 4093, 0, 0, 1, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3144, 4095, 0, 0, 828, 0, 3767, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 1, 4092, 0, 1, 0, 4094, 1, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 5, 20, 0, 0, 4095, 0, 1, 4095, 0, 4089, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17612, 15589, 16388, 16376, 16382, 15931, 16383, 16411] avg=16384.0000 max=17612 max_rank=0 min=15589 min_rank=1 imbalance_ratio=1.074951 max_load_reduction=113460 max_load_reduction_pct=0.865631 moved_to_echo=65107 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3472, 0, 0, 1960, 0, 0, 0, 4094, 1863, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 824, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2125, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3144, 4096, 0, 0, 2035, 0, 4095, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15485, 16189, 16383, 16385, 16389, 17467, 16384, 16390] avg=16384.0000 max=17467 max_rank=5 min=15485 min_rank=0 imbalance_ratio=1.066101 max_load_reduction=113605 max_load_reduction_pct=0.866737 moved_to_echo=66630 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1999, 0, 0, 1949, 0, 0, 0, 4096, 3343, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2115, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2132, 953, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3143, 4096, 0, 0, 752, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15483, 17494, 16392, 16391, 16382, 16182, 16371, 16377] avg=16384.0000 max=17494 max_rank=1 min=15483 min_rank=0 imbalance_ratio=1.067749 max_load_reduction=113578 max_load_reduction_pct=0.866531 moved_to_echo=65312 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3716, 0, 0, 1960, 0, 0, 0, 4096, 1822, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1637, 4096, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 2125, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3144, 4096, 0, 0, 1020, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 6, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15692, 17003, 16383, 16385, 16390, 16452, 16384, 16383] avg=16384.0000 max=17003 max_rank=1 min=15692 min_rank=0 imbalance_ratio=1.037781 max_load_reduction=114069 max_load_reduction_pct=0.870277 moved_to_echo=65609 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 32768, 128, 128, 0, 32768, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[26496, 128, 128, 15744, 128, 0, 128, 32768, 23424, 0, 0, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 0, 128, 128, 0, 32768, 32768, 128, 128, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 25216, 32768, 0, 0, 8064, 0, 32512, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 0, 0, 32768, 32768, 0, 128, 128, 128, 0, 128, 128, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 32768, 32768, 0, 128, 0, 0, 128, 32768, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 32768, 0, 128, 32768, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 128, 8192, 32768, 128, 0, 128, 0, 0, 32768, 32768, 128, 128, 17024, 7680, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 2, 4094, 4096, 0, 4096, 6, 0, 0, 0, 2530, 4096, 0, 0, 4094, 0, 0, 0, 0, 2064, 0, 2, 0, 4091, 0, 0, 0, 0, 1, 13, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 4096, 3704, 0, 2, 0, 0, 0, 0, 0, 2, 0, 4095, 4096, 0, 4095, 33, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 2101, 4096, 1882, 4085, 0, 0, 0, 3, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4068, 0, 4096, 0, 1, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 1624, 4096, 0, 0, 4096, 0, 0, 0, 0, 3360, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3164, 4096, 2540, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 2, 0, 0, 0, 2409, 4096, 0, 0, 4096, 0, 0, 0, 0, 3136, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 11, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 2381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3454, 4096, 910, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 1, 0, 0, 0, 1906, 4096, 0, 0, 4096, 0, 0, 0, 0, 3073, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 2442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2957, 4096, 1916, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 2712, 4096, 0, 0, 4096, 0, 0, 0, 0, 4036, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3531, 4096, 815, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 1, 0, 0, 0, 1931, 4096, 0, 0, 4096, 0, 0, 0, 0, 3983, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 4092, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 2881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2999, 4096, 480, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 2, 0, 0, 0, 1367, 4096, 0, 0, 4096, 0, 0, 0, 0, 3354, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 2563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 32, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 2318, 4096, 2661, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1, 4091, 0, 0, 7, 0, 0, 1, 4095, 1, 1, 5, 4084, 1, 1, 0, 4093, 0, 0, 4095, 4095, 1, 4092, 0, 1, 0, 1, 1080, 4095, 0, 0, 4095, 1, 9, 0, 1, 2955, 0, 0, 0, 4095, 0, 0, 1, 1, 0, 0, 18, 4095, 1, 4059, 1, 4095, 0, 1, 0, 0, 4095, 1, 1, 4093, 1384, 0, 0, 0, 3, 0, 1, 1, 0, 1, 4095, 4094, 0, 4094, 0, 0, 0, 0, 1, 1, 0, 4095, 1, 4095, 0, 0, 1, 4095, 1, 0, 259, 3714, 4095, 2911, 4095, 0, 0, 1, 0, 4095, 3, 0, 1, 0, 0, 4095, 1, 0, 0, 1, 1, 4089, 1, 4093, 3, 0, 0, 0, 0, 0, 4095, 2, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3253, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 2, 4094, 0, 0, 4096, 6, 0, 0, 0, 2530, 0, 0, 0, 2150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2064, 0, 2, 0, 4091, 0, 0, 0, 0, 1, 13, 0, 4096, 4096, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1834, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 3704, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 4096, 0, 4095, 33, 0, 0, 0, 0, 1944, 1417, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 2101, 3371, 1882, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3253, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 4096, 0, 1, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 5, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16976, 15206, 17825, 16403, 15532, 16382, 16365] avg=16384.0000 max=17825 max_rank=3 min=15206 min_rank=2 imbalance_ratio=1.087952 max_load_reduction=113247 max_load_reduction_pct=0.864006 moved_to_echo=64682 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3253, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 1906, 0, 0, 0, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3073, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1834, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 1, 0, 0, 0, 0, 1945, 1417, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2957, 3371, 1916, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3253, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16346, 16204, 16563, 16372, 16436, 16384, 16381] avg=16384.0000 max=16563 max_rank=3 min=16204 min_rank=2 imbalance_ratio=1.010925 max_load_reduction=114509 max_load_reduction_pct=0.873634 moved_to_echo=65573 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3253, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1624, 0, 0, 0, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3360, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 4096, 4096, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1834, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4, 0, 0, 0, 0, 1945, 1417, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3164, 3371, 2540, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3253, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16063, 16494, 15719, 16375, 17263, 16384, 16388] avg=16384.0000 max=17263 max_rank=5 min=15719 min_rank=3 imbalance_ratio=1.053650 max_load_reduction=113809 max_load_reduction_pct=0.868294 moved_to_echo=66410 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 4091, 0, 0, 7, 0, 0, 1, 4095, 1, 1, 5, 4084, 1, 1, 0, 3253, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4095, 0, 1, 4092, 0, 1, 0, 1, 1080, 0, 0, 0, 2150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 0, 1, 2955, 0, 0, 0, 4095, 0, 0, 1, 1, 0, 0, 18, 4095, 4095, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1834, 1, 4059, 1, 0, 0, 1, 0, 0, 4095, 1, 1, 4093, 1384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 1, 0, 1, 4095, 4094, 0, 4094, 0, 0, 0, 0, 1, 1945, 1417, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 1, 0, 1, 0, 259, 3714, 3370, 2911, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 3, 0, 1, 0, 0, 4095, 1, 0, 0, 1, 1, 4095, 3252, 842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 4093, 3, 0, 0, 0, 0, 0, 4095, 2, 4095, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 15513, 16114, 15470, 16373, 18448, 16387, 16382] avg=16384.0000 max=18448 max_rank=5 min=15470 min_rank=3 imbalance_ratio=1.125977 max_load_reduction=112624 max_load_reduction_pct=0.859253 moved_to_echo=67590 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3253, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 1931, 0, 0, 0, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 4096, 4096, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1834, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 12, 0, 0, 0, 0, 1945, 1417, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2999, 3371, 480, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3252, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16371, 17121, 16999, 16383, 15041, 16383, 16388] avg=16384.0000 max=17121 max_rank=2 min=15041 min_rank=5 imbalance_ratio=1.044983 max_load_reduction=113951 max_load_reduction_pct=0.869377 moved_to_echo=64195 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3253, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 0, 2409, 0, 0, 0, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3136, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 11, 0, 4096, 4096, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1834, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 1, 0, 0, 0, 0, 1945, 1417, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3454, 3371, 910, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3253, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16850, 16278, 16503, 16372, 15915, 16384, 16388] avg=16384.0000 max=16850 max_rank=1 min=15915 min_rank=5 imbalance_ratio=1.028442 max_load_reduction=114222 max_load_reduction_pct=0.871445 moved_to_echo=65059 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3253, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 2712, 0, 0, 0, 2145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1828, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 2, 0, 0, 0, 0, 1951, 1422, 728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3531, 3360, 815, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3252, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 17145, 17168, 15308, 16391, 15898, 16383, 16392] avg=16384.0000 max=17168 max_rank=2 min=15308 min_rank=3 imbalance_ratio=1.047852 max_load_reduction=113904 max_load_reduction_pct=0.869019 moved_to_echo=65064 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3252, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 0, 1367, 0, 0, 0, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3354, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 4096, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1834, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 32, 0, 0, 0, 0, 1945, 1417, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 2318, 3371, 2661, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3253, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 15808, 16487, 16685, 16403, 16539, 16385, 16388] avg=16384.0000 max=16685 max_rank=3 min=15808 min_rank=1 imbalance_ratio=1.018372 max_load_reduction=114387 max_load_reduction_pct=0.872704 moved_to_echo=65715 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[14720, 128, 32768, 128, 0, 0, 128, 0, 0, 32768, 128, 128, 32768, 18176, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 128, 0, 128, 128, 384, 24320, 27008, 14208, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 32768, 0, 128, 32768, 128, 128, 0, 128, 15616, 0, 0, 0, 17280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 32768, 128, 128, 0, 0, 0, 0, 32768, 128, 32768, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 25984, 0, 128, 0, 32768, 0, 0, 128, 128, 128, 128, 128, 32768, 32768, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 0, 0, 128, 0, 0, 128, 32768, 128, 128, 128, 32768, 128, 128, 0, 26112, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 0, 128, 0, 0, 32768, 128, 0, 0, 128, 128, 32768, 26112, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 32768, 32768, 0, 32768, 128, 0, 0, 0, 128, 15616, 11392, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 142, 4096, 0, 8, 0, 0, 4096, 4095, 4047, 0, 0, 0, 0, 4096, 0, 0, 3401, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 366, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 4087, 0, 0, 0, 4096, 0, 4096, 229, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 64, 0, 0, 0, 4095, 0, 1, 0, 4095, 4048, 1, 397, 0, 1, 0, 1, 0, 1, 0, 115, 4087, 0, 111, 0, 0, 4094, 4095, 4095, 0, 1, 1, 0, 4095, 1, 1, 820, 0, 4, 1, 0, 1, 0, 21, 0, 4080, 4075, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 1, 1, 1, 0, 0, 1, 4095, 0, 1, 4089, 0, 1, 0, 4092, 2648, 4093, 3, 1, 0, 0, 7, 4095, 4095, 0, 4095, 0, 4082, 1, 4091, 0, 3948, 1, 0, 1, 4095, 12, 4089, 146, 0, 0, 22, 0, 4095, 4093, 4095, 5, 4095, 0, 1, 3, 1, 4091, 0, 0, 0, 0, 1, 0, 4082, 4095, 1, 1, 1, 1, 2, 3, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 4096, 0, 0, 0, 0, 4096, 4096, 4042, 0, 0, 0, 0, 4096, 0, 0, 1260, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2718, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 3976, 0, 0, 0, 4096, 0, 4096, 187, 0, 0, 1, 0, 4096, 4096, 4094, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 61, 0, 0, 0, 0, 0, 0, 0, 129, 4096, 0, 2, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 4096, 0, 0, 1175, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2988, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 3731, 0, 0, 0, 4096, 0, 4096, 107, 0, 0, 5, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 314, 4096, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 4096, 0, 0, 1647, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2593, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 3631, 0, 0, 0, 4096, 0, 4096, 41, 0, 0, 0, 0, 4096, 4096, 4079, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3, 4068, 0, 0, 0, 4096, 4096, 0, 16, 0, 0, 0, 0, 0, 0, 0, 83, 4096, 0, 6, 0, 0, 4096, 4096, 4053, 0, 0, 0, 0, 4096, 0, 0, 2175, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1373, 4096, 0, 0, 0, 0, 0, 4096, 4093, 0, 4096, 0, 4096, 0, 4096, 0, 4030, 0, 0, 0, 4096, 0, 4096, 605, 0, 0, 3, 0, 4090, 4096, 4078, 0, 4096, 0, 0, 0, 0, 4093, 0, 2, 0, 0, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4087, 0, 0, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 95, 4096, 0, 0, 0, 0, 4096, 4095, 4085, 0, 0, 0, 0, 4096, 0, 0, 2799, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1169, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 4045, 0, 0, 0, 4096, 0, 4096, 104, 0, 0, 0, 0, 4096, 4096, 4093, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 4096, 4096, 0, 6, 0, 0, 1, 0, 0, 0, 0, 191, 4096, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 3487, 0, 45, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 338, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 4061, 0, 0, 0, 4096, 0, 4096, 64, 0, 0, 0, 0, 4096, 4096, 4095, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 64, 0, 0, 0, 4095, 0, 1, 0, 4095, 4048, 1, 397, 1944, 1773, 182, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 115, 4087, 0, 111, 0, 0, 3952, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 1, 1, 820, 0, 4, 1, 0, 1, 0, 21, 0, 4095, 4095, 1988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4075, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 1, 1, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2322, 0, 1, 4089, 0, 1, 0, 4092, 2648, 4093, 3, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 4082, 1, 4091, 0, 3948, 1, 0, 1, 0, 12, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 22, 0, 4095, 3911, 4095, 5, 0, 0, 1, 3, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4082, 4095, 1, 1, 1, 1, 2, 3, 0, 1, 0, 4095, 2107, 1991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16741, 16458, 15124, 16355, 17258, 16385, 16370, 16381] avg=16384.0000 max=17258 max_rank=4 min=15124 min_rank=2 imbalance_ratio=1.053345 max_load_reduction=113814 max_load_reduction_pct=0.868332 moved_to_echo=66394 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4, 1944, 1773, 182, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 4096, 0, 8, 0, 0, 3954, 4095, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3401, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 0, 0, 4096, 0, 0, 0, 4096, 366, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 4096, 0, 4096, 0, 4087, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 0, 0, 4096, 3914, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2106, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16331, 16342, 17683, 16388, 14976, 16535, 16431, 16386] avg=16384.0000 max=17683 max_rank=2 min=14976 min_rank=4 imbalance_ratio=1.079285 max_load_reduction=113389 max_load_reduction_pct=0.865089 moved_to_echo=64328 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 4096, 4096, 0, 0, 1944, 1773, 182, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 4096, 0, 0, 0, 0, 3954, 4096, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1260, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4096, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 0, 0, 4096, 0, 0, 0, 4096, 2718, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 4096, 0, 4096, 0, 3976, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 1, 0, 4096, 3914, 4094, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2106, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16309, 16316, 15540, 16387, 17329, 16424, 16387, 16380] avg=16384.0000 max=17329 max_rank=4 min=15540 min_rank=2 imbalance_ratio=1.057678 max_load_reduction=113743 max_load_reduction_pct=0.867790 moved_to_echo=66520 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4096, 4096, 0, 1, 1944, 1773, 182, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 4096, 0, 0, 0, 0, 3954, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1647, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 0, 0, 4096, 0, 0, 0, 4096, 2593, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 4096, 0, 4096, 0, 3631, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 4096, 3914, 4079, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2107, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16313, 16553, 15924, 16387, 17204, 16079, 16226, 16386] avg=16384.0000 max=17204 max_rank=4 min=15924 min_rank=2 imbalance_ratio=1.050049 max_load_reduction=113868 max_load_reduction_pct=0.868744 moved_to_echo=65895 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3, 4068, 0, 0, 0, 4096, 4096, 0, 16, 1943, 1773, 182, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 4096, 0, 6, 0, 0, 3954, 4096, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2175, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4096, 1989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 0, 0, 4096, 0, 0, 0, 4096, 1373, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 4096, 0, 4096, 0, 4030, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 605, 0, 0, 3, 0, 4090, 3914, 4078, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 2107, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16316, 16288, 16455, 16386, 15984, 16478, 16783, 16382] avg=16384.0000 max=16783 max_rank=6 min=15984 min_rank=4 imbalance_ratio=1.024353 max_load_reduction=114289 max_load_reduction_pct=0.871956 moved_to_echo=65627 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 61, 1944, 1773, 182, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 4096, 0, 2, 0, 0, 3954, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1175, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 4096, 1989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 0, 0, 4096, 0, 0, 0, 4096, 2988, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 4096, 0, 4096, 0, 3731, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 5, 0, 4096, 3914, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2107, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16370, 15458, 16387, 17599, 16179, 16310, 16382] avg=16384.0000 max=17599 max_rank=4 min=15458 min_rank=2 imbalance_ratio=1.074158 max_load_reduction=113473 max_load_reduction_pct=0.865730 moved_to_echo=66470 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 0, 4093, 0, 0, 0, 4096, 4096, 0, 6, 1944, 1773, 182, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 191, 4096, 0, 0, 0, 0, 3953, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3487, 0, 45, 0, 0, 0, 0, 0, 0, 4096, 4096, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 0, 0, 4096, 0, 0, 0, 4096, 338, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 4096, 0, 4096, 0, 4061, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 4096, 3914, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2106, 1992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16332, 16433, 17810, 16389, 14948, 16509, 16265, 16386] avg=16384.0000 max=17810 max_rank=2 min=14948 min_rank=4 imbalance_ratio=1.087036 max_load_reduction=113262 max_load_reduction_pct=0.864120 moved_to_echo=64108 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4087, 0, 0, 0, 4096, 4096, 0, 2, 1951, 1779, 185, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 4096, 0, 0, 0, 0, 3941, 4095, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2799, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2317, 0, 0, 4096, 0, 0, 0, 4096, 1169, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 0, 4096, 0, 4096, 0, 4045, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 4096, 3911, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2106, 1995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16343, 16312, 17078, 16393, 15774, 16483, 16300, 16389] avg=16384.0000 max=17078 max_rank=2 min=15774 min_rank=4 imbalance_ratio=1.042358 max_load_reduction=113994 max_load_reduction_pct=0.869705 moved_to_echo=64946 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 0, 128, 0, 32768, 32768, 128, 128, 128, 128, 128, 128, 128, 128, 0, 32768, 16896, 16000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 128, 16768, 0, 128, 128, 0, 128, 0, 128, 0, 32768, 32768, 16000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 32768, 128, 128, 128, 128, 32768, 128, 0, 0, 32768, 128, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 1280, 0, 0, 0, 32768, 128, 32768, 0, 31616, 128, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 18688, 0, 128, 32768, 0, 128, 0, 32768, 14208, 32768, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[1536, 0, 0, 128, 0, 32768, 31360, 32768, 128, 0, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 128, 128, 128, 0, 128, 0, 1280, 32768, 0, 128, 0, 0, 31616, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 128, 0, 0, 128, 32768, 0, 128, 0, 32768, 32768, 128, 512, 15616, 14208, 1536, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 1163, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 2194, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 807, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4060, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 2, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 1887, 0, 4096, 0, 0, 4083, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 2200, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 417, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3765, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2253, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 1423, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 648, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3991, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2053, 0, 4096, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 2055, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 462, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3662, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2175, 0, 4096, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 11, 0, 1995, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 3, 231, 0, 3, 0, 4096, 0, 0, 4094, 1, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4, 4096, 0, 0, 3, 3992, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 3897, 0, 0, 0, 0, 4096, 4091, 0, 0, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 3223, 0, 4096, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 11, 0, 973, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 323, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3814, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 3954, 0, 0, 0, 0, 4096, 4095, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 4093, 4095, 0, 4095, 0, 4072, 1, 0, 4095, 1, 1365, 0, 4094, 1, 0, 4016, 0, 0, 0, 1, 1, 148, 4095, 333, 1, 4095, 4078, 0, 0, 858, 1, 1, 1, 1, 0, 1, 1, 4095, 13, 4095, 0, 0, 0, 0, 1835, 0, 0, 1, 4094, 1, 0, 4095, 0, 0, 0, 4095, 0, 1, 1, 0, 4050, 1, 4082, 2, 1, 2, 4094, 0, 2, 0, 3841, 1, 1, 0, 4095, 1, 31, 0, 0, 1, 0, 6, 1, 4094, 67, 4095, 1, 1, 0, 1, 0, 1, 3888, 0, 1, 31, 0, 4089, 4094, 0, 0, 22, 4095, 1, 26, 0, 1, 0, 1, 1, 0, 1, 1, 0, 4066, 0, 0, 4095, 4094, 4093, 34, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2347, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 1231, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 575, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4068, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2042, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1423, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2464, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 648, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2003, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 205, 0, 0, 0, 3991, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 51, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16583, 16343, 16188, 16366, 16484, 16372, 16352, 16384] avg=16384.0000 max=16583 max_rank=0 min=16188 min_rank=2 imbalance_ratio=1.012146 max_load_reduction=114489 max_load_reduction_pct=0.873482 moved_to_echo=65592 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2042, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1995, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 3, 4096, 2464, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 3, 0, 4096, 0, 0, 4094, 1, 0, 6, 4096, 0, 0, 0, 0, 2003, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4, 205, 0, 0, 3, 3992, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3897, 0, 0, 0, 0, 4096, 4091, 0, 0, 3, 4092, 0, 0, 0, 0, 0, 51, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16505, 16341, 16760, 15957, 16486, 16368, 16271, 16384] avg=16384.0000 max=16760 max_rank=2 min=15957 min_rank=3 imbalance_ratio=1.022949 max_load_reduction=114312 max_load_reduction_pct=0.872131 moved_to_echo=65509 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2042, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2055, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2464, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 4096, 0, 0, 0, 0, 2003, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 205, 0, 0, 0, 3662, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 51, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16310, 16820, 16183, 16155, 16371, 16466, 16384] avg=16384.0000 max=16820 max_rank=2 min=16155 min_rank=4 imbalance_ratio=1.026611 max_load_reduction=114252 max_load_reduction_pct=0.871674 moved_to_echo=65376 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2042, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4055, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2194, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2464, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 807, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2003, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 205, 0, 0, 0, 4060, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 2, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 51, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15493, 16344, 16959, 16525, 16553, 16374, 16440, 16384] avg=16384.0000 max=16959 max_rank=2 min=15493 min_rank=0 imbalance_ratio=1.035095 max_load_reduction=114113 max_load_reduction_pct=0.870613 moved_to_echo=65751 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2042, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 973, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2464, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 2003, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 205, 0, 0, 0, 3814, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 0, 0, 0, 4096, 4095, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 51, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17553, 16343, 15738, 16043, 16307, 16370, 16334, 16384] avg=16384.0000 max=17553 max_rank=0 min=15738 min_rank=2 imbalance_ratio=1.071350 max_load_reduction=113519 max_load_reduction_pct=0.866081 moved_to_echo=65395 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2034, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4083, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4049, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2200, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2471, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2007, 1434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 191, 0, 0, 0, 3765, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 55, 47, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16209, 16325, 16973, 16146, 16244, 16370, 16423, 16382] avg=16384.0000 max=16973 max_rank=2 min=16146 min_rank=3 imbalance_ratio=1.035950 max_load_reduction=114099 max_load_reduction_pct=0.870506 moved_to_echo=65419 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2041, 0, 0, 1, 4093, 0, 0, 0, 0, 4072, 1, 0, 4095, 1, 1365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 4016, 0, 0, 0, 1, 1, 148, 0, 333, 1, 4054, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 858, 1, 1, 1, 1, 0, 1, 1, 4095, 13, 4095, 0, 0, 0, 0, 4095, 2463, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1835, 0, 0, 1, 4094, 1, 0, 4095, 0, 0, 0, 4095, 0, 1, 1, 0, 2003, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 1, 4082, 2, 1, 2, 204, 0, 2, 0, 3841, 1, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 1, 0, 6, 1, 4094, 67, 4095, 1, 1, 0, 1, 0, 1, 4095, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3888, 0, 1, 31, 0, 4089, 4094, 0, 0, 22, 4095, 1, 26, 0, 1, 0, 51, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 4066, 0, 0, 4095, 4094, 4093, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15669, 16727, 15638, 17553, 16283, 16476, 16340, 16386] avg=16384.0000 max=17553 max_rank=3 min=15638 min_rank=2 imbalance_ratio=1.071350 max_load_reduction=113519 max_load_reduction_pct=0.866081 moved_to_echo=65485 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2042, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1231, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2464, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 6, 4096, 0, 0, 0, 0, 2003, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 205, 0, 0, 0, 4068, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 51, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16677, 16339, 15996, 16299, 16560, 16371, 16446, 16384] avg=16384.0000 max=16677 max_rank=0 min=15996 min_rank=2 imbalance_ratio=1.017883 max_load_reduction=114395 max_load_reduction_pct=0.872765 moved_to_echo=65761 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 32768, 128, 128, 128, 1664, 0, 128, 128, 31232, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32000, 0, 128, 128, 0, 32768, 32768, 0, 0, 128, 32768, 128, 128, 0, 128, 0, 512, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 13056, 128, 128, 128, 128, 0, 128, 128, 32768, 128, 32768, 0, 0, 0, 128, 32768, 19840, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[5376, 0, 128, 128, 32768, 128, 0, 32768, 128, 0, 128, 32768, 0, 128, 128, 0, 16128, 11520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[16384, 0, 0, 128, 32768, 0, 0, 0, 0, 32768, 128, 0, 32768, 128, 16512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 0, 128, 128, 32768, 128, 32768, 128, 128, 0, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 32640, 0, 0, 0, 128, 128, 256, 0, 384, 128, 32512, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 128, 0, 32768, 0, 0, 32768, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 26, 0, 4096, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4045, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 107, 3974, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4096, 0, 0, 0, 0, 260, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4083, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 15, 4015, 0, 0, 0, 0, 0, 0, 3886, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 68, 0, 4096, 0, 0, 0, 0, 376, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 4096, 3, 4087, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4093, 4096, 0, 0, 4090, 1, 0, 4095, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4090, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 6, 0, 0, 14, 0, 4096, 0, 0, 0, 0, 67, 3795, 0, 0, 0, 0, 4, 0, 3885, 5, 0, 0, 4096, 0, 4096, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 25, 0, 4096, 0, 0, 0, 0, 932, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4085, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 38, 3689, 0, 0, 0, 0, 0, 0, 3516, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 25, 0, 4096, 0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 14, 4033, 0, 0, 0, 0, 0, 0, 4016, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 624, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4083, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4047, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 62, 3742, 0, 0, 0, 0, 0, 0, 3822, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 571, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 3526, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 1, 4093, 1, 1, 1, 0, 0, 4094, 1, 1, 1, 3, 1, 62, 1, 2, 1, 4095, 0, 0, 0, 1, 272, 0, 0, 1, 1, 1, 4095, 4084, 1, 1, 1, 0, 0, 4094, 0, 4095, 0, 0, 4080, 2, 0, 4094, 1, 0, 6, 0, 0, 0, 0, 0, 4090, 0, 4094, 4095, 0, 1, 0, 4095, 4095, 3, 0, 3588, 0, 0, 4047, 1, 1, 0, 1, 1, 4088, 1, 1, 1, 7, 0, 4095, 2, 3, 4095, 4095, 4094, 1, 0, 4095, 4095, 4091, 0, 0, 0, 0, 0, 4095, 4080, 1, 0, 4092, 0, 0, 6, 0, 0, 4095, 3, 0, 0, 1, 980, 3641, 0, 1, 0, 0, 0, 1, 3804, 0, 2, 1, 4094, 14, 4093, 0, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 4096, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 4096, 4096, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 107, 3974, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 276, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16096, 16388, 16385, 16420, 16384, 16431, 16593] avg=16384.0000 max=16593 max_rank=7 min=16096 min_rank=1 imbalance_ratio=1.012756 max_load_reduction=114479 max_load_reduction_pct=0.873405 moved_to_echo=65828 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 4096, 0, 0, 0, 0, 376, 0, 0, 0, 0, 0, 4092, 4096, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4087, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 4095, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4095, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 6, 0, 0, 14, 0, 4096, 0, 0, 0, 0, 67, 3795, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3885, 5, 0, 0, 4096, 0, 4096, 0, 0, 2, 4094, 275, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16394, 16384, 16383, 16457, 16381, 16230, 16466] avg=16384.0000 max=16466 max_rank=7 min=16230 min_rank=6 imbalance_ratio=1.005005 max_load_reduction=114606 max_load_reduction_pct=0.874374 moved_to_echo=65534 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 4096, 0, 0, 0, 0, 932, 0, 0, 0, 0, 0, 4096, 4096, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 38, 3689, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3516, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 275, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16376, 16911, 16388, 16384, 16461, 16383, 16082, 16087] avg=16384.0000 max=16911 max_rank=1 min=16082 min_rank=6 imbalance_ratio=1.032166 max_load_reduction=114161 max_load_reduction_pct=0.870979 moved_to_echo=65013 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4093, 1, 1, 1, 0, 0, 4094, 1, 1, 1, 3, 1, 62, 1, 4095, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4095, 0, 0, 0, 1, 272, 0, 0, 1, 1, 1, 4095, 4084, 1, 3665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4094, 0, 4095, 0, 0, 4080, 2, 0, 4094, 1, 0, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4094, 2, 0, 1, 0, 4095, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3588, 0, 0, 4047, 1, 1, 0, 1, 1, 4088, 1, 1, 1, 7, 0, 4095, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 4, 4094, 1, 0, 0, 4095, 4091, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 1, 0, 4092, 0, 0, 6, 0, 0, 4095, 3, 0, 0, 1, 980, 3641, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 3804, 0, 2, 1, 4094, 14, 4093, 0, 0, 3, 4093, 276, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16442, 16219, 16377, 16380, 15918, 16385, 16960, 16391] avg=16384.0000 max=16960 max_rank=6 min=15918 min_rank=4 imbalance_ratio=1.035156 max_load_reduction=114112 max_load_reduction_pct=0.870605 moved_to_echo=65654 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4096, 0, 0, 0, 0, 260, 0, 0, 0, 0, 0, 4096, 4096, 0, 3669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 15, 4015, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3886, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 275, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16376, 16247, 16388, 16384, 16457, 16384, 16379, 16457] avg=16384.0000 max=16457 max_rank=4 min=16247 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=114615 max_load_reduction_pct=0.874443 moved_to_echo=65677 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 624, 0, 0, 0, 0, 0, 4096, 4096, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 62, 3742, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3822, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4094, 278, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16582, 16375, 16386, 16421, 16384, 16154, 16395] avg=16384.0000 max=16582 max_rank=1 min=16154 min_rank=6 imbalance_ratio=1.012085 max_load_reduction=114490 max_load_reduction_pct=0.873489 moved_to_echo=65354 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 4096, 0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 4096, 4096, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 14, 4033, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4094, 275, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16090, 16383, 16386, 16468, 16387, 16397, 16586] avg=16384.0000 max=16586 max_rank=7 min=16090 min_rank=1 imbalance_ratio=1.012329 max_load_reduction=114486 max_load_reduction_pct=0.873459 moved_to_echo=65838 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 571, 0, 0, 0, 0, 0, 4096, 4096, 0, 3668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4090, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3526, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 275, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16376, 16533, 16389, 16384, 16470, 16384, 16439, 16097] avg=16384.0000 max=16533 max_rank=1 min=16097 min_rank=7 imbalance_ratio=1.009094 max_load_reduction=114539 max_load_reduction_pct=0.873863 moved_to_echo=65390 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 0, 32768, 128, 0, 128, 128, 0, 32768, 128, 0, 0, 128, 1408, 31104, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32128, 128, 0, 32768, 128, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 0, 32768, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 128, 0, 32768, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 32768, 0, 32768, 128, 0, 128, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 0, 128, 128, 30592, 128, 128, 128, 32768, 128, 32768, 0, 0, 128, 32768, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 128, 32768, 0, 0, 0, 128, 3328, 0, 0, 128, 128, 128, 32768, 32768, 128, 29440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 32768, 0, 128, 32768, 128, 0, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2432, 4096, 0, 4096, 0, 0, 0, 0, 4094, 4096, 0, 19, 0, 0, 2727, 0, 0, 0, 20, 6, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 148, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 11, 4090, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4015, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 181, 0, 4096, 4, 4096, 0, 4086, 4096, 0, 0, 0, 0, 0, 4077, 0, 4092, 2762, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3130, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 17, 0, 0, 1076, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 65, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4049, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 139, 0, 4096, 0, 4096, 0, 4088, 4096, 0, 0, 0, 0, 0, 4095, 0, 4096, 3813, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2311, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 1456, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 771, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 10, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3640, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2658, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 44, 0, 0, 2347, 0, 0, 0, 30, 11, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 20, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4044, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 85, 0, 4096, 0, 4096, 0, 4072, 4096, 0, 0, 0, 0, 0, 4085, 0, 4096, 3084, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 1796, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 36, 0, 0, 2890, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4051, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 191, 0, 4096, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 4078, 0, 4096, 3317, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4096, 1295, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 18, 0, 0, 2889, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4083, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 77, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4060, 0, 4096, 3949, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 0, 0, 3, 0, 4093, 4094, 0, 0, 57, 4095, 3674, 4092, 0, 4094, 0, 0, 0, 0, 4095, 4095, 1, 203, 1, 1, 703, 0, 0, 1, 0, 13, 0, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 2, 0, 6, 0, 2, 0, 1, 1, 2, 4013, 0, 1, 4095, 1, 0, 1, 50, 0, 0, 4095, 1, 2, 2, 1, 1, 1, 0, 4095, 0, 1, 4085, 4091, 1, 4095, 4089, 4093, 4092, 15, 5, 0, 4079, 4095, 0, 1, 0, 4094, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 4095, 4095, 0, 1, 4095, 0, 4094, 0, 4095, 4095, 0, 1, 0, 1, 10, 4094, 0, 4095, 3574, 1, 0, 0, 1, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2401, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 24, 0, 0, 3526, 0, 0, 0, 9, 1, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4077, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 80, 0, 4096, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 4028, 0, 4096, 2220, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2432, 4096, 0, 3484, 0, 0, 0, 0, 4094, 0, 0, 19, 0, 0, 2727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 6, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 148, 0, 4096, 3952, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 11, 4090, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 34, 4096, 0, 0, 0, 4096, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 4007, 4, 4096, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4092, 2762, 0, 0, 0, 0, 0, 0, 0, 4062, 612, 144, 89, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 16852, 16396, 16407, 16388, 16337, 16470, 15844] avg=16384.0000 max=16852 max_rank=1 min=15844 min_rank=7 imbalance_ratio=1.028564 max_load_reduction=114220 max_load_reduction_pct=0.871429 moved_to_echo=65039 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2311, 4096, 0, 3484, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 1456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 771, 0, 4096, 3953, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 34, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4007, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3640, 0, 0, 0, 0, 0, 0, 0, 4062, 612, 143, 89, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 15446, 16370, 17027, 16384, 16418, 16305, 16744] avg=16384.0000 max=17027 max_rank=3 min=15446 min_rank=1 imbalance_ratio=1.039246 max_load_reduction=114045 max_load_reduction_pct=0.870094 moved_to_echo=65851 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2658, 4096, 0, 3484, 0, 0, 0, 0, 4096, 0, 0, 44, 0, 0, 2347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 11, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 20, 0, 4096, 3953, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 34, 4096, 0, 0, 0, 4096, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 4007, 0, 4096, 0, 4072, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4096, 3084, 0, 0, 0, 0, 0, 0, 0, 4062, 612, 143, 89, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 16725, 16409, 16277, 16384, 16366, 16356, 16177] avg=16384.0000 max=16725 max_rank=1 min=16177 min_rank=7 imbalance_ratio=1.020813 max_load_reduction=114347 max_load_reduction_pct=0.872398 moved_to_echo=65283 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4089, 0, 0, 0, 3, 0, 4093, 4094, 0, 0, 57, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3674, 4092, 0, 3482, 0, 0, 0, 0, 4095, 0, 1, 203, 1, 1, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 0, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 2, 0, 4095, 4095, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 1, 1, 2, 4013, 0, 1, 4095, 1, 0, 1, 50, 0, 4095, 3950, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 2, 2, 1, 1, 1, 0, 4095, 0, 1, 4085, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 34, 4092, 15, 5, 0, 4079, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 0, 0, 0, 1, 4006, 0, 4094, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 10, 4094, 0, 4095, 3574, 1, 0, 0, 1, 0, 1, 0, 4059, 612, 144, 89, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16432, 16252, 16384, 16235, 16376, 16410, 16295, 16688] avg=16384.0000 max=16688 max_rank=7 min=16235 min_rank=3 imbalance_ratio=1.018555 max_load_reduction=114384 max_load_reduction_pct=0.872681 moved_to_echo=65769 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3130, 4096, 0, 3479, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 65, 0, 4096, 3953, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 27, 4096, 0, 0, 0, 4096, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 4000, 0, 4096, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 3813, 0, 0, 0, 0, 0, 0, 0, 4069, 617, 143, 96, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 15894, 16377, 16321, 16383, 16364, 16419, 16941] avg=16384.0000 max=16941 max_rank=7 min=15894 min_rank=1 imbalance_ratio=1.033997 max_load_reduction=114131 max_load_reduction_pct=0.870750 moved_to_echo=66107 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1796, 4096, 0, 3484, 0, 0, 0, 0, 4096, 0, 0, 36, 0, 0, 2890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 8, 0, 4096, 3952, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 34, 4096, 0, 0, 0, 4096, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 4007, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4096, 3317, 0, 0, 0, 0, 0, 0, 0, 4062, 612, 144, 89, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 16398, 16387, 16264, 16385, 16373, 16483, 16404] avg=16384.0000 max=16483 max_rank=6 min=16264 min_rank=3 imbalance_ratio=1.006042 max_load_reduction=114589 max_load_reduction_pct=0.874245 moved_to_echo=65645 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1295, 4096, 0, 3484, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 2889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 12, 0, 4096, 3953, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 34, 4096, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 4007, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 4096, 3949, 0, 0, 0, 0, 0, 0, 0, 4062, 612, 143, 89, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 15878, 16370, 16269, 16384, 16405, 16372, 17017] avg=16384.0000 max=17017 max_rank=7 min=15878 min_rank=1 imbalance_ratio=1.038635 max_load_reduction=114055 max_load_reduction_pct=0.870171 moved_to_echo=66178 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2401, 4096, 0, 3484, 0, 0, 0, 0, 4096, 0, 0, 24, 0, 0, 3526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 15, 0, 4096, 3952, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 34, 4096, 0, 0, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 4007, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 4096, 2220, 0, 0, 0, 0, 0, 0, 0, 4062, 612, 144, 89, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 17627, 16379, 16272, 16388, 16399, 16372, 15257] avg=16384.0000 max=17627 max_rank=1 min=15257 min_rank=7 imbalance_ratio=1.075867 max_load_reduction=113445 max_load_reduction_pct=0.865517 moved_to_echo=64416 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[19712, 32768, 0, 27904, 0, 0, 0, 0, 32768, 0, 128, 384, 128, 128, 17664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 32768, 128, 128, 32768, 128, 0, 128, 1152, 0, 32768, 31744, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 32768, 0, 0, 0, 128, 0, 32768, 32768, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 384, 32768, 128, 128, 0, 32768, 32512, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 128, 128, 128, 32768, 0, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 768, 128, 32128, 128, 32768, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 32640, 0, 32768, 26368, 128, 0, 0, 128, 0, 128, 0, 32512, 4992, 1152, 768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 128, 0, 32768, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4053, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1130, 3256, 0, 0, 4096, 3346, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4034, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4096, 0, 4096, 4095, 0, 0, 1, 4096, 3, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 4096, 0, 3675, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 1745, 3079, 0, 0, 4096, 3464, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4092, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 3721, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1397, 2964, 0, 0, 4096, 3837, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4072, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 141, 3428, 0, 0, 4096, 3900, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 20, 0, 0, 0, 0, 0, 4091, 4096, 0, 4096, 0, 3671, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 837, 3716, 0, 0, 4096, 4019, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4086, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 352, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 3955, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 320, 3585, 0, 0, 4096, 4048, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4094, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 0, 68, 0, 4074, 0, 4095, 4095, 0, 0, 0, 4095, 0, 1, 0, 4043, 4095, 21, 0, 2, 0, 53, 1, 4094, 4095, 0, 4093, 4, 2606, 1, 1, 4092, 1, 0, 0, 1, 1, 0, 321, 0, 1, 1, 1, 2, 0, 1, 0, 5, 4094, 4093, 6, 0, 2, 2868, 2838, 0, 0, 4094, 3830, 0, 2, 1, 4017, 1, 0, 0, 4087, 0, 1, 0, 0, 1, 1, 4090, 4092, 0, 0, 0, 0, 0, 1, 0, 1, 4094, 4095, 1, 0, 0, 4091, 0, 1, 2, 4095, 3865, 1, 4095, 0, 0, 1, 4093, 1, 1, 1, 0, 4, 1, 4095, 4095, 0, 0, 4095, 1, 0, 1, 64, 1, 2, 0, 4095, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 3981, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 156, 4054, 0, 0, 4096, 4085, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4091, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 4096, 0, 4096, 4096, 0, 3943, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4053, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 404, 0, 0, 4096, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1130, 3256, 0, 0, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3346, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4034, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 25, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16421, 16379, 16898, 16329, 15925, 16384, 16360, 16376] avg=16384.0000 max=16898 max_rank=2 min=15925 min_rank=4 imbalance_ratio=1.031372 max_load_reduction=114174 max_load_reduction_pct=0.871078 moved_to_echo=65045 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4096, 0, 4096, 4095, 0, 3943, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3675, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 289, 0, 0, 4096, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 1745, 3079, 0, 0, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3464, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 26, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16294, 16379, 16405, 16768, 16042, 16395, 16414, 16375] avg=16384.0000 max=16768 max_rank=3 min=16042 min_rank=4 imbalance_ratio=1.023438 max_load_reduction=114304 max_load_reduction_pct=0.872070 moved_to_echo=65226 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 4096, 0, 4096, 4096, 0, 3943, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3721, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 187, 0, 0, 4096, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1397, 2964, 0, 0, 3752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3837, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 25, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16436, 16383, 16349, 16305, 16415, 16387, 16421, 16376] avg=16384.0000 max=16436 max_rank=0 min=16305 min_rank=3 imbalance_ratio=1.003174 max_load_reduction=114636 max_load_reduction_pct=0.874603 moved_to_echo=65599 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 0, 68, 0, 4074, 0, 4095, 4095, 0, 3943, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1, 0, 4043, 0, 21, 0, 2, 0, 53, 1, 4094, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 2606, 1, 1, 4092, 1, 0, 0, 1, 1, 0, 321, 0, 1, 4095, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 0, 1, 0, 5, 4094, 4093, 6, 0, 2, 2868, 2838, 0, 0, 3750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3830, 0, 2, 1, 4017, 1, 0, 0, 4087, 0, 1, 0, 0, 1, 1, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4092, 0, 0, 0, 0, 0, 1, 0, 1, 4094, 0, 1, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4095, 3865, 1, 4095, 0, 0, 1, 4093, 1, 1, 1, 0, 4, 26, 8, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4087, 0, 0, 4095, 1, 0, 1, 64, 1, 2, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16306, 16401, 15369, 17661, 16325, 16370, 16198, 16442] avg=16384.0000 max=17661 max_rank=3 min=15369 min_rank=2 imbalance_ratio=1.077942 max_load_reduction=113411 max_load_reduction_pct=0.865257 moved_to_echo=65335 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 4096, 4096, 0, 3943, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 20, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3671, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 34, 0, 0, 4096, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 837, 3716, 0, 0, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4019, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 25, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16265, 16394, 16146, 16496, 16598, 16385, 16412, 16376] avg=16384.0000 max=16598 max_rank=4 min=16146 min_rank=2 imbalance_ratio=1.013062 max_load_reduction=114474 max_load_reduction_pct=0.873367 moved_to_echo=65771 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 352, 0, 4096, 0, 4096, 4096, 0, 3943, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3955, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 30, 0, 0, 4096, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 320, 3585, 0, 0, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4048, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 25, 9, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16613, 16378, 16426, 15848, 16626, 16379, 16427, 16375] avg=16384.0000 max=16626 max_rank=4 min=15848 min_rank=3 imbalance_ratio=1.014771 max_load_reduction=114446 max_load_reduction_pct=0.873154 moved_to_echo=65807 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 4096, 0, 4096, 4096, 0, 3943, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4072, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 540, 0, 0, 4096, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 141, 3428, 0, 0, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3900, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 25, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16468, 16379, 17053, 15512, 16478, 16384, 16422, 16376] avg=16384.0000 max=17053 max_rank=2 min=15512 min_rank=3 imbalance_ratio=1.040833 max_load_reduction=114019 max_load_reduction_pct=0.869896 moved_to_echo=65660 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 4096, 4096, 0, 3942, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3981, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 156, 4054, 0, 0, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 26, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16269, 16379, 16426, 16153, 16663, 16388, 16418, 16376] avg=16384.0000 max=16663 max_rank=4 min=16153 min_rank=3 imbalance_ratio=1.017029 max_load_reduction=114409 max_load_reduction_pct=0.872871 moved_to_echo=65845 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 128, 128, 0, 32768, 0, 128, 0, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 32768, 0, 0, 32768, 128, 0, 128, 128, 128, 128, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 32768, 32512, 128, 32768, 0, 0, 128, 32768, 128, 128, 128, 0, 128, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 32768, 32768, 128, 128, 128, 8704, 27008, 0, 0, 30080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 30592, 0, 128, 128, 32768, 128, 0, 0, 32768, 0, 128, 0, 0, 128, 128, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 128, 29824, 128, 128, 32768, 128, 0, 0, 128, 128, 0, 1920, 0, 128, 32768, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 32768, 128, 0, 128, 0, 0, 128, 0, 128, 32768, 0, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 0, 0, 128, 0, 0, 1024, 0, 32768, 0, 32768, 32768, 0, 31616, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[519, 1346, 1062, 2, 123, 3, 2972, 29, 400, 1076, 200, 563, 2128, 3243, 4, 2526, 1725, 1410, 1728, 2824, 41, 940, 107, 983, 18, 11, 3471, 1813, 1176, 3, 2118, 899, 3184, 0, 251, 429, 245, 567, 1861, 134, 169, 0, 1786, 3111, 5, 3142, 342, 924, 1693, 2108, 1209, 123, 2, 7, 35, 2809, 2, 511, 97, 322, 1264, 3615, 1908, 56, 1188, 386, 0, 1335, 2357, 168, 568, 845, 3157, 2653, 1578, 18, 945, 0, 242, 2196, 954, 995, 3, 65, 606, 1037, 644, 55, 2197, 17, 2401, 1818, 124, 2310, 312, 0, 1504, 726, 74, 785, 2, 411, 1364, 550, 291, 1062, 5, 1085, 703, 2429, 1260, 1580, 1931, 2540, 508, 2019, 795, 522, 1644, 1220, 1598, 2997, 903, 1354, 176, 3, 482, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[639, 1162, 1054, 1, 124, 0, 3303, 108, 814, 1120, 262, 752, 2344, 2958, 0, 2338, 1017, 1245, 1881, 2487, 158, 899, 114, 1020, 42, 17, 3613, 1957, 2071, 10, 2107, 549, 2255, 11, 357, 687, 523, 381, 1516, 225, 156, 5, 1959, 3069, 10, 2881, 378, 784, 1959, 2582, 1452, 169, 9, 18, 17, 2442, 0, 313, 125, 474, 2005, 3093, 1253, 27, 1276, 359, 0, 2239, 1918, 1036, 308, 949, 2335, 2712, 1709, 32, 272, 16, 234, 1960, 1072, 2127, 0, 38, 504, 307, 727, 202, 2415, 69, 2205, 1108, 210, 2234, 422, 0, 1861, 1339, 28, 1051, 6, 427, 1314, 1260, 346, 976, 5, 760, 605, 2177, 696, 1299, 2016, 2297, 209, 2324, 563, 332, 1467, 1454, 1654, 3140, 502, 1711, 255, 3, 688, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[455, 1649, 1319, 0, 29, 0, 3395, 52, 340, 1923, 238, 278, 2099, 3383, 0, 2602, 1992, 1540, 1846, 2298, 95, 860, 144, 869, 8, 44, 3711, 2047, 1994, 28, 2362, 1028, 2610, 0, 122, 774, 447, 392, 1915, 133, 95, 4, 1887, 3188, 2, 2825, 467, 893, 2167, 2423, 955, 61, 1, 4, 35, 2359, 0, 555, 12, 283, 1650, 3619, 2031, 8, 1438, 244, 0, 1490, 2030, 264, 217, 682, 2275, 2255, 1818, 36, 742, 3, 539, 1609, 975, 1650, 1, 24, 713, 631, 405, 46, 1848, 11, 2327, 1725, 160, 2184, 314, 0, 1295, 1795, 13, 705, 0, 795, 1504, 731, 212, 1285, 9, 1282, 362, 2128, 1441, 1488, 1766, 2709, 285, 1520, 611, 586, 1465, 1031, 1533, 2635, 929, 1064, 122, 2, 568, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[384, 1392, 1538, 0, 136, 0, 2610, 50, 567, 1312, 218, 515, 2714, 3068, 0, 2531, 2225, 1409, 2019, 2438, 88, 862, 23, 1056, 9, 12, 3691, 2140, 1554, 12, 2109, 697, 2662, 0, 254, 495, 333, 463, 2048, 163, 150, 11, 1786, 3145, 0, 2990, 340, 796, 1798, 2292, 1011, 197, 0, 5, 10, 2913, 0, 640, 39, 385, 1915, 2930, 2088, 10, 1066, 337, 0, 1561, 2473, 500, 226, 532, 2825, 2515, 1867, 14, 719, 2, 242, 2281, 362, 924, 0, 22, 611, 813, 923, 76, 2175, 11, 1681, 1305, 106, 2394, 393, 0, 1358, 1379, 52, 944, 0, 819, 1477, 586, 118, 1012, 1, 912, 642, 2546, 936, 827, 1352, 2532, 396, 2748, 1490, 512, 1666, 1651, 1694, 2538, 760, 1006, 229, 0, 685, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[567, 1199, 1908, 1, 26, 0, 3192, 77, 686, 1458, 257, 489, 2736, 3069, 0, 2843, 1263, 1421, 2018, 2286, 170, 1520, 145, 836, 12, 23, 3690, 2162, 1770, 18, 2106, 661, 2638, 12, 240, 1008, 148, 382, 1761, 138, 95, 0, 2144, 3062, 2, 2756, 505, 1193, 1785, 2071, 1384, 74, 2, 5, 42, 2681, 0, 301, 91, 269, 1691, 3382, 1822, 41, 971, 250, 1, 1597, 1636, 444, 280, 1082, 2642, 2366, 2103, 9, 837, 3, 172, 2098, 936, 1127, 5, 101, 540, 299, 374, 70, 2098, 25, 1910, 1544, 169, 2103, 459, 4, 1493, 1161, 34, 642, 10, 346, 1279, 437, 218, 1096, 34, 770, 363, 2435, 1328, 1258, 1571, 2398, 413, 1654, 710, 603, 2235, 1543, 1607, 3177, 914, 1761, 144, 3, 840, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[300, 1434, 1492, 0, 74, 0, 3067, 46, 831, 1863, 201, 328, 2868, 3516, 0, 2562, 1347, 1672, 1673, 2213, 72, 956, 140, 1236, 6, 10, 3710, 2402, 1962, 26, 2434, 995, 2636, 0, 163, 874, 399, 452, 1827, 120, 144, 1, 1897, 3055, 0, 2841, 522, 903, 1521, 2730, 919, 77, 0, 6, 26, 2444, 0, 450, 35, 268, 2209, 3144, 2100, 14, 1217, 302, 0, 1614, 1996, 426, 228, 468, 2340, 1931, 1895, 8, 762, 6, 371, 1870, 604, 1081, 0, 33, 793, 644, 419, 51, 2136, 19, 1757, 1059, 184, 2305, 196, 0, 1732, 1432, 14, 910, 6, 547, 1628, 761, 305, 832, 7, 1089, 426, 2361, 1115, 1378, 1679, 2626, 500, 2050, 654, 607, 2014, 1232, 1466, 3015, 734, 927, 218, 1, 876, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[581, 1784, 731, 0, 21, 1, 3299, 60, 659, 2093, 220, 282, 2152, 3372, 0, 2752, 1439, 976, 2042, 2387, 77, 739, 120, 1060, 1, 4, 3675, 1915, 2201, 17, 1986, 661, 2266, 0, 381, 916, 641, 496, 1841, 65, 388, 0, 1680, 3315, 0, 3135, 313, 918, 2195, 2530, 997, 122, 0, 0, 10, 2651, 1, 298, 21, 244, 1895, 3134, 2176, 0, 1230, 240, 1, 1445, 1824, 532, 140, 729, 2603, 2172, 2060, 19, 816, 2, 286, 1655, 995, 801, 0, 7, 898, 591, 394, 28, 1959, 24, 1642, 1478, 158, 2257, 316, 0, 1777, 2011, 13, 722, 1, 990, 1330, 789, 245, 1186, 2, 800, 447, 2884, 1572, 798, 1896, 2799, 314, 1789, 585, 446, 1860, 1222, 1202, 3501, 839, 915, 135, 0, 764, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[673, 1099, 1804, 1, 151, 3, 2995, 33, 756, 1371, 234, 530, 2008, 3101, 3, 3198, 1649, 1499, 2585, 2614, 82, 862, 167, 1561, 11, 3, 3510, 1940, 1196, 15, 2034, 601, 2661, 0, 181, 815, 291, 660, 1848, 59, 110, 23, 1638, 3147, 0, 3354, 451, 491, 1784, 2628, 1106, 161, 0, 2, 32, 2407, 0, 464, 114, 286, 2062, 3400, 1248, 42, 970, 210, 0, 1606, 1970, 202, 275, 609, 2546, 2603, 1629, 3, 519, 5, 328, 1831, 791, 1181, 1, 38, 553, 474, 830, 148, 2106, 11, 1607, 1437, 266, 2084, 421, 1, 1348, 1149, 10, 1224, 5, 724, 1794, 889, 320, 1268, 7, 874, 569, 2819, 1563, 1489, 1708, 2123, 375, 2345, 830, 434, 1824, 1160, 1379, 2657, 795, 1592, 191, 17, 579, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[519, 1346, 1062, 2, 123, 3, 2972, 29, 400, 1076, 200, 563, 2128, 1990, 4, 2526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1725, 1410, 1728, 2809, 41, 940, 107, 983, 18, 11, 0, 1813, 1176, 3, 2118, 899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3184, 0, 251, 429, 245, 567, 1861, 134, 169, 0, 1786, 3111, 5, 3142, 342, 924, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1693, 2108, 1209, 123, 2, 7, 35, 2809, 2, 511, 97, 322, 1264, 3615, 1908, 56, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1188, 386, 0, 1335, 2357, 168, 568, 845, 3157, 2653, 1578, 18, 945, 0, 242, 2196, 23, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 954, 995, 3, 65, 606, 1037, 644, 55, 2197, 17, 2401, 1818, 124, 2310, 312, 0, 3471, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1504, 726, 74, 785, 2, 411, 1364, 550, 291, 1062, 5, 1085, 703, 2429, 1260, 1580, 1687, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1931, 2540, 508, 2019, 795, 522, 1644, 1220, 1598, 871, 903, 1354, 176, 3, 482, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14943, 15781, 16696, 16187, 17674, 17448, 15776, 16567] avg=16384.0000 max=17674 max_rank=4 min=14943 min_rank=0 imbalance_ratio=1.078735 max_load_reduction=113398 max_load_reduction_pct=0.865158 moved_to_echo=67465 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[455, 1649, 1319, 0, 29, 0, 3395, 52, 340, 1923, 238, 278, 2099, 2074, 0, 2602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1992, 1540, 1846, 2286, 95, 860, 144, 869, 8, 44, 0, 2047, 1994, 28, 2362, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2610, 0, 122, 774, 447, 392, 1915, 133, 95, 4, 1887, 3188, 2, 2825, 467, 893, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2167, 2423, 955, 61, 1, 4, 35, 2359, 0, 555, 12, 283, 1650, 3619, 2031, 8, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 244, 0, 1490, 2030, 264, 217, 682, 2275, 2255, 1818, 36, 742, 3, 539, 1609, 24, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 975, 1650, 1, 24, 713, 631, 405, 46, 1848, 11, 2327, 1725, 160, 2184, 314, 0, 3711, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1295, 1795, 13, 705, 0, 795, 1504, 731, 212, 1285, 9, 1282, 362, 2128, 1441, 1488, 1483, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1766, 2709, 285, 1520, 611, 586, 1465, 1031, 1533, 766, 929, 1064, 122, 2, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16453, 17143, 16324, 16608, 15678, 17111, 16798, 14957] avg=16384.0000 max=17143 max_rank=1 min=14957 min_rank=7 imbalance_ratio=1.046326 max_load_reduction=113929 max_load_reduction_pct=0.869209 moved_to_echo=64544 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[384, 1392, 1538, 0, 136, 0, 2610, 50, 567, 1312, 218, 515, 2714, 1882, 0, 2531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2225, 1409, 2019, 2425, 88, 862, 23, 1056, 9, 12, 0, 2140, 1554, 12, 2109, 697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2662, 0, 254, 495, 333, 463, 2048, 163, 150, 11, 1786, 3145, 0, 2990, 340, 796, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1798, 2292, 1011, 197, 0, 5, 10, 2913, 0, 640, 39, 385, 1915, 2930, 2088, 10, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1066, 337, 0, 1561, 2473, 500, 226, 532, 2825, 2515, 1867, 14, 719, 2, 242, 2281, 22, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 924, 0, 22, 611, 813, 923, 76, 2175, 11, 1681, 1305, 106, 2394, 393, 0, 3691, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1358, 1379, 52, 944, 0, 819, 1477, 586, 118, 1012, 1, 912, 642, 2546, 936, 827, 1428, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1352, 2532, 396, 2748, 1490, 512, 1666, 1651, 1694, 738, 760, 1006, 229, 0, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15849, 16640, 16153, 16636, 17195, 15859, 15281, 17459] avg=16384.0000 max=17459 max_rank=7 min=15281 min_rank=6 imbalance_ratio=1.065613 max_load_reduction=113613 max_load_reduction_pct=0.866798 moved_to_echo=65794 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[639, 1162, 1054, 1, 124, 0, 3303, 108, 814, 1120, 262, 752, 2344, 1814, 0, 2338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1245, 1881, 2474, 158, 899, 114, 1020, 42, 17, 0, 1957, 2071, 10, 2107, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2255, 11, 357, 687, 523, 381, 1516, 225, 156, 5, 1959, 3069, 10, 2881, 378, 784, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1959, 2582, 1452, 169, 9, 18, 17, 2442, 0, 313, 125, 474, 2005, 3093, 1253, 27, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1276, 359, 0, 2239, 1918, 1036, 308, 949, 2335, 2712, 1709, 32, 272, 16, 234, 1960, 21, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 2127, 0, 38, 504, 307, 727, 202, 2415, 69, 2205, 1108, 210, 2234, 422, 0, 3613, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1861, 1339, 28, 1051, 6, 427, 1314, 1260, 346, 976, 5, 760, 605, 2177, 696, 1299, 1767, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2016, 2297, 209, 2324, 563, 332, 1467, 1454, 1654, 913, 502, 1711, 255, 3, 688, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15835, 15561, 15695, 16327, 17389, 17713, 16153, 16399] avg=16384.0000 max=17713 max_rank=5 min=15561 min_rank=1 imbalance_ratio=1.081116 max_load_reduction=113359 max_load_reduction_pct=0.864861 moved_to_echo=67654 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[581, 1784, 731, 0, 21, 1, 3299, 60, 659, 2093, 220, 282, 2152, 2068, 0, 2752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1439, 976, 2042, 2375, 77, 739, 120, 1060, 1, 4, 0, 1915, 2201, 17, 1986, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2266, 0, 381, 916, 641, 496, 1841, 65, 388, 0, 1680, 3315, 0, 3135, 313, 918, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2195, 2530, 997, 122, 0, 0, 10, 2651, 1, 298, 21, 244, 1895, 3134, 2176, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1230, 240, 1, 1445, 1824, 532, 140, 729, 2603, 2172, 2060, 19, 816, 2, 286, 1655, 24, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 995, 801, 0, 7, 898, 591, 394, 28, 1959, 24, 1642, 1478, 158, 2257, 316, 0, 3675, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1777, 2011, 13, 722, 1, 990, 1330, 789, 245, 1186, 2, 800, 447, 2884, 1572, 798, 1970, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1896, 2799, 314, 1789, 585, 446, 1860, 1222, 1202, 1018, 839, 915, 135, 0, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16703, 15613, 16923, 16717, 15790, 15736, 17806, 15784] avg=16384.0000 max=17806 max_rank=6 min=15613 min_rank=1 imbalance_ratio=1.086792 max_load_reduction=113266 max_load_reduction_pct=0.864151 moved_to_echo=65116 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[300, 1434, 1492, 0, 74, 0, 3067, 46, 831, 1863, 201, 328, 2868, 2157, 0, 2562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1347, 1672, 1673, 2202, 72, 956, 140, 1236, 6, 10, 0, 2402, 1962, 26, 2434, 995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2636, 0, 163, 874, 399, 452, 1827, 120, 144, 1, 1897, 3055, 0, 2841, 522, 903, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 2730, 919, 77, 0, 6, 26, 2444, 0, 450, 35, 268, 2209, 3144, 2100, 14, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1217, 302, 0, 1614, 1996, 426, 228, 468, 2340, 1931, 1895, 8, 762, 6, 371, 1870, 25, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 604, 1081, 0, 33, 793, 644, 419, 51, 2136, 19, 1757, 1059, 184, 2305, 196, 0, 3710, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1732, 1432, 14, 910, 6, 547, 1628, 761, 305, 832, 7, 1089, 426, 2361, 1115, 1378, 1697, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1679, 2626, 500, 2050, 654, 607, 2014, 1232, 1466, 876, 734, 927, 218, 1, 876, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17223, 17133, 16426, 16405, 15470, 15433, 16520, 16462] avg=16384.0000 max=17223 max_rank=0 min=15433 min_rank=5 imbalance_ratio=1.051208 max_load_reduction=113849 max_load_reduction_pct=0.868599 moved_to_echo=63885 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[567, 1199, 1908, 1, 26, 0, 3192, 77, 686, 1458, 257, 489, 2736, 1866, 0, 2843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1263, 1421, 2018, 2271, 170, 1520, 145, 836, 12, 23, 0, 2162, 1770, 18, 2106, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2638, 12, 240, 1008, 148, 382, 1761, 138, 95, 0, 2144, 3062, 2, 2756, 505, 1193, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1785, 2071, 1384, 74, 2, 5, 42, 2681, 0, 301, 91, 269, 1691, 3382, 1822, 41, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 971, 250, 1, 1597, 1636, 444, 280, 1082, 2642, 2366, 2103, 9, 837, 3, 172, 2098, 28, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 936, 1127, 5, 101, 540, 299, 374, 70, 2098, 25, 1910, 1544, 169, 2103, 459, 4, 3690, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1493, 1161, 34, 642, 10, 346, 1279, 437, 218, 1096, 34, 770, 363, 2435, 1328, 1258, 1792, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1571, 2398, 413, 1654, 710, 603, 2235, 1543, 1607, 915, 914, 1761, 144, 3, 840, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17305, 16396, 16604, 16048, 16534, 15924, 14944, 17317] avg=16384.0000 max=17317 max_rank=7 min=14944 min_rank=6 imbalance_ratio=1.056946 max_load_reduction=113755 max_load_reduction_pct=0.867882 moved_to_echo=64719 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[673, 1099, 1804, 1, 151, 3, 2995, 33, 756, 1371, 234, 530, 2008, 1902, 3, 3198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1649, 1499, 2585, 2600, 82, 862, 167, 1561, 11, 3, 0, 1940, 1196, 15, 2034, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2661, 0, 181, 815, 291, 660, 1848, 59, 110, 23, 1638, 3147, 0, 3354, 451, 491, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1784, 2628, 1106, 161, 0, 2, 32, 2407, 0, 464, 114, 286, 2062, 3400, 1248, 42, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 970, 210, 0, 1606, 1970, 202, 275, 609, 2546, 2603, 1629, 3, 519, 5, 328, 1831, 22, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 791, 1181, 1, 38, 553, 474, 830, 148, 2106, 11, 1607, 1437, 266, 2084, 421, 1, 3510, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1348, 1149, 10, 1224, 5, 724, 1794, 889, 320, 1268, 7, 874, 569, 2819, 1563, 1489, 1495, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1708, 2123, 375, 2345, 830, 434, 1824, 1160, 1379, 773, 795, 1592, 191, 17, 579, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16761, 16805, 16251, 16144, 15342, 15848, 17794, 16127] avg=16384.0000 max=17794 max_rank=6 min=15342 min_rank=4 imbalance_ratio=1.086060 max_load_reduction=113278 max_load_reduction_pct=0.864243 moved_to_echo=65111 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[12672, 11264, 15872, 19456, 896, 7680, 1024, 8704, 128, 128, 0, 16384, 13952, 256, 17280, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[20992, 128, 2048, 6016, 3072, 3840, 14720, 1152, 1408, 128, 14848, 25216, 128, 23936, 3328, 6912, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[4224, 11136, 11008, 128, 768, 128, 24960, 512, 5120, 12288, 1920, 3840, 19072, 15872, 128, 21376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[14976, 19456, 9088, 1024, 128, 128, 256, 20736, 128, 3584, 640, 2560, 14720, 26368, 14720, 256, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[12416, 11008, 256, 7040, 128, 5120, 11776, 6016, 2176, 8832, 128, 7680, 4224, 19840, 9984, 10240, 13440, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[9472, 2432, 128, 12928, 16256, 3584, 2304, 6016, 20736, 19328, 14720, 256, 5632, 128, 2432, 15616, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[13952, 20096, 3072, 16512, 6272, 4096, 14208, 10624, 12160, 6912, 6400, 10368, 1536, 128, 5504, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[6784, 9984, 128, 384, 5248, 4864, 4736, 768, 17024, 256, 15616, 11520, 1408, 17920, 2944, 128, 29184, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 2454, 0, 0, 4096, 4096, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4093, 2, 91, 0, 4083, 0, 4094, 4, 4095, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 0, 0, 4083, 0, 0, 604, 4093, 0, 0, 1463, 1, 0, 4096, 2374, 0, 1, 291, 0, 0, 4096, 0, 0, 4090, 0, 0, 3202, 3539, 0, 528, 0, 2, 25, 0, 0, 0, 0, 0, 0, 3572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1899, 0, 0, 0, 4096, 4096, 3997, 0, 0, 0, 4096, 0, 44, 1, 0, 0, 434, 4096, 2, 0, 4086, 0, 0, 2763, 0, 4087, 7, 1, 1444, 0, 0, 0, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 901, 3, 0, 4095, 4068, 0, 0, 4090, 0, 0, 0, 1, 1, 1, 0, 4095, 0, 0, 4094, 4093, 0, 99, 0, 4096, 0, 4095, 502, 4096, 1, 6, 4095, 2, 1, 4096, 0, 4090, 0, 1, 0, 0, 4, 0, 4069, 0, 0, 994, 4095, 0, 1, 151, 26, 1, 4096, 2479, 0, 0, 375, 1, 0, 4095, 1, 0, 4093, 0, 0, 3039, 3324, 0, 74, 0, 0, 19, 0, 1, 0, 0, 0, 0, 3892, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4094, 1, 1754, 1, 1, 0, 4095, 4090, 3822, 0, 0, 0, 4096, 0, 98, 0, 1, 0, 638, 4096, 0, 0, 4093, 0, 1, 3053, 0, 4093, 0, 0, 3512, 0, 1, 0, 4077] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[3, 9, 0, 1292, 0, 1, 4070, 3903, 18, 4, 4082, 0, 5, 3, 0, 0, 0, 0, 4079, 3, 0, 4092, 4091, 0, 25, 1, 4040, 0, 4095, 61, 4084, 4, 1, 4083, 15, 4, 4074, 10, 4094, 0, 0, 0, 3, 14, 0, 4062, 8, 2, 1792, 4091, 0, 3, 1172, 0, 3, 4086, 3302, 1, 22, 2079, 0, 1, 4096, 1, 0, 4094, 2, 10, 3465, 3011, 10, 35, 5, 6, 11, 0, 2, 2, 0, 2, 11, 3489, 0, 0, 10, 0, 8, 31, 0, 8, 0, 0, 2, 4084, 16, 1715, 0, 4, 0, 4093, 4093, 4088, 0, 3, 16, 4078, 1, 51, 0, 2, 0, 353, 4081, 0, 4, 4083, 0, 0, 1119, 11, 4093, 0, 0, 1796, 0, 9, 3, 4078] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1480, 0, 0, 4089, 4090, 0, 8, 4086, 1, 0, 0, 0, 5, 0, 6, 4093, 0, 0, 4090, 4082, 2, 104, 0, 4093, 5, 4096, 8, 4089, 0, 2, 4091, 0, 8, 4091, 2, 4096, 0, 3, 21, 0, 5, 0, 4005, 0, 2, 814, 4095, 0, 0, 227, 0, 0, 4095, 3710, 0, 11, 199, 0, 0, 4095, 0, 0, 4091, 0, 0, 3815, 2882, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3290, 2, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 4096, 1, 2951, 4, 0, 0, 4096, 4057, 4094, 0, 0, 4, 4094, 0, 18, 0, 0, 0, 389, 4091, 0, 0, 4094, 0, 0, 3301, 0, 4096, 0, 0, 1428, 0, 0, 0, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1486, 9, 0, 4096, 4096, 0, 0, 4094, 1, 0, 0, 1, 12, 0, 0, 4096, 0, 0, 4081, 4093, 0, 150, 0, 4094, 0, 4096, 89, 4096, 0, 0, 4096, 2, 0, 4096, 1, 4094, 0, 3, 0, 0, 0, 0, 4090, 0, 0, 453, 4095, 0, 0, 313, 13, 0, 4096, 3528, 0, 2, 453, 0, 0, 4096, 0, 0, 4091, 0, 0, 4041, 3552, 0, 4, 0, 2, 4, 0, 0, 0, 0, 0, 0, 3102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2455, 0, 0, 0, 4096, 4096, 4059, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 966, 4096, 0, 0, 4096, 0, 1, 1804, 0, 4094, 0, 1, 2202, 0, 0, 0, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1307, 0, 2, 4091, 3779, 0, 3, 4081, 0, 0, 0, 0, 2, 0, 0, 4095, 0, 0, 4096, 4088, 20, 31, 0, 4078, 0, 4094, 1, 4083, 0, 0, 4090, 7, 4, 4095, 8, 4095, 0, 0, 2, 0, 1, 1, 4051, 0, 3, 643, 4084, 0, 0, 1904, 19, 52, 4091, 2739, 0, 9, 676, 0, 0, 4095, 6, 0, 4089, 0, 5, 3735, 3782, 1, 455, 5, 11, 8, 1, 0, 0, 7, 0, 0, 3796, 1, 4, 0, 6, 0, 0, 0, 0, 2, 3, 0, 4073, 0, 1003, 0, 0, 6, 4093, 4009, 4076, 0, 0, 0, 4092, 1, 8, 0, 0, 0, 1661, 4094, 23, 0, 4095, 0, 0, 1075, 0, 4096, 0, 0, 2135, 0, 3, 0, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1746, 0, 0, 4096, 4093, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4078, 4095, 1, 9, 0, 4095, 0, 4071, 4, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 0, 0, 4088, 0, 0, 1082, 4093, 0, 0, 511, 62, 0, 4096, 2681, 0, 2, 416, 0, 0, 4096, 0, 0, 4095, 0, 0, 3767, 3595, 0, 22, 0, 1, 78, 0, 0, 0, 0, 0, 0, 3803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 1780, 0, 0, 0, 4096, 4096, 4084, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 359, 4096, 1, 0, 4067, 0, 0, 2761, 0, 4096, 10, 1, 2056, 0, 0, 0, 4051] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1, 1, 0, 2708, 0, 0, 4094, 4090, 0, 0, 4094, 3, 0, 1, 0, 15, 0, 0, 4095, 1, 2, 4095, 4096, 4, 8, 1, 4076, 0, 4095, 89, 4092, 0, 0, 4095, 0, 5, 4095, 5, 4094, 1, 12, 0, 0, 0, 0, 4088, 0, 0, 786, 4087, 0, 0, 1220, 7, 2, 4095, 2739, 0, 6, 659, 1, 0, 4095, 1, 0, 4070, 1, 1, 2896, 2667, 1, 130, 1, 1, 25, 0, 1, 0, 0, 0, 0, 4087, 2, 0, 0, 0, 0, 2, 0, 1, 0, 1, 1, 4095, 0, 955, 0, 1, 0, 4094, 4096, 3951, 0, 1, 0, 4094, 0, 9, 0, 1, 0, 773, 4095, 0, 1, 4094, 0, 0, 3788, 0, 4070, 0, 0, 1249, 1, 2, 1, 4059] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 901, 3, 0, 4095, 4068, 0, 0, 4090, 0, 0, 0, 1, 1, 1282, 636, 319, 130, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 4093, 0, 99, 0, 4096, 0, 0, 502, 3960, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4095, 2, 1, 4096, 0, 4090, 0, 1, 0, 0, 4, 0, 4069, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 994, 4095, 0, 1, 151, 26, 1, 4096, 2479, 0, 0, 375, 1, 0, 2813, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3039, 3324, 0, 74, 0, 0, 19, 0, 1, 0, 0, 0, 3965, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4094, 1, 1754, 4095, 2749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 3459, 4090, 3822, 0, 0, 0, 4096, 0, 98, 0, 1, 0, 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 3053, 0, 3774, 0, 0, 3512, 0, 1, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15647, 16846, 16379, 15033, 15862, 16588, 16206, 18511] avg=16384.0000 max=18511 max_rank=7 min=15033 min_rank=3 imbalance_ratio=1.129822 max_load_reduction=112561 max_load_reduction_pct=0.858772 moved_to_echo=67167 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 9, 0, 1292, 0, 1, 4070, 3903, 18, 4, 4082, 0, 5, 3, 0, 0, 1282, 636, 319, 131, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4092, 4091, 0, 25, 1, 4040, 0, 0, 61, 3949, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4083, 15, 4, 4074, 10, 4094, 0, 0, 0, 3, 14, 0, 4062, 8, 2, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1792, 4091, 0, 3, 1172, 0, 3, 4086, 3302, 1, 22, 2079, 0, 1, 2814, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 10, 3465, 3011, 10, 35, 5, 6, 11, 0, 2, 2, 0, 2, 3964, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3489, 0, 0, 10, 0, 8, 31, 0, 8, 0, 0, 2, 4084, 16, 1715, 4079, 2738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3457, 4093, 4088, 0, 3, 16, 4078, 1, 51, 0, 2, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4083, 0, 0, 1119, 11, 3774, 0, 0, 1796, 0, 9, 3, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15878, 16266, 16385, 19367, 15962, 16191, 16146, 14877] avg=16384.0000 max=19367 max_rank=3 min=14877 min_rank=7 imbalance_ratio=1.182068 max_load_reduction=111705 max_load_reduction_pct=0.852242 moved_to_echo=63176 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1486, 9, 0, 4096, 4096, 0, 0, 4094, 1, 0, 0, 1, 12, 1282, 637, 319, 131, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4093, 0, 150, 0, 4094, 0, 0, 89, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 1, 4094, 0, 3, 0, 0, 0, 0, 4090, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 4095, 0, 0, 313, 13, 0, 4096, 3528, 0, 2, 453, 0, 0, 2814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4041, 3552, 0, 4, 0, 2, 4, 0, 0, 0, 0, 0, 3965, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2455, 4096, 2748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3459, 4096, 4059, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 1804, 0, 3775, 0, 1, 2202, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16285, 16467, 16397, 15767, 17007, 16497, 16681, 15971] avg=16384.0000 max=17007 max_rank=4 min=15767 min_rank=3 imbalance_ratio=1.038025 max_load_reduction=114065 max_load_reduction_pct=0.870247 moved_to_echo=66156 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2454, 0, 0, 4096, 4096, 0, 0, 4095, 2, 0, 0, 0, 0, 1282, 637, 319, 130, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4093, 2, 91, 0, 4083, 0, 0, 4, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 0, 0, 4083, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 604, 4093, 0, 0, 1463, 1, 0, 4096, 2374, 0, 1, 291, 0, 0, 2814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 3202, 3539, 0, 528, 0, 2, 25, 0, 0, 0, 0, 0, 3964, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1899, 4096, 2749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3459, 4096, 3997, 0, 0, 0, 4096, 0, 44, 1, 0, 0, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4086, 0, 0, 2763, 0, 3768, 7, 1, 1444, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17232, 16327, 16388, 15737, 16697, 16409, 16127, 16155] avg=16384.0000 max=17232 max_rank=0 min=15737 min_rank=3 imbalance_ratio=1.051758 max_load_reduction=113840 max_load_reduction_pct=0.868530 moved_to_echo=65388 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1480, 0, 0, 4089, 4090, 0, 8, 4086, 1, 0, 0, 0, 5, 1282, 637, 319, 131, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4090, 4082, 2, 104, 0, 4093, 5, 0, 8, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4091, 0, 8, 4091, 2, 4096, 0, 3, 21, 0, 5, 0, 4005, 0, 2, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 814, 4095, 0, 0, 227, 0, 0, 4095, 3710, 0, 11, 199, 0, 0, 2813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 3815, 2882, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3290, 2, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 4096, 1, 2951, 4093, 2745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3459, 4057, 4094, 0, 0, 4, 4094, 0, 18, 0, 0, 0, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3301, 0, 3777, 0, 0, 1428, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16248, 16344, 16341, 15964, 16182, 17182, 16119, 16692] avg=16384.0000 max=17182 max_rank=5 min=15964 min_rank=3 imbalance_ratio=1.048706 max_load_reduction=113890 max_load_reduction_pct=0.868912 moved_to_echo=66175 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1746, 0, 0, 4096, 4093, 0, 0, 4095, 1, 0, 0, 0, 0, 1282, 637, 319, 129, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4095, 1, 9, 0, 4095, 0, 0, 4, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 0, 0, 4088, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 4093, 0, 0, 511, 62, 0, 4096, 2681, 0, 2, 416, 0, 0, 2814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3767, 3595, 0, 22, 0, 1, 78, 0, 0, 0, 0, 0, 3942, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 1780, 4096, 2749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3459, 4096, 4084, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4067, 0, 0, 2761, 0, 3777, 10, 1, 2056, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16519, 16242, 16393, 15757, 16847, 16488, 16102, 16724] avg=16384.0000 max=16847 max_rank=4 min=15757 min_rank=3 imbalance_ratio=1.028259 max_load_reduction=114225 max_load_reduction_pct=0.871468 moved_to_echo=66161 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1307, 0, 2, 4091, 3779, 0, 3, 4081, 0, 0, 0, 0, 2, 1287, 639, 325, 131, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 20, 31, 0, 4078, 0, 0, 1, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 7, 4, 4095, 8, 4095, 0, 0, 2, 0, 1, 1, 4051, 0, 3, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 643, 4084, 0, 0, 1904, 19, 52, 4091, 2739, 0, 9, 676, 0, 0, 2808, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 5, 3735, 3782, 1, 455, 5, 11, 8, 1, 0, 0, 7, 0, 3963, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 1, 4, 0, 6, 0, 0, 0, 0, 2, 3, 0, 4073, 0, 1003, 4095, 2747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3454, 4009, 4076, 0, 0, 0, 4092, 1, 8, 0, 0, 0, 1661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4095, 0, 0, 1075, 0, 3771, 0, 0, 2135, 0, 3, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15770, 16252, 16379, 17031, 17409, 15730, 17307, 15194] avg=16384.0000 max=17409 max_rank=4 min=15194 min_rank=7 imbalance_ratio=1.062561 max_load_reduction=113663 max_load_reduction_pct=0.867180 moved_to_echo=65640 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 2708, 0, 0, 4094, 4090, 0, 0, 4094, 3, 0, 1, 0, 15, 1282, 636, 317, 130, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4095, 4096, 4, 8, 1, 4076, 0, 0, 89, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 5, 4095, 5, 4094, 1, 12, 0, 0, 0, 0, 4088, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 786, 4087, 0, 0, 1220, 7, 2, 4095, 2739, 0, 6, 659, 1, 0, 2813, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 1, 1, 2896, 2667, 1, 130, 1, 1, 25, 0, 1, 0, 0, 0, 3965, 1347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 2, 0, 0, 0, 0, 2, 0, 1, 0, 1, 1, 4095, 0, 955, 4095, 2748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3458, 4096, 3951, 0, 1, 0, 4094, 0, 9, 0, 1, 0, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 3788, 0, 3753, 0, 0, 1249, 1, 2, 1, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17493, 16328, 16410, 16416, 15106, 15987, 16384, 16948] avg=16384.0000 max=17493 max_rank=0 min=15106 min_rank=4 imbalance_ratio=1.067688 max_load_reduction=113579 max_load_reduction_pct=0.866539 moved_to_echo=64425 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[0, 32768, 128, 128, 28032, 26368, 128, 1408, 128, 128, 256, 128, 128, 128, 128, 128, 31744, 10880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 128, 32768, 32768, 128, 640, 128, 32768, 128, 0, 768, 31744, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 27776, 32640, 32256, 0, 128, 128, 32768, 128, 256, 128, 128, 0, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[7168, 32768, 0, 128, 7040, 128, 128, 32768, 23552, 128, 128, 5248, 128, 128, 22528, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 128, 128, 32768, 128, 32768, 128, 128, 128, 128, 128, 128, 32640, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 0, 13440, 128, 128, 32768, 32256, 128, 128, 32768, 128, 128, 128, 128, 128, 10368, 5120, 2560, 1152, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 29056, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 14592, 32768, 22016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 32768, 0, 128, 19712, 128, 30208, 128, 128, 15872, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1, 4091, 123, 2, 4074, 3, 4, 2417, 12, 1, 51, 6, 4, 4040, 0, 4089, 5, 5, 2, 4052, 407, 3, 4085, 7, 4088, 4091, 0, 0, 4093, 4081, 1570, 1, 3819, 12, 4085, 0, 6, 1, 4092, 3, 4, 3915, 4087, 5, 0, 4, 2, 4078, 8, 84, 4, 7, 0, 0, 2355, 1526, 0, 6, 4080, 4093, 0, 6, 0, 0, 0, 5, 4086, 8, 0, 2, 0, 0, 11, 4090, 0, 3, 4092, 0, 2, 47, 4070, 0, 3, 280, 7, 0, 28, 5, 4090, 0, 0, 1, 6, 4075, 6, 4056, 0, 16, 6, 5, 0, 0, 4082, 0, 9, 8, 2, 0, 0, 5, 4091, 28, 7, 5, 12, 0, 3, 8, 4088, 4, 0, 0, 895, 0, 3971, 0, 0, 3164] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 4090, 5, 0, 4089, 0, 0, 1732, 1, 1, 45, 0, 0, 3783, 0, 4095, 0, 0, 0, 4085, 443, 0, 4091, 0, 4096, 4096, 0, 0, 4089, 4093, 2867, 0, 3660, 0, 4096, 3, 0, 3, 4090, 0, 0, 4073, 4096, 0, 6, 0, 0, 4094, 0, 512, 0, 0, 0, 0, 1747, 889, 2, 0, 4087, 4069, 0, 0, 5, 0, 0, 0, 4096, 7, 0, 0, 0, 2, 10, 4096, 0, 0, 4096, 0, 7, 67, 4087, 0, 0, 885, 0, 0, 33, 0, 4094, 12, 0, 0, 0, 4044, 0, 4084, 3, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 26, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 290, 6, 3829, 0, 0, 3868] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[5, 4023, 12, 1, 4096, 0, 0, 2369, 0, 0, 379, 6, 0, 3734, 0, 4096, 0, 0, 0, 4066, 117, 0, 4089, 0, 4089, 4095, 0, 0, 4083, 4090, 2147, 0, 3926, 12, 4095, 8, 0, 1, 4095, 0, 0, 3834, 4096, 0, 13, 0, 0, 4083, 0, 1248, 0, 0, 0, 1, 1908, 412, 0, 0, 4095, 4081, 0, 0, 6, 0, 0, 0, 4089, 6, 1, 0, 0, 0, 23, 4096, 0, 0, 4087, 0, 6, 175, 4090, 0, 0, 376, 0, 4, 199, 0, 4095, 7, 0, 0, 0, 4014, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 6, 1, 0, 4096, 64, 0, 0, 9, 0, 0, 0, 4093, 0, 1, 0, 212, 6, 3776, 0, 0, 3943] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 4094, 0, 2, 4095, 0, 0, 3764, 0, 2, 10, 0, 0, 3842, 0, 4095, 1, 0, 1, 4092, 271, 0, 4096, 0, 4096, 4095, 0, 0, 4095, 4094, 1497, 0, 3565, 0, 4096, 0, 0, 43, 4094, 1, 1, 3601, 4096, 0, 0, 0, 0, 4095, 0, 1910, 0, 0, 0, 0, 1984, 611, 0, 0, 4094, 4094, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 4096, 1, 0, 141, 4096, 0, 1, 312, 0, 0, 35, 0, 4095, 10, 0, 0, 0, 4085, 1, 4082, 2, 0, 0, 0, 1, 0, 4095, 1, 0, 0, 1, 1, 0, 0, 4096, 71, 0, 0, 2, 0, 0, 0, 4095, 0, 0, 0, 206, 1, 3374, 0, 1, 3449] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 4061, 11, 0, 4096, 0, 0, 2787, 0, 0, 13, 0, 0, 4056, 0, 4096, 0, 1, 0, 4094, 41, 0, 4096, 0, 4096, 4096, 0, 0, 4093, 4096, 2379, 0, 3353, 0, 4096, 0, 1, 13, 4081, 0, 0, 3853, 4096, 0, 0, 0, 0, 4095, 0, 1036, 0, 0, 0, 0, 1438, 934, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 20, 4096, 0, 0, 224, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 4091, 0, 4093, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 13, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 975, 0, 3927, 0, 0, 3663] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 4089, 1, 0, 4096, 0, 0, 2620, 0, 0, 62, 0, 0, 3273, 0, 4096, 0, 0, 0, 4082, 469, 0, 4096, 0, 4096, 4096, 0, 0, 4095, 4096, 2774, 0, 3913, 0, 4096, 0, 1, 5, 4090, 0, 0, 3965, 4096, 0, 0, 0, 0, 4096, 0, 90, 0, 0, 0, 0, 2545, 277, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 38, 4096, 0, 0, 215, 0, 0, 105, 0, 4095, 11, 0, 0, 0, 4008, 0, 4070, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 388, 0, 4026, 0, 0, 4042] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 4041, 11, 0, 4092, 1, 1, 3672, 0, 1, 3, 0, 1, 3980, 0, 4094, 1, 1, 1, 4072, 376, 0, 4095, 1, 4095, 4095, 0, 0, 4062, 4094, 1985, 0, 3743, 0, 4095, 1, 0, 151, 4069, 1, 1, 4065, 4095, 1, 2, 1, 0, 4095, 0, 1668, 1, 0, 0, 0, 1527, 785, 0, 1, 4090, 4095, 0, 1, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 2, 4095, 1, 1, 4094, 0, 0, 128, 4095, 0, 1, 138, 1, 0, 1, 1, 4095, 8, 0, 1, 0, 4085, 1, 4076, 0, 0, 0, 1, 1, 0, 4086, 0, 1, 1, 1, 0, 0, 1, 4094, 204, 1, 1, 0, 0, 1, 1, 4093, 1, 0, 1, 111, 0, 3265, 0, 0, 3023] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 4082, 1, 0, 4096, 0, 0, 3642, 0, 1, 23, 0, 0, 3834, 0, 4096, 0, 0, 0, 4092, 93, 0, 4096, 0, 4096, 4096, 0, 0, 4094, 4096, 1751, 0, 2977, 0, 4096, 0, 1, 43, 4079, 0, 0, 4018, 4096, 0, 0, 0, 0, 4096, 0, 2300, 0, 0, 0, 0, 1999, 341, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 12, 4096, 0, 0, 624, 0, 0, 5, 0, 4095, 1, 0, 0, 0, 3893, 0, 4093, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 70, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 160, 0, 3959, 0, 0, 3063] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 4091, 123, 2, 4074, 3, 4, 2417, 12, 1, 51, 6, 4, 4040, 0, 1426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 2, 4052, 407, 3, 1719, 7, 0, 0, 0, 0, 4093, 4081, 1570, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3819, 12, 627, 0, 6, 1, 4092, 3, 4, 3915, 0, 5, 0, 4, 2, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 84, 4, 7, 0, 0, 2355, 1526, 0, 6, 4080, 4093, 0, 6, 0, 0, 3033, 1388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4086, 8, 0, 2, 0, 0, 11, 4090, 0, 3, 4092, 0, 2, 47, 2366, 1275, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 3, 280, 7, 0, 28, 5, 3729, 0, 0, 1, 6, 4075, 6, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 6, 5, 0, 0, 4082, 0, 9, 8, 2, 0, 0, 5, 4091, 28, 4091, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 12, 0, 3, 8, 4088, 4, 0, 0, 895, 0, 3971, 0, 0, 3164, 4088, 425, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16255, 15945, 16568, 16590, 16348, 16266, 16369, 16731] avg=16384.0000 max=16731 max_rank=7 min=15945 min_rank=1 imbalance_ratio=1.021179 max_load_reduction=114341 max_load_reduction_pct=0.872353 moved_to_echo=65714 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 5, 0, 4089, 0, 0, 1732, 1, 1, 45, 0, 0, 3783, 0, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 443, 0, 1722, 0, 0, 0, 0, 0, 4089, 4093, 2867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3660, 0, 628, 3, 0, 3, 4090, 0, 0, 4073, 0, 0, 6, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 0, 0, 1747, 889, 2, 0, 4087, 4069, 0, 0, 5, 0, 3041, 1390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 2, 10, 4096, 0, 0, 4096, 0, 7, 67, 2369, 1277, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 885, 0, 0, 33, 0, 3733, 12, 0, 0, 0, 4044, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 26, 4096, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 290, 6, 3829, 0, 0, 3868, 4096, 427, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15174, 17299, 16557, 15742, 16388, 16878, 16356, 16678] avg=16384.0000 max=17299 max_rank=1 min=15174 min_rank=0 imbalance_ratio=1.055847 max_load_reduction=113773 max_load_reduction_pct=0.868019 moved_to_echo=66300 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 4041, 11, 0, 4092, 1, 1, 3672, 0, 1, 3, 0, 1, 3980, 0, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4072, 376, 0, 1723, 1, 0, 0, 0, 0, 4062, 4094, 1985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3743, 0, 629, 1, 0, 151, 4069, 1, 1, 4065, 0, 1, 2, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1668, 1, 0, 0, 0, 1527, 785, 0, 1, 4090, 4095, 0, 1, 0, 0, 3040, 1390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 2, 4095, 1, 1, 4094, 0, 0, 128, 2372, 1276, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 138, 1, 0, 1, 1, 3734, 8, 0, 1, 0, 4085, 1, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4086, 0, 1, 1, 1, 0, 0, 1, 4094, 204, 4095, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 4093, 1, 0, 1, 111, 0, 3265, 0, 0, 3023, 4095, 426, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17232, 16316, 16759, 16598, 16426, 16142, 16519, 15080] avg=16384.0000 max=17232 max_rank=0 min=15080 min_rank=7 imbalance_ratio=1.051758 max_load_reduction=113840 max_load_reduction_pct=0.868530 moved_to_echo=64167 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 2, 4095, 0, 0, 3764, 0, 2, 10, 0, 0, 3842, 0, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4092, 271, 0, 1724, 0, 0, 0, 0, 0, 4095, 4094, 1497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3565, 0, 628, 0, 0, 43, 4094, 1, 1, 3601, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1910, 0, 0, 0, 0, 1984, 611, 0, 0, 4094, 4094, 1, 0, 0, 0, 3041, 1390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 4096, 1, 0, 141, 2372, 1277, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 312, 0, 0, 35, 0, 3734, 10, 0, 0, 0, 4085, 1, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 4095, 1, 0, 0, 1, 1, 0, 0, 4096, 71, 4095, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 0, 0, 0, 206, 1, 3374, 0, 1, 3449, 4096, 427, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17237, 15775, 16028, 17125, 16441, 16356, 16398, 15712] avg=16384.0000 max=17237 max_rank=0 min=15712 min_rank=7 imbalance_ratio=1.052063 max_load_reduction=113835 max_load_reduction_pct=0.868492 moved_to_echo=64907 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4061, 11, 0, 4096, 0, 0, 2787, 0, 0, 13, 0, 0, 4056, 0, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 41, 0, 1724, 0, 0, 0, 0, 0, 4093, 4096, 2379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3353, 0, 628, 0, 1, 13, 4081, 0, 0, 3853, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 0, 0, 0, 0, 1438, 934, 0, 0, 4095, 4096, 0, 0, 0, 0, 3041, 1391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 20, 2372, 1277, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 224, 0, 0, 0, 0, 3733, 2, 0, 0, 0, 4091, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 13, 4096, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 975, 0, 3927, 0, 0, 3663, 4096, 427, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16452, 16428, 16024, 16031, 16318, 16239, 16335, 17245] avg=16384.0000 max=17245 max_rank=7 min=16024 min_rank=2 imbalance_ratio=1.052551 max_load_reduction=113827 max_load_reduction_pct=0.868431 moved_to_echo=66137 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[5, 4023, 12, 1, 4096, 0, 0, 2369, 0, 0, 379, 6, 0, 3734, 0, 1421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 117, 0, 1716, 0, 0, 0, 0, 0, 4083, 4090, 2147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 12, 624, 8, 0, 1, 4095, 0, 0, 3834, 0, 0, 13, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1248, 0, 0, 0, 1, 1908, 412, 0, 0, 4095, 4081, 0, 0, 6, 0, 3042, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 6, 1, 0, 0, 0, 23, 4096, 0, 0, 4087, 0, 6, 175, 2373, 1281, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 376, 0, 4, 199, 0, 3727, 7, 0, 0, 0, 4014, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 6, 1, 0, 4096, 64, 4095, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4093, 0, 1, 0, 212, 6, 3776, 0, 0, 3943, 4089, 429, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16046, 16219, 16596, 16187, 16505, 16506, 16394, 16619] avg=16384.0000 max=16619 max_rank=7 min=16046 min_rank=0 imbalance_ratio=1.014343 max_load_reduction=114453 max_load_reduction_pct=0.873207 moved_to_echo=66024 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 1, 0, 4096, 0, 0, 2620, 0, 0, 62, 0, 0, 3273, 0, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 469, 0, 1724, 0, 0, 0, 0, 0, 4095, 4096, 2774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 628, 0, 1, 5, 4090, 0, 0, 3965, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 2545, 277, 0, 0, 4096, 4096, 0, 0, 0, 0, 3041, 1391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 38, 2372, 1277, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 215, 0, 0, 105, 0, 3734, 11, 0, 0, 0, 4008, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 388, 0, 4026, 0, 0, 4042, 4096, 427, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15569, 17240, 16698, 15536, 16336, 16239, 16312, 17142] avg=16384.0000 max=17240 max_rank=1 min=15536 min_rank=3 imbalance_ratio=1.052246 max_load_reduction=113832 max_load_reduction_pct=0.868469 moved_to_echo=66029 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4082, 1, 0, 4096, 0, 0, 3642, 0, 1, 23, 0, 0, 3834, 0, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 93, 0, 1724, 0, 0, 0, 0, 0, 4094, 4096, 1751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2977, 0, 628, 0, 1, 43, 4079, 0, 0, 4018, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2300, 0, 0, 0, 0, 1999, 341, 0, 0, 4096, 4095, 0, 0, 0, 0, 3041, 1391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 12, 2372, 1277, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 624, 0, 0, 5, 0, 3734, 1, 0, 0, 0, 3893, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 70, 4096, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 160, 0, 3959, 0, 0, 3063, 4096, 427, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17107, 15850, 15842, 17263, 16310, 16446, 16389, 15865] avg=16384.0000 max=17263 max_rank=3 min=15842 min_rank=2 imbalance_ratio=1.053650 max_load_reduction=113809 max_load_reduction_pct=0.868294 moved_to_echo=65010 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32640, 256, 128, 32768, 128, 128, 23040, 128, 128, 640, 128, 128, 30592, 0, 11520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[29056, 128, 5120, 128, 128, 384, 32768, 128, 128, 31360, 0, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32640, 2304, 128, 13824, 128, 0, 0, 0, 0, 32768, 32768, 17024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 0, 128, 128, 32768, 128, 128, 32768, 128, 128, 640, 19072, 10240, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 128, 128, 128, 128, 128, 128, 128, 32768, 512, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 8960, 128, 128, 0, 128, 15616, 5888, 128, 128, 32768, 32768, 128, 128, 128, 0, 24320, 11136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 0, 128, 3072, 128, 128, 512, 128, 29952, 128, 0, 128, 128, 32384, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 128, 128, 128, 3328, 128, 30208, 0, 128, 28288, 32768, 3456, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[7, 0, 7, 8, 0, 3, 0, 3048, 3389, 4090, 2103, 0, 0, 1, 4084, 7, 4076, 0, 7, 4087, 6, 0, 0, 218, 0, 0, 5, 1, 1, 536, 4, 0, 4088, 0, 0, 7, 9, 1, 4090, 5, 0, 4089, 4089, 4089, 4, 4072, 4089, 8, 4088, 4086, 4085, 6, 0, 4089, 0, 618, 0, 7, 4, 0, 12, 0, 13, 4085, 11, 0, 6, 4089, 34, 7, 0, 4089, 4076, 7, 7, 7, 0, 5, 0, 5, 4087, 0, 4092, 4089, 1, 0, 5, 1, 4088, 3038, 2, 1024, 4, 0, 0, 6, 5, 5, 0, 3992, 9, 0, 7, 0, 3, 5, 0, 0, 6, 4, 0, 2946, 0, 7, 5, 2, 3596, 0, 0, 7, 0, 8, 4089, 4088, 0, 3, 4084, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3374, 2858, 4096, 2343, 0, 0, 0, 4096, 0, 4096, 5, 0, 4095, 0, 0, 0, 106, 0, 0, 0, 0, 0, 1171, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 4096, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 2039, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 6, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3619, 0, 2645, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 1, 0, 0, 23, 0, 0, 0, 0, 1499, 0, 0, 0, 0, 827, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3410, 3491, 4096, 1126, 0, 0, 0, 4096, 0, 4086, 0, 0, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 369, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4095, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 1592, 0, 0, 0, 0, 0, 14, 0, 4096, 0, 0, 0, 4096, 57, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 2245, 0, 3327, 0, 0, 0, 0, 0, 0, 0, 3939, 0, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 2266, 3, 0, 0, 0, 2746, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 3390, 3186, 4095, 1671, 0, 0, 0, 4096, 0, 4093, 4, 0, 4096, 0, 0, 1, 326, 0, 0, 0, 1, 0, 833, 0, 0, 4094, 1, 1, 0, 0, 0, 4095, 0, 0, 4095, 4092, 4094, 0, 4096, 4096, 0, 4095, 4095, 4096, 0, 1, 4096, 0, 331, 0, 0, 0, 0, 1, 4, 0, 4096, 0, 0, 0, 4095, 38, 0, 1, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4096, 1635, 1, 3619, 0, 1, 3, 0, 0, 0, 0, 3907, 0, 0, 0, 4, 0, 0, 34, 1, 0, 0, 0, 3689, 6, 0, 0, 0, 1921, 1, 0, 0, 0, 0, 4096, 4082, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 2512, 2869, 4095, 2422, 0, 0, 0, 4096, 0, 4076, 0, 0, 4096, 0, 0, 1, 115, 0, 0, 7, 0, 0, 444, 0, 0, 4096, 0, 5, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4051, 4096, 0, 4091, 4096, 4096, 0, 7, 4096, 0, 440, 0, 0, 0, 10, 5, 1, 0, 4096, 0, 0, 0, 4096, 14, 0, 1, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 4091, 0, 0, 0, 0, 4095, 1401, 0, 3050, 0, 0, 0, 0, 0, 0, 5, 3818, 0, 0, 0, 14, 0, 0, 57, 0, 0, 0, 0, 3741, 21, 0, 0, 0, 3729, 0, 0, 0, 1, 0, 4094, 4078, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 0, 1, 0, 3659, 3618, 4095, 595, 0, 0, 1, 4094, 1, 4095, 19, 1, 4095, 1, 0, 0, 99, 0, 0, 0, 0, 1, 1519, 1, 0, 4095, 0, 0, 0, 1, 1, 4095, 0, 1, 4095, 4095, 4095, 0, 4093, 4095, 1, 4095, 4095, 4095, 1, 0, 4095, 0, 78, 0, 1, 1, 0, 1, 0, 2, 4095, 1, 0, 0, 4095, 43, 1, 0, 4095, 4087, 1, 1, 1, 0, 0, 0, 0, 4094, 0, 4095, 4095, 0, 0, 1, 0, 4095, 802, 0, 3151, 0, 0, 0, 0, 1, 0, 0, 4057, 1, 0, 0, 0, 1, 1, 10, 0, 1, 1, 1, 3617, 0, 1, 0, 1, 3332, 0, 0, 1, 0, 1, 4095, 4073, 0, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3090, 3696, 4096, 440, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 96, 0, 0, 0, 0, 0, 1309, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 1306, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 85, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 2018, 0, 3110, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3088, 0, 0, 0, 0, 2306, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 0, 1525, 3504, 4096, 2310, 0, 0, 0, 4096, 0, 4078, 0, 0, 4096, 0, 0, 0, 71, 0, 0, 4, 0, 0, 41, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4083, 4096, 0, 4094, 4096, 4095, 0, 0, 4096, 0, 2623, 0, 0, 0, 2, 8, 13, 2, 4096, 0, 0, 0, 4096, 238, 0, 0, 4096, 4068, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4095, 4091, 0, 0, 0, 0, 4096, 3131, 0, 3649, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 0, 7, 0, 0, 31, 0, 0, 0, 0, 1915, 0, 0, 0, 0, 1567, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 7, 8, 0, 3, 0, 3048, 3389, 4090, 2103, 0, 0, 1, 4084, 7, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 7, 4087, 6, 0, 0, 218, 0, 0, 5, 1, 1, 536, 4, 0, 4088, 2752, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 7, 9, 1, 0, 5, 0, 10, 4089, 4089, 4, 4072, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4086, 4085, 6, 0, 0, 0, 618, 0, 7, 4, 0, 12, 0, 13, 2953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 6, 4089, 34, 7, 0, 4089, 4076, 7, 7, 7, 0, 5, 0, 5, 3645, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3005, 0, 4092, 4089, 1, 0, 5, 1, 0, 3038, 2, 1024, 4, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 3992, 9, 0, 7, 0, 3, 5, 0, 0, 6, 4, 0, 2946, 4089, 4090, 1337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 2, 3596, 0, 0, 7, 0, 8, 4089, 4088, 0, 3, 4084, 0, 848, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16981, 16215, 16382, 15872, 16373, 15267, 16498, 17484] avg=16384.0000 max=17484 max_rank=7 min=15267 min_rank=5 imbalance_ratio=1.067139 max_load_reduction=113588 max_load_reduction_pct=0.866608 moved_to_echo=65622 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 0, 1, 0, 3659, 3618, 4095, 595, 0, 0, 1, 4094, 1, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 19, 1, 4095, 1, 0, 0, 99, 0, 0, 0, 0, 1, 1519, 1, 0, 4095, 2755, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 1, 0, 0, 1, 10, 4095, 4095, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 4095, 1, 0, 0, 0, 78, 0, 1, 1, 0, 1, 0, 2, 2960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 43, 1, 0, 4095, 4087, 1, 1, 1, 0, 0, 0, 0, 3650, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3010, 0, 4095, 4095, 0, 0, 1, 0, 0, 802, 0, 3151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4057, 1, 0, 0, 0, 1, 1, 10, 0, 1, 1, 1, 3617, 4095, 4095, 1340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3332, 0, 0, 1, 0, 1, 4095, 4073, 0, 0, 4095, 0, 849, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16302, 17116, 16392, 15329, 16361, 15154, 17221, 17197] avg=16384.0000 max=17221 max_rank=6 min=15154 min_rank=5 imbalance_ratio=1.051086 max_load_reduction=113851 max_load_reduction_pct=0.868614 moved_to_echo=65933 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3374, 2858, 4096, 2343, 0, 0, 0, 4096, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 4095, 0, 0, 0, 106, 0, 0, 0, 0, 0, 1171, 0, 0, 4096, 2757, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4094, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 2039, 0, 0, 0, 0, 0, 0, 0, 2960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3651, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3011, 0, 4096, 4096, 0, 0, 0, 0, 0, 3619, 0, 2645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 1, 0, 0, 23, 0, 0, 0, 0, 1499, 4096, 4096, 1339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 827, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 4096, 0, 849, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17002, 16761, 16391, 17287, 16332, 17467, 15127, 14705] avg=16384.0000 max=17467 max_rank=5 min=14705 min_rank=7 imbalance_ratio=1.066101 max_load_reduction=113605 max_load_reduction_pct=0.866737 moved_to_echo=63631 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3410, 3491, 4096, 1126, 0, 0, 0, 4096, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 369, 0, 0, 4096, 2757, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1592, 0, 0, 0, 0, 0, 14, 0, 2960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 57, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 3651, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3012, 0, 4096, 4096, 0, 0, 0, 0, 0, 2245, 0, 3327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3939, 0, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 2266, 4096, 4096, 1339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2746, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 4096, 0, 849, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16454, 15850, 16393, 16854, 16374, 16776, 15741, 16630] avg=16384.0000 max=16854 max_rank=3 min=15741 min_rank=6 imbalance_ratio=1.028687 max_load_reduction=114218 max_load_reduction_pct=0.871414 moved_to_echo=65521 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 3390, 3186, 4095, 1671, 0, 0, 0, 4096, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 0, 4096, 0, 0, 1, 326, 0, 0, 0, 1, 0, 833, 0, 0, 4096, 2756, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 0, 0, 0, 0, 0, 10, 4092, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 4096, 0, 1, 0, 0, 331, 0, 0, 0, 0, 1, 4, 0, 2960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 38, 0, 1, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 3650, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3012, 0, 4095, 4096, 0, 0, 0, 0, 0, 1635, 1, 3619, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 0, 0, 0, 4, 0, 0, 34, 1, 0, 0, 0, 3689, 4096, 4095, 1340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1921, 1, 0, 0, 0, 0, 4096, 4082, 0, 0, 4096, 0, 849, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16674, 16641, 16388, 15583, 16358, 16462, 17166, 15800] avg=16384.0000 max=17166 max_rank=6 min=15583 min_rank=3 imbalance_ratio=1.047729 max_load_reduction=113906 max_load_reduction_pct=0.869034 moved_to_echo=65786 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 2512, 2869, 4095, 2422, 0, 0, 0, 4096, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4096, 0, 0, 1, 115, 0, 0, 7, 0, 0, 444, 0, 0, 4095, 2757, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 7, 4096, 4096, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 4096, 0, 7, 0, 0, 440, 0, 0, 0, 10, 5, 1, 0, 2952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 1, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 3653, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3004, 0, 4090, 4091, 0, 0, 0, 0, 0, 1401, 0, 3050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3818, 0, 0, 0, 14, 0, 0, 57, 0, 0, 0, 0, 3741, 4096, 4096, 1339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 3729, 0, 0, 0, 1, 0, 4094, 4078, 0, 0, 4096, 0, 855, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16235, 16027, 16351, 15698, 16330, 15636, 17166, 17629] avg=16384.0000 max=17629 max_rank=7 min=15636 min_rank=5 imbalance_ratio=1.075989 max_load_reduction=113443 max_load_reduction_pct=0.865501 moved_to_echo=66761 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 0, 1525, 3504, 4096, 2310, 0, 0, 0, 4096, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 4096, 0, 0, 0, 71, 0, 0, 4, 0, 0, 41, 0, 0, 4096, 2757, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 4096, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 4095, 0, 0, 0, 0, 2623, 0, 0, 0, 2, 8, 13, 2, 2960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 238, 0, 0, 4096, 4068, 0, 0, 0, 0, 2, 0, 0, 3651, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3012, 0, 4095, 4091, 0, 0, 0, 0, 0, 3131, 0, 3649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 0, 7, 0, 0, 31, 0, 0, 0, 0, 1915, 4096, 4096, 1339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1567, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 4096, 0, 849, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15771, 15578, 16381, 17893, 16538, 17978, 15494, 15439] avg=16384.0000 max=17978 max_rank=5 min=15439 min_rank=7 imbalance_ratio=1.097290 max_load_reduction=113094 max_load_reduction_pct=0.862839 moved_to_echo=65449 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3090, 3696, 4096, 440, 0, 0, 0, 4096, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 96, 0, 0, 0, 0, 0, 1309, 0, 0, 4096, 2756, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1306, 0, 0, 0, 0, 0, 2, 0, 2960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 85, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 3651, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3012, 0, 4096, 4096, 0, 0, 0, 0, 0, 2018, 0, 3110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 3088, 4096, 4096, 1340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2306, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 4096, 0, 849, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15653, 16884, 16394, 16556, 16406, 16332, 16659, 16188] avg=16384.0000 max=16884 max_rank=1 min=15653 min_rank=0 imbalance_ratio=1.030518 max_load_reduction=114188 max_load_reduction_pct=0.871185 moved_to_echo=65585 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 32768, 128, 0, 128, 1152, 0, 0, 128, 128, 128, 6272, 128, 0, 32768, 22144, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 32768, 640, 128, 128, 32768, 32768, 128, 128, 128, 0, 128, 0, 128, 29312, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 31872, 128, 0, 128, 128, 128, 128, 256, 128, 128, 128, 128, 22784, 32768, 32768, 10752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 20096, 128, 0, 128, 128, 128, 32768, 32768, 0, 128, 32768, 0, 6912, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 24064, 26624, 32768, 13056, 0, 0, 128, 32768, 128, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 32768, 128, 128, 0, 0, 9088, 0, 128, 128, 128, 128, 128, 128, 23680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 128, 128, 128, 32768, 32768, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[24192, 0, 32768, 32768, 128, 0, 128, 128, 0, 17920, 128, 23680, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 2386, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 4095, 0, 1, 0, 4096, 2, 0, 0, 0, 2930, 4095, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 0, 0, 0, 4096, 32, 0, 0, 4096, 4096, 4096, 0, 4091, 6, 4096, 4096, 4083, 4088, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 768, 0, 4096, 4091, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 1, 19, 4096, 4093, 0, 0, 4096, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 4096, 0, 6, 2065, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[3966, 2, 0, 0, 5, 0, 0, 6, 0, 6, 6, 233, 1, 7, 4, 0, 4, 8, 0, 6, 0, 0, 8, 7, 1, 9, 8, 4, 0, 4088, 4087, 4091, 0, 12, 4, 4088, 0, 0, 0, 9, 2746, 4085, 1, 10, 4, 0, 5, 0, 0, 8, 4083, 9, 5, 0, 4088, 192, 13, 8, 4079, 4088, 4088, 0, 4086, 0, 4085, 4052, 4070, 4082, 2, 3, 0, 4, 4, 4069, 0, 8, 0, 3315, 0, 4077, 4076, 0, 7, 8, 5, 0, 8, 0, 0, 5, 8, 4087, 8, 1, 17, 4079, 4087, 7, 7, 4088, 0, 3, 2, 0, 4089, 4, 10, 0, 6, 0, 4083, 6, 14, 1892, 5, 1, 0, 4088, 0, 4082, 0, 0, 6, 4086, 4087, 1, 1, 4069] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 4092, 0, 11, 0, 4096, 0, 0, 0, 0, 3643, 4096, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 0, 4096, 7, 0, 0, 4096, 4096, 4096, 0, 4096, 4, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 1542, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 31, 4096, 4066, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 8, 1082, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 4096, 0, 7, 0, 4096, 0, 0, 0, 0, 3291, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 0, 4096, 49, 0, 0, 4096, 4096, 4096, 0, 4096, 2, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1749, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 4096, 4083, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1806, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 13, 0, 0, 0, 10, 1022, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 4084, 0, 9, 0, 4096, 0, 0, 0, 0, 3118, 4090, 0, 0, 0, 0, 33, 0, 0, 0, 4096, 9, 0, 0, 4096, 45, 0, 0, 4096, 4096, 4096, 0, 4095, 7, 4096, 4096, 4081, 4091, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 2101, 0, 4096, 4073, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 5, 35, 4092, 4048, 0, 0, 4096, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 4096, 0, 8, 1912, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 6, 0, 4096, 0, 0, 0, 0, 2160, 4096, 1, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 117, 0, 0, 4096, 4096, 4096, 0, 4095, 0, 4096, 4095, 4096, 4093, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3343, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1628, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 745, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4096, 4084, 0, 0, 0, 4096, 0, 0, 0, 0, 2568, 4096, 0, 0, 0, 0, 15, 0, 0, 0, 4096, 0, 0, 0, 4096, 45, 0, 0, 4092, 4096, 4096, 0, 4096, 7, 4096, 4096, 4058, 4096, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 1642, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 48, 4088, 4049, 0, 0, 4096, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 4096, 0, 17, 3240, 0, 0, 0, 4096, 5, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 1, 0, 0, 0, 0, 1, 2, 1258, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 4095, 4095, 4095, 0, 0, 1, 4095, 0, 0, 0, 1, 3075, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 4095, 74, 1, 1, 4095, 4095, 4095, 0, 4095, 0, 4095, 4091, 4095, 4090, 0, 0, 0, 1, 0, 4086, 0, 1, 0, 2540, 0, 4093, 4089, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 4095, 1, 0, 0, 4095, 4090, 1, 1, 4095, 0, 1, 0, 0, 4095, 1, 1, 0, 1, 0, 4085, 1, 2, 1281, 1, 0, 0, 4095, 0, 4095, 0, 0, 2, 4094, 4095, 1, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3966, 2, 0, 0, 5, 0, 0, 6, 0, 6, 6, 233, 1, 7, 4, 0, 4088, 4088, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, 0, 6, 0, 0, 8, 7, 1, 9, 8, 4, 0, 4088, 4087, 4091, 4010, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4, 4088, 0, 0, 0, 9, 2746, 4085, 1, 10, 4, 0, 5, 0, 4009, 1210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4083, 9, 5, 0, 0, 192, 13, 8, 4079, 0, 4017, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 4070, 1994, 2, 3, 0, 4, 4, 4069, 0, 8, 0, 3315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 7, 8, 5, 0, 8, 0, 0, 5, 8, 4087, 8, 1, 17, 4079, 2088, 1861, 67, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 7, 7, 4088, 0, 3, 2, 0, 4089, 4, 10, 0, 6, 0, 4083, 6, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1892, 5, 1, 0, 2227, 0, 4082, 0, 0, 6, 4086, 0, 1, 1, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15289, 16407, 16183, 16500, 17521, 16379, 16409, 16384] avg=16384.0000 max=17521 max_rank=4 min=15289 min_rank=0 imbalance_ratio=1.069397 max_load_reduction=113551 max_load_reduction_pct=0.866325 moved_to_echo=66693 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 2386, 0, 0, 0, 0, 4096, 4096, 2883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 4095, 4029, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 2, 0, 0, 0, 2930, 4095, 0, 0, 0, 0, 27, 0, 4020, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 32, 0, 0, 4096, 0, 4024, 0, 4091, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 1997, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 1, 19, 4096, 2091, 1865, 67, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2065, 0, 0, 0, 2231, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17557, 16388, 16384, 16345, 15028, 16381, 16399, 16590] avg=16384.0000 max=17557 max_rank=0 min=15028 min_rank=4 imbalance_ratio=1.071594 max_load_reduction=113515 max_load_reduction_pct=0.866051 moved_to_echo=64398 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 943, 0, 0, 0, 0, 4096, 4096, 2884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4029, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 2160, 4096, 1, 0, 1, 0, 0, 0, 4019, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 117, 0, 0, 4096, 0, 4024, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 1999, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 4096, 2094, 1864, 67, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1628, 0, 0, 0, 2230, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16113, 16394, 15591, 16428, 17628, 16371, 16401, 16146] avg=16384.0000 max=17628 max_rank=4 min=15591 min_rank=2 imbalance_ratio=1.075928 max_load_reduction=113444 max_load_reduction_pct=0.865509 moved_to_echo=66546 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1918, 0, 0, 0, 0, 4096, 4096, 2883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 4092, 4029, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 3643, 4096, 0, 0, 0, 0, 16, 0, 4020, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 4024, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1998, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 1542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 31, 4096, 2094, 1865, 67, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1082, 0, 0, 0, 2231, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17089, 16369, 17095, 16323, 15816, 16399, 16372, 15609] avg=16384.0000 max=17095 max_rank=2 min=15609 min_rank=7 imbalance_ratio=1.043396 max_load_reduction=113977 max_load_reduction_pct=0.869576 moved_to_echo=64196 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 1, 0, 0, 0, 0, 1, 2, 1258, 1, 0, 0, 0, 4095, 4095, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 4095, 4095, 4095, 4026, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 1, 3075, 4095, 0, 0, 0, 0, 0, 0, 4019, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 74, 1, 1, 4095, 0, 4023, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 1998, 0, 0, 0, 1, 0, 4086, 0, 1, 0, 2540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 4095, 1, 0, 0, 4095, 2092, 1864, 67, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 1, 4095, 0, 1, 0, 0, 4095, 1, 1, 0, 1, 0, 4085, 1, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1281, 1, 0, 0, 2231, 0, 4095, 0, 0, 2, 4094, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16428, 16392, 16499, 16386, 16812, 16364, 16390, 15801] avg=16384.0000 max=16812 max_rank=4 min=15801 min_rank=7 imbalance_ratio=1.026123 max_load_reduction=114260 max_load_reduction_pct=0.871735 moved_to_echo=65367 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1286, 0, 0, 0, 0, 4096, 4096, 2883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 4096, 4029, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 3291, 4096, 0, 0, 0, 0, 14, 0, 4020, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 49, 0, 0, 4096, 0, 4024, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2000, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 4096, 2096, 1865, 67, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1806, 0, 0, 0, 2231, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16457, 16388, 16737, 16363, 16036, 16377, 16389, 16325] avg=16384.0000 max=16737 max_rank=2 min=16036 min_rank=4 imbalance_ratio=1.021545 max_load_reduction=114335 max_load_reduction_pct=0.872307 moved_to_echo=65127 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 13, 0, 0, 0, 10, 1022, 0, 0, 0, 0, 4096, 4096, 2884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 4084, 4030, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 3118, 4090, 0, 0, 0, 0, 33, 0, 4019, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 45, 0, 0, 4096, 0, 4019, 0, 4095, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 1994, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 2101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 5, 35, 4092, 2097, 1868, 66, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 4096, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1912, 0, 0, 0, 2228, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16217, 16369, 16577, 16367, 16360, 16392, 16354, 16436] avg=16384.0000 max=16577 max_rank=2 min=16217 min_rank=0 imbalance_ratio=1.011780 max_load_reduction=114495 max_load_reduction_pct=0.873528 moved_to_echo=65542 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 745, 0, 0, 0, 0, 4096, 4096, 2884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4096, 4084, 4029, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2568, 4096, 0, 0, 0, 0, 15, 0, 4019, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 45, 0, 0, 4092, 0, 4024, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4058, 2000, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 48, 4088, 2096, 1865, 67, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 4096, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3240, 0, 0, 0, 2231, 5, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15922, 16365, 16006, 16360, 15871, 16409, 16358, 17781] avg=16384.0000 max=17781 max_rank=7 min=15871 min_rank=4 imbalance_ratio=1.085266 max_load_reduction=113291 max_load_reduction_pct=0.864342 moved_to_echo=66419 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 32768, 16000, 128, 128, 0, 128, 128, 32768, 0, 128, 0, 17024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32640, 128, 0, 0, 128, 0, 128, 128, 0, 128, 128, 9856, 128, 128, 128, 0, 32768, 32768, 23168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 128, 128, 0, 0, 640, 128, 128, 32768, 0, 32256, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32640, 128, 128, 32768, 0, 128, 128, 128, 32768, 128, 128, 0, 128, 0, 32768, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 14976, 128, 128, 0, 17920, 128, 32768, 0, 0, 128, 32768, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 128, 128, 128, 128, 128, 0, 32768, 32768, 32768, 32256, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 32768, 128, 0, 0, 128, 23552, 32768, 128, 128, 128, 0, 128, 0, 32256, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 128, 0, 128, 128, 128, 32768, 128, 128, 256, 32768, 16768, 14976, 640, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4086, 17, 5, 9, 9, 8, 4085, 3, 4087, 9, 0, 46, 1, 0, 0, 0, 0, 0, 7, 0, 9, 3, 0, 4087, 0, 8, 4081, 0, 7, 4060, 0, 4088, 4088, 4085, 4087, 0, 4029, 0, 0, 9, 0, 4086, 3430, 118, 0, 9, 5, 4087, 4085, 4087, 6, 2737, 0, 3927, 9, 0, 0, 9, 9, 2, 6, 9, 0, 4081, 0, 0, 253, 5, 8, 38, 7, 8, 0, 4086, 5, 8, 7, 4079, 1881, 36, 5, 0, 35, 0, 2, 0, 4089, 0, 4, 4086, 4087, 9, 0, 7, 0, 4086, 4087, 8, 0, 8, 3, 0, 0, 0, 0, 4085, 0, 0, 6, 4088, 5, 0, 4087, 3, 3915, 9, 0, 8, 20, 9, 0, 3, 4083, 3, 0, 4087, 2, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 94, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 31, 0, 2, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4096, 1, 0, 4096, 1, 0, 4086, 0, 4095, 4095, 4096, 4096, 0, 3984, 1, 1, 0, 0, 4096, 3958, 839, 0, 2, 0, 4095, 4095, 4095, 6, 1926, 0, 4091, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 169, 0, 0, 3, 0, 0, 0, 4094, 0, 0, 0, 4096, 1307, 0, 0, 1, 7, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 4092, 0, 0, 4095, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 64, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 48, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4095, 0, 4096, 4096, 4096, 4096, 0, 3905, 0, 0, 0, 0, 4096, 3189, 227, 0, 0, 0, 4096, 4096, 4096, 13, 3226, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 764, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 4096, 833, 0, 0, 0, 113, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4095, 0, 0, 4095, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 87, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4095, 4096, 4096, 4096, 0, 3694, 0, 0, 0, 0, 4096, 4015, 225, 0, 0, 0, 4096, 4096, 4096, 0, 1838, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 93, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 2378, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4045, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4095, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4096, 8, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 9, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 4096, 4, 0, 4096, 27, 0, 4070, 0, 4095, 4090, 4096, 4096, 0, 4087, 7, 3, 0, 0, 4096, 3662, 150, 0, 7, 0, 4094, 4096, 4096, 10, 1245, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 971, 0, 0, 21, 0, 0, 0, 4071, 0, 0, 0, 4096, 2145, 0, 0, 8, 98, 0, 0, 0, 4089, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4086, 0, 0, 4096, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4091, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 35, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 3661, 0, 0, 0, 0, 4096, 4060, 511, 0, 0, 0, 4096, 4096, 4096, 0, 2442, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 67, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1499, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4096, 21, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 4, 0, 8, 0, 0, 0, 4096, 5, 0, 4096, 0, 0, 4080, 0, 4096, 4096, 4090, 4096, 0, 3914, 5, 0, 0, 0, 4096, 3535, 604, 0, 6, 0, 4096, 4096, 4096, 14, 1510, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 243, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 2475, 0, 0, 12, 48, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4091, 0, 0, 4096, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4095, 56, 0, 1, 1, 1, 4095, 1, 4095, 1, 0, 7, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 4095, 0, 1, 4095, 0, 1, 4092, 0, 4095, 4095, 4095, 4095, 0, 3847, 0, 0, 1, 0, 4095, 4087, 919, 0, 2, 0, 4094, 4095, 4095, 0, 689, 1, 4092, 1, 0, 0, 1, 1, 0, 0, 1, 0, 4095, 0, 0, 209, 0, 1, 18, 1, 1, 0, 4095, 1, 1, 1, 4095, 2472, 3, 1, 0, 3, 0, 0, 0, 4095, 0, 1, 4095, 4095, 1, 0, 1, 0, 4095, 4094, 1, 0, 0, 1, 0, 0, 8, 0, 4095, 0, 0, 1, 4095, 0, 0, 4095, 0, 4067, 1, 0, 1, 1, 1, 0, 0, 4095, 1, 0, 4092, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4086, 17, 5, 9, 9, 8, 4085, 3, 4087, 9, 0, 46, 1, 0, 0, 0, 2111, 1906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 3, 0, 4087, 0, 8, 4081, 0, 7, 4060, 0, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4085, 0, 0, 4029, 0, 0, 9, 0, 0, 3430, 118, 0, 9, 5, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 2175, 6, 2737, 0, 3927, 9, 0, 0, 9, 9, 2, 6, 9, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 253, 5, 8, 38, 7, 8, 0, 4086, 5, 8, 7, 4079, 1881, 36, 4087, 1862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 35, 0, 2, 0, 4089, 0, 4, 4086, 4048, 9, 0, 7, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 8, 0, 8, 3, 0, 0, 0, 0, 4085, 0, 0, 6, 4088, 5, 0, 2224, 1868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 3, 3915, 9, 0, 8, 20, 9, 0, 3, 4083, 3, 0, 4087, 2, 7, 36, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16353, 15881, 17055, 16370, 16371, 16382, 16278] avg=16384.0000 max=17055 max_rank=3 min=15881 min_rank=2 imbalance_ratio=1.040955 max_load_reduction=114017 max_load_reduction_pct=0.869881 moved_to_echo=65401 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 94, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 31, 0, 2, 0, 0, 2115, 1910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4096, 1, 0, 4096, 1, 0, 4086, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 3984, 1, 1, 0, 0, 0, 3958, 839, 0, 2, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2179, 6, 1926, 0, 4091, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 3, 0, 0, 0, 4094, 0, 0, 0, 4096, 1307, 0, 4096, 1865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 0, 4095, 0, 0, 4096, 4057, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 4092, 0, 0, 2231, 1871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 36, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16439, 16380, 17085, 16391, 15630, 16352, 16384, 16411] avg=16384.0000 max=17085 max_rank=2 min=15630 min_rank=4 imbalance_ratio=1.042786 max_load_reduction=113987 max_load_reduction_pct=0.869652 moved_to_echo=64777 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 87, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 9, 0, 0, 0, 0, 2115, 1910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3694, 0, 0, 0, 0, 0, 4015, 225, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2180, 0, 1838, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 2378, 0, 4096, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 4096, 4057, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 4096, 0, 0, 2230, 1872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4045, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 36, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16409, 16386, 16235, 16291, 16626, 16356, 16394, 16375] avg=16384.0000 max=16626 max_rank=4 min=16235 min_rank=2 imbalance_ratio=1.014771 max_load_reduction=114446 max_load_reduction_pct=0.873154 moved_to_echo=65751 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 64, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 48, 0, 1, 0, 0, 2115, 1910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4095, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3905, 0, 0, 0, 0, 0, 3189, 227, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2180, 13, 3226, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 764, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 4096, 833, 0, 4096, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 4096, 0, 0, 4096, 4057, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4095, 0, 0, 2230, 1872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 36, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16426, 16387, 15622, 17659, 15731, 16458, 16385, 16404] avg=16384.0000 max=17659 max_rank=3 min=15622 min_rank=2 imbalance_ratio=1.077820 max_load_reduction=113413 max_load_reduction_pct=0.865273 moved_to_echo=64978 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4095, 56, 0, 1, 1, 1, 4095, 1, 4095, 1, 0, 7, 1, 0, 0, 0, 2114, 1910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 4095, 0, 1, 4095, 0, 1, 4092, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 3847, 0, 0, 1, 0, 0, 4087, 919, 0, 2, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2179, 0, 689, 1, 4092, 1, 0, 0, 1, 1, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 1, 18, 1, 1, 0, 4095, 1, 1, 1, 4095, 2472, 3, 4095, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 4095, 0, 1, 4095, 4056, 1, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 1, 0, 0, 8, 0, 4095, 0, 0, 1, 4095, 0, 0, 2229, 1871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4067, 1, 0, 1, 1, 1, 0, 0, 4095, 1, 0, 4092, 0, 1, 36, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 16385, 17155, 15155, 16859, 16348, 16395, 16397] avg=16384.0000 max=17155 max_rank=2 min=15155 min_rank=3 imbalance_ratio=1.047058 max_load_reduction=113917 max_load_reduction_pct=0.869118 moved_to_echo=65999 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 8, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 9, 0, 0, 0, 0, 2119, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 4096, 4, 0, 4096, 27, 0, 4070, 0, 4095, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 4087, 7, 3, 0, 0, 0, 3662, 150, 0, 7, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2169, 10, 1245, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 971, 0, 0, 21, 0, 0, 0, 4071, 0, 0, 0, 4096, 2145, 0, 4096, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 98, 0, 0, 0, 4089, 0, 0, 4096, 4048, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4086, 0, 0, 2230, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4091, 0, 0, 38, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16332, 16406, 16202, 15669, 17266, 16434, 16371, 16392] avg=16384.0000 max=17266 max_rank=4 min=15669 min_rank=3 imbalance_ratio=1.053833 max_load_reduction=113806 max_load_reduction_pct=0.868271 moved_to_echo=66463 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 21, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 2115, 1910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8, 0, 0, 0, 4096, 5, 0, 4096, 0, 0, 4080, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 3914, 5, 0, 0, 0, 0, 3535, 604, 0, 6, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2180, 14, 1510, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 2475, 0, 4096, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 48, 0, 0, 0, 4096, 0, 0, 4096, 4057, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4091, 0, 0, 2230, 1872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 36, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16341, 16388, 16359, 15944, 16871, 16405, 16368, 16396] avg=16384.0000 max=16871 max_rank=4 min=15944 min_rank=3 imbalance_ratio=1.029724 max_load_reduction=114201 max_load_reduction_pct=0.871284 moved_to_echo=66040 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 35, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 17, 0, 0, 0, 0, 2115, 1910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3661, 0, 0, 0, 0, 0, 4060, 511, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2180, 0, 2442, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1499, 0, 4096, 1865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 4096, 4057, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 0, 0, 2231, 1872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 36, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16365, 16387, 16533, 16908, 15719, 16348, 16393, 16419] avg=16384.0000 max=16908 max_rank=3 min=15719 min_rank=4 imbalance_ratio=1.031982 max_load_reduction=114164 max_load_reduction_pct=0.871002 moved_to_echo=64879 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 0, 128, 0, 32768, 0, 0, 128, 32768, 128, 0, 17920, 14976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 32768, 128, 128, 32768, 128, 128, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 32768, 0, 0, 31232, 128, 128, 128, 0, 0, 29952, 3712, 0, 128, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 17536, 128, 15616, 128, 32512, 128, 0, 0, 128, 128, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 384, 128, 128, 128, 128, 32768, 128, 32768, 128, 0, 256, 128, 128, 0, 0, 17024, 15360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 32512, 128, 0, 128, 128, 128, 0, 128, 32768, 128, 128, 32768, 128, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 384, 0, 128, 0, 32768, 0, 128, 32768, 32512, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 2816, 128, 128, 128, 128, 128, 0, 32768, 128, 128, 128, 32768, 15104, 128, 32768, 14976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2782, 0, 0, 4095, 0, 0, 0, 0, 1, 9, 0, 0, 4044, 0, 0, 0, 4086, 0, 0, 4052, 4090, 0, 4096, 4086, 4096, 0, 0, 0, 0, 0, 0, 4090, 4096, 87, 4074, 0, 0, 0, 4096, 0, 0, 6, 0, 4094, 4096, 0, 0, 2709, 1395, 0, 0, 4089, 67, 0, 12, 4096, 0, 8, 0, 4096, 4096, 4091, 1178, 0, 4096, 0, 6, 0, 4096, 0, 300, 0, 0, 4017, 3894, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 9, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2340, 0, 0, 4096, 0, 0, 0, 0, 1, 2, 1, 0, 4067, 0, 0, 0, 4094, 0, 0, 4079, 4086, 0, 4096, 4096, 4096, 1, 0, 0, 0, 0, 0, 4095, 4096, 90, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4095, 1, 0, 2618, 1763, 0, 0, 4096, 11, 0, 304, 4096, 0, 4, 0, 4096, 4095, 4095, 1174, 0, 4096, 0, 1, 0, 4095, 0, 53, 0, 0, 4085, 3986, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1781, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 4096, 0, 0, 3878, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 157, 4096, 0, 4, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 2873, 2199, 0, 0, 4087, 10, 0, 37, 4096, 0, 0, 0, 4096, 4096, 4096, 1380, 0, 4096, 0, 0, 0, 4096, 0, 71, 0, 0, 4063, 4003, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[8, 0, 0, 4087, 9, 0, 9, 0, 4087, 0, 9, 0, 7, 3, 2, 0, 1, 0, 7, 4086, 8, 9, 4084, 9, 8, 9, 2907, 0, 1, 4086, 9, 1, 9, 0, 0, 0, 0, 8, 4083, 0, 9, 2, 4084, 0, 9, 2544, 4089, 9, 4087, 4087, 4083, 0, 0, 10, 0, 0, 4, 4087, 4084, 33, 4087, 39, 0, 0, 4085, 1, 0, 0, 8, 4086, 4043, 0, 0, 3900, 1139, 9, 11, 4075, 0, 2, 16, 4081, 0, 0, 0, 4086, 4086, 4087, 983, 5, 4079, 7, 0, 7, 4078, 6, 713, 0, 7, 4088, 4087, 4084, 2, 7, 0, 0, 7, 9, 8, 0, 41, 8, 3, 4081, 4085, 3, 4, 9, 4, 5, 9, 13, 7, 15, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2296, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3930, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 39, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4095, 0, 0, 3763, 472, 0, 0, 4095, 1, 0, 6, 4096, 0, 0, 0, 4096, 4096, 4096, 1129, 0, 4096, 0, 0, 0, 4096, 0, 656, 0, 0, 4096, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2514, 2, 0, 4096, 0, 0, 0, 0, 5, 2, 0, 0, 4065, 0, 0, 0, 4087, 0, 0, 3935, 4093, 0, 4096, 4081, 4096, 0, 0, 0, 0, 0, 0, 4092, 4096, 38, 4088, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 3327, 920, 0, 0, 4089, 43, 0, 22, 4096, 0, 6, 5, 4096, 4096, 4096, 1350, 0, 4096, 0, 3, 0, 4095, 0, 286, 0, 0, 4021, 4032, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 23, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 1, 3, 1, 0, 4095, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 4095, 1, 1, 4095, 1, 1, 1, 2122, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 1, 0, 4095, 0, 0, 4039, 4095, 1, 4095, 4093, 4094, 0, 0, 1, 1, 0, 0, 4095, 4095, 122, 4095, 3, 0, 0, 4095, 0, 0, 0, 0, 4095, 4078, 0, 0, 3706, 1198, 0, 1, 4095, 0, 1, 34, 4095, 0, 0, 0, 4095, 4095, 4095, 964, 1, 4095, 1, 0, 1, 4093, 1, 111, 0, 1, 4095, 4095, 4095, 1, 1, 0, 0, 1, 1, 0, 0, 5, 1, 0, 4095, 4095, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2638, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 3946, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 372, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 3464, 504, 0, 0, 4096, 2, 0, 19, 4096, 0, 0, 0, 4096, 4096, 4096, 1301, 0, 4096, 0, 0, 0, 4096, 0, 67, 0, 0, 4094, 4070, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 23, 4096, 4070, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2340, 0, 0, 4096, 0, 0, 1237, 232, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 0, 4067, 0, 0, 0, 4094, 0, 0, 4079, 4086, 0, 203, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3986, 1, 0, 0, 0, 0, 0, 4095, 0, 90, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3693, 4095, 1, 0, 2618, 1763, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 4096, 0, 4, 0, 0, 0, 2858, 1174, 0, 4096, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 4085, 3986, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 3864, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16296, 16643, 16363, 16277, 16628, 16111, 16374] avg=16384.0000 max=16643 max_rank=2 min=16111 min_rank=6 imbalance_ratio=1.015808 max_load_reduction=114429 max_load_reduction_pct=0.873024 moved_to_echo=65390 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 4087, 9, 0, 9, 0, 4087, 0, 9, 0, 7, 3, 2, 0, 4085, 4061, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 4086, 8, 9, 4084, 9, 8, 9, 2907, 0, 1, 4086, 9, 1, 1235, 232, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8, 4083, 0, 9, 2, 4084, 0, 9, 2544, 4089, 9, 202, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 3973, 0, 0, 10, 0, 0, 4, 4087, 0, 33, 4087, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 3685, 4043, 0, 0, 3900, 1139, 9, 11, 4075, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4081, 0, 0, 0, 0, 0, 2852, 983, 5, 4079, 7, 0, 7, 4078, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 713, 0, 7, 4088, 4087, 4084, 2, 7, 0, 0, 7, 9, 8, 0, 41, 8, 3852, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4081, 4085, 3, 4, 9, 4, 5, 9, 13, 7, 15, 0, 0, 0, 0, 4087, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16370, 16891, 15158, 16320, 16873, 16114, 16938, 16408] avg=16384.0000 max=16938 max_rank=6 min=15158 min_rank=2 imbalance_ratio=1.033813 max_load_reduction=114134 max_load_reduction_pct=0.870773 moved_to_echo=66333 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 62, 4096, 4072, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2782, 0, 0, 4095, 0, 0, 1241, 233, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 0, 0, 4044, 0, 0, 0, 4086, 0, 0, 4052, 4090, 0, 207, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 3979, 0, 0, 0, 0, 0, 0, 4090, 0, 87, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3683, 4096, 0, 0, 2709, 1395, 0, 0, 4089, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 8, 0, 0, 0, 2850, 1178, 0, 4096, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 0, 4017, 3894, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 9, 0, 3863, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16421, 16747, 16606, 16316, 16045, 16342, 16209, 16386] avg=16384.0000 max=16747 max_rank=1 min=16045 min_rank=4 imbalance_ratio=1.022156 max_load_reduction=114325 max_load_reduction_pct=0.872231 moved_to_echo=64982 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 63, 4096, 4071, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1781, 0, 0, 4096, 0, 0, 1237, 232, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 4096, 0, 0, 3878, 4096, 0, 203, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3986, 0, 0, 0, 0, 0, 0, 4096, 0, 157, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3694, 4096, 0, 0, 2873, 2199, 0, 0, 4087, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4096, 0, 0, 0, 0, 0, 2859, 1380, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 4063, 4003, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16425, 15737, 16449, 16435, 16959, 16564, 16122, 16381] avg=16384.0000 max=16959 max_rank=4 min=15737 min_rank=1 imbalance_ratio=1.035095 max_load_reduction=114113 max_load_reduction_pct=0.870613 moved_to_echo=66026 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4071, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2296, 0, 0, 4096, 0, 0, 1237, 233, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3930, 4096, 0, 203, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3986, 0, 0, 0, 0, 0, 0, 4096, 0, 39, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3694, 4095, 0, 0, 3763, 472, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 2859, 1129, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 656, 0, 0, 4096, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3863, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16362, 16253, 16531, 16313, 16120, 16282, 16830, 16381] avg=16384.0000 max=16830 max_rank=6 min=16120 min_rank=4 imbalance_ratio=1.027222 max_load_reduction=114242 max_load_reduction_pct=0.871597 moved_to_echo=65613 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 1, 3, 1, 0, 4095, 0, 1, 0, 0, 1, 0, 0, 4095, 4070, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 4095, 1, 1, 1, 2122, 0, 0, 4094, 1, 0, 1237, 232, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 1, 0, 4095, 0, 0, 4039, 4095, 1, 203, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3984, 0, 0, 1, 1, 0, 0, 4095, 0, 122, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3693, 4078, 0, 0, 3706, 1198, 0, 1, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4095, 0, 0, 0, 0, 0, 2858, 964, 1, 4095, 1, 0, 1, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 1, 4095, 4095, 4095, 1, 1, 0, 0, 1, 1, 0, 0, 5, 1, 3863, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16366, 16081, 16639, 16394, 16772, 16143, 16295, 16382] avg=16384.0000 max=16772 max_rank=4 min=16081 min_rank=1 imbalance_ratio=1.023682 max_load_reduction=114300 max_load_reduction_pct=0.872040 moved_to_echo=65592 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 37, 4096, 4071, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2514, 2, 0, 4096, 0, 0, 1237, 233, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 4065, 0, 0, 0, 4087, 0, 0, 3935, 4093, 0, 203, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 3986, 0, 0, 0, 0, 0, 0, 4092, 0, 38, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3694, 4096, 0, 0, 3327, 920, 0, 0, 4089, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4096, 0, 6, 5, 0, 0, 2859, 1350, 0, 4096, 0, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 0, 0, 4021, 4032, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 23, 0, 3863, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16473, 16500, 16285, 16170, 16532, 16351, 16378] avg=16384.0000 max=16532 max_rank=5 min=16170 min_rank=4 imbalance_ratio=1.009033 max_load_reduction=114540 max_load_reduction_pct=0.873871 moved_to_echo=65431 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 4096, 4071, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2638, 0, 0, 4096, 0, 0, 1237, 232, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 3946, 4096, 0, 203, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3986, 0, 0, 0, 0, 0, 0, 4096, 0, 372, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3694, 4096, 0, 0, 3464, 504, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 0, 2859, 1301, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 4094, 4070, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16365, 16594, 16546, 16646, 15856, 16467, 16216, 16382] avg=16384.0000 max=16646 max_rank=3 min=15856 min_rank=4 imbalance_ratio=1.015991 max_load_reduction=114426 max_load_reduction_pct=0.873001 moved_to_echo=64921 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 128, 128, 128, 128, 128, 128, 0, 128, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 32768, 128, 128, 128, 19456, 128, 128, 32768, 128, 128, 9984, 1920, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 31872, 128, 0, 128, 128, 0, 128, 32768, 0, 1024, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[512, 32768, 0, 128, 128, 0, 0, 22912, 9472, 128, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 0, 32768, 128, 128, 128, 0, 32768, 0, 128, 0, 128, 128, 128, 256, 32768, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[2304, 0, 128, 32640, 32384, 32768, 128, 128, 0, 128, 128, 128, 128, 0, 128, 128, 30976, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 128, 128, 32640, 0, 128, 128, 32768, 0, 128, 30464, 32768, 128, 1664, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 29568, 32768, 128, 0, 26368, 9600, 128, 128, 32768, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[3876, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 4095, 2604, 0, 0, 0, 0, 4096, 0, 4096, 372, 1, 4093, 0, 0, 0, 0, 4082, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3696, 0, 4090, 0, 0, 0, 0, 4095, 0, 4096, 1, 0, 0, 0, 1928, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 119, 0, 35, 0, 4096, 4096, 7, 3969, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, 0, 4096, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3877, 4096, 4, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4086, 9, 0, 0, 0, 7, 4077, 0, 8, 11, 0, 9, 8, 4, 4084, 9, 0, 4, 0, 0, 0, 4088, 4, 0, 4, 0, 4085, 10, 0, 0, 0, 0, 14, 4087, 2573, 41, 3, 4, 0, 4082, 5, 4085, 90, 0, 4087, 7, 0, 0, 10, 4073, 0, 0, 10, 1, 4084, 0, 7, 8, 1, 0, 8, 4085, 4085, 10, 0, 4024, 9, 4036, 7, 11, 0, 0, 4088, 2, 4085, 0, 9, 10, 4, 1686, 4086, 3, 4085, 4083, 6, 4082, 11, 5, 0, 10, 222, 0, 4063, 4084, 0, 3861, 8, 10, 10, 7, 0, 10, 0, 0, 7, 0, 9, 3973, 0, 4085, 8, 4, 3, 0, 8, 0, 10, 4, 4086, 0, 6, 4072, 0, 4043, 4080, 1, 4089, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4059, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3929, 0, 0, 0, 0, 4096, 0, 4096, 41, 0, 4093, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4054, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 227, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 20, 0, 12, 0, 4095, 4096, 0, 4092, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4046, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3894, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4091, 0, 8, 0, 0, 0, 0, 4089, 3141, 0, 0, 0, 0, 4096, 0, 4094, 76, 0, 4095, 0, 7, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4026, 0, 4086, 0, 0, 0, 0, 4090, 0, 4096, 6, 0, 0, 1, 777, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 8, 0, 16, 0, 4096, 4096, 9, 4073, 0, 0, 0, 0, 4, 0, 50, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4037, 4096, 383, 4087, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4005, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3090, 0, 0, 0, 0, 4096, 0, 4096, 274, 0, 4095, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3902, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1163, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 76, 0, 36, 0, 4096, 4096, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3979, 4096, 11, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4003, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3560, 0, 0, 0, 0, 4096, 0, 4096, 20, 0, 4095, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4070, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 608, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 20, 0, 19, 0, 4096, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4072, 4096, 15, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[3958, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4093, 0, 8, 0, 0, 5, 0, 4089, 3260, 0, 0, 0, 0, 4096, 0, 4096, 74, 0, 4096, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4029, 0, 4092, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 1127, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 10, 0, 26, 0, 4096, 4096, 7, 3964, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4015, 4096, 5, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4077, 1, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 1, 1, 4095, 0, 10, 0, 0, 0, 0, 4095, 14, 0, 1, 0, 4095, 1, 1, 0, 0, 0, 2, 4095, 2219, 2, 0, 0, 0, 4095, 0, 4095, 292, 0, 4095, 0, 0, 0, 1, 4095, 0, 0, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 4095, 4095, 1, 0, 3526, 1, 4077, 1, 1, 0, 0, 4095, 1, 4095, 0, 1, 1, 0, 2253, 4095, 1, 4095, 4095, 1, 4094, 1, 0, 27, 1, 15, 0, 4081, 4095, 0, 4022, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 4050, 0, 4095, 1, 1, 0, 0, 0, 0, 1, 0, 4095, 0, 1, 4094, 0, 4093, 4095, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3876, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 3167, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4080, 4068, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2604, 0, 0, 0, 0, 4096, 0, 893, 372, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3696, 0, 4090, 0, 0, 0, 0, 4095, 0, 3064, 1, 0, 0, 0, 1928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 119, 0, 35, 0, 4096, 13, 7, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, 0, 4096, 1, 4095, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 4096, 0, 3877, 4096, 4, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16261, 16381, 16154, 16379, 16874, 16431, 16396, 16196] avg=16384.0000 max=16874 max_rank=4 min=16154 min_rank=2 imbalance_ratio=1.029907 max_load_reduction=114198 max_load_reduction_pct=0.871262 moved_to_echo=65897 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4086, 9, 0, 0, 0, 7, 4077, 0, 8, 11, 0, 9, 8, 4, 4084, 9, 3159, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4088, 4, 0, 4, 0, 4085, 10, 0, 0, 0, 0, 4068, 4058, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4087, 2573, 41, 3, 4, 0, 4082, 5, 890, 90, 0, 4087, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4073, 0, 0, 10, 1, 4084, 0, 7, 8, 1, 0, 8, 4085, 4085, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 9, 4036, 7, 11, 0, 0, 4088, 2, 3055, 0, 9, 10, 4, 1686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4083, 6, 4082, 11, 5, 0, 10, 222, 0, 4063, 13, 0, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 10, 10, 7, 0, 10, 0, 0, 7, 0, 9, 3973, 0, 4085, 8, 4, 4086, 4085, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8, 0, 10, 4, 28, 0, 6, 4072, 0, 4043, 4080, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16493, 16357, 15883, 16390, 16941, 16359, 16305, 16344] avg=16384.0000 max=16941 max_rank=4 min=15883 min_rank=2 imbalance_ratio=1.033997 max_load_reduction=114131 max_load_reduction_pct=0.870750 moved_to_echo=65949 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4059, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 3167, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 4068, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3929, 0, 0, 0, 0, 4096, 0, 893, 41, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 4095, 0, 0, 0, 0, 4096, 0, 3064, 1, 0, 0, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 20, 0, 12, 0, 4095, 13, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 4096, 0, 4046, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16443, 16382, 17148, 16388, 15537, 16424, 16388, 16362] avg=16384.0000 max=17148 max_rank=2 min=15537 min_rank=4 imbalance_ratio=1.046631 max_load_reduction=113924 max_load_reduction_pct=0.869171 moved_to_echo=64711 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4003, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3167, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 4068, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3560, 0, 0, 0, 0, 4096, 0, 893, 20, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 4096, 0, 0, 0, 0, 4096, 0, 3064, 0, 0, 0, 0, 608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 20, 0, 19, 0, 4096, 13, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 4096, 0, 4072, 4096, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16384, 16760, 16385, 15934, 16433, 16387, 16403] avg=16384.0000 max=16760 max_rank=2 min=15934 min_rank=4 imbalance_ratio=1.022949 max_load_reduction=114312 max_load_reduction_pct=0.872131 moved_to_echo=65157 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3894, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 3172, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4091, 0, 8, 0, 0, 0, 4081, 4074, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3141, 0, 0, 0, 0, 4096, 0, 879, 76, 0, 4095, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 4086, 0, 0, 0, 0, 4090, 0, 3052, 6, 0, 0, 1, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 8, 0, 16, 0, 4096, 10, 9, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 50, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 4096, 0, 4037, 4096, 383, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16293, 16399, 16383, 16396, 16038, 16404, 16438, 16721] avg=16384.0000 max=16721 max_rank=7 min=16038 min_rank=4 imbalance_ratio=1.020569 max_load_reduction=114351 max_load_reduction_pct=0.872429 moved_to_echo=65601 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4077, 1, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 1, 1, 4095, 0, 3167, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4095, 14, 0, 1, 0, 4095, 1, 1, 0, 0, 0, 4079, 4067, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 2219, 2, 0, 0, 0, 4095, 0, 892, 292, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 4095, 4095, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3526, 1, 4077, 1, 1, 0, 0, 4095, 1, 3063, 0, 1, 1, 0, 2253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 4094, 1, 0, 27, 1, 15, 0, 4081, 13, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 4050, 0, 4095, 1, 1, 4095, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 28, 0, 1, 4094, 0, 4093, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16464, 16399, 15692, 16393, 17020, 16351, 16346, 16407] avg=16384.0000 max=17020 max_rank=4 min=15692 min_rank=2 imbalance_ratio=1.038818 max_load_reduction=114052 max_load_reduction_pct=0.870148 moved_to_echo=66124 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4005, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3167, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 4068, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3090, 0, 0, 0, 0, 4096, 0, 893, 274, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 4096, 0, 0, 0, 0, 4096, 0, 3064, 0, 0, 0, 0, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 76, 0, 36, 0, 4096, 13, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 4096, 0, 3979, 4096, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16377, 16544, 16387, 16321, 16362, 16387, 16306] avg=16384.0000 max=16544 max_rank=2 min=16306 min_rank=7 imbalance_ratio=1.009766 max_load_reduction=114528 max_load_reduction_pct=0.873779 moved_to_echo=65376 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3958, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 3167, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4093, 0, 8, 0, 0, 5, 4080, 4068, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3260, 0, 0, 0, 0, 4096, 0, 893, 74, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 4092, 0, 0, 0, 0, 4095, 0, 3064, 0, 0, 0, 0, 1127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 10, 0, 26, 0, 4096, 13, 7, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 4096, 0, 4015, 4096, 5, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16344, 16393, 16508, 16354, 16407, 16308, 16425, 16333] avg=16384.0000 max=16508 max_rank=2 min=16308 min_rank=5 imbalance_ratio=1.007568 max_load_reduction=114564 max_load_reduction_pct=0.874054 moved_to_echo=65473 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32000, 128, 0, 0, 0, 128, 32768, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 25344, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 0, 0, 32768, 128, 0, 128, 0, 32768, 128, 128, 0, 0, 128, 32640, 32640, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 32768, 128, 32768, 128, 128, 384, 128, 384, 0, 32768, 128, 128, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 24448, 128, 128, 128, 0, 32768, 128, 7168, 1280, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 0, 128, 0, 128, 32640, 128, 32768, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 0, 0, 128, 128, 32768, 0, 128, 128, 128, 0, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 128, 128, 256, 0, 128, 32768, 0, 32256, 32768, 512, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 31360, 128, 32768, 128, 128, 0, 0, 32768, 128, 24576, 128, 128, 128, 128, 9856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 10, 0, 1467, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 670, 0, 4083, 0, 0, 1, 0, 0, 4079, 0, 0, 4095, 0, 0, 2, 0, 0, 0, 0, 4084, 0, 1, 3546, 0, 4096, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 4096, 1, 4096, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4094, 4096, 3, 1, 4071, 0, 0, 4096, 0, 1, 4096, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 4096, 4095, 4096, 1, 4096, 2546, 0, 4095, 0, 13] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2760, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 292, 0, 4092, 0, 0, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3041, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 2113, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 7, 0, 1179, 5, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1498, 0, 4094, 0, 0, 0, 0, 0, 4069, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 2714, 0, 4096, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 1, 0, 4082, 0, 0, 4096, 0, 30, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 6, 4096, 2800, 0, 4091, 0, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[10, 0, 10, 0, 2, 4086, 11, 11, 2, 0, 0, 0, 7, 2609, 34, 10, 4086, 4081, 10, 6, 10, 0, 2, 4087, 8, 10, 0, 10, 4086, 0, 0, 857, 13, 3988, 13, 9, 0, 9, 10, 4087, 9, 0, 4078, 0, 0, 0, 0, 12, 0, 2, 4094, 0, 0, 2084, 0, 4080, 4085, 6, 0, 4087, 1, 0, 0, 0, 1, 0, 7, 4086, 0, 0, 0, 9, 0, 0, 9, 9, 4086, 4079, 0, 4086, 0, 0, 9, 0, 0, 0, 9, 9, 8, 29, 9, 0, 4051, 12, 1, 10, 4086, 1, 4086, 3406, 0, 0, 4093, 0, 10, 4080, 0, 0, 4079, 0, 4084, 3, 0, 0, 4086, 15, 0, 11, 4085, 4086, 4086, 0, 4083, 3368, 2, 4083, 8, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 39, 0, 2077, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 211, 0, 4093, 0, 0, 2, 0, 0, 4067, 0, 0, 4092, 0, 11, 6, 0, 0, 0, 0, 4050, 0, 0, 2692, 0, 4096, 4091, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 4095, 4096, 0, 4096, 2, 5, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 4096, 0, 0, 0, 4096, 0, 4088, 4096, 7, 4, 4054, 0, 0, 4096, 0, 28, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4090, 4096, 12, 4096, 3210, 0, 4082, 0, 15] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1386, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 765, 0, 4094, 0, 0, 0, 0, 0, 4066, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3076, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 2977, 0, 4096, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 25, 0, 280, 2, 0, 4096, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 1535, 0, 4092, 0, 0, 0, 0, 0, 4079, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 4067, 0, 0, 2831, 0, 4096, 4096, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 4096, 4, 0, 4070, 0, 0, 4096, 0, 15, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 7, 4096, 3565, 0, 4084, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 4095, 1, 1, 0, 0, 0, 1, 0, 2761, 3, 1, 4095, 4095, 1, 0, 1, 0, 1, 4095, 1, 1, 0, 1, 4095, 0, 0, 403, 1, 4095, 1, 1, 0, 1, 1, 4093, 1, 0, 4095, 0, 1, 0, 0, 1, 0, 0, 4095, 0, 0, 2264, 0, 4095, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 1, 0, 0, 1, 1, 4095, 4095, 0, 4095, 0, 0, 1, 0, 0, 0, 1, 1, 1, 3, 0, 0, 4091, 1, 0, 1, 4095, 0, 4095, 4030, 0, 0, 4095, 0, 1, 4093, 0, 0, 4095, 0, 4094, 1, 0, 0, 4095, 2, 0, 1, 4095, 4095, 4095, 0, 4095, 2828, 0, 4094, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 10, 0, 1467, 0, 0, 4096, 3996, 2343, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3317, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 1, 0, 0, 4079, 0, 0, 4095, 0, 0, 2, 0, 0, 2757, 779, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 1, 3546, 0, 1339, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 3, 1, 4071, 0, 0, 4096, 0, 1, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1169, 4095, 0, 1, 4096, 2546, 0, 4095, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16027, 16274, 16380, 17160, 16385, 16368, 16462, 16016] avg=16384.0000 max=17160 max_rank=3 min=16016 min_rank=7 imbalance_ratio=1.047363 max_load_reduction=113912 max_load_reduction_pct=0.869080 moved_to_echo=65231 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2760, 0, 0, 4096, 3996, 2343, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3317, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 0, 0, 0, 2757, 779, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3041, 0, 1339, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1169, 4096, 0, 0, 4096, 2113, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17309, 15897, 16393, 16668, 16384, 16366, 16485, 15570] avg=16384.0000 max=17309 max_rank=0 min=15570 min_rank=7 imbalance_ratio=1.056458 max_load_reduction=113763 max_load_reduction_pct=0.867943 moved_to_echo=64805 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 10, 0, 2, 4086, 11, 11, 2, 0, 0, 0, 7, 2609, 34, 10, 4084, 3979, 2337, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3309, 4081, 10, 6, 10, 0, 2, 4087, 8, 10, 0, 10, 4086, 0, 0, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3988, 13, 9, 0, 9, 10, 4087, 9, 0, 4078, 0, 0, 0, 0, 12, 2747, 777, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 2084, 0, 1333, 4085, 6, 0, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 4086, 0, 0, 0, 9, 0, 0, 9, 9, 4086, 4079, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 9, 9, 8, 29, 9, 0, 4051, 12, 1, 10, 4086, 4086, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 3406, 0, 0, 4093, 0, 10, 4080, 0, 0, 100, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 11, 1165, 4086, 0, 0, 4083, 3368, 2, 4083, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17210, 16476, 16335, 15692, 16372, 16387, 15779, 16821] avg=16384.0000 max=17210 max_rank=0 min=15692 min_rank=3 imbalance_ratio=1.050415 max_load_reduction=113862 max_load_reduction_pct=0.868698 moved_to_echo=65359 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 7, 0, 1179, 5, 0, 4096, 3996, 2343, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3317, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4069, 0, 0, 4096, 0, 0, 0, 0, 0, 2757, 779, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 2714, 0, 1339, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 4082, 0, 0, 4096, 0, 30, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1169, 4096, 0, 6, 4096, 2800, 0, 4091, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15740, 17103, 16379, 16296, 16384, 16404, 16501, 16265] avg=16384.0000 max=17103 max_rank=1 min=15740 min_rank=0 imbalance_ratio=1.043884 max_load_reduction=113969 max_load_reduction_pct=0.869514 moved_to_echo=65554 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 4095, 1, 1, 0, 0, 0, 1, 0, 2761, 3, 1, 4094, 3995, 2343, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3317, 4095, 1, 0, 1, 0, 1, 4095, 1, 1, 0, 1, 4095, 0, 0, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 0, 1, 1, 4093, 1, 0, 4095, 0, 1, 0, 0, 1, 2757, 778, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2264, 0, 1338, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 1, 0, 0, 1, 1, 4095, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 3, 0, 0, 4091, 1, 0, 1, 4095, 4095, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4030, 0, 0, 4095, 0, 1, 4093, 0, 0, 100, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1168, 4095, 0, 0, 4095, 2828, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17314, 16011, 16410, 15887, 16384, 16367, 16415, 16284] avg=16384.0000 max=17314 max_rank=0 min=15887 min_rank=3 imbalance_ratio=1.056763 max_load_reduction=113758 max_load_reduction_pct=0.867905 moved_to_echo=65450 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 39, 0, 2077, 0, 0, 4096, 3998, 2350, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3314, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 2, 0, 0, 4067, 0, 0, 4092, 0, 11, 6, 0, 0, 2763, 781, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 2692, 0, 1333, 4091, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 4095, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 7, 4, 4054, 0, 0, 4096, 0, 28, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1157, 4090, 0, 12, 4096, 3210, 0, 4082, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16679, 15807, 16404, 16251, 16392, 16406, 16471, 16662] avg=16384.0000 max=16679 max_rank=0 min=15807 min_rank=1 imbalance_ratio=1.018005 max_load_reduction=114393 max_load_reduction_pct=0.872749 moved_to_echo=65931 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1386, 0, 0, 4096, 3996, 2343, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3317, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4066, 0, 0, 4096, 0, 0, 0, 0, 0, 2757, 779, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3076, 0, 1339, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1169, 4096, 0, 0, 4096, 2977, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15935, 16370, 16376, 16702, 16383, 16388, 16483, 16435] avg=16384.0000 max=16702 max_rank=3 min=15935 min_rank=0 imbalance_ratio=1.019409 max_load_reduction=114370 max_load_reduction_pct=0.872574 moved_to_echo=65689 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 25, 0, 280, 2, 0, 4096, 3996, 2343, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3317, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 1535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4079, 0, 0, 4096, 0, 8, 0, 0, 0, 2757, 779, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 2831, 0, 1339, 4096, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4, 0, 4070, 0, 0, 4096, 0, 15, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1169, 4096, 0, 7, 4096, 3565, 0, 4084, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14858, 17134, 16395, 16416, 16388, 16386, 16476, 17019] avg=16384.0000 max=17134 max_rank=1 min=14858 min_rank=0 imbalance_ratio=1.045776 max_load_reduction=113938 max_load_reduction_pct=0.869278 moved_to_echo=66269 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[26624, 32768, 128, 128, 128, 0, 128, 32768, 128, 128, 0, 128, 32768, 0, 0, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 128, 128, 0, 128, 128, 32768, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 32768, 32128, 128, 128, 32768, 128, 128, 32768, 0, 128, 896, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 0, 128, 22272, 0, 10752, 32768, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 0, 128, 32768, 128, 128, 128, 0, 128, 128, 128, 14592, 128, 128, 32768, 32000, 18816, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 128, 128, 9344, 32768, 0, 128, 32768, 23424, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32640, 128, 128, 128, 128, 128, 32640, 128, 0, 32768, 0, 128, 128, 0, 128, 22144, 6272, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 0, 0, 128, 128, 128, 256, 128, 0, 32768, 128, 128, 128, 32768, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4050, 4, 46, 4083, 12, 3, 118, 3874, 3266, 0, 5, 4020, 1, 6, 10, 0, 0, 0, 25, 156, 10, 244, 4084, 2, 0, 11, 0, 0, 4086, 4051, 0, 0, 4066, 0, 4, 33, 4087, 4086, 0, 19, 5, 7, 0, 9, 30, 0, 7, 12, 10, 0, 10, 0, 0, 7, 6, 4085, 4086, 10, 0, 4064, 0, 9, 10, 4084, 2, 0, 0, 0, 0, 4062, 4086, 1, 10, 0, 0, 13, 3951, 0, 0, 10, 10, 0, 10, 4084, 4086, 5, 0, 1816, 0, 2861, 2, 0, 8, 0, 4071, 3, 0, 4076, 10, 11, 0, 5, 4047, 0, 10, 4, 4082, 9, 0, 59, 4080, 0, 4086, 4084, 33, 182, 4081, 0, 4083, 10, 11, 4086, 0, 3978, 11, 12, 8, 10] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[3132, 0, 0, 4096, 0, 0, 0, 4096, 3228, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1506, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 1, 4096, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4092, 4095, 0, 2, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 334, 0, 4096, 1, 0, 0, 0, 4096, 0, 10, 4096, 0, 0, 1, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 4084, 4095, 0, 0, 4096, 0, 4094, 0, 0, 4095, 1, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[2752, 0, 0, 4096, 0, 0, 0, 4096, 3350, 0, 0, 4096, 0, 0, 0, 9, 3, 0, 0, 0, 0, 1458, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4054, 4096, 0, 6, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 654, 0, 4096, 0, 0, 0, 0, 4096, 0, 14, 4096, 0, 0, 0, 0, 4096, 7, 0, 0, 4096, 0, 0, 0, 4096, 0, 4085, 4095, 0, 0, 4096, 0, 4094, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[3377, 0, 0, 4096, 0, 0, 0, 4092, 3187, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 789, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 839, 0, 4096, 1, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 4096, 4096, 0, 1, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1224, 0, 0, 4096, 0, 0, 0, 4096, 3623, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1563, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1783, 0, 4096, 1, 0, 0, 0, 4096, 0, 11, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4082, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3877, 0, 1, 4096, 0, 0, 0, 4096, 2842, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 657, 4096, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4090, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 4092, 0, 0, 824, 0, 4096, 0, 0, 0, 5, 4096, 0, 7, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4079, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[2944, 0, 0, 4096, 0, 0, 0, 4096, 3079, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1491, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 676, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 4096, 0, 1, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[3793, 0, 5, 4095, 1, 0, 0, 4000, 2322, 0, 1, 4094, 0, 1, 1, 0, 0, 0, 4, 22, 1, 1844, 4095, 0, 0, 1, 0, 0, 4095, 4090, 0, 0, 4094, 0, 0, 0, 4095, 4095, 0, 2, 0, 1, 0, 1, 8, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 4095, 4095, 1, 0, 4089, 0, 1, 1, 4095, 1, 1, 0, 0, 0, 4095, 4095, 0, 1, 0, 0, 1, 4081, 0, 0, 1, 1, 0, 1, 4095, 4095, 0, 0, 316, 0, 4078, 0, 0, 1, 0, 4094, 0, 0, 4095, 1, 1, 0, 1, 4094, 0, 1, 1, 4095, 1, 0, 15, 4092, 0, 4095, 4095, 6, 13, 4095, 0, 4095, 0, 1, 4095, 0, 4077, 1, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4050, 4, 46, 1954, 12, 3, 118, 3874, 3266, 0, 5, 4020, 1, 6, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 156, 10, 244, 4084, 2, 0, 11, 0, 0, 4086, 4051, 0, 0, 2114, 744, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 4, 33, 4087, 4086, 0, 19, 5, 7, 0, 9, 30, 0, 7, 12, 4062, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 10, 0, 0, 7, 6, 4085, 4086, 10, 0, 4064, 0, 9, 10, 4084, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4062, 4086, 1, 10, 0, 0, 13, 3951, 0, 0, 10, 4086, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 10, 3340, 4086, 5, 0, 1816, 0, 2861, 2, 0, 8, 0, 4071, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 10, 11, 0, 5, 4047, 0, 10, 4, 4073, 9, 0, 59, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 33, 182, 4074, 0, 4083, 10, 11, 0, 0, 3978, 11, 12, 8, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17369, 15540, 16442, 16384, 16243, 16212, 16384, 16498] avg=16384.0000 max=17369 max_rank=0 min=15540 min_rank=1 imbalance_ratio=1.060120 max_load_reduction=113703 max_load_reduction_pct=0.867485 moved_to_echo=65337 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3132, 0, 0, 1960, 0, 0, 0, 4096, 3228, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1506, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 1, 2121, 747, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4092, 4095, 0, 2, 4096, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4095, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3349, 4094, 0, 0, 334, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 1, 0, 4096, 2, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4089, 0, 4094, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16512, 16676, 16371, 16387, 16404, 15970, 16388, 16364] avg=16384.0000 max=16676 max_rank=1 min=15970 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=114396 max_load_reduction_pct=0.872772 moved_to_echo=65126 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2752, 0, 0, 1951, 0, 0, 0, 4096, 3350, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1458, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2124, 750, 14, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4054, 4096, 0, 6, 4096, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3346, 4089, 0, 0, 654, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 4096, 7, 0, 0, 4082, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4080, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16254, 16645, 16373, 16367, 16406, 16281, 16391, 16355] avg=16384.0000 max=16645 max_rank=1 min=16254 min_rank=0 imbalance_ratio=1.015930 max_load_reduction=114427 max_load_reduction_pct=0.873009 moved_to_echo=65433 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3377, 0, 0, 1960, 0, 0, 0, 4092, 3187, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 789, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2121, 747, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3349, 4096, 0, 0, 839, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4089, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16712, 15958, 16377, 16387, 16406, 16477, 16377, 16378] avg=16384.0000 max=16712 max_rank=0 min=15958 min_rank=1 imbalance_ratio=1.020020 max_load_reduction=114360 max_load_reduction_pct=0.872498 moved_to_echo=65638 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1224, 0, 0, 1960, 0, 0, 0, 4096, 3623, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1563, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 2121, 747, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3349, 4096, 0, 0, 1783, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14999, 16733, 16377, 16387, 16406, 17421, 16386, 16363] avg=16384.0000 max=17421 max_rank=5 min=14999 min_rank=0 imbalance_ratio=1.063293 max_load_reduction=113651 max_load_reduction_pct=0.867088 moved_to_echo=66576 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3793, 0, 5, 1960, 1, 0, 0, 4000, 2322, 0, 1, 4094, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 22, 1, 1844, 4095, 0, 0, 1, 0, 0, 4095, 4090, 0, 0, 2120, 746, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4095, 4095, 0, 2, 0, 1, 0, 1, 8, 0, 1, 1, 4073, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 4095, 4095, 1, 0, 4089, 0, 1, 1, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4095, 4095, 0, 1, 0, 0, 1, 4081, 0, 0, 1, 4095, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3349, 4095, 0, 0, 316, 0, 4078, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 1, 4094, 0, 1, 1, 4086, 1, 0, 15, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 6, 13, 4088, 0, 4095, 0, 1, 0, 0, 4077, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16178, 17031, 16386, 16384, 16393, 15935, 16388, 16377] avg=16384.0000 max=17031 max_rank=1 min=15935 min_rank=5 imbalance_ratio=1.039490 max_load_reduction=114041 max_load_reduction_pct=0.870064 moved_to_echo=65093 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3877, 0, 1, 1960, 0, 0, 0, 4096, 2842, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 4096, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 2121, 747, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4090, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3349, 4092, 0, 0, 824, 0, 4096, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16873, 15828, 16370, 16389, 16407, 16462, 16383, 16360] avg=16384.0000 max=16873 max_rank=0 min=15828 min_rank=1 imbalance_ratio=1.029846 max_load_reduction=114199 max_load_reduction_pct=0.871269 moved_to_echo=65612 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2944, 0, 0, 1960, 0, 0, 0, 4096, 3079, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1491, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 2121, 747, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3349, 4096, 0, 0, 676, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 4089, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16175, 16661, 16376, 16387, 16407, 16314, 16375, 16377] avg=16384.0000 max=16661 max_rank=1 min=16175 min_rank=0 imbalance_ratio=1.016907 max_load_reduction=114411 max_load_reduction_pct=0.872887 moved_to_echo=65473 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[25216, 128, 128, 15744, 128, 128, 128, 32512, 24960, 0, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 32768, 128, 128, 128, 32768, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 0, 128, 128, 32768, 32768, 0, 128, 128, 128, 0, 128, 128, 0, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 26880, 32768, 128, 0, 7296, 0, 31616, 128, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 256, 128, 9600, 32768, 128, 0, 128, 0, 0, 32768, 32768, 0, 128, 17024, 6016, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 0, 128, 256, 32768, 0, 32768, 128, 128, 0, 128, 32640, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 32768, 32768, 128, 128, 32768, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 0, 0, 32768, 32768, 128, 128, 0, 0, 128, 32640, 128, 128, 128, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 2978, 4096, 0, 0, 4096, 0, 0, 0, 0, 3545, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 2652, 4096, 1661, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4086, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 1279, 4096, 0, 0, 4096, 0, 0, 0, 0, 3576, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 18, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 4096, 3102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 13, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4092, 0, 2, 0, 3761, 4096, 562, 4089, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4075, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 4096, 4096, 0, 4095, 0, 0, 0, 0, 2665, 4096, 0, 0, 4096, 0, 0, 0, 0, 3923, 0, 5, 0, 4091, 0, 0, 0, 0, 0, 24, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1094, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 4086, 0, 0, 0, 2850, 4096, 1768, 4083, 0, 0, 0, 4, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4085, 0, 4096, 0, 1, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 6, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 2298, 4096, 0, 0, 4096, 0, 0, 0, 0, 3793, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 91, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2986, 4096, 1874, 4080, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[11, 4070, 5, 0, 133, 4, 3, 8, 4085, 44, 11, 33, 3952, 10, 11, 0, 4059, 0, 0, 4085, 4086, 19, 4054, 0, 11, 0, 8, 812, 4065, 0, 8, 4086, 11, 79, 1, 13, 1731, 0, 0, 0, 4086, 0, 14, 2, 11, 0, 0, 417, 4085, 10, 3779, 10, 4084, 0, 11, 0, 0, 4085, 11, 9, 4075, 3270, 0, 0, 0, 0, 0, 11, 9, 0, 4, 4085, 4063, 2, 4081, 3, 0, 4, 11, 9, 7, 0, 4081, 11, 4085, 0, 0, 12, 4079, 7, 0, 1417, 3222, 4085, 1668, 4084, 0, 3, 10, 0, 4081, 24, 0, 8, 2, 0, 4080, 10, 0, 0, 10, 9, 4010, 11, 4081, 7, 0, 0, 2, 0, 19, 4083, 27, 4083, 5, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 4096, 4096, 0, 4095, 0, 0, 0, 0, 2820, 4096, 0, 0, 4095, 0, 0, 0, 0, 2933, 0, 1, 0, 4095, 0, 0, 0, 0, 1, 6, 0, 4096, 0, 4096, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 4096, 320, 0, 1, 0, 0, 0, 0, 0, 1, 0, 4095, 4096, 0, 4096, 70, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4095, 0, 2, 0, 2935, 4096, 3211, 4090, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 1, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1, 4093, 1, 0, 12, 1, 1, 0, 4095, 14, 1, 3, 4088, 1, 1, 0, 4092, 0, 0, 4095, 4095, 2, 4094, 0, 1, 0, 1, 1183, 4094, 0, 0, 4095, 1, 7, 1, 1, 2917, 0, 0, 0, 4095, 0, 1, 0, 1, 0, 0, 60, 4095, 1, 4062, 1, 4095, 0, 1, 0, 0, 4095, 1, 1, 4094, 1494, 0, 0, 0, 3, 0, 1, 1, 0, 0, 4095, 4093, 0, 4095, 39, 0, 0, 1, 1, 1, 0, 4095, 1, 4095, 0, 0, 1, 4095, 1, 0, 4, 3455, 4095, 3142, 4095, 0, 0, 0, 0, 4095, 2, 0, 1, 1, 0, 4095, 1, 0, 0, 1, 1, 4089, 1, 4095, 0, 0, 0, 0, 0, 1, 4095, 2, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 1922, 4096, 0, 0, 4096, 0, 0, 0, 0, 3467, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 68, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2979, 4096, 2786, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3318, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2112, 0, 0, 4096, 0, 0, 0, 0, 2978, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3545, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2507, 0, 0, 0, 0, 4096, 0, 0, 4096, 1438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 15, 0, 0, 0, 0, 1589, 1254, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2835, 0, 0, 0, 4096, 0, 0, 1, 2652, 0, 1661, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3315, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16374, 17378, 16615, 16232, 16360, 15341, 16381, 16391] avg=16384.0000 max=17378 max_rank=1 min=15341 min_rank=5 imbalance_ratio=1.060669 max_load_reduction=113694 max_load_reduction_pct=0.867416 moved_to_echo=64473 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 4093, 1, 0, 12, 1, 1, 0, 4095, 14, 1, 3, 4088, 1, 1, 0, 3316, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 2111, 0, 2, 4094, 0, 1, 0, 1, 1183, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 1, 1, 2917, 0, 0, 0, 4095, 0, 1, 0, 1, 0, 0, 60, 4095, 4095, 780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4062, 1, 2507, 0, 1, 0, 0, 4095, 1, 1, 4094, 1494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 1, 0, 0, 4095, 4093, 0, 4095, 39, 0, 0, 1, 1, 1588, 1254, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 2834, 0, 0, 1, 4095, 1, 0, 4, 3455, 0, 3142, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 1, 1, 0, 4095, 1, 0, 0, 1, 1, 4095, 3315, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 4095, 0, 0, 0, 0, 0, 1, 4095, 2, 4095, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16398, 15578, 16054, 16257, 16385, 17629, 16386, 16385] avg=16384.0000 max=17629 max_rank=5 min=15578 min_rank=1 imbalance_ratio=1.075989 max_load_reduction=113443 max_load_reduction_pct=0.865501 moved_to_echo=66785 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3317, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2112, 0, 0, 4096, 0, 0, 0, 0, 2298, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3793, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2507, 0, 0, 0, 0, 4096, 0, 0, 4096, 1265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 91, 0, 0, 0, 0, 1589, 1254, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2835, 0, 0, 0, 4096, 0, 0, 0, 2986, 0, 1874, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3315, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16698, 16862, 16057, 16436, 15871, 16382, 16391] avg=16384.0000 max=16862 max_rank=2 min=15871 min_rank=5 imbalance_ratio=1.029175 max_load_reduction=114210 max_load_reduction_pct=0.871353 moved_to_echo=65080 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3317, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 2112, 0, 0, 4095, 0, 0, 0, 0, 2820, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2933, 0, 1, 0, 4095, 0, 0, 0, 0, 1, 6, 0, 4096, 4095, 781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2507, 0, 0, 0, 0, 4095, 0, 0, 4096, 320, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 4096, 0, 4096, 70, 0, 0, 0, 0, 1588, 1254, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2834, 0, 0, 0, 4095, 0, 2, 0, 2935, 0, 3211, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3315, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 1, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 17220, 16008, 15115, 16414, 17167, 16383, 16392] avg=16384.0000 max=17220 max_rank=1 min=15115 min_rank=3 imbalance_ratio=1.051025 max_load_reduction=113852 max_load_reduction_pct=0.868622 moved_to_echo=66356 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4086, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3318, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2112, 0, 0, 4096, 0, 0, 0, 0, 1279, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3576, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 18, 0, 4096, 4096, 782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2507, 0, 0, 0, 0, 4094, 0, 0, 4096, 3102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 13, 0, 0, 0, 0, 1589, 1254, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2835, 0, 0, 0, 4092, 0, 2, 0, 3761, 0, 562, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3314, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16366, 15679, 16664, 17894, 16358, 15341, 16376, 16394] avg=16384.0000 max=17894 max_rank=3 min=15341 min_rank=5 imbalance_ratio=1.092163 max_load_reduction=113178 max_load_reduction_pct=0.863480 moved_to_echo=64469 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4075, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3318, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 2102, 0, 0, 4095, 0, 0, 0, 0, 2665, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 5, 0, 4091, 0, 0, 0, 0, 0, 24, 0, 4096, 4096, 782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2504, 0, 0, 0, 0, 4096, 0, 0, 4096, 1094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 15, 0, 0, 0, 0, 1592, 1259, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2819, 0, 0, 0, 4086, 0, 0, 0, 2850, 0, 1768, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3314, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4096, 0, 1, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 6, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16359, 17055, 17017, 15886, 16378, 15606, 16378, 16393] avg=16384.0000 max=17055 max_rank=1 min=15606 min_rank=5 imbalance_ratio=1.040955 max_load_reduction=114017 max_load_reduction_pct=0.869881 moved_to_echo=64755 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[11, 4070, 5, 0, 133, 4, 3, 8, 4085, 44, 11, 33, 3952, 10, 11, 0, 3305, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 2106, 0, 19, 4054, 0, 11, 0, 8, 812, 4065, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 79, 1, 13, 1731, 0, 0, 0, 4086, 0, 14, 2, 11, 0, 0, 417, 4086, 4086, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3779, 10, 2500, 0, 11, 0, 0, 4085, 11, 9, 4075, 3270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 9, 0, 4, 4085, 4063, 2, 4081, 3, 0, 4, 11, 9, 1584, 1251, 1211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 11, 2827, 0, 0, 12, 4079, 7, 0, 1417, 3222, 0, 1668, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 10, 0, 4081, 24, 0, 8, 2, 0, 4080, 10, 0, 0, 10, 9, 4085, 3306, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 11, 4081, 7, 0, 0, 2, 0, 19, 4083, 27, 4083, 5, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16453, 15142, 15316, 17760, 16328, 17334, 16404, 16335] avg=16384.0000 max=17760 max_rank=3 min=15142 min_rank=1 imbalance_ratio=1.083984 max_load_reduction=113312 max_load_reduction_pct=0.864502 moved_to_echo=66401 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3317, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2112, 0, 0, 4096, 0, 0, 0, 0, 1922, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3467, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2507, 0, 0, 0, 0, 4096, 0, 0, 4096, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 68, 0, 0, 0, 0, 1589, 1254, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2835, 0, 0, 0, 4096, 0, 0, 0, 2979, 0, 2786, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3315, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16374, 16322, 16536, 15871, 16413, 16783, 16382, 16391] avg=16384.0000 max=16783 max_rank=5 min=15871 min_rank=3 imbalance_ratio=1.024353 max_load_reduction=114289 max_load_reduction_pct=0.871956 moved_to_echo=65969 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 128, 128, 0, 32768, 128, 0, 0, 128, 128, 32768, 26624, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 16896, 0, 128, 32768, 0, 128, 0, 128, 16000, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 128, 0, 256, 128, 128, 128, 32768, 128, 128, 128, 32640, 128, 128, 0, 26624, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 32512, 128, 20096, 0, 128, 0, 0, 32768, 128, 128, 32768, 13184, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 25984, 0, 128, 0, 32768, 0, 128, 128, 128, 128, 128, 512, 32768, 32768, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 32768, 32768, 128, 32768, 384, 0, 128, 128, 128, 12800, 10112, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 22656, 0, 0, 128, 32768, 128, 128, 1536, 24960, 0, 16768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 32768, 128, 128, 128, 128, 0, 128, 32768, 128, 32768, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4078, 0, 0, 0, 4096, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 138, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3730, 4096, 0, 0, 0, 0, 1, 4096, 4095, 0, 4096, 0, 4096, 0, 4096, 0, 3926, 0, 0, 0, 4096, 0, 4096, 167, 0, 0, 1, 0, 4094, 4096, 4084, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[7, 12, 2, 13, 360, 0, 0, 0, 4084, 3, 12, 0, 4080, 3756, 11, 1798, 11, 11, 0, 15, 0, 0, 0, 56, 4017, 0, 916, 0, 0, 4072, 4075, 4081, 0, 12, 9, 0, 4084, 14, 10, 1446, 0, 49, 10, 0, 10, 0, 247, 1, 3970, 3951, 14, 11, 11, 12, 4085, 0, 8, 0, 4082, 16, 11, 11, 0, 0, 11, 4084, 0, 10, 3842, 9, 11, 0, 3936, 443, 4003, 113, 14, 0, 0, 80, 4080, 4082, 0, 4083, 0, 3997, 12, 4029, 0, 3701, 10, 5, 14, 4082, 112, 3994, 0, 0, 2, 109, 0, 4086, 4055, 4080, 72, 4084, 0, 6, 32, 0, 4054, 0, 0, 0, 0, 4, 0, 4068, 4084, 15, 13, 10, 10, 25, 27, 0, 13, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 0, 4094, 0, 0, 0, 4096, 4096, 0, 9, 0, 0, 1, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3002, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 4037, 0, 0, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4014, 0, 0, 0, 4096, 4096, 0, 15, 0, 0, 7, 0, 0, 0, 0, 185, 4096, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 1566, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2414, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 3861, 0, 0, 0, 4096, 0, 4096, 250, 0, 0, 8, 0, 4089, 4096, 4087, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4081, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4027, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 1430, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2223, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 3965, 0, 0, 0, 4096, 0, 4096, 358, 0, 0, 0, 0, 4096, 4096, 4088, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 22, 0, 0, 0, 0, 0, 0, 0, 389, 4096, 0, 0, 0, 0, 4096, 4096, 4087, 0, 0, 0, 0, 4096, 0, 0, 2514, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1281, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 3867, 0, 0, 0, 4096, 0, 4096, 164, 0, 0, 8, 0, 4088, 4096, 4075, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4081, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4087, 0, 0, 0, 4096, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 235, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 1756, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2084, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 4019, 0, 0, 0, 4096, 0, 4096, 100, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 1, 1, 105, 0, 0, 0, 4090, 0, 0, 0, 4095, 4034, 1, 353, 0, 1, 0, 1, 1, 0, 0, 17, 4092, 0, 146, 0, 0, 4095, 4095, 4095, 0, 1, 1, 0, 4095, 1, 1, 1803, 0, 3, 1, 0, 1, 0, 44, 0, 4081, 4082, 1, 1, 1, 1, 4095, 0, 1, 0, 4095, 3, 1, 1, 0, 0, 1, 4095, 0, 1, 4057, 0, 1, 0, 4092, 1879, 4087, 6, 1, 0, 0, 11, 4094, 4095, 0, 4095, 0, 4088, 1, 4084, 0, 3919, 1, 0, 2, 4095, 14, 4091, 33, 0, 1, 19, 0, 4095, 4094, 4095, 12, 4095, 0, 0, 4, 0, 4082, 0, 0, 1, 0, 1, 0, 4090, 4095, 1, 1, 1, 1, 0, 3, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[7, 12, 2, 13, 360, 0, 0, 0, 4084, 3, 12, 0, 4080, 3756, 11, 1798, 2088, 1331, 282, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 11, 0, 15, 0, 0, 0, 56, 4017, 0, 916, 0, 0, 4072, 3793, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 9, 0, 4084, 14, 10, 1446, 0, 49, 10, 0, 10, 0, 247, 1, 4084, 4083, 2555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 3951, 14, 11, 11, 12, 4085, 0, 8, 0, 4082, 16, 11, 11, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1996, 0, 10, 3842, 9, 11, 0, 3936, 443, 4003, 113, 14, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0, 3997, 12, 4029, 0, 3701, 10, 5, 14, 0, 112, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 109, 0, 3944, 4055, 4080, 72, 0, 0, 6, 32, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4068, 4084, 15, 13, 10, 10, 25, 27, 0, 13, 0, 4082, 2549, 1527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17954, 16972, 16614, 16209, 14468, 16074, 16354, 16427] avg=16384.0000 max=17954 max_rank=0 min=14468 min_rank=4 imbalance_ratio=1.095825 max_load_reduction=113118 max_load_reduction_pct=0.863022 moved_to_echo=63323 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 0, 4094, 0, 0, 0, 4096, 4096, 0, 9, 2094, 1337, 283, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 4095, 3813, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2002, 0, 0, 4096, 0, 0, 0, 4096, 3002, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0, 4096, 0, 4096, 0, 4037, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 3954, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2559, 1533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16128, 16105, 15983, 16412, 17291, 16525, 16248, 16380] avg=16384.0000 max=17291 max_rank=4 min=15983 min_rank=2 imbalance_ratio=1.055359 max_load_reduction=113781 max_load_reduction_pct=0.868080 moved_to_echo=66444 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 1, 105, 0, 0, 0, 4090, 0, 0, 0, 4095, 4034, 1, 353, 2094, 1336, 283, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 17, 4092, 0, 146, 0, 0, 4095, 3812, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 1, 1, 1803, 0, 3, 1, 0, 1, 0, 44, 0, 4095, 4095, 2562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4082, 1, 1, 1, 1, 4095, 0, 1, 0, 4095, 3, 1, 1, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2001, 0, 1, 4057, 0, 1, 0, 4092, 1879, 4087, 6, 1, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0, 4088, 1, 4084, 0, 3919, 1, 0, 2, 0, 14, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 1, 19, 0, 3953, 4094, 4095, 12, 0, 0, 0, 4, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4090, 4095, 1, 1, 1, 1, 0, 3, 0, 1, 0, 4095, 2558, 1533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16508, 16260, 16703, 16390, 16137, 16400, 16293, 16381] avg=16384.0000 max=16703 max_rank=2 min=16137 min_rank=4 imbalance_ratio=1.019470 max_load_reduction=114369 max_load_reduction_pct=0.872566 moved_to_echo=65211 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4014, 0, 0, 0, 4096, 4096, 0, 15, 2101, 1339, 289, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 185, 4096, 0, 0, 0, 0, 4096, 3807, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1566, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1995, 0, 0, 4096, 0, 0, 0, 4096, 2414, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 0, 0, 4096, 0, 4096, 0, 3861, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 0, 0, 8, 0, 3938, 4096, 4087, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 2562, 1533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16070, 16286, 16417, 16417, 16697, 16344, 16469, 16372] avg=16384.0000 max=16697 max_rank=4 min=16070 min_rank=0 imbalance_ratio=1.019104 max_load_reduction=114375 max_load_reduction_pct=0.872612 moved_to_echo=65882 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4027, 0, 0, 0, 4096, 4096, 0, 0, 2094, 1337, 283, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 4096, 0, 0, 0, 0, 4096, 3813, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1430, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2002, 0, 0, 4096, 0, 0, 0, 4096, 2223, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0, 4096, 0, 4096, 0, 3965, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 0, 0, 0, 0, 3954, 4096, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2559, 1533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16049, 16393, 16281, 16411, 16513, 16453, 16592, 16380] avg=16384.0000 max=16592 max_rank=6 min=16049 min_rank=0 imbalance_ratio=1.012695 max_load_reduction=114480 max_load_reduction_pct=0.873413 moved_to_echo=65938 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 22, 2094, 1337, 283, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 389, 4096, 0, 0, 0, 0, 4096, 3813, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2514, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 2563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2002, 0, 0, 4096, 0, 0, 0, 4096, 1281, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0, 4096, 0, 4096, 0, 3867, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 8, 0, 3946, 4096, 4075, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 2559, 1533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16138, 16481, 17366, 16411, 15571, 16355, 16379, 16371] avg=16384.0000 max=17366 max_rank=2 min=15571 min_rank=4 imbalance_ratio=1.059937 max_load_reduction=113706 max_load_reduction_pct=0.867508 moved_to_echo=64676 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4087, 0, 0, 0, 4096, 4096, 0, 4, 2094, 1337, 283, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 4096, 0, 0, 0, 0, 4096, 3813, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1756, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4096, 2562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2002, 0, 0, 4096, 0, 0, 0, 4096, 2084, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0, 4096, 0, 4096, 0, 4019, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 3954, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2559, 1534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16113, 16336, 16608, 16411, 16374, 16507, 16342, 16381] avg=16384.0000 max=16608 max_rank=2 min=16113 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=114464 max_load_reduction_pct=0.873291 moved_to_echo=65604 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4078, 0, 0, 0, 4096, 4096, 0, 12, 2094, 1337, 283, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 4096, 0, 0, 0, 0, 4096, 3813, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2002, 0, 0, 4096, 0, 0, 0, 4096, 3730, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0, 4096, 0, 4096, 0, 3926, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 1, 0, 3952, 4096, 4084, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 2559, 1534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16112, 16239, 15100, 16411, 18021, 16414, 16395, 16380] avg=16384.0000 max=18021 max_rank=4 min=15100 min_rank=2 imbalance_ratio=1.099915 max_load_reduction=113051 max_load_reduction_pct=0.862511 moved_to_echo=67210 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1664, 0, 0, 0, 0, 32768, 128, 32768, 0, 31360, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 0, 32768, 32768, 128, 128, 128, 128, 128, 128, 128, 128, 0, 32768, 20480, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32640, 32640, 128, 128, 128, 128, 32768, 128, 128, 0, 32768, 128, 128, 128, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 128, 11904, 0, 128, 128, 0, 128, 0, 384, 128, 32768, 32768, 20608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 1408, 32768, 0, 1152, 0, 0, 32768, 30592, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 512, 0, 0, 128, 32640, 128, 128, 0, 32768, 32384, 128, 2304, 16768, 10752, 2304, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[1152, 0, 128, 256, 0, 31616, 32768, 32768, 128, 0, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 16128, 0, 128, 32512, 128, 128, 0, 32640, 17152, 32768, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2630, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 4096, 0, 0, 1116, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 515, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3932, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2833, 0, 4096, 0, 0, 4083, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 6, 0, 1853, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 930, 0, 4, 0, 4096, 0, 0, 4094, 1, 0, 10, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2914, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4095, 0, 0, 0, 0, 1, 0, 3761, 0, 0, 0, 0, 4096, 4092, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4078, 0, 0, 21, 4069, 4084, 0, 4083, 0, 3906, 14, 0, 4081, 11, 331, 0, 4067, 0, 5, 4004, 0, 1, 0, 12, 13, 770, 4082, 1748, 14, 4080, 3957, 0, 0, 671, 16, 7, 13, 5, 0, 11, 12, 4085, 140, 4083, 0, 11, 8, 0, 1239, 0, 0, 11, 4076, 13, 0, 4086, 0, 0, 0, 4085, 7, 12, 12, 0, 3485, 17, 3941, 16, 14, 23, 4070, 0, 13, 0, 3702, 11, 10, 0, 4085, 11, 250, 7, 2, 10, 19, 99, 8, 3985, 185, 4082, 10, 0, 0, 17, 0, 9, 3788, 0, 10, 230, 0, 3978, 4083, 0, 0, 116, 4085, 8, 350, 0, 5, 0, 9, 11, 0, 13, 27, 0, 3791, 0, 0, 4083, 4066, 4063, 241, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2879, 0, 4096, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 17, 0, 1899, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 795, 0, 6, 0, 4096, 0, 0, 4096, 1, 0, 24, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 3066, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 3657, 0, 0, 0, 0, 4096, 4089, 0, 0, 11, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2658, 0, 4096, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 2, 0, 1856, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 562, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3566, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 3673, 0, 0, 3, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2191, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 668, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1521, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3879, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 4, 0, 4096, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2603, 0, 4096, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 8, 0, 964, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 4096, 0, 0, 0, 0, 784, 0, 4, 0, 4096, 0, 0, 4096, 0, 0, 16, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 3945, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 0, 4096, 4091, 0, 0, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 4094, 4095, 0, 4095, 0, 4069, 1, 0, 4095, 1, 1492, 0, 4093, 1, 1, 4046, 0, 1, 0, 1, 1, 209, 4095, 325, 1, 4095, 4083, 0, 0, 495, 1, 1, 1, 0, 0, 1, 1, 4095, 12, 4095, 0, 1, 1, 0, 1909, 0, 0, 1, 4094, 1, 0, 4095, 0, 0, 0, 4095, 0, 1, 1, 0, 4022, 2, 4081, 3, 1, 4, 4094, 0, 1, 0, 3900, 1, 1, 0, 4095, 1, 37, 0, 1, 1, 2, 9, 1, 4094, 120, 4095, 1, 0, 0, 0, 0, 1, 3873, 1, 1, 57, 0, 4092, 4095, 0, 0, 24, 4095, 1, 28, 0, 1, 0, 1, 1, 0, 1, 0, 0, 4058, 0, 0, 4094, 4078, 4092, 20, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1922, 0, 2833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4083, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3761, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 2804, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 930, 0, 4, 0, 4096, 0, 0, 4094, 1, 0, 10, 4096, 0, 0, 0, 0, 2174, 710, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 582, 0, 0, 0, 2914, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4095, 0, 0, 0, 0, 1, 0, 4096, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 0, 0, 0, 0, 4096, 4092, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17043, 16043, 17022, 16286, 15778, 16307, 16195, 16398] avg=16384.0000 max=17043 max_rank=0 min=15778 min_rank=4 imbalance_ratio=1.040222 max_load_reduction=114029 max_load_reduction_pct=0.869972 moved_to_echo=64678 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1922, 0, 2630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3761, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1116, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2804, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2174, 710, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 582, 0, 0, 0, 3932, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 4096, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16840, 16051, 16287, 15858, 16802, 16309, 16527, 16398] avg=16384.0000 max=16840 max_rank=0 min=15858 min_rank=3 imbalance_ratio=1.027832 max_load_reduction=114232 max_load_reduction_pct=0.871521 moved_to_echo=66036 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4078, 0, 0, 21, 4069, 0, 0, 0, 0, 3906, 14, 0, 1915, 11, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 5, 4004, 0, 1, 0, 12, 13, 770, 0, 1748, 14, 3747, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 671, 16, 7, 13, 5, 0, 11, 12, 4085, 140, 4083, 0, 11, 8, 0, 4082, 2786, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1239, 0, 0, 11, 4076, 13, 0, 4086, 0, 0, 0, 4085, 7, 12, 12, 0, 2166, 706, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3485, 17, 3941, 16, 14, 23, 578, 0, 13, 0, 3702, 11, 10, 0, 4085, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 7, 2, 10, 19, 99, 8, 3985, 185, 4082, 10, 0, 0, 17, 0, 9, 4084, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3788, 0, 10, 230, 0, 3978, 4083, 0, 0, 116, 4085, 8, 350, 0, 5, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 11, 0, 13, 27, 0, 3791, 0, 0, 4083, 4066, 4063, 241, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14345, 18338, 16009, 16583, 15906, 16771, 16802, 16318] avg=16384.0000 max=18338 max_rank=1 min=14345 min_rank=0 imbalance_ratio=1.119263 max_load_reduction=112734 max_load_reduction_pct=0.860092 moved_to_echo=65797 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1922, 0, 2658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1856, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2804, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2174, 710, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 582, 0, 0, 0, 3566, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3673, 0, 0, 3, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16868, 16018, 17026, 15905, 16436, 16306, 16115, 16398] avg=16384.0000 max=17026 max_rank=2 min=15905 min_rank=3 imbalance_ratio=1.039185 max_load_reduction=114046 max_load_reduction_pct=0.870102 moved_to_echo=65255 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1920, 0, 2879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3747, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1899, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2810, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 795, 0, 6, 0, 4096, 0, 0, 4096, 1, 0, 24, 4096, 0, 0, 0, 0, 2176, 714, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 572, 0, 0, 0, 3066, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3657, 0, 0, 0, 0, 4096, 4089, 0, 0, 11, 4092, 0, 0, 0, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17087, 16020, 17076, 16178, 15908, 16299, 16099, 16405] avg=16384.0000 max=17087 max_rank=0 min=15908 min_rank=4 imbalance_ratio=1.042908 max_load_reduction=113985 max_load_reduction_pct=0.869637 moved_to_echo=64711 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 4094, 0, 0, 0, 0, 4069, 1, 0, 1922, 1, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 1, 4046, 0, 1, 0, 1, 1, 209, 0, 325, 1, 3760, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 1, 1, 1, 0, 0, 1, 1, 4095, 12, 4095, 0, 1, 1, 0, 4095, 2803, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1909, 0, 0, 1, 4094, 1, 0, 4095, 0, 0, 0, 4095, 0, 1, 1, 0, 2173, 710, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 2, 4081, 3, 1, 4, 581, 0, 1, 0, 3900, 1, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 1, 1, 2, 9, 1, 4094, 120, 4095, 1, 0, 0, 0, 0, 1, 4095, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3873, 1, 1, 57, 0, 4092, 4095, 0, 0, 24, 4095, 1, 28, 0, 1, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 4058, 0, 0, 4094, 4078, 4092, 20, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15675, 16522, 15680, 17251, 16693, 16474, 16418, 16359] avg=16384.0000 max=17251 max_rank=3 min=15675 min_rank=0 imbalance_ratio=1.052917 max_load_reduction=113821 max_load_reduction_pct=0.868385 moved_to_echo=65944 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1922, 0, 2191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2804, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2174, 710, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 582, 0, 0, 0, 3879, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 4, 0, 4096, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16401, 16043, 15838, 16864, 16749, 16306, 16473, 16398] avg=16384.0000 max=16864 max_rank=3 min=15838 min_rank=2 imbalance_ratio=1.029297 max_load_reduction=114208 max_load_reduction_pct=0.871338 moved_to_echo=65926 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1922, 0, 2603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 964, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 4096, 0, 0, 0, 0, 4096, 2804, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 784, 0, 4, 0, 4096, 0, 0, 4096, 0, 0, 16, 4096, 0, 0, 0, 0, 2174, 710, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 582, 0, 0, 0, 3945, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 0, 4096, 4091, 0, 0, 3, 4094, 0, 0, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16813, 16037, 16134, 16147, 16800, 16300, 16443, 16398] avg=16384.0000 max=16813 max_rank=0 min=16037 min_rank=1 imbalance_ratio=1.026184 max_load_reduction=114259 max_load_reduction_pct=0.871727 moved_to_echo=65941 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 0, 0, 128, 32768, 0, 0, 0, 0, 32640, 128, 0, 15488, 128, 17664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 32640, 0, 128, 0, 128, 128, 1024, 0, 2176, 128, 30080, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 9600, 128, 128, 128, 128, 0, 128, 128, 32768, 256, 32768, 0, 128, 128, 0, 32768, 22528, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32128, 128, 32640, 128, 128, 128, 4736, 0, 128, 0, 28928, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[30976, 128, 128, 384, 0, 32768, 32768, 0, 0, 256, 32768, 128, 384, 0, 128, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[8320, 0, 128, 128, 32768, 128, 0, 32768, 128, 0, 128, 32768, 128, 128, 128, 0, 17408, 5760, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[384, 128, 128, 128, 128, 128, 128, 32768, 384, 32768, 128, 0, 0, 128, 128, 128, 32768, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 0, 32512, 0, 0, 32768, 32768, 32768, 384, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 77, 0, 4096, 0, 0, 0, 0, 312, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 1, 4091, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4094, 1, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 4096, 0, 0, 4095, 4093, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 2, 0, 0, 6, 0, 4095, 0, 0, 0, 0, 80, 3656, 0, 0, 0, 0, 1, 0, 4067, 2, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[3, 14, 4061, 13, 0, 14, 0, 0, 4072, 11, 11, 12, 34, 11, 376, 21, 0, 13, 4084, 4, 0, 0, 0, 12, 0, 0, 0, 12, 13, 4083, 4013, 9, 12, 18, 0, 0, 4074, 0, 4085, 7, 0, 3904, 33, 1, 4080, 12, 0, 35, 0, 25, 9, 0, 5, 4042, 5, 4082, 4074, 0, 11, 0, 4080, 4084, 28, 0, 2002, 0, 0, 3768, 0, 12, 0, 13, 10, 4052, 12, 5, 13, 56, 16, 4084, 9, 34, 4084, 4084, 4078, 7, 0, 4083, 4081, 4046, 0, 0, 0, 0, 0, 4082, 3993, 9, 1, 4032, 0, 0, 130, 0, 2, 4082, 29, 0, 0, 8, 2354, 4049, 0, 12, 0, 0, 0, 15, 3852, 0, 18, 12, 4053, 89, 4053, 0, 4, 42] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 171, 0, 4096, 0, 0, 0, 0, 387, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 814, 3067, 0, 0, 0, 0, 0, 0, 3756, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 131, 0, 4096, 0, 0, 0, 0, 656, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 4096, 4, 4079, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4090, 4096, 0, 0, 4071, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4092, 4088, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 7, 0, 0, 15, 0, 4096, 0, 0, 0, 0, 46, 3640, 0, 0, 0, 0, 4, 0, 3749, 3, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 64, 0, 4096, 0, 0, 0, 0, 531, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4087, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 281, 3980, 0, 0, 0, 0, 0, 0, 3347, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 72, 0, 4096, 0, 0, 0, 0, 406, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 422, 3264, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 226, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 0, 4083, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4089, 4096, 0, 0, 4057, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4091, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 6, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 136, 3941, 0, 0, 0, 0, 0, 0, 3922, 2, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 2, 4093, 1, 0, 0, 0, 0, 4094, 1, 1, 0, 3, 1, 63, 2, 7, 1, 4095, 1, 0, 0, 1, 23, 0, 0, 3, 1, 1, 4094, 4083, 1, 1, 2, 0, 0, 4094, 0, 4095, 0, 0, 4077, 3, 0, 4094, 1, 0, 2, 0, 1, 0, 0, 0, 4090, 1, 4094, 4094, 0, 1, 0, 4094, 4095, 17, 0, 3679, 0, 0, 4048, 0, 1, 0, 1, 1, 4091, 1, 2, 1, 5, 1, 4095, 0, 3, 4095, 4095, 4094, 1, 0, 4095, 4095, 4091, 0, 0, 0, 0, 0, 4095, 4078, 1, 0, 4088, 0, 0, 8, 0, 1, 4095, 2, 0, 0, 1, 608, 3947, 0, 1, 0, 0, 0, 1, 4023, 0, 2, 1, 4092, 12, 4092, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 4096, 0, 0, 0, 0, 312, 0, 0, 0, 0, 0, 4096, 4096, 0, 3719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4091, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 190, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4093, 4096, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 6, 0, 3901, 0, 0, 0, 0, 80, 3656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4067, 2, 0, 0, 4096, 0, 4096, 0, 0, 0, 4092, 176, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16320, 16396, 16403, 16389, 16741, 16391, 15837, 16595] avg=16384.0000 max=16741 max_rank=4 min=15837 min_rank=6 imbalance_ratio=1.021790 max_load_reduction=114331 max_load_reduction_pct=0.872276 moved_to_echo=65564 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 14, 4061, 13, 0, 14, 0, 0, 4072, 11, 11, 12, 34, 11, 376, 21, 4084, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4084, 4, 0, 0, 0, 12, 0, 0, 0, 12, 13, 4083, 4013, 9, 3707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 18, 0, 0, 4074, 0, 4085, 7, 0, 3904, 33, 1, 4080, 12, 0, 35, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 9, 0, 5, 4042, 5, 4082, 4074, 0, 11, 0, 4080, 4, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2002, 0, 0, 3768, 0, 12, 0, 13, 10, 4052, 12, 5, 13, 56, 16, 4084, 189, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 34, 0, 0, 4078, 7, 0, 4083, 4081, 4046, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 9, 1, 4032, 0, 0, 130, 0, 2, 3889, 29, 0, 0, 8, 2354, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 15, 3852, 0, 18, 12, 4053, 89, 4053, 0, 4, 42, 4080, 175, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16757, 15950, 16284, 16365, 14402, 16349, 18496, 16469] avg=16384.0000 max=18496 max_rank=6 min=14402 min_rank=4 imbalance_ratio=1.128906 max_load_reduction=112576 max_load_reduction_pct=0.858887 moved_to_echo=65716 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 4096, 0, 0, 0, 0, 387, 0, 0, 0, 0, 0, 4096, 4096, 0, 3719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 190, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 2, 0, 3902, 0, 0, 0, 0, 814, 3067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3756, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4092, 176, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16320, 16565, 16407, 16388, 16742, 16395, 15975, 16280] avg=16384.0000 max=16742 max_rank=4 min=15975 min_rank=6 imbalance_ratio=1.021851 max_load_reduction=114330 max_load_reduction_pct=0.872269 moved_to_echo=65392 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 4096, 0, 0, 0, 0, 531, 0, 0, 0, 0, 0, 4096, 4096, 0, 3719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 190, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 8, 0, 3902, 0, 0, 0, 0, 281, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3347, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4092, 176, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16319, 16602, 16397, 16388, 16736, 16395, 16363, 15872] avg=16384.0000 max=16736 max_rank=4 min=15872 min_rank=7 imbalance_ratio=1.021484 max_load_reduction=114336 max_load_reduction_pct=0.872314 moved_to_echo=65366 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4093, 1, 0, 0, 0, 0, 4094, 1, 1, 0, 3, 1, 63, 2, 4095, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 4095, 1, 0, 0, 1, 23, 0, 0, 3, 1, 1, 4094, 4083, 1, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 4094, 0, 4095, 0, 0, 4077, 3, 0, 4094, 1, 0, 2, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4090, 1, 4094, 4094, 0, 1, 0, 4094, 4, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3679, 0, 0, 4048, 0, 1, 0, 1, 1, 4091, 1, 2, 1, 5, 1, 4095, 190, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4094, 1, 0, 4095, 4095, 4091, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 1, 0, 4088, 0, 0, 8, 0, 1, 3901, 2, 0, 0, 1, 608, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4023, 0, 2, 1, 4092, 12, 4092, 0, 1, 0, 4091, 176, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16029, 16392, 16396, 16287, 16390, 16635, 16557] avg=16384.0000 max=16635 max_rank=6 min=16029 min_rank=1 imbalance_ratio=1.015320 max_load_reduction=114437 max_load_reduction_pct=0.873085 moved_to_echo=65869 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 4096, 0, 0, 0, 0, 656, 0, 0, 0, 0, 0, 4093, 4096, 0, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4079, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 195, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 7, 0, 0, 15, 0, 3899, 0, 0, 0, 0, 46, 3640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3749, 3, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 176, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16326, 16797, 16395, 16378, 16721, 16372, 15799, 16284] avg=16384.0000 max=16797 max_rank=1 min=15799 min_rank=6 imbalance_ratio=1.025208 max_load_reduction=114275 max_load_reduction_pct=0.871849 moved_to_echo=65176 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 4096, 0, 0, 0, 0, 406, 0, 0, 0, 0, 0, 4096, 4096, 0, 3719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 190, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 3902, 0, 0, 0, 0, 422, 3264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4092, 176, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16319, 16485, 16400, 16388, 16743, 16395, 15781, 16561] avg=16384.0000 max=16743 max_rank=4 min=15781 min_rank=6 imbalance_ratio=1.021912 max_load_reduction=114329 max_load_reduction_pct=0.872261 moved_to_echo=65480 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 226, 0, 0, 0, 0, 0, 4096, 4095, 0, 3719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4089, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 190, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4091, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 6, 0, 0, 9, 0, 3902, 0, 0, 0, 0, 136, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3922, 2, 0, 0, 4096, 0, 4096, 0, 0, 0, 4093, 181, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16325, 16248, 16394, 16380, 16700, 16385, 16186, 16454] avg=16384.0000 max=16700 max_rank=4 min=16186 min_rank=6 imbalance_ratio=1.019287 max_load_reduction=114372 max_load_reduction_pct=0.872589 moved_to_echo=65725 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 32768, 128, 0, 128, 128, 0, 32768, 128, 128, 128, 128, 128, 512, 128, 32768, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[30208, 128, 0, 32512, 0, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 128, 32768, 1536, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 32768, 128, 0, 256, 128, 128, 31232, 128, 0, 0, 128, 4864, 29568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[640, 128, 32768, 128, 0, 0, 128, 2560, 0, 0, 128, 128, 128, 32768, 32768, 128, 29824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 32768, 128, 0, 32640, 128, 128, 32768, 128, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 0, 0, 128, 128, 30848, 128, 128, 128, 32768, 128, 32768, 0, 128, 128, 32768, 1536, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 128, 32768, 128, 32768, 32768, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2940, 4096, 0, 4096, 0, 0, 0, 0, 4095, 4096, 0, 43, 0, 0, 1336, 0, 0, 0, 19, 6, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 18, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 4095, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4044, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 215, 0, 4096, 1, 4096, 0, 4090, 4096, 0, 0, 0, 0, 0, 4083, 0, 4095, 3679, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4053, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 70, 0, 0, 114, 0, 0, 0, 47, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 3988, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 184, 0, 4096, 0, 4096, 0, 4089, 4096, 0, 0, 0, 0, 0, 4046, 0, 4096, 3867, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 4081, 0, 0, 7, 81, 1, 4060, 4076, 0, 0, 519, 4082, 3934, 4053, 0, 4073, 1, 2, 6, 0, 4086, 4085, 2, 99, 13, 11, 1042, 0, 0, 14, 0, 19, 0, 14, 7, 3, 11, 4073, 0, 10, 12, 10, 23, 2, 74, 0, 24, 0, 24, 10, 22, 3605, 0, 15, 4082, 12, 0, 27, 128, 6, 0, 4082, 13, 18, 31, 11, 11, 10, 0, 4085, 5, 13, 3960, 4051, 10, 4086, 3974, 4062, 4059, 123, 26, 0, 3988, 4082, 0, 8, 0, 4062, 0, 0, 0, 0, 17, 0, 12, 0, 20, 0, 4084, 4084, 0, 13, 4083, 0, 4078, 0, 4083, 4078, 0, 12, 0, 13, 135, 4084, 0, 4081, 2844, 11, 0, 0, 6, 0, 11, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3875, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 42, 0, 0, 318, 0, 0, 0, 41, 10, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 16, 4088, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 3934, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 206, 0, 4096, 5, 4096, 0, 4079, 4096, 0, 0, 0, 0, 0, 4061, 0, 4089, 3900, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4096, 2485, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 6, 0, 0, 2034, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 24, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4077, 0, 4096, 3656, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3497, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 45, 0, 0, 1835, 0, 0, 0, 21, 12, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 136, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 14, 4089, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4044, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 145, 0, 4096, 0, 4096, 0, 4083, 4096, 0, 0, 0, 0, 0, 4077, 0, 4095, 2571, 0, 0, 0, 0, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2663, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 13, 0, 0, 1536, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 70, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4063, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 72, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4081, 0, 4096, 3873, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4094, 0, 0, 1, 10, 0, 4092, 4094, 0, 0, 57, 4095, 3439, 4092, 0, 4094, 1, 0, 1, 0, 4095, 4095, 0, 218, 1, 1, 1100, 0, 0, 1, 0, 6, 0, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 2, 1, 11, 0, 2, 0, 2, 1, 1, 4014, 1, 1, 4095, 1, 0, 3, 6, 0, 0, 4095, 1, 1, 3, 1, 1, 1, 0, 4095, 1, 0, 4087, 4092, 1, 4095, 4078, 4093, 4092, 18, 2, 0, 4083, 4095, 1, 1, 0, 4093, 0, 0, 0, 0, 1, 0, 1, 0, 3, 0, 4095, 4095, 0, 1, 4095, 0, 4094, 0, 4095, 4094, 0, 1, 0, 1, 14, 4095, 0, 4095, 3430, 1, 0, 0, 1, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2940, 4096, 0, 4096, 0, 0, 0, 0, 3605, 0, 0, 43, 0, 0, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 6, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 18, 0, 4096, 4064, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 4095, 0, 0, 4096, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4044, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 215, 0, 3995, 1, 4096, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4095, 3679, 0, 0, 0, 0, 0, 0, 0, 4027, 490, 101, 74, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16310, 16116, 16376, 16405, 16399, 16401, 16493, 16572] avg=16384.0000 max=16572 max_rank=7 min=16116 min_rank=1 imbalance_ratio=1.011475 max_load_reduction=114500 max_load_reduction_pct=0.873566 moved_to_echo=65865 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 4081, 0, 0, 7, 81, 1, 4060, 4076, 0, 0, 519, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 4053, 0, 4073, 1, 2, 6, 0, 3597, 0, 2, 99, 13, 11, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 19, 0, 14, 7, 3, 11, 4073, 0, 10, 12, 10, 23, 2, 4085, 4084, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 24, 0, 24, 10, 22, 3605, 0, 15, 4082, 12, 0, 27, 128, 6, 4062, 4054, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 13, 18, 31, 11, 11, 10, 0, 4085, 5, 13, 3960, 4051, 10, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 4059, 123, 26, 0, 3988, 4082, 0, 8, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 12, 0, 20, 0, 0, 0, 0, 13, 3983, 0, 4078, 0, 4083, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 13, 135, 4084, 0, 4081, 2844, 11, 0, 0, 6, 0, 11, 0, 3993, 489, 100, 74, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16845, 16833, 16417, 16179, 16309, 16329, 16284, 15876] avg=16384.0000 max=16845 max_rank=0 min=15876 min_rank=7 imbalance_ratio=1.028137 max_load_reduction=114227 max_load_reduction_pct=0.871483 moved_to_echo=64798 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 4096, 0, 4096, 0, 0, 0, 0, 3606, 0, 0, 70, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 13, 0, 4096, 4064, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 3988, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 3995, 0, 4096, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 4096, 3867, 0, 0, 0, 0, 0, 0, 0, 4027, 490, 101, 74, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16310, 16035, 16406, 16399, 16393, 16345, 16460, 16724] avg=16384.0000 max=16724 max_rank=7 min=16035 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=114348 max_load_reduction_pct=0.872406 moved_to_echo=65922 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2485, 4096, 0, 4096, 0, 0, 0, 0, 3606, 0, 0, 6, 0, 0, 2034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 9, 0, 4096, 4064, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4092, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 3995, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4096, 3656, 0, 0, 0, 0, 0, 0, 0, 4027, 490, 101, 74, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16309, 16323, 16352, 16395, 16393, 16449, 16307, 16544] avg=16384.0000 max=16544 max_rank=7 min=16307 min_rank=6 imbalance_ratio=1.009766 max_load_reduction=114528 max_load_reduction_pct=0.873779 moved_to_echo=65693 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4094, 0, 0, 1, 10, 0, 4092, 4094, 0, 0, 57, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 4092, 0, 4094, 1, 0, 1, 0, 3605, 0, 0, 218, 1, 1, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 2, 1, 4095, 4095, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 2, 0, 2, 1, 1, 4014, 1, 1, 4095, 1, 0, 3, 6, 0, 4093, 4063, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 3, 1, 1, 1, 0, 4095, 1, 0, 4087, 4092, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4092, 18, 2, 0, 4083, 4095, 1, 1, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 3, 0, 0, 0, 0, 1, 3994, 0, 4094, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 14, 4095, 0, 4095, 3430, 1, 0, 0, 1, 0, 1, 0, 4024, 490, 101, 74, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16370, 16552, 16361, 16328, 16388, 16439, 16283, 16351] avg=16384.0000 max=16552 max_rank=1 min=16283 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=114520 max_load_reduction_pct=0.873718 moved_to_echo=65461 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3875, 4096, 0, 4096, 0, 0, 0, 0, 3601, 0, 0, 42, 0, 0, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 10, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 4065, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 16, 4088, 0, 0, 4096, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 3934, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 3993, 5, 4096, 0, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 4089, 3900, 0, 0, 0, 0, 0, 0, 0, 4030, 495, 103, 76, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16308, 16028, 16408, 16388, 16397, 16288, 16475, 16780] avg=16384.0000 max=16780 max_rank=7 min=16028 min_rank=1 imbalance_ratio=1.024170 max_load_reduction=114292 max_load_reduction_pct=0.871979 moved_to_echo=65940 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3497, 4096, 0, 4096, 0, 0, 0, 0, 3606, 0, 0, 45, 0, 0, 1835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 12, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 136, 0, 4096, 4064, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 14, 4089, 0, 0, 4096, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4044, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 3995, 0, 4096, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4095, 2571, 0, 0, 0, 0, 0, 0, 2, 4027, 490, 101, 74, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16310, 17175, 16389, 16522, 16400, 16401, 16415, 15460] avg=16384.0000 max=17175 max_rank=1 min=15460 min_rank=7 imbalance_ratio=1.048279 max_load_reduction=113897 max_load_reduction_pct=0.868965 moved_to_echo=64676 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2663, 4096, 0, 4096, 0, 0, 0, 0, 3606, 0, 0, 13, 0, 0, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 70, 0, 4096, 4064, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4063, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 3995, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 4096, 3873, 0, 0, 0, 0, 0, 0, 0, 4027, 490, 101, 74, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16310, 16010, 16363, 16456, 16393, 16420, 16355, 16765] avg=16384.0000 max=16765 max_rank=7 min=16010 min_rank=1 imbalance_ratio=1.023254 max_load_reduction=114307 max_load_reduction_pct=0.872093 moved_to_echo=65933 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 32768, 0, 0, 128, 128, 128, 32768, 32768, 0, 0, 640, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 32256, 128, 128, 32768, 128, 0, 128, 384, 128, 32768, 32512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 896, 128, 32000, 128, 32768, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 256, 32640, 0, 32768, 27904, 128, 0, 0, 128, 0, 128, 128, 32256, 3968, 896, 640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 256, 128, 128, 128, 128, 128, 128, 32768, 0, 128, 128, 128, 128, 128, 32768, 32768, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 32640, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[27008, 32768, 0, 32768, 128, 128, 128, 0, 28928, 0, 128, 640, 128, 128, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32640, 0, 32768, 256, 128, 0, 32768, 32384, 128, 128, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[3, 0, 1, 0, 0, 0, 0, 0, 0, 301, 0, 4096, 0, 4096, 4095, 0, 0, 0, 4096, 1, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 4090, 4096, 0, 4096, 0, 3148, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 1661, 3168, 0, 0, 4096, 3991, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 0, 0, 0, 0, 920, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 4096, 0, 3597, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 7, 0, 1850, 1796, 0, 0, 4096, 4010, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 13, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 15, 0, 0, 0, 0, 18, 0, 3, 110, 0, 3895, 0, 4077, 4085, 0, 0, 0, 4083, 0, 14, 4, 3740, 4080, 162, 0, 17, 0, 432, 9, 4079, 4083, 0, 4077, 56, 3205, 8, 9, 4077, 11, 0, 0, 14, 16, 0, 1755, 0, 14, 11, 15, 1, 0, 12, 0, 45, 4076, 4033, 23, 0, 23, 3859, 2760, 6, 0, 4051, 2810, 0, 19, 12, 3365, 11, 0, 13, 4000, 4, 13, 0, 0, 17, 7, 4042, 4060, 0, 0, 0, 0, 0, 32, 1, 1, 4066, 4083, 13, 0, 0, 4060, 1, 12, 36, 4079, 2159, 14, 4082, 0, 0, 12, 4074, 13, 17, 14, 0, 27, 17, 4080, 4078, 0, 0, 4084, 10, 0, 10, 351, 10, 24, 11, 4081, 4084, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4050, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 429, 3694, 0, 0, 4096, 4089, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 3340, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1706, 2074, 0, 0, 4096, 4092, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 2, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 4096, 0, 3922, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2239, 1919, 0, 0, 4096, 4021, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 16, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4094, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 3668, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 753, 3448, 0, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4089, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 0, 0, 2, 0, 0, 248, 0, 4076, 0, 4094, 4095, 0, 0, 0, 4095, 0, 1, 1, 4047, 4095, 9, 0, 1, 0, 64, 1, 4095, 4095, 0, 4093, 6, 2870, 1, 1, 4094, 1, 0, 1, 1, 1, 0, 190, 0, 1, 1, 1, 0, 0, 1, 0, 16, 4094, 4092, 3, 0, 2, 1753, 3495, 1, 0, 4094, 3939, 0, 1, 1, 3953, 1, 0, 0, 4086, 1, 1, 0, 0, 1, 1, 4088, 4093, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 4095, 1, 0, 0, 4093, 1, 1, 3, 4094, 3947, 1, 4095, 0, 0, 1, 4094, 1, 2, 1, 0, 2, 2, 4094, 4095, 0, 0, 4095, 0, 0, 1, 65, 1, 2, 0, 4095, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 0, 0, 0, 0, 18, 0, 3, 110, 0, 3895, 0, 4077, 4085, 0, 3730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4, 3740, 4080, 162, 0, 17, 0, 432, 9, 4079, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 56, 3205, 8, 9, 4077, 11, 0, 0, 14, 16, 0, 1755, 0, 14, 4083, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 15, 1, 0, 12, 0, 45, 4076, 4033, 23, 0, 23, 3859, 2760, 6, 0, 3579, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 2810, 0, 19, 12, 3365, 11, 0, 13, 4000, 4, 13, 0, 0, 17, 7, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 4060, 0, 0, 0, 0, 0, 32, 1, 1, 4066, 12, 13, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 36, 4079, 2159, 14, 4082, 0, 0, 12, 4074, 13, 17, 14, 0, 27, 163, 54, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4080, 4078, 0, 0, 4084, 10, 0, 10, 351, 10, 24, 11, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15933, 16589, 17662, 18460, 14651, 16287, 14788, 16702] avg=16384.0000 max=18460 max_rank=3 min=14651 min_rank=4 imbalance_ratio=1.126709 max_load_reduction=112612 max_load_reduction_pct=0.859161 moved_to_echo=62428 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 1, 0, 0, 0, 0, 0, 0, 301, 0, 4096, 0, 4096, 4095, 0, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 4090, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3148, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4096, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 1661, 3168, 0, 0, 3590, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3991, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 164, 54, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16332, 16351, 15785, 16631, 16609, 16402, 16632, 16330] avg=16384.0000 max=16632 max_rank=6 min=15785 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=114440 max_load_reduction_pct=0.873108 moved_to_echo=65973 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 4096, 4096, 0, 3742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4050, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 22, 0, 0, 4096, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 429, 3694, 0, 0, 3590, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 164, 54, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16035, 16353, 16697, 15922, 16707, 16396, 16632, 16330] avg=16384.0000 max=16707 max_rank=4 min=15922 min_rank=3 imbalance_ratio=1.019714 max_load_reduction=114365 max_load_reduction_pct=0.872536 moved_to_echo=66065 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 0, 0, 0, 0, 920, 0, 4096, 0, 4096, 4096, 0, 3742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4089, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3597, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 99, 0, 0, 4096, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 7, 0, 1850, 1796, 0, 0, 3593, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4010, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 13, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 166, 60, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16956, 16346, 16320, 15455, 16635, 16397, 16639, 16324] avg=16384.0000 max=16956 max_rank=0 min=15455 min_rank=3 imbalance_ratio=1.034912 max_load_reduction=114116 max_load_reduction_pct=0.870636 moved_to_echo=65995 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 0, 0, 2, 0, 0, 248, 0, 4076, 0, 4094, 4095, 0, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4047, 4095, 9, 0, 1, 0, 64, 1, 4095, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 6, 2870, 1, 1, 4094, 1, 0, 1, 1, 1, 0, 190, 0, 1, 4095, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 16, 4094, 4092, 3, 0, 2, 1753, 3495, 1, 0, 3589, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3939, 0, 1, 1, 3953, 1, 0, 0, 4086, 1, 1, 0, 0, 1, 1, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4093, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 12, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 4094, 3947, 1, 4095, 0, 0, 1, 4094, 1, 2, 1, 0, 2, 164, 54, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 4095, 0, 0, 4095, 0, 0, 1, 65, 1, 2, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16257, 16378, 15691, 17067, 16409, 16382, 16492, 16396] avg=16384.0000 max=17067 max_rank=3 min=15691 min_rank=2 imbalance_ratio=1.041687 max_load_reduction=114005 max_load_reduction_pct=0.869789 moved_to_echo=65679 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 0, 4096, 0, 4096, 4096, 0, 3741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3340, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4096, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1706, 2074, 0, 0, 3590, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 164, 54, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17092, 16353, 15976, 15579, 16710, 16400, 16632, 16330] avg=16384.0000 max=17092 max_rank=0 min=15579 min_rank=3 imbalance_ratio=1.043213 max_load_reduction=113980 max_load_reduction_pct=0.869598 moved_to_echo=66072 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 4096, 0, 4096, 4096, 0, 3742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4090, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3922, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 82, 0, 0, 4096, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2239, 1919, 0, 0, 3590, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4021, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 16, 0, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 164, 54, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16125, 16349, 16629, 15957, 16639, 16412, 16631, 16330] avg=16384.0000 max=16639 max_rank=4 min=15957 min_rank=3 imbalance_ratio=1.015564 max_load_reduction=114433 max_load_reduction_pct=0.873055 moved_to_echo=66012 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 0, 4096, 0, 4096, 4096, 0, 3741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3668, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 19, 0, 0, 4096, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 753, 3448, 0, 0, 3590, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 164, 54, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16342, 16353, 16312, 16001, 16712, 16396, 16626, 16330] avg=16384.0000 max=16712 max_rank=4 min=16001 min_rank=3 imbalance_ratio=1.020020 max_load_reduction=114360 max_load_reduction_pct=0.872498 moved_to_echo=66064 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 32768, 128, 0, 128, 0, 0, 128, 128, 128, 32768, 128, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 128, 32384, 32768, 256, 0, 128, 0, 512, 128, 32768, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32768, 128, 27904, 128, 128, 32768, 128, 0, 128, 128, 128, 0, 2304, 0, 128, 32768, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 32768, 30720, 128, 32768, 0, 0, 128, 32768, 128, 128, 128, 0, 128, 1408, 512, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 31104, 0, 128, 128, 32000, 128, 0, 128, 32768, 128, 128, 0, 0, 128, 128, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 0, 0, 128, 0, 128, 3072, 0, 32640, 0, 32768, 32768, 0, 29952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 32768, 0, 0, 32768, 128, 0, 128, 512, 128, 128, 128, 32384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 32768, 32768, 128, 128, 128, 14336, 22400, 128, 0, 28800, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True + [2026-08-30 05:05:01] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 738.9 | throughput per GPU (TFLOP/s/GPU): 451.2 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.308530E+00 | loss scale: 1.0 | grad norm: 5.511 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1328, 1985, 2099, 2, 263, 0, 2189, 224, 287, 2479, 360, 608, 1938, 2398, 0, 1804, 1391, 1687, 1567, 1768, 26, 576, 154, 2052, 170, 17, 3536, 2119, 1312, 41, 1724, 1388, 937, 4, 1227, 503, 161, 1089, 1509, 142, 327, 6, 2098, 3109, 0, 2085, 950, 797, 1081, 867, 484, 448, 3, 12, 14, 2760, 0, 120, 16, 606, 1645, 2999, 1423, 169, 1076, 16, 0, 1730, 2577, 907, 471, 912, 1621, 2066, 1915, 90, 1416, 6, 637, 1631, 657, 320, 3, 22, 1842, 530, 694, 9, 2195, 71, 1290, 985, 419, 1813, 395, 0, 1431, 1305, 7, 2199, 5, 624, 1295, 783, 212, 1407, 69, 1668, 324, 2124, 2423, 1947, 2361, 2190, 379, 1348, 2334, 764, 1879, 1761, 1230, 2181, 1284, 1115, 535, 1, 481, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1842, 2182, 1727, 0, 305, 0, 2497, 217, 669, 2434, 626, 594, 1848, 2593, 0, 1259, 792, 1541, 1943, 1393, 24, 1060, 143, 1784, 57, 42, 3501, 2028, 1362, 68, 1746, 1541, 702, 0, 1429, 256, 138, 1604, 1431, 196, 48, 1, 1925, 2777, 0, 2154, 841, 703, 1154, 1246, 470, 312, 5, 7, 2, 2123, 0, 108, 8, 283, 1846, 3002, 1022, 110, 748, 34, 0, 1373, 2414, 1041, 431, 1006, 1273, 2048, 2243, 95, 1823, 2, 584, 1972, 638, 274, 2, 12, 1194, 443, 716, 28, 2707, 17, 1857, 1075, 233, 2508, 277, 0, 1520, 1081, 5, 1745, 1, 592, 1976, 1038, 126, 1637, 27, 1331, 553, 2042, 2424, 1996, 2360, 2019, 275, 862, 2811, 761, 2418, 1733, 1146, 2285, 1378, 806, 833, 5, 494, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1965, 2132, 2001, 3, 211, 0, 1953, 544, 1081, 2854, 516, 307, 1776, 2455, 0, 866, 723, 1529, 1089, 1685, 12, 803, 192, 2105, 113, 22, 3482, 2059, 1576, 82, 2298, 1371, 404, 8, 1527, 903, 362, 1049, 1250, 200, 280, 12, 2297, 2741, 13, 2197, 748, 452, 1415, 772, 580, 615, 11, 9, 6, 2371, 0, 459, 91, 645, 2097, 2195, 1396, 254, 989, 30, 0, 2021, 1944, 945, 611, 1051, 1215, 1391, 2157, 126, 1653, 11, 679, 1761, 607, 375, 7, 3, 1336, 387, 1622, 75, 2296, 44, 873, 480, 219, 1967, 373, 2, 1871, 1154, 36, 1883, 11, 944, 1785, 786, 261, 1339, 57, 1421, 707, 2399, 2308, 1137, 1797, 2257, 200, 1695, 2264, 955, 3019, 1832, 1215, 2191, 1046, 678, 818, 8, 543, 41] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1729, 2323, 1399, 0, 365, 0, 2712, 662, 1015, 2985, 506, 647, 1702, 2509, 0, 1228, 1163, 1208, 1238, 1586, 33, 589, 165, 1763, 146, 41, 3329, 1852, 1646, 42, 1903, 1103, 595, 1, 1167, 673, 184, 1779, 1403, 143, 119, 15, 2340, 2895, 6, 2456, 793, 649, 1244, 900, 652, 376, 9, 20, 45, 2271, 2, 250, 41, 417, 1562, 2733, 1433, 130, 981, 14, 0, 1775, 2356, 1290, 545, 832, 1468, 1774, 2193, 124, 1698, 14, 748, 2057, 685, 381, 1, 19, 1960, 542, 1277, 17, 2413, 35, 1446, 871, 160, 1660, 453, 3, 1756, 821, 8, 1694, 14, 610, 1028, 866, 147, 1350, 101, 1018, 366, 2477, 2678, 1338, 1643, 2114, 476, 1661, 1960, 955, 2853, 1696, 1089, 2717, 1184, 545, 514, 13, 647, 49] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1937, 2112, 1766, 0, 212, 1, 1670, 492, 883, 2253, 555, 253, 1265, 2851, 1, 1689, 1019, 1712, 1561, 1557, 22, 604, 169, 1967, 153, 23, 3551, 1840, 1097, 38, 2129, 1259, 744, 2, 1030, 242, 462, 1666, 2081, 142, 141, 4, 2727, 2935, 1, 2406, 622, 1114, 1155, 680, 453, 697, 3, 14, 20, 2332, 0, 174, 23, 271, 1901, 3052, 1450, 141, 1237, 16, 1, 1539, 2513, 1483, 638, 720, 1680, 2357, 2300, 228, 1407, 3, 741, 2067, 536, 470, 10, 11, 1814, 358, 1145, 25, 2756, 30, 923, 705, 119, 1955, 318, 1, 1237, 913, 4, 1910, 18, 436, 1792, 688, 138, 1638, 52, 1752, 527, 1923, 2360, 1577, 1861, 1992, 275, 1228, 2661, 1035, 1757, 1665, 1567, 1705, 1225, 565, 661, 13, 459, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1599, 2229, 1930, 2, 256, 1, 2390, 279, 752, 2900, 627, 251, 1128, 3149, 0, 1400, 1109, 1807, 1564, 1432, 8, 720, 165, 1801, 92, 19, 3597, 2170, 1616, 43, 1795, 1054, 438, 2, 1371, 538, 125, 1396, 1260, 157, 186, 1, 2042, 3047, 1, 2507, 837, 775, 1249, 860, 744, 228, 7, 28, 23, 1749, 1, 688, 1, 342, 1619, 2932, 1390, 87, 1069, 31, 0, 1491, 1798, 1049, 462, 1025, 1166, 2156, 2357, 134, 1157, 23, 1221, 2065, 802, 529, 7, 14, 1312, 572, 995, 24, 1963, 111, 1151, 977, 153, 2515, 455, 2, 1408, 1100, 9, 2196, 2, 692, 1809, 837, 252, 1345, 78, 1055, 279, 2143, 2667, 1839, 2417, 2441, 332, 1123, 2071, 712, 2824, 1521, 1156, 2393, 1060, 726, 834, 5, 443, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1822, 1805, 1681, 1, 251, 1, 2523, 222, 488, 3174, 1332, 468, 1335, 3060, 0, 1658, 705, 2118, 1461, 1519, 11, 848, 163, 1726, 36, 70, 3747, 2551, 1092, 39, 1370, 882, 848, 2, 1271, 608, 143, 1809, 1436, 215, 114, 1, 2081, 2880, 0, 2399, 704, 1016, 1542, 1041, 491, 413, 1, 6, 12, 2436, 0, 281, 31, 388, 1242, 3283, 1346, 69, 764, 15, 1, 1584, 2660, 772, 319, 710, 1253, 1878, 1894, 131, 1491, 6, 1036, 2455, 797, 486, 0, 18, 1451, 548, 655, 37, 2279, 60, 1542, 1112, 140, 2715, 401, 0, 1104, 1075, 8, 1418, 2, 529, 1440, 821, 138, 1617, 44, 1128, 158, 2332, 2755, 2073, 2436, 1787, 388, 1555, 2244, 871, 2152, 939, 826, 2331, 1389, 493, 1022, 4, 507, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1936, 2530, 2048, 0, 54, 0, 2585, 399, 894, 3059, 931, 338, 1792, 3235, 0, 1449, 833, 1652, 1362, 1860, 52, 813, 178, 1836, 103, 18, 3538, 2213, 1963, 84, 2133, 818, 527, 3, 930, 549, 108, 1298, 1101, 328, 58, 2, 2387, 3194, 0, 2404, 616, 1066, 724, 828, 677, 204, 0, 21, 14, 1877, 0, 320, 22, 328, 1938, 3228, 1470, 64, 964, 6, 1, 1860, 2015, 1268, 570, 991, 1249, 1767, 2170, 71, 1516, 11, 968, 2043, 620, 549, 5, 41, 1902, 399, 1149, 25, 2052, 133, 1431, 580, 168, 2041, 376, 1, 1584, 1154, 8, 1416, 5, 471, 1163, 668, 171, 1366, 19, 1344, 213, 1812, 2843, 1082, 2013, 2310, 211, 1284, 2079, 998, 2557, 1661, 979, 2510, 1012, 812, 793, 6, 586, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1328, 1985, 2099, 2, 263, 0, 2189, 224, 287, 2479, 360, 608, 1938, 19, 0, 1804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1391, 1687, 1567, 1768, 26, 576, 154, 2052, 170, 17, 1038, 2119, 1312, 41, 1724, 1388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 937, 4, 1227, 503, 161, 1089, 1509, 142, 327, 6, 2098, 3109, 0, 2085, 950, 797, 695, 676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 867, 484, 448, 3, 12, 14, 2760, 0, 120, 16, 606, 1645, 2999, 1423, 169, 1879, 1147, 400, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 16, 0, 1730, 1430, 907, 471, 912, 1621, 2066, 1915, 90, 1416, 6, 637, 1631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 320, 3, 22, 1842, 530, 694, 9, 2195, 71, 1290, 985, 419, 1813, 395, 0, 2379, 2098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1431, 1305, 7, 2199, 5, 624, 1295, 783, 212, 1407, 69, 1668, 324, 2124, 1728, 1947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2361, 2190, 379, 1348, 1353, 764, 0, 1761, 1230, 2181, 1284, 1115, 535, 1, 481, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15585, 17030, 16315, 16378, 15924, 15722, 17128, 16990] avg=16384.0000 max=17128 max_rank=6 min=15585 min_rank=0 imbalance_ratio=1.045410 max_load_reduction=113944 max_load_reduction_pct=0.869324 moved_to_echo=65764 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1842, 2182, 1727, 0, 305, 0, 2497, 217, 669, 2434, 626, 594, 1848, 26, 0, 1259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 792, 1541, 1943, 1393, 24, 1060, 143, 1784, 57, 42, 1037, 2028, 1362, 68, 1746, 1541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 702, 0, 1429, 256, 138, 1604, 1431, 196, 48, 1, 1925, 2777, 0, 2154, 841, 703, 693, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1154, 1246, 470, 312, 5, 7, 2, 2123, 0, 108, 8, 283, 1846, 3002, 1022, 110, 2418, 1071, 391, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 748, 34, 0, 1373, 1343, 1041, 431, 1006, 1273, 2048, 2243, 95, 1823, 2, 584, 1972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 638, 274, 2, 12, 1194, 443, 716, 28, 2707, 17, 1857, 1075, 233, 2508, 277, 0, 2567, 2073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1520, 1081, 5, 1745, 1, 592, 1976, 1038, 126, 1637, 27, 1331, 553, 2042, 1731, 1996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2360, 2019, 275, 862, 1639, 761, 0, 1733, 1146, 2285, 1378, 806, 833, 5, 494, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16226, 16561, 15708, 15940, 16016, 16621, 17401, 16599] avg=16384.0000 max=17401 max_rank=6 min=15708 min_rank=2 imbalance_ratio=1.062073 max_load_reduction=113671 max_load_reduction_pct=0.867241 moved_to_echo=66637 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1729, 2323, 1399, 0, 365, 0, 2712, 662, 1015, 2985, 506, 647, 1702, 25, 0, 1228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1163, 1208, 1238, 1586, 33, 589, 165, 1763, 146, 41, 986, 1852, 1646, 42, 1903, 1103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 595, 1, 1167, 673, 184, 1779, 1403, 143, 119, 15, 2340, 2895, 6, 2456, 793, 649, 766, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1244, 900, 652, 376, 9, 20, 45, 2271, 2, 250, 41, 417, 1562, 2733, 1433, 130, 2853, 1045, 372, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 981, 14, 0, 1775, 1311, 1290, 545, 832, 1468, 1774, 2193, 124, 1698, 14, 748, 2057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 685, 381, 1, 19, 1960, 542, 1277, 17, 2413, 35, 1446, 871, 160, 1660, 453, 3, 2484, 1971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1756, 821, 8, 1694, 14, 610, 1028, 866, 147, 1350, 101, 1018, 366, 2477, 1912, 1338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1643, 2114, 476, 1661, 1143, 955, 0, 1696, 1089, 2717, 1184, 545, 514, 13, 647, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17298, 15464, 16548, 16608, 16824, 16378, 15506, 16446] avg=16384.0000 max=17298 max_rank=0 min=15464 min_rank=1 imbalance_ratio=1.055786 max_load_reduction=113774 max_load_reduction_pct=0.868027 moved_to_echo=65154 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1937, 2112, 1766, 0, 212, 1, 1670, 492, 883, 2253, 555, 253, 1265, 28, 1, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1019, 1712, 1561, 1557, 22, 604, 169, 1967, 153, 23, 1051, 1840, 1097, 38, 2129, 1259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 744, 2, 1030, 242, 462, 1666, 2081, 142, 141, 4, 2727, 2935, 1, 2406, 622, 1114, 675, 766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 680, 453, 697, 3, 14, 20, 2332, 0, 174, 23, 271, 1901, 3052, 1450, 141, 1757, 1115, 397, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1237, 16, 1, 1539, 1398, 1483, 638, 720, 1680, 2357, 2300, 228, 1407, 3, 741, 2067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 470, 10, 11, 1814, 358, 1145, 25, 2756, 30, 923, 705, 119, 1955, 318, 1, 2823, 2103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1237, 913, 4, 1910, 18, 436, 1792, 688, 138, 1638, 52, 1752, 527, 1923, 1685, 1577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1861, 1992, 275, 1228, 1552, 1035, 0, 1665, 1567, 1705, 1225, 565, 661, 13, 459, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15117, 16201, 17760, 15978, 17815, 16102, 16290, 15809] avg=16384.0000 max=17815 max_rank=4 min=15117 min_rank=0 imbalance_ratio=1.087341 max_load_reduction=113257 max_load_reduction_pct=0.864082 moved_to_echo=66016 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1599, 2229, 1930, 2, 256, 1, 2390, 279, 752, 2900, 627, 251, 1128, 31, 0, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1109, 1807, 1564, 1432, 8, 720, 165, 1801, 92, 19, 1065, 2170, 1616, 43, 1795, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 2, 1371, 538, 125, 1396, 1260, 157, 186, 1, 2042, 3047, 1, 2507, 837, 775, 763, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1249, 860, 744, 228, 7, 28, 23, 1749, 1, 688, 1, 342, 1619, 2932, 1390, 87, 2824, 798, 402, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 31, 0, 1491, 1000, 1049, 462, 1025, 1166, 2156, 2357, 134, 1157, 23, 1221, 2065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 802, 529, 7, 14, 1312, 572, 995, 24, 1963, 111, 1151, 977, 153, 2515, 455, 2, 3118, 2130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1408, 1100, 9, 2196, 2, 692, 1809, 837, 252, 1345, 78, 1055, 279, 2143, 1904, 1839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2417, 2441, 332, 1123, 1208, 712, 0, 1521, 1156, 2393, 1060, 726, 834, 5, 443, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15775, 16460, 16042, 16239, 16406, 16830, 16948, 16372] avg=16384.0000 max=16948 max_rank=6 min=15775 min_rank=0 imbalance_ratio=1.034424 max_load_reduction=114124 max_load_reduction_pct=0.870697 moved_to_echo=66556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1965, 2132, 2001, 3, 211, 0, 1953, 544, 1081, 2854, 516, 307, 1776, 24, 0, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 723, 1529, 1089, 1685, 12, 803, 192, 2105, 113, 22, 1031, 2059, 1576, 82, 2298, 1371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 404, 8, 1527, 903, 362, 1049, 1250, 200, 280, 12, 2297, 2741, 13, 2197, 748, 452, 660, 652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1415, 772, 580, 615, 11, 9, 6, 2371, 0, 459, 91, 645, 2097, 2195, 1396, 254, 3019, 862, 389, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 989, 30, 0, 2021, 1082, 945, 611, 1051, 1215, 1391, 2157, 126, 1653, 11, 679, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 375, 7, 3, 1336, 387, 1622, 75, 2296, 44, 873, 480, 219, 1967, 373, 2, 2431, 2062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1871, 1154, 36, 1883, 11, 944, 1785, 786, 261, 1339, 57, 1421, 707, 2399, 1648, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1797, 2257, 200, 1695, 1320, 955, 0, 1832, 1215, 2191, 1046, 678, 818, 8, 543, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16233, 16690, 15755, 17478, 15722, 15159, 17439, 16596] avg=16384.0000 max=17478 max_rank=3 min=15159 min_rank=5 imbalance_ratio=1.066772 max_load_reduction=113594 max_load_reduction_pct=0.866653 moved_to_echo=64916 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1936, 2530, 2048, 0, 54, 0, 2585, 399, 894, 3059, 931, 338, 1792, 32, 0, 1449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 1652, 1362, 1860, 52, 813, 178, 1836, 103, 18, 1048, 2213, 1963, 84, 2133, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 3, 930, 549, 108, 1298, 1101, 328, 58, 2, 2387, 3194, 0, 2404, 616, 1066, 813, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 724, 828, 677, 204, 0, 21, 14, 1877, 0, 320, 22, 328, 1938, 3228, 1470, 64, 2557, 894, 395, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 964, 6, 1, 1860, 1121, 1268, 570, 991, 1249, 1767, 2170, 71, 1516, 11, 968, 2043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 620, 549, 5, 41, 1902, 399, 1149, 25, 2052, 133, 1431, 580, 168, 2041, 376, 1, 3203, 2095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1584, 1154, 8, 1416, 5, 471, 1163, 668, 171, 1366, 19, 1344, 213, 1812, 2030, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2013, 2310, 211, 1284, 1212, 998, 0, 1661, 979, 2510, 1012, 812, 793, 6, 586, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18047, 16966, 15983, 15829, 16576, 16770, 14506, 16395] avg=16384.0000 max=18047 max_rank=0 min=14506 min_rank=6 imbalance_ratio=1.101501 max_load_reduction=113025 max_load_reduction_pct=0.862312 moved_to_echo=64247 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1822, 1805, 1681, 1, 251, 1, 2523, 222, 488, 3174, 1332, 468, 1335, 30, 0, 1658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 705, 2118, 1461, 1519, 11, 848, 163, 1726, 36, 70, 1109, 2551, 1092, 39, 1370, 882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 848, 2, 1271, 608, 143, 1809, 1436, 215, 114, 1, 2081, 2880, 0, 2399, 704, 1016, 788, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1542, 1041, 491, 413, 1, 6, 12, 2436, 0, 281, 31, 388, 1242, 3283, 1346, 69, 2152, 1180, 419, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 764, 15, 1, 1584, 1480, 772, 319, 710, 1253, 1878, 1894, 131, 1491, 6, 1036, 2455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 797, 486, 0, 18, 1451, 548, 655, 37, 2279, 60, 1542, 1112, 140, 2715, 401, 0, 3030, 2219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1104, 1075, 8, 1418, 2, 529, 1440, 821, 138, 1617, 44, 1128, 158, 2332, 1967, 2073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2436, 1787, 388, 1555, 1309, 871, 0, 939, 826, 2331, 1389, 493, 1022, 4, 507, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16791, 15700, 16961, 16622, 15789, 17490, 15854, 15865] avg=16384.0000 max=17490 max_rank=5 min=15700 min_rank=1 imbalance_ratio=1.067505 max_load_reduction=113582 max_load_reduction_pct=0.866562 moved_to_echo=64998 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[7808, 13312, 11904, 12800, 256, 6016, 1408, 15104, 896, 256, 8448, 16896, 11776, 512, 15104, 9472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[14208, 17408, 14720, 128, 1920, 128, 18560, 3072, 6144, 22144, 5504, 3584, 12800, 256, 128, 11392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12032, 8704, 128, 14464, 128, 4992, 12288, 6528, 1536, 11776, 512, 10752, 3200, 17280, 14720, 13056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[5248, 128, 9984, 4352, 1792, 11776, 11520, 1536, 1280, 128, 17920, 23680, 128, 18688, 6144, 6656, 5888, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[16896, 17152, 2560, 10880, 10752, 7168, 0, 12928, 9216, 18432, 9600, 5760, 6016, 128, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9600, 7296, 4608, 3328, 128, 128, 256, 17920, 128, 2432, 256, 3328, 13952, 23424, 11008, 1024, 19584, 8192, 3200, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[5376, 3456, 128, 256, 12928, 3840, 8320, 256, 18688, 512, 10624, 6912, 1664, 17280, 3072, 128, 22144, 16768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[7936, 256, 128, 13440, 10240, 8832, 4096, 7296, 11008, 15488, 17280, 1024, 12288, 128, 6656, 16128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 4, 0, 18, 4089, 0, 3676, 0, 0, 0, 3993, 0, 0, 0, 0, 4094, 0, 2, 4094, 0, 0, 4095, 4, 102, 3, 0, 4067, 0, 4095, 4096, 62, 0, 2, 4095, 0, 8, 4088, 7, 151, 0, 16, 0, 1, 0, 0, 1388, 0, 0, 4059, 4083, 0, 0, 4093, 0, 12, 4096, 1, 0, 4093, 0, 0, 0, 4090, 0, 7, 4005, 0, 0, 2469, 0, 0, 4096, 0, 89, 4091, 0, 0, 0, 0, 0, 0, 3855, 0, 0, 0, 0, 1, 3330, 0, 0, 0, 0, 0, 4096, 2, 3, 0, 0, 0, 4096, 4087, 22, 0, 0, 8, 4093, 0, 4089, 0, 0, 0, 4095, 4090, 0, 0, 50, 0, 24, 0, 0, 4079, 68, 0, 1368, 0, 0, 0, 4082] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 4074, 0, 2595, 6, 0, 0, 3794, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 137, 2, 0, 4085, 0, 4095, 4096, 11, 0, 0, 4095, 0, 1, 4094, 7, 207, 0, 23, 0, 0, 0, 0, 273, 0, 0, 4045, 4090, 0, 0, 4096, 0, 2, 4096, 0, 0, 4096, 1, 0, 0, 4096, 0, 1, 4076, 0, 0, 3758, 1, 0, 4096, 0, 45, 4096, 0, 0, 0, 0, 0, 0, 3868, 0, 0, 0, 0, 0, 3835, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4095, 4096, 0, 0, 1318, 0, 18, 2, 0, 4084, 20, 1, 689, 0, 0, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 3995, 2, 2430, 0, 0, 1, 3815, 4, 0, 0, 0, 4096, 1, 0, 4095, 1, 0, 4095, 2, 173, 0, 0, 4076, 0, 4086, 4092, 162, 0, 0, 4095, 0, 10, 4096, 1, 12, 0, 8, 0, 0, 0, 0, 367, 0, 0, 4088, 4081, 1, 0, 4095, 8, 5, 4093, 0, 0, 4092, 42, 3, 0, 4086, 0, 0, 4091, 0, 1, 3646, 9, 0, 4093, 0, 13, 4096, 0, 0, 3, 0, 0, 0, 3687, 3, 1, 0, 0, 0, 4072, 0, 0, 0, 1, 0, 4092, 0, 0, 0, 0, 0, 4096, 4092, 149, 0, 0, 0, 4095, 0, 4094, 0, 0, 4, 4093, 4096, 2, 0, 454, 0, 44, 1, 0, 4095, 0, 0, 1547, 0, 2, 0, 4089] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4067, 0, 2820, 1, 0, 0, 3999, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4076, 0, 316, 1, 0, 4090, 0, 4094, 4093, 0, 0, 2, 4096, 0, 6, 4090, 0, 103, 0, 6, 0, 0, 4, 0, 1271, 0, 2, 4069, 4094, 0, 0, 4096, 0, 34, 4096, 0, 0, 4096, 7, 0, 0, 4094, 0, 0, 4056, 0, 0, 3215, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 3831, 0, 2, 0, 0, 0, 3725, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 1, 10, 0, 4, 4096, 0, 4095, 6, 0, 0, 4096, 4096, 0, 0, 794, 0, 14, 0, 0, 4093, 0, 0, 448, 0, 0, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4092, 0, 522, 26, 0, 1, 4081, 0, 0, 0, 1, 4096, 1, 0, 4095, 1, 0, 4090, 0, 97, 2, 0, 4088, 0, 4095, 4096, 7, 1, 0, 4093, 0, 12, 4094, 4, 186, 0, 10, 0, 1, 2, 0, 196, 0, 0, 4088, 4091, 0, 1, 4095, 4, 4, 4095, 1, 1, 4095, 16, 0, 0, 4096, 0, 0, 4038, 0, 0, 3895, 6, 1, 4094, 0, 15, 4094, 0, 1, 0, 0, 0, 0, 3542, 0, 3, 1, 0, 2, 3966, 2, 1, 0, 0, 1, 4095, 1, 1, 0, 1, 0, 4095, 4096, 0, 0, 1, 1, 4095, 1, 4094, 0, 0, 1, 4092, 4095, 0, 1, 2899, 0, 1, 8, 0, 4083, 0, 0, 1082, 0, 0, 0, 4089] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4088, 0, 1625, 3, 0, 0, 3992, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 4091, 8, 237, 1, 0, 4079, 0, 4086, 4091, 48, 0, 0, 4096, 0, 18, 4059, 2, 115, 0, 3, 0, 0, 0, 1, 523, 0, 4, 4089, 4096, 0, 0, 4096, 0, 107, 4096, 0, 0, 4095, 1, 1, 0, 4088, 0, 8, 4059, 0, 0, 3783, 1, 0, 4096, 0, 10, 4061, 0, 0, 0, 4, 0, 0, 4009, 1, 0, 0, 0, 0, 3867, 0, 0, 0, 0, 0, 4096, 0, 1, 1, 0, 0, 4096, 4083, 13, 3, 0, 10, 4088, 0, 4096, 0, 0, 1, 4096, 4091, 0, 0, 1033, 1, 23, 0, 0, 4096, 1, 0, 1217, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 3950, 0, 1144, 2, 0, 0, 3721, 0, 0, 0, 0, 4067, 0, 0, 4095, 0, 0, 4058, 113, 41, 4, 0, 4039, 8, 4090, 4096, 0, 0, 1, 4096, 0, 49, 4018, 0, 26, 0, 1, 29, 0, 60, 0, 754, 0, 0, 4040, 4096, 0, 0, 4092, 18, 19, 4096, 0, 0, 4053, 28, 17, 0, 4048, 0, 1, 4095, 0, 0, 3678, 13, 0, 4096, 0, 9, 4095, 0, 0, 0, 0, 0, 0, 3695, 0, 0, 0, 0, 0, 3609, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4093, 59, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 4096, 4096, 0, 0, 785, 0, 87, 1, 0, 4094, 1, 0, 3062, 0, 0, 0, 4057] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4031, 0, 3500, 3, 0, 1, 3835, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 5, 48, 0, 0, 4094, 2, 4088, 4096, 15, 0, 0, 4095, 0, 0, 4096, 5, 24, 0, 14, 0, 1, 4, 1, 1010, 0, 0, 4096, 4094, 0, 0, 4096, 1, 7, 4095, 0, 0, 4093, 0, 2, 0, 4096, 0, 2, 4020, 0, 0, 3469, 21, 0, 4093, 0, 34, 4093, 0, 0, 0, 0, 0, 0, 3957, 0, 3, 0, 0, 0, 3800, 0, 0, 0, 0, 0, 4096, 2, 1, 0, 0, 3, 4091, 4041, 7, 1, 0, 3, 4096, 1, 4091, 0, 0, 0, 4096, 4095, 0, 0, 266, 0, 50, 0, 0, 4086, 2, 0, 620, 0, 0, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 18, 4089, 0, 3676, 0, 0, 0, 3993, 0, 0, 0, 0, 4094, 1800, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 4, 102, 3, 0, 4067, 0, 4095, 3928, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 8, 4088, 7, 151, 0, 16, 0, 1, 0, 0, 1388, 0, 0, 4096, 3246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4083, 0, 0, 3, 0, 12, 0, 1, 0, 4093, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4005, 0, 0, 2469, 0, 0, 4096, 0, 89, 4091, 0, 0, 0, 0, 0, 427, 168, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3855, 0, 0, 0, 0, 1, 3330, 0, 0, 0, 0, 0, 4096, 2, 3, 3857, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 22, 0, 0, 8, 4093, 0, 4089, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 50, 0, 24, 0, 0, 4079, 68, 0, 1368, 0, 0, 0, 4082, 1863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17913, 16358, 17098, 16341, 15373, 15992, 16373, 15624] avg=16384.0000 max=17913 max_rank=0 min=15373 min_rank=4 imbalance_ratio=1.093323 max_load_reduction=113159 max_load_reduction_pct=0.863335 moved_to_echo=63362 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 3995, 2, 2430, 0, 0, 1, 3815, 4, 0, 0, 0, 4096, 1800, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4095, 2, 173, 0, 0, 4076, 0, 4086, 3924, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 10, 4096, 1, 12, 0, 8, 0, 0, 0, 0, 367, 0, 0, 4093, 3247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4081, 1, 0, 4, 8, 5, 0, 0, 0, 4092, 42, 3, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1, 3646, 9, 0, 4093, 0, 13, 4096, 0, 0, 3, 0, 0, 427, 168, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3687, 3, 1, 0, 0, 0, 4072, 0, 0, 0, 1, 0, 4092, 0, 0, 3857, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 149, 0, 0, 0, 4095, 0, 4094, 0, 0, 4, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 454, 0, 44, 1, 0, 4095, 0, 0, 1547, 0, 2, 0, 4089, 1864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16520, 15929, 16410, 16568, 16561, 16506, 16194] avg=16384.0000 max=16568 max_rank=4 min=15929 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=114504 max_load_reduction_pct=0.873596 moved_to_echo=65829 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 4074, 0, 2595, 6, 0, 0, 3794, 2, 0, 0, 0, 4096, 1801, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 137, 2, 0, 4085, 0, 4095, 3928, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 4094, 7, 207, 0, 23, 0, 0, 0, 0, 273, 0, 0, 4096, 3248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 4090, 0, 0, 4, 0, 2, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4076, 0, 0, 3758, 1, 0, 4096, 0, 45, 4096, 0, 0, 0, 0, 0, 427, 168, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3868, 0, 0, 0, 0, 0, 3835, 0, 0, 0, 0, 0, 4096, 0, 0, 3857, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1318, 0, 18, 2, 0, 4084, 20, 1, 689, 0, 0, 0, 4093, 1864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16610, 16353, 16044, 16334, 16689, 16504, 16353, 16185] avg=16384.0000 max=16689 max_rank=4 min=16044 min_rank=2 imbalance_ratio=1.018616 max_load_reduction=114383 max_load_reduction_pct=0.872673 moved_to_echo=65731 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4092, 0, 522, 26, 0, 1, 4081, 0, 0, 0, 1, 4096, 1800, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4090, 0, 97, 2, 0, 4088, 0, 4095, 3928, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 12, 4094, 4, 186, 0, 10, 0, 1, 2, 0, 196, 0, 0, 4095, 3248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4091, 0, 1, 2, 4, 4, 0, 1, 1, 4095, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 3895, 6, 1, 4094, 0, 15, 4094, 0, 1, 0, 0, 0, 427, 168, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 0, 3, 1, 0, 2, 3966, 2, 1, 0, 0, 1, 4095, 1, 1, 3856, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 1, 1, 4095, 1, 4094, 0, 0, 1, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 2899, 0, 1, 8, 0, 4083, 0, 0, 1082, 0, 0, 0, 4089, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14859, 16310, 15941, 16399, 16760, 16318, 16361, 18124] avg=16384.0000 max=18124 max_rank=7 min=14859 min_rank=0 imbalance_ratio=1.106201 max_load_reduction=112948 max_load_reduction_pct=0.861725 moved_to_echo=67563 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4088, 0, 1625, 3, 0, 0, 3992, 0, 0, 0, 0, 4096, 1805, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 8, 237, 1, 0, 4079, 0, 4086, 3922, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 18, 4059, 2, 115, 0, 3, 0, 0, 0, 1, 523, 0, 4, 4096, 3248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 107, 0, 0, 0, 4095, 1, 1, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4059, 0, 0, 3783, 1, 0, 4096, 0, 10, 4061, 0, 0, 0, 4, 0, 427, 169, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 1, 0, 0, 0, 0, 3867, 0, 0, 0, 0, 0, 4096, 0, 1, 3856, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4083, 13, 3, 0, 10, 4088, 0, 4096, 0, 0, 1, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 1033, 1, 23, 0, 0, 4096, 1, 0, 1217, 0, 0, 0, 4096, 1864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15850, 16472, 16165, 16477, 16646, 16677, 16363, 16422] avg=16384.0000 max=16677 max_rank=5 min=15850 min_rank=0 imbalance_ratio=1.017883 max_load_reduction=114395 max_load_reduction_pct=0.872765 moved_to_echo=66108 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4067, 0, 2820, 1, 0, 0, 3999, 0, 0, 0, 0, 4096, 1801, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 316, 1, 0, 4090, 0, 4094, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 6, 4090, 0, 103, 0, 6, 0, 0, 4, 0, 1271, 0, 2, 4096, 3249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4094, 0, 0, 0, 0, 34, 0, 0, 0, 4096, 7, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 3215, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 0, 0, 430, 168, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3831, 0, 2, 0, 0, 0, 3725, 0, 0, 0, 0, 0, 4096, 0, 0, 3856, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 10, 0, 4, 4096, 0, 4095, 6, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 794, 0, 14, 0, 0, 4093, 0, 0, 448, 0, 0, 0, 4091, 1865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17024, 16502, 16925, 16394, 16086, 16357, 16383, 15401] avg=16384.0000 max=17024 max_rank=0 min=15401 min_rank=7 imbalance_ratio=1.039062 max_load_reduction=114048 max_load_reduction_pct=0.870117 moved_to_echo=64227 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4031, 0, 3500, 3, 0, 1, 3835, 0, 0, 0, 0, 4096, 1801, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 48, 0, 0, 4094, 2, 4088, 3928, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 5, 24, 0, 14, 0, 1, 4, 1, 1010, 0, 0, 4095, 3248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 4, 1, 7, 0, 0, 0, 4093, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4020, 0, 0, 3469, 21, 0, 4093, 0, 34, 4093, 0, 0, 0, 0, 0, 427, 168, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3957, 0, 3, 0, 0, 0, 3800, 0, 0, 0, 0, 0, 4096, 2, 1, 3852, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4041, 7, 1, 0, 3, 4096, 1, 4091, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 266, 0, 50, 0, 0, 4086, 2, 0, 620, 0, 0, 0, 4094, 1864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17508, 16276, 16593, 16393, 16348, 16559, 16318, 15077] avg=16384.0000 max=17508 max_rank=0 min=15077 min_rank=7 imbalance_ratio=1.068604 max_load_reduction=113564 max_load_reduction_pct=0.866425 moved_to_echo=64302 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 3950, 0, 1144, 2, 0, 0, 3721, 0, 0, 0, 0, 4067, 1799, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 113, 41, 4, 0, 4039, 8, 4090, 3928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 49, 4018, 0, 26, 0, 1, 29, 0, 60, 0, 754, 0, 0, 4096, 3247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 4096, 0, 0, 5, 18, 19, 0, 0, 0, 4053, 28, 17, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 3678, 13, 0, 4096, 0, 9, 4095, 0, 0, 0, 0, 0, 426, 168, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3695, 0, 0, 0, 0, 0, 3609, 0, 0, 0, 0, 0, 4096, 0, 0, 3856, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 59, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 785, 0, 87, 1, 0, 4094, 1, 0, 3062, 0, 0, 0, 4057, 1862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14924, 16281, 16377, 16324, 16602, 16104, 16415, 18045] avg=16384.0000 max=18045 max_rank=7 min=14924 min_rank=0 imbalance_ratio=1.101379 max_load_reduction=113027 max_load_reduction_pct=0.862328 moved_to_echo=67166 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 30464, 128, 128, 128, 0, 128, 30208, 128, 128, 0, 128, 128, 32768, 128, 128, 30848, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 32512, 128, 18432, 128, 0, 128, 31232, 128, 0, 0, 128, 32768, 14464, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 32768, 256, 128, 128, 128, 32768, 128, 32768, 128, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32512, 0, 128, 28032, 128, 128, 32768, 0, 256, 32768, 0, 128, 128, 128, 0, 3456, 1408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 0, 32768, 256, 1152, 128, 0, 32640, 128, 32768, 31488, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32640, 32768, 128, 128, 128, 128, 256, 0, 128, 128, 32768, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 7680, 128, 384, 128, 0, 32768, 128, 128, 10112, 0, 128, 0, 32768, 14976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 0, 128, 32640, 128, 896, 0, 128, 128, 128, 128, 128, 5888, 0, 128, 32768, 25984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2289, 0, 372, 0, 0, 8, 0, 4087, 13, 3, 0, 0, 0, 4091, 1, 0, 0, 3126, 4084, 0, 4092, 0, 4096, 4096, 0, 0, 0, 4092, 1498, 0, 10, 0, 4096, 0, 0, 4096, 84, 0, 0, 0, 4096, 0, 11, 0, 0, 4095, 4088, 3288, 0, 1, 0, 2, 4096, 4091, 0, 0, 4089, 4085, 0, 0, 4, 1699, 0, 0, 4095, 7, 0, 0, 0, 0, 4, 4096, 0, 0, 4093, 4, 7, 4034, 4089, 0, 0, 0, 4048, 0, 387, 0, 3312, 4, 0, 4066, 0, 0, 0, 4087, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 446, 4094, 0, 0, 6, 0, 0, 0, 4094, 0, 1, 0, 4091, 8, 3, 3, 0, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3485, 0, 544, 0, 0, 42, 0, 4095, 14, 0, 0, 0, 0, 4096, 0, 0, 0, 3883, 4095, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 1002, 0, 0, 0, 4096, 0, 1, 4096, 8, 0, 0, 1, 4096, 0, 0, 0, 0, 4095, 4085, 3215, 0, 0, 7, 0, 4095, 4096, 0, 0, 4094, 4096, 0, 0, 0, 3336, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4072, 4096, 0, 0, 0, 4058, 0, 11, 0, 753, 1, 0, 4093, 0, 0, 0, 4075, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 176, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 1557, 0, 506, 1, 1, 28, 1, 4092, 3, 0, 1, 0, 0, 4095, 0, 1, 0, 3808, 4089, 1, 4095, 1, 4095, 4095, 0, 0, 2, 4095, 1367, 0, 0, 0, 4095, 1, 0, 4095, 39, 1, 1, 0, 4095, 1, 1, 1, 0, 4093, 4090, 2653, 1, 0, 0, 0, 4093, 4094, 1, 1, 4089, 4094, 0, 1, 0, 2638, 0, 1, 4095, 2, 0, 1, 0, 0, 0, 4095, 0, 0, 4095, 0, 2, 3953, 4095, 0, 1, 5, 4054, 0, 461, 1, 3332, 1, 0, 4096, 0, 3, 0, 4092, 0, 2, 0, 1, 8, 0, 4094, 0, 1, 1, 0, 0, 0, 1, 172, 4095, 1, 1, 0, 0, 0, 0, 4095, 1, 0, 0, 4095, 3, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1975, 0, 1043, 0, 0, 115, 0, 4091, 13, 1, 0, 0, 0, 4093, 0, 0, 0, 3770, 4084, 0, 4092, 0, 4096, 4094, 0, 0, 0, 4092, 1243, 0, 3, 4, 4096, 7, 0, 4095, 41, 0, 0, 0, 4096, 0, 10, 0, 0, 4095, 4087, 3087, 0, 0, 2, 1, 4092, 4093, 0, 0, 4079, 4088, 0, 0, 2, 2840, 0, 0, 4096, 4, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 6, 5, 4074, 4093, 0, 0, 0, 4045, 0, 49, 0, 2101, 4, 0, 4080, 0, 0, 0, 4087, 2, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4, 0, 0, 209, 4093, 0, 0, 4, 0, 0, 0, 4094, 0, 2, 0, 4094, 10, 0, 3, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1578, 1, 2107, 0, 0, 33, 0, 4088, 6, 0, 0, 0, 0, 4091, 0, 0, 0, 3169, 3998, 0, 4089, 0, 4096, 4095, 0, 0, 0, 4095, 2052, 0, 1, 0, 4096, 8, 0, 4096, 7, 0, 0, 0, 4096, 0, 12, 0, 0, 4095, 4096, 3450, 0, 0, 0, 0, 4095, 4096, 0, 0, 4066, 4080, 0, 0, 0, 2766, 0, 0, 4096, 10, 0, 0, 0, 0, 6, 4096, 0, 0, 4096, 0, 16, 4089, 4087, 0, 0, 0, 4026, 0, 51, 0, 1278, 0, 0, 4094, 0, 0, 0, 4066, 1, 23, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 84, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 7, 0, 4086, 3, 0, 6, 0, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3003, 0, 1502, 0, 0, 98, 0, 4096, 11, 0, 0, 0, 0, 4096, 0, 0, 0, 3547, 4068, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 761, 0, 0, 0, 4096, 0, 1, 4096, 20, 0, 0, 0, 4096, 0, 11, 0, 0, 4096, 4092, 3428, 0, 0, 0, 0, 4096, 4089, 0, 0, 4081, 4077, 0, 0, 0, 3244, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 12, 4077, 4039, 0, 0, 37, 4019, 0, 121, 0, 690, 0, 0, 4063, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 151, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 12, 0, 4, 0, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2107, 0, 870, 0, 0, 113, 0, 4087, 82, 0, 0, 0, 0, 4095, 0, 0, 0, 3514, 3987, 0, 4094, 0, 4096, 4096, 0, 0, 0, 4096, 851, 0, 7, 0, 4096, 10, 0, 4096, 141, 0, 0, 0, 4096, 0, 5, 0, 0, 4096, 4089, 3030, 0, 0, 6, 0, 4094, 4096, 0, 0, 4044, 4019, 0, 0, 5, 3101, 0, 0, 4092, 23, 0, 0, 0, 0, 2, 4096, 0, 0, 4093, 0, 11, 4044, 4068, 0, 0, 34, 3893, 0, 71, 0, 2931, 4, 0, 4055, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 106, 4094, 0, 0, 4, 0, 0, 0, 4083, 0, 1, 0, 4096, 17, 0, 19, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 2684, 0, 587, 0, 0, 2, 0, 4090, 177, 0, 0, 0, 0, 4091, 0, 0, 0, 3030, 3988, 0, 4096, 0, 4096, 4096, 0, 0, 3, 4096, 841, 0, 40, 0, 4096, 0, 0, 4096, 1112, 0, 0, 0, 4096, 0, 43, 0, 0, 4096, 4071, 1725, 0, 0, 1, 0, 4096, 4090, 0, 0, 4010, 4046, 0, 0, 1, 2630, 0, 0, 4096, 47, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 1, 3865, 4079, 0, 0, 66, 3991, 0, 558, 0, 3321, 0, 0, 4092, 0, 0, 0, 3991, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 246, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 1, 0, 4085, 10, 0, 6, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2289, 0, 372, 0, 0, 8, 0, 4087, 13, 3, 0, 0, 0, 4091, 2308, 1616, 537, 215, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3126, 4084, 0, 4092, 0, 0, 0, 0, 0, 0, 3555, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3893, 0, 0, 4096, 84, 0, 0, 0, 4096, 0, 11, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2472, 3288, 0, 1, 0, 2, 0, 0, 0, 0, 4089, 4085, 0, 0, 4, 1699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 0, 0, 0, 0, 4, 4096, 0, 0, 4093, 4, 7, 4034, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1781, 0, 0, 0, 4048, 0, 387, 0, 3312, 4, 0, 4066, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 446, 4094, 4096, 3876, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4094, 0, 1, 0, 4091, 8, 3, 3, 0, 9, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15694, 16356, 16285, 15640, 16388, 17685, 16631, 16393] avg=16384.0000 max=17685 max_rank=5 min=15640 min_rank=3 imbalance_ratio=1.079407 max_load_reduction=113387 max_load_reduction_pct=0.865074 moved_to_echo=67097 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3485, 0, 544, 0, 0, 42, 0, 4095, 14, 0, 0, 0, 0, 4096, 2312, 1615, 537, 215, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3883, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 3559, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3893, 0, 1, 4096, 8, 0, 0, 1, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2470, 3215, 0, 0, 7, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 3336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4072, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1784, 0, 0, 0, 4058, 0, 11, 0, 753, 1, 0, 4093, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 176, 4096, 4095, 3881, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 1, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17110, 16635, 16190, 17218, 16409, 14775, 16362, 16373] avg=16384.0000 max=17218 max_rank=3 min=14775 min_rank=5 imbalance_ratio=1.050903 max_load_reduction=113854 max_load_reduction_pct=0.868637 moved_to_echo=63919 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1578, 1, 2107, 0, 0, 33, 0, 4088, 6, 0, 0, 0, 0, 4091, 2307, 1619, 537, 215, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3169, 3998, 0, 4089, 0, 0, 0, 0, 0, 0, 3558, 2052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3893, 8, 0, 4096, 7, 0, 0, 0, 4096, 0, 12, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2477, 3450, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4080, 0, 0, 0, 2766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 6, 4096, 0, 0, 4096, 0, 16, 4089, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1780, 0, 0, 0, 4026, 0, 51, 0, 1278, 0, 0, 4094, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 23, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 84, 4096, 4095, 3881, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 7, 0, 4086, 3, 0, 6, 0, 9, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16737, 16866, 16208, 16839, 16457, 15295, 16289, 16381] avg=16384.0000 max=16866 max_rank=1 min=15295 min_rank=5 imbalance_ratio=1.029419 max_load_reduction=114206 max_load_reduction_pct=0.871323 moved_to_echo=64422 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 1557, 0, 506, 1, 1, 28, 1, 4092, 3, 0, 1, 0, 0, 4095, 2311, 1617, 537, 216, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3808, 4089, 1, 4095, 1, 0, 0, 0, 0, 2, 3558, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 1, 0, 4095, 39, 1, 1, 0, 4095, 1, 1, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2473, 2653, 1, 0, 0, 0, 0, 0, 1, 1, 4089, 4094, 0, 1, 0, 2638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 2, 0, 1, 0, 0, 0, 4095, 0, 0, 4095, 0, 2, 3953, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1784, 0, 1, 5, 4054, 0, 461, 1, 3332, 1, 0, 4096, 0, 3, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 8, 0, 4094, 0, 1, 1, 0, 0, 0, 1, 172, 4095, 4093, 3878, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4095, 1, 0, 0, 4095, 3, 1, 0, 0, 0, 4095, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15124, 16922, 16220, 15951, 16292, 17830, 16360, 16373] avg=16384.0000 max=17830 max_rank=5 min=15124 min_rank=0 imbalance_ratio=1.088257 max_load_reduction=113242 max_load_reduction_pct=0.863968 moved_to_echo=66855 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1975, 0, 1043, 0, 0, 115, 0, 4091, 13, 1, 0, 0, 0, 4093, 2310, 1616, 537, 214, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3770, 4084, 0, 4092, 0, 0, 0, 0, 0, 0, 3555, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 3893, 7, 0, 4095, 41, 0, 0, 0, 4096, 0, 10, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2471, 3087, 0, 0, 2, 1, 0, 0, 0, 0, 4079, 4088, 0, 0, 2, 2840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 6, 5, 4074, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1783, 0, 0, 0, 4045, 0, 49, 0, 2101, 4, 0, 4080, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4, 0, 0, 209, 4093, 4092, 3879, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4094, 0, 2, 0, 4094, 10, 0, 3, 0, 2, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16163, 16744, 16244, 16570, 16429, 16149, 16387, 16386] avg=16384.0000 max=16744 max_rank=1 min=16149 min_rank=5 imbalance_ratio=1.021973 max_load_reduction=114328 max_load_reduction_pct=0.872253 moved_to_echo=65351 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3003, 0, 1502, 0, 0, 98, 0, 4096, 11, 0, 0, 0, 0, 4096, 2279, 1618, 537, 215, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3547, 4068, 0, 4096, 0, 0, 0, 0, 0, 0, 3559, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3893, 0, 1, 4096, 20, 0, 0, 0, 4096, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2474, 3428, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4077, 0, 0, 0, 3244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 12, 4077, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1760, 0, 0, 37, 4019, 0, 121, 0, 690, 0, 0, 4063, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 151, 4096, 4096, 3874, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 12, 0, 4, 0, 7, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17610, 16031, 16213, 17304, 16431, 14768, 16329, 16386] avg=16384.0000 max=17610 max_rank=0 min=14768 min_rank=5 imbalance_ratio=1.074829 max_load_reduction=113462 max_load_reduction_pct=0.865646 moved_to_echo=63914 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2107, 0, 870, 0, 0, 113, 0, 4087, 82, 0, 0, 0, 0, 4095, 2299, 1619, 541, 216, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3514, 3987, 0, 4094, 0, 0, 0, 0, 0, 0, 3555, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3885, 10, 0, 4096, 141, 0, 0, 0, 4096, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2470, 3030, 0, 0, 6, 0, 0, 0, 0, 0, 4044, 4019, 0, 0, 5, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 23, 0, 0, 0, 0, 2, 4096, 0, 0, 4093, 0, 11, 4044, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1769, 0, 0, 34, 3893, 0, 71, 0, 2931, 4, 0, 4055, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 106, 4094, 4094, 3880, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4083, 0, 1, 0, 4096, 17, 0, 19, 0, 3, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16185, 16001, 16336, 16675, 16416, 16774, 16285, 16400] avg=16384.0000 max=16774 max_rank=5 min=16001 min_rank=1 imbalance_ratio=1.023804 max_load_reduction=114298 max_load_reduction_pct=0.872025 moved_to_echo=65875 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2684, 0, 587, 0, 0, 2, 0, 4090, 177, 0, 0, 0, 0, 4091, 2302, 1609, 537, 214, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3030, 3988, 0, 4096, 0, 0, 0, 0, 0, 3, 3559, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 3893, 0, 0, 4096, 1112, 0, 0, 0, 4096, 0, 43, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2462, 1725, 0, 0, 1, 0, 0, 0, 0, 0, 4010, 4046, 0, 0, 1, 2630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 47, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 1, 3865, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1777, 0, 0, 66, 3991, 0, 558, 0, 3321, 0, 0, 4092, 0, 0, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 246, 4096, 4096, 3876, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 1, 0, 4085, 10, 0, 6, 0, 3, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16449, 15517, 17376, 14875, 16250, 17796, 16429, 16380] avg=16384.0000 max=17796 max_rank=5 min=14875 min_rank=3 imbalance_ratio=1.086182 max_load_reduction=113276 max_load_reduction_pct=0.864227 moved_to_echo=66855 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 31232, 128, 128, 32768, 1536, 128, 128, 128, 32768, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 128, 0, 32768, 0, 128, 128, 0, 128, 0, 128, 1664, 32768, 32768, 31104, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[19840, 23936, 128, 128, 128, 128, 0, 0, 128, 128, 32640, 32640, 0, 128, 128, 22272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 0, 0, 128, 32768, 0, 0, 32768, 128, 128, 32256, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 0, 0, 0, 32768, 128, 128, 0, 32768, 128, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[14336, 0, 128, 256, 32256, 0, 1792, 128, 17792, 128, 0, 32640, 0, 128, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 18688, 128, 7552, 128, 128, 512, 128, 32768, 384, 128, 128, 0, 0, 32768, 18432, 13056, 4352, 1792, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 27904, 32512, 128, 32768, 128, 0, 0, 0, 0, 128, 28544, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4095, 0, 0, 2209, 4096, 4096, 0, 0, 0, 4095, 0, 4096, 6, 0, 4093, 0, 0, 0, 4096, 0, 0, 4089, 0, 0, 4096, 0, 0, 4095, 1, 0, 1, 0, 0, 4096, 0, 1, 4096, 3715, 4096, 0, 7, 2631, 0, 4096, 4096, 4083, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 1, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 142, 0, 0, 0, 0, 0, 4096, 3584, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 4090, 3, 0, 825, 4094, 4096, 0, 0, 0, 4096, 0, 4071, 0, 0, 4096, 0, 0, 0, 4091, 0, 2, 4096, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3, 0, 4096, 3896, 4093, 0, 0, 3727, 0, 4096, 4096, 4094, 0, 0, 4096, 35, 8, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 10, 4087, 0, 0, 4096, 4087, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4091, 4094, 0, 0, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 5, 0, 0, 4096, 1, 0, 0, 0, 31, 4096, 0, 0, 0, 362, 0, 0, 0, 0, 0, 4096, 3388, 1, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 1, 0, 1452, 4093, 4096, 0, 0, 0, 4096, 0, 4068, 0, 0, 4096, 0, 0, 1, 4091, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 3337, 4085, 0, 0, 3299, 0, 4096, 4096, 4096, 0, 0, 4096, 36, 0, 0, 0, 0, 1, 6, 1, 0, 4096, 0, 0, 0, 1, 4089, 0, 0, 4094, 4087, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 4091, 4096, 0, 0, 0, 0, 4096, 0, 0, 26, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 6, 0, 0, 4096, 4, 0, 0, 0, 33, 4096, 0, 0, 0, 390, 0, 0, 0, 0, 0, 4096, 3753, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 1, 1, 2, 4095, 0, 0, 1520, 4095, 4095, 0, 0, 1, 4093, 1, 4095, 2, 1, 4093, 0, 0, 0, 4095, 0, 0, 4089, 0, 0, 4094, 0, 0, 4095, 3, 0, 2, 1, 0, 4095, 0, 15, 4095, 3447, 4094, 0, 5, 2546, 1, 4095, 4095, 4095, 1, 1, 4095, 87, 0, 0, 1, 1, 0, 1, 1, 0, 4095, 1, 0, 0, 23, 4095, 1, 0, 4095, 4093, 1, 1, 1, 0, 0, 0, 1, 4092, 0, 4095, 4095, 0, 0, 1, 0, 4094, 0, 0, 2, 0, 1, 0, 0, 1, 0, 7, 0, 1, 0, 1, 0, 1, 1, 4095, 0, 1, 1, 0, 9, 4091, 1, 0, 0, 612, 0, 0, 1, 0, 1, 4095, 4023, 0, 1, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8, 4087, 2, 0, 614, 4088, 4096, 0, 0, 0, 4096, 0, 4029, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 3466, 4077, 0, 0, 2999, 0, 4096, 4096, 4096, 0, 0, 4096, 6, 10, 0, 0, 0, 0, 26, 9, 0, 4096, 0, 0, 0, 7, 4095, 0, 0, 4096, 4059, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 4090, 0, 0, 0, 0, 4096, 0, 0, 70, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 46, 4096, 0, 0, 0, 1772, 0, 0, 0, 0, 0, 4096, 3394, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 4096, 0, 0, 3226, 4096, 4096, 0, 0, 0, 4096, 0, 4046, 0, 0, 4095, 0, 0, 8, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3744, 4075, 0, 0, 2509, 0, 4096, 4096, 4093, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 2, 4088, 0, 0, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 4096, 0, 0, 0, 0, 4094, 0, 0, 30, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 64, 4096, 0, 0, 0, 227, 0, 0, 0, 0, 0, 4096, 2543, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4093, 0, 0, 1882, 4092, 4096, 0, 0, 0, 4096, 0, 4041, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3060, 4089, 0, 0, 3265, 0, 4096, 4096, 4096, 0, 0, 4096, 40, 6, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 9, 4087, 0, 0, 4095, 4080, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 0, 1, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 57, 4096, 0, 0, 0, 559, 0, 0, 0, 0, 0, 4096, 3479, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 13, 4096, 0, 0, 1765, 4096, 4096, 0, 0, 0, 4096, 0, 4011, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3366, 4096, 0, 0, 1612, 0, 4096, 4096, 4096, 0, 0, 4096, 14, 6, 0, 0, 0, 0, 50, 0, 0, 4096, 0, 0, 0, 1, 4070, 0, 0, 4096, 4030, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4080, 4096, 0, 0, 0, 0, 4096, 0, 0, 73, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 81, 4096, 0, 0, 0, 1786, 0, 0, 0, 0, 0, 4096, 3679, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4095, 0, 0, 2209, 4096, 2410, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 43, 0, 0, 0, 4096, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 0, 0, 0, 1, 1868, 3715, 4096, 0, 7, 2631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4051, 4083, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 2228, 1686, 115, 45, 21, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 1, 0, 4075, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 4096, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 142, 0, 0, 0, 0, 0, 3981, 3584, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16905, 16426, 16415, 16337, 16403, 16364, 16324, 15898] avg=16384.0000 max=16905 max_rank=0 min=15898 min_rank=7 imbalance_ratio=1.031799 max_load_reduction=114167 max_load_reduction_pct=0.871025 moved_to_echo=64989 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 4090, 3, 0, 825, 4094, 2410, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 43, 0, 0, 0, 4091, 0, 2, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 1868, 3896, 4093, 0, 0, 3727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4051, 4094, 0, 0, 0, 35, 8, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4087, 0, 0, 4096, 4087, 0, 0, 0, 0, 2, 0, 0, 2228, 1686, 115, 45, 21, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 4094, 0, 0, 0, 0, 4075, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 5, 0, 0, 4096, 1, 0, 0, 0, 31, 4096, 4096, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 362, 0, 0, 0, 0, 0, 3981, 3388, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15522, 16395, 17683, 16383, 16396, 16368, 16401, 15924] avg=16384.0000 max=17683 max_rank=2 min=15522 min_rank=0 imbalance_ratio=1.079285 max_load_reduction=113389 max_load_reduction_pct=0.865089 moved_to_echo=65089 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4093, 0, 0, 1882, 4092, 2410, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 43, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1868, 3060, 4089, 0, 0, 3265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4051, 4096, 0, 0, 0, 40, 6, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4087, 0, 0, 4095, 4080, 0, 0, 0, 0, 0, 0, 0, 2228, 1686, 115, 45, 21, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 4075, 0, 0, 15, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 57, 4096, 4096, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 559, 0, 0, 0, 0, 0, 3981, 3479, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16573, 16368, 16378, 16393, 16385, 16376, 16388, 16211] avg=16384.0000 max=16573 max_rank=0 min=16211 min_rank=7 imbalance_ratio=1.011536 max_load_reduction=114499 max_load_reduction_pct=0.873558 moved_to_echo=65360 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 1, 0, 1452, 4093, 2410, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 43, 0, 0, 1, 4091, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 1868, 3337, 4085, 0, 0, 3299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4051, 4096, 0, 0, 0, 36, 0, 0, 0, 0, 1, 6, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 4094, 4087, 0, 0, 0, 0, 3, 0, 0, 2228, 1686, 115, 45, 21, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 4096, 0, 0, 0, 0, 4075, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 6, 0, 0, 4096, 4, 0, 0, 0, 33, 4096, 4096, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 390, 0, 0, 0, 0, 0, 3981, 3753, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16148, 16395, 16686, 16383, 16388, 16384, 16372, 16316] avg=16384.0000 max=16686 max_rank=2 min=16148 min_rank=0 imbalance_ratio=1.018433 max_load_reduction=114386 max_load_reduction_pct=0.872696 moved_to_echo=65460 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 1, 2, 4095, 0, 0, 1520, 4095, 2409, 0, 0, 1, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 1, 43, 0, 0, 0, 4095, 0, 0, 4089, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 2, 1, 0, 0, 0, 15, 1868, 3447, 4094, 0, 5, 2546, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4050, 4095, 1, 1, 0, 87, 0, 0, 1, 1, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 23, 4095, 1, 0, 4095, 4093, 1, 1, 1, 0, 0, 0, 1, 2227, 1686, 115, 45, 21, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4095, 4095, 0, 0, 1, 0, 4073, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 1, 0, 1, 0, 1, 1, 4095, 0, 1, 1, 0, 9, 4095, 4095, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 612, 0, 0, 1, 0, 1, 3980, 4023, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16220, 16419, 16077, 16428, 16425, 16359, 16339, 16805] avg=16384.0000 max=16805 max_rank=7 min=16077 min_rank=2 imbalance_ratio=1.025696 max_load_reduction=114267 max_load_reduction_pct=0.871788 moved_to_echo=65928 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8, 4087, 2, 0, 614, 4088, 2406, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 35, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 1864, 3466, 4077, 0, 0, 2999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4047, 4096, 0, 0, 0, 6, 10, 0, 0, 0, 0, 26, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4095, 0, 0, 4096, 4059, 0, 0, 0, 0, 0, 0, 0, 2232, 1690, 117, 49, 25, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 4090, 0, 0, 0, 0, 4071, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 46, 4096, 4096, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1772, 0, 0, 0, 0, 0, 3979, 3394, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15301, 16342, 16503, 16386, 16392, 16415, 16396, 17337] avg=16384.0000 max=17337 max_rank=7 min=15301 min_rank=0 imbalance_ratio=1.058167 max_load_reduction=113735 max_load_reduction_pct=0.867729 moved_to_echo=66540 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 4096, 0, 0, 3226, 4096, 2410, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 43, 0, 0, 8, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1868, 3744, 4075, 0, 0, 2509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4051, 4093, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4088, 0, 0, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 2228, 1686, 115, 45, 21, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 4096, 0, 0, 0, 0, 4073, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 64, 4096, 4096, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 227, 0, 0, 0, 0, 0, 3981, 2543, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17927, 16385, 16292, 16353, 16372, 16386, 16414, 14943] avg=16384.0000 max=17927 max_rank=0 min=14943 min_rank=7 imbalance_ratio=1.094177 max_load_reduction=113145 max_load_reduction_pct=0.863228 moved_to_echo=64115 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 13, 4096, 0, 0, 1765, 4096, 2410, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 43, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1868, 3366, 4096, 0, 0, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4051, 4096, 0, 0, 0, 14, 6, 0, 0, 0, 0, 50, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4070, 0, 0, 4096, 4030, 0, 0, 0, 0, 0, 0, 0, 2228, 1686, 115, 45, 21, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4080, 4096, 0, 0, 0, 0, 4075, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 81, 4096, 4096, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1786, 0, 0, 0, 0, 0, 3981, 3679, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16476, 16342, 15038, 16409, 16311, 16420, 16438, 17638] avg=16384.0000 max=17638 max_rank=7 min=15038 min_rank=2 imbalance_ratio=1.076538 max_load_reduction=113434 max_load_reduction_pct=0.865433 moved_to_echo=66807 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32512, 32768, 128, 128, 0, 256, 128, 0, 128, 128, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32512, 128, 128, 384, 0, 0, 128, 32768, 0, 128, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 128, 0, 13568, 32768, 19328, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 0, 128, 128, 14976, 28032, 32768, 0, 128, 22656, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 0, 0, 5888, 0, 0, 128, 0, 128, 31872, 27904, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 0, 32768, 32768, 0, 0, 128, 0, 32640, 0, 0, 256, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 256, 0, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 0, 384, 32768, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 32768, 32640, 128, 128, 128, 0, 128, 0, 128, 17920, 13568, 1024, 384, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[3188, 0, 4087, 0, 0, 4094, 1324, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 4096, 0, 4096, 135, 0, 0, 0, 0, 29, 4, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 0, 0, 4094, 4096, 0, 0, 4088, 4096, 0, 0, 5, 45, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 3805, 4096, 4096, 0, 4083, 0, 4096, 0, 0, 0, 0, 0, 106, 0, 3479, 4096, 0, 1, 0, 260, 0, 4096, 0, 0, 4027, 3, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[2425, 0, 4076, 0, 0, 4068, 1940, 0, 0, 0, 17, 27, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4096, 4092, 0, 4096, 0, 4096, 1, 0, 0, 0, 7, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4086, 4096, 0, 0, 66, 105, 4096, 8, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4017, 0, 0, 3552, 4096, 4084, 0, 4088, 2, 4095, 0, 0, 4, 0, 0, 83, 0, 3902, 4091, 0, 0, 0, 414, 0, 4096, 0, 0, 3982, 25, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[2313, 1, 4084, 0, 0, 4091, 1831, 1, 0, 0, 5, 2, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 4095, 4095, 4095, 1, 4095, 1, 4095, 28, 0, 0, 1, 0, 26, 0, 0, 0, 0, 4095, 0, 5, 1, 4092, 0, 0, 0, 4095, 4095, 1, 1, 4084, 4095, 0, 0, 240, 2, 4095, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 4095, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 4095, 1, 0, 4095, 4095, 0, 1, 3809, 4095, 4095, 1, 4094, 0, 4095, 1, 1, 0, 1, 0, 89, 1, 3964, 4095, 0, 0, 0, 53, 0, 4095, 0, 0, 4042, 0, 4095, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[3412, 0, 4060, 0, 0, 4023, 2690, 0, 0, 0, 12, 14, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 4092, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 20, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4090, 4092, 0, 0, 4040, 4096, 0, 0, 260, 90, 4096, 5, 4081, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4079, 0, 0, 1954, 4096, 4089, 0, 4056, 0, 4096, 0, 0, 8, 0, 0, 219, 0, 4037, 4096, 0, 0, 0, 123, 0, 4096, 0, 0, 3774, 23, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[3256, 0, 4083, 0, 0, 4040, 3601, 0, 0, 0, 35, 16, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4096, 4094, 0, 4096, 0, 4096, 28, 0, 0, 0, 8, 100, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4092, 0, 0, 4067, 4096, 0, 0, 113, 82, 4096, 15, 4020, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 1596, 4096, 4082, 0, 4070, 0, 4096, 0, 0, 5, 0, 0, 165, 0, 3655, 4084, 0, 0, 0, 139, 0, 4096, 0, 0, 3743, 74, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[2873, 0, 4085, 0, 0, 4013, 3270, 0, 0, 0, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 4096, 0, 4096, 0, 4096, 69, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 3873, 4096, 0, 0, 18, 76, 4096, 8, 4071, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4092, 0, 0, 2607, 4096, 4089, 0, 4088, 0, 4096, 0, 0, 15, 0, 0, 227, 0, 3606, 4096, 0, 0, 0, 133, 0, 4096, 0, 0, 3761, 54, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[2769, 0, 4075, 0, 0, 3999, 1306, 0, 0, 0, 69, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 4096, 4091, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 112, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 3903, 4096, 0, 0, 54, 38, 4096, 43, 4014, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4007, 0, 0, 2273, 4096, 4069, 0, 4053, 0, 4096, 0, 0, 67, 0, 0, 2347, 0, 3726, 4096, 0, 0, 0, 612, 0, 4096, 0, 0, 3425, 110, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[3471, 0, 4072, 0, 0, 4070, 3461, 0, 0, 0, 21, 17, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4096, 4088, 0, 4096, 0, 4096, 5, 0, 0, 0, 10, 171, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 0, 0, 4058, 4096, 0, 0, 4, 361, 4096, 8, 4077, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4086, 0, 0, 1131, 4096, 4077, 0, 4041, 0, 4094, 0, 0, 7, 0, 0, 295, 0, 4025, 4096, 0, 0, 0, 39, 0, 4096, 0, 0, 3500, 47, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2425, 0, 4076, 0, 0, 4068, 1940, 0, 0, 0, 17, 27, 0, 0, 0, 0, 2721, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4096, 4092, 4096, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 7, 3, 0, 0, 0, 0, 4096, 0, 3709, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 0, 0, 4096, 4096, 0, 0, 4086, 0, 0, 0, 66, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4017, 4073, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3552, 1086, 4084, 0, 4088, 2, 4095, 0, 0, 4, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 4091, 0, 0, 0, 414, 0, 351, 0, 0, 3982, 25, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15377, 16389, 16297, 16424, 16394, 16336, 16994, 16861] avg=16384.0000 max=16994 max_rank=6 min=15377 min_rank=0 imbalance_ratio=1.037231 max_load_reduction=114078 max_load_reduction_pct=0.870346 moved_to_echo=66585 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3188, 0, 4087, 0, 0, 4094, 1324, 0, 0, 0, 1, 1, 0, 0, 0, 0, 2721, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 135, 0, 0, 0, 0, 29, 4, 0, 0, 0, 4096, 0, 3709, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3975, 0, 0, 0, 4094, 4096, 0, 0, 4088, 0, 0, 0, 5, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4074, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3805, 1086, 4096, 0, 4083, 0, 4096, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3479, 4096, 0, 1, 0, 260, 0, 351, 0, 0, 4027, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15519, 16405, 16454, 16304, 16401, 16404, 17272, 16313] avg=16384.0000 max=17272 max_rank=6 min=15519 min_rank=0 imbalance_ratio=1.054199 max_load_reduction=113800 max_load_reduction_pct=0.868225 moved_to_echo=66390 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2313, 1, 4084, 0, 0, 4091, 1831, 1, 0, 0, 5, 2, 1, 1, 1, 0, 2720, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 4095, 4095, 4095, 4095, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 4095, 28, 0, 0, 1, 0, 26, 0, 0, 0, 0, 4095, 0, 3709, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 3971, 0, 0, 0, 4095, 4095, 1, 1, 4084, 0, 0, 0, 240, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 4095, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 4095, 1, 0, 4095, 4095, 4072, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3809, 1086, 4095, 1, 4094, 0, 4095, 1, 1, 0, 1, 0, 89, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 4095, 0, 0, 0, 53, 0, 350, 0, 0, 4042, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15154, 16408, 16340, 16495, 16400, 16402, 17274, 16599] avg=16384.0000 max=17274 max_rank=6 min=15154 min_rank=0 imbalance_ratio=1.054321 max_load_reduction=113798 max_load_reduction_pct=0.868210 moved_to_echo=66675 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3412, 0, 4060, 0, 0, 4023, 2690, 0, 0, 0, 12, 14, 0, 0, 0, 0, 2721, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 4092, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 20, 0, 0, 0, 0, 4096, 0, 3709, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 0, 0, 4090, 4092, 0, 0, 4040, 0, 0, 0, 260, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 4081, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4079, 4074, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1954, 1086, 4089, 0, 4056, 0, 4096, 0, 0, 8, 0, 0, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 4096, 0, 0, 0, 123, 0, 351, 0, 0, 3774, 23, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17035, 16393, 16311, 16547, 16390, 16388, 15508, 16500] avg=16384.0000 max=17035 max_rank=0 min=15508 min_rank=6 imbalance_ratio=1.039734 max_load_reduction=114037 max_load_reduction_pct=0.870033 moved_to_echo=64786 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3471, 0, 4072, 0, 0, 4070, 3461, 0, 0, 0, 21, 17, 0, 0, 0, 0, 2721, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4096, 4088, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 0, 0, 10, 171, 0, 0, 0, 0, 4096, 0, 3709, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 0, 0, 4095, 4096, 0, 0, 4058, 0, 0, 0, 4, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 4077, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4086, 4074, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1131, 1086, 4077, 0, 4041, 0, 4094, 0, 0, 7, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 4096, 0, 0, 0, 39, 0, 351, 0, 0, 3500, 47, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17936, 16381, 16472, 16589, 16391, 16418, 14731, 16154] avg=16384.0000 max=17936 max_rank=0 min=14731 min_rank=6 imbalance_ratio=1.094727 max_load_reduction=113136 max_load_reduction_pct=0.863159 moved_to_echo=63694 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3256, 0, 4083, 0, 0, 4040, 3601, 0, 0, 0, 35, 16, 0, 0, 0, 0, 2723, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4096, 4094, 4096, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 28, 0, 0, 0, 8, 100, 0, 0, 0, 0, 4096, 0, 3716, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 4096, 4092, 0, 0, 4067, 0, 0, 0, 113, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 4020, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4073, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1596, 1080, 4082, 0, 4070, 0, 4096, 0, 0, 5, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3655, 4084, 0, 0, 0, 139, 0, 341, 0, 0, 3743, 74, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17858, 16372, 16433, 16417, 16348, 16418, 15094, 16132] avg=16384.0000 max=17858 max_rank=0 min=15094 min_rank=6 imbalance_ratio=1.089966 max_load_reduction=113214 max_load_reduction_pct=0.863754 moved_to_echo=63992 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2873, 0, 4085, 0, 0, 4013, 3270, 0, 0, 0, 17, 17, 0, 0, 0, 0, 2721, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 4096, 4096, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 69, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 0, 3709, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 0, 0, 4096, 4096, 0, 0, 3873, 0, 0, 0, 18, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 4071, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4092, 4073, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2607, 1086, 4089, 0, 4088, 0, 4096, 0, 0, 15, 0, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3606, 4096, 0, 0, 0, 133, 0, 351, 0, 0, 3761, 54, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17099, 16391, 16361, 16134, 16385, 16397, 16208, 16097] avg=16384.0000 max=17099 max_rank=0 min=16097 min_rank=7 imbalance_ratio=1.043640 max_load_reduction=113973 max_load_reduction_pct=0.869545 moved_to_echo=65087 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2769, 0, 4075, 0, 0, 3999, 1306, 0, 0, 0, 69, 52, 0, 0, 0, 0, 2721, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 4096, 4091, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 112, 0, 0, 0, 0, 4096, 0, 3709, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 0, 0, 4096, 4096, 0, 0, 3903, 0, 0, 0, 54, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 43, 4014, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4007, 4074, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2273, 1086, 4069, 0, 4053, 0, 4096, 0, 0, 67, 0, 0, 2347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3726, 4096, 0, 0, 0, 612, 0, 351, 0, 0, 3425, 110, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15094, 16333, 16404, 16162, 16363, 16309, 17991, 16416] avg=16384.0000 max=17991 max_rank=6 min=15094 min_rank=0 imbalance_ratio=1.098083 max_load_reduction=113081 max_load_reduction_pct=0.862740 moved_to_echo=67079 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[23808, 128, 32640, 0, 0, 32512, 19456, 128, 0, 0, 256, 256, 128, 128, 128, 0, 21888, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 0, 128, 128, 128, 32768, 128, 0, 32768, 32640, 32640, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 128, 32640, 32768, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 32768, 384, 0, 0, 128, 128, 512, 128, 0, 0, 0, 32768, 0, 29696, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[30464, 32768, 0, 128, 0, 1792, 0, 2816, 0, 0, 30336, 384, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 20736, 8704, 32768, 128, 32640, 128, 32768, 128, 128, 128, 128, 0, 3584, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 128, 128, 32640, 32768, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 31872, 0, 0, 0, 32768, 32768, 128, 128, 32256, 0, 0, 0, 768, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 65, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 15, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3743, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3298, 4021, 0, 0, 0, 1169, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 4, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 1, 1, 4095, 4095, 4095, 1, 0, 53, 1, 4046, 0, 0, 0, 0, 1, 0, 1, 0, 1, 4094, 1, 1, 4095, 4095, 1, 0, 0, 1, 4095, 4095, 4095, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 4095, 4095, 4095, 0, 0, 0, 0, 1, 0, 0, 1, 1, 3290, 0, 1, 0, 4093, 0, 0, 4095, 0, 1, 4095, 1, 1, 0, 0, 0, 0, 1, 4094, 4092, 0, 1, 3630, 3863, 0, 0, 0, 1546, 4094, 1, 4095, 4095, 1, 0, 1, 4095, 4095, 0, 1, 0, 1, 1, 0, 0, 4094, 0, 4095, 0, 0, 0, 4038, 0, 0, 7, 0, 4, 1, 0, 1, 4, 1, 0, 1, 4094, 1, 0, 1, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4089, 4092, 0, 0, 171, 0, 3598, 0, 0, 0, 3, 0, 6, 0, 0, 0, 4093, 4, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4095, 4096, 0, 2, 5, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 4096, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 4069, 0, 0, 2939, 3943, 0, 0, 0, 1970, 4089, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 4047, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4088, 4096, 0, 0, 110, 0, 4034, 0, 0, 0, 10, 0, 29, 0, 0, 0, 4095, 4, 0, 4096, 4096, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 4096, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 4063, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4042, 0, 0, 2521, 4081, 0, 0, 0, 1759, 4088, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 4096, 0, 0, 0, 4051, 0, 0, 0, 0, 1, 0, 0, 0, 22, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4094, 4092, 4092, 0, 0, 212, 0, 3973, 0, 0, 0, 4, 0, 6, 0, 0, 0, 4096, 3, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 1, 5, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 4096, 4093, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3942, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4077, 0, 0, 3281, 4020, 0, 0, 0, 994, 4090, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4095, 0, 0, 0, 4080, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4090, 4092, 0, 0, 97, 0, 4077, 0, 0, 0, 4, 0, 11, 0, 0, 0, 4088, 8, 0, 4096, 4096, 0, 0, 0, 0, 4093, 4096, 4096, 0, 0, 3, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4095, 4096, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 4096, 4080, 0, 0, 1232, 3922, 0, 0, 0, 3119, 4084, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 4080, 0, 0, 2, 0, 10, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4091, 4095, 0, 0, 154, 0, 4094, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4095, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4080, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4061, 0, 0, 1122, 4057, 0, 0, 0, 2939, 4075, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4096, 0, 0, 0, 4048, 0, 0, 10, 0, 27, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 429, 0, 4042, 0, 0, 0, 0, 0, 47, 0, 0, 0, 4078, 14, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 4096, 4096, 125, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 0, 0, 0, 4056, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3994, 0, 0, 2978, 3901, 0, 0, 0, 2128, 4083, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 4096, 0, 0, 0, 3820, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4046, 4089, 4092, 0, 0, 171, 0, 3598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 0, 0, 0, 4093, 4, 0, 4096, 4096, 0, 0, 0, 0, 4034, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 2, 5, 0, 0, 0, 4093, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 0, 0, 0, 4085, 308, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 4069, 0, 28, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2939, 3943, 0, 0, 0, 1970, 4089, 0, 0, 0, 0, 0, 0, 3743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 4047, 0, 0, 4096, 62, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15996, 16377, 16389, 16458, 16386, 16684, 16408, 16374] avg=16384.0000 max=16684 max_rank=5 min=15996 min_rank=0 imbalance_ratio=1.018311 max_load_reduction=114388 max_load_reduction_pct=0.872711 moved_to_echo=65852 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4046, 4096, 4096, 0, 0, 65, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 0, 4033, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 15, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3743, 0, 0, 0, 4096, 308, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 28, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3298, 4021, 0, 0, 0, 1169, 4096, 0, 0, 0, 0, 0, 0, 3743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4093, 0, 0, 4096, 63, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16365, 16365, 16395, 16362, 16414, 16327, 16460, 16384] avg=16384.0000 max=16460 max_rank=6 min=16327 min_rank=5 imbalance_ratio=1.004639 max_load_reduction=114612 max_load_reduction_pct=0.874420 moved_to_echo=65585 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1, 1, 4045, 4095, 4095, 1, 0, 53, 1, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 4094, 1, 1, 4095, 4095, 1, 0, 0, 1, 4033, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 1, 0, 4092, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 1, 0, 0, 1, 1, 3290, 0, 1, 0, 4093, 307, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 4095, 1, 1, 0, 0, 0, 0, 1, 4094, 4092, 0, 28, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3630, 3863, 0, 0, 0, 1546, 4094, 1, 0, 0, 1, 0, 1, 3743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 4094, 0, 4095, 0, 0, 0, 4038, 0, 0, 4095, 62, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4, 1, 0, 1, 4, 1, 0, 1, 4094, 1, 0, 1, 1, 1, 4095, 4095, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16339, 16369, 16378, 15906, 16411, 16880, 16403, 16386] avg=16384.0000 max=16880 max_rank=5 min=15906 min_rank=3 imbalance_ratio=1.030273 max_load_reduction=114192 max_load_reduction_pct=0.871216 moved_to_echo=66080 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4046, 4090, 4092, 0, 0, 97, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 11, 0, 0, 0, 4088, 8, 0, 4096, 4096, 0, 0, 0, 0, 4034, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 3, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 4085, 308, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 4096, 4080, 0, 28, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1232, 3922, 0, 0, 0, 3119, 4084, 0, 0, 0, 0, 0, 0, 3743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 4080, 0, 0, 4096, 62, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 10, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16402, 16382, 16382, 16582, 16403, 16100, 16439, 16382] avg=16384.0000 max=16582 max_rank=3 min=16100 min_rank=5 imbalance_ratio=1.012085 max_load_reduction=114490 max_load_reduction_pct=0.873489 moved_to_echo=65324 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4040, 4088, 4096, 0, 0, 110, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 29, 0, 0, 0, 4095, 4, 0, 4096, 4096, 0, 0, 0, 0, 4034, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 8, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 4063, 309, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4042, 0, 30, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2521, 4081, 0, 0, 0, 1759, 4088, 0, 0, 0, 0, 0, 0, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 4096, 0, 0, 0, 4051, 0, 0, 4096, 62, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 22, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16368, 16416, 16379, 16586, 16369, 16184, 16379, 16391] avg=16384.0000 max=16586 max_rank=3 min=16184 min_rank=5 imbalance_ratio=1.012329 max_load_reduction=114486 max_load_reduction_pct=0.873459 moved_to_echo=65323 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4044, 4092, 4092, 0, 0, 212, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6, 0, 0, 0, 4096, 3, 0, 4096, 4096, 0, 0, 0, 0, 4034, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 5, 0, 0, 0, 4093, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3942, 0, 0, 0, 4088, 308, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4077, 0, 28, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3281, 4020, 0, 0, 0, 994, 4090, 0, 0, 0, 0, 0, 0, 3743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4095, 0, 0, 0, 4080, 0, 0, 4096, 62, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16413, 16380, 16389, 16554, 16396, 16128, 16441, 16371] avg=16384.0000 max=16554 max_rank=3 min=16128 min_rank=5 imbalance_ratio=1.010376 max_load_reduction=114518 max_load_reduction_pct=0.873703 moved_to_echo=65336 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4046, 4091, 4095, 0, 0, 154, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4034, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4080, 308, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4061, 0, 28, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 4057, 0, 0, 0, 2939, 4075, 0, 0, 0, 0, 0, 0, 3743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4096, 0, 0, 0, 4048, 0, 0, 4096, 62, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 27, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16480, 16374, 16381, 16715, 16380, 15936, 16395, 16411] avg=16384.0000 max=16715 max_rank=3 min=15936 min_rank=5 imbalance_ratio=1.020203 max_load_reduction=114357 max_load_reduction_pct=0.872475 moved_to_echo=65122 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4046, 4096, 4096, 0, 0, 429, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 4078, 14, 0, 4096, 4096, 0, 0, 0, 0, 4033, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 125, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 0, 0, 0, 4056, 308, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3994, 0, 28, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2978, 3901, 0, 0, 0, 2128, 4083, 0, 0, 0, 0, 0, 0, 3743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 4096, 0, 0, 0, 3820, 0, 0, 4096, 63, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16709, 16409, 16379, 15909, 16313, 16833, 16147, 16373] avg=16384.0000 max=16833 max_rank=5 min=15909 min_rank=3 imbalance_ratio=1.027405 max_load_reduction=114239 max_load_reduction_pct=0.871574 moved_to_echo=65666 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 32768, 0, 0, 128, 128, 0, 128, 0, 32768, 128, 0, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 32768, 0, 128, 32768, 128, 128, 0, 128, 0, 0, 128, 32768, 32512, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 128, 128, 32384, 32768, 32768, 128, 0, 1408, 128, 32000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 256, 0, 0, 128, 128, 0, 0, 128, 128, 30080, 0, 128, 0, 32768, 2560, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 128, 0, 128, 32768, 128, 128, 32768, 32768, 128, 0, 0, 128, 32384, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 21120, 31872, 0, 0, 0, 15744, 32768, 128, 0, 0, 128, 0, 128, 29952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 32768, 0, 32768, 0, 0, 0, 32384, 0, 0, 32768, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 0, 128, 128, 128, 0, 128, 32768, 128, 0, 128, 128, 128, 32768, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 1, 4096, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 3521, 4096, 4096, 0, 583, 4078, 0, 0, 4096, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 22, 4096, 4096, 4095, 4096, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 13, 4096, 0, 30, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 8, 4094, 0, 174, 4079, 4096, 0, 0, 0, 0, 0, 3653, 4087, 4096, 0, 318, 4077, 0, 0, 4096, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 10, 26, 4096, 4096, 4092, 4096, 0, 4096, 0, 3, 0, 4096, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 4078, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 9, 4096, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 12, 4096, 0, 17, 4086, 4096, 0, 0, 0, 0, 0, 3349, 4091, 4096, 0, 709, 4085, 0, 0, 4096, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 8, 69, 4096, 4096, 4091, 4096, 0, 4096, 0, 4, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 7, 4096, 0, 46, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 3, 4096, 0, 129, 4079, 4096, 0, 0, 0, 0, 0, 3813, 4088, 4096, 0, 211, 4048, 0, 0, 4096, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 8, 38, 4096, 4096, 4096, 4093, 0, 4096, 0, 0, 0, 4079, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 0, 0, 4074, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 22, 4096, 0, 67, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 5, 4094, 0, 136, 4077, 4096, 0, 0, 0, 0, 0, 3310, 4076, 4096, 0, 772, 4085, 0, 0, 4096, 0, 0, 0, 0, 4096, 4060, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 15, 56, 4096, 4096, 4088, 4096, 0, 4096, 0, 8, 0, 4095, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 25, 0, 0, 0, 0, 4096, 0, 0, 4034, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 1, 0, 1, 0, 4094, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 4095, 1, 1, 4093, 1, 1, 1, 0, 4095, 0, 17, 1, 0, 1, 0, 0, 4095, 0, 1, 0, 5, 1, 0, 4094, 0, 0, 0, 4095, 0, 5, 4094, 4095, 0, 0, 1, 1, 0, 3820, 4095, 4095, 0, 277, 4093, 0, 0, 4095, 0, 0, 0, 0, 4095, 4059, 0, 0, 1, 4095, 1, 1, 0, 4095, 0, 4095, 4093, 0, 0, 23, 4095, 4095, 4095, 4094, 1, 4092, 0, 0, 1, 4092, 0, 4095, 1, 1, 0, 1, 4095, 0, 1, 0, 0, 1, 1, 0, 0, 4095, 0, 0, 4091, 4095, 0, 1, 1, 2, 1, 1, 0, 1, 1, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 19, 4096, 0, 366, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 11, 4081, 0, 220, 4096, 4096, 0, 0, 0, 0, 0, 1975, 3964, 4096, 0, 1764, 3926, 0, 0, 4096, 0, 0, 0, 0, 4096, 4066, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 29, 163, 4096, 4096, 4078, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 19, 0, 0, 0, 0, 4096, 0, 0, 3997, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 16, 4096, 0, 20, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 4082, 0, 0, 9, 4096, 0, 30, 4088, 4096, 0, 0, 0, 0, 0, 3391, 4086, 4096, 0, 633, 4083, 0, 0, 4096, 0, 0, 0, 0, 4096, 4076, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 9, 59, 4096, 4096, 4096, 4096, 0, 4096, 0, 3, 0, 4091, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 0, 4081, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4025, 0, 0, 0, 0, 0, 0, 15, 4096, 4068, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 13, 4096, 0, 30, 0, 0, 3955, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 8, 4094, 0, 4044, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 4079, 0, 0, 0, 0, 0, 0, 3653, 4087, 64, 0, 318, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 21, 4090, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 26, 0, 0, 4092, 4096, 0, 4036, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 0, 4008, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16352, 16346, 16388, 16452, 16399, 16359, 16381, 16395] avg=16384.0000 max=16452 max_rank=3 min=16346 min_rank=1 imbalance_ratio=1.004150 max_load_reduction=114620 max_load_reduction_pct=0.874481 moved_to_echo=65534 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4068, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 1, 0, 0, 3955, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 1, 4096, 0, 4045, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 3521, 4096, 64, 0, 583, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 21, 4085, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 4095, 4096, 0, 4035, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4008, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16406, 16305, 16403, 16439, 16391, 16344, 16372, 16412] avg=16384.0000 max=16439 max_rank=3 min=16305 min_rank=1 imbalance_ratio=1.003357 max_load_reduction=114633 max_load_reduction_pct=0.874580 moved_to_echo=65519 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 14, 4096, 4068, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 16, 4096, 0, 20, 0, 0, 3955, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 4082, 0, 0, 9, 4096, 0, 4045, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4088, 0, 0, 0, 0, 0, 0, 3391, 4086, 64, 0, 633, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 21, 4076, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 59, 0, 0, 4096, 4096, 0, 4036, 0, 3, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 4008, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16408, 16339, 16395, 16375, 16385, 16390, 16383, 16397] avg=16384.0000 max=16408 max_rank=0 min=16339 min_rank=1 imbalance_ratio=1.001465 max_load_reduction=114664 max_load_reduction_pct=0.874817 moved_to_echo=65555 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 14, 4096, 4068, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 9, 4096, 0, 9, 0, 0, 3955, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 12, 4096, 0, 4044, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4086, 0, 0, 0, 0, 0, 0, 3349, 4091, 64, 0, 709, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 21, 4083, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 69, 0, 0, 4091, 4096, 0, 4036, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 4008, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16321, 16399, 16401, 16392, 16400, 16378, 16402] avg=16384.0000 max=16402 max_rank=7 min=16321 min_rank=1 imbalance_ratio=1.001099 max_load_reduction=114670 max_load_reduction_pct=0.874863 moved_to_echo=65572 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 1, 0, 1, 0, 4094, 0, 1, 0, 0, 1, 0, 0, 4095, 4067, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 4093, 1, 1, 1, 0, 4095, 0, 17, 1, 0, 3954, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 1, 0, 5, 1, 0, 4094, 0, 0, 0, 4095, 0, 4041, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4094, 0, 0, 0, 1, 1, 0, 3820, 4095, 64, 0, 277, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 22, 4059, 0, 0, 1, 4095, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 4095, 4094, 1, 4032, 0, 0, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 1, 4095, 0, 1, 0, 0, 1, 1, 0, 0, 4095, 0, 4007, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 0, 1, 1, 2, 1, 1, 0, 1, 1, 0, 0, 0, 0, 4095, 4095, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16408, 16321, 16399, 16450, 16369, 16338, 16375, 16412] avg=16384.0000 max=16450 max_rank=3 min=16321 min_rank=1 imbalance_ratio=1.004028 max_load_reduction=114622 max_load_reduction_pct=0.874496 moved_to_echo=65494 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 33, 4096, 4068, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 22, 4096, 0, 67, 0, 0, 3958, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 5, 4094, 0, 4044, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 4077, 0, 0, 0, 0, 0, 0, 3310, 4076, 58, 0, 772, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 17, 4060, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 56, 0, 0, 4088, 4096, 0, 4035, 0, 8, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 25, 0, 0, 0, 0, 4096, 0, 4011, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16301, 16396, 16348, 16514, 16365, 16393, 16405, 16350] avg=16384.0000 max=16514 max_rank=3 min=16301 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=114558 max_load_reduction_pct=0.874008 moved_to_echo=65513 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 8, 4096, 4068, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 7, 4096, 0, 46, 0, 0, 3955, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 3, 4096, 0, 4044, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 4079, 0, 0, 0, 0, 0, 0, 3813, 4088, 64, 0, 211, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 21, 4087, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 38, 0, 0, 4096, 4093, 0, 4036, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 0, 4008, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16356, 16387, 16432, 16396, 16350, 16386, 16390] avg=16384.0000 max=16432 max_rank=3 min=16350 min_rank=5 imbalance_ratio=1.002930 max_load_reduction=114640 max_load_reduction_pct=0.874634 moved_to_echo=65522 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 98, 4096, 4067, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 19, 4096, 0, 366, 0, 0, 3955, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 11, 4081, 0, 4045, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 4096, 0, 0, 0, 0, 0, 0, 1975, 3964, 64, 0, 1764, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 21, 4066, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 163, 0, 0, 4078, 4096, 0, 4036, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 19, 0, 0, 0, 0, 4096, 0, 4008, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16443, 16688, 16353, 16009, 16375, 16498, 16392, 16314] avg=16384.0000 max=16688 max_rank=1 min=16009 min_rank=3 imbalance_ratio=1.018555 max_load_reduction=114384 max_load_reduction_pct=0.872681 moved_to_echo=65579 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 0, 128, 32768, 0, 128, 0, 128, 128, 128, 0, 0, 32768, 0, 32128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 32768, 0, 640, 128, 0, 31744, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 0, 0, 256, 32640, 0, 0, 128, 32768, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[768, 32768, 0, 0, 0, 128, 128, 0, 26880, 32640, 512, 0, 5376, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 128, 0, 32512, 0, 128, 0, 0, 128, 0, 256, 32768, 32640, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 512, 0, 0, 32768, 32768, 128, 32384, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32640, 32768, 0, 128, 128, 128, 128, 128, 0, 128, 128, 0, 128, 0, 0, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 0, 32768, 0, 128, 0, 128, 128, 0, 32640, 0, 0, 128, 32768, 0, 32384, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4083, 0, 0, 35, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 1454, 0, 0, 0, 0, 4096, 506, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4064, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1745, 0, 4096, 0, 0, 1, 0, 0, 0, 362, 4096, 0, 5, 0, 0, 0, 0, 0, 3910, 4096, 0, 0, 0, 0, 177, 0, 4096, 0, 0, 4096, 0, 38, 0, 0, 2, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4016, 0, 0, 44, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1921, 0, 0, 0, 0, 4071, 231, 0, 0, 0, 4087, 0, 0, 0, 0, 1, 0, 4096, 3965, 0, 4096, 0, 0, 4096, 0, 4085, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 29, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1152, 0, 4096, 0, 0, 0, 0, 0, 0, 540, 4096, 3, 0, 0, 0, 0, 0, 0, 3663, 4096, 0, 0, 0, 0, 861, 0, 4096, 0, 0, 4089, 0, 0, 8, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4053, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2485, 0, 0, 0, 0, 4092, 270, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4093, 3657, 0, 4095, 0, 0, 4096, 0, 4093, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1797, 0, 4096, 0, 0, 0, 0, 0, 0, 170, 4096, 3, 4, 0, 0, 0, 4, 0, 3460, 4096, 0, 0, 0, 0, 483, 0, 4096, 0, 0, 4091, 0, 2, 4, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4092, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4055, 0, 0, 20, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1155, 0, 0, 0, 0, 4089, 190, 0, 0, 0, 4092, 0, 2, 0, 0, 0, 0, 4096, 3933, 5, 4096, 0, 0, 4096, 0, 4091, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 2608, 0, 4096, 0, 0, 0, 0, 0, 0, 230, 4096, 4, 3, 0, 0, 0, 0, 0, 3673, 4096, 0, 0, 0, 0, 501, 0, 4096, 0, 0, 4089, 0, 17, 6, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4091, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4047, 0, 0, 35, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2785, 0, 0, 0, 0, 4087, 342, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 3849, 0, 4096, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 755, 0, 4096, 0, 0, 0, 0, 0, 0, 378, 4096, 0, 7, 0, 0, 0, 0, 0, 3810, 4096, 0, 0, 0, 0, 376, 0, 4096, 0, 0, 4095, 0, 0, 1, 0, 5, 4096, 0, 0, 4096, 0, 1, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4095, 1, 0, 2, 1, 1, 4094, 0, 1, 1, 0, 1, 1, 0, 4094, 0, 835, 0, 0, 0, 0, 4095, 328, 0, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 4095, 3883, 2, 4095, 0, 0, 4095, 0, 4095, 4095, 0, 0, 0, 4088, 0, 1, 1, 0, 0, 1, 0, 4095, 0, 1, 1, 0, 0, 1, 4093, 4095, 1, 0, 0, 1, 0, 1, 1, 0, 0, 4095, 0, 4095, 4094, 1, 1, 0, 1, 4095, 1, 4095, 2063, 1, 4095, 1, 1, 0, 1, 0, 0, 929, 4095, 0, 1, 1, 1, 1, 6, 0, 3968, 4095, 0, 0, 0, 1, 237, 0, 4095, 1, 1, 4095, 0, 6, 0, 1, 35, 4095, 0, 0, 4095, 0, 4, 4095, 4091, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[3954, 0, 0, 43, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2323, 0, 0, 0, 0, 4066, 383, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3460, 0, 4096, 0, 0, 4096, 0, 4089, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 22, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 832, 0, 4096, 0, 0, 0, 0, 0, 0, 739, 4096, 0, 0, 0, 0, 0, 0, 0, 3071, 4096, 0, 0, 0, 0, 1558, 0, 4096, 0, 0, 4086, 0, 37, 9, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4062, 0, 0, 136, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1455, 0, 0, 0, 0, 4089, 808, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 4065, 0, 4096, 0, 0, 4096, 0, 4090, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 997, 0, 4096, 0, 0, 0, 0, 0, 0, 722, 4096, 0, 3, 0, 0, 0, 0, 0, 3851, 4096, 0, 0, 0, 0, 247, 0, 4096, 0, 0, 4086, 0, 26, 7, 0, 24, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4062, 0, 0, 136, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2171, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1455, 0, 0, 0, 0, 4089, 808, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 1925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4065, 0, 243, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2161, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2162, 1923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 997, 0, 2090, 0, 0, 0, 0, 0, 0, 722, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 4096, 0, 0, 0, 0, 247, 0, 4096, 0, 0, 2006, 1930, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 26, 7, 0, 24, 0, 0, 0, 0, 0, 0, 4074, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16496, 16464, 16591, 16383, 16383, 16100, 16248, 16407] avg=16384.0000 max=16591 max_rank=2 min=16100 min_rank=5 imbalance_ratio=1.012634 max_load_reduction=114481 max_load_reduction_pct=0.873421 moved_to_echo=65138 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4095, 1, 0, 2, 1, 1, 4094, 0, 1, 1, 0, 1, 1, 0, 4094, 0, 2171, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 835, 0, 0, 0, 0, 4095, 328, 0, 1, 0, 4095, 1, 0, 0, 0, 0, 4095, 1924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3883, 2, 242, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 4095, 0, 1, 1, 0, 0, 1, 4093, 4095, 1, 2161, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 4095, 0, 4095, 4094, 1, 1, 0, 1, 2161, 1923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4095, 2063, 1, 2089, 1, 1, 0, 1, 0, 0, 929, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 6, 0, 3968, 4095, 0, 0, 0, 1, 237, 0, 4095, 1, 1, 2006, 1930, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 6, 0, 1, 35, 0, 0, 0, 0, 0, 4, 4074, 4091, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 15374, 16405, 16385, 16374, 17372, 16364, 16401] avg=16384.0000 max=17372 max_rank=5 min=15374 min_rank=1 imbalance_ratio=1.060303 max_load_reduction=113700 max_load_reduction_pct=0.867462 moved_to_echo=66511 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4083, 0, 0, 35, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 2170, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1454, 0, 0, 0, 0, 4096, 506, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4064, 0, 243, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2161, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2162, 1923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1745, 0, 2090, 0, 0, 1, 0, 0, 0, 362, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3910, 4096, 0, 0, 0, 0, 177, 0, 4096, 0, 0, 2006, 1930, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 38, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4074, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16418, 16174, 16594, 16383, 16373, 16491, 16237, 16402] avg=16384.0000 max=16594 max_rank=2 min=16174 min_rank=1 imbalance_ratio=1.012817 max_load_reduction=114478 max_load_reduction_pct=0.873398 moved_to_echo=65503 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4047, 0, 0, 35, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2171, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2785, 0, 0, 0, 0, 4087, 342, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 1925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3849, 0, 243, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2161, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2161, 1923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 755, 0, 2090, 0, 0, 0, 0, 0, 0, 378, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 4096, 0, 0, 0, 0, 376, 0, 4096, 0, 0, 2006, 1930, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 1, 4074, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 17329, 16379, 16384, 16378, 15518, 16336, 16368] avg=16384.0000 max=17329 max_rank=1 min=15518 min_rank=5 imbalance_ratio=1.057678 max_load_reduction=113743 max_load_reduction_pct=0.867790 moved_to_echo=64600 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4016, 0, 0, 44, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2171, 1936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1921, 0, 0, 0, 0, 4071, 231, 0, 0, 0, 4087, 0, 0, 0, 0, 1, 4096, 1925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3965, 0, 232, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 2160, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2161, 1928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1152, 0, 2084, 0, 0, 0, 0, 0, 0, 540, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3663, 4096, 0, 0, 0, 0, 861, 0, 4096, 0, 0, 2012, 1936, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 4073, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16359, 16332, 16474, 16384, 16406, 16067, 16687, 16363] avg=16384.0000 max=16687 max_rank=6 min=16067 min_rank=5 imbalance_ratio=1.018494 max_load_reduction=114385 max_load_reduction_pct=0.872688 moved_to_echo=65523 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4055, 0, 0, 20, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2171, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 0, 0, 0, 0, 4089, 190, 0, 0, 0, 4092, 0, 2, 0, 0, 0, 4096, 1925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3933, 5, 243, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 2161, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2161, 1923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2608, 0, 2090, 0, 0, 0, 0, 0, 0, 230, 4096, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3673, 4096, 0, 0, 0, 0, 501, 0, 4096, 0, 0, 2006, 1930, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 17, 6, 0, 1, 0, 0, 0, 0, 0, 0, 4074, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 15549, 16464, 16385, 16380, 17223, 16324, 16374] avg=16384.0000 max=17223 max_rank=5 min=15549 min_rank=1 imbalance_ratio=1.051208 max_load_reduction=113849 max_load_reduction_pct=0.868599 moved_to_echo=66301 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4053, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2171, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2485, 0, 0, 0, 0, 4092, 270, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 1925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3657, 0, 242, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2161, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2162, 1923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1797, 0, 2090, 0, 0, 0, 0, 0, 0, 170, 4096, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3460, 4096, 0, 0, 0, 0, 483, 0, 4096, 0, 0, 2006, 1930, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16355, 16960, 16181, 16385, 16383, 16352, 16097, 16359] avg=16384.0000 max=16960 max_rank=1 min=16097 min_rank=6 imbalance_ratio=1.035156 max_load_reduction=114112 max_load_reduction_pct=0.870605 moved_to_echo=65191 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3954, 0, 0, 43, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2170, 1935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 0, 0, 0, 0, 4066, 383, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3460, 0, 243, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2161, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2162, 1923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 832, 0, 2090, 0, 0, 0, 0, 0, 0, 739, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3071, 4096, 0, 0, 0, 0, 1558, 0, 4096, 0, 0, 2006, 1930, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 37, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16294, 16890, 15984, 16383, 16395, 15949, 16779, 16398] avg=16384.0000 max=16890 max_rank=1 min=15949 min_rank=5 imbalance_ratio=1.030884 max_load_reduction=114182 max_load_reduction_pct=0.871140 moved_to_echo=65521 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32384, 128, 0, 384, 128, 128, 32768, 0, 128, 128, 128, 128, 128, 0, 32768, 0, 17408, 15488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 0, 128, 128, 32768, 0, 128, 128, 0, 128, 128, 32768, 32768, 128, 17408, 15488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 32768, 12032, 128, 16768, 128, 128, 128, 128, 0, 0, 4096, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 0, 29440, 32768, 0, 0, 0, 128, 4480, 0, 32768, 128, 128, 16128, 15488, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[14464, 0, 0, 0, 0, 32768, 3072, 0, 128, 0, 32768, 128, 128, 0, 0, 128, 32768, 15488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 32768, 30976, 128, 2048, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 128, 32640, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 128, 128, 0, 0, 32768, 0, 32768, 32768, 128, 128, 0, 128, 17408, 15488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4044, 4096, 0, 0, 0, 0, 0, 4096, 0, 39, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4045, 0, 0, 4084, 0, 54, 4, 226, 0, 0, 0, 1220, 0, 0, 3288, 0, 3373, 4096, 10, 0, 4094, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4066, 0, 121, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 1, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3524, 4096, 0, 0, 0, 0, 0, 4096, 0, 35, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4085, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 4051, 0, 0, 4090, 0, 30, 5, 351, 0, 0, 0, 1757, 0, 0, 2638, 0, 3879, 4094, 0, 0, 4093, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 2, 4096, 6, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4091, 0, 75, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 4096, 4086, 4096, 4, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3235, 4096, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4081, 0, 0, 4096, 0, 0, 0, 54, 0, 0, 0, 2341, 0, 0, 2648, 0, 3928, 4096, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 96, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1893, 4096, 0, 0, 0, 0, 0, 4096, 0, 32, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 4057, 0, 0, 4093, 0, 10, 0, 32, 0, 0, 0, 2771, 0, 0, 3555, 0, 3818, 4096, 3, 0, 4094, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 4, 4096, 3, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 0, 198, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 4096, 4096, 2, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3781, 4095, 1, 0, 0, 0, 0, 4093, 0, 5, 1, 1, 4095, 4094, 1, 0, 1, 0, 0, 4095, 1, 1, 0, 1, 4095, 0, 0, 0, 1, 88, 1, 1, 0, 1, 1, 4095, 1, 0, 4095, 0, 3, 0, 1525, 1, 0, 0, 3764, 0, 0, 1726, 0, 1510, 4095, 0, 0, 4095, 4088, 0, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 0, 0, 11, 1, 1, 54, 4094, 0, 4095, 4059, 0, 1, 0, 0, 0, 1, 1, 1, 4095, 0, 0, 4092, 1, 1, 1, 4095, 0, 4095, 0, 28, 0, 17, 0, 1, 3914, 0, 4092, 4094, 0, 4095, 0, 0, 0, 4095, 3, 0, 1, 4095, 4095, 4095, 0, 4095, 5, 0, 1, 1, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3236, 4096, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4087, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 171, 0, 0, 0, 0, 0, 4052, 0, 0, 4096, 0, 17, 2, 120, 0, 0, 0, 2411, 0, 0, 2192, 0, 3725, 4096, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 6, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4082, 0, 450, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2310, 4096, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4032, 0, 0, 4079, 0, 42, 0, 80, 0, 0, 0, 2182, 0, 0, 3376, 0, 3973, 4096, 5, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 385, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3735, 4096, 0, 0, 0, 0, 0, 4096, 0, 49, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 3965, 0, 0, 4096, 0, 104, 0, 580, 0, 0, 0, 1552, 0, 0, 2825, 0, 3223, 4096, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4080, 0, 375, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3524, 4096, 0, 0, 0, 0, 0, 4096, 0, 35, 0, 0, 4096, 452, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4085, 0, 0, 4, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 4051, 0, 0, 4090, 0, 30, 5, 351, 0, 4096, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1757, 0, 0, 2638, 0, 3879, 0, 0, 0, 3831, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 2, 4096, 6, 4096, 3638, 262, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3644, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 75, 0, 0, 0, 0, 4096, 0, 4096, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4086, 0, 4, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16699, 16387, 16381, 16196, 16372, 16388, 16269, 16380] avg=16384.0000 max=16699 max_rank=0 min=16196 min_rank=3 imbalance_ratio=1.019226 max_load_reduction=114373 max_load_reduction_pct=0.872597 moved_to_echo=65409 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3236, 4096, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 4096, 451, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4087, 0, 0, 6, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 0, 0, 0, 4052, 0, 0, 4096, 0, 17, 2, 120, 0, 4096, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2411, 0, 0, 2192, 0, 3725, 0, 0, 0, 3834, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 6, 4096, 3639, 262, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3645, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 450, 0, 0, 0, 0, 4096, 0, 4096, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16395, 16391, 16250, 16246, 16381, 16390, 16635, 16384] avg=16384.0000 max=16635 max_rank=6 min=16246 min_rank=3 imbalance_ratio=1.015320 max_load_reduction=114437 max_load_reduction_pct=0.873085 moved_to_echo=65790 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4044, 4096, 0, 0, 0, 0, 0, 4096, 0, 39, 0, 0, 4096, 452, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 4096, 0, 0, 0, 12, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4045, 0, 0, 4084, 0, 54, 4, 226, 0, 4096, 3695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1220, 0, 0, 3288, 0, 3373, 0, 10, 0, 3830, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4096, 0, 4096, 3640, 264, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3644, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 121, 0, 0, 0, 0, 4096, 0, 4096, 3907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17224, 16381, 16206, 15813, 16396, 16388, 16286, 16378] avg=16384.0000 max=17224 max_rank=0 min=15813 min_rank=3 imbalance_ratio=1.051270 max_load_reduction=113848 max_load_reduction_pct=0.868591 moved_to_echo=65448 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3781, 4095, 1, 0, 0, 0, 0, 4093, 0, 5, 1, 1, 4095, 452, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4094, 1, 0, 1, 0, 0, 4095, 1, 1, 0, 1, 4095, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 88, 1, 1, 0, 1, 1, 4095, 1, 0, 4095, 0, 3, 0, 1525, 1, 4095, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3764, 0, 0, 1726, 0, 1510, 0, 0, 0, 3833, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 0, 0, 11, 1, 1, 54, 4094, 0, 4095, 3639, 262, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 1, 0, 0, 0, 1, 1, 1, 4095, 0, 0, 4092, 1, 1, 1, 3643, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 28, 0, 17, 0, 1, 3914, 0, 4092, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 1, 0, 4095, 0, 0, 4095, 5, 0, 1, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16924, 16394, 17604, 14921, 16430, 16352, 16056, 16391] avg=16384.0000 max=17604 max_rank=2 min=14921 min_rank=3 imbalance_ratio=1.074463 max_load_reduction=113468 max_load_reduction_pct=0.865692 moved_to_echo=65229 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3235, 4096, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 4096, 451, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4081, 0, 0, 4096, 0, 0, 0, 54, 0, 4096, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2341, 0, 0, 2648, 0, 3928, 0, 0, 0, 3834, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3639, 262, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3645, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 96, 0, 0, 0, 0, 4096, 0, 4096, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16374, 16394, 16025, 16846, 16364, 16390, 16295, 16384] avg=16384.0000 max=16846 max_rank=3 min=16025 min_rank=2 imbalance_ratio=1.028198 max_load_reduction=114226 max_load_reduction_pct=0.871475 moved_to_echo=65433 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1893, 4096, 0, 0, 0, 0, 0, 4096, 0, 32, 0, 0, 4096, 452, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 4057, 0, 0, 4093, 0, 10, 0, 32, 0, 4096, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2771, 0, 0, 3555, 0, 3818, 0, 3, 0, 3832, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 4, 4096, 3, 4096, 3640, 262, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3644, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 198, 0, 0, 0, 0, 4096, 0, 4096, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15065, 16386, 16013, 18071, 16374, 16386, 16390, 16387] avg=16384.0000 max=18071 max_rank=3 min=15065 min_rank=0 imbalance_ratio=1.102966 max_load_reduction=113001 max_load_reduction_pct=0.862129 moved_to_echo=65537 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2310, 4096, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 0, 4096, 452, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4032, 0, 0, 4079, 0, 42, 0, 80, 0, 4096, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2182, 0, 0, 3376, 0, 3973, 0, 5, 0, 3834, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4096, 0, 4096, 3640, 262, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3644, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 385, 0, 0, 0, 0, 4096, 0, 4096, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15468, 16387, 16028, 17450, 16383, 16388, 16584, 16384] avg=16384.0000 max=17450 max_rank=3 min=15468 min_rank=0 imbalance_ratio=1.065063 max_load_reduction=113622 max_load_reduction_pct=0.866867 moved_to_echo=65739 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3735, 4096, 0, 0, 0, 0, 0, 4096, 0, 49, 0, 0, 4096, 451, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 3965, 0, 0, 4096, 0, 104, 0, 580, 0, 4096, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1552, 0, 0, 2825, 0, 3223, 0, 0, 0, 3834, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 4096, 3639, 262, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3645, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 375, 0, 0, 0, 0, 4095, 0, 4096, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16923, 16352, 16565, 15529, 16372, 16390, 16557, 16384] avg=16384.0000 max=16923 max_rank=0 min=15529 min_rank=3 imbalance_ratio=1.032898 max_load_reduction=114149 max_load_reduction_pct=0.870888 moved_to_echo=65703 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 128, 128, 128, 32768, 0, 0, 32768, 128, 128, 128, 29184, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 0, 32768, 0, 128, 32768, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 25856, 32768, 128, 0, 0, 0, 0, 32768, 0, 256, 128, 128, 32768, 3712, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 18048, 0, 0, 22272, 0, 27520, 0, 128, 0, 30720, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 0, 128, 0, 0, 32768, 128, 128, 128, 128, 32768, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 32768, 0, 1792, 0, 128, 0, 128, 32640, 0, 32768, 31360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 512, 128, 128, 0, 128, 128, 32384, 128, 0, 32768, 0, 384, 128, 3072, 128, 32768, 29568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 128, 0, 0, 128, 128, 128, 128, 32768, 128, 32768, 29184, 2176, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4032, 9, 0, 0, 0, 94, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4091, 3244, 0, 0, 4096, 0, 0, 4059, 4096, 4096, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 894, 4096, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 30, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4082, 4067, 4096, 4096, 4096, 0, 4081, 0, 0, 4096, 0, 67, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4090, 1, 0, 4095, 1, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 1, 4094, 0, 0, 1, 130, 1, 4095, 4094, 0, 0, 1, 0, 0, 4095, 3504, 0, 0, 4094, 0, 0, 4095, 4095, 4095, 0, 0, 1, 1, 0, 1, 4080, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 4095, 1, 0, 4072, 0, 1, 1, 4095, 1, 0, 0, 0, 0, 507, 4095, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 4095, 0, 0, 4095, 1, 1, 0, 0, 4089, 0, 1, 1, 4095, 1, 0, 9, 4094, 0, 4095, 4095, 4095, 4095, 4095, 0, 4095, 0, 1, 4095, 0, 0, 1, 3, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 3996, 62, 0, 0, 0, 174, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4071, 3313, 0, 0, 4096, 0, 0, 4040, 4096, 4096, 0, 0, 0, 0, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 906, 4094, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4070, 4014, 4096, 4096, 4096, 0, 4084, 0, 0, 4096, 0, 56, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4088, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 4094, 3911, 0, 0, 4096, 0, 0, 4083, 4096, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 201, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4082, 4095, 4096, 4096, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4089, 0, 0, 4096, 0, 0, 3956, 8, 0, 0, 0, 170, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4082, 3380, 0, 0, 4096, 0, 0, 4035, 4096, 4096, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 765, 4095, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 18, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4079, 4095, 4096, 4090, 4096, 0, 4080, 0, 0, 4096, 0, 28, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4048, 4, 0, 0, 0, 290, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 4091, 2944, 0, 0, 4096, 0, 0, 4077, 4096, 4096, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 997, 4092, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 4096, 0, 2, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4088, 4060, 4096, 4096, 4096, 0, 4084, 0, 0, 4096, 0, 53, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4060, 3, 0, 0, 0, 52, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 4091, 3469, 0, 0, 4096, 0, 0, 4076, 4096, 4096, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 686, 4093, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 4096, 0, 5, 4096, 0, 0, 1, 0, 4096, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 4086, 4083, 4096, 4096, 4096, 0, 4090, 0, 0, 4096, 0, 25, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 3961, 76, 0, 0, 0, 113, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4076, 3482, 0, 0, 4096, 0, 0, 3977, 4096, 4096, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 810, 4096, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 76, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4031, 4014, 4096, 4096, 4096, 0, 4035, 0, 0, 4096, 0, 124, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4030, 0, 0, 4032, 9, 0, 0, 0, 94, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4091, 3244, 0, 0, 595, 66, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4059, 4096, 125, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3376, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 894, 4096, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 4096, 3362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 0, 30, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4067, 22, 4096, 0, 0, 4081, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16357, 16207, 16373, 16381, 16560, 16396, 16383, 16415] avg=16384.0000 max=16560 max_rank=4 min=16207 min_rank=1 imbalance_ratio=1.010742 max_load_reduction=114512 max_load_reduction_pct=0.873657 moved_to_echo=65754 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4088, 0, 0, 4030, 0, 0, 4093, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 4094, 3911, 0, 0, 595, 66, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4083, 4096, 125, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3376, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 22, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16334, 16878, 16492, 16376, 15850, 16366, 16383, 16393] avg=16384.0000 max=16878 max_rank=1 min=15850 min_rank=4 imbalance_ratio=1.030151 max_load_reduction=114194 max_load_reduction_pct=0.871231 moved_to_echo=64992 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4089, 0, 0, 4030, 0, 0, 3956, 8, 0, 0, 0, 170, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4082, 3380, 0, 0, 595, 66, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4035, 4096, 125, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3376, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 765, 4095, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 4096, 3362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 0, 18, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4095, 22, 4090, 0, 0, 4080, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16349, 16334, 16417, 16386, 16425, 16384, 16383, 16394] avg=16384.0000 max=16425 max_rank=4 min=16334 min_rank=1 imbalance_ratio=1.002502 max_load_reduction=114647 max_load_reduction_pct=0.874687 moved_to_echo=65586 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4090, 1, 0, 4029, 1, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 130, 1, 4095, 4094, 0, 0, 1, 0, 0, 4095, 3504, 0, 0, 595, 66, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4095, 4095, 125, 0, 0, 1, 1, 0, 1, 4080, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 4095, 1, 0, 4072, 0, 1, 1, 4095, 3375, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 507, 4095, 1, 1, 0, 0, 1, 0, 0, 0, 1, 4095, 4095, 3361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4077, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 0, 4089, 0, 1, 1, 4095, 1, 0, 9, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 22, 4095, 0, 0, 4095, 0, 1, 0, 0, 0, 1, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16312, 16601, 16494, 16357, 16158, 16356, 16386, 16408] avg=16384.0000 max=16601 max_rank=1 min=16158 min_rank=4 imbalance_ratio=1.013245 max_load_reduction=114471 max_load_reduction_pct=0.873344 moved_to_echo=65308 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4030, 0, 0, 4048, 4, 0, 0, 0, 290, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 4091, 2944, 0, 0, 595, 66, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 4096, 125, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 4096, 3376, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 997, 4092, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 4096, 3362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4091, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4060, 22, 4096, 0, 0, 4084, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16564, 15910, 16385, 16389, 16652, 16383, 16386, 16403] avg=16384.0000 max=16652 max_rank=4 min=15910 min_rank=1 imbalance_ratio=1.016357 max_load_reduction=114420 max_load_reduction_pct=0.872955 moved_to_echo=65824 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4023, 0, 0, 3996, 62, 0, 0, 0, 174, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4071, 3313, 0, 0, 600, 73, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4040, 4096, 117, 0, 0, 0, 0, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 4096, 3379, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 906, 4094, 0, 0, 0, 0, 0, 25, 0, 0, 0, 4096, 4096, 3364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 4096, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 4014, 15, 4096, 0, 0, 4084, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16447, 16281, 16268, 16412, 16581, 16372, 16376, 16335] avg=16384.0000 max=16581 max_rank=4 min=16268 min_rank=2 imbalance_ratio=1.012024 max_load_reduction=114491 max_load_reduction_pct=0.873497 moved_to_echo=65664 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4030, 0, 0, 4060, 3, 0, 0, 0, 52, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 4091, 3469, 0, 0, 595, 66, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 4096, 125, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3376, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 686, 4093, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 4096, 3362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4093, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4095, 0, 0, 1, 0, 4096, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4083, 22, 4096, 0, 0, 4090, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16337, 16431, 16414, 16382, 16341, 16373, 16392, 16402] avg=16384.0000 max=16431 max_rank=1 min=16337 min_rank=0 imbalance_ratio=1.002869 max_load_reduction=114641 max_load_reduction_pct=0.874641 moved_to_echo=65508 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4030, 0, 0, 3961, 76, 0, 0, 0, 113, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4076, 3482, 0, 0, 595, 66, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3977, 4096, 125, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3376, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 810, 4096, 0, 0, 0, 0, 0, 45, 0, 0, 0, 4096, 4096, 3362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 0, 76, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 4014, 22, 4096, 0, 0, 4035, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 16430, 16229, 16389, 16505, 16442, 16383, 16322] avg=16384.0000 max=16505 max_rank=4 min=16229 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=114567 max_load_reduction_pct=0.874077 moved_to_echo=65652 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 32768, 128, 128, 128, 32768, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32640, 32768, 0, 0, 0, 0, 256, 0, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 0, 0, 0, 5888, 32768, 128, 128, 0, 0, 128, 128, 0, 0, 128, 32768, 32768, 27008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 0, 32256, 128, 0, 32256, 256, 0, 0, 128, 1024, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 256, 128, 32768, 32768, 0, 0, 128, 0, 0, 32768, 27264, 0, 0, 4864, 640, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 128, 32768, 128, 128, 32768, 0, 128, 128, 32768, 27136, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 0, 32512, 32768, 1024, 0, 0, 128, 128, 0, 128, 32128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32640, 32512, 256, 32768, 0, 0, 32768, 0, 128, 0, 128, 384, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2509, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 23, 27, 4062, 4096, 0, 6, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 326, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 4096, 4096, 0, 4093, 3730, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 4096, 4091, 1632, 4096, 30, 4089, 0, 0, 0, 0, 4035, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 1988, 2, 0, 1, 0, 1, 0, 4, 4092, 1, 1, 0, 4092, 0, 0, 4095, 4095, 13, 0, 0, 1, 0, 1, 4031, 4090, 0, 0, 4095, 1, 12, 1, 1, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 0, 0, 4095, 4095, 1, 11, 1, 4095, 0, 1, 0, 0, 4095, 1, 1, 4094, 1229, 0, 4028, 0, 2, 0, 1, 1, 0, 1, 4092, 4093, 0, 4095, 2969, 0, 4, 1, 1, 1, 0, 4095, 1, 4095, 0, 0, 1, 42, 1, 4038, 4095, 1405, 4095, 2, 4094, 0, 0, 0, 0, 4095, 690, 0, 1, 0, 0, 4093, 1, 0, 0, 1, 1, 4095, 1, 4095, 1, 0, 0, 0, 0, 4, 4095, 28, 4095, 0, 0, 0, 4086] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2441, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 166, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4094, 4056, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 4089, 4096, 1500, 4096, 39, 4094, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3009, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 50, 47, 4037, 4096, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 113, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4092, 3692, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4094, 4091, 1384, 4096, 27, 4083, 0, 0, 0, 0, 4029, 13, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3026, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 34, 12, 4074, 4096, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 47, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 4094, 4096, 0, 4095, 3785, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4091, 4092, 1263, 4096, 46, 4088, 0, 0, 0, 0, 4050, 53, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2290, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 45, 23, 4066, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 8, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 4096, 4096, 0, 4092, 4085, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4094, 4096, 1724, 4096, 81, 4083, 0, 0, 0, 0, 4062, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2128, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 26, 16, 4070, 4096, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 67, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 4092, 4066, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4092, 4089, 1891, 4096, 66, 4085, 0, 0, 0, 0, 4064, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2559, 0, 0, 0, 29, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 33, 83, 3985, 4096, 0, 11, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 291, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 3952, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 0, 1, 0, 4095, 4096, 1251, 4096, 213, 4065, 0, 0, 0, 0, 3986, 21, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2509, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 23, 27, 4062, 4096, 2503, 1573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3820, 0, 6, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 326, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 4096, 0, 4093, 3730, 0, 0, 0, 0, 276, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2544, 0, 0, 0, 0, 0, 4096, 4091, 1632, 0, 30, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 2523, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16399, 16394, 16385, 16439, 16323, 16482, 16269, 16381] avg=16384.0000 max=16482 max_rank=5 min=16269 min_rank=6 imbalance_ratio=1.005981 max_load_reduction=114590 max_load_reduction_pct=0.874252 moved_to_echo=65455 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 1988, 2, 0, 1, 0, 1, 0, 4, 4092, 1, 1, 0, 4095, 4095, 1592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 7, 0, 13, 0, 0, 1, 0, 1, 4031, 4090, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 1, 1, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 0, 0, 4095, 4095, 2503, 1572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3819, 1, 11, 1, 0, 0, 1, 0, 0, 4095, 1, 1, 4094, 1229, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 0, 1, 4092, 4093, 0, 4095, 2969, 0, 4, 1, 1, 276, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 2545, 0, 0, 1, 42, 1, 4038, 4095, 1405, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 690, 0, 1, 0, 0, 4093, 1, 0, 0, 1, 1, 4088, 2523, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 4095, 1, 0, 0, 0, 0, 4, 4095, 28, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15874, 16330, 16378, 17281, 15564, 16225, 17018, 16402] avg=16384.0000 max=17281 max_rank=3 min=15564 min_rank=4 imbalance_ratio=1.054749 max_load_reduction=113791 max_load_reduction_pct=0.868156 moved_to_echo=65209 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2441, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 4096, 2503, 1573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3820, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 166, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4094, 4056, 0, 0, 0, 0, 276, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2545, 0, 0, 0, 1, 0, 4089, 4096, 1500, 0, 39, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 2523, 1526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16322, 16392, 16366, 16275, 16646, 16364, 16326, 16381] avg=16384.0000 max=16646 max_rank=4 min=16275 min_rank=3 imbalance_ratio=1.015991 max_load_reduction=114426 max_load_reduction_pct=0.873001 moved_to_echo=65717 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2128, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 26, 16, 4070, 4096, 2503, 1573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3820, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 67, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 4092, 4066, 0, 0, 0, 0, 276, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2545, 0, 0, 0, 0, 0, 4092, 4089, 1891, 0, 66, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 2523, 1526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16018, 16392, 16383, 16176, 16650, 16768, 16302, 16383] avg=16384.0000 max=16768 max_rank=5 min=16018 min_rank=0 imbalance_ratio=1.023438 max_load_reduction=114304 max_load_reduction_pct=0.872070 moved_to_echo=66103 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3026, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 34, 12, 4074, 4096, 2503, 1573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3820, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 47, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4094, 4096, 0, 4095, 3785, 0, 0, 0, 0, 276, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2545, 0, 0, 0, 0, 0, 4091, 4092, 1263, 0, 46, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 53, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 2523, 1526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16910, 16394, 16387, 16156, 16380, 16125, 16336, 16384] avg=16384.0000 max=16910 max_rank=0 min=16125 min_rank=5 imbalance_ratio=1.032104 max_load_reduction=114162 max_load_reduction_pct=0.870987 moved_to_echo=65225 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3009, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 50, 47, 4037, 4096, 2502, 1574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3817, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 113, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4092, 3692, 0, 0, 0, 0, 279, 28, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2539, 0, 0, 0, 0, 0, 4094, 4091, 1384, 0, 27, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 13, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 2522, 1529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16895, 16385, 16402, 16219, 16290, 16218, 16284, 16379] avg=16384.0000 max=16895 max_rank=0 min=16218 min_rank=5 imbalance_ratio=1.031189 max_load_reduction=114177 max_load_reduction_pct=0.871101 moved_to_echo=65171 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2290, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 45, 23, 4066, 4096, 2502, 1574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3820, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 4096, 0, 4092, 4085, 0, 0, 0, 0, 276, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2545, 0, 0, 0, 0, 0, 4094, 4096, 1724, 0, 81, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 2522, 1526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16185, 16392, 16402, 16116, 16676, 16623, 16297, 16381] avg=16384.0000 max=16676 max_rank=4 min=16116 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=114396 max_load_reduction_pct=0.872772 moved_to_echo=65977 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2559, 0, 0, 0, 29, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 33, 83, 3985, 4096, 2503, 1573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3820, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 291, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 3952, 0, 0, 0, 0, 276, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2545, 0, 1, 0, 1, 0, 4095, 4096, 1251, 0, 213, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 21, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 2523, 1526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16469, 16393, 16369, 16410, 16543, 16267, 16240, 16381] avg=16384.0000 max=16543 max_rank=4 min=16240 min_rank=6 imbalance_ratio=1.009705 max_load_reduction=114529 max_load_reduction_pct=0.873787 moved_to_echo=65431 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 19968, 128, 0, 128, 128, 128, 0, 128, 32768, 128, 128, 0, 32768, 32768, 12800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 0, 0, 128, 0, 128, 0, 0, 128, 128, 128, 32768, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 0, 0, 32512, 896, 0, 128, 0, 0, 32768, 128, 0, 0, 128, 128, 32768, 20224, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[30592, 128, 128, 128, 0, 0, 128, 0, 0, 32768, 128, 128, 32768, 2304, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 20480, 0, 128, 128, 128, 128, 32768, 32768, 12160, 0, 512, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 128, 32768, 128, 0, 0, 0, 0, 128, 32768, 128, 0, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 0, 32768, 0, 128, 128, 128, 256, 256, 32512, 32768, 20096, 12672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 32768, 32768, 0, 32768, 30336, 0, 128, 128, 128, 2304, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 2, 0, 8, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 19, 0, 0, 0, 3, 1655, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 58, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3711, 4088, 0, 0, 0, 0, 0, 4096, 4096, 0, 2759, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 16, 0, 4093, 4096, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4074, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 0, 4, 2, 0, 0, 4096, 4096, 4096, 0, 4096, 2, 0, 13, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 6, 527, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 407, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3935, 4038, 0, 0, 0, 0, 0, 4096, 4096, 0, 3380, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 22, 0, 4093, 4096, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4062, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 1, 0, 11, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 11, 0, 0, 0, 4, 726, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 670, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3859, 3949, 0, 0, 0, 0, 0, 4096, 4096, 0, 3092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 12, 0, 4085, 4096, 0, 0, 4096, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 4072, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4095, 0, 0, 0, 56, 1, 0, 4094, 4095, 4071, 1, 4095, 251, 1, 0, 1, 1, 0, 0, 4095, 4068, 0, 14, 0, 0, 4091, 1, 8, 0, 1, 1, 0, 4095, 1, 1, 0, 0, 68, 1, 1, 1, 0, 3221, 0, 4049, 4081, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 7, 1, 1, 0, 118, 1, 4095, 0, 1, 3974, 0, 1, 0, 4093, 2680, 3762, 0, 1, 0, 0, 2, 4094, 4095, 0, 2351, 0, 0, 1, 4089, 0, 0, 1, 0, 1, 4095, 0, 4092, 4073, 0, 1, 77, 0, 4095, 4093, 34, 16, 4095, 0, 1, 2, 5, 4023, 0, 0, 0, 0, 1, 0, 4035, 4095, 0, 1, 1, 1, 2, 15, 4093, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 1, 0, 1, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1585, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 170, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3841, 3966, 0, 0, 0, 0, 0, 4096, 4096, 0, 2715, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 2, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 3, 0, 7, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 4, 2190, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 124, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3725, 4039, 0, 0, 0, 0, 0, 4096, 4096, 0, 2206, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 14, 0, 4086, 4096, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 3, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 3, 1323, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 119, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3773, 4085, 0, 0, 0, 0, 0, 4096, 4096, 0, 3006, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 8, 0, 4092, 4096, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4075, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 65, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 117, 0, 0, 0, 0, 1506, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 68, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2803, 4065, 0, 0, 0, 0, 0, 4096, 4096, 0, 3706, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 45, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 4027, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4, 0, 0, 0, 4096, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 1, 0, 2901, 928, 210, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 19, 0, 0, 0, 3, 1655, 0, 4096, 4096, 2472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3886, 0, 0, 0, 4096, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 4096, 0, 0, 0, 4096, 3711, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2759, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1195, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 16, 0, 4093, 4089, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2500, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16349, 16437, 16232, 16659, 16242, 16381, 16390] avg=16384.0000 max=16659 max_rank=4 min=16232 min_rank=3 imbalance_ratio=1.016785 max_load_reduction=114413 max_load_reduction_pct=0.872902 moved_to_echo=65672 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4095, 0, 0, 2901, 928, 210, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 10, 0, 0, 0, 0, 1585, 0, 4096, 4096, 2472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3886, 0, 0, 0, 4096, 0, 0, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 4096, 0, 0, 0, 4096, 3841, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2715, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1195, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2500, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16338, 16355, 16344, 16667, 16198, 16378, 16412] avg=16384.0000 max=16667 max_rank=4 min=16198 min_rank=5 imbalance_ratio=1.017273 max_load_reduction=114405 max_load_reduction_pct=0.872841 moved_to_echo=65655 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 2, 0, 0, 0, 4096, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 2901, 928, 210, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 4, 2190, 0, 4096, 4096, 2472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3886, 0, 0, 0, 4096, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 4096, 0, 0, 0, 4096, 3725, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2206, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 1195, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 4086, 4089, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2500, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16348, 16957, 16298, 16624, 15693, 16373, 16400] avg=16384.0000 max=16957 max_rank=2 min=15693 min_rank=5 imbalance_ratio=1.034973 max_load_reduction=114115 max_load_reduction_pct=0.870628 moved_to_echo=65090 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4095, 0, 0, 0, 56, 1, 0, 4094, 0, 4071, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 1, 0, 1, 1, 0, 0, 4095, 4068, 0, 14, 0, 0, 4091, 1, 8, 2900, 928, 210, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 1, 1, 0, 0, 68, 1, 1, 1, 0, 3221, 0, 4095, 4095, 2471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 4081, 1, 1, 1, 1, 3885, 0, 0, 0, 4095, 7, 1, 1, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 668, 0, 1, 3974, 0, 1, 0, 4093, 2680, 3762, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2351, 0, 0, 1, 4089, 0, 0, 1, 0, 1, 1195, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 1, 77, 0, 4095, 4086, 34, 16, 0, 0, 1, 2, 5, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4035, 4095, 0, 1, 1, 1, 2, 15, 4093, 1, 0, 2499, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 16580, 18052, 16241, 15183, 15824, 16413, 16368] avg=16384.0000 max=18052 max_rank=2 min=15183 min_rank=4 imbalance_ratio=1.101807 max_load_reduction=113020 max_load_reduction_pct=0.862274 moved_to_echo=63788 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 0, 4, 2, 0, 0, 4096, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 13, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 2903, 932, 213, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 6, 527, 0, 4096, 4096, 2473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3883, 0, 0, 0, 4096, 0, 0, 0, 0, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 0, 0, 4096, 0, 0, 0, 4096, 3935, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3380, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 1193, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 22, 0, 4093, 4088, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2504, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16365, 15297, 16578, 16825, 16862, 16385, 16381] avg=16384.0000 max=16862 max_rank=5 min=15297 min_rank=2 imbalance_ratio=1.029175 max_load_reduction=114210 max_load_reduction_pct=0.871353 moved_to_echo=66453 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 2901, 928, 210, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 11, 0, 0, 0, 4, 726, 0, 4096, 4096, 2473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3886, 0, 0, 0, 4096, 0, 0, 0, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 4096, 0, 0, 0, 4096, 3859, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1195, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 0, 4085, 4089, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2500, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16350, 15502, 16844, 16668, 16575, 16366, 16387] avg=16384.0000 max=16844 max_rank=3 min=15502 min_rank=2 imbalance_ratio=1.028076 max_load_reduction=114228 max_load_reduction_pct=0.871490 moved_to_echo=65996 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 3, 0, 0, 0, 4096, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 2901, 928, 210, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 3, 1323, 0, 4096, 4096, 2472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3886, 0, 0, 0, 4096, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 4096, 0, 0, 0, 4096, 3773, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3006, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1195, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 4092, 4089, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2500, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16339, 16089, 16293, 16718, 16489, 16372, 16391] avg=16384.0000 max=16718 max_rank=4 min=16089 min_rank=2 imbalance_ratio=1.020386 max_load_reduction=114354 max_load_reduction_pct=0.872452 moved_to_echo=65970 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 2901, 928, 210, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 117, 0, 0, 0, 0, 1506, 0, 4096, 4096, 2472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3886, 0, 0, 0, 4096, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 4096, 0, 0, 0, 4096, 2803, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3706, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1195, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 45, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2500, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16403, 16383, 16242, 15728, 17189, 16404, 16343] avg=16384.0000 max=17189 max_rank=5 min=15728 min_rank=4 imbalance_ratio=1.049133 max_load_reduction=113883 max_load_reduction_pct=0.868858 moved_to_echo=65664 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 0, 128, 256, 0, 32768, 32768, 128, 128, 0, 0, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 128, 128, 128, 31104, 0, 0, 0, 32768, 128, 128, 128, 0, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 0, 0, 23296, 0, 128, 128, 32768, 0, 0, 128, 0, 128, 9600, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 128, 128, 128, 0, 32768, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 128, 0, 0, 256, 128, 128, 128, 128, 12800, 0, 32768, 32768, 19840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 5376, 0, 128, 32768, 0, 128, 0, 32768, 28416, 32000, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[384, 128, 128, 128, 128, 0, 0, 32768, 32768, 0, 128, 0, 0, 32768, 128, 128, 23296, 7552, 1792, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 0, 128, 0, 32640, 32768, 0, 128, 128, 128, 128, 128, 32768, 128, 0, 20096, 13056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 1856, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 19, 0, 4096, 0, 0, 0, 4096, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 8, 4096, 0, 94, 0, 0, 0, 0, 0, 0, 0, 2188, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4095, 0, 4096, 0, 0, 4096, 0, 1564, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 4096, 3991, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 9, 4096, 0, 195, 0, 0, 0, 0, 0, 0, 0, 2473, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 1811, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 4096, 0, 1337, 0, 0, 0, 0, 0, 0, 0, 1204, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4094, 4093, 0, 4095, 0, 4075, 1, 0, 4095, 1, 205, 1, 4094, 0, 1, 0, 0, 1, 0, 1, 1, 4095, 4095, 0, 1, 4095, 4085, 4043, 0, 3707, 1, 1, 1, 0, 0, 1, 1, 4095, 15, 20, 0, 3, 1, 0, 4061, 0, 0, 1, 4094, 1, 3976, 59, 0, 0, 4087, 4095, 1, 1, 1, 0, 0, 1, 4071, 9, 1, 2, 4094, 3317, 2, 0, 0, 1, 1, 0, 5, 1, 133, 0, 0, 1, 1, 52, 1, 0, 4093, 4095, 1, 0, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 4080, 4095, 0, 0, 26, 4095, 1, 2847, 0, 0, 0, 1, 1, 0, 1, 2200, 0, 0, 0, 0, 4095, 4090, 4063, 8, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2586, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 4096, 3810, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4094, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 1, 4096, 0, 208, 0, 0, 0, 0, 0, 0, 0, 1588, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 1748, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 480, 0, 0, 0, 0, 0, 0, 0, 1877, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2793, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4095, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 9, 4096, 0, 405, 0, 0, 0, 0, 0, 0, 0, 1664, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4095, 0, 4096, 0, 0, 4096, 0, 2295, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 67, 0, 4096, 0, 0, 0, 4096, 3655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 469, 0, 0, 0, 0, 0, 0, 0, 1813, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2243, 4096, 0, 4095, 0, 4096, 0, 0, 0, 0, 1564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 4096, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 4096, 3991, 0, 2, 0, 0, 0, 0, 0, 0, 4026, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 4092, 0, 0, 0, 0, 0, 0, 4090, 4043, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 0, 4096, 4094, 0, 0, 9, 4096, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2473, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1465, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16094, 16393, 16430, 16395, 16607, 16341, 15828, 16984] avg=16384.0000 max=16984 max_rank=7 min=15828 min_rank=6 imbalance_ratio=1.036621 max_load_reduction=114088 max_load_reduction_pct=0.870422 moved_to_echo=65760 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2234, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 1856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4096, 4096, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4096, 0, 0, 0, 4096, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 4095, 0, 0, 0, 0, 0, 0, 4096, 4043, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3334, 0, 4096, 4096, 0, 0, 8, 4096, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2188, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1468, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 16384, 16427, 16404, 16693, 16356, 15724, 16706] avg=16384.0000 max=16706 max_rank=7 min=15724 min_rank=6 imbalance_ratio=1.019653 max_load_reduction=114366 max_load_reduction_pct=0.872543 moved_to_echo=65479 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2243, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 1811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4090, 4044, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 0, 4096, 4096, 0, 0, 1, 4096, 0, 1337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1204, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1465, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16342, 16394, 16436, 16389, 16445, 16387, 16964, 15715] avg=16384.0000 max=16964 max_rank=6 min=15715 min_rank=7 imbalance_ratio=1.035400 max_load_reduction=114108 max_load_reduction_pct=0.870575 moved_to_echo=65511 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2242, 4093, 0, 4095, 0, 4075, 1, 0, 0, 1, 205, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 1, 0, 1, 1, 9, 0, 0, 1, 4095, 4085, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3707, 1, 1, 1, 0, 0, 1, 1, 4095, 15, 20, 0, 3, 1, 0, 4095, 4095, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 1, 4094, 1, 3976, 59, 0, 0, 4087, 6, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4071, 9, 1, 2, 4094, 3317, 2, 0, 0, 1, 1, 0, 5, 1, 4025, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 1, 1, 52, 1, 0, 4093, 4095, 1, 0, 0, 0, 0, 1, 4089, 4042, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3337, 0, 4080, 4095, 0, 0, 26, 4095, 1, 2847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 2200, 0, 0, 0, 0, 4095, 4090, 4063, 8, 0, 0, 0, 1464, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14713, 16331, 16089, 16288, 15918, 16570, 18482, 16681] avg=16384.0000 max=18482 max_rank=6 min=14713 min_rank=0 imbalance_ratio=1.128052 max_load_reduction=112590 max_load_reduction_pct=0.858994 moved_to_echo=67651 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2243, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 2793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 4096, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 4096, 0, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3345, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4095, 0, 0, 0, 0, 2, 0, 4090, 4043, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 0, 4096, 4096, 0, 0, 9, 4096, 0, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1664, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1465, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17324, 16393, 16414, 16398, 15951, 16377, 16040, 16175] avg=16384.0000 max=17324 max_rank=0 min=15951 min_rank=4 imbalance_ratio=1.057373 max_load_reduction=113748 max_load_reduction_pct=0.867828 moved_to_echo=64543 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2243, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 2586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 4096, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 4096, 0, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 4096, 3810, 0, 2, 0, 0, 0, 0, 0, 0, 4026, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4094, 0, 0, 0, 0, 1, 0, 4090, 4043, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 0, 4096, 4094, 0, 0, 1, 4096, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1588, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1465, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17117, 16393, 16430, 16396, 16426, 16378, 15833, 16099] avg=16384.0000 max=17117 max_rank=0 min=15833 min_rank=6 imbalance_ratio=1.044739 max_load_reduction=113955 max_load_reduction_pct=0.869408 moved_to_echo=64736 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2243, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 1748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 4096, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 0, 0, 4091, 4043, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 0, 4096, 4096, 0, 0, 0, 4096, 0, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1877, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1465, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16279, 16391, 16433, 16394, 16704, 16377, 16106, 16388] avg=16384.0000 max=16704 max_rank=4 min=16106 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=114368 max_load_reduction_pct=0.872559 moved_to_echo=65575 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2243, 4096, 0, 4095, 0, 4096, 0, 0, 0, 0, 2295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 4096, 4096, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 4096, 0, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 4096, 0, 0, 0, 4096, 3655, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 4096, 0, 0, 0, 0, 0, 0, 4090, 4044, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 0, 4096, 4096, 0, 0, 0, 4096, 0, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1813, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1465, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16825, 16393, 16413, 16408, 16328, 16286, 16095, 16324] avg=16384.0000 max=16825 max_rank=0 min=16095 min_rank=6 imbalance_ratio=1.026917 max_load_reduction=114247 max_load_reduction_pct=0.871635 moved_to_echo=65033 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 0, 128, 0, 128, 128, 0, 128, 128, 128, 0, 0, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 0, 0, 128, 128, 128, 128, 0, 32640, 32768, 128, 0, 0, 0, 128, 128, 32768, 32384, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 18048, 32768, 0, 32768, 0, 32768, 128, 0, 0, 128, 14976, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 128, 26752, 0, 32768, 32768, 0, 0, 128, 32768, 128, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32384, 128, 128, 128, 0, 0, 128, 128, 32768, 128, 128, 0, 128, 128, 0, 32768, 32768, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 32768, 30208, 128, 128, 0, 128, 128, 0, 128, 128, 32256, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 128, 128, 32768, 128, 32768, 128, 0, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 15104, 0, 0, 0, 0, 32768, 32768, 32768, 128, 0, 0, 0, 11776, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4068, 0, 0, 188, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1197, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 54, 4096, 4089, 0, 2636, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4092, 4096, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 4096, 74, 0, 0, 4096, 0, 4093, 1, 0, 0, 0, 4096, 4071, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1241, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1462, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 80, 4096, 4096, 0, 1274, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 21, 0, 0, 4096, 0, 4096, 21, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 1, 4083, 1, 0, 0, 0, 0, 4094, 1, 1, 5, 2, 1, 14, 2, 4090, 1, 4095, 0, 0, 0, 0, 4095, 0, 1, 4095, 1, 1, 1500, 4083, 1, 1, 1, 0, 0, 4094, 0, 17, 0, 712, 4062, 3, 0, 4093, 1, 0, 5, 0, 0, 0, 0, 0, 4091, 1, 4095, 4094, 0, 1, 0, 797, 4095, 4095, 0, 1082, 0, 0, 0, 0, 1, 0, 1, 1, 4092, 1, 1, 1, 2, 4095, 4095, 4, 5, 4095, 14, 4095, 1, 0, 4095, 4095, 4089, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 4079, 0, 0, 20, 3945, 0, 4095, 15, 0, 0, 1, 4095, 4093, 0, 1, 0, 0, 0, 1, 7, 0, 2, 1, 4086, 153, 4090, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4052, 0, 0, 2230, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 295, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 6, 4096, 4086, 0, 1466, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4087, 4096, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 4096, 151, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4081, 0, 0, 496, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 916, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 27, 4096, 4094, 0, 2501, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 198, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 0, 502, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 601, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 36, 4096, 4092, 0, 2912, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4092, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4092, 4096, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 4096, 89, 0, 0, 4096, 0, 4094, 2, 0, 0, 0, 4096, 4061, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4084, 0, 0, 386, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1954, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 39, 4096, 4091, 0, 1709, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4090, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4089, 4096, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 4096, 45, 0, 0, 4096, 0, 4096, 18, 0, 0, 0, 4096, 4050, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4013, 0, 0, 1534, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 800, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 9, 4096, 4093, 0, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4092, 4096, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 4096, 232, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3113, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 188, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1197, 4096, 0, 0, 4096, 0, 0, 0, 3079, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 54, 3970, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2636, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4095, 4096, 1016, 983, 126, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 74, 0, 0, 4096, 0, 4019, 1, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4070, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 15561, 16590, 16308, 17118, 16381, 16361, 16368] avg=16384.0000 max=17118 max_rank=4 min=15561 min_rank=1 imbalance_ratio=1.044800 max_load_reduction=113954 max_load_reduction_pct=0.869400 moved_to_echo=66228 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3113, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1241, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1462, 4096, 0, 0, 4096, 0, 0, 0, 3079, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 80, 3970, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1274, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 1014, 983, 126, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 0, 4096, 0, 4022, 21, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4071, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16642, 16854, 16338, 15759, 16388, 16348, 16362] avg=16384.0000 max=16854 max_rank=2 min=15759 min_rank=4 imbalance_ratio=1.028687 max_load_reduction=114218 max_load_reduction_pct=0.871414 moved_to_echo=64857 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4083, 1, 0, 0, 0, 0, 4094, 1, 1, 5, 2, 1, 14, 2, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 3112, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 1500, 4083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4094, 0, 17, 0, 712, 4062, 3, 0, 4093, 1, 0, 5, 3078, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 4094, 0, 1, 0, 797, 3969, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 0, 0, 0, 0, 1, 0, 1, 1, 4092, 1, 1, 1, 2, 4095, 4095, 1017, 983, 126, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 4095, 14, 0, 1, 0, 0, 4095, 4089, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4079, 0, 0, 20, 3945, 0, 4021, 15, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 7, 0, 2, 1, 4086, 153, 4090, 0, 1, 0, 4095, 4069, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 16885, 16093, 17048, 15572, 16398, 16175, 16509] avg=16384.0000 max=17048 max_rank=3 min=15572 min_rank=4 imbalance_ratio=1.040527 max_load_reduction=114024 max_load_reduction_pct=0.869934 moved_to_echo=64654 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3113, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 386, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1954, 4096, 0, 0, 4096, 0, 0, 0, 3079, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 39, 3970, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1709, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4090, 4096, 1014, 983, 126, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 45, 0, 0, 4096, 0, 4022, 18, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4071, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 15775, 17346, 16292, 16186, 16380, 16334, 16371] avg=16384.0000 max=17346 max_rank=2 min=15775 min_rank=1 imbalance_ratio=1.058716 max_load_reduction=113726 max_load_reduction_pct=0.867661 moved_to_echo=65271 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3113, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 502, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 601, 4096, 0, 0, 4096, 0, 0, 0, 3079, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 36, 3970, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2912, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4092, 4096, 1014, 983, 126, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 89, 0, 0, 4096, 0, 4020, 2, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4070, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 15894, 15994, 16292, 17390, 16383, 16368, 16367] avg=16384.0000 max=17390 max_rank=4 min=15894 min_rank=1 imbalance_ratio=1.061401 max_load_reduction=113682 max_load_reduction_pct=0.867325 moved_to_echo=66508 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3107, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 2230, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 295, 4096, 0, 0, 4096, 0, 0, 0, 3082, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 3963, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1466, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 1014, 989, 133, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4096, 151, 0, 0, 4096, 0, 4017, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4070, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 17581, 15691, 16247, 15969, 16375, 16457, 16371] avg=16384.0000 max=17581 max_rank=1 min=15691 min_rank=2 imbalance_ratio=1.073059 max_load_reduction=113491 max_load_reduction_pct=0.865868 moved_to_echo=65172 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3113, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 496, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 916, 4096, 0, 0, 4096, 0, 0, 0, 3082, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 27, 3970, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2501, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 1014, 983, 126, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 198, 0, 0, 4096, 0, 4022, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4070, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 15882, 16312, 16283, 16986, 16384, 16483, 16361] avg=16384.0000 max=16986 max_rank=4 min=15882 min_rank=1 imbalance_ratio=1.036743 max_load_reduction=114086 max_load_reduction_pct=0.870407 moved_to_echo=66214 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3113, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 1534, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 800, 4096, 0, 0, 4096, 0, 0, 0, 3079, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 9, 3970, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 4096, 1014, 983, 126, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4096, 232, 0, 0, 4096, 0, 4022, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4071, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16852, 16192, 16264, 16092, 16383, 16546, 16363] avg=16384.0000 max=16852 max_rank=1 min=16092 min_rank=4 imbalance_ratio=1.028564 max_load_reduction=114220 max_load_reduction_pct=0.871429 moved_to_echo=65384 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 128, 0, 32768, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 0, 0, 32768, 32768, 0, 0, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 24960, 0, 0, 0, 0, 0, 0, 128, 32640, 128, 128, 8192, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 896, 0, 128, 32640, 0, 32256, 128, 0, 0, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[15232, 0, 0, 0, 0, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 32768, 32768, 8192, 7936, 1024, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 32768, 128, 0, 32768, 0, 128, 0, 1152, 31872, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 0, 128, 128, 128, 0, 128, 128, 32768, 256, 32768, 0, 128, 0, 32768, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 0, 32768, 0, 128, 0, 8064, 32768, 128, 0, 32768, 128, 0, 128, 24704, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 4063, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4096, 0, 0, 4096, 4096, 0, 4096, 13, 4096, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4076, 0, 0, 4072, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4085, 0, 0, 5, 0, 0, 0, 0, 0, 0, 27, 4092, 0, 0, 4096, 4096, 0, 4096, 17, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4092, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 4035, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 4093, 0, 0, 4096, 4096, 0, 4093, 19, 4096, 4096, 0, 4096, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4095, 0, 0, 1, 7, 0, 4091, 4041, 0, 0, 4095, 4095, 0, 4092, 0, 4094, 0, 1, 1, 0, 38, 4095, 0, 0, 1, 1, 0, 0, 0, 1, 4094, 4095, 4085, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 2, 0, 4, 4094, 2, 0, 1, 1, 1, 0, 0, 1, 4095, 1, 0, 0, 4060, 0, 0, 70, 0, 1, 4, 1, 1, 1, 0, 4095, 1, 1, 4044, 4091, 1, 4095, 0, 4089, 4092, 28, 4095, 0, 0, 36, 1, 1, 0, 4094, 0, 0, 0, 0, 1, 0, 1, 0, 3, 0, 4095, 4095, 4061, 1, 4095, 0, 4093, 0, 4095, 4095, 0, 1, 0, 1, 4095, 12, 0, 4095, 0, 1, 0, 0, 2, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4069, 0, 0, 3767, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 4096, 0, 0, 4096, 4096, 0, 4096, 44, 4096, 4096, 0, 4096, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4088, 4096, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4084, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 8, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 4096, 4096, 0, 4096, 2, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 4000, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4083, 0, 0, 7, 0, 0, 0, 0, 0, 0, 89, 4096, 0, 0, 4096, 4096, 0, 4096, 15, 4096, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4067, 0, 0, 3920, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4067, 0, 0, 2, 0, 0, 0, 0, 0, 0, 175, 4096, 0, 0, 4096, 4096, 0, 4096, 50, 4096, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 4035, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 3966, 93, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4083, 0, 4089, 19, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 4093, 0, 0, 4096, 4096, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 4096, 0, 4067, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16424, 16379, 16385, 16390, 16344, 16377, 16388, 16385] avg=16384.0000 max=16424 max_rank=0 min=16344 min_rank=4 imbalance_ratio=1.002441 max_load_reduction=114648 max_load_reduction_pct=0.874695 moved_to_echo=65494 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 111, 0, 0, 1, 7, 0, 4091, 4041, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4094, 0, 1, 1, 0, 38, 4095, 0, 0, 1, 1, 0, 0, 3965, 93, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 4095, 4085, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 2, 0, 1, 1, 1, 0, 0, 1, 4095, 1, 0, 0, 4060, 0, 4088, 19, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 1, 4, 1, 1, 1, 0, 4095, 1, 1, 4044, 4091, 1, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4092, 28, 4067, 0, 0, 36, 1, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 3, 0, 0, 0, 4061, 1, 7, 0, 4093, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 12, 0, 4095, 0, 1, 0, 0, 2, 0, 1, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16443, 16409, 16377, 16372, 16313, 16408, 16357, 16393] avg=16384.0000 max=16443 max_rank=0 min=16313 min_rank=4 imbalance_ratio=1.003601 max_load_reduction=114629 max_load_reduction_pct=0.874550 moved_to_echo=65471 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 4096, 4076, 0, 0, 4072, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 3966, 93, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4085, 0, 4089, 19, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 27, 4092, 0, 0, 4096, 4096, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 4096, 0, 4067, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 4096, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16451, 16379, 16385, 16390, 16319, 16376, 16387, 16385] avg=16384.0000 max=16451 max_rank=0 min=16319 min_rank=4 imbalance_ratio=1.004089 max_load_reduction=114621 max_load_reduction_pct=0.874489 moved_to_echo=65467 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 4063, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3966, 93, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4074, 0, 4089, 19, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4096, 0, 0, 4096, 4096, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 4096, 0, 4067, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16459, 16375, 16385, 16379, 16330, 16369, 16391, 16384] avg=16384.0000 max=16459 max_rank=0 min=16330 min_rank=4 imbalance_ratio=1.004578 max_load_reduction=114613 max_load_reduction_pct=0.874428 moved_to_echo=65474 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 4000, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 3966, 93, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4083, 0, 4089, 19, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 89, 4096, 0, 0, 4096, 4096, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 4096, 0, 4067, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16381, 16385, 16388, 16387, 16372, 16389, 16380] avg=16384.0000 max=16390 max_rank=0 min=16372 min_rank=5 imbalance_ratio=1.000366 max_load_reduction=114682 max_load_reduction_pct=0.874954 moved_to_echo=65528 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 4096, 4069, 0, 0, 3767, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 3967, 97, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4067, 0, 4094, 24, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 4096, 0, 0, 4096, 4096, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 4096, 4096, 0, 4062, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16133, 16399, 16386, 16383, 16607, 16407, 16378, 16379] avg=16384.0000 max=16607 max_rank=4 min=16133 min_rank=0 imbalance_ratio=1.013611 max_load_reduction=114465 max_load_reduction_pct=0.873299 moved_to_echo=65771 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4084, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3966, 93, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 4089, 19, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 4096, 4096, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 4067, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16482, 16375, 16385, 16398, 16304, 16357, 16391, 16380] avg=16384.0000 max=16482 max_rank=0 min=16304 min_rank=4 imbalance_ratio=1.005981 max_load_reduction=114590 max_load_reduction_pct=0.874252 moved_to_echo=65432 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 4096, 4067, 0, 0, 3920, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3966, 93, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4067, 0, 4089, 19, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 175, 4096, 0, 0, 4096, 4096, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 4096, 4096, 0, 4067, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16290, 16375, 16384, 16372, 16468, 16406, 16391, 16386] avg=16384.0000 max=16468 max_rank=4 min=16290 min_rank=0 imbalance_ratio=1.005127 max_load_reduction=114604 max_load_reduction_pct=0.874359 moved_to_echo=65651 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 896, 0, 0, 128, 128, 0, 32768, 32640, 0, 0, 32128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 0, 32768, 0, 128, 0, 0, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[256, 32768, 32768, 128, 32640, 0, 128, 128, 128, 128, 0, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 32768, 32768, 32768, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 0, 32768, 0, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 0, 0, 31744, 768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 128, 0, 128, 128, 32768, 128, 0, 0, 32640, 0, 32768, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 32768, 128, 128, 0, 32768, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 128, 768, 32768, 128, 128, 32768, 32768, 128, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 4096, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 7, 4096, 4090, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 15, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 31, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 4083, 4096, 0, 4093, 0, 4054, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 14, 4096, 4093, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4039, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 31, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4095, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 4096, 2, 0, 0, 0, 4096, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 4095, 0, 4088, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 7, 4096, 4094, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4079, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 12, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 69, 0, 4095, 81, 1, 0, 0, 4095, 4093, 1, 0, 0, 4095, 0, 0, 1, 2, 8, 1, 4095, 4095, 0, 4092, 6, 4095, 1, 1, 1, 1, 0, 1, 0, 1, 0, 10, 0, 1, 1, 1, 1, 0, 1, 0, 5, 4094, 4086, 3, 4095, 4094, 4095, 0, 0, 0, 3922, 0, 0, 3, 1, 3901, 1, 0, 0, 4087, 2, 1, 0, 0, 1, 1, 4086, 4092, 4073, 0, 4038, 0, 37, 1, 0, 1, 4094, 60, 1, 0, 0, 4093, 1, 1, 4095, 4095, 0, 1, 4095, 0, 0, 1, 4094, 1, 1, 1, 0, 10, 1, 4095, 4094, 0, 0, 4095, 0, 0, 1, 199, 1, 1, 0, 4095, 4095, 4083] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4082, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 10, 4096, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 4096, 0, 4076, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 9, 4096, 4088, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4081, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 23, 0, 0, 0, 4096, 2, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 4073, 4096, 0, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 36, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4056, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 30, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 4096, 0, 0, 4096, 4096, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4076, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 7, 0, 4090, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 21, 20, 15, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 4096, 0, 4096, 0, 15, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16362, 16385, 16375, 16378, 16444, 16385, 16383, 16360] avg=16384.0000 max=16444 max_rank=4 min=16360 min_rank=7 imbalance_ratio=1.003662 max_load_reduction=114628 max_load_reduction_pct=0.874542 moved_to_echo=65572 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 4096, 4096, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4075, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 21, 21, 15, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 4096, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16361, 16387, 16386, 16377, 16445, 16369, 16383, 16364] avg=16384.0000 max=16445 max_rank=4 min=16361 min_rank=0 imbalance_ratio=1.003723 max_load_reduction=114627 max_load_reduction_pct=0.874535 moved_to_echo=65561 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 10, 3, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4075, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 9, 0, 4088, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4081, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 21, 21, 15, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 4096, 0, 4096, 0, 23, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16357, 16389, 16379, 16378, 16431, 16394, 16383, 16361] avg=16384.0000 max=16431 max_rank=4 min=16357 min_rank=0 imbalance_ratio=1.002869 max_load_reduction=114641 max_load_reduction_pct=0.874641 moved_to_echo=65569 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 31, 0, 4096, 0, 0, 4096, 4096, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4054, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4076, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 14, 0, 4093, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4039, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 28, 20, 21, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4075, 4096, 0, 4096, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 16375, 16383, 16387, 16409, 16394, 16379, 16353] avg=16384.0000 max=16409 max_rank=4 min=16353 min_rank=7 imbalance_ratio=1.001526 max_load_reduction=114663 max_load_reduction_pct=0.874809 moved_to_echo=65535 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 69, 0, 4095, 81, 1, 4095, 4095, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4093, 1, 0, 0, 3, 0, 0, 1, 2, 8, 1, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 6, 4095, 1, 1, 1, 1, 0, 1, 0, 1, 0, 10, 0, 1, 4094, 4073, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 5, 4094, 4086, 3, 0, 4094, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3922, 0, 0, 3, 1, 3901, 1, 0, 0, 4087, 2, 1, 0, 0, 1, 1, 4092, 21, 20, 15, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4077, 4073, 0, 4038, 0, 37, 1, 0, 1, 0, 60, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4094, 0, 1, 4095, 0, 0, 1, 4094, 1, 1, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4073, 0, 0, 4095, 0, 0, 1, 199, 1, 1, 0, 0, 4095, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16506, 16394, 16404, 16375, 16075, 16374, 16395, 16549] avg=16384.0000 max=16549 max_rank=7 min=16075 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=114523 max_load_reduction_pct=0.873741 moved_to_echo=65393 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 4096, 2, 0, 4096, 4096, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4088, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4076, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 7, 0, 4094, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4079, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 21, 20, 15, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 4096, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16370, 16384, 16385, 16382, 16428, 16381, 16383, 16359] avg=16384.0000 max=16428 max_rank=4 min=16359 min_rank=7 imbalance_ratio=1.002686 max_load_reduction=114644 max_load_reduction_pct=0.874664 moved_to_echo=65551 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 4096, 4096, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4076, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 21, 20, 15, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 4096, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16363, 16391, 16380, 16382, 16434, 16376, 16383, 16363] avg=16384.0000 max=16434 max_rank=4 min=16363 min_rank=0 imbalance_ratio=1.003052 max_load_reduction=114638 max_load_reduction_pct=0.874619 moved_to_echo=65556 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 4096, 0, 0, 4096, 4096, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 4073, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4075, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 36, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4056, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 21, 21, 15, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 4096, 0, 4096, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16361, 16367, 16380, 16413, 16406, 16399, 16383, 16363] avg=16384.0000 max=16413 max_rank=3 min=16361 min_rank=0 imbalance_ratio=1.001770 max_load_reduction=114659 max_load_reduction_pct=0.874779 moved_to_echo=65551 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32640, 0, 0, 128, 128, 32512, 128, 0, 0, 32768, 128, 128, 0, 0, 128, 128, 32768, 256, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 32768, 0, 0, 128, 32768, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32768, 128, 32768, 128, 128, 128, 128, 0, 128, 0, 128, 0, 128, 0, 128, 32768, 32640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 32768, 32768, 128, 0, 128, 128, 0, 0, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 32640, 0, 0, 32768, 0, 0, 128, 256, 128, 128, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 32640, 32768, 0, 32768, 0, 256, 128, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 128, 0, 0, 0, 0, 128, 0, 0, 0, 128, 128, 0, 32768, 128, 128, 32768, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 32768, 32768, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1869, 2171, 1845, 0, 284, 0, 2643, 394, 733, 2607, 474, 520, 1526, 2971, 0, 1212, 969, 1471, 1291, 1789, 79, 641, 164, 1732, 130, 10, 3479, 2334, 1508, 48, 1915, 1007, 694, 4, 1080, 369, 202, 1018, 1697, 281, 155, 43, 1971, 2858, 1, 2941, 705, 853, 1042, 608, 662, 527, 1, 8, 14, 1676, 0, 276, 69, 271, 1625, 2620, 1468, 102, 1170, 14, 0, 1675, 2447, 1139, 552, 808, 1668, 1889, 2229, 108, 1681, 11, 835, 2357, 607, 501, 20, 24, 1849, 397, 869, 40, 2184, 178, 1046, 730, 257, 1699, 551, 0, 1355, 896, 9, 2316, 3, 403, 1596, 1000, 255, 1254, 69, 1261, 461, 2238, 2490, 1507, 1955, 2196, 252, 1348, 2199, 973, 2008, 1680, 1394, 2531, 1547, 1357, 604, 3, 707, 13] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[953, 2564, 1808, 0, 283, 9, 1833, 512, 1009, 2757, 611, 236, 1257, 2068, 0, 1266, 848, 2251, 675, 1879, 50, 529, 170, 1916, 124, 44, 3454, 2036, 2043, 66, 1764, 1188, 559, 1, 1175, 382, 174, 966, 1538, 186, 272, 2, 2440, 2924, 1, 2167, 997, 1071, 1072, 1216, 832, 339, 18, 2, 23, 2361, 1, 283, 13, 659, 1583, 3483, 1510, 174, 758, 8, 0, 1419, 2434, 977, 898, 1247, 1970, 1554, 2407, 227, 1901, 13, 940, 1800, 954, 287, 39, 17, 1644, 543, 500, 17, 2288, 69, 1278, 778, 277, 2190, 626, 5, 1675, 564, 6, 1903, 22, 601, 1579, 719, 419, 1310, 244, 1612, 323, 1915, 2234, 1808, 1487, 2152, 599, 1142, 1912, 835, 2350, 1585, 1347, 2095, 1482, 883, 481, 9, 1085, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1739, 2141, 1689, 0, 215, 0, 2670, 444, 976, 2799, 898, 173, 1627, 3027, 1, 1442, 872, 1376, 1454, 1800, 18, 813, 150, 1736, 91, 22, 3722, 2047, 1566, 25, 1767, 945, 704, 0, 1308, 457, 94, 1396, 1457, 176, 191, 6, 1942, 2849, 2, 2351, 788, 698, 1314, 745, 542, 516, 0, 4, 6, 1789, 1, 188, 61, 278, 1550, 2857, 1129, 148, 838, 6, 0, 1732, 2563, 842, 396, 880, 1644, 1603, 2338, 107, 1471, 2, 423, 2464, 535, 283, 2, 4, 1547, 558, 909, 8, 2833, 26, 1156, 464, 185, 1482, 345, 0, 1526, 915, 13, 2141, 1, 467, 2266, 702, 223, 1603, 42, 1304, 478, 2730, 2943, 1348, 2196, 2174, 447, 1472, 2852, 1154, 2230, 1646, 1148, 2707, 1125, 700, 507, 18, 553, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1648, 2313, 2345, 1, 243, 0, 2173, 590, 579, 2488, 548, 616, 2007, 2522, 0, 1939, 657, 1202, 2043, 2048, 13, 475, 166, 1726, 80, 43, 3465, 2217, 1557, 19, 1288, 1087, 741, 0, 1152, 413, 407, 1319, 1416, 156, 202, 0, 2180, 2989, 0, 1616, 776, 908, 1386, 850, 389, 851, 8, 36, 31, 2454, 0, 271, 14, 221, 2013, 2974, 1586, 119, 1228, 5, 0, 1364, 1800, 1416, 452, 1205, 1871, 2070, 2493, 354, 1371, 5, 592, 1881, 572, 593, 5, 31, 1927, 865, 598, 30, 2138, 152, 669, 920, 215, 1772, 521, 1, 771, 1252, 5, 2068, 17, 463, 1532, 1084, 211, 1428, 101, 2152, 554, 1864, 2031, 1302, 1610, 2495, 361, 1393, 2501, 973, 1855, 1703, 1018, 1993, 1183, 891, 727, 6, 812, 24] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1239, 1733, 1997, 6, 436, 4, 2059, 147, 911, 2520, 571, 406, 2243, 2984, 0, 1392, 686, 1686, 1313, 1719, 49, 638, 194, 2264, 67, 10, 3435, 2305, 1485, 56, 2119, 1196, 741, 2, 957, 197, 167, 1300, 1020, 393, 227, 0, 1714, 2757, 2, 2450, 790, 679, 1231, 705, 318, 630, 5, 33, 6, 2211, 1, 176, 138, 253, 1757, 3054, 870, 174, 1164, 15, 0, 2240, 1978, 1294, 620, 1152, 1387, 2221, 2191, 76, 1382, 1, 831, 1423, 524, 529, 8, 20, 1478, 578, 886, 33, 2370, 24, 703, 1071, 323, 2419, 358, 0, 1495, 1154, 6, 1584, 3, 849, 1694, 988, 171, 1554, 67, 1880, 282, 2486, 2542, 2009, 2025, 1866, 156, 1592, 2359, 675, 2024, 1937, 1320, 2041, 1853, 704, 1190, 1, 413, 25] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1732, 1697, 1479, 1, 309, 0, 2500, 291, 1080, 2700, 955, 353, 1301, 2730, 0, 1395, 711, 1991, 1722, 1993, 38, 1005, 153, 2242, 92, 33, 3439, 1852, 1593, 36, 1977, 992, 455, 0, 1458, 643, 119, 1604, 1485, 175, 199, 5, 2591, 3074, 0, 1962, 890, 791, 1247, 1205, 616, 374, 2, 21, 3, 2032, 0, 173, 60, 277, 1830, 2936, 1653, 123, 734, 13, 0, 1503, 1912, 1308, 394, 885, 1290, 1871, 2225, 131, 1637, 4, 790, 2535, 646, 298, 4, 0, 1512, 432, 989, 28, 2088, 72, 2021, 733, 115, 1905, 205, 0, 1314, 1093, 5, 2150, 1, 228, 1556, 901, 181, 1541, 43, 1195, 438, 1880, 2357, 1438, 2226, 2305, 569, 1438, 2052, 860, 1981, 1581, 1509, 2466, 1131, 596, 700, 6, 649, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1387, 2112, 1535, 0, 220, 0, 2634, 456, 610, 2924, 929, 736, 1868, 2842, 0, 1461, 929, 1781, 1127, 1612, 16, 625, 160, 1860, 77, 64, 3562, 1948, 1201, 38, 1548, 1576, 819, 0, 985, 515, 63, 1351, 1159, 146, 56, 4, 1997, 2868, 0, 2774, 970, 718, 1185, 844, 587, 628, 0, 29, 3, 2206, 0, 191, 19, 347, 1658, 3461, 1306, 126, 881, 8, 0, 1738, 2121, 856, 381, 1027, 1152, 1791, 2444, 78, 1794, 87, 986, 2004, 885, 226, 6, 10, 1402, 465, 893, 4, 2301, 34, 1226, 868, 253, 1975, 552, 0, 1720, 1128, 3, 1766, 4, 305, 1492, 780, 155, 1693, 51, 1293, 304, 1920, 2593, 1677, 2034, 2263, 430, 767, 2434, 1107, 2810, 1599, 958, 3081, 1605, 669, 460, 2, 667, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1782, 2093, 1834, 0, 257, 0, 2021, 281, 755, 2689, 731, 266, 904, 3180, 0, 1615, 1188, 1761, 1427, 1442, 37, 563, 169, 1498, 75, 39, 3532, 1949, 857, 39, 1297, 1133, 1028, 0, 1130, 357, 144, 1827, 1910, 262, 157, 2, 2153, 2832, 0, 2555, 869, 920, 887, 1068, 1095, 397, 0, 23, 14, 2476, 0, 241, 22, 631, 1610, 3065, 1741, 71, 987, 16, 0, 1252, 2547, 1467, 705, 1037, 1905, 2378, 2358, 481, 1433, 3, 666, 2473, 678, 648, 16, 24, 970, 389, 500, 40, 2464, 69, 1510, 936, 82, 2469, 423, 0, 1309, 685, 17, 1808, 2, 394, 1788, 713, 194, 1482, 74, 1490, 248, 1966, 2713, 1927, 2259, 1947, 236, 1822, 1990, 874, 1440, 1180, 1217, 2160, 1515, 523, 729, 3, 537, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1869, 2171, 1845, 0, 284, 0, 2643, 394, 733, 2607, 474, 520, 1526, 245, 0, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 969, 1471, 1291, 1789, 79, 641, 164, 1732, 130, 10, 1344, 2334, 1508, 48, 1915, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 694, 4, 1080, 369, 202, 1018, 1697, 281, 155, 43, 1971, 2858, 1, 2941, 705, 853, 912, 714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 608, 662, 527, 1, 8, 14, 1676, 0, 276, 69, 271, 1625, 2620, 1468, 102, 1763, 1639, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 14, 0, 1675, 2447, 1139, 552, 808, 1668, 1889, 590, 108, 1681, 11, 835, 2357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 501, 20, 24, 1849, 397, 869, 40, 2184, 178, 1046, 730, 257, 1699, 551, 0, 2726, 2531, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1355, 896, 9, 2316, 3, 403, 1596, 1000, 255, 1254, 69, 1261, 461, 2238, 1578, 1507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1955, 2196, 252, 1348, 983, 973, 2008, 1680, 1394, 0, 1547, 1357, 604, 3, 707, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16523, 16432, 16498, 14873, 16944, 16581, 16201, 17020] avg=16384.0000 max=17020 max_rank=7 min=14873 min_rank=3 imbalance_ratio=1.038818 max_load_reduction=114052 max_load_reduction_pct=0.870148 moved_to_echo=66746 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1739, 2141, 1689, 0, 215, 0, 2670, 444, 976, 2799, 898, 173, 1627, 249, 1, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 872, 1376, 1454, 1800, 18, 813, 150, 1736, 91, 22, 1438, 2047, 1566, 25, 1767, 945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 704, 0, 1308, 457, 94, 1396, 1457, 176, 191, 6, 1942, 2849, 2, 2351, 788, 698, 1078, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1314, 745, 542, 516, 0, 4, 6, 1789, 1, 188, 61, 278, 1550, 2857, 1129, 148, 1886, 1719, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 838, 6, 0, 1732, 2563, 842, 396, 880, 1644, 1603, 619, 107, 1471, 2, 423, 2464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 283, 2, 4, 1547, 558, 909, 8, 2833, 26, 1156, 464, 185, 1482, 345, 0, 2778, 2707, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1526, 915, 13, 2141, 1, 467, 2266, 702, 223, 1603, 42, 1304, 478, 2730, 1865, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2196, 2174, 447, 1472, 1275, 1154, 2230, 1646, 1148, 0, 1125, 700, 507, 18, 553, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17063, 16120, 16423, 15384, 15590, 16220, 17624, 16648] avg=16384.0000 max=17624 max_rank=6 min=15384 min_rank=3 imbalance_ratio=1.075684 max_load_reduction=113448 max_load_reduction_pct=0.865540 moved_to_echo=66082 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[953, 2564, 1808, 0, 283, 9, 1833, 512, 1009, 2757, 611, 236, 1257, 170, 0, 1266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 848, 2251, 675, 1879, 50, 529, 170, 1916, 124, 44, 1334, 2036, 2043, 66, 1764, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 1, 1175, 382, 174, 966, 1538, 186, 272, 2, 2440, 2924, 1, 2167, 997, 1071, 818, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 1216, 832, 339, 18, 2, 23, 2361, 1, 283, 13, 659, 1583, 3483, 1510, 174, 1750, 1770, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 758, 8, 0, 1419, 2434, 977, 898, 1247, 1970, 1554, 637, 227, 1901, 13, 940, 1800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 954, 287, 39, 17, 1644, 543, 500, 17, 2288, 69, 1278, 778, 277, 2190, 626, 5, 1898, 2095, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1675, 564, 6, 1903, 22, 601, 1579, 719, 419, 1310, 244, 1612, 323, 1915, 1416, 1808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1487, 2152, 599, 1142, 856, 835, 2350, 1585, 1347, 0, 1482, 883, 481, 9, 1085, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15268, 16917, 16293, 17525, 16783, 15875, 16116, 16295] avg=16384.0000 max=17525 max_rank=3 min=15268 min_rank=0 imbalance_ratio=1.069641 max_load_reduction=113547 max_load_reduction_pct=0.866295 moved_to_echo=65069 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1239, 1733, 1997, 6, 436, 4, 2059, 147, 911, 2520, 571, 406, 2243, 246, 0, 1392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 686, 1686, 1313, 1719, 49, 638, 194, 2264, 67, 10, 1328, 2305, 1485, 56, 2119, 1196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 741, 2, 957, 197, 167, 1300, 1020, 393, 227, 0, 1714, 2757, 2, 2450, 790, 679, 931, 766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1231, 705, 318, 630, 5, 33, 6, 2211, 1, 176, 138, 253, 1757, 3054, 870, 174, 1740, 1611, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1164, 15, 0, 2240, 1978, 1294, 620, 1152, 1387, 2221, 580, 76, 1382, 1, 831, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 529, 8, 20, 1478, 578, 886, 33, 2370, 24, 703, 1071, 323, 2419, 358, 0, 2738, 2041, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1495, 1154, 6, 1584, 3, 849, 1694, 988, 171, 1554, 67, 1880, 282, 2486, 1611, 2009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2025, 1866, 156, 1592, 1055, 675, 2024, 1937, 1320, 0, 1853, 704, 1190, 1, 413, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15910, 17115, 15093, 15451, 16364, 16470, 17833, 16836] avg=16384.0000 max=17833 max_rank=6 min=15093 min_rank=2 imbalance_ratio=1.088440 max_load_reduction=113239 max_load_reduction_pct=0.863945 moved_to_echo=67503 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1648, 2313, 2345, 1, 243, 0, 2173, 590, 579, 2488, 548, 616, 2007, 208, 0, 1939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 1202, 2043, 2048, 13, 475, 166, 1726, 80, 43, 1339, 2217, 1557, 19, 1288, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 741, 0, 1152, 413, 407, 1319, 1416, 156, 202, 0, 2180, 2989, 0, 1616, 776, 908, 744, 812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1386, 850, 389, 851, 8, 36, 31, 2454, 0, 271, 14, 221, 2013, 2974, 1586, 119, 1755, 1833, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1228, 5, 0, 1364, 1800, 1416, 452, 1205, 1871, 2070, 660, 354, 1371, 5, 592, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 572, 593, 5, 31, 1927, 865, 598, 30, 2138, 152, 669, 920, 215, 1772, 521, 1, 2314, 1993, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 771, 1252, 5, 2068, 17, 463, 1532, 1084, 211, 1428, 101, 2152, 554, 1864, 1287, 1302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1610, 2495, 361, 1393, 1118, 973, 1855, 1703, 1018, 0, 1183, 891, 727, 6, 812, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17698, 15960, 15831, 17362, 16274, 15687, 16091, 16169] avg=16384.0000 max=17698 max_rank=0 min=15687 min_rank=5 imbalance_ratio=1.080200 max_load_reduction=113374 max_load_reduction_pct=0.864975 moved_to_echo=64221 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1387, 2112, 1535, 0, 220, 0, 2634, 456, 610, 2924, 929, 736, 1868, 230, 0, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 929, 1781, 1127, 1612, 16, 625, 160, 1860, 77, 64, 1368, 1948, 1201, 38, 1548, 1576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 819, 0, 985, 515, 63, 1351, 1159, 146, 56, 4, 1997, 2868, 0, 2774, 970, 718, 952, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1185, 844, 587, 628, 0, 29, 3, 2206, 0, 191, 19, 347, 1658, 3461, 1306, 126, 1809, 1801, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 881, 8, 0, 1738, 2121, 856, 381, 1027, 1152, 1791, 643, 78, 1794, 87, 986, 2004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 885, 226, 6, 10, 1402, 465, 893, 4, 2301, 34, 1226, 868, 253, 1975, 552, 0, 2612, 3081, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1720, 1128, 3, 1766, 4, 305, 1492, 780, 155, 1693, 51, 1293, 304, 1920, 1641, 1677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2034, 2263, 430, 767, 1081, 1107, 2810, 1599, 958, 0, 1605, 669, 460, 2, 667, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17102, 15930, 16169, 16761, 15547, 17178, 15932, 16453] avg=16384.0000 max=17178 max_rank=5 min=15547 min_rank=4 imbalance_ratio=1.048462 max_load_reduction=113894 max_load_reduction_pct=0.868942 moved_to_echo=65110 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1732, 1697, 1479, 1, 309, 0, 2500, 291, 1080, 2700, 955, 353, 1301, 225, 0, 1395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 711, 1991, 1722, 1993, 38, 1005, 153, 2242, 92, 33, 1329, 1852, 1593, 36, 1977, 992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, 1458, 643, 119, 1604, 1485, 175, 199, 5, 2591, 3074, 0, 1962, 890, 791, 863, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1247, 1205, 616, 374, 2, 21, 3, 2032, 0, 173, 60, 277, 1830, 2936, 1653, 123, 1742, 1636, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 734, 13, 0, 1503, 1912, 1308, 394, 885, 1290, 1871, 589, 131, 1637, 4, 790, 2535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 646, 298, 4, 0, 1512, 432, 989, 28, 2088, 72, 2021, 733, 115, 1905, 205, 0, 2505, 2466, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1314, 1093, 5, 2150, 1, 228, 1556, 901, 181, 1541, 43, 1195, 438, 1880, 1494, 1438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2226, 2305, 569, 1438, 918, 860, 1981, 1581, 1509, 0, 1131, 596, 700, 6, 649, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16018, 17759, 16980, 16398, 15596, 16387, 15458, 16476] avg=16384.0000 max=17759 max_rank=1 min=15458 min_rank=6 imbalance_ratio=1.083923 max_load_reduction=113313 max_load_reduction_pct=0.864510 moved_to_echo=63917 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1782, 2093, 1834, 0, 257, 0, 2021, 281, 755, 2689, 731, 266, 904, 262, 0, 1615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1188, 1761, 1427, 1442, 37, 563, 169, 1498, 75, 39, 1365, 1949, 857, 39, 1297, 1133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 0, 1130, 357, 144, 1827, 1910, 262, 157, 2, 2153, 2832, 0, 2555, 869, 920, 993, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 887, 1068, 1095, 397, 0, 23, 14, 2476, 0, 241, 22, 631, 1610, 3065, 1741, 71, 1789, 1734, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 987, 16, 0, 1252, 2547, 1467, 705, 1037, 1905, 2378, 624, 481, 1433, 3, 666, 2473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 678, 648, 16, 24, 970, 389, 500, 40, 2464, 69, 1510, 936, 82, 2469, 423, 0, 2918, 2160, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1309, 685, 17, 1808, 2, 394, 1788, 713, 194, 1482, 74, 1490, 248, 1966, 1720, 1927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2259, 1947, 236, 1822, 890, 874, 1440, 1180, 1217, 0, 1515, 523, 729, 3, 537, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15490, 14839, 17785, 17318, 17974, 16674, 15817, 15175] avg=16384.0000 max=17974 max_rank=4 min=14839 min_rank=1 imbalance_ratio=1.097046 max_load_reduction=113098 max_load_reduction_pct=0.862869 moved_to_echo=65640 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12416, 16896, 14592, 128, 2304, 128, 18560, 3200, 6656, 21504, 5760, 3328, 12800, 1920, 128, 11776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[6912, 13568, 11136, 14336, 384, 5376, 1408, 14976, 768, 384, 10880, 16768, 11904, 384, 13696, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[11264, 7808, 128, 15744, 128, 3712, 13568, 6912, 1920, 11904, 768, 12288, 3200, 17024, 12672, 13056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[7808, 128, 0, 12928, 17920, 9344, 4480, 8320, 12928, 15488, 4992, 1664, 12672, 128, 6144, 17024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[15872, 17408, 3072, 11008, 8192, 7552, 16768, 12928, 9984, 0, 11520, 6400, 5504, 128, 5504, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[5760, 128, 9344, 3456, 1408, 10880, 11776, 1792, 1536, 128, 17024, 23168, 128, 18816, 6912, 6656, 7296, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[9472, 7296, 5120, 4352, 128, 256, 128, 17280, 128, 1920, 512, 2944, 13696, 24576, 11264, 1152, 14336, 13824, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[5504, 3456, 128, 256, 12416, 4352, 6144, 256, 18688, 640, 9728, 6528, 1792, 16000, 3584, 128, 20608, 19200, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 4084, 0, 3316, 0, 0, 0, 3981, 0, 0, 0, 0, 4090, 0, 4, 4093, 0, 0, 4087, 14, 72, 0, 2, 4092, 0, 4088, 4091, 8, 0, 1, 4091, 3, 0, 4096, 0, 29, 0, 1, 0, 0, 11, 0, 444, 1, 1, 4095, 4082, 1, 0, 4092, 0, 0, 4094, 0, 0, 4092, 12, 0, 0, 4094, 0, 7, 3869, 4, 0, 3493, 1, 0, 4095, 0, 255, 4092, 0, 0, 4, 0, 1, 4, 3925, 5, 9, 0, 0, 0, 3771, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 4091, 4051, 3, 4, 0, 3, 4096, 0, 4090, 3, 1, 4, 4095, 4096, 0, 0, 403, 4, 256, 1, 0, 4088, 3, 0, 813, 0, 1, 0, 4070] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4022, 0, 2815, 9, 0, 0, 3805, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4087, 2, 52, 3, 0, 4073, 0, 4096, 4096, 19, 0, 0, 4095, 0, 87, 4095, 1, 95, 0, 14, 0, 0, 3, 0, 1085, 0, 8, 4075, 4094, 0, 0, 4096, 0, 1, 4096, 0, 0, 4095, 94, 0, 0, 4096, 0, 0, 3955, 0, 0, 3368, 2, 0, 4096, 0, 17, 4096, 0, 0, 0, 0, 0, 0, 3671, 0, 0, 0, 0, 0, 3264, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 4096, 0, 0, 474, 0, 40, 2, 0, 4087, 0, 1, 1831, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 20, 4088, 0, 2316, 0, 0, 0, 3854, 5, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 4095, 63, 156, 0, 0, 4079, 0, 4087, 4086, 14, 0, 0, 4096, 0, 7, 4091, 0, 23, 0, 5, 0, 0, 0, 0, 964, 0, 1, 4092, 4085, 0, 0, 4096, 17, 1, 4096, 0, 0, 4096, 8, 0, 0, 4080, 0, 2, 4050, 0, 0, 4004, 19, 0, 4096, 0, 21, 4096, 0, 0, 0, 0, 0, 0, 4047, 0, 5, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 4089, 66, 15, 0, 3, 4096, 0, 4084, 6, 0, 0, 4096, 4096, 1, 0, 612, 0, 57, 1, 0, 4088, 15, 0, 295, 0, 0, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 21, 0, 0, 3499, 15, 1090, 1, 1, 5, 3908, 0, 0, 0, 0, 4089, 1, 26, 4095, 0, 0, 4096, 9, 23, 0, 0, 3958, 0, 4096, 4081, 163, 0, 0, 4089, 10, 139, 3886, 0, 70, 0, 0, 2, 0, 0, 0, 308, 0, 59, 4096, 4059, 31, 1, 4089, 0, 84, 4080, 0, 4, 4094, 16, 0, 4, 4095, 0, 0, 4067, 0, 12, 3456, 151, 16, 4082, 4, 144, 4056, 0, 1, 11, 0, 4, 0, 4086, 0, 0, 0, 1, 0, 4021, 0, 0, 15, 1, 1, 4092, 15, 0, 25, 0, 2, 4071, 3915, 185, 0, 1, 27, 4068, 4, 4091, 0, 4, 0, 4019, 4054, 22, 1, 1566, 0, 0, 0, 2, 4096, 0, 0, 2181, 0, 14, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4093, 0, 2684, 9, 0, 0, 3849, 1, 0, 1, 0, 4095, 0, 0, 4095, 0, 0, 4091, 5, 246, 0, 0, 4093, 1, 4096, 4095, 65, 0, 0, 4096, 0, 3, 4095, 3, 40, 0, 30, 0, 0, 20, 0, 484, 0, 1, 4068, 4091, 0, 0, 4094, 4, 15, 4095, 0, 0, 4078, 1, 1, 0, 4096, 0, 1, 4002, 0, 0, 3268, 0, 0, 4096, 0, 329, 4095, 0, 0, 0, 0, 0, 0, 3954, 0, 0, 1, 1, 0, 3836, 0, 0, 0, 0, 1, 4096, 1, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 1, 4096, 4095, 0, 0, 468, 2, 35, 0, 0, 4080, 12, 1, 1288, 0, 0, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 4028, 0, 1204, 0, 0, 0, 3874, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 76, 0, 0, 4077, 0, 4096, 4096, 6, 0, 6, 4095, 0, 40, 4077, 0, 31, 0, 1, 0, 0, 2, 0, 1116, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 4096, 1, 0, 4096, 1, 0, 0, 4094, 0, 1, 4080, 0, 0, 3902, 23, 0, 4096, 0, 3, 4092, 0, 0, 0, 0, 0, 0, 3932, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 4096, 4096, 17, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 4095, 0, 0, 882, 0, 0, 0, 0, 4096, 0, 0, 1424, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4039, 0, 1488, 1, 0, 0, 4011, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4090, 7, 936, 0, 0, 4093, 0, 4095, 4096, 13, 0, 0, 4096, 0, 9, 4096, 2, 73, 0, 11, 0, 0, 0, 0, 508, 0, 0, 4075, 4093, 0, 0, 4096, 2, 0, 4096, 0, 0, 4096, 7, 0, 0, 4096, 0, 0, 4032, 0, 0, 3803, 0, 0, 4096, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 3563, 0, 2, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 4092, 7, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 736, 0, 24, 3, 0, 4091, 2, 0, 1378, 0, 0, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 4092, 0, 956, 16, 0, 9, 4014, 11, 0, 0, 7, 4089, 0, 0, 4092, 0, 0, 4096, 7, 784, 1, 0, 4096, 0, 4083, 4093, 18, 0, 3, 4092, 1, 13, 4060, 5, 350, 0, 32, 0, 0, 39, 0, 833, 0, 0, 4088, 4086, 0, 0, 4094, 0, 85, 4096, 0, 0, 4018, 17, 0, 0, 4093, 0, 25, 4058, 1, 0, 3949, 113, 0, 4081, 0, 4, 4084, 0, 0, 0, 8, 0, 0, 3498, 0, 0, 0, 0, 0, 3712, 3, 0, 8, 4, 0, 4082, 0, 0, 0, 0, 0, 4095, 3705, 4, 0, 0, 0, 4096, 0, 4064, 0, 0, 1, 4095, 4093, 7, 0, 1770, 0, 93, 0, 0, 4087, 0, 0, 767, 0, 0, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 20, 4088, 0, 2316, 0, 0, 0, 3854, 5, 0, 0, 0, 4094, 2056, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3784, 63, 156, 0, 0, 4079, 0, 4087, 4086, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 4091, 0, 23, 0, 5, 0, 0, 0, 0, 964, 0, 1, 4096, 3257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4085, 0, 0, 0, 17, 1, 0, 0, 0, 4096, 8, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4050, 0, 0, 4004, 19, 0, 4096, 0, 21, 4096, 0, 0, 0, 0, 0, 311, 58, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 5, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 4094, 0, 0, 3767, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 66, 15, 0, 3, 53, 0, 4084, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 612, 0, 57, 1, 0, 4088, 15, 0, 295, 0, 0, 0, 4094, 1929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16762, 16269, 16540, 16365, 16671, 16769, 16508, 15188] avg=16384.0000 max=16769 max_rank=5 min=15188 min_rank=7 imbalance_ratio=1.023499 max_load_reduction=114303 max_load_reduction_pct=0.872063 moved_to_echo=65136 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 4084, 0, 3316, 0, 0, 0, 3981, 0, 0, 0, 0, 4090, 2056, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3776, 14, 72, 0, 2, 4092, 0, 4088, 4091, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 3, 0, 4096, 0, 29, 0, 1, 0, 0, 11, 0, 444, 1, 1, 4093, 3254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4082, 1, 0, 0, 0, 0, 0, 0, 0, 4092, 12, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3869, 4, 0, 3493, 1, 0, 4095, 0, 255, 4092, 0, 0, 4, 0, 1, 311, 58, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3925, 5, 9, 0, 0, 0, 3771, 0, 0, 0, 0, 0, 4094, 0, 0, 3765, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 4051, 3, 4, 0, 3, 53, 0, 4090, 3, 1, 4, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 403, 4, 256, 1, 0, 4088, 3, 0, 813, 0, 1, 0, 4070, 1929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17860, 16147, 16025, 16362, 16204, 16411, 16399, 15664] avg=16384.0000 max=17860 max_rank=0 min=15664 min_rank=7 imbalance_ratio=1.090088 max_load_reduction=113212 max_load_reduction_pct=0.863739 moved_to_echo=64678 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 21, 0, 0, 3499, 15, 1090, 1, 1, 5, 3908, 0, 0, 0, 0, 4089, 2042, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 26, 0, 0, 0, 3784, 9, 23, 0, 0, 3958, 0, 4096, 4081, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 10, 139, 3886, 0, 70, 0, 0, 2, 0, 0, 0, 308, 0, 59, 4095, 3250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4059, 31, 1, 0, 0, 84, 0, 0, 4, 4094, 16, 0, 4, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 12, 3456, 151, 16, 4082, 4, 144, 4056, 0, 1, 11, 0, 4, 312, 58, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 1, 0, 4021, 0, 0, 15, 1, 1, 4092, 15, 0, 3753, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 2, 4071, 3915, 185, 0, 1, 27, 52, 4, 4091, 0, 4, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 22, 1, 1566, 0, 0, 0, 2, 4096, 0, 0, 2181, 0, 14, 0, 4095, 1916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14998, 16141, 15908, 16470, 16388, 16824, 16396, 17947] avg=16384.0000 max=17947 max_rank=7 min=14998 min_rank=0 imbalance_ratio=1.095398 max_load_reduction=113125 max_load_reduction_pct=0.863075 moved_to_echo=67555 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4022, 0, 2815, 9, 0, 0, 3805, 1, 0, 0, 0, 4096, 2056, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3776, 2, 52, 3, 0, 4073, 0, 4096, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 87, 4095, 1, 95, 0, 14, 0, 0, 3, 0, 1085, 0, 8, 4096, 3257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4094, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 94, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 3368, 2, 0, 4096, 0, 17, 4096, 0, 0, 0, 0, 0, 311, 58, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3671, 0, 0, 0, 0, 0, 3264, 0, 0, 0, 0, 0, 4096, 0, 0, 3766, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 53, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 474, 0, 40, 2, 0, 4087, 0, 1, 1831, 0, 0, 0, 4095, 1929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17134, 16117, 16836, 16441, 15917, 15636, 16436, 16555] avg=16384.0000 max=17134 max_rank=0 min=15636 min_rank=5 imbalance_ratio=1.045776 max_load_reduction=113938 max_load_reduction_pct=0.869278 moved_to_echo=64544 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4093, 0, 2684, 9, 0, 0, 3849, 1, 0, 1, 0, 4095, 2062, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3777, 5, 246, 0, 0, 4093, 1, 4096, 4095, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 4095, 3, 40, 0, 30, 0, 0, 20, 0, 484, 0, 1, 4095, 3255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 4091, 0, 0, 0, 4, 15, 0, 0, 0, 4078, 1, 1, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4002, 0, 0, 3268, 0, 0, 4096, 0, 329, 4095, 0, 0, 0, 0, 0, 314, 64, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 0, 1, 1, 0, 3836, 0, 0, 0, 0, 1, 4096, 1, 2, 3765, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 38, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 468, 2, 35, 0, 0, 4080, 12, 1, 1288, 0, 0, 0, 4093, 1931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17125, 16378, 16122, 16337, 16186, 16496, 16423, 16005] avg=16384.0000 max=17125 max_rank=0 min=16005 min_rank=7 imbalance_ratio=1.045227 max_load_reduction=113947 max_load_reduction_pct=0.869347 moved_to_echo=65110 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 4092, 0, 956, 16, 0, 9, 4014, 11, 0, 0, 7, 4089, 2056, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3784, 7, 784, 1, 0, 4096, 0, 4083, 4093, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4092, 1, 13, 4060, 5, 350, 0, 32, 0, 0, 39, 0, 833, 0, 0, 4092, 3254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4086, 0, 0, 0, 0, 85, 0, 0, 0, 4018, 17, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4058, 1, 0, 3949, 113, 0, 4081, 0, 4, 4084, 0, 0, 0, 8, 0, 312, 58, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3498, 0, 0, 0, 0, 0, 3712, 3, 0, 8, 4, 0, 4082, 0, 0, 3767, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3705, 4, 0, 0, 0, 53, 0, 4064, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 7, 0, 1770, 0, 93, 0, 0, 4087, 0, 0, 767, 0, 0, 0, 4093, 1929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15582, 16866, 16774, 16373, 16707, 15914, 16017, 16839] avg=16384.0000 max=16866 max_rank=1 min=15582 min_rank=0 imbalance_ratio=1.029419 max_load_reduction=114206 max_load_reduction_pct=0.871323 moved_to_echo=65477 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 4028, 0, 1204, 0, 0, 0, 3874, 0, 0, 0, 0, 4096, 2056, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3784, 0, 76, 0, 0, 4077, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4095, 0, 40, 4077, 0, 31, 0, 1, 0, 0, 2, 0, 1116, 0, 0, 4096, 3256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 1, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4080, 0, 0, 3902, 23, 0, 4096, 0, 3, 4092, 0, 0, 0, 0, 0, 312, 58, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 4096, 0, 4, 3767, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 17, 0, 0, 0, 53, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 882, 0, 0, 0, 0, 4096, 0, 0, 1424, 0, 0, 0, 4096, 1929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15591, 16135, 16720, 16369, 16581, 16701, 16453, 16522] avg=16384.0000 max=16720 max_rank=2 min=15591 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=114352 max_load_reduction_pct=0.872437 moved_to_echo=66257 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4039, 0, 1488, 1, 0, 0, 4011, 0, 0, 0, 0, 4096, 2056, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3779, 7, 936, 0, 0, 4093, 0, 4095, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 4096, 2, 73, 0, 11, 0, 0, 0, 0, 508, 0, 0, 4096, 3256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4093, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 7, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 3803, 0, 0, 4096, 0, 8, 4096, 0, 0, 0, 0, 0, 311, 58, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3563, 0, 2, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 4096, 0, 1, 3767, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 7, 0, 0, 0, 53, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 736, 0, 24, 3, 0, 4091, 2, 0, 1378, 0, 0, 0, 4093, 1929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16020, 17019, 16147, 16355, 16418, 16321, 16440, 16352] avg=16384.0000 max=17019 max_rank=1 min=16020 min_rank=0 imbalance_ratio=1.038757 max_load_reduction=114053 max_load_reduction_pct=0.870155 moved_to_echo=65531 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 32768, 128, 128, 0, 128, 256, 0, 128, 128, 32768, 256, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 32768, 32256, 384, 128, 128, 128, 512, 128, 32768, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 384, 32512, 128, 768, 0, 128, 128, 0, 128, 0, 5760, 128, 128, 32768, 26112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 6912, 128, 512, 128, 128, 32768, 128, 128, 9984, 0, 128, 0, 32768, 15488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 30720, 128, 128, 128, 128, 0, 30848, 128, 0, 128, 128, 128, 32768, 128, 128, 30208, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 0, 0, 30336, 128, 2432, 128, 128, 32640, 128, 32768, 32768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32128, 128, 128, 29312, 384, 128, 32768, 128, 896, 32768, 0, 128, 128, 128, 128, 2560, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 32000, 128, 15872, 128, 128, 128, 31360, 128, 0, 128, 128, 32768, 16512, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3104, 0, 1363, 0, 0, 52, 0, 4095, 4, 0, 0, 0, 0, 4095, 1, 0, 0, 3316, 4092, 0, 4095, 0, 4096, 4095, 0, 0, 0, 4096, 1429, 0, 2, 1, 4096, 0, 0, 4095, 20, 0, 0, 0, 4096, 0, 1, 0, 0, 4094, 4094, 3335, 0, 0, 2, 1, 4095, 4096, 0, 0, 4090, 4095, 0, 0, 0, 2481, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 1, 4072, 4094, 0, 0, 1, 4064, 1, 1, 0, 1189, 2, 0, 4080, 0, 0, 0, 4083, 0, 2, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 158, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 4095, 0, 0, 1, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1212, 0, 2121, 0, 0, 122, 0, 4096, 30, 0, 0, 0, 0, 4096, 0, 0, 0, 3466, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 1698, 0, 0, 0, 4096, 0, 1, 4096, 30, 0, 0, 0, 4096, 0, 5, 0, 0, 4096, 4093, 2106, 0, 0, 0, 0, 4096, 4079, 0, 0, 4084, 4047, 0, 0, 2, 2691, 0, 0, 4096, 13, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 1, 4092, 4096, 0, 0, 74, 3912, 0, 113, 0, 2971, 0, 0, 4063, 0, 0, 0, 3964, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 147, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 14, 0, 4096, 1, 0, 1, 0, 15] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2324, 4, 2370, 0, 1, 58, 0, 4092, 13, 0, 0, 0, 2, 4092, 4, 0, 0, 3285, 4059, 0, 4091, 0, 4095, 4093, 0, 0, 0, 4096, 1126, 0, 7, 3, 4096, 4, 0, 4096, 3, 0, 0, 1, 4095, 1, 9, 0, 0, 4080, 4087, 3346, 0, 0, 0, 0, 4092, 4092, 0, 0, 4087, 4087, 0, 0, 2, 3083, 0, 0, 4090, 0, 2, 0, 0, 0, 1, 4096, 1, 0, 4096, 6, 1, 4054, 4093, 0, 1, 9, 4049, 0, 15, 0, 878, 1, 0, 4090, 0, 0, 0, 4002, 4, 8, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 5, 0, 1, 99, 4091, 1, 0, 3, 0, 0, 1, 4096, 0, 0, 0, 4090, 7, 2, 3, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[23, 12, 3613, 0, 232, 1, 1, 76, 1, 4052, 308, 9, 1, 0, 0, 4076, 0, 3, 2, 3367, 3829, 0, 4073, 3, 4091, 4096, 1, 0, 18, 4051, 590, 1, 7, 0, 4095, 41, 0, 4095, 281, 0, 1, 0, 4091, 2, 12, 3, 0, 4064, 4096, 1723, 4, 0, 0, 0, 4094, 4084, 0, 1, 4017, 4028, 1, 1, 0, 2624, 6, 0, 4069, 26, 0, 0, 0, 0, 11, 4093, 0, 0, 4088, 0, 76, 3973, 3948, 1, 0, 87, 4047, 2, 224, 2, 3533, 19, 0, 4089, 1, 0, 2, 4089, 0, 58, 3, 1, 0, 0, 4088, 0, 4, 1, 4, 0, 6, 4, 300, 4068, 3, 1, 25, 0, 0, 5, 4093, 1, 20, 0, 4079, 25, 0, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 2415, 0, 386, 0, 0, 34, 0, 4096, 25, 0, 0, 0, 0, 4096, 0, 0, 0, 3431, 4094, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 1482, 0, 0, 0, 4096, 11, 1, 4096, 126, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4093, 2251, 0, 0, 6, 0, 4094, 4096, 0, 0, 4076, 4063, 0, 0, 0, 2648, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 6, 4079, 4089, 0, 0, 5, 4023, 0, 36, 0, 3141, 0, 0, 4086, 0, 0, 0, 4084, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 544, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4094, 0, 0, 2, 0, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 3566, 0, 382, 0, 0, 5, 0, 4096, 20, 0, 0, 1, 0, 4096, 0, 0, 0, 3880, 4092, 0, 4096, 0, 4096, 4096, 0, 0, 1, 4096, 909, 0, 0, 0, 4096, 0, 0, 4096, 9, 0, 0, 1, 4096, 0, 2, 0, 0, 4094, 4094, 2278, 0, 0, 1, 0, 4096, 4096, 0, 0, 4095, 4096, 0, 0, 0, 1971, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4075, 4096, 0, 0, 0, 4094, 0, 90, 0, 2761, 1, 0, 4089, 0, 0, 0, 4073, 0, 1, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 562, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2272, 0, 1454, 0, 0, 16, 0, 4095, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 3735, 4095, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 1333, 0, 0, 0, 4096, 0, 2, 4096, 7, 0, 0, 1, 4096, 0, 0, 0, 0, 4095, 4094, 3582, 0, 0, 4, 0, 4095, 4096, 0, 0, 4093, 4096, 0, 0, 0, 2386, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4035, 4096, 0, 0, 0, 3961, 0, 36, 0, 1786, 1, 0, 4055, 0, 0, 0, 4052, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 45, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 1, 0, 0, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2353, 0, 170, 0, 0, 54, 0, 4087, 250, 0, 0, 0, 0, 4089, 0, 0, 0, 2734, 4093, 0, 4095, 0, 4095, 4091, 0, 0, 0, 4088, 1378, 0, 10, 7, 4096, 19, 0, 4093, 885, 0, 0, 0, 4096, 0, 3, 0, 0, 4096, 4037, 2926, 0, 0, 9, 0, 4096, 4087, 0, 0, 4079, 4064, 0, 0, 10, 2128, 0, 0, 4089, 19, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 10, 4006, 4096, 1, 0, 92, 3870, 0, 736, 0, 2887, 4, 0, 4058, 0, 0, 0, 4050, 2, 14, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 11, 0, 0, 197, 4082, 0, 0, 7, 0, 0, 0, 4089, 0, 3, 0, 4094, 8, 0, 19, 3, 19] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1212, 0, 2121, 0, 0, 122, 0, 4096, 30, 0, 0, 0, 0, 4096, 2400, 1053, 503, 280, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3466, 4096, 0, 3593, 0, 0, 0, 0, 0, 0, 4096, 1698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3915, 0, 1, 4096, 30, 0, 0, 0, 4096, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3040, 2106, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4047, 0, 0, 2, 2691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 1, 4092, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 74, 3912, 0, 113, 0, 2971, 0, 0, 1663, 0, 0, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 147, 4096, 4096, 3799, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 14, 0, 4096, 1, 0, 1, 0, 15, 4096, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16064, 16949, 16239, 15970, 16425, 16793, 16240, 16392] avg=16384.0000 max=16949 max_rank=1 min=15970 min_rank=3 imbalance_ratio=1.034485 max_load_reduction=114123 max_load_reduction_pct=0.870689 moved_to_echo=65850 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2324, 4, 2370, 0, 1, 58, 0, 4092, 13, 0, 0, 0, 2, 4092, 2416, 1052, 502, 281, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3285, 4059, 0, 3589, 0, 0, 0, 0, 0, 0, 4096, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 3915, 4, 0, 4096, 3, 0, 0, 1, 4095, 1, 9, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3035, 3346, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4087, 0, 0, 2, 3083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 2, 0, 0, 0, 1, 4096, 1, 0, 4096, 6, 1, 4054, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 9, 4049, 0, 15, 0, 878, 1, 0, 1674, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 5, 0, 1, 99, 4091, 4092, 3811, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 1, 4096, 0, 0, 0, 4090, 7, 2, 3, 0, 5, 4095, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17358, 16159, 16214, 17640, 16377, 14722, 16227, 16375] avg=16384.0000 max=17640 max_rank=3 min=14722 min_rank=5 imbalance_ratio=1.076660 max_load_reduction=113432 max_load_reduction_pct=0.865417 moved_to_echo=63701 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[23, 12, 3613, 0, 232, 1, 1, 76, 1, 4052, 308, 9, 1, 0, 0, 4076, 2415, 1054, 500, 280, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 3367, 3829, 0, 3573, 3, 0, 0, 1, 0, 18, 4051, 590, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3914, 41, 0, 4095, 281, 0, 1, 0, 4091, 2, 12, 3, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3042, 1723, 4, 0, 0, 0, 0, 0, 0, 1, 4017, 4028, 1, 1, 0, 2624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4069, 26, 0, 0, 0, 0, 11, 4093, 0, 0, 4088, 0, 76, 3973, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 1, 0, 87, 4047, 2, 224, 2, 3533, 19, 0, 1674, 1, 0, 2, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 3, 1, 0, 0, 4088, 0, 4, 1, 4, 0, 6, 4, 300, 4068, 4094, 3804, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 25, 0, 0, 5, 4093, 1, 20, 0, 4079, 25, 0, 2, 0, 0, 4091, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16805, 15438, 16511, 15441, 16372, 17629, 16457, 16419] avg=16384.0000 max=17629 max_rank=5 min=15438 min_rank=1 imbalance_ratio=1.075989 max_load_reduction=113443 max_load_reduction_pct=0.865501 moved_to_echo=66877 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3104, 0, 1363, 0, 0, 52, 0, 4095, 4, 0, 0, 0, 0, 4095, 2415, 1057, 507, 281, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3316, 4092, 0, 3588, 0, 0, 0, 0, 0, 0, 4096, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 3909, 0, 0, 4095, 20, 0, 0, 0, 4096, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3037, 3335, 0, 0, 2, 1, 0, 0, 0, 0, 4090, 4095, 0, 0, 0, 2481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 1, 4072, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 4064, 1, 1, 0, 1189, 2, 0, 1665, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 158, 4096, 4095, 3815, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 4095, 0, 0, 1, 0, 5, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17126, 16522, 16218, 17041, 16397, 15100, 16291, 16377] avg=16384.0000 max=17126 max_rank=0 min=15100 min_rank=5 imbalance_ratio=1.045288 max_load_reduction=113946 max_load_reduction_pct=0.869339 moved_to_echo=64165 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2353, 0, 170, 0, 0, 54, 0, 4087, 250, 0, 0, 0, 0, 4089, 2397, 1039, 503, 280, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2734, 4093, 0, 3592, 0, 0, 0, 0, 0, 0, 4088, 1378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 7, 3915, 19, 0, 4093, 885, 0, 0, 0, 4096, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2998, 2926, 0, 0, 9, 0, 0, 0, 0, 0, 4079, 4064, 0, 0, 10, 2128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 19, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 10, 4006, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 92, 3870, 0, 736, 0, 2887, 4, 0, 1661, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 14, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 11, 0, 0, 197, 4082, 4096, 3807, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4089, 0, 3, 0, 4094, 8, 0, 19, 3, 19, 4095, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15373, 15885, 17124, 16214, 16348, 17397, 16324, 16407] avg=16384.0000 max=17397 max_rank=5 min=15373 min_rank=0 imbalance_ratio=1.061829 max_load_reduction=113675 max_load_reduction_pct=0.867271 moved_to_echo=66476 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2415, 0, 386, 0, 0, 34, 0, 4096, 25, 0, 0, 0, 0, 4096, 2414, 1053, 503, 281, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3431, 4094, 0, 3593, 0, 0, 0, 0, 0, 0, 4096, 1482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3915, 11, 1, 4096, 126, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3040, 2251, 0, 0, 6, 0, 0, 0, 0, 0, 4076, 4063, 0, 0, 0, 2648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 6, 4079, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 5, 4023, 0, 36, 0, 3141, 0, 0, 1672, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 544, 4096, 4094, 3815, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4094, 0, 0, 2, 0, 7, 4096, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15455, 16696, 16341, 16084, 16405, 17050, 16669, 16372] avg=16384.0000 max=17050 max_rank=5 min=15455 min_rank=0 imbalance_ratio=1.040649 max_load_reduction=114022 max_load_reduction_pct=0.869919 moved_to_echo=66496 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2272, 0, 1454, 0, 0, 16, 0, 4095, 4, 0, 0, 0, 0, 4096, 2395, 1054, 503, 281, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3735, 4095, 0, 3593, 0, 0, 0, 0, 0, 0, 4096, 1333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3915, 0, 2, 4096, 7, 0, 0, 1, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3040, 3582, 0, 0, 4, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 2386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4035, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3961, 0, 36, 0, 1786, 1, 0, 1660, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 45, 4096, 4095, 3815, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 1, 0, 0, 0, 4, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16321, 16852, 16212, 17201, 16354, 15592, 16172, 16368] avg=16384.0000 max=17201 max_rank=3 min=15592 min_rank=5 imbalance_ratio=1.049866 max_load_reduction=113871 max_load_reduction_pct=0.868767 moved_to_echo=64486 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3566, 0, 382, 0, 0, 5, 0, 4096, 20, 0, 0, 1, 0, 4096, 2415, 1054, 503, 280, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3880, 4092, 0, 3593, 0, 0, 0, 0, 0, 1, 4096, 909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3915, 0, 0, 4096, 9, 0, 0, 1, 4096, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3040, 2278, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 1971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4075, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 90, 0, 2761, 1, 0, 1674, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 562, 4096, 4096, 3816, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16570, 16571, 16213, 15481, 16394, 16789, 16692, 16362] avg=16384.0000 max=16789 max_rank=5 min=15481 min_rank=3 imbalance_ratio=1.024719 max_load_reduction=114283 max_load_reduction_pct=0.871910 moved_to_echo=66237 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 32768, 128, 128, 0, 0, 0, 128, 32768, 128, 0, 32768, 128, 128, 32512, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132992 observed_loss=-1920 tokens_per_expert=[128, 128, 20864, 128, 8576, 128, 128, 512, 128, 32768, 768, 128, 128, 128, 128, 32768, 19328, 8448, 4096, 2304, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32640, 128, 128, 384, 32128, 128, 1280, 128, 19200, 128, 0, 13440, 128, 0, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 27264, 32512, 0, 28800, 128, 0, 0, 128, 0, 128, 32768, 9984, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[24320, 21632, 128, 0, 128, 128, 0, 0, 0, 128, 32640, 32640, 128, 128, 128, 20096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 31360, 128, 128, 32768, 1408, 0, 128, 128, 32768, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 0, 128, 32768, 128, 128, 0, 32768, 128, 128, 128, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 0, 128, 128, 128, 128, 128, 128, 2176, 32768, 32768, 30592, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132992 slack_slots=7040 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 2689, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 5, 0, 4096, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 4096, 0, 0, 4095, 1, 0, 2, 0, 0, 4096, 0, 0, 4096, 3582, 4096, 0, 0, 1476, 0, 4096, 4096, 4094, 0, 0, 4096, 36, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4094, 0, 0, 0, 505, 0, 0, 0, 0, 0, 4096, 3986, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8, 4091, 2, 0, 2819, 4092, 4096, 0, 0, 0, 4096, 0, 4083, 2, 0, 4096, 0, 0, 5, 4093, 0, 2, 4096, 0, 0, 4094, 0, 0, 4096, 0, 1, 0, 0, 0, 4092, 4, 0, 4096, 3833, 4091, 0, 0, 2053, 0, 4096, 4096, 4091, 0, 0, 4096, 9, 9, 0, 0, 0, 0, 3, 5, 0, 4096, 0, 1, 0, 0, 4089, 0, 0, 4092, 4084, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 4091, 4092, 0, 0, 0, 0, 4096, 1, 0, 22, 0, 0, 0, 1, 0, 0, 20, 3, 0, 0, 0, 5, 0, 0, 4096, 4, 0, 0, 0, 3, 4094, 0, 0, 0, 213, 0, 0, 0, 0, 0, 4094, 3336, 0, 0, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 439, 4095, 4096, 0, 0, 0, 4096, 0, 4086, 0, 0, 4096, 0, 0, 1, 4096, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3388, 4086, 0, 0, 2934, 0, 4096, 4096, 4096, 0, 0, 4096, 35, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 4091, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 32, 4096, 0, 0, 0, 1648, 0, 0, 0, 0, 0, 4096, 3807, 4, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4095, 0, 0, 3342, 4096, 4096, 0, 0, 0, 4095, 0, 4093, 3, 0, 4094, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 4096, 0, 0, 4095, 1, 0, 1, 0, 0, 4096, 0, 14, 4096, 3936, 4096, 0, 6, 925, 0, 4096, 4096, 4087, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 124, 4094, 0, 0, 0, 128, 0, 0, 0, 0, 0, 4096, 3820, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 1933, 4096, 4096, 0, 0, 0, 4096, 0, 4011, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 4090, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3816, 4072, 0, 0, 3242, 0, 4096, 4096, 4093, 0, 0, 4096, 22, 0, 0, 0, 0, 0, 20, 5, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 4096, 0, 0, 0, 0, 4096, 0, 0, 80, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 110, 4096, 0, 0, 0, 367, 0, 0, 0, 0, 0, 4096, 2827, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[3, 0, 3, 2, 30, 4080, 2, 0, 1975, 4094, 4093, 1, 0, 3, 4090, 3, 3966, 0, 2, 4093, 3, 2, 1, 4065, 0, 7, 4093, 1, 0, 4090, 1, 0, 4092, 2, 11, 3, 2, 0, 4093, 8, 0, 4093, 2729, 4066, 3, 0, 3407, 2, 4093, 4093, 4088, 2, 0, 4093, 8, 17, 1, 3, 1, 4, 83, 0, 0, 4092, 4, 0, 4, 109, 4071, 3, 0, 4094, 4036, 3, 3, 3, 0, 0, 0, 0, 4093, 0, 4095, 4075, 2, 0, 3, 0, 4070, 0, 0, 51, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 2, 0, 1, 3, 4093, 14, 3, 3, 0, 52, 4092, 2, 0, 1, 1564, 0, 0, 1, 0, 4, 4083, 2591, 0, 2, 4048, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 4089, 7, 0, 813, 4090, 4096, 0, 0, 0, 4096, 0, 3995, 0, 0, 4096, 0, 0, 2, 4088, 0, 0, 4096, 0, 0, 4090, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 4013, 4095, 0, 0, 1290, 0, 4096, 4096, 4088, 0, 1, 4096, 16, 20, 0, 0, 0, 3, 11, 9, 0, 4096, 0, 0, 0, 186, 4084, 0, 0, 4086, 4074, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 4083, 4089, 0, 0, 0, 0, 4096, 0, 0, 85, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 13, 0, 0, 4096, 0, 0, 0, 0, 44, 4096, 0, 0, 0, 2355, 0, 0, 0, 0, 0, 4096, 3605, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4093, 0, 0, 3489, 4095, 4096, 0, 0, 0, 4096, 0, 4093, 1, 0, 4096, 0, 0, 1, 4096, 0, 1, 4088, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2754, 4095, 0, 1, 1573, 0, 4096, 4096, 4088, 0, 0, 4096, 40, 1, 0, 0, 0, 1, 1, 0, 0, 4096, 0, 0, 0, 2, 4094, 0, 0, 4095, 4092, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4096, 1, 0, 5, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 2, 0, 0, 4095, 1, 0, 0, 0, 15, 4096, 0, 0, 0, 438, 0, 0, 0, 0, 0, 4095, 3988, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 1933, 4096, 1908, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 49, 0, 0, 0, 4091, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2576, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3816, 4072, 0, 0, 3242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 4056, 22, 0, 0, 0, 0, 0, 20, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 2188, 1520, 293, 40, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 4092, 4096, 0, 0, 0, 0, 4096, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 110, 4096, 4096, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3803, 0, 0, 0, 367, 0, 0, 0, 0, 0, 4096, 2827, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16129, 16337, 17802, 16388, 16348, 16438, 16441, 15189] avg=16384.0000 max=17802 max_rank=2 min=15189 min_rank=7 imbalance_ratio=1.086548 max_load_reduction=113270 max_load_reduction_pct=0.864182 moved_to_echo=64416 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 4089, 7, 0, 813, 4090, 1908, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 0, 49, 0, 0, 2, 4088, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2576, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4013, 4095, 0, 0, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 1, 4056, 16, 20, 0, 0, 0, 3, 11, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 4084, 0, 0, 4086, 4074, 0, 0, 0, 0, 6, 0, 0, 2188, 1520, 293, 40, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 4083, 4089, 0, 0, 0, 0, 4096, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 13, 0, 0, 4096, 0, 0, 0, 0, 44, 4096, 4096, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3803, 0, 0, 0, 2355, 0, 0, 0, 0, 0, 4096, 3605, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15005, 16320, 16071, 16396, 16509, 16427, 16389, 17955] avg=16384.0000 max=17955 max_rank=7 min=15005 min_rank=0 imbalance_ratio=1.095886 max_load_reduction=113117 max_load_reduction_pct=0.863014 moved_to_echo=67280 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8, 4091, 2, 0, 2819, 4092, 1908, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 2, 0, 49, 0, 0, 5, 4093, 0, 2, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2576, 0, 1, 0, 0, 0, 4092, 4, 0, 0, 3833, 4091, 0, 0, 2053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 4056, 9, 9, 0, 0, 0, 0, 3, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 0, 0, 4092, 4084, 0, 0, 0, 0, 3, 0, 0, 2188, 1520, 293, 40, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 4091, 4092, 0, 0, 0, 0, 4096, 1, 0, 22, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 3, 0, 0, 0, 5, 0, 0, 4096, 4, 0, 0, 0, 3, 4096, 4096, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 0, 0, 0, 213, 0, 0, 0, 0, 0, 4094, 3336, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17016, 16424, 16650, 16365, 16342, 16377, 16360, 15538] avg=16384.0000 max=17016 max_rank=0 min=15538 min_rank=7 imbalance_ratio=1.038574 max_load_reduction=114056 max_load_reduction_pct=0.870178 moved_to_echo=64617 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3, 2, 30, 4080, 2, 0, 1975, 4094, 1906, 1, 0, 3, 4090, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 2, 49, 3, 2, 1, 4065, 0, 7, 4093, 1, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2573, 2, 11, 3, 2, 0, 4093, 8, 0, 0, 2729, 4066, 3, 0, 3407, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4088, 2, 0, 4053, 8, 17, 1, 3, 1, 4, 83, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 109, 4071, 3, 0, 4094, 4036, 3, 3, 3, 0, 0, 0, 0, 2187, 1519, 293, 40, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 4095, 4075, 2, 0, 3, 0, 4070, 0, 0, 51, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 1, 3, 4093, 14, 3, 3, 0, 52, 4093, 4093, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3799, 2, 0, 1, 1564, 0, 0, 1, 0, 4, 4083, 2591, 0, 2, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16192, 16280, 16899, 16445, 16401, 16368, 16392, 16095] avg=16384.0000 max=16899 max_rank=2 min=16095 min_rank=7 imbalance_ratio=1.031433 max_load_reduction=114173 max_load_reduction_pct=0.871071 moved_to_echo=65256 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4093, 0, 0, 3489, 4095, 1902, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 45, 0, 0, 1, 4096, 0, 1, 4088, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2573, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2754, 4095, 0, 1, 1573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 4053, 40, 1, 0, 0, 0, 1, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 4095, 4092, 0, 0, 0, 0, 1, 0, 0, 2194, 1523, 297, 43, 28, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 4095, 4096, 0, 0, 0, 0, 4096, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 2, 0, 0, 4095, 1, 0, 0, 0, 15, 4096, 4096, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3799, 0, 0, 0, 438, 0, 0, 0, 0, 0, 4095, 3988, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17675, 16420, 15092, 16376, 16382, 16361, 16350, 16416] avg=16384.0000 max=17675 max_rank=0 min=15092 min_rank=2 imbalance_ratio=1.078796 max_load_reduction=113397 max_load_reduction_pct=0.865150 moved_to_echo=65509 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 439, 4095, 1908, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 49, 0, 0, 1, 4096, 0, 1, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2576, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3388, 4086, 0, 0, 2934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4056, 35, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4091, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 2188, 1520, 293, 40, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 32, 4096, 4096, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3803, 0, 0, 0, 1648, 0, 0, 0, 0, 0, 4096, 3807, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14634, 16425, 17080, 16383, 16360, 16379, 16357, 17454] avg=16384.0000 max=17454 max_rank=7 min=14634 min_rank=0 imbalance_ratio=1.065308 max_load_reduction=113618 max_load_reduction_pct=0.866837 moved_to_echo=66550 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 2689, 4096, 1908, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 49, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2575, 1, 0, 2, 0, 0, 4096, 0, 0, 0, 3582, 4096, 0, 0, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 4056, 36, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 2188, 1520, 293, 40, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4096, 4096, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 0, 0, 0, 505, 0, 0, 0, 0, 0, 4096, 3986, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16885, 16436, 15828, 16378, 16368, 16360, 16333, 16484] avg=16384.0000 max=16885 max_rank=0 min=15828 min_rank=2 imbalance_ratio=1.030579 max_load_reduction=114187 max_load_reduction_pct=0.871178 moved_to_echo=65545 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4095, 0, 0, 3342, 4096, 1908, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 49, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2575, 1, 0, 1, 0, 0, 4096, 0, 14, 0, 3936, 4096, 0, 6, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 4056, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 2188, 1520, 293, 40, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 4096, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 124, 4096, 4096, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 0, 0, 0, 128, 0, 0, 0, 0, 0, 4096, 3820, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17536, 16430, 15650, 16341, 16362, 16362, 16450, 15941] avg=16384.0000 max=17536 max_rank=0 min=15650 min_rank=2 imbalance_ratio=1.070312 max_load_reduction=113536 max_load_reduction_pct=0.866211 moved_to_echo=65115 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32512, 128, 128, 512, 128, 128, 128, 32768, 0, 128, 32768, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 32768, 128, 0, 17536, 32768, 15360, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 384, 32768, 128, 0, 32768, 32768, 128, 128, 128, 0, 128, 0, 0, 17536, 12288, 2432, 384, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 32768, 128, 128, 32512, 256, 128, 128, 128, 128, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32640, 0, 32768, 32768, 128, 0, 128, 0, 32768, 128, 0, 384, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[30464, 128, 0, 128, 7296, 0, 0, 128, 0, 128, 32768, 28032, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 0, 384, 32768, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[20608, 128, 128, 128, 128, 0, 32768, 128, 128, 0, 28160, 32768, 128, 128, 17024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[3616, 0, 4068, 0, 0, 3977, 2659, 0, 0, 0, 31, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4096, 4095, 0, 4096, 0, 4096, 0, 0, 0, 0, 2, 23, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4055, 4096, 0, 0, 261, 78, 4096, 0, 4050, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 1337, 4096, 4082, 0, 4015, 0, 4096, 0, 0, 0, 0, 0, 447, 0, 3896, 4093, 0, 0, 0, 464, 0, 4096, 0, 0, 3835, 85, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[2840, 0, 4045, 0, 0, 3988, 2002, 0, 0, 0, 24, 37, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 4096, 4085, 0, 4096, 0, 4096, 1, 0, 0, 0, 10, 33, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4089, 0, 0, 3876, 4096, 0, 0, 152, 6, 4096, 11, 4026, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 3402, 4096, 4079, 0, 4078, 0, 4091, 0, 0, 0, 0, 0, 577, 0, 3729, 4088, 0, 0, 0, 207, 0, 4096, 0, 0, 3779, 90, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[2215, 0, 4077, 0, 0, 4077, 3053, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 4094, 0, 4096, 0, 4096, 35, 0, 0, 0, 1, 20, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 0, 0, 4093, 4096, 0, 0, 4036, 4096, 0, 0, 6, 152, 4096, 1, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 4093, 0, 0, 2900, 4095, 4094, 0, 4079, 1, 4095, 0, 0, 1, 0, 0, 164, 0, 4011, 4095, 0, 0, 0, 63, 0, 4096, 0, 0, 3862, 20, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[3210, 1, 3999, 0, 0, 3788, 2582, 7, 0, 3, 81, 20, 0, 2, 1, 0, 0, 3, 8, 3, 0, 0, 2, 3, 0, 3, 3, 1, 0, 4071, 4092, 4079, 1, 4093, 1, 4093, 0, 0, 0, 4, 53, 15, 5, 2, 0, 0, 4089, 4, 0, 3, 4090, 2, 2, 0, 4093, 4093, 4, 3, 3883, 4093, 4, 0, 338, 14, 4092, 12, 3962, 4093, 0, 1, 0, 1, 8, 2, 0, 3, 0, 5, 0, 4091, 0, 0, 3, 3, 3, 1, 3, 0, 19, 0, 3, 4092, 3, 0, 4091, 4043, 0, 3, 1814, 4093, 4039, 0, 4081, 0, 4093, 3, 3, 28, 1, 2, 289, 2, 4009, 4095, 0, 0, 6, 1143, 1, 4092, 0, 0, 3489, 129, 4092, 1, 0, 4086] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[3571, 0, 4092, 0, 0, 4096, 2030, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 12, 0, 0, 0, 0, 18, 3, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4049, 4096, 0, 0, 20, 86, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 2585, 4096, 4096, 0, 4056, 0, 4096, 0, 0, 0, 0, 0, 213, 0, 4014, 4096, 0, 1, 0, 73, 0, 4096, 0, 0, 3828, 14, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[2727, 0, 4086, 0, 0, 4092, 2442, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 4096, 0, 4096, 133, 0, 0, 0, 0, 6, 3, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 0, 0, 3831, 4096, 0, 0, 74, 156, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 2784, 4096, 4096, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 147, 0, 3920, 4096, 0, 0, 0, 219, 0, 4096, 0, 0, 4043, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[3888, 0, 4084, 0, 0, 4058, 772, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 4096, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 169, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 0, 0, 4087, 4096, 0, 0, 107, 28, 4096, 8, 4084, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4092, 0, 0, 3381, 4096, 4088, 0, 4095, 0, 4096, 0, 0, 9, 0, 0, 253, 0, 3939, 4096, 0, 0, 0, 130, 0, 4096, 0, 0, 3758, 17, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[3195, 0, 4074, 0, 0, 4093, 3312, 0, 0, 0, 17, 6, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 4091, 0, 4096, 0, 4096, 30, 0, 0, 0, 4, 40, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 4090, 4096, 0, 0, 4052, 4096, 0, 1, 16, 32, 4096, 7, 4072, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4067, 0, 0, 1891, 4095, 4082, 0, 4068, 1, 4092, 0, 0, 0, 0, 0, 71, 0, 3762, 4089, 0, 0, 0, 128, 0, 4096, 0, 0, 4015, 24, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2840, 0, 4045, 0, 0, 3988, 2002, 0, 0, 0, 24, 37, 0, 0, 0, 0, 2703, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 4096, 4085, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 10, 33, 0, 0, 0, 5, 4096, 0, 3973, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 4096, 4089, 0, 0, 3876, 0, 0, 0, 152, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 4026, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4080, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3402, 1349, 4079, 0, 4078, 0, 4091, 0, 0, 0, 0, 0, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3729, 4088, 0, 0, 0, 207, 0, 0, 0, 0, 3779, 90, 109, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15686, 16361, 16354, 16238, 16355, 16404, 17576, 16098] avg=16384.0000 max=17576 max_rank=6 min=15686 min_rank=0 imbalance_ratio=1.072754 max_load_reduction=113496 max_load_reduction_pct=0.865906 moved_to_echo=66433 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3616, 0, 4068, 0, 0, 3977, 2659, 0, 0, 0, 31, 18, 0, 0, 0, 0, 2703, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4096, 4095, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 2, 23, 0, 0, 0, 0, 4096, 0, 3973, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 4096, 4096, 0, 0, 4055, 0, 0, 0, 261, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4050, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4079, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1337, 1349, 4082, 0, 4015, 0, 4096, 0, 0, 0, 0, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 4093, 0, 0, 0, 464, 0, 0, 0, 0, 3835, 85, 109, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17119, 16363, 16330, 16605, 16368, 16383, 15326, 16578] avg=16384.0000 max=17119 max_rank=0 min=15326 min_rank=6 imbalance_ratio=1.044861 max_load_reduction=113953 max_load_reduction_pct=0.869392 moved_to_echo=64655 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3210, 1, 3999, 0, 0, 3788, 2582, 7, 0, 3, 81, 20, 0, 2, 1, 0, 2701, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 3, 0, 0, 2, 3, 0, 3, 3, 1, 0, 4071, 4092, 4079, 4093, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 1, 4093, 0, 0, 0, 4, 53, 15, 5, 2, 0, 0, 4089, 4, 3969, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4014, 2, 2, 0, 4093, 4093, 4, 3, 3883, 0, 4, 0, 338, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 12, 3962, 4093, 0, 1, 0, 1, 8, 2, 0, 3, 0, 5, 0, 4091, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 3, 0, 19, 0, 3, 4092, 3, 0, 4091, 4043, 4076, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1814, 1348, 4039, 0, 4081, 0, 4093, 3, 3, 28, 1, 2, 289, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 4095, 0, 0, 6, 1143, 1, 0, 0, 0, 3489, 129, 109, 1, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16441, 16377, 16373, 16453, 16300, 16354, 15706, 17068] avg=16384.0000 max=17068 max_rank=7 min=15706 min_rank=6 imbalance_ratio=1.041748 max_load_reduction=114004 max_load_reduction_pct=0.869781 moved_to_echo=65428 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2215, 0, 4077, 0, 0, 4077, 3053, 0, 0, 0, 5, 0, 0, 0, 0, 0, 2705, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 4094, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 35, 0, 0, 0, 1, 20, 0, 0, 0, 0, 4096, 0, 3980, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4016, 0, 0, 0, 4093, 4096, 0, 0, 4036, 0, 0, 0, 6, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 4093, 4079, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2900, 1343, 4094, 0, 4079, 1, 4095, 0, 0, 1, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 4095, 0, 0, 0, 63, 0, 0, 0, 0, 3862, 20, 99, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16181, 16395, 16371, 16400, 16412, 16390, 16677, 16246] avg=16384.0000 max=16677 max_rank=6 min=16181 min_rank=0 imbalance_ratio=1.017883 max_load_reduction=114395 max_load_reduction_pct=0.872765 moved_to_echo=65725 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3571, 0, 4092, 0, 0, 4096, 2030, 0, 0, 0, 1, 3, 0, 0, 0, 0, 2703, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 12, 0, 0, 0, 0, 18, 3, 0, 0, 0, 4096, 0, 3973, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4019, 0, 0, 0, 4096, 4096, 0, 0, 4049, 0, 0, 0, 20, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4079, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2585, 1349, 4096, 0, 4056, 0, 4096, 0, 0, 0, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 4096, 0, 1, 0, 73, 0, 0, 0, 0, 3828, 14, 109, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16543, 16401, 16338, 16367, 16413, 16384, 16395, 16231] avg=16384.0000 max=16543 max_rank=0 min=16231 min_rank=7 imbalance_ratio=1.009705 max_load_reduction=114529 max_load_reduction_pct=0.873787 moved_to_echo=65423 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3195, 0, 4074, 0, 0, 4093, 3312, 0, 0, 0, 17, 6, 0, 0, 0, 0, 2702, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 4091, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 30, 0, 0, 0, 4, 40, 0, 0, 0, 0, 4096, 0, 3973, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 4090, 4096, 0, 0, 4052, 0, 0, 1, 16, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 4072, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4067, 4079, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1891, 1349, 4082, 0, 4068, 1, 4092, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3762, 4089, 0, 0, 0, 128, 0, 0, 0, 0, 4015, 24, 109, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17445, 16382, 16379, 16306, 16398, 16385, 15554, 16223] avg=16384.0000 max=17445 max_rank=0 min=15554 min_rank=6 imbalance_ratio=1.064758 max_load_reduction=113627 max_load_reduction_pct=0.866905 moved_to_echo=64560 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2727, 0, 4086, 0, 0, 4092, 2442, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2703, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 133, 0, 0, 0, 0, 6, 3, 0, 0, 0, 4096, 0, 3973, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 4095, 4096, 0, 0, 3831, 0, 0, 0, 74, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4080, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2784, 1349, 4096, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3920, 4096, 0, 0, 0, 219, 0, 0, 0, 0, 4043, 0, 109, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16098, 16399, 16447, 16271, 16416, 16391, 16567, 16483] avg=16384.0000 max=16567 max_rank=6 min=16098 min_rank=0 imbalance_ratio=1.011169 max_load_reduction=114505 max_load_reduction_pct=0.873604 moved_to_echo=65857 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3888, 0, 4084, 0, 0, 4058, 772, 0, 0, 0, 7, 0, 0, 0, 0, 0, 2703, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 4096, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 169, 0, 0, 0, 0, 4096, 0, 3973, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 4095, 4096, 0, 0, 4087, 0, 0, 0, 107, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 4084, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4092, 4079, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3381, 1349, 4088, 0, 4095, 0, 4096, 0, 0, 9, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3939, 4096, 0, 0, 0, 130, 0, 0, 0, 0, 3758, 17, 109, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15559, 16394, 16480, 16432, 16410, 16381, 17271, 16145] avg=16384.0000 max=17271 max_rank=6 min=15559 min_rank=0 imbalance_ratio=1.054138 max_load_reduction=113801 max_load_reduction_pct=0.868233 moved_to_echo=66207 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[31360, 32768, 0, 128, 128, 2432, 128, 0, 0, 0, 30720, 384, 896, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 32768, 256, 0, 0, 128, 128, 384, 128, 128, 0, 128, 32768, 128, 31872, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 32256, 128, 128, 0, 32768, 32768, 128, 128, 31872, 0, 128, 128, 1024, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[25344, 128, 32640, 0, 0, 32256, 18944, 128, 0, 128, 256, 128, 0, 128, 128, 128, 21632, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 32512, 32768, 0, 128, 0, 128, 128, 128, 0, 128, 0, 128, 0, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 0, 128, 0, 128, 32768, 128, 128, 32768, 32768, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 0, 0, 128, 128, 0, 128, 128, 128, 0, 32640, 32768, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 20096, 10880, 32768, 0, 32640, 128, 32768, 128, 128, 128, 128, 128, 2176, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 67, 0, 3879, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 0, 0, 4074, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4064, 0, 0, 3392, 3918, 0, 0, 0, 1347, 4066, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 3979, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4094, 4096, 0, 0, 161, 0, 3899, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4095, 1, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 2794, 4047, 0, 0, 0, 1608, 4095, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 4083, 0, 0, 3, 0, 3, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4093, 0, 3, 3, 3, 0, 4093, 4070, 4093, 3, 0, 31, 1, 3970, 1, 0, 0, 0, 3, 0, 3, 5, 1, 4093, 0, 3, 4093, 4093, 2, 0, 0, 11, 4093, 4093, 4092, 0, 10, 22, 0, 3, 0, 4093, 0, 0, 0, 10, 3, 4093, 4087, 4073, 114, 7, 0, 0, 3, 0, 0, 3, 3, 3965, 1, 3, 0, 4049, 0, 0, 4093, 2, 3, 4093, 1, 3, 0, 25, 3, 0, 3, 4079, 4017, 4, 3, 3817, 4063, 0, 4, 0, 563, 4081, 3, 4092, 4093, 2, 0, 3, 4093, 4090, 0, 3, 0, 3, 1, 0, 0, 4093, 0, 4093, 0, 0, 1, 3946, 0, 0, 75, 0, 0, 3, 0, 3, 23, 1, 0, 0, 4092, 0, 0, 8, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 70, 0, 3788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3013, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2368, 3807, 0, 0, 0, 3350, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4077, 0, 0, 3, 0, 1, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4085, 4094, 0, 0, 67, 0, 3918, 0, 0, 0, 11, 0, 15, 0, 0, 0, 4093, 11, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4090, 4096, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 4096, 4096, 81, 0, 0, 2, 0, 0, 0, 0, 0, 3765, 0, 0, 0, 4075, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 4025, 0, 0, 3026, 4007, 0, 0, 0, 1852, 4080, 0, 4096, 4096, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4096, 0, 0, 0, 3864, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4094, 4096, 0, 0, 178, 0, 3883, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4085, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4095, 41, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 3202, 2884, 0, 0, 0, 2307, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 3948, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 24, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3689, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3928, 3931, 0, 0, 0, 773, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4057, 0, 0, 0, 0, 4, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4093, 4089, 4096, 0, 0, 298, 0, 3925, 0, 0, 0, 4, 0, 6, 0, 0, 0, 4096, 4, 0, 4096, 4096, 0, 0, 0, 0, 4095, 4092, 4096, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 4095, 4094, 11, 0, 0, 20, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4072, 0, 0, 2084, 3799, 0, 0, 0, 2304, 4089, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 1, 0, 0, 4061, 1, 0, 1, 0, 8, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 1, 4, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[76, 0, 3, 3, 3, 0, 4093, 4070, 4093, 3, 0, 31, 1, 3970, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 5, 1, 4093, 0, 3, 4093, 4093, 2, 0, 0, 11, 3939, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4087, 0, 10, 22, 0, 3, 0, 4093, 0, 0, 0, 10, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4073, 114, 7, 0, 0, 3, 0, 0, 3, 3, 3965, 1, 3, 0, 4049, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 3, 4093, 1, 3, 0, 25, 3, 0, 3, 4079, 4017, 4, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3817, 4063, 0, 4, 0, 563, 4081, 3, 0, 0, 2, 0, 3, 0, 3645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 1, 0, 0, 4093, 0, 4093, 0, 0, 1, 3946, 0, 0, 4093, 78, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 3, 0, 3, 23, 1, 0, 0, 4092, 0, 0, 8, 0, 3, 4093, 4093, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16347, 16389, 16414, 16589, 16352, 16184, 16330, 16467] avg=16384.0000 max=16589 max_rank=3 min=16184 min_rank=5 imbalance_ratio=1.012512 max_load_reduction=114483 max_load_reduction_pct=0.873436 moved_to_echo=65333 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[75, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 67, 0, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 3942, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 0, 0, 4074, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4064, 0, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3392, 3918, 0, 0, 0, 1347, 4066, 0, 0, 0, 0, 0, 0, 0, 3650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 3979, 0, 0, 4096, 79, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16309, 16393, 16379, 16508, 16378, 16373, 16365, 16367] avg=16384.0000 max=16508 max_rank=3 min=16309 min_rank=0 imbalance_ratio=1.007568 max_load_reduction=114564 max_load_reduction_pct=0.874054 moved_to_echo=65483 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[75, 0, 0, 0, 0, 0, 4095, 4085, 4094, 0, 0, 67, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 15, 0, 0, 0, 4093, 11, 0, 4096, 4096, 0, 0, 0, 0, 3942, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4091, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 81, 0, 0, 2, 0, 0, 0, 0, 0, 3765, 0, 0, 0, 4075, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 4025, 0, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3026, 4007, 0, 0, 0, 1852, 4080, 0, 0, 0, 0, 0, 0, 0, 3650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4096, 0, 0, 0, 3864, 0, 0, 4095, 79, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16334, 16408, 16382, 16396, 16337, 16615, 16231, 16369] avg=16384.0000 max=16615 max_rank=5 min=16231 min_rank=6 imbalance_ratio=1.014099 max_load_reduction=114457 max_load_reduction_pct=0.873238 moved_to_echo=65552 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[75, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 70, 0, 3788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 0, 3942, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3013, 0, 0, 0, 4096, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2368, 3807, 0, 0, 0, 3350, 4096, 0, 0, 0, 0, 0, 0, 0, 3650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4077, 0, 0, 4096, 79, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16221, 16373, 16381, 15583, 16408, 17271, 16464, 16371] avg=16384.0000 max=17271 max_rank=5 min=15583 min_rank=3 imbalance_ratio=1.054138 max_load_reduction=113801 max_load_reduction_pct=0.868233 moved_to_echo=66514 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[69, 0, 0, 0, 0, 0, 4096, 4094, 4096, 0, 0, 161, 0, 3899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4095, 1, 0, 4096, 4096, 0, 0, 0, 0, 3947, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4090, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 0, 0, 0, 4095, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 22, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2794, 4047, 0, 0, 0, 1608, 4095, 0, 0, 0, 0, 0, 0, 0, 3642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 4083, 0, 0, 4096, 80, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16415, 16383, 16377, 16463, 16407, 16186, 16470, 16371] avg=16384.0000 max=16470 max_rank=6 min=16186 min_rank=5 imbalance_ratio=1.005249 max_load_reduction=114602 max_load_reduction_pct=0.874344 moved_to_echo=65434 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[75, 0, 0, 0, 0, 0, 4096, 4094, 4096, 0, 0, 178, 0, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4085, 0, 0, 4096, 4096, 0, 0, 0, 0, 3942, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 41, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 4089, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3202, 2884, 0, 0, 0, 2307, 4096, 0, 0, 0, 0, 0, 0, 0, 3650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 3948, 0, 0, 4096, 79, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16422, 16364, 16379, 16678, 16395, 16139, 16324, 16371] avg=16384.0000 max=16678 max_rank=3 min=16139 min_rank=5 imbalance_ratio=1.017944 max_load_reduction=114394 max_load_reduction_pct=0.872757 moved_to_echo=65229 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[75, 0, 0, 0, 0, 0, 4093, 4089, 4096, 0, 0, 298, 0, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6, 0, 0, 0, 4096, 4, 0, 4096, 4096, 0, 0, 0, 0, 3942, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4091, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4094, 11, 0, 0, 20, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 4086, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4072, 0, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2084, 3799, 0, 0, 0, 2304, 4089, 0, 0, 0, 0, 0, 0, 0, 3650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 1, 0, 0, 4061, 1, 0, 4096, 79, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 1, 4, 0, 0, 4096, 4095, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16576, 16388, 16377, 16596, 16385, 15926, 16445, 16379] avg=16384.0000 max=16596 max_rank=3 min=15926 min_rank=5 imbalance_ratio=1.012939 max_load_reduction=114476 max_load_reduction_pct=0.873383 moved_to_echo=65135 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[75, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 24, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 3942, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3689, 0, 0, 0, 4096, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 21, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3928, 3931, 0, 0, 0, 773, 4096, 0, 0, 0, 0, 0, 0, 0, 3650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4057, 0, 0, 4096, 79, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16448, 16374, 16383, 16259, 16410, 16378, 16443, 16377] avg=16384.0000 max=16448 max_rank=0 min=16259 min_rank=3 imbalance_ratio=1.003906 max_load_reduction=114624 max_load_reduction_pct=0.874512 moved_to_echo=65608 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 24704, 30464, 0, 128, 0, 14208, 32768, 128, 0, 0, 128, 0, 128, 0, 29312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 32768, 0, 128, 128, 0, 128, 0, 32768, 128, 0, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[640, 0, 128, 128, 128, 0, 32768, 32768, 32768, 128, 0, 896, 128, 31360, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 32768, 128, 128, 32768, 128, 128, 0, 128, 128, 0, 128, 32768, 32640, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 32768, 384, 128, 0, 128, 128, 0, 0, 128, 128, 30464, 128, 128, 0, 32768, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 32768, 0, 32768, 128, 0, 128, 32128, 128, 0, 32768, 640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 32768, 128, 0, 0, 128, 31616, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 0, 128, 128, 128, 0, 0, 32768, 0, 128, 128, 0, 128, 32768, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 57, 4096, 4096, 0, 0, 0, 0, 0, 3123, 4046, 4096, 0, 1054, 4049, 0, 0, 4096, 0, 0, 0, 0, 4096, 4069, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4, 4096, 4096, 4095, 4096, 0, 4096, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 18, 4096, 0, 384, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 4051, 1, 0, 19, 4093, 0, 51, 4075, 4096, 0, 0, 0, 0, 0, 1748, 4022, 4096, 0, 1975, 4063, 0, 0, 4096, 0, 0, 0, 0, 4095, 4093, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 22, 72, 4096, 4096, 4086, 4095, 0, 4096, 0, 7, 0, 4086, 0, 4096, 3, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 0, 0, 4066, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[3, 0, 0, 4093, 3, 0, 3, 0, 4065, 0, 3, 0, 0, 2, 0, 67, 0, 0, 3, 4093, 3, 3, 4092, 3, 3, 3, 46, 4093, 0, 35, 3, 0, 10, 0, 0, 3959, 0, 3, 2, 8, 3, 0, 4064, 0, 0, 10, 4094, 2, 293, 4071, 4091, 15, 0, 4, 2, 0, 3702, 4037, 4090, 0, 266, 4093, 0, 0, 4092, 0, 0, 0, 0, 4093, 4005, 0, 0, 1, 4093, 4, 4, 0, 4092, 0, 4093, 4092, 3, 0, 41, 4093, 4090, 4093, 4092, 0, 4091, 2, 16, 3, 4051, 3, 4091, 1, 3, 2, 4, 4093, 0, 3, 0, 63, 1, 3, 0, 0, 4092, 5, 1, 4015, 4093, 0, 1, 3, 0, 2, 3, 5, 3, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 82, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 24, 4092, 4096, 0, 0, 0, 0, 0, 3477, 4091, 4096, 0, 584, 4027, 0, 0, 4096, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 2, 36, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 4076, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 4096, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 4083, 0, 0, 12, 4096, 0, 100, 4096, 4096, 0, 0, 0, 0, 0, 2667, 4070, 4096, 0, 1386, 4023, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 2, 68, 4096, 4096, 4095, 4089, 0, 4096, 0, 0, 0, 4075, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 4096, 0, 0, 4054, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 12, 4095, 0, 18, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 4082, 0, 0, 7, 4090, 0, 38, 4092, 4096, 3, 0, 0, 0, 0, 3812, 4090, 4096, 0, 313, 4010, 0, 0, 4096, 0, 0, 1, 0, 4093, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 6, 20, 4096, 4093, 4093, 4094, 0, 4096, 0, 6, 0, 4089, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 4085, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3, 4096, 0, 16, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 5, 4095, 0, 12, 4095, 4096, 0, 0, 0, 0, 0, 3755, 4083, 4096, 0, 266, 4067, 0, 0, 4096, 0, 0, 1, 0, 4096, 4095, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4, 89, 4096, 4095, 4094, 4096, 0, 4096, 0, 1, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 4095, 0, 0, 0, 4096, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 3779, 4096, 4096, 0, 342, 4083, 0, 0, 4096, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 10, 4096, 4096, 4096, 4096, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 4093, 3, 0, 3, 0, 4065, 0, 3, 0, 0, 2, 0, 67, 4091, 4072, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 3, 3, 4092, 3, 3, 3, 46, 4093, 0, 35, 3, 0, 3979, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 3959, 0, 3, 2, 8, 3, 0, 4064, 0, 0, 10, 4094, 2, 4081, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 4071, 0, 15, 0, 4, 2, 0, 3702, 4037, 70, 0, 266, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 4093, 4005, 0, 0, 1, 4093, 4, 4, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 41, 0, 4057, 4093, 4092, 0, 0, 2, 16, 3, 4051, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 3, 2, 4, 4093, 0, 3, 0, 63, 1, 3, 0, 0, 4092, 5, 4037, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4015, 4093, 0, 1, 3, 0, 2, 3, 5, 3, 0, 0, 0, 0, 0, 4093, 4093, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 16392, 16271, 16553, 16312, 16361, 16439, 16332] avg=16384.0000 max=16553 max_rank=3 min=16271 min_rank=2 imbalance_ratio=1.010315 max_load_reduction=114519 max_load_reduction_pct=0.873711 moved_to_echo=65444 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 77, 4096, 4076, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 18, 4096, 0, 384, 0, 0, 3984, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 4051, 1, 0, 19, 4093, 0, 4086, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 4075, 0, 0, 0, 0, 0, 0, 1748, 4022, 71, 0, 1975, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 4095, 4093, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 72, 0, 4063, 4086, 4095, 0, 0, 0, 7, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 0, 4041, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 4096, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16405, 16707, 16364, 16005, 16400, 16431, 16381, 16379] avg=16384.0000 max=16707 max_rank=1 min=16005 min_rank=3 imbalance_ratio=1.019714 max_load_reduction=114365 max_load_reduction_pct=0.872536 moved_to_echo=65591 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 32, 4096, 4076, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 3984, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 4083, 0, 0, 12, 4096, 0, 4085, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 4096, 0, 0, 0, 0, 0, 0, 2667, 4070, 71, 0, 1386, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 68, 0, 4063, 4095, 4089, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 4096, 0, 4041, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16405, 16306, 16407, 16413, 16402, 16392, 16385, 16362] avg=16384.0000 max=16413 max_rank=3 min=16306 min_rank=1 imbalance_ratio=1.001770 max_load_reduction=114659 max_load_reduction_pct=0.874779 moved_to_echo=65541 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 4096, 4077, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 3, 0, 0, 3984, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 4085, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4096, 0, 0, 0, 0, 0, 0, 3123, 4046, 71, 0, 1054, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 4096, 4069, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4063, 4095, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4041, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16363, 16309, 16410, 16496, 16370, 16351, 16370, 16403] avg=16384.0000 max=16496 max_rank=3 min=16309 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=114576 max_load_reduction_pct=0.874146 moved_to_echo=65494 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 11, 4096, 4076, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 82, 0, 0, 3984, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 4085, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4092, 0, 0, 0, 0, 0, 0, 3477, 4091, 71, 0, 584, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 36, 0, 4063, 4096, 4096, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 4096, 0, 4041, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 16387, 16398, 16366, 16400, 16369, 16379, 16400] avg=16384.0000 max=16400 max_rank=4 min=16366 min_rank=3 imbalance_ratio=1.000977 max_load_reduction=114672 max_load_reduction_pct=0.874878 moved_to_echo=65548 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 11, 4096, 4076, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 12, 4095, 0, 18, 0, 0, 3984, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 4082, 0, 0, 7, 4090, 0, 4085, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4092, 0, 3, 0, 0, 0, 0, 3812, 4090, 71, 0, 313, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 1, 0, 4093, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 20, 0, 4060, 4093, 4094, 0, 0, 0, 6, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 4041, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 4096, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16371, 16334, 16395, 16429, 16403, 16368, 16375, 16397] avg=16384.0000 max=16429 max_rank=3 min=16334 min_rank=1 imbalance_ratio=1.002747 max_load_reduction=114643 max_load_reduction_pct=0.874657 moved_to_echo=65543 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 9, 4096, 4077, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3, 4096, 0, 16, 0, 0, 3990, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 5, 4095, 0, 4085, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4095, 0, 0, 0, 0, 0, 0, 3755, 4083, 64, 0, 266, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 1, 0, 4096, 4095, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 89, 0, 4060, 4094, 4096, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4042, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16332, 16417, 16342, 16394, 16439, 16373, 16395] avg=16384.0000 max=16439 max_rank=5 min=16332 min_rank=1 imbalance_ratio=1.003357 max_load_reduction=114633 max_load_reduction_pct=0.874580 moved_to_echo=65601 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4096, 4076, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3984, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 4095, 0, 0, 0, 4096, 0, 4085, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 3779, 4096, 71, 0, 342, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4063, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4041, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16363, 16305, 16410, 16468, 16391, 16361, 16370, 16404] avg=16384.0000 max=16468 max_rank=3 min=16305 min_rank=1 imbalance_ratio=1.005127 max_load_reduction=114604 max_load_reduction_pct=0.874359 moved_to_echo=65526 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32640, 32768, 0, 128, 128, 0, 128, 128, 128, 128, 0, 0, 128, 0, 0, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 32768, 0, 640, 128, 0, 32000, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 128, 0, 32640, 0, 128, 0, 0, 128, 0, 256, 32768, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 128, 128, 32768, 0, 128, 0, 128, 128, 128, 0, 0, 32768, 128, 32384, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 384, 0, 32512, 32768, 32768, 0, 0, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 0, 32640, 128, 128, 128, 128, 128, 0, 32768, 128, 0, 128, 32768, 128, 32768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[256, 0, 0, 128, 0, 32768, 32640, 128, 0, 128, 32768, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[640, 32768, 0, 128, 0, 128, 128, 0, 26112, 32640, 640, 0, 6272, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4040, 0, 0, 77, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 1244, 0, 0, 0, 0, 4081, 259, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 3978, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 2252, 0, 4096, 0, 0, 1, 0, 0, 0, 206, 4096, 0, 10, 0, 0, 0, 0, 0, 3341, 4096, 0, 0, 0, 0, 976, 0, 4096, 0, 0, 4081, 0, 10, 0, 0, 7, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4013, 3, 0, 0, 0, 3, 4092, 0, 3, 3, 0, 3, 3, 1, 4092, 3, 1291, 4, 0, 0, 0, 4060, 842, 0, 1, 0, 4080, 4, 0, 0, 0, 0, 1, 4077, 3746, 0, 4093, 1, 4, 4093, 2, 4079, 4092, 1, 0, 3, 4067, 4, 3, 12, 1, 0, 3, 14, 4092, 0, 2, 4, 0, 0, 3, 4068, 4093, 4, 1, 44, 3, 0, 3, 3, 0, 1, 4093, 0, 4091, 4089, 3, 3, 0, 20, 4092, 1, 4093, 2240, 1, 4092, 4, 2, 0, 3, 0, 0, 691, 4092, 19, 0, 3, 3, 4, 3, 0, 2981, 4092, 0, 0, 4, 3, 342, 0, 4092, 3, 2, 4092, 0, 183, 0, 3, 57, 4092, 0, 0, 4092, 0, 0, 4092, 4089, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[3961, 0, 0, 81, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2755, 0, 0, 0, 0, 4074, 201, 0, 0, 0, 4088, 0, 0, 0, 0, 2, 0, 4095, 3867, 0, 4091, 0, 0, 4096, 0, 4088, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 18, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1328, 0, 4096, 0, 0, 0, 0, 0, 0, 120, 4096, 6, 11, 0, 0, 0, 5, 0, 3612, 4096, 0, 0, 0, 0, 466, 0, 4096, 0, 0, 4083, 0, 1, 11, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4092, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4092, 0, 0, 71, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 1553, 0, 0, 0, 0, 4096, 452, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3739, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1659, 0, 4096, 0, 0, 1, 0, 0, 0, 128, 4096, 0, 0, 0, 0, 0, 0, 0, 3615, 4096, 0, 0, 0, 0, 1067, 0, 4096, 0, 0, 4096, 0, 2, 0, 0, 3, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4037, 0, 0, 39, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 578, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4019, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 2255, 0, 4096, 0, 0, 0, 0, 0, 0, 932, 4096, 0, 0, 0, 0, 0, 0, 0, 3382, 4096, 0, 0, 0, 0, 1124, 0, 4096, 0, 0, 4093, 0, 1, 1, 0, 3, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4077, 0, 0, 37, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 1474, 0, 0, 0, 0, 4095, 460, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 4095, 3914, 0, 4095, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1435, 0, 4096, 0, 0, 2, 0, 0, 0, 892, 4096, 1, 5, 0, 0, 0, 0, 0, 3798, 4096, 0, 0, 0, 0, 285, 0, 4095, 0, 0, 4095, 0, 6, 1, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4082, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1213, 0, 0, 0, 0, 4096, 358, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4047, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 2457, 0, 4096, 0, 0, 0, 0, 0, 0, 210, 4096, 0, 0, 0, 0, 0, 0, 0, 3872, 4096, 0, 0, 0, 0, 96, 0, 4096, 0, 0, 4096, 0, 40, 0, 0, 9, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4031, 0, 0, 21, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 1387, 0, 0, 0, 0, 4087, 318, 0, 0, 0, 4092, 0, 0, 0, 0, 5, 0, 4091, 4052, 0, 4093, 0, 0, 4096, 0, 4091, 4095, 1, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 2, 0, 0, 4096, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1561, 0, 4096, 0, 0, 0, 0, 0, 0, 731, 4096, 5, 0, 0, 0, 0, 0, 3, 3832, 4096, 0, 0, 1, 0, 397, 0, 4094, 0, 0, 4091, 0, 12, 5, 0, 51, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4089, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4040, 0, 0, 77, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 2321, 1787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1244, 0, 0, 0, 0, 4081, 259, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 2309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3978, 0, 185, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2319, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2307, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1702, 2252, 0, 4096, 0, 0, 1, 0, 0, 0, 206, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3341, 4096, 0, 0, 0, 0, 976, 0, 4096, 0, 0, 2394, 1604, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 10, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4054, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16418, 16083, 16451, 16382, 16386, 16459, 16549, 16344] avg=16384.0000 max=16549 max_rank=6 min=16083 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=114523 max_load_reduction_pct=0.873741 moved_to_echo=65738 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3961, 0, 0, 81, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2320, 1787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2755, 0, 0, 0, 0, 4074, 201, 0, 0, 0, 4088, 0, 0, 0, 0, 2, 4096, 2309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3867, 0, 185, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 2318, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 2304, 1778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1702, 1328, 0, 4096, 0, 0, 0, 0, 0, 0, 120, 4096, 6, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3612, 4096, 0, 0, 0, 0, 466, 0, 4096, 0, 0, 2394, 1602, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 1, 11, 0, 1, 0, 0, 0, 0, 0, 0, 4054, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16341, 17525, 16331, 16383, 16386, 15455, 16313, 16338] avg=16384.0000 max=17525 max_rank=1 min=15455 min_rank=5 imbalance_ratio=1.069641 max_load_reduction=113547 max_load_reduction_pct=0.866295 moved_to_echo=64492 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4013, 3, 0, 0, 0, 3, 4092, 0, 3, 3, 0, 3, 3, 1, 4092, 3, 2318, 1785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1291, 4, 0, 0, 0, 4060, 842, 0, 1, 0, 4080, 4, 0, 0, 0, 0, 4093, 2307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4077, 3746, 0, 185, 1, 4, 0, 2, 4079, 0, 1, 0, 3, 4067, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 12, 1, 0, 3, 14, 4092, 0, 2, 4, 0, 0, 3, 4068, 4093, 4, 2316, 1774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 44, 3, 0, 3, 3, 0, 1, 4093, 0, 4091, 4089, 3, 3, 0, 20, 2305, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1701, 2240, 1, 4092, 4, 2, 0, 3, 0, 0, 691, 4092, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 4, 3, 0, 2981, 4092, 0, 0, 4, 3, 342, 0, 4092, 3, 2, 2392, 1603, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 183, 0, 3, 57, 0, 0, 0, 0, 0, 0, 4050, 4089, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16322, 16682, 16170, 16389, 16435, 16938, 15569, 16567] avg=16384.0000 max=16938 max_rank=5 min=15569 min_rank=6 imbalance_ratio=1.033813 max_load_reduction=114134 max_load_reduction_pct=0.870773 moved_to_echo=65509 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4037, 0, 0, 39, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2321, 1787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4019, 0, 185, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2319, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2307, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1702, 2255, 0, 4096, 0, 0, 0, 0, 0, 0, 932, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3382, 4096, 0, 0, 0, 0, 1124, 0, 4096, 0, 0, 2394, 1604, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 1, 0, 3, 0, 0, 0, 0, 0, 0, 4054, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16376, 15189, 16492, 16382, 16374, 17177, 16738, 16344] avg=16384.0000 max=17177 max_rank=5 min=15189 min_rank=1 imbalance_ratio=1.048401 max_load_reduction=113895 max_load_reduction_pct=0.868950 moved_to_echo=66633 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 71, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 2321, 1787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1553, 0, 0, 0, 0, 4096, 452, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3739, 0, 185, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2319, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2307, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1702, 1659, 0, 4096, 0, 0, 1, 0, 0, 0, 128, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3615, 4096, 0, 0, 0, 0, 1067, 0, 4096, 0, 0, 2394, 1604, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4054, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16466, 16602, 16211, 16382, 16372, 15778, 16914, 16347] avg=16384.0000 max=16914 max_rank=6 min=15778 min_rank=5 imbalance_ratio=1.032349 max_load_reduction=114158 max_load_reduction_pct=0.870956 moved_to_echo=65411 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4031, 0, 0, 21, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 2321, 1787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1387, 0, 0, 0, 0, 4087, 318, 0, 0, 0, 4092, 0, 0, 0, 0, 5, 4096, 2309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4052, 0, 185, 0, 0, 0, 0, 4091, 0, 1, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 2, 0, 0, 4096, 4096, 0, 2319, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 4096, 0, 0, 0, 0, 2305, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1702, 1561, 0, 4096, 0, 0, 0, 0, 0, 0, 731, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3832, 4096, 0, 0, 1, 0, 397, 0, 4094, 0, 0, 2394, 1603, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 12, 5, 0, 51, 0, 0, 0, 0, 0, 0, 4054, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16352, 16294, 16519, 16388, 16372, 16287, 16462, 16398] avg=16384.0000 max=16519 max_rank=2 min=16287 min_rank=5 imbalance_ratio=1.008240 max_load_reduction=114553 max_load_reduction_pct=0.873970 moved_to_echo=65519 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4077, 0, 0, 37, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 2321, 1788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1474, 0, 0, 0, 0, 4095, 460, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4096, 2308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3914, 0, 178, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 2319, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 2309, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1698, 1435, 0, 4096, 0, 0, 2, 0, 0, 0, 892, 4096, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 4096, 0, 0, 0, 0, 285, 0, 4095, 0, 0, 2398, 1608, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 6, 1, 0, 1, 0, 0, 0, 0, 0, 0, 4049, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16416, 16529, 16378, 16384, 16374, 16321, 16327, 16343] avg=16384.0000 max=16529 max_rank=1 min=16321 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=114543 max_load_reduction_pct=0.873894 moved_to_echo=65365 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4082, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2320, 1787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1213, 0, 0, 0, 0, 4096, 358, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4047, 0, 185, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2318, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2307, 1778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1702, 2457, 0, 4096, 0, 0, 0, 0, 0, 0, 210, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3872, 4096, 0, 0, 0, 0, 96, 0, 4096, 0, 0, 2394, 1604, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 40, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4054, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16168, 16520, 16382, 16373, 16657, 16200, 16391] avg=16384.0000 max=16657 max_rank=5 min=16168 min_rank=1 imbalance_ratio=1.016663 max_load_reduction=114415 max_load_reduction_pct=0.872917 moved_to_echo=65621 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 31488, 0, 1536, 128, 128, 0, 128, 32768, 0, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 13696, 15232, 128, 32768, 128, 128, 128, 128, 0, 0, 3968, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32384, 128, 0, 384, 0, 128, 32768, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 18688, 14336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[11520, 128, 0, 0, 0, 32768, 2944, 0, 128, 0, 32768, 128, 0, 0, 0, 128, 32768, 18560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 28544, 32768, 0, 0, 128, 128, 4864, 0, 32768, 128, 128, 19200, 12928, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 128, 32768, 0, 32768, 32768, 128, 128, 0, 128, 18560, 14336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 0, 128, 128, 128, 0, 128, 32768, 32768, 128, 18560, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 0, 256, 128, 128, 256, 0, 0, 0, 0, 0, 0, 32512, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3925, 4096, 0, 0, 0, 0, 0, 4096, 0, 29, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4072, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 4045, 0, 0, 4090, 0, 62, 0, 199, 0, 0, 0, 769, 0, 0, 3317, 0, 3915, 4096, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4090, 0, 92, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 3961, 4088, 4, 3, 0, 0, 0, 4093, 11, 72, 17, 3, 4093, 4091, 3, 0, 3, 0, 1, 4094, 2, 3, 0, 3, 4092, 0, 0, 0, 4, 58, 4, 4, 0, 4, 3, 4093, 3, 1, 4093, 0, 159, 0, 1965, 4, 0, 0, 3686, 0, 0, 701, 0, 1413, 4092, 12, 2, 4093, 4068, 0, 0, 0, 3, 0, 3, 4092, 0, 0, 0, 2, 0, 0, 3, 3, 269, 4090, 0, 4092, 3972, 0, 1, 0, 0, 0, 3, 3, 2, 4093, 0, 0, 4078, 4, 1, 3, 4093, 5, 4041, 0, 333, 0, 5, 0, 3, 3942, 0, 4044, 4082, 0, 4092, 0, 0, 0, 4093, 5, 0, 3, 4093, 4092, 4092, 0, 4090, 49, 0, 0, 4, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3525, 4096, 0, 0, 0, 0, 0, 4095, 0, 3, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 4072, 0, 0, 4095, 0, 7, 0, 28, 0, 0, 0, 2884, 0, 0, 1874, 0, 3599, 4096, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 291, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 4096, 0, 4096, 1, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3581, 4096, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 4069, 0, 0, 4088, 0, 1, 1, 44, 0, 0, 0, 2556, 0, 0, 2071, 0, 3658, 4095, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 7, 4096, 2, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 348, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 4094, 4096, 1, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 0, 4096, 0, 25, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4071, 0, 0, 20, 0, 4096, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 4028, 0, 0, 4058, 0, 106, 8, 53, 0, 0, 0, 949, 0, 0, 3104, 0, 3877, 4095, 0, 0, 4087, 4069, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 45, 4096, 5, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4084, 0, 142, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 4090, 4096, 5, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3864, 4096, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 4077, 0, 0, 4086, 0, 19, 0, 296, 0, 0, 0, 1965, 0, 0, 2550, 0, 3503, 4096, 1, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4092, 0, 54, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3408, 4095, 0, 0, 0, 0, 1, 4096, 0, 42, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 50, 0, 0, 4, 0, 0, 4058, 0, 0, 4091, 0, 16, 6, 64, 0, 0, 0, 2361, 0, 0, 2124, 0, 3818, 4091, 0, 0, 4092, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 1, 0, 0, 0, 0, 4096, 7, 4093, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 4080, 1, 458, 3, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4095, 0, 1, 0, 4096, 4094, 4096, 3, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3069, 4096, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 143, 0, 0, 0, 2126, 0, 0, 2983, 0, 3579, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 366, 0, 0, 0, 0, 4091, 0, 4094, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 1, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3925, 4096, 0, 0, 0, 0, 0, 4096, 0, 29, 0, 0, 4068, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4072, 0, 0, 1, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 4045, 0, 0, 4090, 0, 62, 0, 199, 0, 4096, 3678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 769, 0, 0, 3317, 0, 3915, 280, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 0, 4096, 3816, 222, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4084, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 92, 0, 0, 0, 0, 4096, 0, 4096, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16632, 16365, 16240, 16456, 16352, 16400, 16248, 16379] avg=16384.0000 max=16632 max_rank=0 min=16240 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=114440 max_load_reduction_pct=0.873108 moved_to_echo=65379 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 3961, 4088, 4, 3, 0, 0, 0, 4093, 11, 72, 17, 3, 4064, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4091, 3, 0, 3, 0, 1, 4094, 2, 3, 0, 3, 4092, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 58, 4, 4, 0, 4, 3, 4093, 3, 1, 4093, 0, 159, 0, 1965, 4, 4093, 3675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3686, 0, 0, 701, 0, 1413, 280, 12, 2, 4093, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 4092, 0, 0, 0, 2, 0, 0, 3, 3, 269, 4090, 0, 4092, 3812, 221, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 0, 1, 0, 0, 0, 3, 3, 2, 4093, 0, 0, 4078, 4, 1, 3, 4080, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4041, 0, 333, 0, 5, 0, 3, 3942, 0, 4044, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 3, 0, 4092, 0, 0, 4090, 49, 0, 0, 4, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16738, 16388, 18163, 14255, 16603, 16268, 16234, 16423] avg=16384.0000 max=18163 max_rank=2 min=14255 min_rank=3 imbalance_ratio=1.108582 max_load_reduction=112909 max_load_reduction_pct=0.861427 moved_to_echo=65528 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 0, 4096, 0, 25, 0, 0, 4068, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4071, 0, 0, 20, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 4028, 0, 0, 4058, 0, 106, 8, 53, 0, 4096, 3678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 949, 0, 0, 3104, 0, 3877, 280, 0, 0, 4087, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 45, 4096, 5, 4096, 3815, 222, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4084, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 142, 0, 0, 0, 0, 4096, 0, 4096, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 0, 4090, 0, 5, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16787, 16383, 16075, 16366, 16389, 16401, 16292, 16379] avg=16384.0000 max=16787 max_rank=0 min=16075 min_rank=2 imbalance_ratio=1.024597 max_load_reduction=114285 max_load_reduction_pct=0.871925 moved_to_echo=65461 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3525, 4096, 0, 0, 0, 0, 0, 4095, 0, 3, 0, 0, 4069, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 4072, 0, 0, 4095, 0, 7, 0, 28, 0, 4096, 3678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2884, 0, 0, 1874, 0, 3599, 280, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 4096, 3816, 222, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4084, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 291, 0, 0, 0, 0, 4096, 0, 4096, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4095, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16206, 16388, 16077, 16828, 16342, 16399, 16453, 16379] avg=16384.0000 max=16828 max_rank=3 min=16077 min_rank=2 imbalance_ratio=1.027100 max_load_reduction=114244 max_load_reduction_pct=0.871613 moved_to_echo=65573 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3408, 4095, 0, 0, 0, 0, 1, 4096, 0, 42, 0, 0, 4069, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 2, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 4, 0, 0, 4058, 0, 0, 4091, 0, 16, 6, 64, 0, 4096, 3678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2361, 0, 0, 2124, 0, 3818, 280, 0, 0, 4092, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 1, 0, 0, 0, 0, 4096, 7, 4093, 3811, 222, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4084, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 1, 458, 3, 0, 0, 0, 4096, 0, 4096, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1, 0, 0, 4094, 0, 3, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16130, 16382, 16063, 16764, 16341, 16403, 16608, 16381] avg=16384.0000 max=16764 max_rank=3 min=16063 min_rank=2 imbalance_ratio=1.023193 max_load_reduction=114308 max_load_reduction_pct=0.872101 moved_to_echo=65733 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3581, 4096, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 4069, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 4069, 0, 0, 4088, 0, 1, 1, 44, 0, 4096, 3678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2556, 0, 0, 2071, 0, 3658, 279, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 7, 4096, 2, 4096, 3816, 227, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4084, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 348, 0, 0, 0, 0, 4096, 0, 4096, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4084, 0, 0, 0, 0, 4094, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16270, 16389, 16022, 16753, 16362, 16400, 16504, 16372] avg=16384.0000 max=16753 max_rank=3 min=16022 min_rank=2 imbalance_ratio=1.022522 max_load_reduction=114319 max_load_reduction_pct=0.872185 moved_to_echo=65638 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3864, 4096, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 4068, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 4077, 0, 0, 4086, 0, 19, 0, 296, 0, 4096, 3678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1965, 0, 0, 2550, 0, 3503, 280, 1, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 4096, 3816, 222, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4084, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 54, 0, 0, 0, 0, 4096, 0, 4096, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16553, 16388, 16305, 16490, 16345, 16400, 16212, 16379] avg=16384.0000 max=16553 max_rank=0 min=16212 min_rank=6 imbalance_ratio=1.010315 max_load_reduction=114519 max_load_reduction_pct=0.873711 moved_to_echo=65336 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3069, 4096, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 4068, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 143, 0, 4096, 3679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2126, 0, 0, 2983, 0, 3579, 280, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3816, 222, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4085, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 366, 0, 0, 0, 0, 4091, 0, 4094, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15756, 16389, 16127, 17160, 16338, 16401, 16521, 16380] avg=16384.0000 max=17160 max_rank=3 min=15756 min_rank=0 imbalance_ratio=1.047363 max_load_reduction=113912 max_load_reduction_pct=0.869080 moved_to_echo=65640 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 0, 128, 0, 128, 32768, 128, 128, 128, 128, 32768, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 17408, 0, 0, 18816, 0, 27392, 2304, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 0, 32768, 0, 128, 32768, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 0, 29440, 32768, 128, 128, 0, 0, 128, 32768, 128, 256, 128, 128, 32640, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 128, 128, 128, 32768, 0, 0, 32768, 128, 128, 128, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 128, 2176, 128, 128, 0, 128, 32640, 0, 32768, 30976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 128, 128, 128, 128, 128, 128, 384, 32768, 128, 32768, 30592, 1792, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 512, 128, 128, 128, 128, 128, 32640, 128, 128, 32768, 0, 384, 128, 2816, 128, 32768, 29440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4092, 0, 0, 4096, 0, 0, 3985, 29, 0, 0, 0, 121, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4081, 3684, 0, 0, 4096, 0, 0, 4026, 4096, 4096, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 546, 4096, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4065, 4069, 4096, 4096, 4096, 0, 4082, 0, 0, 4096, 0, 71, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 0, 4096, 0, 0, 4071, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 3598, 0, 0, 4096, 0, 0, 4059, 4096, 4096, 0, 0, 0, 0, 0, 0, 3684, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 924, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 4085, 4096, 4096, 4096, 0, 4092, 0, 0, 4096, 0, 33, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4092, 1, 15, 4089, 4, 3, 4089, 6, 0, 0, 3, 14, 4, 0, 3, 4091, 8, 0, 16, 16, 1, 4091, 4076, 3, 0, 4, 6, 0, 4093, 3390, 0, 0, 4090, 0, 0, 4091, 4093, 4092, 0, 2, 0, 3, 0, 3, 3481, 0, 0, 4, 3, 0, 3, 0, 30, 3, 2, 4, 4093, 4, 0, 4091, 0, 3, 4, 4092, 4, 0, 0, 1, 0, 1211, 4093, 14, 3, 1, 0, 5, 7, 0, 0, 3, 3, 0, 4, 4092, 4093, 2, 0, 0, 0, 0, 0, 0, 3, 4082, 4092, 0, 0, 4091, 3, 4, 0, 1, 4067, 0, 3, 1, 4092, 3, 0, 6, 4079, 0, 4093, 4087, 4092, 4092, 4092, 0, 4068, 3, 4, 4092, 0, 69, 3, 2, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 137, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 4096, 4096, 0, 0, 0, 0, 0, 4094, 3408, 0, 0, 4096, 0, 0, 4086, 4096, 4096, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 597, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4092, 4095, 4096, 4096, 0, 4096, 0, 0, 4096, 1, 12, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4079, 0, 0, 4096, 0, 0, 4053, 1, 0, 0, 0, 44, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4092, 4020, 0, 0, 4096, 0, 0, 4051, 4095, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 140, 4095, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 1, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4094, 4094, 4096, 4096, 4096, 0, 4089, 0, 0, 4095, 1, 5, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 1, 4096, 0, 0, 3999, 102, 0, 0, 0, 143, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 4078, 3842, 0, 0, 4096, 0, 0, 4004, 4095, 4096, 0, 0, 0, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 7, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 204, 4090, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 59, 0, 0, 0, 4096, 4096, 0, 2, 4096, 0, 0, 1, 0, 4096, 6, 0, 0, 4096, 0, 0, 0, 4096, 0, 4051, 4057, 4096, 4096, 4096, 0, 4050, 0, 0, 4096, 0, 151, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4093, 0, 0, 4096, 0, 0, 4037, 11, 0, 0, 0, 43, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 4089, 3311, 0, 3, 4096, 0, 0, 4037, 4092, 4096, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 6, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 927, 4090, 0, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 4096, 0, 6, 4096, 0, 0, 2, 0, 4096, 4, 0, 0, 4096, 0, 0, 0, 4096, 0, 4085, 4084, 4091, 4096, 4096, 0, 4084, 0, 0, 4092, 4, 32, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4090, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 3714, 0, 0, 4096, 0, 0, 4088, 4096, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 396, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4092, 4096, 4096, 4096, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 4096, 0, 0, 3985, 29, 0, 0, 0, 121, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4081, 3684, 0, 0, 419, 40, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4026, 4096, 218, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3459, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 4096, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 4096, 3470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4069, 4089, 0, 0, 0, 4082, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16432, 16404, 16375, 16320, 16401, 16385, 16376] avg=16384.0000 max=16432 max_rank=1 min=16320 min_rank=4 imbalance_ratio=1.002930 max_load_reduction=114640 max_load_reduction_pct=0.874634 moved_to_echo=65482 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4092, 1, 15, 4089, 4, 3, 4089, 6, 0, 0, 3, 14, 4, 0, 3, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 16, 16, 1, 4091, 4076, 3, 0, 4, 6, 0, 4093, 3390, 0, 0, 419, 40, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4091, 4093, 217, 0, 2, 0, 3, 0, 3, 3481, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 30, 3, 2, 4, 4093, 4, 0, 4091, 0, 3, 4, 4092, 3456, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 1211, 4093, 14, 3, 1, 0, 5, 7, 0, 0, 3, 4092, 4092, 3467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 4082, 4093, 2, 0, 0, 0, 0, 0, 0, 3, 4082, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 4, 0, 1, 4067, 0, 3, 1, 4092, 3, 0, 6, 4079, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4087, 4085, 0, 0, 0, 4068, 3, 4, 0, 0, 69, 3, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16374, 16179, 15984, 16413, 16993, 16361, 16351, 16417] avg=16384.0000 max=16993 max_rank=4 min=15984 min_rank=2 imbalance_ratio=1.037170 max_load_reduction=114079 max_load_reduction_pct=0.870354 moved_to_echo=66122 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 4096, 0, 0, 4071, 0, 0, 0, 0, 23, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 3598, 0, 0, 419, 40, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4059, 4096, 218, 0, 0, 0, 0, 0, 0, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3459, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 924, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 4096, 3471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4085, 4089, 0, 0, 0, 4092, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16341, 16361, 16153, 16379, 16686, 16379, 16385, 16388] avg=16384.0000 max=16686 max_rank=4 min=16153 min_rank=2 imbalance_ratio=1.018433 max_load_reduction=114386 max_load_reduction_pct=0.872696 moved_to_echo=65838 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 1, 4096, 0, 0, 3999, 102, 0, 0, 0, 143, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 4078, 3842, 0, 0, 419, 40, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4004, 4095, 218, 0, 0, 0, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 7, 4096, 0, 0, 0, 4096, 3459, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 4090, 0, 0, 0, 0, 0, 26, 0, 0, 0, 4096, 4096, 3471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4089, 0, 0, 0, 0, 59, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 1, 0, 4096, 6, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4057, 4089, 0, 0, 0, 4050, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16493, 16584, 16397, 16397, 15983, 16426, 16394, 16398] avg=16384.0000 max=16584 max_rank=1 min=15983 min_rank=4 imbalance_ratio=1.012207 max_load_reduction=114488 max_load_reduction_pct=0.873474 moved_to_echo=65201 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 137, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 4096, 0, 0, 0, 0, 0, 4094, 3408, 0, 0, 419, 40, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 4096, 218, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3459, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 597, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 4088, 0, 0, 0, 4096, 0, 0, 0, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16476, 16172, 16550, 16373, 16355, 16376, 16385, 16385] avg=16384.0000 max=16550 max_rank=2 min=16172 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=114522 max_load_reduction_pct=0.873734 moved_to_echo=65501 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 4096, 0, 0, 4037, 11, 0, 0, 0, 43, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 4089, 3311, 0, 3, 419, 40, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4037, 4092, 218, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 6, 4096, 0, 0, 0, 4096, 3459, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 927, 4090, 0, 0, 0, 0, 0, 6, 1, 0, 0, 4096, 4096, 3466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4090, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 2, 0, 4096, 4, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4084, 4084, 0, 0, 0, 4084, 0, 0, 0, 4, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16336, 16070, 16450, 16385, 16682, 16379, 16397, 16373] avg=16384.0000 max=16682 max_rank=4 min=16070 min_rank=1 imbalance_ratio=1.018188 max_load_reduction=114390 max_load_reduction_pct=0.872726 moved_to_echo=65831 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4079, 0, 0, 4096, 0, 0, 4053, 1, 0, 0, 0, 44, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4092, 4020, 0, 0, 423, 46, 13, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4051, 4095, 211, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 4096, 3462, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 4095, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4096, 4096, 3469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4095, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 1, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 4081, 0, 0, 0, 4089, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16323, 16797, 16546, 16378, 15898, 16376, 16390, 16364] avg=16384.0000 max=16797 max_rank=1 min=15898 min_rank=4 imbalance_ratio=1.025208 max_load_reduction=114275 max_load_reduction_pct=0.871849 moved_to_echo=65028 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 14, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 3714, 0, 0, 419, 40, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 4096, 218, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3459, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 4089, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16350, 16477, 16588, 16372, 16155, 16374, 16385, 16371] avg=16384.0000 max=16588 max_rank=2 min=16155 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=114484 max_load_reduction_pct=0.873444 moved_to_echo=65285 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 32768, 128, 128, 32512, 256, 0, 0, 128, 640, 128, 0, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 128, 128, 32768, 32768, 128, 0, 128, 128, 0, 32768, 29056, 0, 128, 3456, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 0, 32512, 32768, 1792, 0, 128, 0, 128, 0, 128, 31488, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 128, 32768, 128, 128, 32768, 0, 128, 128, 32768, 27776, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 32768, 128, 128, 128, 32768, 128, 0, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 32768, 32768, 0, 0, 0, 32768, 128, 128, 0, 128, 384, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 0, 128, 0, 4992, 32768, 128, 128, 128, 0, 128, 128, 128, 128, 128, 32768, 32768, 27776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 128, 32768, 32768, 128, 0, 0, 0, 128, 0, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[4, 36, 0, 2, 2351, 0, 0, 3, 51, 1, 4, 16, 4089, 4, 4, 0, 4091, 1, 0, 4093, 4092, 29, 0, 0, 4, 0, 2, 3954, 4066, 1, 9, 4092, 4, 8, 1, 4, 2, 0, 0, 0, 4093, 0, 5, 3, 4, 0, 19, 4045, 4092, 4, 0, 3, 4092, 0, 3, 0, 0, 4089, 3, 3, 4091, 566, 0, 3758, 0, 0, 0, 3, 3, 0, 4, 4092, 4084, 0, 4079, 2472, 0, 1, 0, 3, 3, 0, 4090, 4, 4092, 0, 0, 3, 114, 2, 4009, 4092, 2127, 4092, 30, 4093, 1, 1, 3, 0, 4001, 1130, 0, 3, 0, 0, 4091, 3, 0, 0, 3, 3, 4092, 4, 4092, 5, 0, 0, 0, 0, 5, 4091, 18, 4090, 3, 0, 0, 4050] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1997, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 7, 3, 4086, 4096, 0, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 55, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4094, 3905, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 10, 0, 4092, 4094, 2190, 4096, 35, 4093, 0, 0, 0, 1, 4089, 14, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3087, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 95, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4094, 3915, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 1028, 4096, 62, 4096, 0, 0, 0, 0, 4094, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2416, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 3, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 57, 54, 4048, 4096, 0, 58, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 57, 0, 4096, 0, 28, 0, 0, 0, 0, 0, 4096, 4096, 0, 4093, 3557, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2016, 4096, 40, 4076, 0, 0, 0, 0, 4037, 36, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3160, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 27, 4073, 4096, 0, 30, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 52, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4090, 3132, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 1788, 4096, 9, 4096, 0, 0, 0, 0, 4084, 27, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3294, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 6, 0, 4092, 0, 0, 0, 0, 10, 19, 4068, 4096, 0, 4, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 4096, 205, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 4089, 3505, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 13, 0, 4091, 4089, 1155, 4096, 73, 4084, 0, 0, 0, 4, 4065, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 4, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2072, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 9, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 107, 49, 4035, 4096, 0, 97, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 693, 0, 4096, 0, 13, 0, 0, 0, 0, 0, 4096, 4096, 0, 4092, 3874, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4091, 4086, 1433, 4096, 81, 4078, 0, 0, 0, 0, 3939, 8, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 7, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1014, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 97, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 3657, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4088, 4096, 3413, 4096, 12, 4096, 0, 0, 0, 0, 4095, 9, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4, 36, 0, 2, 2351, 0, 0, 3, 51, 1, 4, 16, 4089, 4, 4, 0, 4093, 4092, 1655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 0, 0, 4092, 29, 0, 0, 4, 0, 2, 3954, 4066, 1, 9, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, 1, 4, 2, 0, 0, 0, 4093, 0, 5, 3, 4, 0, 19, 4045, 4090, 2437, 1635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 4, 0, 3, 0, 0, 3, 0, 0, 4089, 3, 3, 4091, 566, 0, 3758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 4, 4092, 4084, 0, 4079, 2472, 0, 1, 0, 3, 385, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4, 2164, 0, 0, 3, 114, 2, 4009, 4092, 2127, 0, 30, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 0, 4001, 1130, 0, 3, 0, 0, 4091, 3, 0, 0, 3, 3, 4075, 2454, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 4092, 5, 0, 0, 0, 0, 5, 4091, 18, 4090, 3, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16405, 16266, 16350, 16404, 15334, 16641, 17311, 16361] avg=16384.0000 max=17311 max_rank=6 min=15334 min_rank=4 imbalance_ratio=1.056580 max_load_reduction=113761 max_load_reduction_pct=0.867928 moved_to_echo=65647 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2416, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 57, 54, 4048, 4096, 2439, 1637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 0, 58, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 57, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 4096, 4096, 0, 4093, 3557, 0, 0, 0, 0, 385, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2166, 0, 0, 0, 0, 0, 4096, 4096, 2016, 0, 40, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 36, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 2456, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16361, 16403, 16427, 16290, 16464, 16490, 16250, 16387] avg=16384.0000 max=16490 max_rank=5 min=16250 min_rank=6 imbalance_ratio=1.006470 max_load_reduction=114582 max_load_reduction_pct=0.874191 moved_to_echo=65591 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2072, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 107, 49, 4035, 4096, 2440, 1637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 0, 97, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 693, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 4096, 4096, 0, 4092, 3874, 0, 0, 0, 0, 385, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2166, 0, 0, 0, 0, 0, 4091, 4086, 1433, 0, 81, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3939, 8, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 2456, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 7, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16027, 16409, 16457, 16965, 16765, 15935, 16124, 16390] avg=16384.0000 max=16965 max_rank=3 min=15935 min_rank=5 imbalance_ratio=1.035461 max_load_reduction=114107 max_load_reduction_pct=0.870567 moved_to_echo=65214 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1997, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 7, 3, 4086, 4096, 2439, 1639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3885, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 55, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4094, 3905, 0, 0, 0, 0, 389, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2158, 0, 0, 0, 10, 0, 4092, 4094, 2190, 0, 35, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 14, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4085, 2457, 1549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15943, 16395, 16366, 16230, 16791, 16672, 16291, 16384] avg=16384.0000 max=16791 max_rank=4 min=15943 min_rank=0 imbalance_ratio=1.024841 max_load_reduction=114281 max_load_reduction_pct=0.871895 moved_to_echo=66138 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3160, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 27, 4073, 4096, 2439, 1637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 0, 30, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 52, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4090, 3132, 0, 0, 0, 0, 385, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2166, 0, 0, 0, 0, 0, 4096, 4096, 1788, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 27, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 2456, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17106, 16401, 16374, 16257, 16008, 16251, 16288, 16387] avg=16384.0000 max=17106 max_rank=0 min=16008 min_rank=4 imbalance_ratio=1.044067 max_load_reduction=113966 max_load_reduction_pct=0.869492 moved_to_echo=64934 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3087, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 2439, 1637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 95, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4094, 3915, 0, 0, 0, 0, 385, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2166, 0, 0, 0, 0, 0, 4095, 4096, 1028, 0, 62, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 2459, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17032, 16400, 16365, 16272, 16795, 15543, 16281, 16384] avg=16384.0000 max=17032 max_rank=0 min=15543 min_rank=5 imbalance_ratio=1.039551 max_load_reduction=114040 max_load_reduction_pct=0.870056 moved_to_echo=65003 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3294, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 4092, 0, 0, 0, 0, 10, 19, 4068, 4096, 2439, 1637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 0, 4, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 205, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 4089, 3505, 0, 0, 0, 0, 385, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2164, 0, 0, 0, 13, 0, 4091, 4089, 1155, 0, 73, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4065, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 2456, 1544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 1, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 4, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17240, 16399, 16368, 16382, 16376, 15669, 16246, 16392] avg=16384.0000 max=17240 max_rank=0 min=15669 min_rank=5 imbalance_ratio=1.052246 max_load_reduction=113832 max_load_reduction_pct=0.868469 moved_to_echo=64683 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1014, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 2440, 1637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 97, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 3657, 0, 0, 0, 0, 385, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2166, 0, 0, 0, 0, 0, 4088, 4096, 3413, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 9, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4080, 2456, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14958, 16399, 16365, 16272, 16539, 17871, 16281, 16387] avg=16384.0000 max=17871 max_rank=5 min=14958 min_rank=0 imbalance_ratio=1.090759 max_load_reduction=113201 max_load_reduction_pct=0.863655 moved_to_echo=67078 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[31104, 128, 256, 128, 0, 0, 128, 0, 0, 32768, 128, 128, 32768, 1920, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 32512, 1280, 0, 128, 0, 0, 32768, 128, 0, 0, 128, 128, 32640, 19712, 12416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 0, 0, 128, 128, 128, 32640, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 19456, 0, 0, 128, 128, 128, 128, 128, 32768, 128, 128, 0, 32768, 32768, 13312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 0, 0, 128, 32768, 128, 32768, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 17408, 0, 0, 128, 256, 128, 32768, 32768, 15232, 0, 384, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 32768, 32768, 0, 32768, 28032, 0, 128, 0, 128, 3200, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 0, 32768, 0, 128, 128, 128, 256, 256, 32640, 32768, 19584, 13184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2098, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 742, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2623, 3697, 0, 0, 0, 0, 0, 4096, 4096, 0, 3132, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 3, 0, 4093, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1734, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 441, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3635, 4057, 0, 0, 0, 0, 0, 4096, 4096, 0, 2447, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 23, 0, 4087, 4096, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4060, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2701, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 110, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3762, 4084, 0, 0, 0, 0, 0, 4096, 4096, 0, 1645, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4082, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4, 8, 0, 4, 4092, 0, 0, 0, 196, 1, 0, 4088, 4091, 4053, 4, 4091, 232, 4, 127, 5, 1, 0, 2, 4092, 3971, 0, 14, 0, 0, 4091, 15, 28, 0, 4, 3, 0, 4090, 4, 3, 0, 0, 107, 4, 3, 3, 1, 3828, 2, 4005, 4045, 5, 3, 4, 4, 4092, 14, 1, 0, 4091, 3, 4, 4, 0, 132, 4, 4092, 0, 3, 3943, 0, 4, 0, 4075, 1087, 3203, 3, 3, 0, 0, 4, 4092, 4092, 0, 3808, 0, 0, 4, 4084, 0, 0, 4, 1, 5, 4091, 9, 4077, 4085, 1, 1, 230, 0, 4092, 4090, 0, 51, 4092, 0, 3, 3, 0, 3974, 0, 0, 1, 0, 1, 0, 3860, 4092, 4, 4, 4, 4, 0, 23, 4091, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4081, 0, 0, 4, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 98, 0, 0, 0, 3, 1471, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 306, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2615, 3989, 0, 0, 0, 0, 0, 4096, 4096, 0, 3847, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 19, 0, 4089, 4096, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4051, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 14, 0, 3, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 27, 0, 0, 0, 0, 1953, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 235, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3731, 3908, 0, 0, 0, 0, 0, 4096, 4096, 0, 2427, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 2, 0, 4092, 4096, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 6, 0, 0, 0, 4095, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 4, 2753, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 371, 0, 4093, 0, 0, 4096, 0, 0, 0, 4096, 3019, 3897, 0, 0, 0, 0, 0, 4096, 4095, 1, 2248, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 10, 0, 4086, 4096, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4082, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 2, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 171, 0, 0, 0, 0, 2286, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1185, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3766, 4048, 0, 0, 0, 0, 0, 4096, 4096, 0, 829, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 2547, 1045, 426, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1734, 0, 4096, 4096, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3670, 0, 0, 0, 4096, 0, 0, 0, 0, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1322, 0, 0, 4096, 0, 0, 0, 4096, 3635, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2447, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1549, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 4087, 4078, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2407, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16366, 16342, 15713, 16399, 17206, 16284, 16374, 16388] avg=16384.0000 max=17206 max_rank=4 min=15713 min_rank=2 imbalance_ratio=1.050171 max_load_reduction=113866 max_load_reduction_pct=0.868729 moved_to_echo=66252 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4, 8, 0, 4, 4092, 0, 0, 0, 196, 1, 0, 4088, 0, 4053, 4, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 4, 127, 5, 1, 0, 2, 4092, 3971, 0, 14, 0, 0, 4091, 15, 28, 2543, 1044, 426, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 4090, 4, 3, 0, 0, 107, 4, 3, 3, 1, 3828, 2, 4092, 4091, 1686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 4045, 5, 3, 4, 4, 3666, 14, 1, 0, 4091, 3, 4, 4, 0, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1321, 0, 3, 3943, 0, 4, 0, 4075, 1087, 3203, 3, 3, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3808, 0, 0, 4, 4084, 0, 0, 4, 1, 5, 1548, 9, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 1, 230, 0, 4092, 4072, 0, 51, 0, 0, 3, 3, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 3860, 4092, 4, 4, 4, 4, 0, 23, 4091, 5, 0, 2406, 1727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16523, 16631, 17921, 15981, 13650, 17632, 16512, 16222] avg=16384.0000 max=17921 max_rank=2 min=13650 min_rank=4 imbalance_ratio=1.093811 max_load_reduction=113151 max_load_reduction_pct=0.863274 moved_to_echo=64016 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4081, 0, 0, 4, 0, 0, 0, 4096, 0, 4096, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 6, 0, 2547, 1045, 426, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 98, 0, 0, 0, 3, 1471, 0, 4096, 4096, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3670, 0, 0, 0, 4096, 0, 0, 0, 0, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1322, 0, 0, 4096, 0, 0, 0, 4096, 2615, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3847, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 1549, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 0, 4089, 4078, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2408, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16355, 16352, 15548, 16264, 16118, 17685, 16370, 16380] avg=16384.0000 max=17685 max_rank=5 min=15548 min_rank=2 imbalance_ratio=1.079407 max_load_reduction=113387 max_load_reduction_pct=0.865074 moved_to_echo=66553 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 1, 2548, 1047, 430, 21, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2098, 0, 4096, 4096, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3665, 0, 0, 0, 4096, 0, 0, 0, 0, 742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1316, 0, 0, 4096, 0, 0, 0, 4096, 2623, 3697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3132, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 1548, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 4093, 4076, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2407, 1733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16363, 16356, 16078, 16695, 15828, 16969, 16363, 16420] avg=16384.0000 max=16969 max_rank=5 min=15828 min_rank=4 imbalance_ratio=1.035706 max_load_reduction=114103 max_load_reduction_pct=0.870537 moved_to_echo=65580 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 3, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 2547, 1045, 426, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 27, 0, 0, 0, 0, 1953, 0, 4096, 4096, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3670, 0, 0, 0, 4096, 0, 0, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1322, 0, 0, 4096, 0, 0, 0, 4096, 3731, 3908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2427, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1549, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4092, 4078, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2407, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16366, 16359, 15957, 16193, 17153, 16264, 16361, 16419] avg=16384.0000 max=17153 max_rank=4 min=15957 min_rank=2 imbalance_ratio=1.046936 max_load_reduction=113919 max_load_reduction_pct=0.869133 moved_to_echo=66197 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 6, 0, 0, 0, 4095, 0, 4096, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4, 0, 2547, 1044, 426, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 4, 2753, 0, 4096, 4096, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3670, 0, 0, 0, 4096, 0, 0, 0, 0, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1321, 0, 0, 4096, 0, 0, 0, 4096, 3019, 3897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 2248, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 1549, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 4086, 4078, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2407, 1728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16367, 16345, 16743, 16329, 16429, 16090, 16360, 16409] avg=16384.0000 max=16743 max_rank=2 min=16090 min_rank=5 imbalance_ratio=1.021912 max_load_reduction=114329 max_load_reduction_pct=0.872261 moved_to_echo=65288 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4095, 0, 0, 2547, 1045, 426, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2701, 0, 4096, 4096, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3670, 0, 0, 0, 4096, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1322, 0, 0, 4096, 0, 0, 0, 4096, 3762, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1645, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1549, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2407, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16366, 16341, 16679, 16068, 17360, 15482, 16366, 16410] avg=16384.0000 max=17360 max_rank=4 min=15482 min_rank=5 imbalance_ratio=1.059570 max_load_reduction=113712 max_load_reduction_pct=0.867554 moved_to_echo=65618 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 4096, 0, 0, 2547, 1045, 426, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 171, 0, 0, 0, 0, 2286, 0, 4096, 4096, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3670, 0, 0, 0, 4096, 0, 0, 0, 0, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1322, 0, 0, 4096, 0, 0, 0, 4096, 3766, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 829, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1549, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2407, 1729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16366, 16346, 16433, 17143, 17328, 14666, 16366, 16424] avg=16384.0000 max=17328 max_rank=4 min=14666 min_rank=5 imbalance_ratio=1.057617 max_load_reduction=113744 max_load_reduction_pct=0.867798 moved_to_echo=64784 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32768, 128, 20480, 0, 128, 128, 32768, 0, 0, 128, 128, 128, 12416, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 128, 0, 0, 512, 128, 128, 128, 128, 18944, 128, 32768, 32768, 13568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 128, 256, 128, 0, 32768, 0, 32768, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 0, 32512, 32768, 128, 128, 128, 128, 0, 128, 32768, 128, 0, 19328, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 384, 0, 32768, 32640, 0, 128, 0, 0, 128, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 32768, 128, 128, 128, 128, 29440, 128, 128, 0, 32768, 128, 128, 128, 0, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 10624, 0, 128, 32640, 0, 128, 0, 32768, 24320, 30976, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[256, 128, 256, 128, 128, 0, 128, 32768, 32768, 0, 128, 128, 0, 32768, 128, 128, 20480, 8448, 3456, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4095, 0, 4096, 0, 0, 4096, 0, 2248, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 30, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 2, 4096, 0, 289, 0, 0, 0, 0, 0, 0, 0, 1821, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4091, 0, 0, 8, 4083, 4092, 0, 4092, 0, 4023, 4, 0, 4091, 4, 721, 3, 4088, 3, 0, 0, 0, 2, 0, 4, 5, 4092, 4089, 0, 4, 4091, 4087, 4090, 0, 2924, 6, 3, 5, 0, 0, 4, 4, 4093, 39, 25, 0, 13, 3, 0, 4032, 0, 0, 4, 4088, 4, 3820, 138, 0, 0, 4090, 4092, 5, 4, 4, 0, 0, 7, 3987, 0, 4, 24, 4079, 3970, 4, 0, 0, 3, 5, 0, 8, 4, 268, 8, 1, 3, 15, 107, 1, 0, 4061, 4069, 0, 0, 0, 1, 0, 3, 0, 0, 3, 4091, 0, 4078, 4091, 0, 0, 93, 4092, 3, 2395, 0, 1, 0, 3, 3, 0, 5, 2268, 0, 0, 0, 0, 4092, 4091, 3960, 2, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2879, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 1, 4096, 3975, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 2, 4095, 0, 144, 0, 0, 0, 0, 0, 0, 0, 1192, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 1864, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 13, 0, 4096, 0, 0, 1, 4096, 3783, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4091, 0, 0, 8, 4096, 0, 447, 0, 0, 0, 0, 0, 0, 0, 2184, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2622, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 2, 4096, 0, 73, 0, 0, 0, 0, 0, 0, 0, 1875, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4095, 0, 4096, 0, 0, 4096, 0, 1340, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 1, 4096, 3592, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4092, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4096, 4092, 0, 0, 1, 4095, 0, 522, 0, 0, 0, 0, 0, 0, 0, 2737, 0, 1, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 1493, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 4096, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 208, 0, 0, 0, 0, 0, 0, 0, 2440, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 827, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 4096, 0, 1023, 0, 0, 0, 0, 0, 0, 0, 3031, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2357, 0, 0, 0, 4096, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 2248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 30, 0, 4096, 0, 4096, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 4096, 0, 0, 0, 0, 0, 0, 4094, 3957, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3450, 0, 4096, 4096, 0, 0, 2, 4096, 0, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1821, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1272, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16892, 16386, 16523, 16431, 16481, 16303, 16029, 16027] avg=16384.0000 max=16892 max_rank=0 min=16027 min_rank=7 imbalance_ratio=1.031006 max_load_reduction=114180 max_load_reduction_pct=0.871124 moved_to_echo=64840 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2357, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 4096, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 4096, 0, 4096, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4096, 0, 0, 1, 4096, 3783, 0, 5, 0, 0, 0, 0, 0, 0, 3954, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 4094, 0, 0, 0, 0, 0, 0, 4094, 3958, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3450, 0, 4096, 4091, 0, 0, 8, 4096, 0, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2184, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1272, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16509, 16386, 16514, 16422, 16415, 16248, 16188, 16390] avg=16384.0000 max=16514 max_rank=2 min=16188 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=114558 max_load_reduction_pct=0.874008 moved_to_echo=65241 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2357, 0, 0, 0, 4096, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 1340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 4096, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 4096, 0, 4096, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 1, 4096, 3592, 0, 4, 0, 0, 0, 0, 0, 0, 3954, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4092, 0, 0, 0, 0, 4, 0, 4096, 3956, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3450, 0, 4096, 4092, 0, 0, 1, 4095, 0, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2737, 0, 1, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1272, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15984, 16384, 16514, 16410, 16214, 16366, 16256, 16944] avg=16384.0000 max=16944 max_rank=7 min=15984 min_rank=0 imbalance_ratio=1.034180 max_load_reduction=114128 max_load_reduction_pct=0.870728 moved_to_echo=65780 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2354, 0, 0, 8, 4083, 0, 0, 4092, 0, 4023, 4, 0, 0, 4, 721, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 0, 2, 0, 4, 5, 0, 4089, 0, 4, 2, 4087, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2924, 6, 3, 5, 0, 0, 4, 4, 4093, 39, 25, 0, 13, 3, 0, 4092, 4091, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 4, 4088, 4, 3820, 138, 0, 0, 4090, 17, 5, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3987, 0, 4, 24, 4079, 3970, 4, 0, 0, 3, 5, 0, 8, 4, 3950, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 268, 8, 1, 3, 15, 107, 1, 0, 4061, 4069, 0, 0, 0, 1, 0, 3, 4089, 3954, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3446, 0, 4078, 4091, 0, 0, 93, 4092, 3, 2395, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 5, 2268, 0, 0, 0, 0, 4092, 4091, 3960, 2, 2, 0, 0, 1270, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15292, 16374, 15440, 16206, 16512, 16705, 18202, 16341] avg=16384.0000 max=18202 max_rank=6 min=15292 min_rank=0 imbalance_ratio=1.110962 max_load_reduction=112870 max_load_reduction_pct=0.861130 moved_to_echo=67760 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2351, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 2879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 1, 4096, 3975, 0, 1, 0, 0, 0, 0, 0, 0, 3955, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 4096, 3957, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3444, 0, 4096, 4095, 0, 0, 2, 4095, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1274, 652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17518, 16384, 16520, 16398, 16598, 16372, 15876, 15406] avg=16384.0000 max=17518 max_rank=0 min=15406 min_rank=7 imbalance_ratio=1.069214 max_load_reduction=113554 max_load_reduction_pct=0.866348 moved_to_echo=64252 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2357, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 2622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 4096, 3957, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3450, 0, 4096, 4096, 0, 0, 2, 4096, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1875, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1272, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17267, 16385, 16517, 16400, 16243, 16366, 15813, 16081] avg=16384.0000 max=17267 max_rank=0 min=15813 min_rank=6 imbalance_ratio=1.053894 max_load_reduction=113805 max_load_reduction_pct=0.868263 moved_to_echo=64503 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2357, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 4096, 0, 4096, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 4096, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 4096, 0, 0, 0, 0, 0, 0, 4094, 3958, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3450, 0, 4096, 4096, 0, 0, 0, 4096, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2440, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1272, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16138, 16386, 16522, 16405, 16686, 16343, 15946, 16646] avg=16384.0000 max=16686 max_rank=4 min=15946 min_rank=6 imbalance_ratio=1.018433 max_load_reduction=114386 max_load_reduction_pct=0.872696 moved_to_echo=65621 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2357, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3310, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 3958, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3450, 0, 4096, 4096, 0, 0, 1, 4096, 0, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3031, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1272, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15472, 16387, 16522, 16400, 15923, 16369, 16762, 17237] avg=16384.0000 max=17237 max_rank=7 min=15472 min_rank=0 imbalance_ratio=1.052063 max_load_reduction=113835 max_load_reduction_pct=0.868492 moved_to_echo=66291 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 0, 128, 128, 0, 128, 128, 0, 32768, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[384, 128, 128, 128, 128, 128, 128, 0, 32640, 32768, 0, 0, 0, 128, 128, 128, 32768, 31744, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[18944, 0, 0, 128, 32768, 0, 0, 32768, 0, 32768, 128, 0, 0, 128, 14080, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 31616, 128, 128, 128, 0, 0, 128, 128, 32768, 128, 128, 0, 128, 128, 0, 32768, 32768, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 128, 27648, 0, 32768, 32768, 0, 0, 128, 32768, 128, 5120, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 128, 128, 32768, 128, 32512, 256, 0, 0, 32768, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 32768, 30208, 128, 128, 0, 128, 128, 0, 128, 128, 31744, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 17664, 0, 128, 0, 0, 32768, 32768, 32640, 128, 128, 0, 0, 10240, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4044, 0, 0, 621, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 764, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 76, 4096, 4096, 0, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4094, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 182, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4079, 0, 0, 650, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 1011, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4, 136, 4096, 4090, 0, 2285, 3, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4092, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4095, 4090, 4096, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 4096, 48, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 4092, 4074, 0, 0, 0, 0, 11, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4043, 0, 0, 1066, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1435, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4, 4096, 4086, 0, 1189, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4088, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4081, 4096, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 4096, 467, 0, 0, 4096, 0, 4091, 5, 0, 0, 0, 4091, 4094, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1223, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 615, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 69, 4096, 4096, 0, 2105, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 95, 0, 0, 4096, 0, 4096, 3, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 2233, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 942, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 71, 4096, 4095, 0, 860, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 22, 0, 0, 4096, 0, 4095, 1, 0, 0, 0, 4095, 4064, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1909, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1554, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 49, 4096, 4096, 0, 643, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 10, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4082, 0, 0, 466, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 382, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 6, 4096, 4096, 0, 3181, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 4096, 70, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[3, 5, 4085, 4, 0, 6, 14, 0, 4086, 4, 4, 0, 10, 3, 54, 6, 4088, 4, 4092, 0, 0, 1, 4, 4091, 0, 1, 3979, 4, 5, 2019, 4050, 4, 4, 7, 0, 0, 4088, 0, 13, 2, 1002, 4077, 8, 0, 4090, 5, 0, 0, 2, 0, 0, 0, 4, 4082, 0, 4091, 4077, 0, 4, 0, 390, 4092, 4090, 0, 671, 0, 0, 0, 1, 4, 0, 4, 4, 4041, 4, 3, 5, 10, 4092, 4091, 2, 2, 4092, 27, 4090, 3, 0, 4092, 4072, 4067, 0, 0, 0, 0, 0, 4092, 36, 3, 0, 4037, 31, 0, 112, 3789, 0, 4088, 27, 0, 0, 3, 4092, 4093, 0, 4, 0, 0, 0, 5, 52, 0, 6, 4, 4069, 308, 4088, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2859, 0, 0, 4044, 0, 0, 621, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 764, 4096, 0, 0, 4096, 0, 0, 0, 3075, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4001, 4096, 0, 0, 0, 76, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 4096, 1237, 1014, 95, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 182, 0, 0, 4096, 0, 4023, 1, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4039, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 15716, 16184, 16365, 17233, 16389, 16474, 16339] avg=16384.0000 max=17233 max_rank=4 min=15716 min_rank=1 imbalance_ratio=1.051819 max_load_reduction=113839 max_load_reduction_pct=0.868523 moved_to_echo=66435 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2859, 0, 0, 4079, 0, 0, 650, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 1011, 4096, 0, 0, 4096, 0, 0, 0, 3075, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4001, 4096, 0, 0, 4, 136, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2285, 3, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4092, 4096, 1237, 1014, 95, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4090, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 48, 0, 0, 4096, 0, 4021, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4035, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 15780, 16432, 16427, 16986, 16384, 16338, 16347] avg=16384.0000 max=16986 max_rank=4 min=15780 min_rank=1 imbalance_ratio=1.036743 max_load_reduction=114086 max_load_reduction_pct=0.870407 moved_to_echo=66055 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2859, 0, 0, 4043, 0, 0, 1066, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1435, 4096, 0, 0, 4096, 0, 0, 0, 3075, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4001, 4096, 0, 0, 0, 4, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1189, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4088, 4096, 1237, 1014, 95, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4081, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 467, 0, 0, 4096, 0, 4019, 5, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 4096, 4035, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16160, 16854, 16284, 15886, 16376, 16782, 16351] avg=16384.0000 max=16854 max_rank=2 min=15886 min_rank=4 imbalance_ratio=1.028687 max_load_reduction=114218 max_load_reduction_pct=0.871414 moved_to_echo=65395 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2859, 0, 0, 4096, 0, 0, 1223, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 615, 4096, 0, 0, 4096, 0, 0, 0, 3075, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4001, 4096, 0, 0, 0, 69, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2105, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 1237, 1014, 95, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 95, 0, 0, 4096, 0, 4023, 3, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4038, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 16370, 16036, 16358, 16812, 16392, 16394, 16338] avg=16384.0000 max=16812 max_rank=4 min=16036 min_rank=2 imbalance_ratio=1.026123 max_load_reduction=114260 max_load_reduction_pct=0.871735 moved_to_echo=65936 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2857, 0, 0, 4094, 0, 0, 2233, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 942, 4096, 0, 0, 4096, 0, 0, 0, 3078, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4000, 4096, 0, 0, 1, 71, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 860, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 4096, 1239, 1015, 96, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 22, 0, 0, 4096, 0, 4019, 1, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4037, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 17376, 16366, 16360, 15573, 16386, 16299, 16339] avg=16384.0000 max=17376 max_rank=1 min=15573 min_rank=4 imbalance_ratio=1.060547 max_load_reduction=113696 max_load_reduction_pct=0.867432 moved_to_echo=64597 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2859, 0, 0, 4082, 0, 0, 466, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 382, 4096, 0, 0, 4096, 0, 0, 0, 3075, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4001, 4096, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3181, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4096, 1237, 1014, 95, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 70, 0, 0, 4096, 0, 4023, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4039, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 15599, 15802, 16295, 17887, 16391, 16387, 16339] avg=16384.0000 max=17887 max_rank=4 min=15599 min_rank=1 imbalance_ratio=1.091736 max_load_reduction=113185 max_load_reduction_pct=0.863533 moved_to_echo=67004 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 5, 4085, 4, 0, 6, 14, 0, 4086, 4, 4, 0, 10, 3, 54, 6, 4092, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 0, 0, 0, 1, 4, 2856, 0, 1, 3979, 4, 5, 2019, 4050, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 0, 0, 4088, 0, 13, 2, 1002, 4077, 8, 0, 4090, 5, 0, 0, 3072, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 4082, 0, 3996, 4077, 0, 4, 0, 390, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 671, 0, 0, 0, 1, 4, 0, 4, 4, 4041, 4, 3, 5, 10, 4092, 4091, 1235, 1013, 95, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 27, 4090, 3, 0, 4092, 4072, 4067, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 3, 0, 4037, 31, 0, 112, 3789, 0, 4016, 27, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 5, 52, 0, 6, 4, 4069, 308, 4088, 0, 2, 0, 4092, 4036, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16455, 17015, 16424, 16645, 15345, 16362, 16147, 16679] avg=16384.0000 max=17015 max_rank=1 min=15345 min_rank=4 imbalance_ratio=1.038513 max_load_reduction=114057 max_load_reduction_pct=0.870186 moved_to_echo=64533 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2859, 0, 0, 4096, 0, 0, 1909, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1554, 4096, 0, 0, 4096, 0, 0, 0, 3075, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4001, 4096, 0, 0, 0, 49, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 643, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 1237, 1014, 95, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 4096, 0, 4023, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4039, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16371, 17056, 16974, 16338, 15350, 16392, 16251, 16340] avg=16384.0000 max=17056 max_rank=1 min=15350 min_rank=4 imbalance_ratio=1.041016 max_load_reduction=114016 max_load_reduction_pct=0.869873 moved_to_echo=64333 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 0, 0, 128, 32768, 0, 32128, 32768, 0, 128, 128, 896, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 1024, 0, 128, 32512, 0, 32256, 128, 0, 0, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 32768, 128, 0, 128, 128, 0, 32768, 128, 128, 0, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 128, 128, 7808, 32768, 128, 0, 32768, 128, 0, 0, 24704, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[13568, 128, 0, 0, 128, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 32768, 32768, 9984, 8192, 768, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 32768, 32768, 32768, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 0, 0, 0, 128, 128, 22912, 0, 128, 32640, 128, 128, 10240, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 0, 128, 128, 128, 128, 128, 128, 32768, 384, 32768, 0, 128, 0, 32768, 32384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3993, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4070, 0, 0, 7, 0, 0, 0, 0, 0, 0, 102, 4096, 0, 0, 4096, 4096, 0, 4096, 13, 4096, 4096, 0, 4096, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 3977, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 4096, 0, 0, 4096, 4096, 0, 4096, 20, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 22, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4077, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 8, 0, 0, 0, 0, 0, 0, 13, 4096, 0, 0, 4096, 4096, 0, 4096, 1, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 4, 0, 4088, 0, 0, 3, 7, 0, 4085, 3893, 0, 0, 4092, 4092, 0, 4085, 0, 4090, 0, 0, 3, 0, 178, 4092, 0, 0, 4, 4, 0, 0, 0, 5, 4091, 4092, 4091, 5, 3, 0, 3, 4090, 0, 4, 4, 4, 6, 0, 9, 4068, 11, 0, 24, 3, 4, 0, 14, 4, 4091, 4, 1, 4, 3992, 6, 0, 95, 3, 7, 10, 4, 4, 3, 0, 4093, 2, 5, 4037, 4070, 3, 4093, 0, 4089, 4083, 105, 4092, 0, 0, 0, 0, 3, 0, 4078, 0, 0, 0, 0, 5, 0, 4, 0, 0, 0, 4092, 4092, 4032, 4, 4092, 0, 4089, 0, 4092, 4091, 0, 4, 0, 5, 4092, 50, 0, 4086, 0, 3, 0, 0, 3, 0, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 3981, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 5, 0, 4096, 0, 0, 0, 4079, 0, 0, 2, 0, 0, 0, 0, 0, 0, 113, 4090, 0, 0, 4096, 4096, 0, 4091, 20, 4096, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 2, 4096, 0, 4093, 4096, 0, 0, 0, 0, 4094, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 4047, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4090, 0, 0, 14, 0, 0, 0, 0, 0, 0, 39, 4095, 0, 0, 4096, 4096, 0, 4095, 5, 4096, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 4008, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 4087, 0, 0, 4096, 4096, 0, 4095, 21, 4096, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4091, 4096, 0, 0, 0, 0, 4090, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 4012, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 3, 0, 0, 0, 0, 0, 0, 84, 4096, 0, 0, 4096, 4096, 0, 4096, 4, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3993, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 3977, 73, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4070, 0, 4086, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 102, 4096, 0, 0, 4096, 4096, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 4096, 0, 4076, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16363, 16383, 16374, 16420, 16379, 16394, 16375] avg=16384.0000 max=16420 max_rank=4 min=16363 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=114652 max_load_reduction_pct=0.874725 moved_to_echo=65568 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 3977, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3977, 73, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4086, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 4096, 0, 0, 4096, 4096, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 4096, 0, 4076, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 22, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16361, 16359, 16383, 16400, 16395, 16384, 16394, 16396] avg=16384.0000 max=16400 max_rank=3 min=16359 min_rank=1 imbalance_ratio=1.000977 max_load_reduction=114672 max_load_reduction_pct=0.874878 moved_to_echo=65569 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4, 0, 4088, 0, 0, 3, 7, 0, 4085, 3893, 0, 0, 4092, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4090, 0, 0, 3, 0, 178, 4092, 0, 0, 4, 4, 0, 0, 3973, 73, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4091, 4091, 4091, 5, 3, 0, 3, 4090, 0, 4, 4, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4068, 11, 0, 24, 3, 4, 0, 14, 4, 4091, 4, 1, 4, 3992, 6, 4082, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 3, 7, 10, 4, 4, 3, 0, 4093, 2, 5, 4037, 4070, 3, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4083, 105, 4072, 0, 0, 0, 0, 3, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4, 0, 0, 0, 0, 0, 4032, 4, 10, 0, 4089, 0, 4092, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 4092, 50, 0, 4086, 0, 3, 0, 0, 3, 0, 3, 0, 4092, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16276, 16523, 16397, 16343, 16356, 16430, 16327, 16420] avg=16384.0000 max=16523 max_rank=1 min=16276 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=114549 max_load_reduction_pct=0.873940 moved_to_echo=65533 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 4008, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 3977, 73, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4067, 0, 4086, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 4087, 0, 0, 4096, 4096, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 4096, 0, 4076, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 4096, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16369, 16383, 16371, 16395, 16398, 16389, 16380] avg=16384.0000 max=16398 max_rank=5 min=16369 min_rank=1 imbalance_ratio=1.000854 max_load_reduction=114674 max_load_reduction_pct=0.874893 moved_to_echo=65562 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4077, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3977, 73, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 4086, 16, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 13, 4096, 0, 0, 4096, 4096, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16468, 16359, 16383, 16398, 16332, 16364, 16394, 16374] avg=16384.0000 max=16468 max_rank=0 min=16332 min_rank=4 imbalance_ratio=1.005127 max_load_reduction=114604 max_load_reduction_pct=0.874359 moved_to_echo=65464 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 3981, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 3977, 73, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 5, 0, 4096, 0, 0, 0, 4079, 0, 4086, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 113, 4090, 0, 0, 4096, 4096, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 4096, 0, 4076, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 2, 4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16366, 16363, 16383, 16388, 16415, 16385, 16393, 16379] avg=16384.0000 max=16415 max_rank=4 min=16363 min_rank=1 imbalance_ratio=1.001892 max_load_reduction=114657 max_load_reduction_pct=0.874763 moved_to_echo=65572 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 4047, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 3981, 74, 26, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4090, 0, 4092, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 39, 4095, 0, 0, 4096, 4096, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 0, 4070, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16429, 16377, 16377, 16403, 16361, 16364, 16387, 16374] avg=16384.0000 max=16429 max_rank=0 min=16361 min_rank=4 imbalance_ratio=1.002747 max_load_reduction=114643 max_load_reduction_pct=0.874657 moved_to_echo=65486 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 4012, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3977, 73, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 4086, 16, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 84, 4096, 0, 0, 4096, 4096, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 4076, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16401, 16359, 16383, 16395, 16398, 16368, 16394, 16374] avg=16384.0000 max=16401 max_rank=0 min=16359 min_rank=1 imbalance_ratio=1.001038 max_load_reduction=114671 max_load_reduction_pct=0.874870 moved_to_echo=65534 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 128, 128, 0, 32768, 32640, 0, 0, 32256, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 32768, 128, 128, 128, 32768, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 128, 32768, 0, 128, 0, 0, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 32768, 32768, 128, 32640, 0, 128, 0, 0, 128, 0, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 0, 32768, 0, 0, 128, 0, 256, 32768, 0, 128, 128, 128, 0, 0, 31872, 640, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 32768, 32768, 32768, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 32640, 128, 32768, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 256, 128, 128, 128, 128, 128, 128, 640, 32768, 128, 128, 32768, 32768, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 13, 0, 4096, 0, 0, 0, 1, 4096, 4096, 0, 0, 7, 4096, 0, 0, 0, 2, 0, 0, 4088, 4096, 0, 4094, 0, 4071, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4093, 0, 17, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4093, 4096, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 5, 5, 0, 4, 0, 6, 0, 0, 0, 0, 443, 0, 4090, 108, 3, 0, 0, 4091, 4091, 5, 0, 0, 4091, 5, 0, 9, 6, 0, 3, 4079, 4091, 0, 4083, 14, 4029, 1, 4, 40, 3, 0, 0, 2, 4, 0, 41, 1, 5, 3, 5, 0, 0, 4, 0, 1, 4089, 4085, 26, 4062, 4089, 4092, 0, 0, 0, 4050, 0, 0, 6, 4, 3173, 3, 0, 4, 4086, 7, 5, 0, 0, 8, 3, 4081, 4072, 3995, 0, 4030, 0, 9, 0, 0, 1, 4087, 161, 4, 0, 0, 4067, 1, 4, 4089, 4090, 0, 5, 4091, 0, 0, 4, 4088, 3, 6, 5, 0, 42, 5, 4091, 4091, 0, 0, 4092, 4, 0, 3, 469, 4, 8, 5, 4091, 4092, 4015] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4095, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 4096, 0, 0, 0, 1, 0, 0, 4093, 4096, 0, 4095, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4095, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 8, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 2, 0, 0, 0, 4096, 4096, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 4084, 4096, 0, 4091, 0, 4089, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4096, 19, 4096, 4088, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4074, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 10, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4079, 4096, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 21, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 4096, 0, 4090, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4050, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4089, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4095, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 5, 0, 4, 0, 6, 0, 0, 0, 0, 443, 0, 4090, 108, 3, 4092, 4091, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4091, 5, 0, 0, 3, 5, 0, 9, 6, 0, 3, 4079, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 14, 4029, 1, 4, 40, 3, 0, 0, 2, 4, 0, 41, 1, 5, 4087, 4006, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 0, 0, 4, 0, 1, 4089, 4085, 26, 4062, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 6, 4, 3173, 3, 0, 4, 4086, 7, 5, 0, 0, 8, 3, 4088, 75, 48, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 3995, 0, 4030, 0, 9, 0, 0, 1, 0, 161, 4, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 4089, 4090, 0, 5, 4087, 0, 0, 4, 4088, 3, 6, 5, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4043, 4091, 0, 0, 4092, 4, 0, 3, 469, 4, 8, 5, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16858, 16383, 16400, 16365, 15566, 16337, 16424, 16739] avg=16384.0000 max=16858 max_rank=0 min=15566 min_rank=4 imbalance_ratio=1.028931 max_load_reduction=114214 max_load_reduction_pct=0.871384 moved_to_echo=65066 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 13, 0, 4096, 0, 0, 4096, 4096, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 7, 3, 0, 0, 0, 2, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4071, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4020, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 76, 48, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4093, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 4091, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 4095, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16326, 16389, 16365, 16388, 16503, 16396, 16375, 16330] avg=16384.0000 max=16503 max_rank=4 min=16326 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=114569 max_load_reduction_pct=0.874092 moved_to_echo=65604 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 2, 0, 4096, 4096, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 13, 3, 0, 0, 0, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4089, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4020, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4096, 19, 4096, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4074, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 76, 48, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16313, 16388, 16383, 16398, 16490, 16392, 16379, 16329] avg=16384.0000 max=16490 max_rank=4 min=16313 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=114582 max_load_reduction_pct=0.874191 moved_to_echo=65590 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 4096, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4020, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 76, 48, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4095, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16305, 16387, 16386, 16384, 16510, 16384, 16379, 16337] avg=16384.0000 max=16510 max_rank=4 min=16305 min_rank=0 imbalance_ratio=1.007690 max_load_reduction=114562 max_load_reduction_pct=0.874039 moved_to_echo=65610 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 4096, 4096, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4020, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 76, 48, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4096, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16309, 16370, 16384, 16384, 16507, 16403, 16379, 16336] avg=16384.0000 max=16507 max_rank=4 min=16309 min_rank=0 imbalance_ratio=1.007507 max_load_reduction=114565 max_load_reduction_pct=0.874062 moved_to_echo=65625 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 4096, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 2, 0, 0, 0, 1, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4020, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 76, 54, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 4096, 0, 8, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16306, 16386, 16387, 16383, 16517, 16386, 16378, 16329] avg=16384.0000 max=16517 max_rank=4 min=16306 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=114555 max_load_reduction_pct=0.873985 moved_to_echo=65610 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 4096, 0, 0, 4096, 4096, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4020, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4050, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 76, 48, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16350, 16382, 16386, 16385, 16466, 16388, 16379, 16336] avg=16384.0000 max=16466 max_rank=4 min=16336 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=114606 max_load_reduction_pct=0.874374 moved_to_echo=65569 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 4096, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4019, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 77, 48, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4095, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16305, 16387, 16381, 16385, 16513, 16386, 16379, 16336] avg=16384.0000 max=16513 max_rank=4 min=16305 min_rank=0 imbalance_ratio=1.007874 max_load_reduction=114559 max_load_reduction_pct=0.874016 moved_to_echo=65614 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 32768, 0, 0, 128, 32768, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32384, 32768, 0, 0, 32768, 128, 0, 128, 512, 128, 128, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 0, 0, 0, 128, 512, 0, 32768, 128, 128, 32768, 32768, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 32768, 32768, 128, 0, 128, 128, 128, 0, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 32768, 128, 32768, 128, 128, 128, 128, 0, 0, 128, 128, 0, 128, 128, 128, 32768, 32256, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 0, 0, 128, 128, 31872, 128, 0, 128, 32768, 128, 128, 0, 0, 128, 128, 32768, 640, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 32768, 32768, 0, 32768, 0, 128, 0, 0, 128, 0, 256, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 32768, 32768, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True + [2026-08-30 05:05:02] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 950.4 | throughput per GPU (TFLOP/s/GPU): 350.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.055100E+00 | loss scale: 1.0 | grad norm: 5.380 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2218, 2055, 1712, 0, 613, 0, 1967, 711, 1055, 2650, 1206, 907, 1611, 1764, 0, 1054, 1013, 1456, 1266, 1515, 46, 561, 129, 2144, 151, 22, 2992, 1731, 1265, 49, 1214, 1020, 395, 1, 1587, 411, 266, 1664, 1205, 241, 100, 24, 1951, 2873, 6, 1935, 996, 886, 917, 776, 597, 1180, 2, 52, 33, 1978, 0, 188, 126, 587, 1350, 2930, 804, 734, 1121, 16, 0, 1254, 2202, 1759, 941, 765, 1345, 1652, 1945, 315, 1676, 14, 851, 2048, 849, 784, 9, 18, 1711, 495, 1002, 59, 2307, 125, 1332, 1059, 135, 2051, 854, 0, 943, 517, 19, 2250, 4, 447, 1949, 827, 279, 1982, 198, 1150, 756, 2053, 2300, 1637, 2030, 1931, 372, 1748, 2980, 906, 2003, 1428, 1195, 2055, 1345, 815, 827, 2, 533, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1534, 2025, 1960, 2, 505, 0, 1813, 691, 950, 2378, 1076, 782, 1633, 2517, 0, 1267, 721, 1497, 1386, 1765, 24, 863, 159, 2628, 374, 47, 2983, 1845, 1785, 41, 1501, 1532, 560, 42, 1578, 335, 434, 2053, 1477, 220, 315, 7, 2105, 2919, 2, 1818, 1129, 1308, 1125, 424, 615, 661, 5, 21, 35, 1993, 2, 187, 7, 481, 1520, 2940, 1049, 67, 1429, 2, 0, 1268, 2273, 1956, 1034, 1110, 1456, 1867, 2116, 352, 1798, 24, 836, 2496, 662, 371, 18, 36, 2315, 566, 1099, 2, 1582, 370, 741, 907, 318, 1517, 462, 0, 1173, 577, 4, 1639, 5, 553, 1104, 1121, 210, 1252, 124, 1569, 742, 1829, 2361, 958, 2288, 2336, 380, 818, 2670, 794, 2144, 1311, 1482, 1767, 1016, 561, 1036, 16, 403, 128] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2503, 2113, 1463, 0, 342, 0, 2198, 1050, 1113, 2949, 1532, 974, 1251, 2959, 1, 1164, 947, 1496, 1277, 1115, 17, 1192, 171, 1733, 99, 57, 3503, 1898, 1065, 66, 1449, 859, 270, 0, 1786, 901, 192, 2031, 1514, 99, 145, 0, 1779, 2822, 0, 2372, 861, 1025, 1251, 554, 933, 914, 12, 15, 12, 1954, 0, 393, 39, 442, 1058, 2795, 1396, 446, 813, 6, 0, 1912, 2787, 1096, 537, 918, 1173, 1323, 1658, 125, 1952, 11, 760, 2475, 547, 305, 14, 12, 1968, 534, 1086, 5, 2127, 198, 1724, 584, 261, 1704, 573, 0, 1187, 750, 5, 1986, 8, 390, 1380, 668, 284, 1381, 83, 1314, 322, 1134, 2701, 1845, 2121, 1797, 832, 963, 2535, 1244, 2194, 1536, 1258, 2254, 1133, 686, 711, 12, 598, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2280, 2521, 1826, 0, 425, 0, 2790, 715, 344, 3048, 1098, 597, 1492, 2559, 0, 989, 787, 1497, 1084, 1573, 16, 807, 156, 2046, 86, 49, 3482, 1682, 1406, 43, 1252, 1068, 396, 0, 1922, 750, 160, 1509, 1225, 303, 126, 4, 1800, 2755, 0, 1846, 890, 603, 1297, 472, 830, 528, 3, 18, 7, 2357, 0, 138, 17, 361, 1644, 3040, 1181, 138, 1086, 33, 0, 1487, 2131, 1313, 1014, 1338, 1704, 1535, 2467, 334, 2214, 9, 827, 2474, 1219, 566, 3, 38, 1825, 467, 554, 4, 2187, 244, 1114, 1032, 211, 1891, 699, 0, 689, 1407, 13, 2309, 8, 278, 1388, 1059, 138, 1128, 192, 1283, 412, 2122, 2119, 1849, 2289, 1524, 246, 696, 2537, 1220, 2237, 1415, 807, 2115, 1163, 668, 935, 3, 759, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1742, 2122, 1319, 0, 409, 0, 2484, 735, 1465, 2759, 1426, 570, 1241, 3077, 0, 1159, 1003, 1598, 1235, 1308, 9, 585, 161, 2317, 119, 168, 3247, 2168, 1397, 195, 1973, 991, 155, 0, 1920, 601, 221, 1897, 932, 321, 168, 3, 2246, 3087, 0, 2044, 770, 851, 845, 823, 753, 778, 8, 53, 2, 1685, 0, 149, 25, 561, 1369, 2945, 867, 198, 747, 15, 0, 1214, 1917, 1353, 648, 1023, 1316, 1682, 2132, 262, 1744, 5, 1256, 2452, 605, 349, 10, 4, 1632, 496, 915, 14, 2436, 140, 1018, 797, 407, 2261, 610, 0, 1331, 1156, 8, 2116, 5, 463, 1435, 1200, 142, 1740, 93, 1249, 112, 1683, 2693, 2277, 2467, 2101, 447, 738, 2497, 979, 1603, 1303, 1003, 2404, 1080, 497, 938, 11, 577, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2123, 2348, 1589, 0, 658, 0, 2623, 590, 928, 2764, 825, 795, 1026, 2625, 0, 1552, 690, 1400, 1477, 1494, 22, 826, 155, 1961, 106, 44, 3175, 1852, 1509, 75, 1853, 1239, 474, 2, 1555, 610, 347, 1587, 1534, 213, 125, 0, 2310, 2728, 0, 1811, 1178, 769, 1038, 818, 892, 934, 1, 51, 4, 2005, 0, 312, 61, 528, 1756, 2747, 1455, 216, 725, 57, 0, 1117, 1614, 1843, 577, 828, 1280, 1341, 1963, 394, 1646, 19, 572, 2298, 1202, 542, 14, 5, 2080, 607, 1582, 36, 2173, 106, 1051, 782, 142, 2338, 338, 0, 1365, 849, 6, 2101, 10, 460, 1572, 1161, 272, 1231, 99, 789, 594, 1723, 2435, 1932, 2114, 2080, 200, 792, 2643, 784, 2283, 1404, 1160, 1900, 1235, 580, 968, 5, 665, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2304, 1741, 1577, 1, 612, 1, 2135, 659, 1159, 2293, 1227, 429, 1088, 2606, 0, 1050, 892, 2212, 1081, 1244, 25, 534, 152, 2042, 72, 28, 3076, 2070, 1142, 87, 1252, 856, 536, 0, 1695, 546, 212, 2223, 1499, 286, 298, 5, 2187, 3010, 1, 1629, 1253, 1368, 1032, 707, 497, 470, 1, 14, 26, 1942, 0, 151, 80, 430, 684, 2856, 1190, 184, 891, 24, 0, 1701, 2482, 1852, 890, 955, 1324, 1981, 2224, 356, 1759, 4, 1420, 2002, 988, 582, 8, 16, 1589, 720, 1016, 24, 2711, 144, 1865, 695, 313, 1492, 571, 0, 958, 279, 55, 1467, 7, 304, 1167, 1010, 193, 1877, 196, 1519, 474, 1899, 2972, 2099, 2030, 2021, 507, 1639, 2445, 888, 1321, 1108, 954, 2635, 1187, 1099, 1016, 4, 457, 27] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2123, 1792, 1756, 2, 268, 0, 2588, 1104, 1076, 2121, 1149, 835, 1001, 1955, 0, 1019, 675, 1601, 1650, 1338, 21, 755, 180, 1843, 388, 49, 3365, 1998, 1598, 60, 1899, 1050, 218, 0, 2000, 975, 272, 1854, 1398, 533, 167, 0, 2490, 2836, 0, 1751, 1017, 831, 920, 758, 843, 449, 15, 39, 9, 1465, 1, 232, 41, 354, 1882, 2458, 1587, 433, 972, 15, 0, 1618, 1922, 2175, 1038, 1185, 880, 1727, 2282, 629, 1223, 6, 1029, 1825, 1120, 742, 10, 27, 2119, 491, 1702, 7, 2056, 163, 1490, 963, 82, 1507, 439, 4, 1074, 639, 2, 1968, 41, 416, 1335, 1190, 203, 1299, 45, 1308, 339, 1174, 1909, 1489, 1905, 2204, 353, 1593, 2331, 681, 2520, 1483, 1201, 2469, 1078, 738, 966, 11, 550, 23] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2218, 2055, 1712, 0, 613, 0, 1967, 711, 1055, 0, 1206, 907, 1611, 1093, 0, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1456, 1266, 1515, 46, 561, 129, 2144, 151, 22, 1805, 1731, 1265, 49, 1214, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 395, 1, 1587, 411, 266, 1664, 1205, 241, 100, 24, 1951, 2873, 6, 1935, 996, 886, 450, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 917, 776, 597, 1180, 2, 52, 33, 1978, 0, 188, 126, 587, 1350, 2930, 804, 734, 2650, 2434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1121, 16, 0, 1254, 2202, 1759, 941, 765, 1345, 1652, 1945, 315, 1676, 14, 851, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 849, 784, 9, 18, 1711, 495, 1002, 59, 2307, 125, 1332, 1059, 135, 2051, 854, 0, 1907, 1187, 546, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 943, 517, 19, 2250, 4, 447, 1949, 827, 279, 1982, 198, 1150, 756, 2053, 2300, 1637, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2030, 1931, 372, 1748, 0, 906, 2003, 1428, 1195, 1695, 1345, 815, 827, 2, 533, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16202, 15387, 15172, 17338, 15997, 16651, 17490, 16835] avg=16384.0000 max=17490 max_rank=6 min=15172 min_rank=2 imbalance_ratio=1.067505 max_load_reduction=113582 max_load_reduction_pct=0.866562 moved_to_echo=66973 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2503, 2113, 1463, 0, 342, 0, 2198, 1050, 1113, 0, 1532, 974, 1251, 1848, 1, 1164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 947, 1496, 1277, 1115, 17, 1192, 171, 1733, 99, 57, 2117, 1898, 1065, 66, 1449, 859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 0, 1786, 901, 192, 2031, 1514, 99, 145, 0, 1779, 2822, 0, 2372, 861, 1025, 747, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1251, 554, 933, 914, 12, 15, 12, 1954, 0, 393, 39, 442, 1058, 2795, 1396, 446, 2949, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 813, 6, 0, 1912, 2787, 1096, 537, 918, 1173, 1323, 1658, 125, 1952, 11, 760, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 305, 14, 12, 1968, 534, 1086, 5, 2127, 198, 1724, 584, 261, 1704, 573, 0, 2299, 1386, 465, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1187, 750, 5, 1986, 8, 390, 1380, 668, 284, 1381, 83, 1314, 322, 1134, 2701, 1845, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2121, 1797, 832, 963, 0, 1244, 2194, 1536, 1258, 1868, 1133, 686, 711, 12, 598, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17552, 15558, 16739, 17233, 15247, 16156, 15629, 16958] avg=16384.0000 max=17552 max_rank=0 min=15247 min_rank=4 imbalance_ratio=1.071289 max_load_reduction=113520 max_load_reduction_pct=0.866089 moved_to_echo=63990 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2280, 2521, 1826, 0, 425, 0, 2790, 715, 344, 0, 1098, 597, 1492, 1598, 0, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 1497, 1084, 1573, 16, 807, 156, 2046, 86, 49, 2105, 1682, 1406, 43, 1252, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 1922, 750, 160, 1509, 1225, 303, 126, 4, 1800, 2755, 0, 1846, 890, 603, 646, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1297, 472, 830, 528, 3, 18, 7, 2357, 0, 138, 17, 361, 1644, 3040, 1181, 138, 3048, 2072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 33, 0, 1487, 2131, 1313, 1014, 1338, 1704, 1535, 2467, 334, 2214, 9, 827, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1219, 566, 3, 38, 1825, 467, 554, 4, 2187, 244, 1114, 1032, 211, 1891, 699, 0, 2299, 1377, 465, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, 1407, 13, 2309, 8, 278, 1388, 1059, 138, 1128, 192, 1283, 412, 2122, 2119, 1849, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2289, 1524, 246, 696, 0, 1220, 2237, 1415, 807, 1752, 1163, 668, 935, 3, 759, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16675, 15657, 15118, 17151, 17667, 16510, 16574, 15720] avg=16384.0000 max=17667 max_rank=4 min=15118 min_rank=2 imbalance_ratio=1.078308 max_load_reduction=113405 max_load_reduction_pct=0.865211 moved_to_echo=66471 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2123, 2348, 1589, 0, 658, 0, 2623, 590, 928, 0, 825, 795, 1026, 1639, 0, 1552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 690, 1400, 1477, 1494, 22, 826, 155, 1961, 106, 44, 1919, 1852, 1509, 75, 1853, 1239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 2, 1555, 610, 347, 1587, 1534, 213, 125, 0, 2310, 2728, 0, 1811, 1178, 769, 663, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 818, 892, 934, 1, 51, 4, 2005, 0, 312, 61, 528, 1756, 2747, 1455, 216, 2764, 2157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 725, 57, 0, 1117, 1614, 1843, 577, 828, 1280, 1341, 1963, 394, 1646, 19, 572, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1202, 542, 14, 5, 2080, 607, 1582, 36, 2173, 106, 1051, 782, 142, 2338, 338, 0, 2135, 1256, 486, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1365, 849, 6, 2101, 10, 460, 1572, 1161, 272, 1231, 99, 789, 594, 1723, 2435, 1932, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2114, 2080, 200, 792, 0, 784, 2283, 1404, 1160, 1575, 1235, 580, 968, 5, 665, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16696, 16622, 16070, 17739, 14139, 17198, 16760, 15848] avg=16384.0000 max=17739 max_rank=3 min=14139 min_rank=4 imbalance_ratio=1.082703 max_load_reduction=113333 max_load_reduction_pct=0.864662 moved_to_echo=63945 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1534, 2025, 1960, 2, 505, 0, 1813, 691, 950, 0, 1076, 782, 1633, 1572, 0, 1267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 721, 1497, 1386, 1765, 24, 863, 159, 2628, 374, 47, 1803, 1845, 1785, 41, 1501, 1532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 42, 1578, 335, 434, 2053, 1477, 220, 315, 7, 2105, 2919, 2, 1818, 1129, 1308, 635, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1125, 424, 615, 661, 5, 21, 35, 1993, 2, 187, 7, 481, 1520, 2940, 1049, 67, 2378, 2179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1429, 2, 0, 1268, 2273, 1956, 1034, 1110, 1456, 1867, 2116, 352, 1798, 24, 836, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 371, 18, 36, 2315, 566, 1099, 2, 1582, 370, 741, 907, 318, 1517, 462, 0, 2319, 1180, 491, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1173, 577, 4, 1639, 5, 553, 1104, 1121, 210, 1252, 124, 1569, 742, 1829, 2361, 958, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2288, 2336, 380, 818, 0, 794, 2144, 1311, 1482, 1464, 1016, 561, 1036, 16, 403, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15810, 17971, 17090, 15689, 17698, 15266, 15371, 16177] avg=16384.0000 max=17971 max_rank=1 min=15266 min_rank=5 imbalance_ratio=1.096863 max_load_reduction=113101 max_load_reduction_pct=0.862892 moved_to_echo=64512 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2304, 1741, 1577, 1, 612, 1, 2135, 659, 1159, 0, 1227, 429, 1088, 1627, 0, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 892, 2212, 1081, 1244, 25, 534, 152, 2042, 72, 28, 1859, 2070, 1142, 87, 1252, 856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 0, 1695, 546, 212, 2223, 1499, 286, 298, 5, 2187, 3010, 1, 1629, 1253, 1368, 658, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 707, 497, 470, 1, 14, 26, 1942, 0, 151, 80, 430, 684, 2856, 1190, 184, 2293, 1997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 891, 24, 0, 1701, 2482, 1852, 890, 955, 1324, 1981, 2224, 356, 1759, 4, 1420, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 988, 582, 8, 16, 1589, 720, 1016, 24, 2711, 144, 1865, 695, 313, 1492, 571, 0, 1860, 1217, 448, 321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 958, 279, 55, 1467, 7, 304, 1167, 1010, 193, 1877, 196, 1519, 474, 1899, 2972, 2099, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2030, 2021, 507, 1639, 0, 888, 1321, 1108, 954, 2183, 1187, 1099, 1016, 4, 457, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15610, 15548, 17634, 14554, 18005, 16580, 16700, 16441] avg=16384.0000 max=18005 max_rank=4 min=14554 min_rank=3 imbalance_ratio=1.098938 max_load_reduction=113067 max_load_reduction_pct=0.862633 moved_to_echo=67726 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1742, 2122, 1319, 0, 409, 0, 2484, 735, 1465, 0, 1426, 570, 1241, 1921, 0, 1159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1598, 1235, 1308, 9, 585, 161, 2317, 119, 168, 1963, 2168, 1397, 195, 1973, 991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 1920, 601, 221, 1897, 932, 321, 168, 3, 2246, 3087, 0, 2044, 770, 851, 777, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 845, 823, 753, 778, 8, 53, 2, 1685, 0, 149, 25, 561, 1369, 2945, 867, 198, 2759, 2038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 747, 15, 0, 1214, 1917, 1353, 648, 1023, 1316, 1682, 2132, 262, 1744, 5, 1256, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 605, 349, 10, 4, 1632, 496, 915, 14, 2436, 140, 1018, 797, 407, 2261, 610, 0, 2278, 1284, 459, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1331, 1156, 8, 2116, 5, 463, 1435, 1200, 142, 1740, 93, 1249, 112, 1683, 2693, 2277, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2467, 2101, 447, 738, 0, 979, 1603, 1303, 1003, 1992, 1080, 497, 938, 11, 577, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16593, 17190, 16201, 15858, 15488, 16094, 17907, 15741] avg=16384.0000 max=17907 max_rank=6 min=15488 min_rank=4 imbalance_ratio=1.092957 max_load_reduction=113165 max_load_reduction_pct=0.863380 moved_to_echo=65230 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2123, 1792, 1756, 2, 268, 0, 2588, 1104, 1076, 0, 1149, 835, 1001, 1221, 0, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 675, 1601, 1650, 1338, 21, 755, 180, 1843, 388, 49, 2034, 1998, 1598, 60, 1899, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 2000, 975, 272, 1854, 1398, 533, 167, 0, 2490, 2836, 0, 1751, 1017, 831, 493, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 920, 758, 843, 449, 15, 39, 9, 1465, 1, 232, 41, 354, 1882, 2458, 1587, 433, 2121, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 972, 15, 0, 1618, 1922, 2175, 1038, 1185, 880, 1727, 2282, 629, 1223, 6, 1029, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1120, 742, 10, 27, 2119, 491, 1702, 7, 2056, 163, 1490, 963, 82, 1507, 439, 4, 1695, 1331, 428, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 639, 2, 1968, 41, 416, 1335, 1190, 203, 1299, 45, 1308, 339, 1174, 1909, 1489, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1905, 2204, 353, 1593, 0, 681, 2520, 1483, 1201, 2046, 1078, 738, 966, 11, 550, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15934, 17139, 17048, 15510, 16831, 16617, 14641, 17352] avg=16384.0000 max=17352 max_rank=7 min=14641 min_rank=6 imbalance_ratio=1.059082 max_load_reduction=113720 max_load_reduction_pct=0.867615 moved_to_echo=65441 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[6784, 12800, 10496, 11392, 256, 6144, 1280, 16768, 1408, 512, 15616, 15360, 11264, 640, 12416, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[8448, 5376, 6016, 6016, 128, 384, 128, 15488, 128, 1792, 512, 3840, 11264, 22784, 9600, 2432, 20992, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[7296, 4352, 128, 256, 15360, 4480, 8960, 256, 17664, 1536, 10368, 6912, 1920, 14848, 4608, 128, 16896, 10240, 3840, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3072, 128, 14080, 5248, 2176, 14848, 10880, 2304, 1536, 128, 16896, 23040, 128, 15232, 8192, 7680, 5120, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[16896, 16768, 13312, 128, 3840, 128, 18688, 6272, 8192, 0, 9600, 6016, 10368, 12544, 128, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[8832, 6272, 128, 15872, 128, 3328, 11392, 8320, 1792, 11904, 1152, 10240, 3840, 13696, 19584, 14208, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[7808, 256, 0, 11648, 17408, 13440, 6784, 8192, 10496, 13184, 16896, 2816, 14080, 128, 7552, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[17280, 16000, 3456, 9088, 0, 7552, 16384, 11008, 9088, 14592, 9344, 5760, 7424, 128, 4608, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4076, 1, 265, 5, 0, 1, 101, 18, 0, 1, 0, 4096, 0, 0, 4094, 0, 0, 4090, 0, 3403, 0, 1, 4077, 0, 4092, 4094, 39, 0, 1, 4096, 0, 2964, 3778, 31, 0, 1, 2224, 0, 1, 0, 0, 270, 0, 3, 3983, 4068, 0, 0, 4096, 6, 35, 4095, 0, 1, 4096, 42, 0, 0, 4095, 0, 326, 1966, 0, 0, 3083, 6, 1, 4091, 0, 3759, 4095, 0, 0, 0, 1, 0, 0, 2366, 1, 18, 0, 0, 2, 3796, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4054, 0, 5, 0, 25, 4096, 0, 4078, 0, 0, 0, 4092, 4095, 1, 1, 44, 0, 101, 0, 0, 4068, 215, 0, 82, 0, 0, 1, 4069] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 59, 5, 0, 0, 379, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 4089, 0, 3258, 3, 0, 4087, 0, 4096, 4095, 29, 0, 0, 4096, 0, 1483, 3956, 7, 0, 0, 3150, 0, 0, 0, 0, 445, 0, 20, 4029, 4085, 0, 0, 4096, 0, 26, 4096, 0, 0, 4096, 2, 0, 0, 4096, 0, 32, 2370, 0, 0, 3108, 0, 0, 4090, 0, 3334, 4093, 0, 0, 0, 0, 0, 0, 2937, 0, 1, 0, 0, 0, 2426, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3749, 8, 0, 0, 5, 4096, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 91, 0, 355, 0, 0, 4094, 124, 0, 1516, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 0, 14, 0, 0, 0, 153, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 3563, 3, 0, 4068, 0, 4094, 4096, 34, 0, 0, 4096, 0, 2694, 3911, 14, 1, 0, 2762, 0, 0, 2, 0, 421, 0, 0, 4056, 4093, 0, 0, 4096, 0, 125, 4096, 0, 0, 4096, 45, 0, 0, 4093, 0, 89, 2424, 0, 0, 2269, 0, 0, 4096, 0, 3791, 4096, 0, 0, 0, 1, 0, 0, 3688, 0, 4, 0, 0, 0, 1885, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3914, 8, 0, 0, 18, 4096, 0, 4096, 0, 0, 0, 4094, 4096, 0, 0, 90, 0, 224, 1, 0, 4095, 116, 0, 597, 0, 0, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4091, 0, 1, 6, 8, 2, 793, 3, 0, 0, 1, 4096, 0, 0, 4095, 0, 0, 4088, 0, 3548, 2, 0, 4076, 0, 4096, 4093, 3, 1, 0, 4096, 0, 3393, 3854, 10, 4, 0, 2984, 0, 0, 1, 0, 510, 0, 116, 4084, 4091, 0, 0, 4095, 0, 48, 4096, 0, 0, 4096, 102, 0, 0, 4096, 0, 2, 2557, 0, 0, 2031, 4, 0, 4091, 0, 2834, 4080, 1, 0, 0, 0, 0, 0, 2505, 5, 0, 0, 0, 0, 2360, 0, 0, 1, 0, 0, 4095, 1, 0, 0, 0, 1, 4095, 3874, 10, 29, 0, 30, 4096, 1, 4091, 1, 0, 0, 4095, 4095, 0, 0, 63, 0, 80, 1, 0, 4090, 149, 0, 930, 0, 0, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4096, 0, 23, 0, 0, 0, 422, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 4078, 3, 3744, 0, 0, 4092, 0, 4080, 4088, 0, 0, 0, 4096, 0, 2850, 3921, 2, 0, 0, 2658, 0, 0, 0, 0, 311, 0, 0, 4063, 4095, 0, 0, 4096, 25, 60, 4096, 0, 0, 4092, 320, 1, 0, 4068, 0, 17, 2412, 0, 0, 3262, 6, 0, 4090, 0, 1657, 4096, 0, 0, 0, 0, 0, 0, 2066, 0, 2, 0, 0, 0, 2901, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4002, 0, 76, 0, 127, 4096, 0, 4089, 5, 0, 0, 4094, 4096, 0, 0, 27, 0, 170, 8, 0, 4096, 8, 0, 1822, 0, 0, 0, 4085] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 4072, 0, 253, 1, 0, 0, 229, 10, 0, 0, 0, 4094, 1, 0, 4096, 0, 1, 4091, 2, 3731, 0, 1, 4002, 0, 4094, 4092, 47, 0, 0, 4094, 0, 2301, 3884, 37, 0, 0, 3162, 4, 1, 3, 0, 228, 0, 0, 4060, 4064, 0, 0, 4096, 1, 60, 4094, 0, 0, 4096, 45, 0, 0, 4056, 0, 71, 2896, 0, 0, 2139, 7, 0, 4094, 1, 3484, 4095, 0, 0, 0, 4, 0, 0, 2897, 0, 35, 0, 1, 1, 3131, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 4096, 4085, 15, 2, 0, 7, 4095, 0, 4090, 7, 0, 0, 4096, 4096, 0, 0, 20, 0, 74, 0, 0, 4071, 151, 0, 10, 2, 1, 0, 4088] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4072, 0, 613, 0, 0, 0, 162, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 2398, 0, 0, 4083, 0, 4095, 4096, 24, 0, 0, 4096, 0, 3332, 3937, 15, 0, 0, 3699, 0, 0, 3, 0, 392, 0, 0, 4034, 4074, 0, 0, 4096, 34, 97, 4096, 0, 0, 4096, 25, 0, 0, 4096, 0, 35, 2360, 0, 0, 1806, 2, 0, 4096, 0, 3520, 4096, 0, 0, 0, 0, 0, 0, 3214, 0, 52, 0, 0, 0, 1704, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 4072, 1, 4, 0, 43, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 132, 1, 0, 4068, 500, 1, 738, 0, 0, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4075, 0, 4, 0, 0, 2, 301, 0, 0, 0, 1, 4090, 0, 0, 4096, 0, 0, 4096, 1, 4020, 1, 0, 4026, 0, 4094, 4095, 13, 0, 0, 4096, 0, 3344, 3860, 5, 2, 0, 1293, 0, 0, 0, 0, 302, 0, 77, 4090, 4079, 0, 0, 4096, 1, 36, 4096, 0, 0, 4093, 238, 0, 0, 4096, 0, 30, 3239, 0, 0, 3332, 4, 0, 4095, 0, 1797, 4095, 0, 0, 0, 0, 0, 0, 3164, 0, 9, 0, 0, 0, 3072, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 3825, 41, 10, 0, 49, 4095, 0, 4061, 0, 0, 0, 4096, 4096, 1, 0, 335, 0, 236, 0, 0, 4090, 11, 2, 288, 0, 0, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4076, 1, 265, 5, 0, 1, 101, 18, 0, 1, 0, 4096, 4096, 3627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3403, 0, 1, 4077, 0, 4092, 654, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 2964, 3778, 31, 0, 1, 2224, 0, 1, 0, 0, 270, 0, 3, 2297, 101, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 4068, 0, 0, 0, 6, 35, 0, 0, 1, 3995, 42, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 1966, 0, 0, 3083, 6, 1, 4091, 0, 3759, 4035, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2366, 1, 18, 0, 0, 2, 3796, 0, 1, 0, 0, 0, 4095, 0, 0, 4095, 2195, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 4054, 0, 5, 0, 25, 4096, 0, 4078, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 44, 0, 101, 0, 0, 4068, 215, 0, 82, 0, 0, 1, 4069, 1807, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16289, 16356, 15827, 16225, 17268, 17036, 16444, 15627] avg=16384.0000 max=17268 max_rank=4 min=15627 min_rank=7 imbalance_ratio=1.053955 max_load_reduction=113804 max_load_reduction_pct=0.868256 moved_to_echo=66375 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 59, 5, 0, 0, 379, 0, 0, 0, 0, 4094, 4096, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 3258, 3, 0, 4087, 0, 4096, 655, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1483, 3956, 7, 0, 0, 3150, 0, 0, 0, 0, 445, 0, 20, 2297, 101, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 4085, 0, 0, 0, 0, 26, 0, 0, 0, 3995, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 2370, 0, 0, 3108, 0, 0, 4090, 0, 3334, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2937, 0, 1, 0, 0, 0, 2426, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2195, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 3749, 8, 0, 0, 5, 4096, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 91, 0, 355, 0, 0, 4094, 124, 0, 1516, 0, 0, 0, 4095, 1807, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16358, 16217, 15615, 16233, 16967, 16218, 16143, 17321] avg=16384.0000 max=17321 max_rank=7 min=15615 min_rank=2 imbalance_ratio=1.057190 max_load_reduction=113751 max_load_reduction_pct=0.867851 moved_to_echo=66649 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 0, 14, 0, 0, 0, 153, 0, 0, 0, 0, 4096, 4096, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3563, 3, 0, 4068, 0, 4094, 654, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2694, 3911, 14, 1, 0, 2762, 0, 0, 2, 0, 421, 0, 0, 2298, 101, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 4093, 0, 0, 0, 0, 125, 0, 0, 0, 3995, 45, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 2424, 0, 0, 2269, 0, 0, 4096, 0, 3791, 4036, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3688, 0, 4, 0, 0, 0, 1885, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2195, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 3914, 8, 0, 0, 18, 4096, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 90, 0, 224, 1, 0, 4095, 116, 0, 597, 0, 0, 0, 4093, 1807, 1144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16075, 16510, 16360, 16407, 16706, 16431, 16320, 16263] avg=16384.0000 max=16706 max_rank=4 min=16075 min_rank=0 imbalance_ratio=1.019653 max_load_reduction=114366 max_load_reduction_pct=0.872543 moved_to_echo=65720 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4091, 0, 1, 6, 8, 2, 793, 3, 0, 0, 1, 4096, 4095, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 3548, 2, 0, 4076, 0, 4096, 654, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3393, 3854, 10, 4, 0, 2984, 0, 0, 1, 0, 510, 0, 116, 2296, 101, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4091, 0, 0, 0, 0, 48, 0, 0, 0, 3995, 102, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2557, 0, 0, 2031, 4, 0, 4091, 0, 2834, 4020, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2505, 5, 0, 0, 0, 0, 2360, 0, 0, 1, 0, 0, 4095, 1, 0, 4096, 2194, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 95, 3874, 10, 29, 0, 30, 4096, 1, 4091, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 63, 0, 80, 1, 0, 4090, 149, 0, 930, 0, 0, 0, 4094, 1806, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16726, 16468, 17425, 16416, 15540, 15723, 16323, 16451] avg=16384.0000 max=17425 max_rank=2 min=15540 min_rank=4 imbalance_ratio=1.063538 max_load_reduction=113647 max_load_reduction_pct=0.867058 moved_to_echo=64037 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 11, 4072, 0, 253, 1, 0, 0, 229, 10, 0, 0, 0, 4094, 4096, 3630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4091, 2, 3731, 0, 1, 4002, 0, 4094, 654, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2301, 3884, 37, 0, 0, 3162, 4, 1, 3, 0, 228, 0, 0, 2296, 101, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 4064, 0, 0, 0, 1, 60, 0, 0, 0, 3995, 45, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 2896, 0, 0, 2139, 7, 0, 4094, 1, 3484, 4035, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2897, 0, 35, 0, 1, 1, 3131, 0, 0, 0, 0, 0, 4094, 2, 0, 4094, 2195, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 4085, 15, 2, 0, 7, 4095, 0, 4090, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 0, 74, 0, 0, 4071, 151, 0, 10, 2, 1, 0, 4088, 1807, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16396, 16624, 16171, 16281, 16731, 16916, 16491, 15462] avg=16384.0000 max=16916 max_rank=5 min=15462 min_rank=7 imbalance_ratio=1.032471 max_load_reduction=114156 max_load_reduction_pct=0.870941 moved_to_echo=65600 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4096, 0, 23, 0, 0, 0, 422, 0, 0, 0, 0, 4092, 4096, 3630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 3, 3744, 0, 0, 4092, 0, 4080, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2850, 3921, 2, 0, 0, 2658, 0, 0, 0, 0, 311, 0, 0, 2298, 102, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4095, 0, 0, 0, 25, 60, 0, 0, 0, 3990, 320, 1, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2412, 0, 0, 3262, 6, 0, 4090, 0, 1657, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2066, 0, 2, 0, 0, 0, 2901, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2199, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 4002, 0, 76, 0, 127, 4096, 0, 4089, 5, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 27, 0, 170, 8, 0, 4096, 8, 0, 1822, 0, 0, 0, 4085, 1808, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16361, 16642, 16305, 16622, 15473, 15826, 16578, 17265] avg=16384.0000 max=17265 max_rank=7 min=15473 min_rank=4 imbalance_ratio=1.053772 max_load_reduction=113807 max_load_reduction_pct=0.868279 moved_to_echo=65142 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4072, 0, 613, 0, 0, 0, 162, 0, 0, 0, 0, 4096, 4096, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2398, 0, 0, 4083, 0, 4095, 654, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3332, 3937, 15, 0, 0, 3699, 0, 0, 3, 0, 392, 0, 0, 2298, 101, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 4074, 0, 0, 0, 34, 97, 0, 0, 0, 3995, 25, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 2360, 0, 0, 1806, 2, 0, 4096, 0, 3520, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3214, 0, 52, 0, 0, 0, 1704, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 2195, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 4072, 1, 4, 0, 43, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 132, 1, 0, 4068, 500, 1, 738, 0, 0, 0, 4093, 1807, 1144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16669, 15349, 17933, 16355, 15855, 15825, 16502, 16584] avg=16384.0000 max=17933 max_rank=2 min=15349 min_rank=1 imbalance_ratio=1.094543 max_load_reduction=113139 max_load_reduction_pct=0.863182 moved_to_echo=64766 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4075, 0, 4, 0, 0, 2, 301, 0, 0, 0, 1, 4090, 4096, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4020, 1, 0, 4026, 0, 4094, 655, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3344, 3860, 5, 2, 0, 1293, 0, 0, 0, 0, 302, 0, 77, 2297, 100, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4079, 0, 0, 0, 1, 36, 0, 0, 0, 3993, 238, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 3239, 0, 0, 3332, 4, 0, 4095, 0, 1797, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3164, 0, 9, 0, 0, 0, 3072, 0, 0, 1, 0, 0, 4093, 0, 0, 4096, 2195, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 3825, 41, 10, 0, 49, 4095, 0, 4061, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 335, 0, 236, 0, 0, 4090, 11, 2, 288, 0, 0, 0, 4090, 1807, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16198, 16906, 15436, 16533, 16532, 17097, 16271, 16099] avg=16384.0000 max=17097 max_rank=5 min=15436 min_rank=2 imbalance_ratio=1.043518 max_load_reduction=113975 max_load_reduction_pct=0.869560 moved_to_echo=65999 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 22912, 128, 128, 0, 128, 128, 21376, 0, 128, 128, 0, 0, 32768, 128, 128, 32768, 17664, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 1280, 128, 128, 128, 2560, 128, 0, 128, 128, 32768, 32768, 29056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32512, 32768, 0, 0, 0, 128, 512, 0, 0, 128, 32000, 896, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 768, 0, 1408, 128, 0, 32768, 1280, 128, 6016, 128, 128, 128, 32768, 14464, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[640, 20224, 0, 0, 21120, 128, 128, 32768, 128, 24192, 32384, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 0, 22400, 31104, 128, 128, 128, 22016, 128, 128, 128, 0, 2944, 0, 256, 18432, 896, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 128, 768, 31616, 128, 128, 0, 384, 32768, 128, 32768, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 0, 128, 32768, 128, 27776, 128, 128, 32512, 0, 32768, 5248, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2391, 0, 168, 0, 0, 229, 0, 4096, 92, 0, 0, 0, 0, 4096, 0, 0, 0, 157, 4075, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 3402, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 1, 4096, 0, 0, 0, 0, 4093, 4096, 1402, 0, 0, 1826, 0, 4096, 4096, 0, 0, 4090, 4096, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4091, 4096, 0, 0, 31, 4062, 0, 178, 0, 433, 0, 0, 4087, 0, 0, 0, 3866, 0, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 80, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 2193, 0, 4096, 2, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2969, 0, 113, 0, 0, 105, 0, 4093, 237, 1, 0, 0, 0, 4095, 0, 0, 0, 147, 4062, 0, 4095, 0, 4096, 4095, 0, 0, 2, 4096, 2413, 0, 5, 1, 4096, 0, 0, 4096, 2, 0, 0, 0, 4095, 0, 0, 0, 0, 4088, 4096, 592, 0, 1, 911, 1, 4095, 4092, 0, 0, 4080, 4054, 0, 0, 3, 4096, 0, 0, 4094, 2, 0, 0, 0, 0, 1, 4096, 0, 0, 4095, 0, 11, 4068, 4088, 0, 0, 59, 4062, 0, 889, 0, 523, 1, 1, 4084, 0, 0, 0, 3855, 1, 17, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 1, 0, 45, 4095, 0, 0, 2, 0, 0, 0, 4095, 0, 3661, 0, 4094, 9, 0, 1, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2758, 0, 301, 0, 0, 358, 0, 4093, 4, 0, 0, 0, 0, 4096, 2, 0, 0, 38, 4070, 0, 4093, 0, 4096, 4095, 0, 0, 0, 4095, 2757, 0, 1, 1, 4095, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 4093, 4086, 1664, 0, 0, 2074, 0, 4096, 4092, 0, 0, 4079, 4087, 0, 0, 0, 4092, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 10, 8, 3867, 4094, 0, 0, 14, 3985, 0, 312, 0, 468, 0, 1, 4063, 0, 0, 1, 2909, 2, 13, 1, 0, 12, 0, 4095, 1, 0, 0, 0, 1, 2, 0, 4, 4096, 0, 0, 3, 0, 0, 0, 4094, 0, 3112, 0, 4094, 7, 1, 2, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2761, 0, 1739, 0, 0, 719, 0, 4090, 321, 1, 0, 0, 0, 4094, 0, 0, 0, 6, 4023, 0, 4092, 0, 4096, 4093, 0, 0, 0, 4094, 2655, 0, 1, 0, 4096, 4, 0, 4096, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 4095, 4081, 1080, 0, 0, 721, 0, 4096, 4092, 0, 0, 4063, 4026, 0, 0, 4, 4096, 0, 0, 4096, 12, 0, 0, 0, 0, 2, 4095, 0, 0, 4096, 1, 25, 3976, 4094, 0, 0, 24, 4004, 0, 111, 0, 246, 1, 0, 4066, 0, 0, 0, 2916, 2, 0, 0, 0, 4, 0, 4071, 0, 0, 0, 0, 2, 0, 0, 19, 4090, 0, 0, 7, 0, 0, 0, 4087, 0, 3447, 0, 4096, 14, 1, 3, 0, 18] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2530, 0, 795, 0, 0, 1036, 0, 4096, 176, 0, 0, 1, 0, 4096, 0, 0, 0, 33, 4087, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 1180, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 1, 0, 0, 4087, 4091, 3341, 0, 0, 795, 0, 4096, 4096, 0, 0, 4094, 4091, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 6, 0, 4013, 4096, 0, 0, 7, 4059, 0, 36, 0, 90, 0, 0, 4040, 0, 0, 0, 3690, 0, 3, 0, 0, 29, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 2838, 0, 4096, 0, 0, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2155, 0, 730, 0, 0, 1192, 0, 4096, 25, 0, 0, 0, 0, 4096, 0, 0, 0, 116, 3986, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 1745, 0, 0, 0, 4096, 0, 0, 4096, 8, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 4095, 2856, 0, 0, 2151, 0, 4096, 4096, 0, 0, 4088, 4083, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 1, 4042, 4095, 0, 0, 5, 4026, 0, 103, 0, 221, 0, 0, 4052, 0, 0, 0, 3217, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 2142, 0, 4096, 3, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2539, 0, 417, 0, 0, 192, 0, 4096, 457, 0, 0, 0, 0, 4095, 0, 0, 0, 67, 4057, 0, 4095, 0, 4096, 4096, 0, 0, 0, 4092, 1726, 0, 6, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 4094, 4080, 1131, 0, 0, 1472, 0, 4096, 4096, 0, 0, 4073, 4051, 0, 0, 15, 4095, 0, 0, 4093, 27, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 12, 3997, 4092, 0, 0, 58, 3991, 0, 482, 0, 831, 0, 0, 4042, 0, 0, 0, 3922, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 139, 4091, 0, 0, 9, 0, 0, 0, 4089, 0, 3254, 0, 4085, 12, 0, 11, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2803, 0, 95, 0, 0, 227, 0, 4094, 100, 0, 0, 0, 0, 4095, 1, 0, 0, 81, 4089, 0, 4095, 0, 4096, 4095, 0, 0, 0, 4093, 1243, 0, 2, 0, 4096, 0, 0, 4095, 1, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 4087, 1600, 0, 0, 2006, 0, 4094, 4082, 0, 0, 4076, 4038, 0, 0, 2, 4096, 0, 0, 4092, 4, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 24, 4059, 4096, 0, 0, 220, 4021, 0, 1298, 0, 1008, 1, 0, 4042, 0, 0, 0, 3660, 0, 1, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 3, 0, 0, 36, 4096, 0, 0, 2, 0, 0, 0, 4089, 0, 2257, 0, 4092, 7, 0, 4, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2391, 0, 168, 0, 0, 229, 0, 4096, 92, 0, 0, 0, 0, 4096, 2178, 1977, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 4075, 0, 4096, 0, 0, 1918, 0, 0, 0, 4096, 3402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 2, 0, 0, 1, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1402, 0, 0, 1826, 0, 0, 945, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4091, 58, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3840, 0, 0, 31, 4062, 0, 178, 0, 433, 0, 0, 4087, 0, 0, 0, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 80, 4096, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2193, 0, 4096, 2, 0, 0, 0, 0, 4095, 1174, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15425, 17744, 16378, 16455, 16443, 16497, 16425, 15705] avg=16384.0000 max=17744 max_rank=1 min=15425 min_rank=0 imbalance_ratio=1.083008 max_load_reduction=113328 max_load_reduction_pct=0.864624 moved_to_echo=65070 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2758, 0, 301, 0, 0, 358, 0, 4093, 4, 0, 0, 0, 0, 4096, 2178, 1975, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 38, 4070, 0, 4093, 0, 0, 1917, 0, 0, 0, 4095, 2757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4089, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1664, 0, 0, 2074, 0, 0, 944, 0, 0, 4079, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 10, 8, 3867, 58, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 0, 0, 14, 3985, 0, 312, 0, 468, 0, 1, 4063, 0, 0, 1, 2909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 13, 1, 0, 12, 0, 4095, 1, 0, 0, 0, 1, 2, 0, 4, 4096, 4096, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4094, 0, 3112, 0, 4094, 7, 1, 2, 0, 1, 4092, 1173, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15961, 16972, 16379, 16934, 16237, 15591, 16368, 16630] avg=16384.0000 max=16972 max_rank=1 min=15591 min_rank=5 imbalance_ratio=1.035889 max_load_reduction=114100 max_load_reduction_pct=0.870514 moved_to_echo=64826 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2969, 0, 113, 0, 0, 105, 0, 4093, 237, 1, 0, 0, 0, 4095, 2178, 1975, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 4062, 0, 4095, 0, 0, 1917, 0, 0, 2, 4096, 2413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 4090, 0, 0, 4096, 2, 0, 0, 0, 4095, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 592, 0, 1, 911, 1, 0, 944, 0, 0, 4080, 4054, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 1, 4096, 0, 0, 4095, 0, 11, 4068, 58, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3833, 0, 0, 59, 4062, 0, 889, 0, 523, 1, 1, 4084, 0, 0, 0, 3855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 17, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 1, 0, 45, 4095, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 0, 3661, 0, 4094, 9, 0, 1, 0, 3, 4096, 1173, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15963, 16732, 16377, 14682, 16431, 17307, 16397, 17183] avg=16384.0000 max=17307 max_rank=5 min=14682 min_rank=3 imbalance_ratio=1.056335 max_load_reduction=113765 max_load_reduction_pct=0.867958 moved_to_echo=67318 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2155, 0, 730, 0, 0, 1192, 0, 4096, 25, 0, 0, 0, 0, 4096, 2178, 1977, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 3986, 0, 4096, 0, 0, 1918, 0, 0, 0, 4096, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 8, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2856, 0, 0, 2151, 0, 0, 945, 0, 0, 4088, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 1, 4042, 58, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3839, 0, 0, 5, 4026, 0, 103, 0, 221, 0, 0, 4052, 0, 0, 0, 3217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2142, 0, 4096, 3, 0, 0, 0, 0, 4096, 1174, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16647, 15957, 16387, 18218, 16396, 15463, 16347, 15657] avg=16384.0000 max=18218 max_rank=3 min=15463 min_rank=5 imbalance_ratio=1.111938 max_load_reduction=112854 max_load_reduction_pct=0.861008 moved_to_echo=63863 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2761, 0, 1739, 0, 0, 719, 0, 4090, 321, 1, 0, 0, 0, 4094, 2177, 1975, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4023, 0, 4092, 0, 0, 1916, 0, 0, 0, 4094, 2655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 4, 0, 4096, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 1080, 0, 0, 721, 0, 0, 944, 0, 0, 4063, 4026, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 2, 4095, 0, 0, 4096, 1, 25, 3976, 58, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 0, 0, 24, 4004, 0, 111, 0, 246, 1, 0, 4066, 0, 0, 0, 2916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 4071, 0, 0, 0, 0, 2, 0, 0, 19, 4090, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4087, 0, 3447, 0, 4096, 14, 1, 3, 0, 18, 4096, 1173, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18075, 16786, 16397, 14919, 16367, 15206, 16330, 16992] avg=16384.0000 max=18075 max_rank=0 min=14919 min_rank=3 imbalance_ratio=1.103210 max_load_reduction=112997 max_load_reduction_pct=0.862099 moved_to_echo=64895 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2530, 0, 795, 0, 0, 1036, 0, 4096, 176, 0, 0, 1, 0, 4096, 2178, 1977, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4087, 0, 4096, 0, 0, 1918, 0, 0, 0, 4096, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 1, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3341, 0, 0, 795, 0, 0, 945, 0, 0, 4094, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 6, 0, 4013, 58, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3840, 0, 0, 7, 4059, 0, 36, 0, 90, 0, 0, 4040, 0, 0, 0, 3690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 29, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2838, 0, 4096, 0, 0, 0, 0, 2, 4095, 1174, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17083, 15410, 16371, 17357, 16371, 15762, 16367, 16351] avg=16384.0000 max=17357 max_rank=3 min=15410 min_rank=1 imbalance_ratio=1.059387 max_load_reduction=113715 max_load_reduction_pct=0.867577 moved_to_echo=64851 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2539, 0, 417, 0, 0, 192, 0, 4096, 457, 0, 0, 0, 0, 4095, 2181, 1982, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 4057, 0, 4095, 0, 0, 1915, 0, 0, 0, 4092, 1726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4089, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 1131, 0, 0, 1472, 0, 0, 934, 0, 0, 4073, 4051, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 27, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 12, 3997, 63, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3830, 0, 0, 58, 3991, 0, 482, 0, 831, 0, 0, 4042, 0, 0, 0, 3922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 139, 4091, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4089, 0, 3254, 0, 4085, 12, 0, 11, 0, 0, 4095, 1180, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16158, 15952, 16402, 15756, 16391, 17156, 16472, 16785] avg=16384.0000 max=17156 max_rank=5 min=15756 min_rank=3 imbalance_ratio=1.047119 max_load_reduction=113916 max_load_reduction_pct=0.869110 moved_to_echo=66804 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2803, 0, 95, 0, 0, 227, 0, 4094, 100, 0, 0, 0, 0, 4095, 2178, 1970, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 81, 4089, 0, 4095, 0, 0, 1917, 0, 0, 0, 4093, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4090, 0, 0, 4095, 1, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1600, 0, 0, 2006, 0, 0, 942, 0, 0, 4076, 4038, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 24, 4059, 58, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3840, 0, 0, 220, 4021, 0, 1298, 0, 1008, 1, 0, 4042, 0, 0, 0, 3660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 3, 0, 0, 36, 4096, 4096, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4089, 0, 2257, 0, 4092, 7, 0, 4, 0, 2, 4096, 1170, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15760, 15519, 16381, 16751, 16436, 18090, 16366, 15769] avg=16384.0000 max=18090 max_rank=5 min=15519 min_rank=1 imbalance_ratio=1.104126 max_load_reduction=112982 max_load_reduction_pct=0.861984 moved_to_echo=66661 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 0, 32768, 128, 0, 0, 128, 32768, 0, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[30720, 0, 0, 512, 32256, 0, 3456, 0, 3840, 128, 128, 32512, 0, 0, 128, 28160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 128, 128, 0, 384, 32768, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 13696, 0, 128, 12032, 128, 0, 7552, 0, 0, 32768, 32640, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 20992, 0, 4480, 0, 0, 4096, 0, 32768, 1536, 128, 0, 128, 0, 32768, 17536, 15872, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 0, 0, 32768, 0, 22912, 0, 32768, 128, 128, 128, 0, 128, 32768, 9472, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 0, 0, 768, 32512, 0, 32768, 0, 0, 15360, 0, 0, 128, 32768, 17152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0, 128, 32768, 0, 0, 32768, 128, 128, 32128, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4094, 0, 0, 11, 4096, 4096, 0, 0, 0, 4096, 0, 4087, 3, 0, 4094, 0, 0, 0, 4096, 0, 271, 4096, 0, 0, 4096, 0, 0, 4096, 3936, 0, 0, 0, 0, 4096, 0, 279, 4096, 0, 4091, 0, 1, 17, 0, 4096, 4096, 2503, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 39, 1, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 800, 0, 0, 23, 0, 0, 0, 4085, 0, 0, 2201, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 2223, 0, 0, 0, 0, 0, 4096, 3, 19, 0, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 4081, 3, 0, 189, 4092, 4096, 0, 0, 0, 4096, 0, 4022, 0, 0, 4052, 0, 0, 13, 4086, 0, 186, 4096, 0, 0, 4095, 0, 0, 4096, 3963, 0, 0, 0, 0, 4096, 1, 633, 4096, 53, 4089, 0, 0, 39, 0, 4096, 4096, 2188, 0, 0, 4096, 4068, 2, 0, 0, 0, 0, 13, 7, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 4096, 4076, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4086, 4094, 0, 0, 0, 0, 1170, 0, 0, 40, 0, 0, 0, 4017, 0, 0, 3272, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 30, 4096, 0, 0, 0, 767, 0, 0, 0, 0, 0, 4095, 2, 5, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 4086, 1, 0, 112, 4095, 4096, 0, 0, 0, 4096, 0, 4085, 5, 0, 4073, 0, 0, 4, 4092, 0, 1084, 4096, 0, 0, 4094, 0, 0, 4096, 3682, 1, 0, 0, 0, 4095, 1, 486, 4096, 13, 4087, 0, 0, 19, 0, 4096, 4096, 3729, 0, 0, 4096, 4074, 1, 0, 0, 0, 0, 10, 5, 0, 4096, 0, 0, 0, 4, 4095, 0, 0, 4094, 4087, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4091, 4095, 0, 0, 0, 0, 784, 0, 0, 15, 0, 0, 2, 4033, 0, 0, 1804, 1, 0, 5, 0, 4, 0, 0, 4096, 1, 0, 0, 0, 35, 4096, 0, 0, 0, 583, 0, 0, 0, 0, 0, 4096, 37, 22, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 18, 4096, 4096, 0, 0, 0, 4096, 0, 4071, 2, 0, 4096, 0, 0, 0, 4096, 0, 193, 4093, 0, 0, 4096, 0, 0, 4096, 4001, 0, 0, 0, 0, 4096, 0, 302, 4096, 5, 4096, 0, 0, 48, 0, 4096, 4096, 2022, 0, 0, 4096, 4066, 0, 0, 0, 0, 0, 3, 3, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 1591, 0, 0, 18, 0, 0, 0, 4077, 0, 0, 3283, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 872, 0, 0, 0, 0, 0, 4096, 2, 4, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 145, 4096, 4096, 0, 0, 0, 4095, 0, 4096, 17, 0, 4012, 0, 0, 0, 4096, 0, 194, 4071, 0, 0, 4096, 0, 0, 4093, 3982, 0, 3, 0, 0, 4096, 0, 242, 4096, 3, 4095, 0, 0, 3, 0, 4096, 4096, 2453, 0, 0, 4096, 4085, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 2, 0, 1482, 0, 0, 0, 0, 0, 1, 4077, 0, 0, 2576, 0, 0, 32, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 27, 4095, 0, 0, 0, 1258, 0, 0, 0, 0, 0, 4096, 6, 9, 0, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 4092, 0, 0, 9, 4094, 4096, 0, 0, 1, 4096, 0, 3902, 0, 0, 4095, 0, 0, 10, 4093, 0, 481, 4096, 0, 0, 4095, 0, 0, 4096, 4009, 0, 0, 0, 0, 4095, 0, 925, 4096, 36, 3651, 0, 0, 50, 0, 4096, 4096, 3520, 0, 1, 4096, 4091, 2, 0, 0, 0, 0, 42, 10, 0, 4096, 0, 0, 0, 2, 4094, 0, 0, 4093, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 4095, 0, 0, 0, 0, 171, 0, 0, 52, 0, 1, 0, 4048, 0, 0, 1367, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 51, 4096, 0, 0, 0, 1385, 0, 0, 0, 0, 0, 4096, 18, 867, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 11, 4082, 0, 0, 54, 4095, 4096, 0, 0, 0, 4096, 0, 4076, 0, 0, 4008, 0, 0, 5, 4092, 0, 511, 4096, 0, 0, 4095, 0, 0, 4096, 4048, 1, 0, 0, 0, 4095, 2, 556, 4096, 3, 4052, 0, 2, 1, 0, 4096, 4096, 3180, 0, 0, 4096, 4091, 0, 0, 0, 0, 1, 16, 0, 0, 4096, 0, 0, 0, 1, 4094, 0, 0, 4096, 4086, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4094, 4095, 0, 0, 0, 0, 955, 0, 0, 12, 0, 0, 0, 4055, 0, 0, 2531, 0, 0, 0, 0, 2, 0, 0, 4096, 3, 0, 0, 0, 23, 4096, 0, 0, 0, 601, 0, 0, 0, 0, 0, 4095, 2, 4, 0, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 4091, 5, 0, 20, 4087, 4096, 0, 0, 0, 4096, 0, 3839, 0, 0, 4076, 0, 0, 14, 4085, 0, 477, 4096, 0, 0, 4094, 0, 0, 4096, 3513, 0, 0, 0, 0, 4096, 0, 352, 4096, 45, 4037, 0, 0, 0, 0, 4096, 4096, 2763, 0, 0, 4096, 4046, 0, 0, 0, 0, 0, 46, 11, 0, 4096, 0, 0, 0, 2, 4086, 0, 0, 4096, 4074, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4084, 4096, 0, 0, 0, 0, 1535, 0, 0, 54, 0, 12, 0, 4044, 0, 0, 3006, 0, 0, 0, 0, 5, 0, 0, 4096, 2, 0, 0, 0, 109, 4096, 0, 0, 0, 693, 0, 0, 0, 0, 0, 4096, 6, 115, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4094, 0, 0, 11, 4096, 4030, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 3, 0, 4094, 0, 0, 0, 4096, 0, 271, 3776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3936, 0, 0, 0, 0, 3857, 0, 279, 0, 0, 4091, 0, 1, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1296, 2503, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 39, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 2548, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3053, 0, 4096, 4096, 0, 0, 1, 0, 800, 0, 0, 23, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2201, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4096, 4096, 1548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2223, 0, 0, 0, 0, 0, 4096, 3, 19, 0, 4094, 0, 1241, 1043, 320, 239, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16328, 16327, 16277, 16110, 16396, 16154, 16040, 17440] avg=16384.0000 max=17440 max_rank=7 min=16040 min_rank=6 imbalance_ratio=1.064453 max_load_reduction=113632 max_load_reduction_pct=0.866943 moved_to_echo=66030 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 18, 4096, 4030, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 2, 0, 4096, 0, 0, 0, 4096, 0, 193, 3774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4001, 0, 0, 0, 0, 3857, 0, 302, 0, 5, 4096, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1296, 2022, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 3, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 2547, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3052, 0, 4096, 4096, 0, 0, 0, 0, 1591, 0, 0, 18, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3283, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 1549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 872, 0, 0, 0, 0, 0, 4096, 2, 4, 0, 4095, 0, 1241, 1042, 319, 239, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16336, 16232, 16405, 15582, 16394, 16930, 17121, 16072] avg=16384.0000 max=17121 max_rank=6 min=15582 min_rank=3 imbalance_ratio=1.044983 max_load_reduction=113951 max_load_reduction_pct=0.869377 moved_to_echo=66517 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 4086, 1, 0, 112, 4095, 4030, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 5, 0, 4073, 0, 0, 4, 4092, 0, 1084, 3776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3682, 1, 0, 0, 0, 3857, 1, 486, 0, 13, 4087, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1296, 3729, 0, 0, 0, 4074, 1, 0, 0, 0, 0, 10, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 0, 0, 4094, 4087, 0, 0, 0, 0, 1, 0, 0, 2546, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3053, 0, 4091, 4095, 0, 0, 0, 0, 784, 0, 0, 15, 0, 0, 2, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1804, 1, 0, 5, 0, 4, 0, 0, 4096, 1, 0, 0, 0, 35, 4096, 4096, 1548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 583, 0, 0, 0, 0, 0, 4096, 37, 22, 0, 4095, 0, 1241, 1043, 320, 238, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16422, 17119, 16242, 17307, 16386, 16073, 15686, 15837] avg=16384.0000 max=17307 max_rank=3 min=15686 min_rank=6 imbalance_ratio=1.056335 max_load_reduction=113765 max_load_reduction_pct=0.867958 moved_to_echo=63982 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 145, 4096, 4030, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 0, 4012, 0, 0, 0, 4096, 0, 194, 3753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3982, 0, 3, 0, 0, 3857, 0, 242, 0, 3, 4095, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1296, 2453, 0, 0, 0, 4085, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 2548, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3052, 0, 4096, 4096, 0, 0, 2, 0, 1482, 0, 0, 0, 0, 0, 1, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2576, 0, 0, 32, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 27, 4096, 4096, 1548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1258, 0, 0, 0, 0, 0, 4096, 6, 9, 0, 4094, 0, 1241, 1042, 318, 239, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16462, 16168, 16278, 16027, 16396, 16806, 16471, 16464] avg=16384.0000 max=16806 max_rank=5 min=16027 min_rank=3 imbalance_ratio=1.025757 max_load_reduction=114266 max_load_reduction_pct=0.871780 moved_to_echo=66137 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 11, 4082, 0, 0, 54, 4095, 4030, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4008, 0, 0, 5, 4092, 0, 511, 3776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4048, 1, 0, 0, 0, 3857, 2, 556, 0, 3, 4052, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1296, 3180, 0, 0, 0, 4091, 0, 0, 0, 0, 1, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 4096, 4086, 0, 0, 0, 0, 1, 0, 0, 2547, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3053, 0, 4094, 4095, 0, 0, 0, 0, 955, 0, 0, 12, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2531, 0, 0, 0, 0, 2, 0, 0, 4096, 3, 0, 0, 0, 23, 4096, 4096, 1548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 601, 0, 0, 0, 0, 0, 4095, 2, 4, 0, 4093, 0, 1241, 1043, 320, 238, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16368, 16468, 16618, 16776, 16384, 16264, 16395, 15799] avg=16384.0000 max=16776 max_rank=3 min=15799 min_rank=7 imbalance_ratio=1.023926 max_load_reduction=114296 max_load_reduction_pct=0.872009 moved_to_echo=64842 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 4091, 5, 0, 20, 4087, 4026, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3839, 0, 0, 4076, 0, 0, 14, 4085, 0, 477, 3775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3513, 0, 0, 0, 0, 3854, 0, 352, 0, 45, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1292, 2763, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 46, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4086, 0, 0, 4096, 4074, 0, 0, 0, 0, 0, 0, 0, 2547, 1563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3048, 0, 4084, 4096, 0, 0, 0, 0, 1535, 0, 0, 54, 0, 12, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3006, 0, 0, 0, 0, 5, 0, 0, 4096, 2, 0, 0, 0, 109, 4096, 4096, 1547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 693, 0, 0, 0, 0, 0, 4096, 6, 115, 0, 4096, 0, 1241, 1048, 321, 242, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16337, 16266, 15897, 16350, 16368, 16873, 16957, 16024] avg=16384.0000 max=16957 max_rank=6 min=15897 min_rank=2 imbalance_ratio=1.034973 max_load_reduction=114115 max_load_reduction_pct=0.870628 moved_to_echo=66222 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 4092, 0, 0, 9, 4094, 4030, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 0, 4095, 0, 0, 10, 4093, 0, 481, 3776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4009, 0, 0, 0, 0, 3857, 0, 925, 0, 36, 3651, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1296, 3520, 0, 1, 0, 4091, 2, 0, 0, 0, 0, 42, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 4093, 4082, 0, 0, 0, 0, 0, 0, 0, 2546, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3053, 0, 4092, 4095, 0, 0, 0, 0, 171, 0, 0, 52, 0, 1, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1367, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 51, 4096, 4096, 1549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1385, 0, 0, 0, 0, 0, 4096, 18, 867, 0, 4096, 0, 1241, 1043, 320, 238, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16325, 16357, 16624, 17154, 16376, 15512, 15258, 17466] avg=16384.0000 max=17466 max_rank=7 min=15258 min_rank=6 imbalance_ratio=1.066040 max_load_reduction=113606 max_load_reduction_pct=0.866745 moved_to_echo=64612 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 4081, 3, 0, 189, 4092, 4030, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 4052, 0, 0, 13, 4086, 0, 186, 3776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3963, 0, 0, 0, 0, 3857, 1, 633, 0, 53, 4089, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1296, 2188, 0, 0, 0, 4068, 2, 0, 0, 0, 0, 13, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 4076, 0, 0, 0, 0, 2, 0, 0, 2547, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3053, 0, 4086, 4094, 0, 0, 0, 0, 1170, 0, 0, 40, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3272, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 30, 4096, 4096, 1548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 767, 0, 0, 0, 0, 0, 4095, 2, 5, 0, 4096, 0, 1241, 1043, 320, 239, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16494, 16135, 16731, 15766, 16372, 16460, 17144, 15970] avg=16384.0000 max=17144 max_rank=6 min=15766 min_rank=3 imbalance_ratio=1.046387 max_load_reduction=113928 max_load_reduction_pct=0.869202 moved_to_echo=65946 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 10368, 22400, 0, 128, 0, 32640, 128, 0, 0, 0, 128, 256, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 32768, 128, 0, 640, 32768, 32256, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[32256, 128, 0, 32512, 0, 0, 128, 32768, 0, 3456, 30208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 0, 8448, 0, 0, 0, 0, 0, 32768, 128, 1152, 0, 32768, 0, 9984, 8448, 2560, 1920, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 20096, 128, 0, 128, 0, 128, 0, 0, 32768, 128, 0, 0, 0, 384, 32768, 32768, 12416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[24448, 0, 32768, 32768, 0, 0, 128, 0, 8576, 0, 0, 256, 0, 128, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 31232, 128, 128, 0, 0, 30976, 128, 3840, 0, 256, 32256, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 32768, 0, 0, 32768, 32768, 0, 0, 0, 0, 128, 0, 0, 20480, 12544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[62, 0, 4084, 0, 0, 4092, 4092, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 4043, 0, 0, 0, 0, 736, 1331, 0, 0, 0, 4096, 0, 0, 0, 35, 0, 0, 0, 343, 4096, 0, 0, 0, 4096, 0, 0, 3, 4029, 4096, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4086, 0, 0, 4080, 4096, 4096, 0, 2385, 0, 4096, 0, 0, 177, 0, 0, 5, 0, 3297, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4078, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[37, 0, 4069, 0, 0, 4093, 4096, 0, 0, 0, 43, 3, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 4091, 0, 4096, 0, 4095, 3963, 0, 0, 0, 2, 231, 315, 0, 0, 2, 4096, 0, 0, 0, 85, 0, 0, 0, 225, 4096, 0, 0, 0, 4096, 0, 0, 1, 4046, 4096, 2, 4075, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 4094, 4096, 4091, 0, 3744, 0, 4093, 0, 0, 270, 0, 0, 13, 0, 3573, 4088, 0, 0, 0, 5, 0, 4096, 0, 0, 3989, 17, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[108, 0, 4090, 0, 0, 3919, 4096, 0, 0, 0, 94, 31, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4096, 4093, 0, 4096, 0, 4096, 3711, 0, 0, 0, 0, 360, 235, 0, 0, 2, 4096, 0, 0, 0, 102, 0, 0, 0, 612, 4096, 0, 0, 0, 4096, 0, 0, 0, 3983, 4096, 9, 4070, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4063, 0, 0, 4060, 4096, 4083, 0, 3737, 0, 4091, 0, 0, 583, 0, 0, 48, 0, 3213, 4093, 0, 0, 0, 53, 0, 4096, 0, 0, 3911, 39, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[22, 0, 4089, 0, 0, 4085, 4089, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 4034, 0, 0, 0, 0, 471, 322, 0, 0, 0, 4096, 0, 0, 0, 34, 0, 0, 0, 207, 4096, 0, 0, 0, 4096, 0, 0, 0, 4027, 4096, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 4096, 4096, 4096, 0, 3810, 0, 4096, 0, 0, 648, 0, 0, 11, 0, 2862, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 4054, 1, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[53, 0, 4089, 0, 0, 4090, 4096, 0, 0, 0, 16, 6, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4096, 4094, 0, 4096, 0, 4096, 3940, 0, 0, 0, 1, 840, 753, 0, 0, 0, 4096, 0, 0, 0, 68, 0, 0, 0, 223, 4095, 0, 0, 0, 4096, 0, 0, 2, 3820, 4096, 2, 4074, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4083, 0, 0, 4090, 4096, 4092, 0, 3637, 0, 4094, 0, 0, 125, 0, 0, 19, 0, 2965, 4090, 0, 0, 0, 13, 0, 4096, 0, 0, 4073, 11, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 4071, 0, 0, 4060, 4096, 0, 0, 0, 61, 17, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 4096, 4089, 0, 4096, 0, 4096, 3755, 0, 0, 0, 3, 2045, 648, 0, 0, 0, 4096, 0, 0, 0, 86, 0, 0, 0, 146, 4096, 0, 0, 0, 4096, 0, 0, 6, 3846, 4096, 4, 4070, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4078, 0, 0, 4069, 4096, 4085, 0, 3421, 0, 4095, 0, 0, 255, 0, 0, 44, 0, 2255, 4091, 0, 0, 0, 9, 0, 4096, 0, 0, 4029, 28, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[9, 0, 4095, 0, 0, 4043, 4096, 0, 0, 0, 41, 5, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4096, 4094, 0, 4096, 0, 4096, 3704, 0, 0, 0, 0, 979, 1207, 0, 0, 1, 4096, 0, 0, 0, 310, 0, 0, 0, 166, 4096, 0, 0, 0, 4096, 0, 0, 4, 4047, 4096, 2, 4061, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 0, 0, 4038, 4096, 4093, 0, 3820, 0, 4095, 0, 0, 203, 0, 0, 80, 0, 2038, 4095, 0, 0, 0, 1, 0, 4096, 0, 0, 4019, 22, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[224, 0, 4087, 0, 0, 4095, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 4096, 0, 4096, 0, 4096, 3976, 0, 0, 0, 0, 347, 509, 0, 0, 0, 4096, 0, 2, 0, 10, 0, 0, 0, 74, 4096, 0, 0, 0, 4096, 0, 0, 0, 4085, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4092, 4096, 4096, 0, 3639, 0, 4096, 0, 0, 477, 0, 0, 0, 0, 3067, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4078, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[37, 0, 4069, 0, 0, 4093, 4096, 0, 0, 0, 43, 3, 0, 0, 0, 0, 3853, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 4091, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2885, 0, 4095, 3963, 0, 0, 0, 2, 231, 315, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 225, 4096, 0, 0, 0, 4096, 0, 0, 1, 4046, 2678, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4075, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 227, 4091, 0, 3744, 0, 4093, 0, 0, 270, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3573, 4088, 0, 0, 0, 5, 0, 1233, 0, 0, 3989, 17, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16383, 15589, 16412, 16381, 16395, 16532, 17001] avg=16384.0000 max=17001 max_rank=7 min=15589 min_rank=2 imbalance_ratio=1.037659 max_load_reduction=114071 max_load_reduction_pct=0.870293 moved_to_echo=66309 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 4089, 0, 0, 4085, 4089, 0, 0, 0, 8, 0, 0, 0, 0, 0, 3853, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2885, 0, 4096, 4034, 0, 0, 0, 0, 471, 322, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 207, 4096, 0, 0, 0, 4096, 0, 0, 0, 4027, 2678, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 227, 4096, 0, 3810, 0, 4096, 0, 0, 648, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2862, 4096, 0, 0, 0, 3, 0, 1233, 0, 0, 4054, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16331, 16400, 15904, 16323, 16392, 16393, 16984, 16345] avg=16384.0000 max=16984 max_rank=6 min=15904 min_rank=2 imbalance_ratio=1.036621 max_load_reduction=114088 max_load_reduction_pct=0.870422 moved_to_echo=66114 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[62, 0, 4084, 0, 0, 4092, 4092, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3853, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2885, 0, 4096, 4043, 0, 0, 0, 0, 736, 1331, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 343, 4096, 0, 0, 0, 4096, 0, 0, 3, 4029, 2678, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4086, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 227, 4096, 0, 2385, 0, 4096, 0, 0, 177, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3297, 4096, 0, 0, 0, 1, 0, 1233, 0, 0, 4078, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16370, 16400, 17187, 16465, 16398, 16385, 15066, 16801] avg=16384.0000 max=17187 max_rank=2 min=15066 min_rank=6 imbalance_ratio=1.049011 max_load_reduction=113885 max_load_reduction_pct=0.868874 moved_to_echo=64650 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[108, 0, 4090, 0, 0, 3919, 4096, 0, 0, 0, 94, 31, 0, 0, 0, 0, 3855, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4096, 4093, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2876, 0, 4096, 3711, 0, 0, 0, 0, 360, 235, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 612, 4096, 0, 0, 0, 4096, 0, 0, 0, 3983, 2681, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 4070, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4063, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 225, 4083, 0, 3737, 0, 4091, 0, 0, 583, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3213, 4093, 0, 0, 0, 53, 0, 1224, 0, 0, 3911, 39, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16359, 15376, 16758, 16382, 16357, 16827, 16629] avg=16384.0000 max=16827 max_rank=6 min=15376 min_rank=2 imbalance_ratio=1.027039 max_load_reduction=114245 max_load_reduction_pct=0.871620 moved_to_echo=66195 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[53, 0, 4089, 0, 0, 4090, 4096, 0, 0, 0, 16, 6, 0, 0, 0, 0, 3853, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4096, 4094, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2885, 0, 4096, 3940, 0, 0, 0, 1, 840, 753, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 223, 4095, 0, 0, 0, 4096, 0, 0, 2, 3820, 2678, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4074, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4083, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 227, 4092, 0, 3637, 0, 4094, 0, 0, 125, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2965, 4090, 0, 0, 0, 13, 0, 1233, 0, 0, 4073, 11, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16380, 16611, 16167, 16380, 16381, 16284, 16481] avg=16384.0000 max=16611 max_rank=2 min=16167 min_rank=3 imbalance_ratio=1.013855 max_load_reduction=114461 max_load_reduction_pct=0.873268 moved_to_echo=65526 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4071, 0, 0, 4060, 4096, 0, 0, 0, 61, 17, 0, 0, 0, 0, 3853, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 4096, 4089, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2885, 0, 4096, 3755, 0, 0, 0, 3, 2045, 648, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 146, 4096, 0, 0, 0, 4096, 0, 0, 6, 3846, 2678, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4070, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4078, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 227, 4085, 0, 3421, 0, 4095, 0, 0, 255, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2255, 4091, 0, 0, 0, 9, 0, 1233, 0, 0, 4029, 28, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16344, 16373, 17528, 16139, 16377, 16374, 16196, 15741] avg=16384.0000 max=17528 max_rank=2 min=15741 min_rank=7 imbalance_ratio=1.069824 max_load_reduction=113544 max_load_reduction_pct=0.866272 moved_to_echo=64688 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[224, 0, 4087, 0, 0, 4095, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 5, 3853, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2885, 0, 4096, 3976, 0, 0, 0, 0, 347, 509, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 10, 0, 0, 0, 74, 4096, 0, 0, 0, 4096, 0, 0, 0, 4085, 2678, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 227, 4096, 0, 3639, 0, 4096, 0, 0, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3067, 4096, 0, 0, 0, 0, 0, 1233, 0, 0, 4078, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16549, 16396, 15909, 16226, 16401, 16394, 16627, 16570] avg=16384.0000 max=16627 max_rank=6 min=15909 min_rank=2 imbalance_ratio=1.014832 max_load_reduction=114445 max_load_reduction_pct=0.873146 moved_to_echo=65992 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 4095, 0, 0, 4043, 4096, 0, 0, 0, 41, 5, 0, 0, 0, 0, 3853, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4096, 4094, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2885, 0, 4096, 3704, 0, 0, 0, 0, 979, 1207, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 0, 0, 166, 4096, 0, 0, 0, 4096, 0, 0, 4, 4047, 2678, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4061, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 227, 4093, 0, 3820, 0, 4095, 0, 0, 203, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2038, 4095, 0, 0, 0, 1, 0, 1233, 0, 0, 4019, 22, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16327, 16381, 16968, 16582, 16361, 16393, 16556, 15504] avg=16384.0000 max=16968 max_rank=2 min=15504 min_rank=7 imbalance_ratio=1.035645 max_load_reduction=114104 max_load_reduction_pct=0.870544 moved_to_echo=64814 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[23296, 32768, 0, 0, 0, 128, 0, 9856, 0, 0, 32256, 128, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[640, 0, 32768, 0, 0, 32512, 32768, 0, 0, 0, 384, 128, 0, 0, 0, 128, 30848, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 32768, 0, 0, 128, 0, 0, 32768, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[32768, 128, 32640, 32768, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 768, 0, 0, 0, 2048, 32768, 0, 0, 0, 32768, 0, 0, 128, 32000, 21504, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 32640, 32768, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 23168, 0, 32768, 31232, 0, 0, 0, 128, 6016, 5376, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 32640, 1920, 32768, 0, 28288, 0, 32768, 0, 0, 2816, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131200 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1724, 0, 2145, 0, 0, 0, 0, 0, 257, 0, 0, 0, 268, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4096, 4096, 4095, 28, 0, 0, 0, 0, 70, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 4094, 0, 4038, 4017, 4088, 0, 0, 0, 81, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 860, 0, 0, 2732, 0, 0, 0, 0, 3, 0, 0, 0, 176, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 844, 0, 3157, 0, 0, 0, 0, 0, 452, 0, 0, 0, 697, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 14, 0, 0, 0, 0, 4096, 4096, 4093, 13, 0, 0, 2, 0, 63, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4039, 3861, 4082, 0, 0, 0, 158, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 815, 0, 0, 1946, 0, 0, 0, 0, 6, 0, 0, 0, 337, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4085, 0, 0, 597, 0, 3830, 0, 0, 0, 8, 0, 870, 0, 0, 0, 3, 9, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 9, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 4096, 4096, 293, 0, 0, 0, 0, 332, 0, 0, 0, 4096, 0, 0, 0, 4078, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4066, 0, 3906, 4072, 4094, 0, 0, 0, 113, 4088, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4096, 280, 0, 0, 2073, 0, 0, 12, 0, 6, 0, 0, 0, 57, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4094, 4092, 0, 0, 2319, 0, 2196, 0, 0, 0, 7, 0, 115, 0, 0, 0, 442, 2, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4094, 4096, 0, 0, 4, 1, 0, 0, 4096, 38, 0, 0, 0, 0, 4092, 4095, 4096, 44, 0, 0, 2, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 1, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 4084, 0, 3976, 3824, 4049, 0, 0, 0, 493, 4093, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 191, 0, 0, 2770, 0, 0, 2, 0, 18, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4096, 4094, 0, 0, 417, 0, 3277, 0, 0, 0, 8, 0, 636, 0, 0, 0, 315, 5, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 5, 0, 0, 0, 4096, 2, 0, 0, 1, 0, 4094, 4096, 4096, 18, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 0, 0, 0, 4079, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 0, 3923, 4002, 4090, 0, 0, 0, 433, 4084, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 950, 0, 0, 2430, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 4094, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1720, 0, 3375, 0, 0, 0, 0, 0, 85, 0, 0, 0, 143, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 4096, 4095, 105, 0, 0, 0, 0, 136, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 3960, 3845, 4055, 0, 0, 0, 314, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 602, 0, 0, 1924, 0, 0, 0, 0, 4, 0, 0, 0, 220, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4095, 4095, 0, 0, 899, 0, 2843, 0, 0, 0, 3, 0, 289, 0, 0, 0, 8, 4, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 2, 1, 0, 0, 4096, 0, 0, 0, 1, 0, 4071, 4096, 4096, 97, 0, 0, 0, 0, 25, 0, 0, 0, 4095, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 4002, 3975, 4095, 0, 0, 0, 687, 4092, 0, 4096, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 762, 0, 0, 2821, 0, 0, 2, 0, 1, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4095, 4093, 0, 0, 1185, 0, 2347, 0, 0, 0, 4, 0, 467, 0, 0, 0, 28, 3, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 749, 0, 0, 0, 0, 240, 0, 0, 0, 4096, 0, 0, 0, 4077, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4068, 0, 3935, 4094, 4095, 0, 0, 0, 180, 4088, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 1232, 0, 0, 1951, 0, 0, 0, 0, 10, 0, 0, 0, 19, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4087, 4096, 4096, 0, 0, 1724, 0, 2145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 268, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3342, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 4095, 28, 0, 0, 0, 0, 70, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 4094, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 4017, 4088, 0, 0, 0, 81, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 860, 0, 0, 2732, 0, 0, 4091, 754, 272, 29, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 176, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16148, 16268, 16378, 16209, 16395, 16291, 16933, 16450] avg=16384.0000 max=16933 max_rank=6 min=16148 min_rank=0 imbalance_ratio=1.033508 max_load_reduction=114139 max_load_reduction_pct=0.870811 moved_to_echo=66069 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4087, 4096, 4096, 0, 0, 1720, 0, 3375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 143, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3342, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 4095, 105, 0, 0, 0, 0, 136, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 3845, 4055, 0, 0, 0, 314, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 602, 0, 0, 1924, 0, 0, 4087, 754, 272, 29, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 220, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17374, 15971, 16384, 16352, 16393, 16241, 15862, 16495] avg=16384.0000 max=17374 max_rank=0 min=15862 min_rank=6 imbalance_ratio=1.060425 max_load_reduction=113698 max_load_reduction_pct=0.867447 moved_to_echo=64991 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4086, 4094, 4092, 0, 0, 2319, 0, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 115, 0, 0, 0, 442, 2, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3341, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4090, 0, 0, 4, 1, 0, 0, 4096, 38, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3823, 4096, 44, 0, 0, 2, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 4084, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 3824, 4049, 0, 0, 0, 493, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 191, 0, 0, 2770, 0, 0, 4096, 755, 272, 29, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 18, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16787, 16308, 16415, 16161, 16383, 16406, 16303, 16309] avg=16384.0000 max=16787 max_rank=0 min=16161 min_rank=3 imbalance_ratio=1.024597 max_load_reduction=114285 max_load_reduction_pct=0.871925 moved_to_echo=65401 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4079, 4096, 4085, 0, 0, 597, 0, 3830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 870, 0, 0, 0, 3, 9, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3342, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 9, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3823, 4096, 293, 0, 0, 0, 0, 332, 0, 0, 0, 4096, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4066, 0, 12, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3906, 4072, 4094, 0, 0, 0, 113, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4096, 280, 0, 0, 2073, 0, 0, 4096, 754, 273, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 6, 0, 0, 0, 57, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16687, 16634, 16384, 16718, 16376, 16239, 15689, 16345] avg=16384.0000 max=16718 max_rank=3 min=15689 min_rank=6 imbalance_ratio=1.020386 max_load_reduction=114354 max_load_reduction_pct=0.872452 moved_to_echo=64649 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4087, 4096, 4096, 0, 0, 844, 0, 3157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 0, 0, 0, 697, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3342, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 4095, 14, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 4093, 13, 0, 0, 2, 0, 63, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 3861, 4082, 0, 0, 0, 158, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 815, 0, 0, 1946, 0, 0, 4096, 754, 272, 29, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 337, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16280, 16892, 16391, 16186, 16395, 16207, 16107, 16614] avg=16384.0000 max=16892 max_rank=1 min=16107 min_rank=6 imbalance_ratio=1.031006 max_load_reduction=114180 max_load_reduction_pct=0.871124 moved_to_echo=65323 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4086, 4096, 4094, 0, 0, 417, 0, 3277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 636, 0, 0, 0, 315, 5, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3342, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 5, 0, 0, 0, 4096, 2, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 4096, 18, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 4002, 4090, 0, 0, 0, 433, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 950, 0, 0, 2430, 0, 0, 4096, 754, 272, 29, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 4096, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15970, 16708, 16384, 16135, 16378, 16503, 16722, 16272] avg=16384.0000 max=16722 max_rank=6 min=15970 min_rank=0 imbalance_ratio=1.020630 max_load_reduction=114350 max_load_reduction_pct=0.872421 moved_to_echo=65875 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4087, 4095, 4095, 0, 0, 899, 0, 2843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 289, 0, 0, 0, 8, 4, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3342, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 2, 1, 0, 0, 4096, 0, 0, 0, 1, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 4096, 97, 0, 0, 0, 0, 25, 0, 0, 0, 4095, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 3975, 4095, 0, 0, 0, 687, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 762, 0, 0, 2821, 0, 0, 4095, 754, 272, 29, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16019, 16047, 16357, 16229, 16384, 16822, 16927, 16287] avg=16384.0000 max=16927 max_rank=6 min=16019 min_rank=0 imbalance_ratio=1.033142 max_load_reduction=114145 max_load_reduction_pct=0.870857 moved_to_echo=66420 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4087, 4095, 4093, 0, 0, 1185, 0, 2347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 467, 0, 0, 0, 28, 3, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3341, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3824, 4096, 749, 0, 0, 0, 0, 240, 0, 0, 0, 4096, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4068, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 4094, 4095, 0, 0, 0, 180, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 1232, 0, 0, 1951, 0, 0, 4096, 755, 272, 29, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 19, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15807, 16244, 16379, 17082, 16368, 16363, 16529, 16300] avg=16384.0000 max=17082 max_rank=3 min=15807 min_rank=0 imbalance_ratio=1.042603 max_load_reduction=113990 max_load_reduction_pct=0.869675 moved_to_echo=65560 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[30592, 32768, 1408, 0, 0, 128, 0, 1024, 0, 0, 0, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[31872, 31744, 32768, 0, 0, 0, 2560, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 32768, 32768, 0, 0, 128, 128, 0, 0, 32768, 128, 0, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 32768, 32768, 32768, 0, 0, 9728, 0, 23296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 0, 0, 32768, 0, 0, 0, 128, 0, 0, 0, 32768, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 32768, 0, 32768, 5760, 0, 0, 18688, 0, 0, 32768, 6144, 2304, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 128, 0, 3200, 0, 0, 0, 1920, 128, 0, 32768, 32768, 0, 0, 0, 0, 32768, 26752, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[128, 0, 128, 0, 0, 0, 896, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 32768, 32768, 31872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2392, 0, 0, 1, 0, 0, 0, 10, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3, 4096, 0, 23, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4087, 2, 0, 6, 4095, 0, 16, 4049, 4096, 0, 0, 0, 0, 0, 4080, 4067, 21, 0, 1798, 4087, 0, 0, 4096, 0, 0, 0, 0, 4094, 4087, 1, 0, 0, 88, 0, 0, 0, 4096, 0, 4096, 4096, 0, 8, 4096, 4096, 4095, 4094, 4096, 0, 11, 0, 2, 0, 3835, 0, 4096, 3953, 0, 0, 0, 4096, 0, 0, 0, 441, 0, 0, 0, 0, 4096, 0, 0, 3897, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2585, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 7, 4096, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 4080, 0, 0, 13, 4094, 0, 33, 4080, 4096, 0, 0, 0, 0, 2, 4088, 4065, 18, 0, 708, 4084, 0, 0, 4096, 0, 0, 0, 0, 4086, 4036, 0, 0, 0, 91, 0, 0, 0, 4096, 0, 4096, 4096, 0, 10, 4096, 4096, 4096, 4088, 4096, 0, 37, 0, 5, 0, 3659, 0, 4096, 3963, 0, 0, 0, 4096, 0, 0, 0, 1683, 0, 0, 0, 0, 4096, 0, 0, 3720, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 4094, 4089, 21, 0, 612, 4094, 0, 0, 4096, 0, 0, 0, 0, 4096, 4033, 0, 0, 0, 426, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 4096, 0, 12, 0, 0, 0, 3608, 0, 4096, 4080, 0, 0, 0, 4096, 0, 0, 0, 1366, 0, 0, 0, 0, 4096, 0, 0, 4021, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 1, 0, 0, 2898, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 4096, 0, 2, 4094, 4096, 0, 0, 0, 0, 0, 4096, 4093, 9, 0, 998, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 198, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 4096, 0, 3, 0, 0, 0, 3765, 0, 4096, 4046, 0, 0, 0, 4096, 0, 0, 0, 423, 0, 0, 0, 0, 4096, 0, 0, 4052, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 1, 0, 0, 3272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4065, 5, 0, 636, 4096, 0, 0, 4096, 0, 1, 0, 0, 4096, 4070, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 3798, 0, 4096, 4039, 0, 0, 0, 4096, 0, 0, 0, 875, 0, 0, 0, 0, 4096, 0, 0, 3813, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 2803, 0, 0, 0, 0, 0, 0, 10, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 4088, 0, 0, 7, 4095, 0, 16, 4086, 4096, 1, 0, 0, 0, 1, 4090, 4088, 57, 0, 776, 4089, 0, 0, 4096, 0, 0, 1, 0, 4091, 4086, 0, 0, 0, 127, 0, 0, 0, 4096, 0, 4096, 4096, 0, 12, 4096, 4096, 4095, 4090, 4096, 0, 16, 0, 4, 0, 2233, 0, 4095, 4059, 0, 0, 0, 4096, 0, 0, 0, 2763, 0, 0, 0, 0, 4096, 0, 0, 3561, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2669, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 12, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4063, 0, 0, 4070, 0, 0, 12, 4096, 0, 60, 4052, 4096, 0, 0, 0, 0, 0, 4094, 4055, 6, 0, 1145, 4079, 0, 0, 4096, 0, 0, 0, 0, 4094, 4081, 0, 0, 0, 288, 0, 0, 0, 4096, 0, 4096, 4092, 0, 20, 4096, 4096, 4096, 4087, 4096, 0, 90, 0, 7, 0, 3986, 0, 4096, 4068, 0, 0, 0, 4096, 0, 0, 0, 301, 0, 0, 0, 0, 4096, 0, 0, 3874, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 802, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 5, 4096, 0, 6, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 4070, 0, 0, 5, 4092, 0, 13, 4077, 4096, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 422, 4084, 0, 0, 4096, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 1531, 0, 0, 0, 4096, 0, 4096, 4096, 0, 10, 4096, 4096, 4096, 4092, 4095, 0, 32, 0, 2, 0, 4034, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 1446, 0, 0, 0, 0, 4096, 0, 0, 4037, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 1, 0, 0, 2898, 0, 0, 0, 0, 0, 0, 3, 4096, 4096, 1617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 1804, 1104, 874, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 3222, 4093, 9, 0, 998, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 198, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 452, 0, 3, 0, 0, 0, 3765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4046, 0, 0, 0, 2992, 0, 0, 0, 423, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2479, 1840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16807, 16377, 16394, 16513, 16257, 16508, 15653, 16563] avg=16384.0000 max=16807 max_rank=0 min=15653 min_rank=6 imbalance_ratio=1.025818 max_load_reduction=114265 max_load_reduction_pct=0.871773 moved_to_echo=64981 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2310, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1804, 1104, 874, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 3220, 4089, 21, 0, 612, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 0, 0, 0, 0, 4096, 4033, 0, 0, 0, 426, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 452, 0, 12, 0, 0, 0, 3608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 2992, 0, 0, 0, 1366, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2479, 1840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16215, 16374, 16397, 16133, 16431, 16360, 16630, 16532] avg=16384.0000 max=16630 max_rank=6 min=16133 min_rank=3 imbalance_ratio=1.015015 max_load_reduction=114442 max_load_reduction_pct=0.873123 moved_to_echo=65953 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2392, 0, 0, 1, 0, 0, 0, 10, 4096, 4096, 1617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3, 4096, 0, 23, 0, 0, 1804, 1104, 871, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4087, 2, 0, 6, 4095, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4049, 0, 0, 0, 0, 0, 0, 3209, 4067, 21, 0, 1798, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 0, 0, 0, 0, 4094, 4087, 1, 0, 0, 88, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 4095, 4094, 452, 0, 11, 0, 2, 0, 3835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3953, 0, 0, 0, 2992, 0, 0, 0, 441, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3897, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 2479, 1840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16308, 16397, 16394, 17247, 16146, 16593, 15578, 16409] avg=16384.0000 max=17247 max_rank=3 min=15578 min_rank=6 imbalance_ratio=1.052673 max_load_reduction=113825 max_load_reduction_pct=0.868416 moved_to_echo=64726 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 2803, 0, 0, 0, 0, 0, 0, 10, 4096, 4096, 1617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 1804, 1104, 873, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 4088, 0, 0, 7, 4095, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4086, 0, 1, 0, 0, 0, 1, 3217, 4088, 57, 0, 776, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 0, 0, 1, 0, 4091, 4086, 0, 0, 0, 127, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 4095, 4090, 452, 0, 16, 0, 4, 0, 2233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4059, 0, 0, 0, 2992, 0, 0, 0, 2763, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3561, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 2479, 1840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16717, 16378, 16389, 16331, 16181, 14998, 18005, 16073] avg=16384.0000 max=18005 max_rank=6 min=14998 min_rank=5 imbalance_ratio=1.098938 max_load_reduction=113067 max_load_reduction_pct=0.862633 moved_to_echo=65257 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2585, 0, 0, 0, 0, 0, 0, 12, 4096, 4096, 1617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 7, 4096, 0, 4, 0, 0, 1804, 1104, 872, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 4080, 0, 0, 13, 4094, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4080, 0, 0, 0, 0, 0, 2, 3216, 4065, 18, 0, 708, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 0, 0, 0, 0, 4086, 4036, 0, 0, 0, 91, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 4096, 4088, 452, 0, 37, 0, 5, 0, 3659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3963, 0, 0, 0, 2992, 0, 0, 0, 1683, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 2479, 1840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16502, 16383, 16387, 16206, 16089, 16443, 16830, 16232] avg=16384.0000 max=16830 max_rank=6 min=16089 min_rank=4 imbalance_ratio=1.027222 max_load_reduction=114242 max_load_reduction_pct=0.871597 moved_to_echo=65594 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 1, 0, 0, 3272, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1804, 1104, 874, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3222, 4065, 5, 0, 636, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 0, 1, 0, 0, 4096, 4070, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 452, 0, 0, 0, 0, 0, 3798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4039, 0, 0, 0, 2992, 0, 0, 0, 875, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3813, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2479, 1840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17178, 16374, 16395, 16120, 16045, 16538, 16098, 16324] avg=16384.0000 max=17178 max_rank=0 min=16045 min_rank=4 imbalance_ratio=1.048462 max_load_reduction=113894 max_load_reduction_pct=0.868942 moved_to_echo=65005 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2669, 0, 0, 0, 0, 0, 0, 38, 4096, 4096, 1617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 12, 4096, 0, 1, 0, 0, 1810, 1105, 877, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4063, 0, 0, 4070, 0, 0, 12, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 4052, 0, 0, 0, 0, 0, 0, 3217, 4055, 6, 0, 1145, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3773, 0, 0, 0, 0, 4094, 4081, 0, 0, 0, 288, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 20, 0, 0, 4096, 4087, 441, 0, 90, 0, 7, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4068, 0, 0, 0, 2991, 0, 0, 0, 301, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3874, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2479, 1845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16612, 16404, 16349, 16614, 16332, 16819, 15552, 16390] avg=16384.0000 max=16819 max_rank=5 min=15552 min_rank=6 imbalance_ratio=1.026550 max_load_reduction=114253 max_load_reduction_pct=0.871681 moved_to_echo=65093 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 802, 0, 0, 0, 0, 0, 0, 26, 4096, 4096, 1617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 5, 4096, 0, 6, 0, 0, 1804, 1104, 874, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 4070, 0, 0, 5, 4092, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4077, 0, 0, 0, 0, 0, 0, 3222, 4090, 0, 0, 422, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3780, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 1531, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 4096, 4092, 451, 0, 32, 0, 2, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2992, 0, 0, 0, 1446, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 2479, 1840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14733, 16385, 16367, 15908, 17591, 16813, 16726, 16549] avg=16384.0000 max=17591 max_rank=4 min=14733 min_rank=0 imbalance_ratio=1.073669 max_load_reduction=113481 max_load_reduction_pct=0.865791 moved_to_echo=67679 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 0, 32768, 0, 128, 0, 0, 19840, 0, 0, 128, 0, 0, 0, 128, 32768, 32768, 13056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[32768, 32384, 0, 0, 0, 23936, 0, 0, 0, 9344, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 30976, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 32768, 19840, 14848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[30336, 0, 128, 128, 0, 32768, 32640, 128, 0, 0, 2816, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 0, 32768, 0, 0, 32768, 128, 0, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 32768, 0, 128, 0, 0, 32768, 32768, 3712, 0, 256, 0, 128, 0, 28928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 32768, 0, 0, 0, 128, 32768, 0, 128, 0, 0, 14464, 8960, 7040, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[256, 32640, 0, 128, 0, 0, 0, 128, 25856, 32640, 256, 0, 7168, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131200 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[2016, 0, 0, 3254, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 138, 0, 0, 0, 0, 4096, 3119, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 3921, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 4003, 0, 0, 4086, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1925, 0, 0, 3134, 0, 0, 3855, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 468, 0, 0, 0, 0, 4096, 3003, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 4096, 0, 4014, 0, 0, 4088, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1791, 0, 0, 3062, 0, 0, 3900, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 176, 0, 0, 0, 0, 4083, 3703, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 4096, 0, 4084, 4096, 0, 0, 0, 3990, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 4089, 0, 3876, 14, 0, 4064, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[2322, 0, 0, 3217, 0, 0, 3382, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 202, 0, 0, 0, 0, 4084, 3545, 0, 0, 0, 4087, 0, 0, 0, 0, 6, 0, 4094, 0, 0, 4096, 0, 0, 4096, 0, 4088, 4096, 0, 0, 0, 3926, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 15, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 4090, 0, 3893, 11, 0, 4070, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[3165, 0, 0, 3029, 0, 0, 3588, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 115, 0, 0, 0, 0, 4084, 2704, 0, 0, 0, 4092, 0, 0, 0, 0, 3, 0, 4094, 0, 0, 4095, 0, 0, 4096, 0, 4091, 4096, 0, 0, 0, 3921, 1, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 1, 0, 0, 4094, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 2, 0, 0, 0, 0, 2, 1, 0, 4096, 0, 0, 0, 0, 8, 0, 4095, 0, 0, 4094, 0, 3981, 5, 0, 4075, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[2692, 0, 0, 2754, 0, 0, 3524, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 180, 0, 0, 0, 0, 4088, 3270, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 4093, 0, 0, 4096, 0, 0, 4096, 0, 4093, 4096, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4094, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 4095, 0, 0, 1, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 4096, 0, 4036, 3, 0, 4092, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1003, 0, 0, 4026, 0, 0, 4024, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 177, 0, 0, 0, 0, 4096, 3280, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3986, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[2203, 0, 0, 3524, 0, 0, 3709, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 416, 0, 0, 0, 0, 4082, 2755, 0, 0, 0, 4090, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 4096, 0, 4091, 4096, 0, 0, 0, 3931, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 2, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 19, 0, 4096, 0, 0, 4094, 0, 3949, 13, 0, 4070, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2016, 0, 0, 3254, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3140, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 4096, 3119, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 688, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 130, 0, 0, 0, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3408, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3279, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4003, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 143, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16548, 16244, 16339, 16383, 16380, 16380, 16374, 16424] avg=16384.0000 max=16548 max_rank=0 min=16244 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=114524 max_load_reduction_pct=0.873749 moved_to_echo=65558 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1925, 0, 0, 3134, 0, 0, 3855, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3140, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 4096, 3003, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 687, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 130, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3409, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3279, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4014, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 143, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16154, 16457, 16499, 16384, 16380, 16380, 16381, 16437] avg=16384.0000 max=16499 max_rank=2 min=16154 min_rank=0 imbalance_ratio=1.007019 max_load_reduction=114573 max_load_reduction_pct=0.874123 moved_to_echo=65578 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3165, 0, 0, 3029, 0, 0, 3588, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 3140, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 0, 0, 4084, 2704, 0, 0, 0, 4092, 0, 0, 0, 0, 3, 4096, 687, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4095, 0, 0, 0, 0, 4091, 130, 0, 0, 0, 3921, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 1, 0, 0, 4094, 4096, 0, 3409, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 3279, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 4096, 0, 0, 0, 0, 8, 0, 4095, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3981, 5, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 143, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17023, 15792, 16332, 16385, 16379, 16386, 16383, 16392] avg=16384.0000 max=17023 max_rank=0 min=15792 min_rank=1 imbalance_ratio=1.039001 max_load_reduction=114049 max_load_reduction_pct=0.870125 moved_to_echo=65540 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2692, 0, 0, 2754, 0, 0, 3524, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3140, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 4088, 3270, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 4096, 688, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 4093, 130, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4094, 4096, 0, 3408, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 3279, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 2, 0, 4095, 0, 0, 1, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4036, 3, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 143, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16210, 16429, 16438, 16383, 16382, 16383, 16382, 16465] avg=16384.0000 max=16465 max_rank=7 min=16210 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=114607 max_load_reduction_pct=0.874382 moved_to_echo=65612 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2322, 0, 0, 3217, 0, 0, 3382, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3140, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 0, 0, 4084, 3545, 0, 0, 0, 4087, 0, 0, 0, 0, 6, 4096, 688, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 4088, 130, 0, 0, 0, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3408, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3279, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 4, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3893, 11, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 143, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16161, 16720, 16334, 16385, 16390, 16387, 16395, 16300] avg=16384.0000 max=16720 max_rank=1 min=16161 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=114352 max_load_reduction_pct=0.872437 moved_to_echo=65472 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1791, 0, 0, 3062, 0, 0, 3900, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3145, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, 0, 0, 4083, 3703, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 688, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 4084, 126, 0, 0, 0, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3408, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 3282, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 5, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 3876, 14, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 134, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16003, 16849, 16393, 16384, 16398, 16388, 16384, 16273] avg=16384.0000 max=16849 max_rank=1 min=16003 min_rank=0 imbalance_ratio=1.028381 max_load_reduction=114223 max_load_reduction_pct=0.871452 moved_to_echo=65443 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1003, 0, 0, 4026, 0, 0, 4024, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 3140, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 4096, 3280, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 687, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 4096, 130, 0, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3409, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3279, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3986, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 143, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16297, 16443, 16395, 16384, 16380, 16380, 16376, 16417] avg=16384.0000 max=16443 max_rank=1 min=16297 min_rank=0 imbalance_ratio=1.003601 max_load_reduction=114629 max_load_reduction_pct=0.874550 moved_to_echo=65553 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2203, 0, 0, 3524, 0, 0, 3709, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3140, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 416, 0, 0, 0, 0, 4082, 2755, 0, 0, 0, 4090, 0, 1, 0, 0, 0, 4096, 687, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 4091, 130, 0, 0, 0, 3931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 3409, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3279, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 19, 0, 4096, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3949, 13, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 143, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16676, 16138, 16342, 16384, 16383, 16388, 16397, 16364] avg=16384.0000 max=16676 max_rank=0 min=16138 min_rank=1 imbalance_ratio=1.017822 max_load_reduction=114396 max_load_reduction_pct=0.872772 moved_to_echo=65532 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[17152, 0, 0, 26112, 0, 0, 30080, 128, 0, 0, 128, 0, 0, 0, 32768, 0, 25216, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 32768, 0, 32768, 32768, 0, 0, 0, 0, 26240, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[32768, 0, 31744, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 1152, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 128, 32768, 0, 0, 0, 0, 32768, 1152, 0, 0, 0, 31872, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 128, 0, 32768, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[1920, 0, 0, 0, 0, 32768, 25472, 0, 0, 0, 32768, 0, 128, 0, 0, 128, 32768, 5504, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 32768, 128, 0, 32768, 0, 0, 128, 0, 0, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 128, 32768, 0, 0, 0, 128, 0, 0, 32768, 32768, 0, 27392, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131200 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3779, 4096, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 775, 0, 0, 0, 0, 0, 3785, 0, 0, 4094, 0, 62, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 12, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3890, 0, 0, 2, 4096, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3806, 4096, 0, 0, 0, 0, 0, 4096, 0, 31, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4079, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 908, 0, 0, 2, 0, 0, 3335, 0, 0, 3964, 0, 320, 1, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 92, 4096, 251, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3775, 0, 0, 9, 4096, 2, 4096, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4063, 0, 4096, 0, 0, 0, 0, 23, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4084, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3767, 4096, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 18, 0, 4096, 0, 0, 0, 0, 1302, 0, 0, 2, 0, 0, 3442, 0, 0, 3710, 0, 56, 3, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 56, 4094, 5, 0, 4094, 4087, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4029, 0, 0, 11, 4096, 6, 4095, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4084, 1, 4095, 1, 0, 0, 0, 6, 0, 4096, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 4088, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3719, 4096, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 1358, 0, 0, 0, 0, 0, 3613, 0, 0, 3645, 0, 9, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 25, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4002, 0, 0, 1, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3669, 4096, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4074, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 1100, 0, 0, 7, 0, 0, 3679, 0, 0, 3825, 1, 91, 6, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 89, 4094, 34, 0, 4094, 4084, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 3895, 0, 0, 26, 4096, 0, 4096, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4083, 0, 4096, 0, 0, 0, 0, 11, 0, 4096, 4095, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 4096, 4079, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3886, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 892, 0, 0, 0, 0, 0, 3495, 0, 0, 4034, 0, 132, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4096, 40, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3891, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4086, 0, 4095, 0, 0, 0, 0, 5, 0, 4094, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4051, 4096, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4075, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 1075, 0, 0, 1, 0, 0, 2870, 0, 0, 4038, 0, 477, 1, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 53, 4095, 60, 0, 4095, 4057, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3790, 0, 0, 18, 4096, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4083, 0, 4096, 0, 0, 0, 0, 10, 0, 4095, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 4095, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3594, 4096, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 774, 0, 0, 1, 0, 0, 3839, 0, 0, 4084, 0, 111, 2, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4095, 59, 0, 4095, 4083, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3887, 0, 0, 0, 4096, 3, 4096, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4093, 0, 4096, 0, 0, 0, 0, 12, 0, 4096, 4096, 0, 4096, 0, 1, 1, 4096, 0, 0, 0, 4096, 4094, 4096, 1, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3779, 4096, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 4096, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 775, 0, 0, 0, 0, 0, 3785, 0, 0, 4094, 0, 62, 0, 4079, 0, 3691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 12, 0, 4096, 4096, 0, 0, 0, 3796, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3890, 0, 0, 2, 4096, 0, 4096, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3798, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 6, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 16385, 16486, 16304, 16371, 16382, 16382, 16390] avg=16384.0000 max=16486 max_rank=2 min=16304 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=114586 max_load_reduction_pct=0.874222 moved_to_echo=65525 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3767, 4096, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 4096, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 18, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1302, 0, 0, 2, 0, 0, 3442, 0, 0, 3710, 0, 56, 3, 4080, 0, 3691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 4094, 5, 0, 4094, 4087, 1, 0, 0, 3796, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4029, 0, 0, 11, 4096, 6, 4095, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3799, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1, 4095, 1, 0, 0, 0, 6, 0, 4096, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4088, 6, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16396, 16286, 16340, 16524, 16384, 16384, 16383] avg=16384.0000 max=16524 max_rank=4 min=16286 min_rank=2 imbalance_ratio=1.008545 max_load_reduction=114548 max_load_reduction_pct=0.873932 moved_to_echo=65675 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3806, 4096, 0, 0, 0, 0, 0, 4096, 0, 31, 0, 0, 4096, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4079, 0, 0, 11, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 908, 0, 0, 2, 0, 0, 3335, 0, 0, 3964, 0, 320, 1, 4021, 0, 3694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 4096, 251, 0, 4096, 4079, 0, 0, 0, 3799, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3775, 0, 0, 9, 4096, 2, 4096, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3798, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 4096, 0, 0, 0, 0, 23, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16423, 16382, 16245, 16620, 16265, 16391, 16374, 16372] avg=16384.0000 max=16620 max_rank=3 min=16245 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=114452 max_load_reduction_pct=0.873199 moved_to_echo=65402 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3719, 4096, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 4096, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1358, 0, 0, 0, 0, 0, 3613, 0, 0, 3645, 0, 9, 0, 4092, 0, 3691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 25, 0, 4096, 4094, 0, 0, 0, 3796, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4002, 0, 0, 1, 4096, 0, 4096, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3799, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 6, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16314, 16390, 16408, 16320, 16482, 16382, 16387, 16389] avg=16384.0000 max=16482 max_rank=4 min=16314 min_rank=0 imbalance_ratio=1.005981 max_load_reduction=114590 max_load_reduction_pct=0.874252 moved_to_echo=65640 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3886, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 892, 0, 0, 0, 0, 0, 3495, 0, 0, 4034, 0, 132, 0, 4094, 0, 3691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4096, 40, 0, 4096, 4094, 0, 0, 0, 3796, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3891, 0, 0, 0, 4096, 0, 4096, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3799, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4095, 0, 0, 0, 0, 5, 0, 4094, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 6, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16471, 16386, 16338, 16353, 16370, 16383, 16381, 16390] avg=16384.0000 max=16471 max_rank=0 min=16338 min_rank=2 imbalance_ratio=1.005310 max_load_reduction=114601 max_load_reduction_pct=0.874336 moved_to_echo=65524 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3594, 4096, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 4096, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 7, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 774, 0, 0, 1, 0, 0, 3839, 0, 0, 4084, 0, 111, 2, 4086, 0, 3691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4095, 59, 0, 4095, 4083, 0, 0, 0, 3796, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3887, 0, 0, 0, 4096, 3, 4096, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3798, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 12, 0, 4096, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4096, 0, 0, 0, 0, 4094, 4, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16186, 16389, 16588, 16370, 16370, 16382, 16398, 16389] avg=16384.0000 max=16588 max_rank=2 min=16186 min_rank=0 imbalance_ratio=1.012451 max_load_reduction=114484 max_load_reduction_pct=0.873444 moved_to_echo=65539 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3669, 4096, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 4096, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4074, 0, 0, 8, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1100, 0, 0, 7, 0, 0, 3679, 0, 0, 3825, 1, 91, 6, 4084, 0, 3694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 4094, 34, 0, 4094, 4084, 1, 0, 0, 3796, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 3895, 0, 0, 26, 4096, 0, 4096, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3798, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 0, 0, 0, 0, 11, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4079, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16275, 16371, 16487, 16402, 16401, 16386, 16381, 16369] avg=16384.0000 max=16487 max_rank=2 min=16275 min_rank=0 imbalance_ratio=1.006287 max_load_reduction=114585 max_load_reduction_pct=0.874214 moved_to_echo=65537 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4051, 4096, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 4096, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4075, 0, 0, 9, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 0, 0, 1, 0, 0, 2870, 0, 0, 4038, 0, 477, 1, 4081, 0, 3691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 4095, 60, 0, 4095, 4057, 0, 0, 0, 3796, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3790, 0, 0, 18, 4096, 2, 4096, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3799, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 0, 0, 0, 0, 10, 0, 4095, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4095, 6, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16656, 16373, 16234, 16363, 16289, 16382, 16385, 16390] avg=16384.0000 max=16656 max_rank=0 min=16234 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=114416 max_load_reduction_pct=0.872925 moved_to_echo=65446 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 32768, 128, 32768, 128, 0, 0, 0, 128, 0, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 0, 32768, 128, 128, 32768, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 30336, 32768, 0, 0, 0, 0, 0, 32768, 0, 128, 0, 0, 32768, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 0, 128, 0, 0, 0, 0, 32768, 0, 0, 32768, 0, 0, 0, 30464, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[32768, 32768, 0, 0, 0, 0, 0, 32768, 0, 0, 128, 0, 32768, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 128, 0, 0, 31232, 0, 0, 128, 32768, 128, 32768, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 384, 32768, 512, 0, 32768, 32768, 128, 0, 0, 30464, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 8192, 0, 0, 128, 0, 0, 28160, 0, 0, 31488, 128, 1280, 128, 32640, 0, 29568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131200 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4094, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 0, 3955, 0, 4096, 4096, 0, 0, 0, 4088, 0, 4042, 5, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 3915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 376, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 1, 4096, 4096, 4096, 0, 4086, 0, 0, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3954, 0, 0, 4096, 0, 0, 4069, 5, 0, 0, 0, 40, 0, 0, 0, 4096, 0, 2, 0, 3946, 0, 4096, 4095, 0, 0, 0, 4069, 0, 4056, 6, 0, 0, 4096, 0, 0, 4076, 4096, 4096, 0, 0, 0, 0, 0, 0, 3272, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1214, 4089, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 19, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4084, 0, 4096, 4096, 4096, 0, 3976, 0, 0, 4095, 16, 22, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4061, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 4, 1, 0, 3887, 0, 4096, 4093, 0, 0, 0, 4064, 0, 4068, 23, 0, 0, 4096, 0, 0, 4076, 4096, 4096, 0, 0, 0, 0, 0, 0, 2659, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1612, 4085, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 45, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 20, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4082, 0, 4096, 4096, 4096, 0, 4054, 0, 0, 4096, 13, 47, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[3996, 0, 0, 4095, 0, 0, 4081, 5, 0, 0, 0, 67, 0, 0, 0, 4096, 1, 5, 0, 4059, 0, 4096, 4093, 0, 0, 0, 4065, 0, 3924, 8, 0, 0, 4096, 0, 0, 4082, 4095, 4096, 0, 0, 0, 0, 0, 0, 3925, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 831, 4091, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 4, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 4090, 0, 4096, 4095, 4096, 0, 3676, 0, 0, 4094, 17, 12, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[3937, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 3890, 0, 4096, 4096, 0, 0, 0, 4089, 0, 4041, 7, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 2707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1918, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 1, 4096, 4096, 4096, 0, 3979, 0, 0, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4094, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 46, 0, 0, 0, 4096, 0, 0, 0, 3966, 0, 4096, 4096, 0, 0, 0, 4093, 0, 4050, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 3059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1256, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 1, 4096, 4096, 4096, 0, 4006, 0, 0, 4096, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 0, 4096, 0, 0, 4086, 14, 0, 0, 0, 10, 0, 0, 0, 4095, 0, 0, 0, 4056, 0, 4096, 4095, 0, 0, 0, 4092, 0, 4057, 2, 0, 0, 4096, 0, 0, 4082, 4095, 4096, 0, 0, 0, 0, 0, 0, 3210, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 946, 4093, 1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 9, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4090, 1, 4096, 4096, 4096, 0, 4079, 0, 0, 4095, 3, 15, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4071, 32, 0, 0, 0, 48, 0, 0, 0, 4096, 0, 0, 0, 3985, 0, 4096, 4094, 0, 0, 0, 4062, 0, 4074, 17, 0, 0, 4096, 0, 0, 4089, 4096, 4096, 0, 0, 0, 0, 0, 0, 3673, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 561, 4092, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 3, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4087, 2, 4096, 4096, 4096, 0, 3996, 0, 0, 4096, 0, 23, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 13, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 195, 4096, 0, 0, 0, 4088, 0, 4042, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 804, 0, 0, 0, 0, 0, 0, 3915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3288, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 376, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3997, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 99, 4096, 0, 0, 4086, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16420, 16381, 17007, 16376, 15666, 16371, 16376, 16475] avg=16384.0000 max=17007 max_rank=2 min=15666 min_rank=4 imbalance_ratio=1.038025 max_load_reduction=114065 max_load_reduction_pct=0.870247 moved_to_echo=64888 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3996, 0, 0, 4095, 0, 0, 4081, 5, 0, 0, 0, 67, 0, 0, 0, 4096, 13, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 4059, 0, 195, 4093, 0, 0, 0, 4065, 0, 3924, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4082, 4095, 804, 0, 0, 0, 0, 0, 0, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 4096, 3288, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 831, 4091, 0, 0, 0, 0, 0, 2, 0, 1, 0, 4096, 3997, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 16, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4, 0, 4096, 2, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 99, 4095, 0, 0, 3676, 0, 0, 4094, 17, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16364, 16350, 17002, 16385, 16119, 16385, 16384, 16083] avg=16384.0000 max=17002 max_rank=2 min=16083 min_rank=7 imbalance_ratio=1.037720 max_load_reduction=114070 max_load_reduction_pct=0.870285 moved_to_echo=64971 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 46, 0, 0, 0, 4096, 13, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 195, 4096, 0, 0, 0, 4093, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 804, 0, 0, 0, 0, 0, 0, 3059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3288, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1256, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3997, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 99, 4096, 0, 0, 4006, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16455, 16400, 16151, 16376, 16546, 16371, 16377, 16396] avg=16384.0000 max=16546 max_rank=4 min=16151 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=114526 max_load_reduction_pct=0.873764 moved_to_echo=65690 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3954, 0, 0, 4096, 0, 0, 4069, 5, 0, 0, 0, 40, 0, 0, 0, 4096, 13, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3946, 0, 195, 4095, 0, 0, 0, 4069, 0, 4056, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 4096, 804, 0, 0, 0, 0, 0, 0, 3272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 0, 5, 4096, 0, 0, 0, 4096, 3288, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1214, 4089, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 3997, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 27, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 0, 4096, 1, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 99, 4096, 0, 0, 3976, 0, 0, 4095, 16, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16284, 16369, 16344, 16391, 16506, 16393, 16397, 16388] avg=16384.0000 max=16506 max_rank=4 min=16284 min_rank=0 imbalance_ratio=1.007446 max_load_reduction=114566 max_load_reduction_pct=0.874069 moved_to_echo=65684 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4061, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 18, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 3887, 0, 190, 4093, 0, 0, 0, 4064, 0, 4068, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 4096, 797, 0, 0, 0, 0, 0, 0, 2659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 4096, 3294, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1612, 4085, 0, 0, 0, 0, 0, 29, 0, 0, 0, 4096, 3999, 3106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 45, 0, 0, 0, 4078, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 0, 4096, 0, 0, 0, 4084, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 97, 4096, 0, 0, 4054, 0, 0, 4096, 13, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 16330, 15724, 16397, 16927, 16406, 16392, 16485] avg=16384.0000 max=16927 max_rank=4 min=15724 min_rank=2 imbalance_ratio=1.033142 max_load_reduction=114145 max_load_reduction_pct=0.870857 moved_to_echo=66210 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3937, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 13, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3890, 0, 195, 4096, 0, 0, 0, 4089, 0, 4041, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 804, 0, 0, 0, 0, 0, 0, 2707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3288, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1918, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3997, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 99, 4096, 0, 0, 3979, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16255, 16318, 15799, 16376, 17208, 16371, 16377, 16368] avg=16384.0000 max=17208 max_rank=4 min=15799 min_rank=2 imbalance_ratio=1.050293 max_load_reduction=113864 max_load_reduction_pct=0.868713 moved_to_echo=66324 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 4096, 0, 0, 4086, 14, 0, 0, 0, 10, 0, 0, 0, 4095, 13, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 195, 4095, 0, 0, 0, 4092, 0, 4057, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4082, 4095, 804, 0, 0, 0, 0, 0, 0, 3210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 4096, 3288, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 946, 4093, 1, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 3997, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 9, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 9, 0, 4096, 1, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 99, 4096, 0, 0, 4079, 0, 0, 4095, 3, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16420, 16497, 16287, 16384, 16240, 16378, 16388, 16478] avg=16384.0000 max=16497 max_rank=1 min=16240 min_rank=4 imbalance_ratio=1.006897 max_load_reduction=114575 max_load_reduction_pct=0.874138 moved_to_echo=65484 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4071, 32, 0, 0, 0, 48, 0, 0, 0, 4096, 13, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 195, 4094, 0, 0, 0, 4062, 0, 4074, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 4096, 804, 0, 0, 0, 0, 0, 0, 3673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 4096, 3288, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 4092, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 3997, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 27, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 3, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 2, 99, 4096, 0, 0, 3996, 0, 0, 4096, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16463, 16427, 16758, 16387, 15860, 16397, 16381, 16399] avg=16384.0000 max=16758 max_rank=2 min=15860 min_rank=4 imbalance_ratio=1.022827 max_load_reduction=114314 max_load_reduction_pct=0.872147 moved_to_echo=65037 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 32768, 0, 0, 0, 0, 128, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 0, 128, 0, 32768, 128, 0, 0, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32384, 0, 0, 32768, 0, 0, 32768, 128, 0, 0, 0, 384, 0, 0, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 128, 0, 31744, 0, 1664, 32768, 0, 0, 0, 32640, 0, 32384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 32768, 32768, 6528, 0, 0, 0, 0, 0, 0, 26496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[32768, 128, 896, 32768, 0, 0, 31872, 0, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 32768, 0, 128, 32768, 0, 0, 0, 32768, 26368, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 8832, 32768, 128, 0, 0, 0, 0, 128, 0, 128, 0, 32768, 32000, 24832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131200 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3186, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 3933, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 53, 138, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4032, 0, 0, 0, 0, 4096, 356, 4096, 200, 4092, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 413, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3707, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 3858, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4090, 21, 4042, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 172, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 4088, 4096, 0, 55, 96, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 4009, 0, 0, 0, 30, 4095, 47, 4096, 261, 4030, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 166, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2928, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 3600, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4019, 0, 4089, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 106, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 9, 53, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3942, 0, 0, 0, 0, 4096, 1161, 4096, 47, 4091, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 534, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2586, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 3655, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 4095, 22, 3991, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1090, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 4087, 4096, 0, 44, 143, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3790, 0, 0, 0, 5, 4092, 513, 4096, 38, 4071, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 529, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4072, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 6, 0, 4094, 0, 0, 0, 0, 3816, 9, 4071, 4096, 0, 3, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 208, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 4091, 4096, 0, 55, 203, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 4072, 0, 0, 0, 6, 4094, 438, 4096, 156, 4081, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 292, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2635, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 3491, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4000, 0, 4086, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 453, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 68, 135, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4061, 0, 0, 0, 0, 4096, 1161, 4096, 106, 4089, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 298, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3511, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 3880, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 4095, 14, 4072, 4096, 0, 1, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 384, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4093, 4096, 0, 5, 143, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 3936, 0, 0, 0, 3, 4094, 163, 4096, 25, 4088, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 260, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3306, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 3828, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 4082, 20, 4053, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 780, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 4096, 4096, 0, 9, 139, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3640, 0, 0, 0, 0, 4095, 22, 4096, 15, 4081, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 590, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2928, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 3600, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4019, 0, 4089, 3364, 412, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3697, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 106, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 9, 53, 0, 0, 0, 0, 4096, 3119, 807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3942, 0, 0, 0, 0, 3684, 1161, 0, 47, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 534, 0, 0, 0, 4096, 3289, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16193, 16197, 16379, 16091, 16276, 17021, 16531, 16384] avg=16384.0000 max=17021 max_rank=5 min=16091 min_rank=3 imbalance_ratio=1.038879 max_load_reduction=114051 max_load_reduction_pct=0.870140 moved_to_echo=66212 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3186, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 3933, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 4096, 3364, 412, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3697, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 53, 138, 0, 0, 0, 0, 4096, 3119, 807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4032, 0, 0, 0, 0, 3684, 356, 0, 200, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 413, 0, 0, 0, 4096, 3289, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16451, 16524, 16455, 15988, 16404, 16460, 16406, 16384] avg=16384.0000 max=16524 max_rank=1 min=15988 min_rank=3 imbalance_ratio=1.008545 max_load_reduction=114548 max_load_reduction_pct=0.873932 moved_to_echo=65654 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 4072, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 4094, 0, 0, 0, 0, 3816, 9, 4071, 3364, 412, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3697, 0, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 208, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4091, 4096, 0, 55, 203, 0, 0, 0, 0, 4096, 3120, 807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4072, 0, 0, 0, 6, 3682, 438, 0, 156, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 292, 0, 0, 0, 4096, 3289, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16360, 16669, 16172, 16195, 16471, 16527, 16293, 16385] avg=16384.0000 max=16669 max_rank=1 min=16172 min_rank=2 imbalance_ratio=1.017395 max_load_reduction=114403 max_load_reduction_pct=0.872826 moved_to_echo=65676 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2635, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 3491, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4000, 0, 4086, 3364, 412, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3697, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 453, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 68, 135, 0, 0, 0, 0, 4096, 3120, 807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4061, 0, 0, 0, 0, 3684, 1161, 0, 106, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 298, 0, 0, 0, 4096, 3289, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15899, 16086, 16357, 16438, 16417, 17197, 16294, 16384] avg=16384.0000 max=17197 max_rank=5 min=15899 min_rank=0 imbalance_ratio=1.049622 max_load_reduction=113875 max_load_reduction_pct=0.868797 moved_to_echo=66292 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3707, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 3858, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4090, 21, 4042, 3364, 412, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3697, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 172, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4088, 4096, 0, 55, 96, 0, 0, 0, 0, 4096, 3119, 807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4009, 0, 0, 0, 30, 3683, 47, 0, 261, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 166, 0, 0, 0, 4096, 3289, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16972, 16450, 16429, 16157, 16365, 16151, 16166, 16382] avg=16384.0000 max=16972 max_rank=0 min=16151 min_rank=5 imbalance_ratio=1.035889 max_load_reduction=114100 max_load_reduction_pct=0.870514 moved_to_echo=65064 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2586, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 301, 0, 3655, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 4095, 22, 3991, 3369, 418, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3691, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4087, 4096, 0, 44, 143, 0, 0, 0, 0, 4096, 3119, 807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3790, 0, 0, 0, 5, 3674, 513, 0, 38, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 529, 0, 0, 0, 4096, 3289, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15851, 16244, 16402, 17069, 16404, 16187, 16532, 16383] avg=16384.0000 max=17069 max_rank=3 min=15851 min_rank=0 imbalance_ratio=1.041809 max_load_reduction=114003 max_load_reduction_pct=0.869774 moved_to_echo=65506 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3511, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 3880, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 4095, 14, 4072, 3364, 412, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3697, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 384, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4093, 4096, 0, 5, 143, 0, 0, 0, 0, 4096, 3119, 807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3936, 0, 0, 0, 3, 3682, 163, 0, 25, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 260, 0, 0, 0, 4096, 3289, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16776, 16477, 16452, 16369, 16361, 15992, 16261, 16384] avg=16384.0000 max=16776 max_rank=0 min=15992 min_rank=5 imbalance_ratio=1.023926 max_load_reduction=114296 max_load_reduction_pct=0.872009 moved_to_echo=64998 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3306, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 3828, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 4082, 20, 4053, 3364, 412, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3697, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 780, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 4096, 0, 9, 139, 0, 0, 0, 0, 4096, 3120, 807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3640, 0, 0, 0, 0, 3683, 22, 0, 15, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 590, 0, 0, 0, 4095, 3289, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16570, 16425, 16426, 16765, 16374, 15537, 16589, 16386] avg=16384.0000 max=16765 max_rank=3 min=15537 min_rank=5 imbalance_ratio=1.023254 max_load_reduction=114307 max_load_reduction_pct=0.872093 moved_to_echo=64886 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[32768, 0, 32768, 0, 0, 0, 0, 0, 0, 128, 0, 32768, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 24960, 0, 0, 0, 0, 0, 0, 0, 32768, 0, 0, 0, 32768, 32768, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 0, 0, 0, 0, 0, 2560, 0, 30336, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 32768, 32768, 0, 384, 1152, 0, 0, 0, 0, 32768, 24960, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[29696, 0, 128, 0, 0, 0, 0, 0, 0, 32768, 0, 0, 32768, 3200, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 31488, 0, 0, 0, 128, 29568, 3968, 0, 896, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 32384, 128, 32512, 27008, 3328, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 32768, 0, 0, 0, 0, 32768, 0, 3200, 0, 0, 0, 32768, 26368, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131200 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131200 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 9, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4070, 0, 64, 0, 0, 0, 0, 4096, 2900, 0, 0, 820, 0, 4096, 1, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 3504, 0, 0, 0, 4, 4069, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 3262, 18, 0, 0, 0, 0, 0, 4096, 4095, 2, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3146, 0, 0, 5, 0, 4092, 4096, 0, 0, 1300, 0, 0, 0, 0, 4091, 0, 1, 0, 0, 0, 0, 1414, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4089, 0, 44, 0, 0, 0, 0, 4096, 1807, 0, 0, 1175, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3992, 0, 0, 0, 0, 4080, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4084, 0, 0, 0, 4096, 3625, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 2151, 0, 0, 3, 0, 4096, 4096, 0, 0, 1111, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2515, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4085, 0, 73, 0, 0, 0, 0, 4096, 3665, 0, 0, 1438, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3604, 0, 0, 0, 0, 3988, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3716, 1, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 1563, 0, 0, 0, 0, 4096, 4096, 0, 0, 470, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1974, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4085, 0, 25, 0, 0, 0, 0, 4096, 1102, 0, 0, 735, 0, 4096, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 0, 3529, 0, 0, 0, 4, 4025, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3745, 57, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 2779, 0, 0, 17, 0, 4080, 4096, 0, 0, 2951, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 1567, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 0, 17, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4056, 0, 263, 0, 0, 0, 0, 4096, 1969, 0, 0, 1059, 0, 4096, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 3527, 0, 0, 0, 3, 4017, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 2828, 36, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 2658, 0, 0, 10, 0, 4087, 4096, 0, 0, 2011, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 2146, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 3, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4091, 0, 29, 0, 0, 0, 0, 4096, 2160, 0, 0, 1870, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 3322, 0, 0, 0, 0, 3640, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 3161, 19, 0, 0, 0, 0, 0, 4096, 4096, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 2562, 0, 0, 8, 0, 4080, 4096, 0, 0, 2177, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 1566, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 0, 0, 7, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4084, 0, 104, 0, 0, 0, 0, 4096, 3017, 0, 0, 690, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3365, 0, 0, 0, 1, 4040, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 3035, 13, 0, 0, 0, 0, 0, 4096, 4096, 1, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 2528, 0, 0, 5, 0, 4090, 4096, 0, 0, 1840, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1861, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 52, 0, 0, 0, 0, 4096, 3166, 0, 0, 1562, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3576, 0, 0, 0, 0, 4054, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2945, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 1787, 0, 0, 0, 0, 4096, 4096, 0, 0, 463, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2875, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 9, 0, 0, 0, 0, 4096, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 64, 0, 0, 0, 0, 4096, 2900, 0, 0, 820, 0, 4096, 1, 0, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 3504, 0, 0, 0, 4, 4069, 0, 2752, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 3262, 18, 0, 0, 0, 0, 0, 790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3146, 0, 0, 5, 0, 4092, 4096, 0, 0, 1300, 0, 0, 0, 0, 4091, 0, 164, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1414, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16433, 16419, 16385, 16357, 16385, 16898, 15809] avg=16384.0000 max=16898 max_rank=6 min=15809 min_rank=7 imbalance_ratio=1.031372 max_load_reduction=114174 max_load_reduction_pct=0.871078 moved_to_echo=65449 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 73, 0, 0, 0, 0, 4096, 3665, 0, 0, 1438, 0, 4095, 0, 0, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3604, 0, 0, 0, 0, 3988, 0, 2752, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3716, 1, 0, 0, 0, 0, 0, 790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1563, 0, 0, 0, 0, 4096, 4096, 0, 0, 470, 0, 0, 0, 0, 4096, 0, 168, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1974, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 17838, 16430, 16384, 16795, 16383, 14493, 16368] avg=16384.0000 max=17838 max_rank=1 min=14493 min_rank=6 imbalance_ratio=1.088745 max_load_reduction=113234 max_load_reduction_pct=0.863907 moved_to_echo=64039 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 44, 0, 0, 0, 0, 4096, 1807, 0, 0, 1175, 0, 4092, 0, 0, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3992, 0, 0, 0, 0, 4080, 0, 2752, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 0, 0, 0, 4096, 3625, 0, 0, 0, 0, 0, 0, 790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2151, 0, 0, 3, 0, 4096, 4096, 0, 0, 1111, 0, 0, 0, 0, 4096, 0, 165, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2515, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 15689, 16910, 16387, 16691, 16383, 15722, 16909] avg=16384.0000 max=16910 max_rank=2 min=15689 min_rank=1 imbalance_ratio=1.032104 max_load_reduction=114162 max_load_reduction_pct=0.870987 moved_to_echo=65705 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 52, 0, 0, 0, 0, 4096, 3166, 0, 0, 1562, 0, 4096, 0, 0, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3576, 0, 0, 0, 0, 4054, 0, 2752, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2945, 0, 0, 0, 0, 0, 0, 790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1787, 0, 0, 0, 0, 4096, 4096, 0, 0, 463, 0, 0, 0, 0, 4096, 0, 164, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2875, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 17454, 16468, 16388, 16023, 16383, 14706, 17269] avg=16384.0000 max=17454 max_rank=1 min=14706 min_rank=6 imbalance_ratio=1.065308 max_load_reduction=113618 max_load_reduction_pct=0.866837 moved_to_echo=64381 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 25, 0, 0, 0, 0, 4096, 1102, 0, 0, 735, 0, 4096, 0, 0, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 0, 3529, 0, 0, 0, 4, 4025, 0, 2752, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3745, 57, 0, 0, 0, 0, 0, 790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2779, 0, 0, 17, 0, 4080, 4096, 0, 0, 2951, 0, 0, 0, 0, 4055, 0, 164, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1567, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 14529, 16407, 16375, 16880, 16381, 18154, 15961] avg=16384.0000 max=18154 max_rank=6 min=14529 min_rank=1 imbalance_ratio=1.108032 max_load_reduction=112918 max_load_reduction_pct=0.861496 moved_to_echo=67376 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 0, 0, 7, 0, 0, 0, 0, 4096, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 104, 0, 0, 0, 0, 4096, 3017, 0, 0, 690, 0, 4096, 0, 0, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3365, 0, 0, 0, 1, 4040, 0, 2753, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 3035, 13, 0, 0, 0, 0, 0, 793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2528, 0, 0, 5, 0, 4090, 4096, 0, 0, 1840, 0, 0, 0, 0, 4094, 0, 164, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1861, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16473, 16245, 16383, 16128, 16385, 16821, 16255] avg=16384.0000 max=16821 max_rank=6 min=16128 min_rank=4 imbalance_ratio=1.026672 max_load_reduction=114251 max_load_reduction_pct=0.871666 moved_to_echo=65589 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 3, 0, 0, 0, 0, 4096, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 29, 0, 0, 0, 0, 4096, 2160, 0, 0, 1870, 0, 4096, 0, 0, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 3322, 0, 0, 0, 0, 3640, 0, 2752, 1989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 3161, 19, 0, 0, 0, 0, 0, 790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2562, 0, 0, 8, 0, 4080, 4096, 0, 0, 2177, 0, 0, 0, 0, 4076, 0, 164, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1566, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16728, 15801, 16388, 16257, 16387, 17167, 15961] avg=16384.0000 max=17167 max_rank=6 min=15801 min_rank=2 imbalance_ratio=1.047791 max_load_reduction=113905 max_load_reduction_pct=0.869026 moved_to_echo=65772 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 0, 17, 0, 0, 0, 0, 4096, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 263, 0, 0, 0, 0, 4096, 1969, 0, 0, 1059, 0, 4096, 0, 0, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 3527, 0, 0, 0, 3, 4017, 0, 2753, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 2828, 36, 0, 0, 0, 0, 0, 790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2658, 0, 0, 10, 0, 4087, 4096, 0, 0, 2011, 0, 0, 0, 0, 4081, 0, 164, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2146, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16393, 15928, 16392, 16382, 15941, 16385, 17111, 16540] avg=16384.0000 max=17111 max_rank=6 min=15928 min_rank=1 imbalance_ratio=1.044373 max_load_reduction=113961 max_load_reduction_pct=0.869453 moved_to_echo=65977 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[128, 32768, 0, 0, 0, 0, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 128, 0, 0, 0, 0, 32768, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 0, 768, 0, 0, 0, 0, 32768, 19840, 0, 0, 9472, 0, 32768, 128, 0, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[19200, 0, 0, 128, 0, 32768, 32768, 0, 0, 12416, 0, 0, 0, 0, 32768, 0, 1408, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 0, 16000, 32768, 0, 0, 0, 0, 0, 0, 32768, 0, 0, 32768, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[32768, 32768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 32768, 0, 0, 0, 0, 28544, 0, 0, 0, 128, 32000, 0, 22144, 16000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 0, 32768, 0, 0, 0, 32768, 26368, 256, 0, 0, 0, 0, 0, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131200 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131200 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 14, 0, 1, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 190, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 4096, 4096, 0, 380, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 3011, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 0, 0, 0, 0, 3, 0, 0, 543, 0, 4096, 0, 4096, 4095, 1, 0, 3, 4096, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 4086, 0, 4096, 0, 0, 0, 206, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 275, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3630, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 3963, 0, 4096, 0, 0, 0, 724, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 1178, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 2354, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 1, 4096, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4032, 0, 4096, 0, 0, 0, 159, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 261, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3747, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 21, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4065, 0, 4096, 0, 0, 0, 566, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 485, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3065, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 45, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 16, 0, 39, 0, 4096, 0, 0, 4096, 0, 4076, 0, 4096, 0, 0, 0, 1635, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 375, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2123, 0, 0, 0, 0, 0, 0, 0, 4068, 4096, 0, 2, 0, 0, 0, 0, 0, 15, 0, 4096, 0, 4096, 4096, 0, 0, 3, 4096, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 24, 0, 1, 0, 4096, 0, 0, 4096, 0, 4081, 0, 4096, 0, 0, 0, 994, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 310, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 2826, 0, 0, 0, 0, 0, 0, 0, 4073, 4096, 0, 0, 0, 0, 1, 0, 0, 34, 0, 4096, 0, 4096, 4095, 0, 0, 2, 4096, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 7, 0, 5, 0, 4096, 0, 0, 4096, 0, 4091, 0, 4096, 0, 0, 0, 488, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 895, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 21, 0, 4096, 0, 0, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 2682, 0, 0, 0, 0, 0, 0, 0, 4085, 4094, 0, 0, 0, 0, 0, 0, 0, 76, 0, 4096, 0, 4096, 4094, 0, 0, 5, 4096, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 22, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 206, 0, 0, 0, 0, 4096, 0, 0, 0, 3476, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3414, 620, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3630, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 660, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4040, 0, 0, 4096, 0, 0, 0, 4096, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 15970, 16145, 16379, 16378, 17092, 16326, 16385] avg=16384.0000 max=17092 max_rank=5 min=15970 min_rank=1 imbalance_ratio=1.043213 max_load_reduction=113980 max_load_reduction_pct=0.869598 moved_to_echo=66181 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 22, 14, 0, 1, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 190, 0, 0, 0, 0, 4095, 0, 0, 0, 3476, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 380, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 4096, 3582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 4091, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 3414, 620, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3011, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 0, 0, 0, 0, 3, 0, 4096, 660, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 0, 4040, 0, 0, 4095, 1, 0, 3, 4096, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16421, 15953, 16246, 16383, 16381, 16470, 16837, 16381] avg=16384.0000 max=16837 max_rank=6 min=15953 min_rank=1 imbalance_ratio=1.027649 max_load_reduction=114235 max_load_reduction_pct=0.871544 moved_to_echo=66069 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 22, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 566, 0, 0, 0, 0, 4096, 0, 0, 0, 3476, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3414, 620, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3065, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 660, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 4040, 0, 0, 4096, 0, 0, 0, 4096, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16330, 16355, 16379, 16378, 16527, 16345, 16383] avg=16384.0000 max=16527 max_rank=5 min=16330 min_rank=1 imbalance_ratio=1.008728 max_load_reduction=114545 max_load_reduction_pct=0.873909 moved_to_echo=65633 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 22, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 3963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 724, 0, 0, 0, 0, 4096, 0, 0, 0, 3476, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1178, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 4096, 3582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4091, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 3414, 620, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2354, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 0, 0, 0, 0, 4, 0, 4096, 660, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 4095, 0, 0, 1, 4096, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16274, 16488, 17041, 16381, 16386, 15814, 16301, 16387] avg=16384.0000 max=17041 max_rank=2 min=15814 min_rank=5 imbalance_ratio=1.040100 max_load_reduction=114031 max_load_reduction_pct=0.869987 moved_to_echo=64888 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 22, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 159, 0, 0, 0, 0, 4096, 0, 0, 0, 3476, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3414, 620, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3747, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 4096, 660, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4040, 0, 0, 4096, 0, 0, 0, 4096, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16342, 15923, 16131, 16379, 16378, 17211, 16321, 16387] avg=16384.0000 max=17211 max_rank=5 min=15923 min_rank=1 imbalance_ratio=1.050476 max_load_reduction=113861 max_load_reduction_pct=0.868690 moved_to_echo=66297 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 15, 16, 0, 39, 0, 4096, 0, 0, 4096, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1635, 0, 0, 0, 0, 4096, 0, 0, 0, 3475, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 4096, 4096, 3582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 4084, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3420, 621, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2123, 0, 0, 0, 0, 0, 0, 0, 4068, 4096, 0, 2, 0, 0, 0, 0, 4096, 661, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4039, 0, 0, 4096, 0, 0, 3, 4096, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16434, 17398, 16226, 16390, 16390, 15560, 16292, 16382] avg=16384.0000 max=17398 max_rank=1 min=15560 min_rank=5 imbalance_ratio=1.061890 max_load_reduction=113674 max_load_reduction_pct=0.867264 moved_to_echo=64624 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 22, 7, 0, 5, 0, 4096, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 488, 0, 0, 0, 0, 4096, 0, 0, 0, 3476, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 895, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4096, 4096, 3582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 4091, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4096, 0, 0, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 3414, 620, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2682, 0, 0, 0, 0, 0, 0, 0, 4085, 4094, 0, 0, 0, 0, 0, 0, 4096, 660, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 4040, 0, 0, 4094, 0, 0, 5, 4096, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16413, 16252, 16751, 16392, 16401, 16131, 16350, 16382] avg=16384.0000 max=16751 max_rank=2 min=16131 min_rank=5 imbalance_ratio=1.022400 max_load_reduction=114321 max_load_reduction_pct=0.872200 moved_to_echo=65264 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 22, 24, 0, 1, 0, 4096, 0, 0, 4096, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 994, 0, 0, 0, 0, 4096, 0, 0, 0, 3476, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 3581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 4091, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 3414, 620, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2826, 0, 0, 0, 0, 0, 0, 0, 4073, 4096, 0, 0, 0, 0, 1, 0, 4096, 660, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4040, 0, 0, 4095, 0, 0, 2, 4096, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16416, 16758, 16177, 16389, 16380, 16267, 16300, 16385] avg=16384.0000 max=16758 max_rank=1 min=16177 min_rank=2 imbalance_ratio=1.022827 max_load_reduction=114314 max_load_reduction_pct=0.872147 moved_to_echo=65332 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 0, 256, 128, 0, 128, 0, 32768, 0, 0, 32768, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 768, 0, 32384, 0, 0, 32768, 128, 0, 128, 32768, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 4992, 0, 0, 0, 0, 32768, 0, 0, 0, 27904, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[23552, 0, 0, 0, 0, 0, 0, 0, 32768, 32768, 0, 128, 0, 0, 128, 0, 32768, 5376, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 0, 32768, 0, 0, 128, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 27392, 4992, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0, 0, 32768, 32768, 32768, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 4224, 0, 0, 0, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 32768, 32768, 28672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131200 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131200 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131200 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131200 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 162, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 43, 0, 0, 4096, 0, 0, 0, 4096, 3892, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2473, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 563, 0, 0, 4096, 0, 0, 0, 4096, 1057, 0, 0, 4096, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 944, 4096, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 3, 0, 4096, 4095, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4090, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4090, 4096, 0, 0, 213, 0, 0, 4096, 0, 0, 0, 4096, 2950, 0, 0, 4096, 0, 4092, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 611, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 666, 0, 0, 4096, 0, 0, 0, 4096, 2819, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 7, 0, 4074, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1620, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4089, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4090, 4096, 0, 0, 838, 0, 0, 4096, 4, 0, 0, 4096, 1672, 0, 0, 4096, 0, 4087, 4079, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4, 0, 4082, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 809, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 2, 0, 4096, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4095, 4094, 4096, 0, 0, 645, 0, 0, 4096, 1, 0, 0, 4095, 2655, 0, 0, 4096, 0, 4091, 4093, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 10, 0, 4070, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 644, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4076, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4080, 4096, 0, 0, 702, 0, 0, 4096, 0, 0, 0, 4096, 2792, 0, 0, 4096, 0, 4086, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 3, 0, 4070, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 520, 4096, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4094, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4092, 4096, 0, 0, 120, 0, 0, 4096, 1, 0, 0, 4096, 3483, 0, 0, 4095, 0, 4091, 4095, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 162, 4096, 0, 0, 4096, 0, 0, 0, 2652, 469, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4086, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3627, 0, 0, 43, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 3892, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16394, 15572, 16383, 16388, 15958, 17617, 16377] avg=16384.0000 max=17617 max_rank=6 min=15572 min_rank=2 imbalance_ratio=1.075256 max_load_reduction=113455 max_load_reduction_pct=0.865593 moved_to_echo=66340 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 944, 4096, 0, 0, 4096, 0, 0, 0, 2652, 469, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 3, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4090, 4096, 4086, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 3627, 0, 0, 213, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 2950, 0, 0, 0, 0, 4092, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16384, 16355, 16390, 16383, 16122, 16671, 16384] avg=16384.0000 max=16671 max_rank=6 min=16122 min_rank=5 imbalance_ratio=1.017517 max_load_reduction=114401 max_load_reduction_pct=0.872810 moved_to_echo=65560 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2473, 4096, 0, 0, 4096, 0, 0, 0, 2652, 469, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4086, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3627, 0, 0, 563, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 1057, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16394, 17883, 16383, 16388, 16478, 14781, 16382] avg=16384.0000 max=17883 max_rank=2 min=14781 min_rank=6 imbalance_ratio=1.091492 max_load_reduction=113189 max_load_reduction_pct=0.863564 moved_to_echo=64029 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 611, 4096, 0, 0, 4096, 0, 0, 0, 2652, 469, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4086, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3627, 0, 0, 666, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 2819, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16395, 16021, 16383, 16388, 16581, 16544, 16377] avg=16384.0000 max=16581 max_rank=5 min=16021 min_rank=2 imbalance_ratio=1.012024 max_load_reduction=114491 max_load_reduction_pct=0.873497 moved_to_echo=65890 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 4096, 7, 0, 4074, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1620, 4096, 0, 0, 4096, 0, 0, 0, 2652, 469, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4089, 4096, 4086, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 3627, 0, 0, 838, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1438, 1672, 0, 0, 0, 0, 4087, 4079, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16379, 17030, 16381, 16376, 16747, 15385, 16387] avg=16384.0000 max=17030 max_rank=2 min=15385 min_rank=6 imbalance_ratio=1.039429 max_load_reduction=114042 max_load_reduction_pct=0.870071 moved_to_echo=64895 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 4096, 4, 0, 4082, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 809, 4096, 0, 0, 4096, 0, 0, 0, 2651, 469, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 2, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4095, 4096, 4086, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4094, 3627, 0, 0, 645, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1438, 2655, 0, 0, 0, 0, 4091, 4093, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16384, 16219, 16383, 16386, 16557, 16373, 16385] avg=16384.0000 max=16557 max_rank=5 min=16219 min_rank=2 imbalance_ratio=1.010559 max_load_reduction=114515 max_load_reduction_pct=0.873680 moved_to_echo=65701 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4096, 10, 0, 4070, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 644, 4096, 0, 0, 4096, 0, 0, 0, 2657, 472, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4076, 4096, 4091, 8, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 3624, 0, 0, 702, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1429, 2792, 0, 0, 0, 0, 4086, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16373, 16063, 16383, 16379, 16598, 16495, 16397] avg=16384.0000 max=16598 max_rank=5 min=16063 min_rank=2 imbalance_ratio=1.013062 max_load_reduction=114474 max_load_reduction_pct=0.873367 moved_to_echo=65869 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 4096, 3, 0, 4070, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 520, 4096, 0, 0, 4095, 0, 0, 0, 2652, 469, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4094, 4096, 4086, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 3627, 0, 0, 120, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1438, 3483, 0, 0, 0, 0, 4091, 4095, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16369, 15929, 16386, 16384, 16031, 17206, 16383] avg=16384.0000 max=17206 max_rank=6 min=15929 min_rank=2 imbalance_ratio=1.050171 max_load_reduction=113866 max_load_reduction_pct=0.868729 moved_to_echo=66004 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 0, 0, 0, 0, 32768, 128, 0, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 0, 32768, 0, 0, 32768, 0, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 32768, 32768, 29056, 0, 0, 3840, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[128, 0, 0, 11520, 21376, 0, 0, 0, 0, 32768, 32768, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 0, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 32768, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 32768, 0, 0, 0, 7808, 32768, 0, 0, 32768, 0, 0, 0, 21248, 3840, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 32768, 128, 32768, 0, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131200 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131200 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 55, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 3915, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 23, 0, 4088, 0, 0, 0, 2635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1570, 4094, 0, 0, 4096, 4096, 0, 4093, 7, 4096, 4096, 1, 4096, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4094, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 207, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3897, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3750, 4096, 0, 0, 4096, 4096, 0, 4096, 1, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 16] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 504, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3868, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 10, 0, 4083, 0, 0, 0, 2893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 927, 4096, 0, 0, 4096, 4096, 0, 4087, 13, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4073, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 23] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 185, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3985, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 20, 0, 4095, 0, 0, 0, 2185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1810, 4096, 0, 0, 4096, 4096, 0, 4096, 1, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 18] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 76, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 3755, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 75, 0, 4085, 0, 0, 0, 2748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1556, 4093, 0, 0, 4096, 4096, 0, 4093, 10, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4072, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 10] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 163, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 3663, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 33, 0, 4089, 0, 0, 0, 2141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2198, 4095, 0, 0, 4096, 4096, 0, 4096, 2, 4096, 4096, 1, 4096, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4095, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 403, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3711, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3149, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 61] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 83, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 3944, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 7, 0, 4094, 0, 0, 0, 1894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2264, 4095, 0, 0, 4096, 4096, 0, 4095, 2, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4085, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 207, 4096, 1990, 1896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2100, 1995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3897, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 315, 0, 4096, 2101, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3750, 4096, 0, 0, 1947, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16383, 16438, 14731, 17985, 16389, 16384, 16381] avg=16384.0000 max=17985 max_rank=4 min=14731 min_rank=3 imbalance_ratio=1.097717 max_load_reduction=113087 max_load_reduction_pct=0.862785 moved_to_echo=67139 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 55, 4096, 1990, 1896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 2100, 1995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3915, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 23, 0, 4088, 0, 0, 0, 2635, 0, 4096, 2100, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1570, 4094, 0, 0, 1947, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 6, 1, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16229, 16385, 16456, 17056, 15800, 16387, 16384, 16375] avg=16384.0000 max=17056 max_rank=3 min=15800 min_rank=4 imbalance_ratio=1.041016 max_load_reduction=114016 max_load_reduction_pct=0.869873 moved_to_echo=64946 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 163, 4096, 1990, 1896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 2100, 1995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3663, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 33, 0, 4089, 0, 0, 0, 2141, 0, 4096, 2101, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2198, 4095, 0, 0, 1947, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 6, 1, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16337, 16385, 16204, 16575, 16432, 16391, 16383, 16365] avg=16384.0000 max=16575 max_rank=3 min=16204 min_rank=2 imbalance_ratio=1.011658 max_load_reduction=114497 max_load_reduction_pct=0.873543 moved_to_echo=65571 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 403, 4096, 1990, 1896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2100, 1995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3711, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 863, 0, 4096, 2100, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3149, 4096, 0, 0, 1947, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 61, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16577, 16383, 16252, 15275, 17384, 16389, 16385, 16427] avg=16384.0000 max=17384 max_rank=4 min=15275 min_rank=3 imbalance_ratio=1.061035 max_load_reduction=113688 max_load_reduction_pct=0.867371 moved_to_echo=66585 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 185, 4096, 1990, 1896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2100, 1995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3985, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 20, 0, 4095, 0, 0, 0, 2185, 0, 4096, 2100, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1810, 4096, 0, 0, 1947, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 18, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16359, 16383, 16526, 16611, 16045, 16380, 16385, 16383] avg=16384.0000 max=16611 max_rank=3 min=16045 min_rank=4 imbalance_ratio=1.013855 max_load_reduction=114461 max_load_reduction_pct=0.873268 moved_to_echo=65193 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 504, 4096, 1991, 1899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2100, 1995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3868, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 10, 0, 4083, 0, 0, 0, 2893, 0, 4096, 2101, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 927, 4096, 0, 0, 1938, 4096, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 4073, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 23, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16682, 16383, 16415, 17298, 15144, 16379, 16384, 16387] avg=16384.0000 max=17298 max_rank=3 min=15144 min_rank=4 imbalance_ratio=1.055786 max_load_reduction=113774 max_load_reduction_pct=0.868027 moved_to_echo=64294 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 76, 4096, 1990, 1896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 2100, 1995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3755, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 75, 0, 4085, 0, 0, 0, 2748, 0, 4096, 2101, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1556, 4093, 0, 0, 1947, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4072, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16250, 16386, 16296, 17220, 15785, 16376, 16383, 16376] avg=16384.0000 max=17220 max_rank=3 min=15785 min_rank=4 imbalance_ratio=1.051025 max_load_reduction=113852 max_load_reduction_pct=0.868622 moved_to_echo=64920 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 83, 4096, 1990, 1896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2100, 1995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3944, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 7, 0, 4094, 0, 0, 0, 1894, 0, 4096, 2100, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2264, 4095, 0, 0, 1947, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16257, 16384, 16485, 16306, 16497, 16381, 16384, 16378] avg=16384.0000 max=16497 max_rank=4 min=16257 min_rank=0 imbalance_ratio=1.006897 max_load_reduction=114575 max_load_reduction_pct=0.874138 moved_to_echo=65640 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 256, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 32768, 0, 0, 128, 0, 0, 0, 256, 0, 32768, 0, 0, 0, 15744, 0, 32768, 16896, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 30848, 32768, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 32768, 0, 0, 0, 0, 0, 32768, 0, 128, 0, 0, 0, 0, 16896, 16000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 32768, 0, 0, 0, 1792, 32768, 16000, 15232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 17280, 32768, 0, 0, 15616, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 32768, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 32768, 0, 0, 0, 0, 32768, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131200 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131200 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131200 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3725, 1, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 4096, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 3740, 0, 803, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4059, 3, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 4083, 4096, 0, 4092, 0, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 10, 4096, 4095, 4096, 0, 0, 0, 17, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4026, 0, 263, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 1, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4042, 4096, 0, 4089, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4096, 17, 4095, 4096, 4096, 0, 0, 0, 46, 0, 0, 0, 0, 38, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4077, 0, 141, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 4080, 4096, 0, 4094, 0, 3797, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 11, 4096, 4095, 4096, 0, 0, 0, 35, 1, 0, 0, 0, 59, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 3922, 0, 579, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 1, 0, 4095, 0, 0, 0, 1, 4096, 4096, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 4081, 4096, 0, 4094, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 10, 4096, 4095, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 39, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4020, 0, 287, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3894, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 3, 4096, 0, 0, 0, 4096, 0, 0, 4061, 4096, 0, 4087, 0, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 12, 4096, 4095, 4096, 0, 0, 0, 31, 0, 0, 0, 0, 179, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 3990, 0, 320, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4093, 4096, 0, 4096, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 3903, 0, 119, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 1, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 4096, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 4096, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 87, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4039, 0, 191, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3725, 1, 0, 4096, 0, 0, 4096, 4096, 90, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4038, 206, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3740, 0, 803, 0, 0, 0, 3890, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16159, 16400, 16424, 16376, 16317, 16625, 16384, 16387] avg=16384.0000 max=16625 max_rank=5 min=16159 min_rank=0 imbalance_ratio=1.014709 max_load_reduction=114447 max_load_reduction_pct=0.873161 moved_to_echo=65713 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4059, 3, 0, 4096, 0, 0, 4096, 4096, 90, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4037, 206, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 10, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4026, 0, 263, 0, 0, 0, 3890, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16497, 16389, 16347, 16386, 16316, 16371, 16383, 16383] avg=16384.0000 max=16497 max_rank=0 min=16316 min_rank=4 imbalance_ratio=1.006897 max_load_reduction=114575 max_load_reduction_pct=0.874138 moved_to_echo=65453 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 1, 0, 4096, 0, 0, 4096, 4096, 90, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4042, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4037, 210, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4081, 0, 17, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 38, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 0, 141, 0, 0, 0, 3886, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16406, 16341, 16499, 16387, 16378, 16296, 16385, 16380] avg=16384.0000 max=16499 max_rank=2 min=16296 min_rank=5 imbalance_ratio=1.007019 max_load_reduction=114573 max_load_reduction_pct=0.874123 moved_to_echo=65439 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 1, 0, 4096, 0, 0, 4096, 4096, 90, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4037, 206, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 1, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 87, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4039, 0, 191, 0, 0, 0, 3890, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16460, 16400, 16365, 16378, 16385, 16312, 16384, 16388] avg=16384.0000 max=16460 max_rank=0 min=16312 min_rank=5 imbalance_ratio=1.004639 max_load_reduction=114612 max_load_reduction_pct=0.874420 moved_to_echo=65469 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 1, 0, 4095, 0, 0, 4096, 4096, 90, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 17, 4096, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4081, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4041, 206, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4089, 0, 10, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 39, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4020, 0, 287, 0, 0, 0, 3890, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16363, 16389, 16431, 16387, 16348, 16389, 16384, 16381] avg=16384.0000 max=16431 max_rank=2 min=16348 min_rank=4 imbalance_ratio=1.002869 max_load_reduction=114641 max_load_reduction_pct=0.874641 moved_to_echo=65502 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3894, 0, 0, 4096, 0, 0, 4096, 4096, 90, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 4061, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4039, 206, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4089, 0, 12, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 179, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3990, 0, 320, 0, 0, 0, 3890, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16329, 16369, 16327, 16389, 16504, 16392, 16384, 16378] avg=16384.0000 max=16504 max_rank=4 min=16327 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=114568 max_load_reduction_pct=0.874084 moved_to_echo=65658 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 0, 4095, 0, 0, 4096, 4096, 89, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3797, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4037, 206, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4089, 0, 11, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 1, 0, 0, 0, 59, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3922, 0, 579, 0, 0, 0, 3890, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16333, 16386, 16221, 16388, 16390, 16583, 16384, 16387] avg=16384.0000 max=16583 max_rank=5 min=16221 min_rank=2 imbalance_ratio=1.012146 max_load_reduction=114489 max_load_reduction_pct=0.873482 moved_to_echo=65744 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 4096, 4096, 90, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4037, 206, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3903, 0, 119, 0, 0, 0, 3890, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16525, 16398, 16458, 16381, 16434, 16104, 16384, 16388] avg=16384.0000 max=16525 max_rank=0 min=16104 min_rank=5 imbalance_ratio=1.008606 max_load_reduction=114547 max_load_reduction_pct=0.873924 moved_to_echo=65310 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 128, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 0, 31744, 0, 2816, 0, 0, 0, 31232, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 32768, 0, 31744, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32640, 32384, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 128, 256, 32768, 0, 0, 128, 0, 0, 0, 0, 32768, 0, 0, 32640, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 32768, 0, 0, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 31616, 128, 0, 32768, 0, 0, 32768, 32768, 768, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[256, 128, 0, 0, 0, 640, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 32768, 32768, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 32768, 0, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131200 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131200 slack_slots=8832 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131328 slack_slots=8704 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1418, 1935, 1892, 1, 472, 2, 1850, 942, 1069, 1769, 1165, 470, 1618, 2502, 0, 1156, 926, 1566, 1176, 1495, 6, 855, 159, 2173, 223, 23, 2970, 1677, 1353, 42, 1623, 1141, 292, 0, 1599, 425, 232, 1761, 1548, 261, 277, 3, 2228, 2673, 0, 2167, 1190, 1610, 1370, 740, 509, 774, 7, 58, 15, 1989, 0, 304, 40, 557, 1026, 2792, 1357, 266, 1181, 9, 0, 1497, 2024, 1453, 932, 900, 1446, 2061, 2233, 351, 1890, 0, 944, 1941, 727, 214, 7, 15, 2092, 713, 1002, 6, 1745, 152, 1132, 947, 208, 1926, 553, 0, 1117, 606, 9, 2317, 5, 448, 1406, 883, 247, 1749, 147, 1383, 240, 1858, 2479, 1506, 1913, 1863, 476, 1692, 2803, 1234, 2011, 1544, 1361, 2117, 1603, 587, 818, 6, 581, 23] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1621, 2271, 1432, 0, 226, 0, 1855, 518, 884, 2928, 1639, 363, 1180, 3318, 0, 1414, 1048, 2160, 1211, 1565, 49, 883, 146, 2180, 116, 263, 2587, 1861, 1759, 170, 1674, 1215, 201, 0, 2094, 591, 263, 2068, 1216, 718, 215, 2, 1512, 3151, 0, 1925, 911, 973, 1192, 711, 594, 499, 0, 77, 6, 1861, 5, 149, 10, 321, 1453, 2989, 1389, 125, 916, 4, 0, 1542, 1482, 1711, 634, 862, 1546, 1454, 1848, 132, 1662, 6, 1112, 2307, 664, 368, 3, 8, 2134, 502, 926, 7, 1990, 126, 1183, 624, 405, 2090, 413, 0, 1458, 1063, 14, 1857, 9, 651, 1761, 1199, 155, 1796, 44, 1344, 179, 1953, 2801, 1354, 2153, 2104, 462, 1537, 1670, 1365, 1574, 1238, 1028, 2826, 1079, 587, 1036, 1, 345, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2355, 2035, 1805, 0, 363, 0, 1987, 1122, 1153, 2324, 744, 409, 1401, 1764, 0, 845, 706, 1431, 1057, 1604, 16, 640, 185, 2359, 424, 115, 3581, 2033, 1784, 50, 1466, 1156, 199, 6, 1765, 941, 176, 1593, 1198, 261, 277, 5, 2148, 2558, 0, 1359, 787, 866, 1240, 848, 890, 940, 1, 11, 14, 1545, 0, 683, 107, 417, 1729, 2445, 1432, 675, 1077, 8, 0, 1902, 2045, 2031, 705, 1505, 1200, 1902, 2268, 245, 1151, 21, 676, 1735, 865, 667, 6, 11, 2026, 345, 1653, 20, 2419, 125, 864, 611, 116, 1690, 391, 0, 1409, 1244, 9, 2069, 16, 388, 1714, 1103, 248, 1545, 64, 1105, 388, 1889, 1857, 1589, 2205, 2143, 150, 1718, 2265, 807, 2747, 1592, 1188, 1648, 885, 923, 872, 14, 958, 10] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1791, 2459, 1381, 0, 656, 3, 1874, 996, 931, 2312, 1092, 300, 1167, 2721, 0, 1192, 961, 1250, 1286, 1351, 10, 1428, 201, 1996, 295, 165, 3307, 1657, 1104, 32, 1399, 1425, 228, 4, 1944, 355, 122, 1826, 1152, 230, 197, 3, 1985, 2848, 0, 2310, 861, 873, 926, 802, 787, 1130, 7, 71, 7, 1897, 0, 231, 65, 675, 1318, 3203, 1084, 190, 1225, 11, 0, 1491, 2237, 1891, 634, 1389, 1216, 1844, 1843, 390, 1162, 5, 1431, 2015, 891, 379, 7, 23, 2061, 609, 979, 16, 2550, 72, 966, 798, 278, 2295, 446, 3, 1355, 794, 8, 1799, 17, 429, 1762, 1037, 233, 1964, 58, 1394, 754, 1312, 2223, 1513, 2196, 1568, 565, 1250, 2350, 651, 2002, 1834, 1033, 2360, 1069, 1035, 790, 6, 467, 14] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1660, 2149, 2212, 1, 467, 2, 1570, 389, 882, 1650, 910, 708, 1880, 2669, 0, 1332, 966, 1402, 1642, 1582, 5, 859, 146, 2239, 130, 59, 3113, 1491, 990, 31, 1387, 1167, 530, 2, 1799, 242, 444, 1702, 1255, 281, 234, 3, 1639, 2935, 0, 1937, 1069, 908, 1502, 413, 438, 930, 1, 19, 6, 1913, 0, 253, 111, 348, 1216, 2772, 1117, 457, 1221, 4, 1, 1749, 2245, 1760, 1080, 789, 1425, 2180, 1819, 304, 1868, 6, 923, 2019, 907, 310, 12, 61, 2176, 749, 824, 15, 2189, 125, 681, 915, 249, 2138, 588, 3, 817, 629, 4, 2287, 7, 939, 1814, 1140, 216, 1397, 215, 1610, 697, 2133, 1837, 1899, 2310, 2260, 235, 1241, 3134, 1043, 2108, 1576, 951, 1646, 1306, 850, 860, 17, 373, 20] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1821, 1724, 1425, 1, 318, 0, 2669, 423, 704, 3036, 1224, 381, 1174, 2969, 2, 1141, 790, 1475, 1428, 1206, 24, 723, 152, 2140, 153, 44, 3405, 1927, 1793, 55, 1575, 1211, 210, 3, 1589, 549, 190, 1571, 1124, 431, 142, 1, 1916, 2822, 0, 2403, 1091, 1437, 1249, 499, 854, 816, 0, 5, 6, 1387, 1, 177, 16, 319, 958, 2721, 1248, 231, 961, 9, 3, 1789, 2084, 1749, 955, 1409, 1070, 2172, 2424, 265, 2157, 15, 914, 1812, 574, 185, 9, 2, 2433, 455, 1101, 20, 2112, 47, 971, 824, 296, 2138, 538, 4, 1016, 836, 0, 2173, 3, 443, 1781, 1314, 272, 1491, 118, 1328, 421, 1683, 2659, 1894, 2259, 1907, 206, 1026, 2076, 989, 2326, 1262, 984, 2767, 1494, 509, 702, 10, 544, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2168, 2671, 1764, 0, 406, 0, 2068, 929, 1058, 2932, 1005, 699, 1271, 2540, 0, 1282, 1111, 1408, 1063, 1654, 25, 710, 148, 1991, 96, 46, 3256, 2047, 1332, 38, 1314, 1065, 559, 11, 1483, 357, 176, 1530, 1327, 215, 70, 8, 1781, 2700, 0, 2248, 757, 637, 1186, 472, 796, 726, 7, 42, 42, 2621, 0, 264, 30, 472, 1251, 2828, 1046, 166, 784, 19, 0, 1652, 2091, 1701, 676, 1154, 1027, 2104, 2158, 289, 1598, 3, 1218, 1764, 1078, 308, 11, 5, 2055, 640, 926, 37, 2315, 45, 842, 832, 293, 2178, 546, 3, 1329, 675, 1, 1887, 7, 428, 1617, 1069, 318, 1734, 73, 1197, 451, 2086, 3011, 1641, 2271, 1947, 371, 767, 2569, 819, 2271, 1489, 1073, 2194, 1408, 798, 779, 30, 499, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1609, 2138, 1825, 1, 202, 0, 2045, 396, 1059, 2457, 1038, 706, 2192, 3019, 0, 985, 639, 1814, 1045, 1269, 10, 610, 151, 2303, 85, 40, 3126, 2084, 1089, 39, 1515, 1300, 587, 3, 1887, 502, 241, 2026, 698, 500, 258, 0, 1549, 2627, 0, 2049, 999, 1082, 1202, 685, 453, 611, 3, 89, 6, 2035, 0, 206, 23, 485, 1342, 2976, 1149, 103, 1215, 6, 1, 1484, 1936, 1671, 911, 1132, 1552, 2052, 1934, 177, 1932, 8, 1185, 1620, 982, 200, 13, 6, 2067, 341, 657, 18, 2034, 106, 981, 1037, 328, 2421, 471, 1, 1733, 919, 12, 1774, 7, 485, 1597, 938, 136, 1642, 74, 1078, 458, 2445, 2616, 1806, 2642, 2078, 444, 931, 2362, 1040, 1409, 1440, 1134, 2384, 1377, 801, 942, 5, 695, 22] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1418, 1935, 1892, 1, 472, 2, 1850, 942, 1069, 1726, 1165, 470, 1618, 0, 0, 1156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 926, 1566, 1176, 1495, 6, 855, 159, 2173, 223, 23, 1631, 1677, 1353, 42, 1623, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 0, 1599, 425, 232, 1761, 1548, 261, 277, 3, 2228, 2673, 0, 2167, 1190, 1610, 836, 247, 164, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1370, 740, 509, 774, 7, 58, 15, 1989, 0, 304, 40, 557, 1026, 2792, 1357, 266, 2502, 2291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 9, 0, 1497, 1777, 1453, 932, 900, 1446, 2061, 0, 351, 1890, 0, 944, 1941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 727, 214, 7, 15, 2092, 713, 1002, 6, 1745, 152, 1132, 947, 208, 1926, 553, 0, 2233, 1339, 544, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1117, 606, 9, 2317, 5, 448, 1406, 883, 247, 1749, 147, 1383, 240, 1858, 1643, 1506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1205, 1863, 476, 1692, 0, 1234, 2011, 1544, 1361, 2117, 1603, 587, 818, 6, 581, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15716, 16069, 17556, 16597, 16382, 16067, 15564, 17121] avg=16384.0000 max=17556 max_rank=2 min=15564 min_rank=6 imbalance_ratio=1.071533 max_load_reduction=113516 max_load_reduction_pct=0.866058 moved_to_echo=65134 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1621, 2271, 1432, 0, 226, 0, 1855, 518, 884, 2857, 1639, 363, 1180, 0, 0, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 2160, 1211, 1565, 49, 883, 146, 2180, 116, 263, 1421, 1861, 1759, 170, 1674, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 2094, 591, 263, 2068, 1216, 718, 215, 2, 1512, 3151, 0, 1925, 911, 973, 944, 180, 185, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 711, 594, 499, 0, 77, 6, 1861, 5, 149, 10, 321, 1453, 2989, 1389, 125, 3318, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 916, 4, 0, 1542, 1302, 1711, 634, 862, 1546, 1454, 0, 132, 1662, 6, 1112, 2307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 664, 368, 3, 8, 2134, 502, 926, 7, 1990, 126, 1183, 624, 405, 2090, 413, 0, 1848, 1166, 612, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1458, 1063, 14, 1857, 9, 651, 1761, 1199, 155, 1796, 44, 1344, 179, 1953, 1857, 1354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1356, 2104, 462, 1537, 0, 1365, 1574, 1238, 1028, 2826, 1079, 587, 1036, 1, 345, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16260, 17721, 17220, 16063, 15190, 15375, 16694, 16549] avg=16384.0000 max=17721 max_rank=1 min=15190 min_rank=4 imbalance_ratio=1.081604 max_load_reduction=113351 max_load_reduction_pct=0.864799 moved_to_echo=63808 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2355, 2035, 1805, 0, 363, 0, 1987, 1122, 1153, 2268, 744, 409, 1401, 0, 0, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 706, 1431, 1057, 1604, 16, 640, 185, 2359, 424, 115, 1967, 2033, 1784, 50, 1466, 1156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 6, 1765, 941, 176, 1593, 1198, 261, 277, 5, 2148, 2558, 0, 1359, 787, 866, 626, 249, 189, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1240, 848, 890, 940, 1, 11, 14, 1545, 0, 683, 107, 417, 1729, 2445, 1432, 675, 1764, 1852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 8, 0, 1902, 1796, 2031, 705, 1505, 1200, 1902, 0, 245, 1151, 21, 676, 1735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 865, 667, 6, 11, 2026, 345, 1653, 20, 2419, 125, 864, 611, 116, 1690, 391, 0, 2268, 1614, 627, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1409, 1244, 9, 2069, 16, 388, 1714, 1103, 248, 1545, 64, 1105, 388, 1889, 1231, 1589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1389, 2143, 150, 1718, 0, 807, 2747, 1592, 1188, 1648, 885, 923, 872, 14, 958, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16487, 16993, 15259, 16593, 15954, 16731, 16011, 17044] avg=16384.0000 max=17044 max_rank=7 min=15259 min_rank=2 imbalance_ratio=1.040283 max_load_reduction=114028 max_load_reduction_pct=0.869965 moved_to_echo=65740 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1791, 2459, 1381, 0, 656, 3, 1874, 996, 931, 2256, 1092, 300, 1167, 0, 0, 1192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 961, 1250, 1286, 1351, 10, 1428, 201, 1996, 295, 165, 1816, 1657, 1104, 32, 1399, 1425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 4, 1944, 355, 122, 1826, 1152, 230, 197, 3, 1985, 2848, 0, 2310, 861, 873, 749, 273, 188, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 926, 802, 787, 1130, 7, 71, 7, 1897, 0, 231, 65, 675, 1318, 3203, 1084, 190, 2721, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1225, 11, 0, 1491, 1964, 1891, 634, 1389, 1216, 1844, 0, 390, 1162, 5, 1431, 2015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 891, 379, 7, 23, 2061, 609, 979, 16, 2550, 72, 966, 798, 278, 2295, 446, 3, 1843, 1491, 625, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1355, 794, 8, 1799, 17, 429, 1762, 1037, 233, 1964, 58, 1394, 754, 1312, 1474, 1513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1383, 1568, 565, 1250, 0, 651, 2002, 1834, 1033, 2360, 1069, 1035, 790, 6, 467, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16098, 16376, 16204, 17034, 16668, 16762, 15903, 16027] avg=16384.0000 max=17034 max_rank=3 min=15903 min_rank=6 imbalance_ratio=1.039673 max_load_reduction=114038 max_load_reduction_pct=0.870041 moved_to_echo=65360 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1660, 2149, 2212, 1, 467, 2, 1570, 389, 882, 1605, 910, 708, 1880, 0, 0, 1332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1402, 1642, 1582, 5, 859, 146, 2239, 130, 59, 1705, 1491, 990, 31, 1387, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 2, 1799, 242, 444, 1702, 1255, 281, 234, 3, 1639, 2935, 0, 1937, 1069, 908, 624, 277, 202, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1502, 413, 438, 930, 1, 19, 6, 1913, 0, 253, 111, 348, 1216, 2772, 1117, 457, 2669, 2562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1221, 4, 1, 1749, 1968, 1760, 1080, 789, 1425, 2180, 0, 304, 1868, 6, 923, 2019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 907, 310, 12, 61, 2176, 749, 824, 15, 2189, 125, 681, 915, 249, 2138, 588, 3, 1819, 1408, 660, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 817, 629, 4, 2287, 7, 939, 1814, 1140, 216, 1397, 215, 1610, 697, 2133, 1213, 1899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1448, 2260, 235, 1241, 0, 1043, 2108, 1576, 951, 1646, 1306, 850, 860, 17, 373, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15767, 15801, 16128, 16727, 17297, 16401, 17017, 15934] avg=16384.0000 max=17297 max_rank=4 min=15767 min_rank=0 imbalance_ratio=1.055725 max_load_reduction=113775 max_load_reduction_pct=0.868034 moved_to_echo=66649 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1821, 1724, 1425, 1, 318, 0, 2669, 423, 704, 2962, 1224, 381, 1174, 0, 2, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 790, 1475, 1428, 1206, 24, 723, 152, 2140, 153, 44, 1870, 1927, 1793, 55, 1575, 1211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 3, 1589, 549, 190, 1571, 1124, 431, 142, 1, 1916, 2822, 0, 2403, 1091, 1437, 896, 254, 194, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1249, 499, 854, 816, 0, 5, 6, 1387, 1, 177, 16, 319, 958, 2721, 1248, 231, 2969, 1697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 961, 9, 3, 1789, 1830, 1749, 955, 1409, 1070, 2172, 0, 265, 2157, 15, 914, 1812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 185, 9, 2, 2433, 455, 1101, 20, 2112, 47, 971, 824, 296, 2138, 538, 4, 2424, 1535, 643, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 836, 0, 2173, 3, 443, 1781, 1314, 272, 1491, 118, 1328, 421, 1683, 1763, 1894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1422, 1907, 206, 1026, 0, 989, 2326, 1262, 984, 2767, 1494, 509, 702, 10, 544, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15969, 16566, 16897, 15153, 17110, 16690, 16536, 16151] avg=16384.0000 max=17110 max_rank=4 min=15153 min_rank=3 imbalance_ratio=1.044312 max_load_reduction=113962 max_load_reduction_pct=0.869461 moved_to_echo=66487 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2168, 2671, 1764, 0, 406, 0, 2068, 929, 1058, 2861, 1005, 699, 1271, 0, 0, 1282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1111, 1408, 1063, 1654, 25, 710, 148, 1991, 96, 46, 1788, 2047, 1332, 38, 1314, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 11, 1483, 357, 176, 1530, 1327, 215, 70, 8, 1781, 2700, 0, 2248, 757, 637, 1015, 255, 195, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1186, 472, 796, 726, 7, 42, 42, 2621, 0, 264, 30, 472, 1251, 2828, 1046, 166, 2540, 2099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 784, 19, 0, 1652, 1836, 1701, 676, 1154, 1027, 2104, 0, 289, 1598, 3, 1218, 1764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 308, 11, 5, 2055, 640, 926, 37, 2315, 45, 842, 832, 293, 2178, 546, 3, 2158, 1468, 646, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1329, 675, 1, 1887, 7, 428, 1617, 1069, 318, 1734, 73, 1197, 451, 2086, 1996, 1641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1430, 1947, 371, 767, 0, 819, 2271, 1489, 1073, 2194, 1408, 798, 779, 30, 499, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18182, 15836, 15395, 16588, 15825, 16856, 16509, 15881] avg=16384.0000 max=18182 max_rank=0 min=15395 min_rank=2 imbalance_ratio=1.109741 max_load_reduction=112890 max_load_reduction_pct=0.861282 moved_to_echo=65071 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1609, 2138, 1825, 1, 202, 0, 2045, 396, 1059, 2397, 1038, 706, 2192, 0, 0, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 639, 1814, 1045, 1269, 10, 610, 151, 2303, 85, 40, 1717, 2084, 1089, 39, 1515, 1300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 587, 3, 1887, 502, 241, 2026, 698, 500, 258, 0, 1549, 2627, 0, 2049, 999, 1082, 882, 236, 227, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1202, 685, 453, 611, 3, 89, 6, 2035, 0, 206, 23, 485, 1342, 2976, 1149, 103, 3019, 1930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1215, 6, 1, 1484, 1700, 1671, 911, 1132, 1552, 2052, 0, 177, 1932, 8, 1185, 1620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 982, 200, 13, 6, 2067, 341, 657, 18, 2034, 106, 981, 1037, 328, 2421, 471, 1, 1934, 1409, 752, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1733, 919, 12, 1774, 7, 485, 1597, 938, 136, 1642, 74, 1078, 458, 2445, 1734, 1806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1663, 2078, 444, 931, 0, 1040, 1409, 1440, 1134, 2384, 1377, 801, 942, 5, 695, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16593, 15710, 16413, 16317, 16646, 16190, 16838, 16365] avg=16384.0000 max=16838 max_rank=6 min=15710 min_rank=1 imbalance_ratio=1.027710 max_load_reduction=114234 max_load_reduction_pct=0.871536 moved_to_echo=66039 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[7168, 12544, 9984, 11776, 256, 6784, 1408, 17408, 1536, 768, 13952, 14848, 11264, 512, 12032, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[9984, 5248, 5376, 6528, 128, 384, 128, 15360, 128, 2304, 512, 3712, 10368, 22784, 9856, 2304, 21504, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[11392, 15872, 2944, 10240, 0, 8064, 16512, 12032, 8832, 18048, 10240, 6144, 6912, 128, 4480, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[14464, 17408, 13824, 128, 3200, 128, 16000, 5760, 7808, 18944, 8832, 4096, 11904, 0, 128, 9472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[8704, 128, 128, 13184, 14208, 14080, 6528, 9216, 10496, 15872, 0, 2176, 13440, 128, 8448, 15232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[10240, 6784, 128, 16256, 128, 4224, 13568, 8704, 1920, 13440, 896, 10496, 3712, 15360, 12928, 13312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2816, 128, 14208, 3968, 1920, 14080, 9600, 2944, 1792, 128, 14848, 22400, 0, 16512, 7680, 8448, 6656, 2048, 1664, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[6784, 2688, 128, 256, 17152, 4480, 8192, 256, 17408, 896, 7680, 6656, 2176, 16896, 3968, 128, 16640, 11520, 5120, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4085, 1, 11, 2, 0, 0, 85, 0, 0, 0, 1, 4095, 0, 0, 4096, 0, 0, 4096, 0, 3614, 0, 0, 4038, 0, 4096, 4095, 23, 0, 0, 4095, 0, 3181, 3388, 18, 3, 0, 3781, 0, 0, 0, 0, 257, 0, 2, 4094, 4078, 0, 0, 4094, 0, 89, 4096, 0, 0, 4095, 144, 0, 0, 4096, 0, 26, 2393, 0, 1, 3132, 0, 0, 4094, 2, 2630, 4096, 0, 0, 0, 0, 1, 4, 2499, 2, 0, 0, 0, 1, 2675, 0, 0, 3, 1, 0, 4094, 0, 0, 0, 0, 1, 4094, 3926, 18, 18, 0, 33, 4095, 0, 4096, 0, 0, 0, 4096, 4094, 0, 0, 6, 0, 320, 0, 0, 4082, 54, 0, 545, 0, 0, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 4090, 0, 183, 0, 0, 0, 313, 5, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 4078, 1, 3760, 0, 0, 4064, 0, 4078, 4096, 9, 0, 0, 4095, 0, 2516, 3289, 0, 4, 0, 3811, 0, 0, 2, 0, 775, 0, 1, 4087, 4072, 0, 0, 4096, 7, 48, 4096, 0, 0, 4094, 18, 0, 0, 4081, 0, 4, 2450, 0, 0, 1237, 2, 0, 4096, 0, 3650, 4096, 0, 0, 0, 0, 0, 0, 3364, 0, 11, 0, 0, 0, 2598, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 4026, 2, 10, 0, 5, 4096, 0, 4092, 4, 0, 0, 4096, 4096, 1, 0, 1, 0, 36, 0, 0, 4089, 161, 0, 602, 0, 0, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[3, 0, 0, 32, 4086, 0, 0, 0, 0, 0, 161, 0, 1, 3, 6, 4091, 0, 5, 4091, 0, 0, 4093, 0, 4028, 8, 0, 3769, 2, 4087, 4091, 9, 1, 3, 4090, 1, 2489, 3351, 18, 0, 4, 3573, 0, 5, 1, 5, 255, 5, 260, 4096, 4058, 5, 4, 4091, 0, 14, 4090, 5, 5, 4091, 62, 0, 5, 4091, 0, 13, 1910, 0, 0, 3238, 3, 1, 4091, 7, 1752, 4091, 0, 5, 0, 0, 0, 2, 3482, 0, 0, 1, 0, 0, 3456, 4, 5, 2, 0, 5, 4089, 0, 5, 0, 4, 2, 4091, 3895, 15, 2, 4, 1, 4090, 5, 4086, 0, 1, 0, 4091, 4089, 3, 5, 568, 0, 100, 0, 4, 4089, 0, 0, 440, 1, 0, 0, 4080] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 20, 4072, 3, 304, 0, 0, 0, 492, 8, 0, 0, 0, 4087, 0, 1, 4093, 0, 2, 4090, 2, 3674, 0, 0, 4035, 0, 4094, 4093, 74, 0, 0, 4096, 0, 2655, 3755, 2, 0, 0, 3503, 1, 0, 1, 0, 689, 0, 1, 4016, 4087, 0, 0, 4094, 8, 15, 4096, 2, 0, 4093, 3, 0, 0, 4095, 0, 15, 1719, 0, 2, 1948, 4, 0, 4093, 0, 3806, 4084, 0, 0, 3, 5, 1, 0, 3255, 0, 9, 0, 0, 0, 2361, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 1, 4093, 3978, 21, 8, 0, 3, 4093, 2, 4052, 0, 0, 3, 4094, 4093, 0, 0, 11, 0, 164, 2, 3, 4079, 159, 0, 359, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 50, 4055, 0, 459, 0, 0, 0, 290, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4063, 0, 3055, 0, 0, 4095, 0, 4073, 4096, 13, 0, 0, 4096, 0, 3216, 3921, 5, 0, 0, 2955, 0, 0, 14, 0, 956, 0, 0, 3925, 4050, 0, 0, 4096, 4, 46, 4096, 0, 0, 4096, 12, 0, 0, 4081, 0, 123, 3154, 0, 0, 2745, 0, 0, 4096, 0, 2458, 4096, 0, 0, 0, 0, 0, 0, 1895, 1, 130, 0, 0, 0, 2549, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 4080, 1, 8, 0, 15, 4096, 0, 4092, 0, 0, 0, 4096, 4096, 0, 0, 20, 0, 284, 1, 0, 4072, 168, 1, 499, 0, 0, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 20, 4064, 0, 8, 5, 0, 0, 321, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 4096, 0, 3002, 0, 0, 4066, 0, 4092, 4096, 1, 0, 0, 4092, 0, 2915, 2261, 9, 5, 0, 2258, 0, 0, 0, 0, 550, 0, 120, 4066, 4069, 0, 0, 4096, 0, 432, 4096, 0, 0, 4092, 61, 0, 0, 4096, 0, 15, 3303, 0, 1, 3621, 14, 0, 4095, 0, 2135, 4096, 0, 0, 0, 0, 0, 2, 3591, 0, 9, 0, 0, 0, 2648, 0, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 4095, 3817, 159, 0, 0, 46, 4095, 0, 4094, 0, 0, 0, 4094, 4096, 0, 0, 55, 0, 174, 4, 0, 4089, 35, 0, 1335, 0, 2, 0, 4082] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 4079, 0, 144, 0, 0, 0, 232, 0, 0, 2, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 3750, 6, 0, 4078, 1, 4093, 4093, 7, 0, 0, 4096, 0, 2477, 3802, 13, 2, 0, 3159, 0, 0, 0, 0, 89, 0, 43, 4088, 4080, 0, 0, 4094, 5, 30, 4096, 0, 0, 4093, 20, 1, 0, 4092, 0, 5, 1380, 0, 0, 2956, 0, 0, 4093, 0, 3844, 4096, 0, 0, 0, 0, 0, 0, 3335, 0, 0, 0, 0, 1, 1794, 0, 0, 1, 0, 0, 4096, 0, 4, 0, 0, 0, 4096, 4060, 0, 14, 0, 16, 4095, 0, 4075, 2, 0, 0, 4096, 4096, 1, 0, 60, 1, 514, 3, 0, 4093, 258, 0, 836, 0, 0, 0, 4088] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4094, 0, 62, 3, 1, 0, 203, 6, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4071, 3, 3845, 2, 0, 4086, 0, 4090, 4096, 9, 0, 1, 4094, 0, 2084, 4051, 3, 0, 0, 3814, 1, 0, 4, 0, 406, 0, 1, 4054, 4077, 0, 0, 4096, 1, 56, 4096, 0, 0, 4094, 65, 0, 0, 4096, 0, 60, 1540, 0, 0, 1425, 7, 0, 4094, 0, 3844, 4096, 0, 0, 0, 0, 0, 0, 3141, 0, 7, 0, 0, 0, 2840, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 4096, 4027, 0, 1, 0, 3, 4095, 0, 4081, 0, 0, 0, 4096, 4096, 0, 0, 159, 0, 287, 1, 0, 4083, 182, 2, 765, 0, 1, 0, 4088] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 4090, 0, 183, 0, 0, 0, 313, 5, 0, 0, 0, 4092, 4096, 3686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 1, 3760, 0, 0, 4064, 0, 4078, 570, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2516, 3289, 0, 4, 0, 3811, 0, 0, 2, 0, 775, 0, 1, 2198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4072, 0, 0, 0, 7, 48, 0, 0, 0, 4023, 18, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2450, 0, 0, 1237, 2, 0, 4096, 0, 3650, 4096, 0, 0, 0, 0, 0, 301, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3364, 0, 11, 0, 0, 0, 2598, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 2073, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4026, 2, 10, 0, 5, 4096, 0, 4092, 4, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 1, 0, 36, 0, 0, 4089, 161, 0, 602, 0, 0, 0, 4091, 1940, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16470, 16560, 16691, 16336, 15907, 16650, 16414, 16044] avg=16384.0000 max=16691 max_rank=2 min=15907 min_rank=4 imbalance_ratio=1.018738 max_load_reduction=114381 max_load_reduction_pct=0.872658 moved_to_echo=65015 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4085, 1, 11, 2, 0, 0, 85, 0, 0, 0, 1, 4095, 4096, 3686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3614, 0, 0, 4038, 0, 4096, 570, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3181, 3388, 18, 3, 0, 3781, 0, 0, 0, 0, 257, 0, 2, 2197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4078, 0, 0, 0, 0, 89, 0, 0, 0, 4024, 144, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 2393, 0, 1, 3132, 0, 0, 4094, 2, 2630, 4096, 0, 0, 0, 0, 1, 301, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2499, 2, 0, 0, 0, 1, 2675, 0, 0, 3, 1, 0, 4094, 0, 0, 4094, 2073, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 3926, 18, 18, 0, 33, 4095, 0, 4096, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 6, 0, 320, 0, 0, 4082, 54, 0, 545, 0, 0, 0, 4090, 1940, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16063, 16437, 16922, 16525, 16747, 15856, 16364, 16158] avg=16384.0000 max=16922 max_rank=2 min=15856 min_rank=5 imbalance_ratio=1.032837 max_load_reduction=114150 max_load_reduction_pct=0.870895 moved_to_echo=65125 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 50, 4055, 0, 459, 0, 0, 0, 290, 0, 0, 0, 0, 4096, 4096, 3686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 3055, 0, 0, 4095, 0, 4073, 570, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3216, 3921, 5, 0, 0, 2955, 0, 0, 14, 0, 956, 0, 0, 2198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3925, 4050, 0, 0, 0, 4, 46, 0, 0, 0, 4025, 12, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 3154, 0, 0, 2745, 0, 0, 4096, 0, 2458, 4096, 0, 0, 0, 0, 0, 301, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1895, 1, 130, 0, 0, 0, 2549, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 2073, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 1, 8, 0, 15, 4096, 0, 4092, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 0, 284, 1, 0, 4072, 168, 1, 499, 0, 0, 0, 4093, 1940, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16732, 15869, 17361, 16143, 17044, 15251, 16471, 16201] avg=16384.0000 max=17361 max_rank=2 min=15251 min_rank=5 imbalance_ratio=1.059631 max_load_reduction=113711 max_load_reduction_pct=0.867546 moved_to_echo=64967 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 32, 4086, 0, 0, 0, 0, 0, 161, 0, 1, 3, 6, 4091, 4090, 3682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4093, 0, 4028, 8, 0, 3769, 2, 4087, 560, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4090, 1, 2489, 3351, 18, 0, 4, 3573, 0, 5, 1, 5, 255, 5, 260, 2197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4058, 5, 4, 0, 0, 14, 0, 5, 5, 4017, 62, 0, 5, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1910, 0, 0, 3238, 3, 1, 4091, 7, 1752, 4091, 0, 5, 0, 0, 0, 304, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3482, 0, 0, 1, 0, 0, 3456, 4, 5, 2, 0, 5, 4089, 0, 5, 4091, 2073, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 4091, 3895, 15, 2, 4, 1, 4090, 5, 4086, 0, 1, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 5, 568, 0, 100, 0, 4, 4089, 0, 0, 440, 1, 0, 0, 4080, 1944, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16155, 16562, 16257, 16362, 15489, 17624, 16270, 16353] avg=16384.0000 max=17624 max_rank=5 min=15489 min_rank=4 imbalance_ratio=1.075684 max_load_reduction=113448 max_load_reduction_pct=0.865540 moved_to_echo=65736 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 20, 4072, 3, 304, 0, 0, 0, 492, 8, 0, 0, 0, 4087, 4096, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 4090, 2, 3674, 0, 0, 4035, 0, 4094, 570, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2655, 3755, 2, 0, 0, 3503, 1, 0, 1, 0, 689, 0, 1, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 4087, 0, 0, 0, 8, 15, 0, 2, 0, 4023, 3, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1719, 0, 2, 1948, 4, 0, 4093, 0, 3806, 4084, 0, 0, 3, 5, 1, 301, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3255, 0, 9, 0, 0, 0, 2361, 0, 0, 0, 0, 0, 4094, 3, 0, 4094, 2072, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 3978, 21, 8, 0, 3, 4093, 2, 4052, 0, 0, 3, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 11, 0, 164, 2, 3, 4079, 159, 0, 359, 0, 0, 0, 4096, 1939, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16766, 16542, 16899, 16249, 16051, 16297, 16337, 15931] avg=16384.0000 max=16899 max_rank=2 min=15931 min_rank=7 imbalance_ratio=1.031433 max_load_reduction=114173 max_load_reduction_pct=0.871071 moved_to_echo=64616 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4094, 0, 62, 3, 1, 0, 203, 6, 0, 0, 0, 4096, 4096, 3686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 3, 3845, 2, 0, 4086, 0, 4090, 570, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 2084, 4051, 3, 0, 0, 3814, 1, 0, 4, 0, 406, 0, 1, 2198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 4077, 0, 0, 0, 1, 56, 0, 0, 0, 4023, 65, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 1540, 0, 0, 1425, 7, 0, 4094, 0, 3844, 4096, 0, 0, 0, 0, 0, 301, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3141, 0, 7, 0, 0, 0, 2840, 0, 0, 0, 0, 0, 4095, 0, 2, 4096, 2073, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4027, 0, 1, 0, 3, 4095, 0, 4081, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 159, 0, 287, 1, 0, 4083, 182, 2, 765, 0, 1, 0, 4088, 1940, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16248, 16676, 16657, 16372, 15438, 16664, 16386, 16631] avg=16384.0000 max=16676 max_rank=1 min=15438 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=114396 max_load_reduction_pct=0.872772 moved_to_echo=65119 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 20, 4064, 0, 8, 5, 0, 0, 321, 0, 0, 0, 0, 4091, 4096, 3686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3002, 0, 0, 4066, 0, 4092, 570, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2915, 2261, 9, 5, 0, 2258, 0, 0, 0, 0, 550, 0, 120, 2198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4069, 0, 0, 0, 0, 432, 0, 0, 0, 4022, 61, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3303, 0, 1, 3621, 14, 0, 4095, 0, 2135, 4096, 0, 0, 0, 0, 0, 301, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3591, 0, 9, 0, 0, 0, 2648, 0, 0, 0, 1, 0, 4093, 0, 0, 4096, 2072, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3817, 159, 0, 0, 46, 4095, 0, 4094, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 55, 0, 174, 4, 0, 4089, 35, 0, 1335, 0, 2, 0, 4082, 1939, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16291, 15827, 14408, 16746, 17651, 16922, 16389, 16838] avg=16384.0000 max=17651 max_rank=4 min=14408 min_rank=2 imbalance_ratio=1.077332 max_load_reduction=113421 max_load_reduction_pct=0.865334 moved_to_echo=67800 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 4079, 0, 144, 0, 0, 0, 232, 0, 0, 2, 0, 4096, 4096, 3686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3750, 6, 0, 4078, 1, 4093, 570, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2477, 3802, 13, 2, 0, 3159, 0, 0, 0, 0, 89, 0, 43, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4080, 0, 0, 0, 5, 30, 0, 0, 0, 4023, 20, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1380, 0, 0, 2956, 0, 0, 4093, 0, 3844, 4096, 0, 0, 0, 0, 0, 301, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3335, 0, 0, 0, 0, 1, 1794, 0, 0, 1, 0, 0, 4096, 0, 4, 4094, 2073, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4060, 0, 14, 0, 16, 4095, 0, 4075, 2, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 60, 1, 514, 3, 0, 4093, 258, 0, 836, 0, 0, 0, 4088, 1940, 1026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16347, 16599, 15877, 16339, 16745, 15808, 16441, 16916] avg=16384.0000 max=16916 max_rank=7 min=15808 min_rank=5 imbalance_ratio=1.032471 max_load_reduction=114156 max_load_reduction_pct=0.870941 moved_to_echo=65910 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[384, 17920, 0, 128, 20352, 128, 128, 32768, 128, 24192, 32768, 0, 128, 128, 128, 128, 2432, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32512, 32640, 128, 128, 0, 128, 768, 0, 128, 128, 32256, 512, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 128, 896, 128, 1920, 128, 128, 32768, 1024, 128, 5504, 128, 128, 0, 32768, 15616, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 24576, 128, 256, 128, 0, 128, 20992, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 16640, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 32768, 128, 21632, 27904, 128, 128, 128, 26880, 128, 128, 128, 128, 4096, 128, 512, 17664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 256, 32640, 128, 1280, 128, 128, 0, 2176, 128, 128, 128, 128, 32768, 32768, 29568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 32768, 31872, 256, 128, 128, 128, 32768, 128, 32768, 128, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 0, 128, 32768, 128, 28800, 128, 0, 32256, 128, 32768, 4608, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3101, 0, 987, 0, 0, 495, 0, 4096, 1, 0, 0, 1, 0, 4096, 0, 0, 0, 60, 4026, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 1422, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 5, 0, 0, 4087, 4092, 1546, 0, 0, 1719, 0, 4096, 4096, 0, 0, 4093, 3967, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4044, 4096, 0, 0, 118, 4078, 0, 251, 0, 730, 0, 0, 3937, 0, 0, 0, 3086, 0, 16, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 3267, 0, 4094, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3709, 0, 1391, 0, 0, 174, 0, 4094, 34, 0, 0, 0, 0, 4094, 0, 0, 0, 6, 3826, 0, 4095, 0, 4096, 4094, 0, 0, 0, 4094, 1985, 0, 2, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 4096, 0, 10, 0, 0, 4096, 4096, 3355, 0, 0, 735, 1, 4096, 4094, 0, 0, 4079, 4081, 0, 0, 2, 4095, 0, 0, 4093, 6, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 18, 4091, 4088, 0, 0, 16, 4052, 0, 115, 0, 126, 0, 0, 4075, 0, 0, 0, 2432, 0, 3, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 25, 4093, 0, 0, 2, 0, 0, 0, 4096, 0, 2618, 0, 4096, 4, 0, 2, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2261, 0, 700, 0, 0, 1069, 0, 4096, 334, 0, 0, 0, 0, 4096, 0, 0, 0, 22, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 2329, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 1868, 0, 0, 891, 0, 4096, 4095, 0, 0, 4095, 4086, 0, 0, 0, 4096, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4, 2, 4083, 4095, 0, 0, 89, 4049, 0, 118, 0, 284, 0, 0, 4061, 0, 0, 0, 2708, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 59, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 3756, 0, 4096, 1, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2575, 0, 1032, 0, 0, 577, 0, 4092, 200, 3, 0, 0, 0, 4093, 0, 0, 0, 27, 3962, 0, 4093, 0, 4095, 4093, 0, 0, 0, 4093, 1909, 0, 1, 1, 4096, 3, 0, 4094, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4095, 4090, 2493, 0, 0, 1503, 3, 4096, 4093, 0, 0, 4079, 4060, 0, 0, 4, 4096, 0, 0, 4091, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 14, 4088, 4096, 0, 0, 52, 3873, 1, 147, 0, 234, 2, 0, 4036, 0, 0, 0, 2141, 1, 1, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4, 0, 0, 55, 4093, 0, 0, 4, 0, 0, 0, 4094, 0, 3893, 0, 4092, 5, 0, 6, 0, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 2103, 0, 96, 5, 5, 360, 2, 4088, 131, 0, 5, 2, 2, 4091, 0, 5, 0, 71, 4074, 4, 4091, 5, 4090, 4091, 0, 0, 6, 4089, 1998, 0, 9, 0, 4091, 0, 0, 4091, 3, 6, 5, 3, 4091, 6, 0, 5, 0, 4090, 4092, 493, 5, 0, 2698, 0, 4091, 4091, 0, 5, 4089, 4082, 0, 5, 0, 4088, 0, 5, 4090, 0, 1, 5, 0, 0, 0, 4091, 0, 5, 4089, 0, 0, 4086, 4069, 0, 5, 70, 4089, 0, 360, 5, 1368, 0, 0, 4054, 5, 3, 0, 3740, 0, 21, 4, 5, 7, 0, 4068, 0, 6, 2, 0, 0, 0, 0, 77, 4091, 5, 5, 4, 0, 0, 5, 4085, 5, 2895, 0, 4091, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 3152, 0, 574, 0, 0, 447, 0, 4095, 199, 1, 0, 0, 0, 4094, 0, 0, 0, 13, 4086, 0, 4095, 0, 4096, 4095, 0, 0, 1, 4094, 1059, 0, 0, 1, 4096, 5, 0, 4095, 1, 0, 0, 0, 4096, 0, 2, 0, 0, 4092, 4091, 2587, 0, 0, 1537, 1, 4096, 4095, 1, 0, 4064, 4092, 0, 0, 1, 4096, 0, 0, 4095, 4, 0, 0, 0, 1, 2, 4096, 0, 0, 4095, 0, 5, 4034, 4093, 0, 0, 31, 4027, 0, 229, 0, 424, 0, 0, 4080, 0, 0, 0, 3623, 0, 1, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 4095, 0, 0, 1, 0, 0, 0, 4096, 0, 2674, 0, 4095, 2, 0, 1, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1972, 0, 156, 0, 0, 270, 0, 4096, 246, 0, 0, 0, 0, 4096, 0, 0, 0, 433, 3956, 0, 4091, 0, 4096, 4092, 0, 0, 3, 4091, 951, 0, 0, 1, 4096, 9, 0, 4096, 19, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 1223, 0, 0, 1938, 0, 4096, 4090, 0, 0, 4078, 4072, 0, 0, 4, 4091, 0, 0, 4095, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 17, 4037, 4084, 0, 0, 27, 4066, 0, 799, 0, 1606, 1, 0, 4050, 0, 0, 0, 3932, 3, 10, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 18, 4096, 0, 0, 5, 0, 0, 0, 4091, 0, 3074, 0, 4092, 12, 0, 5, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3037, 0, 800, 0, 0, 623, 0, 4093, 14, 0, 0, 0, 0, 4094, 0, 0, 0, 151, 4045, 0, 4095, 0, 4096, 4096, 0, 0, 0, 4095, 2345, 0, 1, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4092, 4093, 1937, 0, 0, 1081, 0, 4096, 4096, 0, 0, 4087, 4085, 0, 0, 2, 4095, 0, 0, 4096, 5, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4, 2, 4074, 4095, 0, 0, 14, 4065, 0, 2, 0, 161, 2, 0, 4076, 0, 0, 0, 2822, 0, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 3, 4095, 0, 0, 1, 0, 0, 0, 4094, 0, 3509, 0, 4095, 8, 0, 1, 0, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2261, 0, 700, 0, 0, 1069, 0, 4096, 334, 0, 0, 0, 0, 4096, 2568, 1522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4096, 0, 4096, 0, 0, 2339, 0, 0, 0, 4096, 2329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1868, 0, 0, 891, 0, 0, 689, 0, 0, 4095, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4, 2, 4083, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 89, 4049, 0, 118, 0, 284, 0, 0, 4061, 0, 0, 0, 2708, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 59, 4096, 4096, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3756, 0, 4096, 1, 0, 0, 0, 1, 4096, 838, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16646, 16978, 16376, 15723, 16400, 15630, 16395, 16924] avg=16384.0000 max=16978 max_rank=1 min=15630 min_rank=5 imbalance_ratio=1.036255 max_load_reduction=114094 max_load_reduction_pct=0.870468 moved_to_echo=65349 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3152, 0, 574, 0, 0, 447, 0, 4095, 199, 1, 0, 0, 0, 4094, 2568, 1521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4086, 0, 4095, 0, 0, 2339, 0, 0, 1, 4094, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 5, 0, 4095, 1, 0, 0, 0, 4096, 0, 2, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2587, 0, 0, 1537, 1, 0, 689, 1, 0, 4064, 4092, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 1, 2, 4096, 0, 0, 4095, 0, 5, 4034, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 31, 4027, 0, 229, 0, 424, 0, 0, 4080, 0, 0, 0, 3623, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 4095, 4096, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 2674, 0, 4095, 2, 0, 1, 0, 5, 4096, 838, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16652, 15687, 16380, 17063, 16349, 16733, 16360, 15848] avg=16384.0000 max=17063 max_rank=3 min=15687 min_rank=1 imbalance_ratio=1.041443 max_load_reduction=114009 max_load_reduction_pct=0.869820 moved_to_echo=65290 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3101, 0, 987, 0, 0, 495, 0, 4096, 1, 0, 0, 1, 0, 4096, 2569, 1522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 4026, 0, 4096, 0, 0, 2339, 0, 0, 0, 4096, 1422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 5, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1546, 0, 0, 1719, 0, 0, 689, 0, 0, 4093, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4044, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 118, 4078, 0, 251, 0, 730, 0, 0, 3937, 0, 0, 0, 3086, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 4096, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3267, 0, 4094, 0, 0, 0, 0, 0, 4095, 838, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16868, 16039, 16373, 16106, 16350, 16522, 16384, 16430] avg=16384.0000 max=16868 max_rank=0 min=16039 min_rank=1 imbalance_ratio=1.029541 max_load_reduction=114204 max_load_reduction_pct=0.871307 moved_to_echo=65686 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2575, 0, 1032, 0, 0, 577, 0, 4092, 200, 3, 0, 0, 0, 4093, 2567, 1521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 3962, 0, 4093, 0, 0, 2337, 0, 0, 0, 4093, 1909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4088, 3, 0, 4094, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2493, 0, 0, 1503, 3, 0, 689, 0, 0, 4079, 4060, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 14, 4088, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 52, 3873, 1, 147, 0, 234, 2, 0, 4036, 0, 0, 0, 2141, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4, 0, 0, 55, 4093, 4096, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4094, 0, 3893, 0, 4092, 5, 0, 6, 0, 11, 4096, 837, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16660, 16421, 16380, 16921, 16407, 14808, 16397, 17078] avg=16384.0000 max=17078 max_rank=7 min=14808 min_rank=5 imbalance_ratio=1.042358 max_load_reduction=113994 max_load_reduction_pct=0.869705 moved_to_echo=64690 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3037, 0, 800, 0, 0, 623, 0, 4093, 14, 0, 0, 0, 0, 4094, 2569, 1522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 4045, 0, 4095, 0, 0, 2339, 0, 0, 0, 4095, 2345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4088, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1937, 0, 0, 1081, 0, 0, 689, 0, 0, 4087, 4085, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4, 2, 4074, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 14, 4065, 0, 2, 0, 161, 2, 0, 4076, 0, 0, 0, 2822, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 3, 4095, 4096, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 3509, 0, 4095, 8, 0, 1, 0, 7, 4095, 838, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16752, 17070, 16376, 15974, 16391, 15463, 16358, 16688] avg=16384.0000 max=17070 max_rank=1 min=15463 min_rank=5 imbalance_ratio=1.041870 max_load_reduction=114002 max_load_reduction_pct=0.869766 moved_to_echo=64900 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3709, 0, 1391, 0, 0, 174, 0, 4094, 34, 0, 0, 0, 0, 4094, 2568, 1521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3826, 0, 4095, 0, 0, 2338, 0, 0, 0, 4094, 1985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4088, 0, 0, 4096, 3, 0, 0, 0, 4096, 0, 10, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3355, 0, 0, 735, 1, 0, 689, 0, 0, 4079, 4081, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 6, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 18, 4091, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 16, 4052, 0, 115, 0, 126, 0, 0, 4075, 0, 0, 0, 2432, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 25, 4093, 4096, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 2618, 0, 4096, 4, 0, 2, 0, 3, 4095, 837, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17585, 16344, 16391, 17038, 16418, 15130, 16372, 15794] avg=16384.0000 max=17585 max_rank=0 min=15130 min_rank=5 imbalance_ratio=1.073303 max_load_reduction=113487 max_load_reduction_pct=0.865837 moved_to_echo=63714 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1972, 0, 156, 0, 0, 270, 0, 4096, 246, 0, 0, 0, 0, 4096, 2565, 1520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 3956, 0, 4091, 0, 0, 2337, 0, 0, 3, 4091, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 9, 0, 4096, 19, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1223, 0, 0, 1938, 0, 0, 689, 0, 0, 4078, 4072, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 17, 4037, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 27, 4066, 0, 799, 0, 1606, 1, 0, 4050, 0, 0, 0, 3932, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 10, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 18, 4096, 4096, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4091, 0, 3074, 0, 4092, 12, 0, 5, 0, 1, 4091, 836, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14921, 15862, 16405, 16098, 16364, 18791, 16384, 16247] avg=16384.0000 max=18791 max_rank=5 min=14921 min_rank=0 imbalance_ratio=1.146912 max_load_reduction=112281 max_load_reduction_pct=0.856636 moved_to_echo=67786 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 2103, 0, 96, 5, 5, 360, 2, 4088, 131, 0, 5, 2, 2, 4091, 2571, 1524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 71, 4074, 4, 4091, 5, 0, 2328, 0, 0, 6, 4089, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4082, 0, 0, 4091, 3, 6, 5, 3, 4091, 6, 0, 5, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 493, 5, 0, 2698, 0, 0, 680, 0, 5, 4089, 4082, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4090, 0, 1, 5, 0, 0, 0, 4091, 0, 5, 4089, 0, 0, 4086, 12, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 5, 70, 4089, 0, 360, 5, 1368, 0, 0, 4054, 5, 3, 0, 3740, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4, 5, 7, 0, 4068, 0, 6, 2, 0, 0, 0, 0, 77, 4091, 4091, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 4, 0, 0, 5, 4085, 5, 2895, 0, 4091, 0, 0, 0, 0, 0, 4088, 840, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14988, 16671, 16391, 16149, 16393, 17995, 16422, 16063] avg=16384.0000 max=17995 max_rank=5 min=14988 min_rank=0 imbalance_ratio=1.098328 max_load_reduction=113077 max_load_reduction_pct=0.862709 moved_to_echo=66873 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 0, 128, 512, 32384, 128, 2048, 128, 4992, 128, 0, 32384, 128, 128, 0, 24576, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 0, 128, 128, 0, 128, 0, 0, 384, 32768, 32768, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 896, 32128, 128, 32768, 128, 0, 18816, 0, 0, 128, 32768, 14080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 15616, 128, 0, 12160, 128, 0, 5504, 128, 128, 32768, 32640, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 32768, 128, 0, 32768, 128, 128, 128, 128, 32768, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 0, 128, 128, 32768, 0, 128, 32768, 128, 128, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 22016, 0, 5760, 128, 128, 4096, 128, 32768, 1280, 128, 128, 128, 128, 32768, 20608, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 0, 0, 128, 32768, 128, 25728, 0, 32768, 128, 0, 128, 0, 128, 32768, 6784, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 4090, 3, 0, 126, 4094, 4096, 0, 0, 0, 4096, 0, 4019, 0, 0, 4093, 0, 0, 9, 4091, 0, 115, 4095, 0, 0, 4091, 0, 0, 4096, 3956, 0, 0, 0, 0, 4092, 3, 143, 4096, 39, 4069, 0, 0, 3, 0, 4096, 4096, 3487, 0, 0, 4096, 4082, 4, 0, 0, 0, 6, 38, 9, 0, 4096, 0, 0, 0, 2, 4090, 0, 0, 4093, 4076, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 4091, 4093, 0, 0, 0, 0, 609, 0, 0, 42, 0, 0, 0, 4026, 0, 0, 2771, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 188, 4096, 0, 0, 0, 973, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 4096, 0, 0, 30, 4094, 4096, 0, 0, 0, 4096, 0, 4043, 0, 0, 4052, 0, 0, 5, 4091, 0, 196, 4096, 0, 0, 4093, 0, 0, 4095, 3907, 0, 0, 0, 0, 4096, 2, 353, 4096, 14, 3942, 0, 0, 4, 0, 4096, 4096, 3353, 0, 0, 4096, 4086, 0, 0, 0, 0, 1, 15, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 4094, 0, 0, 0, 0, 772, 0, 0, 22, 0, 0, 0, 4067, 0, 0, 3059, 0, 0, 1, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 832, 0, 0, 0, 0, 0, 4095, 1, 30, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 4095, 1, 0, 4, 4096, 4096, 0, 0, 0, 4095, 0, 4051, 0, 0, 4048, 0, 0, 1, 4095, 0, 396, 4096, 0, 0, 4096, 0, 0, 4096, 4025, 0, 0, 0, 0, 4095, 0, 1079, 4096, 15, 4092, 0, 0, 26, 0, 4096, 4096, 2974, 0, 1, 4096, 4085, 0, 0, 0, 0, 1, 20, 1, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 4090, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4094, 4095, 0, 0, 0, 0, 799, 0, 0, 8, 0, 0, 0, 4063, 0, 0, 2305, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 65, 4096, 0, 0, 0, 650, 0, 0, 0, 0, 0, 4095, 32, 37, 0, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 4095, 0, 0, 12, 4095, 4096, 0, 0, 0, 4096, 0, 4081, 1, 0, 4095, 0, 0, 0, 4095, 0, 218, 4091, 0, 0, 4094, 0, 0, 4096, 4002, 0, 0, 0, 0, 4096, 1, 381, 4096, 2, 4090, 0, 0, 14, 0, 4096, 4096, 2913, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 4092, 0, 0, 0, 0, 1, 2, 0, 4096, 0, 4093, 4095, 0, 0, 0, 0, 497, 0, 0, 8, 0, 0, 1, 4081, 0, 0, 3495, 0, 0, 0, 0, 1, 0, 0, 4096, 1, 0, 0, 0, 19, 4096, 0, 0, 0, 738, 0, 0, 0, 0, 0, 4096, 18, 25, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 24, 4096, 4096, 0, 0, 0, 4095, 0, 4070, 15, 0, 4083, 0, 0, 2, 4096, 0, 829, 4094, 0, 0, 4096, 0, 0, 4095, 4005, 0, 1, 0, 0, 4096, 0, 858, 4096, 16, 4092, 0, 0, 0, 0, 4096, 4096, 2100, 0, 0, 4096, 4075, 0, 0, 0, 0, 1, 3, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 1, 0, 680, 0, 0, 58, 0, 0, 21, 4068, 0, 0, 2261, 0, 0, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 1404, 0, 0, 0, 0, 0, 4096, 11, 86, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[5, 6, 6, 5, 0, 4091, 0, 0, 2, 4091, 4091, 0, 1, 1, 4080, 5, 4085, 2, 5, 3992, 5, 1, 0, 4091, 0, 233, 4090, 0, 5, 4091, 0, 0, 4090, 4023, 0, 5, 6, 0, 4091, 0, 911, 4090, 44, 4070, 1, 0, 53, 5, 4090, 4091, 3501, 5, 3, 4090, 4075, 0, 0, 5, 5, 0, 4, 0, 1, 4090, 5, 0, 0, 13, 4091, 5, 0, 4089, 4081, 5, 5, 5, 0, 0, 0, 0, 4087, 0, 4091, 4091, 0, 1, 7, 3, 67, 0, 0, 0, 0, 1, 0, 4049, 5, 6, 1779, 0, 5, 0, 0, 0, 5, 5, 4090, 1, 5, 0, 2, 144, 4089, 5, 0, 5, 542, 6, 0, 5, 0, 5, 4091, 48, 1108, 5, 4087, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4094, 0, 0, 1, 4096, 4096, 0, 0, 0, 4096, 0, 4069, 0, 0, 4039, 0, 0, 10, 4088, 0, 990, 4096, 0, 0, 4096, 0, 0, 4096, 3590, 0, 0, 0, 0, 4096, 0, 1580, 4096, 11, 4055, 0, 1, 0, 0, 4096, 4096, 1902, 0, 0, 4096, 4086, 1, 0, 0, 0, 5, 14, 5, 0, 4096, 0, 0, 0, 2, 4093, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 4096, 0, 0, 0, 0, 862, 0, 0, 8, 0, 0, 2, 4071, 0, 0, 847, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 49, 4096, 0, 0, 0, 2247, 0, 0, 0, 0, 0, 4096, 21, 331, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 44, 4096, 4096, 0, 0, 0, 4096, 0, 4075, 0, 0, 4080, 0, 0, 0, 4096, 0, 421, 4093, 0, 0, 4096, 0, 0, 4096, 3924, 0, 0, 0, 0, 4096, 0, 1161, 4096, 17, 4093, 0, 0, 20, 0, 4096, 4096, 2700, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 3, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 863, 0, 0, 5, 0, 0, 0, 4077, 0, 0, 2266, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 43, 4096, 0, 0, 0, 870, 0, 0, 0, 0, 0, 4096, 19, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 4095, 1, 0, 4, 4096, 4065, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 4048, 0, 0, 1, 4095, 0, 396, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3453, 4025, 0, 0, 0, 0, 4095, 0, 1079, 0, 15, 4092, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1225, 0, 2974, 0, 1, 4096, 4085, 0, 0, 0, 0, 1, 20, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4090, 0, 0, 0, 0, 1, 0, 0, 2415, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 4094, 4095, 0, 0, 0, 0, 799, 0, 0, 8, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2305, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 65, 4096, 4096, 1681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 650, 0, 0, 0, 0, 0, 4095, 32, 37, 0, 4093, 0, 1183, 643, 628, 357, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16361, 16330, 16785, 16499, 16385, 16527, 16340, 15845] avg=16384.0000 max=16785 max_rank=2 min=15845 min_rank=7 imbalance_ratio=1.024475 max_load_reduction=114287 max_load_reduction_pct=0.871941 moved_to_echo=65097 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 4095, 0, 0, 12, 4095, 4065, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 1, 0, 4095, 0, 0, 0, 4095, 0, 218, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3453, 4002, 0, 0, 0, 0, 4096, 1, 381, 0, 2, 4090, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1225, 0, 2913, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4092, 0, 0, 0, 0, 1, 2, 0, 2414, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 4093, 4095, 0, 0, 0, 0, 497, 0, 0, 8, 0, 0, 1, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3495, 0, 0, 0, 0, 1, 0, 0, 4096, 1, 0, 0, 0, 19, 4096, 4096, 1680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 738, 0, 0, 0, 0, 0, 4096, 18, 25, 0, 4096, 0, 1183, 643, 628, 356, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16367, 16225, 16039, 16416, 16388, 16243, 17484, 15910] avg=16384.0000 max=17484 max_rank=6 min=15910 min_rank=7 imbalance_ratio=1.067139 max_load_reduction=113588 max_load_reduction_pct=0.866608 moved_to_echo=66025 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 24, 4096, 4065, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 15, 0, 4083, 0, 0, 2, 4096, 0, 829, 3737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3452, 4005, 0, 1, 0, 0, 4096, 0, 858, 0, 16, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1225, 0, 2100, 0, 0, 4096, 4075, 0, 0, 0, 0, 1, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 2416, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3467, 0, 4096, 4096, 0, 0, 1, 0, 680, 0, 0, 58, 0, 0, 21, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2261, 0, 0, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4096, 4096, 1680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1404, 0, 0, 0, 0, 0, 4096, 11, 86, 0, 4095, 0, 1183, 643, 627, 357, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16376, 16832, 16520, 15596, 16393, 16487, 16239, 16629] avg=16384.0000 max=16832 max_rank=1 min=15596 min_rank=3 imbalance_ratio=1.027344 max_load_reduction=114240 max_load_reduction_pct=0.871582 moved_to_echo=65748 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 4090, 3, 0, 126, 4094, 4065, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 4093, 0, 0, 9, 4091, 0, 115, 3738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3453, 3956, 0, 0, 0, 0, 4092, 3, 143, 0, 39, 4069, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1225, 0, 3487, 0, 0, 4096, 4082, 4, 0, 0, 0, 6, 38, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4090, 0, 0, 4093, 4076, 0, 0, 0, 0, 5, 0, 0, 2413, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 4091, 4093, 0, 0, 0, 0, 609, 0, 0, 42, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2771, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 188, 4096, 4096, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 973, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 4094, 0, 1183, 643, 628, 357, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16480, 16065, 15758, 17043, 16367, 16329, 16929, 16101] avg=16384.0000 max=17043 max_rank=3 min=15758 min_rank=2 imbalance_ratio=1.040222 max_load_reduction=114029 max_load_reduction_pct=0.869972 moved_to_echo=65726 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 4096, 0, 0, 30, 4094, 4065, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 4052, 0, 0, 5, 4091, 0, 196, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3452, 3907, 0, 0, 0, 0, 4096, 2, 353, 0, 14, 3942, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1225, 0, 3353, 0, 0, 4096, 4086, 0, 0, 0, 0, 1, 15, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 2413, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 4094, 4094, 0, 0, 0, 0, 772, 0, 0, 22, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3059, 0, 0, 1, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 4096, 1680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 832, 0, 0, 0, 0, 0, 4095, 1, 30, 0, 4095, 0, 1183, 643, 628, 357, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16126, 15770, 16872, 16378, 16517, 17030, 15991] avg=16384.0000 max=17030 max_rank=6 min=15770 min_rank=2 imbalance_ratio=1.039429 max_load_reduction=114042 max_load_reduction_pct=0.870071 moved_to_echo=65916 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4094, 0, 0, 1, 4096, 4065, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 4039, 0, 0, 10, 4088, 0, 990, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3453, 3590, 0, 0, 0, 0, 4096, 0, 1580, 0, 11, 4055, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1225, 0, 1902, 0, 0, 4096, 4086, 1, 0, 0, 0, 5, 14, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 2415, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 4090, 4096, 0, 0, 0, 0, 862, 0, 0, 8, 0, 0, 2, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 847, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 49, 4096, 4096, 1681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2247, 0, 0, 0, 0, 0, 4096, 21, 331, 0, 4096, 0, 1183, 643, 628, 357, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16353, 16935, 16786, 15430, 16377, 16597, 14865, 17729] avg=16384.0000 max=17729 max_rank=7 min=14865 min_rank=6 imbalance_ratio=1.082092 max_load_reduction=113343 max_load_reduction_pct=0.864738 moved_to_echo=65568 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[5, 6, 6, 5, 0, 4091, 0, 0, 2, 4091, 4057, 0, 1, 1, 4080, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 2, 5, 3992, 5, 1, 0, 4091, 0, 233, 3731, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3441, 4023, 0, 5, 6, 0, 4091, 0, 911, 0, 44, 4070, 1, 0, 53, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1222, 0, 3501, 5, 3, 4090, 4075, 0, 0, 5, 5, 0, 4, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 13, 4091, 5, 0, 4089, 4081, 5, 5, 5, 0, 0, 0, 0, 2413, 1686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3458, 0, 4091, 4091, 0, 1, 7, 3, 67, 0, 0, 0, 0, 1, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 1779, 0, 5, 0, 0, 0, 5, 5, 4090, 1, 5, 0, 2, 144, 4091, 4090, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 0, 5, 542, 6, 0, 5, 0, 5, 4091, 48, 1108, 5, 4087, 0, 1182, 649, 629, 359, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16350, 16150, 16650, 17001, 16398, 15768, 15906, 16849] avg=16384.0000 max=17001 max_rank=3 min=15768 min_rank=5 imbalance_ratio=1.037659 max_load_reduction=114071 max_load_reduction_pct=0.870293 moved_to_echo=64921 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 44, 4096, 4065, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 4080, 0, 0, 0, 4096, 0, 421, 3737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3453, 3924, 0, 0, 0, 0, 4096, 0, 1161, 0, 17, 4093, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1225, 0, 2700, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 3, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 2415, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 4095, 4096, 0, 0, 0, 0, 863, 0, 0, 5, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2266, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 43, 4096, 4096, 1681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 870, 0, 0, 0, 0, 0, 4096, 19, 0, 0, 4096, 0, 1183, 643, 628, 356, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 16409, 16764, 16215, 16386, 16604, 16279, 16018] avg=16384.0000 max=16764 max_rank=2 min=16018 min_rank=7 imbalance_ratio=1.023193 max_load_reduction=114308 max_load_reduction_pct=0.872101 moved_to_echo=65287 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[27776, 0, 32768, 32768, 0, 128, 128, 128, 5248, 0, 0, 256, 0, 128, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[27648, 31488, 0, 128, 128, 0, 32768, 128, 6528, 0, 256, 32512, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 128, 0, 256, 32768, 32512, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32512, 128, 128, 32512, 128, 128, 128, 32768, 0, 3456, 29952, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 0, 128, 8320, 128, 0, 128, 0, 128, 32768, 256, 1664, 128, 32768, 0, 9472, 5248, 5120, 2944, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[9856, 0, 23040, 128, 128, 32768, 32768, 128, 0, 128, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 18816, 0, 128, 128, 0, 128, 128, 128, 32768, 128, 128, 0, 128, 640, 32768, 32768, 13440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 32768, 128, 0, 32768, 32768, 128, 128, 128, 0, 128, 128, 0, 19328, 13568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[14, 0, 4089, 0, 0, 4086, 4095, 0, 0, 0, 42, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 4094, 0, 4096, 0, 4096, 3897, 0, 0, 0, 1, 583, 768, 0, 0, 0, 4096, 0, 0, 0, 202, 0, 0, 0, 253, 4095, 0, 0, 0, 4096, 0, 0, 2, 4004, 4096, 1, 4066, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 4089, 0, 0, 4085, 4096, 4095, 0, 3925, 0, 4094, 0, 0, 125, 0, 0, 39, 0, 2624, 4095, 0, 0, 0, 27, 0, 4096, 0, 0, 4040, 12, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[8, 0, 4085, 0, 0, 4084, 4095, 0, 0, 0, 10, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4096, 4096, 0, 4096, 0, 4096, 3454, 0, 0, 0, 0, 1004, 627, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 162, 4096, 0, 0, 0, 4096, 0, 0, 3, 4064, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4089, 4096, 4096, 0, 3852, 0, 4096, 0, 0, 225, 0, 0, 25, 0, 3036, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 4040, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[41, 0, 4089, 0, 0, 4059, 4096, 0, 0, 0, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 4096, 0, 4096, 0, 4096, 4012, 0, 0, 0, 0, 608, 513, 0, 0, 3, 4096, 0, 0, 0, 57, 0, 0, 0, 521, 4096, 0, 0, 0, 4096, 0, 0, 1, 4009, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4093, 0, 0, 3945, 4096, 4096, 0, 3328, 0, 4096, 0, 0, 277, 0, 0, 23, 0, 3306, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 3961, 12, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[59, 5, 4091, 1, 0, 4060, 4054, 2, 0, 5, 36, 4, 4, 1, 5, 0, 0, 5, 0, 6, 0, 0, 5, 5, 0, 1, 0, 5, 5, 4091, 4090, 4091, 5, 4091, 5, 4091, 3726, 0, 0, 4, 0, 351, 958, 1, 0, 3, 4090, 1, 22, 5, 118, 1, 2, 0, 315, 4091, 6, 5, 0, 4090, 0, 0, 43, 3871, 4090, 0, 4091, 4087, 0, 1, 0, 5, 0, 0, 0, 5, 0, 14, 4, 4089, 0, 0, 5, 5, 5, 4079, 5, 0, 32, 2, 5, 4088, 5, 0, 4089, 4079, 1, 5, 4063, 4091, 4091, 6, 3794, 0, 4091, 2, 6, 626, 2, 0, 0, 4, 2416, 4091, 1, 2, 0, 126, 0, 4070, 0, 0, 4094, 0, 4090, 4, 0, 4085] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[61, 0, 4085, 0, 0, 4072, 4096, 0, 0, 0, 24, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 4095, 0, 4096, 0, 4096, 3881, 0, 0, 0, 1, 323, 880, 0, 0, 0, 4096, 0, 2, 0, 208, 0, 0, 0, 141, 4095, 0, 0, 0, 4096, 0, 0, 2, 4082, 4096, 1, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 0, 0, 4093, 4096, 4095, 0, 3840, 0, 4094, 0, 0, 270, 0, 0, 9, 0, 2773, 4095, 0, 0, 0, 1, 0, 4096, 0, 0, 4032, 9, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[47, 0, 4090, 0, 0, 4045, 4096, 0, 0, 0, 57, 4, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 4090, 0, 4096, 0, 4096, 3858, 0, 0, 0, 4, 396, 805, 0, 0, 3, 4096, 0, 0, 0, 296, 0, 0, 0, 154, 4093, 0, 0, 0, 4096, 0, 0, 7, 4005, 4096, 4, 4062, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4074, 0, 0, 4021, 4096, 4092, 0, 3576, 0, 4092, 0, 0, 419, 0, 0, 51, 0, 3017, 4093, 0, 0, 0, 2, 0, 4096, 0, 0, 3961, 29, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 4086, 0, 0, 4073, 4096, 0, 0, 0, 35, 9, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4096, 4095, 0, 4096, 0, 4096, 3941, 0, 0, 0, 1, 398, 532, 0, 0, 0, 4096, 0, 0, 0, 85, 0, 0, 0, 612, 4096, 0, 0, 0, 4096, 0, 0, 1, 4029, 4096, 5, 4082, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4053, 0, 0, 4041, 4096, 4089, 0, 4023, 0, 4096, 0, 0, 209, 0, 0, 35, 0, 2686, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 4022, 20, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[8, 0, 4078, 0, 0, 4067, 4087, 0, 0, 0, 34, 5, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4096, 4092, 0, 4096, 0, 4096, 3961, 0, 0, 0, 1, 536, 57, 0, 0, 1, 4096, 0, 0, 0, 363, 0, 0, 0, 163, 4096, 0, 0, 0, 4096, 0, 0, 2, 4053, 4096, 4, 4071, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4008, 0, 0, 4046, 4096, 4093, 0, 4003, 0, 4093, 0, 0, 341, 0, 0, 68, 0, 3072, 4095, 0, 0, 0, 16, 0, 4096, 0, 0, 4026, 15, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 4089, 0, 0, 4086, 4095, 0, 0, 0, 42, 1, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 4094, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3183, 0, 4096, 3897, 0, 0, 0, 1, 583, 768, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 0, 253, 4095, 0, 0, 0, 4096, 0, 0, 2, 4004, 2822, 884, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4066, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 4089, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 12, 4095, 0, 3925, 0, 4094, 0, 0, 125, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2624, 4095, 0, 0, 0, 27, 0, 4096, 0, 0, 4040, 12, 0, 0, 0, 1274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 16386, 16624, 16360, 16366, 16396, 16375, 16168] avg=16384.0000 max=16624 max_rank=2 min=16168 min_rank=7 imbalance_ratio=1.014648 max_load_reduction=114448 max_load_reduction_pct=0.873169 moved_to_echo=65305 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[41, 0, 4089, 0, 0, 4059, 4096, 0, 0, 0, 24, 4, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 4096, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3183, 0, 4096, 4012, 0, 0, 0, 0, 608, 513, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 521, 4096, 0, 0, 0, 4096, 0, 0, 1, 4009, 2822, 884, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4093, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 12, 4096, 0, 3328, 0, 4096, 0, 0, 277, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3306, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 3961, 12, 0, 0, 0, 1274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16378, 16511, 16488, 16395, 16397, 15777, 16743] avg=16384.0000 max=16743 max_rank=7 min=15777 min_rank=6 imbalance_ratio=1.021912 max_load_reduction=114329 max_load_reduction_pct=0.872261 moved_to_echo=65312 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 4085, 0, 0, 4084, 4095, 0, 0, 0, 10, 5, 0, 0, 0, 1, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4096, 4096, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3183, 0, 4096, 3454, 0, 0, 0, 0, 1004, 627, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 162, 4096, 0, 0, 0, 4096, 0, 0, 3, 4064, 2822, 884, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 12, 4096, 0, 3852, 0, 4096, 0, 0, 225, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3036, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 4040, 0, 0, 0, 0, 1274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16358, 16375, 16460, 16142, 16396, 16402, 16395, 16544] avg=16384.0000 max=16544 max_rank=7 min=16142 min_rank=3 imbalance_ratio=1.009766 max_load_reduction=114528 max_load_reduction_pct=0.873779 moved_to_echo=65737 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[61, 0, 4085, 0, 0, 4072, 4096, 0, 0, 0, 24, 2, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 4095, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3183, 0, 4096, 3881, 0, 0, 0, 1, 323, 880, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 208, 0, 0, 0, 141, 4095, 0, 0, 0, 4096, 0, 0, 2, 4082, 2822, 884, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 12, 4095, 0, 3840, 0, 4094, 0, 0, 270, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2773, 4095, 0, 0, 0, 1, 0, 4096, 0, 0, 4032, 9, 0, 0, 0, 1274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16410, 16385, 16460, 16334, 16390, 16400, 16413, 16280] avg=16384.0000 max=16460 max_rank=2 min=16280 min_rank=7 imbalance_ratio=1.004639 max_load_reduction=114612 max_load_reduction_pct=0.874420 moved_to_echo=65483 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[59, 5, 4091, 1, 0, 4060, 4054, 2, 0, 5, 36, 4, 4, 1, 5, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 0, 0, 5, 5, 0, 1, 0, 5, 5, 4091, 4090, 4091, 4090, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3163, 5, 4091, 3726, 0, 0, 4, 0, 351, 958, 1, 0, 3, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 5, 118, 1, 2, 0, 315, 4091, 6, 5, 0, 4090, 0, 0, 43, 3871, 2817, 889, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4091, 4087, 0, 1, 0, 5, 0, 0, 0, 5, 0, 14, 4, 4089, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 4079, 5, 0, 32, 2, 5, 4088, 5, 0, 4089, 4079, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 4063, 0, 4091, 6, 3794, 0, 4091, 2, 6, 626, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2416, 4091, 1, 2, 0, 126, 0, 4070, 0, 0, 4094, 0, 0, 4, 0, 1268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16396, 16413, 16398, 16278, 16400, 16424, 16691, 16072] avg=16384.0000 max=16691 max_rank=6 min=16072 min_rank=7 imbalance_ratio=1.018738 max_load_reduction=114381 max_load_reduction_pct=0.872658 moved_to_echo=65587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[47, 0, 4090, 0, 0, 4045, 4096, 0, 0, 0, 57, 4, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 4090, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3183, 0, 4096, 3858, 0, 0, 0, 4, 396, 805, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 0, 0, 154, 4093, 0, 0, 0, 4096, 0, 0, 7, 4005, 2822, 884, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4062, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4074, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 9, 4092, 0, 3576, 0, 4092, 0, 0, 419, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3017, 4093, 0, 0, 0, 2, 0, 4096, 0, 0, 3961, 29, 0, 0, 0, 1274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16409, 16383, 16441, 16362, 16365, 16380, 16260, 16472] avg=16384.0000 max=16472 max_rank=7 min=16260 min_rank=6 imbalance_ratio=1.005371 max_load_reduction=114600 max_load_reduction_pct=0.874329 moved_to_echo=65477 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4086, 0, 0, 4073, 4096, 0, 0, 0, 35, 9, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4096, 4095, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3183, 0, 4096, 3941, 0, 0, 0, 1, 398, 532, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 612, 4096, 0, 0, 0, 4096, 0, 0, 1, 4029, 2822, 884, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 4082, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4053, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 12, 4089, 0, 4023, 0, 4096, 0, 0, 209, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2686, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 4022, 20, 0, 0, 0, 1274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16370, 16379, 16247, 16627, 16386, 16359, 16505, 16199] avg=16384.0000 max=16627 max_rank=3 min=16199 min_rank=7 imbalance_ratio=1.014832 max_load_reduction=114445 max_load_reduction_pct=0.873146 moved_to_echo=65449 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 4078, 0, 0, 4067, 4087, 0, 0, 0, 34, 5, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4096, 4092, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3183, 0, 4096, 3961, 0, 0, 0, 1, 536, 57, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 0, 0, 0, 163, 4096, 0, 0, 0, 4096, 0, 0, 2, 4053, 2822, 884, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4071, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4008, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 12, 4093, 0, 4003, 0, 4093, 0, 0, 341, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3072, 4095, 0, 0, 0, 16, 0, 4096, 0, 0, 4026, 15, 0, 0, 0, 1274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16349, 16373, 15931, 16481, 16374, 16314, 16656, 16594] avg=16384.0000 max=16656 max_rank=6 min=15931 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=114416 max_load_reduction_pct=0.872925 moved_to_echo=65938 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[256, 128, 32768, 128, 0, 32640, 32768, 128, 0, 128, 384, 128, 128, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 128, 128, 32768, 128, 0, 128, 128, 128, 32768, 128, 128, 32768, 32640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 32768, 32768, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[23040, 32768, 128, 128, 0, 256, 0, 32768, 0, 0, 32256, 128, 0, 128, 0, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 25472, 128, 32768, 30848, 0, 0, 128, 128, 4224, 5248, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 1408, 128, 128, 0, 2432, 32768, 128, 128, 0, 32768, 0, 0, 128, 32128, 22656, 7168, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32384, 128, 32768, 128, 30464, 0, 32768, 128, 128, 2560, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 0, 0, 128, 128, 0, 128, 0, 128, 128, 32768, 32768, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1832, 0, 2878, 0, 0, 0, 0, 0, 190, 0, 0, 0, 399, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 17, 0, 0, 0, 0, 4096, 4096, 4095, 120, 0, 0, 2, 0, 56, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 3761, 3981, 4041, 0, 0, 0, 46, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 121, 0, 0, 2965, 0, 0, 0, 0, 20, 0, 0, 0, 60, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4095, 4095, 0, 0, 1314, 0, 3131, 0, 0, 0, 4, 0, 319, 0, 0, 0, 220, 1, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 1, 3, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4095, 4096, 148, 0, 0, 1, 0, 48, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 1, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 4087, 0, 3978, 3970, 4093, 0, 0, 0, 241, 4093, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 394, 0, 0, 2500, 0, 0, 1, 0, 1, 0, 0, 0, 135, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4090, 0, 7, 3, 6, 4, 4091, 4091, 4090, 6, 0, 881, 5, 3103, 2, 0, 0, 0, 6, 16, 6, 7, 4, 129, 0, 5, 4088, 4090, 5, 0, 0, 2, 4090, 4091, 4089, 0, 0, 0, 0, 5, 0, 4086, 1, 1, 0, 0, 0, 4091, 4090, 4091, 47, 2, 0, 0, 5, 386, 0, 6, 6, 4088, 0, 5, 0, 4073, 0, 0, 4090, 1, 5, 4090, 5, 5, 1, 4, 5, 0, 5, 4089, 4094, 16, 4090, 4065, 4087, 0, 0, 0, 133, 4077, 3, 4090, 4091, 5, 0, 3, 4088, 4088, 0, 5, 0, 5, 5, 0, 0, 4090, 0, 4091, 197, 0, 0, 2819, 1, 0, 12, 0, 0, 6, 0, 6, 512, 5, 0, 0, 4086, 1, 0, 0, 6, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 837, 0, 2824, 0, 0, 0, 5, 0, 1189, 0, 0, 0, 2, 1, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 382, 0, 0, 0, 0, 282, 0, 0, 0, 4096, 0, 0, 0, 4084, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 4008, 4002, 4095, 0, 0, 0, 17, 4090, 0, 4096, 4096, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 68, 0, 0, 2632, 0, 0, 0, 0, 2, 0, 0, 0, 177, 0, 0, 0, 4096, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4092, 4092, 0, 0, 802, 0, 3153, 0, 0, 0, 5, 0, 132, 0, 0, 0, 266, 4, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 2, 6, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 4095, 4096, 4096, 136, 0, 0, 0, 0, 20, 0, 0, 0, 4095, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4077, 0, 3949, 3776, 4096, 0, 0, 0, 360, 4091, 0, 4096, 4096, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 826, 0, 0, 2752, 0, 0, 3, 0, 8, 0, 0, 0, 232, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 2258, 0, 1908, 0, 0, 0, 0, 0, 108, 0, 0, 0, 72, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 47, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 4018, 3687, 4090, 0, 0, 0, 140, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1297, 0, 0, 2859, 0, 0, 0, 0, 1, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4095, 4095, 0, 0, 1483, 0, 2486, 0, 0, 0, 2, 0, 131, 0, 0, 0, 732, 1, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 4096, 4096, 4096, 80, 0, 0, 0, 0, 112, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 4024, 3996, 4095, 0, 0, 0, 21, 4094, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 973, 0, 0, 1970, 0, 0, 1, 0, 7, 0, 0, 0, 381, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4095, 4094, 0, 0, 1781, 0, 2703, 0, 0, 0, 5, 0, 235, 0, 0, 0, 65, 4, 0, 4096, 4096, 0, 0, 0, 0, 4094, 4096, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 4096, 117, 0, 0, 0, 0, 171, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4076, 0, 4002, 4009, 4096, 0, 0, 0, 118, 4085, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 571, 0, 0, 2356, 0, 0, 1, 0, 1, 0, 0, 0, 288, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4021, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 1314, 0, 3131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 319, 0, 0, 0, 220, 1, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3333, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 3, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3835, 148, 0, 0, 1, 0, 48, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 4087, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 3970, 4093, 0, 0, 0, 241, 4093, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 394, 0, 0, 2500, 0, 0, 3939, 763, 261, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 135, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16656, 16398, 16386, 16315, 16382, 16532, 16116, 16287] avg=16384.0000 max=16656 max_rank=0 min=16116 min_rank=6 imbalance_ratio=1.016602 max_load_reduction=114416 max_load_reduction_pct=0.872925 moved_to_echo=65317 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4021, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1832, 0, 2878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 0, 0, 399, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3333, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 17, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3834, 120, 0, 0, 2, 0, 56, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 3981, 4041, 0, 0, 0, 46, 4096, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 121, 0, 0, 2965, 0, 0, 3939, 763, 261, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 60, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16923, 16443, 16399, 16297, 16388, 16082, 16309, 16231] avg=16384.0000 max=16923 max_rank=0 min=16082 min_rank=5 imbalance_ratio=1.032898 max_load_reduction=114149 max_load_reduction_pct=0.870888 moved_to_echo=65010 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4021, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2258, 0, 1908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 72, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3332, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3835, 47, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 3687, 4090, 0, 0, 0, 140, 4096, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1297, 0, 0, 2859, 0, 0, 3938, 764, 261, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16033, 16383, 16166, 16376, 16188, 17379, 16168] avg=16384.0000 max=17379 max_rank=6 min=16033 min_rank=1 imbalance_ratio=1.060730 max_load_reduction=113693 max_load_reduction_pct=0.867409 moved_to_echo=66111 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4021, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 837, 0, 2824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1189, 0, 0, 0, 2, 1, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3332, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3835, 382, 0, 0, 0, 0, 282, 0, 0, 0, 4096, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 4002, 4095, 0, 0, 0, 17, 4090, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 68, 0, 0, 2632, 0, 0, 3932, 764, 261, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 177, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 4096, 3863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15874, 17050, 16385, 16775, 16376, 16369, 15912, 16331] avg=16384.0000 max=17050 max_rank=1 min=15874 min_rank=0 imbalance_ratio=1.040649 max_load_reduction=114022 max_load_reduction_pct=0.869919 moved_to_echo=64988 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4008, 0, 7, 3, 6, 4, 0, 4091, 4090, 6, 0, 881, 5, 3103, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 16, 6, 7, 4, 129, 0, 5, 4088, 4090, 5, 0, 0, 2, 4090, 3326, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4089, 0, 0, 0, 0, 5, 0, 4086, 1, 1, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3826, 47, 2, 0, 0, 5, 386, 0, 6, 6, 4088, 0, 5, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 5, 4090, 5, 5, 1, 4, 5, 0, 5, 4089, 4094, 16, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4065, 4087, 0, 0, 0, 133, 4077, 3, 0, 0, 5, 0, 3, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5, 5, 0, 0, 4090, 0, 4091, 197, 0, 0, 2819, 1, 0, 3933, 762, 265, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 6, 0, 6, 512, 5, 0, 0, 4086, 1, 0, 0, 6, 5, 4091, 4091, 3858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16206, 16007, 16359, 16534, 16421, 16618, 16248, 16679] avg=16384.0000 max=16679 max_rank=7 min=16007 min_rank=1 imbalance_ratio=1.018005 max_load_reduction=114393 max_load_reduction_pct=0.872749 moved_to_echo=65966 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4021, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 1483, 0, 2486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 131, 0, 0, 0, 732, 1, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3332, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3835, 80, 0, 0, 0, 0, 112, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 3996, 4095, 0, 0, 0, 21, 4094, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 973, 0, 0, 1970, 0, 0, 3939, 764, 261, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 381, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16180, 16719, 16383, 16312, 16385, 16387, 16166, 16540] avg=16384.0000 max=16719 max_rank=1 min=16166 min_rank=6 imbalance_ratio=1.020447 max_load_reduction=114353 max_load_reduction_pct=0.872444 moved_to_echo=65478 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4021, 0, 0, 0, 0, 0, 0, 4092, 4092, 0, 0, 802, 0, 3153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 132, 0, 0, 0, 266, 4, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3333, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2, 6, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3835, 136, 0, 0, 0, 0, 20, 0, 0, 0, 4095, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4077, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 3776, 4096, 0, 0, 0, 360, 4091, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 826, 0, 0, 2752, 0, 0, 3937, 763, 261, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8, 0, 0, 0, 232, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16160, 16261, 16395, 16268, 16372, 16429, 16793, 16394] avg=16384.0000 max=16793 max_rank=6 min=16160 min_rank=0 imbalance_ratio=1.024963 max_load_reduction=114279 max_load_reduction_pct=0.871880 moved_to_echo=65988 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4021, 0, 0, 0, 0, 0, 0, 4095, 4094, 0, 0, 1781, 0, 2703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 235, 0, 0, 0, 65, 4, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3332, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3835, 117, 0, 0, 0, 0, 171, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4076, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 4009, 4096, 0, 0, 0, 118, 4085, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 571, 0, 0, 2356, 0, 0, 3939, 764, 261, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 288, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16694, 16161, 16382, 16405, 16372, 16467, 16149, 16442] avg=16384.0000 max=16694 max_rank=0 min=16149 min_rank=6 imbalance_ratio=1.018921 max_load_reduction=114378 max_load_reduction_pct=0.872635 moved_to_echo=65430 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 0, 32768, 0, 128, 128, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 2432, 128, 128, 128, 1920, 128, 128, 32768, 32768, 128, 0, 0, 128, 32768, 26752, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[31872, 31488, 32768, 0, 0, 0, 1152, 32768, 128, 0, 0, 128, 0, 128, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 128, 0, 128, 32768, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32256, 0, 128, 128, 128, 128, 0, 32768, 32768, 128, 0, 11264, 128, 22272, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 30720, 1152, 128, 0, 128, 128, 1152, 0, 128, 128, 32768, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 0, 128, 1920, 128, 0, 0, 32768, 128, 0, 128, 128, 128, 32768, 32768, 30976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 32768, 0, 32768, 4480, 0, 0, 20864, 128, 0, 31616, 6144, 2176, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 1, 0, 0, 1546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4079, 0, 0, 4095, 0, 0, 0, 4096, 0, 4, 4088, 4096, 0, 0, 0, 0, 0, 4093, 4096, 4, 0, 896, 4083, 0, 0, 4096, 0, 1, 0, 0, 4096, 4063, 0, 0, 0, 100, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 4096, 4096, 4096, 4096, 4094, 0, 0, 0, 0, 0, 3129, 0, 4096, 3882, 0, 0, 0, 4096, 0, 0, 0, 3220, 0, 0, 0, 0, 4096, 0, 0, 3680, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2702, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 14, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 4091, 1, 0, 4, 4095, 0, 10, 4082, 4095, 1, 0, 0, 0, 1, 4089, 4075, 10, 0, 596, 4087, 0, 0, 4096, 0, 0, 0, 0, 4095, 4080, 0, 0, 0, 44, 0, 0, 0, 4096, 0, 4096, 4096, 0, 6, 4096, 4096, 4095, 4096, 4096, 0, 32, 0, 2, 0, 4052, 0, 4095, 4060, 0, 0, 0, 4096, 0, 0, 0, 955, 0, 0, 0, 0, 4096, 0, 0, 3969, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[5, 0, 0, 4091, 5, 0, 6, 0, 2071, 0, 6, 0, 1, 5, 0, 0, 1, 0, 5, 4090, 5, 5, 4088, 6, 6, 5, 0, 4090, 0, 2, 6, 1, 23, 0, 0, 4064, 13, 5, 62, 4088, 6, 5, 4091, 6, 0, 0, 4091, 2, 1, 4088, 4090, 0, 0, 6, 5, 0, 4090, 4090, 0, 0, 960, 4091, 0, 25, 4089, 2, 0, 0, 5, 4090, 4030, 0, 4, 0, 930, 4, 6, 0, 4087, 0, 4089, 4082, 0, 0, 4088, 4089, 4090, 4091, 4045, 0, 2, 1, 0, 5, 3933, 5, 4090, 3907, 5, 5, 7, 4090, 0, 5, 37, 659, 5, 5, 1, 0, 4090, 10, 0, 3862, 4090, 0, 5, 6, 6, 0, 6, 3, 5, 4, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 1707, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 4096, 0, 8, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4081, 0, 0, 4083, 0, 0, 7, 4093, 0, 23, 4083, 4096, 0, 0, 0, 0, 0, 4080, 4085, 9, 0, 376, 4087, 0, 0, 4096, 0, 0, 0, 0, 4086, 4063, 0, 0, 0, 980, 0, 0, 0, 4096, 0, 4096, 4096, 0, 12, 4096, 4096, 4096, 4095, 4094, 0, 22, 0, 3, 0, 4082, 0, 4096, 4090, 0, 0, 0, 4096, 0, 0, 0, 1222, 0, 0, 0, 0, 4096, 0, 0, 3946, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3134, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 4096, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4083, 0, 0, 4079, 3, 0, 8, 4092, 0, 28, 4093, 4096, 0, 0, 0, 0, 0, 4086, 4087, 8, 0, 1155, 4083, 0, 0, 4096, 0, 0, 0, 0, 4086, 4081, 0, 0, 0, 78, 0, 0, 0, 4096, 0, 4096, 4096, 0, 10, 4096, 4096, 4096, 4093, 4096, 0, 51, 0, 5, 0, 3310, 0, 4096, 4072, 0, 0, 0, 4096, 0, 0, 0, 706, 0, 0, 0, 0, 4096, 0, 0, 3884, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3429, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3, 4096, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4092, 1, 0, 4, 4094, 0, 2, 4089, 4096, 0, 0, 0, 0, 0, 4095, 4093, 10, 0, 598, 4090, 0, 0, 4096, 0, 0, 0, 0, 4094, 4054, 0, 0, 0, 95, 0, 0, 0, 4096, 0, 4096, 4096, 0, 6, 4096, 4096, 4096, 4095, 4096, 0, 5, 0, 1, 0, 3934, 0, 4096, 4058, 0, 0, 0, 4096, 0, 0, 0, 470, 0, 0, 0, 0, 4096, 0, 0, 3821, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2686, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 7, 4096, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4075, 0, 0, 5, 4094, 0, 20, 4092, 4096, 0, 0, 0, 0, 0, 4096, 4077, 14, 0, 1320, 4081, 0, 0, 4096, 0, 0, 0, 0, 4058, 3971, 0, 0, 0, 223, 0, 0, 0, 4096, 0, 4096, 4096, 0, 3, 4096, 4096, 4096, 4096, 4095, 0, 67, 0, 2, 0, 3872, 0, 4096, 4034, 0, 0, 0, 4096, 0, 0, 0, 507, 0, 0, 0, 0, 4096, 0, 0, 3822, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 2, 0, 4096, 0, 0, 0, 0, 2555, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 4095, 0, 1, 4086, 0, 0, 0, 4096, 0, 20, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 1412, 4086, 0, 0, 4096, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 120, 0, 0, 0, 4096, 0, 4096, 4096, 0, 3, 4096, 4096, 4096, 4094, 4093, 0, 33, 0, 0, 0, 3674, 0, 4096, 3815, 0, 0, 0, 4096, 0, 0, 0, 1159, 0, 0, 0, 0, 4096, 0, 0, 3602, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2702, 0, 0, 0, 0, 0, 0, 6, 4096, 4095, 1605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 14, 0, 0, 1896, 1013, 906, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 4091, 1, 0, 4, 4095, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4082, 0, 1, 0, 0, 0, 1, 3183, 4075, 10, 0, 596, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 0, 0, 0, 0, 4095, 4080, 0, 0, 0, 44, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 325, 0, 0, 4096, 4096, 0, 32, 0, 2, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4060, 0, 0, 0, 3083, 0, 0, 0, 955, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 2491, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16600, 16386, 16380, 16045, 16140, 16705, 16289, 16527] avg=16384.0000 max=16705 max_rank=5 min=16045 min_rank=3 imbalance_ratio=1.019592 max_load_reduction=114367 max_load_reduction_pct=0.872551 moved_to_echo=65661 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 1, 0, 0, 1546, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1896, 1013, 907, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4079, 0, 0, 4095, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4088, 0, 0, 0, 0, 0, 0, 3186, 4096, 4, 0, 896, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 0, 1, 0, 0, 4096, 4063, 0, 0, 0, 100, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 325, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 3129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3882, 0, 0, 0, 3083, 0, 0, 0, 3220, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3680, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2491, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15440, 16372, 16369, 16357, 16181, 15738, 18377, 16238] avg=16384.0000 max=18377 max_rank=6 min=15440 min_rank=0 imbalance_ratio=1.121643 max_load_reduction=112695 max_load_reduction_pct=0.859795 moved_to_echo=66534 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3429, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 1605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3, 4096, 0, 10, 0, 0, 1896, 1013, 908, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4092, 1, 0, 4, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4089, 0, 0, 0, 0, 0, 0, 3187, 4093, 10, 0, 598, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 0, 0, 0, 0, 4094, 4054, 0, 0, 0, 95, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 325, 0, 0, 4095, 4096, 0, 5, 0, 1, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4058, 0, 0, 0, 3083, 0, 0, 0, 470, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3821, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2491, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17327, 16386, 16386, 16069, 16164, 16558, 15803, 16379] avg=16384.0000 max=17327 max_rank=0 min=15803 min_rank=6 imbalance_ratio=1.057556 max_load_reduction=113745 max_load_reduction_pct=0.867805 moved_to_echo=64904 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 1707, 0, 0, 0, 0, 0, 0, 16, 4096, 4096, 1605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 4096, 0, 8, 0, 0, 1896, 1013, 904, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4081, 0, 0, 4083, 0, 0, 7, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4083, 0, 0, 0, 0, 0, 0, 3176, 4085, 9, 0, 376, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 0, 0, 0, 0, 4086, 4063, 0, 0, 0, 980, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 325, 0, 0, 4095, 4094, 0, 22, 0, 3, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 3083, 0, 0, 0, 1222, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2491, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15616, 16383, 16364, 15839, 17050, 16729, 16587, 16504] avg=16384.0000 max=17050 max_rank=4 min=15616 min_rank=0 imbalance_ratio=1.040649 max_load_reduction=114022 max_load_reduction_pct=0.869919 moved_to_echo=66870 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 4091, 5, 0, 6, 0, 2071, 0, 6, 0, 1, 5, 0, 0, 4089, 4090, 1603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 4090, 5, 5, 4088, 6, 6, 5, 0, 4090, 0, 2, 6, 1, 1895, 1018, 911, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 4064, 13, 5, 62, 4088, 6, 5, 4091, 6, 0, 0, 4091, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 0, 0, 0, 6, 5, 0, 3179, 4090, 0, 0, 960, 4091, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 2, 0, 0, 5, 4090, 4030, 0, 4, 0, 930, 4, 6, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 318, 0, 0, 4091, 4045, 0, 2, 1, 0, 5, 3933, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3907, 5, 5, 7, 3072, 0, 5, 37, 659, 5, 5, 1, 0, 4090, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3862, 4090, 0, 5, 6, 6, 0, 6, 3, 5, 4, 0, 0, 0, 0, 4090, 2486, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15972, 16409, 16460, 16445, 16968, 16482, 15898, 16438] avg=16384.0000 max=16968 max_rank=4 min=15898 min_rank=6 imbalance_ratio=1.035645 max_load_reduction=114104 max_load_reduction_pct=0.870544 moved_to_echo=65786 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 4096, 0, 0, 0, 0, 2555, 0, 0, 0, 0, 0, 0, 21, 4096, 4096, 1605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1896, 1013, 908, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 4095, 0, 1, 4086, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 0, 0, 0, 0, 0, 0, 3188, 4089, 0, 0, 1412, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 120, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 325, 0, 0, 4094, 4093, 0, 33, 0, 0, 0, 3674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3815, 0, 0, 0, 3083, 0, 0, 0, 1159, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3602, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2491, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16471, 16373, 16380, 16891, 16230, 16318, 16249, 16160] avg=16384.0000 max=16891 max_rank=3 min=16160 min_rank=7 imbalance_ratio=1.030945 max_load_reduction=114181 max_load_reduction_pct=0.871132 moved_to_echo=64957 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3134, 0, 0, 0, 0, 0, 0, 16, 4096, 4096, 1605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 4096, 0, 4, 0, 0, 1896, 1013, 906, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4083, 0, 0, 4079, 3, 0, 8, 4092, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4093, 0, 0, 0, 0, 0, 0, 3180, 4087, 8, 0, 1155, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 0, 0, 0, 0, 4086, 4081, 0, 0, 0, 78, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 325, 0, 0, 4093, 4096, 0, 51, 0, 5, 0, 3310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4072, 0, 0, 0, 3083, 0, 0, 0, 706, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 2491, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17043, 16381, 16364, 16634, 16166, 15986, 16053, 16445] avg=16384.0000 max=17043 max_rank=0 min=15986 min_rank=5 imbalance_ratio=1.040222 max_load_reduction=114029 max_load_reduction_pct=0.869972 moved_to_echo=64650 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2686, 0, 0, 0, 0, 0, 0, 25, 4096, 4096, 1604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 7, 4096, 0, 2, 0, 0, 1896, 1013, 908, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4075, 0, 0, 5, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4092, 0, 0, 0, 0, 0, 0, 3188, 4077, 14, 0, 1320, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 0, 0, 0, 0, 4058, 3971, 0, 0, 0, 223, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 325, 0, 0, 4096, 4095, 0, 67, 0, 2, 0, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4034, 0, 0, 0, 3083, 0, 0, 0, 507, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3822, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2492, 1875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16603, 16382, 16369, 16792, 16173, 16556, 15816, 16381] avg=16384.0000 max=16792 max_rank=3 min=15816 min_rank=6 imbalance_ratio=1.024902 max_load_reduction=114280 max_load_reduction_pct=0.871887 moved_to_echo=64926 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 30592, 32768, 0, 128, 128, 128, 0, 128, 128, 128, 128, 0, 128, 0, 0, 32768, 19968, 15104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 128, 0, 19840, 0, 128, 0, 128, 128, 0, 128, 32768, 32768, 12928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[30592, 128, 128, 0, 128, 32768, 32512, 0, 128, 0, 2688, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 0, 32768, 128, 128, 128, 32768, 128, 128, 32768, 128, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 32000, 128, 128, 128, 24704, 0, 128, 128, 8960, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 0, 128, 0, 128, 128, 128, 25472, 32768, 128, 0, 7424, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 0, 128, 2688, 0, 0, 32768, 32768, 0, 256, 128, 128, 128, 30080, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 32768, 0, 128, 128, 128, 15232, 8192, 7296, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[2807, 0, 0, 2946, 0, 0, 4050, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 31, 0, 0, 0, 0, 4096, 2939, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3718, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[2532, 0, 0, 3288, 0, 0, 3820, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 345, 0, 0, 0, 0, 4092, 2769, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 4094, 4096, 0, 0, 0, 3877, 1, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 6, 0, 4095, 0, 0, 4093, 0, 3887, 4, 0, 4052, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[2801, 0, 0, 2451, 0, 0, 3939, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 207, 0, 0, 0, 0, 4090, 3335, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4090, 4096, 0, 0, 0, 3803, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 4094, 0, 3956, 5, 0, 4082, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3424, 4, 0, 2796, 5, 5, 4046, 0, 6, 1, 0, 5, 5, 0, 4089, 2, 224, 1, 0, 0, 0, 4091, 2638, 0, 3, 0, 4090, 6, 0, 0, 0, 0, 0, 4091, 0, 17, 4090, 5, 6, 4087, 5, 4091, 4091, 0, 9, 5, 3267, 0, 6, 4, 0, 0, 6, 0, 4089, 0, 5, 2, 0, 0, 5, 4077, 4090, 7, 0, 0, 5, 0, 5, 6, 0, 0, 4091, 0, 4090, 4079, 6, 6, 0, 3, 4090, 5, 4090, 0, 5, 4088, 6, 6, 0, 6, 0, 0, 0, 4089, 0, 0, 5, 6, 6, 107, 0, 18, 4074, 0, 0, 0, 6, 0, 0, 4090, 5, 5, 4091, 0, 3952, 0, 6, 4090, 4090, 0, 0, 4066, 0, 27, 4087, 4078, 4091, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1031, 0, 0, 3758, 0, 0, 3927, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 178, 0, 0, 0, 0, 4087, 3714, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 4095, 0, 1, 4096, 0, 0, 4096, 0, 4086, 4096, 0, 0, 0, 3832, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 16, 0, 4096, 0, 0, 4093, 0, 4058, 8, 0, 4071, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1710, 0, 0, 3428, 0, 0, 3825, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 266, 0, 0, 0, 0, 4094, 3227, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 4094, 4096, 0, 0, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4095, 0, 4058, 2, 0, 4088, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[2309, 0, 0, 3988, 0, 0, 4070, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 273, 0, 0, 0, 0, 4090, 2344, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 3790, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 4096, 0, 3729, 0, 0, 4070, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[857, 0, 0, 3848, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 481, 0, 0, 0, 0, 4086, 3575, 0, 0, 0, 4092, 0, 0, 0, 0, 4, 0, 4094, 0, 0, 4096, 0, 0, 4096, 0, 4088, 4096, 0, 0, 0, 3664, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 22, 0, 4096, 0, 0, 4089, 0, 4051, 9, 0, 4071, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2532, 0, 0, 3288, 0, 0, 3820, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2823, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 0, 0, 0, 0, 4092, 2769, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4096, 761, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 324, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 3877, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3334, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3145, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 6, 0, 4095, 0, 0, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3887, 4, 0, 4052, 0, 0, 0, 4096, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16562, 16182, 16487, 16385, 16384, 16384, 16367, 16321] avg=16384.0000 max=16562 max_rank=0 min=16182 min_rank=1 imbalance_ratio=1.010864 max_load_reduction=114510 max_load_reduction_pct=0.873642 moved_to_echo=65456 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2807, 0, 0, 2946, 0, 0, 4050, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 2823, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 4096, 2939, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 762, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 324, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3334, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3145, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3718, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16726, 16043, 16599, 16384, 16382, 16381, 16362, 16195] avg=16384.0000 max=16726 max_rank=0 min=16043 min_rank=1 imbalance_ratio=1.020874 max_load_reduction=114346 max_load_reduction_pct=0.872391 moved_to_echo=65320 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2801, 0, 0, 2451, 0, 0, 3939, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2823, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 4090, 3335, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 762, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 324, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 3803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3334, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 4096, 0, 0, 0, 0, 3145, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3956, 5, 0, 4082, 0, 0, 0, 4096, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16113, 16602, 16409, 16384, 16386, 16388, 16368, 16422] avg=16384.0000 max=16602 max_rank=1 min=16113 min_rank=0 imbalance_ratio=1.013306 max_load_reduction=114470 max_load_reduction_pct=0.873337 moved_to_echo=65564 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2309, 0, 0, 3988, 0, 0, 4070, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 2823, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 0, 0, 0, 4090, 2344, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 762, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 324, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 3790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3334, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 3145, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3729, 0, 0, 4070, 0, 0, 0, 4096, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17291, 15684, 16402, 16384, 16381, 16384, 16366, 16180] avg=16384.0000 max=17291 max_rank=0 min=15684 min_rank=1 imbalance_ratio=1.055359 max_load_reduction=113781 max_load_reduction_pct=0.868080 moved_to_echo=65311 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3424, 4, 0, 2796, 5, 5, 4046, 0, 6, 1, 0, 5, 5, 0, 4089, 2, 2825, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 1, 0, 0, 0, 4091, 2638, 0, 3, 0, 4090, 6, 0, 0, 0, 0, 4087, 762, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 17, 310, 5, 6, 4087, 5, 4091, 0, 0, 9, 5, 3267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 6, 0, 4089, 0, 5, 2, 0, 0, 5, 4077, 4090, 7, 3329, 766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5, 6, 0, 0, 4091, 0, 4090, 4079, 6, 6, 0, 3, 3138, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 5, 4090, 0, 5, 4088, 6, 6, 0, 6, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 6, 107, 0, 18, 4074, 0, 0, 0, 6, 0, 0, 4090, 5, 5, 4091, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3952, 0, 6, 4090, 0, 0, 0, 4066, 0, 27, 0, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17220, 15925, 15893, 16386, 16384, 16378, 16480, 16406] avg=16384.0000 max=17220 max_rank=0 min=15893 min_rank=2 imbalance_ratio=1.051025 max_load_reduction=113852 max_load_reduction_pct=0.868622 moved_to_echo=65648 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1710, 0, 0, 3428, 0, 0, 3825, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2823, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 4094, 3227, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 4096, 762, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 323, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3334, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 3145, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4058, 2, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15885, 16563, 16582, 16384, 16381, 16384, 16365, 16528] avg=16384.0000 max=16582 max_rank=2 min=15885 min_rank=0 imbalance_ratio=1.012085 max_load_reduction=114490 max_load_reduction_pct=0.873489 moved_to_echo=65658 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1031, 0, 0, 3758, 0, 0, 3927, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2823, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 4087, 3714, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 4096, 761, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 324, 0, 0, 4096, 0, 4086, 0, 0, 0, 0, 3832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 3333, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3145, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 16, 0, 4096, 0, 0, 4096, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4058, 8, 0, 4071, 0, 0, 0, 4096, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15638, 16954, 16434, 16381, 16385, 16385, 16380, 16515] avg=16384.0000 max=16954 max_rank=1 min=15638 min_rank=0 imbalance_ratio=1.034790 max_load_reduction=114118 max_load_reduction_pct=0.870651 moved_to_echo=65665 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[857, 0, 0, 3848, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2823, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 0, 0, 0, 0, 4086, 3575, 0, 0, 0, 4092, 0, 0, 0, 0, 4, 4096, 762, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 324, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 3664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 3331, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3145, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 22, 0, 4096, 0, 0, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4051, 9, 0, 4071, 0, 0, 0, 4096, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15637, 17119, 16266, 16384, 16389, 16388, 16384, 16505] avg=16384.0000 max=17119 max_rank=1 min=15637 min_rank=0 imbalance_ratio=1.044861 max_load_reduction=113953 max_load_reduction_pct=0.869392 moved_to_echo=65666 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[17536, 128, 0, 26624, 128, 128, 31744, 0, 128, 128, 128, 128, 128, 0, 32768, 128, 22656, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 32768, 128, 128, 32768, 128, 128, 0, 128, 0, 0, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 0, 31488, 128, 128, 32640, 0, 0, 0, 32768, 0, 128, 0, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 128, 128, 0, 0, 32768, 0, 32768, 32768, 128, 128, 0, 128, 25216, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32768, 0, 0, 128, 128, 128, 0, 32768, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32768, 0, 128, 2688, 128, 128, 32768, 128, 32768, 0, 0, 128, 128, 30208, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[2048, 128, 0, 0, 0, 32768, 24576, 0, 128, 0, 32768, 128, 0, 0, 0, 128, 32768, 6144, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 0, 128, 128, 32768, 0, 128, 128, 0, 0, 128, 32768, 32768, 128, 26752, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3874, 4095, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 748, 0, 0, 1, 0, 0, 3799, 0, 0, 4031, 0, 128, 2, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 50, 4095, 78, 0, 4095, 4089, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 3689, 0, 0, 2, 4096, 3, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4085, 0, 4096, 0, 0, 0, 0, 8, 0, 4096, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 4090, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3956, 4096, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 870, 0, 0, 1, 0, 0, 3383, 0, 0, 4040, 0, 46, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 41, 4095, 33, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4007, 0, 0, 2, 4096, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4087, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3075, 4096, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 2578, 0, 0, 0, 0, 0, 2631, 0, 0, 3992, 0, 26, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 90, 4096, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3982, 0, 0, 1, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3960, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4088, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 997, 0, 0, 1, 0, 0, 3289, 0, 0, 3839, 0, 198, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4096, 279, 0, 4096, 4064, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3833, 0, 0, 0, 4096, 5, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4079, 0, 4096, 0, 0, 0, 0, 6, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4090, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3753, 4096, 0, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4082, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 2648, 0, 0, 0, 0, 0, 3412, 0, 0, 3347, 0, 63, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4096, 24, 0, 4096, 4071, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3164, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 7, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4053, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 3676, 4089, 6, 5, 5, 1, 0, 4055, 0, 4, 3, 6, 4090, 4086, 6, 6, 5, 0, 0, 4091, 5, 6, 0, 5, 4090, 0, 0, 4, 8, 966, 8, 6, 0, 7, 6, 3912, 6, 1, 3075, 0, 132, 0, 3899, 7, 0, 0, 425, 0, 0, 27, 0, 0, 4090, 101, 4, 4090, 4072, 0, 0, 0, 0, 0, 5, 4089, 0, 0, 0, 4, 0, 3938, 5, 1, 153, 4060, 0, 4090, 3968, 0, 6, 0, 487, 0, 6, 6, 5, 4090, 1, 0, 4085, 6, 4, 6, 4090, 23, 4090, 0, 4033, 0, 43, 0, 6, 0, 0, 3933, 4089, 4, 4083, 0, 0, 0, 4090, 14, 0, 6, 4090, 4090, 4090, 0, 4090, 9, 1, 3, 0, 4088] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3330, 4096, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 1118, 0, 0, 2, 0, 0, 3810, 0, 0, 3864, 0, 201, 2, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 36, 4094, 47, 0, 4094, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4018, 0, 0, 5, 4096, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4058, 0, 4096, 0, 0, 0, 0, 7, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4091, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3993, 4096, 0, 0, 0, 0, 0, 4096, 0, 24, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4082, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 827, 0, 0, 4, 0, 0, 3464, 0, 0, 3852, 0, 174, 3, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 84, 4093, 9, 0, 4093, 4086, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 3993, 0, 0, 11, 4096, 5, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4082, 0, 4096, 0, 0, 0, 0, 10, 0, 4096, 4093, 0, 4096, 0, 0, 3, 4096, 0, 0, 0, 4096, 4079, 4096, 3, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3874, 4095, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 748, 0, 0, 1, 0, 0, 3799, 0, 0, 4031, 0, 128, 2, 4080, 0, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 4095, 78, 0, 4095, 4089, 0, 1, 0, 3752, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 3689, 0, 0, 2, 4096, 3, 4096, 245, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3712, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4096, 0, 0, 0, 0, 8, 0, 4096, 29, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 0, 0, 0, 0, 4090, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16553, 16382, 16424, 16347, 16230, 16338, 16410, 16388] avg=16384.0000 max=16553 max_rank=0 min=16230 min_rank=4 imbalance_ratio=1.010315 max_load_reduction=114519 max_load_reduction_pct=0.873711 moved_to_echo=65366 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3753, 4096, 0, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 4096, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 4082, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2648, 0, 0, 0, 0, 0, 3412, 0, 0, 3347, 0, 63, 0, 4088, 0, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4096, 24, 0, 4096, 4071, 0, 0, 0, 3752, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3164, 0, 0, 0, 4096, 0, 4096, 245, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3713, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 7, 0, 4096, 29, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4053, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16438, 16371, 17193, 16264, 15699, 16337, 16420, 16350] avg=16384.0000 max=17193 max_rank=2 min=15699 min_rank=4 imbalance_ratio=1.049377 max_load_reduction=113879 max_load_reduction_pct=0.868828 moved_to_echo=64806 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3956, 4096, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 4096, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 870, 0, 0, 1, 0, 0, 3383, 0, 0, 4040, 0, 46, 1, 4092, 0, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 4095, 33, 0, 4096, 4094, 0, 0, 0, 3752, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4007, 0, 0, 2, 4096, 0, 4096, 245, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3713, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 29, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16642, 16385, 16068, 16298, 16544, 16337, 16406, 16392] avg=16384.0000 max=16642 max_rank=0 min=16068 min_rank=2 imbalance_ratio=1.015747 max_load_reduction=114430 max_load_reduction_pct=0.873032 moved_to_echo=65679 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3075, 4096, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 0, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2578, 0, 0, 0, 0, 0, 2631, 0, 0, 3992, 0, 26, 0, 4085, 0, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 4096, 1, 0, 4096, 4096, 0, 0, 0, 3752, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3982, 0, 0, 1, 4096, 0, 4096, 245, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3712, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 29, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15765, 16384, 16947, 16318, 16518, 16335, 16412, 16393] avg=16384.0000 max=16947 max_rank=2 min=15765 min_rank=0 imbalance_ratio=1.034363 max_load_reduction=114125 max_load_reduction_pct=0.870705 moved_to_echo=65658 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3960, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 4088, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 997, 0, 0, 1, 0, 0, 3289, 0, 0, 3839, 0, 198, 0, 4089, 0, 3634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4096, 279, 0, 4096, 4064, 0, 0, 0, 3752, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3833, 0, 0, 0, 4096, 5, 4096, 245, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3713, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 4096, 0, 0, 0, 0, 6, 0, 4096, 29, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4090, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16633, 16390, 16047, 16504, 16373, 16336, 16402, 16387] avg=16384.0000 max=16633 max_rank=0 min=16047 min_rank=2 imbalance_ratio=1.015198 max_load_reduction=114439 max_load_reduction_pct=0.873100 moved_to_echo=65498 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 3676, 4089, 6, 5, 5, 1, 0, 4055, 0, 4, 3, 6, 4090, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4086, 6, 6, 5, 0, 0, 4091, 5, 6, 0, 5, 4090, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 966, 8, 6, 0, 7, 6, 3912, 6, 1, 3075, 0, 132, 0, 3899, 7, 3632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 425, 0, 0, 27, 0, 0, 4090, 101, 4, 4090, 4072, 0, 0, 0, 3747, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4089, 0, 0, 0, 4, 0, 3938, 5, 1, 153, 4060, 0, 4090, 252, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 6, 0, 487, 0, 6, 6, 5, 4090, 1, 0, 4085, 6, 4, 6, 3706, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4090, 0, 4033, 0, 43, 0, 6, 0, 0, 3933, 16, 4, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 6, 0, 4090, 4090, 0, 4090, 9, 1, 3, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16330, 16389, 15665, 16745, 16602, 16718, 16231, 16392] avg=16384.0000 max=16745 max_rank=3 min=15665 min_rank=2 imbalance_ratio=1.022034 max_load_reduction=114327 max_load_reduction_pct=0.872246 moved_to_echo=65943 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3993, 4096, 0, 0, 0, 0, 0, 4096, 0, 24, 0, 0, 4096, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 4082, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 827, 0, 0, 4, 0, 0, 3464, 0, 0, 3852, 0, 174, 3, 4066, 0, 3632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 4093, 9, 0, 4093, 4086, 0, 0, 0, 3752, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 3993, 0, 0, 11, 4096, 5, 4096, 245, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3712, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4096, 0, 0, 0, 0, 10, 0, 4096, 29, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 0, 0, 0, 0, 4079, 4096, 3, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16689, 16384, 16022, 16304, 16547, 16335, 16409, 16382] avg=16384.0000 max=16689 max_rank=0 min=16022 min_rank=2 imbalance_ratio=1.018616 max_load_reduction=114383 max_load_reduction_pct=0.872673 moved_to_echo=65673 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3330, 4096, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 4096, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1118, 0, 0, 2, 0, 0, 3810, 0, 0, 3864, 0, 201, 2, 4074, 0, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 4094, 47, 0, 4094, 4082, 0, 0, 0, 3752, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4018, 0, 0, 5, 4096, 1, 4096, 245, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3713, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 4096, 0, 0, 0, 0, 7, 0, 4096, 29, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4091, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16022, 16387, 16706, 16292, 16559, 16336, 16382, 16388] avg=16384.0000 max=16706 max_rank=2 min=16022 min_rank=0 imbalance_ratio=1.019653 max_load_reduction=114366 max_load_reduction_pct=0.872543 moved_to_echo=65665 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32640, 128, 128, 0, 512, 0, 128, 128, 128, 32768, 128, 0, 32768, 128, 128, 128, 29696, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 0, 32768, 0, 128, 0, 128, 128, 0, 32640, 256, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 0, 29696, 32768, 128, 128, 128, 128, 0, 32768, 0, 128, 128, 128, 32768, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 128, 128, 0, 0, 32768, 128, 128, 128, 128, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 10752, 128, 128, 128, 128, 128, 27776, 128, 128, 30080, 0, 1024, 128, 32512, 128, 29184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 512, 0, 0, 128, 0, 384, 32768, 640, 128, 32768, 32768, 0, 128, 0, 30080, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 0, 128, 0, 32768, 32768, 128, 32768, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 128, 128, 0, 30720, 128, 128, 256, 32768, 128, 32768, 2048, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4088, 0, 0, 4096, 0, 0, 4081, 13, 0, 0, 0, 39, 0, 0, 0, 4096, 0, 1, 0, 4009, 0, 4096, 4095, 0, 0, 0, 4076, 0, 4040, 7, 0, 0, 4096, 0, 0, 4089, 4095, 4096, 0, 0, 0, 0, 0, 0, 3017, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1226, 4094, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 17, 0, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 2, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4092, 4, 4096, 4095, 4096, 0, 4043, 0, 0, 4094, 4, 10, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4085, 2, 0, 4089, 6, 5, 4090, 0, 0, 0, 5, 0, 0, 0, 6, 4087, 0, 0, 10, 4082, 5, 4088, 4040, 0, 0, 6, 4090, 0, 4091, 0, 0, 0, 4083, 3, 3, 4090, 4090, 4085, 0, 6, 2, 6, 0, 5, 3498, 0, 3, 6, 6, 0, 6, 0, 0, 5, 5, 13, 4090, 6, 0, 3975, 0, 5, 6, 4090, 6, 0, 0, 2, 0, 483, 4091, 0, 5, 0, 0, 8, 0, 0, 0, 6, 6, 0, 6, 4089, 4090, 2, 0, 0, 0, 0, 0, 0, 6, 4064, 4086, 6, 0, 4075, 6, 6, 0, 2, 3725, 0, 5, 5, 4087, 5, 0, 22, 4087, 0, 4090, 896, 4090, 4090, 4086, 0, 3772, 3, 7, 4090, 0, 0, 6, 76, 5, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4007, 0, 0, 4096, 0, 0, 4083, 1, 0, 0, 0, 21, 0, 0, 0, 4096, 0, 2, 0, 4003, 0, 4096, 4095, 0, 0, 0, 4089, 0, 4071, 5, 0, 0, 4096, 0, 0, 4089, 4096, 4096, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 282, 4095, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4093, 0, 4096, 4096, 4096, 0, 4054, 0, 0, 4096, 5, 4, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4080, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 3984, 0, 4096, 4096, 0, 0, 0, 4084, 0, 3981, 37, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 3337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1174, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 0, 3892, 0, 0, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4083, 0, 0, 4096, 0, 0, 4080, 2, 0, 0, 0, 39, 0, 0, 0, 4096, 0, 2, 0, 3723, 0, 4096, 4096, 0, 0, 0, 4090, 0, 4043, 1, 0, 0, 4096, 0, 0, 4088, 4096, 4096, 0, 0, 0, 0, 0, 0, 3124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1462, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 8, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4093, 0, 4096, 4096, 4096, 0, 3999, 0, 0, 4096, 1, 5, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4043, 0, 0, 4093, 0, 0, 4075, 36, 0, 0, 0, 104, 0, 0, 0, 4096, 1, 8, 0, 3450, 0, 4096, 4093, 0, 0, 0, 4067, 0, 3941, 11, 0, 0, 4096, 0, 0, 4080, 4096, 4096, 0, 0, 0, 0, 0, 0, 3592, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1244, 4089, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 15, 0, 0, 0, 4096, 4096, 0, 3, 4096, 0, 0, 2, 0, 4096, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 4088, 0, 4096, 4094, 4096, 0, 4068, 0, 0, 4092, 10, 19, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4079, 0, 0, 0, 0, 44, 0, 0, 0, 4096, 0, 5, 0, 3997, 0, 4096, 4094, 0, 0, 0, 4076, 0, 4089, 16, 0, 0, 4096, 0, 0, 4083, 4096, 4096, 0, 0, 0, 0, 0, 0, 3617, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 6, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 567, 4091, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 15, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 1, 4096, 4096, 4096, 0, 4050, 0, 0, 4096, 9, 18, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4079, 0, 0, 4096, 0, 0, 4085, 10, 0, 0, 0, 41, 0, 0, 0, 4096, 0, 2, 0, 3922, 0, 4096, 4094, 0, 0, 0, 4084, 0, 4084, 5, 0, 0, 4096, 0, 0, 4076, 4096, 4096, 0, 0, 0, 0, 0, 0, 3153, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1231, 4093, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 18, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4086, 0, 4096, 4094, 4096, 0, 3940, 0, 0, 4096, 2, 22, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4088, 0, 0, 4096, 0, 0, 4081, 13, 0, 0, 0, 39, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4009, 0, 261, 4095, 0, 0, 0, 4076, 0, 4040, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 683, 4096, 0, 0, 0, 0, 0, 0, 3017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 4096, 3398, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1226, 4094, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 4096, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 0, 0, 0, 0, 17, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 2, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 14, 12, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 4083, 0, 0, 4043, 0, 0, 4094, 4, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16489, 15981, 16394, 16651, 16395, 16425, 16330] avg=16384.0000 max=16651 max_rank=4 min=15981 min_rank=2 imbalance_ratio=1.016296 max_load_reduction=114421 max_load_reduction_pct=0.872963 moved_to_echo=65801 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4007, 0, 0, 4096, 0, 0, 4083, 1, 0, 0, 0, 21, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4003, 0, 261, 4095, 0, 0, 0, 4089, 0, 4071, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 683, 4096, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 4096, 3399, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 4095, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 4096, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 14, 12, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4084, 0, 0, 4054, 0, 0, 4096, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16298, 16526, 17008, 16392, 15706, 16384, 16422, 16336] avg=16384.0000 max=17008 max_rank=2 min=15706 min_rank=4 imbalance_ratio=1.038086 max_load_reduction=114064 max_load_reduction_pct=0.870239 moved_to_echo=64848 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4085, 2, 0, 4089, 6, 5, 4090, 0, 0, 0, 5, 0, 0, 0, 6, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4082, 5, 252, 4040, 0, 0, 6, 4090, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 3, 3, 4090, 672, 4085, 0, 6, 2, 6, 0, 5, 3498, 0, 3, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 5, 5, 13, 4090, 6, 0, 3975, 0, 5, 6, 4090, 3397, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 2, 0, 483, 4091, 0, 5, 0, 0, 8, 0, 0, 0, 6, 4090, 4086, 3135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 4079, 4090, 2, 0, 0, 0, 0, 0, 0, 6, 4064, 4086, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 6, 6, 0, 2, 3725, 0, 5, 5, 4087, 5, 0, 22, 4087, 0, 21, 16, 8, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 896, 0, 4074, 0, 0, 3772, 3, 7, 4090, 0, 0, 6, 76, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16367, 16576, 16462, 16305, 15912, 16345, 16080, 17025] avg=16384.0000 max=17025 max_rank=7 min=15912 min_rank=4 imbalance_ratio=1.039124 max_load_reduction=114047 max_load_reduction_pct=0.870110 moved_to_echo=65362 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4083, 0, 0, 4096, 0, 0, 4080, 2, 0, 0, 0, 39, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3723, 0, 261, 4096, 0, 0, 0, 4090, 0, 4043, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 683, 4096, 0, 0, 0, 0, 0, 0, 3124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3399, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1462, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 4096, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 14, 12, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4084, 0, 0, 3999, 0, 0, 4096, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16216, 16087, 16388, 16889, 16395, 16429, 16278] avg=16384.0000 max=16889 max_rank=4 min=16087 min_rank=2 imbalance_ratio=1.030823 max_load_reduction=114183 max_load_reduction_pct=0.871147 moved_to_echo=65991 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4080, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 0, 261, 4096, 0, 0, 0, 4084, 0, 3981, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 683, 4096, 0, 0, 0, 0, 0, 0, 3337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3399, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1174, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 14, 12, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 0, 0, 3892, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16363, 16443, 16308, 16388, 16596, 16380, 16425, 16169] avg=16384.0000 max=16596 max_rank=4 min=16169 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=114476 max_load_reduction_pct=0.873383 moved_to_echo=65570 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4079, 0, 0, 0, 0, 44, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3997, 0, 261, 4094, 0, 0, 0, 4076, 0, 4089, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4083, 683, 4096, 0, 0, 0, 0, 0, 0, 3617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 6, 4096, 0, 0, 0, 4096, 3399, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 4091, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 4096, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 15, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 14, 12, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 4084, 0, 0, 4050, 0, 0, 4096, 9, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16405, 16538, 16575, 16401, 15990, 16394, 16422, 16347] avg=16384.0000 max=16575 max_rank=2 min=15990 min_rank=4 imbalance_ratio=1.011658 max_load_reduction=114497 max_load_reduction_pct=0.873543 moved_to_echo=65153 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4043, 0, 0, 4093, 0, 0, 4075, 36, 0, 0, 0, 104, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 3450, 0, 261, 4093, 0, 0, 0, 4067, 0, 3941, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 683, 4096, 0, 0, 0, 0, 0, 0, 3592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 4096, 3399, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1244, 4089, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 4096, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4092, 0, 0, 0, 0, 15, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 2, 0, 4096, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 14, 12, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4082, 0, 0, 4068, 0, 0, 4092, 10, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16441, 15832, 16547, 16405, 16669, 16390, 16429, 16359] avg=16384.0000 max=16669 max_rank=4 min=15832 min_rank=1 imbalance_ratio=1.017395 max_load_reduction=114403 max_load_reduction_pct=0.872826 moved_to_echo=65847 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4079, 0, 0, 4096, 0, 0, 4085, 10, 0, 0, 0, 41, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3922, 0, 261, 4094, 0, 0, 0, 4084, 0, 4084, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 683, 4096, 0, 0, 0, 0, 0, 0, 3153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 4096, 3399, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1231, 4093, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 4096, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4094, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 18, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 14, 12, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4082, 0, 0, 3940, 0, 0, 4096, 2, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16401, 16452, 16104, 16399, 16659, 16389, 16440, 16228] avg=16384.0000 max=16659 max_rank=4 min=16104 min_rank=2 imbalance_ratio=1.016785 max_load_reduction=114413 max_load_reduction_pct=0.872902 moved_to_echo=65716 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 128, 32768, 32768, 128, 0, 0, 0, 128, 0, 0, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 31232, 128, 2176, 32768, 0, 0, 128, 32768, 0, 32384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 32768, 5504, 32768, 0, 128, 128, 128, 0, 128, 27392, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 1024, 0, 32768, 0, 0, 31872, 128, 128, 32768, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 128, 32768, 128, 128, 32768, 0, 128, 128, 32768, 27264, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32640, 128, 0, 32768, 128, 128, 32768, 128, 0, 0, 128, 384, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 7680, 32768, 0, 128, 0, 0, 128, 128, 0, 0, 128, 32768, 32768, 25088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 32512, 128, 128, 128, 32768, 128, 0, 128, 32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3441, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 3911, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 4023, 10, 4073, 4096, 0, 3, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 394, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 227, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 4014, 0, 0, 0, 1, 4095, 168, 4096, 68, 4094, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 4096, 0, 156, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2847, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 3950, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3982, 2, 4084, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 228, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 129, 145, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4032, 0, 0, 0, 1, 4096, 948, 4096, 33, 4056, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 153, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[6, 0, 0, 7, 3658, 11, 2, 10, 0, 1, 0, 37, 3932, 6, 6, 0, 4069, 0, 0, 4090, 4090, 204, 0, 0, 6, 4037, 5, 1918, 3960, 1, 2, 4090, 6, 108, 5, 7, 0, 0, 0, 0, 4090, 0, 8, 6, 5, 852, 0, 4091, 4090, 6, 0, 6, 4089, 0, 6, 0, 0, 4090, 6, 6, 4084, 828, 0, 3760, 0, 3, 0, 6, 5, 0, 7, 3926, 4072, 0, 1835, 14, 0, 14, 3, 6, 5, 0, 4086, 6, 4090, 0, 3783, 7, 20, 5, 18, 4088, 1415, 4090, 1, 4090, 0, 1, 7, 0, 189, 4090, 0, 5, 0, 0, 4082, 6, 2007, 0, 6, 6, 4089, 6, 4088, 7, 0, 0, 0, 0, 80, 4089, 263, 4087, 6, 0, 0, 4066] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3538, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 3242, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4064, 8, 4030, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 578, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 4091, 4096, 0, 16, 98, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3884, 0, 0, 0, 1, 4096, 62, 4096, 13, 4081, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 957, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3545, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 3233, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 4093, 0, 0, 0, 0, 3926, 13, 4029, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 218, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 4092, 4096, 0, 9, 241, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4021, 0, 0, 0, 2, 4092, 419, 4096, 602, 4057, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 4096, 0, 263, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2776, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 3397, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 4015, 4, 4091, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 442, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4094, 4096, 0, 14, 91, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4073, 0, 0, 0, 2, 4096, 1356, 4096, 32, 4086, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 199, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2659, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 3903, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 4075, 16, 4030, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 494, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4092, 4096, 0, 19, 103, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4082, 0, 0, 0, 0, 4095, 662, 4096, 150, 4081, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 302, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3685, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 3959, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 5, 4085, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 5, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 4096, 4096, 0, 15, 292, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3914, 0, 0, 0, 3, 4096, 267, 4096, 113, 4087, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 54, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3441, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3911, 4096, 0, 0, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 4023, 10, 4073, 3437, 657, 357, 80, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3739, 0, 0, 0, 0, 4095, 0, 0, 4096, 394, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 227, 0, 0, 0, 0, 4096, 3257, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 0, 0, 0, 4014, 0, 0, 0, 1, 4095, 168, 0, 68, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 4096, 0, 156, 0, 0, 0, 4014, 3642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4058, 0, 4096, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16568, 16858, 16771, 16423, 16224, 15879, 16003, 16346] avg=16384.0000 max=16858 max_rank=1 min=15879 min_rank=5 imbalance_ratio=1.028931 max_load_reduction=114214 max_load_reduction_pct=0.871384 moved_to_echo=64452 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2847, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3950, 4095, 0, 0, 658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3982, 2, 4084, 3437, 657, 357, 80, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3739, 0, 0, 0, 0, 4096, 0, 0, 4096, 228, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 129, 145, 0, 0, 0, 0, 4096, 3257, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 0, 0, 0, 4032, 0, 0, 0, 1, 4096, 948, 0, 33, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 153, 0, 0, 0, 4016, 3642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4058, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15974, 16895, 16733, 16255, 16266, 16605, 15998, 16346] avg=16384.0000 max=16895 max_rank=1 min=15974 min_rank=0 imbalance_ratio=1.031189 max_load_reduction=114177 max_load_reduction_pct=0.871101 moved_to_echo=65215 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3538, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3242, 4095, 0, 0, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4064, 8, 4030, 3437, 657, 357, 80, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3739, 0, 0, 0, 0, 4096, 0, 0, 4096, 578, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4091, 4096, 0, 16, 98, 0, 0, 0, 0, 4096, 3257, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 0, 0, 0, 3884, 0, 0, 0, 1, 4096, 62, 0, 13, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 957, 0, 0, 0, 4016, 3642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4058, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16665, 16188, 16772, 16605, 16113, 15576, 16807, 16346] avg=16384.0000 max=16807 max_rank=6 min=15576 min_rank=5 imbalance_ratio=1.025818 max_load_reduction=114265 max_load_reduction_pct=0.871773 moved_to_echo=64842 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2776, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3397, 4093, 0, 0, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 4015, 4, 4091, 3437, 657, 357, 80, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3739, 0, 0, 0, 0, 4096, 0, 0, 4096, 442, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 4096, 0, 14, 91, 0, 0, 0, 0, 4096, 3258, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 0, 0, 0, 4073, 0, 0, 0, 2, 4096, 1356, 0, 32, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 199, 0, 0, 0, 4015, 3642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4058, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15903, 16341, 16775, 16469, 16105, 17084, 16048, 16347] avg=16384.0000 max=17084 max_rank=5 min=15903 min_rank=0 imbalance_ratio=1.042725 max_load_reduction=113988 max_load_reduction_pct=0.869659 moved_to_echo=65584 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 7, 3658, 11, 2, 10, 0, 1, 0, 37, 3932, 6, 6, 0, 4090, 4090, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 204, 0, 0, 6, 4037, 5, 1918, 3960, 1, 2, 652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 108, 5, 7, 0, 0, 0, 0, 4090, 0, 8, 6, 5, 852, 0, 4091, 3438, 656, 358, 81, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 3731, 0, 6, 0, 0, 4090, 6, 6, 4084, 828, 0, 3760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 5, 0, 7, 3926, 4072, 0, 1835, 14, 0, 14, 3, 6, 4090, 3252, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3430, 6, 0, 0, 3783, 7, 20, 5, 18, 4088, 1415, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 189, 4090, 0, 5, 0, 0, 4082, 6, 2007, 0, 6, 6, 4009, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4088, 7, 0, 0, 0, 0, 80, 4046, 263, 4087, 6, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16694, 14854, 13754, 16523, 17687, 16868, 18043, 16649] avg=16384.0000 max=18043 max_rank=6 min=13754 min_rank=2 imbalance_ratio=1.101257 max_load_reduction=113029 max_load_reduction_pct=0.862343 moved_to_echo=69247 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3685, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3959, 4096, 0, 0, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 5, 4085, 3437, 657, 357, 80, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3739, 0, 0, 0, 0, 4096, 0, 0, 4096, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 4096, 0, 15, 292, 0, 0, 0, 0, 4096, 3258, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 0, 0, 0, 3914, 0, 0, 0, 3, 4096, 267, 0, 113, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 54, 0, 0, 0, 4016, 3642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4058, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16811, 16906, 16844, 16032, 16310, 15919, 15904, 16346] avg=16384.0000 max=16906 max_rank=1 min=15904 min_rank=6 imbalance_ratio=1.031860 max_load_reduction=114166 max_load_reduction_pct=0.871017 moved_to_echo=64479 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3545, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3233, 4096, 0, 0, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4093, 0, 0, 0, 0, 3926, 13, 4029, 3437, 657, 357, 80, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3739, 0, 0, 0, 0, 4095, 0, 0, 4096, 218, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4092, 4096, 0, 9, 241, 0, 0, 0, 0, 4096, 3257, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 0, 0, 0, 4021, 0, 0, 0, 2, 4092, 419, 0, 602, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 4096, 0, 263, 0, 0, 0, 4016, 3642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4058, 0, 4096, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16672, 16180, 16634, 16244, 16248, 16632, 16116, 16346] avg=16384.0000 max=16672 max_rank=0 min=16116 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=114400 max_load_reduction_pct=0.872803 moved_to_echo=65342 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2659, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3903, 4096, 0, 0, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 4075, 16, 4030, 3437, 657, 357, 80, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3739, 0, 0, 0, 0, 4096, 0, 0, 4096, 494, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4092, 4096, 0, 19, 103, 0, 0, 0, 0, 4096, 3258, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 0, 0, 0, 4082, 0, 0, 0, 0, 4095, 662, 0, 150, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 302, 0, 0, 0, 4016, 3643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4058, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15785, 16850, 16789, 16521, 16119, 16509, 16153, 16346] avg=16384.0000 max=16850 max_rank=1 min=15785 min_rank=0 imbalance_ratio=1.028442 max_load_reduction=114222 max_load_reduction_pct=0.871445 moved_to_echo=65127 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 0, 0, 0, 256, 0, 0, 128, 32768, 128, 27520, 32640, 128, 128, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 27520, 128, 0, 0, 31872, 128, 128, 128, 128, 32768, 5376, 0, 1024, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 29952, 0, 128, 0, 0, 32768, 128, 128, 32768, 3200, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0, 128, 32512, 384, 32768, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 0, 128, 26240, 128, 128, 128, 128, 128, 0, 128, 32640, 128, 128, 0, 32768, 32768, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 256, 32768, 0, 128, 0, 0, 32768, 128, 4096, 0, 128, 128, 32128, 29184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 32640, 32768, 0, 2048, 1280, 0, 128, 128, 128, 32768, 26112, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 0, 32768, 0, 128, 128, 128, 29056, 128, 32640, 27520, 5376, 2944, 768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4091, 0, 10, 0, 0, 0, 0, 4096, 3587, 0, 0, 2492, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 2987, 0, 0, 0, 1, 4052, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 4085, 0, 0, 0, 4096, 3216, 6, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 730, 0, 0, 5, 0, 4095, 4096, 0, 0, 144, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3268, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 14, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4050, 0, 63, 0, 0, 0, 0, 4096, 2431, 0, 0, 1934, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3758, 0, 0, 0, 1, 4033, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 2084, 24, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 2526, 0, 0, 8, 0, 4091, 4096, 0, 0, 926, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 2735, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[5, 1, 4, 7, 4090, 0, 0, 0, 362, 42, 0, 4086, 4088, 3990, 6, 4089, 4089, 6, 1756, 8, 0, 0, 0, 4085, 1282, 0, 75, 2668, 0, 4079, 0, 19, 0, 7, 5, 0, 4089, 7, 6, 0, 0, 4001, 6, 6, 6, 0, 4090, 0, 3748, 3991, 8, 6, 6, 7, 4090, 0, 5, 0, 4089, 46, 6, 6, 0, 4055, 6, 4089, 0, 6, 2565, 2, 6, 0, 4037, 1906, 0, 4, 4, 0, 0, 3, 4082, 4089, 0, 141, 0, 0, 7, 4037, 0, 0, 6, 6, 9, 4087, 5, 4068, 3137, 0, 5, 593, 0, 4090, 4030, 93, 130, 1893, 0, 5, 19, 21, 3628, 0, 0, 0, 1, 1, 0, 696, 4090, 9, 7, 6, 6, 61, 172, 4084, 7, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4087, 0, 320, 0, 0, 0, 0, 4096, 2706, 0, 0, 1646, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 2679, 0, 0, 0, 1, 3993, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4092, 0, 0, 0, 4096, 3673, 11, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 1812, 0, 0, 9, 0, 4088, 4096, 0, 0, 592, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 3045, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 15, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4094, 0, 160, 0, 0, 0, 0, 4096, 3056, 0, 0, 1062, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3124, 0, 0, 0, 1, 3777, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2708, 39, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 2429, 0, 0, 8, 0, 4080, 4096, 0, 0, 2441, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 1689, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 11, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4085, 0, 159, 0, 0, 0, 0, 4096, 1482, 0, 0, 2467, 0, 4096, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 3705, 0, 0, 0, 3, 4041, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 3983, 49, 0, 0, 0, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 1642, 0, 0, 6, 0, 4068, 4096, 0, 0, 995, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 1979, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4065, 0, 201, 0, 0, 0, 0, 4096, 3227, 0, 0, 1080, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3929, 0, 0, 0, 0, 4078, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4094, 0, 0, 0, 4096, 2090, 2, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 1860, 0, 0, 4, 0, 4096, 4096, 0, 0, 1551, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2491, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4095, 0, 78, 0, 0, 0, 0, 4096, 1327, 0, 0, 1899, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3466, 0, 0, 0, 3, 3967, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3035, 8, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 2412, 0, 0, 1, 0, 4095, 4096, 0, 0, 1612, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 2677, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 4052, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 10, 0, 0, 0, 0, 3560, 3587, 0, 0, 2492, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 2987, 0, 0, 0, 1, 4052, 0, 2469, 2352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 53, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4085, 0, 0, 0, 4096, 3216, 6, 0, 0, 0, 0, 0, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 0, 0, 5, 0, 4095, 4096, 0, 0, 144, 0, 0, 0, 0, 4093, 0, 536, 134, 44, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3268, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16342, 17836, 15958, 16437, 16938, 16376, 13885, 17300] avg=16384.0000 max=17836 max_rank=1 min=13885 min_rank=6 imbalance_ratio=1.088623 max_load_reduction=113236 max_load_reduction_pct=0.863922 moved_to_echo=64499 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 14, 0, 0, 0, 4096, 4052, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 63, 0, 0, 0, 0, 3560, 2431, 0, 0, 1934, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3758, 0, 0, 0, 1, 4033, 0, 2469, 2353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 53, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 2084, 24, 0, 0, 0, 0, 0, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2526, 0, 0, 8, 0, 4091, 4096, 0, 0, 926, 0, 0, 0, 0, 4091, 0, 536, 134, 44, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2735, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16353, 16134, 16710, 16437, 15834, 16378, 16460, 16766] avg=16384.0000 max=16766 max_rank=7 min=15834 min_rank=4 imbalance_ratio=1.023315 max_load_reduction=114306 max_load_reduction_pct=0.872086 moved_to_echo=65438 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 15, 0, 0, 0, 4096, 4052, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 160, 0, 0, 0, 0, 3560, 3056, 0, 0, 1062, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3124, 0, 0, 0, 1, 3777, 0, 2469, 2353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 53, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2708, 39, 0, 0, 0, 0, 0, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2429, 0, 0, 8, 0, 4080, 4096, 0, 0, 2441, 0, 0, 0, 0, 4088, 0, 536, 134, 44, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1689, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16355, 16028, 15820, 16434, 16475, 16376, 17864, 15720] avg=16384.0000 max=17864 max_rank=6 min=15720 min_rank=7 imbalance_ratio=1.090332 max_load_reduction=113208 max_load_reduction_pct=0.863708 moved_to_echo=66435 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 11, 0, 0, 0, 4096, 4052, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 159, 0, 0, 0, 0, 3560, 1482, 0, 0, 2467, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 3705, 0, 0, 0, 3, 4041, 0, 2469, 2353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 53, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 3983, 49, 0, 0, 0, 0, 0, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1642, 0, 0, 6, 0, 4068, 4096, 0, 0, 995, 0, 0, 0, 0, 4090, 0, 536, 134, 44, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1979, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16349, 15849, 16671, 16436, 17759, 16379, 15619, 16010] avg=16384.0000 max=17759 max_rank=4 min=15619 min_rank=6 imbalance_ratio=1.083923 max_load_reduction=113313 max_load_reduction_pct=0.864510 moved_to_echo=65767 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 0, 4096, 4052, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 320, 0, 0, 0, 0, 3560, 2706, 0, 0, 1646, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 2679, 0, 0, 0, 1, 3993, 0, 2469, 2352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 53, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 4096, 3673, 11, 0, 0, 0, 0, 0, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1812, 0, 0, 9, 0, 4088, 4096, 0, 0, 592, 0, 0, 0, 0, 4090, 0, 536, 134, 44, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3045, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16358, 16415, 15592, 16437, 17408, 16376, 15409, 17077] avg=16384.0000 max=17408 max_rank=4 min=15409 min_rank=6 imbalance_ratio=1.062500 max_load_reduction=113664 max_load_reduction_pct=0.867188 moved_to_echo=66270 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4052, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 78, 0, 0, 0, 0, 3560, 1327, 0, 0, 1899, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3466, 0, 0, 0, 3, 3967, 0, 2469, 2353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 53, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3035, 8, 0, 0, 0, 0, 0, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2412, 0, 0, 1, 0, 4095, 4096, 0, 0, 1612, 0, 0, 0, 0, 4093, 0, 536, 134, 44, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2677, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16340, 15055, 16354, 16437, 16771, 16376, 17031, 16708] avg=16384.0000 max=17031 max_rank=6 min=15055 min_rank=1 imbalance_ratio=1.039490 max_load_reduction=114041 max_load_reduction_pct=0.870064 moved_to_echo=66886 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4052, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 201, 0, 0, 0, 0, 3560, 3227, 0, 0, 1080, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3929, 0, 0, 0, 0, 4078, 0, 2469, 2353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 53, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 4096, 2090, 2, 0, 0, 0, 0, 0, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1860, 0, 0, 4, 0, 4096, 4096, 0, 0, 1551, 0, 0, 0, 0, 4096, 0, 536, 134, 44, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2491, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16340, 16229, 16925, 16437, 15818, 16376, 16425, 16522] avg=16384.0000 max=16925 max_rank=2 min=15818 min_rank=4 imbalance_ratio=1.033020 max_load_reduction=114147 max_load_reduction_pct=0.870872 moved_to_echo=65141 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[5, 1, 4, 7, 4090, 0, 0, 0, 362, 42, 0, 4086, 4042, 3990, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 6, 1756, 8, 0, 0, 0, 3544, 1282, 0, 75, 2668, 0, 4079, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 0, 4089, 7, 6, 0, 0, 4001, 6, 6, 6, 0, 4090, 0, 2470, 2349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 3991, 8, 6, 6, 7, 44, 0, 5, 0, 4089, 46, 6, 6, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4089, 0, 6, 2565, 2, 6, 0, 4037, 1906, 0, 4, 4, 0, 0, 3, 1441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 141, 0, 0, 7, 4037, 0, 0, 6, 6, 9, 4074, 5, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3137, 0, 5, 593, 0, 4090, 4030, 93, 130, 1893, 0, 5, 19, 21, 3628, 0, 541, 135, 46, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 696, 4090, 9, 7, 6, 6, 61, 172, 4084, 7, 0, 4089, 1740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16635, 17526, 17042, 16017, 14069, 16435, 18379, 14969] avg=16384.0000 max=18379 max_rank=6 min=14069 min_rank=4 imbalance_ratio=1.121765 max_load_reduction=112693 max_load_reduction_pct=0.859779 moved_to_echo=63852 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 2816, 128, 0, 0, 0, 28544, 19200, 0, 128, 15360, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 128, 384, 128, 0, 32768, 32512, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 0, 128, 31232, 128, 128, 0, 32768, 22784, 256, 128, 128, 0, 0, 128, 11520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32512, 32768, 128, 128, 128, 128, 512, 0, 128, 0, 32768, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 0, 128, 256, 0, 128, 128, 32768, 0, 0, 128, 128, 128, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 128, 0, 0, 27776, 128, 128, 128, 128, 32128, 0, 19840, 18944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 0, 18688, 32768, 128, 128, 128, 128, 128, 256, 32768, 128, 0, 32768, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[16640, 0, 128, 640, 0, 32768, 32768, 128, 256, 10240, 0, 128, 128, 128, 32384, 0, 4352, 1152, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 43, 0, 13, 0, 4096, 0, 0, 4096, 0, 4072, 0, 4096, 0, 0, 0, 44, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 414, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 3567, 0, 0, 0, 0, 0, 0, 0, 4084, 4093, 0, 1, 0, 0, 1, 0, 0, 82, 0, 4096, 0, 4096, 4093, 0, 0, 5, 4096, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 351, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 138, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3148, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 471, 0, 4096, 0, 4096, 4096, 0, 0, 1, 4096, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4086, 0, 0, 14, 4077, 788, 0, 318, 0, 4015, 8, 0, 4088, 6, 1506, 6, 4083, 0, 0, 0, 2806, 10, 0, 6, 8, 4090, 4088, 0, 7, 4088, 4047, 4020, 0, 16, 10, 5, 7, 0, 0, 6, 7, 4090, 60, 18, 2, 34, 5, 0, 3972, 0, 0, 6, 4082, 7, 3854, 162, 0, 0, 4056, 4089, 2, 6, 6, 0, 0, 13, 3921, 52, 7, 8, 4069, 4085, 10, 0, 0, 5, 8, 0, 14, 6, 3960, 2, 4, 6, 1, 670, 5, 0, 4079, 4089, 0, 73, 0, 4, 0, 5, 0, 239, 6, 4089, 0, 3958, 4090, 0, 0, 146, 4090, 5, 4091, 0, 0, 0, 7, 6, 0, 7, 4090, 0, 0, 0, 4, 4089, 4072, 2308, 12, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 13, 0, 0, 0, 4096, 0, 0, 4096, 0, 3987, 0, 4096, 0, 0, 0, 499, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 454, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3246, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 1, 0, 0, 0, 0, 0, 19, 0, 4096, 0, 4096, 4096, 0, 0, 1, 4096, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 11, 0, 0, 0, 4096, 0, 0, 4096, 0, 3979, 0, 4096, 0, 0, 0, 453, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 128, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 24, 0, 4096, 0, 0, 1, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 3637, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 0, 0, 0, 0, 2, 0, 0, 8, 0, 4096, 0, 4096, 4095, 0, 0, 4, 4096, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 2, 0, 5, 0, 4096, 0, 0, 4096, 0, 4034, 0, 4096, 0, 0, 0, 114, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 257, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 3770, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 0, 1, 0, 0, 45, 0, 4096, 0, 4096, 4096, 0, 0, 3, 4096, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 4096, 4086, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 7, 0, 3, 0, 4096, 0, 0, 4096, 0, 4090, 0, 4096, 0, 0, 0, 535, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 1496, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 1918, 0, 0, 0, 0, 0, 0, 0, 4088, 4094, 0, 0, 0, 0, 2, 0, 0, 179, 0, 4096, 0, 4096, 4095, 0, 0, 3, 4096, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 5, 0, 19, 0, 4096, 0, 0, 4096, 0, 3990, 0, 4096, 0, 0, 0, 1476, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 771, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1618, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 1, 0, 0, 0, 0, 0, 354, 0, 4096, 0, 4096, 4096, 0, 0, 7, 4096, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 234, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 351, 0, 0, 0, 0, 0, 3326, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3409, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3148, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4071, 471, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 0, 3946, 0, 4096, 4096, 0, 0, 1, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 150, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16617, 15965, 16051, 16409, 16390, 16335, 16695, 16610] avg=16384.0000 max=16695 max_rank=6 min=15965 min_rank=1 imbalance_ratio=1.018982 max_load_reduction=114377 max_load_reduction_pct=0.872627 moved_to_echo=66030 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 11, 0, 0, 0, 4096, 0, 0, 234, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 453, 0, 0, 0, 0, 0, 3326, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 4096, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 4096, 0, 0, 1, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 3409, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3637, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 0, 0, 0, 0, 2, 0, 4071, 471, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3946, 0, 4096, 4095, 0, 0, 4, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 150, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16512, 16067, 16037, 16411, 16416, 16820, 16202, 16607] avg=16384.0000 max=16820 max_rank=5 min=16037 min_rank=2 imbalance_ratio=1.026611 max_load_reduction=114252 max_load_reduction_pct=0.871674 moved_to_echo=66045 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 13, 0, 0, 0, 4096, 0, 0, 234, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 499, 0, 0, 0, 0, 0, 3326, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3409, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3246, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 1, 0, 0, 0, 0, 4071, 472, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 3946, 0, 4096, 4096, 0, 0, 1, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 150, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16522, 16113, 16366, 16409, 16390, 16434, 16228, 16610] avg=16384.0000 max=16610 max_rank=7 min=16113 min_rank=1 imbalance_ratio=1.013794 max_load_reduction=114462 max_load_reduction_pct=0.873276 moved_to_echo=65662 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4086, 0, 0, 14, 4077, 788, 0, 318, 0, 4015, 8, 0, 226, 6, 1506, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 2806, 10, 0, 6, 8, 0, 3315, 0, 7, 0, 4047, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 10, 5, 7, 0, 0, 6, 7, 4090, 60, 18, 2, 34, 5, 0, 4090, 4090, 3618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 0, 0, 6, 4082, 7, 3854, 162, 0, 0, 4056, 22, 2, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3921, 52, 7, 8, 4069, 4085, 10, 0, 0, 5, 8, 0, 14, 6, 3407, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 2, 4, 6, 1, 670, 5, 0, 4079, 4089, 0, 73, 0, 4, 0, 5, 4067, 470, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 6, 3939, 0, 3958, 4090, 0, 0, 146, 0, 5, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 7, 4090, 0, 0, 0, 4, 4089, 4072, 2308, 12, 0, 0, 0, 150, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15050, 18302, 16058, 16175, 16300, 17890, 16474, 14823] avg=16384.0000 max=18302 max_rank=1 min=14823 min_rank=7 imbalance_ratio=1.117065 max_load_reduction=112770 max_load_reduction_pct=0.860367 moved_to_echo=65487 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 43, 0, 13, 0, 4096, 0, 0, 234, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 44, 0, 0, 0, 0, 0, 3326, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 414, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 4096, 4096, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 4096, 0, 4096, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 3409, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3567, 0, 0, 0, 0, 0, 0, 0, 4084, 4093, 0, 1, 0, 0, 1, 0, 4071, 471, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 3946, 0, 4096, 4093, 0, 0, 5, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 150, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16650, 15658, 16315, 16424, 16395, 16741, 16282, 16607] avg=16384.0000 max=16741 max_rank=5 min=15658 min_rank=1 imbalance_ratio=1.021790 max_load_reduction=114331 max_load_reduction_pct=0.872276 moved_to_echo=66025 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 2, 0, 5, 0, 4096, 0, 0, 234, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 114, 0, 0, 0, 0, 0, 3326, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 4096, 3624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 3409, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3770, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 0, 1, 0, 4071, 472, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 3946, 0, 4096, 4096, 0, 0, 3, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 4096, 4086, 0, 0, 0, 0, 150, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16563, 15728, 16165, 16411, 16391, 16956, 16260, 16598] avg=16384.0000 max=16956 max_rank=5 min=15728 min_rank=1 imbalance_ratio=1.034912 max_load_reduction=114116 max_load_reduction_pct=0.870636 moved_to_echo=66205 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 5, 0, 19, 0, 4096, 0, 0, 234, 0, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1476, 0, 0, 0, 0, 0, 3326, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 771, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 4096, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3409, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1618, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 1, 0, 0, 0, 0, 4071, 471, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 3946, 0, 4096, 4096, 0, 0, 7, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 150, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16536, 17090, 16679, 16414, 16394, 14798, 16552, 16609] avg=16384.0000 max=17090 max_rank=1 min=14798 min_rank=5 imbalance_ratio=1.043091 max_load_reduction=113982 max_load_reduction_pct=0.869614 moved_to_echo=64353 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 7, 0, 3, 0, 4096, 0, 0, 234, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 535, 0, 0, 0, 0, 0, 3326, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1496, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 4096, 3624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 4096, 0, 4096, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 3409, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1918, 0, 0, 0, 0, 0, 0, 0, 4088, 4094, 0, 0, 0, 0, 2, 0, 4071, 472, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 3946, 0, 4096, 4095, 0, 0, 3, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 150, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16622, 16149, 17401, 16419, 16396, 15098, 16379, 16608] avg=16384.0000 max=17401 max_rank=2 min=15098 min_rank=5 imbalance_ratio=1.062073 max_load_reduction=113671 max_load_reduction_pct=0.867241 moved_to_echo=64481 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[24960, 128, 128, 128, 128, 768, 128, 0, 32768, 32768, 0, 128, 0, 128, 128, 128, 32640, 3840, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 0, 0, 6400, 128, 0, 128, 128, 0, 26624, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 0, 128, 32768, 896, 0, 384, 0, 32768, 128, 0, 1920, 128, 29824, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 1408, 128, 31616, 0, 32640, 32768, 0, 0, 256, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 0, 128, 128, 32768, 128, 32640, 256, 0, 0, 32768, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 3712, 128, 128, 128, 0, 0, 128, 128, 32768, 128, 128, 128, 128, 128, 0, 32768, 32768, 29056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 32640, 128, 128, 128, 32768, 32768, 128, 128, 0, 128, 128, 0, 128, 128, 27392, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0, 128, 32768, 32768, 30976, 128, 0, 0, 0, 1280, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 995, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4095, 4096, 0, 0, 964, 0, 0, 4096, 0, 0, 0, 4096, 2139, 0, 0, 4096, 0, 4094, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2, 0, 4078, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 1434, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 4096, 4094, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4095, 4095, 0, 0, 4096, 0, 4096, 0, 0, 4095, 4092, 4096, 0, 0, 235, 0, 0, 4096, 1, 0, 0, 4096, 2449, 0, 0, 4096, 0, 4095, 4083, 0, 0, 0, 4095, 9, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4, 8, 4002, 7, 1, 1, 0, 0, 4083, 6, 6, 242, 25, 6, 61, 13, 4069, 7, 4089, 0, 0, 0, 0, 4088, 0, 4, 4085, 6, 7, 78, 4048, 5, 7, 10, 0, 0, 4053, 0, 46, 2, 313, 3755, 28, 5, 4071, 3, 0, 28, 0, 0, 0, 0, 0, 4049, 1, 4086, 4077, 0, 6, 0, 0, 4089, 4089, 0, 0, 0, 3, 0, 0, 7, 0, 7, 6, 4076, 6, 7, 7, 20, 4089, 4089, 116, 44, 4089, 16, 4084, 5, 0, 4090, 4090, 4028, 5, 0, 300, 0, 0, 4089, 0, 5, 2, 3998, 1567, 1, 238, 2850, 3, 4088, 4089, 0, 0, 5, 4090, 1471, 0, 6, 0, 0, 0, 8, 20, 0, 7, 6, 3932, 1679, 4060, 0, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 10, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 501, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4095, 4096, 0, 0, 1358, 0, 0, 4096, 1, 0, 0, 4096, 2243, 0, 0, 4096, 0, 4089, 4095, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4, 0, 4067, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 621, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 5, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4091, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4089, 4096, 0, 0, 876, 0, 0, 4096, 3, 0, 0, 4096, 2628, 0, 0, 4096, 0, 4089, 4093, 0, 0, 0, 4094, 6, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 512, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 390, 0, 0, 4096, 0, 0, 0, 4096, 3194, 0, 0, 4096, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4082, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 824, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 4096, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4093, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4095, 4095, 4096, 0, 0, 517, 0, 0, 4096, 0, 0, 0, 4096, 2772, 0, 0, 4096, 0, 4094, 4076, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2067, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 400, 0, 0, 4096, 0, 0, 0, 4096, 1631, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 4096, 2, 0, 4078, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 1434, 4096, 0, 0, 4096, 0, 0, 0, 2371, 639, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 11, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4095, 4095, 3873, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3456, 4092, 4096, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 2449, 0, 0, 4096, 0, 1724, 4083, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 3866, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16352, 16375, 16945, 16395, 16381, 15975, 16458, 16191] avg=16384.0000 max=16945 max_rank=2 min=15975 min_rank=5 imbalance_ratio=1.034241 max_load_reduction=114127 max_load_reduction_pct=0.870720 moved_to_echo=65005 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 4096, 2, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 995, 4096, 0, 0, 4096, 0, 0, 0, 2371, 639, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 11, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 4096, 3873, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3457, 4095, 4096, 0, 0, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2139, 0, 0, 4096, 0, 1723, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 3866, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16351, 16392, 16505, 16393, 16381, 16708, 16151, 16191] avg=16384.0000 max=16708 max_rank=5 min=16151 min_rank=6 imbalance_ratio=1.019775 max_load_reduction=114364 max_load_reduction_pct=0.872528 moved_to_echo=65431 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 4096, 10, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 501, 4096, 0, 0, 4096, 0, 0, 0, 2368, 639, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 11, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 4096, 3873, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3457, 4095, 4096, 0, 0, 1358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 2243, 0, 0, 4096, 0, 1721, 4095, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 3866, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16350, 16389, 16008, 16390, 16378, 17102, 16260, 16195] avg=16384.0000 max=17102 max_rank=5 min=16008 min_rank=2 imbalance_ratio=1.043823 max_load_reduction=113970 max_load_reduction_pct=0.869522 moved_to_echo=65935 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2067, 4096, 0, 0, 4096, 0, 0, 0, 2372, 639, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4096, 3873, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3457, 4096, 4096, 0, 0, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1631, 0, 0, 4096, 0, 1724, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 3866, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16350, 16386, 17578, 16395, 16382, 16145, 15643, 16193] avg=16384.0000 max=17578 max_rank=2 min=15643 min_rank=6 imbalance_ratio=1.072876 max_load_reduction=113494 max_load_reduction_pct=0.865891 moved_to_echo=64363 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4, 8, 4002, 7, 1, 1, 0, 0, 4083, 6, 6, 242, 25, 6, 61, 13, 4089, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 7, 0, 0, 0, 0, 0, 4088, 0, 4, 4085, 6, 7, 78, 4048, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 10, 0, 0, 4053, 0, 46, 2, 313, 3755, 28, 5, 4071, 3, 0, 28, 2369, 645, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 1, 4086, 4077, 0, 6, 0, 0, 2, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7, 0, 7, 6, 4076, 6, 7, 7, 20, 4089, 4089, 3871, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 44, 0, 16, 4084, 5, 0, 3445, 4090, 4028, 5, 0, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 3998, 1567, 1, 238, 2850, 3, 1719, 4089, 0, 0, 5, 0, 1471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8, 20, 0, 7, 6, 3932, 1679, 4060, 0, 5, 0, 4090, 3864, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16610, 16397, 15551, 16310, 16413, 16133, 15948, 17710] avg=16384.0000 max=17710 max_rank=7 min=15551 min_rank=2 imbalance_ratio=1.080933 max_load_reduction=113362 max_load_reduction_pct=0.864883 moved_to_echo=66204 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 512, 4096, 0, 0, 4096, 0, 0, 0, 2372, 639, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 3873, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3457, 4096, 4096, 0, 0, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3194, 0, 0, 4096, 0, 1724, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 3866, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16351, 16393, 16023, 16395, 16383, 16135, 17204, 16188] avg=16384.0000 max=17204 max_rank=6 min=16023 min_rank=2 imbalance_ratio=1.050049 max_load_reduction=113868 max_load_reduction_pct=0.868744 moved_to_echo=65910 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 4096, 4, 0, 4067, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 621, 4096, 0, 0, 4096, 0, 0, 0, 2368, 639, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 5, 0, 11, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4091, 4096, 3873, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3457, 4089, 4096, 0, 0, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 2628, 0, 0, 4096, 0, 1721, 4093, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4094, 3869, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16356, 16362, 16128, 16398, 16374, 16614, 16643, 16197] avg=16384.0000 max=16643 max_rank=6 min=16128 min_rank=2 imbalance_ratio=1.015808 max_load_reduction=114429 max_load_reduction_pct=0.873024 moved_to_echo=65828 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 4096, 0, 0, 4082, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 824, 4096, 0, 0, 4096, 0, 0, 0, 2371, 639, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 11, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4093, 4096, 3873, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3456, 4095, 4096, 0, 0, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2772, 0, 0, 4096, 0, 1723, 4076, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 4096, 3866, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16352, 16378, 16334, 16396, 16380, 16260, 16765, 16207] avg=16384.0000 max=16765 max_rank=6 min=16207 min_rank=7 imbalance_ratio=1.023254 max_load_reduction=114307 max_load_reduction_pct=0.872093 moved_to_echo=65612 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 128, 0, 32768, 128, 128, 256, 128, 128, 128, 128, 32768, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 32768, 128, 32768, 32768, 0, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 32768, 128, 128, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 0, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 32768, 32768, 31104, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 32768, 18688, 128, 256, 31616, 128, 13824, 32768, 0, 0, 128, 0, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 0, 128, 128, 128, 0, 128, 128, 32640, 1792, 32768, 0, 128, 0, 32768, 30976, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 128, 128, 7296, 32512, 128, 128, 32768, 128, 0, 128, 19072, 5120, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 27648, 32768, 32768, 128, 0, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 535, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 3796, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 41, 0, 4091, 0, 0, 0, 2866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 955, 4095, 0, 0, 4096, 4096, 0, 4095, 1, 4096, 4096, 1, 4096, 0, 1, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 264, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3908, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 235, 0, 4096, 0, 0, 0, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2483, 4096, 0, 0, 4096, 4096, 0, 4096, 1, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 505, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 3787, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 371, 0, 4087, 0, 0, 0, 2138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1401, 4094, 0, 0, 4096, 4096, 0, 4094, 3, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4094, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 293, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 3920, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 55, 0, 4092, 0, 0, 0, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3521, 4096, 0, 0, 4096, 4096, 0, 4095, 3, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 6, 7, 0, 4087, 0, 0, 5, 52, 0, 4070, 3, 0, 0, 1327, 4086, 0, 4066, 0, 4080, 0, 11, 5, 0, 115, 4090, 0, 0, 7, 6, 0, 0, 0, 6, 4044, 4081, 3948, 8, 5, 1, 6, 4085, 0, 6, 7, 5, 13, 3, 27, 4084, 18, 0, 378, 6, 15, 0, 0, 8, 4080, 6, 3, 10, 3281, 2, 0, 168, 4, 11, 52, 6, 6, 6, 39, 4090, 5, 7, 3877, 4035, 5, 4090, 0, 3972, 4076, 231, 4089, 0, 0, 186, 0, 5, 3653, 4063, 0, 0, 3184, 0, 8, 0, 6, 0, 24, 0, 4089, 4089, 3929, 7, 4089, 0, 4074, 0, 4089, 4083, 0, 6, 0, 7, 4090, 359, 0, 4088, 0, 6, 2, 0, 3, 0, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 244, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 3455, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 30, 0, 4093, 0, 0, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3064, 4093, 0, 0, 4096, 4096, 0, 4090, 3, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 412, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 3599, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 15, 0, 4088, 0, 0, 0, 2828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1347, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4080, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 14] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 213, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3909, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 25, 0, 4096, 0, 0, 0, 3000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 10] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 535, 4096, 2200, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2331, 1753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3796, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 41, 0, 4091, 0, 0, 0, 2866, 0, 4096, 1742, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 955, 2368, 0, 0, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 143, 1, 0, 0, 1, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16442, 16373, 16392, 16985, 15610, 16529, 16406, 16335] avg=16384.0000 max=16985 max_rank=3 min=15610 min_rank=4 imbalance_ratio=1.036682 max_load_reduction=114087 max_load_reduction_pct=0.870415 moved_to_echo=64880 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 264, 4096, 2200, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2331, 1753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3908, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 235, 0, 4096, 0, 0, 0, 1303, 0, 4096, 1742, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2483, 2369, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 143, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16170, 16372, 16504, 15621, 17140, 16521, 16407, 16337] avg=16384.0000 max=17140 max_rank=4 min=15621 min_rank=3 imbalance_ratio=1.046143 max_load_reduction=113932 max_load_reduction_pct=0.869232 moved_to_echo=66405 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 7, 0, 4087, 0, 0, 5, 52, 0, 4070, 3, 0, 0, 1327, 4086, 2197, 1420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 4080, 0, 11, 5, 0, 115, 4090, 0, 0, 7, 6, 0, 0, 2335, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4044, 4081, 3948, 8, 5, 1, 6, 4085, 0, 6, 7, 5, 13, 3, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4084, 18, 0, 378, 6, 15, 0, 0, 8, 4080, 6, 3, 10, 3281, 2, 4089, 1747, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 4, 11, 52, 6, 6, 6, 39, 2361, 5, 7, 3877, 4035, 5, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 134, 231, 0, 0, 0, 186, 0, 5, 3653, 4063, 0, 0, 3184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 6, 0, 24, 0, 0, 0, 3929, 7, 7, 0, 4074, 0, 4089, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7, 4090, 359, 0, 4088, 0, 6, 2, 0, 3, 0, 5, 0, 4089, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17260, 16460, 16527, 17807, 14672, 15428, 16227, 16691] avg=16384.0000 max=17807 max_rank=3 min=14672 min_rank=4 imbalance_ratio=1.086853 max_load_reduction=113265 max_load_reduction_pct=0.864143 moved_to_echo=63018 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 505, 4096, 2200, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 2331, 1753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3787, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 371, 0, 4087, 0, 0, 0, 2138, 0, 4096, 1742, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1401, 2368, 0, 0, 4096, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 143, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 16374, 16383, 16582, 16055, 16520, 16407, 16340] avg=16384.0000 max=16582 max_rank=3 min=16055 min_rank=4 imbalance_ratio=1.012085 max_load_reduction=114490 max_load_reduction_pct=0.873489 moved_to_echo=65322 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 412, 4096, 2200, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2331, 1753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3599, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 15, 0, 4088, 0, 0, 0, 2828, 0, 4096, 1742, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1347, 2369, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 143, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16319, 16373, 16195, 16918, 16004, 16511, 16407, 16345] avg=16384.0000 max=16918 max_rank=3 min=16004 min_rank=4 imbalance_ratio=1.032593 max_load_reduction=114154 max_load_reduction_pct=0.870926 moved_to_echo=65267 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 293, 4096, 2200, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2331, 1753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3920, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 55, 0, 4092, 0, 0, 0, 399, 0, 4096, 1742, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3521, 2369, 0, 0, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 143, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16200, 16373, 16516, 14533, 18177, 16523, 16407, 16343] avg=16384.0000 max=18177 max_rank=4 min=14533 min_rank=3 imbalance_ratio=1.109436 max_load_reduction=112895 max_load_reduction_pct=0.861320 moved_to_echo=67450 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 244, 4096, 2200, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 2331, 1753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3455, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 30, 0, 4093, 0, 0, 0, 1408, 0, 4096, 1742, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3064, 2367, 0, 0, 4096, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 143, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16150, 16375, 16051, 15518, 17713, 16521, 16404, 16340] avg=16384.0000 max=17713 max_rank=4 min=15518 min_rank=3 imbalance_ratio=1.081116 max_load_reduction=113359 max_load_reduction_pct=0.864861 moved_to_echo=66978 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 213, 4096, 2200, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2331, 1753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3909, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 25, 0, 4096, 0, 0, 0, 3000, 0, 4096, 1742, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 2369, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 143, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16120, 16372, 16504, 17108, 15701, 16519, 16407, 16341] avg=16384.0000 max=17108 max_rank=3 min=15701 min_rank=4 imbalance_ratio=1.044189 max_load_reduction=113964 max_load_reduction_pct=0.869476 moved_to_echo=64968 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 32768, 384, 128, 32768, 0, 128, 128, 0, 128, 0, 128, 128, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 0, 32768, 0, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 0, 0, 18688, 14080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 0, 384, 128, 128, 0, 896, 128, 32768, 128, 128, 128, 17280, 128, 32768, 13952, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 128, 128, 0, 32768, 128, 0, 0, 3840, 32768, 17664, 11392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 32768, 1152, 256, 0, 0, 128, 256, 0, 128, 32384, 32768, 0, 0, 31872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 32768, 30464, 32640, 128, 128, 128, 128, 32768, 0, 128, 128, 128, 128, 128, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 32640, 128, 256, 0, 32768, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 256, 128, 128, 128, 128, 128, 128, 13952, 18944, 128, 128, 32640, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 2, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 4095, 0, 0, 4083, 4096, 0, 4095, 0, 4011, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3, 4096, 4095, 4096, 0, 0, 0, 33, 0, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4085, 0, 73, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 0, 11, 0, 2, 0, 2484, 0, 0, 4086, 111, 5, 0, 0, 4089, 4080, 8, 0, 0, 4088, 0, 0, 5, 4054, 63, 10, 4088, 4089, 0, 4085, 42, 3911, 5, 6, 17, 6, 0, 21, 0, 8, 0, 49, 0, 7, 6, 8, 2, 0, 5, 0, 34, 4033, 4015, 33, 3981, 4084, 4089, 0, 0, 0, 0, 0, 0, 9, 7, 0, 6, 0, 0, 4033, 52, 8, 0, 0, 8, 6, 4055, 4068, 3996, 0, 2784, 0, 1913, 10, 0, 5, 4082, 73, 7, 0, 0, 4067, 5, 6, 4089, 4087, 0, 8, 4089, 0, 0, 5, 4087, 7, 9, 7, 0, 143, 9, 4087, 4075, 0, 0, 4090, 3, 0, 6, 1158, 7, 15, 4, 4088, 4090, 3982] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3665, 2, 0, 4093, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 4082, 4096, 0, 4096, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 9, 4095, 4095, 4096, 0, 0, 0, 29, 0, 0, 0, 0, 213, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4084, 0, 259, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4086, 4096, 0, 4096, 0, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 4096, 4096, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 53, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4037, 0, 265, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 4074, 4096, 0, 4094, 0, 4025, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 7, 4096, 4095, 4096, 0, 0, 0, 37, 0, 0, 0, 0, 156, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 3895, 0, 174, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 4094, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4090, 4096, 0, 4096, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4096, 4096, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 3952, 0, 192, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 0, 0, 4090, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 4066, 4096, 0, 4090, 0, 3604, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 11, 4096, 4093, 4096, 0, 0, 0, 32, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4053, 0, 669, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 4096, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4083, 4096, 0, 4096, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 14, 1, 0, 0, 0, 24, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4088, 0, 202, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 2, 0, 4095, 0, 0, 4096, 4096, 210, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 4095, 0, 0, 4083, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4011, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3874, 235, 129, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3, 4096, 4095, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4085, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16663, 16383, 16442, 16405, 16350, 16211, 16365, 16253] avg=16384.0000 max=16663 max_rank=0 min=16211 min_rank=5 imbalance_ratio=1.017029 max_load_reduction=114409 max_load_reduction_pct=0.872871 moved_to_echo=65179 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 4096, 0, 0, 4096, 4096, 210, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4086, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3874, 235, 129, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 4096, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 53, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4037, 0, 265, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16636, 16386, 16308, 16408, 16359, 16355, 16364, 16256] avg=16384.0000 max=16636 max_rank=0 min=16256 min_rank=7 imbalance_ratio=1.015381 max_load_reduction=114436 max_load_reduction_pct=0.873077 moved_to_echo=65334 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 4094, 0, 0, 4096, 4096, 210, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4090, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3874, 235, 129, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4096, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3952, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16673, 16389, 16460, 16407, 16328, 16197, 16364, 16254] avg=16384.0000 max=16673 max_rank=0 min=16197 min_rank=5 imbalance_ratio=1.017639 max_load_reduction=114399 max_load_reduction_pct=0.872795 moved_to_echo=65143 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3665, 2, 0, 4093, 0, 0, 4096, 4096, 210, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 4082, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3874, 235, 129, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 9, 4095, 4095, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 213, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4084, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16245, 16383, 16480, 16411, 16543, 16396, 16364, 16250] avg=16384.0000 max=16543 max_rank=4 min=16245 min_rank=0 imbalance_ratio=1.009705 max_load_reduction=114529 max_load_reduction_pct=0.873787 moved_to_echo=65553 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 0, 11, 0, 2, 0, 2484, 0, 0, 4086, 111, 5, 4082, 4068, 210, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 8, 0, 0, 4088, 0, 0, 5, 4054, 63, 10, 4088, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 42, 3911, 5, 6, 17, 6, 0, 21, 0, 8, 0, 49, 0, 7, 4072, 3871, 237, 135, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8, 2, 0, 5, 0, 34, 4033, 4015, 33, 3981, 4084, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 7, 0, 6, 0, 0, 4033, 52, 8, 0, 0, 8, 6, 4089, 4088, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 3996, 0, 2784, 0, 1913, 10, 0, 5, 0, 73, 7, 0, 0, 3830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 4089, 4087, 0, 8, 4067, 0, 0, 5, 4087, 7, 9, 7, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3952, 4075, 0, 0, 4090, 3, 0, 6, 1158, 7, 15, 4, 0, 0, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15150, 16404, 16494, 16218, 16312, 16673, 16520, 17301] avg=16384.0000 max=17301 max_rank=7 min=15150 min_rank=0 imbalance_ratio=1.055969 max_load_reduction=113771 max_load_reduction_pct=0.868004 moved_to_echo=66806 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 4096, 0, 0, 4096, 4096, 210, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4083, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3874, 235, 129, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 0, 24, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4088, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16584, 16383, 16400, 16403, 16340, 16343, 16364, 16255] avg=16384.0000 max=16584 max_rank=0 min=16255 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=114488 max_load_reduction_pct=0.873474 moved_to_echo=65302 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 4096, 0, 0, 4096, 4096, 210, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 4074, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4025, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3874, 235, 129, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 7, 4096, 4095, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 156, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3895, 0, 174, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16599, 16376, 16455, 16409, 16494, 16122, 16364, 16253] avg=16384.0000 max=16599 max_rank=0 min=16122 min_rank=5 imbalance_ratio=1.013123 max_load_reduction=114473 max_load_reduction_pct=0.873360 moved_to_echo=65233 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 0, 0, 4090, 0, 0, 4096, 4096, 210, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 4066, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3604, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3874, 235, 129, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 11, 4096, 4093, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4053, 0, 669, 0, 0, 0, 0, 0, 0, 0, 0, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16522, 16368, 16033, 16411, 16346, 16775, 16367, 16250] avg=16384.0000 max=16775 max_rank=5 min=16033 min_rank=2 imbalance_ratio=1.023865 max_load_reduction=114297 max_load_reduction_pct=0.872017 moved_to_echo=65738 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 128, 32768, 0, 0, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 32768, 128, 31488, 128, 128, 128, 128, 0, 128, 0, 128, 0, 128, 0, 128, 32640, 31104, 1920, 1152, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 128, 0, 128, 128, 512, 128, 0, 0, 32768, 128, 128, 0, 0, 128, 128, 32768, 32768, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 30464, 128, 0, 32768, 128, 128, 32768, 32768, 1792, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 0, 32768, 0, 31104, 0, 3840, 128, 0, 128, 0, 128, 128, 0, 0, 30976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 32768, 32768, 128, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 31744, 32768, 0, 0, 32768, 128, 0, 128, 1280, 128, 128, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 32640, 0, 0, 128, 32768, 128, 128, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True + [2026-08-30 05:05:03] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 742.7 | throughput per GPU (TFLOP/s/GPU): 448.9 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.902580E+00 | loss scale: 1.0 | grad norm: 5.186 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2194, 1863, 1113, 1, 563, 0, 1742, 1510, 1127, 2106, 1040, 706, 1131, 2810, 0, 1190, 862, 1352, 1151, 641, 12, 823, 159, 1882, 314, 82, 3174, 2039, 1081, 211, 1457, 1359, 168, 2, 2347, 732, 478, 2644, 1499, 1005, 243, 7, 2069, 2619, 1, 1529, 1302, 1147, 1363, 509, 768, 1156, 19, 225, 45, 2047, 0, 259, 28, 651, 1386, 2419, 1661, 203, 579, 11, 0, 1649, 1917, 1998, 1055, 1233, 1074, 1557, 2266, 356, 1353, 9, 1582, 2013, 515, 549, 46, 19, 1647, 535, 1531, 17, 2054, 188, 939, 1009, 237, 1773, 951, 0, 1084, 1074, 13, 1957, 18, 366, 1432, 1161, 267, 1701, 110, 921, 338, 1761, 2334, 1346, 2124, 1778, 517, 911, 2018, 1571, 1748, 1280, 1009, 1842, 1639, 327, 988, 19, 496, 34] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2340, 1733, 1410, 1, 554, 0, 1956, 851, 1478, 1995, 1761, 588, 1086, 2576, 0, 980, 796, 1036, 1131, 778, 49, 1009, 194, 2099, 649, 111, 2769, 1716, 811, 215, 1483, 1092, 97, 0, 2231, 387, 185, 2488, 1037, 842, 451, 18, 1769, 2779, 0, 1888, 877, 919, 1090, 447, 1145, 1527, 42, 142, 4, 1068, 2, 344, 194, 865, 1276, 2568, 1457, 554, 833, 4, 0, 1325, 2078, 1717, 1153, 1069, 1354, 1329, 1535, 327, 1562, 33, 1246, 1940, 950, 702, 9, 17, 1708, 774, 1272, 26, 2531, 231, 1055, 898, 385, 2021, 566, 6, 1318, 968, 1, 1735, 25, 552, 1400, 1273, 320, 1834, 91, 1149, 397, 1364, 2566, 1692, 2014, 1009, 742, 1803, 2094, 1241, 2115, 1507, 1020, 2055, 1283, 762, 1474, 17, 605, 50] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1852, 1613, 1100, 0, 390, 0, 1785, 1000, 1616, 3035, 1722, 276, 1003, 2958, 0, 906, 1065, 1913, 496, 1329, 12, 1066, 189, 2599, 347, 104, 2752, 1553, 1178, 158, 1426, 1114, 172, 1, 1952, 586, 159, 2392, 1351, 713, 339, 24, 1632, 2774, 0, 1667, 1133, 1389, 1156, 662, 986, 989, 4, 107, 4, 1067, 0, 413, 28, 402, 1155, 2462, 1480, 768, 744, 16, 0, 1676, 2366, 1765, 551, 980, 912, 761, 2033, 317, 2357, 16, 1412, 2019, 1176, 434, 6, 5, 2017, 498, 817, 24, 2241, 276, 1409, 658, 447, 1863, 489, 0, 1688, 352, 1, 1667, 9, 665, 1504, 922, 174, 1540, 173, 932, 452, 2584, 2904, 1822, 1894, 1374, 743, 1309, 1851, 913, 1999, 1518, 881, 1876, 1421, 712, 1756, 9, 592, 26] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1466, 1793, 1734, 0, 515, 1, 2128, 619, 1048, 3023, 1491, 807, 1557, 2682, 1, 830, 653, 1348, 856, 911, 18, 550, 194, 2299, 438, 39, 3405, 1870, 1362, 161, 1316, 1204, 435, 3, 1763, 468, 207, 2348, 1426, 1205, 189, 36, 1811, 2209, 0, 1572, 1467, 1104, 961, 536, 701, 905, 11, 98, 7, 1661, 0, 269, 17, 938, 1101, 2586, 1575, 479, 1086, 45, 0, 1206, 1836, 1649, 1476, 1096, 1119, 1558, 2008, 323, 2269, 19, 1394, 1689, 985, 369, 26, 12, 1956, 639, 974, 12, 1785, 269, 1208, 1103, 540, 1994, 727, 0, 1528, 640, 8, 1916, 14, 692, 1194, 880, 229, 1439, 352, 1186, 316, 1610, 2702, 1655, 2367, 1337, 583, 945, 2499, 1711, 1963, 1145, 903, 1790, 1303, 372, 1353, 16, 625, 20] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1658, 2165, 1571, 11, 516, 5, 1691, 946, 1329, 2187, 1239, 420, 1668, 2112, 0, 1305, 798, 1344, 1320, 1227, 16, 770, 228, 2420, 351, 75, 2919, 1705, 1312, 222, 1594, 1101, 240, 21, 2065, 516, 138, 2260, 961, 805, 330, 17, 1949, 2584, 1, 1272, 1921, 1027, 973, 401, 902, 1231, 11, 70, 23, 1964, 7, 488, 128, 646, 1154, 2417, 970, 674, 831, 39, 2, 1606, 2046, 2017, 1344, 1262, 1080, 2005, 2032, 346, 1390, 17, 1080, 1786, 1075, 576, 30, 56, 1679, 704, 1413, 47, 2129, 254, 1375, 886, 388, 1814, 501, 4, 827, 801, 28, 1822, 74, 614, 1990, 1201, 268, 1713, 117, 1137, 209, 1668, 2473, 1730, 2310, 1202, 301, 690, 2078, 938, 1995, 1280, 932, 1903, 1584, 868, 1540, 15, 510, 49] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1751, 1347, 1499, 5, 471, 0, 1909, 1024, 746, 2210, 782, 1151, 1867, 2214, 0, 1321, 647, 1377, 1604, 1274, 15, 666, 229, 2148, 457, 120, 2861, 1908, 1879, 191, 1301, 1093, 206, 0, 1956, 397, 163, 2285, 1354, 833, 310, 0, 2033, 2031, 2, 1265, 1070, 1427, 1288, 531, 676, 967, 17, 148, 85, 1339, 0, 220, 56, 379, 1666, 2263, 1032, 810, 1367, 7, 2, 1668, 1873, 2273, 1167, 1316, 930, 1715, 1883, 704, 1667, 6, 995, 1712, 1139, 699, 36, 14, 2480, 805, 1248, 19, 1523, 383, 924, 1254, 389, 1636, 980, 3, 1501, 1272, 3, 1974, 39, 601, 1507, 1434, 316, 1450, 440, 1457, 786, 718, 1982, 1256, 1469, 1420, 644, 1296, 2449, 1365, 1819, 1588, 917, 1928, 1283, 662, 1159, 17, 604, 23] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1964, 1237, 1968, 1, 541, 0, 2259, 1290, 1263, 2805, 1487, 476, 1409, 2385, 1, 1074, 696, 1439, 722, 1348, 11, 766, 167, 1715, 590, 70, 3223, 1656, 1689, 98, 1790, 1100, 157, 7, 2110, 688, 128, 2765, 1460, 1050, 283, 5, 1916, 2277, 0, 1414, 1499, 1105, 1127, 611, 738, 919, 1, 95, 23, 1061, 0, 464, 95, 384, 1412, 2218, 1103, 494, 1299, 10, 0, 1995, 1454, 2050, 1398, 717, 1380, 1610, 2084, 451, 1369, 23, 1124, 1494, 865, 602, 19, 15, 2115, 439, 1179, 4, 1894, 229, 1299, 732, 437, 1959, 914, 0, 1121, 1109, 8, 2310, 23, 444, 1203, 1187, 253, 1355, 117, 1116, 443, 1540, 2192, 1843, 1993, 1820, 463, 954, 2244, 1497, 2099, 1224, 851, 1555, 1153, 934, 1289, 20, 678, 25] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1871, 1465, 1580, 0, 932, 1, 2353, 891, 883, 1991, 1186, 598, 1392, 1750, 0, 743, 530, 1466, 1567, 1228, 19, 661, 214, 1981, 348, 77, 3132, 1984, 1156, 81, 1571, 1536, 320, 2, 1357, 437, 218, 1899, 1047, 1046, 267, 31, 1893, 1958, 0, 1781, 1400, 1020, 1234, 709, 511, 1195, 21, 54, 20, 1422, 1, 333, 65, 621, 1583, 2291, 1259, 733, 1096, 13, 0, 1961, 2074, 1842, 1103, 1455, 1120, 1679, 2055, 747, 1735, 15, 1024, 1287, 837, 400, 43, 47, 1712, 830, 1902, 36, 2101, 234, 693, 1036, 372, 1694, 743, 1, 1076, 981, 13, 1953, 25, 848, 1378, 1452, 334, 1475, 269, 1286, 1077, 1849, 2230, 1837, 2278, 1506, 309, 766, 2292, 1151, 2151, 1359, 1040, 1806, 1777, 953, 1164, 43, 563, 29] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2194, 1863, 1113, 1, 563, 0, 1742, 1510, 1127, 1920, 1040, 706, 1131, 0, 0, 1190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 862, 1352, 1151, 641, 12, 823, 159, 1882, 314, 82, 2494, 2039, 1081, 211, 1457, 1359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 2, 2347, 732, 478, 2644, 1499, 1005, 243, 7, 2069, 2619, 1, 1529, 1302, 1147, 186, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1363, 509, 768, 1156, 19, 225, 45, 2047, 0, 259, 28, 651, 1386, 2419, 1661, 203, 2810, 1872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 579, 11, 0, 1649, 1707, 1998, 1055, 1233, 1074, 1557, 0, 356, 1353, 9, 1582, 2013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 549, 46, 19, 1647, 535, 1531, 17, 2054, 188, 939, 1009, 237, 1773, 951, 0, 2266, 680, 361, 274, 199, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 1074, 13, 1957, 18, 366, 1432, 1161, 267, 1701, 110, 921, 338, 1761, 1973, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1850, 1778, 517, 911, 0, 1571, 1748, 1280, 1009, 1842, 1639, 327, 988, 19, 496, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16100, 15919, 17989, 17421, 16176, 15936, 15522, 16009] avg=16384.0000 max=17989 max_rank=2 min=15522 min_rank=6 imbalance_ratio=1.097961 max_load_reduction=113083 max_load_reduction_pct=0.862755 moved_to_echo=63643 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2340, 1733, 1410, 1, 554, 0, 1956, 851, 1478, 1819, 1761, 588, 1086, 0, 0, 980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 796, 1036, 1131, 778, 49, 1009, 194, 2099, 649, 111, 2176, 1716, 811, 215, 1483, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 2231, 387, 185, 2488, 1037, 842, 451, 18, 1769, 2779, 0, 1888, 877, 919, 176, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 447, 1145, 1527, 42, 142, 4, 1068, 2, 344, 194, 865, 1276, 2568, 1457, 554, 2576, 1943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 4, 0, 1325, 1851, 1717, 1153, 1069, 1354, 1329, 0, 327, 1562, 33, 1246, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 950, 702, 9, 17, 1708, 774, 1272, 26, 2531, 231, 1055, 898, 385, 2021, 566, 6, 1535, 593, 397, 260, 215, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1318, 968, 1, 1735, 25, 552, 1400, 1273, 320, 1834, 91, 1149, 397, 1364, 2169, 1692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1754, 1009, 742, 1803, 0, 1241, 2115, 1507, 1020, 2055, 1283, 762, 1474, 17, 605, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16557, 15345, 16156, 17244, 15743, 16302, 16288, 17437] avg=16384.0000 max=17437 max_rank=7 min=15345 min_rank=1 imbalance_ratio=1.064270 max_load_reduction=113635 max_load_reduction_pct=0.866966 moved_to_echo=65770 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1658, 2165, 1571, 11, 516, 5, 1691, 946, 1329, 1990, 1239, 420, 1668, 0, 0, 1305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 798, 1344, 1320, 1227, 16, 770, 228, 2420, 351, 75, 2291, 1705, 1312, 222, 1594, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 21, 2065, 516, 138, 2260, 961, 805, 330, 17, 1949, 2584, 1, 1272, 1921, 1027, 197, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 973, 401, 902, 1231, 11, 70, 23, 1964, 7, 488, 128, 646, 1154, 2417, 970, 674, 2112, 1929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 831, 39, 2, 1606, 1814, 2017, 1344, 1262, 1080, 2005, 0, 346, 1390, 17, 1080, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 576, 30, 56, 1679, 704, 1413, 47, 2129, 254, 1375, 886, 388, 1814, 501, 4, 2032, 628, 386, 302, 217, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 827, 801, 28, 1822, 74, 614, 1990, 1201, 268, 1713, 117, 1137, 209, 1668, 2087, 1730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2008, 1202, 301, 690, 0, 938, 1995, 1280, 932, 1903, 1584, 868, 1540, 15, 510, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16514, 16774, 16319, 16100, 16619, 16645, 16286, 15815] avg=16384.0000 max=16774 max_rank=1 min=15815 min_rank=7 imbalance_ratio=1.023804 max_load_reduction=114298 max_load_reduction_pct=0.872025 moved_to_echo=65365 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1466, 1793, 1734, 0, 515, 1, 2128, 619, 1048, 2756, 1491, 807, 1557, 0, 1, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 653, 1348, 856, 911, 18, 550, 194, 2299, 438, 39, 2676, 1870, 1362, 161, 1316, 1204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 3, 1763, 468, 207, 2348, 1426, 1205, 189, 36, 1811, 2209, 0, 1572, 1467, 1104, 267, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 961, 536, 701, 905, 11, 98, 7, 1661, 0, 269, 17, 938, 1101, 2586, 1575, 479, 2682, 2318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 45, 0, 1206, 1636, 1649, 1476, 1096, 1119, 1558, 0, 323, 2269, 19, 1394, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 985, 369, 26, 12, 1956, 639, 974, 12, 1785, 269, 1208, 1103, 540, 1994, 727, 0, 2008, 729, 418, 305, 190, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1528, 640, 8, 1916, 14, 692, 1194, 880, 229, 1439, 352, 1186, 316, 1610, 2284, 1655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2062, 1337, 583, 945, 0, 1711, 1963, 1145, 903, 1790, 1303, 372, 1353, 16, 625, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16746, 15895, 16520, 16845, 16565, 16430, 15943, 16128] avg=16384.0000 max=16845 max_rank=3 min=15895 min_rank=1 imbalance_ratio=1.028137 max_load_reduction=114227 max_load_reduction_pct=0.871483 moved_to_echo=65066 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1964, 1237, 1968, 1, 541, 0, 2259, 1290, 1263, 2557, 1487, 476, 1409, 0, 1, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 1439, 722, 1348, 11, 766, 167, 1715, 590, 70, 2533, 1656, 1689, 98, 1790, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 7, 2110, 688, 128, 2765, 1460, 1050, 283, 5, 1916, 2277, 0, 1414, 1499, 1105, 248, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1127, 611, 738, 919, 1, 95, 23, 1061, 0, 464, 95, 384, 1412, 2218, 1103, 494, 2385, 2082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1299, 10, 0, 1995, 1295, 2050, 1398, 717, 1380, 1610, 0, 451, 1369, 23, 1124, 1494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 865, 602, 19, 15, 2115, 439, 1179, 4, 1894, 229, 1299, 732, 437, 1959, 914, 0, 2084, 690, 339, 257, 151, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1121, 1109, 8, 2310, 23, 444, 1203, 1187, 253, 1355, 117, 1116, 443, 1540, 1853, 1843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1736, 1820, 463, 954, 0, 1497, 2099, 1224, 851, 1555, 1153, 934, 1289, 20, 678, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17527, 16390, 17120, 15212, 16215, 16385, 15925, 16298] avg=16384.0000 max=17527 max_rank=0 min=15212 min_rank=3 imbalance_ratio=1.069763 max_load_reduction=113545 max_load_reduction_pct=0.866280 moved_to_echo=64823 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1751, 1347, 1499, 5, 471, 0, 1909, 1024, 746, 2015, 782, 1151, 1867, 0, 0, 1321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 647, 1377, 1604, 1274, 15, 666, 229, 2148, 457, 120, 2248, 1908, 1879, 191, 1301, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 1956, 397, 163, 2285, 1354, 833, 310, 0, 2033, 2031, 2, 1265, 1070, 1427, 195, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1288, 531, 676, 967, 17, 148, 85, 1339, 0, 220, 56, 379, 1666, 2263, 1032, 810, 2214, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1367, 7, 2, 1668, 1668, 2273, 1167, 1316, 930, 1715, 0, 704, 1667, 6, 995, 1712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1139, 699, 36, 14, 2480, 805, 1248, 19, 1523, 383, 924, 1254, 389, 1636, 980, 3, 1883, 613, 306, 189, 194, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1501, 1272, 3, 1974, 39, 601, 1507, 1434, 316, 1450, 440, 1457, 786, 718, 1676, 1256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1280, 1420, 644, 1296, 0, 1365, 1819, 1588, 917, 1928, 1283, 662, 1159, 17, 604, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15888, 17157, 15538, 15963, 17197, 16894, 16430, 16005] avg=16384.0000 max=17197 max_rank=4 min=15538 min_rank=2 imbalance_ratio=1.049622 max_load_reduction=113875 max_load_reduction_pct=0.868797 moved_to_echo=66526 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1871, 1465, 1580, 0, 932, 1, 2353, 891, 883, 1815, 1186, 598, 1392, 0, 0, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 1466, 1567, 1228, 19, 661, 214, 1981, 348, 77, 2461, 1984, 1156, 81, 1571, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 2, 1357, 437, 218, 1899, 1047, 1046, 267, 31, 1893, 1958, 0, 1781, 1400, 1020, 176, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 709, 511, 1195, 21, 54, 20, 1422, 1, 333, 65, 621, 1583, 2291, 1259, 733, 1750, 2127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1096, 13, 0, 1961, 1847, 1842, 1103, 1455, 1120, 1679, 0, 747, 1735, 15, 1024, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 837, 400, 43, 47, 1712, 830, 1902, 36, 2101, 234, 693, 1036, 372, 1694, 743, 1, 2055, 671, 345, 294, 215, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 981, 13, 1953, 25, 848, 1378, 1452, 334, 1475, 269, 1286, 1077, 1849, 1885, 1837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1984, 1506, 309, 766, 0, 1151, 2151, 1359, 1040, 1806, 1777, 953, 1164, 43, 563, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15710, 16880, 14864, 15929, 16924, 16426, 17738, 16601] avg=16384.0000 max=17738 max_rank=6 min=14864 min_rank=2 imbalance_ratio=1.082642 max_load_reduction=113334 max_load_reduction_pct=0.864670 moved_to_echo=67689 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1852, 1613, 1100, 0, 390, 0, 1785, 1000, 1616, 2767, 1722, 276, 1003, 0, 0, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 1913, 496, 1329, 12, 1066, 189, 2599, 347, 104, 2163, 1553, 1178, 158, 1426, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 1, 1952, 586, 159, 2392, 1351, 713, 339, 24, 1632, 2774, 0, 1667, 1133, 1389, 268, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1156, 662, 986, 989, 4, 107, 4, 1067, 0, 413, 28, 402, 1155, 2462, 1480, 768, 2958, 1717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 744, 16, 0, 1676, 2107, 1765, 551, 980, 912, 761, 0, 317, 2357, 16, 1412, 2019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1176, 434, 6, 5, 2017, 498, 817, 24, 2241, 276, 1409, 658, 447, 1863, 489, 0, 2033, 589, 449, 244, 245, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1688, 352, 1, 1667, 9, 665, 1504, 922, 174, 1540, 173, 932, 452, 2584, 2455, 1822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 1374, 743, 1309, 0, 913, 1999, 1518, 881, 1876, 1421, 712, 1756, 9, 592, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16030, 16712, 16566, 16358, 15633, 16054, 16940, 16779] avg=16384.0000 max=16940 max_rank=6 min=15633 min_rank=4 imbalance_ratio=1.033936 max_load_reduction=114132 max_load_reduction_pct=0.870758 moved_to_echo=65406 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[15104, 13312, 12032, 128, 4608, 128, 15872, 8192, 9600, 17664, 10752, 5120, 11136, 0, 128, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[14336, 11520, 4352, 8704, 0, 10496, 16000, 11008, 7680, 14848, 11520, 5632, 10752, 256, 4736, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[10240, 7296, 128, 15360, 256, 4864, 11648, 9600, 2176, 12544, 1792, 9216, 4096, 13184, 16384, 13184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[6144, 11392, 8960, 8832, 256, 6400, 1664, 17152, 3584, 768, 19072, 14464, 10496, 1408, 12032, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9216, 4480, 6528, 8960, 128, 1024, 256, 11648, 128, 2816, 640, 4992, 10752, 19328, 10624, 4736, 19584, 16384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[7552, 4352, 256, 256, 15360, 5248, 10368, 256, 16384, 2176, 8960, 7680, 3200, 14848, 5888, 128, 16000, 5248, 3072, 2176, 1664, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[1920, 128, 15872, 4224, 1792, 19200, 10240, 7552, 2432, 256, 15104, 19328, 128, 12416, 10752, 9216, 1792, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[7936, 256, 128, 13184, 13952, 15360, 9344, 9216, 9088, 12288, 0, 3584, 13824, 256, 9984, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3303, 0, 10, 1, 0, 0, 2, 13, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 3964, 1, 3836, 1, 0, 2256, 0, 4096, 4095, 5, 0, 0, 4096, 0, 3711, 154, 976, 0, 0, 4092, 0, 0, 101, 0, 666, 0, 1797, 1878, 3949, 0, 0, 4096, 0, 2273, 4096, 0, 0, 3844, 517, 0, 0, 3900, 0, 3624, 0, 0, 0, 161, 0, 0, 4096, 0, 4079, 4096, 0, 0, 0, 0, 0, 0, 275, 4, 4066, 0, 0, 0, 32, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 889, 145, 4096, 0, 132, 4096, 0, 2216, 0, 0, 0, 4095, 4096, 0, 0, 3, 0, 1387, 0, 0, 3698, 3410, 0, 167, 0, 0, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 1260, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 4081, 0, 0, 4096, 0, 0, 4044, 13, 3281, 2, 0, 2568, 0, 4092, 4061, 7, 0, 0, 4096, 0, 2436, 1, 369, 0, 0, 3647, 0, 0, 223, 0, 635, 0, 3042, 3837, 3922, 0, 0, 4096, 0, 2626, 4096, 0, 0, 3977, 1475, 0, 0, 4094, 0, 3568, 66, 0, 0, 1948, 1, 0, 4082, 0, 3497, 4010, 0, 0, 0, 0, 0, 0, 178, 2, 1324, 0, 0, 0, 968, 0, 0, 0, 0, 0, 4096, 115, 0, 0, 0, 0, 4095, 160, 1678, 3970, 0, 901, 4093, 0, 2599, 0, 0, 0, 4096, 4096, 0, 0, 70, 0, 1288, 2, 0, 4092, 1537, 0, 353, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2814, 0, 1, 13, 0, 0, 6, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4034, 0, 4058, 34, 0, 825, 0, 4095, 4094, 0, 0, 0, 4096, 0, 3640, 0, 1288, 0, 0, 3752, 0, 0, 336, 0, 1518, 0, 3274, 2763, 3970, 0, 0, 4095, 0, 1346, 4096, 0, 0, 3642, 1883, 0, 0, 3830, 0, 3186, 12, 0, 0, 1649, 0, 0, 4096, 0, 3668, 4031, 0, 0, 0, 0, 0, 0, 219, 0, 1526, 0, 0, 4, 719, 0, 0, 0, 0, 0, 4096, 234, 1, 0, 0, 0, 4096, 1345, 694, 4088, 0, 256, 4089, 0, 2272, 0, 0, 0, 4096, 4096, 0, 0, 28, 0, 1801, 0, 0, 4080, 570, 0, 327, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 10, 2970, 0, 0, 0, 0, 0, 10, 6, 1, 0, 0, 4095, 0, 1, 4095, 0, 0, 4079, 18, 4075, 0, 1, 3372, 0, 4066, 4095, 30, 1, 0, 4095, 0, 2828, 141, 1161, 1, 1, 3881, 0, 1, 117, 1, 93, 1, 2582, 3010, 3910, 0, 1, 4095, 0, 1007, 4094, 0, 1, 3416, 1880, 0, 1, 3804, 1, 3222, 7, 0, 1, 1021, 1, 1, 4095, 0, 3681, 4089, 1, 1, 0, 0, 0, 1, 143, 2, 3201, 1, 1, 1, 1309, 0, 1, 0, 0, 1, 4095, 128, 1, 0, 1, 1, 4095, 85, 1107, 4079, 1, 156, 4075, 1, 2377, 0, 1, 0, 4095, 4095, 0, 1, 84, 0, 1709, 0, 0, 4084, 724, 2, 158, 0, 0, 1, 4086] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 2, 0, 1621, 0, 1, 0, 4, 0, 0, 1, 2, 0, 1, 4096, 0, 0, 4095, 0, 1, 4078, 0, 3832, 2, 0, 2976, 0, 4096, 4090, 1, 2, 0, 4094, 0, 3456, 1, 1603, 0, 0, 3693, 0, 0, 31, 0, 317, 0, 3035, 2693, 3963, 2, 0, 4095, 1, 1363, 4096, 0, 0, 2959, 2001, 2, 0, 4072, 2, 3693, 52, 0, 0, 1235, 5, 0, 4094, 0, 4023, 4008, 0, 1, 0, 0, 0, 2, 376, 0, 2650, 2, 0, 0, 182, 0, 0, 0, 2, 0, 4094, 16, 2, 0, 1, 2, 4093, 411, 755, 4078, 1, 159, 4083, 0, 2905, 0, 1, 0, 4095, 4071, 3, 0, 49, 0, 1756, 0, 0, 4083, 1549, 0, 64, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 2281, 0, 2, 0, 0, 0, 0, 1, 1, 0, 0, 4095, 1, 1, 4095, 1, 0, 4061, 0, 3855, 0, 1, 1868, 0, 4090, 4088, 9, 1, 0, 4095, 0, 2523, 21, 1999, 2, 1, 4009, 1, 0, 178, 1, 400, 0, 2516, 2270, 3980, 0, 1, 4095, 0, 1369, 4095, 0, 1, 3303, 1626, 0, 1, 4040, 0, 3902, 18, 0, 1, 1043, 29, 1, 4095, 1, 3591, 4095, 0, 1, 0, 2, 0, 0, 203, 1, 2569, 0, 1, 0, 1152, 0, 1, 1, 0, 1, 4095, 7, 0, 1, 1, 1, 4095, 1023, 309, 4031, 1, 122, 4088, 1, 3156, 0, 0, 0, 4095, 4095, 0, 1, 121, 0, 2100, 0, 0, 4093, 1791, 0, 63, 0, 0, 1, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2773, 0, 5, 0, 0, 7, 0, 6, 0, 0, 3, 4051, 5, 1, 4089, 3, 5, 4078, 4, 4085, 12, 1, 3448, 0, 4077, 4096, 0, 0, 0, 4089, 0, 2405, 97, 1024, 0, 0, 3868, 1, 0, 89, 0, 1097, 0, 3077, 1864, 3957, 1, 0, 4096, 13, 1739, 4088, 8, 0, 3701, 1921, 9, 2, 4075, 0, 3369, 43, 2, 0, 812, 53, 0, 4096, 0, 3851, 4096, 3, 0, 0, 0, 8, 32, 312, 16, 2960, 0, 0, 0, 325, 10, 0, 0, 0, 6, 4081, 151, 1, 3, 4, 1, 4090, 523, 254, 4052, 0, 318, 4095, 0, 2770, 2, 1, 0, 4094, 4096, 0, 0, 109, 0, 1692, 4, 0, 4087, 551, 1, 42, 6, 0, 0, 4079] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2334, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 4078, 10, 3920, 1, 0, 2835, 0, 4093, 4089, 3, 0, 1, 4095, 0, 1590, 0, 1168, 0, 0, 3980, 0, 0, 60, 0, 376, 0, 879, 2421, 3757, 0, 0, 4095, 0, 2339, 4096, 0, 0, 3988, 1762, 0, 0, 4017, 0, 4080, 154, 0, 0, 267, 0, 0, 4096, 0, 4030, 4091, 0, 0, 0, 5, 0, 0, 222, 7, 3479, 0, 0, 1, 580, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4093, 527, 156, 3945, 0, 1220, 4096, 0, 3656, 1, 0, 0, 4095, 4096, 0, 0, 228, 0, 1022, 1, 0, 4088, 1935, 0, 529, 0, 0, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3303, 0, 10, 1, 0, 0, 2, 13, 0, 0, 0, 4095, 4096, 4095, 1669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 1, 3836, 1, 0, 2256, 0, 4096, 1840, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3711, 154, 976, 0, 0, 4092, 0, 0, 101, 0, 666, 0, 1797, 827, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1878, 3949, 0, 0, 0, 0, 2273, 0, 0, 0, 3844, 517, 0, 0, 2810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3624, 0, 0, 0, 161, 0, 0, 3863, 0, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 4, 4066, 0, 0, 0, 32, 0, 0, 0, 0, 0, 4096, 6, 0, 4096, 2427, 2056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 889, 145, 4096, 0, 132, 4096, 0, 2216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 1387, 0, 0, 3698, 3410, 0, 167, 0, 0, 0, 4091, 263, 254, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17284, 15999, 16619, 15271, 15823, 17058, 15416, 17602] avg=16384.0000 max=17602 max_rank=7 min=15271 min_rank=3 imbalance_ratio=1.074341 max_load_reduction=113470 max_load_reduction_pct=0.865707 moved_to_echo=65899 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2814, 0, 1, 13, 0, 0, 6, 2, 0, 0, 0, 4096, 4095, 4096, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 4058, 34, 0, 825, 0, 4095, 1839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3640, 0, 1288, 0, 0, 3752, 0, 0, 336, 0, 1518, 0, 3274, 812, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2763, 3970, 0, 0, 0, 0, 1346, 0, 0, 0, 3642, 1883, 0, 0, 2759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3186, 12, 0, 0, 1649, 0, 0, 3863, 0, 3668, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 1526, 0, 0, 4, 719, 0, 0, 0, 0, 0, 4096, 234, 1, 4096, 2426, 2056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 1345, 694, 4088, 0, 256, 4089, 0, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 28, 0, 1801, 0, 0, 4080, 570, 0, 327, 0, 0, 0, 4096, 259, 254, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16793, 14885, 18915, 16363, 16409, 15377, 16586, 15744] avg=16384.0000 max=18915 max_rank=2 min=14885 min_rank=1 imbalance_ratio=1.154480 max_load_reduction=112157 max_load_reduction_pct=0.855690 moved_to_echo=64116 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 1260, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 4081, 4096, 4096, 1669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 13, 3281, 2, 0, 2568, 0, 4092, 1824, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2436, 1, 369, 0, 0, 3647, 0, 0, 223, 0, 635, 0, 3042, 868, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3837, 3922, 0, 0, 0, 0, 2626, 0, 0, 0, 3977, 1475, 0, 0, 2949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3568, 66, 0, 0, 1948, 1, 0, 3850, 0, 3497, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 2, 1324, 0, 0, 0, 968, 0, 0, 0, 0, 0, 4096, 115, 0, 4096, 2427, 2039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 160, 1678, 3970, 0, 901, 4093, 0, 2599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 70, 0, 1288, 2, 0, 4092, 1537, 0, 353, 0, 0, 0, 4096, 277, 254, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15216, 15831, 15515, 18786, 16940, 15245, 17242, 16297] avg=16384.0000 max=18786 max_rank=3 min=15216 min_rank=0 imbalance_ratio=1.146606 max_load_reduction=112286 max_load_reduction_pct=0.856674 moved_to_echo=65724 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 2281, 0, 2, 0, 0, 0, 0, 1, 1, 0, 0, 4095, 4095, 4095, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 4061, 0, 3855, 0, 1, 1868, 0, 4090, 1836, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2523, 21, 1999, 2, 1, 4009, 1, 0, 178, 1, 400, 0, 2516, 857, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2270, 3980, 0, 1, 0, 0, 1369, 0, 0, 1, 3303, 1626, 0, 1, 2910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 18, 0, 1, 1043, 29, 1, 3862, 1, 3591, 4095, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 1, 2569, 0, 1, 0, 1152, 0, 1, 1, 0, 1, 4095, 7, 0, 4095, 2425, 2053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 3841, 1023, 309, 4031, 1, 122, 4088, 1, 3156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 121, 0, 2100, 0, 0, 4093, 1791, 0, 63, 0, 0, 1, 4094, 273, 254, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16241, 15724, 16802, 15461, 16546, 16604, 16575, 17119] avg=16384.0000 max=17119 max_rank=7 min=15461 min_rank=3 imbalance_ratio=1.044861 max_load_reduction=113953 max_load_reduction_pct=0.869392 moved_to_echo=66844 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2, 0, 1621, 0, 1, 0, 4, 0, 0, 1, 2, 0, 1, 4096, 4095, 4095, 1669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4078, 0, 3832, 2, 0, 2976, 0, 4096, 1837, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3456, 1, 1603, 0, 0, 3693, 0, 0, 31, 0, 317, 0, 3035, 863, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2693, 3963, 2, 0, 0, 1, 1363, 0, 0, 0, 2959, 2001, 2, 0, 2934, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3693, 52, 0, 0, 1235, 5, 0, 3861, 0, 4023, 4008, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 376, 0, 2650, 2, 0, 0, 182, 0, 0, 0, 2, 0, 4094, 16, 2, 4096, 2426, 2054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3839, 411, 755, 4078, 1, 159, 4083, 0, 2905, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 3, 0, 49, 0, 1756, 0, 0, 4083, 1549, 0, 64, 0, 0, 0, 4095, 275, 254, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15588, 16825, 17292, 15920, 16878, 15902, 16235, 16432] avg=16384.0000 max=17292 max_rank=2 min=15588 min_rank=0 imbalance_ratio=1.055420 max_load_reduction=113780 max_load_reduction_pct=0.868073 moved_to_echo=65447 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 10, 2970, 0, 0, 0, 0, 0, 10, 6, 1, 0, 0, 4095, 4095, 4095, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4079, 18, 4075, 0, 1, 3372, 0, 4066, 1840, 30, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2828, 141, 1161, 1, 1, 3881, 0, 1, 117, 1, 93, 1, 2582, 807, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3010, 3910, 0, 1, 0, 0, 1007, 0, 0, 1, 3416, 1880, 0, 1, 2740, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3222, 7, 0, 1, 1021, 1, 1, 3862, 0, 3681, 4089, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 143, 2, 3201, 1, 1, 1, 1309, 0, 1, 0, 0, 1, 4095, 128, 1, 4094, 2425, 2056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3841, 85, 1107, 4079, 1, 156, 4075, 1, 2377, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 84, 0, 1709, 0, 0, 4084, 724, 2, 158, 0, 0, 1, 4086, 257, 254, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16953, 17483, 15909, 15967, 15887, 17460, 15725, 15688] avg=16384.0000 max=17483 max_rank=1 min=15688 min_rank=7 imbalance_ratio=1.067078 max_load_reduction=113589 max_load_reduction_pct=0.866615 moved_to_echo=64760 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2773, 0, 5, 0, 0, 7, 0, 6, 0, 0, 3, 4051, 4096, 4094, 1667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 3, 5, 4078, 4, 4085, 12, 1, 3448, 0, 4077, 1829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 2405, 97, 1024, 0, 0, 3868, 1, 0, 89, 0, 1097, 0, 3077, 868, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1864, 3957, 1, 0, 0, 13, 1739, 0, 8, 0, 3701, 1921, 9, 2, 2928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3369, 43, 2, 0, 812, 53, 0, 3862, 0, 3851, 4096, 3, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 312, 16, 2960, 0, 0, 0, 325, 10, 0, 0, 0, 6, 4081, 151, 1, 4088, 2422, 2062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 1, 3830, 523, 254, 4052, 0, 318, 4095, 0, 2770, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 109, 0, 1692, 4, 0, 4087, 551, 1, 42, 6, 0, 0, 4079, 279, 260, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16703, 17548, 16820, 16143, 16099, 16466, 15853, 15440] avg=16384.0000 max=17548 max_rank=1 min=15440 min_rank=7 imbalance_ratio=1.071045 max_load_reduction=113524 max_load_reduction_pct=0.866119 moved_to_echo=63858 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2334, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4095, 4095, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 10, 3920, 1, 0, 2835, 0, 4093, 1837, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1590, 0, 1168, 0, 0, 3980, 0, 0, 60, 0, 376, 0, 879, 852, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2421, 3757, 0, 0, 0, 0, 2339, 0, 0, 0, 3988, 1762, 0, 0, 2894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 154, 0, 0, 267, 0, 0, 3863, 0, 4030, 4091, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 7, 3479, 0, 0, 1, 580, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 2425, 2053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3839, 527, 156, 3945, 0, 1220, 4096, 0, 3656, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 228, 0, 1022, 1, 0, 4088, 1935, 0, 529, 0, 0, 0, 4093, 271, 254, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16294, 16777, 13200, 17161, 16490, 16960, 17440, 16750] avg=16384.0000 max=17440 max_rank=6 min=13200 min_rank=2 imbalance_ratio=1.064453 max_load_reduction=113632 max_load_reduction_pct=0.866943 moved_to_echo=67640 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 0, 22656, 512, 9600, 128, 128, 30976, 128, 128, 1152, 128, 5120, 128, 20224, 6784, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 768, 0, 12800, 128, 0, 32384, 12160, 128, 1792, 128, 0, 128, 32768, 2176, 2048, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[20736, 31488, 128, 128, 0, 128, 14080, 0, 128, 128, 28928, 13184, 128, 128, 23040, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 128, 32512, 128, 30976, 128, 128, 20224, 0, 32768, 14720, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 30720, 4992, 5120, 32384, 128, 3328, 32768, 128, 22016, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132992 observed_loss=-1920 tokens_per_expert=[128, 2048, 128, 21888, 128, 128, 128, 5376, 128, 128, 128, 128, 128, 32768, 768, 128, 32768, 19456, 16512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[28672, 384, 128, 128, 8192, 128, 128, 30976, 128, 30464, 32640, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 128, 19456, 0, 128, 128, 128, 128, 128, 128, 128, 0, 128, 32768, 32768, 32768, 13440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132992 slack_slots=7040 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 498, 0, 3, 1, 1, 2424, 3, 4092, 2871, 0, 1, 0, 0, 4095, 0, 1, 1, 0, 969, 0, 4095, 1, 4095, 4095, 0, 0, 0, 4095, 1182, 0, 4, 0, 4095, 2, 0, 4095, 5, 1, 1, 0, 4095, 1, 5, 1, 0, 4089, 4094, 299, 1, 0, 4090, 0, 4095, 4081, 0, 1, 4079, 1790, 1, 1, 3, 4092, 0, 1, 4095, 12, 0, 1, 0, 0, 0, 4095, 0, 1, 4094, 3, 370, 2003, 4054, 0, 1, 3096, 4081, 0, 3546, 1, 1216, 0, 0, 2793, 1, 1, 0, 84, 0, 63, 0, 1, 1568, 0, 4085, 0, 1, 1, 1, 0, 0, 1, 8, 4095, 1, 1, 7, 1, 1, 1, 3986, 1, 4054, 0, 4095, 4, 0, 2, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 307, 0, 1508, 0, 0, 2556, 0, 4096, 2853, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 361, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 1472, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4072, 4092, 1188, 0, 0, 4014, 0, 4096, 4095, 0, 0, 4082, 1720, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4, 190, 3588, 4096, 0, 0, 3381, 3747, 0, 2110, 0, 141, 0, 0, 1713, 0, 0, 0, 11, 0, 5, 0, 0, 3265, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 2940, 0, 3929, 0, 4096, 3, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 146, 0, 192, 0, 0, 586, 0, 4096, 2923, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1915, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 679, 0, 1, 0, 4096, 23, 0, 4096, 41, 0, 0, 0, 4096, 0, 1, 0, 0, 4093, 4095, 39, 0, 0, 4095, 0, 4096, 3779, 0, 0, 4005, 1830, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1653, 2633, 3662, 0, 0, 4072, 3863, 0, 3425, 0, 2709, 0, 0, 1926, 0, 0, 0, 422, 0, 205, 0, 0, 245, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 167, 4096, 0, 0, 0, 0, 0, 0, 4022, 0, 3996, 0, 4096, 9, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 568, 1, 110, 0, 0, 2744, 2, 4094, 1302, 2, 0, 0, 0, 4094, 2, 0, 0, 0, 536, 0, 4094, 1, 4094, 4094, 0, 0, 2, 4096, 1499, 0, 3, 0, 4096, 6, 0, 4094, 54, 0, 1, 0, 4095, 1, 7, 1, 0, 4091, 4096, 170, 0, 0, 4093, 1, 4094, 4095, 0, 1, 3984, 1252, 0, 0, 1, 4096, 0, 0, 4096, 1, 0, 1, 0, 0, 0, 4095, 0, 1, 4094, 0, 262, 3137, 4069, 2, 1, 3931, 4015, 0, 3027, 1, 1556, 0, 2, 2835, 0, 0, 0, 104, 1, 399, 0, 1, 2212, 0, 4096, 0, 1, 0, 0, 0, 0, 1, 41, 4076, 0, 1, 1, 0, 0, 0, 3254, 1, 3986, 0, 4083, 16, 0, 0, 1, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 406, 0, 245, 0, 0, 1925, 0, 4084, 2013, 4, 0, 0, 0, 4085, 9, 0, 0, 0, 500, 0, 4081, 0, 4096, 4085, 0, 0, 0, 4090, 585, 0, 20, 10, 4096, 18, 0, 4093, 18, 0, 0, 0, 4096, 0, 7, 0, 0, 4096, 4079, 158, 0, 0, 4031, 5, 4092, 4047, 0, 0, 4041, 1327, 0, 0, 19, 4069, 0, 0, 4087, 42, 0, 0, 0, 4, 8, 4096, 0, 0, 4093, 2, 1609, 2140, 3953, 0, 0, 3793, 3991, 0, 3820, 0, 1876, 1, 0, 3487, 0, 0, 0, 58, 10, 139, 0, 0, 853, 0, 4086, 0, 0, 0, 0, 9, 6, 0, 96, 4087, 0, 0, 12, 0, 0, 0, 3959, 0, 4079, 0, 4082, 32, 5, 9, 0, 18] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 97, 0, 289, 0, 0, 2958, 1, 4095, 1796, 1, 0, 0, 0, 4096, 1, 0, 0, 0, 576, 0, 4095, 0, 4096, 4095, 1, 0, 0, 4096, 276, 0, 0, 1, 4096, 0, 0, 4095, 2, 0, 0, 0, 4096, 0, 2, 0, 0, 4091, 4092, 564, 0, 1, 4092, 1, 4096, 4092, 0, 0, 4084, 1844, 0, 0, 11, 4096, 0, 0, 4095, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 518, 3038, 4086, 0, 0, 4009, 3946, 0, 3146, 0, 613, 0, 0, 3484, 0, 0, 0, 13, 1, 38, 0, 0, 1974, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 53, 4095, 0, 0, 1, 0, 0, 0, 3562, 0, 4081, 0, 4089, 2, 0, 8, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 186, 0, 5, 0, 0, 1865, 0, 4096, 2894, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1542, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 540, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 14, 0, 0, 4060, 0, 4096, 4092, 0, 0, 4012, 741, 0, 0, 4, 4095, 0, 0, 4096, 27, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4, 591, 2120, 4092, 0, 0, 4058, 3907, 0, 3460, 0, 3198, 0, 0, 2879, 0, 0, 0, 22, 0, 205, 0, 0, 736, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 85, 4096, 0, 0, 0, 0, 0, 0, 3878, 0, 4096, 0, 4026, 8, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1093, 0, 76, 1, 1, 1135, 0, 4093, 1327, 0, 1, 2, 0, 4095, 0, 1, 0, 1, 546, 0, 4092, 1, 4095, 4095, 1, 0, 0, 4095, 897, 0, 5, 0, 4095, 0, 0, 4095, 0, 1, 1, 1, 4095, 1, 42, 1, 0, 4051, 4095, 37, 1, 0, 3992, 0, 4094, 4095, 0, 1, 4072, 2090, 1, 1, 0, 4091, 0, 1, 4094, 9, 0, 1, 0, 0, 0, 4095, 0, 1, 4094, 1, 743, 2542, 4072, 0, 1, 3652, 4083, 0, 3653, 1, 1843, 0, 0, 2752, 1, 1, 1, 185, 0, 769, 0, 1, 1487, 0, 4071, 0, 1, 1, 0, 0, 0, 0, 104, 4095, 1, 1, 0, 1, 1, 0, 4086, 1, 3916, 0, 4095, 8, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 498, 0, 3, 1, 1, 2424, 3, 4092, 2871, 0, 1, 0, 0, 4095, 1755, 1081, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 969, 0, 4095, 1, 2340, 4095, 0, 0, 0, 4095, 1182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4072, 2, 0, 4095, 5, 1, 1, 0, 4095, 1, 5, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 1, 0, 4090, 0, 0, 4081, 0, 1, 4079, 1790, 1, 1, 3, 2345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 12, 0, 1, 0, 0, 0, 4095, 0, 1, 4094, 3, 370, 2003, 666, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 927, 0, 1, 3096, 4081, 0, 3546, 1, 1216, 0, 0, 2793, 1, 1, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 1, 1568, 0, 4085, 0, 1, 1, 1, 0, 0, 1, 8, 4095, 4095, 2266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 7, 1, 1, 1, 3986, 1, 4054, 0, 4095, 4, 0, 2, 0, 1, 2754, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17198, 16779, 16371, 16691, 15364, 15747, 16185, 16737] avg=16384.0000 max=17198 max_rank=0 min=15364 min_rank=4 imbalance_ratio=1.049683 max_load_reduction=113874 max_load_reduction_pct=0.868790 moved_to_echo=64033 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 146, 0, 192, 0, 0, 586, 0, 4096, 2923, 0, 0, 0, 0, 4096, 1756, 1081, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1915, 0, 4096, 0, 2340, 4096, 0, 0, 0, 4096, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4073, 23, 0, 4096, 41, 0, 0, 0, 4096, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 4095, 0, 0, 3779, 0, 0, 4005, 1830, 0, 0, 0, 2346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1653, 2633, 666, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 837, 0, 0, 4072, 3863, 0, 3425, 0, 2709, 0, 0, 1926, 0, 0, 0, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 245, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 167, 4096, 4096, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 3996, 0, 4096, 9, 0, 0, 0, 0, 2488, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15213, 17222, 16424, 16094, 17263, 17254, 15163, 16439] avg=16384.0000 max=17263 max_rank=4 min=15163 min_rank=6 imbalance_ratio=1.053650 max_load_reduction=113809 max_load_reduction_pct=0.868294 moved_to_echo=66119 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 406, 0, 245, 0, 0, 1925, 0, 4084, 2013, 4, 0, 0, 0, 4085, 1758, 1077, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 500, 0, 4081, 0, 2338, 4085, 0, 0, 0, 4090, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 10, 4068, 18, 0, 4093, 18, 0, 0, 0, 4096, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 4031, 5, 0, 4047, 0, 0, 4041, 1327, 0, 0, 19, 2327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 42, 0, 0, 0, 4, 8, 4096, 0, 0, 4093, 2, 1609, 2140, 665, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 896, 0, 0, 3793, 3991, 0, 3820, 0, 1876, 1, 0, 3487, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 139, 0, 0, 853, 0, 4086, 0, 0, 0, 0, 9, 6, 0, 96, 4087, 4092, 2258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 3959, 0, 4079, 0, 4082, 32, 5, 9, 0, 18, 2690, 1821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15964, 15688, 16426, 15955, 16774, 17922, 15636, 16707] avg=16384.0000 max=17922 max_rank=5 min=15636 min_rank=6 imbalance_ratio=1.093872 max_load_reduction=113150 max_load_reduction_pct=0.863266 moved_to_echo=67039 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 568, 1, 110, 0, 0, 2744, 2, 4094, 1302, 2, 0, 0, 0, 4094, 1755, 1082, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 536, 0, 4094, 1, 2339, 4094, 0, 0, 2, 4096, 1499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4073, 6, 0, 4094, 54, 0, 1, 0, 4095, 1, 7, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 0, 4093, 1, 0, 4095, 0, 1, 3984, 1252, 0, 0, 1, 2347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 1, 0, 0, 0, 4095, 0, 1, 4094, 0, 262, 3137, 667, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 929, 2, 1, 3931, 4015, 0, 3027, 1, 1556, 0, 2, 2835, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 399, 0, 1, 2212, 0, 4096, 0, 1, 0, 0, 0, 0, 1, 41, 4076, 4094, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 3254, 1, 3986, 0, 4083, 16, 0, 0, 1, 4, 2765, 1829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16129, 16663, 16426, 15944, 16377, 16403, 17189, 15941] avg=16384.0000 max=17189 max_rank=6 min=15941 min_rank=7 imbalance_ratio=1.049133 max_load_reduction=113883 max_load_reduction_pct=0.868858 moved_to_echo=65910 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 97, 0, 289, 0, 0, 2958, 1, 4095, 1796, 1, 0, 0, 0, 4096, 1756, 1082, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 576, 0, 4095, 0, 2340, 4095, 1, 0, 0, 4096, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4073, 0, 0, 4095, 2, 0, 0, 0, 4096, 0, 2, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 0, 1, 4092, 1, 0, 4092, 0, 0, 4084, 1844, 0, 0, 11, 2347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 518, 3038, 667, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 934, 0, 0, 4009, 3946, 0, 3146, 0, 613, 0, 0, 3484, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 38, 0, 0, 1974, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 53, 4095, 4096, 2265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3562, 0, 4081, 0, 4089, 2, 0, 8, 0, 0, 2776, 1827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16547, 15480, 16360, 17036, 16542, 16145, 16616, 16346] avg=16384.0000 max=17036 max_rank=3 min=15480 min_rank=1 imbalance_ratio=1.039795 max_load_reduction=114036 max_load_reduction_pct=0.870026 moved_to_echo=65649 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1093, 0, 76, 1, 1, 1135, 0, 4093, 1327, 0, 1, 2, 0, 4095, 1755, 1080, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 546, 0, 4092, 1, 2340, 4095, 1, 0, 0, 4095, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4072, 0, 0, 4095, 0, 1, 1, 1, 4095, 1, 42, 1, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1, 0, 3992, 0, 0, 4095, 0, 1, 4072, 2090, 1, 1, 0, 2345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 9, 0, 1, 0, 0, 0, 4095, 0, 1, 4094, 1, 743, 2542, 666, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 930, 0, 1, 3652, 4083, 0, 3653, 1, 1843, 0, 0, 2752, 1, 1, 1, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 769, 0, 1, 1487, 0, 4071, 0, 1, 1, 0, 0, 0, 0, 104, 4095, 4094, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 4086, 1, 3916, 0, 4095, 8, 0, 0, 0, 1, 2767, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15034, 16069, 16365, 16635, 16270, 17103, 16890, 16706] avg=16384.0000 max=17103 max_rank=5 min=15034 min_rank=0 imbalance_ratio=1.043884 max_load_reduction=113969 max_load_reduction_pct=0.869514 moved_to_echo=66969 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 186, 0, 5, 0, 0, 1865, 0, 4096, 2894, 0, 0, 0, 0, 4096, 1756, 1081, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1542, 0, 4096, 0, 2340, 4096, 0, 0, 0, 4096, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4060, 0, 0, 4092, 0, 0, 4012, 741, 0, 0, 4, 2348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 27, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4, 591, 2120, 666, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 935, 0, 0, 4058, 3907, 0, 3460, 0, 3198, 0, 0, 2879, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 736, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 85, 4096, 4096, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3878, 0, 4096, 0, 4026, 8, 0, 0, 0, 0, 2780, 1829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16356, 16710, 16363, 15271, 15719, 18459, 15577, 16617] avg=16384.0000 max=18459 max_rank=5 min=15271 min_rank=3 imbalance_ratio=1.126648 max_load_reduction=112613 max_load_reduction_pct=0.859169 moved_to_echo=66372 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 307, 0, 1508, 0, 0, 2556, 0, 4096, 2853, 0, 0, 0, 0, 4096, 1756, 1082, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 0, 4096, 0, 2340, 4096, 0, 0, 0, 4096, 1472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1188, 0, 0, 4014, 0, 0, 4095, 0, 0, 4082, 1720, 0, 0, 0, 2347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4, 190, 3588, 667, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 936, 0, 0, 3381, 3747, 0, 2110, 0, 141, 0, 0, 1713, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3265, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 2265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2940, 0, 3929, 0, 4096, 3, 0, 0, 0, 1, 2783, 1827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18631, 16461, 16337, 17446, 16763, 12039, 17816, 15579] avg=16384.0000 max=18631 max_rank=0 min=12039 min_rank=5 imbalance_ratio=1.137146 max_load_reduction=112441 max_load_reduction_pct=0.857857 moved_to_echo=62197 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32640, 128, 0, 32768, 128, 128, 128, 128, 32768, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 0, 128, 128, 32768, 0, 128, 32768, 128, 6016, 21248, 5376, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 2560, 128, 128, 32512, 128, 0, 32384, 0, 128, 32384, 12672, 128, 128, 128, 18816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[7424, 128, 128, 30080, 31744, 0, 26240, 128, 13184, 128, 128, 21888, 128, 128, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 128, 128, 29696, 128, 32256, 0, 32768, 128, 128, 128, 128, 128, 21888, 14720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 3328, 128, 2432, 128, 128, 16256, 128, 32768, 18048, 128, 128, 128, 0, 32768, 14080, 8704, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 7040, 0, 32768, 128, 18816, 32768, 128, 0, 128, 32768, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 1920, 0, 128, 12416, 0, 32768, 0, 128, 128, 128, 128, 128, 128, 640, 32768, 32768, 18176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 8, 4095, 0, 0, 1, 4094, 4095, 0, 0, 4026, 645, 1, 2, 1, 1, 1, 1, 0, 27, 4093, 0, 3958, 4087, 1, 1, 4093, 1, 0, 4094, 4092, 0, 1, 1, 0, 4095, 0, 2896, 4094, 0, 3, 0, 0, 3, 1, 4095, 4095, 0, 1, 1, 4095, 628, 0, 0, 1, 1, 2, 2737, 2, 0, 4095, 1, 0, 0, 11, 4095, 1, 0, 4095, 4084, 1, 1, 1, 3289, 0, 0, 0, 4094, 1, 4095, 4095, 0, 1, 328, 0, 4, 0, 0, 21, 0, 2, 2842, 4079, 1, 0, 3955, 0, 1, 3770, 1, 0, 1, 1, 4095, 6, 1, 0, 0, 6, 4095, 1, 0, 1, 24, 0, 0, 1, 0, 0, 4095, 0, 3606, 1, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 4084, 4096, 0, 0, 4064, 637, 0, 4, 1, 0, 18, 0, 0, 4, 4096, 0, 4062, 4046, 0, 0, 4096, 0, 0, 4095, 4091, 0, 0, 0, 0, 4096, 0, 2013, 4096, 0, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 227, 0, 0, 0, 0, 0, 3990, 7, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 4096, 4095, 0, 0, 0, 2846, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 134, 0, 2, 0, 0, 54, 0, 2, 3312, 4094, 0, 0, 3668, 0, 0, 3701, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 34, 4096, 0, 0, 0, 26, 0, 0, 0, 0, 0, 4096, 0, 4015, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 11, 4094, 0, 0, 0, 4095, 4095, 0, 1, 4018, 338, 1, 3, 3, 0, 6, 1, 1, 0, 4096, 0, 3983, 4084, 0, 1, 4093, 0, 0, 4094, 4086, 1, 0, 0, 1, 4095, 0, 1579, 4094, 5, 0, 1, 0, 9, 1, 4095, 4095, 8, 0, 2, 4095, 125, 1, 0, 1, 0, 1, 3766, 0, 1, 4095, 1, 0, 0, 1, 4092, 1, 0, 4096, 4082, 1, 1, 1, 3737, 0, 0, 0, 4095, 0, 4095, 4094, 1, 0, 484, 0, 0, 0, 0, 10, 0, 4, 2784, 4070, 0, 1, 3873, 1, 0, 3775, 0, 1, 0, 1, 4095, 37, 0, 0, 0, 43, 4095, 1, 1, 0, 134, 0, 0, 0, 0, 1, 4096, 0, 4089, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8, 4092, 0, 0, 0, 4079, 4096, 0, 0, 3775, 774, 0, 3, 0, 0, 73, 0, 0, 26, 4089, 0, 4009, 4091, 0, 0, 4095, 0, 0, 4096, 3970, 0, 0, 0, 0, 4096, 1, 2764, 4096, 20, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 477, 0, 0, 0, 0, 1, 3988, 1, 0, 4096, 0, 0, 0, 5, 4094, 0, 0, 4095, 4074, 0, 0, 0, 3035, 1, 0, 0, 4096, 0, 4095, 4095, 0, 0, 338, 0, 6, 0, 0, 50, 0, 14, 1486, 4092, 0, 0, 4042, 0, 0, 4006, 0, 2, 0, 0, 4096, 23, 0, 0, 0, 27, 4096, 0, 0, 0, 84, 0, 0, 0, 0, 0, 4096, 0, 3919, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 14, 4096, 0, 0, 1, 4096, 4096, 0, 0, 4087, 506, 0, 69, 0, 0, 0, 0, 0, 47, 4096, 0, 4076, 4096, 0, 0, 4096, 0, 0, 4096, 4093, 0, 1, 0, 0, 4096, 0, 2521, 4096, 1, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 4096, 140, 0, 0, 0, 0, 0, 3397, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4084, 0, 0, 0, 3817, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 11, 0, 8, 0, 0, 19, 0, 0, 2382, 4093, 0, 0, 4076, 0, 0, 3643, 0, 0, 0, 0, 4096, 84, 0, 0, 0, 11, 4096, 0, 0, 0, 40, 0, 0, 0, 0, 0, 4096, 0, 3834, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 3, 4095, 0, 0, 1, 4095, 4095, 0, 0, 4072, 300, 1, 3, 12, 1, 2, 1, 0, 16, 4094, 0, 4050, 4095, 1, 1, 4095, 0, 0, 4094, 4095, 0, 59, 1, 0, 4095, 0, 1722, 4094, 0, 20, 1, 0, 9, 1, 4095, 4095, 0, 1, 2, 4095, 52, 0, 0, 1, 1, 2, 3258, 0, 0, 4095, 1, 0, 0, 28, 4095, 1, 1, 4094, 4073, 1, 1, 1, 3972, 0, 0, 0, 4089, 1, 4095, 4095, 0, 0, 1000, 0, 1, 0, 0, 27, 1, 13, 2351, 4080, 1, 0, 4082, 0, 1, 3565, 0, 0, 1, 1, 4095, 60, 1, 0, 0, 4, 4092, 1, 0, 1, 37, 0, 0, 1, 0, 0, 4095, 0, 4087, 1, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 4083, 15, 0, 0, 4069, 4096, 0, 0, 3972, 438, 0, 1, 0, 0, 84, 0, 0, 30, 4077, 0, 3895, 4094, 0, 0, 4083, 0, 0, 4096, 4093, 0, 0, 0, 0, 4082, 4, 1790, 4096, 19, 0, 0, 0, 7, 0, 4096, 4096, 0, 0, 0, 4096, 1256, 19, 0, 0, 0, 15, 3738, 51, 0, 4096, 0, 0, 0, 2, 4080, 0, 0, 4085, 4064, 0, 0, 0, 3508, 9, 0, 0, 4096, 0, 4076, 4091, 0, 0, 332, 0, 22, 0, 0, 43, 0, 16, 2491, 4054, 0, 0, 4084, 0, 0, 3244, 0, 13, 0, 0, 4096, 23, 0, 0, 0, 53, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4089, 0, 3794, 0, 17, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 4096, 0, 0, 5, 4096, 4096, 0, 0, 4030, 2928, 0, 24, 4, 0, 0, 0, 0, 66, 4096, 0, 3785, 4095, 0, 0, 4096, 0, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 525, 4096, 0, 67, 0, 0, 2, 0, 4096, 4096, 0, 0, 0, 4096, 420, 0, 0, 0, 0, 0, 2217, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4092, 0, 0, 0, 3705, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 592, 0, 0, 0, 0, 6, 0, 19, 2737, 4091, 0, 0, 4072, 0, 0, 4094, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 24, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3454, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 8, 4095, 0, 0, 1, 4094, 3366, 0, 0, 4026, 645, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 1, 1, 0, 27, 4093, 0, 3958, 4087, 1, 1, 4093, 1, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4092, 0, 1, 1, 0, 4095, 0, 2896, 2111, 0, 3, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 1, 1, 280, 628, 0, 0, 1, 1, 2, 2737, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 11, 4095, 1, 0, 4095, 4084, 1, 1, 1, 3289, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 1, 4095, 4095, 0, 1, 328, 0, 4, 0, 0, 21, 0, 2, 2842, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3955, 0, 1, 3770, 1, 0, 1, 1, 4095, 6, 1, 0, 0, 6, 3815, 705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 24, 0, 0, 1, 0, 0, 4095, 0, 3606, 1, 0, 1, 2267, 1983, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16239, 16334, 17297, 15938, 16196, 16590, 16358, 16120] avg=16384.0000 max=17297 max_rank=2 min=15938 min_rank=3 imbalance_ratio=1.055725 max_load_reduction=113775 max_load_reduction_pct=0.868034 moved_to_echo=65264 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 11, 4094, 0, 0, 0, 4095, 3366, 0, 1, 4018, 338, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 6, 1, 1, 0, 4096, 0, 3983, 4084, 0, 1, 4093, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4086, 1, 0, 0, 1, 4095, 0, 1579, 2111, 5, 0, 1, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 8, 0, 2, 280, 125, 1, 0, 1, 0, 1, 3766, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4092, 1, 0, 4096, 4082, 1, 1, 1, 3737, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1121, 0, 4095, 4094, 1, 0, 484, 0, 0, 0, 0, 10, 0, 4, 2784, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3873, 1, 0, 3775, 0, 1, 0, 1, 4095, 37, 0, 0, 0, 43, 3815, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 134, 0, 0, 0, 0, 1, 4096, 0, 4089, 0, 0, 0, 2268, 1983, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15927, 16338, 15983, 16470, 16630, 16663, 16348, 16713] avg=16384.0000 max=16713 max_rank=7 min=15927 min_rank=0 imbalance_ratio=1.020081 max_load_reduction=114359 max_load_reduction_pct=0.872490 moved_to_echo=66354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 8, 4092, 0, 0, 0, 4079, 3366, 0, 0, 3775, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 73, 0, 0, 26, 4089, 0, 4009, 4091, 0, 0, 4095, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3970, 0, 0, 0, 0, 4096, 1, 2764, 2112, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 280, 477, 0, 0, 0, 0, 1, 3988, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4094, 0, 0, 4095, 4074, 0, 0, 0, 3035, 1, 0, 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 0, 4095, 4095, 0, 0, 338, 0, 6, 0, 0, 50, 0, 14, 1486, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 4006, 0, 2, 0, 0, 4096, 23, 0, 0, 0, 27, 3816, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 84, 0, 0, 0, 0, 0, 4096, 0, 3919, 0, 1, 0, 2268, 1984, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16094, 16453, 17059, 17035, 15922, 15298, 16718, 16493] avg=16384.0000 max=17059 max_rank=2 min=15298 min_rank=5 imbalance_ratio=1.041199 max_load_reduction=114013 max_load_reduction_pct=0.869850 moved_to_echo=64431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 4084, 3366, 0, 0, 4064, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 18, 0, 0, 4, 4096, 0, 4062, 4046, 0, 0, 4096, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4091, 0, 0, 0, 0, 4096, 0, 2013, 2112, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 280, 227, 0, 0, 0, 0, 0, 3990, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4095, 0, 0, 0, 2846, 0, 0, 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 0, 4096, 4096, 0, 0, 134, 0, 2, 0, 0, 54, 0, 2, 3312, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3668, 0, 0, 3701, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 34, 3816, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 26, 0, 0, 0, 0, 0, 4096, 0, 4015, 0, 0, 0, 2268, 1984, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16247, 16394, 16409, 16792, 15752, 16912, 16036, 16530] avg=16384.0000 max=16912 max_rank=5 min=15752 min_rank=4 imbalance_ratio=1.032227 max_load_reduction=114160 max_load_reduction_pct=0.870972 moved_to_echo=65230 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 14, 4096, 0, 0, 1, 4096, 3366, 0, 0, 4087, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 47, 4096, 0, 4076, 4096, 0, 0, 4096, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 1, 0, 0, 4096, 0, 2521, 2112, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 280, 140, 0, 0, 0, 0, 0, 3397, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4084, 0, 0, 0, 3817, 0, 0, 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 0, 4096, 4096, 0, 0, 11, 0, 8, 0, 0, 19, 0, 0, 2382, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 3643, 0, 0, 0, 0, 4096, 84, 0, 0, 0, 11, 3816, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 40, 0, 0, 0, 0, 0, 4096, 0, 3834, 0, 0, 0, 2268, 1984, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16166, 16547, 16921, 16105, 16711, 15827, 16432, 16363] avg=16384.0000 max=16921 max_rank=2 min=15827 min_rank=5 imbalance_ratio=1.032776 max_load_reduction=114151 max_load_reduction_pct=0.870903 moved_to_echo=65333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 3, 4095, 0, 0, 1, 4095, 3366, 0, 0, 4072, 300, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 12, 1, 2, 1, 0, 16, 4094, 0, 4050, 4095, 1, 1, 4095, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 0, 59, 1, 0, 4095, 0, 1722, 2111, 0, 20, 1, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 1, 2, 280, 52, 0, 0, 1, 1, 2, 3258, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 28, 4095, 1, 1, 4094, 4073, 1, 1, 1, 3972, 0, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1120, 1, 4095, 4095, 0, 0, 1000, 0, 1, 0, 0, 27, 1, 13, 2351, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4082, 0, 1, 3565, 0, 0, 1, 1, 4095, 60, 1, 0, 0, 4, 3815, 705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 1, 37, 0, 0, 1, 0, 0, 4095, 0, 4087, 1, 0, 1, 2264, 1983, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15936, 16438, 16208, 15882, 16885, 16784, 16331, 16608] avg=16384.0000 max=16885 max_rank=4 min=15882 min_rank=3 imbalance_ratio=1.030579 max_load_reduction=114187 max_load_reduction_pct=0.871178 moved_to_echo=66608 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 4083, 15, 0, 0, 4069, 3357, 0, 0, 3972, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 84, 0, 0, 30, 4077, 0, 3895, 4094, 0, 0, 4083, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 4082, 4, 1790, 2111, 19, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 278, 1256, 19, 0, 0, 0, 15, 3738, 51, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4080, 0, 0, 4085, 4064, 0, 0, 0, 3508, 9, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 0, 4076, 4091, 0, 0, 332, 0, 22, 0, 0, 43, 0, 16, 2491, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 3244, 0, 13, 0, 0, 4096, 23, 0, 0, 0, 53, 3818, 709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4089, 0, 3794, 0, 17, 0, 2273, 1985, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15940, 16335, 16202, 17645, 16369, 16239, 16040, 16302] avg=16384.0000 max=17645 max_rank=3 min=15940 min_rank=0 imbalance_ratio=1.076965 max_load_reduction=113427 max_load_reduction_pct=0.865379 moved_to_echo=64950 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 4096, 0, 0, 5, 4096, 3366, 0, 0, 4030, 2928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4, 0, 0, 0, 0, 66, 4096, 0, 3785, 4095, 0, 0, 4096, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 525, 2112, 0, 67, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 280, 420, 0, 0, 0, 0, 0, 2217, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4092, 0, 0, 0, 3705, 0, 0, 0, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1122, 0, 4096, 4096, 0, 0, 592, 0, 0, 0, 0, 6, 0, 19, 2737, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 4094, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 24, 3816, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3454, 0, 0, 0, 2268, 1984, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18523, 16233, 14993, 15205, 16607, 16759, 16809, 15943] avg=16384.0000 max=18523 max_rank=0 min=14993 min_rank=2 imbalance_ratio=1.130554 max_load_reduction=112549 max_load_reduction_pct=0.858681 moved_to_echo=66118 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 128, 384, 0, 0, 128, 0, 128, 32768, 0, 30848, 128, 128, 128, 18176, 15872, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 32768, 128, 128, 128, 2304, 3328, 128, 0, 128, 128, 128, 27136, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 31872, 128, 128, 29824, 128, 128, 128, 128, 32768, 256, 128, 0, 0, 256, 30592, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 128, 32768, 128, 0, 128, 32768, 27008, 0, 128, 32128, 6656, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9088, 128, 32768, 32768, 128, 128, 3328, 0, 128, 0, 0, 256, 128, 128, 20480, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 32640, 128, 128, 128, 128, 32768, 128, 15872, 16896, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 0, 128, 32768, 128, 128, 32768, 32768, 128, 128, 128, 28032, 128, 0, 0, 4992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 256, 128, 128, 256, 32768, 0, 31872, 32768, 128, 128, 32768, 128, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[8, 0, 4087, 0, 0, 2898, 4093, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 4096, 4096, 0, 4096, 0, 4096, 4007, 0, 0, 0, 0, 1515, 4093, 0, 0, 0, 4096, 0, 1, 0, 17, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 0, 3465, 4096, 0, 4037, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 2044, 0, 0, 4096, 0, 0, 4096, 1812, 0, 0, 3027, 4096, 4096, 0, 32, 0, 4096, 0, 0, 4096, 0, 0, 33, 0, 1706, 4096, 0, 0, 0, 28, 0, 1, 0, 0, 4077, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 4092, 0, 0, 2715, 4096, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4096, 4095, 0, 4096, 0, 4096, 3926, 0, 0, 0, 1, 1887, 4062, 0, 0, 1, 4096, 0, 2, 0, 68, 0, 0, 0, 73, 4096, 0, 0, 0, 4096, 0, 0, 0, 3933, 4096, 0, 4008, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4095, 0, 0, 663, 0, 0, 4096, 0, 0, 4096, 3336, 0, 0, 2983, 4096, 4095, 0, 12, 0, 4096, 0, 0, 4096, 0, 0, 65, 0, 830, 4090, 0, 0, 0, 24, 0, 147, 0, 0, 4054, 10, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[6, 0, 4076, 0, 0, 3269, 4096, 0, 0, 0, 4096, 33, 0, 0, 0, 0, 0, 0, 22, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 4096, 4083, 0, 4096, 0, 4096, 3836, 0, 0, 0, 8, 1183, 4065, 0, 0, 19, 4096, 0, 0, 0, 22, 0, 0, 0, 119, 4089, 0, 0, 0, 4096, 0, 0, 0, 3921, 4096, 11, 3904, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 5, 931, 0, 0, 4096, 0, 0, 4096, 2983, 0, 0, 3812, 4091, 4077, 0, 71, 0, 4079, 0, 0, 4096, 0, 0, 28, 0, 546, 4084, 0, 0, 0, 55, 0, 38, 0, 0, 3996, 28, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 4082, 0, 1, 3718, 4078, 0, 0, 1, 4095, 2, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 4095, 4095, 4095, 0, 4095, 1, 4095, 3916, 0, 0, 0, 0, 780, 4067, 0, 0, 9, 4095, 0, 18, 1, 0, 0, 1, 0, 13, 4095, 1, 1, 0, 4095, 0, 0, 39, 3554, 4095, 0, 4091, 4093, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 4093, 0, 0, 1, 1, 1, 4095, 1, 0, 1127, 1, 1, 4094, 1, 0, 4095, 2993, 0, 1, 3459, 4095, 4093, 1, 40, 0, 4095, 0, 1, 4093, 1, 0, 6, 0, 769, 4095, 0, 0, 0, 81, 0, 22, 0, 0, 4093, 0, 4095, 1, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 4086, 0, 0, 3026, 4080, 1, 0, 0, 4095, 6, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 4090, 4095, 4095, 1, 4095, 1, 4095, 3921, 0, 1, 0, 0, 1315, 4089, 0, 0, 7, 4095, 0, 25, 1, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 4095, 0, 0, 4, 4003, 4095, 0, 4087, 4094, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 1, 1, 1, 4095, 1, 0, 1631, 1, 1, 4095, 1, 0, 4095, 2141, 0, 1, 3953, 4095, 4094, 1, 4, 0, 4095, 0, 1, 4094, 1, 0, 0, 1, 459, 4095, 0, 0, 0, 21, 0, 9, 0, 0, 4091, 0, 4095, 1, 0, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4087, 0, 0, 2705, 4074, 1, 0, 1, 4095, 11, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 4095, 4095, 4093, 1, 4095, 0, 4094, 3822, 0, 0, 0, 1, 1380, 4079, 0, 1, 4, 4095, 1, 0, 1, 4, 0, 0, 0, 61, 4095, 1, 1, 0, 4095, 0, 0, 6, 3654, 4095, 1, 3823, 4095, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 4095, 0, 0, 1, 1, 0, 4095, 1, 0, 1174, 0, 1, 4095, 1, 0, 4095, 3242, 0, 1, 3389, 4095, 4094, 1, 173, 0, 4092, 0, 1, 4094, 1, 0, 33, 1, 1063, 4095, 1, 0, 3, 42, 1, 53, 0, 0, 4072, 5, 4095, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4, 0, 4087, 0, 0, 3140, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 4096, 0, 4096, 0, 4096, 4072, 0, 0, 0, 0, 2175, 4095, 0, 0, 0, 4096, 0, 54, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3870, 4096, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 27, 0, 0, 4096, 0, 0, 4096, 3179, 0, 0, 3172, 4096, 4096, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 1, 1, 412, 4096, 0, 0, 0, 0, 0, 397, 0, 0, 4090, 1, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 2231, 4084, 0, 0, 0, 4096, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 4096, 4096, 0, 4096, 0, 4096, 3391, 0, 0, 0, 0, 3117, 4059, 0, 0, 4, 4096, 0, 0, 0, 22, 0, 0, 0, 231, 4096, 0, 0, 0, 4096, 0, 0, 25, 3912, 4096, 0, 3922, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 2035, 0, 0, 4096, 0, 0, 4096, 2313, 0, 0, 2842, 4096, 4096, 0, 71, 0, 4096, 0, 0, 4096, 0, 0, 32, 0, 456, 4096, 0, 0, 0, 40, 0, 56, 0, 0, 4041, 2, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 4087, 0, 0, 2898, 4093, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 638, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 4096, 4096, 3193, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4007, 0, 0, 0, 0, 1515, 4093, 0, 0, 0, 2675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 17, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 0, 3465, 4096, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4037, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2044, 0, 0, 4096, 0, 0, 4096, 1812, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3027, 4096, 4096, 0, 32, 0, 4096, 0, 0, 698, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1706, 4096, 0, 0, 0, 28, 0, 1, 0, 0, 4077, 0, 3219, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16322, 16374, 16386, 15981, 16424, 16284, 16078, 17223] avg=16384.0000 max=17223 max_rank=7 min=15981 min_rank=3 imbalance_ratio=1.051208 max_load_reduction=113849 max_load_reduction_pct=0.868599 moved_to_echo=66009 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4082, 0, 1, 3718, 4078, 0, 0, 1, 4095, 2, 1, 0, 1, 1, 637, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 4095, 4095, 4095, 3191, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 3916, 0, 0, 0, 0, 780, 4067, 0, 0, 9, 2675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1, 0, 0, 1, 0, 13, 4095, 1, 1, 0, 4095, 0, 0, 39, 3554, 4095, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4091, 4093, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 4093, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4095, 1, 0, 1127, 1, 1, 4094, 1, 0, 4095, 2993, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3459, 4095, 4093, 1, 40, 0, 4095, 0, 1, 697, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 769, 4095, 0, 0, 0, 81, 0, 22, 0, 0, 4093, 0, 3219, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17118, 16403, 15543, 16118, 16476, 16551, 16489, 16374] avg=16384.0000 max=17118 max_rank=0 min=15543 min_rank=2 imbalance_ratio=1.044800 max_load_reduction=113954 max_load_reduction_pct=0.869400 moved_to_echo=65890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4092, 0, 0, 2715, 4096, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 638, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4096, 4095, 3193, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3926, 0, 0, 0, 1, 1887, 4062, 0, 0, 1, 2675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 68, 0, 0, 0, 73, 4096, 0, 0, 0, 4096, 0, 0, 0, 3933, 4096, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4008, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 663, 0, 0, 4096, 0, 0, 4096, 3336, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2983, 4096, 4095, 0, 12, 0, 4096, 0, 0, 698, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 830, 4090, 0, 0, 0, 24, 0, 147, 0, 0, 4054, 10, 3219, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16142, 16376, 16648, 16569, 16395, 16427, 16045, 16470] avg=16384.0000 max=16648 max_rank=2 min=16045 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=114424 max_load_reduction_pct=0.872986 moved_to_echo=65337 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4087, 0, 0, 2705, 4074, 1, 0, 1, 4095, 11, 0, 0, 1, 0, 637, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 4095, 4095, 4093, 3191, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 3822, 0, 0, 0, 1, 1380, 4079, 0, 1, 4, 2675, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 61, 4095, 1, 1, 0, 4095, 0, 0, 6, 3654, 4095, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 3823, 4095, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 4095, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 1, 0, 1174, 0, 1, 4095, 1, 0, 4095, 3242, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3389, 4095, 4094, 1, 173, 0, 4092, 0, 1, 698, 1, 0, 33, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1063, 4095, 1, 0, 3, 42, 1, 53, 0, 0, 4072, 5, 3219, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16112, 16399, 16058, 16218, 16211, 16846, 16579, 16649] avg=16384.0000 max=16846 max_rank=5 min=16058 min_rank=2 imbalance_ratio=1.028198 max_load_reduction=114226 max_load_reduction_pct=0.871475 moved_to_echo=66285 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 4076, 0, 0, 3269, 4096, 0, 0, 0, 4096, 33, 0, 0, 0, 0, 641, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 4096, 4083, 3198, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3836, 0, 0, 0, 8, 1183, 4065, 0, 0, 19, 2670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 119, 4089, 0, 0, 0, 4096, 0, 0, 0, 3921, 4096, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 3904, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 931, 0, 0, 4096, 0, 0, 4096, 2983, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3812, 4091, 4077, 0, 71, 0, 4079, 0, 0, 687, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 4084, 0, 0, 0, 55, 0, 38, 0, 0, 3996, 28, 3209, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16719, 16370, 15877, 16554, 16305, 16350, 16845, 16052] avg=16384.0000 max=16845 max_rank=6 min=15877 min_rank=2 imbalance_ratio=1.028137 max_load_reduction=114227 max_load_reduction_pct=0.871483 moved_to_echo=65552 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4086, 0, 0, 3026, 4080, 1, 0, 0, 4095, 6, 1, 0, 1, 0, 637, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 4090, 4095, 4095, 3191, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 3921, 0, 1, 0, 0, 1315, 4089, 0, 0, 7, 2675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 1, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 4095, 0, 0, 4, 4003, 4095, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4087, 4094, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 4095, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4095, 1, 0, 1631, 1, 1, 4095, 1, 0, 4095, 2141, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3953, 4095, 4094, 1, 4, 0, 4095, 0, 1, 698, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 4095, 0, 0, 0, 21, 0, 9, 0, 0, 4091, 0, 3219, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16434, 16399, 16105, 16525, 16474, 16204, 16944, 15987] avg=16384.0000 max=16944 max_rank=6 min=15987 min_rank=7 imbalance_ratio=1.034180 max_load_reduction=114128 max_load_reduction_pct=0.870728 moved_to_echo=65609 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4087, 0, 0, 3140, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 638, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 4096, 3193, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4072, 0, 0, 0, 0, 2175, 4095, 0, 0, 0, 2675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3870, 4096, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 27, 0, 0, 4096, 0, 0, 4096, 3179, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3172, 4096, 4096, 0, 1, 0, 4096, 0, 0, 698, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 412, 4096, 0, 0, 0, 0, 0, 397, 0, 0, 4090, 1, 3219, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16560, 16392, 17113, 16424, 16477, 15634, 16161, 16311] avg=16384.0000 max=17113 max_rank=2 min=15634 min_rank=5 imbalance_ratio=1.044495 max_load_reduction=113959 max_load_reduction_pct=0.869438 moved_to_echo=64583 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 2231, 4084, 0, 0, 0, 4096, 23, 0, 0, 0, 0, 638, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 4096, 4096, 3193, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3391, 0, 0, 0, 0, 3117, 4059, 0, 0, 4, 2675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 231, 4096, 0, 0, 0, 4096, 0, 0, 25, 3912, 4096, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3922, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2035, 0, 0, 4096, 0, 0, 4096, 2313, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2842, 4096, 4096, 0, 71, 0, 4096, 0, 0, 698, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 4096, 0, 0, 0, 40, 0, 56, 0, 0, 4041, 2, 3219, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15665, 16359, 17342, 16683, 16310, 16776, 15931, 16006] avg=16384.0000 max=17342 max_rank=2 min=15665 min_rank=0 imbalance_ratio=1.058472 max_load_reduction=113730 max_load_reduction_pct=0.867691 moved_to_echo=65023 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 32768, 0, 128, 23808, 32768, 128, 0, 128, 32768, 128, 128, 0, 128, 128, 5120, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 128, 128, 9728, 128, 128, 32768, 128, 0, 32768, 22016, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 32768, 30976, 0, 128, 0, 128, 13440, 32640, 0, 128, 128, 21504, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 32000, 32768, 128, 128, 0, 128, 0, 128, 0, 128, 0, 128, 128, 32768, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 256, 0, 128, 0, 512, 32768, 128, 128, 0, 32768, 0, 0, 128, 30336, 32768, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[6272, 32768, 128, 0, 128, 384, 128, 768, 0, 0, 32640, 128, 25856, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 26752, 32768, 32768, 128, 512, 0, 32768, 0, 128, 5632, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 0, 0, 128, 128, 128, 32640, 32768, 32768, 25600, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 1, 1, 4095, 4095, 4093, 1, 0, 25, 1, 0, 1, 0, 0, 43, 1, 4070, 1, 0, 1, 0, 0, 1, 4093, 4095, 1, 0, 0, 0, 4087, 4095, 4095, 0, 0, 0, 0, 1, 3937, 4094, 0, 0, 0, 0, 0, 15, 4095, 4042, 620, 0, 1, 1, 1, 4095, 0, 1, 1, 4056, 0, 1, 0, 4074, 0, 4093, 4095, 2878, 1, 91, 1, 1, 0, 1, 1, 0, 1, 4095, 4086, 113, 4038, 451, 3508, 1, 0, 0, 12, 4095, 1, 4095, 4095, 1, 0, 0, 4095, 794, 0, 1, 0, 1, 1, 1, 0, 4095, 0, 4095, 4095, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 10, 1, 0, 0, 4094, 1, 0, 1, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4080, 0, 0, 146, 0, 42, 0, 0, 0, 63, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 1, 0, 0, 3732, 4094, 0, 0, 0, 0, 0, 10, 4096, 4093, 488, 0, 0, 0, 0, 4095, 0, 0, 0, 4087, 0, 0, 0, 4043, 0, 4091, 4096, 3339, 0, 24, 0, 0, 0, 0, 0, 0, 0, 4096, 4067, 9, 3927, 419, 3850, 0, 0, 0, 2, 4074, 0, 4096, 4096, 0, 0, 0, 4096, 466, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4095, 4096, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4089, 0, 0, 19, 0, 61, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4048, 4096, 4096, 0, 0, 0, 0, 0, 3963, 4095, 0, 0, 0, 0, 0, 0, 4096, 4086, 661, 0, 0, 0, 0, 4096, 0, 0, 0, 3733, 0, 0, 0, 4093, 0, 4095, 4096, 2965, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 7, 4051, 764, 4057, 0, 0, 0, 1, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 233, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 12, 0, 0, 3, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4088, 4057, 0, 0, 46, 0, 36, 0, 0, 1, 67, 0, 4096, 0, 0, 0, 0, 17, 0, 4096, 4096, 0, 0, 0, 0, 4001, 4096, 4096, 0, 1, 19, 6, 0, 3480, 4080, 0, 0, 0, 0, 0, 4, 4096, 4095, 516, 0, 0, 0, 0, 4085, 0, 0, 0, 3666, 0, 0, 0, 4045, 0, 4090, 4096, 3269, 0, 87, 0, 0, 0, 0, 0, 0, 0, 4088, 4060, 22, 3957, 710, 3748, 0, 0, 0, 2, 4087, 0, 4096, 4096, 0, 0, 0, 4096, 1130, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 24, 0, 0, 0, 4095, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 1, 0, 4094, 4094, 4022, 1, 0, 9, 0, 1, 0, 0, 0, 20, 1, 4095, 1, 0, 1, 0, 1, 1, 4095, 4095, 0, 1, 0, 1, 4015, 4094, 4095, 0, 1, 1, 0, 1, 3884, 4086, 0, 0, 0, 0, 0, 15, 4095, 4093, 172, 1, 0, 0, 1, 4095, 0, 1, 1, 3777, 0, 1, 0, 4071, 0, 4094, 4095, 3079, 1, 97, 0, 1, 0, 0, 1, 0, 1, 4095, 4079, 15, 4081, 1055, 3507, 0, 0, 0, 0, 4094, 1, 4095, 4095, 1, 0, 1, 4095, 969, 0, 1, 0, 1, 0, 0, 0, 4092, 0, 4095, 4095, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 5, 0, 0, 0, 4095, 0, 0, 0, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4083, 0, 0, 70, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 0, 3901, 4093, 49, 0, 0, 0, 0, 41, 4096, 4095, 78, 0, 0, 0, 0, 4096, 0, 0, 0, 4014, 0, 0, 0, 4094, 0, 4085, 4096, 3295, 0, 1244, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 279, 3935, 76, 2904, 0, 0, 0, 3, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 521, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 9, 0, 0, 0, 0, 4, 0, 0, 0, 79, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4018, 0, 0, 109, 0, 0, 0, 0, 1, 39, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3979, 4096, 4096, 0, 0, 0, 0, 0, 3746, 4094, 1, 0, 0, 0, 0, 5, 4096, 4030, 1105, 0, 0, 0, 0, 4096, 0, 0, 0, 4053, 0, 0, 0, 4048, 0, 4095, 4096, 2790, 0, 30, 0, 0, 0, 0, 0, 0, 0, 4096, 4069, 20, 4051, 627, 3987, 0, 0, 0, 2, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 309, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 1, 1, 4095, 4095, 4062, 1, 0, 4, 1, 0, 1, 0, 0, 46, 1, 4077, 1, 0, 1, 0, 0, 1, 4093, 4095, 1, 0, 0, 0, 4067, 4095, 4095, 0, 0, 0, 0, 1, 4030, 4094, 0, 0, 0, 0, 0, 5, 4095, 4019, 589, 0, 1, 1, 1, 4095, 0, 1, 1, 3810, 0, 1, 0, 4076, 0, 4087, 4095, 2463, 1, 131, 1, 1, 0, 1, 1, 0, 1, 4095, 4089, 69, 3921, 1343, 3764, 0, 0, 0, 6, 4095, 1, 4095, 4095, 1, 0, 0, 4095, 468, 0, 1, 0, 1, 1, 1, 0, 4095, 0, 4095, 4095, 0, 0, 5, 0, 0, 6, 0, 4, 1, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 1, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4058, 0, 1, 0, 1, 1, 4095, 4095, 4093, 1, 0, 25, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 1, 4070, 1, 0, 1, 0, 0, 1, 4093, 4095, 1, 0, 0, 0, 2986, 760, 277, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4095, 293, 0, 0, 0, 0, 1, 3937, 4094, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 620, 0, 1, 1, 1, 3818, 0, 1, 1, 4056, 0, 1, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 827, 2878, 1, 91, 1, 1, 0, 1, 1, 0, 1, 4095, 4086, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 451, 3508, 1, 0, 0, 12, 4095, 1, 0, 0, 1, 0, 0, 3335, 794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 4095, 0, 4095, 4095, 0, 0, 2, 0, 0, 3802, 282, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 1, 0, 0, 4094, 1, 0, 1, 1, 1, 4095, 4095, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 16366, 16522, 16616, 16189, 16236, 16398, 16373] avg=16384.0000 max=16616 max_rank=3 min=16189 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=114456 max_load_reduction_pct=0.873230 moved_to_echo=65196 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4059, 0, 0, 0, 0, 0, 4096, 4096, 4080, 0, 0, 146, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 2987, 760, 277, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 293, 0, 0, 1, 0, 0, 3732, 4094, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 488, 0, 0, 0, 0, 3818, 0, 0, 0, 4087, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 826, 3339, 0, 24, 0, 0, 0, 0, 0, 0, 0, 4096, 4067, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 419, 3850, 0, 0, 0, 2, 4074, 0, 0, 0, 0, 0, 0, 3336, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4095, 4096, 0, 0, 1, 0, 0, 3803, 283, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16519, 16413, 16321, 16529, 16452, 16074, 16393, 16371] avg=16384.0000 max=16529 max_rank=3 min=16074 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=114543 max_load_reduction_pct=0.873894 moved_to_echo=65290 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4059, 0, 0, 0, 0, 0, 4096, 4096, 4089, 0, 0, 19, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 2987, 760, 277, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 4096, 293, 0, 0, 0, 0, 0, 3963, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 661, 0, 0, 0, 0, 3819, 0, 0, 0, 3733, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 826, 2965, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 764, 4057, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 3336, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 12, 0, 0, 3803, 283, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16420, 16350, 16495, 16392, 16092, 16538, 16407, 16378] avg=16384.0000 max=16538 max_rank=5 min=16092 min_rank=4 imbalance_ratio=1.009399 max_load_reduction=114534 max_load_reduction_pct=0.873825 moved_to_echo=65415 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4056, 0, 0, 0, 0, 0, 4095, 4088, 4057, 0, 0, 46, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 67, 0, 4096, 0, 0, 0, 0, 17, 0, 4096, 4096, 0, 0, 0, 0, 2991, 763, 281, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 4096, 290, 0, 1, 19, 6, 0, 3480, 4080, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 516, 0, 0, 0, 0, 3804, 0, 0, 0, 3666, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 819, 3269, 0, 87, 0, 0, 0, 0, 0, 0, 0, 4088, 4060, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3957, 710, 3748, 0, 0, 0, 2, 4087, 0, 0, 0, 0, 0, 0, 3333, 1130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 4096, 4096, 3, 0, 0, 0, 0, 3806, 286, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 24, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 16448, 15977, 16126, 16435, 16967, 16345, 16396] avg=16384.0000 max=16967 max_rank=5 min=15977 min_rank=2 imbalance_ratio=1.035583 max_load_reduction=114105 max_load_reduction_pct=0.870552 moved_to_echo=66143 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4058, 0, 1, 0, 1, 0, 4094, 4094, 4022, 1, 0, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1, 4095, 1, 0, 1, 0, 1, 1, 4095, 4095, 0, 1, 0, 1, 2986, 760, 277, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 4094, 293, 0, 1, 1, 0, 1, 3884, 4086, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 172, 1, 0, 0, 1, 3818, 0, 1, 1, 3777, 0, 1, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 827, 3079, 1, 97, 0, 1, 0, 0, 1, 0, 1, 4095, 4079, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 1055, 3507, 0, 0, 0, 0, 4094, 1, 0, 0, 1, 0, 1, 3335, 969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4092, 0, 4095, 4095, 0, 0, 0, 1, 0, 3802, 282, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 5, 0, 0, 0, 4095, 0, 0, 0, 1, 1, 4095, 4095, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16281, 16372, 16390, 15936, 16290, 17044, 16392, 16367] avg=16384.0000 max=17044 max_rank=5 min=15936 min_rank=3 imbalance_ratio=1.040283 max_load_reduction=114028 max_load_reduction_pct=0.869965 moved_to_echo=66093 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4059, 0, 0, 0, 0, 0, 4096, 4096, 4083, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 2987, 760, 277, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 293, 0, 0, 0, 0, 0, 3901, 4093, 49, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 78, 0, 0, 0, 0, 3819, 0, 0, 0, 4014, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 826, 3295, 0, 1244, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 76, 2904, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 3336, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 9, 0, 0, 3803, 283, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 79, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16404, 16358, 16568, 16100, 17921, 14871, 16406, 16444] avg=16384.0000 max=17921 max_rank=4 min=14871 min_rank=5 imbalance_ratio=1.093811 max_load_reduction=113151 max_load_reduction_pct=0.863274 moved_to_echo=65642 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4058, 0, 1, 0, 1, 1, 4095, 4095, 4062, 1, 0, 4, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1, 4077, 1, 0, 1, 0, 0, 1, 4093, 4095, 1, 0, 0, 0, 2986, 760, 277, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4095, 293, 0, 0, 0, 0, 1, 4030, 4094, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 589, 0, 1, 1, 1, 3818, 0, 1, 1, 3810, 0, 1, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 827, 2463, 1, 131, 1, 1, 0, 1, 1, 0, 1, 4095, 4089, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 1343, 3764, 0, 0, 0, 6, 4095, 1, 0, 0, 1, 0, 0, 3335, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 4095, 0, 4095, 4095, 0, 0, 5, 0, 0, 3802, 282, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4, 1, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 1, 1, 1, 4095, 4095, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16320, 16376, 16585, 16318, 15767, 16934, 16401, 16371] avg=16384.0000 max=16934 max_rank=5 min=15767 min_rank=4 imbalance_ratio=1.033569 max_load_reduction=114138 max_load_reduction_pct=0.870804 moved_to_echo=65473 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4059, 0, 0, 0, 0, 0, 4096, 4096, 4018, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 39, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 2987, 760, 277, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 4096, 293, 0, 0, 0, 0, 0, 3746, 4094, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 1105, 0, 0, 0, 0, 3819, 0, 0, 0, 4053, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 826, 2790, 0, 30, 0, 0, 0, 0, 0, 0, 0, 4096, 4069, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 627, 3987, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 3336, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 4096, 4096, 0, 0, 0, 0, 0, 3803, 283, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 16389, 16214, 17055, 15926, 16408, 16330, 16372] avg=16384.0000 max=17055 max_rank=3 min=15926 min_rank=4 imbalance_ratio=1.040955 max_load_reduction=114017 max_load_reduction_pct=0.869881 moved_to_echo=65036 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 6656, 24192, 128, 1792, 128, 128, 0, 128, 128, 0, 128, 32768, 32640, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32000, 5504, 29440, 128, 0, 0, 128, 32768, 128, 0, 0, 128, 0, 128, 26752, 4992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32512, 32768, 2432, 0, 128, 128, 128, 128, 30720, 32768, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32512, 0, 128, 0, 128, 128, 32768, 32768, 32512, 128, 0, 512, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32640, 4352, 128, 128, 128, 128, 30592, 0, 128, 128, 31232, 0, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 0, 128, 256, 128, 0, 0, 32768, 128, 0, 128, 128, 128, 32768, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 128, 128, 0, 32640, 0, 32768, 32768, 128, 0, 128, 128, 0, 30464, 2304, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 384, 128, 32768, 128, 0, 128, 0, 128, 128, 32768, 32768, 128, 128, 0, 128, 23936, 6144, 2304, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1, 3923, 0, 4095, 1, 27, 1, 0, 6, 0, 1, 0, 0, 1, 0, 2994, 0, 0, 1, 4095, 1, 1, 4094, 1, 1, 1, 0, 4095, 0, 1, 1, 0, 4093, 1, 0, 3949, 4092, 1, 41, 4095, 1, 197, 4094, 284, 0, 0, 4094, 0, 11, 4081, 4095, 0, 0, 1, 1, 0, 4095, 4070, 0, 0, 60, 4095, 0, 1, 4095, 1, 1, 0, 0, 80, 657, 0, 1, 0, 85, 1, 1, 0, 4094, 0, 4095, 4029, 0, 0, 4095, 4095, 4095, 4094, 4062, 1, 0, 0, 0, 1, 48, 0, 4095, 4095, 1, 1, 1, 4095, 0, 1, 7, 4094, 1, 1, 0, 0, 4095, 0, 0, 60, 4095, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 3804, 0, 4096, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3751, 0, 4, 0, 4096, 0, 0, 4096, 0, 0, 0, 15, 4096, 0, 18, 0, 0, 4054, 0, 0, 4096, 4074, 0, 36, 4094, 0, 34, 4056, 130, 0, 21, 4083, 0, 53, 4043, 4096, 13, 0, 0, 0, 4, 4078, 3955, 10, 0, 47, 4075, 0, 0, 4096, 0, 0, 0, 0, 0, 346, 2, 0, 0, 21, 0, 0, 0, 4096, 0, 4096, 4096, 0, 68, 4096, 4096, 4083, 4084, 4096, 0, 25, 0, 16, 0, 8, 0, 4096, 4079, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 37, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 3662, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2228, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4094, 4095, 0, 275, 4096, 0, 207, 4096, 315, 0, 0, 4096, 0, 2, 4094, 4096, 0, 0, 0, 0, 0, 4095, 3945, 0, 0, 85, 4095, 0, 0, 4096, 0, 0, 0, 0, 6, 1556, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 4096, 4096, 0, 6, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4, 4096, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 3750, 0, 4096, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 26, 0, 0, 4093, 0, 0, 4095, 4095, 0, 42, 4096, 0, 26, 4089, 78, 0, 3, 4093, 0, 27, 4095, 4096, 1, 0, 0, 0, 1, 4079, 4045, 0, 0, 71, 4081, 0, 0, 4096, 0, 0, 0, 0, 0, 184, 0, 0, 0, 94, 0, 0, 0, 4096, 0, 4096, 4096, 0, 7, 4096, 4096, 4095, 4094, 4096, 0, 0, 0, 2, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 3567, 0, 4096, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3344, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4088, 4095, 0, 2, 4096, 0, 227, 4095, 351, 0, 2, 4096, 0, 16, 4030, 4096, 0, 0, 0, 0, 0, 4091, 3797, 0, 0, 37, 4092, 0, 0, 4096, 0, 0, 0, 0, 7, 933, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 4096, 4096, 0, 6, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 12, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1, 3586, 0, 4095, 1, 286, 1, 0, 5, 0, 1, 0, 0, 0, 0, 3530, 2, 1, 1, 4095, 1, 1, 4095, 1, 1, 1, 1, 4095, 0, 10, 1, 0, 4091, 0, 0, 3952, 4086, 1, 7, 4095, 1, 247, 4093, 207, 0, 0, 4094, 0, 26, 3985, 4095, 1, 0, 0, 0, 0, 4095, 4063, 0, 0, 114, 4093, 0, 1, 4095, 1, 0, 1, 1, 64, 339, 0, 0, 1, 65, 1, 1, 0, 4095, 0, 4095, 4058, 0, 0, 4086, 4095, 4094, 4095, 4090, 0, 0, 0, 1, 1, 2, 1, 4095, 4086, 1, 1, 1, 4095, 0, 1, 0, 4095, 0, 1, 0, 0, 4095, 0, 0, 59, 4095, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 3935, 0, 4095, 1, 132, 1, 0, 9, 0, 1, 0, 0, 1, 0, 3096, 0, 0, 1, 4095, 1, 1, 4094, 1, 1, 1, 0, 4095, 0, 0, 1, 0, 4052, 0, 0, 3965, 4092, 1, 66, 4095, 1, 249, 4093, 174, 0, 0, 4095, 0, 14, 4042, 4095, 0, 0, 1, 1, 0, 4095, 4058, 0, 0, 101, 4095, 0, 1, 4095, 1, 5, 0, 1, 40, 569, 0, 1, 0, 126, 1, 1, 0, 4095, 0, 4095, 3957, 0, 0, 4095, 4095, 4095, 4094, 4063, 1, 0, 0, 0, 1, 23, 0, 4095, 4095, 1, 1, 1, 4094, 0, 1, 0, 4095, 1, 1, 0, 0, 4095, 0, 0, 90, 4095, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 3474, 0, 4096, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3236, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 3937, 4077, 0, 13, 4096, 0, 177, 4086, 316, 0, 0, 4096, 0, 52, 4084, 4096, 0, 0, 0, 0, 0, 4095, 4015, 0, 0, 69, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 974, 0, 0, 0, 70, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4, 4095, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 46, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 3923, 0, 4095, 1, 27, 1, 0, 6, 0, 1, 0, 0, 1, 0, 2994, 4080, 819, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 4094, 1, 1, 1, 0, 4095, 0, 1, 1, 0, 3720, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 3949, 3728, 1, 41, 0, 1, 197, 4094, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4081, 51, 0, 0, 1, 1, 0, 4095, 4070, 0, 0, 60, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 80, 657, 0, 1, 0, 85, 1, 1, 0, 4094, 0, 4093, 3275, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 15, 0, 4095, 4094, 4062, 1, 0, 0, 0, 1, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 1, 1, 1, 4095, 0, 1, 7, 4094, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 4095, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4095, 4095, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16273, 16376, 16389, 16466, 16425, 16345, 16394, 16404] avg=16384.0000 max=16466 max_rank=3 min=16273 min_rank=0 imbalance_ratio=1.005005 max_load_reduction=114606 max_load_reduction_pct=0.874374 moved_to_echo=65568 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 3586, 0, 4095, 1, 286, 1, 0, 5, 0, 1, 0, 0, 0, 0, 3530, 4071, 818, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 4095, 1, 1, 4095, 1, 1, 1, 1, 4095, 0, 10, 1, 0, 3720, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 3952, 3722, 1, 7, 0, 1, 247, 4093, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 3985, 51, 1, 0, 0, 0, 0, 4095, 4063, 0, 0, 114, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 1, 64, 339, 0, 0, 1, 65, 1, 1, 0, 4095, 0, 4093, 3276, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 15, 0, 4094, 4095, 4090, 0, 0, 0, 1, 1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4086, 1, 1, 1, 4095, 0, 1, 0, 4095, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 4095, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 4095, 4095, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16719, 16390, 16321, 16429, 16074, 16357, 16378, 16404] avg=16384.0000 max=16719 max_rank=0 min=16074 min_rank=4 imbalance_ratio=1.020447 max_load_reduction=114353 max_load_reduction_pct=0.872444 moved_to_echo=65213 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3567, 0, 4096, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3344, 4081, 819, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3720, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 3731, 0, 2, 0, 0, 227, 4095, 351, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4030, 52, 0, 0, 0, 0, 0, 4091, 3797, 0, 0, 37, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 933, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 4094, 3277, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 15, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16270, 16372, 16592, 16115, 16563, 16417, 16386, 16357] avg=16384.0000 max=16592 max_rank=2 min=16115 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=114480 max_load_reduction_pct=0.873413 moved_to_echo=65723 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3750, 0, 4096, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 4081, 819, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 26, 0, 0, 3720, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4095, 3731, 0, 42, 0, 0, 26, 4089, 78, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4095, 52, 1, 0, 0, 0, 1, 4079, 4045, 0, 0, 71, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 184, 0, 0, 0, 94, 0, 0, 0, 4096, 0, 4094, 3274, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 15, 0, 4095, 4094, 4096, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17036, 16399, 16157, 16452, 15879, 16406, 16386, 16357] avg=16384.0000 max=17036 max_rank=0 min=15879 min_rank=4 imbalance_ratio=1.039795 max_load_reduction=114036 max_load_reduction_pct=0.870026 moved_to_echo=65028 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3804, 0, 4096, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3751, 4085, 817, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 4096, 0, 0, 0, 15, 4096, 0, 18, 0, 0, 3725, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 4096, 3706, 0, 36, 0, 0, 34, 4056, 130, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 4043, 43, 13, 0, 0, 0, 4, 4078, 3955, 10, 0, 47, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 346, 2, 0, 0, 21, 0, 0, 0, 4096, 0, 4096, 3266, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 68, 11, 0, 4083, 4084, 4096, 0, 25, 0, 16, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 4096, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16997, 16418, 16133, 16321, 15965, 16487, 16367, 16384] avg=16384.0000 max=16997 max_rank=0 min=15965 min_rank=4 imbalance_ratio=1.037415 max_load_reduction=114075 max_load_reduction_pct=0.870323 moved_to_echo=65203 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3662, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2228, 4081, 818, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3720, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 3731, 0, 275, 0, 0, 207, 4096, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 52, 0, 0, 0, 0, 0, 4095, 3945, 0, 0, 85, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 1556, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 4094, 3278, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 15, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 4, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15210, 16371, 16813, 16368, 17169, 16406, 16390, 16345] avg=16384.0000 max=17169 max_rank=4 min=15210 min_rank=0 imbalance_ratio=1.047913 max_load_reduction=113903 max_load_reduction_pct=0.869011 moved_to_echo=66310 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 3935, 0, 4095, 1, 132, 1, 0, 9, 0, 1, 0, 0, 1, 0, 3096, 4080, 819, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 4094, 1, 1, 1, 0, 4095, 0, 0, 1, 0, 3720, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 3965, 3728, 1, 66, 0, 1, 249, 4093, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4042, 51, 0, 0, 1, 1, 0, 4095, 4058, 0, 0, 101, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 5, 0, 1, 40, 569, 0, 1, 0, 126, 1, 1, 0, 4095, 0, 4095, 3276, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3957, 0, 0, 15, 0, 4095, 4094, 4063, 1, 0, 0, 0, 1, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 1, 4094, 0, 1, 0, 4095, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 4095, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4095, 4095, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16495, 16375, 16329, 16459, 16348, 16249, 16385, 16432] avg=16384.0000 max=16495 max_rank=0 min=16249 min_rank=5 imbalance_ratio=1.006775 max_load_reduction=114577 max_load_reduction_pct=0.874153 moved_to_echo=65414 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3474, 0, 4096, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3236, 4080, 818, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3720, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3937, 3714, 0, 13, 0, 0, 177, 4086, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 4084, 52, 0, 0, 0, 0, 0, 4095, 4015, 0, 0, 69, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 974, 0, 0, 0, 70, 0, 0, 0, 4096, 0, 4094, 3278, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 15, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16072, 16371, 16338, 16462, 16649, 16405, 16386, 16389] avg=16384.0000 max=16649 max_rank=4 min=16072 min_rank=0 imbalance_ratio=1.016174 max_load_reduction=114423 max_load_reduction_pct=0.872978 moved_to_echo=65829 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 29824, 0, 32768, 128, 768, 128, 0, 128, 0, 128, 0, 0, 128, 0, 26112, 32640, 6656, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 384, 32768, 0, 128, 128, 128, 128, 128, 0, 128, 0, 0, 128, 0, 0, 32768, 32768, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 128, 128, 256, 5632, 128, 128, 128, 512, 128, 128, 0, 32768, 0, 32768, 26240, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 128, 128, 32768, 0, 128, 128, 32768, 128, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 32256, 29824, 128, 512, 0, 128, 1408, 32768, 1920, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 32768, 0, 128, 128, 0, 29824, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 32512, 512, 128, 0, 128, 128, 128, 32768, 32000, 128, 0, 640, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32640, 0, 128, 128, 0, 32768, 32768, 32768, 128, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[5, 0, 0, 3937, 0, 0, 3, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4070, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 4096, 3898, 4061, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3411, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1415, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3853, 17, 0, 4094, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4, 0, 0, 4055, 0, 0, 14, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 4089, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4095, 0, 0, 4096, 4037, 4007, 4096, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4095, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 2504, 0, 0, 0, 0, 0, 1, 0, 4096, 1, 0, 0, 0, 0, 1975, 0, 0, 4083, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4095, 0, 3810, 68, 0, 4059, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[17, 1, 0, 4037, 1, 1, 12, 0, 1, 0, 27, 1, 1, 0, 4095, 0, 3931, 0, 0, 0, 0, 4095, 4083, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 4095, 3, 5, 4095, 0, 1, 4089, 4050, 4095, 4095, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 4084, 4095, 1, 3, 0, 1, 0, 1, 1, 0, 0, 4095, 1, 4093, 4092, 1, 1, 0, 1, 4095, 1, 4095, 0, 1, 2751, 1, 1, 0, 1, 0, 0, 0, 4095, 0, 3, 1, 1, 1, 1887, 0, 1, 3959, 0, 0, 0, 1, 0, 0, 4095, 1, 1, 4095, 0, 3918, 0, 1, 4095, 4095, 0, 0, 4093, 0, 2, 4095, 4093, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[24, 1, 0, 3997, 0, 1, 36, 1, 1, 1, 19, 1, 1, 0, 4095, 1, 4085, 0, 1, 0, 0, 4094, 4091, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 4093, 0, 3, 4094, 0, 1, 4095, 4079, 4078, 4095, 0, 0, 0, 4, 0, 1, 0, 1, 0, 1, 1, 4095, 0, 1, 1, 0, 0, 1, 4075, 4095, 1, 3, 2, 1, 0, 1, 1, 0, 0, 4095, 0, 4080, 4094, 0, 1, 0, 1, 4095, 0, 4095, 0, 0, 3094, 1, 1, 0, 1, 1, 0, 0, 4095, 0, 0, 1, 1, 1, 1335, 0, 1, 4067, 0, 0, 1, 1, 1, 0, 4095, 1, 0, 4094, 0, 3852, 25, 1, 4092, 4095, 0, 0, 4092, 0, 10, 4095, 4094, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4051, 0, 0, 3, 0, 0, 0, 294, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 3821, 4069, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2978, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1660, 0, 0, 3861, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3855, 2, 0, 4083, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[32, 1, 0, 3764, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 4095, 0, 3975, 0, 1, 0, 0, 4095, 4095, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 4095, 8, 10, 4095, 0, 1, 4094, 4053, 4095, 4095, 0, 0, 0, 15, 0, 1, 0, 1, 0, 1, 0, 4095, 0, 1, 0, 0, 1, 1, 4084, 4095, 1, 3, 0, 1, 0, 1, 1, 0, 0, 4095, 1, 4085, 4092, 1, 1, 0, 2, 4095, 1, 4095, 0, 1, 3145, 1, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 1, 1, 1, 1584, 0, 0, 4068, 0, 0, 0, 1, 0, 0, 4095, 1, 1, 4095, 0, 3941, 0, 1, 4095, 4095, 0, 0, 4091, 0, 0, 4095, 4092, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[5, 0, 0, 4082, 0, 0, 7, 0, 0, 0, 33, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 4090, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4062, 4022, 4096, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 293, 0, 0, 4074, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 4096, 0, 3972, 62, 0, 4088, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[12, 0, 0, 3928, 0, 0, 85, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 4092, 0, 0, 7, 0, 4058, 4036, 0, 0, 0, 4079, 0, 0, 0, 0, 12, 0, 4086, 0, 2, 4093, 0, 0, 4096, 3976, 4022, 4096, 0, 0, 0, 33, 7, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 3, 2, 0, 4082, 4096, 0, 0, 15, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 3806, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 3, 0, 0, 0, 951, 0, 0, 4082, 0, 0, 2, 0, 18, 0, 4094, 0, 0, 4082, 0, 3808, 57, 0, 3977, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4078, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[17, 1, 0, 4037, 1, 1, 12, 0, 1, 0, 27, 1, 1, 0, 4095, 0, 4095, 4095, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3931, 0, 0, 0, 0, 4095, 4083, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 5, 120, 0, 1, 4089, 4050, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 4084, 4095, 1, 2929, 1168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 1, 1, 0, 0, 4095, 1, 4093, 4092, 1, 1, 0, 1, 2807, 1291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4095, 0, 1, 2751, 1, 1, 0, 1, 0, 0, 0, 4095, 0, 3, 886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1887, 0, 1, 3959, 0, 0, 0, 1, 0, 0, 4095, 1, 1, 4068, 2802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3918, 0, 1, 4095, 0, 0, 0, 219, 0, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 16264, 16458, 16378, 16388, 15930, 16818, 16425] avg=16384.0000 max=16818 max_rank=6 min=15930 min_rank=5 imbalance_ratio=1.026489 max_load_reduction=114254 max_load_reduction_pct=0.871689 moved_to_echo=65561 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 3937, 0, 0, 3, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 4096, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4070, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 120, 0, 0, 4096, 3898, 4061, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2931, 1168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2808, 1293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3411, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1415, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4068, 2803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3853, 17, 0, 4094, 0, 0, 0, 219, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16269, 16417, 16273, 16387, 16389, 16586, 16376, 16375] avg=16384.0000 max=16586 max_rank=5 min=16269 min_rank=0 imbalance_ratio=1.012329 max_load_reduction=114486 max_load_reduction_pct=0.873459 moved_to_echo=65726 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 4055, 0, 0, 14, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 4096, 4096, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4089, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 120, 0, 0, 4096, 4037, 4007, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4095, 4096, 0, 2931, 1168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 2807, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 2504, 0, 0, 0, 0, 0, 1, 0, 4096, 1, 0, 887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1975, 0, 0, 4083, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4069, 2804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3810, 68, 0, 4059, 0, 0, 0, 219, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16406, 16430, 16404, 16388, 16387, 15682, 17029, 16346] avg=16384.0000 max=17029 max_rank=6 min=15682 min_rank=5 imbalance_ratio=1.039368 max_load_reduction=114043 max_load_reduction_pct=0.870079 moved_to_echo=65444 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[24, 1, 0, 3997, 0, 1, 36, 1, 1, 1, 19, 1, 1, 0, 4095, 1, 4095, 4095, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 1, 0, 0, 4094, 4091, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 120, 0, 1, 4095, 4079, 4078, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 4095, 0, 1, 1, 0, 0, 1, 4075, 4095, 1, 2928, 1168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1, 0, 1, 1, 0, 0, 4095, 0, 4080, 4094, 0, 1, 0, 1, 2806, 1291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 3094, 1, 1, 0, 1, 1, 0, 0, 4095, 0, 0, 886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1335, 0, 1, 4067, 0, 0, 1, 1, 1, 0, 4095, 1, 0, 4068, 2803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3852, 25, 1, 4092, 0, 0, 0, 219, 0, 10, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16396, 16425, 16473, 16369, 16376, 16269, 16376, 16388] avg=16384.0000 max=16473 max_rank=2 min=16269 min_rank=5 imbalance_ratio=1.005432 max_load_reduction=114599 max_load_reduction_pct=0.874321 moved_to_echo=65409 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4051, 0, 0, 3, 0, 0, 0, 294, 0, 0, 0, 4096, 0, 4096, 4096, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 120, 0, 0, 4096, 3821, 4069, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2931, 1168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2808, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2978, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1660, 0, 0, 3861, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4069, 2804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3855, 2, 0, 4083, 0, 0, 0, 219, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16663, 16435, 16205, 16387, 16388, 16153, 16490, 16351] avg=16384.0000 max=16663 max_rank=0 min=16153 min_rank=5 imbalance_ratio=1.017029 max_load_reduction=114409 max_load_reduction_pct=0.872871 moved_to_echo=65382 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[32, 1, 0, 3764, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 4095, 0, 4095, 4095, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 1, 0, 0, 4095, 4095, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 8, 10, 120, 0, 1, 4094, 4053, 4095, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 4095, 0, 1, 0, 0, 1, 1, 4084, 4095, 1, 2928, 1168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 1, 1, 0, 0, 4095, 1, 4085, 4092, 1, 1, 0, 2, 2807, 1291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4095, 0, 1, 3145, 1, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1584, 0, 0, 4068, 0, 0, 0, 1, 0, 0, 4095, 1, 1, 4067, 2801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3941, 0, 1, 4095, 0, 0, 0, 218, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16116, 16321, 16491, 16377, 16381, 16320, 16621, 16445] avg=16384.0000 max=16621 max_rank=6 min=16116 min_rank=0 imbalance_ratio=1.014465 max_load_reduction=114451 max_load_reduction_pct=0.873192 moved_to_echo=65767 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 4082, 0, 0, 7, 0, 0, 0, 33, 0, 0, 0, 4096, 0, 4096, 4096, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4090, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 120, 0, 0, 4096, 4062, 4022, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2931, 1168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 4096, 4096, 0, 0, 0, 0, 2808, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 4074, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 4069, 2804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3972, 62, 0, 4088, 0, 0, 0, 219, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16442, 16434, 16435, 16387, 16366, 17131, 15344, 16533] avg=16384.0000 max=17131 max_rank=5 min=15344 min_rank=6 imbalance_ratio=1.045593 max_load_reduction=113941 max_load_reduction_pct=0.869301 moved_to_echo=65374 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 3928, 0, 0, 85, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 4096, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 7, 0, 4058, 4036, 0, 0, 0, 4079, 0, 0, 0, 0, 12, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 2, 111, 0, 0, 4096, 3976, 4022, 0, 0, 0, 0, 33, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 3, 2, 0, 4082, 4096, 0, 2936, 1173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 4096, 0, 0, 0, 0, 2809, 1293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 3806, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 3, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 0, 0, 4082, 0, 0, 2, 0, 18, 0, 4094, 0, 0, 4068, 2803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 3808, 57, 0, 3977, 0, 0, 0, 207, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16369, 16346, 16333, 16399, 16397, 17001, 16018, 16209] avg=16384.0000 max=17001 max_rank=5 min=16018 min_rank=6 imbalance_ratio=1.037659 max_load_reduction=114071 max_load_reduction_pct=0.870293 moved_to_echo=65625 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 0, 31104, 256, 128, 32640, 0, 0, 0, 1792, 0, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32512, 0, 128, 128, 0, 32768, 32768, 0, 0, 0, 32768, 128, 0, 0, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 32768, 128, 128, 25728, 128, 128, 0, 128, 128, 128, 0, 32768, 128, 128, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 0, 32768, 128, 32768, 32768, 128, 128, 0, 128, 22528, 10368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 31872, 128, 128, 256, 128, 128, 128, 512, 128, 128, 0, 32768, 128, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 128, 128, 1024, 0, 128, 32768, 32000, 32512, 0, 0, 0, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 0, 128, 128, 128, 128, 32768, 0, 128, 128, 128, 128, 128, 32768, 32768, 128, 23552, 9472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 11136, 0, 128, 32256, 0, 0, 128, 128, 128, 0, 32768, 128, 128, 32640, 22528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4058, 4096, 0, 0, 0, 0, 0, 4096, 0, 124, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 706, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2910, 0, 0, 0, 0, 4096, 0, 3802, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 4095, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 0, 0, 2, 4096, 3, 4096, 4096, 0, 0, 0, 4049, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 3326, 0, 3923, 0, 0, 0, 0, 0, 0, 22, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 1632, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 0, 4096, 0, 246, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4080, 0, 0, 322, 0, 4096, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 3228, 0, 0, 3, 0, 4096, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 0, 4096, 189, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 42, 4096, 42, 4096, 4096, 0, 0, 0, 3907, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 2461, 0, 3739, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 2294, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3723, 4095, 1, 0, 1, 0, 0, 4094, 0, 503, 0, 1, 4095, 4094, 1, 1, 1, 0, 0, 4095, 1, 1, 70, 1, 4095, 0, 1, 2, 1, 4065, 1, 1, 0, 1, 1, 2101, 1, 0, 8, 0, 4095, 0, 3548, 1, 0, 0, 71, 0, 0, 1, 0, 0, 4095, 4095, 0, 4095, 420, 0, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 1, 0, 4090, 1, 0, 82, 4039, 0, 4090, 4023, 0, 1, 0, 4060, 0, 1, 1, 1, 4095, 0, 0, 4093, 1, 0, 1, 4095, 2, 3189, 0, 4002, 0, 3, 0, 1, 0, 0, 0, 4095, 0, 4045, 0, 0, 0, 4095, 1, 0, 1, 4095, 3005, 4095, 0, 4095, 2, 0, 1, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4047, 4096, 0, 0, 0, 0, 0, 4096, 0, 112, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4064, 0, 0, 464, 0, 4096, 0, 0, 0, 0, 4077, 0, 0, 26, 0, 0, 2307, 0, 0, 0, 10, 4096, 16, 3922, 0, 0, 0, 0, 0, 0, 0, 0, 55, 4083, 4095, 0, 4087, 577, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 4063, 0, 0, 0, 4096, 76, 4084, 4090, 13, 0, 0, 4039, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 2347, 0, 4094, 0, 0, 0, 0, 2, 0, 39, 4085, 0, 4096, 0, 2, 10, 4094, 0, 0, 0, 4096, 2546, 4096, 0, 4090, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3752, 4095, 1, 0, 1, 0, 0, 4094, 0, 684, 0, 1, 4095, 4094, 1, 0, 1, 0, 0, 4095, 1, 1, 285, 1, 4095, 0, 0, 2, 1, 4093, 1, 1, 0, 1, 1, 2196, 1, 0, 3, 0, 4095, 0, 3810, 1, 0, 1, 39, 0, 0, 1, 0, 0, 4095, 4093, 1, 4095, 31, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 1, 0, 4087, 1, 0, 75, 3988, 0, 4095, 4024, 0, 1, 0, 4037, 0, 1, 1, 1, 4095, 0, 0, 4095, 1, 0, 1, 4095, 38, 2395, 0, 4060, 0, 4, 0, 1, 0, 0, 0, 4095, 1, 4076, 1, 0, 0, 4095, 1, 0, 1, 4095, 3368, 4095, 0, 4095, 1, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4049, 4096, 0, 0, 0, 0, 0, 4096, 0, 1250, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4059, 0, 0, 355, 0, 4096, 0, 0, 0, 0, 4065, 0, 0, 4, 0, 0, 2015, 0, 0, 0, 1, 4096, 1, 3547, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4096, 4096, 0, 4095, 783, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4086, 0, 0, 23, 4096, 21, 4096, 4095, 2, 0, 0, 3919, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 1730, 0, 4093, 0, 0, 0, 0, 0, 0, 21, 4095, 0, 4096, 0, 0, 1, 4095, 0, 0, 0, 4096, 2805, 4096, 0, 4095, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3963, 4096, 0, 0, 0, 0, 0, 4096, 0, 587, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 318, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 2936, 0, 0, 0, 0, 4096, 0, 1847, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 0, 4096, 636, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 9, 4096, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 3239, 0, 4091, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 2855, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3860, 4094, 1, 1, 0, 0, 0, 4095, 1, 384, 1, 1, 4095, 4094, 1, 0, 1, 0, 0, 4096, 1, 1, 41, 1, 4095, 0, 0, 5, 1, 4054, 1, 1, 1, 1, 1, 1734, 1, 1, 0, 1, 4094, 0, 3980, 1, 0, 0, 12, 0, 0, 0, 0, 1, 4094, 4094, 0, 4095, 143, 1, 0, 0, 0, 0, 1, 4095, 0, 0, 1, 1, 0, 4089, 1, 1, 118, 4069, 0, 4085, 4050, 0, 0, 0, 4076, 0, 1, 0, 0, 4095, 1, 0, 4091, 1, 0, 1, 4095, 0, 2665, 0, 4026, 0, 1, 0, 1, 2, 0, 12, 4094, 0, 4028, 0, 0, 1, 4095, 1, 0, 1, 4095, 3697, 4095, 0, 4095, 13, 0, 1, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3723, 4095, 1, 0, 1, 0, 0, 4094, 0, 503, 0, 1, 3763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3785, 4094, 1, 1, 1, 0, 0, 4095, 1, 1, 70, 1, 4095, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4065, 1, 1, 0, 1, 1, 2101, 1, 0, 8, 0, 4095, 0, 3548, 1, 1964, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 1, 0, 0, 4095, 4095, 0, 4095, 420, 0, 0, 0, 2776, 610, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4068, 0, 0, 0, 1, 0, 4090, 1, 0, 82, 4039, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 1, 0, 4060, 0, 1, 1, 1, 4095, 0, 0, 4093, 1, 0, 1, 67, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 2, 3189, 0, 4002, 0, 3, 0, 1, 0, 0, 0, 4095, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 1319, 3005, 0, 0, 4095, 2, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16182, 16148, 16031, 16495, 16373, 16371, 16858, 16614] avg=16384.0000 max=16858 max_rank=6 min=16031 min_rank=2 imbalance_ratio=1.028931 max_load_reduction=114214 max_load_reduction_pct=0.871384 moved_to_echo=66216 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4058, 4096, 0, 0, 0, 0, 0, 4096, 0, 124, 0, 0, 3764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3786, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 706, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2910, 0, 0, 0, 0, 4096, 0, 3802, 0, 1964, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 4095, 0, 4096, 16, 0, 0, 0, 2777, 611, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 4091, 0, 0, 2, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4049, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 67, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 0, 3326, 0, 3923, 0, 0, 0, 0, 0, 0, 22, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1319, 1632, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16138, 16780, 17111, 16032, 16356, 16432, 16984, 15239] avg=16384.0000 max=17111 max_rank=2 min=15239 min_rank=7 imbalance_ratio=1.044373 max_load_reduction=113961 max_load_reduction_pct=0.869453 moved_to_echo=65011 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4049, 4096, 0, 0, 0, 0, 0, 4096, 0, 1250, 0, 0, 3763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3786, 4096, 0, 0, 0, 0, 0, 4059, 0, 0, 355, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 4, 0, 0, 2015, 0, 0, 0, 1, 4096, 1, 3547, 0, 1964, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4096, 4096, 0, 4095, 783, 0, 0, 0, 2777, 611, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 1, 0, 0, 4086, 0, 0, 23, 4096, 21, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 3919, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 67, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 0, 1730, 0, 4093, 0, 0, 0, 0, 0, 0, 21, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 1319, 2805, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17254, 16392, 15936, 16829, 16391, 16303, 15556, 16411] avg=16384.0000 max=17254 max_rank=0 min=15556 min_rank=6 imbalance_ratio=1.053101 max_load_reduction=113818 max_load_reduction_pct=0.868362 moved_to_echo=64661 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 0, 4096, 0, 246, 0, 0, 3763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3786, 4096, 0, 0, 0, 0, 0, 4080, 0, 0, 322, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 3228, 0, 0, 3, 0, 4096, 0, 4045, 0, 1964, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 0, 4096, 189, 0, 0, 0, 2777, 611, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 4096, 0, 0, 42, 4096, 42, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3907, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 67, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 0, 2461, 0, 3739, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1319, 2294, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16285, 16380, 17660, 16203, 16440, 16290, 15913, 15901] avg=16384.0000 max=17660 max_rank=2 min=15901 min_rank=7 imbalance_ratio=1.077881 max_load_reduction=113412 max_load_reduction_pct=0.865265 moved_to_echo=64544 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3963, 4096, 0, 0, 0, 0, 0, 4096, 0, 587, 0, 0, 3763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3786, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 318, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 2936, 0, 0, 0, 0, 4096, 0, 1847, 0, 1964, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 0, 4096, 636, 0, 0, 0, 2777, 611, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 67, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 0, 3239, 0, 4091, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1319, 2855, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16505, 16392, 15175, 16650, 16367, 16477, 17044, 16462] avg=16384.0000 max=17044 max_rank=6 min=15175 min_rank=2 imbalance_ratio=1.040283 max_load_reduction=114028 max_load_reduction_pct=0.869965 moved_to_echo=66350 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4047, 4096, 0, 0, 0, 0, 0, 4096, 0, 112, 0, 0, 3764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 4096, 0, 0, 0, 0, 0, 4064, 0, 0, 464, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 26, 0, 0, 2307, 0, 0, 0, 10, 4096, 16, 3922, 0, 1970, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 4083, 4095, 0, 4087, 577, 5, 0, 0, 2782, 614, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 11, 0, 0, 4063, 0, 0, 0, 4096, 76, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 13, 0, 0, 4039, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 73, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1512, 0, 2347, 0, 4094, 0, 0, 0, 0, 2, 0, 39, 4085, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 10, 4094, 0, 0, 0, 1314, 2546, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16115, 16498, 16669, 16630, 16395, 16438, 16175, 16152] avg=16384.0000 max=16669 max_rank=2 min=16115 min_rank=0 imbalance_ratio=1.017395 max_load_reduction=114403 max_load_reduction_pct=0.872826 moved_to_echo=65160 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3752, 4095, 1, 0, 1, 0, 0, 4094, 0, 684, 0, 1, 3763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3785, 4094, 1, 0, 1, 0, 0, 4095, 1, 1, 285, 1, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 1, 1, 0, 1, 1, 2196, 1, 0, 3, 0, 4095, 0, 3810, 1, 1964, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 39, 0, 0, 1, 0, 0, 4095, 4093, 1, 4095, 31, 0, 0, 0, 2776, 610, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4068, 0, 0, 0, 1, 0, 4087, 1, 0, 75, 3988, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 1, 0, 4037, 0, 1, 1, 1, 4095, 0, 0, 4095, 1, 0, 1, 67, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 38, 2395, 0, 4060, 0, 4, 0, 1, 0, 0, 0, 4095, 1, 4076, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 1319, 3368, 0, 0, 4095, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 16361, 16411, 16074, 16316, 16351, 16192, 16975] avg=16384.0000 max=16975 max_rank=7 min=16074 min_rank=3 imbalance_ratio=1.036072 max_load_reduction=114097 max_load_reduction_pct=0.870491 moved_to_echo=65834 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3860, 4094, 1, 1, 0, 0, 0, 4095, 1, 384, 1, 1, 3762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3785, 4094, 1, 0, 1, 0, 0, 4096, 1, 1, 41, 1, 4095, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4054, 1, 1, 1, 1, 1, 1734, 1, 1, 0, 1, 4094, 0, 3980, 1, 1964, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1, 4094, 4094, 0, 4095, 143, 1, 0, 0, 2776, 610, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4068, 0, 0, 1, 1, 0, 4089, 1, 1, 118, 4069, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 4076, 0, 1, 0, 0, 4095, 1, 0, 4091, 1, 0, 1, 67, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 0, 2665, 0, 4026, 0, 1, 0, 1, 2, 0, 12, 4094, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 1, 1319, 3697, 0, 0, 4095, 13, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16201, 16121, 16079, 16159, 16434, 16410, 16350, 17318] avg=16384.0000 max=17318 max_rank=7 min=16079 min_rank=2 imbalance_ratio=1.057007 max_load_reduction=113754 max_load_reduction_pct=0.867874 moved_to_echo=66512 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[12160, 128, 21376, 0, 32128, 0, 128, 0, 128, 128, 0, 128, 32768, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[30336, 32768, 128, 128, 128, 0, 0, 32768, 128, 128, 2688, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 0, 31616, 32768, 128, 128, 128, 0, 0, 32768, 128, 3968, 128, 128, 30208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 32640, 0, 0, 128, 128, 0, 32768, 128, 128, 384, 32640, 256, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 0, 128, 10624, 22272, 0, 0, 32768, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 32640, 128, 128, 128, 128, 128, 19456, 128, 128, 128, 128, 32768, 128, 28544, 128, 15744, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 128, 0, 128, 32768, 32768, 128, 32768, 2816, 128, 0, 0, 22272, 4992, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32640, 128, 128, 0, 32256, 0, 128, 128, 128, 32768, 128, 0, 32768, 128, 0, 128, 640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[993, 0, 0, 4096, 0, 0, 4092, 1, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 4096, 0, 208, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 4096, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3051, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4091, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 3934, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[3247, 0, 0, 4095, 1, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 1, 4093, 17, 2827, 4, 600, 1, 4095, 4012, 0, 1, 1, 4095, 0, 264, 0, 0, 0, 4094, 1, 1, 4095, 4095, 4095, 0, 0, 0, 1, 0, 1, 50, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 4095, 1, 0, 4050, 0, 1, 1, 4095, 1, 0, 0, 0, 0, 2075, 4095, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 4095, 4095, 1, 247, 0, 0, 0, 0, 0, 1, 4090, 4095, 1, 0, 4095, 1, 1, 3563, 1, 3718, 0, 1, 1, 4094, 1, 0, 2, 4094, 0, 4095, 69, 4095, 4095, 4095, 0, 2, 0, 1, 4095, 3923, 0, 1, 4095, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[3017, 0, 0, 4096, 0, 0, 4094, 170, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4083, 0, 35, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1027, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 163, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4082, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 3990, 7, 0, 3916, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3267, 0, 0, 4095, 0, 1, 4095, 0, 0, 0, 1, 0, 0, 0, 1, 4094, 21, 3079, 1, 669, 1, 4095, 4058, 0, 1, 1, 4095, 0, 346, 0, 0, 0, 4094, 1, 2, 4095, 4095, 4095, 0, 0, 0, 1, 0, 1, 104, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 4095, 1, 0, 3974, 0, 1, 1, 4095, 1, 0, 0, 0, 0, 1838, 4095, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 4095, 4095, 0, 17, 0, 0, 0, 0, 0, 1, 4093, 4094, 3, 0, 4095, 1, 1, 3654, 0, 3581, 0, 1, 1, 4094, 1, 0, 1, 4094, 0, 4095, 20, 4095, 4095, 4095, 0, 0, 0, 1, 4095, 4034, 0, 1, 4094, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[3395, 0, 0, 4096, 0, 0, 4096, 87, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 4094, 0, 13, 0, 4096, 4031, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 848, 4090, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 18, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4091, 0, 4051, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4074, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 4029, 0, 0, 3992, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[3196, 0, 0, 4088, 0, 0, 4063, 29, 0, 0, 0, 34, 0, 0, 0, 4096, 7, 4093, 0, 0, 0, 4096, 3983, 0, 0, 0, 4082, 0, 0, 1, 0, 0, 4096, 0, 0, 4062, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 4096, 0, 15, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1289, 4065, 2, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 54, 0, 0, 24, 0, 12, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4008, 8, 0, 0, 4096, 0, 0, 0, 4096, 0, 4056, 0, 4095, 4094, 4096, 0, 0, 0, 0, 4079, 3959, 41, 0, 3955, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[2842, 0, 0, 4096, 0, 0, 4075, 67, 0, 0, 0, 26, 0, 0, 0, 4096, 0, 4095, 0, 83, 0, 4096, 4025, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 4056, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1839, 4075, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 19, 0, 4, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4082, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4072, 0, 4095, 4095, 4096, 0, 0, 0, 0, 4095, 3748, 21, 0, 3792, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[3854, 1, 3, 4094, 1, 1, 4094, 1, 0, 0, 1, 0, 0, 0, 1, 4093, 16, 2706, 2, 412, 0, 4094, 4049, 1, 0, 1, 4095, 0, 356, 0, 0, 0, 4095, 0, 0, 4094, 4095, 4095, 0, 1, 2, 1, 0, 1, 14, 0, 0, 1, 1, 0, 1, 0, 22, 1, 0, 1, 4095, 1, 0, 4082, 0, 0, 1, 4095, 1, 0, 0, 0, 1, 1695, 4095, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 4094, 4094, 0, 176, 0, 0, 1, 0, 0, 1, 4095, 4095, 1, 0, 4095, 1, 1, 3587, 1, 3660, 0, 1, 0, 4095, 1, 0, 0, 4094, 0, 4094, 9, 4095, 4095, 4095, 0, 1, 0, 1, 4095, 4024, 0, 1, 4095, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3247, 0, 0, 4095, 1, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 1, 4093, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2827, 4, 600, 1, 4095, 4012, 0, 1, 1, 4095, 0, 264, 0, 0, 0, 90, 33, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 4095, 4095, 4087, 0, 0, 0, 1, 0, 1, 50, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 0, 4095, 1, 0, 4050, 0, 1, 1, 4095, 2616, 1478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2075, 4095, 1, 1, 0, 0, 1, 0, 0, 0, 1, 4095, 4095, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4005, 4095, 1, 247, 0, 0, 0, 0, 0, 1, 4090, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 1, 1, 3563, 1, 3718, 0, 1, 1, 4094, 1, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 69, 0, 227, 0, 0, 2, 0, 1, 4095, 3923, 0, 1, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16607, 16048, 16426, 16341, 16755, 16537, 15849, 16509] avg=16384.0000 max=16755 max_rank=4 min=15849 min_rank=6 imbalance_ratio=1.022644 max_load_reduction=114317 max_load_reduction_pct=0.872169 moved_to_echo=65650 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[993, 0, 0, 4096, 0, 0, 4092, 1, 0, 0, 0, 9, 0, 0, 0, 4096, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 208, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 90, 34, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 4096, 4087, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2617, 1478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3051, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 0, 4096, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 228, 0, 0, 0, 0, 0, 4096, 3934, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14361, 16725, 16376, 16385, 17729, 16306, 16747, 16443] avg=16384.0000 max=17729 max_rank=4 min=14361 min_rank=0 imbalance_ratio=1.082092 max_load_reduction=113343 max_load_reduction_pct=0.864738 moved_to_echo=67225 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3017, 0, 0, 4096, 0, 0, 4094, 170, 0, 0, 0, 1, 0, 0, 0, 4096, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 35, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 90, 34, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2617, 1478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 4096, 0, 163, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 0, 4096, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 228, 0, 0, 0, 0, 0, 4096, 3990, 7, 0, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16548, 16539, 16356, 16396, 15707, 16457, 16750, 16319] avg=16384.0000 max=16750 max_rank=6 min=15707 min_rank=4 imbalance_ratio=1.022339 max_load_reduction=114322 max_load_reduction_pct=0.872208 moved_to_echo=65233 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3267, 0, 0, 4095, 0, 1, 4095, 0, 0, 0, 1, 0, 0, 0, 1, 4094, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 3079, 1, 669, 1, 4095, 4058, 0, 1, 1, 4095, 0, 346, 0, 0, 0, 90, 33, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 2, 4095, 4095, 4087, 0, 0, 0, 1, 0, 1, 104, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 4095, 1, 0, 3974, 0, 1, 1, 4095, 2616, 1478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1838, 4095, 1, 1, 0, 0, 1, 0, 0, 0, 1, 4095, 4095, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4005, 4095, 0, 17, 0, 0, 0, 0, 0, 1, 4093, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 1, 1, 3654, 0, 3581, 0, 1, 1, 4094, 1, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 20, 0, 227, 0, 0, 0, 0, 1, 4095, 4034, 0, 1, 4094, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16628, 16498, 16482, 16266, 16518, 16310, 15801, 16569] avg=16384.0000 max=16628 max_rank=0 min=15801 min_rank=6 imbalance_ratio=1.014893 max_load_reduction=114444 max_load_reduction_pct=0.873138 moved_to_echo=65198 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2842, 0, 0, 4096, 0, 0, 4075, 67, 0, 0, 0, 26, 0, 0, 0, 4096, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 83, 0, 4096, 4025, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 90, 34, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4056, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 4096, 2617, 1478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1839, 4075, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 4095, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 4095, 0, 0, 0, 0, 19, 0, 4, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 0, 4096, 0, 4082, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 227, 0, 0, 0, 0, 0, 4095, 3748, 21, 0, 3792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16276, 16529, 16335, 16421, 16499, 16316, 16741, 15955] avg=16384.0000 max=16741 max_rank=6 min=15955 min_rank=7 imbalance_ratio=1.021790 max_load_reduction=114331 max_load_reduction_pct=0.872276 moved_to_echo=65511 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3854, 1, 3, 4094, 1, 1, 4094, 1, 0, 0, 1, 0, 0, 0, 1, 4093, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2706, 2, 412, 0, 4094, 4049, 1, 0, 1, 4095, 0, 356, 0, 0, 0, 90, 33, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 4095, 4087, 0, 1, 2, 1, 0, 1, 14, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 22, 1, 0, 1, 4095, 1, 0, 4082, 0, 0, 1, 4095, 2616, 1478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1695, 4095, 1, 1, 0, 0, 1, 0, 0, 0, 1, 4095, 4095, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4004, 4094, 0, 176, 0, 0, 1, 0, 0, 1, 4095, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 1, 1, 3587, 1, 3660, 0, 1, 0, 4095, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 9, 0, 227, 0, 0, 1, 0, 1, 4095, 4024, 0, 1, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17218, 15863, 16391, 16394, 16376, 16469, 15813, 16548] avg=16384.0000 max=17218 max_rank=0 min=15813 min_rank=6 imbalance_ratio=1.050903 max_load_reduction=113854 max_load_reduction_pct=0.868637 moved_to_echo=65206 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3196, 0, 0, 4088, 0, 0, 4063, 29, 0, 0, 0, 34, 0, 0, 0, 4096, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4093, 0, 0, 0, 4096, 3983, 0, 0, 0, 4082, 0, 0, 1, 0, 0, 96, 37, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4062, 4091, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 4096, 0, 15, 4096, 0, 0, 0, 4096, 2620, 1481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1289, 4065, 2, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 4095, 2393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 4083, 0, 54, 0, 0, 24, 0, 12, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 0, 4096, 0, 4008, 8, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 220, 0, 0, 0, 0, 0, 4079, 3959, 41, 0, 3955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16583, 16407, 16333, 16446, 15962, 16365, 16666, 16310] avg=16384.0000 max=16666 max_rank=6 min=15962 min_rank=4 imbalance_ratio=1.017212 max_load_reduction=114406 max_load_reduction_pct=0.872849 moved_to_echo=65303 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3395, 0, 0, 4096, 0, 0, 4096, 87, 0, 0, 0, 7, 0, 0, 0, 4096, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 13, 0, 4096, 4031, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 90, 34, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2617, 1478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 848, 4090, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 4096, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 4096, 0, 18, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 0, 4091, 0, 4051, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 228, 0, 0, 0, 0, 0, 4096, 4029, 0, 0, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16851, 16463, 16373, 16423, 15526, 16312, 16705, 16419] avg=16384.0000 max=16851 max_rank=0 min=15526 min_rank=4 imbalance_ratio=1.028503 max_load_reduction=114221 max_load_reduction_pct=0.871437 moved_to_echo=64962 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 32128, 32768, 128, 768, 0, 0, 128, 0, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 29184, 128, 2048, 128, 32768, 32384, 128, 128, 128, 32768, 0, 1024, 128, 0, 0, 768, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[23936, 128, 128, 32768, 128, 128, 32768, 384, 0, 0, 128, 128, 0, 0, 128, 32768, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 0, 256, 128, 128, 128, 32768, 128, 128, 32640, 0, 128, 128, 32768, 20992, 11904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 0, 128, 13696, 32768, 128, 128, 0, 0, 128, 128, 0, 0, 128, 32768, 32768, 19200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 32768, 32768, 32768, 0, 128, 128, 128, 0, 128, 256, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 3072, 128, 128, 31360, 128, 31360, 128, 128, 128, 32768, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 0, 1920, 0, 0, 128, 0, 128, 32768, 31744, 128, 128, 32128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4093, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4073, 43, 2527, 4096, 0, 6, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 819, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4082, 4096, 518, 1064, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3244, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 3993, 1, 3929, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4004, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 2897, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 0, 0, 4096, 4091, 4096, 741, 297, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 518, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 0, 7, 241, 2, 0, 0, 0, 0, 0, 9, 4056, 1, 1, 0, 4086, 0, 0, 4095, 4095, 4085, 0, 0, 1, 4093, 1, 0, 0, 0, 0, 4095, 1, 34, 1, 1, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 1755, 0, 3870, 4095, 1, 53, 1, 4095, 0, 1, 0, 0, 4095, 1, 1, 4094, 4036, 0, 3973, 0, 4, 0, 1, 1, 0, 1, 0, 4091, 0, 65, 89, 0, 0, 0, 1, 1, 0, 4095, 1, 4095, 0, 4063, 1, 0, 1, 0, 4095, 4095, 4095, 333, 3455, 0, 0, 0, 0, 5, 4095, 0, 1, 0, 0, 4092, 1, 4095, 1, 1, 1, 4095, 1, 4094, 1, 0, 0, 0, 0, 33, 4095, 2618, 4095, 0, 1, 0, 4089] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 0, 3, 409, 1, 0, 0, 0, 0, 0, 9, 4042, 1, 1, 0, 4086, 0, 0, 4095, 4095, 4094, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 4095, 1, 38, 1, 1, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 1649, 0, 4059, 4095, 1, 0, 1, 4095, 0, 1, 0, 0, 4095, 1, 1, 4094, 4054, 0, 3881, 0, 4, 0, 1, 1, 0, 1, 0, 4091, 0, 13, 466, 0, 1, 1, 1, 1, 0, 4095, 1, 4095, 0, 4077, 1, 5, 1, 0, 4095, 4066, 4095, 1514, 1714, 0, 0, 4, 0, 4, 4095, 0, 1, 0, 0, 4092, 1, 4095, 1, 1, 1, 4095, 1, 4092, 0, 0, 0, 0, 0, 53, 4095, 2721, 4094, 0, 1, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4094, 3993, 0, 0, 0, 4096, 0, 20, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 19, 0, 4079, 0, 0, 0, 0, 4081, 107, 3578, 4096, 0, 52, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1434, 0, 0, 0, 0, 0, 0, 4096, 0, 4078, 0, 4086, 0, 0, 0, 1, 4094, 4086, 4096, 443, 1376, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 4076, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 1335, 4093, 0, 0, 11, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4081, 0, 0, 0, 4096, 0, 1, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0, 0, 4045, 37, 3299, 4096, 0, 11, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4093, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1491, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4094, 0, 0, 0, 1, 4096, 4090, 4096, 1637, 503, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 1290, 4096, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4008, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3885, 71, 3392, 4096, 0, 39, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 984, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4089, 4096, 799, 1996, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1219, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 383, 0, 0, 1, 1, 0, 1, 6, 4067, 1, 1, 0, 4093, 0, 0, 4095, 4095, 4089, 0, 0, 0, 4087, 1, 1, 2, 1, 1, 4095, 1, 29, 0, 1, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 2490, 1, 4014, 4095, 1, 1, 1, 4095, 0, 1, 0, 0, 4094, 1, 1, 4090, 4092, 0, 3930, 0, 3, 0, 0, 1, 0, 1, 0, 4091, 0, 13, 169, 0, 0, 1, 1, 1, 0, 4094, 1, 4095, 0, 4083, 1, 20, 0, 0, 4094, 4084, 4095, 881, 2465, 0, 1, 1, 0, 1, 4095, 0, 1, 0, 0, 4092, 1, 4093, 1, 1, 1, 4095, 1, 4093, 1, 0, 0, 0, 0, 9, 4095, 2053, 4094, 1, 1, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 3993, 1, 3929, 4027, 1150, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4004, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 2897, 0, 0, 0, 0, 4096, 4096, 2919, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 1649, 4091, 0, 741, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2214, 1883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2213, 518, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16265, 16412, 17430, 16361, 18231, 14969, 16385, 15019] avg=16384.0000 max=18231 max_rank=4 min=14969 min_rank=5 imbalance_ratio=1.112732 max_load_reduction=112841 max_load_reduction_pct=0.860909 moved_to_echo=64604 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3, 409, 1, 0, 0, 0, 0, 0, 9, 4042, 1, 1, 0, 4095, 4095, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4095, 4094, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 38, 1, 1, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 1649, 0, 4059, 4026, 1149, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 1, 0, 1, 0, 0, 1, 0, 0, 4095, 1, 1, 4094, 4054, 0, 3881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 1, 0, 1, 0, 4091, 0, 13, 466, 0, 1, 1, 1, 4095, 4095, 2919, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 4077, 1, 5, 1, 0, 1648, 4066, 0, 1514, 1714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 4095, 0, 1, 0, 0, 4092, 1, 4095, 1, 1, 1, 2214, 1882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 0, 0, 0, 0, 53, 2213, 2721, 4094, 0, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16634, 16399, 15255, 16198, 15807, 17123, 16391, 17265] avg=16384.0000 max=17265 max_rank=7 min=15255 min_rank=2 imbalance_ratio=1.053772 max_load_reduction=113807 max_load_reduction_pct=0.868279 moved_to_echo=66586 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4093, 0, 0, 0, 4096, 0, 0, 11, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4073, 43, 2527, 4027, 1150, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 819, 0, 0, 0, 0, 4096, 4096, 2919, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1649, 4082, 0, 518, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2214, 1883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2213, 3244, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16265, 16419, 16149, 16459, 16145, 15505, 16385, 17745] avg=16384.0000 max=17745 max_rank=7 min=15505 min_rank=5 imbalance_ratio=1.083069 max_load_reduction=113327 max_load_reduction_pct=0.864616 moved_to_echo=65780 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 3993, 0, 0, 0, 4096, 0, 20, 16, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4079, 0, 0, 0, 0, 4081, 107, 3578, 4031, 1153, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 52, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1434, 0, 0, 0, 0, 4096, 4096, 2924, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4086, 0, 0, 0, 1, 1639, 4086, 0, 443, 1376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 4076, 0, 0, 0, 2217, 1888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 2208, 1335, 4093, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16265, 16334, 17286, 16500, 16773, 15709, 16374, 15831] avg=16384.0000 max=17286 max_rank=2 min=15709 min_rank=5 imbalance_ratio=1.055054 max_load_reduction=113786 max_load_reduction_pct=0.868118 moved_to_echo=64687 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 7, 241, 2, 0, 0, 0, 0, 0, 9, 4056, 1, 1, 0, 4095, 4095, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4095, 4085, 0, 0, 1, 4093, 1, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 34, 1, 1, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 1755, 0, 3870, 4026, 1149, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 1, 53, 1, 0, 0, 1, 0, 0, 4095, 1, 1, 4094, 4036, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 1, 0, 1, 0, 4091, 0, 65, 89, 0, 0, 0, 1, 4095, 4095, 2919, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 4063, 1, 0, 1, 0, 1648, 4095, 0, 333, 3455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4095, 0, 1, 0, 0, 4092, 1, 4095, 1, 1, 1, 2214, 1882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 0, 0, 0, 0, 33, 2213, 2618, 4095, 0, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16485, 16388, 15168, 16325, 15480, 17693, 16388, 17145] avg=16384.0000 max=17693 max_rank=5 min=15168 min_rank=2 imbalance_ratio=1.079895 max_load_reduction=113379 max_load_reduction_pct=0.865013 moved_to_echo=66706 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4081, 0, 0, 0, 4096, 0, 1, 1, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0, 0, 4045, 37, 3299, 4027, 1150, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1491, 0, 0, 0, 0, 4096, 4096, 2919, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4094, 0, 0, 0, 1, 1649, 4090, 0, 1637, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 2214, 1883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 2213, 1290, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16265, 16398, 16889, 16461, 16818, 16069, 16381, 15791] avg=16384.0000 max=16889 max_rank=2 min=15791 min_rank=7 imbalance_ratio=1.030823 max_load_reduction=114183 max_load_reduction_pct=0.871147 moved_to_echo=65059 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 383, 0, 0, 1, 1, 0, 1, 6, 4067, 1, 1, 0, 4095, 4095, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4095, 4089, 0, 0, 0, 4087, 1, 1, 2, 1, 1, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 29, 0, 1, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 2490, 1, 4014, 4026, 1149, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 1, 1, 1, 0, 0, 1, 0, 0, 4094, 1, 1, 4090, 4092, 0, 3930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 1, 0, 4091, 0, 13, 169, 0, 0, 1, 1, 4095, 4094, 2919, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 4083, 1, 20, 0, 0, 1648, 4084, 0, 881, 2465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 4095, 0, 1, 0, 0, 4092, 1, 4093, 1, 1, 1, 2213, 1882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 1, 0, 0, 0, 0, 9, 2213, 2053, 4094, 1, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16628, 16397, 16041, 16281, 15507, 17279, 16383, 16556] avg=16384.0000 max=17279 max_rank=5 min=15507 min_rank=4 imbalance_ratio=1.054626 max_load_reduction=113793 max_load_reduction_pct=0.868172 moved_to_echo=65725 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4008, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3885, 71, 3392, 4027, 1150, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 39, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 984, 0, 0, 0, 0, 4096, 4096, 2919, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1649, 4089, 0, 799, 1996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2214, 1883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2213, 1219, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16265, 16325, 16854, 16487, 16311, 16725, 16385, 15720] avg=16384.0000 max=16854 max_rank=2 min=15720 min_rank=7 imbalance_ratio=1.028687 max_load_reduction=114218 max_load_reduction_pct=0.871414 moved_to_echo=65141 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 0, 128, 1152, 128, 0, 128, 128, 0, 128, 128, 32768, 128, 128, 0, 32768, 32768, 31872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0, 128, 17792, 15104, 32768, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 32768, 128, 128, 128, 128, 13184, 32768, 0, 6912, 12928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[640, 128, 256, 128, 0, 0, 128, 0, 0, 32768, 128, 128, 32768, 32640, 0, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 0, 0, 0, 32768, 32640, 0, 0, 128, 32768, 128, 128, 128, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 0, 128, 0, 0, 32768, 128, 32768, 128, 128, 128, 17792, 15104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 0, 32768, 0, 128, 128, 128, 25984, 384, 28672, 32256, 9216, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 128, 32768, 0, 128, 8448, 0, 128, 128, 128, 32768, 32768, 23424, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4095, 1, 0, 4045, 162, 44, 0, 4095, 4094, 4075, 1, 4095, 4095, 1, 4092, 1, 0, 0, 0, 4095, 0, 0, 11, 4093, 0, 4062, 0, 3, 4073, 1, 0, 0, 4095, 1, 1, 0, 0, 2278, 1, 1, 1, 573, 4095, 1, 820, 4025, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 12, 1, 1, 0, 4084, 1, 4095, 0, 1, 0, 0, 1, 0, 4081, 3775, 0, 0, 0, 0, 0, 0, 4088, 4095, 0, 0, 0, 0, 1, 4084, 0, 0, 1, 0, 1, 4095, 0, 4092, 38, 3, 1, 496, 0, 4095, 4066, 5, 35, 129, 0, 1, 2, 10, 3984, 0, 0, 0, 0, 1, 0, 0, 4095, 1, 1, 1, 1, 19, 160, 4095, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4088, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2414, 0, 0, 0, 1419, 3939, 0, 315, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 108, 0, 0, 49, 0, 4002, 4096, 0, 0, 53, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4095, 0, 0, 4032, 152, 47, 0, 4095, 4094, 4074, 1, 4095, 4081, 1, 4092, 1, 0, 0, 0, 4094, 0, 0, 9, 4092, 0, 4077, 0, 4, 4065, 1, 0, 0, 4095, 1, 1, 0, 0, 2961, 1, 1, 1, 376, 4095, 0, 430, 4025, 1, 1, 1, 1, 4095, 0, 1, 0, 4095, 14, 1, 1, 0, 4089, 1, 4095, 0, 1, 0, 0, 1, 0, 4090, 3630, 0, 0, 0, 0, 0, 1, 4082, 4095, 0, 1, 0, 0, 1, 4086, 0, 0, 1, 0, 2, 4095, 1, 4092, 79, 6, 1, 612, 0, 4095, 4075, 2, 36, 60, 0, 1, 3, 5, 3960, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 1, 1, 1, 20, 155, 4095, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4087, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2543, 0, 0, 0, 1512, 4077, 0, 25, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 0, 0, 30, 0, 4095, 4096, 0, 0, 14, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4081, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3270, 0, 0, 0, 597, 4007, 0, 186, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 4096, 9, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 95, 0, 0, 5, 0, 4029, 4096, 0, 0, 112, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4078, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4058, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4085, 0, 4079, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2774, 0, 0, 0, 1128, 3843, 0, 167, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 27, 0, 0, 0, 4096, 4094, 34, 0, 0, 0, 0, 0, 4096, 4096, 11, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 251, 0, 0, 28, 0, 4058, 4096, 0, 0, 90, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4081, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3416, 0, 0, 0, 212, 3984, 0, 337, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 40, 0, 0, 10, 0, 4090, 4096, 0, 0, 216, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1, 1, 0, 1, 4095, 0, 0, 4061, 146, 12, 1, 4095, 4094, 4081, 1, 4095, 4094, 1, 4093, 1, 0, 0, 0, 4095, 0, 0, 5, 4081, 0, 4033, 1, 0, 4092, 1, 0, 0, 4094, 1, 1, 0, 0, 2430, 1, 1, 1, 803, 4095, 0, 313, 4046, 1, 1, 1, 1, 4095, 1, 0, 0, 4095, 6, 1, 1, 0, 4091, 1, 4095, 0, 1, 0, 0, 1, 0, 4087, 3825, 0, 1, 0, 0, 0, 2, 4093, 4094, 0, 2, 0, 0, 1, 4087, 0, 0, 1, 0, 1, 4095, 1, 4090, 122, 5, 0, 564, 0, 4095, 4075, 4, 27, 116, 0, 1, 3, 9, 3943, 0, 0, 1, 0, 0, 0, 0, 4095, 2, 1, 1, 1, 12, 160, 4095, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4095, 1, 0, 4045, 162, 44, 0, 0, 4052, 4075, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4092, 1, 0, 0, 0, 32, 0, 0, 11, 4093, 0, 4062, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 1, 0, 0, 4095, 1, 1, 0, 0, 2278, 1, 1, 1, 573, 4095, 1, 302, 246, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 820, 4025, 1, 1, 1, 1, 4095, 0, 0, 0, 3793, 12, 1, 1, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 0, 4081, 3775, 0, 0, 0, 0, 0, 0, 4089, 68, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4095, 0, 0, 0, 0, 1, 4084, 0, 0, 1, 0, 1, 6, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 3, 1, 496, 0, 4095, 4066, 5, 35, 129, 0, 1, 2, 10, 3984, 0, 3749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 1, 1, 1, 1, 19, 160, 4095, 1, 0, 4095, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16478, 16390, 15711, 16835, 16175, 16368, 16614, 16501] avg=16384.0000 max=16835 max_rank=3 min=15711 min_rank=2 imbalance_ratio=1.027527 max_load_reduction=114237 max_load_reduction_pct=0.871559 moved_to_echo=65658 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4054, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 32, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2543, 0, 0, 0, 1512, 4077, 0, 302, 246, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3794, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4090, 68, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 30, 0, 4095, 4096, 0, 0, 14, 0, 0, 0, 0, 4096, 0, 3750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16342, 16407, 16914, 16107, 16510, 16390, 16087, 16315] avg=16384.0000 max=16914 max_rank=2 min=16087 min_rank=6 imbalance_ratio=1.032349 max_load_reduction=114158 max_load_reduction_pct=0.870956 moved_to_echo=65302 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4054, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 4096, 0, 0, 0, 0, 32, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3270, 0, 0, 0, 597, 4007, 0, 302, 246, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3794, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 4096, 9, 0, 0, 0, 0, 0, 4090, 68, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 5, 0, 4029, 4096, 0, 0, 112, 0, 0, 0, 0, 4093, 0, 3750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16342, 16397, 16656, 16268, 16521, 16391, 16180, 16317] avg=16384.0000 max=16656 max_rank=2 min=16180 min_rank=6 imbalance_ratio=1.016602 max_load_reduction=114416 max_load_reduction_pct=0.872925 moved_to_echo=65409 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4095, 0, 0, 4032, 152, 47, 0, 0, 4052, 4074, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 1, 4092, 1, 0, 0, 0, 32, 0, 0, 9, 4092, 0, 4077, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 1, 0, 0, 4095, 1, 1, 0, 0, 2961, 1, 1, 1, 376, 4095, 0, 302, 246, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 430, 4025, 1, 1, 1, 1, 4095, 0, 1, 0, 3793, 14, 1, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 0, 4090, 3630, 0, 0, 0, 0, 0, 1, 4089, 68, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4095, 0, 1, 0, 0, 1, 4086, 0, 0, 1, 0, 2, 6, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 6, 1, 612, 0, 4095, 4075, 2, 36, 60, 0, 1, 3, 5, 3960, 0, 3748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 1, 1, 1, 20, 155, 4095, 1, 0, 4095, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16456, 16389, 16188, 16453, 16040, 16367, 16683, 16496] avg=16384.0000 max=16683 max_rank=6 min=16040 min_rank=4 imbalance_ratio=1.018250 max_load_reduction=114389 max_load_reduction_pct=0.872719 moved_to_echo=65586 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4054, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 32, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2414, 0, 0, 0, 1419, 3939, 0, 302, 246, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3794, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4090, 68, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 49, 0, 4002, 4096, 0, 0, 53, 0, 0, 0, 0, 4094, 0, 3750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16342, 16407, 16554, 16397, 16509, 16390, 16152, 16321] avg=16384.0000 max=16554 max_rank=2 min=16152 min_rank=6 imbalance_ratio=1.010376 max_load_reduction=114518 max_load_reduction_pct=0.873703 moved_to_echo=65372 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 1, 4095, 0, 0, 4061, 146, 12, 1, 0, 4052, 4081, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 4093, 1, 0, 0, 0, 32, 0, 0, 5, 4081, 0, 4033, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 4094, 1, 1, 0, 0, 2430, 1, 1, 1, 803, 4095, 0, 302, 246, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 4046, 1, 1, 1, 1, 4095, 1, 0, 0, 3793, 6, 1, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 0, 4087, 3825, 0, 1, 0, 0, 0, 2, 4089, 68, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4094, 0, 2, 0, 0, 1, 4087, 0, 0, 1, 0, 1, 6, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 5, 0, 564, 0, 4095, 4075, 4, 27, 116, 0, 1, 3, 9, 3943, 0, 3749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 2, 1, 1, 1, 12, 160, 4095, 1, 0, 4095, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16452, 16341, 16110, 16351, 16234, 16376, 16713, 16495] avg=16384.0000 max=16713 max_rank=6 min=16110 min_rank=2 imbalance_ratio=1.020081 max_load_reduction=114359 max_load_reduction_pct=0.872490 moved_to_echo=65818 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4054, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 32, 0, 0, 0, 4096, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3416, 0, 0, 0, 212, 3984, 0, 302, 246, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3794, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4090, 68, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 10, 0, 4090, 4096, 0, 0, 216, 0, 0, 0, 0, 4096, 0, 3750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16342, 16401, 16394, 16419, 16510, 16390, 16298, 16318] avg=16384.0000 max=16510 max_rank=4 min=16298 min_rank=6 imbalance_ratio=1.007690 max_load_reduction=114562 max_load_reduction_pct=0.874039 moved_to_echo=65516 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4078, 0, 0, 4096, 0, 0, 0, 0, 4048, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 4085, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2774, 0, 0, 0, 1128, 3843, 0, 308, 252, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 3788, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 27, 0, 0, 0, 4096, 4094, 34, 0, 0, 0, 0, 0, 4091, 71, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 11, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 28, 0, 4058, 4096, 0, 0, 90, 0, 0, 0, 0, 4071, 0, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16318, 16340, 16545, 16242, 16573, 16400, 16345, 16309] avg=16384.0000 max=16573 max_rank=4 min=16242 min_rank=3 imbalance_ratio=1.011536 max_load_reduction=114499 max_load_reduction_pct=0.873558 moved_to_echo=65627 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 32640, 512, 128, 128, 0, 32512, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 32768, 128, 128, 0, 0, 128, 32768, 0, 0, 128, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[2688, 32640, 128, 128, 128, 128, 32768, 128, 128, 0, 30464, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 32768, 128, 0, 0, 0, 256, 0, 0, 128, 32768, 0, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[768, 128, 128, 1920, 0, 32640, 32768, 128, 128, 896, 0, 128, 128, 128, 32384, 0, 30080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 0, 0, 32768, 128, 128, 128, 128, 128, 512, 32768, 128, 0, 32768, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 0, 0, 32768, 128, 128, 0, 0, 22144, 128, 128, 128, 6656, 32256, 128, 2432, 2048, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 128, 0, 32768, 31744, 128, 128, 0, 0, 0, 128, 32768, 640, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4093, 27, 0, 26, 0, 4070, 1, 0, 4095, 1, 0, 1, 4094, 0, 0, 0, 4090, 2, 0, 1, 1, 4095, 4095, 0, 1, 4095, 4082, 4077, 0, 0, 1, 1, 1, 0, 0, 1, 1, 79, 8, 1, 0, 95, 1, 0, 4019, 0, 0, 1, 4094, 1, 4054, 22, 0, 0, 4073, 4095, 1, 1, 1, 0, 0, 1, 4011, 14, 1, 4, 4076, 4083, 1, 0, 0, 1, 1, 0, 1, 1, 4095, 0, 1, 1, 2, 607, 0, 0, 4091, 4095, 0, 4070, 0, 1, 0, 1, 0, 4050, 1, 4095, 0, 4065, 4095, 0, 0, 22, 4095, 1, 4095, 0, 0, 0, 1, 1, 0, 1, 3609, 0, 0, 0, 1, 4095, 4092, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 8, 0, 0, 0, 4096, 0, 0, 4096, 0, 138, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 0, 73, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 8, 4096, 4096, 0, 14, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 3976, 4085, 0, 4096, 0, 0, 49, 0, 0, 4084, 0, 4096, 0, 4096, 4091, 0, 0, 1, 4096, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 4094, 17, 0, 20, 0, 4073, 1, 0, 4095, 1, 0, 1, 4094, 0, 0, 0, 4087, 1, 0, 1, 1, 4095, 4095, 0, 1, 4095, 4070, 4093, 0, 0, 2, 1, 1, 0, 0, 1, 1, 40, 2, 1, 0, 98, 1, 0, 4017, 0, 0, 1, 4094, 1, 4060, 24, 0, 0, 4077, 4095, 0, 1, 1, 0, 0, 2, 4035, 17, 1, 3, 4082, 4095, 1, 0, 0, 1, 1, 0, 0, 1, 4094, 0, 1, 1, 6, 559, 0, 0, 4089, 4095, 0, 4056, 0, 0, 0, 1, 0, 3830, 1, 4095, 0, 4070, 4095, 0, 0, 31, 4095, 1, 4089, 0, 1, 0, 1, 1, 0, 1, 3881, 0, 0, 0, 1, 4095, 4092, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 176, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 35, 0, 0, 4096, 0, 8, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 24, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 3986, 4096, 0, 4096, 0, 0, 58, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 54, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 55, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 32, 0, 0, 3978, 4096, 0, 4096, 0, 0, 12, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 127, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 4096, 0, 7, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 67, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 4013, 4095, 0, 4096, 0, 0, 4, 0, 0, 4095, 0, 4096, 0, 4096, 4096, 0, 0, 2, 4096, 0, 3970, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 2, 4093, 32, 0, 26, 0, 4067, 1, 0, 4094, 1, 0, 1, 4094, 1, 0, 0, 4083, 0, 0, 1, 1, 4095, 4094, 0, 1, 4095, 4091, 4086, 0, 4, 1, 1, 1, 1, 0, 1, 1, 14, 8, 0, 0, 140, 1, 0, 4012, 0, 0, 1, 4093, 1, 4070, 19, 1, 0, 4093, 4095, 0, 1, 1, 0, 0, 1, 4031, 2, 1, 4, 4079, 4094, 1, 1, 0, 0, 1, 0, 0, 1, 4094, 1, 0, 1, 6, 556, 0, 0, 4085, 4094, 1, 4089, 0, 1, 0, 1, 0, 4010, 1, 4095, 0, 4079, 4094, 1, 0, 18, 4095, 1, 4085, 0, 0, 0, 1, 1, 0, 1, 3638, 1, 0, 0, 1, 4095, 4094, 0, 2, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 5, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 15, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4024, 0, 0, 0, 4093, 27, 0, 26, 0, 4070, 1, 0, 4095, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4090, 2, 0, 1, 1, 0, 13, 0, 1, 0, 4082, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 79, 8, 1, 0, 95, 1, 0, 4095, 4095, 4095, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 1, 4094, 1, 4054, 22, 0, 0, 4073, 29, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4011, 14, 1, 4, 4076, 4083, 1, 0, 0, 1, 1, 0, 1, 1, 3883, 143, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 2, 607, 0, 0, 4091, 3952, 0, 4070, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 1, 0, 0, 4065, 4095, 0, 0, 22, 100, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 3609, 0, 0, 0, 1, 4095, 4092, 0, 1, 0, 0, 0, 4066, 112, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16338, 16361, 16493, 16296, 16292, 16821, 16429, 16042] avg=16384.0000 max=16821 max_rank=5 min=16042 min_rank=7 imbalance_ratio=1.026672 max_load_reduction=114251 max_load_reduction_pct=0.871666 moved_to_echo=65584 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4025, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 5, 0, 0, 4096, 4096, 4096, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 143, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 32, 0, 0, 3978, 3953, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 100, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4067, 112, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16367, 16398, 16333, 16413, 16441, 16158, 16432, 16530] avg=16384.0000 max=16530 max_rank=7 min=16158 min_rank=5 imbalance_ratio=1.008911 max_load_reduction=114542 max_load_reduction_pct=0.873886 moved_to_echo=65561 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4018, 0, 0, 0, 4096, 8, 0, 0, 0, 4096, 0, 0, 4096, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 4096, 4096, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 73, 0, 4096, 0, 4096, 0, 0, 0, 4096, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 8, 4096, 4096, 0, 14, 0, 0, 0, 0, 0, 0, 3889, 146, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 3976, 3939, 0, 4096, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4096, 4091, 0, 0, 1, 92, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4070, 115, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16452, 16387, 16326, 16483, 16425, 16109, 16363, 16527] avg=16384.0000 max=16527 max_rank=7 min=16109 min_rank=5 imbalance_ratio=1.008728 max_load_reduction=114545 max_load_reduction_pct=0.873909 moved_to_echo=65424 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4025, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 35, 0, 0, 4096, 4096, 4096, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 4096, 0, 4096, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 143, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 0, 0, 0, 0, 0, 3986, 3953, 0, 4096, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 100, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4067, 112, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16489, 16398, 16349, 16421, 16410, 16059, 16417, 16529] avg=16384.0000 max=16529 max_rank=7 min=16059 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=114543 max_load_reduction_pct=0.873894 moved_to_echo=65415 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4025, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 4096, 4096, 4096, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 4096, 0, 4096, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 3884, 143, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 4013, 3952, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 2, 100, 0, 3970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4067, 112, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16440, 16398, 16333, 16420, 16454, 16140, 16359, 16528] avg=16384.0000 max=16528 max_rank=7 min=16140 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=114544 max_load_reduction_pct=0.873901 moved_to_echo=65481 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4024, 0, 0, 1, 4094, 17, 0, 20, 0, 4073, 1, 0, 4095, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4087, 1, 0, 1, 1, 0, 13, 0, 1, 0, 4070, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 0, 1, 1, 40, 2, 1, 0, 98, 1, 0, 4095, 4095, 4095, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 0, 1, 4094, 1, 4060, 24, 0, 0, 4077, 29, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4035, 17, 1, 3, 4082, 4095, 1, 0, 0, 1, 1, 0, 0, 1, 3883, 143, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 1, 6, 559, 0, 0, 4089, 3952, 0, 4056, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3830, 1, 0, 0, 4070, 4095, 0, 0, 31, 100, 1, 4089, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 3881, 0, 0, 0, 1, 4095, 4092, 0, 0, 1, 0, 0, 4066, 112, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16327, 16361, 16452, 16305, 16336, 16759, 16218, 16314] avg=16384.0000 max=16759 max_rank=5 min=16218 min_rank=6 imbalance_ratio=1.022888 max_load_reduction=114313 max_load_reduction_pct=0.872139 moved_to_echo=65627 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4024, 0, 0, 2, 4093, 32, 0, 26, 0, 4067, 1, 0, 4094, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 4083, 0, 0, 1, 1, 0, 13, 0, 1, 0, 4091, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 1, 1, 1, 0, 1, 1, 14, 8, 0, 0, 140, 1, 0, 4095, 4095, 4095, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 1, 4093, 1, 4070, 19, 1, 0, 4093, 29, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4031, 2, 1, 4, 4079, 4094, 1, 1, 0, 0, 1, 0, 0, 1, 3883, 143, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 1, 6, 556, 0, 0, 4085, 3951, 1, 4089, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 1, 0, 0, 4079, 4094, 1, 0, 18, 100, 1, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 3638, 1, 0, 0, 1, 4095, 4094, 0, 2, 0, 0, 0, 4066, 112, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16341, 16371, 16476, 16321, 16313, 16786, 16389, 16075] avg=16384.0000 max=16786 max_rank=5 min=16075 min_rank=7 imbalance_ratio=1.024536 max_load_reduction=114286 max_load_reduction_pct=0.871933 moved_to_echo=65563 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4025, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4096, 4096, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 143, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 3953, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 100, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4067, 112, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16318, 16398, 16310, 16413, 16401, 16240, 16465, 16527] avg=16384.0000 max=16527 max_rank=7 min=16240 min_rank=5 imbalance_ratio=1.008728 max_load_reduction=114545 max_load_reduction_pct=0.873909 moved_to_echo=65633 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 0, 0, 32768, 128, 0, 128, 128, 0, 128, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32256, 0, 0, 128, 32768, 128, 0, 128, 0, 32768, 128, 0, 32768, 128, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32640, 0, 128, 128, 32768, 128, 32768, 128, 128, 0, 32768, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32384, 128, 0, 0, 32768, 32768, 128, 0, 128, 896, 128, 32512, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[256, 128, 32640, 128, 128, 128, 32768, 32768, 128, 128, 0, 128, 128, 0, 128, 128, 31104, 1152, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32640, 128, 128, 128, 128, 1792, 0, 0, 32384, 31616, 128, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 128, 256, 128, 128, 0, 512, 128, 0, 32768, 32768, 32768, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 31616, 128, 0, 0, 128, 32768, 32768, 0, 128, 128, 0, 0, 32640, 1024, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 152, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3943, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2683, 3805, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 26, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2984, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 3855, 4096, 0, 0, 4068, 0, 0, 4096, 0, 0, 0, 4096, 85, 0, 0, 2, 0, 619, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2912, 0, 0, 0, 0, 0, 2394, 1151, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 12, 0, 4096, 0, 0, 52, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3901, 0, 0, 0, 4090, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 3168, 3837, 0, 0, 4055, 0, 0, 0, 25, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 45, 0, 4096, 4064, 0, 1, 5, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 3049, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4091, 3986, 4096, 0, 0, 4063, 0, 0, 4096, 11, 0, 0, 4096, 128, 0, 0, 27, 0, 1115, 4074, 0, 0, 0, 4090, 6, 0, 0, 0, 0, 3388, 0, 0, 0, 0, 0, 1053, 920, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 2, 4057, 1, 1, 0, 0, 0, 4094, 1, 1, 2053, 5, 1, 9, 2, 4091, 1, 4095, 0, 0, 0, 0, 4095, 4095, 1, 4093, 1, 1, 5, 4090, 1, 1, 1, 0, 0, 3935, 0, 5, 0, 399, 2126, 4, 0, 4035, 0, 0, 3, 0, 0, 0, 0, 0, 4087, 1, 4094, 4092, 0, 1, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 4095, 1, 1, 1, 0, 3941, 4095, 190, 16, 4095, 1, 4095, 1, 0, 4095, 4092, 4076, 1, 0, 3600, 0, 0, 4095, 0, 1, 0, 4066, 0, 0, 38, 0, 1, 1762, 4093, 0, 0, 1, 4095, 227, 0, 1, 46, 0, 779, 1, 3, 0, 1, 1, 2565, 3034, 4092, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 250, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4066, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2442, 3808, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 38, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3024, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 3986, 4096, 0, 0, 4054, 0, 0, 4096, 0, 0, 0, 4096, 101, 0, 0, 1, 0, 519, 4084, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 1853, 547, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1, 1, 4068, 1, 1, 0, 0, 0, 4094, 1, 1, 1099, 3, 1, 17, 2, 4092, 1, 4095, 0, 0, 0, 0, 4095, 4095, 1, 4088, 1, 1, 7, 4088, 1, 1, 1, 0, 0, 4061, 0, 4, 0, 75, 3099, 7, 1, 4048, 0, 0, 4, 0, 0, 0, 0, 0, 4087, 1, 4094, 4093, 0, 1, 0, 0, 4095, 4095, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 4093, 1, 1, 1, 2, 2925, 4095, 249, 19, 4095, 1, 4094, 1, 0, 4095, 4091, 4079, 1, 0, 3518, 0, 0, 4095, 0, 1, 0, 4067, 0, 0, 26, 0, 1, 2389, 4082, 0, 0, 1, 4095, 215, 0, 1, 42, 0, 1531, 0, 3, 0, 1, 1, 2344, 3097, 4092, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3186, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3240, 2025, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3943, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4094, 4096, 0, 0, 4048, 0, 0, 4096, 0, 0, 0, 4096, 37, 0, 0, 0, 0, 391, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2545, 0, 0, 0, 0, 0, 759, 319, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 1050, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4033, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2786, 3726, 0, 0, 4082, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 51, 0, 4096, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3257, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 3927, 4096, 0, 0, 4022, 0, 0, 4096, 0, 0, 0, 4096, 84, 0, 0, 8, 0, 327, 4094, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 3394, 0, 0, 0, 0, 0, 473, 1548, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[2, 1, 4078, 1, 1, 1, 1, 0, 4094, 1, 1, 1378, 4, 1, 12, 1, 4092, 1, 4095, 0, 0, 0, 1, 4094, 4095, 0, 4070, 1, 1, 15, 4088, 1, 1, 1, 0, 0, 4003, 0, 7, 0, 378, 2869, 7, 2, 4087, 0, 0, 0, 1, 0, 0, 0, 0, 4090, 0, 4095, 4092, 0, 1, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 4088, 1, 1, 1, 4, 3795, 4095, 308, 4, 4095, 2, 4094, 0, 0, 4095, 4078, 4087, 0, 0, 3449, 0, 0, 4095, 1, 1, 0, 4070, 11, 0, 27, 0, 0, 2078, 4059, 1, 0, 1, 4095, 201, 0, 1, 82, 0, 1161, 1, 3, 0, 2, 1, 1459, 3471, 4093, 0, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4057, 1, 1, 0, 0, 0, 4094, 1, 1, 2053, 5, 1, 9, 2, 4095, 2943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 4095, 61, 1, 4093, 1, 1, 5, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 3935, 0, 5, 0, 399, 2126, 4, 0, 4035, 0, 0, 3, 2123, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 4080, 4092, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 4095, 1, 1, 1, 0, 3941, 4095, 2914, 1911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 16, 0, 1, 234, 1, 0, 4095, 4092, 4076, 1, 0, 3600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4066, 0, 0, 38, 0, 1, 1762, 4093, 0, 0, 1, 4095, 227, 2798, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 46, 0, 779, 1, 3, 0, 1, 1, 2565, 3034, 4092, 0, 1, 0, 4095, 1181, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17265, 16440, 13695, 16356, 16962, 16306, 17096, 16952] avg=16384.0000 max=17265 max_rank=0 min=13695 min_rank=2 imbalance_ratio=1.053772 max_load_reduction=113807 max_load_reduction_pct=0.868279 moved_to_echo=67316 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 12, 0, 4096, 0, 0, 52, 0, 0, 0, 0, 4096, 2945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 51, 0, 3901, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 3168, 3837, 0, 0, 4055, 0, 0, 0, 2128, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 4096, 0, 4076, 4096, 0, 0, 45, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 3049, 4096, 2916, 1917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 0, 0, 4091, 3986, 4096, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 128, 0, 0, 27, 0, 1115, 4074, 0, 0, 0, 4090, 6, 2801, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3388, 0, 0, 0, 0, 0, 1053, 920, 4096, 0, 0, 0, 4096, 1180, 1151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15297, 16234, 18355, 16402, 16069, 16463, 16368, 15884] avg=16384.0000 max=18355 max_rank=2 min=15297 min_rank=0 imbalance_ratio=1.120300 max_load_reduction=112717 max_load_reduction_pct=0.859962 moved_to_echo=64784 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3186, 0, 0, 0, 0, 4096, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 62, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3240, 2025, 0, 0, 4085, 0, 0, 0, 2123, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3943, 4096, 2915, 1911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 4096, 4094, 4096, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 37, 0, 0, 0, 0, 391, 4095, 0, 0, 0, 4096, 0, 2799, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2545, 0, 0, 0, 0, 0, 759, 319, 4096, 0, 0, 0, 4096, 1181, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18418, 16446, 16632, 16371, 16961, 16568, 15528, 14148] avg=16384.0000 max=18418 max_rank=0 min=14148 min_rank=7 imbalance_ratio=1.124146 max_load_reduction=112654 max_load_reduction_pct=0.859482 moved_to_echo=63205 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 250, 0, 0, 0, 0, 4096, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 62, 0, 4066, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2442, 3808, 0, 0, 4095, 0, 0, 0, 2123, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 4096, 0, 0, 38, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3024, 4096, 2915, 1911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 4096, 3986, 4096, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 101, 0, 0, 1, 0, 519, 4084, 0, 0, 0, 4096, 0, 2799, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 1853, 547, 4096, 0, 0, 0, 4096, 1181, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15482, 16416, 17627, 16408, 16042, 16466, 15710, 16921] avg=16384.0000 max=17627 max_rank=2 min=15482 min_rank=0 imbalance_ratio=1.075867 max_load_reduction=113445 max_load_reduction_pct=0.865517 moved_to_echo=65139 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 152, 0, 0, 0, 0, 4096, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 62, 0, 3943, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2683, 3805, 0, 0, 4089, 0, 0, 0, 2123, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 4096, 0, 0, 26, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2984, 4096, 2915, 1911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 4096, 3855, 4096, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 85, 0, 0, 2, 0, 619, 4096, 0, 0, 0, 4096, 0, 2799, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2912, 0, 0, 0, 0, 0, 2394, 1151, 4096, 0, 0, 0, 4096, 1181, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15384, 16293, 17859, 16396, 16002, 16349, 15807, 16982] avg=16384.0000 max=17859 max_rank=2 min=15384 min_rank=0 imbalance_ratio=1.090027 max_load_reduction=113213 max_load_reduction_pct=0.863747 moved_to_echo=65140 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 1050, 0, 0, 0, 0, 4096, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 62, 0, 4033, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2786, 3726, 0, 0, 4082, 0, 0, 0, 2123, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4082, 4096, 0, 0, 51, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3257, 4096, 2915, 1911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 4096, 3927, 4096, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 84, 0, 0, 8, 0, 327, 4094, 0, 0, 0, 4096, 1, 2799, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3394, 0, 0, 0, 0, 0, 473, 1548, 4096, 0, 0, 0, 4096, 1181, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16283, 16382, 17876, 16422, 16275, 16375, 15519, 15940] avg=16384.0000 max=17876 max_rank=2 min=15519 min_rank=6 imbalance_ratio=1.091064 max_load_reduction=113196 max_load_reduction_pct=0.863617 moved_to_echo=64109 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 4068, 1, 1, 0, 0, 0, 4094, 1, 1, 1099, 3, 1, 17, 2, 4095, 2943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 4095, 61, 1, 4088, 1, 1, 7, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4061, 0, 4, 0, 75, 3099, 7, 1, 4048, 0, 0, 4, 2123, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 4080, 4093, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 4093, 1, 1, 1, 2, 2925, 4095, 2914, 1911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 19, 0, 1, 234, 1, 0, 4095, 4091, 4079, 1, 0, 3518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4067, 0, 0, 26, 0, 1, 2389, 4082, 0, 0, 1, 4095, 215, 2797, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 42, 0, 1531, 0, 3, 0, 1, 1, 2344, 3097, 4092, 0, 1, 0, 4095, 1181, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16328, 16436, 14487, 16357, 15947, 16288, 17688, 17541] avg=16384.0000 max=17688 max_rank=6 min=14487 min_rank=2 imbalance_ratio=1.079590 max_load_reduction=113384 max_load_reduction_pct=0.865051 moved_to_echo=67464 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 4078, 1, 1, 1, 1, 0, 4094, 1, 1, 1378, 4, 1, 12, 1, 4095, 2943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 1, 4094, 61, 0, 4070, 1, 1, 15, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4003, 0, 7, 0, 378, 2869, 7, 2, 4087, 0, 0, 0, 2123, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4090, 0, 4081, 4092, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 4088, 1, 1, 1, 4, 3795, 4095, 2914, 1911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 308, 4, 0, 2, 234, 0, 0, 4095, 4078, 4087, 0, 0, 3449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4070, 11, 0, 27, 0, 0, 2078, 4059, 1, 0, 1, 4095, 201, 2797, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 82, 0, 1161, 1, 3, 0, 2, 1, 1459, 3471, 4093, 0, 1, 1, 4095, 1181, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16615, 16425, 14541, 16360, 16814, 16257, 17356, 16704] avg=16384.0000 max=17356 max_rank=6 min=14541 min_rank=2 imbalance_ratio=1.059326 max_load_reduction=113716 max_load_reduction_pct=0.867584 moved_to_echo=67131 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 128, 0, 32768, 128, 128, 9344, 128, 128, 128, 128, 32768, 23552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 128, 32768, 512, 128, 32384, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 32768, 128, 32768, 32768, 0, 128, 256, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 32768, 512, 0, 128, 128, 128, 9216, 32768, 128, 0, 128, 32768, 768, 22400, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 27008, 32768, 23424, 15360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 256, 0, 19712, 128, 128, 0, 128, 128, 12928, 14208, 32768, 0, 128, 128, 32768, 9472, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 0, 32512, 0, 128, 0, 15232, 25344, 128, 128, 32640, 0, 0, 128, 17024, 8576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[768, 128, 0, 128, 1920, 128, 0, 32768, 32128, 32768, 128, 0, 30848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 0, 1, 6, 0, 4092, 0, 0, 0, 500, 4092, 0, 4091, 0, 4094, 0, 6, 1, 0, 11, 4095, 0, 0, 1, 1, 0, 0, 0, 1, 4079, 62, 4079, 1, 1, 0, 1, 4095, 0, 1, 1, 1, 2, 1, 3, 4094, 2, 0, 2797, 1, 2, 0, 4074, 1, 6, 1, 0, 0, 0, 0, 0, 29, 0, 2, 15, 1, 1, 1, 799, 4095, 1, 0, 4029, 4083, 1, 4095, 0, 3937, 4093, 128, 4095, 0, 0, 37, 0, 1, 4052, 4080, 0, 0, 3974, 0, 1, 0, 1, 0, 4, 0, 4095, 4095, 4058, 1, 4095, 0, 4093, 0, 4095, 4095, 0, 1, 0, 1, 4095, 308, 0, 4095, 0, 1, 1, 0, 1, 0, 1, 3993] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1809, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 347, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1943, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 466, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1718, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1814, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 104, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1381, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1887, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 820, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 156, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 1, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2135, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1817, 4095, 0, 0, 4096, 4096, 0, 4095, 1, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 555, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 5, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1526, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2017, 4084, 0, 0, 4096, 4096, 0, 4076, 16, 4096, 4096, 1, 4096, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 4086, 8, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 0, 1, 8, 0, 4090, 0, 0, 0, 630, 4091, 0, 4091, 0, 4094, 0, 7, 1, 0, 12, 4095, 0, 0, 1, 1, 0, 0, 0, 1, 4084, 73, 4090, 1, 1, 0, 1, 4095, 0, 1, 1, 1, 2, 1, 3, 4094, 1, 0, 2240, 1, 2, 0, 4074, 1, 5, 1, 0, 1, 0, 0, 0, 25, 0, 2, 14, 1, 1, 1, 1125, 4095, 1, 0, 4011, 4087, 1, 4095, 0, 3979, 4092, 136, 4095, 0, 0, 69, 0, 1, 4047, 4089, 0, 0, 3979, 0, 1, 0, 1, 0, 3, 0, 4095, 4095, 4063, 1, 4095, 0, 4092, 0, 4095, 4095, 0, 1, 0, 1, 4095, 277, 0, 4095, 0, 1, 1, 0, 0, 0, 1, 4025] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 0, 1, 5, 0, 4092, 0, 0, 0, 382, 4095, 0, 4091, 0, 4094, 0, 5, 0, 0, 11, 4095, 0, 1, 1, 1, 0, 0, 0, 0, 4088, 47, 4089, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 1, 0, 2, 4094, 3, 0, 2908, 1, 2, 0, 4095, 1, 2, 1, 1, 1, 0, 2, 0, 21, 0, 1, 11, 1, 1, 1, 744, 4095, 0, 1, 4017, 4085, 1, 4095, 1, 3993, 4093, 125, 4095, 0, 0, 5, 0, 1, 4043, 4086, 0, 1, 4011, 0, 1, 0, 1, 0, 3, 0, 4095, 4095, 4075, 1, 4095, 0, 4092, 0, 4095, 4095, 0, 1, 0, 1, 4095, 237, 0, 4094, 0, 1, 0, 0, 1, 0, 1, 4057] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 0, 1, 6, 0, 4092, 0, 0, 0, 500, 4092, 1996, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4094, 0, 6, 1, 0, 11, 4095, 0, 0, 1, 1, 0, 0, 2147, 1943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4079, 62, 4079, 1, 1, 0, 1, 4095, 0, 1, 1, 1, 2, 1, 2131, 1948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 2, 0, 2797, 1, 2, 0, 4074, 1, 6, 1, 0, 0, 0, 0, 4095, 2152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 2, 15, 1, 1, 1, 799, 2731, 1, 0, 4029, 4083, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3937, 47, 128, 0, 0, 0, 37, 0, 1, 4052, 4080, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4, 0, 0, 0, 4058, 1, 13, 0, 4093, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 308, 0, 4095, 0, 1, 1, 0, 1, 0, 1, 3993, 2050, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16147, 16390, 16404, 17228, 15788, 16256, 16361, 16498] avg=16384.0000 max=17228 max_rank=3 min=15788 min_rank=4 imbalance_ratio=1.051514 max_load_reduction=113844 max_load_reduction_pct=0.868561 moved_to_echo=64903 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1809, 4096, 1998, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2146, 1943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2132, 1950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 347, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1943, 2732, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 46, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2052, 1952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17459, 16377, 16365, 14788, 16963, 16430, 16396, 16294] avg=16384.0000 max=17459 max_rank=0 min=14788 min_rank=3 imbalance_ratio=1.065613 max_load_reduction=113613 max_load_reduction_pct=0.866798 moved_to_echo=66083 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 555, 4096, 2001, 1362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 2147, 1944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2136, 1949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1526, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 2152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2017, 2722, 0, 0, 4096, 4096, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 41, 1, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 8, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2054, 1955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16206, 16390, 16378, 15963, 17007, 16435, 16387, 16306] avg=16384.0000 max=17007 max_rank=4 min=15963 min_rank=3 imbalance_ratio=1.038025 max_load_reduction=114065 max_load_reduction_pct=0.870247 moved_to_echo=66135 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 156, 4096, 1998, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2147, 1943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2132, 1949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2135, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1817, 2731, 0, 0, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 46, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2052, 1952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15806, 16379, 16358, 16576, 16835, 16431, 16396, 16291] avg=16384.0000 max=16835 max_rank=4 min=15806 min_rank=0 imbalance_ratio=1.027527 max_load_reduction=114237 max_load_reduction_pct=0.871559 moved_to_echo=65953 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1381, 4096, 1998, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2147, 1944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2132, 1949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1887, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 820, 2732, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 46, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2052, 1952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17031, 16379, 16367, 16327, 15840, 16430, 16396, 16302] avg=16384.0000 max=17031 max_rank=0 min=15840 min_rank=4 imbalance_ratio=1.039490 max_load_reduction=114041 max_load_reduction_pct=0.870064 moved_to_echo=64968 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 466, 4096, 1998, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2146, 1943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2132, 1950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1718, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1814, 2732, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 46, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 104, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 2052, 1952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16116, 16377, 16365, 16159, 16834, 16430, 16396, 16395] avg=16384.0000 max=16834 max_rank=4 min=16116 min_rank=0 imbalance_ratio=1.027466 max_load_reduction=114238 max_load_reduction_pct=0.871567 moved_to_echo=66055 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 0, 1, 5, 0, 4092, 0, 0, 0, 382, 4095, 1996, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4094, 0, 5, 0, 0, 11, 4095, 0, 1, 1, 1, 0, 0, 2147, 1942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 47, 4089, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 1, 0, 2131, 1948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 3, 0, 2908, 1, 2, 0, 4095, 1, 2, 1, 1, 1, 0, 2, 4095, 2153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 1, 11, 1, 1, 1, 744, 2731, 0, 1, 4017, 4085, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3993, 47, 125, 0, 0, 0, 5, 0, 1, 4043, 4086, 0, 1, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 3, 0, 0, 0, 4075, 1, 13, 0, 4092, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 237, 0, 4094, 0, 1, 0, 0, 1, 0, 1, 4057, 2050, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16031, 16388, 16404, 17361, 15710, 16313, 16376, 16489] avg=16384.0000 max=17361 max_rank=3 min=15710 min_rank=4 imbalance_ratio=1.059631 max_load_reduction=113711 max_load_reduction_pct=0.867546 moved_to_echo=64888 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 0, 1, 8, 0, 4090, 0, 0, 0, 630, 4091, 1996, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4094, 0, 7, 1, 0, 12, 4095, 0, 0, 1, 1, 0, 0, 2147, 1943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4084, 73, 4090, 1, 1, 0, 1, 4095, 0, 1, 1, 1, 2, 1, 2131, 1948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 1, 0, 2240, 1, 2, 0, 4074, 1, 5, 1, 0, 1, 0, 0, 4095, 2152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 2, 14, 1, 1, 1, 1125, 2731, 1, 0, 4011, 4087, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 47, 136, 0, 0, 0, 69, 0, 1, 4047, 4089, 0, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 3, 0, 0, 0, 4063, 1, 13, 0, 4092, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 277, 0, 4095, 0, 1, 1, 0, 0, 0, 1, 4025, 2049, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16276, 16392, 16431, 16670, 16095, 16347, 16364, 16497] avg=16384.0000 max=16670 max_rank=3 min=16095 min_rank=4 imbalance_ratio=1.017456 max_load_reduction=114402 max_load_reduction_pct=0.872818 moved_to_echo=65303 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 0, 32768, 0, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 0, 0, 17280, 15616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 32512, 384, 512, 0, 0, 0, 128, 0, 128, 32640, 32768, 0, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 32768, 256, 32768, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 128, 128, 17152, 15616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 0, 15616, 128, 128, 0, 32768, 128, 128, 128, 128, 128, 0, 128, 32768, 17280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 32768, 128, 128, 0, 32768, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 128, 128, 0, 32768, 0, 0, 0, 5888, 32768, 16000, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 32768, 1024, 128, 32768, 0, 128, 128, 0, 128, 0, 128, 32640, 16512, 15616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 128, 11136, 21888, 128, 128, 32640, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 175, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1919, 4096, 0, 4088, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 395, 4096, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 856, 0, 4096, 0, 4094, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1084, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 3294, 0, 0, 0, 4096, 4094, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4069, 192, 1, 145, 13, 1, 0, 0, 4095, 4093, 1, 0, 0, 4095, 0, 0, 2, 4091, 33, 0, 2148, 4095, 0, 4024, 7, 1094, 1, 1, 7, 1, 0, 19, 0, 1, 0, 6, 0, 1, 1, 1, 3, 0, 1, 0, 4, 0, 4078, 4095, 583, 4095, 4095, 0, 0, 0, 0, 3697, 0, 1, 1, 19, 1, 0, 0, 4084, 9, 1, 0, 0, 0, 1, 4085, 4089, 4071, 0, 735, 0, 4095, 0, 3827, 1, 4094, 10, 1, 0, 0, 4089, 1, 1, 4095, 4095, 0, 1, 4095, 0, 0, 1, 4095, 1, 1, 1, 0, 235, 1, 4095, 4066, 0, 0, 4095, 1, 0, 1, 3873, 1, 2, 0, 4095, 4095, 4020] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 131, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1191, 4096, 0, 4061, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1683, 4096, 4096, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 69, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 46, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 3931, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 4078, 251, 1, 141, 12, 1, 0, 0, 4095, 4093, 1, 0, 0, 4095, 0, 0, 1, 4091, 38, 1, 2030, 4095, 0, 4012, 6, 1346, 1, 1, 16, 1, 0, 13, 0, 1, 0, 4, 0, 1, 1, 1, 3, 0, 1, 0, 4, 0, 4080, 4092, 181, 4094, 4095, 0, 0, 0, 0, 3601, 0, 2, 1, 27, 1, 0, 0, 4084, 11, 1, 0, 0, 0, 1, 4083, 4089, 4072, 0, 857, 0, 4094, 0, 3808, 1, 4094, 10, 1, 0, 0, 4090, 1, 1, 4095, 4095, 0, 1, 4095, 0, 0, 1, 4095, 0, 1, 1, 0, 448, 1, 4095, 4055, 0, 0, 4095, 1, 0, 1, 3838, 1, 3, 0, 4095, 4095, 4049] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 133, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 2561, 4096, 0, 4006, 0, 230, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 4096, 1599, 4096, 4096, 0, 0, 0, 11, 4076, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 750, 0, 4096, 0, 4089, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 70, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 2952, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 373, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3, 4096, 0, 14, 0, 4096, 0, 0, 857, 4096, 0, 4002, 0, 375, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 4096, 4096, 356, 4094, 4096, 0, 0, 0, 28, 4068, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 4096, 4096, 4095, 0, 1710, 0, 4096, 0, 4085, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1083, 0, 4096, 4085, 0, 6, 4078, 0, 0, 0, 3529, 0, 0, 0, 4096, 4067, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1149, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 926, 4096, 0, 4096, 0, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 832, 4096, 4096, 0, 0, 0, 1, 3992, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 1643, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 310, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 2750, 0, 0, 0, 4096, 4094, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 1, 0, 2, 0, 0, 0, 4065, 148, 1, 153, 11, 1, 0, 0, 4095, 4095, 1, 0, 0, 4095, 0, 0, 2, 4093, 5, 1, 2028, 4095, 0, 4074, 6, 422, 1, 1, 23, 1, 0, 39, 0, 1, 0, 4, 0, 1, 1, 1, 3, 0, 0, 0, 7, 0, 4084, 4093, 198, 4094, 4095, 0, 0, 0, 3, 3700, 0, 2, 1, 0, 1, 0, 0, 4088, 13, 1, 0, 0, 2, 1, 4086, 4090, 4078, 0, 1581, 0, 4095, 0, 3958, 0, 4094, 15, 1, 0, 0, 4089, 0, 1, 4094, 4094, 0, 1, 4095, 0, 0, 0, 4094, 1, 1, 1, 0, 436, 1, 4095, 4049, 0, 0, 4095, 2, 0, 1, 3853, 1, 2, 0, 4094, 4095, 4024] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4069, 192, 1, 145, 13, 1, 4095, 4095, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2379, 4093, 1, 0, 0, 0, 0, 0, 2, 4091, 33, 0, 2148, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 7, 1094, 1, 1, 7, 1, 0, 19, 0, 1, 0, 6, 0, 1, 4095, 4094, 3025, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 0, 1, 0, 4, 0, 4078, 4095, 583, 4095, 3368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3697, 0, 1, 1, 19, 1, 0, 0, 4084, 9, 1, 0, 0, 0, 1, 3459, 1716, 1064, 924, 727, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 3165, 4071, 0, 735, 0, 0, 0, 3827, 1, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4095, 0, 1, 3631, 0, 0, 1, 4095, 1, 1, 1, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4066, 0, 0, 636, 1, 0, 1, 3873, 1, 2, 0, 0, 4095, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16337, 16842, 16747, 16229, 16168, 15895, 16158, 16696] avg=16384.0000 max=16842 max_rank=1 min=15895 min_rank=5 imbalance_ratio=1.027954 max_load_reduction=114230 max_load_reduction_pct=0.871506 moved_to_echo=64917 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 175, 0, 0, 0, 0, 4096, 4096, 3726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2379, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1919, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3030, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 395, 4096, 3369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3460, 1717, 1066, 926, 727, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3170, 4096, 0, 856, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3632, 0, 0, 0, 4096, 0, 0, 0, 0, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 636, 0, 0, 0, 3294, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16189, 16586, 16168, 16052, 16545, 16312, 17004, 16216] avg=16384.0000 max=17004 max_rank=6 min=16052 min_rank=3 imbalance_ratio=1.037842 max_load_reduction=114068 max_load_reduction_pct=0.870270 moved_to_echo=66077 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 131, 0, 0, 0, 0, 4096, 4096, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2379, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1191, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3029, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1683, 4096, 3369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3460, 1717, 1067, 926, 727, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3170, 4096, 0, 69, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3632, 0, 0, 0, 4096, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 636, 0, 0, 0, 3931, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16144, 15858, 16830, 17340, 16552, 15527, 15966, 16855] avg=16384.0000 max=17340 max_rank=3 min=15527 min_rank=5 imbalance_ratio=1.058350 max_load_reduction=113732 max_load_reduction_pct=0.867706 moved_to_echo=64900 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 133, 0, 0, 0, 0, 4096, 4096, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2379, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2561, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 230, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3030, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 4096, 1599, 4096, 3369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4076, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3458, 1717, 1066, 926, 727, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3170, 4096, 0, 750, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3632, 0, 0, 0, 4096, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 636, 0, 0, 0, 2952, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16146, 17228, 15830, 17258, 16543, 16201, 15990, 15876] avg=16384.0000 max=17258 max_rank=3 min=15830 min_rank=2 imbalance_ratio=1.053345 max_load_reduction=113814 max_load_reduction_pct=0.868332 moved_to_echo=64610 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 373, 0, 0, 0, 0, 4096, 4096, 3726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2375, 4096, 0, 0, 3, 0, 0, 14, 0, 4096, 0, 0, 857, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 375, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3030, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 4096, 4096, 356, 4094, 3368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4068, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 3448, 1721, 1066, 930, 728, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3166, 4095, 0, 1710, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3629, 0, 0, 0, 4096, 0, 0, 0, 0, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 6, 630, 0, 0, 0, 3529, 0, 0, 0, 0, 4067, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16395, 15537, 15978, 16030, 16565, 17154, 17000, 16413] avg=16384.0000 max=17154 max_rank=5 min=15537 min_rank=1 imbalance_ratio=1.046997 max_load_reduction=113918 max_load_reduction_pct=0.869125 moved_to_echo=67132 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 4078, 251, 1, 141, 12, 1, 4095, 4095, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2379, 4093, 1, 0, 0, 0, 0, 0, 1, 4091, 38, 1, 2030, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 6, 1346, 1, 1, 16, 1, 0, 13, 0, 1, 0, 4, 0, 1, 4095, 4094, 3026, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 0, 1, 0, 4, 0, 4080, 4092, 181, 4094, 3368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3601, 0, 2, 1, 27, 1, 0, 0, 4084, 11, 1, 0, 0, 0, 1, 3459, 1716, 1064, 924, 727, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 3165, 4072, 0, 857, 0, 0, 0, 3808, 1, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4095, 0, 1, 3631, 0, 0, 1, 4095, 0, 1, 1, 0, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4055, 0, 0, 636, 1, 0, 1, 3838, 1, 3, 0, 0, 4095, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16401, 16729, 16987, 15825, 16083, 15997, 16370, 16680] avg=16384.0000 max=16987 max_rank=2 min=15825 min_rank=3 imbalance_ratio=1.036804 max_load_reduction=114085 max_load_reduction_pct=0.870399 moved_to_echo=65130 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 1, 0, 2, 0, 0, 0, 4065, 148, 1, 153, 11, 1, 4095, 4095, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2379, 4095, 1, 0, 0, 0, 0, 0, 2, 4093, 5, 1, 2028, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 6, 422, 1, 1, 23, 1, 0, 39, 0, 1, 0, 4, 0, 1, 4094, 4094, 3025, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 0, 0, 0, 7, 0, 4084, 4093, 198, 4094, 3368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3700, 0, 2, 1, 0, 1, 0, 0, 4088, 13, 1, 0, 0, 2, 1, 3459, 1716, 1064, 924, 727, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 3166, 4078, 0, 1581, 0, 0, 0, 3958, 0, 0, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 4094, 0, 1, 3631, 0, 0, 0, 4094, 1, 1, 1, 0, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4049, 0, 0, 636, 2, 0, 1, 3853, 1, 2, 0, 0, 4095, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16298, 16699, 16157, 15849, 16166, 16885, 16354, 16664] avg=16384.0000 max=16885 max_rank=5 min=15849 min_rank=3 imbalance_ratio=1.030579 max_load_reduction=114187 max_load_reduction_pct=0.871178 moved_to_echo=66069 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1149, 0, 0, 0, 0, 4096, 4096, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2379, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 926, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3029, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 832, 4096, 3369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3992, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3460, 1717, 1067, 926, 727, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3170, 4096, 0, 1643, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3632, 0, 0, 0, 4096, 0, 0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 636, 0, 0, 0, 2750, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17162, 15593, 16375, 16489, 16450, 17101, 16230, 15672] avg=16384.0000 max=17162 max_rank=0 min=15593 min_rank=1 imbalance_ratio=1.047485 max_load_reduction=113910 max_load_reduction_pct=0.869064 moved_to_echo=65453 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 19072, 32768, 128, 0, 128, 0, 0, 128, 128, 32768, 128, 128, 13696, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 29056, 0, 0, 128, 32768, 128, 128, 128, 0, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 32640, 0, 128, 5120, 128, 0, 128, 28032, 128, 128, 0, 0, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 25344, 32768, 0, 8320, 0, 0, 0, 32128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 0, 32768, 32768, 5888, 32768, 27008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 128, 0, 128, 0, 0, 0, 32768, 2560, 128, 512, 128, 128, 32768, 32768, 29824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32384, 128, 5888, 128, 128, 128, 128, 0, 128, 0, 128, 0, 128, 0, 128, 32768, 32768, 24320, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 31360, 0, 128, 128, 128, 128, 0, 0, 32768, 128, 128, 128, 0, 128, 128, 27776, 13824, 8576, 7424, 5888, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2224, 2071, 1446, 0, 517, 0, 1886, 2179, 1569, 2373, 1954, 874, 1271, 2823, 0, 1166, 610, 1501, 1553, 1005, 43, 833, 187, 2122, 476, 297, 2772, 1893, 2078, 484, 1389, 812, 96, 1, 2554, 986, 451, 1894, 1124, 735, 222, 7, 2352, 2591, 0, 1325, 990, 1231, 997, 513, 904, 1116, 28, 117, 58, 1599, 1, 442, 56, 527, 1287, 1734, 1153, 396, 791, 8, 0, 1549, 1198, 2433, 1007, 1423, 1222, 1138, 1470, 786, 1756, 37, 1555, 1452, 1268, 810, 11, 47, 2081, 1097, 1896, 13, 2040, 601, 920, 712, 220, 1361, 560, 1, 1652, 951, 6, 2129, 47, 757, 1197, 970, 186, 1138, 116, 1434, 320, 620, 2596, 815, 1231, 1623, 463, 1170, 2524, 1192, 1358, 1247, 985, 1422, 1012, 812, 1272, 33, 361, 75] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1751, 1263, 1819, 1, 783, 1, 1833, 1369, 1244, 2360, 1497, 687, 1229, 1861, 0, 778, 950, 1311, 1579, 1583, 12, 679, 133, 1849, 457, 101, 3011, 1887, 1297, 76, 1392, 1129, 170, 2, 1492, 664, 497, 2321, 1566, 882, 307, 0, 1801, 2436, 0, 1553, 1303, 1273, 1390, 408, 628, 1139, 3, 18, 26, 1415, 0, 290, 25, 692, 1116, 2604, 1495, 667, 1076, 6, 0, 1630, 1864, 2076, 1367, 1380, 1367, 1788, 2140, 717, 1726, 7, 1400, 1630, 1065, 398, 18, 21, 2087, 531, 1152, 13, 2185, 246, 1033, 1031, 223, 1651, 906, 0, 687, 536, 11, 1772, 11, 559, 1104, 1084, 347, 1289, 386, 1443, 542, 1039, 2666, 1692, 1930, 1456, 616, 728, 2626, 1413, 1750, 1046, 1119, 2101, 1628, 669, 1182, 24, 770, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2009, 1581, 2032, 0, 494, 0, 1919, 1659, 1419, 2603, 1177, 517, 1135, 1821, 0, 698, 919, 1315, 1151, 1573, 2, 856, 161, 1639, 651, 67, 3355, 1579, 1591, 68, 1302, 1076, 371, 3, 1860, 412, 176, 1792, 1258, 870, 212, 15, 1520, 2387, 0, 1263, 1444, 1125, 1002, 340, 545, 1243, 22, 27, 39, 1453, 0, 581, 79, 452, 1100, 2655, 831, 721, 1083, 4, 0, 1609, 1991, 1900, 979, 1631, 1216, 1158, 2414, 646, 1050, 7, 1038, 2022, 1050, 545, 21, 19, 2355, 627, 1321, 9, 2562, 151, 1287, 807, 276, 1372, 1135, 1, 1342, 972, 0, 1891, 28, 821, 1302, 1071, 295, 1053, 305, 1229, 638, 936, 2672, 1826, 1829, 2072, 424, 891, 2799, 1633, 2288, 1377, 1046, 2179, 1198, 1005, 890, 27, 562, 18] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1763, 1611, 1474, 0, 514, 0, 2669, 512, 950, 3117, 2177, 446, 830, 2767, 0, 496, 649, 1974, 726, 1192, 17, 1096, 155, 1571, 499, 175, 3109, 1658, 846, 135, 1533, 1234, 262, 10, 2089, 733, 180, 2265, 900, 1071, 184, 2, 1866, 2586, 0, 1652, 1413, 885, 1197, 613, 482, 892, 69, 222, 1, 1530, 0, 353, 18, 1277, 1148, 3126, 1007, 681, 801, 6, 0, 1288, 2282, 1333, 1643, 875, 850, 1488, 2080, 370, 1289, 6, 1431, 1536, 912, 276, 9, 9, 2251, 974, 915, 9, 2141, 207, 1482, 883, 430, 2473, 798, 1, 1208, 680, 6, 1506, 16, 347, 1347, 957, 178, 1502, 356, 891, 833, 1425, 2425, 1950, 2150, 1732, 697, 533, 2323, 1672, 2062, 957, 1016, 2594, 1137, 555, 1532, 36, 786, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2204, 1275, 2090, 0, 564, 0, 1438, 1127, 1144, 1890, 1265, 1010, 1423, 1345, 0, 610, 514, 1700, 1485, 898, 9, 770, 150, 2740, 692, 76, 2728, 2095, 1542, 72, 1630, 1395, 428, 15, 2052, 893, 298, 1915, 915, 457, 202, 6, 1850, 2756, 6, 1624, 1263, 942, 1125, 635, 540, 1334, 21, 77, 34, 1479, 0, 609, 88, 996, 1344, 2548, 1559, 581, 1274, 8, 0, 1966, 1782, 1461, 1251, 1563, 841, 1756, 1367, 759, 1302, 15, 984, 1453, 1293, 766, 19, 24, 2110, 692, 1181, 75, 1687, 343, 1341, 898, 333, 1913, 622, 8, 1407, 938, 3, 2347, 42, 600, 1616, 1057, 280, 1493, 334, 1189, 666, 1267, 2352, 1404, 1723, 1795, 344, 1113, 2092, 1119, 1614, 1728, 1125, 1860, 1589, 800, 876, 75, 583, 81] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2358, 1447, 1203, 3, 367, 0, 1865, 986, 1482, 2666, 1708, 1141, 1011, 2615, 0, 1052, 842, 1064, 1475, 623, 24, 760, 207, 1862, 346, 76, 2923, 1690, 1108, 168, 1259, 1176, 102, 2, 2647, 760, 329, 2531, 882, 300, 339, 5, 1582, 2230, 0, 1566, 1258, 694, 1214, 792, 814, 1613, 39, 63, 17, 1161, 0, 624, 63, 812, 1478, 2136, 1417, 459, 713, 1, 0, 1586, 1637, 2092, 509, 800, 1094, 1818, 1987, 659, 1325, 25, 1645, 1839, 969, 291, 11, 10, 1850, 1038, 1264, 35, 2240, 304, 729, 776, 447, 1305, 648, 1, 1442, 1200, 2, 2110, 13, 822, 1753, 1565, 369, 1893, 174, 1397, 429, 1799, 2459, 1666, 1557, 1270, 582, 1545, 2314, 1333, 2107, 1390, 1044, 2315, 759, 843, 1171, 25, 639, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2630, 1935, 1552, 0, 570, 0, 1708, 1312, 1212, 2184, 1744, 654, 1070, 2215, 0, 886, 757, 1381, 1142, 814, 33, 793, 156, 1983, 441, 59, 2835, 2461, 1796, 217, 1340, 977, 96, 0, 2188, 941, 458, 2037, 802, 492, 183, 8, 1768, 2677, 0, 1501, 1414, 1274, 937, 667, 867, 1224, 34, 82, 6, 1388, 0, 492, 41, 812, 1368, 2354, 1404, 722, 901, 8, 0, 1598, 1722, 1891, 973, 918, 818, 1091, 1730, 359, 1415, 39, 1573, 2082, 1113, 455, 13, 13, 2213, 565, 1303, 19, 2320, 283, 1166, 637, 314, 1554, 801, 0, 1398, 1001, 6, 1759, 19, 634, 1403, 1360, 309, 1463, 505, 1318, 350, 1302, 2224, 1515, 2190, 1628, 355, 1391, 2187, 1081, 1926, 1456, 928, 2256, 1565, 615, 1122, 28, 775, 22] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2170, 1685, 1573, 0, 652, 3, 2193, 1130, 1392, 2387, 1708, 937, 924, 2161, 0, 862, 750, 1291, 688, 888, 62, 904, 138, 1826, 702, 52, 3073, 1676, 1804, 125, 1328, 846, 103, 0, 2070, 625, 206, 2983, 1533, 892, 312, 7, 1958, 2791, 1, 1329, 930, 1592, 1172, 582, 1068, 921, 9, 131, 8, 1197, 0, 811, 35, 446, 1333, 2083, 1524, 463, 529, 5, 0, 1918, 1969, 2361, 1467, 1252, 992, 1310, 1752, 272, 1757, 81, 1535, 2146, 1402, 330, 17, 4, 2106, 1078, 1083, 7, 2097, 418, 1720, 646, 353, 1578, 1008, 0, 1320, 579, 25, 1886, 36, 445, 1082, 1094, 188, 1512, 333, 1100, 408, 994, 2785, 1652, 1487, 1561, 533, 1113, 1625, 1422, 2140, 1061, 787, 1785, 1396, 648, 1091, 8, 705, 33] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1751, 1263, 1819, 1, 783, 1, 1833, 1369, 1244, 0, 1497, 687, 1229, 1301, 0, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 950, 1311, 1579, 1583, 12, 679, 133, 1849, 457, 101, 2286, 1887, 1297, 76, 1392, 1129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 2, 1492, 664, 497, 2321, 1566, 882, 307, 0, 1801, 2436, 0, 1553, 1303, 1273, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1390, 408, 628, 1139, 3, 18, 26, 1415, 0, 290, 25, 692, 1116, 2604, 1495, 667, 2360, 2143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 6, 0, 1630, 1864, 83, 1367, 1380, 1367, 1788, 2140, 717, 1726, 7, 1400, 1630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1065, 398, 18, 21, 2087, 531, 1152, 13, 2185, 246, 1033, 1031, 223, 1651, 906, 0, 1993, 725, 483, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, 536, 11, 1772, 11, 559, 1104, 1084, 347, 1289, 386, 1443, 542, 1039, 2666, 1692, 81, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1930, 1456, 616, 728, 0, 1413, 1750, 1046, 1119, 2020, 1628, 669, 1182, 24, 770, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15556, 16721, 16628, 16419, 18181, 15936, 15273, 16358] avg=16384.0000 max=18181 max_rank=4 min=15273 min_rank=6 imbalance_ratio=1.109680 max_load_reduction=112891 max_load_reduction_pct=0.861290 moved_to_echo=65748 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2224, 2071, 1446, 0, 517, 0, 1886, 2179, 1569, 0, 1954, 874, 1271, 1974, 0, 1166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 1501, 1553, 1005, 43, 833, 187, 2122, 476, 297, 2104, 1893, 2078, 484, 1389, 812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 1, 2554, 986, 451, 1894, 1124, 735, 222, 7, 2352, 2591, 0, 1325, 990, 1231, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 997, 513, 904, 1116, 28, 117, 58, 1599, 1, 442, 56, 527, 1287, 1734, 1153, 396, 2373, 2059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 791, 8, 0, 1549, 1198, 97, 1007, 1423, 1222, 1138, 1470, 786, 1756, 37, 1555, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1268, 810, 11, 47, 2081, 1097, 1896, 13, 2040, 601, 920, 712, 220, 1361, 560, 1, 2336, 668, 465, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1652, 951, 6, 2129, 47, 757, 1197, 970, 186, 1138, 116, 1434, 320, 620, 2596, 815, 55, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1231, 1623, 463, 1170, 0, 1192, 1358, 1247, 985, 1367, 1012, 812, 1272, 33, 361, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19131, 17387, 17107, 15360, 15489, 17372, 15025, 14201] avg=16384.0000 max=19131 max_rank=0 min=14201 min_rank=7 imbalance_ratio=1.167664 max_load_reduction=111941 max_load_reduction_pct=0.854042 moved_to_echo=62087 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2009, 1581, 2032, 0, 494, 0, 1919, 1659, 1419, 0, 1177, 517, 1135, 1274, 0, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 919, 1315, 1151, 1573, 2, 856, 161, 1639, 651, 67, 2547, 1579, 1591, 68, 1302, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 3, 1860, 412, 176, 1792, 1258, 870, 212, 15, 1520, 2387, 0, 1263, 1444, 1125, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 340, 545, 1243, 22, 27, 39, 1453, 0, 581, 79, 452, 1100, 2655, 831, 721, 2603, 2285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 4, 0, 1609, 1991, 76, 979, 1631, 1216, 1158, 2414, 646, 1050, 7, 1038, 2022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 545, 21, 19, 2355, 627, 1321, 9, 2562, 151, 1287, 807, 276, 1372, 1135, 1, 1824, 808, 514, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1342, 972, 0, 1891, 28, 821, 1302, 1071, 295, 1053, 305, 1229, 638, 936, 2672, 1826, 84, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1829, 2072, 424, 891, 0, 1633, 2288, 1377, 1046, 2095, 1198, 1005, 890, 27, 562, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15914, 16497, 15061, 15978, 16924, 16855, 16488, 17355] avg=16384.0000 max=17355 max_rank=7 min=15061 min_rank=2 imbalance_ratio=1.059265 max_load_reduction=113717 max_load_reduction_pct=0.867592 moved_to_echo=67622 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1763, 1611, 1474, 0, 514, 0, 2669, 512, 950, 0, 2177, 446, 830, 1935, 0, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 1974, 726, 1192, 17, 1096, 155, 1571, 499, 175, 2360, 1658, 846, 135, 1533, 1234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 10, 2089, 733, 180, 2265, 900, 1071, 184, 2, 1866, 2586, 0, 1652, 1413, 885, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1197, 613, 482, 892, 69, 222, 1, 1530, 0, 353, 18, 1277, 1148, 3126, 1007, 681, 3117, 1895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 801, 6, 0, 1288, 2282, 53, 1643, 875, 850, 1488, 2080, 370, 1289, 6, 1431, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 912, 276, 9, 9, 2251, 974, 915, 9, 2141, 207, 1482, 883, 430, 2473, 798, 1, 1280, 749, 428, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1208, 680, 6, 1506, 16, 347, 1347, 957, 178, 1502, 356, 891, 833, 1425, 2425, 1950, 100, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2150, 1732, 697, 533, 0, 1672, 2062, 957, 1016, 2494, 1137, 555, 1532, 36, 786, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15377, 15820, 16635, 17628, 15998, 16487, 15762, 17365] avg=16384.0000 max=17628 max_rank=3 min=15377 min_rank=0 imbalance_ratio=1.075928 max_load_reduction=113444 max_load_reduction_pct=0.865509 moved_to_echo=65612 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2358, 1447, 1203, 3, 367, 0, 1865, 986, 1482, 0, 1708, 1141, 1011, 1829, 0, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 842, 1064, 1475, 623, 24, 760, 207, 1862, 346, 76, 2219, 1690, 1108, 168, 1259, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 2, 2647, 760, 329, 2531, 882, 300, 339, 5, 1582, 2230, 0, 1566, 1258, 694, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1214, 792, 814, 1613, 39, 63, 17, 1161, 0, 624, 63, 812, 1478, 2136, 1417, 459, 2666, 1888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 713, 1, 0, 1586, 1637, 84, 509, 800, 1094, 1818, 1987, 659, 1325, 25, 1645, 1839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 969, 291, 11, 10, 1850, 1038, 1264, 35, 2240, 304, 729, 776, 447, 1305, 648, 1, 2008, 704, 426, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1442, 1200, 2, 2110, 13, 822, 1753, 1565, 369, 1893, 174, 1397, 429, 1799, 2459, 1666, 89, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1557, 1270, 582, 1545, 0, 1333, 2107, 1390, 1044, 2226, 759, 843, 1171, 25, 639, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16452, 14899, 15734, 17256, 15722, 15302, 19215, 16492] avg=16384.0000 max=19215 max_rank=6 min=14899 min_rank=1 imbalance_ratio=1.172791 max_load_reduction=111857 max_load_reduction_pct=0.853401 moved_to_echo=66731 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2204, 1275, 2090, 0, 564, 0, 1438, 1127, 1144, 0, 1265, 1010, 1423, 929, 0, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 1700, 1485, 898, 9, 770, 150, 2740, 692, 76, 2065, 2095, 1542, 72, 1630, 1395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 15, 2052, 893, 298, 1915, 915, 457, 202, 6, 1850, 2756, 6, 1624, 1263, 942, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1125, 635, 540, 1334, 21, 77, 34, 1479, 0, 609, 88, 996, 1344, 2548, 1559, 581, 1890, 1708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1274, 8, 0, 1966, 1782, 55, 1251, 1563, 841, 1756, 1367, 759, 1302, 15, 984, 1453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1293, 766, 19, 24, 2110, 692, 1181, 75, 1687, 343, 1341, 898, 333, 1913, 622, 8, 1406, 663, 384, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1407, 938, 3, 2347, 42, 600, 1616, 1057, 280, 1493, 334, 1189, 666, 1267, 2352, 1404, 76, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1723, 1795, 344, 1113, 0, 1119, 1614, 1728, 1125, 1784, 1589, 800, 876, 75, 583, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15079, 17833, 15887, 16568, 16376, 15887, 17093, 16349] avg=16384.0000 max=17833 max_rank=1 min=15079 min_rank=0 imbalance_ratio=1.088440 max_load_reduction=113239 max_load_reduction_pct=0.863945 moved_to_echo=65705 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2630, 1935, 1552, 0, 570, 0, 1708, 1312, 1212, 0, 1744, 654, 1070, 1549, 0, 886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 1381, 1142, 814, 33, 793, 156, 1983, 441, 59, 2152, 2461, 1796, 217, 1340, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 2188, 941, 458, 2037, 802, 492, 183, 8, 1768, 2677, 0, 1501, 1414, 1274, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 937, 667, 867, 1224, 34, 82, 6, 1388, 0, 492, 41, 812, 1368, 2354, 1404, 722, 2184, 1784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 901, 8, 0, 1598, 1722, 76, 973, 918, 818, 1091, 1730, 359, 1415, 39, 1573, 2082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1113, 455, 13, 13, 2213, 565, 1303, 19, 2320, 283, 1166, 637, 314, 1554, 801, 0, 1815, 683, 403, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1398, 1001, 6, 1759, 19, 634, 1403, 1360, 309, 1463, 505, 1318, 350, 1302, 2224, 1515, 87, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2190, 1628, 355, 1391, 0, 1081, 1926, 1456, 928, 2169, 1565, 615, 1122, 28, 775, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16822, 16502, 16269, 16366, 15303, 15878, 16681, 17251] avg=16384.0000 max=17251 max_rank=7 min=15303 min_rank=4 imbalance_ratio=1.052917 max_load_reduction=113821 max_load_reduction_pct=0.868385 moved_to_echo=65113 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2170, 1685, 1573, 0, 652, 3, 2193, 1130, 1392, 0, 1708, 937, 924, 1512, 0, 862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 750, 1291, 688, 888, 62, 904, 138, 1826, 702, 52, 2333, 1676, 1804, 125, 1328, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 2070, 625, 206, 2983, 1533, 892, 312, 7, 1958, 2791, 1, 1329, 930, 1592, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1172, 582, 1068, 921, 9, 131, 8, 1197, 0, 811, 35, 446, 1333, 2083, 1524, 463, 2387, 1327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 5, 0, 1918, 1969, 94, 1467, 1252, 992, 1310, 1752, 272, 1757, 81, 1535, 2146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 330, 17, 4, 2106, 1078, 1083, 7, 2097, 418, 1720, 646, 353, 1578, 1008, 0, 2267, 740, 298, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1320, 579, 25, 1886, 36, 445, 1082, 1094, 188, 1512, 333, 1100, 408, 994, 2785, 1652, 69, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1487, 1561, 533, 1113, 0, 1422, 2140, 1061, 787, 1716, 1396, 648, 1091, 8, 705, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16741, 15413, 17751, 15497, 17079, 17355, 15535, 15701] avg=16384.0000 max=17751 max_rank=2 min=15413 min_rank=1 imbalance_ratio=1.083435 max_load_reduction=113321 max_load_reduction_pct=0.864571 moved_to_echo=65670 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[6016, 11648, 9856, 8576, 256, 6784, 1408, 15616, 4352, 1024, 18176, 14976, 12160, 1408, 11264, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[14208, 13184, 4096, 8576, 0, 10880, 15360, 10368, 8064, 15872, 10368, 6016, 9216, 256, 5248, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[7168, 128, 0, 13184, 14464, 640, 9216, 9856, 8448, 11648, 14976, 4608, 11648, 256, 11264, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[17152, 12928, 13312, 128, 4480, 128, 15616, 10368, 10496, 0, 13312, 6272, 8960, 12416, 0, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[9088, 4608, 5888, 9600, 256, 768, 256, 11264, 128, 4224, 512, 6016, 10240, 19328, 10496, 4736, 19584, 15104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[10496, 6912, 128, 15488, 256, 4992, 10880, 9216, 2176, 11392, 2560, 10112, 4224, 9472, 20224, 12544, 768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1664, 128, 17024, 6016, 2688, 17792, 9088, 5760, 2048, 128, 14720, 20480, 128, 11904, 10112, 9088, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9088, 3968, 128, 256, 17152, 6656, 10240, 256, 17280, 2560, 9728, 6400, 2688, 13312, 6528, 128, 14976, 5760, 3456, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 29, 3112, 0, 1, 4, 1, 0, 0, 3, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 3934, 0, 3684, 0, 0, 3511, 0, 4092, 4096, 6, 0, 3, 4096, 0, 3167, 2, 426, 0, 0, 4063, 0, 0, 434, 0, 1116, 0, 2226, 1707, 3936, 0, 0, 4096, 3, 1669, 4096, 0, 0, 3852, 1708, 0, 0, 4092, 0, 3743, 3, 0, 0, 670, 3, 0, 4083, 0, 4036, 4080, 0, 0, 0, 0, 0, 0, 248, 0, 2838, 0, 0, 0, 259, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4094, 932, 355, 4084, 0, 1268, 4096, 0, 1805, 0, 0, 0, 4096, 4095, 2, 0, 43, 0, 404, 0, 0, 4087, 1735, 0, 460, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2217, 0, 63, 2, 0, 0, 0, 35, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 3994, 1, 3608, 0, 0, 2720, 0, 4071, 4096, 7, 0, 0, 4093, 0, 1966, 324, 809, 2, 0, 4083, 0, 0, 29, 0, 464, 0, 1959, 1771, 3858, 0, 0, 4095, 3, 2266, 4096, 0, 0, 3848, 614, 0, 0, 3983, 0, 3779, 4, 0, 0, 327, 17, 0, 4095, 0, 4087, 4094, 0, 0, 0, 0, 0, 0, 156, 0, 3650, 0, 0, 0, 518, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4094, 1275, 163, 4096, 0, 1423, 4096, 0, 2921, 0, 0, 0, 4096, 4096, 0, 0, 17, 0, 1222, 3, 0, 3814, 3463, 0, 108, 1, 0, 0, 4086] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2731, 0, 1, 1, 0, 0, 1, 10, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 4075, 0, 4032, 2, 0, 3645, 0, 4096, 4093, 1, 0, 0, 4096, 0, 2333, 52, 826, 0, 0, 3815, 0, 0, 149, 0, 139, 0, 905, 2596, 4050, 0, 0, 4096, 0, 1741, 4096, 0, 0, 4079, 1027, 0, 0, 4087, 0, 3728, 0, 0, 0, 1190, 0, 0, 4090, 0, 4064, 4038, 0, 0, 0, 0, 0, 0, 677, 0, 3823, 0, 0, 0, 305, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 606, 366, 4083, 0, 599, 4096, 0, 2871, 0, 0, 0, 4096, 4096, 0, 0, 70, 0, 608, 0, 0, 4078, 2094, 0, 241, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2523, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4069, 8, 3706, 5, 0, 3023, 0, 4096, 4096, 3, 0, 2, 4096, 0, 2126, 23, 2413, 0, 0, 4055, 0, 0, 445, 0, 232, 0, 1168, 2057, 3859, 0, 0, 4096, 0, 1266, 4096, 0, 0, 4013, 1030, 0, 0, 4015, 0, 3786, 0, 0, 0, 417, 0, 0, 4095, 0, 4039, 4090, 0, 0, 0, 0, 0, 2, 38, 0, 3878, 0, 0, 0, 406, 0, 0, 2, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 748, 270, 4089, 0, 160, 4096, 0, 3518, 0, 0, 0, 4096, 4096, 0, 0, 25, 0, 1582, 1, 0, 4079, 2516, 0, 43, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 4, 2940, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 4091, 1, 0, 4095, 1, 0, 4006, 10, 3960, 1, 0, 3434, 0, 4092, 4089, 39, 1, 0, 4095, 0, 3008, 73, 822, 0, 1, 3944, 0, 0, 11, 0, 369, 1, 3690, 1036, 3863, 1, 1, 4095, 26, 322, 4088, 2, 1, 3837, 651, 1, 0, 3889, 1, 3724, 0, 0, 0, 1209, 0, 0, 4095, 0, 4087, 4028, 0, 1, 0, 1, 2, 0, 124, 9, 3335, 1, 1, 0, 694, 0, 1, 1, 6, 1, 4095, 4, 0, 0, 0, 0, 4095, 433, 286, 3866, 1, 1252, 4082, 1, 2995, 0, 0, 5, 4095, 4094, 0, 1, 228, 1, 1444, 2, 1, 4078, 2009, 0, 11, 0, 0, 1, 4074] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 2628, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3982, 0, 4029, 0, 0, 1190, 0, 4088, 4095, 3, 0, 0, 4096, 0, 3916, 18, 687, 1, 0, 4090, 0, 0, 97, 0, 814, 0, 1157, 2847, 3860, 0, 0, 4096, 0, 1177, 4096, 0, 0, 4070, 1427, 0, 0, 3932, 0, 3934, 8, 0, 0, 290, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 162, 2, 4052, 0, 0, 0, 207, 0, 0, 0, 0, 0, 4096, 3, 1, 0, 0, 0, 4096, 2422, 170, 4089, 0, 656, 4096, 0, 2483, 0, 0, 0, 4096, 4096, 0, 0, 11, 0, 850, 0, 0, 4034, 1932, 0, 307, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3, 9, 2097, 0, 9, 2, 1, 1, 11, 0, 0, 1, 0, 4093, 0, 0, 4094, 0, 0, 3987, 4, 3954, 0, 0, 2571, 1, 4080, 4096, 0, 2, 0, 4092, 0, 2407, 25, 1302, 0, 0, 4089, 0, 0, 353, 0, 518, 0, 915, 2010, 3718, 0, 0, 4096, 7, 2029, 4096, 0, 0, 3642, 978, 0, 0, 4082, 0, 3911, 31, 0, 0, 544, 12, 0, 4091, 0, 4085, 4094, 0, 0, 0, 0, 0, 0, 652, 0, 3327, 0, 0, 2, 483, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 1, 4092, 1260, 104, 4091, 0, 631, 4094, 0, 3498, 4, 0, 0, 4092, 4096, 3, 0, 1, 0, 897, 6, 0, 4089, 3195, 0, 129, 0, 0, 0, 4089] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 12, 2167, 0, 15, 1, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4095, 0, 1, 3991, 0, 3913, 0, 0, 3020, 0, 4063, 4096, 11, 0, 0, 4095, 1, 1775, 117, 524, 4, 0, 4090, 1, 0, 123, 0, 559, 0, 1234, 975, 3834, 0, 0, 4095, 5, 2299, 4094, 0, 0, 4013, 932, 1, 0, 3846, 1, 3926, 4, 0, 0, 250, 0, 0, 4095, 0, 4090, 4094, 0, 0, 1, 0, 0, 0, 126, 0, 3772, 0, 0, 0, 432, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 1, 0, 4095, 2543, 113, 4093, 0, 393, 4096, 0, 3626, 0, 0, 1, 4096, 4096, 0, 0, 47, 0, 1884, 3, 0, 4070, 2560, 2, 277, 0, 1, 0, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 29, 3112, 0, 1, 4, 1, 0, 0, 3, 0, 0, 0, 4094, 4096, 4096, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 0, 3684, 0, 0, 3511, 0, 4092, 1533, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 3167, 2, 426, 0, 0, 4063, 0, 0, 434, 0, 1116, 0, 2226, 1146, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1707, 3936, 0, 0, 0, 3, 1669, 79, 0, 0, 3852, 1708, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3743, 3, 0, 0, 670, 3, 0, 3787, 0, 4036, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 2838, 0, 0, 0, 259, 0, 0, 0, 0, 0, 4096, 6, 0, 4017, 2580, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2959, 932, 355, 4084, 0, 1268, 4096, 0, 1805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 43, 0, 404, 0, 0, 4087, 1735, 0, 460, 0, 0, 0, 4096, 296, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16952, 16760, 17760, 17046, 16322, 15461, 15499, 15272] avg=16384.0000 max=17760 max_rank=2 min=15272 min_rank=7 imbalance_ratio=1.083984 max_load_reduction=113312 max_load_reduction_pct=0.864502 moved_to_echo=62554 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2217, 0, 63, 2, 0, 0, 0, 35, 1, 0, 0, 4096, 4095, 4096, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 1, 3608, 0, 0, 2720, 0, 4071, 1533, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1966, 324, 809, 2, 0, 4083, 0, 0, 29, 0, 464, 0, 1959, 1146, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1771, 3858, 0, 0, 0, 3, 2266, 79, 0, 0, 3848, 614, 0, 0, 3983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3779, 4, 0, 0, 327, 17, 0, 3798, 0, 4087, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 3650, 0, 0, 0, 518, 0, 0, 0, 0, 0, 4096, 4, 0, 4017, 2579, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2959, 1275, 163, 4096, 0, 1423, 4096, 0, 2921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 1222, 3, 0, 3814, 3463, 0, 108, 1, 0, 0, 4086, 297, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16123, 15934, 15956, 16422, 16106, 16437, 16933, 17161] avg=16384.0000 max=17161 max_rank=7 min=15934 min_rank=1 imbalance_ratio=1.047424 max_load_reduction=113911 max_load_reduction_pct=0.869072 moved_to_echo=66637 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2731, 0, 1, 1, 0, 0, 1, 10, 0, 0, 0, 4093, 4096, 4096, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4032, 2, 0, 3645, 0, 4096, 1532, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2333, 52, 826, 0, 0, 3815, 0, 0, 149, 0, 139, 0, 905, 1145, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2596, 4050, 0, 0, 0, 0, 1741, 79, 0, 0, 4079, 1027, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3728, 0, 0, 0, 1190, 0, 0, 3793, 0, 4064, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 677, 0, 3823, 0, 0, 0, 305, 0, 0, 0, 0, 0, 4096, 1, 0, 4017, 2579, 1416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2961, 606, 366, 4083, 0, 599, 4096, 0, 2871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 70, 0, 608, 0, 0, 4078, 2094, 0, 241, 0, 0, 0, 4095, 297, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16547, 17383, 14541, 17659, 16813, 16914, 15582, 15633] avg=16384.0000 max=17659 max_rank=3 min=14541 min_rank=2 imbalance_ratio=1.077820 max_load_reduction=113413 max_load_reduction_pct=0.865273 moved_to_echo=64942 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 2523, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4096, 4096, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 8, 3706, 5, 0, 3023, 0, 4096, 1533, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 2126, 23, 2413, 0, 0, 4055, 0, 0, 445, 0, 232, 0, 1168, 1146, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2057, 3859, 0, 0, 0, 0, 1266, 79, 0, 0, 4013, 1030, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3786, 0, 0, 0, 417, 0, 0, 3798, 0, 4039, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 38, 0, 3878, 0, 0, 0, 406, 0, 0, 2, 0, 0, 4093, 0, 0, 4017, 2580, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2961, 748, 270, 4089, 0, 160, 4096, 0, 3518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 25, 0, 1582, 1, 0, 4079, 2516, 0, 43, 0, 0, 0, 4095, 297, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16330, 16443, 16787, 16319, 16130, 16433, 15842, 16788] avg=16384.0000 max=16788 max_rank=7 min=15842 min_rank=6 imbalance_ratio=1.024658 max_load_reduction=114284 max_load_reduction_pct=0.871918 moved_to_echo=65193 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4, 2940, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 4091, 4095, 4095, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4006, 10, 3960, 1, 0, 3434, 0, 4092, 1519, 39, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3008, 73, 822, 0, 1, 3944, 0, 0, 11, 0, 369, 1, 3690, 1150, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 3863, 1, 1, 0, 26, 322, 73, 2, 1, 3837, 651, 1, 0, 3889, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3724, 0, 0, 0, 1209, 0, 0, 3792, 0, 4087, 4028, 0, 1, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 9, 3335, 1, 1, 0, 694, 0, 1, 1, 6, 1, 4095, 4, 0, 4015, 2578, 1420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2950, 433, 286, 3866, 1, 1252, 4082, 1, 2995, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 228, 1, 1444, 2, 1, 4078, 2009, 0, 11, 0, 0, 1, 4074, 303, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16748, 17064, 18248, 13704, 16844, 16285, 15871, 16308] avg=16384.0000 max=18248 max_rank=2 min=13704 min_rank=3 imbalance_ratio=1.113770 max_load_reduction=112824 max_load_reduction_pct=0.860779 moved_to_echo=65308 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 2628, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 4096, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 0, 4029, 0, 0, 1190, 0, 4088, 1532, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3916, 18, 687, 1, 0, 4090, 0, 0, 97, 0, 814, 0, 1157, 1146, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2847, 3860, 0, 0, 0, 0, 1177, 79, 0, 0, 4070, 1427, 0, 0, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 8, 0, 0, 290, 0, 0, 3799, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 2, 4052, 0, 0, 0, 207, 0, 0, 0, 0, 0, 4096, 3, 1, 4017, 2580, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2961, 2422, 170, 4089, 0, 656, 4096, 0, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 850, 0, 0, 4034, 1932, 0, 307, 0, 0, 0, 4095, 297, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16440, 14824, 17103, 17392, 16223, 16537, 16877, 15676] avg=16384.0000 max=17392 max_rank=3 min=14824 min_rank=1 imbalance_ratio=1.061523 max_load_reduction=113680 max_load_reduction_pct=0.867310 moved_to_echo=65313 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 12, 2167, 0, 15, 1, 0, 0, 0, 1, 0, 0, 0, 4096, 4095, 4095, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3991, 0, 3913, 0, 0, 3020, 0, 4063, 1533, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1775, 117, 524, 4, 0, 4090, 1, 0, 123, 0, 559, 0, 1234, 1146, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 975, 3834, 0, 0, 0, 5, 2299, 79, 0, 0, 4013, 932, 1, 0, 3846, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 4, 0, 0, 250, 0, 0, 3798, 0, 4090, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 3772, 0, 0, 0, 432, 0, 0, 0, 0, 0, 4095, 1, 0, 4015, 2580, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 2960, 2543, 113, 4093, 0, 393, 4096, 0, 3626, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 47, 0, 1884, 3, 0, 4070, 2560, 2, 277, 0, 1, 0, 4087, 297, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15999, 16532, 14750, 15985, 16163, 16438, 17827, 17378] avg=16384.0000 max=17827 max_rank=6 min=14750 min_rank=2 imbalance_ratio=1.088074 max_load_reduction=113245 max_load_reduction_pct=0.863991 moved_to_echo=67806 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 9, 2097, 0, 9, 2, 1, 1, 11, 0, 0, 1, 0, 4093, 4096, 4094, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 4, 3954, 0, 0, 2571, 1, 4080, 1533, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2407, 25, 1302, 0, 0, 4089, 0, 0, 353, 0, 518, 0, 915, 1146, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2010, 3718, 0, 0, 0, 7, 2029, 79, 0, 0, 3642, 978, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3911, 31, 0, 0, 544, 12, 0, 3794, 0, 4085, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 652, 0, 3327, 0, 0, 2, 483, 0, 0, 0, 0, 0, 4093, 0, 0, 4017, 2576, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2958, 1260, 104, 4091, 0, 631, 4094, 0, 3498, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 1, 0, 897, 6, 0, 4089, 3195, 0, 129, 0, 0, 0, 4089, 297, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15933, 16132, 15927, 16545, 16471, 16567, 16641, 16856] avg=16384.0000 max=16856 max_rank=7 min=15927 min_rank=2 imbalance_ratio=1.028809 max_load_reduction=114216 max_load_reduction_pct=0.871399 moved_to_echo=66535 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[15104, 31104, 128, 128, 0, 128, 12800, 640, 128, 128, 31360, 8448, 128, 0, 32000, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[30592, 128, 0, 0, 4992, 128, 0, 30464, 0, 32640, 32640, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 23680, 10240, 1920, 32512, 128, 6400, 32768, 128, 23808, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 20480, 0, 128, 128, 128, 128, 128, 128, 128, 128, 0, 32768, 32768, 32768, 12160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 0, 128, 128, 32128, 128, 30976, 128, 0, 23168, 128, 32768, 12288, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 20736, 640, 7936, 128, 128, 32256, 128, 0, 1664, 0, 4224, 128, 13312, 9216, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 128, 128, 512, 128, 8960, 128, 128, 32384, 19584, 128, 1664, 128, 128, 128, 32768, 2432, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132992 observed_loss=-1920 tokens_per_expert=[128, 2304, 128, 28800, 128, 128, 128, 3328, 0, 128, 128, 128, 128, 32768, 128, 128, 32256, 20736, 11392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132992 slack_slots=7040 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 806, 0, 506, 0, 0, 2171, 0, 4090, 1200, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 1628, 0, 4091, 0, 4096, 4088, 1, 0, 0, 4095, 930, 0, 3, 1, 4096, 10, 0, 4096, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 4092, 4088, 998, 0, 0, 4016, 0, 4096, 3907, 0, 0, 4036, 1286, 0, 0, 10, 4096, 0, 0, 4089, 60, 0, 0, 0, 0, 1, 4095, 0, 0, 4096, 9, 256, 2759, 4096, 0, 0, 3634, 4001, 0, 2584, 0, 1017, 0, 0, 2772, 0, 0, 0, 14, 5, 32, 0, 0, 2809, 0, 4086, 0, 0, 0, 0, 6, 0, 0, 77, 4086, 0, 0, 5, 0, 0, 0, 3517, 0, 4096, 0, 4092, 31, 0, 5, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 768, 0, 696, 0, 0, 3171, 0, 4094, 2204, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1733, 1, 4094, 0, 4096, 4095, 0, 0, 0, 4095, 343, 0, 1, 0, 4094, 1, 2, 4095, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 4093, 1118, 0, 0, 4024, 0, 4096, 4085, 0, 0, 4071, 1719, 0, 0, 4, 4096, 0, 0, 4094, 3, 0, 0, 0, 0, 1, 4096, 1, 0, 4096, 25, 130, 1523, 4096, 0, 0, 3490, 3832, 0, 2741, 0, 135, 1, 0, 2472, 0, 0, 0, 37, 1, 5, 0, 0, 2905, 0, 4068, 1, 0, 0, 0, 1, 0, 0, 28, 4095, 0, 0, 1, 0, 0, 0, 3809, 0, 4093, 0, 4093, 15, 0, 1, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 730, 0, 65, 0, 0, 1700, 0, 4096, 1856, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 589, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 1503, 0, 0, 0, 4096, 1, 0, 4096, 5, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 4095, 265, 0, 0, 4027, 0, 4096, 4096, 0, 0, 4070, 1411, 0, 0, 0, 4096, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 459, 2446, 4091, 0, 0, 3794, 4085, 0, 3723, 0, 1581, 0, 0, 3033, 0, 0, 0, 105, 0, 102, 0, 0, 1398, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 0, 0, 0, 0, 0, 0, 4031, 0, 4064, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1248, 0, 271, 0, 0, 2601, 0, 4096, 1163, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1321, 0, 4094, 0, 4096, 4095, 0, 0, 0, 4096, 758, 0, 2, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 4092, 4096, 1493, 0, 0, 4085, 0, 4093, 4082, 0, 0, 4065, 2231, 0, 0, 6, 4094, 0, 0, 4096, 8, 0, 0, 0, 0, 1, 4096, 1, 0, 4096, 1, 427, 3669, 4035, 0, 0, 4029, 4055, 0, 1971, 0, 391, 0, 0, 2062, 0, 0, 0, 12, 0, 69, 0, 0, 1037, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 104, 4096, 0, 0, 0, 0, 0, 0, 3966, 0, 4077, 0, 4094, 4, 0, 0, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1367, 0, 937, 0, 0, 3546, 0, 4096, 685, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 810, 0, 4090, 0, 4096, 4096, 0, 0, 0, 4096, 611, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 7, 0, 0, 4081, 4095, 866, 0, 0, 4081, 0, 4096, 4096, 0, 0, 4024, 1313, 0, 0, 0, 4094, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 4096, 1, 0, 4095, 8, 110, 3590, 4096, 0, 0, 3458, 4046, 0, 2366, 0, 330, 0, 0, 2579, 0, 0, 0, 108, 0, 17, 0, 0, 2676, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3517, 0, 4064, 0, 4069, 0, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 897, 0, 84, 1, 1, 1273, 0, 4093, 1707, 0, 1, 1, 0, 4095, 0, 1, 0, 0, 1205, 0, 4094, 1, 4094, 4095, 1, 0, 2, 4091, 1019, 1, 6, 0, 4095, 5, 0, 4095, 2, 0, 0, 0, 4095, 1, 27, 1, 0, 4089, 4094, 535, 1, 0, 4030, 0, 4092, 4087, 0, 1, 3959, 2359, 1, 1, 8, 4095, 0, 1, 4089, 16, 0, 0, 0, 0, 1, 4095, 1, 1, 4095, 0, 448, 2287, 4088, 2, 1, 3607, 4040, 0, 3411, 1, 1099, 0, 0, 3634, 1, 0, 0, 42, 0, 264, 0, 1, 1818, 0, 4093, 0, 1, 0, 1, 1, 1, 0, 48, 4094, 1, 1, 13, 1, 1, 1, 3433, 1, 3848, 0, 4014, 23, 0, 4, 0, 12] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 756, 0, 968, 0, 0, 3332, 0, 4094, 387, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 814, 0, 4095, 0, 4095, 4095, 0, 0, 0, 4095, 2240, 0, 2, 1, 4096, 1, 0, 4096, 1, 0, 0, 0, 4095, 0, 0, 0, 1, 4090, 4094, 286, 0, 0, 4076, 0, 4095, 4045, 1, 0, 4074, 2869, 0, 0, 2, 4095, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 4096, 3, 0, 4095, 7, 112, 3139, 4095, 0, 0, 3749, 3972, 0, 1977, 0, 28, 0, 0, 2417, 0, 0, 0, 0, 1, 0, 1, 0, 1921, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 8, 4095, 0, 0, 1, 0, 0, 1, 3963, 0, 4094, 1, 4094, 15, 2, 1, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 93, 0, 356, 0, 0, 2512, 0, 4096, 1683, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 983, 0, 4096, 0, 4096, 4096, 0, 0, 1, 4094, 886, 0, 0, 0, 4096, 3, 1, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4089, 4094, 1242, 0, 0, 4089, 0, 4096, 4096, 0, 0, 4014, 2588, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 10, 180, 2803, 4083, 0, 0, 3792, 4066, 0, 2631, 0, 1592, 0, 0, 2600, 0, 0, 0, 104, 0, 7, 0, 0, 736, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 107, 4096, 0, 0, 3, 0, 0, 0, 3907, 0, 4082, 0, 4093, 7, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 768, 0, 696, 0, 0, 3171, 0, 4094, 2204, 0, 0, 0, 0, 4096, 2167, 579, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1733, 1, 4094, 0, 1929, 4095, 0, 0, 0, 4095, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 1, 2, 4088, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1403, 1118, 0, 0, 4024, 0, 0, 4085, 0, 0, 4071, 1719, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 1, 4096, 1, 0, 4096, 25, 130, 1523, 1029, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2488, 0, 0, 3490, 3832, 0, 2741, 0, 135, 1, 0, 2472, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 2905, 0, 4068, 1, 0, 0, 0, 1, 0, 0, 28, 4095, 4096, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3809, 0, 4093, 0, 4093, 15, 0, 1, 0, 4, 2460, 2002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18005, 16290, 16380, 16424, 15005, 15196, 17294, 16478] avg=16384.0000 max=18005 max_rank=0 min=15005 min_rank=4 imbalance_ratio=1.098938 max_load_reduction=113067 max_load_reduction_pct=0.862633 moved_to_echo=63973 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 730, 0, 65, 0, 0, 1700, 0, 4096, 1856, 0, 0, 0, 0, 4096, 2167, 578, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 4096, 0, 1929, 4096, 0, 0, 0, 4096, 1503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4089, 5, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1404, 265, 0, 0, 4027, 0, 0, 4096, 0, 0, 4070, 1411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 459, 2446, 1027, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2486, 0, 0, 3794, 4085, 0, 3723, 0, 1581, 0, 0, 3033, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 1398, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 4096, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 4064, 0, 4096, 0, 0, 0, 0, 0, 2461, 2002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15518, 16309, 16384, 15273, 16229, 18807, 15898, 16654] avg=16384.0000 max=18807 max_rank=5 min=15273 min_rank=3 imbalance_ratio=1.147888 max_load_reduction=112265 max_load_reduction_pct=0.856514 moved_to_echo=67588 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1367, 0, 937, 0, 0, 3546, 0, 4096, 685, 0, 0, 0, 0, 4096, 2167, 579, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 810, 0, 4090, 0, 1929, 4096, 0, 0, 0, 4096, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 0, 0, 0, 1, 4096, 0, 7, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1404, 866, 0, 0, 4081, 0, 0, 4096, 0, 0, 4024, 1313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 4096, 1, 0, 4095, 8, 110, 3590, 1029, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2488, 0, 0, 3458, 4046, 0, 2366, 0, 330, 0, 0, 2579, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 2676, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3517, 0, 4064, 0, 4069, 0, 0, 0, 0, 1, 2461, 2002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17703, 15632, 16370, 15784, 17046, 15375, 17048, 16114] avg=16384.0000 max=17703 max_rank=0 min=15375 min_rank=5 imbalance_ratio=1.080505 max_load_reduction=113369 max_load_reduction_pct=0.864937 moved_to_echo=65583 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 756, 0, 968, 0, 0, 3332, 0, 4094, 387, 0, 0, 1, 0, 4095, 2167, 579, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 814, 0, 4095, 0, 1928, 4095, 0, 0, 0, 4095, 2240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4096, 1, 0, 4089, 1, 0, 0, 0, 4095, 0, 0, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1404, 286, 0, 0, 4076, 0, 0, 4045, 1, 0, 4074, 2869, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 4096, 3, 0, 4095, 7, 112, 3139, 1028, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2488, 0, 0, 3749, 3972, 0, 1977, 0, 28, 0, 0, 2417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1921, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 8, 4095, 4095, 2093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 3963, 0, 4094, 1, 4094, 15, 2, 1, 0, 3, 2460, 2002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16610, 17267, 16376, 16757, 16583, 14631, 16211, 16637] avg=16384.0000 max=17267 max_rank=1 min=14631 min_rank=5 imbalance_ratio=1.053894 max_load_reduction=113805 max_load_reduction_pct=0.868263 moved_to_echo=64062 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 806, 0, 506, 0, 0, 2171, 0, 4090, 1200, 0, 0, 0, 0, 4096, 2167, 579, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1628, 0, 4091, 0, 1929, 4088, 1, 0, 0, 4095, 930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 4096, 10, 0, 4089, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 998, 0, 0, 4016, 0, 0, 3907, 0, 0, 4036, 1286, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 60, 0, 0, 0, 0, 1, 4095, 0, 0, 4096, 9, 256, 2759, 1029, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2488, 0, 0, 3634, 4001, 0, 2584, 0, 1017, 0, 0, 2772, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 32, 0, 0, 2809, 0, 4086, 0, 0, 0, 0, 6, 0, 0, 77, 4086, 4096, 2095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3517, 0, 4096, 0, 4092, 31, 0, 5, 0, 5, 2457, 2001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15844, 16764, 16397, 15655, 16401, 16510, 17292, 16209] avg=16384.0000 max=17292 max_rank=6 min=15655 min_rank=3 imbalance_ratio=1.055420 max_load_reduction=113780 max_load_reduction_pct=0.868073 moved_to_echo=66412 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1248, 0, 271, 0, 0, 2601, 0, 4096, 1163, 0, 0, 0, 0, 4096, 2167, 570, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1321, 0, 4094, 0, 1929, 4095, 0, 0, 0, 4096, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 2, 0, 4089, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1405, 1493, 0, 0, 4085, 0, 0, 4082, 0, 0, 4065, 2231, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 1, 4096, 1, 0, 4096, 1, 427, 3669, 1013, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2452, 0, 0, 4029, 4055, 0, 1971, 0, 391, 0, 0, 2062, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 1037, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 104, 4096, 4094, 2092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 4077, 0, 4094, 4, 0, 0, 0, 5, 2461, 2001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16442, 16293, 16387, 17367, 17415, 14972, 15588, 16608] avg=16384.0000 max=17415 max_rank=4 min=14972 min_rank=5 imbalance_ratio=1.062927 max_load_reduction=113657 max_load_reduction_pct=0.867134 moved_to_echo=64583 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 897, 0, 84, 1, 1, 1273, 0, 4093, 1707, 0, 1, 1, 0, 4095, 2172, 581, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1205, 0, 4094, 1, 1922, 4095, 1, 0, 2, 4091, 1019, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4095, 5, 0, 4082, 2, 0, 0, 0, 4095, 1, 27, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1397, 535, 1, 0, 4030, 0, 0, 4087, 0, 1, 3959, 2359, 1, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 16, 0, 0, 0, 0, 1, 4095, 1, 1, 4095, 0, 448, 2287, 1030, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2477, 2, 1, 3607, 4040, 0, 3411, 1, 1099, 0, 0, 3634, 1, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 1, 1818, 0, 4093, 0, 1, 0, 1, 1, 1, 0, 48, 4094, 4095, 2092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 13, 1, 1, 1, 3433, 1, 3848, 0, 4014, 23, 0, 4, 0, 12, 2464, 2000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15140, 16432, 16403, 16379, 16077, 18315, 16509, 15817] avg=16384.0000 max=18315 max_rank=5 min=15140 min_rank=0 imbalance_ratio=1.117859 max_load_reduction=112757 max_load_reduction_pct=0.860268 moved_to_echo=66718 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 93, 0, 356, 0, 0, 2512, 0, 4096, 1683, 0, 0, 0, 0, 4096, 2167, 577, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 983, 0, 4096, 0, 1929, 4096, 0, 0, 1, 4094, 886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 1, 4089, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1404, 1242, 0, 0, 4089, 0, 0, 4096, 0, 0, 4014, 2588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 10, 180, 2803, 1025, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2481, 0, 0, 3792, 4066, 0, 2631, 0, 1592, 0, 0, 2600, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 736, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 107, 4096, 4096, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3907, 0, 4082, 0, 4093, 7, 0, 1, 0, 0, 2460, 2002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15810, 16085, 16375, 17433, 16316, 17266, 15232, 16555] avg=16384.0000 max=17433 max_rank=3 min=15232 min_rank=6 imbalance_ratio=1.064026 max_load_reduction=113639 max_load_reduction_pct=0.866997 moved_to_echo=65369 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 6784, 0, 3968, 128, 128, 20352, 0, 32768, 11008, 0, 128, 128, 0, 32768, 17408, 4736, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 512, 128, 128, 15360, 0, 32640, 128, 128, 0, 128, 128, 128, 0, 512, 32768, 32768, 16768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 0, 9088, 128, 32768, 128, 15488, 32768, 128, 0, 128, 32768, 8320, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 128, 128, 30208, 128, 32512, 128, 32768, 128, 128, 128, 0, 128, 19712, 16128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[11264, 6912, 128, 0, 32512, 0, 0, 32512, 128, 128, 32384, 15872, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 32768, 128, 128, 32768, 128, 0, 0, 128, 32768, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0, 128, 32768, 128, 128, 32768, 128, 2176, 22272, 8320, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[19968, 128, 128, 29568, 32128, 0, 21504, 128, 6272, 128, 0, 21632, 128, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 11, 4096, 0, 0, 0, 4093, 4096, 0, 0, 4053, 1676, 0, 83, 0, 0, 22, 0, 0, 18, 4094, 0, 3994, 4059, 0, 0, 4096, 0, 0, 4096, 4086, 0, 0, 0, 0, 4096, 0, 1390, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 1, 0, 0, 4096, 643, 0, 0, 0, 0, 1, 3701, 3, 0, 4096, 0, 0, 0, 5, 4095, 0, 0, 4096, 4090, 0, 0, 0, 2097, 0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 461, 0, 1, 0, 0, 29, 0, 1, 2647, 4090, 0, 0, 4041, 0, 0, 3943, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 11, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 3989, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 4094, 1, 0, 50, 4094, 4096, 0, 0, 3706, 2198, 0, 42, 32, 0, 3, 0, 0, 20, 4090, 0, 3745, 4024, 0, 0, 4094, 0, 0, 4094, 4096, 0, 1, 0, 0, 4093, 0, 718, 4096, 4, 51, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 859, 0, 0, 0, 0, 0, 3638, 9, 0, 4096, 0, 0, 0, 1, 4091, 0, 0, 4096, 4083, 0, 0, 0, 3585, 0, 1, 0, 4096, 0, 4090, 4095, 0, 0, 401, 0, 10, 0, 0, 14, 0, 81, 2195, 4088, 0, 0, 4052, 0, 0, 3969, 0, 3, 0, 0, 4096, 1, 0, 0, 0, 37, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3447, 0, 11, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 13, 4094, 3, 0, 0, 4077, 4096, 0, 0, 3677, 1645, 0, 0, 0, 0, 19, 0, 0, 69, 4074, 0, 3936, 4051, 0, 0, 4090, 0, 0, 4096, 4077, 0, 0, 0, 0, 4095, 1, 1657, 4096, 13, 0, 0, 0, 6, 0, 4096, 4096, 0, 0, 0, 4096, 979, 1, 0, 0, 0, 0, 3322, 9, 0, 4096, 0, 0, 0, 2, 4085, 0, 0, 4094, 4075, 0, 0, 0, 3718, 2, 0, 0, 4096, 0, 4085, 4095, 0, 0, 218, 0, 17, 0, 0, 49, 0, 40, 2428, 4085, 0, 0, 4021, 0, 0, 3907, 0, 6, 0, 0, 4096, 5, 0, 0, 0, 57, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3104, 0, 16, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 35, 4091, 0, 0, 1, 4095, 4096, 0, 0, 3524, 2242, 0, 17, 0, 0, 1, 0, 0, 50, 4095, 0, 3770, 4087, 0, 0, 4096, 0, 0, 4096, 4095, 0, 1, 0, 0, 4096, 0, 2592, 4096, 10, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 0, 4096, 166, 0, 0, 0, 0, 1, 3913, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4081, 0, 0, 0, 1912, 0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 223, 0, 0, 0, 0, 14, 0, 25, 2105, 4083, 0, 0, 4029, 0, 0, 4039, 0, 0, 0, 0, 4096, 28, 0, 0, 0, 48, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 4063, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 3991, 1435, 0, 4, 2, 0, 27, 0, 0, 3, 4096, 0, 3988, 3981, 0, 0, 4096, 0, 0, 4096, 4083, 0, 0, 0, 0, 4096, 0, 1198, 4096, 5, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 183, 0, 0, 0, 0, 0, 3012, 5, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4087, 0, 0, 0, 3906, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 722, 0, 2, 0, 0, 10, 0, 0, 2797, 4095, 0, 0, 3902, 0, 0, 3553, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 2, 4096, 0, 0, 0, 63, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 52, 4091, 0, 0, 0, 4094, 4095, 0, 0, 4071, 505, 1, 10, 0, 1, 55, 1, 1, 14, 4095, 0, 3841, 3991, 0, 1, 4086, 0, 0, 4093, 4095, 0, 141, 1, 0, 4095, 3, 1552, 4095, 7, 0, 0, 0, 23, 1, 4095, 4095, 2, 1, 0, 4095, 207, 4, 0, 1, 1, 0, 3413, 0, 0, 4095, 0, 0, 0, 11, 4095, 1, 0, 4088, 4067, 1, 1, 1, 3247, 0, 0, 0, 4095, 0, 4088, 4094, 0, 0, 1900, 1, 4, 0, 0, 44, 1, 1, 2049, 4059, 1, 0, 4058, 0, 1, 3564, 1, 0, 1, 1, 4095, 59, 1, 0, 0, 27, 4093, 1, 0, 1, 3, 0, 0, 1, 0, 0, 4095, 1, 4094, 1, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 27, 4096, 0, 0, 2, 4096, 4096, 0, 0, 3777, 2584, 0, 2, 3, 0, 0, 0, 0, 14, 4096, 0, 3861, 4092, 0, 0, 4096, 0, 0, 4095, 4096, 0, 3, 0, 0, 4096, 0, 718, 4095, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 167, 0, 0, 0, 0, 1, 3934, 1, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 4096, 4077, 0, 0, 0, 3864, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 560, 0, 3, 0, 0, 7, 0, 3, 1098, 4095, 0, 2, 4017, 0, 0, 4051, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 29, 4095, 0, 0, 0, 23, 0, 0, 0, 0, 0, 4096, 0, 4018, 0, 15, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 4096, 0, 0, 3900, 2325, 0, 2, 6, 0, 15, 0, 0, 27, 4092, 1, 3748, 4079, 0, 0, 4094, 0, 0, 4094, 4091, 1, 0, 0, 0, 4095, 1, 1553, 4094, 1, 7, 0, 0, 1, 0, 4096, 4096, 1, 0, 0, 4096, 179, 1, 0, 0, 0, 5, 2516, 28, 0, 4096, 0, 2, 0, 1, 4095, 0, 0, 4096, 4092, 0, 0, 0, 3185, 0, 1, 0, 4096, 0, 4095, 4095, 0, 0, 751, 0, 2, 0, 0, 7, 0, 10, 2320, 4088, 0, 0, 4052, 0, 0, 4079, 0, 1, 0, 0, 4096, 18, 0, 0, 0, 18, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4095, 0, 4050, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 4094, 1, 0, 50, 4094, 2509, 0, 0, 3706, 2198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 32, 0, 3, 0, 0, 20, 4090, 0, 3745, 4024, 0, 0, 4094, 0, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 1, 0, 0, 4093, 0, 718, 2648, 4, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 233, 859, 0, 0, 0, 0, 0, 3638, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 4096, 4083, 0, 0, 0, 3585, 0, 1, 0, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1206, 0, 4090, 4095, 0, 0, 401, 0, 10, 0, 0, 14, 0, 81, 2195, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 3969, 0, 3, 0, 0, 4096, 1, 0, 0, 0, 37, 3863, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3447, 0, 11, 0, 2422, 1587, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16657, 16272, 15705, 17027, 16777, 16180, 16489, 15965] avg=16384.0000 max=17027 max_rank=3 min=15705 min_rank=2 imbalance_ratio=1.039246 max_load_reduction=114045 max_load_reduction_pct=0.870094 moved_to_echo=65411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 11, 4096, 0, 0, 0, 4093, 2509, 0, 0, 4053, 1676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 22, 0, 0, 18, 4094, 0, 3994, 4059, 0, 0, 4096, 0, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 4096, 0, 1390, 2648, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 233, 643, 0, 0, 0, 0, 1, 3701, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4095, 0, 0, 4096, 4090, 0, 0, 0, 2097, 0, 0, 0, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1206, 0, 4094, 4096, 0, 0, 461, 0, 1, 0, 0, 29, 0, 1, 2647, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 3943, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 11, 3863, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 3989, 0, 0, 0, 2422, 1587, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16438, 16588, 16317, 16870, 15303, 16625, 16428, 16503] avg=16384.0000 max=16870 max_rank=3 min=15303 min_rank=4 imbalance_ratio=1.029663 max_load_reduction=114202 max_load_reduction_pct=0.871292 moved_to_echo=64859 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 2509, 0, 0, 3991, 1435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 27, 0, 0, 3, 4096, 0, 3988, 3981, 0, 0, 4096, 0, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 4096, 0, 1198, 2648, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 233, 183, 0, 0, 0, 0, 0, 3012, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4087, 0, 0, 0, 3906, 0, 0, 0, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1205, 0, 4096, 4096, 0, 0, 722, 0, 2, 0, 0, 10, 0, 0, 2797, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 0, 3553, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 2, 3863, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 63, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 2421, 1587, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16128, 16419, 16127, 15721, 17105, 17023, 15886, 16663] avg=16384.0000 max=17105 max_rank=4 min=15721 min_rank=3 imbalance_ratio=1.044006 max_load_reduction=113967 max_load_reduction_pct=0.869499 moved_to_echo=66677 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 13, 4094, 3, 0, 0, 4077, 2509, 0, 0, 3677, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 69, 4074, 0, 3936, 4051, 0, 0, 4090, 0, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4077, 0, 0, 0, 0, 4095, 1, 1657, 2648, 13, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 233, 979, 1, 0, 0, 0, 0, 3322, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4085, 0, 0, 4094, 4075, 0, 0, 0, 3718, 2, 0, 0, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1206, 0, 4085, 4095, 0, 0, 218, 0, 17, 0, 0, 49, 0, 40, 2428, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 3907, 0, 6, 0, 0, 4096, 5, 0, 0, 0, 57, 3863, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3104, 0, 16, 0, 2422, 1587, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16018, 16461, 16593, 16832, 16896, 16223, 16423, 15626] avg=16384.0000 max=16896 max_rank=4 min=15626 min_rank=7 imbalance_ratio=1.031250 max_load_reduction=114176 max_load_reduction_pct=0.871094 moved_to_echo=65168 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 52, 4091, 0, 0, 0, 4094, 2502, 0, 0, 4071, 505, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 1, 55, 1, 1, 14, 4095, 0, 3841, 3991, 0, 1, 4086, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4095, 0, 141, 1, 0, 4095, 3, 1552, 2633, 7, 0, 0, 0, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 2, 1, 0, 232, 207, 4, 0, 1, 1, 0, 3413, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4095, 1, 0, 4088, 4067, 1, 1, 1, 3247, 0, 0, 0, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1197, 0, 4088, 4094, 0, 0, 1900, 1, 4, 0, 0, 44, 1, 1, 2049, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4058, 0, 1, 3564, 1, 0, 1, 1, 4095, 59, 1, 0, 0, 27, 3863, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 1, 3, 0, 0, 1, 0, 0, 4095, 1, 4094, 1, 1, 1, 2426, 1593, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15319, 16322, 16644, 16146, 16439, 17438, 16144, 16620] avg=16384.0000 max=17438 max_rank=5 min=15319 min_rank=0 imbalance_ratio=1.064331 max_load_reduction=113634 max_load_reduction_pct=0.866959 moved_to_echo=66641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 27, 4096, 0, 0, 2, 4096, 2509, 0, 0, 3777, 2584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 14, 4096, 0, 3861, 4092, 0, 0, 4096, 0, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 3, 0, 0, 4096, 0, 718, 2647, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 233, 167, 0, 0, 0, 0, 1, 3934, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 4077, 0, 0, 0, 3864, 0, 0, 0, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1205, 0, 4096, 4096, 0, 0, 560, 0, 3, 0, 0, 7, 0, 3, 1098, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4017, 0, 0, 4051, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 29, 3863, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 23, 0, 0, 0, 0, 0, 4096, 0, 4018, 0, 15, 0, 2421, 1587, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17091, 16386, 15656, 16624, 17055, 15163, 16536, 16561] avg=16384.0000 max=17091 max_rank=0 min=15163 min_rank=5 imbalance_ratio=1.043152 max_load_reduction=113981 max_load_reduction_pct=0.869606 moved_to_echo=65315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 2509, 0, 0, 3900, 2325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 15, 0, 0, 27, 4092, 1, 3748, 4079, 0, 0, 4094, 0, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4091, 1, 0, 0, 0, 4095, 1, 1553, 2646, 1, 7, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 233, 179, 1, 0, 0, 0, 5, 2516, 28, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4095, 0, 0, 4096, 4092, 0, 0, 0, 3185, 0, 1, 0, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1206, 0, 4095, 4095, 0, 0, 751, 0, 2, 0, 0, 7, 0, 10, 2320, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 4079, 0, 1, 0, 0, 4096, 18, 0, 0, 0, 18, 3863, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4095, 0, 4050, 0, 0, 0, 2422, 1587, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16924, 16286, 16490, 15251, 16392, 16574, 16595, 16560] avg=16384.0000 max=16924 max_rank=0 min=15251 min_rank=3 imbalance_ratio=1.032959 max_load_reduction=114148 max_load_reduction_pct=0.870880 moved_to_echo=66121 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 35, 4091, 0, 0, 1, 4095, 2509, 0, 0, 3524, 2242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 1, 0, 0, 50, 4095, 0, 3770, 4087, 0, 0, 4096, 0, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 1, 0, 0, 4096, 0, 2592, 2648, 10, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 233, 166, 0, 0, 0, 0, 1, 3913, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4081, 0, 0, 0, 1912, 0, 0, 0, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1206, 0, 4094, 4096, 0, 0, 223, 0, 0, 0, 0, 14, 0, 25, 2105, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 4039, 0, 0, 0, 0, 4096, 28, 0, 0, 0, 48, 3863, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 4063, 0, 2, 0, 2422, 1587, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16497, 16338, 17540, 16601, 15105, 15846, 16571, 16574] avg=16384.0000 max=17540 max_rank=2 min=15105 min_rank=4 imbalance_ratio=1.070557 max_load_reduction=113532 max_load_reduction_pct=0.866180 moved_to_echo=64096 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 256, 32768, 128, 0, 128, 32768, 20096, 0, 0, 30720, 14720, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[9728, 0, 32768, 32768, 0, 0, 5248, 128, 128, 0, 0, 256, 128, 256, 17664, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 128, 0, 1920, 3456, 128, 0, 128, 128, 128, 27520, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 32256, 0, 128, 31232, 128, 128, 128, 128, 32768, 256, 128, 0, 0, 256, 30976, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 256, 128, 0, 32768, 128, 11392, 21248, 128, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 128, 128, 256, 128, 128, 256, 32768, 128, 30976, 32384, 0, 128, 32768, 0, 0, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 0, 32768, 32768, 128, 128, 128, 25600, 128, 128, 0, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 0, 128, 0, 0, 32768, 128, 30976, 128, 128, 128, 19456, 12800, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[27, 0, 4090, 0, 0, 2328, 4092, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 4096, 0, 4096, 0, 4096, 3976, 0, 0, 0, 0, 1763, 4093, 0, 0, 0, 4096, 0, 3, 0, 4, 0, 0, 0, 14, 4096, 0, 0, 0, 4096, 0, 0, 2, 4059, 4096, 0, 4052, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 716, 0, 0, 4096, 0, 0, 4096, 1726, 0, 0, 3407, 4096, 4096, 0, 1, 0, 4095, 0, 0, 4096, 0, 0, 4, 0, 2123, 4096, 0, 0, 0, 7, 0, 393, 0, 0, 4077, 5, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 0, 3734, 4077, 0, 0, 0, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 4096, 4096, 0, 4096, 0, 4096, 3992, 0, 0, 0, 0, 2711, 4080, 0, 0, 3, 4096, 0, 0, 0, 36, 0, 0, 0, 6, 4096, 0, 0, 0, 4096, 0, 0, 0, 4050, 4096, 0, 4026, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 379, 0, 0, 4095, 0, 0, 4096, 3476, 0, 0, 1689, 4096, 4096, 0, 7, 0, 4093, 0, 0, 4096, 0, 0, 23, 0, 153, 4096, 0, 0, 0, 26, 0, 363, 0, 0, 4050, 4, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[6, 0, 4062, 0, 0, 2355, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 4096, 4093, 0, 4096, 0, 4096, 4006, 0, 0, 0, 1, 2465, 4087, 0, 0, 13, 4096, 0, 63, 0, 21, 0, 0, 0, 5, 4095, 0, 0, 0, 4096, 0, 0, 1, 3933, 4096, 1, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 103, 0, 0, 4096, 0, 0, 4096, 2548, 0, 0, 3537, 4096, 4095, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 5, 0, 776, 4092, 0, 0, 0, 1, 0, 816, 0, 0, 4005, 7, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4076, 0, 1, 2265, 4090, 0, 0, 1, 4095, 12, 1, 1, 1, 1, 0, 1, 9, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 4076, 4095, 4086, 0, 4095, 1, 4095, 3918, 0, 0, 1, 2, 2679, 4072, 1, 0, 18, 4095, 0, 34, 1, 20, 0, 0, 0, 14, 4095, 1, 1, 0, 4095, 0, 0, 4, 4092, 4095, 0, 4018, 4091, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 1, 1, 1, 4091, 1, 0, 1637, 0, 1, 4095, 1, 0, 4095, 1075, 0, 1, 3617, 4095, 4095, 1, 41, 0, 4084, 0, 1, 4094, 1, 0, 21, 22, 1052, 4095, 0, 0, 0, 28, 0, 94, 0, 0, 4082, 7, 4095, 0, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 3659, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4096, 4096, 0, 4096, 0, 4096, 3899, 0, 0, 0, 0, 1391, 4083, 0, 0, 0, 4096, 0, 11, 0, 16, 0, 0, 0, 26, 4096, 0, 0, 0, 4096, 0, 0, 1, 3937, 4096, 0, 4043, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 135, 0, 0, 4096, 0, 0, 4096, 3593, 0, 0, 3380, 4096, 4096, 0, 23, 0, 4096, 0, 0, 4096, 0, 0, 5, 2, 329, 4096, 0, 0, 0, 12, 0, 173, 0, 0, 4080, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 3096, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 4096, 4096, 0, 4096, 0, 4096, 3992, 0, 0, 0, 0, 2538, 4087, 0, 0, 0, 4096, 0, 2, 0, 1, 0, 0, 0, 30, 4096, 0, 0, 0, 4096, 0, 0, 3, 4033, 4096, 0, 4083, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 31, 0, 0, 4092, 0, 0, 4096, 3330, 0, 0, 3019, 4096, 4096, 0, 4, 0, 4096, 0, 0, 4096, 0, 0, 7, 3, 264, 4096, 0, 0, 0, 9, 0, 259, 0, 0, 4032, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[9, 0, 4093, 0, 0, 3579, 4095, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 4096, 4094, 0, 4096, 0, 4096, 4011, 0, 0, 0, 1, 1503, 4056, 0, 0, 6, 4096, 0, 17, 0, 29, 0, 0, 0, 64, 4091, 0, 0, 0, 4096, 0, 0, 0, 3901, 4096, 2, 4022, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3, 95, 0, 0, 4096, 0, 0, 4096, 3093, 0, 0, 3578, 4096, 4091, 0, 1, 0, 4085, 0, 0, 4096, 0, 0, 19, 0, 638, 4088, 0, 0, 0, 6, 0, 148, 0, 0, 4035, 15, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 4077, 0, 0, 2763, 4093, 0, 0, 0, 4096, 5, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4096, 4095, 0, 4096, 0, 4096, 4047, 0, 0, 0, 0, 1627, 4085, 0, 0, 20, 4095, 0, 0, 0, 3, 0, 0, 0, 4, 4095, 0, 0, 0, 4096, 0, 0, 0, 4078, 4096, 1, 4094, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 511, 0, 0, 4096, 0, 0, 4096, 3242, 0, 0, 3368, 4095, 4094, 0, 13, 1, 4094, 0, 0, 4096, 0, 0, 16, 0, 737, 4093, 0, 0, 0, 14, 0, 130, 0, 0, 4064, 3, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 4062, 0, 0, 2355, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 1027, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 4096, 4093, 3189, 933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2136, 4006, 0, 0, 0, 1, 2465, 4087, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 21, 0, 0, 0, 5, 4095, 0, 0, 0, 4096, 0, 0, 1, 3933, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 103, 0, 0, 4096, 0, 0, 4096, 2548, 885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3537, 875, 4095, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 4092, 0, 0, 0, 1, 0, 816, 0, 0, 4005, 7, 3068, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15745, 16382, 16804, 16342, 16415, 15824, 16699, 16861] avg=16384.0000 max=16861 max_rank=7 min=15745 min_rank=0 imbalance_ratio=1.029114 max_load_reduction=114211 max_load_reduction_pct=0.871361 moved_to_echo=65799 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[27, 0, 4090, 0, 0, 2328, 4092, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 1027, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 4096, 3189, 933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2136, 3976, 0, 0, 0, 0, 1763, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 0, 0, 0, 14, 4096, 0, 0, 0, 4096, 0, 0, 2, 4059, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4052, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 716, 0, 0, 4096, 0, 0, 4096, 1726, 885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3407, 875, 4096, 0, 1, 0, 4095, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2123, 4096, 0, 0, 0, 7, 0, 393, 0, 0, 4077, 5, 3068, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15761, 16408, 16064, 16402, 16383, 15615, 16574, 17865] avg=16384.0000 max=17865 max_rank=7 min=15615 min_rank=5 imbalance_ratio=1.090393 max_load_reduction=113207 max_load_reduction_pct=0.863701 moved_to_echo=66437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4077, 0, 0, 2763, 4093, 0, 0, 0, 4096, 5, 0, 0, 0, 1, 1027, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4096, 4095, 3188, 933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2136, 4047, 0, 0, 0, 0, 1627, 4085, 0, 0, 20, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4, 4095, 0, 0, 0, 4096, 0, 0, 0, 4078, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4094, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 511, 0, 0, 4096, 0, 0, 4096, 3242, 885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3368, 875, 4094, 0, 13, 1, 4094, 0, 0, 4096, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 737, 4093, 0, 0, 0, 14, 0, 130, 0, 0, 4064, 3, 3068, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16163, 16379, 16010, 16404, 16427, 16927, 16557, 16205] avg=16384.0000 max=16927 max_rank=5 min=16010 min_rank=2 imbalance_ratio=1.033142 max_load_reduction=114145 max_load_reduction_pct=0.870857 moved_to_echo=66116 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 3659, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1027, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4096, 4096, 3189, 933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2136, 3899, 0, 0, 0, 0, 1391, 4083, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 16, 0, 0, 0, 26, 4096, 0, 0, 0, 4096, 0, 0, 1, 3937, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4043, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 135, 0, 0, 4096, 0, 0, 4096, 3593, 885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3380, 875, 4096, 0, 23, 0, 4096, 0, 0, 4096, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 4096, 0, 0, 0, 12, 0, 173, 0, 0, 4080, 0, 3068, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17067, 16387, 15605, 16311, 16374, 16901, 16573, 15854] avg=16384.0000 max=17067 max_rank=0 min=15605 min_rank=2 imbalance_ratio=1.041687 max_load_reduction=114005 max_load_reduction_pct=0.869789 moved_to_echo=65702 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 0, 3734, 4077, 0, 0, 0, 4096, 21, 0, 0, 0, 0, 1027, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 4096, 4096, 3189, 933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2136, 3992, 0, 0, 0, 0, 2711, 4080, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 6, 4096, 0, 0, 0, 4096, 0, 0, 0, 4050, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4026, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 379, 0, 0, 4095, 0, 0, 4096, 3476, 885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1689, 875, 4096, 0, 7, 0, 4093, 0, 0, 4096, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 4096, 0, 0, 0, 26, 0, 363, 0, 0, 4050, 4, 3068, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17149, 16374, 17018, 16412, 16357, 17027, 14879, 15856] avg=16384.0000 max=17149 max_rank=0 min=14879 min_rank=6 imbalance_ratio=1.046692 max_load_reduction=113923 max_load_reduction_pct=0.869164 moved_to_echo=64119 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4076, 0, 1, 2265, 4090, 0, 0, 1, 4095, 12, 1, 1, 1, 1, 1029, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 4076, 4095, 4086, 3190, 940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2126, 3918, 0, 0, 1, 2, 2679, 4072, 1, 0, 18, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 1, 20, 0, 0, 0, 14, 4095, 1, 1, 0, 4095, 0, 0, 4, 4092, 4095, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4018, 4091, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 4095, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4091, 1, 0, 1637, 0, 1, 4095, 1, 0, 4095, 1075, 885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3617, 871, 4095, 1, 41, 0, 4084, 0, 1, 4094, 1, 0, 21, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 4095, 0, 0, 0, 28, 0, 94, 0, 0, 4082, 7, 3062, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15677, 16403, 16913, 16486, 16347, 15884, 16849, 16513] avg=16384.0000 max=16913 max_rank=2 min=15677 min_rank=0 imbalance_ratio=1.032288 max_load_reduction=114159 max_load_reduction_pct=0.870964 moved_to_echo=65593 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 3096, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 1027, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 4096, 4096, 3189, 933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2136, 3992, 0, 0, 0, 0, 2538, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 30, 4096, 0, 0, 0, 4096, 0, 0, 3, 4033, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4093, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 31, 0, 0, 4092, 0, 0, 4096, 3330, 885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3019, 875, 4096, 0, 4, 0, 4096, 0, 0, 4096, 0, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 4096, 0, 0, 0, 9, 0, 259, 0, 0, 4032, 0, 3068, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16504, 16366, 16849, 16389, 16414, 16530, 16196, 15824] avg=16384.0000 max=16849 max_rank=2 min=15824 min_rank=7 imbalance_ratio=1.028381 max_load_reduction=114223 max_load_reduction_pct=0.871452 moved_to_echo=64964 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 4093, 0, 0, 3579, 4095, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 1027, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 4096, 4094, 3189, 933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2136, 4011, 0, 0, 0, 1, 1503, 4056, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 29, 0, 0, 0, 64, 4091, 0, 0, 0, 4096, 0, 0, 0, 3901, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4022, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 95, 0, 0, 4096, 0, 0, 4096, 3093, 885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3578, 875, 4091, 0, 1, 0, 4085, 0, 0, 4096, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 638, 4088, 0, 0, 0, 6, 0, 148, 0, 0, 4035, 15, 3068, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17006, 16373, 15809, 16326, 16355, 16364, 16745, 16094] avg=16384.0000 max=17006 max_rank=0 min=15809 min_rank=2 imbalance_ratio=1.037964 max_load_reduction=114066 max_load_reduction_pct=0.870255 moved_to_echo=65558 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 25600, 7040, 32768, 128, 128, 128, 32768, 0, 128, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[6144, 32768, 0, 0, 0, 128, 0, 2432, 0, 0, 32512, 128, 24576, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 128, 256, 0, 0, 0, 256, 32768, 128, 128, 0, 32768, 0, 0, 128, 32128, 32768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 128, 128, 3712, 0, 128, 32768, 128, 0, 32768, 22144, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 17152, 31872, 0, 0, 128, 128, 16768, 32768, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 32512, 32768, 128, 128, 0, 128, 0, 128, 0, 128, 0, 0, 128, 32768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 0, 128, 128, 128, 0, 0, 0, 128, 128, 32640, 32768, 32768, 25600, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 32768, 0, 128, 23808, 32768, 0, 0, 128, 32768, 128, 128, 128, 128, 128, 8320, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131456 slack_slots=8576 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3978, 0, 0, 30, 0, 3, 0, 0, 1, 42, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4073, 4096, 4096, 0, 0, 1, 0, 0, 3770, 4088, 3, 0, 0, 0, 0, 4, 4096, 4081, 250, 0, 0, 0, 0, 4095, 0, 0, 0, 4089, 0, 0, 0, 4055, 0, 4092, 4096, 3131, 0, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 279, 4024, 1650, 2508, 0, 0, 0, 8, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 863, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 3, 0, 0, 4, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4095, 4089, 0, 0, 51, 0, 2, 0, 0, 1, 15, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 4081, 4096, 4096, 0, 0, 2, 0, 0, 3898, 4087, 308, 0, 0, 1, 0, 10, 4096, 4095, 93, 0, 0, 4, 0, 4095, 0, 0, 0, 4083, 0, 0, 0, 4083, 0, 4090, 4096, 3692, 0, 146, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 75, 3971, 513, 2535, 0, 0, 0, 1, 4093, 0, 4096, 4096, 0, 0, 0, 4096, 1055, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 9, 0, 0, 0, 0, 7, 0, 0, 0, 72, 0, 0, 0, 4095, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 1, 0, 4095, 4095, 3876, 1, 0, 44, 1, 6, 0, 0, 0, 60, 1, 4090, 1, 0, 1, 0, 3, 1, 4095, 4095, 1, 0, 0, 1, 3956, 4095, 4095, 0, 0, 6, 0, 1, 3339, 4089, 1, 0, 0, 0, 0, 19, 4095, 3907, 32, 0, 1, 0, 1, 4095, 0, 1, 1, 3966, 0, 1, 0, 4057, 0, 4087, 4095, 3868, 1, 349, 1, 1, 0, 0, 1, 0, 1, 4091, 4079, 64, 3955, 941, 3162, 2, 0, 0, 18, 4088, 1, 4095, 4095, 1, 0, 0, 4095, 1229, 0, 1, 0, 1, 1, 0, 0, 4086, 0, 4095, 4083, 0, 1, 8, 0, 0, 14, 0, 0, 1, 0, 1, 40, 1, 0, 0, 4094, 1, 0, 0, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4055, 0, 0, 61, 0, 18, 0, 0, 0, 103, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4079, 4096, 4096, 0, 0, 0, 0, 0, 3950, 4094, 0, 0, 0, 0, 0, 3, 4096, 4018, 108, 0, 0, 1, 0, 4096, 0, 0, 0, 4007, 0, 0, 0, 4077, 0, 4096, 4096, 3925, 0, 26, 0, 0, 0, 1, 0, 0, 0, 4096, 4086, 95, 3943, 562, 3255, 0, 0, 0, 24, 4094, 0, 4096, 4096, 0, 0, 0, 4096, 485, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4095, 0, 0, 50, 0, 0, 0, 0, 1, 0, 0, 0, 34, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3988, 0, 0, 62, 0, 4, 0, 0, 0, 76, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4065, 4096, 4096, 0, 0, 0, 0, 0, 3926, 4095, 1, 0, 0, 0, 0, 6, 4096, 4089, 279, 0, 0, 0, 0, 4096, 0, 0, 0, 4076, 0, 0, 0, 4074, 0, 4096, 4096, 3783, 0, 92, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 228, 3983, 657, 3074, 0, 0, 0, 2, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 402, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4079, 0, 0, 2, 0, 0, 0, 0, 0, 14, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 0, 3749, 4095, 3, 0, 0, 0, 0, 28, 4096, 4095, 58, 0, 0, 1, 0, 4096, 0, 0, 0, 4056, 0, 0, 0, 4084, 0, 4085, 4096, 3865, 0, 50, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 126, 3890, 64, 3899, 0, 0, 0, 17, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 668, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 8, 0, 0, 0, 0, 1, 0, 0, 0, 26, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4093, 4075, 0, 0, 35, 0, 16, 0, 0, 0, 28, 0, 4096, 0, 0, 0, 0, 6, 0, 4096, 4096, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 5, 0, 0, 3583, 4092, 4, 0, 0, 0, 0, 1, 4096, 4096, 286, 0, 0, 4, 0, 4087, 0, 0, 0, 4014, 0, 0, 0, 4033, 0, 4091, 4096, 3593, 0, 18, 0, 0, 0, 0, 0, 0, 0, 4095, 4065, 51, 4010, 525, 3791, 0, 0, 0, 53, 4078, 0, 4096, 4096, 0, 0, 0, 4096, 621, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4095, 4088, 0, 0, 19, 0, 4, 0, 0, 0, 26, 0, 4095, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 2, 1, 0, 3881, 4092, 4, 0, 0, 0, 0, 2, 4095, 4081, 457, 0, 0, 3, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 4086, 0, 4095, 4096, 3736, 0, 22, 0, 0, 0, 0, 0, 0, 0, 4095, 4083, 235, 3938, 81, 3501, 0, 0, 0, 1, 4095, 0, 4096, 4096, 0, 0, 0, 4096, 332, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 4096, 0, 0, 35, 0, 0, 1, 0, 9, 0, 0, 0, 158, 0, 0, 0, 4095, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4095, 4089, 0, 0, 51, 0, 2, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 15, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 3660, 330, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 316, 4096, 0, 0, 2, 0, 0, 3898, 4087, 308, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 93, 0, 0, 4, 0, 4014, 0, 0, 0, 4083, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 180, 3692, 0, 146, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 513, 2535, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 3766, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 9, 0, 0, 3916, 120, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 72, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16452, 16353, 16799, 16372, 16368, 15934, 16383, 16411] avg=16384.0000 max=16799 max_rank=2 min=15934 min_rank=5 imbalance_ratio=1.025330 max_load_reduction=114273 max_load_reduction_pct=0.871834 moved_to_echo=65096 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3978, 0, 0, 30, 0, 3, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 42, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3660, 330, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 316, 4096, 0, 0, 1, 0, 0, 3770, 4088, 3, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 250, 0, 0, 0, 0, 4014, 0, 0, 0, 4089, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 180, 3131, 0, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 1650, 2508, 0, 0, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 3766, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 3, 0, 0, 3916, 120, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16322, 16379, 16351, 16489, 15885, 16915, 16375, 16356] avg=16384.0000 max=16915 max_rank=5 min=15885 min_rank=4 imbalance_ratio=1.032410 max_load_reduction=114157 max_load_reduction_pct=0.870949 moved_to_echo=65531 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 1, 0, 4095, 4095, 3876, 1, 0, 44, 1, 6, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 1, 4090, 1, 0, 1, 0, 3, 1, 4095, 4095, 1, 0, 0, 1, 3660, 336, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 309, 4095, 0, 0, 6, 0, 1, 3339, 4089, 1, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 32, 0, 1, 0, 1, 4004, 0, 1, 1, 3966, 0, 1, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 173, 3868, 1, 349, 1, 1, 0, 0, 1, 0, 1, 4091, 4079, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 941, 3162, 2, 0, 0, 18, 4088, 1, 0, 0, 1, 0, 0, 3759, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 4086, 0, 4095, 4083, 0, 1, 8, 0, 0, 3922, 126, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1, 0, 1, 40, 1, 0, 0, 4094, 1, 0, 0, 1, 1, 4095, 4095, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16242, 16409, 15815, 15971, 16716, 17156, 16375, 16388] avg=16384.0000 max=17156 max_rank=5 min=15815 min_rank=2 imbalance_ratio=1.047119 max_load_reduction=113916 max_load_reduction_pct=0.869110 moved_to_echo=66635 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4055, 0, 0, 61, 0, 18, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3660, 330, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 316, 4096, 0, 0, 0, 0, 0, 3950, 4094, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 108, 0, 0, 1, 0, 4015, 0, 0, 0, 4007, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 180, 3925, 0, 26, 0, 0, 0, 1, 0, 0, 0, 4096, 4086, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 562, 3255, 0, 0, 0, 24, 4094, 0, 0, 0, 0, 0, 0, 3766, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4095, 0, 0, 50, 0, 0, 3916, 120, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 34, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16445, 16439, 16538, 16226, 16505, 16129, 16421, 16369] avg=16384.0000 max=16538 max_rank=2 min=16129 min_rank=5 imbalance_ratio=1.009399 max_load_reduction=114534 max_load_reduction_pct=0.873825 moved_to_echo=65424 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3988, 0, 0, 62, 0, 4, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3660, 330, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 316, 4096, 0, 0, 0, 0, 0, 3926, 4095, 1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 279, 0, 0, 0, 0, 4015, 0, 0, 0, 4076, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 180, 3783, 0, 92, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 657, 3074, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 3766, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 0, 3916, 120, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16365, 16412, 16505, 16533, 16561, 15980, 16374, 16342] avg=16384.0000 max=16561 max_rank=4 min=15980 min_rank=5 imbalance_ratio=1.010803 max_load_reduction=114511 max_load_reduction_pct=0.873650 moved_to_echo=65257 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4093, 4075, 0, 0, 35, 0, 16, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 4096, 0, 0, 0, 0, 6, 0, 4096, 4096, 0, 0, 0, 0, 3660, 330, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 316, 4096, 0, 0, 5, 0, 0, 3583, 4092, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 286, 0, 0, 4, 0, 4006, 0, 0, 0, 4014, 0, 0, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 180, 3593, 0, 18, 0, 0, 0, 0, 0, 0, 0, 4095, 4065, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 525, 3791, 0, 0, 0, 53, 4078, 0, 0, 0, 0, 0, 0, 3766, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4096, 4095, 0, 0, 0, 0, 0, 3916, 120, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16434, 16370, 16192, 16439, 16093, 16844, 16355, 16345] avg=16384.0000 max=16844 max_rank=5 min=16093 min_rank=4 imbalance_ratio=1.028076 max_load_reduction=114228 max_load_reduction_pct=0.871490 moved_to_echo=65637 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4095, 4088, 0, 0, 19, 0, 4, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 4095, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 3660, 330, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 316, 4096, 0, 0, 2, 1, 0, 3881, 4092, 4, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 457, 0, 0, 3, 0, 4015, 0, 0, 0, 4091, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 180, 3736, 0, 22, 0, 0, 0, 0, 0, 0, 0, 4095, 4083, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3938, 81, 3501, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 3766, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 4096, 0, 0, 35, 0, 0, 3916, 120, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 158, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16420, 16362, 16490, 16733, 16446, 15714, 16407, 16500] avg=16384.0000 max=16733 max_rank=3 min=15714 min_rank=5 imbalance_ratio=1.021301 max_load_reduction=114339 max_load_reduction_pct=0.872337 moved_to_echo=65067 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4079, 0, 0, 2, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3660, 330, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 316, 4096, 0, 0, 0, 0, 0, 3749, 4095, 3, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 58, 0, 0, 1, 0, 4015, 0, 0, 0, 4056, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 180, 3865, 0, 50, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3890, 64, 3899, 0, 0, 0, 17, 4096, 0, 0, 0, 0, 0, 0, 3766, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 8, 0, 0, 3916, 120, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 26, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 16348, 16382, 16309, 16498, 16400, 16382, 16361] avg=16384.0000 max=16498 max_rank=4 min=16309 min_rank=3 imbalance_ratio=1.006958 max_load_reduction=114574 max_load_reduction_pct=0.874130 moved_to_echo=65641 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32768, 1536, 29696, 128, 768, 128, 128, 0, 128, 128, 0, 128, 32768, 32768, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[31744, 5120, 25728, 128, 0, 0, 128, 32768, 128, 0, 0, 128, 0, 0, 30208, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32640, 2560, 32768, 0, 0, 128, 128, 128, 30208, 32768, 384, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 384, 128, 32768, 128, 0, 128, 0, 128, 128, 32768, 32768, 128, 0, 0, 128, 29312, 2688, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 128, 0, 128, 0, 32768, 32768, 32256, 128, 0, 384, 128, 128, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32512, 1664, 0, 128, 128, 128, 32128, 0, 128, 128, 32384, 0, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 32768, 0, 32768, 32768, 0, 128, 128, 0, 0, 31360, 1024, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 0, 128, 384, 128, 0, 0, 32768, 128, 0, 0, 128, 128, 32768, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 3948, 0, 4096, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2563, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 154, 4096, 0, 5, 4090, 374, 0, 0, 4096, 0, 7, 4087, 4096, 0, 0, 0, 0, 0, 4094, 4057, 0, 0, 418, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 734, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 4096, 4096, 0, 6, 4096, 4096, 4096, 4096, 4096, 0, 2, 0, 1, 0, 1, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1, 3602, 0, 4095, 1, 319, 1, 0, 1, 0, 1, 0, 0, 1, 0, 2089, 0, 0, 1, 4095, 1, 1, 4094, 1, 1, 1, 8, 4095, 0, 6, 1, 0, 4085, 0, 0, 4015, 4073, 1, 100, 4094, 1, 370, 4074, 216, 0, 1, 4095, 0, 20, 3996, 4095, 0, 0, 1, 1, 0, 4095, 4088, 0, 0, 5, 4095, 0, 0, 4095, 1, 4, 0, 0, 75, 596, 0, 1, 1, 115, 2, 0, 0, 4095, 0, 4095, 4006, 0, 4, 4095, 4095, 4087, 4095, 4095, 1, 0, 0, 0, 1, 110, 0, 4095, 4095, 1, 1, 1, 4095, 0, 1, 0, 4095, 1, 1, 0, 0, 4095, 0, 0, 885, 4095, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 3962, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2340, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 4092, 0, 0, 4096, 4085, 0, 7, 4095, 0, 370, 4093, 598, 0, 5, 4096, 0, 13, 4075, 4096, 0, 0, 0, 0, 0, 4093, 4047, 0, 0, 380, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 526, 0, 0, 0, 34, 0, 0, 0, 4096, 0, 4096, 4096, 0, 5, 4096, 4096, 4095, 4095, 4096, 0, 2, 0, 0, 0, 10, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 28, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 3587, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2482, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4094, 0, 0, 4096, 4096, 0, 88, 4096, 0, 379, 4094, 398, 0, 0, 4096, 0, 19, 4060, 4096, 0, 0, 0, 0, 0, 4096, 4047, 0, 0, 446, 4096, 0, 0, 4096, 0, 1, 0, 0, 2, 716, 0, 0, 0, 29, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4093, 4096, 4096, 4095, 4096, 0, 0, 0, 0, 0, 11, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 3, 4095, 0, 0, 0, 0, 4096, 0, 0, 118, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 4017, 0, 4096, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2984, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 3, 4096, 0, 29, 4087, 357, 0, 1, 4096, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 4096, 4069, 0, 0, 91, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 316, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 205, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 3955, 0, 4096, 0, 26, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2670, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 2, 0, 0, 4085, 0, 0, 4096, 4093, 0, 116, 4096, 0, 390, 4093, 288, 0, 6, 4094, 0, 4, 4081, 4096, 0, 0, 0, 0, 0, 4093, 4082, 0, 0, 136, 4081, 0, 0, 4096, 0, 0, 0, 0, 0, 548, 1, 0, 0, 2, 0, 0, 0, 4096, 0, 4096, 4094, 0, 11, 4096, 4096, 4095, 4096, 4096, 0, 2, 0, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 7, 4096, 0, 0, 0, 0, 4096, 0, 0, 89, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 3788, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3060, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 4096, 0, 8, 0, 0, 4089, 0, 0, 4096, 4093, 0, 5, 4096, 0, 127, 4076, 408, 0, 8, 4092, 0, 33, 4066, 4096, 5, 0, 0, 0, 0, 4089, 4058, 0, 0, 116, 4064, 0, 0, 4096, 0, 0, 0, 0, 1, 687, 0, 0, 0, 36, 0, 0, 0, 4096, 0, 4096, 4096, 0, 26, 4096, 4096, 4091, 4084, 4090, 0, 5, 0, 7, 0, 0, 0, 4096, 4092, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 26, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 4071, 0, 4095, 0, 40, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2170, 0, 1, 0, 4096, 0, 0, 4095, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4090, 0, 0, 4096, 4094, 0, 153, 4095, 0, 369, 4088, 493, 0, 4, 4094, 0, 5, 4073, 4096, 1, 0, 0, 0, 1, 4095, 4090, 0, 0, 244, 4093, 0, 0, 4096, 0, 0, 0, 0, 2, 530, 1, 0, 0, 4, 0, 0, 0, 4096, 0, 4096, 4096, 0, 8, 4095, 4096, 4095, 4095, 4096, 0, 2, 0, 1, 0, 1, 0, 4096, 4094, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 145, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3955, 0, 4096, 0, 26, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2670, 4096, 889, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 2, 0, 0, 3205, 697, 174, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3388, 0, 0, 4096, 4093, 0, 116, 0, 0, 390, 4093, 288, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4081, 0, 0, 0, 0, 0, 0, 3919, 4082, 0, 0, 136, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 548, 1, 0, 0, 2, 0, 0, 0, 4096, 0, 4096, 3288, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 11, 0, 0, 4095, 4096, 4085, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 7, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 3908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16543, 16379, 16470, 16303, 16315, 16385, 16391, 16286] avg=16384.0000 max=16543 max_rank=0 min=16286 min_rank=7 imbalance_ratio=1.009705 max_load_reduction=114529 max_load_reduction_pct=0.873787 moved_to_echo=65377 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4017, 0, 4096, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2984, 4096, 889, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3207, 699, 174, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3397, 0, 0, 4096, 4096, 0, 3, 0, 0, 29, 4087, 357, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3922, 4069, 0, 0, 91, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 316, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 3287, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4085, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17119, 16378, 16066, 16271, 16080, 16374, 16383, 16401] avg=16384.0000 max=17119 max_rank=0 min=16066 min_rank=2 imbalance_ratio=1.044861 max_load_reduction=113953 max_load_reduction_pct=0.869392 moved_to_echo=65238 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3948, 0, 4096, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2563, 4096, 888, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3208, 699, 174, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3397, 0, 0, 4096, 4095, 0, 154, 0, 0, 5, 4090, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4087, 0, 0, 0, 0, 0, 0, 3920, 4057, 0, 0, 418, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 734, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 4096, 3288, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 4096, 4096, 4085, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16435, 16380, 16211, 16578, 16504, 16383, 16383, 16198] avg=16384.0000 max=16578 max_rank=3 min=16198 min_rank=7 imbalance_ratio=1.011841 max_load_reduction=114494 max_load_reduction_pct=0.873520 moved_to_echo=65468 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 3602, 0, 4095, 1, 319, 1, 0, 1, 0, 1, 0, 0, 1, 0, 2089, 4095, 889, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 4094, 1, 1, 1, 8, 4095, 0, 6, 1, 0, 3206, 700, 181, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3385, 0, 0, 4015, 4073, 1, 100, 0, 1, 370, 4074, 216, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 3996, 0, 0, 0, 1, 1, 0, 3914, 4088, 0, 0, 5, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4, 0, 0, 75, 596, 0, 1, 1, 115, 2, 0, 0, 4095, 0, 4095, 3285, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 4, 0, 0, 4087, 4095, 4083, 1, 0, 0, 0, 1, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 1, 4095, 0, 1, 0, 4095, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 885, 4095, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 4095, 4095, 3907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15904, 16404, 16236, 16120, 16553, 16387, 16386, 17082] avg=16384.0000 max=17082 max_rank=7 min=15904 min_rank=0 imbalance_ratio=1.042603 max_load_reduction=113990 max_load_reduction_pct=0.869675 moved_to_echo=66408 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3587, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2482, 4093, 889, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 3207, 698, 174, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3396, 0, 0, 4096, 4096, 0, 88, 0, 0, 379, 4094, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4060, 0, 0, 0, 0, 0, 0, 3922, 4047, 0, 0, 446, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 2, 716, 0, 0, 0, 29, 0, 0, 0, 4096, 0, 4096, 3288, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4095, 4085, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 3, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15963, 16378, 16547, 16590, 16512, 16383, 16385, 16314] avg=16384.0000 max=16590 max_rank=3 min=15963 min_rank=0 imbalance_ratio=1.012573 max_load_reduction=114482 max_load_reduction_pct=0.873428 moved_to_echo=65594 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3962, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2340, 4096, 888, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 3208, 698, 174, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3394, 0, 0, 4096, 4085, 0, 7, 0, 0, 370, 4093, 598, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4075, 0, 0, 0, 0, 0, 0, 3919, 4047, 0, 0, 380, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 526, 0, 0, 0, 34, 0, 0, 0, 4096, 0, 4095, 3287, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 4095, 4095, 4085, 0, 2, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16196, 16386, 16648, 16525, 16321, 16388, 16383, 16225] avg=16384.0000 max=16648 max_rank=2 min=16196 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=114424 max_load_reduction_pct=0.872986 moved_to_echo=65317 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4071, 0, 4095, 0, 40, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2170, 4095, 888, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4095, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 3206, 698, 174, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3392, 0, 0, 4096, 4094, 0, 153, 0, 0, 369, 4088, 493, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4073, 0, 1, 0, 0, 0, 1, 3921, 4090, 0, 0, 244, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 530, 1, 0, 0, 4, 0, 0, 0, 4096, 0, 4096, 3287, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 4095, 4095, 4085, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4096, 4096, 3908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16169, 16379, 16689, 16428, 16300, 16383, 16381, 16343] avg=16384.0000 max=16689 max_rank=2 min=16169 min_rank=0 imbalance_ratio=1.018616 max_load_reduction=114383 max_load_reduction_pct=0.872673 moved_to_echo=65407 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3788, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3060, 4096, 888, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 4096, 0, 8, 0, 0, 3204, 697, 174, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3392, 0, 0, 4096, 4093, 0, 5, 0, 0, 127, 4076, 408, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4066, 0, 5, 0, 0, 0, 0, 3915, 4058, 0, 0, 116, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 687, 0, 0, 0, 36, 0, 0, 0, 4096, 0, 4096, 3287, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 26, 0, 0, 4091, 4084, 4080, 0, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 3908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16743, 16388, 16205, 16257, 16487, 16389, 16380, 16223] avg=16384.0000 max=16743 max_rank=0 min=16205 min_rank=2 imbalance_ratio=1.021912 max_load_reduction=114329 max_load_reduction_pct=0.872261 moved_to_echo=65479 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[27264, 0, 0, 32768, 32768, 128, 640, 0, 128, 2048, 32768, 3200, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 1536, 32768, 0, 128, 128, 0, 0, 128, 128, 128, 0, 0, 128, 128, 0, 32768, 32768, 31360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32640, 0, 128, 0, 128, 128, 128, 31360, 32640, 0, 0, 1920, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 30976, 0, 32768, 128, 768, 128, 0, 128, 0, 128, 0, 0, 128, 0, 20480, 32768, 7168, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 0, 0, 128, 4736, 128, 128, 128, 256, 128, 0, 0, 32768, 0, 32768, 26368, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 128, 128, 32768, 0, 128, 128, 32768, 128, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 32768, 0, 128, 128, 0, 25728, 5632, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 0, 128, 0, 0, 32768, 32768, 32768, 128, 128, 0, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[9, 0, 0, 4094, 0, 0, 8, 0, 0, 0, 389, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4075, 4089, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4090, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 2433, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1856, 0, 0, 3932, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 4096, 0, 3691, 3, 0, 4088, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[15, 0, 0, 4091, 0, 0, 17, 0, 0, 0, 595, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 4089, 4090, 0, 0, 0, 4094, 0, 1, 0, 0, 1, 0, 4095, 0, 0, 4096, 0, 0, 4096, 3981, 4081, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2804, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2611, 0, 0, 2525, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4095, 0, 3869, 4, 0, 4090, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[21, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 4096, 0, 4079, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 4096, 0, 0, 4096, 4055, 4087, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2343, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3092, 0, 1, 3435, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3331, 12, 0, 4090, 4096, 0, 0, 4096, 0, 0, 4096, 4095, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[156, 0, 0, 4050, 0, 0, 1, 0, 0, 0, 37, 0, 0, 0, 4096, 0, 4078, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4090, 4061, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2721, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1478, 0, 0, 4090, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 0, 3879, 15, 0, 4089, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[88, 1, 0, 4032, 0, 1, 34, 0, 1, 0, 94, 1, 1, 0, 4095, 0, 3990, 0, 1, 0, 0, 4094, 4095, 0, 0, 0, 4091, 1, 2, 0, 0, 0, 0, 4087, 0, 29, 4095, 0, 1, 4089, 4082, 4080, 4095, 0, 0, 0, 9, 0, 1, 2, 1, 0, 1, 0, 4095, 0, 1, 0, 0, 1, 1, 4069, 4095, 1, 0, 4, 1, 0, 1, 1, 0, 0, 4095, 1, 4094, 4094, 1, 1, 1, 1, 4095, 1, 4095, 0, 1, 3056, 1, 1, 0, 1, 0, 0, 0, 4095, 0, 0, 1, 1, 1, 1585, 0, 0, 3966, 0, 0, 0, 1, 1, 0, 4095, 1, 1, 4095, 0, 3645, 15, 1, 4093, 4095, 0, 0, 4095, 0, 1, 4095, 4091, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[37, 0, 0, 4093, 0, 0, 25, 0, 0, 0, 221, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 4077, 4015, 0, 0, 0, 4081, 0, 3, 0, 0, 4, 0, 4095, 0, 0, 4095, 0, 0, 4096, 3939, 4051, 4096, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4091, 0, 4094, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 2844, 0, 0, 0, 0, 0, 1, 0, 4096, 3, 5, 0, 0, 0, 1658, 0, 0, 3977, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4091, 0, 3772, 46, 0, 4069, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4, 0, 0, 4078, 0, 0, 5, 0, 0, 0, 203, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4090, 4092, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 4096, 3886, 4073, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2818, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1372, 0, 0, 4046, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 4096, 0, 4051, 34, 0, 4094, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[5, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 107, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4090, 4090, 0, 0, 0, 4093, 0, 0, 0, 0, 1, 0, 4095, 4, 0, 4095, 0, 0, 4096, 3980, 4055, 4095, 1, 0, 0, 37, 1, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4095, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 0, 4086, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 2379, 0, 0, 1, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 2192, 1, 0, 3928, 0, 0, 1, 0, 7, 0, 4095, 0, 0, 4095, 0, 3808, 6, 0, 4092, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 4094, 0, 0, 8, 0, 0, 0, 389, 0, 0, 0, 4096, 0, 4096, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 87, 0, 0, 4096, 4075, 4089, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2218, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4090, 0, 4095, 4096, 0, 0, 0, 0, 2306, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 2433, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1856, 0, 0, 3932, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 4096, 2215, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3691, 3, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4096, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16532, 16414, 16454, 16387, 16379, 16141, 16457, 16308] avg=16384.0000 max=16532 max_rank=0 min=16141 min_rank=5 imbalance_ratio=1.009033 max_load_reduction=114540 max_load_reduction_pct=0.873871 moved_to_echo=65285 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[15, 0, 0, 4091, 0, 0, 17, 0, 0, 0, 595, 0, 0, 0, 4096, 0, 4096, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4089, 4090, 0, 0, 0, 4094, 0, 1, 0, 0, 1, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 87, 0, 0, 4096, 3981, 4081, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 2218, 1882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 4096, 0, 0, 0, 0, 2305, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2804, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2611, 0, 0, 2525, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 2214, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3869, 4, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16750, 16406, 16347, 16384, 16385, 16512, 15800, 16488] avg=16384.0000 max=16750 max_rank=0 min=15800 min_rank=6 imbalance_ratio=1.022339 max_load_reduction=114322 max_load_reduction_pct=0.872208 moved_to_echo=65185 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 4096, 0, 4096, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 87, 0, 0, 4096, 4055, 4087, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2218, 1882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2306, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2343, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3092, 0, 1, 3435, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2214, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3331, 12, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4095, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16263, 16404, 16434, 16388, 16385, 16050, 17190, 15958] avg=16384.0000 max=17190 max_rank=6 min=15958 min_rank=7 imbalance_ratio=1.049194 max_load_reduction=113882 max_load_reduction_pct=0.868851 moved_to_echo=65583 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[156, 0, 0, 4050, 0, 0, 1, 0, 0, 0, 37, 0, 0, 0, 4096, 0, 4096, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 87, 0, 0, 4096, 4090, 4061, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2218, 1882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 2306, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2721, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1478, 0, 0, 4090, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 2214, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3879, 15, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16276, 16403, 16453, 16388, 16383, 16428, 16232, 16509] avg=16384.0000 max=16509 max_rank=7 min=16232 min_rank=6 imbalance_ratio=1.007629 max_load_reduction=114563 max_load_reduction_pct=0.874046 moved_to_echo=65552 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 107, 0, 0, 0, 4096, 0, 4096, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 4090, 0, 0, 0, 4093, 0, 0, 0, 0, 1, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 87, 0, 0, 4096, 3980, 4055, 0, 1, 0, 0, 37, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4095, 4096, 0, 2217, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 4086, 0, 4095, 4096, 0, 0, 0, 0, 2305, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 2379, 0, 0, 1, 0, 0, 0, 0, 4096, 1, 0, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2192, 1, 0, 3928, 0, 0, 1, 0, 7, 0, 4095, 0, 0, 4096, 2214, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3808, 6, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16238, 16407, 16356, 16387, 16375, 16088, 16790, 16431] avg=16384.0000 max=16790 max_rank=6 min=16088 min_rank=5 imbalance_ratio=1.024780 max_load_reduction=114282 max_load_reduction_pct=0.871902 moved_to_echo=65684 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[88, 1, 0, 4032, 0, 1, 34, 0, 1, 0, 94, 1, 1, 0, 4095, 0, 4095, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 0, 1, 0, 0, 4094, 4095, 0, 0, 0, 4091, 1, 2, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 29, 82, 0, 1, 4089, 4082, 4080, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 0, 1, 0, 4095, 0, 1, 0, 0, 1, 1, 4069, 4095, 1, 2219, 1882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 1, 1, 0, 0, 4095, 1, 4094, 4094, 1, 1, 1, 1, 2307, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4095, 0, 1, 3056, 1, 1, 0, 1, 0, 0, 0, 4095, 0, 0, 1422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1585, 0, 0, 3966, 0, 0, 0, 1, 1, 0, 4095, 1, 1, 4095, 2213, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3645, 15, 1, 4093, 0, 0, 0, 0, 0, 1, 0, 4091, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16283, 16317, 16459, 16369, 16396, 16768, 16216, 16264] avg=16384.0000 max=16768 max_rank=5 min=16216 min_rank=6 imbalance_ratio=1.023438 max_load_reduction=114304 max_load_reduction_pct=0.872070 moved_to_echo=65644 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[37, 0, 0, 4093, 0, 0, 25, 0, 0, 0, 221, 0, 0, 0, 4096, 0, 4096, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4077, 4015, 0, 0, 0, 4081, 0, 3, 0, 0, 4, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 87, 0, 0, 4096, 3939, 4051, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 2217, 1882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4091, 0, 4094, 4096, 0, 0, 0, 0, 2305, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 2844, 0, 0, 0, 0, 0, 1, 0, 4096, 3, 5, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1658, 0, 0, 3977, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 2214, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3772, 46, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 4096, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16408, 16314, 16316, 16382, 16384, 16560, 16299, 16409] avg=16384.0000 max=16560 max_rank=5 min=16299 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=114512 max_load_reduction_pct=0.873657 moved_to_echo=65652 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 4078, 0, 0, 5, 0, 0, 0, 203, 0, 0, 0, 4096, 0, 4096, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 4092, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 87, 0, 0, 4096, 3886, 4073, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2218, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2306, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2818, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1372, 0, 0, 4046, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 4096, 2215, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4051, 34, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16322, 16407, 16253, 16387, 16385, 16525, 16088, 16705] avg=16384.0000 max=16705 max_rank=7 min=16088 min_rank=6 imbalance_ratio=1.019592 max_load_reduction=114367 max_load_reduction_pct=0.872551 moved_to_echo=65703 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 32768, 128, 128, 21504, 128, 128, 128, 128, 0, 128, 0, 32768, 128, 128, 11392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[384, 128, 0, 32640, 0, 128, 128, 128, 128, 0, 1920, 128, 128, 0, 32768, 0, 32768, 30720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32640, 0, 128, 128, 0, 32768, 32768, 0, 0, 0, 32768, 128, 128, 0, 0, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 0, 30080, 256, 128, 32768, 0, 0, 0, 0, 0, 128, 0, 32768, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 128, 128, 768, 0, 128, 32768, 32128, 32640, 0, 128, 0, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 15872, 128, 128, 29952, 0, 0, 128, 128, 128, 0, 32768, 128, 128, 32768, 17792, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 0, 32768, 128, 32768, 32768, 128, 128, 128, 128, 18560, 14336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 0, 128, 0, 128, 128, 128, 32768, 32768, 128, 17792, 15104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3706, 4096, 0, 0, 0, 0, 0, 4096, 0, 257, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 141, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 3, 0, 0, 1735, 0, 0, 0, 0, 4096, 1, 3761, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 4096, 0, 4096, 263, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 16, 4096, 18, 4096, 4096, 0, 0, 0, 3958, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 3305, 0, 4024, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 3383, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3855, 4096, 0, 0, 0, 0, 0, 4096, 0, 725, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4083, 0, 0, 155, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 2, 0, 0, 1900, 0, 0, 0, 0, 4096, 1, 3086, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4095, 4087, 0, 4095, 363, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4088, 0, 0, 4, 4096, 16, 4095, 4096, 4, 0, 0, 4080, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 3549, 0, 4076, 0, 0, 0, 0, 1, 0, 9, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 2774, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3924, 4096, 0, 0, 0, 0, 0, 4096, 0, 542, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 373, 0, 4096, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 2507, 0, 0, 6, 0, 4096, 0, 3775, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 4096, 0, 4096, 607, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 19, 4096, 4096, 0, 0, 0, 3874, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 1340, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 3556, 4096, 0, 4096, 1, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3701, 4096, 0, 0, 0, 0, 0, 4096, 0, 637, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 567, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 1730, 0, 0, 3, 0, 4096, 0, 2781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 864, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 2, 4096, 28, 4096, 4096, 0, 0, 0, 4059, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 3333, 0, 4094, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 2784, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3861, 4094, 1, 1, 1, 0, 0, 4095, 6, 1377, 2, 1, 4095, 4094, 1, 0, 1, 0, 0, 4095, 1, 1, 32, 1, 4095, 0, 0, 2, 1, 3755, 1, 1, 1, 1, 1, 1228, 1, 0, 19, 0, 4095, 0, 3729, 1, 0, 0, 1, 0, 0, 0, 0, 8, 4095, 4095, 0, 4095, 176, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 1, 1, 0, 4080, 1, 0, 118, 4036, 0, 4095, 4059, 0, 1, 0, 4009, 0, 1, 1, 1, 4095, 0, 0, 4094, 1, 1, 1, 4095, 24, 2427, 0, 4065, 0, 1, 0, 1, 0, 0, 0, 4094, 0, 4053, 1, 0, 1, 4095, 0, 0, 1, 4095, 3979, 4095, 0, 4095, 1, 0, 1, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2018, 4096, 0, 0, 0, 0, 0, 4096, 0, 627, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4091, 0, 0, 602, 0, 4096, 0, 0, 0, 0, 4071, 0, 0, 4, 0, 0, 3130, 0, 0, 0, 0, 4096, 0, 3842, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 4096, 0, 4096, 58, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 6, 4096, 35, 4096, 4096, 1, 0, 0, 4054, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 2984, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 3164, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3377, 4096, 0, 0, 0, 0, 0, 4096, 0, 231, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4075, 0, 0, 733, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 15, 0, 0, 3020, 0, 0, 0, 0, 4096, 3, 3394, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4094, 4095, 0, 4095, 409, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4087, 0, 0, 0, 4096, 38, 4094, 4095, 6, 0, 0, 4015, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 3232, 0, 4068, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 4096, 0, 0, 2, 4095, 0, 0, 0, 4096, 2026, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 3028, 4095, 0, 0, 0, 0, 1, 4096, 0, 392, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4081, 0, 0, 275, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 5, 0, 0, 3323, 0, 0, 0, 0, 4096, 3, 3318, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4095, 4090, 0, 4095, 317, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4091, 0, 0, 0, 4096, 19, 4095, 4096, 5, 0, 0, 3993, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 3298, 1, 4075, 0, 0, 0, 0, 1, 0, 9, 4096, 0, 4096, 0, 1, 1, 4096, 0, 0, 0, 4096, 2534, 4095, 0, 4095, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3855, 4096, 0, 0, 0, 0, 0, 4096, 0, 725, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 4096, 0, 0, 0, 0, 0, 4083, 0, 0, 155, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 2, 0, 0, 1900, 0, 0, 0, 0, 4096, 1, 3086, 0, 2167, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4095, 4087, 0, 4095, 363, 0, 0, 0, 2962, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 1, 0, 0, 4088, 0, 0, 4, 4096, 16, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 4080, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 66, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1189, 0, 3549, 0, 4076, 0, 0, 0, 0, 1, 0, 9, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2774, 4096, 0, 1072, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16930, 16174, 15626, 16359, 16370, 16464, 17015, 16134] avg=16384.0000 max=17015 max_rank=6 min=15626 min_rank=2 imbalance_ratio=1.038513 max_load_reduction=114057 max_load_reduction_pct=0.870186 moved_to_echo=65983 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3706, 4096, 0, 0, 0, 0, 0, 4096, 0, 257, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 141, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 3, 0, 0, 1735, 0, 0, 0, 0, 4096, 1, 3761, 0, 2167, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 4096, 0, 4096, 263, 0, 0, 0, 2962, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 4092, 0, 0, 16, 4096, 18, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3958, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 66, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1189, 0, 3305, 0, 4024, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3383, 4096, 0, 1072, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16313, 16173, 16139, 16264, 16388, 16338, 16714, 16743] avg=16384.0000 max=16743 max_rank=7 min=16139 min_rank=2 imbalance_ratio=1.021912 max_load_reduction=114329 max_load_reduction_pct=0.872261 moved_to_echo=66183 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3924, 4096, 0, 0, 0, 0, 0, 4096, 0, 542, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 373, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 2507, 0, 0, 6, 0, 4096, 0, 3775, 0, 2167, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 4096, 0, 4096, 607, 0, 0, 0, 2962, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 19, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3874, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 66, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1189, 0, 1340, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3556, 4096, 0, 1072, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16816, 16405, 16905, 16604, 16378, 16254, 14793, 16917] avg=16384.0000 max=16917 max_rank=7 min=14793 min_rank=6 imbalance_ratio=1.032532 max_load_reduction=114155 max_load_reduction_pct=0.870934 moved_to_echo=64342 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3701, 4096, 0, 0, 0, 0, 0, 4096, 0, 637, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 567, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 1730, 0, 0, 3, 0, 4096, 0, 2781, 0, 2167, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 864, 0, 0, 0, 2962, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 4095, 0, 0, 2, 4096, 28, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4059, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 66, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1189, 0, 3333, 0, 4094, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2784, 4096, 0, 1072, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16688, 16599, 15151, 16853, 16387, 16439, 16811, 16144] avg=16384.0000 max=16853 max_rank=3 min=15151 min_rank=2 imbalance_ratio=1.028625 max_load_reduction=114219 max_load_reduction_pct=0.871422 moved_to_echo=65781 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3861, 4094, 1, 1, 1, 0, 0, 4095, 6, 1377, 2, 1, 4095, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3733, 4094, 1, 0, 1, 0, 0, 4095, 1, 1, 32, 1, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3755, 1, 1, 1, 1, 1, 1228, 1, 0, 19, 0, 4095, 0, 3729, 1, 2168, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8, 4095, 4095, 0, 4095, 176, 0, 0, 0, 2968, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4065, 0, 0, 1, 1, 0, 4080, 1, 0, 118, 4036, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 1, 0, 4009, 0, 1, 1, 1, 4095, 0, 0, 4094, 1, 1, 1, 71, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 24, 2427, 0, 4065, 0, 1, 0, 1, 0, 0, 0, 4094, 0, 4053, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 1, 0, 3979, 4095, 0, 1062, 1, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17600, 16056, 15293, 16184, 16398, 16365, 15847, 17329] avg=16384.0000 max=17600 max_rank=0 min=15293 min_rank=2 imbalance_ratio=1.074219 max_load_reduction=113472 max_load_reduction_pct=0.865723 moved_to_echo=65939 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 3028, 4095, 0, 0, 0, 0, 1, 4096, 0, 392, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 4096, 0, 0, 0, 0, 0, 4081, 0, 0, 275, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 5, 0, 0, 3323, 0, 0, 0, 0, 4096, 3, 3318, 0, 2167, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4095, 4090, 0, 4095, 317, 1, 0, 0, 2962, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 1, 0, 0, 4091, 0, 0, 0, 4096, 19, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 3993, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 66, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1189, 0, 3298, 1, 4075, 0, 0, 0, 0, 1, 0, 9, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4096, 0, 0, 0, 0, 2534, 4095, 0, 1071, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15772, 16292, 17286, 16314, 16372, 16377, 16765, 15894] avg=16384.0000 max=17286 max_rank=2 min=15772 min_rank=0 imbalance_ratio=1.055054 max_load_reduction=113786 max_load_reduction_pct=0.868118 moved_to_echo=65408 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2018, 4096, 0, 0, 0, 0, 0, 4096, 0, 627, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 4096, 0, 0, 0, 0, 0, 4091, 0, 0, 602, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 4, 0, 0, 3130, 0, 0, 0, 0, 4096, 0, 3842, 0, 2167, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 4096, 0, 4096, 58, 0, 0, 0, 2962, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 4090, 0, 0, 6, 4096, 35, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4054, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 66, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1189, 0, 2984, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3164, 4096, 0, 1072, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14995, 16629, 17596, 16060, 16393, 16435, 16440, 16524] avg=16384.0000 max=17596 max_rank=2 min=14995 min_rank=0 imbalance_ratio=1.073975 max_load_reduction=113476 max_load_reduction_pct=0.865753 moved_to_echo=65792 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3377, 4096, 0, 0, 0, 0, 0, 4096, 0, 231, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 4096, 0, 0, 0, 0, 0, 4075, 0, 0, 733, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 15, 0, 0, 3020, 0, 0, 0, 0, 4096, 3, 3394, 0, 2167, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4094, 4095, 0, 4095, 409, 1, 0, 0, 2962, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 1, 0, 0, 4087, 0, 0, 0, 4096, 38, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 0, 0, 4015, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 66, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1189, 0, 3232, 0, 4068, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 2026, 4096, 0, 1072, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15958, 16744, 17076, 16434, 16386, 16400, 16687, 15387] avg=16384.0000 max=17076 max_rank=2 min=15387 min_rank=7 imbalance_ratio=1.042236 max_load_reduction=113996 max_load_reduction_pct=0.869720 moved_to_echo=64860 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9600, 128, 23552, 128, 32640, 0, 128, 0, 128, 128, 0, 128, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[29952, 32768, 128, 0, 128, 0, 0, 32768, 128, 128, 2944, 128, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 0, 27520, 32768, 128, 128, 128, 0, 128, 32768, 128, 4864, 128, 128, 32768, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 0, 32128, 0, 128, 128, 128, 32768, 0, 0, 32768, 128, 128, 128, 640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 128, 32768, 32768, 0, 32768, 3072, 128, 0, 0, 23808, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 0, 24320, 32768, 0, 8576, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 32640, 0, 0, 128, 128, 0, 32768, 128, 0, 256, 32768, 256, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32384, 128, 128, 128, 128, 128, 18688, 128, 0, 128, 0, 32768, 128, 27776, 128, 17408, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1600, 0, 0, 4095, 0, 0, 4080, 0, 0, 0, 0, 46, 0, 0, 0, 4096, 0, 4094, 0, 94, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 4096, 4096, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 3, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2602, 4090, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 13, 0, 0, 6, 0, 1, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4091, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4085, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4093, 3912, 2, 0, 4052, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[3398, 0, 0, 4094, 1, 1, 4095, 0, 0, 0, 1, 0, 0, 0, 1, 4092, 29, 3013, 2, 695, 0, 4094, 4034, 0, 1, 1, 4095, 0, 329, 0, 0, 0, 4094, 0, 1, 4095, 4095, 4095, 0, 1, 0, 1, 0, 1, 102, 0, 0, 1, 1, 0, 1, 0, 2, 1, 1, 1, 4095, 1, 0, 4019, 0, 0, 1, 4095, 1, 0, 0, 0, 3, 1893, 4095, 9, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 4095, 4095, 1, 54, 0, 0, 0, 0, 0, 1, 4088, 4095, 0, 0, 4094, 1, 1, 3559, 1, 3567, 0, 1, 1, 4095, 0, 0, 0, 4093, 1, 4095, 34, 4095, 4095, 4095, 0, 0, 1, 1, 4095, 3935, 0, 1, 4095, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1934, 0, 0, 4096, 0, 0, 4092, 3, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 4095, 0, 139, 0, 4096, 4080, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2601, 4093, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 17, 0, 0, 4, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4092, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4094, 3528, 3, 0, 4082, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1885, 0, 0, 4096, 0, 0, 4090, 12, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 4094, 0, 117, 0, 4096, 4067, 0, 0, 0, 4096, 0, 1, 1, 0, 0, 4096, 0, 0, 4081, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2670, 4086, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 15, 0, 0, 2, 0, 2, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 3971, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4088, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 3707, 6, 0, 4051, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1377, 0, 0, 4095, 0, 0, 4070, 7, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 4091, 0, 256, 0, 4096, 4034, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 4057, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 6, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2676, 4070, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 65, 0, 0, 31, 0, 5, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4088, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 4073, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4091, 3916, 26, 0, 4070, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[2416, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 4094, 0, 25, 0, 4096, 4081, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2094, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 21, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 3704, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4086, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 4054, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[673, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4095, 0, 58, 0, 4096, 4088, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 1, 0, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3470, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4095, 0, 4045, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4095, 4062, 0, 0, 4063, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1195, 0, 0, 4095, 0, 0, 4090, 1, 0, 0, 0, 11, 0, 0, 0, 4096, 1, 4095, 0, 146, 0, 4095, 4084, 0, 0, 0, 4094, 0, 0, 5, 0, 0, 4096, 0, 0, 4091, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3186, 4092, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 24, 0, 0, 10, 0, 2, 0, 4096, 4096, 0, 1, 4096, 0, 0, 4096, 0, 3986, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4083, 0, 4095, 4095, 4096, 0, 0, 0, 0, 4094, 3777, 4, 0, 4077, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1600, 0, 0, 4095, 0, 0, 4080, 0, 0, 0, 0, 46, 0, 0, 0, 4096, 2168, 73, 34, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 94, 0, 4023, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 4096, 4093, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 3, 4095, 0, 0, 0, 4096, 2430, 1669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2602, 4090, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 4096, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4094, 0, 13, 0, 0, 6, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4091, 1, 0, 0, 220, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 259, 0, 0, 0, 0, 0, 4093, 3912, 2, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16195, 16401, 16374, 16392, 16339, 16368, 16600, 16403] avg=16384.0000 max=16600 max_rank=6 min=16195 min_rank=0 imbalance_ratio=1.013184 max_load_reduction=114472 max_load_reduction_pct=0.873352 moved_to_echo=65710 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1934, 0, 0, 4096, 0, 0, 4092, 3, 0, 0, 0, 7, 0, 0, 0, 4096, 2168, 73, 34, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 139, 0, 4023, 4080, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 4096, 4093, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2430, 1669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2601, 4093, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 4096, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4096, 0, 17, 0, 0, 4, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4094, 0, 0, 0, 220, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 259, 0, 0, 0, 0, 0, 4094, 3528, 3, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16506, 16433, 16375, 16391, 16336, 16371, 16602, 16058] avg=16384.0000 max=16602 max_rank=6 min=16058 min_rank=7 imbalance_ratio=1.013306 max_load_reduction=114470 max_load_reduction_pct=0.873337 moved_to_echo=65367 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3398, 0, 0, 4094, 1, 1, 4095, 0, 0, 0, 1, 0, 0, 0, 1, 4092, 2172, 75, 41, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 3013, 2, 695, 0, 4019, 4034, 0, 1, 1, 4095, 0, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 4095, 4095, 4091, 0, 1, 0, 1, 0, 1, 102, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 1, 1, 1, 4095, 1, 0, 4019, 0, 0, 1, 4095, 2430, 1672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 1893, 4095, 9, 1, 0, 0, 1, 0, 0, 0, 1, 4095, 4095, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4054, 4095, 1, 54, 0, 0, 0, 0, 0, 1, 4088, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 3559, 1, 3567, 0, 1, 1, 214, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 34, 0, 251, 0, 0, 0, 1, 1, 4095, 3935, 0, 1, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17975, 16218, 16482, 16320, 15645, 16390, 15533, 16509] avg=16384.0000 max=17975 max_rank=0 min=15533 min_rank=6 imbalance_ratio=1.097107 max_load_reduction=113097 max_load_reduction_pct=0.862862 moved_to_echo=64077 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2416, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 2168, 73, 34, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 25, 0, 4023, 4081, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2430, 1669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2094, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4096, 0, 21, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3704, 0, 0, 0, 220, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 259, 0, 0, 0, 0, 0, 4096, 4054, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16990, 16319, 16373, 16391, 15825, 16371, 16212, 16591] avg=16384.0000 max=16990 max_rank=0 min=15825 min_rank=4 imbalance_ratio=1.036987 max_load_reduction=114082 max_load_reduction_pct=0.870377 moved_to_echo=64999 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[673, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 2168, 73, 34, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 58, 0, 4023, 4088, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4095, 4096, 4093, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2430, 1669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3470, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 4045, 0, 0, 0, 220, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 259, 0, 0, 0, 0, 0, 4095, 4062, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15240, 16361, 16384, 16388, 17204, 16369, 16552, 16574] avg=16384.0000 max=17204 max_rank=4 min=15240 min_rank=0 imbalance_ratio=1.050049 max_load_reduction=113868 max_load_reduction_pct=0.868744 moved_to_echo=66699 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1195, 0, 0, 4095, 0, 0, 4090, 1, 0, 0, 0, 11, 0, 0, 0, 4096, 2167, 73, 34, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 146, 0, 4022, 4084, 0, 0, 0, 4094, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 4096, 2430, 1668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3186, 4092, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4095, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4095, 0, 24, 0, 0, 10, 0, 2, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 3986, 1, 0, 0, 220, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 260, 0, 0, 0, 0, 0, 4094, 3777, 4, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15765, 16447, 16376, 16392, 16916, 16385, 16496, 16295] avg=16384.0000 max=16916 max_rank=4 min=15765 min_rank=0 imbalance_ratio=1.032471 max_load_reduction=114156 max_load_reduction_pct=0.870941 moved_to_echo=66092 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1885, 0, 0, 4096, 0, 0, 4090, 12, 0, 0, 0, 3, 0, 0, 0, 4096, 2168, 73, 34, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 117, 0, 4023, 4067, 0, 0, 0, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4081, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2430, 1669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2670, 4086, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 4096, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4096, 0, 15, 0, 0, 2, 0, 2, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3971, 0, 0, 0, 220, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 259, 0, 0, 0, 0, 0, 4096, 3707, 6, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16460, 16399, 16366, 16393, 16399, 16369, 16479, 16207] avg=16384.0000 max=16479 max_rank=6 min=16207 min_rank=7 imbalance_ratio=1.005798 max_load_reduction=114593 max_load_reduction_pct=0.874275 moved_to_echo=65454 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1377, 0, 0, 4095, 0, 0, 4070, 7, 0, 0, 0, 18, 0, 0, 0, 4096, 2168, 73, 34, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 256, 0, 4023, 4034, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4057, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 6, 4096, 0, 0, 0, 4096, 2430, 1669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2676, 4070, 0, 0, 0, 0, 0, 24, 0, 0, 0, 4096, 4096, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4094, 0, 65, 0, 0, 31, 0, 5, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4088, 2, 0, 0, 220, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 259, 0, 0, 0, 0, 0, 4091, 3916, 26, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15941, 16494, 16342, 16405, 16408, 16449, 16598, 16435] avg=16384.0000 max=16598 max_rank=6 min=15941 min_rank=0 imbalance_ratio=1.013062 max_load_reduction=114474 max_load_reduction_pct=0.873367 moved_to_echo=65890 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 128, 32768, 128, 128, 32768, 0, 0, 128, 32768, 19456, 13440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[14592, 0, 0, 32768, 128, 128, 32768, 128, 0, 0, 128, 128, 0, 0, 128, 32768, 17408, 640, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 0, 128, 21248, 32768, 128, 128, 0, 0, 128, 128, 0, 0, 128, 32768, 32768, 11648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 31744, 128, 1536, 0, 32256, 32640, 0, 128, 128, 32768, 0, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 128, 32512, 32768, 128, 256, 0, 0, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 128, 32256, 128, 31616, 128, 128, 128, 1792, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 32768, 32768, 32768, 0, 128, 0, 128, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 0, 2176, 0, 0, 0, 128, 128, 32768, 30976, 128, 128, 32640, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4090, 0, 0, 0, 4096, 0, 5, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 3821, 19, 3983, 4096, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1753, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 1, 4096, 4089, 4096, 348, 267, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2090, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4089, 0, 0, 0, 4096, 0, 2, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 3911, 4, 3960, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4012, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 2089, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4095, 0, 0, 0, 0, 4096, 4089, 4096, 680, 951, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 778, 4096, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4088, 0, 0, 0, 4096, 0, 1, 6, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 3692, 22, 3869, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4055, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 1593, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 995, 199, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1952, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 0, 5, 407, 1, 1, 0, 1, 0, 0, 10, 4049, 1, 1, 0, 4093, 0, 0, 4095, 4095, 4094, 0, 0, 1, 4092, 1, 0, 0, 0, 2, 4095, 1, 52, 1, 1, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 2413, 0, 3918, 4095, 1, 0, 1, 4095, 0, 1, 0, 0, 4095, 1, 1, 4093, 4048, 0, 4023, 0, 7, 1, 1, 1, 0, 0, 0, 4094, 0, 54, 316, 0, 0, 0, 1, 1, 0, 4093, 1, 4095, 0, 4069, 1, 18, 1, 0, 4095, 4085, 4095, 490, 3089, 0, 0, 1, 0, 0, 4095, 0, 1, 0, 0, 4094, 1, 4093, 1, 1, 1, 4095, 1, 4088, 1, 0, 0, 0, 0, 2, 4095, 1728, 4095, 0, 1, 0, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4092, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 3883, 9, 3931, 4096, 0, 10, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1114, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4095, 4096, 609, 558, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2169, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 3925, 48, 3580, 4096, 0, 19, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1694, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4092, 0, 0, 0, 0, 4096, 4096, 4096, 735, 245, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2042, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4086, 0, 0, 0, 4096, 0, 8, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 4091, 0, 0, 0, 0, 3962, 40, 3842, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4053, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2470, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4090, 4096, 875, 85, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 1068, 4094, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4094, 4077, 0, 0, 0, 4096, 0, 4, 6, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0, 0, 3981, 23, 3725, 4096, 0, 13, 0, 4096, 1, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 0, 0, 2331, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 1, 4095, 4095, 4096, 996, 136, 0, 0, 0, 1, 0, 4092, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 1066, 4095, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 4089, 0, 0, 0, 4096, 0, 2, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 3911, 4, 3960, 2976, 1731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4012, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 2089, 0, 0, 0, 0, 4096, 4096, 2362, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2694, 0, 4095, 0, 0, 0, 0, 0, 4089, 0, 680, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 1609, 1401, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 2487, 778, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16334, 16387, 16681, 16332, 16797, 16605, 16383, 15553] avg=16384.0000 max=16797 max_rank=4 min=15553 min_rank=7 imbalance_ratio=1.025208 max_load_reduction=114275 max_load_reduction_pct=0.871849 moved_to_echo=65338 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 4090, 0, 0, 0, 4096, 0, 5, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 3821, 19, 3983, 2976, 1731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1753, 0, 0, 0, 0, 4096, 4096, 2362, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2695, 0, 4096, 0, 0, 0, 1, 0, 4089, 0, 348, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 1609, 1401, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2487, 2090, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16342, 16387, 16631, 16415, 16456, 15592, 16384, 16865] avg=16384.0000 max=16865 max_rank=7 min=15592 min_rank=5 imbalance_ratio=1.029358 max_load_reduction=114207 max_load_reduction_pct=0.871330 moved_to_echo=65297 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 4088, 0, 0, 0, 4096, 0, 1, 6, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 3692, 22, 3869, 2976, 1731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4055, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 1593, 0, 0, 0, 0, 4096, 4096, 2362, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2695, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 995, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1609, 1401, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2487, 1952, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16333, 16385, 16390, 16375, 16300, 16177, 16385, 16727] avg=16384.0000 max=16727 max_rank=7 min=16177 min_rank=5 imbalance_ratio=1.020935 max_load_reduction=114345 max_load_reduction_pct=0.872383 moved_to_echo=65589 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4092, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 3883, 9, 3931, 2976, 1731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1114, 0, 0, 0, 0, 4096, 4096, 2364, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2695, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 609, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1609, 1401, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2487, 2169, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16339, 16385, 16627, 16425, 15818, 16149, 16385, 16944] avg=16384.0000 max=16944 max_rank=7 min=15818 min_rank=4 imbalance_ratio=1.034180 max_load_reduction=114128 max_load_reduction_pct=0.870728 moved_to_echo=65296 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 5, 407, 1, 1, 0, 1, 0, 0, 10, 4049, 1, 1, 0, 4095, 4095, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4094, 0, 0, 1, 4092, 1, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 52, 1, 1, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 2413, 0, 3918, 2979, 1734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 1, 0, 1, 0, 0, 1, 0, 0, 4095, 1, 1, 4093, 4048, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 1, 1, 0, 0, 0, 4094, 0, 54, 316, 0, 0, 0, 1, 4095, 4095, 2361, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 1, 2691, 0, 4069, 1, 18, 1, 0, 0, 4085, 0, 490, 3089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 1, 0, 0, 4094, 1, 4093, 1, 1, 1, 1609, 1404, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 1, 0, 0, 0, 0, 2, 2486, 1728, 4095, 0, 1, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16710, 16378, 15197, 16289, 15078, 18539, 16392, 16489] avg=16384.0000 max=18539 max_rank=5 min=15078 min_rank=4 imbalance_ratio=1.131531 max_load_reduction=112533 max_load_reduction_pct=0.858559 moved_to_echo=66498 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 8, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4091, 0, 0, 0, 0, 3962, 40, 3842, 2976, 1734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4053, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2470, 0, 0, 0, 0, 4096, 4096, 2362, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2695, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 875, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 1609, 1401, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 2487, 1068, 4094, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16332, 16386, 16653, 16373, 17179, 15937, 16377, 15835] avg=16384.0000 max=17179 max_rank=4 min=15835 min_rank=7 imbalance_ratio=1.048523 max_load_reduction=113893 max_load_reduction_pct=0.868935 moved_to_echo=65328 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 4077, 0, 0, 0, 4096, 0, 4, 6, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0, 0, 3981, 23, 3725, 2976, 1730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 13, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 0, 0, 2331, 0, 0, 0, 0, 4096, 4095, 2361, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2694, 0, 4096, 0, 0, 0, 1, 0, 4095, 0, 996, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 1609, 1401, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2487, 1066, 4095, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16351, 16376, 16533, 16429, 17047, 16114, 16380, 15842] avg=16384.0000 max=17047 max_rank=4 min=15842 min_rank=7 imbalance_ratio=1.040466 max_load_reduction=114025 max_load_reduction_pct=0.869942 moved_to_echo=65383 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 4095, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 3925, 48, 3580, 2976, 1731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 19, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1694, 0, 0, 0, 0, 4096, 4096, 2362, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2695, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 735, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1609, 1401, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2487, 2042, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16331, 16388, 16360, 16434, 16397, 15959, 16386, 16817] avg=16384.0000 max=16817 max_rank=7 min=15959 min_rank=5 imbalance_ratio=1.026428 max_load_reduction=114255 max_load_reduction_pct=0.871696 moved_to_echo=65559 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 0, 0, 128, 32768, 0, 0, 128, 32768, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 128, 128, 0, 0, 0, 128, 19968, 12928, 32768, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 128, 0, 0, 128, 32768, 0, 128, 13440, 0, 0, 0, 128, 32768, 32768, 18944, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 128, 128, 128, 0, 128, 128, 0, 0, 32768, 128, 128, 32768, 32640, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 512, 128, 128, 0, 128, 0, 0, 128, 32768, 128, 128, 0, 32768, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 32768, 128, 21632, 0, 32768, 128, 128, 128, 128, 0, 32768, 0, 5760, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 0, 32768, 0, 128, 128, 128, 29696, 256, 30848, 23936, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 0, 32768, 0, 128, 0, 0, 32768, 128, 32768, 128, 128, 128, 12928, 11264, 8832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 4092, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3367, 0, 0, 0, 805, 3991, 0, 7, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 13, 0, 0, 5, 0, 4086, 4096, 0, 0, 21, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4091, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2457, 0, 0, 0, 1457, 4050, 0, 189, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 2, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 9, 0, 0, 36, 0, 4093, 4096, 0, 0, 6, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4090, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 4088, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3737, 0, 0, 0, 407, 3958, 0, 19, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 5, 0, 0, 0, 4096, 4095, 2, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 54, 0, 0, 17, 0, 4062, 4096, 0, 0, 45, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3757, 0, 0, 0, 298, 4035, 0, 48, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 29, 0, 0, 4, 0, 4091, 4096, 0, 0, 34, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4095, 1, 0, 4039, 241, 27, 0, 4094, 4095, 4066, 1, 4095, 4089, 1, 4095, 1, 0, 0, 0, 4095, 0, 0, 4, 4083, 0, 4073, 1, 10, 4093, 1, 0, 0, 4095, 1, 1, 0, 0, 3102, 1, 1, 1, 501, 4092, 0, 92, 4012, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 11, 1, 1, 0, 4087, 1, 4095, 0, 1, 0, 0, 1, 0, 4087, 3685, 4, 1, 0, 0, 0, 0, 4094, 4095, 0, 1, 0, 0, 1, 4089, 0, 0, 1, 0, 1, 4095, 1, 4088, 11, 0, 1, 778, 0, 4095, 4072, 0, 28, 101, 0, 1, 2, 0, 3822, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 3, 127, 4095, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2771, 0, 0, 0, 1157, 4068, 0, 154, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 28, 0, 0, 11, 0, 4092, 4096, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4088, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4079, 0, 4091, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3082, 0, 0, 0, 799, 4047, 0, 201, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 8, 0, 0, 0, 4096, 4094, 8, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 71, 0, 0, 19, 0, 4064, 4096, 0, 0, 37, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4086, 0, 4095, 0, 0, 0, 0, 4096, 2, 0, 0, 4095, 0, 4090, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3479, 0, 0, 0, 478, 4057, 0, 158, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 5, 0, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 4095, 4095, 1, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 29, 0, 0, 10, 0, 4085, 4096, 0, 0, 6, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4074, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 15, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2457, 0, 0, 0, 1457, 4050, 0, 98, 76, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3998, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 4094, 34, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 36, 0, 4093, 4096, 0, 0, 6, 0, 0, 0, 0, 4093, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16361, 16391, 16352, 16475, 16434, 16387, 16304, 16368] avg=16384.0000 max=16475 max_rank=3 min=16304 min_rank=6 imbalance_ratio=1.005554 max_load_reduction=114597 max_load_reduction_pct=0.874306 moved_to_echo=65493 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4074, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 15, 0, 0, 0, 4093, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3367, 0, 0, 0, 805, 3991, 0, 98, 76, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3998, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4094, 34, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 5, 0, 4086, 4096, 0, 0, 21, 0, 0, 0, 0, 4093, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16362, 16392, 16551, 16293, 16434, 16386, 16285, 16369] avg=16384.0000 max=16551 max_rank=2 min=16285 min_rank=6 imbalance_ratio=1.010193 max_load_reduction=114521 max_load_reduction_pct=0.873726 moved_to_echo=65474 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4074, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 15, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3757, 0, 0, 0, 298, 4035, 0, 98, 76, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3998, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 34, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 4, 0, 4091, 4096, 0, 0, 34, 0, 0, 0, 0, 4096, 0, 3970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16362, 16397, 16478, 16334, 16432, 16386, 16320, 16363] avg=16384.0000 max=16478 max_rank=2 min=16320 min_rank=6 imbalance_ratio=1.005737 max_load_reduction=114594 max_load_reduction_pct=0.874283 moved_to_echo=65501 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4074, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 15, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2771, 0, 0, 0, 1157, 4068, 0, 98, 76, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3998, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 34, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 11, 0, 4092, 4096, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16362, 16398, 16384, 16440, 16433, 16384, 16304, 16367] avg=16384.0000 max=16440 max_rank=3 min=16304 min_rank=6 imbalance_ratio=1.003418 max_load_reduction=114632 max_load_reduction_pct=0.874573 moved_to_echo=65488 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4074, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4095, 0, 0, 0, 0, 15, 2, 0, 0, 4095, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3479, 0, 0, 0, 478, 4057, 0, 98, 76, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 3998, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 5, 0, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 4094, 34, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 1, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 10, 0, 4085, 4096, 0, 0, 6, 0, 0, 0, 0, 4094, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16361, 16383, 16402, 16443, 16438, 16386, 16291, 16368] avg=16384.0000 max=16443 max_rank=3 min=16291 min_rank=6 imbalance_ratio=1.003601 max_load_reduction=114629 max_load_reduction_pct=0.874550 moved_to_echo=65483 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4074, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 15, 0, 0, 0, 4094, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3737, 0, 0, 0, 407, 3958, 0, 98, 76, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3998, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 4096, 4095, 2, 0, 0, 0, 0, 0, 4094, 34, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 17, 0, 4062, 4096, 0, 0, 45, 0, 0, 0, 0, 4096, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16362, 16383, 16490, 16305, 16436, 16386, 16341, 16369] avg=16384.0000 max=16490 max_rank=2 min=16305 min_rank=3 imbalance_ratio=1.006470 max_load_reduction=114582 max_load_reduction_pct=0.874191 moved_to_echo=65532 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4074, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 15, 0, 0, 0, 4079, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3082, 0, 0, 0, 799, 4047, 0, 98, 76, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3998, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 8, 0, 0, 0, 4096, 4094, 8, 0, 0, 0, 0, 0, 4096, 34, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 19, 0, 4064, 4096, 0, 0, 37, 0, 0, 0, 0, 4081, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16361, 16369, 16316, 16487, 16445, 16386, 16339, 16369] avg=16384.0000 max=16487 max_rank=3 min=16316 min_rank=2 imbalance_ratio=1.006287 max_load_reduction=114585 max_load_reduction_pct=0.874214 moved_to_echo=65539 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4095, 1, 0, 4039, 241, 27, 0, 4068, 0, 4066, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 4095, 1, 0, 0, 0, 2, 0, 0, 4, 4083, 0, 4073, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 4095, 1, 1, 0, 0, 3102, 1, 1, 1, 501, 4092, 0, 103, 81, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 4012, 1, 1, 1, 1, 4095, 0, 0, 0, 3992, 11, 1, 1, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 0, 4087, 3685, 4, 1, 0, 0, 0, 0, 4095, 35, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 0, 1, 0, 0, 1, 4089, 0, 0, 1, 0, 1, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1, 778, 0, 4095, 4072, 0, 28, 101, 0, 1, 2, 0, 3822, 0, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 3, 127, 4095, 1, 0, 4095, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16541, 16359, 16099, 16295, 16020, 16371, 16888, 16499] avg=16384.0000 max=16888 max_rank=6 min=16020 min_rank=4 imbalance_ratio=1.030762 max_load_reduction=114184 max_load_reduction_pct=0.871155 moved_to_echo=65778 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[256, 0, 128, 896, 0, 32768, 32768, 0, 128, 384, 0, 128, 128, 0, 32512, 0, 31872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[896, 32768, 128, 128, 128, 128, 32768, 0, 0, 0, 32000, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 32768, 128, 0, 0, 0, 128, 128, 0, 128, 32768, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 128, 0, 128, 128, 32768, 0, 0, 128, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 0, 0, 32768, 128, 128, 0, 0, 25856, 128, 128, 128, 6016, 32384, 0, 896, 640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 32768, 256, 128, 0, 32640, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 0, 0, 128, 32768, 0, 128, 128, 128, 128, 128, 32768, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 128, 0, 32768, 32384, 128, 128, 0, 0, 0, 0, 32768, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 26, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 4096, 0, 0, 2, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 23, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 4096, 0, 0, 10, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 98, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 2, 0, 0, 4096, 0, 29, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 12, 0, 4096, 0, 0, 1, 4096, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 4038, 4095, 0, 4096, 0, 0, 14, 0, 0, 4095, 0, 4096, 0, 4096, 4095, 0, 0, 1, 4096, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 31, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 16, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4051, 4096, 0, 4096, 0, 0, 21, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 10, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 4096, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 18, 0, 4096, 0, 0, 1, 4096, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 0, 4096, 0, 0, 2, 0, 0, 4095, 0, 4096, 0, 4096, 4095, 0, 0, 1, 4096, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 4093, 19, 0, 26, 0, 4067, 1, 0, 4095, 1, 0, 1, 4094, 1, 0, 0, 4086, 0, 0, 1, 1, 4095, 4094, 0, 1, 4095, 4079, 4094, 0, 1, 1, 1, 1, 1, 0, 1, 1, 5, 7, 2, 0, 212, 1, 0, 3996, 0, 0, 1, 4094, 1, 4045, 27, 0, 0, 4092, 4095, 1, 1, 1, 0, 0, 1, 4037, 9, 1, 6, 4086, 4091, 1, 0, 0, 1, 1, 0, 0, 1, 4095, 1, 1, 1, 10, 241, 0, 0, 4083, 4088, 0, 4072, 0, 0, 0, 1, 0, 3954, 1, 4095, 0, 4073, 4095, 0, 0, 41, 4095, 1, 4092, 0, 0, 0, 1, 1, 0, 1, 3973, 3, 0, 0, 1, 4095, 4093, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 48, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 4096, 0, 14, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4052, 4096, 0, 4096, 0, 0, 14, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 27, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 13, 0, 4096, 0, 0, 1, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4090, 4095, 0, 4095, 0, 0, 15, 0, 0, 4096, 0, 4096, 0, 4096, 4095, 0, 0, 0, 4096, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4062, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6, 0, 0, 4096, 4096, 4096, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 34, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4090, 4092, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 47, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4080, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16376, 16389, 16353, 16395, 16388, 16368, 16403, 16400] avg=16384.0000 max=16403 max_rank=6 min=16353 min_rank=2 imbalance_ratio=1.001160 max_load_reduction=114669 max_load_reduction_pct=0.874855 moved_to_echo=65559 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4062, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 4096, 4096, 4096, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4096, 0, 0, 1, 4096, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 4049, 34, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4094, 4091, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4095, 0, 0, 1, 47, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4080, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16360, 16389, 16348, 16398, 16401, 16376, 16404, 16396] avg=16384.0000 max=16404 max_rank=6 min=16348 min_rank=2 imbalance_ratio=1.001221 max_load_reduction=114668 max_load_reduction_pct=0.874847 moved_to_echo=65577 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4062, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 4096, 4096, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 34, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 4088, 4092, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 47, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4080, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 16389, 16356, 16395, 16380, 16367, 16410, 16402] avg=16384.0000 max=16410 max_rank=6 min=16356 min_rank=2 imbalance_ratio=1.001587 max_load_reduction=114662 max_load_reduction_pct=0.874802 moved_to_echo=65559 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4062, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 16, 0, 0, 4096, 4096, 4096, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 34, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4051, 4092, 0, 4096, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 47, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4080, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16386, 16364, 16395, 16384, 16355, 16402, 16405] avg=16384.0000 max=16405 max_rank=7 min=16355 min_rank=5 imbalance_ratio=1.001282 max_load_reduction=114667 max_load_reduction_pct=0.874840 moved_to_echo=65546 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4062, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 4096, 4096, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4096, 0, 0, 1, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4049, 34, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4090, 4091, 0, 4095, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 47, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4080, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 16388, 16353, 16396, 16395, 16373, 16391, 16398] avg=16384.0000 max=16398 max_rank=7 min=16353 min_rank=2 imbalance_ratio=1.000854 max_load_reduction=114674 max_load_reduction_pct=0.874893 moved_to_echo=65557 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4062, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 2, 0, 0, 4096, 4096, 4096, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 29, 0, 4096, 0, 4096, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4096, 0, 0, 1, 4096, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 4049, 34, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 4038, 4091, 0, 4096, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4095, 0, 0, 1, 47, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4080, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16448, 16384, 16362, 16424, 16398, 16306, 16351, 16399] avg=16384.0000 max=16448 max_rank=0 min=16306 min_rank=5 imbalance_ratio=1.003906 max_load_reduction=114624 max_load_reduction_pct=0.874512 moved_to_echo=65454 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4054, 0, 0, 1, 4093, 19, 0, 26, 0, 4067, 1, 0, 4095, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 4086, 0, 0, 1, 1, 4095, 0, 0, 1, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 5, 7, 2, 0, 212, 1, 0, 4095, 4095, 4094, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 1, 4094, 1, 4045, 27, 0, 0, 4092, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4037, 9, 1, 6, 4086, 4091, 1, 0, 0, 1, 1, 0, 0, 1, 4053, 41, 7, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 1, 10, 241, 0, 0, 4083, 4079, 0, 4072, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 1, 0, 0, 4073, 4095, 0, 0, 41, 42, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 3973, 3, 0, 0, 1, 4095, 4093, 0, 1, 0, 0, 0, 4087, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16358, 16358, 16578, 16260, 16345, 16584, 16299, 16290] avg=16384.0000 max=16584 max_rank=5 min=16260 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=114488 max_load_reduction_pct=0.873474 moved_to_echo=65518 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4062, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 4096, 4096, 4096, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 14, 0, 4096, 0, 4096, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 34, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4052, 4092, 0, 4096, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 47, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4080, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16398, 16389, 16358, 16409, 16381, 16343, 16412, 16382] avg=16384.0000 max=16412 max_rank=6 min=16343 min_rank=5 imbalance_ratio=1.001709 max_load_reduction=114660 max_load_reduction_pct=0.874786 moved_to_echo=65518 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 128, 0, 0, 32768, 0, 0, 128, 128, 32768, 0, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32512, 0, 0, 128, 32768, 128, 128, 128, 0, 32768, 128, 0, 32768, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 32640, 128, 0, 0, 128, 32768, 32768, 0, 128, 0, 0, 0, 32768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 32640, 128, 0, 0, 32768, 32768, 0, 0, 128, 384, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 128, 128, 32768, 128, 32768, 128, 0, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 32768, 32768, 128, 128, 128, 128, 128, 0, 0, 128, 32512, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 128, 256, 0, 0, 32640, 32768, 0, 32768, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 0, 384, 128, 0, 32768, 32768, 32768, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 252, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3436, 3108, 0, 0, 4078, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3409, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4042, 4096, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 4096, 21, 0, 0, 5, 0, 354, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3007, 0, 0, 0, 0, 0, 879, 2001, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 1, 4068, 1, 1, 1, 0, 0, 4094, 1, 1, 1369, 3, 1, 13, 2, 4093, 1, 4095, 0, 0, 0, 0, 4095, 4094, 1, 4087, 1, 1, 13, 4086, 1, 1, 1, 0, 0, 4081, 0, 3, 0, 574, 3580, 4, 2, 3980, 0, 0, 2, 0, 0, 0, 0, 1, 4091, 0, 4095, 4091, 0, 1, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 4083, 1, 1, 1, 7, 3465, 4095, 307, 4, 4095, 1, 4094, 1, 0, 4095, 4080, 4083, 1, 0, 3335, 0, 0, 4095, 0, 1, 0, 4061, 8, 0, 23, 0, 0, 2598, 4068, 0, 0, 1, 4095, 213, 0, 1, 74, 0, 773, 0, 7, 0, 1, 1, 1656, 2727, 4094, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 1361, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4030, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2444, 3184, 0, 0, 4076, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 51, 0, 4096, 4086, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 3540, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4018, 4096, 0, 0, 4035, 0, 0, 4096, 0, 0, 0, 4096, 106, 0, 0, 15, 0, 604, 4096, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 3777, 0, 0, 0, 0, 0, 1209, 320, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1057, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1850, 3170, 0, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 35, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3940, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4037, 4096, 0, 0, 4083, 0, 0, 4096, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 1484, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3717, 0, 0, 0, 0, 0, 524, 796, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 415, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4082, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1397, 3776, 0, 0, 4092, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 21, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3449, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4058, 4096, 0, 0, 4068, 0, 0, 4096, 0, 0, 0, 4096, 23, 0, 0, 0, 0, 2376, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3293, 0, 0, 0, 0, 0, 955, 765, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 670, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4057, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1181, 3335, 0, 0, 4004, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 21, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3419, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4041, 4096, 0, 0, 4055, 0, 0, 4096, 0, 0, 0, 4096, 41, 0, 0, 4, 0, 1162, 4083, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 3467, 0, 0, 0, 0, 0, 833, 2492, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 3592, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4091, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2594, 3391, 0, 0, 3876, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 11, 0, 4096, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3836, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4085, 4096, 0, 0, 4055, 0, 0, 4096, 0, 0, 0, 4096, 60, 0, 0, 5, 0, 289, 4096, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 1285, 0, 0, 0, 0, 0, 967, 630, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 1373, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4072, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 2284, 3503, 0, 0, 4020, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 27, 0, 4096, 4094, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3410, 4095, 0, 0, 4096, 0, 4096, 0, 0, 4095, 4081, 4096, 0, 0, 4091, 0, 0, 4096, 1, 0, 0, 4096, 70, 0, 0, 6, 0, 90, 4090, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3072, 0, 0, 0, 0, 0, 519, 2154, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 252, 0, 0, 0, 0, 4096, 1693, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 0, 0, 0, 4096, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3436, 3108, 0, 0, 4078, 0, 0, 0, 1841, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 4076, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3409, 4096, 2403, 2232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 4096, 4042, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 0, 5, 0, 354, 4096, 0, 0, 0, 4096, 0, 2889, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3007, 0, 0, 0, 0, 0, 879, 2001, 4096, 0, 0, 0, 4096, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15374, 16396, 17642, 16366, 16236, 16447, 15577, 17034] avg=16384.0000 max=17642 max_rank=2 min=15374 min_rank=0 imbalance_ratio=1.076782 max_load_reduction=113430 max_load_reduction_pct=0.865402 moved_to_echo=65294 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4068, 1, 1, 1, 0, 0, 4094, 1, 1, 1369, 3, 1, 13, 2, 4095, 1694, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 16, 0, 0, 0, 0, 0, 4094, 1, 4087, 1, 1, 13, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4081, 0, 3, 0, 574, 3580, 4, 2, 3980, 0, 0, 2, 1847, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 4091, 0, 1, 0, 0, 4074, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 4083, 1, 1, 1, 7, 3465, 4095, 2401, 2232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 4, 0, 1, 116, 1, 0, 4095, 4080, 4083, 1, 0, 3335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4061, 8, 0, 23, 0, 0, 2598, 4068, 0, 0, 1, 4095, 213, 2894, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 74, 0, 773, 0, 7, 0, 1, 1, 1656, 2727, 4094, 0, 1, 0, 4095, 2953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16487, 16394, 15159, 16353, 16290, 16023, 17983, 16383] avg=16384.0000 max=17983 max_rank=6 min=15159 min_rank=2 imbalance_ratio=1.097595 max_load_reduction=113089 max_load_reduction_pct=0.862801 moved_to_echo=66679 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1057, 0, 0, 0, 0, 4096, 1694, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1850, 3170, 0, 0, 3973, 0, 0, 0, 1841, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 35, 0, 4076, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3940, 4096, 2402, 2232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 4096, 4037, 4096, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 0, 0, 0, 0, 1484, 4087, 0, 0, 0, 4096, 0, 2889, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3717, 0, 0, 0, 0, 0, 524, 796, 4096, 0, 0, 0, 4096, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16180, 16397, 16013, 16399, 16766, 16436, 16697, 16184] avg=16384.0000 max=16766 max_rank=4 min=16013 min_rank=2 imbalance_ratio=1.023315 max_load_reduction=114306 max_load_reduction_pct=0.872086 moved_to_echo=66083 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 3592, 0, 0, 0, 0, 4096, 1694, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2594, 3391, 0, 0, 3876, 0, 0, 0, 1841, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 11, 0, 4076, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3836, 4096, 2402, 2232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 4096, 4085, 4096, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 60, 0, 0, 5, 0, 289, 4096, 0, 0, 0, 4095, 1, 2889, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1285, 0, 0, 0, 0, 0, 967, 630, 4096, 0, 0, 0, 4096, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18716, 16401, 16881, 16377, 16661, 16456, 15551, 14029] avg=16384.0000 max=18716 max_rank=0 min=14029 min_rank=7 imbalance_ratio=1.142334 max_load_reduction=112356 max_load_reduction_pct=0.857208 moved_to_echo=62697 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 415, 0, 0, 0, 0, 4096, 1694, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 0, 0, 0, 4096, 0, 4082, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1397, 3776, 0, 0, 4092, 0, 0, 0, 1841, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 21, 0, 4076, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3449, 4096, 2402, 2232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 4096, 4058, 4096, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 23, 0, 0, 0, 0, 2376, 4093, 0, 0, 0, 4096, 0, 2889, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3293, 0, 0, 0, 0, 0, 955, 765, 4096, 0, 0, 0, 4096, 2954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15539, 16393, 16285, 16386, 16275, 16442, 17593, 16159] avg=16384.0000 max=17593 max_rank=6 min=15539 min_rank=0 imbalance_ratio=1.073792 max_load_reduction=113479 max_load_reduction_pct=0.865776 moved_to_echo=66469 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 1361, 0, 0, 0, 0, 4096, 1694, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 0, 0, 0, 4096, 0, 4030, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2444, 3184, 0, 0, 4076, 0, 0, 0, 1841, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 51, 0, 4076, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 3540, 4096, 2402, 2232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 4096, 4018, 4096, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 106, 0, 0, 15, 0, 604, 4096, 0, 0, 0, 4095, 1, 2889, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3777, 0, 0, 0, 0, 0, 1209, 320, 4096, 0, 0, 0, 4096, 2954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16488, 16340, 16724, 16415, 16362, 16369, 15922, 16452] avg=16384.0000 max=16724 max_rank=2 min=15922 min_rank=6 imbalance_ratio=1.020752 max_load_reduction=114348 max_load_reduction_pct=0.872406 moved_to_echo=65105 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 1373, 0, 0, 0, 0, 4096, 1693, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 0, 0, 0, 4096, 0, 4072, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 2284, 3503, 0, 0, 4020, 0, 0, 0, 1841, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 27, 0, 4076, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3410, 4095, 2403, 2232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 4095, 4081, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 70, 0, 0, 6, 0, 90, 4090, 0, 0, 0, 4095, 0, 2889, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3072, 0, 0, 0, 0, 0, 519, 2154, 4096, 0, 0, 0, 4096, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16496, 16383, 16828, 16392, 16237, 16487, 15357, 16892] avg=16384.0000 max=16892 max_rank=7 min=15357 min_rank=6 imbalance_ratio=1.031006 max_load_reduction=114180 max_load_reduction_pct=0.871124 moved_to_echo=64973 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 670, 0, 0, 0, 0, 4096, 1693, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 0, 0, 0, 4096, 0, 4057, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1181, 3335, 0, 0, 4004, 0, 0, 0, 1841, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 21, 0, 4076, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3419, 4096, 2403, 2232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 4096, 4041, 4096, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 41, 0, 0, 4, 0, 1162, 4083, 0, 0, 0, 4096, 1, 2889, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3467, 0, 0, 0, 0, 0, 833, 2492, 4096, 0, 0, 0, 4096, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15792, 16368, 15540, 16384, 16245, 16412, 16392, 17939] avg=16384.0000 max=17939 max_rank=7 min=15540 min_rank=2 imbalance_ratio=1.094910 max_load_reduction=113133 max_load_reduction_pct=0.863136 moved_to_echo=66988 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 128, 0, 32768, 128, 128, 10112, 128, 128, 128, 128, 32768, 13568, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 32768, 0, 0, 32768, 0, 128, 256, 0, 32640, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 0, 32768, 384, 0, 128, 128, 0, 8960, 32768, 0, 0, 128, 32768, 256, 23168, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 128, 0, 15872, 27136, 128, 128, 32128, 0, 0, 128, 14848, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 256, 0, 0, 0, 0, 0, 32768, 128, 32640, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 0, 128, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 28544, 32768, 19328, 17920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 22400, 0, 128, 0, 128, 128, 7552, 11904, 32768, 0, 128, 0, 32768, 23680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[384, 128, 0, 128, 1024, 128, 0, 32768, 32512, 32768, 128, 0, 31872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1703, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4039, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1102, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1313, 4096, 0, 0, 4096, 4096, 0, 4094, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 34, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2018, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4058, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1307, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 806, 4096, 0, 0, 4096, 4096, 0, 4093, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4093, 0, 0, 1, 6, 0, 4095, 0, 0, 0, 229, 4095, 0, 4092, 0, 4094, 0, 5, 1, 0, 11, 4095, 0, 2, 1, 1, 0, 0, 0, 1, 4095, 71, 4094, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 2, 0, 2, 4093, 1, 0, 3216, 1, 1, 0, 4093, 1, 3, 1, 0, 1, 0, 0, 0, 20, 0, 1, 19, 1, 1, 1, 669, 4095, 1, 1, 4074, 4086, 1, 4095, 0, 3986, 4092, 131, 4095, 0, 0, 2, 0, 1, 4047, 4087, 0, 0, 3980, 0, 1, 0, 1, 0, 4, 0, 4095, 4095, 4074, 1, 4095, 0, 4088, 0, 4095, 4095, 0, 1, 0, 1, 4095, 65, 0, 4095, 0, 1, 1, 0, 1, 0, 1, 4082] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 684, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1059, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2356, 4096, 0, 0, 4096, 4096, 0, 4096, 1, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 292, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 2, 4030, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 573, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3293, 4095, 0, 0, 4096, 4096, 0, 4093, 2, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4095, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1099, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1968, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1507, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 5, 3951, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1601, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1134, 4091, 0, 0, 4096, 4096, 0, 4086, 10, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 439, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1299, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2369, 4095, 0, 0, 4096, 4096, 0, 4093, 3, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 4096, 1, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4095, 2, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1703, 4096, 1618, 1481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2607, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4039, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2610, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1102, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1313, 2478, 0, 0, 4096, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 34, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2597, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17090, 16381, 16360, 15970, 16077, 16402, 16389, 16403] avg=16384.0000 max=17090 max_rank=0 min=15970 min_rank=3 imbalance_ratio=1.043091 max_load_reduction=113982 max_load_reduction_pct=0.869614 moved_to_echo=65271 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 684, 4096, 1618, 1481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2607, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2610, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1059, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2356, 2478, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2597, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16071, 16381, 16411, 15927, 17122, 16402, 16389, 16369] avg=16384.0000 max=17122 max_rank=4 min=15927 min_rank=3 imbalance_ratio=1.045044 max_load_reduction=113950 max_load_reduction_pct=0.869370 moved_to_echo=66282 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 292, 4096, 1618, 1481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2607, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4030, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2610, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 573, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3293, 2477, 0, 0, 4096, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2597, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15679, 16382, 16350, 15441, 18055, 16404, 16389, 16372] avg=16384.0000 max=18055 max_rank=4 min=15441 min_rank=3 imbalance_ratio=1.101990 max_load_reduction=113017 max_load_reduction_pct=0.862251 moved_to_echo=67220 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1099, 4096, 1618, 1481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2607, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2610, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1968, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 2478, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2597, 1489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16486, 16381, 16412, 16836, 15786, 16402, 16384, 16385] avg=16384.0000 max=16836 max_rank=3 min=15786 min_rank=4 imbalance_ratio=1.027588 max_load_reduction=114236 max_load_reduction_pct=0.871552 moved_to_echo=64957 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4093, 0, 0, 1, 6, 0, 4095, 0, 0, 0, 229, 4095, 1620, 1482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4094, 0, 5, 1, 0, 11, 4095, 0, 2, 1, 1, 0, 0, 2611, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 71, 4094, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 2, 0, 2609, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 1, 0, 3216, 1, 1, 0, 4093, 1, 3, 1, 0, 1, 0, 0, 4095, 2579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 1, 19, 1, 1, 1, 669, 2475, 1, 1, 4074, 4086, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 10, 131, 0, 0, 0, 2, 0, 1, 4047, 4087, 0, 0, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4, 0, 0, 4095, 4074, 1, 0, 0, 4088, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 65, 0, 4095, 0, 1, 1, 0, 1, 0, 1, 4082, 2600, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15623, 16399, 16488, 18087, 15445, 16244, 16359, 16427] avg=16384.0000 max=18087 max_rank=3 min=15445 min_rank=4 imbalance_ratio=1.103943 max_load_reduction=112985 max_load_reduction_pct=0.862007 moved_to_echo=64475 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1507, 4096, 1616, 1481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2610, 1487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 3951, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2609, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1601, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1134, 2475, 0, 0, 4096, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2597, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16892, 16386, 16274, 16468, 15887, 16411, 16384, 16370] avg=16384.0000 max=16892 max_rank=0 min=15887 min_rank=4 imbalance_ratio=1.031006 max_load_reduction=114180 max_load_reduction_pct=0.871124 moved_to_echo=65052 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2018, 4096, 1618, 1481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2607, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4058, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2610, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1307, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 806, 2478, 0, 0, 4096, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2597, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17405, 16381, 16376, 16175, 15569, 16402, 16389, 16375] avg=16384.0000 max=17405 max_rank=0 min=15569 min_rank=4 imbalance_ratio=1.062317 max_load_reduction=113667 max_load_reduction_pct=0.867210 moved_to_echo=64735 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 439, 4096, 1618, 1481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2607, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2610, 1516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1299, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2369, 2477, 0, 0, 4096, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 1, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2597, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15826, 16381, 16401, 16168, 17131, 16405, 16389, 16371] avg=16384.0000 max=17131 max_rank=4 min=15826 min_rank=0 imbalance_ratio=1.045593 max_load_reduction=113941 max_load_reduction_pct=0.869301 moved_to_echo=66296 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 0, 32768, 0, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 0, 0, 20864, 11904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 128, 32768, 0, 128, 128, 0, 128, 0, 128, 32768, 20864, 11904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 0, 12160, 128, 128, 128, 32768, 128, 128, 128, 0, 128, 0, 0, 32768, 20736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 128, 128, 0, 32768, 0, 0, 0, 8064, 32768, 13056, 11904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 32768, 128, 32512, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 128, 0, 20992, 12160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 32768, 32768, 128, 0, 128, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 128, 13056, 19840, 128, 128, 32768, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32768, 256, 256, 0, 0, 0, 128, 0, 128, 32768, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 187, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 4096, 0, 1, 0, 4096, 0, 0, 1265, 4096, 0, 4053, 0, 144, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4096, 1002, 4096, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 1290, 0, 4096, 0, 4092, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 418, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 3935, 0, 0, 0, 4096, 4095, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 224, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 2682, 4096, 0, 4094, 0, 149, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 667, 4096, 4096, 0, 0, 0, 7, 4085, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 390, 0, 4096, 0, 4082, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 739, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 3357, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 137, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1010, 4096, 0, 4072, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2229, 4096, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 611, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 70, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 3896, 0, 0, 0, 4096, 4095, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4082, 396, 1, 175, 12, 1, 0, 0, 4095, 4094, 1, 0, 0, 4095, 0, 0, 1, 4091, 23, 1, 2211, 4095, 0, 3974, 5, 1529, 1, 1, 31, 1, 0, 7, 0, 1, 0, 3, 0, 1, 1, 1, 4, 0, 1, 0, 9, 0, 4091, 4093, 260, 4093, 4095, 0, 0, 0, 9, 3715, 0, 1, 1, 4, 1, 0, 0, 4091, 4, 1, 0, 0, 2, 1, 4082, 4090, 4060, 0, 937, 0, 4094, 0, 3916, 1, 4094, 12, 1, 0, 0, 4087, 1, 1, 4094, 4094, 0, 1, 4095, 0, 0, 1, 4095, 0, 1, 1, 0, 108, 1, 4095, 4021, 0, 0, 4095, 2, 0, 1, 3317, 1, 1, 1, 4095, 4095, 4073] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 276, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1465, 4096, 0, 4084, 0, 948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1331, 4096, 4096, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 378, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 494, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 3336, 0, 0, 0, 4096, 4095, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 57, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 3, 0, 4096, 0, 0, 1620, 4096, 0, 4029, 0, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 4096, 4096, 1097, 4096, 4096, 0, 0, 0, 11, 4068, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 4096, 4096, 4096, 0, 324, 0, 4096, 0, 4078, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 372, 0, 4096, 4095, 0, 0, 4090, 0, 0, 0, 3812, 0, 0, 0, 4096, 4092, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 186, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 3, 0, 4096, 0, 0, 914, 4096, 0, 4061, 0, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4096, 2436, 4096, 4096, 0, 0, 0, 2, 4094, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 4096, 4096, 0, 228, 0, 4096, 0, 4089, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 49, 0, 4096, 4095, 0, 0, 4094, 0, 0, 0, 4056, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 301, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1888, 4096, 0, 4055, 0, 332, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 1821, 4095, 4095, 0, 0, 0, 5, 4093, 1, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 4096, 4096, 0, 319, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 61, 0, 4096, 4094, 0, 0, 4094, 0, 0, 0, 3503, 0, 0, 0, 4096, 4089, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 187, 0, 0, 0, 0, 4096, 4096, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2461, 4096, 0, 0, 1, 0, 0, 1, 0, 4096, 0, 0, 1265, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 144, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3299, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4096, 1002, 4096, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3637, 1635, 1357, 797, 526, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1290, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3808, 0, 0, 0, 4096, 0, 0, 0, 0, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 459, 0, 0, 0, 3935, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16328, 16016, 15933, 16030, 16426, 17144, 16514, 16681] avg=16384.0000 max=17144 max_rank=5 min=15933 min_rank=2 imbalance_ratio=1.046387 max_load_reduction=113928 max_load_reduction_pct=0.869202 moved_to_echo=66765 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 137, 0, 0, 0, 0, 4096, 4096, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2461, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1010, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3299, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2229, 4096, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3637, 1635, 1357, 797, 526, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 611, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3808, 0, 0, 0, 4096, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 459, 0, 0, 0, 3896, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16278, 15759, 16067, 17256, 16436, 16468, 16166, 16642] avg=16384.0000 max=17256 max_rank=3 min=15759 min_rank=1 imbalance_ratio=1.053223 max_load_reduction=113816 max_load_reduction_pct=0.868347 moved_to_echo=65712 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 186, 0, 0, 0, 0, 4096, 4096, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2461, 4096, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 914, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3300, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4096, 2436, 4096, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 3635, 1635, 1357, 796, 526, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 228, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3808, 0, 0, 0, 4096, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 459, 0, 0, 0, 4056, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16327, 15666, 16157, 17464, 16432, 16079, 16145, 16802] avg=16384.0000 max=17464 max_rank=3 min=15666 min_rank=1 imbalance_ratio=1.065918 max_load_reduction=113608 max_load_reduction_pct=0.866760 moved_to_echo=65458 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 224, 0, 0, 0, 0, 4096, 4096, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2461, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2682, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 149, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3299, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 667, 4096, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4085, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3637, 1635, 1357, 797, 526, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 390, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3808, 0, 0, 0, 4096, 0, 0, 0, 0, 739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 459, 0, 0, 0, 3357, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16365, 17431, 15981, 15694, 16428, 16234, 16835, 16104] avg=16384.0000 max=17431 max_rank=1 min=15694 min_rank=3 imbalance_ratio=1.063904 max_load_reduction=113641 max_load_reduction_pct=0.867012 moved_to_echo=65601 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 276, 0, 0, 0, 0, 4096, 4096, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2461, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1465, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3300, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1331, 4096, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3637, 1635, 1357, 796, 526, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 378, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3808, 0, 0, 0, 4096, 0, 0, 0, 0, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 459, 0, 0, 0, 3336, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16417, 16214, 16767, 16358, 16408, 16236, 16590, 16082] avg=16384.0000 max=16767 max_rank=2 min=16082 min_rank=7 imbalance_ratio=1.023376 max_load_reduction=114305 max_load_reduction_pct=0.872078 moved_to_echo=65316 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4082, 396, 1, 175, 12, 1, 4095, 4095, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2459, 4094, 1, 0, 0, 0, 0, 0, 1, 4091, 23, 1, 2211, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 5, 1529, 1, 1, 31, 1, 0, 7, 0, 1, 0, 3, 0, 1, 4094, 4094, 3295, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4, 0, 1, 0, 9, 0, 4091, 4093, 260, 4093, 2732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3715, 0, 1, 1, 4, 1, 0, 0, 4091, 4, 1, 0, 0, 2, 1, 3641, 1636, 1363, 795, 526, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4060, 0, 937, 0, 0, 0, 3916, 1, 0, 12, 1, 0, 0, 3561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4094, 4094, 0, 1, 3801, 0, 0, 1, 4095, 0, 1, 1, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4021, 0, 0, 454, 2, 0, 1, 3317, 1, 1, 1, 0, 4095, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16712, 16976, 17279, 15285, 16085, 16570, 16198, 15967] avg=16384.0000 max=17279 max_rank=2 min=15285 min_rank=3 imbalance_ratio=1.054626 max_load_reduction=113793 max_load_reduction_pct=0.868172 moved_to_echo=64820 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 301, 0, 0, 0, 0, 4096, 4096, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2461, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1888, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 332, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3299, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 1821, 4095, 2738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4093, 1, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 3635, 1635, 1357, 797, 526, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 319, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 3808, 0, 0, 0, 4096, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 459, 0, 0, 0, 3503, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16445, 16638, 16125, 16847, 16443, 16177, 16156, 16241] avg=16384.0000 max=16847 max_rank=3 min=16125 min_rank=2 imbalance_ratio=1.028259 max_load_reduction=114225 max_load_reduction_pct=0.871468 moved_to_echo=65017 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 57, 0, 0, 0, 0, 4096, 4096, 3854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2461, 4096, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 1620, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3300, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 4096, 4096, 1097, 4096, 2739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4068, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 3632, 1635, 1357, 796, 526, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 324, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3808, 0, 0, 0, 4096, 0, 0, 0, 0, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 458, 0, 0, 0, 3812, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16200, 16372, 16763, 16138, 16414, 16164, 16468, 16553] avg=16384.0000 max=16763 max_rank=2 min=16138 min_rank=3 imbalance_ratio=1.023132 max_load_reduction=114309 max_load_reduction_pct=0.872108 moved_to_echo=65599 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 0, 0, 128, 0, 0, 0, 32768, 1792, 128, 256, 128, 128, 32768, 32768, 30848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 19712, 32768, 128, 0, 128, 0, 0, 128, 128, 32768, 128, 128, 13056, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 32768, 0, 0, 3712, 128, 0, 128, 29312, 128, 128, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 0, 32768, 0, 4480, 0, 0, 0, 32640, 128, 0, 128, 128, 0, 0, 28672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 32512, 128, 4736, 128, 128, 128, 128, 0, 128, 0, 128, 0, 128, 0, 128, 32768, 32768, 26496, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 30464, 0, 0, 128, 32768, 0, 128, 128, 0, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 128, 32768, 32768, 10880, 32768, 22016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 32384, 128, 128, 128, 128, 128, 0, 0, 32768, 128, 128, 128, 0, 128, 128, 29184, 13184, 10880, 6400, 4224, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True + [2026-08-30 05:05:03] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 750.3 | throughput per GPU (TFLOP/s/GPU): 444.4 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.791005E+00 | loss scale: 1.0 | grad norm: 4.947 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2424, 1143, 1392, 4, 959, 0, 1635, 2201, 1320, 2494, 2209, 1118, 855, 2116, 0, 631, 786, 1481, 707, 649, 19, 1010, 157, 1819, 768, 239, 2357, 1407, 1125, 137, 1074, 1008, 76, 0, 2366, 583, 401, 2736, 1519, 1182, 207, 3, 1468, 2109, 4, 1326, 1858, 1367, 972, 453, 1249, 1473, 29, 299, 30, 1245, 0, 666, 55, 1365, 667, 1767, 966, 1178, 1051, 16, 1, 1458, 1963, 2218, 1773, 1010, 1241, 1301, 1137, 754, 1801, 35, 2167, 1124, 1131, 637, 19, 5, 1753, 1133, 1196, 8, 2072, 767, 1536, 639, 606, 1440, 1244, 0, 895, 419, 17, 1729, 17, 733, 1312, 1003, 250, 1095, 541, 982, 519, 1707, 2333, 1882, 1871, 1280, 791, 1191, 1711, 1142, 1182, 1333, 1065, 1732, 1039, 507, 1908, 29, 689, 69] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1960, 1462, 1236, 0, 889, 2, 1606, 1804, 1437, 2637, 1669, 1163, 1083, 2148, 0, 697, 638, 1094, 767, 580, 57, 1051, 186, 2287, 823, 334, 2362, 1839, 1718, 336, 1378, 1058, 102, 5, 2089, 1156, 314, 2107, 958, 914, 521, 20, 1454, 2173, 0, 823, 1264, 939, 1147, 582, 1084, 1223, 40, 143, 19, 1235, 0, 580, 32, 932, 1038, 2037, 1287, 1154, 704, 1, 0, 1438, 1576, 2003, 770, 1054, 756, 1005, 1149, 1241, 1512, 70, 1793, 1701, 1264, 820, 34, 3, 2482, 1042, 1186, 9, 2270, 623, 1098, 356, 590, 2019, 1210, 0, 1536, 1390, 8, 1657, 38, 955, 1220, 1457, 267, 1134, 509, 997, 941, 1135, 2228, 1551, 1935, 1477, 656, 1157, 2051, 1490, 1646, 1273, 888, 1528, 1126, 726, 1512, 20, 1071, 41] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2371, 1684, 1392, 5, 518, 0, 1364, 1781, 1367, 2340, 2450, 636, 822, 2669, 0, 802, 971, 1556, 731, 798, 18, 996, 208, 1937, 704, 368, 2718, 1677, 1054, 541, 1138, 876, 112, 8, 1817, 1221, 365, 2098, 1222, 1217, 443, 60, 1716, 2032, 0, 1071, 1805, 1677, 1389, 725, 1418, 1456, 29, 321, 5, 892, 0, 715, 33, 1582, 982, 2408, 1104, 971, 737, 2, 0, 1129, 1832, 1542, 1218, 884, 905, 1174, 1748, 1109, 2232, 131, 1997, 1642, 1071, 465, 40, 44, 1986, 1075, 932, 15, 1451, 632, 754, 560, 726, 1575, 953, 2, 1228, 796, 16, 1706, 16, 1020, 1028, 1028, 302, 1050, 505, 899, 374, 1070, 2636, 1727, 1473, 1130, 645, 941, 1782, 1925, 1362, 1148, 979, 1497, 1423, 657, 2188, 13, 512, 47] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1615, 924, 1515, 0, 1080, 4, 1150, 1824, 1055, 1534, 1113, 1276, 1214, 1091, 0, 997, 651, 1002, 953, 1022, 21, 825, 248, 1841, 808, 65, 1991, 1171, 1182, 192, 937, 1246, 233, 25, 1999, 406, 631, 1829, 967, 945, 449, 43, 1205, 1596, 1, 1278, 1121, 837, 1422, 612, 447, 1968, 55, 292, 74, 1384, 0, 530, 232, 885, 1026, 2040, 988, 1141, 1388, 10, 0, 1440, 2033, 1711, 1965, 1184, 1080, 1497, 2581, 1184, 1810, 18, 1127, 1307, 1301, 387, 96, 129, 2213, 1044, 1244, 56, 2257, 569, 1122, 1175, 1169, 1367, 935, 32, 924, 826, 47, 2103, 105, 981, 1612, 1313, 360, 1265, 640, 1658, 1442, 1785, 1541, 2102, 1526, 1703, 713, 960, 2458, 1287, 1440, 1465, 1228, 1749, 1169, 922, 1848, 143, 1042, 76] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1988, 1308, 1383, 1, 913, 0, 1848, 1999, 1530, 1829, 2294, 912, 1470, 1560, 0, 1049, 566, 1299, 895, 792, 14, 1119, 232, 1946, 979, 167, 2249, 1416, 1324, 190, 1662, 1302, 96, 2, 2305, 776, 185, 1290, 1127, 993, 208, 73, 1627, 2012, 0, 1118, 1191, 976, 1006, 368, 1820, 1394, 99, 497, 23, 1428, 0, 662, 174, 1328, 1130, 1761, 911, 1182, 826, 16, 0, 1160, 1197, 2360, 2000, 1670, 1535, 1697, 1498, 867, 2092, 111, 1223, 1834, 1144, 1092, 47, 25, 1684, 568, 972, 35, 1936, 573, 1137, 691, 401, 1911, 873, 2, 1169, 1192, 9, 1844, 43, 627, 1841, 1069, 402, 1250, 503, 701, 690, 724, 1881, 1713, 1897, 1583, 558, 1275, 2167, 1011, 1192, 1501, 1112, 1565, 1109, 1256, 1003, 57, 841, 112] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2072, 1595, 1584, 4, 1668, 1, 1192, 1504, 1186, 2096, 1601, 825, 1368, 1094, 1, 882, 572, 1054, 1093, 815, 8, 1007, 182, 2221, 803, 149, 2448, 1274, 1296, 311, 1294, 1117, 211, 30, 1905, 749, 390, 1472, 1477, 567, 544, 43, 1157, 1741, 0, 1312, 972, 1156, 1487, 559, 734, 1558, 22, 300, 74, 1806, 0, 917, 232, 834, 1168, 1587, 933, 1084, 953, 70, 0, 1257, 1871, 2213, 1907, 1401, 1424, 1481, 1740, 1079, 1916, 120, 852, 1862, 1479, 739, 36, 17, 1784, 818, 1527, 41, 2181, 337, 1192, 544, 411, 998, 854, 0, 677, 678, 49, 2531, 77, 830, 1570, 1159, 373, 1285, 590, 802, 1159, 1933, 2298, 1938, 2017, 1205, 362, 1382, 2325, 877, 1661, 1187, 1062, 1692, 1321, 1366, 1090, 51, 1012, 71] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2131, 1747, 1142, 0, 1392, 1, 1067, 1356, 1066, 2284, 1858, 1393, 1282, 2149, 1, 699, 487, 1421, 544, 702, 7, 1035, 228, 1900, 966, 139, 2149, 1110, 815, 472, 1129, 1377, 345, 13, 2114, 873, 135, 1932, 1155, 1696, 409, 46, 1212, 1436, 1, 1055, 1164, 934, 1216, 478, 923, 1520, 76, 390, 25, 1428, 2, 966, 128, 1137, 979, 1762, 1143, 1090, 648, 7, 0, 2010, 1999, 1205, 1790, 801, 1101, 1429, 1208, 1197, 1517, 29, 1416, 1512, 1156, 472, 74, 26, 1833, 992, 1483, 28, 1790, 427, 1373, 915, 750, 1891, 866, 11, 1524, 1026, 13, 1924, 105, 774, 2111, 1504, 283, 1476, 481, 931, 602, 1680, 2536, 1925, 1596, 1263, 953, 1014, 2090, 1509, 1553, 1010, 1054, 1557, 1706, 977, 1267, 38, 730, 72] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1732, 1073, 1224, 0, 1668, 0, 1339, 1573, 1394, 1554, 1308, 1128, 1077, 1132, 1, 1151, 745, 1746, 579, 1050, 1, 943, 206, 1965, 639, 94, 2570, 1577, 1271, 120, 1121, 995, 208, 35, 1927, 362, 220, 2392, 1388, 770, 391, 13, 1427, 2586, 5, 1064, 1743, 1500, 1565, 604, 624, 1395, 33, 145, 19, 1357, 0, 715, 114, 1116, 854, 1590, 861, 1779, 1240, 7, 0, 1447, 2695, 2302, 1641, 1202, 1668, 2147, 1795, 1029, 1349, 10, 1900, 1285, 1445, 470, 41, 30, 2151, 588, 1206, 60, 1784, 422, 1208, 1142, 540, 1511, 980, 1, 828, 899, 22, 1660, 55, 850, 1157, 1068, 266, 1286, 555, 999, 1311, 1232, 2037, 2299, 1594, 1013, 336, 848, 2620, 1111, 1597, 1233, 1580, 1341, 1122, 666, 1538, 50, 786, 39] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1722, 1143, 1392, 4, 959, 0, 1635, 2201, 1320, 0, 2209, 1118, 855, 2116, 0, 631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 786, 1481, 707, 649, 19, 1010, 157, 1819, 768, 239, 2357, 1407, 1125, 137, 1074, 1008, 602, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 2366, 583, 401, 2736, 1519, 1182, 207, 3, 1468, 2109, 4, 1326, 1858, 1367, 987, 262, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 972, 453, 1249, 1473, 29, 299, 30, 1245, 0, 666, 55, 1365, 667, 1767, 966, 1178, 2494, 1300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 16, 1, 1458, 976, 0, 1773, 1010, 1241, 1301, 1137, 754, 1801, 35, 2167, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1131, 637, 19, 5, 1753, 1133, 1196, 8, 2072, 767, 1536, 639, 606, 1440, 1244, 0, 1427, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 895, 419, 17, 1729, 17, 733, 1312, 1003, 250, 1095, 541, 982, 519, 1707, 1731, 1882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1871, 1280, 791, 1191, 191, 1142, 1182, 1333, 1065, 1732, 1039, 507, 1908, 29, 689, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17305, 15785, 18547, 16208, 15845, 16531, 14832, 16019] avg=16384.0000 max=18547 max_rank=2 min=14832 min_rank=6 imbalance_ratio=1.132019 max_load_reduction=112525 max_load_reduction_pct=0.858498 moved_to_echo=63227 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1677, 1684, 1392, 5, 518, 0, 1364, 1781, 1367, 0, 2450, 636, 822, 2669, 0, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1556, 731, 798, 18, 996, 208, 1937, 704, 368, 2718, 1677, 1054, 541, 1138, 876, 685, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 8, 1817, 1221, 365, 2098, 1222, 1217, 443, 60, 1716, 2032, 0, 1071, 1805, 1677, 925, 261, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1389, 725, 1418, 1456, 29, 321, 5, 892, 0, 715, 33, 1582, 982, 2408, 1104, 971, 2340, 905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 737, 2, 0, 1129, 907, 0, 1218, 884, 905, 1174, 1748, 1109, 2232, 131, 1997, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 465, 40, 44, 1986, 1075, 932, 15, 1451, 632, 754, 560, 726, 1575, 953, 2, 1491, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1228, 796, 16, 1706, 16, 1020, 1028, 1028, 302, 1050, 505, 899, 374, 1070, 1951, 1727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1473, 1130, 645, 941, 189, 1925, 1362, 1148, 979, 1497, 1423, 657, 2188, 13, 512, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17167, 17409, 18152, 17275, 15815, 14409, 14716, 16129] avg=16384.0000 max=18152 max_rank=2 min=14409 min_rank=5 imbalance_ratio=1.107910 max_load_reduction=112920 max_load_reduction_pct=0.861511 moved_to_echo=61069 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1392, 1462, 1236, 0, 889, 2, 1606, 1804, 1437, 0, 1669, 1163, 1083, 2148, 0, 697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 638, 1094, 767, 580, 57, 1051, 186, 2287, 823, 334, 2362, 1839, 1718, 336, 1378, 1058, 575, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 5, 2089, 1156, 314, 2107, 958, 914, 521, 20, 1454, 2173, 0, 823, 1264, 939, 793, 212, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1147, 582, 1084, 1223, 40, 143, 19, 1235, 0, 580, 32, 932, 1038, 2037, 1287, 1154, 2637, 1174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 704, 1, 0, 1438, 783, 0, 770, 1054, 756, 1005, 1149, 1241, 1512, 70, 1793, 1701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1264, 820, 34, 3, 2482, 1042, 1186, 9, 2270, 623, 1098, 356, 590, 2019, 1210, 0, 1711, 829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1536, 1390, 8, 1657, 38, 955, 1220, 1457, 267, 1134, 509, 997, 941, 1135, 1653, 1551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1935, 1477, 656, 1157, 228, 1490, 1646, 1273, 888, 1528, 1126, 726, 1512, 20, 1071, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16588, 17439, 15956, 16344, 13977, 17546, 16448, 16774] avg=16384.0000 max=17546 max_rank=5 min=13977 min_rank=4 imbalance_ratio=1.070923 max_load_reduction=113526 max_load_reduction_pct=0.866135 moved_to_echo=64745 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1412, 1308, 1383, 1, 913, 0, 1848, 1999, 1530, 0, 2294, 912, 1470, 1560, 0, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 1299, 895, 792, 14, 1119, 232, 1946, 979, 167, 2249, 1416, 1324, 190, 1662, 1302, 485, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 2, 2305, 776, 185, 1290, 1127, 993, 208, 73, 1627, 2012, 0, 1118, 1191, 976, 602, 215, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1006, 368, 1820, 1394, 99, 497, 23, 1428, 0, 662, 174, 1328, 1130, 1761, 911, 1182, 1829, 1383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 16, 0, 1160, 595, 0, 2000, 1670, 1535, 1697, 1498, 867, 2092, 111, 1223, 1834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1144, 1092, 47, 25, 1684, 568, 972, 35, 1936, 573, 1137, 691, 401, 1911, 873, 2, 1807, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1169, 1192, 9, 1844, 43, 627, 1841, 1069, 402, 1250, 503, 701, 690, 724, 1396, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1897, 1583, 558, 1275, 241, 1011, 1192, 1501, 1112, 1565, 1109, 1256, 1003, 57, 841, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17679, 16998, 14915, 16995, 17124, 15875, 15173, 16313] avg=16384.0000 max=17679 max_rank=0 min=14915 min_rank=2 imbalance_ratio=1.079041 max_load_reduction=113393 max_load_reduction_pct=0.865120 moved_to_echo=64485 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1147, 924, 1515, 0, 1080, 4, 1150, 1824, 1055, 0, 1113, 1276, 1214, 1091, 0, 997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 1002, 953, 1022, 21, 825, 248, 1841, 808, 65, 1991, 1171, 1182, 192, 937, 1246, 397, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 25, 1999, 406, 631, 1829, 967, 945, 449, 43, 1205, 1596, 1, 1278, 1121, 837, 1023, 175, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1422, 612, 447, 1968, 55, 292, 74, 1384, 0, 530, 232, 885, 1026, 2040, 988, 1141, 1534, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1388, 10, 0, 1440, 1010, 0, 1965, 1184, 1080, 1497, 2581, 1184, 1810, 18, 1127, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1301, 387, 96, 129, 2213, 1044, 1244, 56, 2257, 569, 1122, 1175, 1169, 1367, 935, 32, 2050, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 924, 826, 47, 2103, 105, 981, 1612, 1313, 360, 1265, 640, 1658, 1442, 1785, 1144, 2102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1526, 1703, 713, 960, 273, 1287, 1440, 1465, 1228, 1749, 1169, 922, 1848, 143, 1042, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14390, 14845, 14898, 15633, 17601, 17854, 18307, 17544] avg=16384.0000 max=18307 max_rank=6 min=14390 min_rank=0 imbalance_ratio=1.117371 max_load_reduction=112765 max_load_reduction_pct=0.860329 moved_to_echo=71306 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1514, 1747, 1142, 0, 1392, 1, 1067, 1356, 1066, 0, 1858, 1393, 1282, 2149, 1, 699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 1421, 544, 702, 7, 1035, 228, 1900, 966, 139, 2149, 1110, 815, 472, 1129, 1377, 654, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 13, 2114, 873, 135, 1932, 1155, 1696, 409, 46, 1212, 1436, 1, 1055, 1164, 934, 1005, 230, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 478, 923, 1520, 76, 390, 25, 1428, 2, 966, 128, 1137, 979, 1762, 1143, 1090, 2284, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 648, 7, 0, 2010, 994, 0, 1790, 801, 1101, 1429, 1208, 1197, 1517, 29, 1416, 1512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1156, 472, 74, 26, 1833, 992, 1483, 28, 1790, 427, 1373, 915, 750, 1891, 866, 11, 1743, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1524, 1026, 13, 1924, 105, 774, 2111, 1504, 283, 1476, 481, 931, 602, 1680, 1882, 1925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1596, 1263, 953, 1014, 233, 1509, 1553, 1010, 1054, 1557, 1706, 977, 1267, 38, 730, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16667, 15522, 15869, 16254, 15659, 16328, 18241, 16532] avg=16384.0000 max=18241 max_rank=6 min=15522 min_rank=1 imbalance_ratio=1.113342 max_load_reduction=112831 max_load_reduction_pct=0.860832 moved_to_echo=66760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1472, 1595, 1584, 4, 1668, 1, 1192, 1504, 1186, 0, 1601, 825, 1368, 1094, 1, 882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 572, 1054, 1093, 815, 8, 1007, 182, 2221, 803, 149, 2448, 1274, 1296, 311, 1294, 1117, 593, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 30, 1905, 749, 390, 1472, 1477, 567, 544, 43, 1157, 1741, 0, 1312, 972, 1156, 941, 224, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1487, 559, 734, 1558, 22, 300, 74, 1806, 0, 917, 232, 834, 1168, 1587, 933, 1084, 2096, 1297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 70, 0, 1257, 930, 0, 1907, 1401, 1424, 1481, 1740, 1079, 1916, 120, 852, 1862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1479, 739, 36, 17, 1784, 818, 1527, 41, 2181, 337, 1192, 544, 411, 998, 854, 0, 1939, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 677, 678, 49, 2531, 77, 830, 1570, 1159, 373, 1285, 590, 802, 1159, 1933, 1705, 1938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2017, 1205, 362, 1382, 259, 877, 1661, 1187, 1062, 1692, 1321, 1366, 1090, 51, 1012, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15977, 16613, 15018, 16688, 16992, 15813, 17356, 16615] avg=16384.0000 max=17356 max_rank=6 min=15018 min_rank=2 imbalance_ratio=1.059326 max_load_reduction=113716 max_load_reduction_pct=0.867584 moved_to_echo=66776 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1231, 1073, 1224, 0, 1668, 0, 1339, 1573, 1394, 0, 1308, 1128, 1077, 1132, 1, 1151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 1746, 579, 1050, 1, 943, 206, 1965, 639, 94, 2570, 1577, 1271, 120, 1121, 995, 525, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 35, 1927, 362, 220, 2392, 1388, 770, 391, 13, 1427, 2586, 5, 1064, 1743, 1500, 1356, 187, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1565, 604, 624, 1395, 33, 145, 19, 1357, 0, 715, 114, 1116, 854, 1590, 861, 1779, 1554, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1240, 7, 0, 1447, 1339, 0, 1641, 1202, 1668, 2147, 1795, 1029, 1349, 10, 1900, 1285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1445, 470, 41, 30, 2151, 588, 1206, 60, 1784, 422, 1208, 1142, 540, 1511, 980, 1, 2185, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 899, 22, 1660, 55, 850, 1157, 1068, 266, 1286, 555, 999, 1311, 1232, 1512, 2299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1594, 1013, 336, 848, 292, 1111, 1597, 1233, 1580, 1341, 1122, 666, 1538, 50, 786, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15299, 16461, 17717, 15675, 18059, 16716, 15999, 15146] avg=16384.0000 max=18059 max_rank=4 min=15146 min_rank=7 imbalance_ratio=1.102234 max_load_reduction=113013 max_load_reduction_pct=0.862221 moved_to_echo=65920 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[8832, 7296, 256, 15232, 512, 6784, 11904, 9728, 2560, 9856, 4352, 8064, 7040, 11392, 13056, 15232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[11648, 11008, 10880, 128, 9088, 128, 11264, 14080, 10368, 0, 14592, 8576, 9216, 14080, 128, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[5504, 10752, 6272, 6528, 256, 8064, 1664, 16000, 6528, 1664, 18944, 11520, 9856, 2304, 9856, 9088, 4608, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[7552, 256, 128, 11392, 7552, 0, 13184, 9216, 9728, 11776, 12928, 8576, 14336, 640, 12544, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[10240, 4480, 8320, 12032, 384, 2432, 384, 10880, 128, 5760, 1024, 9216, 7936, 14976, 8320, 9600, 16768, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1408, 128, 16640, 6144, 2688, 15872, 9856, 8320, 3200, 384, 11392, 15744, 128, 9088, 11136, 9472, 7680, 1792, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[10112, 5120, 512, 384, 16000, 7296, 9856, 256, 15744, 4352, 9472, 6144, 5248, 12800, 7936, 128, 14464, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[13952, 10752, 5120, 8832, 1920, 10368, 11648, 10240, 9088, 12672, 10112, 7168, 12416, 512, 6784, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 638, 669, 0, 17, 3, 0, 0, 0, 3632, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1987, 0, 3195, 6, 0, 838, 0, 1362, 3875, 11, 0, 0, 4095, 0, 1181, 45, 4094, 0, 0, 4057, 0, 0, 2767, 0, 3354, 0, 3948, 0, 1804, 0, 0, 4096, 51, 4016, 4096, 0, 0, 11, 3848, 0, 0, 23, 0, 4072, 0, 0, 0, 2143, 1, 0, 2365, 1, 4071, 3019, 0, 0, 0, 0, 0, 132, 416, 6, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 1717, 0, 0, 0, 0, 4096, 22, 3981, 4095, 0, 3946, 3864, 0, 17, 0, 0, 0, 4095, 4096, 0, 0, 66, 2, 1507, 0, 0, 2199, 2675, 1, 270, 0, 0, 0, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[6, 19, 0, 206, 100, 9, 0, 0, 11, 0, 0, 2505, 5, 0, 3, 4086, 0, 13, 4088, 3, 0, 2482, 0, 4058, 4, 0, 339, 0, 3712, 4007, 0, 0, 27, 4086, 0, 1110, 0, 4089, 0, 8, 4040, 0, 3, 1799, 0, 2574, 8, 4088, 14, 2508, 21, 2, 4092, 0, 3902, 4073, 3, 0, 40, 4030, 0, 0, 10, 0, 3693, 13, 0, 7, 736, 42, 8, 2943, 16, 3992, 3457, 0, 7, 3, 0, 0, 179, 77, 0, 4056, 3, 0, 0, 140, 0, 7, 10, 0, 1, 4089, 3897, 0, 0, 7, 0, 4076, 41, 4096, 4067, 7, 3255, 1763, 8, 25, 0, 8, 1, 4074, 4066, 0, 7, 411, 0, 2979, 0, 0, 2714, 1758, 0, 64, 0, 0, 0, 4086] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1, 3, 0, 347, 44, 0, 11, 0, 1, 2, 16, 3999, 0, 0, 0, 4095, 0, 3, 4094, 3, 0, 1064, 0, 3913, 0, 0, 30, 0, 3032, 3982, 85, 0, 0, 4087, 0, 1101, 17, 4092, 0, 0, 4090, 0, 0, 3026, 1, 2181, 0, 4021, 10, 834, 0, 0, 4096, 0, 4006, 4093, 3, 0, 610, 3554, 0, 0, 142, 0, 4084, 0, 0, 0, 378, 0, 3, 3746, 1, 4060, 3896, 1, 0, 2, 0, 2, 7, 236, 0, 4093, 0, 1, 0, 2, 0, 0, 0, 0, 0, 4096, 2817, 0, 2, 0, 0, 4096, 342, 3428, 4091, 0, 3384, 4081, 2, 432, 10, 3, 3, 4089, 4093, 0, 0, 94, 0, 2342, 0, 4, 567, 3625, 0, 187, 0, 3, 0, 4080] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 92, 533, 0, 3, 0, 0, 0, 3, 3246, 0, 0, 0, 4091, 0, 0, 4095, 0, 0, 1736, 3, 3096, 1, 0, 911, 0, 3166, 4032, 1, 1, 1, 4096, 0, 268, 13, 4096, 1, 0, 4056, 0, 0, 2776, 0, 2706, 0, 4031, 2, 1722, 0, 0, 4096, 5, 2399, 4095, 0, 0, 7, 3875, 1, 0, 8, 0, 3698, 0, 0, 0, 2225, 0, 0, 2755, 0, 3895, 3502, 0, 0, 0, 1, 0, 256, 335, 8, 4095, 1, 0, 0, 5, 1, 0, 0, 1, 0, 4096, 1491, 0, 1, 0, 0, 4095, 10, 3660, 4094, 0, 3871, 4046, 0, 10, 4, 0, 0, 4095, 4096, 0, 0, 828, 1, 2765, 6, 0, 2383, 3230, 0, 154, 0, 0, 1, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 901, 204, 0, 0, 0, 0, 0, 0, 4017, 0, 1, 0, 4084, 0, 1, 4095, 0, 0, 2059, 19, 3170, 0, 0, 36, 0, 2191, 4041, 123, 0, 2, 4096, 0, 421, 7, 4096, 2, 0, 4082, 2, 2, 3367, 0, 2826, 0, 3990, 38, 1144, 1, 0, 4096, 4, 3720, 4096, 0, 0, 58, 3704, 0, 0, 259, 1, 4042, 0, 0, 1, 163, 36, 0, 3309, 0, 4040, 3818, 0, 0, 2, 0, 0, 66, 249, 30, 4088, 0, 0, 0, 21, 0, 0, 9, 0, 0, 4089, 2789, 2, 1, 0, 0, 4081, 214, 3031, 4095, 0, 3761, 4038, 0, 101, 6, 0, 0, 4095, 4094, 4, 1, 193, 1, 1898, 0, 0, 838, 4002, 0, 820, 0, 0, 0, 4088] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 852, 30, 0, 27, 44, 0, 0, 0, 4073, 0, 0, 0, 4096, 0, 0, 4083, 0, 0, 1043, 0, 2291, 8, 0, 354, 0, 1247, 4042, 46, 0, 2, 4093, 2, 616, 0, 4092, 4, 0, 4093, 0, 0, 3419, 0, 3328, 0, 3674, 6, 1048, 0, 0, 4087, 0, 4039, 4092, 0, 0, 59, 3733, 0, 0, 348, 0, 3903, 0, 0, 1, 221, 34, 0, 3729, 0, 4096, 3986, 0, 0, 0, 21, 0, 21, 134, 1, 4096, 0, 0, 3, 11, 0, 0, 0, 0, 0, 4096, 1202, 0, 0, 0, 0, 4093, 583, 3507, 4080, 0, 3972, 4059, 0, 193, 3, 0, 0, 4085, 4092, 0, 0, 59, 0, 2729, 0, 0, 2551, 3689, 0, 748, 0, 7, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 6, 0, 511, 1212, 0, 0, 0, 1, 1, 0, 2161, 0, 0, 0, 4092, 0, 0, 4096, 0, 2, 2792, 8, 4077, 1, 0, 587, 0, 3758, 3857, 27, 0, 11, 4096, 0, 390, 0, 4094, 0, 0, 4062, 0, 0, 2266, 0, 2517, 0, 4091, 14, 1104, 1, 0, 4094, 0, 2641, 4096, 0, 0, 45, 3860, 0, 0, 16, 0, 3915, 1, 0, 0, 2576, 2, 0, 2583, 0, 3202, 2233, 0, 0, 0, 0, 0, 474, 132, 0, 4096, 0, 0, 8, 91, 0, 0, 0, 0, 0, 4096, 3593, 0, 0, 0, 0, 4094, 2, 4022, 4096, 0, 3967, 3364, 0, 15, 1, 0, 0, 4096, 4087, 0, 0, 687, 0, 2122, 0, 0, 3165, 1492, 0, 178, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2, 7, 2, 234, 71, 1, 0, 9, 4, 0, 21, 3359, 2, 2, 0, 4094, 0, 4, 4094, 2, 0, 2005, 0, 3860, 56, 1, 372, 2, 3802, 3991, 3, 0, 0, 4095, 0, 618, 2, 4093, 0, 3, 4085, 0, 2, 2844, 2, 1684, 4, 4025, 3, 2347, 0, 2, 4092, 0, 2528, 4094, 3, 3, 19, 3894, 0, 2, 1, 1, 3993, 0, 0, 4, 859, 10, 2, 2820, 3, 3784, 3087, 0, 2, 2, 2, 2, 57, 79, 5, 4089, 0, 2, 3, 141, 0, 2, 0, 0, 3, 4094, 3537, 0, 2, 2, 0, 4094, 22, 3936, 4089, 2, 3593, 3799, 3, 95, 0, 2, 0, 4091, 4088, 0, 2, 1567, 0, 1138, 0, 0, 1844, 3505, 1, 76, 0, 0, 2, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 3, 0, 347, 44, 0, 11, 0, 1, 2, 16, 3999, 0, 0, 0, 4095, 4092, 3970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3783, 3, 0, 1064, 0, 3913, 0, 0, 30, 0, 3032, 3982, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2164, 0, 1101, 17, 0, 0, 0, 4090, 0, 0, 3026, 1, 2181, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 834, 0, 0, 3281, 0, 4006, 4093, 3, 0, 610, 3554, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 378, 0, 3, 3746, 1, 4060, 3896, 1, 0, 2, 0, 2, 815, 127, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 236, 0, 4093, 0, 1, 0, 2, 0, 0, 0, 0, 0, 4096, 2817, 0, 3121, 1897, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 342, 3428, 4091, 0, 3384, 4081, 2, 432, 10, 3, 3, 968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 94, 0, 2342, 0, 4, 567, 3625, 0, 187, 0, 3, 0, 4080, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16581, 15895, 16601, 16533, 17141, 16396, 16746, 15179] avg=16384.0000 max=17141 max_rank=4 min=15179 min_rank=7 imbalance_ratio=1.046204 max_load_reduction=113931 max_load_reduction_pct=0.869225 moved_to_echo=65462 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 638, 669, 0, 17, 3, 0, 0, 0, 3632, 0, 0, 0, 4096, 4094, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3785, 0, 0, 1987, 0, 3195, 6, 0, 838, 0, 1362, 3875, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2168, 0, 1181, 45, 0, 0, 0, 4057, 0, 0, 2767, 0, 3354, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1804, 0, 0, 3281, 51, 4016, 4096, 0, 0, 11, 3848, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 2143, 1, 0, 2365, 1, 4071, 3019, 0, 0, 0, 0, 0, 815, 127, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 416, 6, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 1717, 0, 3126, 1901, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 3981, 4095, 0, 3946, 3864, 0, 17, 0, 0, 0, 969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 66, 2, 1507, 0, 0, 2199, 2675, 1, 270, 0, 0, 0, 4092, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17118, 15059, 17520, 17130, 16640, 15619, 16894, 15092] avg=16384.0000 max=17520 max_rank=2 min=15059 min_rank=1 imbalance_ratio=1.069336 max_load_reduction=113552 max_load_reduction_pct=0.866333 moved_to_echo=64245 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[6, 19, 0, 206, 100, 9, 0, 0, 11, 0, 0, 2505, 5, 0, 3, 4086, 4089, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3777, 3, 0, 2482, 0, 4058, 4, 0, 339, 0, 3712, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2163, 0, 1110, 0, 0, 0, 8, 4040, 0, 3, 1799, 0, 2574, 8, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2508, 21, 2, 3278, 0, 3902, 4073, 3, 0, 40, 4030, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3693, 13, 0, 7, 736, 42, 8, 2943, 16, 3992, 3457, 0, 7, 3, 0, 0, 814, 127, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 77, 0, 4056, 3, 0, 0, 140, 0, 7, 10, 0, 1, 4089, 3897, 0, 3110, 1897, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 41, 4096, 4067, 7, 3255, 1763, 8, 25, 0, 8, 1, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 7, 411, 0, 2979, 0, 0, 2714, 1758, 0, 64, 0, 0, 0, 4086, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14989, 18395, 15820, 17881, 15884, 17592, 14242, 16269] avg=16384.0000 max=18395 max_rank=1 min=14242 min_rank=6 imbalance_ratio=1.122742 max_load_reduction=112677 max_load_reduction_pct=0.859657 moved_to_echo=63987 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 92, 533, 0, 3, 0, 0, 0, 3, 3246, 0, 0, 0, 4091, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3784, 0, 0, 1736, 3, 3096, 1, 0, 911, 0, 3166, 4032, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2168, 0, 268, 13, 0, 1, 0, 4056, 0, 0, 2776, 0, 2706, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1722, 0, 0, 3281, 5, 2399, 4095, 0, 0, 7, 3875, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3698, 0, 0, 0, 2225, 0, 0, 2755, 0, 3895, 3502, 0, 0, 0, 1, 0, 815, 127, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 335, 8, 4095, 1, 0, 0, 5, 1, 0, 0, 1, 0, 4096, 1491, 0, 3126, 1902, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 10, 3660, 4094, 0, 3871, 4046, 0, 10, 4, 0, 0, 969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 828, 1, 2765, 6, 0, 2383, 3230, 0, 154, 0, 0, 1, 4093, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16032, 16731, 16020, 15395, 17044, 15444, 16665, 17741] avg=16384.0000 max=17741 max_rank=7 min=15395 min_rank=3 imbalance_ratio=1.082825 max_load_reduction=113331 max_load_reduction_pct=0.864647 moved_to_echo=66894 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 901, 204, 0, 0, 0, 0, 0, 0, 4017, 0, 1, 0, 4084, 4096, 3955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3774, 0, 0, 2059, 19, 3170, 0, 0, 36, 0, 2191, 4041, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2163, 0, 421, 7, 0, 2, 0, 4082, 2, 2, 3367, 0, 2826, 0, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 1144, 1, 0, 3277, 4, 3720, 4096, 0, 0, 58, 3704, 0, 0, 259, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 1, 163, 36, 0, 3309, 0, 4040, 3818, 0, 0, 2, 0, 0, 819, 132, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 249, 30, 4088, 0, 0, 0, 21, 0, 0, 9, 0, 0, 4089, 2789, 2, 3130, 1905, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 214, 3031, 4095, 0, 3761, 4038, 0, 101, 6, 0, 0, 965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4, 1, 193, 1, 1898, 0, 0, 838, 4002, 0, 820, 0, 0, 0, 4088, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17258, 15414, 16864, 16302, 16390, 16504, 16212, 16128] avg=16384.0000 max=17258 max_rank=0 min=15414 min_rank=1 imbalance_ratio=1.053345 max_load_reduction=113814 max_load_reduction_pct=0.868332 moved_to_echo=65234 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 511, 1212, 0, 0, 0, 1, 1, 0, 2161, 0, 0, 0, 4092, 4094, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3785, 0, 2, 2792, 8, 4077, 1, 0, 587, 0, 3758, 3857, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2168, 0, 390, 0, 0, 0, 0, 4062, 0, 0, 2266, 0, 2517, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1104, 1, 0, 3280, 0, 2641, 4096, 0, 0, 45, 3860, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3915, 1, 0, 0, 2576, 2, 0, 2583, 0, 3202, 2233, 0, 0, 0, 0, 0, 814, 127, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 132, 0, 4096, 0, 0, 8, 91, 0, 0, 0, 0, 0, 4096, 3593, 0, 3127, 1902, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4022, 4096, 0, 3967, 3364, 0, 15, 1, 0, 0, 969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 687, 0, 2122, 0, 0, 3165, 1492, 0, 178, 0, 0, 0, 4095, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16045, 18894, 15505, 15057, 15479, 17646, 16436, 16010] avg=16384.0000 max=18894 max_rank=1 min=15057 min_rank=3 imbalance_ratio=1.153198 max_load_reduction=112178 max_load_reduction_pct=0.855850 moved_to_echo=65571 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 852, 30, 0, 27, 44, 0, 0, 0, 4073, 0, 0, 0, 4096, 4092, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3772, 0, 0, 1043, 0, 2291, 8, 0, 354, 0, 1247, 4042, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2167, 2, 616, 0, 0, 4, 0, 4093, 0, 0, 3419, 0, 3328, 0, 3674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1048, 0, 0, 3274, 0, 4039, 4092, 0, 0, 59, 3733, 0, 0, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3903, 0, 0, 1, 221, 34, 0, 3729, 0, 4096, 3986, 0, 0, 0, 21, 0, 813, 127, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 134, 1, 4096, 0, 0, 3, 11, 0, 0, 0, 0, 0, 4096, 1202, 0, 3118, 1900, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 3507, 4080, 0, 3972, 4059, 0, 193, 3, 0, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 59, 0, 2729, 0, 0, 2551, 3689, 0, 748, 0, 7, 0, 4096, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17180, 12803, 17305, 16599, 16957, 14709, 17364, 18155] avg=16384.0000 max=18155 max_rank=7 min=12803 min_rank=1 imbalance_ratio=1.108093 max_load_reduction=112917 max_load_reduction_pct=0.861488 moved_to_echo=67185 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 7, 2, 234, 71, 1, 0, 9, 4, 0, 21, 3359, 2, 2, 0, 4094, 4093, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3783, 2, 0, 2005, 0, 3860, 56, 1, 372, 2, 3802, 3991, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2168, 0, 618, 2, 0, 0, 3, 4085, 0, 2, 2844, 2, 1684, 4, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2347, 0, 2, 3278, 0, 2528, 4094, 3, 3, 19, 3894, 0, 2, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 4, 859, 10, 2, 2820, 3, 3784, 3087, 0, 2, 2, 2, 2, 814, 127, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 79, 5, 4089, 0, 2, 3, 141, 0, 2, 0, 0, 3, 4094, 3537, 0, 3123, 1901, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 22, 3936, 4089, 2, 3593, 3799, 3, 95, 0, 2, 0, 968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 2, 1567, 0, 1138, 0, 0, 1844, 3505, 1, 76, 0, 0, 2, 4091, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15869, 17881, 15437, 16175, 15537, 17162, 16513, 16498] avg=16384.0000 max=17881 max_rank=1 min=15437 min_rank=2 imbalance_ratio=1.091370 max_load_reduction=113191 max_load_reduction_pct=0.863579 moved_to_echo=65710 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 30336, 128, 128, 15232, 128, 27776, 128, 128, 3584, 128, 22272, 31872, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 128, 3968, 128, 17536, 128, 128, 16384, 24064, 128, 2560, 0, 128, 128, 32768, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[1280, 1664, 128, 32768, 128, 128, 128, 512, 128, 128, 128, 128, 128, 32768, 21120, 128, 25088, 15232, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 17408, 128, 5760, 128, 0, 128, 128, 32640, 128, 128, 22272, 128, 21248, 128, 31872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 12544, 128, 128, 26240, 128, 27264, 32768, 128, 128, 896, 30592, 128, 128, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 0, 1280, 29696, 32768, 128, 29824, 29056, 128, 896, 128, 128, 128, 7808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 3840, 2944, 128, 128, 128, 128, 128, 128, 27008, 128, 128, 128, 32768, 32768, 31744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[31488, 128, 0, 128, 9344, 128, 128, 24320, 128, 31232, 27008, 128, 128, 128, 128, 128, 6528, 1024, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1355, 1, 600, 0, 0, 1172, 0, 4090, 3899, 3, 0, 0, 0, 4095, 1, 0, 0, 0, 4, 4083, 4092, 0, 4093, 4092, 0, 0, 0, 4092, 258, 0, 0, 3, 4065, 1, 0, 4094, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 2496, 4084, 2297, 0, 1, 1869, 3, 4095, 100, 0, 0, 1898, 20, 0, 0, 3945, 4093, 3, 0, 4092, 276, 5, 0, 0, 0, 0, 4096, 2882, 0, 4096, 4096, 4095, 1540, 4091, 0, 0, 4076, 1149, 2, 1384, 0, 2072, 0, 0, 3, 0, 0, 0, 2, 0, 2183, 0, 0, 4092, 0, 1253, 4, 0, 0, 0, 4, 0, 0, 219, 4096, 0, 0, 5, 0, 0, 3, 28, 0, 4018, 0, 4076, 22, 0, 4, 0, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1369, 0, 10, 0, 0, 1245, 2, 4091, 3452, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4019, 4096, 0, 4096, 4096, 0, 0, 0, 4092, 105, 0, 0, 0, 4096, 2, 3, 4094, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 1538, 4094, 280, 0, 0, 2976, 0, 4096, 94, 0, 0, 2353, 0, 0, 0, 4037, 4096, 0, 0, 4096, 311, 0, 0, 0, 0, 0, 4096, 197, 0, 4096, 3345, 4096, 274, 3996, 6, 0, 4073, 3079, 0, 3888, 0, 3534, 0, 0, 0, 0, 0, 0, 809, 0, 2925, 0, 0, 4090, 0, 1136, 1, 0, 0, 0, 5, 3, 0, 242, 4094, 0, 0, 1, 0, 0, 0, 1, 0, 4012, 0, 3966, 26, 0, 35, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2182, 0, 341, 0, 0, 497, 0, 4096, 3980, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 4063, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 76, 0, 0, 0, 3892, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2096, 4084, 1181, 0, 0, 3186, 0, 4095, 40, 0, 0, 2537, 49, 0, 0, 4095, 4096, 0, 0, 4095, 196, 0, 0, 0, 0, 0, 4096, 668, 0, 4096, 3849, 4095, 363, 4094, 0, 0, 4049, 1710, 0, 3384, 0, 3080, 1, 0, 0, 0, 0, 0, 184, 0, 725, 0, 0, 4096, 0, 1377, 17, 0, 0, 0, 0, 0, 0, 1396, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 4006, 0, 4039, 85, 0, 0, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1235, 0, 814, 0, 0, 3769, 16, 4094, 3664, 4, 0, 0, 0, 4092, 6, 0, 0, 0, 2, 3895, 4075, 0, 4096, 4094, 0, 0, 0, 4087, 524, 0, 0, 0, 3874, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 2892, 4082, 3333, 0, 0, 2301, 1, 4091, 10, 0, 0, 1483, 31, 0, 0, 4096, 4094, 0, 0, 4085, 952, 1, 0, 0, 0, 8, 4096, 1561, 0, 4095, 4067, 4096, 1739, 4095, 4, 0, 3998, 553, 2, 1765, 0, 633, 0, 3, 0, 0, 0, 0, 6, 2, 160, 0, 0, 4081, 0, 1454, 1, 0, 0, 0, 2, 7, 0, 224, 4075, 0, 0, 24, 0, 0, 0, 39, 0, 3941, 0, 4080, 237, 0, 25, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[1, 2, 2054, 0, 17, 7, 7, 294, 3, 4081, 4077, 0, 7, 1, 0, 4087, 0, 8, 0, 0, 39, 4069, 4082, 7, 4087, 4089, 15, 0, 8, 4082, 77, 0, 0, 0, 4086, 0, 0, 4089, 28, 7, 5, 8, 4088, 9, 0, 7, 0, 1614, 4091, 523, 8, 4, 3142, 0, 4089, 118, 0, 5, 1859, 174, 5, 7, 4082, 4087, 3, 7, 4079, 90, 0, 6, 0, 0, 1, 4089, 424, 2, 4084, 3836, 4087, 329, 4010, 17, 7, 4066, 2373, 0, 3639, 8, 3548, 0, 0, 40, 8, 2, 1, 307, 0, 3157, 9, 6, 4055, 0, 835, 1, 8, 0, 7, 0, 0, 9, 752, 4084, 8, 7, 14, 0, 0, 7, 0, 7, 3886, 1, 3816, 0, 0, 0, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1289, 1, 509, 0, 0, 285, 0, 4095, 4063, 1, 0, 0, 0, 4095, 1, 0, 0, 0, 4, 4081, 4091, 0, 4096, 4095, 0, 0, 0, 4095, 79, 0, 1, 0, 4083, 2, 0, 4095, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 2289, 4093, 1408, 0, 0, 3184, 1, 4096, 2, 0, 0, 2392, 76, 0, 0, 4087, 4095, 0, 0, 4096, 441, 0, 0, 0, 0, 0, 4095, 1884, 0, 4096, 3899, 4096, 320, 4096, 1, 0, 4068, 2193, 0, 3390, 0, 2563, 0, 0, 0, 0, 0, 0, 336, 1, 1790, 0, 0, 4094, 1, 552, 10, 0, 0, 0, 1, 0, 0, 798, 4095, 0, 0, 1, 0, 0, 0, 40, 0, 3133, 0, 4067, 26, 0, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1443, 0, 2040, 0, 0, 2800, 0, 4090, 3960, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 3968, 4074, 0, 4096, 4092, 0, 0, 5, 4091, 259, 0, 1, 0, 4027, 22, 28, 4096, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 3257, 4063, 3130, 0, 0, 1632, 0, 4096, 8, 0, 0, 2023, 2, 0, 0, 4096, 4096, 0, 0, 4091, 1356, 0, 0, 0, 1, 4, 4096, 468, 0, 4096, 4027, 4096, 767, 4028, 2, 0, 4094, 343, 0, 2720, 0, 1514, 0, 0, 0, 0, 0, 0, 12, 1, 197, 0, 0, 4056, 0, 770, 5, 0, 0, 0, 3, 4, 0, 206, 4076, 0, 0, 22, 0, 0, 0, 11, 0, 4091, 0, 4084, 114, 0, 15, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 815, 0, 47, 2, 3, 1082, 0, 4087, 3650, 0, 3, 2, 0, 4092, 4, 3, 0, 2, 0, 4039, 4094, 3, 4093, 4091, 0, 0, 0, 4091, 405, 0, 11, 0, 4086, 9, 3, 4093, 1, 0, 3, 3, 4093, 3, 22, 3, 3, 2654, 4093, 426, 2, 0, 2813, 0, 4089, 22, 0, 2, 3111, 31, 1, 3, 4014, 4092, 0, 3, 4085, 840, 0, 2, 0, 0, 0, 4093, 517, 3, 4093, 4064, 4093, 628, 4069, 0, 3, 4059, 2357, 0, 2826, 3, 1583, 0, 0, 0, 3, 0, 1, 507, 0, 2273, 0, 3, 4061, 1, 2285, 7, 3, 3, 2, 0, 0, 0, 182, 4072, 3, 3, 11, 2, 2, 0, 31, 3, 3889, 0, 4071, 23, 0, 16, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1355, 1, 600, 0, 0, 1172, 0, 4090, 3899, 3, 0, 0, 0, 4095, 787, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 4083, 4092, 0, 0, 3939, 0, 0, 0, 4092, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4065, 1, 0, 4094, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 2496, 1465, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 2297, 0, 1, 1869, 3, 1724, 100, 0, 0, 1898, 20, 0, 0, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4092, 276, 5, 0, 0, 0, 0, 0, 2882, 0, 4096, 4096, 1920, 1540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4076, 1149, 2, 1384, 0, 2072, 0, 0, 3, 0, 0, 0, 2, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2183, 0, 0, 4092, 0, 1253, 4, 0, 0, 0, 4, 0, 0, 219, 4096, 3910, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3, 28, 0, 4018, 0, 4076, 22, 0, 4, 0, 9, 4096, 4093, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16155, 16469, 16546, 15941, 18910, 13847, 16667, 16537] avg=16384.0000 max=18910 max_rank=4 min=13847 min_rank=5 imbalance_ratio=1.154175 max_load_reduction=112162 max_load_reduction_pct=0.855728 moved_to_echo=65961 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2182, 0, 341, 0, 0, 497, 0, 4096, 3980, 0, 0, 0, 0, 4096, 787, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4063, 4096, 0, 0, 3943, 0, 0, 0, 4096, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2096, 1465, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1181, 0, 0, 3186, 0, 1724, 40, 0, 0, 2537, 49, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 196, 0, 0, 0, 0, 0, 0, 668, 0, 4096, 3849, 1920, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4049, 1710, 0, 3384, 0, 3080, 1, 0, 0, 0, 0, 0, 184, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 725, 0, 0, 4096, 0, 1377, 17, 0, 0, 0, 0, 0, 0, 1396, 4096, 3913, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4006, 0, 4039, 85, 0, 0, 0, 5, 4096, 4096, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16132, 16282, 15966, 16896, 15187, 17570, 16526, 16513] avg=16384.0000 max=17570 max_rank=5 min=15187 min_rank=4 imbalance_ratio=1.072388 max_load_reduction=113502 max_load_reduction_pct=0.865952 moved_to_echo=65796 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1369, 0, 10, 0, 0, 1245, 2, 4091, 3452, 0, 0, 0, 0, 4096, 787, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4019, 4096, 0, 0, 3943, 0, 0, 0, 4092, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 3, 4094, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 1538, 1465, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 280, 0, 0, 2976, 0, 1725, 94, 0, 0, 2353, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 311, 0, 0, 0, 0, 0, 0, 197, 0, 4096, 3345, 1920, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 6, 0, 4073, 3079, 0, 3888, 0, 3534, 0, 0, 0, 0, 0, 0, 809, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2925, 0, 0, 4090, 0, 1136, 1, 0, 0, 0, 5, 3, 0, 242, 4094, 3913, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 4012, 0, 3966, 26, 0, 35, 0, 0, 4096, 4096, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15205, 16256, 15628, 15559, 14239, 20454, 17315, 16416] avg=16384.0000 max=20454 max_rank=5 min=14239 min_rank=4 imbalance_ratio=1.248413 max_load_reduction=110618 max_load_reduction_pct=0.843948 moved_to_echo=68424 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1289, 1, 509, 0, 0, 285, 0, 4095, 4063, 1, 0, 0, 0, 4095, 787, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 4081, 4091, 0, 0, 3942, 0, 0, 0, 4095, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4083, 2, 0, 4095, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 2289, 1465, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1408, 0, 0, 3184, 1, 1725, 2, 0, 0, 2392, 76, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 441, 0, 0, 0, 0, 0, 0, 1884, 0, 4096, 3899, 1920, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4068, 2193, 0, 3390, 0, 2563, 0, 0, 0, 0, 0, 0, 336, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1790, 0, 0, 4094, 1, 552, 10, 0, 0, 0, 1, 0, 0, 798, 4095, 3912, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 40, 0, 3133, 0, 4067, 26, 0, 2, 0, 0, 4095, 4096, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15278, 16293, 16358, 16968, 16656, 17716, 16160, 15643] avg=16384.0000 max=17716 max_rank=5 min=15278 min_rank=0 imbalance_ratio=1.081299 max_load_reduction=113356 max_load_reduction_pct=0.864838 moved_to_echo=66175 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 2054, 0, 17, 7, 7, 294, 3, 4081, 4077, 0, 7, 1, 0, 4087, 785, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 39, 4069, 4082, 7, 0, 3936, 15, 0, 8, 4082, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4089, 28, 7, 5, 8, 4088, 9, 0, 7, 0, 1614, 1463, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 523, 8, 4, 3142, 0, 1721, 118, 0, 5, 1859, 174, 5, 7, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 4079, 90, 0, 6, 0, 0, 1, 0, 424, 2, 4084, 3836, 1917, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 17, 7, 4066, 2373, 0, 3639, 8, 3548, 0, 0, 40, 8, 2, 1, 307, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3157, 9, 6, 4055, 0, 835, 1, 8, 0, 7, 0, 0, 9, 752, 4084, 3904, 905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 7, 14, 0, 0, 7, 0, 7, 3886, 1, 3816, 0, 0, 0, 0, 4, 4089, 4087, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15576, 16323, 15723, 15739, 14778, 19092, 17732, 16109] avg=16384.0000 max=19092 max_rank=5 min=14778 min_rank=4 imbalance_ratio=1.165283 max_load_reduction=111980 max_load_reduction_pct=0.854340 moved_to_echo=67711 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1235, 0, 814, 0, 0, 3769, 16, 4094, 3664, 4, 0, 0, 0, 4092, 791, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2, 3895, 4075, 0, 0, 3938, 0, 0, 0, 4087, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3874, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 2892, 1467, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 3333, 0, 0, 2301, 1, 1714, 10, 0, 0, 1483, 31, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 952, 1, 0, 0, 0, 8, 0, 1561, 0, 4095, 4067, 1910, 1739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 3998, 553, 2, 1765, 0, 633, 0, 3, 0, 0, 0, 0, 6, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 160, 0, 0, 4081, 0, 1454, 1, 0, 0, 0, 2, 7, 0, 224, 4075, 3912, 910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 39, 0, 3941, 0, 4080, 237, 0, 25, 0, 0, 4096, 4096, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18635, 16527, 16761, 17051, 18418, 12132, 14828, 16720] avg=16384.0000 max=18635 max_rank=0 min=12132 min_rank=5 imbalance_ratio=1.137390 max_load_reduction=112437 max_load_reduction_pct=0.857826 moved_to_echo=62098 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 815, 0, 47, 2, 3, 1082, 0, 4087, 3650, 0, 3, 2, 0, 4092, 786, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 2, 0, 4039, 4094, 3, 0, 3938, 0, 0, 0, 4091, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4086, 9, 3, 4093, 1, 0, 3, 3, 4093, 3, 22, 3, 3, 2654, 1463, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 426, 2, 0, 2813, 0, 1721, 22, 0, 2, 3111, 31, 1, 3, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4085, 840, 0, 2, 0, 0, 0, 0, 517, 3, 4093, 4064, 1919, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 3, 4059, 2357, 0, 2826, 3, 1583, 0, 0, 0, 3, 0, 1, 507, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2273, 0, 3, 4061, 1, 2285, 7, 3, 3, 2, 0, 0, 0, 182, 4072, 3910, 905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 11, 2, 2, 0, 31, 3, 3889, 0, 4071, 23, 0, 16, 0, 0, 4093, 4093, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14722, 16579, 16770, 16239, 16154, 16479, 17707, 16422] avg=16384.0000 max=17707 max_rank=6 min=14722 min_rank=0 imbalance_ratio=1.080750 max_load_reduction=113365 max_load_reduction_pct=0.864906 moved_to_echo=66762 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1443, 0, 2040, 0, 0, 2800, 0, 4090, 3960, 0, 0, 0, 0, 4096, 787, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3968, 4074, 0, 0, 3939, 0, 0, 5, 4091, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4027, 22, 28, 4096, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 3257, 1465, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 3130, 0, 0, 1632, 0, 1725, 8, 0, 0, 2023, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1356, 0, 0, 0, 1, 4, 0, 468, 0, 4096, 4027, 1920, 767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 2, 0, 4094, 343, 0, 2720, 0, 1514, 0, 0, 0, 0, 0, 0, 12, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 197, 0, 0, 4056, 0, 770, 5, 0, 0, 0, 3, 4, 0, 206, 4076, 3913, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 11, 0, 4091, 0, 4084, 114, 0, 15, 0, 0, 4096, 4096, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19369, 16343, 17320, 16679, 16730, 13782, 14137, 16712] avg=16384.0000 max=19369 max_rank=0 min=13782 min_rank=5 imbalance_ratio=1.182190 max_load_reduction=111703 max_load_reduction_pct=0.852226 moved_to_echo=61361 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 13440, 128, 128, 32640, 128, 9728, 128, 128, 128, 128, 128, 128, 128, 4096, 32768, 31360, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 32256, 128, 128, 32768, 128, 128, 128, 128, 32768, 128, 128, 128, 128, 18944, 11776, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 12672, 128, 128, 21120, 128, 13824, 512, 0, 128, 17664, 384, 128, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 11776, 128, 4480, 128, 128, 11264, 128, 32768, 30848, 128, 128, 128, 0, 32768, 6400, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 128, 32256, 32768, 128, 0, 31616, 128, 0, 128, 32768, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 128, 256, 128, 30976, 128, 32256, 640, 0, 128, 0, 128, 32768, 32768, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 4480, 128, 128, 0, 128, 128, 0, 8704, 128, 32768, 31232, 15360, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32512, 128, 128, 32512, 13824, 128, 23040, 128, 18560, 128, 128, 128, 128, 128, 128, 2176, 8576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 60, 4070, 0, 0, 2, 3139, 4096, 0, 0, 4051, 0, 0, 1, 20, 0, 0, 0, 0, 2980, 3990, 0, 2253, 0, 0, 0, 4096, 0, 0, 4092, 4008, 0, 226, 1, 0, 4095, 0, 18, 4095, 0, 2, 0, 4, 0, 0, 4096, 4096, 0, 0, 0, 3645, 1, 0, 0, 0, 0, 1, 2010, 2589, 0, 4096, 0, 1, 0, 0, 4089, 0, 0, 4091, 3258, 0, 0, 0, 4093, 0, 106, 5, 4079, 0, 4069, 4096, 0, 0, 4096, 0, 0, 0, 1, 118, 0, 3247, 482, 4095, 0, 3399, 2256, 0, 0, 3958, 0, 0, 0, 0, 4096, 3901, 0, 0, 0, 31, 200, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 3375, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 144, 4080, 2, 0, 3, 3596, 4096, 0, 0, 4064, 12, 0, 0, 3, 0, 0, 0, 0, 3452, 3987, 0, 1071, 8, 0, 0, 4094, 0, 3, 3986, 4092, 3, 156, 0, 0, 4089, 5, 1188, 4096, 9, 7, 0, 0, 14, 0, 4096, 4096, 0, 0, 0, 3046, 0, 2, 0, 0, 0, 6, 2954, 1163, 0, 4096, 0, 1, 0, 4, 4091, 0, 0, 4095, 3875, 0, 0, 0, 4085, 1, 229, 7, 4056, 0, 4073, 4092, 0, 0, 4002, 0, 5, 0, 0, 18, 0, 3624, 176, 4083, 0, 3739, 3301, 0, 0, 3962, 0, 5, 0, 0, 4096, 3695, 0, 0, 0, 65, 38, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4094, 0, 1832, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[8, 1, 10, 8, 59, 4088, 0, 9, 0, 2989, 4087, 0, 2, 4069, 0, 7, 11, 6, 8, 0, 9, 2, 3029, 4072, 0, 3434, 104, 9, 6, 4083, 0, 0, 4078, 3847, 0, 554, 10, 0, 4087, 0, 78, 4087, 1, 1, 16, 0, 14, 7, 4086, 4088, 0, 8, 0, 3246, 0, 3, 0, 8, 2, 0, 2673, 335, 1, 4084, 11, 0, 3, 27, 4088, 8, 0, 4088, 3538, 9, 9, 8, 4077, 0, 3, 3, 4086, 1, 4096, 4087, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 3513, 916, 4065, 7, 3589, 1786, 0, 8, 3575, 0, 0, 6, 9, 4085, 4034, 8, 5, 3, 28, 21, 8, 0, 1, 80, 0, 0, 8, 0, 9, 4087, 0, 3550, 8, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 337, 4088, 5, 0, 0, 3374, 4096, 0, 0, 3926, 31, 0, 0, 0, 0, 7, 0, 0, 3849, 3627, 0, 2114, 5, 0, 0, 4090, 0, 0, 3819, 3964, 0, 98, 0, 0, 4092, 3, 110, 4096, 11, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 0, 3272, 0, 4, 0, 0, 0, 44, 2543, 3454, 0, 4096, 0, 0, 0, 9, 4046, 0, 0, 4093, 3883, 0, 0, 0, 4051, 0, 120, 50, 4082, 0, 4053, 4094, 0, 0, 4010, 0, 9, 0, 0, 380, 0, 4037, 255, 4087, 0, 2953, 1781, 0, 0, 4068, 0, 9, 0, 0, 4096, 2587, 0, 0, 0, 143, 509, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4092, 0, 2222, 0, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 202, 4096, 2, 0, 0, 3420, 4096, 0, 0, 4088, 4, 0, 0, 2, 0, 0, 0, 0, 3256, 4080, 0, 3283, 51, 0, 0, 4096, 0, 0, 4054, 4067, 0, 160, 0, 0, 4091, 0, 238, 4096, 1, 0, 0, 0, 4, 0, 4096, 4096, 0, 0, 3, 3130, 0, 0, 0, 0, 0, 8, 3197, 700, 0, 4096, 0, 1, 0, 0, 4072, 0, 0, 4095, 3658, 0, 0, 0, 4090, 0, 24, 29, 4095, 0, 4080, 4096, 0, 0, 4046, 0, 0, 0, 0, 53, 0, 2348, 669, 4073, 0, 2550, 2971, 0, 0, 2783, 0, 0, 0, 0, 4095, 4090, 0, 0, 0, 28, 245, 0, 0, 0, 16, 0, 0, 0, 0, 0, 4096, 0, 4054, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 118, 4065, 7, 0, 0, 2986, 4096, 0, 0, 4051, 19, 0, 0, 0, 0, 7, 0, 0, 3971, 4026, 0, 499, 25, 0, 0, 4090, 0, 0, 4049, 4081, 1, 81, 0, 0, 4085, 3, 181, 4096, 12, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4022, 0, 2, 0, 0, 0, 7, 2648, 2475, 0, 4096, 0, 0, 0, 1, 4080, 0, 0, 4091, 3600, 0, 0, 0, 4092, 4, 590, 17, 4038, 0, 4064, 4094, 0, 0, 4067, 0, 4, 0, 0, 98, 0, 4078, 429, 4081, 0, 2988, 3148, 0, 0, 4009, 0, 11, 0, 0, 4096, 3336, 0, 0, 0, 61, 317, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4093, 0, 1584, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[3, 0, 3, 3, 340, 4038, 0, 0, 0, 2035, 4093, 0, 0, 4043, 0, 3, 1, 2, 3, 0, 0, 3, 3203, 4055, 0, 2422, 47, 3, 2, 4090, 0, 0, 4030, 4080, 0, 1351, 3, 1, 4091, 0, 410, 4093, 1, 1, 0, 2, 7, 3, 4093, 4093, 0, 3, 1, 3417, 0, 5, 0, 3, 3, 7, 2693, 918, 1, 4093, 2, 0, 3, 13, 4093, 3, 2, 4075, 3215, 3, 3, 3, 4054, 0, 5, 3, 4082, 2, 4087, 4093, 0, 2, 4048, 0, 0, 0, 1, 58, 0, 3975, 2032, 4079, 3, 3567, 2333, 0, 3, 2641, 3, 0, 2, 3, 4093, 3834, 3, 0, 0, 29, 152, 3, 0, 3, 1, 0, 0, 3, 0, 0, 4093, 0, 2567, 3, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 57, 4094, 1, 0, 0, 3324, 4096, 0, 0, 4084, 4, 0, 0, 7, 0, 3, 0, 0, 3220, 4074, 0, 1847, 1, 0, 0, 4095, 0, 0, 4084, 4055, 1, 470, 0, 0, 4095, 1, 238, 4096, 1, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3702, 0, 1, 0, 0, 0, 5, 2010, 725, 0, 4096, 0, 0, 0, 1, 4085, 0, 0, 4083, 3224, 0, 0, 0, 4093, 1, 187, 31, 4071, 0, 4095, 4095, 0, 0, 4090, 0, 5, 0, 0, 32, 0, 3978, 198, 4092, 0, 3119, 3836, 0, 0, 3680, 0, 1, 0, 0, 4096, 3975, 0, 0, 0, 36, 456, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4095, 0, 2734, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 144, 4080, 2, 0, 3, 3596, 4096, 0, 0, 4064, 12, 0, 555, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3452, 3987, 0, 1071, 8, 0, 0, 4094, 0, 3, 2375, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 4092, 3, 156, 0, 0, 4089, 5, 1188, 3534, 9, 7, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 609, 0, 0, 0, 3046, 0, 2, 0, 0, 0, 6, 2954, 1163, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 4091, 0, 0, 4095, 3875, 0, 0, 0, 4085, 1, 229, 7, 244, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 3907, 0, 0, 0, 4002, 0, 5, 0, 0, 18, 0, 3624, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3739, 3301, 0, 0, 3962, 0, 5, 0, 0, 3143, 3695, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4094, 0, 1832, 0, 0, 0, 4092, 4083, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16870, 15391, 17083, 15972, 16798, 15788, 17910, 15260] avg=16384.0000 max=17910 max_rank=6 min=15260 min_rank=7 imbalance_ratio=1.093140 max_load_reduction=113162 max_load_reduction_pct=0.863358 moved_to_echo=65756 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 60, 4070, 0, 0, 2, 3139, 4096, 0, 0, 4051, 0, 0, 555, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 0, 0, 0, 0, 2980, 3990, 0, 2253, 0, 0, 0, 4096, 0, 0, 2375, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4008, 0, 226, 1, 0, 4095, 0, 18, 3533, 0, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 609, 0, 0, 0, 3645, 1, 0, 0, 0, 0, 1, 2010, 2589, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 0, 0, 4091, 3258, 0, 0, 0, 4093, 0, 106, 5, 244, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 3903, 0, 0, 0, 4096, 0, 0, 0, 1, 118, 0, 3247, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3399, 2256, 0, 0, 3958, 0, 0, 0, 0, 3143, 3901, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 3375, 0, 0, 0, 4096, 4095, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16291, 16113, 15979, 17047, 16053, 15926, 16688, 16975] avg=16384.0000 max=17047 max_rank=3 min=15926 min_rank=5 imbalance_ratio=1.040466 max_load_reduction=114025 max_load_reduction_pct=0.869942 moved_to_echo=65642 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[8, 1, 10, 8, 59, 4088, 0, 9, 0, 2989, 4087, 0, 2, 4069, 0, 7, 554, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 6, 8, 0, 9, 2, 3029, 4072, 0, 3434, 104, 9, 6, 4083, 0, 0, 2370, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 3847, 0, 554, 10, 0, 4087, 0, 78, 3526, 1, 1, 16, 0, 14, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 609, 0, 8, 0, 3246, 0, 3, 0, 8, 2, 0, 2673, 335, 1, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 3, 27, 4088, 8, 0, 4088, 3538, 9, 9, 8, 4077, 0, 3, 3, 244, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 3929, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 3513, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3589, 1786, 0, 8, 3575, 0, 0, 6, 9, 3134, 4034, 8, 5, 3, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8, 0, 1, 80, 0, 0, 8, 0, 9, 4087, 0, 3550, 8, 0, 0, 4087, 4065, 1109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16208, 17540, 16219, 15055, 16283, 16542, 16192, 17033] avg=16384.0000 max=17540 max_rank=1 min=15055 min_rank=3 imbalance_ratio=1.070557 max_load_reduction=113532 max_load_reduction_pct=0.866180 moved_to_echo=66050 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 202, 4096, 2, 0, 0, 3420, 4096, 0, 0, 4088, 4, 0, 555, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3256, 4080, 0, 3283, 51, 0, 0, 4096, 0, 0, 2375, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 4067, 0, 160, 0, 0, 4091, 0, 238, 3534, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 609, 0, 0, 3, 3130, 0, 0, 0, 0, 0, 8, 3197, 700, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4072, 0, 0, 4095, 3658, 0, 0, 0, 4090, 0, 24, 29, 244, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3914, 0, 0, 0, 4046, 0, 0, 0, 0, 53, 0, 2348, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2550, 2971, 0, 0, 2783, 0, 0, 0, 0, 3142, 4090, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, 16, 0, 0, 0, 0, 0, 4096, 0, 4054, 0, 2, 0, 4096, 4073, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16781, 17541, 16149, 15839, 16379, 15125, 15564, 17694] avg=16384.0000 max=17694 max_rank=7 min=15125 min_rank=5 imbalance_ratio=1.079956 max_load_reduction=113378 max_load_reduction_pct=0.865005 moved_to_echo=64762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 57, 4094, 1, 0, 0, 3324, 4096, 0, 0, 4084, 4, 0, 555, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 0, 0, 3220, 4074, 0, 1847, 1, 0, 0, 4095, 0, 0, 2375, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4055, 1, 470, 0, 0, 4095, 1, 238, 3534, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 609, 0, 0, 0, 3702, 0, 1, 0, 0, 0, 5, 2010, 725, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 0, 0, 4083, 3224, 0, 0, 0, 4093, 1, 187, 31, 244, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 3928, 0, 0, 0, 4090, 0, 5, 0, 0, 32, 0, 3978, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3119, 3836, 0, 0, 3680, 0, 1, 0, 0, 3143, 3975, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4095, 0, 2734, 0, 0, 0, 4095, 4092, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16533, 16020, 16480, 15244, 16116, 16302, 17790, 16587] avg=16384.0000 max=17790 max_rank=6 min=15244 min_rank=3 imbalance_ratio=1.085815 max_load_reduction=113282 max_load_reduction_pct=0.864273 moved_to_echo=66795 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 118, 4065, 7, 0, 0, 2986, 4096, 0, 0, 4051, 19, 0, 561, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3971, 4026, 0, 499, 25, 0, 0, 4090, 0, 0, 2377, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 4081, 1, 81, 0, 0, 4085, 3, 181, 3525, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 605, 0, 0, 0, 4022, 0, 2, 0, 0, 0, 7, 2648, 2475, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4080, 0, 0, 4091, 3600, 0, 0, 0, 4092, 4, 590, 17, 249, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 3894, 0, 0, 0, 4067, 0, 4, 0, 0, 98, 0, 4078, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2988, 3148, 0, 0, 4009, 0, 11, 0, 0, 3130, 3336, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4093, 0, 1584, 0, 1, 0, 4094, 4081, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16225, 15400, 16018, 17951, 16894, 16608, 16683, 15293] avg=16384.0000 max=17951 max_rank=3 min=15293 min_rank=7 imbalance_ratio=1.095642 max_load_reduction=113121 max_load_reduction_pct=0.863045 moved_to_echo=65478 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3, 3, 340, 4038, 0, 0, 0, 2035, 4093, 0, 0, 4043, 0, 3, 555, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 0, 0, 3, 3203, 4055, 0, 2422, 47, 3, 2, 4090, 0, 0, 2373, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 4080, 0, 1351, 3, 1, 4091, 0, 410, 3531, 1, 1, 0, 2, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 609, 0, 3, 1, 3417, 0, 5, 0, 3, 3, 7, 2693, 918, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 13, 4093, 3, 2, 4075, 3215, 3, 3, 3, 4054, 0, 5, 3, 244, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 2, 3920, 0, 0, 2, 4048, 0, 0, 0, 1, 58, 0, 3975, 2032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3567, 2333, 0, 3, 2641, 3, 0, 2, 3, 3140, 3834, 3, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 3, 0, 3, 1, 0, 0, 3, 0, 0, 4093, 0, 2567, 3, 0, 2, 4093, 4079, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15434, 16602, 17511, 15846, 15888, 18120, 15561, 16110] avg=16384.0000 max=18120 max_rank=5 min=15434 min_rank=0 imbalance_ratio=1.105957 max_load_reduction=112952 max_load_reduction_pct=0.861755 moved_to_echo=65679 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 337, 4088, 5, 0, 0, 3374, 4096, 0, 0, 3926, 31, 0, 555, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3849, 3627, 0, 2114, 5, 0, 0, 4090, 0, 0, 2375, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3819, 3964, 0, 98, 0, 0, 4092, 3, 110, 3534, 11, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 609, 0, 0, 0, 3272, 0, 4, 0, 0, 0, 44, 2543, 3454, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4046, 0, 0, 4093, 3883, 0, 0, 0, 4051, 0, 120, 50, 244, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 3888, 0, 0, 0, 4010, 0, 9, 0, 0, 380, 0, 4037, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2953, 1781, 0, 0, 4068, 0, 9, 0, 0, 3143, 2587, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4092, 0, 2222, 0, 3, 0, 4094, 4087, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16730, 16465, 15633, 18118, 16661, 16661, 14684, 16120] avg=16384.0000 max=18118 max_rank=3 min=14684 min_rank=6 imbalance_ratio=1.105835 max_load_reduction=112954 max_load_reduction_pct=0.861771 moved_to_echo=64126 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 4992, 0, 128, 128, 27520, 128, 128, 0, 128, 128, 128, 20736, 12416, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 1408, 32640, 128, 128, 128, 24960, 32768, 0, 128, 32384, 128, 128, 4480, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32640, 128, 31360, 0, 0, 128, 32512, 0, 128, 0, 128, 768, 0, 28800, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32256, 32256, 128, 3200, 128, 128, 32768, 128, 2560, 28288, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 25984, 21504, 0, 128, 28800, 128, 128, 128, 128, 25216, 29568, 128, 128, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 128, 27008, 32000, 0, 17024, 256, 128, 128, 32768, 0, 128, 19072, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[2048, 128, 0, 128, 128, 128, 0, 128, 0, 128, 32768, 0, 22016, 128, 128, 128, 32768, 32768, 8960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 32768, 28288, 128, 128, 128, 32640, 128, 1280, 256, 2048, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 4094, 0, 0, 0, 163, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 26, 0, 1, 0, 0, 0, 0, 4088, 0, 0, 0, 3770, 4096, 4096, 0, 4096, 0, 4096, 1428, 2, 0, 0, 0, 3779, 4045, 0, 0, 4096, 4096, 510, 4093, 0, 28, 0, 0, 0, 48, 4096, 0, 0, 0, 4096, 0, 0, 1, 3035, 4096, 0, 103, 4095, 3609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 77, 0, 0, 4096, 3, 0, 0, 1, 4096, 4095, 0, 0, 0, 4009, 0, 0, 4096, 0, 0, 32, 4068, 84, 4096, 0, 0, 0, 19, 0, 3, 0, 0, 3945, 8, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 4085, 0, 0, 3, 356, 0, 1, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3910, 4096, 4096, 0, 4096, 0, 4096, 955, 0, 0, 0, 0, 3445, 4095, 0, 0, 4096, 4096, 240, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3468, 4096, 0, 1963, 4095, 3444, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4089, 0, 24, 0, 0, 0, 4096, 0, 0, 4088, 0, 0, 223, 0, 0, 4096, 34, 0, 0, 0, 4096, 4096, 0, 0, 0, 3588, 0, 0, 4096, 0, 0, 0, 3595, 10, 4096, 0, 1, 0, 0, 0, 7, 0, 0, 3429, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4086, 0, 0, 0, 472, 0, 1, 0, 4096, 5, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3356, 4096, 4094, 0, 4096, 0, 4095, 908, 0, 0, 0, 1, 3823, 4091, 0, 0, 4096, 4096, 405, 4094, 0, 7, 0, 0, 0, 2, 4095, 0, 0, 0, 4096, 0, 0, 0, 2228, 4096, 1, 3286, 4096, 2611, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4086, 0, 17, 0, 0, 0, 4096, 0, 1, 4092, 0, 0, 22, 0, 0, 4096, 4, 0, 0, 3, 4095, 4095, 0, 1, 0, 3916, 0, 0, 4096, 0, 0, 15, 4062, 12, 4095, 0, 0, 1, 4, 0, 7, 0, 0, 3531, 3, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4076, 0, 0, 0, 651, 0, 0, 0, 4096, 23, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 3541, 4096, 4085, 0, 4096, 0, 4096, 2546, 0, 0, 0, 2, 3732, 4074, 0, 0, 4096, 4096, 1317, 4089, 0, 7, 0, 0, 0, 28, 4096, 0, 0, 0, 4096, 0, 0, 0, 3491, 4096, 2, 2077, 4096, 2981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 32, 0, 0, 0, 4096, 0, 2, 4077, 0, 0, 518, 0, 0, 4096, 30, 0, 0, 11, 4094, 4094, 0, 0, 0, 2912, 0, 0, 4096, 0, 0, 28, 3671, 68, 4084, 0, 0, 0, 44, 0, 22, 0, 0, 998, 20, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[18, 0, 4055, 0, 0, 0, 260, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 3987, 4096, 4089, 0, 4096, 0, 4096, 302, 3, 0, 0, 3, 4094, 4073, 0, 0, 4096, 4096, 1290, 4087, 0, 20, 0, 0, 0, 51, 4096, 0, 0, 0, 4096, 0, 0, 28, 3771, 4096, 0, 1840, 4094, 3276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 641, 0, 0, 4096, 214, 0, 0, 0, 4096, 4093, 0, 0, 0, 3977, 0, 0, 4096, 0, 0, 56, 3316, 181, 4085, 0, 0, 0, 16, 0, 6, 0, 0, 1373, 34, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 4043, 2, 0, 4, 130, 10, 0, 8, 4087, 0, 0, 0, 8, 0, 1, 9, 0, 6, 0, 0, 4, 8, 0, 4085, 8, 7, 7, 4086, 4086, 4088, 1, 4087, 8, 4087, 1959, 17, 0, 2, 4, 3730, 4089, 8, 0, 4084, 4086, 1084, 4082, 9, 0, 2, 0, 0, 28, 4088, 10, 8, 0, 4087, 0, 0, 139, 2159, 4086, 0, 1170, 4086, 1711, 6, 0, 3, 14, 0, 0, 9, 0, 20, 8, 3967, 0, 1, 8, 8, 7, 4078, 9, 0, 4088, 0, 9, 4, 9, 0, 4085, 2, 4, 9, 45, 4086, 4083, 7, 13, 0, 4081, 7, 10, 4067, 0, 0, 0, 4094, 226, 4089, 0, 8, 0, 93, 0, 0, 0, 0, 4069, 0, 4087, 6, 0, 4054] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 3, 4045, 0, 2, 0, 345, 2, 0, 2, 4093, 8, 2, 0, 3, 0, 0, 3, 8, 3, 1, 0, 2, 3, 0, 4076, 0, 3, 3, 4027, 4093, 4090, 2, 4093, 3, 4093, 1399, 2, 0, 1, 3, 4004, 4056, 1, 1, 4093, 4093, 554, 4093, 3, 0, 1, 0, 0, 22, 4093, 3, 3, 0, 4093, 0, 0, 16, 2360, 4093, 0, 521, 4085, 3006, 2, 0, 2, 0, 0, 0, 3, 0, 1, 3, 4092, 0, 0, 3, 3, 3, 4092, 3, 0, 4093, 2, 3, 44, 3, 0, 4092, 1, 1, 3, 25, 4093, 4093, 3, 14, 0, 4082, 1, 3, 4091, 3, 0, 4, 4041, 187, 4093, 0, 0, 0, 94, 0, 0, 0, 0, 4006, 0, 4093, 2, 0, 4088] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[26, 0, 4081, 0, 0, 0, 100, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 13, 0, 3, 0, 0, 0, 0, 4091, 0, 0, 0, 3864, 4096, 4090, 0, 4096, 0, 4093, 1225, 0, 0, 0, 4, 3951, 4091, 0, 0, 4096, 4095, 1502, 4090, 0, 8, 0, 0, 0, 8, 4093, 0, 0, 0, 4096, 0, 0, 1, 2068, 4096, 2, 2002, 4096, 2881, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4058, 0, 14, 0, 0, 0, 4096, 0, 2, 4084, 0, 0, 54, 0, 3, 4096, 135, 0, 0, 0, 4095, 4092, 0, 0, 0, 3631, 0, 0, 4096, 0, 1, 4, 3784, 93, 4087, 0, 0, 0, 4, 0, 6, 0, 0, 3378, 11, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4086, 0, 0, 0, 472, 0, 1, 0, 4096, 5, 0, 0, 0, 1, 4096, 3797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3356, 4096, 4094, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 908, 0, 0, 0, 1, 3823, 4091, 0, 0, 2185, 0, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 7, 0, 0, 0, 2, 4095, 0, 0, 0, 4096, 0, 0, 0, 2228, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3653, 1, 3286, 4096, 2611, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 4096, 0, 1, 4092, 0, 0, 22, 0, 0, 4096, 4, 3536, 299, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 559, 4095, 0, 1, 0, 3916, 0, 0, 4096, 0, 0, 15, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4095, 0, 0, 1, 4, 0, 7, 0, 0, 3531, 3, 4096, 0, 0, 4096, 688, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16554, 15917, 15508, 15744, 17736, 16164, 16747, 16702] avg=16384.0000 max=17736 max_rank=4 min=15508 min_rank=2 imbalance_ratio=1.082520 max_load_reduction=113336 max_load_reduction_pct=0.864685 moved_to_echo=67349 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4085, 0, 0, 3, 356, 0, 1, 0, 4096, 0, 0, 0, 0, 1, 4096, 3797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3910, 4096, 4096, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 955, 0, 0, 0, 0, 3445, 4095, 0, 0, 2185, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3468, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3653, 0, 1963, 4095, 3444, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 4096, 0, 0, 4088, 0, 0, 223, 0, 0, 4096, 34, 3537, 299, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 4096, 0, 0, 0, 3588, 0, 0, 4096, 0, 0, 0, 3595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 1, 0, 0, 0, 7, 0, 0, 3429, 0, 4096, 0, 0, 4096, 688, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16436, 16471, 15016, 16979, 17246, 16398, 15934, 16592] avg=16384.0000 max=17246 max_rank=4 min=15016 min_rank=2 imbalance_ratio=1.052612 max_load_reduction=113826 max_load_reduction_pct=0.868423 moved_to_echo=66170 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4094, 0, 0, 0, 163, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 3798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 1, 0, 0, 0, 0, 4088, 0, 0, 0, 3770, 4096, 4096, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1428, 2, 0, 0, 0, 3779, 4045, 0, 0, 2185, 0, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 28, 0, 0, 0, 48, 4096, 0, 0, 0, 4096, 0, 0, 1, 3035, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3653, 0, 103, 4095, 3609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 77, 0, 0, 4096, 3, 3537, 298, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 559, 4095, 0, 0, 0, 4009, 0, 0, 4096, 0, 0, 32, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 4096, 0, 0, 0, 19, 0, 3, 0, 0, 3945, 8, 4096, 0, 0, 4096, 688, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16249, 16351, 16045, 16619, 15539, 16205, 16860, 17204] avg=16384.0000 max=17204 max_rank=7 min=15539 min_rank=4 imbalance_ratio=1.050049 max_load_reduction=113868 max_load_reduction_pct=0.868744 moved_to_echo=65808 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4076, 0, 0, 0, 651, 0, 0, 0, 4096, 23, 0, 0, 0, 0, 4096, 3798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 3541, 4096, 4085, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2546, 0, 0, 0, 2, 3732, 4074, 0, 0, 2170, 0, 1317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 7, 0, 0, 0, 28, 4096, 0, 0, 0, 4096, 0, 0, 0, 3491, 1227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3645, 2, 2077, 4096, 2981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 4096, 0, 2, 4077, 0, 0, 518, 0, 0, 4096, 30, 3540, 298, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 554, 4094, 0, 0, 0, 2912, 0, 0, 4096, 0, 0, 28, 3671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 4084, 0, 0, 0, 44, 0, 22, 0, 0, 998, 20, 4096, 0, 0, 4096, 692, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16740, 16108, 17937, 17034, 16896, 16696, 15366, 14295] avg=16384.0000 max=17937 max_rank=2 min=14295 min_rank=7 imbalance_ratio=1.094788 max_load_reduction=113135 max_load_reduction_pct=0.863152 moved_to_echo=63253 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4043, 2, 0, 4, 130, 10, 0, 8, 4087, 0, 0, 0, 8, 0, 4087, 3789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 0, 6, 0, 0, 4, 8, 0, 4085, 8, 7, 7, 4086, 4086, 4088, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 4087, 1959, 17, 0, 2, 4, 3730, 4089, 8, 0, 2178, 0, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 9, 0, 2, 0, 0, 28, 4088, 10, 8, 0, 4087, 0, 0, 139, 2159, 1219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3644, 0, 1170, 4086, 1711, 6, 0, 3, 14, 0, 0, 9, 0, 20, 8, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 8, 7, 4078, 9, 0, 4088, 0, 9, 4, 9, 0, 4085, 2, 3528, 297, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 9, 45, 558, 4083, 7, 13, 0, 4081, 7, 10, 4067, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 4089, 0, 8, 0, 93, 0, 0, 0, 0, 4069, 0, 4087, 6, 0, 4054, 686, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16169, 16668, 17167, 15831, 14638, 16134, 16978, 17487] avg=16384.0000 max=17487 max_rank=7 min=14638 min_rank=4 imbalance_ratio=1.067322 max_load_reduction=113585 max_load_reduction_pct=0.866585 moved_to_echo=65237 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[18, 0, 4055, 0, 0, 0, 260, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 4096, 3797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 3987, 4096, 4089, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 302, 3, 0, 0, 3, 4094, 4073, 0, 0, 2185, 0, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 20, 0, 0, 0, 51, 4096, 0, 0, 0, 4096, 0, 0, 28, 3771, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3653, 0, 1840, 4094, 3276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 641, 0, 0, 4096, 214, 3537, 299, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 4093, 0, 0, 0, 3977, 0, 0, 4096, 0, 0, 56, 3316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 4085, 0, 0, 0, 16, 0, 6, 0, 0, 1373, 34, 4096, 0, 0, 4096, 688, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16334, 16540, 16046, 17371, 16959, 16981, 16097, 14744] avg=16384.0000 max=17371 max_rank=3 min=14744 min_rank=7 imbalance_ratio=1.060242 max_load_reduction=113701 max_load_reduction_pct=0.867470 moved_to_echo=64781 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4045, 0, 2, 0, 345, 2, 0, 2, 4093, 8, 2, 0, 3, 0, 4093, 3795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 3, 1, 0, 2, 3, 0, 4076, 0, 3, 3, 4027, 4093, 4090, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 4093, 1399, 2, 0, 1, 3, 4004, 4056, 1, 1, 2183, 0, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 1, 0, 0, 22, 4093, 3, 3, 0, 4093, 0, 0, 16, 2360, 1221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3650, 0, 521, 4085, 3006, 2, 0, 2, 0, 0, 0, 3, 0, 1, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 4092, 3, 0, 4093, 2, 3, 44, 3, 0, 4092, 1, 3534, 298, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 25, 559, 4093, 3, 14, 0, 4082, 1, 3, 4091, 3, 0, 4, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 4093, 0, 0, 0, 94, 0, 0, 0, 0, 4006, 0, 4093, 2, 0, 4088, 688, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16393, 16586, 16302, 15908, 15365, 16175, 16923, 17420] avg=16384.0000 max=17420 max_rank=7 min=15365 min_rank=4 imbalance_ratio=1.063232 max_load_reduction=113652 max_load_reduction_pct=0.867096 moved_to_echo=65883 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[26, 0, 4081, 0, 0, 0, 100, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 3797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 3, 0, 0, 0, 0, 4091, 0, 0, 0, 3864, 4096, 4090, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1225, 0, 0, 0, 4, 3951, 4091, 0, 0, 2185, 0, 1502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 8, 0, 0, 0, 8, 4093, 0, 0, 0, 4096, 0, 0, 1, 2068, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3653, 2, 2002, 4096, 2881, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 2, 4084, 0, 0, 54, 0, 3, 4096, 135, 3536, 298, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 4092, 0, 0, 0, 3631, 0, 0, 4096, 0, 1, 4, 3784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 4087, 0, 0, 0, 4, 0, 6, 0, 0, 3378, 11, 4096, 0, 0, 4096, 688, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16197, 16431, 17051, 15586, 16693, 16319, 16167, 16628] avg=16384.0000 max=17051 max_rank=2 min=15586 min_rank=3 imbalance_ratio=1.040710 max_load_reduction=114021 max_load_reduction_pct=0.869911 moved_to_echo=65807 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[29312, 128, 13056, 32768, 23552, 128, 0, 128, 128, 128, 0, 128, 0, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 128, 0, 32768, 128, 128, 128, 30592, 32768, 32768, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 32640, 128, 128, 128, 2560, 128, 128, 128, 32768, 128, 128, 0, 128, 128, 32768, 30464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 128, 1664, 128, 128, 32768, 512, 28288, 2432, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 256, 32768, 128, 128, 0, 32768, 0, 0, 256, 22656, 9856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 4480, 32768, 128, 128, 0, 30208, 128, 128, 32768, 128, 128, 256, 30720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 32768, 10752, 128, 0, 128, 128, 30592, 32640, 128, 128, 17536, 0, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[896, 32768, 0, 128, 128, 384, 0, 128, 0, 0, 24832, 128, 32768, 128, 0, 32768, 5632, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 48, 0, 0, 0, 0, 648, 4095, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 2571, 4096, 4096, 0, 0, 0, 0, 0, 4082, 3, 3865, 0, 0, 0, 0, 0, 4096, 3546, 187, 0, 0, 64, 0, 4096, 0, 0, 0, 3018, 0, 0, 4, 170, 0, 4094, 4095, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 699, 4096, 4084, 3680, 1761, 3397, 0, 0, 0, 2532, 4051, 0, 4096, 4096, 0, 0, 0, 4096, 806, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 7, 0, 0, 2, 0, 2, 0, 0, 0, 1735, 0, 0, 0, 4095, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4093, 0, 0, 0, 31, 0, 6, 0, 0, 21, 4096, 0, 4096, 0, 0, 0, 2, 4, 0, 4096, 4096, 0, 0, 0, 0, 2247, 4096, 4096, 0, 0, 8, 4, 0, 4080, 66, 3831, 0, 0, 1, 0, 0, 4092, 2641, 108, 0, 0, 53, 0, 4092, 0, 0, 0, 3571, 0, 0, 72, 1017, 0, 4096, 4096, 4088, 0, 13, 0, 0, 0, 0, 0, 0, 0, 2749, 4080, 4073, 3555, 2132, 2749, 0, 0, 0, 1340, 4073, 0, 4096, 4096, 0, 0, 0, 4096, 2122, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4093, 4096, 3, 0, 15, 0, 0, 4, 0, 8, 0, 0, 0, 420, 0, 0, 0, 4091, 0, 3, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 36, 0, 14, 0, 0, 709, 4096, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 1475, 4096, 4096, 0, 0, 5, 0, 0, 4070, 85, 1861, 0, 0, 0, 0, 2, 4096, 2336, 756, 0, 0, 0, 0, 4096, 0, 0, 0, 3582, 0, 0, 11, 290, 0, 4090, 4096, 4077, 0, 1, 0, 0, 0, 0, 0, 0, 0, 472, 4077, 4076, 4003, 3933, 3687, 0, 0, 0, 2438, 4072, 0, 4096, 4096, 0, 0, 0, 4096, 1303, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 4096, 4096, 0, 0, 2, 0, 0, 15, 0, 9, 0, 0, 0, 1840, 0, 0, 0, 4089, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 215, 0, 9, 0, 0, 180, 4096, 0, 4096, 0, 0, 0, 0, 8, 0, 4096, 4096, 0, 0, 0, 0, 2732, 4096, 4096, 0, 0, 16, 0, 0, 4064, 4, 3776, 0, 0, 2, 0, 0, 4094, 3029, 27, 0, 0, 124, 0, 4095, 0, 0, 0, 2962, 0, 0, 8, 251, 0, 4096, 4096, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1906, 4076, 4084, 2943, 1155, 3962, 0, 0, 0, 2781, 4058, 0, 4096, 4096, 0, 0, 0, 4096, 1538, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 4095, 4096, 0, 0, 0, 0, 0, 7, 0, 35, 0, 0, 0, 1160, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 124, 0, 11, 0, 0, 559, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 4096, 4096, 0, 0, 0, 0, 3424, 4096, 4096, 0, 0, 9, 1, 0, 4050, 7, 3945, 0, 0, 0, 0, 0, 4096, 1418, 251, 0, 0, 4, 0, 4095, 0, 0, 0, 3921, 0, 0, 3, 32, 0, 4096, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1767, 4063, 4090, 3060, 2289, 3581, 0, 0, 0, 2611, 3998, 0, 4096, 4096, 0, 0, 0, 4096, 663, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 4096, 4096, 0, 0, 62, 0, 0, 8, 0, 4, 0, 0, 0, 1230, 0, 0, 0, 4095, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4093, 0, 3, 0, 3, 0, 4093, 4093, 3, 3, 0, 0, 2, 18, 1, 0, 216, 4091, 3, 4091, 3, 0, 3, 0, 0, 3, 4092, 4093, 3, 0, 0, 0, 3355, 4093, 4093, 0, 0, 6, 0, 3, 4083, 7, 2782, 0, 0, 0, 0, 0, 4093, 2808, 226, 0, 2, 3, 3, 4093, 0, 3, 3, 2272, 0, 3, 0, 453, 0, 4093, 4093, 4061, 3, 127, 3, 3, 0, 1, 3, 0, 3, 762, 4078, 4093, 3919, 2798, 3212, 1, 1, 0, 2112, 4084, 3, 4093, 4093, 3, 0, 1, 4093, 1270, 0, 3, 0, 3, 2, 2, 0, 4060, 0, 4093, 4088, 0, 1, 21, 0, 0, 76, 0, 1, 3, 0, 0, 2356, 2, 0, 2, 4050, 3, 0, 0, 3, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4084, 0, 9, 0, 9, 4, 4087, 4088, 0, 9, 0, 0, 8, 0, 2, 0, 1251, 4080, 9, 4050, 10, 36, 8, 0, 0, 8, 4072, 4087, 8, 0, 0, 4, 2856, 4087, 4084, 0, 0, 0, 0, 9, 4086, 166, 2027, 4, 0, 1, 0, 4, 4085, 3403, 559, 9, 0, 0, 9, 4085, 0, 9, 9, 1823, 0, 9, 11, 778, 0, 4081, 4087, 4079, 9, 67, 7, 9, 0, 48, 8, 0, 9, 515, 4096, 4090, 4082, 2936, 3777, 0, 4, 0, 2619, 4080, 5, 4087, 4087, 0, 0, 0, 4087, 1014, 0, 9, 0, 9, 6, 1, 0, 4086, 0, 4087, 4075, 0, 0, 44, 0, 2, 41, 5, 0, 9, 0, 8, 693, 0, 0, 0, 4051, 5, 0, 5, 9, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 178, 0, 1, 0, 0, 1310, 4096, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 0, 0, 1350, 4095, 4096, 0, 0, 3, 1, 0, 4084, 33, 3454, 0, 0, 0, 0, 0, 4096, 2314, 443, 0, 0, 85, 0, 4095, 0, 0, 0, 3097, 0, 0, 2, 664, 0, 4095, 4096, 4092, 0, 22, 0, 0, 0, 0, 0, 0, 0, 1432, 4090, 4087, 3757, 3156, 3809, 0, 0, 0, 1427, 4071, 0, 4096, 4096, 0, 0, 0, 4096, 559, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 13, 0, 0, 0, 1616, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4093, 0, 0, 0, 31, 0, 6, 0, 0, 1630, 1384, 995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 0, 4096, 0, 0, 0, 2, 4, 0, 4096, 3484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2247, 2466, 4096, 0, 0, 8, 4, 0, 4080, 66, 3831, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2641, 108, 0, 0, 53, 0, 4092, 0, 0, 0, 3571, 0, 0, 72, 1017, 739, 612, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2811, 0, 4088, 0, 13, 0, 0, 0, 0, 0, 0, 0, 2749, 4080, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3555, 2132, 2749, 0, 0, 0, 1340, 3334, 0, 0, 0, 0, 0, 0, 0, 2122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4093, 4096, 3, 0, 15, 0, 0, 2712, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8, 0, 0, 0, 420, 0, 0, 0, 4091, 0, 3, 0, 0, 0, 4096, 4096, 2695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16330, 15799, 16799, 17287, 17814, 15232, 16398, 15413] avg=16384.0000 max=17814 max_rank=4 min=15232 min_rank=5 imbalance_ratio=1.087280 max_load_reduction=113258 max_load_reduction_pct=0.864090 moved_to_echo=64857 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 48, 0, 0, 0, 0, 1630, 1383, 994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 648, 4095, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 3484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2571, 2466, 4096, 0, 0, 0, 0, 0, 4082, 3, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3546, 187, 0, 0, 64, 0, 4096, 0, 0, 0, 3018, 0, 0, 4, 170, 735, 612, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2811, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 699, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3680, 1761, 3397, 0, 0, 0, 2532, 3316, 0, 0, 0, 0, 0, 0, 0, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 7, 0, 0, 2712, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 1735, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 2695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16345, 16420, 17083, 16817, 15787, 15492, 16407, 16721] avg=16384.0000 max=17083 max_rank=2 min=15492 min_rank=5 imbalance_ratio=1.042664 max_load_reduction=113989 max_load_reduction_pct=0.869667 moved_to_echo=64407 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 178, 0, 1, 0, 0, 1629, 1384, 995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1310, 4096, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 3484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1350, 2466, 4096, 0, 0, 3, 1, 0, 4084, 33, 3454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2314, 443, 0, 0, 85, 0, 4095, 0, 0, 0, 3097, 0, 0, 2, 664, 739, 612, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2810, 0, 4092, 0, 22, 0, 0, 0, 0, 0, 0, 0, 1432, 4090, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3757, 3156, 3809, 0, 0, 0, 1427, 3332, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4, 0, 0, 2712, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 1616, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16475, 17084, 15487, 16437, 16533, 16040, 16404, 16612] avg=16384.0000 max=17084 max_rank=1 min=15487 min_rank=2 imbalance_ratio=1.042725 max_load_reduction=113988 max_load_reduction_pct=0.869659 moved_to_echo=65589 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 215, 0, 9, 0, 0, 1633, 1385, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 4096, 0, 4096, 0, 0, 0, 0, 8, 0, 4096, 3478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2732, 2463, 4096, 0, 0, 16, 0, 0, 4064, 4, 3776, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3029, 27, 0, 0, 124, 0, 4095, 0, 0, 0, 2962, 0, 0, 8, 251, 741, 618, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2803, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1906, 4076, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2943, 1155, 3962, 0, 0, 0, 2781, 3317, 0, 0, 0, 0, 0, 0, 0, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 4095, 4096, 0, 0, 0, 0, 0, 2711, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 35, 0, 0, 0, 1160, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16530, 15954, 17153, 16242, 16930, 15696, 16382, 16185] avg=16384.0000 max=17153 max_rank=2 min=15696 min_rank=5 imbalance_ratio=1.046936 max_load_reduction=113919 max_load_reduction_pct=0.869133 moved_to_echo=65193 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 3, 0, 3, 0, 4093, 4093, 3, 3, 0, 0, 2, 18, 1, 0, 1628, 1383, 994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 4091, 3, 4091, 3, 0, 3, 0, 0, 3, 4092, 3481, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3355, 2465, 4093, 0, 0, 6, 0, 3, 4083, 7, 2782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2808, 226, 0, 2, 3, 3, 4093, 0, 3, 3, 2272, 0, 3, 0, 453, 741, 612, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2810, 0, 4061, 3, 127, 3, 3, 0, 1, 3, 0, 3, 762, 4078, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3919, 2798, 3212, 1, 1, 0, 2112, 3343, 3, 0, 0, 3, 0, 1, 0, 1270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 2, 2, 0, 4060, 0, 4093, 4088, 0, 1, 21, 0, 0, 2710, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 1, 3, 0, 0, 2356, 2, 0, 2, 4050, 3, 0, 0, 3, 3, 4093, 4093, 2693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16317, 15986, 16794, 15604, 15947, 16663, 16383, 17378] avg=16384.0000 max=17378 max_rank=7 min=15604 min_rank=3 imbalance_ratio=1.060669 max_load_reduction=113694 max_load_reduction_pct=0.867416 moved_to_echo=66371 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 36, 0, 14, 0, 0, 1630, 1384, 993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 709, 4096, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 3484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1475, 2466, 4096, 0, 0, 5, 0, 0, 4070, 85, 1861, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2336, 756, 0, 0, 0, 0, 4096, 0, 0, 0, 3582, 0, 0, 11, 290, 739, 612, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2808, 0, 4077, 0, 1, 0, 0, 0, 0, 0, 0, 0, 472, 4077, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 3933, 3687, 0, 0, 0, 2438, 3333, 0, 0, 0, 0, 0, 0, 0, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 4096, 4096, 0, 0, 2, 0, 0, 2712, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 9, 0, 0, 0, 1840, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 4096, 2695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16345, 16482, 14060, 16807, 15511, 18697, 16330, 16840] avg=16384.0000 max=18697 max_rank=5 min=14060 min_rank=2 imbalance_ratio=1.141174 max_load_reduction=112375 max_load_reduction_pct=0.857353 moved_to_echo=67378 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 9, 0, 9, 4, 4087, 4088, 0, 9, 0, 0, 8, 0, 2, 0, 1626, 1381, 991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1251, 4080, 9, 4050, 10, 36, 8, 0, 0, 8, 4072, 3476, 8, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2856, 2461, 4084, 0, 0, 0, 0, 9, 4086, 166, 2027, 4, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 3403, 559, 9, 0, 0, 9, 4085, 0, 9, 9, 1823, 0, 9, 11, 778, 741, 611, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2801, 0, 4079, 9, 67, 7, 9, 0, 48, 8, 0, 9, 515, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 2936, 3777, 0, 4, 0, 2619, 3339, 5, 0, 0, 0, 0, 0, 0, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 9, 6, 1, 0, 4086, 0, 4087, 4075, 0, 0, 44, 0, 2, 2706, 1398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 5, 0, 9, 0, 8, 693, 0, 0, 0, 4051, 5, 0, 5, 9, 8, 4087, 4087, 2689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16298, 17012, 15698, 16430, 15738, 17776, 16423, 15697] avg=16384.0000 max=17776 max_rank=5 min=15697 min_rank=7 imbalance_ratio=1.084961 max_load_reduction=113296 max_load_reduction_pct=0.864380 moved_to_echo=65634 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 124, 0, 11, 0, 0, 1630, 1384, 995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 4096, 3484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3424, 2466, 4096, 0, 0, 9, 1, 0, 4050, 7, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1418, 251, 0, 0, 4, 0, 4095, 0, 0, 0, 3921, 0, 0, 3, 32, 726, 612, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2811, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1767, 4063, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3060, 2289, 3581, 0, 0, 0, 2611, 3272, 0, 0, 0, 0, 0, 0, 0, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 4096, 4096, 0, 0, 62, 0, 0, 2712, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4, 0, 0, 0, 1230, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 4096, 4096, 2696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16432, 16335, 17998, 15448, 16812, 15476, 16345, 16226] avg=16384.0000 max=17998 max_rank=2 min=15448 min_rank=3 imbalance_ratio=1.098511 max_load_reduction=113074 max_load_reduction_pct=0.862686 moved_to_echo=64859 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[4992, 32768, 128, 32768, 128, 128, 128, 128, 128, 128, 32768, 27904, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 22528, 0, 32640, 128, 256, 128, 128, 128, 128, 128, 0, 128, 10368, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 128, 0, 32512, 0, 32768, 32768, 128, 128, 256, 0, 128, 21760, 11264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[256, 128, 128, 128, 0, 128, 11136, 128, 0, 128, 32768, 128, 128, 128, 128, 128, 32768, 32768, 21632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 0, 128, 0, 128, 128, 32768, 32768, 128, 128, 0, 640, 128, 128, 128, 0, 13056, 11136, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[20096, 19840, 32768, 0, 0, 128, 128, 128, 32640, 384, 25600, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[29056, 20224, 28288, 128, 128, 0, 17920, 26624, 128, 0, 0, 128, 0, 128, 0, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 21504, 2560, 128, 128, 384, 128, 32768, 0, 128, 128, 24320, 0, 128, 128, 3712, 6016, 4992, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 3560, 0, 4096, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1523, 0, 231, 0, 4096, 0, 0, 290, 0, 0, 0, 3, 4096, 0, 5, 0, 0, 4092, 0, 0, 742, 4093, 0, 3987, 4095, 0, 4096, 4061, 4095, 0, 51, 4088, 0, 41, 1224, 4096, 4, 0, 0, 0, 18, 4022, 1420, 0, 0, 18, 3506, 0, 0, 4096, 0, 3876, 0, 0, 0, 0, 7, 0, 0, 31, 261, 0, 0, 4096, 0, 4096, 3333, 0, 100, 2719, 4096, 2, 4091, 1986, 0, 1, 0, 17, 0, 0, 0, 4063, 4095, 0, 0, 0, 4096, 0, 0, 4012, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 3412, 0, 4093, 0, 4065, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1296, 0, 55, 0, 4096, 0, 0, 19, 0, 0, 0, 5, 4092, 0, 0, 0, 0, 4090, 0, 0, 86, 4092, 0, 4046, 4096, 0, 4096, 4076, 4096, 0, 24, 4091, 0, 15, 1546, 4096, 6, 0, 0, 0, 9, 4044, 1957, 0, 0, 30, 3678, 0, 0, 4096, 0, 4030, 0, 0, 0, 0, 19, 0, 0, 2, 1737, 0, 0, 4096, 0, 4096, 2011, 0, 27, 2080, 4096, 3, 4088, 2710, 0, 4, 0, 8, 0, 0, 0, 4095, 4096, 0, 0, 0, 4095, 0, 0, 4081, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 3720, 0, 4096, 0, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3444, 0, 45, 0, 4096, 0, 0, 56, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 354, 4096, 0, 3967, 4096, 0, 4096, 4096, 4096, 0, 18, 4096, 0, 0, 718, 4096, 0, 0, 0, 0, 0, 3953, 822, 0, 0, 9, 3798, 0, 0, 4096, 0, 4092, 0, 0, 1, 0, 0, 0, 0, 0, 359, 0, 0, 4096, 0, 4096, 1235, 0, 92, 2615, 4096, 2, 4081, 3619, 0, 0, 0, 0, 0, 1, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 3845, 0, 4096, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3226, 0, 292, 0, 4096, 0, 0, 365, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 4096, 0, 0, 64, 4095, 0, 3977, 4096, 0, 4096, 4063, 4096, 0, 1, 4084, 0, 18, 445, 4096, 1, 0, 0, 0, 0, 4022, 1699, 0, 0, 1, 3925, 0, 0, 4096, 0, 4027, 0, 0, 0, 0, 47, 0, 0, 23, 530, 0, 0, 4096, 0, 4096, 743, 0, 65, 1952, 4096, 0, 4090, 3631, 0, 2, 0, 7, 0, 0, 0, 4094, 4095, 0, 0, 0, 4096, 0, 0, 4028, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[9, 1613, 8, 4087, 9, 4054, 9, 0, 0, 0, 9, 0, 1, 8, 0, 2361, 0, 188, 9, 4087, 9, 9, 1292, 11, 9, 9, 0, 4087, 0, 0, 9, 3, 4087, 0, 0, 28, 4085, 9, 4081, 4087, 9, 4086, 4088, 4087, 0, 0, 4088, 23, 18, 196, 4085, 0, 0, 9, 9, 0, 4071, 722, 0, 0, 6, 4081, 0, 283, 4085, 8, 4018, 0, 2, 46, 0, 0, 7, 2, 0, 3096, 14, 0, 4082, 0, 4084, 1013, 0, 0, 3006, 4086, 10, 4087, 2567, 0, 0, 7, 0, 9, 0, 0, 4070, 4087, 8, 10, 10, 4087, 0, 9, 4080, 4076, 0, 9, 31, 0, 4086, 13, 0, 0, 4086, 0, 8, 9, 4, 1, 9, 6, 7, 9, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 3356, 0, 4096, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3351, 0, 304, 0, 4096, 0, 0, 39, 0, 0, 0, 5, 4096, 0, 17, 0, 0, 4092, 0, 0, 96, 4094, 0, 3909, 4096, 0, 4096, 4069, 4096, 0, 39, 4090, 0, 24, 339, 4096, 3, 0, 0, 0, 0, 4076, 1543, 0, 0, 146, 3793, 0, 0, 4096, 0, 4073, 0, 0, 0, 0, 45, 0, 0, 9, 1526, 0, 0, 4096, 0, 4096, 2274, 0, 115, 2714, 4096, 2, 4075, 1079, 0, 6, 0, 13, 0, 2, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 4053, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[3, 2282, 3, 4093, 3, 4032, 3, 0, 0, 0, 3, 0, 1, 3, 0, 1754, 0, 100, 3, 4093, 3, 3, 640, 3, 3, 3, 0, 4092, 0, 0, 3, 0, 4092, 2, 0, 0, 4090, 3, 4053, 4093, 3, 4093, 4089, 4093, 0, 0, 4093, 1, 14, 255, 4093, 2, 0, 3, 3, 0, 4069, 2847, 0, 0, 0, 4025, 0, 212, 4093, 3, 4050, 0, 0, 69, 0, 0, 2, 0, 34, 2097, 2, 0, 4090, 0, 4093, 1421, 0, 0, 1462, 4093, 53, 4093, 3386, 2, 0, 0, 0, 3, 0, 0, 4093, 4091, 3, 3, 3, 4092, 0, 3, 4067, 4093, 0, 3, 7, 0, 4092, 3, 0, 0, 4093, 0, 3, 3, 1, 1, 3, 11, 3, 1, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 3489, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2660, 0, 4, 0, 4096, 0, 0, 24, 0, 0, 0, 1, 4096, 0, 1, 0, 0, 4094, 0, 0, 98, 4095, 0, 3998, 4096, 0, 4095, 4088, 4096, 0, 10, 4095, 0, 3, 667, 4096, 2, 0, 0, 0, 1, 4084, 1807, 0, 0, 16, 3682, 0, 0, 4096, 0, 4063, 0, 0, 0, 0, 9, 0, 0, 0, 398, 0, 0, 4096, 0, 4096, 1009, 0, 10, 2767, 4096, 0, 4086, 3999, 0, 1, 0, 4, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4082, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3412, 0, 4093, 0, 4065, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1296, 1697, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 4096, 0, 0, 19, 0, 0, 0, 5, 4092, 0, 0, 0, 0, 4095, 3588, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 86, 0, 0, 4046, 0, 0, 0, 4076, 0, 0, 24, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1546, 4096, 6, 0, 0, 0, 9, 4044, 1957, 0, 0, 30, 3678, 0, 0, 1958, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4030, 0, 0, 0, 0, 19, 0, 0, 2, 1737, 0, 0, 4096, 0, 2365, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2011, 0, 27, 2080, 1217, 3, 4088, 2710, 0, 4, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 4081, 4096, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15487, 15955, 16326, 17426, 16849, 16244, 16402, 16383] avg=16384.0000 max=17426 max_rank=3 min=15487 min_rank=0 imbalance_ratio=1.063599 max_load_reduction=113646 max_load_reduction_pct=0.867050 moved_to_echo=65878 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3720, 0, 4096, 0, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3444, 1697, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 4096, 0, 0, 56, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3590, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 354, 0, 0, 3967, 0, 0, 0, 4096, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 718, 4096, 0, 0, 0, 0, 0, 3953, 822, 0, 0, 9, 3798, 0, 0, 1958, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 1, 0, 0, 0, 0, 0, 359, 0, 0, 4096, 0, 2365, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1235, 0, 92, 2615, 1217, 2, 4081, 3619, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17840, 15983, 16539, 15441, 15515, 16958, 16415, 16381] avg=16384.0000 max=17840 max_rank=0 min=15441 min_rank=3 imbalance_ratio=1.088867 max_load_reduction=113232 max_load_reduction_pct=0.863892 moved_to_echo=65269 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3845, 0, 4096, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3226, 1697, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 0, 4096, 0, 0, 365, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 4096, 3589, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 64, 0, 0, 3977, 0, 0, 0, 4063, 0, 0, 1, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 445, 4096, 1, 0, 0, 0, 0, 4022, 1699, 0, 0, 1, 3925, 0, 0, 1958, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4027, 0, 0, 0, 0, 47, 0, 0, 23, 530, 0, 0, 4096, 0, 2365, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 743, 0, 65, 1952, 1217, 0, 4090, 3631, 0, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 4028, 4096, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17861, 16541, 16198, 16252, 15690, 15803, 16347, 16380] avg=16384.0000 max=17861 max_rank=0 min=15690 min_rank=4 imbalance_ratio=1.090149 max_load_reduction=113211 max_load_reduction_pct=0.863731 moved_to_echo=64220 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3560, 0, 4096, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1523, 1697, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 4096, 0, 0, 290, 0, 0, 0, 3, 4096, 0, 5, 0, 0, 4096, 3588, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 742, 0, 0, 3987, 0, 0, 0, 4061, 0, 0, 51, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 1224, 4096, 4, 0, 0, 0, 18, 4022, 1420, 0, 0, 18, 3506, 0, 0, 1958, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3876, 0, 0, 0, 0, 7, 0, 0, 31, 261, 0, 0, 4096, 0, 2365, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3333, 0, 100, 2719, 1217, 2, 4091, 1986, 0, 1, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4095, 0, 0, 0, 0, 0, 0, 4012, 4096, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4095, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15856, 16409, 16934, 16394, 15237, 17562, 16300, 16380] avg=16384.0000 max=17562 max_rank=5 min=15237 min_rank=4 imbalance_ratio=1.071899 max_load_reduction=113510 max_load_reduction_pct=0.866013 moved_to_echo=65479 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3489, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2660, 1697, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 24, 0, 0, 0, 1, 4096, 0, 1, 0, 0, 4096, 3589, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 98, 0, 0, 3998, 0, 0, 0, 4088, 0, 0, 10, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 667, 4096, 2, 0, 0, 0, 1, 4084, 1807, 0, 0, 16, 3682, 0, 0, 1958, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4063, 0, 0, 0, 0, 9, 0, 0, 0, 398, 0, 0, 4096, 0, 2365, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1009, 0, 10, 2767, 1217, 0, 4086, 3999, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4082, 4096, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16956, 15911, 16296, 16403, 15533, 17189, 16404, 16380] avg=16384.0000 max=17189 max_rank=5 min=15533 min_rank=4 imbalance_ratio=1.049133 max_load_reduction=113883 max_load_reduction_pct=0.868858 moved_to_echo=65506 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[9, 1613, 8, 4087, 9, 4054, 9, 0, 0, 0, 9, 0, 1, 8, 0, 2361, 1693, 919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 9, 4087, 9, 9, 1292, 11, 9, 9, 0, 4087, 0, 0, 9, 3, 4087, 3582, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 28, 0, 9, 4081, 0, 9, 0, 4088, 0, 0, 0, 4088, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 196, 4085, 0, 0, 9, 9, 0, 4071, 722, 0, 0, 6, 4081, 0, 283, 1953, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 8, 4018, 0, 2, 46, 0, 0, 7, 2, 0, 3096, 14, 0, 4082, 0, 2359, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1013, 0, 0, 3006, 1214, 10, 4087, 2567, 0, 0, 7, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 4087, 8, 10, 10, 0, 0, 9, 4080, 4076, 0, 9, 31, 0, 34, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 8, 9, 4, 1, 9, 6, 7, 9, 0, 0, 0, 0, 4087, 4087, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14780, 17396, 16326, 15520, 18222, 15997, 16437, 16394] avg=16384.0000 max=18222 max_rank=4 min=14780 min_rank=0 imbalance_ratio=1.112183 max_load_reduction=112850 max_load_reduction_pct=0.860977 moved_to_echo=67050 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3356, 0, 4096, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3351, 1701, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 4096, 0, 0, 39, 0, 0, 0, 5, 4096, 0, 17, 0, 0, 4096, 3590, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 96, 0, 0, 3909, 0, 0, 0, 4069, 0, 0, 39, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 339, 4096, 3, 0, 0, 0, 0, 4076, 1543, 0, 0, 146, 3793, 0, 0, 1959, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4073, 0, 0, 0, 0, 45, 0, 0, 9, 1526, 0, 0, 4096, 0, 2370, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2274, 0, 115, 2714, 1213, 2, 4075, 1079, 0, 6, 0, 13, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4053, 4096, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17497, 16248, 16204, 16070, 16719, 15589, 16364, 16381] avg=16384.0000 max=17497 max_rank=0 min=15589 min_rank=5 imbalance_ratio=1.067932 max_load_reduction=113575 max_load_reduction_pct=0.866508 moved_to_echo=65053 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 2282, 3, 4093, 3, 4032, 3, 0, 0, 0, 3, 0, 1, 3, 0, 1754, 1695, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 3, 4093, 3, 3, 640, 3, 3, 3, 0, 4092, 0, 0, 3, 0, 4092, 3586, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 2, 0, 0, 0, 3, 4053, 0, 3, 0, 4089, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 255, 4093, 2, 0, 3, 3, 0, 4069, 2847, 0, 0, 0, 4025, 0, 212, 1956, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 4050, 0, 0, 69, 0, 0, 2, 0, 34, 2097, 2, 0, 4090, 0, 2363, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1421, 0, 0, 1462, 1217, 53, 4093, 3386, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4091, 3, 3, 3, 0, 0, 3, 4067, 4093, 0, 3, 7, 0, 34, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 3, 1, 1, 3, 11, 3, 1, 0, 0, 0, 0, 4093, 4093, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14795, 16629, 16249, 17566, 17307, 15730, 16403, 16393] avg=16384.0000 max=17566 max_rank=3 min=14795 min_rank=0 imbalance_ratio=1.072144 max_load_reduction=113506 max_load_reduction_pct=0.865982 moved_to_echo=65833 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 32256, 0, 128, 128, 0, 128, 128, 128, 128, 10112, 128, 0, 32768, 0, 18944, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 32768, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 0, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 128, 128, 0, 0, 128, 32512, 32768, 0, 128, 128, 0, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32128, 128, 0, 1536, 0, 128, 32128, 0, 128, 0, 32640, 0, 0, 256, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 13056, 0, 512, 19328, 9856, 128, 32768, 23040, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 1280, 128, 32768, 128, 128, 2816, 128, 128, 128, 128, 32768, 0, 128, 128, 128, 32768, 28800, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 5504, 32768, 128, 0, 128, 128, 128, 32384, 12928, 0, 0, 256, 30592, 0, 512, 15744, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 25344, 128, 32768, 128, 32512, 128, 0, 0, 0, 128, 128, 128, 128, 0, 19712, 13696, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[663, 0, 0, 4095, 0, 0, 0, 0, 0, 1088, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 4096, 9, 200, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4085, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1638, 0, 2075, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2580, 4093, 0, 4092, 4096, 0, 0, 4096, 0, 0, 4096, 4079, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[354, 0, 0, 4096, 0, 0, 9, 0, 0, 723, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4082, 4090, 0, 0, 0, 4012, 0, 0, 0, 0, 1, 0, 4095, 2, 179, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4081, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1247, 0, 3427, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 8, 0, 0, 0, 4096, 0, 0, 65, 0, 0, 0, 0, 37, 0, 4096, 0, 0, 4095, 0, 2286, 4096, 0, 4045, 4096, 0, 0, 4096, 0, 0, 4096, 4095, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1684, 7, 1, 4003, 1, 9, 29, 0, 9, 503, 4054, 9, 9, 0, 4085, 5, 3922, 11, 0, 0, 0, 4088, 4087, 0, 0, 0, 4085, 11, 0, 0, 0, 0, 0, 4087, 0, 335, 4087, 6, 10, 3952, 4087, 118, 4087, 0, 0, 7, 0, 0, 9, 3, 8, 0, 9, 0, 4085, 0, 8, 10, 0, 0, 8, 3739, 4086, 12, 4087, 0, 9, 0, 8, 10, 0, 0, 3368, 0, 1142, 3924, 10, 9, 0, 8, 4086, 6, 4087, 0, 5, 7, 9, 3, 3, 10, 6, 0, 0, 4084, 0, 4, 8, 10, 10, 4086, 0, 118, 4, 0, 3, 0, 9, 0, 0, 4086, 8, 7, 4087, 0, 1954, 4054, 9, 4080, 4086, 0, 0, 4005, 0, 66, 4082, 3962, 4088, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[49, 0, 0, 4096, 0, 0, 15, 0, 0, 18, 4092, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 4072, 4096, 0, 0, 0, 4073, 0, 0, 0, 0, 7, 0, 4086, 3, 35, 4096, 0, 0, 4096, 4096, 0, 4096, 1, 0, 0, 39, 3, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4, 0, 0, 4064, 4096, 0, 4094, 14, 0, 0, 0, 0, 0, 0, 2962, 0, 3800, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 2, 6, 0, 0, 0, 4096, 0, 0, 37, 0, 0, 2, 0, 10, 0, 4094, 0, 0, 4094, 0, 1331, 4096, 0, 4049, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[649, 0, 0, 4096, 0, 0, 18, 0, 0, 51, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4051, 4095, 0, 0, 0, 3973, 0, 0, 0, 0, 6, 0, 4093, 0, 26, 4096, 0, 0, 4096, 4096, 0, 4095, 0, 0, 0, 63, 1, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 1, 0, 0, 4075, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 2423, 0, 2889, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 51, 0, 0, 5, 0, 69, 0, 4094, 0, 0, 4086, 0, 2183, 4096, 0, 4051, 4096, 0, 0, 4096, 0, 0, 4096, 4092, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[558, 1, 0, 4085, 0, 3, 19, 0, 3, 1141, 4066, 3, 3, 0, 4092, 1, 4070, 0, 3, 0, 0, 4093, 4093, 0, 2, 0, 4093, 3, 11, 0, 0, 0, 0, 4072, 0, 373, 4093, 0, 3, 4064, 4093, 13, 4093, 0, 0, 2, 24, 0, 3, 0, 3, 0, 3, 0, 4092, 0, 3, 1, 0, 2, 3, 3840, 4093, 4, 4093, 4, 3, 0, 3, 3, 0, 0, 2394, 0, 2909, 4074, 3, 3, 2, 6, 4093, 3, 4093, 0, 3, 0, 3, 1, 0, 3, 3, 0, 0, 4093, 0, 22, 3, 3, 3, 4093, 0, 5, 36, 0, 8, 0, 3, 0, 1, 4093, 3, 2, 4093, 0, 1070, 4092, 3, 4093, 4093, 0, 0, 4076, 0, 11, 4092, 4069, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[181, 0, 0, 4096, 0, 0, 15, 2, 0, 83, 4091, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4066, 4093, 0, 0, 0, 4058, 0, 0, 0, 0, 5, 0, 4091, 7, 6, 4093, 0, 0, 4096, 4093, 0, 4096, 4, 0, 0, 20, 4, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 3, 0, 0, 4079, 4096, 0, 4093, 5, 0, 0, 0, 0, 0, 0, 3015, 0, 3382, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 28, 0, 0, 5, 0, 13, 0, 4092, 0, 0, 4092, 0, 1530, 4096, 0, 4086, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4091, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[125, 0, 0, 4095, 0, 0, 4, 0, 0, 402, 4095, 0, 0, 0, 4095, 0, 4094, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 4047, 0, 0, 0, 0, 1, 0, 4095, 0, 957, 4095, 0, 0, 4096, 4096, 0, 4096, 1, 0, 0, 4, 1, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4082, 4096, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 1462, 0, 2235, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4096, 2, 1, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 1, 0, 10, 0, 4095, 0, 0, 4095, 0, 3059, 4096, 0, 4091, 4096, 0, 0, 4096, 0, 0, 4096, 4095, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[663, 0, 0, 4095, 0, 0, 0, 0, 0, 1088, 4096, 0, 0, 0, 4096, 0, 1946, 926, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 4036, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 200, 4096, 0, 0, 4096, 3816, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4085, 4096, 0, 4096, 65, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1638, 0, 2075, 4096, 0, 0, 0, 0, 3161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 8, 4079, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2580, 2147, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17105, 16407, 16339, 16455, 15066, 16386, 16320, 16994] avg=16384.0000 max=17105 max_rank=0 min=15066 min_rank=4 imbalance_ratio=1.044006 max_load_reduction=113967 max_load_reduction_pct=0.869499 moved_to_echo=64766 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1684, 7, 1, 4003, 1, 9, 29, 0, 9, 503, 4054, 9, 9, 0, 4085, 5, 1927, 924, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3922, 11, 0, 0, 0, 4088, 4087, 0, 0, 0, 4085, 11, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 335, 4087, 6, 10, 3952, 3807, 118, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 8, 0, 9, 0, 4085, 0, 8, 10, 0, 0, 8, 3739, 4086, 12, 4082, 65, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 9, 0, 8, 10, 0, 0, 3368, 0, 1142, 3924, 10, 9, 0, 8, 3156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 6, 4087, 0, 5, 7, 9, 3, 3, 10, 6, 0, 0, 4084, 0, 4, 4071, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 10, 10, 4086, 0, 118, 4, 0, 3, 0, 9, 0, 0, 4086, 8, 7, 4087, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1954, 2127, 9, 4080, 0, 0, 0, 4005, 0, 66, 0, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17454, 16289, 16409, 16141, 15731, 16388, 16457, 16203] avg=16384.0000 max=17454 max_rank=0 min=15731 min_rank=4 imbalance_ratio=1.065308 max_load_reduction=113618 max_load_reduction_pct=0.866837 moved_to_echo=64779 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[181, 0, 0, 4096, 0, 0, 15, 2, 0, 83, 4091, 0, 0, 0, 4096, 0, 1947, 925, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4066, 4093, 0, 0, 0, 4058, 0, 0, 0, 0, 5, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 7, 6, 4093, 0, 0, 4096, 3813, 0, 0, 4, 0, 0, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 3, 0, 0, 4079, 4096, 0, 4096, 65, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 0, 0, 0, 0, 0, 0, 3015, 0, 3382, 4096, 0, 0, 0, 0, 3160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 6, 0, 4074, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 28, 0, 0, 5, 0, 13, 0, 4092, 0, 0, 4096, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1530, 2149, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15631, 16403, 16134, 16456, 17751, 16379, 16361, 15957] avg=16384.0000 max=17751 max_rank=4 min=15631 min_rank=0 imbalance_ratio=1.083435 max_load_reduction=113321 max_load_reduction_pct=0.864571 moved_to_echo=66448 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[125, 0, 0, 4095, 0, 0, 4, 0, 0, 402, 4095, 0, 0, 0, 4095, 0, 1947, 926, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 4047, 0, 0, 0, 0, 1, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 957, 4095, 0, 0, 4096, 3816, 0, 0, 1, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4082, 4096, 0, 4096, 65, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 1462, 0, 2235, 4096, 0, 0, 0, 0, 3161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4096, 2, 1, 4078, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 1, 0, 10, 0, 4095, 0, 0, 4096, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3059, 2149, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15884, 16416, 17065, 16454, 15050, 16379, 16334, 17490] avg=16384.0000 max=17490 max_rank=7 min=15050 min_rank=4 imbalance_ratio=1.067505 max_load_reduction=113582 max_load_reduction_pct=0.866562 moved_to_echo=65253 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[354, 0, 0, 4096, 0, 0, 9, 0, 0, 723, 4096, 0, 0, 0, 4096, 0, 1947, 926, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4082, 4090, 0, 0, 0, 4012, 0, 0, 0, 0, 1, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 179, 4096, 0, 0, 4096, 3816, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4081, 4096, 0, 4096, 65, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1247, 0, 3427, 4096, 0, 0, 0, 0, 3161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 8, 4079, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 65, 0, 0, 0, 0, 37, 0, 4096, 0, 0, 4096, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2286, 2149, 0, 4045, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16442, 16366, 16309, 16451, 16027, 16385, 16421, 16671] avg=16384.0000 max=16671 max_rank=7 min=16027 min_rank=4 imbalance_ratio=1.017517 max_load_reduction=114401 max_load_reduction_pct=0.872810 moved_to_echo=65504 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[49, 0, 0, 4096, 0, 0, 15, 0, 0, 18, 4092, 0, 0, 0, 4095, 0, 1951, 927, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4072, 4096, 0, 0, 0, 4073, 0, 0, 0, 0, 7, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 3, 35, 4096, 0, 0, 4096, 3809, 0, 0, 1, 0, 0, 39, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4, 0, 0, 4064, 4096, 0, 4096, 66, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 14, 0, 0, 0, 0, 0, 0, 2962, 0, 3800, 4096, 0, 0, 0, 0, 3160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 2, 6, 4075, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 37, 0, 0, 2, 0, 10, 0, 4094, 0, 0, 4096, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1331, 2145, 0, 4049, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15442, 16432, 16168, 16442, 18126, 16380, 16365, 15717] avg=16384.0000 max=18126 max_rank=4 min=15442 min_rank=0 imbalance_ratio=1.106323 max_load_reduction=112946 max_load_reduction_pct=0.861710 moved_to_echo=66588 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[649, 0, 0, 4096, 0, 0, 18, 0, 0, 51, 4096, 0, 0, 0, 4096, 0, 1947, 924, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4051, 4095, 0, 0, 0, 3973, 0, 0, 0, 0, 6, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 26, 4096, 0, 0, 4096, 3816, 0, 0, 0, 0, 0, 63, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 1, 0, 0, 4075, 4096, 0, 4096, 65, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 2423, 0, 2889, 4096, 0, 0, 0, 0, 3154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 4076, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 51, 0, 0, 5, 0, 69, 0, 4094, 0, 0, 4095, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2183, 2149, 0, 4051, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16072, 16306, 16191, 16450, 16665, 16376, 16441, 16571] avg=16384.0000 max=16665 max_rank=4 min=16072 min_rank=0 imbalance_ratio=1.017151 max_load_reduction=114407 max_load_reduction_pct=0.872856 moved_to_echo=66053 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[558, 1, 0, 4085, 0, 3, 19, 0, 3, 1141, 4066, 3, 3, 0, 4092, 1, 1945, 927, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 3, 0, 0, 4093, 4093, 0, 2, 0, 4093, 3, 11, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 373, 4093, 0, 3, 4064, 3813, 13, 0, 0, 0, 2, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 3, 0, 4092, 0, 3, 1, 0, 2, 3, 3840, 4093, 4, 4092, 66, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 3, 0, 3, 3, 0, 0, 2394, 0, 2909, 4074, 3, 3, 2, 6, 3159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 4093, 0, 3, 0, 3, 1, 0, 3, 3, 0, 0, 4093, 0, 22, 4075, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 4093, 0, 5, 36, 0, 8, 0, 3, 0, 1, 4093, 3, 2, 4093, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 2147, 3, 4093, 0, 0, 0, 4076, 0, 11, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17042, 16453, 16457, 16223, 16656, 16399, 16373, 15469] avg=16384.0000 max=17042 max_rank=0 min=15469 min_rank=7 imbalance_ratio=1.040161 max_load_reduction=114030 max_load_reduction_pct=0.869980 moved_to_echo=64897 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 32768, 0, 128, 256, 0, 128, 128, 128, 256, 128, 32768, 128, 128, 32768, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 0, 18560, 0, 21888, 32640, 128, 128, 128, 128, 25344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 32768, 128, 128, 128, 128, 128, 128, 128, 128, 128, 0, 32768, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 128, 2176, 32768, 128, 128, 32640, 30592, 256, 0, 128, 0, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[4352, 128, 128, 32768, 128, 128, 128, 128, 128, 4096, 32768, 128, 128, 0, 32768, 128, 15616, 7424, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 16000, 17280, 128, 32640, 0, 0, 0, 32768, 0, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32640, 128, 128, 0, 0, 32640, 32768, 0, 128, 0, 32384, 128, 128, 0, 0, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 0, 128, 128, 128, 128, 128, 32128, 32768, 128, 32768, 640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131584 slack_slots=8448 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 2088, 4096, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 58, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 4096, 4043, 0, 4096, 0, 0, 2261, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 2067, 29, 4096, 4026, 4096, 0, 0, 0, 3959, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2849, 3032, 0, 59, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 107, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 4096, 0, 4096, 1, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 16, 0, 0, 1822, 4093, 0, 0, 0, 0, 4, 4096, 2, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4068, 0, 0, 4092, 0, 4096, 4, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 456, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4091, 4078, 0, 4093, 0, 3, 2939, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4085, 0, 0, 1720, 11, 4096, 4090, 4095, 33, 0, 0, 4001, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3289, 1782, 0, 24, 3, 0, 0, 0, 3, 0, 1, 4093, 0, 334, 0, 3, 4, 4092, 0, 1, 0, 4096, 0, 4094, 0, 4092, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3511, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4075, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 117, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4095, 4094, 0, 4096, 0, 0, 3205, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4081, 0, 0, 998, 22, 4096, 4061, 4096, 2, 0, 0, 3572, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3617, 1206, 0, 5, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 185, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 2313, 4095, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4068, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 557, 0, 0, 0, 5, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4093, 4092, 0, 4096, 0, 0, 2877, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 4087, 0, 0, 2442, 18, 4096, 4028, 4091, 63, 0, 0, 3452, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2113, 2334, 0, 14, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 263, 0, 3, 5, 4092, 0, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 1302, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4050, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4078, 0, 0, 4096, 0, 0, 631, 0, 0, 0, 9, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4086, 4091, 0, 4095, 0, 2, 3066, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 4085, 0, 0, 1920, 63, 4096, 4063, 4094, 29, 0, 0, 3787, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2863, 2160, 0, 81, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 539, 0, 2, 4, 4094, 0, 0, 0, 4096, 0, 4096, 1, 4094, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 1351, 4085, 10, 0, 8, 0, 0, 4068, 8, 4086, 4, 9, 4086, 4077, 9, 41, 9, 0, 0, 4091, 4, 9, 3793, 9, 4087, 0, 0, 2, 16, 4081, 16, 10, 2278, 12, 10, 1825, 9, 0, 27, 0, 4080, 0, 2, 12, 0, 0, 368, 0, 0, 60, 0, 0, 4088, 4037, 9, 4087, 178, 0, 850, 0, 2, 0, 9, 4083, 0, 1, 0, 8, 0, 4062, 9, 0, 3294, 0, 3342, 3809, 3752, 0, 0, 0, 3881, 0, 9, 10, 7, 4086, 4, 0, 4040, 10, 3, 10, 4087, 4052, 3212, 0, 8, 0, 77, 0, 9, 0, 0, 0, 4082, 5, 0, 7, 0, 0, 4087, 30, 0, 10, 4086, 592, 4086, 0, 4085, 141, 0, 10, 0, 4065] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 1593, 4091, 3, 0, 3, 0, 0, 4092, 11, 4093, 7, 3, 4093, 4091, 3, 14, 3, 0, 0, 4094, 2, 3, 3989, 3, 4093, 0, 0, 14, 3, 4093, 4, 3, 3032, 4, 3, 1311, 3, 1, 0, 1, 4093, 0, 2, 3, 0, 0, 161, 0, 0, 14, 0, 0, 4093, 4072, 3, 4093, 3, 0, 1675, 0, 0, 0, 3, 4092, 1, 0, 0, 2, 0, 4082, 3, 0, 2247, 0, 3746, 3934, 3858, 0, 1, 0, 3589, 0, 3, 3, 3, 4093, 1, 0, 4078, 3, 2, 3, 4093, 3906, 3275, 0, 112, 0, 5, 0, 3, 0, 0, 0, 4089, 4, 0, 2, 0, 0, 4093, 3, 0, 3, 4093, 290, 4093, 0, 4093, 104, 0, 3, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 2029, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 0, 888, 0, 0, 0, 1, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4095, 4096, 0, 4096, 0, 1, 2797, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4095, 0, 0, 2366, 13, 4096, 4092, 4095, 5, 0, 0, 3989, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2733, 1370, 0, 49, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 156, 0, 1, 1, 4095, 0, 0, 0, 4096, 2, 4096, 0, 4095, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 16, 0, 0, 1822, 4093, 0, 0, 0, 0, 4, 4096, 2, 4095, 0, 0, 1953, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4068, 0, 0, 4092, 0, 58, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 456, 0, 0, 0, 0, 4096, 8, 0, 0, 3714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4091, 4078, 0, 4093, 0, 3, 2939, 0, 1550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2173, 0, 0, 4, 0, 0, 4085, 0, 0, 1720, 11, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 33, 0, 0, 4001, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 236, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3289, 1782, 0, 24, 3, 0, 0, 0, 3, 0, 1, 4093, 0, 334, 0, 2085, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 4092, 0, 1, 0, 0, 0, 3955, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16218, 16414, 16456, 16774, 16179, 16696, 16092, 16243] avg=16384.0000 max=16774 max_rank=3 min=16092 min_rank=6 imbalance_ratio=1.023804 max_load_reduction=114298 max_load_reduction_pct=0.872025 moved_to_echo=65210 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 2088, 4096, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 1953, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 0, 0, 58, 0, 0, 0, 0, 4095, 0, 0, 0, 3714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 4096, 4043, 0, 4096, 0, 0, 2261, 0, 1549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2171, 0, 0, 0, 0, 0, 4096, 0, 0, 2067, 29, 4096, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3959, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 236, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2849, 3032, 0, 59, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 107, 0, 2085, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 3956, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16467, 16441, 16055, 16049, 16485, 16623, 16706, 16246] avg=16384.0000 max=16706 max_rank=6 min=16049 min_rank=3 imbalance_ratio=1.019653 max_load_reduction=114366 max_load_reduction_pct=0.872543 moved_to_echo=66060 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3511, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1953, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4075, 0, 0, 4096, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 117, 0, 0, 0, 0, 4096, 1, 0, 0, 3714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4095, 4094, 0, 4096, 0, 0, 3205, 0, 1550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2173, 0, 0, 1, 0, 0, 4081, 0, 0, 998, 22, 4096, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 3572, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 236, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3617, 1206, 0, 5, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 185, 0, 2085, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 3956, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17890, 16421, 16116, 17058, 15432, 16238, 15671, 16246] avg=16384.0000 max=17890 max_rank=0 min=15432 min_rank=4 imbalance_ratio=1.091919 max_load_reduction=113182 max_load_reduction_pct=0.863510 moved_to_echo=63587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 1351, 4085, 10, 0, 8, 0, 0, 4068, 8, 4086, 4, 9, 1948, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4077, 9, 41, 9, 0, 0, 4091, 4, 9, 3793, 9, 59, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4081, 16, 10, 2278, 12, 10, 1825, 9, 0, 27, 0, 4080, 0, 2, 12, 3706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 60, 0, 0, 4088, 4037, 9, 4087, 178, 0, 850, 0, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 9, 2167, 0, 1, 0, 8, 0, 4062, 9, 0, 3294, 0, 3342, 3809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3752, 0, 0, 0, 3881, 0, 9, 10, 7, 4086, 4, 0, 4040, 10, 3, 10, 235, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4052, 3212, 0, 8, 0, 77, 0, 9, 0, 0, 0, 4082, 5, 0, 7, 2080, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 30, 0, 10, 0, 592, 3947, 0, 4085, 141, 0, 10, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15722, 16189, 16084, 15222, 16703, 16186, 17999, 16967] avg=16384.0000 max=17999 max_rank=6 min=15222 min_rank=3 imbalance_ratio=1.098572 max_load_reduction=113073 max_load_reduction_pct=0.862679 moved_to_echo=67855 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 2029, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1953, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 0, 888, 0, 0, 0, 1, 4096, 1, 0, 0, 3714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4095, 4096, 0, 4096, 0, 1, 2797, 0, 1550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2173, 0, 0, 1, 0, 0, 4095, 0, 0, 2366, 13, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 0, 3989, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 236, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2733, 1370, 0, 49, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 156, 0, 2085, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 0, 2, 3956, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16435, 16883, 16641, 16836, 16657, 14967, 16246] avg=16384.0000 max=16883 max_rank=2 min=14967 min_rank=6 imbalance_ratio=1.030457 max_load_reduction=114189 max_load_reduction_pct=0.871193 moved_to_echo=64706 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 1302, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1956, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4050, 0, 0, 4096, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 4096, 0, 0, 631, 0, 0, 0, 9, 4096, 15, 0, 0, 3715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4086, 4091, 0, 4095, 0, 2, 3066, 0, 1555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2161, 0, 0, 9, 0, 0, 4085, 0, 0, 1920, 63, 4096, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 29, 0, 0, 3787, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 241, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2863, 2160, 0, 81, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 539, 0, 2088, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 4094, 0, 0, 0, 0, 0, 3953, 1, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15689, 16390, 16640, 16922, 16397, 16486, 16304, 16244] avg=16384.0000 max=16922 max_rank=3 min=15689 min_rank=0 imbalance_ratio=1.032837 max_load_reduction=114150 max_load_reduction_pct=0.870895 moved_to_echo=65431 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 2313, 4095, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 1953, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4068, 0, 0, 4096, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 557, 0, 0, 0, 5, 4096, 12, 0, 0, 3714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4093, 4092, 0, 4096, 0, 0, 2877, 0, 1550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2173, 0, 0, 8, 0, 0, 4087, 0, 0, 2442, 18, 4096, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 63, 0, 0, 3452, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 236, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2113, 2334, 0, 14, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 263, 0, 2085, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 4092, 0, 0, 0, 0, 0, 3956, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16692, 16414, 16570, 16743, 16852, 16174, 15380, 16247] avg=16384.0000 max=16852 max_rank=4 min=15380 min_rank=6 imbalance_ratio=1.028564 max_load_reduction=114220 max_load_reduction_pct=0.871429 moved_to_echo=64653 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 1593, 4091, 3, 0, 3, 0, 0, 4092, 11, 4093, 7, 3, 1951, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4091, 3, 14, 3, 0, 0, 4094, 2, 3, 3989, 3, 59, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 4, 3, 3032, 4, 3, 1311, 3, 1, 0, 1, 4093, 0, 2, 3, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 14, 0, 0, 4093, 4072, 3, 4093, 3, 0, 1675, 0, 1549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2170, 1, 0, 0, 2, 0, 4082, 3, 0, 2247, 0, 3746, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3858, 0, 1, 0, 3589, 0, 3, 3, 3, 4093, 1, 0, 4078, 3, 2, 3, 236, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3906, 3275, 0, 112, 0, 5, 0, 3, 0, 0, 0, 4089, 4, 0, 2, 2083, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 3, 0, 290, 3954, 0, 4093, 104, 0, 3, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15987, 16368, 16268, 15663, 16188, 16012, 17953, 16633] avg=16384.0000 max=17953 max_rank=6 min=15663 min_rank=3 imbalance_ratio=1.095764 max_load_reduction=113119 max_load_reduction_pct=0.863029 moved_to_echo=66786 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 32768, 128, 128, 29952, 128, 128, 5888, 128, 128, 128, 128, 32768, 128, 128, 128, 29824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 128, 128, 0, 0, 32640, 128, 128, 32384, 128, 512, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 0, 896, 31744, 128, 32768, 256, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 0, 16128, 32768, 128, 0, 128, 0, 128, 32768, 128, 32768, 128, 128, 15744, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 25472, 18432, 0, 384, 128, 128, 0, 128, 128, 0, 128, 32768, 128, 1664, 128, 16768, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 17408, 128, 128, 128, 128, 0, 32768, 128, 0, 17152, 256, 31744, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32256, 256, 128, 0, 30336, 0, 128, 128, 128, 32768, 128, 0, 32768, 128, 128, 128, 1920, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 640, 0, 0, 128, 0, 128, 32768, 32640, 128, 32768, 256, 128, 19712, 0, 12416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1833, 0, 0, 4092, 0, 0, 4082, 4096, 0, 0, 0, 3187, 0, 0, 0, 4096, 4, 4096, 0, 217, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 4, 0, 0, 4096, 3982, 0, 4081, 4094, 2298, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 14, 484, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4091, 6, 3, 0, 0, 0, 0, 217, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 4091, 0, 0, 68, 0, 4096, 0, 4096, 4096, 0, 3, 41, 0, 0, 4096, 0, 2, 5, 0, 0, 4096, 0, 0, 0, 4095, 0, 4058, 0, 4090, 4093, 4096, 0, 0, 0, 0, 4075, 4096, 13, 0, 4087, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[372, 0, 0, 4096, 0, 0, 4096, 4045, 0, 0, 0, 3404, 0, 0, 0, 4096, 0, 4096, 0, 378, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3585, 0, 4088, 4096, 2824, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 1, 1302, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4085, 1, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 0, 0, 369, 0, 4096, 0, 4096, 4096, 0, 0, 53, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 4095, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1207, 0, 0, 4094, 0, 0, 4069, 4078, 0, 0, 0, 3686, 0, 0, 0, 4096, 0, 4096, 0, 429, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 3401, 0, 4056, 4096, 2523, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 4096, 0, 5, 784, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 144, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 4093, 0, 0, 189, 0, 4096, 0, 4096, 4096, 0, 0, 15, 0, 0, 4096, 0, 4, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 4047, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4050, 4096, 30, 0, 4093, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[3192, 0, 0, 4096, 0, 0, 4081, 4094, 0, 0, 0, 1751, 0, 0, 0, 4096, 0, 4096, 0, 219, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3695, 0, 4072, 4096, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 4096, 0, 2, 2236, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3923, 0, 0, 0, 0, 0, 0, 289, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4095, 0, 0, 9, 0, 4096, 0, 4096, 4096, 0, 0, 98, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4044, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4066, 4095, 15, 0, 4090, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[2903, 3, 2, 4084, 6, 0, 4088, 2139, 10, 0, 8, 25, 0, 0, 9, 4083, 2, 2435, 25, 2459, 0, 4082, 0, 0, 5, 10, 4086, 0, 479, 0, 0, 0, 4077, 3701, 11, 4087, 4086, 3850, 1, 14, 4, 9, 0, 9, 252, 0, 0, 10, 10, 0, 9, 0, 544, 9, 8, 14, 4087, 10, 0, 237, 0, 0, 10, 4086, 9, 0, 0, 0, 0, 4014, 1342, 11, 9, 0, 0, 14, 35, 0, 0, 9, 9, 0, 10, 4084, 4087, 2, 4086, 0, 0, 0, 0, 3663, 8, 4058, 4081, 14, 0, 790, 10, 10, 3785, 12, 0, 0, 9, 6, 4073, 8, 0, 69, 4079, 0, 4088, 108, 4086, 4086, 4074, 0, 3, 0, 11, 4087, 3849, 0, 10, 4087, 100, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[646, 0, 0, 4093, 0, 0, 4076, 4000, 0, 0, 0, 3766, 0, 0, 0, 4096, 4, 4096, 0, 495, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 4096, 3752, 0, 4046, 4094, 2709, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 4096, 0, 3, 757, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 148, 0, 1, 0, 0, 0, 0, 4096, 4084, 0, 4093, 0, 0, 153, 0, 4096, 0, 4096, 4096, 0, 0, 71, 0, 0, 4096, 0, 1, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 4025, 0, 4096, 4094, 4096, 0, 0, 0, 0, 4064, 4096, 16, 0, 4080, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[2910, 1, 0, 4092, 1, 1, 4093, 2315, 3, 2, 3, 151, 0, 0, 3, 4087, 36, 3203, 14, 1892, 0, 4090, 0, 1, 2, 3, 4093, 0, 245, 0, 0, 0, 4088, 3965, 6, 4093, 4093, 3708, 1, 1, 1, 3, 0, 3, 148, 0, 0, 3, 3, 0, 3, 0, 159, 3, 2, 5, 4093, 3, 0, 467, 0, 0, 3, 4093, 3, 0, 0, 0, 1, 3919, 1213, 25, 3, 0, 0, 4, 11, 0, 0, 3, 3, 0, 3, 4093, 4093, 2, 4091, 0, 0, 0, 0, 3855, 3, 4075, 4092, 2, 0, 704, 3, 3, 3899, 3, 0, 0, 3, 2, 4091, 3, 0, 9, 4089, 3, 4093, 26, 4093, 4093, 4091, 0, 0, 0, 3, 4093, 3981, 0, 3, 4093, 6, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1464, 0, 0, 4095, 0, 0, 4092, 4086, 0, 0, 0, 2122, 0, 0, 0, 4095, 0, 4096, 0, 235, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 4096, 3852, 0, 4085, 4095, 2730, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 4096, 0, 1, 1550, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4092, 3, 1, 0, 0, 0, 0, 135, 0, 1, 0, 0, 0, 0, 4096, 4095, 0, 4095, 0, 0, 61, 0, 4096, 0, 4096, 4096, 0, 0, 129, 0, 0, 4096, 0, 1, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4068, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4090, 4096, 5, 0, 4094, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1833, 0, 0, 4092, 0, 0, 4082, 4096, 0, 0, 0, 3187, 0, 0, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 217, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 4, 0, 0, 2390, 1122, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3982, 0, 4081, 1704, 2298, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 14, 484, 0, 0, 0, 4096, 4090, 2587, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4091, 6, 3, 0, 0, 0, 0, 217, 0, 0, 0, 4096, 4096, 3691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4078, 0, 0, 68, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 41, 0, 0, 4096, 0, 2, 5, 0, 0, 4096, 0, 0, 0, 4095, 0, 2456, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 57, 0, 0, 0, 0, 4075, 4096, 13, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17808, 16034, 16179, 15787, 16204, 16428, 16246, 16386] avg=16384.0000 max=17808 max_rank=0 min=15787 min_rank=3 imbalance_ratio=1.086914 max_load_reduction=113264 max_load_reduction_pct=0.864136 moved_to_echo=65264 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[372, 0, 0, 4096, 0, 0, 4096, 4045, 0, 0, 0, 3404, 0, 0, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 378, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2391, 1122, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3585, 0, 4088, 1705, 2824, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 1, 1302, 0, 0, 0, 4096, 4095, 2587, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 0, 0, 0, 0, 66, 0, 0, 0, 4096, 4096, 3695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 0, 0, 369, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 2456, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 57, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16531, 16193, 16300, 16581, 16039, 16739, 16250, 16439] avg=16384.0000 max=16739 max_rank=5 min=16039 min_rank=4 imbalance_ratio=1.021667 max_load_reduction=114333 max_load_reduction_pct=0.872292 moved_to_echo=65467 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2903, 3, 2, 4084, 6, 0, 4088, 2139, 10, 0, 8, 25, 0, 0, 9, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2435, 25, 2459, 0, 4082, 0, 0, 5, 10, 4086, 0, 479, 0, 0, 0, 2385, 1119, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3701, 11, 4087, 1701, 3850, 1, 14, 4, 9, 0, 9, 252, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 9, 0, 544, 9, 8, 14, 4087, 10, 0, 237, 0, 0, 10, 4086, 4086, 2573, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4014, 1342, 11, 9, 0, 0, 14, 35, 0, 0, 9, 4084, 4081, 3685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 10, 0, 4087, 2, 4073, 0, 0, 0, 0, 3663, 8, 4058, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 790, 10, 10, 3785, 12, 0, 0, 9, 6, 4073, 8, 0, 69, 4079, 0, 2448, 1444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 108, 0, 0, 57, 0, 3, 0, 11, 4087, 3849, 0, 10, 4087, 100, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13793, 17100, 17726, 16084, 17293, 15924, 16743, 16409] avg=16384.0000 max=17726 max_rank=2 min=13793 min_rank=0 imbalance_ratio=1.081909 max_load_reduction=113346 max_load_reduction_pct=0.864761 moved_to_echo=66369 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1464, 0, 0, 4095, 0, 0, 4092, 4086, 0, 0, 0, 2122, 0, 0, 0, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 235, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 2390, 1122, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3852, 0, 4085, 1705, 2730, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 4096, 0, 1, 1550, 0, 0, 0, 4096, 4096, 2587, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 3, 1, 0, 0, 0, 0, 135, 0, 1, 0, 4096, 4096, 3694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4082, 0, 0, 61, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 4096, 0, 1, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 2456, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 57, 0, 0, 0, 0, 4090, 4096, 5, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16376, 16048, 16469, 16893, 16119, 16430, 16327, 16410] avg=16384.0000 max=16893 max_rank=3 min=16048 min_rank=1 imbalance_ratio=1.031067 max_load_reduction=114179 max_load_reduction_pct=0.871117 moved_to_echo=65286 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3192, 0, 0, 4096, 0, 0, 4081, 4094, 0, 0, 0, 1751, 0, 0, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 219, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2391, 1122, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3695, 0, 4072, 1705, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 4096, 0, 2, 2236, 0, 0, 0, 4096, 4096, 2587, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 0, 0, 0, 0, 0, 289, 0, 0, 0, 4096, 4096, 3695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4082, 0, 0, 9, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 2456, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 57, 0, 0, 0, 0, 4066, 4095, 15, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17732, 16033, 14582, 17587, 16099, 16378, 16294, 16367] avg=16384.0000 max=17732 max_rank=0 min=14582 min_rank=2 imbalance_ratio=1.082275 max_load_reduction=113340 max_load_reduction_pct=0.864716 moved_to_echo=65138 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[646, 0, 0, 4093, 0, 0, 4076, 4000, 0, 0, 0, 3766, 0, 0, 0, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 495, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 2393, 1127, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3752, 0, 4046, 1701, 2709, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 4096, 0, 3, 757, 0, 0, 0, 4096, 4096, 2589, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 148, 0, 1, 0, 4096, 4096, 3692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4078, 0, 0, 153, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 4096, 0, 1, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 2461, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 0, 49, 0, 0, 0, 0, 4064, 4096, 16, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17089, 16321, 16309, 16103, 16131, 16507, 16282, 16330] avg=16384.0000 max=17089 max_rank=0 min=16103 min_rank=3 imbalance_ratio=1.043030 max_load_reduction=113983 max_load_reduction_pct=0.869621 moved_to_echo=65250 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1207, 0, 0, 4094, 0, 0, 4069, 4078, 0, 0, 0, 3686, 0, 0, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 429, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 2391, 1122, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3401, 0, 4056, 1705, 2523, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 4096, 0, 5, 784, 0, 0, 0, 4096, 4096, 2587, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 144, 0, 0, 0, 4096, 4096, 3695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4080, 0, 0, 189, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4096, 0, 4, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 2456, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 57, 0, 0, 0, 0, 4050, 4096, 30, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17652, 16242, 15783, 16124, 16129, 16552, 16217, 16373] avg=16384.0000 max=17652 max_rank=0 min=15783 min_rank=2 imbalance_ratio=1.077393 max_load_reduction=113420 max_load_reduction_pct=0.865326 moved_to_echo=65271 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2910, 1, 0, 4092, 1, 1, 4093, 2315, 3, 2, 3, 151, 0, 0, 3, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 3203, 14, 1892, 0, 4090, 0, 1, 2, 3, 4093, 0, 245, 0, 0, 0, 2389, 1120, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3965, 6, 4093, 1704, 3708, 1, 1, 1, 3, 0, 3, 148, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 159, 3, 2, 5, 4093, 3, 0, 467, 0, 0, 3, 4093, 4093, 2584, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 3919, 1213, 25, 3, 0, 0, 4, 11, 0, 0, 3, 4093, 4092, 3691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 4093, 2, 4078, 0, 0, 0, 0, 3855, 3, 4075, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 704, 3, 3, 3899, 3, 0, 0, 3, 2, 4091, 3, 0, 9, 4089, 3, 2451, 1450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 26, 0, 0, 57, 0, 0, 0, 3, 4093, 3981, 0, 3, 4093, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14091, 17101, 17724, 15913, 17058, 16114, 16713, 16358] avg=16384.0000 max=17724 max_rank=2 min=14091 min_rank=0 imbalance_ratio=1.081787 max_load_reduction=113348 max_load_reduction_pct=0.864777 moved_to_echo=66243 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 0, 128, 32384, 2688, 128, 128, 0, 0, 128, 1152, 0, 128, 128, 32768, 32768, 29568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 30336, 128, 6400, 0, 32768, 128, 128, 128, 128, 32768, 0, 768, 128, 0, 0, 19200, 9088, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 1920, 128, 128, 32384, 128, 128, 128, 128, 128, 32768, 128, 0, 128, 32768, 128, 19712, 11648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 32640, 0, 0, 896, 0, 32128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32640, 256, 0, 0, 512, 0, 128, 0, 128, 32640, 32512, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 0, 1024, 128, 128, 128, 32768, 128, 128, 7936, 0, 0, 128, 32768, 32768, 20736, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[14592, 128, 128, 32768, 128, 128, 32768, 28928, 128, 128, 128, 18176, 0, 0, 128, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 29952, 128, 32640, 13696, 21760, 128, 128, 128, 128, 0, 128, 512, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 4096, 4079, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 548, 0, 4096, 0, 0, 0, 0, 10, 0, 0, 4096, 0, 2334, 0, 4096, 870, 0, 0, 0, 4096, 0, 0, 4096, 62, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 977, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4095, 4096, 3406, 0, 0, 0, 28, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4085, 0, 0, 0, 0, 0, 4096, 4096, 4082, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4048, 4096, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 759, 0, 4068, 0, 0, 0, 0, 29, 42, 0, 4096, 0, 1679, 0, 4096, 1317, 0, 0, 0, 4096, 0, 0, 4096, 15, 0, 4096, 0, 16, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1977, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 4094, 0, 0, 0, 0, 4095, 4061, 4096, 2518, 0, 0, 0, 70, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 4096, 0, 4082, 0, 0, 0, 0, 0, 4060, 4096, 4039, 4094, 0, 0, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4076, 4094, 0, 0, 0, 4096, 0, 7, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 574, 0, 4064, 0, 0, 0, 0, 9, 16, 0, 4096, 0, 1209, 0, 4096, 2004, 0, 0, 0, 4093, 0, 0, 4096, 23, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 893, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 4096, 0, 0, 0, 0, 4092, 4089, 4096, 3681, 0, 0, 0, 7, 4, 0, 4096, 2, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 4096, 0, 4085, 0, 0, 1, 0, 0, 4085, 4096, 3953, 4092, 0, 0, 4, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[3, 0, 0, 28, 684, 4, 0, 1, 1, 1, 0, 44, 3852, 3, 1, 0, 4054, 0, 0, 4093, 4093, 4093, 0, 0, 2, 4093, 3, 0, 0, 1, 6, 4093, 3, 222, 2, 3, 0, 0, 44, 0, 4093, 0, 4, 3, 2, 0, 0, 65, 4093, 3, 1118, 3, 4093, 3110, 3, 0, 0, 4093, 3, 3, 4084, 263, 0, 3767, 0, 13, 0, 3, 3, 0, 4, 0, 4051, 0, 3, 402, 0, 2, 2, 3, 3, 0, 4086, 3, 4093, 0, 4090, 1, 13, 2, 0, 4092, 4092, 4093, 2094, 402, 0, 0, 398, 0, 0, 4093, 0, 3, 0, 0, 4076, 3, 4091, 3, 3, 3, 4092, 3, 4066, 5, 0, 0, 0, 0, 4093, 4093, 4093, 4087, 3, 3, 0, 4081] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3, 0, 4096, 4026, 4090, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 843, 0, 4080, 0, 0, 0, 0, 40, 28, 0, 4096, 0, 1705, 0, 4096, 562, 0, 0, 0, 4096, 0, 0, 4096, 50, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2648, 0, 0, 0, 0, 0, 0, 4096, 0, 4058, 0, 4091, 0, 0, 0, 0, 4094, 4090, 4096, 2409, 0, 0, 0, 159, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 4071, 4096, 4014, 4094, 0, 0, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4089, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1685, 0, 4071, 0, 0, 0, 0, 175, 61, 0, 4096, 0, 2616, 0, 4096, 2181, 0, 0, 0, 4096, 0, 0, 4096, 93, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 509, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 4091, 0, 0, 0, 0, 4096, 4095, 4096, 952, 1, 0, 0, 16, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 0, 0, 4073, 4096, 4071, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[10, 9, 0, 78, 1306, 20, 0, 10, 6, 5, 2, 117, 3426, 10, 10, 0, 3927, 0, 0, 4086, 4086, 4086, 0, 0, 7, 4085, 8, 0, 0, 1, 11, 4086, 10, 531, 5, 11, 0, 0, 4, 0, 4087, 0, 12, 10, 1, 0, 0, 225, 4086, 10, 874, 9, 4085, 3371, 10, 0, 0, 4086, 10, 9, 4058, 210, 0, 3233, 0, 12, 0, 10, 9, 0, 11, 0, 4006, 0, 2, 51, 0, 12, 4, 9, 9, 0, 4077, 10, 4086, 0, 4035, 11, 9, 7, 0, 4084, 4082, 4086, 1637, 830, 0, 0, 852, 0, 0, 4086, 0, 9, 0, 0, 4001, 10, 4085, 6, 9, 9, 4084, 10, 4040, 14, 0, 0, 0, 0, 4087, 4084, 4087, 4082, 11, 8, 0, 4070] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4095, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1684, 0, 4093, 0, 0, 0, 0, 8, 3, 0, 4096, 0, 1835, 0, 4096, 1764, 0, 0, 0, 4096, 0, 0, 4096, 34, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 841, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 4096, 0, 0, 0, 0, 4095, 4091, 4096, 2101, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 4092, 4096, 4043, 4095, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4076, 4094, 0, 0, 0, 4096, 0, 7, 6, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 0, 4064, 0, 0, 0, 0, 9, 16, 0, 4096, 4096, 2980, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1209, 0, 584, 2004, 0, 0, 0, 4093, 0, 0, 4096, 23, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 893, 0, 0, 0, 0, 4051, 4014, 2075, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1613, 0, 4089, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 3681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 4096, 2, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 2483, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 1, 0, 0, 4085, 0, 3953, 78, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16209, 16420, 16010, 16105, 16243, 17568, 16215, 16302] avg=16384.0000 max=17568 max_rank=5 min=16010 min_rank=2 imbalance_ratio=1.072266 max_load_reduction=113504 max_load_reduction_pct=0.865967 moved_to_echo=66328 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4079, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 0, 4096, 0, 0, 0, 0, 10, 0, 0, 4096, 4096, 2982, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2334, 0, 584, 870, 0, 0, 0, 4096, 0, 0, 4096, 62, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 977, 0, 0, 0, 0, 4050, 4018, 2075, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1613, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 3406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2483, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 4096, 0, 4082, 78, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16209, 16413, 16004, 16138, 16331, 17306, 16234, 16437] avg=16384.0000 max=17306 max_rank=5 min=16004 min_rank=2 imbalance_ratio=1.056274 max_load_reduction=113766 max_load_reduction_pct=0.867966 moved_to_echo=66308 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1684, 0, 4093, 0, 0, 0, 0, 8, 3, 0, 4096, 4096, 2981, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1835, 0, 584, 1764, 0, 0, 0, 4096, 0, 0, 4096, 34, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 841, 0, 0, 0, 0, 4051, 4017, 2075, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1613, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 4091, 0, 2101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 2483, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4092, 0, 4043, 78, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16209, 16429, 17136, 16505, 16195, 15994, 16206, 16398] avg=16384.0000 max=17136 max_rank=2 min=15994 min_rank=5 imbalance_ratio=1.045898 max_load_reduction=113936 max_load_reduction_pct=0.869263 moved_to_echo=64793 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 28, 684, 4, 0, 1, 1, 1, 0, 44, 3852, 3, 1, 0, 4093, 4093, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 4093, 4093, 0, 0, 2, 4093, 3, 0, 0, 1, 6, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 222, 2, 3, 0, 0, 44, 0, 4093, 0, 4, 3, 2, 0, 0, 65, 4093, 4092, 2980, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1118, 3, 584, 3110, 3, 0, 0, 4093, 3, 3, 4084, 263, 0, 3767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 3, 3, 0, 4, 0, 4051, 0, 3, 402, 0, 2, 2, 3, 4048, 4009, 2073, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1609, 3, 4093, 0, 4090, 1, 13, 2, 0, 0, 4092, 0, 2094, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 0, 4093, 0, 3, 0, 0, 4076, 3, 4091, 3, 3, 3, 2477, 1436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4066, 5, 0, 0, 0, 0, 4093, 0, 4093, 78, 3, 3, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16726, 16390, 15781, 17034, 15728, 16402, 16586, 16425] avg=16384.0000 max=17034 max_rank=3 min=15728 min_rank=4 imbalance_ratio=1.039673 max_load_reduction=114038 max_load_reduction_pct=0.870041 moved_to_echo=65141 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4048, 4096, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 759, 0, 4068, 0, 0, 0, 0, 29, 42, 0, 4096, 4096, 2981, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1679, 0, 584, 1317, 0, 0, 0, 4096, 0, 0, 4096, 15, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1977, 0, 0, 0, 0, 4051, 4016, 2075, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1613, 0, 4090, 0, 4094, 0, 0, 0, 0, 0, 4061, 0, 2518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 2483, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 4060, 0, 4039, 78, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16208, 16384, 16247, 15883, 17345, 16376, 16272, 16357] avg=16384.0000 max=17345 max_rank=4 min=15883 min_rank=3 imbalance_ratio=1.058655 max_load_reduction=113727 max_load_reduction_pct=0.867668 moved_to_echo=66350 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 4026, 4090, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 843, 0, 4080, 0, 0, 0, 0, 40, 28, 0, 4096, 4096, 2982, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1705, 0, 577, 562, 0, 0, 0, 4096, 0, 0, 4096, 50, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2648, 0, 0, 0, 0, 4055, 4022, 2077, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1610, 0, 4058, 0, 4091, 0, 0, 0, 0, 0, 4090, 0, 2409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 2486, 1442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4071, 0, 4014, 72, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16209, 16359, 16340, 15182, 18016, 16258, 16365, 16343] avg=16384.0000 max=18016 max_rank=4 min=15182 min_rank=3 imbalance_ratio=1.099609 max_load_reduction=113056 max_load_reduction_pct=0.862549 moved_to_echo=66982 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[10, 9, 0, 78, 1306, 20, 0, 10, 6, 5, 2, 117, 3426, 10, 10, 0, 4086, 4086, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 0, 0, 0, 4086, 4086, 0, 0, 7, 4085, 8, 0, 0, 1, 11, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 531, 5, 11, 0, 0, 4, 0, 4087, 0, 12, 10, 1, 0, 0, 225, 4084, 4084, 2974, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 874, 9, 583, 3371, 10, 0, 0, 4086, 10, 9, 4058, 210, 0, 3233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 10, 9, 0, 11, 0, 4006, 0, 2, 51, 0, 12, 4, 9, 4041, 4004, 2069, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1606, 10, 4086, 0, 4035, 11, 9, 7, 0, 0, 4082, 0, 1637, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 852, 0, 0, 4086, 0, 9, 0, 0, 4001, 10, 4085, 6, 9, 9, 2471, 1433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4040, 14, 0, 0, 0, 0, 4087, 0, 4087, 78, 11, 8, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17093, 16256, 16212, 16463, 15350, 16322, 16971, 16405] avg=16384.0000 max=17093 max_rank=0 min=15350 min_rank=4 imbalance_ratio=1.043274 max_load_reduction=113979 max_load_reduction_pct=0.869591 moved_to_echo=65048 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1685, 0, 4071, 0, 0, 0, 0, 175, 61, 0, 4096, 4096, 2983, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2616, 0, 584, 2181, 0, 0, 0, 4096, 0, 0, 4096, 93, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 509, 0, 0, 0, 0, 4051, 4018, 2075, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1613, 0, 4094, 0, 4091, 0, 0, 0, 0, 0, 4095, 0, 952, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 2483, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 4073, 0, 4071, 78, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16209, 16421, 17342, 17762, 15864, 14846, 16223, 16405] avg=16384.0000 max=17762 max_rank=3 min=14846 min_rank=5 imbalance_ratio=1.084106 max_load_reduction=113310 max_load_reduction_pct=0.864487 moved_to_echo=63338 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32640, 128, 0, 0, 32640, 32768, 0, 0, 128, 32768, 128, 128, 128, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 1536, 128, 0, 32768, 128, 128, 0, 0, 32768, 128, 32768, 128, 128, 128, 19968, 11520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 2048, 128, 0, 128, 128, 128, 128, 256, 31872, 128, 128, 0, 32768, 32768, 31360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32640, 128, 0, 128, 0, 0, 32768, 0, 32384, 640, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 12928, 128, 32768, 0, 32768, 128, 128, 128, 0, 0, 32768, 0, 18816, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 768, 128, 128, 0, 0, 6144, 0, 32768, 0, 128, 128, 128, 384, 256, 384, 32768, 32768, 23936, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 0, 32640, 0, 128, 8320, 0, 128, 128, 128, 32512, 32128, 16640, 8960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 13440, 128, 4736, 15232, 128, 0, 0, 32768, 128, 128, 32768, 768, 0, 31616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 3337, 4096, 0, 4096, 4096, 4096, 0, 4096, 4094, 0, 4096, 0, 0, 0, 0, 470, 0, 0, 0, 4096, 0, 278, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 59, 0, 0, 0, 4096, 1755, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2810, 0, 3582, 4096, 0, 1, 0, 0, 0, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4096, 3548, 4093, 0, 4096, 4096, 4096, 0, 4091, 4070, 0, 4096, 0, 0, 0, 0, 212, 0, 0, 0, 4092, 0, 520, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 178, 0, 0, 0, 4096, 1230, 0, 2, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 3, 0, 0, 0, 4096, 4096, 46, 0, 0, 0, 0, 0, 2, 4096, 11, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 22, 0, 0, 2860, 0, 3654, 4096, 0, 0, 74, 0, 0, 0, 39, 4077, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 4095, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 0, 4096, 3590, 4090, 0, 4096, 4096, 4096, 0, 4096, 4080, 0, 4096, 0, 0, 0, 0, 200, 0, 0, 0, 4092, 0, 644, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 282, 0, 0, 0, 4096, 695, 0, 5, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 5, 0, 0, 0, 4096, 4096, 37, 0, 0, 0, 0, 0, 3, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 35, 0, 0, 3006, 0, 3789, 4096, 0, 0, 10, 0, 0, 0, 91, 4086, 0, 4, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4030, 4092, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4096, 3737, 4096, 0, 4096, 4096, 4096, 0, 4096, 4080, 0, 4096, 0, 0, 0, 0, 204, 0, 0, 0, 4096, 0, 2305, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 697, 0, 0, 0, 4096, 269, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 14, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 37, 0, 0, 2092, 0, 2986, 4096, 0, 0, 15, 0, 0, 0, 2, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 4096, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[9, 7, 4, 13, 4086, 8, 0, 3952, 3509, 4086, 0, 4080, 4076, 3938, 9, 4083, 4086, 10, 4059, 12, 0, 0, 0, 114, 0, 0, 64, 4070, 0, 147, 0, 0, 4079, 12, 0, 0, 4084, 12, 9, 0, 0, 360, 10, 10, 10, 3840, 3413, 2, 0, 3570, 14, 9, 10, 11, 4086, 0, 1, 0, 4085, 168, 10, 11, 6, 4029, 10, 4085, 0, 9, 0, 15, 10, 0, 3836, 2320, 0, 11, 7, 0, 0, 29, 0, 4085, 0, 0, 0, 0, 11, 3941, 0, 0, 10, 0, 16, 4081, 7, 4036, 0, 152, 8, 3956, 0, 3238, 3919, 47, 647, 164, 0, 10, 32, 788, 3006, 0, 0, 0, 0, 0, 0, 0, 336, 52, 11, 10, 10, 3714, 4087, 4081, 12, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3, 1, 2, 4, 4093, 3, 0, 4044, 3459, 4093, 0, 4092, 4091, 4040, 3, 4092, 4093, 3, 4082, 4, 1, 0, 0, 83, 0, 0, 15, 4083, 0, 263, 0, 16, 4090, 3, 0, 0, 4092, 3, 3, 0, 0, 161, 3, 3, 3, 3993, 2705, 0, 0, 3877, 4, 3, 3, 3, 4093, 0, 0, 0, 4092, 61, 3, 3, 0, 4079, 3, 4093, 0, 3, 0, 0, 3, 0, 4049, 3210, 0, 1, 2, 0, 0, 3, 0, 4093, 0, 1, 0, 0, 3, 4052, 0, 0, 3, 0, 4, 4092, 1, 4071, 0, 39, 3, 3668, 0, 3337, 4022, 0, 334, 94, 0, 3, 12, 243, 3696, 0, 0, 1, 1, 3, 0, 0, 176, 7, 3, 3, 3, 3605, 4093, 4092, 4, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4096, 3096, 4092, 0, 4096, 4096, 4096, 0, 4096, 4080, 0, 4096, 0, 0, 0, 0, 591, 0, 0, 0, 4094, 0, 666, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 375, 0, 0, 0, 4096, 1712, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 4, 0, 0, 0, 4096, 4096, 123, 0, 0, 0, 0, 0, 1, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 22, 0, 0, 1882, 0, 3935, 4096, 0, 0, 53, 0, 0, 0, 42, 4074, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3925, 4093, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 3322, 4095, 0, 4096, 4096, 4096, 0, 4096, 4083, 0, 4096, 0, 0, 0, 0, 648, 0, 0, 0, 4095, 0, 442, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 165, 0, 0, 0, 4096, 421, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 2, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 8, 0, 0, 3518, 0, 3790, 4096, 0, 0, 3, 0, 0, 0, 7, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 4096, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 695, 0, 0, 4096, 3590, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 4096, 0, 0, 0, 0, 200, 0, 0, 0, 4092, 0, 644, 0, 1, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 282, 0, 0, 0, 4096, 695, 0, 1779, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 5, 0, 0, 0, 4096, 4096, 37, 0, 0, 0, 0, 0, 4096, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 3006, 0, 3789, 4096, 0, 0, 10, 0, 0, 0, 91, 4086, 0, 1523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4030, 4092, 4096, 0, 0, 3762, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16567, 16247, 15591, 16452, 16757, 16421, 16636, 16401] avg=16384.0000 max=16757 max_rank=4 min=15591 min_rank=2 imbalance_ratio=1.022766 max_load_reduction=114315 max_load_reduction_pct=0.872154 moved_to_echo=66215 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 695, 0, 0, 4096, 3337, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 470, 0, 0, 0, 4096, 0, 278, 0, 0, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 59, 0, 0, 0, 4096, 1755, 0, 1782, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2810, 0, 3582, 4096, 0, 1, 0, 0, 0, 0, 9, 4096, 0, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 4096, 0, 0, 3763, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16320, 16168, 16429, 16450, 16717, 16412, 16119, 16457] avg=16384.0000 max=16717 max_rank=4 min=16119 min_rank=6 imbalance_ratio=1.020325 max_load_reduction=114355 max_load_reduction_pct=0.872459 moved_to_echo=65705 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 677, 0, 0, 4096, 3548, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 4096, 0, 0, 0, 0, 212, 0, 0, 0, 4092, 0, 520, 0, 0, 3136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 178, 0, 0, 0, 4096, 1230, 0, 1783, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3, 0, 0, 0, 4096, 4096, 46, 0, 0, 0, 0, 0, 4096, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 11, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 2860, 0, 3654, 4096, 0, 0, 74, 0, 0, 0, 39, 4077, 0, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 4095, 4096, 0, 0, 3757, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16510, 16126, 16024, 16457, 16766, 16429, 16350, 16410] avg=16384.0000 max=16766 max_rank=4 min=16024 min_rank=2 imbalance_ratio=1.023315 max_load_reduction=114306 max_load_reduction_pct=0.872086 moved_to_echo=65955 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[9, 7, 4, 13, 694, 8, 0, 3952, 3509, 4086, 0, 0, 0, 3938, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 10, 4059, 12, 0, 0, 0, 114, 0, 0, 64, 4070, 0, 147, 0, 0, 3120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 12, 0, 0, 4084, 12, 9, 0, 0, 360, 10, 10, 10, 3840, 3413, 2, 1777, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3570, 14, 9, 10, 11, 4086, 0, 1, 0, 4085, 168, 10, 11, 6, 4029, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4085, 0, 9, 0, 15, 10, 0, 3836, 2320, 0, 11, 7, 0, 0, 29, 4080, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 11, 3941, 0, 0, 10, 0, 16, 4081, 7, 4036, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 8, 3956, 0, 3238, 3919, 47, 647, 164, 0, 10, 32, 788, 3006, 0, 1521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 52, 11, 10, 10, 3714, 4087, 4081, 12, 0, 3750, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16229, 15682, 18161, 16076, 14745, 16215, 17488, 16476] avg=16384.0000 max=18161 max_rank=2 min=14745 min_rank=4 imbalance_ratio=1.108459 max_load_reduction=112911 max_load_reduction_pct=0.861443 moved_to_echo=64924 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 695, 0, 0, 4096, 3322, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 0, 0, 0, 0, 648, 0, 0, 0, 4095, 0, 442, 0, 0, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 165, 0, 0, 0, 4096, 421, 0, 1782, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 4096, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 3518, 0, 3790, 4096, 0, 0, 3, 0, 0, 0, 7, 4094, 0, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 4096, 4096, 0, 0, 3763, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16304, 16498, 15201, 16450, 16720, 16414, 17041, 16444] avg=16384.0000 max=17041 max_rank=6 min=15201 min_rank=2 imbalance_ratio=1.040100 max_load_reduction=114031 max_load_reduction_pct=0.869987 moved_to_echo=66619 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 695, 0, 0, 4096, 3737, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 4096, 0, 0, 0, 0, 204, 0, 0, 0, 4096, 0, 2305, 0, 0, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 697, 0, 0, 0, 4096, 269, 0, 1781, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 14, 0, 0, 0, 0, 0, 4096, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 2092, 0, 2986, 4096, 0, 0, 15, 0, 0, 0, 2, 4083, 0, 1524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 4096, 4096, 0, 0, 3762, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16720, 17915, 15582, 16450, 16732, 16414, 14835, 16424] avg=16384.0000 max=17915 max_rank=1 min=14835 min_rank=6 imbalance_ratio=1.093445 max_load_reduction=113157 max_load_reduction_pct=0.863319 moved_to_echo=64405 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3, 1, 2, 4, 695, 3, 0, 4044, 3459, 4093, 0, 0, 0, 4040, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 4082, 4, 1, 0, 0, 83, 0, 0, 15, 4083, 0, 263, 0, 16, 3132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 0, 0, 4092, 3, 3, 0, 0, 161, 3, 3, 3, 3993, 2705, 0, 1780, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 4, 3, 3, 3, 4093, 0, 0, 0, 4092, 61, 3, 3, 0, 4079, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 3, 0, 0, 3, 0, 4049, 3210, 0, 1, 2, 0, 0, 3, 4092, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 0, 3, 4052, 0, 0, 3, 0, 4, 4092, 1, 4071, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 3, 3668, 0, 3337, 4022, 0, 334, 94, 0, 3, 12, 243, 3696, 0, 1524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 0, 0, 176, 7, 3, 3, 3, 3605, 4093, 4092, 4, 0, 3758, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16347, 15775, 17383, 16287, 15793, 16348, 16975, 16164] avg=16384.0000 max=17383 max_rank=2 min=15775 min_rank=1 imbalance_ratio=1.060974 max_load_reduction=113689 max_load_reduction_pct=0.867378 moved_to_echo=65280 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 695, 0, 0, 4096, 3096, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 4096, 0, 0, 0, 0, 591, 0, 0, 0, 4094, 0, 666, 0, 0, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 375, 0, 0, 0, 4096, 1712, 0, 1781, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4, 0, 0, 0, 4096, 4096, 123, 0, 0, 0, 0, 0, 4096, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 1882, 0, 3935, 4096, 0, 0, 53, 0, 0, 0, 42, 4074, 0, 1524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3925, 4093, 4096, 0, 0, 3763, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16075, 16661, 16701, 16450, 16842, 16419, 15628, 16296] avg=16384.0000 max=16842 max_rank=4 min=15628 min_rank=6 imbalance_ratio=1.027954 max_load_reduction=114230 max_load_reduction_pct=0.871506 moved_to_echo=65185 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32128, 128, 128, 128, 128, 32768, 0, 128, 0, 32768, 256, 128, 128, 128, 32768, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 5632, 128, 0, 32640, 27648, 32768, 0, 0, 0, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 0, 0, 32768, 128, 128, 0, 0, 2304, 128, 128, 128, 32512, 12288, 128, 14336, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 32768, 128, 128, 0, 0, 2560, 0, 0, 128, 32768, 0, 5376, 0, 128, 25088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 0, 128, 128, 128, 128, 0, 32512, 30208, 256, 128, 128, 0, 0, 128, 32768, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 256, 128, 23808, 0, 28416, 32640, 128, 1024, 512, 0, 128, 128, 1280, 31232, 0, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 128, 32640, 0, 0, 128, 0, 128, 32768, 128, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 0, 0, 640, 128, 128, 128, 128, 31616, 32768, 32768, 128, 0, 30080, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 115, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 9, 0, 4096, 0, 4096, 0, 0, 0, 3786, 4096, 0, 0, 0, 0, 4090, 0, 4092, 0, 0, 1, 4096, 4096, 0, 7, 1, 0, 0, 0, 0, 0, 3978, 0, 0, 0, 1, 4031, 0, 0, 124, 4092, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 3, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2188, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4060, 0, 0, 248, 4096, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 3, 0, 0, 5, 0, 0, 0, 0, 1696, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4080, 0, 0, 30, 4048, 111, 0, 74, 0, 3931, 13, 0, 250, 10, 0, 10, 4061, 1, 0, 0, 4064, 29, 0, 10, 12, 4087, 4083, 0, 11, 4083, 3941, 3977, 0, 0, 20, 9, 12, 6, 0, 10, 11, 213, 41, 4, 0, 4085, 7, 0, 3656, 0, 0, 9, 4066, 11, 3953, 147, 0, 0, 523, 4086, 8, 11, 10, 0, 2247, 43, 3729, 308, 13, 39, 3916, 4034, 17, 0, 0, 4, 12, 0, 11, 10, 4087, 4, 2, 9, 115, 4088, 0, 0, 3574, 4079, 0, 4084, 0, 7, 3090, 9, 0, 4077, 9, 4085, 0, 3950, 4085, 0, 0, 136, 4086, 8, 1638, 0, 0, 264, 11, 10, 0, 11, 1221, 0, 0, 0, 10, 4085, 4016, 0, 20, 0, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 3, 0, 0, 4096, 0, 0, 22, 0, 44, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 9, 0, 4096, 0, 4096, 0, 0, 0, 2840, 4096, 0, 0, 0, 0, 4095, 0, 4093, 0, 0, 4, 4096, 4094, 0, 5, 2, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 4067, 0, 0, 717, 4092, 0, 4096, 0, 0, 4096, 0, 0, 4090, 0, 4096, 0, 4096, 4094, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 539, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 53, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 26, 0, 4096, 0, 4096, 0, 0, 0, 3717, 4096, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 4070, 0, 0, 10, 4093, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 64, 0, 0, 0, 0, 0, 0, 0, 312, 5, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4091, 0, 0, 7, 4082, 54, 0, 31, 0, 4025, 4, 0, 137, 3, 0, 3, 4086, 1, 0, 0, 4085, 4, 0, 3, 4, 4093, 4086, 0, 4, 4092, 4053, 4051, 0, 3, 6, 3, 3, 0, 0, 3, 4, 80, 20, 2, 0, 4092, 2, 0, 3918, 0, 0, 3, 4088, 3, 4023, 54, 0, 0, 1235, 4093, 3, 3, 3, 0, 3238, 9, 3935, 112, 3, 13, 4010, 4037, 5, 0, 0, 2, 3, 0, 2, 3, 4093, 3, 2, 3, 49, 4093, 0, 0, 2367, 4091, 0, 4092, 0, 2, 3601, 3, 0, 4090, 3, 4093, 0, 4043, 4092, 0, 0, 75, 4093, 3, 901, 0, 0, 86, 3, 3, 0, 3, 982, 3, 0, 0, 3, 4093, 4080, 0, 6, 2, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 56, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 26, 0, 4096, 0, 4096, 0, 0, 0, 2638, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 5, 4096, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 4030, 0, 0, 0, 0, 4070, 0, 0, 694, 4086, 0, 4096, 0, 0, 4096, 0, 0, 4091, 0, 4096, 0, 4096, 4094, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 774, 3, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 1, 0, 54, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 2993, 4096, 0, 0, 0, 0, 4065, 0, 4095, 0, 0, 1, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 1, 4084, 0, 0, 78, 4095, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 4096, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 3, 0, 0, 4096, 0, 0, 22, 0, 44, 0, 2422, 854, 723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 47, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 4096, 0, 4096, 0, 0, 0, 2840, 1674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4093, 0, 0, 4, 4096, 4094, 0, 5, 2, 0, 0, 0, 0, 0, 366, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 4067, 0, 0, 717, 3357, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3730, 0, 4096, 4094, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4092, 3195, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16356, 16431, 16334, 16811, 16767, 16285, 16020, 16068] avg=16384.0000 max=16811 max_rank=3 min=16020 min_rank=6 imbalance_ratio=1.026062 max_load_reduction=114261 max_load_reduction_pct=0.871742 moved_to_echo=65140 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 2422, 854, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 47, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2188, 1674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 366, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4060, 0, 0, 248, 3360, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3730, 0, 4096, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1696, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 3195, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16291, 16431, 16334, 16149, 16762, 15850, 16026, 17229] avg=16384.0000 max=17229 max_rank=7 min=15850 min_rank=5 imbalance_ratio=1.051575 max_load_reduction=113843 max_load_reduction_pct=0.868553 moved_to_echo=65867 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4080, 0, 0, 30, 4048, 111, 0, 74, 0, 3931, 13, 0, 250, 10, 0, 10, 2416, 851, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 1, 0, 0, 4064, 29, 0, 10, 12, 0, 48, 0, 11, 0, 3941, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 9, 12, 6, 0, 10, 11, 213, 41, 4, 0, 4085, 7, 0, 4086, 4084, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3656, 0, 0, 9, 4066, 11, 3953, 147, 0, 0, 523, 1670, 8, 11, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2247, 43, 3729, 308, 13, 39, 3916, 4034, 17, 0, 0, 4, 12, 0, 11, 10, 365, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4, 2, 9, 115, 4088, 0, 0, 3574, 3346, 0, 0, 0, 7, 3090, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 9, 3720, 0, 3950, 4085, 0, 0, 136, 0, 8, 1638, 0, 0, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 10, 0, 11, 1221, 0, 0, 0, 10, 4085, 4016, 0, 20, 0, 5, 0, 4087, 3184, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16545, 16154, 16621, 14064, 14760, 18331, 17887, 16710] avg=16384.0000 max=18331 max_rank=5 min=14064 min_rank=3 imbalance_ratio=1.118835 max_load_reduction=112741 max_load_reduction_pct=0.860146 moved_to_echo=67688 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 53, 0, 2422, 854, 723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 47, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 26, 0, 4096, 0, 4096, 0, 0, 0, 3717, 1674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 366, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 4070, 0, 0, 10, 3358, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3730, 0, 4096, 4096, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 5, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 3195, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16340, 16431, 16334, 17705, 16759, 15573, 16080, 15850] avg=16384.0000 max=17705 max_rank=3 min=15573 min_rank=5 imbalance_ratio=1.080627 max_load_reduction=113367 max_load_reduction_pct=0.864922 moved_to_echo=64262 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 115, 0, 2422, 854, 723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 47, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 4096, 0, 4096, 0, 0, 0, 3786, 1674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4092, 0, 0, 1, 4096, 4096, 0, 7, 1, 0, 0, 0, 0, 0, 366, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 0, 0, 0, 1, 4031, 0, 0, 124, 3357, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3730, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 3, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 3195, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16431, 16334, 17757, 16761, 15587, 16016, 15779] avg=16384.0000 max=17757 max_rank=3 min=15587 min_rank=5 imbalance_ratio=1.083801 max_load_reduction=113315 max_load_reduction_pct=0.864525 moved_to_echo=64143 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 56, 0, 2426, 858, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 41, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 26, 0, 4096, 0, 4096, 0, 0, 0, 2638, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 5, 4096, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 371, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 0, 0, 0, 4070, 0, 0, 694, 3345, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3725, 0, 4096, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 774, 3, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 3197, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16357, 16425, 16335, 16622, 16781, 16235, 16007, 16310] avg=16384.0000 max=16781 max_rank=4 min=16007 min_rank=6 imbalance_ratio=1.024231 max_load_reduction=114291 max_load_reduction_pct=0.871971 moved_to_echo=65333 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 7, 4082, 54, 0, 31, 0, 4025, 4, 0, 137, 3, 0, 3, 2420, 852, 723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 0, 0, 4085, 4, 0, 3, 4, 0, 47, 0, 4, 0, 4053, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6, 3, 3, 0, 0, 3, 4, 80, 20, 2, 0, 4092, 2, 0, 4093, 4092, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 0, 0, 3, 4088, 3, 4023, 54, 0, 0, 1235, 1673, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3238, 9, 3935, 112, 3, 13, 4010, 4037, 5, 0, 0, 2, 3, 0, 2, 3, 366, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 2, 3, 49, 4093, 0, 0, 2367, 3356, 0, 0, 0, 2, 3601, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 3727, 0, 4043, 4092, 0, 0, 75, 0, 3, 901, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 3, 982, 3, 0, 0, 3, 4093, 4080, 0, 6, 2, 1, 0, 4093, 3187, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16432, 16338, 16446, 15006, 15750, 17572, 17020, 16508] avg=16384.0000 max=17572 max_rank=5 min=15006 min_rank=3 imbalance_ratio=1.072510 max_load_reduction=113500 max_load_reduction_pct=0.865936 moved_to_echo=66850 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 1, 0, 54, 0, 2422, 854, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 47, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 2993, 1674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 4095, 0, 0, 1, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 366, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 1, 4084, 0, 0, 78, 3359, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3730, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 3195, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16344, 16431, 16334, 16958, 16732, 15639, 16016, 16618] avg=16384.0000 max=16958 max_rank=3 min=15639 min_rank=5 imbalance_ratio=1.035034 max_load_reduction=114114 max_load_reduction_pct=0.870621 moved_to_echo=65005 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 0, 0, 32768, 128, 0, 128, 128, 0, 384, 0, 128, 0, 32640, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 0, 0, 128, 32768, 256, 128, 128, 0, 32640, 128, 0, 512, 128, 384, 128, 19456, 6912, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32384, 128, 128, 128, 256, 32640, 0, 0, 7936, 26880, 0, 0, 0, 128, 31360, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[30080, 128, 32256, 512, 128, 128, 32512, 32768, 128, 128, 128, 128, 128, 0, 128, 128, 2944, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 6912, 128, 0, 0, 128, 32768, 32768, 0, 128, 128, 128, 0, 32768, 25600, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32256, 0, 128, 128, 32768, 128, 32640, 256, 0, 0, 19968, 13440, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 29824, 0, 32640, 32768, 128, 0, 256, 0, 128, 2688, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 128, 384, 128, 128, 0, 32768, 128, 0, 32768, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 4085, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3974, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 8, 0, 0, 0, 1337, 19, 0, 0, 0, 0, 0, 0, 636, 0, 0, 6, 0, 4096, 0, 4096, 4094, 0, 0, 4096, 0, 4096, 4036, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 2, 4092, 3950, 0, 4096, 0, 4092, 0, 0, 4093, 0, 4096, 0, 4, 3992, 0, 0, 4096, 2, 0, 0, 4096, 3375, 0, 0, 9, 0, 0, 3451, 0, 0, 0, 4092, 0, 0, 0, 3881, 0, 4096, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[6, 14, 3778, 11, 11, 11, 0, 0, 4076, 10, 9, 4085, 61, 9, 130, 23, 4045, 12, 4086, 0, 0, 0, 0, 4083, 4073, 8, 4036, 11, 11, 173, 4072, 9, 11, 17, 0, 0, 0, 0, 33, 0, 1610, 0, 84, 12, 0, 0, 0, 56, 1301, 0, 0, 205, 3, 4006, 0, 4070, 3876, 0, 9, 3871, 0, 4085, 4085, 0, 0, 0, 8, 0, 0, 11, 0, 10, 10, 4054, 10, 9, 11, 20, 1033, 4083, 4086, 262, 4085, 10, 4072, 8, 0, 4086, 469, 3813, 7, 0, 1964, 0, 0, 4085, 0, 9, 0, 3951, 1853, 0, 381, 0, 0, 46, 3915, 0, 0, 8, 4086, 1093, 0, 10, 4084, 0, 3046, 12, 17, 0, 19, 10, 0, 4087, 4043, 0, 9, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3875, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 15, 0, 0, 0, 1638, 55, 0, 0, 0, 0, 0, 0, 2033, 0, 0, 2, 0, 4096, 0, 4096, 4091, 0, 0, 4096, 0, 4096, 3792, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 4009, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4085, 0, 0, 4096, 0, 0, 0, 4096, 4069, 0, 0, 7, 0, 0, 1042, 0, 0, 0, 4096, 4, 0, 0, 4066, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3605, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1755, 0, 0, 0, 0, 0, 0, 0, 1320, 0, 0, 4, 0, 4096, 0, 4096, 4091, 0, 0, 4096, 0, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 15, 4096, 3640, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 3735, 0, 0, 4096, 0, 0, 0, 4095, 3847, 0, 0, 0, 0, 0, 2733, 0, 0, 0, 4096, 0, 0, 0, 4055, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3847, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 16, 0, 0, 0, 885, 43, 0, 0, 0, 0, 0, 0, 1879, 0, 0, 0, 0, 4096, 0, 4096, 4083, 0, 0, 4096, 0, 4096, 3995, 0, 5, 3, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 7, 4096, 3720, 0, 4096, 0, 4092, 0, 0, 4096, 0, 4096, 0, 0, 3832, 0, 0, 4096, 0, 0, 0, 4096, 4015, 0, 0, 7, 0, 0, 2528, 0, 0, 0, 4096, 4, 0, 0, 3915, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 6, 0, 4096, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3981, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 26, 0, 0, 0, 1675, 41, 0, 0, 0, 0, 0, 0, 1346, 0, 0, 11, 0, 4096, 0, 4096, 4094, 0, 0, 4096, 0, 4096, 3963, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 4094, 3972, 0, 4096, 0, 4091, 0, 0, 4094, 0, 4096, 0, 0, 3922, 0, 0, 4096, 2, 0, 0, 4096, 2936, 0, 0, 12, 0, 0, 3216, 0, 0, 0, 4089, 0, 0, 0, 3586, 0, 4096, 0, 0, 0, 0, 0, 6, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[3, 5, 3916, 3, 4, 0, 0, 0, 4089, 3, 3, 4093, 21, 3, 43, 7, 4089, 3, 4093, 0, 0, 0, 0, 4091, 4092, 3, 4075, 3, 3, 57, 4086, 3, 3, 4, 0, 0, 0, 0, 13, 0, 1495, 0, 15, 5, 0, 1, 0, 10, 1373, 0, 0, 140, 3, 4072, 0, 4091, 3973, 0, 3, 3990, 0, 4093, 4061, 0, 0, 0, 1, 0, 3, 3, 0, 3, 3, 4078, 3, 3, 3, 8, 542, 4093, 4093, 112, 4092, 3, 4086, 3, 0, 4093, 107, 3996, 3, 0, 2724, 0, 0, 4093, 0, 3, 0, 4029, 2602, 0, 131, 0, 2, 0, 3566, 0, 0, 3, 4093, 582, 0, 3, 4084, 0, 3482, 1, 21, 0, 5, 3, 0, 4093, 4082, 0, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3995, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 9, 0, 0, 0, 1964, 9, 0, 0, 0, 0, 0, 0, 1214, 0, 0, 1, 0, 4096, 0, 4096, 4086, 0, 0, 4096, 0, 4096, 3978, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4, 4095, 4059, 0, 4096, 0, 4095, 0, 0, 4095, 0, 4096, 0, 0, 3952, 0, 0, 4096, 0, 0, 0, 4096, 4014, 0, 0, 3, 0, 0, 1449, 0, 0, 0, 4095, 1, 0, 0, 4032, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 4096, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 2469, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 151, 4096, 0, 3605, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1755, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 2491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1320, 0, 0, 4, 0, 2871, 0, 0, 4091, 0, 0, 4096, 0, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 15, 4096, 4096, 2293, 1605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3640, 0, 0, 0, 4096, 0, 0, 657, 0, 4096, 0, 0, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3847, 0, 0, 0, 0, 0, 2733, 0, 0, 0, 4096, 0, 970, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16409, 16044, 16535, 16350, 16201, 16224, 16630, 16679] avg=16384.0000 max=16679 max_rank=7 min=16044 min_rank=1 imbalance_ratio=1.018005 max_load_reduction=114393 max_load_reduction_pct=0.872749 moved_to_echo=65734 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6, 14, 3778, 11, 11, 11, 0, 0, 4076, 10, 9, 4085, 61, 9, 130, 23, 2463, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 12, 0, 0, 0, 0, 0, 151, 4073, 8, 4036, 11, 11, 173, 4072, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 17, 0, 0, 0, 0, 33, 0, 1610, 0, 84, 12, 0, 0, 0, 56, 4086, 4085, 4085, 2485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1301, 0, 0, 205, 3, 2807, 0, 0, 3876, 0, 9, 3871, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 11, 0, 10, 10, 4054, 10, 9, 11, 20, 1033, 4083, 4070, 2286, 1600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 262, 0, 10, 4072, 8, 0, 655, 469, 3813, 7, 0, 1964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3951, 1853, 0, 381, 0, 0, 46, 3915, 0, 0, 8, 4086, 1093, 968, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4084, 0, 3046, 12, 17, 0, 19, 10, 0, 4087, 4043, 0, 9, 0, 329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16343, 16601, 16564, 16157, 17215, 15346, 17180, 15666] avg=16384.0000 max=17215 max_rank=4 min=15346 min_rank=5 imbalance_ratio=1.050720 max_load_reduction=113857 max_load_reduction_pct=0.868660 moved_to_echo=65407 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 4085, 0, 0, 0, 0, 2467, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 151, 4096, 0, 3974, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8, 0, 0, 0, 1337, 19, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 2492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 636, 0, 0, 6, 0, 2871, 0, 0, 4094, 0, 0, 4096, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 2, 4092, 4096, 2293, 1604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3950, 0, 0, 0, 4092, 0, 0, 656, 0, 4096, 0, 4, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 3375, 0, 0, 9, 0, 0, 3451, 0, 0, 0, 4092, 0, 970, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 4096, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 1, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16401, 16413, 16154, 15739, 16181, 16790, 16884, 16510] avg=16384.0000 max=16884 max_rank=6 min=15739 min_rank=3 imbalance_ratio=1.030518 max_load_reduction=114188 max_load_reduction_pct=0.871185 moved_to_echo=66365 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 0, 0, 0, 0, 2469, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 151, 4096, 0, 3875, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 15, 0, 0, 0, 1638, 55, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 2491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2033, 0, 0, 2, 0, 2871, 0, 0, 4091, 0, 0, 4096, 0, 0, 3792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 4096, 2293, 1605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 4096, 0, 0, 657, 0, 4096, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4069, 0, 0, 7, 0, 0, 1042, 0, 0, 0, 4096, 4, 970, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16393, 16314, 16489, 16885, 16185, 16943, 15173, 16690] avg=16384.0000 max=16943 max_rank=5 min=15173 min_rank=6 imbalance_ratio=1.034119 max_load_reduction=114129 max_load_reduction_pct=0.870735 moved_to_echo=64991 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 2468, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 151, 4096, 0, 3995, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 1964, 9, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 2491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1214, 0, 0, 1, 0, 2871, 0, 0, 4086, 0, 0, 4096, 0, 0, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4, 4095, 4096, 2293, 1605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 4095, 0, 0, 657, 0, 4096, 0, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4014, 0, 0, 3, 0, 0, 1449, 0, 0, 0, 4095, 1, 970, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 4096, 1, 0, 0, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16404, 16434, 16762, 16246, 16192, 16859, 15517, 16658] avg=16384.0000 max=16859 max_rank=5 min=15517 min_rank=6 imbalance_ratio=1.028992 max_load_reduction=114213 max_load_reduction_pct=0.871376 moved_to_echo=65226 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 6, 0, 4096, 0, 0, 4084, 0, 0, 0, 0, 2471, 1653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 146, 4096, 0, 3981, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 26, 0, 0, 0, 1675, 41, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 2492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1346, 0, 0, 11, 0, 2864, 0, 0, 4094, 0, 0, 4096, 0, 0, 3963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 4094, 4096, 2297, 1604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 0, 0, 0, 4091, 0, 0, 649, 0, 4096, 0, 0, 3922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 2936, 0, 0, 12, 0, 0, 3216, 0, 0, 0, 4089, 0, 974, 893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3586, 0, 4096, 0, 0, 0, 0, 0, 6, 4096, 4096, 0, 0, 0, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16406, 16415, 16530, 16374, 16180, 16730, 16218, 16219] avg=16384.0000 max=16730 max_rank=5 min=16180 min_rank=4 imbalance_ratio=1.021118 max_load_reduction=114342 max_load_reduction_pct=0.872360 moved_to_echo=65347 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 5, 3916, 3, 4, 0, 0, 0, 4089, 3, 3, 4093, 21, 3, 43, 7, 2467, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 0, 0, 0, 0, 0, 151, 4092, 3, 4075, 3, 3, 57, 4086, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 13, 0, 1495, 0, 15, 5, 0, 1, 0, 10, 4093, 4093, 4093, 2489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1373, 0, 0, 140, 3, 2854, 0, 0, 3973, 0, 3, 3990, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 3, 0, 3, 3, 4078, 3, 3, 3, 8, 542, 4093, 4091, 2290, 1603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 112, 0, 3, 4086, 3, 0, 656, 107, 3996, 3, 0, 2724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4029, 2602, 0, 131, 0, 2, 0, 3566, 0, 0, 3, 4093, 582, 970, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4084, 0, 3482, 1, 21, 0, 5, 3, 0, 4093, 4082, 0, 3, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16310, 16565, 16314, 16397, 16727, 15783, 16865, 16111] avg=16384.0000 max=16865 max_rank=6 min=15783 min_rank=5 imbalance_ratio=1.029358 max_load_reduction=114207 max_load_reduction_pct=0.871330 moved_to_echo=65486 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 4088, 0, 0, 0, 0, 2469, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 151, 4096, 0, 3847, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 885, 43, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 2491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1879, 0, 0, 0, 0, 2871, 0, 0, 4083, 0, 0, 4096, 0, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 7, 4096, 4096, 2293, 1605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 0, 0, 0, 4092, 0, 0, 657, 0, 4096, 0, 0, 3832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4015, 0, 0, 7, 0, 0, 2528, 0, 0, 0, 4096, 4, 970, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3915, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16406, 16286, 15724, 16924, 16191, 16397, 16605, 16539] avg=16384.0000 max=16924 max_rank=3 min=15724 min_rank=2 imbalance_ratio=1.032959 max_load_reduction=114148 max_load_reduction_pct=0.870880 moved_to_echo=65732 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 1280, 32768, 128, 31488, 128, 128, 256, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[11136, 0, 0, 384, 128, 22912, 0, 0, 32512, 0, 128, 32512, 0, 0, 32000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[31616, 384, 0, 128, 32768, 128, 0, 5248, 640, 32512, 128, 128, 28288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 32384, 128, 128, 128, 128, 0, 32768, 128, 128, 32768, 128, 128, 256, 128, 19840, 13312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 31744, 0, 31104, 128, 128, 0, 128, 128, 128, 32768, 32768, 128, 128, 128, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 1664, 32768, 32768, 18432, 12928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 32640, 26752, 0, 512, 128, 128, 128, 22016, 0, 0, 128, 32768, 1792, 7808, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 0, 12416, 256, 128, 128, 0, 128, 0, 128, 32768, 32768, 32768, 19968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 3532, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3117, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 18, 4096, 0, 0, 4096, 4096, 0, 4095, 4, 98, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 1661, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 5, 0, 0, 0, 4096, 3834, 22, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 2737, 0, 4096, 0, 4096, 0, 4086, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 2, 2119, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4092, 0, 0, 4096, 4096, 0, 4079, 19, 636, 4096, 10, 4096, 0, 0, 0, 0, 0, 4096, 2979, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4094, 4096, 6, 0, 0, 0, 4091, 3811, 5, 4095, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 3, 3, 0, 4086, 0, 0, 3, 47, 0, 4089, 2, 0, 0, 1081, 3707, 0, 4078, 0, 4087, 0, 4092, 3, 0, 20, 4093, 0, 1, 3, 3, 0, 0, 0, 0, 4034, 29, 1605, 4, 3, 0, 3, 4091, 0, 3, 3, 3, 6, 1, 8, 4090, 11, 0, 4093, 2, 16, 0, 4046, 4, 0, 3, 1, 3, 0, 1, 0, 84, 2, 7, 76, 3, 3, 3, 0, 4093, 3, 3, 3950, 4051, 3, 4093, 1, 132, 4084, 1537, 4093, 0, 0, 167, 0, 3, 4035, 795, 0, 3, 3976, 0, 4, 0, 3, 0, 19, 0, 4093, 4093, 4001, 3, 4093, 0, 4077, 0, 4092, 4092, 2, 3, 0, 3, 4093, 3559, 0, 4092, 0, 3, 3, 0, 5, 0, 3, 4072] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 3468, 0, 4096, 0, 4096, 0, 4088, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 1141, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4093, 0, 0, 4096, 4096, 0, 4071, 11, 490, 4096, 21, 4096, 0, 0, 0, 0, 0, 4096, 3380, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 87, 0, 0, 0, 4090, 3718, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 3287, 0, 4096, 0, 4096, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2934, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4095, 0, 0, 4096, 4096, 0, 4080, 57, 178, 4096, 4, 4096, 0, 0, 0, 0, 0, 4096, 2319, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 10, 0, 0, 0, 4095, 3517, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 9, 11, 0, 4081, 0, 0, 9, 127, 0, 4050, 7, 0, 0, 728, 3882, 0, 4028, 0, 4053, 0, 4083, 8, 0, 75, 4085, 0, 0, 11, 10, 0, 0, 0, 1, 3694, 101, 1395, 12, 9, 0, 10, 4079, 0, 10, 11, 9, 24, 0, 34, 4078, 42, 0, 4086, 10, 56, 0, 4007, 14, 14, 10, 6, 12, 0, 4, 0, 177, 7, 24, 212, 10, 10, 9, 0, 4086, 8, 12, 3799, 3978, 9, 4087, 4, 0, 4064, 2785, 4085, 0, 0, 340, 0, 9, 3974, 40, 0, 10, 3866, 0, 13, 0, 8, 0, 65, 0, 4084, 4084, 3889, 11, 4084, 0, 4061, 0, 4084, 4057, 0, 10, 0, 12, 4086, 3800, 0, 4081, 0, 9, 9, 0, 1, 0, 8, 4026] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 3477, 0, 4096, 0, 4096, 0, 4081, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 1631, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4091, 0, 0, 4096, 4096, 0, 4077, 98, 505, 4096, 12, 4096, 0, 0, 0, 0, 0, 4096, 3023, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 38, 0, 0, 0, 4094, 3523, 6, 4094, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 35, 3246, 0, 4096, 0, 4096, 0, 4093, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 3198, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4095, 0, 0, 4096, 4096, 0, 4086, 29, 252, 4096, 2, 4096, 0, 0, 0, 0, 0, 4096, 2150, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 31, 0, 0, 0, 4094, 3351, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 2737, 2418, 2110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4086, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 2119, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2451, 1641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4092, 0, 0, 4096, 4096, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 636, 4096, 10, 1200, 0, 0, 0, 0, 0, 4096, 2979, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4096, 0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4091, 3811, 5, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15455, 16386, 16395, 16371, 16373, 17132, 16419, 16541] avg=16384.0000 max=17132 max_rank=5 min=15455 min_rank=0 imbalance_ratio=1.045654 max_load_reduction=113940 max_load_reduction_pct=0.869293 moved_to_echo=66465 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 3532, 2418, 2109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3117, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2451, 1641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 18, 4096, 0, 0, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 98, 4096, 0, 1200, 0, 0, 0, 0, 0, 4096, 1661, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 3834, 22, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16251, 16392, 17392, 16377, 16402, 15251, 16421, 16586] avg=16384.0000 max=17392 max_rank=2 min=15251 min_rank=5 imbalance_ratio=1.061523 max_load_reduction=113680 max_load_reduction_pct=0.867310 moved_to_echo=64660 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 11, 0, 4081, 0, 0, 9, 127, 0, 4050, 7, 0, 0, 728, 3882, 2410, 2104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 4053, 0, 4083, 8, 0, 75, 4085, 0, 0, 11, 10, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3694, 101, 1395, 12, 9, 0, 10, 4079, 0, 10, 11, 9, 24, 0, 4084, 1980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4078, 42, 0, 4086, 10, 56, 0, 4007, 14, 14, 10, 6, 12, 0, 4, 2445, 1636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 7, 24, 212, 10, 10, 9, 0, 4086, 8, 12, 3799, 3978, 9, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4064, 2785, 1196, 0, 0, 340, 0, 9, 3974, 40, 0, 10, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 8, 0, 65, 0, 0, 38, 3889, 11, 0, 0, 4061, 0, 4084, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 12, 4086, 3800, 0, 4081, 0, 9, 9, 0, 1, 0, 8, 4026, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17418, 16361, 15419, 16454, 16428, 16288, 16226, 16478] avg=16384.0000 max=17418 max_rank=0 min=15419 min_rank=2 imbalance_ratio=1.063110 max_load_reduction=113654 max_load_reduction_pct=0.867111 moved_to_echo=65420 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 3, 0, 4086, 0, 0, 3, 47, 0, 4089, 2, 0, 0, 1081, 3707, 2416, 2108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4087, 0, 4092, 3, 0, 20, 4093, 0, 1, 3, 3, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 29, 1605, 4, 3, 0, 3, 4091, 0, 3, 3, 3, 6, 1, 4093, 1985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4090, 11, 0, 4093, 2, 16, 0, 4046, 4, 0, 3, 1, 3, 0, 1, 2450, 1640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 2, 7, 76, 3, 3, 3, 0, 4093, 3, 3, 3950, 4051, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 132, 4084, 1537, 1198, 0, 0, 167, 0, 3, 4035, 795, 0, 3, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 0, 19, 0, 0, 37, 4001, 3, 0, 0, 4077, 0, 4092, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 3, 4093, 3559, 0, 4092, 0, 3, 3, 0, 5, 0, 3, 4072, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17545, 16388, 15863, 16368, 16374, 15931, 16328, 16275] avg=16384.0000 max=17545 max_rank=0 min=15863 min_rank=2 imbalance_ratio=1.070862 max_load_reduction=113527 max_load_reduction_pct=0.866142 moved_to_echo=64908 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 35, 3246, 2418, 2109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4093, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3198, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2451, 1641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4095, 0, 0, 4096, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 252, 4096, 2, 1200, 0, 0, 0, 0, 0, 4096, 2150, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 4094, 3351, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15999, 16390, 17473, 16374, 16378, 15921, 16421, 16116] avg=16384.0000 max=17473 max_rank=2 min=15921 min_rank=5 imbalance_ratio=1.066467 max_load_reduction=113599 max_load_reduction_pct=0.866692 moved_to_echo=64836 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 3287, 2418, 2109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2934, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2451, 1641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4095, 0, 0, 4096, 4096, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 178, 4096, 4, 1200, 0, 0, 0, 0, 0, 4096, 2319, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4095, 3517, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16007, 16384, 17209, 16379, 16375, 16046, 16421, 16251] avg=16384.0000 max=17209 max_rank=2 min=16007 min_rank=0 imbalance_ratio=1.050354 max_load_reduction=113863 max_load_reduction_pct=0.868706 moved_to_echo=65093 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 3477, 2420, 2110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4081, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1631, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2457, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4090, 0, 0, 4096, 4096, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 505, 4096, 12, 1187, 0, 0, 0, 0, 0, 4096, 3023, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 4094, 3523, 6, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16209, 16384, 15905, 16381, 16374, 17113, 16415, 16291] avg=16384.0000 max=17113 max_rank=5 min=15905 min_rank=2 imbalance_ratio=1.044495 max_load_reduction=113959 max_load_reduction_pct=0.869438 moved_to_echo=66193 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 3468, 2417, 2109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4088, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1141, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2451, 1641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4093, 0, 0, 4096, 4096, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 490, 4096, 21, 1200, 0, 0, 0, 0, 0, 4096, 3380, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 4090, 3718, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16188, 16387, 15416, 16368, 16368, 17390, 16421, 16534] avg=16384.0000 max=17390 max_rank=5 min=15416 min_rank=2 imbalance_ratio=1.061401 max_load_reduction=113682 max_load_reduction_pct=0.867325 moved_to_echo=66713 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 32384, 256, 17152, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 128, 128, 32768, 16000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 0, 32768, 128, 128, 0, 32640, 128, 128, 128, 128, 128, 0, 128, 19712, 13184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 128, 128, 256, 0, 32768, 128, 0, 0, 1920, 27392, 19456, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[256, 2304, 32768, 4480, 9600, 0, 0, 512, 0, 128, 32640, 16384, 0, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 384, 32512, 128, 0, 0, 32768, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 128, 0, 128, 32768, 29184, 128, 32768, 0, 128, 128, 0, 128, 0, 128, 32768, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 384, 128, 128, 384, 128, 128, 128, 128, 32768, 128, 128, 32384, 32640, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 0, 32768, 0, 32768, 128, 0, 128, 32768, 0, 128, 128, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3949, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 3419, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 2512, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 1967, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 471, 0, 0, 0, 4096, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 3988, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4007, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 2230, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4094, 0, 0, 0, 2, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 0, 4096, 4096, 4096, 0, 24, 0, 1525, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4069, 0, 0, 0, 569, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 4053, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 3831, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 1512, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 4096, 0, 0, 0, 9, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 4096, 4094, 0, 20, 0, 1638, 0, 4095, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 4074, 0, 0, 0, 1266, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4096, 4072, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 3169, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 916, 0, 0, 0, 0, 0, 4096, 4096, 6, 4094, 4090, 0, 0, 0, 3, 4091, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 4096, 4094, 0, 14, 0, 3166, 0, 4096, 0, 4096, 0, 0, 0, 3, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4093, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4082, 0, 0, 0, 970, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 12, 0, 0, 5, 0, 19, 0, 2, 0, 3961, 3085, 9, 97, 62, 9, 0, 0, 4085, 4071, 13, 0, 0, 4082, 0, 273, 8, 4059, 279, 18, 658, 4084, 0, 17, 82, 1027, 10, 10, 50, 9, 0, 4078, 0, 13, 0, 41, 0, 12, 10, 13, 343, 0, 5, 0, 120, 0, 3053, 4018, 0, 4079, 4085, 0, 0, 0, 0, 3592, 0, 14, 11, 89, 10, 0, 0, 3899, 235, 12, 0, 0, 7, 10, 4022, 3892, 3961, 0, 0, 0, 3939, 7, 3942, 9, 4074, 22, 11, 0, 0, 4021, 7, 10, 4083, 4081, 0, 12, 4084, 3993, 0, 8, 4080, 0, 14, 13, 0, 4086, 14, 4082, 152, 841, 0, 4086, 13, 0, 10, 3593, 11, 23, 9, 4084, 1420, 2518] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 4058, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 3136, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 97, 0, 0, 8, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 2529, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 28, 0, 2180, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 20, 0, 0, 4081, 0, 0, 0, 252, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 0, 7, 0, 0, 0, 4026, 3850, 3, 40, 18, 3, 0, 0, 4093, 4089, 4, 0, 0, 4092, 0, 1377, 3, 4081, 178, 2, 314, 4093, 0, 0, 25, 158, 3, 3, 34, 3, 0, 4092, 0, 4, 0, 12, 0, 3, 3, 4, 161, 0, 2, 0, 44, 0, 3450, 4088, 0, 4090, 4093, 0, 0, 0, 0, 3846, 0, 5, 3, 26, 3, 0, 1, 4055, 97, 4, 0, 0, 4, 3, 4060, 4033, 4015, 0, 0, 0, 3827, 4, 4027, 3, 4089, 12, 3, 0, 0, 4067, 3, 3, 4092, 4092, 0, 4, 4092, 4020, 0, 2, 4092, 0, 4, 3, 0, 4093, 4, 4092, 674, 346, 0, 4093, 4, 2, 3, 2503, 3, 6, 5, 4092, 653, 3356] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4051, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 2962, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 2117, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4095, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 6, 0, 2253, 0, 4096, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 4093, 0, 0, 0, 899, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3949, 0, 0, 0, 0, 4096, 4076, 178, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3419, 0, 4096, 0, 0, 0, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 4095, 4096, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2512, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 2758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2894, 2488, 1537, 1338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1608, 4096, 4096, 0, 0, 0, 1967, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 20, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 471, 0, 0, 0, 2559, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16443, 16861, 16176, 17558, 16449, 15863, 16404, 15318] avg=16384.0000 max=17558 max_rank=3 min=15318 min_rank=7 imbalance_ratio=1.071655 max_load_reduction=113514 max_load_reduction_pct=0.866043 moved_to_echo=64034 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4096, 4072, 0, 0, 0, 0, 4096, 4076, 178, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3169, 0, 4096, 0, 0, 0, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 4096, 4096, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 916, 0, 0, 0, 0, 0, 4096, 4096, 6, 4094, 2754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4091, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 2894, 2488, 1537, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1608, 4096, 4094, 0, 14, 0, 3166, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 20, 4093, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4082, 0, 0, 0, 970, 0, 0, 0, 2559, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16579, 16611, 16193, 15962, 16447, 17077, 16400, 15803] avg=16384.0000 max=17077 max_rank=5 min=15803 min_rank=7 imbalance_ratio=1.042297 max_load_reduction=113995 max_load_reduction_pct=0.869713 moved_to_echo=65727 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 4053, 0, 0, 0, 0, 4096, 4080, 179, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3831, 0, 4096, 0, 0, 0, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 4096, 4096, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1512, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 2753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 2898, 2493, 1541, 1343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1603, 4096, 4094, 0, 20, 0, 1638, 0, 4095, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 16, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4074, 0, 0, 0, 1266, 0, 0, 0, 2555, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16563, 17264, 16182, 16552, 16474, 15548, 16400, 16089] avg=16384.0000 max=17264 max_rank=1 min=15548 min_rank=5 imbalance_ratio=1.053711 max_load_reduction=113808 max_load_reduction_pct=0.868286 moved_to_echo=64511 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 3988, 0, 0, 0, 0, 4096, 4076, 178, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4007, 0, 4096, 0, 0, 0, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 4096, 4096, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2230, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 2756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 0, 2894, 2488, 1537, 1338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1608, 4096, 4096, 0, 24, 0, 1525, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 20, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4069, 0, 0, 0, 569, 0, 0, 0, 2559, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16490, 17449, 16159, 17274, 16462, 15445, 16404, 15389] avg=16384.0000 max=17449 max_rank=1 min=15389 min_rank=7 imbalance_ratio=1.065002 max_load_reduction=113623 max_load_reduction_pct=0.866875 moved_to_echo=63700 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 0, 0, 5, 0, 19, 0, 2, 0, 3961, 3085, 9, 97, 62, 9, 4021, 4064, 177, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 13, 0, 0, 4082, 0, 273, 8, 4059, 279, 18, 658, 1151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 82, 1027, 10, 10, 50, 9, 0, 4078, 0, 13, 0, 41, 0, 12, 4086, 4085, 3897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 13, 343, 0, 5, 0, 120, 0, 3053, 4018, 0, 4079, 2750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3592, 0, 14, 11, 89, 10, 0, 0, 3899, 235, 12, 0, 0, 7, 10, 2885, 2443, 1532, 1335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1579, 3892, 3961, 0, 0, 0, 3939, 7, 3942, 9, 0, 22, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 10, 4083, 4081, 0, 12, 20, 3993, 0, 8, 4080, 0, 14, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4082, 152, 841, 0, 4086, 13, 0, 10, 3593, 11, 23, 9, 2552, 1420, 2518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15571, 14612, 17417, 14391, 16074, 17362, 16321, 19324] avg=16384.0000 max=19324 max_rank=7 min=14391 min_rank=3 imbalance_ratio=1.179443 max_load_reduction=111748 max_load_reduction_pct=0.852570 moved_to_echo=69081 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4051, 0, 0, 0, 0, 4096, 4076, 178, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 2962, 0, 4096, 0, 0, 0, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 4096, 4096, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2117, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 2757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2894, 2488, 1537, 1338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1608, 4096, 4095, 0, 6, 0, 2253, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 20, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4093, 0, 0, 0, 899, 0, 0, 0, 2559, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16549, 16404, 16206, 17162, 16448, 16155, 16404, 15744] avg=16384.0000 max=17162 max_rank=3 min=15744 min_rank=7 imbalance_ratio=1.047485 max_load_reduction=113910 max_load_reduction_pct=0.869064 moved_to_echo=64751 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 4058, 0, 0, 0, 0, 4096, 4076, 178, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3136, 0, 4096, 0, 0, 0, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 8, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 4096, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2529, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 2758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2894, 2488, 1537, 1338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1608, 4096, 4096, 0, 28, 0, 2180, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 20, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 20, 0, 0, 4081, 0, 0, 0, 252, 0, 0, 0, 2559, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16561, 16578, 16305, 17575, 16441, 16104, 16404, 15104] avg=16384.0000 max=17575 max_rank=3 min=15104 min_rank=7 imbalance_ratio=1.072693 max_load_reduction=113497 max_load_reduction_pct=0.865913 moved_to_echo=64053 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 0, 7, 0, 0, 0, 4026, 3850, 3, 40, 18, 3, 4067, 4072, 178, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 0, 0, 4092, 0, 1377, 3, 4081, 178, 2, 314, 1153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 158, 3, 3, 34, 3, 0, 4092, 0, 4, 0, 12, 0, 3, 4093, 4093, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 161, 0, 2, 0, 44, 0, 3450, 4088, 0, 4090, 2756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 0, 5, 3, 26, 3, 0, 1, 4055, 97, 4, 0, 0, 4, 3, 2892, 2466, 1535, 1337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1594, 4033, 4015, 0, 0, 0, 3827, 4, 4027, 3, 0, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 4092, 4092, 0, 4, 20, 4020, 0, 2, 4092, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 674, 346, 0, 4093, 4, 2, 3, 2503, 3, 6, 5, 2557, 653, 3356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16316, 15293, 16434, 14598, 16277, 17518, 16335, 18301] avg=16384.0000 max=18301 max_rank=7 min=14598 min_rank=3 imbalance_ratio=1.117004 max_load_reduction=112771 max_load_reduction_pct=0.860374 moved_to_echo=68431 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 256, 32640, 0, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 896, 1280, 128, 32768, 128, 128, 128, 10624, 128, 128, 128, 20480, 2176, 30464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 0, 128, 0, 32640, 31232, 128, 256, 128, 128, 32768, 32640, 1536, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 0, 32768, 0, 22272, 128, 32768, 512, 128, 1024, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12928, 32512, 32640, 0, 128, 0, 20608, 128, 32640, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 1408, 128, 128, 128, 128, 0, 32640, 0, 128, 0, 128, 0, 128, 32768, 32768, 31360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 12416, 0, 128, 0, 256, 0, 31104, 32768, 128, 32768, 22144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 32000, 128, 128, 128, 128, 128, 0, 128, 32640, 384, 128, 128, 0, 128, 128, 23168, 19968, 12416, 10752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1959, 1413, 1360, 0, 1048, 1, 1231, 2131, 1463, 2229, 2063, 1378, 1073, 1819, 2, 708, 881, 1078, 958, 1010, 10, 793, 247, 1455, 973, 150, 2813, 1335, 1158, 297, 1228, 1318, 163, 0, 1534, 819, 556, 2011, 1177, 343, 438, 78, 1577, 2784, 0, 1917, 1499, 1019, 1243, 438, 1004, 1452, 137, 77, 25, 1119, 2, 685, 119, 1143, 965, 2159, 1907, 991, 1084, 5, 0, 1346, 2044, 1357, 2145, 1080, 1236, 1080, 1403, 1256, 1443, 58, 1445, 1111, 1520, 418, 62, 6, 1818, 746, 1087, 44, 1888, 350, 1104, 593, 506, 975, 804, 4, 1258, 784, 7, 1924, 169, 766, 1354, 737, 350, 1338, 708, 1050, 1121, 1293, 2831, 1162, 1241, 1068, 986, 1104, 2228, 1478, 1133, 1346, 1320, 1911, 1629, 1649, 1154, 84, 843, 65] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2335, 1551, 753, 0, 774, 0, 1349, 2556, 1385, 2029, 1623, 563, 592, 2489, 0, 1171, 656, 1128, 1316, 714, 35, 1474, 396, 2069, 911, 374, 1957, 1204, 1343, 432, 1977, 853, 56, 47, 2112, 1127, 288, 2166, 1287, 1268, 593, 54, 2363, 1618, 0, 1036, 1364, 1258, 967, 499, 784, 2196, 37, 155, 49, 847, 0, 570, 119, 722, 1792, 1133, 1457, 1360, 794, 3, 1, 2108, 1236, 2952, 815, 1097, 904, 1646, 1553, 1167, 1173, 51, 1487, 1371, 651, 1124, 50, 25, 2070, 995, 1647, 43, 1600, 295, 846, 823, 376, 1656, 889, 2, 1136, 1382, 8, 1814, 113, 912, 1178, 1647, 264, 1717, 442, 1199, 1241, 646, 1616, 1223, 1090, 1006, 841, 1094, 2490, 1186, 2186, 1278, 938, 1694, 1112, 554, 1664, 38, 520, 60] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1976, 1846, 1380, 0, 701, 0, 1453, 1816, 1881, 2259, 2771, 873, 1022, 2479, 0, 1007, 554, 1498, 1200, 442, 52, 1314, 198, 2148, 512, 217, 2521, 1220, 930, 200, 1442, 1002, 57, 6, 2186, 903, 158, 1876, 864, 1291, 320, 26, 1302, 2400, 0, 1436, 1458, 461, 1004, 468, 1237, 1573, 20, 347, 9, 1128, 2, 1052, 290, 1127, 1065, 2258, 1003, 1131, 1159, 18, 0, 1396, 1411, 1769, 735, 1107, 918, 1392, 1350, 342, 1903, 109, 1781, 1770, 956, 693, 24, 12, 1901, 972, 600, 23, 2027, 541, 885, 522, 942, 1765, 1277, 4, 1331, 1016, 8, 1921, 38, 1017, 1732, 1380, 213, 1180, 436, 713, 888, 1823, 2460, 1673, 2044, 780, 585, 1253, 1674, 1296, 1766, 1294, 832, 1521, 922, 978, 1955, 26, 600, 41] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2214, 1495, 1104, 6, 1050, 0, 1820, 1106, 1084, 2156, 1735, 1289, 1157, 2024, 1, 954, 541, 1477, 643, 821, 6, 793, 324, 1880, 1000, 248, 2282, 1327, 831, 177, 1253, 1236, 185, 50, 1700, 576, 269, 2331, 1247, 1015, 382, 62, 1852, 1878, 1, 1538, 1564, 880, 1444, 789, 937, 1615, 25, 356, 34, 1635, 0, 868, 276, 1247, 1339, 1377, 714, 1320, 740, 10, 0, 1597, 2355, 1892, 1623, 1081, 1030, 1354, 1751, 1090, 1442, 37, 1251, 1536, 1336, 412, 94, 14, 2070, 810, 1495, 17, 1728, 320, 998, 1209, 777, 1745, 1105, 5, 1465, 661, 5, 2034, 36, 1144, 1170, 1710, 389, 1404, 382, 1093, 885, 799, 1921, 2228, 1649, 1019, 523, 1090, 1512, 972, 1464, 1440, 1174, 1389, 1401, 730, 1879, 64, 932, 44] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1623, 858, 2163, 0, 1422, 1, 807, 1487, 669, 1408, 1234, 1348, 1777, 1608, 0, 929, 1098, 1315, 1882, 745, 19, 854, 239, 2118, 471, 20, 2423, 2216, 268, 135, 927, 1325, 283, 0, 2568, 116, 1342, 1873, 1242, 606, 368, 35, 1028, 2334, 0, 1275, 1304, 1551, 1362, 386, 595, 1408, 25, 85, 131, 2071, 1, 756, 287, 748, 959, 2063, 693, 1011, 1965, 19, 0, 1133, 2252, 2272, 2526, 1086, 1237, 1627, 1526, 1470, 2045, 8, 1583, 561, 947, 781, 46, 104, 2253, 1606, 769, 62, 1091, 243, 430, 938, 596, 1147, 914, 0, 411, 487, 12, 1750, 33, 901, 1580, 1554, 279, 1630, 1218, 2140, 941, 1838, 1579, 1504, 2535, 1118, 227, 1546, 2573, 750, 401, 1261, 1527, 1054, 1229, 549, 2586, 187, 410, 130] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2021, 1199, 1656, 0, 791, 0, 1769, 1631, 1207, 2349, 1630, 1335, 1425, 1759, 3, 761, 649, 1336, 836, 652, 8, 838, 263, 1863, 641, 321, 2270, 1700, 1459, 257, 1157, 1409, 48, 2, 2388, 1214, 526, 1826, 732, 961, 488, 4, 1154, 2128, 0, 1374, 1595, 1276, 923, 577, 673, 1266, 55, 432, 27, 887, 6, 358, 89, 579, 1396, 2499, 737, 811, 1138, 3, 0, 1773, 1808, 2156, 1340, 2078, 1035, 1916, 1699, 831, 1722, 58, 1460, 1634, 1702, 414, 39, 8, 2122, 978, 863, 27, 1819, 505, 991, 944, 825, 1531, 841, 0, 983, 1344, 40, 1408, 43, 931, 1434, 1953, 264, 1291, 422, 1327, 1037, 1869, 2241, 1494, 2579, 1158, 386, 823, 1346, 898, 1302, 1128, 1032, 1888, 857, 644, 1639, 25, 772, 28] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2012, 1503, 1393, 0, 941, 0, 1431, 1906, 1225, 1909, 2091, 1056, 907, 1796, 0, 761, 622, 1354, 887, 837, 60, 1023, 264, 2048, 948, 201, 2794, 1274, 1115, 373, 1210, 772, 150, 4, 1909, 978, 340, 1918, 1172, 540, 339, 31, 1745, 1737, 0, 1240, 1311, 1489, 1097, 684, 1180, 1281, 23, 180, 19, 1433, 1, 634, 123, 1729, 878, 2060, 1022, 1195, 802, 3, 0, 957, 1914, 1594, 1582, 1389, 1179, 1289, 1276, 1363, 1749, 81, 1859, 1521, 1268, 453, 48, 2, 1860, 870, 1294, 17, 2095, 307, 1273, 925, 620, 1734, 985, 3, 1198, 1077, 43, 1334, 25, 1112, 1256, 1116, 395, 1441, 699, 969, 623, 591, 2160, 1843, 1990, 1336, 906, 1178, 2464, 1818, 1270, 1266, 1269, 1874, 1321, 595, 1648, 54, 696, 38] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1773, 1455, 1401, 1, 794, 0, 2021, 1654, 1265, 2532, 1944, 996, 1281, 2081, 0, 758, 813, 1432, 1161, 922, 43, 795, 294, 1963, 693, 186, 2445, 1787, 1501, 570, 1051, 1312, 146, 0, 2270, 521, 383, 2482, 784, 969, 416, 16, 1609, 2037, 0, 1167, 2083, 1235, 1284, 673, 1099, 1293, 49, 494, 16, 1304, 1, 917, 47, 1034, 1129, 2053, 935, 1013, 1020, 5, 0, 1398, 1379, 1770, 1778, 1137, 1007, 938, 1658, 1272, 1294, 33, 1868, 1457, 1379, 396, 46, 18, 1654, 1030, 1169, 28, 1494, 551, 1191, 849, 868, 1368, 1218, 1, 1106, 864, 22, 1419, 26, 1112, 1359, 1066, 385, 1238, 642, 1112, 566, 965, 2266, 1423, 1949, 1315, 513, 995, 1585, 1968, 1528, 1071, 1057, 1375, 1395, 784, 1497, 23, 751, 38] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1959, 1413, 1360, 0, 1048, 1, 1231, 2131, 1463, 0, 2063, 1378, 1073, 968, 2, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 881, 1078, 958, 1010, 10, 793, 247, 1455, 973, 150, 2813, 1335, 1158, 297, 1228, 1318, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 1534, 819, 556, 2011, 1177, 343, 438, 78, 1577, 2784, 0, 1917, 1499, 1019, 806, 44, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1243, 438, 1004, 1452, 137, 77, 25, 1119, 2, 685, 119, 1143, 965, 2159, 1907, 991, 2229, 824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 5, 0, 1346, 1463, 0, 2145, 1080, 1236, 1080, 1403, 1256, 1443, 58, 1445, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1520, 418, 62, 6, 1818, 746, 1087, 44, 1888, 350, 1104, 593, 506, 975, 804, 4, 1813, 807, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1258, 784, 7, 1924, 169, 766, 1354, 737, 350, 1338, 708, 1050, 1121, 1293, 2025, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1241, 1068, 986, 1104, 415, 1478, 1133, 1346, 1320, 1911, 1629, 1649, 1154, 84, 843, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16798, 16240, 16810, 16519, 16155, 15078, 16046, 17426] avg=16384.0000 max=17426 max_rank=7 min=15078 min_rank=5 imbalance_ratio=1.063599 max_load_reduction=113646 max_load_reduction_pct=0.867050 moved_to_echo=64705 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2335, 1551, 753, 0, 774, 0, 1349, 2556, 1385, 0, 1623, 563, 592, 1324, 0, 1171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 656, 1128, 1316, 714, 35, 1474, 396, 2069, 911, 374, 1957, 1204, 1343, 432, 1977, 853, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 47, 2112, 1127, 288, 2166, 1287, 1268, 593, 54, 2363, 1618, 0, 1036, 1364, 1258, 460, 61, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 967, 499, 784, 2196, 37, 155, 49, 847, 0, 570, 119, 722, 1792, 1133, 1457, 1360, 2029, 1795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 794, 3, 1, 2108, 885, 0, 815, 1097, 904, 1646, 1553, 1167, 1173, 51, 1487, 1371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 1124, 50, 25, 2070, 995, 1647, 43, 1600, 295, 846, 823, 376, 1656, 889, 2, 2027, 1104, 1157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1136, 1382, 8, 1814, 113, 912, 1178, 1647, 264, 1717, 442, 1199, 1241, 646, 1156, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 1006, 841, 1094, 463, 1186, 2186, 1278, 938, 1694, 1112, 554, 1664, 38, 520, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15976, 17163, 17185, 16511, 15055, 17380, 16078, 15724] avg=16384.0000 max=17380 max_rank=5 min=15055 min_rank=4 imbalance_ratio=1.060791 max_load_reduction=113692 max_load_reduction_pct=0.867401 moved_to_echo=64237 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1976, 1846, 1380, 0, 701, 0, 1453, 1816, 1881, 0, 2771, 873, 1022, 1319, 0, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 1498, 1200, 442, 52, 1314, 198, 2148, 512, 217, 2521, 1220, 930, 200, 1442, 1002, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 6, 2186, 903, 158, 1876, 864, 1291, 320, 26, 1302, 2400, 0, 1436, 1458, 461, 700, 61, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 468, 1237, 1573, 20, 347, 9, 1128, 2, 1052, 290, 1127, 1065, 2258, 1003, 1131, 2259, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1159, 18, 0, 1396, 1010, 0, 735, 1107, 918, 1392, 1350, 342, 1903, 109, 1781, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 956, 693, 24, 12, 1901, 972, 600, 23, 2027, 541, 885, 522, 942, 1765, 1277, 4, 1362, 1099, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1331, 1016, 8, 1921, 38, 1017, 1732, 1380, 213, 1180, 436, 713, 888, 1823, 1760, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2044, 780, 585, 1253, 312, 1296, 1766, 1294, 832, 1521, 922, 978, 1955, 26, 600, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18045, 15820, 15536, 17048, 14990, 16299, 17129, 16205] avg=16384.0000 max=18045 max_rank=0 min=14990 min_rank=4 imbalance_ratio=1.101379 max_load_reduction=113027 max_load_reduction_pct=0.862328 moved_to_echo=64623 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2214, 1495, 1104, 6, 1050, 0, 1820, 1106, 1084, 0, 1735, 1289, 1157, 1076, 1, 954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 1477, 643, 821, 6, 793, 324, 1880, 1000, 248, 2282, 1327, 831, 177, 1253, 1236, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 50, 1700, 576, 269, 2331, 1247, 1015, 382, 62, 1852, 1878, 1, 1538, 1564, 880, 547, 50, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1444, 789, 937, 1615, 25, 356, 34, 1635, 0, 868, 276, 1247, 1339, 1377, 714, 1320, 2156, 1150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 740, 10, 0, 1597, 1686, 0, 1623, 1081, 1030, 1354, 1751, 1090, 1442, 37, 1251, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1336, 412, 94, 14, 2070, 810, 1495, 17, 1728, 320, 998, 1209, 777, 1745, 1105, 5, 1230, 898, 742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1465, 661, 5, 2034, 36, 1144, 1170, 1710, 389, 1404, 382, 1093, 885, 799, 1374, 2228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1649, 1019, 523, 1090, 282, 972, 1464, 1440, 1174, 1389, 1401, 730, 1879, 64, 932, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16091, 15456, 16179, 17282, 16228, 17005, 16779, 16052] avg=16384.0000 max=17282 max_rank=3 min=15456 min_rank=1 imbalance_ratio=1.054810 max_load_reduction=113790 max_load_reduction_pct=0.868149 moved_to_echo=66064 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1773, 1455, 1401, 1, 794, 0, 2021, 1654, 1265, 0, 1944, 996, 1281, 1107, 0, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 813, 1432, 1161, 922, 43, 795, 294, 1963, 693, 186, 2445, 1787, 1501, 570, 1051, 1312, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 2270, 521, 383, 2482, 784, 969, 416, 16, 1609, 2037, 0, 1167, 2083, 1235, 645, 51, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 673, 1099, 1293, 49, 494, 16, 1304, 1, 917, 47, 1034, 1129, 2053, 935, 1013, 2532, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 5, 0, 1398, 988, 0, 1778, 1137, 1007, 938, 1658, 1272, 1294, 33, 1868, 1457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1379, 396, 46, 18, 1654, 1030, 1169, 28, 1494, 551, 1191, 849, 868, 1368, 1218, 1, 1290, 923, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1106, 864, 22, 1419, 26, 1112, 1359, 1066, 385, 1238, 642, 1112, 566, 965, 1621, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1949, 1315, 513, 995, 295, 1968, 1528, 1071, 1057, 1375, 1395, 784, 1497, 23, 751, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16450, 17329, 16844, 16949, 15853, 16167, 14926, 16554] avg=16384.0000 max=17329 max_rank=1 min=14926 min_rank=6 imbalance_ratio=1.057678 max_load_reduction=113743 max_load_reduction_pct=0.867790 moved_to_echo=63500 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1623, 858, 2163, 0, 1422, 1, 807, 1487, 669, 0, 1234, 1348, 1777, 848, 0, 929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1098, 1315, 1882, 745, 19, 854, 239, 2118, 471, 20, 2423, 2216, 268, 135, 927, 1325, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 0, 2568, 116, 1342, 1873, 1242, 606, 368, 35, 1028, 2334, 0, 1275, 1304, 1551, 453, 43, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1362, 386, 595, 1408, 25, 85, 131, 2071, 1, 756, 287, 748, 959, 2063, 693, 1011, 1408, 1382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1965, 19, 0, 1133, 1603, 0, 2526, 1086, 1237, 1627, 1526, 1470, 2045, 8, 1583, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 947, 781, 46, 104, 2253, 1606, 769, 62, 1091, 243, 430, 938, 596, 1147, 914, 0, 2099, 717, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 411, 487, 12, 1750, 33, 901, 1580, 1554, 279, 1630, 1218, 2140, 941, 1838, 1126, 1504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2535, 1118, 227, 1546, 474, 750, 401, 1261, 1527, 1054, 1229, 549, 2586, 187, 410, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15166, 16648, 16477, 15371, 18389, 15633, 17404, 15984] avg=16384.0000 max=18389 max_rank=4 min=15166 min_rank=0 imbalance_ratio=1.122375 max_load_reduction=112683 max_load_reduction_pct=0.859703 moved_to_echo=67410 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2021, 1199, 1656, 0, 791, 0, 1769, 1631, 1207, 0, 1630, 1335, 1425, 936, 3, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 1336, 836, 652, 8, 838, 263, 1863, 641, 321, 2270, 1700, 1459, 257, 1157, 1409, 474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 2, 2388, 1214, 526, 1826, 732, 961, 488, 4, 1154, 2128, 0, 1374, 1595, 1276, 638, 43, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 923, 577, 673, 1266, 55, 432, 27, 887, 6, 358, 89, 579, 1396, 2499, 737, 811, 2349, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1138, 3, 0, 1773, 1294, 0, 1340, 2078, 1035, 1916, 1699, 831, 1722, 58, 1460, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1702, 414, 39, 8, 2122, 978, 863, 27, 1819, 505, 991, 944, 825, 1531, 841, 0, 1095, 780, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 983, 1344, 40, 1408, 43, 931, 1434, 1953, 264, 1291, 422, 1327, 1037, 1869, 1603, 1494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2579, 1158, 386, 823, 251, 898, 1302, 1128, 1032, 1888, 857, 644, 1639, 25, 772, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16364, 16133, 16437, 14975, 17981, 16329, 17443, 15410] avg=16384.0000 max=17981 max_rank=4 min=14975 min_rank=3 imbalance_ratio=1.097473 max_load_reduction=113091 max_load_reduction_pct=0.862816 moved_to_echo=67163 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2012, 1503, 1393, 0, 941, 0, 1431, 1906, 1225, 0, 2091, 1056, 907, 956, 0, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 622, 1354, 887, 837, 60, 1023, 264, 2048, 948, 201, 2794, 1274, 1115, 373, 1210, 772, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 4, 1909, 978, 340, 1918, 1172, 540, 339, 31, 1745, 1737, 0, 1240, 1311, 1489, 615, 44, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 684, 1180, 1281, 23, 180, 19, 1433, 1, 634, 123, 1729, 878, 2060, 1022, 1195, 1909, 969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 802, 3, 0, 957, 1371, 0, 1582, 1389, 1179, 1289, 1276, 1363, 1749, 81, 1859, 1521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1268, 453, 48, 2, 1860, 870, 1294, 17, 2095, 307, 1273, 925, 620, 1734, 985, 3, 2006, 796, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1198, 1077, 43, 1334, 25, 1112, 1256, 1116, 395, 1441, 699, 969, 623, 591, 1545, 1843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1990, 1336, 906, 1178, 458, 1818, 1270, 1266, 1269, 1874, 1321, 595, 1648, 54, 696, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16182, 16283, 15604, 16417, 16421, 17181, 15267, 17717] avg=16384.0000 max=17717 max_rank=7 min=15267 min_rank=6 imbalance_ratio=1.081360 max_load_reduction=113355 max_load_reduction_pct=0.864830 moved_to_echo=66586 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[5888, 10624, 8960, 6144, 256, 7936, 2304, 15616, 6272, 1792, 19584, 12160, 8704, 2560, 10368, 9344, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[15104, 8832, 4992, 9088, 3072, 10368, 11136, 10112, 9216, 12800, 9984, 6528, 14080, 512, 5632, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[16000, 11392, 11264, 128, 7552, 128, 11904, 14336, 10240, 0, 15104, 8960, 9344, 8576, 128, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[8704, 128, 128, 11776, 10368, 0, 12544, 10112, 8576, 11264, 12288, 8832, 12800, 512, 12800, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[9856, 4736, 512, 256, 15872, 8064, 8960, 384, 13824, 3200, 7808, 6912, 5632, 12032, 8064, 128, 12928, 7168, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[8960, 7680, 256, 13696, 512, 7936, 11136, 11264, 2560, 11264, 4992, 9728, 7424, 9856, 12288, 12672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1152, 128, 16768, 6272, 3968, 16512, 8576, 7040, 3456, 384, 12672, 17024, 128, 11008, 12288, 9216, 4864, 512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[9344, 4608, 7552, 12160, 384, 2176, 384, 10496, 128, 5888, 1408, 8448, 9600, 15616, 8576, 8832, 16896, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 355, 65, 1, 4, 0, 2, 0, 0, 3738, 0, 0, 0, 4095, 0, 0, 4095, 1, 0, 248, 12, 3489, 1, 0, 237, 0, 2638, 4068, 8, 0, 14, 4095, 0, 872, 5, 4070, 0, 0, 4084, 0, 1, 3063, 0, 3127, 0, 4057, 27, 1339, 3, 1, 4096, 0, 4052, 4095, 0, 1, 165, 3546, 1, 0, 183, 0, 3761, 0, 0, 0, 900, 1, 0, 2916, 0, 4043, 3729, 0, 0, 0, 1, 0, 76, 343, 0, 4082, 0, 1, 3, 14, 0, 0, 0, 0, 0, 4094, 1642, 0, 0, 1, 2, 4095, 388, 3356, 4095, 0, 3082, 3883, 1, 37, 0, 1, 0, 4096, 4095, 1, 1, 152, 0, 3439, 0, 0, 2143, 3514, 0, 1039, 0, 0, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 82, 494, 0, 0, 0, 1, 0, 1, 3328, 0, 0, 1, 4095, 0, 0, 4095, 0, 0, 3266, 1, 4087, 0, 0, 15, 0, 3998, 3961, 10, 1, 1, 4093, 1, 1282, 0, 4090, 0, 0, 3598, 0, 1, 2233, 0, 1537, 0, 4087, 61, 1224, 1, 0, 4095, 0, 1271, 4095, 0, 0, 3, 3816, 1, 0, 174, 0, 4049, 1, 0, 0, 3053, 11, 0, 3155, 0, 2812, 525, 0, 0, 0, 0, 1, 21, 39, 2, 3643, 1, 0, 0, 815, 0, 1, 0, 0, 0, 4095, 3859, 0, 0, 1, 0, 4082, 0, 3230, 4093, 0, 3001, 2701, 0, 23, 0, 0, 0, 4093, 4095, 0, 0, 3245, 0, 3958, 1, 1, 2057, 3198, 0, 13, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 556, 183, 0, 0, 0, 0, 0, 8, 4046, 0, 0, 0, 4089, 0, 0, 4095, 1, 0, 731, 9, 3354, 0, 1, 348, 1, 3544, 3491, 99, 0, 2, 4095, 0, 694, 33, 4093, 1, 0, 4062, 0, 0, 3225, 1, 3164, 0, 4063, 1, 252, 1, 1, 4095, 0, 4079, 4095, 0, 1, 14, 3849, 0, 0, 108, 0, 3679, 0, 0, 0, 380, 3, 1, 3303, 2, 3987, 3887, 0, 0, 0, 0, 0, 14, 792, 16, 4093, 1, 0, 1, 36, 0, 0, 4, 0, 1, 4093, 2707, 0, 1, 0, 0, 4090, 245, 3535, 4088, 0, 3318, 4029, 0, 6, 10, 0, 1, 4094, 4095, 0, 1, 377, 1, 2952, 0, 0, 1202, 3319, 2, 141, 0, 0, 0, 4079] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 489, 80, 0, 5, 1, 0, 2, 1, 3772, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1141, 0, 3989, 2, 2, 35, 0, 2838, 4046, 39, 0, 4, 4094, 0, 887, 8, 4092, 1, 0, 4096, 0, 1, 2705, 0, 3401, 0, 3995, 0, 1046, 0, 0, 4096, 0, 4050, 4091, 1, 0, 144, 3573, 0, 0, 125, 2, 3974, 2, 0, 0, 375, 16, 0, 3223, 0, 4071, 3285, 0, 0, 2, 1, 0, 14, 306, 4, 4096, 0, 0, 0, 143, 2, 0, 0, 2, 0, 4096, 3130, 2, 2, 0, 0, 4094, 371, 3021, 4093, 0, 3651, 4032, 0, 29, 0, 0, 0, 4093, 4093, 0, 0, 109, 0, 1994, 2, 2, 1492, 4025, 0, 85, 3, 1, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 875, 103, 1, 10, 0, 0, 0, 0, 3744, 1, 0, 1, 4095, 1, 0, 4095, 1, 0, 747, 0, 3357, 0, 0, 275, 0, 1180, 3736, 30, 0, 0, 4095, 0, 1319, 12, 4096, 1, 0, 4088, 0, 1, 2670, 1, 3418, 1, 3907, 6, 966, 0, 1, 4095, 13, 4007, 4094, 0, 1, 124, 3805, 0, 1, 82, 1, 3916, 0, 1, 1, 833, 25, 1, 3332, 0, 4074, 3666, 1, 1, 0, 11, 1, 58, 174, 39, 4095, 1, 1, 0, 8, 0, 1, 0, 0, 1, 4095, 2556, 0, 0, 1, 0, 4095, 111, 3980, 4091, 1, 3835, 4045, 0, 14, 0, 0, 0, 4095, 4095, 0, 1, 217, 0, 1665, 0, 0, 2947, 3712, 1, 125, 0, 1, 1, 4089] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 370, 54, 0, 1, 1, 0, 0, 0, 2270, 0, 0, 0, 4095, 1, 0, 4095, 0, 0, 1125, 0, 2058, 5, 1, 127, 0, 2863, 3690, 7, 1, 5, 4094, 0, 1590, 0, 4094, 0, 0, 3792, 0, 0, 3663, 1, 3027, 0, 4079, 9, 2025, 1, 0, 4096, 0, 4036, 4095, 0, 1, 4, 3783, 3, 0, 762, 1, 3982, 8, 0, 0, 1377, 21, 0, 2131, 0, 3779, 3628, 0, 0, 0, 0, 0, 1157, 261, 0, 4095, 0, 0, 12, 11, 0, 0, 0, 0, 0, 4095, 1579, 2, 0, 0, 0, 4092, 103, 3953, 4092, 0, 3914, 3642, 0, 359, 0, 0, 1, 4095, 4074, 0, 0, 169, 0, 1886, 1, 0, 2732, 2937, 0, 865, 0, 0, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 4, 0, 329, 106, 0, 0, 0, 3, 0, 30, 2514, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1687, 7, 3672, 1, 0, 247, 0, 3827, 3833, 89, 0, 18, 4094, 1, 2120, 0, 4076, 0, 0, 4083, 3, 0, 3416, 0, 2237, 1, 4094, 154, 800, 1, 0, 4095, 1, 3382, 4096, 1, 0, 2, 4050, 0, 0, 70, 0, 3737, 0, 0, 0, 1824, 2, 0, 2538, 2, 3367, 3734, 1, 0, 0, 0, 0, 705, 163, 5, 4087, 1, 1, 3, 67, 0, 0, 0, 0, 0, 4095, 4033, 0, 0, 0, 0, 4096, 49, 3904, 4092, 0, 3151, 2876, 0, 176, 0, 1, 0, 4092, 4058, 0, 0, 998, 1, 1855, 0, 1, 1638, 2229, 0, 59, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 662, 568, 0, 8, 1, 0, 2, 0, 3394, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 2372, 0, 2782, 7, 0, 225, 0, 3514, 4078, 46, 0, 6, 4092, 2, 617, 4, 4096, 0, 0, 4026, 0, 0, 3422, 0, 2337, 0, 3994, 45, 1590, 2, 0, 4096, 3, 3428, 4093, 0, 0, 51, 3696, 0, 0, 204, 0, 4045, 15, 1, 0, 1118, 2, 0, 2327, 1, 4070, 2714, 2, 0, 1, 1, 0, 426, 129, 2, 4082, 0, 2, 0, 176, 0, 0, 0, 0, 0, 4094, 2486, 0, 0, 0, 0, 4095, 140, 4077, 4083, 0, 3631, 3696, 0, 217, 4, 0, 2, 4095, 4096, 0, 0, 660, 2, 1563, 5, 0, 2158, 2761, 1, 348, 0, 0, 2, 4088] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 489, 80, 0, 5, 1, 0, 2, 1, 3772, 0, 0, 0, 4096, 4093, 4094, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1141, 0, 3989, 2, 2, 35, 0, 2838, 4046, 39, 0, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 887, 8, 1354, 1, 0, 4096, 0, 1, 2705, 0, 3401, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 0, 0, 3512, 0, 4050, 4091, 1, 0, 144, 3573, 0, 0, 125, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 2, 0, 0, 375, 16, 0, 3223, 0, 4071, 3285, 0, 0, 2, 1, 0, 700, 584, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 306, 4, 4096, 0, 0, 0, 143, 2, 0, 0, 2, 0, 4096, 3130, 2, 2712, 2038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1315, 371, 3021, 4093, 0, 3651, 4032, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 109, 0, 1994, 2, 2, 1492, 4025, 0, 85, 3, 1, 0, 3546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16700, 16563, 16452, 16544, 16402, 16545, 16514, 15352] avg=16384.0000 max=16700 max_rank=0 min=15352 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=114372 max_load_reduction_pct=0.872589 moved_to_echo=64813 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 355, 65, 1, 4, 0, 2, 0, 0, 3738, 0, 0, 0, 4095, 4096, 4095, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 248, 12, 3489, 1, 0, 237, 0, 2638, 4068, 8, 0, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 872, 5, 1347, 0, 0, 4084, 0, 1, 3063, 0, 3127, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 1339, 3, 1, 3512, 0, 4052, 4095, 0, 1, 165, 3546, 1, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3761, 0, 0, 0, 900, 1, 0, 2916, 0, 4043, 3729, 0, 0, 0, 1, 0, 696, 584, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 343, 0, 4082, 0, 1, 3, 14, 0, 0, 0, 0, 0, 4094, 1642, 0, 2713, 2027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1315, 388, 3356, 4095, 0, 3082, 3883, 1, 37, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 152, 0, 3439, 0, 0, 2143, 3514, 0, 1039, 0, 0, 0, 3547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16518, 15172, 16570, 16925, 16800, 14995, 16161, 17931] avg=16384.0000 max=17931 max_rank=7 min=14995 min_rank=5 imbalance_ratio=1.094421 max_load_reduction=113141 max_load_reduction_pct=0.863197 moved_to_echo=65887 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 370, 54, 0, 1, 1, 0, 0, 0, 2270, 0, 0, 0, 4095, 4095, 4094, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 1125, 0, 2058, 5, 1, 127, 0, 2863, 3690, 7, 1, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1590, 0, 1354, 0, 0, 3792, 0, 0, 3663, 1, 3027, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2025, 1, 0, 3512, 0, 4036, 4095, 0, 1, 4, 3783, 3, 0, 762, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 8, 0, 0, 1377, 21, 0, 2131, 0, 3779, 3628, 0, 0, 0, 0, 0, 701, 584, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1157, 261, 0, 4095, 0, 0, 12, 11, 0, 0, 0, 0, 0, 4095, 1579, 2, 2711, 2039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1314, 103, 3953, 4092, 0, 3914, 3642, 0, 359, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 169, 0, 1886, 1, 0, 2732, 2937, 0, 865, 0, 0, 0, 3550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15047, 14348, 17511, 18232, 16380, 15962, 17378, 16214] avg=16384.0000 max=18232 max_rank=3 min=14348 min_rank=1 imbalance_ratio=1.112793 max_load_reduction=112840 max_load_reduction_pct=0.860901 moved_to_echo=65934 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 875, 103, 1, 10, 0, 0, 0, 0, 3744, 1, 0, 1, 4095, 4095, 4095, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 747, 0, 3357, 0, 0, 275, 0, 1180, 3736, 30, 0, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1319, 12, 1355, 1, 0, 4088, 0, 1, 2670, 1, 3418, 1, 3907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 966, 0, 1, 3511, 13, 4007, 4094, 0, 1, 124, 3805, 0, 1, 82, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3916, 0, 1, 1, 833, 25, 1, 3332, 0, 4074, 3666, 1, 1, 0, 11, 1, 701, 584, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 174, 39, 4095, 1, 1, 0, 8, 0, 1, 0, 0, 1, 4095, 2556, 0, 2713, 2040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1315, 111, 3980, 4091, 1, 3835, 4045, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 217, 0, 1665, 0, 0, 2947, 3712, 1, 125, 0, 1, 1, 3545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17088, 13797, 16773, 16612, 17317, 15782, 17393, 16310] avg=16384.0000 max=17393 max_rank=6 min=13797 min_rank=1 imbalance_ratio=1.061584 max_load_reduction=113679 max_load_reduction_pct=0.867302 moved_to_echo=66802 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 82, 494, 0, 0, 0, 1, 0, 1, 3328, 0, 0, 1, 4095, 4093, 4093, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3266, 1, 4087, 0, 0, 15, 0, 3998, 3961, 10, 1, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1282, 0, 1343, 0, 0, 3598, 0, 1, 2233, 0, 1537, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 1224, 1, 0, 3509, 0, 1271, 4095, 0, 0, 3, 3816, 1, 0, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 1, 0, 0, 3053, 11, 0, 3155, 0, 2812, 525, 0, 0, 0, 0, 1, 704, 586, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 39, 2, 3643, 1, 0, 0, 815, 0, 1, 0, 0, 0, 4095, 3859, 0, 2708, 2043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1302, 0, 3230, 4093, 0, 3001, 2701, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3245, 0, 3958, 1, 1, 2057, 3198, 0, 13, 0, 0, 0, 3542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16261, 19814, 14083, 14155, 15071, 17227, 14351, 20110] avg=16384.0000 max=20110 max_rank=7 min=14083 min_rank=2 imbalance_ratio=1.227417 max_load_reduction=110962 max_load_reduction_pct=0.846573 moved_to_echo=66759 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 556, 183, 0, 0, 0, 0, 0, 8, 4046, 0, 0, 0, 4089, 4094, 4095, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 731, 9, 3354, 0, 1, 348, 1, 3544, 3491, 99, 0, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 694, 33, 1354, 1, 0, 4062, 0, 0, 3225, 1, 3164, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 252, 1, 1, 3511, 0, 4079, 4095, 0, 1, 14, 3849, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3679, 0, 0, 0, 380, 3, 1, 3303, 2, 3987, 3887, 0, 0, 0, 0, 0, 700, 584, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 792, 16, 4093, 1, 0, 1, 36, 0, 0, 4, 0, 1, 4093, 2707, 0, 2710, 2039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1313, 245, 3535, 4088, 0, 3318, 4029, 0, 6, 10, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 377, 1, 2952, 0, 0, 1202, 3319, 2, 141, 0, 0, 0, 3536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17139, 16048, 16599, 15912, 16695, 16507, 16546, 15626] avg=16384.0000 max=17139 max_rank=0 min=15626 min_rank=7 imbalance_ratio=1.046082 max_load_reduction=113933 max_load_reduction_pct=0.869240 moved_to_echo=65374 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 662, 568, 0, 8, 1, 0, 2, 0, 3394, 0, 0, 0, 4095, 4095, 4092, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2372, 0, 2782, 7, 0, 225, 0, 3514, 4078, 46, 0, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 617, 4, 1355, 0, 0, 4026, 0, 0, 3422, 0, 2337, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 1590, 2, 0, 3512, 3, 3428, 4093, 0, 0, 51, 3696, 0, 0, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 15, 1, 0, 1118, 2, 0, 2327, 1, 4070, 2714, 2, 0, 1, 1, 0, 701, 584, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 426, 129, 2, 4082, 0, 2, 0, 176, 0, 0, 0, 0, 0, 4094, 2486, 0, 2713, 2040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1315, 140, 4077, 4083, 0, 3631, 3696, 0, 217, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 660, 2, 1563, 5, 0, 2158, 2761, 1, 348, 0, 0, 2, 3544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16984, 17495, 15763, 16624, 15751, 16150, 17165, 15140] avg=16384.0000 max=17495 max_rank=1 min=15140 min_rank=7 imbalance_ratio=1.067810 max_load_reduction=113577 max_load_reduction_pct=0.866524 moved_to_echo=64206 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 329, 106, 0, 0, 0, 3, 0, 30, 2514, 0, 0, 0, 4096, 4092, 4094, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1687, 7, 3672, 1, 0, 247, 0, 3827, 3833, 89, 0, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1, 2120, 0, 1348, 0, 0, 4083, 3, 0, 3416, 0, 2237, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 800, 1, 0, 3511, 1, 3382, 4096, 1, 0, 2, 4050, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3737, 0, 0, 0, 1824, 2, 0, 2538, 2, 3367, 3734, 1, 0, 0, 0, 0, 698, 584, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 705, 163, 5, 4087, 1, 1, 3, 67, 0, 0, 0, 0, 0, 4095, 4033, 0, 2714, 2030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1315, 49, 3904, 4092, 0, 3151, 2876, 0, 176, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 998, 1, 1855, 0, 1, 1638, 2229, 0, 59, 0, 0, 0, 3550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15335, 17835, 17321, 16068, 16656, 17904, 15564, 14389] avg=16384.0000 max=17904 max_rank=5 min=14389 min_rank=7 imbalance_ratio=1.092773 max_load_reduction=113168 max_load_reduction_pct=0.863403 moved_to_echo=64513 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 3840, 1664, 128, 128, 128, 128, 128, 128, 26880, 128, 0, 128, 32768, 32768, 32768, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 9472, 128, 10880, 128, 0, 31872, 128, 128, 24448, 128, 22272, 128, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[384, 9344, 128, 128, 28160, 128, 28416, 32768, 128, 128, 512, 30208, 128, 128, 1792, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 32768, 128, 0, 11392, 128, 26880, 128, 128, 1536, 128, 24448, 30976, 384, 128, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[31232, 128, 128, 128, 9984, 128, 128, 23040, 128, 30208, 25216, 128, 128, 128, 128, 128, 5632, 4736, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 10624, 1408, 29056, 32768, 128, 27648, 28928, 128, 896, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[2560, 2304, 128, 32384, 128, 128, 128, 1280, 128, 128, 128, 128, 128, 32768, 22016, 128, 21760, 16384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 6016, 128, 19328, 128, 128, 16384, 25728, 128, 2688, 128, 128, 128, 28416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 161, 0, 196, 0, 0, 1275, 0, 4093, 4094, 2, 0, 0, 0, 4095, 0, 0, 0, 0, 5, 3964, 4090, 0, 4096, 4089, 0, 0, 0, 4094, 57, 0, 1, 0, 4078, 16, 1, 4095, 7, 0, 0, 0, 4096, 0, 55, 0, 0, 3141, 4089, 2942, 0, 1, 3260, 1, 4096, 57, 1, 0, 2694, 4, 0, 0, 4062, 4092, 3, 0, 4096, 846, 0, 0, 0, 0, 0, 4096, 188, 0, 4096, 3739, 4096, 528, 3876, 4, 0, 4091, 205, 0, 3859, 0, 3486, 0, 0, 0, 0, 0, 0, 29, 0, 379, 0, 0, 4016, 0, 1980, 0, 0, 0, 0, 1, 3, 0, 130, 4086, 0, 0, 9, 0, 0, 0, 140, 0, 3978, 0, 3972, 29, 0, 10, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 990, 0, 1507, 0, 0, 2131, 0, 4095, 4093, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 31, 4057, 4091, 0, 4096, 4095, 0, 0, 3, 4095, 354, 0, 0, 0, 4076, 1, 2, 4095, 0, 0, 0, 0, 4095, 0, 4, 0, 0, 2848, 4095, 2933, 0, 0, 3533, 0, 4094, 0, 0, 0, 1804, 5, 0, 0, 4085, 4096, 1, 0, 4096, 288, 0, 0, 0, 0, 1, 4095, 1127, 0, 4096, 4018, 4096, 889, 4073, 0, 0, 4095, 176, 1, 1691, 0, 2185, 0, 1, 0, 0, 0, 1, 38, 0, 631, 0, 0, 4081, 1, 1458, 17, 0, 0, 0, 0, 1, 0, 118, 4090, 0, 0, 1, 0, 0, 0, 16, 0, 4093, 0, 4062, 32, 0, 3, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1292, 0, 463, 1, 1, 1693, 0, 4092, 4071, 0, 1, 1, 0, 4095, 0, 1, 0, 1, 6, 4068, 4095, 1, 4094, 4095, 1, 0, 0, 4095, 95, 0, 0, 0, 4066, 1, 0, 4095, 0, 0, 1, 0, 4095, 1, 8, 1, 0, 3039, 4087, 1075, 1, 0, 2081, 0, 4094, 111, 1, 1, 2574, 14, 1, 1, 4095, 4094, 0, 1, 4095, 921, 0, 1, 0, 0, 0, 4095, 453, 1, 4094, 4003, 4095, 821, 4083, 1, 1, 4091, 2142, 0, 2367, 1, 3301, 0, 0, 0, 1, 12, 1, 7, 0, 831, 0, 1, 4095, 0, 1306, 35, 1, 1, 1, 0, 0, 0, 113, 4090, 1, 1, 10, 1, 1, 1, 13, 1, 4025, 0, 4057, 95, 0, 4, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 1371, 0, 260, 0, 0, 917, 0, 4093, 3443, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4050, 4094, 0, 4096, 4094, 0, 0, 0, 4094, 234, 0, 1, 0, 4084, 3, 0, 4095, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 2702, 4088, 339, 0, 2, 1880, 2, 4094, 152, 0, 0, 1340, 1, 0, 0, 3847, 4096, 0, 0, 4096, 252, 0, 0, 0, 0, 1, 4096, 1094, 0, 4095, 3851, 4096, 1060, 3933, 0, 0, 4036, 2728, 1, 3370, 0, 3595, 0, 0, 0, 0, 0, 0, 8, 2, 1721, 0, 0, 4071, 0, 2293, 7, 0, 0, 0, 2, 2, 0, 727, 4083, 0, 0, 2, 0, 0, 0, 0, 0, 4043, 0, 3982, 130, 2, 4, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 598, 0, 41, 1, 1, 685, 0, 4094, 3323, 0, 1, 0, 0, 4095, 0, 1, 0, 0, 5, 4084, 4095, 1, 4095, 4095, 0, 0, 2, 4096, 505, 0, 0, 0, 4089, 0, 3, 4095, 0, 1, 1, 0, 4095, 1, 1, 1, 0, 2795, 4094, 53, 1, 0, 2692, 0, 4095, 51, 0, 0, 2194, 33, 0, 1, 3570, 4094, 0, 0, 4095, 100, 0, 0, 0, 0, 0, 4095, 1022, 0, 4095, 3983, 4094, 715, 4092, 0, 1, 4064, 3189, 0, 3797, 1, 3052, 0, 0, 2, 0, 0, 1, 165, 0, 2967, 1, 1, 4090, 0, 1781, 0, 1, 1, 1, 0, 0, 0, 591, 4093, 1, 1, 0, 1, 0, 1, 34, 1, 3311, 0, 3804, 44, 0, 5, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 1669, 0, 92, 0, 0, 852, 1, 4089, 4036, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4036, 4063, 0, 4095, 4090, 0, 0, 0, 4071, 184, 0, 2, 0, 4092, 53, 0, 4095, 0, 0, 0, 0, 4095, 0, 39, 0, 1, 2405, 4095, 797, 1, 2, 3353, 0, 4096, 0, 0, 0, 2715, 74, 0, 0, 3995, 4096, 0, 1, 4095, 264, 0, 1, 1, 0, 26, 4095, 862, 0, 4095, 3170, 4095, 801, 4026, 14, 0, 4085, 1355, 0, 3824, 0, 3271, 0, 0, 0, 0, 1, 0, 16, 0, 1923, 0, 0, 3987, 1, 1123, 5, 1, 0, 1, 0, 3, 1, 491, 4064, 1, 1, 30, 0, 0, 1, 0, 1, 3806, 0, 4002, 117, 0, 26, 5, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 1726, 0, 881, 0, 0, 2864, 0, 4093, 4056, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 35, 4061, 4092, 0, 4096, 4094, 0, 0, 0, 4094, 287, 0, 1, 0, 4069, 9, 0, 4095, 1, 0, 0, 0, 4096, 0, 13, 0, 0, 3286, 4093, 2026, 0, 3, 2706, 0, 4096, 8, 1, 0, 2246, 84, 0, 0, 4080, 4093, 0, 0, 4092, 500, 0, 0, 0, 2, 2, 4096, 1568, 0, 4095, 4054, 4096, 643, 4094, 0, 0, 4006, 1357, 0, 1978, 0, 516, 0, 0, 0, 0, 0, 0, 91, 2, 1062, 1, 0, 4094, 0, 790, 7, 0, 0, 0, 5, 0, 0, 177, 4096, 0, 0, 2, 0, 0, 0, 58, 0, 4077, 0, 4012, 7, 0, 3, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 505, 0, 1336, 1, 1, 627, 0, 4092, 4093, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 20, 4060, 4095, 1, 4095, 4094, 0, 1, 0, 4095, 66, 1, 2, 0, 3875, 0, 0, 4095, 0, 0, 1, 1, 4095, 1, 1, 1, 1, 3504, 4095, 803, 1, 0, 2245, 0, 4095, 0, 0, 1, 3183, 184, 0, 1, 4061, 4095, 0, 0, 4091, 640, 0, 0, 0, 0, 0, 4095, 1222, 1, 4095, 4083, 4095, 1076, 4066, 0, 1, 4007, 352, 0, 3009, 1, 3440, 0, 0, 0, 1, 0, 0, 40, 0, 1543, 0, 1, 4067, 0, 488, 4, 1, 1, 1, 4, 0, 0, 766, 4075, 1, 1, 9, 1, 0, 1, 15, 1, 4062, 0, 4054, 10, 0, 11, 0, 20] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 161, 0, 196, 0, 0, 1275, 0, 4093, 4094, 2, 0, 0, 0, 4095, 1197, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3964, 4090, 0, 0, 3914, 0, 0, 0, 4094, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4078, 16, 1, 4095, 7, 0, 0, 0, 4096, 0, 55, 0, 0, 3141, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 2942, 0, 1, 3260, 1, 0, 57, 1, 0, 2694, 4, 0, 0, 4062, 1784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 846, 0, 0, 0, 0, 0, 0, 188, 0, 4096, 3739, 2092, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3876, 4, 0, 4091, 205, 0, 3859, 0, 3486, 0, 0, 0, 0, 0, 0, 29, 783, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 4016, 0, 1980, 0, 0, 0, 0, 1, 3, 0, 130, 4086, 3937, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 140, 0, 3978, 0, 3972, 29, 0, 10, 1, 0, 4096, 4096, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15191, 16124, 16633, 18895, 15588, 16508, 15643, 16490] avg=16384.0000 max=18895 max_rank=3 min=15191 min_rank=0 imbalance_ratio=1.153259 max_load_reduction=112177 max_load_reduction_pct=0.855843 moved_to_echo=64229 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1371, 0, 260, 0, 0, 917, 0, 4093, 3443, 1, 0, 0, 0, 4094, 1198, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 4094, 0, 0, 3919, 0, 0, 0, 4094, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4084, 3, 0, 4095, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 2702, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 339, 0, 2, 1880, 2, 0, 152, 0, 0, 1340, 1, 0, 0, 3847, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 252, 0, 0, 0, 0, 1, 0, 1094, 0, 4095, 3851, 2092, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 0, 0, 4036, 2728, 1, 3370, 0, 3595, 0, 0, 0, 0, 0, 0, 8, 783, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1721, 0, 0, 4071, 0, 2293, 7, 0, 0, 0, 2, 2, 0, 727, 4083, 3935, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4043, 0, 3982, 130, 2, 4, 0, 6, 4096, 4096, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15456, 16391, 16143, 13437, 16541, 18629, 17955, 16520] avg=16384.0000 max=18629 max_rank=5 min=13437 min_rank=3 imbalance_ratio=1.137024 max_load_reduction=112443 max_load_reduction_pct=0.857872 moved_to_echo=69645 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 1726, 0, 881, 0, 0, 2864, 0, 4093, 4056, 2, 0, 0, 0, 4094, 1198, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4061, 4092, 0, 0, 3919, 0, 0, 0, 4094, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4069, 9, 0, 4095, 1, 0, 0, 0, 4096, 0, 13, 0, 0, 3286, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2026, 0, 3, 2706, 0, 0, 8, 1, 0, 2246, 84, 0, 0, 4080, 1784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 500, 0, 0, 0, 2, 2, 0, 1568, 0, 4095, 4054, 2092, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4006, 1357, 0, 1978, 0, 516, 0, 0, 0, 0, 0, 0, 91, 783, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1062, 1, 0, 4094, 0, 790, 7, 0, 0, 0, 5, 0, 0, 177, 4096, 3937, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 58, 0, 4077, 0, 4012, 7, 0, 3, 0, 6, 4096, 4096, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18994, 16488, 16713, 17031, 17048, 13000, 15282, 16516] avg=16384.0000 max=18994 max_rank=0 min=13000 min_rank=5 imbalance_ratio=1.159302 max_load_reduction=112078 max_load_reduction_pct=0.855087 moved_to_echo=61846 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 990, 0, 1507, 0, 0, 2131, 0, 4095, 4093, 0, 0, 0, 0, 4095, 1198, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4057, 4091, 0, 0, 3920, 0, 0, 3, 4095, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 1, 2, 4095, 0, 0, 0, 0, 4095, 0, 4, 0, 0, 2848, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2933, 0, 0, 3533, 0, 0, 0, 0, 0, 1804, 5, 0, 0, 4085, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 288, 0, 0, 0, 0, 1, 0, 1127, 0, 4096, 4018, 2092, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 4095, 176, 1, 1691, 0, 2185, 0, 1, 0, 0, 0, 1, 38, 783, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 631, 0, 0, 4081, 1, 1458, 17, 0, 0, 0, 0, 1, 0, 118, 4090, 3935, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 16, 0, 4093, 0, 4062, 32, 0, 3, 1, 0, 4095, 4096, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18187, 16551, 16264, 18241, 16608, 13219, 15444, 16558] avg=16384.0000 max=18241 max_rank=3 min=13219 min_rank=5 imbalance_ratio=1.113342 max_load_reduction=112831 max_load_reduction_pct=0.860832 moved_to_echo=61829 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1292, 0, 463, 1, 1, 1693, 0, 4092, 4071, 0, 1, 1, 0, 4095, 1198, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 6, 4068, 4095, 1, 0, 3920, 1, 0, 0, 4095, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 1, 0, 4095, 0, 0, 1, 0, 4095, 1, 8, 1, 0, 3039, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1075, 1, 0, 2081, 0, 0, 111, 1, 1, 2574, 14, 1, 1, 4095, 1785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 921, 0, 1, 0, 0, 0, 0, 453, 1, 4094, 4003, 2093, 821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 1, 1, 4091, 2142, 0, 2367, 1, 3301, 0, 0, 0, 1, 12, 1, 7, 783, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 831, 0, 1, 4095, 0, 1306, 35, 1, 1, 1, 0, 0, 0, 113, 4090, 3935, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 10, 1, 1, 1, 13, 1, 4025, 0, 4057, 95, 0, 4, 0, 1, 4095, 4094, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16985, 16283, 16449, 15827, 16483, 16966, 15520, 16559] avg=16384.0000 max=16985 max_rank=0 min=15520 min_rank=6 imbalance_ratio=1.036682 max_load_reduction=114087 max_load_reduction_pct=0.870415 moved_to_echo=65528 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 505, 0, 1336, 1, 1, 627, 0, 4092, 4093, 0, 1, 0, 0, 4095, 1198, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4060, 4095, 1, 0, 3919, 0, 1, 0, 4095, 66, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3875, 0, 0, 4095, 0, 0, 1, 1, 4095, 1, 1, 1, 1, 3504, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 803, 1, 0, 2245, 0, 0, 0, 0, 1, 3183, 184, 0, 1, 4061, 1785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 640, 0, 0, 0, 0, 0, 0, 1222, 1, 4095, 4083, 2093, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 1, 4007, 352, 0, 3009, 1, 3440, 0, 0, 0, 1, 0, 0, 40, 783, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1543, 0, 1, 4067, 0, 488, 4, 1, 1, 1, 4, 0, 0, 766, 4075, 3936, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 9, 1, 0, 1, 15, 1, 4062, 0, 4054, 10, 0, 11, 0, 20, 4095, 4095, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16026, 16258, 16719, 16359, 17301, 15875, 15999, 16535] avg=16384.0000 max=17301 max_rank=4 min=15875 min_rank=5 imbalance_ratio=1.055969 max_load_reduction=113771 max_load_reduction_pct=0.868004 moved_to_echo=65710 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 598, 0, 41, 1, 1, 685, 0, 4094, 3323, 0, 1, 0, 0, 4095, 1203, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 4084, 4095, 1, 0, 3918, 0, 0, 2, 4096, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 3, 4095, 0, 1, 1, 0, 4095, 1, 1, 1, 0, 2795, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 53, 1, 0, 2692, 0, 0, 51, 0, 0, 2194, 33, 0, 1, 3570, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 100, 0, 0, 0, 0, 0, 0, 1022, 0, 4095, 3983, 2084, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 4064, 3189, 0, 3797, 1, 3052, 0, 0, 2, 0, 0, 1, 165, 784, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2967, 1, 1, 4090, 0, 1781, 0, 1, 1, 1, 0, 0, 0, 591, 4093, 3937, 1116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 1, 34, 1, 3311, 0, 3804, 44, 0, 5, 0, 1, 4095, 4095, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14123, 16707, 16227, 14464, 16094, 19325, 18580, 15552] avg=16384.0000 max=19325 max_rank=5 min=14123 min_rank=0 imbalance_ratio=1.179504 max_load_reduction=111747 max_load_reduction_pct=0.852562 moved_to_echo=69551 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1669, 0, 92, 0, 0, 852, 1, 4089, 4036, 0, 0, 0, 0, 4095, 1198, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4036, 4063, 0, 0, 3915, 0, 0, 0, 4071, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 53, 0, 4095, 0, 0, 0, 0, 4095, 0, 39, 0, 1, 2405, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 797, 1, 2, 3353, 0, 0, 0, 0, 0, 2715, 74, 0, 0, 3995, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 264, 0, 1, 1, 0, 26, 0, 862, 0, 4095, 3170, 2093, 801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 14, 0, 4085, 1355, 0, 3824, 0, 3271, 0, 0, 0, 0, 1, 0, 16, 783, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1923, 0, 0, 3987, 1, 1123, 5, 1, 0, 1, 0, 3, 1, 491, 4064, 3937, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 30, 0, 0, 1, 0, 1, 3806, 0, 4002, 117, 0, 26, 5, 3, 4095, 4095, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16110, 16270, 15924, 16818, 15409, 17550, 16649, 16342] avg=16384.0000 max=17550 max_rank=5 min=15409 min_rank=4 imbalance_ratio=1.071167 max_load_reduction=113522 max_load_reduction_pct=0.866104 moved_to_echo=65950 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 11008, 128, 128, 21760, 128, 0, 384, 128, 128, 18816, 512, 128, 128, 31872, 14336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 128, 32384, 32768, 128, 0, 31360, 128, 128, 128, 32768, 1792, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 32512, 128, 128, 32768, 128, 128, 128, 128, 32768, 128, 256, 128, 128, 23808, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 8320, 0, 4864, 128, 128, 11136, 128, 32768, 31232, 128, 128, 128, 0, 32768, 9600, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 3840, 0, 128, 128, 128, 128, 0, 7552, 128, 32768, 30976, 16768, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 384, 128, 31488, 0, 32000, 512, 128, 128, 128, 128, 32768, 32768, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 11136, 128, 128, 32512, 128, 11264, 128, 128, 128, 128, 128, 128, 128, 3200, 32768, 31616, 8960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32256, 128, 128, 32512, 11520, 128, 23936, 128, 22912, 0, 128, 128, 128, 128, 128, 512, 6272, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132864 slack_slots=7168 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 63, 4080, 0, 0, 1, 3294, 4095, 0, 0, 4030, 0, 1, 0, 1, 1, 0, 1, 0, 3889, 3774, 0, 2878, 15, 1, 1, 4093, 0, 0, 4085, 3989, 0, 217, 1, 0, 4095, 0, 583, 4094, 0, 5, 4, 0, 3, 1, 4095, 4095, 0, 1, 1, 3614, 0, 0, 0, 1, 1, 5, 2069, 1328, 0, 4095, 1, 0, 0, 13, 4094, 1, 1, 4087, 3509, 1, 1, 1, 4093, 0, 409, 9, 3971, 1, 4090, 4095, 0, 1, 4072, 0, 0, 0, 0, 69, 1, 2837, 931, 4086, 1, 3132, 1952, 0, 1, 3825, 0, 0, 1, 1, 4095, 3974, 1, 0, 0, 107, 73, 1, 0, 1, 8, 0, 0, 1, 0, 0, 4095, 0, 2824, 1, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 252, 4081, 0, 0, 0, 3394, 4095, 0, 0, 4051, 30, 0, 0, 0, 1, 8, 1, 0, 2870, 3998, 0, 3162, 109, 0, 0, 4096, 0, 0, 4094, 4067, 1, 578, 1, 1, 4057, 1, 298, 4095, 10, 2, 0, 0, 0, 0, 4095, 4095, 0, 1, 0, 3048, 0, 0, 1, 1, 0, 5, 2685, 889, 0, 4095, 1, 0, 0, 9, 4082, 1, 0, 4082, 3926, 0, 0, 1, 4094, 0, 6, 125, 4096, 0, 4043, 4095, 1, 0, 3982, 0, 1, 0, 0, 88, 1, 2484, 1898, 4071, 0, 2101, 1766, 2, 1, 3355, 1, 0, 0, 1, 4095, 3957, 0, 0, 0, 60, 317, 1, 0, 1, 90, 1, 0, 1, 1, 1, 4094, 0, 3871, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 208, 4058, 4, 0, 0, 2516, 4096, 0, 0, 4084, 8, 0, 0, 5, 0, 3, 0, 0, 3990, 3908, 0, 2605, 14, 0, 0, 4092, 0, 0, 4072, 3889, 2, 32, 0, 0, 4093, 3, 696, 4096, 6, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2935, 8, 3, 0, 0, 0, 5, 3356, 1732, 0, 4096, 0, 0, 0, 0, 4026, 0, 0, 4092, 3741, 0, 0, 0, 3978, 2, 20, 447, 4079, 0, 4071, 4094, 0, 0, 4046, 0, 6, 0, 0, 247, 0, 3142, 267, 4089, 0, 2601, 3414, 0, 0, 2828, 0, 5, 0, 0, 4096, 3602, 0, 0, 0, 36, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2992, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 1, 1, 293, 4093, 0, 0, 0, 2926, 4095, 0, 0, 4070, 2, 1, 0, 1, 1, 0, 1, 0, 2262, 4089, 0, 2355, 96, 0, 1, 4095, 1, 0, 4093, 4028, 0, 1094, 1, 0, 4094, 0, 460, 4095, 0, 0, 0, 0, 1, 1, 4095, 4095, 0, 1, 1, 4040, 3, 0, 0, 1, 1, 2, 3945, 904, 0, 4095, 0, 0, 0, 14, 4095, 1, 1, 4091, 3968, 1, 1, 1, 4031, 0, 3, 15, 4095, 0, 4083, 4095, 0, 0, 4062, 0, 0, 0, 0, 58, 0, 3790, 608, 4077, 1, 2250, 2206, 0, 1, 1863, 0, 0, 1, 1, 4095, 3712, 1, 0, 0, 30, 215, 1, 0, 0, 1, 0, 0, 1, 1, 0, 4095, 0, 4069, 1, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 209, 4084, 3, 0, 0, 2816, 4096, 0, 0, 4073, 8, 0, 0, 0, 0, 2, 0, 0, 3710, 3953, 0, 3601, 120, 0, 0, 4094, 0, 0, 4090, 3444, 1, 58, 0, 0, 4094, 2, 445, 4096, 5, 0, 0, 0, 1, 0, 4096, 4096, 1, 0, 1, 2947, 1, 7, 0, 0, 0, 3, 1804, 2064, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 4093, 3855, 0, 0, 0, 4051, 0, 16, 96, 3996, 0, 4079, 4095, 0, 0, 4043, 0, 4, 0, 0, 237, 0, 4015, 3130, 4076, 0, 2033, 1144, 0, 0, 3954, 0, 2, 0, 0, 4096, 3317, 0, 0, 0, 24, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1815, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 150, 4092, 3, 0, 0, 3221, 4096, 0, 0, 4044, 6, 0, 0, 1, 0, 7, 0, 0, 3705, 3840, 0, 2077, 19, 0, 0, 4095, 0, 0, 4016, 4048, 1, 772, 0, 0, 4094, 2, 220, 4096, 2, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 3986, 0, 0, 0, 0, 0, 13, 870, 1567, 0, 4096, 0, 0, 0, 4, 4079, 0, 0, 4096, 3553, 0, 0, 0, 4078, 0, 12, 144, 4092, 0, 4083, 4095, 1, 0, 4066, 0, 5, 0, 0, 249, 0, 3974, 981, 4089, 0, 2701, 3781, 1, 0, 4029, 0, 1, 0, 0, 4096, 3142, 0, 0, 1, 53, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1925, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 18, 4095, 0, 1, 0, 3280, 4095, 0, 0, 3963, 11, 1, 0, 6, 1, 0, 1, 1, 3583, 4009, 0, 2728, 17, 1, 1, 4089, 0, 0, 4028, 4009, 0, 64, 1, 0, 4095, 3, 387, 4095, 13, 2, 0, 0, 6, 1, 4095, 4095, 0, 1, 0, 2264, 0, 1, 0, 1, 0, 1, 2524, 2030, 0, 4095, 1, 0, 0, 16, 4092, 1, 0, 4081, 3555, 1, 1, 1, 4090, 3, 336, 89, 4048, 0, 4080, 4095, 0, 0, 4008, 0, 8, 0, 0, 50, 0, 3970, 546, 4080, 1, 1970, 3106, 0, 1, 2816, 0, 0, 1, 1, 4093, 3879, 1, 0, 0, 118, 463, 1, 0, 1, 4, 0, 0, 1, 0, 0, 4095, 0, 3651, 1, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 77, 4093, 4, 0, 5, 3053, 4096, 0, 0, 4091, 2, 0, 0, 4, 0, 3, 0, 0, 3875, 3980, 0, 1997, 12, 0, 0, 4094, 0, 0, 3935, 4041, 1, 409, 0, 0, 4090, 2, 210, 4096, 2, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 0, 3588, 0, 3, 0, 0, 0, 10, 3004, 1188, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 4093, 3766, 0, 0, 0, 4088, 2, 391, 12, 4048, 0, 4090, 4094, 0, 0, 4086, 0, 5, 0, 0, 56, 0, 4083, 1004, 4083, 0, 2758, 1387, 0, 0, 3988, 0, 4, 0, 0, 4096, 3548, 0, 0, 0, 50, 74, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4094, 0, 2819, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 208, 4058, 4, 0, 0, 2516, 4096, 0, 0, 4084, 8, 0, 634, 268, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3, 0, 0, 3990, 3908, 0, 2605, 14, 0, 0, 4092, 0, 0, 2128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 3889, 2, 32, 0, 0, 4093, 3, 696, 3479, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 887, 4096, 0, 0, 0, 2935, 8, 3, 0, 0, 0, 5, 3356, 1732, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 4092, 3741, 0, 0, 0, 3978, 2, 20, 447, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 3437, 0, 0, 0, 4046, 0, 6, 0, 0, 247, 0, 3142, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2601, 3414, 0, 0, 2828, 0, 5, 0, 0, 4096, 3602, 0, 0, 0, 36, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2992, 0, 0, 0, 4094, 4089, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15891, 16745, 16272, 17118, 16445, 15224, 17045, 16332] avg=16384.0000 max=17118 max_rank=3 min=15224 min_rank=5 imbalance_ratio=1.044800 max_load_reduction=113954 max_load_reduction_pct=0.869400 moved_to_echo=65046 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 63, 4080, 0, 0, 1, 3294, 4095, 0, 0, 4030, 0, 1, 637, 268, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 3889, 3774, 0, 2878, 15, 1, 1, 4093, 0, 0, 2128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 3989, 0, 217, 1, 0, 4095, 0, 583, 3477, 0, 5, 4, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 886, 4095, 0, 1, 1, 3614, 0, 0, 0, 1, 1, 5, 2069, 1328, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 13, 4094, 1, 1, 4087, 3509, 1, 1, 1, 4093, 0, 409, 9, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 1, 3453, 0, 0, 1, 4072, 0, 0, 0, 0, 69, 1, 2837, 931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3132, 1952, 0, 1, 3825, 0, 0, 1, 1, 4095, 3974, 1, 0, 0, 107, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 1, 0, 1, 8, 0, 0, 1, 0, 0, 4095, 0, 2824, 1, 0, 1, 4095, 4086, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16487, 16782, 16460, 16096, 16359, 15336, 17553, 15999] avg=16384.0000 max=17553 max_rank=6 min=15336 min_rank=5 imbalance_ratio=1.071350 max_load_reduction=113519 max_load_reduction_pct=0.866081 moved_to_echo=65247 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 209, 4084, 3, 0, 0, 2816, 4096, 0, 0, 4073, 8, 0, 636, 268, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3710, 3953, 0, 3601, 120, 0, 0, 4094, 0, 0, 2128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3444, 1, 58, 0, 0, 4094, 2, 445, 3479, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 887, 4096, 1, 0, 1, 2947, 1, 7, 0, 0, 0, 3, 1804, 2064, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4093, 3855, 0, 0, 0, 4051, 0, 16, 96, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 3443, 0, 0, 0, 4043, 0, 4, 0, 0, 237, 0, 4015, 3130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2033, 1144, 0, 0, 3954, 0, 2, 0, 0, 4096, 3317, 0, 0, 0, 24, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1815, 0, 0, 0, 4095, 4076, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16208, 17608, 15619, 15907, 16344, 18868, 15033, 15485] avg=16384.0000 max=18868 max_rank=5 min=15033 min_rank=6 imbalance_ratio=1.151611 max_load_reduction=112204 max_load_reduction_pct=0.856049 moved_to_echo=65730 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 1, 293, 4093, 0, 0, 0, 2926, 4095, 0, 0, 4070, 2, 1, 640, 271, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 2262, 4089, 0, 2355, 96, 0, 1, 4095, 1, 0, 2132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4028, 0, 1094, 1, 0, 4094, 0, 460, 3468, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 872, 4095, 0, 1, 1, 4040, 3, 0, 0, 1, 1, 2, 3945, 904, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4095, 1, 1, 4091, 3968, 1, 1, 1, 4031, 0, 3, 15, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3443, 0, 0, 0, 4062, 0, 0, 0, 0, 58, 0, 3790, 608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2250, 2206, 0, 1, 1863, 0, 0, 1, 1, 4095, 3712, 1, 0, 0, 30, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 215, 1, 0, 0, 1, 0, 0, 1, 1, 0, 4095, 0, 4069, 1, 0, 1, 4095, 4077, 820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 15034, 17240, 17960, 16366, 16056, 14627, 17377] avg=16384.0000 max=17960 max_rank=3 min=14627 min_rank=6 imbalance_ratio=1.096191 max_load_reduction=113112 max_load_reduction_pct=0.862976 moved_to_echo=64426 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 252, 4081, 0, 0, 0, 3394, 4095, 0, 0, 4051, 30, 0, 630, 268, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 1, 0, 2870, 3998, 0, 3162, 109, 0, 0, 4096, 0, 0, 2128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4067, 1, 578, 1, 1, 4057, 1, 298, 3478, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 886, 4095, 0, 1, 0, 3048, 0, 0, 1, 1, 0, 5, 2685, 889, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 9, 4082, 1, 0, 4082, 3926, 0, 0, 1, 4094, 0, 6, 125, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3413, 0, 1, 0, 3982, 0, 1, 0, 0, 88, 1, 2484, 1898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2101, 1766, 2, 1, 3355, 1, 0, 0, 1, 4095, 3957, 0, 0, 0, 60, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 1, 0, 1, 90, 1, 0, 1, 1, 1, 4094, 0, 3871, 0, 0, 0, 4095, 4071, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16816, 16373, 16588, 15706, 16466, 15964, 15802, 17357] avg=16384.0000 max=17357 max_rank=7 min=15706 min_rank=3 imbalance_ratio=1.059387 max_load_reduction=113715 max_load_reduction_pct=0.867577 moved_to_echo=65589 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 77, 4093, 4, 0, 5, 3053, 4096, 0, 0, 4091, 2, 0, 637, 268, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 0, 0, 3875, 3980, 0, 1997, 12, 0, 0, 4094, 0, 0, 2128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 4041, 1, 409, 0, 0, 4090, 2, 210, 3479, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 887, 4096, 0, 0, 0, 3588, 0, 3, 0, 0, 0, 10, 3004, 1188, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4093, 3766, 0, 0, 0, 4088, 2, 391, 12, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 3453, 0, 0, 0, 4086, 0, 5, 0, 0, 56, 0, 4083, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2758, 1387, 0, 0, 3988, 0, 4, 0, 0, 4096, 3548, 0, 0, 0, 50, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4094, 0, 2819, 0, 0, 0, 4094, 4083, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16341, 16093, 16171, 16872, 16581, 16735, 16294, 15985] avg=16384.0000 max=16872 max_rank=3 min=15985 min_rank=7 imbalance_ratio=1.029785 max_load_reduction=114200 max_load_reduction_pct=0.871277 moved_to_echo=65595 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 150, 4092, 3, 0, 0, 3221, 4096, 0, 0, 4044, 6, 0, 636, 268, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 3705, 3840, 0, 2077, 19, 0, 0, 4095, 0, 0, 2128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 4048, 1, 772, 0, 0, 4094, 2, 220, 3479, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 887, 4096, 0, 0, 0, 3986, 0, 0, 0, 0, 0, 13, 870, 1567, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4079, 0, 0, 4096, 3553, 0, 0, 0, 4078, 0, 12, 144, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3447, 0, 1, 0, 4066, 0, 5, 0, 0, 249, 0, 3974, 981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2701, 3781, 1, 0, 4029, 0, 1, 0, 0, 4096, 3142, 0, 0, 1, 53, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1925, 0, 0, 0, 4095, 4089, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16531, 15872, 16635, 15515, 16105, 16815, 18268, 15331] avg=16384.0000 max=18268 max_rank=6 min=15331 min_rank=7 imbalance_ratio=1.114990 max_load_reduction=112804 max_load_reduction_pct=0.860626 moved_to_echo=66519 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 18, 4095, 0, 1, 0, 3280, 4095, 0, 0, 3963, 11, 1, 636, 268, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 1, 1, 3583, 4009, 0, 2728, 17, 1, 1, 4089, 0, 0, 2128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 4009, 0, 64, 1, 0, 4095, 3, 387, 3478, 13, 2, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 886, 4095, 0, 1, 0, 2264, 0, 1, 0, 1, 0, 1, 2524, 2030, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 16, 4092, 1, 0, 4081, 3555, 1, 1, 1, 4090, 3, 336, 89, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 3444, 0, 0, 0, 4008, 0, 8, 0, 0, 50, 0, 3970, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1970, 3106, 0, 1, 2816, 0, 0, 1, 1, 4093, 3879, 1, 0, 0, 118, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 1, 0, 1, 4, 0, 0, 1, 0, 0, 4095, 0, 3651, 1, 1, 0, 4095, 4080, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16565, 16087, 15898, 16406, 16074, 16450, 17206] avg=16384.0000 max=17206 max_rank=7 min=15898 min_rank=3 imbalance_ratio=1.050171 max_load_reduction=113866 max_load_reduction_pct=0.868729 moved_to_echo=66136 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[7168, 32768, 128, 128, 128, 26496, 128, 128, 128, 128, 128, 128, 20352, 11776, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 19584, 18816, 128, 128, 26752, 128, 128, 128, 128, 32768, 29184, 128, 0, 128, 512, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32512, 128, 27648, 0, 128, 128, 32384, 0, 128, 0, 0, 1152, 128, 28416, 9472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[2304, 128, 0, 128, 128, 128, 0, 128, 128, 128, 32768, 0, 24064, 128, 128, 128, 32768, 32768, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32512, 31616, 128, 3328, 128, 128, 32768, 128, 3328, 27904, 128, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 128, 128, 27904, 31616, 0, 21504, 512, 128, 128, 32768, 128, 0, 17152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 1280, 32768, 128, 128, 128, 24576, 32768, 0, 0, 32512, 128, 128, 5120, 2176, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 0, 128, 32768, 128, 128, 32768, 29952, 128, 128, 128, 32512, 128, 1280, 1024, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 1, 0, 228, 1, 0, 1, 4095, 6, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 4088, 1, 1, 1, 4092, 4095, 4095, 1, 4095, 1, 4095, 1968, 1, 0, 0, 0, 3875, 4074, 0, 0, 4095, 4095, 1034, 4095, 1, 0, 0, 0, 0, 4, 4095, 1, 1, 0, 4095, 0, 0, 8, 2730, 4095, 0, 630, 4091, 2900, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 4083, 0, 1, 1, 1, 1, 4095, 1, 0, 4095, 1, 1, 170, 1, 0, 4095, 12, 0, 1, 0, 4095, 4094, 1, 1, 0, 3919, 1, 1, 4094, 1, 0, 0, 4035, 63, 4095, 0, 0, 0, 28, 0, 0, 0, 0, 3006, 1, 4095, 1, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[21, 1, 4062, 0, 1, 0, 103, 1, 0, 0, 4095, 24, 0, 1, 1, 0, 0, 1, 32, 0, 0, 0, 1, 1, 0, 4056, 0, 0, 1, 3805, 4095, 4088, 0, 4095, 1, 4095, 2050, 0, 0, 0, 0, 3829, 4012, 0, 1, 4095, 4095, 891, 4095, 1, 1, 0, 1, 0, 45, 4095, 1, 0, 0, 4095, 0, 0, 118, 2006, 4095, 0, 517, 4087, 3025, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4077, 1, 26, 1, 0, 0, 4095, 1, 0, 4095, 1, 1, 241, 1, 0, 4095, 3, 1, 0, 75, 4094, 4093, 0, 0, 0, 3932, 0, 1, 4095, 1, 1, 35, 3934, 256, 4095, 0, 0, 1, 132, 0, 0, 0, 1, 3740, 18, 4095, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4078, 0, 0, 0, 318, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 21, 0, 1, 0, 0, 0, 0, 4086, 0, 0, 0, 3892, 4096, 4091, 0, 4096, 0, 4095, 1088, 0, 0, 0, 1, 3855, 4046, 0, 0, 4096, 4096, 615, 4049, 0, 19, 0, 0, 0, 8, 4095, 0, 0, 0, 4096, 0, 1, 0, 3479, 4096, 1, 1295, 4096, 3136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 6, 0, 0, 0, 4096, 0, 6, 4095, 0, 0, 1377, 0, 0, 4096, 4, 0, 0, 0, 4095, 4095, 0, 2, 0, 3991, 0, 0, 4095, 0, 0, 73, 2661, 247, 4089, 0, 0, 0, 81, 0, 0, 0, 0, 2624, 5, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 1, 1748, 0, 1, 0, 4095, 2, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 4076, 0, 1, 1, 4046, 4095, 4095, 1, 4095, 1, 4095, 1261, 4, 1, 1, 0, 2952, 4092, 1, 0, 4095, 4094, 287, 4095, 1, 0, 0, 0, 0, 8, 4095, 1, 1, 0, 4095, 0, 0, 129, 3699, 4095, 0, 28, 4094, 2251, 1, 0, 1, 0, 2, 0, 1, 0, 1, 0, 4021, 0, 0, 1, 1, 1, 4093, 1, 0, 4095, 0, 1, 165, 1, 0, 4095, 4, 0, 1, 1, 4095, 4095, 0, 0, 0, 3958, 1, 1, 4094, 1, 0, 5, 4063, 36, 4095, 0, 1, 0, 45, 0, 0, 0, 0, 4079, 0, 4095, 0, 0, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 0, 108, 0, 2, 0, 4096, 7, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 3925, 4096, 4091, 0, 4096, 0, 4094, 2183, 10, 0, 0, 2, 3934, 4093, 0, 0, 4096, 4096, 1186, 4094, 0, 7, 0, 0, 0, 13, 4094, 0, 0, 0, 4096, 0, 0, 38, 2437, 4096, 2, 73, 4094, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 196, 0, 0, 4096, 2, 0, 0, 2, 4096, 4094, 0, 0, 0, 3930, 0, 0, 4096, 0, 0, 31, 3896, 51, 4088, 0, 0, 0, 2, 0, 6, 0, 0, 3071, 6, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[23, 0, 4066, 0, 0, 0, 227, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 3820, 4096, 4092, 0, 4096, 0, 4095, 991, 1, 0, 0, 1, 3523, 4081, 0, 0, 4095, 4096, 835, 4094, 0, 3, 0, 0, 0, 22, 4096, 0, 0, 0, 4096, 0, 1, 0, 3445, 4096, 0, 2354, 4096, 3173, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 17, 0, 0, 0, 4096, 0, 2, 4095, 0, 0, 213, 0, 0, 4096, 50, 0, 0, 2, 4095, 4094, 0, 0, 1, 3127, 0, 0, 4096, 0, 0, 26, 3943, 131, 4094, 0, 0, 0, 0, 0, 4, 0, 0, 2786, 3, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4084, 0, 0, 0, 185, 0, 0, 1, 4095, 24, 0, 0, 1, 0, 0, 1, 11, 1, 0, 0, 1, 1, 0, 4084, 0, 0, 1, 3983, 4095, 4095, 0, 4095, 1, 4095, 650, 0, 1, 1, 0, 4074, 4059, 0, 0, 4095, 4095, 305, 4095, 1, 0, 0, 0, 0, 14, 4095, 1, 1, 0, 4095, 0, 0, 12, 3400, 4095, 0, 1834, 4090, 3657, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 4079, 0, 132, 1, 1, 1, 4095, 1, 0, 4093, 1, 1, 161, 1, 0, 4095, 74, 0, 1, 0, 4095, 4095, 1, 0, 0, 3936, 1, 1, 4094, 1, 0, 8, 3352, 210, 4095, 0, 1, 0, 42, 0, 0, 0, 0, 2685, 3, 4095, 0, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[3, 0, 4073, 0, 0, 0, 237, 0, 0, 0, 4096, 13, 0, 0, 0, 1, 0, 0, 8, 0, 2, 0, 0, 0, 0, 4086, 0, 0, 0, 3651, 4096, 4093, 0, 4096, 0, 4095, 872, 1, 0, 0, 2, 3972, 4091, 0, 0, 4096, 4096, 967, 4090, 0, 8, 1, 0, 0, 14, 4094, 0, 0, 0, 4096, 0, 1, 0, 3221, 4096, 2, 1563, 4094, 2673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 6, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 138, 0, 0, 4096, 129, 0, 0, 0, 4094, 4092, 0, 0, 0, 3931, 0, 0, 4096, 0, 0, 17, 3979, 14, 4093, 0, 0, 2, 3, 0, 28, 0, 0, 3279, 7, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4078, 0, 0, 0, 318, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 4096, 3701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 1, 0, 0, 0, 0, 4086, 0, 0, 0, 3892, 4096, 4091, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1088, 0, 0, 0, 1, 3855, 4046, 0, 0, 2319, 0, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 19, 0, 0, 0, 8, 4095, 0, 0, 0, 4096, 0, 1, 0, 3479, 789, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 1295, 4096, 3136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 6, 4095, 0, 0, 1377, 0, 0, 4096, 4, 3307, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 2, 0, 3991, 0, 0, 586, 0, 0, 73, 2661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 4089, 0, 0, 0, 81, 0, 0, 0, 0, 2624, 5, 4096, 0, 0, 4096, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16299, 16385, 16019, 16738, 16745, 17382, 15503, 16001] avg=16384.0000 max=17382 max_rank=5 min=15503 min_rank=6 imbalance_ratio=1.060913 max_load_reduction=113690 max_load_reduction_pct=0.867386 moved_to_echo=65631 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 1, 0, 228, 1, 0, 1, 4095, 6, 1, 0, 1, 0, 4095, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 4088, 1, 1, 1, 4092, 4095, 4095, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 1968, 1, 0, 0, 0, 3875, 4074, 0, 0, 2319, 0, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4, 4095, 1, 1, 0, 4095, 0, 0, 8, 2730, 789, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 630, 4091, 2900, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 4083, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 4095, 1, 0, 4095, 1, 1, 170, 1, 0, 4095, 12, 3306, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 4094, 1, 1, 0, 3919, 1, 1, 586, 1, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 4095, 0, 0, 0, 28, 0, 0, 0, 0, 3006, 1, 4095, 1, 0, 4094, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16221, 16575, 17368, 16021, 15829, 16177, 16735, 16146] avg=16384.0000 max=17368 max_rank=2 min=15829 min_rank=4 imbalance_ratio=1.060059 max_load_reduction=113704 max_load_reduction_pct=0.867493 moved_to_echo=64887 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 0, 108, 0, 2, 0, 4096, 7, 0, 0, 0, 1, 4096, 3701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 3925, 4096, 4091, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2183, 10, 0, 0, 2, 3934, 4093, 0, 0, 2319, 0, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 7, 0, 0, 0, 13, 4094, 0, 0, 0, 4096, 0, 0, 38, 2437, 789, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 73, 4094, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 196, 0, 0, 4096, 2, 3308, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4094, 0, 0, 0, 3930, 0, 0, 586, 0, 0, 31, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 4088, 0, 0, 0, 2, 0, 6, 0, 0, 3071, 6, 4096, 0, 0, 4096, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16100, 16407, 17821, 15770, 15969, 16191, 16635, 16179] avg=16384.0000 max=17821 max_rank=2 min=15770 min_rank=3 imbalance_ratio=1.087708 max_load_reduction=113251 max_load_reduction_pct=0.864037 moved_to_echo=64974 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4073, 0, 0, 0, 237, 0, 0, 0, 4096, 13, 0, 0, 0, 1, 4096, 3700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 0, 0, 0, 0, 4086, 0, 0, 0, 3651, 4096, 4093, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 872, 1, 0, 0, 2, 3972, 4091, 0, 0, 2319, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 8, 1, 0, 0, 14, 4094, 0, 0, 0, 4096, 0, 1, 0, 3221, 789, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 1563, 4094, 2673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 138, 0, 0, 4096, 129, 3308, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4092, 0, 0, 0, 3931, 0, 0, 586, 0, 0, 17, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4093, 0, 0, 2, 3, 0, 28, 0, 0, 3279, 7, 4096, 0, 0, 4096, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16219, 16134, 16319, 16516, 16541, 16263, 16699, 16381] avg=16384.0000 max=16699 max_rank=6 min=16134 min_rank=1 imbalance_ratio=1.019226 max_load_reduction=114373 max_load_reduction_pct=0.872597 moved_to_echo=65884 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4084, 0, 0, 0, 185, 0, 0, 1, 4095, 24, 0, 0, 1, 0, 4095, 3700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 1, 0, 0, 1, 1, 0, 4084, 0, 0, 1, 3983, 4095, 4095, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 650, 0, 1, 1, 0, 4074, 4059, 0, 0, 2319, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 14, 4095, 1, 1, 0, 4095, 0, 0, 12, 3400, 789, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1834, 4090, 3657, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 4079, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 1, 1, 1, 4095, 1, 0, 4093, 1, 1, 161, 1, 0, 4095, 74, 3306, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 4095, 1, 0, 0, 3936, 1, 1, 586, 1, 0, 8, 3352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 4095, 0, 1, 0, 42, 0, 0, 0, 0, 2685, 3, 4095, 0, 0, 4091, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16185, 16470, 15505, 16705, 17787, 16358, 16077, 15985] avg=16384.0000 max=17787 max_rank=4 min=15505 min_rank=2 imbalance_ratio=1.085632 max_load_reduction=113285 max_load_reduction_pct=0.864296 moved_to_echo=66207 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[23, 0, 4066, 0, 0, 0, 227, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 3700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 3820, 4096, 4092, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 991, 1, 0, 0, 1, 3523, 4081, 0, 0, 2319, 0, 835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3, 0, 0, 0, 22, 4096, 0, 0, 0, 4096, 0, 1, 0, 3445, 789, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2354, 4096, 3173, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 4096, 0, 2, 4095, 0, 0, 213, 0, 0, 4096, 50, 3308, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 4094, 0, 0, 1, 3127, 0, 0, 586, 0, 0, 26, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 4094, 0, 0, 0, 0, 0, 4, 0, 0, 2786, 3, 4096, 0, 0, 4096, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16211, 16304, 15846, 16748, 17843, 16273, 15874, 15973] avg=16384.0000 max=17843 max_rank=4 min=15846 min_rank=2 imbalance_ratio=1.089050 max_load_reduction=113229 max_load_reduction_pct=0.863869 moved_to_echo=65963 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 1, 1748, 0, 1, 0, 4095, 2, 0, 0, 1, 0, 4095, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 4076, 0, 1, 1, 4046, 4095, 4095, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 1261, 4, 1, 1, 0, 2952, 4092, 1, 0, 2300, 0, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 8, 4095, 1, 1, 0, 4095, 0, 0, 129, 3699, 793, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 28, 4094, 2251, 1, 0, 1, 0, 2, 0, 1, 0, 1, 0, 4021, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4093, 1, 0, 4095, 0, 1, 165, 1, 0, 4095, 4, 3311, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4095, 0, 0, 0, 3958, 1, 1, 575, 1, 0, 5, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 4095, 0, 1, 0, 45, 0, 0, 0, 0, 4079, 0, 4095, 0, 0, 4092, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17733, 16520, 14996, 17125, 14527, 16164, 16796, 17211] avg=16384.0000 max=17733 max_rank=0 min=14527 min_rank=4 imbalance_ratio=1.082336 max_load_reduction=113339 max_load_reduction_pct=0.864708 moved_to_echo=64698 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[21, 1, 4062, 0, 1, 0, 103, 1, 0, 0, 4095, 24, 0, 1, 1, 0, 4095, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 32, 0, 0, 0, 1, 1, 0, 4056, 0, 0, 1, 3805, 4095, 4088, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 2050, 0, 0, 0, 0, 3829, 4012, 0, 1, 2319, 0, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 1, 0, 45, 4095, 1, 0, 0, 4095, 0, 0, 118, 2006, 789, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 517, 4087, 3025, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4077, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 26, 1, 0, 0, 4095, 1, 0, 4095, 1, 1, 241, 1, 0, 4095, 3, 3307, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 75, 4094, 4093, 0, 0, 0, 3932, 0, 1, 586, 1, 1, 35, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 4095, 0, 0, 1, 132, 0, 0, 0, 1, 3740, 18, 4095, 0, 0, 4095, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16104, 16277, 17198, 15449, 15831, 16264, 16753, 17196] avg=16384.0000 max=17198 max_rank=2 min=15449 min_rank=3 imbalance_ratio=1.049683 max_load_reduction=113874 max_load_reduction_pct=0.868790 moved_to_echo=66044 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 32640, 0, 128, 128, 3200, 128, 128, 128, 32768, 128, 128, 128, 128, 128, 32768, 29696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 32768, 11136, 128, 128, 128, 128, 30080, 32640, 128, 128, 18560, 0, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 128, 128, 0, 128, 32768, 128, 128, 0, 32768, 0, 128, 384, 24448, 6400, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 32768, 32768, 128, 128, 128, 30848, 128, 128, 4736, 128, 128, 256, 29952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 128, 0, 32768, 128, 128, 128, 31232, 32768, 32768, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 8320, 32768, 24448, 128, 0, 128, 0, 128, 0, 128, 0, 128, 128, 32640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1024, 32768, 0, 128, 128, 384, 0, 128, 0, 128, 25344, 128, 32768, 128, 0, 32768, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 256, 128, 128, 128, 32768, 128, 128, 32768, 128, 128, 2688, 128, 0, 32768, 384, 26496, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4095, 1, 0, 0, 164, 0, 3, 0, 0, 205, 4096, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 3259, 4096, 4096, 0, 0, 9, 2, 0, 4065, 0, 3460, 0, 0, 0, 0, 0, 4095, 2713, 154, 0, 0, 12, 0, 4096, 0, 0, 0, 2601, 0, 0, 8, 401, 0, 4096, 4096, 4083, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2423, 4075, 4090, 3762, 804, 3427, 0, 1, 0, 1954, 4078, 0, 4096, 4096, 0, 0, 0, 4096, 1728, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 4095, 4096, 1, 0, 53, 0, 0, 5, 0, 16, 0, 0, 0, 1640, 0, 0, 0, 4088, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 1, 4095, 4095, 1, 1, 0, 42, 1, 0, 1, 1, 654, 4095, 1, 4094, 1, 0, 1, 0, 0, 1, 4084, 4095, 1, 0, 0, 0, 2422, 4095, 4095, 0, 0, 0, 0, 1, 4094, 109, 3630, 0, 0, 0, 0, 4, 4095, 3254, 523, 0, 1, 4, 1, 4095, 0, 1, 1, 3785, 0, 1, 4, 177, 0, 4093, 4095, 4090, 1, 17, 1, 1, 0, 0, 1, 0, 1, 1214, 4090, 4095, 3849, 1474, 3428, 1, 0, 0, 2927, 4082, 1, 4095, 4095, 1, 0, 0, 4094, 305, 0, 1, 0, 1, 1, 0, 0, 4085, 0, 4095, 4095, 0, 0, 1, 0, 1, 14, 0, 0, 1, 0, 0, 881, 1, 0, 0, 4088, 1, 0, 0, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 1, 1, 1, 1, 4095, 4096, 0, 1, 0, 43, 0, 33, 0, 0, 446, 4095, 0, 4095, 1, 0, 0, 0, 1, 1, 4095, 4095, 1, 0, 0, 0, 2895, 4095, 4095, 0, 0, 31, 3, 1, 4039, 4, 2370, 1, 0, 0, 0, 0, 4095, 2324, 1173, 0, 0, 0, 1, 4094, 0, 1, 1, 1579, 0, 1, 7, 1180, 0, 4095, 4095, 4048, 1, 51, 0, 1, 0, 0, 0, 0, 1, 464, 4064, 4093, 3839, 3217, 4050, 0, 0, 0, 2048, 4050, 1, 4095, 4095, 1, 0, 1, 4095, 1403, 0, 1, 0, 1, 0, 0, 0, 3990, 1, 4095, 4091, 0, 0, 0, 0, 0, 1, 0, 18, 1, 0, 1, 1813, 1, 0, 0, 4061, 0, 0, 1, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 1, 1, 1, 0, 4095, 4095, 0, 1, 0, 35, 1, 0, 0, 0, 324, 4095, 1, 4089, 1, 0, 1, 0, 0, 1, 4093, 4095, 1, 0, 0, 0, 2327, 4095, 4095, 0, 0, 0, 0, 1, 4086, 18, 1280, 1, 0, 0, 0, 4, 4095, 1912, 2185, 0, 1, 0, 1, 4095, 0, 1, 1, 2474, 0, 1, 9, 1671, 0, 4095, 4095, 4085, 1, 51, 1, 1, 0, 2, 0, 0, 1, 874, 4091, 4095, 3961, 3997, 4045, 0, 0, 0, 1283, 4090, 1, 4095, 4095, 1, 0, 0, 4095, 641, 0, 1, 0, 1, 0, 1, 0, 3852, 0, 4095, 4095, 0, 0, 4, 0, 1, 13, 0, 0, 1, 0, 1, 1877, 0, 0, 0, 4053, 0, 0, 0, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 135, 0, 38, 0, 0, 1648, 4096, 0, 4096, 0, 0, 0, 0, 3, 0, 4096, 4096, 0, 0, 0, 0, 2484, 4096, 4096, 0, 0, 8, 1, 0, 4084, 33, 2370, 0, 0, 1, 0, 0, 4095, 2630, 645, 0, 0, 4, 0, 4096, 0, 0, 0, 3408, 0, 0, 2, 177, 0, 4095, 4096, 4082, 0, 18, 0, 0, 0, 1, 0, 0, 0, 305, 4074, 4084, 3523, 3125, 4042, 0, 0, 0, 2445, 3932, 0, 4096, 4096, 0, 0, 0, 4095, 577, 0, 0, 0, 0, 0, 0, 0, 3808, 0, 4095, 4096, 1, 0, 94, 0, 0, 12, 0, 6, 0, 0, 0, 1457, 0, 0, 0, 4093, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 32, 0, 3, 0, 0, 1237, 4096, 0, 4096, 0, 0, 0, 2, 6, 0, 4096, 4096, 0, 0, 0, 0, 1397, 4096, 4096, 0, 0, 4, 2, 0, 4085, 1, 3178, 0, 0, 2, 0, 0, 4095, 1968, 752, 0, 0, 13, 0, 4096, 0, 0, 0, 3342, 0, 0, 2, 585, 0, 4094, 4096, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1582, 4088, 4091, 3643, 3734, 3081, 1, 0, 0, 2050, 4027, 0, 4096, 4096, 0, 0, 0, 4096, 1295, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 4096, 4096, 0, 0, 14, 0, 0, 0, 0, 2, 0, 0, 0, 895, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4094, 0, 0, 0, 56, 0, 1, 0, 0, 109, 4096, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 0, 0, 3438, 4096, 4096, 0, 0, 8, 2, 0, 4054, 52, 3932, 0, 0, 0, 0, 0, 4095, 3527, 66, 0, 0, 29, 0, 4095, 0, 0, 0, 3554, 0, 0, 25, 492, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1772, 4083, 4082, 3645, 580, 3004, 0, 0, 0, 3313, 4091, 0, 4096, 4096, 0, 0, 0, 4096, 647, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4095, 4096, 2, 0, 17, 0, 0, 3, 0, 23, 0, 0, 0, 461, 0, 0, 0, 4095, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 1, 1, 4095, 4095, 6, 1, 0, 107, 1, 3, 0, 0, 193, 4094, 1, 4095, 1, 1, 1, 0, 3, 1, 4094, 4095, 1, 0, 0, 1, 3757, 4095, 4095, 0, 0, 3, 0, 1, 4090, 52, 2685, 0, 0, 0, 0, 0, 4095, 2880, 97, 0, 1, 5, 1, 4095, 0, 1, 1, 3416, 0, 1, 10, 380, 0, 4092, 4095, 4087, 1, 19, 1, 1, 0, 0, 0, 0, 1, 1461, 4083, 4095, 3376, 2505, 3498, 1, 0, 0, 1561, 4076, 0, 4095, 4095, 1, 0, 0, 4094, 722, 0, 1, 0, 1, 0, 0, 0, 4076, 0, 4095, 4095, 0, 1, 4, 0, 0, 0, 0, 0, 1, 0, 1, 2010, 1, 0, 0, 4075, 1, 0, 0, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 1, 4095, 4095, 1, 1, 0, 42, 1, 0, 1, 1, 1536, 1307, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 654, 4095, 1, 4094, 1, 0, 1, 0, 0, 1, 4084, 3494, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2422, 2559, 4095, 0, 0, 0, 0, 1, 4094, 109, 3630, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3254, 523, 0, 1, 4, 1, 4095, 0, 1, 1, 3785, 0, 1, 4, 177, 601, 487, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2846, 0, 4090, 1, 17, 1, 1, 0, 0, 1, 0, 1, 1214, 4090, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 1474, 3428, 1, 0, 0, 2927, 3595, 1, 0, 0, 1, 0, 0, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 4085, 0, 4095, 4095, 0, 0, 1, 0, 1, 2787, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1, 0, 0, 881, 1, 0, 0, 4088, 1, 0, 0, 1, 1, 4095, 4095, 2716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16339, 16426, 16914, 17115, 16357, 15581, 16446, 15894] avg=16384.0000 max=17115 max_rank=3 min=15581 min_rank=5 imbalance_ratio=1.044617 max_load_reduction=113957 max_load_reduction_pct=0.869423 moved_to_echo=64278 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4095, 1, 0, 0, 164, 0, 3, 0, 0, 1537, 1308, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 4096, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 3494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3259, 2559, 4096, 0, 0, 9, 2, 0, 4065, 0, 3460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2713, 154, 0, 0, 12, 0, 4096, 0, 0, 0, 2601, 0, 0, 8, 401, 602, 487, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2848, 0, 4083, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2423, 4075, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3762, 804, 3427, 0, 1, 0, 1954, 3591, 0, 0, 0, 0, 0, 0, 0, 1728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 4095, 4096, 1, 0, 53, 0, 0, 2788, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 16, 0, 0, 0, 1640, 0, 0, 0, 4088, 0, 1, 0, 0, 0, 4096, 4096, 2716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16462, 15988, 17450, 15254, 17522, 15267, 16471, 16658] avg=16384.0000 max=17522 max_rank=4 min=15254 min_rank=3 imbalance_ratio=1.069458 max_load_reduction=113550 max_load_reduction_pct=0.866318 moved_to_echo=65918 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 32, 0, 3, 0, 0, 1537, 1309, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1237, 4096, 0, 4096, 0, 0, 0, 2, 6, 0, 4096, 3494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1397, 2559, 4096, 0, 0, 4, 2, 0, 4085, 1, 3178, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1968, 752, 0, 0, 13, 0, 4096, 0, 0, 0, 3342, 0, 0, 2, 585, 602, 481, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2847, 0, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1582, 4088, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3643, 3734, 3081, 1, 0, 0, 2050, 3546, 0, 0, 0, 0, 0, 0, 0, 1295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 4096, 4096, 0, 0, 14, 0, 0, 2787, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 895, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16330, 17027, 15324, 16021, 16699, 17350, 16420, 15901] avg=16384.0000 max=17350 max_rank=5 min=15324 min_rank=2 imbalance_ratio=1.058960 max_load_reduction=113722 max_load_reduction_pct=0.867630 moved_to_echo=66370 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4094, 0, 0, 0, 56, 0, 1, 0, 0, 1537, 1308, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 4096, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 3494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3438, 2559, 4096, 0, 0, 8, 2, 0, 4054, 52, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3527, 66, 0, 0, 29, 0, 4095, 0, 0, 0, 3554, 0, 0, 25, 492, 602, 488, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2848, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1772, 4083, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3645, 580, 3004, 0, 0, 0, 3313, 3603, 0, 0, 0, 0, 0, 0, 0, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4095, 4096, 2, 0, 17, 0, 0, 2788, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 23, 0, 0, 0, 461, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 4096, 4096, 2716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16350, 15893, 18141, 17058, 16875, 14792, 16472, 15491] avg=16384.0000 max=18141 max_rank=2 min=14792 min_rank=5 imbalance_ratio=1.107239 max_load_reduction=112931 max_load_reduction_pct=0.861595 moved_to_echo=63630 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 135, 0, 38, 0, 0, 1537, 1308, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1648, 4096, 0, 4096, 0, 0, 0, 0, 3, 0, 4096, 3494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2484, 2559, 4096, 0, 0, 8, 1, 0, 4084, 33, 2370, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2630, 645, 0, 0, 4, 0, 4096, 0, 0, 0, 3408, 0, 0, 2, 177, 602, 469, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2847, 0, 4082, 0, 18, 0, 0, 0, 1, 0, 0, 0, 305, 4074, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3523, 3125, 4042, 0, 0, 0, 2445, 3463, 0, 0, 0, 0, 0, 0, 0, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3808, 0, 4095, 4096, 1, 0, 94, 0, 0, 2787, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 6, 0, 0, 0, 1457, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 4096, 2716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16467, 17433, 15636, 16213, 15411, 17175, 16261, 16476] avg=16384.0000 max=17433 max_rank=1 min=15411 min_rank=4 imbalance_ratio=1.064026 max_load_reduction=113639 max_load_reduction_pct=0.866997 moved_to_echo=65323 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 1, 1, 1, 4095, 4096, 0, 1, 0, 43, 0, 33, 0, 0, 1536, 1307, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 446, 4095, 0, 4095, 1, 0, 0, 0, 1, 1, 4095, 3494, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2895, 2559, 4095, 0, 0, 31, 3, 1, 4039, 4, 2370, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2324, 1173, 0, 0, 0, 1, 4094, 0, 1, 1, 1579, 0, 1, 7, 1180, 601, 483, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2847, 0, 4048, 1, 51, 0, 1, 0, 0, 0, 0, 1, 464, 4064, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3839, 3217, 4050, 0, 0, 0, 2048, 3567, 1, 0, 0, 1, 0, 1, 0, 1403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 3990, 1, 4095, 4091, 0, 0, 0, 0, 0, 2788, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 18, 1, 0, 1, 1813, 1, 0, 0, 4061, 0, 0, 1, 1, 0, 4095, 4095, 2716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 16229, 15998, 15625, 15570, 18127, 16346, 16804] avg=16384.0000 max=18127 max_rank=5 min=15570 min_rank=4 imbalance_ratio=1.106384 max_load_reduction=112945 max_load_reduction_pct=0.861702 moved_to_echo=66847 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 1, 1, 0, 4095, 4095, 0, 1, 0, 35, 1, 0, 0, 0, 1540, 1308, 1166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 4095, 1, 4089, 1, 0, 1, 0, 0, 1, 4093, 3490, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2327, 2555, 4095, 0, 0, 0, 0, 1, 4086, 18, 1280, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1912, 2185, 0, 1, 0, 1, 4095, 0, 1, 1, 2474, 0, 1, 9, 1671, 605, 493, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2842, 0, 4085, 1, 51, 1, 1, 0, 2, 0, 0, 1, 874, 4091, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 3997, 4045, 0, 0, 0, 1283, 3597, 1, 0, 0, 1, 0, 0, 0, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 3852, 0, 4095, 4095, 0, 0, 4, 0, 1, 2787, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 1, 0, 1, 1877, 0, 0, 0, 4053, 0, 0, 0, 1, 1, 4095, 4095, 2715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16339, 16096, 14367, 17631, 16044, 17526, 16217, 16852] avg=16384.0000 max=17631 max_rank=3 min=14367 min_rank=2 imbalance_ratio=1.076111 max_load_reduction=113441 max_load_reduction_pct=0.865486 moved_to_echo=66639 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 1, 1, 4095, 4095, 6, 1, 0, 107, 1, 3, 0, 0, 1536, 1308, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 4094, 1, 4095, 1, 1, 1, 0, 3, 1, 4094, 3494, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3757, 2559, 4095, 0, 0, 3, 0, 1, 4090, 52, 2685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2880, 97, 0, 1, 5, 1, 4095, 0, 1, 1, 3416, 0, 1, 10, 380, 601, 486, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2845, 0, 4087, 1, 19, 1, 1, 0, 0, 0, 0, 1, 1461, 4083, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3376, 2505, 3498, 1, 0, 0, 1561, 3590, 0, 0, 0, 1, 0, 0, 0, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4076, 0, 4095, 4095, 0, 1, 4, 0, 0, 2786, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 2010, 1, 0, 0, 4075, 1, 0, 0, 1, 1, 4095, 4095, 2715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 15980, 17242, 16155, 16594, 15254, 16439, 16996] avg=16384.0000 max=17242 max_rank=2 min=15254 min_rank=5 imbalance_ratio=1.052368 max_load_reduction=113830 max_load_reduction_pct=0.868454 moved_to_echo=65283 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 32768, 32768, 128, 128, 0, 640, 128, 128, 128, 128, 12416, 10496, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[4864, 32768, 128, 32768, 128, 128, 128, 128, 128, 128, 32768, 28032, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 22784, 0, 32768, 128, 256, 128, 128, 0, 128, 128, 0, 128, 10112, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 128, 0, 128, 11136, 128, 0, 0, 32768, 128, 128, 128, 128, 128, 32768, 32768, 21760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[22016, 20480, 32768, 0, 0, 128, 128, 128, 32640, 384, 22912, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 21248, 5632, 0, 128, 128, 128, 32768, 0, 128, 128, 24192, 0, 128, 128, 5120, 4864, 3968, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[29696, 19456, 28672, 128, 128, 0, 17664, 28672, 128, 0, 0, 128, 0, 128, 0, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 0, 128, 128, 128, 0, 32128, 128, 32768, 32768, 128, 128, 256, 0, 128, 22400, 11136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 3342, 0, 4095, 0, 4079, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1813, 0, 109, 0, 4096, 0, 0, 141, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 4095, 0, 0, 175, 4096, 0, 3945, 4095, 0, 4095, 4079, 4096, 0, 23, 4091, 0, 18, 721, 4096, 1, 0, 0, 0, 3, 4005, 1452, 0, 0, 16, 3991, 0, 0, 4096, 0, 4057, 0, 0, 0, 0, 37, 0, 0, 1, 754, 0, 0, 4096, 0, 4096, 3631, 0, 70, 2843, 4096, 2, 4095, 1685, 0, 1, 0, 11, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4057, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 3311, 0, 4094, 0, 4086, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2633, 0, 154, 0, 4096, 0, 0, 182, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4093, 0, 0, 59, 4094, 0, 3745, 4096, 0, 4095, 4088, 4096, 0, 13, 4092, 0, 7, 1428, 4096, 3, 0, 0, 0, 3, 4048, 1968, 0, 0, 22, 2571, 0, 0, 4096, 0, 3991, 0, 0, 0, 0, 21, 0, 0, 2, 1049, 0, 0, 4096, 0, 4096, 1207, 0, 21, 3221, 4096, 0, 4084, 3152, 0, 3, 0, 4, 0, 1, 0, 4094, 4093, 0, 0, 0, 4096, 0, 0, 4087, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 1109, 0, 4095, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2419, 0, 200, 0, 4096, 0, 0, 124, 0, 0, 0, 2, 4096, 0, 1, 0, 0, 4094, 0, 0, 28, 4095, 0, 4006, 4096, 0, 4095, 4076, 4096, 0, 32, 4094, 0, 20, 1142, 4096, 3, 0, 0, 0, 2, 3943, 461, 0, 0, 33, 3690, 0, 0, 4096, 0, 4052, 0, 0, 0, 0, 23, 0, 0, 5, 448, 0, 0, 4096, 0, 4096, 3888, 0, 40, 3262, 4096, 0, 4094, 3950, 0, 1, 0, 3, 0, 7, 0, 4093, 4096, 0, 0, 0, 4096, 0, 0, 4002, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1, 3307, 0, 4095, 1, 4067, 1, 0, 19, 0, 1, 0, 0, 1, 0, 2694, 0, 27, 1, 4095, 1, 1, 648, 1, 1, 1, 0, 4095, 0, 0, 1, 0, 4095, 1, 0, 210, 4095, 1, 4059, 4095, 1, 4095, 4089, 4094, 0, 0, 4095, 2, 8, 1377, 4095, 0, 1, 1, 1, 0, 4027, 1199, 0, 0, 9, 3989, 0, 3, 4095, 1, 4006, 0, 0, 6, 0, 0, 1, 0, 28, 1219, 0, 0, 4094, 0, 4095, 2490, 0, 0, 1310, 4095, 20, 4092, 2153, 0, 0, 0, 0, 1, 0, 0, 4095, 4095, 1, 1, 1, 4094, 0, 1, 4084, 4094, 1, 1, 0, 0, 4095, 1, 0, 0, 4095, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1, 1857, 1, 4095, 1, 4070, 1, 0, 3, 0, 1, 0, 1, 1, 0, 3357, 0, 18, 1, 4095, 1, 1, 104, 1, 1, 1, 0, 4095, 0, 0, 1, 0, 4093, 0, 0, 0, 4084, 1, 4008, 4094, 1, 4095, 4091, 4095, 0, 0, 4095, 3, 15, 491, 4095, 0, 0, 1, 1, 0, 4002, 2554, 0, 0, 0, 4056, 0, 3, 4095, 1, 4050, 0, 0, 1, 0, 0, 1, 0, 34, 2159, 0, 0, 4095, 0, 4095, 185, 0, 0, 1884, 4095, 0, 4095, 4021, 0, 0, 0, 0, 1, 0, 0, 4095, 4095, 1, 1, 1, 4095, 0, 1, 4070, 4095, 1, 1, 18, 0, 4095, 1, 0, 0, 4095, 0, 1, 1, 0, 0, 1, 4, 1, 1, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1, 2873, 1, 4095, 1, 4051, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1882, 0, 19, 1, 4095, 1, 1, 477, 1, 1, 1, 1, 4095, 0, 37, 1, 0, 4092, 0, 0, 116, 4064, 1, 3577, 4095, 1, 4095, 4075, 4095, 0, 0, 4095, 1, 38, 487, 4095, 5, 0, 1, 0, 0, 4032, 1109, 0, 0, 3, 4066, 0, 0, 4095, 0, 3870, 0, 0, 17, 0, 222, 0, 1, 61, 1557, 1, 0, 4095, 0, 4095, 2327, 0, 51, 2256, 4095, 4, 4093, 3839, 0, 0, 0, 21, 1, 0, 0, 4095, 4092, 1, 1, 1, 4095, 0, 1, 3987, 4095, 1, 1, 24, 1, 4095, 1, 1, 0, 4095, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 3797, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3480, 0, 70, 0, 4096, 0, 0, 216, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 4092, 0, 0, 263, 4092, 0, 4027, 4096, 0, 4096, 4078, 4096, 0, 9, 4091, 0, 16, 1035, 4096, 3, 0, 0, 0, 2, 4000, 1150, 0, 0, 2, 3918, 0, 0, 4096, 0, 4084, 0, 0, 0, 0, 24, 0, 0, 7, 683, 0, 0, 4096, 0, 4096, 2844, 0, 30, 1410, 4096, 2, 4081, 1764, 0, 2, 0, 6, 0, 1, 0, 4094, 4094, 0, 0, 0, 4096, 0, 0, 4066, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 3886, 1, 4095, 1, 3864, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1978, 0, 123, 1, 4095, 1, 1, 300, 1, 1, 1, 0, 4094, 0, 0, 1, 0, 4095, 1, 0, 331, 4093, 1, 4086, 4095, 1, 4095, 4088, 4095, 0, 0, 4092, 0, 15, 629, 4095, 3, 1, 1, 1, 0, 4081, 179, 0, 0, 6, 3855, 0, 0, 4095, 1, 3991, 0, 0, 9, 0, 0, 1, 0, 32, 2474, 0, 0, 4094, 0, 4095, 1726, 0, 4, 2268, 4095, 14, 4094, 3027, 0, 0, 0, 3, 1, 0, 0, 4095, 4095, 1, 1, 1, 4095, 0, 1, 4079, 4095, 0, 1, 1, 0, 4095, 1, 0, 0, 4095, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3311, 0, 4094, 0, 4086, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2633, 1581, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 4096, 0, 0, 182, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4095, 3722, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 59, 4094, 0, 3745, 0, 0, 0, 4088, 0, 0, 13, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1428, 4096, 3, 0, 0, 0, 3, 4048, 1968, 0, 0, 22, 2571, 0, 0, 2297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3991, 0, 0, 0, 0, 21, 0, 0, 2, 1049, 0, 0, 4096, 0, 2477, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1207, 0, 21, 3221, 615, 0, 4084, 3152, 0, 3, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4093, 0, 0, 0, 4096, 0, 0, 4087, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16890, 16349, 16120, 16443, 16074, 16404, 16408, 16384] avg=16384.0000 max=16890 max_rank=0 min=16074 min_rank=4 imbalance_ratio=1.030884 max_load_reduction=114182 max_load_reduction_pct=0.871140 moved_to_echo=65270 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1109, 0, 4095, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2419, 1581, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 4096, 0, 0, 124, 0, 0, 0, 2, 4096, 0, 1, 0, 0, 4095, 3723, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 28, 4095, 0, 4006, 0, 0, 0, 4076, 0, 0, 32, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1142, 4096, 3, 0, 0, 0, 2, 3943, 461, 0, 0, 33, 3690, 0, 0, 2297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4052, 0, 0, 0, 0, 23, 0, 0, 5, 448, 0, 0, 4096, 0, 2477, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3888, 0, 40, 3262, 615, 0, 4094, 3950, 0, 1, 0, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 4096, 0, 0, 4002, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14483, 16339, 16359, 15687, 15540, 19956, 16325, 16383] avg=16384.0000 max=19956 max_rank=5 min=14483 min_rank=0 imbalance_ratio=1.218018 max_load_reduction=111116 max_load_reduction_pct=0.847748 moved_to_echo=68204 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3342, 0, 4095, 0, 4079, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1813, 1581, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 4096, 0, 0, 141, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 4095, 3724, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 175, 4096, 0, 3945, 0, 0, 0, 4079, 0, 0, 23, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 721, 4096, 1, 0, 0, 0, 3, 4005, 1452, 0, 0, 16, 3991, 0, 0, 2297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4057, 0, 0, 0, 0, 37, 0, 0, 1, 754, 0, 0, 4096, 0, 2477, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3631, 0, 70, 2843, 615, 2, 4095, 1685, 0, 1, 0, 11, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4057, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16095, 16263, 16437, 16600, 15861, 17050, 16383, 16383] avg=16384.0000 max=17050 max_rank=5 min=15861 min_rank=4 imbalance_ratio=1.040649 max_load_reduction=114022 max_load_reduction_pct=0.869919 moved_to_echo=65677 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 3307, 0, 4095, 1, 4067, 1, 0, 19, 0, 1, 0, 0, 1, 0, 2694, 1580, 1183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 1, 4095, 1, 1, 648, 1, 1, 1, 0, 4095, 0, 0, 1, 0, 4095, 3724, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 1, 0, 210, 4095, 1, 4059, 0, 1, 0, 4089, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1377, 4095, 0, 1, 1, 1, 0, 4027, 1199, 0, 0, 9, 3989, 0, 3, 2297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4006, 0, 0, 6, 0, 0, 1, 0, 28, 1219, 0, 0, 4094, 0, 2476, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2490, 0, 0, 1310, 615, 20, 4092, 2153, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 1, 1, 1, 4094, 0, 1, 4084, 38, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 4095, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16950, 16692, 16581, 17007, 16269, 14776, 16413, 16384] avg=16384.0000 max=17007 max_rank=3 min=14776 min_rank=5 imbalance_ratio=1.038025 max_load_reduction=114065 max_load_reduction_pct=0.870247 moved_to_echo=63842 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3797, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3480, 1581, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 4096, 0, 0, 216, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 4096, 3721, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 263, 4092, 0, 4027, 0, 0, 0, 4078, 0, 0, 9, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1035, 4096, 3, 0, 0, 0, 2, 4000, 1150, 0, 0, 2, 3918, 0, 0, 2297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4084, 0, 0, 0, 0, 24, 0, 0, 7, 683, 0, 0, 4096, 0, 2477, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2844, 0, 30, 1410, 615, 2, 4081, 1764, 0, 2, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 4096, 0, 0, 4066, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18230, 16301, 16589, 16519, 15809, 14851, 16388, 16385] avg=16384.0000 max=18230 max_rank=0 min=14851 min_rank=5 imbalance_ratio=1.112671 max_load_reduction=112842 max_load_reduction_pct=0.860916 moved_to_echo=63433 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 1857, 1, 4095, 1, 4070, 1, 0, 3, 0, 1, 0, 1, 1, 0, 3357, 1584, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1, 4095, 1, 1, 104, 1, 1, 1, 0, 4095, 0, 0, 1, 0, 4095, 3724, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4084, 1, 4008, 0, 1, 0, 4091, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 491, 4095, 0, 0, 1, 1, 0, 4002, 2554, 0, 0, 0, 4056, 0, 3, 2302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4050, 0, 0, 1, 0, 0, 1, 0, 34, 2159, 0, 0, 4095, 0, 2482, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 185, 0, 0, 1884, 605, 0, 4095, 4021, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 1, 1, 1, 4095, 0, 1, 4070, 29, 1, 1, 18, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 1, 4, 1, 1, 0, 0, 0, 0, 4095, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16161, 16140, 16306, 17520, 17264, 14886, 16409, 16386] avg=16384.0000 max=17520 max_rank=3 min=14886 min_rank=5 imbalance_ratio=1.069336 max_load_reduction=113552 max_load_reduction_pct=0.866333 moved_to_echo=64945 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 2873, 1, 4095, 1, 4051, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1882, 1581, 1183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1, 4095, 1, 1, 477, 1, 1, 1, 1, 4095, 0, 37, 1, 0, 4095, 3721, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 116, 4064, 1, 3577, 0, 1, 0, 4075, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 487, 4095, 5, 0, 1, 0, 0, 4032, 1109, 0, 0, 3, 4066, 0, 0, 2297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3870, 0, 0, 17, 0, 222, 0, 1, 61, 1557, 1, 0, 4095, 0, 2477, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2327, 0, 51, 2256, 615, 4, 4093, 3839, 0, 0, 0, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4092, 1, 1, 1, 4095, 0, 1, 3987, 37, 1, 1, 24, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15670, 16548, 15959, 16133, 16738, 17302, 16338, 16384] avg=16384.0000 max=17302 max_rank=5 min=15670 min_rank=0 imbalance_ratio=1.056030 max_load_reduction=113770 max_load_reduction_pct=0.867996 moved_to_echo=66762 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 3886, 1, 4095, 1, 3864, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1978, 1581, 1183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 1, 4095, 1, 1, 300, 1, 1, 1, 0, 4094, 0, 0, 1, 0, 4095, 3724, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 1, 0, 331, 4093, 1, 4086, 0, 1, 0, 4088, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 629, 4095, 3, 1, 1, 1, 0, 4081, 179, 0, 0, 6, 3855, 0, 0, 2297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 3991, 0, 0, 9, 0, 0, 1, 0, 32, 2474, 0, 0, 4094, 0, 2477, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1726, 0, 4, 2268, 615, 14, 4094, 3027, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 1, 1, 1, 4095, 0, 1, 4079, 37, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4095, 4095, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16593, 16440, 16721, 15163, 17517, 15847, 16408, 16383] avg=16384.0000 max=17517 max_rank=4 min=15163 min_rank=3 imbalance_ratio=1.069153 max_load_reduction=113555 max_load_reduction_pct=0.866356 moved_to_echo=66155 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 23552, 128, 32768, 128, 32512, 128, 0, 128, 0, 128, 128, 128, 128, 128, 20352, 12672, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 32768, 0, 128, 128, 128, 128, 128, 128, 128, 128, 0, 128, 128, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 32128, 0, 0, 128, 0, 384, 128, 128, 256, 10368, 128, 0, 32768, 0, 19840, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 32768, 128, 128, 128, 32768, 0, 128, 32512, 384, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 768, 128, 32768, 128, 128, 2304, 128, 128, 128, 128, 32768, 0, 128, 128, 0, 32768, 29824, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[256, 128, 0, 1280, 32768, 128, 31488, 0, 128, 0, 32768, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 18304, 0, 256, 18560, 4992, 128, 32768, 23680, 0, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 7424, 32768, 128, 128, 128, 128, 128, 32256, 10112, 0, 0, 128, 30208, 0, 128, 18432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[320, 0, 0, 4096, 0, 0, 3, 0, 0, 29, 4094, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4088, 4093, 0, 0, 0, 4005, 0, 0, 0, 0, 4, 0, 4094, 0, 76, 4094, 0, 0, 4096, 4096, 0, 4095, 1, 0, 0, 49, 2, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 2, 0, 0, 4092, 4096, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 3007, 0, 2813, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 0, 2, 0, 40, 0, 4094, 0, 0, 4094, 0, 1993, 4096, 0, 4043, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[713, 0, 0, 4095, 0, 1, 0, 0, 1, 376, 4087, 1, 1, 0, 4095, 0, 4065, 0, 1, 0, 0, 4095, 4095, 0, 1, 0, 4095, 1, 1, 0, 0, 0, 0, 4095, 18, 79, 4095, 1, 1, 4093, 4095, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 3977, 4095, 1, 4095, 0, 1, 0, 1, 1, 0, 0, 2727, 1, 3187, 4085, 0, 1, 0, 2, 4095, 1, 4095, 0, 1, 3, 0, 0, 0, 1, 0, 1, 0, 4095, 0, 37, 1, 1, 1, 4095, 0, 1, 5, 0, 0, 0, 1, 0, 1, 4095, 1, 1, 4095, 0, 1275, 4094, 1, 4041, 4095, 0, 1, 4085, 0, 10, 4095, 4083, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[905, 0, 0, 4096, 0, 0, 9, 1, 0, 319, 4094, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4087, 4095, 0, 0, 0, 4032, 0, 0, 0, 0, 3, 0, 4093, 1, 214, 4095, 0, 0, 4096, 4095, 0, 4095, 1, 0, 0, 14, 1, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4077, 4096, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 1635, 0, 2790, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 1, 0, 0, 0, 0, 4095, 1, 0, 27, 0, 0, 2, 0, 20, 0, 4095, 0, 0, 4094, 0, 2379, 4096, 0, 4067, 4095, 1, 0, 4096, 0, 0, 4096, 4096, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[137, 0, 0, 4093, 0, 1, 30, 0, 1, 1501, 4095, 1, 1, 0, 4095, 0, 4069, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 4095, 11, 297, 4095, 1, 1, 4075, 4095, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 3975, 4095, 1, 4095, 0, 1, 0, 1, 1, 0, 0, 1899, 0, 3735, 4082, 1, 1, 1, 0, 4095, 1, 4095, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 4095, 0, 0, 1, 1, 1, 4095, 0, 4, 0, 0, 0, 0, 1, 0, 0, 4095, 1, 1, 4095, 0, 755, 4094, 1, 4095, 4095, 0, 0, 4091, 0, 17, 4095, 4084, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1549, 0, 0, 4096, 0, 0, 9, 0, 0, 59, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4075, 4095, 0, 0, 0, 4027, 0, 1, 0, 0, 2, 0, 4090, 0, 247, 4095, 0, 0, 4096, 4096, 0, 4094, 1, 0, 0, 14, 1, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 2, 0, 0, 4067, 4096, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 2932, 0, 3192, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 71, 0, 0, 2, 0, 19, 0, 4096, 0, 0, 4093, 0, 269, 4096, 0, 4049, 4094, 0, 0, 4096, 0, 0, 4096, 4096, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[826, 1, 0, 4093, 0, 0, 77, 0, 1, 775, 4056, 1, 1, 0, 4095, 1, 4095, 1, 1, 0, 0, 4071, 4095, 0, 0, 0, 4038, 1, 0, 0, 0, 0, 0, 4069, 2, 163, 4095, 0, 1, 4095, 4095, 0, 4095, 0, 0, 0, 39, 0, 1, 0, 1, 0, 1, 2, 4095, 0, 0, 1, 0, 0, 1, 3984, 4095, 1, 4095, 10, 1, 0, 0, 1, 0, 0, 2899, 1, 3180, 4079, 1, 1, 0, 0, 4095, 0, 4095, 0, 0, 2, 1, 1, 0, 1, 0, 7, 0, 4095, 0, 30, 0, 1, 1, 4095, 0, 1, 40, 0, 6, 0, 1, 39, 0, 4095, 1, 0, 4095, 0, 406, 4095, 1, 4051, 4095, 0, 0, 4095, 0, 12, 4095, 4091, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[549, 0, 0, 4090, 0, 0, 4, 0, 0, 148, 4092, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4076, 4095, 0, 0, 0, 4072, 0, 0, 0, 0, 6, 0, 4090, 0, 47, 4094, 0, 0, 4096, 4095, 0, 4094, 0, 0, 0, 5, 3, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 2, 0, 0, 4093, 4096, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 2127, 0, 2546, 4096, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 2, 0, 0, 2, 0, 4096, 3, 0, 0, 0, 0, 4096, 1, 0, 17, 0, 0, 0, 0, 9, 0, 4094, 0, 0, 4094, 0, 2803, 4096, 0, 4082, 4096, 0, 0, 4096, 0, 0, 4096, 4094, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1022, 1, 0, 4095, 0, 1, 30, 0, 1, 710, 4093, 1, 1, 0, 4094, 1, 4094, 0, 1, 0, 0, 4085, 4095, 0, 0, 0, 4064, 1, 1, 0, 0, 0, 0, 4095, 13, 40, 4095, 0, 1, 4074, 4095, 0, 4095, 0, 0, 0, 2, 0, 1, 0, 1, 0, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 4015, 4095, 1, 4095, 3, 1, 0, 1, 1, 0, 0, 1930, 0, 3194, 4084, 1, 1, 0, 1, 4095, 0, 4095, 0, 1, 6, 1, 0, 0, 1, 1, 1, 0, 4095, 0, 30, 1, 1, 1, 4095, 0, 1, 4, 0, 0, 1, 1, 0, 0, 4095, 1, 1, 4095, 0, 1343, 4094, 1, 4095, 4095, 0, 0, 4095, 0, 22, 4095, 4090, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[320, 0, 0, 4096, 0, 0, 3, 0, 0, 29, 4094, 0, 0, 0, 4096, 0, 2841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 4093, 0, 0, 0, 4005, 0, 0, 0, 0, 4, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 76, 4094, 0, 0, 4096, 0, 0, 3941, 1, 0, 0, 49, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 2, 0, 0, 4092, 4096, 0, 4096, 41, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 3007, 0, 2813, 4096, 0, 0, 0, 0, 1371, 1252, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 3, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 0, 2, 0, 40, 0, 4094, 0, 0, 4096, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2723, 0, 1993, 0, 0, 4043, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15479, 16346, 16353, 16442, 16732, 16372, 16397, 16951] avg=16384.0000 max=16951 max_rank=7 min=15479 min_rank=0 imbalance_ratio=1.034607 max_load_reduction=114121 max_load_reduction_pct=0.870674 moved_to_echo=66452 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[905, 0, 0, 4096, 0, 0, 9, 1, 0, 319, 4094, 0, 0, 0, 4096, 0, 2840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 4095, 0, 0, 0, 4032, 0, 0, 0, 0, 3, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 214, 4095, 0, 0, 4096, 0, 0, 3941, 1, 0, 0, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4077, 4096, 0, 4096, 41, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 1635, 0, 2790, 4096, 0, 0, 0, 0, 1371, 1254, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 1, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 27, 0, 0, 2, 0, 20, 0, 4095, 0, 0, 4096, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2723, 0, 2379, 0, 0, 4067, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16360, 16373, 16456, 16425, 15338, 16368, 16390, 17362] avg=16384.0000 max=17362 max_rank=7 min=15338 min_rank=4 imbalance_ratio=1.059692 max_load_reduction=113710 max_load_reduction_pct=0.867538 moved_to_echo=65458 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1549, 0, 0, 4096, 0, 0, 9, 0, 0, 59, 4095, 0, 0, 0, 4096, 0, 2841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4075, 4095, 0, 0, 0, 4027, 0, 1, 0, 0, 2, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 247, 4095, 0, 0, 4096, 0, 0, 3941, 1, 0, 0, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 2, 0, 0, 4067, 4096, 0, 4096, 41, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 2932, 0, 3192, 4096, 0, 0, 0, 0, 1371, 1253, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 1, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 71, 0, 0, 2, 0, 19, 0, 4096, 0, 0, 4096, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2722, 0, 269, 0, 0, 4049, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16745, 16356, 16485, 16417, 17034, 16368, 16435, 15232] avg=16384.0000 max=17034 max_rank=4 min=15232 min_rank=7 imbalance_ratio=1.039673 max_load_reduction=114038 max_load_reduction_pct=0.870041 moved_to_echo=65069 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[137, 0, 0, 4093, 0, 1, 30, 0, 1, 1501, 4095, 1, 1, 0, 4095, 0, 2842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 11, 297, 4095, 1, 1, 4075, 0, 0, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 3975, 4095, 1, 4094, 41, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 1, 1, 0, 0, 1899, 0, 3735, 4082, 1, 1, 1, 0, 1376, 1253, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4095, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 4095, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4095, 0, 4, 0, 0, 0, 0, 1, 0, 0, 4095, 1, 1, 4095, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2719, 0, 755, 0, 1, 4095, 0, 0, 0, 4091, 0, 17, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16797, 16415, 16513, 16324, 16542, 16370, 16349, 15762] avg=16384.0000 max=16797 max_rank=0 min=15762 min_rank=7 imbalance_ratio=1.025208 max_load_reduction=114275 max_load_reduction_pct=0.871849 moved_to_echo=65023 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[713, 0, 0, 4095, 0, 1, 0, 0, 1, 376, 4087, 1, 1, 0, 4095, 0, 2841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 1, 0, 0, 4095, 4095, 0, 1, 0, 4095, 1, 1, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 18, 79, 4095, 1, 1, 4093, 0, 0, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 3977, 4095, 1, 4094, 40, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 1, 1, 0, 0, 2727, 1, 3187, 4085, 0, 1, 0, 2, 1372, 1254, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4095, 0, 1, 3, 0, 0, 0, 1, 0, 1, 0, 4095, 0, 37, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4095, 0, 1, 5, 0, 0, 0, 1, 0, 1, 4095, 1, 1, 4095, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2723, 0, 1275, 0, 1, 4041, 0, 0, 1, 4085, 0, 10, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16211, 16414, 16323, 16324, 16821, 16407, 16353, 16219] avg=16384.0000 max=16821 max_rank=4 min=16211 min_rank=0 imbalance_ratio=1.026672 max_load_reduction=114251 max_load_reduction_pct=0.871666 moved_to_echo=65800 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[549, 0, 0, 4090, 0, 0, 4, 0, 0, 148, 4092, 0, 0, 0, 4096, 0, 2840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4076, 4095, 0, 0, 0, 4072, 0, 0, 0, 0, 6, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 47, 4094, 0, 0, 4096, 0, 0, 3941, 0, 0, 0, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 2, 0, 0, 4093, 4096, 0, 4096, 41, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 2127, 0, 2546, 4096, 0, 0, 0, 0, 1371, 1253, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 2, 0, 0, 2, 0, 4096, 3, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 17, 0, 0, 0, 0, 9, 0, 4094, 0, 0, 4096, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2723, 0, 2803, 0, 0, 4082, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15819, 16405, 16276, 16444, 15587, 16376, 16367, 17798] avg=16384.0000 max=17798 max_rank=7 min=15587 min_rank=4 imbalance_ratio=1.086304 max_load_reduction=113274 max_load_reduction_pct=0.864212 moved_to_echo=66128 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[826, 1, 0, 4093, 0, 0, 77, 0, 1, 775, 4056, 1, 1, 0, 4095, 1, 2842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 4071, 4095, 0, 0, 0, 4038, 1, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 2, 163, 4095, 0, 1, 4095, 0, 0, 3941, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 2, 4095, 0, 0, 1, 0, 0, 1, 3984, 4095, 1, 4095, 40, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 10, 1, 0, 0, 1, 0, 0, 2899, 1, 3180, 4079, 1, 1, 0, 0, 1372, 1253, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 2, 1, 1, 0, 1, 0, 7, 0, 4095, 0, 30, 4079, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 1, 40, 0, 6, 0, 1, 39, 0, 4095, 1, 0, 4095, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2723, 0, 406, 0, 1, 4051, 0, 0, 0, 4095, 0, 12, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16769, 16362, 16405, 16333, 16987, 16407, 16430, 15379] avg=16384.0000 max=16987 max_rank=4 min=15379 min_rank=7 imbalance_ratio=1.036804 max_load_reduction=114085 max_load_reduction_pct=0.870399 moved_to_echo=65203 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1022, 1, 0, 4095, 0, 1, 30, 0, 1, 710, 4093, 1, 1, 0, 4094, 1, 2842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 4085, 4095, 0, 0, 0, 4064, 1, 1, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 13, 40, 4095, 0, 1, 4074, 0, 0, 3941, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 4015, 4095, 1, 4094, 41, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 1, 0, 1, 1, 0, 0, 1930, 0, 3194, 4084, 1, 1, 0, 1, 1372, 1253, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 1, 6, 1, 0, 0, 1, 1, 1, 0, 4095, 0, 30, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4095, 0, 1, 4, 0, 0, 1, 1, 0, 0, 4095, 1, 1, 4095, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2723, 0, 1343, 0, 1, 4095, 0, 0, 0, 4095, 0, 22, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16892, 16401, 16261, 16363, 16031, 16404, 16351, 16369] avg=16384.0000 max=16892 max_rank=0 min=16031 min_rank=4 imbalance_ratio=1.031006 max_load_reduction=114180 max_load_reduction_pct=0.871124 moved_to_echo=65155 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[6144, 128, 0, 32768, 0, 128, 256, 128, 128, 3968, 32768, 128, 128, 0, 32768, 128, 22784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 128, 1280, 32768, 128, 128, 32768, 0, 0, 31616, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 128, 128, 128, 32768, 0, 128, 128, 128, 0, 128, 32384, 32768, 128, 32768, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 128, 32768, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 0, 0, 32768, 32768, 0, 128, 0, 32512, 128, 128, 0, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 256, 0, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 0, 0, 19200, 128, 24704, 32768, 128, 128, 128, 128, 11008, 10112, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[21888, 0, 11264, 0, 128, 32640, 0, 128, 128, 32768, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1447, 4094, 1, 0, 1, 0, 0, 4095, 2, 4095, 0, 1, 4095, 4093, 1, 10, 1, 0, 0, 4095, 1, 1, 4058, 1, 4095, 0, 0, 19, 1, 4095, 1, 1, 3791, 0, 1, 954, 1, 0, 0, 0, 4094, 0, 2, 1, 0, 1, 49, 0, 0, 1, 0, 0, 4095, 4060, 0, 4095, 5, 0, 1585, 0, 1, 0, 1, 4095, 0, 0, 0, 1, 0, 4092, 1, 0, 2328, 0, 3885, 3799, 3972, 0, 1, 0, 3983, 0, 1, 1, 1, 4095, 0, 0, 4093, 1, 3, 1, 4095, 3884, 2800, 0, 110, 0, 4, 0, 1, 0, 0, 0, 4095, 0, 75, 1, 0, 0, 4095, 2, 0, 1, 4095, 114, 4095, 0, 4095, 20, 0, 1, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 1793, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4075, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 725, 0, 0, 0, 1, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4094, 4088, 0, 4095, 0, 1, 2349, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 4080, 0, 0, 2485, 8, 4096, 4088, 4094, 44, 0, 0, 3371, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3351, 1969, 0, 55, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 257, 0, 1, 2, 4095, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 1412, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4076, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 699, 0, 0, 0, 2, 4096, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4093, 4095, 0, 4094, 4, 2, 2230, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 4092, 0, 0, 2423, 17, 4096, 4083, 4094, 23, 0, 0, 3612, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3374, 2236, 0, 25, 1, 0, 0, 0, 0, 0, 2, 4094, 0, 351, 0, 2, 2, 4093, 0, 1, 0, 4096, 0, 4096, 0, 4094, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 1193, 4095, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4069, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 206, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4094, 4087, 0, 4095, 0, 1, 2307, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 3262, 18, 4096, 4073, 4096, 27, 0, 0, 3788, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3430, 1839, 1, 9, 1, 0, 0, 0, 1, 0, 0, 4095, 0, 355, 0, 1, 1, 4095, 0, 0, 0, 4096, 0, 4095, 1, 4095, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 2121, 4095, 1, 1, 0, 0, 0, 4095, 26, 4095, 1, 1, 4095, 4095, 1, 2, 1, 0, 0, 4096, 0, 1, 4084, 1, 4095, 0, 0, 4, 1, 4095, 1, 1, 4079, 1, 1, 1396, 1, 0, 1, 18, 4095, 0, 55, 1, 0, 1, 1, 0, 0, 0, 0, 0, 4095, 4068, 1, 4095, 1, 0, 2063, 0, 1, 0, 1, 4095, 0, 0, 1, 0, 0, 4090, 1, 0, 1935, 1, 4063, 4071, 4003, 0, 0, 0, 3750, 0, 1, 0, 0, 4095, 1, 0, 4092, 1, 1, 1, 4095, 3814, 1304, 0, 33, 0, 1, 0, 1, 0, 0, 9, 4091, 0, 33, 0, 0, 0, 4095, 1, 0, 1, 4095, 10, 4095, 0, 4095, 16, 0, 1, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 2361, 4095, 1, 0, 1, 0, 0, 4091, 0, 4095, 1, 1, 4095, 4094, 1, 4, 1, 0, 0, 4095, 0, 1, 4074, 1, 4095, 0, 0, 29, 1, 4095, 1, 1, 3774, 1, 1, 608, 1, 0, 0, 0, 4094, 0, 19, 1, 0, 0, 8, 0, 0, 2, 0, 0, 4095, 4095, 1, 4095, 40, 0, 2945, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 0, 4093, 1, 0, 2628, 0, 3957, 4091, 3991, 0, 1, 0, 3185, 0, 1, 0, 1, 4095, 0, 0, 4092, 1, 2, 1, 4095, 4016, 946, 0, 15, 0, 4, 0, 1, 0, 0, 0, 4095, 0, 2, 1, 0, 0, 4095, 1, 0, 1, 4095, 110, 4095, 0, 4095, 57, 0, 1, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 833, 4094, 1, 1, 1, 0, 0, 4095, 5, 4095, 3, 1, 4095, 4095, 1, 2, 1, 0, 0, 4095, 1, 1, 4087, 1, 4095, 0, 0, 0, 1, 4095, 1, 1, 3773, 1, 1, 371, 1, 0, 0, 6, 4095, 0, 1, 1, 0, 0, 38, 0, 0, 0, 0, 1, 4095, 4051, 1, 4095, 0, 0, 471, 0, 1, 0, 1, 4095, 0, 0, 3, 0, 0, 4091, 1, 0, 3737, 0, 3973, 3844, 3943, 0, 1, 0, 4049, 0, 1, 0, 1, 4095, 0, 0, 4092, 1, 1, 1, 4095, 3795, 3695, 0, 219, 0, 1, 0, 1, 0, 0, 0, 4094, 1, 9, 0, 0, 0, 4095, 1, 0, 1, 4095, 23, 4095, 0, 4095, 25, 1, 1, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 2102, 4094, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4069, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 548, 0, 0, 0, 3, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4093, 4091, 0, 4094, 0, 2, 2646, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 4088, 0, 0, 2310, 2, 4096, 4089, 4094, 22, 0, 0, 3914, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3030, 919, 0, 290, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 638, 0, 2, 2, 4094, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 1412, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1849, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 4076, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 699, 0, 0, 0, 2, 4096, 4, 1, 0, 3513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4093, 4095, 0, 4094, 4, 2, 2230, 0, 2014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1598, 0, 0, 2, 0, 0, 4092, 0, 0, 2423, 17, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 23, 0, 0, 3612, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 387, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3374, 2236, 0, 25, 1, 0, 0, 0, 0, 0, 2, 4094, 0, 351, 0, 2233, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4093, 0, 1, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16137, 16378, 16502, 16542, 16311, 16356, 16509, 16337] avg=16384.0000 max=16542 max_rank=3 min=16137 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=114530 max_load_reduction_pct=0.873795 moved_to_echo=65513 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 1793, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1849, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 4075, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 725, 0, 0, 0, 1, 4096, 10, 0, 0, 3513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4094, 4088, 0, 4095, 0, 1, 2349, 0, 2014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1598, 0, 0, 2, 0, 0, 4080, 0, 0, 2485, 8, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 44, 0, 0, 3371, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 387, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3351, 1969, 0, 55, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 257, 0, 2233, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16518, 16377, 16534, 16658, 16357, 16135, 16154, 16339] avg=16384.0000 max=16658 max_rank=3 min=16135 min_rank=5 imbalance_ratio=1.016724 max_load_reduction=114414 max_load_reduction_pct=0.872910 moved_to_echo=64985 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1447, 4094, 1, 0, 1, 0, 0, 4095, 2, 4095, 0, 1, 1849, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4093, 1, 10, 1, 0, 0, 4095, 1, 1, 4058, 1, 4095, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 3791, 0, 1, 954, 1, 0, 0, 0, 4094, 0, 2, 1, 3513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 49, 0, 0, 1, 0, 0, 4095, 4060, 0, 4095, 5, 0, 1585, 0, 2014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1598, 0, 0, 0, 1, 0, 4092, 1, 0, 2328, 0, 3885, 3799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 0, 1, 0, 3983, 0, 1, 1, 1, 4095, 0, 0, 4093, 1, 3, 1, 386, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3884, 2800, 0, 110, 0, 4, 0, 1, 0, 0, 0, 4095, 0, 75, 1, 2233, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 1, 0, 114, 4095, 0, 4095, 20, 0, 1, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16168, 16388, 16455, 15905, 15706, 16584, 17395, 16471] avg=16384.0000 max=17395 max_rank=6 min=15706 min_rank=4 imbalance_ratio=1.061707 max_load_reduction=113677 max_load_reduction_pct=0.867287 moved_to_echo=66156 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 1193, 4095, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 1849, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 4069, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 206, 0, 0, 0, 0, 4096, 4, 0, 0, 3513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4094, 4087, 0, 4095, 0, 1, 2307, 0, 2014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1598, 0, 0, 0, 0, 0, 4093, 0, 0, 3262, 18, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 27, 0, 0, 3788, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 387, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3430, 1839, 1, 9, 1, 0, 0, 0, 1, 0, 0, 4095, 0, 355, 0, 2233, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 0, 0, 4095, 1, 4095, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15915, 16370, 16005, 16611, 17140, 16537, 16157, 16337] avg=16384.0000 max=17140 max_rank=4 min=15915 min_rank=0 imbalance_ratio=1.046143 max_load_reduction=113932 max_load_reduction_pct=0.869232 moved_to_echo=66171 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 2102, 4094, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 1849, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 4069, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 548, 0, 0, 0, 3, 4096, 6, 0, 0, 3513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4093, 4091, 0, 4094, 0, 2, 2646, 0, 2014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1598, 0, 0, 2, 0, 0, 4088, 0, 0, 2310, 2, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 22, 0, 0, 3914, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 387, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3030, 919, 0, 290, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 638, 0, 2233, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4094, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16823, 16371, 16348, 16953, 16185, 16656, 15397, 16339] avg=16384.0000 max=16953 max_rank=3 min=15397 min_rank=6 imbalance_ratio=1.034729 max_load_reduction=114119 max_load_reduction_pct=0.870659 moved_to_echo=64577 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 2121, 4095, 1, 1, 0, 0, 0, 4095, 26, 4095, 1, 1, 1849, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4095, 1, 2, 1, 0, 0, 4096, 0, 1, 4084, 1, 4095, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 4079, 1, 1, 1396, 1, 0, 1, 18, 4095, 0, 55, 1, 3513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 4095, 4068, 1, 4095, 1, 0, 2063, 0, 2014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1598, 0, 0, 1, 0, 0, 4090, 1, 0, 1935, 1, 4063, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 0, 3750, 0, 1, 0, 0, 4095, 1, 0, 4092, 1, 1, 1, 386, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3814, 1304, 0, 33, 0, 1, 0, 1, 0, 0, 9, 4091, 0, 33, 0, 2233, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 10, 4095, 0, 4095, 16, 0, 1, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16868, 16393, 17259, 16339, 15762, 16377, 15711, 16363] avg=16384.0000 max=17259 max_rank=2 min=15711 min_rank=6 imbalance_ratio=1.053406 max_load_reduction=113813 max_load_reduction_pct=0.868324 moved_to_echo=64213 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 2361, 4095, 1, 0, 1, 0, 0, 4091, 0, 4095, 1, 1, 1851, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4094, 1, 4, 1, 0, 0, 4095, 0, 1, 4074, 1, 4095, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 3774, 1, 1, 608, 1, 0, 0, 0, 4094, 0, 19, 1, 3512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 2, 0, 0, 4095, 4095, 1, 4095, 40, 0, 2945, 0, 2017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1590, 0, 0, 0, 0, 0, 4093, 1, 0, 2628, 0, 3957, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 1, 0, 3185, 0, 1, 0, 1, 4095, 0, 0, 4092, 1, 2, 1, 390, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4016, 946, 0, 15, 0, 4, 0, 1, 0, 0, 0, 4095, 0, 2, 1, 2236, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 110, 4095, 0, 4095, 57, 0, 1, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17081, 16403, 16109, 17298, 16362, 15810, 15509, 16500] avg=16384.0000 max=17298 max_rank=3 min=15509 min_rank=6 imbalance_ratio=1.055786 max_load_reduction=113774 max_load_reduction_pct=0.868027 moved_to_echo=64181 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 833, 4094, 1, 1, 1, 0, 0, 4095, 5, 4095, 3, 1, 1849, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4095, 1, 2, 1, 0, 0, 4095, 1, 1, 4087, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 3773, 1, 1, 371, 1, 0, 0, 6, 4095, 0, 1, 1, 3512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 1, 4095, 4051, 1, 4095, 0, 0, 471, 0, 2014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1598, 0, 0, 3, 0, 0, 4091, 1, 0, 3737, 0, 3973, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 0, 1, 0, 4049, 0, 1, 0, 1, 4095, 0, 0, 4092, 1, 1, 1, 386, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3795, 3695, 0, 219, 0, 1, 0, 1, 0, 0, 0, 4094, 1, 9, 0, 2233, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 23, 4095, 0, 4095, 25, 1, 1, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15562, 16392, 15860, 14766, 17249, 16617, 18240, 16386] avg=16384.0000 max=18240 max_rank=6 min=14766 min_rank=3 imbalance_ratio=1.113281 max_load_reduction=112832 max_load_reduction_pct=0.860840 moved_to_echo=68492 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32384, 128, 128, 0, 29696, 128, 128, 128, 128, 32768, 128, 0, 32768, 128, 128, 128, 3200, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 0, 13312, 32768, 128, 128, 128, 0, 128, 32768, 128, 32768, 128, 128, 14848, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 0, 32768, 128, 128, 32768, 128, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 128, 128, 31872, 128, 128, 5632, 128, 0, 128, 128, 32768, 128, 128, 128, 28160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 0, 384, 32768, 128, 32768, 128, 128, 128, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 128, 0, 128, 32768, 32640, 128, 32768, 128, 128, 16640, 0, 16128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 12800, 0, 0, 128, 128, 0, 32768, 128, 0, 21120, 128, 32384, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 28800, 15744, 128, 768, 128, 128, 0, 128, 128, 0, 128, 32768, 128, 1792, 128, 17920, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1456, 0, 0, 4094, 0, 0, 4090, 4085, 0, 0, 0, 2525, 0, 0, 0, 4096, 3, 4096, 0, 23, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 4096, 3888, 0, 4037, 4096, 2318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 4096, 0, 4, 1273, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 3990, 0, 2, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 4094, 0, 0, 14, 0, 4096, 0, 4096, 4096, 0, 0, 635, 0, 0, 4096, 0, 1, 3, 0, 0, 4096, 0, 0, 0, 4095, 0, 4059, 0, 4092, 4094, 4096, 0, 0, 0, 0, 4088, 4096, 6, 0, 4093, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1206, 0, 0, 4094, 0, 0, 4082, 4096, 0, 0, 0, 1962, 0, 0, 0, 4096, 3, 4096, 0, 225, 0, 4095, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 4096, 2288, 0, 4057, 4095, 3110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 4096, 0, 5, 2853, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3990, 0, 1, 0, 0, 0, 0, 324, 0, 1, 0, 0, 0, 0, 4096, 4093, 0, 4096, 0, 0, 18, 0, 4096, 0, 4096, 4096, 0, 0, 309, 0, 0, 4096, 0, 2, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 4012, 0, 4095, 4095, 4096, 0, 0, 0, 0, 4072, 4096, 17, 0, 4092, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[3529, 0, 0, 4095, 0, 0, 4095, 3733, 1, 0, 1, 79, 0, 0, 1, 4094, 19, 3536, 1, 1045, 0, 4094, 0, 0, 0, 1, 4095, 0, 86, 0, 0, 0, 4094, 3950, 0, 4095, 4095, 2853, 1, 1, 0, 1, 0, 1, 45, 0, 0, 1, 1, 0, 1, 0, 251, 1, 1, 1, 4095, 1, 0, 1584, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 3452, 263, 12, 1, 0, 0, 1, 8, 0, 0, 1, 1, 0, 1, 4095, 4095, 1, 4095, 0, 0, 0, 0, 4001, 1, 4086, 4095, 0, 0, 312, 1, 1, 4020, 1, 0, 0, 1, 1, 4095, 1, 0, 4, 4093, 0, 4092, 2, 4095, 4095, 4094, 0, 0, 0, 1, 4095, 4081, 0, 1, 4095, 6, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[584, 0, 0, 4095, 0, 0, 4082, 4096, 0, 0, 0, 3051, 0, 0, 0, 4096, 1, 4096, 0, 1012, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 4096, 2767, 0, 4071, 4095, 3306, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 4096, 0, 3, 1335, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4087, 2, 0, 0, 0, 0, 0, 74, 1, 1, 0, 0, 0, 0, 4096, 4091, 0, 4094, 0, 0, 69, 0, 4096, 0, 4096, 4096, 0, 1, 79, 0, 0, 4096, 0, 4, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4037, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4072, 4096, 14, 0, 4093, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1273, 0, 0, 4094, 0, 1, 4095, 3320, 1, 0, 1, 1323, 0, 0, 1, 4093, 25, 3659, 3, 1664, 0, 4094, 0, 0, 1, 1, 4095, 0, 83, 0, 0, 0, 4093, 3936, 1, 4095, 4095, 3254, 1, 0, 0, 1, 0, 1, 50, 0, 0, 1, 1, 0, 1, 0, 28, 1, 1, 1, 4095, 1, 0, 1046, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 4090, 429, 5, 1, 0, 0, 1, 8, 0, 0, 1, 1, 0, 1, 4095, 4095, 1, 4094, 0, 0, 0, 0, 4027, 1, 4087, 4094, 2, 0, 501, 1, 1, 4066, 0, 0, 0, 1, 1, 4093, 1, 0, 2, 4094, 0, 4095, 0, 4095, 4095, 4094, 0, 0, 0, 1, 4095, 4075, 0, 1, 4095, 4, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[2274, 1, 2, 4095, 1, 0, 4095, 3672, 0, 0, 1, 436, 1, 0, 1, 4093, 56, 3890, 12, 634, 0, 4095, 0, 0, 0, 1, 4095, 0, 95, 0, 0, 0, 4094, 3071, 1, 4095, 4095, 3725, 1, 1, 0, 1, 0, 1, 8, 0, 0, 1, 1, 0, 1, 0, 498, 1, 0, 1, 4095, 1, 0, 1612, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 3966, 57, 31, 1, 0, 0, 1, 39, 0, 0, 1, 1, 0, 1, 4095, 4095, 0, 4095, 0, 0, 1, 0, 3926, 1, 4095, 4095, 1, 0, 738, 1, 1, 4056, 1, 0, 0, 1, 0, 4095, 0, 0, 1, 4094, 0, 4092, 0, 4095, 4095, 4095, 0, 0, 0, 1, 4095, 4058, 3, 1, 4095, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[1211, 0, 0, 4093, 0, 0, 4089, 3997, 0, 0, 0, 3651, 0, 0, 0, 4096, 3, 4096, 0, 384, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 2, 0, 0, 4096, 4020, 0, 4084, 4096, 2794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 4096, 0, 5, 168, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 2, 2, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 4096, 0, 0, 29, 0, 4096, 0, 4096, 4096, 0, 0, 101, 0, 0, 4096, 0, 2, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 4051, 0, 4092, 4094, 4096, 0, 0, 0, 0, 4065, 4096, 6, 0, 4090, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[499, 1, 1, 4095, 0, 0, 4095, 3505, 1, 0, 1, 1506, 0, 0, 1, 4092, 30, 3746, 10, 1157, 0, 4094, 0, 0, 1, 1, 4095, 0, 72, 0, 0, 0, 4094, 3929, 1, 4095, 4095, 3899, 0, 1, 0, 1, 0, 1, 83, 0, 0, 1, 1, 0, 1, 0, 62, 1, 0, 1, 4095, 1, 0, 936, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 4091, 468, 10, 1, 0, 0, 1, 71, 0, 0, 1, 1, 0, 1, 4095, 4094, 1, 4091, 0, 0, 2, 0, 4011, 1, 4085, 4095, 2, 0, 658, 1, 1, 4043, 1, 0, 0, 1, 1, 4094, 1, 0, 1, 4094, 0, 4095, 0, 4095, 4095, 4095, 0, 0, 3, 1, 4095, 4079, 0, 1, 4095, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1456, 0, 0, 4094, 0, 0, 4090, 4085, 0, 0, 0, 2525, 0, 0, 0, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 23, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 2549, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3888, 0, 4037, 1547, 2318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 0, 0, 0, 4096, 0, 4, 1273, 0, 0, 0, 4096, 4054, 2467, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3990, 0, 2, 0, 0, 0, 0, 42, 0, 0, 0, 4096, 4096, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4094, 0, 0, 14, 0, 4096, 0, 4096, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 635, 0, 0, 4096, 0, 1, 3, 0, 0, 4096, 0, 0, 0, 4095, 0, 2120, 1579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 46, 0, 0, 0, 0, 0, 0, 4088, 4096, 6, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17315, 15773, 15886, 16445, 16205, 16435, 16625, 16388] avg=16384.0000 max=17315 max_rank=0 min=15773 min_rank=1 imbalance_ratio=1.056824 max_load_reduction=113757 max_load_reduction_pct=0.867897 moved_to_echo=65653 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1206, 0, 0, 4094, 0, 0, 4082, 4096, 0, 0, 0, 1962, 0, 0, 0, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 225, 0, 4095, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 2549, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2288, 0, 4057, 1546, 3110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 4096, 0, 5, 2853, 0, 0, 0, 4096, 4054, 2468, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 0, 1, 0, 0, 0, 0, 324, 0, 1, 0, 4096, 4096, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 18, 0, 4096, 0, 4096, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 0, 4096, 0, 2, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 2120, 1580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 47, 0, 0, 0, 0, 0, 0, 4072, 4096, 17, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16505, 15974, 15097, 17932, 16486, 16441, 16301, 16336] avg=16384.0000 max=17932 max_rank=3 min=15097 min_rank=2 imbalance_ratio=1.094482 max_load_reduction=113140 max_load_reduction_pct=0.863190 moved_to_echo=65564 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1273, 0, 0, 4094, 0, 1, 4095, 3320, 1, 0, 1, 1323, 0, 0, 1, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 3659, 3, 1664, 0, 4094, 0, 0, 1, 1, 4095, 0, 83, 0, 0, 0, 2549, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3936, 1, 4095, 1546, 3254, 1, 0, 0, 1, 0, 1, 50, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 28, 1, 1, 1, 4095, 1, 0, 1046, 0, 0, 1, 4095, 4052, 2468, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4090, 429, 5, 1, 0, 0, 1, 8, 0, 0, 1, 4095, 4094, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4095, 1, 4094, 0, 0, 0, 0, 4027, 1, 4087, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 1, 1, 4066, 0, 0, 0, 1, 1, 4093, 1, 0, 2, 4094, 0, 2119, 1580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 47, 0, 0, 0, 0, 0, 1, 4095, 4075, 0, 1, 4095, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15172, 17085, 16979, 15908, 16703, 16351, 16460, 16414] avg=16384.0000 max=17085 max_rank=1 min=15172 min_rank=0 imbalance_ratio=1.042786 max_load_reduction=113987 max_load_reduction_pct=0.869652 moved_to_echo=65928 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3529, 0, 0, 4095, 0, 0, 4095, 3733, 1, 0, 1, 79, 0, 0, 1, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 3536, 1, 1045, 0, 4094, 0, 0, 0, 1, 4095, 0, 86, 0, 0, 0, 2551, 915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3950, 0, 4095, 1544, 2853, 1, 1, 0, 1, 0, 1, 45, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 251, 1, 1, 1, 4095, 1, 0, 1584, 0, 0, 1, 4095, 4059, 2473, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3452, 263, 12, 1, 0, 0, 1, 8, 0, 0, 1, 4095, 4094, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4095, 1, 4095, 0, 0, 0, 0, 4001, 1, 4086, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 1, 1, 4020, 1, 0, 0, 1, 1, 4095, 1, 0, 4, 4093, 0, 2123, 1586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 36, 0, 0, 0, 0, 0, 1, 4095, 4081, 0, 1, 4095, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16590, 16343, 16586, 16682, 15905, 16317, 16239, 16410] avg=16384.0000 max=16682 max_rank=3 min=15905 min_rank=4 imbalance_ratio=1.018188 max_load_reduction=114390 max_load_reduction_pct=0.872726 moved_to_echo=64871 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[584, 0, 0, 4095, 0, 0, 4082, 4096, 0, 0, 0, 3051, 0, 0, 0, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 1012, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 2549, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2767, 0, 4071, 1546, 3306, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 4096, 0, 3, 1335, 0, 0, 0, 4095, 4054, 2468, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 2, 0, 0, 0, 0, 0, 74, 1, 1, 0, 4096, 4096, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4094, 0, 0, 69, 0, 4096, 0, 4096, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 79, 0, 0, 4096, 0, 4, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 2120, 1580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 47, 0, 0, 0, 0, 0, 0, 4072, 4096, 14, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16973, 16760, 15787, 16296, 16336, 16488, 16073, 16359] avg=16384.0000 max=16973 max_rank=0 min=15787 min_rank=2 imbalance_ratio=1.035950 max_load_reduction=114099 max_load_reduction_pct=0.870506 moved_to_echo=65256 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1211, 0, 0, 4093, 0, 0, 4089, 3997, 0, 0, 0, 3651, 0, 0, 0, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 384, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 2, 0, 0, 2549, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4020, 0, 4084, 1547, 2794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 4096, 0, 5, 168, 0, 0, 0, 4096, 4054, 2467, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 2, 0, 0, 0, 0, 96, 0, 0, 0, 4096, 4096, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 29, 0, 4096, 0, 4096, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 4096, 0, 2, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 2120, 1579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 46, 0, 0, 0, 0, 0, 0, 4065, 4096, 6, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18106, 16134, 16541, 15032, 16361, 16451, 16093, 16354] avg=16384.0000 max=18106 max_rank=0 min=15032 min_rank=3 imbalance_ratio=1.105103 max_load_reduction=112966 max_load_reduction_pct=0.861862 moved_to_echo=65259 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2274, 1, 2, 4095, 1, 0, 4095, 3672, 0, 0, 1, 436, 1, 0, 1, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 3890, 12, 634, 0, 4095, 0, 0, 0, 1, 4095, 0, 95, 0, 0, 0, 2549, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3071, 1, 4095, 1546, 3725, 1, 1, 0, 1, 0, 1, 8, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 498, 1, 0, 1, 4095, 1, 0, 1612, 0, 0, 1, 4095, 4053, 2468, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3966, 57, 31, 1, 0, 0, 1, 39, 0, 0, 1, 4095, 4095, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4095, 0, 4095, 0, 0, 1, 0, 3926, 1, 4095, 42, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 738, 1, 1, 4056, 1, 0, 0, 1, 0, 4095, 0, 0, 1, 4094, 0, 2119, 1580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 47, 0, 0, 0, 0, 0, 1, 4095, 4058, 3, 1, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15642, 16338, 16545, 16944, 16265, 16258, 16687, 16393] avg=16384.0000 max=16944 max_rank=3 min=15642 min_rank=0 imbalance_ratio=1.034180 max_load_reduction=114128 max_load_reduction_pct=0.870728 moved_to_echo=65603 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[499, 1, 1, 4095, 0, 0, 4095, 3505, 1, 0, 1, 1506, 0, 0, 1, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 3746, 10, 1157, 0, 4094, 0, 0, 1, 1, 4095, 0, 72, 0, 0, 0, 2549, 910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3929, 1, 4095, 1546, 3899, 0, 1, 0, 1, 0, 1, 83, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 62, 1, 0, 1, 4095, 1, 0, 936, 0, 0, 1, 4095, 4053, 2468, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4091, 468, 10, 1, 0, 0, 1, 71, 0, 0, 1, 4095, 4095, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4094, 1, 4091, 0, 0, 2, 0, 4011, 1, 4085, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 658, 1, 1, 4043, 1, 0, 0, 1, 1, 4094, 1, 0, 1, 4094, 0, 2118, 1580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 47, 0, 0, 0, 0, 3, 1, 4095, 4079, 0, 1, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14769, 16665, 17651, 15833, 16811, 16331, 16594, 16418] avg=16384.0000 max=17651 max_rank=2 min=14769 min_rank=0 imbalance_ratio=1.077332 max_load_reduction=113421 max_load_reduction_pct=0.865334 moved_to_echo=66154 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[12032, 128, 128, 32768, 128, 128, 32768, 30592, 128, 0, 128, 14592, 128, 0, 128, 8576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 27904, 128, 32640, 12416, 25344, 128, 128, 0, 128, 0, 128, 256, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[256, 31232, 128, 6144, 0, 32768, 0, 0, 128, 128, 32768, 0, 384, 128, 0, 0, 20480, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 32768, 0, 0, 256, 0, 32384, 128, 32768, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 3456, 128, 128, 32640, 128, 128, 128, 128, 128, 32768, 128, 0, 128, 32768, 0, 17024, 12672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 0, 1664, 128, 128, 128, 32768, 128, 128, 10880, 0, 0, 128, 32768, 32512, 19840, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32640, 128, 384, 0, 0, 0, 0, 128, 128, 32768, 32768, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 0, 0, 128, 31872, 1280, 128, 128, 0, 0, 128, 768, 128, 128, 128, 32768, 32768, 31872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1, 1, 0, 12, 312, 2, 0, 1, 0, 0, 0, 18, 4008, 1, 0, 0, 4086, 0, 0, 4095, 4095, 4095, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 4095, 1, 130, 1, 1, 0, 0, 295, 0, 4095, 0, 1, 1, 1, 0, 0, 14, 4095, 1, 1928, 1, 4095, 2270, 1, 0, 0, 4095, 1, 1, 4094, 33, 0, 3942, 0, 7, 0, 1, 1, 0, 0, 0, 4083, 0, 0, 357, 1, 0, 1, 1, 1, 0, 4094, 1, 4095, 0, 4082, 0, 3, 1, 0, 4095, 4095, 4095, 2807, 95, 0, 0, 202, 0, 1, 4095, 0, 1, 0, 0, 4093, 1, 4095, 1, 1, 1, 4095, 1, 4079, 0, 0, 0, 1, 0, 4095, 4095, 4095, 4094, 1, 1, 0, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4081, 4096, 0, 0, 0, 4096, 0, 2, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 354, 0, 4072, 0, 0, 0, 0, 22, 69, 0, 4096, 0, 3467, 0, 4096, 1560, 0, 0, 0, 4095, 0, 0, 4096, 50, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 784, 0, 0, 0, 0, 0, 0, 4096, 0, 4049, 0, 4094, 0, 0, 0, 2, 4095, 4091, 4096, 1791, 0, 0, 0, 276, 1, 0, 4095, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 4096, 0, 4087, 0, 0, 1, 0, 0, 4041, 4096, 4067, 4095, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4084, 4095, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 348, 0, 4093, 0, 0, 0, 0, 20, 30, 0, 4096, 0, 1752, 0, 4096, 1820, 0, 0, 0, 4094, 0, 0, 4096, 161, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 847, 0, 0, 0, 0, 0, 0, 4096, 0, 4080, 0, 4096, 0, 0, 0, 2, 4094, 4091, 4096, 3220, 0, 0, 0, 66, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 4096, 0, 4088, 0, 0, 2, 0, 0, 4077, 4096, 4082, 4094, 0, 0, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 0, 13, 143, 2, 0, 1, 0, 0, 0, 25, 4000, 1, 1, 0, 4085, 0, 0, 4095, 4095, 4095, 0, 0, 0, 4095, 1, 0, 0, 1, 0, 4095, 1, 104, 1, 1, 0, 0, 187, 0, 4095, 0, 1, 1, 0, 0, 0, 19, 4095, 1, 1656, 1, 4095, 2617, 1, 0, 0, 4095, 1, 1, 4093, 1922, 0, 3927, 0, 3, 0, 1, 1, 0, 1, 0, 4087, 0, 0, 264, 1, 0, 1, 1, 1, 0, 4095, 1, 4095, 0, 4070, 1, 3, 1, 0, 4095, 4095, 4095, 1245, 174, 0, 0, 138, 0, 5, 4095, 0, 1, 0, 0, 4094, 1, 4094, 1, 1, 1, 4095, 1, 4079, 1, 0, 0, 0, 0, 4095, 4095, 4095, 4093, 0, 0, 0, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 0, 3, 200, 2, 0, 1, 0, 0, 1, 21, 4031, 1, 1, 0, 4092, 0, 0, 4095, 4095, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 2, 4095, 1, 98, 0, 1, 0, 0, 55, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 4095, 1, 2076, 1, 4095, 2715, 1, 0, 0, 4095, 1, 1, 4089, 337, 0, 3884, 0, 13, 0, 0, 1, 0, 1, 0, 4093, 0, 2, 235, 0, 1, 1, 1, 1, 0, 4093, 1, 4095, 0, 4084, 1, 10, 0, 0, 4095, 4092, 4095, 2568, 104, 0, 0, 92, 0, 0, 4095, 0, 1, 0, 0, 4094, 1, 4095, 1, 1, 1, 4095, 1, 4072, 1, 0, 0, 0, 0, 4095, 4095, 4094, 4091, 1, 1, 0, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 1, 4096, 4094, 4094, 0, 0, 0, 4096, 0, 3, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 840, 0, 4078, 0, 0, 0, 0, 9, 25, 0, 4096, 0, 1639, 0, 4096, 1514, 0, 0, 0, 4095, 0, 0, 4096, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1151, 0, 0, 0, 0, 0, 0, 4095, 0, 4056, 0, 4095, 0, 1, 0, 1, 4095, 4092, 4096, 3128, 0, 0, 0, 35, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 4096, 0, 4092, 0, 1, 0, 0, 0, 4074, 4096, 4030, 4095, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 4096, 4082, 4096, 0, 0, 0, 4096, 0, 3, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 385, 0, 4088, 0, 0, 0, 0, 7, 18, 0, 4096, 0, 1127, 0, 4096, 1161, 0, 0, 0, 4095, 0, 0, 4096, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1880, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 4088, 0, 0, 0, 1, 4094, 4095, 4096, 3688, 0, 0, 0, 89, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 4096, 0, 4094, 0, 0, 1, 0, 0, 4078, 4096, 3977, 4093, 0, 0, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 0, 16, 228, 1, 0, 1, 0, 0, 0, 22, 4027, 1, 1, 0, 4089, 0, 0, 4095, 4095, 4095, 0, 0, 0, 4095, 1, 0, 0, 0, 3, 4095, 1, 122, 1, 1, 0, 0, 395, 0, 4095, 0, 1, 1, 1, 1, 0, 20, 4095, 1, 2333, 1, 4095, 1933, 1, 0, 0, 4095, 1, 1, 4093, 16, 0, 3949, 0, 9, 0, 0, 1, 0, 1, 0, 4091, 0, 1, 149, 0, 0, 1, 1, 1, 0, 4092, 1, 4095, 0, 4074, 1, 9, 1, 2, 4095, 4095, 4095, 2935, 19, 0, 0, 257, 0, 0, 4095, 0, 1, 0, 0, 4093, 1, 4095, 1, 1, 1, 4095, 1, 4080, 1, 0, 0, 0, 0, 4095, 4095, 4095, 4091, 0, 0, 0, 4086] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4084, 4095, 0, 0, 0, 8, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 0, 4093, 0, 0, 0, 0, 20, 30, 0, 4096, 4096, 3568, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1752, 0, 0, 1820, 0, 0, 0, 4013, 0, 0, 4096, 161, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 847, 0, 0, 0, 0, 4094, 4088, 2865, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1396, 0, 4080, 0, 4096, 0, 0, 0, 2, 0, 4091, 0, 3220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 2700, 1171, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 2, 0, 0, 4077, 0, 4082, 58, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16296, 16382, 16339, 15938, 16521, 16885, 16306, 16405] avg=16384.0000 max=16885 max_rank=5 min=15938 min_rank=3 imbalance_ratio=1.030579 max_load_reduction=114187 max_load_reduction_pct=0.871178 moved_to_echo=66117 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 12, 312, 2, 0, 1, 0, 0, 0, 18, 4008, 1, 0, 0, 4095, 4095, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4095, 4095, 0, 0, 1, 8, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 130, 1, 1, 0, 0, 295, 0, 4095, 0, 1, 1, 1, 0, 0, 14, 4095, 4095, 3566, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1928, 1, 0, 2270, 1, 0, 0, 4014, 1, 1, 4094, 33, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 1, 0, 0, 0, 4083, 0, 0, 357, 1, 0, 1, 1, 4095, 4087, 2865, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1396, 1, 4095, 0, 4082, 0, 3, 1, 0, 0, 4095, 0, 2807, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 1, 4095, 0, 1, 0, 0, 4093, 1, 4095, 1, 1, 1, 2698, 1171, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4079, 0, 0, 0, 1, 0, 4095, 0, 4095, 58, 1, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16552, 16381, 16385, 16286, 16028, 16576, 16441, 16423] avg=16384.0000 max=16576 max_rank=5 min=16028 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=114496 max_load_reduction_pct=0.873535 moved_to_echo=65468 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4081, 4096, 0, 0, 0, 8, 0, 2, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 4072, 0, 0, 0, 0, 22, 69, 0, 4096, 4096, 3568, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3467, 0, 0, 1560, 0, 0, 0, 4014, 0, 0, 4096, 50, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 784, 0, 0, 0, 0, 4095, 4088, 2866, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1396, 0, 4049, 0, 4094, 0, 0, 0, 2, 0, 4091, 0, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 1, 0, 4095, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 2700, 1171, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 1, 0, 0, 4041, 0, 4067, 58, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16296, 16381, 16365, 17283, 16459, 15423, 16514, 16351] avg=16384.0000 max=17283 max_rank=3 min=15423 min_rank=5 imbalance_ratio=1.054871 max_load_reduction=113789 max_load_reduction_pct=0.868141 moved_to_echo=64747 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 1, 0, 4094, 4094, 0, 0, 0, 8, 0, 3, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 840, 0, 4078, 0, 0, 0, 0, 9, 25, 0, 4096, 4096, 3568, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1639, 0, 0, 1514, 0, 0, 0, 4014, 0, 0, 4096, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1151, 0, 0, 0, 0, 4095, 4088, 2866, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1396, 0, 4056, 0, 4095, 0, 1, 0, 1, 0, 4092, 0, 3128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 2699, 1171, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 0, 0, 0, 4074, 0, 4030, 58, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16296, 16393, 16800, 15366, 16824, 16769, 16272, 16352] avg=16384.0000 max=16824 max_rank=4 min=15366 min_rank=3 imbalance_ratio=1.026855 max_load_reduction=114248 max_load_reduction_pct=0.871643 moved_to_echo=66217 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4082, 4096, 0, 0, 0, 8, 0, 3, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 0, 4088, 0, 0, 0, 0, 7, 18, 0, 4096, 4096, 3568, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1127, 0, 0, 1161, 0, 0, 0, 4014, 0, 0, 4096, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1880, 0, 0, 0, 0, 4094, 4088, 2864, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1396, 0, 4092, 0, 4088, 0, 0, 0, 1, 0, 4095, 0, 3688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 2700, 1171, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 4078, 0, 3977, 58, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16295, 16384, 16347, 14505, 17550, 17360, 16325, 16306] avg=16384.0000 max=17550 max_rank=4 min=14505 min_rank=3 imbalance_ratio=1.071167 max_load_reduction=113522 max_load_reduction_pct=0.866104 moved_to_echo=67541 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 13, 143, 2, 0, 1, 0, 0, 0, 25, 4000, 1, 1, 0, 4095, 4095, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4095, 4095, 0, 0, 0, 6, 1, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 104, 1, 1, 0, 0, 187, 0, 4095, 0, 1, 1, 0, 0, 0, 19, 4095, 4095, 3567, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1656, 1, 0, 2617, 1, 0, 0, 4011, 1, 1, 4093, 1922, 0, 3927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 1, 0, 1, 0, 4087, 0, 0, 264, 1, 0, 1, 1, 4095, 4089, 2869, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1394, 1, 4095, 0, 4070, 1, 3, 1, 0, 0, 4095, 0, 1245, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 5, 4095, 0, 1, 0, 0, 4094, 1, 4094, 1, 1, 1, 2701, 1175, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4079, 1, 0, 0, 0, 0, 4095, 0, 4095, 49, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16378, 16255, 18231, 15941, 15080, 16391, 16412] avg=16384.0000 max=18231 max_rank=3 min=15080 min_rank=5 imbalance_ratio=1.112732 max_load_reduction=112841 max_load_reduction_pct=0.860909 moved_to_echo=63824 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3, 200, 2, 0, 1, 0, 0, 1, 21, 4031, 1, 1, 0, 4095, 4095, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4095, 4095, 0, 0, 0, 8, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 98, 0, 1, 0, 0, 55, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 4095, 4095, 3566, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2076, 1, 0, 2715, 1, 0, 0, 4014, 1, 1, 4089, 337, 0, 3884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 1, 0, 1, 0, 4093, 0, 2, 235, 0, 1, 1, 1, 4095, 4087, 2863, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1395, 1, 4095, 0, 4084, 1, 10, 0, 0, 0, 4092, 0, 2568, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 4095, 0, 1, 0, 0, 4094, 1, 4095, 1, 1, 1, 2698, 1170, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4072, 1, 0, 0, 0, 0, 4095, 0, 4094, 58, 1, 1, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16458, 16387, 16097, 17120, 15922, 16351, 16330, 16407] avg=16384.0000 max=17120 max_rank=3 min=15922 min_rank=4 imbalance_ratio=1.044922 max_load_reduction=113952 max_load_reduction_pct=0.869385 moved_to_echo=65010 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 16, 228, 1, 0, 1, 0, 0, 0, 22, 4027, 1, 1, 0, 4095, 4095, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4095, 4095, 0, 0, 0, 8, 1, 0, 0, 0, 3, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 122, 1, 1, 0, 0, 395, 0, 4095, 0, 1, 1, 1, 1, 0, 20, 4095, 4095, 3567, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2333, 1, 0, 1933, 1, 0, 0, 4014, 1, 1, 4093, 16, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 0, 1, 0, 4091, 0, 1, 149, 0, 0, 1, 1, 4095, 4087, 2863, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1395, 1, 4095, 0, 4074, 1, 9, 1, 2, 0, 4095, 0, 2935, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 4095, 0, 1, 0, 0, 4093, 1, 4095, 1, 1, 1, 2697, 1170, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4080, 1, 0, 0, 0, 0, 4095, 0, 4095, 58, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16495, 16386, 16484, 16343, 15827, 16628, 16493, 16416] avg=16384.0000 max=16628 max_rank=5 min=15827 min_rank=4 imbalance_ratio=1.014893 max_load_reduction=114444 max_load_reduction_pct=0.873138 moved_to_echo=65364 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 0, 32768, 32768, 0, 0, 128, 128, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 11264, 128, 32768, 0, 32768, 128, 128, 128, 128, 0, 32768, 0, 21504, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 128, 0, 32768, 0, 32640, 512, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 0, 32768, 0, 128, 5760, 128, 128, 128, 128, 32768, 32768, 23040, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 896, 128, 0, 128, 0, 0, 128, 128, 32512, 128, 128, 0, 32768, 32768, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 1280, 128, 128, 32768, 0, 128, 0, 0, 32768, 128, 32768, 128, 128, 128, 21632, 9472, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 16000, 128, 0, 15616, 128, 0, 0, 32128, 128, 128, 32768, 2560, 0, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 512, 128, 128, 0, 0, 2944, 0, 32768, 0, 128, 128, 128, 128, 256, 128, 32768, 32768, 28544, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4095, 1, 0, 4068, 2683, 4095, 0, 4095, 4094, 4077, 1, 4095, 4081, 1, 4091, 1, 0, 0, 0, 68, 0, 0, 5, 4092, 0, 143, 0, 7, 4093, 1, 0, 0, 4095, 1, 1, 0, 0, 57, 1, 1, 1, 4077, 3022, 0, 0, 4022, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 27, 1, 1, 0, 4082, 1, 4095, 0, 1, 0, 0, 1, 0, 4047, 3638, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4080, 0, 0, 1, 0, 1, 4095, 0, 4092, 0, 20, 1, 3792, 0, 3315, 4068, 3, 177, 42, 0, 1, 3, 305, 3856, 0, 0, 0, 0, 1, 0, 0, 111, 2, 1, 1, 1, 3559, 4095, 4095, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 3785, 4093, 0, 4096, 4096, 4096, 0, 4096, 4088, 0, 4096, 0, 0, 0, 0, 170, 0, 0, 0, 4094, 0, 1526, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 270, 0, 0, 0, 4096, 272, 0, 2, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 2, 0, 0, 0, 4096, 4096, 21, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 21, 0, 0, 2638, 0, 3712, 4096, 0, 0, 9, 0, 0, 0, 2, 4092, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 4094, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4096, 3370, 4095, 0, 4096, 4096, 4096, 0, 4096, 4092, 0, 4096, 0, 0, 0, 0, 624, 0, 0, 0, 4094, 0, 401, 0, 1, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 547, 0, 0, 0, 4096, 772, 0, 3, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 4096, 20, 0, 0, 0, 0, 0, 1, 4095, 6, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 24, 0, 0, 2598, 0, 3945, 4096, 0, 0, 22, 0, 0, 0, 20, 4086, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 4093, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4095, 1, 0, 4080, 3846, 4095, 0, 4095, 4094, 4079, 1, 4095, 4095, 1, 4095, 1, 0, 0, 0, 70, 0, 0, 5, 4090, 0, 1393, 0, 9, 4091, 1, 0, 0, 4095, 1, 1, 0, 0, 466, 1, 1, 1, 4080, 1302, 0, 0, 4010, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 19, 1, 1, 0, 4082, 1, 4095, 0, 1, 0, 0, 1, 0, 4045, 3641, 0, 1, 0, 0, 0, 5, 0, 4095, 0, 0, 0, 0, 1, 4082, 0, 0, 1, 0, 1, 4095, 0, 4090, 0, 13, 1, 3794, 0, 2241, 4069, 0, 116, 61, 0, 1, 2, 29, 3652, 0, 0, 0, 0, 0, 0, 0, 123, 3, 1, 1, 1, 4029, 4095, 4095, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4096, 3371, 4095, 0, 4096, 4096, 4096, 0, 4096, 4077, 0, 4096, 0, 0, 0, 0, 1163, 0, 0, 0, 4094, 0, 1765, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 183, 0, 0, 0, 4096, 1343, 0, 2, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 2, 0, 0, 0, 4096, 4096, 12, 0, 0, 0, 0, 0, 1, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 20, 0, 0, 698, 0, 3810, 4096, 0, 0, 80, 0, 0, 0, 22, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 4090, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1, 1, 0, 1, 4095, 0, 0, 4083, 3842, 4095, 0, 4095, 4094, 4081, 1, 4094, 4081, 1, 4093, 1, 0, 0, 0, 260, 0, 0, 1, 4075, 0, 580, 0, 5, 4095, 1, 0, 0, 4095, 1, 1, 0, 0, 175, 1, 1, 1, 4091, 1642, 0, 0, 4024, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 20, 1, 1, 1, 4092, 1, 4095, 0, 1, 0, 0, 1, 0, 4064, 3765, 30, 1, 0, 0, 0, 6, 0, 4095, 0, 0, 0, 0, 1, 4087, 0, 0, 1, 0, 1, 4095, 0, 4088, 0, 18, 1, 3687, 0, 2768, 4069, 0, 120, 144, 0, 1, 1, 66, 3928, 0, 0, 0, 0, 0, 0, 0, 79, 3, 1, 1, 1, 3644, 4095, 4095, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4095, 1, 0, 4069, 2768, 4095, 0, 4095, 4094, 4076, 1, 4095, 4092, 1, 4093, 1, 0, 0, 0, 191, 0, 0, 2, 4085, 0, 577, 0, 11, 4094, 1, 0, 0, 4095, 1, 1, 0, 0, 47, 1, 1, 1, 4091, 2283, 0, 0, 4028, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 24, 1, 1, 0, 4084, 1, 4095, 0, 1, 0, 0, 1, 0, 4061, 3852, 1, 1, 0, 0, 0, 4, 0, 4095, 0, 1, 0, 0, 1, 4083, 0, 0, 1, 0, 1, 4095, 0, 4090, 0, 21, 1, 3210, 0, 3885, 4067, 0, 149, 37, 0, 1, 2, 124, 3835, 0, 0, 1, 0, 0, 0, 0, 136, 2, 1, 1, 1, 3631, 4095, 4095, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4096, 3811, 4096, 0, 4096, 4096, 4096, 0, 4096, 4077, 0, 4096, 0, 0, 0, 0, 148, 0, 0, 0, 4094, 0, 614, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 156, 0, 0, 0, 4096, 791, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 4094, 0, 4091, 0, 0, 3, 0, 0, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 1, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 13, 0, 0, 2847, 0, 3933, 4096, 0, 0, 18, 0, 0, 0, 6, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 4094, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 3785, 681, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 170, 0, 0, 0, 4094, 0, 1526, 0, 0, 2896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 270, 0, 0, 0, 4096, 272, 0, 1605, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2, 0, 0, 0, 4096, 4096, 21, 0, 0, 0, 0, 0, 4096, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 2638, 0, 3712, 4096, 0, 0, 9, 0, 0, 0, 2, 4092, 0, 1773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 4094, 4096, 0, 0, 3903, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16754, 16870, 15265, 16413, 16599, 16393, 16343, 16435] avg=16384.0000 max=16870 max_rank=1 min=15265 min_rank=2 imbalance_ratio=1.029663 max_load_reduction=114202 max_load_reduction_pct=0.871292 moved_to_echo=65770 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4096, 3370, 682, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 624, 0, 0, 0, 4094, 0, 401, 0, 1, 2896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 547, 0, 0, 0, 4096, 772, 0, 1605, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 4096, 20, 0, 0, 0, 0, 0, 4096, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 6, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 2598, 0, 3945, 4096, 0, 0, 22, 0, 0, 0, 20, 4086, 0, 1773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 4093, 4096, 0, 0, 3904, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16339, 16204, 16041, 16415, 16598, 16397, 16564, 16514] avg=16384.0000 max=16598 max_rank=4 min=16041 min_rank=2 imbalance_ratio=1.013062 max_load_reduction=114474 max_load_reduction_pct=0.873367 moved_to_echo=66073 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4095, 1, 0, 4068, 2683, 682, 0, 0, 0, 4077, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 1, 4091, 1, 0, 0, 0, 68, 0, 0, 5, 4092, 0, 143, 0, 7, 2895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 4095, 1, 1, 0, 0, 57, 1, 1, 1, 4077, 3022, 0, 1605, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 27, 1, 1, 0, 4082, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 0, 4047, 3638, 0, 0, 1, 0, 0, 0, 4095, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4080, 0, 0, 1, 0, 1, 4095, 0, 4092, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1, 3792, 0, 3315, 4068, 3, 177, 42, 0, 1, 3, 305, 3856, 0, 1773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 111, 2, 1, 1, 1, 3559, 4095, 4095, 1, 0, 3903, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15610, 15384, 17783, 16356, 16071, 16371, 17356, 16141] avg=16384.0000 max=17783 max_rank=2 min=15384 min_rank=1 imbalance_ratio=1.085388 max_load_reduction=113289 max_load_reduction_pct=0.864326 moved_to_echo=65939 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4095, 1, 0, 4080, 3846, 667, 0, 0, 0, 4079, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4095, 1, 0, 0, 0, 70, 0, 0, 5, 4090, 0, 1393, 0, 9, 2896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 4095, 1, 1, 0, 0, 466, 1, 1, 1, 4080, 1302, 0, 1611, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 19, 1, 1, 0, 4082, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 0, 4045, 3641, 0, 1, 0, 0, 0, 5, 4095, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4082, 0, 0, 1, 0, 1, 4095, 0, 4090, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 3794, 0, 2241, 4069, 0, 116, 61, 0, 1, 2, 29, 3652, 0, 1779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 3, 1, 1, 1, 4029, 4095, 4095, 1, 0, 3902, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16771, 16655, 16479, 16337, 16078, 16373, 15758, 16621] avg=16384.0000 max=16771 max_rank=0 min=15758 min_rank=6 imbalance_ratio=1.023621 max_load_reduction=114301 max_load_reduction_pct=0.872047 moved_to_echo=64830 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4096, 3371, 682, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4096, 0, 0, 0, 0, 1163, 0, 0, 0, 4094, 0, 1765, 0, 0, 2898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 183, 0, 0, 0, 4096, 1343, 0, 1605, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2, 0, 0, 0, 4096, 4096, 12, 0, 0, 0, 0, 0, 4096, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 698, 0, 3810, 4096, 0, 0, 80, 0, 0, 0, 22, 4090, 0, 1773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 4090, 4096, 0, 0, 3903, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16339, 18093, 16247, 16415, 16590, 16394, 14589, 16405] avg=16384.0000 max=18093 max_rank=1 min=14589 min_rank=6 imbalance_ratio=1.104309 max_load_reduction=112979 max_load_reduction_pct=0.861961 moved_to_echo=63978 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4096, 3811, 681, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4096, 0, 0, 0, 0, 148, 0, 0, 0, 4094, 0, 614, 0, 0, 2896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 156, 0, 0, 0, 4096, 791, 0, 1606, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 4094, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 3, 0, 0, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 4096, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 2847, 0, 3933, 4096, 0, 0, 18, 0, 0, 0, 6, 4094, 0, 1774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 4094, 4096, 0, 0, 3904, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16778, 15925, 15671, 16413, 16580, 16397, 16781, 16527] avg=16384.0000 max=16781 max_rank=6 min=15671 min_rank=2 imbalance_ratio=1.024231 max_load_reduction=114291 max_load_reduction_pct=0.871971 moved_to_echo=66285 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 1, 4095, 0, 0, 4083, 3842, 682, 0, 0, 0, 4081, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 1, 4093, 1, 0, 0, 0, 260, 0, 0, 1, 4075, 0, 580, 0, 5, 2895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4095, 1, 1, 0, 0, 175, 1, 1, 1, 4091, 1642, 0, 1605, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 20, 1, 1, 1, 4092, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 0, 4064, 3765, 30, 1, 0, 0, 0, 6, 4095, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4087, 0, 0, 1, 0, 1, 4095, 0, 4088, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1, 3687, 0, 2768, 4069, 0, 120, 144, 0, 1, 1, 66, 3928, 0, 1773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 3, 1, 1, 1, 3644, 4095, 4095, 1, 0, 3902, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16787, 15992, 16537, 16362, 16251, 16374, 16576, 16193] avg=16384.0000 max=16787 max_rank=0 min=15992 min_rank=1 imbalance_ratio=1.024597 max_load_reduction=114285 max_load_reduction_pct=0.871925 moved_to_echo=65394 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4095, 1, 0, 4069, 2768, 682, 0, 0, 0, 4076, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 4093, 1, 0, 0, 0, 191, 0, 0, 2, 4085, 0, 577, 0, 11, 2896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 4095, 1, 1, 0, 0, 47, 1, 1, 1, 4091, 2283, 0, 1605, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 24, 1, 1, 0, 4084, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 0, 4061, 3852, 1, 1, 0, 0, 0, 4, 4095, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 1, 4083, 0, 0, 1, 0, 1, 4095, 0, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1, 3210, 0, 3885, 4067, 0, 149, 37, 0, 1, 2, 124, 3835, 0, 1773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 136, 2, 1, 1, 1, 3631, 4095, 4095, 1, 0, 3902, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15694, 15949, 17049, 16361, 16305, 16373, 17105, 16236] avg=16384.0000 max=17105 max_rank=6 min=15694 min_rank=0 imbalance_ratio=1.044006 max_load_reduction=113967 max_load_reduction_pct=0.869499 moved_to_echo=66019 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 32768, 128, 0, 0, 0, 2816, 0, 0, 128, 32768, 0, 7040, 0, 128, 23168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 32768, 27520, 5504, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 0, 0, 32768, 128, 128, 0, 0, 1920, 128, 128, 128, 32768, 11520, 0, 12928, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 128, 32768, 0, 0, 128, 0, 128, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 23296, 0, 27648, 32768, 128, 640, 512, 0, 128, 128, 640, 31744, 0, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32512, 128, 128, 128, 128, 32768, 0, 0, 0, 32768, 128, 128, 128, 128, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 128, 0, 32640, 31360, 128, 128, 128, 0, 0, 128, 32768, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 0, 0, 512, 128, 128, 128, 128, 30976, 32768, 32768, 128, 0, 31232, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 100, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 20, 0, 4096, 0, 4096, 0, 0, 0, 3666, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 3, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 4077, 0, 0, 14, 4094, 0, 4096, 0, 0, 4094, 0, 0, 4096, 0, 4096, 0, 4096, 4095, 1, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 122, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 3264, 4096, 0, 0, 0, 0, 4091, 0, 4095, 0, 0, 2, 4096, 4096, 0, 2, 1, 0, 0, 0, 0, 0, 3936, 0, 0, 0, 0, 4086, 0, 0, 144, 4094, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 4096, 0, 4096, 4094, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 733, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 4091, 20, 0, 12, 0, 4069, 1, 0, 43, 1, 0, 1, 4093, 0, 0, 0, 4090, 3, 0, 1, 1, 4095, 4094, 0, 1, 4095, 4089, 4086, 0, 0, 2, 1, 1, 1, 0, 1, 1, 19, 3, 0, 0, 4095, 1, 0, 3990, 0, 0, 1, 4094, 1, 4072, 21, 0, 0, 3129, 4095, 1, 1, 1, 0, 3743, 3, 4039, 35, 1, 6, 4065, 4095, 1, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 1, 13, 4095, 0, 0, 1049, 4095, 0, 4095, 0, 1, 3838, 1, 0, 4095, 1, 4095, 0, 4076, 4095, 0, 0, 15, 4095, 1, 306, 0, 0, 34, 1, 1, 0, 1, 287, 0, 0, 0, 1, 4095, 4085, 0, 1, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 4093, 21, 0, 11, 0, 4068, 1, 0, 83, 1, 0, 1, 4094, 0, 0, 0, 4092, 3, 0, 1, 1, 4095, 4094, 0, 1, 4095, 4084, 4085, 0, 0, 2, 1, 1, 1, 0, 1, 1, 28, 5, 0, 0, 4095, 1, 0, 4011, 0, 0, 1, 4094, 1, 4068, 24, 0, 1, 1605, 4095, 1, 1, 1, 0, 3727, 3, 4039, 49, 1, 5, 4067, 4094, 2, 0, 0, 1, 1, 0, 0, 1, 4095, 0, 1, 1, 11, 4095, 0, 0, 1696, 4095, 0, 4095, 0, 0, 3861, 1, 0, 4093, 1, 4095, 0, 4073, 4095, 0, 0, 28, 4095, 1, 293, 0, 0, 7, 1, 1, 0, 1, 1104, 0, 0, 0, 1, 4095, 4086, 0, 2, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 1, 0, 50, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4095, 0, 0, 4096, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 17, 0, 4096, 0, 4096, 0, 0, 0, 2383, 4096, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 1, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 4076, 0, 0, 439, 4094, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 4095, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1267, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4090, 26, 0, 16, 0, 4068, 1, 0, 82, 1, 0, 1, 4093, 0, 0, 0, 4087, 2, 0, 1, 1, 4095, 4095, 0, 1, 4095, 4092, 4093, 0, 0, 1, 1, 1, 1, 0, 1, 1, 7, 2, 0, 0, 4095, 1, 0, 4004, 0, 0, 1, 4094, 1, 4075, 14, 0, 0, 2173, 4095, 1, 1, 1, 0, 3566, 3, 4031, 20, 1, 12, 4069, 4095, 1, 0, 0, 1, 1, 0, 0, 1, 4095, 1, 0, 1, 8, 4095, 0, 0, 1696, 4094, 0, 4095, 0, 1, 3862, 1, 0, 4091, 1, 4095, 0, 4078, 4095, 0, 0, 25, 4095, 1, 192, 0, 0, 120, 1, 1, 0, 1, 739, 0, 0, 0, 1, 4095, 4085, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 0, 0, 1, 0, 36, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 4094, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 16, 0, 4096, 0, 4096, 0, 0, 0, 3064, 4096, 0, 0, 0, 0, 4083, 0, 4093, 0, 0, 4, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 1, 4056, 0, 0, 201, 4093, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 4096, 0, 4096, 4094, 2, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 858, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 2, 4091, 40, 0, 10, 0, 4066, 1, 0, 31, 1, 0, 1, 4095, 0, 0, 0, 4089, 0, 0, 1, 1, 4095, 4091, 0, 1, 4095, 4093, 4094, 0, 0, 1, 0, 1, 0, 0, 1, 1, 8, 6, 0, 0, 4095, 1, 0, 4006, 0, 0, 1, 4094, 1, 4086, 21, 0, 1, 2280, 4095, 1, 1, 1, 0, 3461, 1, 4023, 4, 1, 6, 4061, 4091, 1, 0, 0, 1, 1, 0, 0, 1, 4094, 3, 0, 1, 33, 4095, 0, 0, 1486, 4069, 0, 4095, 0, 1, 4013, 1, 0, 4089, 1, 4095, 0, 4089, 4095, 0, 0, 28, 4095, 0, 259, 0, 0, 0, 1, 1, 0, 1, 873, 16, 0, 0, 1, 4095, 4094, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 122, 0, 2657, 790, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 3264, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4095, 0, 0, 2, 4096, 4096, 0, 2, 1, 0, 0, 0, 0, 0, 152, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 0, 0, 0, 4086, 0, 0, 144, 3408, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 4094, 1, 0, 0, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 733, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 3296, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16436, 16392, 16373, 16996, 16646, 15670, 16229, 16330] avg=16384.0000 max=16996 max_rank=3 min=15670 min_rank=5 imbalance_ratio=1.037354 max_load_reduction=114076 max_load_reduction_pct=0.870331 moved_to_echo=64875 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 100, 0, 2657, 790, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 4096, 0, 4096, 0, 0, 0, 3666, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 3, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 152, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 4077, 0, 0, 14, 3408, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 1, 0, 1, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 457, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 3296, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 16393, 16373, 17413, 16649, 15549, 16233, 16051] avg=16384.0000 max=17413 max_rank=3 min=15549 min_rank=5 imbalance_ratio=1.062805 max_load_reduction=113659 max_load_reduction_pct=0.867149 moved_to_echo=64482 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 4093, 21, 0, 11, 0, 4068, 1, 0, 83, 1, 0, 1, 2656, 790, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4092, 3, 0, 1, 1, 0, 4094, 0, 1, 0, 4084, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 1, 0, 1, 1, 28, 5, 0, 0, 4095, 1, 0, 4095, 4095, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 1, 4094, 1, 4068, 24, 0, 1, 1605, 1439, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3727, 3, 4039, 49, 1, 5, 4067, 4094, 2, 0, 0, 1, 1, 0, 0, 1, 152, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 11, 4095, 0, 0, 1696, 3409, 0, 0, 0, 0, 3861, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 4073, 4095, 0, 0, 28, 3943, 1, 293, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1104, 0, 0, 0, 1, 4095, 4086, 0, 2, 0, 0, 0, 4095, 3294, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16396, 16371, 16409, 15247, 16253, 17170, 16534, 16692] avg=16384.0000 max=17170 max_rank=5 min=15247 min_rank=3 imbalance_ratio=1.047974 max_load_reduction=113902 max_load_reduction_pct=0.869003 moved_to_echo=66649 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 4091, 20, 0, 12, 0, 4069, 1, 0, 43, 1, 0, 1, 2660, 793, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4090, 3, 0, 1, 1, 0, 4094, 0, 1, 0, 4089, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 1, 0, 1, 1, 19, 3, 0, 0, 4095, 1, 0, 4095, 4095, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 0, 0, 1, 4094, 1, 4072, 21, 0, 0, 3129, 1435, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3743, 3, 4039, 35, 1, 6, 4065, 4095, 1, 0, 0, 0, 1, 0, 0, 1, 158, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 13, 4095, 0, 0, 1049, 3406, 0, 0, 0, 1, 3838, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4076, 4095, 0, 0, 15, 3937, 1, 306, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 287, 0, 0, 0, 1, 4095, 4085, 0, 1, 0, 1, 0, 4095, 3297, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16364, 16368, 16399, 16746, 16260, 16499, 16560, 15876] avg=16384.0000 max=16746 max_rank=3 min=15876 min_rank=7 imbalance_ratio=1.022095 max_load_reduction=114326 max_load_reduction_pct=0.872238 moved_to_echo=65195 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 1, 0, 50, 0, 2657, 790, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 17, 0, 4096, 0, 4096, 0, 0, 0, 2383, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 1, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 152, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 4076, 0, 0, 439, 3408, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 3944, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1267, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 3296, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16362, 16393, 16374, 16127, 16648, 16072, 16234, 16862] avg=16384.0000 max=16862 max_rank=7 min=16072 min_rank=5 imbalance_ratio=1.029175 max_load_reduction=114210 max_load_reduction_pct=0.871353 moved_to_echo=65816 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 0, 0, 1, 0, 36, 0, 2657, 790, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 16, 0, 4096, 0, 4096, 0, 0, 0, 3064, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4093, 0, 0, 4, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 152, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 1, 4056, 0, 0, 201, 3408, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 4094, 2, 0, 0, 3944, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 858, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4094, 3296, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16348, 16392, 16372, 16807, 16638, 15829, 16232, 16454] avg=16384.0000 max=16807 max_rank=3 min=15829 min_rank=5 imbalance_ratio=1.025818 max_load_reduction=114265 max_load_reduction_pct=0.871773 moved_to_echo=65153 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4090, 26, 0, 16, 0, 4068, 1, 0, 82, 1, 0, 1, 2656, 789, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4087, 2, 0, 1, 1, 0, 4095, 0, 1, 0, 4092, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1, 7, 2, 0, 0, 4095, 1, 0, 4095, 4095, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 0, 1, 4094, 1, 4075, 14, 0, 0, 2173, 1439, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3566, 3, 4031, 20, 1, 12, 4069, 4095, 1, 0, 0, 1, 1, 0, 0, 1, 152, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 8, 4095, 0, 0, 1696, 3408, 0, 0, 0, 1, 3862, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 4078, 4095, 0, 0, 25, 3943, 1, 192, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 739, 0, 0, 0, 1, 4095, 4085, 0, 0, 0, 0, 0, 4095, 3293, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16400, 16383, 16385, 15804, 16064, 17168, 16546, 16322] avg=16384.0000 max=17168 max_rank=5 min=15804 min_rank=3 imbalance_ratio=1.047852 max_load_reduction=113904 max_load_reduction_pct=0.869019 moved_to_echo=66100 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 2, 4091, 40, 0, 10, 0, 4066, 1, 0, 31, 1, 0, 1, 2656, 790, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4089, 0, 0, 1, 1, 0, 4091, 0, 1, 0, 4093, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 8, 6, 0, 0, 4095, 1, 0, 4095, 4095, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 1, 4094, 1, 4086, 21, 0, 1, 2280, 1439, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3461, 1, 4023, 4, 1, 6, 4061, 4091, 1, 0, 0, 1, 1, 0, 0, 1, 152, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 1, 33, 4095, 0, 0, 1486, 3388, 0, 0, 0, 1, 4013, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 4089, 4095, 0, 0, 28, 3943, 0, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 873, 16, 0, 0, 1, 4095, 4094, 0, 1, 0, 0, 0, 4095, 3295, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16355, 16380, 16387, 15932, 15914, 17115, 16504, 16485] avg=16384.0000 max=17115 max_rank=5 min=15914 min_rank=4 imbalance_ratio=1.044617 max_load_reduction=113957 max_load_reduction_pct=0.869423 moved_to_echo=66018 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 6400, 128, 0, 0, 128, 32768, 32768, 0, 128, 0, 128, 0, 32768, 26368, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32512, 0, 128, 128, 32768, 128, 32768, 128, 0, 128, 21632, 11520, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32512, 128, 128, 128, 128, 32768, 0, 0, 6784, 27264, 0, 0, 0, 128, 32000, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 128, 0, 0, 32768, 32768, 128, 0, 128, 31616, 128, 1152, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 32768, 128, 0, 128, 128, 0, 32768, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 0, 0, 128, 32768, 128, 128, 128, 0, 32768, 128, 0, 256, 128, 384, 128, 21376, 6400, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[30976, 128, 32512, 128, 128, 128, 32768, 32768, 128, 128, 128, 128, 128, 0, 0, 128, 1280, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 128, 128, 128, 0, 0, 32768, 128, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3071, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 13, 0, 0, 0, 2788, 70, 0, 0, 0, 0, 0, 0, 904, 0, 0, 5, 0, 4096, 0, 4096, 4091, 0, 0, 4096, 0, 4096, 3964, 0, 14, 2, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4095, 3747, 0, 4096, 0, 4096, 0, 0, 4095, 0, 4096, 0, 1, 4020, 0, 0, 4096, 0, 0, 0, 4096, 3844, 0, 0, 3, 0, 0, 2294, 0, 0, 0, 4095, 1, 0, 0, 3939, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3713, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 12, 0, 0, 0, 1958, 18, 0, 0, 0, 0, 0, 0, 2087, 0, 0, 6, 0, 4096, 0, 4096, 4087, 0, 0, 4096, 0, 4096, 4075, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 7, 4096, 3973, 0, 4096, 0, 4094, 0, 0, 4095, 0, 4096, 0, 0, 3986, 0, 0, 4096, 0, 0, 0, 4096, 3744, 0, 0, 4, 0, 0, 1194, 0, 0, 0, 4094, 0, 0, 0, 3904, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1, 1, 4030, 1, 1, 1, 2, 0, 4094, 1, 1, 4095, 5, 1, 19, 2, 4092, 1, 4095, 0, 0, 0, 0, 4094, 4095, 0, 3934, 1, 1, 17, 4090, 1, 1, 1, 0, 0, 0, 0, 4, 0, 1766, 0, 6, 1, 1, 0, 0, 1, 691, 0, 0, 26, 1, 4091, 0, 4095, 4049, 0, 1, 4087, 0, 4095, 4085, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 4067, 1, 1, 1, 8, 107, 4095, 4092, 5, 4095, 0, 4093, 0, 0, 4095, 0, 4063, 0, 0, 3513, 0, 0, 4095, 0, 1, 0, 4074, 2882, 0, 43, 0, 0, 0, 3596, 24, 0, 1, 4095, 278, 0, 1, 4018, 0, 3843, 1, 10, 0, 1, 1, 0, 4095, 4094, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1, 1, 4021, 1, 1, 0, 0, 1, 4094, 1, 1, 4095, 4, 1, 14, 2, 4090, 1, 4095, 0, 0, 0, 0, 4095, 4095, 1, 3991, 1, 1, 25, 4092, 1, 1, 1, 0, 0, 0, 0, 3, 0, 1185, 0, 9, 0, 0, 0, 0, 5, 1181, 0, 0, 69, 0, 4086, 0, 4095, 4060, 0, 1, 4060, 0, 4095, 4067, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 4093, 1, 1, 1, 0, 219, 4095, 4087, 35, 4095, 0, 4093, 1, 0, 4095, 30, 4054, 1, 0, 3202, 0, 0, 4095, 0, 1, 0, 4082, 3119, 0, 49, 0, 1, 0, 3722, 0, 0, 1, 4095, 266, 0, 1, 3882, 0, 3813, 0, 2, 0, 2, 1, 0, 4095, 4093, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 1, 4013, 1, 1, 0, 0, 0, 4094, 1, 1, 4095, 8, 1, 17, 2, 4092, 1, 4095, 0, 0, 0, 0, 4095, 4094, 1, 4050, 1, 1, 10, 4093, 1, 1, 1, 0, 0, 0, 0, 7, 0, 2139, 0, 10, 1, 0, 0, 0, 3, 281, 0, 0, 47, 0, 4087, 0, 4093, 4055, 0, 1, 4069, 0, 4095, 3892, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 4094, 1, 1, 1, 4, 101, 4095, 4095, 31, 4095, 1, 4094, 1, 0, 4095, 28, 4057, 1, 0, 3348, 0, 0, 4095, 0, 1, 0, 4072, 3020, 0, 46, 0, 0, 0, 3721, 0, 0, 1, 4095, 296, 0, 1, 3978, 0, 3852, 0, 3, 0, 2, 1, 0, 4095, 4093, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3766, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 7, 0, 0, 0, 1060, 40, 0, 0, 0, 0, 0, 0, 1794, 0, 0, 3, 0, 4096, 0, 4096, 4071, 0, 0, 4096, 0, 4096, 3767, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 7, 4095, 3706, 0, 4096, 0, 4095, 0, 0, 4096, 0, 4096, 0, 1, 4017, 0, 0, 4096, 0, 0, 0, 4095, 3198, 0, 0, 5, 0, 0, 3507, 0, 0, 0, 4094, 0, 0, 0, 3816, 0, 4096, 0, 0, 0, 0, 0, 2, 4095, 4096, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4000, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 10, 0, 0, 0, 1687, 27, 0, 0, 0, 0, 0, 0, 1151, 0, 0, 11, 0, 4096, 0, 4096, 4092, 0, 0, 4096, 0, 4096, 3596, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4094, 4010, 0, 4096, 0, 4094, 0, 0, 4094, 0, 4096, 0, 0, 3560, 0, 0, 4096, 0, 0, 0, 4096, 3966, 0, 0, 5, 0, 0, 2978, 0, 0, 0, 4094, 2, 0, 0, 3676, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1, 1, 4021, 1, 1, 1, 0, 0, 4094, 1, 1, 4095, 4, 1, 19, 2, 4094, 1, 4095, 0, 0, 0, 0, 4095, 4095, 1, 3980, 1, 1, 12, 4092, 1, 1, 1, 0, 0, 0, 0, 5, 0, 1593, 0, 5, 0, 0, 0, 0, 1, 589, 0, 0, 70, 0, 4090, 0, 4094, 4036, 0, 1, 4081, 0, 4095, 4068, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 4085, 1, 1, 1, 10, 235, 4095, 4037, 35, 4095, 1, 4091, 1, 0, 4095, 15, 4067, 1, 0, 3027, 0, 0, 4095, 0, 1, 0, 4072, 3253, 0, 53, 0, 0, 0, 3832, 1, 0, 1, 4095, 259, 0, 1, 4040, 0, 3835, 0, 3, 0, 2, 1, 0, 4095, 4094, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 2506, 1613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 278, 0, 3713, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 12, 0, 0, 0, 1958, 18, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 2291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2087, 0, 0, 6, 0, 3181, 0, 0, 4087, 0, 0, 4096, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 7, 4096, 4096, 2205, 1805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 0, 0, 0, 4094, 0, 0, 637, 0, 4096, 0, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3744, 0, 0, 4, 0, 0, 1194, 0, 0, 0, 4094, 0, 952, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 1, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16406, 16279, 16569, 17532, 16306, 16786, 14684, 16510] avg=16384.0000 max=17532 max_rank=3 min=14684 min_rank=6 imbalance_ratio=1.070068 max_load_reduction=113540 max_load_reduction_pct=0.866241 moved_to_echo=64286 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 4021, 1, 1, 0, 0, 1, 4094, 1, 1, 4095, 4, 1, 14, 2, 2506, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 4095, 278, 1, 3991, 1, 1, 25, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 3, 0, 1185, 0, 9, 0, 0, 0, 0, 5, 4095, 4095, 4095, 2290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 0, 0, 69, 0, 3174, 0, 0, 4060, 0, 1, 4060, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 4093, 1, 1, 1, 0, 219, 4095, 4095, 2205, 1805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 35, 0, 0, 4093, 1, 0, 637, 30, 4054, 1, 0, 3202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4082, 3119, 0, 49, 0, 1, 0, 3722, 0, 0, 1, 4095, 266, 952, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3882, 0, 3813, 0, 2, 0, 2, 1, 0, 4095, 4093, 0, 1, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16356, 16576, 15779, 16612, 16519, 16140, 16886, 16204] avg=16384.0000 max=16886 max_rank=6 min=15779 min_rank=2 imbalance_ratio=1.030640 max_load_reduction=114186 max_load_reduction_pct=0.871170 moved_to_echo=65749 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 2506, 1613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 278, 0, 3071, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 13, 0, 0, 0, 2788, 70, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 2291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 904, 0, 0, 5, 0, 3181, 0, 0, 4091, 0, 0, 4096, 0, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4095, 4096, 2205, 1805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3747, 0, 0, 0, 4096, 0, 0, 637, 0, 4096, 0, 1, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3844, 0, 0, 3, 0, 0, 2294, 0, 0, 0, 4095, 1, 952, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3939, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16404, 15637, 17454, 16241, 16310, 16597, 15885, 16544] avg=16384.0000 max=17454 max_rank=2 min=15637 min_rank=1 imbalance_ratio=1.065308 max_load_reduction=113618 max_load_reduction_pct=0.866837 moved_to_echo=65336 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 2506, 1613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 278, 0, 4000, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 10, 0, 0, 0, 1687, 27, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 2291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1151, 0, 0, 11, 0, 3181, 0, 0, 4092, 0, 0, 4096, 0, 0, 3596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4094, 4096, 2205, 1805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 0, 4094, 0, 0, 636, 0, 4096, 0, 0, 3560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3966, 0, 0, 5, 0, 0, 2978, 0, 0, 0, 4094, 2, 952, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3676, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16406, 16566, 16306, 16127, 16297, 16396, 16693, 16281] avg=16384.0000 max=16693 max_rank=6 min=16127 min_rank=3 imbalance_ratio=1.018860 max_load_reduction=114379 max_load_reduction_pct=0.872643 moved_to_echo=65667 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 4030, 1, 1, 1, 2, 0, 4094, 1, 1, 4095, 5, 1, 19, 2, 2506, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 4094, 278, 0, 3934, 1, 1, 17, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4, 0, 1766, 0, 6, 1, 1, 0, 0, 1, 4095, 4095, 4095, 2290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, 0, 26, 1, 3177, 0, 0, 4049, 0, 1, 4087, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 4067, 1, 1, 1, 8, 107, 4095, 4095, 2205, 1805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 5, 0, 0, 4093, 0, 0, 637, 0, 4063, 0, 0, 3513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4074, 2882, 0, 43, 0, 0, 0, 3596, 24, 0, 1, 4095, 278, 952, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4018, 0, 3843, 1, 10, 0, 1, 1, 0, 4095, 4094, 0, 1, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 16509, 16356, 16117, 16389, 16403, 16545, 16380] avg=16384.0000 max=16545 max_rank=6 min=16117 min_rank=3 imbalance_ratio=1.009827 max_load_reduction=114527 max_load_reduction_pct=0.873772 moved_to_echo=65717 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 2507, 1613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 278, 0, 3766, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7, 0, 0, 0, 1060, 40, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 2291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1794, 0, 0, 3, 0, 3181, 0, 0, 4071, 0, 0, 4096, 0, 0, 3767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 7, 4095, 4096, 2205, 1805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3706, 0, 0, 0, 4095, 0, 0, 637, 0, 4096, 0, 1, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3198, 0, 0, 5, 0, 0, 3507, 0, 0, 0, 4094, 0, 952, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3816, 0, 4096, 0, 0, 0, 0, 0, 2, 4095, 4096, 0, 0, 1, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16406, 16332, 15688, 16912, 16310, 16552, 16451, 16421] avg=16384.0000 max=16912 max_rank=3 min=15688 min_rank=2 imbalance_ratio=1.032227 max_load_reduction=114160 max_load_reduction_pct=0.870972 moved_to_echo=65734 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4013, 1, 1, 0, 0, 0, 4094, 1, 1, 4095, 8, 1, 17, 2, 2510, 1615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 4095, 272, 1, 4050, 1, 1, 10, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 7, 0, 2139, 0, 10, 1, 0, 0, 0, 3, 4095, 4095, 4095, 2291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 0, 47, 0, 3165, 0, 0, 4055, 0, 1, 4069, 0, 0, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 4094, 1, 1, 1, 4, 101, 4095, 4093, 2207, 1804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 31, 0, 1, 4094, 1, 0, 630, 28, 4057, 1, 0, 3348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4072, 3020, 0, 46, 0, 0, 0, 3721, 0, 0, 1, 4095, 296, 955, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3978, 0, 3852, 0, 3, 0, 2, 1, 0, 4095, 4093, 0, 1, 0, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16360, 16617, 16738, 15510, 16406, 16286, 16810, 16345] avg=16384.0000 max=16810 max_rank=6 min=15510 min_rank=3 imbalance_ratio=1.026001 max_load_reduction=114262 max_load_reduction_pct=0.871750 moved_to_echo=65847 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 4021, 1, 1, 1, 0, 0, 4094, 1, 1, 4095, 4, 1, 19, 2, 2506, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 4095, 278, 1, 3980, 1, 1, 12, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 5, 0, 1593, 0, 5, 0, 0, 0, 0, 1, 4095, 4095, 4095, 2291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 0, 70, 0, 3176, 0, 0, 4036, 0, 1, 4081, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 4085, 1, 1, 1, 10, 235, 4095, 4094, 2205, 1804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 35, 0, 1, 4091, 1, 0, 637, 15, 4067, 1, 0, 3027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4072, 3253, 0, 53, 0, 0, 0, 3832, 1, 0, 1, 4095, 259, 952, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4040, 0, 3835, 0, 3, 0, 2, 1, 0, 4095, 4094, 0, 1, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16361, 16556, 16182, 16021, 16535, 15912, 17118, 16387] avg=16384.0000 max=17118 max_rank=6 min=15912 min_rank=5 imbalance_ratio=1.044800 max_load_reduction=113954 max_load_reduction_pct=0.869400 moved_to_echo=65952 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 32768, 27136, 0, 256, 128, 128, 0, 24960, 128, 0, 128, 32768, 1152, 7680, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[8704, 0, 0, 256, 128, 25472, 0, 0, 32640, 0, 128, 32768, 0, 0, 31616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 32768, 2304, 128, 30592, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 32512, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 20096, 12928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[31872, 128, 0, 128, 32768, 128, 0, 5120, 128, 32640, 128, 128, 28800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 31360, 0, 31744, 128, 128, 0, 128, 128, 128, 32768, 32768, 0, 128, 128, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 768, 32768, 32768, 17664, 14464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 0, 14208, 256, 128, 128, 128, 0, 0, 128, 32768, 32768, 32768, 18432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131968 slack_slots=8064 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 25, 3033, 0, 4096, 0, 4096, 0, 4092, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 3270, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4094, 0, 0, 4096, 4096, 0, 4087, 19, 138, 4096, 4, 4096, 0, 0, 0, 0, 0, 4096, 1817, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 17, 0, 0, 0, 4094, 3964, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 2631, 0, 4096, 0, 4096, 0, 4089, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 2131, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4095, 0, 0, 4096, 4096, 0, 4082, 43, 418, 4096, 2, 4095, 0, 0, 0, 0, 0, 4096, 3572, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 4096, 0, 4096, 1, 4096, 0, 4096, 4096, 26, 0, 0, 0, 4095, 3474, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3917, 0, 4096, 0, 4096, 0, 4082, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 1679, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 4095, 0, 0, 4096, 4096, 0, 4075, 12, 221, 4096, 4, 4096, 0, 0, 0, 0, 0, 4096, 2509, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 60, 0, 0, 0, 4094, 3899, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4091, 0, 0, 1, 6, 0, 4095, 0, 0, 0, 144, 3507, 0, 4091, 0, 4094, 0, 4095, 0, 0, 8, 4095, 0, 4, 1, 1, 0, 0, 0, 0, 4095, 34, 2077, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 1, 0, 2, 4093, 3, 0, 4095, 1, 2, 0, 4094, 1, 1, 1, 1, 2, 0, 1, 0, 9, 0, 1, 30, 1, 1, 1, 0, 4095, 0, 1, 4068, 4084, 1, 4095, 0, 366, 4092, 746, 4095, 0, 0, 0, 0, 0, 4064, 1522, 0, 1, 4074, 0, 1, 0, 1, 0, 3, 0, 4095, 4095, 4087, 1, 4095, 0, 4092, 0, 4095, 4095, 5, 1, 0, 1, 4095, 3934, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 3077, 0, 4096, 0, 4096, 0, 4087, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 2489, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4094, 0, 0, 4096, 4096, 0, 4083, 39, 403, 4096, 2, 4096, 0, 0, 0, 0, 0, 4096, 2459, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 20, 0, 0, 0, 4094, 3790, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 0, 1, 15, 0, 4092, 0, 0, 0, 382, 3570, 0, 4091, 0, 4094, 0, 4095, 1, 0, 7, 4095, 0, 0, 1, 1, 0, 0, 0, 0, 4060, 13, 3175, 1, 1, 0, 1, 4095, 0, 1, 1, 1, 2, 0, 3, 4094, 3, 0, 4095, 1, 4, 0, 4060, 1, 2, 1, 0, 1, 0, 0, 0, 43, 0, 2, 25, 1, 1, 1, 1, 4095, 1, 1, 4043, 4080, 1, 4095, 0, 15, 4093, 699, 4095, 0, 0, 47, 0, 1, 4072, 784, 0, 1, 4041, 0, 1, 0, 1, 0, 5, 0, 4095, 4095, 4074, 1, 4095, 0, 4091, 0, 4095, 4095, 0, 1, 0, 1, 4095, 3742, 0, 4095, 0, 1, 1, 0, 1, 0, 1, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4091, 0, 0, 1, 17, 0, 4093, 0, 0, 0, 477, 4022, 0, 4091, 0, 4094, 0, 4095, 1, 0, 8, 4095, 0, 5, 1, 1, 0, 0, 0, 0, 4089, 22, 1399, 1, 1, 0, 1, 4095, 0, 1, 1, 1, 1, 0, 1, 4094, 2, 0, 4095, 1, 4, 0, 4088, 1, 1, 1, 0, 0, 0, 0, 0, 48, 0, 2, 29, 1, 1, 1, 0, 4095, 1, 1, 4062, 4076, 1, 4095, 4, 63, 4093, 1008, 4095, 0, 0, 3, 0, 1, 4071, 2489, 0, 1, 4052, 0, 1, 0, 1, 0, 3, 0, 4095, 4095, 4077, 1, 4095, 0, 4089, 0, 4095, 4093, 4, 1, 0, 1, 4095, 2846, 0, 4095, 0, 1, 1, 0, 2, 0, 1, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 0, 1, 15, 0, 4092, 1, 0, 0, 596, 3488, 0, 4090, 0, 4094, 0, 4095, 1, 0, 8, 4095, 0, 0, 1, 1, 0, 0, 0, 1, 4067, 22, 1185, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 2, 1, 3, 4094, 2, 0, 4095, 0, 4, 0, 4086, 1, 1, 1, 0, 0, 0, 0, 0, 39, 0, 3, 24, 1, 1, 1, 0, 4095, 1, 0, 4066, 4081, 1, 4095, 1, 85, 4092, 1069, 4095, 0, 0, 51, 0, 1, 4071, 2304, 0, 1, 4054, 0, 1, 0, 1, 0, 5, 0, 4095, 4095, 4070, 1, 4095, 0, 4092, 0, 4095, 4095, 0, 1, 0, 1, 4095, 3569, 0, 4095, 0, 1, 1, 0, 1, 0, 1, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 25, 3033, 2396, 2181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4092, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3270, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2415, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4082, 0, 0, 4096, 4096, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 138, 4096, 4, 1267, 0, 0, 0, 0, 0, 4096, 1817, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 4094, 3964, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 414, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15825, 16383, 17473, 16383, 16378, 15533, 16395, 16702] avg=16384.0000 max=17473 max_rank=2 min=15533 min_rank=5 imbalance_ratio=1.066467 max_load_reduction=113599 max_load_reduction_pct=0.866692 moved_to_echo=65008 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 3077, 2396, 2180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4087, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2489, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2415, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4082, 0, 0, 4096, 4096, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 403, 4096, 2, 1267, 0, 0, 0, 0, 0, 4096, 2459, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4094, 3790, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 414, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15857, 16378, 16693, 16376, 16391, 16458, 16395, 16524] avg=16384.0000 max=16693 max_rank=2 min=15857 min_rank=0 imbalance_ratio=1.018860 max_load_reduction=114379 max_load_reduction_pct=0.872643 moved_to_echo=65768 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 2631, 2396, 2181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4089, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2131, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2414, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4083, 0, 0, 4096, 4096, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 418, 4096, 2, 1267, 0, 0, 0, 0, 0, 4096, 3572, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 1, 4096, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 4095, 3474, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 414, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15404, 16379, 16334, 16387, 16368, 17590, 16394, 16216] avg=16384.0000 max=17590 max_rank=5 min=15404 min_rank=0 imbalance_ratio=1.073608 max_load_reduction=113482 max_load_reduction_pct=0.865799 moved_to_echo=66568 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3917, 2396, 2181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4082, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1679, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2415, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 4083, 0, 0, 4096, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 221, 4096, 4, 1267, 0, 0, 0, 0, 0, 4096, 2509, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 4094, 3899, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 414, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16686, 16372, 15882, 16372, 16393, 16301, 16395, 16671] avg=16384.0000 max=16686 max_rank=0 min=15882 min_rank=2 imbalance_ratio=1.018433 max_load_reduction=114386 max_load_reduction_pct=0.872696 moved_to_echo=65760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4091, 0, 0, 1, 6, 0, 4095, 0, 0, 0, 144, 3507, 2396, 2179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4094, 0, 4095, 0, 0, 8, 4095, 0, 4, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 34, 2077, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 1, 0, 4095, 1916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 3, 0, 4095, 1, 2, 0, 4094, 1, 1, 1, 1, 2, 0, 1, 2414, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1, 30, 1, 1, 1, 0, 4083, 0, 1, 4068, 4084, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 366, 4092, 746, 1267, 0, 0, 0, 0, 0, 4064, 1522, 0, 1, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 3, 0, 0, 4095, 4087, 1, 0, 0, 4092, 0, 11, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 1, 4095, 3934, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 4093, 414, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16421, 16390, 16318, 16399, 16375, 16132, 16386, 16651] avg=16384.0000 max=16651 max_rank=7 min=16132 min_rank=5 imbalance_ratio=1.016296 max_load_reduction=114421 max_load_reduction_pct=0.872963 moved_to_echo=65544 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 0, 1, 15, 0, 4092, 0, 0, 0, 382, 3570, 2400, 2180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4094, 0, 4095, 1, 0, 7, 4095, 0, 0, 1, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 13, 3175, 1, 1, 0, 1, 4095, 0, 1, 1, 1, 2, 0, 4095, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 3, 0, 4095, 1, 4, 0, 4060, 1, 2, 1, 0, 1, 0, 0, 2418, 1693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 2, 25, 1, 1, 1, 1, 4075, 1, 1, 4043, 4080, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4093, 699, 1257, 0, 0, 47, 0, 1, 4072, 784, 0, 1, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 5, 0, 0, 4095, 4074, 1, 0, 0, 4091, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 3742, 0, 4095, 0, 1, 1, 0, 1, 0, 1, 4091, 420, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16736, 16392, 17361, 16376, 16370, 15010, 16365, 16462] avg=16384.0000 max=17361 max_rank=2 min=15010 min_rank=5 imbalance_ratio=1.059631 max_load_reduction=113711 max_load_reduction_pct=0.867546 moved_to_echo=64207 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 0, 1, 15, 0, 4092, 1, 0, 0, 596, 3488, 2396, 2179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4094, 0, 4095, 1, 0, 8, 4095, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4067, 22, 1185, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 2, 1, 4095, 1916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 2, 0, 4095, 0, 4, 0, 4086, 1, 1, 1, 0, 0, 0, 0, 2414, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 3, 24, 1, 1, 1, 0, 4083, 1, 0, 4066, 4081, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 85, 4092, 1069, 1267, 0, 0, 51, 0, 1, 4071, 2304, 0, 1, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 5, 0, 0, 4095, 4070, 1, 0, 0, 4092, 0, 11, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 3569, 0, 4095, 0, 1, 1, 0, 1, 0, 1, 4091, 414, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16864, 16386, 15389, 16389, 16396, 16996, 16371, 16281] avg=16384.0000 max=16996 max_rank=5 min=15389 min_rank=2 imbalance_ratio=1.037354 max_load_reduction=114076 max_load_reduction_pct=0.870331 moved_to_echo=66044 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4091, 0, 0, 1, 17, 0, 4093, 0, 0, 0, 477, 4022, 2396, 2180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4094, 0, 4095, 1, 0, 8, 4095, 0, 5, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 22, 1399, 1, 1, 0, 1, 4095, 0, 1, 1, 1, 1, 0, 4095, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 2, 0, 4095, 1, 4, 0, 4088, 1, 1, 1, 0, 0, 0, 0, 2414, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 2, 29, 1, 1, 1, 0, 4083, 1, 1, 4062, 4076, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 63, 4093, 1008, 1267, 0, 0, 3, 0, 1, 4071, 2489, 0, 1, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 3, 0, 0, 4095, 4077, 1, 0, 0, 4089, 0, 11, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 1, 4095, 2846, 0, 4095, 0, 1, 1, 0, 2, 0, 1, 4093, 414, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17279, 16392, 15622, 16390, 16401, 17052, 16371, 15565] avg=16384.0000 max=17279 max_rank=0 min=15565 min_rank=7 imbalance_ratio=1.054626 max_load_reduction=113793 max_load_reduction_pct=0.868172 moved_to_echo=65389 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 32768, 128, 17408, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 128, 128, 32768, 15360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 1792, 32768, 3584, 10240, 0, 0, 128, 0, 128, 32768, 17536, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 128, 128, 0, 32768, 128, 0, 0, 1664, 27264, 19200, 17536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 0, 32768, 128, 128, 0, 32768, 128, 128, 128, 128, 128, 0, 128, 19328, 13568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 128, 0, 128, 32768, 29312, 128, 32768, 0, 128, 128, 0, 128, 0, 128, 32768, 3328, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 0, 32768, 0, 32768, 128, 0, 128, 32768, 0, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 256, 0, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 32640, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 32768, 32768, 128, 0, 128, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=132480 slack_slots=7552 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132224 slack_slots=7808 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4062, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 24, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 2604, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 4092, 0, 0, 0, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 4096, 4094, 0, 8, 0, 1288, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 1, 4093, 0, 4096, 2, 0, 2, 4089, 0, 0, 0, 253, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4052, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 3711, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 3064, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 4096, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 4096, 4096, 4095, 0, 44, 0, 1214, 0, 4096, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 1, 4095, 0, 4096, 0, 0, 1, 4076, 0, 0, 0, 228, 0, 0, 0, 4096, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4028, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 4096, 0, 3079, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 870, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 4096, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 4096, 4095, 0, 42, 0, 2800, 0, 4096, 0, 4096, 0, 0, 0, 3, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 4079, 0, 0, 0, 1489, 0, 0, 0, 4095, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 3829, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 1206, 0, 0, 0, 0, 0, 4094, 4096, 1, 4096, 4088, 0, 0, 0, 3, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 4096, 4096, 4093, 0, 13, 0, 2987, 0, 4096, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 4096, 38, 0, 2, 4082, 0, 0, 0, 209, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4081, 3943, 1, 9, 8, 1, 0, 0, 4095, 4094, 1, 0, 0, 4095, 0, 377, 1, 4092, 39, 0, 147, 4095, 0, 0, 6, 83, 1, 1, 10, 1, 0, 4092, 0, 1, 0, 4, 0, 1, 1, 1, 652, 0, 1, 0, 12, 0, 3995, 4095, 0, 4094, 4095, 0, 0, 0, 0, 3920, 0, 3, 1, 17, 1, 0, 0, 4084, 16, 1, 0, 0, 2, 1, 4084, 4085, 4064, 0, 42, 0, 4050, 0, 4064, 1, 4094, 7, 1, 0, 0, 4087, 1, 1, 4095, 4094, 0, 1, 4095, 4082, 0, 1, 4095, 0, 1, 1, 0, 4095, 1, 4095, 304, 231, 0, 4095, 1, 0, 1, 2766, 1, 2, 0, 4095, 289, 3777] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 4072, 3896, 1, 4, 6, 1, 0, 0, 4095, 4094, 1, 0, 0, 4095, 0, 2769, 1, 4092, 63, 0, 142, 4095, 0, 0, 7, 60, 1, 1, 11, 1, 0, 4095, 0, 1, 0, 3, 0, 1, 1, 1, 388, 0, 1, 0, 15, 0, 3680, 4071, 1, 4094, 4095, 0, 0, 0, 0, 4016, 0, 3, 1, 27, 1, 0, 0, 4083, 41, 1, 0, 0, 1, 1, 4084, 4071, 4064, 0, 0, 0, 3190, 0, 4058, 1, 4094, 4, 1, 0, 0, 4089, 1, 1, 4095, 4095, 0, 1, 4095, 4081, 0, 1, 4095, 0, 1, 1, 0, 4095, 1, 4095, 330, 192, 0, 4095, 1, 0, 1, 1877, 1, 1, 0, 4095, 304, 3728] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 0, 2, 0, 1, 0, 4089, 3910, 1, 11, 6, 1, 0, 0, 4095, 4094, 1, 0, 0, 4095, 0, 1652, 2, 4092, 69, 0, 14, 4095, 0, 0, 5, 47, 1, 1, 0, 1, 0, 4095, 0, 1, 0, 2, 0, 1, 1, 1, 189, 0, 0, 0, 10, 0, 4043, 4095, 0, 4094, 4095, 0, 0, 0, 1, 3981, 0, 2, 1, 6, 1, 0, 0, 4091, 13, 1, 0, 0, 0, 1, 4085, 4089, 4053, 0, 68, 0, 3868, 0, 4066, 1, 4094, 4, 1, 0, 1, 4083, 0, 1, 4095, 4095, 0, 1, 4095, 4070, 0, 0, 4095, 0, 1, 1, 0, 4095, 1, 4095, 224, 171, 0, 4095, 1, 1, 1, 2512, 1, 2, 0, 4095, 279, 3620] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 0, 2, 0, 0, 0, 4090, 3793, 1, 15, 7, 1, 0, 0, 4095, 4094, 1, 0, 0, 4095, 0, 3205, 1, 4093, 72, 1, 90, 4095, 0, 0, 5, 58, 1, 1, 2, 1, 0, 4093, 0, 1, 0, 3, 0, 1, 1, 1, 344, 0, 1, 0, 11, 0, 3347, 4073, 0, 4094, 4095, 0, 0, 0, 0, 4034, 0, 1, 1, 1, 1, 0, 0, 4090, 17, 1, 0, 0, 0, 1, 4080, 4082, 4072, 0, 0, 0, 3383, 0, 4055, 1, 4094, 5, 1, 0, 0, 4084, 1, 1, 4095, 4095, 0, 1, 4095, 4082, 0, 1, 4095, 0, 1, 1, 0, 4095, 1, 4095, 45, 195, 0, 4095, 1, 0, 1, 1996, 1, 2, 1, 4095, 329, 3762] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 4096, 4077, 0, 0, 0, 0, 4096, 4087, 102, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 4096, 0, 0, 0, 0, 0, 4062, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 4096, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2604, 0, 0, 0, 0, 0, 4096, 4096, 0, 3098, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 2835, 2810, 1590, 997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 4096, 4094, 0, 8, 0, 1288, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2506, 2, 0, 2, 4089, 0, 0, 0, 253, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16494, 17534, 16356, 17986, 16425, 14843, 16390, 15044] avg=16384.0000 max=17986 max_rank=3 min=14843 min_rank=5 imbalance_ratio=1.097778 max_load_reduction=113086 max_load_reduction_pct=0.862778 moved_to_echo=62702 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4052, 0, 0, 0, 0, 4096, 4087, 102, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 4096, 0, 0, 0, 0, 0, 3711, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4096, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3064, 0, 0, 0, 0, 0, 4096, 4096, 0, 3098, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 2835, 2810, 1590, 997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 4096, 4095, 0, 44, 0, 1214, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2506, 0, 0, 1, 4076, 0, 0, 0, 228, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16464, 17183, 16344, 18450, 16430, 14807, 16392, 15002] avg=16384.0000 max=18450 max_rank=3 min=14807 min_rank=5 imbalance_ratio=1.126099 max_load_reduction=112622 max_load_reduction_pct=0.859238 moved_to_echo=62631 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 4096, 4087, 102, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 4096, 0, 0, 0, 0, 0, 3829, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 0, 0, 4096, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1206, 0, 0, 0, 0, 0, 4094, 4096, 1, 3098, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 2835, 2810, 1590, 998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 4096, 4093, 0, 13, 0, 2987, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2506, 38, 0, 2, 4082, 0, 0, 0, 209, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16492, 17301, 16301, 16583, 16427, 16548, 16391, 15029] avg=16384.0000 max=17301 max_rank=1 min=15029 min_rank=7 imbalance_ratio=1.055969 max_load_reduction=113771 max_load_reduction_pct=0.868004 moved_to_echo=64395 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4028, 0, 0, 0, 0, 4096, 4087, 102, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 4096, 0, 0, 1, 0, 0, 3079, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 870, 0, 0, 0, 0, 0, 4096, 4096, 1, 3098, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 2835, 2810, 1590, 998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 4096, 4095, 0, 42, 0, 2800, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2506, 0, 0, 0, 4079, 0, 0, 0, 1489, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16443, 16552, 16348, 16257, 16424, 16393, 16390, 16265] avg=16384.0000 max=16552 max_rank=1 min=16257 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=114520 max_load_reduction_pct=0.873718 moved_to_echo=65472 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 4072, 3896, 1, 4, 6, 1, 4089, 4086, 102, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 4094, 1, 0, 0, 0, 0, 2769, 1, 4092, 63, 0, 142, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 60, 1, 1, 11, 1, 0, 4095, 0, 1, 0, 3, 0, 1, 4095, 4095, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 388, 0, 1, 0, 15, 0, 3680, 4071, 1, 3097, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 3, 1, 27, 1, 0, 0, 4083, 41, 1, 0, 0, 1, 1, 2826, 2809, 1590, 997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1258, 4071, 4064, 0, 0, 0, 3190, 0, 4058, 1, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 9, 4095, 0, 1, 0, 4081, 0, 1, 4095, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2505, 330, 192, 0, 4095, 1, 0, 1, 1877, 1, 1, 0, 4095, 304, 3728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16288, 16441, 16437, 15350, 16397, 16647, 16381, 17131] avg=16384.0000 max=17131 max_rank=7 min=15350 min_rank=3 imbalance_ratio=1.045593 max_load_reduction=113941 max_load_reduction_pct=0.869301 moved_to_echo=66556 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4081, 3943, 1, 9, 8, 1, 4087, 4089, 107, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1176, 4094, 1, 0, 0, 0, 0, 377, 1, 4092, 39, 0, 147, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 83, 1, 1, 10, 1, 0, 4092, 0, 1, 0, 4, 0, 1, 4095, 4095, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 652, 0, 1, 0, 12, 0, 3995, 4095, 0, 3093, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3920, 0, 3, 1, 17, 1, 0, 0, 4084, 16, 1, 0, 0, 2, 1, 2829, 2812, 1594, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1255, 4085, 4064, 0, 42, 0, 4050, 0, 4064, 1, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 6, 4094, 0, 1, 0, 4082, 0, 1, 4095, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2501, 304, 231, 0, 4095, 1, 0, 1, 2766, 1, 2, 0, 4095, 289, 3777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16357, 14022, 16455, 15945, 16282, 17569, 16378, 18064] avg=16384.0000 max=18064 max_rank=7 min=14022 min_rank=1 imbalance_ratio=1.102539 max_load_reduction=113008 max_load_reduction_pct=0.862183 moved_to_echo=68293 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 0, 2, 0, 0, 0, 4090, 3793, 1, 15, 7, 1, 4084, 4086, 102, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 4094, 1, 0, 0, 0, 0, 3205, 1, 4093, 72, 1, 90, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 58, 1, 1, 2, 1, 0, 4093, 0, 1, 0, 3, 0, 1, 4095, 4095, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 344, 0, 1, 0, 11, 0, 3347, 4073, 0, 3097, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 1, 1, 1, 1, 0, 0, 4090, 17, 1, 0, 0, 0, 1, 2823, 2809, 1590, 997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1257, 4082, 4072, 0, 0, 0, 3383, 0, 4055, 1, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 9, 4095, 0, 1, 0, 4082, 0, 1, 4095, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2505, 45, 195, 0, 4095, 1, 0, 1, 1996, 1, 2, 1, 4095, 329, 3762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16212, 16836, 16421, 14970, 16366, 16856, 16382, 17029] avg=16384.0000 max=17029 max_rank=7 min=14970 min_rank=3 imbalance_ratio=1.039368 max_load_reduction=114043 max_load_reduction_pct=0.870079 moved_to_echo=66633 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 0, 2, 0, 1, 0, 4089, 3910, 1, 11, 6, 1, 4083, 4086, 102, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 4094, 1, 0, 0, 0, 0, 1652, 2, 4092, 69, 0, 14, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 47, 1, 1, 0, 1, 0, 4095, 0, 1, 0, 2, 0, 1, 4095, 4095, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 189, 0, 0, 0, 10, 0, 4043, 4095, 0, 3097, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3981, 0, 2, 1, 6, 1, 0, 0, 4091, 13, 1, 0, 0, 0, 1, 2827, 2809, 1590, 997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1258, 4089, 4053, 0, 68, 0, 3868, 0, 4066, 1, 0, 4, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 4095, 0, 1, 0, 4070, 0, 0, 4095, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2505, 224, 171, 0, 4095, 1, 1, 1, 2512, 1, 2, 0, 4095, 279, 3620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16322, 15203, 16410, 15531, 16321, 17409, 16368, 17508] avg=16384.0000 max=17508 max_rank=7 min=15203 min_rank=1 imbalance_ratio=1.068604 max_load_reduction=113564 max_load_reduction_pct=0.866425 moved_to_echo=67606 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=140032 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 9472, 32768, 128, 0, 128, 0, 0, 22784, 128, 32768, 256, 128, 512, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=140032 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 384, 128, 128, 128, 128, 0, 32768, 0, 128, 0, 128, 0, 128, 32768, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=140032 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32768, 0, 128, 0, 32768, 0, 128, 32768, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=140032 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 9344, 0, 128, 0, 128, 0, 31488, 32768, 128, 24832, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 20096, 1024, 896, 128, 32768, 128, 128, 128, 11392, 128, 128, 128, 32768, 1280, 31360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=140032 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[10112, 32768, 32640, 0, 256, 0, 22784, 0, 32640, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=140032 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32384, 128, 128, 128, 128, 128, 0, 0, 32768, 128, 128, 128, 0, 128, 128, 22656, 22528, 12800, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=140032 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 0, 128, 128, 0, 128, 128, 128, 0, 32768, 31872, 128, 128, 128, 128, 32768, 32768, 896, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=140032 valid_num_tokens=132352 slack_slots=7680 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=140032 valid_num_tokens=132096 slack_slots=7936 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=140032 valid_num_tokens=131840 slack_slots=8192 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=140032 valid_num_tokens=131712 slack_slots=8320 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=140032 valid_num_tokens=132608 slack_slots=7424 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=140032 valid_num_tokens=132736 slack_slots=7296 device_initiated_grouped_gemm=True + [2026-08-30 05:05:04] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 745.8 | throughput per GPU (TFLOP/s/GPU): 447.0 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.674554E+00 | loss scale: 1.0 | grad norm: 4.880 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-30 05:05:04 +[rank0]:[W830 05:05:05.062259053 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 05:05:08.629000 470792 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 05:05:08.630000 470792 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 470885 closing signal SIGTERM +W0830 05:05:08.631000 470792 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 470887 closing signal SIGTERM +W0830 05:05:08.631000 470792 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 470890 closing signal SIGTERM +W0830 05:05:08.631000 470792 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 470892 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 470792 got signal: 15 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..2d1f2e728d6091d136003bf2e09208772f93ab9d --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal/rep1/mem_samples.csv @@ -0,0 +1,256 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 27 +4, 14 +5, 45 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 2177 +1, 2177 +2, 2177 +3, 2177 +4, 2177 +5, 2177 +6, 2177 +7, 2177 +0, 39485 +1, 39949 +2, 39939 +3, 39923 +4, 39511 +5, 39957 +6, 39941 +7, 39975 +0, 40095 +1, 40509 +2, 40539 +3, 40505 +4, 40105 +5, 40559 +6, 40549 +7, 40547 +0, 42733 +1, 43181 +2, 43181 +3, 43181 +4, 42733 +5, 43181 +6, 43181 +7, 43181 +0, 42971 +1, 43419 +2, 43419 +3, 43419 +4, 42971 +5, 43419 +6, 43419 +7, 43419 +0, 43175 +1, 43603 +2, 43623 +3, 43619 +4, 43187 +5, 43641 +6, 43647 +7, 43637 +0, 44029 +1, 44477 +2, 44477 +3, 44477 +4, 44029 +5, 44477 +6, 44477 +7, 44477 +0, 45123 +1, 45571 +2, 45571 +3, 45571 +4, 45123 +5, 45571 +6, 45571 +7, 45571 +0, 46707 +1, 47155 +2, 47155 +3, 47155 +4, 46707 +5, 47155 +6, 47155 +7, 47155 +0, 47807 +1, 48255 +2, 48255 +3, 48255 +4, 47807 +5, 48255 +6, 48255 +7, 48255 +0, 84631 +1, 84657 +2, 84657 +3, 84657 +4, 84651 +5, 84657 +6, 84657 +7, 84657 +0, 85727 +1, 85755 +2, 85753 +3, 85753 +4, 85747 +5, 85753 +6, 85753 +7, 85753 +0, 85729 +1, 85755 +2, 85755 +3, 85755 +4, 85749 +5, 85755 +6, 85755 +7, 85755 +0, 85729 +1, 85755 +2, 85755 +3, 85755 +4, 85749 +5, 85755 +6, 85755 +7, 85755 +0, 102899 +1, 102935 +2, 102943 +3, 102937 +4, 102947 +5, 102941 +6, 102941 +7, 102945 +0, 103761 +1, 103805 +2, 103847 +3, 103855 +4, 103869 +5, 103835 +6, 103901 +7, 103867 +0, 104525 +1, 104573 +2, 104573 +3, 104573 +4, 104565 +5, 104573 +6, 104573 +7, 104573 +0, 105191 +1, 105273 +2, 105265 +3, 105241 +4, 105235 +5, 105303 +6, 105293 +7, 105351 +0, 128441 +1, 128525 +2, 128505 +3, 128505 +4, 128547 +5, 128525 +6, 129579 +7, 128485 +0, 128441 +1, 128525 +2, 128505 +3, 128505 +4, 128547 +5, 128525 +6, 129579 +7, 129579 +0, 128441 +1, 128525 +2, 128505 +3, 128505 +4, 128547 +5, 128525 +6, 129579 +7, 129579 +0, 128441 +1, 128525 +2, 128505 +3, 128505 +4, 128547 +5, 128525 +6, 129579 +7, 129579 +0, 127495 +1, 127579 +2, 127559 +3, 127559 +4, 127601 +5, 127579 +6, 128633 +7, 128633 +0, 1965 +1, 2222 +2, 127559 +3, 3246 +4, 1966 +5, 1966 +6, 1966 +7, 1966 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal/rep1/run.env b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..80e60cc4edecc17959b9813365c66fd6f580cc52 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal/rep1/run.env @@ -0,0 +1,70 @@ +backend=echo +config=G4 +family=tierA +repo=/workspace/Megatron-MoE-Echo +branch=token-drop-analysis +commit=b099fd84 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=896 +num_experts=128 +topk=32 +experts_per_rank=16 +expert_params=11010048 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=131072 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=1.068359375 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=1 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal/rep1/train.log b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..4013c984abceb530103cc38f5f6f88aaacff74f6 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal/rep1/train.log @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea28d70324acbb70c38001ca3dd0acecfca66d1018e81f44300004b450edbd26 +size 10844923 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1/mem_samples.csv index 2cdae5f06275aecda9ee05125a454dc885514640..5ed2bf7faa15758c97561e784d91b987714f82b2 100644 --- a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1/mem_samples.csv +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1/mem_samples.csv @@ -38,3 +38,227 @@ 5, 4 6, 4 7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 27 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 2231 +1, 2231 +2, 2231 +3, 2231 +4, 2231 +5, 2231 +6, 2231 +7, 2231 +0, 39429 +1, 39857 +2, 39889 +3, 39881 +4, 39445 +5, 39877 +6, 39887 +7, 39893 +0, 40135 +1, 40583 +2, 40583 +3, 40583 +4, 40135 +5, 40583 +6, 40583 +7, 40583 +0, 42733 +1, 43181 +2, 43181 +3, 43181 +4, 42733 +5, 43181 +6, 43181 +7, 43181 +0, 42971 +1, 43419 +2, 43419 +3, 43419 +4, 42971 +5, 43419 +6, 43419 +7, 43419 +0, 43083 +1, 43531 +2, 43531 +3, 43531 +4, 43083 +5, 43531 +6, 43531 +7, 43531 +0, 44029 +1, 44477 +2, 44477 +3, 44477 +4, 44029 +5, 44477 +6, 44477 +7, 44477 +0, 46077 +1, 46525 +2, 46525 +3, 46525 +4, 46077 +5, 46525 +6, 46525 +7, 46525 +0, 49125 +1, 49573 +2, 49573 +3, 49573 +4, 49125 +5, 49573 +6, 49573 +7, 49573 +0, 51179 +1, 51627 +2, 51627 +3, 51627 +4, 51179 +5, 51627 +6, 51627 +7, 51627 +0, 98665 +1, 98741 +2, 96659 +3, 98741 +4, 98685 +5, 98741 +6, 98741 +7, 98741 +0, 102459 +1, 102515 +2, 102515 +3, 102515 +4, 102479 +5, 102515 +6, 102515 +7, 102515 +0, 103485 +1, 103541 +2, 103541 +3, 103541 +4, 103505 +5, 103541 +6, 103541 +7, 103541 +0, 103485 +1, 103541 +2, 103541 +3, 103541 +4, 103505 +5, 103541 +6, 103541 +7, 103541 +0, 103485 +1, 103541 +2, 103541 +3, 103541 +4, 103505 +5, 103541 +6, 103541 +7, 103541 +0, 120929 +1, 120995 +2, 121007 +3, 120993 +4, 120993 +5, 120991 +6, 121029 +7, 121023 +0, 121787 +1, 121845 +2, 121869 +3, 121849 +4, 121847 +5, 121893 +6, 121903 +7, 121901 +0, 122545 +1, 122601 +2, 122601 +3, 122601 +4, 122565 +5, 122601 +6, 122601 +7, 122601 +0, 123149 +1, 123241 +2, 123225 +3, 123215 +4, 123211 +5, 123291 +6, 123291 +7, 123295 +0, 145563 +1, 146037 +2, 146037 +3, 146037 +4, 146001 +5, 146037 +6, 146037 +7, 146037 +0, 145563 +1, 146037 +2, 146037 +3, 146037 +4, 146001 +5, 146037 +6, 146037 +7, 146037 +0, 145563 +1, 146037 +2, 146037 +3, 146037 +4, 146001 +5, 146037 +6, 146037 +7, 146037 +0, 145563 +1, 146037 +2, 146037 +3, 146037 +4, 146001 +5, 146037 +6, 146037 +7, 146037 +0, 145271 +1, 145777 +2, 146033 +3, 146033 +4, 145709 +5, 146033 +6, 146033 +7, 146033 +0, 1966 +1, 1966 +2, 1966 +3, 2222 +4, 145055 +5, 3246 +6, 1966 +7, 1966 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1/train.log b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1/train.log index 78dd598df00ef4e32eabf9f9bfc4268123423118..d0dc9338651a34a32a0b53c50fae671405fd6169 100644 --- a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1/train.log +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1/train.log @@ -11,3 +11,16889 @@ W0830 05:02:25.817000 466818 torch/distributed/run.py:766] W0830 05:02:25.817000 466818 torch/distributed/run.py:766] ***************************************** W0830 05:02:25.817000 466818 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. W0830 05:02:25.817000 466818 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 896 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 128 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 2.0 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 32 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 57 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 76 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank5]:[W830 05:02:43.913947997 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 05:02:43.913990365 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 05:02:43.914033487 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 05:02:43.914081327 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 05:02:43.914081717 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 05:02:43.914271588 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 05:02:43.915653946 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.036 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 05:02:43.077773226 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.986 seconds +time to initialize megatron (seconds): 7.476 +[after megatron is initialized] datetime: 2026-08-30 05:02:49 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 3910275072 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (42479616 elements, 42479616 padded size): + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.final_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight +Params for bucket 3 (42475520 elements, 42475520 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_qkv.weight +Params for bucket 4 (42475520 elements, 42475520 padded size): + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight +Params for bucket 5 (42475520 elements, 42475520 padded size): + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight +Params for bucket 6 (42475520 elements, 42475520 padded size): + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_qkv.weight +Params for bucket 7 (42475520 elements, 42475520 padded size): + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight +Params for bucket 8 (42475520 elements, 42475520 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_qkv.weight +Params for bucket 9 (42475520 elements, 42475520 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 10 (42475520 elements, 42475520 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.pre_mlp_layernorm.weight +Params for bucket 11 (42475520 elements, 42475520 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.pre_mlp_layernorm.weight +Params for bucket 12 (42475520 elements, 42475520 padded size): + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight +Params for bucket 13 (42475520 elements, 42475520 padded size): + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight +Params for bucket 14 (42475520 elements, 42475520 padded size): + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight +Params for bucket 15 (42475520 elements, 42475520 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight +Params for bucket 16 (42475520 elements, 42475520 padded size): + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 17 (42475520 elements, 42475520 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_proj.weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 67 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 +Params for bucket 3 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 +Params for bucket 23 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 +Params for bucket 24 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 +Params for bucket 25 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 +Params for bucket 28 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 +Params for bucket 29 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 +Params for bucket 36 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 +Params for bucket 37 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 +Params for bucket 42 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 +Params for bucket 43 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 +Params for bucket 44 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 46 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 +Params for bucket 47 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 +Params for bucket 48 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 +Params for bucket 49 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 +Params for bucket 50 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 +Params for bucket 51 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 +Params for bucket 52 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 +Params for bucket 53 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 54 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 55 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 +Params for bucket 56 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 +Params for bucket 59 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 +Params for bucket 60 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 +Params for bucket 61 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 +Params for bucket 62 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 63 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 +Params for bucket 64 (40370176 elements, 40370176 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 +Params for bucket 66 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 +Params for bucket 67 (29360128 elements, 29360128 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 05:02:50 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 192 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 05:02:50 +done with setup ... +training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (662.71, 678.08) + train/valid/test-data-iterators-setup ..........: (63.71, 68.67) +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 05:02:50 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[141, 2558, 988, 249, 1113, 448, 983, 919, 1184, 173, 857, 140, 931, 576, 155, 423, 246, 333, 1081, 736, 926, 110, 2822, 1473, 471, 777, 1198, 557, 1420, 563, 1395, 1552, 3062, 442, 134, 491, 158, 1928, 2291, 451, 3111, 253, 1039, 1009, 1419, 1203, 1139, 643, 240, 234, 272, 1952, 1042, 413, 648, 75, 513, 436, 1083, 1028, 1203, 661, 1013, 687, 575, 1738, 530, 983, 1419, 205, 690, 1588, 1018, 2034, 309, 1815, 636, 513, 1677, 1079, 3201, 1196, 2883, 665, 1986, 147, 951, 1193, 2571, 210, 551, 1186, 1186, 501, 578, 1070, 1622, 368, 1693, 687, 1700, 483, 2077, 2087, 1139, 967, 926, 2171, 656, 460, 1744, 1119, 272, 1225, 1504, 1451, 326, 686, 1375, 1319, 812, 1739, 1527, 821, 1480, 893, 559, 529] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[51, 2962, 486, 178, 1629, 530, 1087, 696, 956, 179, 1546, 284, 882, 666, 179, 296, 69, 810, 559, 545, 1006, 220, 3203, 1941, 551, 1196, 885, 1138, 864, 292, 1426, 1838, 3083, 743, 254, 224, 81, 2271, 1615, 427, 2349, 536, 1500, 1092, 1380, 1643, 1204, 1395, 187, 553, 265, 1864, 625, 464, 391, 65, 830, 267, 932, 1808, 1548, 487, 564, 1312, 859, 1836, 923, 1021, 1246, 103, 946, 1661, 1203, 1853, 216, 1691, 809, 1012, 2237, 810, 2848, 886, 3088, 592, 1753, 342, 550, 1810, 1547, 244, 703, 1309, 926, 664, 395, 1181, 2208, 163, 1559, 631, 1359, 764, 1227, 2110, 813, 925, 508, 2016, 502, 299, 2244, 1368, 344, 818, 1293, 1195, 307, 468, 1030, 1401, 801, 1699, 1434, 746, 1307, 441, 414, 305] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[76, 2952, 694, 262, 1513, 618, 1031, 992, 1025, 182, 955, 181, 603, 562, 225, 525, 146, 647, 639, 919, 902, 147, 2914, 1759, 460, 702, 1022, 795, 1334, 422, 1610, 2017, 3080, 356, 206, 440, 121, 1948, 2197, 575, 2977, 808, 1488, 441, 771, 1133, 1184, 1302, 243, 370, 325, 2000, 852, 458, 499, 149, 325, 631, 711, 1631, 1865, 592, 1057, 1093, 514, 1945, 731, 837, 1397, 395, 791, 1551, 1039, 1566, 121, 1937, 676, 1405, 1955, 1046, 2077, 1063, 2890, 384, 2231, 485, 921, 803, 2187, 230, 762, 840, 1052, 376, 440, 858, 2159, 267, 1789, 547, 1502, 975, 1717, 1837, 998, 815, 587, 2342, 766, 479, 1770, 793, 177, 928, 1614, 1372, 368, 826, 783, 1310, 850, 1749, 1477, 998, 1764, 412, 691, 246] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[159, 2994, 594, 592, 1464, 573, 1045, 499, 1166, 153, 1871, 113, 1131, 1001, 173, 467, 115, 361, 1232, 695, 769, 446, 2851, 1928, 502, 841, 941, 991, 1290, 242, 1469, 1433, 2961, 509, 231, 218, 232, 2712, 2014, 503, 2559, 205, 1276, 831, 1172, 1312, 1106, 1111, 333, 202, 427, 1617, 696, 876, 685, 61, 735, 581, 1194, 1557, 1580, 602, 323, 961, 912, 1668, 513, 1097, 973, 235, 914, 1703, 927, 2110, 324, 2125, 589, 893, 2331, 721, 2409, 1802, 2839, 613, 2029, 126, 730, 1151, 1357, 294, 609, 1374, 783, 579, 463, 874, 2035, 267, 1729, 907, 1245, 710, 1828, 2892, 926, 969, 392, 2184, 750, 437, 2161, 1321, 226, 959, 958, 1347, 128, 397, 646, 1174, 778, 1173, 1460, 551, 1848, 362, 401, 361] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[113, 2241, 727, 458, 1232, 233, 987, 685, 1242, 138, 1420, 629, 711, 647, 125, 431, 113, 642, 517, 935, 1080, 109, 2755, 1808, 482, 796, 864, 778, 1095, 121, 1712, 1818, 2940, 448, 351, 468, 480, 2422, 1571, 608, 2569, 332, 1246, 886, 1503, 1109, 1450, 801, 279, 510, 292, 2473, 801, 421, 547, 86, 546, 612, 1255, 1489, 2007, 291, 548, 1192, 488, 2195, 800, 1593, 813, 223, 849, 1518, 909, 2339, 231, 1743, 730, 803, 2068, 706, 2538, 1120, 2702, 627, 1482, 538, 862, 913, 1963, 336, 537, 1073, 915, 964, 397, 668, 2294, 273, 1816, 582, 1573, 702, 1931, 2417, 533, 1028, 486, 1741, 771, 555, 1783, 1262, 285, 1261, 834, 1398, 268, 482, 691, 1503, 1107, 1395, 1767, 747, 2315, 364, 673, 415] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[128, 3053, 828, 361, 1245, 611, 692, 1162, 778, 324, 972, 388, 1142, 1006, 134, 413, 76, 489, 1473, 718, 818, 185, 2596, 1701, 840, 956, 1105, 1373, 1484, 312, 1738, 2054, 2803, 418, 300, 598, 253, 1995, 2029, 441, 2673, 300, 1548, 725, 1319, 1272, 875, 1260, 369, 361, 221, 1659, 1189, 598, 553, 104, 577, 311, 849, 1426, 1436, 700, 1145, 1174, 1123, 2130, 530, 1243, 785, 386, 596, 2042, 705, 2221, 469, 1756, 580, 553, 2311, 624, 2421, 1242, 3026, 409, 2030, 284, 1034, 989, 1213, 289, 519, 1046, 624, 661, 486, 1104, 1884, 238, 1902, 570, 1491, 932, 1575, 2465, 701, 941, 554, 2330, 766, 467, 2071, 644, 243, 1099, 1097, 956, 486, 576, 1053, 1496, 629, 1077, 1577, 614, 1356, 488, 423, 304] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[48, 2776, 896, 457, 1357, 502, 747, 852, 921, 155, 996, 561, 1182, 583, 326, 487, 135, 444, 1177, 865, 1046, 110, 2956, 2088, 460, 975, 832, 1248, 1287, 388, 1995, 2383, 2787, 798, 233, 482, 184, 1944, 1574, 292, 2405, 885, 1059, 1011, 1220, 1572, 1040, 1203, 267, 201, 581, 2076, 744, 371, 546, 90, 536, 195, 1054, 1556, 1391, 358, 649, 1089, 1031, 2023, 445, 1103, 937, 455, 744, 1282, 908, 2335, 219, 1968, 696, 838, 2770, 679, 2262, 1212, 2768, 494, 2135, 480, 593, 958, 1343, 316, 749, 903, 1061, 561, 389, 611, 1996, 235, 1308, 472, 1663, 1203, 1798, 2457, 837, 840, 624, 2328, 624, 157, 2319, 963, 310, 1400, 741, 1077, 350, 524, 1202, 1287, 932, 1379, 1432, 956, 1299, 480, 335, 648] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[79, 2956, 568, 182, 1217, 425, 769, 1156, 1136, 291, 1028, 174, 953, 828, 309, 385, 124, 402, 314, 1101, 1058, 118, 2707, 1734, 263, 1062, 1145, 1025, 1490, 431, 1436, 2206, 3142, 484, 266, 958, 141, 2349, 1831, 722, 2931, 416, 1187, 583, 1091, 1250, 1266, 999, 175, 228, 228, 2144, 1386, 342, 285, 69, 587, 493, 916, 1613, 1522, 380, 1460, 840, 680, 2009, 462, 1059, 1312, 203, 704, 1509, 912, 1852, 152, 2179, 786, 674, 2880, 795, 2722, 1040, 2331, 461, 2377, 349, 823, 1082, 2009, 261, 548, 1037, 1266, 463, 404, 853, 2051, 229, 2306, 464, 1339, 1104, 1963, 2514, 823, 671, 593, 2129, 661, 453, 1923, 862, 265, 851, 1301, 1087, 147, 543, 832, 1232, 792, 1410, 1471, 680, 1578, 388, 609, 251] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[141, 2558, 988, 249, 1113, 448, 983, 919, 1184, 173, 857, 140, 931, 576, 155, 423, 1712, 975, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 333, 1081, 736, 926, 110, 2543, 1473, 471, 777, 1198, 557, 1420, 563, 1395, 1552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 442, 134, 491, 158, 1928, 2291, 451, 3111, 253, 1039, 1009, 1419, 1203, 1139, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 234, 272, 1952, 1042, 413, 648, 75, 513, 436, 1083, 1028, 1203, 661, 1013, 687, 2798, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 1738, 530, 983, 1419, 205, 690, 1588, 1018, 2034, 309, 1815, 636, 513, 497, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3201, 1196, 1171, 665, 1986, 147, 951, 1193, 2571, 210, 551, 1186, 1186, 501, 578, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1622, 368, 1693, 687, 1700, 483, 2077, 0, 1139, 967, 926, 1156, 656, 460, 1744, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 1225, 1504, 1451, 326, 686, 1375, 1319, 812, 1739, 1527, 821, 1480, 893, 559, 529, 1015, 482, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15223, 15381, 15975, 15410, 15629, 18363, 16797, 18294] avg=16384.0000 max=18363 max_rank=5 min=15223 min_rank=0 imbalance_ratio=1.120789 max_load_reduction=112709 max_load_reduction_pct=0.859901 moved_to_echo=69083 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[51, 2962, 486, 178, 1629, 530, 1087, 696, 956, 179, 1546, 284, 882, 666, 179, 296, 1829, 985, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 810, 559, 545, 1006, 220, 2892, 1941, 551, 1196, 885, 1138, 864, 292, 1426, 1838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 743, 254, 224, 81, 2271, 1615, 427, 2349, 536, 1500, 1092, 1380, 1643, 1204, 1395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 553, 265, 1864, 625, 464, 391, 65, 830, 267, 932, 1808, 1548, 487, 564, 1312, 2813, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 859, 1836, 923, 1021, 1246, 103, 946, 1661, 1203, 1853, 216, 1691, 809, 1012, 671, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2848, 886, 1259, 592, 1753, 342, 550, 1810, 1547, 244, 703, 1309, 926, 664, 395, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2208, 163, 1559, 631, 1359, 764, 1227, 0, 813, 925, 508, 1078, 502, 299, 2244, 1368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 818, 1293, 1195, 307, 468, 1030, 1401, 801, 1699, 1434, 746, 1307, 441, 414, 305, 938, 638, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16349, 16232, 16984, 16100, 16860, 17009, 15648, 15890] avg=16384.0000 max=17009 max_rank=5 min=15648 min_rank=6 imbalance_ratio=1.038147 max_load_reduction=114063 max_load_reduction_pct=0.870232 moved_to_echo=65407 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[76, 2952, 694, 262, 1513, 618, 1031, 992, 1025, 182, 955, 181, 603, 562, 225, 525, 1712, 858, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 647, 639, 919, 902, 147, 2631, 1759, 460, 702, 1022, 795, 1334, 422, 1610, 2017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 356, 206, 440, 121, 1948, 2197, 575, 2977, 808, 1488, 441, 771, 1133, 1184, 1302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 370, 325, 2000, 852, 458, 499, 149, 325, 631, 711, 1631, 1865, 592, 1057, 1093, 2811, 979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 1945, 731, 837, 1397, 395, 791, 1551, 1039, 1566, 121, 1937, 676, 1405, 587, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2077, 1063, 1178, 384, 2231, 485, 921, 803, 2187, 230, 762, 840, 1052, 376, 440, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2159, 267, 1789, 547, 1502, 975, 1717, 0, 998, 815, 587, 1252, 766, 479, 1770, 793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 928, 1614, 1372, 368, 826, 783, 1310, 850, 1749, 1477, 998, 1764, 412, 691, 246, 1090, 557, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15777, 16152, 16216, 16591, 16538, 15887, 16416, 17495] avg=16384.0000 max=17495 max_rank=7 min=15777 min_rank=0 imbalance_ratio=1.067810 max_load_reduction=113577 max_load_reduction_pct=0.866524 moved_to_echo=66336 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[113, 2241, 727, 458, 1232, 233, 987, 685, 1242, 138, 1420, 629, 711, 647, 125, 431, 1600, 1128, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 642, 517, 935, 1080, 109, 2487, 1808, 482, 796, 864, 778, 1095, 121, 1712, 1818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 448, 351, 468, 480, 2422, 1571, 608, 2569, 332, 1246, 886, 1503, 1109, 1450, 801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 510, 292, 2473, 801, 421, 547, 86, 546, 612, 1255, 1489, 2007, 291, 548, 1192, 2683, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 2195, 800, 1593, 813, 223, 849, 1518, 909, 2339, 231, 1743, 730, 803, 620, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2538, 1120, 1102, 627, 1482, 538, 862, 913, 1963, 336, 537, 1073, 915, 964, 397, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2294, 273, 1816, 582, 1573, 702, 1931, 0, 533, 1028, 486, 931, 771, 555, 1783, 1262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 1261, 834, 1398, 268, 482, 691, 1503, 1107, 1395, 1767, 747, 2315, 364, 673, 415, 810, 590, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15605, 15357, 16501, 17321, 16560, 16035, 16520, 17173] avg=16384.0000 max=17321 max_rank=3 min=15357 min_rank=1 imbalance_ratio=1.057190 max_load_reduction=113751 max_load_reduction_pct=0.867851 moved_to_echo=66288 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[159, 2994, 594, 592, 1464, 573, 1045, 499, 1166, 153, 1871, 113, 1131, 1001, 173, 467, 1681, 1351, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 361, 1232, 695, 769, 446, 2574, 1928, 502, 841, 941, 991, 1290, 242, 1469, 1433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 509, 231, 218, 232, 2712, 2014, 503, 2559, 205, 1276, 831, 1172, 1312, 1106, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 202, 427, 1617, 696, 876, 685, 61, 735, 581, 1194, 1557, 1580, 602, 323, 961, 2702, 1541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 912, 1668, 513, 1097, 973, 235, 914, 1703, 927, 2110, 324, 2125, 589, 893, 699, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2409, 1802, 1158, 613, 2029, 126, 730, 1151, 1357, 294, 609, 1374, 783, 579, 463, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2035, 267, 1729, 907, 1245, 710, 1828, 0, 926, 969, 392, 1168, 750, 437, 2161, 1321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 959, 958, 1347, 128, 397, 646, 1174, 778, 1173, 1460, 551, 1848, 362, 401, 361, 1016, 665, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17994, 15829, 16250, 16673, 16403, 16351, 16845, 14727] avg=16384.0000 max=17994 max_rank=0 min=14727 min_rank=7 imbalance_ratio=1.098267 max_load_reduction=113078 max_load_reduction_pct=0.862717 moved_to_echo=64326 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[128, 3053, 828, 361, 1245, 611, 692, 1162, 778, 324, 972, 388, 1142, 1006, 134, 413, 1792, 1150, 959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 489, 1473, 718, 818, 185, 2344, 1701, 840, 956, 1105, 1373, 1484, 312, 1738, 2054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 418, 300, 598, 253, 1995, 2029, 441, 2673, 300, 1548, 725, 1319, 1272, 875, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 361, 221, 1659, 1189, 598, 553, 104, 577, 311, 849, 1426, 1436, 700, 1145, 1174, 2558, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 2130, 530, 1243, 785, 386, 596, 2042, 705, 2221, 469, 1756, 580, 553, 693, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2421, 1242, 1234, 409, 2030, 284, 1034, 989, 1213, 289, 519, 1046, 624, 661, 486, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1884, 238, 1902, 570, 1491, 932, 1575, 0, 701, 941, 554, 1246, 766, 467, 2071, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 1099, 1097, 956, 486, 576, 1053, 1496, 629, 1077, 1577, 614, 1356, 488, 423, 304, 1084, 659, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17138, 17666, 16251, 16545, 16436, 15585, 15982, 15469] avg=16384.0000 max=17666 max_rank=1 min=15469 min_rank=7 imbalance_ratio=1.078247 max_load_reduction=113406 max_load_reduction_pct=0.865219 moved_to_echo=63472 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[48, 2776, 896, 457, 1357, 502, 747, 852, 921, 155, 996, 561, 1182, 583, 326, 487, 1639, 1147, 1149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 444, 1177, 865, 1046, 110, 2669, 2088, 460, 975, 832, 1248, 1287, 388, 1995, 2383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 798, 233, 482, 184, 1944, 1574, 292, 2405, 885, 1059, 1011, 1220, 1572, 1040, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 201, 581, 2076, 744, 371, 546, 90, 536, 195, 1054, 1556, 1391, 358, 649, 1089, 2543, 1310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1031, 2023, 445, 1103, 937, 455, 744, 1282, 908, 2335, 219, 1968, 696, 838, 831, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2262, 1212, 1129, 494, 2135, 480, 593, 958, 1343, 316, 749, 903, 1061, 561, 389, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1996, 235, 1308, 472, 1663, 1203, 1798, 0, 837, 840, 624, 1245, 624, 157, 2319, 963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 1400, 741, 1077, 350, 524, 1202, 1287, 932, 1379, 1432, 956, 1299, 480, 335, 648, 1083, 790, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16781, 18102, 16146, 15557, 16494, 15196, 16284, 16512] avg=16384.0000 max=18102 max_rank=1 min=15196 min_rank=5 imbalance_ratio=1.104858 max_load_reduction=112970 max_load_reduction_pct=0.861893 moved_to_echo=64486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[79, 2956, 568, 182, 1217, 425, 769, 1156, 1136, 291, 1028, 174, 953, 828, 309, 385, 1380, 1174, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 402, 314, 1101, 1058, 118, 2444, 1734, 263, 1062, 1145, 1025, 1490, 431, 1436, 2206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 484, 266, 958, 141, 2349, 1831, 722, 2931, 416, 1187, 583, 1091, 1250, 1266, 999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 228, 228, 2144, 1386, 342, 285, 69, 587, 493, 916, 1613, 1522, 380, 1460, 840, 2867, 1340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 680, 2009, 462, 1059, 1312, 203, 704, 1509, 912, 1852, 152, 2179, 786, 674, 864, 795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2722, 1040, 951, 461, 2377, 349, 823, 1082, 2009, 261, 548, 1037, 1266, 463, 404, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2051, 229, 2306, 464, 1339, 1104, 1963, 0, 823, 671, 593, 1138, 661, 453, 1923, 862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 851, 1301, 1087, 147, 543, 832, 1232, 792, 1410, 1471, 680, 1578, 388, 609, 251, 991, 821, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16205, 16353, 16749, 16875, 16152, 16646, 16580, 15512] avg=16384.0000 max=16875 max_rank=3 min=15512 min_rank=7 imbalance_ratio=1.029968 max_load_reduction=114197 max_load_reduction_pct=0.871254 moved_to_echo=64890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[896, 22528, 5888, 2816, 10880, 3968, 7424, 7040, 8448, 1664, 9728, 2560, 7552, 5888, 1664, 3456, 13440, 8832, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[16256, 2048, 14208, 4864, 11904, 6912, 14208, 0, 6784, 7168, 4736, 9216, 5504, 3328, 16128, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[6272, 15616, 4992, 8960, 8960, 2304, 6272, 12928, 7680, 16384, 2048, 15232, 5504, 6784, 5504, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2176, 2688, 2688, 15872, 7424, 3968, 4224, 768, 4736, 3584, 8064, 12160, 12672, 4096, 6784, 8448, 21888, 10112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1024, 4224, 7040, 6528, 7680, 1536, 20608, 14464, 4096, 7424, 8064, 7936, 10368, 2816, 12800, 15360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[20480, 9600, 9216, 4352, 16128, 2816, 6528, 8960, 14208, 2304, 4992, 8832, 7936, 4864, 3584, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2176, 8576, 9344, 9984, 2432, 4608, 7680, 10752, 6784, 11648, 12160, 6144, 13056, 3840, 4224, 3072, 8064, 5248, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2176, 4224, 2048, 3968, 1664, 17664, 15232, 4096, 21632, 3840, 10368, 6656, 9984, 10496, 9344, 8832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[3556, 0, 2087, 258, 4030, 1, 604, 1422, 3948, 54, 4, 221, 1265, 226, 4, 486, 4, 216, 621, 0, 877, 4004, 203, 936, 1808, 467, 0, 1603, 357, 3358, 4072, 13, 2181, 855, 3117, 0, 93, 16, 0, 7, 1114, 491, 22, 300, 0, 23, 57, 43, 0, 7, 40, 0, 11, 983, 2, 3, 0, 4083, 315, 1266, 2606, 285, 23, 0, 45, 1552, 2858, 88, 240, 0, 4019, 6, 521, 2261, 802, 249, 729, 43, 3015, 0, 3351, 2389, 1272, 1331, 3093, 217, 16, 4094, 4, 224, 319, 4025, 1145, 8, 2, 487, 1, 0, 4081, 16, 3048, 37, 3658, 0, 148, 1473, 2969, 882, 3526, 2938, 4, 314, 5, 333, 1676, 22, 0, 1828, 200, 614, 848, 8, 0, 4063, 1428, 100, 0, 3799] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1270, 173, 3446, 925, 2094, 0, 37, 3247, 3386, 176, 1818, 254, 1188, 207, 0, 2327, 5, 1, 3144, 40, 507, 1856, 516, 5, 15, 4, 0, 116, 0, 88, 2369, 0, 1052, 986, 3666, 0, 285, 0, 3382, 2, 19, 18, 0, 111, 0, 0, 299, 0, 43, 62, 98, 0, 2725, 142, 23, 58, 58, 4064, 134, 1441, 2867, 0, 3, 3, 1, 78, 3128, 1262, 0, 0, 3807, 43, 3103, 2159, 370, 347, 114, 0, 1144, 0, 4085, 1113, 921, 3800, 3584, 135, 669, 3650, 15, 106, 2659, 4047, 6, 123, 0, 3250, 0, 220, 3574, 60, 3530, 0, 4036, 0, 1, 3533, 162, 166, 2939, 1604, 227, 2149, 17, 1438, 798, 0, 0, 4066, 285, 438, 1670, 0, 0, 1524, 63, 17, 12, 4069] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[3977, 0, 1241, 68, 2433, 5, 22, 1831, 4061, 50, 1243, 35, 3970, 625, 33, 2095, 40, 0, 2111, 6, 87, 1711, 633, 45, 1, 46, 0, 38, 1, 2683, 2901, 1, 664, 808, 2084, 0, 11, 3, 1, 0, 38, 3, 7, 0, 110, 10, 1032, 96, 0, 267, 481, 0, 37, 2276, 0, 0, 4, 3426, 688, 2257, 140, 2, 16, 239, 0, 0, 2284, 68, 1000, 58, 2438, 39, 2660, 1613, 9, 0, 219, 5, 4084, 0, 4050, 2211, 3499, 67, 3466, 1907, 4024, 3979, 0, 15, 2667, 4096, 70, 25, 0, 2, 0, 1382, 3914, 361, 3867, 4, 245, 22, 84, 623, 4067, 9, 1948, 1483, 41, 3977, 193, 882, 845, 1561, 0, 983, 2849, 2, 1323, 77, 0, 3864, 1306, 0, 0, 3912] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[990, 14, 3154, 1945, 2336, 5, 301, 3891, 3720, 3235, 182, 3, 1718, 1518, 0, 2768, 28, 11, 3447, 3, 27, 3045, 16, 73, 0, 7, 6, 245, 47, 2984, 3998, 0, 3206, 1935, 3493, 0, 0, 40, 19, 0, 565, 871, 0, 1, 9, 1, 0, 8, 0, 33, 255, 0, 276, 3704, 1, 0, 1, 3471, 363, 3469, 261, 0, 84, 318, 0, 1, 3952, 393, 1347, 10, 2666, 0, 1608, 2923, 1, 25, 1, 10, 3879, 6, 4075, 1984, 134, 2, 3921, 18, 1411, 3612, 2, 21, 2274, 4054, 9, 392, 50, 51, 0, 2020, 4094, 1357, 1067, 6, 325, 0, 77, 2298, 3922, 10, 242, 677, 106, 657, 30, 52, 1019, 5, 4, 2484, 1975, 11, 31, 0, 3, 3487, 108, 17, 29, 4026] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[3103, 0, 3339, 14, 2220, 0, 28, 4051, 3884, 223, 69, 183, 2736, 7, 8, 1257, 0, 0, 2650, 0, 151, 3876, 1515, 2859, 46, 168, 21, 505, 0, 1301, 1858, 0, 197, 1870, 3428, 1, 5, 20, 274, 0, 733, 1741, 36, 0, 5, 60, 701, 29, 0, 0, 522, 15, 306, 2862, 19, 20, 4, 4090, 264, 241, 1068, 128, 13, 0, 0, 498, 3762, 688, 100, 0, 3822, 3, 260, 659, 392, 344, 118, 0, 3904, 0, 3873, 1773, 2494, 501, 3990, 14, 1874, 4076, 0, 2772, 2187, 4091, 14, 42, 3, 680, 10, 79, 3867, 117, 3530, 0, 3990, 0, 65, 3680, 3647, 678, 1524, 2178, 937, 1191, 105, 6, 571, 0, 0, 3943, 209, 402, 276, 0, 39, 1059, 0, 0, 2, 1309] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[3514, 0, 4093, 543, 2684, 0, 51, 1774, 4089, 632, 0, 3, 2014, 1361, 0, 3929, 0, 20, 72, 0, 0, 2138, 293, 1553, 3, 422, 1, 3, 15, 3492, 2689, 1, 2212, 2304, 2724, 5, 324, 6, 0, 0, 2545, 0, 0, 128, 92, 87, 5, 71, 0, 2, 0, 3, 48, 3436, 0, 8, 1, 3969, 90, 3133, 895, 0, 28, 154, 1, 0, 3250, 166, 1680, 4, 3981, 0, 1303, 0, 15, 61, 1, 0, 2384, 0, 4040, 3191, 566, 2070, 3523, 117, 680, 4011, 0, 1235, 749, 4090, 35, 0, 1, 889, 1, 686, 3091, 1735, 3670, 66, 3597, 6, 4, 273, 4073, 4, 3422, 5, 297, 281, 119, 107, 0, 669, 53, 2635, 181, 90, 3867, 16, 0, 3217, 49, 59, 9, 3088] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[3736, 0, 3318, 653, 3891, 1, 0, 3772, 3468, 2256, 1124, 0, 461, 521, 0, 3667, 0, 0, 2172, 41, 515, 3858, 0, 303, 0, 0, 3, 23, 0, 934, 3850, 0, 3307, 53, 3006, 3, 3, 0, 0, 5, 90, 4, 0, 3, 0, 0, 68, 5, 1, 43, 521, 0, 169, 1881, 0, 0, 1, 4002, 259, 3093, 6, 5, 1, 9, 1, 0, 3528, 2, 295, 11, 4065, 0, 3018, 3202, 0, 135, 779, 0, 1549, 5, 4051, 775, 556, 1742, 2629, 848, 767, 3409, 2, 24, 744, 4064, 1863, 147, 4, 230, 9, 33, 4038, 1534, 3927, 0, 1882, 1, 1, 567, 776, 27, 1801, 97, 134, 2801, 34, 33, 3006, 89, 1, 3482, 115, 494, 2386, 15, 0, 3974, 995, 1236, 192, 3842] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[3545, 3, 2947, 927, 3120, 2, 0, 1930, 1675, 19, 22, 2, 2072, 0, 0, 2627, 111, 17, 454, 135, 6, 4048, 337, 891, 16, 12, 2, 10, 12, 1151, 4091, 8, 4068, 59, 3371, 2, 0, 0, 242, 39, 1292, 0, 0, 85, 0, 42, 51, 5, 14, 702, 1313, 17, 35, 66, 495, 38, 394, 3786, 3011, 583, 432, 0, 2, 8, 132, 104, 2996, 4, 95, 0, 4096, 26, 1636, 2788, 1, 1414, 1173, 0, 580, 1, 3764, 2907, 2560, 3122, 3132, 51, 575, 3928, 68, 515, 366, 4078, 1877, 17, 0, 758, 10, 193, 4096, 2756, 3300, 2, 4062, 115, 1, 736, 1654, 428, 4019, 1024, 0, 1927, 157, 117, 621, 83, 322, 4065, 2, 283, 1507, 37, 1, 4019, 328, 0, 110, 1989] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[274, 173, 3446, 925, 2094, 0, 37, 3247, 0, 176, 1818, 254, 1188, 207, 0, 2327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 3144, 40, 507, 1856, 516, 5, 15, 4, 0, 116, 0, 88, 2369, 0, 868, 1782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 986, 3666, 0, 285, 0, 3382, 2, 19, 18, 0, 111, 0, 0, 299, 0, 4047, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 62, 98, 0, 2725, 142, 23, 58, 58, 4064, 134, 1441, 2867, 0, 3, 3, 3574, 1604, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 78, 3128, 1262, 0, 0, 3807, 43, 3103, 2159, 370, 347, 114, 0, 1144, 0, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1113, 921, 3800, 3584, 135, 669, 2399, 15, 106, 2659, 0, 6, 123, 0, 3250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 60, 2028, 0, 4036, 0, 1, 3533, 162, 166, 2939, 1604, 227, 2149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1438, 798, 0, 0, 4066, 285, 438, 1670, 0, 0, 1524, 63, 17, 12, 4069, 1502, 128, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16166, 11316, 17780, 17071, 16599, 18780, 17125, 16235] avg=16384.0000 max=18780 max_rank=5 min=11316 min_rank=1 imbalance_ratio=1.146240 max_load_reduction=112292 max_load_reduction_pct=0.856720 moved_to_echo=68739 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[881, 0, 1241, 68, 2433, 5, 22, 1831, 0, 50, 1243, 35, 3970, 625, 33, 2095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 2111, 6, 87, 1711, 633, 45, 1, 46, 0, 38, 1, 2683, 2901, 1, 2705, 2137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 664, 808, 2084, 0, 11, 3, 1, 0, 38, 3, 7, 0, 110, 10, 1032, 96, 4096, 3878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 481, 0, 37, 2276, 0, 0, 4, 3426, 688, 2257, 140, 2, 16, 239, 3914, 1924, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2284, 68, 1000, 58, 2438, 39, 2660, 1613, 9, 0, 219, 5, 4084, 0, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2211, 3499, 67, 3466, 1907, 4024, 2625, 0, 15, 2667, 0, 70, 25, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1382, 0, 361, 2226, 4, 245, 22, 84, 623, 4067, 9, 1948, 1483, 41, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 882, 845, 1561, 0, 983, 2849, 2, 1323, 77, 0, 3864, 1306, 0, 0, 3912, 1641, 391, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14532, 15146, 12841, 15843, 15609, 20578, 16472, 20051] avg=16384.0000 max=20578 max_rank=5 min=12841 min_rank=2 imbalance_ratio=1.255981 max_load_reduction=110494 max_load_reduction_pct=0.843002 moved_to_echo=72710 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[220, 14, 3154, 1945, 2336, 5, 301, 3891, 0, 3235, 182, 3, 1718, 1518, 0, 2768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 11, 3447, 3, 27, 3045, 16, 73, 0, 7, 6, 245, 47, 2984, 3998, 0, 673, 1957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 1935, 3493, 0, 0, 40, 19, 0, 565, 871, 0, 1, 9, 1, 0, 8, 4054, 3902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 255, 0, 276, 3704, 1, 0, 1, 3471, 363, 3469, 261, 0, 84, 318, 4094, 1763, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3952, 393, 1347, 10, 2666, 0, 1608, 2923, 1, 25, 1, 10, 3879, 6, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1984, 134, 2, 3921, 18, 1411, 2383, 2, 21, 2274, 0, 9, 392, 50, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2020, 0, 1357, 615, 6, 325, 0, 77, 2298, 3922, 10, 242, 677, 106, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 52, 1019, 5, 4, 2484, 1975, 11, 31, 0, 3, 3487, 108, 17, 29, 4026, 452, 97, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21290, 16567, 18104, 18266, 17850, 12652, 12312, 14031] avg=16384.0000 max=21290 max_rank=0 min=12312 min_rank=6 imbalance_ratio=1.299438 max_load_reduction=109782 max_load_reduction_pct=0.837570 moved_to_echo=56845 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[788, 0, 2087, 258, 4030, 1, 604, 1422, 0, 54, 4, 221, 1265, 226, 4, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 216, 621, 0, 877, 4004, 203, 936, 1808, 467, 0, 1603, 357, 3358, 4072, 13, 2419, 2077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 855, 3117, 0, 93, 16, 0, 7, 1114, 491, 22, 300, 0, 23, 57, 43, 4025, 3209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 40, 0, 11, 983, 2, 3, 0, 4083, 315, 1266, 2606, 285, 23, 0, 4081, 1871, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 1552, 2858, 88, 240, 0, 4019, 6, 521, 2261, 802, 249, 729, 43, 3015, 0, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2389, 1272, 1331, 3093, 217, 16, 2701, 4, 224, 319, 0, 1145, 8, 2, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 16, 1755, 37, 3658, 0, 148, 1473, 2969, 882, 3526, 2938, 4, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 333, 1676, 22, 0, 1828, 200, 614, 848, 8, 0, 4063, 1428, 100, 0, 3799, 1293, 349, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11450, 23035, 15553, 15718, 17593, 13208, 17721, 16794] avg=16384.0000 max=23035 max_rank=1 min=11450 min_rank=0 imbalance_ratio=1.405945 max_load_reduction=108037 max_load_reduction_pct=0.824257 moved_to_echo=65316 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[687, 0, 3339, 14, 2220, 0, 28, 4051, 0, 223, 69, 183, 2736, 7, 8, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2650, 0, 151, 3876, 1515, 2859, 46, 168, 21, 505, 0, 1301, 1858, 0, 2111, 2044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 1870, 3428, 1, 5, 20, 274, 0, 733, 1741, 36, 0, 5, 60, 701, 29, 4091, 3710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 15, 306, 2862, 19, 20, 4, 4090, 264, 241, 1068, 128, 13, 0, 3867, 1840, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 3762, 688, 100, 0, 3822, 3, 260, 659, 392, 344, 118, 0, 3904, 0, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1773, 2494, 501, 3990, 14, 1874, 2689, 0, 2772, 2187, 0, 14, 42, 3, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 79, 0, 117, 2032, 0, 3990, 0, 65, 3680, 3647, 678, 1524, 2178, 937, 1191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 6, 571, 0, 0, 3943, 209, 402, 276, 0, 39, 1059, 0, 0, 2, 1309, 1498, 305, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14822, 19105, 16901, 15422, 15710, 19033, 20128, 9951] avg=16384.0000 max=20128 max_rank=6 min=9951 min_rank=7 imbalance_ratio=1.228516 max_load_reduction=110944 max_load_reduction_pct=0.846436 moved_to_echo=64822 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[779, 0, 4093, 543, 2684, 0, 51, 1774, 0, 632, 0, 3, 2014, 1361, 0, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 72, 0, 0, 2138, 293, 1553, 3, 422, 1, 3, 15, 3492, 2689, 1, 2390, 2152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2212, 2304, 2724, 5, 324, 6, 0, 0, 2545, 0, 0, 128, 92, 87, 5, 71, 4090, 3870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 48, 3436, 0, 8, 1, 3969, 90, 3133, 895, 0, 28, 154, 3091, 1937, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3250, 166, 1680, 4, 3981, 0, 1303, 0, 15, 61, 1, 0, 2384, 0, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3191, 566, 2070, 3523, 117, 680, 2646, 0, 1235, 749, 0, 35, 0, 1, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 686, 0, 1735, 2113, 66, 3597, 6, 4, 273, 4073, 4, 3422, 5, 297, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 107, 0, 669, 53, 2635, 181, 90, 3867, 16, 0, 3217, 49, 59, 9, 3088, 1557, 345, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17863, 15244, 18463, 16965, 13987, 15702, 16563, 16285] avg=16384.0000 max=18463 max_rank=2 min=13987 min_rank=4 imbalance_ratio=1.126892 max_load_reduction=112609 max_load_reduction_pct=0.859138 moved_to_echo=62537 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[828, 0, 3318, 653, 3891, 1, 0, 3772, 0, 2256, 1124, 0, 461, 521, 0, 3667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2172, 41, 515, 3858, 0, 303, 0, 0, 3, 23, 0, 934, 3850, 0, 2541, 1825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3307, 53, 3006, 3, 3, 0, 0, 5, 90, 4, 0, 3, 0, 0, 68, 5, 4064, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 43, 521, 0, 169, 1881, 0, 0, 1, 4002, 259, 3093, 6, 5, 1, 9, 4038, 1643, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3528, 2, 295, 11, 4065, 0, 3018, 3202, 0, 135, 779, 0, 1549, 5, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 775, 556, 1742, 2629, 848, 767, 2249, 2, 24, 744, 0, 1863, 147, 4, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 33, 0, 1534, 2261, 0, 1882, 1, 1, 567, 776, 27, 1801, 97, 134, 2801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 33, 3006, 89, 1, 3482, 115, 494, 2386, 15, 0, 3974, 995, 1236, 192, 3842, 1666, 367, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20492, 16065, 14491, 15843, 17560, 12580, 11924, 22117] avg=16384.0000 max=22117 max_rank=7 min=11924 min_rank=6 imbalance_ratio=1.349915 max_load_reduction=108955 max_load_reduction_pct=0.831261 moved_to_echo=64181 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[786, 3, 2947, 927, 3120, 2, 0, 1930, 0, 19, 22, 2, 2072, 0, 0, 2627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 17, 454, 135, 6, 4048, 337, 891, 16, 12, 2, 10, 12, 1151, 4091, 8, 2411, 882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 59, 3371, 2, 0, 0, 242, 39, 1292, 0, 0, 85, 0, 42, 51, 5, 4078, 3605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 702, 1313, 17, 35, 66, 495, 38, 394, 3786, 3011, 583, 432, 0, 2, 8, 4096, 793, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 104, 2996, 4, 95, 0, 4096, 26, 1636, 2788, 1, 1414, 1173, 0, 580, 1, 1118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2907, 2560, 3122, 3132, 51, 575, 2591, 68, 515, 366, 0, 1877, 17, 0, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 193, 0, 2756, 1900, 2, 4062, 115, 1, 736, 1654, 428, 4019, 1024, 0, 1927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 117, 621, 83, 322, 4065, 2, 283, 1507, 37, 1, 4019, 328, 0, 110, 1989, 1400, 348, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14457, 14594, 16939, 15944, 16164, 18539, 18827, 15608] avg=16384.0000 max=18827 max_rank=6 min=14457 min_rank=0 imbalance_ratio=1.149109 max_load_reduction=112245 max_load_reduction_pct=0.856361 moved_to_echo=69138 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[5248, 256, 23680, 5376, 22912, 128, 1152, 22016, 0, 6656, 4480, 768, 15488, 4480, 128, 19200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 4736, 0, 7936, 14976, 128, 21888, 256, 384, 13184, 21376, 2304, 19456, 10112, 1792, 13312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[768, 3072, 8576, 2432, 384, 23552, 5888, 2432, 12032, 256, 128, 25216, 4352, 1536, 384, 26112, 11136, 2432, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 16384, 12032, 12672, 27392, 3328, 10112, 20352, 128, 4992, 12032, 0, 5120, 768, 128, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 1152, 3328, 128, 3712, 15360, 640, 128, 512, 30976, 5248, 15488, 8320, 512, 256, 768, 30848, 13440, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[16896, 8960, 24960, 128, 768, 128, 3968, 128, 6400, 3200, 128, 640, 256, 256, 2304, 384, 32640, 30080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 2304, 25856, 2688, 4864, 128, 28928, 128, 14208, 15616, 1664, 2688, 3200, 128, 20608, 128, 8832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 384, 14720, 256, 2176, 24576, 3584, 6784, 1920, 1152, 128, 2560, 512, 16000, 25856, 128, 16128, 14976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[3860, 11, 42, 4043, 2569, 3574, 0, 72, 0, 4077, 6, 1985, 4040, 169, 30, 3223, 0, 0, 0, 132, 4048, 11, 0, 0, 1344, 3172, 807, 21, 2, 4022, 33, 0, 2904, 0, 15, 236, 3, 2834, 0, 1, 3255, 44, 1037, 141, 863, 536, 138, 539, 1, 3865, 4089, 0, 182, 0, 0, 2912, 0, 0, 0, 258, 4096, 3700, 1386, 4096, 3597, 262, 3778, 0, 53, 0, 2031, 0, 65, 14, 723, 1270, 0, 2, 8, 0, 1239, 435, 2, 156, 193, 4007, 3113, 4037, 0, 1738, 0, 47, 0, 0, 13, 0, 27, 0, 2699, 263, 0, 0, 0, 0, 118, 0, 3604, 233, 0, 4093, 19, 0, 0, 0, 4096, 4074, 0, 0, 0, 2683, 1365, 46, 142, 0, 0, 0, 52, 2351] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[981, 0, 4012, 278, 988, 83, 70, 3523, 1, 3722, 0, 3553, 1642, 3557, 0, 578, 0, 111, 1, 2, 376, 27, 637, 0, 138, 0, 0, 1, 37, 4084, 67, 4007, 4096, 12, 0, 502, 54, 2, 1092, 0, 4091, 45, 51, 230, 4090, 1728, 355, 1, 1, 3528, 843, 0, 6, 0, 2, 0, 1331, 87, 0, 4085, 3861, 3903, 3332, 3111, 0, 46, 53, 0, 0, 3871, 3198, 2, 2555, 2, 2371, 367, 13, 91, 0, 0, 0, 4061, 1, 3198, 1, 3685, 3922, 4074, 0, 175, 0, 0, 35, 0, 529, 1, 1, 20, 3349, 283, 0, 404, 0, 0, 64, 0, 109, 1, 0, 4091, 111, 297, 388, 992, 4078, 4096, 2763, 570, 0, 30, 185, 4, 0, 5, 181, 0, 63, 3821] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1997, 0, 3798, 123, 0, 4068, 0, 61, 0, 4080, 0, 3957, 69, 1660, 24, 16, 5, 1730, 0, 2, 12, 0, 676, 539, 2, 928, 36, 36, 0, 4095, 0, 3025, 3053, 11, 9, 0, 5, 0, 3757, 0, 2803, 0, 0, 0, 4072, 125, 0, 211, 0, 3848, 0, 357, 0, 0, 86, 1, 16, 0, 3, 4004, 3642, 4096, 6, 4014, 1536, 0, 1619, 16, 275, 1277, 4079, 991, 1469, 3, 3790, 243, 23, 0, 4, 0, 438, 3752, 872, 0, 3275, 4055, 1127, 4071, 0, 3495, 10, 1976, 0, 653, 2747, 16, 0, 0, 4080, 0, 683, 1, 0, 380, 2, 22, 58, 0, 4, 3514, 0, 0, 3025, 624, 3972, 2713, 35, 158, 20, 0, 342, 573, 8, 0, 18, 1, 9, 3960] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4081, 0, 4090, 3, 2675, 126, 5, 4063, 0, 4036, 0, 3062, 3495, 574, 0, 3, 0, 294, 0, 0, 0, 1, 23, 25, 41, 0, 2887, 31, 622, 4023, 1, 144, 3180, 2, 78, 0, 0, 3000, 1778, 0, 3974, 0, 1504, 3355, 4058, 2379, 3, 0, 0, 3952, 1, 594, 0, 0, 74, 0, 76, 0, 0, 549, 3672, 4094, 0, 1419, 1858, 0, 4022, 3, 1, 2978, 3715, 20, 29, 0, 1218, 778, 0, 3, 0, 0, 0, 1907, 1671, 132, 4093, 4089, 3796, 4096, 5, 2551, 0, 971, 0, 89, 20, 3, 0, 52, 0, 0, 5, 654, 0, 145, 4, 0, 1961, 21, 0, 11, 134, 0, 0, 7, 4019, 3670, 3963, 135, 21, 0, 13, 88, 2875, 66, 1, 0, 329, 2803] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[3872, 0, 4096, 0, 286, 4093, 0, 2810, 0, 3941, 0, 4096, 1656, 1, 0, 0, 584, 1, 0, 0, 0, 0, 0, 178, 401, 0, 3756, 224, 141, 4052, 0, 475, 2896, 0, 4, 3859, 0, 3381, 613, 0, 4084, 1, 2009, 2, 3843, 0, 93, 0, 0, 3838, 3, 150, 0, 0, 185, 0, 0, 0, 16, 0, 2721, 3570, 786, 809, 5, 0, 3290, 144, 0, 891, 3804, 222, 11, 4046, 3092, 42, 0, 2614, 37, 0, 0, 765, 7, 11, 191, 4081, 4089, 4091, 0, 4073, 0, 0, 0, 90, 3893, 5, 4, 0, 4094, 0, 82, 18, 0, 0, 8, 1, 8, 2443, 198, 107, 2, 3, 961, 3, 4044, 357, 4058, 785, 187, 8, 4094, 0, 0, 0, 2400, 0, 0, 187] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[3494, 0, 3429, 0, 551, 4069, 1, 3255, 0, 4038, 154, 3395, 60, 238, 318, 13, 114, 3649, 0, 0, 1097, 0, 289, 0, 1332, 4003, 9, 2, 17, 3233, 6, 25, 7, 28, 0, 0, 0, 1735, 2585, 458, 1210, 816, 25, 3, 3795, 141, 30, 0, 170, 4081, 20, 3183, 8, 22, 4046, 2191, 18, 0, 554, 4095, 2916, 3756, 385, 3587, 4065, 0, 3958, 0, 0, 5, 3813, 562, 0, 0, 4084, 0, 0, 8, 0, 1902, 91, 3836, 3370, 15, 2, 4058, 3272, 3559, 17, 289, 35, 106, 6, 0, 2600, 12, 0, 0, 1811, 64, 0, 103, 939, 0, 403, 608, 0, 17, 77, 1811, 0, 3, 922, 0, 3881, 1697, 0, 709, 0, 0, 0, 26, 805, 0, 0, 258, 0, 617] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4096, 0, 2890, 22, 4, 4082, 0, 4092, 0, 4090, 0, 4081, 3477, 0, 2, 0, 146, 0, 3, 0, 31, 2018, 0, 0, 3123, 643, 0, 0, 0, 2412, 2, 14, 3877, 0, 0, 50, 0, 49, 2, 0, 3929, 0, 150, 4069, 3967, 244, 114, 0, 46, 4089, 0, 0, 4, 23, 16, 640, 10, 0, 19, 0, 3945, 3898, 21, 3408, 796, 1, 853, 6, 0, 288, 4095, 4, 4, 1063, 3457, 21, 0, 134, 0, 0, 23, 3197, 2436, 3371, 1157, 4038, 4054, 4096, 0, 19, 1090, 0, 0, 4, 2, 0, 0, 344, 4091, 0, 86, 0, 1, 936, 0, 0, 4071, 77, 0, 4003, 0, 0, 334, 2, 4089, 2351, 3614, 3782, 0, 9, 2, 0, 14, 7, 940, 0, 275, 37] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4085, 0, 4042, 729, 40, 3947, 0, 1395, 2, 4086, 0, 4035, 7, 393, 0, 0, 1, 485, 1, 0, 47, 10, 0, 0, 2892, 3027, 3890, 0, 53, 3696, 3, 2182, 3080, 2, 114, 147, 19, 2358, 14, 0, 4093, 5, 0, 22, 924, 7, 0, 0, 46, 4094, 0, 2, 16, 10, 8, 0, 444, 0, 0, 2989, 4086, 4080, 29, 4024, 1, 1, 1744, 1, 50, 1362, 3955, 3259, 1501, 236, 2512, 13, 0, 0, 0, 18, 0, 3175, 723, 0, 13, 4096, 661, 4095, 0, 4090, 0, 9, 4, 0, 32, 0, 0, 1, 4079, 0, 0, 0, 38, 1, 0, 5, 120, 900, 8, 2784, 12, 0, 56, 0, 4066, 4091, 3484, 0, 6, 0, 1466, 103, 2784, 0, 2057, 10, 58, 3731] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[874, 0, 4012, 278, 988, 83, 70, 3523, 1, 0, 0, 0, 1642, 3557, 0, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 1, 2, 376, 27, 637, 0, 138, 0, 0, 1, 37, 4084, 67, 4007, 3174, 1562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 502, 54, 2, 1092, 0, 4091, 45, 51, 230, 4090, 1728, 355, 1, 496, 126, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 241, 843, 0, 6, 0, 2, 0, 1331, 87, 0, 4085, 3861, 3903, 3332, 3111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 53, 0, 0, 3871, 3198, 2, 2555, 2, 2371, 367, 13, 91, 0, 0, 1865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 1, 3198, 1, 3189, 3922, 0, 0, 175, 0, 0, 35, 0, 529, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 20, 3349, 283, 0, 404, 0, 0, 64, 0, 109, 1, 0, 4091, 111, 297, 4074, 3722, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 992, 4078, 4096, 2763, 570, 0, 30, 185, 4, 0, 5, 181, 0, 63, 3821, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15606, 14224, 16994, 20803, 14434, 15112, 16639, 17260] avg=16384.0000 max=20803 max_rank=3 min=14224 min_rank=1 imbalance_ratio=1.269714 max_load_reduction=110269 max_load_reduction_pct=0.841286 moved_to_echo=63445 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3436, 11, 42, 4043, 2569, 3574, 0, 72, 0, 0, 6, 0, 4040, 169, 30, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 4048, 11, 0, 0, 1344, 3172, 807, 21, 2, 4022, 33, 0, 3477, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2904, 0, 15, 236, 3, 2834, 0, 1, 3255, 44, 1037, 141, 863, 536, 138, 539, 540, 70, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 264, 4089, 0, 182, 0, 0, 2912, 0, 0, 0, 258, 4096, 3700, 1386, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3597, 262, 3778, 0, 53, 0, 2031, 0, 65, 14, 723, 1270, 0, 2, 8, 0, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1239, 435, 2, 156, 193, 3467, 3113, 0, 0, 1738, 0, 47, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 2699, 263, 0, 0, 0, 0, 118, 0, 3604, 233, 0, 4093, 19, 0, 4037, 4077, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4074, 0, 0, 0, 2683, 1365, 46, 142, 0, 0, 0, 52, 2351, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21215, 17942, 13249, 20984, 12845, 10403, 19294, 15140] avg=16384.0000 max=21215 max_rank=0 min=10403 min_rank=5 imbalance_ratio=1.294861 max_load_reduction=109857 max_load_reduction_pct=0.838142 moved_to_echo=57682 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3439, 0, 4096, 0, 286, 4093, 0, 2810, 0, 0, 0, 0, 1656, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 584, 1, 0, 0, 0, 0, 0, 178, 401, 0, 3756, 224, 141, 4052, 0, 475, 3457, 1803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2896, 0, 4, 3859, 0, 3381, 613, 0, 4084, 1, 2009, 2, 3843, 0, 93, 0, 554, 144, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 3, 150, 0, 0, 185, 0, 0, 0, 16, 0, 2721, 3570, 786, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3290, 144, 0, 891, 3804, 222, 11, 4046, 3092, 42, 0, 2614, 37, 0, 2149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 765, 7, 11, 191, 3527, 4089, 0, 0, 4073, 0, 0, 0, 90, 3893, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4094, 0, 82, 18, 0, 0, 8, 1, 8, 2443, 198, 107, 2, 3, 4091, 3941, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 961, 3, 4044, 357, 4058, 785, 187, 8, 4094, 0, 0, 0, 2400, 0, 0, 187, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 15072, 21580, 8495, 20347, 16651, 15126, 17420] avg=16384.0000 max=21580 max_rank=2 min=8495 min_rank=3 imbalance_ratio=1.317139 max_load_reduction=109492 max_load_reduction_pct=0.835358 moved_to_echo=69544 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3111, 0, 3429, 0, 551, 4069, 1, 3255, 0, 0, 154, 0, 60, 238, 318, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 3649, 0, 0, 1097, 0, 289, 0, 1332, 4003, 9, 2, 17, 3233, 6, 25, 3672, 1493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 28, 0, 0, 0, 1735, 2585, 458, 1210, 816, 25, 3, 3795, 141, 30, 0, 547, 120, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 278, 20, 3183, 8, 22, 4046, 2191, 18, 0, 554, 4095, 2916, 3756, 385, 3587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 3958, 0, 0, 5, 3813, 562, 0, 0, 4084, 0, 0, 8, 0, 1902, 1782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 3836, 3370, 15, 2, 3511, 3272, 0, 17, 289, 35, 106, 6, 0, 2600, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1811, 64, 0, 103, 939, 0, 403, 608, 0, 17, 77, 1811, 0, 3, 3559, 4038, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 922, 0, 3881, 1697, 0, 709, 0, 0, 0, 26, 805, 0, 0, 258, 0, 617, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15199, 18941, 11584, 25229, 20179, 17162, 13564, 9214] avg=16384.0000 max=25229 max_rank=3 min=9214 min_rank=7 imbalance_ratio=1.539856 max_load_reduction=105843 max_load_reduction_pct=0.807518 moved_to_echo=60119 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3637, 0, 4042, 729, 40, 3947, 0, 1395, 2, 0, 0, 0, 7, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 485, 1, 0, 47, 10, 0, 0, 2892, 3027, 3890, 0, 53, 3696, 3, 2182, 3683, 1774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3080, 2, 114, 147, 19, 2358, 14, 0, 4093, 5, 0, 22, 924, 7, 0, 0, 552, 142, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 280, 0, 2, 16, 10, 8, 0, 444, 0, 0, 2989, 4086, 4080, 29, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1744, 1, 50, 1362, 3955, 3259, 1501, 236, 2512, 13, 0, 0, 0, 18, 2119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3175, 723, 0, 13, 3544, 661, 0, 0, 4090, 0, 9, 4, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4079, 0, 0, 0, 38, 1, 0, 5, 120, 900, 8, 2784, 12, 0, 4095, 4086, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 4066, 4091, 3484, 0, 6, 0, 1466, 103, 2784, 0, 2057, 10, 58, 3731, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14192, 21744, 11577, 16014, 16772, 12251, 16260, 22262] avg=16384.0000 max=22262 max_rank=7 min=11577 min_rank=2 imbalance_ratio=1.358765 max_load_reduction=108810 max_load_reduction_pct=0.830154 moved_to_echo=67545 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3633, 0, 4090, 3, 2675, 126, 5, 4063, 0, 0, 0, 0, 3495, 574, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 0, 0, 1, 23, 25, 41, 0, 2887, 31, 622, 4023, 1, 144, 3555, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3180, 2, 78, 0, 0, 3000, 1778, 0, 3974, 0, 1504, 3355, 4058, 2379, 3, 0, 551, 110, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 1, 594, 0, 0, 74, 0, 76, 0, 0, 549, 3672, 4094, 0, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1858, 0, 4022, 3, 1, 2978, 3715, 20, 29, 0, 1218, 778, 0, 3, 0, 0, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1907, 1671, 132, 4093, 3538, 3796, 0, 5, 2551, 0, 971, 0, 89, 20, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 5, 654, 0, 145, 4, 0, 1961, 21, 0, 11, 134, 0, 4096, 4036, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4019, 3670, 3963, 135, 21, 0, 13, 88, 2875, 66, 1, 0, 329, 2803, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18667, 12993, 24070, 10750, 16231, 18776, 11245, 18340] avg=16384.0000 max=24070 max_rank=2 min=10750 min_rank=3 imbalance_ratio=1.469116 max_load_reduction=107002 max_load_reduction_pct=0.816360 moved_to_echo=64592 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1778, 0, 3798, 123, 0, 4068, 0, 61, 0, 0, 0, 0, 69, 1660, 24, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1730, 0, 2, 12, 0, 676, 539, 2, 928, 36, 36, 0, 4095, 0, 3025, 3462, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3053, 11, 9, 0, 5, 0, 3757, 0, 2803, 0, 0, 0, 4072, 125, 0, 211, 546, 139, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 0, 357, 0, 0, 86, 1, 16, 0, 3, 4004, 3642, 4096, 6, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1536, 0, 1619, 16, 275, 1277, 4079, 991, 1469, 3, 3790, 243, 23, 0, 4, 0, 2076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 3752, 872, 0, 3275, 3509, 1127, 0, 0, 3495, 10, 1976, 0, 653, 2747, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 683, 1, 0, 380, 2, 22, 58, 0, 4, 3514, 0, 0, 4071, 4080, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3025, 624, 3972, 2713, 35, 158, 20, 0, 342, 573, 8, 0, 18, 1, 9, 3960, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11597, 16290, 14779, 16488, 17401, 21870, 17018, 15629] avg=16384.0000 max=21870 max_rank=5 min=11597 min_rank=0 imbalance_ratio=1.334839 max_load_reduction=109202 max_load_reduction_pct=0.833145 moved_to_echo=71918 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3646, 0, 2890, 22, 4, 4082, 0, 4092, 0, 0, 0, 0, 3477, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 3, 0, 31, 2018, 0, 0, 3123, 643, 0, 0, 0, 2412, 2, 14, 3679, 1795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 0, 50, 0, 49, 2, 0, 3929, 0, 150, 4069, 3967, 244, 114, 0, 544, 145, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 279, 0, 0, 4, 23, 16, 640, 10, 0, 19, 0, 3945, 3898, 21, 3408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 796, 1, 853, 6, 0, 288, 4095, 4, 4, 1063, 3457, 21, 0, 134, 0, 0, 2141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3197, 2436, 3371, 1157, 3494, 4054, 0, 0, 19, 1090, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 4091, 0, 86, 0, 1, 936, 0, 0, 4071, 77, 0, 4003, 0, 0, 4096, 4090, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 2, 4089, 2351, 3614, 3782, 0, 9, 2, 0, 14, 7, 940, 0, 275, 37, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18215, 13866, 17239, 12309, 12863, 18847, 21926, 15807] avg=16384.0000 max=21926 max_rank=6 min=12309 min_rank=3 imbalance_ratio=1.338257 max_load_reduction=109146 max_load_reduction_pct=0.832718 moved_to_echo=69443 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[23680, 128, 26496, 5248, 7168, 24064, 128, 19328, 128, 0, 256, 0, 14464, 6656, 384, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[11904, 384, 19328, 256, 384, 10752, 28800, 5120, 5760, 5376, 21248, 2816, 128, 2944, 128, 1920, 14848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[5760, 1664, 32256, 23168, 17920, 6144, 256, 2816, 7552, 896, 6656, 128, 5632, 384, 896, 17536, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[23168, 128, 256, 4864, 128, 13440, 9856, 512, 27520, 1024, 4864, 7936, 25728, 5248, 768, 768, 4352, 1024, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1792, 21248, 9088, 6912, 8960, 27904, 24064, 0, 128, 16512, 1152, 3200, 128, 896, 9856, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 512, 24320, 640, 896, 1280, 1024, 1536, 640, 640, 9984, 3712, 384, 20480, 384, 384, 32128, 32128, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[384, 2176, 4992, 4352, 256, 128, 4480, 5760, 1920, 128, 640, 16000, 29056, 31104, 6016, 24576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[896, 6272, 128, 256, 5632, 2176, 1664, 768, 9344, 11776, 11392, 384, 896, 29696, 128, 9984, 28160, 12416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 140, 68, 728, 0, 4, 0, 1, 486, 4068, 0, 4026, 4096, 3463, 0, 4, 3377, 3670, 0, 1, 4094, 1, 8, 0, 0, 0, 16, 4, 0, 0, 3723, 29, 0, 141, 1184, 1184, 418, 3, 3815, 0, 0, 45, 489, 27, 3, 0, 0, 7, 200, 387, 2110, 3865, 2413, 3787, 24, 4033, 0, 0, 1, 4096, 0, 27, 0, 0, 3589, 22, 0, 0, 4048, 10, 971, 0, 1, 978, 1739, 4070, 2, 0, 172, 569, 3866, 0, 2, 3786, 4096, 0, 0, 3986, 1, 345, 3883, 0, 0, 4091, 54, 30, 0, 12, 4041, 0, 1231, 2411, 0, 21, 0, 0, 2592, 0, 0, 6, 3966, 0, 83, 3972, 329, 19, 13, 3571, 0, 0, 1665, 0, 3629, 0, 2149, 0, 785, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1, 2, 2778, 2, 0, 126, 104, 8, 4096, 2308, 3793, 4087, 3758, 785, 1581, 0, 4092, 4000, 23, 2451, 4019, 3, 0, 0, 4, 0, 0, 40, 2325, 0, 3422, 2949, 0, 2, 296, 53, 5, 0, 0, 0, 0, 3821, 2847, 1471, 63, 1678, 69, 0, 0, 0, 0, 51, 4072, 0, 4019, 1790, 9, 260, 1924, 3908, 1, 15, 12, 1062, 57, 2, 0, 1, 0, 0, 0, 535, 248, 113, 0, 225, 0, 583, 3, 4090, 106, 5, 0, 459, 3444, 0, 3769, 4096, 4051, 0, 3340, 0, 0, 4089, 0, 437, 3, 1156, 2756, 9, 3923, 86, 10, 4082, 0, 266, 18, 2, 0, 0, 0, 1884, 0, 858, 0, 1, 262, 0, 0, 0, 1, 0, 3596, 143, 4096, 0, 4082, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[8, 225, 4020, 4051, 0, 560, 29, 1879, 119, 3909, 163, 4091, 3183, 2755, 9, 0, 3294, 0, 47, 1424, 3749, 29, 5, 3, 0, 4096, 0, 0, 31, 24, 0, 3949, 188, 4, 0, 4094, 13, 1, 1, 0, 782, 3984, 813, 12, 364, 32, 182, 153, 0, 1, 0, 0, 3566, 1, 611, 1285, 3439, 1, 689, 3750, 55, 15, 0, 0, 7, 3203, 13, 36, 1, 0, 611, 4094, 4, 0, 43, 3622, 61, 91, 0, 156, 0, 0, 0, 3818, 6, 0, 116, 4048, 5, 451, 2144, 0, 1610, 142, 320, 1451, 11, 3908, 3327, 18, 395, 2217, 4090, 604, 7, 723, 3570, 0, 0, 0, 1533, 877, 1, 3116, 0, 0, 3553, 2717, 0, 161, 3, 3, 3998, 21, 4050, 0, 425, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 2, 2438, 0, 23, 482, 110, 8, 2126, 0, 17, 2853, 3647, 2665, 3, 0, 3847, 4010, 0, 2982, 1605, 1022, 10, 0, 0, 0, 0, 0, 1088, 133, 2751, 432, 1547, 1, 0, 0, 1, 11, 3923, 2341, 0, 0, 13, 3875, 678, 2617, 36, 0, 0, 3981, 115, 286, 4083, 4034, 3890, 0, 34, 4, 1179, 4055, 3958, 835, 1, 444, 13, 236, 0, 169, 4079, 0, 529, 0, 105, 569, 0, 85, 2, 0, 0, 0, 0, 0, 0, 3359, 3939, 0, 276, 4095, 12, 0, 402, 970, 0, 2732, 1678, 0, 0, 31, 3323, 0, 3909, 3986, 0, 46, 2, 2929, 5, 0, 0, 708, 372, 3321, 0, 3737, 446, 0, 733, 4, 10, 2, 901, 1, 2020, 28, 3236, 4, 3854, 18] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 29, 3727, 0, 77, 0, 40, 0, 4062, 4051, 0, 1651, 3832, 4095, 0, 0, 4096, 0, 0, 6, 4021, 0, 0, 24, 420, 3523, 1, 2, 0, 0, 2, 4095, 2, 0, 0, 2436, 0, 254, 609, 0, 0, 2, 3850, 296, 3960, 0, 1, 6, 804, 1376, 1446, 0, 4030, 6, 2212, 3503, 0, 0, 1, 1845, 0, 3, 21, 399, 4096, 0, 0, 0, 1023, 20, 4, 416, 8, 699, 4094, 478, 5, 190, 1012, 3123, 8, 275, 0, 4068, 270, 0, 18, 4088, 47, 1, 4052, 0, 0, 4096, 176, 0, 11, 1682, 4086, 11, 2063, 3527, 0, 0, 20, 0, 9, 4, 0, 58, 4092, 2290, 110, 3618, 0, 1, 4083, 536, 9, 0, 1, 6, 4070, 0, 3289, 3, 410, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[29, 0, 2763, 697, 3392, 0, 144, 0, 4093, 608, 2563, 3868, 3716, 1308, 0, 20, 3555, 0, 18, 119, 256, 0, 49, 0, 0, 4076, 0, 0, 3200, 0, 19, 4052, 0, 862, 40, 17, 653, 29, 2479, 0, 0, 2, 1821, 0, 3809, 65, 0, 3, 6, 1, 74, 0, 2596, 2, 3779, 3566, 788, 0, 2597, 3955, 13, 2, 1, 155, 3, 3814, 55, 0, 48, 0, 1683, 163, 213, 0, 1024, 4093, 0, 0, 961, 4, 0, 0, 10, 4087, 119, 0, 1515, 4078, 3557, 1369, 215, 19, 0, 4085, 1323, 149, 0, 2190, 4025, 0, 4096, 0, 0, 4032, 637, 127, 1, 0, 0, 0, 182, 2797, 0, 2269, 12, 0, 4083, 0, 0, 0, 0, 0, 4096, 0, 4060, 18, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 836, 799, 650, 0, 0, 3135, 0, 3753, 142, 4, 4045, 4093, 3255, 18, 0, 20, 4071, 433, 3947, 4077, 419, 0, 2, 3, 0, 0, 2, 3692, 267, 22, 61, 3, 29, 355, 4063, 26, 2, 912, 85, 4, 2705, 1311, 4086, 3951, 864, 318, 0, 1, 3787, 0, 0, 1848, 2688, 4068, 69, 0, 3, 719, 3928, 0, 0, 0, 0, 0, 4096, 0, 1738, 3619, 88, 5, 1, 209, 4, 6, 949, 0, 195, 0, 2, 0, 0, 0, 1, 3943, 2, 2646, 4096, 4094, 0, 97, 0, 0, 2392, 891, 4092, 0, 46, 3167, 283, 35, 3507, 0, 46, 6, 860, 0, 1, 0, 0, 153, 6, 0, 580, 3645, 0, 3810, 0, 0, 0, 3939, 0, 4082, 0, 4, 0, 146, 19] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 1, 43, 19, 4084, 3156, 3908, 4074, 1996, 4093, 0, 68, 4063, 1, 0, 3316, 537, 0, 50, 94, 278, 4096, 0, 0, 1273, 0, 3400, 4096, 0, 562, 0, 98, 1303, 1, 0, 0, 0, 2341, 7, 0, 316, 6, 0, 3, 0, 687, 0, 3737, 3783, 1, 73, 0, 34, 0, 2333, 3567, 1, 0, 0, 69, 3810, 564, 0, 0, 0, 0, 392, 31, 15, 0, 19, 217, 0, 0, 7, 0, 6, 16, 0, 4096, 3614, 0, 44, 3885, 1903, 1831, 109, 821, 1505, 3210, 15, 398, 0, 0, 4086, 0, 2645, 4041, 3374, 3340, 0, 3324, 2, 1, 42, 865, 3607, 935, 0, 3992, 2811, 0, 272, 660, 0, 260, 286, 0, 2416, 2, 1712, 16, 201, 12] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[8, 225, 4020, 4051, 0, 560, 29, 1879, 119, 3909, 163, 0, 1931, 2755, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1861, 0, 47, 1424, 3749, 29, 5, 3, 0, 4096, 0, 0, 31, 24, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 4, 0, 4094, 13, 1, 1, 0, 782, 3984, 813, 12, 364, 32, 182, 153, 1531, 1433, 1475, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3566, 1, 611, 1285, 3439, 1, 689, 2219, 55, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3203, 13, 36, 1, 0, 611, 4094, 4, 0, 43, 3622, 61, 91, 0, 156, 3722, 2616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3818, 6, 0, 116, 326, 5, 451, 2144, 0, 1610, 142, 320, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3908, 2447, 18, 395, 2217, 4090, 604, 7, 723, 3570, 0, 0, 0, 1533, 877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3116, 0, 0, 3553, 2717, 0, 161, 3, 3, 3998, 21, 4050, 0, 425, 3, 880, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19658, 15218, 16134, 11882, 18280, 10389, 20400, 19111] avg=16384.0000 max=20400 max_rank=6 min=10389 min_rank=5 imbalance_ratio=1.245117 max_load_reduction=110672 max_load_reduction_pct=0.844360 moved_to_echo=68180 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 140, 68, 728, 0, 4, 0, 1, 486, 4068, 0, 0, 2476, 3463, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1905, 3670, 0, 1, 4094, 1, 8, 0, 0, 0, 16, 4, 0, 0, 3723, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 1184, 1184, 418, 3, 3815, 0, 0, 45, 489, 27, 3, 0, 0, 7, 1676, 1472, 1452, 1384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 387, 2110, 3865, 2413, 3787, 24, 4033, 0, 0, 1, 2420, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3589, 22, 0, 0, 4048, 10, 971, 0, 1, 978, 1739, 4070, 2, 0, 172, 569, 3670, 2574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3866, 0, 2, 3786, 4096, 0, 0, 316, 1, 345, 3883, 0, 0, 4091, 54, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2967, 0, 1231, 2411, 0, 21, 0, 0, 2592, 0, 0, 6, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 3972, 329, 19, 13, 3571, 0, 0, 1665, 0, 3629, 0, 2149, 0, 785, 0, 1074, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11438, 13451, 13300, 19267, 22415, 20470, 13206, 17525] avg=16384.0000 max=22415 max_rank=4 min=11438 min_rank=0 imbalance_ratio=1.368103 max_load_reduction=108657 max_load_reduction_pct=0.828987 moved_to_echo=73616 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 2438, 0, 23, 482, 110, 8, 2126, 0, 17, 0, 2212, 2665, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2173, 4010, 0, 2982, 1605, 1022, 10, 0, 0, 0, 0, 0, 1088, 133, 2751, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1547, 1, 0, 0, 1, 11, 3923, 2341, 0, 0, 13, 3875, 678, 2617, 36, 0, 1655, 1674, 1028, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 115, 286, 4083, 4034, 3890, 0, 34, 4, 1179, 2400, 3958, 835, 1, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 236, 0, 169, 4079, 0, 529, 0, 105, 569, 0, 85, 2, 0, 0, 0, 3765, 1825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3359, 3939, 0, 276, 330, 12, 0, 402, 970, 0, 2732, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 2444, 0, 3909, 3986, 0, 46, 2, 2929, 5, 0, 0, 708, 372, 3321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3737, 446, 0, 733, 4, 10, 2, 901, 1, 2020, 28, 3236, 4, 3854, 18, 879, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10086, 16206, 20629, 25244, 11377, 13698, 17753, 16079] avg=16384.0000 max=25244 max_rank=3 min=10086 min_rank=0 imbalance_ratio=1.540771 max_load_reduction=105828 max_load_reduction_pct=0.807404 moved_to_echo=58907 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 2778, 2, 0, 126, 104, 8, 4096, 2308, 3793, 0, 2279, 785, 1581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2312, 4000, 23, 2451, 4019, 3, 0, 0, 4, 0, 0, 40, 2325, 0, 3422, 2949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 296, 53, 5, 0, 0, 0, 0, 3821, 2847, 1471, 63, 1678, 69, 0, 1595, 1780, 1474, 1266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 4072, 0, 4019, 1790, 9, 260, 1924, 2313, 1, 15, 12, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 2, 0, 1, 0, 0, 0, 535, 248, 113, 0, 225, 0, 583, 3, 4090, 3766, 2613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 5, 0, 459, 3444, 0, 3769, 330, 4051, 0, 3340, 0, 0, 4089, 0, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1156, 2027, 9, 3923, 86, 10, 4082, 0, 266, 18, 2, 0, 0, 0, 1884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 858, 0, 1, 262, 0, 0, 0, 1, 0, 3596, 143, 4096, 0, 4082, 0, 729, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17863, 21548, 16420, 15528, 12236, 20030, 13466, 13981] avg=16384.0000 max=21548 max_rank=1 min=12236 min_rank=4 imbalance_ratio=1.315186 max_load_reduction=109524 max_load_reduction_pct=0.835602 moved_to_echo=59713 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 2763, 697, 3392, 0, 144, 0, 4093, 608, 2563, 0, 2254, 1308, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2008, 0, 18, 119, 256, 0, 49, 0, 0, 4076, 0, 0, 3200, 0, 19, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 862, 40, 17, 653, 29, 2479, 0, 0, 2, 1821, 0, 3809, 65, 0, 3, 1614, 1547, 1394, 1252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 74, 0, 2596, 2, 3779, 3566, 788, 0, 2597, 2341, 13, 2, 1, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3814, 55, 0, 48, 0, 1683, 163, 213, 0, 1024, 4093, 0, 0, 961, 4, 3750, 2474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4087, 119, 0, 1515, 328, 3557, 1369, 215, 19, 0, 4085, 1323, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2190, 2960, 0, 4096, 0, 0, 4032, 637, 127, 1, 0, 0, 0, 182, 2797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2269, 12, 0, 4083, 0, 0, 0, 0, 0, 4096, 0, 4060, 18, 0, 0, 1065, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17871, 13797, 15587, 15921, 18285, 16776, 17022, 15813] avg=16384.0000 max=18285 max_rank=4 min=13797 min_rank=1 imbalance_ratio=1.116028 max_load_reduction=112787 max_load_reduction_pct=0.860497 moved_to_echo=67896 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 29, 3727, 0, 77, 0, 40, 0, 4062, 4051, 0, 0, 2324, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2314, 0, 0, 6, 4021, 0, 0, 24, 420, 3523, 1, 2, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2436, 0, 254, 609, 0, 0, 2, 3850, 296, 3960, 0, 1, 6, 753, 1782, 596, 1291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 1376, 1446, 0, 4030, 6, 2212, 3503, 0, 0, 1, 1092, 0, 3, 21, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1023, 20, 4, 416, 8, 699, 4094, 478, 5, 190, 1012, 3123, 3759, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 275, 0, 4068, 270, 0, 18, 329, 47, 1, 4052, 0, 0, 4096, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1682, 3005, 11, 2063, 3527, 0, 0, 20, 0, 9, 4, 0, 58, 4092, 2290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 3618, 0, 1, 4083, 536, 9, 0, 1, 6, 4070, 0, 3289, 3, 410, 0, 1081, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18405, 14408, 15838, 14893, 19982, 13340, 16772, 17434] avg=16384.0000 max=19982 max_rank=4 min=13340 min_rank=5 imbalance_ratio=1.219604 max_load_reduction=111090 max_load_reduction_pct=0.847549 moved_to_echo=67528 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 1, 43, 19, 4084, 3156, 3908, 0, 1211, 4093, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2295, 1, 0, 3316, 537, 0, 50, 94, 278, 4096, 0, 0, 1273, 0, 3400, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 0, 98, 1303, 1, 0, 0, 0, 2341, 7, 0, 316, 6, 0, 3, 1456, 1768, 1468, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, 0, 3737, 3783, 1, 73, 0, 34, 0, 2333, 2111, 1, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 564, 0, 0, 0, 0, 392, 31, 15, 0, 19, 217, 0, 0, 7, 0, 3572, 2606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 16, 0, 4096, 3614, 0, 44, 313, 1903, 1831, 109, 821, 1505, 3210, 15, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3005, 0, 2645, 4041, 3374, 3340, 0, 3324, 2, 1, 42, 865, 3607, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 2811, 0, 272, 660, 0, 260, 286, 0, 2416, 2, 1712, 16, 201, 12, 1081, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20677, 19436, 10001, 12829, 11233, 17881, 25181, 13834] avg=16384.0000 max=25181 max_rank=6 min=10001 min_rank=2 imbalance_ratio=1.536926 max_load_reduction=105891 max_load_reduction_pct=0.807884 moved_to_echo=68129 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 836, 799, 650, 0, 0, 3135, 0, 3753, 142, 4, 0, 2482, 3255, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4071, 433, 3947, 4077, 419, 0, 2, 3, 0, 0, 2, 3692, 267, 22, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 29, 355, 4063, 26, 2, 912, 85, 4, 2705, 1311, 4086, 3951, 864, 318, 0, 1603, 8, 1459, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3787, 0, 0, 1848, 2688, 4068, 69, 0, 3, 719, 2325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1738, 3619, 88, 5, 1, 209, 4, 6, 949, 0, 195, 0, 2, 3766, 2586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3943, 2, 2646, 330, 4094, 0, 97, 0, 0, 2392, 891, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 2329, 283, 35, 3507, 0, 46, 6, 860, 0, 1, 0, 0, 153, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 580, 3645, 0, 3810, 0, 0, 0, 3939, 0, 4082, 0, 4, 0, 146, 19, 838, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15074, 17008, 23163, 15508, 17264, 18488, 7272, 17295] avg=16384.0000 max=23163 max_rank=2 min=7272 min_rank=6 imbalance_ratio=1.413757 max_load_reduction=107909 max_load_reduction_pct=0.823280 moved_to_echo=60319 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 1280, 16640, 6144, 7680, 1280, 3712, 1920, 22912, 18304, 10496, 0, 17280, 22528, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[14976, 15872, 640, 14336, 22400, 1536, 128, 128, 768, 15872, 128, 128, 11648, 512, 13440, 19712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1024, 10240, 3840, 8064, 26496, 10624, 18688, 14336, 4352, 384, 9472, 17280, 4096, 1024, 128, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1792, 1664, 1920, 12032, 2432, 384, 11776, 2432, 896, 12928, 11264, 9856, 13184, 5376, 640, 256, 11904, 11520, 10368, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[256, 22144, 7296, 128, 16896, 7552, 128, 512, 6912, 128, 28032, 256, 22656, 128, 9984, 128, 7680, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 9088, 21248, 384, 18304, 19840, 7552, 12288, 768, 8320, 6272, 128, 128, 1664, 13952, 12160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[4096, 384, 128, 23680, 19456, 128, 8448, 2688, 13696, 4096, 14336, 1920, 3200, 24960, 4480, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[11648, 12032, 128, 2048, 12928, 128, 4224, 5248, 896, 2432, 7040, 13824, 128, 1152, 2176, 8064, 29824, 18432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4047, 1191, 3958, 0, 179, 0, 0, 404, 57, 0, 3578, 4085, 0, 0, 2385, 392, 0, 0, 0, 0, 3799, 0, 0, 4093, 4060, 7, 1288, 0, 0, 4091, 599, 159, 5, 3254, 1, 20, 476, 0, 3078, 4072, 4096, 2617, 22, 42, 0, 1331, 0, 295, 0, 4006, 1, 573, 4060, 118, 58, 3168, 1003, 4, 3093, 0, 1, 25, 2210, 4090, 32, 16, 171, 2565, 19, 6, 7, 34, 2, 0, 0, 374, 49, 8, 138, 4, 0, 758, 228, 1847, 3385, 1, 0, 3426, 402, 0, 8, 2701, 302, 0, 1926, 111, 1933, 3718, 3190, 0, 58, 82, 4092, 0, 0, 4, 1, 1, 0, 36, 108, 3764, 77, 3318, 0, 0, 0, 0, 40, 0, 19, 0, 0, 4060, 3978, 343, 42, 3597] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[3971, 3811, 4096, 135, 0, 9, 0, 1, 728, 4, 0, 4092, 0, 4, 0, 1510, 186, 0, 0, 4094, 6, 0, 0, 4071, 4096, 302, 530, 257, 855, 12, 3848, 0, 0, 290, 1796, 426, 0, 4, 4048, 218, 4061, 3961, 2531, 0, 0, 2, 2, 3731, 0, 3922, 12, 3995, 3990, 0, 202, 4080, 0, 2, 0, 1, 0, 124, 2818, 3459, 5, 15, 5, 107, 0, 0, 154, 5, 4096, 0, 411, 43, 0, 2, 1, 0, 0, 0, 5, 158, 2230, 1, 9, 3024, 0, 28, 1, 18, 4094, 1, 32, 0, 0, 597, 171, 3739, 0, 0, 4096, 27, 3900, 171, 0, 3291, 0, 0, 0, 3940, 106, 4096, 568, 0, 11, 4, 4005, 7, 0, 20, 122, 3929, 3307, 11, 0, 216] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[903, 2, 3886, 0, 2123, 350, 47, 0, 3325, 0, 310, 18, 1, 2692, 6, 3534, 0, 0, 0, 111, 3365, 5, 4091, 3879, 1389, 176, 19, 0, 894, 657, 0, 0, 2825, 3907, 54, 0, 4096, 4089, 3098, 0, 405, 0, 4016, 0, 125, 32, 3372, 34, 0, 760, 3276, 4081, 675, 0, 0, 2339, 1198, 2, 0, 0, 1, 90, 3796, 640, 12, 1743, 0, 0, 1608, 10, 4094, 35, 1, 0, 0, 0, 0, 443, 166, 0, 3, 85, 3628, 0, 4096, 1724, 6, 2063, 597, 26, 1282, 67, 4089, 0, 719, 2, 0, 58, 0, 0, 2574, 2, 3659, 12, 0, 0, 581, 4048, 14, 20, 0, 4048, 0, 4093, 0, 1, 14, 1, 2688, 1111, 11, 3953, 3, 2565, 7, 416, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4095, 4043, 4086, 0, 382, 313, 4, 578, 1, 0, 0, 1946, 192, 0, 0, 241, 4096, 0, 22, 666, 4070, 0, 0, 4095, 4082, 3422, 3805, 0, 8, 3097, 8, 0, 0, 1, 0, 1, 0, 0, 4090, 4095, 45, 636, 2431, 4015, 0, 0, 0, 3578, 403, 3605, 0, 3901, 1885, 359, 2, 3823, 0, 0, 62, 0, 3475, 80, 0, 3914, 1, 222, 13, 178, 0, 0, 886, 2, 0, 18, 0, 7, 340, 0, 0, 16, 0, 162, 0, 3711, 217, 1, 20, 1142, 14, 0, 0, 0, 3699, 0, 2611, 0, 0, 67, 752, 3644, 0, 0, 3907, 0, 0, 0, 0, 3701, 0, 1632, 4, 4076, 0, 4085, 313, 1, 0, 0, 0, 139, 199, 0, 37, 3411, 4093, 0, 3, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4040, 2616, 4093, 0, 532, 1, 11, 3, 3957, 171, 45, 4096, 1, 0, 3838, 1492, 0, 0, 0, 3200, 1, 0, 0, 4096, 102, 94, 4094, 0, 0, 4091, 1082, 12, 0, 4072, 0, 23, 1, 0, 3106, 4094, 0, 0, 4075, 24, 3369, 3901, 1064, 4070, 0, 4007, 1651, 4064, 86, 0, 10, 333, 0, 0, 0, 0, 3826, 35, 3587, 502, 0, 1604, 1, 24, 0, 53, 3510, 0, 4091, 0, 39, 1, 4, 282, 8, 0, 3065, 0, 0, 0, 4090, 3, 0, 161, 0, 0, 0, 4003, 3916, 0, 12, 0, 0, 0, 0, 159, 10, 0, 4086, 7, 67, 18, 0, 311, 0, 2723, 21, 4015, 0, 150, 0, 28, 0, 127, 4070, 0, 1, 0, 445, 2287, 37, 0, 6, 69] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[3849, 2238, 4021, 0, 0, 652, 0, 0, 0, 0, 0, 4086, 0, 0, 1, 4079, 0, 3, 0, 3533, 2973, 67, 0, 3886, 3601, 4084, 2753, 87, 1611, 0, 0, 0, 0, 120, 785, 0, 3, 179, 4091, 518, 2895, 307, 4012, 3450, 0, 30, 0, 862, 783, 4096, 2, 3869, 3264, 713, 0, 2, 43, 0, 4, 0, 1174, 0, 0, 0, 36, 1, 0, 3516, 2, 4008, 2961, 37, 30, 1218, 0, 0, 1, 403, 975, 23, 0, 0, 11, 4077, 4091, 4032, 0, 3375, 20, 0, 12, 13, 3863, 38, 44, 0, 0, 0, 4085, 0, 5, 0, 3340, 0, 0, 1, 0, 3943, 0, 0, 0, 385, 0, 4071, 0, 0, 25, 4, 643, 0, 919, 0, 476, 4064, 3386, 0, 195, 17] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4039, 3483, 3255, 12, 0, 254, 0, 0, 109, 0, 0, 22, 0, 25, 0, 4013, 3926, 0, 0, 1, 4064, 0, 4006, 9, 0, 233, 520, 263, 0, 3280, 2636, 0, 1938, 3063, 30, 0, 2, 0, 3268, 690, 2911, 3715, 2119, 15, 0, 20, 4096, 4095, 0, 3865, 3, 4096, 4090, 958, 0, 2803, 291, 0, 0, 0, 1540, 0, 3953, 3, 0, 36, 59, 0, 1, 58, 4088, 0, 678, 2, 3792, 12, 0, 14, 827, 884, 0, 0, 0, 0, 1530, 0, 0, 3, 0, 0, 1, 47, 3264, 0, 0, 2454, 851, 220, 3820, 66, 0, 0, 4089, 8, 1299, 0, 0, 1192, 0, 2881, 24, 4096, 0, 454, 302, 0, 0, 11, 3993, 0, 148, 5, 2, 4096, 4027, 0, 0, 24] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[3813, 4096, 2965, 342, 0, 0, 147, 0, 4033, 1, 79, 3976, 0, 0, 0, 4096, 2, 68, 0, 4084, 43, 3, 0, 447, 2416, 0, 3581, 35, 0, 346, 4054, 420, 0, 3031, 0, 555, 568, 45, 4094, 584, 98, 3763, 3980, 0, 6, 0, 0, 21, 0, 4076, 1, 0, 4093, 0, 0, 3328, 0, 0, 3022, 0, 665, 0, 0, 4061, 0, 0, 194, 1, 748, 0, 40, 0, 1613, 0, 0, 97, 247, 11, 0, 13, 0, 3790, 667, 0, 143, 0, 994, 4096, 1076, 0, 0, 3909, 4058, 199, 4068, 17, 1, 404, 2634, 4, 0, 7, 3250, 3, 712, 0, 0, 2504, 0, 0, 1, 3821, 67, 4092, 0, 0, 8, 0, 2164, 78, 0, 3, 0, 4094, 85, 1766, 268, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4047, 1191, 931, 0, 179, 0, 0, 404, 57, 0, 3578, 4085, 0, 0, 2385, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3799, 0, 0, 589, 4060, 7, 1288, 0, 0, 4091, 599, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3254, 1, 20, 476, 0, 0, 4072, 4096, 2617, 11, 42, 0, 1331, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 1, 573, 4060, 118, 58, 3168, 1003, 4, 3093, 0, 1, 25, 2210, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 16, 171, 2565, 19, 6, 7, 34, 2, 0, 0, 374, 49, 8, 138, 4, 3078, 3563, 2957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 758, 228, 1847, 3385, 1, 0, 3426, 402, 0, 8, 2701, 302, 0, 1926, 111, 6, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1933, 3718, 3190, 0, 58, 82, 4092, 0, 0, 4, 1, 1, 0, 36, 108, 3764, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 3318, 0, 0, 0, 0, 40, 0, 19, 0, 0, 4060, 3978, 343, 42, 3597, 2768, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17249, 14592, 16220, 18847, 13023, 15837, 16992, 18312] avg=16384.0000 max=18847 max_rank=3 min=13023 min_rank=4 imbalance_ratio=1.150330 max_load_reduction=112225 max_load_reduction_pct=0.856209 moved_to_echo=64164 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3971, 3811, 963, 135, 0, 9, 0, 1, 728, 4, 0, 4092, 0, 4, 0, 1510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 4094, 6, 0, 0, 586, 4096, 302, 530, 257, 855, 12, 3848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 1796, 426, 0, 4, 0, 218, 4061, 3961, 1098, 0, 0, 2, 2, 3731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 12, 3995, 3990, 0, 202, 4080, 0, 2, 0, 1, 0, 124, 2818, 3459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 15, 5, 107, 0, 0, 154, 5, 4096, 0, 411, 43, 0, 2, 1, 0, 4048, 3489, 3060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 158, 2230, 1, 9, 3024, 0, 28, 1, 18, 4094, 1, 32, 0, 747, 732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 597, 171, 3739, 0, 0, 4096, 27, 3900, 171, 0, 3291, 0, 0, 0, 3940, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 4096, 568, 0, 11, 4, 4005, 7, 0, 20, 122, 3929, 3307, 11, 0, 216, 2753, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15228, 14772, 15589, 19116, 15441, 11080, 20618, 19228] avg=16384.0000 max=20618 max_rank=6 min=11080 min_rank=5 imbalance_ratio=1.258423 max_load_reduction=110454 max_load_reduction_pct=0.842697 moved_to_echo=66367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4039, 3483, 759, 12, 0, 254, 0, 0, 109, 0, 0, 22, 0, 25, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 0, 0, 1, 4064, 0, 4006, 0, 0, 233, 520, 263, 0, 3280, 2636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1938, 3063, 30, 0, 2, 0, 0, 690, 2911, 3715, 910, 15, 0, 20, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 3, 4096, 4090, 958, 0, 2803, 291, 0, 0, 0, 1540, 0, 3953, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 59, 0, 1, 58, 4088, 0, 678, 2, 3792, 12, 0, 14, 827, 884, 3268, 3441, 2434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1530, 0, 0, 3, 0, 0, 1, 47, 3264, 0, 0, 2454, 629, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 851, 220, 3820, 66, 0, 0, 4089, 8, 1299, 0, 0, 1192, 0, 2881, 24, 4096, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 302, 0, 0, 11, 3993, 0, 148, 5, 2, 4096, 4027, 0, 0, 24, 6, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12716, 18929, 21485, 18161, 19594, 7931, 19126, 13130] avg=16384.0000 max=21485 max_rank=2 min=7931 min_rank=5 imbalance_ratio=1.311340 max_load_reduction=109587 max_load_reduction_pct=0.836082 moved_to_echo=59781 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3813, 4096, 698, 342, 0, 0, 147, 0, 4033, 1, 79, 3976, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 68, 0, 4084, 43, 3, 0, 65, 2416, 0, 3581, 35, 0, 346, 4054, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3031, 0, 555, 568, 45, 0, 584, 98, 3763, 1726, 0, 6, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 1, 0, 4093, 0, 0, 3328, 0, 0, 3022, 0, 665, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 1, 748, 0, 40, 0, 1613, 0, 0, 97, 247, 11, 0, 13, 4094, 3626, 2215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3790, 667, 0, 143, 0, 994, 4096, 1076, 0, 0, 3909, 4058, 199, 4068, 17, 1175, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 404, 2634, 4, 0, 7, 3250, 3, 712, 0, 0, 2504, 0, 0, 1, 3821, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 4092, 0, 0, 8, 0, 2164, 78, 0, 3, 0, 4094, 85, 1766, 268, 4087, 302, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21281, 15117, 10397, 15620, 12899, 24272, 14420, 17066] avg=16384.0000 max=24272 max_rank=5 min=10397 min_rank=2 imbalance_ratio=1.481445 max_load_reduction=106800 max_load_reduction_pct=0.814819 moved_to_echo=68657 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[903, 2, 914, 0, 2123, 350, 47, 0, 3325, 0, 310, 18, 1, 2692, 6, 3534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 3365, 5, 4091, 558, 1389, 176, 19, 0, 894, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2825, 3907, 54, 0, 4096, 4089, 0, 0, 405, 0, 1742, 0, 125, 32, 3372, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 3276, 4081, 675, 0, 0, 2339, 1198, 2, 0, 0, 1, 90, 3796, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1743, 0, 0, 1608, 10, 4094, 35, 1, 0, 0, 0, 0, 443, 166, 0, 3098, 676, 2903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 85, 3628, 0, 4096, 1724, 6, 2063, 597, 26, 1282, 67, 4089, 0, 719, 2, 1185, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 2574, 2, 3659, 12, 0, 0, 581, 4048, 14, 20, 0, 4048, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 14, 1, 2688, 1111, 11, 3953, 3, 2565, 7, 416, 0, 0, 2623, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14225, 11265, 20681, 16182, 14789, 20270, 16105, 17555] avg=16384.0000 max=20681 max_rank=2 min=11265 min_rank=1 imbalance_ratio=1.262268 max_load_reduction=110391 max_load_reduction_pct=0.842216 moved_to_echo=68719 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4095, 4043, 962, 0, 382, 313, 4, 578, 1, 0, 0, 1946, 192, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 22, 666, 4070, 0, 0, 589, 4082, 3422, 3805, 0, 8, 3097, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4095, 45, 636, 1055, 4015, 0, 0, 0, 3578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 398, 0, 3901, 1885, 359, 2, 3823, 0, 0, 62, 0, 3475, 80, 0, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 222, 13, 178, 0, 0, 886, 2, 0, 18, 0, 7, 340, 0, 0, 16, 4090, 3207, 3052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 3711, 217, 1, 20, 1142, 14, 0, 0, 0, 3699, 0, 2611, 0, 717, 737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 752, 3644, 0, 0, 3907, 0, 0, 0, 0, 3701, 0, 1632, 4, 4076, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 313, 1, 0, 0, 0, 139, 199, 0, 37, 3411, 4093, 0, 3, 4095, 2769, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12757, 23865, 13426, 18302, 12032, 13031, 18442, 19217] avg=16384.0000 max=23865 max_rank=1 min=12032 min_rank=4 imbalance_ratio=1.456604 max_load_reduction=107207 max_load_reduction_pct=0.817924 moved_to_echo=62722 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3849, 2238, 946, 0, 0, 652, 0, 0, 0, 0, 0, 4086, 0, 0, 1, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3533, 2973, 67, 0, 555, 3601, 4084, 2753, 87, 1611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 785, 0, 3, 179, 0, 518, 2895, 307, 1740, 3450, 0, 30, 0, 862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 783, 453, 2, 3869, 3264, 713, 0, 2, 43, 0, 4, 0, 1174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 1, 0, 3516, 2, 4008, 2961, 37, 30, 1218, 0, 0, 1, 403, 975, 23, 4091, 3643, 3004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4077, 4091, 4032, 0, 3375, 20, 0, 12, 13, 3863, 38, 44, 0, 1184, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 5, 0, 3340, 0, 0, 1, 0, 3943, 0, 0, 0, 385, 1088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 25, 4, 643, 0, 919, 0, 476, 4064, 3386, 0, 195, 17, 2630, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15851, 19267, 10889, 10307, 23949, 21461, 12847, 16501] avg=16384.0000 max=23949 max_rank=4 min=10307 min_rank=3 imbalance_ratio=1.461731 max_load_reduction=107123 max_load_reduction_pct=0.817284 moved_to_echo=74758 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4040, 2616, 962, 0, 532, 1, 11, 3, 3957, 171, 45, 4096, 1, 0, 3838, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3200, 1, 0, 0, 589, 102, 94, 4094, 0, 0, 4091, 1082, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 23, 1, 0, 0, 4094, 0, 0, 1767, 24, 3369, 3901, 1064, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 1651, 4064, 86, 0, 10, 333, 0, 0, 0, 0, 3826, 35, 3587, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1604, 1, 24, 0, 53, 3510, 0, 4091, 0, 39, 1, 4, 282, 8, 0, 3106, 3564, 3058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3065, 0, 0, 0, 4090, 3, 0, 161, 0, 0, 0, 4003, 3916, 0, 12, 0, 1203, 737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 10, 0, 4086, 7, 67, 18, 0, 311, 0, 2723, 21, 4015, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 28, 0, 127, 4070, 0, 1, 0, 445, 2287, 37, 0, 6, 69, 2770, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21765, 13265, 22385, 14537, 19345, 17190, 12522, 10063] avg=16384.0000 max=22385 max_rank=2 min=10063 min_rank=7 imbalance_ratio=1.366272 max_load_reduction=108687 max_load_reduction_pct=0.829216 moved_to_echo=59120 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[28800, 21504, 7168, 512, 3328, 1664, 256, 1024, 12288, 256, 4096, 22400, 256, 2816, 6272, 19456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[4864, 17792, 2688, 1152, 5248, 4352, 0, 14336, 14592, 15104, 10112, 7552, 3584, 5376, 8576, 16768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2816, 5120, 14720, 7680, 2688, 128, 30592, 128, 6016, 256, 640, 19072, 128, 7296, 256, 28160, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[8320, 128, 128, 15744, 18432, 128, 8192, 3584, 19840, 8320, 16640, 768, 3456, 15616, 12288, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 24448, 1280, 128, 128, 256, 17664, 1408, 1408, 4096, 1152, 28544, 18944, 2560, 640, 12160, 16640, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 3712, 512, 6400, 2432, 4224, 15744, 128, 10624, 1280, 4352, 640, 768, 1280, 2176, 1024, 28928, 25216, 22784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[3072, 4864, 4608, 9856, 19840, 5888, 1152, 17408, 2176, 128, 1408, 10880, 27392, 256, 9472, 2688, 6912, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1280, 3200, 4992, 24704, 22144, 2176, 384, 19968, 2560, 128, 6272, 128, 10752, 384, 16384, 16768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[296, 25, 119, 2, 2205, 0, 0, 0, 0, 129, 0, 1403, 3141, 0, 0, 449, 1335, 0, 342, 681, 0, 3031, 1323, 4058, 0, 2633, 3931, 85, 0, 0, 4030, 0, 3561, 0, 78, 0, 3955, 0, 0, 2572, 9, 0, 3818, 1977, 3854, 3088, 0, 2618, 16, 0, 545, 3979, 7, 854, 3465, 1, 13, 4094, 0, 0, 0, 3077, 0, 0, 139, 0, 19, 0, 1666, 1506, 25, 2915, 0, 4078, 0, 742, 4063, 0, 25, 0, 39, 81, 3492, 0, 4096, 2737, 17, 35, 0, 435, 546, 1274, 8, 0, 0, 3691, 518, 653, 80, 0, 114, 190, 2144, 554, 3894, 11, 0, 9, 4087, 164, 3083, 79, 0, 0, 0, 4089, 0, 0, 4067, 19, 0, 723, 108, 3593, 1, 464, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 28, 909, 0, 0, 4092, 156, 2939, 0, 4096, 254, 0, 3, 0, 0, 858, 0, 1, 0, 2, 802, 3349, 4031, 2307, 0, 3689, 4047, 174, 4040, 0, 3859, 0, 355, 3673, 3, 0, 2155, 0, 3670, 4087, 2, 13, 1, 4096, 3623, 4079, 0, 5, 1, 0, 22, 83, 0, 4027, 4075, 2, 0, 2, 1950, 13, 0, 1158, 4035, 0, 0, 514, 0, 0, 1, 0, 38, 2, 0, 18, 2983, 1, 1575, 747, 219, 1110, 4026, 45, 4002, 4094, 3193, 0, 0, 0, 4083, 14, 3, 69, 0, 2021, 88, 646, 0, 0, 0, 10, 1085, 0, 59, 0, 4085, 3090, 9, 0, 28, 523, 0, 0, 0, 2, 4039, 3974, 0, 0, 1097, 1, 6, 0, 6, 2800, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4005, 60, 0, 4077, 11, 3530, 0, 4045, 136, 0, 0, 339, 8, 94, 2344, 9, 0, 17, 4, 3072, 753, 768, 0, 3519, 1697, 2, 3974, 0, 1921, 1013, 2, 4068, 0, 18, 10, 0, 177, 2878, 1, 1180, 0, 595, 0, 3920, 0, 254, 3, 3393, 0, 2330, 10, 4095, 2669, 0, 803, 0, 2844, 0, 0, 0, 4016, 695, 0, 3342, 1, 1, 782, 2782, 12, 3831, 0, 4, 4027, 0, 6, 0, 3380, 1, 3434, 0, 273, 4074, 3896, 44, 3, 21, 0, 5, 60, 265, 0, 2190, 0, 1790, 6, 0, 0, 0, 1008, 2, 91, 0, 2753, 3953, 0, 3953, 67, 4061, 0, 49, 0, 0, 2758, 242, 2, 0, 4085, 164, 0, 11, 0, 4094, 0, 0, 176, 14] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 1186, 1396, 83, 0, 4036, 188, 4094, 0, 1031, 720, 69, 0, 0, 0, 2057, 0, 1337, 0, 0, 195, 0, 2, 3953, 0, 3701, 4079, 0, 3447, 0, 2748, 0, 59, 3977, 14, 15, 0, 1, 1421, 3277, 2, 696, 4, 4096, 2362, 1, 0, 0, 0, 0, 0, 31, 3136, 55, 0, 0, 1318, 0, 4067, 0, 2082, 0, 1689, 0, 0, 0, 0, 124, 4028, 4096, 69, 2969, 1, 94, 4096, 1500, 1716, 4073, 0, 0, 3604, 3, 4041, 150, 1335, 17, 29, 0, 8, 0, 0, 0, 0, 2529, 2679, 2844, 121, 2, 0, 0, 4096, 4, 0, 0, 0, 3012, 2, 0, 50, 4072, 3424, 1322, 0, 10, 3, 0, 0, 0, 4096, 2736, 0, 0, 8, 4068, 813, 4, 597, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 4085, 66, 4063, 0, 3852, 0, 4, 3571, 0, 0, 204, 0, 0, 338, 4074, 371, 104, 2968, 4091, 27, 4088, 4069, 0, 493, 0, 4024, 0, 0, 4095, 0, 0, 0, 0, 27, 2040, 4, 0, 0, 4093, 2905, 11, 0, 670, 0, 0, 0, 0, 0, 3968, 3837, 229, 34, 4095, 1, 0, 0, 397, 2622, 31, 8, 2927, 0, 0, 59, 11, 0, 4093, 0, 0, 3922, 0, 2363, 1006, 2, 4030, 4055, 0, 3792, 1032, 3930, 0, 0, 32, 33, 12, 549, 0, 0, 2723, 98, 554, 0, 0, 0, 0, 2, 0, 0, 0, 3951, 1, 911, 4096, 0, 3944, 20, 30, 0, 2, 6, 3731, 0, 19, 0, 0, 18, 93, 5, 4083, 0, 0, 1284, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1, 4, 4, 2, 0, 0, 0, 4066, 0, 188, 0, 0, 1, 0, 0, 3301, 0, 2672, 0, 21, 0, 4096, 0, 2183, 12, 4096, 1432, 0, 1655, 0, 3257, 0, 5, 4094, 2347, 193, 4068, 163, 294, 1876, 193, 0, 0, 4096, 4096, 4093, 2378, 6, 346, 3, 0, 3919, 0, 4088, 49, 0, 65, 286, 2736, 23, 890, 3982, 0, 2880, 0, 0, 0, 949, 0, 55, 0, 3877, 0, 242, 0, 0, 17, 6, 3, 4093, 170, 26, 3884, 26, 4087, 525, 0, 0, 3, 985, 3240, 0, 67, 4086, 1198, 4095, 9, 4090, 14, 103, 8, 0, 2, 0, 101, 4064, 9, 4095, 0, 0, 0, 270, 0, 1, 1, 1082, 0, 101, 3569, 61, 0, 0, 0, 3926, 0, 0, 3771, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 15, 0, 0, 3967, 66, 4096, 0, 4093, 0, 4, 2128, 0, 0, 3080, 4019, 1080, 133, 4088, 0, 38, 916, 0, 23, 4096, 3355, 0, 1, 1903, 4087, 0, 101, 0, 114, 1, 1196, 0, 125, 4042, 1757, 0, 0, 4095, 675, 10, 8, 326, 565, 512, 16, 4077, 0, 3994, 3883, 14, 5, 29, 7, 0, 59, 3811, 1, 9, 1, 31, 0, 0, 0, 0, 0, 4072, 0, 4012, 0, 0, 3987, 4093, 310, 3441, 3247, 0, 144, 0, 3959, 48, 14, 0, 4047, 3413, 0, 2, 24, 100, 5, 16, 0, 4096, 0, 0, 1771, 0, 14, 0, 3440, 3097, 0, 212, 956, 2965, 0, 3089, 959, 2, 143, 355, 1, 443, 0, 0, 0, 0, 29, 3644, 0, 268, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[112, 0, 0, 0, 0, 22, 0, 3618, 0, 4093, 0, 0, 4096, 0, 42, 4031, 0, 0, 1, 819, 121, 4048, 5, 4081, 1, 3329, 4096, 1007, 4094, 0, 685, 1, 0, 12, 198, 0, 3957, 0, 0, 3306, 0, 40, 1, 4093, 1, 4029, 0, 0, 0, 0, 0, 273, 0, 0, 872, 8, 1919, 4009, 2385, 0, 0, 2015, 0, 0, 0, 64, 0, 748, 4096, 0, 615, 3838, 0, 181, 0, 244, 1330, 3703, 648, 0, 3889, 1, 3803, 2477, 3604, 886, 1, 9, 3544, 3308, 0, 0, 1675, 3405, 5, 4096, 0, 0, 651, 0, 0, 16, 542, 0, 0, 2528, 1, 3282, 495, 4026, 454, 0, 0, 99, 8, 4095, 0, 0, 3177, 1, 0, 0, 105, 0, 0, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 28, 909, 0, 0, 4092, 156, 2939, 0, 4096, 254, 0, 3, 0, 0, 858, 1496, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 802, 3349, 4031, 2307, 0, 0, 1596, 174, 4040, 0, 3859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 3673, 3, 0, 2155, 0, 3670, 4087, 2, 13, 1, 1217, 3623, 4079, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 22, 83, 0, 4027, 4075, 2, 0, 2, 1950, 13, 0, 1158, 4035, 0, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 0, 0, 1, 0, 38, 2, 0, 18, 2983, 1, 1575, 747, 219, 1110, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 45, 4002, 4094, 253, 0, 0, 0, 4083, 14, 3, 69, 0, 2021, 88, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1085, 0, 59, 0, 4085, 3090, 9, 0, 28, 523, 0, 0, 1383, 1217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4039, 3974, 0, 0, 1097, 1, 6, 0, 6, 2800, 0, 0, 0, 0, 3689, 2040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15731, 20161, 22883, 15894, 7916, 19344, 11489, 17654] avg=16384.0000 max=22883 max_rank=2 min=7916 min_rank=4 imbalance_ratio=1.396667 max_load_reduction=108189 max_load_reduction_pct=0.825417 moved_to_echo=56403 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4005, 60, 0, 4077, 11, 3530, 0, 4045, 136, 0, 0, 339, 8, 94, 217, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2344, 9, 0, 17, 4, 3072, 753, 768, 0, 0, 670, 2, 3974, 0, 1921, 1013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4068, 0, 18, 10, 0, 177, 2878, 1, 1180, 0, 178, 0, 3920, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3393, 0, 2330, 10, 4095, 2669, 0, 803, 0, 2844, 0, 0, 0, 4016, 695, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3342, 1, 1, 782, 2782, 12, 3831, 0, 4, 4027, 0, 6, 0, 3380, 1, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3434, 0, 273, 4074, 309, 44, 3, 21, 0, 5, 60, 265, 0, 2190, 0, 1790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1008, 2, 91, 0, 2753, 3953, 0, 3953, 67, 4061, 0, 49, 200, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2758, 242, 2, 0, 4085, 164, 0, 11, 0, 4094, 0, 0, 176, 14, 3519, 2489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17620, 14547, 12686, 21078, 18466, 12468, 16653, 17554] avg=16384.0000 max=21078 max_rank=3 min=12468 min_rank=5 imbalance_ratio=1.286499 max_load_reduction=109994 max_load_reduction_pct=0.839188 moved_to_echo=65141 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[296, 25, 119, 2, 2205, 0, 0, 0, 0, 129, 0, 1403, 3141, 0, 0, 449, 722, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1335, 0, 342, 681, 0, 3031, 1323, 4058, 0, 0, 1551, 85, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3561, 0, 78, 0, 3955, 0, 0, 2572, 9, 0, 3818, 588, 3854, 3088, 0, 2618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 545, 3979, 7, 854, 3465, 1, 13, 4094, 0, 0, 0, 3077, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 19, 0, 1666, 1506, 25, 2915, 0, 4078, 0, 742, 4063, 0, 25, 0, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 81, 3492, 0, 325, 2737, 17, 35, 0, 435, 546, 1274, 8, 0, 0, 3691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 653, 80, 0, 114, 190, 2144, 554, 3894, 11, 0, 9, 4087, 164, 3083, 79, 667, 1182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4067, 19, 0, 723, 108, 3593, 1, 464, 0, 0, 2633, 2617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9645, 16436, 24141, 16562, 15865, 12680, 17429, 18314] avg=16384.0000 max=24141 max_rank=2 min=9645 min_rank=0 imbalance_ratio=1.473450 max_load_reduction=106931 max_load_reduction_pct=0.815819 moved_to_echo=64288 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 4, 4, 2, 0, 0, 0, 4066, 0, 188, 0, 0, 1, 0, 0, 3301, 1499, 1155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2672, 0, 21, 0, 4096, 0, 2183, 12, 0, 555, 0, 1655, 0, 3257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4094, 2347, 193, 4068, 163, 294, 1876, 193, 0, 0, 1210, 4096, 4093, 2378, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 346, 3, 0, 3919, 0, 4088, 49, 0, 65, 286, 2736, 23, 890, 3982, 0, 2880, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 949, 0, 55, 0, 3877, 0, 242, 0, 0, 17, 6, 3, 4093, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 26, 3884, 26, 316, 525, 0, 0, 3, 985, 3240, 0, 67, 4086, 1198, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4090, 14, 103, 8, 0, 2, 0, 101, 4064, 9, 4095, 0, 0, 0, 270, 1387, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1082, 0, 101, 3569, 61, 0, 0, 0, 3926, 0, 0, 3771, 1, 4096, 2616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10221, 14451, 25016, 19456, 9496, 18621, 14586, 19225] avg=16384.0000 max=25016 max_rank=2 min=9496 min_rank=4 imbalance_ratio=1.526855 max_load_reduction=106056 max_load_reduction_pct=0.809143 moved_to_echo=61928 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1186, 1396, 83, 0, 4036, 188, 4094, 0, 1031, 720, 69, 0, 0, 0, 2057, 1496, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1337, 0, 0, 195, 0, 2, 3953, 0, 0, 1610, 0, 3447, 0, 2748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 3977, 14, 15, 0, 1, 1421, 3277, 2, 696, 4, 1217, 2362, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3136, 55, 0, 0, 1318, 0, 4067, 0, 2082, 0, 1689, 0, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 4028, 4096, 69, 2969, 1, 94, 4096, 1500, 1716, 4073, 0, 0, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3604, 3, 4041, 150, 106, 17, 29, 0, 8, 0, 0, 0, 0, 2529, 2679, 2844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 2, 0, 0, 4096, 4, 0, 0, 0, 3012, 2, 0, 50, 4072, 3424, 1322, 1383, 1226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3, 0, 0, 0, 4096, 2736, 0, 0, 8, 4068, 813, 4, 597, 2, 3701, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16732, 13292, 13046, 12908, 23479, 16010, 18714, 16891] avg=16384.0000 max=23479 max_rank=4 min=12908 min_rank=3 imbalance_ratio=1.433044 max_load_reduction=107593 max_load_reduction_pct=0.820869 moved_to_echo=75094 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 4085, 66, 4063, 0, 3852, 0, 4, 3571, 0, 0, 204, 1495, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 338, 4074, 371, 104, 2968, 4091, 27, 0, 1605, 0, 493, 0, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 27, 2040, 4, 0, 0, 1216, 2905, 11, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 3837, 229, 34, 4095, 1, 0, 0, 397, 2622, 31, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2927, 0, 0, 59, 11, 0, 4093, 0, 0, 3922, 0, 2363, 1006, 2, 4030, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 3792, 1032, 312, 0, 0, 32, 33, 12, 549, 0, 0, 2723, 98, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3951, 1, 911, 4096, 0, 3944, 20, 30, 1382, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 3731, 0, 19, 0, 0, 18, 93, 5, 4083, 0, 0, 1284, 3, 4088, 2511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22538, 18095, 10968, 15743, 19133, 13192, 15560, 15843] avg=16384.0000 max=22538 max_rank=0 min=10968 min_rank=2 imbalance_ratio=1.375610 max_load_reduction=108534 max_load_reduction_pct=0.828049 moved_to_echo=63728 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 15, 0, 0, 3967, 66, 4096, 0, 4093, 0, 4, 2128, 0, 0, 3080, 1495, 1116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 1080, 133, 4088, 0, 38, 916, 0, 23, 0, 1323, 0, 1, 1903, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 114, 1, 1196, 0, 125, 4042, 1757, 0, 0, 1217, 675, 10, 8, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 565, 512, 16, 4077, 0, 3994, 3883, 14, 5, 29, 7, 0, 59, 3811, 1, 9, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 31, 0, 0, 0, 0, 0, 4072, 0, 4012, 0, 0, 3987, 4093, 310, 3441, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3247, 0, 144, 0, 313, 48, 14, 0, 4047, 3413, 0, 2, 24, 100, 5, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1771, 0, 14, 0, 3440, 3097, 0, 212, 956, 2965, 0, 3089, 1383, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 959, 2, 143, 355, 1, 443, 0, 0, 0, 0, 29, 3644, 0, 268, 2, 0, 4096, 2530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20060, 17611, 9572, 17418, 20534, 11373, 22032, 12472] avg=16384.0000 max=22032 max_rank=6 min=9572 min_rank=2 imbalance_ratio=1.344727 max_load_reduction=109040 max_load_reduction_pct=0.831909 moved_to_echo=66411 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[112, 0, 0, 0, 0, 22, 0, 3618, 0, 4093, 0, 0, 4096, 0, 42, 4031, 1495, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 819, 121, 4048, 5, 4081, 1, 0, 1616, 1007, 4094, 0, 685, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 198, 0, 3957, 0, 0, 3306, 0, 40, 1, 1216, 1, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 0, 872, 8, 1919, 4009, 2385, 0, 0, 2015, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 748, 4096, 0, 615, 3838, 0, 181, 0, 244, 1330, 3703, 648, 0, 716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 1, 3803, 2477, 285, 886, 1, 9, 3544, 3308, 0, 0, 1675, 3405, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 0, 0, 16, 542, 0, 0, 2528, 1, 3282, 495, 4026, 454, 0, 1382, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 8, 4095, 0, 0, 3177, 1, 0, 0, 105, 0, 0, 2, 0, 0, 3329, 2303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18525, 16479, 12760, 12013, 16183, 27384, 14609, 13119] avg=16384.0000 max=27384 max_rank=5 min=12013 min_rank=3 imbalance_ratio=1.671387 max_load_reduction=103688 max_load_reduction_pct=0.791077 moved_to_echo=71295 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1024, 128, 7040, 17664, 128, 640, 20096, 3072, 128, 896, 384, 26240, 896, 768, 5888, 128, 29184, 18048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[768, 8960, 768, 128, 8192, 256, 2944, 640, 18304, 19840, 1024, 15744, 5760, 19840, 7040, 4864, 9216, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[512, 1280, 10624, 256, 2304, 20352, 512, 26496, 0, 21632, 1152, 1536, 13056, 384, 128, 14080, 9984, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[7808, 5120, 896, 9728, 1536, 17792, 10112, 21504, 128, 0, 10624, 1280, 17792, 1920, 24704, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[22528, 256, 23552, 11904, 2304, 4352, 128, 128, 11776, 8192, 4480, 1664, 1792, 17152, 4096, 17792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 6912, 128, 1920, 10752, 8576, 768, 25600, 128, 8704, 15104, 2560, 15104, 13696, 4608, 12800, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1024, 3968, 640, 14720, 3200, 21120, 18944, 256, 4224, 12544, 14080, 128, 3072, 14464, 12416, 3712, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[4096, 19968, 2816, 256, 15360, 256, 5760, 24192, 2048, 2048, 3840, 8064, 17536, 19328, 2432, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[3915, 0, 2, 2255, 0, 0, 26, 0, 0, 235, 24, 0, 0, 111, 20, 4046, 1599, 4015, 2, 361, 0, 0, 24, 0, 0, 63, 4085, 4074, 0, 0, 0, 0, 0, 21, 1, 0, 4095, 2722, 0, 218, 3133, 1993, 0, 178, 0, 1924, 2849, 369, 52, 0, 2185, 4064, 108, 1, 0, 4, 228, 1872, 3344, 3738, 0, 3699, 81, 435, 4011, 41, 119, 77, 51, 1006, 42, 0, 1, 1, 0, 4074, 13, 3, 0, 0, 4085, 0, 0, 0, 4073, 259, 4074, 2099, 813, 4, 0, 3906, 3879, 0, 0, 63, 1575, 1088, 0, 3890, 0, 4014, 800, 1412, 0, 0, 11, 20, 1, 465, 0, 0, 886, 4076, 0, 231, 0, 44, 3331, 0, 557, 2151, 3959, 75, 3494, 4096, 31, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3674, 0, 4096, 3370, 3, 1666, 2308, 18, 4040, 3, 2, 1294, 630, 3444, 0, 0, 52, 1636, 4093, 913, 2904, 171, 2024, 837, 0, 57, 4083, 4085, 4080, 4039, 32, 3387, 26, 98, 0, 16, 0, 2706, 0, 4094, 2598, 1, 2919, 0, 712, 0, 792, 5, 4093, 5, 0, 157, 43, 1, 0, 0, 31, 385, 3540, 0, 270, 0, 0, 1316, 317, 0, 0, 0, 0, 1162, 705, 2747, 8, 0, 113, 427, 3, 0, 0, 1164, 8, 0, 0, 0, 131, 3301, 74, 0, 4096, 0, 3, 12, 4095, 2555, 0, 0, 2949, 2546, 0, 122, 3490, 396, 0, 1, 0, 3928, 7, 0, 0, 0, 2, 0, 464, 3858, 3142, 0, 6, 0, 4, 0, 1, 247, 119, 4088, 1014, 3018, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[57, 0, 0, 4046, 12, 1, 306, 211, 392, 0, 1, 0, 0, 3830, 1659, 4067, 1337, 0, 0, 0, 0, 0, 0, 33, 0, 0, 4052, 3807, 15, 0, 2, 4084, 12, 0, 3, 0, 84, 4070, 0, 2801, 4002, 34, 2048, 0, 211, 246, 0, 28, 7, 17, 161, 3547, 0, 0, 3064, 779, 3586, 0, 4096, 3722, 40, 895, 0, 0, 4025, 0, 2, 40, 2, 0, 0, 29, 63, 0, 0, 2688, 0, 0, 0, 2, 3767, 109, 0, 3, 2448, 4090, 4038, 239, 562, 0, 0, 3955, 3471, 0, 0, 0, 3583, 4095, 0, 3752, 3, 4096, 0, 19, 1, 1176, 9, 186, 4075, 42, 433, 0, 2, 1, 4095, 0, 4089, 0, 0, 0, 0, 0, 4084, 0, 4093, 2340, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4092, 0, 863, 1180, 0, 0, 459, 0, 46, 0, 0, 0, 0, 4068, 0, 4096, 4088, 7, 0, 0, 0, 0, 0, 0, 0, 251, 461, 3990, 2, 3174, 0, 3914, 0, 0, 0, 1875, 3, 3699, 2481, 610, 3894, 4095, 0, 6, 0, 0, 180, 9, 0, 0, 0, 4065, 139, 0, 2, 136, 3061, 0, 4096, 4096, 0, 3484, 3, 3, 4096, 0, 0, 1, 4074, 9, 50, 0, 175, 1, 1572, 4014, 0, 0, 72, 9, 1, 0, 0, 7, 3699, 3979, 1693, 1, 4093, 0, 80, 4088, 1498, 115, 142, 68, 0, 3850, 8, 38, 0, 4096, 0, 0, 0, 4065, 0, 3, 4096, 0, 0, 0, 0, 58, 2, 0, 3905, 125, 4081, 0, 335, 0, 1, 0, 2, 3030, 3212, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4056, 0, 4088, 0, 0, 6, 0, 109, 75, 0, 233, 4092, 12, 30, 47, 4093, 4094, 0, 0, 7, 1, 0, 0, 56, 0, 3200, 15, 3451, 0, 3635, 0, 3529, 0, 0, 3930, 1266, 666, 4074, 14, 41, 4095, 410, 0, 1, 0, 0, 0, 42, 3, 4087, 717, 3879, 0, 4, 3383, 2425, 4034, 1296, 3226, 4095, 0, 0, 0, 0, 4096, 0, 565, 0, 6, 0, 41, 2799, 0, 0, 1, 3991, 0, 0, 422, 0, 1078, 0, 372, 0, 4090, 4096, 4067, 0, 0, 653, 0, 2754, 3436, 0, 0, 1, 7, 1995, 1733, 109, 17, 0, 12, 0, 1881, 40, 539, 0, 2, 0, 0, 18, 0, 4096, 1442, 0, 3212, 15, 3632, 1653, 0, 1, 0, 145, 7, 1528, 2, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[266, 0, 0, 2497, 0, 131, 90, 4067, 0, 1, 0, 4027, 0, 18, 0, 3822, 3959, 4, 0, 1, 62, 13, 1591, 1349, 0, 0, 1534, 4027, 0, 0, 0, 3995, 25, 17, 0, 0, 509, 3128, 322, 0, 4096, 58, 0, 0, 0, 3930, 0, 15, 0, 4073, 1, 1570, 0, 0, 96, 0, 0, 0, 4058, 1909, 1, 4091, 0, 8, 4089, 18, 3328, 0, 0, 0, 155, 478, 0, 3217, 0, 3360, 0, 57, 2, 0, 2334, 0, 0, 0, 115, 3703, 4068, 1, 0, 5, 0, 4096, 0, 372, 1878, 1174, 1325, 4095, 0, 4092, 0, 6, 0, 17, 0, 11, 0, 2, 4096, 0, 0, 228, 0, 675, 3777, 0, 3759, 235, 3098, 2, 0, 1565, 4085, 738, 4096, 3353, 6, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 18, 291, 1177, 4096, 129, 0, 5, 0, 1970, 169, 4008, 4000, 1, 0, 1, 81, 0, 32, 7, 0, 182, 3883, 3738, 2457, 0, 0, 4096, 0, 76, 5, 0, 0, 4096, 0, 2880, 1074, 0, 4, 3, 2, 1126, 0, 144, 0, 0, 46, 3267, 1, 0, 20, 0, 0, 3918, 4096, 255, 277, 2262, 1, 37, 27, 0, 2888, 49, 0, 0, 0, 206, 0, 0, 425, 3813, 0, 0, 148, 763, 4035, 536, 0, 0, 2576, 3223, 3521, 4095, 0, 0, 0, 4088, 1323, 0, 0, 3235, 0, 4090, 1089, 460, 0, 79, 0, 0, 0, 0, 2840, 1612, 1126, 11, 4049, 0, 432, 2737, 3573, 0, 6, 0, 2514, 0, 0, 1425, 4045, 0, 3529, 4091, 132, 254] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[3826, 1, 0, 0, 3610, 0, 3, 0, 44, 3, 3946, 11, 0, 4096, 159, 4084, 2725, 0, 153, 0, 24, 2, 0, 0, 67, 6, 4053, 4046, 168, 51, 3, 2302, 303, 0, 227, 0, 0, 753, 1, 3954, 720, 3807, 0, 1, 1, 1732, 0, 6, 31, 1, 0, 4088, 27, 3719, 4096, 94, 1569, 497, 950, 368, 0, 3502, 4059, 0, 2065, 0, 2189, 0, 4011, 0, 0, 7, 2551, 0, 48, 536, 0, 0, 7, 643, 236, 0, 0, 10, 20, 4065, 0, 0, 3900, 0, 1, 1998, 0, 0, 149, 24, 144, 4075, 3557, 110, 0, 2588, 3879, 5, 28, 13, 25, 1259, 3088, 0, 0, 527, 1404, 11, 1817, 2066, 4096, 1, 4096, 0, 4, 5, 4079, 0, 3601, 0, 227, 18] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3915, 0, 2, 2255, 0, 0, 26, 0, 0, 235, 24, 0, 0, 111, 20, 3903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1599, 4015, 2, 361, 0, 0, 24, 0, 0, 63, 4085, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1, 0, 4095, 2722, 0, 218, 3133, 1993, 0, 178, 0, 1924, 2849, 369, 1957, 732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 2185, 4064, 108, 1, 0, 4, 228, 1872, 1183, 3738, 0, 3699, 81, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 41, 119, 77, 51, 1006, 42, 0, 1, 1, 0, 4074, 13, 3, 0, 0, 232, 4096, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4073, 27, 4074, 2099, 813, 4, 0, 3906, 3879, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1575, 1088, 0, 3890, 0, 4014, 800, 1412, 0, 0, 11, 20, 1, 465, 0, 0, 749, 819, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 886, 4076, 0, 231, 0, 44, 2512, 0, 557, 2151, 3959, 75, 3494, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10491, 12742, 20192, 17650, 13971, 23023, 14987, 18016] avg=16384.0000 max=23023 max_rank=5 min=10491 min_rank=0 imbalance_ratio=1.405212 max_load_reduction=108049 max_load_reduction_pct=0.824348 moved_to_echo=69997 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[57, 0, 0, 4046, 12, 1, 306, 211, 392, 0, 1, 0, 0, 3830, 1659, 3924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1337, 0, 0, 0, 0, 0, 0, 33, 0, 0, 4052, 2423, 15, 0, 2, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 3, 0, 84, 4070, 0, 2801, 4002, 34, 2048, 0, 211, 246, 0, 28, 2397, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 17, 161, 3547, 0, 0, 3064, 779, 3586, 0, 1449, 3722, 40, 895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 2, 40, 2, 0, 0, 29, 63, 0, 0, 2688, 0, 0, 0, 2, 3663, 2340, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3767, 109, 0, 3, 2448, 427, 4038, 239, 562, 0, 0, 3955, 3471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3583, 4095, 0, 3752, 3, 4096, 0, 19, 1, 1176, 9, 186, 4075, 42, 433, 0, 700, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4095, 0, 4089, 0, 0, 0, 0, 0, 4084, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14439, 11946, 16620, 17267, 13104, 19019, 22313, 16364] avg=16384.0000 max=22313 max_rank=6 min=11946 min_rank=1 imbalance_ratio=1.361877 max_load_reduction=108759 max_load_reduction_pct=0.829765 moved_to_echo=70800 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3674, 0, 4096, 3370, 3, 1666, 2308, 18, 4040, 3, 2, 1294, 630, 3444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 1636, 4093, 913, 2904, 171, 2024, 837, 0, 57, 4083, 2599, 4080, 4039, 32, 3387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 98, 0, 16, 0, 2706, 0, 4094, 2598, 1, 2919, 0, 712, 0, 792, 5, 2072, 734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 0, 157, 43, 1, 0, 0, 31, 385, 1252, 0, 270, 0, 0, 1316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 0, 0, 0, 0, 1162, 705, 2747, 8, 0, 113, 427, 3, 0, 0, 1164, 2957, 3018, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 131, 344, 74, 0, 4096, 0, 3, 12, 4095, 2555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2949, 2546, 0, 122, 3490, 396, 0, 1, 0, 3928, 7, 0, 0, 0, 2, 0, 752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 3858, 3142, 0, 6, 0, 4, 0, 1, 247, 119, 4088, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[24548, 30907, 16773, 7553, 12837, 11318, 14193, 12943] avg=16384.0000 max=30907 max_rank=1 min=7553 min_rank=3 imbalance_ratio=1.886414 max_load_reduction=100165 max_load_reduction_pct=0.764198 moved_to_echo=51291 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4056, 0, 4088, 0, 0, 6, 0, 109, 75, 0, 233, 4092, 12, 30, 47, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 7, 1, 0, 0, 56, 0, 3200, 15, 2187, 0, 3635, 0, 3529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 1266, 666, 4074, 14, 41, 4095, 410, 0, 1, 0, 0, 0, 42, 1891, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4087, 717, 3879, 0, 4, 3383, 2425, 4034, 1296, 1136, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 565, 0, 6, 0, 41, 2799, 0, 0, 1, 3991, 0, 0, 422, 0, 3671, 1528, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 0, 372, 0, 4090, 425, 4067, 0, 0, 653, 0, 2754, 3436, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1995, 1733, 109, 17, 0, 12, 0, 1881, 40, 539, 0, 2, 0, 0, 18, 639, 897, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1442, 0, 3212, 15, 2735, 1653, 0, 1, 0, 145, 7, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16693, 16724, 17055, 25059, 17319, 16876, 8037, 13309] avg=16384.0000 max=25059 max_rank=3 min=8037 min_rank=6 imbalance_ratio=1.529480 max_load_reduction=106013 max_load_reduction_pct=0.808815 moved_to_echo=55541 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 18, 291, 1177, 4096, 129, 0, 5, 0, 1970, 169, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 1, 0, 1, 81, 0, 32, 7, 0, 182, 3883, 2378, 2457, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 5, 0, 0, 4096, 0, 2880, 1074, 0, 4, 3, 2, 1126, 0, 144, 2397, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 3267, 1, 0, 20, 0, 0, 3918, 1449, 255, 277, 2262, 1, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 2888, 49, 0, 0, 0, 206, 0, 0, 425, 3813, 0, 0, 148, 763, 2887, 4091, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 536, 0, 0, 2576, 336, 3521, 4095, 0, 0, 0, 4088, 1323, 0, 0, 3235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1089, 460, 0, 79, 0, 0, 0, 0, 2840, 1612, 1126, 11, 4049, 0, 688, 618, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 2737, 3573, 0, 6, 0, 1896, 0, 0, 1425, 4045, 0, 3529, 0, 132, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15818, 17118, 12479, 11533, 15547, 23745, 16803, 18029] avg=16384.0000 max=23745 max_rank=5 min=11533 min_rank=3 imbalance_ratio=1.449280 max_load_reduction=107327 max_load_reduction_pct=0.818840 moved_to_echo=74124 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 863, 1180, 0, 0, 459, 0, 46, 0, 0, 0, 0, 4068, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 0, 0, 0, 0, 0, 0, 251, 461, 2539, 2, 3174, 0, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1875, 3, 3699, 2481, 610, 3894, 4095, 0, 6, 0, 0, 180, 9, 2397, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 139, 0, 2, 136, 3061, 0, 1449, 4096, 0, 3484, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4074, 9, 50, 0, 175, 1, 1572, 4014, 0, 0, 72, 9, 3564, 3030, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 3699, 415, 1693, 1, 4093, 0, 80, 4088, 1498, 115, 142, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3850, 8, 38, 0, 4096, 0, 0, 0, 4065, 0, 3, 4096, 0, 0, 0, 734, 1003, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 2, 0, 3905, 125, 3078, 0, 335, 0, 1, 0, 2, 0, 3212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14660, 14436, 19966, 16438, 20917, 15900, 18037, 10718] avg=16384.0000 max=20917 max_rank=4 min=10718 min_rank=7 imbalance_ratio=1.276672 max_load_reduction=110155 max_load_reduction_pct=0.840416 moved_to_echo=65572 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[266, 0, 0, 2497, 0, 131, 90, 4067, 0, 1, 0, 4027, 0, 18, 0, 3687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 4, 0, 1, 62, 13, 1591, 1349, 0, 0, 1534, 2562, 0, 0, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 17, 0, 0, 509, 3128, 322, 0, 4096, 58, 0, 0, 0, 3930, 0, 15, 2375, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 1, 1570, 0, 0, 96, 0, 0, 0, 1436, 1909, 1, 4091, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 18, 3328, 0, 0, 0, 155, 478, 0, 3217, 0, 3360, 0, 57, 2, 0, 3317, 3353, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2334, 0, 0, 0, 115, 386, 4068, 1, 0, 5, 0, 4096, 0, 372, 1878, 1174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 4095, 0, 4092, 0, 6, 0, 17, 0, 11, 0, 2, 4096, 0, 0, 228, 741, 761, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 675, 3777, 0, 3759, 235, 2337, 2, 0, 1565, 4085, 738, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14784, 15070, 15199, 13185, 21621, 14429, 15509, 21275] avg=16384.0000 max=21621 max_rank=4 min=13185 min_rank=3 imbalance_ratio=1.319641 max_load_reduction=109451 max_load_reduction_pct=0.835045 moved_to_echo=72834 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3826, 1, 0, 0, 3610, 0, 3, 0, 44, 3, 3946, 11, 0, 4096, 159, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2725, 0, 153, 0, 24, 2, 0, 0, 67, 6, 4053, 2575, 168, 51, 3, 2302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 227, 0, 0, 753, 1, 3954, 720, 3807, 0, 1, 1, 1732, 0, 6, 556, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 1, 0, 4088, 27, 3719, 4096, 94, 1569, 497, 336, 368, 0, 3502, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2065, 0, 2189, 0, 4011, 0, 0, 7, 2551, 0, 48, 536, 0, 0, 7, 643, 3641, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 10, 20, 424, 0, 0, 3900, 0, 1, 1998, 0, 0, 149, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 4075, 3557, 110, 0, 2588, 3879, 5, 28, 13, 25, 1259, 3088, 0, 0, 527, 744, 1007, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1404, 11, 1817, 2066, 4096, 1, 3089, 0, 4, 5, 4079, 0, 3601, 0, 227, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19639, 12129, 12788, 22387, 15756, 6762, 21193, 20418] avg=16384.0000 max=22387 max_rank=3 min=6762 min_rank=5 imbalance_ratio=1.366394 max_load_reduction=108685 max_load_reduction_pct=0.829201 moved_to_echo=64129 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[22784, 128, 9216, 256, 8192, 2304, 1024, 6272, 2816, 3328, 2176, 22912, 128, 128, 768, 2688, 23936, 21504, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[21888, 5760, 4352, 1408, 3072, 256, 3712, 2304, 128, 3840, 22272, 19968, 6784, 11008, 128, 25344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[19968, 128, 9088, 17536, 3712, 1920, 3584, 5632, 8704, 384, 4224, 9472, 768, 17664, 2176, 27264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9600, 25856, 6400, 12672, 3584, 15360, 4736, 1536, 1920, 9344, 3456, 3200, 16512, 640, 4608, 896, 5760, 5120, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3200, 15616, 17920, 2304, 19200, 512, 15744, 1664, 1024, 5504, 20480, 5120, 19840, 0, 3712, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[15616, 768, 384, 128, 17152, 2816, 21632, 6528, 13568, 768, 128, 24960, 17792, 3072, 2176, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[4224, 8192, 3200, 24704, 384, 3840, 10752, 3456, 12544, 8064, 9728, 18304, 640, 18048, 4224, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[384, 256, 4224, 3200, 5376, 25344, 2944, 14720, 23680, 10496, 4992, 256, 1024, 8960, 3840, 640, 16128, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 3178, 4071, 1, 3632, 0, 549, 426, 4096, 288, 4, 1774, 0, 30, 1110, 34, 2174, 0, 703, 50, 3887, 1072, 2685, 7, 4066, 0, 0, 206, 0, 1, 0, 0, 0, 0, 4091, 0, 3191, 11, 0, 4073, 11, 1292, 4091, 0, 27, 0, 0, 445, 0, 1, 0, 2, 0, 4085, 50, 1832, 89, 0, 0, 49, 2884, 224, 31, 0, 0, 2, 1940, 2, 0, 4020, 1875, 1171, 1, 4090, 21, 1599, 2, 3628, 2936, 1, 3853, 3993, 2714, 14, 3589, 16, 0, 8, 0, 0, 0, 2984, 130, 0, 824, 0, 0, 0, 833, 2846, 0, 93, 0, 3626, 5, 373, 2472, 3457, 0, 0, 4096, 136, 4096, 0, 0, 1, 15, 4078, 1, 0, 3987, 76, 937, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 618, 3965, 0, 3812, 13, 18, 4078, 0, 4, 0, 0, 4002, 0, 28, 1, 4096, 3125, 1, 0, 4096, 1702, 73, 0, 4023, 87, 0, 41, 155, 2375, 0, 0, 1022, 4092, 3708, 4096, 119, 2552, 3522, 3413, 3514, 1995, 0, 0, 5, 0, 0, 0, 1, 0, 0, 474, 1961, 479, 0, 0, 0, 6, 0, 4091, 17, 85, 48, 1, 3861, 0, 14, 5, 4047, 3965, 224, 0, 3937, 222, 0, 0, 780, 943, 2946, 64, 0, 2, 2531, 4069, 0, 4096, 0, 0, 0, 0, 3074, 0, 1, 55, 20, 9, 911, 0, 4092, 0, 0, 0, 2, 48, 0, 729, 4026, 578, 100, 2, 0, 3953, 0, 3984, 134, 30, 1887, 9, 293, 3778, 2, 0, 131] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 1, 3442, 0, 3143, 4050, 0, 3902, 3944, 33, 104, 550, 3489, 141, 18, 0, 0, 0, 0, 1, 0, 0, 21, 4090, 3972, 14, 225, 953, 2656, 7, 4, 3894, 0, 4022, 3976, 999, 0, 3226, 0, 3661, 0, 1281, 4083, 1, 4063, 4059, 1931, 0, 14, 0, 0, 0, 2, 0, 1, 6, 264, 0, 0, 1771, 45, 0, 99, 0, 0, 36, 0, 2020, 4038, 0, 0, 0, 3779, 3539, 28, 1225, 3210, 0, 4061, 2, 4096, 3, 219, 6, 0, 4041, 0, 20, 0, 4036, 0, 41, 1558, 785, 2, 545, 0, 0, 9, 0, 0, 0, 4072, 372, 1, 4069, 0, 0, 0, 4084, 0, 0, 0, 0, 190, 4096, 62, 4037, 1, 35, 0, 2, 0, 0, 594, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 282, 0, 0, 1, 58, 1563, 180, 4095, 1, 0, 81, 4084, 3567, 0, 135, 0, 0, 371, 0, 0, 0, 0, 1183, 2339, 0, 4095, 4094, 4058, 0, 2, 0, 1, 0, 4081, 0, 1, 3941, 0, 153, 258, 51, 4088, 0, 3784, 4096, 3117, 2376, 1589, 2, 0, 0, 1, 454, 0, 0, 1, 0, 0, 859, 121, 0, 11, 9, 4024, 0, 9, 0, 4096, 0, 0, 0, 4095, 4049, 0, 37, 4054, 0, 0, 0, 4041, 0, 1, 3992, 3604, 3876, 2063, 1345, 777, 237, 0, 0, 0, 34, 6, 0, 91, 0, 3617, 0, 31, 3373, 3873, 1, 0, 1399, 0, 6, 0, 1, 5, 0, 21, 4096, 5, 4094, 1, 187, 0, 0, 3995, 2402, 385, 3966, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 3290, 4033, 4095, 0, 3865, 40, 0, 0, 19, 0, 2, 120, 586, 0, 6, 0, 0, 0, 0, 216, 4096, 0, 813, 4065, 0, 2, 2, 450, 0, 1794, 0, 0, 4094, 4061, 50, 3658, 0, 4012, 4096, 1294, 3654, 262, 3890, 0, 0, 131, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 1, 0, 3, 3, 0, 0, 0, 5, 4008, 0, 327, 0, 4096, 0, 1, 134, 4077, 4041, 3322, 0, 4084, 0, 180, 0, 0, 0, 35, 4096, 0, 10, 0, 167, 976, 3101, 0, 3798, 0, 0, 3998, 1812, 650, 1366, 4096, 1063, 441, 3988, 8, 6, 34, 3843, 0, 15, 412, 0, 705, 3216, 1962, 3245, 1608, 0, 0, 0, 263, 1092, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 50, 0, 29, 0, 3729, 4096, 0, 2931, 949, 481, 509, 0, 0, 59, 1808, 3745, 4, 0, 0, 1972, 0, 0, 0, 3560, 412, 4046, 28, 4091, 5, 0, 146, 166, 0, 3751, 0, 2122, 259, 0, 0, 18, 0, 3952, 17, 3725, 4085, 4096, 106, 0, 0, 0, 4, 1, 1, 181, 0, 3536, 4000, 31, 2, 0, 0, 0, 913, 13, 321, 11, 0, 3856, 1505, 0, 0, 4021, 4096, 144, 625, 4004, 157, 1, 0, 2948, 0, 4072, 640, 0, 169, 57, 4094, 3938, 3522, 0, 0, 0, 0, 3, 4085, 0, 0, 3845, 0, 0, 0, 4075, 403, 0, 1790, 0, 2668, 0, 2527, 8, 0, 0, 374, 1329, 4041, 0, 36, 0, 0, 664, 0, 390, 3023, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3780, 0, 0, 0, 0, 4, 3368, 4096, 4082, 2781, 0, 236, 0, 31, 7, 0, 0, 0, 150, 0, 1362, 347, 53, 0, 1437, 4043, 0, 18, 0, 4095, 0, 0, 3, 3, 5, 4096, 0, 0, 88, 0, 333, 0, 0, 0, 953, 3343, 4096, 3471, 0, 0, 0, 0, 0, 0, 84, 0, 0, 4073, 3400, 5, 0, 0, 0, 0, 4095, 0, 36, 0, 969, 4079, 0, 0, 0, 2075, 4096, 0, 413, 3136, 0, 0, 64, 4094, 0, 3555, 0, 70, 4073, 0, 2, 67, 2977, 1, 0, 22, 0, 4096, 37, 17, 0, 2193, 0, 1, 4096, 4096, 3860, 0, 0, 0, 0, 0, 2146, 3978, 0, 0, 8, 0, 4029, 1695, 0, 0, 0, 4087, 0, 3501, 4070, 777, 0, 718] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 2, 134, 14, 0, 4094, 0, 3482, 2923, 299, 1, 103, 65, 0, 2004, 0, 212, 4091, 3, 0, 421, 3899, 458, 214, 6, 2087, 2833, 1113, 0, 2, 132, 0, 125, 4030, 0, 0, 3568, 3552, 127, 31, 3897, 3994, 0, 0, 1813, 2514, 4002, 4096, 117, 349, 64, 0, 20, 0, 0, 8, 24, 0, 3995, 0, 187, 0, 0, 0, 0, 36, 4096, 88, 0, 0, 0, 3, 0, 0, 4094, 1, 0, 4066, 3124, 4096, 33, 3942, 844, 1163, 497, 0, 2683, 4083, 0, 1, 0, 1, 0, 4, 2, 3957, 0, 1256, 0, 2542, 0, 1331, 2250, 4067, 2344, 0, 2, 0, 92, 0, 0, 0, 282, 0, 13, 1025, 4070, 0, 0, 1880, 0, 0, 0, 3335, 4076, 4, 584, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 3442, 0, 3143, 4050, 0, 3902, 3944, 33, 104, 550, 3489, 141, 18, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 21, 4090, 3972, 14, 225, 953, 2656, 7, 4, 1256, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3894, 0, 4022, 3976, 999, 0, 3226, 0, 3661, 0, 1281, 0, 1, 0, 0, 1888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 2, 0, 1, 6, 264, 0, 0, 1771, 45, 0, 99, 2918, 4059, 4063, 1574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 2020, 4038, 0, 0, 0, 3779, 3539, 28, 1225, 292, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1979, 3, 219, 6, 0, 4041, 0, 20, 0, 4036, 0, 41, 1558, 785, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 545, 0, 0, 9, 0, 0, 0, 4072, 372, 1, 4069, 0, 0, 0, 4084, 0, 251, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 4096, 62, 4037, 1, 35, 0, 2, 0, 0, 594, 0, 0, 2411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23427, 13297, 22948, 14816, 19018, 12692, 13446, 11428] avg=16384.0000 max=23427 max_rank=0 min=11428 min_rank=7 imbalance_ratio=1.429871 max_load_reduction=107645 max_load_reduction_pct=0.821266 moved_to_echo=56584 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 618, 3965, 0, 3812, 13, 18, 4078, 0, 4, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 28, 1, 4096, 3125, 1, 0, 4096, 1702, 73, 0, 4023, 87, 0, 239, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 155, 2375, 0, 0, 1022, 4092, 3708, 4096, 119, 2552, 0, 3413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 474, 1961, 479, 0, 0, 0, 6, 0, 3579, 1995, 3514, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 17, 85, 48, 1, 3861, 0, 14, 5, 4047, 3965, 224, 0, 358, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 943, 2946, 64, 0, 2, 2531, 4069, 0, 4096, 0, 0, 0, 0, 3074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 55, 20, 9, 911, 0, 4092, 0, 0, 0, 2, 48, 0, 729, 4026, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 100, 2, 0, 3953, 0, 3984, 134, 30, 1887, 9, 293, 3778, 2, 0, 131, 2081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12628, 21557, 21573, 13371, 16938, 18103, 9940, 16962] avg=16384.0000 max=21573 max_rank=2 min=9940 min_rank=6 imbalance_ratio=1.316711 max_load_reduction=109499 max_load_reduction_pct=0.835411 moved_to_echo=61943 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 3290, 4033, 4095, 0, 3865, 40, 0, 0, 19, 0, 2, 608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 586, 0, 6, 0, 0, 0, 0, 216, 4096, 0, 813, 4065, 0, 2, 2, 1252, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 0, 1794, 0, 0, 4094, 4061, 50, 3658, 0, 4012, 0, 1294, 0, 0, 3803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 1, 0, 3, 3706, 262, 3654, 1578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 5, 4008, 0, 327, 0, 4096, 0, 1, 134, 371, 4041, 3322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1974, 0, 180, 0, 0, 0, 35, 4096, 0, 10, 0, 167, 976, 3101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 0, 3998, 1812, 650, 1366, 4096, 1063, 441, 3988, 8, 6, 34, 3843, 0, 250, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 412, 0, 705, 3216, 1962, 3245, 1608, 0, 0, 0, 263, 1092, 0, 0, 0, 2420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15964, 11256, 23216, 13411, 16308, 10539, 25440, 14938] avg=16384.0000 max=25440 max_rank=6 min=10539 min_rank=5 imbalance_ratio=1.552734 max_load_reduction=105632 max_load_reduction_pct=0.805908 moved_to_echo=67225 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 282, 0, 0, 1, 58, 1563, 180, 4095, 1, 0, 81, 4084, 3567, 0, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 371, 0, 0, 0, 0, 1183, 2339, 0, 4095, 4094, 4058, 0, 2, 1239, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4081, 0, 1, 3941, 0, 153, 258, 51, 0, 0, 0, 0, 3047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2376, 1589, 2, 0, 0, 1, 454, 0, 0, 1, 0, 0, 859, 121, 0, 11, 3685, 4096, 3784, 1575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4024, 0, 9, 0, 4096, 0, 0, 0, 4095, 4049, 0, 37, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1952, 0, 1, 3992, 3604, 3876, 2063, 1345, 777, 237, 0, 0, 0, 34, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 3617, 0, 31, 3373, 3873, 1, 0, 1399, 0, 6, 0, 1, 5, 248, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 5, 4094, 1, 187, 0, 0, 3995, 2402, 385, 3966, 0, 0, 0, 2414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14514, 17615, 11533, 18554, 16688, 17887, 12715, 21566] avg=16384.0000 max=21566 max_rank=7 min=11533 min_rank=2 imbalance_ratio=1.316284 max_load_reduction=109506 max_load_reduction_pct=0.835464 moved_to_echo=68856 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3780, 0, 0, 0, 0, 4, 3368, 4096, 4082, 2781, 0, 236, 0, 31, 7, 0, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 1362, 347, 53, 0, 1437, 4043, 0, 18, 0, 4095, 0, 0, 1258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 5, 4096, 0, 0, 88, 0, 333, 0, 0, 0, 953, 0, 0, 3392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 4073, 3400, 5, 0, 0, 0, 0, 2856, 4096, 3343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 36, 0, 969, 4079, 0, 0, 0, 2075, 4096, 0, 413, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 1969, 0, 3555, 0, 70, 4073, 0, 2, 67, 2977, 1, 0, 22, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 17, 0, 2193, 0, 1, 4096, 4096, 3860, 0, 0, 0, 0, 0, 2146, 3978, 255, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4029, 1695, 0, 0, 0, 4087, 0, 3501, 4070, 777, 0, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18997, 12763, 8873, 17857, 16043, 16896, 20758, 18885] avg=16384.0000 max=20758 max_rank=6 min=8873 min_rank=2 imbalance_ratio=1.266968 max_load_reduction=110314 max_load_reduction_pct=0.841629 moved_to_echo=72582 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 50, 0, 29, 0, 3729, 4096, 0, 2931, 949, 481, 509, 0, 0, 59, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1808, 3745, 4, 0, 0, 1972, 0, 0, 0, 3560, 412, 4046, 28, 4091, 5, 0, 904, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 166, 0, 3751, 0, 2122, 259, 0, 0, 18, 0, 0, 17, 0, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 4, 1, 1, 181, 0, 3536, 4000, 31, 2, 0, 0, 0, 3640, 4085, 3725, 1523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 913, 13, 321, 11, 0, 3856, 1505, 0, 0, 4021, 4096, 144, 625, 364, 157, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1424, 0, 4072, 640, 0, 169, 57, 4094, 3938, 3522, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 3845, 0, 0, 0, 4075, 403, 0, 1790, 0, 2668, 0, 2527, 8, 181, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 1329, 4041, 0, 36, 0, 0, 664, 0, 390, 3023, 0, 0, 0, 2333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13273, 20671, 10483, 20835, 16027, 17919, 19674, 12190] avg=16384.0000 max=20835 max_rank=3 min=10483 min_rank=2 imbalance_ratio=1.271667 max_load_reduction=110237 max_load_reduction_pct=0.841042 moved_to_echo=65810 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 3178, 4071, 1, 3632, 0, 549, 426, 4096, 288, 4, 1774, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 1110, 34, 2174, 0, 703, 50, 3887, 1072, 2685, 7, 4066, 0, 0, 206, 1112, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4091, 0, 3191, 11, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 445, 0, 1, 0, 2, 0, 4085, 50, 1832, 89, 0, 0, 49, 3718, 4091, 1292, 1571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2884, 224, 31, 0, 0, 2, 1940, 2, 0, 4020, 1875, 1171, 1, 372, 21, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1754, 2936, 1, 3853, 3993, 2714, 14, 3589, 16, 0, 8, 0, 0, 0, 2984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 824, 0, 0, 0, 833, 2846, 0, 93, 0, 3626, 5, 373, 2472, 3457, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 136, 4096, 0, 0, 1, 15, 4078, 1, 0, 3987, 76, 937, 6, 2405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18561, 17233, 7305, 17252, 14142, 21864, 14881, 19834] avg=16384.0000 max=21864 max_rank=5 min=7305 min_rank=2 imbalance_ratio=1.334473 max_load_reduction=109208 max_load_reduction_pct=0.833191 moved_to_echo=70721 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 134, 14, 0, 4094, 0, 3482, 2923, 299, 1, 103, 65, 0, 2004, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 4091, 3, 0, 421, 3899, 458, 214, 6, 2087, 2833, 1113, 0, 2, 132, 0, 1209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 4030, 0, 0, 3568, 3552, 127, 31, 3897, 3994, 0, 0, 1813, 0, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 349, 64, 0, 20, 0, 0, 8, 24, 0, 3995, 0, 187, 0, 0, 0, 3696, 4002, 2514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 4096, 88, 0, 0, 0, 3, 0, 0, 4094, 1, 0, 370, 3124, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 1904, 844, 1163, 497, 0, 2683, 4083, 0, 1, 0, 1, 0, 4, 2, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1256, 0, 2542, 0, 1331, 2250, 4067, 2344, 0, 2, 0, 92, 0, 0, 0, 242, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 13, 1025, 4070, 0, 0, 1880, 0, 0, 0, 3335, 4076, 4, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13708, 16680, 25141, 14976, 15908, 15172, 14218, 15269] avg=16384.0000 max=25141 max_rank=2 min=13708 min_rank=0 imbalance_ratio=1.534485 max_load_reduction=105931 max_load_reduction_pct=0.808189 moved_to_echo=60567 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[12032, 4352, 4608, 256, 3072, 24064, 3456, 384, 128, 26240, 25728, 1664, 2560, 2816, 7680, 13184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[3840, 128, 512, 3456, 128, 13824, 19968, 21376, 14720, 21760, 1664, 1408, 9472, 7936, 5760, 1920, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 13440, 4736, 12160, 9088, 7680, 17664, 8832, 17280, 4864, 14976, 128, 256, 2560, 3968, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[8704, 1408, 896, 16256, 1920, 2944, 12032, 31232, 8064, 640, 11264, 3712, 2944, 512, 15872, 11520, 1792, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[2688, 2048, 256, 512, 128, 128, 640, 256, 512, 18048, 12032, 1920, 2944, 256, 128, 256, 27904, 26752, 25984, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[896, 640, 8704, 3456, 31616, 3840, 11520, 3712, 128, 14720, 2432, 8192, 24064, 1536, 1536, 896, 14080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2304, 12544, 1280, 512, 3968, 10368, 4352, 384, 10880, 25344, 7680, 10496, 13312, 18944, 256, 256, 8576, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[4736, 4480, 8320, 16000, 4608, 10880, 19968, 3840, 19072, 4480, 7936, 0, 7552, 0, 0, 20224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[405, 1401, 437, 3, 0, 92, 613, 1, 824, 53, 0, 207, 4076, 1569, 38, 0, 0, 2, 0, 4, 4007, 907, 0, 0, 2383, 3384, 4064, 0, 4016, 371, 0, 88, 1042, 0, 0, 0, 0, 1510, 2806, 4093, 1784, 0, 0, 4088, 0, 2719, 9, 0, 2903, 12, 0, 0, 0, 4051, 126, 41, 0, 1, 0, 0, 0, 1577, 0, 353, 0, 0, 0, 458, 0, 75, 0, 0, 267, 2634, 0, 54, 99, 3998, 3, 0, 490, 3, 330, 3335, 0, 3997, 4066, 3908, 10, 1679, 2934, 193, 0, 0, 196, 1424, 0, 38, 4093, 0, 240, 1043, 4096, 0, 0, 306, 0, 0, 3494, 34, 0, 0, 4, 3546, 3591, 3841, 7, 1, 4096, 119, 4096, 0, 3122, 3805, 5, 0, 1323, 3959] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 3950, 718, 0, 861, 0, 2968, 0, 0, 4083, 0, 0, 0, 0, 0, 235, 3903, 4094, 9, 3992, 1054, 675, 0, 0, 3962, 0, 0, 48, 1, 1, 26, 0, 4005, 4074, 0, 301, 3861, 890, 0, 0, 0, 3709, 0, 8, 36, 0, 0, 0, 4095, 0, 87, 20, 2736, 0, 0, 3423, 13, 3, 547, 4, 3357, 8, 0, 0, 1, 89, 2487, 0, 0, 63, 3278, 2, 4072, 8, 0, 3369, 2, 677, 0, 0, 3729, 0, 0, 1720, 0, 2463, 0, 0, 0, 0, 2022, 4048, 29, 3, 0, 0, 3274, 1102, 4050, 0, 0, 3758, 637, 0, 281, 26, 2821, 1688, 3626, 100, 1, 4096, 148, 3855, 0, 4095, 69, 16, 3517, 0, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 2, 4, 176, 0, 0, 4078, 0, 13, 2963, 0, 525, 4094, 0, 0, 0, 4082, 0, 32, 0, 394, 0, 2000, 61, 1989, 0, 3907, 40, 3967, 3781, 0, 3858, 3938, 44, 24, 0, 74, 0, 113, 65, 523, 0, 3158, 4008, 0, 4048, 0, 0, 3348, 1, 0, 4, 6, 0, 2231, 0, 0, 0, 2, 1376, 40, 0, 2198, 0, 3238, 126, 0, 3340, 1000, 0, 165, 7, 0, 12, 0, 0, 0, 3949, 38, 919, 0, 0, 71, 0, 1201, 1950, 0, 4096, 154, 3, 0, 0, 0, 0, 0, 1, 62, 3924, 0, 3834, 892, 29, 2068, 0, 0, 3750, 23, 4, 0, 0, 50, 0, 3435, 3943, 4014, 204, 3275, 0, 4092, 520, 3691, 52, 4054, 3016, 378, 1806, 0, 2519] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[2591, 97, 89, 42, 0, 1716, 3992, 195, 0, 406, 0, 3782, 4091, 4024, 3312, 0, 9, 0, 0, 0, 3923, 3779, 0, 2670, 121, 23, 1421, 0, 3470, 2726, 1121, 41, 3, 740, 0, 73, 0, 0, 810, 25, 1996, 0, 1, 2657, 0, 0, 1, 0, 3596, 0, 0, 0, 1, 0, 1182, 0, 55, 0, 3, 0, 2, 85, 1433, 3, 2, 197, 1, 0, 4096, 0, 0, 0, 0, 0, 16, 1, 0, 4065, 10, 1617, 0, 0, 1104, 4068, 3070, 433, 4093, 3459, 0, 3993, 22, 0, 0, 0, 1, 44, 11, 4095, 4066, 4073, 0, 100, 4027, 1010, 0, 3943, 3919, 44, 18, 0, 82, 0, 13, 2701, 3660, 0, 62, 0, 2976, 0, 4096, 0, 4096, 479, 1, 0, 1, 1000] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[2076, 0, 356, 338, 0, 0, 4096, 45, 0, 3501, 0, 4053, 3178, 0, 9, 0, 4060, 0, 0, 0, 0, 411, 7, 0, 4085, 2232, 277, 0, 4082, 393, 1554, 4096, 1070, 1125, 0, 4061, 0, 0, 0, 13, 356, 0, 0, 4096, 1, 3602, 3, 0, 7, 0, 54, 0, 0, 0, 3051, 54, 0, 0, 569, 0, 0, 0, 28, 0, 100, 230, 0, 44, 2, 0, 1, 9, 0, 0, 0, 4096, 0, 893, 3902, 1683, 0, 53, 152, 4095, 3465, 3433, 0, 4096, 0, 0, 323, 0, 0, 0, 3772, 0, 0, 667, 43, 72, 2263, 544, 3859, 3803, 0, 0, 0, 1039, 149, 0, 0, 11, 3846, 3940, 4034, 0, 0, 12, 3925, 3455, 4093, 1, 4096, 3911, 0, 0, 0, 21] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 186, 0, 4096, 22, 4096, 1, 0, 3975, 4087, 0, 0, 54, 145, 0, 74, 0, 0, 4096, 22, 0, 4095, 28, 45, 3938, 96, 63, 5, 3981, 2, 0, 0, 48, 0, 0, 3559, 0, 156, 0, 10, 4096, 77, 0, 31, 20, 0, 0, 731, 28, 125, 3910, 0, 4095, 78, 0, 0, 4, 0, 4047, 4, 26, 12, 1, 0, 1, 2, 0, 0, 48, 4, 0, 0, 125, 0, 4066, 3711, 3035, 0, 112, 0, 3381, 4056, 0, 1165, 197, 0, 2195, 0, 0, 0, 0, 3820, 4096, 4049, 4054, 0, 3377, 163, 0, 4026, 4071, 200, 2823, 0, 0, 421, 97, 0, 0, 4095, 4050, 3234, 526, 1, 0, 4096, 1035, 0, 0, 3818, 1, 578, 0, 246, 1932] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4068, 0, 186, 49, 0, 0, 343, 191, 4089, 4012, 0, 0, 2971, 1, 0, 0, 3002, 0, 222, 1, 22, 3153, 3554, 0, 4, 977, 105, 0, 4011, 0, 2809, 0, 0, 33, 0, 589, 0, 0, 897, 0, 937, 0, 2124, 0, 4091, 4055, 3, 0, 367, 0, 4038, 77, 9, 1538, 0, 0, 41, 0, 348, 0, 0, 3318, 2007, 19, 87, 0, 9, 0, 0, 32, 0, 0, 285, 62, 1, 0, 350, 2452, 0, 2578, 0, 9, 257, 4087, 0, 3688, 0, 3879, 4, 4025, 3548, 1952, 2165, 1435, 4093, 215, 13, 0, 624, 2196, 746, 15, 2763, 0, 0, 72, 3983, 67, 4096, 0, 14, 0, 2338, 1045, 1, 3979, 0, 0, 22, 3517, 4046, 3, 3418, 8, 0, 4001, 631, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 3638, 3955, 1806, 4091, 4096, 4096, 3696, 26, 4096, 4000, 3213, 136, 0, 0, 813, 32, 0, 0, 2296, 20, 0, 3533, 0, 4096, 5, 1, 0, 0, 0, 4050, 0, 1651, 465, 0, 176, 3468, 0, 623, 0, 3635, 4095, 0, 7, 0, 0, 267, 1, 0, 0, 0, 5, 45, 689, 0, 0, 0, 0, 4, 0, 1614, 0, 0, 5, 0, 0, 13, 1, 695, 0, 24, 126, 135, 0, 0, 4096, 0, 351, 41, 32, 82, 0, 185, 3306, 0, 4096, 0, 0, 292, 0, 0, 0, 724, 4, 23, 4040, 4, 4029, 0, 3709, 3908, 0, 0, 2791, 128, 470, 0, 14, 318, 1, 2774, 3752, 3867, 26, 4096, 0, 3312, 421, 4067, 1, 3307, 1605, 308, 0, 64, 1389] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[405, 1401, 437, 3, 0, 92, 613, 1, 824, 53, 0, 207, 2497, 1569, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 4007, 907, 0, 0, 2383, 3384, 4064, 0, 60, 371, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 0, 0, 0, 0, 1510, 2806, 4093, 1784, 0, 0, 4088, 0, 2719, 9, 0, 1579, 1103, 694, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2903, 12, 0, 0, 0, 4051, 126, 41, 0, 1, 0, 0, 0, 1577, 0, 353, 3122, 4096, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 0, 75, 0, 0, 267, 2634, 0, 54, 99, 3998, 3, 0, 3609, 3956, 1662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 3, 330, 3335, 0, 3997, 4066, 2805, 10, 1679, 2934, 193, 0, 0, 196, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4093, 0, 240, 1043, 2234, 0, 0, 306, 0, 0, 3494, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2852, 3591, 3841, 7, 1, 0, 119, 0, 0, 0, 3805, 5, 0, 1323, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8140, 15270, 21627, 16769, 16815, 21462, 11482, 19507] avg=16384.0000 max=21627 max_rank=2 min=8140 min_rank=0 imbalance_ratio=1.320007 max_load_reduction=109445 max_load_reduction_pct=0.834999 moved_to_echo=69266 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4, 176, 0, 0, 4078, 0, 13, 2963, 0, 525, 2508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 32, 0, 394, 0, 2000, 61, 1989, 0, 3907, 40, 59, 3781, 0, 3858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3938, 44, 24, 0, 74, 0, 113, 65, 523, 0, 3158, 4008, 0, 4048, 0, 0, 1586, 1156, 772, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3348, 1, 0, 4, 6, 0, 2231, 0, 0, 0, 2, 1376, 40, 0, 2198, 0, 4054, 3691, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3238, 126, 0, 3340, 1000, 0, 165, 7, 0, 12, 0, 0, 0, 3949, 38, 919, 3606, 3908, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 1201, 1950, 0, 2940, 154, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 3924, 0, 3834, 892, 29, 1128, 0, 0, 3750, 23, 4, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3435, 3171, 4014, 204, 3275, 0, 0, 520, 0, 52, 0, 3016, 378, 1806, 0, 2519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10269, 20203, 19610, 17437, 21147, 6320, 13696, 22390] avg=16384.0000 max=22390 max_rank=7 min=6320 min_rank=5 imbalance_ratio=1.366577 max_load_reduction=108682 max_load_reduction_pct=0.829178 moved_to_echo=63553 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 3950, 718, 0, 861, 0, 2968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 235, 3903, 4094, 9, 3992, 1054, 10, 0, 0, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1, 1, 26, 0, 4005, 4074, 0, 301, 3861, 890, 0, 0, 0, 0, 1052, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3709, 0, 8, 36, 0, 0, 0, 4095, 0, 87, 20, 2736, 0, 0, 3423, 13, 4095, 3855, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 547, 4, 3357, 8, 0, 0, 1, 89, 2487, 0, 0, 63, 3278, 2, 4072, 3609, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3369, 2, 677, 0, 0, 2677, 0, 0, 1720, 0, 2463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2022, 4048, 29, 3, 0, 0, 3274, 1102, 4050, 0, 0, 3758, 637, 0, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 2269, 1688, 3626, 100, 1, 0, 148, 0, 0, 0, 69, 16, 3517, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8498, 21342, 14811, 22564, 18185, 10916, 19204, 15552] avg=16384.0000 max=22564 max_rank=3 min=8498 min_rank=0 imbalance_ratio=1.377197 max_load_reduction=108508 max_load_reduction_pct=0.827850 moved_to_echo=63857 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2076, 0, 356, 338, 0, 0, 4096, 45, 0, 3501, 0, 4053, 1944, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 411, 7, 0, 4085, 2232, 277, 0, 57, 393, 1554, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 1125, 0, 4061, 0, 0, 0, 13, 356, 0, 0, 4096, 1, 3602, 3, 0, 1234, 1161, 777, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 54, 0, 0, 0, 3051, 54, 0, 0, 569, 0, 0, 0, 28, 0, 4096, 4093, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 230, 0, 44, 2, 0, 1, 9, 0, 0, 0, 4096, 0, 893, 3902, 1683, 3457, 4025, 1568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 152, 4095, 3465, 3433, 0, 2935, 0, 0, 323, 0, 0, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, 43, 72, 2263, 544, 2099, 3803, 0, 0, 0, 1039, 149, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 3163, 4034, 0, 0, 12, 0, 3455, 0, 1, 0, 3911, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16418, 17172, 17691, 12420, 20010, 18228, 10690, 18443] avg=16384.0000 max=20010 max_rank=4 min=10690 min_rank=6 imbalance_ratio=1.221313 max_load_reduction=111062 max_load_reduction_pct=0.847336 moved_to_echo=67371 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4068, 0, 186, 49, 0, 0, 343, 191, 4089, 4012, 0, 0, 1821, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3002, 0, 222, 1, 22, 3153, 3554, 0, 4, 977, 105, 0, 60, 0, 2809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 589, 0, 0, 897, 0, 937, 0, 2124, 0, 4091, 4055, 3, 0, 1150, 1095, 204, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 4038, 77, 9, 1538, 0, 0, 41, 0, 348, 0, 0, 3318, 2007, 19, 3418, 4046, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 9, 0, 0, 32, 0, 0, 285, 62, 1, 0, 350, 2452, 0, 2578, 20, 3951, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 257, 4087, 0, 3688, 0, 2784, 4, 4025, 3548, 1952, 2165, 1435, 4093, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 624, 2196, 746, 15, 1507, 0, 0, 72, 3983, 67, 4096, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2338, 841, 1, 3979, 0, 0, 0, 3517, 0, 3, 0, 8, 0, 4001, 631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14760, 13909, 15313, 19228, 10948, 28262, 13333, 15319] avg=16384.0000 max=28262 max_rank=5 min=10948 min_rank=4 imbalance_ratio=1.724976 max_load_reduction=102810 max_load_reduction_pct=0.784378 moved_to_echo=67862 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2591, 97, 89, 42, 0, 1716, 3992, 195, 0, 406, 0, 3782, 2507, 4024, 3312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3923, 3779, 0, 2670, 121, 23, 1421, 0, 52, 2726, 1121, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 740, 0, 73, 0, 0, 810, 25, 1996, 0, 1, 2657, 0, 0, 1, 0, 1584, 976, 529, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3596, 0, 0, 0, 1, 0, 1182, 0, 55, 0, 3, 0, 2, 85, 1433, 3, 4096, 4096, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 197, 1, 0, 4096, 0, 0, 0, 0, 0, 16, 1, 0, 4065, 10, 1617, 2622, 3418, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1104, 4068, 3070, 433, 4093, 2483, 0, 3993, 22, 0, 0, 0, 1, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4095, 4066, 4073, 0, 100, 2196, 1010, 0, 3943, 3919, 44, 18, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2172, 3660, 0, 62, 0, 0, 0, 0, 0, 0, 479, 1, 0, 1, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22753, 15886, 9592, 14906, 17679, 19311, 23557, 7388] avg=16384.0000 max=23557 max_rank=6 min=7388 min_rank=7 imbalance_ratio=1.437805 max_load_reduction=107515 max_load_reduction_pct=0.820274 moved_to_echo=67935 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 186, 0, 4096, 22, 4096, 1, 0, 3975, 2504, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 74, 0, 0, 4096, 22, 0, 4095, 28, 45, 3938, 2, 63, 5, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 48, 0, 0, 3559, 0, 156, 0, 10, 4096, 77, 0, 31, 20, 1583, 55, 793, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 731, 28, 125, 3910, 0, 4095, 78, 0, 0, 4, 0, 4047, 4, 26, 3818, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 1, 2, 0, 0, 48, 4, 0, 0, 125, 0, 4066, 3711, 3035, 3608, 94, 1633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 3381, 4056, 0, 1165, 142, 0, 2195, 0, 0, 0, 0, 3820, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 4054, 0, 3377, 163, 0, 2196, 4071, 200, 2823, 0, 0, 421, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3257, 3234, 526, 1, 0, 0, 1035, 0, 0, 0, 1, 578, 0, 246, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14934, 16494, 10627, 17354, 16340, 18967, 21451, 14905] avg=16384.0000 max=21451 max_rank=6 min=10627 min_rank=2 imbalance_ratio=1.309265 max_load_reduction=109621 max_load_reduction_pct=0.836342 moved_to_echo=71663 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3638, 3955, 1806, 4091, 4096, 4096, 3696, 26, 4096, 2451, 3213, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 813, 32, 0, 0, 2296, 20, 0, 3533, 0, 4096, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 1651, 465, 0, 176, 3468, 0, 623, 0, 3635, 4095, 0, 7, 0, 0, 1549, 1156, 735, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 1, 0, 0, 0, 5, 45, 689, 0, 0, 0, 0, 4, 0, 1614, 0, 3307, 4067, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 13, 1, 695, 0, 24, 126, 135, 0, 0, 4096, 0, 351, 2917, 0, 1585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 32, 82, 0, 185, 3306, 0, 2940, 0, 0, 292, 0, 0, 0, 724, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4040, 4, 4029, 0, 3709, 2132, 0, 0, 2791, 128, 470, 0, 14, 318, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2774, 3017, 3867, 26, 4096, 0, 0, 421, 0, 1, 0, 1605, 308, 0, 64, 1389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[35300, 10796, 21801, 10394, 9948, 7606, 17659, 17568] avg=16384.0000 max=35300 max_rank=0 min=7606 min_rank=5 imbalance_ratio=2.154541 max_load_reduction=95772 max_load_reduction_pct=0.730682 moved_to_echo=52781 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[640, 256, 5376, 19072, 12672, 16896, 9344, 19712, 256, 11904, 8960, 2176, 4736, 1536, 12672, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[16640, 20864, 24192, 12288, 7552, 128, 0, 9216, 0, 128, 0, 12928, 1408, 9344, 2304, 14976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[3456, 1152, 128, 7296, 5248, 128, 896, 128, 768, 5376, 256, 4352, 512, 26880, 7680, 14336, 23552, 20096, 10112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[15488, 896, 384, 128, 8448, 14720, 5888, 6656, 20352, 6656, 17920, 5120, 384, 7424, 5504, 16128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[9216, 1536, 1152, 4352, 4224, 3712, 21376, 4608, 17152, 15360, 128, 17536, 16256, 11776, 3584, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[14208, 128, 4864, 256, 256, 9600, 6656, 9088, 256, 128, 1024, 4224, 128, 9088, 10752, 512, 30080, 28032, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[4224, 18944, 12928, 17664, 4352, 5504, 13568, 12160, 1408, 17792, 8064, 1664, 12032, 896, 512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[10112, 2048, 1792, 5248, 128, 1792, 11776, 8320, 10496, 0, 9344, 27008, 5120, 14464, 128, 128, 10368, 7808, 5120, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4014, 0, 0, 3395, 0, 181, 0, 1, 1171, 186, 96, 0, 0, 0, 0, 3395, 2240, 1, 4032, 0, 4096, 4093, 3595, 3980, 0, 4096, 18, 828, 3981, 0, 0, 0, 788, 4096, 0, 39, 0, 3980, 0, 0, 7, 0, 848, 3699, 0, 0, 1043, 204, 2755, 4096, 501, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 4016, 0, 0, 0, 0, 55, 0, 4094, 0, 0, 0, 0, 17, 4091, 0, 84, 213, 58, 3305, 0, 4, 3364, 0, 0, 0, 3873, 30, 4089, 0, 0, 0, 3938, 0, 3696, 0, 0, 0, 0, 87, 12, 0, 2, 3945, 0, 0, 1, 0, 0, 243, 0, 0, 4096, 0, 0, 0, 4025, 0, 2, 16, 343, 1958, 0, 4090, 17, 0, 0, 3660, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4088, 0, 0, 3975, 456, 4019, 0, 2065, 86, 3981, 25, 1269, 177, 0, 0, 59, 5, 1, 1784, 0, 523, 0, 0, 486, 0, 4096, 107, 0, 4092, 138, 17, 11, 4067, 4090, 1, 212, 4, 4090, 2744, 0, 0, 3712, 20, 0, 7, 4088, 54, 1, 0, 0, 0, 0, 0, 1157, 0, 4049, 80, 21, 0, 743, 0, 195, 0, 0, 0, 2677, 0, 30, 204, 0, 3983, 3, 0, 3933, 0, 0, 0, 0, 1239, 1927, 0, 4096, 0, 0, 3943, 0, 126, 4096, 14, 33, 0, 3829, 576, 3315, 3842, 0, 3569, 0, 0, 20, 0, 0, 4094, 186, 0, 0, 547, 65, 0, 1, 89, 0, 3637, 0, 2, 4096, 2, 0, 2946, 1, 3713, 0, 459, 0, 3772, 1301, 3911, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 313, 0, 4037, 0, 31, 0, 0, 0, 4093, 0, 0, 4086, 82, 0, 1655, 935, 11, 0, 0, 4096, 521, 0, 0, 0, 1, 38, 0, 4096, 0, 0, 28, 1, 3, 96, 9, 0, 4096, 3932, 46, 98, 146, 0, 3922, 0, 4095, 0, 0, 297, 1606, 0, 0, 0, 0, 4096, 7, 1, 4096, 4, 0, 3769, 0, 42, 0, 4096, 16, 0, 3158, 0, 71, 0, 0, 10, 4040, 4, 0, 3894, 52, 3757, 39, 14, 0, 0, 1138, 4074, 3333, 4096, 12, 238, 0, 154, 4029, 91, 0, 4004, 0, 0, 0, 0, 25, 0, 0, 3989, 0, 11, 0, 2784, 0, 2728, 3, 0, 1, 4067, 99, 0, 4029, 4095, 1, 0, 8, 4096, 68, 4020, 0, 0, 2247, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4080, 3900, 3789, 0, 0, 231, 3741, 0, 3706, 38, 57, 4096, 0, 0, 4096, 0, 231, 0, 295, 4010, 4092, 0, 92, 36, 4041, 22, 5, 4057, 0, 98, 16, 0, 0, 0, 3475, 392, 4095, 0, 0, 0, 0, 0, 0, 4087, 4090, 29, 0, 1971, 0, 0, 0, 0, 0, 0, 0, 541, 4096, 3804, 258, 4096, 0, 472, 0, 2650, 0, 0, 111, 0, 0, 0, 2, 0, 3824, 0, 29, 4092, 0, 4090, 0, 0, 1, 0, 3, 4095, 3002, 51, 18, 0, 14, 0, 4046, 0, 2294, 3553, 0, 0, 0, 74, 21, 0, 0, 4066, 0, 0, 0, 241, 0, 3551, 141, 0, 9, 1655, 1, 86, 4072, 139, 0, 36, 0, 49, 4078, 0, 0, 970, 0, 3873, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[23, 0, 0, 30, 2, 0, 1, 0, 0, 3889, 0, 923, 4082, 418, 0, 0, 0, 0, 4074, 0, 0, 4057, 3991, 3218, 48, 0, 64, 307, 4096, 62, 0, 3172, 521, 3868, 0, 0, 22, 4090, 834, 1048, 0, 4086, 185, 4060, 2, 3032, 1000, 1, 1883, 3563, 38, 0, 0, 0, 0, 0, 72, 0, 430, 0, 4092, 0, 0, 153, 4072, 0, 0, 1623, 0, 0, 711, 0, 0, 1914, 0, 0, 2479, 0, 3853, 897, 0, 8, 0, 0, 27, 4092, 608, 4090, 2, 0, 0, 4096, 676, 6, 0, 0, 0, 0, 753, 1985, 11, 0, 4055, 0, 0, 13, 3943, 5, 3719, 7, 0, 171, 0, 4096, 0, 3475, 4096, 0, 29, 319, 0, 0, 890, 4084, 11, 1946, 2868, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4074, 0, 6, 126, 2, 0, 176, 26, 978, 3608, 0, 4096, 0, 282, 1088, 3084, 97, 4017, 0, 4047, 3777, 3274, 3229, 0, 274, 39, 0, 0, 4085, 1732, 0, 0, 2786, 4053, 15, 0, 0, 166, 4009, 415, 61, 0, 2674, 4075, 7, 28, 0, 0, 407, 0, 0, 0, 94, 4096, 25, 0, 0, 799, 0, 0, 3773, 107, 5, 0, 5, 0, 69, 3, 0, 4022, 4020, 3790, 0, 2838, 11, 0, 5, 0, 4016, 3, 1, 4073, 3306, 0, 0, 3358, 262, 3156, 1072, 10, 0, 4096, 245, 10, 1894, 0, 21, 56, 56, 0, 0, 0, 3187, 3, 271, 594, 0, 3868, 4, 26, 5, 0, 0, 0, 2, 2547, 4091, 0, 4, 0, 570, 0, 0, 41, 0, 0, 3875, 1874] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 6, 0, 3840, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 4, 3836, 0, 1512, 0, 4096, 3450, 0, 859, 4076, 0, 20, 0, 44, 10, 0, 3547, 0, 3836, 4009, 1, 286, 940, 0, 3167, 0, 0, 2328, 3, 0, 3987, 0, 0, 35, 162, 15, 0, 0, 1118, 11, 0, 4092, 0, 4082, 0, 2818, 4, 1905, 617, 0, 0, 4089, 99, 0, 4096, 2, 3, 3601, 126, 4096, 0, 0, 4082, 2785, 0, 3981, 4087, 164, 0, 26, 0, 0, 4095, 17, 2962, 4068, 93, 982, 0, 88, 0, 0, 11, 3761, 0, 1, 0, 0, 4088, 0, 0, 0, 0, 312, 0, 0, 183, 0, 5, 4045, 64, 4096, 0, 4095, 0, 0, 4093, 4026, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4093, 3959, 0, 13, 2888, 0, 0, 1264, 50, 17, 0, 0, 0, 1431, 1, 0, 512, 3292, 32, 2232, 3026, 121, 400, 3351, 0, 4088, 4095, 5, 0, 6, 133, 90, 34, 125, 4060, 4048, 0, 0, 0, 205, 1, 0, 0, 4096, 69, 5, 3909, 0, 0, 116, 0, 0, 3, 3909, 2, 3703, 0, 3669, 3979, 1, 1, 39, 4089, 0, 0, 72, 0, 302, 0, 0, 1, 4054, 8, 374, 4063, 0, 3722, 1, 0, 4086, 1137, 0, 0, 31, 3, 394, 0, 8, 0, 1695, 34, 3996, 0, 0, 20, 26, 0, 4087, 0, 0, 3769, 0, 0, 0, 136, 135, 162, 36, 0, 0, 4092, 4030, 204, 4061, 4096, 119, 0, 598, 0, 0, 4071, 924, 886, 363, 4094, 20] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4088, 0, 0, 3975, 456, 4019, 0, 2065, 86, 3981, 25, 1269, 177, 0, 0, 59, 1308, 241, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 1784, 0, 523, 0, 0, 486, 0, 4096, 107, 0, 1010, 138, 17, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4090, 1, 212, 4, 2448, 2744, 0, 0, 3712, 20, 0, 7, 4088, 54, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1157, 0, 4049, 80, 21, 0, 743, 0, 195, 0, 0, 2749, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2677, 0, 30, 204, 0, 3983, 3, 0, 3763, 0, 0, 0, 0, 1239, 1927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3943, 0, 126, 4096, 14, 33, 0, 892, 576, 3315, 3842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3569, 0, 0, 20, 0, 0, 4094, 186, 0, 0, 547, 65, 0, 1, 89, 0, 3911, 3082, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3637, 0, 2, 3855, 2, 0, 2946, 1, 3713, 0, 459, 0, 3772, 1301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21919, 8178, 21448, 9328, 13826, 20933, 15752, 19688] avg=16384.0000 max=21919 max_rank=0 min=8178 min_rank=1 imbalance_ratio=1.337830 max_load_reduction=109153 max_load_reduction_pct=0.832771 moved_to_echo=70199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4014, 0, 0, 3395, 0, 181, 0, 1, 1171, 186, 96, 0, 0, 0, 0, 3395, 1272, 237, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2240, 1, 4032, 0, 4096, 4093, 3595, 3980, 0, 4096, 18, 828, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 788, 4096, 0, 39, 0, 2383, 0, 0, 7, 0, 848, 3699, 0, 0, 1043, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2755, 4096, 501, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 4016, 0, 0, 0, 2827, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 4094, 0, 0, 0, 0, 17, 3914, 0, 84, 213, 58, 3305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3364, 0, 0, 0, 3873, 30, 4089, 0, 0, 0, 917, 0, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 12, 0, 2, 3945, 0, 0, 1, 0, 0, 243, 0, 0, 4096, 3660, 2999, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3788, 0, 2, 16, 343, 1958, 0, 4090, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14125, 27961, 13107, 22713, 11740, 15973, 15239, 10214] avg=16384.0000 max=27961 max_rank=1 min=10214 min_rank=7 imbalance_ratio=1.706604 max_load_reduction=103111 max_load_reduction_pct=0.786674 moved_to_echo=53166 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 3959, 0, 13, 2888, 0, 0, 1264, 50, 17, 0, 0, 0, 1431, 1294, 239, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 512, 3292, 32, 2232, 3026, 121, 400, 3351, 0, 4088, 1010, 5, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 90, 34, 125, 4060, 2423, 0, 0, 0, 205, 1, 0, 0, 4096, 69, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 0, 0, 116, 0, 0, 3, 3909, 2, 3703, 0, 3669, 3979, 1, 1, 39, 1217, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 72, 0, 302, 0, 0, 1, 3879, 8, 374, 4063, 0, 3722, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1137, 0, 0, 31, 3, 394, 0, 8, 0, 395, 34, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 26, 0, 4087, 0, 0, 3769, 0, 0, 0, 136, 135, 162, 36, 0, 0, 4094, 3085, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4030, 204, 3822, 4096, 119, 0, 598, 0, 0, 4071, 924, 886, 363, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15423, 18076, 11241, 20879, 16511, 10084, 15633, 23225] avg=16384.0000 max=23225 max_rank=7 min=10084 min_rank=5 imbalance_ratio=1.417542 max_load_reduction=107847 max_load_reduction_pct=0.822807 moved_to_echo=65453 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4074, 0, 6, 126, 2, 0, 176, 26, 978, 3608, 0, 4096, 0, 282, 1088, 3084, 53, 150, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 4017, 0, 4047, 3777, 3274, 3229, 0, 274, 39, 0, 0, 1008, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2786, 4053, 15, 0, 0, 100, 4009, 415, 61, 0, 2674, 4075, 7, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 0, 0, 94, 4096, 25, 0, 0, 799, 0, 0, 3773, 107, 5, 0, 2940, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 69, 3, 0, 4022, 4020, 3790, 0, 2715, 11, 0, 5, 0, 4016, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4073, 3306, 0, 0, 3358, 262, 3156, 1072, 10, 0, 955, 245, 10, 1894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 56, 56, 0, 0, 0, 3187, 3, 271, 594, 0, 3868, 4, 26, 5, 0, 3875, 3077, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2397, 4091, 0, 4, 0, 570, 0, 0, 41, 0, 0, 0, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17872, 21494, 18223, 12259, 18659, 18342, 15244, 8979] avg=16384.0000 max=21494 max_rank=1 min=8979 min_rank=7 imbalance_ratio=1.311890 max_load_reduction=109578 max_load_reduction_pct=0.836014 moved_to_echo=61224 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4080, 3900, 3789, 0, 0, 231, 3741, 0, 3706, 38, 57, 4096, 0, 0, 4096, 1313, 243, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 295, 4010, 4092, 0, 92, 36, 4041, 22, 5, 997, 0, 98, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3475, 392, 2442, 0, 0, 0, 0, 0, 0, 4087, 4090, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1971, 0, 0, 0, 0, 0, 0, 0, 541, 4096, 3804, 258, 4096, 0, 472, 0, 2909, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2650, 0, 0, 111, 0, 0, 0, 2, 0, 3655, 0, 29, 4092, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 4095, 3002, 51, 18, 0, 14, 0, 933, 0, 2294, 3553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 21, 0, 0, 4066, 0, 0, 0, 241, 0, 3551, 141, 0, 9, 3873, 3060, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1655, 1, 86, 3829, 139, 0, 36, 0, 49, 4078, 0, 0, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[29459, 13935, 14515, 18487, 14629, 13964, 15240, 10843] avg=16384.0000 max=29459 max_rank=0 min=10843 min_rank=7 imbalance_ratio=1.798035 max_load_reduction=101613 max_load_reduction_pct=0.775246 moved_to_echo=54676 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 313, 0, 4037, 0, 31, 0, 0, 0, 4093, 0, 0, 4086, 82, 0, 1655, 1309, 237, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 935, 11, 0, 0, 4096, 521, 0, 0, 0, 1, 38, 0, 1011, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 96, 9, 0, 2452, 3932, 46, 98, 146, 0, 3922, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 1606, 0, 0, 0, 0, 4096, 7, 1, 4096, 4, 0, 3769, 0, 42, 0, 2892, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 3158, 0, 71, 0, 0, 10, 3865, 4, 0, 3894, 52, 3757, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1138, 4074, 3333, 4096, 12, 238, 0, 154, 939, 91, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 3989, 0, 11, 0, 2784, 0, 2728, 3, 0, 1, 4096, 3085, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 99, 0, 3792, 4095, 1, 0, 8, 4096, 68, 4020, 0, 0, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16018, 6641, 14800, 17145, 18962, 18093, 16920, 22493] avg=16384.0000 max=22493 max_rank=7 min=6641 min_rank=1 imbalance_ratio=1.372864 max_load_reduction=108579 max_load_reduction_pct=0.828392 moved_to_echo=76468 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 0, 0, 6, 0, 3840, 0, 0, 0, 0, 0, 0, 30, 0, 1226, 10, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3836, 0, 1512, 0, 4096, 3450, 0, 859, 1006, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 10, 0, 3547, 0, 2296, 4009, 1, 286, 940, 0, 3167, 0, 0, 2328, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 0, 35, 162, 15, 0, 0, 1118, 11, 0, 4092, 0, 4082, 0, 2940, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2818, 4, 1905, 617, 0, 0, 4089, 99, 0, 3919, 2, 3, 3601, 126, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 2785, 0, 3981, 4087, 164, 0, 26, 0, 0, 954, 17, 2962, 4068, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 982, 0, 88, 0, 0, 11, 3761, 0, 1, 0, 0, 4088, 0, 0, 0, 0, 4026, 3070, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 0, 0, 173, 0, 5, 4045, 64, 4096, 0, 4095, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5293, 14783, 16631, 16756, 21279, 23219, 16228, 16883] avg=16384.0000 max=23219 max_rank=5 min=5293 min_rank=0 imbalance_ratio=1.417175 max_load_reduction=107853 max_load_reduction_pct=0.822853 moved_to_echo=77609 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[23, 0, 0, 30, 2, 0, 1, 0, 0, 3889, 0, 923, 4082, 418, 0, 0, 1308, 204, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 4057, 3991, 3218, 48, 0, 64, 307, 1011, 62, 0, 3172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 3868, 0, 0, 22, 2448, 834, 1048, 0, 4086, 185, 4060, 2, 3032, 1000, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1883, 3563, 38, 0, 0, 0, 0, 0, 72, 0, 430, 0, 4092, 0, 0, 153, 2940, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 1623, 0, 0, 711, 0, 0, 1831, 0, 0, 2479, 0, 3853, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 27, 4092, 608, 4090, 2, 0, 0, 955, 676, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 753, 1985, 11, 0, 4055, 0, 0, 13, 3943, 5, 3719, 7, 0, 171, 2868, 3085, 201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3271, 4096, 0, 29, 319, 0, 0, 890, 4084, 11, 1946, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10963, 20004, 21107, 13505, 15466, 10464, 20816, 18747] avg=16384.0000 max=21107 max_rank=2 min=10464 min_rank=5 imbalance_ratio=1.288269 max_load_reduction=109965 max_load_reduction_pct=0.838966 moved_to_echo=65493 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[4608, 128, 1152, 6272, 128, 128, 30976, 256, 384, 640, 7680, 8192, 10496, 256, 128, 4352, 30464, 24576, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[11264, 13312, 640, 128, 256, 5504, 4224, 12160, 768, 18048, 4352, 4736, 27904, 384, 4608, 256, 21504, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[8448, 16256, 256, 7424, 4480, 17024, 15616, 1536, 512, 9216, 3840, 18944, 4224, 19456, 4608, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[13824, 8320, 384, 24960, 16640, 128, 7168, 1408, 14592, 4224, 17664, 5120, 5760, 9984, 0, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 19712, 7296, 1152, 16128, 21888, 5376, 15872, 1408, 128, 256, 7040, 1664, 16384, 17408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[3328, 4352, 10496, 7680, 20480, 18304, 15360, 7936, 4864, 19200, 256, 6144, 8064, 2048, 256, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[12288, 4480, 8064, 19328, 512, 4352, 3328, 9728, 2304, 20736, 256, 6400, 12544, 896, 1152, 13824, 9088, 1664, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[17792, 2816, 2048, 9728, 256, 4480, 12928, 3968, 128, 27648, 128, 512, 18432, 256, 28160, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4083, 133, 0, 9, 0, 0, 11, 155, 1023, 0, 17, 50, 4079, 19, 0, 0, 22, 0, 0, 1400, 0, 0, 0, 0, 0, 1121, 0, 0, 0, 3122, 0, 0, 22, 0, 3807, 2822, 89, 492, 2759, 0, 75, 3, 4085, 0, 224, 0, 36, 48, 0, 4095, 2775, 4096, 0, 4096, 10, 109, 4095, 0, 61, 0, 1467, 4096, 0, 247, 0, 4032, 1298, 0, 3941, 0, 0, 2675, 1054, 3, 4016, 630, 0, 0, 0, 4086, 349, 189, 1586, 0, 8, 0, 0, 4095, 0, 0, 279, 4094, 13, 55, 0, 0, 0, 1480, 3843, 0, 3910, 11, 4056, 0, 2321, 4072, 0, 1248, 0, 0, 0, 2627, 76, 0, 4092, 0, 3220, 4057, 0, 4096, 248, 4095, 3, 0, 361, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[3, 711, 4096, 3684, 11, 1, 2065, 0, 0, 0, 10, 4095, 0, 0, 357, 0, 35, 13, 199, 0, 1, 662, 3733, 2, 125, 0, 0, 0, 0, 8, 2985, 535, 0, 2398, 0, 4074, 0, 3561, 0, 4089, 0, 0, 68, 641, 2966, 0, 0, 8, 0, 273, 3896, 0, 3687, 0, 0, 0, 0, 4091, 1543, 0, 0, 16, 162, 0, 1374, 0, 4071, 4037, 4056, 191, 12, 0, 3741, 0, 3974, 0, 0, 3891, 3575, 0, 4074, 4, 31, 530, 3259, 407, 878, 3996, 85, 0, 0, 17, 2582, 743, 0, 0, 85, 430, 0, 3331, 27, 4077, 4022, 3011, 1097, 0, 2824, 0, 0, 59, 0, 67, 35, 22, 0, 1159, 13, 0, 3854, 0, 3229, 0, 19, 0, 2032, 1328, 0, 19] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[78, 3126, 4063, 888, 4087, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 160, 0, 1, 4, 0, 13, 0, 0, 0, 1, 0, 1, 0, 0, 14, 0, 357, 1224, 45, 0, 0, 4096, 0, 0, 703, 332, 1787, 0, 0, 0, 4083, 2270, 0, 0, 25, 1114, 671, 0, 4096, 3969, 1, 3063, 0, 2978, 0, 1032, 936, 0, 3922, 0, 2263, 186, 3029, 152, 5, 4091, 0, 0, 3992, 0, 4079, 3297, 3, 157, 0, 0, 3673, 2805, 0, 62, 336, 4096, 0, 3733, 3581, 52, 2206, 145, 2523, 0, 248, 26, 0, 2, 4041, 2086, 0, 4096, 0, 3821, 0, 1, 0, 0, 5, 187, 3201, 0, 470, 0, 0, 0, 0, 4091, 3881, 620, 3902, 0, 0, 1661, 0, 1030, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4093, 0, 469, 3883, 0, 0, 0, 1581, 9, 0, 4095, 19, 0, 36, 0, 0, 0, 0, 43, 0, 45, 139, 0, 0, 379, 605, 4091, 0, 19, 3993, 0, 1646, 3500, 0, 4095, 3812, 0, 2, 4095, 0, 0, 1, 5, 104, 133, 314, 103, 0, 0, 4096, 0, 0, 143, 1, 5, 3, 4078, 0, 48, 5, 0, 2900, 237, 3130, 2928, 44, 153, 3780, 4058, 0, 0, 4092, 0, 0, 2, 0, 2806, 2580, 0, 1704, 3864, 4, 0, 29, 3849, 0, 178, 3603, 6, 0, 0, 3968, 0, 696, 0, 28, 0, 0, 0, 3718, 3966, 2001, 4095, 0, 0, 3367, 190, 0, 0, 0, 24, 173, 2, 0, 3470, 20, 0, 3492, 0, 3348, 3984, 3929, 0, 15, 0, 0, 951] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 2454, 1, 0, 472, 0, 0, 0, 0, 116, 0, 4096, 4006, 0, 1, 31, 0, 0, 4, 0, 0, 6, 2894, 0, 0, 0, 1412, 1921, 2270, 460, 3957, 0, 14, 0, 0, 4007, 0, 1080, 679, 14, 63, 0, 0, 5, 0, 4041, 4044, 0, 0, 48, 2197, 0, 4090, 85, 0, 978, 194, 4027, 4096, 115, 4, 3289, 1928, 492, 0, 575, 10, 4096, 148, 0, 1862, 251, 3467, 4091, 4091, 1083, 160, 0, 0, 0, 0, 0, 0, 3660, 3865, 4067, 0, 0, 0, 1, 0, 3536, 4074, 0, 1233, 0, 33, 49, 352, 0, 2145, 11, 791, 3783, 1, 0, 592, 0, 3867, 1356, 69, 9, 4090, 0, 0, 0, 33, 2680, 3285, 0, 3852, 3507, 0, 0, 0, 0, 167, 539] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[49, 0, 4093, 41, 1, 0, 0, 0, 4096, 3120, 13, 0, 3, 301, 1, 0, 0, 0, 0, 0, 2692, 15, 4084, 6, 0, 1805, 0, 0, 15, 1, 1373, 0, 22, 0, 1, 677, 2749, 4013, 3823, 0, 0, 25, 322, 1544, 181, 9, 259, 5, 283, 1, 4096, 0, 0, 7, 0, 0, 0, 4096, 2, 0, 0, 2771, 0, 30, 4096, 0, 4096, 4095, 2, 0, 17, 0, 4096, 2720, 2781, 196, 0, 0, 0, 0, 13, 0, 0, 0, 4094, 1, 78, 4094, 2789, 0, 0, 0, 0, 1137, 0, 0, 0, 3715, 388, 3317, 4096, 3952, 4073, 4096, 135, 4095, 3306, 31, 0, 0, 0, 0, 95, 0, 0, 3656, 377, 113, 168, 0, 3949, 2390, 0, 4096, 3938, 0, 0, 156] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 3, 4051, 0, 4096, 0, 159, 0, 2052, 0, 0, 4096, 0, 0, 0, 0, 0, 79, 0, 0, 429, 0, 3986, 0, 0, 3563, 317, 4094, 0, 41, 3825, 30, 8, 0, 0, 4084, 0, 99, 0, 40, 4096, 188, 3934, 0, 339, 0, 4064, 0, 0, 0, 3921, 14, 4, 150, 142, 872, 0, 4088, 0, 178, 0, 3937, 3150, 0, 4078, 0, 0, 1656, 0, 2773, 17, 0, 3621, 3, 696, 0, 58, 4096, 0, 0, 4092, 123, 0, 0, 23, 35, 0, 0, 3914, 402, 0, 0, 2914, 30, 0, 0, 11, 772, 2301, 4090, 4074, 4040, 0, 4081, 0, 0, 3252, 0, 4035, 0, 45, 0, 857, 1, 0, 649, 0, 1511, 3439, 0, 3643, 5, 6, 285, 33, 1278, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 25, 37, 1355, 108, 0, 347, 4096, 0, 288, 0, 12, 0, 0, 0, 3, 0, 643, 0, 93, 0, 1, 0, 0, 4096, 0, 4080, 0, 0, 246, 1675, 3864, 0, 0, 0, 0, 2440, 0, 488, 23, 10, 11, 0, 0, 1109, 3, 3420, 0, 0, 0, 2593, 0, 1209, 3961, 0, 0, 925, 0, 4011, 0, 4023, 2799, 0, 0, 4021, 9, 0, 1, 3268, 0, 4096, 116, 731, 14, 4066, 0, 4067, 4096, 0, 53, 4080, 0, 4067, 0, 220, 0, 59, 3994, 2630, 0, 0, 0, 0, 10, 0, 4007, 0, 0, 4096, 4094, 4096, 0, 4064, 0, 0, 0, 741, 4092, 0, 0, 0, 1, 0, 26, 3728, 0, 0, 112, 2455, 3776, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 469, 3883, 0, 0, 0, 1581, 9, 0, 4095, 19, 0, 36, 0, 0, 1587, 535, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 45, 139, 0, 0, 379, 605, 4091, 0, 19, 3993, 0, 4095, 3809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1646, 3500, 0, 4095, 3812, 0, 2, 4095, 0, 0, 1, 5, 104, 133, 314, 103, 958, 924, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 143, 1, 5, 3, 3120, 0, 48, 5, 0, 2900, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3130, 2928, 44, 153, 3780, 4058, 0, 0, 0, 0, 0, 2, 0, 2806, 2580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1704, 3864, 4, 0, 29, 3849, 0, 178, 3603, 6, 0, 0, 2381, 0, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 3718, 2507, 2001, 0, 0, 0, 3367, 190, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 2, 0, 3470, 20, 0, 3492, 0, 3149, 3984, 3929, 0, 15, 0, 0, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16590, 17218, 19891, 10558, 19481, 16314, 11835, 19185] avg=16384.0000 max=19891 max_rank=2 min=10558 min_rank=3 imbalance_ratio=1.214050 max_load_reduction=111181 max_load_reduction_pct=0.848244 moved_to_echo=66815 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4083, 133, 0, 9, 0, 0, 11, 155, 1023, 0, 17, 50, 4079, 19, 0, 3, 1641, 530, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 1400, 0, 0, 0, 0, 0, 1121, 0, 0, 0, 3122, 0, 4056, 2489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 3807, 2822, 89, 492, 2759, 0, 75, 3, 4085, 0, 224, 0, 36, 967, 915, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 4095, 2775, 4096, 0, 4096, 10, 109, 3128, 0, 61, 0, 1467, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 4032, 1298, 0, 3941, 0, 0, 0, 1054, 0, 4016, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 349, 189, 1586, 0, 8, 0, 0, 4095, 0, 0, 279, 2453, 13, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1480, 3843, 0, 2465, 11, 0, 0, 2321, 4072, 0, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2627, 76, 0, 4092, 0, 3220, 4057, 0, 3850, 248, 4095, 3, 0, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11939, 12210, 16542, 23981, 15218, 13113, 15440, 22629] avg=16384.0000 max=23981 max_rank=3 min=11939 min_rank=0 imbalance_ratio=1.463684 max_load_reduction=107091 max_load_reduction_pct=0.817039 moved_to_echo=66400 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 711, 4096, 3684, 11, 1, 2065, 0, 0, 0, 10, 4095, 0, 0, 357, 0, 2760, 1033, 550, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 13, 199, 0, 1, 662, 3733, 2, 125, 0, 0, 0, 0, 8, 2985, 535, 3011, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2398, 0, 4074, 0, 3561, 0, 4089, 0, 0, 68, 641, 2966, 0, 0, 8, 961, 950, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 3896, 0, 3687, 0, 0, 0, 0, 3130, 1543, 0, 0, 16, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1374, 0, 4071, 4037, 4056, 191, 12, 0, 0, 0, 1214, 0, 0, 3891, 3575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4, 31, 530, 3259, 407, 878, 3996, 85, 0, 0, 17, 1549, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 430, 0, 3331, 27, 2577, 4022, 0, 1097, 0, 2824, 0, 0, 59, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 22, 0, 1159, 13, 0, 3854, 0, 3037, 0, 19, 0, 2032, 1328, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19634, 14792, 19908, 12707, 22421, 15573, 14519, 11518] avg=16384.0000 max=22421 max_rank=4 min=11518 min_rank=7 imbalance_ratio=1.368469 max_load_reduction=108651 max_load_reduction_pct=0.828941 moved_to_echo=64031 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2454, 1, 0, 472, 0, 0, 0, 0, 116, 0, 4096, 4006, 0, 1, 31, 2841, 1629, 1, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 6, 2894, 0, 0, 0, 1412, 1921, 2270, 460, 3957, 0, 3783, 3228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4007, 0, 1080, 679, 14, 63, 0, 0, 5, 0, 4041, 4044, 0, 946, 2, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 2197, 0, 4090, 85, 0, 978, 194, 3081, 4096, 115, 4, 3289, 1928, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 10, 4096, 148, 0, 1862, 251, 0, 4091, 1250, 1083, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3660, 3865, 4067, 0, 0, 0, 1, 0, 3536, 2445, 0, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 49, 352, 0, 2145, 8, 791, 0, 1, 0, 592, 0, 3867, 1356, 69, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 33, 2680, 3285, 0, 3623, 3507, 0, 0, 0, 0, 167, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15887, 19935, 15124, 20597, 13526, 18807, 9272, 17924] avg=16384.0000 max=20597 max_rank=3 min=9272 min_rank=6 imbalance_ratio=1.257141 max_load_reduction=110475 max_load_reduction_pct=0.842857 moved_to_echo=59529 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[78, 3126, 4063, 888, 4087, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 160, 0, 2834, 1009, 552, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 13, 0, 0, 0, 1, 0, 1, 0, 0, 14, 0, 357, 1224, 3821, 3715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4096, 0, 0, 703, 332, 1787, 0, 0, 0, 4083, 2270, 0, 0, 699, 955, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 1114, 671, 0, 4096, 3969, 1, 3063, 0, 2279, 0, 1032, 936, 0, 3922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2263, 186, 3029, 152, 5, 4091, 0, 0, 0, 0, 1245, 3297, 3, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3673, 2805, 0, 62, 336, 4096, 0, 3733, 3581, 52, 2206, 145, 1514, 0, 248, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4041, 2086, 0, 2589, 0, 0, 0, 1, 0, 0, 5, 187, 3201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 470, 0, 0, 0, 0, 4091, 3881, 620, 3670, 0, 0, 1661, 0, 1030, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21169, 9151, 15202, 21108, 14428, 22477, 12112, 15425] avg=16384.0000 max=22477 max_rank=5 min=9151 min_rank=1 imbalance_ratio=1.371887 max_load_reduction=108595 max_load_reduction_pct=0.828514 moved_to_echo=64442 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[49, 0, 4093, 41, 1, 0, 0, 0, 4096, 3120, 13, 0, 3, 301, 1, 0, 1931, 0, 533, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2692, 15, 4084, 6, 0, 1805, 0, 0, 15, 1, 1373, 0, 4096, 3812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 1, 677, 2749, 4013, 3823, 0, 0, 25, 322, 1544, 181, 9, 259, 5, 962, 921, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 1, 4096, 0, 0, 7, 0, 0, 0, 3134, 2, 0, 0, 2771, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 2, 0, 17, 0, 0, 2720, 850, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4094, 1, 78, 4094, 2789, 0, 0, 0, 0, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3715, 388, 3317, 4096, 2498, 4073, 0, 135, 4095, 3306, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 3656, 377, 113, 168, 0, 3714, 2390, 0, 4096, 3938, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14466, 17899, 15748, 10324, 16072, 12206, 25654, 18703] avg=16384.0000 max=25654 max_rank=6 min=10324 min_rank=3 imbalance_ratio=1.565796 max_load_reduction=105418 max_load_reduction_pct=0.804276 moved_to_echo=72635 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 25, 37, 1355, 108, 0, 347, 4096, 0, 288, 0, 12, 2844, 1052, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 643, 0, 93, 0, 1, 0, 0, 4096, 0, 4080, 0, 4094, 3042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 1675, 3864, 0, 0, 0, 0, 2440, 0, 488, 23, 10, 11, 0, 0, 930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1109, 3, 3420, 0, 0, 0, 2593, 0, 1209, 3031, 0, 0, 925, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 2799, 0, 0, 4021, 9, 0, 1, 0, 0, 1252, 116, 731, 14, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4096, 0, 53, 4080, 0, 4067, 0, 220, 0, 59, 3994, 1578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4007, 0, 0, 4096, 0, 4096, 0, 4064, 0, 0, 0, 741, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 26, 3728, 0, 0, 112, 2455, 3776, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14486, 16052, 9687, 16301, 17032, 22214, 21106, 14194] avg=16384.0000 max=22214 max_rank=5 min=9687 min_rank=2 imbalance_ratio=1.355835 max_load_reduction=108858 max_load_reduction_pct=0.830521 moved_to_echo=74546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4051, 0, 4096, 0, 159, 0, 2052, 0, 0, 4096, 0, 0, 0, 0, 483, 1165, 545, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 429, 0, 3986, 0, 0, 3563, 317, 4094, 0, 41, 3825, 30, 4081, 3370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4084, 0, 99, 0, 40, 4096, 188, 3934, 0, 339, 0, 4064, 0, 960, 941, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 14, 4, 150, 142, 872, 0, 3128, 0, 178, 0, 3937, 3150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 1656, 0, 2773, 17, 0, 0, 3, 213, 0, 58, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 123, 0, 0, 23, 35, 0, 0, 3914, 402, 0, 0, 1749, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 772, 2301, 4090, 4074, 2554, 0, 0, 0, 0, 3252, 0, 4035, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 857, 1, 0, 649, 0, 1511, 3439, 0, 3426, 5, 6, 285, 33, 1278, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16901, 23815, 18970, 15496, 12894, 10368, 21134, 11494] avg=16384.0000 max=23815 max_rank=1 min=10368 min_rank=5 imbalance_ratio=1.453552 max_load_reduction=107257 max_load_reduction_pct=0.818306 moved_to_echo=55890 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1536, 1536, 26496, 2816, 16000, 4480, 6912, 4992, 1536, 24064, 5760, 1536, 1920, 11520, 20224, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[19328, 6528, 15360, 15488, 12032, 15104, 1920, 256, 0, 7936, 6144, 8832, 1664, 11008, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[256, 14592, 16512, 5120, 16768, 128, 2304, 1408, 8064, 4352, 384, 24704, 4096, 4736, 640, 128, 13696, 9216, 3328, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 256, 128, 4608, 1408, 14848, 128, 128, 5760, 3456, 10112, 6400, 640, 23808, 1792, 31104, 27008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[8448, 128, 0, 13056, 512, 11648, 25984, 640, 24576, 10368, 10624, 9856, 6144, 8192, 256, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[21760, 11264, 256, 6016, 15744, 12544, 5120, 12032, 18304, 512, 2304, 8064, 13696, 2048, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[256, 4992, 8576, 20736, 14080, 15232, 15104, 0, 5376, 6528, 21504, 256, 9216, 1664, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[1792, 6272, 1792, 28800, 9472, 8960, 5760, 11392, 8448, 384, 4864, 6400, 7808, 6784, 8704, 256, 7424, 5632, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3651, 0, 0, 0, 3290, 3870, 1222, 21, 4007, 331, 1541, 79, 0, 0, 0, 1407, 1, 69, 0, 0, 0, 5, 0, 18, 0, 1560, 0, 105, 4034, 0, 0, 0, 412, 15, 0, 0, 0, 0, 0, 10, 0, 0, 4001, 2393, 0, 0, 7, 4096, 0, 0, 0, 3944, 20, 828, 3715, 0, 0, 0, 284, 140, 0, 69, 0, 0, 0, 1672, 3776, 0, 382, 793, 0, 4094, 347, 1, 2938, 31, 0, 0, 0, 1403, 0, 3399, 48, 473, 4095, 1, 3933, 0, 4060, 0, 1858, 0, 93, 0, 0, 0, 4049, 0, 4077, 1872, 992, 0, 4096, 18, 74, 2, 46, 125, 3732, 4096, 15, 4096, 0, 0, 3845, 4004, 0, 697, 0, 0, 3478, 251, 4096, 0, 2353, 3667, 0, 2849] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[18, 14, 3989, 5, 2246, 2578, 1174, 3555, 6, 873, 0, 3903, 149, 0, 1034, 0, 4096, 1573, 1, 4095, 152, 4096, 0, 0, 0, 466, 738, 1, 292, 4, 0, 399, 0, 0, 0, 66, 1388, 4081, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3962, 0, 0, 4043, 4069, 0, 0, 4030, 4095, 0, 25, 134, 0, 3852, 0, 2669, 0, 115, 0, 0, 1648, 2544, 0, 1970, 4092, 3728, 0, 0, 304, 11, 279, 165, 0, 1, 0, 662, 1, 4041, 0, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 3, 0, 4092, 3778, 95, 0, 770, 39, 0, 3925, 0, 67, 0, 3, 3210, 162, 0, 176, 0, 4096, 4096, 3316, 0, 0, 0, 3505, 4090, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4096, 167, 3855, 48, 4094, 4096, 89, 1067, 3329, 7, 4096, 0, 0, 0, 2, 4, 3889, 2, 0, 2067, 14, 1841, 0, 0, 130, 0, 286, 0, 295, 5, 403, 0, 0, 0, 0, 0, 0, 2983, 4, 22, 0, 0, 4096, 0, 0, 0, 0, 2767, 35, 0, 667, 3801, 0, 4083, 0, 0, 2412, 490, 1248, 0, 0, 11, 7, 2762, 3117, 3562, 2, 0, 0, 0, 116, 4093, 522, 0, 4, 0, 1, 1, 40, 3849, 3, 1391, 0, 509, 30, 6, 4016, 0, 0, 0, 122, 0, 0, 151, 0, 1783, 3199, 1010, 0, 3687, 0, 1109, 3912, 7, 4053, 0, 2, 0, 4039, 0, 479, 3531, 3462, 4088, 2591, 0, 58, 0, 4081, 0, 0, 0, 2, 814, 3213, 1052, 4, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4075, 0, 4094, 947, 261, 20, 4, 0, 0, 1, 4092, 0, 0, 9, 0, 0, 2544, 0, 0, 0, 0, 0, 183, 921, 0, 312, 0, 0, 0, 1553, 0, 0, 385, 0, 0, 3456, 0, 1, 56, 0, 947, 0, 2791, 2871, 757, 13, 78, 4095, 0, 166, 1325, 2324, 984, 4082, 0, 929, 3865, 4096, 5, 140, 0, 0, 1, 3514, 4096, 4074, 5, 0, 0, 26, 15, 3460, 3950, 0, 1570, 0, 25, 0, 0, 0, 0, 0, 0, 0, 150, 0, 4086, 3920, 0, 4095, 349, 0, 3962, 228, 0, 0, 1743, 13, 7, 3287, 1, 8, 1079, 0, 4090, 275, 751, 2, 3297, 92, 0, 4059, 0, 3994, 812, 2084, 0, 0, 0, 588, 156, 3074, 0, 0, 3958, 4095, 0, 3699] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 3678, 264, 0, 4055, 36, 17, 0, 192, 4096, 0, 0, 43, 0, 29, 301, 0, 86, 4, 0, 0, 0, 99, 0, 4033, 0, 0, 157, 0, 297, 58, 3506, 486, 0, 0, 0, 0, 4096, 660, 189, 0, 4095, 3666, 0, 0, 0, 31, 0, 322, 2925, 0, 3, 3579, 0, 0, 3483, 4072, 0, 0, 0, 0, 0, 3919, 283, 4096, 3191, 0, 0, 0, 0, 4096, 3381, 0, 3, 302, 3969, 1, 0, 3572, 22, 4092, 1, 3835, 4003, 2, 3924, 0, 0, 0, 0, 0, 0, 0, 4072, 2641, 275, 4094, 12, 1292, 0, 0, 4095, 0, 4096, 19, 0, 0, 3364, 1527, 631, 1, 1, 838, 4071, 0, 0, 0, 0, 0, 32, 0, 225, 34, 3716, 0, 2, 688] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[9, 1, 3949, 3576, 83, 2471, 663, 5, 0, 0, 3825, 3938, 0, 0, 0, 0, 4042, 169, 11, 3918, 223, 366, 0, 238, 103, 23, 41, 0, 3, 0, 44, 0, 0, 269, 0, 3949, 12, 0, 4074, 41, 3992, 0, 0, 2928, 0, 1, 0, 1, 27, 0, 1302, 6, 0, 0, 4096, 62, 3161, 4095, 0, 0, 0, 0, 0, 4085, 0, 0, 3270, 1793, 9, 0, 4027, 3697, 0, 0, 0, 4096, 0, 460, 0, 72, 0, 4096, 1, 0, 4096, 0, 3668, 4028, 1324, 0, 0, 0, 0, 4028, 1, 0, 0, 1, 0, 1206, 0, 0, 4094, 296, 0, 1479, 0, 0, 4091, 0, 4096, 0, 1, 0, 3832, 0, 4022, 0, 9, 3751, 0, 0, 3546, 0, 0, 2160, 0, 20] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4053, 0, 3651, 4051, 2, 120, 0, 0, 1, 0, 437, 0, 0, 0, 0, 0, 3, 0, 0, 1586, 0, 0, 0, 0, 3, 185, 1, 53, 0, 0, 0, 0, 0, 0, 56, 4095, 792, 0, 5, 110, 4093, 4087, 4094, 4072, 287, 3, 29, 4096, 2161, 288, 2247, 4078, 0, 4093, 1165, 2, 0, 3367, 0, 0, 0, 79, 2, 4083, 0, 4074, 0, 0, 0, 0, 0, 4089, 7, 184, 1528, 4094, 3613, 0, 0, 3991, 123, 1235, 0, 4096, 245, 1, 4039, 4032, 0, 3956, 0, 419, 0, 0, 28, 4078, 197, 1907, 0, 4088, 0, 0, 28, 0, 2064, 0, 0, 0, 2470, 6, 0, 2011, 718, 8, 3278, 7, 0, 0, 1, 4, 4008, 0, 0, 0, 0, 908, 0, 4007] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4096, 36, 4096, 0, 0, 594, 45, 6, 3835, 0, 3571, 0, 432, 0, 0, 176, 4064, 2, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1441, 0, 0, 2624, 0, 4, 0, 3780, 2318, 3, 0, 122, 4096, 0, 4053, 3684, 46, 0, 0, 1, 557, 0, 390, 4086, 1344, 4074, 174, 1, 473, 4096, 4095, 0, 101, 0, 0, 1426, 1, 0, 0, 0, 0, 0, 0, 8, 1, 18, 3935, 9, 1816, 161, 1, 3731, 1157, 4069, 2, 0, 3903, 0, 1, 4024, 0, 0, 230, 0, 8, 7, 31, 322, 4, 5, 0, 4096, 4076, 0, 1862, 1162, 1, 0, 0, 0, 4096, 3982, 269, 3761, 0, 4054, 1869, 60, 3483, 0, 0, 0, 3852, 0, 351, 401, 1927, 295, 0, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 167, 2364, 48, 4094, 4096, 89, 1067, 3329, 7, 4096, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3889, 2, 0, 2067, 14, 1841, 0, 0, 130, 0, 286, 0, 295, 5, 403, 0, 4091, 3267, 2735, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2983, 4, 22, 0, 0, 4096, 0, 0, 0, 0, 2767, 871, 486, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 667, 3801, 0, 1348, 0, 0, 2412, 490, 1248, 0, 0, 11, 7, 2762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3117, 3562, 2, 0, 0, 0, 116, 3222, 522, 0, 4, 0, 1, 1, 40, 3849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1391, 0, 509, 30, 6, 4016, 0, 0, 0, 122, 0, 0, 151, 0, 1783, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3199, 1010, 0, 3687, 0, 1109, 3912, 7, 4053, 0, 2, 0, 772, 0, 479, 3531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3462, 4088, 2141, 0, 58, 0, 4081, 0, 0, 0, 2, 814, 3213, 1052, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23459, 20030, 11261, 12781, 14436, 8429, 21761, 18915] avg=16384.0000 max=23459 max_rank=0 min=8429 min_rank=5 imbalance_ratio=1.431824 max_load_reduction=107613 max_load_reduction_pct=0.821022 moved_to_echo=63541 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3651, 0, 0, 0, 3290, 3870, 1222, 21, 4007, 331, 1541, 79, 0, 0, 0, 1407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 69, 0, 0, 0, 5, 0, 18, 0, 1560, 0, 105, 4034, 0, 0, 0, 2849, 3019, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 412, 15, 0, 0, 0, 0, 0, 10, 0, 0, 4001, 2393, 0, 0, 7, 4096, 871, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3944, 20, 274, 3715, 0, 0, 0, 284, 140, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1672, 3776, 0, 382, 793, 0, 3223, 347, 1, 2938, 31, 0, 0, 0, 1403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3399, 48, 473, 4095, 1, 3933, 0, 4060, 0, 1858, 0, 93, 0, 0, 0, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 4077, 1872, 992, 0, 4096, 18, 74, 2, 46, 125, 713, 4096, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3176, 4004, 0, 697, 0, 0, 3478, 251, 4096, 0, 2353, 3667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19419, 12214, 11853, 8446, 14566, 18581, 24271, 21722] avg=16384.0000 max=24271 max_rank=6 min=8446 min_rank=3 imbalance_ratio=1.481384 max_load_reduction=106801 max_load_reduction_pct=0.814827 moved_to_echo=79140 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[18, 14, 2446, 5, 2246, 2578, 1174, 3555, 6, 873, 0, 3903, 149, 0, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1573, 1, 4095, 152, 4096, 0, 0, 0, 466, 738, 1, 292, 4, 0, 399, 4096, 3175, 2708, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 1388, 4081, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 541, 503, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3962, 0, 0, 1335, 4069, 0, 0, 4030, 4095, 0, 25, 134, 0, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2669, 0, 115, 0, 0, 1648, 2003, 0, 1970, 4092, 3728, 0, 0, 304, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 165, 0, 1, 0, 662, 1, 4041, 0, 0, 1, 0, 14, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4092, 3778, 95, 0, 770, 39, 0, 750, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3210, 134, 0, 176, 0, 4096, 4096, 3316, 0, 0, 0, 3505, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18001, 26932, 10686, 21503, 16540, 5190, 9594, 22626] avg=16384.0000 max=26932 max_rank=1 min=5190 min_rank=5 imbalance_ratio=1.643799 max_load_reduction=104140 max_load_reduction_pct=0.794525 moved_to_echo=53950 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[9, 1, 2421, 3576, 83, 2471, 663, 5, 0, 0, 3825, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 169, 11, 3918, 223, 366, 0, 238, 103, 23, 41, 0, 3, 0, 44, 0, 20, 3309, 0, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 0, 3949, 12, 0, 4074, 41, 3992, 0, 0, 2928, 0, 1, 0, 1, 787, 498, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 1302, 6, 0, 0, 4096, 62, 3161, 4095, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3270, 1793, 9, 0, 4027, 2910, 0, 0, 0, 4096, 0, 460, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 3668, 4028, 1324, 0, 0, 0, 0, 4028, 1, 0, 619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1206, 0, 0, 4094, 296, 0, 1479, 0, 0, 782, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3166, 0, 4022, 0, 9, 3751, 0, 0, 3546, 0, 0, 2160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16992, 13540, 16599, 16834, 16637, 21861, 11954, 16655] avg=16384.0000 max=21861 max_rank=5 min=11954 min_rank=6 imbalance_ratio=1.334290 max_load_reduction=109211 max_load_reduction_pct=0.833214 moved_to_echo=67107 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 2256, 264, 0, 4055, 36, 17, 0, 192, 4096, 0, 0, 43, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 301, 0, 86, 4, 0, 0, 0, 99, 0, 4033, 0, 0, 157, 0, 297, 58, 688, 2721, 2398, 959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3506, 486, 0, 0, 0, 0, 4096, 660, 189, 0, 4095, 3666, 0, 0, 0, 31, 872, 463, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 2925, 0, 3, 1181, 0, 0, 3483, 4072, 0, 0, 0, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 4096, 3191, 0, 0, 0, 0, 3224, 3381, 0, 3, 302, 3969, 1, 0, 3572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4092, 1, 3835, 4003, 2, 3924, 0, 0, 0, 0, 0, 0, 0, 4072, 2641, 658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 4094, 12, 1292, 0, 0, 4095, 0, 4096, 19, 0, 0, 643, 1527, 631, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 838, 3363, 0, 0, 0, 0, 0, 32, 0, 225, 34, 3716, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15084, 11801, 18114, 15905, 22022, 23250, 16685, 8211] avg=16384.0000 max=23250 max_rank=5 min=8211 min_rank=7 imbalance_ratio=1.419067 max_load_reduction=107822 max_load_reduction_pct=0.822617 moved_to_echo=70168 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4075, 0, 2510, 947, 261, 20, 4, 0, 0, 1, 4092, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2544, 0, 0, 0, 0, 0, 183, 921, 0, 312, 0, 0, 0, 1553, 0, 0, 3699, 2667, 2735, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 0, 0, 3456, 0, 1, 56, 0, 947, 0, 2791, 2871, 757, 13, 78, 4095, 736, 516, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 1325, 2324, 984, 1347, 0, 929, 3865, 4096, 5, 140, 0, 0, 1, 3514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4074, 5, 0, 0, 26, 15, 2724, 3950, 0, 1570, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 4086, 3920, 0, 4095, 349, 0, 3962, 228, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1743, 13, 7, 3287, 1, 8, 1079, 0, 4090, 275, 751, 2, 630, 92, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 671, 2084, 0, 0, 0, 588, 156, 3074, 0, 0, 3958, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11919, 15682, 16712, 18696, 16485, 16921, 16037, 18620] avg=16384.0000 max=18696 max_rank=3 min=11919 min_rank=0 imbalance_ratio=1.141113 max_load_reduction=112376 max_load_reduction_pct=0.857361 moved_to_echo=68063 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4053, 0, 2231, 4051, 2, 120, 0, 0, 1, 0, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1586, 0, 0, 0, 0, 3, 185, 1, 53, 0, 0, 0, 0, 4007, 2002, 2746, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 4095, 792, 0, 5, 110, 4093, 4087, 4094, 4072, 287, 3, 29, 4096, 874, 463, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2161, 288, 2247, 4078, 0, 1347, 1165, 2, 0, 3367, 0, 0, 0, 79, 2, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 3215, 7, 184, 1528, 4094, 3613, 0, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 1235, 0, 4096, 245, 1, 4039, 4032, 0, 3956, 0, 419, 0, 0, 28, 4078, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 1907, 0, 4088, 0, 0, 28, 0, 2064, 0, 0, 0, 468, 6, 0, 2011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 718, 8, 2702, 7, 0, 0, 1, 4, 4008, 0, 0, 0, 0, 908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10895, 11543, 27200, 18819, 20706, 22784, 10769, 8356] avg=16384.0000 max=27200 max_rank=2 min=8356 min_rank=7 imbalance_ratio=1.660156 max_load_reduction=103872 max_load_reduction_pct=0.792480 moved_to_echo=62615 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 36, 2512, 0, 0, 594, 45, 6, 3835, 0, 3571, 0, 432, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 2, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1441, 0, 0, 2624, 4087, 3313, 2729, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3780, 2318, 3, 0, 122, 4096, 0, 4053, 3684, 46, 0, 0, 1, 1, 516, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 0, 390, 4086, 1344, 1345, 174, 1, 473, 4096, 4095, 0, 101, 0, 0, 1426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 1, 18, 3935, 9, 1816, 161, 1, 3731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1157, 4069, 2, 0, 3903, 0, 1, 4024, 0, 0, 230, 0, 8, 7, 31, 322, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 4096, 4076, 0, 1862, 1162, 1, 0, 0, 0, 783, 3982, 269, 3761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 1544, 60, 3483, 0, 0, 0, 3852, 0, 351, 401, 1927, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15303, 19330, 18647, 18088, 9680, 14056, 20001, 15967] avg=16384.0000 max=20001 max_rank=6 min=9680 min_rank=4 imbalance_ratio=1.220764 max_load_reduction=111071 max_load_reduction_pct=0.847404 moved_to_echo=59704 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[24192, 256, 16768, 8960, 9984, 17920, 3328, 4736, 11264, 1408, 21760, 7936, 640, 128, 1152, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[9472, 7040, 4096, 19584, 5120, 5248, 23040, 1664, 14464, 2560, 896, 128, 5632, 9728, 5632, 17536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[4224, 16256, 17024, 6272, 7808, 768, 8192, 8448, 14848, 3328, 8320, 1280, 18688, 16384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2816, 896, 12928, 18304, 2432, 8192, 13312, 1024, 13440, 24320, 9728, 384, 128, 384, 128, 23680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[7552, 20224, 10368, 1920, 512, 896, 5888, 20608, 8320, 2176, 14080, 12288, 9472, 640, 384, 16640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[4352, 896, 128, 15360, 4608, 7168, 12416, 1024, 13440, 4096, 23168, 19712, 1152, 128, 128, 15104, 5632, 3456, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1664, 18560, 128, 8960, 16640, 768, 23680, 20096, 5504, 8064, 2560, 512, 4096, 4480, 4224, 8832, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[18944, 1920, 128, 11776, 512, 6400, 256, 1280, 256, 6656, 1152, 256, 6272, 1664, 768, 3200, 23552, 23552, 16640, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[2, 3871, 0, 0, 56, 4094, 0, 46, 4088, 1744, 3968, 4096, 0, 0, 0, 1170, 63, 958, 3314, 20, 1148, 0, 272, 4094, 0, 0, 1958, 4090, 1, 0, 1, 0, 167, 0, 3471, 0, 2016, 0, 730, 0, 0, 4072, 408, 4092, 60, 232, 4088, 0, 200, 0, 0, 395, 0, 0, 0, 40, 477, 0, 0, 243, 0, 0, 23, 0, 8, 6, 1740, 2, 16, 0, 201, 0, 3989, 0, 4096, 42, 2, 0, 3463, 0, 45, 0, 0, 3937, 18, 3137, 0, 0, 2, 110, 4039, 3703, 0, 1, 3, 0, 23, 35, 6, 4090, 0, 0, 4005, 1658, 2790, 0, 0, 853, 4091, 0, 0, 0, 900, 3794, 0, 0, 1693, 7, 3957, 1, 3703, 3678, 0, 3847, 0, 3613, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[40, 22, 7, 0, 1342, 0, 0, 0, 0, 44, 0, 0, 201, 3873, 0, 0, 0, 0, 820, 4096, 1058, 2439, 0, 837, 0, 0, 4050, 473, 0, 197, 0, 0, 0, 3669, 3921, 0, 0, 0, 0, 3835, 0, 2028, 2, 1, 3233, 13, 380, 4094, 0, 70, 4096, 3755, 3391, 0, 0, 1, 4096, 0, 0, 3009, 0, 3896, 0, 0, 3842, 2, 0, 0, 0, 0, 0, 1975, 3944, 0, 4061, 0, 0, 0, 0, 0, 1, 0, 3, 4081, 0, 3686, 0, 0, 0, 3009, 0, 0, 628, 4096, 0, 0, 0, 0, 143, 2655, 3541, 0, 17, 4096, 3729, 0, 4096, 12, 287, 4086, 4, 195, 2, 25, 486, 10, 42, 0, 470, 1, 4090, 1234, 50, 4081, 2911, 0, 0, 2492] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 3476, 167, 431, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 279, 0, 0, 0, 3891, 0, 0, 646, 0, 0, 1, 0, 3802, 0, 0, 0, 0, 0, 0, 3, 4095, 0, 0, 0, 4090, 4092, 0, 2756, 0, 1337, 0, 4063, 0, 3510, 103, 524, 0, 51, 3940, 0, 0, 17, 1656, 0, 429, 301, 16, 4096, 0, 0, 0, 4092, 0, 1, 3093, 13, 58, 0, 3490, 0, 0, 0, 7, 26, 4039, 122, 0, 0, 3537, 13, 0, 3354, 0, 33, 8, 0, 0, 4087, 0, 3955, 2, 0, 0, 0, 1713, 0, 19, 0, 4038, 4072, 3908, 0, 0, 0, 0, 4094, 4096, 5, 232, 3868, 2839, 0, 0, 906, 4001, 0, 3712, 181, 1, 2649, 0, 0, 0, 748] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 1990, 168, 0, 0, 2892, 0, 1569, 0, 2586, 0, 0, 65, 0, 0, 123, 50, 11, 0, 0, 0, 0, 783, 3982, 0, 0, 3068, 3846, 0, 836, 241, 0, 0, 4088, 3250, 3343, 13, 0, 984, 2589, 84, 0, 0, 0, 0, 4095, 0, 0, 93, 0, 0, 0, 368, 0, 0, 0, 99, 4020, 608, 4096, 0, 4089, 6, 707, 4090, 3943, 0, 2446, 4096, 0, 0, 2446, 2447, 41, 0, 0, 0, 0, 4089, 34, 299, 0, 0, 4076, 0, 3115, 2432, 0, 0, 9, 0, 83, 0, 797, 0, 376, 288, 0, 4096, 260, 0, 0, 3717, 4095, 3731, 1, 130, 77, 0, 0, 1175, 0, 3787, 2659, 8, 3453, 17, 3971, 8, 1403, 3264, 4096, 0, 0, 6, 215, 0, 1054] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 4033, 2, 0, 0, 85, 0, 6, 12, 4067, 0, 0, 1, 4095, 0, 39, 454, 0, 4042, 2626, 0, 3877, 5, 4096, 0, 3, 4096, 0, 287, 7, 0, 6, 0, 3905, 4096, 171, 13, 0, 3698, 15, 0, 2859, 0, 0, 0, 4, 0, 4095, 0, 1, 4066, 0, 4, 0, 0, 76, 2829, 0, 0, 0, 0, 21, 0, 0, 0, 4092, 4086, 1, 2573, 128, 0, 0, 1984, 24, 0, 4061, 0, 3569, 4074, 0, 334, 22, 4096, 0, 3740, 4062, 0, 1346, 0, 1308, 385, 2529, 0, 3797, 0, 2, 0, 0, 16, 0, 772, 0, 3243, 0, 4040, 0, 0, 0, 0, 0, 4096, 0, 0, 670, 644, 394, 37, 4083, 0, 0, 0, 20, 0, 4096, 4017, 0, 0, 1039] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 183, 114, 5, 4086, 352, 141, 184, 3677, 78, 4065, 454, 0, 3813, 112, 0, 0, 630, 0, 0, 31, 0, 0, 33, 306, 0, 0, 102, 0, 672, 3493, 0, 0, 0, 3950, 14, 3664, 291, 0, 3543, 2153, 2268, 996, 304, 439, 3455, 0, 1165, 0, 0, 383, 3980, 0, 1, 272, 3456, 4095, 2000, 0, 0, 0, 2709, 32, 0, 406, 0, 3632, 0, 0, 4096, 286, 0, 4043, 4058, 0, 5, 0, 0, 3852, 0, 1, 18, 5, 3835, 541, 0, 0, 0, 1126, 0, 0, 0, 0, 3460, 1720, 0, 0, 0, 3541, 955, 4096, 48, 0, 0, 3766, 55, 0, 0, 825, 4096, 16, 147, 0, 0, 3112, 302, 10, 3820, 8, 25, 977, 4095, 792, 3933, 3668, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[3510, 0, 0, 0, 0, 0, 2, 1, 3109, 0, 702, 3, 206, 0, 0, 0, 15, 121, 0, 22, 0, 3491, 28, 4052, 1486, 298, 4094, 3670, 0, 402, 531, 0, 54, 3, 4089, 76, 2187, 0, 0, 0, 0, 0, 283, 0, 515, 4067, 396, 3770, 0, 0, 0, 0, 4092, 0, 10, 0, 3949, 6, 36, 285, 0, 4096, 396, 0, 14, 3264, 3825, 0, 32, 0, 4086, 0, 3987, 0, 2, 0, 418, 1520, 251, 0, 0, 0, 4096, 92, 3, 1619, 0, 0, 0, 2, 683, 4040, 0, 3627, 0, 2467, 0, 4096, 0, 2319, 401, 0, 0, 1, 2529, 3948, 3532, 17, 0, 2, 3952, 2, 731, 4014, 3770, 2880, 0, 19, 4012, 0, 833, 1897, 0, 3850, 41, 3, 0, 142] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[2, 0, 681, 498, 0, 3143, 2535, 0, 4, 0, 9, 327, 7, 0, 2, 0, 1846, 0, 1744, 2, 4096, 210, 234, 0, 8, 648, 3820, 768, 0, 910, 0, 18, 4094, 4088, 3865, 871, 1, 0, 0, 0, 4, 4096, 117, 0, 0, 3610, 3707, 258, 0, 0, 0, 0, 766, 0, 0, 12, 0, 0, 1097, 9, 0, 3950, 0, 0, 1502, 4087, 0, 0, 1, 0, 1, 2571, 4051, 0, 1, 28, 0, 906, 208, 2, 0, 0, 0, 4096, 0, 23, 0, 0, 13, 0, 16, 4035, 0, 3500, 0, 0, 3, 0, 68, 3675, 4041, 224, 4096, 0, 2377, 2680, 3954, 3819, 2153, 3830, 1240, 0, 0, 4096, 0, 1, 0, 4096, 2271, 0, 3261, 0, 0, 4049, 3251, 28, 0, 761] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 3871, 0, 0, 56, 4094, 0, 46, 4088, 1744, 3968, 4096, 0, 0, 0, 1170, 1877, 2025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 958, 3314, 20, 1148, 0, 272, 4094, 0, 0, 1958, 4090, 1, 0, 1, 0, 1822, 1455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 0, 2016, 0, 730, 0, 0, 4072, 408, 4092, 60, 205, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 395, 0, 0, 0, 40, 477, 0, 0, 243, 0, 0, 23, 0, 3471, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 1740, 2, 16, 0, 201, 0, 2474, 0, 4096, 42, 2, 0, 3463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 3937, 18, 3137, 0, 0, 2, 110, 4039, 3703, 0, 1, 3, 0, 1182, 975, 27, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 35, 6, 4090, 0, 0, 3030, 1658, 0, 0, 0, 853, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 900, 3794, 0, 0, 1693, 7, 3957, 1, 2521, 3678, 0, 0, 0, 3613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[27037, 19196, 15838, 5762, 12050, 17239, 13786, 20164] avg=16384.0000 max=27037 max_rank=0 min=5762 min_rank=3 imbalance_ratio=1.650208 max_load_reduction=104035 max_load_reduction_pct=0.793724 moved_to_echo=63239 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 3476, 167, 431, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 279, 0, 2628, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3891, 0, 0, 646, 0, 0, 1, 0, 3802, 0, 0, 0, 0, 0, 1255, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4090, 4092, 0, 2756, 0, 1337, 0, 3576, 0, 3510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 524, 0, 51, 3940, 0, 0, 17, 1656, 0, 429, 301, 16, 4096, 0, 0, 4095, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 3093, 13, 58, 0, 2165, 0, 0, 0, 7, 26, 4039, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3537, 13, 0, 3354, 0, 33, 8, 0, 0, 4087, 0, 3955, 2, 0, 1185, 983, 487, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1713, 0, 19, 0, 3055, 4072, 0, 0, 0, 0, 0, 4094, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 3868, 2839, 0, 0, 906, 4001, 0, 2527, 181, 1, 0, 0, 0, 0, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20663, 10868, 19364, 16508, 13616, 17696, 17054, 15303] avg=16384.0000 max=20663 max_rank=0 min=10868 min_rank=1 imbalance_ratio=1.261169 max_load_reduction=110409 max_load_reduction_pct=0.842354 moved_to_echo=63669 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4033, 2, 0, 0, 85, 0, 6, 12, 4067, 0, 0, 1, 4095, 0, 39, 2718, 2157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 0, 4042, 2626, 0, 3877, 5, 4096, 0, 3, 4096, 0, 287, 7, 0, 6, 1939, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 0, 171, 13, 0, 3698, 15, 0, 2859, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4066, 0, 4, 0, 0, 76, 2829, 0, 0, 0, 0, 21, 0, 0, 4096, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4086, 1, 2573, 128, 0, 0, 1224, 24, 0, 4061, 0, 3569, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 22, 4096, 0, 3740, 4062, 0, 1346, 0, 1308, 385, 2529, 0, 3797, 0, 2, 0, 792, 4, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 772, 0, 2451, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 670, 644, 394, 37, 4083, 0, 0, 0, 20, 0, 0, 4017, 0, 0, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17215, 22165, 14756, 12415, 23832, 22450, 7335, 10904] avg=16384.0000 max=23832 max_rank=4 min=7335 min_rank=6 imbalance_ratio=1.454590 max_load_reduction=107240 max_load_reduction_pct=0.818176 moved_to_echo=64521 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[40, 22, 7, 0, 1342, 0, 0, 0, 0, 44, 0, 0, 201, 3873, 0, 0, 2509, 2149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 820, 4096, 1058, 2439, 0, 837, 0, 0, 4050, 473, 0, 197, 0, 0, 1932, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3669, 0, 0, 0, 0, 0, 3835, 0, 2028, 2, 1, 3233, 12, 380, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 4096, 3755, 3391, 0, 0, 1, 4096, 0, 0, 3009, 0, 3896, 0, 0, 3921, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 2, 0, 0, 0, 0, 0, 1975, 2446, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 4081, 0, 3686, 0, 0, 0, 3009, 0, 0, 628, 4096, 0, 0, 1306, 4, 1, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 2655, 3541, 0, 13, 4096, 0, 0, 4096, 12, 287, 4086, 4, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 25, 486, 10, 42, 0, 470, 1, 2784, 1234, 50, 0, 2911, 0, 0, 2492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10187, 17341, 17254, 27455, 12326, 16874, 19128, 10507] avg=16384.0000 max=27455 max_rank=3 min=10187 min_rank=0 imbalance_ratio=1.675720 max_load_reduction=103617 max_load_reduction_pct=0.790535 moved_to_echo=58835 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 183, 114, 5, 4086, 352, 141, 184, 3677, 78, 4065, 454, 0, 3813, 112, 0, 2533, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 0, 0, 31, 0, 0, 33, 306, 0, 0, 102, 0, 672, 3493, 0, 1863, 1475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3664, 291, 0, 3543, 2153, 2268, 996, 304, 439, 3041, 0, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 3980, 0, 1, 272, 3456, 4095, 2000, 0, 0, 0, 2709, 32, 0, 3950, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 406, 0, 3632, 0, 0, 4096, 286, 0, 2507, 4058, 0, 5, 0, 0, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 18, 5, 3835, 541, 0, 0, 0, 1126, 0, 0, 0, 0, 3460, 1720, 0, 312, 0, 414, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3541, 955, 4096, 48, 0, 0, 0, 55, 0, 0, 825, 4096, 16, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3112, 302, 10, 3820, 8, 25, 665, 4095, 792, 0, 3668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21867, 8605, 17878, 22111, 18842, 11493, 13779, 16497] avg=16384.0000 max=22111 max_rank=3 min=8605 min_rank=1 imbalance_ratio=1.349548 max_load_reduction=108961 max_load_reduction_pct=0.831306 moved_to_echo=60611 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1990, 168, 0, 0, 2892, 0, 1569, 0, 2586, 0, 0, 65, 0, 0, 123, 2509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 11, 0, 0, 0, 0, 783, 3982, 0, 0, 3068, 3846, 0, 836, 241, 0, 0, 893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 3343, 13, 0, 984, 2589, 84, 0, 0, 0, 0, 3604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 368, 0, 0, 0, 99, 4020, 608, 4096, 0, 4089, 6, 707, 3250, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3943, 0, 2446, 4096, 0, 0, 2446, 1518, 41, 0, 0, 0, 0, 4089, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 4076, 0, 3115, 2432, 0, 0, 9, 0, 83, 0, 797, 0, 376, 1042, 905, 491, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 4096, 260, 0, 0, 2812, 4095, 0, 1, 130, 77, 0, 0, 1175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3787, 2659, 8, 3453, 17, 3971, 8, 1403, 2222, 4096, 0, 0, 6, 215, 0, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11902, 13710, 14705, 18558, 22703, 13661, 12934, 22899] avg=16384.0000 max=22899 max_rank=7 min=11902 min_rank=0 imbalance_ratio=1.397644 max_load_reduction=108173 max_load_reduction_pct=0.825294 moved_to_echo=72197 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3510, 0, 0, 0, 0, 0, 2, 1, 3109, 0, 702, 3, 206, 0, 0, 0, 1702, 2027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 121, 0, 22, 0, 3491, 28, 4052, 1486, 298, 4094, 3670, 0, 402, 531, 0, 1823, 1455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 3, 0, 76, 2187, 0, 0, 0, 0, 0, 283, 0, 515, 3580, 396, 3770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 10, 0, 3949, 6, 36, 285, 0, 4096, 396, 0, 4089, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3264, 3825, 0, 32, 0, 4086, 0, 2472, 0, 2, 0, 418, 1520, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 92, 3, 1619, 0, 0, 0, 2, 683, 4040, 0, 3627, 0, 2467, 266, 0, 487, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2319, 401, 0, 0, 1, 0, 3948, 3532, 17, 0, 2, 3952, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 731, 4014, 3770, 2880, 0, 19, 4012, 0, 567, 1897, 0, 0, 41, 3, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11262, 21488, 10864, 17786, 15884, 17442, 18270, 18076] avg=16384.0000 max=21488 max_rank=1 min=10864 min_rank=2 imbalance_ratio=1.311523 max_load_reduction=109584 max_load_reduction_pct=0.836060 moved_to_echo=69672 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 681, 498, 0, 3143, 2535, 0, 4, 0, 9, 327, 7, 0, 2, 0, 1599, 2132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1846, 0, 1744, 2, 4096, 210, 234, 0, 8, 648, 3820, 768, 0, 910, 0, 18, 1917, 1478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4088, 0, 871, 1, 0, 0, 0, 4, 4096, 117, 0, 0, 3177, 3707, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 766, 0, 0, 12, 0, 0, 1097, 9, 0, 3950, 0, 0, 3865, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1502, 4087, 0, 0, 1, 0, 1, 2571, 2512, 0, 1, 28, 0, 906, 208, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 13, 0, 16, 4035, 0, 3500, 0, 0, 1043, 997, 433, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 68, 3675, 4041, 224, 3099, 0, 0, 2680, 3954, 3819, 2153, 3830, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 2271, 0, 2218, 0, 0, 0, 3251, 28, 0, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10939, 17699, 20413, 10477, 11819, 14217, 28786, 16722] avg=16384.0000 max=28786 max_rank=6 min=10477 min_rank=3 imbalance_ratio=1.756958 max_load_reduction=102286 max_load_reduction_pct=0.780380 moved_to_echo=71544 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[7680, 13696, 1152, 1024, 5504, 10624, 2688, 1920, 11008, 12672, 8832, 4992, 512, 16000, 512, 1408, 18176, 14080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[9984, 19584, 13312, 2560, 9856, 4352, 4736, 7040, 17408, 4224, 8192, 4224, 512, 6144, 20096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[768, 128, 11776, 20224, 4352, 19072, 2432, 1408, 1152, 4480, 5248, 18560, 640, 23296, 1792, 2944, 6400, 4736, 2432, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[512, 640, 8576, 8192, 12672, 128, 384, 3712, 17280, 6144, 2176, 8064, 128, 22912, 512, 768, 30848, 8832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[384, 4224, 9600, 14080, 12928, 384, 14464, 13952, 0, 6784, 11776, 4864, 7424, 16128, 14592, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[4352, 15872, 0, 4480, 7936, 384, 9600, 14080, 2304, 18176, 1920, 5760, 4352, 17280, 8576, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[5760, 19200, 10880, 7040, 1920, 17024, 14848, 1536, 13568, 15232, 896, 0, 13952, 3968, 0, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2432, 1792, 13824, 6784, 6400, 10752, 1408, 17152, 1920, 1024, 24960, 13056, 384, 3072, 4352, 128, 12672, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4050, 0, 0, 3, 72, 58, 61, 982, 0, 0, 37, 1559, 1584, 2453, 72, 0, 4096, 3, 1682, 368, 595, 0, 0, 202, 0, 0, 507, 0, 0, 397, 4084, 0, 0, 0, 3914, 0, 0, 1789, 2029, 0, 3822, 3655, 0, 2860, 3257, 0, 1016, 3918, 6, 422, 0, 0, 186, 0, 0, 326, 0, 20, 250, 4096, 3154, 4068, 38, 3616, 3841, 4022, 939, 119, 0, 185, 23, 394, 285, 0, 0, 0, 823, 3567, 27, 0, 0, 4096, 0, 0, 3766, 0, 243, 2, 4073, 0, 0, 0, 0, 1959, 777, 4096, 1, 0, 2422, 0, 4095, 0, 2787, 0, 469, 0, 4044, 2987, 2, 0, 4043, 0, 0, 0, 0, 0, 3798, 0, 0, 15, 2, 3472, 642, 0, 19, 0, 4, 3726] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[167, 42, 3, 0, 0, 10, 0, 1413, 0, 0, 0, 4069, 3364, 3917, 0, 19, 3322, 4069, 0, 1906, 0, 2, 0, 1, 31, 699, 0, 0, 0, 1, 4083, 0, 3010, 0, 12, 1, 0, 4074, 3944, 150, 48, 1587, 0, 11, 4078, 1474, 3831, 4095, 4, 0, 0, 0, 2192, 3574, 1, 3672, 332, 204, 2, 0, 975, 3223, 0, 0, 0, 0, 70, 33, 17, 2, 0, 66, 4075, 2, 4, 0, 0, 186, 4096, 0, 120, 3344, 0, 418, 2821, 841, 0, 0, 4091, 0, 0, 3883, 0, 4063, 0, 3772, 0, 6, 4096, 4088, 0, 2638, 0, 0, 0, 3236, 3919, 0, 32, 0, 1697, 0, 15, 544, 2128, 0, 13, 0, 3962, 4096, 0, 0, 0, 0, 29, 1033, 24, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1814, 0, 2192, 4077, 788, 4082, 0, 0, 3788, 0, 55, 0, 0, 590, 0, 1, 2365, 0, 0, 0, 3, 301, 597, 4093, 1, 0, 0, 0, 0, 0, 4096, 2, 20, 0, 0, 32, 0, 3917, 2814, 0, 4096, 2779, 3616, 1, 2195, 3173, 4059, 0, 3891, 280, 3994, 0, 0, 4, 0, 0, 0, 29, 0, 1, 4088, 1918, 1260, 0, 0, 4096, 4067, 0, 0, 4093, 1427, 697, 2, 2006, 0, 50, 4091, 4096, 17, 0, 0, 0, 1, 100, 0, 0, 31, 3704, 0, 0, 0, 0, 0, 3535, 8, 0, 7, 0, 1729, 0, 0, 489, 0, 0, 0, 0, 0, 4095, 0, 1435, 1788, 13, 31, 0, 0, 0, 0, 6, 230, 3687, 11, 4093, 4085, 1, 337, 3975, 1350, 677] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 1353, 0, 0, 17, 0, 591, 0, 0, 0, 0, 0, 12, 3670, 0, 0, 3502, 953, 4096, 0, 344, 4, 0, 0, 0, 0, 0, 0, 1, 4095, 3754, 2, 485, 3401, 919, 293, 1636, 4063, 7, 0, 4096, 4096, 0, 0, 66, 30, 0, 0, 769, 54, 4088, 262, 3205, 3979, 0, 3987, 85, 0, 0, 489, 0, 433, 78, 4, 0, 100, 4094, 0, 0, 3640, 0, 4096, 4078, 4028, 0, 0, 1, 232, 1787, 1329, 0, 653, 0, 0, 2368, 0, 4, 0, 4096, 1753, 1, 0, 4051, 3937, 441, 1985, 0, 0, 3551, 0, 1, 3740, 153, 3503, 3821, 0, 3977, 4094, 0, 5, 4018, 0, 0, 0, 2614, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[293, 4073, 0, 0, 0, 0, 2309, 21, 0, 0, 2336, 0, 505, 3277, 0, 0, 4058, 0, 0, 3253, 3835, 2, 0, 0, 0, 0, 1924, 0, 0, 0, 3409, 0, 335, 2651, 0, 701, 2, 4092, 3988, 1, 905, 648, 114, 4096, 3873, 1, 3843, 1, 4092, 234, 4088, 0, 4068, 0, 7, 0, 968, 28, 0, 0, 4096, 7, 150, 12, 0, 4084, 0, 2, 0, 3841, 0, 1, 3943, 0, 0, 0, 3642, 1102, 2421, 183, 0, 4080, 0, 0, 0, 0, 0, 50, 3079, 0, 0, 3, 4082, 1344, 3950, 3383, 0, 0, 2095, 0, 2363, 41, 0, 0, 82, 366, 4096, 0, 0, 4041, 421, 0, 0, 47, 4036, 15, 0, 0, 0, 0, 160, 0, 626, 1196, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 89, 0, 270, 4096, 1, 4004, 365, 0, 4, 37, 1, 1335, 1514, 0, 4095, 0, 93, 3, 3229, 0, 0, 134, 0, 0, 1446, 37, 4, 0, 4085, 0, 204, 82, 68, 3526, 0, 706, 0, 0, 0, 4, 0, 0, 2362, 0, 0, 4071, 4059, 25, 0, 0, 4096, 3840, 0, 4096, 11, 1, 0, 0, 953, 3892, 1885, 0, 0, 357, 0, 3421, 0, 0, 0, 1945, 0, 0, 0, 1, 160, 4088, 1, 3778, 4, 4096, 0, 0, 0, 23, 0, 3811, 4096, 0, 0, 3, 0, 241, 1, 3107, 0, 0, 2849, 3608, 3998, 4096, 7, 91, 70, 100, 3278, 4096, 0, 0, 4083, 3943, 0, 3408, 1775, 5, 3366, 0, 0, 0, 19, 1, 360, 0, 2, 0, 0, 4061] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 4091, 29, 188, 4070, 37, 78, 2456, 0, 0, 254, 0, 0, 3478, 0, 263, 3994, 6, 0, 938, 3826, 4, 3695, 2313, 0, 0, 0, 0, 0, 208, 8, 0, 0, 7, 1126, 0, 0, 4034, 3606, 0, 1921, 1911, 0, 3439, 4059, 747, 0, 0, 4077, 0, 48, 0, 526, 4055, 0, 111, 11, 0, 544, 0, 5, 0, 25, 4090, 2779, 4084, 2071, 3, 208, 4096, 0, 0, 3187, 52, 10, 3047, 526, 0, 4094, 4086, 5, 4095, 0, 0, 0, 0, 4070, 0, 1092, 0, 0, 0, 0, 0, 549, 0, 0, 0, 3997, 4087, 60, 0, 0, 13, 0, 3876, 0, 3310, 1592, 92, 4, 0, 0, 0, 3219, 0, 2365, 99, 1741, 0, 4051, 0, 0, 216, 10, 0, 8, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 3448, 0, 0, 3232, 3722, 18, 0, 1480, 1133, 672, 0, 510, 0, 2223, 692, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 24, 3521, 0, 1389, 4095, 0, 1102, 888, 0, 0, 4096, 2958, 0, 0, 0, 76, 1052, 1, 954, 741, 406, 3734, 0, 4049, 3184, 0, 4095, 47, 0, 0, 38, 0, 0, 25, 46, 0, 3818, 0, 0, 3365, 3287, 4003, 0, 78, 4079, 0, 0, 4081, 944, 3650, 0, 3672, 0, 17, 373, 0, 16, 0, 3559, 564, 0, 4096, 2751, 4093, 0, 11, 0, 22, 4044, 4096, 0, 953, 0, 0, 115, 2, 0, 1, 0, 4096, 0, 2331, 0, 0, 2527, 4051, 1, 534, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4050, 0, 0, 3, 72, 58, 61, 982, 0, 0, 37, 1559, 1584, 2453, 72, 0, 2639, 1534, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 1682, 368, 595, 0, 0, 202, 0, 0, 507, 0, 0, 397, 4084, 0, 2206, 2, 602, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 0, 0, 0, 2029, 0, 3822, 3655, 0, 2860, 618, 0, 1016, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 422, 0, 0, 186, 0, 0, 326, 0, 20, 250, 4096, 3154, 4068, 38, 3616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 282, 939, 119, 0, 185, 23, 394, 285, 0, 0, 0, 823, 3567, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3494, 0, 0, 3766, 0, 243, 2, 4073, 0, 0, 0, 0, 1959, 777, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 2787, 0, 469, 0, 3600, 2987, 2, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 0, 15, 2, 3472, 642, 0, 19, 0, 4, 3726, 2422, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15530, 15188, 21832, 16180, 10485, 18410, 17984, 15463] avg=16384.0000 max=21832 max_rank=2 min=10485 min_rank=4 imbalance_ratio=1.332520 max_load_reduction=109240 max_load_reduction_pct=0.833435 moved_to_echo=62342 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[167, 42, 3, 0, 0, 10, 0, 1413, 0, 0, 0, 4069, 3364, 3917, 0, 19, 3304, 0, 972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3322, 4069, 0, 1906, 0, 2, 0, 1, 31, 699, 0, 0, 0, 1, 4083, 0, 0, 1, 491, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3010, 0, 12, 1, 0, 0, 3944, 150, 48, 1587, 0, 11, 774, 1474, 3831, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2192, 3574, 1, 3672, 332, 204, 2, 0, 975, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 33, 17, 2, 0, 66, 4075, 2, 4, 0, 0, 186, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 2853, 0, 418, 2821, 841, 0, 0, 4091, 0, 0, 3883, 0, 4063, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4088, 0, 2638, 0, 0, 0, 3236, 3489, 0, 32, 0, 1697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 544, 2128, 0, 13, 0, 3962, 4096, 0, 0, 0, 0, 29, 1033, 24, 0, 4096, 3102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17280, 15036, 18937, 14178, 8551, 22862, 15186, 19042] avg=16384.0000 max=22862 max_rank=5 min=8551 min_rank=4 imbalance_ratio=1.395386 max_load_reduction=108210 max_load_reduction_pct=0.825577 moved_to_echo=65641 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1353, 0, 0, 17, 0, 591, 0, 0, 0, 0, 0, 12, 3670, 0, 0, 57, 40, 969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3502, 953, 4096, 0, 344, 4, 0, 0, 0, 0, 0, 0, 1, 4095, 3754, 2, 57, 296, 99, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 3401, 919, 293, 1636, 0, 7, 0, 4096, 4096, 0, 0, 9, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 473, 54, 4088, 262, 3205, 3979, 0, 3987, 85, 0, 0, 489, 0, 433, 78, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 0, 3640, 0, 4096, 4078, 4028, 0, 0, 1, 232, 1787, 1329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 0, 0, 2368, 0, 4, 0, 4096, 1753, 1, 0, 4051, 3937, 441, 1985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3740, 153, 3503, 3821, 0, 3537, 4094, 0, 5, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2614, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3551, 3094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6709, 17643, 14972, 17137, 23288, 19190, 22872, 9261] avg=16384.0000 max=23288 max_rank=4 min=6709 min_rank=0 imbalance_ratio=1.421387 max_load_reduction=107784 max_load_reduction_pct=0.822327 moved_to_echo=74611 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1814, 0, 2192, 4077, 788, 4082, 0, 0, 3788, 0, 55, 0, 0, 590, 0, 1, 1778, 1562, 934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2365, 0, 0, 0, 3, 301, 597, 4093, 1, 0, 0, 0, 0, 0, 4096, 2, 2247, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 32, 0, 0, 2814, 0, 4096, 2779, 3616, 1, 417, 3173, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2415, 280, 3994, 0, 0, 4, 0, 0, 0, 29, 0, 1, 4088, 1918, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 4067, 0, 0, 4093, 1427, 697, 2, 2006, 0, 50, 4091, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 100, 0, 0, 31, 3704, 0, 0, 0, 0, 0, 3535, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 489, 0, 0, 0, 0, 0, 4095, 0, 1435, 1788, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 6, 230, 3687, 11, 4093, 4085, 1, 337, 3975, 1350, 677, 1729, 2983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21661, 15181, 21007, 13989, 20833, 7379, 7827, 23195] avg=16384.0000 max=23195 max_rank=7 min=7379 min_rank=5 imbalance_ratio=1.415710 max_load_reduction=107877 max_load_reduction_pct=0.823036 moved_to_echo=59234 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[293, 4073, 0, 0, 0, 0, 2309, 21, 0, 0, 2336, 0, 505, 3277, 0, 0, 3138, 1558, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 3253, 3835, 2, 0, 0, 0, 0, 1924, 0, 0, 0, 3409, 0, 2240, 1553, 599, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 2651, 0, 701, 2, 0, 3988, 1, 905, 648, 114, 4096, 735, 1, 3843, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2539, 234, 4088, 0, 4068, 0, 7, 0, 968, 28, 0, 0, 4096, 7, 150, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 0, 2, 0, 3841, 0, 1, 3943, 0, 0, 0, 3642, 1102, 2421, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3481, 0, 0, 0, 0, 0, 50, 3079, 0, 0, 3, 4082, 1344, 3950, 3383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2363, 41, 0, 0, 82, 366, 3646, 0, 0, 4041, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4036, 15, 0, 0, 0, 0, 160, 0, 626, 1196, 1, 0, 0, 0, 2095, 3116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18486, 21323, 18021, 16197, 15421, 19372, 10960, 11292] avg=16384.0000 max=21323 max_rank=1 min=10960 min_rank=6 imbalance_ratio=1.301453 max_load_reduction=109749 max_load_reduction_pct=0.837318 moved_to_echo=57045 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 29, 188, 4070, 37, 78, 2456, 0, 0, 254, 0, 0, 3478, 0, 263, 3289, 1558, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 6, 0, 938, 3826, 4, 3695, 2313, 0, 0, 0, 0, 0, 208, 8, 0, 2240, 1547, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1126, 0, 0, 0, 3606, 0, 1921, 1911, 0, 3439, 770, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2530, 0, 48, 0, 526, 4055, 0, 111, 11, 0, 544, 0, 5, 0, 25, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2779, 286, 2071, 3, 208, 4096, 0, 0, 3187, 52, 10, 3047, 526, 0, 4094, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3493, 0, 0, 0, 0, 4070, 0, 1092, 0, 0, 0, 0, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 60, 0, 0, 13, 0, 3876, 0, 3310, 1592, 92, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3219, 0, 2365, 99, 1741, 0, 4051, 0, 0, 216, 10, 0, 8, 0, 3997, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20753, 19381, 13527, 11945, 24445, 9209, 13034, 18778] avg=16384.0000 max=24445 max_rank=4 min=9209 min_rank=5 imbalance_ratio=1.492004 max_load_reduction=106627 max_load_reduction_pct=0.813499 moved_to_echo=65466 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 89, 0, 270, 4096, 1, 4004, 365, 0, 4, 37, 1, 1335, 1514, 0, 1914, 136, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 93, 3, 3229, 0, 0, 134, 0, 0, 1446, 37, 4, 0, 4085, 0, 195, 1540, 602, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 82, 68, 3526, 0, 0, 0, 0, 0, 4, 0, 0, 448, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2519, 25, 0, 0, 4096, 3840, 0, 4096, 11, 1, 0, 0, 953, 3892, 1885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 3421, 0, 0, 0, 1945, 0, 0, 0, 1, 160, 4088, 1, 3778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3494, 0, 0, 0, 23, 0, 3811, 4096, 0, 0, 3, 0, 241, 1, 3107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3608, 3998, 4096, 7, 91, 70, 100, 2918, 4096, 0, 0, 4083, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3408, 1775, 5, 3366, 0, 0, 0, 19, 1, 360, 0, 2, 0, 0, 4061, 2849, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13935, 15823, 8403, 21318, 13420, 14780, 27010, 16383] avg=16384.0000 max=27010 max_rank=6 min=8403 min_rank=2 imbalance_ratio=1.648560 max_load_reduction=104062 max_load_reduction_pct=0.793930 moved_to_echo=71593 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 3448, 0, 0, 3232, 3722, 18, 0, 1480, 719, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1133, 672, 0, 510, 0, 2223, 692, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1122, 599, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3521, 0, 1389, 4095, 0, 1102, 169, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 0, 0, 0, 76, 1052, 1, 954, 741, 406, 3734, 0, 4049, 3184, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 38, 0, 0, 25, 46, 0, 3818, 0, 0, 3365, 3287, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 3480, 0, 0, 4081, 944, 3650, 0, 3672, 0, 17, 373, 0, 16, 0, 3559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 564, 0, 0, 2751, 4093, 0, 11, 0, 22, 4044, 3646, 0, 953, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 4096, 0, 2331, 0, 0, 2527, 4051, 1, 534, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16718, 11497, 14373, 20128, 14629, 19870, 16199, 17658] avg=16384.0000 max=20128 max_rank=3 min=11497 min_rank=1 imbalance_ratio=1.228516 max_load_reduction=110944 max_load_reduction_pct=0.846436 moved_to_echo=68356 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[6400, 9600, 2432, 4352, 9344, 8320, 3072, 8960, 7680, 0, 2688, 8960, 9216, 18816, 1664, 1792, 16896, 6400, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[26624, 5760, 5888, 7040, 11904, 2560, 4992, 6784, 128, 768, 3968, 128, 128, 4736, 27648, 128, 9216, 7552, 3712, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[6784, 1280, 11264, 3712, 256, 15872, 1536, 7296, 15616, 9984, 128, 3200, 12672, 16640, 16512, 9472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 4096, 13824, 128, 13696, 128, 8320, 7808, 4352, 10112, 9856, 1536, 1024, 5120, 1408, 8576, 24960, 17408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12416, 1024, 12288, 384, 14464, 16512, 128, 13184, 2176, 768, 4608, 4608, 17408, 16768, 3456, 11904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[640, 128, 0, 14592, 14720, 11008, 3072, 3712, 4480, 11648, 20864, 18688, 2688, 5632, 16128, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 20864, 128, 640, 13056, 1920, 8064, 7680, 24320, 1792, 128, 4352, 8192, 15104, 5760, 19968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[4096, 6144, 6144, 4608, 1664, 0, 19968, 256, 16384, 18816, 3840, 11520, 3968, 5504, 12800, 16256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 4036, 3932, 3891, 1695, 0, 0, 10, 0, 35, 0, 0, 4096, 2, 21, 0, 63, 509, 4095, 0, 0, 0, 785, 196, 3488, 0, 0, 730, 348, 0, 4095, 0, 0, 0, 2, 0, 68, 5, 231, 3, 1868, 2883, 4077, 51, 6, 44, 3712, 810, 0, 923, 4008, 46, 3, 0, 0, 1501, 0, 2798, 2249, 563, 3839, 3952, 398, 11, 0, 2589, 0, 0, 0, 3275, 3184, 669, 338, 4071, 0, 3051, 4096, 1314, 1516, 697, 962, 19, 0, 0, 878, 21, 250, 0, 2740, 600, 3822, 122, 63, 0, 0, 3537, 0, 2394, 10, 0, 0, 0, 1, 162, 0, 3931, 126, 2913, 3225, 0, 52, 3904, 209, 0, 2732, 0, 0, 0, 1, 0, 0, 4056, 0, 3460, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 3869, 1204, 22, 88, 4086, 0, 4095, 0, 0, 0, 350, 160, 0, 0, 0, 128, 0, 0, 4, 0, 3563, 127, 0, 4007, 0, 818, 0, 2063, 0, 0, 1641, 0, 0, 2, 13, 0, 0, 4064, 4096, 159, 38, 2929, 4, 155, 312, 4090, 1, 0, 0, 4084, 3998, 0, 0, 23, 21, 0, 0, 0, 0, 99, 0, 0, 4095, 4054, 3, 0, 410, 0, 4085, 3918, 4091, 0, 0, 3018, 3775, 0, 3565, 2, 0, 0, 0, 0, 0, 0, 3087, 0, 0, 3766, 0, 222, 4007, 0, 0, 716, 1, 0, 1280, 3736, 0, 0, 0, 0, 0, 1321, 4090, 0, 0, 2870, 494, 0, 1, 4078, 3336, 3987, 1, 4093, 275, 4, 0, 214, 4060, 0, 4092, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3278, 3036, 0, 0, 2374, 0, 10, 225, 655, 2376, 3755, 0, 0, 255, 0, 0, 4028, 0, 0, 0, 17, 3712, 48, 4050, 0, 4074, 0, 1212, 0, 3796, 0, 1029, 0, 0, 1235, 11, 1794, 989, 5, 686, 4043, 1, 3454, 312, 566, 2939, 3, 61, 9, 4096, 3942, 1, 83, 3289, 7, 3, 1, 2643, 0, 261, 0, 4006, 0, 4095, 12, 4082, 258, 0, 3604, 3923, 0, 0, 0, 2, 20, 0, 0, 0, 58, 3424, 0, 0, 0, 0, 1195, 5, 303, 3873, 347, 0, 91, 0, 4096, 0, 0, 2496, 236, 4096, 3531, 173, 0, 0, 0, 0, 13, 0, 72, 891, 0, 442, 0, 2952, 2710, 583, 243, 0, 634, 2047, 30, 0, 4043, 0, 4096, 26] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[44, 0, 1, 1, 2372, 2, 0, 33, 930, 345, 0, 0, 0, 43, 4091, 0, 8, 1402, 0, 0, 36, 1795, 0, 0, 0, 1146, 0, 3189, 1310, 24, 12, 157, 39, 440, 0, 0, 3, 2, 0, 0, 0, 2239, 0, 4096, 192, 4096, 4096, 0, 9, 8, 3993, 111, 0, 4096, 1114, 0, 0, 20, 198, 0, 4071, 0, 0, 3755, 92, 0, 4094, 2627, 3411, 4067, 3357, 0, 3851, 229, 0, 0, 0, 0, 1773, 160, 0, 0, 0, 0, 0, 0, 0, 0, 3770, 0, 0, 3930, 1, 0, 22, 2, 4096, 3566, 0, 4094, 154, 403, 3, 87, 720, 0, 4060, 0, 4096, 0, 3766, 4082, 0, 73, 4096, 4096, 4090, 23, 1037, 259, 0, 0, 3409, 0, 0, 0, 3930, 27] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 499, 4096, 0, 0, 2890, 187, 2759, 0, 0, 3900, 4094, 0, 1, 1, 162, 3578, 0, 0, 0, 220, 20, 4096, 0, 4096, 0, 0, 2, 4096, 0, 1703, 4096, 0, 0, 0, 314, 0, 0, 2, 4092, 4086, 0, 4096, 2186, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 79, 1932, 0, 0, 1, 4096, 687, 3994, 4093, 167, 0, 1, 1614, 18, 4096, 4080, 65, 0, 1, 0, 2892, 0, 0, 70, 26, 0, 0, 0, 611, 0, 0, 0, 0, 3494, 0, 307, 30, 0, 0, 3968, 4096, 4032, 4, 2713, 5, 0, 68, 335, 4060, 1131, 139, 0, 3991, 3160, 0, 0, 0, 1116, 3665, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 1662, 1110] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 3714, 3750, 3937, 0, 0, 4091, 136, 343, 0, 1, 0, 0, 0, 2581, 175, 2, 0, 4027, 0, 0, 0, 2, 662, 3968, 0, 0, 4096, 0, 276, 3965, 1774, 32, 0, 84, 0, 4096, 22, 313, 0, 0, 0, 3849, 4090, 0, 305, 0, 4068, 0, 0, 0, 712, 1, 0, 0, 352, 0, 0, 4096, 95, 2389, 267, 0, 1, 0, 3434, 0, 0, 0, 4086, 3, 62, 0, 4096, 0, 2851, 0, 3933, 0, 0, 4096, 0, 0, 0, 54, 6, 0, 0, 2087, 0, 644, 0, 0, 27, 3637, 4095, 3524, 0, 4000, 0, 0, 3, 0, 884, 0, 3991, 160, 3033, 4061, 0, 0, 2954, 4096, 4079, 0, 0, 0, 0, 614, 0, 0, 4025, 258, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1642, 4, 25, 2, 0, 4096, 0, 0, 0, 0, 82, 0, 4096, 0, 3114, 0, 0, 4096, 0, 96, 0, 0, 0, 285, 0, 2848, 51, 148, 3888, 0, 0, 3313, 0, 4096, 18, 2970, 0, 3280, 0, 0, 0, 0, 4080, 3932, 6, 3782, 0, 0, 0, 4093, 0, 0, 0, 534, 749, 1, 959, 245, 0, 0, 0, 0, 0, 370, 0, 562, 1649, 3454, 0, 1650, 0, 0, 4045, 4063, 0, 71, 0, 177, 4093, 0, 0, 8, 0, 0, 0, 1206, 0, 0, 2171, 0, 0, 3827, 147, 0, 15, 2841, 2059, 2043, 3818, 1095, 0, 4096, 1, 2, 0, 4051, 4094, 0, 4, 0, 23, 48, 0, 661, 3929, 0, 4093, 3857, 4096, 609, 0, 0, 0, 674, 732, 0, 4029, 178] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[150, 0, 0, 3607, 4067, 0, 0, 0, 4092, 0, 3826, 257, 1325, 318, 3720, 0, 0, 0, 0, 29, 563, 0, 272, 0, 3120, 482, 535, 2495, 1723, 89, 0, 4038, 0, 4067, 0, 107, 3, 4092, 0, 0, 0, 415, 4063, 961, 0, 4086, 0, 139, 19, 2414, 7, 4091, 714, 0, 286, 0, 8, 2, 98, 0, 3487, 8, 4096, 4096, 0, 1, 3520, 4081, 0, 4094, 465, 0, 4087, 3902, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3663, 2, 2, 3174, 0, 0, 4092, 0, 12, 3035, 0, 0, 3912, 9, 1, 1849, 0, 0, 2322, 207, 4096, 0, 0, 3998, 0, 0, 5, 4096, 43, 5, 0, 1635, 0, 0, 0, 2726, 0, 5, 63, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 4036, 3932, 3891, 1695, 0, 0, 10, 0, 35, 0, 0, 4096, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 21, 0, 63, 509, 4095, 0, 0, 0, 785, 196, 3488, 0, 0, 730, 348, 3384, 515, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 68, 5, 231, 3, 1868, 2883, 3010, 51, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 3712, 810, 0, 923, 4008, 46, 3, 0, 0, 1501, 0, 2798, 2249, 563, 3839, 1130, 892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 398, 11, 0, 2589, 0, 0, 0, 0, 1687, 669, 338, 4071, 0, 3051, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1314, 1516, 697, 962, 19, 0, 0, 878, 21, 250, 0, 2740, 600, 3822, 122, 63, 3275, 2710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3537, 0, 2394, 10, 0, 0, 0, 1, 162, 0, 547, 126, 2913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 3904, 209, 0, 2732, 0, 0, 0, 1, 0, 0, 4056, 0, 3460, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17872, 14503, 12222, 22518, 20862, 18989, 9690, 14416] avg=16384.0000 max=22518 max_rank=3 min=9690 min_rank=6 imbalance_ratio=1.374390 max_load_reduction=108554 max_load_reduction_pct=0.828201 moved_to_echo=63957 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[44, 0, 1, 1, 2372, 2, 0, 33, 930, 345, 0, 0, 0, 43, 4091, 0, 1101, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1402, 0, 0, 36, 1795, 0, 0, 0, 1146, 0, 3189, 1310, 24, 12, 157, 3526, 653, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 440, 0, 0, 3, 2, 0, 0, 0, 2239, 0, 4096, 192, 3025, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 3993, 111, 0, 4096, 1114, 0, 0, 20, 198, 0, 4071, 0, 0, 3755, 81, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 4094, 2627, 3411, 4067, 3357, 0, 0, 122, 0, 0, 0, 0, 1773, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3770, 0, 0, 3930, 1, 0, 22, 2, 3851, 3429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3566, 0, 4094, 154, 403, 3, 87, 720, 0, 4060, 0, 570, 0, 3766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 4096, 4096, 2989, 23, 1037, 259, 0, 0, 3409, 0, 0, 0, 3930, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9138, 13284, 14132, 18352, 19703, 15005, 21519, 19939] avg=16384.0000 max=21519 max_rank=6 min=9138 min_rank=0 imbalance_ratio=1.313416 max_load_reduction=109553 max_load_reduction_pct=0.835823 moved_to_echo=76166 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3278, 3036, 0, 0, 2374, 0, 10, 225, 655, 2376, 3755, 0, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 4028, 0, 0, 0, 17, 3712, 48, 4050, 0, 4074, 0, 1212, 11, 142, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 0, 1029, 0, 0, 1235, 11, 1794, 989, 5, 686, 4043, 1, 3454, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 2939, 3, 61, 9, 4096, 3942, 1, 83, 3289, 7, 3, 1, 2643, 0, 261, 1393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 4095, 12, 4082, 258, 0, 0, 2077, 0, 0, 0, 2, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 3424, 0, 0, 0, 0, 1195, 5, 303, 3873, 347, 0, 91, 0, 3604, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2496, 236, 4096, 3531, 173, 0, 0, 0, 0, 2, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 442, 0, 2952, 1980, 583, 243, 0, 634, 2047, 30, 0, 4043, 0, 4096, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16439, 18002, 17355, 19297, 14552, 13649, 14702, 17076] avg=16384.0000 max=19297 max_rank=3 min=13649 min_rank=5 imbalance_ratio=1.177795 max_load_reduction=111775 max_load_reduction_pct=0.852776 moved_to_echo=59979 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 3714, 3750, 3937, 0, 0, 4091, 136, 343, 0, 1, 0, 0, 1098, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2581, 175, 2, 0, 4027, 0, 0, 0, 2, 662, 3968, 0, 0, 4096, 0, 3436, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 3965, 1774, 32, 0, 84, 0, 4096, 22, 313, 0, 0, 0, 2842, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 0, 4068, 0, 0, 0, 712, 1, 0, 0, 352, 0, 0, 4096, 95, 2389, 1, 842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 0, 1, 0, 3434, 0, 0, 0, 0, 2, 62, 0, 4096, 0, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 0, 0, 4096, 0, 0, 0, 54, 6, 0, 0, 2087, 0, 644, 0, 0, 4086, 3412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 3637, 4095, 3524, 0, 4000, 0, 0, 3, 0, 884, 0, 555, 160, 3033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2954, 4096, 2981, 0, 0, 0, 0, 614, 0, 0, 4025, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17242, 19598, 17494, 12861, 10713, 18318, 19918, 14928] avg=16384.0000 max=19918 max_rank=6 min=10713 min_rank=4 imbalance_ratio=1.215698 max_load_reduction=111154 max_load_reduction_pct=0.848038 moved_to_echo=63877 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 0, 499, 4096, 0, 0, 2890, 187, 2759, 0, 0, 3900, 4094, 0, 987, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 162, 3578, 0, 0, 0, 220, 20, 4096, 0, 4096, 0, 0, 2, 4096, 119, 506, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1703, 4096, 0, 0, 0, 314, 0, 0, 2, 4092, 4086, 0, 3025, 2186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 0, 0, 79, 1932, 0, 0, 1, 4096, 1449, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 687, 3994, 4093, 167, 0, 1, 1614, 18, 0, 2160, 65, 0, 1, 0, 2892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 26, 0, 0, 0, 611, 0, 0, 0, 0, 3494, 0, 307, 30, 0, 4096, 2654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 4096, 4032, 4, 2713, 5, 0, 68, 335, 4060, 1131, 20, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1116, 2678, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 1662, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19591, 17368, 19504, 12549, 15692, 11288, 24423, 10657] avg=16384.0000 max=24423 max_rank=6 min=10657 min_rank=7 imbalance_ratio=1.490662 max_load_reduction=106649 max_load_reduction_pct=0.813667 moved_to_echo=62060 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[12, 0, 0, 0, 3869, 1204, 22, 88, 4086, 0, 4095, 0, 0, 0, 350, 160, 898, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 4, 0, 3563, 127, 0, 4007, 0, 818, 0, 2063, 3521, 460, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1641, 0, 0, 2, 13, 0, 0, 4064, 4096, 159, 38, 2163, 4, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 312, 4090, 1, 0, 0, 4084, 3998, 0, 0, 23, 21, 0, 0, 0, 0, 99, 1391, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4054, 3, 0, 410, 0, 0, 2075, 4091, 0, 0, 3018, 3775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3565, 2, 0, 0, 0, 0, 0, 0, 3087, 0, 0, 3766, 0, 222, 4007, 0, 4085, 2410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 716, 1, 0, 1280, 3736, 0, 0, 0, 0, 0, 1321, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 0, 1, 4078, 2438, 3987, 1, 4093, 275, 4, 0, 214, 4060, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14909, 15143, 12335, 14660, 21521, 21144, 7623, 23737] avg=16384.0000 max=23737 max_rank=7 min=7623 min_rank=6 imbalance_ratio=1.448792 max_load_reduction=107335 max_load_reduction_pct=0.818901 moved_to_echo=74025 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[150, 0, 0, 3607, 4067, 0, 0, 0, 4092, 0, 3826, 257, 1325, 318, 3720, 0, 15, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 563, 0, 272, 0, 3120, 482, 535, 2495, 1723, 89, 0, 4038, 3529, 640, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 107, 3, 4092, 0, 0, 0, 415, 4063, 961, 0, 3009, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2414, 7, 4091, 714, 0, 286, 0, 8, 2, 98, 0, 3487, 8, 4096, 4096, 1389, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3520, 4081, 0, 4094, 465, 0, 0, 2059, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3663, 2, 2, 3174, 0, 0, 4092, 0, 4087, 3358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3035, 0, 0, 3912, 9, 1, 1849, 0, 0, 2322, 207, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 28, 5, 0, 1635, 0, 0, 0, 2726, 0, 5, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21556, 17969, 16856, 21613, 14222, 18379, 11914, 8563] avg=16384.0000 max=21613 max_rank=3 min=8563 min_rank=7 imbalance_ratio=1.319153 max_load_reduction=109459 max_load_reduction_pct=0.835106 moved_to_echo=53078 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1642, 4, 25, 2, 0, 4096, 0, 0, 0, 0, 82, 0, 4096, 0, 3114, 0, 1102, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 96, 0, 0, 0, 285, 0, 2848, 51, 148, 3888, 0, 0, 3313, 3, 8, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 18, 2970, 0, 3280, 0, 0, 0, 0, 4080, 3932, 6, 2792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 534, 749, 1, 959, 245, 0, 0, 0, 0, 0, 370, 1443, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 1649, 3454, 0, 1650, 0, 0, 0, 2151, 0, 71, 0, 177, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1206, 0, 0, 2171, 0, 0, 3827, 147, 0, 15, 2841, 4045, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2059, 2043, 3818, 1095, 0, 4096, 1, 2, 0, 4051, 4094, 0, 1, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 661, 3929, 0, 2991, 3857, 4096, 609, 0, 0, 0, 674, 732, 0, 4029, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14325, 15205, 21174, 9222, 13807, 14300, 21283, 21756] avg=16384.0000 max=21756 max_rank=7 min=9222 min_rank=3 imbalance_ratio=1.327881 max_load_reduction=109316 max_load_reduction_pct=0.834015 moved_to_echo=71146 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[1920, 128, 128, 3712, 21888, 20224, 7936, 1920, 14464, 4736, 11008, 896, 6144, 6656, 19200, 4352, 6016, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[10368, 17024, 15616, 15360, 8064, 19072, 3584, 2176, 896, 4480, 15616, 2688, 2944, 384, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 8448, 384, 3968, 5248, 9984, 384, 512, 6784, 13312, 1536, 25472, 7040, 5120, 4864, 15232, 17536, 3584, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1280, 17280, 8960, 8448, 1664, 16896, 10880, 128, 1152, 3584, 2304, 2048, 10368, 9088, 4864, 18944, 8320, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[5120, 9088, 17536, 18560, 9472, 13952, 6144, 128, 0, 12416, 4992, 512, 8192, 3200, 18560, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[8832, 1664, 896, 8576, 128, 1280, 640, 1024, 13952, 384, 384, 27008, 1152, 5120, 8448, 2944, 31232, 18816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[512, 1280, 14976, 20736, 16128, 11264, 5504, 6656, 1024, 2688, 3456, 3712, 21120, 384, 21376, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 22272, 7552, 4224, 128, 7552, 1664, 4224, 1920, 8320, 16384, 15872, 7168, 19968, 13952, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4038, 0, 4096, 29, 614, 0, 315, 0, 0, 0, 526, 3137, 4096, 4077, 0, 0, 0, 0, 48, 20, 210, 1, 0, 0, 0, 0, 4028, 0, 4096, 0, 211, 4055, 4078, 3902, 0, 46, 0, 24, 39, 47, 18, 3773, 29, 0, 1, 20, 8, 4, 3985, 2077, 0, 0, 154, 4089, 0, 15, 0, 0, 4080, 0, 0, 0, 8, 2080, 0, 4096, 91, 0, 0, 1, 16, 0, 0, 28, 4015, 0, 0, 0, 0, 1085, 3301, 3985, 2, 4055, 3254, 2879, 0, 0, 0, 4, 0, 92, 3852, 0, 73, 1958, 0, 3554, 4093, 0, 4096, 4096, 217, 0, 0, 2, 6, 395, 0, 1, 18, 0, 4092, 215, 3516, 3658, 4036, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 84, 48] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[348, 0, 3678, 3296, 0, 0, 4096, 0, 69, 0, 0, 0, 4096, 657, 2378, 0, 0, 58, 4089, 1094, 3061, 0, 168, 0, 0, 1, 0, 0, 3473, 0, 0, 0, 4073, 4096, 1835, 0, 0, 554, 0, 0, 72, 4096, 75, 294, 15, 43, 1255, 0, 0, 0, 0, 0, 0, 0, 3835, 157, 113, 115, 0, 0, 0, 0, 0, 4007, 1, 3489, 0, 0, 4, 1, 4093, 0, 91, 4093, 4094, 0, 0, 14, 0, 0, 4096, 4096, 0, 0, 4083, 0, 0, 621, 0, 0, 1, 3990, 4040, 1, 3426, 0, 2, 3718, 4096, 4084, 0, 3933, 0, 3, 0, 0, 0, 0, 0, 0, 2, 0, 0, 528, 7, 20, 3800, 3852, 3948, 0, 0, 4096, 0, 2548, 18, 0, 267, 719] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[419, 0, 1434, 1494, 57, 0, 1, 0, 133, 0, 138, 0, 4096, 2821, 1523, 0, 0, 929, 3146, 0, 4009, 197, 0, 0, 0, 0, 3917, 4062, 2025, 0, 5, 4083, 1371, 126, 4045, 85, 1093, 563, 0, 0, 0, 111, 290, 298, 694, 0, 4028, 0, 3782, 3840, 0, 0, 0, 109, 73, 0, 0, 0, 0, 25, 0, 26, 432, 3070, 27, 3670, 785, 301, 0, 49, 0, 0, 0, 2684, 0, 172, 17, 0, 0, 4096, 0, 0, 1, 0, 4095, 3949, 0, 0, 3596, 0, 2670, 2380, 3313, 0, 3537, 3753, 267, 4061, 2937, 0, 1900, 4054, 0, 3747, 0, 0, 8, 1, 0, 2, 152, 194, 0, 0, 0, 712, 4083, 3944, 4013, 6, 5, 1182, 46, 15, 2098, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 243, 1093, 0, 0, 0, 0, 28, 0, 0, 4050, 4094, 0, 0, 0, 0, 4067, 8, 3930, 647, 1631, 0, 0, 4037, 3344, 0, 4085, 0, 13, 0, 58, 3271, 3842, 0, 0, 0, 0, 0, 4093, 4095, 1904, 0, 65, 996, 4096, 0, 4071, 4077, 0, 0, 4094, 0, 0, 0, 81, 3, 2862, 0, 0, 10, 0, 56, 1, 659, 0, 0, 45, 0, 4096, 0, 712, 0, 27, 0, 0, 0, 3087, 3947, 2512, 806, 2, 0, 0, 4079, 0, 0, 2, 1897, 0, 4096, 4089, 0, 0, 4095, 0, 0, 3388, 0, 0, 4094, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 1332, 230, 4096, 0, 164, 2326, 18, 0, 88, 4042, 0, 4092, 0, 15, 0, 4086] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 4052, 0, 0, 0, 0, 3580, 4090, 0, 0, 0, 249, 4093, 558, 0, 1488, 0, 4074, 398, 4052, 1691, 3, 4095, 635, 3087, 0, 0, 0, 0, 0, 422, 1078, 3980, 1713, 2122, 0, 32, 0, 1, 0, 1260, 11, 131, 0, 0, 4096, 0, 0, 0, 0, 1059, 4090, 1306, 2003, 15, 209, 0, 0, 4014, 0, 2, 0, 1398, 3884, 0, 314, 0, 0, 5, 22, 0, 4094, 3764, 30, 4088, 3962, 784, 25, 0, 1832, 4094, 0, 414, 2879, 0, 249, 0, 0, 5, 4095, 18, 0, 20, 14, 0, 1065, 0, 2813, 3945, 0, 1, 1, 0, 0, 0, 0, 0, 0, 68, 1918, 5, 0, 2849, 0, 3809, 38, 3600, 2478, 4077, 1, 1, 0, 173, 397, 60, 0, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[1464, 0, 4076, 50, 4095, 4096, 0, 0, 103, 12, 0, 0, 3381, 0, 0, 405, 0, 0, 0, 0, 4083, 589, 0, 2, 0, 0, 961, 0, 3750, 72, 2748, 2, 0, 4023, 3867, 1176, 1, 0, 0, 0, 133, 1528, 0, 0, 336, 131, 3817, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 228, 712, 2975, 4096, 0, 164, 0, 2430, 255, 3792, 125, 0, 15, 0, 0, 1, 3929, 0, 0, 0, 0, 0, 0, 2152, 3658, 3904, 0, 0, 0, 2600, 0, 720, 4096, 0, 1490, 0, 4096, 5, 582, 2253, 3834, 4021, 4059, 2115, 3503, 221, 2, 0, 0, 0, 0, 3307, 71, 1, 75, 0, 3728, 620, 0, 516, 2994, 4096, 3968, 149, 0, 208, 0, 2, 0, 24, 69, 298] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4071, 0, 3, 97, 2757, 29, 3977, 1, 102, 0, 8, 1958, 4084, 6, 10, 0, 0, 0, 4066, 22, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 0, 0, 3510, 4015, 3808, 4070, 0, 0, 0, 1, 169, 3893, 0, 986, 0, 0, 716, 0, 0, 2135, 0, 31, 0, 4094, 4093, 1722, 0, 0, 40, 2492, 5, 780, 0, 0, 2975, 1232, 56, 0, 3598, 1557, 0, 0, 1512, 0, 0, 1, 4079, 0, 0, 4084, 0, 1, 4093, 0, 16, 3326, 513, 442, 418, 3906, 37, 6, 4096, 2417, 10, 14, 674, 0, 4086, 4041, 3150, 0, 4096, 19, 0, 3, 0, 0, 0, 0, 29, 0, 0, 728, 156, 0, 26, 4044, 28, 4095, 0, 0, 4094, 0, 0, 0, 1195, 0, 364] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 3789, 0, 4096, 4, 3921, 220, 0, 138, 739, 358, 4092, 1994, 0, 53, 1, 0, 0, 0, 4096, 0, 4063, 77, 84, 0, 4096, 0, 4022, 0, 1933, 0, 1009, 4072, 3896, 1, 0, 13, 0, 0, 3853, 4096, 0, 3, 0, 0, 0, 0, 2756, 0, 1391, 0, 61, 4096, 0, 8, 0, 31, 3089, 0, 0, 24, 0, 2603, 0, 4054, 0, 0, 0, 1216, 3711, 668, 2, 0, 4096, 0, 0, 355, 0, 0, 0, 1280, 271, 0, 0, 31, 0, 0, 4004, 978, 0, 4057, 3321, 161, 12, 4078, 0, 4063, 4083, 3017, 0, 56, 273, 0, 7, 0, 2282, 0, 0, 2287, 0, 0, 4095, 0, 3177, 0, 1849, 185, 4096, 0, 0, 0, 1, 0, 0, 527, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4071, 0, 3, 97, 2757, 29, 3977, 1, 102, 0, 8, 1958, 3779, 6, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 22, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 0, 0, 3510, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 3808, 4070, 0, 0, 0, 1, 169, 3893, 0, 986, 0, 0, 716, 0, 0, 390, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2135, 0, 31, 0, 4094, 4093, 1722, 0, 0, 40, 2492, 5, 780, 0, 0, 2975, 2550, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1232, 56, 0, 3598, 1557, 0, 0, 1512, 0, 0, 1, 4079, 0, 0, 4084, 0, 2121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 16, 3326, 513, 442, 418, 3906, 37, 6, 839, 0, 10, 14, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4041, 3150, 0, 4096, 19, 0, 3, 0, 0, 0, 0, 29, 0, 0, 1340, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 728, 156, 0, 26, 4044, 28, 2365, 0, 0, 4094, 0, 0, 0, 1195, 0, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16798, 11766, 18289, 21213, 18240, 14295, 17471, 13000] avg=16384.0000 max=21213 max_rank=3 min=11766 min_rank=1 imbalance_ratio=1.294739 max_load_reduction=109859 max_load_reduction_pct=0.838158 moved_to_echo=63006 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[419, 0, 1434, 1494, 57, 0, 1, 0, 133, 0, 138, 0, 3789, 2821, 1523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 929, 3146, 0, 4009, 197, 0, 0, 0, 0, 3917, 4062, 2025, 0, 5, 4083, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1371, 126, 4045, 85, 1093, 563, 0, 0, 0, 111, 290, 298, 694, 0, 4028, 0, 383, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3782, 3840, 0, 0, 0, 109, 73, 0, 0, 0, 0, 25, 0, 26, 432, 3070, 1481, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 3670, 785, 301, 0, 49, 0, 0, 0, 2684, 0, 172, 17, 0, 0, 4096, 2908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 3949, 0, 0, 3596, 0, 2670, 489, 0, 0, 3537, 3753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 4061, 2937, 0, 1900, 4054, 0, 3747, 0, 0, 8, 1, 0, 2, 152, 194, 1313, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 712, 4083, 3944, 2317, 6, 5, 1182, 46, 15, 2098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11809, 22438, 13329, 13243, 14709, 22090, 19046, 14408] avg=16384.0000 max=22438 max_rank=1 min=11809 min_rank=0 imbalance_ratio=1.369507 max_load_reduction=108634 max_load_reduction_pct=0.828812 moved_to_echo=70253 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[348, 0, 3678, 3296, 0, 0, 4096, 0, 69, 0, 0, 0, 3789, 657, 2378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 4089, 1094, 3061, 0, 168, 0, 0, 1, 0, 0, 3473, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4096, 1835, 0, 0, 554, 0, 0, 72, 4096, 75, 294, 15, 43, 1255, 0, 376, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3835, 157, 113, 115, 0, 0, 0, 0, 0, 4007, 2484, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3489, 0, 0, 4, 1, 4093, 0, 91, 4093, 4094, 0, 0, 14, 0, 0, 3546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4083, 0, 0, 621, 0, 0, 1, 818, 0, 1, 3426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3718, 4096, 4084, 0, 3933, 0, 3, 0, 0, 0, 0, 0, 0, 2, 0, 1292, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 7, 20, 3800, 3852, 2280, 0, 0, 4096, 0, 2548, 18, 0, 267, 719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18311, 12009, 17026, 11205, 19426, 17142, 17818, 18135] avg=16384.0000 max=19426 max_rank=4 min=11205 min_rank=3 imbalance_ratio=1.185669 max_load_reduction=111646 max_load_reduction_pct=0.851791 moved_to_echo=72521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4038, 0, 4096, 29, 614, 0, 315, 0, 0, 0, 526, 3137, 3789, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 20, 210, 1, 0, 0, 0, 0, 4028, 0, 4096, 0, 211, 4055, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 3902, 0, 46, 0, 24, 39, 47, 18, 3773, 29, 0, 1, 20, 8, 4, 389, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 2077, 0, 0, 154, 4089, 0, 15, 0, 0, 4080, 0, 0, 0, 8, 2080, 57, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 91, 0, 0, 1, 16, 0, 0, 28, 4015, 0, 0, 0, 0, 1085, 3381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3301, 3985, 2, 4055, 3254, 2879, 0, 0, 0, 4, 0, 20, 0, 0, 73, 1958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3554, 4093, 0, 4096, 4096, 217, 0, 0, 2, 6, 395, 0, 1, 18, 0, 1337, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 215, 3516, 3658, 4036, 0, 2358, 0, 0, 0, 0, 0, 0, 0, 84, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20621, 12734, 12620, 17016, 12713, 19531, 17830, 18007] avg=16384.0000 max=20621 max_rank=0 min=12620 min_rank=2 imbalance_ratio=1.258606 max_load_reduction=110451 max_load_reduction_pct=0.842674 moved_to_echo=68081 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1464, 0, 4076, 50, 4095, 4096, 0, 0, 103, 12, 0, 0, 3129, 0, 0, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 589, 0, 2, 0, 0, 961, 0, 3750, 72, 2748, 2, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 3867, 1176, 1, 0, 0, 0, 133, 1528, 0, 0, 336, 131, 3817, 0, 378, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 228, 712, 2975, 4096, 0, 164, 0, 2430, 0, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 3792, 125, 0, 15, 0, 0, 1, 3929, 0, 0, 0, 0, 0, 0, 2152, 3596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3658, 3904, 0, 0, 0, 2600, 0, 720, 4096, 0, 1490, 0, 0, 5, 582, 2253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3834, 4021, 4059, 2115, 3503, 221, 2, 0, 0, 0, 0, 3307, 71, 1, 75, 0, 1299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3728, 620, 0, 516, 2994, 4096, 2291, 149, 0, 208, 0, 2, 0, 24, 69, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17430, 12260, 15589, 15117, 13865, 19308, 22508, 14995] avg=16384.0000 max=22508 max_rank=6 min=12260 min_rank=1 imbalance_ratio=1.373779 max_load_reduction=108564 max_load_reduction_pct=0.828278 moved_to_echo=70676 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 243, 1093, 0, 0, 0, 0, 28, 0, 0, 3741, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 8, 3930, 647, 1631, 0, 0, 4037, 3344, 0, 4085, 0, 13, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 3271, 3842, 0, 0, 0, 0, 0, 4093, 4095, 1904, 0, 65, 996, 4096, 0, 6, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 4077, 0, 0, 4094, 0, 0, 0, 81, 3, 2862, 0, 0, 10, 0, 56, 2552, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 659, 0, 0, 45, 0, 4096, 0, 712, 0, 27, 0, 0, 0, 3087, 3947, 3589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2512, 806, 2, 0, 0, 4079, 0, 0, 2, 1897, 0, 834, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3388, 0, 0, 4094, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 9, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1332, 230, 4096, 0, 164, 2326, 3, 0, 88, 4042, 0, 4092, 0, 15, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9200, 21829, 22668, 18306, 16163, 14227, 8205, 20474] avg=16384.0000 max=22668 max_rank=2 min=8205 min_rank=6 imbalance_ratio=1.383545 max_load_reduction=108404 max_load_reduction_pct=0.827057 moved_to_echo=59069 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4052, 0, 0, 0, 0, 3580, 4090, 0, 0, 0, 249, 3787, 558, 0, 1488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 398, 4052, 1691, 3, 4095, 635, 3087, 0, 0, 0, 0, 0, 422, 1078, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 1713, 2122, 0, 32, 0, 1, 0, 1260, 11, 131, 0, 0, 4096, 0, 0, 389, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1059, 4090, 1306, 2003, 15, 209, 0, 0, 4014, 0, 2, 0, 1398, 3884, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 5, 22, 0, 4094, 3764, 30, 4088, 3962, 784, 25, 0, 1832, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 414, 2879, 0, 249, 0, 0, 5, 4095, 18, 0, 0, 14, 0, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2813, 3945, 0, 1, 1, 0, 0, 0, 0, 0, 0, 68, 1918, 5, 0, 1334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2849, 0, 3809, 38, 3600, 2478, 2354, 1, 1, 0, 173, 397, 60, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17804, 19600, 13976, 17982, 18938, 12833, 10085, 19854] avg=16384.0000 max=19854 max_rank=7 min=10085 min_rank=6 imbalance_ratio=1.211792 max_load_reduction=111218 max_load_reduction_pct=0.848526 moved_to_echo=61710 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3789, 0, 4096, 4, 3921, 220, 0, 138, 739, 358, 3787, 1994, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4063, 77, 84, 0, 4096, 0, 4022, 0, 1933, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 4072, 3896, 1, 0, 13, 0, 0, 3853, 4096, 0, 3, 0, 0, 0, 0, 391, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2756, 0, 1391, 0, 61, 4096, 0, 8, 0, 31, 3089, 0, 0, 24, 0, 2603, 2526, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 1216, 3711, 668, 2, 0, 4096, 0, 0, 355, 0, 0, 2916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1280, 271, 0, 0, 31, 0, 0, 4004, 978, 0, 831, 0, 161, 12, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4083, 3017, 0, 56, 273, 0, 7, 0, 2282, 0, 0, 2287, 0, 0, 1341, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3177, 0, 1849, 185, 2364, 0, 0, 0, 1, 0, 0, 527, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19099, 18436, 17575, 16990, 17018, 11646, 18109, 12199] avg=16384.0000 max=19099 max_rank=0 min=11646 min_rank=5 imbalance_ratio=1.165710 max_load_reduction=111973 max_load_reduction_pct=0.854286 moved_to_echo=58972 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[16768, 10112, 2560, 4096, 9728, 18432, 5760, 512, 512, 1024, 19584, 4224, 896, 256, 1920, 21120, 11776, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[1536, 20224, 1024, 3968, 1664, 1408, 12032, 6400, 8576, 6912, 16384, 8320, 896, 512, 7296, 13184, 22144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[17664, 18176, 768, 7040, 14848, 14336, 512, 1792, 15616, 7040, 4224, 3840, 0, 256, 7680, 17920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[10368, 4096, 17152, 5248, 12800, 4224, 16000, 4352, 512, 256, 1536, 5760, 29696, 14208, 3968, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 5120, 15872, 5248, 21120, 1536, 9984, 768, 3200, 4096, 16384, 4096, 25600, 128, 5376, 12800, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[16896, 1792, 14720, 4992, 24576, 17024, 16384, 256, 128, 13696, 256, 7168, 2176, 1792, 512, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[4224, 26368, 30720, 12416, 9600, 20608, 512, 3840, 128, 128, 2304, 3712, 256, 4352, 256, 256, 9344, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[18688, 25088, 23680, 1408, 1152, 1280, 128, 256, 13440, 17792, 3456, 640, 1152, 6016, 13312, 128, 2816, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[389, 2702, 324, 486, 1591, 295, 707, 929, 1379, 142, 1439, 164, 1073, 893, 168, 380, 67, 507, 975, 639, 673, 213, 3055, 2131, 629, 847, 817, 993, 968, 487, 1710, 1533, 2696, 1113, 345, 780, 67, 1772, 1712, 700, 2492, 739, 1796, 824, 1686, 1359, 824, 944, 320, 353, 502, 2197, 990, 487, 305, 80, 619, 372, 1434, 1708, 1876, 438, 426, 1132, 663, 1594, 774, 1185, 1268, 296, 316, 1956, 866, 1953, 229, 1948, 411, 792, 1980, 1089, 2270, 1559, 2542, 518, 1635, 49, 773, 1645, 1727, 284, 824, 918, 990, 419, 373, 940, 1945, 484, 1124, 617, 1574, 324, 2503, 2406, 524, 1293, 371, 1646, 531, 338, 1838, 1087, 419, 1189, 602, 1730, 364, 386, 1083, 1287, 626, 1380, 1535, 1253, 1949, 532, 535, 418] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[52, 3102, 472, 255, 1436, 313, 831, 895, 1431, 80, 1521, 126, 1034, 1091, 452, 382, 48, 414, 734, 434, 1129, 212, 2704, 1965, 834, 892, 889, 863, 1158, 396, 1815, 1861, 2843, 544, 360, 279, 78, 2372, 1937, 523, 3002, 214, 1478, 546, 1274, 1699, 1059, 922, 292, 322, 251, 1579, 582, 498, 363, 80, 678, 352, 1058, 1983, 1480, 362, 665, 1279, 662, 1861, 477, 1512, 850, 174, 679, 2227, 897, 2064, 206, 1735, 441, 1201, 2320, 1035, 2543, 747, 2598, 611, 2006, 160, 598, 1139, 1727, 204, 792, 1016, 773, 810, 382, 888, 2410, 319, 1713, 953, 1191, 836, 1476, 2877, 649, 949, 532, 2136, 750, 576, 2396, 1176, 214, 957, 1194, 1002, 153, 518, 888, 1459, 769, 1656, 1937, 706, 1322, 333, 528, 387] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[52, 2341, 450, 213, 1368, 264, 1031, 879, 942, 133, 1099, 232, 983, 684, 278, 267, 32, 340, 494, 685, 1184, 66, 3321, 1817, 657, 854, 965, 993, 1245, 144, 1300, 2077, 3458, 809, 212, 757, 278, 1857, 2592, 679, 2691, 145, 1093, 551, 1251, 1275, 1081, 620, 217, 366, 265, 1894, 726, 494, 649, 103, 635, 407, 562, 2336, 1895, 405, 997, 1119, 712, 2040, 615, 1591, 1488, 183, 764, 1859, 882, 2401, 422, 1710, 987, 770, 2055, 637, 2356, 947, 2726, 395, 2062, 147, 689, 1168, 1790, 140, 774, 1165, 819, 569, 470, 764, 2427, 206, 2123, 812, 1273, 823, 2086, 2316, 744, 545, 441, 2336, 690, 377, 2190, 1281, 257, 797, 1494, 1248, 426, 592, 756, 1351, 799, 1772, 1575, 653, 1632, 351, 466, 327] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[122, 2812, 827, 244, 1398, 396, 751, 1024, 800, 63, 1166, 224, 1324, 655, 143, 428, 249, 432, 1318, 1426, 670, 58, 2675, 1366, 450, 708, 878, 995, 1637, 490, 1135, 1904, 3364, 711, 350, 393, 456, 1893, 2137, 405, 2577, 327, 1294, 899, 1482, 1301, 885, 1399, 373, 135, 383, 1916, 1006, 466, 782, 220, 613, 191, 787, 1689, 1395, 715, 648, 1047, 893, 1616, 579, 867, 1287, 312, 1012, 1659, 1369, 2273, 374, 2253, 990, 540, 2323, 818, 2576, 1249, 2829, 253, 2140, 101, 620, 1265, 1549, 181, 928, 920, 865, 378, 367, 848, 1632, 276, 2197, 469, 1454, 889, 1638, 1902, 714, 961, 711, 2642, 668, 265, 1872, 1025, 294, 1524, 931, 1428, 358, 522, 696, 1404, 787, 1727, 1516, 432, 1490, 403, 389, 615] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[233, 2944, 699, 404, 1195, 420, 744, 949, 1376, 172, 1461, 325, 1101, 664, 116, 342, 166, 637, 1488, 591, 890, 144, 2621, 1649, 266, 875, 1101, 813, 1459, 490, 1673, 1679, 2789, 547, 307, 502, 104, 2086, 2062, 437, 2777, 368, 1441, 945, 1196, 1468, 1167, 1154, 311, 291, 252, 2158, 890, 576, 467, 115, 395, 860, 898, 1550, 1379, 757, 706, 1213, 801, 2272, 639, 1111, 848, 259, 455, 1983, 560, 1323, 179, 1845, 552, 989, 2945, 609, 2740, 1185, 2682, 510, 1598, 353, 834, 1222, 1531, 238, 560, 1160, 726, 410, 452, 873, 2101, 300, 1839, 559, 1725, 611, 1844, 2723, 592, 1004, 448, 1412, 574, 247, 2227, 787, 217, 1237, 987, 1693, 275, 522, 1594, 1311, 750, 1184, 1558, 884, 1746, 514, 566, 742] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[44, 2609, 811, 329, 1564, 442, 1026, 716, 746, 115, 1547, 484, 1141, 494, 159, 370, 147, 346, 751, 584, 896, 40, 3344, 1861, 573, 884, 862, 702, 921, 206, 969, 1432, 2941, 481, 330, 525, 224, 1465, 1485, 540, 2750, 167, 959, 648, 1069, 1571, 1409, 878, 254, 366, 277, 1570, 738, 366, 766, 95, 818, 762, 1381, 1638, 1622, 503, 608, 822, 821, 2079, 360, 1345, 1599, 236, 712, 2081, 837, 2726, 277, 1400, 780, 896, 2421, 1353, 3025, 1102, 2577, 488, 1928, 213, 484, 1691, 2029, 295, 709, 864, 1277, 419, 350, 1135, 1828, 354, 1913, 810, 1931, 849, 2071, 3016, 747, 776, 669, 1989, 908, 475, 2126, 1311, 561, 553, 747, 1096, 156, 403, 960, 1502, 619, 1427, 1553, 982, 1166, 873, 703, 346] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[39, 2602, 463, 363, 1762, 362, 708, 607, 1047, 90, 1789, 224, 979, 616, 160, 477, 249, 870, 512, 777, 616, 181, 2622, 2107, 503, 1378, 997, 1059, 889, 311, 1246, 2251, 3261, 641, 459, 420, 169, 2123, 2026, 653, 2578, 350, 1250, 615, 1626, 1830, 960, 1319, 304, 315, 391, 2004, 674, 668, 557, 76, 1128, 348, 896, 1817, 1316, 603, 399, 895, 1081, 1459, 650, 1519, 1666, 303, 822, 1226, 1237, 1919, 317, 2175, 1016, 636, 2625, 650, 2562, 1316, 2709, 210, 1869, 304, 587, 793, 1659, 274, 665, 1167, 760, 584, 508, 665, 2230, 165, 1590, 530, 1254, 684, 1491, 2660, 696, 942, 590, 2095, 647, 310, 1912, 849, 244, 950, 856, 1256, 234, 403, 883, 1300, 1129, 1983, 1757, 697, 1988, 310, 462, 515] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[62, 2296, 774, 214, 1195, 281, 862, 614, 1380, 172, 1527, 234, 418, 907, 269, 397, 94, 510, 1589, 699, 942, 325, 2782, 2017, 567, 1164, 848, 1085, 1366, 333, 2061, 1603, 2793, 435, 541, 396, 121, 2269, 1723, 1057, 2378, 272, 1862, 992, 1572, 1256, 800, 896, 277, 369, 218, 1789, 607, 527, 612, 85, 440, 394, 817, 1815, 1801, 388, 930, 1025, 866, 1718, 849, 1175, 615, 306, 570, 1341, 532, 2355, 296, 1879, 854, 1073, 2496, 482, 2289, 1097, 2363, 507, 2083, 411, 1065, 950, 1630, 224, 777, 1054, 459, 648, 416, 781, 2463, 342, 1535, 615, 1559, 755, 1362, 2959, 769, 865, 621, 2161, 498, 473, 1985, 1010, 172, 1600, 1389, 1887, 208, 591, 1143, 1047, 819, 1565, 1553, 382, 2066, 307, 492, 674] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[52, 3102, 472, 255, 1436, 313, 831, 895, 1431, 80, 1521, 126, 1034, 1091, 452, 382, 2774, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 414, 734, 434, 1129, 212, 2704, 1965, 834, 892, 889, 863, 1158, 396, 1815, 1861, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 544, 360, 279, 78, 2372, 1937, 523, 3002, 214, 1478, 546, 1274, 1699, 1059, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 322, 251, 1579, 582, 498, 363, 80, 678, 352, 1058, 1983, 1480, 362, 665, 1279, 2042, 1809, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 1861, 477, 1512, 850, 174, 679, 2227, 897, 2064, 206, 1735, 441, 1201, 278, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2543, 747, 1798, 611, 2006, 160, 598, 1139, 1727, 204, 792, 1016, 773, 810, 382, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1423, 319, 1713, 953, 1191, 836, 1476, 0, 649, 949, 532, 2136, 750, 576, 2396, 1176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 957, 1194, 1002, 153, 518, 888, 1459, 769, 1656, 1937, 706, 1322, 333, 528, 387, 888, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17315, 16385, 16356, 15774, 16299, 16194, 17075, 15674] avg=16384.0000 max=17315 max_rank=0 min=15674 min_rank=7 imbalance_ratio=1.056824 max_load_reduction=113757 max_load_reduction_pct=0.867897 moved_to_echo=65242 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[389, 2702, 324, 486, 1591, 295, 707, 929, 1379, 142, 1439, 164, 1073, 893, 168, 380, 2635, 894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 507, 975, 639, 673, 213, 3055, 2131, 629, 847, 817, 993, 968, 487, 1710, 1533, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 1113, 345, 780, 67, 1772, 1712, 700, 2492, 739, 1796, 824, 1686, 1359, 824, 944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 353, 502, 2197, 990, 487, 305, 80, 619, 372, 1434, 1708, 1876, 438, 426, 1132, 1746, 1512, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 663, 1594, 774, 1185, 1268, 296, 316, 1956, 866, 1953, 229, 1948, 411, 792, 234, 1089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2270, 1559, 1750, 518, 1635, 49, 773, 1645, 1727, 284, 824, 918, 990, 419, 373, 940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1142, 484, 1124, 617, 1574, 324, 2503, 0, 524, 1293, 371, 1646, 531, 338, 1838, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 419, 1189, 602, 1730, 364, 386, 1083, 1287, 626, 1380, 1535, 1253, 1949, 532, 535, 418, 720, 752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16590, 16284, 17214, 16580, 15574, 16674, 15396, 16760] avg=16384.0000 max=17214 max_rank=2 min=15396 min_rank=6 imbalance_ratio=1.050659 max_load_reduction=113858 max_load_reduction_pct=0.868668 moved_to_echo=64404 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[52, 2341, 450, 213, 1368, 264, 1031, 879, 942, 133, 1099, 232, 983, 684, 278, 267, 3374, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 340, 494, 685, 1184, 66, 3321, 1817, 657, 854, 965, 993, 1245, 144, 1300, 2077, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 809, 212, 757, 278, 1857, 2592, 679, 2691, 145, 1093, 551, 1251, 1275, 1081, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 366, 265, 1894, 726, 494, 649, 103, 635, 407, 562, 2336, 1895, 405, 997, 1119, 1809, 1456, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 712, 2040, 615, 1591, 1488, 183, 764, 1859, 882, 2401, 422, 1710, 987, 770, 246, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2356, 947, 1886, 395, 2062, 147, 689, 1168, 1790, 140, 774, 1165, 819, 569, 470, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1433, 206, 2123, 812, 1273, 823, 2086, 0, 744, 545, 441, 2336, 690, 377, 2190, 1281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 797, 1494, 1248, 426, 592, 756, 1351, 799, 1772, 1575, 653, 1632, 351, 466, 327, 894, 801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15450, 16213, 15975, 16435, 17307, 16141, 17360, 16191] avg=16384.0000 max=17360 max_rank=6 min=15450 min_rank=0 imbalance_ratio=1.059570 max_load_reduction=113712 max_load_reduction_pct=0.867554 moved_to_echo=66999 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[233, 2944, 699, 404, 1195, 420, 744, 949, 1376, 172, 1461, 325, 1101, 664, 116, 342, 2721, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 637, 1488, 591, 890, 144, 2621, 1649, 266, 875, 1101, 813, 1459, 490, 1673, 1679, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 547, 307, 502, 104, 2086, 2062, 437, 2777, 368, 1441, 945, 1196, 1468, 1167, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 291, 252, 2158, 890, 576, 467, 115, 395, 860, 898, 1550, 1379, 757, 706, 1213, 2592, 1711, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 801, 2272, 639, 1111, 848, 259, 455, 1983, 560, 1323, 179, 1845, 552, 989, 353, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2740, 1185, 1856, 510, 1598, 353, 834, 1222, 1531, 238, 560, 1160, 726, 410, 452, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1241, 300, 1839, 559, 1725, 611, 1844, 0, 592, 1004, 448, 1412, 574, 247, 2227, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 1237, 987, 1693, 275, 522, 1594, 1311, 750, 1184, 1558, 884, 1746, 514, 566, 742, 774, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16878, 16580, 16629, 17207, 14778, 16248, 15410, 17342] avg=16384.0000 max=17342 max_rank=7 min=14778 min_rank=4 imbalance_ratio=1.058472 max_load_reduction=113730 max_load_reduction_pct=0.867691 moved_to_echo=63778 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[122, 2812, 827, 244, 1398, 396, 751, 1024, 800, 63, 1166, 224, 1324, 655, 143, 428, 3283, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 432, 1318, 1426, 670, 58, 2675, 1366, 450, 708, 878, 995, 1637, 490, 1135, 1904, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 711, 350, 393, 456, 1893, 2137, 405, 2577, 327, 1294, 899, 1482, 1301, 885, 1399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 135, 383, 1916, 1006, 466, 782, 220, 613, 191, 787, 1689, 1395, 715, 648, 1047, 2044, 1195, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 893, 1616, 579, 867, 1287, 312, 1012, 1659, 1369, 2273, 374, 2253, 990, 540, 279, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2576, 1249, 1958, 253, 2140, 101, 620, 1265, 1549, 181, 928, 920, 865, 378, 367, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 964, 276, 2197, 469, 1454, 889, 1638, 0, 714, 961, 711, 2642, 668, 265, 1872, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 1524, 931, 1428, 358, 522, 696, 1404, 787, 1727, 1516, 432, 1490, 403, 389, 615, 601, 831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16367, 16431, 16590, 15672, 17121, 16198, 16745, 15948] avg=16384.0000 max=17121 max_rank=4 min=15672 min_rank=3 imbalance_ratio=1.044983 max_load_reduction=113951 max_load_reduction_pct=0.869377 moved_to_echo=66012 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[39, 2602, 463, 363, 1762, 362, 708, 607, 1047, 90, 1789, 224, 979, 616, 160, 477, 3182, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 870, 512, 777, 616, 181, 2622, 2107, 503, 1378, 997, 1059, 889, 311, 1246, 2251, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 641, 459, 420, 169, 2123, 2026, 653, 2578, 350, 1250, 615, 1626, 1830, 960, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 315, 391, 2004, 674, 668, 557, 76, 1128, 348, 896, 1817, 1316, 603, 399, 895, 2310, 1672, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 1459, 650, 1519, 1666, 303, 822, 1226, 1237, 1919, 317, 2175, 1016, 636, 315, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2562, 1316, 1875, 210, 1869, 304, 587, 793, 1659, 274, 665, 1167, 760, 584, 508, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 165, 1590, 530, 1254, 684, 1491, 0, 696, 942, 590, 2095, 647, 310, 1912, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 950, 856, 1256, 234, 403, 883, 1300, 1129, 1983, 1757, 697, 1988, 310, 462, 515, 821, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16458, 16606, 17098, 16465, 16991, 15798, 15072, 16584] avg=16384.0000 max=17098 max_rank=2 min=15072 min_rank=6 imbalance_ratio=1.043579 max_load_reduction=113974 max_load_reduction_pct=0.869553 moved_to_echo=64445 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[44, 2609, 811, 329, 1564, 442, 1026, 716, 746, 115, 1547, 484, 1141, 494, 159, 370, 2870, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 346, 751, 584, 896, 40, 3344, 1861, 573, 884, 862, 702, 921, 206, 969, 1432, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 481, 330, 525, 224, 1465, 1485, 540, 2750, 167, 959, 648, 1069, 1571, 1409, 878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 366, 277, 1570, 738, 366, 766, 95, 818, 762, 1381, 1638, 1622, 503, 608, 822, 2131, 1896, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 821, 2079, 360, 1345, 1599, 236, 712, 2081, 837, 2726, 277, 1400, 780, 896, 290, 1353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3025, 1102, 1783, 488, 1928, 213, 484, 1691, 2029, 295, 709, 864, 1277, 419, 350, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 354, 1913, 810, 1931, 849, 2071, 0, 747, 776, 669, 1989, 908, 475, 2126, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 553, 747, 1096, 156, 403, 960, 1502, 619, 1427, 1553, 982, 1166, 873, 703, 346, 673, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16587, 14555, 14572, 16688, 17792, 17792, 18009, 15077] avg=16384.0000 max=18009 max_rank=6 min=14555 min_rank=1 imbalance_ratio=1.099182 max_load_reduction=113063 max_load_reduction_pct=0.862602 moved_to_echo=68670 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[62, 2296, 774, 214, 1195, 281, 862, 614, 1380, 172, 1527, 234, 418, 907, 269, 397, 2725, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 510, 1589, 699, 942, 325, 2782, 2017, 567, 1164, 848, 1085, 1366, 333, 2061, 1603, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 435, 541, 396, 121, 2269, 1723, 1057, 2378, 272, 1862, 992, 1572, 1256, 800, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 369, 218, 1789, 607, 527, 612, 85, 440, 394, 817, 1815, 1801, 388, 930, 1025, 2197, 1859, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 866, 1718, 849, 1175, 615, 306, 570, 1341, 532, 2355, 296, 1879, 854, 1073, 299, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2289, 1097, 1636, 507, 2083, 411, 1065, 950, 1630, 224, 777, 1054, 459, 648, 416, 781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1455, 342, 1535, 615, 1559, 755, 1362, 0, 769, 865, 621, 2161, 498, 473, 1985, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 1600, 1389, 1887, 208, 591, 1143, 1047, 819, 1565, 1553, 382, 2066, 307, 492, 674, 907, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15427, 18018, 16638, 16251, 15210, 16027, 16005, 17496] avg=16384.0000 max=18018 max_rank=1 min=15210 min_rank=4 imbalance_ratio=1.099731 max_load_reduction=113054 max_load_reduction_pct=0.862534 moved_to_echo=64738 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[6528, 14720, 4992, 10368, 9728, 2176, 5376, 14336, 7296, 17024, 2304, 14976, 6144, 6912, 2304, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1024, 21504, 4864, 2560, 11520, 2816, 6784, 6656, 9216, 1024, 11648, 2048, 8064, 6016, 1792, 3072, 23680, 7808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[20480, 9216, 14592, 3584, 15360, 1792, 5760, 9984, 13696, 1920, 6144, 8320, 6784, 4352, 3328, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[10112, 2560, 14080, 5376, 12032, 5888, 14592, 0, 5504, 7424, 4480, 16512, 5376, 3072, 16640, 8576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1152, 4096, 7936, 5888, 7040, 1280, 23168, 14976, 4480, 7680, 7424, 7552, 9728, 2944, 12032, 14464, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2432, 2560, 2560, 15232, 6272, 4096, 4608, 896, 5376, 3712, 7936, 14592, 12800, 4224, 5504, 8576, 16896, 13184, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2432, 8832, 8320, 11392, 2176, 3968, 8064, 10752, 6400, 12800, 13056, 6016, 13440, 3712, 4224, 4096, 6400, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[640, 5376, 2944, 4096, 1536, 15872, 15744, 5120, 21248, 2688, 11264, 6144, 11264, 11776, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[3462, 2, 4023, 135, 342, 0, 9, 4039, 985, 1144, 297, 9, 73, 692, 15, 2970, 41, 3, 2831, 19, 29, 3378, 33, 195, 0, 59, 19, 379, 86, 2623, 1477, 8, 3223, 310, 3663, 22, 0, 0, 109, 42, 2250, 216, 15, 0, 0, 1, 1903, 12, 0, 1338, 0, 1, 295, 2022, 1, 18, 0, 3996, 2720, 3960, 342, 0, 0, 0, 3, 237, 2666, 636, 63, 266, 3849, 0, 3396, 15, 62, 170, 482, 10, 3052, 1, 3923, 3137, 2854, 328, 1671, 40, 9, 3844, 0, 693, 3168, 4092, 0, 133, 6, 81, 0, 91, 3968, 2263, 1896, 0, 4083, 0, 180, 1513, 3642, 4, 3555, 5, 0, 3201, 340, 599, 404, 7, 6, 3060, 11, 0, 1388, 185, 2, 1537, 80, 36, 445, 3848] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[3651, 52, 731, 1583, 860, 0, 736, 75, 3105, 0, 2171, 52, 3703, 2022, 27, 3740, 397, 0, 2187, 0, 3, 1640, 3636, 46, 7, 0, 0, 100, 2, 878, 3870, 179, 959, 972, 3330, 4, 12, 0, 1, 35, 3540, 191, 17, 7, 1149, 249, 568, 0, 0, 909, 4, 0, 402, 3152, 0, 0, 249, 4092, 2928, 2627, 1379, 0, 5, 2, 2, 1103, 4095, 2, 191, 0, 3376, 235, 261, 134, 8, 17, 7, 0, 1754, 13, 3784, 3471, 449, 267, 3055, 462, 3896, 4088, 0, 406, 2846, 4036, 1746, 753, 0, 822, 0, 757, 3577, 2300, 1623, 3, 350, 2, 0, 122, 1631, 0, 2290, 1502, 38, 1775, 91, 89, 36, 3, 5, 3702, 22, 0, 1263, 4, 0, 1291, 701, 12, 360, 4008] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[2452, 1, 1598, 570, 3784, 0, 339, 3309, 1668, 1307, 1754, 212, 911, 428, 59, 885, 418, 50, 953, 30, 16, 509, 23, 985, 372, 1021, 0, 128, 13, 4061, 3946, 0, 3538, 1926, 4058, 0, 0, 1, 11, 1, 4, 14, 0, 0, 3, 0, 521, 167, 2, 0, 167, 0, 2307, 3054, 0, 4, 10, 4017, 1173, 87, 3017, 0, 554, 8, 6, 64, 1536, 1068, 315, 114, 3271, 8, 199, 828, 80, 175, 1237, 4, 4081, 0, 3817, 1498, 2432, 1111, 3209, 86, 432, 4080, 64, 553, 2017, 3866, 698, 462, 0, 235, 3, 55, 4070, 882, 2152, 0, 3452, 0, 236, 86, 440, 11, 3532, 2192, 187, 1458, 1970, 120, 27, 1, 0, 3263, 2964, 22, 3771, 79, 1, 2379, 40, 11, 0, 3676] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1094, 4, 4047, 3936, 3585, 0, 39, 1374, 3429, 436, 17, 4, 3856, 1985, 69, 1491, 107, 0, 2032, 6, 805, 3963, 619, 61, 15, 1544, 28, 298, 1, 1532, 4039, 0, 610, 363, 4017, 0, 0, 0, 1505, 8, 262, 1, 2, 0, 5, 152, 249, 1, 0, 17, 35, 0, 358, 2971, 4, 178, 1, 4058, 2544, 2870, 2183, 0, 3, 5, 0, 25, 3988, 79, 35, 1, 3208, 2, 136, 320, 57, 153, 147, 12, 2586, 0, 3976, 3926, 2639, 2157, 3066, 93, 144, 3908, 0, 445, 719, 4096, 259, 287, 5, 1143, 1, 4, 3932, 3191, 570, 33, 3269, 0, 13, 1485, 2210, 336, 3826, 888, 2, 561, 926, 67, 25, 17, 0, 2494, 475, 0, 1785, 3, 7, 920, 243, 7, 298, 3054] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2788, 0, 4094, 539, 2525, 0, 160, 3929, 3874, 1780, 0, 1039, 2285, 2086, 33, 1690, 0, 0, 506, 0, 10, 3985, 2, 1139, 115, 4, 0, 6, 0, 4078, 704, 31, 2324, 1340, 3966, 0, 0, 3, 23, 23, 1045, 303, 0, 5, 12, 28, 27, 6, 0, 360, 7, 1, 23, 445, 60, 1, 12, 3419, 0, 2028, 2474, 0, 26, 38, 0, 2, 3147, 912, 220, 8, 4090, 4, 2630, 0, 2410, 978, 474, 0, 87, 1, 4085, 1559, 23, 8, 3697, 0, 350, 3982, 0, 1484, 532, 3990, 15, 956, 0, 2, 19, 1727, 4095, 278, 3918, 96, 2833, 65, 0, 2404, 3683, 3, 1097, 7, 13, 3994, 477, 105, 3110, 7, 0, 1272, 600, 102, 1757, 0, 0, 3750, 24, 0, 510, 4079] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[3415, 2, 2921, 125, 1233, 0, 13, 203, 1113, 239, 3585, 16, 1003, 272, 89, 3594, 106, 0, 2968, 1, 686, 246, 1540, 8, 0, 120, 55, 375, 0, 856, 3473, 0, 3748, 3038, 4066, 4, 27, 14, 22, 0, 11, 2377, 7, 89, 731, 0, 2117, 0, 11, 600, 341, 0, 42, 3399, 2, 1, 11, 4022, 0, 1405, 2255, 0, 111, 32, 0, 1, 3826, 1, 350, 2, 3556, 1, 792, 4009, 1, 0, 745, 0, 3451, 0, 3439, 3828, 1890, 42, 4058, 65, 3125, 4094, 7, 85, 3383, 4079, 31, 2252, 1, 104, 4, 51, 4025, 620, 2128, 3, 2702, 14, 8, 1689, 3826, 2, 2604, 1757, 2332, 54, 341, 40, 137, 8, 0, 380, 2296, 0, 400, 7, 13, 19, 66, 49, 14, 1525] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[4078, 2, 1334, 424, 3338, 1, 118, 3488, 3228, 299, 124, 0, 320, 3, 40, 3958, 36, 14, 30, 0, 161, 2295, 1473, 2211, 263, 1473, 0, 1406, 0, 3697, 3977, 0, 1134, 352, 1235, 0, 0, 0, 1, 4, 474, 102, 0, 0, 2, 2923, 4, 2, 66, 42, 68, 0, 972, 1951, 12, 0, 35, 3921, 2053, 376, 236, 109, 79, 11, 31, 502, 3863, 224, 281, 0, 4089, 0, 4044, 37, 509, 565, 208, 17, 4037, 21, 4002, 282, 3620, 3033, 3281, 175, 2879, 3947, 2, 388, 2884, 1794, 22, 67, 8, 286, 1, 398, 4096, 496, 4095, 12, 3555, 101, 5, 228, 709, 88, 4030, 917, 1540, 3117, 293, 6, 45, 8, 0, 591, 3, 0, 1407, 91, 60, 1535, 18, 543, 9, 4022] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[3650, 0, 962, 2259, 2457, 1, 200, 294, 2167, 82, 1370, 1, 1081, 2247, 4, 2789, 219, 239, 3821, 5, 17, 3855, 32, 71, 0, 51, 0, 305, 1, 2929, 3941, 1, 488, 3470, 2966, 0, 1, 0, 745, 36, 130, 26, 5, 0, 78, 36, 22, 1, 0, 197, 167, 0, 82, 2088, 0, 129, 5, 3179, 1101, 3165, 2198, 1, 24, 27, 8, 40, 3717, 13, 218, 1, 4014, 21, 1448, 482, 0, 55, 144, 0, 3637, 10, 4092, 3113, 2041, 3993, 3790, 0, 12, 4094, 51, 32, 881, 4096, 661, 2746, 57, 60, 0, 13, 3974, 2517, 1331, 5, 2271, 1, 0, 995, 1276, 24, 3497, 558, 3, 1021, 2982, 140, 163, 2, 0, 3150, 669, 0, 1954, 0, 0, 1907, 382, 8, 20, 3264] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 4047, 3936, 3585, 0, 39, 1374, 3429, 436, 17, 4, 3856, 1985, 69, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 2032, 6, 805, 3963, 619, 61, 15, 1544, 28, 298, 1, 1532, 4039, 0, 2564, 271, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 363, 4017, 0, 0, 0, 1505, 8, 262, 1, 2, 0, 5, 152, 249, 1, 3908, 2381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 35, 0, 358, 2971, 4, 178, 1, 4058, 2544, 2870, 2183, 0, 3, 5, 823, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 3988, 79, 35, 1, 3208, 2, 136, 320, 57, 153, 147, 12, 2586, 0, 969, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 2639, 2157, 3066, 93, 144, 0, 0, 445, 719, 1532, 259, 287, 5, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 3191, 570, 33, 3269, 0, 13, 1485, 2210, 336, 3268, 888, 2, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 926, 67, 25, 17, 0, 2494, 475, 0, 1785, 3, 7, 920, 243, 7, 298, 3054, 2814, 1551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23239, 17949, 13464, 17212, 12276, 16415, 15831, 14686] avg=16384.0000 max=23239 max_rank=0 min=12276 min_rank=4 imbalance_ratio=1.418396 max_load_reduction=107833 max_load_reduction_pct=0.822701 moved_to_echo=59208 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 52, 731, 1583, 860, 0, 736, 75, 3105, 0, 2171, 52, 3703, 2022, 27, 1146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 0, 2187, 0, 3, 1640, 3636, 46, 7, 0, 0, 100, 2, 878, 3870, 179, 2526, 906, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 959, 972, 3330, 4, 12, 0, 1, 35, 3540, 191, 17, 7, 1149, 249, 568, 0, 4088, 2166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 909, 4, 0, 402, 3152, 0, 0, 249, 4092, 2928, 2627, 1379, 0, 5, 2, 2745, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1103, 4095, 2, 191, 0, 3376, 235, 261, 134, 8, 17, 7, 0, 1754, 13, 2432, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3471, 449, 267, 3055, 462, 3896, 0, 0, 406, 2846, 1510, 1746, 753, 0, 822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 0, 2300, 1623, 3, 350, 2, 0, 122, 1631, 0, 1956, 1502, 38, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 89, 36, 3, 5, 3702, 22, 0, 1263, 4, 0, 1291, 701, 12, 360, 4008, 2678, 1411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16263, 16539, 17288, 19600, 13964, 19683, 12059, 15676] avg=16384.0000 max=19683 max_rank=5 min=12059 min_rank=6 imbalance_ratio=1.201355 max_load_reduction=111389 max_load_reduction_pct=0.849831 moved_to_echo=61382 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4023, 135, 342, 0, 9, 4039, 985, 1144, 297, 9, 73, 692, 15, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 3, 2831, 19, 29, 3378, 33, 195, 0, 59, 19, 379, 86, 2623, 1477, 8, 2561, 859, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3223, 310, 3663, 22, 0, 0, 109, 42, 2250, 216, 15, 0, 0, 1, 1903, 12, 3844, 2403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1338, 0, 1, 295, 2022, 1, 18, 0, 3996, 2720, 3960, 342, 0, 0, 0, 2603, 1147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 237, 2666, 636, 63, 266, 3849, 0, 3396, 15, 62, 170, 482, 10, 3052, 1, 1931, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3137, 2854, 328, 1671, 40, 9, 0, 0, 693, 3168, 1531, 0, 133, 6, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 2263, 1896, 0, 4083, 0, 180, 1513, 3642, 4, 3037, 5, 0, 3201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 599, 404, 7, 6, 3060, 11, 0, 1388, 185, 2, 1537, 80, 36, 445, 3848, 2776, 1565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12676, 14728, 18013, 18443, 17357, 13651, 19915, 16289] avg=16384.0000 max=19915 max_rank=6 min=12676 min_rank=0 imbalance_ratio=1.215515 max_load_reduction=111157 max_load_reduction_pct=0.848061 moved_to_echo=67212 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 539, 2525, 0, 160, 3929, 3874, 1780, 0, 1039, 2285, 2086, 33, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 506, 0, 10, 3985, 2, 1139, 115, 4, 0, 6, 0, 4078, 704, 31, 2500, 693, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2324, 1340, 3966, 0, 0, 3, 23, 23, 1045, 303, 0, 5, 12, 28, 27, 6, 3982, 2481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 7, 1, 23, 445, 60, 1, 12, 3419, 0, 2028, 2474, 0, 26, 38, 2095, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3147, 912, 220, 8, 4090, 4, 2630, 0, 2410, 978, 474, 0, 87, 1, 1102, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1559, 23, 8, 3697, 0, 350, 0, 0, 1484, 532, 1490, 15, 956, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1727, 0, 278, 3918, 96, 2833, 65, 0, 2404, 3683, 3, 934, 7, 13, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 105, 3110, 7, 0, 1272, 600, 102, 1757, 0, 0, 3750, 24, 0, 510, 4079, 2890, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22855, 13850, 15568, 12184, 16228, 10116, 19974, 20297] avg=16384.0000 max=22855 max_rank=0 min=10116 min_rank=5 imbalance_ratio=1.394958 max_load_reduction=108217 max_load_reduction_pct=0.825630 moved_to_echo=66615 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 1334, 424, 3338, 1, 118, 3488, 3228, 299, 124, 0, 320, 3, 40, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 14, 30, 0, 161, 2295, 1473, 2211, 263, 1473, 0, 1406, 0, 3697, 3977, 0, 1123, 1013, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1134, 352, 1235, 0, 0, 0, 1, 4, 474, 102, 0, 0, 2, 2923, 4, 2, 3947, 2481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 42, 68, 0, 972, 1951, 12, 0, 35, 3921, 2053, 376, 236, 109, 79, 11, 3065, 1171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 502, 3863, 224, 281, 0, 4089, 0, 4044, 37, 509, 565, 208, 17, 4037, 21, 2574, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 3620, 3033, 3281, 175, 2879, 0, 2, 388, 2884, 671, 22, 67, 8, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 398, 0, 496, 4095, 12, 3555, 101, 5, 228, 709, 88, 3443, 917, 1540, 3117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 6, 45, 8, 0, 591, 3, 0, 1407, 91, 60, 1535, 18, 543, 9, 4022, 2831, 1615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13932, 19343, 12661, 14167, 21589, 17598, 18705, 13077] avg=16384.0000 max=21589 max_rank=4 min=12661 min_rank=2 imbalance_ratio=1.317688 max_load_reduction=109483 max_load_reduction_pct=0.835289 moved_to_echo=70969 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 2921, 125, 1233, 0, 13, 203, 1113, 239, 3585, 16, 1003, 272, 89, 1101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 2968, 1, 686, 246, 1540, 8, 0, 120, 55, 375, 0, 856, 3473, 0, 2553, 849, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 3038, 4066, 4, 27, 14, 22, 0, 11, 2377, 7, 89, 731, 0, 2117, 0, 4094, 2438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 600, 341, 0, 42, 3399, 2, 1, 11, 4022, 0, 1405, 2255, 0, 111, 32, 2566, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3826, 1, 350, 2, 3556, 1, 792, 4009, 1, 0, 745, 0, 3451, 0, 2337, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3828, 1890, 42, 4058, 65, 3125, 0, 7, 85, 3383, 1526, 31, 2252, 1, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 51, 0, 620, 2128, 3, 2702, 14, 8, 1689, 3826, 2, 2225, 1757, 2332, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 40, 137, 8, 0, 380, 2296, 0, 400, 7, 13, 19, 66, 49, 14, 1525, 2433, 1587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11915, 13992, 22783, 15804, 19451, 20397, 17415, 9315] avg=16384.0000 max=22783 max_rank=2 min=9315 min_rank=7 imbalance_ratio=1.390564 max_load_reduction=108289 max_load_reduction_pct=0.826180 moved_to_echo=66578 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1598, 570, 3784, 0, 339, 3309, 1668, 1307, 1754, 212, 911, 428, 59, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 50, 953, 30, 16, 509, 23, 985, 372, 1021, 0, 128, 13, 4061, 3946, 0, 2420, 609, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3538, 1926, 4058, 0, 0, 1, 11, 1, 4, 14, 0, 0, 3, 0, 521, 167, 4080, 2466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 167, 0, 2307, 3054, 0, 4, 10, 4017, 1173, 87, 3017, 0, 554, 8, 1843, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 64, 1536, 1068, 315, 114, 3271, 8, 199, 828, 80, 175, 1237, 4, 4081, 0, 575, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1498, 2432, 1111, 3209, 86, 432, 0, 64, 553, 2017, 1446, 698, 462, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 55, 0, 882, 2152, 0, 3452, 0, 236, 86, 440, 11, 3017, 2192, 187, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1970, 120, 27, 1, 0, 3263, 2964, 22, 3771, 79, 1, 2379, 40, 11, 0, 3676, 2700, 1604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16212, 15592, 16790, 17360, 14076, 14243, 14171, 22628] avg=16384.0000 max=22628 max_rank=7 min=14076 min_rank=4 imbalance_ratio=1.381104 max_load_reduction=108444 max_load_reduction_pct=0.827362 moved_to_echo=65118 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 962, 2259, 2457, 1, 200, 294, 2167, 82, 1370, 1, 1081, 2247, 4, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 239, 3821, 5, 17, 3855, 32, 71, 0, 51, 0, 305, 1, 2929, 3941, 1, 2564, 907, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 488, 3470, 2966, 0, 1, 0, 745, 36, 130, 26, 5, 0, 78, 36, 22, 1, 4094, 2407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 167, 0, 82, 2088, 0, 129, 5, 3179, 1101, 3165, 2198, 1, 24, 27, 2743, 1196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 40, 3717, 13, 218, 1, 4014, 21, 1448, 482, 0, 55, 144, 0, 3637, 10, 1813, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3113, 2041, 3993, 3790, 0, 12, 0, 51, 32, 881, 1532, 661, 2746, 57, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 2517, 1331, 5, 2271, 1, 0, 995, 1276, 24, 2987, 558, 3, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2982, 140, 163, 2, 0, 3150, 669, 0, 1954, 0, 0, 1907, 382, 8, 20, 3264, 2896, 1567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13980, 19079, 14505, 16302, 16131, 18969, 13002, 19104] avg=16384.0000 max=19104 max_rank=7 min=13002 min_rank=6 imbalance_ratio=1.166016 max_load_reduction=111968 max_load_reduction_pct=0.854248 moved_to_echo=67206 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[7424, 1280, 3968, 128, 128, 17920, 7040, 128, 13824, 384, 128, 13440, 1664, 768, 1664, 27520, 22144, 12544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[16128, 11776, 27392, 128, 128, 128, 2432, 256, 7808, 3328, 128, 128, 2048, 3456, 5504, 256, 32128, 19328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 19712, 9600, 18176, 128, 1664, 16768, 19584, 5376, 9344, 1408, 13312, 9856, 384, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 20864, 16000, 11008, 25856, 1024, 10880, 0, 128, 4096, 16512, 11264, 3456, 7680, 128, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1408, 384, 15360, 128, 1792, 19968, 7424, 4736, 896, 4352, 128, 3072, 128, 20736, 25472, 256, 18816, 6144, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 3200, 0, 12672, 17792, 256, 22528, 256, 512, 8576, 17536, 512, 20992, 7936, 4224, 15232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 2048, 26880, 2944, 1792, 512, 29568, 384, 12928, 5888, 3200, 2176, 3456, 128, 22784, 128, 13824, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 3584, 896, 128, 4608, 19200, 128, 384, 384, 30720, 12544, 16640, 14208, 128, 896, 128, 18560, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[253, 68, 3955, 5, 1725, 4046, 17, 1183, 0, 3590, 0, 4094, 3942, 3510, 25, 5, 367, 1107, 3, 0, 1158, 36, 0, 0, 433, 0, 2833, 0, 5, 3236, 2, 3785, 3985, 0, 2, 0, 4, 52, 170, 1, 3941, 0, 107, 0, 3123, 347, 2, 0, 0, 4023, 0, 56, 1, 0, 10, 69, 3911, 3, 236, 4096, 4089, 4073, 2, 4066, 0, 0, 409, 4, 43, 2, 4016, 10, 815, 0, 4096, 8, 1, 20, 2, 8, 152, 4096, 919, 19, 0, 2353, 149, 4064, 15, 3245, 28, 6, 13, 0, 591, 0, 86, 22, 3235, 7, 114, 61, 1, 0, 27, 0, 2881, 2822, 0, 4077, 0, 18, 341, 415, 4020, 3838, 394, 0, 15, 0, 3196, 0, 36, 0, 1508, 0, 3117, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[4093, 37, 0, 246, 5, 3591, 0, 4094, 1, 3966, 0, 3990, 3991, 35, 60, 2, 0, 34, 0, 0, 12, 704, 7, 1, 1739, 3866, 92, 0, 661, 3611, 4004, 0, 392, 0, 2, 283, 190, 4093, 0, 24, 2857, 35, 0, 2809, 143, 2, 789, 0, 72, 2915, 1, 8, 4, 0, 16, 574, 50, 23, 328, 3642, 4090, 1368, 1, 4034, 2796, 13, 4096, 2, 0, 4089, 1811, 0, 325, 1, 4, 11, 0, 1, 116, 63, 3, 3709, 4093, 2, 218, 4059, 2832, 4090, 4, 2546, 0, 0, 0, 1, 100, 1634, 0, 16, 711, 8, 1468, 1025, 0, 0, 1, 19, 2382, 0, 0, 689, 2, 0, 3986, 4, 4080, 3700, 307, 7, 301, 2, 4075, 3, 0, 0, 0, 2, 149, 3929] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4096, 0, 3691, 2793, 0, 3690, 11, 114, 0, 3954, 0, 2816, 173, 0, 9, 4, 0, 0, 0, 826, 0, 0, 301, 2, 3546, 3853, 0, 0, 227, 3612, 0, 3873, 3, 0, 653, 3513, 1, 3165, 0, 0, 4084, 0, 5, 14, 873, 24, 0, 0, 6, 2980, 5, 217, 0, 0, 1259, 0, 1109, 0, 233, 3486, 4086, 3572, 9, 4087, 3726, 0, 3690, 27, 0, 26, 4024, 2, 490, 26, 34, 10, 0, 0, 0, 0, 37, 3877, 1863, 298, 3882, 4071, 974, 4096, 26, 3920, 1, 62, 0, 0, 4093, 0, 1, 0, 1416, 5, 78, 0, 22, 164, 387, 0, 3037, 3363, 0, 122, 0, 0, 0, 11, 4091, 3957, 253, 0, 275, 0, 0, 12, 9, 7, 20, 1, 135, 3476] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[3853, 0, 4095, 85, 4, 4090, 65, 3395, 14, 4095, 0, 4088, 77, 34, 0, 15, 4, 0, 27, 294, 7, 10, 0, 8, 3931, 35, 3697, 109, 344, 4093, 657, 5, 0, 21, 241, 4011, 32, 1618, 296, 0, 3692, 14, 0, 5, 4081, 0, 122, 0, 0, 3847, 11, 1, 5, 19, 1796, 62, 0, 20, 295, 3050, 4093, 1243, 92, 4081, 708, 11, 3615, 0, 125, 3220, 3810, 0, 996, 0, 721, 31, 0, 15, 1, 0, 37, 4021, 848, 55, 580, 4094, 3809, 3994, 0, 4062, 0, 2, 0, 68, 1034, 164, 0, 1, 4094, 0, 0, 0, 55, 5, 0, 1, 2142, 0, 3, 4096, 5, 0, 799, 3, 4092, 3216, 1939, 1771, 4, 11, 103, 4, 0, 0, 1294, 0, 2, 1532] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4096, 21, 3530, 2227, 0, 3190, 13, 2779, 0, 3528, 0, 1751, 2821, 0, 0, 164, 16, 344, 7, 1, 222, 0, 4, 417, 4029, 722, 0, 50, 1091, 4094, 819, 3806, 41, 105, 476, 2084, 21, 2620, 3, 9, 156, 0, 47, 3824, 2924, 2, 0, 0, 3, 4063, 521, 0, 201, 0, 0, 134, 719, 0, 1, 2783, 3411, 3617, 5, 2146, 883, 2, 4089, 8, 0, 284, 3798, 0, 65, 1318, 4061, 0, 0, 1, 13, 106, 375, 307, 3160, 2, 99, 4091, 215, 4047, 68, 428, 0, 0, 1, 0, 2756, 71, 244, 0, 3392, 1, 1641, 4, 0, 4, 180, 0, 2378, 190, 3, 4096, 0, 0, 2879, 0, 4078, 3908, 72, 79, 0, 0, 3465, 0, 3893, 0, 7, 0, 0, 652] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[3962, 0, 3954, 0, 0, 4024, 4, 0, 0, 4078, 0, 6, 59, 0, 4, 2644, 0, 29, 0, 13, 1556, 0, 0, 1, 1, 25, 3967, 0, 47, 3212, 226, 11, 4096, 0, 0, 2334, 0, 1598, 0, 0, 4093, 0, 0, 151, 2378, 4088, 2051, 0, 1, 4090, 10, 135, 0, 1, 1645, 1976, 0, 0, 414, 4070, 3718, 3360, 2891, 1, 48, 0, 4096, 0, 0, 134, 3807, 0, 154, 356, 3826, 3719, 0, 0, 0, 0, 4, 4082, 111, 434, 6, 4082, 0, 4096, 18, 3060, 1, 13, 0, 13, 4091, 67, 0, 0, 3426, 2307, 27, 0, 3, 4, 0, 0, 2747, 5, 0, 3047, 38, 0, 1, 2133, 4096, 4026, 3, 0, 23, 0, 1794, 13, 191, 0, 1, 0, 44, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[138, 1, 4096, 0, 797, 4096, 6, 3926, 0, 4096, 1, 4051, 3839, 4046, 0, 0, 37, 2652, 6, 0, 0, 3, 0, 3181, 1962, 108, 419, 0, 0, 4086, 3537, 4060, 9, 20, 0, 0, 0, 118, 0, 0, 2191, 0, 5, 3119, 2862, 4, 15, 0, 0, 3751, 1, 35, 3, 0, 121, 15, 1, 0, 0, 31, 3807, 3908, 0, 3913, 61, 0, 1459, 22, 3, 172, 3925, 455, 1352, 20, 4085, 616, 0, 2, 36, 0, 5, 2675, 1092, 1962, 188, 4090, 1044, 4095, 0, 2136, 0, 1, 0, 579, 3247, 6, 0, 1, 4092, 13, 428, 0, 0, 79, 0, 1, 0, 9, 103, 3515, 320, 1, 3648, 2511, 4093, 2109, 2290, 0, 0, 0, 0, 13, 1266, 0, 170, 0, 0, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4096, 7, 4082, 5, 111, 2311, 0, 3985, 0, 4089, 0, 4075, 2321, 1863, 1, 2, 0, 139, 0, 11, 0, 0, 0, 155, 233, 2682, 420, 14, 113, 4080, 619, 352, 0, 6, 7, 109, 2, 4081, 0, 6, 3011, 0, 0, 73, 4022, 0, 844, 0, 1, 3951, 138, 0, 64, 0, 0, 0, 489, 0, 88, 22, 4096, 3987, 1, 4084, 3331, 1, 1062, 0, 11, 3704, 3178, 0, 774, 51, 3700, 0, 0, 0, 3, 0, 56, 3734, 809, 2, 597, 4087, 3946, 3966, 0, 4034, 0, 1557, 0, 0, 1690, 2, 7, 0, 4086, 0, 1798, 0, 0, 0, 29, 0, 15, 0, 0, 91, 1203, 0, 291, 0, 4086, 4096, 0, 117, 0, 17, 3988, 3782, 0, 0, 0, 0, 4, 2319] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4093, 37, 0, 246, 5, 0, 0, 4094, 1, 0, 0, 3990, 3991, 35, 60, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 12, 704, 7, 1, 1739, 3866, 92, 0, 661, 3611, 4004, 0, 1423, 877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 2, 283, 190, 4093, 0, 24, 2857, 35, 0, 2809, 143, 2, 789, 0, 2062, 1385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 2915, 1, 8, 4, 0, 16, 574, 50, 23, 328, 3642, 1123, 1368, 1, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2796, 13, 4096, 2, 0, 4089, 1811, 0, 325, 1, 4, 11, 0, 1, 116, 63, 1529, 1544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3709, 4093, 2, 218, 3182, 2832, 0, 4, 2546, 0, 0, 0, 1, 100, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 711, 8, 1468, 1025, 0, 0, 1, 19, 2382, 0, 0, 689, 2, 0, 4090, 2581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 4, 4080, 3700, 307, 7, 301, 2, 4075, 3, 0, 0, 0, 2, 149, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16554, 17031, 15066, 14159, 16401, 18324, 12992, 20545] avg=16384.0000 max=20545 max_rank=7 min=12992 min_rank=6 imbalance_ratio=1.253967 max_load_reduction=110527 max_load_reduction_pct=0.843254 moved_to_echo=68262 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[253, 68, 3497, 5, 1725, 0, 17, 1183, 0, 0, 0, 4094, 3942, 3510, 25, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 1107, 3, 0, 1158, 36, 0, 0, 433, 0, 2833, 0, 5, 3236, 2, 3785, 1423, 508, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 2, 0, 4, 52, 170, 1, 3941, 0, 107, 0, 3123, 347, 2, 0, 2323, 1254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 56, 1, 0, 10, 69, 3911, 3, 236, 4096, 1123, 4073, 2, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 4, 43, 2, 4016, 10, 815, 0, 4096, 8, 1, 20, 2, 8, 1723, 1543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 4096, 919, 19, 0, 1845, 149, 0, 15, 3245, 28, 6, 13, 0, 591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 22, 3235, 7, 114, 61, 1, 0, 27, 0, 2881, 2822, 0, 4077, 0, 18, 4064, 2336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 415, 4020, 3838, 394, 0, 15, 0, 3196, 0, 36, 0, 1508, 0, 3117, 5, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18324, 15289, 15311, 21669, 12700, 11078, 19751, 16950] avg=16384.0000 max=21669 max_rank=3 min=11078 min_rank=5 imbalance_ratio=1.322571 max_load_reduction=109403 max_load_reduction_pct=0.834679 moved_to_echo=60479 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3853, 0, 3621, 85, 4, 0, 65, 3395, 14, 0, 0, 4088, 77, 34, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 27, 294, 7, 10, 0, 8, 3931, 35, 3697, 109, 344, 4093, 657, 5, 1424, 885, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 241, 4011, 32, 1618, 296, 0, 3692, 14, 0, 5, 4081, 0, 122, 0, 2348, 1430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3847, 11, 1, 5, 19, 1796, 62, 0, 20, 295, 3050, 1124, 1243, 92, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 708, 11, 3615, 0, 125, 3220, 3810, 0, 996, 0, 721, 31, 0, 15, 1, 0, 1742, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4021, 848, 55, 580, 3209, 3809, 0, 0, 4062, 0, 2, 0, 68, 1034, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 55, 5, 0, 1, 2142, 0, 3, 4096, 5, 0, 3994, 2665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 799, 3, 4092, 3216, 1939, 1771, 4, 11, 103, 4, 0, 0, 1294, 0, 2, 1532, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15251, 15936, 17911, 15646, 16540, 17889, 17061, 14838] avg=16384.0000 max=17911 max_rank=2 min=14838 min_rank=7 imbalance_ratio=1.093201 max_load_reduction=113161 max_load_reduction_pct=0.863350 moved_to_echo=66328 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 3264, 2793, 0, 0, 11, 114, 0, 0, 0, 2816, 173, 0, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 0, 0, 301, 2, 3546, 3853, 0, 0, 227, 3612, 0, 3873, 1422, 880, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 653, 3513, 1, 3165, 0, 0, 4084, 0, 5, 14, 873, 24, 0, 0, 2120, 1382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2980, 5, 217, 0, 0, 1259, 0, 1109, 0, 233, 3486, 1122, 3572, 9, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3726, 0, 3690, 27, 0, 26, 4024, 2, 490, 26, 34, 10, 0, 0, 0, 0, 1570, 1542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 3877, 1863, 298, 3882, 3191, 974, 0, 26, 3920, 1, 62, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1416, 5, 78, 0, 22, 164, 387, 0, 3037, 3363, 0, 122, 0, 0, 4096, 2572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4091, 3957, 253, 0, 275, 0, 0, 12, 9, 7, 20, 1, 135, 3476, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13280, 18908, 15837, 18085, 15167, 22224, 15263, 12308] avg=16384.0000 max=22224 max_rank=5 min=12308 min_rank=7 imbalance_ratio=1.356445 max_load_reduction=108848 max_load_reduction_pct=0.830444 moved_to_echo=64962 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 21, 3122, 2227, 0, 0, 13, 2779, 0, 0, 0, 1751, 2821, 0, 0, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 344, 7, 1, 222, 0, 4, 417, 4029, 722, 0, 50, 1091, 4094, 819, 3806, 1187, 884, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 105, 476, 2084, 21, 2620, 3, 9, 156, 0, 47, 3824, 2924, 2, 0, 0, 1832, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4063, 521, 0, 201, 0, 0, 134, 719, 0, 1, 2783, 937, 3617, 5, 2146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 883, 2, 4089, 8, 0, 284, 3798, 0, 65, 1318, 4061, 0, 0, 1, 13, 106, 1358, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 307, 3160, 2, 99, 3207, 215, 0, 68, 428, 0, 0, 1, 0, 2756, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 3392, 1, 1641, 4, 0, 4, 180, 0, 2378, 190, 3, 4096, 0, 0, 4047, 2295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2879, 0, 4078, 3908, 72, 79, 0, 0, 3465, 0, 3893, 0, 7, 0, 0, 652, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16994, 18043, 15377, 15130, 17273, 10689, 18475, 19091] avg=16384.0000 max=19091 max_rank=7 min=10689 min_rank=5 imbalance_ratio=1.165222 max_load_reduction=111981 max_load_reduction_pct=0.854347 moved_to_echo=65528 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3962, 0, 3489, 0, 0, 0, 4, 0, 0, 0, 0, 6, 59, 0, 4, 2644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 13, 1556, 0, 0, 1, 1, 25, 3967, 0, 47, 3212, 226, 11, 1297, 886, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2334, 0, 1598, 0, 0, 4093, 0, 0, 151, 2378, 4088, 2051, 0, 2312, 1425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 10, 135, 0, 1, 1645, 1976, 0, 0, 414, 4070, 1015, 3360, 2891, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 4096, 0, 0, 134, 3807, 0, 154, 356, 3826, 3719, 0, 0, 0, 0, 1712, 1406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4082, 111, 434, 6, 3196, 0, 0, 18, 3060, 1, 13, 0, 13, 4091, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3426, 2307, 27, 0, 3, 4, 0, 0, 2747, 5, 0, 3047, 38, 0, 4096, 2653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2133, 4096, 4026, 3, 0, 23, 0, 1794, 13, 191, 0, 1, 0, 44, 1, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10168, 11667, 24526, 19609, 19258, 15096, 18353, 12395] avg=16384.0000 max=24526 max_rank=2 min=10168 min_rank=0 imbalance_ratio=1.496948 max_load_reduction=106546 max_load_reduction_pct=0.812881 moved_to_echo=65102 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[138, 1, 3621, 0, 797, 0, 6, 3926, 0, 0, 1, 4051, 3839, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 2652, 6, 0, 0, 3, 0, 3181, 1962, 108, 419, 0, 0, 4086, 3537, 4060, 1325, 884, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 20, 0, 0, 0, 118, 0, 0, 2191, 0, 5, 3119, 2862, 4, 15, 0, 2353, 1432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3751, 1, 35, 3, 0, 121, 15, 1, 0, 0, 31, 1045, 3908, 0, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 1459, 22, 3, 172, 3925, 455, 1352, 20, 4085, 616, 0, 2, 36, 0, 1743, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2675, 1092, 1962, 188, 3206, 1044, 0, 0, 2136, 0, 1, 0, 579, 3247, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 13, 428, 0, 0, 79, 0, 1, 0, 9, 103, 3515, 320, 1, 4095, 2664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3648, 2511, 4093, 2109, 2290, 0, 0, 0, 0, 13, 1266, 0, 170, 0, 0, 9, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20426, 22667, 12128, 12824, 15388, 16141, 15321, 16177] avg=16384.0000 max=22667 max_rank=1 min=12128 min_rank=2 imbalance_ratio=1.383484 max_load_reduction=108405 max_load_reduction_pct=0.827065 moved_to_echo=63027 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 7, 3609, 5, 111, 0, 0, 3985, 0, 0, 0, 4075, 2321, 1863, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 11, 0, 0, 0, 155, 233, 2682, 420, 14, 113, 4080, 619, 352, 1425, 883, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 109, 2, 4081, 0, 6, 3011, 0, 0, 73, 4022, 0, 844, 0, 1327, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3951, 138, 0, 64, 0, 0, 0, 489, 0, 88, 22, 1125, 3987, 1, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3331, 1, 1062, 0, 11, 3704, 3178, 0, 774, 51, 3700, 0, 0, 0, 3, 0, 984, 1546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 3734, 809, 2, 597, 3204, 3946, 0, 0, 4034, 0, 1557, 0, 0, 1690, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4086, 0, 1798, 0, 0, 0, 29, 0, 15, 0, 0, 91, 1203, 0, 3966, 2661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 0, 4086, 4096, 0, 117, 0, 17, 3988, 3782, 0, 0, 0, 0, 4, 2319, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20075, 11531, 14916, 13950, 18345, 19631, 13856, 18768] avg=16384.0000 max=20075 max_rank=0 min=11531 min_rank=1 imbalance_ratio=1.225281 max_load_reduction=110997 max_load_reduction_pct=0.846840 moved_to_echo=70600 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[12032, 5120, 32640, 28928, 5376, 2048, 640, 128, 16640, 3840, 5504, 128, 3072, 128, 3456, 12032, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[11648, 128, 22528, 128, 256, 11648, 28416, 512, 4992, 1792, 20608, 4480, 128, 128, 256, 256, 12416, 11904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[24704, 256, 24320, 5376, 2688, 0, 128, 19584, 128, 0, 128, 24960, 17280, 9600, 128, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 29696, 768, 512, 384, 128, 4864, 2944, 6400, 128, 1664, 21248, 8704, 25216, 3072, 26496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[768, 26624, 12928, 2816, 5632, 24320, 13056, 0, 256, 23552, 128, 1664, 128, 768, 17664, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 128, 24576, 2432, 5632, 1152, 128, 256, 640, 128, 15616, 6400, 128, 19840, 1664, 128, 32512, 20480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[8576, 256, 1408, 12416, 256, 17408, 512, 128, 24064, 128, 256, 10112, 20480, 4480, 3840, 0, 16768, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[512, 4352, 128, 1152, 3072, 768, 384, 3840, 16000, 11392, 11520, 256, 2560, 30080, 9984, 16000, 11008, 6784, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 3954, 3689, 420, 27, 547, 1207, 0, 4095, 1981, 9, 2011, 3677, 4079, 0, 0, 4059, 3754, 90, 23, 3934, 2070, 101, 0, 0, 32, 0, 238, 117, 0, 28, 190, 2, 0, 18, 3962, 0, 0, 0, 0, 0, 1, 1, 4000, 3, 285, 4, 0, 0, 3779, 0, 346, 4090, 6, 4058, 4096, 287, 11, 0, 3997, 13, 0, 95, 0, 2375, 1044, 0, 0, 3698, 0, 417, 0, 4, 0, 0, 641, 307, 6, 38, 33, 0, 0, 0, 4084, 2616, 4, 2324, 4095, 4064, 0, 4052, 0, 0, 2906, 1236, 13, 3, 4060, 4084, 0, 4079, 1, 0, 629, 0, 7, 7, 8, 0, 26, 166, 0, 0, 4058, 31, 21, 2374, 99, 0, 0, 1, 0, 4079, 5, 3991, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 197, 3859, 4073, 0, 15, 378, 3, 70, 4083, 0, 43, 4093, 3332, 3433, 0, 1238, 407, 0, 3804, 4096, 24, 0, 0, 1311, 56, 2, 523, 0, 0, 298, 3702, 1, 0, 0, 1419, 152, 3, 883, 233, 439, 3840, 501, 4057, 0, 0, 0, 1, 4, 4095, 0, 0, 4095, 0, 4095, 3221, 154, 131, 176, 4068, 34, 344, 0, 0, 113, 505, 71, 4020, 13, 19, 18, 89, 4069, 410, 3999, 4002, 0, 0, 2, 25, 14, 64, 6, 18, 4000, 0, 1090, 4046, 225, 57, 3168, 0, 0, 4078, 0, 3379, 30, 681, 3405, 1582, 3, 0, 209, 269, 0, 0, 0, 235, 0, 0, 4043, 0, 269, 3319, 8, 0, 263, 81, 0, 1, 207, 0, 3952, 0, 3964, 24, 8, 33] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4085, 0, 0, 71, 0, 1396, 0, 0, 1499, 406, 4059, 3595, 0, 0, 6, 1, 0, 4092, 4008, 0, 1005, 0, 0, 3968, 96, 731, 118, 0, 241, 2, 5, 0, 0, 1, 8, 4077, 1661, 25, 20, 0, 4, 0, 619, 45, 4, 8, 0, 1877, 157, 0, 2458, 64, 4093, 3998, 7, 0, 17, 3843, 0, 10, 18, 2, 523, 4089, 4091, 0, 3859, 0, 1, 0, 2213, 96, 0, 3417, 0, 0, 0, 96, 477, 0, 292, 3781, 3842, 4, 5, 3737, 0, 0, 1089, 4, 1, 234, 1, 0, 0, 3159, 3890, 0, 4042, 4074, 361, 28, 28, 0, 3507, 43, 10, 0, 3354, 0, 1, 4062, 415, 0, 4083, 4095, 0, 0, 1, 0, 4093, 155, 4096, 7, 3316, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 34, 0, 0, 2586, 0, 146, 3013, 0, 0, 3647, 4096, 3993, 0, 0, 0, 0, 0, 3844, 240, 1155, 124, 227, 0, 38, 1, 0, 10, 0, 0, 800, 0, 0, 0, 1111, 258, 0, 45, 166, 0, 340, 85, 3480, 13, 0, 0, 4096, 67, 1631, 0, 288, 4028, 2, 4012, 2522, 275, 0, 3325, 3710, 1, 1, 1, 0, 200, 3926, 1, 5, 3712, 0, 1908, 0, 0, 0, 3602, 3791, 227, 0, 0, 39, 106, 5, 5, 4096, 4085, 37, 0, 4090, 0, 3977, 4019, 0, 603, 0, 0, 0, 9, 1406, 4030, 1, 2075, 4057, 0, 1706, 0, 7, 0, 942, 2, 282, 3439, 1, 46, 3847, 9, 1, 4093, 3983, 0, 70, 444, 0, 4090, 12, 4094, 0, 0, 581] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 51, 4096, 6, 0, 39, 3853, 0, 3436, 3589, 2422, 4060, 4093, 3100, 1, 0, 4083, 898, 3, 0, 2491, 0, 0, 0, 0, 4079, 59, 0, 100, 3, 28, 47, 451, 0, 12, 2820, 0, 0, 266, 2422, 13, 2647, 2, 0, 45, 0, 0, 0, 0, 4063, 0, 2, 4064, 528, 382, 3744, 3724, 0, 42, 3947, 41, 0, 0, 0, 0, 4095, 737, 6, 3355, 0, 3875, 7, 8, 0, 1344, 4093, 4, 11, 0, 0, 0, 0, 0, 4013, 1, 0, 354, 4055, 392, 0, 101, 0, 0, 1, 1, 4096, 0, 194, 4088, 302, 3994, 3323, 1278, 4092, 0, 11, 0, 1, 0, 0, 27, 0, 0, 2979, 0, 0, 883, 691, 0, 2, 3079, 0, 4095, 1, 121, 0, 1610, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 567, 0, 3266, 0, 3074, 2611, 3810, 2550, 7, 3782, 4078, 3958, 3, 0, 192, 0, 7, 4093, 3832, 7, 3, 599, 53, 4058, 0, 2984, 230, 2, 1339, 4095, 295, 22, 4, 269, 7, 1, 1976, 0, 4, 14, 283, 493, 3460, 6, 0, 0, 0, 3043, 0, 2, 2688, 20, 0, 3784, 382, 0, 7, 4082, 0, 0, 497, 0, 0, 386, 201, 1744, 3320, 0, 587, 0, 0, 422, 5, 4024, 13, 0, 1232, 0, 26, 190, 0, 3996, 3764, 0, 138, 4096, 1941, 0, 54, 0, 93, 368, 347, 117, 0, 30, 4096, 73, 3634, 210, 0, 3563, 0, 0, 1, 25, 8, 5, 2116, 15, 3, 3902, 0, 0, 4090, 10, 0, 104, 1, 27, 4072, 0, 4092, 0, 17, 3375] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 697, 4, 14, 4051, 21, 133, 9, 4047, 4030, 107, 602, 3933, 4093, 1, 0, 4095, 184, 37, 3879, 2053, 0, 0, 0, 0, 3840, 0, 0, 3734, 0, 4096, 0, 0, 0, 1, 118, 330, 0, 0, 0, 1, 0, 0, 10, 40, 0, 60, 5, 0, 0, 0, 660, 4092, 3, 17, 51, 3, 28, 0, 4095, 5, 0, 12, 0, 3763, 891, 0, 40, 2711, 0, 213, 3, 0, 4, 0, 3148, 2, 307, 135, 2899, 0, 459, 0, 3698, 3988, 0, 2828, 3917, 1, 968, 0, 5, 0, 2832, 4053, 4078, 0, 3953, 4095, 0, 4044, 6, 2, 808, 0, 8, 2462, 40, 0, 0, 4009, 5, 151, 3782, 839, 0, 4072, 3, 0, 421, 26, 0, 4086, 30, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4011, 160, 25, 32, 0, 4094, 4088, 4002, 3016, 2626, 4086, 4095, 0, 119, 2192, 0, 0, 36, 4010, 0, 39, 50, 1, 544, 0, 5, 0, 0, 886, 20, 0, 10, 31, 4, 26, 1255, 4034, 66, 0, 789, 0, 234, 1, 0, 2, 50, 17, 258, 58, 872, 3191, 1, 4080, 2809, 0, 0, 0, 4022, 1, 0, 0, 23, 4077, 3223, 1244, 0, 4057, 0, 0, 0, 3903, 0, 1846, 4070, 0, 0, 27, 0, 7, 3, 0, 4096, 4091, 0, 4, 3929, 878, 142, 386, 1, 0, 27, 117, 0, 0, 3540, 4047, 2, 2247, 6, 1, 1215, 0, 6, 3411, 2, 5, 3026, 4074, 0, 0, 4093, 10, 0, 138, 17, 1, 63, 74, 0, 4094, 1, 4041, 5, 589, 265] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 3954, 3689, 420, 27, 547, 1207, 0, 4095, 1981, 9, 2011, 0, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 3754, 90, 23, 3934, 2070, 101, 0, 0, 32, 0, 238, 117, 0, 28, 190, 738, 671, 559, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 18, 3962, 0, 0, 0, 0, 0, 1, 1, 4000, 3, 285, 4, 0, 3677, 3676, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3779, 0, 346, 4090, 6, 4058, 4096, 287, 11, 0, 3259, 13, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2375, 1044, 0, 0, 3698, 0, 417, 0, 4, 0, 0, 641, 307, 6, 38, 33, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 2616, 4, 2324, 1840, 4064, 0, 4052, 0, 0, 2906, 1236, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4060, 3440, 0, 4079, 1, 0, 629, 0, 7, 7, 8, 0, 26, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 31, 21, 2374, 99, 0, 0, 1, 0, 3408, 5, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18343, 16820, 17325, 20040, 8991, 23139, 12426, 13988] avg=16384.0000 max=23139 max_rank=5 min=8991 min_rank=4 imbalance_ratio=1.412292 max_load_reduction=107933 max_load_reduction_pct=0.823463 moved_to_echo=58544 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4085, 0, 0, 71, 0, 1396, 0, 0, 1499, 406, 0, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 4092, 4008, 0, 1005, 0, 0, 3968, 96, 731, 118, 0, 241, 2, 710, 673, 510, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 8, 4077, 1661, 25, 20, 0, 4, 0, 619, 45, 4, 8, 4059, 3240, 1547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1877, 157, 0, 2458, 64, 4093, 3998, 7, 0, 17, 3133, 0, 10, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 4089, 4091, 0, 3859, 0, 1, 0, 2213, 96, 0, 3417, 0, 0, 0, 96, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 0, 292, 3781, 3842, 4, 5, 1680, 0, 0, 1089, 4, 1, 234, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3159, 3277, 0, 4042, 4074, 361, 28, 28, 0, 3507, 43, 10, 0, 3354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4062, 415, 0, 4083, 4095, 0, 0, 1, 0, 3420, 155, 4096, 7, 3316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7812, 16367, 15323, 15834, 18792, 11410, 21883, 23651] avg=16384.0000 max=23651 max_rank=7 min=7812 min_rank=0 imbalance_ratio=1.443542 max_load_reduction=107421 max_load_reduction_pct=0.819557 moved_to_echo=75736 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 197, 3859, 4073, 0, 15, 378, 3, 70, 4083, 0, 43, 0, 329, 3433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1238, 407, 0, 3804, 4096, 24, 0, 0, 1311, 56, 2, 523, 0, 0, 298, 3702, 751, 650, 553, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1419, 152, 3, 883, 233, 439, 3840, 501, 4057, 0, 0, 0, 1, 4093, 3003, 1675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 0, 0, 4095, 0, 4095, 3221, 154, 131, 176, 3317, 34, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 505, 71, 4020, 13, 19, 18, 89, 4069, 410, 3999, 4002, 0, 0, 2, 25, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 64, 6, 18, 4000, 0, 1090, 1818, 225, 57, 3168, 0, 0, 4078, 0, 3379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 681, 2869, 1582, 3, 0, 209, 269, 0, 0, 0, 235, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 3319, 8, 0, 263, 81, 0, 1, 207, 0, 3302, 0, 3964, 24, 8, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16483, 17595, 20300, 19666, 17711, 17917, 9921, 11479] avg=16384.0000 max=20300 max_rank=2 min=9921 min_rank=6 imbalance_ratio=1.239014 max_load_reduction=110772 max_load_reduction_pct=0.845123 moved_to_echo=57028 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 34, 0, 0, 2586, 0, 146, 3013, 0, 0, 3647, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3844, 240, 1155, 124, 227, 0, 38, 1, 0, 10, 0, 0, 800, 685, 673, 559, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1111, 258, 0, 45, 166, 0, 340, 85, 3480, 13, 0, 0, 4096, 4096, 3599, 1694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 1631, 0, 288, 4028, 2, 4012, 2522, 275, 0, 3325, 3025, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 3926, 1, 5, 3712, 0, 1908, 0, 0, 0, 3602, 3791, 227, 0, 0, 39, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 5, 5, 4096, 4085, 37, 0, 1837, 0, 3977, 4019, 0, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1406, 3395, 1, 2075, 4057, 0, 1706, 0, 7, 0, 942, 2, 282, 3439, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 3847, 9, 1, 4093, 3983, 0, 70, 444, 0, 3417, 12, 4094, 0, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9820, 8569, 18983, 19178, 17833, 18770, 17322, 20597] avg=16384.0000 max=20597 max_rank=7 min=8569 min_rank=1 imbalance_ratio=1.257141 max_load_reduction=110475 max_load_reduction_pct=0.842857 moved_to_echo=74522 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 567, 0, 3266, 0, 3074, 2611, 3810, 2550, 7, 3782, 0, 391, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 7, 4093, 3832, 7, 3, 599, 53, 4058, 0, 2984, 230, 2, 1339, 4095, 754, 670, 559, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 22, 4, 269, 7, 1, 1976, 0, 4, 14, 283, 493, 3460, 6, 0, 0, 4078, 3567, 1696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3043, 0, 2, 2688, 20, 0, 3784, 382, 0, 7, 3328, 0, 0, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 386, 201, 1744, 3320, 0, 587, 0, 0, 422, 5, 4024, 13, 0, 1232, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 190, 0, 3996, 3764, 0, 138, 1841, 1941, 0, 54, 0, 93, 368, 347, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 3450, 73, 3634, 210, 0, 3563, 0, 0, 1, 25, 8, 5, 2116, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3902, 0, 0, 4090, 10, 0, 104, 1, 27, 3402, 0, 4092, 0, 17, 3375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20061, 23694, 16175, 13751, 12363, 12875, 13130, 19023] avg=16384.0000 max=23694 max_rank=1 min=12363 min_rank=4 imbalance_ratio=1.446167 max_load_reduction=107378 max_load_reduction_pct=0.819229 moved_to_echo=57391 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 697, 4, 14, 4051, 21, 133, 9, 4047, 4030, 107, 602, 0, 404, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 184, 37, 3879, 2053, 0, 0, 0, 0, 3840, 0, 0, 3734, 0, 4096, 0, 756, 672, 535, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 118, 330, 0, 0, 0, 1, 0, 0, 10, 40, 0, 60, 5, 3933, 3689, 1622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 4092, 3, 17, 51, 3, 28, 0, 3339, 5, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3763, 891, 0, 40, 2711, 0, 213, 3, 0, 4, 0, 3148, 2, 307, 135, 2899, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 0, 3698, 3988, 0, 2828, 1760, 1, 968, 0, 5, 0, 2832, 4053, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3953, 3449, 0, 4044, 6, 2, 808, 0, 8, 2462, 40, 0, 0, 4009, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 3782, 839, 0, 4072, 3, 0, 421, 26, 0, 3414, 30, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14120, 24098, 9809, 8210, 14545, 24670, 18786, 16834] avg=16384.0000 max=24670 max_rank=5 min=8210 min_rank=3 imbalance_ratio=1.505737 max_load_reduction=106402 max_load_reduction_pct=0.811783 moved_to_echo=74835 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 51, 4096, 6, 0, 39, 3853, 0, 3436, 3589, 2422, 4060, 0, 303, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 898, 3, 0, 2491, 0, 0, 0, 0, 4079, 59, 0, 100, 3, 28, 47, 733, 678, 557, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 0, 12, 2820, 0, 0, 266, 2422, 13, 2647, 2, 0, 45, 0, 0, 0, 4093, 2797, 1683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 2, 4064, 528, 382, 3744, 3724, 0, 42, 3214, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 737, 6, 3355, 0, 3875, 7, 8, 0, 1344, 4093, 4, 11, 0, 0, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 1, 0, 354, 1815, 392, 0, 101, 0, 0, 1, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 3438, 302, 3994, 3323, 1278, 4092, 0, 11, 0, 1, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2979, 0, 0, 883, 691, 0, 2, 3079, 0, 3417, 1, 121, 0, 1610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21856, 13978, 17251, 19804, 17966, 10774, 16660, 12783] avg=16384.0000 max=21856 max_rank=0 min=10774 min_rank=5 imbalance_ratio=1.333984 max_load_reduction=109216 max_load_reduction_pct=0.833252 moved_to_echo=58183 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4011, 160, 25, 32, 0, 4094, 4088, 4002, 3016, 2626, 0, 404, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2192, 0, 0, 36, 4010, 0, 39, 50, 1, 544, 0, 5, 0, 0, 886, 20, 743, 674, 537, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 31, 4, 26, 1255, 4034, 66, 0, 789, 0, 234, 1, 0, 2, 50, 4086, 3691, 1627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 258, 58, 872, 3191, 1, 4080, 2809, 0, 0, 0, 3279, 1, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3223, 1244, 0, 4057, 0, 0, 0, 3903, 0, 1846, 4070, 0, 0, 27, 0, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 0, 4096, 4091, 0, 4, 1765, 878, 142, 386, 1, 0, 27, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3540, 3409, 2, 2247, 6, 1, 1215, 0, 6, 3411, 2, 5, 3026, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 10, 0, 138, 17, 1, 63, 74, 0, 3420, 1, 4041, 5, 589, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22577, 9951, 15906, 14589, 22871, 11517, 20944, 12717] avg=16384.0000 max=22871 max_rank=4 min=9951 min_rank=1 imbalance_ratio=1.395935 max_load_reduction=108201 max_load_reduction_pct=0.825508 moved_to_echo=68049 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 4992, 20352, 4736, 7424, 3328, 8704, 8320, 22656, 20352, 7168, 17280, 0, 3072, 3456, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[768, 128, 128, 9728, 896, 5376, 8960, 2944, 512, 7680, 896, 12288, 4224, 384, 128, 4224, 32128, 27264, 13312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[512, 30080, 1408, 128, 20096, 9088, 128, 768, 3840, 128, 27264, 256, 28544, 128, 5632, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 17024, 26752, 2048, 24192, 11776, 1920, 12416, 128, 128, 9472, 1408, 128, 3456, 21248, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[640, 768, 384, 27904, 26496, 128, 6784, 14464, 7552, 5248, 12928, 128, 768, 10496, 5760, 11776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[11136, 18176, 6400, 5888, 24832, 128, 7040, 128, 10240, 1024, 10880, 27264, 640, 384, 1536, 3200, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[15872, 5248, 256, 19840, 24704, 3328, 1280, 896, 1408, 16640, 256, 4608, 4352, 128, 7040, 8960, 5888, 5376, 4480, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 18816, 256, 2176, 28800, 640, 20864, 24320, 4864, 256, 3584, 25984, 128, 384, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[3044, 536, 3822, 46, 0, 0, 261, 0, 550, 0, 0, 3891, 0, 0, 1, 735, 52, 0, 0, 313, 0, 20, 0, 1733, 1, 0, 4079, 6, 0, 0, 128, 0, 3676, 0, 8, 155, 4004, 0, 4096, 170, 91, 3804, 3879, 4020, 150, 0, 4021, 423, 0, 4047, 1, 3833, 3944, 0, 2, 3736, 0, 6, 0, 268, 3, 0, 2493, 4096, 3, 0, 511, 79, 0, 0, 102, 1, 4005, 9, 3997, 301, 0, 151, 0, 3890, 0, 1, 0, 135, 4025, 300, 0, 52, 0, 276, 0, 2, 3414, 63, 0, 3210, 1190, 1418, 489, 1974, 0, 0, 4067, 1397, 2474, 0, 0, 4095, 0, 0, 0, 4092, 18, 3473, 0, 22, 8, 176, 4078, 0, 0, 3, 0, 4096, 306, 305, 2700, 20] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[2089, 797, 4095, 0, 3, 562, 4, 0, 3898, 7, 0, 4094, 0, 3808, 0, 3542, 4096, 0, 25, 0, 7, 0, 0, 4079, 2925, 0, 3814, 6, 1590, 3815, 1083, 0, 1, 789, 941, 400, 1442, 0, 4027, 0, 2324, 4071, 0, 0, 0, 1, 259, 0, 44, 4057, 0, 4095, 4094, 0, 48, 4095, 0, 0, 416, 0, 0, 0, 0, 4046, 1521, 669, 0, 414, 1, 1017, 0, 0, 0, 1, 3621, 0, 1635, 58, 0, 149, 0, 898, 61, 1626, 4033, 0, 0, 0, 0, 0, 0, 2130, 3751, 0, 0, 1821, 2, 3696, 0, 347, 0, 0, 4013, 0, 51, 117, 0, 0, 86, 0, 44, 4061, 0, 3319, 472, 0, 2, 9, 4064, 0, 21, 2148, 0, 4085, 1639, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4091, 3795, 4058, 4, 3919, 9, 1, 3, 1369, 99, 0, 0, 0, 0, 139, 0, 38, 0, 0, 0, 256, 34, 1, 4067, 0, 3976, 4096, 7, 3570, 3918, 0, 0, 3845, 0, 3662, 0, 256, 0, 4058, 3485, 0, 3801, 99, 96, 0, 0, 0, 112, 1202, 4096, 1244, 2980, 4059, 0, 39, 3735, 0, 0, 0, 0, 29, 592, 3434, 4092, 9, 975, 12, 3, 12, 3998, 218, 7, 7, 780, 0, 5, 0, 32, 404, 1077, 1, 21, 1518, 555, 4096, 0, 0, 0, 0, 3, 4, 161, 3534, 4, 0, 0, 19, 6, 4076, 3599, 962, 0, 3549, 0, 3, 1, 1, 4090, 0, 2275, 740, 2341, 0, 19, 14, 0, 0, 311, 69, 0, 347, 1, 695, 4061, 4091, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[3173, 3493, 4074, 0, 0, 559, 0, 0, 118, 0, 51, 3417, 0, 3, 4, 3470, 4016, 47, 28, 3627, 4092, 6, 0, 1681, 19, 0, 4000, 0, 2833, 2120, 4, 0, 0, 4, 7, 1, 41, 0, 4070, 369, 32, 2814, 3980, 823, 0, 257, 210, 0, 1, 4060, 0, 4060, 3837, 0, 188, 4063, 0, 36, 0, 33, 4088, 84, 0, 3643, 428, 0, 2, 0, 0, 4044, 207, 0, 1387, 0, 0, 0, 36, 0, 1, 4091, 1479, 454, 9, 1536, 4085, 2079, 24, 3802, 0, 0, 0, 0, 3889, 0, 528, 0, 82, 40, 0, 4030, 828, 0, 239, 0, 3967, 103, 0, 9, 0, 1534, 1, 10, 2262, 4095, 36, 55, 0, 0, 4096, 0, 1, 0, 128, 1004, 47, 754, 11, 2123] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[356, 1919, 3701, 15, 567, 3338, 1923, 26, 4050, 0, 2, 4037, 0, 139, 0, 3745, 4059, 1, 0, 4031, 0, 0, 0, 3908, 1, 3, 4090, 0, 78, 0, 4081, 2, 0, 0, 4056, 3606, 0, 1, 4096, 0, 3749, 3774, 4094, 16, 2, 367, 1, 4090, 0, 1898, 0, 1265, 0, 0, 0, 63, 5, 0, 0, 4, 4070, 0, 0, 4065, 4, 86, 947, 4046, 0, 0, 0, 0, 504, 409, 77, 3882, 0, 222, 82, 44, 2, 0, 552, 3120, 174, 58, 22, 0, 0, 16, 206, 3009, 3689, 0, 0, 1, 68, 1960, 0, 4096, 1, 8, 0, 0, 4004, 4, 20, 1, 0, 0, 2, 2674, 153, 3577, 0, 35, 38, 0, 3952, 52, 37, 146, 2, 4059, 1708, 3, 2, 24] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[23, 4095, 3967, 0, 122, 0, 0, 51, 3778, 0, 0, 4089, 0, 0, 2, 0, 0, 0, 87, 4093, 402, 95, 52, 1, 15, 0, 610, 2, 304, 0, 3186, 82, 1, 3999, 586, 0, 0, 5, 4082, 0, 2, 2480, 700, 0, 9, 58, 4096, 3962, 69, 3949, 0, 4090, 4068, 0, 26, 9, 0, 4, 4067, 0, 3248, 1, 4092, 2497, 0, 0, 2, 35, 82, 0, 3, 18, 1749, 1897, 1210, 306, 0, 552, 165, 2041, 0, 0, 30, 0, 2043, 144, 0, 3766, 0, 0, 108, 3433, 4069, 0, 0, 0, 29, 4086, 1, 2510, 0, 0, 4094, 0, 13, 0, 10, 4027, 430, 0, 16, 4093, 1565, 12, 0, 0, 80, 1, 4094, 0, 405, 3843, 0, 3422, 463, 38, 3671, 1360] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[3533, 7, 4043, 1, 4, 231, 23, 0, 18, 0, 0, 2863, 0, 9, 0, 3942, 4041, 50, 0, 4059, 22, 0, 0, 0, 0, 0, 0, 2220, 0, 3061, 4089, 10, 0, 4078, 15, 21, 124, 0, 4094, 3828, 1427, 4084, 3978, 0, 52, 0, 851, 4096, 1324, 3552, 3, 4053, 1794, 0, 0, 4060, 133, 0, 0, 0, 164, 0, 3152, 229, 0, 0, 391, 16, 0, 0, 2708, 0, 1053, 206, 0, 967, 0, 78, 0, 2590, 0, 0, 846, 1025, 4061, 0, 3, 3708, 0, 10, 7, 3278, 4086, 3, 0, 0, 1, 773, 31, 2184, 0, 33, 4091, 823, 3880, 0, 0, 668, 175, 0, 0, 3811, 435, 4001, 0, 0, 3, 0, 3936, 0, 11, 4, 0, 3818, 0, 0, 0, 20] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4091, 3652, 3836, 0, 817, 2, 343, 0, 3331, 1, 0, 2785, 0, 0, 0, 205, 2312, 2, 15, 0, 357, 0, 18, 4095, 3676, 79, 767, 2690, 0, 3980, 772, 0, 0, 3, 158, 7, 44, 0, 4059, 68, 3355, 821, 1966, 366, 404, 2, 0, 3856, 1754, 3836, 0, 41, 3905, 0, 74, 4096, 0, 0, 375, 0, 2466, 0, 0, 515, 59, 0, 0, 596, 3, 32, 0, 0, 0, 257, 45, 3306, 8, 1499, 0, 4093, 0, 0, 2940, 488, 2025, 7, 232, 0, 0, 0, 0, 3998, 3095, 16, 8, 104, 25, 5, 3028, 2491, 0, 0, 3859, 1, 3985, 126, 0, 3764, 2, 0, 0, 4079, 12, 4086, 37, 2, 30, 57, 994, 0, 4015, 476, 1543, 4042, 31, 129, 3, 1442] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2089, 797, 2246, 0, 3, 562, 4, 0, 3898, 7, 0, 4094, 0, 3808, 0, 3542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 25, 0, 7, 0, 0, 4079, 2925, 0, 2986, 6, 1590, 3815, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 789, 941, 400, 1442, 0, 0, 0, 2324, 3955, 0, 0, 0, 1, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 4095, 3233, 0, 48, 4095, 0, 0, 416, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 669, 0, 414, 1, 1017, 0, 0, 0, 1, 3621, 0, 1635, 58, 0, 149, 4027, 3075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 898, 61, 1626, 4033, 0, 0, 0, 0, 0, 0, 2130, 3751, 0, 0, 1821, 1849, 982, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3696, 0, 347, 0, 0, 4013, 0, 51, 117, 0, 0, 86, 0, 44, 3245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3319, 472, 0, 2, 9, 4064, 0, 21, 2148, 0, 4085, 1639, 0, 1, 0, 816, 828, 584, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21050, 20612, 10112, 15977, 16188, 17428, 11601, 18104] avg=16384.0000 max=21050 max_rank=0 min=10112 min_rank=2 imbalance_ratio=1.284790 max_load_reduction=110022 max_load_reduction_pct=0.839401 moved_to_echo=63321 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3044, 536, 2099, 46, 0, 0, 261, 0, 550, 0, 0, 3891, 0, 0, 1, 735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 313, 0, 20, 0, 1733, 1, 0, 3199, 6, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3676, 0, 8, 155, 4004, 0, 0, 170, 91, 3699, 3879, 4020, 150, 0, 4021, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3833, 3121, 0, 2, 3736, 0, 6, 0, 268, 3, 0, 2493, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 511, 79, 0, 0, 102, 1, 4005, 9, 3997, 301, 0, 151, 0, 3890, 4096, 3066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 135, 4025, 300, 0, 52, 0, 276, 0, 2, 3414, 63, 0, 3210, 1723, 981, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1190, 1418, 489, 1974, 0, 0, 4067, 1397, 2474, 0, 0, 4095, 0, 0, 0, 3275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3473, 0, 22, 8, 176, 4078, 0, 0, 3, 0, 4096, 306, 305, 2700, 20, 817, 880, 559, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11163, 5452, 24296, 17559, 20211, 14446, 20379, 17566] avg=16384.0000 max=24296 max_rank=2 min=5452 min_rank=1 imbalance_ratio=1.482910 max_load_reduction=106776 max_load_reduction_pct=0.814636 moved_to_echo=72602 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3173, 3493, 2238, 0, 0, 559, 0, 0, 118, 0, 51, 3417, 0, 3, 4, 3470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 47, 28, 3627, 4092, 6, 0, 1681, 19, 0, 3137, 0, 2833, 2120, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 1, 41, 0, 0, 369, 32, 2737, 3980, 823, 0, 257, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4060, 3037, 0, 188, 4063, 0, 36, 0, 33, 4088, 84, 0, 3643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 0, 2, 0, 0, 4044, 207, 0, 1387, 0, 0, 0, 36, 0, 1, 4091, 4070, 3076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1479, 454, 9, 1536, 4085, 2079, 24, 3802, 0, 0, 0, 0, 3889, 0, 528, 0, 1836, 984, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 40, 0, 4030, 828, 0, 239, 0, 3967, 103, 0, 9, 0, 1534, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2262, 4095, 36, 55, 0, 0, 4096, 0, 1, 0, 128, 1004, 47, 754, 11, 2123, 1, 863, 544, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16526, 21610, 8461, 19233, 17342, 20961, 10842, 16097] avg=16384.0000 max=21610 max_rank=1 min=8461 min_rank=2 imbalance_ratio=1.318970 max_load_reduction=109462 max_load_reduction_pct=0.835129 moved_to_echo=65242 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4091, 3795, 2229, 4, 3919, 9, 1, 3, 1369, 99, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 256, 34, 1, 4067, 0, 3976, 3212, 7, 3570, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3845, 0, 3662, 0, 256, 0, 0, 3485, 0, 3696, 99, 96, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1202, 0, 1244, 2980, 3213, 0, 39, 3735, 0, 0, 0, 0, 29, 592, 3434, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 975, 12, 3, 12, 3998, 218, 7, 7, 780, 0, 5, 0, 32, 404, 1077, 4058, 3104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 21, 1518, 555, 4096, 0, 0, 0, 0, 3, 4, 161, 3534, 4, 0, 0, 1829, 992, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 6, 4076, 3599, 962, 0, 3549, 0, 3, 1, 1, 4090, 0, 2275, 740, 1874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 14, 0, 0, 311, 69, 0, 347, 1, 695, 4061, 4091, 0, 0, 0, 467, 884, 575, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15658, 19079, 15251, 20560, 14701, 12989, 21195, 11639] avg=16384.0000 max=21195 max_rank=6 min=11639 min_rank=7 imbalance_ratio=1.293640 max_load_reduction=109877 max_load_reduction_pct=0.838295 moved_to_echo=60524 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[23, 4095, 2179, 0, 122, 0, 0, 51, 3778, 0, 0, 4089, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 4093, 402, 95, 52, 1, 15, 0, 479, 2, 304, 0, 3186, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3999, 586, 0, 0, 5, 0, 0, 2, 2412, 700, 0, 9, 58, 4096, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 4090, 3219, 0, 26, 9, 0, 4, 4067, 0, 3248, 1, 4092, 2497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 35, 82, 0, 3, 18, 1749, 1897, 1210, 306, 0, 552, 165, 2041, 4082, 2992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 2043, 144, 0, 3766, 0, 0, 108, 3433, 4069, 0, 0, 0, 1788, 957, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4086, 1, 2510, 0, 0, 4094, 0, 13, 0, 10, 4027, 430, 0, 16, 3276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1565, 12, 0, 0, 80, 1, 4094, 0, 405, 3843, 0, 3422, 463, 38, 3671, 1360, 817, 131, 577, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14339, 8798, 15830, 21322, 15134, 16610, 18492, 20547] avg=16384.0000 max=21322 max_rank=3 min=8798 min_rank=1 imbalance_ratio=1.301392 max_load_reduction=109750 max_load_reduction_pct=0.837326 moved_to_echo=70783 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[356, 1919, 2033, 15, 567, 3338, 1923, 26, 4050, 0, 2, 4037, 0, 139, 0, 3745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 1, 0, 4031, 0, 0, 0, 3908, 1, 3, 3207, 0, 78, 0, 4081, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 3606, 0, 1, 0, 0, 3749, 3670, 4094, 16, 2, 367, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1265, 0, 0, 0, 63, 5, 0, 0, 4, 4070, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 86, 947, 4046, 0, 0, 0, 0, 504, 409, 77, 3882, 0, 222, 82, 44, 4096, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 552, 3120, 174, 58, 22, 0, 0, 16, 206, 3009, 3689, 0, 0, 1, 1668, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 1960, 0, 4096, 1, 8, 0, 0, 4004, 4, 20, 1, 0, 0, 2, 2140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 3577, 0, 35, 38, 0, 3952, 52, 37, 146, 2, 4059, 1708, 3, 2, 24, 534, 883, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22150, 19371, 23652, 9472, 15838, 12976, 12304, 15309] avg=16384.0000 max=23652 max_rank=2 min=9472 min_rank=3 imbalance_ratio=1.443604 max_load_reduction=107420 max_load_reduction_pct=0.819550 moved_to_echo=56427 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4091, 3652, 2107, 0, 817, 2, 343, 0, 3331, 1, 0, 2785, 0, 0, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2312, 2, 15, 0, 357, 0, 18, 4095, 3676, 79, 602, 2690, 0, 3980, 772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 158, 7, 44, 0, 0, 68, 3355, 799, 1966, 366, 404, 2, 0, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1754, 0, 0, 41, 3090, 0, 74, 4096, 0, 0, 375, 0, 2466, 0, 0, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 596, 3, 32, 0, 0, 0, 257, 45, 3306, 8, 1499, 0, 4093, 4059, 2907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2940, 488, 2025, 7, 232, 0, 0, 0, 0, 3998, 3095, 16, 8, 104, 1729, 929, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 5, 3028, 2491, 0, 0, 3859, 1, 3985, 126, 0, 3764, 2, 0, 0, 3264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4086, 37, 2, 30, 57, 994, 0, 4015, 476, 1543, 4042, 31, 129, 3, 1442, 815, 165, 554, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17334, 18598, 11028, 12411, 16864, 15832, 20550, 18455] avg=16384.0000 max=20550 max_rank=6 min=11028 min_rank=2 imbalance_ratio=1.254272 max_load_reduction=110522 max_load_reduction_pct=0.843216 moved_to_echo=71701 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3533, 7, 2221, 1, 4, 231, 23, 0, 18, 0, 0, 2863, 0, 9, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 50, 0, 4059, 22, 0, 0, 0, 0, 0, 0, 2220, 0, 3061, 4089, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 15, 21, 124, 0, 0, 3828, 1427, 3972, 3978, 0, 52, 0, 851, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1324, 0, 3, 4053, 1420, 0, 0, 4060, 133, 0, 0, 0, 164, 0, 3152, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 16, 0, 0, 2708, 0, 1053, 206, 0, 967, 0, 78, 0, 2590, 4094, 2691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 846, 1025, 4061, 0, 3, 3708, 0, 10, 7, 3278, 4086, 3, 0, 0, 1822, 861, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 773, 31, 2184, 0, 33, 4091, 823, 3880, 0, 0, 668, 175, 0, 0, 3050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 4001, 0, 0, 3, 0, 3936, 0, 11, 4, 0, 3818, 0, 0, 0, 20, 761, 0, 254, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12852, 17552, 22442, 14538, 14794, 19830, 15709, 13355] avg=16384.0000 max=22442 max_rank=2 min=12852 min_rank=0 imbalance_ratio=1.369751 max_load_reduction=108630 max_load_reduction_pct=0.828781 moved_to_echo=63688 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[20480, 18304, 17408, 128, 5504, 4736, 2560, 128, 17152, 128, 128, 25216, 0, 3968, 256, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[4480, 0, 1280, 24448, 20352, 0, 384, 23936, 256, 128, 4864, 384, 14080, 768, 13184, 23296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1536, 1408, 6016, 8576, 24576, 2688, 384, 11392, 0, 384, 384, 16128, 29568, 128, 640, 5248, 14336, 7168, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1536, 12032, 7680, 21248, 1792, 128, 23936, 2304, 18432, 384, 128, 16768, 768, 3840, 896, 20224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[4480, 22656, 640, 128, 256, 640, 25344, 128, 4864, 6656, 2432, 28672, 8320, 1280, 6400, 4992, 5120, 4736, 3712, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[18688, 128, 256, 16128, 5248, 256, 128, 19584, 6656, 4096, 16896, 4992, 8448, 16896, 13440, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[7552, 8960, 9472, 4224, 6016, 128, 0, 7936, 11008, 24960, 18816, 5376, 640, 768, 9472, 16640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2048, 1792, 1920, 5248, 128, 9216, 3328, 128, 8832, 3584, 8960, 8832, 1792, 2688, 768, 18048, 32640, 22400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[258, 0, 3882, 0, 0, 3, 0, 4030, 0, 3805, 4, 0, 4, 1, 0, 0, 47, 620, 0, 0, 291, 16, 3405, 0, 0, 4094, 3946, 0, 570, 55, 756, 520, 3844, 0, 149, 3962, 3780, 0, 4053, 2139, 399, 1, 0, 4009, 4, 4096, 19, 429, 88, 4030, 40, 2720, 567, 0, 4056, 111, 402, 0, 4053, 348, 0, 0, 1435, 0, 148, 38, 0, 1138, 110, 0, 0, 1622, 0, 3, 3028, 3928, 0, 1, 1420, 0, 4093, 0, 3753, 0, 3404, 0, 982, 11, 2235, 5, 0, 268, 2, 4090, 129, 116, 0, 0, 0, 0, 3642, 0, 1, 14, 930, 4078, 0, 2435, 29, 3651, 1026, 10, 2, 0, 4095, 1389, 2, 0, 4012, 2, 10, 18, 1, 4096, 0, 63, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 869, 3141, 255, 0, 27, 0, 3411, 0, 1635, 327, 0, 69, 4, 0, 1353, 1, 1, 1, 0, 6, 4061, 3918, 3259, 71, 3785, 4076, 0, 3642, 0, 1336, 24, 152, 86, 13, 564, 0, 0, 4095, 2146, 345, 547, 836, 4096, 4060, 3930, 5, 271, 0, 197, 19, 3522, 585, 4082, 0, 11, 0, 0, 3970, 0, 0, 145, 1, 55, 0, 39, 0, 996, 3861, 0, 484, 2036, 0, 28, 4062, 261, 3572, 286, 3, 148, 3885, 1, 3783, 263, 242, 0, 115, 22, 2142, 1158, 0, 0, 0, 0, 160, 2873, 0, 2, 369, 0, 1904, 56, 0, 280, 3420, 4066, 14, 1, 4053, 496, 2, 2, 0, 0, 3055, 60, 0, 0, 4071, 39, 1302, 8, 1, 3480, 4078, 0, 888, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[2, 0, 4095, 26, 1, 4065, 118, 3826, 0, 4096, 0, 0, 0, 0, 0, 937, 11, 2386, 0, 2, 32, 2152, 82, 2892, 68, 3252, 4051, 0, 6, 0, 3764, 0, 418, 1803, 118, 300, 0, 1, 1, 3767, 1, 0, 770, 3448, 4084, 3348, 0, 0, 0, 0, 0, 276, 0, 4054, 1372, 0, 28, 2, 114, 0, 2, 3109, 2950, 0, 0, 0, 6, 4, 3, 0, 0, 4027, 7, 1938, 3237, 1876, 3706, 1728, 3457, 972, 4083, 0, 2966, 64, 12, 199, 6, 94, 0, 0, 0, 24, 0, 778, 10, 3935, 0, 0, 0, 812, 0, 888, 0, 3, 4045, 4020, 2020, 3971, 2, 4054, 0, 0, 33, 120, 0, 2181, 0, 0, 4067, 44, 2851, 14, 0, 2914, 21, 50, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[41, 0, 3, 7, 0, 50, 5, 4090, 0, 3878, 0, 3, 0, 0, 117, 0, 3748, 1, 0, 44, 0, 0, 3896, 2869, 0, 4052, 2592, 0, 4095, 40, 0, 0, 0, 3523, 0, 133, 4025, 0, 168, 0, 175, 4, 0, 0, 750, 4092, 0, 4, 0, 44, 0, 4084, 0, 148, 4071, 90, 3834, 0, 1614, 14, 23, 198, 4096, 0, 0, 0, 0, 430, 920, 65, 0, 4090, 0, 0, 3125, 0, 4093, 0, 3295, 1556, 4055, 1, 4087, 3, 3890, 168, 4, 0, 4094, 3733, 3566, 0, 289, 3992, 6, 4096, 0, 0, 186, 0, 104, 288, 0, 0, 149, 4019, 0, 238, 1, 3840, 132, 0, 4, 4, 1199, 4096, 0, 0, 4094, 0, 0, 0, 0, 2384, 154, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 25, 140, 0, 0, 4073, 26, 4080, 15, 195, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 3806, 0, 4072, 3470, 0, 673, 795, 3913, 2324, 1, 183, 142, 0, 0, 2527, 1, 0, 4060, 6, 158, 3368, 2778, 1696, 12, 4095, 2549, 2420, 0, 0, 0, 82, 10, 3959, 0, 3992, 1051, 0, 834, 4066, 0, 0, 354, 142, 0, 1057, 0, 1577, 6, 0, 0, 0, 599, 4096, 0, 1075, 3, 0, 3547, 0, 3549, 4039, 4083, 191, 3735, 0, 2931, 1, 95, 0, 49, 2, 0, 0, 64, 481, 405, 4074, 1, 0, 0, 191, 0, 4050, 1, 974, 4088, 1693, 233, 0, 14, 2969, 4087, 45, 0, 1, 1, 274, 0, 743, 86, 11, 3883, 0, 443, 721, 0, 0, 703, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1, 0, 68, 429, 0, 4044, 289, 3659, 6, 1065, 652, 0, 0, 0, 0, 2121, 218, 3776, 0, 2, 0, 4096, 119, 186, 0, 431, 4001, 0, 2467, 0, 3484, 0, 93, 4089, 0, 0, 827, 9, 595, 3762, 3920, 0, 1296, 4093, 0, 4087, 0, 11, 33, 4035, 0, 4001, 0, 4096, 3024, 21, 60, 0, 4096, 6, 0, 4045, 117, 6, 0, 0, 1, 0, 0, 4057, 0, 3789, 0, 286, 0, 82, 3455, 252, 37, 2, 4063, 0, 2556, 4096, 2953, 0, 344, 2, 0, 2068, 0, 3, 51, 0, 1881, 57, 12, 3896, 6, 744, 0, 0, 0, 0, 1519, 4095, 0, 0, 0, 38, 0, 92, 0, 913, 4028, 9, 0, 536, 4063, 0, 4, 0, 0, 3632, 14, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[2, 1, 3101, 824, 0, 3960, 19, 2490, 0, 4080, 0, 22, 0, 0, 27, 0, 0, 4, 0, 2168, 0, 3614, 823, 4067, 0, 3064, 607, 132, 1994, 195, 0, 0, 0, 4058, 6, 2282, 89, 0, 4096, 2699, 13, 7, 1692, 3868, 794, 4074, 0, 609, 41, 1, 0, 3588, 0, 1586, 1262, 14, 1, 577, 3918, 56, 120, 242, 4096, 0, 0, 0, 0, 39, 113, 90, 0, 2700, 10, 0, 4084, 2, 91, 26, 1229, 1517, 3966, 188, 3602, 4096, 4096, 0, 26, 0, 0, 20, 0, 0, 633, 2007, 33, 154, 0, 0, 0, 0, 778, 13, 0, 0, 4042, 3321, 8, 0, 0, 3981, 3332, 2552, 73, 1049, 940, 4096, 0, 0, 3313, 0, 0, 0, 0, 3715, 154, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1, 481, 4086, 313, 3, 3540, 37, 4050, 0, 4065, 1050, 0, 3882, 0, 68, 859, 0, 3951, 0, 0, 2, 4055, 1581, 4069, 1, 3495, 3968, 0, 3307, 0, 3295, 0, 3, 169, 1, 5, 0, 31, 1, 355, 103, 26, 3, 4096, 3599, 3814, 0, 137, 0, 3, 0, 3653, 0, 3269, 1670, 0, 3, 0, 4057, 0, 246, 0, 0, 0, 0, 0, 0, 106, 0, 72, 449, 4031, 1, 0, 18, 419, 739, 197, 0, 72, 4096, 24, 4069, 3663, 4095, 577, 2, 0, 744, 0, 0, 50, 0, 3302, 726, 3939, 0, 3902, 0, 78, 98, 7, 0, 0, 3644, 3285, 27, 16, 0, 11, 796, 63, 14, 222, 3383, 354, 0, 0, 4096, 70, 3467, 0, 11, 731, 0, 0, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4095, 26, 1, 4065, 118, 3826, 0, 4096, 0, 0, 0, 0, 0, 937, 3085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2386, 0, 2, 32, 2152, 82, 2892, 68, 2848, 0, 0, 6, 0, 3764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 1803, 118, 300, 0, 1, 1, 3767, 1, 0, 770, 3310, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 0, 4054, 1372, 0, 28, 2, 114, 0, 2, 3109, 2950, 0, 270, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 3, 0, 0, 4027, 7, 1938, 3237, 1876, 3706, 1728, 3457, 972, 1634, 884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 913, 0, 2966, 64, 12, 199, 6, 94, 0, 0, 0, 24, 0, 778, 10, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 812, 0, 888, 0, 3, 4045, 4020, 2020, 3971, 2, 4054, 0, 0, 1536, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 120, 0, 2181, 0, 0, 4067, 44, 2851, 14, 0, 2914, 21, 50, 0, 0, 3348, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20251, 14243, 14573, 12315, 23479, 9001, 21485, 15725] avg=16384.0000 max=23479 max_rank=4 min=9001 min_rank=5 imbalance_ratio=1.433044 max_load_reduction=107593 max_load_reduction_pct=0.820869 moved_to_echo=69690 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[258, 0, 3882, 0, 0, 3, 0, 4030, 0, 3805, 4, 0, 4, 1, 0, 0, 3005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 620, 0, 0, 291, 16, 3405, 0, 0, 3585, 0, 0, 570, 55, 756, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3844, 0, 149, 3962, 3780, 0, 4053, 2139, 399, 1, 0, 3849, 4, 0, 19, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 4030, 40, 2720, 567, 0, 4056, 111, 402, 0, 4053, 348, 0, 0, 1435, 0, 340, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 38, 0, 1138, 110, 0, 0, 1622, 0, 3, 3028, 3928, 0, 1, 1420, 0, 1638, 861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 915, 0, 3753, 0, 3404, 0, 982, 11, 2235, 5, 0, 268, 2, 4090, 129, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3642, 0, 1, 14, 930, 4078, 0, 2435, 29, 3651, 1026, 10, 1540, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 1389, 2, 0, 4012, 2, 10, 18, 1, 4096, 0, 63, 1, 0, 4096, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14992, 9865, 22628, 18350, 13935, 15910, 17525, 17867] avg=16384.0000 max=22628 max_rank=2 min=9865 min_rank=1 imbalance_ratio=1.381104 max_load_reduction=108444 max_load_reduction_pct=0.827362 moved_to_echo=65237 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[41, 0, 3, 7, 0, 50, 5, 4090, 0, 3878, 0, 3, 0, 0, 117, 0, 1975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 1, 0, 44, 0, 0, 3896, 2869, 0, 3538, 0, 0, 4095, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3523, 0, 133, 4025, 0, 168, 0, 175, 4, 0, 0, 750, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 4084, 0, 148, 4071, 90, 3834, 0, 1614, 14, 23, 198, 4096, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 430, 920, 65, 0, 4090, 0, 0, 3125, 0, 4093, 0, 3295, 1556, 1629, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 896, 1, 4087, 3, 3890, 168, 4, 0, 4094, 3733, 3566, 0, 289, 3992, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 104, 288, 0, 0, 149, 4019, 0, 238, 1, 3840, 132, 0, 1530, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 1199, 4096, 0, 0, 4094, 0, 0, 0, 0, 2384, 154, 0, 1, 0, 4092, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10169, 18231, 8782, 18557, 19768, 28825, 10660, 16080] avg=16384.0000 max=28825 max_rank=5 min=8782 min_rank=2 imbalance_ratio=1.759338 max_load_reduction=102247 max_load_reduction_pct=0.780083 moved_to_echo=75333 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 869, 3141, 255, 0, 27, 0, 3411, 0, 1635, 327, 0, 69, 4, 0, 1353, 3104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 6, 4061, 3918, 3259, 71, 3315, 0, 0, 3642, 0, 1336, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 86, 13, 564, 0, 0, 4095, 2146, 345, 547, 836, 3932, 4060, 0, 5, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 19, 3522, 585, 4082, 0, 11, 0, 0, 3970, 0, 0, 145, 1, 55, 314, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 996, 3861, 0, 484, 2036, 0, 28, 4062, 261, 3572, 286, 3, 148, 1555, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 868, 1, 3783, 263, 242, 0, 115, 22, 2142, 1158, 0, 0, 0, 0, 160, 2873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 369, 0, 1904, 56, 0, 280, 3420, 4066, 14, 1, 4053, 496, 2, 2, 1462, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3055, 60, 0, 0, 4071, 39, 1302, 8, 1, 3480, 4078, 0, 888, 0, 3930, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14195, 19635, 17052, 13065, 18221, 11627, 16283, 20994] avg=16384.0000 max=20994 max_rank=7 min=11627 min_rank=5 imbalance_ratio=1.281372 max_load_reduction=110078 max_load_reduction_pct=0.839828 moved_to_echo=67125 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 68, 429, 0, 4044, 289, 3659, 6, 1065, 652, 0, 0, 0, 0, 2121, 3047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 3776, 0, 2, 0, 4096, 119, 186, 0, 379, 0, 0, 2467, 0, 3484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 4089, 0, 0, 827, 9, 595, 3762, 3920, 0, 1296, 3929, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4035, 0, 4001, 0, 4096, 3024, 21, 60, 0, 4096, 6, 0, 4045, 117, 6, 35, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4057, 0, 3789, 0, 286, 0, 82, 3455, 252, 37, 2, 1626, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 908, 0, 2556, 4096, 2953, 0, 344, 2, 0, 2068, 0, 3, 51, 0, 1881, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3896, 6, 744, 0, 0, 0, 0, 1519, 4095, 0, 0, 0, 38, 0, 92, 1529, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 913, 4028, 9, 0, 536, 4063, 0, 4, 0, 0, 3632, 14, 0, 0, 0, 4087, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15381, 14727, 18531, 23739, 14460, 14919, 11948, 17367] avg=16384.0000 max=23739 max_rank=3 min=11948 min_rank=6 imbalance_ratio=1.448914 max_load_reduction=107333 max_load_reduction_pct=0.818886 moved_to_echo=58694 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 25, 140, 0, 0, 4073, 26, 4080, 15, 195, 0, 0, 4096, 0, 0, 8, 2980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3806, 0, 4072, 3470, 0, 673, 697, 0, 2324, 1, 183, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2527, 1, 0, 4060, 6, 158, 3368, 2778, 1696, 12, 3931, 2549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 10, 3959, 0, 3992, 1051, 0, 834, 4066, 0, 0, 354, 142, 0, 1057, 66, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1577, 6, 0, 0, 0, 599, 4096, 0, 1075, 3, 0, 3547, 0, 3549, 4039, 1634, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 913, 191, 3735, 0, 2931, 1, 95, 0, 49, 2, 0, 0, 64, 481, 405, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 191, 0, 4050, 1, 974, 4088, 1693, 233, 0, 14, 2969, 4087, 45, 1536, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 274, 0, 743, 86, 11, 3883, 0, 443, 721, 0, 0, 703, 4, 2420, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15638, 15368, 21086, 15777, 20980, 12941, 19914, 9368] avg=16384.0000 max=21086 max_rank=2 min=9368 min_rank=7 imbalance_ratio=1.286987 max_load_reduction=109986 max_load_reduction_pct=0.839127 moved_to_echo=63203 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 3101, 824, 0, 3960, 19, 2490, 0, 4080, 0, 22, 0, 0, 27, 0, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2168, 0, 3614, 823, 4067, 0, 2684, 0, 132, 1994, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 6, 2282, 89, 0, 4096, 2699, 13, 7, 1692, 3710, 794, 0, 0, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 1, 0, 3588, 0, 1586, 1262, 14, 1, 577, 3918, 56, 120, 242, 4096, 0, 254, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 113, 90, 0, 2700, 10, 0, 4084, 2, 91, 26, 1229, 1517, 1587, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 887, 188, 3602, 4096, 4096, 0, 26, 0, 0, 20, 0, 0, 633, 2007, 33, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 778, 13, 0, 0, 4042, 3321, 8, 0, 0, 3981, 3332, 2552, 1492, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 1049, 940, 4096, 0, 0, 3313, 0, 0, 0, 0, 3715, 154, 0, 0, 0, 4074, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14989, 15681, 20055, 15914, 11620, 15742, 19645, 17426] avg=16384.0000 max=20055 max_rank=2 min=11620 min_rank=4 imbalance_ratio=1.224060 max_load_reduction=111017 max_load_reduction_pct=0.846992 moved_to_echo=64433 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 481, 4086, 313, 3, 3540, 37, 4050, 0, 4065, 1050, 0, 3882, 0, 68, 859, 3022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 0, 0, 2, 4055, 1581, 4069, 1, 3061, 0, 0, 3307, 0, 3295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 169, 1, 5, 0, 31, 1, 355, 103, 26, 3, 3932, 3599, 0, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3653, 0, 3269, 1670, 0, 3, 0, 4057, 0, 246, 0, 0, 0, 290, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 72, 449, 4031, 1, 0, 18, 419, 739, 197, 0, 72, 1639, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 916, 24, 4069, 3663, 4095, 577, 2, 0, 744, 0, 0, 50, 0, 3302, 726, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 0, 78, 98, 7, 0, 0, 3644, 3285, 27, 16, 0, 11, 796, 63, 1541, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 222, 3383, 354, 0, 0, 4096, 70, 3467, 0, 11, 731, 0, 0, 0, 3, 3814, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[25457, 23322, 8365, 13355, 8609, 22107, 13612, 16245] avg=16384.0000 max=25457 max_rank=0 min=8365 min_rank=2 imbalance_ratio=1.553772 max_load_reduction=105615 max_load_reduction_pct=0.805779 moved_to_echo=60573 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 2432, 16768, 12544, 128, 1280, 27904, 256, 11520, 128, 512, 21760, 4480, 128, 1664, 128, 29952, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 1408, 18560, 1920, 128, 19840, 512, 29696, 128, 22912, 2048, 128, 8064, 128, 256, 5376, 20736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 7808, 640, 1920, 6528, 5376, 128, 1280, 21888, 28672, 2304, 6784, 4224, 19072, 9472, 2816, 12288, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[7296, 512, 28672, 12288, 21632, 1024, 1664, 256, 9344, 7040, 3584, 384, 1152, 14720, 3456, 19328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[4608, 16256, 384, 7296, 12800, 128, 13184, 18304, 7808, 2304, 4736, 26624, 15872, 0, 128, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[4096, 10752, 128, 6144, 384, 22144, 17408, 17408, 896, 20224, 0, 2560, 16128, 512, 12800, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[256, 8448, 128, 25856, 1152, 21248, 16512, 256, 5248, 4736, 21888, 512, 768, 7936, 12800, 1152, 1920, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[256, 1664, 128, 2816, 5120, 4352, 1536, 26496, 128, 3456, 17664, 6656, 19328, 2560, 13056, 8320, 13056, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[3985, 0, 0, 33, 0, 0, 0, 0, 0, 29, 0, 0, 0, 4095, 0, 4093, 1502, 0, 3, 0, 0, 0, 0, 322, 0, 2183, 4095, 3791, 0, 114, 0, 190, 0, 0, 105, 269, 3376, 3800, 0, 1, 4096, 1269, 0, 0, 0, 3, 0, 0, 109, 0, 756, 4096, 0, 7, 103, 159, 3308, 4092, 4081, 4080, 0, 0, 2, 0, 1979, 4, 4093, 316, 3456, 120, 438, 0, 0, 0, 0, 4096, 98, 0, 1, 1, 3974, 11, 2033, 1, 3539, 3129, 1302, 3886, 704, 63, 0, 2078, 202, 5, 87, 143, 2, 2950, 2, 50, 0, 0, 111, 0, 0, 0, 1, 0, 4096, 0, 0, 532, 2225, 4089, 2367, 0, 4059, 0, 4051, 0, 1, 0, 4068, 587, 4056, 3919, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[3989, 0, 51, 3815, 0, 0, 0, 0, 420, 19, 0, 0, 0, 8, 0, 3485, 3379, 26, 6, 0, 0, 0, 0, 65, 0, 1920, 8, 4096, 0, 4089, 0, 4096, 0, 0, 0, 28, 0, 3742, 1, 0, 4001, 3945, 0, 0, 0, 4092, 213, 0, 1, 0, 37, 4096, 0, 1, 618, 4, 32, 3928, 4096, 4096, 0, 0, 0, 0, 0, 1, 1, 2656, 4096, 594, 13, 2882, 479, 0, 5, 3851, 1, 0, 3790, 0, 40, 0, 0, 0, 4096, 1694, 57, 217, 272, 0, 0, 4078, 64, 81, 2003, 4079, 4, 4086, 72, 4096, 0, 3427, 0, 5, 0, 1117, 0, 0, 2442, 0, 66, 50, 0, 4091, 1368, 1, 4090, 4, 0, 467, 3, 0, 4062, 0, 3051, 1117, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[3999, 0, 2, 0, 3, 0, 0, 6, 26, 0, 2029, 0, 0, 2708, 106, 3952, 3784, 0, 1, 0, 0, 0, 0, 238, 1, 1177, 1, 2624, 28, 0, 1, 4096, 5, 0, 0, 143, 9, 4096, 8, 1, 3899, 4086, 1339, 178, 31, 4095, 3574, 0, 6, 0, 0, 4072, 7, 0, 468, 123, 2472, 0, 0, 3271, 16, 857, 0, 0, 2704, 0, 3772, 0, 352, 0, 27, 0, 0, 734, 0, 4032, 0, 0, 0, 0, 4015, 0, 26, 60, 3, 3954, 0, 92, 3729, 0, 188, 4028, 344, 6, 0, 0, 4095, 4095, 15, 3894, 0, 4023, 3977, 0, 1368, 5, 0, 0, 4042, 62, 81, 1797, 1, 3888, 38, 62, 4096, 0, 3847, 0, 0, 167, 1813, 0, 0, 12, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3, 0, 0, 2798, 81, 3923, 32, 0, 4091, 92, 0, 146, 0, 4072, 196, 4096, 568, 6, 0, 0, 1526, 0, 3200, 0, 0, 0, 3989, 3105, 159, 39, 0, 4004, 189, 0, 71, 1537, 152, 3694, 0, 4084, 3958, 66, 199, 0, 6, 0, 0, 2, 0, 599, 2513, 3367, 0, 443, 4096, 0, 1, 4090, 4077, 1824, 0, 2645, 0, 0, 4088, 10, 88, 2588, 132, 0, 54, 3616, 0, 0, 0, 3490, 0, 2, 0, 0, 170, 224, 323, 0, 0, 40, 0, 1172, 0, 359, 0, 2401, 3026, 1, 0, 807, 4078, 4093, 3974, 1616, 1, 4092, 0, 0, 101, 73, 896, 470, 0, 0, 0, 0, 2, 0, 3024, 137, 3913, 0, 0, 6, 0, 78, 0, 0, 4074, 4091, 23, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[2526, 0, 0, 0, 21, 0, 4, 35, 4067, 0, 0, 3, 204, 4011, 0, 4085, 3701, 2, 19, 15, 0, 0, 6, 0, 0, 0, 3853, 2, 0, 9, 0, 4095, 0, 0, 749, 469, 0, 4093, 237, 4036, 1386, 25, 4058, 0, 1, 140, 0, 0, 0, 0, 18, 3696, 0, 0, 1396, 0, 4095, 2292, 4096, 4089, 6, 3364, 0, 60, 4037, 43, 3373, 0, 0, 19, 0, 259, 0, 0, 58, 3682, 1, 0, 0, 0, 0, 191, 1, 0, 1928, 4091, 0, 4068, 0, 17, 71, 88, 3874, 0, 0, 1128, 3, 3005, 1505, 448, 1, 4003, 742, 0, 114, 0, 0, 962, 3862, 0, 271, 0, 520, 3445, 4047, 0, 4096, 13, 0, 0, 82, 0, 3976, 65, 3927, 4092, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[4073, 0, 104, 0, 0, 111, 8, 36, 412, 0, 0, 3714, 0, 1346, 0, 3459, 4023, 65, 1, 53, 0, 0, 136, 4096, 32, 8, 0, 4015, 0, 4087, 0, 3871, 0, 2, 4016, 3166, 0, 19, 0, 0, 3942, 1350, 0, 12, 0, 2155, 0, 0, 266, 3184, 33, 4096, 0, 174, 14, 27, 117, 2873, 1863, 4083, 5, 572, 0, 81, 863, 1, 4045, 0, 718, 29, 14, 0, 318, 1642, 10, 622, 0, 0, 0, 0, 0, 0, 0, 345, 3755, 4096, 1067, 0, 0, 0, 0, 4022, 0, 0, 166, 0, 25, 3998, 0, 3787, 44, 0, 4034, 0, 265, 3830, 7, 0, 3930, 0, 0, 3819, 3, 60, 875, 0, 3945, 1900, 4085, 1901, 0, 0, 0, 79, 4090, 2707, 112, 163] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 24, 0, 0, 18, 0, 4096, 14, 0, 4, 0, 0, 4043, 2460, 3976, 0, 0, 0, 1596, 0, 0, 123, 0, 13, 3836, 2762, 848, 0, 376, 3731, 0, 0, 378, 0, 0, 4096, 98, 4022, 3940, 0, 336, 377, 0, 1, 1710, 0, 0, 37, 7, 3908, 1, 0, 4, 4074, 3104, 4090, 4096, 2006, 0, 42, 0, 0, 4094, 0, 2, 34, 0, 0, 4, 3827, 0, 26, 0, 4091, 5, 0, 1555, 171, 72, 1, 137, 0, 784, 3437, 2483, 1585, 0, 0, 1, 4034, 0, 0, 0, 235, 3476, 4013, 3044, 998, 120, 0, 0, 0, 0, 2429, 3681, 254, 0, 7, 105, 0, 23, 2493, 952, 0, 6, 47, 4061, 430, 181, 0, 1834, 3900, 4069, 4057, 0, 67] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[58, 0, 22, 1, 0, 2, 0, 0, 4086, 0, 0, 1556, 1, 121, 1, 3722, 4096, 64, 0, 0, 1345, 0, 131, 0, 0, 0, 37, 3187, 2, 0, 1, 4096, 9, 0, 0, 0, 116, 4095, 0, 4096, 4054, 0, 0, 0, 0, 38, 4051, 0, 0, 45, 5, 3821, 0, 0, 1393, 0, 4072, 3346, 4068, 3538, 2, 4087, 6, 975, 4096, 41, 0, 92, 0, 0, 1, 1, 0, 0, 0, 3720, 0, 0, 4039, 25, 52, 0, 1037, 0, 125, 3422, 4075, 399, 0, 0, 0, 4012, 0, 0, 177, 0, 3, 4025, 0, 4093, 14, 4001, 0, 1669, 518, 0, 0, 241, 3912, 27, 22, 0, 11, 1988, 0, 8, 1648, 0, 3499, 382, 378, 2798, 3980, 292, 4002, 3901, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3985, 0, 0, 33, 0, 0, 0, 0, 0, 29, 0, 0, 0, 4095, 0, 4093, 3614, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1502, 0, 3, 0, 0, 0, 0, 322, 0, 2183, 4095, 3791, 0, 114, 0, 190, 1015, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 269, 3376, 3800, 0, 1, 4096, 1269, 0, 0, 0, 3, 0, 0, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 756, 301, 0, 7, 103, 159, 3308, 4092, 4081, 4080, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1979, 4, 4093, 316, 3456, 120, 438, 0, 0, 0, 0, 4096, 98, 0, 1, 1, 3535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 11, 2033, 1, 3539, 3129, 1302, 3886, 704, 63, 0, 2078, 202, 5, 87, 143, 1008, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1560, 2, 50, 0, 0, 111, 0, 0, 0, 1, 0, 4096, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2225, 4089, 2367, 0, 2187, 0, 4051, 0, 1, 0, 4068, 587, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16109, 13597, 13776, 16998, 18137, 22607, 6354, 23494] avg=16384.0000 max=23494 max_rank=7 min=6354 min_rank=6 imbalance_ratio=1.433960 max_load_reduction=107578 max_load_reduction_pct=0.820755 moved_to_echo=70592 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3999, 0, 2, 0, 3, 0, 0, 6, 26, 0, 2029, 0, 0, 2708, 106, 3952, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3784, 0, 1, 0, 0, 0, 0, 238, 1, 1177, 1, 2624, 28, 0, 1, 4096, 1022, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 143, 9, 4096, 8, 1, 3899, 4086, 1339, 178, 31, 4095, 3574, 0, 861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 308, 7, 0, 468, 123, 2472, 0, 0, 3271, 16, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2704, 0, 3772, 0, 352, 0, 27, 0, 0, 734, 0, 4032, 0, 0, 0, 0, 3510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 26, 60, 3, 3954, 0, 92, 3729, 0, 188, 4028, 344, 6, 0, 0, 1395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2176, 15, 3894, 0, 4023, 3977, 0, 1368, 5, 0, 0, 4042, 62, 81, 1797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3888, 38, 62, 2213, 0, 3847, 0, 0, 167, 1813, 0, 0, 12, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13085, 13497, 22325, 7528, 15131, 17840, 25535, 16131] avg=16384.0000 max=25535 max_rank=6 min=7528 min_rank=3 imbalance_ratio=1.558533 max_load_reduction=105537 max_load_reduction_pct=0.805183 moved_to_echo=74637 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4073, 0, 104, 0, 0, 111, 8, 36, 412, 0, 0, 3714, 0, 1346, 0, 3459, 3644, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 65, 1, 53, 0, 0, 136, 4096, 32, 8, 0, 4015, 0, 4087, 0, 3871, 984, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4016, 3166, 0, 19, 0, 0, 3942, 1350, 0, 12, 0, 2155, 0, 0, 829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 3184, 33, 310, 0, 174, 14, 27, 117, 2873, 1863, 4083, 5, 572, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 863, 1, 4045, 0, 718, 29, 14, 0, 318, 1642, 10, 622, 0, 0, 0, 0, 3530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 3755, 4096, 1067, 0, 0, 0, 0, 4022, 0, 0, 166, 0, 1362, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2124, 0, 3787, 44, 0, 4034, 0, 265, 3830, 7, 0, 3930, 0, 0, 3819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 60, 875, 0, 2132, 1900, 4085, 1901, 0, 0, 0, 79, 0, 2707, 112, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17163, 21883, 15491, 13602, 11792, 15259, 21865, 14017] avg=16384.0000 max=21883 max_rank=1 min=11792 min_rank=4 imbalance_ratio=1.335632 max_load_reduction=109189 max_load_reduction_pct=0.833046 moved_to_echo=62933 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 2798, 81, 3923, 32, 0, 4091, 92, 0, 146, 0, 4072, 196, 4096, 3629, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 6, 0, 0, 1526, 0, 3200, 0, 0, 0, 3989, 3105, 159, 39, 0, 4004, 976, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 71, 1537, 152, 3694, 0, 4084, 3958, 66, 199, 0, 6, 0, 0, 2, 822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 599, 2513, 255, 0, 443, 4096, 0, 1, 4090, 4077, 1824, 0, 2645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 10, 88, 2588, 132, 0, 54, 3616, 0, 0, 0, 3490, 0, 2, 0, 0, 2902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 224, 323, 0, 0, 40, 0, 1172, 0, 359, 0, 2401, 3026, 1, 0, 807, 1394, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 2175, 3974, 1616, 1, 4092, 0, 0, 101, 73, 896, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3024, 137, 2115, 0, 0, 6, 0, 78, 0, 0, 0, 4091, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23369, 18096, 14780, 20543, 16970, 10362, 17476, 9476] avg=16384.0000 max=23369 max_rank=0 min=9476 min_rank=7 imbalance_ratio=1.426331 max_load_reduction=107703 max_load_reduction_pct=0.821709 moved_to_echo=54284 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2526, 0, 0, 0, 21, 0, 4, 35, 4067, 0, 0, 3, 204, 4011, 0, 4085, 3498, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3701, 2, 19, 15, 0, 0, 6, 0, 0, 0, 3853, 2, 0, 9, 0, 4095, 1022, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 749, 469, 0, 4093, 237, 4036, 1386, 25, 4058, 0, 1, 140, 0, 0, 861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 279, 0, 0, 1396, 0, 4095, 2292, 4096, 4089, 6, 3364, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 43, 3373, 0, 0, 19, 0, 259, 0, 0, 58, 3682, 1, 0, 0, 0, 3186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 1, 0, 1928, 4091, 0, 4068, 0, 17, 71, 88, 3874, 0, 0, 1128, 1023, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1597, 1505, 448, 1, 4003, 742, 0, 114, 0, 0, 962, 3862, 0, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 3445, 4047, 0, 2213, 13, 0, 0, 82, 0, 3976, 65, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18685, 13109, 16055, 19695, 14658, 16909, 13508, 18453] avg=16384.0000 max=19695 max_rank=3 min=13109 min_rank=1 imbalance_ratio=1.202087 max_load_reduction=111377 max_load_reduction_pct=0.849739 moved_to_echo=63528 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3989, 0, 51, 3815, 0, 0, 0, 0, 420, 19, 0, 0, 0, 8, 0, 3485, 2718, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3379, 26, 6, 0, 0, 0, 0, 65, 0, 1920, 8, 4096, 0, 4089, 0, 4096, 1020, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 3742, 1, 0, 4001, 3945, 0, 0, 0, 4092, 213, 0, 859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 37, 310, 0, 1, 618, 4, 32, 3928, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2656, 4096, 594, 13, 2882, 479, 0, 5, 3851, 1, 0, 3790, 0, 3530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 4096, 1694, 57, 217, 272, 0, 0, 4078, 64, 81, 2003, 4079, 1392, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2171, 72, 4096, 0, 3427, 0, 5, 0, 1117, 0, 0, 2442, 0, 66, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1368, 1, 2211, 4, 0, 467, 3, 0, 4062, 0, 0, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14761, 19228, 16881, 13123, 21899, 18406, 13450, 13324] avg=16384.0000 max=21899 max_rank=4 min=13123 min_rank=3 imbalance_ratio=1.336609 max_load_reduction=109173 max_load_reduction_pct=0.832924 moved_to_echo=67079 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[58, 0, 22, 1, 0, 2, 0, 0, 4086, 0, 0, 1556, 1, 121, 1, 3722, 3564, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 64, 0, 0, 1345, 0, 131, 0, 0, 0, 37, 3187, 2, 0, 1, 4096, 411, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 116, 4095, 0, 4096, 4054, 0, 0, 0, 0, 38, 4051, 0, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 5, 289, 0, 0, 1393, 0, 4072, 3346, 4068, 3538, 2, 4087, 6, 975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 41, 0, 92, 0, 0, 1, 1, 0, 0, 0, 3720, 0, 0, 4039, 25, 3293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 1037, 0, 125, 3422, 4075, 399, 0, 0, 0, 4012, 0, 0, 177, 0, 1371, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2138, 0, 4093, 14, 4001, 0, 1669, 518, 0, 0, 241, 3912, 27, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1988, 0, 8, 891, 0, 3499, 382, 378, 2798, 3980, 292, 0, 3901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13373, 13886, 16805, 21826, 15308, 15108, 16638, 18128] avg=16384.0000 max=21826 max_rank=3 min=13373 min_rank=0 imbalance_ratio=1.332153 max_load_reduction=109246 max_load_reduction_pct=0.833481 moved_to_echo=65182 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 24, 0, 0, 18, 0, 4096, 14, 0, 4, 0, 0, 4043, 2460, 3624, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 0, 0, 0, 1596, 0, 0, 123, 0, 13, 3836, 2762, 848, 0, 376, 3731, 1, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 4096, 98, 4022, 3940, 0, 336, 377, 0, 1, 1710, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 7, 296, 1, 0, 4, 4074, 3104, 4090, 4096, 2006, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 34, 0, 0, 4, 3827, 0, 26, 0, 4091, 5, 0, 1555, 171, 3368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 1, 137, 0, 784, 3437, 2483, 1585, 0, 0, 1, 4034, 0, 0, 0, 235, 1367, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3476, 2132, 3044, 998, 120, 0, 0, 0, 0, 2429, 3681, 254, 0, 7, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2493, 952, 0, 4, 47, 4061, 430, 181, 0, 1834, 3900, 0, 4057, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14527, 17776, 14959, 17757, 17177, 14581, 16246, 18049] avg=16384.0000 max=18049 max_rank=7 min=14527 min_rank=0 imbalance_ratio=1.101624 max_load_reduction=113023 max_load_reduction_pct=0.862297 moved_to_echo=66053 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[2816, 20096, 12672, 256, 14080, 2048, 19584, 3200, 768, 3072, 19840, 4992, 0, 23936, 256, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[11776, 16128, 8704, 19072, 256, 19584, 8960, 1792, 2432, 7552, 4608, 2048, 22400, 128, 640, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[8448, 512, 3584, 512, 14336, 23936, 9088, 11520, 4736, 512, 384, 24832, 7552, 128, 2560, 6400, 10368, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[21888, 128, 15488, 5760, 8832, 768, 640, 10624, 896, 2432, 128, 27648, 128, 128, 9472, 256, 26880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[25088, 256, 128, 128, 4480, 0, 3584, 4864, 128, 5376, 15872, 23680, 1152, 8448, 384, 28288, 6528, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 128, 5376, 5632, 3712, 27648, 384, 16256, 29312, 10752, 6016, 640, 128, 10624, 9600, 128, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[384, 3968, 3456, 2432, 128, 640, 8192, 4480, 17280, 24832, 26496, 27008, 128, 11648, 128, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[18688, 0, 256, 6784, 128, 4096, 128, 128, 17280, 256, 2048, 5504, 256, 16384, 4352, 29440, 24320, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[10, 0, 22, 0, 53, 4081, 238, 3526, 5, 87, 609, 0, 88, 0, 2466, 0, 4079, 4096, 66, 6, 0, 6, 3159, 0, 80, 4087, 0, 130, 0, 4060, 0, 0, 2061, 17, 1, 0, 3211, 2, 4086, 3107, 741, 842, 0, 1480, 0, 3987, 1682, 1, 0, 0, 0, 67, 0, 0, 265, 0, 0, 4049, 3310, 0, 0, 0, 13, 0, 0, 0, 1173, 164, 0, 3749, 0, 0, 231, 4054, 3807, 0, 30, 1083, 0, 0, 0, 2779, 183, 17, 382, 34, 0, 4096, 3991, 0, 4096, 0, 50, 14, 0, 2, 0, 4091, 2396, 3927, 494, 1, 48, 4090, 749, 0, 0, 0, 429, 0, 552, 0, 0, 4034, 4095, 0, 3840, 4038, 0, 253, 0, 3641, 0, 464, 4070, 0, 22, 27] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4085, 0, 6, 0, 74, 4096, 2583, 0, 0, 3858, 69, 101, 17, 0, 0, 3605, 0, 3970, 0, 8, 489, 4005, 0, 3, 0, 4090, 0, 15, 619, 0, 0, 1, 0, 0, 37, 0, 1519, 2943, 582, 3102, 84, 4095, 3891, 1072, 0, 4091, 3831, 20, 88, 0, 1, 1642, 10, 0, 0, 0, 521, 0, 19, 121, 0, 0, 25, 2, 1521, 72, 3420, 4, 0, 2611, 62, 1, 0, 2483, 2698, 22, 0, 4072, 2, 0, 1, 3268, 458, 4095, 0, 0, 4089, 3579, 890, 0, 3950, 5, 77, 0, 0, 3900, 0, 101, 4, 0, 22, 6, 0, 3925, 531, 1, 22, 20, 373, 3905, 1365, 3717, 1270, 0, 0, 418, 4016, 0, 2382, 4052, 0, 6, 230, 14, 4002, 18, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3193, 1, 2227, 191, 0, 3509, 4077, 813, 2986, 37, 0, 0, 0, 4017, 0, 0, 46, 110, 0, 0, 1690, 13, 0, 0, 4087, 2301, 0, 0, 74, 4096, 0, 0, 0, 1, 0, 1731, 4071, 376, 4096, 1, 4081, 3760, 10, 0, 93, 4081, 3932, 0, 3539, 0, 0, 0, 15, 0, 9, 0, 0, 0, 0, 0, 59, 3, 2, 3552, 7, 4054, 17, 0, 0, 0, 0, 0, 0, 1883, 4093, 0, 2, 4096, 0, 4095, 0, 3863, 3466, 4092, 3973, 74, 1723, 0, 5, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 676, 0, 0, 0, 4096, 0, 4067, 3, 0, 62, 0, 2907, 12, 5, 0, 1, 0, 4005, 4053, 0, 0, 0, 0, 0, 20, 0, 0, 4064, 682] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[3967, 0, 1, 0, 0, 4076, 3576, 4057, 0, 4063, 0, 0, 4093, 2521, 112, 4, 0, 0, 140, 619, 180, 109, 0, 8, 0, 3691, 0, 1703, 0, 2488, 0, 0, 0, 0, 5, 0, 0, 2, 3997, 4, 3744, 5, 4083, 3947, 3671, 3806, 3826, 1, 1656, 14, 74, 6, 0, 1, 99, 0, 0, 87, 24, 0, 3922, 0, 0, 0, 0, 0, 4008, 1, 0, 105, 0, 308, 343, 1, 15, 10, 366, 3989, 0, 2409, 15, 4049, 0, 4076, 586, 1, 4096, 38, 4090, 0, 4046, 52, 0, 0, 159, 3597, 18, 0, 0, 284, 245, 0, 7, 4089, 18, 0, 0, 1, 2, 0, 633, 3954, 1008, 1701, 45, 21, 4095, 84, 2, 68, 0, 3, 2202, 3745, 4064, 0, 41, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 4096, 0, 3932, 3458, 2926, 0, 0, 0, 2, 0, 20, 0, 0, 3966, 0, 0, 0, 3279, 1824, 0, 1, 4093, 17, 4085, 0, 2760, 0, 58, 0, 2, 0, 468, 784, 0, 4070, 3, 18, 0, 118, 0, 0, 4096, 4092, 0, 14, 0, 0, 1, 4, 0, 0, 0, 0, 4, 4096, 39, 0, 0, 277, 0, 3259, 0, 2469, 0, 92, 4096, 0, 474, 0, 0, 2359, 16, 1191, 1038, 1033, 0, 0, 3686, 0, 4096, 0, 0, 4095, 0, 3972, 0, 4094, 174, 0, 46, 0, 38, 4079, 3821, 0, 0, 873, 0, 232, 4090, 4073, 0, 1, 0, 0, 716, 3910, 3007, 0, 668, 12, 1, 3714, 67, 892, 4048, 0, 8, 0, 4034, 0, 0, 0, 3986] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[2942, 1, 0, 0, 19, 3593, 4084, 4091, 3849, 3937, 74, 0, 0, 2, 0, 1, 0, 3543, 33, 0, 935, 0, 2193, 0, 0, 4096, 185, 1, 11, 3949, 0, 3, 36, 0, 0, 0, 0, 90, 1, 1084, 0, 0, 1342, 109, 940, 0, 3452, 497, 219, 0, 1337, 0, 0, 0, 0, 0, 0, 157, 3565, 0, 0, 1, 0, 2, 4072, 3619, 32, 1, 829, 4035, 0, 9, 138, 4096, 4066, 4, 690, 4096, 4067, 3255, 634, 2455, 984, 4096, 0, 1238, 0, 419, 51, 0, 4, 0, 8, 170, 0, 4084, 4003, 165, 0, 1269, 4, 61, 0, 4096, 0, 475, 2946, 0, 0, 51, 4089, 376, 47, 0, 0, 0, 3007, 239, 4096, 695, 0, 3913, 0, 0, 164, 0, 0, 3850] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[3210, 1, 99, 12, 0, 0, 247, 4080, 461, 0, 0, 0, 0, 698, 4081, 0, 0, 230, 0, 0, 2, 1, 1, 0, 1310, 3714, 32, 2754, 652, 4096, 0, 0, 0, 427, 0, 3070, 0, 1, 3888, 1, 4088, 0, 3622, 3828, 9, 3796, 4091, 0, 3865, 0, 0, 0, 4, 0, 4046, 0, 0, 0, 0, 0, 0, 3, 1, 1, 4096, 0, 0, 0, 0, 71, 1, 0, 0, 4088, 589, 0, 4089, 4096, 0, 4060, 0, 0, 15, 3739, 0, 2, 0, 0, 1399, 0, 4083, 10, 0, 0, 1, 4075, 0, 0, 0, 4027, 0, 3027, 0, 4096, 22, 88, 3371, 0, 2678, 3810, 0, 1086, 0, 0, 420, 0, 3797, 0, 3762, 0, 0, 0, 0, 3975, 2071, 0, 6, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4087, 0, 2, 1, 0, 0, 2569, 195, 268, 3448, 5, 0, 3734, 1, 17, 0, 3, 4069, 161, 7, 28, 3674, 1468, 0, 0, 553, 3617, 1, 23, 4096, 0, 0, 8, 0, 0, 1785, 17, 0, 3736, 3800, 3247, 3598, 24, 2754, 45, 3596, 4092, 4041, 4084, 0, 601, 498, 419, 1, 0, 557, 0, 11, 0, 30, 0, 0, 0, 23, 589, 201, 0, 0, 0, 0, 88, 0, 1, 6, 4095, 0, 0, 3888, 0, 0, 0, 4088, 1026, 6, 42, 0, 2980, 3588, 550, 4091, 4017, 65, 0, 0, 0, 3910, 1251, 2246, 0, 0, 0, 0, 275, 4095, 0, 1, 0, 0, 4096, 9, 10, 3712, 121, 7, 3946, 0, 3945, 3, 0, 29, 0, 1967, 0, 2, 0, 86, 2824, 222] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2114, 0, 6, 0, 74, 4096, 2583, 0, 0, 3858, 69, 101, 17, 0, 0, 3605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 0, 8, 489, 4005, 0, 3, 0, 4090, 0, 15, 619, 0, 0, 1, 1387, 121, 166, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 1519, 2943, 582, 3102, 84, 4095, 3891, 1072, 0, 1334, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 1, 1642, 10, 0, 0, 0, 521, 0, 19, 121, 0, 0, 25, 2, 3831, 3758, 2757, 1805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 72, 3420, 4, 0, 2611, 62, 1, 0, 2483, 2698, 22, 0, 2685, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3268, 458, 4095, 0, 0, 4089, 3579, 890, 0, 192, 5, 77, 0, 0, 3900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 4, 0, 22, 6, 0, 3804, 531, 1, 22, 20, 373, 3905, 1365, 3717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1270, 0, 0, 418, 3970, 0, 2382, 4052, 0, 6, 230, 14, 4002, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16523, 14920, 18679, 14580, 15581, 20554, 13871, 16364] avg=16384.0000 max=20554 max_rank=5 min=13871 min_rank=6 imbalance_ratio=1.254517 max_load_reduction=110518 max_load_reduction_pct=0.843185 moved_to_echo=66370 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 22, 0, 53, 4081, 238, 3526, 5, 87, 609, 0, 88, 0, 2466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4096, 66, 6, 0, 6, 3159, 0, 80, 4087, 0, 130, 0, 4060, 0, 0, 369, 126, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2061, 17, 1, 0, 3211, 2, 4086, 3107, 741, 842, 0, 1480, 0, 1300, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 265, 0, 0, 4049, 3310, 0, 0, 0, 13, 0, 1682, 3897, 2687, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1173, 164, 0, 3749, 0, 0, 231, 4054, 3807, 0, 30, 714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2779, 183, 17, 382, 34, 0, 4096, 3991, 0, 199, 0, 50, 14, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2396, 3927, 494, 1, 48, 3964, 749, 0, 0, 0, 429, 0, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 4095, 0, 3796, 4038, 0, 253, 0, 3641, 0, 464, 4070, 0, 22, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11181, 20308, 16849, 15974, 13922, 11747, 16651, 24440] avg=16384.0000 max=24440 max_rank=7 min=11181 min_rank=0 imbalance_ratio=1.491699 max_load_reduction=106632 max_load_reduction_pct=0.813538 moved_to_echo=66760 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 0, 4096, 0, 3932, 3458, 2926, 0, 0, 0, 2, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 0, 3279, 1824, 0, 1, 4093, 17, 4085, 0, 2760, 0, 58, 353, 126, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 468, 784, 0, 4070, 3, 18, 0, 118, 0, 0, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1, 4, 0, 0, 0, 0, 4, 4096, 39, 0, 0, 277, 0, 4092, 3895, 2760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3259, 0, 2469, 0, 92, 4096, 0, 474, 0, 0, 2359, 16, 1191, 685, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3686, 0, 4096, 0, 0, 4095, 0, 3972, 0, 199, 174, 0, 46, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 3821, 0, 0, 873, 0, 232, 3964, 4073, 0, 1, 0, 0, 716, 3910, 3007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 12, 1, 3671, 67, 892, 4048, 0, 8, 0, 4034, 0, 0, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14443, 20605, 6799, 15182, 15674, 16306, 24676, 17387] avg=16384.0000 max=24676 max_rank=6 min=6799 min_rank=2 imbalance_ratio=1.506104 max_load_reduction=106396 max_load_reduction_pct=0.811737 moved_to_echo=74043 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2115, 0, 2, 1, 0, 0, 2569, 195, 268, 3448, 5, 0, 3734, 1, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4069, 161, 7, 28, 3674, 1468, 0, 0, 553, 3617, 1, 23, 4096, 0, 0, 1325, 126, 166, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1785, 17, 0, 3736, 3800, 3247, 3598, 24, 2754, 45, 1173, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 601, 498, 419, 1, 0, 557, 0, 11, 0, 30, 0, 0, 0, 23, 4092, 3822, 2423, 1806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 201, 0, 0, 0, 0, 88, 0, 1, 6, 4095, 0, 0, 2563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1026, 6, 42, 0, 2980, 3588, 550, 4091, 195, 65, 0, 0, 0, 3910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1251, 2246, 0, 0, 0, 0, 275, 3969, 0, 1, 0, 0, 4096, 9, 10, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 7, 3946, 0, 3899, 3, 0, 29, 0, 1967, 0, 2, 0, 86, 2824, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12355, 19363, 24228, 18367, 7543, 20541, 15569, 13106] avg=16384.0000 max=24228 max_rank=2 min=7543 min_rank=4 imbalance_ratio=1.478760 max_load_reduction=106844 max_load_reduction_pct=0.815155 moved_to_echo=56759 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2052, 0, 1, 0, 0, 4076, 3576, 4057, 0, 4063, 0, 0, 4093, 2521, 112, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 619, 180, 109, 0, 8, 0, 3691, 0, 1703, 0, 2488, 0, 0, 1359, 126, 162, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2, 3997, 4, 3744, 5, 4083, 3947, 3671, 1241, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1656, 14, 74, 6, 0, 1, 99, 0, 0, 87, 24, 0, 3922, 0, 0, 0, 3826, 3850, 2565, 1753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 1, 0, 105, 0, 308, 343, 1, 15, 10, 366, 2630, 0, 2409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4049, 0, 4076, 586, 1, 4096, 38, 4090, 0, 196, 52, 0, 0, 159, 3597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 284, 245, 0, 7, 3963, 18, 0, 0, 1, 2, 0, 633, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1701, 45, 21, 4048, 84, 2, 68, 0, 3, 2202, 3745, 4064, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[24555, 10632, 20700, 17877, 10196, 20955, 9125, 17032] avg=16384.0000 max=24555 max_rank=0 min=9125 min_rank=6 imbalance_ratio=1.498718 max_load_reduction=106517 max_load_reduction_pct=0.812660 moved_to_echo=57308 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1645, 1, 2227, 191, 0, 3509, 4077, 813, 2986, 37, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 110, 0, 0, 1690, 13, 0, 0, 4087, 2301, 0, 0, 74, 4096, 0, 0, 1400, 131, 133, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1731, 4071, 376, 4096, 1, 4081, 3760, 10, 0, 93, 1326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3539, 0, 0, 0, 15, 0, 9, 0, 0, 0, 0, 0, 59, 3, 2, 3552, 3932, 3902, 2755, 1415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4054, 17, 0, 0, 0, 0, 0, 0, 1883, 4093, 0, 2, 2696, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3863, 3466, 4092, 3973, 74, 1723, 0, 5, 0, 193, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 0, 0, 0, 3965, 0, 4067, 3, 0, 62, 0, 2907, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 3955, 4053, 0, 0, 0, 0, 0, 20, 0, 0, 4064, 682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19503, 14131, 19546, 19183, 16847, 17390, 11692, 12780] avg=16384.0000 max=19546 max_rank=2 min=11692 min_rank=6 imbalance_ratio=1.192993 max_load_reduction=111526 max_load_reduction_pct=0.850876 moved_to_echo=58709 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1522, 1, 0, 0, 19, 3593, 4084, 4091, 3849, 3937, 74, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3543, 33, 0, 935, 0, 2193, 0, 0, 4096, 185, 1, 11, 3949, 0, 3, 1395, 126, 120, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 90, 1, 1084, 0, 0, 1342, 109, 940, 0, 0, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 1337, 0, 0, 0, 0, 0, 0, 157, 3565, 0, 0, 1, 0, 2, 3452, 3, 0, 1300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 3619, 32, 1, 829, 4035, 0, 9, 138, 4096, 4066, 4, 690, 2701, 4067, 3255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 634, 2455, 984, 4096, 0, 1238, 0, 419, 51, 0, 1, 0, 8, 170, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 165, 0, 1269, 4, 61, 0, 3970, 0, 475, 2946, 0, 0, 51, 4089, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 2972, 239, 4096, 695, 0, 3913, 0, 0, 164, 0, 0, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21173, 16625, 4099, 10036, 31614, 14140, 17409, 15976] avg=16384.0000 max=31614 max_rank=4 min=4099 min_rank=2 imbalance_ratio=1.929565 max_load_reduction=99458 max_load_reduction_pct=0.758804 moved_to_echo=79139 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1660, 1, 99, 12, 0, 0, 247, 4080, 461, 0, 0, 0, 0, 698, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 2, 1, 1, 0, 1310, 3714, 32, 2754, 652, 4096, 0, 0, 1395, 126, 131, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 0, 3070, 0, 1, 3888, 1, 4088, 0, 3622, 3828, 9, 1238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 0, 0, 0, 4, 0, 4046, 0, 0, 0, 0, 0, 0, 3, 1, 1, 4091, 3885, 2558, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 71, 1, 0, 0, 4088, 589, 0, 4089, 2701, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 3739, 0, 2, 0, 0, 1399, 0, 198, 10, 0, 0, 1, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 3027, 0, 3970, 22, 88, 3371, 0, 2678, 3810, 0, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 0, 3753, 0, 3762, 0, 0, 0, 0, 3975, 2071, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11339, 14488, 20172, 19873, 19695, 9439, 22079, 13987] avg=16384.0000 max=22079 max_rank=6 min=9439 min_rank=5 imbalance_ratio=1.347595 max_load_reduction=108993 max_load_reduction_pct=0.831551 moved_to_echo=65200 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[11136, 128, 2432, 256, 4352, 19456, 21376, 20224, 10496, 15488, 768, 128, 7936, 7296, 6784, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2176, 512, 128, 7168, 9728, 3456, 24576, 11136, 16128, 12416, 13184, 13312, 4864, 8960, 0, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[4224, 20096, 512, 640, 3328, 11136, 8704, 128, 5504, 26752, 3968, 8704, 1408, 25600, 0, 128, 9088, 1024, 896, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[768, 24192, 6144, 24320, 4992, 1408, 17024, 11776, 14976, 4096, 1408, 384, 256, 256, 256, 19712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9472, 10496, 2432, 10240, 1664, 3200, 640, 31616, 5504, 4736, 6400, 128, 7680, 8576, 13568, 15872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[13568, 128, 2048, 2304, 512, 128, 4480, 640, 640, 4352, 11136, 256, 4096, 128, 384, 3584, 29056, 27136, 18560, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[13568, 8064, 11136, 256, 1024, 14720, 256, 896, 768, 16640, 21760, 128, 6400, 17408, 5120, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[2560, 6528, 8576, 512, 30080, 8576, 11136, 9216, 0, 9600, 2432, 12288, 14464, 128, 7040, 8832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[146, 0, 30, 212, 0, 0, 3949, 0, 0, 3572, 0, 4090, 379, 4019, 8, 0, 0, 3974, 2, 14, 0, 1358, 1168, 0, 0, 121, 4096, 16, 14, 0, 0, 1756, 1375, 1, 0, 1218, 8, 446, 1, 90, 4095, 0, 2273, 3961, 0, 90, 3, 2, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 3178, 0, 0, 231, 4092, 0, 7, 190, 0, 0, 5, 0, 0, 0, 0, 0, 0, 60, 4088, 0, 1559, 0, 0, 847, 0, 4079, 2325, 0, 4090, 0, 22, 4045, 0, 0, 0, 125, 1, 0, 3075, 0, 3979, 136, 1362, 3169, 0, 0, 3484, 3961, 3995, 0, 0, 0, 0, 3737, 3969, 3796, 30, 529, 13, 4071, 0, 4088, 39, 3902, 3883, 1, 228, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1814, 0, 8, 18, 0, 32, 0, 1902, 4059, 78, 40, 32, 4096, 0, 5, 4045, 4092, 1096, 4064, 35, 3650, 0, 3863, 4, 3914, 0, 530, 3308, 26, 3, 0, 3531, 492, 52, 1576, 7, 52, 2, 2207, 3644, 9, 3360, 0, 0, 4056, 0, 2447, 0, 0, 4, 3669, 4040, 708, 0, 100, 0, 49, 0, 2263, 2367, 0, 10, 3841, 0, 0, 26, 391, 0, 3, 0, 0, 4, 0, 4096, 0, 3907, 258, 1, 0, 0, 1029, 2289, 0, 3745, 746, 2, 3955, 76, 22, 0, 0, 17, 0, 0, 86, 0, 0, 2, 1670, 1250, 1, 2, 0, 0, 4019, 53, 29, 0, 4094, 340, 3735, 4088, 59, 451, 3290, 4091, 269, 44, 3476, 22, 0, 188, 44, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 39, 0, 2632, 0, 309, 4083, 3517, 4096, 0, 4095, 3358, 0, 0, 180, 0, 0, 0, 142, 381, 4092, 6, 55, 1336, 0, 374, 0, 1475, 1528, 2668, 22, 2430, 0, 0, 716, 2, 0, 0, 0, 705, 3, 42, 0, 0, 0, 8, 0, 3480, 0, 0, 0, 43, 0, 341, 998, 0, 2, 32, 940, 3, 0, 3470, 1910, 10, 0, 0, 0, 4075, 27, 254, 3810, 0, 0, 1, 4095, 8, 4095, 3754, 0, 0, 2, 0, 3920, 0, 3828, 1, 244, 0, 3801, 0, 0, 0, 0, 458, 4068, 0, 0, 21, 4091, 0, 4021, 4096, 37, 1, 0, 2445, 1738, 3459, 0, 5, 0, 5, 6, 4088, 1834, 633, 845, 4096, 2318, 2335, 0, 4086, 3739, 131, 1, 105, 972] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[3757, 5, 0, 1179, 0, 0, 0, 0, 0, 0, 2, 3052, 2358, 0, 1, 0, 4092, 66, 0, 16, 2401, 0, 129, 3095, 2527, 0, 4001, 0, 16, 9, 0, 3425, 3, 241, 0, 0, 572, 0, 0, 0, 0, 0, 80, 3, 0, 4006, 0, 0, 283, 26, 0, 689, 0, 26, 4017, 74, 0, 0, 1196, 6, 0, 0, 4065, 555, 697, 4096, 235, 0, 3834, 0, 135, 2386, 0, 291, 7, 2389, 5, 4063, 0, 959, 109, 2, 3477, 19, 2622, 0, 0, 3593, 0, 17, 4092, 0, 0, 0, 0, 0, 8, 3053, 0, 0, 0, 4087, 3731, 0, 0, 9, 2480, 0, 3662, 3, 0, 2, 192, 237, 3989, 970, 2763, 0, 4092, 4088, 3588, 23, 4045, 4042, 3, 3737, 0, 3267] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 174, 25, 1453, 0, 4091, 4096, 2894, 0, 0, 522, 80, 0, 0, 3502, 3907, 0, 333, 17, 0, 4096, 0, 4084, 0, 0, 0, 4014, 1, 0, 4092, 0, 0, 230, 3675, 0, 3, 159, 0, 0, 4096, 0, 32, 4093, 0, 0, 0, 0, 0, 678, 129, 0, 0, 5, 0, 1, 17, 25, 4096, 0, 0, 3, 47, 55, 6, 0, 0, 33, 0, 0, 0, 0, 4096, 1488, 295, 0, 2, 4033, 853, 4059, 4096, 0, 4083, 38, 0, 3429, 0, 0, 1, 0, 0, 0, 4006, 3804, 484, 0, 2397, 4095, 0, 0, 3919, 7, 0, 0, 2, 11, 0, 4094, 2902, 4079, 1, 55, 43, 4096, 1, 2790, 0, 4088, 4095, 26, 0, 0, 744] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4089, 2, 156, 18, 91, 0, 308, 0, 0, 4077, 0, 1994, 2024, 0, 0, 0, 4013, 0, 0, 819, 16, 344, 12, 1268, 3998, 4044, 4090, 336, 3949, 0, 0, 0, 3842, 0, 0, 139, 0, 2, 0, 5, 0, 168, 0, 254, 0, 3908, 6, 1391, 68, 17, 363, 693, 1893, 0, 0, 3902, 1, 80, 143, 3383, 1, 3190, 0, 0, 225, 43, 17, 45, 3995, 0, 0, 412, 0, 0, 0, 2, 0, 1875, 0, 3939, 0, 3024, 18, 5, 4015, 3533, 10, 3553, 0, 0, 841, 3, 0, 2, 0, 446, 2, 81, 0, 4093, 346, 4096, 4096, 79, 0, 0, 0, 0, 0, 0, 847, 37, 676, 3329, 3067, 0, 3089, 37, 4081, 2344, 3578, 191, 4026, 0, 33, 569, 3188, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3250, 0, 522, 0, 0, 3910, 4096, 0, 17, 0, 2564, 0, 0, 0, 3491, 54, 0, 3, 938, 0, 4094, 0, 37, 363, 0, 2744, 3730, 0, 0, 3253, 0, 0, 4004, 9, 0, 68, 0, 0, 954, 120, 0, 3, 4096, 0, 4084, 720, 31, 0, 133, 0, 0, 0, 3143, 0, 1, 775, 4094, 748, 1550, 0, 1, 4081, 4060, 0, 0, 0, 624, 0, 0, 0, 1, 4096, 0, 4, 0, 1, 4058, 760, 5, 3201, 0, 2711, 3, 0, 40, 0, 0, 0, 0, 1, 7, 3992, 64, 1717, 0, 978, 1959, 0, 0, 3650, 3399, 1, 0, 2, 574, 0, 3936, 1634, 3626, 0, 4071, 3246, 4092, 3, 4096, 0, 4096, 4095, 332, 0, 0, 256] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 2809, 0, 912, 4080, 0, 3858, 3886, 703, 0, 0, 4068, 0, 0, 7, 0, 0, 9, 0, 169, 0, 4063, 3468, 4032, 0, 0, 0, 285, 10, 22, 4, 0, 0, 3391, 2458, 1244, 0, 1088, 1, 75, 4084, 0, 0, 4096, 0, 73, 3936, 0, 0, 3941, 0, 0, 48, 1079, 188, 31, 0, 2785, 7, 36, 220, 93, 49, 4064, 0, 45, 450, 53, 37, 6, 32, 1, 1449, 3, 3815, 1, 0, 2240, 3784, 3979, 489, 0, 4086, 0, 0, 4096, 0, 0, 0, 4035, 0, 928, 9, 0, 133, 5, 3, 3912, 3316, 8, 453, 0, 1137, 232, 6, 0, 82, 3971, 898, 3980, 1, 3908, 8, 4096, 0, 832, 0, 4095, 194, 0, 1411, 0, 3475] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[146, 0, 30, 212, 0, 0, 3949, 0, 0, 3572, 0, 4090, 379, 4019, 8, 0, 2036, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 2, 14, 0, 1358, 1168, 0, 0, 121, 1877, 16, 14, 0, 0, 1756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1375, 1, 0, 1218, 8, 446, 1, 90, 4095, 0, 2273, 3961, 0, 90, 3, 2, 4071, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 3178, 0, 0, 231, 4092, 2091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 190, 0, 0, 5, 0, 0, 0, 0, 0, 0, 60, 4088, 0, 1559, 2553, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 847, 0, 4079, 2325, 0, 4090, 0, 22, 4045, 0, 0, 0, 125, 1, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3075, 0, 3979, 136, 1362, 3169, 0, 0, 3484, 3961, 3995, 0, 0, 0, 0, 681, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3737, 3969, 0, 30, 529, 13, 0, 0, 2723, 39, 0, 3883, 1, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18706, 10300, 18983, 17786, 9902, 16218, 24025, 15152] avg=16384.0000 max=24025 max_rank=6 min=9902 min_rank=4 imbalance_ratio=1.466370 max_load_reduction=107047 max_load_reduction_pct=0.816704 moved_to_echo=65297 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 174, 25, 1453, 0, 4091, 4096, 2894, 0, 0, 2040, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 522, 80, 0, 0, 3502, 3907, 0, 333, 17, 0, 1869, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 1, 0, 4092, 0, 0, 230, 3675, 0, 3, 159, 0, 0, 4096, 0, 32, 4096, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 678, 129, 0, 0, 5, 0, 1, 17, 25, 4096, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 47, 55, 6, 0, 0, 33, 0, 0, 0, 0, 4096, 1488, 295, 2674, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4033, 853, 4059, 4096, 0, 4083, 38, 0, 3429, 0, 0, 1, 0, 0, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 3804, 484, 0, 2397, 4095, 0, 0, 3919, 7, 0, 0, 2, 11, 0, 469, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2902, 0, 1, 55, 43, 0, 1, 1851, 0, 0, 4095, 26, 0, 0, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19156, 14314, 21812, 11291, 10242, 21064, 19381, 13812] avg=16384.0000 max=21812 max_rank=2 min=10242 min_rank=4 imbalance_ratio=1.331299 max_load_reduction=109260 max_load_reduction_pct=0.833588 moved_to_echo=64499 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1814, 0, 8, 18, 0, 32, 0, 1902, 4059, 78, 40, 32, 1920, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 4045, 4092, 1096, 4064, 35, 3650, 0, 1771, 4, 3914, 0, 530, 3308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 3, 0, 3531, 492, 52, 1576, 7, 52, 2, 2207, 3644, 9, 3360, 0, 0, 3290, 1202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 2447, 0, 0, 4, 3669, 4040, 708, 0, 100, 0, 49, 0, 2263, 2367, 2059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3841, 0, 0, 26, 391, 0, 3, 0, 0, 4, 0, 4096, 0, 3907, 2274, 1415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 1, 0, 0, 1029, 2289, 0, 3745, 746, 2, 3955, 76, 22, 0, 0, 17, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 2, 1670, 1250, 1, 2, 0, 0, 4019, 53, 29, 0, 44, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 340, 0, 4088, 59, 451, 0, 4091, 180, 44, 0, 22, 0, 188, 44, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10165, 30610, 19453, 21762, 15967, 12185, 7328, 13602] avg=16384.0000 max=30610 max_rank=1 min=7328 min_rank=6 imbalance_ratio=1.868286 max_load_reduction=100462 max_load_reduction_pct=0.766464 moved_to_echo=49082 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4089, 2, 156, 18, 91, 0, 308, 0, 0, 4077, 0, 1994, 2024, 0, 0, 0, 2033, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 819, 16, 344, 12, 1268, 3998, 4044, 1875, 336, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 0, 0, 139, 0, 2, 0, 5, 0, 168, 0, 254, 0, 3908, 6, 1391, 4081, 1392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 17, 363, 693, 1893, 0, 0, 3902, 1, 80, 143, 3383, 1, 3190, 0, 0, 1690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 43, 17, 45, 3995, 0, 0, 412, 0, 0, 0, 2, 0, 1875, 0, 3939, 2634, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3024, 18, 5, 4015, 3533, 10, 3553, 0, 0, 841, 3, 0, 2, 0, 446, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 81, 0, 4093, 346, 4096, 4096, 79, 0, 0, 0, 0, 0, 0, 847, 37, 596, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 3329, 0, 0, 3089, 37, 0, 2344, 2384, 191, 0, 0, 33, 569, 3188, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15006, 20674, 15188, 15424, 14350, 16048, 14455, 19927] avg=16384.0000 max=20674 max_rank=1 min=14350 min_rank=4 imbalance_ratio=1.261841 max_load_reduction=110398 max_load_reduction_pct=0.842270 moved_to_echo=64780 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3757, 5, 0, 1179, 0, 0, 0, 0, 0, 0, 2, 3052, 2358, 0, 1, 0, 1988, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 66, 0, 16, 2401, 0, 129, 3095, 2527, 0, 1834, 0, 16, 9, 0, 3425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 241, 0, 0, 572, 0, 0, 0, 0, 0, 80, 3, 0, 4006, 0, 0, 4092, 1399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 26, 0, 689, 0, 26, 4017, 74, 0, 0, 1196, 6, 0, 0, 4065, 555, 2199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 697, 4096, 235, 0, 3834, 0, 135, 2386, 0, 291, 7, 2389, 5, 4063, 0, 959, 2646, 1511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 2, 3477, 19, 2622, 0, 0, 3593, 0, 17, 4092, 0, 0, 0, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3053, 0, 0, 0, 4087, 3731, 0, 0, 9, 2480, 0, 3662, 3, 0, 2, 598, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 237, 0, 970, 2763, 0, 0, 4088, 2390, 23, 0, 4042, 3, 3737, 0, 3267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12621, 17610, 10396, 13136, 23254, 14531, 17812, 21712] avg=16384.0000 max=23254 max_rank=4 min=10396 min_rank=2 imbalance_ratio=1.419312 max_load_reduction=107818 max_load_reduction_pct=0.822586 moved_to_echo=77309 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 39, 0, 2632, 0, 309, 4083, 3517, 4096, 0, 4095, 3358, 0, 0, 180, 185, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 381, 4092, 6, 55, 1336, 0, 173, 0, 1475, 1528, 2668, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2430, 0, 0, 716, 2, 0, 0, 0, 705, 3, 42, 0, 0, 0, 8, 0, 4096, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3480, 0, 0, 0, 43, 0, 341, 998, 0, 2, 32, 940, 3, 0, 3470, 1910, 2252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4075, 27, 254, 3810, 0, 0, 1, 4095, 8, 4095, 3754, 0, 2672, 1548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3920, 0, 3828, 1, 244, 0, 3801, 0, 0, 0, 0, 458, 4068, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4091, 0, 4021, 4096, 37, 1, 0, 2445, 1738, 3459, 0, 5, 0, 389, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 1834, 633, 845, 0, 2318, 1556, 0, 0, 3739, 131, 1, 105, 972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22782, 11878, 9416, 13471, 24349, 16712, 20319, 12145] avg=16384.0000 max=24349 max_rank=4 min=9416 min_rank=2 imbalance_ratio=1.486145 max_load_reduction=106723 max_load_reduction_pct=0.814232 moved_to_echo=73525 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3250, 0, 522, 0, 0, 3910, 4096, 0, 17, 0, 2035, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2564, 0, 0, 0, 3491, 54, 0, 3, 938, 0, 1876, 0, 37, 363, 0, 2744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3730, 0, 0, 3253, 0, 0, 4004, 9, 0, 68, 0, 0, 954, 120, 0, 3, 4092, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4084, 720, 31, 0, 133, 0, 0, 0, 3143, 0, 1, 775, 4094, 748, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1550, 0, 1, 4081, 4060, 0, 0, 0, 624, 0, 0, 0, 1, 4096, 0, 4, 2679, 1373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4058, 760, 5, 3201, 0, 2711, 3, 0, 40, 0, 0, 0, 0, 1, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3992, 64, 1717, 0, 978, 1959, 0, 0, 3650, 3399, 1, 0, 2, 574, 0, 683, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 1634, 0, 0, 4071, 3246, 0, 3, 2728, 0, 0, 4095, 332, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14085, 12070, 17650, 19823, 18469, 11465, 17209, 20301] avg=16384.0000 max=20301 max_rank=7 min=11465 min_rank=5 imbalance_ratio=1.239075 max_load_reduction=110771 max_load_reduction_pct=0.845116 moved_to_echo=67444 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 2809, 0, 912, 4080, 0, 3858, 3886, 703, 0, 0, 2019, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 7, 0, 0, 9, 0, 169, 0, 1863, 3468, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 10, 22, 4, 0, 0, 3391, 2458, 1244, 0, 1088, 1, 75, 4084, 0, 0, 4096, 1416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 73, 3936, 0, 0, 3941, 0, 0, 48, 1079, 188, 31, 0, 2785, 7, 2195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 220, 93, 49, 4064, 0, 45, 450, 53, 37, 6, 32, 1, 1449, 3, 3815, 2679, 1507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2240, 3784, 3979, 489, 0, 4086, 0, 0, 4096, 0, 0, 0, 4035, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 928, 9, 0, 133, 5, 3, 3912, 3316, 8, 453, 0, 1137, 232, 6, 0, 82, 138, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 898, 0, 1, 3908, 8, 0, 0, 555, 0, 0, 194, 0, 1411, 0, 3475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18551, 13616, 18174, 18379, 14539, 22849, 10543, 14421] avg=16384.0000 max=22849 max_rank=5 min=10543 min_rank=6 imbalance_ratio=1.394592 max_load_reduction=108223 max_load_reduction_pct=0.825676 moved_to_echo=62352 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[20736, 13440, 0, 7040, 15232, 4736, 0, 12928, 14464, 384, 0, 20096, 640, 6144, 3456, 12928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[20224, 128, 7040, 6144, 2048, 4224, 16896, 9216, 768, 256, 5760, 7808, 128, 4096, 17024, 13824, 16768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[8064, 128, 256, 1536, 4608, 0, 14848, 4352, 4992, 17408, 128, 27008, 24320, 7808, 128, 256, 14336, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[19456, 4224, 128, 5120, 13952, 10880, 5504, 4864, 12672, 4224, 13184, 3840, 17536, 1920, 3200, 11264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1024, 14336, 4096, 14592, 512, 17024, 26752, 4736, 128, 11520, 12416, 6912, 11392, 128, 1536, 128, 3712, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2560, 4480, 4480, 4224, 20096, 128, 896, 7168, 768, 384, 128, 6528, 128, 27904, 5248, 14592, 20864, 11520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[15744, 256, 128, 13056, 1152, 512, 9216, 6272, 6144, 256, 5888, 7936, 1152, 19712, 128, 1536, 32000, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[384, 3072, 14720, 9344, 19840, 19840, 128, 26112, 896, 3968, 20608, 128, 128, 128, 4736, 4608, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[6, 0, 0, 2, 34, 1465, 1, 2980, 0, 0, 0, 0, 50, 2, 0, 889, 0, 0, 44, 11, 4028, 337, 3507, 991, 1, 0, 0, 4091, 4069, 159, 2, 1, 997, 2045, 0, 93, 57, 1, 86, 1, 48, 2929, 0, 4092, 212, 125, 0, 0, 26, 4082, 5, 0, 0, 618, 15, 2, 4072, 1332, 3792, 5, 4092, 0, 15, 0, 3963, 0, 3376, 129, 0, 0, 3876, 0, 2, 4090, 0, 2, 3835, 1334, 2232, 19, 98, 0, 0, 0, 3862, 3341, 818, 3, 1, 0, 973, 4085, 0, 2112, 3068, 0, 1024, 0, 0, 0, 0, 6, 0, 19, 0, 0, 519, 4096, 0, 656, 0, 0, 3817, 1, 0, 3602, 4096, 0, 4037, 0, 4077, 0, 4096, 207, 4092, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4092, 0, 0, 3282, 0, 3, 3, 221, 0, 5, 67, 0, 3728, 1, 0, 2, 0, 278, 0, 2713, 2590, 2, 0, 0, 79, 2768, 1, 1178, 3892, 0, 0, 0, 3578, 0, 1, 77, 0, 2060, 3339, 0, 0, 4091, 0, 3855, 0, 1106, 0, 0, 231, 823, 64, 147, 4066, 0, 0, 4094, 2326, 0, 0, 115, 185, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3945, 4, 0, 4050, 0, 0, 1476, 50, 4096, 0, 0, 1522, 4096, 0, 272, 3104, 0, 7, 12, 1718, 2290, 3168, 0, 4009, 2005, 14, 3578, 0, 55, 6, 0, 0, 2, 0, 0, 1265, 1632, 89, 423, 0, 0, 1, 5, 3824, 0, 3984, 4096, 1, 4096, 0, 3065, 0, 4096, 4096, 4096, 181, 1579, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4094, 94, 0, 1, 0, 3, 0, 4087, 0, 56, 0, 677, 581, 0, 0, 3835, 109, 556, 0, 0, 3978, 0, 0, 2959, 0, 0, 52, 0, 4096, 0, 59, 0, 3588, 358, 0, 4086, 42, 0, 0, 0, 841, 0, 0, 2334, 0, 2060, 75, 0, 2687, 3446, 7, 62, 0, 0, 36, 1746, 3581, 0, 0, 0, 3284, 71, 0, 0, 1360, 13, 10, 3889, 0, 69, 218, 3, 0, 4011, 0, 0, 3397, 2747, 3387, 1739, 0, 643, 69, 0, 0, 1850, 2227, 220, 0, 0, 9, 4089, 0, 4091, 4060, 0, 0, 0, 0, 311, 13, 0, 3948, 0, 13, 580, 2269, 2114, 26, 2, 0, 3845, 4038, 0, 0, 4091, 4096, 0, 3871, 0, 4055, 0, 4082, 0, 0, 34, 2042, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[1118, 0, 0, 203, 0, 0, 3424, 3587, 1, 4052, 0, 1, 32, 6, 162, 46, 0, 0, 0, 142, 4081, 2647, 0, 0, 268, 4088, 0, 0, 3533, 38, 12, 0, 0, 3918, 3142, 10, 0, 4094, 0, 82, 1579, 3906, 382, 3, 0, 2603, 4082, 0, 4096, 0, 0, 463, 0, 11, 10, 247, 0, 3701, 0, 0, 3015, 0, 0, 0, 2574, 2, 0, 33, 0, 219, 0, 0, 162, 4096, 96, 28, 3755, 0, 3808, 1, 0, 501, 0, 0, 0, 1626, 4096, 0, 455, 1, 0, 3636, 12, 456, 0, 2, 35, 173, 1716, 529, 0, 0, 725, 436, 0, 171, 3993, 1351, 3834, 0, 0, 519, 4004, 3995, 0, 3323, 4096, 589, 3167, 0, 4078, 0, 2346, 0, 9, 0, 3627, 12] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[424, 0, 732, 3991, 0, 2033, 0, 0, 0, 250, 0, 0, 0, 1, 0, 705, 194, 0, 0, 4096, 3635, 4093, 0, 3882, 425, 0, 8, 4096, 4095, 0, 0, 0, 0, 0, 4, 0, 3171, 3746, 10, 0, 0, 3796, 0, 3019, 0, 317, 0, 0, 1, 4, 0, 0, 0, 0, 0, 3683, 3910, 786, 70, 45, 3989, 8, 318, 0, 4096, 0, 929, 0, 0, 0, 20, 25, 4087, 3274, 0, 0, 126, 13, 4089, 0, 0, 0, 190, 0, 340, 0, 0, 795, 0, 150, 29, 3901, 38, 3468, 7, 1, 0, 0, 4025, 1666, 0, 0, 2018, 154, 0, 53, 42, 0, 0, 19, 0, 2, 102, 3390, 0, 3763, 4096, 1, 1580, 0, 2706, 3827, 4096, 4096, 2130, 4095, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[1793, 0, 1122, 26, 0, 0, 11, 5, 0, 4087, 0, 1319, 140, 0, 0, 3316, 47, 4, 300, 0, 3758, 4, 4068, 12, 2584, 0, 0, 0, 3892, 15, 2941, 34, 0, 0, 0, 19, 1487, 32, 3318, 0, 0, 4095, 587, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 296, 0, 7, 124, 2, 0, 4075, 0, 867, 2839, 4089, 0, 582, 2136, 0, 0, 125, 59, 32, 3491, 2, 0, 2, 3062, 0, 0, 1, 1271, 11, 0, 2614, 3319, 3516, 2824, 811, 104, 95, 4054, 0, 1202, 236, 11, 0, 868, 0, 4033, 0, 0, 2705, 1, 0, 0, 4095, 1, 51, 0, 0, 2, 4094, 4096, 0, 3783, 4096, 1034, 895, 66, 3035, 536, 805, 3851, 3749, 4026, 0, 286] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 719, 0, 0, 14, 0, 1, 0, 0, 162, 0, 2800, 0, 0, 4093, 2, 1347, 1, 0, 4096, 4093, 2535, 4020, 0, 2589, 0, 774, 4096, 0, 0, 0, 2925, 603, 0, 2804, 0, 4096, 1, 0, 66, 2, 0, 8, 0, 3989, 3388, 44, 0, 4096, 267, 0, 0, 0, 0, 1, 4094, 4096, 0, 476, 4002, 0, 0, 0, 3407, 1, 0, 3770, 0, 0, 0, 0, 31, 4096, 0, 46, 2680, 1, 4092, 573, 4, 2503, 0, 0, 0, 173, 4, 187, 0, 0, 0, 3659, 0, 2648, 0, 0, 0, 0, 247, 3833, 0, 0, 4056, 0, 0, 12, 611, 0, 245, 0, 0, 4019, 1992, 525, 0, 4091, 2700, 3, 0, 73, 3924, 0, 16, 4037, 21, 0, 2396, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[3987, 30, 490, 3398, 0, 0, 0, 0, 0, 3720, 0, 4058, 7, 3, 0, 200, 0, 7, 4095, 7, 4045, 2348, 275, 392, 159, 736, 2153, 0, 3850, 5, 7, 0, 1, 254, 6, 4092, 515, 2385, 236, 0, 0, 4, 0, 0, 0, 3877, 3633, 0, 4057, 0, 3, 0, 103, 0, 0, 3, 2435, 0, 0, 46, 4066, 0, 0, 0, 4096, 0, 0, 4076, 0, 0, 2648, 0, 79, 4093, 14, 515, 3, 0, 3288, 0, 4, 0, 0, 886, 135, 1654, 35, 4081, 1, 55, 0, 4096, 0, 4052, 3882, 0, 0, 110, 0, 0, 0, 0, 4094, 31, 1, 2, 561, 1870, 0, 0, 3, 3234, 303, 3745, 280, 3068, 0, 0, 234, 0, 4096, 1, 3767, 3575, 4093, 616, 5, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 2, 34, 1465, 1, 2980, 0, 0, 0, 0, 50, 2, 0, 889, 4096, 1447, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 11, 4028, 337, 3507, 991, 1, 0, 0, 4091, 2766, 159, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 997, 2045, 0, 93, 57, 1, 86, 1, 48, 2929, 0, 4092, 212, 125, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4082, 5, 0, 0, 618, 15, 2, 4072, 1332, 3792, 5, 4092, 0, 15, 0, 2941, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 3376, 129, 0, 0, 3876, 0, 2, 2941, 0, 2, 3835, 1334, 2232, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 3862, 3341, 818, 3, 1, 0, 973, 4085, 0, 2112, 3068, 0, 563, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 0, 0, 0, 0, 6, 0, 19, 0, 0, 519, 4096, 0, 656, 0, 0, 3602, 2630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3817, 1, 0, 0, 0, 0, 4037, 0, 0, 0, 652, 207, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11475, 15938, 11272, 22158, 21709, 19066, 12552, 16902] avg=16384.0000 max=22158 max_rank=3 min=11272 min_rank=2 imbalance_ratio=1.352417 max_load_reduction=108914 max_load_reduction_pct=0.830948 moved_to_echo=70229 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 3282, 0, 3, 3, 221, 0, 5, 67, 0, 3728, 1, 0, 2, 4096, 1088, 507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 0, 2713, 2590, 2, 0, 0, 79, 2768, 1, 1178, 2637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3578, 0, 1, 77, 0, 2060, 3339, 0, 0, 4091, 0, 3855, 0, 1106, 0, 0, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 823, 64, 147, 4066, 0, 0, 4094, 2326, 0, 0, 115, 185, 0, 0, 0, 2945, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3945, 4, 0, 2905, 0, 0, 1476, 50, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1522, 4096, 0, 272, 3104, 0, 7, 12, 1718, 2290, 3168, 0, 4009, 2005, 14, 561, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3578, 0, 55, 6, 0, 0, 2, 0, 0, 1265, 1632, 89, 423, 0, 0, 1, 3984, 1977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3824, 0, 0, 0, 1, 4096, 0, 0, 0, 644, 4096, 4096, 181, 1579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17095, 12246, 18691, 16110, 12477, 22919, 13012, 18522] avg=16384.0000 max=22919 max_rank=5 min=12246 min_rank=1 imbalance_ratio=1.398865 max_load_reduction=108153 max_load_reduction_pct=0.825142 moved_to_echo=66930 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4094, 94, 0, 1, 0, 3, 0, 4087, 0, 56, 0, 677, 581, 0, 0, 3835, 4096, 1439, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 556, 0, 0, 3978, 0, 0, 2959, 0, 0, 52, 0, 2784, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3588, 358, 0, 4086, 42, 0, 0, 0, 841, 0, 0, 2334, 0, 2060, 75, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2687, 3446, 7, 62, 0, 0, 36, 1746, 3581, 0, 0, 0, 3284, 71, 0, 0, 2931, 1169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1360, 13, 10, 3889, 0, 69, 218, 3, 0, 2884, 0, 0, 3397, 2747, 3387, 1739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 643, 69, 0, 0, 1850, 2227, 220, 0, 0, 9, 4089, 0, 4091, 4060, 0, 552, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 13, 0, 3948, 0, 13, 580, 2269, 2114, 26, 2, 0, 3845, 4091, 2616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 3871, 0, 0, 0, 650, 0, 0, 34, 2042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19464, 10497, 13959, 19020, 19716, 17953, 19828, 10635] avg=16384.0000 max=19828 max_rank=6 min=10497 min_rank=1 imbalance_ratio=1.210205 max_load_reduction=111244 max_load_reduction_pct=0.848724 moved_to_echo=68132 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1118, 0, 0, 203, 0, 0, 3424, 3587, 1, 4052, 0, 1, 32, 6, 162, 46, 4096, 1448, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 4081, 2647, 0, 0, 268, 4088, 0, 0, 2402, 38, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 3142, 10, 0, 4094, 0, 82, 1579, 3906, 382, 3, 0, 2603, 4082, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 463, 0, 11, 10, 247, 0, 3701, 0, 0, 3015, 0, 0, 0, 1684, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2574, 2, 0, 33, 0, 219, 0, 0, 162, 2945, 96, 28, 3755, 0, 3808, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 0, 0, 0, 1626, 4096, 0, 455, 1, 0, 3636, 12, 456, 0, 2, 564, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 173, 1716, 529, 0, 0, 725, 436, 0, 171, 3993, 1351, 3834, 0, 0, 519, 3323, 2630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 3995, 0, 0, 0, 589, 3167, 0, 0, 0, 374, 0, 9, 0, 3627, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18464, 13678, 24388, 14235, 13623, 11472, 19435, 15777] avg=16384.0000 max=24388 max_rank=2 min=11472 min_rank=5 imbalance_ratio=1.488525 max_load_reduction=106684 max_load_reduction_pct=0.813934 moved_to_echo=60307 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1793, 0, 1122, 26, 0, 0, 11, 5, 0, 4087, 0, 1319, 140, 0, 0, 3316, 4096, 1078, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4, 300, 0, 3758, 4, 4068, 12, 2584, 0, 0, 0, 2646, 15, 2941, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1487, 32, 3318, 0, 0, 4095, 587, 0, 0, 0, 4089, 0, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 296, 0, 7, 124, 2, 0, 4075, 0, 867, 2839, 578, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 582, 2136, 0, 0, 125, 59, 32, 2511, 2, 0, 2, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1271, 11, 0, 2614, 3319, 3516, 2824, 811, 104, 95, 4054, 0, 1202, 236, 11, 480, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 868, 0, 4033, 0, 0, 2705, 1, 0, 0, 4095, 1, 51, 0, 0, 2, 3783, 1957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 1034, 895, 66, 0, 536, 129, 3851, 3749, 4026, 0, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17091, 16413, 14127, 9898, 12600, 20685, 17496, 22762] avg=16384.0000 max=22762 max_rank=7 min=9898 min_rank=3 imbalance_ratio=1.389282 max_load_reduction=108310 max_load_reduction_pct=0.826340 moved_to_echo=73543 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3987, 30, 490, 3398, 0, 0, 0, 0, 0, 3720, 0, 4058, 7, 3, 0, 200, 0, 1453, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4095, 7, 4045, 2348, 275, 392, 159, 736, 2153, 0, 2617, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 254, 6, 4092, 515, 2385, 236, 0, 0, 4, 0, 0, 0, 3877, 3633, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 3, 0, 103, 0, 0, 3, 2435, 0, 0, 46, 4066, 0, 0, 0, 2705, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 0, 0, 2648, 0, 79, 2943, 14, 515, 3, 0, 3288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 886, 135, 1654, 35, 4081, 1, 55, 0, 4096, 0, 4052, 3882, 0, 563, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 4094, 31, 1, 2, 561, 1870, 0, 0, 3, 3234, 3068, 2643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 3745, 280, 0, 0, 0, 234, 0, 0, 1, 600, 3575, 4093, 616, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17808, 16846, 15590, 14516, 17662, 19579, 15617, 13454] avg=16384.0000 max=19579 max_rank=5 min=13454 min_rank=7 imbalance_ratio=1.195007 max_load_reduction=111493 max_load_reduction_pct=0.850624 moved_to_echo=66312 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[424, 0, 732, 3991, 0, 2033, 0, 0, 0, 250, 0, 0, 0, 1, 0, 705, 4096, 961, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 0, 4096, 3635, 4093, 0, 3882, 425, 0, 8, 4096, 2784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3171, 3746, 10, 0, 0, 3796, 0, 3019, 0, 317, 0, 0, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 3683, 3910, 786, 70, 45, 3989, 8, 318, 0, 2941, 1168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 929, 0, 0, 0, 20, 25, 4087, 2354, 0, 0, 126, 13, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 340, 0, 0, 795, 0, 150, 29, 3901, 38, 3468, 7, 1, 451, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 1666, 0, 0, 2018, 154, 0, 53, 42, 0, 0, 19, 0, 2, 3763, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 3390, 0, 0, 0, 1, 1580, 0, 0, 3827, 652, 4096, 2130, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13696, 23213, 14532, 16923, 15739, 9513, 13487, 23969] avg=16384.0000 max=23969 max_rank=7 min=9513 min_rank=5 imbalance_ratio=1.462952 max_load_reduction=107103 max_load_reduction_pct=0.817131 moved_to_echo=62708 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 719, 0, 0, 14, 0, 1, 0, 0, 162, 0, 2800, 0, 0, 4093, 2700, 1393, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1347, 1, 0, 4096, 4093, 2535, 4020, 0, 2589, 0, 774, 2784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2925, 603, 0, 2804, 0, 4096, 1, 0, 66, 2, 0, 8, 0, 3989, 3388, 44, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 267, 0, 0, 0, 0, 1, 4094, 4096, 0, 476, 4002, 0, 0, 0, 11, 1169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3407, 1, 0, 3770, 0, 0, 0, 0, 31, 2945, 0, 46, 2680, 1, 4092, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2503, 0, 0, 0, 173, 4, 187, 0, 0, 0, 3659, 0, 2648, 0, 0, 564, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 3833, 0, 0, 4056, 0, 0, 12, 611, 0, 245, 0, 0, 4019, 4091, 2531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1992, 525, 0, 0, 0, 3, 0, 73, 0, 0, 4, 4037, 21, 0, 2396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15979, 22241, 18513, 18212, 17546, 9885, 19645, 9051] avg=16384.0000 max=22241 max_rank=1 min=9051 min_rank=7 imbalance_ratio=1.357483 max_load_reduction=108831 max_load_reduction_pct=0.830315 moved_to_echo=56127 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[19712, 128, 3072, 11008, 128, 3584, 3456, 11008, 128, 12288, 256, 6144, 7424, 128, 256, 13184, 27392, 10368, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 6528, 4480, 896, 7296, 15104, 10752, 8192, 1280, 2048, 3456, 30720, 128, 22144, 13312, 128, 4352, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[4736, 1152, 6144, 10496, 128, 128, 17664, 768, 128, 2176, 13824, 9600, 4608, 768, 128, 11648, 29824, 18816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[18432, 19584, 384, 0, 0, 1664, 17920, 256, 0, 4480, 3712, 19968, 18304, 8960, 17920, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[23680, 128, 4992, 14080, 0, 384, 10880, 128, 4480, 22528, 128, 640, 15360, 7296, 25088, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[11136, 7296, 3200, 11264, 5376, 16512, 7040, 128, 2560, 18944, 1024, 13312, 256, 14080, 15360, 128, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[11136, 12544, 384, 768, 4224, 640, 384, 9856, 20480, 10112, 3968, 768, 26752, 128, 1280, 2944, 16768, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 2304, 4480, 7040, 30336, 13568, 10496, 12288, 3584, 10240, 2304, 10240, 21504, 256, 3072, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 3696, 26, 5, 2295, 629, 0, 0, 4082, 0, 10, 3299, 296, 0, 12, 0, 0, 86, 0, 0, 0, 106, 679, 0, 250, 4086, 56, 4090, 0, 0, 3886, 0, 56, 1942, 0, 5, 39, 228, 16, 2790, 680, 45, 6, 3991, 47, 4096, 3, 0, 0, 1424, 3834, 0, 4065, 889, 2589, 1, 1, 4071, 3, 0, 0, 4096, 3412, 0, 0, 0, 0, 31, 3309, 578, 0, 0, 4078, 1085, 3957, 4096, 1, 4096, 0, 134, 4096, 4096, 0, 1786, 2076, 4096, 0, 0, 898, 0, 0, 0, 60, 0, 0, 30, 0, 0, 0, 0, 264, 0, 0, 4096, 89, 0, 0, 0, 0, 0, 4096, 0, 161, 0, 3, 0, 0, 4096, 3977, 3849, 4072, 0, 1893, 53, 0, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 2060, 120, 0, 4083, 0, 63, 0, 4068, 2206, 2244, 4096, 0, 4095, 0, 0, 0, 1, 0, 273, 0, 0, 0, 64, 48, 0, 309, 282, 129, 0, 4059, 48, 43, 4089, 0, 3, 3395, 0, 0, 628, 9, 3475, 0, 34, 2, 1831, 4, 0, 35, 0, 4088, 0, 2658, 5, 0, 1, 0, 4077, 3755, 36, 105, 2587, 4069, 0, 0, 0, 170, 0, 4096, 0, 9, 0, 2519, 0, 4096, 3204, 4, 34, 4051, 0, 4040, 1, 0, 0, 862, 4096, 0, 24, 2437, 17, 4095, 0, 4055, 1, 0, 21, 0, 0, 0, 4096, 2532, 3701, 0, 3831, 1, 10, 24, 0, 3, 0, 0, 1753, 1777, 3, 0, 65, 0, 4096, 1438, 0, 3851, 0, 2356, 0, 343, 0, 183, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 4021, 0, 0, 10, 0, 35, 4096, 0, 0, 4080, 203, 0, 0, 0, 1, 0, 0, 0, 0, 8, 4096, 0, 12, 190, 0, 3653, 41, 0, 0, 0, 13, 55, 32, 8, 3, 476, 4033, 4096, 8, 1, 0, 4096, 0, 2752, 0, 0, 0, 1, 4048, 0, 0, 1, 9, 3929, 0, 4092, 971, 0, 0, 3934, 168, 11, 4041, 0, 2691, 1240, 0, 0, 827, 3730, 3221, 16, 1698, 0, 9, 4067, 81, 0, 4080, 4094, 0, 4029, 0, 635, 0, 998, 4096, 0, 1, 0, 15, 0, 2, 0, 0, 0, 0, 4096, 0, 4025, 4084, 4091, 0, 0, 4092, 0, 1, 0, 0, 0, 3, 0, 0, 5, 2, 0, 2959, 2, 4096, 685, 3067, 4093, 127, 0, 2889, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 3180, 3229, 0, 0, 671, 218, 0, 2, 14, 3912, 4094, 2, 0, 0, 0, 0, 102, 0, 543, 386, 153, 62, 0, 653, 4047, 0, 0, 2, 0, 3377, 0, 0, 2, 0, 388, 4050, 0, 0, 4094, 4096, 203, 0, 2143, 2389, 4072, 0, 636, 0, 59, 4046, 0, 0, 3185, 27, 2389, 0, 4084, 0, 11, 0, 1298, 4094, 6, 2722, 3216, 3944, 4061, 97, 189, 4085, 0, 3945, 385, 669, 0, 107, 0, 4066, 0, 0, 5, 46, 80, 7, 4006, 0, 4081, 87, 0, 36, 11, 1205, 65, 0, 0, 0, 0, 13, 153, 35, 3967, 1, 4096, 0, 0, 0, 1839, 0, 0, 1, 11, 0, 13, 0, 3992, 0, 2, 0, 0, 3824, 1030, 4093, 3251, 0, 0, 0, 1717] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4086, 0, 2178, 154, 0, 3, 0, 3249, 1, 555, 3841, 0, 1729, 0, 0, 0, 0, 0, 323, 0, 39, 4087, 7, 500, 3790, 1237, 677, 908, 0, 3265, 0, 251, 4083, 0, 12, 0, 23, 0, 3, 4028, 0, 0, 162, 3514, 0, 0, 0, 185, 0, 1, 0, 221, 0, 84, 4061, 21, 4073, 1, 0, 0, 4096, 4094, 0, 972, 1, 1889, 105, 41, 3891, 0, 1570, 4096, 0, 3978, 816, 0, 2403, 18, 0, 0, 4095, 0, 3294, 3754, 0, 769, 0, 0, 4, 805, 1996, 4096, 3, 262, 0, 9, 1, 102, 0, 0, 4090, 4067, 3820, 1, 5, 3804, 1264, 2, 0, 3286, 13, 0, 14, 0, 64, 0, 4078, 4093, 0, 3435, 0, 0, 0, 515, 6, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[80, 0, 3853, 0, 0, 0, 1, 18, 4096, 2728, 3, 2, 287, 0, 0, 0, 0, 0, 0, 0, 1851, 0, 4023, 0, 20, 4, 0, 3007, 1, 44, 791, 0, 0, 19, 0, 0, 3009, 0, 663, 670, 3656, 0, 10, 4090, 0, 73, 4094, 0, 0, 1, 3581, 0, 4081, 3940, 0, 0, 27, 4096, 74, 0, 0, 3164, 4049, 0, 2870, 0, 4063, 0, 4036, 2531, 0, 11, 2855, 0, 3, 0, 0, 11, 41, 0, 875, 0, 0, 973, 0, 4096, 0, 2254, 0, 225, 0, 0, 0, 551, 13, 179, 0, 8, 4072, 2900, 0, 3578, 3486, 3896, 0, 0, 3, 3768, 0, 0, 0, 0, 318, 0, 0, 0, 447, 4088, 3794, 407, 4095, 1419, 0, 4092, 3953, 24, 4, 1027] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1, 3903, 223, 0, 4007, 50, 1, 12, 3764, 63, 3799, 4077, 0, 3568, 1, 0, 11, 0, 0, 0, 0, 0, 40, 0, 65, 81, 377, 3904, 0, 0, 3219, 499, 13, 4, 9, 4096, 0, 4, 9, 1575, 4096, 3506, 0, 26, 3498, 0, 355, 0, 0, 166, 4044, 0, 58, 4096, 4096, 1138, 0, 3951, 19, 0, 0, 1, 94, 4, 986, 0, 0, 0, 0, 4085, 0, 0, 0, 7, 4087, 0, 14, 4094, 73, 0, 4009, 0, 0, 0, 0, 1, 107, 1767, 4074, 0, 112, 0, 4087, 0, 1, 0, 668, 0, 306, 4090, 0, 3858, 82, 4041, 102, 0, 3781, 0, 0, 0, 0, 0, 1, 0, 2, 22, 0, 4084, 0, 151, 4034, 0, 3646, 4094, 0, 4072, 10, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[543, 0, 0, 1388, 4096, 1, 0, 47, 3868, 250, 27, 4092, 0, 38, 0, 0, 4, 3250, 0, 0, 62, 0, 0, 65, 1, 17, 0, 203, 10, 0, 4089, 0, 0, 4084, 41, 1873, 0, 20, 0, 4030, 0, 3882, 13, 4061, 0, 2970, 1469, 0, 3838, 3, 3, 0, 1944, 0, 252, 4060, 171, 3996, 502, 0, 885, 2681, 4096, 0, 928, 3052, 3023, 0, 15, 4095, 0, 1, 3388, 0, 44, 0, 3, 7, 4094, 0, 3103, 4089, 0, 4064, 237, 332, 3, 0, 0, 0, 111, 990, 2867, 0, 300, 0, 0, 1883, 0, 4096, 0, 0, 4096, 4046, 60, 22, 3925, 0, 0, 0, 2107, 40, 0, 0, 0, 1, 0, 16, 41, 0, 3780, 3, 1121, 3979, 162, 18, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3696, 26, 5, 2295, 629, 0, 0, 4082, 0, 10, 3299, 296, 0, 12, 0, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 106, 679, 0, 250, 4086, 56, 4090, 0, 0, 3886, 0, 3895, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 1942, 0, 5, 39, 228, 16, 2790, 680, 45, 6, 3557, 47, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1424, 3834, 0, 4065, 889, 2589, 1, 1, 176, 3, 0, 0, 4096, 3412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3309, 578, 0, 0, 0, 1085, 3600, 4096, 1, 4096, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1786, 2076, 4096, 0, 0, 898, 0, 0, 0, 60, 0, 0, 30, 424, 276, 434, 94, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 3672, 89, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 3, 0, 0, 4096, 3977, 3849, 3796, 0, 1893, 53, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14613, 21158, 13510, 20490, 16930, 18420, 8121, 17830] avg=16384.0000 max=21158 max_rank=1 min=8121 min_rank=6 imbalance_ratio=1.291382 max_load_reduction=109914 max_load_reduction_pct=0.838577 moved_to_echo=61301 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2060, 120, 0, 4083, 0, 63, 0, 4068, 2206, 2244, 4096, 0, 4095, 0, 0, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 273, 0, 0, 0, 64, 48, 0, 309, 282, 129, 0, 4059, 48, 3901, 2485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 4089, 0, 3, 3395, 0, 0, 628, 9, 3475, 0, 31, 2, 1831, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4088, 0, 2658, 5, 0, 1, 0, 176, 3755, 36, 105, 2587, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 4096, 0, 9, 0, 0, 0, 3726, 3204, 4, 34, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 1, 0, 0, 862, 4096, 0, 24, 2437, 17, 4095, 0, 4055, 1, 0, 21, 396, 261, 3, 97, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2532, 3701, 0, 3435, 1, 10, 24, 0, 3, 0, 0, 1753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1777, 3, 0, 65, 0, 4096, 1438, 0, 3590, 0, 2356, 0, 343, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23308, 11599, 13510, 17515, 15294, 20440, 15555, 13851] avg=16384.0000 max=23308 max_rank=0 min=11599 min_rank=1 imbalance_ratio=1.422607 max_load_reduction=107764 max_load_reduction_pct=0.822174 moved_to_echo=65140 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4021, 0, 0, 10, 0, 35, 4096, 0, 0, 4080, 203, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8, 4096, 0, 12, 190, 0, 3653, 41, 0, 0, 0, 3918, 3179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 55, 32, 8, 3, 476, 4033, 4096, 8, 1, 0, 3647, 0, 2752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4048, 0, 0, 1, 9, 3929, 0, 174, 971, 0, 0, 3934, 168, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 2691, 1240, 0, 0, 827, 3730, 0, 16, 1537, 0, 9, 4067, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4094, 0, 4029, 0, 635, 0, 998, 4096, 0, 1, 0, 15, 0, 2, 0, 428, 281, 449, 44, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4025, 4084, 3663, 0, 0, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5, 2, 0, 2959, 2, 3815, 685, 3067, 4093, 127, 0, 2889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12563, 15098, 15124, 13246, 18239, 19194, 19961, 17647] avg=16384.0000 max=19961 max_rank=6 min=12563 min_rank=0 imbalance_ratio=1.218323 max_load_reduction=111111 max_load_reduction_pct=0.847710 moved_to_echo=75041 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4086, 0, 2178, 154, 0, 3, 0, 3249, 1, 555, 3841, 0, 1729, 0, 0, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 39, 4087, 7, 500, 3790, 1237, 677, 908, 0, 3265, 0, 3897, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 4083, 0, 12, 0, 23, 0, 3, 4028, 0, 0, 145, 3514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 1, 0, 221, 0, 84, 4061, 21, 176, 1, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 972, 1, 1889, 105, 41, 3891, 0, 1570, 0, 0, 3619, 816, 0, 2403, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3294, 3754, 0, 769, 0, 0, 4, 805, 1996, 4096, 3, 262, 0, 395, 232, 17, 94, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 102, 0, 0, 4090, 4067, 3425, 1, 5, 3804, 1264, 2, 0, 3286, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 64, 0, 4078, 4093, 0, 3203, 0, 0, 0, 515, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16061, 22772, 12059, 12940, 15325, 19870, 20069, 11976] avg=16384.0000 max=22772 max_rank=1 min=11976 min_rank=7 imbalance_ratio=1.389893 max_load_reduction=108300 max_load_reduction_pct=0.826263 moved_to_echo=67240 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3180, 3229, 0, 0, 671, 218, 0, 2, 14, 3912, 4094, 2, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 543, 386, 153, 62, 0, 653, 4047, 0, 0, 2, 0, 3377, 0, 3907, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 388, 4050, 0, 0, 4094, 4096, 203, 0, 1910, 2389, 4072, 0, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 4046, 0, 0, 3185, 27, 2389, 0, 177, 0, 11, 0, 1298, 4094, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2722, 3216, 3944, 4061, 97, 189, 4085, 0, 0, 385, 610, 0, 107, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 46, 80, 7, 4006, 0, 4081, 87, 0, 36, 11, 1205, 65, 0, 0, 424, 259, 233, 15, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 153, 35, 3967, 1, 3672, 0, 0, 0, 1839, 0, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 3992, 0, 2, 0, 0, 3565, 1030, 4093, 3251, 0, 0, 0, 1717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15366, 17124, 21840, 15292, 23482, 10613, 9692, 17663] avg=16384.0000 max=23482 max_rank=4 min=9692 min_rank=6 imbalance_ratio=1.433228 max_load_reduction=107590 max_load_reduction_pct=0.820847 moved_to_echo=61450 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[80, 0, 3853, 0, 0, 0, 1, 18, 4096, 2728, 3, 2, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1851, 0, 4023, 0, 20, 4, 0, 3007, 1, 44, 791, 0, 3919, 2818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 3009, 0, 663, 670, 3656, 0, 10, 3645, 0, 73, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3581, 0, 4081, 3940, 0, 0, 27, 177, 74, 0, 0, 3164, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2870, 0, 4063, 0, 4036, 2531, 0, 11, 0, 0, 3, 0, 0, 11, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 875, 0, 0, 973, 0, 4096, 0, 2254, 0, 225, 0, 0, 0, 551, 13, 179, 403, 277, 445, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4072, 2900, 0, 3578, 3486, 3493, 0, 0, 3, 3768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 0, 447, 4088, 3794, 407, 3818, 1419, 0, 4092, 3953, 24, 4, 1027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11068, 16478, 15839, 19094, 13566, 10328, 21308, 23391] avg=16384.0000 max=23391 max_rank=7 min=10328 min_rank=5 imbalance_ratio=1.427673 max_load_reduction=107681 max_load_reduction_pct=0.821541 moved_to_echo=68593 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 3903, 223, 0, 4007, 50, 1, 12, 3764, 63, 3799, 4077, 0, 3568, 1, 0, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 40, 0, 65, 81, 377, 3904, 0, 0, 3219, 499, 3780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4, 9, 4096, 0, 4, 9, 1575, 4096, 3506, 0, 24, 3498, 0, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 4044, 0, 58, 4096, 4096, 1138, 0, 171, 19, 0, 0, 1, 94, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 986, 0, 0, 0, 0, 4085, 0, 0, 0, 7, 3718, 0, 14, 4094, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 0, 1, 107, 1767, 4074, 0, 112, 0, 4087, 0, 1, 0, 418, 273, 2, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 306, 4090, 0, 3858, 82, 3623, 102, 0, 3781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 22, 0, 4084, 0, 151, 3761, 0, 3646, 4094, 0, 4072, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23741, 11976, 17189, 13887, 12977, 14948, 16510, 19844] avg=16384.0000 max=23741 max_rank=0 min=11976 min_rank=1 imbalance_ratio=1.449036 max_load_reduction=107331 max_load_reduction_pct=0.818871 moved_to_echo=64279 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[543, 0, 0, 1388, 4096, 1, 0, 47, 3868, 250, 27, 4092, 0, 38, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3250, 0, 0, 62, 0, 0, 65, 1, 17, 0, 203, 10, 0, 4089, 0, 3823, 3343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 41, 1873, 0, 20, 0, 4030, 0, 3882, 13, 3619, 0, 2970, 1469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 3, 3, 0, 1944, 0, 252, 4060, 171, 173, 502, 0, 885, 2681, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 928, 3052, 3023, 0, 15, 4095, 0, 1, 0, 0, 41, 0, 3, 7, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3103, 4089, 0, 4064, 237, 332, 3, 0, 0, 0, 111, 990, 2867, 0, 300, 0, 419, 256, 442, 1, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1883, 0, 4096, 0, 0, 4096, 3627, 60, 22, 3925, 0, 0, 0, 2107, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 16, 41, 0, 3524, 3, 1121, 3979, 162, 18, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14352, 14867, 22001, 18608, 15259, 17259, 19856, 8870] avg=16384.0000 max=22001 max_rank=2 min=8870 min_rank=7 imbalance_ratio=1.342834 max_load_reduction=109071 max_load_reduction_pct=0.832146 moved_to_echo=61244 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[640, 17024, 11520, 3584, 14720, 1408, 384, 128, 27264, 5376, 10624, 27648, 896, 9472, 128, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[20224, 16384, 128, 14336, 7040, 17280, 896, 9216, 11648, 256, 5248, 3072, 16512, 640, 640, 256, 3328, 2176, 2048, 512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 3456, 0, 1152, 2304, 384, 13056, 256, 1664, 12288, 2048, 15872, 1152, 128, 22784, 640, 31104, 23808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[4096, 1664, 23680, 0, 13056, 12160, 7168, 15616, 256, 1408, 5376, 128, 1024, 21888, 24192, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[768, 1920, 4608, 19456, 2944, 23296, 15872, 28672, 256, 128, 15744, 6912, 128, 0, 9600, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2304, 128, 128, 4224, 512, 20480, 16384, 4480, 29184, 3200, 16256, 19584, 5120, 4224, 3200, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12544, 6272, 15872, 5504, 11648, 15488, 4992, 5376, 0, 1536, 16896, 8192, 256, 14720, 12544, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[384, 14336, 128, 6400, 10496, 768, 4736, 17920, 16640, 11136, 128, 16640, 9472, 15872, 6016, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4090, 0, 277, 0, 192, 37, 27, 41, 0, 17, 4078, 2764, 0, 0, 0, 0, 5, 0, 0, 1969, 0, 0, 0, 1, 0, 4013, 0, 25, 0, 0, 0, 14, 0, 28, 0, 4086, 0, 0, 9, 4096, 87, 0, 2941, 4092, 17, 255, 0, 0, 4, 0, 91, 4094, 0, 4084, 302, 28, 4096, 4096, 0, 0, 0, 0, 187, 4094, 482, 3690, 0, 1, 15, 0, 0, 4079, 3342, 1, 0, 0, 561, 1, 612, 3455, 0, 4070, 3612, 0, 3943, 0, 853, 1310, 0, 0, 0, 0, 1582, 0, 0, 4096, 1668, 5, 0, 4013, 70, 128, 3773, 4091, 3537, 4095, 441, 0, 4096, 0, 92, 2108, 0, 0, 3748, 126, 0, 0, 2, 4020, 0, 1, 3981, 0, 1133, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4094, 0, 2291, 1999, 0, 359, 110, 4021, 74, 2000, 3848, 0, 57, 0, 23, 9, 4085, 0, 15, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 671, 1086, 4096, 15, 0, 0, 0, 4093, 8, 12, 0, 4096, 1, 0, 0, 0, 1358, 0, 0, 0, 1127, 7, 4096, 3, 2596, 3598, 3993, 0, 0, 0, 0, 0, 3867, 0, 0, 0, 590, 4, 5, 0, 16, 4, 225, 4092, 4096, 1469, 0, 0, 3879, 40, 0, 0, 1527, 3681, 3, 247, 4015, 3, 4, 8, 2086, 1, 0, 106, 0, 9, 0, 0, 3103, 0, 0, 3227, 232, 0, 4092, 85, 1, 3933, 0, 1557, 4084, 0, 3993, 3959, 4069, 224, 0, 3421, 3079, 4096, 0, 0, 2, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4086, 0, 561, 4092, 3926, 0, 0, 0, 409, 0, 3745, 0, 0, 0, 0, 0, 1218, 8, 2644, 96, 2, 6, 0, 4093, 10, 134, 0, 0, 4067, 0, 0, 0, 4077, 1343, 0, 4095, 3, 6, 2245, 0, 709, 16, 1666, 3302, 3, 125, 8, 4090, 0, 0, 0, 4096, 0, 3891, 3, 1, 3885, 2643, 0, 251, 0, 0, 13, 3884, 3791, 3549, 12, 1, 0, 0, 0, 3706, 3951, 432, 0, 0, 2465, 31, 0, 4096, 768, 4058, 0, 0, 1791, 7, 1, 6, 0, 0, 0, 103, 0, 0, 0, 0, 4, 660, 1812, 0, 51, 0, 0, 13, 4089, 1261, 330, 0, 4096, 0, 0, 0, 258, 3507, 4096, 716, 0, 7, 4095, 0, 4095, 0, 0, 8, 3753, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[308, 8, 3351, 4057, 2321, 3939, 0, 0, 4061, 0, 54, 3981, 0, 16, 0, 839, 4096, 698, 0, 3441, 0, 0, 78, 2, 4058, 0, 1, 31, 0, 115, 0, 0, 0, 13, 824, 5, 450, 0, 3996, 2, 0, 0, 0, 0, 4, 9, 0, 2, 0, 0, 563, 2340, 8, 4091, 4091, 0, 2, 4089, 0, 0, 0, 210, 2, 4015, 240, 0, 0, 157, 2, 10, 116, 3809, 2, 35, 3948, 2841, 3778, 0, 445, 0, 1, 0, 0, 0, 2, 118, 6, 0, 0, 38, 5, 0, 19, 104, 0, 4096, 0, 0, 20, 4096, 80, 0, 3803, 419, 1616, 3813, 0, 0, 4096, 4095, 4074, 374, 0, 3870, 6, 664, 0, 0, 4047, 3948, 0, 131, 2, 63, 4081, 3745, 0, 4086] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[2344, 2, 1, 111, 4087, 6, 0, 278, 6, 964, 3470, 4037, 394, 0, 0, 0, 2555, 2, 20, 4095, 174, 712, 0, 0, 0, 0, 0, 37, 0, 0, 6, 1, 0, 0, 0, 0, 1172, 3803, 4096, 13, 0, 0, 3952, 3716, 0, 0, 861, 0, 0, 0, 38, 4091, 0, 3843, 10, 0, 225, 1, 3867, 0, 49, 160, 75, 3944, 0, 0, 0, 5, 0, 0, 0, 4052, 0, 2327, 4096, 4092, 6, 0, 1, 4032, 1112, 1536, 0, 1082, 1, 277, 0, 0, 0, 0, 8, 63, 0, 0, 0, 1, 0, 0, 0, 2941, 0, 3956, 4054, 242, 3487, 4094, 0, 0, 3698, 0, 0, 3864, 140, 4096, 708, 329, 22, 0, 4096, 1321, 3556, 0, 0, 0, 3405, 4092, 0, 3062] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[4081, 0, 4082, 184, 0, 3754, 0, 0, 86, 0, 4096, 0, 62, 2, 0, 350, 4011, 0, 0, 2, 0, 0, 2, 0, 0, 35, 0, 1458, 0, 1, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 10, 5, 198, 1734, 2855, 0, 1, 0, 1475, 3109, 844, 0, 2514, 2, 4047, 4092, 0, 4006, 4090, 0, 516, 0, 0, 32, 4096, 68, 3384, 0, 5, 0, 0, 0, 4095, 276, 3270, 361, 1699, 0, 67, 0, 3975, 2688, 3793, 3807, 0, 0, 0, 0, 2001, 0, 0, 0, 0, 0, 1, 23, 4, 5, 1, 0, 3009, 8, 0, 4084, 1, 3669, 1905, 0, 0, 4096, 4096, 0, 4096, 1, 1573, 0, 721, 32, 0, 0, 4096, 1715, 0, 4096, 0, 1579, 0, 0, 2871] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 4096, 163, 4092, 42, 0, 98, 288, 1674, 1230, 3785, 0, 0, 0, 119, 4096, 0, 0, 46, 106, 0, 0, 0, 168, 36, 0, 0, 2476, 4096, 202, 0, 0, 8, 0, 0, 855, 1, 234, 0, 0, 0, 270, 0, 545, 0, 226, 1, 48, 4050, 0, 3684, 0, 2518, 0, 17, 31, 568, 31, 0, 0, 0, 37, 4096, 0, 3904, 4096, 0, 8, 0, 569, 4096, 8, 0, 4095, 0, 2781, 0, 0, 3630, 0, 0, 921, 0, 0, 21, 3619, 109, 106, 0, 5, 0, 368, 0, 109, 0, 0, 0, 3971, 0, 0, 3826, 3915, 0, 0, 63, 33, 11, 4065, 0, 4091, 3976, 2532, 3959, 3818, 4032, 443, 0, 4082, 3056, 3495, 1, 0, 54, 3043, 4082, 0, 2046] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[4009, 3, 4092, 0, 4, 0, 0, 0, 214, 2340, 4096, 0, 3027, 2, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 275, 0, 28, 0, 51, 0, 0, 0, 0, 0, 0, 0, 828, 0, 0, 0, 730, 4022, 4, 4059, 1193, 9, 22, 0, 449, 0, 4, 3537, 1, 365, 3960, 0, 0, 2347, 4091, 3659, 0, 0, 0, 58, 1046, 4092, 3754, 0, 91, 0, 0, 0, 4075, 3, 6, 3928, 0, 2625, 311, 0, 70, 4087, 0, 0, 0, 759, 0, 4084, 0, 0, 2479, 0, 0, 0, 1, 0, 0, 350, 84, 0, 4077, 88, 0, 4026, 1, 0, 4077, 0, 3, 4096, 4096, 0, 4096, 10, 4096, 0, 182, 4087, 0, 0, 4096, 4018, 0, 2593, 0, 47, 0, 0, 3965] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4081, 0, 4082, 184, 0, 3754, 0, 0, 86, 0, 3283, 0, 62, 2, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 2, 0, 0, 2, 0, 0, 35, 0, 1458, 0, 1, 0, 0, 4096, 3340, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 10, 5, 198, 1734, 2855, 0, 1, 0, 1475, 2191, 1373, 813, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3109, 844, 0, 2514, 2, 707, 4092, 0, 4006, 4090, 0, 516, 0, 0, 32, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 3384, 0, 5, 0, 0, 0, 1904, 276, 3270, 361, 1699, 0, 67, 0, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2688, 3793, 3807, 0, 0, 0, 0, 2001, 0, 0, 0, 0, 0, 1, 23, 4, 3687, 987, 2723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 3009, 8, 0, 397, 1, 3669, 1905, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 721, 32, 0, 0, 0, 1336, 0, 4096, 0, 1579, 0, 0, 2871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15884, 13531, 15103, 24008, 15009, 19714, 17187, 10636] avg=16384.0000 max=24008 max_rank=3 min=10636 min_rank=7 imbalance_ratio=1.465332 max_load_reduction=107064 max_load_reduction_pct=0.816833 moved_to_echo=62546 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2344, 2, 1, 111, 4087, 6, 0, 278, 6, 964, 2785, 4037, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2555, 2, 20, 4095, 174, 712, 0, 0, 0, 0, 0, 37, 0, 0, 6, 1, 3698, 3168, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1172, 3803, 4096, 13, 0, 0, 3952, 3716, 0, 0, 861, 0, 2164, 443, 685, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4091, 0, 675, 10, 0, 225, 1, 3867, 0, 49, 160, 75, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1888, 0, 2327, 4096, 4092, 6, 0, 1, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1112, 1536, 0, 1082, 1, 277, 0, 0, 0, 0, 8, 63, 0, 0, 0, 1, 3656, 2571, 878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2941, 0, 3956, 398, 242, 3487, 4094, 0, 0, 0, 0, 0, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 708, 329, 22, 0, 4096, 0, 2777, 0, 0, 0, 3405, 4092, 0, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15015, 15993, 21684, 13135, 16447, 11185, 18982, 18631] avg=16384.0000 max=21684 max_rank=2 min=11185 min_rank=5 imbalance_ratio=1.323486 max_load_reduction=109388 max_load_reduction_pct=0.834564 moved_to_echo=65245 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 2291, 1999, 0, 359, 110, 4021, 74, 2000, 3089, 0, 57, 0, 23, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 15, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 671, 3933, 3376, 1487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 4096, 15, 0, 0, 0, 4093, 8, 12, 0, 4096, 1, 0, 0, 0, 1358, 8, 1031, 759, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1127, 7, 720, 3, 2596, 3598, 3993, 0, 0, 0, 0, 0, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 590, 4, 5, 0, 8, 4, 225, 4092, 4096, 1469, 0, 0, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 1527, 3681, 3, 247, 4015, 3, 4, 8, 2086, 1, 0, 106, 0, 2910, 2506, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3103, 0, 0, 317, 232, 0, 4092, 85, 1, 0, 0, 1557, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 4069, 224, 0, 3421, 0, 3199, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18126, 13568, 17460, 15911, 14372, 19185, 13480, 18970] avg=16384.0000 max=19185 max_rank=5 min=13480 min_rank=6 imbalance_ratio=1.170959 max_load_reduction=111887 max_load_reduction_pct=0.853630 moved_to_echo=66007 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4086, 0, 561, 4092, 3926, 0, 0, 0, 409, 0, 3006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1218, 8, 2644, 96, 2, 6, 0, 4093, 10, 134, 0, 0, 4067, 0, 0, 0, 4096, 3207, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 1343, 0, 4095, 3, 6, 2245, 0, 709, 16, 1666, 3302, 3, 125, 8, 4090, 1979, 0, 739, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 684, 3, 1, 3885, 2643, 0, 251, 0, 0, 13, 3884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3791, 3549, 12, 1, 0, 0, 0, 1727, 3951, 432, 0, 0, 2465, 31, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 768, 4058, 0, 0, 1791, 7, 1, 6, 0, 0, 0, 103, 0, 0, 0, 0, 0, 2200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 660, 1812, 0, 51, 0, 0, 13, 4089, 1261, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 258, 0, 4096, 716, 0, 7, 4095, 0, 3198, 0, 0, 8, 3753, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16080, 20888, 25303, 15460, 20055, 8934, 8220, 16132] avg=16384.0000 max=25303 max_rank=2 min=8220 min_rank=6 imbalance_ratio=1.544373 max_load_reduction=105769 max_load_reduction_pct=0.806953 moved_to_echo=53341 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 277, 0, 192, 37, 27, 41, 0, 17, 3273, 2764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1969, 0, 0, 0, 1, 0, 4013, 0, 25, 0, 0, 0, 14, 4096, 3366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 4086, 0, 0, 9, 4096, 87, 0, 2941, 4092, 17, 255, 0, 0, 2178, 1346, 805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 91, 4094, 0, 718, 302, 28, 4096, 4096, 0, 0, 0, 0, 187, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 3690, 0, 1, 15, 0, 0, 1901, 3342, 1, 0, 0, 561, 1, 612, 3455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 3612, 0, 3943, 0, 853, 1310, 0, 0, 0, 0, 1582, 0, 0, 4096, 3402, 0, 2674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1668, 5, 0, 4013, 70, 128, 371, 4091, 3537, 4095, 441, 0, 0, 0, 92, 2108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 126, 0, 0, 2, 0, 0, 1, 3981, 0, 1133, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10718, 13489, 19940, 17710, 14061, 25542, 20619, 8993] avg=16384.0000 max=25542 max_rank=5 min=8993 min_rank=7 imbalance_ratio=1.558960 max_load_reduction=105530 max_load_reduction_pct=0.805130 moved_to_echo=69215 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 163, 4092, 42, 0, 98, 288, 1674, 988, 3785, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 46, 106, 0, 0, 0, 168, 36, 0, 0, 2476, 4096, 202, 0, 4065, 2075, 1474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 855, 1, 234, 0, 0, 0, 270, 0, 545, 0, 226, 1, 2187, 1024, 242, 765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 4050, 0, 3684, 0, 443, 0, 17, 31, 568, 31, 0, 0, 0, 37, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3904, 4096, 0, 8, 0, 569, 1909, 8, 0, 4095, 0, 2781, 0, 0, 3630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 921, 0, 0, 21, 3619, 109, 106, 0, 5, 0, 368, 0, 109, 0, 3530, 2485, 2032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 3826, 385, 0, 0, 63, 33, 11, 0, 0, 4091, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2532, 0, 3818, 4032, 443, 0, 4082, 0, 2730, 1, 0, 54, 3043, 4082, 0, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15345, 18840, 6358, 13005, 21000, 13305, 16356, 26863] avg=16384.0000 max=26863 max_rank=7 min=6358 min_rank=2 imbalance_ratio=1.639587 max_load_reduction=104209 max_load_reduction_pct=0.795052 moved_to_echo=77524 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4009, 3, 4092, 0, 4, 0, 0, 0, 214, 2340, 3288, 0, 3027, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 0, 275, 0, 28, 0, 51, 0, 0, 0, 0, 4096, 3264, 1526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 0, 0, 0, 730, 4022, 4, 4059, 1193, 9, 22, 0, 449, 2176, 1373, 808, 880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3537, 1, 365, 696, 0, 0, 2347, 4091, 3659, 0, 0, 0, 58, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3754, 0, 91, 0, 0, 0, 1899, 3, 6, 3928, 0, 2625, 311, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 759, 0, 4084, 0, 0, 2479, 0, 0, 0, 1, 0, 0, 3631, 2570, 2723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 84, 0, 4077, 88, 0, 395, 1, 0, 4077, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 182, 4087, 0, 0, 0, 3138, 0, 2593, 0, 47, 0, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16979, 13334, 16553, 15804, 16779, 20334, 17267, 14022] avg=16384.0000 max=20334 max_rank=5 min=13334 min_rank=1 imbalance_ratio=1.241089 max_load_reduction=110738 max_load_reduction_pct=0.844864 moved_to_echo=68402 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[308, 8, 3351, 4057, 2321, 3939, 0, 0, 4061, 0, 44, 3981, 0, 16, 0, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 698, 0, 3441, 0, 0, 78, 2, 4058, 0, 1, 31, 0, 115, 0, 0, 4096, 3372, 1441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 824, 5, 450, 0, 3996, 2, 0, 0, 0, 0, 4, 9, 0, 2, 2034, 1322, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 563, 2340, 8, 719, 4091, 0, 2, 4089, 0, 0, 0, 210, 2, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 157, 2, 10, 116, 1775, 2, 35, 3948, 2841, 3778, 0, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 118, 6, 0, 0, 38, 5, 0, 19, 104, 0, 4096, 3429, 2429, 2626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 80, 0, 374, 419, 1616, 3813, 0, 0, 0, 4095, 4074, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 664, 0, 0, 4047, 0, 0, 131, 2, 63, 4081, 3745, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22925, 21429, 8671, 16039, 13349, 12873, 18961, 16825] avg=16384.0000 max=22925 max_rank=0 min=8671 min_rank=2 imbalance_ratio=1.399231 max_load_reduction=108147 max_load_reduction_pct=0.825096 moved_to_echo=62008 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[5248, 5504, 896, 13184, 2560, 3840, 14720, 4864, 4864, 256, 18816, 15232, 640, 512, 1152, 7424, 14976, 7936, 4864, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[23040, 128, 18816, 10624, 14720, 8192, 256, 4480, 5248, 7040, 19840, 14592, 3584, 128, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[3200, 4992, 4352, 22016, 384, 5376, 8576, 2688, 18304, 23680, 7680, 768, 128, 384, 512, 29056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2944, 0, 16384, 10880, 4864, 128, 19840, 0, 16384, 256, 10752, 128, 17152, 12032, 128, 20224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[24192, 768, 2688, 9728, 384, 768, 128, 4480, 4352, 4352, 128, 1664, 6656, 4224, 256, 768, 32256, 25216, 9472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[8704, 18304, 4224, 896, 128, 128, 768, 13056, 7680, 6400, 20608, 12800, 13696, 512, 1152, 23168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[8704, 13568, 8448, 2688, 10240, 512, 8832, 7552, 128, 2560, 128, 2304, 2048, 128, 256, 8320, 24320, 15872, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[2048, 768, 5888, 21248, 384, 7936, 2688, 5120, 16512, 23424, 896, 128, 0, 12288, 9856, 22656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[1120, 3505, 200, 97, 518, 71, 0, 0, 4092, 3906, 0, 0, 0, 1, 0, 0, 0, 7, 4095, 0, 0, 0, 0, 3327, 0, 0, 3744, 87, 0, 1, 0, 2, 370, 3258, 4052, 0, 0, 0, 3742, 4096, 7, 581, 0, 0, 1788, 4003, 2397, 31, 168, 76, 0, 0, 4022, 6, 0, 109, 35, 2497, 132, 0, 18, 4080, 0, 0, 4090, 4096, 0, 0, 0, 1873, 0, 0, 4081, 0, 2936, 0, 0, 5, 3336, 0, 0, 55, 0, 66, 0, 0, 26, 1825, 0, 0, 0, 4095, 0, 4096, 0, 2, 0, 0, 307, 0, 0, 142, 3900, 852, 2410, 0, 58, 405, 33, 135, 4096, 0, 15, 4096, 4094, 0, 1, 4096, 3673, 0, 4041, 0, 0, 3766, 45, 740, 0, 3344] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 4085, 0, 0, 4089, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 2313, 4073, 0, 141, 0, 4095, 0, 0, 1, 0, 141, 3271, 4, 0, 0, 4096, 4049, 0, 506, 0, 4096, 0, 0, 4096, 0, 0, 466, 3770, 0, 4096, 34, 2, 1976, 4095, 3855, 0, 0, 224, 0, 37, 0, 0, 1, 3755, 0, 0, 3949, 0, 2654, 0, 0, 0, 0, 0, 4096, 0, 156, 1, 4080, 3610, 339, 3, 0, 8, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4071, 0, 4095, 0, 22, 0, 0, 0, 777, 0, 0, 4091, 3910, 3094, 0, 2604, 250, 3576, 0, 0, 0, 3420, 4052, 958, 0, 611, 2, 0, 8, 2034, 0, 19, 1060, 0, 0, 0, 3953] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 1552, 18, 7, 0, 0, 11, 0, 4080, 0, 0, 0, 167, 0, 2, 116, 0, 0, 2951, 1, 0, 248, 0, 0, 14, 3295, 600, 6, 16, 314, 8, 0, 0, 3236, 0, 0, 198, 0, 0, 0, 299, 0, 0, 1874, 0, 564, 1032, 2717, 833, 473, 4004, 3769, 0, 2142, 26, 20, 0, 544, 0, 0, 4096, 24, 1, 3895, 4064, 2, 21, 9, 4096, 0, 383, 362, 78, 4096, 4052, 0, 160, 1044, 3, 126, 0, 4087, 0, 4065, 3745, 0, 1309, 2, 0, 3604, 422, 0, 4096, 0, 29, 1, 0, 0, 3486, 3954, 0, 4095, 0, 1946, 0, 0, 0, 45, 1562, 2717, 54, 1541, 0, 4094, 0, 0, 4063, 0, 0, 2377, 602, 2749, 4093, 2556, 0, 0, 4033] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4096, 3944, 2, 0, 7, 278, 0, 0, 4085, 3678, 2998, 0, 0, 4066, 0, 379, 0, 0, 1983, 3492, 4, 4020, 0, 2541, 0, 0, 4096, 2311, 0, 997, 0, 0, 0, 4087, 4091, 0, 6, 1, 4096, 61, 0, 4091, 0, 0, 0, 4096, 4094, 900, 0, 0, 1, 0, 2085, 1, 0, 0, 4089, 12, 0, 0, 0, 713, 6, 95, 4025, 63, 0, 4, 0, 275, 0, 0, 0, 0, 0, 4017, 2468, 0, 4062, 0, 0, 0, 3, 0, 0, 9, 0, 393, 0, 3486, 0, 4032, 0, 2, 1467, 0, 0, 3123, 0, 0, 1150, 0, 0, 0, 4096, 0, 0, 0, 76, 353, 161, 0, 3762, 0, 3841, 3928, 0, 0, 2941, 0, 2553, 0, 379, 4025, 0, 21, 0, 855] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[62, 3761, 3985, 0, 129, 4083, 0, 10, 3052, 4094, 0, 0, 0, 1, 0, 0, 0, 61, 0, 134, 0, 1741, 678, 0, 0, 0, 4092, 0, 0, 4089, 379, 958, 0, 1, 10, 1514, 0, 17, 0, 279, 0, 3930, 344, 0, 0, 3995, 0, 3594, 4077, 11, 38, 0, 4072, 10, 7, 5, 3858, 0, 1, 1898, 16, 3986, 0, 0, 3681, 4059, 0, 0, 4095, 205, 0, 0, 3879, 0, 0, 21, 2, 0, 18, 0, 2, 7, 0, 40, 0, 0, 3902, 12, 113, 0, 0, 1740, 16, 4096, 68, 0, 537, 0, 4096, 0, 0, 172, 2674, 0, 3805, 0, 0, 47, 3110, 928, 4096, 0, 4096, 0, 0, 0, 29, 13, 3915, 0, 2, 4085, 5, 4096, 2336, 9, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[55, 3652, 4091, 0, 0, 3554, 41, 0, 238, 4000, 0, 0, 18, 77, 0, 725, 0, 0, 74, 617, 1199, 1, 0, 0, 0, 0, 1001, 4092, 0, 377, 0, 0, 0, 2765, 3786, 0, 32, 3, 449, 741, 186, 21, 0, 0, 0, 3613, 4096, 3859, 0, 447, 2514, 0, 4070, 4, 0, 0, 0, 112, 3421, 0, 0, 4036, 2, 0, 3904, 4093, 3548, 130, 3213, 0, 0, 0, 3249, 0, 0, 0, 0, 0, 0, 2029, 523, 0, 4096, 0, 153, 1, 0, 20, 0, 84, 0, 4037, 0, 4093, 4, 0, 0, 4, 3783, 0, 75, 0, 0, 0, 3844, 0, 0, 0, 4096, 4096, 1506, 0, 0, 3984, 3946, 3876, 0, 29, 1535, 0, 4085, 346, 19, 1, 3975, 60, 0, 666] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[3, 3970, 119, 0, 0, 4020, 0, 18, 35, 18, 122, 23, 0, 374, 257, 918, 0, 0, 0, 163, 484, 33, 2871, 4058, 4094, 0, 3957, 0, 0, 1, 2, 0, 0, 41, 4096, 0, 0, 0, 2, 3848, 0, 0, 31, 243, 0, 3795, 16, 0, 2748, 0, 0, 344, 4063, 0, 18, 0, 0, 0, 0, 0, 977, 4093, 1113, 0, 0, 4092, 0, 3855, 4085, 89, 0, 0, 3648, 6, 299, 0, 0, 0, 1811, 76, 0, 0, 2737, 4047, 4057, 9, 3, 0, 0, 4086, 50, 3743, 0, 4032, 0, 0, 0, 1, 0, 459, 0, 0, 1, 0, 0, 648, 3995, 0, 0, 4096, 4067, 1, 4070, 3940, 2994, 4090, 0, 0, 3528, 95, 43, 4095, 0, 1, 0, 0, 0, 3255] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 4077, 0, 0, 0, 0, 0, 2019, 20, 102, 0, 0, 35, 3376, 0, 48, 0, 6, 6, 0, 0, 0, 434, 15, 0, 0, 3935, 0, 8, 109, 99, 0, 0, 1682, 3601, 0, 0, 0, 388, 0, 0, 0, 0, 0, 0, 758, 0, 4081, 2041, 119, 0, 0, 4008, 0, 0, 1104, 3231, 38, 18, 3559, 0, 1863, 86, 0, 4050, 3845, 690, 333, 536, 3003, 0, 0, 1013, 276, 0, 0, 0, 4090, 4085, 42, 4010, 0, 3118, 0, 4057, 0, 2, 394, 0, 388, 0, 3377, 0, 4032, 0, 0, 20, 0, 800, 0, 3607, 0, 3647, 50, 2896, 8, 104, 0, 4082, 1164, 3718, 0, 46, 4089, 3384, 4096, 650, 2546, 1061, 2, 1180, 4096, 0, 3475, 0, 48, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 4085, 0, 0, 4089, 0, 4096, 9, 0, 0, 0, 0, 462, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2313, 4073, 0, 141, 0, 4095, 0, 0, 1, 0, 141, 3271, 4, 0, 3953, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3001, 0, 506, 0, 4096, 0, 0, 4096, 0, 0, 466, 3770, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 2, 1976, 4095, 3855, 0, 0, 224, 0, 37, 0, 0, 1, 3755, 0, 0, 1384, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 879, 0, 2654, 0, 0, 0, 0, 0, 4096, 0, 156, 1, 4080, 3610, 339, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4071, 0, 4095, 0, 22, 1686, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 777, 0, 0, 4091, 3910, 3094, 0, 2604, 250, 3576, 0, 0, 0, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3420, 4052, 0, 0, 611, 2, 0, 8, 2034, 0, 19, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17376, 18595, 24127, 15718, 15818, 10136, 18810, 10492] avg=16384.0000 max=24127 max_rank=2 min=10136 min_rank=5 imbalance_ratio=1.472595 max_load_reduction=106945 max_load_reduction_pct=0.815926 moved_to_echo=55256 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1120, 3505, 200, 97, 518, 71, 0, 0, 4092, 3906, 0, 0, 0, 1, 0, 0, 1642, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4095, 0, 0, 0, 0, 3327, 0, 0, 3744, 87, 0, 1, 0, 2, 3344, 2580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 3258, 3004, 0, 0, 0, 3742, 4096, 7, 581, 0, 0, 1788, 4003, 2397, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 76, 0, 0, 4022, 6, 0, 109, 35, 2497, 132, 0, 18, 4080, 0, 0, 1433, 1514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 910, 4096, 0, 0, 0, 1873, 0, 0, 4081, 0, 2936, 0, 0, 5, 3336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 66, 0, 0, 26, 1825, 0, 0, 0, 4095, 0, 4096, 0, 2, 1747, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 0, 0, 142, 3900, 852, 2410, 0, 58, 405, 33, 135, 4096, 0, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 1, 4096, 3673, 0, 4041, 0, 0, 1226, 45, 740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15692, 17187, 23277, 14090, 17237, 12810, 12846, 17933] avg=16384.0000 max=23277 max_rank=2 min=12810 min_rank=5 imbalance_ratio=1.420715 max_load_reduction=107795 max_load_reduction_pct=0.822411 moved_to_echo=60826 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1552, 18, 7, 0, 0, 11, 0, 4080, 0, 0, 0, 167, 0, 2, 1785, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 2951, 1, 0, 248, 0, 0, 14, 3295, 600, 6, 16, 314, 8, 4033, 2581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2399, 0, 0, 198, 0, 0, 0, 299, 0, 0, 1874, 0, 564, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2717, 833, 473, 4004, 3769, 0, 2142, 26, 20, 0, 544, 0, 0, 4096, 24, 1, 1365, 1513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 867, 4064, 2, 21, 9, 4096, 0, 383, 362, 78, 4096, 4052, 0, 160, 1044, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 4087, 0, 4065, 3745, 0, 1309, 2, 0, 3604, 422, 0, 4096, 0, 29, 1663, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3486, 3954, 0, 4095, 0, 1946, 0, 0, 0, 45, 1562, 2717, 54, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1541, 0, 0, 0, 0, 4063, 0, 0, 2377, 602, 2749, 1332, 2556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12149, 14183, 6366, 21527, 19237, 24124, 18266, 15220] avg=16384.0000 max=24124 max_rank=5 min=6366 min_rank=2 imbalance_ratio=1.472412 max_load_reduction=106948 max_load_reduction_pct=0.815948 moved_to_echo=76847 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[62, 3761, 3985, 0, 129, 4083, 0, 10, 3052, 4094, 0, 0, 0, 1, 0, 0, 1786, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 134, 0, 1741, 678, 0, 0, 0, 4092, 0, 0, 4089, 379, 958, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 1514, 0, 17, 0, 279, 0, 3930, 344, 0, 0, 3995, 0, 3594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 11, 38, 0, 4072, 10, 7, 5, 3858, 0, 1, 1898, 16, 3986, 0, 0, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 819, 4059, 0, 0, 4095, 205, 0, 0, 3879, 0, 0, 21, 2, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7, 0, 40, 0, 0, 3902, 12, 113, 0, 0, 1740, 16, 4096, 68, 0, 1572, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 537, 0, 4096, 0, 0, 172, 2674, 0, 3805, 0, 0, 47, 3110, 928, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 29, 13, 3915, 0, 2, 4085, 5, 1333, 2336, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20964, 16225, 13682, 19269, 13098, 12545, 19466, 15823] avg=16384.0000 max=20964 max_rank=0 min=12545 min_rank=5 imbalance_ratio=1.279541 max_load_reduction=110108 max_load_reduction_pct=0.840057 moved_to_echo=60932 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3, 3970, 119, 0, 0, 4020, 0, 18, 35, 18, 122, 23, 0, 374, 257, 918, 0, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 484, 33, 2871, 4058, 4094, 0, 3957, 0, 0, 1, 2, 0, 3255, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 3036, 0, 0, 0, 2, 3848, 0, 0, 31, 243, 0, 3795, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2748, 0, 0, 344, 4063, 0, 18, 0, 0, 0, 0, 0, 977, 4093, 1113, 0, 0, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3855, 4085, 89, 0, 0, 3648, 6, 299, 0, 0, 0, 1811, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2737, 4047, 4057, 9, 3, 0, 0, 4086, 50, 3743, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 459, 0, 0, 1, 0, 0, 648, 3995, 0, 0, 4096, 4067, 1, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 3940, 0, 4090, 0, 0, 3528, 95, 43, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10423, 20805, 11012, 14463, 17961, 22764, 13782, 19862] avg=16384.0000 max=22764 max_rank=5 min=10423 min_rank=0 imbalance_ratio=1.389404 max_load_reduction=108308 max_load_reduction_pct=0.826324 moved_to_echo=74369 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[55, 3652, 4091, 0, 0, 3554, 41, 0, 238, 4000, 0, 0, 18, 77, 0, 725, 0, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 617, 1199, 1, 0, 0, 0, 0, 1001, 4092, 0, 377, 0, 0, 666, 2488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2765, 2806, 0, 32, 3, 449, 741, 186, 21, 0, 0, 0, 3613, 4096, 3859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 447, 2514, 0, 4070, 4, 0, 0, 0, 112, 3421, 0, 0, 4036, 2, 0, 1368, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 869, 4093, 3548, 130, 3213, 0, 0, 0, 3249, 0, 0, 0, 0, 0, 0, 2029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 0, 4096, 0, 153, 1, 0, 20, 0, 84, 0, 4037, 0, 4093, 4, 0, 1667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3783, 0, 75, 0, 0, 0, 3844, 0, 0, 0, 4096, 4096, 1506, 0, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 0, 3876, 0, 29, 1535, 0, 4085, 346, 19, 1, 3975, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16956, 10515, 18571, 17432, 17131, 14678, 17879, 17910] avg=16384.0000 max=18571 max_rank=2 min=10515 min_rank=1 imbalance_ratio=1.133484 max_load_reduction=112501 max_load_reduction_pct=0.858315 moved_to_echo=67598 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 3944, 2, 0, 7, 278, 0, 0, 4085, 3678, 2998, 0, 0, 4066, 0, 379, 1759, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1983, 3492, 4, 4020, 0, 2541, 0, 0, 4096, 2311, 0, 997, 0, 0, 855, 2422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 3027, 0, 6, 1, 4096, 61, 0, 4091, 0, 0, 0, 4096, 4094, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2085, 1, 0, 0, 4089, 12, 0, 0, 0, 713, 6, 95, 1415, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 887, 63, 0, 4, 0, 275, 0, 0, 0, 0, 0, 4017, 2468, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 9, 0, 393, 0, 3486, 0, 4032, 0, 2, 1467, 0, 1723, 963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3123, 0, 0, 1150, 0, 0, 0, 4096, 0, 0, 0, 76, 353, 161, 0, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3762, 0, 0, 3928, 0, 0, 2941, 0, 2553, 0, 379, 1303, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[25840, 22721, 24459, 9836, 11776, 12078, 9475, 14887] avg=16384.0000 max=25840 max_rank=0 min=9475 min_rank=6 imbalance_ratio=1.577148 max_load_reduction=105232 max_load_reduction_pct=0.802856 moved_to_echo=48216 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4077, 0, 0, 0, 0, 0, 2019, 20, 102, 0, 0, 35, 3376, 0, 48, 1515, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 434, 15, 0, 0, 3935, 0, 8, 109, 99, 0, 4096, 2133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1682, 2669, 0, 0, 0, 388, 0, 0, 0, 0, 0, 0, 758, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2041, 119, 0, 0, 4008, 0, 0, 1104, 3231, 38, 18, 3559, 0, 1863, 86, 0, 1419, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 901, 3845, 690, 333, 536, 3003, 0, 0, 1013, 276, 0, 0, 0, 4090, 4085, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 3118, 0, 4057, 0, 2, 394, 0, 388, 0, 3377, 0, 4032, 0, 0, 1730, 829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 800, 0, 3607, 0, 3647, 50, 2896, 8, 104, 0, 4082, 1164, 3718, 0, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 4089, 0, 4096, 650, 2546, 1061, 2, 1180, 4096, 0, 1131, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11672, 10841, 9578, 18737, 18814, 21937, 20548, 18945] avg=16384.0000 max=21937 max_rank=5 min=9578 min_rank=2 imbalance_ratio=1.338928 max_load_reduction=109135 max_load_reduction_pct=0.832634 moved_to_echo=80244 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[17024, 20224, 0, 16000, 1408, 10752, 16768, 128, 16384, 13312, 3200, 6784, 8960, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[11904, 1536, 5120, 8448, 29952, 128, 2176, 1536, 11264, 2816, 4224, 5504, 1024, 26624, 1280, 128, 9728, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[5376, 27008, 9984, 128, 4864, 16128, 128, 2176, 15616, 19968, 7296, 128, 128, 8064, 384, 2176, 8960, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[640, 3200, 9088, 4736, 8832, 384, 18432, 4864, 22144, 768, 6784, 768, 15104, 12416, 20480, 128, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[384, 16000, 19968, 1536, 640, 256, 12800, 9088, 256, 13056, 384, 256, 4224, 24064, 11264, 17664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[4736, 128, 14080, 4224, 12416, 3840, 3968, 3968, 128, 8064, 3712, 25600, 128, 28544, 1664, 128, 11904, 4992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[6144, 24320, 6912, 4352, 12032, 9600, 0, 384, 20352, 384, 7552, 8192, 6656, 7936, 14720, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 8576, 11520, 1792, 6016, 4352, 14080, 4096, 128, 24192, 7168, 256, 8960, 896, 1024, 24320, 14720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 3384, 0, 0, 0, 0, 83, 12, 0, 384, 0, 0, 20, 0, 12, 0, 4096, 3, 4053, 0, 0, 0, 0, 0, 0, 0, 761, 0, 2, 4096, 4090, 0, 14, 2199, 4091, 9, 0, 401, 3935, 10, 3, 4095, 1, 1916, 0, 0, 0, 4096, 4057, 7, 1729, 0, 0, 0, 0, 3982, 192, 0, 282, 1508, 1837, 2050, 0, 2362, 0, 1, 3861, 0, 11, 6, 0, 3849, 4013, 15, 0, 0, 48, 702, 0, 48, 944, 1230, 9, 0, 4095, 0, 0, 4095, 4096, 0, 3798, 0, 100, 3996, 120, 4095, 3, 0, 4095, 0, 236, 28, 8, 351, 4096, 598, 4093, 4096, 2, 3943, 0, 1, 0, 101, 3636, 0, 0, 0, 3804, 14, 0, 0, 1, 154, 0, 0, 0, 2908] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 3964, 1, 0, 0, 0, 0, 190, 0, 17, 0, 0, 3719, 809, 0, 0, 4083, 0, 4095, 34, 0, 432, 0, 3965, 54, 0, 3937, 6, 0, 423, 69, 0, 0, 135, 1228, 2548, 0, 0, 4093, 2, 4096, 3617, 0, 0, 2721, 0, 0, 4096, 0, 1014, 3823, 0, 3914, 1188, 0, 0, 6, 0, 0, 0, 3951, 3910, 302, 4096, 109, 4093, 1, 3071, 0, 0, 0, 4048, 0, 0, 0, 2, 0, 0, 2530, 1937, 498, 3962, 167, 0, 4093, 0, 0, 3993, 2020, 1480, 0, 8, 2044, 3871, 12, 4096, 103, 1, 0, 410, 1181, 3694, 54, 0, 161, 475, 0, 0, 0, 3731, 0, 0, 31, 350, 3332, 772, 0, 0, 0, 8, 0, 9, 0, 0, 0, 0, 91, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 24, 22, 724, 160, 0, 34, 0, 3099, 0, 0, 4, 1, 0, 0, 132, 4096, 4091, 476, 424, 138, 0, 1666, 6, 0, 163, 301, 0, 0, 0, 32, 0, 0, 0, 4020, 0, 0, 2234, 1, 0, 4096, 3678, 0, 2690, 3783, 0, 0, 4040, 4096, 0, 0, 0, 0, 2, 10, 0, 111, 0, 89, 1, 4079, 3805, 0, 3926, 724, 4096, 0, 1255, 0, 358, 3803, 3108, 0, 4095, 0, 62, 4086, 0, 0, 295, 1, 3975, 0, 0, 0, 936, 0, 1506, 3933, 0, 988, 0, 2859, 1040, 64, 3904, 4082, 0, 4006, 0, 19, 2233, 0, 0, 10, 2982, 4092, 8, 17, 0, 57, 0, 7, 0, 77, 0, 4084, 0, 0, 109, 0, 10, 3745, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 1103, 14, 133, 614, 2889, 2, 4096, 0, 0, 80, 0, 0, 3288, 1064, 8, 4096, 453, 4070, 4074, 0, 4096, 1413, 140, 6, 0, 0, 0, 1808, 4081, 3967, 0, 0, 2963, 4089, 0, 1, 0, 3885, 0, 1394, 0, 2, 0, 2610, 526, 0, 2654, 4057, 6, 31, 0, 3521, 0, 25, 2228, 14, 0, 209, 0, 3825, 0, 15, 2435, 0, 3975, 1953, 1312, 109, 3, 929, 2019, 4016, 4049, 0, 167, 0, 0, 0, 0, 42, 4077, 278, 0, 0, 2, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 3994, 3193, 0, 3941, 0, 0, 3791, 0, 1, 0, 2942, 4045, 17, 1, 71, 187, 0, 384, 157, 1120, 0, 3118, 0, 945, 0, 60, 85, 0, 0, 8, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 1731, 0, 0, 3, 1848, 0, 0, 3977, 0, 0, 1, 9, 2448, 0, 27, 1851, 3046, 3985, 599, 0, 4096, 0, 0, 0, 24, 0, 0, 0, 4062, 40, 0, 0, 0, 4088, 0, 0, 3903, 2574, 0, 4061, 334, 0, 2309, 4083, 0, 0, 4042, 4069, 0, 0, 0, 0, 4095, 8, 143, 50, 56, 48, 915, 2796, 6, 0, 4096, 865, 2690, 610, 24, 0, 0, 531, 29, 3950, 5, 0, 0, 4014, 244, 3284, 9, 828, 2284, 0, 0, 2688, 13, 0, 0, 3290, 0, 225, 36, 4021, 2119, 0, 4095, 0, 0, 4096, 1, 10, 0, 2, 0, 2, 2008, 4096, 0, 0, 4065, 0, 7, 1, 36, 3, 0, 4096, 0, 3129, 0, 0, 0, 182, 0, 10, 0, 0, 4051] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 0, 24, 116, 0, 79, 0, 5, 0, 3465, 3828, 0, 875, 4094, 3686, 0, 0, 0, 0, 262, 0, 0, 0, 0, 0, 93, 564, 0, 0, 0, 0, 734, 0, 0, 4092, 4096, 0, 4096, 3977, 28, 0, 4087, 0, 3997, 107, 468, 0, 0, 0, 4054, 47, 0, 111, 7, 2773, 0, 44, 3453, 4096, 0, 245, 1634, 0, 0, 0, 0, 4047, 0, 1517, 3024, 0, 0, 0, 2, 3517, 3938, 0, 0, 4095, 1180, 1, 411, 0, 0, 3033, 1665, 0, 0, 1629, 3685, 4096, 1, 4091, 0, 0, 4096, 0, 1, 4085, 424, 3, 0, 4094, 20, 0, 0, 0, 4059, 3, 0, 0, 8, 0, 737, 0, 4095, 3524, 45, 0, 2637, 0, 0, 0, 1, 4064] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 22, 338, 0, 0, 339, 4054, 233, 0, 0, 1525, 0, 2458, 4096, 0, 19, 4083, 0, 0, 0, 0, 633, 0, 0, 0, 0, 3936, 6, 0, 355, 0, 0, 3174, 887, 0, 96, 6, 3349, 4096, 0, 5, 4009, 6, 4092, 0, 54, 356, 4096, 4096, 0, 0, 0, 4073, 4096, 0, 1, 7, 25, 0, 4090, 4031, 102, 0, 2554, 32, 0, 0, 0, 0, 0, 0, 0, 193, 0, 67, 0, 15, 0, 0, 0, 0, 4083, 811, 4, 4096, 0, 0, 3627, 4095, 0, 0, 86, 6, 192, 0, 4080, 0, 0, 1253, 0, 3954, 2859, 3733, 231, 0, 4095, 4096, 0, 0, 753, 4036, 88, 0, 682, 32, 0, 2597, 0, 103, 16, 4096, 0, 0, 3761, 0, 0, 5, 3897] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 4092, 0, 1, 41, 2607, 6, 3, 0, 0, 1, 3311, 2762, 4089, 0, 6, 4095, 0, 184, 0, 0, 190, 0, 0, 14, 0, 57, 0, 0, 1932, 38, 0, 4064, 0, 265, 4090, 0, 0, 1351, 2433, 3269, 2215, 0, 4067, 3666, 226, 4073, 178, 3888, 0, 20, 0, 265, 4059, 0, 0, 0, 1240, 4, 2, 131, 0, 0, 4056, 5, 0, 320, 3908, 0, 3950, 0, 17, 2870, 83, 0, 2, 0, 4096, 1, 2359, 0, 2596, 0, 0, 0, 0, 0, 3692, 3850, 0, 0, 35, 6, 1755, 8, 4089, 4093, 0, 0, 4078, 4045, 26, 0, 0, 348, 26, 4041, 128, 0, 4030, 299, 0, 2, 38, 29, 0, 1311, 0, 34, 3979, 0, 55, 3842, 1, 0, 0, 0, 34] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1103, 14, 133, 614, 2889, 2, 4096, 0, 0, 80, 0, 0, 3288, 1064, 8, 3994, 3248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 453, 4070, 4074, 0, 4096, 1413, 140, 6, 0, 0, 0, 1808, 4081, 3967, 0, 0, 327, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2963, 4089, 0, 1, 0, 0, 0, 1394, 0, 2, 0, 2610, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2427, 6, 31, 0, 3521, 0, 25, 2228, 14, 0, 209, 0, 3825, 0, 15, 2435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 1953, 1312, 109, 3, 929, 2019, 4016, 4049, 0, 167, 0, 0, 0, 0, 1453, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4077, 278, 0, 0, 2, 0, 0, 3854, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3193, 0, 3941, 0, 0, 3791, 0, 1, 0, 2942, 1854, 17, 1, 71, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 157, 1120, 0, 3118, 0, 945, 0, 60, 85, 0, 0, 8, 0, 0, 0, 2654, 1864, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20533, 28708, 11585, 14736, 20226, 8254, 15998, 11032] avg=16384.0000 max=28708 max_rank=1 min=8254 min_rank=5 imbalance_ratio=1.752197 max_load_reduction=102364 max_load_reduction_pct=0.780975 moved_to_echo=55510 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 24, 22, 724, 160, 0, 34, 0, 3099, 0, 0, 4, 1, 0, 0, 132, 3904, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 476, 424, 138, 0, 1666, 6, 0, 163, 301, 0, 0, 0, 32, 0, 2061, 330, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 2234, 0, 0, 4096, 1617, 0, 2690, 3783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2451, 0, 0, 0, 0, 2, 10, 0, 111, 0, 89, 1, 4079, 3805, 0, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 724, 4096, 0, 1255, 0, 358, 3803, 3108, 0, 4095, 0, 62, 4086, 0, 0, 295, 1467, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3975, 0, 0, 0, 936, 0, 1506, 3701, 0, 988, 0, 2859, 1040, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4006, 0, 19, 2233, 0, 0, 10, 2982, 1876, 8, 17, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 77, 0, 4084, 0, 0, 109, 0, 10, 3745, 0, 4096, 0, 0, 4096, 4040, 1886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8105, 13962, 18440, 14474, 23581, 15070, 15290, 22150] avg=16384.0000 max=23581 max_rank=4 min=8105 min_rank=0 imbalance_ratio=1.439270 max_load_reduction=107491 max_load_reduction_pct=0.820091 moved_to_echo=76091 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 3964, 1, 0, 0, 0, 0, 190, 0, 17, 0, 0, 3719, 809, 0, 0, 4096, 3423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4095, 34, 0, 432, 0, 3965, 54, 0, 3937, 6, 0, 423, 69, 0, 2027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 1228, 2548, 0, 0, 0, 2, 4096, 1590, 0, 0, 2721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1014, 3823, 0, 3914, 1188, 0, 0, 6, 0, 0, 0, 3951, 3910, 302, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 4093, 1, 3071, 0, 0, 0, 4048, 0, 0, 0, 2, 0, 0, 2530, 1937, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 3962, 167, 0, 4093, 0, 0, 3993, 1901, 1480, 0, 8, 2044, 3871, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 1, 0, 410, 1181, 3694, 54, 0, 161, 475, 0, 0, 0, 3731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 350, 3332, 772, 0, 0, 0, 8, 0, 9, 0, 0, 0, 0, 91, 4096, 4096, 0, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16219, 19125, 12320, 22204, 15910, 22029, 9810, 13455] avg=16384.0000 max=22204 max_rank=3 min=9810 min_rank=6 imbalance_ratio=1.355225 max_load_reduction=108868 max_load_reduction_pct=0.830597 moved_to_echo=61204 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 0, 24, 116, 0, 79, 0, 5, 0, 3465, 3828, 0, 875, 4091, 3426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3686, 0, 0, 0, 0, 262, 0, 0, 0, 0, 0, 93, 564, 0, 0, 2233, 4, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 734, 0, 0, 4092, 0, 0, 4096, 1744, 28, 0, 4087, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 0, 0, 0, 4054, 47, 0, 111, 7, 2773, 0, 44, 3453, 4096, 0, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1634, 0, 0, 0, 0, 4047, 0, 1517, 3024, 0, 0, 0, 2, 3517, 3938, 0, 172, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1180, 1, 411, 0, 0, 3033, 1563, 0, 0, 1629, 3685, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4085, 424, 3, 0, 4094, 3, 0, 0, 0, 4059, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 737, 0, 4095, 3524, 45, 0, 2637, 0, 0, 0, 1, 4064, 107, 13, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15916, 10960, 18778, 15102, 17953, 19694, 16768, 15901] avg=16384.0000 max=19694 max_rank=5 min=10960 min_rank=1 imbalance_ratio=1.202026 max_load_reduction=111378 max_load_reduction_pct=0.849747 moved_to_echo=70316 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 1, 41, 2607, 6, 3, 0, 0, 1, 3311, 2762, 4089, 0, 6, 4089, 1129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 184, 0, 0, 190, 0, 0, 14, 0, 57, 0, 0, 1932, 38, 0, 1241, 326, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 265, 4090, 0, 0, 0, 2433, 3269, 974, 0, 4067, 3666, 226, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2326, 0, 20, 0, 265, 4059, 0, 0, 0, 1240, 4, 2, 131, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 320, 3908, 0, 3950, 0, 17, 2870, 83, 0, 2, 0, 4096, 1, 2359, 1393, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2596, 0, 0, 0, 0, 0, 3692, 3623, 0, 0, 35, 6, 1755, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4078, 4045, 26, 0, 0, 348, 26, 1853, 128, 0, 4030, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 38, 29, 0, 1311, 0, 34, 3979, 0, 55, 3842, 1, 0, 0, 0, 34, 178, 1862, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22137, 8246, 27127, 12103, 19231, 11715, 18926, 11587] avg=16384.0000 max=27127 max_rank=2 min=8246 min_rank=1 imbalance_ratio=1.655701 max_load_reduction=103945 max_load_reduction_pct=0.793037 moved_to_echo=61459 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 22, 338, 0, 0, 339, 4054, 233, 0, 0, 1525, 0, 2458, 4096, 0, 19, 4080, 3426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 633, 0, 0, 0, 0, 3936, 6, 0, 355, 0, 0, 2247, 331, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3174, 887, 0, 96, 6, 3349, 0, 0, 5, 1762, 6, 4092, 0, 54, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2451, 0, 0, 0, 4073, 4096, 0, 1, 7, 25, 0, 4090, 4031, 102, 0, 2554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 193, 0, 67, 0, 15, 0, 0, 0, 1467, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 811, 4, 4096, 0, 0, 3627, 3854, 0, 0, 86, 6, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 0, 3954, 2859, 3733, 231, 0, 4095, 1877, 0, 0, 753, 4036, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 682, 32, 0, 2597, 0, 103, 16, 4096, 0, 0, 3761, 0, 0, 5, 3897, 4096, 1888, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20590, 11769, 13787, 21430, 2015, 16759, 22879, 21843] avg=16384.0000 max=22879 max_rank=6 min=2015 min_rank=4 imbalance_ratio=1.396423 max_load_reduction=108193 max_load_reduction_pct=0.825447 moved_to_echo=63496 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 3384, 0, 0, 0, 0, 83, 12, 0, 384, 0, 0, 20, 0, 12, 0, 4095, 3290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4053, 0, 0, 0, 0, 0, 0, 0, 761, 0, 2, 4096, 4090, 0, 2295, 331, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2199, 4091, 9, 0, 401, 0, 10, 3, 1800, 1, 1916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2427, 7, 1729, 0, 0, 0, 0, 3982, 192, 0, 282, 1508, 1837, 2050, 0, 2362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3861, 0, 11, 6, 0, 3849, 4013, 15, 0, 0, 48, 702, 0, 48, 1453, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 944, 1230, 9, 0, 4095, 0, 0, 4095, 3854, 0, 3798, 0, 100, 3996, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 236, 28, 8, 351, 4096, 598, 1876, 4096, 2, 3943, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 3636, 0, 0, 0, 3804, 14, 0, 0, 1, 154, 0, 0, 0, 2908, 4096, 1886, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11280, 19904, 10444, 16376, 14249, 22241, 19333, 17245] avg=16384.0000 max=22241 max_rank=5 min=10444 min_rank=2 imbalance_ratio=1.357483 max_load_reduction=108831 max_load_reduction_pct=0.830315 moved_to_echo=73068 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1731, 0, 0, 3, 1848, 0, 0, 3977, 0, 0, 1, 9, 2448, 0, 27, 4095, 2153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1851, 3046, 3985, 599, 0, 4096, 0, 0, 0, 24, 0, 0, 0, 4062, 40, 0, 187, 331, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 3903, 0, 0, 4061, 147, 0, 2309, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2434, 0, 0, 0, 0, 4095, 8, 143, 50, 56, 48, 915, 2796, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 865, 2690, 610, 24, 0, 0, 531, 29, 3950, 5, 0, 0, 4014, 244, 3284, 9, 1458, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 2284, 0, 0, 2688, 13, 0, 0, 3096, 0, 225, 36, 4021, 2119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 10, 0, 2, 0, 2, 2008, 1877, 0, 0, 4065, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 36, 3, 0, 4096, 0, 3129, 0, 0, 0, 182, 0, 10, 0, 0, 4051, 4042, 1888, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16292, 18398, 18591, 14647, 17907, 15310, 12068, 17859] avg=16384.0000 max=18591 max_rank=2 min=12068 min_rank=6 imbalance_ratio=1.134705 max_load_reduction=112481 max_load_reduction_pct=0.858162 moved_to_echo=63144 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3456, 14976, 6784, 9600, 128, 8448, 5376, 14592, 18176, 8320, 128, 256, 8192, 8576, 9856, 4736, 8960, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[11520, 128, 21504, 4608, 9472, 16768, 4224, 640, 4736, 17280, 11264, 4352, 128, 16640, 8704, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[30592, 11392, 16896, 5248, 256, 9472, 3456, 4224, 128, 256, 9088, 128, 1920, 15616, 8320, 0, 12416, 2048, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[7296, 6272, 18560, 6784, 128, 14080, 0, 2560, 21120, 9728, 128, 15104, 20992, 896, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 14336, 384, 896, 896, 7808, 4352, 4608, 7168, 512, 1664, 3328, 12544, 18560, 1152, 1152, 32512, 20096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2432, 26368, 2560, 128, 15488, 1024, 0, 19968, 25472, 1536, 5120, 1920, 12800, 17152, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[14848, 1152, 5632, 0, 15872, 13568, 128, 6528, 512, 4096, 640, 6656, 24192, 14080, 384, 23808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[512, 1408, 8320, 896, 16000, 0, 12160, 7680, 4224, 256, 10496, 3968, 4224, 0, 128, 23168, 23424, 11392, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[474, 0, 0, 0, 0, 3346, 2901, 4096, 303, 99, 1080, 4017, 7, 0, 4096, 0, 1373, 4086, 1, 0, 0, 0, 3219, 0, 3166, 4073, 0, 4095, 74, 83, 24, 0, 2380, 686, 0, 262, 29, 0, 0, 14, 0, 57, 4079, 3996, 2, 3928, 4096, 0, 74, 5, 0, 2357, 0, 3611, 4, 0, 0, 2, 5, 4024, 1707, 633, 20, 1088, 0, 0, 104, 2523, 3936, 1576, 2, 0, 4077, 0, 72, 116, 0, 0, 4096, 0, 0, 0, 679, 0, 0, 84, 0, 571, 339, 1, 695, 4096, 0, 0, 0, 3, 3716, 300, 30, 0, 0, 0, 0, 0, 0, 248, 3256, 1, 4094, 0, 0, 3734, 0, 937, 3547, 4085, 4060, 534, 10, 0, 0, 0, 0, 3432, 4, 0, 2381, 61] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 1203, 12, 0, 0, 0, 2684, 4096, 138, 4090, 0, 0, 329, 0, 5, 0, 1, 1105, 0, 0, 4077, 0, 511, 3519, 32, 3786, 0, 3, 0, 59, 0, 4093, 4096, 65, 3, 0, 46, 0, 41, 2898, 4096, 3960, 645, 3765, 0, 0, 5, 4035, 0, 2498, 0, 4092, 4096, 0, 0, 1212, 82, 0, 0, 8, 583, 4094, 1, 0, 4090, 3524, 0, 164, 4052, 40, 4093, 4053, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 16, 0, 5, 0, 2958, 3, 57, 3588, 494, 0, 186, 15, 0, 4062, 0, 2, 132, 0, 0, 1329, 1884, 468, 4006, 0, 13, 0, 1832, 4096, 0, 0, 19, 4095, 4092, 0, 524, 0, 0, 0, 39, 4068, 0, 0, 2895, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 32, 3, 0, 0, 0, 0, 4096, 3985, 4014, 0, 0, 272, 3779, 533, 6, 0, 0, 0, 3159, 0, 0, 1, 0, 2150, 0, 4080, 2, 0, 83, 1082, 0, 0, 4079, 1018, 0, 173, 0, 73, 4096, 7, 0, 3725, 0, 4082, 0, 0, 1165, 4095, 0, 1831, 0, 3480, 2840, 0, 8, 22, 0, 0, 5, 4096, 4090, 4096, 0, 147, 0, 4094, 0, 0, 4047, 0, 0, 4095, 63, 0, 0, 118, 6, 0, 0, 0, 371, 0, 10, 2, 0, 0, 1422, 0, 2164, 4056, 3, 0, 13, 0, 3063, 4014, 648, 0, 1819, 4096, 0, 23, 0, 0, 2480, 0, 26, 0, 4094, 4040, 11, 0, 0, 3930, 3980, 0, 0, 0, 356, 0, 0, 0, 3715, 233, 3675, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 3, 3939, 3398, 0, 4092, 0, 4054, 3511, 13, 2529, 0, 0, 10, 0, 286, 136, 0, 3992, 3645, 0, 4096, 0, 4096, 0, 87, 4058, 4072, 1244, 15, 0, 526, 3858, 0, 1131, 0, 4068, 3588, 0, 2801, 3915, 0, 232, 3685, 1, 9, 949, 1778, 4, 1, 0, 2464, 0, 0, 1067, 3625, 2, 1342, 0, 3851, 0, 4088, 4, 0, 4008, 0, 3877, 4094, 424, 0, 1901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 42, 3212, 0, 0, 1378, 134, 0, 859, 0, 1, 0, 157, 4, 0, 2, 3, 4096, 49, 4083, 0, 0, 4080, 0, 0, 0, 4093, 0, 28, 0, 55, 0, 4, 22, 0, 0, 0, 7, 58] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 164, 45, 0, 4071, 4096, 1026, 0, 1168, 0, 0, 0, 3254, 409, 0, 0, 4045, 101, 9, 480, 5, 4000, 0, 1, 0, 0, 547, 4092, 4079, 0, 1597, 0, 141, 0, 0, 4096, 2, 0, 0, 0, 17, 0, 318, 4095, 84, 3843, 0, 4, 3947, 0, 3679, 0, 2761, 5, 0, 867, 42, 361, 0, 378, 0, 2099, 268, 4094, 0, 800, 3751, 3302, 10, 0, 11, 2, 7, 4084, 4, 0, 3660, 806, 0, 0, 0, 0, 0, 3, 129, 0, 0, 0, 738, 0, 2, 4068, 17, 4065, 0, 0, 3542, 317, 0, 0, 0, 2810, 0, 0, 0, 0, 6, 0, 3578, 2, 4093, 3961, 0, 0, 144, 4095, 4096, 2721, 0, 8, 1, 2, 199, 3569, 4082, 0, 4094, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 16, 0, 0, 3970, 0, 0, 0, 4096, 0, 0, 838, 0, 0, 0, 793, 0, 0, 0, 0, 265, 3761, 0, 0, 4095, 0, 0, 4096, 0, 1115, 8, 4091, 0, 4077, 0, 0, 0, 7, 2, 0, 4020, 0, 4006, 4086, 3, 4071, 3390, 5, 0, 106, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 4096, 138, 17, 94, 1, 3921, 4, 4095, 339, 1636, 45, 0, 3143, 0, 10, 0, 1, 91, 0, 0, 0, 2681, 0, 3833, 0, 6, 0, 3, 4096, 0, 167, 2807, 0, 166, 2088, 51, 0, 3375, 2713, 0, 2288, 36, 141, 0, 0, 0, 697, 0, 4042, 0, 4096, 4094, 0, 0, 1855, 4072, 3955, 1336, 230, 0, 0, 2, 0, 3772, 230, 0, 4088, 1506] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 70, 3962, 68, 4096, 0, 2872, 3169, 2, 220, 0, 0, 0, 0, 0, 4004, 4, 1, 0, 0, 0, 0, 3675, 648, 4092, 4096, 461, 0, 1040, 11, 370, 1, 88, 1, 0, 3860, 3769, 2, 45, 4030, 3985, 3968, 0, 3978, 0, 0, 1038, 3367, 3780, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 507, 1, 941, 105, 4003, 0, 3633, 0, 4074, 3691, 0, 4024, 4033, 13, 0, 346, 0, 0, 0, 0, 6, 0, 141, 0, 0, 0, 233, 23, 0, 285, 2741, 0, 104, 1620, 0, 0, 2546, 111, 3719, 0, 679, 0, 0, 0, 0, 4094, 0, 3998, 8, 10, 4096, 0, 0, 36, 4096, 4087, 136, 0, 0, 0, 23, 0, 0, 4082, 426, 3626, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 74, 0, 2, 3507, 3928, 0, 73, 1858, 0, 14, 2828, 454, 0, 1592, 77, 0, 243, 0, 0, 48, 213, 0, 0, 4092, 289, 0, 4096, 0, 2, 3, 4096, 0, 4, 1917, 0, 292, 0, 2, 0, 0, 0, 3961, 4070, 56, 4091, 1676, 12, 0, 205, 4, 0, 0, 4096, 0, 139, 0, 65, 10, 0, 4, 1, 8, 4092, 2, 0, 3823, 4065, 0, 0, 4096, 0, 3496, 651, 2477, 0, 34, 0, 1081, 0, 205, 233, 0, 4092, 0, 11, 1079, 12, 23, 36, 0, 0, 0, 3503, 4096, 1476, 0, 0, 2858, 0, 2710, 4096, 4003, 0, 622, 353, 4053, 0, 1421, 0, 3947, 0, 0, 0, 0, 3681, 2934, 14, 3760, 0, 0, 0, 0, 4071, 1673, 0, 4089, 102] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[474, 0, 0, 0, 0, 3346, 2901, 4096, 303, 99, 1080, 4017, 7, 0, 4096, 0, 361, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1373, 4086, 1, 0, 0, 0, 3219, 0, 3166, 4073, 0, 4095, 74, 83, 24, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2380, 686, 0, 262, 29, 0, 0, 14, 0, 57, 4079, 0, 2, 3460, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 5, 0, 2357, 0, 3611, 4, 0, 0, 2, 5, 4024, 1707, 633, 20, 1088, 733, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 1341, 3936, 1576, 2, 0, 4077, 0, 72, 116, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 679, 0, 0, 84, 0, 571, 339, 1, 695, 4096, 0, 0, 0, 3, 3996, 3040, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3716, 300, 30, 0, 0, 0, 0, 0, 0, 248, 3256, 1, 4094, 0, 0, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 937, 3547, 3479, 4060, 534, 10, 0, 0, 0, 0, 3432, 4, 0, 2381, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21117, 20301, 15065, 14532, 15320, 13953, 12339, 18445] avg=16384.0000 max=21117 max_rank=0 min=12339 min_rank=6 imbalance_ratio=1.288879 max_load_reduction=109955 max_load_reduction_pct=0.838890 moved_to_echo=60057 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1203, 12, 0, 0, 0, 2684, 4096, 138, 4090, 0, 0, 329, 0, 346, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 1105, 0, 0, 4077, 0, 511, 3519, 32, 3786, 0, 3, 0, 59, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 65, 3, 0, 46, 0, 41, 2898, 4096, 0, 645, 3316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4035, 0, 2498, 0, 4092, 4096, 0, 0, 1212, 82, 0, 0, 8, 583, 4094, 1024, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 1873, 0, 164, 4052, 40, 4093, 4053, 0, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 5, 0, 2958, 3, 57, 3588, 494, 0, 186, 15, 3960, 3335, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 2, 132, 0, 0, 1329, 1884, 468, 4006, 0, 13, 0, 1832, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 3488, 4092, 0, 524, 0, 0, 0, 39, 4068, 0, 0, 2895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13236, 13201, 19299, 21998, 18480, 15244, 14489, 15125] avg=16384.0000 max=21998 max_rank=3 min=13201 min_rank=1 imbalance_ratio=1.342651 max_load_reduction=109074 max_load_reduction_pct=0.832169 moved_to_echo=63338 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 3, 3939, 3398, 0, 4092, 0, 4054, 3511, 13, 257, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2529, 0, 0, 10, 0, 286, 136, 0, 3992, 3645, 0, 4096, 0, 4096, 0, 87, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 4072, 1244, 15, 0, 526, 3858, 0, 1131, 0, 4068, 0, 0, 2468, 3915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 3685, 1, 9, 949, 1778, 4, 1, 0, 2464, 0, 0, 1067, 3625, 2, 1342, 1188, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 0, 2172, 4, 0, 4008, 0, 3877, 4094, 424, 0, 1901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 42, 3212, 0, 0, 1378, 134, 3588, 3322, 728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 859, 0, 1, 0, 157, 4, 0, 2, 3, 4096, 49, 4083, 0, 0, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3486, 0, 28, 0, 55, 0, 4, 22, 0, 0, 0, 7, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23701, 18953, 25355, 16616, 20331, 12444, 10012, 3660] avg=16384.0000 max=25355 max_rank=2 min=3660 min_rank=7 imbalance_ratio=1.547546 max_load_reduction=105717 max_load_reduction_pct=0.806557 moved_to_echo=46447 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 16, 0, 0, 3970, 0, 0, 0, 4096, 0, 0, 838, 0, 0, 0, 793, 378, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 3761, 0, 0, 4095, 0, 0, 4096, 0, 1115, 8, 4091, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 7, 2, 0, 4020, 0, 4006, 0, 3, 3578, 3390, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 4096, 138, 17, 94, 1194, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3921, 4, 2168, 339, 1636, 45, 0, 3143, 0, 10, 0, 1, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2681, 0, 3833, 0, 6, 0, 3, 4096, 0, 167, 2807, 0, 166, 2088, 51, 4086, 3337, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3375, 2713, 0, 2288, 36, 141, 0, 0, 0, 697, 0, 4042, 0, 4096, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1855, 3460, 3955, 1336, 230, 0, 0, 2, 0, 3772, 230, 0, 4088, 1506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10431, 17546, 19088, 10015, 11359, 24054, 18145, 20434] avg=16384.0000 max=24054 max_rank=5 min=10015 min_rank=3 imbalance_ratio=1.468140 max_load_reduction=107018 max_load_reduction_pct=0.816483 moved_to_echo=73992 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 164, 45, 0, 4071, 4096, 1026, 0, 1168, 0, 0, 0, 3254, 409, 0, 0, 353, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 101, 9, 480, 5, 4000, 0, 1, 0, 0, 547, 4092, 4079, 0, 1597, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 4096, 2, 0, 0, 0, 17, 0, 318, 0, 84, 3385, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 0, 3679, 0, 2761, 5, 0, 867, 42, 361, 0, 378, 0, 2099, 268, 4094, 959, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 800, 3751, 1755, 10, 0, 11, 2, 7, 4084, 4, 0, 3660, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 129, 0, 0, 0, 738, 0, 2, 4068, 17, 4065, 0, 0, 4095, 3225, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 317, 0, 0, 0, 2810, 0, 0, 0, 0, 6, 0, 3578, 2, 4093, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 3488, 4096, 2721, 0, 8, 1, 2, 199, 3569, 4082, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14924, 19061, 8047, 19729, 14890, 16930, 15084, 22407] avg=16384.0000 max=22407 max_rank=7 min=8047 min_rank=2 imbalance_ratio=1.367615 max_load_reduction=108665 max_load_reduction_pct=0.829048 moved_to_echo=69311 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 32, 3, 0, 0, 0, 0, 4096, 3985, 4014, 0, 0, 272, 3779, 533, 375, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 3159, 0, 0, 1, 0, 2150, 0, 4080, 2, 0, 83, 1082, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 1018, 0, 173, 0, 73, 4096, 7, 0, 0, 0, 3595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1165, 4095, 0, 1831, 0, 3480, 2840, 0, 8, 22, 0, 0, 5, 4096, 4090, 4096, 1189, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 2176, 0, 0, 4047, 0, 0, 4095, 63, 0, 0, 118, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 10, 2, 0, 0, 1422, 0, 2164, 4056, 3, 0, 13, 0, 3725, 3289, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3063, 4014, 648, 0, 1819, 4096, 0, 23, 0, 0, 2480, 0, 26, 0, 4094, 751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 3348, 3980, 0, 0, 0, 356, 0, 0, 0, 3715, 233, 3675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17413, 10675, 13041, 27175, 10652, 15784, 21014, 15318] avg=16384.0000 max=27175 max_rank=3 min=10652 min_rank=4 imbalance_ratio=1.658630 max_load_reduction=103897 max_load_reduction_pct=0.792671 moved_to_echo=62768 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 74, 0, 2, 3507, 3928, 0, 73, 1858, 0, 14, 2828, 454, 0, 1592, 77, 376, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 48, 213, 0, 0, 4092, 289, 0, 4096, 0, 2, 3, 4096, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1917, 0, 292, 0, 2, 0, 0, 0, 3961, 0, 56, 3603, 1676, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 4, 0, 0, 4096, 0, 139, 0, 65, 10, 0, 4, 1, 8, 4092, 1181, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3823, 2160, 0, 0, 4096, 0, 3496, 651, 2477, 0, 34, 0, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 233, 0, 4092, 0, 11, 1079, 12, 23, 36, 0, 0, 0, 3503, 4096, 1476, 4070, 0, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2858, 0, 2710, 4096, 4003, 0, 622, 353, 4053, 0, 1421, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3135, 2934, 14, 3760, 0, 0, 0, 0, 4071, 1673, 0, 4089, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15087, 13194, 11523, 10047, 17820, 19560, 24063, 19778] avg=16384.0000 max=24063 max_rank=6 min=10047 min_rank=3 imbalance_ratio=1.468689 max_load_reduction=107009 max_load_reduction_pct=0.816414 moved_to_echo=81221 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 70, 3962, 68, 4096, 0, 2872, 3169, 2, 220, 0, 0, 0, 0, 366, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 4, 1, 0, 0, 0, 0, 3675, 648, 4092, 4096, 461, 0, 1040, 11, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 1, 88, 1, 0, 3860, 3769, 2, 45, 4030, 3985, 0, 0, 3503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 3367, 3780, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 507, 1, 941, 1055, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 4003, 0, 1931, 0, 4074, 3691, 0, 4024, 4033, 13, 0, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 141, 0, 0, 0, 233, 23, 0, 285, 2741, 0, 104, 1620, 0, 3968, 3335, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2546, 111, 3719, 0, 679, 0, 0, 0, 0, 4094, 0, 3998, 8, 10, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 3489, 4087, 136, 0, 0, 0, 23, 0, 0, 4082, 426, 3626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15163, 18141, 19654, 10960, 22220, 13103, 15926, 15905] avg=16384.0000 max=22220 max_rank=4 min=10960 min_rank=3 imbalance_ratio=1.356201 max_load_reduction=108852 max_load_reduction_pct=0.830475 moved_to_echo=67154 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[8064, 8448, 128, 1664, 3584, 8320, 7552, 128, 19584, 14336, 4736, 32512, 4736, 5376, 2816, 9472, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 1024, 5632, 27392, 27264, 4864, 4608, 128, 384, 128, 384, 18944, 13824, 768, 24960, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[512, 256, 128, 1280, 19712, 11520, 8064, 4224, 21120, 14848, 5248, 16128, 3840, 4736, 13312, 1536, 2816, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[6528, 15616, 7552, 6784, 3712, 21248, 7040, 1024, 128, 4224, 128, 4480, 6912, 11136, 4992, 19968, 8576, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[10368, 15488, 6400, 3840, 7040, 11904, 4224, 1408, 2560, 1152, 22784, 128, 21376, 128, 18176, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 2944, 1152, 8192, 256, 128, 1152, 896, 9600, 128, 3456, 24576, 640, 7936, 9472, 1792, 31488, 22912, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[7040, 13056, 11520, 5504, 384, 4608, 7680, 128, 9472, 7040, 24576, 0, 896, 27008, 13184, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 12800, 11776, 15616, 4352, 7552, 19968, 128, 22784, 21120, 3072, 128, 6016, 1024, 5248, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 54, 496, 0, 0, 0, 4096, 24, 0, 136, 2804, 1, 4073, 3666, 35, 0, 0, 0, 2210, 0, 70, 4084, 4096, 4, 0, 0, 476, 3346, 4095, 15, 0, 116, 0, 701, 3671, 572, 0, 1, 0, 11, 0, 4002, 0, 12, 0, 4082, 3791, 152, 0, 208, 1, 0, 282, 4, 0, 0, 0, 0, 0, 28, 149, 0, 45, 3029, 0, 0, 0, 672, 4024, 419, 2407, 0, 0, 1, 0, 1, 814, 13, 0, 0, 0, 2046, 0, 0, 60, 4082, 0, 0, 4049, 0, 2947, 3887, 3494, 0, 972, 1902, 2438, 4067, 4074, 4091, 0, 4044, 454, 1181, 68, 0, 2, 393, 0, 1, 5, 0, 4089, 0, 4, 0, 4092, 3228, 4089, 0, 1, 4096, 0, 3956, 0, 0, 271, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 3787, 4095, 49, 0, 7, 234, 0, 3525, 4072, 0, 554, 0, 0, 135, 0, 4088, 35, 0, 25, 0, 1712, 0, 3, 4096, 29, 4090, 1, 3481, 4028, 416, 488, 0, 2, 0, 21, 284, 2042, 0, 83, 44, 0, 756, 0, 3909, 3700, 9, 0, 0, 4074, 0, 0, 0, 91, 4087, 0, 0, 19, 0, 3112, 1646, 4060, 0, 0, 0, 186, 4096, 637, 0, 5, 4096, 0, 30, 3, 7, 0, 0, 4096, 0, 0, 2939, 0, 7, 17, 2776, 0, 0, 3997, 3601, 1, 24, 4096, 19, 532, 4035, 0, 4002, 4054, 3167, 100, 0, 0, 0, 84, 0, 4076, 0, 0, 449, 2, 0, 1, 0, 0, 4096, 0, 0, 4061, 0, 707, 90, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 3885, 3594, 477, 11, 0, 865, 0, 4094, 0, 4, 1, 10, 0, 5, 463, 3641, 144, 4055, 960, 39, 0, 63, 201, 977, 4089, 3520, 0, 0, 4096, 2205, 0, 20, 0, 638, 0, 3897, 163, 0, 0, 99, 0, 2342, 0, 4046, 174, 1, 50, 0, 0, 3768, 0, 0, 0, 4093, 0, 29, 0, 16, 1316, 278, 4076, 0, 0, 0, 4066, 0, 0, 0, 1405, 3089, 185, 1, 0, 0, 1, 0, 3254, 4019, 0, 10, 0, 29, 1, 0, 1295, 0, 3169, 4096, 0, 1974, 4045, 0, 4039, 3056, 4079, 0, 3966, 0, 86, 0, 0, 0, 0, 0, 4051, 0, 0, 4074, 0, 3850, 3, 1, 13, 4090, 0, 0, 0, 9, 0, 36, 4075, 5, 2593] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[76, 0, 2556, 0, 4092, 0, 4032, 12, 0, 1389, 0, 4004, 4096, 2775, 0, 0, 0, 0, 37, 0, 4087, 20, 5, 158, 0, 0, 3842, 0, 3548, 0, 3346, 2506, 0, 4096, 3881, 0, 0, 0, 0, 0, 4096, 4057, 0, 0, 0, 1407, 747, 0, 0, 0, 0, 180, 0, 4096, 0, 0, 0, 0, 3220, 3193, 0, 0, 1, 0, 205, 3922, 23, 0, 110, 0, 0, 2931, 0, 36, 380, 294, 19, 0, 0, 19, 0, 1574, 0, 153, 0, 2265, 0, 1057, 4091, 3730, 0, 0, 4077, 0, 291, 1951, 0, 0, 1276, 3985, 0, 3929, 1653, 0, 0, 0, 4072, 2, 89, 0, 497, 1, 3546, 0, 0, 0, 16, 4096, 4096, 5, 0, 0, 0, 3044, 47, 0, 0, 4035] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[3842, 0, 9, 0, 68, 0, 4071, 0, 0, 0, 0, 0, 4025, 4094, 2, 0, 2, 0, 856, 4093, 2317, 0, 1690, 0, 9, 0, 1276, 75, 4025, 1, 0, 4092, 4096, 109, 4026, 0, 0, 0, 0, 0, 260, 4095, 10, 0, 0, 0, 3760, 0, 486, 4072, 0, 0, 0, 3899, 4095, 0, 0, 3900, 0, 0, 977, 0, 0, 4096, 458, 0, 0, 76, 0, 499, 4084, 0, 12, 0, 1, 34, 0, 321, 0, 3596, 3753, 4078, 0, 0, 8, 0, 0, 0, 4091, 3, 0, 2602, 3935, 0, 12, 4088, 0, 4096, 4096, 0, 4071, 55, 2175, 4, 0, 0, 0, 0, 0, 0, 20, 0, 1, 163, 19, 4085, 4087, 4096, 1, 0, 0, 0, 0, 24, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[6, 16, 4095, 4025, 0, 0, 4096, 0, 0, 0, 0, 4067, 4082, 4096, 9, 99, 0, 0, 4070, 0, 4038, 0, 10, 0, 23, 0, 3722, 4086, 0, 2, 2, 420, 4096, 0, 3597, 0, 0, 327, 0, 0, 3837, 1482, 5, 0, 0, 24, 3191, 0, 0, 36, 0, 0, 0, 0, 116, 624, 0, 2448, 2698, 0, 2052, 0, 0, 3800, 0, 3896, 5, 4, 0, 40, 170, 0, 16, 0, 385, 0, 0, 432, 0, 132, 0, 0, 0, 0, 0, 818, 2101, 0, 1, 5, 0, 0, 3639, 0, 1767, 0, 0, 3806, 3817, 2893, 4096, 4096, 1261, 1384, 0, 0, 0, 130, 240, 7, 3086, 0, 2522, 482, 0, 2, 4093, 4096, 4060, 0, 1282, 2143, 0, 3926, 0, 0, 942, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 165, 2583, 0, 0, 4, 3973, 4029, 10, 18, 0, 1, 3437, 4096, 672, 11, 0, 0, 4076, 0, 4096, 158, 0, 0, 0, 0, 6, 288, 4096, 0, 0, 0, 3639, 3839, 4088, 0, 2, 0, 35, 0, 506, 3853, 832, 7, 4062, 0, 3229, 0, 3392, 0, 124, 982, 3929, 3630, 0, 17, 0, 0, 1724, 1117, 0, 0, 0, 4045, 275, 2711, 1314, 0, 44, 0, 2825, 0, 28, 64, 4096, 0, 942, 0, 2911, 2, 0, 1093, 0, 372, 0, 4024, 0, 0, 4096, 1368, 0, 4083, 3970, 0, 11, 301, 0, 0, 3990, 682, 4096, 0, 3933, 0, 21, 0, 0, 0, 17, 114, 0, 0, 0, 6, 0, 0, 0, 0, 4093, 0, 447, 4094, 3, 0, 0, 144, 131, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 1, 24, 4071, 3873, 1547, 0, 2, 2946, 1, 1700, 4096, 4015, 0, 0, 0, 0, 4053, 0, 4096, 198, 0, 83, 0, 0, 935, 0, 1336, 6, 5, 89, 0, 3269, 4024, 0, 45, 0, 0, 0, 38, 4060, 0, 0, 601, 2657, 141, 0, 0, 3355, 0, 0, 4, 3995, 0, 0, 4087, 20, 4090, 2992, 0, 8, 1644, 3988, 63, 0, 5, 0, 3876, 0, 3701, 12, 620, 3, 892, 0, 0, 0, 0, 0, 1, 453, 322, 0, 0, 0, 0, 0, 4096, 0, 0, 3983, 4059, 66, 40, 4060, 22, 427, 4091, 4096, 0, 4096, 4, 0, 0, 2, 0, 0, 137, 8, 10, 3252, 4094, 331, 0, 0, 0, 517, 4094, 0, 0, 3272, 13, 15, 0, 12, 141, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 3787, 4095, 49, 0, 7, 234, 0, 983, 4072, 0, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 4088, 35, 0, 25, 0, 1712, 0, 3, 4096, 29, 4090, 1, 259, 188, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3481, 4028, 398, 488, 0, 2, 0, 21, 284, 2042, 0, 83, 44, 0, 756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 3700, 9, 0, 0, 4074, 0, 0, 0, 91, 4087, 0, 0, 19, 0, 3112, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1646, 4060, 0, 0, 0, 186, 4096, 637, 0, 5, 4096, 0, 30, 3, 7, 0, 2542, 2601, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2939, 0, 7, 17, 2776, 0, 0, 3997, 1000, 1, 24, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 532, 2202, 0, 4002, 4054, 3167, 100, 0, 0, 0, 84, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 2, 0, 1, 0, 0, 3908, 0, 0, 4061, 0, 707, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17875, 14679, 11627, 20093, 20391, 18953, 18236, 9218] avg=16384.0000 max=20391 max_rank=4 min=9218 min_rank=7 imbalance_ratio=1.244568 max_load_reduction=110681 max_load_reduction_pct=0.844429 moved_to_echo=66798 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 54, 496, 0, 0, 0, 4096, 24, 0, 136, 2804, 1, 1135, 3666, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2210, 0, 70, 4084, 4096, 4, 0, 0, 476, 3346, 4095, 15, 0, 116, 262, 188, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 701, 3508, 572, 0, 1, 0, 11, 0, 4002, 0, 12, 0, 4082, 3791, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 1, 0, 282, 4, 0, 0, 0, 0, 0, 28, 149, 0, 45, 3029, 1103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 4024, 419, 2407, 0, 0, 1, 0, 1, 814, 13, 0, 0, 2938, 2524, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2046, 0, 0, 60, 4082, 0, 0, 4049, 0, 2947, 3887, 970, 0, 972, 1902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2438, 4067, 2223, 4091, 0, 4044, 454, 1181, 68, 0, 2, 393, 0, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4, 0, 4092, 3228, 3901, 0, 1, 4096, 0, 3956, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12447, 19125, 16832, 4849, 14299, 20915, 18967, 23638] avg=16384.0000 max=23638 max_rank=7 min=4849 min_rank=3 imbalance_ratio=1.442749 max_load_reduction=107434 max_load_reduction_pct=0.819656 moved_to_echo=77819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[76, 0, 2556, 0, 4092, 0, 4032, 12, 0, 1389, 0, 4004, 1142, 2775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 4087, 20, 5, 158, 0, 0, 3842, 0, 3548, 0, 3346, 2506, 82, 188, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3709, 0, 0, 0, 0, 0, 4096, 4057, 0, 0, 0, 1407, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 4096, 0, 0, 0, 0, 3220, 3193, 0, 0, 1, 0, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 3922, 23, 0, 110, 0, 0, 2931, 0, 36, 380, 294, 19, 0, 0, 19, 2954, 2945, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1574, 0, 153, 0, 2265, 0, 1057, 4091, 3730, 0, 0, 1132, 0, 291, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 697, 3985, 0, 3929, 1653, 0, 0, 0, 4072, 2, 89, 0, 497, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3546, 0, 0, 0, 16, 4096, 3908, 5, 0, 0, 0, 3044, 47, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20078, 17991, 18112, 11035, 13990, 16244, 14925, 18697] avg=16384.0000 max=20078 max_rank=0 min=11035 min_rank=3 imbalance_ratio=1.225464 max_load_reduction=110994 max_load_reduction_pct=0.846817 moved_to_echo=63856 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3842, 0, 9, 0, 68, 0, 4071, 0, 0, 0, 0, 0, 1117, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 856, 4093, 2317, 0, 1690, 0, 9, 0, 1276, 75, 4025, 1, 0, 4092, 267, 1, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 109, 3844, 0, 0, 0, 0, 0, 260, 4095, 10, 0, 0, 0, 3760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 486, 4072, 0, 0, 0, 3899, 4095, 0, 0, 3900, 0, 0, 977, 0, 0, 4096, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 458, 0, 0, 76, 0, 499, 4084, 0, 12, 0, 1, 34, 0, 321, 0, 3596, 2908, 2846, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3753, 4078, 0, 0, 8, 0, 0, 0, 4091, 3, 0, 2602, 1089, 0, 12, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2223, 0, 4071, 55, 2175, 4, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 163, 19, 4085, 4087, 4096, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13203, 18886, 16174, 22638, 15328, 19724, 12644, 12475] avg=16384.0000 max=22638 max_rank=3 min=12475 min_rank=7 imbalance_ratio=1.381714 max_load_reduction=108434 max_load_reduction_pct=0.827286 moved_to_echo=60171 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 165, 2583, 0, 0, 4, 3973, 4029, 10, 18, 0, 1, 958, 4096, 672, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 4096, 158, 0, 0, 0, 0, 6, 288, 4096, 0, 0, 0, 256, 188, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3639, 3839, 3906, 0, 2, 0, 35, 0, 506, 3853, 832, 7, 4062, 0, 3229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3392, 0, 124, 982, 3929, 3630, 0, 17, 0, 0, 1724, 1117, 0, 0, 0, 4045, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 2711, 1314, 0, 44, 0, 2825, 0, 28, 64, 4096, 0, 942, 0, 2911, 2, 2479, 2867, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 0, 372, 0, 4024, 0, 0, 4096, 1368, 0, 4083, 1103, 0, 11, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2178, 682, 4096, 0, 3933, 0, 21, 0, 0, 0, 17, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 3905, 0, 447, 4094, 3, 0, 0, 144, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16520, 13346, 23910, 20040, 21034, 16451, 11041, 8730] avg=16384.0000 max=23910 max_rank=2 min=8730 min_rank=7 imbalance_ratio=1.459351 max_load_reduction=107162 max_load_reduction_pct=0.817581 moved_to_echo=57256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 0, 3885, 3594, 477, 11, 0, 865, 0, 1141, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 5, 463, 3641, 144, 4055, 960, 39, 0, 63, 201, 977, 4089, 3520, 0, 196, 188, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2107, 0, 20, 0, 638, 0, 3897, 163, 0, 0, 99, 0, 2342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 174, 1, 50, 0, 0, 3768, 0, 0, 0, 4093, 0, 29, 0, 16, 1316, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 278, 4076, 0, 0, 0, 4066, 0, 0, 0, 1405, 3089, 185, 1, 0, 0, 1, 2953, 2958, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3254, 4019, 0, 10, 0, 29, 1, 0, 1295, 0, 3169, 1138, 0, 1974, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 1668, 4079, 0, 3966, 0, 86, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 3850, 3, 1, 13, 3902, 0, 0, 0, 9, 0, 36, 4075, 5, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9980, 18649, 13362, 14320, 19377, 18934, 17889, 18561] avg=16384.0000 max=19377 max_rank=4 min=9980 min_rank=0 imbalance_ratio=1.182678 max_load_reduction=111695 max_load_reduction_pct=0.852165 moved_to_echo=74761 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[6, 16, 4095, 4025, 0, 0, 4096, 0, 0, 0, 0, 4067, 1138, 4096, 9, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 4038, 0, 10, 0, 23, 0, 3722, 4086, 0, 2, 2, 420, 245, 187, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3437, 0, 0, 327, 0, 0, 3837, 1482, 5, 0, 0, 24, 3191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 116, 624, 0, 2448, 2698, 0, 2052, 0, 0, 3800, 1033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 5, 4, 0, 40, 170, 0, 16, 0, 385, 0, 0, 432, 0, 132, 2944, 2628, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 818, 2101, 0, 1, 5, 0, 0, 1011, 0, 1767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3806, 2084, 2893, 4096, 4096, 1261, 1384, 0, 0, 0, 130, 240, 7, 3086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2522, 482, 0, 2, 4093, 4096, 3873, 0, 1282, 2143, 0, 3926, 0, 0, 942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21647, 16965, 16399, 12807, 11107, 5703, 23083, 23361] avg=16384.0000 max=23361 max_rank=7 min=5703 min_rank=5 imbalance_ratio=1.425842 max_load_reduction=107711 max_load_reduction_pct=0.821770 moved_to_echo=63254 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 24, 4071, 3873, 1547, 0, 2, 2946, 1, 1700, 1142, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 4096, 198, 0, 83, 0, 0, 935, 0, 1336, 6, 5, 89, 263, 188, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3269, 3845, 0, 45, 0, 0, 0, 38, 4060, 0, 0, 601, 2657, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3355, 0, 0, 4, 3995, 0, 0, 4087, 20, 4090, 2992, 0, 8, 1644, 3988, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 5, 0, 3876, 0, 3701, 12, 620, 3, 892, 0, 0, 0, 0, 0, 2954, 2932, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 453, 322, 0, 0, 0, 0, 0, 4096, 0, 0, 3983, 1127, 66, 40, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 427, 2233, 4096, 0, 4096, 4, 0, 0, 2, 0, 0, 137, 8, 10, 3252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 331, 0, 0, 0, 517, 3906, 0, 0, 3272, 13, 15, 0, 12, 141, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19322, 11431, 14656, 25290, 15546, 14148, 14287, 16392] avg=16384.0000 max=25290 max_rank=3 min=11431 min_rank=1 imbalance_ratio=1.543579 max_load_reduction=105782 max_load_reduction_pct=0.807053 moved_to_echo=60373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 15488, 4608, 26496, 4736, 9856, 1280, 128, 1792, 10368, 8064, 22272, 4224, 11008, 7296, 1920, 1408, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[15360, 20224, 24832, 1152, 128, 384, 768, 128, 12928, 23808, 896, 128, 4864, 8192, 18048, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[3840, 16640, 4352, 640, 3072, 11264, 2176, 1152, 23296, 6528, 3072, 21760, 8576, 128, 5120, 20480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2560, 17024, 15616, 19840, 16384, 24320, 12672, 2816, 128, 128, 4096, 640, 512, 8320, 3712, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[11904, 11648, 256, 1280, 4224, 19712, 8064, 768, 4096, 6528, 19968, 7424, 3328, 128, 1792, 23424, 7808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2944, 18688, 1408, 768, 8064, 5248, 17408, 3584, 768, 1536, 13056, 640, 1920, 896, 2944, 3840, 22784, 22400, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3968, 256, 9856, 4096, 12416, 11648, 29568, 4608, 128, 4608, 3968, 9856, 8832, 26880, 768, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[18816, 1024, 3968, 4096, 12416, 16128, 27392, 128, 1792, 17792, 128, 11776, 256, 4352, 1536, 10752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True + [2026-08-30 05:03:31] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 41130.7 | throughput per GPU (TFLOP/s/GPU): 8.1 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102490E+01 | loss scale: 1.0 | grad norm: 44.116 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 23.22 + +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 64003.4423828125 | max allocated: 113094.310546875 | reserved: 114976.0 | max reserved: 114976.0 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[167, 3514, 693, 277, 1528, 314, 632, 1345, 984, 288, 2131, 255, 651, 823, 187, 337, 140, 362, 748, 913, 1014, 205, 2656, 2092, 650, 1357, 959, 951, 751, 214, 1488, 1571, 2399, 437, 422, 602, 89, 1876, 1783, 255, 2524, 546, 1578, 991, 1120, 1514, 852, 650, 272, 453, 440, 2003, 844, 684, 522, 96, 488, 309, 952, 1739, 1154, 676, 573, 1503, 630, 1703, 763, 941, 1118, 193, 962, 2287, 1221, 2145, 157, 1798, 795, 961, 2142, 754, 2338, 1016, 2379, 821, 1884, 561, 431, 982, 1740, 244, 840, 864, 776, 440, 368, 788, 2188, 204, 1532, 567, 1640, 802, 2100, 2248, 837, 1718, 569, 2038, 668, 427, 2608, 1000, 293, 856, 1106, 2011, 201, 617, 1023, 1333, 853, 1191, 1585, 678, 1491, 443, 316, 344] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[87, 2600, 678, 196, 2087, 629, 888, 1309, 990, 134, 1032, 386, 964, 709, 143, 426, 147, 427, 1251, 1079, 862, 195, 2271, 1496, 374, 1230, 1122, 1059, 841, 276, 1215, 1592, 3310, 551, 247, 698, 99, 2191, 2336, 560, 2064, 194, 1687, 796, 1193, 1056, 1161, 1383, 387, 169, 332, 2314, 737, 188, 631, 128, 647, 173, 953, 1568, 1528, 580, 602, 1209, 846, 1762, 342, 1188, 2203, 281, 924, 1631, 1332, 2059, 249, 1838, 978, 829, 2527, 930, 2333, 1168, 2854, 240, 1705, 155, 1179, 1091, 2151, 242, 934, 1035, 834, 354, 572, 665, 1983, 337, 1975, 444, 1680, 791, 1607, 1862, 631, 736, 548, 2561, 551, 182, 1970, 1118, 281, 1257, 1128, 1061, 281, 901, 669, 1283, 875, 1501, 1304, 424, 1608, 448, 892, 385] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[42, 2583, 305, 168, 1499, 251, 938, 757, 1067, 153, 2009, 211, 459, 819, 197, 232, 64, 391, 1169, 437, 1176, 228, 2890, 1954, 499, 1054, 1018, 785, 1618, 387, 2430, 2218, 3041, 427, 517, 345, 65, 2320, 2101, 324, 2697, 520, 1413, 695, 1415, 1854, 763, 506, 204, 248, 205, 1553, 903, 220, 313, 50, 350, 325, 1011, 2117, 1629, 318, 1747, 1126, 697, 1982, 633, 1309, 741, 203, 369, 1616, 747, 2012, 162, 1810, 610, 1331, 2202, 1099, 2814, 1065, 2509, 424, 1969, 376, 911, 1310, 1943, 181, 670, 907, 677, 691, 427, 996, 2651, 477, 1231, 729, 1279, 864, 1215, 2362, 885, 877, 602, 1443, 325, 336, 2473, 637, 329, 1375, 993, 1778, 169, 717, 1290, 1373, 814, 1552, 1434, 688, 1708, 443, 603, 697] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[74, 2791, 585, 257, 1165, 293, 737, 527, 958, 194, 1058, 116, 1179, 554, 114, 323, 106, 388, 632, 567, 1255, 95, 3591, 1788, 170, 1248, 987, 806, 742, 147, 1482, 2299, 3116, 943, 250, 487, 94, 2465, 2000, 714, 2762, 118, 978, 639, 1791, 1487, 1425, 900, 257, 228, 246, 2254, 445, 350, 295, 70, 687, 370, 890, 1669, 1358, 592, 526, 1401, 678, 2164, 328, 858, 1293, 92, 858, 1539, 1212, 2052, 183, 1490, 887, 984, 2838, 1011, 3399, 995, 2903, 380, 2183, 175, 347, 1139, 2154, 202, 876, 1631, 1173, 627, 409, 511, 2269, 431, 1999, 557, 1328, 643, 1647, 2067, 513, 947, 778, 2276, 872, 272, 2268, 823, 656, 1172, 1052, 900, 221, 554, 543, 1389, 850, 1483, 1437, 1067, 1564, 521, 844, 523] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[133, 2721, 488, 255, 1414, 551, 912, 1444, 1212, 134, 1537, 122, 694, 804, 226, 415, 166, 415, 535, 696, 1173, 278, 2743, 1437, 306, 1048, 1022, 931, 891, 216, 1378, 1762, 2834, 567, 280, 512, 161, 2369, 2262, 512, 2502, 337, 1115, 517, 1278, 1429, 897, 839, 208, 280, 287, 2503, 868, 595, 720, 78, 681, 366, 1541, 1655, 1189, 593, 698, 1483, 641, 2098, 776, 1655, 1224, 177, 624, 1828, 965, 1889, 263, 1631, 879, 733, 2089, 770, 2079, 1252, 2903, 599, 1776, 269, 837, 1046, 2254, 251, 825, 1268, 1077, 634, 662, 897, 2527, 297, 1876, 379, 1397, 900, 1902, 2594, 568, 721, 638, 1875, 693, 859, 2136, 958, 292, 716, 1055, 1075, 165, 723, 1215, 1542, 864, 1491, 1640, 537, 1387, 397, 318, 249] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[84, 2878, 415, 251, 1463, 173, 1107, 1067, 1117, 182, 1463, 143, 673, 677, 98, 325, 97, 791, 883, 683, 819, 95, 2637, 1453, 439, 1460, 800, 992, 1550, 416, 1374, 1889, 2792, 397, 618, 499, 57, 2583, 2104, 681, 2491, 303, 1218, 775, 1478, 1709, 968, 962, 291, 469, 202, 2287, 1178, 513, 575, 88, 280, 163, 959, 2168, 1218, 500, 686, 961, 698, 2056, 541, 1033, 912, 235, 569, 1691, 907, 2042, 156, 2209, 598, 1101, 2465, 1206, 2855, 1008, 2612, 390, 1752, 114, 604, 729, 2323, 119, 780, 1183, 559, 869, 562, 984, 2329, 202, 1904, 586, 1543, 428, 1506, 2737, 428, 1122, 532, 2095, 493, 222, 2405, 886, 227, 1416, 1701, 1250, 279, 445, 826, 1141, 748, 1700, 1478, 815, 1851, 456, 544, 248] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[99, 2426, 600, 311, 1332, 424, 785, 478, 781, 104, 1297, 188, 827, 779, 257, 333, 173, 452, 658, 916, 825, 396, 2737, 1534, 276, 806, 1273, 818, 994, 137, 1587, 2336, 3263, 565, 147, 422, 543, 2335, 2607, 339, 2525, 653, 1211, 827, 1127, 1481, 1012, 1612, 240, 105, 334, 1806, 628, 377, 659, 161, 1006, 345, 1298, 2277, 1368, 483, 550, 1011, 1016, 1630, 701, 892, 1267, 130, 957, 1710, 1159, 2698, 225, 2099, 967, 788, 2761, 830, 2484, 862, 2458, 536, 1675, 134, 684, 1204, 1873, 146, 483, 1024, 1098, 478, 363, 972, 1812, 221, 2298, 982, 1539, 786, 1169, 1539, 672, 823, 842, 2567, 612, 505, 2020, 1119, 385, 1183, 1400, 1379, 548, 578, 565, 1160, 992, 1443, 1619, 494, 1859, 736, 453, 212] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[134, 2637, 412, 345, 908, 313, 824, 589, 1076, 221, 1403, 349, 966, 521, 226, 298, 105, 451, 612, 802, 1033, 576, 2827, 1855, 386, 1435, 1349, 730, 1369, 233, 1524, 1842, 2979, 932, 359, 463, 148, 1976, 1841, 634, 2648, 484, 1233, 710, 1262, 1187, 1059, 744, 286, 498, 398, 1797, 535, 430, 526, 124, 333, 603, 882, 1734, 1571, 399, 716, 992, 419, 1721, 816, 1733, 1422, 135, 845, 1825, 975, 1810, 192, 1699, 599, 1141, 2709, 958, 2790, 1151, 2743, 417, 1559, 133, 1247, 1249, 2006, 308, 986, 1488, 1084, 619, 625, 406, 2315, 293, 1690, 1116, 1385, 584, 1099, 2632, 696, 768, 544, 2009, 639, 340, 1839, 1076, 292, 1137, 1465, 1623, 280, 448, 1350, 1274, 787, 1483, 1535, 1032, 1061, 541, 820, 275] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[87, 2600, 678, 196, 2087, 629, 888, 1309, 990, 134, 1032, 386, 964, 709, 143, 426, 3157, 1410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 427, 1251, 1079, 862, 195, 2271, 1496, 374, 1230, 1122, 1059, 841, 276, 1215, 1592, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 551, 247, 698, 99, 2191, 2336, 560, 2064, 194, 1687, 796, 1193, 1056, 1161, 1383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 169, 332, 2314, 737, 188, 631, 128, 647, 173, 953, 1568, 1528, 580, 602, 1209, 1983, 1009, 741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 846, 1762, 342, 1188, 2203, 281, 924, 1631, 1332, 2059, 249, 1838, 978, 829, 108, 930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2333, 1168, 1353, 240, 1705, 155, 1179, 1091, 2151, 242, 934, 1035, 834, 354, 572, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 1975, 444, 1680, 791, 1607, 1108, 631, 736, 548, 2561, 551, 182, 1970, 1118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 1257, 1128, 1061, 281, 901, 669, 1283, 875, 1501, 1304, 424, 1608, 448, 892, 385, 701, 760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17825, 15490, 16369, 15879, 17500, 16011, 16239, 15759] avg=16384.0000 max=17825 max_rank=0 min=15490 min_rank=1 imbalance_ratio=1.087952 max_load_reduction=113247 max_load_reduction_pct=0.864006 moved_to_echo=65509 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[167, 3514, 693, 277, 1528, 314, 632, 1345, 984, 288, 2131, 255, 651, 823, 187, 337, 2288, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 362, 748, 913, 1014, 205, 2656, 2092, 650, 1357, 959, 951, 751, 214, 1488, 1571, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 437, 422, 602, 89, 1876, 1783, 255, 2524, 546, 1578, 991, 1120, 1514, 852, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 453, 440, 2003, 844, 684, 522, 96, 488, 309, 952, 1739, 1154, 676, 573, 1503, 2188, 855, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 1703, 763, 941, 1118, 193, 962, 2287, 1221, 2145, 157, 1798, 795, 961, 92, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2338, 1016, 1128, 821, 1884, 561, 431, 982, 1740, 244, 840, 864, 776, 440, 368, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 1532, 567, 1640, 802, 2100, 1337, 837, 1718, 569, 2038, 668, 427, 2608, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 856, 1106, 2011, 201, 617, 1023, 1333, 853, 1191, 1585, 678, 1491, 443, 316, 344, 847, 634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17609, 16135, 15350, 16368, 16520, 15221, 18047, 15822] avg=16384.0000 max=18047 max_rank=6 min=15221 min_rank=5 imbalance_ratio=1.101501 max_load_reduction=113025 max_load_reduction_pct=0.862312 moved_to_echo=65610 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[74, 2791, 585, 257, 1165, 293, 737, 527, 958, 194, 1058, 116, 1179, 554, 114, 323, 2974, 1587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 388, 632, 567, 1255, 95, 3591, 1788, 170, 1248, 987, 806, 742, 147, 1482, 2299, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 943, 250, 487, 94, 2465, 2000, 714, 2762, 118, 978, 639, 1791, 1487, 1425, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 228, 246, 2254, 445, 350, 295, 70, 687, 370, 890, 1669, 1358, 592, 526, 1401, 2269, 1138, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 678, 2164, 328, 858, 1293, 92, 858, 1539, 1212, 2052, 183, 1490, 887, 984, 113, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3399, 995, 1368, 380, 2183, 175, 347, 1139, 2154, 202, 876, 1631, 1173, 627, 409, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 431, 1999, 557, 1328, 643, 1647, 1222, 513, 947, 778, 2276, 872, 272, 2268, 823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 656, 1172, 1052, 900, 221, 554, 543, 1389, 850, 1483, 1437, 1067, 1564, 521, 844, 523, 782, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15486, 16366, 17195, 15803, 15742, 17569, 16576, 16335] avg=16384.0000 max=17569 max_rank=5 min=15486 min_rank=0 imbalance_ratio=1.072327 max_load_reduction=113503 max_load_reduction_pct=0.865959 moved_to_echo=66222 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[42, 2583, 305, 168, 1499, 251, 938, 757, 1067, 153, 2009, 211, 459, 819, 197, 232, 2900, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 391, 1169, 437, 1176, 228, 2890, 1954, 499, 1054, 1018, 785, 1618, 387, 2430, 2218, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 427, 517, 345, 65, 2320, 2101, 324, 2697, 520, 1413, 695, 1415, 1854, 763, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 248, 205, 1553, 903, 220, 313, 50, 350, 325, 1011, 2117, 1629, 318, 1747, 1126, 2651, 879, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 697, 1982, 633, 1309, 741, 203, 369, 1616, 747, 2012, 162, 1810, 610, 1331, 94, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2814, 1065, 1190, 424, 1969, 376, 911, 1310, 1943, 181, 670, 907, 677, 691, 427, 996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 1231, 729, 1279, 864, 1215, 1404, 885, 877, 602, 1443, 325, 336, 2473, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 1375, 993, 1778, 169, 717, 1290, 1373, 814, 1552, 1434, 688, 1708, 443, 603, 697, 890, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15819, 18386, 16103, 16500, 15415, 16551, 14777, 17521] avg=16384.0000 max=18386 max_rank=1 min=14777 min_rank=6 imbalance_ratio=1.122192 max_load_reduction=112686 max_load_reduction_pct=0.859726 moved_to_echo=64264 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[133, 2721, 488, 255, 1414, 551, 912, 1444, 1212, 134, 1537, 122, 694, 804, 226, 415, 2703, 1165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 415, 535, 696, 1173, 278, 2743, 1437, 306, 1048, 1022, 931, 891, 216, 1378, 1762, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 567, 280, 512, 161, 2369, 2262, 512, 2502, 337, 1115, 517, 1278, 1429, 897, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 280, 287, 2503, 868, 595, 720, 78, 681, 366, 1541, 1655, 1189, 593, 698, 1483, 2527, 834, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 2098, 776, 1655, 1224, 177, 624, 1828, 965, 1889, 263, 1631, 879, 733, 90, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2079, 1252, 1376, 599, 1776, 269, 837, 1046, 2254, 251, 825, 1268, 1077, 634, 662, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 1876, 379, 1397, 900, 1902, 1543, 568, 721, 638, 1875, 693, 859, 2136, 958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 716, 1055, 1075, 165, 723, 1215, 1542, 864, 1491, 1640, 537, 1387, 397, 318, 249, 977, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16930, 15071, 15708, 17860, 16243, 17102, 16742, 15416] avg=16384.0000 max=17860 max_rank=3 min=15071 min_rank=1 imbalance_ratio=1.090088 max_load_reduction=113212 max_load_reduction_pct=0.863739 moved_to_echo=65503 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[84, 2878, 415, 251, 1463, 173, 1107, 1067, 1117, 182, 1463, 143, 673, 677, 98, 325, 2663, 1375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 791, 883, 683, 819, 95, 2637, 1453, 439, 1460, 800, 992, 1550, 416, 1374, 1889, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 397, 618, 499, 57, 2583, 2104, 681, 2491, 303, 1218, 775, 1478, 1709, 968, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 469, 202, 2287, 1178, 513, 575, 88, 280, 163, 959, 2168, 1218, 500, 686, 961, 2329, 984, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 698, 2056, 541, 1033, 912, 235, 569, 1691, 907, 2042, 156, 2209, 598, 1101, 106, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2855, 1008, 1238, 390, 1752, 114, 604, 729, 2323, 119, 780, 1183, 559, 869, 562, 984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 1904, 586, 1543, 428, 1506, 1627, 428, 1122, 532, 2095, 493, 222, 2405, 886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 1416, 1701, 1250, 279, 445, 826, 1141, 748, 1700, 1478, 815, 1851, 456, 544, 248, 1031, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16154, 16457, 16972, 16529, 16060, 16069, 15979, 16852] avg=16384.0000 max=16972 max_rank=2 min=15979 min_rank=6 imbalance_ratio=1.035889 max_load_reduction=114100 max_load_reduction_pct=0.870514 moved_to_echo=64960 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[99, 2426, 600, 311, 1332, 424, 785, 478, 781, 104, 1297, 188, 827, 779, 257, 333, 3112, 1541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 452, 658, 916, 825, 396, 2737, 1534, 276, 806, 1273, 818, 994, 137, 1587, 2336, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 565, 147, 422, 543, 2335, 2607, 339, 2525, 653, 1211, 827, 1127, 1481, 1012, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 105, 334, 1806, 628, 377, 659, 161, 1006, 345, 1298, 2277, 1368, 483, 550, 1011, 1812, 1102, 638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1630, 701, 892, 1267, 130, 957, 1710, 1159, 2698, 225, 2099, 967, 788, 118, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2484, 862, 1165, 536, 1675, 134, 684, 1204, 1873, 146, 483, 1024, 1098, 478, 363, 972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 2298, 982, 1539, 786, 1169, 915, 672, 823, 842, 2567, 612, 505, 2020, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 1183, 1400, 1379, 548, 578, 565, 1160, 992, 1443, 1619, 494, 1859, 736, 453, 212, 580, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15674, 15962, 17557, 16200, 17187, 15181, 17070, 16241] avg=16384.0000 max=17557 max_rank=2 min=15181 min_rank=5 imbalance_ratio=1.071594 max_load_reduction=113515 max_load_reduction_pct=0.866051 moved_to_echo=65679 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[134, 2637, 412, 345, 908, 313, 824, 589, 1076, 221, 1403, 349, 966, 521, 226, 298, 2841, 1512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 451, 612, 802, 1033, 576, 2827, 1855, 386, 1435, 1349, 730, 1369, 233, 1524, 1842, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 932, 359, 463, 148, 1976, 1841, 634, 2648, 484, 1233, 710, 1262, 1187, 1059, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 498, 398, 1797, 535, 430, 526, 124, 333, 603, 882, 1734, 1571, 399, 716, 992, 2315, 1082, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 419, 1721, 816, 1733, 1422, 135, 845, 1825, 975, 1810, 192, 1699, 599, 1141, 115, 958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2790, 1151, 1300, 417, 1559, 133, 1247, 1249, 2006, 308, 986, 1488, 1084, 619, 625, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 1690, 1116, 1385, 584, 1099, 1564, 696, 768, 544, 2009, 639, 340, 1839, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 1137, 1465, 1623, 280, 448, 1350, 1274, 787, 1483, 1535, 1032, 1061, 541, 820, 275, 992, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15575, 17205, 15818, 15933, 16405, 17368, 15642, 17126] avg=16384.0000 max=17368 max_rank=5 min=15575 min_rank=0 imbalance_ratio=1.060059 max_load_reduction=113704 max_load_reduction_pct=0.867493 moved_to_echo=66541 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[21120, 8576, 10240, 3840, 14592, 1920, 6272, 8832, 16512, 1792, 6400, 9472, 7296, 4736, 4096, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1152, 4864, 2944, 4096, 1280, 18176, 17152, 4096, 20224, 3200, 10496, 6016, 10752, 11776, 8192, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[5632, 15232, 4992, 9728, 10240, 1536, 6144, 14208, 8576, 16768, 1664, 14592, 6400, 7936, 896, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[896, 22272, 4224, 2176, 11520, 3072, 6912, 7552, 8192, 1536, 12032, 1792, 6528, 5760, 1536, 2816, 22656, 11136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[2176, 2560, 2560, 16640, 6144, 3456, 4352, 896, 4480, 2688, 8576, 14976, 11136, 4224, 6144, 9728, 18176, 7936, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 2560, 14592, 5376, 11904, 5888, 12288, 10752, 5248, 7808, 5120, 16896, 4864, 3200, 17792, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[2816, 9216, 9984, 11136, 2176, 4992, 7552, 10496, 6784, 11904, 12032, 5760, 12544, 4096, 4864, 2944, 6912, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1024, 3712, 6528, 6144, 8192, 2176, 22400, 13696, 3200, 9728, 8576, 7168, 8832, 2048, 12544, 15616, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[3555, 2, 1646, 2906, 2089, 0, 1217, 787, 1144, 341, 251, 9, 469, 104, 0, 2583, 7, 1, 2245, 8, 650, 4041, 1923, 1314, 30, 857, 0, 1388, 5, 733, 4018, 0, 164, 59, 4029, 0, 0, 0, 1517, 0, 292, 59, 0, 0, 1, 2, 195, 0, 0, 355, 110, 0, 134, 68, 0, 9, 17, 4071, 2111, 507, 1186, 261, 13, 42, 5, 206, 4043, 12, 2, 52, 4044, 1, 2492, 378, 8, 13, 2553, 0, 2596, 15, 3069, 2969, 1256, 3129, 1565, 1904, 85, 3998, 1, 3803, 2400, 3425, 950, 6, 0, 209, 0, 121, 4095, 3618, 3698, 0, 1031, 34, 5, 157, 3779, 642, 2625, 1651, 18, 790, 556, 828, 2278, 0, 35, 3553, 276, 961, 834, 6, 0, 1149, 10, 0, 53, 3555] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1022, 0, 4095, 163, 1988, 0, 419, 3737, 1942, 357, 27, 129, 759, 70, 0, 2779, 22, 0, 9, 0, 2, 1901, 44, 856, 80, 121, 0, 62, 190, 3284, 1065, 0, 3678, 378, 2974, 0, 0, 0, 695, 0, 1557, 18, 0, 0, 0, 1, 2, 32, 3, 68, 1, 0, 201, 404, 22, 1, 531, 4042, 2279, 1441, 2605, 0, 4, 5, 2, 21, 4080, 1454, 1274, 50, 3835, 6, 4067, 2713, 58, 146, 966, 0, 3135, 0, 4089, 3995, 8, 404, 3941, 9, 1636, 3780, 33, 2195, 751, 4089, 511, 1549, 477, 1502, 25, 2050, 4095, 741, 1304, 0, 4002, 205, 4, 422, 2181, 15, 4094, 345, 6, 4022, 0, 183, 100, 0, 14, 880, 8, 268, 3032, 4, 0, 1214, 496, 74, 357, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[3278, 47, 3999, 1514, 3081, 0, 518, 3793, 2711, 1994, 230, 1, 1262, 670, 1, 363, 252, 2, 958, 5, 2, 4068, 0, 26, 0, 293, 7, 11, 260, 4049, 3983, 20, 2310, 198, 4069, 1, 0, 2, 4, 27, 331, 8, 15, 0, 0, 0, 1, 1, 0, 1101, 45, 0, 9, 2643, 9, 17, 6, 3931, 1360, 932, 832, 0, 0, 270, 5, 147, 2613, 13, 655, 0, 4031, 0, 3982, 753, 1, 371, 298, 0, 3505, 38, 3852, 1252, 1465, 20, 3896, 22, 187, 4026, 0, 125, 3207, 4094, 1356, 54, 5, 26, 0, 80, 4041, 1712, 1097, 6, 2978, 1, 1, 188, 41, 17, 4043, 875, 20, 3917, 31, 659, 3674, 1, 129, 916, 1983, 11, 525, 28, 0, 3932, 650, 21, 53, 3923] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4067, 0, 152, 738, 3586, 12, 2663, 8, 4065, 198, 234, 4, 965, 2060, 0, 875, 17, 8, 3225, 1, 1, 1066, 41, 8, 22, 1855, 1, 38, 4, 2679, 4091, 1, 2913, 2981, 2690, 0, 0, 0, 3, 14, 21, 182, 0, 0, 0, 0, 2835, 84, 16, 1082, 118, 3, 101, 3014, 0, 1, 11, 3672, 236, 2416, 1765, 0, 203, 0, 11, 2344, 2235, 116, 417, 9, 3955, 4, 948, 763, 0, 15, 36, 7, 3716, 1, 4013, 1287, 41, 396, 3449, 85, 817, 3434, 65, 1695, 574, 4050, 1784, 3575, 89, 90, 0, 0, 4091, 583, 2155, 0, 2753, 0, 555, 79, 2128, 11, 3862, 1497, 246, 2834, 122, 92, 20, 0, 1, 2965, 2215, 59, 233, 0, 5, 3440, 34, 3, 51, 3971] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2514, 0, 4083, 1013, 607, 0, 2595, 1895, 3804, 50, 44, 5, 3784, 1964, 0, 625, 0, 0, 1438, 0, 1, 369, 226, 256, 114, 0, 0, 157, 4, 3290, 3529, 48, 717, 919, 3550, 0, 0, 8, 570, 0, 166, 1, 0, 0, 0, 18, 66, 186, 0, 22, 5, 0, 0, 2460, 1, 0, 95, 4064, 253, 1075, 3547, 0, 1, 77, 0, 35, 3732, 1276, 2485, 5, 2473, 143, 3354, 197, 517, 0, 902, 0, 48, 0, 3653, 2010, 269, 363, 4090, 1041, 2426, 3550, 56, 1422, 112, 4092, 16, 103, 281, 17, 3, 3660, 3971, 1430, 4067, 9, 3818, 77, 0, 397, 3817, 5, 3645, 3, 19, 2434, 290, 48, 2619, 0, 3, 4048, 263, 19, 588, 0, 0, 3078, 1, 0, 0, 3876] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[3782, 1, 3766, 406, 2264, 28, 15, 4061, 2910, 2371, 0, 0, 142, 1065, 0, 3622, 8, 4, 2, 40, 4, 1590, 0, 75, 613, 2404, 125, 3, 0, 3996, 2920, 1, 3917, 218, 4068, 0, 0, 830, 0, 13, 56, 19, 12, 0, 3, 596, 298, 0, 1, 251, 2, 36, 3428, 617, 23, 5, 0, 4057, 1612, 1220, 273, 4, 171, 0, 2, 97, 2936, 3264, 542, 218, 4054, 0, 2049, 42, 19, 27, 295, 2, 4081, 64, 3881, 1334, 3834, 16, 3280, 0, 70, 4093, 0, 861, 326, 2566, 47, 50, 0, 0, 2, 2, 3663, 570, 4068, 1901, 2716, 0, 1, 60, 1462, 17, 4093, 276, 0, 1664, 727, 69, 48, 46, 1, 530, 4, 0, 2928, 0, 0, 3653, 764, 1577, 197, 4035] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[2953, 0, 3080, 622, 1275, 0, 384, 1234, 3443, 737, 2716, 130, 3022, 79, 593, 3532, 94, 0, 3543, 0, 683, 1839, 1883, 10, 0, 2860, 0, 625, 6, 1638, 2267, 9, 2889, 3056, 3550, 0, 0, 0, 4, 1, 1652, 2362, 10, 0, 1, 6, 1023, 91, 0, 14, 134, 0, 9, 2430, 1, 0, 146, 4078, 34, 111, 519, 1063, 34, 0, 0, 24, 1340, 15, 187, 49, 3176, 31, 1052, 407, 19, 69, 86, 0, 3794, 4, 4037, 3772, 731, 297, 4035, 19, 3270, 4042, 3, 5, 134, 4096, 1423, 2343, 7, 2950, 0, 2006, 4096, 1182, 974, 0, 34, 412, 3, 1392, 3912, 345, 1689, 1632, 2, 95, 1, 24, 588, 296, 1, 3272, 1562, 0, 1, 0, 0, 1388, 34, 37, 2, 2200] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[3950, 0, 4010, 415, 2316, 0, 769, 1266, 1957, 294, 4, 0, 302, 2624, 32, 2996, 10, 3, 731, 0, 98, 3068, 246, 2787, 24, 1978, 0, 197, 17, 1890, 3896, 3, 191, 209, 4060, 54, 49, 0, 168, 0, 263, 2230, 0, 0, 18, 326, 348, 245, 46, 30, 162, 0, 1533, 3646, 3, 0, 0, 3905, 3403, 3656, 1830, 2, 0, 334, 118, 53, 3916, 537, 1465, 62, 3990, 117, 492, 366, 20, 207, 281, 4, 1905, 7, 3673, 1456, 40, 841, 1766, 1, 686, 4005, 685, 118, 1142, 4095, 0, 109, 8, 251, 0, 360, 4096, 3630, 3172, 11, 671, 7, 0, 546, 641, 202, 4078, 302, 999, 283, 607, 19, 1075, 6, 4, 3448, 1068, 95, 2787, 0, 5, 2530, 59, 1708, 52, 3601] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 743, 2906, 2089, 0, 1217, 787, 1144, 341, 251, 9, 469, 104, 0, 2583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 2245, 8, 650, 4041, 1923, 1314, 30, 857, 0, 1388, 5, 733, 4018, 0, 3487, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 59, 4029, 0, 0, 0, 1517, 0, 292, 59, 0, 0, 1, 2, 195, 0, 4095, 3302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 110, 0, 134, 68, 0, 9, 17, 4071, 2111, 507, 1186, 261, 13, 42, 3425, 696, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 206, 4043, 12, 2, 52, 4044, 1, 2492, 378, 8, 13, 2553, 0, 2596, 15, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2242, 2969, 1256, 3129, 1565, 1904, 85, 0, 1, 3803, 2400, 0, 950, 6, 0, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 3618, 3698, 0, 1031, 34, 5, 157, 3779, 642, 1974, 1651, 18, 790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 828, 2278, 0, 35, 3553, 276, 961, 834, 6, 0, 1149, 10, 0, 53, 3555, 708, 827, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12645, 20902, 13715, 13073, 16905, 20519, 17518, 15795] avg=16384.0000 max=20902 max_rank=1 min=12645 min_rank=0 imbalance_ratio=1.275757 max_load_reduction=110170 max_load_reduction_pct=0.840530 moved_to_echo=70737 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 47, 1804, 1514, 3081, 0, 518, 3793, 2711, 1994, 230, 1, 1262, 670, 1, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 2, 958, 5, 2, 4068, 0, 26, 0, 293, 7, 11, 260, 4049, 3983, 20, 3216, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2310, 198, 4069, 1, 0, 2, 4, 27, 331, 8, 15, 0, 0, 0, 1, 1, 4041, 3325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1101, 45, 0, 9, 2643, 9, 17, 6, 3931, 1360, 932, 832, 0, 0, 270, 4094, 701, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 147, 2613, 13, 655, 0, 4031, 0, 3982, 753, 1, 371, 298, 0, 3505, 38, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2814, 1252, 1465, 20, 3896, 22, 187, 0, 0, 125, 3207, 0, 1356, 54, 5, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 1712, 1097, 6, 2978, 1, 1, 188, 41, 17, 3040, 875, 20, 3917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 659, 3674, 1, 129, 916, 1983, 11, 525, 28, 0, 3932, 650, 21, 53, 3923, 1720, 1038, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17989, 17627, 14333, 16012, 17159, 14429, 13973, 19550] avg=16384.0000 max=19550 max_rank=7 min=13973 min_rank=6 imbalance_ratio=1.193237 max_load_reduction=111522 max_load_reduction_pct=0.850845 moved_to_echo=65111 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 69, 738, 3586, 12, 2663, 8, 4065, 198, 234, 4, 965, 2060, 0, 875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 8, 3225, 1, 1, 1066, 41, 8, 22, 1855, 1, 38, 4, 2679, 4091, 1, 3990, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2913, 2981, 2690, 0, 0, 0, 3, 14, 21, 182, 0, 0, 0, 0, 2835, 84, 4091, 2836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1082, 118, 3, 101, 3014, 0, 1, 11, 3672, 236, 2416, 1765, 0, 203, 0, 4050, 598, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2344, 2235, 116, 417, 9, 3955, 4, 948, 763, 0, 15, 36, 7, 3716, 1, 714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2931, 1287, 41, 396, 3449, 85, 817, 0, 65, 1695, 574, 0, 1784, 3575, 89, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 2155, 0, 2753, 0, 555, 79, 2128, 11, 2904, 1497, 246, 2834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 92, 20, 0, 1, 2965, 2215, 59, 233, 0, 5, 3440, 34, 3, 51, 3971, 65, 1082, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15477, 17066, 18650, 17363, 15291, 16878, 15745, 14602] avg=16384.0000 max=18650 max_rank=2 min=14602 min_rank=7 imbalance_ratio=1.138306 max_load_reduction=112422 max_load_reduction_pct=0.857712 moved_to_echo=62516 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1833, 1013, 607, 0, 2595, 1895, 3804, 50, 44, 5, 3784, 1964, 0, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 0, 1, 369, 226, 256, 114, 0, 0, 157, 4, 3290, 3529, 48, 2467, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 717, 919, 3550, 0, 0, 8, 570, 0, 166, 1, 0, 0, 0, 18, 66, 186, 3971, 2933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 5, 0, 0, 2460, 1, 0, 95, 4064, 253, 1075, 3547, 0, 1, 77, 4092, 617, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 3732, 1276, 2485, 5, 2473, 143, 3354, 197, 517, 0, 902, 0, 48, 0, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2665, 2010, 269, 363, 4090, 1041, 2426, 0, 56, 1422, 112, 0, 16, 103, 281, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3660, 0, 1430, 4067, 9, 3818, 77, 0, 397, 3817, 5, 2732, 3, 19, 2434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 48, 2619, 0, 3, 4048, 263, 19, 588, 0, 0, 3078, 1, 0, 0, 3876, 1760, 988, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18219, 12389, 13105, 16356, 15845, 14871, 22471, 17816] avg=16384.0000 max=22471 max_rank=6 min=12389 min_rank=1 imbalance_ratio=1.371521 max_load_reduction=108601 max_load_reduction_pct=0.828560 moved_to_echo=71003 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1848, 163, 1988, 0, 419, 3737, 1942, 357, 27, 129, 759, 70, 0, 2779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 9, 0, 2, 1901, 44, 856, 80, 121, 0, 62, 190, 3284, 1065, 0, 1003, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3678, 378, 2974, 0, 0, 0, 695, 0, 1557, 18, 0, 0, 0, 1, 2, 32, 4095, 3121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 68, 1, 0, 201, 404, 22, 1, 531, 4042, 2279, 1441, 2605, 0, 4, 5, 4089, 659, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 21, 4080, 1454, 1274, 50, 3835, 6, 4067, 2713, 58, 146, 966, 0, 3135, 0, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2987, 3995, 8, 404, 3941, 9, 1636, 0, 33, 2195, 751, 0, 511, 1549, 477, 1502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2050, 0, 741, 1304, 0, 4002, 205, 4, 422, 2181, 15, 3078, 345, 6, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 100, 0, 14, 880, 8, 268, 3032, 4, 0, 1214, 496, 74, 357, 4090, 1761, 1102, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14218, 9125, 16551, 16374, 22564, 19998, 18400, 13842] avg=16384.0000 max=22564 max_rank=4 min=9125 min_rank=1 imbalance_ratio=1.377197 max_load_reduction=108508 max_load_reduction_pct=0.827850 moved_to_echo=74804 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1699, 406, 2264, 28, 15, 4061, 2910, 2371, 0, 0, 142, 1065, 0, 3622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 2, 40, 4, 1590, 0, 75, 613, 2404, 125, 3, 0, 3996, 2920, 1, 3711, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3917, 218, 4068, 0, 0, 830, 0, 13, 56, 19, 12, 0, 3, 596, 298, 0, 3663, 3381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 251, 2, 36, 3428, 617, 23, 5, 0, 4057, 1612, 1220, 273, 4, 171, 0, 2566, 712, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 97, 2936, 3264, 542, 218, 4054, 0, 2049, 42, 19, 27, 295, 2, 4081, 64, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2835, 1334, 3834, 16, 3280, 0, 70, 0, 0, 861, 326, 0, 47, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 570, 4068, 1901, 2716, 0, 1, 60, 1462, 17, 3077, 276, 0, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 727, 69, 48, 46, 1, 530, 4, 0, 2928, 0, 0, 3653, 764, 1577, 197, 4035, 1620, 1046, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18584, 15943, 17074, 15049, 18449, 12653, 15816, 17504] avg=16384.0000 max=18584 max_rank=0 min=12653 min_rank=5 imbalance_ratio=1.134277 max_load_reduction=112488 max_load_reduction_pct=0.858215 moved_to_echo=64422 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1389, 622, 1275, 0, 384, 1234, 3443, 737, 2716, 130, 3022, 79, 593, 3532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 3543, 0, 683, 1839, 1883, 10, 0, 2860, 0, 625, 6, 1638, 2267, 9, 2897, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2889, 3056, 3550, 0, 0, 0, 4, 1, 1652, 2362, 10, 0, 1, 6, 1023, 91, 4096, 3338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 134, 0, 9, 2430, 1, 0, 146, 4078, 34, 111, 519, 1063, 34, 0, 4096, 704, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 1340, 15, 187, 49, 3176, 31, 1052, 407, 19, 69, 86, 0, 3794, 4, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2949, 3772, 731, 297, 4035, 19, 3270, 0, 3, 5, 134, 0, 1423, 2343, 7, 2950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2006, 0, 1182, 974, 0, 34, 412, 3, 1392, 3912, 345, 1270, 1632, 2, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 24, 588, 296, 1, 3272, 1562, 0, 1, 0, 0, 1388, 34, 37, 2, 2200, 1325, 1088, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19156, 18720, 22079, 13429, 10565, 21938, 13259, 11926] avg=16384.0000 max=22079 max_rank=2 min=10565 min_rank=4 imbalance_ratio=1.347595 max_load_reduction=108993 max_load_reduction_pct=0.831551 moved_to_echo=57688 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1809, 415, 2316, 0, 769, 1266, 1957, 294, 4, 0, 302, 2624, 32, 2996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3, 731, 0, 98, 3068, 246, 2787, 24, 1978, 0, 197, 17, 1890, 3896, 3, 3876, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 209, 4060, 54, 49, 0, 168, 0, 263, 2230, 0, 0, 18, 326, 348, 245, 4096, 3308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 30, 162, 0, 1533, 3646, 3, 0, 0, 3905, 3403, 3656, 1830, 2, 0, 334, 4095, 697, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 53, 3916, 537, 1465, 62, 3990, 117, 492, 366, 20, 207, 281, 4, 1905, 7, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2683, 1456, 40, 841, 1766, 1, 686, 0, 685, 118, 1142, 0, 0, 109, 8, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 0, 3630, 3172, 11, 671, 7, 0, 546, 641, 202, 3066, 302, 999, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 19, 1075, 6, 4, 3448, 1068, 95, 2787, 0, 5, 2530, 59, 1708, 52, 3601, 1725, 990, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14784, 19300, 15565, 23416, 14294, 9786, 13890, 20037] avg=16384.0000 max=23416 max_rank=3 min=9786 min_rank=5 imbalance_ratio=1.429199 max_load_reduction=107656 max_load_reduction_pct=0.821350 moved_to_echo=58007 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2432, 2048, 10496, 384, 256, 19712, 7424, 1536, 11008, 128, 128, 20480, 2048, 3456, 768, 29312, 10752, 8192, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 11264, 7808, 17280, 128, 8704, 16896, 22016, 6400, 3584, 384, 10752, 8704, 640, 17408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 2944, 640, 128, 5504, 15360, 128, 128, 896, 31872, 11392, 11392, 12672, 1408, 512, 768, 30592, 5504, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[512, 128, 12160, 128, 1536, 18048, 4480, 5376, 896, 10368, 256, 2560, 512, 21632, 25856, 128, 24704, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[22144, 18176, 7680, 5504, 26112, 3200, 9216, 0, 896, 10240, 8704, 0, 6144, 7808, 896, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[16896, 8064, 29056, 128, 128, 896, 3072, 128, 4352, 4992, 128, 0, 128, 1024, 4864, 640, 32256, 25600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 2944, 24960, 6784, 7040, 512, 29568, 384, 18560, 5632, 768, 896, 5504, 128, 22784, 256, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 8320, 0, 13568, 20608, 2048, 18048, 768, 640, 3328, 18048, 1280, 21248, 6656, 1408, 16128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3382, 0, 4096, 16, 187, 4093, 0, 3403, 0, 3433, 0, 4096, 0, 118, 0, 22, 0, 0, 0, 0, 0, 4, 0, 248, 1894, 1028, 3483, 5, 7, 4089, 1, 189, 3845, 0, 0, 1, 0, 2735, 2, 0, 3836, 5, 3, 19, 3604, 0, 21, 0, 490, 4094, 444, 926, 10, 0, 8, 19, 3304, 0, 1758, 1688, 3667, 4027, 43, 989, 2440, 0, 3969, 104, 0, 2047, 4067, 2865, 3, 71, 3966, 17, 0, 38, 3, 36, 1, 2272, 1, 3, 2, 4096, 3384, 4096, 0, 1475, 0, 0, 3, 4093, 332, 506, 8, 3, 3070, 0, 1150, 0, 0, 43, 0, 28, 4, 558, 6, 4043, 21, 2, 2325, 126, 4096, 41, 4034, 1345, 59, 453, 36, 0, 11, 12, 370, 0, 2, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[3724, 15, 4077, 841, 0, 2671, 0, 369, 43, 4089, 0, 4092, 3394, 0, 0, 133, 0, 8, 0, 0, 325, 22, 3, 0, 3429, 33, 3320, 129, 946, 4077, 0, 795, 4079, 0, 2, 0, 0, 0, 407, 72, 3805, 0, 44, 2054, 297, 804, 698, 1, 364, 4068, 0, 12, 132, 0, 447, 356, 232, 0, 2028, 3193, 3963, 3257, 2332, 3383, 236, 0, 4014, 0, 0, 890, 3945, 37, 454, 1, 3603, 176, 1, 2, 19, 13, 244, 4096, 177, 754, 66, 4077, 0, 4081, 2, 1592, 1, 0, 9, 1464, 0, 2436, 6, 69, 1146, 0, 60, 219, 7, 13, 0, 0, 3778, 25, 0, 4085, 91, 0, 2, 0, 4096, 2357, 3531, 734, 5, 0, 366, 0, 16, 1, 3, 0, 1642, 3865] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4093, 0, 4001, 14, 1, 3848, 0, 3545, 8, 4094, 0, 4096, 1766, 266, 0, 347, 33, 0, 0, 31, 75, 23, 0, 7, 3794, 4067, 8, 0, 2150, 4084, 0, 16, 370, 0, 63, 35, 0, 2257, 0, 0, 2027, 19, 0, 4, 3420, 10, 71, 0, 0, 4076, 4065, 7, 5, 2, 0, 996, 63, 1, 0, 1541, 4050, 4083, 394, 1936, 4014, 8, 1872, 223, 0, 1619, 4093, 0, 50, 0, 4021, 18, 0, 0, 6, 4, 0, 1756, 3535, 642, 2477, 4094, 9, 4086, 0, 3313, 0, 8, 2, 4, 220, 193, 15, 0, 3097, 0, 6, 263, 0, 0, 45, 0, 143, 4076, 51, 3380, 0, 0, 3633, 19, 4080, 4091, 164, 654, 12, 0, 15, 168, 472, 1, 0, 5, 118, 435] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[3499, 0, 2023, 0, 2, 3881, 0, 1177, 0, 4096, 1, 3524, 336, 1, 5, 148, 0, 1304, 0, 0, 73, 33, 0, 0, 3673, 1, 4084, 0, 0, 4094, 110, 0, 4096, 3, 0, 1009, 99, 3753, 0, 0, 3584, 58, 4, 2, 3952, 315, 3052, 104, 0, 4096, 28, 3484, 0, 18, 4001, 0, 3, 49, 0, 2246, 4096, 2840, 2769, 1650, 1229, 0, 3488, 0, 0, 265, 4003, 3, 2, 1, 4096, 257, 31, 1569, 0, 16, 0, 58, 7, 25, 0, 3855, 25, 4093, 0, 3283, 4, 0, 0, 0, 1, 112, 0, 2, 2644, 56, 1423, 744, 0, 9, 35, 2, 14, 1955, 0, 44, 6, 37, 3, 0, 4077, 4094, 3722, 1, 1, 0, 3896, 2, 519, 6, 1101, 0, 180, 2700] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[516, 7, 4058, 13, 28, 186, 5, 2580, 13, 4021, 0, 1778, 4053, 8, 2, 0, 27, 5, 205, 12, 3794, 0, 0, 0, 6, 38, 745, 0, 65, 3255, 76, 3898, 2836, 466, 24, 0, 479, 200, 4, 362, 3432, 0, 2257, 3998, 4046, 3, 91, 1169, 5, 4065, 0, 28, 129, 6, 3, 0, 2536, 0, 16, 4095, 4056, 4010, 3841, 1041, 28, 22, 2416, 0, 0, 686, 3891, 0, 2586, 3, 2667, 3513, 13, 0, 0, 105, 17, 3628, 9, 2, 903, 4093, 44, 3962, 711, 82, 5, 0, 39, 0, 125, 508, 3, 292, 996, 1385, 0, 2, 1, 0, 0, 22, 17, 0, 0, 4096, 4, 1, 57, 992, 4058, 4079, 50, 0, 0, 1, 72, 63, 3846, 198, 0, 56, 4091, 4040] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4096, 0, 3614, 14, 0, 3950, 1, 1192, 0, 4091, 0, 4096, 1, 9, 0, 17, 0, 13, 0, 1, 95, 105, 0, 0, 3962, 37, 8, 1, 39, 4091, 0, 759, 4081, 0, 0, 0, 0, 239, 0, 0, 4077, 3, 0, 22, 3997, 0, 415, 0, 0, 4095, 0, 85, 16, 0, 1224, 0, 2314, 2, 2, 3906, 4045, 4009, 1833, 3350, 16, 0, 3954, 0, 0, 3, 4094, 9, 48, 136, 4082, 0, 0, 0, 1, 0, 0, 4049, 5, 3836, 1, 3656, 1, 4078, 0, 73, 0, 0, 0, 21, 3104, 1866, 0, 0, 4028, 0, 235, 182, 0, 0, 3, 89, 0, 574, 2, 4094, 5, 3, 66, 8, 4096, 4072, 4093, 3, 2, 0, 863, 0, 3905, 0, 3777, 0, 2, 30] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4087, 0, 998, 0, 0, 4010, 0, 3450, 0, 4096, 0, 3146, 3770, 47, 0, 75, 2, 2, 0, 0, 0, 104, 0, 0, 1397, 4096, 0, 0, 1942, 3944, 30, 313, 19, 0, 0, 4063, 5, 3610, 0, 0, 576, 0, 363, 400, 38, 93, 200, 0, 0, 4096, 4096, 0, 0, 0, 71, 0, 828, 10, 0, 135, 4095, 155, 153, 4094, 4054, 0, 3828, 0, 0, 0, 3213, 103, 35, 1, 1761, 0, 0, 0, 0, 14, 200, 122, 1517, 22, 3337, 4096, 3851, 3517, 0, 3981, 14, 831, 0, 0, 4030, 1392, 0, 3, 18, 2, 1, 2480, 0, 0, 19, 138, 4089, 2816, 0, 198, 41, 1, 1, 323, 4096, 4093, 732, 0, 0, 572, 32, 4042, 891, 0, 2, 0, 19, 3935] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[3737, 0, 3693, 0, 0, 4094, 0, 28, 0, 4074, 0, 4090, 3341, 1763, 0, 26, 2, 528, 0, 0, 155, 64, 0, 85, 173, 1, 1858, 0, 0, 4073, 1258, 619, 4076, 5, 203, 39, 1, 2041, 0, 43, 4096, 0, 8, 779, 3423, 143, 600, 0, 0, 4096, 7, 0, 881, 24, 2141, 572, 569, 25, 0, 3864, 3152, 4017, 14, 0, 2825, 0, 4095, 0, 141, 2283, 4076, 1, 189, 61, 2089, 76, 0, 0, 0, 1, 0, 4096, 2, 4058, 1311, 3278, 1938, 4005, 0, 4052, 0, 6, 0, 10, 23, 114, 0, 2, 4020, 46, 496, 0, 0, 106, 0, 32, 23, 0, 0, 3992, 0, 0, 0, 0, 3972, 3180, 3971, 3, 0, 0, 3928, 60, 3, 0, 11, 0, 11, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3382, 0, 4096, 16, 187, 4093, 0, 3403, 0, 0, 0, 1990, 0, 118, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 248, 1894, 1028, 3483, 5, 7, 4089, 1, 189, 3158, 1166, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3845, 0, 0, 1, 0, 2735, 2, 0, 3836, 5, 3, 19, 3604, 0, 21, 0, 1503, 940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 444, 926, 10, 0, 8, 19, 3304, 0, 1758, 1688, 2045, 4027, 43, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2440, 0, 3969, 104, 0, 2047, 4067, 2865, 3, 71, 3966, 17, 0, 38, 3, 36, 1135, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2272, 1, 3, 2, 4096, 3384, 938, 0, 1475, 0, 0, 3, 4093, 332, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 3070, 0, 1150, 0, 0, 43, 0, 28, 4, 558, 6, 4043, 21, 2, 4094, 2935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2325, 126, 2961, 41, 4034, 1345, 59, 453, 36, 0, 11, 12, 370, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17307, 15770, 16514, 15751, 20880, 17106, 15965, 11779] avg=16384.0000 max=20880 max_rank=4 min=11779 min_rank=7 imbalance_ratio=1.274414 max_load_reduction=110192 max_load_reduction_pct=0.840698 moved_to_echo=65730 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3724, 15, 4077, 841, 0, 2671, 0, 369, 43, 0, 0, 1989, 3394, 0, 0, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 325, 22, 3, 0, 3429, 33, 3320, 129, 946, 4077, 0, 795, 3147, 1164, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 2, 0, 0, 0, 407, 72, 3805, 0, 44, 2054, 297, 804, 698, 1, 1625, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 0, 0, 12, 132, 0, 447, 356, 232, 0, 2028, 3193, 2209, 3257, 2332, 3383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 4014, 0, 0, 890, 3945, 37, 454, 1, 3603, 176, 1, 2, 19, 13, 1135, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 4096, 177, 754, 66, 4077, 0, 934, 2, 1592, 1, 0, 9, 1464, 0, 2436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 69, 1146, 0, 60, 219, 7, 13, 0, 0, 3778, 25, 0, 4085, 91, 0, 4068, 3496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2961, 2357, 3531, 734, 5, 0, 366, 0, 16, 1, 3, 0, 1642, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17256, 17991, 14827, 17945, 14655, 15852, 17063, 15483] avg=16384.0000 max=17991 max_rank=1 min=14655 min_rank=4 imbalance_ratio=1.098083 max_load_reduction=113081 max_load_reduction_pct=0.862740 moved_to_echo=63053 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 4001, 14, 1, 3848, 0, 3545, 8, 0, 0, 1990, 1766, 266, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 31, 75, 23, 0, 7, 3794, 4067, 8, 0, 2150, 4084, 0, 16, 3150, 1166, 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 0, 63, 35, 0, 2257, 0, 0, 2027, 19, 0, 4, 3420, 10, 71, 0, 1661, 940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 7, 5, 2, 0, 996, 63, 1, 0, 1541, 2257, 4083, 394, 1936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 8, 1872, 223, 0, 1619, 4093, 0, 50, 0, 4021, 18, 0, 0, 6, 4, 1131, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1756, 3535, 642, 2477, 4094, 9, 936, 0, 3313, 0, 8, 2, 4, 220, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3097, 0, 6, 263, 0, 0, 45, 0, 143, 4076, 51, 3380, 0, 0, 4076, 3500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3633, 19, 2949, 4091, 164, 654, 12, 0, 15, 168, 472, 1, 0, 5, 118, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19879, 19198, 10877, 15350, 17191, 17189, 18652, 12736] avg=16384.0000 max=19879 max_rank=0 min=10877 min_rank=2 imbalance_ratio=1.213318 max_load_reduction=111193 max_load_reduction_pct=0.848335 moved_to_echo=65768 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3499, 0, 2023, 0, 2, 3881, 0, 1177, 0, 0, 1, 1705, 336, 1, 5, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1304, 0, 0, 73, 33, 0, 0, 3673, 1, 4084, 0, 0, 4094, 110, 0, 3160, 1006, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 1009, 99, 3753, 0, 0, 3584, 58, 4, 2, 3952, 315, 3052, 104, 1684, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3484, 0, 18, 4001, 0, 3, 49, 0, 2246, 2274, 2840, 2769, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1229, 0, 3488, 0, 0, 265, 4003, 3, 2, 1, 4096, 257, 31, 1569, 0, 16, 1134, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 7, 25, 0, 3855, 25, 933, 0, 3283, 4, 0, 0, 0, 1, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2644, 56, 1423, 744, 0, 9, 35, 2, 14, 1955, 0, 44, 6, 37, 4096, 3501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2943, 4094, 3722, 1, 1, 0, 3896, 2, 519, 6, 1101, 0, 180, 2700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12778, 18133, 22528, 19362, 16232, 8303, 14568, 19168] avg=16384.0000 max=22528 max_rank=2 min=8303 min_rank=5 imbalance_ratio=1.375000 max_load_reduction=108544 max_load_reduction_pct=0.828125 moved_to_echo=58271 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[516, 7, 4058, 13, 28, 186, 5, 2580, 13, 0, 0, 864, 4053, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 5, 205, 12, 3794, 0, 0, 0, 6, 38, 745, 0, 65, 3255, 76, 3898, 3055, 506, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2836, 466, 24, 0, 479, 200, 4, 362, 3432, 0, 2257, 3998, 4046, 3, 91, 1169, 1663, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 28, 129, 6, 3, 0, 2536, 0, 16, 4095, 2261, 4010, 3841, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 22, 2416, 0, 0, 686, 3891, 0, 2586, 3, 2667, 3513, 13, 0, 0, 105, 1125, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3628, 9, 2, 903, 4093, 44, 907, 711, 82, 5, 0, 39, 0, 125, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 292, 996, 1385, 0, 2, 1, 0, 0, 22, 17, 0, 0, 4096, 4, 1, 4065, 3436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 992, 2933, 4079, 50, 0, 0, 1, 72, 63, 3846, 198, 0, 56, 4091, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12333, 16272, 21438, 17971, 17187, 11073, 14320, 20478] avg=16384.0000 max=21438 max_rank=2 min=11073 min_rank=5 imbalance_ratio=1.308472 max_load_reduction=109634 max_load_reduction_pct=0.836441 moved_to_echo=63058 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 3614, 14, 0, 3950, 1, 1192, 0, 0, 0, 1990, 1, 9, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 95, 105, 0, 0, 3962, 37, 8, 1, 39, 4091, 0, 759, 3144, 1166, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 239, 0, 0, 4077, 3, 0, 22, 3997, 0, 415, 0, 1658, 940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 16, 0, 1224, 0, 2314, 2, 2, 3906, 2255, 4009, 1833, 3350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3954, 0, 0, 3, 4094, 9, 48, 136, 4082, 0, 0, 0, 1, 0, 1135, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 5, 3836, 1, 3656, 1, 934, 0, 73, 0, 0, 0, 21, 3104, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 235, 182, 0, 0, 3, 89, 0, 574, 2, 4094, 5, 3, 4095, 3496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 8, 2961, 4072, 4093, 3, 2, 0, 863, 0, 3905, 0, 3777, 0, 2, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14884, 14016, 15432, 18996, 13610, 17546, 16806, 19782] avg=16384.0000 max=19782 max_rank=7 min=13610 min_rank=4 imbalance_ratio=1.207397 max_load_reduction=111290 max_load_reduction_pct=0.849075 moved_to_echo=67744 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3737, 0, 3693, 0, 0, 4094, 0, 28, 0, 0, 0, 1987, 3341, 1763, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 528, 0, 0, 155, 64, 0, 85, 173, 1, 1858, 0, 0, 4073, 1258, 619, 3088, 1164, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 5, 203, 39, 1, 2041, 0, 43, 4096, 0, 8, 779, 3423, 143, 600, 0, 1292, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 881, 24, 2141, 572, 569, 25, 0, 3864, 1758, 4017, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2825, 0, 4095, 0, 141, 2283, 4076, 1, 189, 61, 2089, 76, 0, 0, 0, 1, 1101, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4058, 1311, 3278, 1938, 917, 0, 4052, 0, 6, 0, 10, 23, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4020, 46, 496, 0, 0, 106, 0, 32, 23, 0, 0, 3992, 0, 0, 4096, 3482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2871, 3180, 3971, 3, 0, 0, 3928, 60, 3, 0, 11, 0, 11, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18669, 13660, 17688, 13872, 17040, 19805, 16295, 14043] avg=16384.0000 max=19805 max_rank=5 min=13660 min_rank=1 imbalance_ratio=1.208801 max_load_reduction=111267 max_load_reduction_pct=0.848900 moved_to_echo=67183 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4087, 0, 998, 0, 0, 4010, 0, 3450, 0, 0, 0, 1529, 3770, 47, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 104, 0, 0, 1397, 4096, 0, 0, 1942, 3944, 30, 313, 2712, 895, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 4063, 5, 3610, 0, 0, 576, 0, 363, 400, 38, 93, 200, 0, 1679, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 71, 0, 828, 10, 0, 135, 2283, 155, 153, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 3828, 0, 0, 0, 3213, 103, 35, 1, 1761, 0, 0, 0, 0, 14, 1135, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 122, 1517, 22, 3337, 4096, 3851, 805, 0, 3981, 14, 831, 0, 0, 4030, 1392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 18, 2, 1, 2480, 0, 0, 19, 138, 4089, 2816, 0, 198, 41, 1, 4096, 3501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 323, 2961, 4093, 732, 0, 0, 572, 32, 4042, 891, 0, 2, 0, 19, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17966, 16032, 11768, 11825, 14277, 24198, 17403, 17603] avg=16384.0000 max=24198 max_rank=5 min=11768 min_rank=2 imbalance_ratio=1.476929 max_load_reduction=106874 max_load_reduction_pct=0.815384 moved_to_echo=73481 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[27136, 128, 26624, 1024, 256, 26752, 128, 15744, 128, 0, 128, 14080, 16768, 2304, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[23424, 512, 384, 5248, 640, 14848, 512, 512, 25472, 128, 2688, 7296, 22784, 1408, 5248, 1280, 12800, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[6144, 1536, 23552, 26112, 20352, 2816, 128, 1152, 9216, 4352, 9728, 256, 5376, 128, 6144, 15104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[14848, 128, 27648, 384, 256, 7808, 31488, 3072, 3456, 384, 26368, 4096, 128, 1664, 128, 256, 9088, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 384, 19072, 1536, 3456, 3968, 128, 256, 128, 384, 8192, 10112, 128, 23936, 256, 128, 32768, 27392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 1920, 256, 128, 4608, 384, 128, 384, 18432, 9344, 13568, 256, 5248, 31744, 1536, 6656, 24704, 8320, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[896, 0, 8704, 4608, 1280, 128, 7936, 2048, 9856, 128, 3840, 20736, 17408, 26496, 11392, 16512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[512, 20096, 5376, 9344, 8192, 31360, 9344, 7424, 768, 17920, 128, 896, 128, 5632, 7936, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[3, 3728, 0, 10, 6, 0, 0, 0, 4095, 347, 200, 4081, 4096, 3984, 0, 7, 933, 0, 0, 4064, 3768, 0, 12, 0, 0, 0, 3, 0, 4089, 1, 63, 2050, 0, 0, 1903, 88, 697, 12, 3794, 1439, 0, 1307, 296, 1196, 4, 615, 0, 10, 5, 4091, 193, 75, 4028, 16, 4084, 2337, 0, 0, 63, 4090, 0, 0, 0, 0, 2, 865, 32, 24, 2885, 0, 529, 0, 3924, 17, 4033, 2865, 2, 0, 0, 4, 124, 0, 0, 3538, 4096, 0, 0, 3578, 3082, 2, 33, 0, 171, 4096, 3769, 962, 0, 2, 4093, 0, 3566, 23, 33, 1916, 1186, 0, 647, 0, 0, 1, 2837, 0, 232, 4096, 91, 312, 2587, 25, 0, 0, 0, 0, 2920, 0, 60, 0, 1929, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 637, 2, 357, 1659, 0, 3652, 4095, 4094, 3059, 0, 4081, 4026, 0, 6, 124, 5, 4, 7, 31, 2521, 47, 44, 0, 3217, 2936, 3686, 4041, 0, 3958, 441, 0, 950, 0, 0, 2, 0, 3920, 32, 0, 29, 0, 1967, 0, 25, 7, 3, 97, 4061, 1413, 3020, 117, 2, 3687, 4052, 22, 0, 0, 4096, 0, 0, 0, 3, 2260, 2001, 2, 438, 4077, 10, 431, 0, 1856, 0, 0, 3118, 42, 79, 1283, 0, 9, 0, 15, 2012, 4036, 2, 4080, 4070, 4071, 0, 14, 0, 10, 159, 1027, 259, 0, 13, 3944, 0, 40, 17, 0, 13, 21, 2997, 1317, 51, 4, 1, 14, 158, 0, 4096, 0, 0, 4073, 20, 0, 50, 61, 0, 3980, 0, 552, 0, 1, 52] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 462, 2385, 103, 2, 0, 1576, 0, 4077, 3756, 188, 3643, 4009, 0, 0, 0, 5, 2069, 0, 551, 287, 1, 0, 243, 0, 3877, 88, 45, 3465, 0, 1, 4094, 5, 0, 1462, 0, 99, 222, 46, 0, 0, 2123, 1, 3425, 329, 3470, 543, 0, 158, 4096, 1, 15, 3715, 3426, 4075, 3962, 1021, 0, 4086, 4088, 0, 0, 225, 0, 3, 4092, 10, 1, 30, 0, 34, 0, 2460, 4, 0, 3982, 2, 8, 24, 7, 1, 0, 0, 2196, 2333, 0, 78, 4089, 102, 0, 51, 1, 0, 3867, 59, 14, 7, 249, 3917, 0, 3804, 1237, 0, 3806, 0, 0, 114, 0, 0, 69, 26, 0, 0, 2921, 4044, 4, 3590, 1, 20, 0, 369, 0, 4096, 0, 3770, 3, 57, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4, 0, 4096, 838, 499, 2106, 4094, 488, 3238, 1422, 125, 3871, 4096, 193, 472, 0, 3, 4094, 0, 5, 3578, 2, 0, 0, 0, 1752, 22, 0, 2, 10, 1486, 22, 221, 0, 1, 982, 3447, 0, 86, 715, 452, 3, 1679, 2, 9, 15, 113, 0, 0, 10, 3651, 0, 4093, 68, 3785, 4095, 170, 0, 3441, 2614, 21, 0, 0, 0, 0, 958, 1017, 0, 4093, 0, 4, 137, 126, 0, 0, 155, 4, 1, 0, 241, 0, 0, 0, 4029, 3644, 37, 0, 4088, 1, 10, 31, 0, 61, 1026, 0, 3587, 0, 0, 3996, 137, 2871, 4095, 66, 1, 0, 358, 3152, 0, 0, 26, 13, 38, 0, 4013, 3529, 0, 3020, 3330, 0, 0, 0, 1732, 3893, 0, 3279, 0, 4082, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 2358, 3453, 4030, 3201, 167, 4046, 236, 3871, 4096, 0, 1087, 3188, 3665, 4008, 954, 4064, 2172, 0, 11, 1360, 1579, 0, 2, 1, 3887, 0, 12, 66, 2844, 67, 3973, 23, 0, 3, 1, 0, 1897, 1, 2, 1, 4006, 741, 9, 1, 141, 2403, 0, 107, 2029, 0, 0, 4075, 0, 479, 66, 3871, 23, 0, 4076, 731, 0, 72, 3, 21, 147, 5, 1709, 5, 0, 0, 3588, 771, 21, 1, 3995, 0, 123, 0, 520, 1, 370, 1, 294, 3761, 0, 1079, 4050, 963, 3247, 1158, 21, 1, 284, 0, 17, 0, 53, 679, 467, 3, 2008, 360, 4092, 0, 0, 882, 0, 40, 0, 362, 0, 0, 66, 0, 17, 3919, 1, 1, 31, 449, 0, 4096, 0, 3916, 0, 0, 318] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[252, 3920, 4085, 79, 0, 138, 3811, 4089, 4093, 4088, 4016, 1657, 3993, 0, 0, 0, 0, 172, 169, 683, 487, 1770, 0, 0, 0, 4096, 3876, 1, 122, 16, 0, 1869, 484, 34, 40, 0, 266, 0, 1284, 4025, 1, 1546, 3465, 828, 3313, 3, 0, 0, 63, 4095, 0, 0, 1342, 123, 4096, 1391, 2, 0, 4, 4091, 0, 0, 2, 95, 6, 1537, 11, 10, 18, 1884, 9, 0, 68, 0, 0, 638, 3, 3, 0, 38, 22, 2038, 0, 4027, 3740, 3, 0, 4083, 0, 0, 4090, 0, 0, 0, 12, 3045, 0, 2831, 4078, 719, 2867, 5, 0, 3934, 2, 0, 15, 0, 1439, 0, 2, 15, 0, 3537, 0, 0, 29, 181, 0, 0, 799, 3, 4057, 0, 3189, 0, 4, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 1, 4086, 1339, 3, 0, 3139, 0, 2225, 3806, 3664, 4020, 4096, 341, 0, 3, 51, 20, 0, 2, 3900, 5, 22, 0, 0, 4086, 4, 63, 10, 6, 3554, 2833, 23, 1, 1, 0, 0, 297, 3111, 473, 0, 123, 132, 3177, 11, 275, 0, 8, 0, 4053, 1410, 14, 3809, 1593, 4056, 4088, 0, 0, 2, 4096, 0, 0, 0, 0, 122, 56, 4029, 0, 2249, 159, 1722, 0, 1376, 0, 0, 4029, 29, 0, 277, 0, 0, 374, 0, 4085, 4092, 0, 0, 4086, 0, 26, 76, 0, 0, 4096, 1, 1774, 0, 261, 4092, 0, 1820, 114, 4, 2027, 0, 0, 440, 0, 0, 2, 2107, 0, 0, 4039, 8, 0, 4093, 2, 0, 0, 3, 0, 3471, 0, 3874, 0, 25, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 170, 145, 4037, 213, 643, 4086, 3402, 1262, 61, 3807, 4096, 4055, 0, 9, 119, 39, 0, 3944, 4045, 18, 0, 0, 806, 4033, 68, 0, 1361, 1, 3163, 117, 34, 0, 8, 49, 377, 0, 26, 432, 7, 1, 2, 0, 0, 1, 1, 0, 0, 23, 1, 923, 1698, 0, 4096, 399, 43, 0, 0, 2718, 27, 0, 6, 3861, 6, 3837, 39, 0, 2849, 0, 0, 11, 3, 0, 413, 4087, 1, 1, 5, 10, 2403, 0, 257, 3931, 3446, 0, 1966, 4072, 0, 0, 40, 0, 0, 1265, 2747, 20, 0, 3758, 4095, 2216, 228, 79, 68, 0, 29, 289, 1, 693, 619, 20, 4092, 0, 0, 4088, 3570, 8, 2804, 2695, 4, 0, 0, 0, 4021, 0, 3999, 0, 3847, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 3728, 0, 10, 6, 0, 0, 0, 0, 0, 200, 3609, 0, 3984, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 933, 0, 0, 4064, 3768, 0, 12, 0, 0, 0, 3, 0, 4089, 1, 63, 2050, 472, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1903, 88, 697, 12, 3794, 1439, 0, 1307, 296, 1196, 4, 615, 0, 10, 4096, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4091, 193, 75, 4028, 16, 4084, 2337, 0, 0, 63, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 865, 32, 24, 2885, 0, 529, 0, 3924, 17, 4033, 2865, 2, 0, 0, 4, 3703, 226, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 3538, 4096, 0, 0, 1739, 3082, 2, 33, 0, 171, 4096, 3769, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 3566, 23, 33, 1916, 1186, 0, 647, 0, 0, 1, 2837, 0, 1839, 121, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 4096, 91, 312, 2587, 25, 0, 0, 0, 0, 2444, 0, 60, 0, 1929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11547, 15568, 19479, 15279, 19184, 21612, 16627, 11776] avg=16384.0000 max=21612 max_rank=5 min=11547 min_rank=0 imbalance_ratio=1.319092 max_load_reduction=109460 max_load_reduction_pct=0.835114 moved_to_echo=69199 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 637, 2, 357, 1659, 0, 3652, 0, 0, 3059, 0, 0, 4026, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 5, 4, 7, 31, 2521, 47, 44, 0, 3217, 2936, 3686, 4041, 0, 3958, 441, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 950, 0, 0, 2, 0, 3920, 32, 0, 29, 0, 1967, 0, 25, 7, 3, 4081, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 4061, 1413, 3020, 117, 2, 3687, 4052, 22, 0, 0, 387, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2260, 2001, 2, 438, 4077, 10, 431, 0, 1856, 0, 0, 3118, 42, 79, 1283, 0, 3709, 2666, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 15, 2012, 4036, 2, 4080, 1978, 4071, 0, 14, 0, 10, 159, 1027, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3944, 0, 40, 17, 0, 13, 21, 2997, 1317, 51, 4, 1, 14, 158, 2092, 1428, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4073, 20, 0, 50, 61, 0, 3331, 0, 552, 0, 1, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13399, 21216, 15038, 16861, 22045, 17672, 12605, 12236] avg=16384.0000 max=22045 max_rank=4 min=12236 min_rank=7 imbalance_ratio=1.345520 max_load_reduction=109027 max_load_reduction_pct=0.831810 moved_to_echo=64558 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4096, 838, 499, 2106, 4094, 488, 0, 0, 125, 3418, 0, 193, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 5, 3578, 2, 0, 0, 0, 1752, 22, 0, 2, 10, 1486, 22, 453, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 0, 1, 982, 3447, 0, 86, 715, 452, 3, 1679, 2, 9, 15, 113, 0, 4096, 3181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3651, 0, 4093, 68, 3785, 4095, 170, 0, 3441, 244, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 958, 1017, 0, 4093, 0, 4, 137, 126, 0, 0, 155, 4, 1, 0, 241, 2370, 927, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 3644, 37, 0, 1983, 1, 10, 31, 0, 61, 1026, 0, 3587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 137, 2871, 4095, 66, 1, 0, 358, 3152, 0, 0, 26, 13, 38, 2105, 495, 489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 3529, 0, 3020, 3330, 0, 0, 0, 1732, 3248, 0, 3279, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16333, 11585, 15002, 19578, 10090, 14409, 17842, 26233] avg=16384.0000 max=26233 max_rank=7 min=10090 min_rank=4 imbalance_ratio=1.601135 max_load_reduction=104839 max_load_reduction_pct=0.799858 moved_to_echo=68574 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 462, 2385, 103, 2, 0, 1576, 0, 0, 0, 188, 3222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2069, 0, 551, 287, 1, 0, 243, 0, 3877, 88, 45, 3465, 0, 1, 4094, 421, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1462, 0, 99, 222, 46, 0, 0, 2123, 1, 3425, 329, 3470, 543, 0, 4009, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 4096, 1, 15, 3715, 3426, 4075, 3962, 1021, 0, 4086, 387, 0, 0, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4092, 10, 1, 30, 0, 34, 0, 2460, 4, 0, 3982, 2, 8, 24, 7, 3701, 2446, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2196, 2333, 0, 78, 1987, 102, 0, 51, 1, 0, 3867, 59, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 249, 3917, 0, 3804, 1237, 0, 3806, 0, 0, 114, 0, 0, 69, 26, 0, 2102, 1310, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2921, 4044, 4, 3590, 1, 20, 0, 369, 0, 3427, 0, 3770, 3, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7938, 15306, 19739, 25167, 16876, 10689, 17151, 18206] avg=16384.0000 max=25167 max_rank=3 min=7938 min_rank=0 imbalance_ratio=1.536072 max_load_reduction=105905 max_load_reduction_pct=0.807991 moved_to_echo=62922 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[252, 3920, 4085, 79, 0, 138, 3811, 4089, 0, 0, 4016, 1466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 169, 683, 487, 1770, 0, 0, 0, 4096, 3876, 1, 122, 16, 0, 1869, 191, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 34, 40, 0, 266, 0, 1284, 4025, 1, 1546, 3465, 828, 3313, 3, 0, 0, 3993, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 4095, 0, 0, 1342, 123, 4096, 1391, 2, 0, 4, 387, 0, 0, 2, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1537, 11, 10, 18, 1884, 9, 0, 68, 0, 0, 638, 3, 3, 0, 38, 3704, 2663, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 2038, 0, 4027, 3740, 3, 0, 1984, 0, 0, 4090, 0, 0, 0, 12, 3045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2831, 4078, 719, 2867, 5, 0, 3934, 2, 0, 15, 0, 1439, 0, 2, 15, 2099, 1425, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3537, 0, 0, 29, 181, 0, 0, 799, 3, 3395, 0, 3189, 0, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21856, 13609, 23303, 11600, 10664, 18961, 19936, 11143] avg=16384.0000 max=23303 max_rank=2 min=10664 min_rank=4 imbalance_ratio=1.422302 max_load_reduction=107769 max_load_reduction_pct=0.822212 moved_to_echo=60704 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4086, 1339, 3, 0, 3139, 0, 0, 0, 3664, 3555, 0, 341, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 20, 0, 2, 3900, 5, 22, 0, 0, 4086, 4, 63, 10, 6, 3554, 2833, 465, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1, 1, 0, 0, 297, 3111, 473, 0, 123, 132, 3177, 11, 275, 0, 8, 4096, 2186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 1410, 14, 3809, 1593, 4056, 4088, 0, 0, 2, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 56, 4029, 0, 2249, 159, 1722, 0, 1376, 0, 0, 4029, 29, 0, 277, 0, 3709, 2479, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 0, 4085, 4092, 0, 0, 1986, 0, 26, 76, 0, 0, 4096, 1, 1774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 4092, 0, 1820, 114, 4, 2027, 0, 0, 440, 0, 0, 2, 2107, 0, 2100, 1327, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 8, 0, 4093, 2, 0, 0, 3, 0, 2904, 0, 3874, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16131, 15156, 13914, 19412, 20275, 16510, 14726, 14948] avg=16384.0000 max=20275 max_rank=4 min=13914 min_rank=2 imbalance_ratio=1.237488 max_load_reduction=110797 max_load_reduction_pct=0.845314 moved_to_echo=66459 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 2358, 3453, 4030, 3201, 167, 4046, 236, 0, 0, 0, 962, 0, 3665, 4008, 954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 2172, 0, 11, 1360, 1579, 0, 2, 1, 3887, 0, 12, 66, 2844, 67, 3973, 125, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 3, 1, 0, 1897, 1, 2, 1, 4006, 741, 9, 1, 141, 2403, 0, 3188, 3803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 2029, 0, 0, 4075, 0, 479, 66, 3871, 23, 0, 385, 731, 0, 72, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 147, 5, 1709, 5, 0, 0, 3588, 771, 21, 1, 3995, 0, 123, 0, 520, 3691, 2669, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 370, 1, 294, 3761, 0, 1079, 1968, 963, 3247, 1158, 21, 1, 284, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 679, 467, 3, 2008, 360, 4092, 0, 0, 882, 0, 40, 0, 362, 0, 2082, 1427, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 17, 3919, 1, 1, 31, 449, 0, 3427, 0, 3916, 0, 0, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[27080, 20322, 16220, 11841, 17334, 13165, 12965, 12145] avg=16384.0000 max=27080 max_rank=0 min=11841 min_rank=3 imbalance_ratio=1.652832 max_load_reduction=103992 max_load_reduction_pct=0.793396 moved_to_echo=55609 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 170, 145, 4037, 213, 643, 4086, 0, 0, 61, 3367, 0, 4055, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 39, 0, 3944, 4045, 18, 0, 0, 806, 4033, 68, 0, 1361, 1, 3163, 117, 440, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 8, 49, 377, 0, 26, 432, 7, 1, 2, 0, 0, 1, 1, 0, 4096, 3343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1, 923, 1698, 0, 4096, 399, 43, 0, 0, 257, 27, 0, 6, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3837, 39, 0, 2849, 0, 0, 11, 3, 0, 413, 4087, 1, 1, 5, 10, 2461, 822, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2403, 0, 257, 3931, 3446, 0, 1966, 1979, 0, 0, 40, 0, 0, 1265, 2747, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3758, 4095, 2216, 228, 79, 68, 0, 29, 289, 1, 693, 619, 20, 4092, 0, 2093, 440, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3570, 8, 2804, 2695, 4, 0, 0, 0, 3365, 0, 3999, 0, 3847, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16788, 18310, 8377, 11334, 14604, 18054, 19220, 24385] avg=16384.0000 max=24385 max_rank=7 min=8377 min_rank=2 imbalance_ratio=1.488342 max_load_reduction=106687 max_load_reduction_pct=0.813957 moved_to_echo=76263 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[384, 10496, 18944, 6656, 8192, 4352, 17408, 12672, 0, 0, 11392, 19712, 0, 16384, 4480, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[896, 1024, 3456, 1152, 4992, 2432, 12288, 7168, 512, 9216, 6400, 10624, 3712, 4608, 3072, 128, 31744, 28672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[256, 26880, 11264, 384, 24192, 6272, 128, 128, 1792, 1792, 25600, 0, 22656, 128, 9984, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[5376, 8576, 256, 9344, 17536, 6016, 128, 384, 896, 24960, 7040, 3840, 13184, 2944, 12416, 15488, 2688, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[2432, 13568, 5248, 2304, 16256, 2176, 2816, 3840, 10624, 128, 4480, 22912, 128, 256, 1664, 896, 27136, 14976, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[2560, 2816, 384, 24192, 29184, 128, 7296, 15616, 8320, 3328, 5504, 128, 256, 14848, 7680, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 7168, 28928, 3584, 15232, 7680, 640, 15872, 1280, 3712, 6656, 768, 2176, 128, 9472, 256, 16512, 8064, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[512, 22528, 6784, 4096, 22912, 5248, 28416, 20480, 5248, 128, 7680, 2944, 896, 0, 384, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[2283, 3028, 4096, 0, 0, 0, 0, 74, 2356, 0, 6, 3993, 0, 47, 199, 1, 3423, 5, 48, 4094, 0, 6, 2726, 4091, 1, 53, 4046, 833, 0, 3853, 501, 0, 28, 0, 1, 157, 0, 0, 3851, 4039, 3210, 3157, 18, 28, 0, 3065, 4076, 4096, 0, 13, 5, 3630, 0, 0, 3769, 777, 150, 130, 224, 32, 3901, 323, 454, 1, 210, 2069, 215, 3844, 8, 121, 1973, 0, 4039, 3, 0, 0, 0, 2, 0, 0, 1, 0, 10, 0, 4094, 8, 0, 0, 9, 1, 0, 0, 0, 0, 0, 27, 1101, 84, 12, 2933, 166, 0, 4074, 11, 0, 907, 31, 3909, 0, 3, 1104, 4088, 0, 264, 4, 0, 1, 356, 4091, 0, 102, 94, 64, 4069, 3429, 0, 585, 4058] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[3878, 6, 3977, 0, 595, 4, 0, 0, 0, 0, 4061, 4044, 0, 5, 0, 414, 7, 0, 74, 3518, 3679, 0, 0, 4096, 0, 2030, 1433, 3, 0, 4072, 85, 0, 0, 2497, 20, 0, 19, 0, 4096, 4009, 0, 3507, 4015, 14, 65, 0, 2, 4096, 0, 2293, 2, 3002, 4068, 0, 0, 4096, 0, 0, 2, 0, 596, 0, 2716, 21, 0, 1, 6, 1200, 1, 41, 4089, 0, 4093, 0, 34, 0, 7, 1067, 0, 0, 0, 0, 57, 4031, 4070, 0, 0, 0, 0, 0, 3, 0, 4070, 0, 0, 71, 15, 5, 0, 2883, 264, 0, 4094, 337, 11, 0, 0, 4095, 0, 1, 1456, 3977, 0, 4028, 0, 0, 0, 874, 4058, 0, 566, 0, 0, 4089, 1936, 0, 21, 404] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[3698, 3579, 4084, 0, 0, 15, 61, 0, 14, 0, 0, 4077, 114, 0, 0, 5, 0, 0, 1, 3536, 35, 0, 3304, 1497, 0, 27, 401, 992, 201, 0, 4096, 0, 0, 1749, 1514, 109, 0, 0, 2241, 4092, 3971, 10, 4077, 0, 332, 1, 1, 477, 0, 2315, 0, 3960, 4024, 0, 0, 4081, 0, 0, 0, 7, 4095, 23, 0, 4095, 3, 0, 4079, 0, 405, 0, 138, 0, 1986, 0, 0, 0, 6, 39, 0, 7, 6, 50, 0, 0, 4019, 6, 2, 0, 619, 0, 19, 0, 3953, 13, 74, 1065, 0, 32, 0, 4096, 0, 0, 4045, 55, 3936, 0, 0, 3544, 0, 29, 4025, 3761, 4, 3886, 0, 0, 1, 0, 4013, 0, 0, 0, 11, 4064, 3918, 158, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[1369, 0, 3656, 0, 32, 159, 55, 1, 505, 0, 0, 3822, 0, 1, 1, 3448, 3914, 0, 0, 4076, 117, 6, 4, 486, 0, 0, 4096, 0, 4089, 3633, 4091, 0, 8, 1653, 13, 0, 0, 0, 4096, 3148, 1, 4081, 3943, 0, 0, 0, 2572, 4042, 3, 3941, 0, 773, 9, 7, 48, 3520, 0, 0, 0, 0, 3190, 2, 3329, 3852, 157, 0, 4046, 0, 0, 23, 0, 11, 1, 0, 1921, 3319, 0, 0, 309, 3898, 52, 0, 402, 531, 4095, 0, 0, 1, 20, 12, 0, 0, 4076, 0, 58, 0, 0, 0, 0, 4096, 0, 11, 3992, 7, 3927, 0, 0, 0, 782, 120, 104, 4078, 0, 4093, 44, 0, 164, 0, 3795, 0, 0, 1048, 0, 19, 2068, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[3881, 1149, 3198, 0, 0, 4, 115, 0, 4092, 0, 0, 3623, 0, 2, 0, 168, 5, 0, 5, 4089, 306, 0, 4058, 0, 4075, 0, 3305, 2, 65, 4091, 4090, 0, 401, 3753, 28, 0, 1, 0, 3970, 28, 425, 4094, 167, 0, 0, 0, 3669, 136, 1, 4085, 0, 3063, 3996, 901, 3, 126, 874, 0, 2, 0, 1146, 2, 4009, 1800, 20, 3, 0, 1, 4084, 1, 7, 36, 146, 0, 4093, 0, 0, 0, 62, 3063, 0, 3997, 1462, 780, 3965, 0, 0, 4091, 0, 638, 0, 1200, 0, 8, 111, 6, 0, 1178, 0, 144, 0, 173, 4095, 0, 0, 0, 0, 3, 282, 8, 54, 3992, 0, 0, 889, 0, 0, 0, 3093, 0, 0, 1019, 0, 3695, 4042, 0, 3628, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[3959, 3884, 4094, 0, 35, 0, 3, 11, 4047, 0, 0, 3781, 11, 3436, 14, 47, 335, 0, 110, 3229, 4096, 0, 1, 693, 0, 4055, 4041, 1455, 849, 4059, 0, 0, 4084, 3788, 320, 4065, 38, 744, 1659, 0, 410, 4096, 126, 0, 1, 59, 0, 2178, 0, 8, 3, 0, 4012, 3928, 2, 2643, 7, 0, 37, 0, 4048, 2447, 0, 3953, 131, 188, 2586, 4075, 7, 63, 10, 0, 6, 0, 1, 0, 1, 0, 4, 4093, 34, 0, 0, 3969, 237, 431, 0, 0, 116, 41, 0, 1178, 3978, 0, 0, 0, 0, 1952, 199, 84, 44, 0, 3989, 105, 3902, 95, 0, 0, 9, 336, 5, 1582, 0, 3109, 0, 11, 0, 2, 1083, 1, 70, 36, 0, 3972, 0, 430, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4076, 0, 4096, 1, 614, 0, 0, 0, 2891, 0, 0, 4096, 0, 6, 180, 3786, 3918, 0, 0, 3346, 0, 0, 0, 4007, 0, 120, 3998, 348, 2236, 4072, 4078, 0, 117, 2157, 1111, 0, 2, 0, 4095, 2661, 2, 0, 1481, 3, 14, 2, 131, 918, 0, 627, 0, 3652, 203, 0, 379, 4096, 0, 0, 0, 0, 4029, 0, 4095, 4093, 0, 2, 229, 1, 1, 0, 2, 0, 1061, 0, 675, 0, 0, 3, 0, 1709, 0, 104, 0, 0, 4032, 0, 0, 4086, 0, 0, 0, 3859, 4092, 0, 0, 62, 532, 2310, 0, 4094, 0, 0, 4096, 4, 3, 0, 0, 7, 884, 6, 330, 4087, 3, 3963, 0, 0, 0, 14, 4096, 0, 3, 1, 1, 3577, 2731, 0, 0, 675] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[237, 2, 3961, 0, 0, 1, 2137, 0, 4085, 0, 0, 1, 0, 20, 0, 3145, 1270, 0, 381, 2409, 4095, 0, 3141, 518, 0, 0, 15, 1, 46, 2582, 4094, 0, 2484, 4083, 12, 0, 0, 0, 4095, 1, 4003, 4092, 1731, 0, 0, 0, 4, 57, 1, 3518, 0, 3984, 4096, 38, 7, 3855, 245, 0, 0, 0, 4096, 4, 92, 966, 0, 0, 72, 9, 2152, 1, 4067, 0, 30, 0, 196, 0, 0, 3929, 7, 449, 0, 0, 1452, 4, 2299, 0, 0, 4, 0, 4, 0, 2192, 22, 0, 1, 0, 0, 3400, 3576, 200, 0, 0, 3909, 0, 4077, 0, 0, 4084, 1, 20, 144, 4041, 87, 4092, 0, 0, 26, 0, 4095, 0, 0, 1, 935, 4096, 3655, 0, 0, 138] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3878, 6, 3521, 0, 595, 4, 0, 0, 0, 0, 4061, 4044, 0, 5, 0, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 74, 0, 3679, 0, 0, 4096, 0, 2030, 1433, 3, 0, 1972, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2497, 20, 0, 19, 0, 0, 4009, 0, 3124, 4015, 14, 65, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2293, 2, 3002, 4068, 0, 0, 4096, 0, 0, 2, 0, 37, 0, 2716, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 1200, 1, 41, 4089, 0, 4093, 0, 34, 0, 7, 1067, 0, 0, 3518, 3225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4031, 4070, 0, 0, 0, 0, 0, 3, 0, 4070, 0, 0, 71, 559, 2100, 871, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 5, 0, 2883, 264, 0, 3187, 337, 11, 0, 0, 4095, 0, 1, 1456, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 874, 4058, 0, 566, 0, 0, 4089, 1936, 0, 21, 404, 456, 408, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16528, 13379, 17861, 16237, 17282, 16331, 16231, 17223] avg=16384.0000 max=17861 max_rank=2 min=13379 min_rank=1 imbalance_ratio=1.090149 max_load_reduction=113211 max_load_reduction_pct=0.863731 moved_to_echo=67067 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2283, 3028, 3626, 0, 0, 0, 0, 74, 2356, 0, 6, 3993, 0, 47, 199, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3423, 5, 48, 0, 0, 6, 2726, 4091, 1, 53, 4046, 833, 0, 1866, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 1, 157, 0, 0, 0, 4039, 3210, 2812, 18, 28, 0, 3065, 4076, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 5, 3630, 0, 0, 3769, 777, 150, 130, 224, 32, 237, 323, 454, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 2069, 215, 3844, 8, 121, 1973, 0, 4039, 3, 0, 0, 0, 2, 0, 0, 4094, 3032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 10, 0, 4094, 8, 0, 0, 9, 1, 0, 0, 0, 0, 0, 27, 3664, 1987, 819, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1101, 84, 12, 2933, 166, 0, 3172, 11, 0, 907, 31, 3909, 0, 3, 1104, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 4, 0, 1, 356, 4091, 0, 102, 94, 64, 4069, 3429, 0, 585, 4058, 470, 406, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15613, 17599, 21530, 9745, 19610, 11116, 17521, 18338] avg=16384.0000 max=21530 max_rank=2 min=9745 min_rank=3 imbalance_ratio=1.314087 max_load_reduction=109542 max_load_reduction_pct=0.835739 moved_to_echo=66585 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3698, 3579, 3615, 0, 0, 15, 61, 0, 14, 0, 0, 4077, 114, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 35, 0, 3304, 1497, 0, 27, 401, 992, 201, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1749, 1514, 109, 0, 0, 0, 4092, 3971, 9, 4077, 0, 332, 1, 1, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2315, 0, 3960, 4024, 0, 0, 4081, 0, 0, 0, 7, 249, 23, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4079, 0, 405, 0, 138, 0, 1986, 0, 0, 0, 6, 39, 0, 7, 3536, 1764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 50, 0, 0, 4019, 6, 2, 0, 619, 0, 19, 0, 3953, 13, 74, 1065, 3846, 0, 477, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 4096, 0, 0, 3148, 55, 3936, 0, 0, 3544, 0, 29, 4025, 3761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3886, 0, 0, 1, 0, 4013, 0, 0, 0, 11, 4064, 3918, 158, 0, 4094, 469, 404, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15178, 10554, 16332, 18754, 11963, 14642, 22626, 21023] avg=16384.0000 max=22626 max_rank=6 min=10554 min_rank=1 imbalance_ratio=1.380981 max_load_reduction=108446 max_load_reduction_pct=0.827377 moved_to_echo=70254 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1369, 0, 3236, 0, 32, 159, 55, 1, 505, 0, 0, 3822, 0, 1, 1, 3448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 0, 0, 0, 117, 6, 4, 486, 0, 0, 4096, 0, 4089, 1759, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1653, 13, 0, 0, 0, 0, 3148, 1, 3635, 3943, 0, 0, 0, 2572, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3941, 0, 773, 9, 7, 48, 3520, 0, 0, 0, 0, 194, 2, 3329, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 4046, 0, 0, 23, 0, 11, 1, 0, 1921, 3319, 0, 0, 309, 3898, 4076, 3226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 402, 531, 4095, 0, 0, 1, 20, 12, 0, 0, 4076, 0, 58, 0, 2996, 1874, 870, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 3108, 7, 3927, 0, 0, 0, 782, 120, 104, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 44, 0, 164, 0, 3795, 0, 0, 1048, 0, 19, 2068, 0, 0, 0, 420, 398, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12629, 18562, 19015, 15678, 20987, 15473, 16233, 12495] avg=16384.0000 max=20987 max_rank=4 min=12495 min_rank=7 imbalance_ratio=1.280945 max_load_reduction=110085 max_load_reduction_pct=0.839882 moved_to_echo=65188 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3881, 1149, 2828, 0, 0, 4, 115, 0, 4092, 0, 0, 3623, 0, 2, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5, 0, 306, 0, 4058, 0, 4075, 0, 3305, 2, 65, 1977, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 3753, 28, 0, 1, 0, 0, 28, 425, 3645, 167, 0, 0, 0, 3669, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 0, 3063, 3996, 901, 3, 126, 874, 0, 2, 0, 65, 2, 4009, 1800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 3, 0, 1, 4084, 1, 7, 36, 146, 0, 4093, 0, 0, 0, 62, 3063, 4089, 3126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 1462, 780, 3965, 0, 0, 4091, 0, 638, 0, 1200, 0, 8, 111, 6, 1081, 2114, 844, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1178, 0, 144, 0, 173, 3178, 0, 0, 0, 0, 3, 282, 8, 54, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 889, 0, 0, 0, 3093, 0, 0, 1019, 0, 3695, 4042, 0, 3628, 0, 370, 413, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15862, 17888, 12253, 18927, 18731, 20801, 9012, 17598] avg=16384.0000 max=20801 max_rank=5 min=9012 min_rank=6 imbalance_ratio=1.269592 max_load_reduction=110271 max_load_reduction_pct=0.841301 moved_to_echo=66142 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3959, 3884, 3624, 0, 35, 0, 3, 11, 4047, 0, 0, 3781, 11, 3436, 14, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 110, 0, 4096, 0, 1, 693, 0, 4055, 4041, 1455, 849, 1966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 3788, 320, 4065, 38, 744, 0, 0, 410, 3649, 126, 0, 1, 59, 0, 2178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 4012, 3928, 2, 2643, 7, 0, 37, 0, 246, 2447, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 188, 2586, 4075, 7, 63, 10, 0, 6, 0, 1, 0, 1, 0, 4, 4093, 3229, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 3969, 237, 431, 0, 0, 116, 41, 0, 1178, 3978, 0, 0, 0, 3802, 2093, 352, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1952, 199, 84, 44, 0, 3105, 105, 3902, 95, 0, 0, 9, 336, 5, 1582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3109, 0, 11, 0, 2, 1083, 1, 70, 36, 0, 3972, 0, 430, 0, 6, 470, 398, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22852, 17601, 19462, 17286, 15701, 16717, 11418, 10035] avg=16384.0000 max=22852 max_rank=0 min=10035 min_rank=7 imbalance_ratio=1.394775 max_load_reduction=108220 max_load_reduction_pct=0.825653 moved_to_echo=53871 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4076, 0, 3626, 1, 614, 0, 0, 0, 2891, 0, 0, 4096, 0, 6, 180, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 4007, 0, 120, 3998, 348, 2236, 1972, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 2157, 1111, 0, 2, 0, 0, 2661, 2, 0, 1481, 3, 14, 2, 131, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 627, 0, 3652, 203, 0, 379, 4096, 0, 0, 0, 0, 245, 0, 4095, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 229, 1, 1, 0, 2, 0, 1061, 0, 675, 0, 0, 3, 0, 1709, 3346, 3224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 4032, 0, 0, 4086, 0, 0, 0, 3859, 4092, 0, 0, 62, 3784, 2100, 871, 499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 532, 2310, 0, 4094, 0, 0, 3188, 4, 3, 0, 0, 7, 884, 6, 330, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3963, 0, 0, 0, 14, 4096, 0, 3, 1, 1, 3577, 2731, 0, 0, 675, 470, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19276, 20677, 8599, 17390, 10253, 23489, 15445, 15943] avg=16384.0000 max=23489 max_rank=5 min=8599 min_rank=2 imbalance_ratio=1.433655 max_load_reduction=107583 max_load_reduction_pct=0.820793 moved_to_echo=65130 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[237, 2, 3506, 0, 0, 1, 2137, 0, 4085, 0, 0, 1, 0, 20, 0, 3145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1270, 0, 381, 0, 4095, 0, 3141, 518, 0, 0, 15, 1, 46, 1251, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2484, 4083, 12, 0, 0, 0, 0, 1, 4003, 3645, 1731, 0, 0, 0, 4, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3518, 0, 3984, 4096, 38, 7, 3855, 245, 0, 0, 0, 249, 4, 92, 966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 9, 2152, 1, 4067, 0, 30, 0, 196, 0, 0, 3929, 7, 449, 2409, 3224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1452, 4, 2299, 0, 0, 4, 0, 4, 0, 2192, 22, 0, 1, 0, 3847, 1331, 871, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3400, 3576, 200, 0, 0, 3043, 0, 4077, 0, 0, 4084, 1, 20, 144, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 4092, 0, 0, 26, 0, 4095, 0, 0, 1, 935, 4096, 3655, 0, 0, 138, 455, 390, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13134, 14812, 16020, 17055, 16545, 12503, 22586, 18417] avg=16384.0000 max=22586 max_rank=6 min=12503 min_rank=5 imbalance_ratio=1.378540 max_load_reduction=108486 max_load_reduction_pct=0.827682 moved_to_echo=70051 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 16896, 128, 22144, 20480, 4992, 4224, 23296, 1280, 256, 384, 128, 1536, 2816, 14720, 18816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[23424, 11648, 27648, 128, 1280, 256, 2432, 128, 18048, 0, 4096, 27520, 128, 3584, 512, 11136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12928, 128, 640, 0, 12416, 128, 13312, 15488, 4096, 6400, 21376, 3712, 7552, 12800, 21120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1664, 9088, 3840, 18560, 512, 256, 25216, 640, 15872, 1024, 128, 15744, 2048, 640, 7296, 29696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[640, 2304, 11264, 9216, 6784, 256, 10368, 128, 11392, 128, 7040, 3328, 128, 5120, 384, 13312, 28416, 22144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[7168, 19712, 3072, 4352, 128, 768, 0, 18048, 12032, 20608, 15616, 128, 512, 3200, 10496, 16000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 4224, 3456, 9344, 26880, 512, 128, 8192, 768, 768, 128, 8448, 20224, 128, 256, 1280, 23680, 13696, 6016, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 23552, 1024, 128, 256, 1280, 28416, 128, 768, 2304, 1024, 27648, 21888, 640, 4352, 9472, 3584, 3328, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1, 33, 0, 1, 0, 3845, 0, 3218, 67, 2966, 0, 0, 194, 0, 0, 3207, 0, 1223, 0, 0, 0, 4096, 3779, 3204, 0, 3330, 3127, 0, 259, 0, 4096, 21, 0, 76, 55, 1, 3704, 2, 0, 3205, 4, 0, 3141, 4096, 2370, 2348, 1, 12, 1, 0, 0, 3958, 0, 1391, 3120, 0, 113, 4092, 1689, 0, 151, 1944, 0, 418, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 3252, 0, 0, 3966, 4, 2317, 1653, 788, 79, 4044, 0, 1111, 3526, 0, 0, 33, 4088, 0, 0, 306, 3, 523, 4095, 193, 77, 174, 16, 0, 3939, 72, 394, 2, 3893, 0, 3954, 0, 146, 7, 1, 0, 1479, 57, 4095, 0, 0, 3924, 15, 70, 0, 0, 2172, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[25, 1, 0, 0, 1, 440, 0, 0, 0, 122, 0, 0, 0, 0, 3, 0, 0, 3, 0, 3761, 9, 3170, 4047, 4046, 17, 4025, 3862, 0, 1155, 33, 4094, 0, 0, 4067, 2157, 2105, 3556, 0, 53, 2429, 86, 0, 3275, 4087, 4031, 0, 0, 0, 0, 0, 0, 709, 0, 1690, 2860, 341, 87, 54, 349, 0, 16, 4096, 4083, 2057, 3, 0, 0, 0, 0, 5, 0, 3547, 1, 2489, 3798, 0, 3638, 1113, 4048, 3509, 2885, 0, 4094, 217, 8, 217, 2, 45, 0, 4079, 0, 104, 0, 0, 103, 4089, 2, 675, 0, 421, 0, 26, 0, 56, 4058, 4096, 0, 4048, 133, 134, 0, 0, 0, 30, 4063, 0, 0, 0, 61, 0, 0, 3, 794, 3470, 4, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4094, 12, 13, 17, 0, 4093, 11, 4093, 2, 0, 942, 0, 7, 0, 18, 0, 0, 1, 0, 2058, 87, 4085, 0, 4048, 3176, 0, 4096, 7, 155, 0, 4094, 0, 3850, 19, 1667, 0, 0, 4085, 248, 0, 3227, 0, 0, 4094, 0, 0, 0, 2687, 0, 3931, 1, 2817, 280, 0, 28, 0, 59, 0, 0, 0, 4096, 78, 0, 722, 454, 3, 2607, 0, 0, 12, 0, 4, 4087, 9, 4095, 0, 11, 0, 2849, 0, 1377, 0, 4080, 996, 4, 0, 3, 2021, 4057, 0, 4, 3302, 0, 322, 3385, 0, 1, 0, 63, 0, 0, 5, 2099, 4096, 0, 2884, 9, 1426, 5, 0, 105, 0, 4096, 4087, 142, 0, 4094, 1450, 1731, 0, 11, 4087, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[52, 0, 232, 8, 0, 475, 6, 4069, 80, 527, 1, 2, 1820, 7, 202, 2696, 0, 3484, 0, 25, 0, 4095, 3574, 148, 1, 3212, 3852, 1, 0, 0, 4095, 0, 3066, 3742, 436, 0, 9, 117, 2202, 4096, 0, 244, 3448, 3301, 4096, 3383, 0, 0, 0, 31, 0, 0, 30, 356, 0, 45, 4, 0, 1307, 14, 3921, 2, 3980, 0, 0, 0, 0, 0, 6, 0, 21, 4096, 14, 0, 4096, 499, 1478, 424, 19, 22, 4079, 13, 1654, 63, 2961, 0, 0, 3, 119, 4068, 0, 0, 350, 3487, 28, 3889, 103, 0, 355, 0, 0, 3, 0, 780, 2679, 4051, 1409, 2385, 492, 1544, 0, 5, 0, 131, 3881, 3052, 0, 161, 4042, 149, 2, 2, 0, 1825, 2134, 0, 2, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 3877, 71, 57, 0, 0, 4096, 3, 4095, 0, 0, 7, 0, 4, 64, 2796, 0, 0, 8, 0, 2690, 291, 2, 0, 4, 4096, 58, 9, 2, 4027, 0, 31, 0, 685, 701, 0, 161, 1395, 0, 9, 64, 2046, 3643, 415, 4059, 0, 3746, 0, 31, 0, 1, 61, 4083, 730, 65, 44, 0, 3929, 0, 0, 0, 4096, 1298, 117, 162, 2822, 4, 3574, 0, 648, 3889, 3, 0, 4055, 251, 4091, 0, 4055, 2155, 3803, 0, 1617, 1619, 3863, 7, 1, 2188, 4086, 0, 66, 0, 4, 101, 168, 3509, 401, 0, 11, 0, 3, 23, 0, 1377, 3480, 4096, 39, 569, 32, 39, 3946, 0, 0, 0, 4096, 4028, 0, 0, 0, 0, 3645, 0, 215, 185, 265, 0, 214, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 2, 27, 6, 0, 118, 0, 4086, 0, 4065, 0, 0, 3068, 0, 0, 220, 213, 3, 0, 1957, 0, 4016, 4040, 455, 0, 4094, 4063, 346, 75, 0, 3965, 0, 4072, 2, 0, 1, 467, 728, 127, 3716, 0, 0, 3786, 4057, 22, 4089, 0, 3, 3, 2515, 0, 1695, 0, 3980, 0, 0, 0, 0, 4096, 832, 17, 0, 4070, 0, 0, 0, 0, 0, 48, 0, 0, 3544, 0, 258, 9, 110, 4091, 6, 4096, 3886, 4096, 0, 3163, 1238, 3976, 212, 0, 0, 23, 208, 0, 0, 564, 0, 316, 4039, 0, 8, 57, 21, 0, 1, 3, 0, 3977, 4094, 2, 3597, 0, 642, 5, 1, 0, 18, 3919, 2, 0, 6, 4092, 26, 0, 0, 0, 3620, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 952, 0, 0, 1162, 23, 4089, 0, 2974, 0, 308, 3, 0, 767, 0, 4007, 0, 10, 0, 266, 95, 4096, 4092, 5, 3831, 3976, 2691, 3734, 0, 0, 6, 314, 0, 0, 14, 4080, 0, 7, 3662, 0, 78, 93, 660, 2048, 3348, 0, 81, 0, 0, 0, 0, 850, 3149, 3601, 64, 4094, 0, 1, 23, 3297, 0, 3600, 9, 48, 570, 0, 54, 4040, 8, 108, 4010, 0, 727, 4075, 0, 4040, 1816, 158, 0, 1026, 157, 4059, 1, 0, 1, 2, 25, 6, 0, 1298, 676, 495, 4065, 0, 4082, 1518, 4, 0, 0, 1376, 194, 0, 12, 3810, 4094, 137, 4, 3961, 2707, 272, 13, 42, 1, 9, 0, 0, 14, 0, 0, 3825, 11, 4, 2847, 440, 70, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 8, 162, 6, 20, 2728, 0, 4095, 12, 4069, 0, 0, 4, 0, 1, 344, 1, 1174, 0, 0, 0, 0, 205, 4096, 0, 4094, 3688, 22, 4096, 52, 0, 0, 392, 0, 0, 92, 0, 26, 401, 4065, 3356, 2989, 357, 4076, 3335, 2, 0, 0, 0, 4, 0, 4022, 0, 0, 2, 0, 0, 0, 4073, 0, 2902, 0, 4095, 0, 0, 0, 0, 530, 1, 0, 229, 1742, 0, 0, 4092, 4088, 3840, 0, 4, 0, 2987, 0, 4091, 0, 3742, 957, 374, 0, 3915, 0, 2, 1, 3, 4041, 3623, 1, 7, 0, 14, 0, 4091, 15, 0, 24, 1076, 2125, 13, 103, 9, 4077, 4094, 0, 448, 0, 4031, 0, 0, 0, 4088, 0, 5, 0, 0, 4071, 595, 0, 857, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 33, 0, 1, 0, 3845, 0, 3218, 67, 2966, 0, 0, 194, 0, 0, 3207, 3127, 2290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1223, 0, 0, 0, 4096, 3779, 3204, 0, 1817, 0, 0, 259, 0, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 55, 1, 3704, 2, 0, 0, 4, 0, 3141, 2478, 2370, 2348, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3958, 0, 1391, 3120, 0, 113, 4092, 1689, 0, 151, 1944, 0, 418, 684, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 3252, 0, 0, 3282, 4, 2317, 1653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 788, 79, 421, 0, 1111, 3526, 0, 0, 33, 4088, 0, 0, 306, 3, 523, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 77, 174, 16, 0, 3939, 72, 394, 2, 3893, 0, 3954, 0, 146, 7, 1, 1369, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1479, 57, 4095, 0, 0, 3924, 15, 70, 0, 0, 2172, 0, 0, 0, 1, 3623, 915, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18949, 18495, 14192, 18212, 14552, 14973, 15204, 16495] avg=16384.0000 max=18949 max_rank=0 min=14192 min_rank=2 imbalance_ratio=1.156555 max_load_reduction=112123 max_load_reduction_pct=0.855431 moved_to_echo=61224 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[25, 1, 0, 0, 1, 440, 0, 0, 0, 122, 0, 0, 0, 0, 3, 0, 3862, 1736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3761, 9, 3170, 4047, 4046, 17, 2197, 0, 0, 1155, 33, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 2157, 2105, 3556, 0, 53, 0, 86, 0, 3275, 2472, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 709, 0, 1690, 2860, 341, 87, 54, 349, 0, 16, 4096, 4083, 2057, 628, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 5, 0, 3547, 1, 2489, 3798, 0, 3010, 1113, 4048, 3509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2885, 0, 426, 217, 8, 217, 2, 45, 0, 4079, 0, 104, 0, 0, 103, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 675, 0, 421, 0, 26, 0, 56, 4058, 4096, 0, 4048, 133, 134, 0, 0, 1654, 965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4063, 0, 0, 0, 61, 0, 0, 3, 794, 3470, 4, 0, 0, 2, 3668, 693, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6190, 22532, 21802, 17620, 21523, 12175, 16268, 12962] avg=16384.0000 max=22532 max_rank=1 min=6190 min_rank=0 imbalance_ratio=1.375244 max_load_reduction=108540 max_load_reduction_pct=0.828094 moved_to_echo=62928 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 12, 13, 17, 0, 4093, 11, 4093, 2, 0, 942, 0, 7, 0, 3176, 2920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 1, 0, 2058, 87, 4085, 0, 2200, 0, 0, 4096, 7, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3850, 19, 1667, 0, 0, 0, 248, 0, 3227, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2687, 0, 3931, 1, 2817, 280, 0, 28, 0, 59, 0, 0, 0, 4096, 78, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 722, 454, 3, 2607, 0, 0, 12, 0, 4, 4087, 9, 3385, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2849, 0, 140, 0, 4080, 996, 4, 0, 3, 2021, 4057, 0, 4, 3302, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3385, 0, 1, 0, 63, 0, 0, 5, 2099, 4096, 0, 2884, 9, 1426, 5, 0, 1667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 4096, 4087, 142, 0, 4094, 1450, 1731, 0, 11, 4087, 0, 0, 1, 0, 1237, 1165, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19380, 12707, 17199, 14687, 11294, 17778, 15640, 22387] avg=16384.0000 max=22387 max_rank=7 min=11294 min_rank=4 imbalance_ratio=1.366394 max_load_reduction=108685 max_load_reduction_pct=0.829201 moved_to_echo=67099 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[52, 0, 232, 8, 0, 475, 6, 4069, 80, 527, 1, 2, 1820, 7, 202, 2696, 3852, 2927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3484, 0, 25, 0, 4095, 3574, 148, 1, 1753, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3066, 3742, 436, 0, 9, 117, 2202, 0, 0, 244, 3448, 1997, 4096, 3383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 30, 356, 0, 45, 4, 0, 1307, 14, 3921, 2, 3980, 0, 255, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 21, 4096, 14, 0, 4096, 499, 1223, 424, 19, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 13, 172, 63, 2961, 0, 0, 3, 119, 4068, 0, 0, 350, 3487, 28, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 355, 0, 0, 3, 0, 780, 2679, 4051, 1409, 2385, 492, 1544, 0, 5, 1320, 779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 3881, 3052, 0, 161, 4042, 149, 2, 2, 0, 1825, 2134, 0, 2, 2, 1482, 1169, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16956, 17176, 22740, 10470, 10420, 19232, 15905, 18173] avg=16384.0000 max=22740 max_rank=2 min=10420 min_rank=4 imbalance_ratio=1.387939 max_load_reduction=108332 max_load_reduction_pct=0.826508 moved_to_echo=63730 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3877, 71, 57, 0, 0, 4096, 3, 4095, 0, 0, 7, 0, 4, 64, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2796, 0, 0, 8, 0, 2690, 291, 2, 0, 3, 0, 58, 9, 2, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 685, 701, 0, 161, 1395, 0, 9, 64, 2046, 2204, 415, 4059, 0, 3746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 1, 61, 4083, 730, 65, 44, 0, 3929, 0, 0, 0, 4096, 1298, 706, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 162, 2822, 4, 3574, 0, 648, 3889, 3, 0, 4055, 251, 3385, 0, 4055, 2155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3803, 0, 169, 1619, 3863, 7, 1, 2188, 4086, 0, 66, 0, 4, 101, 168, 3509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 0, 11, 0, 3, 23, 0, 1377, 3480, 4096, 39, 569, 32, 39, 3946, 0, 1, 860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4028, 0, 0, 0, 0, 3645, 0, 215, 185, 265, 0, 214, 0, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16370, 9886, 15516, 15623, 25120, 19584, 14877, 14096] avg=16384.0000 max=25120 max_rank=4 min=9886 min_rank=1 imbalance_ratio=1.533203 max_load_reduction=105952 max_load_reduction_pct=0.808350 moved_to_echo=73677 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 27, 6, 0, 118, 0, 4086, 0, 4065, 0, 0, 3068, 0, 0, 220, 4063, 2657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 3, 0, 1957, 0, 4016, 4040, 455, 0, 2234, 0, 346, 75, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 2, 0, 1, 467, 728, 127, 0, 0, 0, 3786, 2454, 22, 4089, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2515, 0, 1695, 0, 3980, 0, 0, 0, 0, 4096, 832, 17, 0, 4070, 0, 706, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 3544, 0, 258, 9, 110, 3385, 6, 4096, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 329, 1238, 3976, 212, 0, 0, 23, 208, 0, 0, 564, 0, 316, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 57, 21, 0, 1, 3, 0, 3977, 4094, 2, 3597, 0, 642, 5, 1, 1683, 958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3919, 2, 0, 6, 4092, 26, 0, 0, 0, 3620, 0, 0, 0, 1, 2834, 1059, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18312, 17304, 15751, 18559, 15342, 15001, 15049, 15754] avg=16384.0000 max=18559 max_rank=3 min=15001 min_rank=5 imbalance_ratio=1.132751 max_load_reduction=112513 max_load_reduction_pct=0.858406 moved_to_echo=61146 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 952, 0, 0, 1162, 23, 4089, 0, 2974, 0, 308, 3, 0, 767, 0, 3976, 2618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 10, 0, 266, 95, 4096, 4092, 5, 2090, 0, 2691, 3734, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 14, 4080, 0, 7, 0, 0, 78, 93, 393, 2048, 3348, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 850, 3149, 3601, 64, 4094, 0, 1, 23, 3297, 0, 3600, 9, 697, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 570, 0, 54, 4040, 8, 108, 4010, 0, 727, 4075, 0, 3343, 1816, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 157, 423, 1, 0, 1, 2, 25, 6, 0, 1298, 676, 495, 4065, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1518, 4, 0, 0, 1376, 194, 0, 12, 3810, 4094, 137, 4, 3961, 2707, 272, 13, 1575, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 1, 9, 0, 0, 14, 0, 0, 3825, 11, 4, 2847, 440, 70, 0, 0, 3636, 1044, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16872, 21092, 10456, 19493, 18957, 12257, 19836, 12109] avg=16384.0000 max=21092 max_rank=1 min=10456 min_rank=2 imbalance_ratio=1.287354 max_load_reduction=109980 max_load_reduction_pct=0.839081 moved_to_echo=63159 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 162, 6, 20, 2728, 0, 4095, 12, 4069, 0, 0, 4, 0, 1, 344, 3688, 2906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1174, 0, 0, 0, 0, 205, 4096, 0, 2234, 0, 22, 4096, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 0, 92, 0, 26, 401, 0, 3356, 2989, 357, 2466, 3335, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4022, 0, 0, 2, 0, 0, 0, 4073, 0, 2902, 0, 4095, 0, 662, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 530, 1, 0, 229, 1742, 0, 0, 4092, 4088, 3178, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2987, 0, 426, 0, 3742, 957, 374, 0, 3915, 0, 2, 1, 3, 4041, 3623, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 14, 0, 4091, 15, 0, 24, 1076, 2125, 13, 103, 9, 4077, 4094, 0, 1683, 962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 448, 0, 4031, 0, 0, 0, 4088, 0, 5, 0, 0, 4071, 595, 0, 857, 0, 3665, 1159, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18043, 11880, 13416, 16408, 13864, 20072, 18293, 19096] avg=16384.0000 max=20072 max_rank=5 min=11880 min_rank=1 imbalance_ratio=1.225098 max_load_reduction=111000 max_load_reduction_pct=0.846863 moved_to_echo=71325 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[22528, 256, 2560, 3200, 19840, 6016, 384, 2304, 8192, 14464, 5504, 896, 1792, 15104, 4864, 24064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12032, 7936, 7296, 2944, 13568, 1152, 4224, 0, 3712, 3456, 19456, 14464, 16384, 21376, 128, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 9344, 128, 128, 8832, 128, 27776, 256, 22912, 128, 384, 6144, 128, 1024, 6656, 29952, 18176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[5632, 768, 640, 512, 5632, 4224, 128, 2688, 21248, 30592, 1664, 17664, 4736, 10752, 8448, 128, 11008, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[640, 1664, 24192, 15360, 256, 256, 20352, 1664, 9344, 128, 1024, 22400, 3456, 128, 1152, 128, 21632, 7296, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 1536, 3328, 640, 10368, 128, 1024, 24960, 128, 6784, 24320, 4992, 24192, 3456, 14720, 11264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[7040, 5888, 128, 5760, 384, 20224, 20224, 20224, 128, 14592, 0, 3200, 13440, 128, 20480, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 5376, 0, 14336, 1024, 17536, 10624, 640, 4480, 4224, 15616, 896, 10368, 6144, 28032, 3968, 5120, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 19, 0, 143, 4068, 0, 2, 2558, 2, 3428, 0, 4090, 4094, 0, 185, 0, 2, 0, 0, 63, 0, 0, 3721, 4072, 2939, 4063, 0, 4007, 0, 0, 38, 0, 725, 11, 73, 5, 4092, 0, 0, 3, 0, 2, 0, 8, 332, 322, 4076, 4096, 0, 0, 0, 3, 4090, 3168, 3999, 4095, 1, 0, 0, 0, 4096, 0, 4069, 0, 0, 0, 1955, 3708, 0, 0, 21, 4094, 0, 0, 2137, 0, 217, 0, 2494, 0, 1, 4096, 466, 0, 0, 63, 0, 0, 824, 0, 599, 21, 3634, 4088, 16, 3503, 27, 3469, 30, 0, 0, 4096, 12, 1638, 3596, 0, 0, 0, 0, 16, 1664, 0, 3707, 1, 1543, 10, 0, 23, 0, 34, 487, 4019, 2, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 3696, 0, 0, 0, 0, 4096, 0, 2, 14, 32, 1909, 1, 3984, 4096, 6, 3317, 0, 0, 0, 16, 2937, 0, 0, 4085, 35, 21, 0, 0, 1228, 0, 0, 7, 1, 2217, 4035, 118, 3821, 4092, 0, 2070, 0, 0, 2, 0, 212, 0, 43, 0, 3777, 191, 92, 77, 2066, 0, 4095, 4069, 3687, 242, 0, 0, 0, 4096, 3, 408, 0, 0, 0, 87, 4067, 0, 10, 92, 4093, 15, 0, 0, 190, 270, 53, 2964, 0, 3646, 4096, 0, 573, 0, 0, 13, 3868, 1249, 0, 15, 0, 4, 1118, 35, 0, 0, 3893, 3609, 5, 0, 1402, 0, 4003, 4035, 0, 11, 90, 4092, 1672, 0, 0, 4061, 0, 101, 25, 0, 6, 1, 3967, 9, 4069, 489, 178] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[103, 0, 4096, 0, 10, 564, 0, 64, 2240, 0, 274, 0, 0, 4068, 0, 4033, 4050, 0, 0, 106, 60, 0, 0, 0, 0, 18, 3555, 3478, 295, 0, 4, 4013, 0, 1, 32, 0, 3930, 1746, 0, 2868, 3935, 99, 1253, 15, 0, 1867, 0, 0, 137, 1, 1, 3873, 639, 0, 3434, 163, 3905, 0, 3235, 918, 167, 5, 0, 3, 4061, 0, 1, 5, 0, 0, 949, 4, 34, 39, 12, 423, 0, 32, 0, 0, 12, 0, 4091, 0, 3831, 4091, 40, 0, 4062, 1, 171, 0, 4079, 0, 4, 0, 1156, 4071, 0, 3730, 128, 3966, 2608, 654, 7, 605, 2, 0, 4043, 336, 0, 211, 79, 0, 4087, 29, 4029, 0, 3854, 0, 160, 42, 0, 4095, 0, 3842, 109, 29] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[8, 10, 3829, 4092, 0, 87, 473, 0, 0, 281, 0, 197, 0, 4096, 37, 3712, 2216, 3953, 0, 9, 0, 0, 0, 50, 0, 4056, 4096, 4096, 0, 0, 0, 0, 11, 0, 0, 3786, 4041, 108, 0, 3434, 408, 214, 2, 0, 0, 7, 2591, 0, 0, 0, 0, 3985, 1, 1013, 827, 1, 3468, 0, 3998, 769, 0, 0, 0, 0, 2750, 0, 4042, 0, 3935, 0, 23, 1, 0, 0, 0, 4071, 4, 0, 0, 400, 4096, 585, 0, 0, 722, 3473, 350, 317, 85, 3089, 0, 3796, 4032, 0, 0, 0, 864, 4096, 0, 2, 0, 3924, 3769, 1, 1, 0, 14, 0, 697, 0, 0, 22, 1197, 3940, 287, 149, 335, 0, 3795, 1, 0, 0, 0, 6, 4096, 4096, 0, 47] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 25, 0, 1503, 677, 19, 0, 2, 2538, 0, 0, 5, 4, 4063, 1148, 4029, 4095, 0, 0, 22, 2, 0, 0, 0, 0, 0, 4071, 3885, 0, 0, 0, 2695, 0, 1, 795, 0, 954, 3245, 0, 2115, 3989, 0, 54, 124, 0, 0, 0, 0, 0, 4081, 0, 4096, 5, 0, 3142, 22, 4092, 4096, 3347, 4085, 220, 0, 0, 0, 4096, 0, 4078, 0, 0, 0, 133, 0, 0, 0, 16, 3818, 0, 0, 9, 0, 167, 0, 4095, 0, 0, 4096, 3, 1, 33, 618, 0, 3845, 75, 0, 0, 5, 356, 4004, 29, 1023, 0, 3357, 919, 0, 1, 6, 1425, 0, 4049, 0, 0, 0, 0, 3979, 3, 0, 3983, 1, 4095, 2, 0, 0, 0, 4096, 3326, 4081, 2, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[3887, 0, 2, 1113, 42, 0, 1760, 0, 35, 186, 744, 0, 0, 3682, 1, 208, 3947, 11, 0, 0, 711, 74, 161, 1, 0, 0, 3647, 744, 45, 0, 144, 3995, 46, 0, 0, 276, 3432, 14, 49, 0, 4096, 333, 4096, 152, 0, 22, 0, 0, 99, 40, 0, 4027, 10, 0, 3900, 3222, 135, 0, 4096, 3745, 0, 734, 2362, 0, 90, 181, 4036, 0, 0, 0, 4096, 5, 40, 0, 0, 4064, 8, 0, 19, 0, 4, 984, 0, 744, 4054, 4094, 0, 4002, 4080, 0, 37, 2166, 0, 0, 0, 1, 1, 4095, 0, 3243, 0, 0, 4057, 0, 1, 0, 0, 0, 2277, 1, 3, 0, 3927, 2892, 358, 35, 3944, 482, 19, 0, 0, 6, 4053, 1459, 4055, 3351, 0, 82] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1, 1030, 3304, 1304, 1788, 182, 353, 0, 3698, 1, 0, 0, 485, 4096, 79, 3698, 975, 122, 0, 0, 0, 25, 0, 0, 0, 282, 4096, 4095, 0, 752, 0, 0, 405, 0, 1, 0, 4068, 2855, 0, 0, 4095, 811, 1, 0, 0, 0, 0, 0, 0, 4052, 0, 3622, 0, 85, 4096, 489, 3686, 4087, 3895, 3865, 322, 0, 0, 0, 3763, 0, 3020, 12, 0, 0, 0, 1, 270, 46, 0, 3292, 2, 0, 0, 746, 1189, 4, 2525, 0, 4096, 3968, 0, 381, 9, 0, 7, 1157, 4095, 0, 0, 0, 3, 4019, 214, 0, 0, 15, 4024, 0, 2271, 0, 6, 0, 1325, 0, 80, 4084, 42, 2, 0, 53, 4096, 10, 4093, 1, 0, 0, 6, 20, 3711, 3612, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1947, 0, 0, 4095, 0, 111, 371, 0, 4066, 1, 0, 4086, 0, 132, 59, 4068, 1557, 0, 0, 0, 0, 0, 0, 90, 1, 6, 4096, 2313, 0, 0, 0, 3989, 0, 0, 0, 0, 7, 3688, 0, 0, 4096, 179, 1, 0, 0, 125, 61, 0, 0, 0, 0, 4087, 9, 8, 12, 1, 0, 4074, 3823, 1105, 0, 0, 0, 105, 4094, 0, 421, 2285, 3667, 0, 0, 4095, 0, 0, 0, 3679, 5, 0, 4016, 23, 3692, 4, 0, 0, 2236, 257, 3793, 237, 37, 817, 0, 4093, 1, 0, 0, 1, 7, 4096, 3327, 4077, 0, 3785, 37, 0, 1, 0, 0, 0, 2, 0, 0, 19, 3303, 4096, 4088, 291, 41, 0, 16, 17, 0, 33, 671, 663, 4029, 4096, 460, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 19, 0, 143, 4068, 0, 2, 2558, 2, 3428, 0, 4090, 501, 184, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 185, 0, 2, 0, 0, 63, 0, 0, 3721, 4072, 2939, 4063, 0, 4007, 609, 380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 725, 11, 73, 5, 4092, 0, 0, 3, 0, 2, 0, 8, 4096, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, 322, 4076, 0, 0, 0, 0, 3, 4090, 3168, 3815, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4069, 0, 0, 0, 1955, 3708, 0, 0, 21, 4094, 0, 0, 2137, 0, 2452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 0, 2494, 0, 1, 3939, 466, 0, 0, 63, 0, 0, 824, 0, 599, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3634, 3207, 16, 3503, 27, 3469, 30, 0, 0, 4096, 12, 1638, 3596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1664, 0, 3707, 1, 1543, 10, 0, 23, 0, 34, 487, 515, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15152, 24135, 9496, 19902, 22532, 8624, 23228, 8003] avg=16384.0000 max=24135 max_rank=1 min=8003 min_rank=7 imbalance_ratio=1.473083 max_load_reduction=106937 max_load_reduction_pct=0.815865 moved_to_echo=62387 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3696, 0, 0, 0, 0, 4096, 0, 2, 14, 32, 1909, 1, 3984, 137, 188, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 3317, 0, 0, 0, 16, 2937, 0, 0, 4085, 35, 21, 0, 0, 1228, 616, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 2217, 4035, 118, 3821, 4092, 0, 2070, 0, 0, 2, 0, 212, 3777, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 191, 92, 77, 2066, 0, 4095, 3881, 3687, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 408, 0, 0, 0, 87, 4067, 0, 10, 92, 4093, 15, 0, 0, 190, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 53, 2964, 0, 3646, 3939, 0, 573, 0, 0, 13, 3868, 1249, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 877, 35, 0, 0, 3893, 3609, 5, 0, 1402, 0, 4003, 4035, 0, 11, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1672, 0, 0, 4061, 0, 101, 25, 0, 6, 1, 3967, 9, 521, 489, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14216, 16461, 20801, 14374, 15544, 16590, 17964, 15122] avg=16384.0000 max=20801 max_rank=2 min=14216 min_rank=0 imbalance_ratio=1.269592 max_load_reduction=110271 max_load_reduction_pct=0.841301 moved_to_echo=65220 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[8, 10, 3829, 4092, 0, 87, 473, 0, 0, 281, 0, 197, 0, 4096, 37, 3712, 507, 188, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2216, 3953, 0, 9, 0, 0, 0, 50, 0, 4056, 4096, 4096, 0, 0, 0, 0, 625, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 3786, 4041, 108, 0, 3434, 408, 214, 2, 0, 0, 7, 2591, 0, 3985, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1013, 827, 1, 3468, 0, 3810, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2750, 0, 4042, 0, 3935, 0, 23, 1, 0, 0, 0, 4071, 4, 0, 0, 400, 2503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 585, 0, 0, 722, 3333, 350, 317, 85, 3089, 0, 3796, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 864, 3204, 0, 2, 0, 3924, 3769, 1, 1, 0, 14, 0, 697, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1197, 3940, 287, 149, 335, 0, 3795, 1, 0, 0, 0, 6, 4096, 512, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17657, 19486, 19043, 9889, 17729, 20405, 12498, 14365] avg=16384.0000 max=20405 max_rank=5 min=9889 min_rank=3 imbalance_ratio=1.245422 max_load_reduction=110667 max_load_reduction_pct=0.844322 moved_to_echo=64997 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[103, 0, 4096, 0, 10, 564, 0, 64, 2240, 0, 274, 0, 0, 4068, 0, 4033, 499, 149, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 106, 60, 0, 0, 0, 0, 18, 3555, 3478, 295, 0, 4, 4013, 582, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 32, 0, 3930, 1746, 0, 2868, 3935, 99, 1253, 15, 0, 1867, 0, 0, 3873, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 1, 1, 0, 639, 0, 3434, 163, 3905, 0, 3086, 918, 167, 5, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 1, 5, 0, 0, 949, 4, 34, 39, 12, 423, 0, 32, 0, 0, 2344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4091, 0, 3831, 3934, 40, 0, 4062, 1, 171, 0, 4079, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1156, 3193, 0, 3730, 128, 3966, 2608, 654, 7, 605, 2, 0, 4043, 336, 0, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 4087, 29, 4029, 0, 3854, 0, 160, 42, 0, 4095, 0, 492, 109, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16257, 16540, 20043, 12459, 7904, 20225, 20639, 17005] avg=16384.0000 max=20639 max_rank=6 min=7904 min_rank=4 imbalance_ratio=1.259705 max_load_reduction=110433 max_load_reduction_pct=0.842537 moved_to_echo=65773 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 25, 0, 1503, 677, 19, 0, 2, 2538, 0, 0, 5, 4, 4063, 1148, 4029, 491, 154, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 22, 2, 0, 0, 0, 0, 0, 4071, 3885, 0, 0, 0, 2695, 618, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 795, 0, 954, 3245, 0, 2115, 3989, 0, 54, 124, 0, 0, 0, 0, 4096, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 5, 0, 3142, 22, 4092, 4096, 3193, 4085, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4078, 0, 0, 0, 133, 0, 0, 0, 16, 3818, 0, 0, 9, 0, 2490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 4095, 0, 0, 3939, 3, 1, 33, 618, 0, 3845, 75, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 356, 3141, 29, 1023, 0, 3357, 919, 0, 1, 6, 1425, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 3, 0, 3983, 1, 4095, 2, 0, 0, 0, 4096, 3326, 523, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14815, 15760, 15823, 22936, 14640, 12781, 14306, 20011] avg=16384.0000 max=22936 max_rank=3 min=12781 min_rank=5 imbalance_ratio=1.399902 max_load_reduction=108136 max_load_reduction_pct=0.825012 moved_to_echo=61738 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 1030, 3304, 1304, 1788, 182, 353, 0, 3698, 1, 0, 0, 485, 4096, 79, 3698, 493, 180, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 975, 122, 0, 0, 0, 25, 0, 0, 0, 282, 4096, 4095, 0, 752, 0, 0, 547, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 0, 1, 0, 4068, 2855, 0, 0, 4095, 811, 1, 0, 0, 0, 0, 0, 3622, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 85, 4096, 489, 3686, 4087, 3715, 3865, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3763, 0, 3020, 12, 0, 0, 0, 1, 270, 46, 0, 3292, 2, 0, 0, 746, 2204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1189, 4, 2525, 0, 4096, 3816, 0, 381, 9, 0, 7, 1157, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3152, 214, 0, 0, 15, 4024, 0, 2271, 0, 6, 0, 1325, 0, 80, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 2, 0, 53, 4096, 10, 4093, 1, 0, 0, 6, 20, 3711, 462, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20844, 11268, 16257, 24397, 13356, 17279, 15174, 12497] avg=16384.0000 max=24397 max_rank=3 min=11268 min_rank=1 imbalance_ratio=1.489075 max_load_reduction=106675 max_load_reduction_pct=0.813866 moved_to_echo=58306 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3887, 0, 2, 1113, 42, 0, 1760, 0, 35, 186, 744, 0, 0, 3682, 1, 208, 502, 189, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 11, 0, 0, 711, 74, 161, 1, 0, 0, 3647, 744, 45, 0, 144, 3995, 508, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 276, 3432, 14, 49, 0, 4096, 333, 4096, 152, 0, 22, 0, 0, 4027, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 40, 0, 0, 10, 0, 3900, 3222, 135, 0, 3907, 3745, 0, 734, 2362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 181, 4036, 0, 0, 0, 4096, 5, 40, 0, 0, 4064, 8, 0, 19, 0, 2044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 984, 0, 744, 4054, 3937, 0, 4002, 4080, 0, 37, 2166, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3212, 0, 3243, 0, 0, 4057, 0, 1, 0, 0, 0, 2277, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 2892, 358, 35, 3944, 482, 19, 0, 0, 6, 4053, 1459, 4055, 429, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12508, 14369, 16913, 18154, 14583, 20009, 12795, 21741] avg=16384.0000 max=21741 max_rank=7 min=12508 min_rank=0 imbalance_ratio=1.326965 max_load_reduction=109331 max_load_reduction_pct=0.834129 moved_to_echo=69128 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1947, 0, 0, 4095, 0, 111, 371, 0, 4066, 1, 0, 4086, 0, 132, 59, 4068, 502, 176, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1557, 0, 0, 0, 0, 0, 0, 90, 1, 6, 4096, 2313, 0, 0, 0, 3989, 620, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3688, 0, 0, 4096, 179, 1, 0, 0, 125, 61, 0, 4087, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 12, 1, 0, 4074, 3647, 1105, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 421, 2285, 3667, 0, 0, 4095, 0, 0, 0, 3679, 5, 0, 4016, 23, 2499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3692, 4, 0, 0, 2236, 248, 3793, 237, 37, 817, 0, 4093, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3213, 3327, 4077, 0, 3785, 37, 0, 1, 0, 0, 0, 2, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3303, 4096, 4088, 291, 41, 0, 16, 17, 0, 33, 671, 663, 4029, 525, 460, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19623, 13053, 12696, 8961, 24784, 15159, 14468, 22328] avg=16384.0000 max=24784 max_rank=4 min=8961 min_rank=3 imbalance_ratio=1.512695 max_load_reduction=106288 max_load_reduction_pct=0.810913 moved_to_echo=76739 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[9728, 1664, 16256, 768, 18688, 27136, 4736, 5632, 8320, 4608, 256, 18944, 14464, 0, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[6016, 1152, 11264, 15872, 2560, 1024, 3072, 256, 20864, 512, 1024, 6912, 640, 25600, 1408, 27904, 3712, 1408, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[27136, 256, 20096, 2304, 7680, 0, 7296, 11904, 384, 128, 256, 27648, 128, 128, 6272, 1408, 19072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[512, 128, 896, 4096, 19456, 15744, 256, 12288, 28928, 1664, 7552, 384, 0, 2048, 2688, 256, 31616, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[25088, 4096, 3584, 256, 896, 128, 256, 3200, 128, 4480, 31488, 22784, 3328, 4864, 256, 19968, 4736, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12672, 16640, 10496, 640, 24320, 512, 17536, 128, 256, 128, 4736, 14464, 19840, 4096, 1152, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[6144, 23296, 3712, 15616, 256, 22528, 19072, 768, 2304, 6144, 1536, 5760, 20096, 384, 128, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[640, 8576, 4096, 0, 896, 1280, 15488, 6016, 19456, 19584, 29056, 22272, 1024, 768, 2432, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 58, 4088, 1057, 1230, 3826, 3904, 1, 41, 0, 1, 0, 0, 657, 646, 0, 114, 1022, 0, 0, 0, 4027, 3541, 3296, 0, 3381, 9, 0, 0, 1075, 727, 11, 3540, 0, 1757, 0, 0, 4072, 0, 3788, 270, 3899, 4096, 4084, 1237, 0, 0, 5, 0, 0, 0, 0, 3, 1332, 4096, 0, 0, 0, 0, 0, 422, 2528, 2298, 4, 121, 273, 0, 0, 0, 0, 3460, 8, 0, 4095, 66, 0, 2, 4093, 77, 255, 156, 3918, 4095, 4038, 1411, 154, 238, 1, 0, 0, 0, 4096, 0, 2478, 0, 4094, 0, 55, 16, 4096, 0, 0, 0, 0, 0, 0, 215, 4068, 0, 0, 0, 0, 4015, 16, 0, 574, 0, 4060, 0, 2541, 4091, 44, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[64, 0, 0, 0, 0, 1128, 3726, 2231, 4096, 4096, 0, 4, 37, 1, 0, 4, 0, 1038, 14, 0, 4092, 4047, 2, 0, 0, 1994, 3, 2516, 0, 0, 0, 11, 37, 40, 767, 0, 3033, 0, 4057, 4096, 550, 1258, 4096, 4047, 1, 2159, 4093, 4029, 0, 12, 0, 655, 0, 0, 0, 2, 0, 0, 0, 2305, 0, 5, 0, 2, 0, 4096, 2947, 3, 2, 1317, 0, 375, 0, 0, 2962, 0, 3973, 3756, 78, 2069, 0, 612, 0, 0, 326, 1214, 2, 3511, 2906, 22, 0, 5, 387, 50, 2, 0, 0, 2154, 55, 71, 0, 0, 0, 4044, 0, 0, 0, 2351, 0, 3281, 28, 3965, 1686, 280, 336, 0, 4092, 0, 464, 4088, 0, 4052, 797, 43, 3301, 4, 38, 979] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 33, 0, 3, 4001, 803, 4091, 549, 3992, 0, 0, 3366, 3947, 0, 0, 441, 205, 71, 1196, 1, 4096, 0, 0, 2147, 4095, 12, 2, 0, 1919, 0, 40, 126, 0, 11, 0, 3637, 0, 1527, 0, 3092, 4053, 0, 363, 2367, 3844, 2920, 0, 10, 445, 0, 166, 0, 0, 630, 0, 1, 3116, 4066, 0, 0, 9, 0, 0, 0, 1353, 3656, 0, 20, 3992, 0, 0, 0, 0, 3501, 0, 94, 3452, 0, 0, 0, 3329, 0, 23, 4095, 713, 3657, 4095, 0, 0, 4096, 0, 81, 0, 0, 1290, 0, 1131, 0, 486, 2, 1, 156, 4096, 3924, 0, 0, 0, 0, 0, 0, 3078, 0, 0, 2, 0, 3955, 0, 0, 64, 0, 3997, 0, 2394, 4096, 0, 4, 846] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[28, 2, 0, 368, 0, 51, 3995, 3577, 0, 4095, 9, 12, 0, 4093, 0, 0, 1, 1426, 0, 0, 3577, 0, 0, 0, 3258, 4025, 0, 5, 0, 4019, 0, 0, 142, 0, 4081, 0, 4039, 95, 2405, 0, 3874, 4096, 15, 1691, 4040, 3723, 4096, 20, 0, 5, 0, 77, 0, 4, 0, 0, 1, 176, 321, 796, 2, 1053, 0, 57, 0, 1, 1738, 0, 0, 0, 1735, 167, 0, 564, 3518, 0, 0, 4094, 0, 2739, 0, 1026, 63, 4010, 3, 0, 3453, 2876, 0, 0, 24, 0, 0, 15, 0, 3740, 0, 419, 2644, 4096, 0, 0, 6, 4081, 0, 285, 0, 2, 0, 3930, 4087, 3984, 0, 0, 0, 269, 4096, 4096, 1641, 3, 0, 186, 0, 0, 0, 0, 93, 38] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[3620, 0, 0, 0, 0, 3999, 4092, 4096, 2, 4096, 3394, 0, 0, 0, 1, 0, 42, 2247, 0, 0, 370, 2925, 26, 0, 0, 4096, 0, 0, 0, 14, 0, 0, 0, 86, 14, 5, 4082, 0, 3786, 3049, 3671, 583, 0, 185, 1068, 3123, 4054, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 6, 3, 0, 0, 0, 0, 14, 181, 4067, 0, 1, 743, 0, 0, 0, 0, 4096, 0, 8, 3719, 0, 3619, 1046, 274, 7, 22, 0, 49, 4084, 4072, 44, 0, 4002, 1, 0, 0, 0, 3767, 0, 339, 0, 1283, 0, 1, 13, 4096, 2, 0, 0, 0, 1909, 0, 4066, 4096, 0, 0, 2882, 0, 4009, 4096, 0, 277, 0, 4094, 0, 2, 3430, 3, 0, 833] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4079, 0, 161, 2137, 0, 356, 0, 4096, 0, 4079, 670, 81, 0, 15, 0, 260, 47, 2, 0, 203, 0, 81, 2, 0, 0, 4009, 0, 0, 0, 6, 0, 0, 0, 0, 738, 3991, 101, 2684, 189, 204, 2, 2365, 4096, 4057, 2537, 3864, 3663, 3717, 0, 381, 0, 38, 0, 6, 45, 0, 2, 7, 3656, 0, 1, 0, 0, 30, 0, 65, 11, 0, 0, 4036, 0, 0, 0, 3690, 3586, 0, 165, 4042, 0, 4096, 34, 88, 3625, 3944, 17, 0, 3883, 1697, 0, 0, 2287, 399, 10, 0, 0, 1265, 0, 4038, 0, 4, 0, 0, 0, 4046, 0, 0, 0, 3105, 10, 1699, 2964, 3472, 0, 1556, 2847, 0, 4091, 476, 0, 381, 0, 3354, 0, 0, 1272, 0, 4095, 64] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[85, 0, 3418, 0, 1763, 23, 0, 2200, 0, 1, 0, 2, 2307, 0, 4019, 0, 1527, 1, 1480, 37, 12, 3060, 12, 0, 225, 0, 923, 4034, 4070, 4096, 0, 7, 37, 0, 3, 0, 32, 4095, 4093, 0, 2841, 4, 4086, 4037, 2133, 4096, 3796, 12, 349, 137, 1018, 238, 0, 2, 0, 0, 0, 407, 0, 0, 0, 18, 0, 0, 0, 305, 0, 73, 486, 255, 56, 0, 0, 5, 4045, 0, 0, 399, 0, 2331, 0, 4096, 0, 4096, 3514, 406, 546, 4093, 3699, 9, 4056, 0, 0, 141, 0, 0, 232, 4096, 0, 0, 1072, 0, 0, 3914, 0, 0, 1, 551, 7, 26, 26, 3, 0, 3821, 4087, 11, 4045, 2058, 0, 87, 0, 4076, 3985, 6, 0, 0, 0, 1621] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4084, 0, 0, 24, 0, 4096, 4065, 4095, 1916, 3528, 2705, 0, 0, 1701, 0, 321, 308, 3452, 0, 0, 0, 34, 0, 0, 4, 4069, 0, 4002, 0, 4096, 0, 0, 228, 0, 543, 0, 558, 74, 2119, 660, 3971, 2519, 3908, 1344, 39, 2824, 4096, 3816, 0, 6, 4, 0, 0, 0, 0, 129, 1, 239, 0, 0, 5, 0, 0, 0, 3465, 1658, 4096, 0, 54, 1, 0, 8, 0, 4086, 48, 0, 0, 4093, 0, 27, 0, 3701, 150, 4090, 2, 0, 0, 4, 2303, 0, 12, 0, 7, 0, 0, 3656, 1234, 0, 0, 87, 0, 31, 0, 4091, 0, 1610, 155, 256, 3, 812, 4089, 3825, 0, 1132, 0, 0, 3638, 970, 4019, 0, 0, 395, 0, 40, 3341, 0, 40, 260] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 58, 4088, 1057, 1230, 1739, 3904, 1, 41, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 646, 0, 114, 1022, 0, 0, 0, 4027, 3541, 3296, 0, 3381, 9, 0, 1438, 1686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 727, 11, 3540, 0, 0, 0, 0, 4072, 0, 3655, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1237, 0, 0, 5, 0, 0, 0, 0, 3, 1332, 4096, 0, 0, 0, 0, 0, 4096, 3899, 1757, 4084, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 2528, 2298, 4, 121, 273, 0, 0, 0, 0, 3460, 8, 0, 4095, 66, 0, 485, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 77, 255, 156, 3918, 4095, 2336, 1411, 154, 238, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2478, 0, 4094, 0, 55, 16, 4096, 0, 0, 0, 0, 0, 0, 215, 4068, 218, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3312, 16, 0, 574, 0, 4060, 0, 2541, 4091, 44, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12125, 19817, 13350, 21158, 13776, 20832, 15373, 14641] avg=16384.0000 max=21158 max_rank=3 min=12125 min_rank=0 imbalance_ratio=1.291382 max_load_reduction=109914 max_load_reduction_pct=0.838577 moved_to_echo=64622 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 33, 0, 3, 4001, 803, 4091, 549, 1813, 0, 0, 3366, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 441, 205, 71, 1196, 1, 4096, 0, 0, 2147, 4095, 12, 2, 0, 1919, 0, 40, 1501, 1710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 11, 0, 3637, 0, 0, 0, 3092, 4053, 0, 351, 2367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 445, 0, 166, 0, 0, 630, 0, 1, 3116, 4066, 0, 0, 9, 0, 0, 2920, 3844, 1527, 0, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1353, 3656, 0, 20, 3992, 0, 0, 0, 0, 3501, 0, 94, 3452, 0, 0, 478, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3329, 0, 23, 4095, 713, 3657, 2369, 0, 0, 4096, 0, 81, 0, 0, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1131, 0, 486, 2, 1, 156, 4096, 3924, 0, 0, 0, 0, 0, 0, 3078, 215, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3262, 0, 0, 64, 0, 3997, 0, 2394, 4096, 0, 4, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18606, 17436, 13637, 17412, 16562, 19653, 13101, 14665] avg=16384.0000 max=19653 max_rank=5 min=13101 min_rank=6 imbalance_ratio=1.199524 max_load_reduction=111419 max_load_reduction_pct=0.850060 moved_to_echo=63981 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[64, 0, 0, 0, 0, 1128, 3726, 2231, 4096, 1861, 0, 4, 37, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 14, 0, 4092, 4047, 2, 0, 0, 1994, 3, 2516, 0, 0, 0, 11, 1540, 1466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 40, 767, 0, 3033, 0, 0, 4096, 550, 1258, 4096, 3905, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 655, 0, 0, 0, 2, 0, 0, 0, 2305, 0, 5, 0, 2, 4093, 2159, 4057, 4029, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2947, 3, 2, 1317, 0, 375, 0, 0, 2962, 0, 3973, 3756, 78, 2069, 495, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 612, 0, 0, 326, 1214, 2, 2031, 2906, 22, 0, 5, 387, 50, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2154, 55, 71, 0, 0, 0, 4044, 0, 0, 0, 2351, 0, 3281, 28, 3965, 223, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1686, 280, 336, 0, 3374, 0, 464, 4088, 0, 4052, 797, 43, 3301, 4, 38, 979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13152, 16723, 17783, 18014, 22087, 7557, 16314, 19442] avg=16384.0000 max=22087 max_rank=4 min=7557 min_rank=5 imbalance_ratio=1.348083 max_load_reduction=108985 max_load_reduction_pct=0.831490 moved_to_echo=65400 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[28, 2, 0, 368, 0, 51, 3995, 3577, 0, 1852, 9, 12, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1426, 0, 0, 3577, 0, 0, 0, 3258, 4025, 0, 5, 0, 4019, 0, 0, 1543, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 0, 4081, 0, 4039, 95, 0, 0, 3874, 4096, 15, 1628, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 77, 0, 4, 0, 0, 1, 176, 321, 796, 2, 1053, 0, 57, 4096, 3723, 2405, 20, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1738, 0, 0, 0, 1735, 167, 0, 564, 3518, 0, 0, 4094, 0, 2739, 498, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1026, 63, 4010, 3, 0, 3453, 1654, 0, 0, 24, 0, 0, 15, 0, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 419, 2644, 4096, 0, 0, 6, 4081, 0, 285, 0, 2, 0, 3930, 4087, 3984, 227, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 3371, 4096, 1641, 3, 0, 186, 0, 0, 0, 0, 93, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13987, 19060, 22010, 13436, 15070, 13988, 23824, 9697] avg=16384.0000 max=23824 max_rank=6 min=9697 min_rank=7 imbalance_ratio=1.454102 max_load_reduction=107248 max_load_reduction_pct=0.818237 moved_to_echo=62579 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4079, 0, 161, 2137, 0, 356, 0, 4096, 0, 1854, 670, 81, 0, 15, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 2, 0, 203, 0, 81, 2, 0, 0, 4009, 0, 0, 0, 6, 0, 0, 1533, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 738, 3991, 101, 2684, 0, 204, 2, 2365, 4096, 3914, 2537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 381, 0, 38, 0, 6, 45, 0, 2, 7, 3656, 0, 1, 0, 0, 30, 3663, 3864, 189, 3717, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 11, 0, 0, 4036, 0, 0, 0, 3690, 3586, 0, 165, 4042, 0, 4096, 495, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 88, 3625, 3944, 17, 0, 3883, 983, 0, 0, 2287, 399, 10, 0, 0, 1265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 4, 0, 0, 0, 4046, 0, 0, 0, 3105, 10, 1699, 2964, 3472, 222, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1556, 2847, 0, 3374, 476, 0, 381, 0, 3354, 0, 0, 1272, 0, 4095, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13709, 6591, 20632, 16291, 20192, 16535, 19703, 17419] avg=16384.0000 max=20632 max_rank=2 min=6591 min_rank=1 imbalance_ratio=1.259277 max_load_reduction=110440 max_load_reduction_pct=0.842590 moved_to_echo=73849 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3620, 0, 0, 0, 0, 3999, 4092, 4096, 2, 1861, 3394, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 2247, 0, 0, 370, 2925, 26, 0, 0, 4096, 0, 0, 0, 14, 0, 0, 1540, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 14, 5, 4082, 0, 0, 3049, 3671, 583, 0, 179, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 6, 3, 0, 0, 0, 0, 4054, 3123, 3786, 4079, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 181, 4067, 0, 1, 743, 0, 0, 0, 0, 4096, 0, 8, 3719, 0, 3619, 485, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 274, 7, 22, 0, 49, 4084, 2356, 44, 0, 4002, 1, 0, 0, 0, 3767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 339, 0, 1283, 0, 1, 13, 4096, 2, 0, 0, 0, 1909, 0, 4066, 4096, 218, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2882, 0, 3306, 4096, 0, 277, 0, 4094, 0, 2, 3430, 3, 0, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21065, 12960, 12737, 16757, 16949, 15652, 16029, 18923] avg=16384.0000 max=21065 max_rank=0 min=12737 min_rank=2 imbalance_ratio=1.285706 max_load_reduction=110007 max_load_reduction_pct=0.839287 moved_to_echo=67553 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[85, 0, 3418, 0, 1763, 23, 0, 2200, 0, 1, 0, 2, 2307, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1527, 1, 1480, 37, 12, 3060, 12, 0, 225, 0, 923, 4034, 4070, 4096, 0, 7, 0, 1709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 3, 0, 32, 4095, 0, 0, 2841, 4, 4086, 3895, 2133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 349, 137, 1018, 238, 0, 2, 0, 0, 0, 407, 0, 0, 0, 18, 0, 0, 3796, 4096, 4093, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 0, 73, 486, 255, 56, 0, 0, 5, 4045, 0, 0, 399, 0, 2331, 489, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3514, 406, 546, 2368, 3699, 9, 4056, 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 4096, 0, 0, 1072, 0, 0, 3914, 0, 0, 1, 551, 7, 26, 26, 3, 220, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3821, 4087, 11, 3336, 2058, 0, 87, 0, 4076, 3985, 6, 0, 0, 0, 1621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13818, 21193, 17126, 14166, 8460, 22931, 10290, 23088] avg=16384.0000 max=23088 max_rank=7 min=8460 min_rank=4 imbalance_ratio=1.409180 max_load_reduction=107984 max_load_reduction_pct=0.823853 moved_to_echo=64769 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 0, 24, 0, 4096, 4065, 4095, 1916, 1603, 2705, 0, 0, 1701, 0, 321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 308, 3452, 0, 0, 0, 34, 0, 0, 4, 4069, 0, 4002, 0, 4096, 0, 0, 1326, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 0, 543, 0, 558, 74, 0, 660, 3971, 2519, 3908, 1297, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 129, 1, 239, 0, 0, 5, 0, 0, 0, 4096, 2824, 2119, 3816, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3465, 1658, 4096, 0, 54, 1, 0, 8, 0, 4086, 48, 0, 0, 4093, 0, 27, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3701, 150, 4090, 2, 0, 0, 3, 2303, 0, 12, 0, 7, 0, 0, 3656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 0, 0, 87, 0, 31, 0, 4091, 0, 1610, 155, 256, 3, 812, 4089, 3825, 198, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 0, 0, 3000, 970, 4019, 0, 0, 395, 0, 40, 3341, 0, 40, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[24610, 17292, 13797, 13838, 17976, 13924, 16438, 13197] avg=16384.0000 max=24610 max_rank=0 min=13197 min_rank=7 imbalance_ratio=1.502075 max_load_reduction=106462 max_load_reduction_pct=0.812241 moved_to_echo=61535 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2432, 9088, 2304, 1536, 8192, 15360, 128, 0, 5760, 26368, 4480, 13952, 4096, 17536, 128, 128, 10496, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1536, 14720, 2816, 10240, 1152, 128, 256, 32512, 3968, 1920, 256, 6272, 2048, 9856, 15488, 26496, 1792, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1152, 17280, 3968, 16512, 8192, 6400, 19840, 14208, 10368, 256, 14720, 512, 512, 256, 128, 17920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[640, 1280, 6912, 4096, 19072, 7040, 0, 8064, 18048, 19072, 16256, 18944, 12544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3968, 10240, 18816, 128, 768, 10624, 1792, 640, 0, 8448, 25216, 128, 4352, 27776, 256, 14976, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1792, 6912, 10240, 384, 26368, 11776, 6144, 5504, 0, 24320, 4864, 5120, 19584, 128, 4352, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12032, 128, 3712, 2560, 1792, 13824, 20864, 25472, 7808, 12672, 10752, 128, 5760, 9856, 4096, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[1664, 1024, 1024, 1280, 0, 128, 768, 256, 128, 6400, 12160, 3200, 128, 1152, 0, 128, 30848, 27648, 19968, 19840, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 14, 0, 0, 0, 363, 4074, 2906, 4018, 1414, 4024, 133, 0, 73, 2, 2, 0, 0, 0, 0, 52, 55, 2879, 4096, 48, 2247, 0, 3562, 1, 19, 16, 4082, 0, 0, 0, 0, 73, 3217, 0, 2847, 0, 0, 4096, 1, 5, 0, 0, 4096, 89, 29, 0, 1, 0, 0, 4084, 0, 0, 745, 493, 0, 0, 4033, 0, 294, 2564, 0, 1, 4057, 0, 1, 8, 0, 568, 44, 33, 0, 2849, 3868, 3907, 0, 0, 4095, 2, 3068, 0, 0, 4096, 0, 1731, 801, 0, 0, 0, 4096, 0, 0, 4028, 0, 4096, 14, 1, 2488, 237, 0, 41, 0, 1641, 0, 0, 0, 2, 2, 2043, 4095, 0, 3582, 1, 152, 666, 3913, 2, 4096, 4018, 1504, 0, 478, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[17, 0, 15, 73, 3678, 9, 4096, 4027, 0, 3994, 46, 675, 4096, 26, 2249, 0, 4049, 0, 0, 0, 492, 2120, 0, 0, 7, 343, 3948, 2, 3305, 0, 0, 2, 4037, 0, 6, 1984, 0, 64, 0, 359, 0, 4, 45, 4026, 22, 69, 0, 4, 4067, 0, 150, 1, 27, 2882, 4039, 15, 0, 0, 0, 989, 0, 4, 0, 27, 0, 3686, 55, 0, 0, 28, 0, 0, 0, 1, 0, 1, 90, 3976, 3, 213, 0, 0, 3750, 3363, 2446, 19, 1887, 4091, 2, 0, 4079, 0, 0, 0, 16, 0, 0, 4093, 0, 3989, 0, 1, 4078, 4046, 0, 25, 4067, 801, 0, 0, 14, 0, 0, 4095, 0, 22, 24, 8, 3166, 4094, 2978, 133, 4096, 3347, 0, 198, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 356, 3980, 0, 4016, 15, 1235, 4092, 161, 4083, 4092, 522, 0, 0, 2794, 0, 1215, 0, 1271, 37, 35, 0, 69, 0, 3513, 6, 0, 0, 0, 2720, 0, 0, 89, 0, 94, 108, 3695, 837, 267, 0, 0, 3916, 0, 318, 0, 10, 3925, 27, 0, 304, 1, 24, 3794, 0, 1, 0, 3739, 3300, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1032, 0, 0, 221, 0, 592, 0, 3285, 4, 0, 2, 1726, 10, 0, 1166, 3273, 0, 4081, 0, 2, 802, 0, 0, 0, 0, 6, 0, 4062, 0, 0, 3, 3989, 1897, 4093, 0, 0, 3769, 0, 4094, 3, 2, 5, 3868, 2140, 3414, 418, 4095, 0, 4093, 4096, 0, 0, 4026, 4082, 9, 0, 0, 4047] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 53, 0, 1612, 37, 44, 4089, 31, 1188, 4090, 54, 2978, 2520, 4096, 3539, 385, 2, 6, 68, 0, 47, 196, 0, 1, 3388, 0, 4074, 0, 2, 0, 4027, 3377, 0, 0, 0, 0, 2129, 712, 7, 1, 535, 0, 151, 3998, 0, 35, 1, 0, 0, 0, 0, 0, 0, 812, 4096, 1, 0, 0, 0, 0, 0, 0, 24, 169, 0, 0, 0, 0, 2901, 0, 8, 1, 0, 0, 0, 19, 23, 4096, 4090, 61, 274, 15, 26, 1, 2979, 4095, 0, 4096, 0, 0, 0, 0, 1464, 0, 0, 1635, 843, 4096, 3480, 0, 0, 4028, 1, 0, 0, 4096, 4096, 3, 4096, 0, 0, 0, 714, 4091, 3743, 20, 3146, 0, 4074, 8, 3951, 0, 0, 13, 4, 3069, 1079, 4062] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 0, 0, 4096, 4093, 0, 4092, 1, 4095, 39, 0, 4068, 0, 918, 0, 0, 0, 0, 3843, 30, 0, 1911, 2914, 4096, 3, 4096, 0, 0, 1, 172, 0, 0, 3900, 0, 0, 169, 391, 0, 0, 0, 3972, 1, 752, 0, 17, 4037, 0, 7, 0, 0, 0, 156, 4, 0, 0, 0, 236, 0, 0, 0, 0, 1433, 0, 0, 2, 27, 0, 2, 18, 0, 0, 0, 4096, 0, 4022, 0, 3972, 0, 7, 4072, 34, 4087, 4088, 0, 4096, 0, 0, 25, 0, 0, 0, 1378, 2980, 0, 4096, 0, 103, 4, 1006, 325, 0, 0, 217, 0, 87, 3082, 0, 0, 0, 4087, 4089, 4091, 3746, 3922, 1, 3644, 1, 3884, 3, 4096, 9, 0, 39, 0, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[225, 0, 9, 3828, 4010, 0, 44, 0, 0, 4054, 0, 4096, 3367, 4096, 1, 0, 1, 0, 812, 228, 1, 142, 0, 4051, 0, 3891, 1554, 1, 1, 2932, 0, 3548, 115, 0, 145, 31, 27, 32, 2934, 4088, 4081, 0, 318, 0, 9, 2279, 0, 0, 3, 0, 0, 0, 16, 0, 13, 1, 0, 27, 125, 134, 80, 97, 3, 1, 409, 0, 0, 0, 4024, 0, 1213, 0, 0, 0, 0, 3915, 1, 3943, 3030, 633, 3, 0, 4036, 781, 0, 4095, 0, 4091, 157, 0, 0, 0, 0, 0, 0, 0, 77, 1233, 90, 3766, 0, 2132, 4096, 0, 5, 4076, 4068, 0, 56, 59, 1069, 10, 0, 2, 2745, 4090, 0, 0, 2970, 4028, 2875, 0, 3085, 4079, 0, 674, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 254, 0, 0, 263, 14, 59, 4096, 0, 3864, 0, 3831, 4096, 5, 23, 0, 4083, 312, 0, 11, 3887, 3719, 0, 16, 3736, 0, 4053, 0, 3916, 0, 18, 0, 0, 0, 0, 0, 253, 2680, 9, 806, 4018, 0, 3972, 286, 0, 0, 0, 4092, 3980, 0, 0, 0, 0, 1700, 146, 4, 0, 0, 0, 191, 0, 0, 8, 0, 0, 17, 0, 4076, 0, 2, 0, 0, 0, 0, 0, 109, 0, 2991, 19, 4087, 0, 2, 4044, 3409, 733, 4095, 0, 4096, 0, 0, 0, 3, 0, 0, 3753, 0, 0, 4096, 3756, 0, 0, 32, 1, 3820, 0, 11, 57, 6, 4064, 0, 0, 0, 1002, 3590, 4045, 2, 268, 0, 913, 0, 4029, 0, 3413, 7, 27, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 32, 0, 10, 42, 172, 2157, 0, 103, 4095, 7, 0, 0, 4061, 6, 0, 0, 4095, 4088, 5, 2656, 3272, 207, 4087, 0, 3183, 0, 569, 4081, 0, 1, 0, 1817, 0, 0, 112, 7, 6, 0, 179, 0, 0, 4090, 23, 0, 4075, 0, 0, 0, 0, 0, 11, 483, 0, 0, 672, 0, 0, 0, 4093, 0, 28, 3, 1, 4047, 4041, 0, 116, 0, 0, 0, 0, 4022, 0, 48, 143, 3846, 0, 152, 4082, 1, 4085, 4095, 0, 3444, 0, 0, 2564, 0, 0, 0, 562, 0, 0, 4096, 0, 0, 0, 4, 1, 3930, 0, 4095, 1913, 24, 4095, 0, 0, 0, 0, 11, 4086, 3338, 1313, 3, 4091, 109, 3996, 0, 4070, 21, 1, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 14, 0, 0, 0, 363, 4074, 2906, 900, 1414, 4024, 133, 0, 73, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 52, 55, 2879, 4096, 48, 1265, 0, 3562, 1, 19, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 73, 3217, 0, 2847, 0, 0, 4096, 1, 5, 0, 0, 3118, 2129, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 89, 29, 0, 1, 0, 0, 4084, 0, 0, 745, 493, 0, 0, 4033, 0, 4096, 4095, 1784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 2564, 0, 1, 4057, 0, 1, 8, 0, 568, 44, 33, 0, 2849, 3868, 3907, 982, 942, 39, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 3068, 0, 0, 3436, 0, 1731, 801, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2284, 0, 4096, 14, 1, 2488, 237, 0, 41, 0, 1641, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2043, 0, 0, 3582, 1, 113, 666, 0, 2, 0, 4018, 1504, 0, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13903, 11995, 20370, 23545, 20817, 17229, 10804, 12409] avg=16384.0000 max=23545 max_rank=3 min=10804 min_rank=6 imbalance_ratio=1.437073 max_load_reduction=107527 max_load_reduction_pct=0.820366 moved_to_echo=61259 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[17, 0, 15, 73, 3678, 9, 4096, 4027, 0, 895, 46, 675, 4096, 26, 2249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 492, 2120, 0, 0, 7, 343, 2222, 2, 3305, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 6, 1984, 0, 64, 0, 359, 0, 4, 45, 4026, 22, 69, 0, 4, 3099, 1620, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 150, 1, 27, 2882, 4039, 15, 0, 0, 0, 989, 0, 4, 0, 27, 4096, 0, 1358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3686, 55, 0, 0, 28, 0, 0, 0, 1, 0, 1, 90, 3976, 3, 213, 1726, 957, 815, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3750, 3363, 2446, 19, 1887, 3432, 2, 0, 4079, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2321, 0, 3989, 0, 1, 4078, 4046, 0, 25, 4067, 801, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 22, 24, 8, 2351, 4094, 0, 133, 0, 3347, 0, 198, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19902, 12542, 16154, 17655, 12210, 18994, 19342, 14273] avg=16384.0000 max=19902 max_rank=0 min=12210 min_rank=4 imbalance_ratio=1.214722 max_load_reduction=111170 max_load_reduction_pct=0.848160 moved_to_echo=64819 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 356, 3980, 0, 4016, 15, 1235, 917, 161, 4083, 4092, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2794, 0, 1215, 0, 1271, 37, 35, 0, 69, 0, 1977, 6, 0, 0, 0, 2720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 94, 108, 3695, 837, 267, 0, 0, 3916, 0, 318, 0, 10, 3175, 0, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3925, 27, 0, 304, 1, 24, 3794, 0, 1, 0, 3739, 3300, 0, 0, 0, 0, 4026, 3414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1032, 0, 0, 221, 0, 592, 0, 3285, 4, 0, 1536, 950, 1054, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1726, 10, 0, 1166, 3273, 0, 3424, 0, 2, 802, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2304, 0, 0, 3, 3989, 1897, 4093, 0, 0, 3769, 0, 4094, 3, 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3868, 2140, 0, 418, 4095, 0, 3039, 4096, 0, 0, 0, 4082, 9, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19377, 10124, 13317, 22555, 9335, 10411, 20159, 25794] avg=16384.0000 max=25794 max_rank=7 min=9335 min_rank=4 imbalance_ratio=1.574341 max_load_reduction=105278 max_load_reduction_pct=0.803207 moved_to_echo=65699 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 254, 0, 0, 263, 14, 59, 4096, 0, 861, 0, 3831, 4096, 5, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 312, 0, 11, 3887, 3719, 0, 16, 3736, 0, 2278, 0, 3916, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 253, 2680, 9, 806, 4018, 0, 3972, 286, 0, 0, 0, 4092, 3003, 2193, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 0, 0, 1700, 146, 4, 0, 0, 0, 191, 0, 0, 8, 0, 3413, 4045, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4076, 0, 2, 0, 0, 0, 0, 0, 109, 0, 2991, 19, 4087, 1775, 959, 239, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4044, 3409, 733, 4095, 0, 3433, 0, 0, 0, 3, 0, 0, 3753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2318, 3756, 0, 0, 32, 1, 3820, 0, 11, 57, 6, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 3590, 0, 2, 268, 0, 674, 0, 0, 0, 0, 7, 27, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13502, 21976, 22131, 15323, 14937, 19472, 14065, 9666] avg=16384.0000 max=22131 max_rank=2 min=9666 min_rank=7 imbalance_ratio=1.350769 max_load_reduction=108941 max_load_reduction_pct=0.831154 moved_to_echo=58140 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 53, 0, 1612, 37, 44, 4089, 31, 1188, 916, 54, 2978, 2520, 4096, 3539, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 68, 0, 47, 196, 0, 1, 3388, 0, 2293, 0, 2, 0, 4027, 3377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2129, 712, 7, 1, 535, 0, 151, 3998, 0, 35, 1, 0, 3174, 2151, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 812, 4096, 1, 0, 0, 0, 0, 0, 0, 24, 169, 0, 3743, 1800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2901, 0, 8, 1, 0, 0, 0, 19, 23, 4096, 4090, 61, 1781, 958, 1049, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 15, 26, 1, 2979, 4095, 0, 3436, 0, 0, 0, 0, 1464, 0, 0, 1635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 843, 2323, 3480, 0, 0, 4028, 1, 0, 0, 4096, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 714, 4091, 0, 20, 3146, 0, 3025, 8, 0, 0, 0, 13, 4, 3069, 1079, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21542, 13407, 13709, 10645, 15647, 13925, 22966, 19231] avg=16384.0000 max=22966 max_rank=6 min=10645 min_rank=3 imbalance_ratio=1.401733 max_load_reduction=108106 max_load_reduction_pct=0.824783 moved_to_echo=71769 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 0, 0, 0, 4096, 4093, 0, 917, 1, 4095, 39, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 918, 0, 0, 0, 0, 3843, 30, 0, 1911, 2914, 2306, 3, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 3900, 0, 0, 169, 391, 0, 0, 0, 3972, 1, 752, 0, 17, 3175, 2114, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 7, 0, 0, 0, 156, 4, 0, 0, 0, 236, 0, 0, 0, 0, 4096, 4091, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1433, 0, 0, 2, 27, 0, 2, 18, 0, 0, 0, 4096, 0, 4022, 0, 3972, 1790, 958, 938, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4072, 34, 4087, 4088, 0, 3436, 0, 0, 25, 0, 0, 0, 1378, 2980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 0, 103, 4, 1006, 325, 0, 0, 217, 0, 87, 3082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4089, 0, 3746, 3922, 1, 2706, 1, 0, 3, 0, 9, 0, 39, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17316, 16022, 15478, 14397, 17918, 20107, 7147, 22687] avg=16384.0000 max=22687 max_rank=7 min=7147 min_rank=6 imbalance_ratio=1.384705 max_load_reduction=108385 max_load_reduction_pct=0.826912 moved_to_echo=67859 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[225, 0, 9, 3828, 4010, 0, 44, 0, 0, 908, 0, 4096, 3367, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 812, 228, 1, 142, 0, 4051, 0, 3891, 875, 1, 1, 2932, 0, 3548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 145, 31, 27, 32, 2934, 4088, 4081, 0, 318, 0, 9, 2279, 0, 0, 3146, 1565, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 16, 0, 13, 1, 0, 27, 125, 134, 80, 97, 3, 1, 3085, 2745, 1310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 0, 0, 0, 4024, 0, 1213, 0, 0, 0, 0, 3915, 1, 3943, 3030, 633, 679, 288, 765, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4036, 781, 0, 4095, 0, 3432, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 700, 90, 3766, 0, 2132, 4096, 0, 5, 4076, 4068, 0, 56, 59, 1069, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4090, 0, 0, 2205, 4028, 0, 0, 0, 4079, 0, 674, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20584, 16483, 19015, 7640, 19559, 12504, 20204, 15083] avg=16384.0000 max=20584 max_rank=0 min=7640 min_rank=3 imbalance_ratio=1.256348 max_load_reduction=110488 max_load_reduction_pct=0.842957 moved_to_echo=67350 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 32, 0, 10, 42, 172, 483, 0, 103, 4095, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 6, 0, 0, 4095, 4088, 5, 2656, 3272, 207, 2300, 0, 3183, 0, 569, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1817, 0, 0, 112, 7, 6, 0, 179, 0, 0, 4090, 23, 0, 1674, 2174, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 11, 483, 0, 0, 672, 0, 0, 0, 4093, 0, 4070, 4086, 1822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3, 1, 4047, 4041, 0, 116, 0, 0, 0, 0, 4022, 0, 48, 143, 3846, 1787, 958, 1054, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 4082, 1, 4085, 4095, 0, 2889, 0, 0, 2564, 0, 0, 0, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2323, 0, 0, 0, 4, 1, 3930, 0, 4095, 1913, 24, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 3338, 1313, 3, 3037, 109, 0, 0, 0, 21, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4946, 28523, 10898, 19312, 20649, 18430, 16385, 11929] avg=16384.0000 max=28523 max_rank=1 min=4946 min_rank=0 imbalance_ratio=1.740906 max_load_reduction=102549 max_load_reduction_pct=0.782387 moved_to_echo=67393 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[24192, 128, 256, 384, 128, 5504, 12288, 4608, 128, 128, 5376, 5376, 128, 128, 8192, 256, 27008, 26240, 11776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1024, 16896, 7424, 12032, 128, 11264, 12928, 16128, 128, 12672, 18048, 2688, 19584, 128, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[9728, 20096, 0, 11648, 16384, 128, 17152, 13056, 0, 256, 0, 15616, 1664, 4096, 1664, 20480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[8448, 128, 256, 7808, 2560, 3712, 10240, 6528, 11776, 128, 4736, 20352, 128, 7552, 128, 4224, 23680, 13952, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[16000, 384, 2176, 256, 9856, 14208, 128, 9728, 16512, 7424, 15616, 128, 18176, 2944, 4736, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 1920, 24192, 7680, 18688, 23808, 1920, 27008, 256, 1792, 8320, 128, 1536, 0, 9856, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 384, 128, 5888, 12032, 128, 16896, 16384, 5504, 6912, 1792, 23936, 22528, 8832, 9984, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2176, 6272, 128, 8192, 15104, 128, 2432, 128, 0, 896, 128, 12800, 128, 25216, 11264, 16768, 12160, 7040, 6016, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[3997, 1, 58, 819, 0, 0, 66, 1899, 0, 3808, 0, 11, 2028, 0, 112, 946, 0, 66, 0, 230, 4094, 1, 273, 1128, 367, 4093, 0, 21, 4084, 0, 613, 7, 0, 0, 0, 109, 0, 241, 2965, 0, 0, 3981, 0, 0, 137, 4095, 0, 0, 0, 0, 0, 651, 0, 0, 0, 3915, 4, 43, 0, 538, 4096, 0, 48, 0, 4095, 0, 0, 3046, 0, 0, 0, 1, 0, 4096, 0, 2, 4094, 0, 4094, 0, 0, 3877, 0, 6, 1607, 3077, 4077, 0, 0, 157, 0, 4096, 0, 4043, 467, 0, 4054, 5, 39, 17, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 224, 0, 21, 477, 3474, 34, 4091, 4077, 0, 4089, 0, 4093, 2, 3866, 1, 4096, 1239, 2697, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[3806, 0, 3071, 119, 0, 2, 97, 3988, 0, 300, 0, 1, 4012, 0, 140, 429, 0, 0, 748, 621, 4088, 1, 4071, 0, 1091, 1225, 0, 0, 4092, 3038, 0, 203, 1, 181, 0, 0, 0, 3965, 2529, 0, 0, 0, 0, 3165, 0, 503, 0, 0, 321, 72, 60, 121, 21, 81, 0, 4096, 0, 0, 283, 0, 3434, 0, 0, 0, 4095, 3352, 0, 3, 0, 0, 347, 0, 3952, 4076, 0, 760, 878, 4066, 752, 0, 0, 21, 3940, 0, 51, 916, 4096, 3594, 0, 0, 1574, 3, 0, 4072, 3575, 0, 0, 0, 12, 0, 166, 0, 3514, 0, 0, 10, 2884, 4088, 0, 110, 1, 0, 1729, 4096, 0, 4, 4096, 0, 4085, 0, 4096, 0, 4058, 20, 1, 3, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[602, 0, 5, 1969, 0, 2735, 1878, 8, 22, 140, 0, 0, 0, 0, 4, 318, 0, 0, 28, 1, 4070, 0, 4096, 4096, 1345, 3693, 0, 4005, 3884, 5, 0, 0, 0, 4091, 0, 0, 58, 4090, 3272, 671, 862, 41, 13, 3068, 0, 114, 191, 22, 3875, 186, 0, 76, 0, 0, 425, 13, 22, 3989, 4077, 494, 3500, 188, 0, 0, 3719, 477, 0, 4095, 0, 96, 0, 0, 0, 3438, 10, 0, 896, 0, 2832, 609, 1, 0, 0, 0, 0, 4073, 1148, 4026, 0, 0, 99, 3207, 0, 1126, 0, 0, 5, 0, 0, 0, 4, 9, 4096, 0, 1, 0, 3699, 3147, 0, 3351, 0, 0, 4096, 0, 0, 3958, 0, 328, 0, 0, 4093, 0, 4093, 59, 4, 1, 4033, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4092, 0, 0, 4096, 0, 0, 0, 0, 0, 234, 0, 2, 0, 0, 3865, 1198, 0, 50, 1173, 0, 4022, 2, 4029, 0, 0, 250, 7, 0, 4090, 64, 0, 0, 293, 1, 0, 0, 0, 3943, 4095, 0, 12, 2998, 0, 1405, 0, 1214, 0, 400, 0, 2284, 304, 640, 0, 1, 0, 4096, 0, 3204, 3, 0, 3804, 20, 0, 0, 3991, 208, 0, 4, 0, 0, 3938, 0, 0, 4092, 0, 0, 1915, 2, 4039, 0, 0, 120, 3847, 0, 4093, 0, 693, 4059, 0, 0, 32, 3941, 0, 2519, 42, 0, 4093, 0, 0, 1, 196, 0, 3722, 0, 0, 321, 2027, 38, 0, 193, 1, 0, 388, 4096, 0, 4096, 207, 0, 2880, 0, 4006, 0, 3747, 7, 4096, 3531, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[3940, 27, 4095, 0, 0, 45, 4062, 28, 0, 4038, 0, 0, 1, 0, 0, 3639, 0, 0, 0, 3147, 3848, 0, 3740, 38, 9, 4091, 0, 257, 3276, 0, 19, 9, 0, 98, 0, 69, 106, 78, 0, 0, 32, 10, 0, 19, 172, 4021, 4095, 0, 0, 1473, 0, 0, 0, 0, 3497, 6, 12, 4091, 0, 4023, 4096, 0, 554, 0, 3880, 0, 0, 564, 0, 0, 2732, 14, 0, 2715, 0, 1, 8, 3616, 4022, 0, 0, 41, 0, 2, 0, 4096, 238, 0, 64, 5, 0, 4096, 0, 1533, 311, 1736, 171, 0, 1508, 13, 0, 0, 4096, 0, 7, 0, 4086, 2621, 1182, 0, 0, 5, 1857, 0, 3, 4004, 3859, 58, 456, 0, 4095, 491, 4021, 0, 9, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 2240, 2381, 367, 0, 92, 0, 5, 0, 1660, 0, 0, 0, 484, 4095, 0, 4096, 32, 82, 144, 13, 3945, 0, 1708, 3802, 0, 0, 0, 4090, 0, 357, 5, 2, 1, 225, 195, 31, 3670, 0, 146, 0, 46, 0, 1155, 53, 1262, 55, 1, 0, 0, 0, 0, 3837, 0, 0, 666, 5, 442, 0, 4062, 3993, 311, 0, 806, 3811, 0, 4095, 103, 0, 963, 0, 0, 3967, 39, 3199, 8, 0, 4016, 315, 1, 4019, 0, 4096, 3690, 0, 6, 2922, 3464, 398, 1839, 79, 0, 0, 0, 0, 148, 2, 0, 3880, 0, 210, 0, 342, 3460, 0, 647, 0, 0, 4035, 3, 103, 4063, 512, 0, 3437, 0, 3914, 1, 30, 392, 2632, 3378, 3841, 354] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[447, 1144, 0, 3088, 51, 0, 4095, 114, 0, 2, 0, 3837, 63, 13, 3, 0, 0, 13, 0, 484, 3014, 3, 0, 0, 4094, 4066, 6, 0, 4094, 35, 2329, 242, 0, 140, 34, 133, 2436, 4088, 1563, 1242, 3930, 504, 0, 1837, 3936, 2884, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 46, 4040, 0, 0, 4004, 0, 104, 0, 3952, 4051, 1, 30, 0, 27, 3846, 4, 0, 4048, 0, 0, 0, 2425, 6, 0, 15, 0, 1, 0, 4045, 2724, 0, 0, 0, 0, 3605, 3574, 0, 4065, 4096, 166, 0, 0, 0, 562, 1, 0, 2637, 181, 2, 0, 187, 4096, 0, 0, 0, 50, 4096, 2, 0, 1607, 3010, 1, 0, 0, 4033, 319, 3266, 64, 23, 0, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 0, 4096, 0, 5, 0, 396, 0, 3204, 0, 1, 0, 0, 15, 4092, 0, 1, 102, 0, 4096, 1230, 4016, 4051, 16, 3768, 3335, 4067, 4076, 0, 0, 0, 1347, 1, 0, 663, 0, 4073, 75, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 2, 2, 4, 49, 0, 4096, 0, 4040, 2, 3352, 4036, 0, 0, 0, 3621, 63, 2, 4045, 20, 0, 32, 0, 0, 4087, 0, 131, 5, 3, 4096, 0, 0, 1603, 4082, 0, 4, 0, 90, 4057, 0, 0, 0, 4093, 0, 244, 0, 0, 46, 0, 4006, 67, 14, 0, 4095, 0, 0, 3, 46, 12, 1934, 60, 2, 0, 8, 287, 0, 4096, 34, 0, 1174, 0, 39, 0, 3159, 4094, 4069, 1, 3034, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3806, 0, 3071, 119, 0, 2, 97, 3988, 0, 300, 0, 1, 4012, 0, 140, 429, 1842, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 748, 621, 4088, 1, 4071, 0, 1091, 1225, 0, 0, 709, 3038, 0, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 181, 0, 0, 0, 3965, 2529, 0, 0, 0, 0, 3165, 0, 503, 0, 0, 2966, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 321, 72, 60, 121, 21, 81, 0, 4096, 0, 0, 283, 0, 3434, 0, 0, 0, 2, 553, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1700, 3352, 0, 3, 0, 0, 347, 0, 3952, 4076, 0, 760, 878, 4066, 752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 3940, 0, 51, 916, 4096, 3594, 0, 0, 1574, 1, 0, 4072, 3575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 166, 0, 3514, 0, 0, 10, 2884, 4088, 0, 110, 1, 0, 4096, 2811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1729, 4096, 0, 3, 4096, 0, 4085, 0, 0, 0, 0, 20, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17808, 15795, 14557, 9461, 19886, 21840, 17692, 14033] avg=16384.0000 max=21840 max_rank=5 min=9461 min_rank=3 imbalance_ratio=1.333008 max_load_reduction=109232 max_load_reduction_pct=0.833374 moved_to_echo=73451 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3940, 27, 4095, 0, 0, 45, 4062, 28, 0, 4038, 0, 0, 1, 0, 0, 3639, 1745, 1022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3147, 3848, 0, 3740, 38, 9, 4091, 0, 257, 568, 0, 19, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 69, 106, 78, 0, 0, 32, 10, 0, 19, 172, 4021, 4095, 0, 2374, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1473, 0, 0, 0, 0, 3497, 6, 12, 4091, 0, 4023, 4096, 0, 554, 0, 2883, 524, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1611, 0, 0, 564, 0, 0, 2732, 14, 0, 2715, 0, 1, 8, 3616, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 2, 0, 4096, 238, 0, 64, 5, 0, 1213, 0, 1533, 311, 1736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 1508, 13, 0, 0, 4096, 0, 7, 0, 4086, 2621, 1182, 0, 0, 5, 4095, 2786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1857, 0, 3, 2982, 3859, 58, 456, 0, 0, 491, 0, 0, 9, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22642, 15726, 12309, 21493, 15283, 9239, 20570, 13810] avg=16384.0000 max=22642 max_rank=0 min=9239 min_rank=5 imbalance_ratio=1.381958 max_load_reduction=108430 max_load_reduction_pct=0.827255 moved_to_echo=58902 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[602, 0, 5, 1969, 0, 2735, 1878, 8, 22, 140, 0, 0, 0, 0, 4, 318, 1672, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 1, 4070, 0, 4096, 4096, 1345, 3693, 0, 4005, 673, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 58, 4090, 3272, 671, 862, 41, 13, 3068, 0, 114, 191, 22, 2815, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3875, 186, 0, 76, 0, 0, 425, 13, 22, 3989, 4077, 494, 3500, 188, 0, 0, 2257, 502, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1545, 477, 0, 4095, 0, 96, 0, 0, 0, 3438, 10, 0, 896, 0, 2832, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4073, 1148, 4026, 0, 0, 99, 950, 0, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4, 9, 4096, 0, 1, 0, 3699, 3147, 0, 3351, 0, 0, 4093, 2836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2948, 0, 328, 0, 0, 0, 0, 0, 59, 4, 1, 4033, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10363, 22012, 20565, 20000, 13998, 11423, 21241, 11470] avg=16384.0000 max=22012 max_rank=1 min=10363 min_rank=0 imbalance_ratio=1.343506 max_load_reduction=109060 max_load_reduction_pct=0.832062 moved_to_echo=58132 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3997, 1, 58, 819, 0, 0, 66, 1899, 0, 3808, 0, 11, 2028, 0, 112, 946, 1842, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 230, 4094, 1, 273, 1128, 367, 4093, 0, 21, 708, 0, 613, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 241, 2965, 0, 0, 3981, 0, 0, 137, 4095, 0, 0, 2960, 1187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 651, 0, 0, 0, 3915, 4, 43, 0, 538, 4096, 0, 48, 0, 2883, 553, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1700, 0, 0, 3046, 0, 0, 0, 1, 0, 4096, 0, 2, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 6, 1607, 3077, 4077, 0, 0, 157, 0, 1213, 0, 4043, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 5, 39, 17, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 224, 0, 21, 4093, 2679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 3474, 34, 3047, 4077, 0, 4089, 0, 0, 2, 0, 1, 4096, 1239, 2697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16631, 11601, 15675, 13147, 17033, 18524, 15228, 23233] avg=16384.0000 max=23233 max_rank=7 min=11601 min_rank=1 imbalance_ratio=1.418030 max_load_reduction=107839 max_load_reduction_pct=0.822746 moved_to_echo=74018 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 4096, 0, 0, 0, 0, 0, 234, 0, 2, 0, 0, 3865, 1198, 1799, 1047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 1173, 0, 4022, 2, 4029, 0, 0, 250, 7, 0, 698, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 1, 0, 0, 0, 3943, 4095, 0, 12, 2998, 0, 1405, 0, 1214, 0, 400, 2970, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2284, 304, 640, 0, 1, 0, 4096, 0, 3204, 3, 0, 3804, 20, 0, 0, 2776, 545, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1647, 208, 0, 4, 0, 0, 3938, 0, 0, 4092, 0, 0, 1915, 2, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 3847, 0, 4093, 0, 693, 4059, 0, 0, 32, 1165, 0, 2519, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 196, 0, 3722, 0, 0, 321, 2027, 38, 0, 193, 1, 0, 4006, 2595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 4096, 0, 3049, 207, 0, 2880, 0, 0, 0, 0, 7, 4096, 3531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16333, 10295, 18483, 18099, 15845, 16570, 17193, 18254] avg=16384.0000 max=18483 max_rank=2 min=10295 min_rank=1 imbalance_ratio=1.128113 max_load_reduction=112589 max_load_reduction_pct=0.858986 moved_to_echo=67862 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[447, 1144, 0, 3088, 51, 0, 4095, 114, 0, 2, 0, 3837, 63, 13, 3, 0, 1777, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 484, 3014, 3, 0, 0, 4094, 4066, 6, 0, 710, 35, 2329, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 34, 133, 2436, 4088, 1563, 1242, 3930, 504, 0, 1837, 3936, 2884, 0, 0, 2967, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 46, 4040, 0, 0, 4004, 0, 104, 0, 2515, 534, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1641, 4051, 1, 30, 0, 27, 3846, 4, 0, 4048, 0, 0, 0, 2425, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 1, 0, 4045, 2724, 0, 0, 0, 0, 3605, 1059, 0, 4065, 4096, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 1, 0, 2637, 181, 2, 0, 187, 4096, 0, 0, 0, 50, 4033, 2263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 1197, 3010, 1, 0, 0, 0, 319, 0, 64, 23, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15044, 14996, 26697, 11662, 16079, 19776, 14012, 12806] avg=16384.0000 max=26697 max_rank=2 min=11662 min_rank=3 imbalance_ratio=1.629456 max_load_reduction=104375 max_load_reduction_pct=0.796318 moved_to_echo=62673 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 2240, 2381, 367, 0, 92, 0, 5, 0, 1660, 0, 0, 1796, 1037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 4095, 0, 4096, 32, 82, 144, 13, 3945, 0, 1708, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 357, 5, 2, 1, 225, 195, 31, 3670, 0, 146, 0, 46, 0, 1155, 2756, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 1262, 55, 1, 0, 0, 0, 0, 3837, 0, 0, 666, 5, 442, 0, 4062, 2438, 539, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1658, 311, 0, 806, 3811, 0, 4095, 103, 0, 963, 0, 0, 3967, 39, 3199, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 315, 1, 4019, 0, 4096, 3690, 0, 6, 2922, 1026, 398, 1839, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 2, 0, 3880, 0, 210, 0, 342, 3460, 0, 647, 0, 0, 3914, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 3, 103, 3026, 512, 0, 3437, 0, 0, 1, 0, 392, 2632, 3378, 3841, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13674, 15258, 12689, 13747, 18960, 22407, 12623, 21714] avg=16384.0000 max=22407 max_rank=5 min=12623 min_rank=6 imbalance_ratio=1.367615 max_load_reduction=108665 max_load_reduction_pct=0.829048 moved_to_echo=75704 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 4096, 0, 5, 0, 396, 0, 3204, 0, 1, 0, 0, 15, 4092, 1628, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 102, 0, 4096, 1230, 4016, 4051, 16, 3768, 3335, 4067, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1347, 1, 0, 663, 0, 4073, 75, 0, 0, 0, 0, 0, 0, 14, 0, 0, 2954, 970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 2, 4, 49, 0, 4096, 0, 4040, 2, 3352, 4036, 0, 0, 0, 2880, 489, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1504, 63, 2, 4045, 20, 0, 32, 0, 0, 4087, 0, 131, 5, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1603, 4082, 0, 4, 0, 90, 4057, 0, 0, 0, 1213, 0, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 4006, 67, 14, 0, 4095, 0, 0, 3, 46, 12, 1934, 60, 2, 0, 39, 2189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 287, 0, 3051, 34, 0, 1174, 0, 0, 0, 0, 4094, 4069, 1, 3034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18577, 25389, 10097, 23463, 13988, 11293, 12513, 15752] avg=16384.0000 max=25389 max_rank=1 min=10097 min_rank=2 imbalance_ratio=1.549622 max_load_reduction=105683 max_load_reduction_pct=0.806297 moved_to_echo=53546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[13056, 8576, 128, 12672, 3840, 128, 15104, 128, 3968, 27520, 128, 896, 11776, 10240, 23040, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 9728, 12288, 128, 13824, 14976, 14464, 19456, 128, 256, 8320, 7936, 512, 19456, 8576, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[5760, 4608, 512, 1024, 2688, 20480, 14848, 2176, 4992, 11264, 128, 9728, 4352, 12928, 4352, 1664, 22784, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[16768, 12032, 256, 19328, 15872, 512, 16128, 0, 0, 896, 0, 4736, 14976, 8192, 21888, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[25088, 1280, 7296, 14208, 128, 5120, 12672, 6912, 128, 11904, 0, 3968, 6144, 1792, 4224, 10624, 14208, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[8448, 128, 5632, 896, 384, 128, 30208, 256, 256, 384, 13312, 17536, 3200, 4608, 128, 128, 28416, 18304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[4352, 9472, 512, 1536, 128, 256, 3968, 16256, 3968, 19456, 4480, 9088, 27008, 768, 768, 4096, 18688, 4352, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 640, 6272, 4608, 31360, 1280, 20352, 9472, 7040, 25216, 3456, 10112, 5504, 3200, 3072, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 17, 4077, 0, 0, 0, 0, 0, 0, 0, 4088, 3997, 0, 0, 0, 1, 0, 0, 1, 0, 1, 16, 0, 388, 4001, 0, 4086, 0, 3, 3150, 0, 0, 2, 0, 3943, 3858, 1, 0, 148, 0, 0, 1183, 135, 3432, 3953, 53, 0, 0, 0, 2311, 648, 1, 0, 0, 4076, 0, 4096, 20, 0, 0, 4096, 2549, 0, 0, 1, 4083, 17, 23, 0, 244, 3109, 4096, 4, 0, 0, 0, 3480, 4003, 0, 2944, 4096, 0, 2886, 4096, 4096, 0, 4096, 4, 0, 0, 2, 3989, 13, 0, 0, 0, 89, 0, 0, 1, 128, 4096, 4095, 3184, 3961, 7, 0, 219, 4, 3877, 0, 0, 9, 0, 0, 3, 2981, 0, 0, 1507, 0, 91, 2016, 0, 0, 1191, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[539, 5, 1819, 4024, 4095, 0, 0, 322, 650, 69, 19, 6, 4071, 8, 0, 0, 0, 1, 0, 0, 3635, 0, 3835, 0, 0, 2785, 1916, 0, 0, 0, 3722, 965, 0, 4, 0, 5, 2521, 0, 6, 4, 3771, 186, 2, 1249, 0, 1, 4084, 363, 4, 0, 3886, 163, 730, 5, 30, 0, 2, 4096, 1, 22, 5, 2200, 4096, 0, 464, 369, 3081, 0, 149, 912, 0, 0, 3978, 0, 3, 4083, 7, 0, 0, 0, 4054, 0, 1, 0, 0, 17, 65, 0, 4095, 0, 9, 2563, 114, 0, 2, 0, 0, 0, 4029, 0, 0, 4046, 4092, 4091, 3867, 112, 2252, 10, 2969, 0, 0, 0, 598, 2, 0, 4033, 0, 4064, 4096, 468, 4096, 82, 831, 426, 0, 2, 0, 3018] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 3211, 591, 0, 4096, 0, 0, 0, 2150, 0, 3, 3568, 2195, 35, 13, 660, 0, 2, 0, 449, 3, 232, 2496, 10, 0, 4094, 0, 3975, 0, 3, 3713, 53, 36, 68, 11, 21, 10, 0, 0, 0, 0, 0, 1619, 0, 4, 916, 4037, 0, 1182, 200, 24, 138, 1859, 0, 4095, 4, 0, 3814, 270, 0, 3, 4011, 4091, 0, 1091, 2760, 1026, 0, 10, 0, 0, 0, 4063, 0, 0, 0, 2470, 0, 505, 0, 2387, 4064, 0, 248, 4094, 1270, 38, 609, 3815, 0, 400, 4083, 4091, 34, 17, 3, 0, 0, 0, 0, 4095, 4089, 4095, 4014, 286, 0, 592, 0, 29, 0, 3872, 0, 0, 34, 4, 3540, 0, 7, 4096, 5, 2902, 0, 3509, 752, 108, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 3977, 216, 2450, 3, 0, 0, 1, 4087, 2197, 0, 3997, 3688, 0, 0, 0, 0, 34, 0, 0, 27, 0, 205, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 0, 26, 0, 687, 0, 33, 113, 172, 0, 25, 9, 1954, 33, 4037, 0, 8, 0, 0, 2866, 329, 4096, 0, 2170, 577, 31, 4096, 112, 17, 0, 4088, 4096, 0, 3595, 4024, 4019, 0, 2, 9, 0, 0, 4089, 0, 1, 4078, 1, 0, 0, 0, 4072, 600, 11, 0, 0, 3688, 0, 1183, 79, 0, 2, 4096, 4096, 66, 2, 0, 0, 0, 0, 4027, 238, 4066, 4096, 4057, 0, 927, 1, 0, 130, 0, 0, 46, 746, 307, 0, 380, 0, 3116, 3777, 0, 4096, 328, 3997, 386, 0, 3, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 41, 0, 3778, 0, 0, 48, 0, 4096, 0, 113, 2820, 0, 193, 0, 0, 4096, 348, 0, 165, 0, 398, 4096, 0, 1, 0, 0, 0, 0, 0, 3849, 1698, 0, 3094, 3, 3815, 4006, 0, 0, 634, 0, 51, 0, 88, 573, 0, 4095, 3890, 0, 0, 3618, 3829, 0, 0, 4037, 3313, 201, 3263, 0, 408, 787, 1435, 62, 795, 4002, 3355, 4065, 3749, 39, 69, 0, 0, 4042, 0, 0, 0, 0, 3831, 0, 0, 1041, 4, 219, 3582, 219, 0, 424, 3723, 2044, 0, 0, 3506, 0, 1050, 420, 12, 0, 0, 0, 28, 46, 0, 4072, 49, 2, 241, 0, 11, 1639, 3516, 0, 0, 0, 0, 408, 0, 3797, 0, 590, 0, 4080, 0, 19, 582, 3909, 0, 0, 950] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4, 0, 30, 245, 0, 0, 0, 0, 3566, 2320, 0, 0, 0, 0, 0, 0, 1, 0, 0, 110, 0, 0, 4096, 0, 161, 4096, 0, 2460, 0, 38, 108, 1531, 0, 1, 0, 0, 3288, 1, 7, 1, 18, 0, 0, 27, 1086, 3010, 3245, 1, 1, 0, 3967, 0, 3980, 0, 0, 44, 0, 2458, 29, 0, 0, 4096, 4095, 1, 4067, 4094, 4072, 1, 4054, 0, 0, 1, 4096, 19, 0, 0, 26, 0, 0, 0, 51, 4096, 3645, 0, 7, 2239, 0, 60, 4, 0, 20, 4088, 4068, 25, 0, 210, 0, 1734, 3900, 538, 4085, 762, 4096, 3438, 26, 3322, 4069, 141, 1, 37, 1842, 604, 941, 144, 0, 0, 1, 3774, 0, 0, 2848, 0, 1, 2272, 4095, 100, 1236, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 3804, 3913, 2802, 469, 0, 0, 0, 0, 77, 0, 3220, 4081, 0, 0, 0, 67, 3600, 0, 0, 4062, 0, 4089, 16, 78, 4096, 214, 0, 0, 0, 2, 0, 0, 0, 0, 3272, 35, 0, 0, 4, 214, 0, 3944, 1, 156, 355, 0, 2271, 0, 445, 4096, 0, 1, 303, 2537, 36, 878, 4033, 0, 222, 0, 3862, 3720, 84, 0, 120, 0, 2, 2163, 0, 25, 0, 4087, 3503, 0, 991, 0, 0, 0, 0, 3701, 0, 891, 0, 0, 5, 0, 3939, 4011, 0, 18, 2201, 3779, 0, 29, 0, 0, 0, 387, 24, 3974, 2599, 1856, 4076, 1, 2, 4, 3620, 2890, 270, 1, 13, 2610, 43, 0, 0, 54, 288, 4096, 0, 4034, 0, 4096, 4, 1393, 0, 16, 197] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 42, 61, 3900, 0, 0, 1209, 43, 0, 0, 4092, 0, 2518, 0, 0, 3, 0, 0, 1, 0, 0, 477, 6, 1, 2033, 0, 91, 0, 2, 4062, 4, 0, 4095, 0, 1, 7, 110, 1701, 185, 46, 0, 7, 959, 0, 1669, 124, 0, 1, 0, 1, 0, 619, 0, 4054, 3271, 370, 4093, 1779, 0, 0, 26, 4094, 0, 3080, 1, 37, 6, 0, 3922, 152, 2, 4096, 14, 3968, 2730, 3, 0, 76, 0, 4033, 4096, 0, 0, 4044, 3849, 0, 37, 988, 0, 0, 4066, 3197, 2, 0, 0, 0, 0, 0, 4089, 2710, 4096, 4096, 4095, 2, 3, 4026, 0, 3, 0, 4045, 18, 0, 3, 0, 222, 0, 3801, 4054, 0, 3352, 0, 3982, 0, 104, 212, 2, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[539, 5, 1819, 4024, 4095, 0, 0, 322, 650, 69, 19, 6, 4071, 8, 0, 0, 1502, 1072, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3635, 0, 3835, 0, 0, 2785, 1916, 0, 0, 0, 3722, 965, 1754, 1653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 2521, 0, 6, 4, 3771, 186, 2, 1249, 0, 1, 4084, 363, 4092, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3886, 163, 730, 5, 30, 0, 2, 941, 1, 22, 5, 2200, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 369, 3081, 0, 149, 912, 0, 0, 3327, 0, 3, 4083, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 1, 0, 0, 17, 65, 0, 4095, 0, 9, 0, 104, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 4046, 0, 3019, 3867, 112, 2252, 10, 2969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 2, 0, 4033, 0, 4064, 4096, 468, 4096, 82, 831, 426, 0, 2, 0, 3018, 610, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18242, 20266, 17097, 12085, 12395, 8347, 20304, 22336] avg=16384.0000 max=22336 max_rank=7 min=8347 min_rank=5 imbalance_ratio=1.363281 max_load_reduction=108736 max_load_reduction_pct=0.829590 moved_to_echo=63382 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 17, 4077, 0, 0, 0, 0, 0, 0, 0, 4088, 3997, 0, 0, 0, 1506, 1076, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 16, 0, 388, 4001, 0, 4086, 0, 3, 3150, 0, 2, 1658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3943, 3858, 1, 0, 148, 0, 0, 1183, 135, 3432, 3953, 53, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2311, 648, 1, 0, 0, 4076, 0, 932, 20, 0, 0, 4096, 2549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4083, 17, 23, 0, 244, 3109, 3415, 4, 0, 0, 0, 3480, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2944, 4096, 0, 2886, 4096, 4096, 0, 4096, 4, 0, 0, 0, 3617, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 1, 128, 0, 3019, 3184, 3961, 7, 0, 219, 4, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 3, 2981, 0, 0, 1507, 0, 91, 2016, 0, 0, 1191, 0, 634, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14808, 13307, 20804, 14633, 18379, 25848, 14489, 8804] avg=16384.0000 max=25848 max_rank=5 min=8804 min_rank=7 imbalance_ratio=1.577637 max_load_reduction=105224 max_load_reduction_pct=0.802795 moved_to_echo=67520 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3977, 216, 2450, 3, 0, 0, 1, 4087, 2197, 0, 3997, 3688, 0, 0, 0, 1502, 1063, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 27, 0, 205, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 2805, 1653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 687, 0, 33, 113, 172, 0, 25, 9, 1954, 33, 4037, 0, 8, 4096, 1291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2866, 329, 4096, 0, 2170, 577, 31, 941, 112, 17, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3595, 4024, 4019, 0, 2, 9, 0, 0, 3420, 0, 1, 4078, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 600, 11, 0, 0, 3688, 0, 1183, 79, 0, 2, 0, 3718, 66, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 238, 4066, 0, 2994, 0, 927, 1, 0, 130, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 746, 307, 0, 380, 0, 3116, 3777, 0, 4096, 328, 3997, 386, 0, 3, 0, 1, 627, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23223, 12901, 12484, 19323, 19149, 13421, 12429, 18142] avg=16384.0000 max=23223 max_rank=0 min=12429 min_rank=6 imbalance_ratio=1.417419 max_load_reduction=107849 max_load_reduction_pct=0.822823 moved_to_echo=63141 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3211, 591, 0, 4096, 0, 0, 0, 2150, 0, 3, 3568, 2195, 35, 13, 660, 1399, 1052, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 449, 3, 232, 2496, 10, 0, 4094, 0, 3975, 0, 3, 3713, 53, 2795, 1539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 68, 11, 21, 10, 0, 0, 0, 0, 0, 1619, 0, 4, 916, 4037, 0, 4095, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1182, 200, 24, 138, 1859, 0, 4095, 4, 0, 876, 270, 0, 3, 4011, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1091, 2760, 1026, 0, 10, 0, 0, 0, 3398, 0, 0, 0, 2470, 0, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2387, 4064, 0, 248, 4094, 1270, 38, 609, 3815, 0, 400, 0, 3714, 34, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4089, 0, 2962, 286, 0, 592, 0, 29, 0, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4, 3540, 0, 7, 4096, 5, 2902, 0, 3509, 752, 108, 0, 0, 0, 623, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19015, 19364, 12105, 16753, 11260, 20693, 15925, 15957] avg=16384.0000 max=20693 max_rank=5 min=11260 min_rank=4 imbalance_ratio=1.263000 max_load_reduction=110379 max_load_reduction_pct=0.842125 moved_to_echo=63835 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 30, 245, 0, 0, 0, 0, 3566, 2320, 0, 0, 0, 0, 0, 0, 901, 901, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 110, 0, 0, 4096, 0, 161, 4096, 0, 2460, 0, 38, 108, 1531, 2799, 992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3288, 1, 7, 1, 18, 0, 0, 27, 1086, 3010, 3245, 1, 4096, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3967, 0, 3980, 0, 0, 44, 0, 565, 29, 0, 0, 4096, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4094, 4072, 1, 4054, 0, 0, 1, 3426, 19, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 4096, 3645, 0, 7, 2239, 0, 60, 4, 0, 20, 0, 3693, 25, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1734, 3900, 538, 4085, 762, 0, 2537, 26, 3322, 4069, 141, 1, 37, 1842, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 941, 144, 0, 0, 1, 3774, 0, 0, 2848, 0, 1, 2272, 4095, 100, 1236, 0, 628, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8009, 16392, 16070, 16778, 19760, 14050, 23598, 16415] avg=16384.0000 max=23598 max_rank=6 min=8009 min_rank=0 imbalance_ratio=1.440308 max_load_reduction=107474 max_load_reduction_pct=0.819962 moved_to_echo=73823 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 41, 0, 3778, 0, 0, 48, 0, 4096, 0, 113, 2820, 0, 193, 0, 0, 1196, 12, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 348, 0, 165, 0, 398, 4096, 0, 1, 0, 0, 0, 0, 0, 3849, 1698, 2401, 1317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3094, 3, 3815, 4006, 0, 0, 634, 0, 51, 0, 88, 573, 0, 4095, 3890, 4072, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3618, 3829, 0, 0, 4037, 3313, 201, 750, 0, 408, 787, 1435, 62, 795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 3355, 4065, 3749, 39, 69, 0, 0, 3380, 0, 0, 0, 0, 3831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 4, 219, 3582, 219, 0, 424, 3723, 2044, 0, 0, 0, 0, 1050, 420, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 46, 0, 0, 37, 2, 241, 0, 11, 1639, 3516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 408, 0, 3797, 0, 590, 0, 4080, 0, 19, 582, 3909, 0, 0, 950, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12339, 18369, 25426, 19235, 22490, 12738, 5520, 14955] avg=16384.0000 max=25426 max_rank=2 min=5520 min_rank=6 imbalance_ratio=1.551880 max_load_reduction=105646 max_load_reduction_pct=0.806015 moved_to_echo=55703 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3804, 3913, 2802, 469, 0, 0, 0, 0, 77, 0, 3220, 4081, 0, 0, 0, 1479, 1068, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 3600, 0, 0, 4062, 0, 4089, 16, 78, 4096, 214, 0, 0, 0, 2, 0, 1506, 1628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3272, 35, 0, 0, 4, 214, 0, 3944, 1, 156, 355, 0, 2271, 1856, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 4096, 0, 1, 303, 2537, 36, 878, 926, 0, 222, 0, 3862, 3720, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 2, 2163, 0, 25, 0, 3418, 3503, 0, 991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3701, 0, 891, 0, 0, 5, 0, 3939, 4011, 0, 18, 0, 3430, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 24, 3974, 2599, 0, 3008, 1, 2, 4, 3620, 2890, 270, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2610, 43, 0, 0, 54, 288, 4096, 0, 4034, 0, 4096, 4, 1393, 0, 16, 197, 627, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20955, 19358, 12803, 17110, 10222, 16024, 16793, 17807] avg=16384.0000 max=20955 max_rank=0 min=10222 min_rank=4 imbalance_ratio=1.278992 max_load_reduction=110117 max_load_reduction_pct=0.840126 moved_to_echo=60846 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 42, 61, 3900, 0, 0, 1209, 43, 0, 0, 4092, 0, 2518, 0, 0, 1501, 1073, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 477, 6, 1, 2033, 0, 91, 0, 2, 4062, 4, 2783, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 7, 110, 1701, 185, 46, 0, 7, 959, 0, 1669, 124, 0, 4096, 1283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 619, 0, 4054, 3271, 370, 940, 1779, 0, 0, 26, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3080, 1, 37, 6, 0, 3922, 152, 2, 3426, 14, 3968, 2730, 3, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 4096, 0, 0, 4044, 3849, 0, 37, 988, 0, 0, 0, 2902, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 2710, 4096, 0, 3022, 2, 3, 4026, 0, 3, 0, 4045, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 222, 0, 3801, 4054, 0, 3352, 0, 3982, 0, 104, 212, 2, 1, 628, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14481, 11115, 14283, 15155, 17417, 19951, 22014, 16656] avg=16384.0000 max=22014 max_rank=6 min=11115 min_rank=1 imbalance_ratio=1.343628 max_load_reduction=109058 max_load_reduction_pct=0.832047 moved_to_echo=76038 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[640, 11136, 6656, 17536, 12672, 0, 128, 1536, 14592, 4736, 256, 21888, 18048, 2816, 128, 768, 11008, 7424, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 7296, 128, 11776, 13824, 256, 1920, 1152, 4096, 384, 6784, 4480, 5376, 13952, 15744, 6656, 30592, 7808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[22400, 17024, 4864, 6784, 12544, 15232, 640, 13696, 15104, 0, 512, 0, 21248, 1280, 512, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[4224, 4096, 0, 768, 7808, 640, 19328, 128, 640, 25216, 2176, 10624, 0, 128, 22784, 4352, 16896, 12160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1280, 768, 20864, 5120, 11392, 384, 17024, 11392, 1536, 6912, 2304, 768, 896, 23936, 26880, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[4992, 640, 512, 8192, 3968, 18048, 20736, 512, 27008, 512, 16640, 6528, 9728, 384, 2560, 4224, 5120, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 1920, 8320, 8832, 15232, 19840, 0, 20608, 7424, 8576, 11008, 3840, 7936, 3840, 13696, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[16384, 14848, 20480, 3840, 6528, 4992, 512, 3200, 27264, 3584, 4096, 11904, 2560, 7424, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 1392, 1306, 120, 0, 39, 218, 0, 0, 3802, 0, 713, 60, 8, 0, 2108, 28, 2, 80, 0, 24, 2561, 53, 641, 3096, 2166, 1626, 31, 632, 71, 0, 205, 0, 0, 23, 31, 651, 4096, 1331, 0, 0, 79, 4093, 0, 4061, 0, 0, 477, 25, 0, 3271, 0, 2743, 25, 174, 166, 4054, 0, 0, 2, 0, 0, 4092, 0, 3777, 0, 0, 0, 0, 3, 4096, 55, 0, 0, 0, 36, 0, 356, 0, 0, 3943, 3508, 4076, 0, 8, 3998, 4096, 0, 85, 6, 0, 0, 38, 98, 2620, 2, 0, 60, 2726, 0, 4096, 0, 0, 3596, 3774, 109, 0, 4066, 0, 12, 84, 4000, 4092, 3938, 2736, 0, 584, 0, 3545, 2786, 7, 0, 0, 2562, 2174, 17, 835] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 537, 21, 1, 3622, 0, 0, 3690, 0, 2334, 0, 0, 0, 0, 0, 4056, 0, 0, 84, 0, 11, 0, 0, 54, 2374, 27, 493, 0, 0, 589, 0, 0, 305, 17, 2110, 0, 3359, 4030, 0, 0, 0, 4050, 1498, 3, 8, 42, 4096, 217, 187, 1327, 4096, 0, 3429, 130, 0, 129, 4096, 0, 439, 0, 4, 7, 4093, 0, 100, 0, 0, 0, 0, 0, 4096, 4083, 1864, 4042, 4089, 0, 0, 0, 1, 0, 4085, 1, 3237, 11, 2, 3808, 3855, 0, 0, 0, 19, 0, 0, 657, 4095, 0, 0, 0, 3899, 0, 0, 3805, 0, 4096, 2304, 0, 0, 4085, 0, 0, 3851, 6, 561, 4089, 52, 0, 0, 0, 0, 237, 0, 0, 0, 2945, 3436, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[67, 0, 0, 0, 3926, 1, 0, 0, 0, 0, 3986, 0, 8, 0, 0, 682, 4095, 0, 4, 4025, 0, 0, 142, 0, 66, 0, 0, 3, 0, 0, 18, 0, 1, 4077, 0, 4096, 0, 3907, 8, 0, 4093, 4094, 0, 3146, 0, 3020, 14, 3, 78, 3254, 569, 4090, 0, 1519, 4096, 2, 3396, 3878, 0, 424, 0, 0, 0, 4096, 11, 4077, 1127, 537, 0, 0, 0, 1320, 0, 0, 1, 2556, 1, 0, 0, 54, 0, 542, 839, 16, 3261, 0, 106, 4081, 98, 1, 0, 0, 3, 0, 6, 4096, 1, 0, 0, 4094, 649, 0, 4095, 0, 160, 2, 7, 0, 4096, 45, 4074, 104, 2179, 0, 3287, 3979, 678, 3, 0, 4000, 813, 0, 4095, 0, 1, 3092, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[3728, 5, 0, 0, 2719, 64, 0, 3567, 0, 39, 4091, 0, 0, 4, 3, 0, 240, 0, 0, 54, 1, 0, 2140, 310, 0, 1992, 0, 381, 0, 0, 0, 102, 0, 0, 363, 279, 0, 0, 0, 1272, 0, 3797, 3632, 1604, 0, 108, 2334, 4075, 1375, 4, 161, 0, 0, 3896, 0, 0, 4065, 4080, 0, 0, 0, 0, 187, 4096, 4052, 3689, 0, 0, 67, 0, 0, 3915, 4096, 4, 4021, 1, 14, 1, 147, 6, 0, 3721, 13, 430, 3893, 0, 2875, 3632, 0, 0, 0, 1, 3211, 0, 0, 897, 2, 92, 0, 3205, 0, 4096, 4016, 69, 3943, 633, 0, 0, 4096, 0, 0, 4073, 0, 1724, 4092, 0, 0, 7, 0, 0, 4092, 91, 0, 9, 3264, 114, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4096, 0, 4018, 1585, 1546, 0, 31, 54, 134, 0, 4061, 0, 0, 0, 0, 0, 4094, 0, 273, 99, 0, 37, 3216, 1124, 3, 4095, 3875, 0, 1161, 0, 4, 2, 0, 10, 8, 4020, 0, 0, 3288, 81, 0, 4090, 4091, 3724, 5, 3, 2, 1070, 1161, 0, 638, 2957, 0, 4087, 2, 0, 894, 3445, 0, 0, 160, 7, 4, 3733, 2153, 114, 0, 0, 0, 0, 0, 4096, 19, 273, 4096, 1, 4094, 0, 0, 36, 583, 4075, 0, 127, 3667, 1, 0, 1736, 0, 0, 0, 6, 0, 0, 232, 2567, 235, 0, 1368, 2799, 0, 59, 4095, 0, 40, 1584, 0, 0, 3028, 0, 3746, 0, 3, 3617, 3437, 3900, 0, 0, 2, 3548, 0, 0, 0, 0, 0, 0, 78, 669] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[2323, 0, 255, 3950, 2775, 1049, 0, 0, 150, 47, 3458, 3955, 0, 0, 0, 3151, 4095, 0, 0, 148, 0, 0, 4096, 49, 501, 0, 0, 1, 141, 31, 13, 1067, 4, 0, 0, 3, 0, 4088, 0, 0, 0, 2921, 2117, 3499, 0, 0, 0, 0, 0, 29, 8, 4079, 0, 289, 31, 1, 25, 2799, 0, 559, 140, 0, 4061, 4096, 2626, 1828, 4096, 0, 0, 328, 34, 4050, 2, 612, 1, 4080, 4074, 0, 0, 0, 0, 6, 1457, 8, 117, 0, 104, 46, 6, 3, 0, 25, 0, 0, 175, 4025, 43, 0, 0, 1599, 142, 0, 4077, 0, 3234, 4096, 1713, 0, 4096, 0, 0, 511, 1251, 800, 51, 1912, 0, 8, 4015, 1502, 3796, 0, 270, 3479, 3625, 0, 0, 3145] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3656, 0, 1, 1379, 4096, 73, 0, 1238, 0, 26, 3977, 0, 99, 3, 15, 0, 826, 25, 955, 0, 399, 0, 0, 4096, 0, 24, 4091, 0, 314, 259, 736, 3, 230, 0, 471, 0, 269, 3992, 3686, 4089, 0, 0, 0, 38, 1, 0, 1, 0, 0, 3865, 25, 1012, 3257, 4043, 3975, 0, 14, 3886, 0, 2847, 9, 0, 136, 2218, 270, 3569, 3506, 49, 0, 0, 1541, 265, 10, 171, 0, 0, 3, 0, 0, 3804, 444, 4096, 3135, 4027, 513, 116, 2104, 0, 0, 0, 189, 0, 25, 702, 0, 0, 0, 1, 9, 7, 8, 4050, 0, 670, 0, 3946, 1, 0, 3881, 0, 0, 342, 0, 4076, 3802, 3669, 0, 2581, 0, 4092, 0, 0, 0, 4096, 0, 2947, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[3824, 0, 3667, 166, 2864, 2, 0, 0, 3984, 4094, 2311, 3119, 2, 1, 0, 410, 4006, 0, 0, 0, 0, 0, 1286, 0, 0, 0, 0, 121, 3870, 0, 4, 112, 103, 194, 0, 719, 0, 0, 4085, 0, 3227, 7, 3959, 1895, 0, 0, 43, 0, 3, 1163, 14, 3764, 0, 4054, 4083, 1, 0, 2, 1, 3, 0, 0, 0, 4096, 32, 4096, 2, 0, 0, 4, 0, 83, 0, 0, 3931, 3805, 2846, 0, 0, 2921, 0, 313, 0, 25, 0, 0, 4032, 4094, 4, 2, 0, 0, 0, 0, 21, 10, 1544, 0, 0, 4095, 119, 0, 4095, 357, 7, 3593, 2, 0, 4096, 1323, 3963, 0, 3485, 4054, 6, 12, 162, 6, 0, 4096, 3636, 0, 0, 0, 941, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 1392, 1306, 120, 0, 39, 218, 0, 0, 3802, 0, 713, 60, 8, 0, 1126, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2108, 28, 2, 80, 0, 24, 2561, 53, 641, 3096, 2166, 1626, 31, 632, 71, 0, 3054, 3938, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 23, 31, 651, 4096, 1331, 0, 0, 79, 2630, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 25, 0, 3271, 0, 2743, 25, 174, 166, 4054, 0, 0, 2, 0, 0, 1270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3777, 0, 0, 0, 0, 3, 4096, 55, 0, 0, 0, 36, 0, 356, 0, 757, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 3508, 4076, 0, 8, 3998, 4096, 0, 85, 6, 0, 0, 38, 98, 2620, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 60, 2726, 0, 4096, 0, 0, 3596, 3774, 109, 0, 1012, 0, 12, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 4092, 0, 2736, 0, 584, 0, 3355, 2786, 7, 0, 0, 2562, 2174, 17, 835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13070, 20261, 13107, 12207, 9417, 24391, 15471, 23148] avg=16384.0000 max=24391 max_rank=5 min=9417 min_rank=4 imbalance_ratio=1.488708 max_load_reduction=106681 max_load_reduction_pct=0.813911 moved_to_echo=72427 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4018, 1585, 1546, 0, 31, 54, 134, 0, 4061, 0, 0, 0, 0, 0, 1024, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 273, 99, 0, 37, 3216, 1124, 3, 4095, 3875, 0, 1161, 0, 4, 2, 2274, 3437, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8, 4020, 0, 0, 3288, 81, 0, 4090, 4091, 2393, 5, 3, 2, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1161, 0, 638, 2957, 0, 4087, 2, 0, 894, 3445, 0, 0, 160, 7, 4, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2153, 114, 0, 0, 0, 0, 0, 4096, 19, 273, 4096, 1, 4094, 0, 0, 36, 690, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 4075, 0, 127, 3667, 1, 0, 1736, 0, 0, 0, 6, 0, 0, 232, 2567, 1747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 235, 0, 1368, 2799, 0, 59, 4095, 0, 40, 1584, 0, 0, 754, 0, 3746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3617, 0, 3900, 0, 0, 2, 3358, 0, 0, 0, 0, 0, 0, 78, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16739, 23830, 19061, 14515, 15879, 14741, 14680, 11627] avg=16384.0000 max=23830 max_rank=1 min=11627 min_rank=7 imbalance_ratio=1.454468 max_load_reduction=107242 max_load_reduction_pct=0.818192 moved_to_echo=56927 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 537, 21, 1, 3622, 0, 0, 3690, 0, 2334, 0, 0, 0, 0, 0, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 84, 0, 11, 0, 0, 54, 2374, 27, 493, 0, 0, 589, 0, 3068, 4089, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 17, 2110, 0, 3359, 4030, 0, 0, 0, 4050, 963, 3, 8, 42, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 187, 1327, 4096, 0, 3429, 130, 0, 129, 4096, 0, 439, 0, 4, 7, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 4096, 4083, 1864, 4042, 4089, 0, 0, 0, 1, 757, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 3237, 11, 2, 3808, 3855, 0, 0, 0, 19, 0, 0, 657, 4095, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3899, 0, 0, 3805, 0, 4096, 2304, 0, 0, 1017, 0, 0, 3851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 561, 0, 52, 0, 0, 0, 0, 237, 0, 0, 0, 2945, 3436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14713, 14995, 18983, 15332, 19155, 21685, 18972, 7237] avg=16384.0000 max=21685 max_rank=5 min=7237 min_rank=7 imbalance_ratio=1.323547 max_load_reduction=109387 max_load_reduction_pct=0.834557 moved_to_echo=67049 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[67, 0, 0, 0, 3926, 1, 0, 0, 0, 0, 3986, 0, 8, 0, 0, 682, 868, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 4025, 0, 0, 142, 0, 66, 0, 0, 3, 0, 0, 18, 0, 3080, 3287, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4077, 0, 4096, 0, 3907, 8, 0, 4093, 4094, 0, 2016, 0, 3020, 14, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 3254, 569, 4090, 0, 1519, 4096, 2, 3396, 3878, 0, 424, 0, 0, 0, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4077, 1127, 537, 0, 0, 0, 1320, 0, 0, 1, 2556, 1, 0, 0, 54, 760, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 839, 16, 3261, 0, 106, 4081, 98, 1, 0, 0, 3, 0, 6, 4096, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 649, 0, 4095, 0, 160, 2, 7, 0, 1016, 45, 4074, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2179, 0, 0, 3979, 678, 3, 0, 3782, 813, 0, 4095, 0, 1, 3092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9756, 14872, 25329, 22570, 10706, 14969, 14247, 18623] avg=16384.0000 max=25329 max_rank=2 min=9756 min_rank=0 imbalance_ratio=1.545959 max_load_reduction=105743 max_load_reduction_pct=0.806755 moved_to_echo=58545 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2323, 0, 255, 3950, 2775, 1049, 0, 0, 150, 47, 3458, 3955, 0, 0, 0, 3151, 962, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 148, 0, 0, 4096, 49, 501, 0, 0, 1, 141, 31, 13, 1067, 3076, 51, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3, 0, 4088, 0, 0, 0, 2921, 2117, 2249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 8, 4079, 0, 289, 31, 1, 25, 2799, 0, 559, 140, 0, 4061, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2626, 1828, 4096, 0, 0, 328, 34, 4050, 2, 612, 1, 4080, 4074, 0, 0, 0, 758, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1457, 8, 117, 0, 104, 46, 6, 3, 0, 25, 0, 0, 175, 4025, 1917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 1599, 142, 0, 4077, 0, 3234, 4096, 1713, 0, 1020, 0, 0, 511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1251, 800, 0, 1912, 0, 8, 4015, 1422, 3796, 0, 270, 3479, 3625, 0, 0, 3145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22155, 13419, 11382, 13292, 22777, 7889, 16435, 23723] avg=16384.0000 max=23723 max_rank=7 min=7889 min_rank=5 imbalance_ratio=1.447937 max_load_reduction=107349 max_load_reduction_pct=0.819008 moved_to_echo=70824 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3728, 5, 0, 0, 2719, 64, 0, 3567, 0, 39, 4091, 0, 0, 4, 3, 0, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 54, 1, 0, 2140, 310, 0, 1992, 0, 381, 0, 0, 0, 102, 3076, 4092, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 363, 279, 0, 0, 0, 1272, 0, 3797, 3632, 1031, 0, 108, 2334, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1375, 4, 161, 0, 0, 3896, 0, 0, 4065, 4080, 0, 0, 0, 0, 187, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 3689, 0, 0, 67, 0, 0, 3915, 4096, 4, 4021, 1, 14, 1, 147, 6, 758, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3721, 13, 430, 3893, 0, 2875, 3632, 0, 0, 0, 1, 3211, 0, 0, 897, 1917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 92, 0, 3205, 0, 4096, 4016, 69, 3943, 633, 0, 0, 1020, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1724, 0, 0, 0, 7, 0, 0, 4092, 91, 0, 9, 3264, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14661, 12538, 16891, 15039, 20903, 20590, 21149, 9301] avg=16384.0000 max=21149 max_rank=6 min=9301 min_rank=7 imbalance_ratio=1.290833 max_load_reduction=109923 max_load_reduction_pct=0.838646 moved_to_echo=71943 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3656, 0, 1, 1379, 4096, 73, 0, 1238, 0, 26, 3977, 0, 99, 3, 15, 0, 10, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 25, 955, 0, 399, 0, 0, 4096, 0, 24, 4091, 0, 314, 259, 736, 3, 2915, 3802, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 471, 0, 269, 3992, 3686, 4089, 0, 0, 0, 25, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 25, 1012, 3257, 4043, 3975, 0, 14, 3886, 0, 2847, 9, 0, 136, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 3569, 3506, 49, 0, 0, 1541, 265, 10, 171, 0, 0, 3, 0, 0, 3804, 410, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 444, 4096, 3135, 4027, 513, 116, 2104, 0, 0, 0, 189, 0, 25, 702, 0, 0, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 7, 8, 4050, 0, 670, 0, 3946, 1, 0, 966, 0, 0, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 3669, 0, 2581, 0, 3872, 0, 0, 0, 4096, 0, 2947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14793, 18526, 12764, 23758, 13601, 16389, 10000, 21241] avg=16384.0000 max=23758 max_rank=3 min=10000 min_rank=6 imbalance_ratio=1.450073 max_load_reduction=107314 max_load_reduction_pct=0.818741 moved_to_echo=61231 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3824, 0, 3667, 166, 2864, 2, 0, 0, 3984, 4094, 2311, 3119, 2, 1, 0, 410, 521, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 0, 0, 1286, 0, 0, 0, 0, 121, 3870, 0, 4, 112, 3076, 6, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 194, 0, 719, 0, 0, 4085, 0, 3227, 7, 3959, 1218, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1163, 14, 3764, 0, 4054, 4083, 1, 0, 2, 1, 3, 0, 0, 0, 1271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4096, 2, 0, 0, 4, 0, 83, 0, 0, 3931, 3805, 2846, 0, 0, 2921, 758, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 0, 25, 0, 0, 4032, 4094, 4, 2, 0, 0, 0, 0, 21, 10, 1917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1544, 0, 0, 4095, 119, 0, 4095, 357, 7, 3593, 2, 0, 1020, 1323, 3963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3485, 4054, 0, 12, 162, 6, 0, 3876, 3636, 0, 0, 0, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[25185, 12631, 13555, 14359, 18634, 10418, 20118, 16172] avg=16384.0000 max=25185 max_rank=0 min=10418 min_rank=5 imbalance_ratio=1.537170 max_load_reduction=105887 max_load_reduction_pct=0.807854 moved_to_echo=65342 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[25984, 128, 9984, 8448, 18048, 4864, 128, 5120, 8064, 4224, 28032, 7168, 896, 128, 128, 4352, 5376, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1920, 128, 1536, 22528, 1024, 12416, 24192, 1152, 15104, 19968, 1920, 0, 7936, 1408, 11904, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[23552, 128, 1280, 4608, 512, 128, 13568, 5632, 1280, 11648, 10240, 2688, 5632, 1024, 1536, 1408, 23680, 22784, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[3328, 8576, 2816, 23296, 3328, 24064, 12416, 256, 8704, 26240, 128, 4352, 384, 128, 4480, 9472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[640, 4608, 896, 11264, 384, 16000, 19200, 6784, 7424, 14976, 18048, 12544, 128, 7296, 2560, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[11008, 18944, 0, 16384, 896, 3200, 4096, 19712, 15360, 128, 4480, 7680, 13440, 11776, 128, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1152, 20864, 8960, 12032, 11520, 128, 17152, 21632, 128, 128, 256, 128, 3328, 768, 1280, 18432, 14336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[9216, 21376, 8832, 640, 128, 384, 1664, 22016, 8320, 2944, 16128, 14592, 11136, 128, 512, 6912, 5760, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 15, 0, 0, 0, 0, 0, 1, 3754, 3, 424, 1, 1023, 0, 0, 2225, 0, 211, 1, 0, 3883, 308, 4096, 0, 0, 4096, 0, 68, 1154, 0, 4065, 0, 1, 4053, 31, 0, 0, 3085, 1151, 0, 0, 0, 0, 0, 27, 158, 3652, 4, 0, 53, 0, 2611, 0, 0, 169, 563, 0, 0, 98, 0, 4095, 0, 0, 17, 3998, 3789, 3, 111, 4095, 0, 3855, 2968, 10, 3653, 3096, 2, 3990, 10, 1, 0, 15, 2785, 2917, 4074, 3277, 88, 3, 0, 4, 64, 3553, 4, 54, 0, 13, 0, 0, 9, 3997, 4087, 0, 18, 3930, 456, 0, 4007, 0, 0, 0, 4096, 0, 0, 1044, 2194, 13, 0, 2464, 0, 0, 4067, 141, 0, 4096, 0, 0, 830, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[1, 334, 1, 0, 140, 3983, 5, 0, 40, 0, 0, 3809, 0, 15, 40, 0, 0, 1, 2341, 4030, 0, 1492, 130, 3996, 26, 8, 4095, 52, 0, 0, 0, 4066, 455, 0, 3001, 0, 1440, 0, 4067, 3951, 0, 4095, 56, 0, 977, 193, 355, 257, 0, 119, 4056, 0, 4040, 0, 0, 1211, 3601, 21, 0, 0, 3202, 2992, 0, 10, 4096, 3976, 0, 15, 81, 4089, 0, 37, 3793, 0, 0, 2807, 128, 0, 1466, 102, 0, 1011, 0, 3759, 2, 1, 0, 1, 232, 0, 1, 4071, 0, 4061, 76, 0, 0, 0, 6, 2796, 181, 0, 0, 1, 1, 0, 384, 0, 0, 50, 4008, 0, 65, 4087, 3750, 0, 0, 0, 76, 0, 3971, 3048, 0, 4082, 0, 0, 1, 4056] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 515, 2285, 9, 3441, 0, 3, 785, 4048, 1, 4, 7, 3589, 292, 0, 20, 0, 3782, 3070, 0, 0, 296, 1518, 0, 0, 4081, 0, 84, 0, 32, 4024, 10, 0, 4096, 0, 0, 0, 0, 389, 170, 4096, 332, 10, 3, 4096, 0, 2962, 6, 1, 4, 64, 4096, 2334, 1, 0, 4096, 0, 193, 4045, 15, 4074, 27, 1, 3797, 59, 539, 0, 3, 24, 0, 0, 4059, 3052, 0, 133, 0, 296, 0, 0, 0, 0, 0, 1957, 3583, 320, 0, 0, 0, 0, 1280, 0, 783, 0, 0, 968, 1, 871, 4093, 4084, 0, 0, 0, 2198, 1772, 0, 0, 55, 0, 15, 2334, 6, 3856, 4095, 3927, 0, 0, 0, 254, 6, 3651, 0, 0, 3997, 3920, 8, 0, 4069] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4095, 4079, 0, 57, 11, 29, 0, 0, 4075, 4092, 844, 770, 315, 0, 0, 0, 0, 682, 3771, 4096, 3970, 19, 1, 4096, 0, 0, 4096, 3976, 0, 3888, 0, 0, 0, 0, 4096, 0, 0, 0, 22, 3393, 0, 4096, 0, 0, 0, 793, 221, 0, 2, 1, 29, 0, 17, 0, 0, 0, 524, 0, 0, 0, 0, 690, 0, 1, 1, 1868, 933, 3931, 0, 19, 602, 0, 4065, 402, 0, 571, 124, 0, 4093, 0, 3864, 0, 0, 4078, 2319, 0, 0, 0, 1, 129, 0, 4096, 0, 0, 100, 0, 0, 239, 1925, 3728, 1, 39, 4004, 133, 4096, 1527, 101, 3, 4021, 3647, 0, 0, 11, 0, 8, 3395, 2176, 0, 2895, 0, 4096, 2, 0, 2981, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1430, 558, 4, 535, 0, 4094, 0, 0, 3982, 2250, 27, 0, 1, 154, 0, 0, 26, 59, 4084, 3414, 0, 0, 0, 4096, 0, 0, 4096, 9, 135, 0, 0, 0, 0, 0, 4096, 0, 689, 0, 0, 39, 0, 3891, 0, 0, 0, 1058, 0, 4088, 0, 0, 237, 0, 4094, 0, 0, 2, 4073, 0, 0, 0, 0, 4095, 0, 0, 2200, 3864, 2384, 21, 2, 4077, 0, 0, 3973, 15, 1, 3890, 2922, 12, 220, 0, 5, 4, 4096, 3269, 0, 19, 0, 5, 0, 9, 51, 4090, 0, 2060, 1, 0, 0, 0, 0, 3922, 4065, 0, 3417, 0, 3912, 0, 0, 0, 1, 0, 4095, 0, 661, 2734, 4086, 182, 0, 1, 0, 0, 3996, 238, 0, 4096, 0, 0, 0, 3160] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[335, 678, 2531, 2511, 13, 7, 0, 0, 3, 126, 4080, 11, 0, 60, 0, 0, 0, 0, 4078, 4088, 0, 331, 7, 3487, 0, 55, 4075, 1156, 0, 4096, 0, 0, 2, 0, 218, 0, 0, 0, 4072, 5, 719, 3808, 289, 0, 0, 4086, 0, 3953, 3980, 0, 0, 7, 3851, 0, 0, 0, 49, 0, 0, 1, 0, 4094, 0, 26, 0, 2242, 529, 0, 1034, 4040, 44, 0, 4096, 0, 0, 2341, 6, 2, 0, 0, 4025, 0, 1, 3503, 3730, 0, 0, 0, 0, 2357, 3562, 4094, 0, 0, 17, 100, 0, 189, 4068, 3857, 3119, 639, 0, 386, 4096, 12, 4096, 3, 1915, 4092, 0, 0, 2900, 0, 54, 0, 0, 0, 0, 0, 6, 1, 0, 4064, 0, 13, 0, 951] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[1, 0, 0, 3667, 11, 3993, 827, 4, 0, 0, 0, 3871, 0, 3881, 4083, 0, 1, 174, 12, 0, 174, 1672, 57, 0, 0, 0, 2720, 9, 901, 0, 118, 1, 932, 52, 850, 0, 55, 0, 3941, 4088, 13, 4096, 35, 4096, 0, 3506, 0, 0, 0, 64, 0, 0, 2566, 12, 4094, 0, 4093, 0, 1422, 1549, 1, 0, 0, 55, 3838, 2356, 134, 0, 43, 0, 145, 438, 23, 0, 1, 0, 3743, 0, 3974, 0, 1, 0, 0, 3928, 0, 3717, 0, 0, 316, 5, 806, 2, 0, 3244, 0, 0, 30, 0, 4044, 0, 1613, 0, 4086, 3960, 4049, 465, 992, 86, 158, 4065, 0, 3293, 3464, 212, 0, 93, 4095, 0, 101, 0, 4002, 133, 6, 6, 0, 0, 0, 3708] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[140, 3626, 0, 1, 5, 22, 2104, 0, 2404, 74, 157, 0, 1261, 0, 0, 0, 3512, 12, 180, 247, 1, 6, 11, 4093, 0, 0, 325, 0, 0, 4003, 23, 0, 0, 0, 4096, 3873, 0, 0, 3089, 4, 0, 127, 0, 0, 0, 3530, 0, 4084, 0, 0, 538, 0, 4096, 0, 0, 0, 4096, 2977, 417, 0, 0, 4095, 0, 0, 2337, 4033, 0, 0, 3922, 12, 0, 3619, 57, 4078, 0, 2, 0, 0, 4015, 2783, 0, 0, 3621, 558, 0, 3700, 851, 0, 0, 274, 0, 4096, 0, 3421, 210, 340, 0, 3, 493, 0, 2, 32, 4073, 4073, 2936, 24, 4096, 0, 866, 26, 3953, 14, 21, 0, 1369, 0, 0, 0, 0, 0, 2279, 3793, 0, 27, 3831, 0, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 15, 0, 0, 0, 0, 0, 1, 3754, 3, 424, 1, 1023, 0, 0, 2968, 1353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2225, 0, 211, 1, 0, 3883, 308, 4096, 0, 0, 2564, 0, 68, 1154, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4021, 31, 0, 0, 3085, 1151, 0, 0, 0, 0, 0, 27, 158, 3652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 53, 0, 2611, 0, 0, 169, 563, 0, 0, 98, 0, 4095, 0, 0, 2644, 1532, 671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3620, 3789, 3, 111, 4095, 0, 3855, 0, 10, 3653, 3096, 2, 3990, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2785, 2917, 4074, 3277, 88, 3, 0, 4, 64, 3553, 4, 54, 0, 13, 145, 677, 378, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4087, 0, 18, 3930, 311, 0, 4007, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 2194, 13, 0, 2464, 0, 0, 2719, 141, 0, 4096, 0, 0, 830, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9542, 18575, 12126, 12440, 26252, 18083, 16458, 17596] avg=16384.0000 max=26252 max_rank=4 min=9542 min_rank=0 imbalance_ratio=1.602295 max_load_reduction=104820 max_load_reduction_pct=0.799713 moved_to_echo=78389 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 334, 1, 0, 140, 3983, 5, 0, 40, 0, 0, 3809, 0, 15, 40, 0, 3793, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2341, 4030, 0, 1492, 130, 3996, 26, 8, 2564, 52, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, 2977, 0, 1440, 0, 4067, 3951, 0, 4095, 56, 0, 977, 193, 355, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 4056, 0, 4040, 0, 0, 1211, 3601, 21, 0, 0, 3202, 2992, 0, 10, 1850, 1531, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3600, 0, 15, 81, 4089, 0, 37, 0, 0, 0, 2807, 128, 0, 1466, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 0, 3759, 2, 1, 0, 1, 232, 0, 1, 4071, 0, 4061, 76, 0, 0, 661, 376, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 181, 0, 0, 1, 1, 0, 384, 0, 0, 50, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4087, 3750, 0, 0, 0, 76, 0, 2655, 3048, 0, 4082, 0, 0, 1, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13107, 18706, 18823, 23288, 16421, 14276, 4631, 21820] avg=16384.0000 max=23288 max_rank=3 min=4631 min_rank=6 imbalance_ratio=1.421387 max_load_reduction=107784 max_load_reduction_pct=0.822327 moved_to_echo=57148 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4095, 4079, 0, 57, 11, 29, 0, 0, 4075, 4092, 844, 770, 315, 0, 0, 0, 4065, 1263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 682, 3771, 4096, 3970, 19, 1, 4096, 0, 0, 2561, 3976, 0, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 22, 3393, 0, 4096, 0, 0, 0, 793, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 29, 0, 17, 0, 0, 0, 524, 0, 0, 0, 0, 690, 0, 1, 2465, 1535, 678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1687, 933, 3931, 0, 19, 602, 0, 0, 402, 0, 571, 124, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 4078, 2319, 0, 0, 0, 1, 129, 0, 4096, 0, 0, 100, 0, 1306, 686, 181, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 1925, 0, 1, 39, 4004, 133, 2790, 1527, 101, 3, 4021, 3647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 8, 3395, 2176, 0, 2895, 0, 2732, 2, 0, 2981, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23695, 27060, 12585, 5942, 12363, 16796, 18430, 14201] avg=16384.0000 max=27060 max_rank=1 min=5942 min_rank=3 imbalance_ratio=1.651611 max_load_reduction=104012 max_load_reduction_pct=0.793549 moved_to_echo=61790 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 515, 2285, 9, 3441, 0, 3, 785, 4048, 1, 4, 7, 3589, 292, 0, 4059, 1383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 3782, 3070, 0, 0, 296, 1518, 0, 0, 2555, 0, 84, 0, 32, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4063, 0, 0, 0, 0, 389, 170, 4096, 332, 10, 3, 4096, 0, 2962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 4, 64, 4096, 2334, 1, 0, 4096, 0, 193, 4045, 15, 4074, 27, 1, 2701, 1526, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3797, 54, 539, 0, 3, 24, 0, 0, 0, 3052, 0, 133, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1957, 3583, 320, 0, 0, 0, 0, 1280, 0, 783, 0, 0, 968, 563, 607, 5, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 871, 4093, 0, 0, 0, 0, 2198, 1209, 0, 0, 55, 0, 15, 2334, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3856, 4095, 3927, 0, 0, 0, 254, 6, 2442, 0, 0, 3997, 3920, 8, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20421, 15381, 16131, 23786, 7898, 10099, 10782, 26574] avg=16384.0000 max=26574 max_rank=7 min=7898 min_rank=4 imbalance_ratio=1.621948 max_load_reduction=104498 max_load_reduction_pct=0.797256 moved_to_echo=55353 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1430, 558, 4, 535, 0, 4094, 0, 0, 3982, 2250, 27, 0, 1, 154, 0, 0, 3973, 1329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 59, 4084, 3414, 0, 0, 0, 4096, 0, 0, 2564, 9, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 689, 0, 0, 39, 0, 3891, 0, 0, 0, 1058, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 4094, 0, 0, 2, 4073, 0, 0, 0, 0, 4095, 0, 0, 2593, 1532, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2200, 3499, 2384, 21, 2, 4077, 0, 0, 0, 15, 1, 3890, 2922, 12, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 4096, 3269, 0, 19, 0, 5, 0, 9, 51, 4090, 0, 2060, 1, 0, 1244, 665, 365, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 3417, 0, 2668, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 661, 2734, 4086, 182, 0, 1, 0, 0, 2672, 238, 0, 4096, 0, 0, 0, 3160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18337, 14387, 13828, 17285, 19243, 15916, 14246, 17830] avg=16384.0000 max=19243 max_rank=4 min=13828 min_rank=2 imbalance_ratio=1.174500 max_load_reduction=111829 max_load_reduction_pct=0.853188 moved_to_echo=67235 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[335, 678, 2531, 2511, 13, 7, 0, 0, 3, 126, 4080, 11, 0, 60, 0, 0, 4096, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4088, 0, 331, 7, 3487, 0, 55, 2551, 1156, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 217, 0, 0, 0, 4072, 5, 719, 3808, 289, 0, 0, 4086, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 7, 3851, 0, 0, 0, 49, 0, 0, 1, 0, 4094, 0, 26, 2550, 1524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2030, 529, 0, 1034, 4040, 44, 0, 0, 0, 0, 2341, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 1, 3503, 3730, 0, 0, 0, 0, 2357, 3562, 4094, 0, 0, 17, 100, 1302, 0, 212, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 4068, 0, 3119, 639, 0, 386, 2794, 12, 4096, 3, 1915, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2900, 0, 54, 0, 0, 0, 0, 0, 6, 1, 0, 4064, 0, 13, 0, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15758, 19849, 17151, 16082, 10026, 22904, 21313, 7989] avg=16384.0000 max=22904 max_rank=5 min=7989 min_rank=7 imbalance_ratio=1.397949 max_load_reduction=108168 max_load_reduction_pct=0.825256 moved_to_echo=62232 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3667, 11, 3993, 827, 4, 0, 0, 0, 3871, 0, 3881, 4083, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 174, 12, 0, 174, 1672, 57, 0, 0, 0, 1703, 9, 901, 0, 118, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 932, 52, 844, 0, 55, 0, 3941, 4088, 13, 4096, 35, 4096, 0, 3506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 2566, 12, 4094, 0, 4093, 0, 1422, 1549, 1, 0, 0, 55, 0, 1017, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 2133, 134, 0, 43, 0, 145, 438, 0, 0, 1, 0, 3743, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3928, 0, 3717, 0, 0, 316, 5, 806, 2, 0, 3244, 0, 0, 1287, 666, 223, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4044, 0, 1613, 0, 4086, 3960, 2762, 465, 992, 86, 158, 4065, 0, 3293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3464, 212, 0, 93, 4095, 0, 101, 0, 2676, 133, 6, 6, 0, 0, 0, 3708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20361, 4822, 21658, 15533, 14449, 14201, 25554, 14494] avg=16384.0000 max=25554 max_rank=6 min=4822 min_rank=1 imbalance_ratio=1.559692 max_load_reduction=105518 max_load_reduction_pct=0.805038 moved_to_echo=68698 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[140, 3626, 0, 1, 5, 22, 2104, 0, 2404, 74, 157, 0, 1261, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3512, 12, 180, 247, 1, 6, 11, 4093, 0, 0, 204, 0, 0, 4003, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 3873, 0, 0, 3089, 4, 0, 127, 0, 0, 0, 3530, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 538, 0, 4096, 0, 0, 0, 4096, 2977, 417, 0, 0, 4095, 0, 0, 0, 121, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2337, 3652, 0, 0, 3922, 12, 0, 3619, 0, 4078, 0, 2, 0, 0, 4015, 2783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3621, 558, 0, 3700, 851, 0, 0, 274, 0, 4096, 0, 3421, 210, 340, 933, 379, 381, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 493, 0, 2, 32, 4073, 4073, 2003, 24, 4096, 0, 866, 26, 3953, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 1369, 0, 0, 0, 0, 0, 1524, 3793, 0, 27, 3831, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9851, 12292, 18770, 16716, 24420, 18797, 19658, 10568] avg=16384.0000 max=24420 max_rank=4 min=9851 min_rank=0 imbalance_ratio=1.490479 max_load_reduction=106652 max_load_reduction_pct=0.813690 moved_to_echo=73443 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[6016, 9344, 3072, 9088, 256, 15616, 2944, 128, 11392, 14464, 5120, 8960, 1664, 8832, 4480, 0, 23040, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[5888, 1024, 18560, 19072, 4224, 7424, 896, 25472, 128, 128, 17280, 5248, 1280, 13184, 256, 12160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 1408, 14720, 0, 13184, 768, 15616, 14720, 14592, 2048, 13696, 256, 7040, 11904, 18560, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[16384, 20352, 8320, 4096, 5248, 16384, 896, 8064, 0, 7680, 3712, 12928, 7040, 4352, 13824, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[11008, 12288, 15488, 3712, 6272, 2560, 3328, 128, 17536, 7424, 128, 23424, 7808, 128, 896, 20096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1408, 128, 24320, 3968, 2304, 0, 18304, 13056, 1024, 24320, 768, 4224, 1024, 17408, 768, 19072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[7936, 1152, 10624, 24064, 13824, 11136, 1024, 128, 640, 2816, 5888, 24064, 896, 12928, 512, 1536, 6784, 4352, 2176, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[4096, 256, 4992, 128, 25472, 2432, 4096, 1408, 21120, 3072, 2048, 5760, 3328, 24192, 128, 128, 14848, 10368, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 2637, 0, 0, 74, 2905, 40, 3, 3706, 0, 0, 0, 20, 3208, 0, 0, 4096, 3838, 481, 0, 4072, 265, 0, 0, 0, 0, 0, 617, 8, 4096, 4073, 0, 4063, 22, 171, 133, 6, 771, 4, 0, 4096, 3707, 0, 0, 12, 4096, 319, 0, 3027, 0, 50, 223, 18, 4096, 0, 0, 65, 0, 0, 3847, 0, 0, 0, 1122, 0, 4084, 622, 3817, 0, 4094, 0, 9, 1161, 4043, 0, 0, 11, 4096, 3075, 0, 0, 5, 0, 0, 3596, 362, 77, 136, 3791, 375, 1, 0, 0, 4, 0, 4096, 0, 0, 0, 105, 10, 4081, 1, 2834, 0, 0, 4072, 0, 0, 1649, 3980, 0, 0, 3886, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 850, 0, 0, 0, 4066] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 3686, 1995, 0, 0, 0, 980, 0, 0, 0, 0, 0, 0, 3432, 0, 0, 4040, 4090, 542, 36, 1352, 0, 0, 0, 142, 0, 0, 1, 0, 4087, 1178, 30, 2, 4051, 141, 59, 3, 1935, 101, 0, 3946, 3000, 0, 2, 3945, 4051, 0, 0, 4095, 0, 556, 8, 0, 1850, 0, 100, 1, 0, 1693, 0, 3867, 0, 27, 4036, 0, 4039, 0, 0, 0, 3599, 1, 4096, 0, 62, 0, 3, 2061, 2976, 4094, 0, 0, 2736, 0, 3, 0, 0, 1, 60, 3986, 0, 0, 0, 3812, 4096, 1496, 4013, 0, 65, 6, 0, 10, 0, 0, 8, 152, 499, 2194, 4095, 0, 4093, 1358, 0, 0, 0, 3729, 0, 0, 0, 0, 1911, 0, 373, 54, 4080, 0, 0, 0, 155] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 968, 0, 81, 0, 2687, 0, 0, 0, 1, 3185, 0, 0, 4096, 4089, 4096, 0, 4, 9, 0, 4039, 19, 0, 3929, 0, 45, 4096, 4087, 32, 4065, 26, 4095, 0, 23, 4065, 843, 0, 4096, 7, 0, 0, 4091, 146, 3990, 0, 4040, 0, 2424, 2, 0, 62, 0, 0, 16, 0, 0, 4, 2533, 13, 0, 4080, 0, 4088, 4088, 0, 0, 4079, 0, 6, 4076, 403, 0, 0, 0, 1580, 4094, 55, 3, 4077, 0, 0, 6, 0, 4, 3113, 3867, 16, 3, 0, 3111, 1, 0, 4096, 0, 0, 0, 341, 0, 3510, 1, 0, 0, 0, 4095, 1446, 0, 2728, 4090, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 25, 0, 0, 3975, 0, 0, 98, 0, 97, 0, 2, 0, 4096, 0, 55, 4017, 4095, 76, 85, 963, 0, 640, 0, 0, 16, 3999, 52, 2, 3213, 3038, 5, 4096, 8, 3180, 48, 503, 1165, 2288, 0, 4084, 0, 1, 0, 3945, 2, 1, 4046, 3960, 73, 4058, 235, 52, 4073, 2100, 0, 0, 0, 0, 18, 0, 1891, 720, 1588, 229, 4096, 0, 0, 0, 0, 0, 180, 4096, 4088, 0, 1, 0, 4001, 4080, 0, 0, 0, 0, 0, 15, 0, 0, 3884, 1455, 4096, 0, 0, 68, 5, 337, 4096, 0, 3, 0, 10, 138, 0, 5, 322, 377, 0, 4096, 0, 7, 0, 3773, 0, 20, 32, 14, 8, 4092, 0, 0, 4075, 0, 12, 4056, 0, 570, 0, 0, 4051] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4070, 4096, 1, 0, 0, 0, 0, 137, 0, 0, 4093, 0, 0, 4096, 1, 2412, 0, 0, 0, 0, 2, 105, 0, 4096, 23, 0, 2, 1, 0, 26, 0, 3606, 0, 0, 0, 4091, 2, 4096, 1, 4096, 3622, 2808, 0, 4095, 3995, 678, 0, 1, 0, 0, 0, 0, 0, 0, 175, 623, 9, 15, 580, 3946, 4092, 2, 45, 0, 1257, 1, 4093, 401, 309, 167, 0, 0, 0, 0, 2, 2319, 4096, 3939, 4096, 0, 292, 3540, 529, 0, 69, 0, 0, 0, 0, 3, 3764, 0, 4096, 4085, 0, 22, 2, 4082, 4, 0, 0, 0, 0, 3805, 0, 371, 57, 3928, 0, 0, 0, 0, 1554, 4071, 3, 7, 26, 0, 0, 63, 0, 4094, 0, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 358, 0, 0, 4096, 3891, 0, 2, 3770, 0, 0, 0, 0, 3822, 0, 0, 3385, 4096, 4096, 10, 0, 0, 262, 0, 40, 0, 24, 2, 0, 1, 4091, 0, 4095, 0, 2, 0, 0, 1, 3929, 3, 4096, 3694, 9, 0, 4090, 0, 381, 252, 4, 0, 111, 0, 163, 3873, 0, 3, 0, 171, 0, 0, 2287, 282, 0, 1732, 16, 1104, 1, 0, 1, 4047, 59, 0, 3744, 53, 0, 0, 0, 3328, 209, 7, 662, 4096, 0, 1693, 3050, 0, 0, 0, 4090, 0, 359, 0, 4096, 2123, 0, 4096, 3986, 0, 1602, 0, 5, 4, 7, 0, 0, 856, 4002, 0, 0, 2558, 4096, 0, 111, 0, 0, 0, 4050, 0, 3916, 4001, 0, 0, 3462, 0, 0, 20, 0, 438] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[49, 7, 140, 1, 0, 0, 0, 0, 1832, 1, 81, 2731, 4024, 1682, 2919, 43, 1336, 3745, 3981, 269, 1086, 0, 1, 571, 2057, 0, 0, 82, 0, 3664, 3936, 0, 1, 0, 0, 2, 0, 0, 10, 0, 46, 4071, 0, 850, 2502, 0, 4090, 0, 110, 4082, 722, 376, 3354, 0, 270, 4085, 0, 0, 0, 0, 0, 2987, 1095, 0, 4, 0, 0, 0, 0, 10, 1301, 13, 0, 0, 0, 2, 3999, 2972, 3992, 96, 0, 4014, 3678, 0, 2709, 0, 0, 17, 2907, 24, 0, 3951, 4029, 526, 1747, 3837, 0, 0, 1881, 0, 4019, 0, 0, 3, 406, 672, 3235, 0, 2017, 3293, 4091, 0, 4, 6, 0, 171, 146, 3819, 93, 5, 0, 191, 4084, 35, 0, 134, 0, 48] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 2195, 0, 0, 92, 1, 0, 0, 790, 0, 0, 0, 0, 4094, 0, 8, 4060, 4096, 518, 239, 0, 1389, 8, 0, 0, 0, 180, 71, 0, 3552, 3703, 0, 4085, 0, 135, 0, 0, 2296, 3490, 29, 4092, 4096, 4, 0, 228, 0, 8, 3829, 43, 0, 2, 0, 1208, 3319, 0, 265, 0, 143, 0, 22, 8, 4016, 0, 4091, 3333, 254, 2081, 0, 7, 19, 650, 3815, 4076, 19, 0, 0, 0, 60, 29, 477, 2193, 4096, 0, 0, 2149, 0, 0, 577, 169, 0, 3986, 0, 835, 837, 0, 4096, 3905, 0, 987, 0, 354, 1195, 0, 0, 0, 4080, 3888, 0, 0, 1607, 3, 0, 0, 0, 1317, 0, 4055, 0, 4096, 3951, 0, 0, 4070, 0, 0, 0, 0, 3401] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 2637, 0, 0, 74, 2905, 40, 3, 3706, 0, 0, 0, 20, 3208, 0, 0, 4096, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 3838, 481, 0, 4072, 265, 0, 0, 0, 0, 0, 617, 8, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 22, 171, 133, 6, 771, 4, 0, 0, 3707, 0, 0, 5, 4096, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3027, 0, 50, 223, 18, 4096, 0, 0, 65, 0, 0, 3847, 0, 0, 0, 1122, 7, 851, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 622, 3817, 0, 4094, 0, 9, 1161, 4043, 0, 0, 11, 4096, 1589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3596, 362, 77, 136, 3791, 375, 1, 0, 0, 4, 0, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 10, 4081, 1, 2834, 0, 0, 4072, 0, 0, 1649, 3980, 0, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3886, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 850, 0, 0, 0, 4066, 2205, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17616, 18734, 13297, 13695, 23526, 9311, 17829, 17064] avg=16384.0000 max=23526 max_rank=4 min=9311 min_rank=5 imbalance_ratio=1.435913 max_load_reduction=107546 max_load_reduction_pct=0.820511 moved_to_echo=67730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 3686, 1995, 0, 0, 0, 980, 0, 0, 0, 0, 0, 0, 3432, 0, 0, 3946, 908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1267, 4090, 542, 36, 1352, 0, 0, 0, 142, 0, 0, 1, 0, 4087, 1178, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4051, 141, 59, 3, 1935, 101, 0, 0, 3000, 0, 2, 1632, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 556, 8, 0, 1850, 0, 100, 1, 0, 1693, 0, 3867, 0, 27, 4036, 2313, 839, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 3599, 1, 4096, 0, 62, 0, 3, 2061, 2976, 2116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2736, 0, 3, 0, 0, 1, 60, 3986, 0, 0, 0, 3812, 4096, 1496, 945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 6, 0, 10, 0, 0, 8, 152, 499, 2194, 4095, 0, 4093, 1358, 0, 1460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3729, 0, 0, 0, 0, 1911, 0, 373, 54, 4080, 0, 0, 0, 155, 2160, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19043, 12725, 14977, 19903, 18953, 17135, 13940, 14396] avg=16384.0000 max=19903 max_rank=3 min=12725 min_rank=1 imbalance_ratio=1.214783 max_load_reduction=111169 max_load_reduction_pct=0.848152 moved_to_echo=64424 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4070, 4096, 1, 0, 0, 0, 0, 137, 0, 0, 4093, 0, 0, 4096, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 1, 2412, 0, 0, 0, 0, 2, 105, 0, 4096, 23, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 3606, 0, 0, 0, 4091, 2, 0, 1, 4096, 3622, 1162, 0, 4095, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 678, 0, 1, 0, 0, 0, 0, 0, 0, 175, 623, 9, 15, 580, 3946, 4092, 1646, 851, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 45, 0, 1257, 1, 4093, 401, 309, 167, 0, 0, 0, 0, 2, 1199, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3939, 4096, 0, 292, 3540, 529, 0, 69, 0, 0, 0, 0, 3, 3764, 0, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 22, 2, 4082, 4, 0, 0, 0, 0, 3805, 0, 371, 57, 3928, 0, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1554, 4071, 3, 7, 26, 0, 0, 63, 0, 4094, 0, 0, 4090, 2205, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21516, 7926, 24696, 12909, 11572, 17196, 17183, 18074] avg=16384.0000 max=24696 max_rank=2 min=7926 min_rank=1 imbalance_ratio=1.507324 max_load_reduction=106376 max_load_reduction_pct=0.811584 moved_to_echo=64025 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 25, 0, 0, 3975, 0, 0, 98, 0, 97, 0, 2, 0, 4096, 0, 55, 4084, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1259, 4095, 76, 85, 963, 0, 640, 0, 0, 16, 3999, 52, 2, 3213, 3038, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 3180, 48, 503, 1165, 2288, 0, 0, 0, 1, 0, 1632, 2, 1, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 73, 4058, 235, 52, 4073, 2100, 0, 0, 0, 0, 18, 0, 1891, 720, 1588, 2313, 835, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 4096, 0, 0, 0, 0, 0, 180, 4096, 4088, 0, 1, 0, 4001, 2109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 3884, 1455, 4096, 0, 0, 68, 5, 337, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 10, 138, 0, 5, 322, 377, 0, 4096, 0, 7, 0, 3773, 0, 1455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 32, 14, 8, 4092, 0, 0, 4075, 0, 12, 4056, 0, 570, 0, 0, 4051, 2205, 1923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13359, 17443, 16970, 22432, 18800, 10824, 10186, 21058] avg=16384.0000 max=22432 max_rank=3 min=10186 min_rank=6 imbalance_ratio=1.369141 max_load_reduction=108640 max_load_reduction_pct=0.828857 moved_to_echo=60868 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 968, 0, 81, 0, 2687, 0, 0, 0, 1, 3185, 0, 0, 4096, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 4089, 4096, 0, 4, 9, 0, 4039, 19, 0, 3929, 0, 45, 4096, 4087, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 26, 4095, 0, 23, 4065, 843, 0, 0, 7, 0, 0, 1692, 146, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 2424, 2, 0, 62, 0, 0, 16, 0, 0, 4, 2533, 13, 0, 4080, 2399, 851, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4088, 0, 0, 4079, 0, 6, 4076, 403, 0, 0, 0, 1580, 2116, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4077, 0, 0, 6, 0, 4, 3113, 3867, 16, 3, 0, 3111, 1, 0, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 0, 3510, 1, 0, 0, 0, 4095, 1446, 0, 2728, 4090, 0, 1460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2205, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11950, 25729, 18952, 16942, 20491, 15165, 17671, 4172] avg=16384.0000 max=25729 max_rank=1 min=4172 min_rank=7 imbalance_ratio=1.570374 max_load_reduction=105343 max_load_reduction_pct=0.803703 moved_to_echo=57499 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 358, 0, 0, 4096, 3891, 0, 2, 3770, 0, 0, 0, 0, 3822, 0, 0, 4096, 930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 4096, 4096, 10, 0, 0, 262, 0, 40, 0, 24, 2, 0, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 1, 3929, 3, 0, 3694, 9, 0, 1688, 0, 381, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 111, 0, 163, 3873, 0, 3, 0, 171, 0, 0, 2287, 282, 0, 1732, 2402, 707, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1104, 1, 0, 1, 4047, 59, 0, 3744, 53, 0, 0, 0, 3328, 103, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 4096, 0, 1693, 3050, 0, 0, 0, 4090, 0, 359, 0, 4096, 2123, 0, 960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 0, 1602, 0, 5, 4, 7, 0, 0, 856, 4002, 0, 0, 2558, 4096, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 0, 4050, 0, 3916, 4001, 0, 0, 3462, 0, 0, 20, 0, 438, 2206, 1625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20965, 13675, 14054, 11764, 12463, 21129, 17193, 19829] avg=16384.0000 max=21129 max_rank=5 min=11764 min_rank=3 imbalance_ratio=1.289612 max_load_reduction=109943 max_load_reduction_pct=0.838799 moved_to_echo=70614 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[49, 7, 140, 1, 0, 0, 0, 0, 1832, 1, 81, 2731, 4024, 1682, 2919, 43, 46, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 420, 3745, 3981, 269, 1086, 0, 1, 571, 2057, 0, 0, 82, 0, 3664, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 10, 0, 0, 4071, 0, 850, 1035, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 4082, 722, 376, 3354, 0, 270, 4085, 0, 0, 0, 0, 0, 2987, 1095, 0, 1467, 277, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 10, 1301, 13, 0, 0, 0, 2, 3999, 2972, 2063, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 3678, 0, 2709, 0, 0, 17, 2907, 24, 0, 3951, 4029, 526, 1747, 904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1881, 0, 4019, 0, 0, 3, 406, 672, 3235, 0, 2017, 3293, 4091, 0, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 0, 171, 146, 3819, 93, 5, 0, 191, 4084, 35, 0, 134, 0, 48, 2065, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14424, 19812, 10059, 19330, 10460, 24506, 21041, 11440] avg=16384.0000 max=24506 max_rank=5 min=10059 min_rank=2 imbalance_ratio=1.495728 max_load_reduction=106566 max_load_reduction_pct=0.813034 moved_to_echo=67447 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 2195, 0, 0, 92, 1, 0, 0, 790, 0, 0, 0, 0, 4094, 0, 8, 4092, 927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1272, 4096, 518, 239, 0, 1389, 8, 0, 0, 0, 180, 71, 0, 3552, 3703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 135, 0, 0, 2296, 3490, 29, 0, 4096, 4, 0, 95, 0, 8, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 2, 0, 1208, 3319, 0, 265, 0, 143, 0, 22, 8, 4016, 0, 4091, 133, 844, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3333, 254, 2081, 0, 7, 19, 650, 3815, 4076, 19, 0, 0, 0, 60, 16, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2193, 4096, 0, 0, 2149, 0, 0, 577, 169, 0, 3986, 0, 835, 837, 0, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 0, 987, 0, 354, 1195, 0, 0, 0, 4080, 3888, 0, 0, 1607, 3, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 0, 4055, 0, 4096, 3951, 0, 0, 4070, 0, 0, 0, 0, 3401, 2205, 1944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12199, 15028, 18067, 14097, 14807, 15806, 16029, 25039] avg=16384.0000 max=25039 max_rank=7 min=12199 min_rank=0 imbalance_ratio=1.528259 max_load_reduction=106033 max_load_reduction_pct=0.808968 moved_to_echo=71681 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 3968, 5120, 1792, 20608, 3840, 8192, 14080, 0, 640, 15872, 4992, 4736, 256, 0, 16256, 17536, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9216, 28160, 16256, 640, 7552, 1664, 1024, 4736, 2432, 128, 12288, 896, 128, 22784, 24192, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[20480, 4224, 11392, 256, 640, 10240, 14848, 128, 0, 18688, 4224, 4480, 8960, 8320, 12928, 12160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[4224, 13056, 2176, 4096, 13312, 6912, 1152, 128, 12800, 128, 256, 2816, 4096, 27648, 2944, 128, 28672, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[6912, 23168, 3712, 2048, 15104, 896, 128, 7936, 20352, 4608, 4352, 3968, 16000, 11392, 3584, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12032, 128, 4608, 512, 8704, 8832, 128, 3200, 1024, 6144, 29440, 5632, 2432, 16000, 25344, 0, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[16000, 4224, 7936, 896, 4864, 17280, 2432, 4480, 128, 512, 2432, 3968, 8832, 9856, 5888, 20864, 12800, 6144, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[3584, 17792, 6912, 5120, 128, 19968, 2432, 8448, 17408, 8704, 0, 128, 6144, 19072, 11392, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[38, 18, 0, 364, 0, 2681, 0, 0, 955, 955, 3783, 4, 3914, 736, 3390, 0, 0, 810, 0, 241, 0, 0, 4083, 668, 0, 2589, 26, 0, 4096, 3948, 0, 3386, 0, 4096, 386, 4096, 67, 100, 0, 0, 0, 484, 4080, 1683, 0, 3917, 0, 0, 1, 2816, 0, 4075, 0, 60, 372, 0, 2113, 33, 0, 0, 0, 8, 2709, 4096, 0, 2499, 0, 4085, 0, 43, 4091, 0, 4095, 4096, 1, 0, 0, 0, 120, 0, 0, 0, 0, 0, 13, 0, 2096, 0, 0, 0, 2, 4043, 0, 542, 2218, 0, 5, 866, 0, 3, 0, 58, 3256, 0, 0, 0, 4095, 0, 2915, 0, 158, 4074, 0, 0, 5, 3711, 4096, 0, 0, 3774, 0, 0, 0, 142, 0, 3, 4090, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[2, 0, 0, 3356, 4093, 35, 130, 758, 4076, 0, 0, 4, 11, 306, 384, 0, 4073, 20, 0, 0, 0, 0, 3841, 0, 31, 380, 152, 3606, 33, 4081, 0, 4095, 0, 4096, 0, 9, 0, 4, 2, 0, 3823, 1, 3905, 4063, 0, 4090, 4087, 2, 6, 31, 65, 4077, 0, 0, 0, 0, 0, 0, 0, 319, 3946, 5, 3990, 2237, 0, 125, 0, 4091, 0, 4095, 53, 0, 4096, 3845, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 2, 4086, 0, 0, 3820, 0, 38, 682, 0, 1470, 0, 7, 13, 0, 0, 0, 4093, 0, 4096, 0, 934, 4093, 0, 3909, 5, 4096, 3971, 1, 0, 0, 0, 0, 0, 4, 3123, 8, 4081, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[325, 0, 0, 0, 0, 140, 4, 2, 3298, 3994, 12, 2, 1332, 2455, 4096, 0, 3928, 398, 0, 0, 0, 515, 0, 316, 0, 0, 0, 3517, 4096, 542, 9, 281, 17, 4036, 12, 4029, 0, 0, 196, 0, 0, 5, 2607, 3214, 0, 4059, 4, 0, 3940, 1362, 0, 4096, 2696, 70, 4021, 0, 0, 1, 0, 3018, 0, 12, 3238, 4080, 0, 3870, 0, 4071, 0, 0, 6, 0, 919, 4096, 70, 0, 18, 0, 6, 218, 9, 0, 99, 0, 782, 598, 177, 0, 0, 25, 609, 3880, 0, 6, 0, 0, 0, 3466, 22, 4074, 0, 9, 3598, 1, 0, 1, 1770, 8, 4078, 0, 0, 2, 10, 4060, 841, 3679, 4093, 3170, 0, 450, 15, 3, 0, 0, 0, 583, 3705, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 233, 362, 4018, 0, 17, 4006, 3313, 0, 35, 4087, 0, 0, 0, 0, 2614, 0, 0, 0, 14, 1345, 2, 0, 1, 1587, 0, 3995, 4090, 65, 268, 3271, 187, 2703, 0, 4010, 0, 0, 0, 0, 4095, 4061, 177, 235, 0, 3248, 19, 0, 0, 4096, 0, 1626, 0, 0, 40, 0, 1471, 0, 0, 15, 4096, 60, 1821, 1942, 17, 3955, 21, 4081, 3712, 3, 1921, 49, 0, 0, 4, 0, 0, 0, 0, 210, 0, 19, 0, 0, 0, 1, 2444, 70, 87, 0, 0, 4095, 3, 0, 1325, 78, 0, 4084, 1139, 147, 0, 1, 0, 0, 0, 2, 4093, 6, 0, 2, 4096, 4096, 0, 0, 3947, 4095, 0, 2920, 1291, 0, 0, 50, 0, 2170, 1610, 0, 4096, 3906] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 1726, 0, 0, 1477, 1169, 0, 3522, 7, 3, 0, 3331, 0, 4, 0, 47, 2691, 0, 0, 0, 753, 0, 0, 2371, 4079, 27, 4096, 459, 18, 3991, 17, 32, 0, 71, 0, 0, 0, 48, 0, 0, 4023, 127, 0, 0, 169, 3852, 0, 398, 347, 0, 4022, 0, 2061, 634, 0, 0, 6, 3197, 578, 4, 0, 0, 0, 4096, 4083, 1830, 0, 4070, 0, 29, 0, 4096, 275, 0, 15, 0, 0, 3860, 3971, 0, 839, 18, 90, 0, 43, 38, 0, 2795, 1680, 3900, 3213, 1, 0, 83, 2703, 0, 3678, 3048, 4089, 1412, 20, 4093, 962, 1, 0, 0, 0, 3991, 0, 590, 4096, 187, 0, 3916, 4096, 3949, 162, 0, 0, 38, 0, 0, 5, 91, 85, 1478, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 2, 6, 8, 807, 636, 0, 1074, 2089, 0, 50, 0, 0, 68, 3952, 65, 0, 27, 0, 0, 0, 2379, 1891, 0, 0, 2634, 0, 812, 3938, 0, 0, 0, 0, 3043, 101, 1180, 0, 18, 0, 0, 4093, 4089, 766, 3490, 0, 4096, 0, 0, 4096, 150, 9, 0, 0, 0, 0, 0, 109, 24, 41, 645, 3679, 22, 0, 4091, 3, 2027, 3354, 4096, 0, 4069, 3581, 8, 4045, 4035, 450, 0, 297, 156, 0, 0, 5, 0, 24, 0, 0, 0, 0, 0, 3838, 41, 9, 4088, 0, 0, 0, 0, 0, 3812, 0, 0, 5, 727, 59, 3, 379, 53, 0, 0, 4036, 38, 4012, 4069, 0, 1883, 4096, 3825, 4094, 0, 0, 0, 0, 0, 0, 80, 3952, 2, 4033, 3608] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 191, 2698, 4063, 68, 2381, 0, 2034, 3973, 0, 0, 0, 0, 0, 0, 3790, 2417, 0, 0, 382, 0, 217, 0, 1, 0, 12, 4093, 0, 2064, 63, 0, 1551, 0, 3525, 1065, 0, 0, 0, 0, 1, 4073, 3455, 2732, 0, 3748, 254, 8, 3722, 3783, 1236, 0, 0, 1, 0, 0, 0, 0, 3673, 3, 35, 4031, 0, 0, 0, 205, 4096, 4083, 781, 2094, 3436, 0, 0, 0, 15, 0, 0, 0, 37, 15, 0, 928, 0, 35, 0, 1, 0, 0, 272, 0, 0, 4084, 0, 0, 3881, 16, 800, 413, 4079, 759, 0, 337, 1, 0, 903, 1, 4078, 1, 4095, 0, 4096, 4096, 0, 0, 384, 4096, 4092, 2165, 21, 0, 0, 1878, 31, 96, 3, 0, 4096, 1228] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1, 0, 1, 0, 4060, 0, 4051, 872, 3329, 0, 0, 7, 0, 0, 0, 0, 3798, 0, 0, 0, 242, 4080, 56, 0, 3923, 0, 26, 4001, 0, 5, 4096, 2543, 0, 4095, 0, 1, 0, 0, 0, 0, 4093, 62, 0, 4073, 0, 4096, 2257, 11, 0, 20, 24, 0, 3, 0, 0, 0, 0, 0, 0, 2, 4096, 1138, 0, 2972, 0, 3839, 433, 785, 3728, 1479, 0, 0, 182, 0, 2, 0, 0, 0, 0, 2, 0, 1246, 0, 0, 0, 1, 0, 2669, 4096, 0, 0, 4095, 0, 1, 1164, 15, 4020, 0, 4078, 0, 0, 1, 3614, 0, 0, 1, 0, 66, 1616, 0, 3741, 4084, 0, 0, 4081, 4096, 0, 4061, 33, 0, 0, 1038, 218, 1974, 4096, 0, 4096, 387] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[38, 18, 0, 364, 0, 2681, 0, 0, 955, 955, 3783, 4, 3914, 736, 3390, 0, 2288, 903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 810, 0, 241, 0, 0, 4083, 668, 0, 2589, 26, 0, 4096, 3948, 0, 3386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 386, 4096, 67, 100, 0, 0, 0, 484, 4080, 1683, 0, 1472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2816, 0, 4075, 0, 60, 372, 0, 2113, 33, 0, 0, 0, 8, 2709, 4096, 0, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2499, 0, 2604, 0, 43, 4091, 0, 4095, 4096, 1, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 2096, 0, 0, 0, 2, 4043, 0, 542, 2218, 0, 3558, 3208, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 866, 0, 3, 0, 58, 3256, 0, 0, 0, 4095, 0, 2915, 0, 158, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 153, 4096, 0, 0, 3774, 0, 0, 0, 142, 0, 3, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20029, 19847, 16464, 16861, 17549, 15837, 12222, 12263] avg=16384.0000 max=20029 max_rank=0 min=12222 min_rank=6 imbalance_ratio=1.222473 max_load_reduction=111043 max_load_reduction_pct=0.847191 moved_to_echo=57871 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[325, 0, 0, 0, 0, 140, 4, 2, 3298, 3994, 12, 2, 1332, 2455, 4096, 0, 2371, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3928, 398, 0, 0, 0, 515, 0, 316, 0, 0, 0, 2646, 4096, 542, 9, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4036, 12, 4029, 0, 0, 196, 0, 0, 5, 2607, 3214, 0, 1525, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 1362, 0, 4096, 2696, 70, 4021, 0, 0, 1, 0, 3018, 0, 12, 3238, 4080, 871, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3870, 0, 2595, 0, 0, 6, 0, 919, 4096, 70, 0, 18, 0, 6, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 99, 0, 782, 598, 177, 0, 0, 25, 609, 3880, 0, 6, 0, 0, 3527, 1, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3466, 22, 4074, 0, 9, 3598, 1, 0, 1, 1770, 8, 4078, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4060, 841, 152, 4093, 3170, 0, 450, 15, 3, 0, 0, 0, 583, 3705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18931, 12731, 15645, 27981, 11798, 9876, 17028, 17082] avg=16384.0000 max=27981 max_rank=3 min=9876 min_rank=5 imbalance_ratio=1.707825 max_load_reduction=103091 max_load_reduction_pct=0.786522 moved_to_echo=55784 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 233, 362, 4018, 0, 17, 4006, 3313, 0, 35, 4087, 0, 0, 0, 0, 1898, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2614, 0, 0, 0, 14, 1345, 2, 0, 1, 1587, 0, 3005, 4090, 65, 268, 3271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 2703, 0, 4010, 0, 0, 0, 0, 4095, 4061, 177, 235, 0, 1220, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1626, 0, 0, 40, 0, 1471, 0, 0, 15, 4096, 60, 1821, 1942, 990, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3955, 21, 2602, 3712, 3, 1921, 49, 0, 0, 4, 0, 0, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 1, 2444, 70, 87, 0, 0, 4095, 3, 0, 1325, 78, 3926, 3225, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1139, 147, 0, 1, 0, 0, 0, 2, 4093, 6, 0, 2, 4096, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 169, 0, 2920, 1291, 0, 0, 50, 0, 2170, 1610, 0, 4096, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18872, 16262, 16707, 16734, 12494, 15403, 14441, 20159] avg=16384.0000 max=20159 max_rank=7 min=12494 min_rank=4 imbalance_ratio=1.230408 max_load_reduction=110913 max_load_reduction_pct=0.846199 moved_to_echo=62497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 3356, 4093, 35, 130, 758, 4076, 0, 0, 4, 11, 306, 384, 0, 2390, 905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 20, 0, 0, 0, 0, 3841, 0, 31, 380, 152, 2712, 33, 4081, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 4, 2, 0, 3823, 1, 3905, 4063, 0, 1536, 4087, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 31, 65, 4077, 0, 0, 0, 0, 0, 0, 0, 319, 3946, 5, 3990, 2237, 894, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 2607, 0, 4095, 53, 0, 4096, 3845, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 2, 4086, 0, 0, 3820, 0, 3927, 3223, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 682, 0, 1470, 0, 7, 13, 0, 0, 0, 4093, 0, 4096, 0, 934, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 5, 169, 3971, 1, 0, 0, 0, 0, 0, 4, 3123, 8, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16450, 19418, 21528, 16149, 14829, 15224, 12203, 15271] avg=16384.0000 max=21528 max_rank=2 min=12203 min_rank=6 imbalance_ratio=1.313965 max_load_reduction=109544 max_load_reduction_pct=0.835754 moved_to_echo=57527 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 6, 8, 807, 636, 0, 1074, 2089, 0, 50, 0, 0, 68, 3952, 65, 2393, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 2379, 1891, 0, 0, 2634, 0, 611, 3938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3043, 101, 1180, 0, 18, 0, 0, 4093, 4089, 766, 3490, 0, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 150, 9, 0, 0, 0, 0, 0, 109, 24, 41, 645, 3679, 22, 0, 4091, 201, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2027, 3354, 2611, 0, 4069, 3581, 8, 4045, 4035, 450, 0, 297, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 24, 0, 0, 0, 0, 0, 3838, 41, 9, 4088, 0, 0, 0, 0, 3667, 3204, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3812, 0, 0, 5, 727, 59, 3, 379, 53, 0, 0, 4036, 38, 4012, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1883, 4096, 158, 4094, 0, 0, 0, 0, 0, 0, 80, 3952, 2, 4033, 3608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12056, 11480, 18318, 13646, 24636, 15041, 13989, 21906] avg=16384.0000 max=24636 max_rank=4 min=11480 min_rank=1 imbalance_ratio=1.503662 max_load_reduction=106436 max_load_reduction_pct=0.812042 moved_to_echo=75572 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 0, 4060, 0, 4051, 872, 3329, 0, 0, 7, 0, 0, 0, 0, 2397, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 0, 0, 242, 4080, 56, 0, 3923, 0, 26, 3006, 0, 5, 4096, 2543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 4093, 62, 0, 4073, 0, 1529, 2257, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 24, 0, 3, 0, 0, 0, 0, 0, 0, 2, 4096, 1138, 0, 2972, 995, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3839, 433, 494, 3728, 1479, 0, 0, 182, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1246, 0, 0, 0, 1, 0, 2669, 4096, 0, 0, 4095, 0, 1, 1164, 15, 3932, 3221, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 4078, 0, 0, 1, 3614, 0, 0, 1, 0, 66, 1616, 0, 3741, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 164, 0, 4061, 33, 0, 0, 1038, 218, 1974, 4096, 0, 4096, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14895, 21775, 16121, 9364, 10159, 20610, 18000, 20148] avg=16384.0000 max=21775 max_rank=1 min=9364 min_rank=3 imbalance_ratio=1.329041 max_load_reduction=109297 max_load_reduction_pct=0.833870 moved_to_echo=68917 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1726, 0, 0, 1477, 1169, 0, 3522, 7, 3, 0, 3331, 0, 4, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 2691, 0, 0, 0, 753, 0, 0, 2371, 4079, 27, 3081, 459, 18, 3991, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 71, 0, 0, 0, 48, 0, 0, 4023, 127, 0, 0, 65, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 347, 0, 4022, 0, 2061, 634, 0, 0, 6, 3197, 578, 4, 0, 0, 0, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 1830, 0, 4070, 0, 29, 0, 4096, 275, 0, 15, 0, 0, 3860, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 18, 90, 0, 43, 38, 0, 2795, 1680, 3900, 3213, 1, 0, 83, 2703, 3927, 3225, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3678, 3048, 4089, 1412, 20, 4093, 962, 1, 0, 0, 0, 3991, 0, 590, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 3916, 169, 3949, 162, 0, 0, 38, 0, 0, 5, 91, 85, 1478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11337, 17534, 8218, 12262, 26325, 22561, 22755, 10080] avg=16384.0000 max=26325 max_rank=4 min=8218 min_rank=2 imbalance_ratio=1.606750 max_load_reduction=104747 max_load_reduction_pct=0.799156 moved_to_echo=81721 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 191, 2698, 4063, 68, 2381, 0, 2034, 3973, 0, 0, 0, 0, 0, 0, 2190, 903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3790, 2417, 0, 0, 382, 0, 217, 0, 1, 0, 12, 3079, 0, 2064, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1551, 0, 3525, 1065, 0, 0, 0, 0, 1, 4073, 3455, 2732, 0, 1407, 254, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3722, 3783, 1236, 0, 0, 1, 0, 0, 0, 0, 3673, 3, 35, 4031, 0, 0, 1014, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 4096, 2603, 781, 2094, 3436, 0, 0, 0, 15, 0, 0, 0, 37, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 928, 0, 35, 0, 1, 0, 0, 272, 0, 0, 4084, 0, 0, 3881, 16, 3927, 3225, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 800, 413, 4079, 759, 0, 337, 1, 0, 903, 1, 4078, 1, 4095, 0, 4096, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 169, 4092, 2165, 21, 0, 0, 1878, 31, 96, 3, 0, 4096, 1228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18502, 12025, 18071, 18075, 13282, 16520, 20434, 14163] avg=16384.0000 max=20434 max_rank=6 min=12025 min_rank=1 imbalance_ratio=1.247192 max_load_reduction=110638 max_load_reduction_pct=0.844101 moved_to_echo=64399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[4224, 20608, 9856, 16128, 12416, 11904, 13184, 128, 17536, 16384, 640, 128, 384, 256, 4096, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[12288, 12672, 1408, 17920, 2816, 2304, 5120, 0, 3712, 128, 6912, 4608, 15872, 5376, 11776, 19456, 6016, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[18304, 6400, 0, 256, 640, 9088, 10112, 1024, 6400, 11392, 256, 18176, 16768, 10752, 8448, 13696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[384, 128, 2176, 6912, 17152, 5120, 7808, 6784, 22656, 8960, 3968, 4224, 8704, 3584, 11904, 128, 16128, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[256, 9856, 17280, 1408, 24320, 12544, 1408, 4224, 128, 3072, 256, 4480, 12928, 768, 29696, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[4864, 17024, 12416, 10624, 1536, 1280, 14720, 1024, 1408, 128, 18176, 128, 24832, 128, 17664, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[1792, 22144, 4096, 14464, 128, 128, 256, 0, 16128, 16896, 15232, 19584, 0, 10368, 10496, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 3072, 256, 128, 896, 768, 4864, 2816, 11136, 1792, 4608, 31616, 128, 640, 12544, 2816, 30464, 22656, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[3587, 0, 0, 0, 0, 4, 3892, 0, 25, 0, 3735, 0, 4084, 4096, 0, 2, 0, 0, 187, 0, 13, 20, 2907, 0, 0, 0, 3626, 0, 4093, 0, 1, 2630, 4059, 4095, 3394, 0, 0, 0, 0, 2, 4038, 3201, 0, 0, 0, 0, 0, 0, 3691, 1112, 0, 12, 861, 4096, 4096, 0, 0, 0, 4036, 108, 0, 159, 388, 2114, 0, 3421, 0, 0, 0, 28, 4043, 172, 1410, 0, 4090, 0, 1609, 0, 0, 1546, 0, 4096, 0, 149, 7, 0, 0, 1453, 3136, 0, 15, 0, 3918, 0, 0, 4074, 644, 0, 4095, 259, 2638, 678, 7, 0, 22, 29, 0, 185, 73, 2, 0, 0, 0, 6, 4064, 4043, 4086, 19, 4011, 0, 0, 423, 0, 0, 146, 0, 106, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[333, 0, 0, 207, 1077, 0, 2977, 404, 53, 0, 1, 3527, 3781, 3375, 0, 0, 0, 0, 37, 4093, 4095, 103, 1, 0, 15, 3690, 4090, 2, 4015, 0, 0, 2759, 2965, 341, 47, 1, 0, 11, 0, 0, 0, 3594, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 29, 0, 835, 3, 128, 0, 2, 3983, 3081, 0, 0, 0, 49, 3902, 3670, 0, 13, 0, 15, 52, 3, 0, 0, 1358, 11, 2688, 3846, 3132, 0, 0, 115, 0, 1457, 2492, 0, 0, 0, 1074, 0, 0, 4096, 4095, 570, 0, 4096, 255, 4096, 2908, 4021, 4071, 3728, 4096, 0, 1, 1221, 0, 0, 0, 0, 3, 801, 739, 1738, 4057, 770, 74, 1, 4002, 0, 0, 0, 0, 1, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[1788, 0, 0, 0, 20, 0, 3850, 3, 0, 0, 3233, 0, 4096, 4083, 19, 0, 0, 0, 5, 12, 2805, 117, 4053, 0, 0, 991, 3997, 0, 4077, 0, 4, 3966, 7, 3953, 4089, 0, 0, 0, 0, 0, 485, 4094, 312, 0, 2310, 2, 15, 0, 4089, 3934, 0, 0, 28, 5, 1, 0, 7, 1418, 8, 4, 0, 18, 0, 3694, 3, 1273, 0, 0, 392, 0, 1090, 0, 22, 0, 3807, 968, 0, 0, 0, 0, 73, 4089, 646, 82, 2968, 1383, 0, 0, 1643, 1, 0, 4096, 4063, 0, 0, 4095, 0, 4069, 4095, 3887, 0, 2720, 4, 2, 1, 0, 0, 0, 0, 0, 0, 2, 4030, 0, 258, 0, 3466, 1, 4055, 0, 0, 0, 0, 4094, 0, 11, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 3327, 534, 314, 0, 26, 0, 1, 3821, 4074, 1, 9, 0, 0, 0, 0, 3756, 1012, 0, 149, 528, 0, 4096, 0, 3187, 0, 3, 4096, 0, 3283, 533, 107, 1, 0, 0, 7, 4084, 3682, 0, 19, 0, 43, 0, 0, 0, 4096, 0, 232, 0, 4092, 358, 6, 3800, 0, 4007, 0, 47, 20, 0, 0, 766, 4092, 0, 750, 0, 0, 3928, 10, 3940, 0, 0, 3342, 202, 0, 0, 0, 16, 4045, 2587, 219, 4077, 1, 3459, 0, 4043, 0, 15, 4067, 262, 0, 0, 2975, 0, 0, 0, 4096, 0, 0, 4090, 51, 3, 0, 526, 118, 0, 0, 1, 3, 673, 138, 0, 4096, 0, 1, 4096, 711, 0, 0, 0, 497, 3657, 13, 1, 4058] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[3926, 0, 3989, 0, 281, 9, 0, 0, 0, 0, 590, 51, 3605, 3944, 712, 0, 0, 0, 78, 352, 9, 0, 0, 0, 0, 0, 1325, 0, 2465, 0, 4, 0, 3660, 2784, 4052, 0, 0, 0, 0, 0, 479, 4092, 76, 0, 2891, 79, 3919, 4, 825, 3798, 230, 0, 4093, 0, 5, 0, 149, 4020, 4096, 0, 0, 150, 1293, 0, 1, 3738, 2107, 0, 570, 0, 4079, 0, 4, 614, 4090, 0, 0, 0, 3900, 0, 10, 4092, 0, 47, 0, 1834, 31, 0, 2, 318, 238, 4085, 1576, 0, 0, 4095, 0, 0, 12, 3898, 0, 28, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 4090, 3982, 4096, 0, 3918, 3306, 3598, 0, 23, 4073, 0, 0, 0, 2579, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1, 1585, 3741, 0, 298, 136, 0, 108, 3903, 0, 2802, 1236, 191, 3585, 0, 0, 0, 245, 49, 0, 3987, 0, 929, 376, 387, 46, 751, 0, 2600, 0, 11, 4079, 4090, 3040, 0, 0, 8, 53, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 2254, 75, 843, 0, 235, 3738, 1, 0, 0, 0, 20, 26, 2, 1182, 3158, 0, 0, 4090, 0, 120, 0, 0, 2839, 4064, 0, 28, 4088, 2282, 24, 0, 10, 0, 34, 4096, 87, 0, 4086, 10, 9, 890, 5, 0, 2656, 1089, 2205, 0, 0, 3957, 3914, 2401, 4070, 4011, 113, 4029, 0, 0, 0, 17, 0, 0, 25, 4096, 0, 4028, 11, 4095, 0, 3865, 0, 5, 3849, 541, 0, 1465, 48, 0, 0, 1, 0, 35] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[3838, 0, 0, 134, 0, 4066, 11, 5, 5, 0, 5, 25, 4088, 41, 0, 0, 0, 0, 434, 1164, 4080, 0, 2, 0, 2, 0, 4077, 1, 21, 44, 44, 2743, 1, 4095, 4096, 0, 0, 0, 0, 0, 484, 4095, 168, 0, 0, 3325, 2940, 0, 27, 2719, 0, 0, 0, 4, 0, 0, 0, 3928, 3999, 0, 1930, 0, 0, 3830, 0, 9, 2610, 199, 0, 1, 565, 0, 4035, 0, 130, 119, 0, 0, 1, 0, 0, 3607, 936, 1, 735, 2006, 3760, 0, 4096, 0, 1301, 69, 4096, 125, 2528, 4096, 0, 121, 4096, 4081, 3426, 3896, 34, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3525, 0, 1, 0, 103, 1849, 4062, 0, 2162, 0, 154, 4059, 0, 0, 148, 3844] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 51, 14, 4068, 0, 0, 4, 28, 3374, 3084, 0, 0, 0, 1772, 0, 4089, 9, 0, 79, 0, 1, 4, 2, 1757, 4096, 0, 2773, 4096, 50, 4039, 4096, 1, 0, 0, 0, 0, 4061, 4095, 57, 0, 3983, 0, 3610, 0, 4096, 3330, 0, 32, 0, 30, 0, 0, 0, 52, 4019, 0, 439, 0, 0, 4096, 946, 3775, 52, 1, 0, 988, 264, 0, 3886, 0, 0, 38, 0, 19, 3, 3903, 2530, 12, 11, 31, 4096, 0, 4015, 0, 4091, 2626, 0, 525, 4, 561, 0, 4096, 1683, 4096, 195, 0, 61, 2, 49, 0, 0, 243, 0, 0, 0, 0, 67, 348, 3351, 0, 107, 0, 4087, 715, 2163, 0, 0, 0, 44, 2, 0, 0, 0, 3903] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3587, 0, 0, 0, 0, 4, 3892, 0, 25, 0, 3735, 0, 4084, 4096, 0, 2, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 13, 20, 2907, 0, 0, 0, 3626, 0, 4093, 0, 1, 2630, 1190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4095, 3394, 0, 0, 0, 0, 2, 4038, 3201, 0, 0, 0, 0, 0, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3691, 1112, 0, 12, 861, 4096, 4096, 0, 0, 0, 4036, 108, 0, 159, 388, 2114, 636, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3421, 0, 0, 0, 28, 4043, 172, 1410, 0, 4090, 0, 1609, 0, 0, 1546, 1419, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2978, 0, 149, 7, 0, 0, 1453, 3136, 0, 15, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 644, 0, 4095, 259, 2638, 678, 7, 0, 22, 29, 0, 185, 73, 2, 0, 0, 2655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4064, 4043, 4086, 19, 2261, 0, 0, 423, 0, 0, 146, 0, 106, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19592, 14667, 19104, 21558, 18049, 11656, 11287, 15159] avg=16384.0000 max=21558 max_rank=3 min=11287 min_rank=6 imbalance_ratio=1.315796 max_load_reduction=109514 max_load_reduction_pct=0.835526 moved_to_echo=56151 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3926, 0, 3989, 0, 281, 9, 0, 0, 0, 0, 590, 51, 3605, 3944, 712, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 352, 9, 0, 0, 0, 0, 0, 1325, 0, 2465, 0, 4, 0, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3660, 2784, 4052, 0, 0, 0, 0, 0, 479, 4092, 76, 0, 2891, 79, 3919, 4, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 825, 3798, 230, 0, 4093, 0, 5, 0, 149, 4020, 4096, 0, 0, 150, 1293, 0, 635, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3738, 2107, 0, 570, 0, 4079, 0, 4, 614, 4090, 0, 0, 0, 3900, 0, 1426, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2976, 0, 47, 0, 1834, 31, 0, 2, 318, 238, 4085, 1576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3898, 0, 28, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 2669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3982, 4096, 0, 3918, 3306, 2028, 0, 23, 4073, 0, 0, 0, 2579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17274, 5301, 22350, 19517, 20808, 11117, 6610, 28095] avg=16384.0000 max=28095 max_rank=7 min=5301 min_rank=1 imbalance_ratio=1.714783 max_load_reduction=102977 max_load_reduction_pct=0.785652 moved_to_echo=66630 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[333, 0, 0, 207, 1077, 0, 2977, 404, 53, 0, 1, 3527, 3781, 3375, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4093, 4095, 103, 1, 0, 15, 3690, 4090, 2, 4015, 0, 0, 2759, 1188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2965, 341, 47, 1, 0, 11, 0, 0, 0, 3594, 0, 0, 0, 0, 3, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 29, 0, 835, 3, 128, 0, 2, 3983, 3081, 0, 0, 0, 49, 17, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3902, 3670, 0, 13, 0, 15, 52, 3, 0, 0, 1358, 11, 2688, 3846, 3132, 0, 1428, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 1457, 2492, 0, 0, 0, 1074, 0, 0, 4096, 4095, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 4096, 2908, 4021, 4071, 3728, 4096, 0, 1, 1221, 0, 0, 0, 0, 3, 801, 2668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 739, 1738, 4057, 770, 74, 1, 2256, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15739, 24088, 6970, 12471, 20428, 13870, 27869, 9637] avg=16384.0000 max=27869 max_rank=6 min=6970 min_rank=2 imbalance_ratio=1.700989 max_load_reduction=103203 max_load_reduction_pct=0.787376 moved_to_echo=71804 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 3327, 534, 314, 0, 26, 0, 1, 3821, 4074, 1, 9, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3756, 1012, 0, 149, 528, 0, 4096, 0, 3187, 0, 3, 4096, 1216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3283, 533, 107, 1, 0, 0, 7, 4084, 3682, 0, 19, 0, 43, 0, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 232, 0, 4092, 358, 6, 3800, 0, 4007, 0, 47, 20, 0, 0, 628, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 766, 4092, 0, 750, 0, 0, 3928, 10, 3940, 0, 0, 3342, 202, 0, 0, 0, 1036, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2941, 2587, 219, 4077, 1, 3459, 0, 4043, 0, 15, 4067, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 51, 3, 0, 526, 118, 0, 0, 1, 3, 1939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 673, 138, 0, 4096, 0, 1, 2309, 711, 0, 0, 0, 497, 3657, 13, 1, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16368, 18043, 12070, 17540, 18383, 21687, 10827, 16154] avg=16384.0000 max=21687 max_rank=5 min=10827 min_rank=6 imbalance_ratio=1.323669 max_load_reduction=109385 max_load_reduction_pct=0.834541 moved_to_echo=67051 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1788, 0, 0, 0, 20, 0, 3850, 3, 0, 0, 3233, 0, 4096, 4083, 19, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 12, 2805, 117, 4053, 0, 0, 991, 3997, 0, 4077, 0, 4, 3966, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3953, 4089, 0, 0, 0, 0, 0, 485, 4094, 312, 0, 2310, 2, 15, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3934, 0, 0, 28, 5, 1, 0, 7, 1418, 8, 4, 0, 18, 0, 3694, 635, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1273, 0, 0, 392, 0, 1090, 0, 22, 0, 3807, 968, 0, 0, 0, 0, 1427, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 2973, 646, 82, 2968, 1383, 0, 0, 1643, 1, 0, 4096, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4095, 3887, 0, 2720, 4, 2, 1, 0, 0, 0, 0, 0, 0, 2, 2668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 258, 0, 3466, 1, 2287, 0, 0, 0, 0, 4094, 0, 11, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17259, 21230, 15581, 14092, 9296, 17928, 17448, 18238] avg=16384.0000 max=21230 max_rank=1 min=9296 min_rank=4 imbalance_ratio=1.295776 max_load_reduction=109842 max_load_reduction_pct=0.838028 moved_to_echo=62910 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 51, 14, 4068, 0, 0, 4, 28, 3374, 3084, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1772, 0, 4089, 9, 0, 79, 0, 1, 4, 2, 1757, 4096, 0, 2773, 4096, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 4039, 4096, 1, 0, 0, 0, 0, 4061, 4095, 57, 0, 3983, 0, 3610, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3330, 0, 32, 0, 30, 0, 0, 0, 52, 4019, 0, 439, 0, 0, 4096, 4, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 946, 3775, 52, 1, 0, 988, 264, 0, 3886, 0, 0, 38, 0, 19, 3, 3903, 1428, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2530, 0, 11, 31, 4096, 0, 4015, 0, 4091, 2626, 0, 525, 4, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1683, 4096, 195, 0, 61, 2, 49, 0, 0, 243, 0, 0, 0, 0, 67, 348, 2668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3351, 0, 107, 0, 4087, 715, 1209, 0, 0, 0, 44, 2, 0, 0, 0, 3903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14722, 19324, 23997, 16235, 15474, 18490, 9412, 13418] avg=16384.0000 max=23997 max_rank=2 min=9412 min_rank=6 imbalance_ratio=1.464661 max_load_reduction=107075 max_load_reduction_pct=0.816917 moved_to_echo=56794 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 1585, 3741, 0, 298, 136, 0, 108, 3903, 0, 2802, 1236, 191, 3585, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 49, 0, 3987, 0, 929, 376, 387, 46, 751, 0, 2600, 0, 11, 4079, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3040, 0, 0, 8, 53, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2254, 75, 843, 0, 235, 3738, 1, 0, 0, 0, 20, 26, 2, 1182, 3158, 0, 637, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 120, 0, 0, 2839, 4064, 0, 28, 4088, 2282, 24, 0, 10, 0, 1379, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 2977, 87, 0, 4086, 10, 9, 890, 5, 0, 2656, 1089, 2205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 2401, 4070, 4011, 113, 4029, 0, 0, 0, 17, 0, 0, 25, 4096, 0, 4028, 2578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4095, 0, 3865, 0, 5, 2170, 541, 0, 1465, 48, 0, 0, 1, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17753, 14602, 11519, 12410, 19222, 14048, 29282, 12236] avg=16384.0000 max=29282 max_rank=6 min=11519 min_rank=2 imbalance_ratio=1.787231 max_load_reduction=101790 max_load_reduction_pct=0.776596 moved_to_echo=74788 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3838, 0, 0, 134, 0, 4066, 11, 5, 5, 0, 5, 25, 4088, 41, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 434, 1164, 4080, 0, 2, 0, 2, 0, 4077, 1, 21, 44, 44, 2743, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4096, 0, 0, 0, 0, 0, 484, 4095, 168, 0, 0, 3325, 2940, 0, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2719, 0, 0, 0, 4, 0, 0, 0, 3928, 3999, 0, 1930, 0, 0, 3830, 560, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2610, 199, 0, 1, 565, 0, 4035, 0, 130, 119, 0, 0, 1, 0, 1428, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2623, 936, 1, 735, 2006, 3760, 0, 4096, 0, 1301, 69, 4096, 125, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 4096, 4081, 3426, 3896, 34, 0, 0, 0, 0, 0, 0, 0, 15, 0, 2668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3525, 0, 1, 0, 103, 1849, 2290, 0, 2162, 0, 154, 4059, 0, 0, 148, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12365, 13817, 19481, 17249, 9412, 22276, 18337, 18135] avg=16384.0000 max=22276 max_rank=5 min=9412 min_rank=4 imbalance_ratio=1.359619 max_load_reduction=108796 max_load_reduction_pct=0.830048 moved_to_echo=68160 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[17664, 1664, 7808, 384, 5888, 7680, 11392, 4992, 4096, 128, 10496, 4992, 27136, 26368, 768, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[19200, 19072, 1152, 384, 5248, 12800, 4480, 256, 3968, 9472, 24192, 3328, 2432, 1536, 4864, 13824, 3840, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[14848, 25728, 20352, 128, 128, 128, 0, 128, 13696, 30976, 640, 128, 9216, 3456, 10496, 128, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 2048, 896, 9728, 18816, 1280, 8064, 640, 1024, 4736, 22016, 1792, 24576, 128, 2944, 24448, 8960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2688, 17664, 4352, 2048, 18560, 5248, 11392, 2432, 18176, 3072, 4352, 18048, 20224, 1280, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[16512, 9984, 12672, 12800, 15744, 6016, 16896, 1280, 2304, 6016, 256, 8704, 3840, 2688, 256, 16000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[5632, 24192, 4864, 1152, 1024, 1152, 16896, 4352, 13312, 768, 17664, 6784, 4608, 3968, 7168, 5504, 11008, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[6528, 14848, 19584, 24320, 10368, 15104, 8320, 128, 128, 1536, 640, 384, 128, 4224, 128, 5248, 20608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[94, 3049, 447, 455, 1064, 323, 960, 731, 980, 118, 1942, 103, 854, 769, 225, 404, 164, 456, 1165, 423, 1010, 294, 3090, 2347, 614, 1567, 806, 1147, 1358, 401, 1756, 1868, 3236, 732, 514, 598, 40, 2361, 1578, 547, 2528, 450, 1574, 943, 1296, 1522, 1034, 1141, 362, 276, 363, 1797, 688, 629, 311, 72, 427, 145, 1157, 2209, 1171, 589, 463, 1167, 594, 1851, 616, 900, 940, 139, 411, 1740, 1068, 2240, 143, 1659, 697, 1798, 2623, 1150, 2825, 1053, 3047, 467, 1658, 218, 705, 1167, 1399, 199, 852, 1164, 772, 920, 342, 926, 2036, 189, 2021, 628, 1195, 255, 1859, 2559, 441, 844, 407, 1207, 386, 429, 2584, 816, 392, 1078, 878, 1043, 198, 637, 1336, 1110, 685, 1543, 1580, 562, 1893, 172, 376, 546] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[95, 2582, 342, 297, 1457, 488, 566, 1230, 930, 174, 1175, 153, 998, 1046, 127, 322, 51, 533, 629, 611, 1337, 85, 3263, 2001, 337, 1052, 696, 1389, 751, 467, 1376, 1810, 3161, 435, 177, 416, 192, 2563, 2824, 321, 2465, 297, 1486, 749, 1295, 1846, 1270, 1451, 361, 305, 258, 2520, 949, 311, 401, 94, 966, 201, 963, 2184, 978, 620, 1087, 747, 1075, 1811, 646, 836, 1454, 209, 1054, 1636, 979, 1631, 211, 1582, 986, 1049, 2725, 1200, 2893, 923, 3149, 382, 1606, 92, 464, 875, 1669, 215, 625, 1450, 761, 674, 462, 1021, 1950, 185, 1601, 490, 1172, 462, 1425, 2150, 506, 1032, 582, 2190, 420, 243, 2399, 1218, 206, 1180, 1710, 1010, 279, 636, 409, 1126, 1172, 2060, 1686, 866, 1127, 203, 320, 449] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[62, 2763, 393, 303, 1043, 441, 561, 1349, 1243, 246, 1236, 176, 1283, 1197, 183, 375, 118, 383, 1719, 648, 994, 271, 2442, 1894, 329, 1186, 907, 1176, 1077, 310, 2163, 2449, 2627, 663, 498, 425, 205, 2445, 1982, 603, 2424, 274, 1638, 609, 1525, 1416, 1230, 1591, 306, 321, 415, 1306, 1048, 608, 596, 64, 756, 533, 748, 1769, 1923, 776, 548, 1026, 976, 1594, 498, 1238, 978, 149, 733, 1716, 1066, 2205, 202, 1807, 740, 808, 2691, 722, 2334, 1334, 3027, 368, 2095, 178, 505, 1235, 1170, 246, 646, 1001, 1022, 586, 398, 696, 1579, 402, 1638, 377, 1356, 675, 1819, 2844, 325, 766, 482, 2522, 488, 345, 1787, 1324, 274, 1165, 883, 1294, 197, 489, 590, 1178, 908, 1043, 1551, 400, 2246, 278, 456, 609] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[186, 2801, 496, 405, 948, 213, 803, 874, 1154, 164, 1707, 517, 623, 772, 483, 391, 48, 530, 922, 555, 807, 341, 2781, 1960, 553, 1044, 907, 1103, 1335, 553, 1856, 1915, 2803, 557, 747, 500, 159, 2025, 1823, 845, 2509, 693, 1755, 927, 1158, 1916, 771, 1009, 207, 412, 327, 2114, 763, 661, 329, 71, 794, 560, 1153, 1779, 1612, 331, 1099, 973, 661, 1738, 630, 1149, 886, 195, 760, 1628, 593, 1639, 232, 1350, 602, 952, 2237, 802, 2557, 779, 2645, 505, 1758, 330, 1075, 1000, 1777, 253, 899, 1016, 891, 617, 492, 1052, 2115, 303, 1512, 750, 1421, 702, 1494, 2124, 704, 1069, 570, 2046, 555, 331, 1942, 1041, 385, 1275, 785, 1507, 304, 582, 1212, 1609, 794, 1454, 1531, 797, 1623, 614, 409, 683] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[61, 2750, 455, 400, 1420, 383, 724, 1127, 1069, 144, 1477, 281, 731, 672, 151, 559, 265, 379, 863, 1104, 732, 146, 3075, 2129, 212, 1061, 820, 1051, 1169, 421, 1440, 1852, 2977, 385, 247, 470, 118, 1882, 2279, 530, 2715, 433, 1396, 970, 1581, 1315, 1085, 669, 269, 271, 293, 2971, 869, 367, 655, 38, 289, 218, 824, 1587, 1372, 458, 1144, 1009, 553, 1738, 509, 836, 823, 211, 689, 1487, 848, 2009, 449, 1783, 555, 963, 2387, 665, 2522, 1103, 2934, 979, 2028, 476, 814, 1178, 1565, 193, 619, 1382, 738, 492, 598, 660, 2318, 179, 1457, 758, 1319, 602, 1464, 2796, 1057, 832, 550, 2199, 687, 313, 2071, 743, 220, 1302, 1358, 1830, 356, 677, 1091, 1211, 792, 1770, 1592, 744, 1880, 340, 440, 529] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[110, 2608, 860, 163, 1302, 276, 819, 1141, 1098, 167, 971, 231, 907, 472, 130, 319, 101, 483, 741, 903, 1036, 37, 3007, 1715, 509, 765, 1023, 992, 1660, 145, 1400, 1750, 2821, 629, 245, 494, 121, 1986, 2123, 328, 2741, 373, 1379, 950, 1355, 1255, 1009, 708, 370, 381, 329, 2659, 615, 427, 380, 110, 378, 187, 780, 1581, 1120, 375, 1130, 1155, 596, 2315, 893, 1137, 643, 259, 445, 1642, 734, 1801, 332, 2144, 898, 901, 2492, 804, 2839, 890, 2632, 645, 1960, 350, 747, 1254, 2451, 322, 638, 1063, 720, 614, 518, 931, 1764, 272, 2095, 525, 1430, 665, 1990, 2493, 701, 973, 904, 2282, 914, 373, 1941, 719, 351, 1189, 1272, 1490, 533, 588, 1046, 1402, 929, 1590, 1804, 1178, 1122, 528, 670, 394] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[117, 3321, 309, 303, 1375, 360, 645, 1332, 1123, 173, 1427, 170, 449, 737, 352, 425, 85, 585, 1132, 619, 919, 235, 2157, 2163, 419, 1076, 651, 1443, 1307, 436, 1714, 1869, 3066, 445, 542, 595, 40, 2303, 2401, 288, 2915, 729, 1339, 844, 1589, 1703, 757, 1140, 245, 475, 385, 2242, 1245, 564, 560, 62, 294, 429, 822, 2024, 1230, 466, 1148, 1334, 822, 2163, 641, 1255, 635, 334, 401, 1711, 794, 1349, 197, 1929, 556, 868, 2880, 542, 2178, 1221, 2921, 329, 1694, 360, 813, 1143, 1479, 185, 783, 1082, 697, 618, 488, 831, 2166, 279, 1723, 592, 1442, 802, 1586, 3036, 810, 961, 417, 2031, 339, 314, 2431, 536, 129, 807, 1239, 1606, 134, 494, 1142, 1348, 1027, 1369, 1816, 738, 1320, 157, 264, 469] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[94, 2832, 834, 398, 1051, 344, 747, 965, 1777, 148, 1274, 406, 1066, 633, 145, 408, 87, 393, 1426, 1057, 1011, 165, 2663, 2104, 550, 836, 738, 768, 1976, 433, 2256, 1828, 2857, 366, 594, 938, 90, 1961, 1517, 601, 2816, 759, 1503, 927, 1408, 1845, 815, 730, 249, 431, 233, 2226, 1022, 501, 624, 68, 524, 547, 1013, 1795, 1695, 361, 866, 921, 952, 1509, 414, 1055, 653, 205, 501, 1640, 601, 1581, 308, 2114, 481, 878, 2255, 560, 2650, 931, 2326, 376, 2184, 476, 836, 1138, 1556, 233, 572, 1309, 722, 594, 451, 841, 2441, 414, 1488, 675, 1710, 1046, 1832, 2694, 901, 710, 574, 2087, 507, 140, 1771, 917, 171, 1261, 799, 1441, 277, 560, 1162, 1086, 785, 1359, 1251, 911, 1445, 666, 471, 402] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[94, 3049, 447, 455, 1064, 323, 960, 731, 980, 118, 1942, 103, 854, 769, 225, 404, 3236, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 456, 1165, 423, 1010, 294, 2108, 2347, 614, 1567, 806, 1147, 1358, 401, 1756, 1868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 732, 514, 598, 40, 2361, 1578, 547, 1958, 450, 1574, 943, 1296, 1522, 1034, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 276, 363, 1797, 688, 629, 311, 72, 427, 145, 1157, 2209, 1171, 589, 463, 1167, 1581, 1244, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 594, 1851, 616, 900, 940, 139, 411, 1740, 1068, 2240, 143, 1659, 697, 1798, 1579, 1150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2825, 1053, 1803, 467, 1658, 218, 705, 1167, 1399, 199, 852, 1164, 772, 920, 342, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2036, 189, 2021, 628, 1195, 255, 1859, 0, 441, 844, 407, 1207, 386, 429, 2542, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 1078, 878, 1043, 198, 637, 1336, 1110, 685, 1543, 1580, 562, 1893, 172, 376, 546, 982, 570, 124, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16732, 17484, 16288, 15571, 17525, 16470, 15255, 15747] avg=16384.0000 max=17525 max_rank=4 min=15255 min_rank=6 imbalance_ratio=1.069641 max_load_reduction=113547 max_load_reduction_pct=0.866295 moved_to_echo=64997 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[95, 2582, 342, 297, 1457, 488, 566, 1230, 930, 174, 1175, 153, 998, 1046, 127, 322, 3161, 822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 533, 629, 611, 1337, 85, 2226, 2001, 337, 1052, 696, 1389, 751, 467, 1376, 1810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 177, 416, 192, 2563, 2824, 321, 1909, 297, 1486, 749, 1295, 1846, 1270, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 305, 258, 2520, 949, 311, 401, 94, 966, 201, 963, 2184, 978, 620, 1087, 747, 1328, 1285, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1075, 1811, 646, 836, 1454, 209, 1054, 1636, 979, 1631, 211, 1582, 986, 1049, 1641, 1200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2893, 923, 1864, 382, 1606, 92, 464, 875, 1669, 215, 625, 1450, 761, 674, 462, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1950, 185, 1601, 490, 1172, 462, 1425, 0, 506, 1032, 582, 2190, 420, 243, 2360, 1218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 1180, 1710, 1010, 279, 636, 409, 1126, 1172, 2060, 1686, 866, 1127, 203, 320, 449, 1037, 556, 129, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15965, 15351, 17231, 16513, 18000, 15976, 15836, 16200] avg=16384.0000 max=18000 max_rank=4 min=15351 min_rank=1 imbalance_ratio=1.098633 max_load_reduction=113072 max_load_reduction_pct=0.862671 moved_to_echo=66012 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[62, 2763, 393, 303, 1043, 441, 561, 1349, 1243, 246, 1236, 176, 1283, 1197, 183, 375, 2627, 1087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 383, 1719, 648, 994, 271, 1666, 1894, 329, 1186, 907, 1176, 1077, 310, 2163, 2449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 663, 498, 425, 205, 2445, 1982, 603, 1877, 274, 1638, 609, 1525, 1416, 1230, 1591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 321, 415, 1306, 1048, 608, 596, 64, 756, 533, 748, 1769, 1923, 776, 548, 1026, 1757, 1236, 944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 976, 1594, 498, 1238, 978, 149, 733, 1716, 1066, 2205, 202, 1807, 740, 808, 1620, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2334, 1334, 1791, 368, 2095, 178, 505, 1235, 1170, 246, 646, 1001, 1022, 586, 398, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1579, 402, 1638, 377, 1356, 675, 1819, 0, 325, 766, 482, 2522, 488, 345, 1758, 1324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 1165, 883, 1294, 197, 489, 590, 1178, 908, 1043, 1551, 400, 2246, 278, 456, 609, 776, 547, 127, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16568, 17290, 16981, 16680, 17052, 15605, 15856, 15040] avg=16384.0000 max=17290 max_rank=1 min=15040 min_rank=7 imbalance_ratio=1.055298 max_load_reduction=113782 max_load_reduction_pct=0.868088 moved_to_echo=63553 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[186, 2801, 496, 405, 948, 213, 803, 874, 1154, 164, 1707, 517, 623, 772, 483, 391, 2803, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 530, 922, 555, 807, 341, 1898, 1960, 553, 1044, 907, 1103, 1335, 553, 1856, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 747, 500, 159, 2025, 1823, 845, 1943, 693, 1755, 927, 1158, 1916, 771, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 412, 327, 2114, 763, 661, 329, 71, 794, 560, 1153, 1779, 1612, 331, 1099, 973, 1311, 1080, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 661, 1738, 630, 1149, 886, 195, 760, 1628, 593, 1639, 232, 1350, 602, 952, 1347, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2557, 779, 1565, 505, 1758, 330, 1075, 1000, 1777, 253, 899, 1016, 891, 617, 492, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2115, 303, 1512, 750, 1421, 702, 1494, 0, 704, 1069, 570, 2046, 555, 331, 1910, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 1275, 785, 1507, 304, 582, 1212, 1609, 794, 1454, 1531, 797, 1623, 614, 409, 683, 883, 566, 106, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16153, 16327, 16828, 16360, 15164, 16566, 16523, 17151] avg=16384.0000 max=17151 max_rank=7 min=15164 min_rank=4 imbalance_ratio=1.046814 max_load_reduction=113921 max_load_reduction_pct=0.869148 moved_to_echo=65404 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[61, 2750, 455, 400, 1420, 383, 724, 1127, 1069, 144, 1477, 281, 731, 672, 151, 559, 2977, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 265, 379, 863, 1104, 732, 146, 2098, 2129, 212, 1061, 820, 1051, 1169, 421, 1440, 1852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 385, 247, 470, 118, 1882, 2279, 530, 2103, 433, 1396, 970, 1581, 1315, 1085, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 269, 271, 293, 2971, 869, 367, 655, 38, 289, 218, 824, 1587, 1372, 458, 1144, 1009, 1727, 1198, 837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 553, 1738, 509, 836, 823, 211, 689, 1487, 848, 2009, 449, 1783, 555, 963, 1437, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2522, 1103, 1736, 979, 2028, 476, 814, 1178, 1565, 193, 619, 1382, 738, 492, 598, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2318, 179, 1457, 758, 1319, 602, 1464, 0, 1057, 832, 550, 2199, 687, 313, 2037, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 1302, 1358, 1830, 356, 677, 1091, 1211, 792, 1770, 1592, 744, 1880, 340, 440, 529, 977, 612, 113, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16450, 15742, 15463, 16396, 15555, 17083, 16515, 17868] avg=16384.0000 max=17868 max_rank=7 min=15463 min_rank=2 imbalance_ratio=1.090576 max_load_reduction=113204 max_load_reduction_pct=0.863678 moved_to_echo=67021 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[110, 2608, 860, 163, 1302, 276, 819, 1141, 1098, 167, 971, 231, 907, 472, 130, 319, 2821, 954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 483, 741, 903, 1036, 37, 2048, 1715, 509, 765, 1023, 992, 1660, 145, 1400, 1750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 245, 494, 121, 1986, 2123, 328, 2119, 373, 1379, 950, 1355, 1255, 1009, 708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 381, 329, 2659, 615, 427, 380, 110, 378, 187, 780, 1581, 1120, 375, 1130, 1155, 1539, 1078, 877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 596, 2315, 893, 1137, 643, 259, 445, 1642, 734, 1801, 332, 2144, 898, 901, 1492, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2839, 890, 1554, 645, 1960, 350, 747, 1254, 2451, 322, 638, 1063, 720, 614, 518, 931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1764, 272, 2095, 525, 1430, 665, 1990, 0, 701, 973, 904, 2282, 914, 373, 1907, 719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 1189, 1272, 1490, 533, 588, 1046, 1402, 929, 1590, 1804, 1178, 1122, 528, 670, 394, 959, 622, 123, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15349, 15308, 15074, 15471, 17036, 17496, 17514, 17824] avg=16384.0000 max=17824 max_rank=7 min=15074 min_rank=2 imbalance_ratio=1.087891 max_load_reduction=113248 max_load_reduction_pct=0.864014 moved_to_echo=69870 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[117, 3321, 309, 303, 1375, 360, 645, 1332, 1123, 173, 1427, 170, 449, 737, 352, 425, 3066, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 585, 1132, 619, 919, 235, 1472, 2163, 419, 1076, 651, 1443, 1307, 436, 1714, 1869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 542, 595, 40, 2303, 2401, 288, 2257, 729, 1339, 844, 1589, 1703, 757, 1140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 475, 385, 2242, 1245, 564, 560, 62, 294, 429, 822, 2024, 1230, 466, 1148, 1334, 1874, 1192, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 822, 2163, 641, 1255, 635, 334, 401, 1711, 794, 1349, 197, 1929, 556, 868, 1734, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2178, 1221, 1729, 329, 1694, 360, 813, 1143, 1479, 185, 783, 1082, 697, 618, 488, 831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2166, 279, 1723, 592, 1442, 802, 1586, 0, 810, 961, 417, 2031, 339, 314, 2391, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 807, 1239, 1606, 134, 494, 1142, 1348, 1027, 1369, 1816, 738, 1320, 157, 264, 469, 685, 658, 136, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16846, 16125, 16972, 17601, 15931, 15630, 16389, 15578] avg=16384.0000 max=17601 max_rank=3 min=15578 min_rank=7 imbalance_ratio=1.074280 max_load_reduction=113471 max_load_reduction_pct=0.865715 moved_to_echo=63528 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[94, 2832, 834, 398, 1051, 344, 747, 965, 1777, 148, 1274, 406, 1066, 633, 145, 408, 2857, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 393, 1426, 1057, 1011, 165, 1817, 2104, 550, 836, 738, 768, 1976, 433, 2256, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 366, 594, 938, 90, 1961, 1517, 601, 2181, 759, 1503, 927, 1408, 1845, 815, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 431, 233, 2226, 1022, 501, 624, 68, 524, 547, 1013, 1795, 1695, 361, 866, 921, 1664, 949, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 952, 1509, 414, 1055, 653, 205, 501, 1640, 601, 1581, 308, 2114, 481, 878, 1357, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2650, 931, 1377, 376, 2184, 476, 836, 1138, 1556, 233, 572, 1309, 722, 594, 451, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2441, 414, 1488, 675, 1710, 1046, 1832, 0, 901, 710, 574, 2087, 507, 140, 1742, 917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 1261, 799, 1441, 277, 560, 1162, 1086, 785, 1359, 1251, 911, 1445, 666, 471, 402, 846, 635, 107, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17009, 17445, 16235, 16480, 14809, 16246, 17184, 15664] avg=16384.0000 max=17445 max_rank=1 min=14809 min_rank=4 imbalance_ratio=1.064758 max_load_reduction=113627 max_load_reduction_pct=0.866905 moved_to_echo=63903 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1024, 3840, 8704, 6016, 7936, 1664, 15360, 16384, 3584, 8704, 6656, 9088, 10752, 3200, 14080, 15360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[896, 22784, 4224, 2816, 9728, 2944, 5888, 8832, 9472, 1408, 11264, 2048, 6912, 6400, 1920, 3328, 23552, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[16384, 2304, 13568, 4864, 11136, 5248, 13568, 0, 5504, 7296, 4608, 16640, 4352, 2560, 16768, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[20864, 8320, 13440, 4096, 15104, 2560, 6016, 9088, 13184, 1920, 5760, 9472, 6400, 5120, 3840, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[2432, 2944, 2688, 17920, 7296, 4096, 3968, 640, 4480, 2944, 7552, 14976, 11136, 4096, 7552, 8448, 12800, 9344, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2176, 9344, 8960, 11264, 2304, 4736, 8064, 10112, 7168, 12288, 12928, 6272, 12672, 3072, 3456, 4096, 7168, 4864, 1024, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 4224, 3584, 4480, 1024, 17536, 16640, 4096, 16384, 4096, 12160, 7040, 11264, 12928, 8064, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[6272, 14720, 4864, 8448, 7040, 1792, 5120, 13312, 6784, 14464, 2176, 14464, 5632, 8320, 12288, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[4036, 0, 4010, 156, 3718, 0, 76, 3578, 2620, 16, 36, 269, 568, 26, 131, 1302, 2, 0, 395, 14, 342, 2860, 534, 2896, 29, 725, 71, 669, 0, 2209, 3884, 67, 701, 1418, 3198, 0, 0, 6, 0, 38, 3278, 35, 2, 2, 90, 383, 15, 1, 4, 2, 26, 3, 0, 2602, 1, 19, 4, 4094, 126, 2296, 1164, 0, 255, 119, 6, 506, 4024, 2018, 1996, 0, 3996, 30, 673, 1325, 1348, 188, 550, 0, 1751, 0, 3972, 2461, 468, 105, 3965, 0, 1396, 4071, 0, 3785, 6, 4069, 1031, 0, 0, 569, 11, 2149, 4080, 1978, 3760, 0, 3674, 0, 0, 461, 3817, 0, 2709, 789, 95, 292, 64, 7, 801, 137, 0, 2810, 11, 411, 333, 61, 0, 419, 15, 28, 424, 2306] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[3921, 1, 4049, 10, 76, 7, 1076, 4015, 2020, 1558, 44, 8, 2937, 31, 7, 3941, 0, 3, 2075, 11, 106, 131, 662, 1922, 132, 833, 48, 76, 497, 2426, 3340, 0, 4057, 91, 2520, 0, 0, 31, 2187, 0, 43, 8, 32, 6, 40, 1, 0, 0, 24, 1280, 1, 428, 38, 2292, 681, 0, 0, 3843, 6, 3605, 160, 0, 35, 35, 0, 9, 4064, 19, 489, 100, 2214, 0, 4058, 378, 0, 109, 921, 1, 651, 0, 4072, 680, 2038, 29, 4066, 1, 424, 4080, 0, 3096, 959, 3065, 811, 1691, 369, 3213, 0, 2725, 4081, 144, 2256, 0, 3351, 11, 227, 196, 3199, 173, 489, 99, 1, 1006, 1046, 4, 3978, 0, 526, 3628, 1, 0, 84, 0, 11, 679, 84, 31, 0, 4028] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[4095, 0, 3409, 2, 2258, 1, 872, 2707, 1629, 35, 192, 18, 978, 39, 62, 3124, 525, 0, 1733, 299, 9, 1887, 89, 2331, 0, 3, 0, 540, 0, 4057, 3003, 12, 1760, 1234, 3879, 78, 4, 0, 112, 16, 2241, 1, 0, 0, 13, 104, 444, 93, 0, 2671, 136, 0, 141, 1898, 934, 0, 28, 3842, 430, 3826, 419, 7, 1, 0, 0, 0, 4082, 9, 141, 6, 4076, 20, 3583, 127, 2978, 665, 338, 0, 3548, 0, 4080, 2320, 163, 34, 2789, 9, 455, 4039, 486, 152, 3358, 3200, 1, 612, 104, 1, 0, 1970, 4082, 510, 2723, 0, 930, 2, 21, 106, 283, 22, 3208, 2999, 613, 2609, 3589, 122, 311, 12, 1, 2249, 428, 117, 1522, 1, 0, 361, 0, 49, 2, 2633] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2876, 0, 4078, 1441, 1245, 0, 112, 3783, 3718, 2887, 2, 0, 2400, 1151, 0, 3561, 0, 3, 2467, 13, 6, 2848, 1287, 4, 22, 49, 0, 4, 279, 2001, 3358, 10, 3932, 971, 4078, 1, 477, 1, 0, 40, 1, 12, 59, 5, 4, 38, 0, 230, 0, 1, 467, 0, 0, 3720, 1, 1, 0, 3806, 691, 3527, 27, 3, 10, 2, 0, 20, 2496, 901, 629, 0, 3861, 112, 3705, 483, 9, 38, 120, 0, 2076, 0, 4086, 2450, 1485, 12, 3818, 1211, 3736, 3219, 0, 520, 415, 4085, 11, 11, 8, 0, 0, 351, 3823, 44, 2848, 0, 3684, 53, 0, 0, 3510, 0, 1731, 298, 695, 3961, 96, 0, 272, 29, 1, 4039, 109, 76, 506, 359, 6, 1364, 0, 48, 1, 1941] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2357, 0, 3367, 139, 2372, 10, 1, 3682, 3274, 1108, 61, 133, 1136, 587, 0, 460, 7, 0, 3490, 95, 1508, 2862, 3, 91, 0, 1549, 5, 265, 0, 3184, 3311, 0, 2043, 11, 3586, 12, 9, 0, 89, 2, 258, 460, 34, 0, 0, 0, 566, 0, 0, 58, 0, 0, 1925, 3340, 0, 0, 120, 4096, 2, 316, 3755, 1, 2, 30, 0, 0, 3031, 91, 1629, 134, 1520, 0, 799, 598, 0, 0, 543, 0, 3725, 0, 3920, 3487, 1255, 583, 4029, 93, 0, 4089, 1, 8, 987, 4055, 3302, 1252, 0, 3196, 0, 401, 3825, 2718, 2884, 0, 2935, 184, 20, 1739, 604, 314, 3696, 6, 68, 128, 88, 499, 261, 423, 15, 1536, 66, 499, 1721, 50, 2, 3935, 76, 312, 127, 3841] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[3742, 9, 4032, 801, 4085, 0, 16, 4055, 1851, 183, 1904, 20, 356, 555, 1, 2287, 0, 0, 703, 0, 685, 3687, 4, 2514, 15, 322, 3, 289, 0, 1341, 4014, 0, 2796, 1591, 3779, 0, 0, 0, 104, 8, 106, 0, 228, 0, 1, 2, 4, 165, 2, 520, 91, 0, 457, 3033, 10, 1, 0, 3773, 352, 1351, 3586, 0, 4, 0, 1, 911, 4091, 330, 1001, 429, 3606, 3, 3342, 658, 234, 483, 21, 0, 3001, 277, 4089, 634, 940, 0, 2891, 176, 332, 3955, 2, 548, 686, 2512, 183, 106, 16, 160, 0, 4, 3907, 222, 3933, 0, 3324, 0, 0, 916, 226, 2, 4081, 2181, 3, 3537, 187, 1613, 2040, 9, 0, 2026, 4, 622, 1333, 583, 6, 1139, 20, 2, 2, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[3510, 0, 3090, 38, 3187, 0, 601, 4015, 3046, 956, 0, 10, 2850, 164, 38, 2702, 70, 927, 212, 5, 1, 2611, 0, 46, 1653, 3287, 16, 157, 0, 2039, 3235, 12, 3048, 97, 4091, 0, 8, 1, 48, 4, 22, 113, 9, 268, 561, 1208, 85, 24, 5, 63, 2, 11, 55, 1568, 297, 7, 47, 4090, 1549, 1662, 55, 22, 36, 0, 29, 840, 3504, 0, 303, 98, 4014, 6, 3592, 901, 0, 18, 520, 8, 4062, 1, 3792, 289, 460, 3129, 3817, 4, 216, 4075, 14, 116, 1222, 4073, 1, 9, 0, 2726, 277, 478, 4077, 42, 2098, 1, 1796, 48, 0, 1151, 322, 561, 3564, 11, 22, 2788, 462, 42, 601, 13, 4, 3636, 233, 0, 3002, 9, 0, 1740, 2210, 216, 0, 2295] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1252, 0, 2166, 1730, 1411, 0, 15, 3547, 3883, 1337, 62, 5, 2195, 2581, 167, 1278, 0, 1, 658, 87, 304, 3707, 554, 14, 9, 7, 0, 19, 0, 3570, 4096, 0, 2743, 176, 4005, 0, 3, 4, 380, 1, 2412, 0, 113, 2, 1184, 991, 3231, 0, 7, 624, 0, 0, 3859, 3316, 1, 43, 114, 4087, 3390, 148, 2560, 243, 0, 2, 1, 275, 4074, 202, 0, 18, 3929, 0, 202, 750, 23, 3, 178, 0, 4079, 125, 4093, 1906, 2748, 2067, 3723, 7, 558, 4044, 7, 22, 1295, 3382, 1, 12, 2, 228, 0, 214, 1292, 998, 4073, 0, 221, 0, 25, 76, 1658, 19, 2045, 428, 78, 111, 2413, 553, 198, 0, 1, 3053, 6, 98, 1300, 3, 0, 2191, 465, 31, 104, 3170] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4036, 0, 1325, 156, 3718, 0, 76, 0, 2620, 16, 36, 269, 568, 26, 131, 1302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 395, 14, 342, 2860, 534, 2896, 29, 725, 71, 669, 0, 2209, 3884, 67, 2504, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 701, 1418, 3198, 0, 0, 6, 0, 38, 3278, 35, 2, 2, 90, 383, 15, 1, 3972, 2613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 26, 3, 0, 2602, 1, 19, 4, 4094, 126, 2296, 1164, 0, 255, 119, 2754, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 506, 4024, 2018, 1996, 0, 3996, 30, 673, 1325, 1348, 188, 550, 0, 1751, 0, 291, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2461, 468, 105, 3674, 0, 1396, 0, 0, 3785, 6, 4069, 1031, 0, 0, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2149, 988, 1978, 3760, 0, 3674, 0, 0, 461, 3817, 0, 2709, 789, 95, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 7, 801, 137, 0, 2810, 11, 411, 333, 61, 0, 419, 15, 28, 424, 2306, 2731, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14279, 17539, 15752, 14316, 18883, 17564, 20723, 12016] avg=16384.0000 max=20723 max_rank=6 min=12016 min_rank=7 imbalance_ratio=1.264832 max_load_reduction=110349 max_load_reduction_pct=0.841896 moved_to_echo=69186 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1128, 2, 2258, 1, 872, 0, 1629, 35, 192, 18, 978, 39, 62, 3124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 1733, 299, 9, 1887, 89, 2331, 0, 3, 0, 540, 0, 4057, 3003, 12, 2128, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1760, 1234, 3879, 78, 4, 0, 112, 16, 2241, 1, 0, 0, 13, 104, 444, 93, 4080, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2671, 136, 0, 141, 1898, 934, 0, 28, 3842, 430, 3826, 419, 7, 1, 0, 2755, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 9, 141, 6, 4076, 20, 3583, 127, 2978, 665, 338, 0, 3548, 0, 205, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2320, 163, 34, 2584, 9, 455, 0, 486, 152, 3358, 3200, 1, 612, 104, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1970, 989, 510, 2723, 0, 930, 2, 21, 106, 283, 22, 3208, 2999, 613, 2609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3589, 122, 311, 12, 1, 2249, 428, 117, 1522, 1, 0, 361, 0, 49, 2, 2633, 2066, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14433, 16954, 16652, 17729, 19931, 13479, 16985, 14909] avg=16384.0000 max=19931 max_rank=4 min=13479 min_rank=5 imbalance_ratio=1.216492 max_load_reduction=111141 max_load_reduction_pct=0.847939 moved_to_echo=65304 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3921, 1, 1339, 10, 76, 7, 1076, 0, 2020, 1558, 44, 8, 2937, 31, 7, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2075, 11, 106, 131, 662, 1922, 132, 833, 48, 76, 497, 2426, 3340, 0, 2528, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 91, 2520, 0, 0, 31, 2187, 0, 43, 8, 32, 6, 40, 1, 0, 0, 4072, 2619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 1280, 1, 428, 38, 2292, 681, 0, 0, 3843, 6, 3605, 160, 0, 35, 35, 2754, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4064, 19, 489, 100, 2214, 0, 4058, 378, 0, 109, 921, 1, 651, 0, 298, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 680, 2038, 29, 3768, 1, 424, 0, 0, 3096, 959, 3065, 811, 1691, 369, 3213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2725, 989, 144, 2256, 0, 3351, 11, 227, 196, 3199, 173, 489, 99, 1, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 4, 3978, 0, 526, 3628, 1, 0, 84, 0, 11, 679, 84, 31, 0, 4028, 3064, 1461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16976, 15128, 15707, 16133, 13493, 20144, 14866, 18625] avg=16384.0000 max=20144 max_rank=5 min=13493 min_rank=4 imbalance_ratio=1.229492 max_load_reduction=110928 max_load_reduction_pct=0.846313 moved_to_echo=67128 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2357, 0, 1105, 139, 2372, 10, 1, 0, 3274, 1108, 61, 133, 1136, 587, 0, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3490, 95, 1508, 2862, 3, 91, 0, 1549, 5, 265, 0, 3184, 3311, 0, 2107, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2043, 11, 3586, 12, 9, 0, 89, 2, 258, 460, 34, 0, 0, 0, 566, 0, 3920, 2626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 1925, 3340, 0, 0, 120, 4096, 2, 316, 3755, 1, 2, 30, 2586, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3031, 91, 1629, 134, 1520, 0, 799, 598, 0, 0, 543, 0, 3725, 0, 300, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3487, 1255, 583, 3729, 93, 0, 0, 1, 8, 987, 4055, 3302, 1252, 0, 3196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 920, 2718, 2884, 0, 2935, 184, 20, 1739, 604, 314, 3696, 6, 68, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 499, 261, 423, 15, 1536, 66, 499, 1721, 50, 2, 3935, 76, 312, 127, 3841, 2809, 1463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12743, 18796, 13616, 17104, 12525, 21948, 16617, 17723] avg=16384.0000 max=21948 max_rank=5 min=12525 min_rank=4 imbalance_ratio=1.339600 max_load_reduction=109124 max_load_reduction_pct=0.832550 moved_to_echo=68813 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3742, 9, 1333, 801, 4085, 0, 16, 0, 1851, 183, 1904, 20, 356, 555, 1, 2287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 703, 0, 685, 3687, 4, 2514, 15, 322, 3, 289, 0, 1341, 4014, 0, 2517, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2796, 1591, 3779, 0, 0, 0, 104, 8, 106, 0, 228, 0, 1, 2, 4, 165, 4089, 2540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 520, 91, 0, 457, 3033, 10, 1, 0, 3773, 352, 1351, 3586, 0, 4, 0, 2637, 961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 911, 4091, 330, 1001, 429, 3606, 3, 3342, 658, 234, 483, 21, 0, 3001, 277, 212, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 634, 940, 0, 2679, 176, 332, 0, 2, 548, 686, 2512, 183, 106, 16, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 946, 222, 3933, 0, 3324, 0, 0, 916, 226, 2, 4081, 2181, 3, 3537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 1613, 2040, 9, 0, 2026, 4, 622, 1333, 583, 6, 1139, 20, 2, 2, 4094, 3094, 1415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17143, 16418, 15413, 16778, 18782, 8974, 19375, 18189] avg=16384.0000 max=19375 max_rank=6 min=8974 min_rank=5 imbalance_ratio=1.182556 max_load_reduction=111697 max_load_reduction_pct=0.852180 moved_to_echo=65320 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3510, 0, 1022, 38, 3187, 0, 601, 0, 3046, 956, 0, 10, 2850, 164, 38, 2702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 927, 212, 5, 1, 2611, 0, 46, 1653, 3287, 16, 157, 0, 2039, 3235, 12, 1929, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3048, 97, 4091, 0, 8, 1, 48, 4, 22, 113, 9, 268, 561, 1208, 85, 24, 3792, 2617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 63, 2, 11, 55, 1568, 297, 7, 47, 4090, 1549, 1662, 55, 22, 36, 0, 2752, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 840, 3504, 0, 303, 98, 4014, 6, 3592, 901, 0, 18, 520, 8, 4062, 1, 280, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 460, 3129, 3537, 4, 216, 0, 14, 116, 1222, 4073, 1, 9, 0, 2726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 478, 987, 42, 2098, 1, 1796, 48, 0, 1151, 322, 561, 3564, 11, 22, 2788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 462, 42, 601, 13, 4, 3636, 233, 0, 3002, 9, 0, 1740, 2210, 216, 0, 2295, 3063, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18124, 16538, 15996, 13173, 18315, 15796, 14146, 18984] avg=16384.0000 max=18984 max_rank=7 min=13173 min_rank=3 imbalance_ratio=1.158691 max_load_reduction=112088 max_load_reduction_pct=0.855164 moved_to_echo=67241 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2876, 0, 1348, 1441, 1245, 0, 112, 0, 3718, 2887, 2, 0, 2400, 1151, 0, 3561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2467, 13, 6, 2848, 1287, 4, 22, 49, 0, 4, 279, 2001, 3358, 10, 2546, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 971, 4078, 1, 477, 1, 0, 40, 1, 12, 59, 5, 4, 38, 0, 230, 4086, 2067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 467, 0, 0, 3720, 1, 1, 0, 3806, 691, 3527, 27, 3, 10, 2, 2580, 897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2496, 901, 629, 0, 3861, 112, 3705, 483, 9, 38, 120, 0, 2076, 0, 280, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2450, 1485, 12, 3538, 1211, 3736, 0, 0, 520, 415, 4085, 11, 11, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 926, 44, 2848, 0, 3684, 53, 0, 0, 3510, 0, 1731, 298, 695, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 272, 29, 1, 4039, 109, 76, 506, 359, 6, 1364, 0, 48, 1, 1941, 2886, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20741, 15214, 16002, 15733, 14914, 17482, 18101, 12885] avg=16384.0000 max=20741 max_rank=0 min=12885 min_rank=7 imbalance_ratio=1.265930 max_load_reduction=110331 max_load_reduction_pct=0.841759 moved_to_echo=63382 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1252, 0, 717, 1730, 1411, 0, 15, 0, 3883, 1337, 62, 5, 2195, 2581, 167, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 658, 87, 304, 3707, 554, 14, 9, 7, 0, 19, 0, 3570, 4096, 0, 1352, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2743, 176, 4005, 0, 3, 4, 380, 1, 2412, 0, 113, 2, 1184, 991, 3231, 0, 4093, 2596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 624, 0, 0, 3859, 3316, 1, 43, 114, 4087, 3390, 148, 2560, 243, 0, 2, 872, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 275, 4074, 202, 0, 18, 3929, 0, 202, 750, 23, 3, 178, 0, 4079, 125, 273, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1906, 2748, 2067, 3450, 7, 558, 0, 7, 22, 1295, 3382, 1, 12, 2, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 313, 998, 4073, 0, 221, 0, 25, 76, 1658, 19, 2045, 428, 78, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2413, 553, 198, 0, 1, 3053, 6, 98, 1300, 3, 0, 2191, 465, 31, 104, 3170, 2707, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16633, 14485, 21934, 20106, 14229, 15685, 10259, 17741] avg=16384.0000 max=21934 max_rank=2 min=10259 min_rank=6 imbalance_ratio=1.338745 max_load_reduction=109138 max_load_reduction_pct=0.832657 moved_to_echo=57914 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[21120, 5632, 29184, 128, 512, 128, 2944, 128, 8448, 640, 512, 384, 1920, 2816, 4352, 640, 32128, 20352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[8064, 2944, 8576, 640, 640, 23040, 896, 1920, 9856, 1152, 128, 11904, 2944, 768, 768, 24320, 22528, 11392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[640, 1024, 11776, 640, 3072, 20608, 3200, 9856, 1920, 6784, 256, 2048, 896, 20864, 28288, 128, 17664, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[25856, 128, 9344, 4352, 18432, 128, 2816, 0, 22144, 8192, 2304, 512, 13440, 5248, 512, 18688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 14336, 9600, 6016, 27008, 1536, 7168, 0, 512, 8320, 8960, 28544, 5376, 3712, 512, 10112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 8320, 7168, 6656, 24576, 128, 19968, 384, 384, 4736, 13696, 1152, 21632, 6912, 1664, 14464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 5248, 768, 512, 6528, 21888, 2048, 128, 384, 31744, 6656, 16768, 11776, 384, 384, 256, 19712, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 2688, 29440, 3584, 6272, 896, 27264, 256, 19968, 5248, 4608, 1536, 3200, 128, 22912, 512, 2176, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[1023, 0, 678, 3237, 0, 4096, 0, 2363, 2, 3800, 0, 4026, 3277, 10, 0, 0, 72, 533, 75, 0, 7, 48, 565, 0, 506, 3642, 0, 0, 77, 4018, 1232, 302, 2133, 1, 0, 689, 66, 1, 64, 0, 4085, 3, 0, 3705, 3605, 276, 3662, 6, 0, 3883, 1928, 0, 0, 18, 558, 6, 1, 0, 0, 3948, 4083, 4089, 3069, 1889, 52, 0, 3716, 0, 15, 2, 3945, 0, 1, 0, 4090, 56, 22, 2, 0, 0, 152, 94, 4018, 755, 1, 4096, 3936, 4070, 264, 0, 4, 166, 0, 0, 0, 0, 0, 1, 4078, 0, 110, 23, 0, 149, 0, 31, 1368, 1293, 4, 4094, 0, 0, 197, 100, 4094, 3652, 226, 4, 0, 142, 87, 3, 28, 0, 4, 4, 2702, 3864] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[1559, 4, 1490, 1, 0, 3661, 18, 209, 0, 4066, 0, 3423, 2132, 128, 0, 0, 8, 26, 16, 1, 170, 533, 9, 393, 2, 101, 2604, 1, 4064, 4090, 0, 0, 4095, 21, 0, 1472, 42, 9, 1, 0, 4040, 0, 0, 1, 67, 500, 32, 0, 0, 4064, 211, 13, 15, 0, 3006, 7, 136, 19, 3, 2367, 4052, 1658, 122, 4088, 3566, 0, 4060, 0, 246, 1, 3916, 11, 845, 6, 4054, 0, 16, 18, 0, 0, 0, 4075, 3136, 0, 0, 4021, 0, 4096, 0, 685, 0, 0, 0, 2, 584, 553, 0, 1, 4079, 0, 0, 0, 1, 237, 0, 600, 7, 3932, 15, 3689, 33, 0, 0, 0, 4090, 4078, 3977, 1, 40, 18, 4085, 313, 3531, 98, 2749, 4, 1104, 1749] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[434, 0, 2783, 967, 59, 4078, 0, 115, 0, 4095, 0, 4066, 1140, 2931, 0, 1053, 148, 644, 0, 0, 3, 1614, 1, 21, 1363, 978, 6, 112, 3435, 4092, 3864, 2740, 4078, 0, 106, 0, 1, 250, 0, 38, 3771, 0, 54, 427, 2880, 30, 0, 1, 6, 4076, 349, 0, 0, 0, 0, 37, 2646, 0, 0, 3973, 3886, 4061, 33, 4095, 73, 2, 2540, 2, 48, 1, 3369, 0, 1, 42, 3682, 40, 0, 10, 66, 54, 411, 3883, 1708, 1, 12, 3766, 0, 3976, 0, 2241, 13, 6, 0, 172, 2683, 19, 8, 1, 3708, 1, 143, 4, 0, 1, 0, 815, 2312, 2044, 0, 1371, 66, 3, 240, 2298, 3997, 4089, 164, 129, 0, 0, 1963, 16, 2694, 0, 628, 13, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[167, 3, 4085, 54, 0, 4062, 148, 1755, 0, 3874, 0, 4096, 31, 42, 0, 0, 12, 103, 0, 180, 3, 0, 5, 0, 3728, 0, 0, 0, 15, 4066, 86, 4088, 3, 0, 399, 559, 0, 0, 902, 0, 4080, 0, 0, 2911, 4071, 161, 5, 1, 0, 3600, 1927, 3526, 0, 189, 0, 456, 4096, 0, 4, 91, 3886, 3878, 45, 3926, 21, 0, 2898, 1, 0, 1616, 4028, 0, 1807, 0, 4071, 0, 0, 0, 0, 0, 7, 4079, 1, 1007, 5, 3912, 63, 4096, 2, 1547, 0, 2, 0, 1, 3638, 1, 0, 0, 1564, 0, 36, 0, 0, 444, 32, 0, 684, 4081, 1, 4035, 197, 0, 0, 3, 4095, 4096, 3749, 0, 134, 0, 711, 759, 144, 8, 205, 0, 0, 3972] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4010, 0, 3475, 1957, 80, 3173, 0, 368, 0, 4056, 0, 2978, 2343, 1, 29, 2889, 0, 401, 1, 8, 4022, 1, 0, 0, 4, 0, 3, 2, 0, 4095, 3, 58, 3688, 0, 10, 1575, 745, 221, 22, 0, 3896, 833, 0, 1, 1931, 4076, 1, 48, 0, 3843, 2, 0, 0, 57, 1473, 1097, 0, 0, 0, 1027, 4080, 2581, 4091, 3663, 464, 1, 3142, 26, 5, 1072, 2501, 370, 29, 0, 2253, 2, 1, 0, 2, 0, 10, 3178, 0, 1307, 202, 3550, 89, 4096, 0, 4034, 0, 3, 1, 0, 2988, 31, 2, 0, 1150, 1108, 625, 152, 0, 251, 21, 0, 4087, 3, 56, 3895, 0, 0, 0, 0, 3992, 3982, 1335, 118, 16, 115, 17, 70, 1730, 12, 1836, 0, 1552, 2672] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[3936, 0, 3760, 0, 1079, 3270, 0, 129, 0, 4095, 0, 2397, 345, 48, 2, 1525, 173, 0, 5, 8, 1, 1, 3, 498, 222, 74, 4096, 7, 923, 4035, 0, 3729, 2423, 0, 0, 0, 0, 2458, 1954, 0, 4064, 56, 115, 75, 4057, 3701, 2015, 11, 3729, 4091, 0, 34, 228, 3, 257, 0, 24, 0, 37, 3984, 2687, 3936, 204, 287, 1439, 0, 4042, 1977, 14, 3867, 4093, 9, 1655, 799, 3056, 34, 0, 1, 4, 3, 18, 4096, 26, 0, 0, 4095, 2, 4094, 0, 3983, 0, 0, 0, 0, 147, 4, 1, 226, 119, 17, 5, 1, 0, 7, 0, 0, 0, 47, 0, 3757, 0, 14, 99, 0, 4058, 2333, 2012, 0, 2, 0, 4073, 317, 893, 0, 0, 8, 0, 834] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4055, 0, 3984, 74, 25, 4073, 0, 958, 0, 4003, 0, 4076, 3864, 38, 5, 0, 0, 309, 0, 46, 1861, 112, 7, 8, 3779, 8, 5, 0, 4093, 4034, 0, 166, 3990, 0, 0, 35, 0, 945, 448, 2, 3686, 0, 234, 3, 1483, 2999, 0, 21, 2, 4087, 0, 0, 1, 0, 1, 194, 18, 0, 284, 1396, 4017, 3875, 9, 425, 3826, 7, 4094, 19, 31, 2047, 4096, 1, 0, 20, 2206, 107, 0, 427, 0, 1, 2229, 2645, 3396, 0, 2, 4040, 6, 3984, 1, 3893, 0, 3, 1, 794, 1142, 742, 24, 0, 180, 0, 3921, 0, 16, 193, 0, 0, 0, 0, 12, 3645, 0, 33, 2013, 1, 4091, 3337, 290, 2, 0, 1, 1237, 1475, 2555, 0, 1, 0, 2, 545] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[3118, 0, 2477, 0, 3928, 680, 1, 3826, 1, 4092, 0, 4096, 2714, 2529, 0, 382, 21, 17, 0, 5, 293, 13, 264, 4, 1200, 365, 590, 2, 3, 3971, 1124, 4024, 1, 0, 0, 489, 0, 4087, 16, 0, 4095, 26, 3950, 3502, 4093, 0, 7, 0, 0, 4024, 2, 74, 21, 0, 18, 214, 3549, 7, 8, 1379, 4046, 4022, 0, 4095, 1179, 0, 2578, 96, 0, 1119, 4083, 0, 407, 10, 3758, 79, 287, 0, 1, 0, 0, 3108, 23, 0, 3101, 3596, 604, 4096, 0, 507, 0, 171, 0, 7, 360, 0, 0, 0, 159, 0, 6, 2, 0, 1, 1, 175, 1706, 304, 2, 3983, 4, 52, 3150, 23, 4096, 4079, 28, 90, 32, 9, 10, 173, 373, 0, 81, 0, 777, 1121] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1023, 0, 678, 3237, 0, 4096, 0, 2363, 2, 0, 0, 1951, 3277, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 533, 75, 0, 7, 48, 565, 0, 506, 3642, 0, 0, 77, 4018, 1232, 302, 1914, 973, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2133, 1, 0, 689, 66, 1, 64, 0, 4085, 3, 0, 3705, 3605, 276, 3662, 6, 711, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1928, 0, 0, 18, 558, 6, 1, 0, 0, 3948, 3226, 4089, 3069, 1889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 3716, 0, 15, 2, 3945, 0, 1, 0, 4090, 56, 22, 2, 0, 0, 1420, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 94, 4018, 755, 1, 4096, 3936, 2156, 264, 0, 4, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4078, 0, 110, 23, 0, 149, 0, 31, 1368, 1293, 4, 4094, 0, 0, 3800, 2910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 100, 3383, 3652, 226, 4, 0, 142, 87, 3, 28, 0, 4, 4, 2702, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16637, 14619, 19010, 18732, 14175, 15642, 17861, 14396] avg=16384.0000 max=19010 max_rank=2 min=14175 min_rank=4 imbalance_ratio=1.160278 max_load_reduction=112062 max_load_reduction_pct=0.854965 moved_to_echo=62074 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1559, 4, 1490, 1, 0, 3661, 18, 209, 0, 0, 0, 1660, 2132, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 26, 16, 1, 170, 533, 9, 393, 2, 101, 2604, 1, 4064, 4090, 0, 0, 1926, 1018, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 21, 0, 1472, 42, 9, 1, 0, 4040, 0, 0, 1, 67, 500, 32, 0, 710, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 13, 15, 0, 3006, 7, 136, 19, 3, 2367, 3202, 1658, 122, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3566, 0, 4060, 0, 246, 1, 3916, 11, 845, 6, 4054, 0, 16, 18, 0, 0, 1207, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 3136, 0, 0, 4021, 0, 2170, 0, 685, 0, 0, 0, 2, 584, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4079, 0, 0, 0, 1, 237, 0, 600, 7, 3932, 15, 3689, 33, 0, 4066, 3046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3380, 4078, 3977, 1, 40, 18, 4085, 313, 3531, 98, 2749, 4, 1104, 1749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10862, 15518, 10993, 14847, 18793, 15226, 19706, 25127] avg=16384.0000 max=25127 max_rank=7 min=10862 min_rank=0 imbalance_ratio=1.533630 max_load_reduction=105945 max_load_reduction_pct=0.808296 moved_to_echo=78852 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[167, 3, 4085, 54, 0, 4062, 148, 1755, 0, 0, 0, 1977, 31, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 103, 0, 180, 3, 0, 5, 0, 3728, 0, 0, 0, 15, 4066, 86, 4088, 1932, 903, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 399, 559, 0, 0, 902, 0, 4080, 0, 0, 2911, 4071, 161, 5, 1, 716, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1927, 3526, 0, 189, 0, 456, 4096, 0, 4, 91, 3059, 3878, 45, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 2898, 1, 0, 1616, 4028, 0, 1807, 0, 4071, 0, 0, 0, 0, 0, 1449, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4079, 1, 1007, 5, 3912, 63, 2164, 2, 1547, 0, 2, 0, 1, 3638, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1564, 0, 36, 0, 0, 444, 32, 0, 684, 4081, 1, 4035, 197, 0, 3874, 2697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3379, 4096, 3749, 0, 134, 0, 711, 759, 144, 8, 205, 0, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12324, 15791, 13818, 21197, 16708, 16429, 17645, 17160] avg=16384.0000 max=21197 max_rank=3 min=12324 min_rank=0 imbalance_ratio=1.293762 max_load_reduction=109875 max_load_reduction_pct=0.838280 moved_to_echo=67942 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[434, 0, 2783, 967, 59, 4078, 0, 115, 0, 0, 0, 1971, 1140, 2931, 0, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 644, 0, 0, 3, 1614, 1, 21, 1363, 978, 6, 112, 3435, 4092, 3864, 2740, 1870, 1021, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 106, 0, 1, 250, 0, 38, 3771, 0, 54, 427, 2880, 30, 0, 1, 694, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 349, 0, 0, 0, 0, 37, 2646, 0, 0, 3973, 3070, 4061, 33, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 2, 2540, 2, 48, 1, 3369, 0, 1, 42, 3682, 40, 0, 10, 66, 54, 1434, 813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 411, 3883, 1708, 1, 12, 3766, 0, 2106, 0, 2241, 13, 6, 0, 172, 2683, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 3708, 1, 143, 4, 0, 1, 0, 815, 2312, 2044, 0, 1371, 66, 3, 4095, 3055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 2298, 3303, 4089, 164, 129, 0, 0, 1963, 16, 2694, 0, 628, 13, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15531, 22573, 12333, 18270, 12177, 17021, 17627, 15540] avg=16384.0000 max=22573 max_rank=1 min=12177 min_rank=4 imbalance_ratio=1.377747 max_load_reduction=108499 max_load_reduction_pct=0.827782 moved_to_echo=62365 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3936, 0, 3760, 0, 1079, 3270, 0, 129, 0, 0, 0, 1163, 345, 48, 2, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 5, 8, 1, 1, 3, 498, 222, 74, 4096, 7, 923, 4035, 0, 3729, 1925, 1025, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2423, 0, 0, 0, 0, 2458, 1954, 0, 4064, 56, 115, 75, 4057, 3701, 2015, 11, 705, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3729, 0, 0, 34, 228, 3, 257, 0, 24, 0, 37, 3984, 2123, 3936, 204, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1439, 0, 4042, 1977, 14, 3867, 4093, 9, 1655, 799, 3056, 34, 0, 1, 4, 3, 845, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4096, 26, 0, 0, 4095, 2, 2169, 0, 3983, 0, 0, 0, 0, 147, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 226, 119, 17, 5, 1, 0, 7, 0, 0, 0, 47, 0, 3757, 0, 14, 4095, 3066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 3353, 2333, 2012, 0, 2, 0, 4073, 317, 893, 0, 0, 8, 0, 834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15257, 17114, 21636, 14846, 22400, 14540, 11355, 13924] avg=16384.0000 max=22400 max_rank=4 min=11355 min_rank=6 imbalance_ratio=1.367188 max_load_reduction=108672 max_load_reduction_pct=0.829102 moved_to_echo=62219 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4055, 0, 3984, 74, 25, 4073, 0, 958, 0, 0, 0, 1975, 3864, 38, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 46, 1861, 112, 7, 8, 3779, 8, 5, 0, 4093, 4034, 0, 166, 1874, 1024, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 0, 0, 35, 0, 945, 448, 2, 3686, 0, 234, 3, 1483, 2999, 0, 21, 710, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1, 194, 18, 0, 284, 1396, 3174, 3875, 9, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3826, 7, 4094, 19, 31, 2047, 4096, 1, 0, 20, 2206, 107, 0, 427, 0, 1, 1438, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2229, 2645, 3396, 0, 2, 4040, 6, 2110, 1, 3893, 0, 3, 1, 794, 1142, 742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 180, 0, 3921, 0, 16, 193, 0, 0, 0, 0, 12, 3645, 0, 33, 4003, 3063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2013, 1, 3381, 3337, 290, 2, 0, 1, 1237, 1475, 2555, 0, 1, 0, 2, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19051, 17989, 14559, 9379, 19160, 21004, 15090, 14840] avg=16384.0000 max=21004 max_rank=5 min=9379 min_rank=3 imbalance_ratio=1.281982 max_load_reduction=110068 max_load_reduction_pct=0.839752 moved_to_echo=70094 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4010, 0, 3475, 1957, 80, 3173, 0, 368, 0, 0, 0, 1444, 2343, 1, 29, 2889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 1, 8, 4022, 1, 0, 0, 4, 0, 3, 2, 0, 4095, 3, 58, 1926, 964, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3688, 0, 10, 1575, 745, 221, 22, 0, 3896, 833, 0, 1, 1931, 4076, 1, 48, 693, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 57, 1473, 1097, 0, 0, 0, 1027, 3224, 2581, 4091, 3663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 464, 1, 3142, 26, 5, 1072, 2501, 370, 29, 0, 2253, 2, 1, 0, 2, 0, 1050, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3178, 0, 1307, 202, 3550, 89, 2170, 0, 4034, 0, 3, 1, 0, 2988, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1150, 1108, 625, 152, 0, 251, 21, 0, 4087, 3, 56, 3895, 0, 0, 4056, 2879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3299, 3982, 1335, 118, 16, 115, 17, 70, 1730, 12, 1836, 0, 1552, 2672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19769, 11972, 17743, 17215, 11771, 17563, 18285, 16754] avg=16384.0000 max=19769 max_rank=0 min=11771 min_rank=4 imbalance_ratio=1.206604 max_load_reduction=111303 max_load_reduction_pct=0.849174 moved_to_echo=64373 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3118, 0, 2477, 0, 3928, 680, 1, 3826, 1, 0, 0, 1985, 2714, 2529, 0, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 17, 0, 5, 293, 13, 264, 4, 1200, 365, 590, 2, 3, 3971, 1124, 4024, 1926, 1008, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 489, 0, 4087, 16, 0, 4095, 26, 3950, 3502, 4093, 0, 7, 0, 711, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 74, 21, 0, 18, 214, 3549, 7, 8, 1379, 3197, 4022, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1179, 0, 2578, 96, 0, 1119, 4083, 0, 407, 10, 3758, 79, 287, 0, 1, 0, 1445, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3108, 23, 0, 3101, 3596, 604, 2170, 0, 507, 0, 171, 0, 7, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 6, 2, 0, 1, 1, 175, 1706, 304, 2, 3983, 4, 52, 4092, 3016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3150, 23, 3385, 4079, 28, 90, 32, 9, 10, 173, 373, 0, 81, 0, 777, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21641, 15496, 20980, 16586, 15888, 13647, 13503, 13331] avg=16384.0000 max=21641 max_rank=0 min=13331 min_rank=7 imbalance_ratio=1.320862 max_load_reduction=109431 max_load_reduction_pct=0.834892 moved_to_echo=56369 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[18304, 128, 22784, 6400, 5248, 27136, 256, 9728, 128, 0, 0, 14208, 15872, 5760, 128, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2944, 25216, 12416, 3072, 3328, 31104, 4736, 17280, 384, 16896, 128, 384, 128, 1024, 11648, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[5760, 2432, 26880, 29696, 11904, 384, 256, 384, 12288, 3200, 12032, 128, 5504, 128, 6144, 14848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 256, 15104, 1152, 4864, 256, 128, 1408, 128, 1664, 10240, 11776, 128, 28544, 384, 128, 32128, 23808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[512, 2048, 128, 256, 6400, 2432, 896, 1024, 10880, 5248, 7424, 128, 12672, 32512, 6400, 15232, 15360, 7936, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[10624, 128, 27136, 2176, 384, 9728, 30080, 512, 4864, 896, 27264, 384, 384, 512, 128, 128, 10368, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[3840, 0, 4480, 3712, 384, 384, 5376, 2048, 10496, 128, 384, 18176, 24320, 28160, 7680, 22528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[20480, 128, 640, 4864, 896, 8064, 3456, 128, 31744, 1024, 4480, 10752, 22272, 11776, 5760, 128, 5760, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 1037, 4047, 6, 8, 23, 0, 1335, 3870, 4072, 4083, 724, 4026, 3660, 13, 0, 521, 0, 492, 4090, 4096, 2, 3, 337, 0, 3781, 2599, 0, 1177, 0, 3, 3546, 228, 0, 158, 0, 25, 9, 67, 2320, 0, 364, 0, 7, 1769, 0, 0, 2, 3, 4088, 2970, 598, 2256, 291, 4040, 3329, 11, 5, 0, 2150, 0, 0, 0, 0, 13, 211, 0, 312, 36, 25, 0, 0, 40, 39, 0, 4079, 18, 0, 4089, 2, 2, 0, 0, 4091, 3936, 0, 15, 4091, 32, 1, 1774, 0, 0, 3897, 65, 2585, 0, 2, 3873, 117, 3306, 116, 0, 41, 1346, 37, 1, 12, 0, 0, 3996, 38, 0, 3814, 0, 0, 4096, 0, 4, 0, 4052, 407, 4096, 2, 122, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[13, 1, 1495, 3545, 338, 274, 52, 5, 4096, 1517, 0, 3832, 3950, 3094, 0, 0, 28, 2725, 0, 3187, 1475, 22, 134, 0, 0, 4091, 0, 0, 5, 0, 229, 1873, 0, 3, 0, 68, 116, 0, 1306, 0, 0, 0, 0, 2799, 3910, 0, 479, 2122, 0, 776, 0, 0, 3963, 0, 4080, 4043, 0, 0, 0, 4040, 1, 0, 23, 13, 597, 230, 0, 8, 4059, 0, 3246, 42, 4, 0, 0, 3964, 0, 1, 70, 0, 0, 164, 586, 3999, 3060, 0, 0, 4096, 561, 58, 0, 0, 0, 3939, 8, 109, 1, 4014, 4096, 0, 4062, 1935, 81, 153, 0, 0, 2023, 0, 1063, 0, 3280, 2, 0, 4034, 1509, 2, 4096, 0, 0, 0, 0, 0, 4089, 5, 4096, 0, 7, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[371, 759, 3002, 2, 0, 1, 493, 3961, 1081, 4091, 4096, 3491, 4088, 4095, 2860, 6, 1157, 3034, 82, 4044, 623, 1702, 4, 0, 0, 0, 5, 0, 1116, 1158, 0, 6, 0, 120, 0, 0, 33, 0, 2, 682, 48, 0, 3907, 2227, 0, 0, 4, 16, 1895, 4073, 3930, 0, 3770, 1772, 425, 1648, 0, 2, 380, 3969, 44, 0, 0, 0, 1666, 12, 0, 0, 0, 407, 394, 0, 2143, 0, 3989, 104, 3, 2, 11, 2309, 0, 0, 0, 3094, 3118, 43, 1, 3893, 0, 0, 2901, 0, 0, 1951, 4090, 12, 0, 2095, 3865, 0, 3843, 3647, 0, 1437, 23, 315, 0, 0, 4, 29, 574, 80, 106, 76, 1, 93, 0, 4025, 4, 7, 464, 185, 3961, 7, 187, 24, 1602, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 135, 4090, 0, 0, 56, 8, 3044, 3646, 3389, 4003, 4089, 4048, 733, 0, 0, 4066, 23, 0, 0, 3870, 3, 122, 19, 153, 3964, 0, 1, 874, 0, 705, 4068, 546, 116, 2, 429, 1323, 6, 44, 0, 302, 3994, 3923, 0, 0, 0, 0, 121, 0, 0, 0, 0, 3659, 132, 37, 1880, 4038, 0, 550, 2810, 0, 2, 0, 80, 3, 3460, 2, 1786, 1230, 9, 460, 0, 2, 60, 0, 3658, 5, 0, 0, 3790, 186, 4, 4, 402, 2, 4, 30, 4039, 1, 18, 0, 67, 1069, 2704, 0, 3687, 23, 3453, 4096, 25, 3135, 4078, 1311, 0, 57, 4, 1, 2, 2, 0, 35, 37, 0, 1, 595, 0, 3905, 3724, 0, 0, 0, 22, 2654, 0, 4092, 1, 4034, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 3871, 3270, 162, 0, 3178, 0, 2158, 3585, 3525, 4093, 3674, 1713, 515, 0, 295, 4, 0, 4082, 3518, 63, 0, 0, 0, 164, 0, 4096, 1077, 0, 1391, 919, 8, 29, 1484, 307, 0, 4017, 201, 0, 198, 3986, 1916, 3749, 1, 717, 0, 0, 0, 107, 0, 0, 3483, 0, 4080, 4085, 0, 0, 0, 4048, 0, 43, 1891, 70, 534, 5, 1335, 0, 15, 0, 2, 3, 358, 0, 3122, 41, 2, 0, 33, 2, 0, 0, 28, 0, 3382, 0, 4096, 4090, 3457, 237, 4075, 20, 39, 4096, 3, 44, 309, 9, 4086, 1, 4096, 82, 1, 1, 3, 5, 0, 0, 13, 0, 9, 453, 67, 726, 1, 0, 0, 39, 0, 0, 1, 0, 4082, 268, 4095, 0, 3888, 45] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[9, 4014, 3950, 21, 0, 0, 0, 4087, 4063, 4082, 26, 3368, 3901, 51, 6, 24, 0, 449, 0, 0, 3053, 1, 0, 434, 0, 3989, 4, 0, 2, 668, 1, 4038, 16, 0, 3821, 793, 6, 94, 4075, 2, 0, 4082, 108, 15, 81, 0, 479, 0, 459, 3681, 0, 119, 3993, 468, 4013, 1803, 0, 1, 2869, 3826, 0, 0, 0, 49, 322, 2319, 150, 0, 485, 0, 0, 0, 1347, 0, 46, 1, 3, 0, 1893, 769, 0, 2289, 1, 302, 4065, 0, 57, 4096, 2, 29, 0, 0, 0, 10, 2726, 3749, 0, 84, 3933, 552, 4088, 7, 83, 300, 2451, 4091, 1, 3, 769, 0, 4000, 0, 51, 16, 3959, 7, 165, 0, 0, 0, 5, 0, 4096, 2, 2647, 6, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 4017, 1866, 0, 0, 1, 573, 3624, 4092, 44, 3822, 4043, 4011, 0, 0, 324, 1933, 0, 4095, 137, 2033, 0, 0, 4, 3820, 119, 5, 6, 1674, 304, 75, 0, 0, 2, 1, 0, 223, 437, 0, 0, 3997, 2278, 31, 0, 0, 168, 1, 0, 0, 0, 0, 4068, 0, 3356, 0, 0, 0, 622, 4035, 0, 0, 1772, 9, 2992, 4, 246, 0, 4095, 0, 3, 23, 3924, 0, 450, 2, 99, 350, 636, 0, 0, 3681, 0, 374, 2, 63, 40, 3999, 2756, 0, 3034, 10, 2, 70, 4075, 3912, 0, 0, 3478, 0, 3422, 3351, 0, 207, 0, 811, 0, 0, 0, 120, 3619, 0, 57, 4086, 3042, 230, 1276, 4096, 0, 0, 0, 0, 4080, 58, 2310, 0, 351, 14] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 3318, 4022, 3393, 0, 288, 0, 536, 4096, 4004, 126, 2810, 4096, 4087, 70, 4, 3517, 0, 0, 3716, 4047, 12, 3, 0, 0, 3943, 36, 310, 70, 0, 4094, 4047, 0, 1, 0, 259, 603, 2756, 391, 129, 0, 4091, 0, 32, 17, 0, 0, 11, 0, 3175, 0, 19, 3174, 0, 2308, 484, 794, 0, 23, 2115, 21, 1, 0, 266, 0, 100, 1252, 25, 363, 85, 2, 0, 4, 20, 0, 3391, 2, 3, 8, 0, 1633, 0, 3, 2636, 3928, 0, 0, 3806, 2929, 968, 0, 0, 121, 4064, 1291, 0, 0, 914, 3740, 0, 13, 3292, 0, 5, 151, 0, 0, 0, 6, 3, 4073, 0, 0, 4096, 5, 0, 3800, 1, 0, 0, 0, 29, 4084, 0, 4095, 0, 816, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1037, 0, 6, 8, 23, 0, 1335, 3461, 0, 4083, 724, 0, 3660, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 521, 0, 492, 4090, 4096, 2, 3, 337, 0, 3668, 2599, 0, 1177, 0, 3, 3546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 0, 158, 0, 25, 9, 67, 2320, 0, 364, 0, 7, 1769, 0, 0, 2, 4047, 2080, 994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4088, 2970, 598, 1777, 291, 4040, 3329, 11, 5, 0, 2150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 211, 0, 312, 36, 25, 0, 0, 40, 39, 0, 4079, 18, 0, 4089, 2, 4026, 3078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4091, 3936, 0, 15, 1339, 32, 1, 1774, 0, 0, 3897, 65, 2585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3873, 117, 3306, 116, 0, 41, 1346, 37, 1, 12, 0, 0, 3996, 38, 672, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3814, 0, 0, 4096, 0, 4, 0, 4052, 407, 4096, 2, 122, 0, 0, 0, 409, 210, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14350, 20534, 12070, 19262, 15968, 17737, 13826, 17325] avg=16384.0000 max=20534 max_rank=1 min=12070 min_rank=2 imbalance_ratio=1.253296 max_load_reduction=110538 max_load_reduction_pct=0.843338 moved_to_echo=64856 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[13, 1, 0, 3545, 338, 274, 52, 5, 3659, 0, 0, 3832, 0, 3094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 2725, 0, 3187, 1475, 22, 134, 0, 0, 3964, 0, 0, 5, 0, 229, 1873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 68, 116, 0, 1306, 0, 0, 0, 0, 2799, 3910, 0, 479, 2122, 1495, 2085, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 0, 0, 3115, 0, 4080, 4043, 0, 0, 0, 4040, 1, 0, 23, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 597, 230, 0, 8, 4059, 0, 3246, 42, 4, 0, 0, 3964, 0, 1, 70, 0, 3950, 1146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 586, 3999, 3060, 0, 0, 1333, 561, 58, 0, 0, 0, 3939, 8, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4014, 4096, 0, 4062, 1935, 81, 153, 0, 0, 2023, 0, 1063, 0, 3280, 2, 678, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 1509, 2, 4096, 0, 0, 0, 0, 0, 4089, 5, 4096, 0, 7, 0, 437, 372, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14813, 13642, 14754, 16091, 17317, 13817, 21864, 18774] avg=16384.0000 max=21864 max_rank=6 min=13642 min_rank=1 imbalance_ratio=1.334473 max_load_reduction=109208 max_load_reduction_pct=0.833191 moved_to_echo=71772 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1866, 0, 0, 1, 573, 3241, 0, 44, 3822, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 1933, 0, 4095, 137, 2033, 0, 0, 4, 3706, 119, 5, 6, 1674, 304, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 223, 437, 0, 0, 3997, 2278, 31, 0, 0, 168, 1, 4017, 2033, 999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3203, 0, 3356, 0, 0, 0, 622, 4035, 0, 0, 1772, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2992, 4, 246, 0, 4095, 0, 3, 23, 3924, 0, 450, 2, 99, 350, 636, 0, 4043, 3093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3681, 0, 374, 2, 63, 40, 1309, 2756, 0, 3034, 10, 2, 70, 4075, 3912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3478, 0, 3422, 3351, 0, 207, 0, 811, 0, 0, 0, 120, 3619, 0, 657, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4086, 3042, 230, 1276, 4096, 0, 0, 0, 0, 4080, 58, 2310, 0, 351, 14, 383, 379, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13558, 14415, 14187, 12997, 19960, 19328, 16151, 20476] avg=16384.0000 max=20476 max_rank=7 min=12997 min_rank=3 imbalance_ratio=1.249756 max_load_reduction=110596 max_load_reduction_pct=0.843781 moved_to_echo=75915 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[9, 4014, 0, 21, 0, 0, 0, 4087, 3633, 0, 26, 3368, 0, 51, 6, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 0, 0, 3053, 1, 0, 434, 0, 3869, 4, 0, 2, 668, 1, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3821, 793, 6, 94, 4075, 2, 0, 4082, 108, 15, 81, 0, 479, 0, 3950, 2082, 996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 3681, 0, 119, 3144, 468, 4013, 1803, 0, 1, 2869, 3826, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 2319, 150, 0, 485, 0, 0, 0, 1347, 0, 46, 1, 3, 0, 1893, 769, 3901, 3086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2289, 1, 302, 4065, 0, 57, 1341, 2, 29, 0, 0, 0, 10, 2726, 3749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 3933, 552, 4088, 7, 83, 300, 2451, 4091, 1, 3, 769, 0, 4000, 0, 673, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 16, 3959, 7, 165, 0, 0, 0, 5, 0, 4096, 2, 2647, 6, 1, 0, 430, 372, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15239, 12519, 20600, 20432, 14322, 14571, 21512, 11877] avg=16384.0000 max=21512 max_rank=6 min=11877 min_rank=7 imbalance_ratio=1.312988 max_load_reduction=109560 max_load_reduction_pct=0.835876 moved_to_echo=62282 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3270, 162, 0, 3178, 0, 1930, 0, 3525, 4093, 0, 1713, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 4, 0, 4082, 3518, 63, 0, 0, 0, 160, 0, 4096, 1077, 0, 1391, 919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 29, 1484, 307, 0, 4017, 201, 0, 198, 3986, 1916, 3749, 1, 717, 0, 0, 3871, 2079, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 2743, 0, 4080, 4085, 0, 0, 0, 4048, 0, 43, 1891, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 5, 1335, 0, 15, 0, 2, 3, 358, 0, 3122, 41, 2, 0, 33, 2, 3674, 2709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 3382, 0, 4096, 1339, 3457, 237, 4075, 20, 39, 4096, 3, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 9, 4086, 1, 4096, 82, 1, 1, 3, 5, 0, 0, 13, 0, 9, 453, 672, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 726, 1, 0, 0, 39, 0, 0, 1, 0, 4082, 268, 4095, 0, 3888, 45, 228, 324, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18386, 15605, 23439, 17067, 11835, 20816, 10156, 13768] avg=16384.0000 max=23439 max_rank=2 min=10156 min_rank=6 imbalance_ratio=1.430603 max_load_reduction=107633 max_load_reduction_pct=0.821175 moved_to_echo=56575 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[371, 759, 0, 2, 0, 1, 493, 3961, 967, 0, 4096, 3491, 0, 4095, 2860, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1157, 3034, 82, 4044, 623, 1702, 4, 0, 0, 0, 5, 0, 1116, 1158, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 33, 0, 2, 682, 48, 0, 3907, 2227, 0, 0, 4, 16, 3002, 1979, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1895, 4073, 3930, 0, 2969, 1772, 425, 1648, 0, 2, 380, 3969, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1666, 12, 0, 0, 0, 407, 394, 0, 2143, 0, 3989, 104, 3, 2, 11, 2309, 4088, 3091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3094, 3118, 43, 1, 1274, 0, 0, 2901, 0, 0, 1951, 4090, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2095, 3865, 0, 3843, 3647, 0, 1437, 23, 315, 0, 0, 4, 29, 574, 80, 640, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 76, 1, 93, 0, 4025, 4, 7, 464, 185, 3961, 7, 187, 24, 1602, 0, 114, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21102, 12931, 13020, 21107, 18219, 16484, 17002, 11207] avg=16384.0000 max=21107 max_rank=3 min=11207 min_rank=7 imbalance_ratio=1.288269 max_load_reduction=109965 max_load_reduction_pct=0.838966 moved_to_echo=62912 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 135, 0, 0, 0, 56, 8, 3044, 3261, 0, 4003, 4089, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 23, 0, 0, 3870, 3, 122, 19, 153, 3845, 0, 1, 874, 0, 705, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 116, 2, 429, 1323, 6, 44, 0, 302, 3994, 3923, 0, 0, 0, 0, 121, 4090, 2053, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2881, 132, 37, 1880, 4038, 0, 550, 2810, 0, 2, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3460, 2, 1786, 1230, 9, 460, 0, 2, 60, 0, 3658, 5, 0, 0, 3790, 4048, 2561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 4, 4, 402, 2, 4, 30, 1322, 1, 18, 0, 67, 1069, 2704, 0, 3687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3453, 4096, 25, 3135, 4078, 1311, 0, 57, 4, 1, 2, 2, 0, 35, 37, 664, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 595, 0, 3905, 3724, 0, 0, 0, 22, 2654, 0, 4092, 1, 4034, 0, 385, 341, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15329, 17749, 17777, 12410, 21074, 9500, 17360, 19873] avg=16384.0000 max=21074 max_rank=4 min=9500 min_rank=5 imbalance_ratio=1.286255 max_load_reduction=109998 max_load_reduction_pct=0.839218 moved_to_echo=67807 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 3318, 0, 3393, 0, 288, 0, 536, 3663, 0, 126, 2810, 0, 4087, 70, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3517, 0, 0, 3716, 4047, 12, 3, 0, 0, 3825, 36, 310, 70, 0, 4094, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 259, 603, 2756, 391, 129, 0, 4091, 0, 32, 17, 0, 0, 11, 4022, 1935, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3175, 0, 19, 2500, 0, 2308, 484, 794, 0, 23, 2115, 21, 1, 0, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 1252, 25, 363, 85, 2, 0, 4, 20, 0, 3391, 2, 3, 8, 0, 4096, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1633, 0, 3, 2636, 3928, 0, 0, 1246, 2929, 968, 0, 0, 121, 4064, 1291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 914, 3740, 0, 13, 3292, 0, 5, 151, 0, 0, 0, 6, 3, 4073, 0, 625, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 3800, 1, 0, 0, 0, 29, 4084, 0, 4095, 0, 816, 0, 433, 295, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18295, 23677, 15225, 11706, 12377, 18819, 13201, 17772] avg=16384.0000 max=23677 max_rank=1 min=11706 min_rank=3 imbalance_ratio=1.445129 max_load_reduction=107395 max_load_reduction_pct=0.819359 moved_to_echo=62169 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[512, 9344, 0, 12160, 512, 768, 3840, 13568, 23936, 0, 16000, 26240, 0, 21504, 3584, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[9984, 8192, 640, 23296, 20864, 3840, 384, 896, 256, 23040, 2816, 4480, 4352, 3584, 6784, 18688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 10624, 31232, 768, 25984, 16512, 1536, 2176, 4096, 5376, 2048, 128, 1920, 256, 19712, 640, 5376, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[384, 16896, 9216, 384, 17408, 11904, 128, 128, 4608, 768, 31232, 384, 21760, 128, 10752, 128, 2944, 2688, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1920, 6144, 640, 14976, 21504, 128, 4352, 10624, 9856, 1408, 11904, 128, 1280, 20736, 12288, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[6144, 6400, 3072, 2176, 10368, 640, 4224, 128, 7936, 128, 7680, 15360, 256, 384, 6784, 6912, 31872, 21888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[896, 384, 5504, 1920, 2176, 7168, 6528, 3200, 640, 20608, 12160, 8960, 5888, 768, 1152, 2304, 28544, 16384, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[2432, 16000, 6912, 768, 22400, 2688, 22400, 17280, 4864, 128, 4480, 27008, 128, 128, 3712, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[85, 335, 2810, 3, 2, 0, 162, 2, 4052, 0, 1, 4076, 0, 0, 287, 1598, 250, 2, 0, 0, 1109, 0, 47, 3605, 0, 0, 3350, 0, 0, 8, 742, 11, 4, 1694, 165, 9, 120, 3, 4087, 752, 0, 3509, 3871, 166, 7, 108, 3938, 4047, 0, 4089, 1, 3171, 3898, 0, 0, 137, 3, 4, 9, 6, 4025, 1, 0, 63, 0, 5, 2766, 12, 0, 1239, 224, 0, 4093, 1250, 4001, 4008, 0, 26, 362, 20, 0, 0, 3924, 3967, 4089, 4009, 0, 0, 3, 21, 0, 1192, 3918, 25, 0, 0, 0, 0, 1351, 3950, 26, 165, 4052, 12, 137, 0, 0, 10, 6, 0, 134, 4094, 3611, 1838, 0, 0, 23, 0, 4090, 0, 59, 6, 0, 4091, 0, 0, 3839, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[149, 0, 4089, 0, 1908, 0, 27, 5, 3549, 172, 8, 2004, 0, 15, 0, 660, 2, 0, 2, 4095, 2462, 49, 3951, 784, 2129, 82, 4096, 1, 279, 4071, 13, 4, 3009, 53, 2526, 443, 0, 0, 3868, 0, 1465, 0, 4095, 1, 43, 3508, 4088, 23, 251, 3682, 2, 3927, 809, 2322, 0, 0, 0, 4, 4070, 0, 0, 0, 10, 4001, 0, 0, 3980, 4069, 1, 0, 1234, 1, 0, 71, 0, 3167, 15, 7, 3, 3069, 0, 3030, 429, 3455, 1901, 4062, 56, 0, 0, 828, 0, 4004, 3, 15, 0, 10, 0, 27, 10, 2008, 0, 3849, 4091, 0, 2104, 135, 39, 0, 337, 21, 0, 396, 0, 411, 185, 0, 0, 0, 3170, 0, 0, 541, 0, 3498, 0, 25, 2, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[611, 0, 4063, 0, 37, 0, 3910, 1, 3337, 0, 0, 3417, 0, 0, 0, 276, 4085, 8, 0, 3800, 2477, 0, 2712, 1400, 2612, 33, 2388, 0, 4087, 4080, 4053, 7, 221, 261, 192, 2310, 0, 0, 4072, 1755, 39, 3851, 2606, 0, 0, 0, 82, 4083, 0, 3700, 0, 3891, 0, 807, 0, 0, 0, 198, 0, 3, 4017, 0, 0, 0, 0, 19, 0, 4022, 0, 817, 1, 0, 0, 65, 0, 3150, 0, 0, 0, 42, 0, 3550, 15, 3655, 2968, 1, 274, 1183, 95, 0, 111, 3505, 15, 0, 604, 192, 1, 171, 94, 4057, 0, 2, 4092, 0, 56, 2, 0, 0, 1, 1491, 1, 4062, 39, 7, 3, 1, 0, 0, 4096, 0, 28, 0, 50, 3924, 1664, 0, 395, 3069] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4026, 3648, 4000, 1, 227, 2842, 0, 23, 0, 3, 3, 4093, 2, 0, 1, 2216, 3619, 0, 1, 4072, 68, 0, 0, 0, 12, 7, 4074, 13, 4029, 3622, 0, 104, 32, 0, 3686, 2440, 0, 170, 3937, 0, 3978, 1525, 4054, 6, 88, 393, 2, 4034, 0, 0, 1, 4093, 25, 5, 312, 20, 0, 0, 46, 0, 14, 0, 1393, 3759, 4, 4019, 14, 2326, 226, 0, 132, 0, 649, 2, 15, 0, 1, 428, 3977, 0, 101, 37, 154, 0, 4067, 424, 0, 1830, 0, 0, 390, 479, 3558, 0, 89, 0, 0, 4035, 0, 469, 734, 697, 3692, 0, 0, 22, 0, 15, 0, 2360, 2070, 4067, 7, 3993, 8, 0, 0, 0, 3243, 0, 29, 0, 2, 1442, 2846, 343, 260, 3097] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4067, 1820, 2234, 0, 115, 0, 2, 1015, 3637, 8, 536, 3644, 0, 0, 3241, 4088, 1751, 6, 42, 10, 3633, 0, 0, 4096, 3659, 0, 3814, 2, 0, 3950, 4073, 469, 1, 2935, 20, 1, 591, 1, 4066, 4, 4005, 2695, 1727, 573, 0, 2105, 1, 1554, 98, 3934, 0, 0, 3182, 5, 1, 175, 1260, 0, 0, 78, 4096, 0, 12, 3975, 0, 4, 1, 7, 2459, 0, 111, 0, 0, 0, 2193, 3, 8, 175, 278, 2592, 0, 4, 1301, 6, 3443, 0, 88, 570, 140, 0, 0, 3, 4, 1, 2446, 0, 1017, 1073, 0, 3121, 294, 17, 3339, 0, 0, 11, 44, 4, 26, 13, 536, 4037, 0, 0, 1775, 381, 0, 0, 4095, 0, 0, 0, 0, 4092, 3519, 0, 33, 801] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4095, 458, 4031, 0, 1672, 197, 851, 695, 98, 0, 0, 4094, 0, 4081, 0, 3880, 4096, 4, 11, 1130, 1479, 0, 175, 4096, 0, 35, 2792, 0, 14, 1, 3317, 0, 286, 0, 0, 6, 0, 0, 4092, 4069, 2380, 3477, 2479, 24, 0, 178, 2756, 57, 14, 4054, 0, 3395, 3491, 0, 1911, 4032, 0, 0, 1614, 0, 0, 0, 3438, 0, 5, 0, 3567, 359, 0, 0, 4096, 0, 0, 9, 0, 839, 1, 1499, 0, 3668, 0, 4064, 0, 562, 392, 2, 83, 1, 0, 0, 0, 1, 3, 0, 6, 0, 283, 3978, 1289, 94, 0, 0, 4053, 0, 332, 0, 0, 233, 26, 16, 0, 4092, 0, 792, 5, 3, 0, 0, 3071, 512, 784, 0, 0, 3825, 6, 1368, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4096, 10, 4067, 5, 0, 0, 196, 0, 3682, 31, 30, 140, 0, 1132, 3, 3735, 3933, 0, 2, 3598, 1413, 20, 3860, 1482, 4019, 0, 3989, 0, 0, 3867, 1610, 10, 2, 4014, 885, 257, 0, 12, 3242, 32, 0, 3536, 4095, 57, 0, 3, 1373, 339, 21, 2858, 10, 3221, 4067, 3981, 84, 18, 0, 0, 0, 0, 997, 0, 1118, 3, 0, 53, 2260, 133, 0, 0, 2759, 7, 49, 2, 2221, 12, 46, 6, 0, 12, 27, 0, 2, 0, 4070, 16, 0, 36, 15, 0, 0, 0, 4031, 0, 0, 147, 2, 4096, 578, 0, 1, 0, 3899, 0, 0, 818, 0, 0, 0, 1, 0, 4094, 4, 360, 1, 0, 1, 1593, 4058, 0, 63, 190, 4085, 4067, 4056, 13, 3923, 110] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[3367, 3466, 3919, 1, 135, 0, 0, 4, 3481, 0, 0, 4084, 0, 7, 357, 4084, 0, 0, 0, 762, 3419, 0, 3869, 3, 0, 1, 3230, 3945, 0, 2, 1, 0, 3989, 278, 0, 21, 15, 2, 4073, 3986, 75, 4012, 3646, 767, 46, 10, 250, 3760, 241, 4086, 0, 2732, 4054, 0, 2, 4085, 0, 0, 0, 0, 3927, 0, 188, 10, 0, 315, 10, 924, 432, 214, 1504, 0, 3175, 1, 1, 3048, 0, 4, 0, 3050, 0, 1, 0, 4083, 3936, 2427, 0, 1774, 13, 0, 171, 0, 1949, 0, 0, 41, 17, 28, 906, 257, 10, 0, 4094, 3, 5, 0, 0, 9, 0, 151, 0, 263, 3539, 3805, 0, 0, 0, 47, 973, 0, 12, 0, 0, 2545, 3888, 0, 55, 1000] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[149, 0, 1255, 0, 1908, 0, 27, 5, 3549, 172, 8, 2004, 0, 15, 0, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 4095, 2462, 49, 3951, 784, 2129, 82, 0, 1, 279, 2121, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3009, 53, 2526, 443, 0, 0, 0, 0, 1465, 0, 2591, 1, 43, 3508, 4088, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 3659, 2, 3927, 809, 2322, 0, 0, 0, 4, 4070, 0, 0, 0, 10, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 4069, 1, 0, 1234, 1, 0, 71, 0, 3167, 15, 7, 3, 3069, 3868, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3030, 429, 3455, 1901, 4062, 56, 0, 0, 828, 0, 4004, 3, 15, 0, 10, 3507, 732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 10, 2008, 0, 3849, 4091, 0, 2104, 135, 39, 0, 337, 21, 0, 396, 2102, 1501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 411, 185, 0, 0, 0, 3170, 0, 0, 541, 0, 3498, 0, 25, 2, 7, 1950, 23, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9752, 15974, 17750, 19055, 20074, 22032, 16620, 9815] avg=16384.0000 max=22032 max_rank=5 min=9752 min_rank=0 imbalance_ratio=1.344727 max_load_reduction=109040 max_load_reduction_pct=0.831909 moved_to_echo=68541 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[611, 0, 1247, 0, 37, 0, 3910, 1, 3337, 0, 0, 3417, 0, 0, 0, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 8, 0, 3800, 2477, 0, 2712, 1400, 2612, 33, 0, 0, 4087, 2126, 4053, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 261, 192, 2310, 0, 0, 0, 1755, 39, 3851, 1649, 0, 0, 0, 82, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3677, 0, 3891, 0, 807, 0, 0, 0, 198, 0, 3, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4022, 0, 817, 1, 0, 0, 65, 0, 3150, 0, 0, 0, 42, 4072, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3550, 15, 3655, 2968, 1, 274, 1183, 95, 0, 111, 3505, 15, 0, 604, 192, 2045, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 171, 94, 4057, 0, 2, 4092, 0, 56, 2, 0, 0, 1, 1491, 1, 4062, 2089, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 7, 3, 1, 0, 0, 4096, 0, 28, 0, 50, 3924, 1664, 0, 395, 3069, 1954, 23, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12836, 27400, 14443, 12593, 12531, 18940, 17074, 15255] avg=16384.0000 max=27400 max_rank=1 min=12531 min_rank=4 imbalance_ratio=1.672363 max_load_reduction=103672 max_load_reduction_pct=0.790955 moved_to_echo=63800 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4095, 458, 1229, 0, 1672, 197, 851, 695, 98, 0, 0, 4094, 0, 4081, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 11, 1130, 1479, 0, 175, 4096, 0, 35, 0, 0, 14, 0, 3317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 0, 0, 6, 0, 0, 0, 4069, 2380, 3477, 1559, 24, 0, 178, 2756, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4026, 0, 3395, 3491, 0, 1911, 4032, 0, 0, 1614, 0, 0, 0, 3438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3567, 359, 0, 0, 4096, 0, 0, 9, 0, 839, 1, 1499, 0, 3668, 4092, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 562, 392, 2, 83, 1, 0, 0, 0, 1, 3, 0, 6, 0, 2390, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 3978, 1289, 94, 0, 0, 4053, 0, 332, 0, 0, 233, 26, 16, 0, 4092, 2077, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 792, 5, 3, 0, 0, 3071, 512, 784, 0, 0, 3825, 6, 1368, 2, 4096, 1, 28, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21350, 14357, 14792, 21921, 18537, 8229, 17386, 14500] avg=16384.0000 max=21921 max_rank=3 min=8229 min_rank=5 imbalance_ratio=1.337952 max_load_reduction=109151 max_load_reduction_pct=0.832756 moved_to_echo=58652 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4026, 3648, 1228, 1, 227, 2842, 0, 23, 0, 3, 3, 4093, 2, 0, 1, 2216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3619, 0, 1, 4072, 68, 0, 0, 0, 12, 7, 0, 13, 4029, 1887, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 3686, 2440, 0, 170, 0, 0, 3978, 1525, 2565, 6, 88, 393, 2, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 25, 5, 312, 20, 0, 0, 46, 0, 14, 0, 1393, 3759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4019, 14, 2326, 226, 0, 132, 0, 649, 2, 15, 0, 1, 428, 3977, 0, 3937, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 37, 154, 0, 4067, 424, 0, 1830, 0, 0, 390, 479, 3558, 0, 89, 0, 3487, 716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 469, 734, 697, 3692, 0, 0, 22, 0, 15, 0, 2360, 2070, 4067, 2056, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3993, 8, 0, 0, 0, 3243, 0, 29, 0, 2, 1442, 2846, 343, 260, 3097, 1735, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18313, 13812, 18919, 9668, 16317, 15332, 21703, 17008] avg=16384.0000 max=21703 max_rank=6 min=9668 min_rank=3 imbalance_ratio=1.324646 max_load_reduction=109369 max_load_reduction_pct=0.834419 moved_to_echo=70360 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 10, 1248, 5, 0, 0, 196, 0, 3682, 31, 30, 140, 0, 1132, 3, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 0, 2, 3598, 1413, 20, 3860, 1482, 4019, 0, 0, 0, 0, 2015, 1610, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4014, 885, 257, 0, 12, 0, 32, 0, 3536, 2591, 57, 0, 3, 1373, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 2841, 10, 3221, 4067, 3981, 84, 18, 0, 0, 0, 0, 997, 0, 1118, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 2260, 133, 0, 0, 2759, 7, 49, 2, 2221, 12, 46, 6, 0, 12, 3242, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 2, 0, 4070, 16, 0, 36, 15, 0, 0, 0, 4031, 0, 0, 147, 3414, 728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 578, 0, 1, 0, 3899, 0, 0, 818, 0, 0, 0, 1, 0, 4094, 2091, 1501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 360, 1, 0, 1, 1593, 4058, 0, 63, 190, 4085, 4067, 4056, 13, 3923, 110, 1852, 17, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14308, 21962, 13101, 16361, 11377, 12486, 17081, 24396] avg=16384.0000 max=24396 max_rank=7 min=11377 min_rank=4 imbalance_ratio=1.489014 max_load_reduction=106676 max_load_reduction_pct=0.813873 moved_to_echo=65340 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4067, 1820, 686, 0, 115, 0, 2, 1015, 3637, 8, 536, 3644, 0, 0, 3241, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1751, 6, 42, 10, 3633, 0, 0, 4096, 3659, 0, 0, 2, 0, 2053, 4073, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2935, 20, 1, 591, 1, 0, 4, 4005, 2695, 1093, 573, 0, 2105, 1, 1554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 3910, 0, 0, 3182, 5, 1, 175, 1260, 0, 0, 78, 4096, 0, 12, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 7, 2459, 0, 111, 0, 0, 0, 2193, 3, 8, 175, 278, 2592, 4066, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1301, 6, 3443, 0, 88, 570, 140, 0, 0, 3, 4, 1, 2446, 0, 3265, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 1073, 0, 3121, 294, 17, 3339, 0, 0, 11, 44, 4, 26, 13, 536, 4037, 1148, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 381, 0, 0, 4095, 0, 0, 0, 0, 4092, 3519, 0, 33, 801, 1897, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22859, 19794, 15579, 16792, 12446, 11671, 15313, 16618] avg=16384.0000 max=22859 max_rank=0 min=11671 min_rank=5 imbalance_ratio=1.395203 max_load_reduction=108213 max_load_reduction_pct=0.825600 moved_to_echo=56048 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[85, 335, 863, 3, 2, 0, 162, 2, 4052, 0, 1, 4076, 0, 0, 287, 1598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 2, 0, 0, 1109, 0, 47, 3605, 0, 0, 0, 0, 0, 5, 742, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1694, 165, 9, 120, 3, 0, 752, 0, 3509, 2449, 166, 7, 108, 3938, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 1, 3171, 3898, 0, 0, 137, 3, 4, 9, 6, 4025, 1, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2766, 12, 0, 1239, 224, 0, 4093, 1250, 4001, 4008, 0, 26, 362, 20, 4087, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3924, 3967, 4089, 4009, 0, 0, 3, 21, 0, 1192, 3918, 25, 0, 0, 2868, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1351, 3950, 26, 165, 4052, 12, 137, 0, 0, 10, 6, 0, 134, 4094, 1444, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3611, 1838, 0, 0, 23, 0, 4090, 0, 59, 6, 0, 4091, 0, 0, 3839, 0, 3, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11466, 5771, 16971, 15382, 22575, 24519, 16800, 17588] avg=16384.0000 max=24519 max_rank=5 min=5771 min_rank=1 imbalance_ratio=1.496521 max_load_reduction=106553 max_load_reduction_pct=0.812935 moved_to_echo=81482 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3367, 3466, 1202, 1, 135, 0, 0, 4, 3481, 0, 0, 4084, 0, 7, 357, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 762, 3419, 0, 3869, 3, 0, 1, 0, 3945, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 278, 0, 21, 15, 2, 0, 3986, 75, 4012, 2306, 767, 46, 10, 250, 3760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 4061, 0, 2732, 4054, 0, 2, 4085, 0, 0, 0, 0, 3927, 0, 188, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 10, 924, 432, 214, 1504, 0, 3175, 1, 1, 3048, 0, 4, 0, 3050, 4073, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4083, 3936, 2427, 0, 1774, 13, 0, 171, 0, 1949, 0, 0, 41, 2766, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 28, 906, 257, 10, 0, 4094, 3, 5, 0, 0, 9, 0, 151, 0, 263, 2015, 1337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3539, 3805, 0, 0, 0, 47, 973, 0, 12, 0, 0, 2545, 3888, 0, 55, 1000, 0, 25, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20188, 12002, 19517, 19300, 17215, 17863, 9095, 15892] avg=16384.0000 max=20188 max_rank=0 min=9095 min_rank=6 imbalance_ratio=1.232178 max_load_reduction=110884 max_load_reduction_pct=0.845978 moved_to_echo=60065 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[20608, 9856, 8960, 128, 4096, 3072, 5248, 1792, 21888, 256, 640, 25600, 128, 5248, 3968, 20608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[1408, 13440, 4352, 14080, 1152, 4736, 31360, 128, 2688, 1024, 128, 384, 512, 4096, 2816, 25216, 15104, 9856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[7552, 9344, 7552, 5504, 768, 256, 0, 10624, 12032, 22656, 16896, 1664, 256, 6400, 12544, 17920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[17792, 128, 128, 17536, 16128, 128, 14720, 15488, 12544, 256, 0, 3968, 8448, 10240, 13824, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[640, 26240, 128, 24448, 19584, 7168, 2432, 8576, 1280, 256, 5760, 128, 17152, 128, 6272, 11904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 10752, 5888, 15744, 24960, 11008, 512, 5504, 384, 896, 768, 9216, 13568, 128, 3200, 512, 23808, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[7296, 11264, 2048, 512, 128, 1664, 26880, 512, 1024, 768, 4224, 27520, 16000, 1792, 8576, 12288, 9472, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 4480, 12672, 11904, 3200, 2304, 10112, 128, 8064, 1408, 8448, 14336, 128, 2176, 4736, 12544, 31488, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 2840, 105, 43, 30, 252, 3147, 517, 3760, 111, 0, 2, 0, 0, 0, 203, 0, 0, 2, 0, 0, 4045, 1, 16, 4096, 4096, 222, 4038, 157, 58, 59, 271, 0, 0, 21, 3999, 0, 0, 1, 1987, 17, 3745, 6, 3983, 4024, 0, 332, 4, 36, 0, 3996, 1, 4044, 4096, 0, 0, 1305, 0, 0, 59, 0, 12, 0, 0, 600, 2, 2188, 4095, 0, 0, 4057, 36, 1715, 3457, 742, 3601, 101, 0, 0, 3657, 0, 2637, 0, 4070, 0, 0, 3, 5, 14, 0, 0, 1, 2599, 995, 4034, 0, 0, 170, 0, 3396, 8, 0, 0, 3786, 3820, 1370, 0, 1153, 4003, 138, 0, 0, 22, 3438, 3975, 0, 0, 3626, 3, 123, 0, 0, 3692, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[89, 0, 4085, 0, 0, 2411, 0, 1066, 24, 4096, 28, 0, 4064, 4, 14, 1, 6, 0, 75, 1606, 1, 3065, 4096, 4096, 0, 3313, 3313, 0, 3347, 30, 0, 486, 2075, 247, 0, 162, 1062, 1, 0, 3846, 2, 0, 4088, 3584, 50, 4088, 86, 907, 0, 497, 0, 268, 0, 1831, 56, 22, 188, 0, 3957, 370, 0, 411, 3277, 2986, 0, 1299, 0, 0, 1395, 0, 4, 2716, 6, 0, 1, 260, 4026, 61, 2919, 0, 3393, 0, 4096, 0, 2135, 9, 0, 0, 2856, 4083, 0, 0, 28, 104, 0, 113, 132, 7, 0, 0, 35, 1128, 31, 47, 972, 4057, 0, 182, 149, 4096, 0, 0, 38, 1955, 4096, 3212, 0, 0, 1845, 0, 0, 0, 5, 3891, 0, 0, 1562, 751] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 4044, 0, 0, 4009, 0, 3574, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 3846, 18, 0, 261, 4050, 179, 4090, 219, 4093, 2138, 3, 1523, 0, 4080, 2, 0, 277, 0, 3854, 0, 0, 3995, 101, 0, 0, 3339, 1, 3624, 2166, 0, 2825, 0, 664, 0, 182, 0, 2, 2872, 0, 0, 40, 1, 0, 69, 3110, 3017, 743, 15, 23, 0, 18, 1568, 14, 0, 4052, 0, 0, 1074, 0, 3315, 2074, 3850, 3722, 4095, 0, 2989, 3, 4096, 0, 0, 601, 0, 2, 330, 0, 251, 1161, 1312, 3560, 0, 0, 0, 0, 1, 153, 0, 0, 3881, 3784, 694, 3826, 32, 0, 3, 132, 0, 229, 4096, 640, 0, 0, 0, 0, 742, 38, 5, 3524, 0, 0, 0, 56] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[2, 5, 512, 0, 0, 1, 9, 4002, 0, 3, 0, 0, 1294, 1, 0, 74, 0, 0, 0, 3142, 2677, 4084, 5, 13, 0, 4048, 4067, 0, 1104, 1, 3226, 0, 0, 3965, 7, 11, 3964, 140, 3500, 17, 0, 2201, 3730, 4096, 4005, 3686, 0, 0, 0, 223, 0, 4, 0, 4040, 1889, 30, 3996, 0, 4049, 0, 183, 2, 17, 206, 0, 0, 0, 10, 3592, 31, 101, 1095, 0, 96, 2710, 2012, 3492, 4096, 3968, 0, 4083, 1, 3896, 35, 3751, 0, 7, 83, 113, 4055, 0, 0, 0, 26, 0, 1, 0, 0, 6, 20, 508, 4073, 0, 362, 4084, 4068, 0, 759, 9, 4093, 48, 7, 0, 0, 222, 0, 0, 517, 2098, 0, 1, 0, 0, 0, 655, 0, 56, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 2, 4094, 0, 0, 2831, 1, 1542, 0, 122, 3, 0, 4011, 0, 0, 4079, 581, 3, 1028, 82, 3469, 72, 2380, 1600, 6, 3911, 4036, 0, 0, 5, 4096, 84, 937, 0, 4092, 3, 0, 0, 16, 2477, 0, 0, 182, 3888, 4096, 3808, 0, 27, 0, 503, 0, 0, 1, 3916, 3681, 207, 2, 0, 0, 26, 1430, 1150, 3489, 0, 24, 278, 0, 0, 3668, 4, 0, 3232, 0, 35, 30, 0, 4065, 57, 0, 3597, 629, 0, 4050, 939, 854, 1, 0, 3, 0, 3931, 0, 0, 10, 14, 0, 3997, 287, 0, 136, 1, 23, 424, 226, 0, 4072, 235, 243, 3528, 74, 4089, 5, 711, 119, 0, 4074, 20, 18, 0, 0, 0, 7, 1, 4039, 4085, 45, 4, 3217, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 148, 3931, 2, 104, 1663, 13, 4091, 3, 0, 0, 0, 1177, 7, 0, 3, 0, 1, 0, 0, 0, 4096, 3229, 0, 0, 0, 4066, 27, 241, 0, 4081, 0, 0, 1, 4096, 21, 4004, 0, 4045, 3342, 0, 6, 1568, 3721, 4096, 4079, 0, 11, 0, 31, 0, 1, 23, 3864, 3756, 196, 0, 0, 0, 0, 855, 2592, 5, 958, 1, 0, 1278, 21, 2747, 0, 0, 694, 92, 638, 4094, 22, 3798, 19, 2518, 4033, 964, 113, 1680, 129, 4037, 0, 5, 396, 1042, 0, 0, 0, 0, 0, 3615, 4096, 100, 12, 2143, 4, 0, 0, 0, 3, 4093, 4047, 7, 38, 5, 4082, 0, 0, 0, 5, 0, 267, 0, 1800, 2168, 0, 15, 0, 0, 4079, 3971, 0, 47, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[513, 0, 0, 286, 2049, 4025, 0, 4056, 1, 4082, 1557, 2, 0, 0, 9, 0, 0, 10, 0, 787, 1, 557, 1, 4059, 0, 2767, 3819, 0, 4036, 5, 4073, 1, 4083, 0, 2, 0, 3590, 0, 4095, 45, 0, 0, 1447, 506, 1656, 4093, 1, 302, 0, 0, 0, 0, 25, 2895, 3758, 13, 861, 0, 1, 0, 9, 4096, 3780, 1202, 3, 1172, 16, 0, 2920, 1298, 0, 4067, 0, 0, 3195, 60, 3, 39, 0, 0, 45, 28, 703, 1889, 4047, 0, 0, 0, 2, 14, 191, 0, 0, 3989, 0, 4, 0, 0, 55, 0, 4086, 0, 2, 88, 9, 2328, 1994, 1005, 414, 4040, 44, 891, 0, 0, 3624, 2314, 82, 0, 4003, 2428, 0, 0, 0, 4096, 0, 12, 2715, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[122, 14, 4055, 2, 0, 0, 0, 4043, 0, 4031, 0, 262, 600, 16, 0, 4088, 0, 3435, 0, 1, 0, 4096, 0, 1680, 0, 4096, 4093, 0, 0, 269, 353, 0, 37, 122, 91, 1, 3715, 0, 0, 3620, 0, 12, 345, 4096, 3885, 90, 9, 49, 2, 805, 0, 27, 0, 2179, 3699, 0, 2917, 78, 4096, 116, 4096, 51, 0, 729, 0, 10, 4, 0, 25, 0, 0, 2757, 0, 1126, 3269, 1499, 3966, 538, 1606, 0, 931, 0, 3937, 0, 4036, 4073, 0, 0, 1, 0, 0, 0, 5, 0, 3934, 4084, 5, 107, 0, 37, 0, 0, 3, 1, 1, 3492, 0, 447, 134, 3944, 1270, 1328, 0, 1423, 4075, 0, 0, 0, 918, 42, 0, 0, 1428, 2920, 3538, 0, 0, 35] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2840, 105, 43, 30, 252, 3147, 517, 3760, 111, 0, 2, 0, 0, 0, 2442, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 0, 0, 2, 0, 0, 4045, 1, 16, 3812, 0, 222, 4038, 157, 58, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 0, 0, 21, 3999, 0, 0, 1, 1987, 17, 3745, 6, 1570, 0, 0, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 36, 0, 3996, 1, 4044, 4096, 0, 0, 1305, 0, 0, 59, 0, 12, 0, 1267, 775, 284, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 600, 2, 2188, 4095, 0, 0, 4057, 36, 1715, 3457, 742, 2826, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3657, 0, 2637, 0, 3976, 0, 0, 3, 5, 14, 0, 0, 1, 2599, 995, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 3396, 8, 0, 0, 3786, 3820, 1370, 0, 1153, 4003, 138, 0, 1582, 1146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 3438, 3975, 0, 0, 3626, 3, 123, 0, 0, 3692, 0, 1, 0, 0, 3363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13982, 12613, 11949, 15973, 19819, 17921, 20572, 18243] avg=16384.0000 max=20572 max_rank=6 min=11949 min_rank=2 imbalance_ratio=1.255615 max_load_reduction=110500 max_load_reduction_pct=0.843048 moved_to_echo=76555 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4044, 0, 0, 4009, 0, 3574, 0, 4096, 0, 0, 4, 0, 0, 0, 1315, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 18, 0, 261, 4050, 179, 4090, 219, 3806, 0, 3, 1523, 0, 4080, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 3854, 0, 0, 3995, 101, 0, 0, 3339, 1, 1420, 0, 0, 2825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 664, 0, 182, 0, 2, 2872, 0, 0, 40, 1, 0, 69, 3110, 3017, 743, 1158, 718, 287, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 23, 0, 18, 1568, 14, 0, 4052, 0, 0, 1074, 0, 2597, 2074, 3850, 3722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2989, 3, 3996, 0, 0, 601, 0, 2, 330, 0, 251, 1161, 1312, 3560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 153, 0, 0, 3881, 3784, 694, 3826, 32, 0, 3, 132, 851, 1046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 4096, 640, 0, 0, 0, 0, 742, 38, 5, 3524, 0, 0, 0, 56, 1754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17426, 22077, 15812, 12963, 19007, 18300, 14403, 11084] avg=16384.0000 max=22077 max_rank=1 min=11084 min_rank=7 imbalance_ratio=1.347473 max_load_reduction=108995 max_load_reduction_pct=0.831566 moved_to_echo=62794 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[89, 0, 4085, 0, 0, 2411, 0, 1066, 24, 4096, 28, 0, 4064, 4, 14, 1, 2481, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 75, 1606, 1, 3065, 4096, 4096, 0, 3083, 0, 0, 3347, 30, 0, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2075, 247, 0, 162, 1062, 1, 0, 3846, 2, 0, 4088, 3584, 21, 0, 86, 907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 0, 268, 0, 1831, 56, 22, 188, 0, 3957, 370, 0, 411, 3277, 2986, 15, 866, 230, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1299, 0, 0, 1395, 0, 4, 2716, 6, 0, 1, 260, 3160, 61, 2919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3393, 0, 4096, 0, 2086, 9, 0, 0, 2856, 4083, 0, 0, 28, 104, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 7, 0, 0, 35, 1128, 31, 47, 972, 4057, 0, 182, 149, 4096, 0, 0, 1607, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 1955, 4096, 3212, 0, 0, 1845, 0, 0, 0, 5, 3891, 0, 0, 1562, 751, 2720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18956, 19891, 16081, 15023, 11821, 16768, 12457, 20075] avg=16384.0000 max=20075 max_rank=7 min=11821 min_rank=4 imbalance_ratio=1.225281 max_load_reduction=110997 max_load_reduction_pct=0.846840 moved_to_echo=61121 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 5, 512, 0, 0, 1, 9, 4002, 0, 3, 0, 0, 1294, 1, 0, 74, 2238, 728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3142, 2677, 4084, 5, 13, 0, 3767, 0, 0, 1104, 1, 3226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 7, 11, 3964, 140, 3500, 17, 0, 2201, 3730, 4096, 1579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 4, 0, 4040, 1889, 30, 3996, 0, 4049, 0, 183, 2, 17, 206, 1274, 751, 281, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3592, 31, 101, 1095, 0, 96, 2710, 2012, 2741, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 1, 3896, 35, 3664, 0, 7, 83, 113, 4055, 0, 0, 0, 26, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 20, 508, 4073, 0, 362, 4084, 4068, 0, 759, 9, 4093, 48, 7, 1448, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 0, 517, 2098, 0, 1, 0, 0, 0, 655, 0, 56, 1, 3339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8869, 18019, 23210, 17032, 20452, 15964, 20637, 6889] avg=16384.0000 max=23210 max_rank=2 min=6889 min_rank=7 imbalance_ratio=1.416626 max_load_reduction=107862 max_load_reduction_pct=0.822922 moved_to_echo=63942 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4094, 0, 0, 2831, 1, 1542, 0, 122, 3, 0, 4011, 0, 0, 4079, 2312, 723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 3, 1028, 82, 3469, 72, 2380, 1600, 6, 3640, 0, 0, 0, 5, 4096, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 937, 0, 4092, 3, 0, 0, 16, 2477, 0, 0, 182, 3888, 1615, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 503, 0, 0, 1, 3916, 3681, 207, 2, 0, 0, 26, 1430, 1150, 3489, 0, 1303, 875, 271, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 278, 0, 0, 3668, 4, 0, 3232, 0, 35, 30, 0, 3190, 57, 0, 3597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 629, 0, 4050, 939, 835, 1, 0, 3, 0, 3931, 0, 0, 10, 14, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 0, 136, 1, 23, 424, 226, 0, 4072, 235, 243, 3528, 74, 4089, 5, 711, 1496, 1178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 0, 4074, 20, 18, 0, 0, 0, 7, 1, 4039, 4085, 45, 4, 3217, 2, 3313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19720, 17046, 13237, 16873, 14115, 14409, 16728, 18944] avg=16384.0000 max=19720 max_rank=0 min=13237 min_rank=2 imbalance_ratio=1.203613 max_load_reduction=111352 max_load_reduction_pct=0.849548 moved_to_echo=64196 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 148, 3931, 2, 104, 1663, 13, 4091, 3, 0, 0, 0, 1177, 7, 0, 3, 2477, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 3229, 0, 0, 0, 0, 27, 241, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 21, 4004, 0, 4045, 3342, 0, 6, 1568, 3721, 1615, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 1, 23, 3864, 3756, 196, 0, 0, 0, 0, 855, 2592, 5, 958, 1303, 817, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1278, 21, 2747, 0, 0, 694, 92, 638, 4094, 22, 2981, 19, 2518, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 964, 113, 1680, 129, 3944, 0, 5, 396, 1042, 0, 0, 0, 0, 0, 3615, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 12, 2143, 4, 0, 0, 0, 3, 4093, 4047, 7, 38, 5, 4082, 0, 0, 1602, 1178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 267, 0, 1800, 2168, 0, 15, 0, 0, 4079, 3971, 0, 47, 0, 3337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14348, 11675, 22430, 14494, 19138, 15984, 17314, 15689] avg=16384.0000 max=22430 max_rank=2 min=11675 min_rank=1 imbalance_ratio=1.369019 max_load_reduction=108642 max_load_reduction_pct=0.828873 moved_to_echo=68125 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[513, 0, 0, 286, 2049, 4025, 0, 4056, 1, 4082, 1557, 2, 0, 0, 9, 0, 2485, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 787, 1, 557, 1, 4059, 0, 2575, 0, 0, 4036, 5, 4073, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 2, 0, 3590, 0, 4095, 45, 0, 0, 1447, 506, 653, 0, 1, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2895, 3758, 13, 861, 0, 1, 0, 9, 4096, 3780, 1202, 527, 0, 192, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1172, 16, 0, 2920, 1298, 0, 4067, 0, 0, 3195, 60, 3, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 28, 703, 1889, 3953, 0, 0, 0, 2, 14, 191, 0, 0, 3989, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 4086, 0, 2, 88, 9, 2328, 1994, 1005, 414, 4040, 44, 891, 1608, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3624, 2314, 82, 0, 4003, 2428, 0, 0, 0, 4096, 0, 12, 2715, 1, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19750, 16105, 14724, 17453, 12773, 10818, 17040, 22409] avg=16384.0000 max=22409 max_rank=7 min=10818 min_rank=5 imbalance_ratio=1.367737 max_load_reduction=108663 max_load_reduction_pct=0.829033 moved_to_echo=63040 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[122, 14, 4055, 2, 0, 0, 0, 4043, 0, 4031, 0, 262, 600, 16, 0, 4088, 55, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3435, 0, 1, 0, 4096, 0, 1680, 0, 3812, 0, 0, 0, 269, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 122, 91, 1, 3715, 0, 0, 3620, 0, 12, 345, 4096, 1532, 0, 9, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 805, 0, 27, 0, 2179, 3699, 0, 2917, 78, 4096, 116, 4096, 51, 0, 729, 1236, 853, 284, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4, 0, 25, 0, 0, 2757, 0, 1126, 3269, 1499, 3113, 538, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 931, 0, 3937, 0, 3943, 4073, 0, 0, 1, 0, 0, 0, 5, 0, 3934, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 107, 0, 37, 0, 0, 3, 1, 1, 3492, 0, 447, 134, 3944, 1270, 1328, 35, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1423, 4075, 0, 0, 0, 918, 42, 0, 0, 1428, 2920, 3538, 0, 0, 35, 3360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18021, 13646, 13629, 21261, 13947, 20908, 11921, 17739] avg=16384.0000 max=21261 max_rank=3 min=11921 min_rank=6 imbalance_ratio=1.297668 max_load_reduction=109811 max_load_reduction_pct=0.837791 moved_to_echo=64515 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[768, 256, 23680, 512, 2304, 14976, 384, 25600, 640, 20224, 1792, 384, 11264, 128, 128, 8320, 15872, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[896, 7296, 1152, 5632, 6528, 20096, 13952, 15616, 256, 24576, 0, 256, 14336, 512, 19968, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[7424, 4736, 8320, 4096, 20352, 256, 15744, 13568, 2048, 2304, 18560, 19968, 10112, 0, 128, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[17920, 256, 24064, 3072, 26496, 4096, 128, 1152, 4096, 12160, 640, 0, 384, 7936, 9856, 19968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 3712, 23680, 10496, 128, 2432, 14720, 2560, 896, 128, 5504, 26368, 8320, 128, 7680, 896, 24320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[640, 128, 2560, 128, 8064, 5888, 384, 512, 20992, 25856, 4352, 9856, 2048, 28416, 1536, 3072, 10240, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 3456, 1408, 2304, 20096, 1408, 128, 22784, 256, 3712, 17920, 4608, 20736, 7040, 14976, 11392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 2816, 0, 4480, 128, 22784, 23808, 512, 8064, 1536, 12160, 512, 6784, 11520, 13696, 6912, 8192, 5760, 1920, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[4095, 159, 491, 2998, 0, 0, 0, 34, 906, 0, 42, 15, 0, 4052, 1517, 4096, 3091, 0, 478, 0, 2833, 0, 4, 0, 8, 0, 1228, 1209, 235, 598, 0, 2476, 0, 0, 0, 0, 14, 0, 0, 0, 3489, 614, 4080, 2, 1, 86, 2441, 0, 0, 0, 0, 3890, 1, 0, 4091, 0, 116, 685, 4053, 3998, 0, 2964, 4062, 11, 0, 19, 113, 0, 0, 4084, 3657, 29, 222, 2969, 0, 3401, 149, 122, 200, 0, 1, 4, 0, 0, 3, 4069, 0, 331, 4077, 0, 0, 17, 3921, 0, 3, 1363, 2975, 4018, 0, 4096, 0, 1739, 351, 0, 3, 0, 40, 0, 599, 71, 3625, 0, 4014, 836, 0, 0, 3937, 0, 743, 2, 50, 0, 2623, 0, 3702, 3729, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[814, 0, 0, 50, 0, 0, 1, 0, 4096, 0, 999, 854, 321, 3941, 0, 4066, 3876, 6, 0, 0, 0, 0, 0, 94, 0, 0, 4081, 2418, 0, 0, 1, 4081, 1, 229, 0, 0, 4058, 3331, 1, 0, 735, 3106, 24, 0, 0, 125, 109, 0, 2, 2, 38, 4096, 0, 11, 472, 0, 3901, 4094, 4059, 1077, 0, 4092, 0, 1, 3657, 0, 2752, 31, 1, 0, 0, 501, 0, 1, 0, 1251, 0, 0, 0, 513, 61, 254, 8, 0, 3673, 4045, 4096, 0, 2497, 4059, 48, 3480, 3891, 0, 30, 327, 180, 4096, 71, 59, 560, 8, 0, 1, 4067, 7, 0, 97, 1, 0, 0, 2, 3926, 0, 3557, 0, 4058, 16, 12, 4090, 0, 0, 385, 0, 3356, 4084, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[35, 0, 1, 3054, 0, 1932, 0, 1, 3735, 0, 0, 2219, 5, 1850, 615, 4028, 3580, 3, 0, 0, 0, 0, 0, 1, 0, 0, 3966, 4096, 0, 186, 0, 2459, 47, 0, 0, 4, 1, 2108, 0, 0, 3727, 126, 0, 1, 0, 0, 1, 12, 151, 0, 3998, 2110, 61, 0, 4096, 0, 3931, 0, 4048, 3895, 19, 1, 2531, 25, 4048, 66, 3407, 0, 38, 510, 0, 287, 1284, 139, 48, 0, 2, 0, 0, 0, 0, 16, 1795, 0, 0, 2566, 4096, 1, 4072, 0, 693, 3281, 0, 0, 0, 865, 0, 3654, 55, 4096, 0, 2295, 7, 1, 1068, 22, 7, 0, 4094, 4, 0, 4092, 93, 184, 4022, 0, 4096, 0, 2665, 0, 0, 36, 0, 3565, 3049, 4094, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 7, 0, 3949, 234, 0, 0, 100, 4088, 0, 1, 2253, 0, 3968, 1, 3808, 4054, 1934, 255, 0, 0, 10, 44, 0, 0, 6, 4096, 4092, 0, 4073, 0, 2215, 0, 1, 2306, 1, 4057, 2709, 0, 450, 4087, 31, 0, 25, 1, 288, 0, 18, 933, 1, 0, 4063, 0, 21, 32, 0, 4095, 4072, 4073, 3349, 829, 42, 2, 0, 4084, 0, 1, 2787, 1, 0, 0, 10, 12, 0, 18, 3055, 0, 0, 0, 0, 10, 36, 0, 0, 442, 4055, 0, 146, 46, 0, 250, 91, 2689, 112, 0, 103, 4056, 1951, 0, 176, 1058, 4096, 252, 0, 0, 698, 78, 3, 4027, 0, 3, 0, 0, 4056, 22, 8, 3295, 0, 4085, 0, 0, 0, 3600, 0, 2891, 4095, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[6, 0, 4071, 3497, 132, 0, 1, 435, 410, 0, 13, 27, 116, 1879, 0, 4057, 4092, 0, 82, 122, 0, 0, 0, 0, 16, 0, 4094, 4096, 3, 744, 0, 4063, 0, 788, 522, 0, 3837, 1836, 0, 0, 4006, 4021, 2, 0, 5, 3741, 1, 23, 1, 4090, 2, 2803, 0, 65, 603, 65, 3587, 4096, 3763, 3260, 6, 0, 898, 0, 0, 4, 978, 22, 1945, 188, 0, 5, 0, 434, 0, 4046, 89, 0, 2, 929, 0, 0, 134, 19, 3992, 471, 0, 1, 0, 0, 10, 2078, 4096, 2, 0, 0, 4096, 4073, 1, 218, 67, 1, 2458, 46, 4096, 1660, 0, 1303, 3461, 2, 2, 4090, 0, 0, 1, 0, 4096, 0, 44, 3241, 12, 0, 1055, 0, 3627, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 3, 320, 0, 0, 0, 270, 3678, 0, 0, 100, 0, 4066, 1368, 4095, 4095, 1, 0, 0, 366, 0, 0, 0, 0, 36, 2729, 4093, 0, 8, 0, 2828, 0, 0, 0, 4096, 0, 4096, 0, 57, 4027, 1071, 0, 20, 0, 3904, 395, 3308, 17, 5, 0, 4096, 0, 0, 2, 1056, 4066, 0, 4096, 4096, 0, 3500, 15, 1, 4096, 0, 0, 31, 475, 0, 811, 0, 112, 270, 0, 427, 0, 0, 0, 2246, 68, 0, 0, 1, 53, 4056, 4089, 1, 21, 0, 12, 3971, 3988, 0, 201, 0, 0, 3981, 1223, 4096, 0, 3849, 772, 0, 1, 3946, 0, 3, 4096, 0, 0, 0, 0, 137, 0, 0, 0, 1613, 4095, 0, 0, 0, 1549, 39, 2806, 3957, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4086, 1, 0, 0, 19, 0, 792, 0, 0, 0, 0, 0, 0, 1152, 9, 4056, 0, 16, 0, 872, 50, 2, 103, 0, 1, 4084, 3314, 3723, 17, 0, 0, 0, 1, 0, 37, 0, 720, 3896, 516, 319, 66, 0, 12, 0, 35, 4095, 3984, 18, 2, 0, 0, 3831, 0, 0, 2153, 2416, 0, 2030, 4059, 3346, 0, 0, 0, 0, 3039, 201, 0, 12, 3960, 0, 0, 676, 32, 0, 1, 4095, 0, 0, 2698, 256, 4077, 214, 0, 0, 3953, 0, 36, 4020, 1679, 162, 1, 3227, 4086, 0, 2, 6, 4077, 348, 0, 3903, 0, 4096, 0, 1912, 0, 0, 0, 1696, 14, 123, 0, 0, 3983, 4096, 4011, 0, 3895, 0, 4085, 1, 0, 0, 0, 0, 428, 4096, 0, 42] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[3773, 0, 437, 3967, 0, 35, 0, 0, 1781, 0, 0, 2263, 1, 1989, 1113, 3641, 3181, 4, 0, 1, 112, 5, 0, 65, 0, 0, 3914, 3836, 7, 351, 0, 4048, 0, 0, 0, 1022, 3850, 8, 0, 4080, 4093, 0, 0, 2, 0, 0, 188, 79, 9, 1802, 0, 3619, 0, 133, 3983, 42, 24, 4096, 3609, 1726, 0, 4083, 0, 0, 3980, 0, 0, 0, 1, 0, 3610, 0, 0, 815, 3, 1815, 0, 0, 297, 419, 52, 0, 239, 0, 3855, 4096, 2566, 3, 103, 47, 0, 4089, 32, 0, 0, 0, 3952, 4026, 4096, 4034, 115, 0, 6, 0, 65, 662, 2540, 0, 2088, 0, 0, 101, 0, 4096, 82, 0, 38, 2, 251, 1913, 1, 3, 178, 74, 1604, 4094, 50, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4095, 159, 491, 2998, 0, 0, 0, 34, 906, 0, 42, 15, 0, 4052, 1517, 4096, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3091, 0, 478, 0, 2833, 0, 4, 0, 8, 0, 1228, 1209, 235, 598, 0, 2476, 1199, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 3489, 614, 4080, 2, 1, 86, 2441, 0, 2376, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1925, 1, 0, 4091, 0, 116, 685, 0, 3998, 0, 2964, 4062, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 113, 0, 0, 4084, 3657, 29, 222, 2969, 0, 3401, 149, 122, 200, 0, 4053, 1698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 3, 4069, 0, 331, 4077, 0, 0, 17, 3921, 0, 3, 1363, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2975, 1440, 0, 4096, 0, 1739, 351, 0, 3, 0, 40, 0, 599, 71, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 836, 0, 0, 3937, 0, 743, 2, 50, 0, 2623, 0, 3702, 1186, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18948, 13561, 13948, 17853, 20716, 14012, 14939, 17095] avg=16384.0000 max=20716 max_rank=4 min=13561 min_rank=1 imbalance_ratio=1.264404 max_load_reduction=110356 max_load_reduction_pct=0.841949 moved_to_echo=66762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[814, 0, 0, 50, 0, 0, 1, 0, 4096, 0, 999, 854, 321, 3941, 0, 4066, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3876, 6, 0, 0, 0, 0, 0, 94, 0, 0, 4081, 2418, 0, 0, 1, 4081, 1263, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 229, 0, 0, 4058, 3331, 1, 0, 735, 3106, 24, 0, 0, 125, 109, 0, 2422, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 38, 2026, 0, 11, 472, 0, 3901, 4094, 0, 1077, 0, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3657, 0, 2752, 31, 1, 0, 0, 501, 0, 1, 0, 1251, 0, 0, 0, 513, 4059, 1860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 254, 8, 0, 3673, 4045, 4096, 0, 2497, 4059, 48, 3480, 3891, 0, 30, 327, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 1468, 71, 59, 560, 8, 0, 1, 4067, 7, 0, 97, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 0, 3557, 0, 4058, 16, 12, 4090, 0, 0, 385, 0, 3356, 1298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15714, 16026, 15067, 15716, 14626, 26704, 6521, 20698] avg=16384.0000 max=26704 max_rank=5 min=6521 min_rank=6 imbalance_ratio=1.629883 max_load_reduction=104368 max_load_reduction_pct=0.796265 moved_to_echo=68549 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[35, 0, 1, 3054, 0, 1932, 0, 1, 3735, 0, 0, 2219, 5, 1850, 615, 4028, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3580, 3, 0, 0, 0, 0, 0, 1, 0, 0, 3966, 4096, 0, 186, 0, 2459, 650, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 4, 1, 2108, 0, 0, 3727, 126, 0, 1, 0, 0, 1, 12, 2160, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 3998, 1045, 61, 0, 4096, 0, 3931, 0, 0, 3895, 19, 1, 2531, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 66, 3407, 0, 38, 510, 0, 287, 1284, 139, 48, 0, 2, 0, 0, 0, 4048, 1864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1795, 0, 0, 2566, 4096, 1, 4072, 0, 693, 3281, 0, 0, 0, 865, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1310, 55, 4096, 0, 2295, 7, 1, 1068, 22, 7, 0, 4094, 4, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 184, 4022, 0, 4096, 0, 2665, 0, 0, 36, 0, 3565, 3049, 1302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17769, 15125, 9115, 19753, 15741, 17506, 17051, 19012] avg=16384.0000 max=19753 max_rank=3 min=9115 min_rank=2 imbalance_ratio=1.205627 max_load_reduction=111319 max_load_reduction_pct=0.849297 moved_to_echo=69310 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 3949, 234, 0, 0, 100, 4088, 0, 1, 2253, 0, 3968, 1, 3808, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 1934, 255, 0, 0, 10, 44, 0, 0, 6, 4096, 4092, 0, 4073, 0, 2215, 1253, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2306, 1, 4057, 2709, 0, 450, 4087, 31, 0, 25, 1, 288, 0, 18, 1153, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 933, 1, 0, 2010, 0, 21, 32, 0, 4095, 4072, 0, 3349, 829, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 1, 2787, 1, 0, 0, 10, 12, 0, 18, 3055, 0, 0, 0, 0, 4073, 1865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 36, 0, 0, 442, 4055, 0, 146, 46, 0, 250, 91, 2689, 112, 0, 103, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 700, 0, 176, 1058, 4096, 252, 0, 0, 698, 78, 3, 4027, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 22, 8, 3295, 0, 4085, 0, 0, 0, 3600, 0, 2891, 1302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18976, 22130, 16055, 15386, 15906, 8213, 15147, 19259] avg=16384.0000 max=22130 max_rank=1 min=8213 min_rank=5 imbalance_ratio=1.350708 max_load_reduction=108942 max_load_reduction_pct=0.831161 moved_to_echo=58525 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 320, 0, 0, 0, 270, 3678, 0, 0, 100, 0, 4066, 1368, 4095, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 366, 0, 0, 0, 0, 36, 2729, 4093, 0, 8, 0, 2828, 1266, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 57, 4027, 1071, 0, 20, 0, 3904, 395, 3308, 2359, 901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 5, 0, 2016, 0, 0, 2, 1056, 4066, 0, 0, 4096, 0, 3500, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 31, 475, 0, 811, 0, 112, 270, 0, 427, 0, 0, 0, 2246, 4096, 1807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 1, 53, 4056, 4089, 1, 21, 0, 12, 3971, 3988, 0, 201, 0, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1418, 1223, 4096, 0, 3849, 772, 0, 1, 3946, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 1613, 4095, 0, 0, 0, 1549, 39, 2806, 1249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14475, 15626, 24234, 14774, 14371, 16700, 19404, 11488] avg=16384.0000 max=24234 max_rank=2 min=11488 min_rank=7 imbalance_ratio=1.479126 max_load_reduction=106838 max_load_reduction_pct=0.815109 moved_to_echo=61963 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 4071, 3497, 132, 0, 1, 435, 410, 0, 13, 27, 116, 1879, 0, 4057, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 82, 122, 0, 0, 0, 0, 16, 0, 4094, 4096, 3, 744, 0, 4063, 864, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 788, 522, 0, 3837, 1836, 0, 0, 4006, 4021, 2, 0, 5, 3741, 1, 23, 2408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 2, 1387, 0, 65, 603, 65, 3587, 4096, 0, 3260, 6, 0, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 978, 22, 1945, 188, 0, 5, 0, 434, 0, 4046, 89, 0, 2, 929, 3763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 19, 3992, 471, 0, 1, 0, 0, 10, 2078, 4096, 2, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1460, 1, 218, 67, 1, 2458, 46, 4096, 1660, 0, 1303, 3461, 2, 2, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 44, 3241, 12, 0, 1055, 0, 3627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15035, 18381, 21190, 18060, 12405, 10964, 22961, 12076] avg=16384.0000 max=22961 max_rank=6 min=10964 min_rank=5 imbalance_ratio=1.401428 max_load_reduction=108111 max_load_reduction_pct=0.824821 moved_to_echo=58406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4086, 1, 0, 0, 19, 0, 792, 0, 0, 0, 0, 0, 0, 1152, 9, 4056, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 872, 50, 2, 103, 0, 1, 4084, 3314, 3723, 17, 0, 0, 0, 1181, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 37, 0, 720, 3896, 516, 319, 66, 0, 12, 0, 35, 4095, 3984, 18, 205, 929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1895, 0, 0, 2153, 2416, 0, 2030, 0, 3346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3039, 201, 0, 12, 3960, 0, 0, 676, 32, 0, 1, 4095, 0, 0, 2698, 256, 4059, 1865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 214, 0, 0, 3953, 0, 36, 4020, 1679, 162, 1, 3227, 4086, 0, 2, 6, 220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 126, 0, 3903, 0, 4096, 0, 1912, 0, 0, 0, 1696, 14, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 4096, 4011, 0, 3895, 0, 4085, 1, 0, 0, 0, 0, 428, 1302, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10650, 13380, 14833, 11842, 20894, 21683, 15947, 21843] avg=16384.0000 max=21843 max_rank=7 min=10650 min_rank=0 imbalance_ratio=1.333191 max_load_reduction=109229 max_load_reduction_pct=0.833351 moved_to_echo=80367 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3773, 0, 437, 3967, 0, 35, 0, 0, 1781, 0, 0, 2263, 1, 1989, 1113, 3641, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3181, 4, 0, 1, 112, 5, 0, 65, 0, 0, 3914, 3836, 7, 351, 0, 4048, 1116, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 3850, 8, 0, 4080, 4093, 0, 0, 2, 0, 0, 188, 79, 2380, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1802, 0, 1790, 0, 133, 3983, 42, 24, 4096, 0, 1726, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 0, 0, 1, 0, 3610, 0, 0, 815, 3, 1815, 0, 0, 297, 419, 3609, 1864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 239, 0, 3855, 4096, 2566, 3, 103, 47, 0, 4089, 32, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 1443, 4096, 4034, 115, 0, 6, 0, 65, 662, 2540, 0, 2088, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 82, 0, 38, 2, 251, 1913, 1, 3, 178, 74, 1604, 1302, 50, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19505, 16843, 16630, 17688, 16413, 15290, 19102, 9601] avg=16384.0000 max=19505 max_rank=0 min=9601 min_rank=7 imbalance_ratio=1.190491 max_load_reduction=111567 max_load_reduction_pct=0.851189 moved_to_echo=60406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[19456, 9472, 5504, 20736, 1920, 16128, 3968, 2048, 9344, 7040, 2688, 3200, 18432, 256, 3712, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[12928, 256, 5120, 17920, 512, 2048, 896, 896, 18816, 0, 1152, 7808, 512, 22912, 4736, 31872, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[12032, 13440, 11776, 128, 23424, 1664, 16000, 9344, 128, 128, 9472, 3712, 21504, 8960, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[22912, 384, 7296, 2944, 6528, 4864, 8192, 1536, 1664, 4736, 128, 18176, 256, 128, 3200, 4480, 31872, 12928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[4352, 640, 2176, 128, 16000, 23424, 14976, 4608, 12544, 4352, 1024, 20352, 22784, 128, 256, 2688, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[25984, 2048, 896, 1024, 3456, 128, 256, 256, 128, 4224, 27520, 27648, 384, 6016, 128, 22272, 8832, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1152, 6016, 4096, 14208, 128, 256, 15488, 3584, 19840, 19200, 0, 24832, 896, 14720, 7552, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 1024, 2944, 5248, 16640, 18048, 640, 4992, 24320, 9088, 4224, 128, 128, 12288, 7168, 3584, 15488, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 475, 0, 0, 0, 4093, 6, 3748, 3954, 0, 0, 3665, 1708, 0, 914, 2, 30, 0, 0, 0, 4036, 0, 0, 130, 3074, 145, 1700, 1384, 27, 435, 0, 3, 23, 0, 1, 0, 0, 4087, 523, 2, 3003, 0, 2, 0, 4095, 4094, 20, 0, 1, 0, 0, 1, 0, 0, 0, 8, 0, 0, 20, 4, 0, 0, 0, 0, 1469, 771, 306, 0, 3945, 48, 0, 0, 855, 4096, 12, 1, 4029, 0, 22, 4003, 3326, 1545, 34, 4075, 4096, 3888, 4041, 4037, 3, 3283, 0, 0, 0, 0, 222, 17, 3152, 1, 0, 77, 1, 3183, 4084, 9, 2, 661, 157, 3, 2, 212, 734, 4073, 3712, 122, 803, 4091, 4096, 0, 255, 44, 4091, 0, 0, 3953, 0, 15, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4055, 0, 0, 0, 0, 4055, 167, 4036, 916, 4093, 4094, 0, 0, 451, 0, 1021, 0, 3505, 11, 270, 0, 4091, 0, 152, 0, 4027, 0, 14, 0, 3995, 0, 2, 0, 0, 7, 0, 0, 2502, 4091, 3059, 0, 94, 4094, 4093, 0, 3972, 4077, 2596, 0, 0, 0, 12, 0, 0, 0, 0, 7, 956, 3919, 1, 245, 0, 0, 0, 5, 1, 10, 0, 0, 3494, 0, 849, 531, 0, 3025, 0, 0, 3884, 0, 3937, 0, 0, 0, 3996, 58, 162, 1036, 2456, 91, 46, 0, 0, 0, 0, 12, 4089, 0, 0, 0, 1950, 0, 184, 3, 4096, 3311, 0, 0, 0, 694, 16, 3211, 4063, 1554, 1, 0, 0, 3819, 0, 143, 0, 0, 3687, 0, 223, 14, 37, 3703, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[58, 0, 1, 0, 0, 489, 1750, 4096, 3593, 4093, 0, 0, 16, 3, 4096, 0, 0, 199, 0, 138, 0, 66, 0, 0, 4, 3206, 0, 3946, 14, 3631, 0, 0, 411, 1, 0, 756, 8, 0, 0, 337, 3581, 0, 4076, 4094, 0, 4081, 3451, 584, 0, 0, 6, 0, 0, 0, 4078, 0, 27, 0, 1, 0, 184, 0, 0, 0, 4000, 3417, 0, 6, 156, 4096, 0, 0, 0, 4029, 4089, 0, 14, 3797, 0, 0, 14, 0, 0, 1, 0, 0, 3959, 0, 3890, 14, 3988, 0, 0, 0, 0, 4096, 49, 0, 1, 1744, 2, 3850, 0, 4096, 2, 0, 1724, 0, 3792, 0, 0, 3827, 0, 922, 2444, 254, 4090, 1453, 4043, 0, 0, 0, 0, 120, 4018, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 82, 3298, 4054, 4096, 4090, 6, 0, 0, 3301, 2, 10, 4050, 2846, 0, 0, 0, 0, 195, 0, 3979, 4079, 0, 2790, 0, 4096, 0, 0, 0, 340, 2, 0, 4079, 46, 4008, 0, 4044, 0, 0, 667, 3813, 1907, 4096, 3798, 20, 0, 0, 22, 0, 0, 1279, 7, 0, 4094, 4092, 0, 0, 0, 33, 0, 41, 558, 0, 0, 106, 3748, 0, 0, 0, 4060, 4050, 0, 1, 3611, 0, 3205, 0, 0, 0, 1491, 2033, 0, 1127, 0, 0, 0, 2162, 2, 488, 0, 0, 5, 0, 3, 0, 4096, 0, 0, 41, 3910, 265, 0, 4075, 0, 968, 0, 0, 1084, 0, 3241, 85, 82, 4094, 0, 1815, 0, 0, 3220, 0, 0, 0, 82, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 820, 38, 0, 0, 3631, 3965, 2657, 0, 0, 178, 0, 3471, 0, 4089, 1, 404, 103, 7, 0, 0, 0, 0, 0, 4096, 0, 9, 5, 2959, 3585, 0, 0, 0, 38, 0, 4056, 4004, 4037, 1882, 9, 326, 0, 4088, 4096, 48, 4018, 3578, 0, 0, 2, 28, 4, 701, 2, 0, 146, 375, 4062, 9, 0, 0, 0, 0, 0, 2, 0, 4096, 2, 0, 428, 167, 2, 0, 2, 3963, 0, 23, 4094, 15, 296, 11, 3000, 142, 2841, 20, 4084, 314, 2339, 3862, 0, 156, 0, 1, 0, 0, 1142, 0, 36, 0, 3676, 0, 2, 0, 4045, 76, 0, 1003, 1230, 5, 0, 3243, 2015, 0, 0, 3654, 79, 4067, 0, 0, 0, 0, 0, 4093, 0, 4000, 0, 2949, 370] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4093, 145, 3780, 0, 332, 2387, 3393, 4067, 4058, 247, 2, 0, 49, 2, 2, 9, 0, 2698, 0, 0, 0, 71, 4096, 48, 3963, 4067, 884, 5, 0, 7, 0, 0, 0, 46, 0, 4094, 0, 0, 3680, 18, 4096, 2217, 3024, 4004, 3696, 4096, 4041, 1, 1, 12, 0, 0, 0, 0, 418, 0, 0, 145, 4032, 178, 0, 0, 205, 0, 4077, 0, 0, 256, 0, 0, 0, 0, 19, 0, 0, 944, 16, 4096, 0, 0, 133, 0, 0, 0, 0, 1674, 3818, 2751, 0, 0, 2333, 184, 0, 0, 55, 3826, 19, 0, 31, 3053, 0, 436, 0, 4058, 4096, 0, 0, 868, 33, 2037, 0, 1093, 77, 0, 1, 0, 3762, 0, 1, 0, 0, 1605, 4, 1224, 3993, 0, 4090, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[221, 0, 725, 3818, 0, 3932, 2606, 0, 4094, 4021, 15, 731, 3344, 3808, 0, 2, 0, 3813, 857, 7, 0, 0, 0, 0, 4094, 3660, 4, 0, 70, 3864, 35, 0, 237, 0, 1408, 4096, 0, 1303, 2105, 0, 3776, 94, 0, 3122, 0, 4096, 4079, 586, 0, 5, 0, 0, 2, 1, 0, 1, 27, 82, 11, 0, 1, 0, 0, 433, 1989, 2, 23, 0, 0, 3313, 231, 0, 0, 2784, 4096, 24, 1, 4018, 0, 3124, 0, 4096, 152, 2865, 50, 0, 0, 1161, 17, 0, 4028, 0, 0, 3927, 734, 0, 231, 0, 0, 2814, 0, 50, 0, 4096, 0, 1718, 4082, 0, 0, 14, 4093, 6, 0, 0, 64, 3728, 4088, 262, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 91, 131, 4096, 3250, 4096, 140, 0, 0, 0, 35, 0, 0, 0, 3996, 0, 0, 33, 4095, 0, 0, 2248, 4093, 27, 114, 0, 4093, 0, 0, 336, 161, 4, 1808, 0, 2842, 1181, 0, 0, 1, 3459, 3953, 16, 4030, 4069, 3321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3573, 262, 9, 0, 0, 0, 28, 3934, 0, 0, 0, 841, 4071, 0, 0, 48, 3949, 4068, 1, 15, 4058, 426, 4033, 2, 1240, 0, 3875, 3801, 127, 3980, 908, 3716, 66, 2957, 0, 0, 0, 0, 1223, 958, 262, 0, 886, 0, 7, 28, 4096, 0, 0, 949, 0, 0, 0, 34, 29, 0, 3932, 48, 0, 4043, 4096, 1236, 48, 0, 2629, 0, 708, 23, 42, 0, 88] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 820, 38, 0, 0, 3631, 3374, 2657, 0, 0, 178, 0, 3471, 0, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 404, 103, 7, 0, 0, 0, 0, 0, 4096, 0, 9, 5, 2959, 3585, 0, 0, 67, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 4056, 4004, 4037, 1882, 9, 326, 0, 4088, 1874, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 28, 4, 701, 2, 0, 146, 375, 4062, 9, 0, 0, 0, 0, 0, 3578, 4018, 0, 1861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 2, 0, 428, 167, 2, 0, 2, 3963, 0, 23, 2702, 15, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3000, 142, 2841, 20, 4084, 314, 2339, 3862, 0, 88, 0, 1, 0, 0, 1142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 3676, 0, 2, 0, 4045, 76, 0, 1003, 1230, 5, 0, 3243, 2015, 1320, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3654, 79, 4067, 0, 0, 0, 0, 0, 4093, 0, 4000, 0, 2949, 370, 591, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18259, 11307, 20362, 14786, 11698, 17844, 17012, 19804] avg=16384.0000 max=20362 max_rank=2 min=11307 min_rank=1 imbalance_ratio=1.242798 max_load_reduction=110710 max_load_reduction_pct=0.844650 moved_to_echo=66358 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[58, 0, 1, 0, 0, 489, 1485, 4096, 0, 4093, 0, 0, 16, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 0, 138, 0, 66, 0, 0, 4, 3206, 0, 3946, 14, 3631, 0, 0, 1718, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 411, 1, 0, 756, 8, 0, 0, 337, 3581, 0, 4076, 1865, 0, 0, 0, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4078, 0, 27, 0, 1, 0, 184, 0, 0, 0, 3451, 4081, 3593, 1863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 3417, 0, 6, 156, 4096, 0, 0, 0, 4029, 4089, 0, 14, 2497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1, 0, 0, 3959, 0, 3890, 14, 2231, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 1, 1744, 2, 3850, 0, 4096, 2, 0, 1724, 0, 3792, 0, 0, 3827, 1229, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 922, 2444, 254, 4090, 1453, 4043, 0, 0, 0, 0, 120, 4018, 0, 0, 0, 265, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14337, 12993, 11619, 17284, 22304, 14205, 20682, 17648] avg=16384.0000 max=22304 max_rank=4 min=11619 min_rank=2 imbalance_ratio=1.361328 max_load_reduction=108768 max_load_reduction_pct=0.829834 moved_to_echo=74839 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4055, 0, 0, 0, 0, 4055, 143, 4036, 0, 4093, 4094, 0, 0, 451, 0, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3505, 11, 270, 0, 4091, 0, 152, 0, 4027, 0, 14, 0, 3995, 0, 2, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2502, 4091, 3059, 0, 94, 4094, 1873, 0, 0, 0, 2596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 7, 956, 3919, 1, 245, 0, 0, 0, 4077, 3972, 916, 1859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 10, 0, 0, 3494, 0, 849, 531, 0, 3025, 0, 0, 2564, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 58, 162, 1036, 2456, 91, 46, 0, 0, 0, 0, 12, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1950, 0, 184, 3, 4096, 3311, 0, 0, 0, 694, 16, 3211, 4063, 1252, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1554, 1, 0, 0, 3819, 0, 143, 0, 0, 3687, 0, 223, 14, 37, 3703, 1, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21948, 16135, 18316, 15964, 14416, 11946, 19141, 13206] avg=16384.0000 max=21948 max_rank=0 min=11946 min_rank=5 imbalance_ratio=1.339600 max_load_reduction=109124 max_load_reduction_pct=0.832550 moved_to_echo=58709 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 82, 2807, 4054, 0, 4090, 6, 0, 0, 3301, 2, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 2846, 0, 0, 0, 0, 195, 0, 3979, 4079, 0, 2790, 0, 4096, 0, 0, 930, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 2, 0, 4079, 46, 4008, 0, 4044, 0, 0, 306, 3813, 0, 0, 3798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 22, 0, 0, 1279, 7, 0, 4094, 4092, 0, 0, 0, 33, 0, 4096, 1907, 4096, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 558, 0, 0, 106, 3748, 0, 0, 0, 4060, 4050, 0, 1, 2383, 0, 3205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1491, 2033, 0, 1127, 0, 0, 0, 1212, 2, 488, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 41, 3910, 265, 0, 4075, 0, 968, 0, 0, 1084, 1164, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3241, 85, 82, 4094, 0, 1815, 0, 0, 3220, 0, 0, 0, 82, 0, 0, 491, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14354, 23029, 20436, 19949, 18152, 6358, 15664, 13130] avg=16384.0000 max=23029 max_rank=1 min=6358 min_rank=5 imbalance_ratio=1.405579 max_load_reduction=108043 max_load_reduction_pct=0.824303 moved_to_echo=53304 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 475, 0, 0, 0, 3483, 6, 0, 3954, 0, 0, 3665, 1708, 0, 914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 30, 0, 0, 0, 4036, 0, 0, 130, 3074, 145, 1700, 1384, 27, 435, 0, 1412, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 23, 0, 1, 0, 0, 4087, 523, 2, 3003, 0, 2, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 8, 0, 0, 20, 4, 0, 0, 0, 4094, 4095, 3748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1469, 771, 306, 0, 3945, 48, 0, 0, 855, 4096, 12, 1, 2659, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 3326, 1545, 34, 4075, 4096, 3888, 4041, 4037, 3, 1841, 0, 0, 0, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3152, 1, 0, 77, 1, 3183, 4084, 9, 2, 661, 157, 3, 2, 212, 734, 1299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 3712, 122, 803, 4091, 4096, 0, 255, 44, 4091, 0, 0, 3953, 0, 15, 2, 610, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14205, 12446, 7664, 11971, 14184, 31111, 13594, 25897] avg=16384.0000 max=31111 max_rank=5 min=7664 min_rank=2 imbalance_ratio=1.898865 max_load_reduction=99961 max_load_reduction_pct=0.762642 moved_to_echo=84786 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4093, 145, 3780, 0, 332, 2387, 2887, 4067, 0, 247, 2, 0, 49, 2, 2, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2698, 0, 0, 0, 71, 4096, 48, 3963, 4067, 884, 5, 0, 7, 0, 0, 1003, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 4094, 0, 0, 3680, 18, 4096, 2217, 3024, 1832, 3696, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0, 0, 0, 0, 418, 0, 0, 145, 4032, 178, 0, 0, 205, 0, 4041, 4096, 4058, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 256, 0, 0, 0, 0, 19, 0, 0, 944, 16, 2703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 1674, 3818, 2751, 0, 0, 1309, 184, 0, 0, 55, 3826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 31, 3053, 0, 436, 0, 4058, 4096, 0, 0, 868, 33, 2037, 0, 1093, 1321, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 1, 0, 3762, 0, 1, 0, 0, 1605, 4, 1224, 3993, 0, 4090, 0, 506, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18002, 16914, 22704, 19005, 8015, 13750, 17398, 15284] avg=16384.0000 max=22704 max_rank=2 min=8015 min_rank=4 imbalance_ratio=1.385742 max_load_reduction=108368 max_load_reduction_pct=0.826782 moved_to_echo=54447 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[221, 0, 725, 3818, 0, 3932, 2218, 0, 0, 4021, 15, 731, 3344, 3808, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3813, 857, 7, 0, 0, 0, 0, 4094, 3660, 4, 0, 70, 3864, 35, 0, 1732, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 1408, 4096, 0, 1303, 2105, 0, 3776, 94, 0, 1429, 0, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2, 1, 0, 1, 27, 82, 11, 0, 1, 0, 0, 433, 4079, 4096, 4094, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1989, 2, 23, 0, 0, 3313, 231, 0, 0, 2784, 4096, 24, 1, 2651, 0, 3124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 152, 2865, 50, 0, 0, 1161, 17, 0, 2259, 0, 0, 3927, 734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 2814, 0, 50, 0, 4096, 0, 1718, 4082, 0, 0, 14, 4093, 6, 1296, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 3728, 4088, 262, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 388, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22835, 18207, 15034, 14250, 18238, 15261, 18675, 8572] avg=16384.0000 max=22835 max_rank=0 min=8572 min_rank=7 imbalance_ratio=1.393738 max_load_reduction=108237 max_load_reduction_pct=0.825783 moved_to_echo=60746 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 91, 131, 3485, 3250, 0, 140, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 33, 4095, 0, 0, 2248, 4093, 27, 114, 0, 4093, 0, 0, 1271, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 161, 4, 1808, 0, 2842, 1181, 0, 0, 1, 3459, 1808, 16, 0, 0, 3321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3573, 262, 9, 0, 0, 0, 28, 4069, 4030, 4096, 1796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 0, 0, 0, 841, 4071, 0, 0, 48, 3949, 4068, 1, 15, 2679, 426, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1240, 0, 3875, 3801, 127, 3980, 908, 3716, 66, 1659, 0, 0, 0, 0, 1223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 958, 262, 0, 886, 0, 7, 28, 4096, 0, 0, 949, 0, 0, 0, 34, 29, 1308, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 48, 0, 4043, 4096, 1236, 48, 0, 2629, 0, 708, 23, 42, 0, 88, 611, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7132, 20041, 14937, 17863, 24065, 20597, 8906, 17531] avg=16384.0000 max=24065 max_rank=4 min=7132 min_rank=0 imbalance_ratio=1.468811 max_load_reduction=107007 max_load_reduction_pct=0.816399 moved_to_echo=71099 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[8448, 1024, 5120, 3840, 512, 14720, 19968, 22272, 0, 20736, 4352, 768, 10624, 9344, 8192, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[4224, 11776, 1920, 15104, 10112, 10240, 18176, 13696, 15616, 256, 10624, 256, 512, 3968, 896, 14720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1280, 3456, 128, 18304, 128, 4608, 3328, 32512, 7808, 1792, 12544, 2304, 5504, 2176, 10880, 12928, 10240, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[5760, 11904, 6528, 4992, 32128, 9984, 7296, 384, 128, 15232, 4224, 2304, 16128, 256, 10880, 512, 3584, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[4480, 17280, 896, 512, 128, 12416, 4352, 256, 18560, 26240, 1152, 8576, 4480, 23424, 512, 128, 8192, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 768, 128, 5888, 256, 512, 12928, 12416, 256, 512, 0, 256, 512, 31488, 30336, 24704, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1024, 640, 1536, 14848, 8192, 10752, 21120, 3968, 15872, 5504, 18816, 11008, 7680, 0, 0, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[14080, 5504, 4992, 640, 1152, 23168, 512, 896, 640, 15744, 27392, 1024, 128, 20864, 512, 14720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[2, 56, 0, 14, 1825, 2, 0, 3765, 4096, 4086, 0, 3998, 3512, 0, 0, 2, 18, 39, 0, 3978, 1, 0, 0, 1596, 0, 3, 169, 0, 3189, 0, 0, 0, 0, 0, 0, 70, 0, 0, 4075, 0, 4066, 49, 5, 3978, 39, 23, 0, 0, 0, 0, 3162, 0, 0, 0, 6, 107, 0, 0, 0, 0, 218, 77, 3953, 3817, 0, 4049, 0, 31, 966, 0, 428, 0, 0, 55, 0, 0, 60, 4092, 3, 39, 0, 702, 3799, 0, 0, 3381, 2023, 3952, 972, 225, 0, 0, 0, 0, 15, 0, 4090, 3893, 303, 12, 2, 0, 3579, 0, 11, 4096, 0, 0, 3857, 0, 3101, 38, 4096, 3767, 4052, 4086, 1, 0, 4069, 61, 4096, 0, 761, 4025, 200, 0, 88, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 22, 3132, 4082, 3493, 4096, 2435, 0, 3996, 3826, 412, 0, 0, 0, 103, 241, 0, 9, 148, 806, 4053, 0, 0, 3296, 0, 2864, 1850, 4096, 2411, 0, 0, 0, 0, 0, 0, 99, 20, 1653, 0, 0, 3623, 428, 0, 0, 23, 0, 1, 0, 3, 2, 302, 3858, 4002, 1600, 4, 0, 0, 0, 0, 4073, 0, 1162, 61, 0, 76, 13, 8, 0, 0, 1, 254, 0, 0, 14, 1177, 0, 0, 0, 175, 4039, 4095, 4054, 0, 259, 4085, 34, 3873, 3538, 101, 0, 0, 548, 13, 1471, 4078, 880, 0, 10, 0, 3343, 4, 27, 28, 0, 0, 43, 0, 0, 0, 4088, 336, 4096, 4087, 0, 0, 4073, 7, 3775, 0, 4096, 39, 3935, 10, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[398, 0, 459, 1, 551, 0, 4051, 0, 0, 0, 22, 2968, 3307, 23, 1, 0, 236, 20, 0, 4042, 0, 1556, 124, 3925, 4002, 0, 4096, 0, 6, 1, 0, 907, 21, 0, 0, 4, 0, 0, 3837, 0, 155, 0, 4095, 118, 0, 79, 33, 9, 4088, 0, 0, 5, 9, 0, 3950, 4096, 0, 0, 244, 147, 0, 0, 4040, 3294, 0, 0, 23, 54, 0, 0, 65, 937, 0, 15, 0, 0, 46, 3740, 1600, 0, 0, 3912, 191, 0, 4095, 0, 1, 3850, 12, 0, 85, 0, 0, 0, 0, 33, 0, 3715, 0, 330, 80, 3124, 4095, 1, 0, 984, 1048, 0, 2522, 10, 3, 0, 3987, 3440, 4096, 0, 4096, 0, 4068, 852, 2, 0, 4096, 4093, 4093, 708, 0, 4050] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[3206, 0, 0, 1, 0, 0, 0, 0, 0, 3825, 0, 9, 3644, 4041, 3, 0, 3206, 0, 0, 0, 118, 0, 0, 0, 4055, 626, 504, 0, 3967, 223, 0, 3987, 4062, 0, 0, 0, 0, 0, 13, 0, 3016, 0, 80, 4094, 0, 0, 1, 0, 3907, 0, 0, 6, 0, 0, 1, 407, 940, 12, 0, 1, 0, 0, 4096, 0, 4096, 0, 1, 4095, 4074, 0, 0, 0, 0, 0, 5, 1, 0, 3910, 0, 4005, 0, 0, 1619, 2, 4095, 3124, 4094, 4096, 0, 1092, 0, 0, 0, 0, 1094, 65, 0, 4095, 22, 4095, 0, 48, 0, 7, 0, 3148, 25, 2572, 1395, 0, 46, 0, 4044, 0, 1619, 1, 2634, 0, 4085, 0, 4096, 0, 4096, 218, 382, 0, 176, 2749] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 1643, 7, 0, 603, 0, 362, 492, 0, 3682, 2649, 4042, 4096, 3801, 250, 0, 0, 3978, 61, 72, 0, 3197, 288, 2911, 4072, 0, 3059, 0, 4078, 0, 0, 1840, 0, 0, 1, 0, 0, 3, 2116, 688, 0, 11, 425, 0, 0, 2795, 1119, 459, 714, 62, 0, 133, 339, 44, 3772, 0, 1, 13, 22, 0, 0, 0, 4066, 76, 2835, 0, 0, 0, 0, 14, 2, 0, 0, 0, 50, 196, 30, 3527, 130, 4096, 1, 0, 1609, 4093, 4036, 3401, 4096, 3911, 1, 0, 0, 0, 0, 4, 755, 10, 0, 4096, 3762, 0, 112, 0, 1465, 0, 0, 1944, 0, 0, 5, 0, 0, 1, 4096, 479, 103, 3455, 3916, 0, 3734, 39, 64, 30, 3960, 393, 556, 0, 0, 4023] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 87, 0, 185, 644, 4096, 869, 0, 0, 23, 4096, 0, 0, 2, 0, 5, 0, 2815, 38, 106, 0, 1, 0, 3371, 0, 91, 1474, 3025, 251, 1, 12, 5, 0, 4093, 4096, 1, 4092, 4, 0, 48, 4083, 0, 3658, 1, 0, 0, 8, 7, 0, 0, 4096, 1, 29, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 363, 0, 0, 710, 4096, 448, 3, 1317, 0, 4082, 0, 0, 4095, 20, 3989, 15, 0, 3845, 145, 0, 1, 1923, 2796, 1, 3904, 4096, 3, 47, 4096, 1175, 4096, 8, 1831, 0, 0, 3535, 44, 0, 0, 4096, 0, 3361, 749, 2983, 0, 3928, 1569, 2384, 0, 3677, 3999, 0, 0, 21, 4073] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[480, 0, 0, 0, 0, 238, 92, 909, 1893, 4090, 0, 4073, 2883, 0, 0, 44, 3872, 0, 0, 1, 60, 3955, 10, 0, 2141, 0, 4096, 0, 4053, 0, 965, 4096, 0, 1, 12, 12, 0, 1154, 4093, 0, 441, 0, 0, 665, 57, 2, 1, 6, 4073, 1, 971, 27, 0, 903, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 43, 30, 0, 257, 0, 3, 39, 0, 0, 0, 3, 0, 0, 5, 3843, 0, 1623, 0, 0, 4094, 4096, 0, 3747, 353, 4016, 117, 0, 217, 0, 0, 0, 0, 3944, 0, 8, 0, 1, 0, 115, 4063, 0, 0, 0, 1504, 3, 3776, 0, 22, 0, 4067, 4095, 0, 0, 4015, 0, 3651, 4054, 4016, 376, 4096, 4080, 4092, 49, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 24, 138, 4074, 0, 6, 0, 0, 16, 368, 3973, 4095, 4, 155, 0, 3715, 0, 0, 458, 3165, 3898, 87, 198, 4096, 4095, 3604, 27, 3274, 0, 832, 3225, 602, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1725, 2987, 352, 1342, 0, 0, 3855, 43, 127, 0, 0, 0, 64, 1078, 0, 0, 0, 1248, 0, 0, 4047, 0, 170, 0, 175, 2, 34, 0, 2, 981, 0, 0, 0, 4096, 0, 3120, 4058, 2384, 0, 233, 445, 4096, 0, 3852, 2275, 4052, 0, 285, 100, 0, 0, 52, 977, 0, 0, 3512, 0, 88, 3, 1, 4096, 3983, 0, 0, 0, 231, 0, 0, 0, 0, 3899, 175, 4095, 558, 0, 1, 3933, 2128, 3832, 2, 4096, 0, 0, 0, 0, 4052] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 56, 0, 14, 1825, 2, 0, 3765, 4096, 4086, 0, 3998, 2742, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 39, 0, 3978, 1, 0, 0, 1596, 0, 3, 2, 0, 3189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 4075, 0, 4066, 49, 5, 3978, 39, 23, 0, 0, 761, 106, 1202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3162, 0, 0, 0, 6, 107, 0, 0, 0, 0, 218, 77, 3953, 3817, 2642, 1668, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 31, 966, 0, 428, 0, 0, 55, 0, 0, 60, 4092, 3, 39, 4096, 2867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 702, 3799, 0, 0, 3381, 2023, 1310, 972, 225, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3893, 303, 12, 2, 0, 3579, 0, 11, 4096, 0, 0, 3857, 0, 3101, 38, 1038, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3767, 4052, 4086, 1, 0, 0, 61, 1390, 0, 0, 4025, 200, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20588, 8826, 14374, 15711, 16686, 12427, 24790, 17670] avg=16384.0000 max=24790 max_rank=6 min=8826 min_rank=1 imbalance_ratio=1.513062 max_load_reduction=106282 max_load_reduction_pct=0.810867 moved_to_echo=71573 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 22, 3132, 4082, 3493, 4096, 2435, 0, 3996, 2987, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 241, 0, 9, 148, 806, 4053, 0, 0, 22, 0, 2864, 1850, 4096, 2411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 20, 1653, 0, 0, 3623, 428, 0, 0, 23, 4096, 2069, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 2, 302, 3858, 4002, 1600, 4, 0, 0, 0, 0, 4073, 0, 2731, 1537, 1205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1162, 61, 0, 76, 13, 8, 0, 0, 1, 254, 0, 0, 14, 1177, 0, 0, 4088, 2868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 4039, 4095, 4054, 0, 259, 1354, 34, 3873, 3538, 101, 0, 0, 548, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1471, 4078, 880, 0, 10, 0, 3343, 4, 27, 28, 0, 0, 43, 0, 0, 0, 957, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 4096, 4087, 0, 0, 0, 7, 1281, 0, 0, 39, 3935, 10, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[24655, 16603, 13216, 19318, 9722, 22083, 11680, 13795] avg=16384.0000 max=24655 max_rank=0 min=9722 min_rank=4 imbalance_ratio=1.504822 max_load_reduction=106417 max_load_reduction_pct=0.811897 moved_to_echo=57280 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[398, 0, 459, 1, 551, 0, 4051, 0, 0, 0, 22, 2968, 2582, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 20, 0, 4042, 0, 1556, 124, 3925, 4002, 0, 26, 0, 6, 1, 0, 907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 4, 0, 0, 3837, 0, 155, 0, 4095, 118, 0, 79, 33, 9, 4096, 2572, 1202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 5, 9, 0, 3950, 4096, 0, 0, 244, 147, 0, 0, 4040, 3294, 2574, 0, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 54, 0, 0, 65, 937, 0, 15, 0, 0, 46, 3740, 1600, 0, 3987, 2866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 191, 0, 4095, 0, 1, 1276, 12, 0, 85, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3715, 0, 330, 80, 3124, 4095, 1, 0, 984, 1048, 0, 2522, 10, 3, 0, 0, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3440, 4096, 0, 4096, 0, 0, 852, 2, 0, 0, 4093, 4093, 708, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11056, 14845, 16221, 23945, 13333, 9605, 16637, 25430] avg=16384.0000 max=25430 max_rank=7 min=9605 min_rank=5 imbalance_ratio=1.552124 max_load_reduction=105642 max_load_reduction_pct=0.805984 moved_to_echo=65005 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1643, 7, 0, 603, 0, 362, 492, 0, 3682, 2649, 4042, 3198, 3801, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 61, 72, 0, 3197, 288, 2911, 4072, 0, 20, 0, 4078, 0, 0, 1840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 2116, 688, 0, 11, 425, 0, 0, 2795, 1119, 459, 3960, 1920, 1103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 714, 62, 0, 133, 339, 44, 3772, 0, 1, 13, 22, 0, 0, 0, 4066, 76, 2615, 26, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2835, 0, 0, 0, 0, 14, 2, 0, 0, 0, 50, 196, 30, 3527, 130, 4096, 4096, 2631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1609, 4093, 4036, 3401, 4096, 1296, 1, 0, 0, 0, 0, 4, 755, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3762, 0, 112, 0, 1465, 0, 0, 1944, 0, 0, 5, 0, 0, 1, 16, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 103, 3455, 3916, 0, 0, 39, 22, 30, 0, 393, 556, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20729, 20517, 14600, 13002, 17607, 19302, 12299, 13016] avg=16384.0000 max=20729 max_rank=0 min=12299 min_rank=6 imbalance_ratio=1.265198 max_load_reduction=110343 max_load_reduction_pct=0.841850 moved_to_echo=62224 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3206, 0, 0, 1, 0, 0, 0, 0, 0, 3825, 0, 9, 2843, 4041, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 0, 0, 0, 118, 0, 0, 0, 4055, 626, 0, 0, 3967, 223, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 13, 0, 3016, 0, 80, 4094, 0, 0, 1, 0, 4096, 319, 1208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 0, 0, 6, 0, 0, 1, 407, 940, 12, 0, 1, 0, 0, 4096, 0, 2744, 1672, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4095, 4074, 0, 0, 0, 0, 0, 5, 1, 0, 3910, 0, 4005, 4044, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1619, 2, 4095, 3124, 4094, 1352, 0, 1092, 0, 0, 0, 0, 1094, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 22, 4095, 0, 48, 0, 7, 0, 3148, 25, 2572, 1395, 0, 46, 0, 1042, 801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1619, 1, 2634, 0, 0, 0, 1382, 0, 0, 218, 382, 0, 176, 2749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13928, 16182, 16889, 13971, 27108, 16537, 17296, 9161] avg=16384.0000 max=27108 max_rank=4 min=9161 min_rank=7 imbalance_ratio=1.654541 max_load_reduction=103964 max_load_reduction_pct=0.793182 moved_to_echo=70102 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[480, 0, 0, 0, 0, 238, 92, 909, 1893, 4090, 0, 4073, 2251, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3872, 0, 0, 1, 60, 3955, 10, 0, 2141, 0, 26, 0, 4053, 0, 965, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 12, 0, 1154, 4093, 0, 441, 0, 0, 665, 57, 2, 1, 6, 4096, 2572, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 1, 971, 27, 0, 903, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 43, 2685, 1635, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 257, 0, 3, 39, 0, 0, 0, 3, 0, 0, 5, 3843, 0, 1623, 4067, 2571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 3747, 353, 1331, 117, 0, 217, 0, 0, 0, 0, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 0, 115, 4063, 0, 0, 0, 1504, 3, 3776, 0, 22, 0, 1018, 632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4015, 0, 0, 4054, 1363, 376, 0, 4080, 4092, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14070, 19179, 14192, 20025, 12441, 17899, 11142, 22124] avg=16384.0000 max=22124 max_rank=7 min=11142 min_rank=6 imbalance_ratio=1.350342 max_load_reduction=108948 max_load_reduction_pct=0.831207 moved_to_echo=63606 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 87, 0, 185, 644, 4096, 869, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 2815, 38, 106, 0, 1, 0, 19, 0, 91, 1474, 3025, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 5, 0, 4093, 4096, 1, 4092, 4, 0, 48, 4083, 0, 3658, 1, 0, 3677, 2116, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 7, 0, 0, 4096, 1, 29, 0, 0, 0, 0, 0, 36, 0, 0, 2667, 971, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 363, 0, 0, 710, 4096, 448, 3, 4096, 2766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 0, 4082, 0, 0, 4095, 20, 1322, 15, 0, 3845, 145, 0, 1, 1923, 2796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3904, 4096, 3, 47, 4096, 1175, 4096, 8, 1831, 0, 0, 3535, 44, 0, 0, 604, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3361, 749, 2983, 0, 0, 1569, 809, 0, 0, 3999, 0, 0, 21, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14091, 7827, 27049, 9051, 12484, 19561, 23445, 17564] avg=16384.0000 max=27049 max_rank=2 min=7827 min_rank=1 imbalance_ratio=1.650940 max_load_reduction=104023 max_load_reduction_pct=0.793633 moved_to_echo=73054 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 24, 138, 4074, 0, 6, 0, 0, 16, 368, 3973, 3197, 4, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3715, 0, 0, 458, 3165, 3898, 87, 198, 4096, 4095, 23, 27, 3274, 0, 832, 3225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1725, 2987, 352, 1342, 0, 0, 4096, 2263, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3855, 43, 127, 0, 0, 0, 64, 1078, 0, 0, 0, 1248, 0, 0, 4047, 0, 2709, 1560, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 0, 175, 2, 34, 0, 2, 981, 0, 0, 0, 4096, 0, 3120, 4058, 2384, 3899, 2770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 445, 4096, 0, 3852, 2275, 1343, 0, 285, 100, 0, 0, 52, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3512, 0, 88, 3, 1, 4096, 3983, 0, 0, 0, 231, 0, 0, 0, 0, 971, 898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 4095, 558, 0, 1, 0, 2128, 1301, 2, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11955, 27093, 14531, 16049, 21691, 13658, 13783, 12312] avg=16384.0000 max=27093 max_rank=1 min=11955 min_rank=0 imbalance_ratio=1.653625 max_load_reduction=103979 max_load_reduction_pct=0.793297 moved_to_echo=61444 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 12416, 21504, 13056, 17664, 128, 0, 8832, 7552, 512, 0, 16896, 13312, 768, 384, 19072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[11136, 4224, 384, 8576, 6272, 12800, 1536, 12800, 18432, 4736, 256, 128, 21632, 3584, 8960, 16768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[1408, 5120, 19968, 16384, 16384, 21632, 13184, 10624, 1152, 5504, 7808, 256, 0, 128, 5376, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[4096, 5888, 512, 256, 7168, 3456, 8832, 9344, 14208, 19072, 3072, 23168, 19840, 12416, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[5632, 27392, 9088, 4608, 256, 7424, 21888, 8192, 128, 12032, 2688, 2816, 15232, 128, 3200, 128, 5760, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[16640, 128, 4352, 256, 384, 5376, 15872, 13824, 2560, 128, 384, 1408, 256, 128, 24320, 7296, 21376, 9088, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[4736, 128, 128, 128, 4096, 5376, 14336, 4864, 9344, 128, 6400, 19584, 896, 7936, 1280, 512, 28928, 13952, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[8320, 4224, 512, 4352, 5120, 128, 512, 1920, 128, 768, 128, 4352, 896, 27520, 6272, 12160, 32384, 22272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[3795, 0, 0, 0, 2, 2, 0, 0, 0, 4096, 1, 0, 0, 11, 0, 4047, 622, 0, 0, 0, 4064, 0, 1980, 0, 2, 4061, 0, 9, 1437, 0, 0, 12, 4033, 3328, 0, 4096, 0, 3830, 3, 0, 2875, 4092, 0, 3796, 94, 1, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 12, 1, 11, 0, 4096, 99, 25, 9, 3942, 467, 3254, 0, 160, 0, 4042, 50, 58, 4087, 2, 1, 0, 368, 4087, 0, 0, 4092, 226, 7, 0, 3840, 4094, 0, 0, 1320, 0, 4070, 1, 0, 24, 7, 19, 0, 109, 2140, 0, 0, 3849, 0, 528, 0, 4096, 4094, 891, 0, 0, 0, 1, 0, 0, 2702, 0, 0, 2941, 113, 4095, 0, 4096, 149, 850, 3535, 4095, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 217, 3258, 0, 0, 0, 150, 0, 3093, 4, 0, 1004, 3351, 0, 18, 0, 0, 2345, 4074, 0, 1258, 14, 88, 42, 4095, 276, 949, 740, 7, 0, 0, 1213, 4096, 0, 49, 0, 4096, 0, 0, 9, 0, 5, 4016, 3268, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 3078, 4086, 0, 19, 3976, 0, 0, 0, 3681, 0, 22, 0, 0, 0, 0, 0, 36, 4084, 544, 450, 1491, 3, 277, 0, 0, 0, 166, 0, 18, 0, 282, 1286, 0, 2, 1, 3174, 0, 4090, 0, 1, 1, 3215, 0, 2156, 0, 11, 4085, 0, 0, 104, 4096, 3983, 634, 0, 0, 72, 4, 4086, 6, 157, 4084, 0, 3815, 0, 4051, 317, 4092, 4049, 259, 2445, 480, 159] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[3808, 2927, 0, 0, 0, 31, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3413, 4094, 0, 876, 205, 1, 5, 4013, 4096, 0, 14, 625, 1, 4095, 25, 26, 0, 4095, 0, 0, 58, 4096, 0, 4096, 477, 4096, 0, 0, 1024, 0, 0, 0, 0, 0, 636, 405, 2, 4096, 18, 0, 4019, 0, 8, 0, 3993, 12, 0, 1216, 0, 0, 123, 0, 8, 758, 0, 0, 4093, 3790, 4094, 4096, 21, 4048, 0, 0, 83, 2125, 131, 2531, 0, 0, 1029, 4093, 0, 0, 2915, 0, 0, 0, 0, 2100, 0, 0, 3419, 15, 0, 0, 0, 0, 0, 70, 0, 0, 4096, 1997, 0, 4095, 4059, 8, 0, 1370, 4096, 53, 3638, 0, 3195, 49, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3691, 0, 0, 4087, 1, 3725, 0, 0, 0, 3624, 0, 11, 0, 56, 0, 4066, 14, 0, 0, 0, 2656, 2158, 0, 4, 0, 10, 0, 0, 4096, 1060, 0, 2, 0, 4064, 5, 0, 8, 4096, 4093, 0, 0, 0, 0, 0, 0, 4088, 36, 1152, 2, 129, 0, 7, 23, 1490, 120, 4055, 0, 3740, 0, 0, 4096, 0, 0, 2046, 0, 0, 0, 3, 1, 0, 0, 1, 0, 4096, 0, 0, 1224, 0, 3534, 1184, 0, 3268, 0, 0, 4033, 1, 3006, 4079, 756, 0, 74, 4091, 0, 141, 4096, 0, 3917, 0, 0, 0, 0, 0, 4047, 423, 34, 1110, 3312, 4, 0, 0, 0, 0, 4, 422, 2, 4095, 202, 92, 3204, 8, 4050, 5, 3612, 28, 4095, 0, 2301, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 3965, 256, 0, 0, 0, 0, 0, 3109, 1701, 0, 3606, 143, 4, 0, 0, 0, 0, 5, 3704, 2966, 0, 1127, 39, 2, 4096, 0, 292, 4096, 0, 3248, 0, 10, 4, 29, 10, 0, 3459, 4015, 0, 0, 4092, 1172, 1885, 27, 0, 0, 0, 1033, 0, 0, 0, 0, 40, 1, 0, 0, 126, 0, 0, 4083, 0, 0, 899, 3822, 3932, 389, 4053, 14, 38, 3880, 294, 1, 4072, 0, 0, 0, 0, 0, 0, 0, 4096, 3966, 0, 0, 4096, 3942, 4072, 65, 1, 15, 4096, 3688, 1, 0, 15, 335, 5, 0, 92, 0, 0, 2978, 0, 842, 0, 0, 3932, 0, 0, 23, 6, 4061, 3654, 0, 1789, 0, 1, 0, 0, 4089, 0, 3333, 0, 0, 16, 4096, 28] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4090, 2, 4079, 4096, 3, 0, 0, 159, 0, 69, 0, 0, 40, 3, 275, 80, 0, 24, 0, 0, 4028, 0, 103, 683, 1824, 2203, 0, 0, 3792, 0, 15, 169, 4056, 304, 300, 0, 82, 4073, 2, 0, 0, 0, 0, 3936, 0, 4096, 5, 0, 1, 4010, 0, 0, 0, 0, 0, 3805, 0, 16, 0, 0, 2697, 0, 0, 0, 4094, 6, 0, 102, 213, 0, 1, 0, 0, 3978, 66, 92, 1848, 4, 3723, 4061, 0, 3832, 0, 0, 348, 6, 4096, 1, 2183, 0, 0, 4062, 1, 0, 0, 0, 1392, 0, 0, 4092, 0, 0, 3901, 18, 0, 0, 3911, 0, 1351, 0, 98, 5, 3378, 3951, 0, 3833, 6, 1, 2589, 182, 4003, 350, 4006, 4093, 0, 0, 4075, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4076, 0, 485, 0, 0, 14, 63, 3, 0, 4093, 13, 3534, 2671, 3, 0, 3893, 141, 0, 846, 833, 1866, 4080, 0, 4031, 682, 3313, 0, 134, 0, 3, 11, 2035, 160, 0, 1, 404, 29, 0, 3707, 1, 0, 29, 0, 0, 3858, 4095, 1, 4, 0, 0, 8, 0, 1123, 0, 0, 569, 1671, 4014, 3544, 0, 4084, 0, 0, 0, 4096, 0, 83, 0, 0, 0, 22, 0, 458, 3977, 0, 0, 6, 0, 4096, 0, 0, 0, 4096, 0, 3909, 133, 537, 3562, 0, 1046, 223, 4005, 0, 62, 0, 2, 0, 0, 1061, 4, 0, 0, 1367, 0, 21, 0, 2091, 4096, 0, 0, 33, 0, 4096, 3791, 5, 3144, 3290, 0, 604, 0, 3774, 0, 4096, 0, 0, 3, 4096, 1062] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[3965, 1, 25, 636, 0, 0, 4, 4053, 0, 452, 1, 223, 59, 0, 0, 4096, 0, 65, 3012, 12, 819, 0, 35, 4048, 0, 3616, 0, 3425, 4094, 8, 882, 235, 1027, 0, 0, 2, 0, 4066, 0, 0, 0, 0, 8, 0, 113, 4066, 3, 0, 6, 3876, 0, 3, 0, 0, 0, 0, 854, 4096, 0, 39, 4096, 0, 187, 0, 4087, 0, 11, 0, 0, 0, 0, 1, 0, 3416, 0, 46, 625, 0, 3491, 1113, 0, 5, 3511, 14, 27, 129, 4058, 4075, 0, 45, 0, 4096, 0, 4090, 262, 0, 744, 0, 776, 4086, 0, 0, 4096, 0, 0, 0, 4080, 1, 0, 324, 2, 3, 1322, 2935, 34, 4046, 0, 0, 4093, 0, 3477, 89, 919, 3934, 4096, 0, 2705, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3795, 0, 0, 0, 2, 2, 0, 0, 0, 4096, 1, 0, 0, 11, 0, 4047, 1761, 942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 622, 0, 0, 0, 4064, 0, 1980, 0, 2, 4061, 0, 9, 1437, 0, 0, 12, 567, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 3328, 0, 4096, 0, 2776, 3, 0, 2875, 4092, 0, 3796, 94, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 12, 1, 11, 0, 4096, 99, 25, 9, 4095, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3942, 467, 3254, 0, 160, 0, 4042, 50, 58, 4087, 2, 1, 0, 368, 4087, 0, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 226, 7, 0, 3840, 4094, 0, 0, 1320, 0, 1936, 1, 0, 24, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 109, 2140, 0, 0, 3849, 0, 528, 0, 4096, 4094, 891, 0, 0, 0, 2747, 2231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2702, 0, 0, 2941, 113, 0, 0, 0, 149, 850, 3535, 922, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14657, 13127, 25094, 9721, 21005, 15547, 20704, 11217] avg=16384.0000 max=25094 max_rank=2 min=9721 min_rank=3 imbalance_ratio=1.531616 max_load_reduction=105978 max_load_reduction_pct=0.808548 moved_to_echo=68473 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3808, 2927, 0, 0, 0, 31, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 1774, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3413, 4094, 0, 876, 205, 1, 5, 4013, 4096, 0, 14, 625, 610, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 25, 26, 0, 2960, 0, 0, 58, 4096, 0, 4096, 477, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 0, 0, 0, 0, 0, 636, 405, 2, 4096, 18, 0, 4019, 0, 8, 0, 4096, 1199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 12, 0, 1216, 0, 0, 123, 0, 8, 758, 0, 0, 4093, 3790, 4094, 4096, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4048, 0, 0, 83, 2125, 131, 2531, 0, 0, 1029, 1940, 0, 0, 2915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2100, 0, 0, 3419, 15, 0, 0, 0, 0, 0, 70, 0, 0, 2439, 2235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1997, 0, 4095, 4059, 8, 0, 1370, 0, 53, 0, 0, 3195, 49, 915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9662, 18331, 19930, 15503, 22708, 14823, 10278, 19837] avg=16384.0000 max=22708 max_rank=4 min=9662 min_rank=0 imbalance_ratio=1.385986 max_load_reduction=108364 max_load_reduction_pct=0.826752 moved_to_echo=67646 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 217, 3258, 0, 0, 0, 150, 0, 3093, 4, 0, 1004, 3351, 0, 18, 1373, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2345, 4074, 0, 1258, 14, 88, 42, 4095, 276, 949, 740, 7, 0, 0, 606, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1213, 4096, 0, 49, 0, 2969, 0, 0, 9, 0, 5, 4016, 3268, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 3078, 4086, 0, 19, 3976, 0, 0, 0, 4051, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3681, 0, 22, 0, 0, 0, 0, 0, 36, 4084, 544, 450, 1491, 3, 277, 0, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 18, 0, 282, 1286, 0, 2, 1, 1510, 0, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3215, 0, 2156, 0, 11, 4085, 0, 0, 104, 4096, 3983, 634, 0, 0, 72, 2744, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4086, 6, 157, 4084, 0, 3815, 0, 0, 317, 0, 4049, 259, 2445, 109, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16673, 14785, 19694, 20603, 11109, 7356, 21362, 19490] avg=16384.0000 max=21362 max_rank=6 min=7356 min_rank=5 imbalance_ratio=1.303833 max_load_reduction=109710 max_load_reduction_pct=0.837021 moved_to_echo=59317 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4090, 2, 4079, 4096, 3, 0, 0, 159, 0, 69, 0, 0, 40, 3, 275, 80, 1757, 937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 4028, 0, 103, 683, 1824, 2203, 0, 0, 3792, 0, 15, 169, 603, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 304, 300, 0, 82, 2952, 2, 0, 0, 0, 0, 3936, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4010, 0, 0, 0, 0, 0, 3805, 0, 16, 0, 0, 2697, 0, 0, 0, 4003, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 102, 213, 0, 1, 0, 0, 3978, 66, 92, 1848, 4, 3723, 4061, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3832, 0, 0, 348, 6, 4096, 1, 2183, 0, 0, 1933, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1392, 0, 0, 4092, 0, 0, 3901, 18, 0, 0, 3911, 0, 1351, 0, 98, 5, 2686, 2220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3378, 3951, 0, 3833, 6, 1, 2589, 182, 0, 350, 0, 4093, 0, 0, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15590, 13816, 15733, 15852, 18706, 12400, 19674, 19301] avg=16384.0000 max=19674 max_rank=6 min=12400 min_rank=5 imbalance_ratio=1.200806 max_load_reduction=111398 max_load_reduction_pct=0.849899 moved_to_echo=70081 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4076, 0, 485, 0, 0, 14, 63, 3, 0, 4093, 13, 3534, 2671, 3, 0, 3893, 1733, 942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 846, 833, 1866, 4080, 0, 4031, 682, 3313, 0, 134, 0, 3, 11, 2035, 0, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 1, 404, 29, 0, 3707, 1, 0, 29, 0, 0, 3858, 4095, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1123, 0, 0, 569, 1671, 4014, 3544, 0, 4084, 0, 0, 0, 3774, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 83, 0, 0, 0, 22, 0, 458, 3977, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3909, 133, 537, 3562, 0, 1046, 223, 1905, 0, 62, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 4, 0, 0, 1367, 0, 21, 0, 2091, 4096, 0, 0, 33, 0, 2746, 2232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3791, 5, 3144, 3290, 0, 604, 0, 0, 0, 0, 0, 0, 3, 922, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21523, 18342, 12289, 20137, 12738, 15475, 13651, 16917] avg=16384.0000 max=21523 max_rank=0 min=12289 min_rank=2 imbalance_ratio=1.313660 max_load_reduction=109549 max_load_reduction_pct=0.835793 moved_to_echo=58781 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3965, 256, 0, 0, 0, 0, 0, 3109, 1701, 0, 3606, 143, 4, 0, 0, 1772, 942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3704, 2966, 0, 1127, 39, 2, 4096, 0, 292, 4096, 0, 3248, 0, 512, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4, 29, 10, 0, 2507, 4015, 0, 0, 4092, 1172, 1885, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 0, 0, 0, 0, 40, 1, 0, 0, 126, 0, 0, 4083, 0, 0, 899, 4089, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3822, 3932, 389, 4053, 14, 38, 3880, 294, 1, 4072, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3966, 0, 0, 4096, 3942, 4072, 65, 1, 15, 1949, 3688, 1, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 5, 0, 92, 0, 0, 2978, 0, 842, 0, 0, 3932, 0, 0, 23, 6, 2234, 2232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 3654, 0, 1789, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 922, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15498, 20462, 13751, 11370, 20935, 25906, 12679, 10471] avg=16384.0000 max=25906 max_rank=5 min=10471 min_rank=7 imbalance_ratio=1.581177 max_load_reduction=105166 max_load_reduction_pct=0.802353 moved_to_echo=69991 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3691, 0, 0, 4087, 1, 3725, 0, 0, 0, 3624, 0, 11, 0, 56, 0, 4066, 1770, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 2656, 2158, 0, 4, 0, 10, 0, 0, 4096, 1060, 0, 2, 606, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 5, 0, 8, 2969, 4093, 0, 0, 0, 0, 0, 0, 4088, 36, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 129, 0, 7, 23, 1490, 120, 4055, 0, 3740, 0, 0, 4096, 0, 0, 2046, 4050, 1191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 1, 0, 4096, 0, 0, 1224, 0, 3534, 1184, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3268, 0, 0, 4033, 1, 3006, 4079, 756, 0, 74, 1946, 0, 141, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3917, 0, 0, 0, 0, 0, 4047, 423, 34, 1110, 3312, 4, 0, 0, 0, 0, 2421, 1254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 422, 2, 4095, 202, 92, 3204, 8, 0, 5, 0, 28, 4095, 0, 518, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21560, 10981, 16415, 20949, 10564, 21400, 16522, 12681] avg=16384.0000 max=21560 max_rank=0 min=10564 min_rank=4 imbalance_ratio=1.315918 max_load_reduction=109512 max_load_reduction_pct=0.835510 moved_to_echo=61167 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3965, 1, 25, 636, 0, 0, 4, 4053, 0, 452, 1, 223, 59, 0, 0, 4096, 1772, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 3012, 12, 819, 0, 35, 4048, 0, 3616, 0, 3425, 4094, 8, 882, 235, 602, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 0, 0, 2, 0, 2947, 0, 0, 0, 0, 8, 0, 113, 4066, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3876, 0, 3, 0, 0, 0, 0, 854, 4096, 0, 39, 4096, 0, 187, 0, 3477, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 11, 0, 0, 0, 0, 1, 0, 3416, 0, 46, 625, 0, 3491, 1113, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3511, 14, 27, 129, 4058, 4075, 0, 45, 0, 1949, 0, 4090, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 744, 0, 776, 4086, 0, 0, 4096, 0, 0, 0, 4080, 1, 0, 324, 2, 3, 616, 1474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1322, 2935, 34, 4046, 0, 0, 4093, 0, 0, 89, 0, 3934, 4096, 0, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15909, 21228, 8166, 16937, 13307, 18165, 16202, 21158] avg=16384.0000 max=21228 max_rank=1 min=8166 min_rank=2 imbalance_ratio=1.295654 max_load_reduction=109844 max_load_reduction_pct=0.838043 moved_to_echo=68832 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[6528, 3328, 2048, 14720, 0, 128, 27776, 512, 1536, 1280, 21632, 16128, 2944, 512, 256, 128, 18688, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[27776, 4480, 3840, 5376, 512, 128, 8192, 384, 640, 28544, 640, 640, 9344, 4224, 23424, 10496, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[10624, 16000, 384, 4608, 128, 20096, 11904, 128, 2944, 12416, 1280, 17792, 7936, 24576, 128, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[27520, 6912, 5120, 12160, 128, 3840, 128, 4480, 3200, 17408, 128, 7424, 3968, 3456, 384, 16256, 13824, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[17024, 20864, 128, 23936, 11648, 128, 17280, 1792, 0, 896, 0, 12288, 12544, 6144, 5888, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2176, 8064, 128, 128, 1152, 1536, 768, 12928, 5632, 20224, 3584, 128, 31232, 128, 256, 3072, 31744, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 19456, 12032, 128, 8448, 10368, 20224, 19712, 3072, 2432, 1408, 15104, 3712, 8448, 7424, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[896, 128, 6272, 8704, 19840, 11648, 3328, 9856, 2816, 21504, 384, 8832, 22400, 1152, 4224, 3200, 4224, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 3810, 2, 3991, 18, 29, 0, 0, 2765, 2, 0, 3879, 1, 2949, 1, 0, 0, 15, 0, 0, 0, 0, 4086, 0, 0, 2654, 2408, 2500, 30, 31, 332, 15, 0, 4025, 0, 3396, 0, 0, 47, 3575, 0, 0, 0, 53, 4075, 286, 0, 0, 1917, 192, 3991, 0, 1760, 5, 264, 0, 859, 4073, 0, 0, 109, 2054, 4093, 0, 9, 4064, 3968, 0, 891, 0, 2, 291, 3906, 0, 4096, 0, 0, 0, 13, 1, 0, 4063, 1, 0, 12, 3742, 92, 353, 1177, 0, 3491, 1612, 4092, 0, 4, 0, 0, 24, 367, 4094, 4095, 3734, 4084, 3791, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3132, 0, 3, 10, 0, 9, 2, 1892, 3259, 81, 63, 2253, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1783, 446, 0, 4058, 0, 0, 1, 76, 1, 124, 0, 3869, 0, 4096, 0, 0, 0, 15, 0, 2628, 0, 0, 2547, 1, 0, 4092, 34, 0, 0, 0, 2323, 0, 2, 0, 60, 4085, 37, 33, 27, 0, 4096, 4, 23, 13, 589, 0, 2, 0, 876, 1004, 0, 0, 45, 0, 0, 0, 0, 3898, 0, 65, 0, 4035, 4089, 0, 3, 0, 4005, 0, 0, 1637, 3623, 2, 3573, 171, 3983, 0, 244, 4096, 8, 0, 4054, 4086, 0, 197, 3995, 44, 0, 3987, 2, 0, 0, 791, 0, 0, 15, 3, 267, 4096, 561, 3762, 4096, 4092, 3428, 4096, 0, 18, 0, 0, 0, 18, 0, 0, 0, 0, 279, 0, 0, 4055, 105, 0, 4096, 0, 3839, 0, 3740, 1, 2923, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[49, 0, 2810, 105, 46, 0, 37, 0, 4019, 0, 23, 3937, 45, 0, 3, 0, 68, 0, 17, 0, 0, 0, 3965, 1, 0, 3898, 0, 3711, 0, 0, 0, 56, 0, 0, 18, 2574, 73, 862, 127, 583, 707, 597, 1, 110, 0, 4064, 6, 56, 0, 0, 4072, 0, 4, 0, 0, 4038, 0, 4052, 0, 0, 297, 4095, 3964, 327, 3919, 1, 3982, 4094, 0, 3782, 1982, 0, 3455, 4096, 4089, 4041, 125, 4086, 0, 0, 1, 4043, 0, 3391, 2168, 4088, 230, 105, 197, 70, 0, 0, 0, 0, 0, 0, 28, 0, 16, 0, 0, 2513, 1780, 367, 16, 737, 4095, 0, 0, 8, 1131, 144, 0, 0, 5, 4096, 0, 2761, 0, 4, 3895, 0, 22, 4092, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 3890, 0, 0, 4096, 0, 1, 4087, 3, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 4, 0, 0, 834, 4096, 3115, 0, 156, 3911, 285, 55, 0, 0, 0, 4061, 0, 0, 488, 41, 4094, 0, 0, 4074, 3619, 2, 3972, 4069, 0, 3456, 664, 532, 0, 4096, 4096, 0, 4088, 0, 0, 0, 2022, 4060, 0, 3265, 0, 0, 29, 4044, 1835, 20, 0, 0, 1, 345, 0, 371, 4068, 87, 0, 4096, 0, 0, 0, 6, 31, 0, 0, 0, 2, 0, 98, 4072, 14, 0, 0, 2, 3, 3, 4093, 0, 4091, 4094, 3787, 0, 0, 65, 0, 0, 0, 0, 0, 2052, 63, 0, 33, 0, 4045, 1127, 0, 3942, 1, 0, 2922, 12, 2117, 0, 126] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[11, 4082, 3824, 21, 218, 164, 0, 0, 0, 3613, 186, 4059, 0, 0, 738, 0, 0, 0, 0, 552, 0, 1195, 4096, 0, 0, 4057, 0, 26, 62, 245, 4066, 0, 10, 0, 0, 4092, 701, 0, 0, 17, 2871, 0, 400, 0, 77, 4074, 3878, 35, 0, 1, 855, 12, 3597, 0, 2, 575, 0, 4092, 1, 171, 0, 4072, 1685, 0, 3195, 0, 0, 4093, 0, 0, 0, 0, 4073, 3, 0, 19, 7, 0, 35, 0, 3002, 3799, 0, 8, 2, 3549, 1, 30, 185, 0, 0, 0, 4096, 3125, 1, 0, 0, 22, 0, 0, 3688, 20, 4043, 4078, 1, 445, 7, 2685, 4013, 32, 12, 67, 0, 0, 0, 0, 0, 4050, 3969, 33, 4023, 4090, 42, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 11, 77, 56, 3, 0, 187, 267, 278, 0, 0, 1974, 0, 4084, 0, 0, 22, 0, 0, 0, 0, 1114, 0, 0, 0, 0, 1, 1, 0, 20, 244, 0, 908, 4052, 0, 3765, 57, 3785, 0, 0, 0, 0, 0, 433, 0, 4, 0, 2, 0, 0, 4096, 0, 0, 7, 0, 420, 3695, 3899, 0, 34, 12, 2536, 4087, 0, 4020, 0, 6, 1, 4096, 2890, 0, 3796, 3358, 6, 0, 0, 132, 0, 4086, 0, 4096, 4020, 0, 1008, 36, 3671, 69, 3122, 4096, 1, 2207, 3666, 3996, 2, 0, 0, 0, 3437, 0, 0, 4013, 4096, 4095, 3964, 0, 0, 4096, 0, 241, 0, 0, 0, 0, 1745, 0, 0, 7, 454, 4023, 0, 3591, 0, 8, 70, 4049, 668, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[14, 0, 4, 14, 0, 547, 0, 2334, 4034, 3539, 0, 137, 3723, 3596, 3682, 0, 72, 0, 0, 0, 80, 0, 4021, 0, 0, 0, 116, 3906, 0, 58, 2084, 74, 0, 2, 6, 4094, 4002, 12, 0, 4096, 0, 0, 0, 0, 1482, 1596, 5, 0, 77, 455, 4096, 0, 716, 0, 60, 0, 312, 3993, 3, 0, 145, 3073, 2558, 0, 1134, 3113, 3766, 11, 2833, 0, 1044, 0, 4017, 29, 4031, 0, 207, 0, 0, 56, 3857, 0, 0, 36, 399, 0, 705, 2879, 262, 0, 0, 205, 3980, 0, 1, 0, 17, 0, 0, 3422, 4074, 0, 1462, 2963, 0, 1326, 4091, 0, 0, 21, 2, 0, 0, 3, 0, 4082, 0, 3586, 0, 0, 4031, 0, 2533, 4076, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 3332, 431, 3721, 0, 0, 0, 5, 2538, 0, 0, 4006, 0, 0, 1, 0, 0, 0, 0, 0, 3196, 0, 0, 0, 0, 1487, 12, 0, 0, 0, 4096, 0, 0, 4096, 0, 3769, 2122, 1072, 32, 4, 539, 92, 1, 0, 1384, 2, 0, 0, 4082, 0, 3977, 22, 809, 0, 4073, 1, 119, 4096, 0, 14, 2, 0, 4096, 0, 2852, 4091, 0, 3301, 209, 43, 0, 0, 4066, 0, 3948, 4096, 1, 1179, 0, 0, 3695, 106, 0, 0, 0, 429, 0, 0, 2424, 0, 1881, 4096, 2274, 68, 2, 29, 0, 0, 1937, 3928, 0, 3763, 4096, 4007, 18, 0, 1263, 0, 32, 0, 0, 2769, 23, 5, 0, 10, 0, 3978, 4096, 0, 4096, 2, 1020, 1, 0, 9, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[49, 0, 2810, 105, 46, 0, 37, 0, 4019, 0, 23, 3937, 45, 0, 3, 0, 1906, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 17, 0, 0, 0, 3965, 1, 0, 3898, 0, 3711, 0, 0, 0, 56, 3455, 1436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2574, 73, 862, 127, 583, 707, 597, 1, 110, 0, 4064, 6, 56, 2261, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 4, 0, 0, 4038, 0, 1432, 0, 0, 297, 4095, 3964, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3919, 1, 3982, 4094, 0, 3782, 1982, 0, 0, 4096, 1970, 4041, 125, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4043, 0, 3391, 2168, 4088, 230, 105, 197, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 16, 0, 0, 2513, 284, 367, 16, 737, 4095, 0, 0, 8, 1131, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 2761, 0, 4, 3895, 0, 22, 4092, 0, 0, 0, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13339, 16607, 12099, 18229, 32078, 14293, 9339, 15088] avg=16384.0000 max=32078 max_rank=4 min=9339 min_rank=6 imbalance_ratio=1.957886 max_load_reduction=98994 max_load_reduction_pct=0.755264 moved_to_echo=70798 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3810, 2, 3991, 18, 29, 0, 0, 2765, 2, 0, 3879, 1, 2949, 1, 0, 1910, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 4086, 0, 0, 2654, 2408, 2500, 30, 31, 332, 15, 3906, 3294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 3396, 0, 0, 47, 3575, 0, 0, 0, 53, 4075, 286, 0, 0, 2272, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1917, 192, 3991, 0, 1760, 5, 264, 0, 859, 1440, 0, 0, 109, 2054, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4064, 3968, 0, 891, 0, 2, 291, 0, 0, 1973, 0, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 1, 0, 12, 3742, 92, 353, 1177, 0, 3491, 1612, 4092, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 367, 4094, 4095, 3734, 651, 3791, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3132, 0, 3, 10, 0, 9, 2, 1892, 3259, 81, 63, 2253, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19718, 19271, 17868, 16684, 11212, 18639, 16763, 10917] avg=16384.0000 max=19718 max_rank=0 min=10917 min_rank=7 imbalance_ratio=1.203491 max_load_reduction=111354 max_load_reduction_pct=0.849564 moved_to_echo=57531 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3890, 0, 0, 4096, 0, 1, 4087, 3, 0, 3, 4096, 0, 0, 0, 0, 163, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 4, 0, 0, 834, 4096, 3115, 0, 156, 3911, 285, 0, 3307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 4061, 0, 0, 488, 41, 4094, 0, 0, 4074, 3619, 2, 3972, 2284, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 3456, 664, 532, 0, 4096, 4096, 0, 1439, 0, 0, 0, 2022, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3265, 0, 0, 29, 4044, 1835, 20, 0, 0, 1, 161, 0, 371, 4068, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3650, 0, 0, 0, 6, 31, 0, 0, 0, 2, 0, 98, 4072, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 3, 4093, 0, 4091, 643, 3787, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2052, 63, 0, 33, 0, 4045, 1127, 0, 3942, 1, 0, 2922, 12, 2117, 0, 126, 446, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16704, 15752, 22834, 24434, 13881, 7873, 12687, 16907] avg=16384.0000 max=24434 max_rank=3 min=7873 min_rank=5 imbalance_ratio=1.491333 max_load_reduction=106638 max_load_reduction_pct=0.813583 moved_to_echo=51348 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1783, 446, 0, 4058, 0, 0, 1, 76, 1, 124, 0, 3869, 0, 4096, 0, 0, 1857, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 2628, 0, 0, 2547, 1, 0, 4092, 34, 0, 0, 0, 2323, 0, 3573, 2765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 60, 4085, 37, 33, 27, 0, 4096, 4, 23, 13, 589, 0, 2, 0, 2175, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 876, 1004, 0, 0, 45, 0, 0, 0, 0, 1378, 0, 65, 0, 4035, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4005, 0, 0, 1637, 3623, 2, 0, 171, 1919, 0, 244, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3616, 4086, 0, 197, 3995, 44, 0, 3987, 2, 0, 0, 791, 0, 0, 15, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 4096, 561, 3762, 4096, 4092, 546, 4096, 0, 18, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 0, 0, 4055, 105, 0, 4096, 0, 3839, 0, 3740, 1, 2923, 4, 438, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16656, 17978, 11263, 11492, 15708, 16736, 21552, 19687] avg=16384.0000 max=21552 max_rank=6 min=11263 min_rank=2 imbalance_ratio=1.315430 max_load_reduction=109520 max_load_reduction_pct=0.835571 moved_to_echo=73683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[11, 4082, 3824, 21, 218, 164, 0, 0, 0, 3613, 186, 4059, 0, 0, 738, 0, 0, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 0, 1195, 4096, 0, 0, 4057, 0, 26, 62, 245, 4066, 0, 4073, 3261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4092, 701, 0, 0, 17, 2871, 0, 400, 0, 77, 4074, 3878, 35, 2283, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 855, 12, 3597, 0, 2, 575, 0, 1447, 1, 171, 0, 4072, 1685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3195, 0, 0, 4093, 0, 0, 0, 0, 0, 3, 0, 19, 7, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2677, 3799, 0, 8, 2, 3549, 1, 30, 185, 0, 0, 0, 4096, 3125, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 3688, 20, 644, 4078, 1, 445, 7, 2685, 4013, 32, 12, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 3969, 33, 4023, 4090, 42, 0, 0, 0, 0, 4096, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17278, 21633, 18576, 12418, 7352, 17473, 15714, 20628] avg=16384.0000 max=21633 max_rank=1 min=7352 min_rank=4 imbalance_ratio=1.320374 max_load_reduction=109439 max_load_reduction_pct=0.834953 moved_to_echo=61167 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 4, 14, 0, 547, 0, 2334, 4034, 3539, 0, 137, 3723, 3596, 3682, 0, 1879, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 80, 0, 4021, 0, 0, 0, 116, 3906, 0, 58, 2084, 74, 4017, 1179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 4094, 4002, 12, 0, 4096, 0, 0, 0, 0, 1482, 1596, 5, 0, 2228, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 455, 4096, 0, 716, 0, 60, 0, 312, 1411, 3, 0, 145, 3073, 2558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1134, 3113, 3766, 11, 2833, 0, 1044, 0, 0, 29, 1942, 0, 207, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3440, 0, 0, 36, 399, 0, 705, 2879, 262, 0, 0, 205, 3980, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 3422, 4074, 0, 233, 2963, 0, 1326, 4091, 0, 0, 21, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4082, 0, 3586, 0, 0, 4031, 0, 2533, 4076, 0, 0, 0, 0, 417, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23857, 15607, 17573, 12906, 14135, 11907, 16149, 18938] avg=16384.0000 max=23857 max_rank=0 min=11907 min_rank=5 imbalance_ratio=1.456116 max_load_reduction=107215 max_load_reduction_pct=0.817986 moved_to_echo=61129 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 77, 56, 3, 0, 187, 267, 278, 0, 0, 1974, 0, 4084, 0, 0, 0, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 1114, 0, 0, 0, 0, 1, 1, 0, 20, 244, 0, 3358, 3303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 908, 4052, 0, 3765, 57, 3785, 0, 0, 0, 0, 0, 433, 0, 4, 0, 2, 2175, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 420, 3695, 1379, 0, 34, 12, 2536, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 6, 1, 4096, 2890, 0, 3796, 0, 6, 0, 0, 132, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3653, 4020, 0, 1008, 36, 3671, 69, 3122, 4096, 1, 2207, 3666, 3996, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3437, 0, 0, 4013, 4096, 652, 3964, 0, 0, 4096, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1745, 0, 0, 7, 454, 4023, 0, 3591, 0, 8, 70, 4049, 668, 3, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7282, 8063, 15321, 16266, 19033, 29547, 20499, 15061] avg=16384.0000 max=29547 max_rank=5 min=7282 min_rank=0 imbalance_ratio=1.803406 max_load_reduction=101525 max_load_reduction_pct=0.774574 moved_to_echo=84140 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3332, 431, 3721, 0, 0, 0, 5, 2538, 0, 0, 4006, 0, 0, 1, 0, 1841, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3196, 0, 0, 0, 0, 1487, 12, 0, 0, 0, 4096, 0, 4066, 3304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3769, 2122, 1072, 32, 4, 539, 92, 1, 0, 1384, 2, 0, 0, 2285, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 3977, 22, 809, 0, 4073, 1, 119, 1448, 0, 14, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2852, 4091, 0, 3301, 209, 43, 0, 0, 0, 0, 1901, 4096, 1, 1179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3295, 106, 0, 0, 0, 429, 0, 0, 2424, 0, 1881, 4096, 2274, 68, 2, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1937, 3928, 0, 3763, 652, 4007, 18, 0, 1263, 0, 32, 0, 0, 2769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 5, 0, 10, 0, 3978, 4096, 0, 4096, 2, 1020, 1, 0, 9, 0, 0, 400, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16238, 16161, 15538, 18643, 17673, 14604, 18369, 13846] avg=16384.0000 max=18643 max_rank=3 min=13846 min_rank=7 imbalance_ratio=1.137878 max_load_reduction=112429 max_load_reduction_pct=0.857765 moved_to_echo=64492 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[384, 7680, 2944, 19328, 19968, 22400, 4352, 27136, 128, 2560, 13696, 2688, 4352, 128, 1152, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1920, 15616, 7168, 12032, 4480, 768, 256, 6784, 13696, 7296, 256, 25984, 3840, 14848, 4480, 0, 9600, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 128, 128, 3200, 3328, 2432, 18816, 128, 0, 17024, 6784, 13312, 128, 512, 17152, 512, 26496, 21888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[11136, 1664, 24576, 768, 7552, 128, 8576, 9216, 4992, 11392, 128, 384, 640, 21888, 28672, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[20352, 20224, 128, 4736, 6656, 15616, 1152, 10496, 8448, 128, 7680, 10496, 22528, 3328, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1024, 12288, 128, 25856, 11136, 5888, 256, 8832, 8320, 4864, 512, 640, 11776, 13696, 3968, 4096, 18048, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[18432, 11392, 15744, 11648, 12160, 10240, 6784, 4096, 0, 4352, 9984, 8192, 1152, 13440, 4352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[2176, 1920, 384, 11392, 128, 23040, 13440, 128, 27776, 4096, 9472, 14464, 7936, 2944, 5248, 4352, 2560, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[38, 0, 4096, 914, 0, 889, 1298, 872, 3811, 31, 1505, 4026, 43, 0, 1, 1821, 4081, 0, 0, 54, 0, 17, 581, 0, 9, 1239, 0, 11, 40, 1107, 38, 199, 0, 3, 0, 3411, 59, 0, 3825, 0, 0, 0, 4096, 3404, 10, 94, 0, 0, 0, 0, 4094, 3715, 2905, 3753, 0, 1093, 3770, 0, 0, 7, 0, 350, 0, 4039, 0, 3964, 0, 0, 3, 0, 483, 4092, 5, 0, 938, 13, 4096, 31, 1, 300, 0, 2345, 0, 5, 0, 11, 4076, 131, 0, 1372, 107, 2097, 0, 0, 201, 1522, 226, 0, 0, 4096, 0, 0, 44, 0, 0, 3322, 0, 0, 3538, 0, 3291, 713, 27, 2981, 3793, 3937, 0, 0, 3906, 254, 4069, 0, 0, 651, 687, 294, 5, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[3952, 5, 1387, 0, 0, 908, 0, 2, 0, 0, 4056, 0, 0, 3827, 312, 99, 3642, 10, 0, 82, 0, 0, 0, 0, 0, 0, 1744, 0, 28, 0, 4, 382, 4096, 0, 1437, 0, 0, 0, 0, 2, 0, 0, 4081, 2592, 2, 1, 0, 0, 0, 4085, 148, 3744, 0, 2304, 3940, 0, 0, 3104, 55, 0, 0, 5, 4, 3246, 0, 4096, 48, 1894, 0, 0, 46, 4040, 0, 0, 4085, 0, 3, 1, 31, 4096, 0, 0, 3, 4094, 2752, 558, 1437, 105, 0, 0, 145, 31, 4096, 0, 0, 1170, 0, 3086, 0, 790, 12, 3, 3568, 0, 4095, 1201, 15, 0, 4066, 3578, 229, 6, 0, 3869, 4034, 4089, 0, 42, 0, 4094, 3973, 6, 142, 0, 0, 91, 37, 4029] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[3592, 0, 0, 2004, 0, 0, 59, 0, 3, 14, 4096, 144, 0, 0, 0, 71, 0, 0, 0, 48, 7, 0, 4090, 0, 0, 0, 0, 0, 0, 42, 435, 933, 3774, 0, 17, 4047, 1, 0, 1757, 0, 0, 0, 3980, 0, 2, 0, 1984, 36, 514, 658, 4096, 2350, 9, 4077, 21, 881, 0, 4062, 3240, 3530, 0, 0, 0, 4096, 0, 0, 39, 0, 0, 0, 0, 3969, 116, 0, 1831, 36, 4016, 0, 5, 848, 0, 4077, 1001, 1310, 0, 1757, 2994, 0, 47, 728, 2, 16, 3714, 87, 0, 3602, 106, 0, 0, 2678, 0, 12, 3199, 0, 281, 0, 0, 0, 4094, 0, 3238, 4096, 1, 4025, 3940, 51, 12, 65, 166, 4096, 4079, 0, 0, 0, 4091, 3977, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[876, 65, 2789, 4094, 567, 3111, 0, 39, 0, 0, 4096, 5, 200, 0, 498, 0, 4029, 0, 0, 3880, 182, 291, 357, 66, 38, 8, 0, 0, 3678, 0, 0, 1785, 0, 1549, 0, 3454, 139, 0, 389, 86, 4096, 3434, 0, 0, 25, 0, 0, 5, 0, 2, 81, 0, 19, 184, 3592, 18, 3334, 2763, 0, 1, 528, 0, 4051, 4023, 6, 4096, 3883, 0, 0, 15, 0, 3291, 138, 0, 0, 75, 4068, 2359, 11, 0, 1092, 3838, 85, 2, 0, 3, 2751, 4081, 3643, 1, 80, 8, 30, 1259, 0, 0, 0, 2, 0, 1327, 0, 1, 4068, 0, 0, 4059, 0, 0, 4078, 0, 3863, 774, 1, 0, 3194, 0, 4095, 0, 230, 4040, 0, 0, 2, 23, 0, 4010, 1, 62] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[2246, 0, 4014, 0, 7, 20, 1, 0, 0, 789, 3860, 3260, 0, 0, 0, 0, 4090, 0, 0, 140, 0, 0, 0, 54, 0, 1522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 1, 4072, 3708, 4096, 3421, 4010, 4093, 1, 718, 518, 4094, 0, 0, 4096, 0, 4096, 4048, 0, 2, 4050, 4096, 5, 1, 14, 0, 2, 1045, 4015, 7, 0, 70, 0, 79, 3, 4077, 0, 3432, 158, 0, 3144, 0, 0, 0, 739, 0, 4075, 65, 3652, 1150, 1582, 0, 0, 4001, 26, 0, 1, 43, 0, 0, 836, 21, 15, 4096, 3, 130, 0, 0, 0, 3599, 0, 0, 532, 2, 5, 1745, 0, 0, 4056, 4086, 0, 0, 0, 3984, 699, 0, 0, 0, 2, 4095, 50, 2559] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[4002, 0, 0, 6, 0, 0, 1161, 4089, 0, 0, 3072, 153, 4, 0, 0, 0, 4096, 241, 0, 8, 0, 6, 0, 0, 55, 4095, 57, 0, 0, 0, 0, 328, 4, 0, 29, 4096, 668, 4060, 0, 499, 1464, 0, 4096, 4094, 0, 8, 0, 0, 0, 0, 892, 0, 0, 4096, 19, 3821, 1, 532, 0, 65, 0, 201, 0, 226, 0, 3516, 3420, 1, 0, 0, 0, 4091, 0, 0, 663, 3974, 2300, 0, 0, 0, 35, 4096, 0, 123, 4092, 0, 3803, 4081, 4, 5, 35, 392, 0, 0, 4096, 4053, 132, 2634, 0, 4096, 884, 0, 3295, 0, 0, 3862, 0, 0, 3682, 0, 3754, 0, 0, 0, 0, 4095, 0, 0, 0, 3352, 4029, 18, 53, 0, 0, 64, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[3605, 10, 4084, 4, 4006, 3036, 0, 18, 0, 0, 4074, 4044, 0, 0, 5, 0, 7, 48, 20, 4091, 0, 0, 0, 191, 3983, 0, 3942, 0, 703, 0, 0, 15, 171, 0, 0, 3285, 2, 1, 0, 28, 116, 26, 471, 4031, 292, 6, 9, 3638, 4091, 0, 3892, 1545, 91, 3795, 17, 1, 3937, 4096, 1409, 0, 0, 0, 0, 4096, 1, 4027, 590, 0, 0, 0, 3927, 4096, 2975, 0, 4044, 0, 0, 0, 58, 0, 277, 3950, 0, 1, 0, 22, 1139, 4088, 0, 0, 67, 14, 22, 196, 0, 0, 0, 0, 4, 1642, 0, 517, 0, 881, 113, 8, 0, 0, 4080, 24, 4096, 0, 0, 4092, 2949, 3405, 0, 0, 0, 0, 6, 5, 210, 7, 4060, 543, 1, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[320, 0, 4036, 3966, 54, 2968, 3995, 0, 4096, 0, 2752, 174, 0, 0, 0, 0, 2866, 0, 345, 831, 0, 418, 0, 0, 205, 4091, 3501, 0, 0, 81, 0, 0, 0, 4, 1, 4095, 0, 4014, 200, 0, 3737, 775, 4003, 4096, 4, 147, 10, 982, 4057, 0, 9, 3296, 0, 4092, 1339, 227, 347, 1438, 4003, 0, 0, 3, 7, 4079, 0, 9, 0, 0, 0, 0, 0, 238, 92, 313, 1871, 61, 4073, 0, 2, 4089, 91, 3983, 0, 104, 11, 778, 0, 0, 0, 0, 0, 246, 0, 0, 737, 1, 0, 0, 0, 4054, 0, 0, 3780, 0, 7, 3943, 0, 0, 3848, 0, 0, 582, 2, 3477, 4089, 2, 0, 0, 0, 3464, 1552, 0, 0, 0, 749, 3046, 0, 2114] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3952, 5, 1387, 0, 0, 908, 0, 2, 0, 0, 3898, 0, 0, 3827, 312, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3642, 10, 0, 82, 0, 0, 0, 0, 0, 0, 1744, 0, 28, 0, 4, 382, 2304, 4034, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1437, 0, 0, 0, 0, 2, 0, 0, 2967, 2592, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 148, 3744, 0, 0, 3940, 0, 0, 3104, 55, 0, 0, 5, 4, 1386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 48, 1894, 0, 0, 46, 4040, 0, 0, 4085, 0, 3, 1, 31, 4096, 457, 230, 203, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 2752, 558, 1437, 105, 0, 0, 145, 31, 4096, 0, 0, 1170, 1142, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3086, 0, 790, 12, 3, 3568, 0, 4095, 1201, 15, 0, 3836, 3578, 229, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3869, 0, 4089, 0, 42, 0, 3891, 3973, 6, 142, 0, 0, 91, 37, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14390, 12948, 11097, 16471, 19388, 16190, 20419, 20169] avg=16384.0000 max=20419 max_rank=6 min=11097 min_rank=2 imbalance_ratio=1.246277 max_load_reduction=110653 max_load_reduction_pct=0.844215 moved_to_echo=76166 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3592, 0, 0, 2004, 0, 0, 59, 0, 3, 14, 3936, 144, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 7, 0, 4090, 0, 0, 0, 0, 0, 0, 42, 435, 933, 4077, 3940, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3774, 0, 17, 4047, 1, 0, 1757, 0, 0, 0, 2894, 0, 2, 0, 1984, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 658, 4096, 2350, 9, 0, 21, 881, 0, 4062, 3240, 3530, 0, 0, 0, 1749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 3969, 116, 0, 1831, 36, 4016, 0, 5, 848, 445, 232, 203, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 1001, 1310, 0, 1757, 2994, 0, 47, 728, 2, 16, 3714, 87, 0, 3602, 1441, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 2678, 0, 12, 3199, 0, 281, 0, 0, 0, 3862, 0, 3238, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4025, 0, 51, 12, 65, 166, 3893, 4079, 0, 0, 0, 4091, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9823, 14478, 14512, 21110, 11900, 21417, 17472, 20360] avg=16384.0000 max=21417 max_rank=5 min=9823 min_rank=0 imbalance_ratio=1.307190 max_load_reduction=109655 max_load_reduction_pct=0.836601 moved_to_echo=71149 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[38, 0, 4096, 914, 0, 889, 1298, 872, 3811, 31, 1447, 4026, 43, 0, 1, 1821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 54, 0, 17, 581, 0, 9, 1239, 0, 11, 40, 1107, 38, 199, 3753, 3793, 894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3411, 59, 0, 3825, 0, 0, 0, 2978, 3404, 10, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3715, 2905, 0, 0, 1093, 3770, 0, 0, 7, 0, 350, 0, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 0, 0, 3, 0, 483, 4092, 5, 0, 938, 13, 4096, 31, 1, 300, 458, 200, 12, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2345, 0, 5, 0, 11, 4076, 131, 0, 1372, 107, 2097, 0, 0, 201, 1522, 1421, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 4096, 0, 0, 44, 0, 0, 3322, 0, 0, 3338, 0, 3291, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2981, 0, 3937, 0, 0, 3906, 242, 4069, 0, 0, 651, 687, 294, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19287, 15816, 13784, 17658, 14654, 13948, 15030, 20895] avg=16384.0000 max=20895 max_rank=7 min=13784 min_rank=2 imbalance_ratio=1.275330 max_load_reduction=110177 max_load_reduction_pct=0.840584 moved_to_echo=64527 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2246, 0, 4014, 0, 7, 20, 1, 0, 0, 789, 3707, 3260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 140, 0, 0, 0, 54, 0, 1522, 0, 0, 0, 0, 0, 0, 4048, 4056, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 1, 4072, 3708, 4096, 3421, 2908, 4093, 1, 718, 518, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 4050, 4096, 5, 1, 14, 0, 2, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 7, 0, 70, 0, 79, 3, 4077, 0, 3432, 158, 0, 3144, 0, 0, 0, 454, 34, 201, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 739, 0, 4075, 65, 3652, 1150, 1582, 0, 0, 4001, 26, 0, 1, 43, 0, 0, 370, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 836, 21, 15, 4096, 3, 130, 0, 0, 0, 3599, 0, 0, 498, 2, 5, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 3783, 699, 0, 0, 0, 2, 4095, 50, 2559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14044, 14145, 27678, 16802, 15827, 16352, 10950, 15274] avg=16384.0000 max=27678 max_rank=2 min=10950 min_rank=6 imbalance_ratio=1.689331 max_load_reduction=103394 max_load_reduction_pct=0.788834 moved_to_echo=58403 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4002, 0, 0, 6, 0, 0, 1161, 4089, 0, 0, 2952, 153, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 241, 0, 8, 0, 6, 0, 0, 55, 4095, 57, 0, 0, 0, 0, 328, 4096, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 29, 4096, 668, 4060, 0, 499, 1464, 0, 2978, 4094, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 892, 0, 0, 0, 19, 3821, 1, 532, 0, 65, 0, 201, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3516, 3420, 1, 0, 0, 0, 4091, 0, 0, 663, 3974, 2300, 0, 0, 0, 458, 209, 166, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4096, 0, 123, 4092, 0, 3803, 4081, 4, 5, 35, 392, 0, 0, 4096, 4053, 79, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 2634, 0, 4096, 884, 0, 3295, 0, 0, 3862, 0, 0, 3473, 0, 3754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 3186, 4029, 18, 53, 0, 0, 64, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12367, 13032, 17900, 5628, 18918, 25554, 22130, 15543] avg=16384.0000 max=25554 max_rank=5 min=5628 min_rank=3 imbalance_ratio=1.559692 max_load_reduction=105518 max_load_reduction_pct=0.805038 moved_to_echo=82145 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3605, 10, 4084, 4, 4006, 3036, 0, 18, 0, 0, 3915, 4044, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 48, 20, 4091, 0, 0, 0, 191, 3983, 0, 3942, 0, 703, 0, 0, 15, 3795, 2949, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 3285, 2, 1, 0, 28, 116, 26, 344, 4031, 292, 6, 9, 3638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3892, 1545, 91, 0, 17, 1, 3937, 4096, 1409, 0, 0, 0, 0, 1749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4027, 590, 0, 0, 0, 3927, 4096, 2975, 0, 4044, 0, 0, 0, 58, 0, 52, 231, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 3950, 0, 1, 0, 22, 1139, 4088, 0, 0, 67, 14, 22, 196, 0, 0, 1441, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1642, 0, 517, 0, 881, 113, 8, 0, 0, 3849, 24, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3405, 0, 0, 0, 0, 6, 5, 210, 7, 4060, 543, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22727, 20650, 11949, 20828, 20160, 11292, 11134, 12332] avg=16384.0000 max=22727 max_rank=0 min=11134 min_rank=6 imbalance_ratio=1.387146 max_load_reduction=108345 max_load_reduction_pct=0.826607 moved_to_echo=54918 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[876, 65, 2789, 4094, 567, 3111, 0, 39, 0, 0, 3936, 5, 200, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 3880, 182, 291, 357, 66, 38, 8, 0, 0, 3678, 0, 0, 1785, 184, 3194, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1549, 0, 3454, 139, 0, 389, 86, 4096, 3434, 0, 0, 25, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 81, 0, 19, 0, 3592, 18, 3334, 2763, 0, 1, 528, 0, 4051, 1718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 3883, 0, 0, 15, 0, 3291, 138, 0, 0, 75, 4068, 2359, 11, 0, 0, 231, 200, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 3838, 85, 2, 0, 3, 2751, 4081, 3643, 1, 80, 8, 30, 1259, 0, 0, 1415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1327, 0, 1, 4068, 0, 0, 4059, 0, 0, 3847, 0, 3863, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 230, 3840, 0, 0, 2, 23, 0, 4010, 1, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16180, 18582, 13177, 16107, 18533, 18288, 17941, 12264] avg=16384.0000 max=18582 max_rank=1 min=12264 min_rank=7 imbalance_ratio=1.134155 max_load_reduction=112490 max_load_reduction_pct=0.858231 moved_to_echo=67026 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[320, 0, 4036, 3966, 54, 2968, 3995, 0, 4096, 0, 2645, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2866, 0, 345, 831, 0, 418, 0, 0, 205, 4091, 3501, 0, 0, 81, 0, 0, 4092, 4089, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 4095, 0, 4014, 200, 0, 3737, 775, 2910, 4096, 4, 147, 10, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 9, 3296, 0, 0, 1339, 227, 347, 1438, 4003, 0, 0, 3, 7, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 238, 92, 313, 1871, 61, 4073, 0, 2, 4089, 448, 218, 171, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 3983, 0, 104, 11, 778, 0, 0, 0, 0, 0, 246, 0, 0, 737, 1, 1435, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 3780, 0, 7, 3943, 0, 0, 3630, 0, 0, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3477, 0, 2, 0, 0, 0, 3293, 1552, 0, 0, 0, 749, 3046, 0, 2114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22254, 21421, 20975, 16468, 11692, 8031, 15996, 14235] avg=16384.0000 max=22254 max_rank=0 min=8031 min_rank=5 imbalance_ratio=1.358276 max_load_reduction=108818 max_load_reduction_pct=0.830215 moved_to_echo=49954 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[8704, 4864, 17408, 14720, 7168, 0, 8960, 6144, 15488, 20096, 8832, 3712, 640, 640, 4096, 10624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[8064, 1664, 1536, 22528, 896, 8192, 10368, 4352, 13568, 7680, 18048, 22400, 384, 1024, 2560, 8832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 18560, 0, 19712, 4224, 128, 4352, 22144, 18432, 128, 512, 768, 9600, 16128, 128, 17024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[18688, 128, 20480, 11008, 4736, 11008, 6528, 5120, 7936, 896, 26496, 11904, 256, 3840, 896, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1408, 5760, 128, 22784, 1024, 768, 18048, 896, 4608, 20096, 128, 0, 26368, 3712, 18560, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2304, 22400, 5248, 5760, 10624, 4352, 17792, 12544, 3712, 6144, 512, 2816, 7936, 1664, 5120, 10368, 8832, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[4096, 19840, 8064, 2048, 128, 128, 4480, 27904, 3328, 3840, 13696, 4224, 21760, 2432, 128, 9344, 2816, 1664, 1280, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[22912, 384, 384, 9216, 256, 768, 5120, 384, 4352, 11008, 9344, 128, 4480, 1280, 512, 3712, 26368, 26112, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[15, 0, 0, 0, 3088, 773, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 142, 103, 105, 0, 4079, 0, 15, 4096, 1729, 0, 4042, 2294, 0, 4044, 39, 6, 21, 15, 3731, 1991, 7, 103, 0, 6, 0, 412, 0, 2, 60, 389, 0, 4019, 4051, 0, 109, 0, 3121, 0, 0, 0, 845, 309, 4067, 24, 78, 0, 151, 272, 1866, 3956, 0, 3, 4095, 3529, 2, 0, 3966, 0, 168, 78, 2769, 46, 195, 0, 0, 0, 2, 1280, 1938, 995, 250, 260, 3, 0, 0, 3907, 0, 4077, 0, 4095, 5, 0, 4080, 941, 0, 2020, 12, 156, 3897, 0, 0, 0, 0, 2679, 4096, 0, 4082, 4096, 693, 49, 0, 3796, 0, 46, 1545, 667, 0, 4072, 1, 103, 4, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[5, 1969, 45, 1381, 0, 4094, 2707, 0, 1126, 0, 3630, 0, 77, 3277, 704, 85, 8, 13, 4076, 0, 34, 0, 716, 0, 4, 0, 0, 59, 0, 3476, 0, 0, 3, 0, 4096, 62, 0, 0, 4095, 2869, 0, 3957, 0, 7, 0, 520, 449, 815, 27, 26, 0, 0, 0, 0, 133, 0, 4032, 4096, 0, 21, 0, 1465, 0, 0, 3461, 4002, 1299, 21, 16, 0, 0, 2480, 4096, 0, 1, 3, 5, 1623, 4075, 27, 0, 0, 0, 25, 1, 3674, 0, 0, 166, 20, 0, 4096, 20, 2332, 20, 0, 0, 0, 1666, 1506, 1444, 0, 0, 0, 3949, 0, 4096, 4, 3409, 3, 0, 0, 4083, 2, 4096, 2, 0, 0, 825, 0, 4066, 4025, 0, 4096, 4089, 0, 0, 4089] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[3908, 17, 7, 29, 1986, 203, 6, 0, 3088, 143, 0, 0, 1633, 130, 0, 0, 0, 0, 2942, 3723, 4093, 35, 39, 4066, 1, 30, 4096, 825, 0, 0, 6, 0, 1169, 0, 4096, 0, 94, 0, 0, 3320, 4, 4011, 0, 0, 429, 0, 0, 1858, 0, 47, 4096, 51, 4057, 0, 0, 1994, 4096, 0, 1299, 3, 0, 849, 6, 0, 0, 4076, 0, 0, 14, 0, 0, 0, 4022, 0, 12, 1, 4, 2, 530, 0, 91, 0, 0, 0, 5, 427, 0, 1, 0, 0, 0, 4035, 0, 4096, 2, 0, 0, 0, 4094, 3981, 1325, 256, 440, 3681, 4091, 0, 4094, 0, 3957, 2245, 1, 0, 3722, 35, 3966, 0, 0, 0, 4092, 0, 4096, 1585, 0, 3027, 834, 1, 0, 1746] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[2346, 3344, 0, 1643, 0, 167, 0, 0, 0, 0, 104, 83, 0, 86, 8, 0, 115, 0, 3999, 3843, 0, 189, 0, 4095, 3, 0, 3787, 4062, 0, 0, 11, 633, 0, 0, 4074, 847, 0, 62, 4096, 4096, 272, 0, 0, 15, 5, 3740, 1, 3680, 0, 0, 0, 57, 0, 0, 0, 899, 3009, 0, 2065, 0, 0, 0, 1, 0, 418, 3664, 48, 406, 7, 2294, 19, 0, 4096, 0, 0, 4, 0, 4089, 0, 34, 226, 0, 0, 13, 4049, 4044, 944, 0, 43, 432, 0, 4095, 0, 7, 0, 0, 0, 35, 3913, 1014, 164, 0, 25, 0, 4033, 0, 0, 370, 0, 3943, 3967, 18, 3634, 473, 4096, 0, 0, 3009, 0, 24, 3589, 0, 2358, 3868, 4078, 0, 194, 3898] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 5, 1962, 0, 146, 0, 0, 0, 4, 708, 0, 0, 1102, 1, 4, 0, 0, 0, 128, 4096, 46, 33, 0, 3776, 1161, 4, 4094, 3645, 0, 589, 0, 6, 1, 0, 1267, 0, 0, 0, 0, 0, 0, 4096, 50, 671, 5, 4, 437, 4065, 0, 0, 1052, 395, 3852, 0, 0, 0, 4096, 0, 0, 4089, 0, 699, 0, 0, 3875, 4078, 1748, 0, 0, 225, 0, 4074, 93, 0, 4087, 0, 0, 2933, 4039, 0, 0, 0, 1, 3258, 0, 3949, 0, 0, 0, 3383, 0, 3977, 340, 4096, 0, 0, 0, 0, 4031, 2091, 0, 0, 4073, 4096, 3974, 3277, 1, 278, 4096, 4039, 0, 0, 26, 461, 225, 0, 83, 0, 268, 0, 362, 328, 0, 1330, 3915, 0, 0, 3673] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[3233, 3791, 26, 2936, 0, 0, 243, 0, 0, 3968, 0, 4096, 0, 1999, 2280, 0, 314, 0, 2537, 4057, 0, 294, 0, 4081, 1, 0, 4059, 4088, 0, 0, 0, 3552, 0, 6, 119, 0, 2799, 1925, 47, 3991, 3062, 80, 0, 17, 0, 0, 812, 0, 0, 0, 13, 0, 4091, 0, 0, 2, 178, 0, 0, 0, 0, 3, 38, 1, 4079, 4095, 0, 0, 72, 0, 0, 1, 4095, 0, 4091, 0, 0, 0, 666, 76, 0, 5, 0, 9, 2030, 345, 0, 0, 0, 0, 0, 2036, 1099, 4096, 0, 0, 0, 3990, 4087, 3136, 4096, 21, 12, 0, 1909, 125, 1301, 0, 1604, 243, 0, 5, 0, 2930, 0, 14, 5, 19, 3982, 0, 4077, 0, 0, 4080, 1836, 0, 0, 4066] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[115, 2260, 0, 0, 0, 3605, 0, 448, 4052, 4094, 461, 354, 0, 0, 1, 0, 0, 1013, 5, 10, 8, 0, 0, 4090, 0, 0, 3788, 12, 0, 4002, 0, 0, 0, 0, 4002, 109, 0, 0, 0, 4072, 4, 3675, 0, 205, 0, 0, 258, 106, 242, 0, 10, 2032, 0, 0, 0, 0, 1987, 0, 0, 0, 0, 4095, 279, 135, 6, 3, 3690, 0, 4096, 2501, 28, 0, 4005, 1, 4066, 3897, 0, 83, 4089, 24, 43, 0, 0, 4096, 0, 0, 801, 0, 11, 4, 4093, 2464, 0, 13, 53, 0, 0, 3, 4096, 3941, 0, 181, 464, 4041, 4096, 0, 0, 74, 0, 0, 1928, 0, 87, 3694, 0, 4047, 3895, 21, 3229, 35, 75, 10, 0, 3196, 1604, 4096, 66, 597] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[31, 4021, 2, 149, 37, 0, 0, 0, 546, 2562, 0, 0, 20, 22, 0, 4, 268, 0, 36, 157, 602, 0, 71, 4096, 0, 0, 2228, 1293, 1651, 1, 0, 4081, 116, 3401, 3994, 0, 0, 18, 2132, 3832, 0, 4086, 0, 0, 0, 3244, 0, 4095, 1200, 0, 159, 0, 4080, 0, 0, 21, 0, 0, 4, 0, 15, 872, 0, 0, 3884, 4093, 34, 67, 6, 1530, 63, 12, 2516, 13, 2, 4055, 7, 129, 4036, 0, 0, 4, 3148, 4056, 722, 28, 0, 14, 0, 6, 112, 3968, 26, 4096, 0, 0, 0, 0, 4077, 177, 0, 0, 4090, 1381, 262, 0, 3985, 0, 4057, 1603, 245, 0, 0, 4096, 208, 1210, 0, 14, 0, 0, 4092, 4, 0, 4096, 3684, 0, 0, 4017] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[5, 1969, 45, 1381, 0, 4094, 2707, 0, 1126, 0, 3630, 0, 77, 3277, 704, 85, 3726, 883, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 13, 4076, 0, 34, 0, 716, 0, 4, 0, 0, 59, 0, 3476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 62, 0, 0, 4095, 2869, 0, 3957, 0, 7, 0, 520, 449, 815, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 26, 0, 0, 0, 0, 133, 0, 4032, 4096, 0, 21, 0, 1465, 0, 0, 1666, 3251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3461, 610, 1299, 21, 16, 0, 0, 2480, 4096, 0, 1, 3, 5, 1623, 4075, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 1, 3674, 0, 0, 166, 20, 0, 4096, 20, 2332, 20, 0, 2509, 1127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1506, 1444, 0, 0, 0, 2793, 0, 4096, 4, 3409, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 2, 4096, 2, 0, 0, 825, 0, 4066, 4025, 0, 0, 4089, 0, 0, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[24554, 8386, 16902, 14717, 17717, 13990, 13255, 21551] avg=16384.0000 max=24554 max_rank=0 min=8386 min_rank=1 imbalance_ratio=1.498657 max_load_reduction=106518 max_load_reduction_pct=0.812668 moved_to_echo=66513 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[15, 0, 0, 0, 3088, 773, 0, 0, 4096, 0, 0, 0, 12, 0, 0, 0, 3728, 873, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 103, 105, 0, 4079, 0, 15, 3145, 1729, 0, 4042, 2294, 0, 4044, 39, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 15, 3731, 1991, 7, 103, 0, 6, 0, 412, 0, 2, 60, 389, 0, 4019, 951, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 109, 0, 3121, 0, 0, 0, 845, 309, 4067, 24, 78, 0, 151, 272, 4080, 3232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1866, 603, 0, 3, 4095, 3529, 2, 0, 3966, 0, 168, 78, 2769, 46, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1280, 1938, 995, 250, 260, 3, 0, 0, 3907, 0, 4077, 0, 4095, 2480, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 941, 0, 2020, 12, 156, 2756, 0, 0, 0, 0, 2679, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 693, 49, 0, 3796, 0, 46, 1545, 667, 0, 0, 1, 103, 4, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13425, 19743, 11736, 20339, 17320, 20399, 12665, 15445] avg=16384.0000 max=20399 max_rank=5 min=11736 min_rank=2 imbalance_ratio=1.245056 max_load_reduction=110673 max_load_reduction_pct=0.844368 moved_to_echo=65829 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3908, 17, 7, 29, 1986, 203, 6, 0, 3088, 143, 0, 0, 1633, 130, 0, 0, 1591, 900, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2942, 3723, 4093, 35, 39, 3122, 1, 30, 4096, 825, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1169, 0, 4096, 0, 94, 0, 0, 3320, 4, 4011, 0, 0, 429, 0, 0, 1858, 944, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4096, 51, 4057, 0, 0, 1994, 4096, 0, 1299, 3, 0, 849, 6, 0, 4094, 2402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 14, 0, 0, 0, 4022, 0, 12, 1, 4, 2, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 5, 427, 0, 1, 0, 0, 0, 4035, 0, 4096, 2, 0, 2555, 1167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 1325, 256, 440, 3681, 2894, 0, 4094, 0, 3957, 2245, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3722, 35, 3966, 0, 0, 0, 4092, 0, 4096, 1585, 0, 0, 834, 1, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14266, 18912, 15955, 22994, 5206, 12379, 22874, 18486] avg=16384.0000 max=22994 max_rank=3 min=5206 min_rank=4 imbalance_ratio=1.403442 max_load_reduction=108078 max_load_reduction_pct=0.824570 moved_to_echo=58945 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 1962, 0, 146, 0, 0, 0, 4, 708, 0, 0, 1102, 1, 4, 0, 3352, 904, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 4096, 46, 33, 0, 2897, 1161, 4, 4094, 3645, 0, 589, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1267, 0, 0, 0, 0, 0, 0, 4096, 50, 671, 5, 4, 437, 4065, 879, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 395, 3852, 0, 0, 0, 4096, 0, 0, 4089, 0, 699, 0, 0, 4031, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3875, 613, 1748, 0, 0, 225, 0, 4074, 93, 0, 4087, 0, 0, 2933, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3258, 0, 3949, 0, 0, 0, 3383, 0, 3977, 340, 4096, 0, 0, 2561, 1138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2091, 0, 0, 4073, 4096, 2802, 3277, 1, 278, 4096, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 461, 225, 0, 83, 0, 268, 0, 362, 328, 0, 0, 3915, 0, 0, 321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8463, 16699, 11509, 19269, 21687, 22703, 24753, 5989] avg=16384.0000 max=24753 max_rank=6 min=5989 min_rank=7 imbalance_ratio=1.510803 max_load_reduction=106319 max_load_reduction_pct=0.811150 moved_to_echo=75132 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2346, 3344, 0, 1643, 0, 167, 0, 0, 0, 0, 104, 83, 0, 86, 8, 0, 3552, 809, 799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 3999, 3843, 0, 189, 0, 3144, 3, 0, 3787, 4062, 0, 0, 11, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 847, 0, 62, 4096, 4096, 272, 0, 0, 15, 5, 3740, 1, 3680, 951, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 899, 3009, 0, 2065, 0, 0, 0, 1, 0, 3913, 3069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 558, 48, 406, 7, 2294, 19, 0, 4096, 0, 0, 4, 0, 4089, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 13, 4049, 4044, 944, 0, 43, 432, 0, 4095, 0, 7, 0, 0, 2297, 1151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 1014, 164, 0, 25, 0, 2852, 0, 0, 370, 0, 3943, 3967, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3634, 473, 4096, 0, 0, 3009, 0, 24, 3589, 0, 2358, 0, 4078, 0, 194, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12941, 19786, 21869, 13013, 11973, 17301, 12388, 21801] avg=16384.0000 max=21869 max_rank=2 min=11973 min_rank=4 imbalance_ratio=1.334778 max_load_reduction=109203 max_load_reduction_pct=0.833153 moved_to_echo=63463 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[115, 2260, 0, 0, 0, 3605, 0, 448, 4052, 4094, 461, 354, 0, 0, 1, 0, 544, 0, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 5, 10, 8, 0, 0, 3140, 0, 0, 3788, 12, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 109, 0, 0, 0, 4072, 4, 3675, 0, 205, 0, 0, 258, 106, 950, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 242, 0, 10, 2032, 0, 0, 0, 0, 1987, 0, 0, 0, 0, 4095, 279, 135, 4096, 2536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 3690, 0, 4096, 2501, 28, 0, 4005, 1, 4066, 3897, 0, 83, 4089, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 4096, 0, 0, 801, 0, 11, 4, 4093, 2464, 0, 13, 53, 0, 1, 1169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3941, 0, 181, 464, 4041, 2897, 0, 0, 74, 0, 0, 1928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 3694, 0, 4047, 3895, 21, 3229, 35, 75, 10, 0, 0, 1604, 4096, 66, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16594, 11978, 13411, 15412, 26488, 12748, 13529, 20912] avg=16384.0000 max=26488 max_rank=4 min=11978 min_rank=1 imbalance_ratio=1.616699 max_load_reduction=104584 max_load_reduction_pct=0.797913 moved_to_echo=73677 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[31, 4021, 2, 149, 37, 0, 0, 0, 546, 2562, 0, 0, 20, 22, 0, 4, 3661, 904, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 268, 0, 36, 157, 602, 0, 71, 3145, 0, 0, 2228, 1293, 1651, 1, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 3401, 3994, 0, 0, 18, 2132, 3832, 0, 4086, 0, 0, 0, 3244, 0, 4095, 951, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1200, 0, 159, 0, 4080, 0, 0, 21, 0, 0, 4, 0, 15, 872, 0, 0, 4077, 3250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 623, 34, 67, 6, 1530, 63, 12, 2516, 13, 2, 4055, 7, 129, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3148, 4056, 722, 28, 0, 14, 0, 6, 112, 3968, 26, 4096, 0, 0, 2566, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 4090, 1381, 187, 0, 3985, 0, 4057, 1603, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 208, 1210, 0, 14, 0, 0, 4092, 4, 0, 0, 3684, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12805, 13533, 25870, 13678, 16977, 18820, 15725, 13664] avg=16384.0000 max=25870 max_rank=2 min=12805 min_rank=0 imbalance_ratio=1.578979 max_load_reduction=105202 max_load_reduction_pct=0.802628 moved_to_echo=65186 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3233, 3791, 26, 2936, 0, 0, 243, 0, 0, 3968, 0, 4096, 0, 1999, 2280, 0, 3705, 904, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 2537, 4057, 0, 294, 0, 3133, 1, 0, 4059, 4088, 0, 0, 0, 3552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 119, 0, 2799, 1925, 47, 3991, 3062, 80, 0, 17, 0, 0, 812, 0, 948, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4091, 0, 0, 2, 178, 0, 0, 0, 0, 3, 38, 1, 4087, 3237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 624, 0, 0, 72, 0, 0, 1, 4095, 0, 4091, 0, 0, 0, 666, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 9, 2030, 345, 0, 0, 0, 0, 0, 2036, 1099, 4096, 0, 0, 2567, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 0, 3136, 4096, 21, 12, 0, 1350, 125, 1301, 0, 1604, 243, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2930, 0, 14, 5, 19, 3982, 0, 4077, 0, 0, 0, 1836, 0, 0, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[28024, 22035, 13820, 11650, 13704, 12732, 15883, 13224] avg=16384.0000 max=28024 max_rank=0 min=11650 min_rank=3 imbalance_ratio=1.710449 max_load_reduction=103048 max_load_reduction_pct=0.786194 moved_to_echo=55543 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[896, 1152, 13952, 16000, 8960, 640, 896, 21760, 2944, 128, 26112, 16384, 1664, 12160, 128, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 4096, 0, 16896, 7040, 2560, 9216, 13440, 18560, 3456, 13568, 768, 17152, 14848, 10240, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1408, 3456, 25472, 3072, 2944, 2176, 10496, 22272, 3456, 20352, 128, 1024, 512, 7936, 2048, 18688, 6656, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[17664, 4352, 6912, 512, 8320, 10112, 128, 6656, 27008, 128, 12544, 8064, 2816, 8960, 17664, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[5632, 128, 5504, 2560, 19328, 0, 256, 2944, 18304, 4480, 7552, 4224, 128, 8064, 512, 512, 30080, 22144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[9728, 15488, 2048, 6144, 5376, 8960, 3072, 512, 12928, 11520, 4224, 4608, 2944, 5632, 3072, 128, 23936, 6272, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 128, 3200, 12800, 8832, 13568, 2048, 384, 256, 3968, 4224, 28672, 1536, 22912, 128, 4096, 17536, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[15744, 15872, 13312, 5376, 4096, 6912, 12416, 128, 22016, 6656, 2432, 0, 20096, 4224, 384, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[7, 0, 2, 0, 563, 0, 0, 3832, 0, 1, 1393, 0, 0, 1691, 38, 159, 2448, 24, 0, 3794, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 50, 129, 1, 8, 0, 0, 4055, 4096, 0, 4069, 0, 2895, 1665, 0, 127, 3288, 4078, 4076, 0, 25, 50, 543, 3947, 0, 0, 0, 3919, 762, 0, 4083, 4096, 76, 0, 0, 286, 4096, 2856, 0, 4042, 0, 563, 2785, 3396, 0, 0, 858, 3697, 0, 2348, 3675, 4096, 2660, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 370, 0, 2043, 793, 14, 3, 24, 0, 0, 0, 0, 5, 4094, 2453, 60, 0, 0, 3669, 0, 480, 4096, 2694, 0, 367, 0, 28, 18, 0, 0, 4066, 1543, 4, 0, 666, 29] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[248, 90, 0, 0, 0, 0, 0, 2775, 2147, 0, 37, 0, 224, 4075, 0, 0, 1046, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 484, 0, 0, 0, 4031, 0, 2, 2889, 454, 0, 0, 219, 3115, 1, 57, 9, 40, 4086, 4095, 1032, 3, 3381, 4096, 1227, 2, 0, 3703, 4054, 556, 0, 0, 6, 774, 59, 21, 4096, 3440, 87, 0, 1, 0, 12, 0, 3986, 0, 0, 4074, 0, 0, 0, 3835, 3186, 0, 0, 0, 3960, 0, 0, 3999, 1444, 0, 4032, 4096, 0, 0, 9, 0, 0, 0, 3968, 0, 0, 1037, 0, 4096, 86, 4077, 523, 4096, 3880, 4096, 0, 0, 4096, 4096, 0, 0, 56, 0, 0, 161, 0, 0, 0, 2699, 0, 553, 90, 0, 69, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4, 3, 0, 0, 3928, 0, 110, 0, 0, 0, 0, 10, 0, 4060, 0, 2748, 4066, 3989, 561, 247, 0, 0, 115, 6, 0, 0, 4096, 0, 100, 0, 49, 0, 0, 0, 4087, 0, 0, 0, 0, 71, 4013, 4040, 0, 0, 0, 7, 0, 3651, 4083, 3, 0, 1, 0, 72, 0, 204, 80, 0, 0, 4019, 2221, 3743, 13, 3984, 122, 4080, 3638, 0, 0, 1, 0, 2, 120, 276, 0, 0, 3478, 1, 0, 4070, 2918, 4001, 110, 0, 1318, 3669, 0, 3993, 3837, 597, 0, 2, 0, 0, 22, 4047, 4096, 16, 7, 409, 50, 4096, 30, 0, 18, 461, 2399, 0, 4048, 173, 2951, 0, 0, 18, 4096, 0, 298, 0, 93, 1, 0, 0, 4096, 0, 941, 0, 0, 4089] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[162, 4089, 241, 407, 4040, 37, 0, 1, 0, 224, 0, 4096, 3481, 3980, 0, 0, 4093, 4090, 0, 0, 1, 18, 1136, 0, 0, 1148, 2, 1, 0, 5, 3988, 0, 3909, 34, 3, 32, 1, 0, 3095, 0, 0, 10, 31, 12, 3931, 0, 4053, 4041, 0, 0, 4, 0, 4091, 3764, 35, 0, 0, 297, 14, 0, 865, 563, 0, 4090, 0, 0, 30, 51, 203, 0, 0, 0, 869, 0, 2, 6, 10, 0, 3128, 0, 951, 4077, 0, 0, 3978, 4094, 103, 0, 4047, 0, 1, 0, 32, 4094, 0, 4096, 1351, 27, 4082, 3848, 7, 1360, 149, 113, 0, 2, 3298, 18, 18, 0, 4041, 3768, 0, 40, 3934, 0, 201, 2, 3862, 2046, 0, 0, 613, 0, 0, 399, 4, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[21, 4096, 3, 28, 0, 3311, 41, 0, 0, 0, 121, 0, 0, 4096, 4, 0, 4093, 137, 0, 85, 0, 0, 4038, 0, 0, 4, 3965, 1053, 0, 0, 2426, 0, 4096, 0, 44, 0, 0, 4096, 3662, 0, 159, 0, 16, 9, 2030, 0, 4026, 3392, 4064, 0, 14, 0, 4087, 0, 2, 186, 0, 0, 0, 0, 738, 3949, 0, 4079, 85, 98, 0, 3758, 16, 1714, 2, 0, 3941, 3910, 0, 0, 24, 0, 61, 3823, 0, 3674, 0, 0, 3573, 0, 0, 1952, 4066, 0, 0, 75, 3372, 0, 0, 1974, 0, 0, 4096, 0, 4089, 280, 0, 1504, 0, 391, 3870, 0, 0, 4092, 3798, 0, 0, 0, 58, 0, 0, 334, 4096, 0, 539, 0, 155, 0, 2, 186, 18, 1275] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[3, 4080, 391, 0, 1740, 0, 0, 725, 3906, 22, 201, 8, 0, 160, 0, 0, 4077, 90, 4084, 4085, 0, 51, 752, 0, 0, 0, 3845, 0, 0, 0, 4096, 0, 979, 0, 0, 0, 187, 11, 45, 0, 3767, 243, 0, 0, 3468, 4096, 366, 4080, 4096, 17, 4082, 0, 0, 0, 2825, 0, 3, 277, 0, 0, 4095, 4096, 0, 0, 0, 4054, 3536, 111, 0, 4096, 0, 2972, 3467, 3377, 0, 0, 0, 69, 3, 30, 16, 4095, 154, 0, 0, 0, 0, 2529, 4025, 0, 1, 0, 0, 4096, 78, 3208, 621, 0, 70, 0, 0, 1635, 0, 291, 0, 0, 3934, 493, 0, 1, 4055, 37, 187, 565, 326, 2179, 4096, 5, 36, 2711, 2, 844, 0, 0, 0, 17, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[380, 0, 1100, 0, 0, 0, 62, 3696, 0, 0, 0, 35, 0, 4096, 0, 380, 3942, 1836, 4015, 0, 0, 139, 7, 0, 0, 0, 1302, 99, 8, 14, 3163, 281, 233, 4, 0, 0, 0, 3, 4021, 4064, 3899, 3901, 0, 20, 0, 4079, 4096, 3843, 2756, 0, 0, 0, 3411, 0, 14, 30, 0, 0, 0, 142, 0, 1066, 5, 3743, 8, 4081, 4096, 0, 128, 4096, 0, 2958, 314, 4094, 16, 12, 0, 0, 939, 4012, 0, 3651, 0, 0, 3189, 0, 3867, 4, 34, 0, 1708, 13, 0, 3910, 2, 4074, 2258, 207, 0, 0, 2216, 104, 415, 117, 3667, 21, 0, 2182, 1006, 0, 3667, 0, 3, 2, 3850, 0, 58, 1, 1, 45, 0, 0, 2, 0, 123, 7, 2029, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 4069, 0, 0, 2, 0, 16, 0, 0, 78, 0, 0, 0, 2, 3, 0, 4096, 4096, 3434, 1558, 0, 17, 0, 0, 0, 0, 0, 0, 48, 402, 4045, 0, 4093, 1395, 643, 0, 2005, 0, 4094, 0, 2557, 4096, 0, 0, 0, 1, 0, 3996, 4056, 0, 5, 28, 0, 4096, 0, 0, 55, 0, 0, 16, 1481, 3478, 0, 4095, 0, 35, 4, 0, 0, 2, 0, 2977, 4093, 0, 0, 0, 1327, 7, 0, 0, 2, 3389, 0, 0, 4096, 6, 12, 3968, 4096, 5, 26, 0, 0, 4093, 0, 4096, 710, 0, 3349, 113, 4085, 3839, 260, 2, 4026, 3774, 4095, 0, 0, 136, 86, 2461, 0, 1420, 0, 834, 3710, 2, 3700, 16, 98, 0, 0, 0, 0, 0, 33, 33] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 2, 0, 563, 0, 0, 3832, 0, 1, 1393, 0, 0, 1691, 38, 159, 3669, 1475, 841, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2448, 24, 0, 3794, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 50, 1313, 689, 195, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 1, 8, 0, 0, 4055, 2783, 0, 4069, 0, 2895, 1665, 0, 127, 3288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2601, 0, 25, 50, 543, 3947, 0, 0, 0, 3919, 762, 0, 4083, 4096, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 4096, 2856, 0, 4042, 0, 563, 2590, 3396, 0, 0, 858, 3697, 0, 2348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3675, 0, 2660, 0, 0, 0, 0, 0, 3003, 0, 0, 0, 0, 370, 0, 2043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 793, 14, 3, 24, 0, 0, 0, 0, 5, 4094, 2387, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 4096, 2694, 0, 367, 0, 28, 18, 0, 0, 4066, 1543, 4, 0, 666, 29, 4096, 3237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14074, 12689, 19020, 20102, 24732, 11751, 7380, 21324] avg=16384.0000 max=24732 max_rank=4 min=7380 min_rank=6 imbalance_ratio=1.509521 max_load_reduction=106340 max_load_reduction_pct=0.811310 moved_to_echo=65187 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[248, 90, 0, 0, 0, 0, 0, 2775, 2147, 0, 37, 0, 224, 4075, 0, 0, 4096, 1487, 698, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 484, 0, 0, 0, 4031, 0, 1002, 693, 290, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2889, 454, 0, 0, 219, 2113, 1, 57, 9, 40, 4086, 4095, 1032, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2609, 1227, 2, 0, 3703, 4054, 556, 0, 0, 6, 774, 59, 21, 4096, 3440, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 3986, 0, 0, 3784, 0, 0, 0, 3835, 3186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 1444, 0, 4032, 2998, 0, 0, 9, 0, 0, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 0, 4096, 86, 4077, 523, 4096, 3880, 3983, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 161, 0, 0, 0, 2699, 0, 553, 90, 0, 69, 0, 0, 3960, 2683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16282, 11757, 15000, 20634, 14804, 16450, 25874, 10271] avg=16384.0000 max=25874 max_rank=6 min=10271 min_rank=7 imbalance_ratio=1.579224 max_load_reduction=105198 max_load_reduction_pct=0.802597 moved_to_echo=67399 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4, 3, 0, 0, 3928, 0, 110, 0, 0, 0, 0, 10, 0, 4060, 0, 2748, 2951, 1478, 753, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 3989, 561, 247, 0, 0, 115, 6, 0, 0, 4096, 0, 100, 0, 49, 0, 0, 645, 8, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 71, 4013, 4040, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2605, 3, 0, 1, 0, 72, 0, 204, 80, 0, 0, 4019, 2221, 3743, 13, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 4080, 3638, 0, 0, 1, 0, 2, 112, 276, 0, 0, 3478, 1, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2918, 0, 110, 0, 1318, 3669, 0, 3993, 2815, 597, 0, 2, 0, 0, 22, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 7, 409, 50, 4096, 30, 0, 18, 461, 2335, 0, 4048, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4096, 0, 298, 0, 93, 1, 0, 0, 4096, 0, 941, 0, 0, 4089, 4001, 2898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16422, 13946, 12218, 16945, 15780, 19491, 15739, 20531] avg=16384.0000 max=20531 max_rank=7 min=12218 min_rank=2 imbalance_ratio=1.253113 max_load_reduction=110541 max_load_reduction_pct=0.843361 moved_to_echo=71541 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[380, 0, 1100, 0, 0, 0, 62, 3696, 0, 0, 0, 35, 0, 4096, 0, 380, 3667, 997, 794, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3942, 1836, 4015, 0, 0, 139, 7, 0, 0, 0, 1302, 99, 8, 14, 3163, 281, 1289, 5, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 4, 0, 0, 0, 3, 2732, 4064, 3899, 3901, 0, 20, 0, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1759, 0, 0, 0, 3411, 0, 14, 30, 0, 0, 0, 142, 0, 1066, 5, 3743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4081, 4096, 0, 128, 4096, 0, 2958, 292, 4094, 16, 12, 0, 0, 939, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3189, 0, 3867, 4, 26, 0, 1708, 13, 0, 3910, 2, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2258, 207, 0, 0, 2216, 104, 415, 117, 3667, 21, 0, 2182, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 3850, 0, 58, 1, 1, 45, 0, 0, 2, 0, 123, 7, 2029, 0, 3651, 3049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15210, 16122, 23031, 10170, 24732, 16793, 12193, 12821] avg=16384.0000 max=24732 max_rank=4 min=10170 min_rank=3 imbalance_ratio=1.509521 max_load_reduction=106340 max_load_reduction_pct=0.811310 moved_to_echo=66539 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[21, 4096, 3, 28, 0, 3311, 41, 0, 0, 0, 121, 0, 0, 4096, 4, 0, 3798, 1471, 699, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 137, 0, 85, 0, 0, 4038, 0, 0, 4, 3965, 1053, 0, 0, 2426, 0, 1174, 684, 276, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 44, 0, 0, 4096, 2488, 0, 159, 0, 16, 9, 2030, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2593, 0, 14, 0, 4087, 0, 2, 186, 0, 0, 0, 0, 738, 3949, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 98, 0, 3758, 16, 1714, 2, 0, 3665, 3910, 0, 0, 24, 0, 61, 3823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3573, 0, 0, 1952, 2982, 0, 0, 75, 3372, 0, 0, 1974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 280, 0, 1504, 0, 391, 3766, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 334, 4096, 0, 539, 0, 155, 0, 2, 186, 18, 1275, 3674, 2693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18089, 18039, 16964, 15648, 17156, 13928, 18218, 13030] avg=16384.0000 max=18218 max_rank=6 min=13030 min_rank=7 imbalance_ratio=1.111938 max_load_reduction=112854 max_load_reduction_pct=0.861008 moved_to_echo=62332 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[162, 4089, 241, 407, 4040, 37, 0, 1, 0, 224, 0, 4096, 3481, 3980, 0, 0, 4041, 0, 834, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4090, 0, 0, 1, 18, 1136, 0, 0, 1148, 2, 1, 0, 5, 3988, 0, 992, 681, 61, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 34, 3, 32, 1, 0, 2103, 0, 0, 10, 31, 12, 3931, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4091, 3764, 35, 0, 0, 297, 14, 0, 865, 563, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 51, 203, 0, 0, 0, 808, 0, 2, 6, 10, 0, 3128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 0, 0, 0, 3978, 4094, 103, 0, 2968, 0, 1, 0, 32, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1351, 27, 4082, 3848, 7, 1360, 149, 113, 0, 2, 3209, 18, 18, 0, 0, 3768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 3934, 0, 201, 2, 3862, 2046, 0, 0, 613, 0, 0, 399, 4, 2, 4077, 3207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[26031, 16305, 14119, 13723, 4238, 20317, 17952, 18387] avg=16384.0000 max=26031 max_rank=0 min=4238 min_rank=4 imbalance_ratio=1.588806 max_load_reduction=105041 max_load_reduction_pct=0.801399 moved_to_echo=60894 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 4080, 391, 0, 1740, 0, 0, 725, 3906, 22, 201, 8, 0, 160, 0, 0, 4055, 1483, 842, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 90, 4084, 4085, 0, 51, 752, 0, 0, 0, 3845, 0, 0, 0, 4096, 0, 14, 677, 243, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 979, 0, 0, 0, 187, 11, 31, 0, 3767, 243, 0, 0, 3468, 4096, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2613, 17, 4082, 0, 0, 0, 2825, 0, 3, 277, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 3536, 111, 0, 4096, 0, 2972, 3224, 3377, 0, 0, 0, 69, 3, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 154, 0, 0, 0, 0, 2529, 2952, 0, 1, 0, 0, 4096, 78, 3208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 70, 0, 0, 1635, 0, 291, 0, 0, 3828, 493, 0, 1, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 565, 326, 2179, 4096, 5, 36, 2711, 2, 844, 0, 0, 0, 17, 0, 1, 4095, 3238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18012, 22120, 13148, 18008, 21472, 13034, 6976, 18302] avg=16384.0000 max=22120 max_rank=1 min=6976 min_rank=6 imbalance_ratio=1.350098 max_load_reduction=108952 max_load_reduction_pct=0.831238 moved_to_echo=59784 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4069, 0, 0, 2, 0, 16, 0, 0, 78, 0, 0, 0, 2, 3, 0, 86, 1468, 825, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3434, 1558, 0, 17, 0, 0, 0, 0, 0, 0, 48, 402, 4045, 0, 1312, 689, 287, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1395, 643, 0, 2005, 0, 2782, 0, 2557, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2588, 0, 5, 28, 0, 4096, 0, 0, 55, 0, 0, 16, 1481, 3478, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4, 0, 0, 2, 0, 2977, 3806, 0, 0, 0, 1327, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 6, 12, 3968, 3004, 5, 26, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 710, 0, 3349, 113, 4085, 3839, 260, 2, 4026, 3774, 3985, 0, 0, 136, 0, 2461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 0, 834, 3710, 2, 3700, 16, 98, 0, 0, 0, 0, 0, 33, 33, 3389, 3171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6952, 20094, 17572, 15842, 8158, 19308, 26740, 16406] avg=16384.0000 max=26740 max_rank=6 min=6952 min_rank=0 imbalance_ratio=1.632080 max_load_reduction=104332 max_load_reduction_pct=0.795990 moved_to_echo=70612 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[17408, 1280, 4224, 128, 15872, 16000, 3456, 512, 256, 4608, 1664, 4352, 13568, 25088, 3584, 20096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[7680, 0, 2944, 0, 20224, 9216, 4096, 16512, 20864, 640, 1792, 128, 3456, 16640, 128, 27520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[13568, 4352, 5248, 128, 2304, 8448, 15104, 4224, 18560, 12416, 3072, 5888, 13568, 9344, 15872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[27904, 14336, 12160, 13952, 128, 256, 6144, 128, 0, 1152, 13696, 1280, 256, 512, 25984, 384, 7168, 4864, 1408, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[768, 6272, 14976, 3072, 8960, 384, 11904, 4864, 3456, 896, 9600, 1664, 1152, 768, 2816, 5504, 30976, 24192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[896, 16512, 1792, 512, 10368, 3456, 256, 11136, 6144, 384, 1792, 4224, 3712, 22272, 128, 3328, 26368, 9984, 6400, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 12672, 15488, 6912, 384, 18048, 128, 9472, 18304, 15104, 128, 128, 9600, 7040, 4224, 14336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[9856, 384, 12672, 4480, 14592, 11520, 4992, 2560, 11904, 12672, 23552, 2816, 5120, 8576, 0, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 24, 4095, 0, 0, 2853, 4065, 4092, 0, 0, 0, 0, 0, 432, 1393, 0, 0, 27, 0, 75, 0, 0, 1, 0, 0, 2012, 0, 0, 0, 4018, 36, 4023, 0, 0, 0, 45, 8, 0, 138, 0, 3986, 3912, 0, 4091, 0, 0, 183, 4096, 2, 2202, 6, 146, 3402, 0, 0, 2, 1, 3, 4095, 4096, 713, 4096, 983, 2713, 1272, 3625, 0, 0, 66, 0, 0, 0, 0, 0, 0, 2152, 0, 200, 1, 0, 921, 0, 0, 0, 15, 61, 870, 0, 694, 3421, 0, 520, 171, 0, 0, 1335, 3267, 0, 290, 803, 1, 0, 0, 0, 4087, 0, 4068, 0, 4046, 4094, 0, 1, 4093, 3957, 4079, 4083, 0, 6, 0, 53, 1, 2964, 3515, 0, 4008, 2267] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 3, 1611, 0, 0, 0, 22, 0, 4061, 39, 0, 0, 3995, 0, 4076, 3, 81, 24, 0, 0, 4091, 3180, 0, 0, 9, 3811, 0, 4084, 0, 48, 1348, 4080, 0, 884, 0, 0, 1, 128, 824, 43, 3836, 0, 2580, 4055, 749, 0, 4056, 438, 955, 1, 0, 2981, 0, 3188, 61, 205, 12, 0, 16, 0, 0, 0, 246, 151, 637, 4079, 0, 0, 2487, 0, 0, 0, 3797, 3821, 0, 492, 0, 0, 0, 34, 404, 4, 1, 4050, 4, 0, 0, 0, 0, 5, 44, 3880, 0, 0, 0, 1283, 3562, 0, 3023, 0, 43, 4096, 4088, 62, 2294, 91, 4094, 0, 2, 0, 4085, 3, 9, 1, 4096, 4077, 4095, 25, 4056, 0, 105, 0, 0, 0, 71, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[1678, 0, 0, 4080, 0, 35, 0, 54, 3336, 3975, 4090, 0, 3, 0, 1, 0, 4016, 46, 72, 0, 0, 0, 0, 0, 3363, 4096, 3698, 4096, 0, 3, 7, 4096, 0, 2, 889, 0, 0, 0, 0, 0, 0, 10, 4096, 4095, 0, 3310, 13, 0, 1, 1188, 1, 4011, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2373, 4096, 3857, 12, 3667, 0, 2332, 432, 4093, 18, 4096, 3337, 47, 0, 1, 0, 10, 0, 0, 481, 48, 89, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 2790, 4090, 0, 16, 2976, 121, 0, 10, 3139, 4096, 0, 5, 1325, 4089, 17, 4052, 0, 0, 0, 0, 0, 0, 4017, 3189, 0, 0, 0, 0, 0, 0, 488, 292, 0, 4096, 913] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[46, 0, 0, 18, 4078, 1, 5, 0, 0, 0, 4093, 3781, 51, 38, 4096, 8, 1, 2010, 0, 0, 53, 0, 162, 733, 0, 0, 23, 2557, 1399, 43, 5, 4096, 0, 0, 4067, 0, 0, 210, 0, 0, 2665, 0, 3865, 4096, 0, 4087, 3852, 73, 0, 323, 0, 1, 0, 1, 0, 0, 1, 0, 632, 2709, 3816, 1739, 0, 1370, 492, 127, 4095, 3576, 0, 0, 0, 0, 10, 3, 0, 0, 0, 594, 991, 38, 0, 0, 232, 0, 4, 1078, 0, 0, 4023, 6, 0, 3415, 0, 0, 0, 0, 0, 0, 4096, 2078, 0, 16, 0, 0, 134, 4059, 3907, 479, 0, 0, 4089, 4096, 0, 12, 3049, 3769, 3403, 4096, 0, 0, 0, 4041, 0, 4049, 1896, 29, 4094, 192] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[27, 19, 118, 0, 3874, 4087, 3969, 4092, 1364, 135, 231, 1156, 1, 0, 0, 0, 4096, 10, 498, 0, 17, 3898, 0, 0, 0, 4096, 0, 4060, 0, 0, 658, 0, 0, 4073, 4076, 0, 0, 3540, 0, 0, 3702, 139, 579, 38, 657, 4095, 1679, 0, 0, 249, 0, 0, 0, 428, 1437, 0, 14, 82, 0, 54, 4096, 4007, 0, 4038, 0, 5, 0, 2840, 0, 0, 0, 0, 204, 0, 0, 0, 3931, 0, 0, 135, 0, 4096, 0, 2677, 2, 0, 0, 745, 4070, 0, 89, 0, 824, 3967, 4096, 0, 953, 41, 3749, 0, 4066, 0, 4, 0, 8, 1, 4, 10, 3371, 5, 4011, 2953, 0, 0, 1, 762, 863, 0, 0, 0, 1014, 2, 0, 0, 4062, 0, 4058, 64] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[1, 4091, 0, 666, 4053, 0, 1605, 4091, 4089, 0, 0, 859, 0, 212, 0, 0, 3591, 0, 0, 39, 0, 4068, 0, 0, 3851, 1, 0, 3543, 3875, 0, 18, 3313, 2, 3854, 0, 4, 163, 30, 0, 0, 0, 47, 0, 4096, 0, 4047, 0, 129, 586, 4096, 0, 3110, 0, 120, 420, 0, 0, 0, 297, 0, 0, 23, 0, 4089, 0, 978, 0, 4069, 4096, 2957, 0, 47, 3464, 5, 37, 0, 0, 0, 0, 1695, 4096, 0, 10, 0, 20, 544, 0, 0, 0, 130, 1, 3580, 0, 0, 2408, 0, 4096, 0, 0, 0, 1484, 2273, 1, 0, 0, 0, 0, 0, 3862, 21, 308, 4048, 0, 126, 4070, 3529, 4096, 1642, 30, 1951, 0, 0, 0, 3, 1, 0, 4094, 221] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[2, 39, 0, 0, 379, 0, 115, 0, 2598, 0, 4089, 4093, 276, 654, 4025, 0, 0, 28, 0, 11, 9, 97, 756, 16, 1, 15, 57, 1874, 1, 355, 0, 695, 0, 4096, 3133, 4096, 0, 156, 0, 11, 4084, 0, 0, 4091, 1156, 4088, 0, 0, 3865, 15, 1, 2399, 96, 3398, 3618, 141, 3, 0, 0, 587, 0, 0, 4026, 4023, 0, 4089, 856, 4042, 4083, 0, 1493, 0, 3587, 4094, 2226, 1, 3, 0, 0, 0, 0, 3, 4, 0, 39, 0, 26, 0, 19, 0, 0, 3671, 5, 20, 0, 120, 4090, 3815, 1, 3905, 0, 1011, 12, 2591, 0, 0, 4, 123, 3, 0, 4063, 0, 0, 4092, 0, 3572, 4070, 0, 0, 5, 0, 0, 259, 7, 0, 0, 3800, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[2, 19, 0, 268, 4080, 0, 0, 0, 4070, 2292, 0, 21, 19, 92, 0, 0, 1898, 3980, 0, 0, 0, 0, 0, 0, 5, 3207, 31, 3888, 3896, 449, 0, 3844, 0, 4091, 0, 214, 0, 99, 223, 0, 5, 4058, 4021, 4092, 0, 4094, 0, 0, 196, 1783, 1, 3274, 0, 2005, 3433, 0, 0, 0, 3797, 0, 0, 1, 597, 3225, 0, 210, 3984, 3488, 0, 259, 1527, 0, 4072, 947, 2412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 4039, 0, 0, 3660, 0, 1113, 2610, 0, 1937, 506, 0, 290, 0, 0, 607, 77, 0, 4096, 0, 12, 3996, 0, 0, 3754, 4095, 4096, 0, 0, 1, 0, 0, 0, 3912, 0, 2, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 1611, 0, 0, 0, 22, 0, 4061, 39, 0, 0, 3995, 0, 4076, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 24, 0, 0, 4091, 3180, 0, 0, 9, 3811, 0, 4084, 0, 48, 1348, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 884, 0, 0, 1, 128, 824, 43, 3836, 0, 2580, 1372, 749, 0, 4056, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 955, 1, 0, 2981, 0, 3188, 61, 205, 12, 0, 16, 0, 0, 0, 246, 151, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 637, 4079, 0, 0, 2487, 0, 0, 0, 3797, 3821, 0, 492, 0, 0, 0, 34, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 404, 4, 1, 4050, 4, 0, 0, 0, 0, 5, 44, 3880, 0, 0, 0, 1283, 3363, 4085, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3562, 0, 3023, 0, 43, 4096, 4088, 62, 2294, 91, 4094, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 4096, 4077, 4095, 25, 4056, 0, 105, 0, 0, 0, 71, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13810, 20756, 14911, 9274, 15945, 17750, 21358, 17268] avg=16384.0000 max=21358 max_rank=6 min=9274 min_rank=3 imbalance_ratio=1.303589 max_load_reduction=109714 max_load_reduction_pct=0.837051 moved_to_echo=72321 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1678, 0, 0, 4080, 0, 35, 0, 54, 3336, 3975, 4090, 0, 3, 0, 1, 0, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 46, 72, 0, 0, 0, 0, 0, 3363, 4096, 3698, 4096, 0, 3, 7, 4096, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 889, 0, 0, 0, 0, 0, 0, 10, 4096, 1374, 0, 3310, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1188, 1, 4011, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2373, 4096, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3857, 12, 3667, 0, 2332, 432, 4093, 18, 4096, 3337, 47, 0, 1, 0, 10, 0, 608, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 48, 89, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 2790, 4090, 0, 3368, 0, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2976, 121, 0, 10, 3139, 4096, 0, 5, 1325, 4089, 17, 2236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 3189, 0, 0, 0, 0, 0, 0, 488, 292, 0, 728, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17746, 24219, 9694, 13226, 23106, 15424, 18030, 9627] avg=16384.0000 max=24219 max_rank=1 min=9627 min_rank=7 imbalance_ratio=1.478210 max_load_reduction=106853 max_load_reduction_pct=0.815224 moved_to_echo=66187 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 24, 4095, 0, 0, 2853, 4065, 4092, 0, 0, 0, 0, 0, 432, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1393, 0, 0, 27, 0, 75, 0, 0, 1, 0, 0, 2012, 0, 0, 0, 4018, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 4023, 0, 0, 0, 45, 8, 0, 138, 0, 3986, 1324, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 4096, 2, 2202, 6, 146, 3402, 0, 0, 2, 1, 3, 4095, 4096, 713, 4096, 1406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 983, 2713, 1272, 3625, 0, 0, 66, 0, 0, 0, 0, 0, 0, 2152, 0, 200, 577, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 921, 0, 0, 0, 15, 61, 870, 0, 694, 3421, 0, 520, 171, 0, 3291, 4046, 605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1335, 3267, 0, 290, 803, 1, 0, 0, 0, 4087, 0, 2255, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 3957, 4079, 4083, 0, 6, 0, 53, 1, 2964, 3515, 0, 717, 2267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16054, 8251, 13651, 24449, 12183, 14616, 16132, 25736] avg=16384.0000 max=25736 max_rank=7 min=8251 min_rank=1 imbalance_ratio=1.570801 max_load_reduction=105336 max_load_reduction_pct=0.803650 moved_to_echo=68667 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[46, 0, 0, 18, 4078, 1, 5, 0, 0, 0, 4093, 3781, 51, 38, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2010, 0, 0, 53, 0, 162, 733, 0, 0, 23, 2557, 1399, 43, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 210, 0, 0, 2665, 0, 3865, 1386, 0, 4087, 3852, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 1, 0, 1, 0, 0, 1, 0, 632, 2709, 3816, 1739, 0, 1370, 1473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 127, 4095, 3576, 0, 0, 0, 0, 10, 3, 0, 0, 0, 594, 991, 38, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 0, 4, 1078, 0, 0, 4023, 6, 0, 3415, 0, 0, 0, 0, 3362, 4089, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2078, 0, 16, 0, 0, 134, 4059, 3907, 479, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3049, 3769, 3403, 4096, 0, 0, 0, 4041, 0, 4049, 1896, 29, 732, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16215, 11082, 20205, 12065, 10530, 16842, 18865, 25268] avg=16384.0000 max=25268 max_rank=7 min=10530 min_rank=4 imbalance_ratio=1.542236 max_load_reduction=105804 max_load_reduction_pct=0.807220 moved_to_echo=71505 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 4091, 0, 666, 4053, 0, 1605, 4091, 4089, 0, 0, 859, 0, 212, 0, 0, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 0, 0, 39, 0, 4068, 0, 0, 3851, 1, 0, 3543, 3875, 0, 18, 3313, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3854, 0, 4, 163, 30, 0, 0, 0, 47, 0, 1386, 0, 4047, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 586, 4096, 0, 3110, 0, 120, 420, 0, 0, 0, 297, 0, 0, 23, 0, 4089, 1473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 978, 0, 4069, 4096, 2957, 0, 47, 3464, 5, 37, 0, 0, 0, 0, 1695, 604, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 20, 544, 0, 0, 0, 130, 1, 3580, 0, 0, 2408, 0, 3362, 308, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1484, 2273, 1, 0, 0, 0, 0, 0, 2141, 21, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 4070, 3529, 4096, 1642, 30, 1951, 0, 0, 0, 3, 1, 0, 732, 221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20135, 22987, 9662, 14214, 18517, 15092, 14064, 16401] avg=16384.0000 max=22987 max_rank=1 min=9662 min_rank=2 imbalance_ratio=1.403015 max_load_reduction=108085 max_load_reduction_pct=0.824623 moved_to_echo=64074 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[27, 19, 118, 0, 3874, 4087, 3969, 4092, 1364, 135, 231, 1156, 1, 0, 0, 0, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 498, 0, 17, 3898, 0, 0, 0, 4096, 0, 4060, 0, 0, 658, 0, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4076, 0, 0, 3540, 0, 0, 3702, 139, 579, 15, 657, 4095, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 0, 0, 428, 1437, 0, 14, 82, 0, 54, 4096, 4007, 0, 4038, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2840, 0, 0, 0, 0, 204, 0, 0, 0, 3931, 0, 0, 135, 5, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2677, 2, 0, 0, 745, 4070, 0, 89, 0, 824, 3967, 4096, 0, 3332, 4011, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 41, 3749, 0, 4066, 0, 4, 0, 8, 1, 4, 10, 1868, 5, 0, 2953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 762, 863, 0, 0, 0, 1014, 2, 0, 0, 4062, 0, 726, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19482, 17934, 22555, 14418, 7613, 27914, 13662, 7494] avg=16384.0000 max=27914 max_rank=5 min=7494 min_rank=7 imbalance_ratio=1.703735 max_load_reduction=103158 max_load_reduction_pct=0.787033 moved_to_echo=56683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 39, 0, 0, 379, 0, 115, 0, 2598, 0, 4089, 4093, 276, 654, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 11, 9, 97, 756, 16, 1, 15, 57, 1874, 1, 355, 0, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3133, 4096, 0, 156, 0, 11, 4084, 0, 0, 1384, 1156, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 15, 1, 2399, 96, 3398, 3618, 141, 3, 0, 0, 587, 0, 0, 4026, 4023, 1471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 856, 4042, 4083, 0, 1493, 0, 3587, 4094, 2226, 1, 3, 0, 0, 0, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 39, 0, 26, 0, 19, 0, 0, 3671, 5, 20, 0, 120, 3120, 4063, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3815, 1, 3905, 0, 1011, 12, 2591, 0, 0, 4, 123, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3572, 4070, 0, 0, 5, 0, 0, 259, 7, 0, 0, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16270, 3915, 22204, 23643, 25077, 11723, 15555, 12685] avg=16384.0000 max=25077 max_rank=4 min=3915 min_rank=1 imbalance_ratio=1.530579 max_load_reduction=105995 max_load_reduction_pct=0.808678 moved_to_echo=65040 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[2, 19, 0, 268, 4080, 0, 0, 0, 4070, 2292, 0, 21, 19, 92, 0, 0, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1898, 3980, 0, 0, 0, 0, 0, 0, 5, 3207, 31, 3888, 3896, 449, 0, 3844, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 214, 0, 99, 223, 0, 5, 4058, 4021, 1385, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 1783, 1, 3274, 0, 2005, 3433, 0, 0, 0, 3797, 0, 0, 1, 597, 3225, 1471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 3984, 3488, 0, 259, 1527, 0, 4072, 947, 2412, 0, 0, 0, 0, 0, 603, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 4039, 0, 0, 3660, 0, 3363, 12, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1113, 2610, 0, 1937, 506, 0, 290, 0, 0, 607, 77, 0, 2270, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3754, 4095, 4096, 0, 0, 1, 0, 0, 0, 3912, 0, 2, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11360, 21928, 18190, 19783, 18101, 11711, 13406, 16593] avg=16384.0000 max=21928 max_rank=1 min=11360 min_rank=0 imbalance_ratio=1.338379 max_load_reduction=109144 max_load_reduction_pct=0.832703 moved_to_echo=59811 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[13952, 10880, 14336, 7936, 6400, 11392, 8576, 2688, 2560, 6144, 16384, 640, 10880, 128, 0, 19200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[15104, 6144, 640, 128, 4224, 11392, 1024, 768, 7296, 15232, 3840, 26240, 9216, 1024, 2048, 24192, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1792, 4224, 1792, 5120, 20608, 4224, 5760, 11136, 23680, 10624, 12544, 9984, 4352, 1024, 12288, 512, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 4352, 19072, 27904, 27904, 9856, 4096, 2048, 1152, 4096, 384, 11520, 9856, 128, 5888, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[4608, 4608, 1280, 6912, 128, 1664, 128, 896, 9088, 256, 896, 25984, 896, 7424, 14464, 1408, 26624, 20736, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[6016, 12288, 13952, 21760, 13056, 3712, 7296, 128, 19328, 12288, 4736, 512, 3968, 2816, 1024, 2176, 4224, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 21120, 12288, 4352, 256, 4224, 1152, 128, 14464, 4352, 19200, 9728, 2688, 27904, 9600, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[5888, 11776, 128, 18048, 256, 9344, 12416, 384, 128, 128, 4864, 3456, 12032, 9984, 8064, 25088, 10368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 3258, 4074, 0, 3595, 0, 0, 138, 3384, 2588, 3712, 0, 271, 452, 2, 647, 0, 0, 3619, 176, 4048, 845, 63, 0, 747, 0, 4021, 0, 40, 3059, 4081, 3616, 136, 268, 0, 3273, 6, 7, 4, 68, 4, 168, 0, 0, 0, 356, 4096, 935, 0, 0, 0, 0, 0, 3, 9, 3441, 4065, 46, 0, 41, 0, 3951, 0, 3734, 0, 0, 0, 33, 3853, 3991, 0, 0, 455, 103, 3303, 0, 0, 0, 0, 3814, 0, 0, 0, 444, 0, 19, 2, 3942, 15, 4093, 380, 0, 739, 4094, 0, 4061, 3121, 4096, 88, 4090, 1288, 606, 6, 0, 0, 0, 2, 3973, 0, 0, 2919, 0, 0, 743, 1071, 36, 0, 0, 298, 4096, 227, 8, 0, 0, 15, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[8, 0, 1409, 0, 1961, 255, 4065, 0, 0, 0, 0, 0, 4094, 2746, 0, 0, 0, 0, 3214, 0, 4094, 34, 319, 0, 0, 0, 3789, 0, 4096, 0, 4095, 0, 1595, 180, 3450, 0, 279, 97, 0, 43, 0, 4088, 3, 24, 0, 174, 0, 7, 3194, 0, 1288, 0, 0, 4056, 1027, 1, 0, 0, 0, 937, 0, 0, 13, 3382, 3984, 4, 349, 183, 2, 0, 3804, 0, 0, 0, 2600, 0, 4096, 4018, 0, 0, 1, 4086, 594, 0, 4040, 58, 0, 0, 157, 0, 14, 3910, 4096, 0, 2857, 3714, 0, 3625, 0, 4096, 0, 393, 447, 0, 10, 0, 3531, 901, 0, 0, 562, 385, 1158, 709, 176, 0, 3944, 0, 4073, 0, 10, 0, 0, 2532, 67, 23, 19, 3827] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[11, 0, 76, 411, 383, 0, 77, 0, 0, 0, 2371, 3790, 4096, 3748, 3695, 0, 0, 58, 0, 0, 4065, 1, 0, 0, 0, 0, 4084, 0, 4096, 0, 0, 375, 3608, 1792, 406, 1, 0, 23, 0, 1, 898, 2, 0, 0, 0, 4092, 232, 7, 255, 3962, 0, 0, 99, 2098, 0, 4, 0, 0, 4082, 0, 0, 0, 0, 3465, 0, 4096, 37, 0, 3779, 0, 4096, 0, 3936, 284, 0, 0, 0, 0, 0, 1861, 0, 101, 93, 0, 0, 71, 3890, 0, 0, 0, 0, 4096, 2575, 0, 0, 3542, 0, 4009, 4094, 3222, 37, 4096, 4085, 2262, 0, 0, 0, 0, 0, 140, 1692, 0, 4024, 965, 3, 0, 4044, 398, 4027, 74, 0, 0, 0, 4043, 0, 0, 416, 620] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 21, 4095, 1207, 4019, 22, 3528, 161, 23, 0, 1994, 3814, 4039, 4096, 0, 0, 0, 0, 0, 157, 3698, 3, 342, 0, 9, 504, 34, 0, 4084, 0, 0, 2386, 2, 46, 3530, 1540, 91, 0, 0, 0, 0, 24, 176, 0, 675, 4, 0, 0, 3957, 4089, 0, 0, 607, 4093, 1756, 0, 0, 0, 2533, 0, 22, 0, 0, 2807, 0, 4010, 0, 1, 0, 0, 4085, 0, 0, 0, 0, 0, 3888, 0, 9, 4055, 0, 4096, 0, 223, 1385, 0, 0, 0, 917, 0, 0, 4096, 0, 0, 370, 4096, 30, 4010, 4096, 0, 3892, 4063, 4096, 0, 6, 0, 0, 22, 2, 0, 0, 206, 3183, 1313, 147, 0, 1884, 0, 149, 562, 0, 0, 0, 4067, 0, 0, 0, 3925] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[211, 70, 34, 0, 0, 0, 0, 36, 0, 0, 0, 4, 3556, 4096, 571, 0, 2, 17, 0, 4084, 4096, 830, 0, 0, 0, 0, 4094, 7, 3809, 0, 1, 3786, 8, 3963, 4096, 0, 0, 8, 0, 0, 0, 3011, 1909, 11, 0, 2119, 4070, 0, 0, 2522, 4072, 2, 0, 150, 0, 0, 0, 0, 3972, 5, 18, 0, 1336, 2059, 2168, 2073, 0, 0, 3, 1, 1514, 0, 4058, 0, 0, 2969, 0, 15, 27, 2916, 0, 0, 0, 135, 4096, 4005, 2030, 2730, 3536, 0, 0, 4020, 0, 328, 4093, 2660, 0, 4022, 62, 0, 3818, 4095, 49, 648, 0, 0, 9, 0, 0, 1, 1388, 0, 697, 0, 74, 520, 3623, 10, 4072, 113, 0, 210, 0, 0, 863, 786, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 3916, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 4096, 3771, 3481, 135, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 1, 3989, 0, 0, 4064, 4096, 4078, 0, 0, 0, 0, 0, 0, 2208, 4085, 27, 5, 4024, 3, 0, 6, 3085, 672, 0, 4082, 11, 4070, 0, 0, 4, 0, 4096, 33, 0, 123, 3879, 2710, 0, 4077, 0, 1887, 209, 4086, 0, 0, 0, 0, 4096, 1401, 0, 0, 0, 429, 4085, 0, 0, 17, 1, 5, 42, 4, 3976, 129, 4, 31, 1315, 0, 0, 4075, 0, 1, 9, 0, 0, 137, 0, 0, 0, 0, 4093, 0, 0, 260, 117, 0, 3449, 0, 2818, 288, 3846, 80, 1859, 0, 0, 0, 6, 3367, 16, 40, 8, 2094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 753, 0, 1079, 31, 0, 4096, 16, 131, 0, 0, 1, 1232, 4044, 0, 2, 0, 0, 0, 2029, 4091, 0, 1, 0, 229, 4091, 4096, 12, 3927, 388, 4094, 3635, 3985, 0, 3895, 0, 0, 0, 2793, 0, 2992, 946, 0, 66, 0, 4096, 0, 2, 3816, 91, 0, 0, 0, 94, 0, 0, 0, 513, 2603, 0, 1546, 0, 0, 4022, 3203, 3057, 0, 5, 16, 39, 684, 2402, 0, 5, 3931, 13, 0, 0, 164, 1336, 0, 0, 0, 4005, 0, 1, 29, 3263, 0, 125, 0, 3222, 0, 0, 3, 4022, 0, 3420, 134, 26, 0, 33, 186, 74, 59, 0, 0, 249, 43, 0, 0, 4093, 4076, 692, 1016, 18, 1957, 0, 3910, 0, 0, 4096, 0, 4096, 0, 3931, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 2121, 1, 15, 78, 4019, 0, 0, 66, 0, 0, 3, 4083, 1, 0, 0, 0, 4094, 34, 4004, 0, 21, 0, 0, 0, 1183, 0, 4060, 0, 8, 1832, 2, 3965, 4052, 0, 0, 0, 0, 0, 28, 4095, 36, 0, 4094, 3, 12, 27, 0, 218, 2, 0, 1169, 0, 3114, 0, 588, 0, 71, 0, 0, 3, 2417, 0, 119, 4095, 1634, 0, 2670, 0, 3181, 0, 3441, 0, 3821, 0, 0, 0, 4074, 232, 3, 3367, 0, 0, 1159, 3628, 12, 0, 126, 3, 0, 4096, 3928, 0, 0, 4095, 1, 0, 4095, 3274, 0, 4096, 0, 0, 1, 0, 410, 0, 0, 0, 0, 0, 3931, 3484, 618, 0, 400, 2592, 3838, 0, 2, 2655, 0, 4032, 0, 282, 68, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 3258, 4074, 0, 3595, 0, 0, 138, 3384, 2588, 3712, 0, 271, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 647, 0, 0, 3619, 176, 4048, 845, 63, 0, 747, 0, 3239, 0, 40, 3059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 3616, 136, 268, 0, 3273, 6, 7, 4, 68, 4, 168, 0, 0, 0, 356, 2155, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 935, 0, 0, 0, 0, 0, 3, 9, 3441, 4065, 46, 0, 41, 0, 3951, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2602, 0, 0, 0, 33, 3853, 3991, 0, 0, 455, 103, 3303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3814, 0, 0, 0, 444, 0, 19, 2, 3942, 15, 4093, 380, 0, 739, 1939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 3121, 4096, 88, 4090, 1288, 606, 6, 0, 0, 0, 2, 3973, 0, 0, 914, 653, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2005, 0, 0, 743, 1071, 36, 0, 0, 298, 4096, 227, 8, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21473, 16485, 14621, 17234, 14340, 15387, 23033, 8499] avg=16384.0000 max=23033 max_rank=6 min=8499 min_rank=7 imbalance_ratio=1.405823 max_load_reduction=108039 max_load_reduction_pct=0.824272 moved_to_echo=61259 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 1409, 0, 1961, 255, 4065, 0, 0, 0, 0, 0, 4094, 2289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3214, 0, 4094, 34, 319, 0, 0, 0, 3789, 0, 3299, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1595, 180, 3450, 0, 279, 97, 0, 43, 0, 4088, 3, 24, 0, 174, 0, 7, 1955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3194, 0, 1288, 0, 0, 4056, 1027, 1, 0, 0, 0, 937, 0, 0, 13, 3382, 659, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 4, 349, 183, 2, 0, 3804, 0, 0, 0, 2600, 0, 4096, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 594, 0, 4040, 58, 0, 0, 157, 0, 14, 3910, 4096, 0, 2857, 1759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3625, 0, 4096, 0, 393, 447, 0, 10, 0, 3531, 901, 0, 0, 562, 385, 362, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 796, 709, 176, 0, 3944, 0, 4073, 0, 10, 0, 0, 2532, 67, 23, 19, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14081, 18844, 11895, 15014, 19040, 21572, 14450, 16176] avg=16384.0000 max=21572 max_rank=5 min=11895 min_rank=2 imbalance_ratio=1.316650 max_load_reduction=109500 max_load_reduction_pct=0.835419 moved_to_echo=71238 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 76, 411, 383, 0, 77, 0, 0, 0, 2371, 3790, 4096, 3120, 3695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 4065, 1, 0, 0, 0, 0, 4084, 0, 3294, 0, 0, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3608, 1792, 406, 1, 0, 23, 0, 1, 898, 2, 0, 0, 0, 4092, 232, 7, 1868, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 3962, 0, 0, 99, 2098, 0, 4, 0, 0, 4082, 0, 0, 0, 0, 3465, 661, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2847, 37, 0, 3779, 0, 4096, 0, 3936, 284, 0, 0, 0, 0, 0, 1861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 93, 0, 0, 71, 3890, 0, 0, 0, 0, 4096, 2575, 0, 0, 1674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 4094, 3222, 37, 4096, 4085, 2262, 0, 0, 0, 0, 0, 140, 1692, 0, 1265, 721, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2759, 965, 3, 0, 4044, 398, 4027, 74, 0, 0, 0, 4043, 0, 0, 416, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18030, 11877, 13458, 15254, 16840, 12500, 25764, 17349] avg=16384.0000 max=25764 max_rank=6 min=11877 min_rank=1 imbalance_ratio=1.572510 max_load_reduction=105308 max_load_reduction_pct=0.803436 moved_to_echo=72453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 21, 4095, 1207, 4019, 22, 3528, 161, 23, 0, 1994, 3814, 4039, 3415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 3698, 3, 342, 0, 9, 504, 34, 0, 3290, 0, 0, 2386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 46, 3530, 1540, 91, 0, 0, 0, 0, 24, 176, 0, 675, 4, 0, 0, 2156, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3957, 4089, 0, 0, 607, 4093, 1756, 0, 0, 0, 2533, 0, 22, 0, 0, 2807, 657, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2795, 0, 1, 0, 0, 4085, 0, 0, 0, 0, 0, 3888, 0, 9, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 223, 1385, 0, 0, 0, 917, 0, 0, 4096, 0, 0, 370, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4010, 4096, 0, 3892, 4063, 4096, 0, 6, 0, 0, 22, 2, 0, 0, 206, 997, 701, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2186, 1313, 147, 0, 1884, 0, 149, 562, 0, 0, 0, 4067, 0, 0, 0, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[26338, 10423, 8758, 21202, 14833, 13027, 22258, 14233] avg=16384.0000 max=26338 max_rank=0 min=8758 min_rank=2 imbalance_ratio=1.607544 max_load_reduction=104734 max_load_reduction_pct=0.799057 moved_to_echo=64351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 753, 0, 1079, 31, 0, 4096, 16, 131, 0, 0, 1, 1232, 3371, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2029, 4091, 0, 1, 0, 229, 4091, 4096, 12, 3163, 388, 4094, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 3895, 0, 0, 0, 2793, 0, 2992, 946, 0, 66, 0, 4096, 0, 2, 2117, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3816, 91, 0, 0, 0, 94, 0, 0, 0, 513, 2603, 0, 1546, 0, 0, 4022, 632, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3203, 2130, 0, 5, 16, 39, 684, 2402, 0, 5, 3931, 13, 0, 0, 164, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 1, 29, 3263, 0, 125, 0, 3222, 0, 0, 3, 1905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3420, 134, 26, 0, 33, 186, 74, 59, 0, 0, 249, 43, 0, 0, 4093, 1277, 535, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2799, 692, 1016, 18, 1957, 0, 3910, 0, 0, 4096, 0, 4096, 0, 3931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10712, 25829, 21284, 13990, 13928, 12553, 10261, 22515] avg=16384.0000 max=25829 max_rank=1 min=10261 min_rank=6 imbalance_ratio=1.576477 max_load_reduction=105243 max_load_reduction_pct=0.802940 moved_to_echo=59257 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 3916, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 4096, 3144, 3481, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 1, 3213, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4078, 0, 0, 0, 0, 0, 0, 2208, 4085, 27, 5, 4024, 3, 0, 6, 2145, 523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3085, 672, 0, 4082, 11, 4070, 0, 0, 4, 0, 4096, 33, 0, 123, 3879, 2710, 642, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2841, 0, 1887, 209, 4086, 0, 0, 0, 0, 4096, 1401, 0, 0, 0, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 17, 1, 5, 42, 4, 3976, 129, 4, 31, 1315, 0, 0, 1930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 0, 0, 137, 0, 0, 0, 0, 4093, 0, 0, 260, 117, 0, 1081, 713, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2368, 0, 2818, 288, 3846, 80, 1859, 0, 0, 0, 6, 3367, 16, 40, 8, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[22833, 13182, 21200, 24034, 14949, 11539, 6545, 16790] avg=16384.0000 max=24034 max_rank=3 min=6545 min_rank=6 imbalance_ratio=1.466919 max_load_reduction=107038 max_load_reduction_pct=0.816635 moved_to_echo=49823 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[211, 70, 34, 0, 0, 0, 0, 36, 0, 0, 0, 4, 3556, 3415, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 17, 0, 4084, 4096, 830, 0, 0, 0, 0, 4094, 7, 3068, 0, 1, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3963, 4096, 0, 0, 8, 0, 0, 0, 3011, 1909, 11, 0, 2119, 4070, 0, 1400, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2522, 4072, 2, 0, 150, 0, 0, 0, 0, 3972, 5, 18, 0, 1336, 2059, 613, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2168, 1445, 0, 0, 3, 1, 1514, 0, 4058, 0, 0, 2969, 0, 15, 27, 2916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 4096, 4005, 2030, 2730, 3536, 0, 0, 4020, 0, 328, 4093, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 62, 0, 3818, 4095, 49, 648, 0, 0, 9, 0, 0, 1, 1388, 0, 218, 362, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 479, 0, 74, 520, 3623, 10, 4072, 113, 0, 210, 0, 0, 863, 786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7897, 19985, 20861, 15430, 15116, 26233, 14800, 10750] avg=16384.0000 max=26233 max_rank=5 min=7897 min_rank=0 imbalance_ratio=1.601135 max_load_reduction=104839 max_load_reduction_pct=0.799858 moved_to_echo=66899 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2121, 1, 15, 78, 4019, 0, 0, 66, 0, 0, 3, 3404, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 34, 4004, 0, 21, 0, 0, 0, 1183, 0, 3271, 0, 8, 1832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3965, 4052, 0, 0, 0, 0, 0, 28, 4095, 36, 0, 4094, 3, 12, 27, 2156, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 2, 0, 1169, 0, 3114, 0, 588, 0, 71, 0, 0, 3, 2417, 0, 653, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 2854, 1634, 0, 2670, 0, 3181, 0, 3441, 0, 3821, 0, 0, 0, 4074, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3367, 0, 0, 1159, 3628, 12, 0, 126, 3, 0, 4096, 3928, 0, 0, 1939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 3274, 0, 4096, 0, 0, 1, 0, 410, 0, 0, 0, 0, 0, 1232, 716, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2699, 3484, 618, 0, 400, 2592, 3838, 0, 2, 2655, 0, 4032, 0, 282, 68, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9708, 14447, 18995, 8914, 22026, 18261, 13961, 24760] avg=16384.0000 max=24760 max_rank=7 min=8914 min_rank=3 imbalance_ratio=1.511230 max_load_reduction=106312 max_load_reduction_pct=0.811096 moved_to_echo=79008 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[4352, 896, 11776, 6016, 14464, 384, 19456, 256, 256, 256, 7808, 10240, 24832, 22272, 8064, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[18432, 12544, 5376, 4096, 1920, 14592, 6016, 128, 640, 3968, 21504, 1024, 1664, 256, 7680, 22400, 5248, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9600, 17536, 2048, 2176, 6784, 4224, 21248, 6400, 11520, 384, 14976, 4608, 11392, 4096, 4352, 10880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 23168, 15616, 14720, 7936, 21120, 10240, 3712, 128, 0, 8064, 1280, 128, 4480, 3840, 4736, 7424, 4480, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[4096, 15488, 768, 4480, 10752, 8320, 6016, 6016, 8832, 4224, 128, 27648, 12416, 384, 8064, 14464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[17408, 17664, 19584, 1920, 384, 3456, 2816, 128, 6144, 16384, 2176, 384, 8832, 10496, 4352, 512, 16000, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 4864, 7424, 6400, 27776, 1152, 4736, 896, 384, 4608, 19968, 128, 25856, 512, 8320, 19200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[16128, 7168, 4864, 1664, 20864, 3200, 22016, 768, 384, 11136, 256, 22272, 1024, 5120, 640, 14592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True + [2026-08-30 05:03:32] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 866.9 | throughput per GPU (TFLOP/s/GPU): 384.6 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101839E+01 | loss scale: 1.0 | grad norm: 45.880 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[22, 2186, 1160, 66, 2029, 43, 827, 251, 620, 85, 1224, 263, 981, 803, 38, 406, 110, 984, 558, 479, 1461, 389, 2534, 1036, 92, 1947, 767, 680, 951, 286, 886, 2068, 3388, 271, 189, 686, 196, 883, 1441, 1148, 2743, 1012, 933, 349, 739, 1736, 1581, 828, 337, 142, 252, 1191, 348, 224, 1782, 54, 283, 424, 1849, 2537, 626, 334, 552, 474, 878, 2921, 38, 1900, 1020, 119, 951, 2422, 598, 2664, 222, 626, 658, 1811, 2589, 1121, 2379, 598, 2509, 1342, 1589, 334, 543, 1832, 958, 190, 709, 1980, 1166, 626, 361, 446, 2081, 445, 2595, 479, 561, 1015, 1484, 2808, 1040, 1384, 791, 1584, 390, 787, 1755, 1219, 297, 1487, 1166, 574, 124, 958, 1252, 1369, 775, 2387, 2218, 987, 1823, 235, 1024, 114] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[45, 2711, 1480, 115, 1845, 79, 660, 458, 766, 144, 975, 234, 1337, 602, 28, 413, 192, 590, 1057, 900, 1893, 130, 2606, 995, 80, 1497, 830, 858, 1518, 371, 1173, 2013, 3363, 245, 136, 408, 98, 426, 1818, 1066, 3279, 1639, 961, 389, 740, 1643, 1090, 614, 330, 373, 246, 1400, 579, 458, 2126, 78, 97, 621, 2010, 1734, 687, 232, 800, 474, 842, 2639, 201, 1523, 819, 163, 698, 1979, 610, 2357, 547, 1537, 699, 1516, 2150, 900, 1677, 1085, 2056, 1585, 1599, 505, 910, 2231, 1010, 214, 527, 1360, 1485, 521, 485, 593, 1700, 530, 2490, 515, 758, 1143, 2043, 2783, 1141, 1774, 637, 1973, 361, 580, 1176, 1320, 134, 1344, 792, 1364, 134, 818, 1131, 1547, 551, 1639, 1716, 690, 1494, 208, 717, 91] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[114, 2340, 657, 76, 2329, 86, 758, 278, 784, 69, 1471, 241, 860, 636, 49, 403, 132, 748, 665, 892, 1531, 326, 2653, 1188, 45, 1649, 391, 549, 870, 193, 1381, 1895, 3580, 291, 115, 613, 158, 897, 1827, 921, 2338, 1456, 1094, 398, 557, 1855, 1377, 842, 316, 185, 918, 1152, 481, 397, 1660, 188, 210, 839, 2516, 2614, 742, 263, 319, 365, 740, 3081, 295, 1240, 925, 370, 1168, 2024, 782, 1919, 170, 1126, 853, 1589, 2725, 1292, 2447, 1226, 2538, 1635, 1228, 338, 654, 3034, 1091, 233, 890, 1517, 1012, 494, 449, 478, 1621, 258, 2859, 782, 846, 786, 1668, 2155, 1397, 1294, 584, 1118, 622, 539, 1202, 1503, 324, 1037, 513, 886, 138, 746, 1147, 1719, 993, 1538, 1814, 1142, 1401, 368, 527, 239] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[119, 2316, 1555, 136, 1877, 35, 728, 422, 962, 93, 1683, 614, 1105, 665, 57, 406, 177, 709, 1166, 837, 1911, 256, 2503, 1076, 108, 1672, 588, 939, 1220, 389, 1541, 1867, 3074, 279, 352, 474, 152, 774, 1726, 627, 2649, 1096, 1200, 871, 578, 1708, 1072, 525, 492, 309, 340, 1234, 295, 245, 1408, 70, 65, 1005, 2013, 2143, 965, 421, 714, 446, 712, 2989, 186, 1590, 719, 175, 1203, 1711, 446, 1643, 213, 882, 814, 1834, 2810, 826, 2208, 1135, 2022, 1273, 1949, 550, 1076, 1801, 917, 347, 939, 1423, 858, 921, 528, 564, 2073, 694, 2133, 496, 499, 860, 1429, 2744, 1000, 1469, 614, 1603, 236, 286, 1175, 1699, 254, 1602, 659, 819, 209, 921, 1426, 1173, 922, 1554, 1917, 1123, 1912, 230, 795, 228] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[49, 2389, 866, 143, 2339, 35, 485, 294, 1032, 90, 1396, 582, 846, 753, 89, 424, 333, 726, 1608, 1198, 1447, 105, 2965, 1085, 94, 1469, 308, 655, 1177, 310, 1354, 1920, 3043, 272, 351, 217, 401, 1038, 1311, 1115, 1923, 1301, 1750, 752, 905, 1177, 883, 1123, 249, 299, 306, 1042, 192, 222, 1914, 138, 117, 1041, 1505, 2620, 647, 379, 648, 483, 840, 2782, 204, 1342, 1141, 361, 1068, 1904, 944, 2094, 135, 1130, 964, 1370, 2589, 773, 2011, 1232, 2127, 1162, 1618, 790, 836, 2344, 722, 346, 674, 1484, 969, 601, 462, 190, 2115, 251, 1471, 970, 854, 1272, 1841, 2606, 940, 1501, 698, 1944, 232, 490, 1193, 1684, 207, 1771, 1102, 1284, 213, 883, 875, 1246, 923, 1660, 1953, 981, 1886, 247, 371, 269] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[58, 2552, 1478, 174, 2233, 34, 728, 341, 1112, 161, 1455, 269, 762, 653, 44, 374, 135, 566, 722, 1340, 1471, 450, 2339, 1283, 135, 1735, 552, 747, 1522, 250, 1830, 2016, 3117, 163, 535, 559, 164, 917, 1627, 657, 2341, 1839, 1667, 686, 431, 1462, 1013, 682, 299, 298, 445, 1377, 311, 425, 1312, 65, 70, 822, 2410, 2117, 614, 192, 1061, 497, 656, 3505, 322, 1644, 726, 210, 973, 2120, 443, 1764, 168, 519, 737, 2122, 1719, 1129, 2216, 488, 2224, 994, 1873, 401, 1050, 2093, 1225, 177, 771, 999, 907, 576, 527, 358, 2070, 386, 2182, 750, 817, 981, 1953, 2593, 1020, 1370, 613, 1648, 406, 362, 1177, 1593, 148, 1288, 1140, 1480, 168, 784, 1225, 1546, 961, 994, 1881, 1223, 1332, 658, 698, 293] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[78, 2721, 1105, 98, 1601, 20, 744, 317, 1007, 246, 1510, 297, 1294, 819, 65, 408, 166, 501, 1117, 551, 1897, 343, 1980, 1255, 108, 1520, 521, 900, 1356, 377, 1541, 1858, 3094, 306, 410, 755, 220, 1120, 1379, 620, 2404, 1260, 1231, 355, 520, 1496, 1132, 569, 262, 199, 340, 1327, 675, 382, 1899, 81, 55, 1286, 2056, 1950, 1040, 254, 859, 601, 869, 2917, 158, 1865, 920, 121, 541, 1758, 165, 1958, 306, 804, 744, 1461, 2190, 1053, 2063, 1339, 2337, 1068, 1685, 531, 866, 1881, 1134, 227, 879, 1217, 1283, 609, 405, 370, 2022, 937, 2461, 558, 750, 1150, 2184, 2895, 1119, 1624, 505, 1832, 253, 487, 2115, 1049, 201, 1213, 894, 1204, 79, 834, 1688, 1135, 777, 1327, 1541, 876, 1856, 215, 950, 159] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[44, 2279, 1407, 139, 1936, 61, 717, 442, 905, 117, 1932, 433, 650, 861, 66, 569, 164, 1019, 974, 1072, 1591, 255, 2458, 1019, 52, 1590, 278, 882, 1259, 183, 1618, 2041, 3416, 247, 393, 595, 400, 1005, 1386, 721, 2508, 1419, 1447, 924, 715, 1681, 1076, 692, 342, 376, 334, 1386, 578, 249, 2094, 85, 228, 583, 2039, 2558, 516, 297, 760, 460, 877, 2656, 158, 1546, 1115, 206, 1141, 1841, 474, 2040, 206, 624, 701, 1822, 2755, 865, 2000, 1078, 2153, 1117, 1729, 671, 973, 2561, 836, 233, 619, 1093, 847, 573, 515, 210, 1909, 212, 2236, 727, 656, 1068, 1802, 2650, 829, 1215, 894, 1629, 337, 408, 2114, 1529, 449, 1216, 581, 1575, 90, 915, 1099, 868, 767, 995, 1742, 898, 1770, 171, 503, 460] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[22, 2186, 1160, 66, 2029, 43, 827, 251, 620, 85, 1224, 263, 981, 803, 38, 406, 2854, 1609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 984, 558, 479, 1461, 389, 2534, 1036, 92, 1947, 767, 680, 951, 286, 886, 2068, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1913, 271, 189, 686, 196, 883, 1441, 1148, 2743, 1012, 933, 349, 739, 1736, 1581, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 142, 252, 1191, 348, 224, 1782, 54, 283, 424, 1849, 2537, 626, 334, 552, 474, 1627, 1442, 1199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 878, 67, 38, 1900, 1020, 119, 951, 2422, 598, 2664, 222, 626, 658, 1811, 2589, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2379, 598, 882, 1342, 1589, 334, 543, 1832, 958, 190, 709, 1980, 1166, 626, 361, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2081, 445, 1270, 479, 561, 1015, 1484, 0, 1040, 1384, 791, 1584, 390, 787, 1755, 1219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 1487, 1166, 574, 124, 958, 1252, 1369, 775, 2387, 2218, 987, 1823, 235, 1024, 114, 1216, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15467, 15337, 16648, 15677, 17684, 15935, 16285, 18039] avg=16384.0000 max=18039 max_rank=7 min=15337 min_rank=1 imbalance_ratio=1.101013 max_load_reduction=113033 max_load_reduction_pct=0.862373 moved_to_echo=67943 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[45, 2711, 1480, 115, 1845, 79, 660, 458, 766, 144, 975, 234, 1337, 602, 28, 413, 2578, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 590, 1057, 900, 1893, 130, 2606, 995, 80, 1497, 830, 858, 1518, 371, 1173, 2013, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1899, 245, 136, 408, 98, 426, 1818, 1066, 3279, 1639, 961, 389, 740, 1643, 1090, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 373, 246, 1400, 579, 458, 2126, 78, 97, 621, 2010, 1734, 687, 232, 800, 474, 1334, 1431, 1187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 842, 61, 201, 1523, 819, 163, 698, 1979, 610, 2357, 547, 1537, 699, 1516, 2150, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1677, 1085, 722, 1585, 1599, 505, 910, 2231, 1010, 214, 527, 1360, 1485, 521, 485, 593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1700, 530, 1218, 515, 758, 1143, 2043, 0, 1141, 1774, 637, 1973, 361, 580, 1176, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 1344, 792, 1364, 134, 818, 1131, 1547, 551, 1639, 1716, 690, 1494, 208, 717, 91, 1167, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16066, 16808, 16451, 16197, 16602, 16509, 16869, 15570] avg=16384.0000 max=16869 max_rank=6 min=15570 min_rank=7 imbalance_ratio=1.029602 max_load_reduction=114203 max_load_reduction_pct=0.871300 moved_to_echo=65550 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[49, 2389, 866, 143, 2339, 35, 485, 294, 1032, 90, 1396, 582, 846, 753, 89, 424, 2718, 1493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 726, 1608, 1198, 1447, 105, 2965, 1085, 94, 1469, 308, 655, 1177, 310, 1354, 1920, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1718, 272, 351, 217, 401, 1038, 1311, 1115, 1923, 1301, 1750, 752, 905, 1177, 883, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 299, 306, 1042, 192, 222, 1914, 138, 117, 1041, 1505, 2620, 647, 379, 648, 483, 1380, 1295, 1113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 840, 64, 204, 1342, 1141, 361, 1068, 1904, 944, 2094, 135, 1130, 964, 1370, 2589, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2011, 1232, 747, 1162, 1618, 790, 836, 2344, 722, 346, 674, 1484, 969, 601, 462, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2115, 251, 720, 970, 854, 1272, 1841, 0, 940, 1501, 698, 1944, 232, 490, 1193, 1684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 1771, 1102, 1284, 213, 883, 875, 1246, 923, 1660, 1953, 981, 1886, 247, 371, 269, 689, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16023, 16816, 16237, 15590, 16923, 16188, 16705, 16590] avg=16384.0000 max=16923 max_rank=4 min=15590 min_rank=3 imbalance_ratio=1.032898 max_load_reduction=114149 max_load_reduction_pct=0.870888 moved_to_echo=66406 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[58, 2552, 1478, 174, 2233, 34, 728, 341, 1112, 161, 1455, 269, 762, 653, 44, 374, 3424, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 566, 722, 1340, 1471, 450, 2339, 1283, 135, 1735, 552, 747, 1522, 250, 1830, 2016, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1760, 163, 535, 559, 164, 917, 1627, 657, 2341, 1839, 1667, 686, 431, 1462, 1013, 682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 298, 445, 1377, 311, 425, 1312, 65, 70, 822, 2410, 2117, 614, 192, 1061, 497, 1443, 1326, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 656, 81, 322, 1644, 726, 210, 973, 2120, 443, 1764, 168, 519, 737, 2122, 1719, 1129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2216, 488, 781, 994, 1873, 401, 1050, 2093, 1225, 177, 771, 999, 907, 576, 527, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2070, 386, 1068, 750, 817, 981, 1953, 0, 1020, 1370, 613, 1648, 406, 362, 1177, 1593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 1288, 1140, 1480, 168, 784, 1225, 1546, 961, 994, 1881, 1223, 1332, 658, 698, 293, 1022, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17338, 17185, 16503, 16191, 15333, 15436, 16214, 16872] avg=16384.0000 max=17338 max_rank=0 min=15333 min_rank=4 imbalance_ratio=1.058228 max_load_reduction=113734 max_load_reduction_pct=0.867722 moved_to_echo=63855 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[119, 2316, 1555, 136, 1877, 35, 728, 422, 962, 93, 1683, 614, 1105, 665, 57, 406, 2924, 1573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 709, 1166, 837, 1911, 256, 2503, 1076, 108, 1672, 588, 939, 1220, 389, 1541, 1867, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1728, 279, 352, 474, 152, 774, 1726, 627, 2649, 1096, 1200, 871, 578, 1708, 1072, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 492, 309, 340, 1234, 295, 245, 1408, 70, 65, 1005, 2013, 2143, 965, 421, 714, 446, 1315, 1312, 1171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 712, 65, 186, 1590, 719, 175, 1203, 1711, 446, 1643, 213, 882, 814, 1834, 2810, 826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2208, 1135, 707, 1273, 1949, 550, 1076, 1801, 917, 347, 939, 1423, 858, 921, 528, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2073, 694, 1036, 496, 499, 860, 1429, 0, 1000, 1469, 614, 1603, 236, 286, 1175, 1699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 1602, 659, 819, 209, 921, 1426, 1173, 922, 1554, 1917, 1123, 1912, 230, 795, 228, 1003, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17270, 17053, 15811, 15963, 15829, 17196, 15169, 16781] avg=16384.0000 max=17270 max_rank=0 min=15169 min_rank=6 imbalance_ratio=1.054077 max_load_reduction=113802 max_load_reduction_pct=0.868240 moved_to_echo=64975 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[78, 2721, 1105, 98, 1601, 20, 744, 317, 1007, 246, 1510, 297, 1294, 819, 65, 408, 2850, 1660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 501, 1117, 551, 1897, 343, 1980, 1255, 108, 1520, 521, 900, 1356, 377, 1541, 1858, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1747, 306, 410, 755, 220, 1120, 1379, 620, 2404, 1260, 1231, 355, 520, 1496, 1132, 569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 199, 340, 1327, 675, 382, 1899, 81, 55, 1286, 2056, 1950, 1040, 254, 859, 601, 1516, 1317, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 869, 67, 158, 1865, 920, 121, 541, 1758, 165, 1958, 306, 804, 744, 1461, 2190, 1053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2063, 1339, 821, 1068, 1685, 531, 866, 1881, 1134, 227, 879, 1217, 1283, 609, 405, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2022, 937, 1204, 558, 750, 1150, 2184, 0, 1119, 1624, 505, 1832, 253, 487, 2115, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 1213, 894, 1204, 79, 834, 1688, 1135, 777, 1327, 1541, 876, 1856, 215, 950, 159, 1153, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16840, 16095, 15524, 17334, 14980, 16378, 17789, 16132] avg=16384.0000 max=17789 max_rank=6 min=14980 min_rank=4 imbalance_ratio=1.085754 max_load_reduction=113283 max_load_reduction_pct=0.864281 moved_to_echo=65279 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[114, 2340, 657, 76, 2329, 86, 758, 278, 784, 69, 1471, 241, 860, 636, 49, 403, 3010, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 748, 665, 892, 1531, 326, 2653, 1188, 45, 1649, 391, 549, 870, 193, 1381, 1895, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2022, 291, 115, 613, 158, 897, 1827, 921, 2338, 1456, 1094, 398, 557, 1855, 1377, 842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 185, 918, 1152, 481, 397, 1660, 188, 210, 839, 2516, 2614, 742, 263, 319, 365, 1646, 1523, 919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 740, 71, 295, 1240, 925, 370, 1168, 2024, 782, 1919, 170, 1126, 853, 1589, 2725, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2447, 1226, 892, 1635, 1228, 338, 654, 3034, 1091, 233, 890, 1517, 1012, 494, 449, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1621, 258, 1398, 782, 846, 786, 1668, 0, 1397, 1294, 584, 1118, 622, 539, 1202, 1503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 1037, 513, 886, 138, 746, 1147, 1719, 993, 1538, 1814, 1142, 1401, 368, 527, 239, 1340, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15397, 15229, 16761, 17253, 17289, 17618, 15618, 15907] avg=16384.0000 max=17618 max_rank=5 min=15229 min_rank=1 imbalance_ratio=1.075317 max_load_reduction=113454 max_load_reduction_pct=0.865585 moved_to_echo=66432 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[44, 2279, 1407, 139, 1936, 61, 717, 442, 905, 117, 1932, 433, 650, 861, 66, 569, 2595, 1518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 1019, 974, 1072, 1591, 255, 2458, 1019, 52, 1590, 278, 882, 1259, 183, 1618, 2041, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1928, 247, 393, 595, 400, 1005, 1386, 721, 2508, 1419, 1447, 924, 715, 1681, 1076, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 376, 334, 1386, 578, 249, 2094, 85, 228, 583, 2039, 2558, 516, 297, 760, 460, 1396, 1454, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 877, 61, 158, 1546, 1115, 206, 1141, 1841, 474, 2040, 206, 624, 701, 1822, 2755, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2000, 1078, 757, 1117, 1729, 671, 973, 2561, 836, 233, 619, 1093, 847, 573, 515, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1909, 212, 1094, 727, 656, 1068, 1802, 0, 829, 1215, 894, 1629, 337, 408, 2114, 1529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 1216, 581, 1575, 90, 915, 1099, 868, 767, 995, 1742, 898, 1770, 171, 503, 460, 1048, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16671, 16549, 17137, 16867, 16432, 15812, 16423, 15181] avg=16384.0000 max=17137 max_rank=2 min=15181 min_rank=7 imbalance_ratio=1.045959 max_load_reduction=113935 max_load_reduction_pct=0.869255 moved_to_echo=63848 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[640, 19584, 9728, 1024, 16256, 512, 5760, 2816, 7296, 1024, 11648, 2944, 7936, 5888, 512, 3456, 23040, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[14720, 2176, 2560, 4352, 1792, 7168, 12544, 6912, 20224, 11136, 10368, 4736, 5248, 12800, 9344, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[15616, 3840, 9088, 5376, 5760, 8320, 14464, 0, 8576, 11648, 5376, 13440, 2944, 3968, 12032, 11648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[6528, 640, 1664, 12672, 7424, 1792, 7808, 15872, 4480, 16512, 2048, 7296, 6272, 13568, 19584, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2688, 2304, 3200, 10112, 3584, 2688, 14208, 768, 1152, 6656, 16512, 18304, 5888, 2432, 5760, 3840, 11776, 11136, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[17024, 8192, 6400, 10240, 13312, 4224, 6912, 17792, 7936, 2048, 6016, 11136, 8576, 4992, 3840, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1536, 5888, 7936, 7296, 13312, 2304, 20096, 8960, 768, 13184, 4352, 6272, 9984, 2432, 11392, 15744, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2048, 11008, 6912, 9216, 1280, 6912, 9856, 10624, 6784, 12160, 14848, 7936, 13568, 2432, 5632, 1920, 8704, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[2, 46, 4069, 0, 3840, 116, 14, 3, 62, 11, 1, 2935, 4091, 3970, 0, 4, 760, 3, 12, 15, 74, 4091, 4, 4, 106, 0, 1, 33, 0, 3917, 4062, 1, 3646, 2, 1770, 0, 64, 0, 3985, 4, 173, 29, 14, 0, 7, 9, 161, 0, 0, 3984, 3964, 1, 113, 51, 125, 17, 4052, 4088, 0, 1, 4092, 3, 11, 0, 1, 465, 42, 3988, 427, 0, 0, 43, 3468, 3085, 36, 1, 0, 8, 1114, 0, 1775, 2322, 56, 2452, 4081, 4043, 65, 3985, 2428, 0, 8, 24, 0, 0, 2143, 0, 33, 4087, 3967, 419, 2853, 0, 58, 3, 0, 25, 0, 0, 440, 18, 0, 0, 4067, 455, 0, 142, 0, 3046, 1, 2687, 577, 968, 0, 2878, 0, 1726, 1, 3948] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3931, 0, 3997, 4, 2, 410, 11, 9, 0, 150, 2886, 4003, 0, 1, 241, 2, 0, 2, 6, 4092, 1, 4, 2662, 6, 0, 41, 0, 4076, 4089, 0, 4077, 0, 1922, 0, 0, 115, 4096, 0, 50, 19, 0, 1, 0, 103, 0, 42, 0, 4093, 3194, 0, 48, 0, 1, 3, 4002, 3964, 7, 3, 1949, 0, 46, 0, 0, 223, 41, 4054, 52, 27, 1700, 0, 3922, 3604, 10, 0, 0, 1, 3895, 25, 3375, 736, 165, 3239, 3753, 4043, 242, 3872, 3030, 0, 68, 1, 9, 0, 1905, 4, 56, 3698, 4087, 223, 4091, 63, 11, 4, 0, 0, 0, 1, 3405, 0, 0, 21, 674, 1, 0, 0, 0, 30, 2, 1865, 898, 562, 0, 3811, 19, 3102, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4095, 4, 3945, 0, 94, 3901, 217, 1061, 0, 423, 2953, 4096, 0, 45, 2, 0, 2, 4, 0, 3983, 0, 0, 1580, 0, 38, 0, 0, 4096, 3657, 15, 3999, 0, 1107, 0, 78, 126, 3942, 1, 4, 1, 0, 0, 0, 1, 0, 0, 18, 3076, 4066, 0, 0, 0, 0, 1, 4096, 4077, 0, 1, 2134, 1, 0, 48, 0, 117, 5, 4092, 240, 59, 176, 20, 2925, 3946, 0, 0, 2, 0, 3694, 0, 3741, 1191, 577, 2668, 4080, 4088, 2309, 4094, 3592, 0, 4, 42, 27, 232, 46, 1, 4, 4093, 4096, 186, 3728, 0, 0, 0, 0, 1, 0, 8, 147, 0, 1, 0, 2097, 1, 0, 0, 0, 433, 260, 1618, 291, 150, 0, 3762, 2, 3143, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 2, 2498, 0, 3730, 3, 35, 1684, 8, 0, 0, 698, 3330, 3990, 2, 6, 945, 0, 78, 42, 563, 3997, 3, 53, 1646, 0, 2, 0, 1, 4057, 3774, 0, 1292, 31, 3, 0, 1, 2, 3748, 1, 6, 68, 408, 0, 16, 0, 21, 1, 0, 4011, 3091, 0, 43, 75, 15, 16, 4063, 4073, 0, 0, 2863, 17, 1, 1, 10, 383, 2, 4076, 45, 0, 1508, 0, 3541, 4042, 464, 19, 0, 6, 3139, 34, 230, 523, 12, 2292, 4089, 4077, 60, 3783, 3992, 9, 10, 0, 14, 4, 1761, 72, 0, 3884, 4042, 46, 4062, 0, 99, 0, 1, 36, 20, 2, 3651, 0, 6, 0, 4008, 165, 0, 18, 20, 2089, 187, 1056, 1607, 727, 16, 4051, 62, 3901, 9, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 7, 3885, 48, 4096, 7, 0, 104, 15, 34, 84, 56, 4058, 4023, 1, 171, 1718, 58, 65, 368, 146, 4081, 0, 0, 3, 16, 15, 0, 0, 3999, 2965, 0, 3623, 0, 1743, 5, 823, 0, 4092, 0, 0, 3603, 0, 1, 13, 0, 26, 0, 4, 3668, 3108, 0, 29, 13, 182, 0, 4092, 3335, 3, 0, 3696, 0, 0, 60, 0, 575, 36, 3592, 589, 203, 1266, 0, 3710, 4062, 0, 39, 4, 187, 3833, 0, 1223, 1410, 256, 529, 3898, 4091, 3, 4045, 3635, 9, 3, 1, 31, 0, 1017, 159, 791, 4076, 4018, 28, 2311, 265, 80, 1, 0, 8, 0, 0, 299, 0, 39, 0, 3957, 141, 0, 0, 3, 646, 18, 3129, 280, 1908, 0, 3904, 48, 689, 0, 3885] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3713, 51, 4070, 0, 0, 1, 221, 0, 4, 922, 4019, 4052, 0, 363, 1076, 0, 1, 0, 336, 4094, 0, 0, 651, 0, 3, 0, 22, 3897, 4079, 57, 4096, 56, 3451, 0, 0, 512, 4025, 3, 56, 12, 69, 14, 0, 30, 4, 88, 20, 4087, 4074, 7, 0, 0, 0, 1, 4096, 3832, 96, 162, 3482, 3, 4, 0, 0, 747, 1, 3525, 11, 16, 89, 13, 2603, 4015, 2, 0, 48, 0, 3891, 5, 1144, 2182, 8, 1355, 4037, 4076, 1452, 4062, 3894, 0, 2, 17, 189, 4, 2854, 58, 2, 3910, 4040, 1, 4087, 2, 1, 4, 0, 0, 546, 0, 1021, 0, 34, 31, 2849, 1, 5, 13, 63, 882, 847, 682, 179, 1703, 9, 109, 134, 1624, 0, 4081] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3148, 0, 3962, 0, 0, 1356, 104, 4, 0, 292, 4032, 3580, 0, 38, 5, 0, 0, 6, 61, 4073, 1, 0, 541, 0, 4, 1, 5, 3809, 1044, 193, 3772, 0, 984, 0, 2, 11, 4085, 2, 122, 3, 98, 0, 0, 1, 0, 3, 0, 4093, 3399, 0, 0, 0, 0, 0, 4091, 4096, 0, 13, 3782, 0, 2, 1, 0, 3006, 0, 3856, 3, 29, 398, 0, 304, 4093, 6, 0, 164, 0, 3195, 0, 310, 1888, 27, 3712, 4096, 4092, 2052, 4084, 3971, 0, 0, 0, 8, 652, 1955, 34, 65, 3807, 4096, 1, 3482, 0, 22, 1, 0, 12, 1, 1, 2547, 0, 1, 0, 2023, 0, 9, 6, 1, 3383, 2, 3744, 819, 245, 0, 2728, 1001, 2263, 0, 4058] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 5, 4076, 18, 4081, 37, 10, 2078, 33, 6, 0, 58, 4069, 3963, 9, 121, 392, 0, 0, 1, 42, 4052, 0, 99, 153, 0, 82, 1, 28, 4056, 2746, 1, 1172, 0, 94, 0, 0, 20, 4096, 1, 1385, 9, 0, 0, 1, 12, 0, 2, 0, 3893, 3956, 0, 0, 1, 126, 2, 4094, 3810, 13, 2, 4007, 0, 15, 3, 2, 59, 604, 4096, 56, 31, 69, 160, 473, 3138, 46, 2, 0, 0, 3981, 16, 1114, 2459, 77, 4013, 4069, 4096, 3282, 4010, 3995, 0, 1, 2, 0, 3, 3779, 736, 19, 4095, 4096, 4, 3473, 0, 6, 0, 1, 0, 0, 1, 1309, 0, 0, 0, 3584, 1, 0, 0, 18, 513, 566, 1078, 1445, 1115, 3, 323, 0, 4056, 0, 4066] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 4, 2777, 0, 94, 3901, 217, 1061, 0, 423, 2953, 4096, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 4, 0, 3983, 0, 0, 1580, 0, 38, 0, 0, 4096, 3657, 15, 1782, 1398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 1107, 0, 78, 126, 3942, 1, 4, 1, 0, 0, 0, 1, 0, 0, 4088, 2682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3076, 4066, 0, 0, 0, 0, 1, 1585, 4077, 0, 1, 2134, 1, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 5, 4092, 240, 59, 176, 20, 2925, 3946, 0, 0, 2, 0, 3694, 0, 490, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3741, 1191, 577, 2668, 0, 0, 2309, 3604, 3592, 0, 4, 42, 27, 232, 46, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4093, 4096, 186, 3728, 0, 0, 0, 0, 1, 0, 8, 147, 0, 1, 0, 1416, 729, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2097, 1, 0, 0, 0, 433, 260, 1618, 291, 150, 0, 3762, 2, 3143, 0, 2679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19666, 16557, 16029, 15007, 16216, 18034, 15127, 14436] avg=16384.0000 max=19666 max_rank=0 min=14436 min_rank=7 imbalance_ratio=1.200317 max_load_reduction=111406 max_load_reduction_pct=0.849960 moved_to_echo=63813 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3931, 0, 2813, 4, 2, 410, 11, 9, 0, 150, 2886, 4003, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 2, 0, 2, 6, 4092, 1, 4, 2662, 6, 0, 41, 0, 4076, 4089, 0, 1741, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 1922, 0, 0, 115, 4096, 0, 50, 19, 0, 1, 0, 103, 0, 42, 4043, 2467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3194, 0, 48, 0, 1, 3, 1549, 3964, 7, 3, 1949, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 223, 41, 4054, 52, 27, 1700, 0, 3922, 3604, 10, 0, 0, 1, 3895, 25, 463, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3375, 736, 165, 3239, 0, 0, 242, 3409, 3030, 0, 68, 1, 9, 0, 1905, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 3698, 4087, 223, 4091, 63, 11, 4, 0, 0, 0, 1, 3405, 0, 0, 21, 1415, 712, 728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 674, 1, 0, 0, 0, 30, 2, 1865, 898, 562, 0, 3811, 19, 3102, 0, 2676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14220, 18249, 16935, 14857, 18473, 16183, 18515, 13640] avg=16384.0000 max=18515 max_rank=6 min=13640 min_rank=7 imbalance_ratio=1.130066 max_load_reduction=112557 max_load_reduction_pct=0.858742 moved_to_echo=66811 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 46, 4069, 0, 2703, 116, 14, 3, 62, 11, 1, 2935, 4091, 3970, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 760, 3, 12, 15, 74, 4091, 4, 4, 106, 0, 1, 33, 0, 3917, 4062, 1, 1763, 1398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3646, 2, 1770, 0, 64, 0, 3985, 4, 173, 29, 14, 0, 7, 9, 161, 0, 4043, 2683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 3964, 1, 113, 51, 125, 17, 1568, 4088, 0, 1, 4092, 3, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 465, 42, 3988, 427, 0, 0, 43, 3468, 3085, 36, 1, 0, 8, 1114, 0, 477, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 2322, 56, 2452, 0, 0, 65, 3508, 2428, 0, 8, 24, 0, 0, 2143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4087, 3967, 419, 2853, 0, 58, 3, 0, 25, 0, 0, 440, 18, 0, 0, 1365, 721, 699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 455, 0, 142, 0, 3046, 1, 2687, 577, 968, 0, 2878, 0, 1726, 1, 2583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18027, 16244, 16590, 18018, 13593, 14781, 14688, 19131] avg=16384.0000 max=19131 max_rank=7 min=13593 min_rank=4 imbalance_ratio=1.167664 max_load_reduction=111941 max_load_reduction_pct=0.854042 moved_to_echo=62193 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3148, 0, 2789, 0, 0, 1356, 104, 4, 0, 292, 4032, 3580, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 6, 61, 4073, 1, 0, 541, 0, 4, 1, 5, 3809, 1044, 193, 1780, 1404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3772, 0, 984, 0, 2, 11, 4085, 2, 122, 3, 98, 0, 0, 1, 0, 3, 4092, 2692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3399, 0, 0, 0, 0, 0, 1583, 4096, 0, 13, 3782, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3006, 0, 3856, 3, 29, 398, 0, 304, 4093, 6, 0, 164, 0, 3195, 0, 489, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 1888, 27, 3712, 0, 0, 2052, 3595, 3971, 0, 0, 0, 8, 652, 1955, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 3807, 4096, 1, 3482, 0, 22, 1, 0, 12, 1, 1, 2547, 0, 1, 0, 1403, 728, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2023, 0, 9, 6, 1, 3383, 2, 3744, 819, 245, 0, 2728, 1001, 2263, 0, 2655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15343, 12927, 15867, 16969, 15995, 18204, 16888, 18879] avg=16384.0000 max=18879 max_rank=7 min=12927 min_rank=1 imbalance_ratio=1.152283 max_load_reduction=112193 max_load_reduction_pct=0.855965 moved_to_echo=69966 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 2498, 0, 2626, 3, 35, 1684, 8, 0, 0, 698, 3330, 3990, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 945, 0, 78, 42, 563, 3997, 3, 53, 1646, 0, 2, 0, 1, 4057, 3774, 0, 1768, 1402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1292, 31, 3, 0, 1, 2, 3748, 1, 6, 68, 408, 0, 16, 0, 21, 1, 4077, 2687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 3091, 0, 43, 75, 15, 16, 1572, 4073, 0, 0, 2863, 17, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 383, 2, 4076, 45, 0, 1508, 0, 3541, 4042, 464, 19, 0, 6, 3139, 34, 452, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 523, 12, 2292, 0, 0, 60, 3331, 3992, 9, 10, 0, 14, 4, 1761, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 4042, 46, 4062, 0, 99, 0, 1, 36, 20, 2, 3651, 0, 6, 0, 1417, 723, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 165, 0, 18, 20, 2089, 187, 1056, 1607, 727, 16, 4051, 62, 3901, 9, 2679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14882, 18331, 12362, 15778, 18146, 12310, 18668, 20595] avg=16384.0000 max=20595 max_rank=7 min=12310 min_rank=5 imbalance_ratio=1.257019 max_load_reduction=110477 max_load_reduction_pct=0.842873 moved_to_echo=69719 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 3885, 48, 2883, 7, 0, 104, 15, 34, 84, 56, 4058, 4023, 1, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1718, 58, 65, 368, 146, 4081, 0, 0, 3, 16, 15, 0, 0, 3999, 2965, 0, 1780, 1337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3623, 0, 1743, 5, 823, 0, 4092, 0, 0, 3603, 0, 1, 13, 0, 26, 0, 4091, 2561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3668, 3108, 0, 29, 13, 182, 0, 1584, 3335, 3, 0, 3696, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 36, 3592, 589, 203, 1266, 0, 3710, 4062, 0, 39, 4, 187, 3833, 0, 484, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1223, 1410, 256, 529, 0, 0, 3, 3561, 3635, 9, 3, 1, 31, 0, 1017, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 791, 4076, 4018, 28, 2311, 265, 80, 1, 0, 8, 0, 0, 299, 0, 39, 0, 1344, 728, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3957, 141, 0, 0, 3, 646, 18, 3129, 280, 1908, 0, 3904, 48, 689, 0, 2541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15376, 16551, 20581, 15682, 19047, 11837, 14734, 17264] avg=16384.0000 max=20581 max_rank=2 min=11837 min_rank=5 imbalance_ratio=1.256165 max_load_reduction=110491 max_load_reduction_pct=0.842979 moved_to_echo=62882 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4076, 18, 2864, 37, 10, 2078, 33, 6, 0, 58, 4069, 3963, 9, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 0, 1, 42, 4052, 0, 99, 153, 0, 82, 1, 28, 4056, 2746, 1, 1784, 1395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1172, 0, 94, 0, 0, 20, 4096, 1, 1385, 9, 0, 0, 1, 12, 0, 2, 4096, 2674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3893, 3956, 0, 0, 1, 126, 2, 1580, 3810, 13, 2, 4007, 0, 15, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 59, 604, 4096, 56, 31, 69, 160, 473, 3138, 46, 2, 0, 0, 3981, 16, 483, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 2459, 77, 4013, 0, 0, 3282, 3527, 3995, 0, 1, 2, 0, 3, 3779, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4095, 4096, 4, 3473, 0, 6, 0, 1, 0, 0, 1, 1309, 0, 0, 0, 1411, 730, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3584, 1, 0, 0, 18, 513, 566, 1078, 1445, 1115, 3, 323, 0, 4056, 0, 2655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17347, 14832, 13562, 17408, 13686, 22988, 15892, 15357] avg=16384.0000 max=22988 max_rank=5 min=13562 min_rank=2 imbalance_ratio=1.403076 max_load_reduction=108084 max_load_reduction_pct=0.824615 moved_to_echo=67923 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3713, 51, 2865, 0, 0, 1, 221, 0, 4, 922, 4019, 4052, 0, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 0, 1, 0, 336, 4094, 0, 0, 651, 0, 3, 0, 22, 3897, 4079, 57, 1782, 1383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 56, 3451, 0, 0, 512, 4025, 3, 56, 12, 69, 14, 0, 30, 4, 88, 4076, 2654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4087, 4074, 7, 0, 0, 0, 1, 1585, 3832, 96, 162, 3482, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 747, 1, 3525, 11, 16, 89, 13, 2603, 4015, 2, 0, 48, 0, 3891, 5, 486, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1144, 2182, 8, 1355, 0, 0, 1452, 3576, 3894, 0, 2, 17, 189, 4, 2854, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3910, 4040, 1, 4087, 2, 1, 4, 0, 0, 546, 0, 1021, 0, 34, 31, 1411, 729, 741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2849, 1, 5, 13, 63, 882, 847, 682, 179, 1703, 9, 109, 134, 1624, 0, 2670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16211, 17381, 19146, 17353, 15916, 16735, 16560, 11770] avg=16384.0000 max=19146 max_rank=2 min=11770 min_rank=7 imbalance_ratio=1.168579 max_load_reduction=111926 max_load_reduction_pct=0.853928 moved_to_echo=60981 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[25728, 128, 11136, 128, 1024, 896, 32128, 128, 1920, 3840, 640, 128, 128, 256, 256, 256, 32640, 21120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 29440, 128, 22400, 256, 256, 9600, 768, 1152, 128, 5632, 29440, 31744, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[23296, 768, 128, 256, 128, 11136, 1920, 15872, 6144, 7424, 128, 21632, 1280, 20608, 128, 21248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[12928, 12800, 1280, 20352, 0, 0, 9472, 28160, 28544, 128, 128, 128, 384, 896, 15488, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 30976, 28928, 128, 256, 256, 512, 128, 12672, 31360, 128, 256, 26112, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[1024, 31744, 32512, 1024, 28160, 384, 384, 128, 128, 128, 640, 128, 12928, 128, 128, 128, 11264, 5888, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 5632, 768, 31360, 1536, 384, 5248, 256, 20992, 30080, 640, 128, 256, 256, 26752, 128, 3840, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[5248, 128, 256, 512, 1280, 32512, 128, 256, 7424, 128, 256, 128, 128, 32000, 26496, 384, 14208, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4096, 189, 0, 0, 4092, 0, 1, 0, 0, 3974, 0, 4096, 2247, 0, 0, 4095, 2193, 4094, 0, 0, 4045, 41, 8, 0, 0, 4095, 0, 0, 0, 4095, 2, 3839, 12, 4096, 0, 4095, 0, 0, 4095, 3968, 0, 1, 0, 12, 1, 0, 0, 0, 0, 3892, 0, 0, 4095, 0, 0, 3, 0, 6, 1, 3365, 4094, 0, 422, 0, 0, 3965, 0, 0, 0, 1, 0, 574, 4096, 3756, 0, 75, 59, 4096, 0, 128, 0, 8, 0, 3682, 48, 4089, 0, 68, 0, 4, 0, 0, 0, 3, 4096, 0, 0, 75, 4096, 0, 0, 1, 0, 289, 0, 0, 4096, 7, 0, 4095, 0, 1, 236, 0, 4096, 3858, 0, 0, 49, 0, 4065, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4016, 0, 15, 2, 435, 0, 320, 0, 0, 3307, 3, 4094, 1706, 0, 0, 3931, 2983, 2391, 0, 0, 1877, 1508, 1677, 0, 0, 3659, 0, 0, 4, 4094, 198, 2353, 12, 4052, 184, 4079, 0, 0, 4096, 4066, 0, 0, 0, 29, 0, 0, 0, 0, 104, 3151, 0, 61, 3841, 0, 0, 0, 96, 0, 278, 3631, 3848, 4, 293, 0, 318, 3288, 0, 11, 27, 0, 21, 1798, 3236, 3923, 11, 1594, 426, 4094, 0, 196, 3, 27, 0, 2926, 998, 4084, 295, 3927, 5, 0, 0, 0, 0, 0, 4091, 3, 0, 409, 4067, 0, 2, 0, 1181, 63, 297, 0, 4052, 387, 0, 4095, 15, 0, 0, 0, 3836, 2793, 5, 0, 32, 0, 4092, 0, 24, 17, 0, 0, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[3979, 0, 0, 0, 3802, 9, 0, 10, 19, 3397, 0, 4029, 129, 24, 0, 3942, 670, 3876, 2, 0, 4073, 3048, 4036, 0, 1, 4095, 0, 0, 0, 4094, 720, 77, 0, 3440, 125, 683, 2, 0, 4076, 4095, 0, 0, 28, 0, 163, 0, 8, 0, 0, 1494, 0, 0, 4087, 3, 15, 0, 0, 0, 0, 4001, 3957, 0, 0, 0, 72, 138, 0, 3, 0, 7, 0, 2412, 3993, 1263, 0, 6, 38, 4096, 16, 2622, 0, 2045, 0, 2478, 15, 4094, 89, 4020, 0, 49, 59, 2, 2058, 0, 4091, 75, 0, 14, 4089, 0, 0, 0, 0, 1112, 0, 0, 4093, 0, 0, 4074, 51, 0, 2431, 0, 4096, 3062, 8, 8, 0, 0, 4057, 0, 0, 0, 3, 0, 0, 24] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[3746, 0, 0, 0, 2255, 0, 0, 4, 5, 4096, 13, 4055, 1905, 0, 0, 4096, 1960, 3555, 0, 0, 2243, 35, 49, 0, 4, 3733, 0, 0, 0, 4096, 922, 480, 6, 4087, 0, 4021, 0, 5, 4091, 4073, 0, 11, 0, 0, 29, 0, 27, 0, 177, 4085, 0, 1, 4096, 0, 1, 0, 227, 4, 3, 3975, 4090, 0, 1312, 0, 0, 4018, 0, 3, 0, 0, 9, 105, 4080, 2286, 204, 1, 0, 4094, 0, 2216, 4, 4089, 0, 4096, 0, 3705, 37, 249, 228, 2, 0, 0, 345, 4, 4086, 49, 0, 98, 3910, 5, 0, 9, 977, 5, 1, 0, 4092, 14, 0, 4096, 0, 0, 22, 2, 4093, 4087, 0, 0, 0, 0, 4076, 0, 0, 0, 36, 0, 55, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[3648, 0, 0, 0, 4093, 0, 0, 0, 0, 4086, 28, 4073, 1457, 0, 0, 4092, 1940, 3844, 0, 0, 4095, 513, 2032, 0, 0, 4049, 0, 0, 0, 4096, 6, 2238, 243, 4096, 0, 2642, 0, 8, 4092, 4068, 0, 0, 0, 1, 0, 0, 0, 0, 67, 2055, 0, 49, 3821, 0, 0, 0, 5, 6, 0, 2343, 4090, 0, 20, 0, 0, 3709, 0, 0, 0, 0, 0, 2868, 4060, 2083, 1, 0, 104, 4096, 0, 1613, 0, 55, 261, 232, 0, 4084, 1, 4004, 0, 7, 0, 0, 633, 0, 4096, 128, 0, 94, 4096, 0, 0, 0, 0, 597, 0, 0, 4096, 31, 0, 4083, 0, 0, 3127, 0, 4095, 2806, 0, 0, 37, 0, 4040, 0, 0, 1, 0, 0, 38, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4063, 5, 1, 0, 1764, 0, 588, 1, 0, 4096, 1, 4039, 0, 0, 0, 3838, 2046, 4093, 0, 0, 4095, 1916, 1707, 1, 0, 3829, 2, 1, 1, 4095, 2012, 723, 96, 4053, 0, 275, 16, 0, 4080, 2393, 0, 0, 0, 8, 0, 0, 0, 4, 2, 1932, 0, 123, 3960, 0, 0, 1, 10, 30, 1, 61, 4090, 0, 197, 0, 41, 3116, 0, 0, 0, 0, 0, 4074, 4095, 4093, 15, 1, 0, 4091, 0, 3951, 13, 4056, 3, 4095, 0, 4086, 4, 1, 0, 0, 0, 1, 257, 1, 4096, 401, 0, 1565, 4060, 22, 0, 2, 173, 70, 0, 28, 4083, 0, 0, 4093, 0, 2, 25, 0, 4072, 4047, 0, 1, 0, 0, 4007, 0, 1, 6, 0, 1, 74, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4055, 94, 0, 53, 3794, 75, 0, 96, 102, 2679, 0, 4047, 2964, 17, 11, 4094, 1413, 4096, 0, 31, 88, 1, 3958, 2, 0, 3883, 3, 26, 8, 4081, 2469, 4080, 0, 3798, 0, 1473, 0, 0, 4070, 3567, 0, 0, 1, 0, 0, 0, 0, 0, 641, 3873, 0, 0, 4071, 2, 0, 3, 1, 0, 3, 1867, 4092, 35, 192, 4, 0, 4093, 0, 2, 0, 0, 0, 1656, 4093, 13, 619, 1, 0, 4057, 0, 3122, 32, 116, 95, 421, 32, 4095, 0, 295, 17, 102, 1, 0, 363, 224, 4015, 78, 180, 0, 4081, 0, 30, 47, 486, 1, 22, 2, 3894, 1390, 1, 4094, 4, 0, 4068, 0, 4094, 3042, 61, 3, 1, 10, 4083, 0, 0, 0, 3, 0, 0, 20] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4095, 0, 34, 0, 2381, 16, 0, 41, 0, 3946, 0, 4096, 3933, 0, 4, 4096, 108, 4045, 0, 0, 649, 192, 1464, 0, 0, 4062, 106, 0, 12, 3866, 4015, 1031, 669, 4090, 0, 4095, 0, 0, 4003, 4044, 8, 3, 32, 1, 55, 3, 0, 6, 0, 3119, 0, 0, 4045, 22, 8, 234, 84, 0, 0, 3984, 4043, 0, 3790, 2, 435, 3948, 256, 9, 28, 0, 10, 910, 4061, 551, 0, 0, 18, 4068, 28, 71, 0, 3706, 9, 153, 2, 3101, 258, 0, 88, 0, 3, 3, 30, 0, 3788, 0, 17, 7, 4096, 0, 0, 1, 0, 1225, 0, 0, 4057, 1, 8, 3873, 0, 40, 1571, 0, 3929, 3985, 0, 31, 0, 0, 4094, 1, 3, 0, 50, 1, 16, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 189, 0, 0, 4092, 0, 1, 0, 0, 3974, 0, 0, 2247, 0, 0, 3952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2193, 4094, 0, 0, 4045, 41, 8, 0, 0, 2484, 0, 0, 0, 0, 2, 3839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 4095, 0, 0, 4095, 3968, 0, 1, 0, 12, 1, 0, 0, 0, 1611, 105, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 0, 0, 4095, 0, 0, 3, 0, 6, 1, 3365, 4094, 0, 422, 0, 1284, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 0, 0, 0, 1, 0, 574, 4096, 3756, 0, 75, 59, 2799, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3682, 48, 4089, 0, 68, 0, 4, 0, 0, 0, 3, 4096, 0, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 4096, 0, 0, 1, 0, 289, 0, 0, 4096, 7, 0, 4095, 0, 1, 2076, 765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 4096, 3858, 0, 0, 49, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 3331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18551, 16706, 18009, 17305, 15453, 13912, 15501, 15635] avg=16384.0000 max=18551 max_rank=0 min=13912 min_rank=5 imbalance_ratio=1.132263 max_load_reduction=112521 max_load_reduction_pct=0.858467 moved_to_echo=60501 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3979, 0, 0, 0, 3802, 9, 0, 10, 19, 3397, 0, 0, 129, 24, 0, 3804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 670, 3876, 2, 0, 4073, 3048, 4036, 0, 1, 2484, 0, 0, 0, 0, 720, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3440, 125, 683, 2, 0, 4076, 4095, 0, 0, 28, 0, 163, 0, 8, 0, 1611, 105, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 0, 0, 4087, 3, 15, 0, 0, 0, 0, 4001, 3957, 0, 0, 0, 1284, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 138, 0, 3, 0, 7, 0, 2412, 3993, 1263, 0, 6, 38, 2799, 16, 2622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2045, 0, 2478, 15, 4094, 89, 4020, 0, 49, 59, 2, 2058, 0, 4091, 75, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4089, 0, 0, 0, 0, 1112, 0, 0, 4093, 0, 0, 4074, 51, 0, 2074, 753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2431, 0, 4096, 3062, 8, 8, 0, 0, 4057, 0, 0, 0, 3, 0, 0, 24, 3276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15173, 18987, 14349, 14979, 13369, 20990, 16260, 16965] avg=16384.0000 max=20990 max_rank=5 min=13369 min_rank=4 imbalance_ratio=1.281128 max_load_reduction=110082 max_load_reduction_pct=0.839859 moved_to_echo=67584 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4016, 0, 15, 2, 435, 0, 320, 0, 0, 3307, 3, 0, 1706, 0, 0, 3794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2983, 2391, 0, 0, 1877, 1508, 1677, 0, 0, 2220, 0, 0, 4, 0, 198, 2353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4052, 184, 4079, 0, 0, 4096, 4066, 0, 0, 0, 29, 0, 0, 0, 0, 1439, 107, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 3151, 0, 61, 3841, 0, 0, 0, 96, 0, 278, 3631, 3848, 4, 293, 0, 1283, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 3288, 0, 11, 27, 0, 21, 1798, 3236, 3923, 11, 1594, 426, 2798, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 27, 0, 2926, 998, 4084, 295, 3927, 5, 0, 0, 0, 0, 0, 4091, 3, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 4067, 0, 2, 0, 1181, 63, 297, 0, 4052, 387, 0, 4095, 15, 0, 2073, 766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 2793, 5, 0, 32, 0, 4092, 0, 24, 17, 0, 0, 0, 5, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13598, 15211, 18077, 16727, 17647, 18273, 17407, 14132] avg=16384.0000 max=18273 max_rank=5 min=13598 min_rank=0 imbalance_ratio=1.115295 max_load_reduction=112799 max_load_reduction_pct=0.860588 moved_to_echo=67459 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3746, 0, 0, 0, 2255, 0, 0, 4, 5, 4096, 13, 0, 1905, 0, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1960, 3555, 0, 0, 2243, 35, 49, 0, 4, 2264, 0, 0, 0, 0, 922, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4087, 0, 4021, 0, 5, 4091, 4073, 0, 11, 0, 0, 29, 0, 27, 0, 1469, 105, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 4085, 0, 1, 4096, 0, 1, 0, 227, 4, 3, 3975, 4090, 0, 1312, 0, 1283, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 3, 0, 0, 9, 105, 4080, 2286, 204, 1, 0, 2798, 0, 2216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4089, 0, 4096, 0, 3705, 37, 249, 228, 2, 0, 0, 345, 4, 4086, 49, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 3910, 5, 0, 9, 977, 5, 1, 0, 4092, 14, 0, 4096, 0, 0, 2076, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 2, 4093, 4087, 0, 0, 0, 0, 4076, 0, 0, 0, 36, 0, 55, 11, 3298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15977, 11512, 17937, 19397, 15720, 18809, 16040, 15680] avg=16384.0000 max=19397 max_rank=3 min=11512 min_rank=1 imbalance_ratio=1.183899 max_load_reduction=111675 max_load_reduction_pct=0.852013 moved_to_echo=66249 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3648, 0, 0, 0, 4093, 0, 0, 0, 0, 4086, 28, 0, 1457, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1940, 3844, 0, 0, 4095, 513, 2032, 0, 0, 2456, 0, 0, 0, 0, 6, 2238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 4096, 0, 2642, 0, 8, 4092, 4068, 0, 0, 0, 1, 0, 0, 0, 0, 1593, 107, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 2055, 0, 49, 3821, 0, 0, 0, 5, 6, 0, 2343, 4090, 0, 20, 0, 1284, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3709, 0, 0, 0, 0, 0, 2868, 4060, 2083, 1, 0, 104, 2799, 0, 1613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 261, 232, 0, 4084, 1, 4004, 0, 7, 0, 0, 633, 0, 4096, 128, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 4096, 0, 0, 0, 0, 597, 0, 0, 4096, 31, 0, 4083, 0, 0, 2074, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3127, 0, 4095, 2806, 0, 0, 37, 0, 4040, 0, 0, 1, 0, 0, 38, 0, 3311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17261, 17124, 16863, 13883, 17237, 15416, 15833, 17455] avg=16384.0000 max=17455 max_rank=7 min=13883 min_rank=3 imbalance_ratio=1.065369 max_load_reduction=113617 max_load_reduction_pct=0.866829 moved_to_echo=65941 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4063, 5, 1, 0, 1764, 0, 588, 1, 0, 4096, 1, 0, 0, 0, 0, 3699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2046, 4093, 0, 0, 4095, 1916, 1707, 1, 0, 2319, 2, 1, 1, 0, 2012, 723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 4053, 0, 275, 16, 0, 4080, 2393, 0, 0, 0, 8, 0, 0, 0, 4, 1510, 107, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1932, 0, 123, 3960, 0, 0, 1, 10, 30, 1, 61, 4090, 0, 197, 0, 1287, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 3116, 0, 0, 0, 0, 0, 4074, 4095, 4093, 15, 1, 0, 2784, 0, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4056, 3, 4095, 0, 4086, 4, 1, 0, 0, 0, 1, 257, 1, 4096, 401, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1565, 4060, 22, 0, 2, 173, 70, 0, 28, 4083, 0, 0, 4093, 0, 2, 2074, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 4072, 4047, 0, 1, 0, 0, 4007, 0, 1, 6, 0, 1, 74, 0, 3284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14218, 18916, 12562, 11833, 22170, 18928, 16927, 15518] avg=16384.0000 max=22170 max_rank=4 min=11833 min_rank=3 imbalance_ratio=1.353149 max_load_reduction=108902 max_load_reduction_pct=0.830856 moved_to_echo=73543 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4055, 94, 0, 53, 3794, 75, 0, 96, 102, 2679, 0, 0, 2964, 17, 11, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1413, 4096, 0, 31, 88, 1, 3958, 2, 0, 2355, 3, 26, 8, 0, 2469, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 0, 1473, 0, 0, 4070, 3567, 0, 0, 1, 0, 0, 0, 0, 0, 1528, 106, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 3873, 0, 0, 4071, 2, 0, 3, 1, 0, 3, 1867, 4092, 35, 192, 4, 1272, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2, 0, 0, 0, 1656, 4093, 13, 619, 1, 0, 2772, 0, 3122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 116, 95, 421, 32, 4095, 0, 295, 17, 102, 1, 0, 363, 224, 4015, 78, 1908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 4081, 0, 30, 47, 486, 1, 22, 2, 3894, 1390, 1, 4094, 4, 0, 2067, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 4094, 3042, 61, 3, 1, 10, 4083, 0, 0, 0, 3, 0, 0, 20, 3290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17891, 18530, 14556, 16199, 16371, 11794, 17056, 18675] avg=16384.0000 max=18675 max_rank=7 min=11794 min_rank=5 imbalance_ratio=1.139832 max_load_reduction=112397 max_load_reduction_pct=0.857521 moved_to_echo=63896 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 34, 0, 2381, 16, 0, 41, 0, 3946, 0, 0, 3933, 0, 4, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 4045, 0, 0, 649, 192, 1464, 0, 0, 2464, 106, 0, 12, 0, 4015, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 669, 4090, 0, 4095, 0, 0, 4003, 4044, 8, 3, 32, 1, 55, 3, 0, 6, 1598, 99, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3119, 0, 0, 4045, 22, 8, 234, 84, 0, 0, 3984, 4043, 0, 3790, 2, 1275, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 3948, 256, 9, 28, 0, 10, 910, 4061, 551, 0, 0, 18, 2780, 28, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3706, 9, 153, 2, 3101, 258, 0, 88, 0, 3, 3, 30, 0, 3788, 0, 1809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 7, 4096, 0, 0, 1, 0, 1225, 0, 0, 4057, 1, 8, 3873, 0, 40, 1958, 765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1571, 0, 3929, 3985, 0, 31, 0, 0, 4094, 1, 3, 0, 50, 1, 16, 0, 3331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18403, 14086, 18719, 20749, 13105, 12950, 16048, 17012] avg=16384.0000 max=20749 max_rank=3 min=12950 min_rank=5 imbalance_ratio=1.266418 max_load_reduction=110323 max_load_reduction_pct=0.841698 moved_to_echo=59115 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[31744, 384, 128, 128, 22656, 128, 1024, 256, 128, 29696, 128, 0, 14464, 128, 128, 31104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[13440, 30080, 128, 128, 21248, 7296, 14976, 128, 128, 19072, 128, 128, 128, 0, 10368, 14848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[11520, 128, 32384, 27776, 128, 128, 128, 128, 32640, 128, 128, 128, 128, 128, 256, 128, 26496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1024, 23680, 0, 256, 32128, 128, 128, 256, 512, 128, 384, 23296, 32384, 128, 6272, 128, 10368, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 2304, 32512, 128, 128, 128, 2944, 3456, 384, 128, 32512, 1920, 128, 32512, 128, 128, 16512, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[896, 26368, 256, 128, 128, 128, 128, 14464, 31744, 18048, 896, 1792, 768, 22400, 128, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[1152, 31744, 384, 21376, 128, 128, 32640, 30336, 128, 128, 128, 128, 256, 128, 128, 128, 12416, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 14208, 384, 18176, 1152, 31360, 768, 12672, 384, 256, 128, 128, 3712, 256, 32384, 768, 15232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4091, 36, 0, 0, 4094, 3, 4081, 4078, 4, 0, 3852, 248, 4096, 0, 0, 4, 1, 1830, 0, 0, 0, 2, 2, 0, 0, 4091, 4096, 1657, 4096, 0, 0, 3919, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 19, 64, 4, 0, 0, 4096, 780, 2539, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 369, 0, 0, 106, 0, 3845, 0, 0, 3507, 0, 0, 256, 0, 0, 0, 0, 0, 4043, 19, 3543, 0, 0, 3865, 117, 4096, 108, 0, 2, 3, 2549, 0, 0, 374, 4062, 0, 2505, 403, 0, 4095, 4095, 4092, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 28, 4096, 0, 4096, 0, 13, 0, 4094, 394] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4085, 484, 0, 0, 3209, 24, 3016, 4096, 2, 1, 303, 222, 3746, 0, 0, 7, 86, 4041, 10, 0, 35, 2, 0, 5, 0, 4061, 4014, 664, 4087, 65, 10, 4085, 0, 0, 0, 0, 21, 4093, 4078, 1, 0, 0, 82, 3689, 0, 0, 2, 3936, 0, 4004, 4068, 0, 0, 0, 31, 4083, 0, 18, 0, 3736, 0, 10, 1, 0, 0, 3988, 0, 3811, 16, 0, 52, 8, 0, 3967, 0, 0, 0, 34, 2, 0, 2, 16, 3830, 143, 3933, 0, 0, 4046, 451, 4091, 27, 346, 91, 0, 0, 0, 1, 36, 3799, 10, 754, 109, 2, 4071, 4070, 3971, 0, 9, 0, 4, 50, 139, 0, 0, 0, 4, 0, 9, 333, 174, 4095, 1, 4088, 0, 35, 0, 4081, 160] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4094, 0, 32, 6, 3826, 555, 364, 3809, 2, 3, 1008, 8, 4085, 2, 1, 0, 0, 4058, 0, 1, 0, 169, 0, 0, 0, 4091, 4085, 981, 4018, 4, 0, 4091, 203, 32, 42, 2, 0, 4077, 4081, 116, 5, 20, 4, 3675, 1, 77, 0, 3302, 0, 3529, 4094, 0, 0, 1, 0, 4073, 1, 0, 12, 4096, 1, 0, 3, 0, 6, 3596, 5, 863, 2332, 0, 3487, 0, 0, 4094, 0, 0, 0, 0, 1439, 0, 1, 0, 119, 19, 22, 24, 2, 4089, 0, 4011, 11, 0, 0, 2, 0, 7, 0, 3208, 4086, 1, 3551, 6, 0, 4093, 4086, 3379, 2096, 0, 0, 2, 2, 0, 0, 2, 0, 0, 1, 0, 0, 0, 4089, 10, 2304, 0, 1, 8, 3754, 1524] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4096, 7, 0, 0, 4077, 369, 3980, 4089, 14, 0, 652, 364, 4096, 0, 0, 49, 0, 891, 3, 1, 0, 0, 5, 0, 0, 4087, 4065, 4096, 4078, 0, 1, 3796, 30, 1, 0, 0, 0, 4067, 4096, 9, 0, 5, 229, 4073, 4, 322, 14, 3709, 0, 4081, 4095, 2, 0, 0, 0, 4084, 0, 0, 0, 4094, 0, 0, 16, 0, 26, 3538, 174, 0, 2707, 0, 4074, 0, 4, 72, 1, 0, 0, 1, 0, 1, 0, 0, 1773, 50, 3186, 63, 0, 4093, 4, 4021, 13, 31, 28, 0, 15, 11, 0, 272, 4028, 0, 863, 45, 0, 4096, 3362, 2822, 2542, 0, 2, 0, 5, 0, 1, 0, 0, 0, 298, 1, 0, 0, 4073, 0, 4045, 0, 0, 229, 4095, 660] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[2834, 0, 0, 0, 3721, 0, 4083, 4023, 0, 176, 0, 489, 4084, 2, 0, 244, 51, 3971, 3, 13, 0, 0, 3, 9, 0, 3832, 4038, 3205, 3920, 0, 0, 4056, 0, 1, 172, 49, 141, 3079, 4065, 1220, 0, 0, 0, 2393, 203, 5, 14, 4096, 686, 4051, 4051, 0, 0, 0, 0, 4043, 125, 0, 0, 3772, 0, 0, 0, 0, 0, 1582, 10, 1448, 2617, 28, 3197, 0, 60, 3591, 12, 0, 0, 0, 9, 3, 2, 8, 3420, 0, 939, 1, 66, 4065, 0, 3278, 0, 1, 1, 416, 0, 37, 38, 0, 4079, 33, 3997, 0, 0, 3963, 4070, 3484, 0, 0, 12, 0, 76, 0, 616, 0, 0, 65, 570, 59, 2, 0, 3718, 1, 1891, 16, 2, 654, 4012, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4095, 51, 0, 0, 3970, 510, 3466, 4092, 0, 0, 2973, 1942, 4096, 0, 0, 0, 0, 1936, 13, 0, 0, 5, 9, 1, 0, 3968, 4080, 1047, 4096, 1, 0, 3171, 6, 0, 0, 0, 0, 4070, 4096, 191, 1, 0, 11, 448, 0, 1, 0, 4021, 0, 950, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2906, 0, 83, 4002, 0, 3836, 0, 0, 4085, 0, 2, 2, 1, 57, 0, 0, 2, 376, 0, 3314, 4, 0, 4066, 19, 4096, 2, 1, 0, 0, 8, 5, 0, 3475, 4078, 0, 2460, 1, 0, 4096, 4096, 4096, 112, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 3, 0, 4091, 1, 3989, 0, 2, 2, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4096, 1139, 0, 0, 4083, 56, 4015, 4096, 85, 1, 4020, 297, 3827, 240, 0, 0, 0, 684, 0, 26, 0, 2, 0, 0, 0, 4, 4072, 3700, 4096, 0, 3, 55, 1, 3, 0, 0, 3, 3651, 4094, 0, 0, 0, 0, 3631, 11, 0, 5, 4096, 598, 4083, 4095, 0, 2, 0, 57, 4087, 0, 0, 0, 3384, 7, 160, 1, 0, 0, 2020, 0, 1050, 4045, 0, 4095, 0, 0, 4092, 0, 0, 0, 0, 245, 0, 0, 98, 348, 12, 1851, 11, 0, 4093, 15, 3871, 0, 0, 0, 72, 103, 0, 5, 3, 4030, 0, 4061, 0, 0, 4096, 4091, 3563, 1668, 0, 63, 0, 0, 189, 0, 7, 0, 0, 73, 0, 0, 58, 3919, 29, 4074, 0, 36, 0, 4094, 426] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4066, 455, 0, 0, 3800, 143, 4078, 3977, 0, 0, 3403, 24, 4096, 49, 16, 0, 432, 3942, 17, 47, 0, 175, 1, 0, 0, 4069, 4089, 3773, 4096, 17, 0, 3678, 54, 5, 2, 0, 0, 3103, 4077, 0, 50, 28, 216, 790, 0, 9, 23, 3194, 0, 3408, 3962, 0, 0, 3, 0, 4067, 0, 0, 0, 4096, 0, 0, 0, 2, 1, 468, 11, 0, 758, 8, 189, 5, 1, 4034, 0, 0, 11, 88, 10, 1, 1, 16, 44, 0, 4091, 0, 115, 4043, 0, 4040, 61, 631, 0, 0, 1461, 0, 14, 3880, 4033, 0, 1347, 84, 0, 3906, 4093, 4095, 3821, 20, 1, 0, 7, 46, 98, 1, 0, 31, 2, 0, 0, 3, 3338, 0, 2478, 1, 47, 10, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4091, 36, 0, 0, 4094, 3, 4081, 0, 4, 0, 3852, 248, 2191, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1830, 0, 0, 0, 2, 2, 0, 0, 4091, 232, 1657, 4096, 0, 0, 3919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 19, 64, 4, 0, 0, 4096, 1862, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 780, 2539, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 632, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 106, 0, 3845, 0, 0, 3507, 0, 0, 256, 0, 0, 0, 4095, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 19, 3543, 0, 0, 3865, 117, 4096, 108, 0, 2, 3, 2549, 0, 3153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 3430, 0, 2505, 403, 0, 0, 4095, 4092, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 28, 4096, 0, 4096, 0, 13, 0, 4094, 394, 3622, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18604, 15830, 14321, 16281, 12634, 21498, 14920, 16984] avg=16384.0000 max=21498 max_rank=5 min=12634 min_rank=4 imbalance_ratio=1.312134 max_load_reduction=109574 max_load_reduction_pct=0.835983 moved_to_echo=66036 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4085, 484, 0, 0, 3209, 24, 3016, 0, 2, 1, 303, 222, 2004, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 4041, 10, 0, 35, 2, 0, 5, 0, 4061, 226, 664, 4087, 65, 10, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4093, 4078, 1, 0, 0, 82, 3689, 0, 0, 2, 3936, 1703, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 4068, 0, 0, 0, 31, 4083, 0, 18, 0, 3736, 0, 10, 1, 0, 591, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 0, 3811, 16, 0, 52, 8, 0, 3967, 0, 0, 0, 34, 2, 0, 4071, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 16, 3830, 143, 3933, 0, 0, 4046, 451, 4091, 27, 346, 91, 0, 0, 0, 3090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 36, 3208, 10, 754, 109, 2, 0, 4070, 3971, 0, 9, 0, 4, 50, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 9, 333, 174, 4095, 1, 4088, 0, 35, 0, 4081, 160, 3638, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13357, 17377, 17686, 16581, 16407, 20066, 12363, 17235] avg=16384.0000 max=20066 max_rank=5 min=12363 min_rank=6 imbalance_ratio=1.224731 max_load_reduction=111006 max_load_reduction_pct=0.846909 moved_to_echo=66071 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 32, 6, 3826, 555, 364, 0, 2, 3, 1008, 8, 2176, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 1, 0, 169, 0, 0, 0, 4091, 219, 981, 4018, 4, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 32, 42, 2, 0, 4077, 4081, 116, 5, 20, 4, 3675, 1, 77, 0, 3302, 1862, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3529, 4094, 0, 0, 1, 0, 4073, 1, 0, 12, 4096, 1, 0, 3, 0, 639, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3596, 5, 863, 2332, 0, 3487, 0, 0, 4094, 0, 0, 0, 0, 1439, 0, 4093, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 119, 19, 22, 24, 2, 4089, 0, 4011, 11, 0, 0, 2, 0, 7, 3148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3208, 3447, 1, 3551, 6, 0, 0, 4086, 3379, 2096, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 4089, 10, 2304, 0, 1, 8, 3754, 1524, 3382, 632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12077, 17632, 17585, 16496, 20342, 11455, 19778, 15707] avg=16384.0000 max=20342 max_rank=4 min=11455 min_rank=5 imbalance_ratio=1.241577 max_load_reduction=110730 max_load_reduction_pct=0.844803 moved_to_echo=67282 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 7, 0, 0, 4077, 369, 3980, 0, 14, 0, 652, 364, 2191, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 891, 3, 1, 0, 0, 5, 0, 0, 4087, 229, 4096, 4078, 0, 1, 3796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 1, 0, 0, 0, 4067, 4096, 9, 0, 5, 229, 4073, 4, 322, 14, 3709, 1862, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4095, 2, 0, 0, 0, 4084, 0, 0, 0, 4094, 0, 0, 16, 0, 627, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 3538, 174, 0, 2707, 0, 4074, 0, 4, 72, 1, 0, 0, 1, 0, 1, 4096, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1773, 50, 3186, 63, 0, 4093, 4, 4021, 13, 31, 28, 0, 15, 11, 3129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 272, 3401, 0, 863, 45, 0, 0, 3362, 2822, 2542, 0, 2, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 298, 1, 0, 0, 4073, 0, 4045, 0, 0, 229, 4095, 660, 3632, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15799, 17187, 18503, 17042, 15151, 16417, 13314, 17659] avg=16384.0000 max=18503 max_rank=2 min=13314 min_rank=6 imbalance_ratio=1.129333 max_load_reduction=112569 max_load_reduction_pct=0.858833 moved_to_echo=62541 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 1139, 0, 0, 4083, 56, 4015, 0, 85, 1, 4020, 297, 2047, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 684, 0, 26, 0, 2, 0, 0, 0, 4, 229, 3700, 4096, 0, 3, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 3, 3651, 4094, 0, 0, 0, 0, 3631, 11, 0, 5, 4096, 1740, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 4083, 4095, 0, 2, 0, 57, 4087, 0, 0, 0, 3384, 7, 160, 1, 0, 627, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2020, 0, 1050, 4045, 0, 4095, 0, 0, 4092, 0, 0, 0, 0, 245, 0, 4096, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 348, 12, 1851, 11, 0, 4093, 15, 3871, 0, 0, 0, 72, 103, 0, 3135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 3403, 0, 4061, 0, 0, 0, 4091, 3563, 1668, 0, 63, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 73, 0, 0, 58, 3919, 29, 4074, 0, 36, 0, 4094, 426, 3637, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20079, 8799, 17317, 17141, 20102, 13609, 17046, 16979] avg=16384.0000 max=20102 max_rank=4 min=8799 min_rank=1 imbalance_ratio=1.226929 max_load_reduction=110970 max_load_reduction_pct=0.846634 moved_to_echo=67736 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4095, 51, 0, 0, 3970, 510, 3466, 0, 0, 0, 2973, 1942, 2191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1936, 13, 0, 0, 5, 9, 1, 0, 3968, 230, 1047, 4096, 1, 0, 3171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4070, 4096, 191, 1, 0, 11, 448, 0, 1, 0, 4021, 1862, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 950, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 635, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2906, 0, 83, 4002, 0, 3836, 0, 0, 4085, 0, 2, 2, 1, 57, 0, 4096, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 376, 0, 3314, 4, 0, 4066, 19, 4096, 2, 1, 0, 0, 8, 5, 3141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3475, 3443, 0, 2460, 1, 0, 0, 4096, 4096, 112, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 4091, 1, 3989, 0, 2, 2, 4096, 0, 3633, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19198, 14477, 14789, 13916, 19529, 15034, 17684, 16445] avg=16384.0000 max=19529 max_rank=4 min=13916 min_rank=3 imbalance_ratio=1.191956 max_load_reduction=111543 max_load_reduction_pct=0.851006 moved_to_echo=68692 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2834, 0, 0, 0, 3721, 0, 4083, 0, 0, 176, 0, 489, 2184, 2, 0, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 3971, 3, 13, 0, 0, 3, 9, 0, 3832, 229, 3205, 3920, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 172, 49, 141, 3079, 4065, 1220, 0, 0, 0, 2393, 203, 5, 14, 4096, 1857, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 686, 4051, 4051, 0, 0, 0, 0, 4043, 125, 0, 0, 3772, 0, 0, 0, 0, 635, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1582, 10, 1448, 2617, 28, 3197, 0, 60, 3591, 12, 0, 0, 0, 9, 3, 3963, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 3420, 0, 939, 1, 66, 4065, 0, 3278, 0, 1, 1, 416, 0, 37, 3108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 3444, 33, 3997, 0, 0, 0, 4070, 3484, 0, 0, 12, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 0, 0, 65, 570, 59, 2, 0, 3718, 1, 1891, 16, 2, 654, 4012, 0, 3572, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13733, 19292, 17376, 17406, 16971, 15342, 15154, 15798] avg=16384.0000 max=19292 max_rank=1 min=13733 min_rank=0 imbalance_ratio=1.177490 max_load_reduction=111780 max_load_reduction_pct=0.852814 moved_to_echo=63265 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4066, 455, 0, 0, 3800, 143, 4078, 0, 0, 0, 3403, 24, 2191, 49, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 3942, 17, 47, 0, 175, 1, 0, 0, 4069, 231, 3773, 4096, 17, 0, 3678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 5, 2, 0, 0, 3103, 4077, 0, 50, 28, 216, 790, 0, 9, 23, 3194, 1862, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3408, 3962, 0, 0, 3, 0, 4067, 0, 0, 0, 4096, 0, 0, 0, 2, 628, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 468, 11, 0, 758, 8, 189, 5, 1, 4034, 0, 0, 11, 88, 10, 1, 3906, 445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 16, 44, 0, 4091, 0, 115, 4043, 0, 4040, 61, 631, 0, 0, 1461, 0, 3148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3880, 3405, 0, 1347, 84, 0, 0, 4093, 4095, 3821, 20, 1, 0, 7, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 1, 0, 31, 2, 0, 0, 3, 3338, 0, 2478, 1, 47, 10, 4096, 0, 3532, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18225, 20478, 13495, 16209, 9936, 17651, 20813, 14265] avg=16384.0000 max=20813 max_rank=6 min=9936 min_rank=4 imbalance_ratio=1.270325 max_load_reduction=110259 max_load_reduction_pct=0.841209 moved_to_echo=62665 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 18560, 256, 7296, 16640, 128, 22784, 128, 128, 27520, 128, 128, 384, 128, 1792, 128, 32512, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[2176, 26752, 32640, 128, 128, 128, 128, 32640, 128, 128, 128, 31488, 128, 256, 128, 128, 5120, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 256, 14080, 256, 20992, 128, 256, 32384, 640, 31616, 256, 1024, 128, 512, 4224, 128, 25088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[31488, 2176, 128, 128, 30848, 1664, 27136, 0, 128, 256, 16256, 3712, 17280, 384, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 11264, 27264, 128, 19584, 768, 128, 0, 32000, 29568, 10368, 128, 128, 128, 256, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[640, 21376, 128, 128, 128, 384, 128, 128, 0, 28288, 1920, 19200, 32512, 128, 128, 26880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[768, 128, 0, 128, 1024, 128, 384, 384, 31488, 128, 27008, 128, 256, 1024, 32384, 3200, 28672, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[384, 128, 256, 128, 256, 30336, 32768, 1664, 128, 128, 640, 18816, 256, 512, 128, 30464, 14720, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 713, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 32, 0, 0, 0, 4096, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 3312, 0, 0, 4096, 0, 4096, 0, 3967, 0, 1188, 1, 4, 0, 4096, 0, 0, 0, 0, 109, 0, 1, 4096, 4096, 0, 0, 0, 0, 4085, 0, 0, 21, 4091, 0, 3682, 0, 3756, 4095, 0, 2, 244, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 110, 4080, 0, 3, 0, 0, 0, 0, 0, 0, 3955, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 3480, 1, 1, 3, 0, 0, 0, 2, 4096, 0, 4, 4094, 0, 4093, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1, 7, 4096, 0, 0, 0, 0, 12, 515, 0, 0, 4094, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 4096, 4092, 0, 4088, 3, 0, 1, 4096, 0, 4095, 0, 53, 3582, 0, 0, 3739, 1, 3213, 0, 3927, 0, 717, 0, 20, 0, 3517, 7, 0, 0, 0, 0, 21, 0, 4083, 4094, 0, 0, 0, 2, 4096, 0, 0, 947, 4096, 0, 219, 0, 4066, 4079, 2, 3156, 3831, 0, 0, 0, 0, 1, 10, 0, 190, 0, 31, 0, 0, 4095, 0, 110, 0, 112, 0, 0, 3, 0, 1081, 4095, 4096, 0, 2, 4089, 0, 0, 4085, 4096, 4083, 0, 1, 0, 0, 0, 3850, 0, 0, 38, 0, 0, 0, 0, 4094, 2, 0, 4090, 0, 4052, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2329, 18, 4, 2, 199, 367, 0, 0, 4080, 0, 0, 147, 3417, 0, 0, 22, 4086, 3, 4010, 4095, 0, 4043, 0, 0, 0, 4095, 0, 4087, 0, 0, 2784, 0, 40, 3929, 0, 4088, 0, 4067, 0, 4096, 1, 4, 1, 4084, 56, 5, 0, 101, 0, 47, 0, 4008, 4010, 15, 0, 96, 0, 4059, 0, 0, 0, 3061, 0, 3392, 8, 2280, 4096, 24, 1, 1233, 158, 1, 1, 0, 38, 36, 0, 878, 91, 38, 5, 1, 4088, 1, 56, 3, 112, 2, 1, 0, 2, 3034, 3974, 4096, 0, 0, 3920, 0, 0, 4094, 4071, 4096, 0, 0, 0, 1, 46, 283, 3, 0, 5, 0, 7, 0, 6, 4066, 8, 0, 4022, 0, 1142, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 4096, 0, 1, 121, 27, 1, 957, 24, 15, 4086, 23, 0, 402, 0, 0, 0, 0, 4096, 0, 4072, 4094, 10, 4095, 4, 0, 0, 4094, 0, 4081, 0, 3, 4095, 0, 55, 3746, 1, 3606, 0, 3264, 0, 472, 2, 29, 2, 3990, 0, 0, 0, 0, 205, 0, 393, 4095, 4080, 32, 0, 0, 10, 4094, 0, 17, 0, 4047, 0, 0, 0, 4089, 4090, 0, 3, 271, 0, 0, 1, 0, 0, 31, 22, 1128, 0, 0, 0, 1, 4065, 5, 1420, 410, 0, 0, 0, 0, 14, 4033, 4083, 4044, 0, 0, 4095, 4, 0, 4092, 4035, 4056, 1, 2, 9, 2, 82, 4095, 0, 0, 0, 0, 0, 100, 0, 4088, 0, 0, 4063, 0, 4094, 2, 0, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4, 0, 2, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 0, 4096, 19, 4095, 4096, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 1, 2161, 0, 112, 4091, 0, 4025, 0, 3336, 0, 3057, 0, 69, 166, 4040, 0, 0, 0, 0, 0, 0, 314, 4096, 4064, 0, 0, 0, 0, 3984, 1, 0, 165, 3748, 2, 24, 0, 4090, 4096, 0, 6, 3761, 0, 0, 0, 0, 0, 151, 0, 321, 0, 90, 0, 0, 4093, 0, 0, 0, 105, 0, 0, 0, 0, 3988, 4096, 4038, 0, 0, 4096, 0, 0, 4096, 4026, 4096, 0, 0, 0, 0, 1, 3376, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[6, 0, 4090, 0, 0, 0, 129, 335, 0, 0, 0, 4096, 0, 0, 386, 27, 5, 0, 2, 4095, 0, 4085, 4083, 97, 4095, 0, 0, 0, 4051, 0, 4096, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 4047, 0, 3277, 0, 329, 2539, 4047, 456, 3, 0, 0, 2363, 0, 0, 4019, 4046, 0, 0, 829, 0, 2461, 0, 0, 0, 3798, 43, 3839, 4, 3950, 3662, 2, 158, 0, 1, 0, 0, 4, 89, 0, 0, 542, 0, 40, 12, 0, 4096, 2, 82, 30, 0, 0, 264, 0, 0, 282, 4096, 3947, 0, 2, 4096, 15, 0, 4096, 3804, 4017, 0, 0, 0, 0, 2, 4018, 0, 0, 25, 1, 0, 0, 0, 3726, 332, 8, 3962, 0, 3744, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[32, 0, 4089, 0, 0, 141, 0, 9, 0, 84, 0, 4090, 67, 30, 6, 1636, 0, 0, 39, 3810, 37, 4040, 4018, 36, 4070, 62, 0, 0, 4072, 0, 4096, 0, 0, 2866, 0, 2724, 3895, 0, 4054, 0, 3948, 0, 3688, 0, 1191, 0, 3984, 103, 1, 0, 247, 1969, 0, 0, 4095, 4096, 0, 0, 615, 0, 4054, 0, 7, 1, 3736, 0, 3909, 120, 541, 3816, 0, 4, 68, 0, 0, 0, 3, 11, 856, 0, 1943, 2, 342, 0, 0, 3783, 0, 10, 0, 0, 0, 0, 0, 24, 13, 4087, 3957, 3, 3, 4096, 194, 0, 3900, 3251, 4087, 7, 0, 0, 2, 23, 269, 0, 295, 29, 0, 0, 0, 0, 4087, 0, 0, 4095, 3, 3485, 2, 14, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 129, 4093, 43, 0, 0, 519, 0, 2, 0, 0, 4018, 0, 3, 0, 0, 0, 0, 0, 4075, 3, 4050, 4030, 2, 3998, 12, 0, 0, 4091, 0, 4092, 1, 10, 4095, 1, 1157, 4061, 39, 4077, 0, 3100, 0, 4096, 0, 0, 67, 3957, 0, 0, 0, 0, 112, 4, 0, 4058, 4063, 34, 0, 0, 77, 3888, 21, 0, 32, 3549, 77, 0, 580, 3913, 4094, 4, 709, 2932, 0, 9, 0, 0, 1, 5, 44, 805, 0, 2, 27, 0, 4043, 0, 171, 480, 0, 0, 0, 0, 0, 0, 4092, 4089, 0, 0, 4086, 0, 19, 4078, 3830, 4092, 0, 1, 42, 8, 0, 2933, 12, 3, 0, 0, 1, 0, 0, 4096, 2, 65, 4079, 0, 3983, 3, 3, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 713, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 32, 4096, 2506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 4096, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3312, 0, 0, 2257, 0, 4096, 0, 3967, 0, 1188, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 1, 4096, 4096, 0, 0, 0, 0, 4085, 0, 0, 21, 2565, 747, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3682, 0, 3756, 4095, 0, 2, 244, 0, 0, 0, 0, 4, 2, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 4080, 0, 3, 0, 0, 0, 0, 0, 0, 3955, 4096, 3388, 1590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1531, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3480, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 2, 4096, 0, 4, 4094, 0, 4093, 0, 0, 0, 0, 4063, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15557, 16451, 14825, 15753, 16968, 17222, 17300, 16996] avg=16384.0000 max=17300 max_rank=6 min=14825 min_rank=2 imbalance_ratio=1.055908 max_load_reduction=113772 max_load_reduction_pct=0.868011 moved_to_echo=68486 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 7, 4096, 0, 0, 0, 0, 12, 515, 0, 0, 4094, 0, 6, 0, 0, 4095, 2498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4092, 0, 4088, 3, 0, 1, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 3582, 0, 0, 2060, 1, 3213, 0, 3927, 0, 717, 0, 20, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4083, 4094, 0, 0, 0, 2, 4096, 0, 0, 947, 2561, 682, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 219, 0, 4066, 4079, 2, 3156, 3831, 0, 0, 0, 0, 1, 10, 0, 997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 31, 0, 0, 4095, 0, 110, 0, 112, 0, 0, 3, 0, 1081, 4095, 3388, 1585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 1528, 0, 0, 4085, 4096, 0, 0, 1, 0, 0, 0, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 4094, 2, 0, 4090, 0, 4052, 0, 0, 0, 0, 3489, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15324, 16444, 13580, 16514, 20457, 14690, 17658, 16405] avg=16384.0000 max=20457 max_rank=4 min=13580 min_rank=2 imbalance_ratio=1.248596 max_load_reduction=110615 max_load_reduction_pct=0.843925 moved_to_echo=69210 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4096, 0, 1, 121, 27, 1, 957, 24, 15, 4086, 23, 0, 402, 0, 4081, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4072, 4094, 10, 4095, 4, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 0, 55, 2057, 1, 3606, 0, 3264, 0, 472, 2, 29, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 393, 4095, 4080, 32, 0, 0, 10, 4094, 0, 17, 0, 2568, 686, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 4089, 4090, 0, 3, 271, 0, 0, 1, 0, 0, 31, 22, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1128, 0, 0, 0, 1, 4065, 5, 1420, 410, 0, 0, 0, 0, 14, 4033, 4083, 3391, 1573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 1527, 4, 0, 4092, 4035, 0, 1, 2, 9, 2, 82, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 4088, 0, 0, 4063, 0, 4094, 2, 0, 0, 4, 3957, 642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16318, 16432, 13586, 16213, 13557, 20123, 17893, 16950] avg=16384.0000 max=20123 max_rank=5 min=13557 min_rank=4 imbalance_ratio=1.228210 max_load_reduction=110949 max_load_reduction_pct=0.846474 moved_to_echo=68523 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4, 0, 2, 0, 0, 4096, 0, 0, 11, 0, 4096, 2507, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 19, 4095, 4096, 0, 4095, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2161, 0, 112, 2254, 0, 4025, 0, 3336, 0, 3057, 0, 69, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 4096, 4064, 0, 0, 0, 0, 3984, 1, 0, 165, 2565, 746, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 2, 24, 0, 4090, 4096, 0, 6, 3761, 0, 0, 0, 0, 0, 151, 0, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 321, 0, 90, 0, 0, 4093, 0, 0, 0, 105, 0, 0, 0, 0, 3988, 4096, 3388, 1589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 1531, 0, 0, 4096, 4026, 0, 0, 0, 0, 0, 1, 3376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4007, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14812, 16469, 15181, 15968, 16969, 17670, 17068, 16935] avg=16384.0000 max=17670 max_rank=5 min=14812 min_rank=0 imbalance_ratio=1.078491 max_load_reduction=113402 max_load_reduction_pct=0.865189 moved_to_echo=68642 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 2329, 18, 4, 2, 199, 367, 0, 0, 4080, 0, 0, 147, 3417, 4087, 2506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4086, 3, 4010, 4095, 0, 4043, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2784, 0, 40, 2165, 0, 4088, 0, 4067, 0, 4096, 1, 4, 1, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 101, 0, 47, 0, 4008, 4010, 15, 0, 96, 0, 4059, 0, 0, 0, 2455, 716, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3061, 0, 3392, 8, 2280, 4096, 24, 1, 1233, 158, 1, 1, 0, 38, 36, 0, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 878, 91, 38, 5, 1, 4088, 1, 56, 3, 112, 2, 1, 0, 2, 3034, 3974, 3387, 1590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1465, 0, 0, 4094, 4071, 0, 0, 0, 0, 1, 46, 283, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7, 0, 6, 4066, 8, 0, 4022, 0, 1142, 0, 0, 0, 0, 4051, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21252, 16328, 17302, 15545, 15377, 17263, 14059, 13946] avg=16384.0000 max=21252 max_rank=0 min=13946 min_rank=7 imbalance_ratio=1.297119 max_load_reduction=109820 max_load_reduction_pct=0.837860 moved_to_echo=60645 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 4090, 0, 0, 0, 129, 335, 0, 0, 0, 4096, 0, 0, 386, 27, 4096, 2459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 4095, 0, 4085, 4083, 97, 4095, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2254, 0, 4096, 0, 4047, 0, 3277, 0, 329, 2539, 0, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2363, 0, 0, 4019, 4046, 0, 0, 829, 0, 2461, 0, 0, 0, 2565, 746, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3798, 43, 3839, 4, 3950, 3662, 2, 158, 0, 1, 0, 0, 4, 89, 0, 0, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 0, 40, 12, 0, 4096, 2, 82, 30, 0, 0, 264, 0, 0, 282, 4096, 3351, 1558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 0, 2, 1531, 15, 0, 4096, 3804, 0, 0, 0, 0, 0, 2, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 1, 0, 0, 0, 3726, 332, 8, 3962, 0, 3744, 0, 0, 0, 0, 4013, 633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15624, 16529, 16998, 17066, 16641, 14355, 17415, 16444] avg=16384.0000 max=17415 max_rank=6 min=14355 min_rank=5 imbalance_ratio=1.062927 max_load_reduction=113657 max_load_reduction_pct=0.867134 moved_to_echo=64855 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 4089, 0, 0, 141, 0, 9, 0, 84, 0, 4090, 67, 30, 6, 1636, 4096, 2502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 3810, 37, 4040, 4018, 36, 4070, 62, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2866, 0, 2724, 2146, 0, 4054, 0, 3948, 0, 3688, 0, 1191, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 247, 1969, 0, 0, 4095, 4096, 0, 0, 615, 0, 4054, 0, 7, 1, 2565, 710, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3736, 0, 3909, 120, 541, 3816, 0, 4, 68, 0, 0, 0, 3, 11, 856, 0, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1943, 2, 342, 0, 0, 3783, 0, 10, 0, 0, 0, 0, 0, 24, 13, 4087, 3368, 1585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3957, 3, 3, 1531, 194, 0, 3900, 3251, 0, 7, 0, 0, 2, 23, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 295, 29, 0, 0, 0, 0, 4087, 0, 0, 4095, 3, 3485, 2, 14, 0, 0, 3951, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16782, 16180, 20720, 18393, 14103, 15157, 13140, 16597] avg=16384.0000 max=20720 max_rank=2 min=13140 min_rank=6 imbalance_ratio=1.264648 max_load_reduction=110352 max_load_reduction_pct=0.841919 moved_to_echo=58997 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 129, 4093, 43, 0, 0, 519, 0, 2, 0, 0, 4018, 0, 3, 0, 0, 4092, 2504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 3, 4050, 4030, 2, 3998, 12, 0, 0, 68, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4095, 1, 1157, 2238, 39, 4077, 0, 3100, 0, 4096, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 4, 0, 4058, 4063, 34, 0, 0, 77, 3888, 21, 0, 32, 2559, 740, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3549, 77, 0, 580, 3913, 4094, 4, 709, 2932, 0, 9, 0, 0, 1, 5, 44, 1083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 805, 0, 2, 27, 0, 4043, 0, 171, 480, 0, 0, 0, 0, 0, 0, 4092, 3384, 1588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 1527, 0, 19, 4078, 3830, 0, 0, 1, 42, 8, 0, 2933, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 4096, 2, 65, 4079, 0, 3983, 3, 3, 0, 0, 3925, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15403, 16239, 18880, 15620, 17000, 14592, 16539, 16799] avg=16384.0000 max=18880 max_rank=2 min=14592 min_rank=5 imbalance_ratio=1.152344 max_load_reduction=112192 max_load_reduction_pct=0.855957 moved_to_echo=64930 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32384, 128, 128, 12288, 256, 128, 32640, 31232, 0, 128, 128, 128, 128, 256, 22400, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[384, 128, 128, 128, 128, 128, 32384, 384, 128, 32512, 128, 28800, 128, 128, 0, 128, 31488, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[30208, 128, 15104, 768, 26752, 32128, 128, 4096, 12416, 256, 128, 128, 128, 256, 1152, 128, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 256, 32768, 3200, 128, 384, 768, 640, 1920, 128, 128, 32768, 128, 128, 1024, 5120, 32768, 19968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 22912, 128, 4096, 17536, 128, 31360, 0, 29696, 0, 20608, 128, 1664, 2816, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 384, 4864, 128, 768, 32640, 32640, 128, 0, 1664, 128, 30848, 128, 128, 1280, 20480, 5888, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[5888, 128, 640, 128, 128, 32384, 128, 1920, 1024, 384, 128, 384, 128, 128, 16512, 32640, 27136, 12672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 32512, 128, 32640, 32640, 256, 32640, 128, 0, 128, 640, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 3932, 0, 0, 0, 0, 0, 121, 4096, 2, 0, 40, 0, 0, 3, 0, 0, 10, 8, 0, 2036, 2977, 4096, 0, 0, 0, 0, 0, 0, 372, 3280, 2733, 0, 0, 1, 6, 4096, 4092, 0, 0, 4095, 598, 0, 0, 0, 0, 0, 3, 0, 0, 9, 0, 4008, 0, 4096, 4096, 0, 57, 4095, 1, 3075, 2580, 0, 7, 0, 3175, 1, 0, 196, 4093, 0, 0, 2, 0, 0, 0, 1, 0, 673, 3775, 4096, 3515, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 1580, 3659, 4096, 0, 4094, 17, 0, 4095, 1, 0, 0, 1, 0, 0, 2, 3975, 0, 4096, 0, 3, 0, 1, 0, 0, 1, 0, 0, 0, 1, 2744, 4092, 0, 4082, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[17, 4018, 4093, 0, 0, 0, 0, 78, 159, 4085, 0, 0, 6, 0, 2, 90, 0, 0, 0, 4035, 0, 3472, 4095, 4092, 0, 6, 5, 0, 21, 27, 0, 482, 578, 13, 0, 0, 0, 4094, 4072, 0, 1, 3808, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1245, 2240, 3, 4090, 4068, 9, 331, 4095, 0, 6, 3032, 5, 357, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 16, 3, 0, 90, 3242, 4096, 4071, 4042, 0, 1, 1086, 0, 0, 4081, 4096, 159, 0, 19, 4041, 1063, 3643, 3402, 0, 4042, 32, 2, 4092, 0, 29, 4, 452, 0, 0, 0, 3997, 3, 4081, 18, 4, 0, 0, 1, 0, 19, 0, 0, 0, 1, 2, 4055, 13, 4051, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 3455, 0, 0, 0, 0, 0, 1793, 4096, 0, 0, 2915, 0, 7, 1, 0, 0, 4096, 52, 3, 4096, 3247, 4096, 2, 0, 0, 51, 0, 0, 2002, 0, 64, 0, 0, 0, 0, 3708, 3143, 0, 0, 2891, 1891, 370, 0, 2, 56, 10, 0, 0, 0, 0, 1243, 3945, 5, 3990, 4042, 0, 0, 4096, 0, 335, 15, 11, 862, 0, 4092, 0, 68, 3954, 4094, 81, 0, 0, 3, 0, 0, 32, 0, 928, 4066, 4095, 2617, 113, 293, 0, 0, 697, 3941, 4088, 1, 0, 0, 3944, 1, 13, 746, 0, 3710, 55, 1, 4096, 0, 0, 0, 1, 18, 36, 7, 4037, 0, 4096, 0, 354, 0, 3711, 0, 0, 0, 0, 0, 1, 0, 205, 4096, 0, 4093, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1, 3899, 4083, 10, 1, 0, 0, 275, 1, 4090, 310, 37, 80, 2, 85, 121, 305, 0, 3545, 104, 0, 1815, 139, 4081, 0, 1, 0, 0, 1, 86, 3971, 4094, 0, 0, 0, 0, 602, 2646, 3793, 0, 0, 4094, 0, 0, 0, 7, 0, 0, 0, 0, 354, 0, 595, 2715, 0, 4066, 4081, 0, 10, 3626, 6, 2944, 3539, 0, 4044, 0, 3061, 5, 318, 0, 4096, 0, 0, 1, 380, 0, 0, 11, 0, 0, 3800, 3962, 0, 3, 3831, 1867, 0, 29, 3953, 4005, 31, 0, 0, 4086, 0, 0, 4084, 5, 4090, 0, 15, 4025, 0, 16, 0, 817, 21, 118, 247, 3418, 0, 4096, 0, 0, 47, 78, 0, 14, 0, 0, 0, 0, 94, 0, 4071, 27, 4091, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[6, 4054, 4005, 0, 0, 1, 0, 1, 4, 4093, 36, 0, 7, 0, 0, 0, 366, 0, 2, 3651, 0, 1776, 3745, 4091, 0, 0, 0, 4, 0, 0, 0, 339, 560, 0, 0, 0, 448, 4095, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 347, 0, 3574, 1532, 0, 4096, 4065, 0, 0, 4060, 0, 1943, 3124, 0, 969, 0, 4094, 0, 0, 0, 4092, 0, 0, 40, 0, 0, 0, 0, 4, 4087, 3966, 4096, 1372, 4, 127, 242, 25, 0, 4092, 2231, 0, 0, 2, 4089, 14, 3379, 4096, 0, 3440, 54, 0, 4096, 0, 4, 0, 388, 2, 0, 54, 4056, 0, 4021, 0, 31, 0, 0, 3, 0, 179, 1, 0, 0, 0, 3396, 4084, 0, 4024, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[153, 4096, 3992, 0, 0, 0, 12, 0, 2656, 3989, 0, 0, 0, 7, 0, 1470, 0, 0, 1, 149, 311, 4085, 132, 3814, 0, 17, 0, 34, 0, 2, 165, 3872, 809, 68, 0, 2, 11, 3890, 4000, 3, 0, 3101, 2253, 0, 413, 1, 349, 8, 0, 0, 0, 0, 0, 3090, 14, 4096, 4081, 0, 0, 3952, 0, 4077, 29, 1146, 3407, 56, 297, 0, 0, 1, 4092, 16, 0, 296, 0, 0, 0, 0, 0, 974, 4093, 4089, 9, 337, 0, 2692, 0, 0, 3828, 4093, 54, 0, 2, 3907, 2115, 3772, 634, 4, 3996, 2, 47, 4031, 2, 0, 0, 214, 0, 174, 520, 2526, 0, 4096, 4, 2, 0, 1055, 10, 4, 1059, 0, 0, 0, 0, 2409, 3907, 67, 3829, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 1711, 0, 2, 0, 0, 0, 95, 4096, 2, 0, 250, 0, 15, 4080, 1, 0, 3848, 3912, 0, 784, 3564, 4096, 0, 0, 2, 10, 1, 20, 3, 15, 2986, 0, 0, 0, 410, 4096, 3575, 0, 0, 4084, 36, 0, 0, 0, 0, 0, 5, 0, 0, 2232, 58, 3700, 0, 4096, 4096, 0, 0, 4096, 0, 4063, 2260, 0, 94, 0, 4094, 0, 0, 0, 4092, 0, 0, 4, 0, 0, 0, 1, 0, 1965, 4056, 4094, 0, 0, 104, 165, 0, 0, 4094, 3988, 0, 0, 0, 4096, 895, 3959, 315, 0, 4096, 1, 0, 4096, 0, 0, 0, 55, 0, 0, 28, 4089, 0, 4092, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 3977, 4094, 21, 22, 0, 35, 132, 30, 4068, 5, 0, 0, 9, 509, 844, 0, 15, 234, 2268, 1, 0, 162, 4082, 14, 2, 5, 0, 71, 4, 53, 0, 4073, 1, 24, 0, 83, 4001, 3494, 0, 1, 4068, 331, 0, 0, 0, 2, 29, 0, 1, 0, 4001, 2, 805, 0, 4088, 4094, 1, 2, 4061, 4, 4020, 574, 29, 0, 0, 4067, 176, 320, 2, 3962, 0, 0, 60, 276, 7, 0, 307, 0, 0, 3970, 4096, 4017, 0, 3685, 108, 5, 8, 4032, 3797, 14, 0, 0, 4082, 3893, 3542, 26, 9, 2732, 0, 2, 4036, 0, 0, 0, 55, 0, 0, 7, 4089, 0, 3940, 89, 3734, 0, 11, 37, 0, 30, 36, 0, 0, 0, 1302, 3994, 0, 4071, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 3932, 0, 0, 0, 0, 0, 121, 4096, 2, 0, 40, 0, 0, 3, 1442, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8, 0, 2036, 2977, 4096, 0, 0, 0, 0, 0, 0, 372, 3280, 1253, 826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2733, 0, 0, 1, 6, 4096, 4092, 0, 0, 4095, 598, 0, 0, 0, 0, 0, 1881, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 9, 0, 4008, 0, 3270, 0, 0, 57, 4095, 1, 3075, 2580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3175, 1, 0, 196, 4093, 0, 0, 2, 0, 0, 0, 1, 0, 673, 2970, 2474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3775, 0, 3515, 0, 0, 0, 0, 0, 773, 4096, 0, 0, 0, 0, 1580, 3659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 17, 0, 2169, 1, 0, 0, 1, 0, 0, 2, 3975, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 1, 0, 0, 0, 1, 2744, 4092, 0, 4082, 0, 4096, 1622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14858, 14858, 18175, 17098, 13592, 17398, 18451, 16642] avg=16384.0000 max=18451 max_rank=6 min=13592 min_rank=4 imbalance_ratio=1.126160 max_load_reduction=112621 max_load_reduction_pct=0.859230 moved_to_echo=66083 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[17, 4018, 4093, 0, 0, 0, 0, 78, 159, 4085, 0, 0, 6, 0, 2, 90, 1441, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 3472, 4095, 4092, 0, 6, 5, 0, 21, 27, 0, 482, 1256, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 13, 0, 0, 0, 4094, 4072, 0, 1, 3808, 0, 0, 0, 0, 0, 1, 1879, 676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1245, 2240, 3, 3262, 0, 9, 331, 4095, 0, 6, 3032, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 16, 3, 0, 90, 3242, 2930, 2460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4042, 0, 1, 1086, 0, 0, 761, 4096, 159, 0, 19, 0, 1063, 3643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3402, 0, 4042, 32, 2, 2160, 0, 29, 4, 452, 0, 0, 0, 3997, 3, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4, 0, 0, 1, 0, 19, 0, 0, 0, 1, 2, 4055, 13, 4051, 0, 4068, 1611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15100, 18319, 15122, 14228, 17290, 18966, 18204, 13843] avg=16384.0000 max=18966 max_rank=5 min=13843 min_rank=7 imbalance_ratio=1.157593 max_load_reduction=112106 max_load_reduction_pct=0.855301 moved_to_echo=68303 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 3455, 0, 0, 0, 0, 0, 1793, 4096, 0, 0, 2915, 0, 7, 1, 1387, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 52, 3, 4096, 3247, 4096, 2, 0, 0, 51, 0, 0, 2002, 0, 1253, 805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 3708, 3143, 0, 0, 2891, 1891, 370, 0, 2, 56, 10, 1810, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1243, 3945, 5, 3185, 0, 0, 0, 4096, 0, 335, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 862, 0, 4092, 0, 68, 3954, 4094, 81, 0, 0, 3, 0, 0, 32, 0, 928, 2860, 2473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 2617, 113, 293, 0, 0, 697, 744, 4088, 1, 0, 0, 0, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 746, 0, 3710, 55, 1, 2170, 0, 0, 0, 1, 18, 36, 7, 4037, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 3711, 0, 0, 0, 0, 0, 1, 0, 205, 4096, 0, 4093, 1, 4042, 1622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18834, 19703, 14618, 12835, 19447, 12633, 14877, 18125] avg=16384.0000 max=19703 max_rank=1 min=12633 min_rank=5 imbalance_ratio=1.202576 max_load_reduction=111369 max_load_reduction_pct=0.849678 moved_to_echo=65082 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 3899, 4083, 10, 1, 0, 0, 275, 1, 4090, 310, 37, 80, 2, 85, 121, 1392, 1123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 0, 3545, 104, 0, 1815, 139, 4081, 0, 1, 0, 0, 1, 86, 3971, 4094, 1231, 820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 2646, 3793, 0, 0, 4094, 0, 0, 0, 7, 0, 0, 1816, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 595, 2715, 0, 3246, 0, 0, 10, 3626, 6, 2944, 3539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 3061, 5, 318, 0, 4096, 0, 0, 1, 380, 0, 0, 11, 0, 0, 2963, 2393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3800, 0, 0, 3, 3831, 1867, 0, 29, 745, 4005, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 5, 4090, 0, 15, 2133, 0, 16, 0, 817, 21, 118, 247, 3418, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 78, 0, 14, 0, 0, 0, 0, 94, 0, 4071, 27, 4091, 0, 4081, 1569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15510, 20193, 13619, 17035, 17272, 14311, 19060, 14072] avg=16384.0000 max=20193 max_rank=1 min=13619 min_rank=2 imbalance_ratio=1.232483 max_load_reduction=110879 max_load_reduction_pct=0.845940 moved_to_echo=64715 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1711, 0, 2, 0, 0, 0, 95, 4096, 2, 0, 250, 0, 15, 4080, 1441, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3848, 3912, 0, 784, 3564, 4096, 0, 0, 2, 10, 1, 20, 3, 15, 1253, 826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2986, 0, 0, 0, 410, 4096, 3575, 0, 0, 4084, 36, 0, 0, 0, 0, 0, 1880, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 2232, 58, 3700, 0, 3270, 0, 0, 0, 4096, 0, 4063, 2260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 4094, 0, 0, 0, 4092, 0, 0, 4, 0, 0, 0, 1, 0, 1965, 2970, 2473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 104, 165, 0, 0, 773, 3988, 0, 0, 0, 0, 895, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 0, 4096, 1, 0, 2170, 0, 0, 0, 55, 0, 0, 28, 4089, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 1621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16914, 18335, 17740, 19684, 15693, 13940, 14846, 13920] avg=16384.0000 max=19684 max_rank=3 min=13920 min_rank=7 imbalance_ratio=1.201416 max_load_reduction=111388 max_load_reduction_pct=0.849823 moved_to_echo=58399 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[6, 4054, 4005, 0, 0, 1, 0, 1, 4, 4093, 36, 0, 7, 0, 0, 0, 1440, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 366, 0, 2, 3651, 0, 1776, 3745, 4091, 0, 0, 0, 4, 0, 0, 0, 339, 1253, 826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 448, 4095, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 1879, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 347, 0, 3574, 1532, 0, 3270, 0, 0, 0, 4060, 0, 1943, 3124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 969, 0, 4094, 0, 0, 0, 4092, 0, 0, 40, 0, 0, 0, 0, 4, 4087, 2965, 2474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 1372, 4, 127, 242, 25, 0, 773, 2231, 0, 0, 2, 0, 14, 3379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3440, 54, 0, 2170, 0, 4, 0, 388, 2, 0, 54, 4056, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 3, 0, 179, 1, 0, 0, 0, 3396, 4084, 0, 4024, 0, 4065, 1622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14771, 16053, 15848, 17850, 18725, 12135, 18285, 17405] avg=16384.0000 max=18725 max_rank=4 min=12135 min_rank=5 imbalance_ratio=1.142883 max_load_reduction=112347 max_load_reduction_pct=0.857140 moved_to_echo=66550 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[153, 4096, 3992, 0, 0, 0, 12, 0, 2656, 3989, 0, 0, 0, 7, 0, 1470, 1348, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 149, 311, 4085, 132, 3814, 0, 17, 0, 34, 0, 2, 165, 3872, 1233, 826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 809, 68, 0, 2, 11, 3890, 4000, 3, 0, 3101, 2253, 0, 413, 1, 349, 8, 1758, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3090, 14, 3270, 0, 0, 0, 3952, 0, 4077, 29, 1146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3407, 56, 297, 0, 0, 1, 4092, 16, 0, 296, 0, 0, 0, 0, 0, 974, 2832, 2471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 9, 337, 0, 2692, 0, 0, 722, 4093, 54, 0, 2, 0, 2115, 3772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 634, 4, 3996, 2, 47, 2136, 2, 0, 0, 214, 0, 174, 520, 2526, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 1055, 10, 4, 1059, 0, 0, 0, 0, 2409, 3907, 67, 3829, 0, 4081, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18798, 14641, 17328, 15578, 14442, 17889, 14351, 18045] avg=16384.0000 max=18798 max_rank=0 min=14351 min_rank=6 imbalance_ratio=1.147339 max_load_reduction=112274 max_load_reduction_pct=0.856583 moved_to_echo=64727 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3977, 4094, 21, 22, 0, 35, 132, 30, 4068, 5, 0, 0, 9, 509, 844, 1419, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 234, 2268, 1, 0, 162, 4082, 14, 2, 5, 0, 71, 4, 53, 0, 1235, 824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 1, 24, 0, 83, 4001, 3494, 0, 1, 4068, 331, 0, 0, 0, 2, 29, 1852, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4001, 2, 805, 0, 3264, 0, 1, 2, 4061, 4, 4020, 574, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 176, 320, 2, 3962, 0, 0, 60, 276, 7, 0, 307, 0, 0, 2960, 2474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 0, 4017, 0, 3685, 108, 5, 8, 761, 3797, 14, 0, 0, 0, 3893, 3542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 9, 2732, 0, 2, 2138, 0, 0, 0, 55, 0, 0, 7, 4089, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 3734, 0, 11, 37, 0, 30, 36, 0, 0, 0, 1302, 3994, 0, 4071, 0, 4094, 1622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16287, 8970, 18622, 16764, 14611, 23800, 12998, 19020] avg=16384.0000 max=23800 max_rank=5 min=8970 min_rank=1 imbalance_ratio=1.452637 max_load_reduction=107272 max_load_reduction_pct=0.818420 moved_to_echo=70429 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[17408, 128, 30208, 256, 128, 17280, 128, 128, 128, 2048, 128, 384, 896, 30208, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9856, 128, 27008, 256, 768, 4224, 32640, 128, 0, 512, 768, 128, 128, 384, 128, 11904, 23552, 19712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[31872, 0, 15616, 512, 8064, 6272, 128, 768, 6144, 30464, 384, 0, 128, 0, 9600, 22016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 32384, 29440, 128, 128, 128, 128, 512, 4864, 32640, 384, 128, 3328, 128, 640, 6656, 11392, 8960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 768, 6272, 6784, 22144, 128, 26112, 0, 128, 512, 32128, 128, 20480, 15232, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[768, 128, 11776, 14208, 384, 18176, 18176, 32512, 128, 128, 128, 128, 128, 256, 6656, 12160, 9984, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 4224, 128, 4864, 128, 128, 1408, 128, 0, 128, 128, 10112, 32512, 128, 32384, 128, 32640, 12928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[11904, 128, 128, 128, 1664, 30720, 30336, 128, 128, 30336, 5120, 384, 512, 128, 512, 128, 14848, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4, 0, 0, 1, 0, 0, 4096, 4094, 4096, 3818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2, 4069, 0, 4096, 0, 0, 0, 0, 0, 4096, 552, 38, 4096, 0, 0, 0, 0, 3852, 0, 0, 4096, 4096, 0, 4091, 0, 4096, 2577, 0, 1794, 0, 1868, 4038, 0, 502, 0, 0, 0, 4088, 4071, 4095, 209, 4096, 0, 0, 1, 0, 0, 38, 0, 4, 0, 4096, 176, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4086, 0, 0, 0, 0, 0, 3, 0, 94, 1, 0, 0, 0, 3996, 3, 1165, 0, 0, 4012, 0, 4096, 1, 4096, 0, 0, 10, 0, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 4069, 0, 0, 7, 4018, 0, 6, 271, 28, 0, 4048, 4079, 3838, 4063, 4096, 495, 0, 9, 0, 14, 0, 0, 65, 9, 0, 3657, 0, 0, 0, 0, 123, 3938, 58, 4096, 0, 3878, 0, 15, 0, 4067, 319, 492, 0, 521, 7, 4, 102, 2189, 0, 0, 3655, 3964, 0, 4080, 0, 4024, 1750, 8, 4090, 0, 3872, 4096, 0, 10, 2, 574, 0, 3899, 907, 4031, 0, 4095, 0, 68, 0, 0, 28, 0, 0, 89, 1, 3822, 15, 0, 4094, 4096, 0, 0, 3815, 1, 0, 65, 0, 130, 22, 0, 4011, 89, 687, 0, 0, 0, 0, 0, 0, 0, 3384, 1, 23, 6, 0, 280, 2, 279, 0, 4, 2128, 76, 4073, 0, 4096, 0, 0, 0, 0, 0, 48] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 4094, 12, 0, 0, 0, 0, 0, 1495, 48, 0, 4094, 4008, 4095, 1, 2824, 4070, 0, 0, 0, 0, 0, 84, 0, 0, 1, 4060, 20, 0, 3, 0, 20, 3849, 0, 4096, 231, 0, 0, 0, 0, 3905, 3767, 3583, 1534, 0, 307, 20, 31, 3329, 0, 0, 4096, 4095, 0, 4096, 0, 4096, 63, 3497, 70, 0, 703, 4096, 0, 19, 0, 0, 31, 3774, 934, 4093, 0, 4096, 0, 1, 0, 0, 0, 14, 0, 0, 0, 3962, 3, 0, 4074, 4096, 0, 0, 3959, 2, 9, 0, 0, 199, 0, 0, 3795, 0, 4096, 0, 0, 0, 0, 0, 741, 0, 6, 203, 0, 2, 285, 4, 405, 1048, 0, 0, 4096, 143, 4096, 428, 4001, 1, 0, 63, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 4001, 73, 0, 0, 369, 1, 0, 0, 1, 0, 4089, 3939, 4093, 3999, 3316, 1424, 14, 0, 0, 0, 3, 802, 0, 2, 0, 3556, 0, 0, 0, 0, 1179, 3633, 0, 3094, 116, 1344, 0, 0, 0, 3264, 1400, 4093, 456, 0, 0, 2, 0, 4002, 0, 57, 4013, 4091, 0, 3837, 0, 4062, 10, 549, 961, 0, 4038, 4090, 0, 4012, 38, 0, 0, 4082, 2523, 4096, 0, 4096, 10, 0, 0, 0, 0, 1, 0, 8, 0, 3974, 0, 69, 4020, 3783, 0, 13, 3721, 119, 0, 0, 0, 4, 1, 0, 4095, 0, 81, 0, 0, 164, 0, 2, 24, 0, 93, 9, 0, 0, 2913, 24, 5, 1, 0, 0, 0, 215, 3976, 330, 4095, 7, 0, 1911, 560, 0, 24] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 4095, 5, 10, 0, 418, 0, 0, 19, 20, 0, 4088, 4089, 4089, 343, 76, 326, 0, 0, 0, 0, 0, 144, 7, 9, 0, 4063, 0, 1, 0, 0, 2, 4094, 0, 4092, 0, 3204, 2, 4, 0, 3953, 4085, 4083, 3190, 16, 0, 0, 0, 412, 3, 0, 4020, 4096, 0, 4046, 0, 4090, 2439, 2, 2238, 2, 4004, 3971, 0, 306, 0, 0, 0, 2704, 3900, 4096, 0, 4040, 0, 2, 6, 0, 0, 0, 7, 0, 1, 4087, 0, 9, 4092, 3747, 0, 11, 4096, 3, 20, 3, 2, 0, 0, 0, 4096, 0, 3246, 0, 1, 0, 0, 20, 0, 0, 0, 0, 0, 0, 1772, 78, 2, 0, 147, 5, 4089, 0, 4091, 1, 4096, 0, 0, 343, 103, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1, 4093, 0, 0, 27, 324, 6, 0, 2, 0, 15, 4069, 4050, 4091, 3988, 1333, 0, 3, 0, 13, 6, 0, 0, 3, 54, 0, 4096, 0, 2, 0, 25, 4, 4094, 13, 4054, 2, 2973, 1, 0, 0, 3991, 3836, 2229, 6, 7, 1, 0, 0, 3624, 0, 0, 4027, 4086, 1, 4095, 0, 4089, 0, 7, 4089, 6, 4088, 1649, 1, 3830, 0, 0, 0, 2459, 353, 4080, 2975, 4082, 0, 0, 1, 2, 0, 0, 211, 0, 72, 3898, 0, 0, 3872, 4073, 0, 4, 4096, 1, 2, 0, 0, 0, 0, 0, 4092, 66, 4096, 1, 6, 48, 0, 0, 3, 19, 0, 1, 0, 0, 854, 1394, 240, 557, 9, 0, 310, 0, 4094, 0, 4091, 0, 2, 1, 2, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 3, 0, 0, 0, 9, 0, 4096, 3967, 4096, 3518, 3780, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4096, 0, 0, 0, 0, 24, 4096, 0, 4034, 0, 969, 0, 0, 0, 2511, 4068, 4096, 0, 0, 0, 0, 0, 3969, 0, 0, 4096, 4096, 0, 4095, 0, 4096, 0, 0, 3495, 0, 4067, 2609, 0, 727, 0, 0, 0, 3995, 3093, 4096, 920, 4089, 0, 0, 31, 0, 0, 2, 0, 0, 1, 4096, 6, 0, 3844, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 3673, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 179, 1310, 0, 1315, 286, 125, 627, 159, 4096, 0, 4096, 0, 0, 27, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[2, 4096, 0, 2, 0, 0, 0, 0, 178, 5, 35, 3977, 4018, 4090, 0, 298, 43, 2, 0, 4, 0, 0, 149, 418, 6, 0, 4074, 114, 1, 11, 0, 3513, 4093, 0, 4067, 1, 72, 0, 0, 0, 4096, 2800, 1531, 0, 423, 0, 2, 2, 4036, 0, 0, 3802, 4092, 9, 4094, 0, 4024, 4017, 3, 2115, 3, 3761, 1498, 4, 6, 58, 3, 0, 4092, 1, 3706, 4096, 4094, 1, 0, 13, 23, 0, 11, 0, 0, 179, 3936, 0, 8, 2725, 2864, 0, 29, 4005, 72, 0, 0, 26, 0, 0, 59, 3987, 7, 4081, 0, 0, 0, 0, 0, 0, 0, 1599, 57, 0, 0, 835, 3351, 5, 3787, 260, 1153, 0, 0, 4085, 0, 4070, 0, 0, 0, 9, 198, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 2954, 4094, 4096, 3818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2, 4096, 4096, 2624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 637, 0, 4096, 0, 0, 0, 0, 0, 4096, 552, 38, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3852, 0, 0, 2493, 0, 0, 0, 0, 0, 2577, 0, 1794, 0, 1868, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 0, 0, 0, 4088, 4071, 4095, 209, 4096, 0, 0, 1, 0, 0, 38, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 176, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4086, 0, 0, 0, 0, 0, 3, 0, 94, 1, 0, 0, 0, 4091, 2180, 1472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 3, 1165, 0, 0, 4012, 0, 4096, 1, 4096, 0, 0, 10, 0, 0, 4, 1603, 1252, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14967, 14913, 13515, 16622, 17405, 17101, 16011, 20538] avg=16384.0000 max=20538 max_rank=7 min=13515 min_rank=2 imbalance_ratio=1.253540 max_load_reduction=110534 max_load_reduction_pct=0.843307 moved_to_echo=71055 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 73, 0, 0, 369, 1, 0, 0, 1, 0, 2949, 3939, 4093, 3999, 3316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1424, 14, 0, 0, 0, 3, 802, 0, 2, 0, 3556, 0, 0, 0, 0, 1179, 4001, 4091, 2603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 569, 0, 3094, 116, 1344, 0, 0, 0, 3264, 1400, 4093, 456, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 57, 2442, 0, 0, 0, 0, 0, 10, 549, 961, 0, 4038, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 38, 0, 0, 4082, 2523, 4096, 0, 4096, 10, 0, 0, 0, 0, 1, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3974, 0, 69, 4020, 3783, 0, 13, 3721, 119, 0, 0, 0, 4, 1, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 81, 0, 0, 164, 0, 2, 24, 0, 93, 9, 0, 0, 2913, 3837, 1946, 1459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 5, 1, 0, 0, 0, 215, 3976, 330, 4095, 7, 0, 1911, 560, 0, 24, 1571, 1118, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18740, 17675, 14338, 16149, 19162, 16248, 14623, 14137] avg=16384.0000 max=19162 max_rank=4 min=14137 min_rank=7 imbalance_ratio=1.169556 max_load_reduction=111910 max_load_reduction_pct=0.853806 moved_to_echo=64170 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 7, 4018, 0, 6, 271, 28, 0, 2920, 4079, 3838, 4063, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 0, 9, 0, 14, 0, 0, 65, 9, 0, 3657, 0, 0, 0, 0, 123, 4069, 3964, 2579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 58, 4096, 0, 3878, 0, 15, 0, 4067, 319, 492, 0, 521, 7, 4, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2189, 0, 0, 2224, 0, 0, 0, 0, 0, 1750, 8, 4090, 0, 3872, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2, 574, 0, 3899, 907, 4031, 0, 4095, 0, 68, 0, 0, 28, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 1, 3822, 15, 0, 4094, 4096, 0, 0, 3815, 1, 0, 65, 0, 130, 22, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 89, 687, 0, 0, 0, 0, 0, 0, 0, 3384, 1, 23, 6, 0, 4080, 2110, 1445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 2, 279, 0, 4, 2128, 76, 4073, 0, 4096, 0, 0, 0, 0, 0, 48, 1431, 1212, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23327, 14984, 14175, 18229, 13915, 16680, 15836, 13926] avg=16384.0000 max=23327 max_rank=0 min=13915 min_rank=4 imbalance_ratio=1.423767 max_load_reduction=107745 max_load_reduction_pct=0.822029 moved_to_echo=60357 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 27, 324, 6, 0, 2, 0, 15, 2923, 4050, 4091, 3988, 1333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 13, 6, 0, 0, 3, 54, 0, 4096, 0, 2, 0, 25, 4, 4093, 4086, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 632, 13, 4054, 2, 2973, 1, 0, 0, 3991, 3836, 2229, 6, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3624, 0, 0, 2446, 0, 1, 0, 0, 0, 0, 7, 4089, 6, 4088, 1649, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3830, 0, 0, 0, 2459, 353, 4080, 2975, 4082, 0, 0, 1, 2, 0, 0, 211, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 3898, 0, 0, 3872, 4073, 0, 4, 4096, 1, 2, 0, 0, 0, 0, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 66, 4096, 1, 6, 48, 0, 0, 3, 19, 0, 1, 0, 0, 854, 4095, 2198, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1394, 240, 557, 9, 0, 310, 0, 4094, 0, 4091, 0, 2, 1, 2, 0, 1, 1581, 1264, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16760, 15006, 17745, 15911, 18300, 16555, 16947, 13848] avg=16384.0000 max=18300 max_rank=4 min=13848 min_rank=7 imbalance_ratio=1.116943 max_load_reduction=112772 max_load_reduction_pct=0.860382 moved_to_echo=65650 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 0, 0, 0, 0, 1495, 48, 0, 2953, 4008, 4095, 1, 2824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 84, 0, 0, 1, 4060, 20, 0, 3, 0, 20, 4094, 4095, 2625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 0, 4096, 231, 0, 0, 0, 0, 3905, 3767, 3583, 1534, 0, 307, 20, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3329, 0, 0, 2493, 0, 0, 0, 0, 0, 63, 3497, 70, 0, 703, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 31, 3774, 934, 4093, 0, 4096, 0, 1, 0, 0, 0, 14, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 3, 0, 4074, 4096, 0, 0, 3959, 2, 9, 0, 0, 199, 0, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3795, 0, 4096, 0, 0, 0, 0, 0, 741, 0, 6, 203, 0, 2, 285, 4096, 2062, 1471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 405, 1048, 0, 0, 4096, 143, 4096, 428, 4001, 1, 0, 63, 0, 0, 0, 1603, 1185, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15436, 19072, 18076, 14251, 13267, 16840, 16757, 17373] avg=16384.0000 max=19072 max_rank=1 min=13267 min_rank=4 imbalance_ratio=1.164062 max_load_reduction=112000 max_load_reduction_pct=0.854492 moved_to_echo=64237 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3, 0, 0, 0, 9, 0, 2954, 3967, 4096, 3518, 3780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4096, 0, 0, 0, 0, 24, 4096, 4096, 2625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 0, 4034, 0, 969, 0, 0, 0, 2511, 4068, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 0, 0, 2493, 0, 0, 0, 0, 0, 0, 0, 3495, 0, 4067, 2609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 727, 0, 0, 0, 3995, 3093, 4096, 920, 4089, 0, 0, 31, 0, 0, 2, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 6, 0, 3844, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 3673, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 179, 4095, 2194, 1471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1310, 0, 1315, 286, 125, 627, 159, 4096, 0, 4096, 0, 0, 27, 0, 0, 0, 1603, 1261, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18327, 14956, 16319, 16633, 17258, 16676, 15698, 15205] avg=16384.0000 max=18327 max_rank=0 min=14956 min_rank=1 imbalance_ratio=1.118591 max_load_reduction=112745 max_load_reduction_pct=0.860176 moved_to_echo=64837 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 10, 0, 418, 0, 0, 19, 20, 0, 2949, 4089, 4089, 343, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 0, 0, 0, 0, 0, 144, 7, 9, 0, 4063, 0, 1, 0, 0, 2, 4095, 4096, 2621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 0, 4092, 0, 3204, 2, 4, 0, 3953, 4085, 4083, 3190, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 412, 3, 0, 2446, 0, 0, 0, 0, 0, 2439, 2, 2238, 2, 4004, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 0, 0, 0, 2704, 3900, 4096, 0, 4040, 0, 2, 6, 0, 0, 0, 7, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 9, 4092, 3747, 0, 11, 4096, 3, 20, 3, 2, 0, 0, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3246, 0, 1, 0, 0, 20, 0, 0, 0, 0, 0, 0, 1772, 4046, 2193, 1469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 2, 0, 147, 5, 4089, 0, 4091, 1, 4096, 0, 0, 343, 103, 0, 0, 1574, 1260, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12018, 15364, 23270, 15517, 15365, 16607, 16843, 16088] avg=16384.0000 max=23270 max_rank=2 min=12018 min_rank=0 imbalance_ratio=1.420288 max_load_reduction=107802 max_load_reduction_pct=0.822464 moved_to_echo=64903 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 2, 0, 0, 0, 0, 178, 5, 35, 2869, 4018, 4090, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 2, 0, 4, 0, 0, 149, 418, 6, 0, 4074, 114, 1, 11, 0, 3513, 4096, 4092, 2579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 640, 0, 4067, 1, 72, 0, 0, 0, 4096, 2800, 1531, 0, 423, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 2314, 0, 9, 0, 0, 0, 4017, 3, 2115, 3, 3761, 1498, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 58, 3, 0, 4092, 1, 3706, 4096, 4094, 1, 0, 13, 23, 0, 11, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 3936, 0, 8, 2725, 2864, 0, 29, 4005, 72, 0, 0, 26, 0, 0, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 3987, 7, 4081, 0, 0, 0, 0, 0, 0, 0, 1599, 57, 0, 0, 835, 4094, 2193, 1445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3351, 5, 3787, 260, 1153, 0, 0, 4085, 0, 4070, 0, 0, 0, 9, 198, 0, 1488, 1260, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11497, 19102, 13634, 17760, 16400, 14365, 18357, 19957] avg=16384.0000 max=19957 max_rank=7 min=11497 min_rank=0 imbalance_ratio=1.218079 max_load_reduction=111115 max_load_reduction_pct=0.847740 moved_to_echo=69079 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 5248, 128, 128, 2048, 128, 128, 23552, 32256, 32512, 19840, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32256, 256, 24064, 128, 128, 256, 0, 128, 896, 128, 5248, 384, 128, 128, 6912, 32512, 17152, 11776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[25472, 128, 128, 19456, 0, 128, 0, 0, 0, 10880, 4096, 18944, 128, 26496, 26112, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 256, 31872, 256, 128, 30848, 30976, 0, 128, 32000, 256, 128, 128, 128, 384, 128, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[9472, 128, 640, 128, 29184, 15872, 32384, 8320, 32768, 128, 128, 128, 128, 128, 128, 256, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[6400, 128, 128, 128, 128, 128, 1280, 512, 128, 128, 31744, 256, 128, 128, 128, 4992, 32640, 32640, 20992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[10496, 768, 8192, 768, 1408, 15360, 640, 32640, 768, 32768, 128, 128, 2432, 768, 256, 128, 12544, 9856, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[4992, 128, 31744, 384, 12544, 128, 128, 0, 29952, 20864, 20224, 9344, 1024, 384, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 7, 0, 0, 4096, 4040, 3339, 0, 310, 5, 0, 0, 3903, 8, 0, 1, 1865, 4080, 262, 0, 3812, 0, 0, 0, 8, 0, 0, 0, 3460, 0, 166, 0, 3, 0, 0, 0, 4068, 0, 4077, 1, 0, 4068, 0, 0, 3858, 31, 3740, 0, 64, 1, 4090, 0, 0, 4045, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 35, 0, 4096, 362, 1, 0, 0, 4028, 0, 0, 0, 25, 4094, 0, 609, 51, 866, 0, 14, 4096, 0, 1326, 4053, 0, 4094, 4059, 4035, 0, 0, 0, 0, 0, 697, 2, 4075, 2, 4096, 0, 4096, 0, 0, 27, 0, 176, 1, 0, 4090, 586, 0, 3749, 0, 3839, 4095, 0, 4095, 4092, 52, 31, 7, 2, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3389, 4096, 4096, 4058, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 4096, 4, 0, 3998, 0, 0, 2, 0, 0, 3275, 11, 4095, 0, 1, 0, 22, 0, 0, 0, 787, 0, 4095, 0, 0, 4096, 0, 0, 3450, 0, 4020, 0, 0, 0, 14, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 4093, 0, 2264, 0, 1, 4096, 0, 2537, 4096, 0, 4096, 2882, 302, 0, 0, 0, 0, 0, 1500, 0, 4096, 3, 4096, 0, 4060, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 262, 98, 0, 4083, 4096, 0, 4096, 4096, 0, 2, 15, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 8, 27, 0, 57, 0, 4019, 3693, 4095, 0, 3, 0, 1, 1, 4019, 37, 0, 0, 7, 4096, 46, 0, 2565, 10, 0, 0, 0, 63, 2863, 0, 4087, 0, 868, 6, 0, 113, 0, 0, 4041, 0, 4096, 0, 1, 4092, 2, 0, 3130, 0, 4096, 0, 92, 0, 2920, 1, 0, 4072, 3, 0, 0, 4, 98, 1, 6, 2, 85, 1, 225, 9, 4095, 0, 17, 0, 1, 4095, 0, 0, 0, 0, 4094, 0, 214, 11, 4094, 0, 0, 4095, 0, 3590, 4036, 23, 4026, 3839, 2, 0, 0, 0, 0, 0, 87, 0, 4051, 0, 4071, 0, 4086, 0, 0, 27, 0, 4047, 6, 1, 4095, 77, 179, 0, 0, 4079, 4094, 13, 4085, 4091, 0, 66, 0, 0, 24] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 28, 0, 0, 631, 4096, 3798, 4096, 0, 412, 0, 8, 0, 3953, 0, 0, 2, 2996, 3272, 0, 0, 4090, 0, 0, 0, 185, 0, 0, 103, 4096, 0, 1204, 0, 0, 0, 0, 0, 2425, 0, 3556, 0, 0, 4030, 57, 1, 3987, 355, 21, 0, 10, 0, 2608, 0, 0, 3734, 0, 0, 0, 0, 5, 0, 2, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 3590, 453, 0, 0, 3741, 4082, 1, 796, 5, 3423, 10, 0, 4070, 0, 338, 4096, 0, 4094, 4096, 3402, 0, 18, 0, 0, 0, 115, 260, 4096, 0, 4096, 0, 3790, 0, 0, 126, 0, 3780, 14, 0, 3437, 0, 10, 0, 0, 4087, 4083, 0, 4066, 4056, 45, 14, 919, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 1, 0, 4096, 3914, 3507, 0, 0, 497, 470, 156, 4028, 0, 0, 0, 4096, 4002, 0, 0, 4092, 0, 0, 70, 4, 0, 3972, 1, 3576, 0, 13, 0, 107, 0, 0, 0, 1621, 2, 3640, 9, 0, 4091, 0, 5, 79, 51, 618, 1203, 0, 0, 312, 23, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 901, 0, 4096, 74, 6, 0, 1, 3319, 0, 0, 0, 21, 4095, 16, 3880, 0, 3, 0, 0, 4096, 0, 35, 4096, 2, 3435, 4062, 4007, 0, 196, 0, 0, 0, 13, 0, 4096, 458, 4096, 0, 3704, 1, 0, 0, 0, 2767, 0, 0, 4095, 704, 571, 4067, 0, 4096, 1513, 0, 4078, 4094, 13, 55, 3956, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 5, 2, 0, 7, 4, 4096, 3886, 3495, 0, 1939, 0, 0, 0, 18, 1, 0, 0, 1105, 3970, 0, 0, 4033, 0, 0, 2304, 6, 0, 2007, 335, 4096, 0, 329, 0, 0, 13, 0, 0, 4075, 0, 337, 9, 0, 4096, 0, 0, 3832, 0, 4096, 0, 1, 0, 3891, 0, 0, 4096, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 3987, 0, 2255, 0, 0, 4096, 61, 14, 4091, 0, 4096, 4094, 3311, 0, 0, 0, 0, 0, 115, 0, 4046, 0, 4095, 0, 4096, 0, 0, 0, 0, 3684, 0, 11, 4039, 151, 0, 53, 0, 4094, 4094, 0, 4096, 4096, 0, 0, 10, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 22, 299, 4056, 4091, 3534, 4, 248, 0, 0, 0, 4093, 33, 21, 12, 4096, 0, 0, 5, 3760, 7, 0, 0, 3658, 14, 0, 66, 4073, 0, 0, 0, 192, 6, 0, 0, 4086, 1, 3311, 0, 46, 4058, 0, 0, 30, 130, 2462, 150, 0, 43, 3988, 44, 0, 4096, 7, 0, 0, 2, 3, 0, 0, 31, 0, 18, 0, 0, 4051, 0, 0, 0, 4, 4078, 0, 172, 0, 0, 4010, 111, 774, 1249, 276, 161, 0, 4086, 750, 2890, 4003, 117, 3925, 3768, 16, 0, 0, 1, 0, 6, 1, 0, 4055, 25, 4096, 3, 3664, 1, 0, 4, 0, 2339, 0, 0, 4096, 1965, 4030, 954, 0, 4040, 3855, 0, 3813, 4054, 0, 671, 161, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[13, 10, 0, 0, 0, 68, 0, 3887, 4086, 4056, 0, 0, 0, 303, 34, 3161, 16, 54, 1, 84, 4091, 7, 0, 0, 1, 8, 6, 0, 0, 214, 305, 4070, 184, 3997, 0, 21, 192, 2967, 38, 3956, 47, 3850, 0, 0, 4032, 0, 7, 56, 21, 4095, 0, 21, 0, 3951, 33, 16, 4063, 22, 0, 13, 13, 0, 2, 6, 0, 0, 0, 0, 0, 4072, 0, 1, 325, 12, 4096, 0, 54, 3, 0, 4096, 0, 4096, 0, 4047, 294, 0, 4068, 427, 861, 3923, 0, 4025, 3868, 0, 36, 0, 0, 0, 7, 3610, 2, 3996, 0, 4096, 105, 4013, 0, 0, 0, 0, 119, 0, 120, 3857, 1000, 33, 1, 18, 4013, 3415, 0, 4090, 4080, 0, 39, 0, 76, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3389, 4096, 4096, 4058, 0, 0, 0, 0, 0, 4019, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 3998, 0, 0, 2, 0, 0, 3275, 11, 4095, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 22, 0, 0, 0, 787, 0, 4095, 0, 0, 4096, 0, 0, 3450, 2017, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 0, 14, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 2337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1759, 1735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 2264, 0, 1, 0, 0, 2537, 344, 0, 4096, 2882, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1500, 0, 4096, 3, 4096, 0, 4060, 0, 0, 0, 0, 4096, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 262, 98, 0, 4083, 4096, 0, 1628, 0, 0, 2, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19743, 16909, 14655, 14564, 15781, 16520, 18619, 14281] avg=16384.0000 max=19743 max_rank=0 min=14281 min_rank=7 imbalance_ratio=1.205017 max_load_reduction=111329 max_load_reduction_pct=0.849373 moved_to_echo=65201 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 8, 27, 0, 57, 0, 4019, 3693, 4095, 0, 3, 0, 1, 1, 4019, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 7, 4096, 46, 0, 2565, 10, 0, 0, 0, 63, 2863, 0, 4087, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 868, 6, 0, 113, 0, 0, 4041, 0, 4096, 0, 1, 4092, 2, 0, 3130, 1991, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 92, 0, 2920, 1, 0, 4072, 3, 0, 0, 4, 98, 1, 6, 4095, 2335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 85, 1, 225, 9, 4095, 0, 17, 0, 1, 4095, 0, 0, 0, 0, 4094, 1756, 1712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 11, 4094, 0, 0, 0, 0, 3590, 333, 23, 4026, 3839, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 4051, 0, 4071, 0, 4086, 0, 0, 27, 0, 4047, 6, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 77, 179, 0, 0, 4079, 4094, 13, 1609, 0, 0, 66, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16012, 15202, 18529, 17723, 16092, 16132, 17145, 14237] avg=16384.0000 max=18529 max_rank=2 min=14237 min_rank=7 imbalance_ratio=1.130920 max_load_reduction=112543 max_load_reduction_pct=0.858635 moved_to_echo=63606 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 7, 0, 0, 4096, 4040, 3339, 0, 310, 5, 0, 0, 3903, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 1865, 4080, 262, 0, 3812, 0, 0, 0, 8, 0, 0, 0, 3460, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 3, 0, 0, 0, 4068, 0, 4077, 1, 0, 4068, 0, 0, 3858, 1996, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3740, 0, 64, 1, 4090, 0, 0, 4045, 0, 0, 1, 0, 0, 0, 0, 4096, 2334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 35, 0, 4096, 362, 1, 0, 0, 4028, 0, 0, 0, 25, 4094, 1758, 1717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 609, 51, 866, 0, 14, 0, 0, 1326, 340, 0, 4094, 4059, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 697, 2, 4075, 2, 4096, 0, 4096, 0, 0, 27, 0, 176, 1, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 586, 0, 3749, 0, 3839, 4095, 0, 1628, 0, 52, 31, 7, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15788, 14923, 18424, 18402, 16121, 15394, 13940, 18080] avg=16384.0000 max=18424 max_rank=2 min=13940 min_rank=6 imbalance_ratio=1.124512 max_load_reduction=112648 max_load_reduction_pct=0.859436 moved_to_echo=63535 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 1, 0, 4096, 3914, 3507, 0, 0, 497, 470, 156, 4028, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4002, 0, 0, 4092, 0, 0, 70, 4, 0, 3972, 1, 3576, 1422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 107, 0, 0, 0, 1621, 2, 3640, 9, 0, 4091, 0, 5, 79, 2017, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 618, 1203, 0, 0, 312, 23, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 901, 0, 4096, 74, 6, 0, 1, 3319, 0, 0, 0, 21, 4095, 1758, 1735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 3880, 0, 3, 0, 0, 0, 0, 35, 344, 2, 3435, 4062, 4007, 0, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4096, 458, 4096, 0, 3704, 1, 0, 0, 0, 2767, 0, 765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 704, 571, 4067, 0, 4096, 1513, 0, 1620, 0, 13, 55, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16756, 21235, 11770, 12735, 16006, 15980, 15900, 20690] avg=16384.0000 max=21235 max_rank=1 min=11770 min_rank=2 imbalance_ratio=1.296082 max_load_reduction=109837 max_load_reduction_pct=0.837990 moved_to_echo=68576 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 28, 0, 0, 631, 4096, 3798, 4096, 0, 412, 0, 8, 0, 3953, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2996, 3272, 0, 0, 4090, 0, 0, 0, 185, 0, 0, 103, 4096, 1417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1204, 0, 0, 0, 0, 0, 2425, 0, 3556, 0, 0, 4030, 57, 1, 3987, 2017, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 21, 0, 10, 0, 2608, 0, 0, 3734, 0, 0, 0, 0, 5, 0, 2, 4070, 2314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 3590, 453, 0, 0, 3741, 4082, 1742, 1735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 796, 5, 3423, 10, 0, 0, 0, 338, 344, 0, 4094, 4096, 3402, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 260, 4096, 0, 4096, 0, 3790, 0, 0, 126, 0, 3780, 14, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3437, 0, 10, 0, 0, 4087, 4083, 0, 1617, 0, 45, 14, 919, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17107, 16161, 17462, 13119, 19443, 16527, 17039, 14214] avg=16384.0000 max=19443 max_rank=4 min=13119 min_rank=3 imbalance_ratio=1.186707 max_load_reduction=111629 max_load_reduction_pct=0.851662 moved_to_echo=67223 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 2, 0, 7, 4, 4096, 3886, 3495, 0, 1939, 0, 0, 0, 18, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1105, 3970, 0, 0, 4033, 0, 0, 2304, 6, 0, 2007, 335, 4096, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 0, 13, 0, 0, 4075, 0, 337, 9, 0, 4096, 0, 0, 3832, 2014, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 3891, 0, 0, 4096, 0, 0, 0, 0, 34, 0, 0, 4096, 2338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1758, 1733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 2255, 0, 0, 0, 61, 14, 344, 0, 4096, 4094, 3311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 4046, 0, 4095, 0, 4096, 0, 0, 0, 0, 3684, 0, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4039, 151, 0, 53, 0, 4094, 4094, 0, 1628, 0, 0, 0, 10, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13537, 19285, 14892, 18552, 15757, 18162, 16804, 14083] avg=16384.0000 max=19285 max_rank=1 min=13537 min_rank=0 imbalance_ratio=1.177063 max_load_reduction=111787 max_load_reduction_pct=0.852867 moved_to_echo=64806 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 22, 299, 4056, 4091, 3534, 4, 248, 0, 0, 0, 4093, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 21, 12, 4096, 0, 0, 5, 3760, 7, 0, 0, 3658, 14, 0, 66, 4073, 1329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 6, 0, 0, 4086, 1, 3311, 0, 46, 4058, 0, 0, 30, 1971, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 2462, 150, 0, 43, 3988, 44, 0, 4096, 7, 0, 0, 2, 3, 0, 0, 4086, 2314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 18, 0, 0, 4051, 0, 0, 0, 4, 4078, 0, 172, 0, 0, 4010, 1740, 1695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 774, 1249, 276, 161, 0, 0, 750, 2890, 337, 117, 3925, 3768, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 1, 0, 4055, 25, 4096, 3, 3664, 1, 0, 4, 0, 2339, 0, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1965, 4030, 954, 0, 4040, 3855, 0, 1516, 0, 0, 671, 161, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16426, 17074, 13875, 17325, 15799, 14374, 14910, 21289] avg=16384.0000 max=21289 max_rank=7 min=13875 min_rank=2 imbalance_ratio=1.299377 max_load_reduction=109783 max_load_reduction_pct=0.837578 moved_to_echo=66372 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[13, 10, 0, 0, 0, 68, 0, 3887, 4086, 4056, 0, 0, 0, 303, 34, 3161, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 54, 1, 84, 4091, 7, 0, 0, 1, 8, 6, 0, 0, 214, 305, 4070, 1426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 3997, 0, 21, 192, 2967, 38, 3956, 47, 3850, 0, 0, 4032, 0, 7, 56, 1932, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4095, 0, 21, 0, 3951, 33, 16, 4063, 22, 0, 13, 13, 0, 2, 6, 4068, 2328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 1, 325, 12, 4096, 0, 54, 3, 0, 4096, 1752, 1662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4047, 294, 0, 0, 427, 861, 329, 0, 4025, 3868, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3610, 2, 3996, 0, 4096, 105, 4013, 0, 0, 0, 0, 119, 0, 767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 3857, 1000, 33, 1, 18, 4013, 3415, 0, 1626, 0, 0, 39, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15703, 10283, 21465, 18652, 16073, 17983, 16715, 14198] avg=16384.0000 max=21465 max_rank=2 min=10283 min_rank=1 imbalance_ratio=1.310120 max_load_reduction=109607 max_load_reduction_pct=0.836235 moved_to_echo=64969 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[640, 23168, 1408, 256, 128, 21888, 128, 128, 32384, 128, 128, 128, 128, 256, 128, 128, 32768, 18688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 6656, 128, 384, 384, 3072, 128, 25088, 128, 27008, 128, 128, 32640, 128, 128, 18432, 16000, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 14336, 27648, 384, 128, 26368, 128, 128, 2432, 3968, 128, 12416, 896, 31616, 11392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 256, 4352, 32512, 31616, 30208, 128, 2944, 512, 896, 256, 27264, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 6144, 384, 32512, 512, 32768, 128, 31616, 128, 0, 256, 0, 21120, 128, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 31872, 4608, 5120, 8960, 128, 32384, 29312, 128, 12928, 0, 128, 896, 5120, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 18560, 1408, 17280, 512, 128, 0, 1280, 11648, 2816, 256, 31872, 30720, 15104, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 1280, 128, 32768, 512, 128, 384, 128, 31488, 512, 256, 128, 3840, 32768, 14080, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4093, 0, 0, 4096, 103, 0, 0, 4003, 27, 4096, 18, 0, 0, 0, 231, 0, 3874, 0, 0, 0, 4071, 0, 0, 0, 0, 4096, 4096, 0, 3962, 6, 7, 23, 0, 0, 0, 30, 0, 0, 0, 102, 0, 2933, 3983, 19, 451, 0, 0, 0, 0, 63, 0, 4096, 1352, 0, 3, 0, 225, 0, 0, 0, 182, 0, 7, 3635, 4096, 0, 0, 3699, 0, 0, 0, 0, 4096, 87, 449, 0, 3871, 0, 0, 4092, 0, 0, 3156, 4093, 46, 4075, 0, 1, 4087, 0, 0, 0, 126, 0, 0, 0, 0, 4095, 319, 4096, 0, 4007, 3811, 610, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 14, 0, 12, 0, 4087, 3922, 4095, 4, 3, 1768, 0, 2180] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4072, 0, 0, 4096, 4096, 172, 0, 4096, 1, 4096, 411, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3871, 0, 1824, 0, 0, 4096, 4096, 0, 312, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4058, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 10, 8, 0, 0, 0, 19, 0, 0, 0, 192, 3943, 0, 4096, 4096, 0, 0, 4036, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 18, 0, 4086, 0, 0, 2, 4096, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 1419, 0, 0, 0, 4096, 4095, 4096, 0, 4096, 208, 0, 0, 0, 0, 0, 0, 0, 2919, 0, 0, 0, 4096, 0, 0, 0, 71, 0, 2838, 2837, 1, 0, 0, 3630, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[3703, 78, 0, 4096, 3821, 5, 0, 4085, 0, 3817, 80, 0, 0, 40, 0, 0, 2399, 0, 0, 0, 3444, 0, 8, 0, 0, 4090, 3976, 27, 0, 258, 74, 0, 0, 282, 10, 578, 397, 0, 0, 4049, 0, 4085, 4020, 0, 327, 2, 0, 0, 0, 0, 5, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 4090, 4095, 1, 0, 1825, 143, 0, 0, 0, 3957, 93, 31, 2, 4088, 12, 667, 4022, 47, 0, 185, 4094, 2691, 4096, 0, 69, 4050, 50, 0, 0, 868, 3882, 0, 0, 0, 4056, 3446, 3791, 0, 4096, 40, 1197, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 4096, 0, 32, 1, 5, 0, 0, 3705, 0, 13, 0, 3996, 8, 3672] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4096, 6, 4, 4096, 4024, 644, 8, 4096, 23, 4096, 40, 0, 0, 13, 330, 0, 4069, 0, 4, 2, 4060, 0, 3285, 0, 0, 4096, 4093, 0, 63, 0, 0, 0, 0, 70, 0, 7, 0, 0, 0, 1933, 0, 4019, 4096, 0, 4, 1, 0, 0, 0, 11, 7, 194, 0, 0, 0, 0, 14, 0, 0, 41, 0, 3018, 0, 3629, 4092, 0, 1, 120, 5, 38, 0, 0, 4073, 109, 22, 0, 4072, 2141, 0, 4092, 0, 73, 3288, 4083, 0, 4083, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 2, 0, 3977, 4096, 3901, 0, 4094, 195, 67, 0, 0, 135, 0, 0, 9, 2985, 0, 0, 5, 4096, 0, 0, 1, 0, 0, 2684, 4060, 0, 0, 1, 4094, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[3083, 0, 0, 3633, 3425, 0, 0, 4019, 49, 4033, 1, 0, 0, 211, 0, 1, 4050, 1, 0, 7, 4068, 0, 3638, 22, 0, 4085, 4080, 80, 0, 192, 1, 0, 1, 0, 555, 4040, 0, 0, 0, 2745, 30, 3899, 2057, 809, 54, 12, 0, 0, 0, 0, 62, 0, 198, 0, 0, 0, 0, 0, 15, 24, 0, 4024, 14, 587, 4051, 25, 90, 335, 0, 0, 0, 2, 4021, 391, 62, 0, 2362, 0, 2796, 3997, 76, 0, 0, 4039, 148, 3724, 8, 0, 3990, 105, 0, 0, 0, 0, 1, 19, 0, 3790, 4071, 4096, 0, 3944, 712, 1115, 1, 73, 1108, 0, 0, 0, 4061, 1, 0, 33, 4069, 52, 0, 0, 24, 36, 4055, 4095, 0, 0, 4, 3788, 18, 3879] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4018, 0, 2, 4094, 4084, 3655, 0, 4095, 0, 4077, 245, 0, 0, 308, 0, 1, 4096, 2, 35, 0, 3921, 1, 3986, 0, 6, 4076, 4079, 12, 84, 19, 146, 32, 0, 15, 1, 3695, 25, 1, 1, 636, 0, 3920, 4092, 0, 0, 0, 719, 23, 0, 32, 0, 139, 0, 2, 11, 0, 392, 3, 0, 4, 1657, 4082, 0, 2921, 4063, 15, 0, 160, 0, 99, 110, 19, 4064, 0, 2, 0, 2952, 0, 0, 4095, 0, 0, 33, 4042, 120, 4092, 0, 0, 4093, 13, 0, 2, 0, 536, 16, 0, 1, 4096, 4092, 4054, 0, 4092, 0, 9, 3, 0, 0, 1, 0, 2, 3843, 0, 0, 0, 4092, 1290, 0, 5, 477, 0, 658, 231, 0, 0, 0, 4082, 0, 4073] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4084, 2, 0, 4096, 4096, 2463, 20, 4096, 2, 4096, 19, 0, 0, 0, 47, 0, 4077, 0, 0, 0, 4095, 0, 90, 2, 30, 4080, 4096, 0, 51, 0, 0, 0, 0, 0, 0, 1090, 0, 0, 0, 396, 0, 4032, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 423, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2151, 0, 4006, 4096, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3348, 413, 4096, 0, 0, 88, 4096, 1110, 4096, 0, 5, 4096, 0, 0, 5, 0, 318, 0, 0, 0, 4096, 4095, 4095, 0, 4096, 4075, 81, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 4096, 123, 5, 0, 0, 0, 3023, 1818, 0, 0, 49, 3874, 0, 3746] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[3856, 0, 0, 4055, 3903, 1657, 3, 4050, 159, 4096, 0, 2, 0, 2, 8, 0, 4093, 332, 0, 3, 4095, 11, 35, 244, 61, 3871, 4096, 6, 3, 0, 1, 63, 12, 122, 0, 0, 72, 0, 0, 4087, 4, 4055, 4029, 10, 585, 91, 150, 0, 0, 158, 39, 3947, 23, 0, 0, 22, 0, 0, 0, 0, 0, 990, 0, 4022, 3997, 1, 159, 1154, 0, 78, 12, 1, 4055, 0, 0, 0, 4081, 2555, 240, 4058, 66, 0, 0, 3859, 1417, 4074, 49, 68, 4001, 0, 2, 1, 740, 0, 0, 0, 0, 3940, 3944, 3662, 0, 3956, 938, 0, 0, 12, 0, 19, 0, 27, 2952, 0, 0, 0, 3989, 171, 0, 0, 55, 1, 1567, 3312, 0, 0, 0, 2888, 8, 3840] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4072, 0, 0, 4096, 4096, 172, 0, 0, 1, 3699, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3871, 0, 1824, 0, 0, 4096, 2153, 0, 312, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4058, 4096, 0, 1, 0, 0, 0, 2311, 1943, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8, 0, 0, 0, 19, 0, 0, 0, 192, 3943, 0, 4096, 4096, 4096, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 0, 0, 4036, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 18, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 1419, 0, 0, 672, 570, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3526, 4095, 4096, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 2919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2998, 0, 0, 0, 71, 0, 2838, 2837, 1, 0, 0, 3630, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16547, 16368, 16929, 16876, 17700, 15342, 14844, 16466] avg=16384.0000 max=17700 max_rank=4 min=14844 min_rank=6 imbalance_ratio=1.080322 max_load_reduction=113372 max_load_reduction_pct=0.864960 moved_to_echo=64352 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 4096, 103, 0, 0, 0, 27, 3699, 18, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3874, 0, 0, 0, 4071, 0, 0, 0, 0, 4096, 2153, 0, 3962, 6, 7, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 102, 0, 2933, 3983, 19, 451, 0, 0, 0, 2311, 1943, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 4096, 1352, 0, 3, 0, 225, 0, 0, 0, 182, 0, 7, 3635, 4003, 4007, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 0, 0, 3699, 0, 0, 0, 0, 4096, 87, 449, 0, 3871, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3156, 4093, 46, 4075, 0, 1, 4087, 0, 0, 0, 126, 0, 0, 0, 672, 570, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3525, 319, 4096, 0, 0, 3811, 610, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2998, 0, 14, 0, 12, 0, 4087, 3922, 4095, 4, 3, 1768, 0, 2180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12267, 18192, 12198, 17989, 17663, 17223, 16457, 19083] avg=16384.0000 max=19083 max_rank=7 min=12198 min_rank=2 imbalance_ratio=1.164734 max_load_reduction=111989 max_load_reduction_pct=0.854408 moved_to_echo=70426 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3703, 78, 0, 4096, 3821, 5, 0, 0, 0, 3447, 80, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2399, 0, 0, 0, 3444, 0, 8, 0, 0, 4090, 2090, 27, 0, 258, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 10, 578, 397, 0, 0, 4049, 0, 4085, 4020, 0, 327, 2, 0, 0, 2310, 1886, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 4090, 4085, 4096, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 1, 0, 1825, 143, 0, 0, 0, 3957, 93, 31, 2, 4088, 12, 667, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 185, 4094, 2691, 4096, 0, 69, 4050, 50, 0, 0, 868, 3882, 0, 0, 672, 565, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3491, 3446, 3791, 0, 0, 40, 1197, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2998, 0, 32, 1, 5, 0, 0, 3705, 0, 13, 0, 3996, 8, 3672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15270, 12390, 18372, 16739, 16210, 21639, 16022, 14430] avg=16384.0000 max=21639 max_rank=5 min=12390 min_rank=1 imbalance_ratio=1.320740 max_load_reduction=109433 max_load_reduction_pct=0.834908 moved_to_echo=68301 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 6, 4, 4096, 4024, 644, 8, 0, 23, 3699, 40, 0, 0, 13, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 4, 2, 4060, 0, 3285, 0, 0, 4096, 2151, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 7, 0, 0, 0, 1933, 0, 4019, 4096, 0, 4, 1, 0, 0, 2308, 1942, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 7, 194, 0, 0, 0, 0, 14, 0, 0, 41, 0, 3018, 0, 3629, 4096, 4094, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 0, 1, 120, 5, 38, 0, 0, 4073, 109, 22, 0, 4072, 2141, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 3288, 4083, 0, 4083, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 2, 672, 554, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3423, 4096, 3901, 0, 0, 195, 67, 0, 0, 135, 0, 0, 9, 2985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2998, 0, 0, 1, 0, 0, 2684, 4060, 0, 0, 1, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16983, 17730, 14806, 15519, 16042, 17244, 14811, 17937] avg=16384.0000 max=17937 max_rank=7 min=14806 min_rank=2 imbalance_ratio=1.094788 max_load_reduction=113135 max_load_reduction_pct=0.863152 moved_to_echo=66034 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4018, 0, 2, 4094, 4084, 3655, 0, 0, 0, 3681, 245, 0, 0, 308, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 35, 0, 3921, 1, 3986, 0, 6, 4076, 2139, 12, 84, 19, 146, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 3695, 25, 1, 1, 636, 0, 3920, 4092, 0, 0, 0, 719, 23, 2296, 1940, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 139, 0, 2, 11, 0, 392, 3, 0, 4, 1657, 4082, 0, 2921, 4095, 4092, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1352, 15, 0, 160, 0, 99, 110, 19, 4064, 0, 2, 0, 2952, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4042, 120, 4092, 0, 0, 4093, 13, 0, 2, 0, 536, 16, 0, 678, 575, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3521, 4092, 4054, 0, 0, 0, 9, 3, 0, 0, 1, 0, 2, 3843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2982, 1290, 0, 5, 477, 0, 658, 231, 0, 0, 0, 4082, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20088, 18555, 17796, 17845, 12868, 14596, 15526, 13798] avg=16384.0000 max=20088 max_rank=0 min=12868 min_rank=4 imbalance_ratio=1.226074 max_load_reduction=110984 max_load_reduction_pct=0.846741 moved_to_echo=56788 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4084, 2, 0, 4096, 4096, 2463, 20, 0, 2, 3699, 19, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 4095, 0, 90, 2, 30, 4080, 2153, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 0, 0, 0, 396, 0, 4032, 4096, 0, 0, 0, 0, 0, 2311, 1943, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 423, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2151, 0, 4006, 4096, 4096, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3348, 413, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 4096, 1110, 4096, 0, 5, 4096, 0, 0, 5, 0, 318, 0, 0, 672, 570, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3526, 4095, 4095, 0, 0, 4075, 81, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2998, 123, 5, 0, 0, 0, 3023, 1818, 0, 0, 49, 3874, 0, 3746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18528, 14578, 14294, 15193, 17423, 15453, 19967, 15636] avg=16384.0000 max=19967 max_rank=6 min=14294 min_rank=2 imbalance_ratio=1.218689 max_load_reduction=111105 max_load_reduction_pct=0.847664 moved_to_echo=68479 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3083, 0, 0, 3633, 3425, 0, 0, 0, 49, 3642, 1, 0, 0, 211, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 1, 0, 7, 4068, 0, 3638, 22, 0, 4085, 2144, 80, 0, 192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 555, 4040, 0, 0, 0, 2745, 30, 3899, 2057, 809, 54, 12, 0, 0, 2285, 1936, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 198, 0, 0, 0, 0, 0, 15, 24, 0, 4024, 14, 587, 4019, 3944, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1355, 25, 90, 335, 0, 0, 0, 2, 4021, 391, 62, 0, 2362, 0, 2796, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 4039, 148, 3724, 8, 0, 3990, 105, 0, 0, 0, 0, 1, 19, 668, 528, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3262, 4071, 4096, 0, 0, 712, 1115, 1, 73, 1108, 0, 0, 0, 4061, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 2977, 52, 0, 0, 24, 36, 4055, 4095, 0, 0, 4, 3788, 18, 3879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14045, 18288, 18847, 13298, 15436, 13697, 18500, 18961] avg=16384.0000 max=18961 max_rank=7 min=13298 min_rank=3 imbalance_ratio=1.157288 max_load_reduction=112111 max_load_reduction_pct=0.855339 moved_to_echo=66594 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3856, 0, 0, 4055, 3903, 1657, 3, 0, 159, 3699, 0, 2, 0, 2, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 332, 0, 3, 4095, 11, 35, 244, 61, 3871, 2153, 6, 3, 0, 1, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 122, 0, 0, 72, 0, 0, 4087, 4, 4055, 4029, 10, 585, 91, 150, 0, 2255, 1943, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 39, 3947, 23, 0, 0, 22, 0, 0, 0, 0, 0, 990, 0, 4022, 4050, 3956, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1336, 1, 159, 1154, 0, 78, 12, 1, 4055, 0, 0, 0, 4081, 2555, 240, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 3859, 1417, 4074, 49, 68, 4001, 0, 2, 1, 740, 0, 0, 0, 655, 549, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3391, 3944, 3662, 0, 0, 938, 0, 0, 12, 0, 19, 0, 27, 2952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2919, 171, 0, 0, 55, 1, 1567, 3312, 0, 0, 0, 2888, 8, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17344, 14971, 17830, 17613, 17730, 15878, 14945, 14761] avg=16384.0000 max=17830 max_rank=2 min=14761 min_rank=7 imbalance_ratio=1.088257 max_load_reduction=113242 max_load_reduction_pct=0.863968 moved_to_echo=63314 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[31104, 128, 128, 32384, 27648, 8704, 128, 0, 384, 29312, 896, 128, 0, 640, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[30848, 384, 128, 128, 31744, 128, 12928, 384, 128, 32512, 17152, 128, 4480, 512, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 23936, 1664, 128, 128, 768, 128, 18944, 24064, 4096, 128, 128, 28160, 128, 29696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 27776, 28160, 31872, 0, 0, 9984, 3200, 128, 128, 1280, 128, 0, 128, 29056, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[11008, 128, 256, 11392, 256, 256, 128, 128, 32512, 768, 640, 128, 29696, 8192, 4224, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 384, 128, 12800, 1664, 128, 128, 128, 768, 128, 128, 128, 2048, 18432, 128, 27008, 32640, 32384, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 512, 640, 9472, 512, 128, 128, 18048, 128, 31104, 30592, 896, 1536, 128, 896, 128, 18432, 15488, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[256, 128, 6784, 32512, 5632, 32384, 128, 256, 32512, 256, 128, 128, 1792, 6272, 128, 128, 5376, 4608, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[11, 0, 0, 4093, 0, 746, 420, 4006, 0, 0, 0, 2, 4, 0, 8, 4078, 0, 4096, 150, 3789, 0, 0, 81, 4067, 0, 2792, 0, 4072, 17, 0, 413, 0, 2, 0, 574, 1411, 0, 0, 1129, 4096, 4096, 4096, 0, 0, 2, 4009, 0, 0, 0, 153, 5, 4094, 4096, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 1, 15, 4065, 0, 0, 0, 3992, 0, 3, 4096, 0, 1, 4085, 4096, 4090, 0, 1, 4091, 1, 4085, 3998, 1, 1, 776, 0, 0, 16, 0, 0, 3880, 0, 0, 2, 0, 0, 1, 0, 3363, 0, 0, 4, 0, 60, 0, 0, 3139, 0, 4085, 0, 90, 4096, 3647, 6, 0, 2491, 0, 2, 0, 29, 3628, 0, 431, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 3, 4095, 1, 2188, 3556, 4093, 0, 0, 649, 0, 0, 0, 0, 4096, 0, 4096, 1, 4096, 719, 0, 0, 4096, 11, 4021, 0, 4096, 0, 0, 1, 14, 0, 0, 4076, 95, 0, 0, 0, 4090, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 2717, 0, 4096, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 4096, 4096, 2, 0, 1, 0, 528, 0, 6, 0, 3744, 0, 0, 0, 1, 0, 0, 0, 2202, 0, 0, 0, 0, 0, 1, 0, 17, 0, 4096, 0, 0, 4096, 11, 0, 0, 4091, 0, 0, 0, 0, 4062, 0, 75, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 90, 42, 4079, 11, 0, 0, 4093, 0, 1, 25, 0, 33, 10, 0, 4042, 0, 4095, 3542, 3667, 8, 0, 739, 4083, 102, 720, 0, 4047, 0, 0, 0, 1, 0, 7, 2499, 4096, 0, 75, 21, 4096, 4068, 4077, 0, 0, 0, 0, 1, 0, 0, 82, 14, 3558, 4096, 0, 27, 1, 4096, 0, 0, 0, 4, 17, 0, 0, 0, 323, 4086, 3, 0, 0, 1444, 0, 0, 4096, 0, 0, 3921, 4096, 4092, 0, 0, 4079, 0, 4075, 4066, 487, 0, 1, 100, 0, 275, 0, 0, 4017, 2, 0, 0, 0, 0, 1, 17, 2323, 0, 0, 0, 0, 1110, 3940, 18, 91, 0, 4090, 0, 0, 4090, 4031, 0, 0, 3685, 0, 0, 0, 15, 4094, 0, 139, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 22, 0, 4094, 0, 0, 3, 4, 0, 0, 1, 4083, 0, 4088, 3777, 207, 1200, 0, 0, 4094, 448, 2775, 0, 4034, 0, 0, 0, 0, 0, 0, 2084, 69, 0, 0, 0, 4088, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 4071, 0, 4096, 4096, 0, 0, 29, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 4084, 3160, 0, 0, 0, 0, 1, 5, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 5, 4091, 0, 0, 0, 0, 3372, 1664, 0, 66, 0, 4096, 0, 2, 4096, 2215, 0, 0, 1151, 0, 0, 0, 0, 4093, 0, 2505, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 100, 263, 103, 3909, 0, 0, 0, 7, 0, 0, 34, 4094, 0, 4093, 3980, 3958, 307, 0, 0, 4082, 0, 0, 3, 3743, 28, 3, 0, 0, 1, 0, 3513, 402, 10, 0, 0, 4050, 4095, 4040, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4090, 4085, 6, 0, 3872, 4080, 1, 0, 0, 0, 0, 0, 0, 0, 100, 4039, 0, 0, 0, 4096, 0, 0, 4095, 11, 0, 4096, 4071, 4095, 0, 0, 4094, 0, 3992, 4095, 15, 0, 0, 18, 1, 4, 18, 0, 0, 0, 0, 0, 1, 1, 27, 49, 4015, 0, 0, 0, 0, 6, 0, 5, 123, 24, 4062, 0, 3809, 4060, 600, 0, 2, 3516, 37, 24, 0, 0, 3967, 13, 745, 13, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4011, 2, 3896, 0, 59, 0, 4054, 0, 0, 0, 275, 0, 0, 255, 3727, 0, 4095, 1, 4076, 132, 2, 0, 4093, 4008, 4078, 0, 77, 0, 0, 1538, 961, 0, 6, 4077, 3464, 0, 7, 0, 4096, 4096, 4096, 0, 426, 0, 25, 0, 0, 0, 567, 0, 2801, 3892, 0, 16, 10, 4096, 0, 207, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 2458, 47, 721, 4096, 372, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 4096, 3854, 1, 1, 4, 284, 0, 1, 9, 10, 3550, 0, 0, 0, 1, 0, 0, 0, 155, 0, 0, 0, 0, 0, 911, 0, 1, 0, 3183, 0, 4096, 4088, 0, 0, 0, 0, 1, 0, 4, 11, 3386, 0, 4032, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[146, 2, 4, 3786, 85, 0, 214, 3895, 0, 415, 33, 0, 0, 23, 0, 4094, 0, 4068, 0, 3988, 4095, 0, 9, 4058, 11, 3619, 14, 4072, 0, 0, 0, 0, 292, 57, 4096, 133, 0, 0, 17, 4087, 4068, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 1, 4094, 4094, 0, 0, 0, 4008, 42, 9, 10, 54, 0, 120, 0, 5, 0, 3923, 0, 0, 53, 4096, 0, 14, 4095, 0, 24, 4096, 4092, 4068, 0, 0, 4093, 0, 3911, 4072, 4009, 165, 0, 139, 68, 0, 1924, 3, 0, 11, 0, 10, 0, 0, 2, 0, 3320, 0, 0, 0, 1, 4066, 0, 0, 328, 0, 3875, 0, 0, 4034, 4052, 0, 25, 11, 0, 0, 0, 0, 2567, 0, 4, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[713, 136, 2, 4088, 0, 2140, 3604, 3980, 0, 10, 0, 0, 60, 0, 2, 4068, 1, 4085, 0, 123, 0, 0, 0, 3712, 0, 0, 0, 4088, 138, 33, 45, 0, 0, 0, 2379, 0, 53, 0, 3750, 3961, 4096, 4096, 0, 0, 0, 3560, 0, 27, 0, 4056, 0, 3856, 4092, 0, 0, 4, 4054, 9, 0, 6, 9, 8, 0, 0, 0, 0, 4086, 179, 0, 0, 4096, 0, 4, 4067, 0, 56, 4028, 4034, 4068, 5, 0, 4048, 1, 3945, 4080, 135, 0, 0, 1334, 1, 399, 0, 9, 3, 0, 0, 0, 0, 0, 35, 0, 1152, 0, 0, 0, 0, 355, 3892, 0, 0, 0, 4096, 6, 0, 4095, 2546, 32, 0, 4091, 11, 0, 0, 0, 3139, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 4095, 1, 2188, 3556, 4093, 0, 0, 649, 0, 0, 0, 0, 4096, 1087, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 1, 4096, 719, 0, 0, 4096, 11, 4021, 0, 4096, 0, 0, 1, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 95, 0, 0, 0, 4090, 2426, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2717, 0, 4096, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 4096, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 2, 0, 1, 0, 528, 0, 6, 0, 3744, 0, 0, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2202, 0, 0, 0, 0, 0, 1, 0, 17, 0, 4096, 4096, 2852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 4091, 0, 0, 0, 0, 4062, 0, 75, 0, 0, 1172, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20422, 17321, 14783, 16881, 16763, 17152, 13265, 14485] avg=16384.0000 max=20422 max_rank=0 min=13265 min_rank=6 imbalance_ratio=1.246460 max_load_reduction=110650 max_load_reduction_pct=0.844193 moved_to_echo=61665 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 0, 4093, 0, 746, 420, 4006, 0, 0, 0, 2, 4, 0, 8, 4078, 1087, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 150, 3789, 0, 0, 81, 4067, 0, 2792, 0, 4072, 17, 0, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 574, 1411, 0, 0, 1129, 4096, 2426, 4096, 0, 0, 2, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 5, 4094, 4096, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 1877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 15, 4065, 0, 0, 0, 3992, 0, 3, 0, 0, 1, 4085, 4096, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 1, 4085, 3998, 1, 1, 776, 0, 0, 16, 0, 0, 3880, 0, 0, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 3363, 0, 0, 4, 0, 60, 0, 0, 3139, 0, 4085, 4096, 2852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 4096, 3647, 6, 0, 2491, 0, 2, 0, 29, 3628, 0, 431, 0, 0, 1171, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15108, 15647, 17745, 14321, 16647, 17433, 17602, 16569] avg=16384.0000 max=17745 max_rank=2 min=14321 min_rank=3 imbalance_ratio=1.083069 max_load_reduction=113327 max_load_reduction_pct=0.864616 moved_to_echo=68251 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 22, 0, 4094, 0, 0, 3, 4, 0, 0, 1, 4083, 1087, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 3777, 207, 1200, 0, 0, 4094, 448, 2775, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2084, 69, 0, 0, 0, 4088, 2426, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 4096, 4096, 0, 0, 29, 4089, 0, 0, 0, 0, 0, 0, 0, 1879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 4096, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4084, 3160, 0, 0, 0, 0, 1, 5, 0, 0, 4053, 0, 0, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4091, 0, 0, 0, 0, 3372, 1664, 0, 66, 0, 4096, 4096, 2846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 2215, 0, 0, 1151, 0, 0, 0, 0, 4093, 0, 2505, 0, 0, 1172, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14044, 16801, 12766, 18260, 16773, 15982, 20236, 16210] avg=16384.0000 max=20236 max_rank=6 min=12766 min_rank=2 imbalance_ratio=1.235107 max_load_reduction=110836 max_load_reduction_pct=0.845612 moved_to_echo=69201 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 90, 42, 4079, 11, 0, 0, 4093, 0, 1, 25, 0, 33, 10, 0, 4042, 1080, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 3542, 3667, 8, 0, 739, 4083, 102, 720, 0, 4047, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2499, 4096, 0, 75, 21, 4096, 2409, 4077, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 14, 3558, 4096, 0, 27, 1, 4096, 0, 0, 0, 4, 17, 0, 0, 1878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 4086, 3, 0, 0, 1444, 0, 0, 0, 0, 0, 3921, 4096, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 4075, 4066, 487, 0, 1, 100, 0, 275, 0, 0, 4017, 2, 0, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 17, 2323, 0, 0, 0, 0, 1110, 3940, 18, 91, 0, 4090, 4096, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4031, 0, 0, 3685, 0, 0, 0, 15, 4094, 0, 139, 0, 0, 1171, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14160, 17175, 17281, 13773, 14262, 17681, 18537, 18203] avg=16384.0000 max=18537 max_rank=6 min=13773 min_rank=3 imbalance_ratio=1.131409 max_load_reduction=112535 max_load_reduction_pct=0.858574 moved_to_echo=68683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 100, 263, 103, 3909, 0, 0, 0, 7, 0, 0, 34, 4094, 1090, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 3980, 3958, 307, 0, 0, 4082, 0, 0, 3, 3743, 28, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3513, 402, 10, 0, 0, 4050, 2419, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4090, 4085, 6, 0, 3872, 4080, 1, 0, 0, 0, 0, 0, 0, 1883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 4039, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 4096, 4071, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3992, 4095, 15, 0, 0, 18, 1, 4, 18, 0, 0, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 27, 49, 4015, 0, 0, 0, 0, 6, 0, 5, 123, 24, 4062, 4095, 2855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3809, 4060, 600, 0, 2, 3516, 37, 24, 0, 0, 3967, 13, 745, 13, 4, 1177, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14351, 16360, 14435, 22102, 16789, 12823, 15263, 18949] avg=16384.0000 max=22102 max_rank=3 min=12823 min_rank=5 imbalance_ratio=1.348999 max_load_reduction=108970 max_load_reduction_pct=0.831375 moved_to_echo=63824 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4011, 2, 3896, 0, 59, 0, 4054, 0, 0, 0, 275, 0, 0, 255, 3727, 1087, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 1, 4076, 132, 2, 0, 4093, 4008, 4078, 0, 77, 0, 0, 1538, 961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4077, 3464, 0, 7, 0, 4096, 2426, 4096, 0, 426, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 0, 2801, 3892, 0, 16, 10, 4096, 0, 207, 0, 0, 0, 0, 0, 1879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 2458, 47, 721, 0, 372, 0, 4096, 4096, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3854, 1, 1, 4, 284, 0, 1, 9, 10, 3550, 0, 0, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 155, 0, 0, 0, 0, 0, 911, 0, 1, 0, 3183, 4096, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 1, 0, 4, 11, 3386, 0, 4032, 1, 0, 1172, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18020, 19232, 18623, 13468, 16273, 16489, 11198, 17769] avg=16384.0000 max=19232 max_rank=1 min=11198 min_rank=6 imbalance_ratio=1.173828 max_load_reduction=111840 max_load_reduction_pct=0.853271 moved_to_echo=61729 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[146, 2, 4, 3786, 85, 0, 214, 3895, 0, 415, 33, 0, 0, 23, 0, 4094, 1080, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 3988, 4095, 0, 9, 4058, 11, 3619, 14, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 57, 4096, 133, 0, 0, 17, 4087, 2409, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 4094, 4094, 0, 0, 0, 4008, 42, 9, 10, 54, 0, 120, 0, 1867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3923, 0, 0, 53, 4096, 0, 14, 0, 0, 24, 4096, 4092, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3911, 4072, 4009, 165, 0, 139, 68, 0, 1924, 3, 0, 11, 0, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 0, 3320, 0, 0, 0, 1, 4066, 0, 0, 328, 0, 3875, 4095, 2833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 4052, 0, 25, 11, 0, 0, 0, 0, 2567, 0, 4, 0, 0, 1164, 971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14427, 20130, 15187, 14306, 16690, 18974, 18530, 12828] avg=16384.0000 max=20130 max_rank=1 min=12828 min_rank=7 imbalance_ratio=1.228638 max_load_reduction=110942 max_load_reduction_pct=0.846420 moved_to_echo=67022 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[713, 136, 2, 4088, 0, 2140, 3604, 3980, 0, 10, 0, 0, 60, 0, 2, 4068, 1087, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 266, 0, 123, 0, 0, 0, 3712, 0, 0, 0, 4088, 138, 33, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2379, 0, 53, 0, 3750, 3961, 2426, 4096, 0, 0, 0, 3560, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 3856, 4092, 0, 0, 4, 4054, 9, 0, 6, 9, 8, 0, 0, 1867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 179, 0, 0, 4096, 0, 4, 0, 0, 56, 4028, 4034, 387, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 1, 3945, 4080, 135, 0, 0, 1334, 1, 399, 0, 9, 3, 0, 0, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 1152, 0, 0, 0, 0, 355, 3892, 0, 0, 0, 4096, 4067, 2844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4095, 2546, 32, 0, 4091, 11, 0, 0, 0, 3139, 0, 0, 0, 0, 1164, 975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20540, 8406, 20252, 17961, 16875, 14538, 16441, 16059] avg=16384.0000 max=20540 max_rank=0 min=8406 min_rank=1 imbalance_ratio=1.253662 max_load_reduction=110532 max_load_reduction_pct=0.843292 moved_to_echo=63913 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[896, 4352, 128, 32256, 256, 5504, 7936, 32128, 0, 512, 768, 384, 128, 128, 384, 32384, 8704, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 8064, 32768, 17152, 128, 128, 19072, 128, 128, 128, 128, 29056, 128, 7936, 128, 128, 9472, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 512, 32512, 256, 0, 128, 28416, 128, 768, 0, 384, 128, 32640, 32768, 3200, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 32384, 31488, 4736, 256, 896, 1920, 640, 768, 2048, 128, 19328, 128, 0, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 15744, 128, 30720, 32640, 128, 128, 3968, 32640, 128, 256, 128, 128, 128, 128, 0, 15104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 2176, 11520, 23936, 6528, 128, 896, 32384, 4608, 18048, 128, 28288, 256, 128, 2048, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[384, 128, 23424, 9728, 128, 128, 4992, 32640, 19456, 32768, 0, 512, 128, 7680, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 20736, 0, 0, 128, 128, 9088, 10496, 128, 3840, 128, 31616, 32768, 22784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 35, 0, 22, 0, 4, 0, 790, 0, 4031, 156, 4, 0, 0, 0, 1652, 0, 130, 0, 4034, 0, 4096, 0, 0, 4095, 0, 0, 3214, 13, 3908, 0, 0, 4065, 4095, 0, 214, 4093, 194, 3995, 1, 0, 0, 0, 0, 77, 73, 0, 4039, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3594, 47, 0, 3985, 0, 0, 4096, 0, 150, 1, 4069, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2557, 0, 3, 4037, 49, 374, 0, 17, 0, 4, 1733, 4048, 0, 0, 0, 0, 44, 2190, 0, 4094, 0, 0, 0, 0, 4096, 0, 7, 0, 4096, 22, 27, 1, 71, 1683, 0, 0, 3917, 3059, 3686, 4096, 4095, 0, 1, 4095, 0, 4089, 3753] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[7, 287, 6, 0, 0, 1, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 2, 0, 3632, 104, 210, 4085, 90, 4082, 0, 33, 4096, 0, 58, 12, 0, 4095, 0, 1, 4070, 4073, 0, 0, 4096, 1649, 4061, 0, 0, 5, 4094, 2, 0, 1249, 372, 4067, 19, 4058, 16, 0, 0, 0, 21, 0, 0, 0, 0, 25, 4096, 0, 18, 3842, 0, 998, 4006, 0, 0, 0, 4095, 31, 0, 0, 0, 5, 0, 0, 3, 0, 4096, 0, 0, 4096, 1, 0, 87, 0, 0, 9, 3675, 4094, 71, 3, 32, 0, 3, 0, 0, 4074, 3967, 2, 0, 25, 4075, 0, 37, 0, 4013, 0, 0, 0, 1, 0, 215, 8, 822, 618, 4035, 4004, 3861, 0, 0, 4085, 0, 3272, 3929] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2337, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4096, 0, 0, 441, 0, 0, 37, 0, 4096, 0, 0, 4096, 4095, 0, 0, 4096, 50, 4096, 0, 0, 0, 4093, 0, 0, 141, 264, 4096, 0, 4095, 0, 1, 0, 0, 5, 0, 1119, 0, 0, 4090, 4096, 1, 0, 4092, 0, 710, 4095, 0, 0, 0, 4092, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3494, 0, 3215, 4030, 0, 1, 0, 0, 0, 0, 4084, 4096, 0, 4, 0, 0, 0, 3, 0, 4096, 17, 0, 0, 0, 4096, 0, 0, 0, 4054, 0, 810, 3, 0, 0, 2668, 0, 1829, 4085, 4096, 150, 4096, 0, 0, 4096, 0, 3326, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 16, 0, 0, 0, 11, 0, 4093, 897, 0, 0, 0, 0, 0, 475, 0, 0, 4096, 0, 4096, 0, 0, 0, 3, 1, 46, 493, 4071, 9, 0, 2277, 4096, 0, 0, 3763, 3, 4096, 60, 0, 0, 2700, 63, 0, 3619, 1, 4096, 0, 4026, 0, 80, 1066, 0, 5, 0, 3913, 0, 0, 359, 4074, 2, 0, 4096, 0, 1864, 3887, 0, 497, 0, 3511, 0, 8, 0, 0, 0, 0, 0, 0, 1, 23, 0, 12, 4079, 0, 4048, 0, 0, 10, 0, 1, 4073, 0, 401, 0, 0, 0, 1, 16, 4096, 2617, 0, 0, 4, 4080, 0, 0, 0, 4096, 2, 0, 433, 0, 409, 106, 0, 4096, 2726, 4096, 4036, 4096, 753, 0, 4096, 11, 4089, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 0, 104, 0, 4095, 0, 0, 0, 0, 0, 0, 1765, 150, 4, 4096, 0, 4096, 0, 0, 1799, 0, 0, 9, 0, 4096, 0, 0, 4096, 4096, 0, 0, 3825, 4094, 4096, 0, 0, 0, 4096, 0, 0, 3873, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3455, 4096, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2681, 0, 77, 4096, 0, 0, 1, 0, 0, 0, 1952, 2761, 2, 0, 0, 0, 0, 4, 31, 4096, 521, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2567, 0, 251, 4018, 4053, 3743, 4096, 0, 0, 4096, 0, 3321, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 46, 3, 4089, 0, 0, 0, 0, 0, 0, 1121, 23, 0, 4094, 160, 4092, 0, 0, 16, 2, 0, 0, 98, 4094, 0, 0, 4073, 4082, 0, 0, 1697, 3381, 4075, 0, 0, 0, 4086, 0, 62, 4096, 13, 4026, 0, 4056, 2, 44, 25, 0, 82, 0, 0, 0, 1, 1625, 4033, 0, 0, 4061, 0, 3653, 4095, 0, 2, 0, 4052, 26, 0, 0, 0, 3, 1, 10, 0, 0, 4002, 0, 197, 4063, 7, 0, 0, 0, 0, 0, 3569, 3338, 5, 6, 0, 0, 0, 634, 0, 4069, 58, 1, 0, 0, 4093, 1, 0, 38, 4085, 0, 1, 0, 0, 41, 0, 0, 1778, 3615, 4096, 3805, 4096, 0, 2, 4081, 0, 4095, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 25, 9, 0, 282, 0, 0, 115, 0, 30, 3872, 0, 0, 3, 0, 0, 96, 0, 0, 0, 3984, 0, 3995, 0, 242, 3823, 395, 0, 1236, 0, 4073, 15, 82, 3750, 4077, 0, 0, 338, 37, 4021, 0, 0, 0, 4046, 0, 0, 27, 0, 3555, 3, 3871, 24, 93, 0, 28, 0, 0, 2680, 0, 163, 487, 4091, 2153, 0, 4094, 34, 13, 4075, 0, 490, 239, 4096, 0, 0, 0, 30, 76, 184, 0, 0, 0, 4057, 0, 3945, 3477, 0, 145, 0, 0, 0, 0, 4096, 4023, 0, 0, 0, 0, 0, 0, 467, 3995, 3913, 0, 1, 0, 4078, 93, 0, 85, 2263, 0, 35, 0, 42, 0, 3, 3, 4064, 3933, 4096, 2588, 3984, 0, 0, 3843, 16, 679, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 0, 115, 0, 24, 0, 4, 48, 0, 3992, 14, 0, 0, 0, 0, 6, 2349, 0, 328, 4086, 9, 3828, 0, 352, 0, 0, 0, 114, 2, 3957, 0, 2, 4096, 4094, 77, 0, 4094, 658, 4096, 0, 0, 0, 4078, 0, 110, 5, 0, 4033, 0, 4095, 0, 65, 0, 18, 0, 0, 0, 8, 0, 4037, 4091, 0, 0, 4019, 0, 0, 4095, 0, 4, 67, 3772, 0, 320, 0, 5, 12, 0, 19, 0, 0, 4089, 0, 4086, 4087, 28, 0, 14, 0, 0, 0, 835, 3839, 61, 0, 0, 66, 0, 21, 0, 4012, 2272, 0, 1, 0, 3877, 76, 1, 234, 4022, 2, 0, 0, 73, 0, 4059, 0, 710, 4022, 2508, 2822, 4096, 7, 1, 4096, 0, 4046, 3810] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 2337, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 3552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4096, 0, 0, 441, 0, 0, 37, 0, 4096, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 50, 2872, 0, 0, 0, 4093, 0, 0, 141, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 1, 0, 0, 5, 0, 1119, 0, 0, 4090, 4096, 1, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 710, 4095, 0, 0, 0, 4092, 0, 0, 0, 3, 0, 0, 0, 0, 2664, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3494, 0, 3215, 4030, 0, 1, 0, 0, 0, 0, 4084, 4096, 0, 4, 0, 495, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 17, 0, 0, 0, 4096, 0, 0, 0, 4054, 0, 810, 1432, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2668, 0, 1829, 4085, 2966, 150, 0, 0, 0, 0, 0, 3326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18176, 12870, 15612, 18126, 16200, 19824, 15237, 15027] avg=16384.0000 max=19824 max_rank=5 min=12870 min_rank=1 imbalance_ratio=1.209961 max_load_reduction=111248 max_load_reduction_pct=0.848755 moved_to_echo=66288 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 35, 0, 22, 0, 4, 0, 790, 0, 4031, 156, 4, 0, 0, 0, 4095, 4095, 3551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1652, 0, 130, 0, 4034, 0, 4096, 0, 0, 4095, 0, 0, 3214, 13, 3908, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 214, 4093, 194, 2801, 1, 0, 0, 0, 0, 77, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3594, 47, 0, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 4096, 0, 150, 1, 4069, 0, 0, 1, 0, 0, 0, 0, 0, 2441, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2557, 0, 3, 4037, 49, 374, 0, 17, 0, 4, 1733, 4048, 0, 0, 0, 483, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 2190, 0, 4094, 0, 0, 0, 0, 4096, 0, 7, 0, 4096, 22, 27, 1312, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 71, 1683, 0, 0, 3917, 3059, 2669, 4096, 0, 0, 1, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16783, 21234, 11518, 16396, 15287, 13669, 16599, 19586] avg=16384.0000 max=21234 max_rank=1 min=11518 min_rank=2 imbalance_ratio=1.296021 max_load_reduction=109838 max_load_reduction_pct=0.837997 moved_to_echo=65141 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[7, 287, 6, 0, 0, 1, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 2, 4073, 4085, 3349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3632, 104, 210, 4085, 90, 4082, 0, 33, 4096, 0, 58, 12, 0, 4095, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4070, 0, 0, 0, 4096, 1649, 2835, 0, 0, 5, 4094, 2, 0, 1249, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 19, 4058, 16, 0, 0, 0, 21, 0, 0, 0, 0, 25, 4096, 0, 18, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 0, 998, 4006, 0, 0, 0, 4095, 31, 0, 0, 0, 5, 0, 0, 3, 2556, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 87, 0, 0, 9, 3675, 4094, 71, 3, 32, 497, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4074, 3967, 2, 0, 25, 4075, 0, 37, 0, 4013, 0, 0, 1373, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 215, 8, 822, 618, 2907, 4004, 0, 0, 0, 0, 0, 3272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15900, 20604, 18373, 12938, 16048, 17064, 18298, 11847] avg=16384.0000 max=20604 max_rank=1 min=11847 min_rank=7 imbalance_ratio=1.257568 max_load_reduction=110468 max_load_reduction_pct=0.842804 moved_to_echo=63257 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 0, 104, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 3552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1765, 150, 4, 4096, 0, 4096, 0, 0, 1799, 0, 0, 9, 0, 4096, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3825, 4094, 2872, 0, 0, 0, 4096, 0, 0, 3873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3455, 4096, 0, 0, 617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 2662, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2681, 0, 77, 4096, 0, 0, 1, 0, 0, 0, 1952, 2761, 2, 0, 0, 495, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 31, 4096, 521, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1432, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2567, 0, 251, 4018, 2934, 3743, 0, 0, 0, 0, 0, 3321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15947, 16116, 22856, 16352, 15496, 12466, 15005, 16834] avg=16384.0000 max=22856 max_rank=2 min=12466 min_rank=5 imbalance_ratio=1.395020 max_load_reduction=108216 max_load_reduction_pct=0.825623 moved_to_echo=59801 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 16, 0, 0, 0, 11, 0, 4093, 897, 0, 0, 0, 0, 4096, 4096, 3552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 4096, 0, 4096, 0, 0, 0, 3, 1, 46, 493, 4071, 9, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2277, 0, 0, 0, 3763, 3, 2872, 60, 0, 0, 2700, 63, 0, 3619, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4026, 0, 80, 1066, 0, 5, 0, 3913, 0, 0, 359, 4074, 2, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1864, 3887, 0, 497, 0, 3511, 0, 8, 0, 0, 0, 0, 0, 0, 2664, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 23, 0, 12, 4079, 0, 4048, 0, 0, 10, 0, 1, 4073, 0, 401, 0, 495, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 16, 4096, 2617, 0, 0, 4, 4080, 0, 0, 0, 4096, 2, 0, 1432, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 0, 409, 106, 0, 4096, 2726, 2966, 4036, 0, 753, 0, 0, 11, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16761, 13392, 15358, 18244, 17071, 13548, 17073, 19625] avg=16384.0000 max=19625 max_rank=7 min=13392 min_rank=1 imbalance_ratio=1.197815 max_load_reduction=111447 max_load_reduction_pct=0.850273 moved_to_echo=67317 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 46, 3, 4089, 0, 0, 0, 0, 0, 4082, 4081, 3552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1121, 23, 0, 4094, 160, 4092, 0, 0, 16, 2, 0, 0, 98, 4094, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 1697, 3381, 2856, 0, 0, 0, 4086, 0, 62, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 4056, 2, 44, 25, 0, 82, 0, 0, 0, 1, 1625, 4033, 0, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 3653, 4095, 0, 2, 0, 4052, 26, 0, 0, 0, 3, 1, 10, 0, 2664, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 197, 4063, 7, 0, 0, 0, 0, 0, 3569, 3338, 5, 6, 0, 493, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 634, 0, 4069, 58, 1, 0, 0, 4093, 1, 0, 38, 4085, 0, 1, 1432, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 1778, 3615, 2966, 3805, 0, 0, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15853, 13802, 20264, 14517, 19111, 16085, 15138, 16302] avg=16384.0000 max=20264 max_rank=2 min=13802 min_rank=1 imbalance_ratio=1.236816 max_load_reduction=110808 max_load_reduction_pct=0.845398 moved_to_echo=66636 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 25, 9, 0, 282, 0, 0, 115, 0, 30, 3872, 0, 0, 3, 0, 0, 4077, 3843, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 3984, 0, 3995, 0, 242, 3823, 395, 0, 1236, 0, 4073, 15, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 3750, 0, 0, 0, 338, 37, 2819, 0, 0, 0, 4046, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3555, 3, 3871, 24, 93, 0, 28, 0, 0, 2680, 0, 163, 487, 4091, 2153, 0, 623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 34, 13, 4075, 0, 490, 239, 4096, 0, 0, 0, 30, 76, 184, 0, 0, 2665, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 3945, 3477, 0, 145, 0, 0, 0, 0, 4096, 4023, 0, 0, 0, 486, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 3995, 3913, 0, 1, 0, 4078, 93, 0, 85, 2263, 0, 35, 1431, 716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 3, 3, 4064, 3933, 2966, 2588, 0, 0, 0, 0, 16, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15712, 17961, 11099, 17771, 16524, 20634, 17077, 14294] avg=16384.0000 max=20634 max_rank=5 min=11099 min_rank=2 imbalance_ratio=1.259399 max_load_reduction=110438 max_load_reduction_pct=0.842575 moved_to_echo=68529 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 115, 0, 24, 0, 4, 48, 0, 3992, 14, 0, 0, 0, 0, 4094, 4096, 3552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2349, 0, 328, 4086, 9, 3828, 0, 352, 0, 0, 0, 114, 2, 3957, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 77, 0, 4094, 658, 2872, 0, 0, 0, 4078, 0, 110, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 4095, 0, 65, 0, 18, 0, 0, 0, 8, 0, 4037, 4091, 0, 0, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 4095, 0, 4, 67, 3772, 0, 320, 0, 5, 12, 0, 19, 0, 2478, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4086, 4087, 28, 0, 14, 0, 0, 0, 835, 3839, 61, 0, 0, 495, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 21, 0, 4012, 2272, 0, 1, 0, 3877, 76, 1, 234, 4022, 2, 0, 1332, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 4059, 0, 710, 4022, 1817, 2822, 0, 7, 1, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15940, 15093, 15992, 16728, 15335, 17782, 16645, 17557] avg=16384.0000 max=17782 max_rank=5 min=15093 min_rank=1 imbalance_ratio=1.085327 max_load_reduction=113290 max_load_reduction_pct=0.864334 moved_to_echo=67319 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1792, 9344, 512, 640, 32640, 384, 32384, 0, 640, 14336, 512, 128, 4736, 640, 32512, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 2944, 640, 32640, 13440, 128, 128, 128, 32512, 256, 128, 384, 30848, 128, 896, 11264, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 30592, 0, 128, 256, 26112, 10112, 22912, 128, 0, 128, 27264, 128, 256, 13184, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 25088, 0, 11648, 32000, 128, 4608, 128, 128, 128, 128, 19968, 30336, 256, 512, 128, 3968, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[512, 256, 2176, 9728, 128, 17536, 26112, 22272, 25344, 0, 768, 128, 0, 128, 27008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32128, 128, 32384, 128, 384, 1152, 128, 128, 0, 7808, 128, 256, 18176, 32256, 2304, 128, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 384, 128, 128, 384, 128, 128, 128, 3456, 128, 32384, 1152, 128, 128, 0, 128, 32768, 32512, 28160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32384, 128, 7296, 32512, 0, 1152, 384, 31872, 128, 384, 128, 128, 128, 256, 128, 128, 20864, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 0, 0, 3943, 44, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3985, 4096, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 823, 4075, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 2, 4095, 4096, 6, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 5, 4095, 86, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 4093, 0, 4096, 0, 1983, 4096, 6, 1, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 2453, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 2960, 0, 14, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 37, 4080, 409, 314, 4096, 0, 12, 251, 0, 1, 4035, 0, 0, 0, 0, 0, 0, 701, 4081, 483, 41, 0, 105, 4089, 0, 1111, 4095, 2980, 38, 4096, 0, 0, 4024, 0, 0, 2, 0, 1797, 0, 0, 0, 0, 0, 4096, 76, 2165, 4060, 18, 0, 0, 0, 1515, 39, 33, 0, 0, 6, 0, 0, 0, 4096, 0, 56, 2372, 0, 156, 0, 0, 0, 0, 26, 3904, 2893, 0, 275, 31, 3848, 2, 0, 0, 4096, 0, 2, 0, 4080, 0, 4084, 0, 3609, 0, 3409, 3605, 0, 1, 0, 3, 0, 0, 4022, 4096, 0, 0, 253, 0, 2479, 0, 106, 4013, 4086, 4096, 4038, 1, 0, 0, 0, 3888, 6, 1912, 0, 542, 4090, 10, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4086, 0, 5, 4090, 0, 0, 79, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 2574, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 2, 4096, 0, 397, 4096, 0, 0, 0, 3, 76, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 29, 2, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 40, 3982, 0, 4070, 0, 3837, 3972, 2253, 0, 0, 0, 0, 0, 3907, 3841, 0, 0, 0, 246, 0, 0, 0, 4093, 4096, 4096, 4096, 0, 0, 1, 2, 3648, 0, 0, 0, 3908, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 52, 26, 4071, 39, 0, 890, 566, 0, 4063, 0, 0, 1484, 0, 7, 0, 3562, 4076, 21, 248, 0, 3, 4096, 5, 0, 4090, 4093, 0, 4085, 223, 17, 4094, 4, 0, 2, 0, 0, 2, 0, 0, 0, 1, 4096, 0, 3542, 4094, 2, 0, 0, 0, 2845, 0, 20, 0, 0, 0, 0, 0, 2, 4091, 0, 0, 4091, 0, 0, 1, 0, 0, 0, 12, 3887, 60, 0, 0, 0, 3945, 0, 0, 0, 4096, 0, 0, 3, 4096, 0, 4096, 0, 4006, 0, 4055, 3743, 480, 15, 22, 0, 122, 0, 4089, 2333, 48, 1, 0, 0, 0, 0, 0, 4046, 4094, 4096, 4096, 0, 5, 0, 0, 4048, 0, 36, 0, 746, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 49, 0, 0, 4088, 0, 0, 2747, 77, 0, 1063, 0, 0, 4096, 1, 20, 2874, 0, 0, 0, 670, 3821, 1, 547, 0, 909, 4026, 50, 562, 3832, 4094, 3479, 4010, 0, 0, 4061, 193, 0, 150, 53, 0, 0, 0, 0, 0, 0, 4093, 0, 3344, 4069, 262, 27, 1, 0, 0, 0, 0, 0, 11, 14, 0, 5, 0, 4080, 0, 0, 3600, 143, 0, 0, 31, 2, 3, 0, 3547, 1, 87, 0, 0, 4074, 0, 0, 0, 4096, 302, 7, 4, 3780, 0, 3823, 0, 4047, 31, 4020, 3895, 2659, 1, 315, 0, 0, 2, 4062, 4073, 0, 194, 0, 459, 2, 34, 0, 3571, 3987, 4096, 4096, 0, 0, 474, 0, 1281, 1, 2113, 0, 8, 2672, 0, 34, 0, 66] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 3, 1, 0, 4073, 0, 0, 3780, 0, 0, 49, 5, 45, 4082, 0, 30, 0, 0, 0, 3, 4096, 4093, 33, 0, 0, 0, 4068, 48, 0, 4093, 4090, 122, 4090, 38, 0, 4084, 0, 0, 11, 0, 1147, 5, 2221, 0, 7, 0, 4075, 83, 4092, 4057, 373, 6, 0, 1, 264, 19, 0, 0, 0, 1, 0, 0, 0, 4096, 4, 0, 4095, 7, 0, 0, 0, 0, 2, 8, 4087, 1042, 8, 0, 0, 4053, 0, 0, 9, 4069, 0, 0, 1, 3986, 1, 4006, 1, 4096, 21, 108, 3997, 3712, 0, 0, 0, 0, 0, 4088, 3964, 0, 0, 0, 0, 1, 1, 18, 3467, 4040, 4077, 4094, 1, 0, 117, 13, 1362, 18, 3039, 4, 0, 4070, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 78, 3282, 0, 126, 4096, 0, 0, 1317, 728, 13, 4095, 0, 0, 1, 0, 0, 0, 4079, 4063, 0, 572, 0, 66, 4015, 0, 175, 4091, 0, 4, 4096, 0, 0, 4096, 0, 0, 0, 0, 139, 0, 1307, 0, 1, 0, 4096, 2817, 852, 4096, 4096, 0, 1, 0, 0, 937, 0, 0, 6, 444, 0, 0, 4, 4080, 0, 0, 3979, 0, 0, 0, 0, 0, 0, 1321, 4068, 0, 0, 0, 0, 3450, 0, 0, 0, 4096, 0, 0, 0, 4063, 0, 4096, 0, 145, 0, 4096, 3631, 0, 34, 0, 0, 744, 7, 4096, 3974, 0, 0, 0, 80, 1004, 0, 0, 3694, 4089, 3947, 4096, 0, 0, 3, 0, 3528, 0, 2855, 0, 15, 4090, 0, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[420, 0, 1, 9, 3985, 0, 343, 4068, 0, 3, 0, 44, 92, 3744, 91, 0, 4066, 0, 0, 13, 3542, 4067, 0, 0, 21, 102, 4085, 18, 0, 4096, 4007, 0, 3705, 0, 3, 4060, 128, 7, 0, 0, 2425, 1, 555, 0, 216, 0, 4077, 0, 4040, 4041, 0, 0, 7, 0, 9, 8, 0, 0, 93, 0, 1, 0, 0, 4071, 0, 0, 3624, 0, 22, 0, 0, 0, 54, 0, 4068, 780, 0, 0, 3, 4092, 2, 148, 0, 4053, 6, 0, 0, 4096, 0, 3716, 0, 3975, 28, 0, 4086, 26, 0, 0, 0, 0, 0, 3454, 3284, 0, 78, 56, 8, 0, 0, 0, 4088, 4096, 4030, 3996, 23, 0, 955, 0, 3027, 0, 2347, 0, 41, 4091, 0, 37, 0, 418] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4096, 0, 0, 3943, 44, 0, 0, 0, 0, 4095, 0, 0, 3353, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 4096, 0, 0, 0, 1, 0, 0, 0, 3778, 4096, 823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 2, 1630, 357, 318, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 6, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2842, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 5, 4095, 86, 0, 0, 0, 4096, 3835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1124, 0, 4093, 0, 4096, 0, 1983, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 2453, 0, 0, 4096, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2466, 0, 0, 0, 0, 4096, 0, 2960, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15793, 16779, 14706, 15974, 16211, 15399, 18482, 17728] avg=16384.0000 max=18482 max_rank=6 min=14706 min_rank=2 imbalance_ratio=1.128052 max_load_reduction=112590 max_load_reduction_pct=0.858994 moved_to_echo=67820 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 37, 4080, 409, 314, 4096, 0, 12, 251, 0, 1, 4035, 0, 0, 3347, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 701, 4081, 483, 41, 0, 105, 0, 0, 1111, 3777, 2980, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4024, 0, 0, 2, 0, 1797, 0, 0, 0, 0, 0, 4096, 76, 1607, 356, 318, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2165, 4060, 18, 0, 0, 0, 1515, 39, 33, 0, 0, 6, 0, 0, 0, 4096, 2831, 742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 2372, 0, 156, 0, 0, 0, 0, 26, 3904, 2893, 0, 275, 31, 3848, 3835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 1120, 0, 4084, 0, 3609, 0, 3409, 3605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 4022, 4096, 0, 0, 253, 0, 2479, 0, 106, 4013, 3730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2431, 1, 0, 0, 0, 3888, 6, 1912, 0, 542, 4090, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16843, 13317, 16501, 15505, 17396, 15831, 18703, 16976] avg=16384.0000 max=18703 max_rank=6 min=13317 min_rank=1 imbalance_ratio=1.141541 max_load_reduction=112369 max_load_reduction_pct=0.857307 moved_to_echo=68906 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4086, 0, 5, 4090, 0, 0, 79, 0, 0, 4096, 0, 0, 3354, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 2574, 4096, 0, 0, 0, 0, 0, 0, 0, 3778, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 397, 4096, 0, 0, 0, 3, 76, 0, 0, 0, 0, 0, 4096, 0, 1630, 357, 318, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 29, 2, 0, 0, 0, 12, 0, 0, 0, 4096, 2842, 742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 3834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1124, 40, 3982, 0, 4070, 0, 3837, 3972, 2253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 3841, 0, 0, 0, 246, 0, 0, 0, 4093, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2466, 0, 0, 1, 2, 3648, 0, 0, 0, 3908, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15972, 14557, 15199, 15915, 16108, 19278, 15826, 18217] avg=16384.0000 max=19278 max_rank=5 min=14557 min_rank=1 imbalance_ratio=1.176636 max_load_reduction=111794 max_load_reduction_pct=0.852921 moved_to_echo=69429 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4095, 52, 26, 4071, 39, 0, 890, 566, 0, 4063, 0, 0, 3353, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1484, 0, 7, 0, 3562, 4076, 21, 248, 0, 3, 0, 5, 0, 3773, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 223, 17, 4094, 4, 0, 2, 0, 0, 2, 0, 0, 0, 1, 4096, 0, 1630, 357, 317, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 4094, 2, 0, 0, 0, 2845, 0, 20, 0, 0, 0, 0, 0, 2, 4091, 2842, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 1, 0, 0, 0, 12, 3887, 60, 0, 0, 0, 3945, 3835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1124, 0, 4096, 0, 4006, 0, 4055, 3743, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 22, 0, 122, 0, 4089, 2333, 48, 1, 0, 0, 0, 0, 0, 4046, 3737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2466, 0, 5, 0, 0, 4048, 0, 36, 0, 746, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17417, 17272, 14958, 18181, 15831, 17507, 14413, 15493] avg=16384.0000 max=18181 max_rank=3 min=14413 min_rank=6 imbalance_ratio=1.109680 max_load_reduction=112891 max_load_reduction_pct=0.861290 moved_to_echo=63244 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 1, 0, 4073, 0, 0, 3780, 0, 0, 49, 5, 45, 4082, 0, 30, 3331, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4093, 33, 0, 0, 0, 0, 48, 0, 3772, 4090, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 38, 0, 4084, 0, 0, 11, 0, 1147, 5, 2221, 0, 7, 0, 4075, 83, 1630, 357, 321, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4057, 373, 6, 0, 1, 264, 19, 0, 0, 0, 1, 0, 0, 0, 4096, 2771, 737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4095, 7, 0, 0, 0, 0, 2, 8, 4087, 1042, 8, 0, 0, 4053, 3809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 1, 1086, 1, 4006, 1, 4096, 21, 108, 3997, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3964, 0, 0, 0, 0, 1, 1, 18, 3467, 3683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 2464, 1, 0, 117, 13, 1362, 18, 3039, 4, 0, 4070, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15659, 16257, 18198, 16417, 17115, 17038, 15222, 15166] avg=16384.0000 max=18198 max_rank=2 min=15166 min_rank=7 imbalance_ratio=1.110718 max_load_reduction=112874 max_load_reduction_pct=0.861160 moved_to_echo=64541 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 49, 0, 0, 4088, 0, 0, 2747, 77, 0, 1063, 0, 0, 4096, 1, 20, 3297, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2874, 0, 0, 0, 670, 3821, 1, 547, 0, 909, 0, 50, 562, 3535, 4094, 3479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 0, 0, 4061, 193, 0, 150, 53, 0, 0, 0, 0, 0, 0, 4093, 0, 1630, 347, 297, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3344, 4069, 262, 27, 1, 0, 0, 0, 0, 0, 11, 14, 0, 5, 0, 4080, 2623, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3600, 143, 0, 0, 31, 2, 3, 0, 3547, 1, 87, 0, 0, 4074, 3834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 302, 7, 4, 1037, 0, 3823, 0, 4047, 31, 4020, 3895, 2659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 315, 0, 0, 2, 4062, 4073, 0, 194, 0, 459, 2, 34, 0, 3571, 3640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2466, 0, 0, 474, 0, 1281, 1, 2113, 0, 8, 2672, 0, 34, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15700, 20542, 14954, 15165, 15322, 19825, 16353, 13211] avg=16384.0000 max=20542 max_rank=1 min=13211 min_rank=7 imbalance_ratio=1.253784 max_load_reduction=110530 max_load_reduction_pct=0.843277 moved_to_echo=64711 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 78, 3282, 0, 126, 4096, 0, 0, 1317, 728, 13, 4095, 0, 0, 3288, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4079, 4063, 0, 572, 0, 66, 0, 0, 175, 3774, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 139, 0, 1307, 0, 1, 0, 4096, 2817, 1630, 356, 317, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 852, 4096, 4096, 0, 1, 0, 0, 937, 0, 0, 6, 444, 0, 0, 4, 4080, 2819, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 0, 1321, 4068, 0, 0, 0, 0, 3450, 3834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1115, 0, 4096, 0, 145, 0, 4096, 3631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 744, 7, 4096, 3974, 0, 0, 0, 80, 1004, 0, 0, 3694, 3733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 2466, 0, 0, 3, 0, 3528, 0, 2855, 0, 15, 4090, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17285, 12734, 18984, 18062, 16652, 13083, 17366, 16906] avg=16384.0000 max=18984 max_rank=2 min=12734 min_rank=1 imbalance_ratio=1.158691 max_load_reduction=112088 max_load_reduction_pct=0.855164 moved_to_echo=64007 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[420, 0, 1, 9, 3985, 0, 343, 4068, 0, 3, 0, 44, 92, 3744, 91, 0, 3344, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 13, 3542, 4067, 0, 0, 21, 102, 0, 18, 0, 3778, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3705, 0, 3, 4060, 128, 7, 0, 0, 2425, 1, 555, 0, 216, 0, 4077, 0, 1590, 357, 318, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 4041, 0, 0, 7, 0, 9, 8, 0, 0, 93, 0, 1, 0, 0, 4071, 2842, 741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3624, 0, 22, 0, 0, 0, 54, 0, 4068, 780, 0, 0, 3, 4092, 3794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 148, 0, 0, 6, 0, 0, 1124, 0, 3716, 0, 3975, 28, 0, 4086, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3454, 3284, 0, 78, 56, 8, 0, 0, 0, 4088, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 2406, 23, 0, 955, 0, 3027, 0, 2347, 0, 41, 4091, 0, 37, 0, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16403, 19614, 17572, 15853, 16437, 13111, 14707, 17375] avg=16384.0000 max=19614 max_rank=1 min=13111 min_rank=5 imbalance_ratio=1.197144 max_load_reduction=111458 max_load_reduction_pct=0.850357 moved_to_echo=61630 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 29952, 256, 256, 128, 128, 128, 128, 1408, 31744, 4864, 128, 384, 128, 31744, 30720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 384, 128, 896, 128, 31872, 29696, 128, 384, 384, 896, 6016, 128, 128, 31104, 29824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[8448, 0, 128, 128, 23296, 32512, 640, 1408, 128, 1280, 0, 128, 1920, 30080, 27520, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32256, 384, 512, 32640, 384, 128, 256, 128, 5632, 128, 4096, 0, 256, 128, 32768, 3072, 13056, 2944, 2560, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 256, 128, 0, 384, 128, 128, 8960, 128, 32000, 128, 28160, 128, 21632, 31104, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32640, 19712, 128, 128, 1664, 128, 24960, 128, 15360, 128, 5376, 31360, 128, 128, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[26240, 32640, 4864, 128, 128, 128, 4864, 1024, 128, 0, 128, 512, 128, 128, 128, 32768, 22528, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[512, 128, 128, 128, 31872, 512, 896, 30976, 256, 128, 3712, 1408, 256, 32384, 128, 128, 26752, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4082, 0, 236, 4095, 0, 0, 306, 103, 44, 0, 3757, 4073, 0, 0, 0, 0, 589, 0, 0, 260, 1, 0, 0, 281, 4096, 0, 4096, 3552, 3782, 2, 514, 0, 0, 0, 0, 0, 29, 31, 4045, 0, 3050, 0, 3386, 4095, 4039, 0, 12, 36, 0, 4035, 4075, 1, 38, 13, 0, 0, 0, 0, 91, 0, 4096, 3971, 0, 2166, 0, 0, 4081, 0, 18, 0, 41, 3463, 940, 0, 4057, 0, 0, 0, 0, 0, 0, 3295, 0, 0, 0, 509, 25, 74, 2085, 0, 0, 4096, 0, 0, 0, 4054, 4078, 12, 4063, 0, 4055, 0, 100, 526, 0, 0, 90, 0, 0, 0, 15, 0, 3, 4045, 0, 4096, 3981, 0, 4096, 0, 3829, 0, 253, 14, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[670, 1, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 94, 0, 4077, 4092, 0, 2, 13, 29, 1926, 0, 11, 36, 283, 0, 0, 0, 4088, 3, 3228, 4061, 4089, 0, 0, 3, 0, 0, 3678, 30, 0, 0, 4049, 0, 85, 11, 2, 4058, 3272, 3, 0, 0, 1, 4089, 4096, 4, 3009, 0, 0, 1, 214, 0, 40, 0, 4093, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3633, 4089, 0, 141, 222, 1, 0, 0, 22, 0, 815, 0, 0, 0, 546, 0, 59, 3426, 2554, 0, 3815, 0, 36, 0, 4094, 4094, 0, 4096, 2, 4096, 0, 5, 0, 0, 5, 0, 0, 0, 0, 1, 1, 5, 1075, 0, 4091, 4073, 0, 4096, 0, 4059, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4093, 0, 0, 1155, 0, 1, 0, 4096, 4096, 0, 0, 0, 965, 1, 0, 0, 0, 4, 0, 0, 60, 4096, 0, 4005, 4037, 4096, 0, 63, 0, 0, 0, 254, 0, 0, 0, 4092, 0, 1, 0, 579, 4093, 3540, 0, 0, 0, 0, 4096, 4096, 0, 137, 4088, 0, 0, 0, 10, 49, 0, 4096, 4096, 0, 4032, 0, 1, 4096, 0, 0, 0, 0, 3753, 4091, 0, 791, 0, 0, 0, 0, 0, 0, 569, 3, 0, 662, 0, 20, 0, 4090, 5, 0, 4096, 0, 10, 153, 4096, 4096, 0, 4095, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 4096, 1273, 0, 4096, 0, 4096, 0, 0, 0, 2600] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 2, 0, 4009, 0, 0, 4036, 0, 0, 0, 0, 3, 0, 4096, 4059, 6, 176, 0, 2913, 3780, 0, 2, 52, 10, 0, 0, 505, 4029, 0, 4071, 4041, 3524, 30, 3918, 30, 0, 0, 0, 0, 0, 0, 3671, 2, 0, 0, 0, 3622, 4096, 0, 23, 174, 0, 4048, 3996, 32, 1999, 4096, 2, 60, 0, 0, 0, 1, 4090, 3930, 0, 2311, 0, 84, 4096, 0, 314, 306, 0, 1351, 4096, 0, 1400, 90, 15, 32, 0, 2, 16, 0, 604, 466, 0, 0, 1, 0, 4086, 195, 0, 2761, 0, 6, 33, 4070, 4087, 0, 4095, 0, 2746, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 5, 577, 64, 4091, 3917, 7, 4089, 0, 3793, 0, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4, 12, 0, 4078, 57, 6, 4089, 0, 27, 140, 0, 0, 0, 3762, 4075, 18, 0, 2, 3164, 1850, 0, 0, 96, 1057, 0, 0, 1432, 4079, 0, 4088, 2428, 3593, 0, 0, 17, 1, 1, 2948, 0, 0, 3, 4095, 3, 6, 35, 157, 4068, 4080, 0, 1, 0, 8, 4092, 4078, 386, 459, 753, 0, 0, 0, 451, 312, 5, 4054, 3988, 0, 2, 0, 0, 3840, 0, 0, 0, 0, 3521, 3039, 12, 0, 28, 2, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 153, 0, 2921, 3942, 0, 4091, 6, 0, 0, 4093, 4094, 0, 4096, 8, 3571, 0, 468, 0, 1, 36, 0, 0, 4, 0, 0, 0, 0, 1119, 0, 4084, 3478, 0, 4096, 0, 4096, 0, 0, 0, 105] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 1, 4096, 0, 0, 2965, 30, 0, 0, 4096, 4096, 0, 0, 0, 3971, 840, 0, 0, 0, 163, 0, 0, 109, 4096, 0, 4096, 4094, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4078, 4096, 3998, 0, 0, 0, 0, 4096, 4096, 0, 350, 4052, 0, 0, 0, 0, 36, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 3395, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 119, 4096, 4096, 0, 4096, 0, 3977, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 4096, 4094, 0, 4068, 0, 4094, 0, 84, 0, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 5, 4096, 15, 0, 4096, 0, 0, 90, 77, 74, 0, 4096, 4096, 0, 0, 0, 4096, 3107, 192, 0, 0, 0, 0, 0, 4065, 4096, 0, 4096, 4089, 4068, 0, 832, 0, 0, 0, 541, 0, 0, 0, 145, 0, 990, 0, 7, 4096, 3925, 0, 0, 185, 0, 4096, 3996, 402, 4094, 148, 2, 0, 0, 33, 20, 0, 3996, 3967, 0, 0, 0, 0, 4096, 0, 81, 1896, 0, 3476, 2070, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4096, 0, 3066, 0, 0, 1, 4096, 4096, 0, 4013, 0, 3667, 247, 0, 2, 2, 11, 96, 0, 0, 0, 0, 7, 0, 4096, 0, 4096, 4061, 8, 4096, 0, 3756, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[80, 1, 0, 4063, 0, 0, 4073, 1, 4, 0, 0, 432, 0, 4095, 3765, 0, 86, 0, 3, 299, 0, 240, 32, 0, 0, 0, 2, 3986, 11, 3156, 4096, 4096, 24, 504, 27, 0, 0, 3850, 1036, 49, 0, 1, 0, 0, 0, 4089, 4024, 3902, 0, 0, 0, 111, 4095, 3977, 56, 81, 142, 0, 1, 0, 0, 76, 0, 4048, 2699, 2, 3799, 8, 10, 4096, 0, 0, 0, 0, 4045, 4096, 0, 3989, 0, 11, 1, 0, 1, 29, 319, 0, 2320, 0, 0, 0, 0, 4096, 0, 0, 3732, 0, 14, 3429, 3953, 3977, 0, 3981, 0, 4029, 3, 0, 66, 0, 10, 0, 15, 0, 6, 0, 1, 40, 2, 0, 4095, 4095, 45, 3595, 83, 3654, 111, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 2917, 0, 0, 1155, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 965, 1, 0, 0, 0, 4, 0, 0, 60, 228, 0, 4005, 4037, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 254, 0, 0, 0, 4092, 0, 1, 0, 579, 4093, 3540, 587, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 137, 4088, 0, 0, 0, 10, 49, 0, 4096, 589, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4032, 0, 1, 2678, 0, 0, 0, 0, 3753, 4091, 0, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 569, 3, 0, 662, 0, 20, 0, 4090, 5, 0, 4096, 0, 3868, 1657, 878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 153, 4096, 3790, 0, 4095, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 2439, 1273, 0, 4096, 0, 4096, 0, 0, 0, 2600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16361, 13396, 13749, 17467, 19442, 15848, 16241, 18568] avg=16384.0000 max=19442 max_rank=4 min=13396 min_rank=1 imbalance_ratio=1.186646 max_load_reduction=111630 max_load_reduction_pct=0.851669 moved_to_echo=70099 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4082, 0, 236, 2917, 0, 0, 306, 103, 44, 0, 3757, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 0, 260, 1, 0, 0, 281, 228, 0, 4096, 3552, 3782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 514, 0, 0, 0, 0, 0, 29, 31, 4045, 0, 3050, 0, 3386, 4095, 4039, 588, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 36, 0, 4035, 4075, 1, 38, 13, 0, 0, 0, 0, 91, 0, 4096, 590, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 2166, 0, 0, 2668, 0, 18, 0, 41, 3463, 940, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3295, 0, 0, 0, 509, 25, 74, 2085, 0, 0, 4096, 0, 3868, 1657, 875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 3774, 12, 4063, 0, 4055, 0, 100, 526, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3, 4045, 0, 2439, 3981, 0, 4096, 0, 3829, 0, 253, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15518, 12789, 20317, 13291, 17324, 16484, 16674, 18675] avg=16384.0000 max=20317 max_rank=2 min=12789 min_rank=1 imbalance_ratio=1.240051 max_load_reduction=110755 max_load_reduction_pct=0.844994 moved_to_echo=69157 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 1, 1, 2918, 0, 0, 2965, 30, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 840, 0, 0, 0, 163, 0, 0, 109, 228, 0, 4096, 4094, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4078, 4096, 3998, 588, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 350, 4052, 0, 0, 0, 0, 36, 0, 4096, 590, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2678, 0, 0, 2, 0, 3395, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 3868, 1657, 878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 4096, 3790, 0, 4096, 0, 3977, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 2439, 4094, 0, 4068, 0, 4094, 0, 84, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18203, 17565, 17396, 17622, 14268, 14674, 16136, 15208] avg=16384.0000 max=18203 max_rank=0 min=14268 min_rank=4 imbalance_ratio=1.111023 max_load_reduction=112869 max_load_reduction_pct=0.861122 moved_to_echo=60286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4, 12, 0, 4078, 57, 6, 2907, 0, 27, 140, 0, 0, 0, 3762, 4075, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3164, 1850, 0, 0, 96, 1057, 0, 0, 1432, 222, 0, 4088, 2428, 3593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 1, 2948, 0, 0, 3, 4095, 3, 6, 35, 157, 4068, 4080, 590, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 4092, 4078, 386, 459, 753, 0, 0, 0, 451, 312, 5, 4054, 592, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 0, 2, 0, 0, 2506, 0, 0, 0, 0, 3521, 3039, 12, 0, 28, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 153, 0, 2921, 3942, 0, 4091, 6, 3857, 1657, 826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3785, 0, 4096, 8, 3571, 0, 468, 0, 1, 36, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1119, 0, 2427, 3478, 0, 4096, 0, 4096, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15086, 17932, 16512, 15500, 13098, 21561, 16062, 15321] avg=16384.0000 max=21561 max_rank=5 min=13098 min_rank=4 imbalance_ratio=1.315979 max_load_reduction=109511 max_load_reduction_pct=0.835503 moved_to_echo=66042 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[670, 1, 0, 4096, 0, 0, 2917, 0, 0, 4096, 0, 94, 0, 4077, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 13, 29, 1926, 0, 11, 36, 283, 0, 0, 0, 227, 3, 3228, 4061, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3678, 30, 0, 0, 4049, 0, 85, 11, 2, 4058, 3272, 588, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 4089, 4096, 4, 3009, 0, 0, 1, 214, 0, 40, 0, 4093, 590, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2678, 0, 0, 0, 0, 3633, 4089, 0, 141, 222, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 815, 0, 0, 0, 546, 0, 59, 3426, 2554, 0, 3815, 0, 3861, 1655, 878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 4094, 3788, 0, 4096, 2, 4096, 0, 5, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5, 1075, 0, 2436, 4073, 0, 4096, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20043, 13908, 16316, 16446, 14860, 17631, 16122, 15746] avg=16384.0000 max=20043 max_rank=0 min=13908 min_rank=1 imbalance_ratio=1.223328 max_load_reduction=111029 max_load_reduction_pct=0.847084 moved_to_echo=64359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 5, 4096, 15, 0, 2918, 0, 0, 90, 77, 74, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3107, 192, 0, 0, 0, 0, 0, 4065, 228, 0, 4096, 4089, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 832, 0, 0, 0, 541, 0, 0, 0, 145, 0, 990, 0, 7, 4096, 3925, 588, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 4096, 3996, 402, 4094, 148, 2, 0, 0, 33, 20, 0, 3996, 590, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 0, 2678, 0, 81, 1896, 0, 3476, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4096, 0, 3066, 0, 3868, 1657, 878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 3790, 0, 4013, 0, 3667, 247, 0, 2, 2, 11, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 2439, 4061, 8, 4096, 0, 3756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15467, 23941, 11664, 17868, 14168, 13576, 15925, 18463] avg=16384.0000 max=23941 max_rank=1 min=11664 min_rank=2 imbalance_ratio=1.461243 max_load_reduction=107131 max_load_reduction_pct=0.817345 moved_to_echo=62132 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 4009, 0, 0, 2876, 0, 0, 0, 0, 3, 0, 4096, 4059, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 2913, 3780, 0, 2, 52, 10, 0, 0, 505, 224, 0, 4071, 4041, 3524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 3918, 30, 0, 0, 0, 0, 0, 0, 3671, 2, 0, 0, 0, 3622, 4096, 579, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 174, 0, 4048, 3996, 32, 1999, 4096, 2, 60, 0, 0, 0, 1, 4090, 581, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 0, 2311, 0, 84, 2678, 0, 314, 306, 0, 1351, 4096, 0, 1400, 90, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 2, 16, 0, 604, 466, 0, 0, 1, 0, 4086, 195, 0, 2761, 0, 3805, 1655, 878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 33, 4070, 3782, 0, 4095, 0, 2746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 5, 577, 64, 2436, 3917, 7, 4089, 0, 3793, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15051, 19298, 16488, 19407, 16575, 14501, 14732, 15020] avg=16384.0000 max=19407 max_rank=3 min=14501 min_rank=5 imbalance_ratio=1.184509 max_load_reduction=111665 max_load_reduction_pct=0.851936 moved_to_echo=60828 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[80, 1, 0, 4063, 0, 0, 2902, 1, 4, 0, 0, 432, 0, 4095, 3765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 3, 299, 0, 240, 32, 0, 0, 0, 2, 222, 11, 3156, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 504, 27, 0, 0, 3850, 1036, 49, 0, 1, 0, 0, 0, 4089, 4024, 3902, 584, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 4095, 3977, 56, 81, 142, 0, 1, 0, 0, 76, 0, 4048, 587, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2699, 2, 3799, 8, 10, 2678, 0, 0, 0, 0, 4045, 4096, 0, 3989, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 29, 319, 0, 2320, 0, 0, 0, 0, 4096, 0, 0, 3732, 0, 3764, 1657, 878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3429, 3953, 3680, 0, 3981, 0, 4029, 3, 0, 66, 0, 10, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 40, 2, 0, 2438, 4095, 45, 3595, 83, 3654, 111, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15343, 12243, 18630, 13471, 21337, 16797, 19180, 14071] avg=16384.0000 max=21337 max_rank=4 min=12243 min_rank=1 imbalance_ratio=1.302307 max_load_reduction=109735 max_load_reduction_pct=0.837212 moved_to_echo=71385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[768, 128, 128, 32640, 128, 256, 23296, 128, 128, 8832, 256, 768, 0, 32128, 32384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 512, 128, 32768, 32512, 896, 10240, 13312, 128, 128, 256, 512, 640, 128, 32640, 4736, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[30848, 128, 12416, 128, 128, 21248, 0, 512, 2304, 128, 26752, 26624, 128, 10496, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 5120, 640, 2816, 4864, 1152, 256, 256, 24832, 10880, 0, 29824, 128, 30848, 13312, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 128, 128, 128, 15488, 128, 19584, 29056, 128, 32256, 128, 31488, 128, 384, 128, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 5888, 128, 128, 128, 11392, 1152, 128, 128, 24320, 128, 4224, 128, 12416, 32256, 30976, 4736, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 3840, 32640, 30208, 128, 32640, 128, 30336, 256, 640, 640, 128, 128, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[384, 128, 15232, 12416, 256, 256, 512, 1536, 0, 0, 6528, 1920, 128, 30848, 30464, 31360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[34, 0, 3910, 2, 0, 0, 0, 0, 4096, 0, 87, 3849, 0, 0, 4007, 298, 48, 0, 0, 4096, 0, 1, 294, 0, 3594, 0, 0, 0, 0, 0, 40, 0, 0, 4094, 4096, 931, 4096, 0, 0, 0, 3551, 4063, 0, 1, 309, 2, 0, 212, 4095, 522, 0, 0, 0, 4073, 0, 4093, 0, 0, 26, 0, 0, 3929, 368, 0, 4096, 1199, 0, 1, 0, 5, 110, 4096, 0, 615, 0, 210, 0, 582, 3838, 0, 0, 4096, 4095, 4088, 49, 0, 0, 0, 94, 4045, 0, 0, 0, 4096, 4026, 0, 3326, 8, 4096, 3568, 0, 0, 0, 0, 0, 4089, 0, 0, 1, 0, 0, 0, 4096, 8, 23, 2, 0, 0, 1, 9, 0, 0, 1512, 0, 0, 4096, 0, 4079] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[627, 0, 623, 0, 0, 0, 0, 44, 4096, 0, 0, 1, 2, 0, 4096, 0, 1, 0, 0, 3494, 98, 0, 0, 0, 3754, 0, 0, 0, 0, 0, 4, 0, 0, 3485, 4096, 3966, 4095, 0, 0, 0, 240, 4096, 0, 3, 1201, 0, 0, 97, 4096, 37, 0, 0, 0, 4096, 0, 4096, 0, 90, 0, 0, 3641, 4096, 0, 2, 4096, 4096, 0, 1, 0, 0, 0, 4083, 0, 0, 0, 4021, 0, 117, 0, 0, 0, 4096, 3427, 4068, 0, 0, 0, 1, 17, 609, 0, 0, 0, 4096, 4081, 0, 3130, 0, 4081, 4063, 0, 0, 0, 0, 101, 4092, 15, 0, 0, 0, 4, 0, 4094, 0, 0, 0, 0, 0, 0, 236, 3892, 0, 0, 0, 4090, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[117, 67, 20, 0, 5, 0, 0, 9, 4096, 0, 0, 4072, 0, 0, 4081, 0, 82, 0, 0, 135, 105, 0, 0, 2, 4089, 0, 0, 0, 0, 5, 5, 0, 0, 247, 4096, 4096, 4018, 0, 4, 0, 4042, 4087, 0, 0, 21, 0, 21, 31, 3967, 0, 7, 0, 0, 4096, 0, 4045, 0, 0, 0, 0, 155, 4001, 0, 5, 4092, 3787, 0, 0, 0, 0, 1, 4080, 0, 0, 0, 7, 0, 24, 4096, 0, 0, 4096, 4096, 4096, 0, 2, 0, 1, 0, 4096, 0, 0, 0, 4090, 4095, 0, 3712, 0, 4023, 4096, 0, 0, 0, 0, 1, 4053, 90, 5, 0, 2, 367, 25, 3891, 0, 0, 20, 3461, 0, 0, 593, 34, 0, 0, 0, 4023, 4092, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 1, 0, 0, 0, 16, 4096, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4066, 0, 3, 7, 4095, 160, 0, 0, 0, 4096, 0, 4096, 14, 0, 0, 0, 151, 4096, 223, 9, 4096, 2447, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 50, 0, 36, 4095, 0, 0, 4095, 4096, 4094, 2, 0, 1, 0, 3, 4096, 0, 0, 0, 4096, 4096, 0, 1713, 0, 4096, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4063, 0, 0, 0, 4019, 2782, 0, 0, 491, 0, 3529, 0, 778, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1882, 54, 0, 0, 0, 68, 0, 427, 4088, 4, 0, 4096, 13, 0, 4095, 0, 0, 19, 0, 83, 0, 0, 698, 13, 4095, 8, 0, 0, 56, 88, 0, 5, 2, 0, 4063, 4077, 4052, 0, 2, 33, 3009, 4037, 0, 0, 189, 0, 38, 0, 4087, 0, 0, 8, 4, 4091, 5, 4087, 2, 0, 0, 11, 0, 4015, 6, 0, 4091, 4094, 0, 0, 0, 0, 60, 3594, 7, 0, 0, 3198, 0, 483, 4088, 1, 21, 4086, 4067, 4096, 0, 0, 0, 0, 0, 2844, 1, 0, 0, 4079, 4061, 0, 65, 214, 4059, 4093, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 1, 41, 0, 3789, 0, 0, 6, 46, 24, 684, 0, 4093, 0, 13, 0, 3353, 4005, 2, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 43, 0, 0, 0, 0, 0, 4096, 0, 0, 1433, 0, 0, 4096, 0, 1569, 0, 0, 2383, 0, 0, 1, 0, 4081, 0, 1, 0, 0, 0, 0, 0, 0, 1, 4096, 3976, 4096, 0, 0, 0, 4069, 4096, 0, 0, 14, 0, 36, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 9, 3922, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 49, 0, 3631, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 4096, 5, 3912, 128, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 27, 0, 22, 0, 431, 0, 3039, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[109, 0, 1506, 87, 0, 0, 0, 476, 4096, 0, 0, 4093, 0, 0, 3728, 0, 0, 0, 0, 2283, 0, 59, 0, 0, 3972, 0, 0, 7, 0, 0, 11, 0, 0, 1156, 3076, 4095, 3999, 8, 0, 0, 616, 3996, 0, 12, 2852, 79, 136, 365, 4021, 0, 0, 0, 131, 4089, 13, 3789, 14, 0, 27, 0, 963, 4096, 21, 17, 4094, 4094, 2, 19, 0, 205, 0, 4068, 0, 4, 0, 1, 0, 1678, 3578, 0, 0, 3936, 4045, 1806, 0, 0, 0, 0, 0, 2532, 0, 51, 0, 4083, 4012, 2, 397, 0, 3787, 3760, 1, 1, 0, 0, 0, 3611, 679, 23, 0, 0, 4, 0, 3960, 0, 96, 0, 3090, 3227, 0, 0, 4068, 0, 0, 14, 2, 4062, 0, 4082] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 84, 0, 0, 115, 0, 0, 4078, 9, 0, 3837, 0, 2, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4069, 0, 79, 0, 5, 80, 5, 0, 0, 271, 4031, 2200, 4078, 418, 90, 0, 3451, 4003, 0, 29, 146, 0, 4, 24, 4096, 3, 0, 0, 0, 4054, 0, 4095, 0, 5, 0, 0, 3258, 3765, 220, 3, 4078, 224, 0, 0, 0, 7, 0, 3974, 42, 0, 0, 3917, 6, 265, 3969, 0, 0, 4053, 4096, 3024, 0, 7, 0, 0, 659, 3459, 0, 0, 7, 4085, 4096, 0, 2884, 123, 4061, 3457, 0, 12, 52, 2, 0, 4093, 0, 0, 30, 1, 0, 1662, 4052, 559, 3, 0, 3805, 0, 2075, 0, 0, 0, 1458, 41, 107, 3994, 16, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[627, 0, 623, 0, 0, 0, 0, 44, 4096, 0, 0, 1, 2, 0, 4096, 0, 2773, 980, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3494, 98, 0, 0, 0, 3754, 0, 0, 0, 0, 0, 4, 0, 4096, 4095, 2278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3485, 4096, 3966, 0, 0, 0, 0, 240, 3682, 0, 3, 1201, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 37, 0, 0, 0, 3017, 0, 4096, 0, 90, 0, 0, 3641, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3484, 4096, 0, 1, 0, 0, 0, 4083, 0, 0, 0, 4021, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3427, 4068, 0, 0, 0, 1, 17, 609, 0, 0, 0, 0, 1308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3130, 0, 4081, 4063, 0, 0, 0, 0, 101, 4092, 15, 0, 0, 0, 4, 0, 612, 561, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 236, 3892, 0, 0, 0, 4090, 4096, 0, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13760, 17820, 16770, 19075, 15802, 13526, 17073, 17246] avg=16384.0000 max=19075 max_rank=3 min=13526 min_rank=5 imbalance_ratio=1.164246 max_load_reduction=111997 max_load_reduction_pct=0.854469 moved_to_echo=63647 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 8, 0, 0, 4096, 0, 2783, 980, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 1, 0, 0, 4096, 4096, 2278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 4096, 0, 0, 0, 0, 4096, 3682, 0, 0, 4066, 0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 160, 0, 0, 0, 3017, 0, 4096, 14, 0, 0, 0, 151, 4096, 223, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3484, 2447, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 50, 0, 36, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4094, 2, 0, 1, 0, 3, 4096, 0, 0, 0, 0, 1313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1713, 0, 4096, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 612, 561, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 4019, 2782, 0, 0, 491, 0, 3529, 0, 778, 4096, 0, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12482, 14576, 20062, 15861, 14208, 17700, 15587, 20596] avg=16384.0000 max=20596 max_rank=7 min=12482 min_rank=0 imbalance_ratio=1.257080 max_load_reduction=110476 max_load_reduction_pct=0.842865 moved_to_echo=68091 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[117, 67, 20, 0, 5, 0, 0, 9, 4096, 0, 0, 4072, 0, 0, 4081, 0, 2782, 980, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 135, 105, 0, 0, 2, 4089, 0, 0, 0, 0, 5, 5, 0, 4090, 4018, 2278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 4096, 4096, 0, 0, 4, 0, 4042, 3674, 0, 0, 21, 0, 21, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 7, 0, 0, 3017, 0, 4045, 0, 0, 0, 0, 155, 4001, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3481, 3787, 0, 0, 0, 0, 1, 4080, 0, 0, 0, 7, 0, 24, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 2, 0, 1, 0, 4096, 0, 0, 0, 0, 1313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3712, 0, 4023, 4096, 0, 0, 0, 0, 1, 4053, 90, 5, 0, 2, 367, 25, 611, 561, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3891, 0, 0, 20, 3461, 0, 0, 593, 34, 0, 0, 0, 4023, 4092, 0, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16747, 14809, 16232, 15197, 15476, 17700, 17959, 16952] avg=16384.0000 max=17959 max_rank=6 min=14809 min_rank=1 imbalance_ratio=1.096130 max_load_reduction=113113 max_load_reduction_pct=0.862984 moved_to_echo=68087 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1882, 54, 0, 0, 0, 68, 0, 427, 4088, 4, 0, 4096, 13, 0, 4095, 0, 2762, 979, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 83, 0, 0, 698, 13, 4095, 8, 0, 0, 56, 88, 0, 5, 4079, 4052, 2276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4063, 4077, 0, 0, 2, 33, 3009, 3624, 0, 0, 189, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 8, 4, 3006, 5, 4087, 2, 0, 0, 11, 0, 4015, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3476, 4094, 0, 0, 0, 0, 60, 3594, 7, 0, 0, 3198, 0, 483, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4086, 4067, 4096, 0, 0, 0, 0, 0, 2844, 1, 0, 0, 0, 1299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 214, 4059, 4093, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 1, 41, 0, 615, 565, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3789, 0, 0, 6, 46, 24, 684, 0, 4093, 0, 13, 0, 3353, 4005, 2, 829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18988, 15472, 15037, 15231, 19001, 16414, 14085, 16844] avg=16384.0000 max=19001 max_rank=4 min=14085 min_rank=6 imbalance_ratio=1.159729 max_load_reduction=112071 max_load_reduction_pct=0.855034 moved_to_echo=66344 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[34, 0, 3910, 2, 0, 0, 0, 0, 4096, 0, 87, 3849, 0, 0, 4007, 298, 2735, 976, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 4096, 0, 1, 294, 0, 3594, 0, 0, 0, 0, 0, 40, 0, 4096, 4096, 2268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 931, 0, 0, 0, 0, 3551, 3652, 0, 1, 309, 2, 0, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 522, 0, 0, 0, 3000, 0, 4093, 0, 0, 26, 0, 0, 3929, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3484, 1199, 0, 1, 0, 5, 110, 4096, 0, 615, 0, 210, 0, 582, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4088, 49, 0, 0, 0, 94, 4045, 0, 0, 0, 0, 1291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3326, 8, 4096, 3568, 0, 0, 0, 0, 0, 4089, 0, 0, 1, 0, 0, 0, 612, 558, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 23, 2, 0, 0, 1, 9, 0, 0, 1512, 0, 0, 4096, 0, 835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20509, 18533, 16848, 16033, 14140, 17758, 16669, 10582] avg=16384.0000 max=20509 max_rank=0 min=10582 min_rank=7 imbalance_ratio=1.251770 max_load_reduction=110563 max_load_reduction_pct=0.843529 moved_to_echo=59149 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 43, 0, 0, 0, 0, 0, 4096, 0, 0, 1433, 0, 0, 4096, 0, 2783, 980, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1569, 0, 0, 2383, 0, 0, 1, 0, 4081, 0, 1, 0, 0, 0, 0, 0, 4096, 4096, 2278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 3976, 0, 0, 0, 0, 4069, 3682, 0, 0, 14, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3017, 0, 4096, 0, 0, 0, 0, 9, 3922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3484, 4088, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 49, 0, 3631, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 1313, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 128, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 612, 561, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 27, 0, 22, 0, 431, 0, 3039, 4096, 0, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13949, 18505, 15874, 15140, 19440, 17698, 17915, 12551] avg=16384.0000 max=19440 max_rank=4 min=12551 min_rank=7 imbalance_ratio=1.186523 max_load_reduction=111632 max_load_reduction_pct=0.851685 moved_to_echo=67604 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[109, 0, 1506, 87, 0, 0, 0, 476, 4096, 0, 0, 4093, 0, 0, 3728, 0, 2726, 976, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2283, 0, 59, 0, 0, 3972, 0, 0, 7, 0, 0, 11, 0, 4083, 3999, 2270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1156, 3076, 4095, 0, 8, 0, 0, 616, 3592, 0, 12, 2852, 79, 136, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 131, 3012, 13, 3789, 14, 0, 27, 0, 963, 4096, 21, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3483, 4094, 2, 19, 0, 205, 0, 4068, 0, 4, 0, 1, 0, 1678, 3578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 4045, 1806, 0, 0, 0, 0, 0, 2532, 0, 51, 0, 0, 1286, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 0, 3787, 3760, 1, 1, 0, 0, 0, 3611, 679, 23, 0, 0, 4, 0, 611, 560, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 0, 96, 0, 3090, 3227, 0, 0, 4068, 0, 0, 14, 2, 4062, 0, 836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18314, 16684, 15987, 16104, 17132, 13658, 13838, 19355] avg=16384.0000 max=19355 max_rank=7 min=13658 min_rank=5 imbalance_ratio=1.181335 max_load_reduction=111717 max_load_reduction_pct=0.852333 moved_to_echo=63983 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 84, 0, 0, 115, 0, 0, 4078, 9, 0, 3837, 0, 2, 3925, 0, 2783, 977, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4069, 0, 79, 0, 5, 80, 5, 0, 4085, 4078, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 4031, 2200, 0, 418, 90, 0, 3451, 3598, 0, 29, 146, 0, 4, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 2986, 0, 4095, 0, 5, 0, 0, 3258, 3765, 220, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3469, 224, 0, 0, 0, 7, 0, 3974, 42, 0, 0, 3917, 6, 265, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 4096, 3024, 0, 7, 0, 0, 659, 3459, 0, 0, 7, 0, 1313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2884, 123, 4061, 3457, 0, 12, 52, 2, 0, 4093, 0, 0, 30, 1, 0, 1662, 609, 555, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 559, 3, 0, 3805, 0, 2075, 0, 0, 0, 1458, 41, 107, 3994, 16, 836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16323, 14673, 14262, 18431, 15873, 16618, 17946, 16946] avg=16384.0000 max=18431 max_rank=3 min=14262 min_rank=2 imbalance_ratio=1.124939 max_load_reduction=112641 max_load_reduction_pct=0.859383 moved_to_echo=67383 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[27904, 24064, 128, 128, 0, 256, 256, 32128, 128, 640, 0, 11520, 128, 6912, 27776, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32640, 768, 128, 128, 256, 24192, 128, 32512, 128, 128, 128, 128, 8192, 32000, 896, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[2816, 128, 6272, 128, 128, 256, 128, 1024, 32768, 128, 128, 21504, 128, 128, 32128, 384, 22144, 7936, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 9344, 31744, 27520, 0, 512, 128, 128, 23168, 29312, 0, 128, 8832, 128, 256, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[19200, 512, 32384, 31232, 128, 128, 128, 128, 128, 32256, 896, 128, 128, 128, 512, 1792, 4992, 4608, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32128, 640, 128, 128, 14464, 6144, 2816, 896, 12672, 0, 7040, 128, 15488, 32640, 128, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32640, 32128, 29440, 128, 128, 128, 128, 896, 25856, 128, 128, 128, 0, 10496, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[1792, 128, 0, 12544, 256, 128, 1024, 128, 31872, 128, 128, 128, 128, 256, 128, 128, 32768, 32640, 18304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 4096, 0, 4096, 0, 0, 0, 4035, 0, 4096, 0, 23, 365, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 609, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3401, 0, 0, 3827, 0, 0, 206, 1, 58, 6, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 3780, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4014, 3771, 0, 0, 0, 4096, 0, 0, 2, 4085, 0, 0, 0, 218, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 3992, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4086, 10, 4060, 0, 0, 0, 8, 1, 4096, 0, 3079, 0, 4, 3927, 0, 4094, 4086, 0, 0, 640, 0, 4096, 0, 4084, 0, 0, 1441, 8, 0, 0, 1, 0, 0, 0, 558, 2204, 0, 0, 486, 2, 0, 24, 4, 166, 1584, 3830, 0, 4096, 0, 0, 199, 0, 7, 37, 0, 96, 0, 0, 4093, 0, 145, 4063, 0, 0, 4096, 5, 0, 359, 0, 3, 66, 0, 0, 3977, 0, 0, 0, 0, 2339, 0, 0, 4095, 4058, 0, 0, 0, 4033, 4096, 0, 0, 3262, 529, 0, 0, 0, 4096, 0, 0, 0, 729, 0, 0, 0, 2910, 3820, 46, 0, 3923, 0, 0, 4018, 0, 0, 1003, 4071, 4096, 0, 0, 0, 0, 0, 3933, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 262, 0, 4096, 0, 3976, 4, 91, 4096, 0, 4096, 4096, 0, 0, 41, 0, 4096, 0, 4096, 0, 0, 5, 0, 0, 0, 87, 0, 0, 0, 2491, 3, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4, 0, 0, 0, 0, 13, 38, 0, 0, 4096, 0, 0, 4092, 0, 0, 6, 0, 0, 40, 2790, 69, 11, 0, 0, 4096, 0, 0, 0, 3, 4096, 0, 0, 4096, 3465, 0, 0, 0, 4096, 4096, 0, 0, 4051, 4088, 0, 0, 0, 4096, 0, 0, 13, 2052, 0, 0, 0, 4085, 4096, 0, 12, 4096, 0, 0, 4096, 0, 0, 0, 4085, 4096, 0, 0, 0, 0, 0, 981, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 3448, 0, 767, 4087, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 5, 3773, 0, 4087, 0, 4096, 3658, 0, 0, 30, 0, 3593, 0, 2511, 0, 0, 0, 527, 0, 0, 1737, 0, 0, 0, 0, 3921, 0, 0, 180, 0, 0, 53, 0, 0, 0, 4087, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 467, 4096, 0, 0, 475, 0, 0, 66, 411, 254, 15, 0, 0, 4004, 0, 0, 0, 2, 4096, 0, 0, 3452, 453, 0, 0, 0, 4094, 4096, 0, 0, 4096, 3620, 0, 0, 4, 4095, 1, 0, 85, 3941, 7, 0, 0, 4078, 4024, 0, 0, 3976, 0, 0, 4060, 0, 9, 0, 56, 4096, 0, 0, 0, 0, 0, 3805, 3, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 3022, 41, 0, 4031, 0, 4091, 0, 0, 0, 6, 0, 4043, 0, 6, 15, 128, 4089, 0, 3926, 4070, 0, 11, 1044, 0, 3635, 0, 4094, 1, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 442, 48, 0, 0, 0, 31, 2, 3990, 0, 3985, 138, 0, 229, 225, 0, 0, 0, 0, 0, 18, 4010, 1, 0, 4008, 0, 20, 3921, 0, 0, 0, 139, 0, 0, 1, 101, 3928, 23, 0, 0, 28, 4096, 0, 0, 3954, 3770, 3, 1, 0, 3828, 3907, 0, 0, 4096, 4095, 2, 4, 0, 4095, 0, 0, 4096, 4095, 27, 27, 153, 4072, 4096, 0, 0, 4076, 43, 0, 4093, 29, 161, 0, 4096, 4043, 0, 0, 78, 0, 38, 54, 248, 4082] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 4093, 7, 4096, 0, 0, 3, 1342, 0, 4096, 0, 3787, 5, 121, 4096, 0, 4096, 4085, 0, 0, 0, 0, 4020, 0, 4068, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 796, 4094, 0, 0, 22, 0, 0, 66, 1, 2, 0, 4065, 0, 4094, 0, 0, 0, 0, 0, 12, 11, 242, 0, 0, 4095, 0, 23, 4051, 0, 77, 1750, 2, 0, 2, 4058, 83, 0, 0, 0, 4086, 0, 0, 0, 0, 3727, 0, 0, 3926, 367, 0, 0, 0, 4092, 4096, 0, 0, 668, 3180, 30, 0, 0, 4096, 0, 0, 806, 4096, 0, 0, 8, 3974, 4096, 0, 0, 4092, 0, 11, 4096, 0, 2, 3, 3809, 4096, 0, 0, 0, 2, 1, 30, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4062, 0, 0, 4085, 0, 4094, 0, 0, 0, 2985, 6, 4029, 0, 4040, 2955, 0, 4096, 26, 4065, 4038, 0, 8, 0, 4, 4096, 30, 4086, 3, 1, 80, 0, 0, 0, 150, 0, 0, 0, 0, 9, 0, 12, 0, 1, 0, 9, 15, 0, 0, 4070, 83, 4082, 0, 45, 0, 0, 0, 1, 0, 93, 0, 0, 4049, 0, 0, 3993, 68, 0, 2, 0, 2, 701, 0, 97, 24, 2, 0, 4095, 0, 0, 0, 53, 3896, 0, 6, 4096, 3966, 16, 1, 0, 4058, 4072, 17, 56, 32, 4002, 1, 4, 0, 4057, 0, 0, 0, 1484, 0, 4, 45, 4096, 4090, 0, 1, 4088, 1, 0, 4092, 0, 0, 797, 4080, 4080, 78, 0, 41, 15, 3, 3260, 0, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 3887, 4, 0, 4067, 0, 4092, 4016, 88, 12, 32, 0, 4062, 0, 4096, 82, 0, 4002, 1, 4096, 3962, 0, 0, 0, 0, 3936, 0, 4002, 0, 13, 683, 4, 8, 2, 0, 0, 0, 6, 0, 3960, 0, 0, 273, 1, 11, 0, 5, 0, 256, 3922, 0, 3651, 6, 30, 0, 94, 0, 0, 0, 58, 0, 0, 4073, 0, 0, 4096, 0, 109, 0, 20, 1, 1206, 0, 4094, 282, 39, 0, 4075, 0, 0, 0, 0, 4054, 0, 53, 4093, 3805, 0, 0, 0, 4096, 3789, 0, 0, 4096, 974, 0, 0, 129, 3960, 0, 0, 23, 18, 0, 0, 0, 2647, 3998, 18, 0, 4096, 1, 39, 4086, 3, 0, 1290, 1447, 4071, 1, 0, 0, 1, 0, 675, 118, 4076] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 2977, 0, 0, 0, 0, 0, 4035, 0, 4096, 0, 23, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3950, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 609, 375, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1684, 1119, 433, 432, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3401, 0, 0, 3827, 0, 0, 206, 1, 58, 6, 0, 0, 4096, 0, 0, 2760, 1880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3347, 4096, 0, 0, 0, 0, 0, 0, 0, 4014, 3771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1784, 0, 0, 2, 4085, 0, 0, 0, 218, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 3992, 4096, 0, 0, 0, 0, 0, 4096, 0, 2412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15593, 16237, 14608, 16101, 16235, 19324, 14281, 18693] avg=16384.0000 max=19324 max_rank=5 min=14281 min_rank=6 imbalance_ratio=1.179443 max_load_reduction=111748 max_load_reduction_pct=0.852570 moved_to_echo=68533 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2970, 10, 0, 0, 0, 0, 8, 1, 4096, 0, 3079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3787, 0, 0, 4086, 0, 0, 640, 0, 4096, 0, 4084, 0, 0, 1441, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 558, 2204, 0, 0, 486, 2, 0, 24, 4, 166, 4060, 4094, 4033, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1584, 3830, 0, 4096, 0, 0, 199, 0, 7, 37, 0, 96, 0, 0, 4093, 0, 1684, 1116, 469, 432, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 4063, 0, 0, 4096, 5, 0, 359, 0, 3, 66, 0, 0, 3977, 0, 0, 2760, 1880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2339, 0, 0, 3626, 4058, 0, 0, 0, 0, 0, 0, 0, 3262, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1784, 0, 0, 0, 729, 0, 0, 0, 2910, 3820, 46, 0, 3923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 1003, 4071, 4096, 0, 0, 0, 0, 0, 3933, 2, 2412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14260, 18146, 16968, 17783, 17354, 13814, 13212, 19535] avg=16384.0000 max=19535 max_rank=7 min=13212 min_rank=6 imbalance_ratio=1.192322 max_load_reduction=111537 max_load_reduction_pct=0.850960 moved_to_echo=63915 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 2977, 0, 0, 0, 0, 0, 262, 0, 4096, 0, 3976, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 3950, 0, 0, 4096, 0, 0, 41, 0, 4096, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 2491, 3, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 4096, 4096, 1337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4, 0, 0, 0, 0, 13, 38, 0, 0, 4096, 0, 1684, 1119, 469, 432, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 6, 0, 0, 40, 2790, 69, 11, 0, 0, 4096, 0, 0, 2759, 1880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 3627, 3465, 0, 0, 0, 0, 0, 0, 0, 4051, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1784, 0, 0, 13, 2052, 0, 0, 0, 4085, 4096, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4085, 4096, 0, 0, 0, 0, 0, 981, 0, 2412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15411, 16375, 16212, 16193, 15743, 19330, 16138, 15670] avg=16384.0000 max=19330 max_rank=5 min=15411 min_rank=0 imbalance_ratio=1.179810 max_load_reduction=111742 max_load_reduction_pct=0.852524 moved_to_echo=66881 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4062, 0, 0, 2965, 0, 0, 0, 0, 0, 2985, 6, 4029, 0, 4040, 2955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 26, 0, 4038, 0, 8, 0, 4, 4096, 30, 4086, 3, 1, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 0, 0, 0, 0, 9, 0, 12, 0, 1, 0, 9, 15, 0, 4094, 4065, 4058, 1329, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 83, 4082, 0, 45, 0, 0, 0, 1, 0, 93, 0, 0, 4049, 0, 1686, 1120, 474, 431, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 68, 0, 2, 0, 2, 701, 0, 97, 24, 2, 0, 4095, 0, 0, 2743, 1868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 3896, 0, 6, 3622, 3966, 16, 1, 0, 0, 0, 17, 56, 32, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 1758, 0, 0, 0, 1484, 0, 4, 45, 4096, 4090, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 0, 0, 797, 4080, 4080, 78, 0, 41, 15, 3, 3260, 0, 2406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21042, 16318, 13742, 16284, 13595, 15667, 15571, 18853] avg=16384.0000 max=21042 max_rank=0 min=13595 min_rank=4 imbalance_ratio=1.284302 max_load_reduction=110030 max_load_reduction_pct=0.839462 moved_to_echo=63686 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 2975, 7, 0, 0, 0, 3, 1342, 0, 4096, 0, 3787, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 3950, 0, 0, 4085, 0, 0, 0, 0, 4020, 0, 4068, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 796, 4094, 0, 0, 22, 0, 0, 66, 1, 2, 4096, 4096, 4092, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 4094, 0, 0, 0, 0, 0, 12, 11, 242, 0, 0, 4095, 0, 1683, 1118, 450, 432, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4051, 0, 77, 1750, 2, 0, 2, 4058, 83, 0, 0, 0, 4086, 0, 0, 2760, 1880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3727, 0, 0, 3476, 367, 0, 0, 0, 0, 0, 0, 0, 668, 3180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 1784, 0, 0, 806, 4096, 0, 0, 8, 3974, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 2, 3, 3809, 4096, 0, 0, 0, 2, 1, 30, 0, 2411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16312, 16274, 18601, 16348, 18772, 11418, 18886, 14461] avg=16384.0000 max=18886 max_rank=6 min=11418 min_rank=5 imbalance_ratio=1.152710 max_load_reduction=112186 max_load_reduction_pct=0.855911 moved_to_echo=63537 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3022, 41, 0, 2930, 0, 0, 0, 0, 0, 6, 0, 4043, 0, 6, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 3944, 0, 0, 4070, 0, 11, 1044, 0, 3635, 0, 4094, 1, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 442, 48, 0, 0, 0, 31, 4091, 3926, 3828, 1275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3990, 0, 3985, 138, 0, 229, 225, 0, 0, 0, 0, 0, 18, 4010, 1, 1678, 1101, 453, 432, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 20, 3921, 0, 0, 0, 139, 0, 0, 1, 101, 3928, 23, 0, 2632, 1880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4096, 0, 0, 3501, 3770, 3, 1, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 1783, 0, 0, 4096, 4095, 27, 27, 153, 4072, 4096, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 4093, 29, 161, 0, 4096, 4043, 0, 0, 78, 0, 38, 54, 248, 2404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10063, 17000, 13641, 16407, 16653, 19590, 22431, 15287] avg=16384.0000 max=22431 max_rank=6 min=10063 min_rank=0 imbalance_ratio=1.369080 max_load_reduction=108641 max_load_reduction_pct=0.828865 moved_to_echo=73961 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3887, 4, 0, 2956, 0, 0, 4016, 88, 12, 32, 0, 4062, 0, 4096, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3860, 1, 0, 3962, 0, 0, 0, 0, 3936, 0, 4002, 0, 13, 683, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2, 0, 0, 0, 6, 0, 3960, 0, 0, 273, 1, 11, 0, 5, 0, 4092, 4096, 4096, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 3922, 0, 3651, 6, 30, 0, 94, 0, 0, 0, 58, 0, 0, 4073, 0, 1676, 1111, 469, 417, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 109, 0, 20, 1, 1206, 0, 4094, 282, 39, 0, 4075, 0, 0, 2553, 1818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 53, 3624, 3805, 0, 0, 0, 0, 0, 0, 0, 4096, 974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 1725, 0, 0, 23, 18, 0, 0, 0, 2647, 3998, 18, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 39, 4086, 3, 0, 1290, 1447, 4071, 1, 0, 0, 1, 0, 675, 118, 2400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19235, 16461, 17786, 15905, 18293, 16606, 12654, 14132] avg=16384.0000 max=19235 max_rank=0 min=12654 min_rank=6 imbalance_ratio=1.174011 max_load_reduction=111837 max_load_reduction_pct=0.853249 moved_to_echo=61685 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3448, 0, 767, 2971, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 5, 3773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3942, 0, 0, 3658, 0, 0, 30, 0, 3593, 0, 2511, 0, 0, 0, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1737, 0, 0, 0, 0, 3921, 0, 0, 180, 0, 0, 53, 0, 0, 4096, 4096, 4094, 1337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1684, 1116, 396, 432, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 4096, 0, 0, 475, 0, 0, 66, 411, 254, 15, 0, 0, 4004, 0, 0, 2759, 1880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 3056, 453, 0, 0, 0, 0, 0, 0, 0, 4096, 3620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1783, 1, 0, 85, 3941, 7, 0, 0, 4078, 4024, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 9, 0, 56, 4096, 0, 0, 0, 0, 0, 3805, 3, 2412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19156, 14261, 19514, 16051, 14427, 15323, 17899, 14441] avg=16384.0000 max=19514 max_rank=2 min=14261 min_rank=1 imbalance_ratio=1.191040 max_load_reduction=111558 max_load_reduction_pct=0.851120 moved_to_echo=62090 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 30464, 0, 128, 27904, 24064, 128, 128, 0, 0, 0, 128, 128, 24320, 24320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 30848, 128, 768, 23808, 128, 0, 4096, 128, 128, 12800, 128, 32640, 0, 19072, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 256, 14208, 128, 0, 5120, 20608, 128, 128, 256, 26112, 32384, 128, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[384, 31360, 128, 0, 32128, 0, 128, 1792, 128, 31616, 128, 31104, 128, 128, 2304, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[640, 31872, 128, 256, 14080, 128, 128, 2688, 7424, 4736, 512, 128, 128, 32384, 128, 0, 21760, 14976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 32640, 128, 256, 3200, 25728, 32768, 128, 0, 128, 128, 128, 16896, 384, 19328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 2048, 0, 0, 128, 4480, 14592, 0, 128, 1536, 128, 128, 256, 128, 256, 32768, 32640, 32512, 10624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[1920, 32256, 128, 32256, 256, 128, 512, 384, 128, 128, 128, 640, 0, 128, 32640, 128, 13568, 8960, 3712, 3456, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4032, 0, 4095, 0, 1, 4096, 0, 0, 0, 4039, 0, 0, 0, 0, 44, 0, 0, 0, 1, 7, 0, 0, 0, 4075, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 393, 4096, 0, 0, 4096, 0, 0, 0, 2611, 0, 0, 0, 0, 2555, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 148, 0, 0, 4096, 0, 5, 0, 3218, 4096, 0, 0, 0, 2, 4096, 0, 0, 2, 0, 1, 0, 0, 4071, 0, 0, 4096, 38, 5, 0, 0, 4064, 0, 1, 4096, 0, 4096, 4096, 0, 0, 0, 3886, 0, 0, 4096, 4096, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 119, 0, 0, 4088, 0, 0, 3560, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 10, 3976, 0, 0, 0, 48, 3843, 0, 0, 22, 25, 0, 0, 0, 0, 3524, 0, 0, 0, 0, 4070, 0, 0, 4096, 27, 4035, 4053, 0, 0, 4094, 30, 0, 0, 7, 0, 0, 0, 0, 100, 228, 0, 4094, 6, 3, 0, 0, 4096, 0, 0, 1, 2, 4096, 11, 0, 0, 4044, 4062, 294, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 135, 4020, 0, 0, 4096, 0, 363, 0, 80, 0, 9, 51, 3933, 0, 4065, 3755, 0, 0, 0, 1, 0, 28, 4087, 4086, 1, 0, 3924, 4096, 0, 1, 0, 0, 137, 0, 222, 17, 0, 4096, 0, 0, 3923, 4018, 0, 171, 0, 174, 0, 35, 0, 4078, 4096, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[2040, 0, 3978, 0, 0, 4088, 45, 29, 0, 923, 0, 269, 0, 23, 0, 0, 0, 1, 0, 704, 1, 0, 129, 4095, 87, 33, 44, 0, 3794, 0, 24, 4095, 0, 4094, 4060, 0, 2, 4067, 1, 0, 0, 347, 52, 0, 0, 3, 2960, 0, 2, 3911, 0, 0, 7, 1, 4045, 15, 0, 0, 16, 4042, 0, 0, 3, 4045, 4045, 0, 0, 3, 0, 4069, 8, 0, 591, 0, 43, 0, 138, 4064, 15, 0, 4093, 124, 3457, 2, 0, 1374, 0, 0, 4075, 28, 4036, 4030, 3, 76, 0, 3779, 0, 0, 4096, 3973, 4, 0, 4066, 3899, 0, 0, 276, 0, 0, 0, 1773, 0, 3, 4071, 0, 0, 0, 4000, 0, 0, 0, 0, 0, 15, 0, 4092, 2595, 4081] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[3834, 0, 3969, 2, 0, 4090, 15, 0, 0, 1814, 0, 168, 0, 19, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4043, 10, 0, 0, 0, 4091, 0, 0, 4095, 0, 4032, 4092, 66, 0, 4082, 45, 0, 4, 2050, 24, 0, 445, 0, 3711, 0, 0, 14, 1, 4, 52, 0, 4038, 0, 62, 33, 0, 4096, 0, 0, 0, 4094, 4087, 0, 0, 0, 0, 4071, 0, 0, 0, 1, 5, 5, 0, 3998, 0, 0, 4088, 9, 4090, 0, 0, 4096, 0, 105, 4096, 0, 4087, 4095, 0, 0, 0, 1378, 0, 0, 4096, 4093, 0, 0, 4096, 1937, 0, 0, 0, 0, 1, 1, 134, 0, 0, 4096, 0, 5, 892, 4086, 0, 638, 0, 0, 2, 9, 37, 4092, 3452, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[2417, 0, 4096, 56, 0, 4060, 0, 0, 0, 3236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4063, 4096, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 3913, 0, 0, 707, 0, 31, 0, 1, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 3204, 4096, 0, 0, 0, 0, 4096, 0, 0, 127, 0, 0, 0, 0, 4096, 0, 0, 4096, 716, 487, 1, 18, 4020, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 3670, 0, 0, 4096, 4068, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 170, 0, 0, 4096, 0, 0, 3923, 4096, 0, 0, 0, 0, 0, 1, 0, 4095, 2117, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[34, 0, 4095, 24, 0, 4092, 0, 0, 0, 349, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 9, 0, 4089, 0, 0, 0, 0, 2698, 0, 0, 4095, 0, 4096, 4080, 406, 6, 4070, 0, 0, 160, 4093, 0, 0, 64, 0, 4096, 0, 0, 0, 7, 0, 0, 3, 4096, 0, 24, 790, 1, 4096, 1, 7, 0, 2980, 3930, 1000, 0, 0, 0, 3621, 1, 36, 4, 107, 1, 0, 0, 3405, 6, 0, 3805, 35, 0, 0, 0, 4096, 3, 0, 4094, 99, 4083, 3506, 0, 14, 0, 733, 0, 0, 3990, 3943, 2, 2, 4029, 4028, 42, 0, 70, 0, 0, 0, 822, 15, 0, 4088, 0, 0, 4054, 4070, 28, 60, 0, 19, 0, 1, 304, 4096, 4095, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 3978, 175, 0, 3614, 95, 0, 0, 297, 0, 0, 0, 127, 121, 6, 129, 9, 34, 1755, 0, 0, 0, 3808, 40, 0, 0, 44, 4067, 0, 458, 4074, 0, 4096, 4093, 0, 30, 4096, 0, 0, 4, 1, 0, 0, 0, 0, 3504, 0, 0, 0, 3, 0, 0, 0, 4058, 0, 18, 42, 0, 3886, 0, 13, 1, 3985, 4085, 4, 0, 14, 41, 4012, 0, 0, 524, 0, 0, 110, 2, 3873, 0, 0, 4096, 4088, 0, 221, 0, 0, 0, 4053, 4096, 21, 3854, 4083, 154, 0, 67, 0, 0, 0, 4059, 4043, 0, 0, 4096, 3956, 7, 0, 0, 0, 0, 28, 3835, 0, 2, 4053, 0, 99, 4016, 4082, 0, 0, 0, 405, 0, 183, 0, 3966, 2221, 3962] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 1337, 12, 4095, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 4096, 0, 0, 4096, 1, 708, 0, 0, 0, 0, 2, 0, 4096, 52, 0, 1799, 0, 0, 504, 0, 4046, 4, 0, 0, 0, 4096, 0, 0, 0, 4093, 3792, 294, 201, 0, 0, 4094, 7, 0, 0, 2, 0, 0, 0, 3602, 0, 0, 4035, 799, 3667, 0, 0, 4096, 0, 0, 4005, 0, 4085, 4096, 0, 0, 0, 1127, 0, 0, 4094, 4087, 0, 0, 4096, 917, 0, 0, 1, 0, 1, 0, 4094, 0, 0, 3012, 0, 0, 2739, 4096, 0, 65, 0, 0, 0, 134, 0, 4035, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4032, 0, 4095, 0, 1, 4096, 0, 0, 0, 4039, 0, 0, 0, 0, 44, 0, 2446, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 0, 4075, 0, 0, 0, 0, 4096, 0, 0, 4096, 3412, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 4096, 0, 0, 4096, 0, 0, 0, 2611, 0, 0, 0, 0, 2555, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 148, 0, 0, 4096, 0, 5, 0, 3218, 2250, 675, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 2, 0, 1, 0, 0, 4071, 0, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 975, 38, 5, 0, 0, 4064, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 4096, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 3560, 4096, 0, 0, 0, 0, 0, 0, 0, 1846, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19437, 15780, 14314, 14582, 16271, 17161, 15841, 17686] avg=16384.0000 max=19437 max_rank=0 min=14314 min_rank=2 imbalance_ratio=1.186340 max_load_reduction=111635 max_load_reduction_pct=0.851707 moved_to_echo=66959 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 10, 3976, 0, 0, 0, 48, 3843, 0, 2446, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 25, 0, 0, 0, 0, 3524, 0, 0, 0, 0, 4070, 0, 0, 4096, 3276, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4035, 4053, 0, 0, 4094, 30, 0, 0, 7, 0, 0, 0, 0, 100, 228, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 3, 0, 0, 4096, 0, 0, 1, 2, 4096, 11, 0, 0, 4044, 2240, 675, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 294, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 135, 4020, 0, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 975, 0, 363, 0, 80, 0, 9, 51, 0, 0, 4065, 3755, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3431, 4086, 1, 0, 3924, 4096, 0, 1, 0, 0, 137, 0, 222, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3923, 4018, 0, 171, 0, 174, 0, 35, 0, 1838, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[23268, 15106, 13136, 19362, 16607, 9299, 15943, 18351] avg=16384.0000 max=23268 max_rank=0 min=9299 min_rank=5 imbalance_ratio=1.420166 max_load_reduction=107804 max_load_reduction_pct=0.822479 moved_to_echo=60200 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2040, 0, 3978, 0, 0, 4088, 45, 29, 0, 923, 0, 269, 0, 23, 0, 0, 2446, 682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 704, 1, 0, 129, 4095, 87, 33, 44, 0, 3794, 0, 24, 4095, 3393, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4060, 0, 2, 4067, 1, 0, 0, 347, 52, 0, 0, 3, 2960, 0, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3911, 0, 0, 7, 1, 4045, 15, 0, 0, 16, 4042, 0, 0, 3, 4045, 2253, 677, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 3, 0, 4069, 8, 0, 591, 0, 43, 0, 138, 4064, 15, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 970, 124, 3457, 2, 0, 1374, 0, 0, 0, 28, 4036, 4030, 3, 76, 0, 3779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3430, 3973, 4, 0, 4066, 3899, 0, 0, 276, 0, 0, 0, 1773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4071, 0, 0, 0, 4000, 0, 0, 0, 0, 0, 15, 0, 1839, 2595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14523, 16494, 16152, 19117, 16963, 17879, 17421, 12523] avg=16384.0000 max=19117 max_rank=3 min=12523 min_rank=7 imbalance_ratio=1.166809 max_load_reduction=111955 max_load_reduction_pct=0.854149 moved_to_echo=64786 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3834, 0, 3969, 2, 0, 4090, 15, 0, 0, 1814, 0, 168, 0, 19, 0, 0, 2441, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4043, 10, 0, 0, 0, 4091, 0, 0, 4095, 3412, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 4092, 66, 0, 4082, 45, 0, 4, 2050, 24, 0, 445, 0, 3711, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 4, 52, 0, 4038, 0, 62, 33, 0, 4096, 0, 0, 0, 4094, 2248, 674, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4071, 0, 0, 0, 1, 5, 5, 0, 3998, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 973, 9, 4090, 0, 0, 4096, 0, 105, 0, 0, 4087, 4095, 0, 0, 0, 1378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 4093, 0, 0, 4096, 1937, 0, 0, 0, 0, 1, 1, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 892, 4086, 0, 638, 0, 0, 2, 9, 37, 1844, 3452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17036, 15748, 19114, 15410, 16169, 18833, 13701, 15061] avg=16384.0000 max=19114 max_rank=2 min=13701 min_rank=6 imbalance_ratio=1.166626 max_load_reduction=111958 max_load_reduction_pct=0.854172 moved_to_echo=63764 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2417, 0, 4096, 56, 0, 4060, 0, 0, 0, 3236, 0, 0, 0, 0, 0, 0, 2446, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 4096, 0, 0, 4096, 3411, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4096, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 3913, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 707, 0, 31, 0, 1, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 3204, 2250, 675, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 127, 0, 0, 0, 0, 4096, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 975, 716, 487, 1, 18, 4020, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 3670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 4068, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3923, 4096, 0, 0, 0, 0, 0, 1, 0, 1845, 2117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16996, 15750, 16741, 15155, 16417, 18078, 15857, 16078] avg=16384.0000 max=18078 max_rank=5 min=15155 min_rank=3 imbalance_ratio=1.103394 max_load_reduction=112994 max_load_reduction_pct=0.862076 moved_to_echo=66430 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[34, 0, 4095, 24, 0, 4092, 0, 0, 0, 349, 0, 0, 0, 0, 0, 0, 2272, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 9, 0, 4089, 0, 0, 0, 0, 2698, 0, 0, 4095, 3409, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 406, 6, 4070, 0, 0, 160, 4093, 0, 0, 64, 0, 4096, 0, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 4096, 0, 24, 790, 1, 4096, 1, 7, 0, 2980, 2250, 627, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 1000, 0, 0, 0, 3621, 1, 36, 4, 107, 1, 0, 0, 3405, 6, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 906, 35, 0, 0, 0, 4096, 3, 0, 0, 99, 4083, 3506, 0, 14, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3350, 3943, 2, 2, 4029, 4028, 42, 0, 70, 0, 0, 0, 822, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4054, 4070, 28, 60, 0, 19, 0, 1, 304, 1846, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11551, 18472, 21619, 14974, 16113, 13475, 16303, 18565] avg=16384.0000 max=21619 max_rank=2 min=11551 min_rank=0 imbalance_ratio=1.319519 max_load_reduction=109453 max_load_reduction_pct=0.835060 moved_to_echo=64456 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 1337, 12, 4095, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2409, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 0, 4096, 3335, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 1, 708, 0, 0, 0, 0, 2, 0, 4096, 52, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1799, 0, 0, 504, 0, 4046, 4, 0, 0, 0, 4096, 0, 0, 0, 4093, 2217, 665, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3792, 294, 201, 0, 0, 4094, 7, 0, 0, 2, 0, 0, 0, 3602, 0, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 961, 799, 3667, 0, 0, 4096, 0, 0, 0, 0, 4085, 4096, 0, 0, 0, 1127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3438, 4087, 0, 0, 4096, 917, 0, 0, 1, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3012, 0, 0, 2739, 4096, 0, 65, 0, 0, 0, 134, 0, 1818, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16717, 15803, 13614, 17518, 15995, 18831, 16634, 15960] avg=16384.0000 max=18831 max_rank=5 min=13614 min_rank=2 imbalance_ratio=1.149353 max_load_reduction=112241 max_load_reduction_pct=0.856331 moved_to_echo=67420 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3978, 175, 0, 3614, 95, 0, 0, 297, 0, 0, 0, 127, 121, 6, 2446, 685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 9, 34, 1755, 0, 0, 0, 3808, 40, 0, 0, 44, 4067, 0, 458, 4074, 3411, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 30, 4096, 0, 0, 4, 1, 0, 0, 0, 0, 3504, 0, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4058, 0, 18, 42, 0, 3886, 0, 13, 1, 3985, 2179, 675, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4, 0, 14, 41, 4012, 0, 0, 524, 0, 0, 110, 2, 3873, 0, 0, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 975, 4088, 0, 221, 0, 0, 0, 4053, 0, 21, 3854, 4083, 154, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3407, 4043, 0, 0, 4096, 3956, 7, 0, 0, 0, 0, 28, 3835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4053, 0, 99, 4016, 4082, 0, 0, 0, 405, 0, 183, 0, 1787, 2221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11544, 17919, 16382, 14954, 16537, 17516, 19372, 16848] avg=16384.0000 max=19372 max_rank=6 min=11544 min_rank=0 imbalance_ratio=1.182373 max_load_reduction=111700 max_load_reduction_pct=0.852203 moved_to_echo=70273 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[20608, 0, 32512, 1664, 128, 32256, 256, 128, 128, 14720, 0, 512, 0, 256, 4096, 128, 19456, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32256, 1664, 256, 128, 128, 32256, 128, 128, 1280, 128, 128, 128, 384, 31232, 128, 0, 31872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 27392, 32512, 128, 128, 32512, 27008, 128, 128, 384, 0, 256, 128, 11264, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[7808, 5888, 12160, 256, 128, 21760, 128, 4224, 0, 256, 32512, 31872, 256, 128, 128, 14592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 10624, 128, 128, 640, 128, 32640, 128, 256, 896, 128, 32512, 128, 128, 128, 29696, 17920, 5376, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 31616, 0, 128, 23168, 32640, 128, 1024, 0, 640, 128, 384, 384, 14720, 26880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 24832, 32768, 512, 128, 32768, 128, 768, 256, 9216, 128, 0, 512, 128, 24960, 384, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[256, 256, 4224, 2560, 128, 128, 256, 31872, 256, 128, 128, 128, 31104, 0, 512, 32768, 27136, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[154, 2951, 1319, 90, 1937, 35, 616, 264, 1120, 105, 1792, 177, 946, 823, 39, 427, 285, 1294, 752, 1067, 1485, 192, 2285, 994, 77, 1499, 444, 986, 1422, 148, 1176, 2226, 3224, 267, 269, 314, 99, 1037, 1941, 1178, 2294, 1378, 1306, 701, 604, 1732, 1365, 611, 386, 462, 357, 1327, 227, 454, 1648, 119, 165, 422, 2649, 2626, 1025, 293, 711, 464, 678, 2730, 216, 1095, 655, 226, 1125, 2308, 808, 1867, 355, 626, 576, 1781, 1966, 640, 2368, 1238, 2064, 1183, 1739, 613, 927, 2629, 1336, 273, 777, 1233, 738, 747, 331, 414, 2154, 288, 2684, 553, 832, 645, 1812, 2880, 983, 1625, 405, 1349, 460, 425, 1104, 1658, 343, 1005, 551, 1018, 197, 604, 1294, 1250, 634, 1419, 1955, 690, 2061, 267, 717, 166] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[53, 2648, 1716, 17, 2036, 79, 799, 488, 613, 117, 1091, 376, 826, 374, 80, 290, 152, 575, 1108, 1485, 1540, 185, 2387, 1173, 133, 1784, 427, 742, 1319, 235, 1213, 1827, 3514, 268, 239, 855, 243, 662, 1513, 771, 2738, 1460, 1219, 695, 536, 1790, 1172, 547, 399, 254, 370, 1347, 497, 270, 1693, 65, 107, 460, 1843, 2628, 861, 213, 839, 548, 726, 2973, 145, 1168, 659, 316, 1031, 2419, 863, 1550, 285, 1103, 600, 1392, 2342, 817, 2738, 883, 2344, 1490, 1932, 499, 1118, 2566, 1019, 173, 512, 1335, 821, 558, 430, 375, 1698, 349, 2277, 525, 887, 1176, 1833, 3136, 1249, 1443, 682, 1526, 504, 267, 1223, 2049, 259, 1309, 526, 1210, 214, 960, 1104, 1247, 881, 1431, 1674, 1114, 1534, 399, 583, 117] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[37, 2781, 1241, 66, 1767, 45, 730, 406, 1031, 151, 1589, 192, 757, 750, 34, 328, 140, 536, 1702, 782, 1773, 480, 2189, 1241, 68, 1992, 450, 667, 1702, 431, 1652, 2108, 2872, 170, 581, 1049, 128, 878, 1683, 652, 2767, 1269, 956, 572, 414, 1315, 1171, 595, 416, 177, 464, 1759, 740, 358, 1193, 66, 147, 611, 2094, 1745, 1117, 268, 1229, 455, 860, 3234, 190, 1455, 685, 408, 958, 2263, 582, 1471, 177, 1543, 603, 1528, 1759, 851, 1999, 1013, 1828, 1198, 1852, 740, 758, 1335, 1014, 194, 704, 1225, 886, 841, 445, 303, 1948, 295, 2079, 986, 421, 1247, 2230, 3392, 1134, 1446, 798, 1614, 455, 352, 1509, 867, 220, 1349, 1084, 819, 91, 957, 1695, 1171, 782, 1257, 1988, 1004, 1956, 325, 808, 162] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[95, 2116, 760, 69, 2224, 89, 659, 599, 787, 113, 1587, 355, 658, 526, 17, 388, 218, 953, 817, 1281, 1518, 243, 2686, 916, 61, 1427, 433, 477, 1086, 235, 1198, 1891, 3609, 198, 244, 242, 447, 732, 2078, 576, 2486, 1022, 1229, 612, 605, 1608, 1244, 736, 585, 222, 424, 1272, 414, 142, 2238, 197, 183, 519, 2106, 2265, 471, 460, 600, 286, 944, 3320, 192, 1248, 1241, 261, 1454, 2020, 574, 1665, 220, 758, 664, 1354, 2487, 686, 2324, 1428, 2103, 1482, 1759, 491, 895, 2576, 1135, 151, 802, 1212, 1136, 321, 575, 248, 1497, 162, 2603, 548, 821, 935, 2071, 2431, 1659, 1505, 900, 2000, 718, 533, 1221, 1757, 228, 1211, 1417, 981, 103, 738, 780, 1379, 908, 1845, 1757, 983, 1631, 723, 480, 287] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[84, 2332, 1069, 91, 2495, 53, 650, 441, 821, 121, 1898, 435, 1090, 759, 88, 301, 158, 582, 1034, 974, 1504, 476, 2433, 1330, 159, 1556, 413, 871, 1333, 457, 1910, 1849, 2755, 160, 276, 684, 247, 880, 1620, 440, 2536, 1757, 1336, 881, 885, 1354, 639, 1270, 540, 112, 338, 1347, 952, 269, 1500, 73, 63, 930, 2318, 2339, 762, 157, 1145, 437, 913, 3011, 294, 941, 737, 320, 818, 1974, 306, 1791, 206, 1149, 572, 1913, 3119, 844, 1831, 1379, 1450, 1044, 1614, 518, 987, 2359, 773, 282, 667, 1319, 929, 388, 347, 285, 1852, 478, 1652, 1209, 695, 995, 2181, 2775, 1517, 1681, 684, 1585, 276, 276, 1041, 910, 190, 1505, 715, 1360, 214, 881, 1646, 1432, 1009, 957, 1846, 1044, 1639, 204, 531, 243] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[66, 2528, 984, 220, 2159, 112, 997, 450, 936, 102, 1826, 147, 819, 607, 124, 503, 139, 601, 1312, 1079, 1473, 428, 1865, 1142, 206, 1568, 722, 391, 1268, 299, 1556, 2009, 2567, 161, 384, 348, 145, 767, 1866, 520, 2785, 1454, 1468, 924, 486, 1553, 1133, 576, 415, 247, 848, 1742, 673, 418, 1574, 100, 84, 885, 2438, 1649, 673, 403, 431, 609, 1104, 3231, 287, 1589, 873, 543, 1095, 2133, 620, 1558, 383, 1081, 912, 1387, 1693, 903, 2177, 1188, 1892, 1007, 1305, 576, 1882, 1822, 1531, 161, 880, 1039, 1041, 436, 570, 547, 1520, 413, 1835, 922, 712, 649, 2593, 2453, 1530, 1509, 690, 1536, 644, 573, 1224, 1260, 167, 1082, 694, 1216, 160, 1142, 1311, 1291, 887, 1518, 1602, 1039, 1355, 552, 458, 125] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[26, 2860, 777, 118, 1272, 133, 675, 170, 523, 280, 1161, 540, 792, 691, 40, 524, 332, 632, 1322, 777, 2063, 103, 2287, 1162, 42, 2048, 397, 657, 1222, 287, 957, 2062, 3193, 621, 182, 845, 248, 1036, 1665, 667, 2377, 830, 1078, 518, 507, 1595, 1620, 784, 469, 165, 445, 1369, 692, 899, 1527, 68, 623, 532, 2405, 1475, 499, 490, 699, 546, 1276, 2406, 268, 1236, 763, 225, 926, 1852, 526, 2291, 763, 1332, 855, 1244, 3127, 1086, 1687, 1206, 2037, 1292, 1200, 589, 513, 1979, 617, 345, 1119, 1951, 1029, 545, 506, 645, 1693, 601, 2100, 304, 945, 1107, 1881, 2473, 1143, 2394, 1013, 2017, 543, 1190, 1665, 1258, 267, 1049, 976, 891, 201, 694, 814, 1074, 886, 1154, 1781, 980, 1575, 331, 907, 128] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[42, 1895, 934, 54, 2191, 60, 932, 202, 1012, 128, 1550, 719, 986, 1079, 18, 410, 137, 1184, 808, 1176, 1698, 254, 2415, 1440, 63, 1740, 300, 897, 1316, 319, 1669, 2425, 3444, 281, 198, 671, 95, 740, 945, 724, 2236, 1327, 1129, 363, 585, 1889, 1119, 659, 274, 235, 450, 994, 456, 236, 1784, 54, 293, 827, 2045, 2739, 710, 264, 583, 544, 947, 2910, 295, 1384, 756, 119, 1157, 2258, 510, 1768, 141, 569, 877, 2026, 2452, 1280, 2014, 917, 2172, 1226, 1475, 526, 667, 2465, 698, 499, 862, 1741, 1328, 758, 438, 299, 1839, 584, 2475, 714, 988, 1229, 1232, 2819, 1204, 1291, 731, 1515, 620, 445, 1374, 1452, 354, 1119, 694, 741, 140, 522, 801, 1280, 799, 1609, 1615, 1177, 1673, 470, 863, 223] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[154, 2951, 1319, 90, 1937, 35, 616, 264, 1120, 105, 1792, 177, 946, 823, 39, 427, 2880, 1520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 1294, 752, 1067, 1485, 192, 2016, 994, 77, 1499, 444, 986, 1422, 148, 1176, 2226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2042, 267, 269, 314, 99, 1037, 1941, 1178, 2294, 1378, 1306, 701, 604, 1732, 1365, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 386, 462, 357, 1327, 227, 454, 1648, 119, 165, 422, 2649, 2626, 1025, 293, 711, 464, 1833, 1316, 834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 678, 376, 216, 1095, 655, 226, 1125, 2308, 808, 1867, 355, 626, 576, 1781, 1966, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2368, 1238, 2064, 1183, 1739, 613, 927, 1018, 1336, 273, 777, 1233, 738, 747, 331, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2154, 288, 851, 553, 832, 645, 1812, 0, 983, 1625, 405, 1349, 460, 425, 1104, 1658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 1005, 551, 1018, 197, 604, 1294, 1250, 634, 1419, 1955, 690, 2061, 267, 717, 166, 1182, 269, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17195, 16063, 17138, 17318, 15298, 16999, 15144, 15917] avg=16384.0000 max=17318 max_rank=3 min=15144 min_rank=6 imbalance_ratio=1.057007 max_load_reduction=113754 max_load_reduction_pct=0.867874 moved_to_echo=63358 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[53, 2648, 1716, 17, 2036, 79, 799, 488, 613, 117, 1091, 376, 826, 374, 80, 290, 3136, 1656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 575, 1108, 1485, 1540, 185, 2106, 1173, 133, 1784, 427, 742, 1319, 235, 1213, 1827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2225, 268, 239, 855, 243, 662, 1513, 771, 2738, 1460, 1219, 695, 536, 1790, 1172, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 254, 370, 1347, 497, 270, 1693, 65, 107, 460, 1843, 2628, 861, 213, 839, 548, 1555, 1285, 909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 726, 408, 145, 1168, 659, 316, 1031, 2419, 863, 1550, 285, 1103, 600, 1392, 2342, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2738, 883, 2344, 1490, 1932, 499, 1118, 993, 1019, 173, 512, 1335, 821, 558, 430, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1698, 349, 722, 525, 887, 1176, 1833, 0, 1249, 1443, 682, 1526, 504, 267, 1223, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 1309, 526, 1210, 214, 960, 1104, 1247, 881, 1431, 1674, 1114, 1534, 399, 583, 117, 1289, 281, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16395, 16004, 16933, 16143, 15824, 17220, 16133, 16420] avg=16384.0000 max=17220 max_rank=5 min=15824 min_rank=4 imbalance_ratio=1.051025 max_load_reduction=113852 max_load_reduction_pct=0.868622 moved_to_echo=65597 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[95, 2116, 760, 69, 2224, 89, 659, 599, 787, 113, 1587, 355, 658, 526, 17, 388, 2431, 1849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 953, 817, 1281, 1518, 243, 2370, 916, 61, 1427, 433, 477, 1086, 235, 1198, 1891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2285, 198, 244, 242, 447, 732, 2078, 576, 2486, 1022, 1229, 612, 605, 1608, 1244, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 585, 222, 424, 1272, 414, 142, 2238, 197, 183, 519, 2106, 2265, 471, 460, 600, 286, 1777, 1290, 1015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 944, 456, 192, 1248, 1241, 261, 1454, 2020, 574, 1665, 220, 758, 664, 1354, 2487, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2324, 1428, 2103, 1482, 1759, 491, 895, 997, 1135, 151, 802, 1212, 1136, 321, 575, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1497, 162, 826, 548, 821, 935, 2071, 0, 1659, 1505, 900, 2000, 718, 533, 1221, 1757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 1211, 1417, 981, 103, 738, 780, 1379, 908, 1845, 1757, 983, 1631, 723, 480, 287, 1324, 316, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15322, 15124, 16344, 16466, 16224, 17059, 17153, 17380] avg=16384.0000 max=17380 max_rank=7 min=15124 min_rank=1 imbalance_ratio=1.060791 max_load_reduction=113692 max_load_reduction_pct=0.867401 moved_to_echo=67816 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[84, 2332, 1069, 91, 2495, 53, 650, 441, 821, 121, 1898, 435, 1090, 759, 88, 301, 2775, 1677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 582, 1034, 974, 1504, 476, 2147, 1330, 159, 1556, 413, 871, 1333, 457, 1910, 1849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1745, 160, 276, 684, 247, 880, 1620, 440, 2536, 1757, 1336, 881, 885, 1354, 639, 1270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 540, 112, 338, 1347, 952, 269, 1500, 73, 63, 930, 2318, 2339, 762, 157, 1145, 437, 1128, 1181, 920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 913, 414, 294, 941, 737, 320, 818, 1974, 306, 1791, 206, 1149, 572, 1913, 3119, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1831, 1379, 1450, 1044, 1614, 518, 987, 913, 773, 282, 667, 1319, 929, 388, 347, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1852, 478, 524, 1209, 695, 995, 2181, 0, 1517, 1681, 684, 1585, 276, 276, 1041, 910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 1505, 715, 1360, 214, 881, 1646, 1432, 1009, 957, 1846, 1044, 1639, 204, 531, 243, 1010, 286, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17180, 16753, 16710, 16511, 16311, 14726, 15904, 16977] avg=16384.0000 max=17180 max_rank=0 min=14726 min_rank=5 imbalance_ratio=1.048584 max_load_reduction=113892 max_load_reduction_pct=0.868927 moved_to_echo=63918 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[37, 2781, 1241, 66, 1767, 45, 730, 406, 1031, 151, 1589, 192, 757, 750, 34, 328, 3392, 1801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 536, 1702, 782, 1773, 480, 1931, 1241, 68, 1992, 450, 667, 1702, 431, 1652, 2108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1819, 170, 581, 1049, 128, 878, 1683, 652, 2767, 1269, 956, 572, 414, 1315, 1171, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 416, 177, 464, 1759, 740, 358, 1193, 66, 147, 611, 2094, 1745, 1117, 268, 1229, 455, 1420, 668, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 860, 444, 190, 1455, 685, 408, 958, 2263, 582, 1471, 177, 1543, 603, 1528, 1759, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1999, 1013, 1828, 1198, 1852, 740, 758, 517, 1014, 194, 704, 1225, 886, 841, 445, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1948, 295, 659, 986, 421, 1247, 2230, 0, 1134, 1446, 798, 1614, 455, 352, 1509, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 1349, 1084, 819, 91, 957, 1695, 1171, 782, 1257, 1988, 1004, 1956, 325, 808, 162, 1053, 258, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17098, 17655, 16019, 15916, 15777, 15517, 15961, 17129] avg=16384.0000 max=17655 max_rank=1 min=15517 min_rank=5 imbalance_ratio=1.077576 max_load_reduction=113417 max_load_reduction_pct=0.865303 moved_to_echo=64384 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[26, 2860, 777, 118, 1272, 133, 675, 170, 523, 280, 1161, 540, 792, 691, 40, 524, 2473, 1344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, 632, 1322, 777, 2063, 103, 2014, 1162, 42, 2048, 397, 657, 1222, 287, 957, 2062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2017, 621, 182, 845, 248, 1036, 1665, 667, 2377, 830, 1078, 518, 507, 1595, 1620, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 469, 165, 445, 1369, 692, 899, 1527, 68, 623, 532, 2405, 1475, 499, 490, 699, 546, 1437, 996, 739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1276, 323, 268, 1236, 763, 225, 926, 1852, 526, 2291, 763, 1332, 855, 1244, 3127, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1687, 1206, 2037, 1292, 1200, 589, 513, 758, 617, 345, 1119, 1951, 1029, 545, 506, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1693, 601, 663, 304, 945, 1107, 1881, 0, 1143, 2394, 1013, 2017, 543, 1190, 1665, 1258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 267, 1049, 976, 891, 201, 694, 814, 1074, 886, 1154, 1781, 980, 1575, 331, 907, 128, 1176, 273, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14399, 16077, 16590, 16075, 18093, 16039, 18417, 15382] avg=16384.0000 max=18417 max_rank=6 min=14399 min_rank=0 imbalance_ratio=1.124084 max_load_reduction=112655 max_load_reduction_pct=0.859489 moved_to_echo=67931 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[66, 2528, 984, 220, 2159, 112, 997, 450, 936, 102, 1826, 147, 819, 607, 124, 503, 2453, 1799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 601, 1312, 1079, 1473, 428, 1646, 1142, 206, 1568, 722, 391, 1268, 299, 1556, 2009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1626, 161, 384, 348, 145, 767, 1866, 520, 2785, 1454, 1468, 924, 486, 1553, 1133, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 415, 247, 848, 1742, 673, 418, 1574, 100, 84, 885, 2438, 1649, 673, 403, 431, 609, 1253, 912, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1104, 444, 287, 1589, 873, 543, 1095, 2133, 620, 1558, 383, 1081, 912, 1387, 1693, 903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2177, 1188, 1892, 1007, 1305, 576, 1882, 706, 1531, 161, 880, 1039, 1041, 436, 570, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1520, 413, 582, 922, 712, 649, 2593, 0, 1530, 1509, 690, 1536, 644, 573, 1224, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 1082, 694, 1216, 160, 1142, 1311, 1291, 887, 1518, 1602, 1039, 1355, 552, 458, 125, 941, 219, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16832, 15839, 16196, 16342, 16605, 16938, 16357, 15963] avg=16384.0000 max=16938 max_rank=5 min=15839 min_rank=1 imbalance_ratio=1.033813 max_load_reduction=114134 max_load_reduction_pct=0.870773 moved_to_echo=65863 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[42, 1895, 934, 54, 2191, 60, 932, 202, 1012, 128, 1550, 719, 986, 1079, 18, 410, 2819, 1620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 1184, 808, 1176, 1698, 254, 2131, 1440, 63, 1740, 300, 897, 1316, 319, 1669, 2425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 281, 198, 671, 95, 740, 945, 724, 2236, 1327, 1129, 363, 585, 1889, 1119, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 235, 450, 994, 456, 236, 1784, 54, 293, 827, 2045, 2739, 710, 264, 583, 544, 1690, 1234, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 947, 401, 295, 1384, 756, 119, 1157, 2258, 510, 1768, 141, 569, 877, 2026, 2452, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2014, 917, 2172, 1226, 1475, 526, 667, 954, 698, 499, 862, 1741, 1328, 758, 438, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1839, 584, 785, 714, 988, 1229, 1232, 0, 1204, 1291, 731, 1515, 620, 445, 1374, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 1119, 694, 741, 140, 522, 801, 1280, 799, 1609, 1615, 1177, 1673, 470, 863, 223, 1263, 284, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16651, 17557, 15142, 16301, 16940, 16574, 16003, 15904] avg=16384.0000 max=17557 max_rank=1 min=15142 min_rank=2 imbalance_ratio=1.071594 max_load_reduction=113515 max_load_reduction_pct=0.866051 moved_to_echo=65421 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[17152, 9344, 16000, 9984, 12928, 4608, 7808, 6912, 8192, 2176, 6400, 11136, 7936, 4608, 3712, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[14208, 3200, 5632, 5888, 6400, 8064, 15872, 0, 10496, 12928, 6016, 13184, 4224, 4096, 10368, 11264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[16000, 2176, 2432, 5120, 1664, 6784, 13312, 5632, 20224, 10624, 9728, 5376, 4736, 12928, 9472, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[640, 20224, 8832, 768, 16128, 640, 6144, 3072, 6912, 1152, 12544, 2944, 6912, 5632, 512, 3200, 22400, 13312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[7552, 3328, 1920, 10240, 6400, 2432, 8576, 17280, 4864, 14080, 2560, 8192, 5760, 12672, 19072, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1664, 6400, 8960, 8704, 13056, 2432, 16384, 9472, 896, 13696, 3712, 5760, 10752, 2432, 11392, 16512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[3584, 1920, 3712, 11264, 4736, 3072, 13184, 768, 1792, 5248, 17920, 17536, 6144, 2560, 6272, 3968, 12160, 8960, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[2048, 9728, 6784, 8320, 1408, 6528, 9472, 10240, 6912, 11264, 14336, 8064, 13440, 3328, 5376, 1536, 9344, 2304, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3239, 0, 3436, 0, 1, 1209, 140, 253, 0, 14, 3496, 4095, 6, 25, 0, 13, 0, 0, 0, 4066, 0, 0, 82, 2, 846, 0, 4, 4096, 3153, 44, 4073, 105, 3857, 19, 0, 548, 4058, 0, 0, 6, 0, 0, 39, 29, 1071, 0, 0, 3777, 4005, 33, 50, 0, 0, 0, 3645, 4073, 60, 0, 3096, 8, 2, 0, 4, 622, 23, 4094, 273, 35, 33, 0, 1693, 4025, 0, 1, 0, 0, 3798, 2, 2286, 2055, 263, 3621, 3983, 4065, 572, 4062, 3295, 0, 0, 0, 0, 11, 726, 0, 0, 2536, 4029, 6, 3722, 81, 1, 0, 2, 0, 0, 11, 928, 0, 12, 0, 467, 0, 0, 0, 7, 1127, 546, 3335, 2338, 364, 0, 4084, 80, 3943, 0, 3137] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3442, 7, 4090, 0, 0, 2041, 52, 397, 0, 9, 2658, 3176, 44, 2, 29, 0, 0, 2, 96, 4096, 0, 4, 180, 3, 0, 92, 29, 4060, 3627, 84, 2827, 3, 263, 0, 0, 1403, 4031, 8, 235, 46, 1, 0, 2, 0, 2, 64, 0, 3896, 3988, 6, 28, 8, 0, 0, 4070, 4093, 7, 0, 2557, 0, 174, 0, 3, 1145, 0, 3969, 375, 559, 1051, 0, 2312, 3803, 148, 0, 22, 0, 3992, 0, 941, 3394, 1, 3093, 3875, 4006, 437, 3692, 2027, 14, 13, 6, 0, 5, 642, 0, 0, 2236, 4073, 80, 3567, 177, 14, 0, 1, 164, 10, 0, 3933, 1, 0, 1, 3152, 1, 0, 38, 0, 2643, 465, 3489, 534, 148, 1, 2674, 225, 3898, 9, 4081] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 4058, 0, 4093, 0, 19, 3655, 141, 11, 1, 1067, 4089, 4032, 4, 19, 9, 0, 0, 2, 1079, 4070, 1, 639, 1630, 12, 26, 0, 0, 4093, 2535, 44, 477, 4, 844, 10, 122, 2, 2763, 8, 254, 333, 95, 22, 4, 0, 52, 91, 3, 4061, 3995, 0, 0, 0, 67, 0, 4082, 4087, 6, 0, 3853, 0, 7, 13, 0, 446, 29, 3972, 460, 7, 257, 22, 3231, 4095, 173, 313, 29, 1, 4020, 5, 662, 1672, 2, 1739, 4075, 4096, 1291, 4095, 4081, 0, 0, 0, 21, 7, 3414, 4, 165, 4095, 4090, 53, 2131, 1, 313, 10, 0, 1, 19, 0, 347, 0, 0, 0, 1369, 9, 0, 0, 0, 1723, 497, 3731, 585, 55, 0, 3039, 0, 1954, 4, 4077] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[8, 145, 933, 1, 3995, 1, 53, 30, 3, 0, 1, 768, 3949, 3994, 17, 5, 0, 4, 27, 1, 0, 4077, 8, 9, 94, 0, 19, 272, 0, 4095, 3524, 0, 4091, 49, 1429, 4, 151, 0, 4010, 0, 39, 105, 138, 0, 0, 60, 15, 6, 0, 3035, 3904, 24, 0, 0, 10, 8, 4018, 4054, 5, 0, 4078, 53, 5, 1, 174, 3623, 1, 3604, 28, 3, 111, 0, 440, 4031, 3, 0, 0, 0, 3972, 3, 2764, 1781, 8, 3859, 4041, 4067, 229, 4091, 4051, 0, 0, 445, 0, 3206, 3152, 19, 1, 4092, 4033, 9, 3200, 0, 1, 7, 35, 4, 0, 69, 911, 71, 8, 9, 1485, 0, 14, 0, 0, 1422, 451, 3475, 3, 261, 0, 3030, 33, 1545, 0, 3872] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3510, 0, 4059, 0, 1, 3410, 17, 40, 0, 69, 4049, 1130, 52, 87, 2, 0, 0, 0, 64, 4029, 0, 2, 1265, 0, 0, 0, 2, 4094, 1759, 1, 2157, 58, 90, 0, 12, 1487, 3869, 3, 495, 34, 4, 0, 0, 189, 0, 1, 0, 4058, 2065, 0, 0, 17, 2, 0, 4074, 4063, 0, 1, 1581, 1, 2, 0, 0, 832, 65, 3248, 191, 1290, 29, 7, 3804, 3742, 12, 6, 0, 0, 3973, 10, 2606, 1073, 119, 4081, 3928, 3899, 34, 4094, 4059, 0, 0, 0, 0, 1182, 196, 0, 0, 4074, 4095, 0, 4096, 180, 0, 1, 0, 38, 2, 98, 4086, 226, 1, 0, 3565, 0, 21, 0, 1, 25, 36, 3146, 1971, 77, 0, 3489, 680, 2683, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 29, 4011, 34, 4075, 0, 0, 4004, 23, 2580, 0, 45, 4034, 3574, 0, 64, 33, 0, 81, 481, 34, 4079, 27, 0, 509, 0, 0, 84, 0, 3873, 2578, 3, 2162, 1, 966, 1, 0, 556, 3754, 0, 0, 3, 1, 1, 0, 0, 58, 4, 73, 3823, 4087, 0, 7, 55, 0, 0, 3400, 3961, 5, 0, 2654, 2, 0, 9, 0, 15, 1, 2684, 12, 0, 169, 6, 2768, 3926, 0, 83, 0, 0, 3992, 0, 4024, 1588, 847, 4044, 4003, 4096, 952, 4096, 2854, 3, 476, 7, 106, 95, 812, 13, 0, 3634, 4077, 69, 2348, 1, 237, 0, 15, 278, 2, 4, 245, 0, 1, 0, 2076, 1, 0, 0, 0, 2059, 676, 3356, 25, 977, 0, 3988, 1, 1383, 34, 4075] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[3, 45, 3589, 0, 3511, 338, 13, 240, 263, 9, 0, 3258, 4024, 2446, 0, 88, 515, 8, 1, 12, 4, 4003, 324, 691, 236, 922, 0, 1, 0, 4015, 4051, 50, 103, 121, 838, 0, 41, 223, 3275, 9, 69, 238, 0, 128, 2, 1128, 337, 0, 832, 2535, 3822, 10, 709, 15, 92, 64, 3988, 4038, 29, 11, 3706, 433, 0, 11, 0, 767, 20, 2062, 1, 548, 292, 30, 1416, 4017, 300, 509, 480, 1, 3484, 18, 1, 3602, 540, 3547, 3717, 3533, 17, 4009, 4011, 0, 745, 4, 0, 11, 1705, 1692, 9, 4076, 3594, 3, 884, 0, 0, 0, 128, 0, 1, 0, 2196, 105, 1, 0, 4024, 2, 0, 7, 0, 3718, 0, 2610, 26, 75, 0, 2772, 31, 2912, 13, 3339] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[2, 103, 4092, 0, 4017, 0, 361, 2464, 1, 11, 0, 118, 4058, 4033, 9, 91, 873, 14, 13, 19, 15, 4082, 4, 275, 22, 16, 0, 0, 25, 4017, 4045, 71, 1588, 0, 421, 0, 2, 7, 4057, 1, 236, 2, 0, 1, 1, 2, 0, 5, 3, 4080, 3603, 0, 7, 1, 1, 40, 4093, 3816, 10, 0, 3998, 2, 0, 52, 7, 223, 109, 3863, 71, 394, 297, 240, 1052, 3988, 15, 0, 0, 83, 4003, 0, 1598, 3497, 6, 68, 3768, 4081, 1250, 4095, 3995, 0, 9, 2, 0, 94, 3961, 21, 2, 4033, 4096, 1, 3303, 6, 0, 9, 0, 0, 26, 0, 384, 0, 2, 15, 3230, 338, 0, 3, 25, 91, 7, 3191, 46, 3238, 0, 1318, 5, 3936, 2, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3442, 7, 2530, 0, 0, 2041, 52, 397, 0, 9, 2658, 3176, 44, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 2, 96, 4096, 0, 4, 180, 3, 0, 92, 29, 4060, 3627, 84, 1934, 1035, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2827, 3, 263, 0, 0, 1403, 4031, 8, 235, 46, 1, 0, 2, 0, 2, 64, 3692, 4006, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 3988, 6, 28, 8, 0, 0, 4070, 1574, 7, 0, 2557, 0, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1145, 0, 3969, 375, 559, 1051, 0, 2312, 3803, 148, 0, 22, 0, 3992, 0, 899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 941, 3394, 1, 3093, 1941, 0, 437, 0, 2027, 14, 13, 6, 0, 5, 642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2236, 4073, 80, 3567, 177, 14, 0, 1, 164, 10, 0, 3933, 1, 0, 1, 2580, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3152, 1, 0, 38, 0, 2643, 465, 3489, 534, 148, 1, 2674, 225, 3898, 9, 1450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14358, 15932, 16634, 16308, 18278, 12514, 18321, 18727] avg=16384.0000 max=18727 max_rank=7 min=12514 min_rank=5 imbalance_ratio=1.143005 max_load_reduction=112345 max_load_reduction_pct=0.857124 moved_to_echo=67840 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4058, 0, 2532, 0, 19, 3655, 141, 11, 1, 1067, 4089, 4032, 4, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 2, 1079, 4070, 1, 639, 1630, 12, 26, 0, 0, 4093, 2535, 44, 2034, 1033, 661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 477, 4, 844, 10, 122, 2, 2763, 8, 254, 333, 95, 22, 4, 0, 52, 91, 4095, 4096, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4061, 3995, 0, 0, 0, 67, 0, 4082, 1572, 6, 0, 3853, 0, 7, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 446, 29, 3972, 460, 7, 257, 22, 3231, 4095, 173, 313, 29, 1, 4020, 5, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 1672, 2, 1739, 2041, 0, 1291, 0, 4081, 0, 0, 0, 21, 7, 3414, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 4095, 4090, 53, 2131, 1, 313, 10, 0, 1, 19, 0, 347, 0, 0, 0, 2577, 1482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 9, 0, 0, 0, 1723, 497, 3731, 585, 55, 0, 3039, 0, 1954, 4, 1449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19629, 17868, 13323, 17659, 17960, 14934, 15284, 14415] avg=16384.0000 max=19629 max_rank=0 min=13323 min_rank=2 imbalance_ratio=1.198059 max_load_reduction=111443 max_load_reduction_pct=0.850243 moved_to_echo=62593 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3239, 0, 2125, 0, 1, 1209, 140, 253, 0, 14, 3496, 4095, 6, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4066, 0, 0, 82, 2, 846, 0, 4, 4096, 3153, 44, 1988, 1030, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 105, 3857, 19, 0, 548, 4058, 0, 0, 6, 0, 0, 39, 29, 1071, 0, 4062, 4065, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3777, 4005, 33, 50, 0, 0, 0, 3645, 1566, 60, 0, 3096, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 622, 23, 4094, 273, 35, 33, 0, 1693, 4025, 0, 1, 0, 0, 3798, 2, 756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2286, 2055, 263, 3621, 1995, 0, 572, 0, 3295, 0, 0, 0, 0, 11, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2536, 4029, 6, 3722, 81, 1, 0, 2, 0, 0, 11, 928, 0, 12, 0, 1983, 1477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 0, 0, 0, 7, 1127, 546, 3335, 2338, 364, 0, 4084, 80, 3943, 0, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14603, 15879, 21971, 16242, 15359, 14824, 14788, 17406] avg=16384.0000 max=21971 max_rank=2 min=14603 min_rank=0 imbalance_ratio=1.341003 max_load_reduction=109101 max_load_reduction_pct=0.832375 moved_to_echo=62377 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[8, 145, 933, 1, 2471, 1, 53, 30, 3, 0, 1, 768, 3949, 3994, 17, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 27, 1, 0, 4077, 8, 9, 94, 0, 19, 272, 0, 4095, 3524, 0, 2017, 1025, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 49, 1429, 4, 151, 0, 4010, 0, 39, 105, 138, 0, 0, 60, 15, 6, 4091, 4067, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3035, 3904, 24, 0, 0, 10, 8, 4018, 1559, 5, 0, 4078, 53, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 3623, 1, 3604, 28, 3, 111, 0, 440, 4031, 3, 0, 0, 0, 3972, 3, 879, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2764, 1781, 8, 3859, 2024, 0, 229, 0, 4051, 0, 0, 445, 0, 3206, 3152, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 4033, 9, 3200, 0, 1, 7, 35, 4, 0, 69, 911, 71, 8, 9, 2448, 1470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1485, 0, 14, 0, 0, 1422, 451, 3475, 3, 261, 0, 3030, 33, 1545, 0, 1376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12379, 15817, 18303, 16700, 16872, 21538, 16368, 13095] avg=16384.0000 max=21538 max_rank=5 min=12379 min_rank=0 imbalance_ratio=1.314575 max_load_reduction=109534 max_load_reduction_pct=0.835678 moved_to_echo=67873 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3510, 0, 2510, 0, 1, 3410, 17, 40, 0, 69, 4049, 1130, 52, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 64, 4029, 0, 2, 1265, 0, 0, 0, 2, 4094, 1759, 1, 1961, 1027, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2157, 58, 90, 0, 12, 1487, 3869, 3, 495, 34, 4, 0, 0, 189, 0, 1, 4094, 3899, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 2065, 0, 0, 17, 2, 0, 4074, 1563, 0, 1, 1581, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 832, 65, 3248, 191, 1290, 29, 7, 3804, 3742, 12, 6, 0, 0, 3973, 10, 893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2606, 1073, 119, 4081, 1967, 0, 34, 0, 4059, 0, 0, 0, 0, 1182, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4095, 0, 4096, 180, 0, 1, 0, 38, 2, 98, 4086, 226, 1, 0, 2589, 1473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3565, 0, 21, 0, 1, 25, 36, 3146, 1971, 77, 0, 3489, 680, 2683, 0, 1456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14875, 14862, 16443, 13364, 18102, 15317, 20959, 17150] avg=16384.0000 max=20959 max_rank=6 min=13364 min_rank=3 imbalance_ratio=1.279236 max_load_reduction=110113 max_load_reduction_pct=0.840096 moved_to_echo=71528 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 45, 3589, 0, 2165, 338, 13, 240, 263, 9, 0, 3258, 4024, 2446, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 515, 8, 1, 12, 4, 4003, 324, 691, 236, 922, 0, 1, 0, 4015, 4051, 50, 1861, 1025, 570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 121, 838, 0, 41, 223, 3275, 9, 69, 238, 0, 128, 2, 1128, 337, 0, 4009, 3533, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 832, 2535, 3822, 10, 709, 15, 92, 64, 3988, 1546, 29, 11, 3706, 433, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 767, 20, 2062, 1, 548, 292, 30, 1416, 4017, 300, 509, 480, 1, 3484, 18, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3602, 540, 3547, 1856, 0, 17, 0, 4011, 0, 745, 4, 0, 11, 1705, 1692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4076, 3594, 3, 884, 0, 0, 0, 128, 0, 1, 0, 2196, 105, 1, 0, 2115, 1467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 2, 0, 7, 0, 3718, 0, 2610, 26, 75, 0, 2772, 31, 2912, 13, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16481, 18289, 14098, 17803, 14721, 17731, 14579, 17370] avg=16384.0000 max=18289 max_rank=1 min=14098 min_rank=2 imbalance_ratio=1.116272 max_load_reduction=112783 max_load_reduction_pct=0.860466 moved_to_echo=64401 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 29, 4011, 34, 2521, 0, 0, 4004, 23, 2580, 0, 45, 4034, 3574, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 81, 481, 34, 4079, 27, 0, 509, 0, 0, 84, 0, 3873, 2578, 3, 1998, 1001, 658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2162, 1, 966, 1, 0, 556, 3754, 0, 0, 3, 1, 1, 0, 0, 58, 4, 4096, 4096, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 3823, 4087, 0, 7, 55, 0, 0, 3400, 1524, 5, 0, 2654, 2, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 2684, 12, 0, 169, 6, 2768, 3926, 0, 83, 0, 0, 3992, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 1588, 847, 4044, 2005, 0, 952, 0, 2854, 3, 476, 7, 106, 95, 812, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3634, 4077, 69, 2348, 1, 237, 0, 15, 278, 2, 4, 245, 0, 1, 0, 2576, 1436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2076, 1, 0, 0, 0, 2059, 676, 3356, 25, 977, 0, 3988, 1, 1383, 34, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20919, 15439, 15750, 15639, 14552, 17826, 14923, 16024] avg=16384.0000 max=20919 max_rank=0 min=14552 min_rank=4 imbalance_ratio=1.276794 max_load_reduction=110153 max_load_reduction_pct=0.840401 moved_to_echo=63325 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 103, 4092, 0, 2485, 0, 361, 2464, 1, 11, 0, 118, 4058, 4033, 9, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 873, 14, 13, 19, 15, 4082, 4, 275, 22, 16, 0, 0, 25, 4017, 4045, 71, 1881, 965, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1588, 0, 421, 0, 2, 7, 4057, 1, 236, 2, 0, 1, 1, 2, 0, 5, 4095, 4081, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4080, 3603, 0, 7, 1, 1, 40, 4093, 1467, 10, 0, 3998, 2, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 223, 109, 3863, 71, 394, 297, 240, 1052, 3988, 15, 0, 0, 83, 4003, 0, 883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1598, 3497, 6, 68, 1887, 0, 1250, 0, 3995, 0, 9, 2, 0, 94, 3961, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4033, 4096, 1, 3303, 6, 0, 9, 0, 0, 26, 0, 384, 0, 2, 15, 2589, 1384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3230, 338, 0, 3, 25, 91, 7, 3191, 46, 3238, 0, 1318, 5, 3936, 2, 1455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17828, 16986, 14550, 17357, 15228, 16388, 15850, 16885] avg=16384.0000 max=17828 max_rank=0 min=14550 min_rank=2 imbalance_ratio=1.088135 max_load_reduction=113244 max_load_reduction_pct=0.863983 moved_to_echo=64351 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[17536, 384, 8832, 128, 384, 4352, 29824, 128, 1408, 768, 256, 256, 128, 1408, 1536, 256, 32256, 31872, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 384, 26880, 128, 19456, 384, 512, 17152, 640, 3328, 128, 5376, 30464, 26496, 256, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[14976, 18688, 1792, 24064, 15744, 0, 4864, 0, 28416, 128, 1280, 512, 128, 4736, 14720, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 28800, 32128, 256, 23296, 512, 640, 128, 256, 512, 128, 256, 13056, 512, 128, 128, 19584, 11776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[19456, 384, 128, 128, 128, 12928, 2688, 26368, 5632, 5248, 128, 24448, 1152, 22272, 128, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 7680, 256, 27520, 1536, 2944, 2304, 384, 16768, 31744, 768, 1024, 640, 128, 31360, 128, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1024, 29312, 29568, 128, 896, 128, 256, 128, 31488, 12416, 128, 128, 25600, 512, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1536, 128, 128, 640, 1408, 32512, 384, 1664, 4096, 1024, 896, 512, 128, 32384, 25344, 384, 15744, 8192, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4000, 0, 2, 0, 3500, 12, 0, 52, 5, 3682, 0, 3244, 2530, 1, 2, 4078, 1294, 2600, 33, 2, 4093, 203, 4000, 0, 0, 4079, 0, 0, 0, 3893, 528, 3827, 75, 4095, 12, 3983, 0, 0, 4087, 3448, 18, 5, 0, 0, 14, 1, 2, 7, 0, 4096, 1, 1, 4092, 11, 0, 153, 22, 0, 3, 849, 4087, 0, 435, 33, 0, 4039, 36, 1, 0, 0, 11, 2835, 4063, 57, 137, 0, 0, 3911, 4, 3765, 2, 1976, 7, 1376, 0, 4051, 0, 1039, 0, 15, 0, 0, 512, 0, 4095, 0, 19, 894, 4077, 0, 0, 0, 203, 97, 3, 0, 4092, 140, 0, 4096, 0, 3, 48, 0, 4055, 2697, 7, 0, 15, 5, 3353, 0, 29, 0, 1, 24, 91, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[2674, 31, 19, 0, 3699, 0, 0, 37, 0, 3625, 0, 3763, 2353, 5, 0, 3881, 3260, 349, 2, 38, 4060, 0, 729, 0, 0, 2722, 0, 0, 0, 4084, 10, 4043, 0, 4096, 0, 3681, 44, 5, 4070, 3106, 0, 22, 215, 39, 164, 0, 0, 37, 26, 4096, 0, 0, 4041, 0, 0, 0, 471, 41, 0, 181, 4094, 92, 1209, 0, 0, 4077, 4, 0, 1, 0, 0, 2652, 4096, 2, 8, 9, 0, 4009, 0, 3838, 0, 2469, 2, 0, 813, 4064, 0, 3943, 392, 13, 0, 0, 0, 0, 4067, 0, 0, 604, 3979, 113, 0, 0, 442, 1966, 3, 0, 4022, 0, 48, 4091, 6, 0, 0, 0, 4047, 4096, 11, 0, 0, 0, 3878, 3, 119, 69, 0, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[2461, 529, 0, 4, 4094, 11, 1575, 543, 110, 777, 0, 2649, 0, 34, 0, 2033, 1719, 2430, 46, 0, 3989, 1047, 309, 86, 0, 4034, 54, 327, 0, 3913, 4094, 57, 0, 3487, 233, 0, 3, 299, 3750, 4085, 0, 0, 121, 173, 0, 0, 4, 1, 0, 4065, 0, 361, 3580, 1, 0, 0, 406, 0, 676, 753, 3949, 0, 1901, 0, 1173, 1679, 41, 57, 5, 367, 0, 3668, 4093, 110, 0, 12, 2024, 3609, 355, 4030, 367, 4032, 0, 1595, 1425, 4035, 68, 18, 0, 1, 719, 0, 2, 1182, 3547, 1004, 92, 2946, 3994, 0, 0, 14, 1533, 34, 0, 0, 4058, 0, 0, 4016, 314, 0, 2577, 5, 3463, 606, 0, 180, 2, 0, 3152, 1, 0, 67, 1, 1, 0, 25] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4026, 0, 0, 0, 103, 0, 3, 0, 0, 3997, 0, 2951, 2785, 0, 0, 4096, 1772, 4061, 0, 0, 3598, 310, 135, 0, 0, 4096, 1, 0, 0, 4064, 60, 3930, 288, 4096, 0, 4051, 0, 0, 4095, 1968, 0, 0, 28, 6, 0, 0, 0, 0, 0, 4093, 0, 14, 4077, 0, 0, 424, 0, 4, 0, 1256, 4077, 0, 2447, 0, 0, 4073, 1, 0, 0, 0, 0, 2, 4096, 163, 68, 0, 0, 3827, 0, 3834, 0, 4090, 12, 750, 0, 4085, 0, 2136, 163, 2, 22, 0, 54, 0, 4095, 204, 0, 344, 3785, 26, 0, 1, 3699, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 79, 0, 4045, 4076, 0, 0, 0, 0, 4049, 169, 0, 0, 0, 0, 22, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4095, 0, 0, 15, 3850, 0, 12, 24, 1, 2713, 0, 3937, 1, 52, 2, 3856, 3027, 4060, 10, 0, 3616, 155, 1377, 0, 0, 3945, 0, 0, 6, 3994, 3874, 998, 2465, 4096, 886, 1071, 7, 10, 4065, 4030, 7, 0, 0, 0, 0, 0, 2, 0, 0, 1914, 5, 0, 3822, 0, 0, 555, 11, 19, 1, 2896, 4096, 6, 0, 3, 82, 1620, 4, 13, 15, 1, 8, 4060, 3670, 1429, 0, 0, 0, 4078, 0, 1539, 1, 4043, 0, 2187, 0, 4087, 397, 14, 0, 0, 0, 4, 132, 0, 4087, 401, 20, 877, 4091, 0, 6, 0, 0, 902, 0, 0, 4096, 0, 0, 4042, 0, 4, 33, 0, 3365, 4088, 0, 22, 0, 2, 4058, 0, 0, 0, 3, 4, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4030, 0, 0, 0, 2128, 54, 77, 142, 1, 4090, 0, 4016, 3273, 0, 0, 4084, 3114, 4051, 6, 17, 0, 310, 2828, 6, 7, 4096, 4, 0, 22, 3968, 3769, 538, 61, 4070, 33, 2512, 2, 116, 4086, 4095, 0, 38, 55, 0, 11, 0, 3, 1, 4, 871, 0, 0, 4068, 21, 1, 20, 0, 23, 134, 1660, 3958, 3, 2496, 0, 93, 184, 0, 0, 22, 17, 0, 3313, 4095, 3610, 0, 105, 0, 4023, 0, 3525, 43, 1518, 0, 3245, 0, 4070, 0, 556, 0, 0, 1, 0, 113, 92, 4040, 152, 0, 171, 4047, 0, 8, 0, 7, 513, 10, 26, 4065, 0, 0, 4086, 26, 23, 341, 0, 4023, 3892, 0, 0, 0, 0, 4018, 4, 11, 0, 0, 11, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4068, 0, 0, 0, 3923, 0, 0, 36, 0, 4085, 0, 4058, 3407, 0, 0, 3933, 113, 1969, 19, 16, 4066, 24, 3404, 0, 0, 787, 325, 0, 0, 3277, 0, 3994, 8, 4096, 0, 3835, 0, 0, 4093, 4080, 2, 117, 1, 4, 40, 0, 2, 1, 0, 446, 0, 1, 4095, 10, 0, 162, 0, 0, 12, 3392, 4091, 0, 108, 0, 0, 4082, 3, 0, 3, 3, 0, 2449, 4081, 999, 1, 605, 0, 4017, 43, 62, 3, 4005, 1, 2012, 453, 4096, 694, 2662, 0, 1, 0, 0, 3, 2, 4081, 112, 0, 372, 4096, 5, 0, 138, 2, 2908, 1, 0, 4089, 283, 6, 3979, 0, 0, 0, 0, 4094, 4033, 57, 23, 367, 0, 2040, 0, 0, 4, 1, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4095, 35, 81, 0, 549, 0, 286, 0, 0, 3990, 2, 4095, 38, 0, 8, 4090, 1763, 1567, 0, 0, 4091, 54, 2817, 1, 2, 2222, 0, 0, 0, 4013, 3993, 3194, 139, 4082, 0, 3, 8, 2, 4088, 4089, 0, 0, 0, 0, 0, 0, 28, 0, 0, 3379, 0, 0, 4096, 53, 0, 0, 0, 103, 1, 1677, 4052, 2, 0, 10, 7, 3932, 86, 0, 16, 0, 0, 3614, 3944, 1472, 0, 0, 7, 4095, 2, 3967, 1, 70, 0, 4088, 9, 4045, 0, 1864, 0, 0, 0, 0, 8, 1, 4079, 2506, 0, 1291, 4075, 2, 0, 0, 0, 699, 4, 0, 4096, 0, 0, 4091, 1, 0, 4, 0, 4096, 3973, 0, 0, 0, 0, 4092, 0, 0, 0, 36, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4000, 0, 2, 0, 3500, 12, 0, 52, 5, 3682, 0, 3244, 2530, 1, 2, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1294, 2600, 33, 2, 1831, 203, 4000, 0, 0, 4079, 0, 0, 0, 0, 528, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 4095, 12, 3983, 0, 0, 4087, 3448, 18, 5, 0, 0, 14, 1, 2, 7, 1607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 1, 4092, 11, 0, 153, 22, 0, 3, 849, 4087, 0, 435, 33, 2176, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 36, 1, 0, 0, 11, 2835, 1945, 57, 137, 0, 0, 3911, 4, 3765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1976, 7, 1376, 0, 4051, 0, 1039, 0, 15, 0, 0, 512, 0, 4095, 0, 691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 894, 4077, 0, 0, 0, 203, 97, 3, 0, 4092, 140, 0, 4096, 0, 3, 1571, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 4055, 2697, 7, 0, 15, 5, 3353, 0, 29, 0, 1, 24, 91, 1, 3893, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17834, 18397, 17354, 16292, 16741, 13764, 15373, 15317] avg=16384.0000 max=18397 max_rank=1 min=13764 min_rank=5 imbalance_ratio=1.122864 max_load_reduction=112675 max_load_reduction_pct=0.859642 moved_to_echo=61195 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2674, 31, 19, 0, 3699, 0, 0, 37, 0, 3625, 0, 3763, 2353, 5, 0, 765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3260, 349, 2, 38, 1816, 0, 729, 0, 0, 2722, 0, 0, 0, 0, 10, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3681, 44, 5, 4070, 3106, 0, 22, 215, 39, 164, 0, 0, 37, 1620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 0, 0, 4041, 0, 0, 0, 471, 41, 0, 181, 4094, 92, 1209, 0, 2071, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4, 0, 1, 0, 0, 2652, 1961, 2, 8, 9, 0, 4009, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2469, 2, 0, 813, 4064, 0, 3943, 392, 13, 0, 0, 0, 0, 4067, 0, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 604, 3979, 113, 0, 0, 442, 1966, 3, 0, 4022, 0, 48, 4091, 6, 0, 1558, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 4096, 11, 0, 0, 0, 3878, 3, 119, 69, 0, 0, 2, 0, 4084, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16971, 12969, 17099, 16658, 16561, 16449, 17011, 17354] avg=16384.0000 max=17354 max_rank=7 min=12969 min_rank=1 imbalance_ratio=1.059204 max_load_reduction=113718 max_load_reduction_pct=0.867599 moved_to_echo=67375 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2461, 529, 0, 4, 4094, 11, 1575, 543, 110, 777, 0, 2649, 0, 34, 0, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1719, 2430, 46, 0, 1779, 1047, 309, 86, 0, 4034, 54, 327, 0, 0, 4094, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3487, 233, 0, 3, 299, 3750, 4085, 0, 0, 121, 173, 0, 0, 4, 1, 1625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 361, 3580, 1, 0, 0, 406, 0, 676, 753, 3949, 0, 1901, 0, 1088, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1173, 1679, 41, 57, 5, 367, 0, 3668, 1945, 110, 0, 12, 2024, 3609, 355, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 4032, 0, 1595, 1425, 4035, 68, 18, 0, 1, 719, 0, 2, 1182, 3547, 1004, 676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 2946, 3994, 0, 0, 14, 1533, 34, 0, 0, 4058, 0, 0, 4016, 314, 0, 1534, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2577, 5, 3463, 606, 0, 180, 2, 0, 3152, 1, 0, 67, 1, 1, 0, 25, 3913, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13180, 15982, 13781, 17119, 19075, 18671, 18719, 14545] avg=16384.0000 max=19075 max_rank=4 min=13180 min_rank=0 imbalance_ratio=1.164246 max_load_reduction=111997 max_load_reduction_pct=0.854469 moved_to_echo=71010 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 15, 3850, 0, 12, 24, 1, 2713, 0, 3937, 1, 52, 2, 760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3027, 4060, 10, 0, 1617, 155, 1377, 0, 0, 3945, 0, 0, 6, 0, 3874, 998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2465, 4096, 886, 1071, 7, 10, 4065, 4030, 7, 0, 0, 0, 0, 0, 2, 0, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1914, 5, 0, 3822, 0, 0, 555, 11, 19, 1, 2896, 4096, 6, 0, 3, 2057, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 1620, 4, 13, 15, 1, 8, 4060, 1757, 1429, 0, 0, 0, 4078, 0, 1539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4043, 0, 2187, 0, 4087, 397, 14, 0, 0, 0, 4, 132, 0, 4087, 401, 611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 877, 4091, 0, 6, 0, 0, 902, 0, 0, 4096, 0, 0, 4042, 0, 4, 1388, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 3365, 4088, 0, 22, 0, 2, 4058, 0, 0, 0, 3, 4, 0, 0, 3994, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15462, 19069, 18091, 15686, 14606, 15964, 15586, 16608] avg=16384.0000 max=19069 max_rank=1 min=14606 min_rank=4 imbalance_ratio=1.163879 max_load_reduction=112003 max_load_reduction_pct=0.854515 moved_to_echo=62764 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4026, 0, 0, 0, 103, 0, 3, 0, 0, 3997, 0, 2951, 2785, 0, 0, 807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1772, 4061, 0, 0, 1610, 310, 135, 0, 0, 4096, 1, 0, 0, 0, 60, 3930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 4096, 0, 4051, 0, 0, 4095, 1968, 0, 0, 28, 6, 0, 0, 0, 0, 1620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 14, 4077, 0, 0, 424, 0, 4, 0, 1256, 4077, 0, 2447, 0, 2186, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 1, 0, 0, 0, 0, 2, 1961, 163, 68, 0, 0, 3827, 0, 3834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 12, 750, 0, 4085, 0, 2136, 163, 2, 22, 0, 54, 0, 4095, 204, 607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 3785, 26, 0, 1, 3699, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 1381, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 4045, 4076, 0, 0, 0, 0, 4049, 169, 0, 0, 0, 0, 22, 0, 4064, 1103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14672, 15975, 16152, 18914, 13929, 16220, 17603, 17607] avg=16384.0000 max=18914 max_rank=3 min=13929 min_rank=4 imbalance_ratio=1.154419 max_load_reduction=112158 max_load_reduction_pct=0.855698 moved_to_echo=65359 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4030, 0, 0, 0, 2128, 54, 77, 142, 1, 4090, 0, 4016, 3273, 0, 0, 805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3114, 4051, 6, 17, 0, 310, 2828, 6, 7, 4096, 4, 0, 22, 0, 3769, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 4070, 33, 2512, 2, 116, 4086, 4095, 0, 38, 55, 0, 11, 0, 3, 1, 1620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 871, 0, 0, 4068, 21, 1, 20, 0, 23, 134, 1660, 3958, 3, 2496, 0, 2179, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 184, 0, 0, 22, 17, 0, 3313, 1960, 3610, 0, 105, 0, 4023, 0, 3525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 1518, 0, 3245, 0, 4070, 0, 556, 0, 0, 1, 0, 113, 92, 4040, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 4047, 0, 8, 0, 7, 513, 10, 26, 4065, 0, 0, 4086, 26, 23, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 0, 4023, 3892, 0, 0, 0, 0, 4018, 4, 11, 0, 0, 11, 0, 0, 3968, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18616, 18768, 16703, 15774, 16852, 13830, 13161, 17368] avg=16384.0000 max=18768 max_rank=1 min=13161 min_rank=6 imbalance_ratio=1.145508 max_load_reduction=112304 max_load_reduction_pct=0.856812 moved_to_echo=61211 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4068, 0, 0, 0, 3923, 0, 0, 36, 0, 4085, 0, 4058, 3407, 0, 0, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 1969, 19, 16, 1818, 24, 3404, 0, 0, 787, 325, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 3835, 0, 0, 4093, 4080, 2, 117, 1, 4, 40, 0, 2, 1, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 446, 0, 1, 4095, 10, 0, 162, 0, 0, 12, 3392, 4091, 0, 108, 0, 2099, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 3, 0, 3, 3, 0, 2449, 1954, 999, 1, 605, 0, 4017, 43, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4005, 1, 2012, 453, 4096, 694, 2662, 0, 1, 0, 0, 3, 2, 4081, 112, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 4096, 5, 0, 138, 2, 2908, 1, 0, 4089, 283, 6, 3979, 0, 0, 1561, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4033, 57, 23, 367, 0, 2040, 0, 0, 4, 1, 0, 0, 1, 3277, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20352, 12469, 17893, 14751, 14221, 18812, 17618, 14956] avg=16384.0000 max=20352 max_rank=0 min=12469 min_rank=1 imbalance_ratio=1.242188 max_load_reduction=110720 max_load_reduction_pct=0.844727 moved_to_echo=65607 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4095, 35, 81, 0, 549, 0, 286, 0, 0, 3990, 2, 4095, 38, 0, 8, 806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1763, 1567, 0, 0, 1830, 54, 2817, 1, 2, 2222, 0, 0, 0, 0, 3993, 3194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 4082, 0, 3, 8, 2, 4088, 4089, 0, 0, 0, 0, 0, 0, 28, 0, 1560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3379, 0, 0, 4096, 53, 0, 0, 0, 103, 1, 1677, 4052, 2, 0, 10, 2182, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3932, 86, 0, 16, 0, 0, 3614, 1889, 1472, 0, 0, 7, 4095, 2, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 70, 0, 4088, 9, 4045, 0, 1864, 0, 0, 0, 0, 8, 1, 4079, 2506, 691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1291, 4075, 2, 0, 0, 0, 699, 4, 0, 4096, 0, 0, 4091, 1, 0, 1570, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 3973, 0, 0, 0, 0, 4092, 0, 0, 0, 36, 0, 0, 1, 4013, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13985, 17443, 13999, 15878, 19087, 17362, 16001, 17317] avg=16384.0000 max=19087 max_rank=4 min=13985 min_rank=0 imbalance_ratio=1.164978 max_load_reduction=111985 max_load_reduction_pct=0.854378 moved_to_echo=69767 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[3200, 128, 31232, 27520, 128, 256, 384, 128, 28672, 256, 256, 256, 128, 128, 128, 128, 31232, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[29568, 640, 128, 128, 21888, 128, 2048, 896, 128, 27008, 128, 28800, 14464, 128, 128, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[256, 7552, 32256, 256, 128, 256, 5888, 7168, 128, 128, 32640, 512, 128, 32512, 384, 128, 10624, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 23040, 128, 384, 31872, 128, 128, 1408, 1024, 256, 896, 12672, 32512, 128, 8704, 128, 16128, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1408, 23808, 256, 128, 128, 512, 128, 22656, 15488, 7936, 256, 768, 2048, 31616, 512, 24576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3072, 32128, 1280, 19200, 128, 512, 32384, 28928, 128, 256, 512, 256, 256, 128, 128, 128, 12800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[16128, 21120, 128, 128, 12416, 2176, 15616, 128, 128, 25984, 384, 384, 128, 0, 16384, 20608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[512, 22272, 128, 15360, 2816, 32640, 1280, 12288, 640, 128, 768, 128, 896, 1280, 32128, 4480, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4025, 1, 0, 343, 4096, 8, 120, 3475, 0, 89, 3007, 0, 3885, 2261, 98, 671, 0, 3853, 3, 0, 0, 413, 0, 4, 0, 3695, 3866, 513, 3798, 62, 0, 4037, 0, 2, 0, 0, 0, 4093, 3970, 0, 0, 78, 7, 4016, 0, 0, 0, 3898, 2, 143, 4081, 1, 253, 0, 0, 4082, 209, 0, 0, 4096, 93, 20, 0, 0, 0, 11, 3, 3988, 1405, 0, 4062, 0, 35, 741, 0, 385, 0, 0, 0, 0, 0, 222, 711, 0, 4091, 0, 28, 4095, 0, 3976, 0, 0, 35, 88, 3839, 0, 0, 3183, 3319, 0, 2000, 208, 2, 4092, 3639, 4055, 491, 0, 0, 0, 7, 0, 0, 0, 0, 0, 6, 0, 28, 53, 4096, 0, 4095, 0, 0, 141, 2076, 499] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4069, 0, 0, 0, 4096, 14, 4060, 3286, 6, 0, 2106, 434, 3937, 1457, 0, 420, 0, 1679, 12, 2, 12, 0, 20, 0, 0, 4087, 4096, 3397, 4095, 30, 0, 3710, 20, 0, 0, 0, 0, 3968, 4051, 380, 7, 10, 34, 3870, 0, 2192, 1314, 1543, 4, 3948, 4024, 0, 256, 0, 6, 4096, 0, 0, 0, 4088, 0, 0, 13, 0, 73, 43, 177, 116, 49, 0, 3949, 0, 0, 20, 0, 59, 16, 19, 0, 0, 25, 0, 2281, 159, 1812, 5, 0, 4050, 0, 3208, 83, 58, 0, 2, 201, 0, 0, 3131, 4071, 0, 2972, 0, 0, 4084, 4084, 4084, 63, 0, 3, 11, 13, 13, 4, 0, 1, 32, 0, 0, 3, 0, 3991, 0, 4090, 7, 0, 0, 3981, 1180] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4058, 419, 0, 0, 4091, 0, 3578, 787, 0, 1, 4069, 4095, 4089, 0, 0, 16, 0, 45, 0, 0, 0, 60, 23, 0, 0, 3947, 2650, 4035, 4046, 6, 95, 2354, 280, 0, 0, 0, 71, 2258, 4043, 0, 93, 2, 71, 0, 863, 1659, 0, 4096, 0, 4089, 4072, 0, 1, 1, 0, 3451, 0, 0, 4, 3854, 49, 0, 0, 2, 8, 8, 2, 3743, 3960, 0, 4039, 0, 0, 3793, 0, 6, 0, 0, 0, 4, 0, 0, 676, 0, 1898, 56, 0, 4003, 0, 4092, 17, 0, 611, 74, 45, 0, 0, 0, 4094, 0, 4094, 190, 0, 4023, 4086, 99, 8, 0, 0, 28, 0, 374, 0, 0, 6, 0, 0, 17, 0, 267, 4096, 0, 4028, 1, 48, 0, 3942, 3213] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 19, 4096, 0, 1400, 4096, 0, 0, 863, 1, 4070, 68, 0, 5, 9, 4078, 0, 12, 0, 0, 0, 0, 2, 1320, 3838, 4056, 4094, 0, 141, 6, 0, 0, 0, 0, 62, 3996, 4092, 738, 0, 0, 80, 3953, 0, 1078, 0, 3862, 0, 4020, 4087, 0, 1, 0, 0, 3768, 0, 0, 0, 4010, 80, 14, 76, 52, 21, 611, 0, 0, 1324, 21, 4087, 58, 8, 4057, 13, 6, 2, 0, 277, 0, 10, 0, 0, 0, 4075, 0, 0, 4012, 4, 4070, 1, 2, 6, 0, 185, 0, 11, 3973, 4009, 0, 3087, 1, 2, 4045, 4079, 4096, 1503, 0, 0, 15, 104, 0, 3, 0, 0, 4, 0, 2, 0, 8, 3724, 0, 4090, 0, 28, 129, 3993, 1077] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4096, 5, 89, 0, 4095, 38, 4092, 4042, 0, 0, 3982, 6, 4096, 3931, 0, 0, 89, 1016, 0, 0, 0, 1, 0, 0, 0, 4096, 1072, 3303, 4096, 1, 0, 4046, 216, 0, 0, 0, 0, 4096, 4096, 7, 13, 0, 30, 2044, 0, 286, 169, 4091, 0, 3770, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 45, 0, 2, 0, 0, 3057, 22, 756, 7, 0, 3888, 0, 0, 0, 0, 2, 1, 0, 14, 0, 0, 0, 0, 0, 0, 2, 0, 3972, 0, 4096, 174, 0, 0, 0, 610, 4, 0, 3665, 4096, 0, 4082, 1, 0, 4082, 4096, 4096, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4067, 0, 4092, 0, 1, 27, 3919, 63] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4088, 4077, 0, 0, 95, 1, 3811, 4096, 1408, 28, 71, 3905, 3959, 21, 0, 547, 12, 3919, 15, 340, 9, 38, 133, 0, 508, 113, 2513, 1401, 3605, 31, 33, 0, 2, 0, 0, 0, 0, 1314, 4072, 3734, 83, 0, 14, 3713, 0, 4, 0, 1380, 20, 4096, 4078, 0, 575, 57, 2441, 3032, 435, 2, 348, 3899, 0, 16, 0, 277, 0, 14, 1, 3006, 0, 0, 0, 0, 53, 4081, 35, 895, 0, 0, 1596, 0, 13, 277, 4093, 50, 3472, 0, 0, 3493, 23, 4041, 65, 6, 2, 0, 3023, 0, 4, 647, 3775, 22, 4060, 3, 22, 3936, 4041, 3604, 1, 231, 0, 8, 0, 52, 0, 0, 0, 1, 126, 19, 6, 3, 4096, 0, 4038, 0, 0, 0, 1554, 215] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4031, 148, 0, 0, 3207, 310, 4074, 3980, 0, 1, 1466, 0, 3722, 0, 0, 0, 31, 3390, 0, 98, 0, 0, 8, 0, 0, 4096, 4092, 3332, 4091, 0, 4, 199, 184, 0, 0, 0, 1, 2988, 4078, 85, 9, 0, 50, 390, 0, 1, 0, 2309, 0, 4076, 4078, 0, 3, 5, 0, 4090, 0, 0, 0, 3998, 16, 434, 53, 3, 11, 2470, 8, 0, 4095, 0, 2078, 7, 0, 4070, 0, 1, 0, 12, 1721, 0, 1, 0, 3937, 0, 843, 253, 0, 3858, 232, 3981, 0, 0, 1, 0, 1989, 107, 1, 507, 4093, 0, 4059, 0, 0, 4063, 4090, 3601, 2589, 0, 0, 6, 12, 0, 0, 4, 0, 25, 22, 5, 0, 4, 3984, 3, 3963, 4, 3, 0, 3194, 34] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4092, 8, 95, 0, 3402, 54, 4018, 4096, 52, 10, 2839, 1, 4084, 0, 0, 0, 3, 961, 0, 0, 8, 685, 41, 0, 0, 4092, 4083, 649, 4094, 0, 3, 4076, 459, 1, 0, 0, 11, 4092, 4093, 0, 0, 4, 0, 1084, 910, 1, 0, 4093, 189, 4090, 4091, 0, 0, 0, 0, 3937, 0, 0, 0, 4094, 93, 0, 0, 5, 0, 1176, 0, 183, 3578, 0, 4060, 7, 8, 4079, 0, 0, 24, 18, 1200, 6, 0, 0, 185, 0, 3082, 0, 0, 4086, 0, 4092, 0, 6, 0, 0, 223, 20, 3, 0, 4095, 0, 539, 20, 0, 4084, 4094, 4092, 132, 0, 2, 0, 0, 1, 0, 0, 0, 4, 152, 0, 0, 5, 4095, 1, 4071, 0, 0, 0, 3889, 3067] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 14, 4060, 3286, 6, 0, 2106, 434, 906, 1457, 0, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1679, 12, 2, 12, 0, 20, 0, 0, 4087, 4096, 3397, 2357, 30, 0, 3710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 3968, 4051, 380, 7, 10, 34, 3870, 0, 2192, 1314, 1543, 782, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3948, 4024, 0, 256, 0, 6, 4096, 0, 0, 0, 4088, 0, 0, 13, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 43, 177, 116, 49, 0, 3949, 0, 0, 20, 0, 59, 16, 19, 0, 0, 4069, 1395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 2281, 159, 1812, 5, 0, 4050, 0, 3208, 83, 58, 0, 2, 201, 0, 2040, 956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3131, 4071, 0, 2972, 0, 0, 0, 3146, 4084, 63, 0, 3, 11, 13, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 32, 0, 0, 3, 0, 3991, 0, 4090, 7, 0, 0, 3981, 1180, 2689, 991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16785, 19402, 18933, 16611, 9985, 14880, 17507, 16969] avg=16384.0000 max=19402 max_rank=1 min=9985 min_rank=4 imbalance_ratio=1.184204 max_load_reduction=111670 max_load_reduction_pct=0.851974 moved_to_echo=59341 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 343, 4096, 8, 120, 3475, 0, 89, 3007, 0, 894, 2261, 98, 671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 3, 0, 0, 413, 0, 4, 0, 3695, 3866, 513, 2187, 62, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4093, 3970, 0, 0, 78, 7, 4016, 0, 0, 0, 3898, 725, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 143, 4081, 1, 253, 0, 0, 4082, 209, 0, 0, 4096, 93, 20, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3, 3988, 1405, 0, 4062, 0, 35, 741, 0, 385, 0, 0, 0, 0, 4025, 1398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 711, 0, 4091, 0, 28, 4095, 0, 3976, 0, 0, 35, 88, 3839, 0, 2013, 886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3183, 3319, 0, 2000, 208, 2, 0, 2803, 4055, 491, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 28, 53, 4096, 0, 4095, 0, 0, 141, 2076, 499, 2694, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15063, 18633, 17468, 13137, 16053, 19984, 16068, 14666] avg=16384.0000 max=19984 max_rank=5 min=13137 min_rank=3 imbalance_ratio=1.219727 max_load_reduction=111088 max_load_reduction_pct=0.847534 moved_to_echo=66771 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 19, 4096, 0, 1400, 4096, 0, 0, 863, 1, 936, 68, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4078, 0, 12, 0, 0, 0, 0, 2, 1320, 3838, 4056, 2356, 0, 141, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 3996, 4092, 738, 0, 0, 80, 3953, 0, 1078, 0, 3862, 782, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 4087, 0, 1, 0, 0, 3768, 0, 0, 0, 4010, 80, 14, 76, 52, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 611, 0, 0, 1324, 21, 4087, 58, 8, 4057, 13, 6, 2, 0, 277, 0, 4096, 1382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4075, 0, 0, 4012, 4, 4070, 1, 2, 6, 0, 185, 0, 2109, 956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3973, 4009, 0, 3087, 1, 2, 0, 3142, 4096, 1503, 0, 0, 15, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4, 0, 2, 0, 8, 3724, 0, 4090, 0, 28, 129, 3993, 1077, 2663, 1025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11484, 15818, 19404, 16284, 15963, 15430, 19943, 16746] avg=16384.0000 max=19943 max_rank=6 min=11484 min_rank=0 imbalance_ratio=1.217224 max_load_reduction=111129 max_load_reduction_pct=0.847847 moved_to_echo=68082 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 148, 0, 0, 3207, 310, 4074, 3980, 0, 1, 1466, 0, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3390, 0, 98, 0, 0, 8, 0, 0, 4096, 4092, 3332, 2355, 0, 4, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 1, 2988, 4078, 85, 9, 0, 50, 390, 0, 1, 0, 2309, 781, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4078, 0, 3, 5, 0, 4090, 0, 0, 0, 3998, 16, 434, 53, 3, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2470, 8, 0, 4095, 0, 2078, 7, 0, 4070, 0, 1, 0, 12, 1721, 0, 4031, 1389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3937, 0, 843, 253, 0, 3858, 232, 3981, 0, 0, 1, 0, 1989, 107, 1928, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 507, 4093, 0, 4059, 0, 0, 0, 3151, 3601, 2589, 0, 0, 6, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 25, 22, 5, 0, 4, 3984, 3, 3963, 4, 3, 0, 3194, 34, 2674, 937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14043, 17605, 11639, 16932, 19893, 18085, 18019, 14856] avg=16384.0000 max=19893 max_rank=4 min=11639 min_rank=2 imbalance_ratio=1.214172 max_load_reduction=111179 max_load_reduction_pct=0.848228 moved_to_echo=70853 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 4077, 0, 0, 95, 1, 3811, 4096, 1408, 28, 71, 3905, 904, 21, 0, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3919, 15, 340, 9, 38, 133, 0, 508, 113, 2513, 1401, 2066, 31, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1314, 4072, 3734, 83, 0, 14, 3713, 0, 4, 0, 1380, 694, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 4078, 0, 575, 57, 2441, 3032, 435, 2, 348, 3899, 0, 16, 0, 277, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 3006, 0, 0, 0, 0, 53, 4081, 35, 895, 0, 0, 1596, 0, 4088, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 277, 4093, 50, 3472, 0, 0, 3493, 23, 4041, 65, 6, 2, 0, 3023, 0, 2054, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 647, 3775, 22, 4060, 3, 22, 0, 3104, 3604, 1, 231, 0, 8, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 126, 19, 6, 3, 4096, 0, 4038, 0, 0, 0, 1554, 215, 2590, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18964, 11131, 15769, 19454, 15115, 21457, 15533, 13649] avg=16384.0000 max=21457 max_rank=5 min=11131 min_rank=1 imbalance_ratio=1.309631 max_load_reduction=109615 max_load_reduction_pct=0.836296 moved_to_echo=65754 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 419, 0, 0, 4091, 0, 3578, 787, 0, 1, 4069, 4095, 941, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 60, 23, 0, 0, 3947, 2650, 4035, 2329, 6, 95, 2354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 0, 71, 2258, 4043, 0, 93, 2, 71, 0, 863, 1659, 0, 4096, 773, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4072, 0, 1, 1, 0, 3451, 0, 0, 4, 3854, 49, 0, 0, 2, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 2, 3743, 3960, 0, 4039, 0, 0, 3793, 0, 6, 0, 0, 0, 4, 4058, 1376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 0, 1898, 56, 0, 4003, 0, 4092, 17, 0, 611, 74, 45, 0, 2118, 944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 190, 0, 0, 3148, 99, 8, 0, 0, 28, 0, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 17, 0, 267, 4096, 0, 4028, 1, 48, 0, 3942, 3213, 2647, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17997, 15544, 14971, 15699, 20997, 14534, 12035, 19295] avg=16384.0000 max=20997 max_rank=4 min=12035 min_rank=6 imbalance_ratio=1.281555 max_load_reduction=110075 max_load_reduction_pct=0.839806 moved_to_echo=66861 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 89, 0, 4095, 38, 4092, 4042, 0, 0, 3982, 6, 942, 3931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 1016, 0, 0, 0, 1, 0, 0, 0, 4096, 1072, 3303, 2358, 1, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 0, 0, 4096, 4096, 7, 13, 0, 30, 2044, 0, 286, 169, 4091, 782, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3770, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 45, 0, 2, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3057, 22, 756, 7, 0, 3888, 0, 0, 0, 0, 2, 1, 0, 14, 0, 4096, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3972, 0, 4096, 174, 0, 0, 0, 610, 4, 2122, 956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3665, 4096, 0, 4082, 1, 0, 0, 3156, 4096, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4067, 0, 4092, 0, 1, 27, 3919, 63, 2686, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21222, 15982, 16594, 16280, 13239, 11936, 19929, 15890] avg=16384.0000 max=21222 max_rank=0 min=11936 min_rank=5 imbalance_ratio=1.295288 max_load_reduction=109850 max_load_reduction_pct=0.838089 moved_to_echo=60994 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 95, 0, 3402, 54, 4018, 4096, 52, 10, 2839, 1, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 961, 0, 0, 8, 685, 41, 0, 0, 4092, 4083, 649, 2356, 0, 3, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 1, 0, 0, 11, 4092, 4093, 0, 0, 4, 0, 1084, 910, 1, 0, 4093, 782, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 4090, 4091, 0, 0, 0, 0, 3937, 0, 0, 0, 4094, 93, 0, 0, 5, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1176, 0, 183, 3578, 0, 4060, 7, 8, 4079, 0, 0, 24, 18, 1200, 6, 4092, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 3082, 0, 0, 4086, 0, 4092, 0, 6, 0, 0, 223, 20, 2116, 956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 539, 20, 0, 0, 3154, 4092, 132, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 152, 0, 0, 5, 4095, 1, 4071, 0, 0, 0, 3889, 3067, 2688, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15514, 16957, 16294, 16675, 19827, 14766, 12038, 19001] avg=16384.0000 max=19827 max_rank=4 min=12038 min_rank=6 imbalance_ratio=1.210144 max_load_reduction=111245 max_load_reduction_pct=0.848732 moved_to_echo=65632 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 15232, 31616, 128, 24960, 512, 128, 0, 24832, 27776, 5632, 256, 128, 128, 256, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 28288, 32640, 128, 1152, 128, 2560, 30592, 768, 128, 384, 32256, 384, 512, 256, 384, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 4736, 256, 384, 27264, 512, 25216, 27904, 1536, 256, 18432, 8448, 7424, 7808, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 18944, 128, 512, 128, 1280, 256, 128, 512, 25472, 26240, 20736, 18432, 256, 384, 18432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 384, 128, 128, 384, 32256, 128, 32512, 128, 128, 384, 26624, 9472, 21376, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1280, 128, 0, 0, 256, 26880, 32512, 4992, 256, 128, 384, 19072, 1792, 5248, 1536, 25344, 6144, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 7424, 256, 11904, 14464, 128, 26240, 128, 128, 20864, 128, 1408, 128, 128, 4864, 128, 32640, 11136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 512, 11904, 256, 19328, 384, 128, 31616, 384, 31616, 384, 128, 768, 256, 10240, 256, 16512, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 4096, 0, 1, 0, 0, 0, 1136, 13, 1, 4093, 0, 0, 0, 2392, 1, 0, 0, 4067, 584, 4022, 4078, 27, 4096, 0, 12, 0, 4053, 105, 3942, 0, 679, 3073, 8, 1602, 3869, 2, 1771, 0, 2952, 0, 4096, 0, 537, 1, 3916, 0, 0, 0, 0, 0, 0, 49, 4096, 4059, 23, 13, 3, 7, 3692, 0, 2, 0, 4087, 0, 10, 0, 4096, 4003, 0, 0, 5, 0, 7, 0, 6, 85, 3, 0, 224, 0, 350, 46, 110, 4063, 0, 0, 721, 16, 0, 360, 4, 3, 4092, 4076, 3925, 24, 33, 3807, 299, 0, 4093, 4069, 4096, 33, 0, 0, 1, 0, 831, 0, 1, 119, 0, 1, 0, 5, 4096, 0, 0, 4071, 0, 4021, 0, 3, 0, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 1, 57, 0, 1071, 0, 0, 4096, 0, 0, 1, 3245, 0, 0, 0, 4090, 10, 4095, 4096, 0, 4096, 0, 0, 0, 4096, 2909, 4096, 0, 30, 1080, 0, 1467, 4096, 0, 2949, 0, 4046, 0, 132, 0, 0, 0, 870, 0, 0, 0, 0, 2569, 0, 2, 4096, 4094, 0, 0, 0, 0, 4096, 0, 0, 55, 4096, 48, 2959, 0, 3674, 3598, 0, 0, 2, 0, 0, 0, 0, 353, 204, 0, 0, 0, 13, 0, 1, 4095, 0, 0, 1, 61, 0, 0, 0, 0, 4064, 4096, 4095, 0, 0, 4096, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 12, 1309, 0, 0, 83, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[1, 18, 4092, 23, 0, 0, 0, 0, 39, 0, 0, 3955, 0, 0, 1, 0, 66, 1, 0, 3998, 161, 4084, 4096, 9, 4006, 421, 0, 0, 3858, 411, 3782, 77, 907, 1959, 60, 508, 4049, 185, 2981, 0, 3933, 194, 3177, 5, 0, 0, 2446, 0, 5, 0, 693, 0, 128, 1509, 4096, 4074, 0, 0, 1, 5, 4096, 94, 0, 18, 3931, 122, 4050, 141, 4057, 3753, 0, 2, 1119, 0, 0, 4, 8, 0, 0, 69, 0, 0, 0, 0, 23, 4048, 40, 7, 1905, 0, 0, 1, 0, 8, 3297, 3967, 4095, 60, 2, 4070, 0, 8, 4074, 3591, 3852, 17, 147, 0, 0, 0, 314, 0, 0, 0, 0, 0, 21, 0, 3917, 0, 0, 3982, 93, 4025, 30, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 97, 5, 5, 1, 0, 0, 4096, 0, 0, 0, 1, 0, 1, 0, 4096, 119, 4071, 4061, 0, 3995, 0, 0, 0, 4094, 0, 4095, 1, 8, 2085, 0, 17, 4095, 4, 3616, 0, 3371, 0, 843, 18, 1768, 34, 1871, 0, 0, 14, 0, 4071, 0, 0, 4089, 3842, 0, 0, 182, 0, 4096, 0, 0, 0, 3811, 22, 2347, 0, 4085, 4096, 1, 3, 11, 0, 1, 8, 0, 49, 2, 18, 396, 0, 72, 0, 0, 4077, 7, 0, 778, 120, 0, 0, 14, 5, 4095, 3992, 3737, 0, 0, 4020, 0, 7, 4083, 4089, 4022, 72, 0, 0, 7, 0, 3865, 0, 0, 0, 0, 0, 0, 13, 4073, 14, 10, 4090, 0, 4096, 0, 7, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 3932, 3067, 0, 7, 16, 1, 0, 0, 0, 4071, 2, 0, 0, 0, 0, 0, 0, 4091, 0, 4080, 3880, 27, 4094, 0, 0, 4, 4096, 0, 4096, 37, 0, 1537, 0, 3, 4094, 3, 3505, 87, 4060, 0, 4096, 118, 23, 0, 4043, 1, 2, 0, 0, 2688, 0, 805, 4086, 3987, 284, 0, 0, 0, 2304, 36, 0, 1, 3250, 0, 2451, 0, 4035, 3735, 345, 7, 1, 131, 151, 0, 0, 0, 0, 0, 6, 0, 359, 0, 0, 3976, 0, 1426, 89, 0, 0, 0, 12, 0, 1, 4051, 3960, 0, 22, 4034, 1, 0, 3994, 4002, 4084, 0, 0, 33, 0, 1, 3162, 170, 46, 0, 25, 0, 0, 0, 4013, 18, 2, 4058, 0, 4095, 48, 0, 11, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 4092, 5, 0, 228, 0, 0, 74, 0, 0, 4019, 0, 0, 999, 421, 1, 0, 0, 4096, 0, 4094, 4092, 0, 4070, 0, 1, 0, 3499, 0, 4052, 0, 0, 0, 0, 0, 4095, 7, 4092, 0, 3951, 2, 614, 2, 0, 11, 4096, 0, 0, 1, 127, 4064, 3, 0, 4094, 4040, 514, 2, 0, 0, 2174, 0, 13, 0, 4096, 54, 0, 7, 4046, 3239, 0, 5, 3962, 0, 1, 0, 24, 16, 0, 0, 379, 0, 12, 64, 0, 3998, 0, 4062, 0, 3, 0, 2, 0, 0, 4013, 4023, 3976, 3, 60, 4093, 0, 1, 4096, 4087, 4094, 0, 0, 0, 5, 7, 0, 0, 0, 0, 0, 0, 2, 229, 4096, 39, 6, 4096, 6, 4042, 482, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 17, 4000, 0, 0, 2, 0, 413, 0, 0, 0, 2691, 0, 0, 0, 4093, 69, 0, 0, 4096, 1, 4049, 3863, 79, 3374, 0, 71, 101, 3704, 0, 3323, 46, 966, 165, 0, 1640, 3955, 55, 4096, 384, 175, 0, 4078, 0, 0, 0, 4086, 22, 0, 1, 272, 3192, 0, 0, 4067, 3316, 0, 350, 66, 114, 3579, 2, 25, 0, 4092, 2, 4096, 0, 4000, 3870, 0, 0, 1151, 3, 0, 0, 425, 710, 3, 0, 0, 79, 1, 302, 10, 4095, 0, 0, 13, 155, 0, 0, 337, 0, 126, 3900, 3564, 4, 0, 3846, 7, 0, 3979, 4086, 3939, 0, 0, 0, 0, 3, 0, 0, 0, 0, 154, 1, 35, 517, 3828, 3189, 95, 4096, 13, 3705, 0, 1, 42, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4091, 21, 0, 75, 47, 0, 15, 4, 2, 4049, 0, 0, 82, 0, 1, 32, 1, 4033, 0, 4096, 4094, 0, 4092, 0, 0, 0, 4095, 5, 4096, 0, 0, 3954, 11, 0, 3955, 0, 4079, 12, 3245, 0, 2218, 3, 1, 998, 4096, 0, 0, 0, 0, 155, 0, 0, 4094, 3890, 0, 0, 14, 0, 4080, 0, 0, 0, 4096, 0, 2224, 0, 3470, 4061, 0, 1, 425, 1, 0, 0, 0, 0, 0, 0, 4058, 0, 21, 0, 0, 4077, 0, 705, 192, 0, 0, 0, 5, 0, 151, 4095, 3715, 1, 1, 4079, 0, 0, 4030, 4063, 4091, 0, 0, 0, 6, 73, 4005, 1, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 5, 3496, 0, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4096, 0, 1, 0, 0, 0, 1136, 13, 1, 4093, 0, 0, 0, 2392, 4022, 2149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 584, 0, 3904, 27, 4096, 0, 12, 0, 4053, 105, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 679, 3073, 8, 1602, 882, 2, 1771, 0, 2952, 0, 4096, 0, 537, 1, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 4096, 4059, 23, 13, 3, 7, 3692, 0, 2, 0, 1720, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 10, 0, 4096, 4003, 0, 0, 5, 0, 7, 0, 6, 85, 3, 0, 1086, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 350, 46, 110, 4063, 0, 0, 721, 16, 0, 360, 4, 3, 4092, 4076, 2330, 1267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3925, 24, 33, 3807, 299, 0, 0, 4069, 2701, 33, 0, 0, 1, 0, 831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 119, 0, 1, 0, 5, 4096, 0, 0, 4071, 0, 4021, 0, 3, 0, 7, 1918, 1763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17904, 16724, 19519, 13973, 13562, 17662, 15723, 16005] avg=16384.0000 max=19519 max_rank=2 min=13562 min_rank=4 imbalance_ratio=1.191345 max_load_reduction=111553 max_load_reduction_pct=0.851082 moved_to_echo=62952 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 18, 4092, 23, 0, 0, 0, 0, 39, 0, 0, 3955, 0, 0, 1, 0, 4084, 2113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 1, 0, 0, 161, 0, 3921, 9, 4006, 421, 0, 0, 3858, 411, 3782, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 907, 1959, 60, 508, 923, 185, 2981, 0, 3933, 194, 3177, 5, 0, 0, 2446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 693, 0, 128, 1509, 4096, 4074, 0, 0, 1, 5, 4096, 94, 0, 18, 1800, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3931, 122, 4050, 141, 4057, 3753, 0, 2, 1119, 0, 0, 4, 8, 0, 0, 69, 1021, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4048, 40, 7, 1905, 0, 0, 1, 0, 8, 3297, 3967, 2319, 1326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 60, 2, 4070, 0, 8, 0, 3591, 2540, 17, 147, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 3917, 0, 0, 3982, 93, 4025, 30, 0, 0, 0, 1885, 1755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14326, 16713, 17278, 16810, 18452, 16941, 14844, 15708] avg=16384.0000 max=18452 max_rank=4 min=14326 min_rank=0 imbalance_ratio=1.126221 max_load_reduction=112620 max_load_reduction_pct=0.859222 moved_to_echo=65945 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 97, 5, 5, 1, 0, 0, 4096, 0, 0, 0, 1, 4071, 2166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 119, 0, 3888, 0, 3995, 0, 0, 0, 4094, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2085, 0, 17, 934, 4, 3616, 0, 3371, 0, 843, 18, 1768, 34, 1871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4071, 0, 0, 4089, 3842, 0, 0, 182, 0, 4096, 0, 0, 0, 1820, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3811, 22, 2347, 0, 4085, 4096, 1, 3, 11, 0, 1, 8, 0, 49, 2, 18, 1066, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 72, 0, 0, 4077, 7, 0, 778, 120, 0, 0, 14, 5, 4095, 3992, 2325, 1341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3737, 0, 0, 4020, 0, 7, 0, 4089, 2652, 72, 0, 0, 7, 0, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4073, 14, 10, 4090, 0, 4096, 0, 7, 0, 0, 1930, 1758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14537, 16193, 14569, 18418, 15693, 17222, 18449, 15991] avg=16384.0000 max=18449 max_rank=6 min=14537 min_rank=0 imbalance_ratio=1.126038 max_load_reduction=112623 max_load_reduction_pct=0.859245 moved_to_echo=67355 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 1, 57, 0, 1071, 0, 0, 4096, 0, 0, 1, 3245, 4095, 2162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3921, 0, 4096, 0, 0, 0, 4096, 2909, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 1080, 0, 1467, 934, 0, 2949, 0, 4046, 0, 132, 0, 0, 0, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2569, 0, 2, 4096, 4094, 0, 0, 0, 0, 4096, 0, 0, 55, 1820, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 48, 2959, 0, 3674, 3598, 0, 0, 2, 0, 0, 0, 0, 353, 204, 0, 1086, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 4095, 0, 0, 1, 61, 0, 0, 0, 0, 4064, 4096, 2332, 1342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4095, 2701, 0, 0, 0, 0, 12, 1309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1928, 1764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18824, 19128, 11508, 17041, 16195, 16005, 16308, 16063] avg=16384.0000 max=19128 max_rank=1 min=11508 min_rank=2 imbalance_ratio=1.167480 max_load_reduction=111944 max_load_reduction_pct=0.854065 moved_to_echo=64571 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 17, 4000, 0, 0, 2, 0, 413, 0, 0, 0, 2691, 0, 0, 0, 4093, 4049, 2166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 1, 0, 3695, 79, 3374, 0, 71, 101, 3704, 0, 3323, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 966, 165, 0, 1640, 894, 55, 4096, 384, 175, 0, 4078, 0, 0, 0, 4086, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 272, 3192, 0, 0, 4067, 3316, 0, 350, 66, 114, 3579, 2, 25, 0, 1761, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 4096, 0, 4000, 3870, 0, 0, 1151, 3, 0, 0, 425, 710, 3, 0, 1049, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 1, 302, 10, 4095, 0, 0, 13, 155, 0, 0, 337, 0, 126, 3900, 2266, 1300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3564, 4, 0, 3846, 7, 0, 0, 4086, 2588, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 1, 35, 517, 3828, 3189, 95, 4096, 13, 3705, 0, 1, 42, 0, 1930, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17431, 14463, 16561, 17047, 19569, 12584, 14098, 19319] avg=16384.0000 max=19569 max_rank=4 min=12584 min_rank=5 imbalance_ratio=1.194397 max_load_reduction=111503 max_load_reduction_pct=0.850700 moved_to_echo=65570 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3932, 3067, 0, 7, 16, 1, 0, 0, 0, 4071, 2, 0, 0, 0, 4080, 2162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3715, 27, 4094, 0, 0, 4, 4096, 0, 4096, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1537, 0, 3, 933, 3, 3505, 87, 4060, 0, 4096, 118, 23, 0, 4043, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2688, 0, 805, 4086, 3987, 284, 0, 0, 0, 2304, 36, 0, 1, 1820, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3250, 0, 2451, 0, 4035, 3735, 345, 7, 1, 131, 151, 0, 0, 0, 0, 0, 1083, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 359, 0, 0, 3976, 0, 1426, 89, 0, 0, 0, 12, 0, 1, 4051, 2274, 1341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 0, 22, 4034, 1, 0, 0, 4002, 2692, 0, 0, 33, 0, 1, 3162, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 25, 0, 0, 0, 4013, 18, 2, 4058, 0, 4095, 48, 0, 11, 3, 1929, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17338, 16069, 18409, 16322, 15354, 13535, 18077, 15968] avg=16384.0000 max=18409 max_rank=2 min=13535 min_rank=5 imbalance_ratio=1.123596 max_load_reduction=112663 max_load_reduction_pct=0.859550 moved_to_echo=62934 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4092, 5, 0, 228, 0, 0, 74, 0, 0, 4019, 0, 0, 999, 421, 4094, 2166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3918, 0, 4070, 0, 1, 0, 3499, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 934, 7, 4092, 0, 3951, 2, 614, 2, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 127, 4064, 3, 0, 4094, 4040, 514, 2, 0, 0, 2174, 0, 13, 0, 1820, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 54, 0, 7, 4046, 3239, 0, 5, 3962, 0, 1, 0, 24, 16, 0, 0, 1085, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 12, 64, 0, 3998, 0, 4062, 0, 3, 0, 2, 0, 0, 4013, 4023, 2333, 1341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 3, 60, 4093, 0, 1, 0, 4087, 2700, 0, 0, 0, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 229, 4096, 39, 6, 4096, 6, 4042, 482, 0, 0, 0, 1930, 1763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16099, 15541, 13709, 17161, 16709, 20230, 14932, 16691] avg=16384.0000 max=20230 max_rank=5 min=13709 min_rank=2 imbalance_ratio=1.234741 max_load_reduction=110842 max_load_reduction_pct=0.845657 moved_to_echo=68562 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 21, 0, 75, 47, 0, 15, 4, 2, 4049, 0, 0, 82, 0, 4096, 2131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 32, 1, 0, 0, 0, 3919, 0, 4092, 0, 0, 0, 4095, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 11, 0, 902, 0, 4079, 12, 3245, 0, 2218, 3, 1, 998, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 4094, 3890, 0, 0, 14, 0, 4080, 0, 0, 0, 1758, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2224, 0, 3470, 4061, 0, 1, 425, 1, 0, 0, 0, 0, 0, 0, 1085, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 21, 0, 0, 4077, 0, 705, 192, 0, 0, 0, 5, 0, 151, 4095, 2294, 1295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3715, 1, 1, 4079, 0, 0, 0, 4063, 2697, 0, 0, 0, 6, 73, 4005, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 5, 3496, 0, 0, 2, 0, 1902, 1736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14613, 16241, 19519, 14300, 15538, 16893, 18641, 15327] avg=16384.0000 max=19519 max_rank=2 min=14300 min_rank=3 imbalance_ratio=1.191345 max_load_reduction=111553 max_load_reduction_pct=0.851082 moved_to_echo=66399 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[31104, 128, 128, 32128, 384, 128, 0, 32128, 21376, 128, 256, 128, 128, 128, 13568, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 32512, 3200, 128, 512, 128, 512, 2432, 128, 128, 31104, 128, 0, 1152, 10240, 32640, 17280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 128, 128, 0, 896, 0, 30976, 256, 31872, 512, 128, 128, 31616, 3456, 31488, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[31488, 256, 18176, 256, 31488, 30464, 384, 128, 6784, 256, 256, 128, 512, 1280, 256, 128, 8576, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 1152, 16768, 256, 2432, 32768, 31360, 896, 384, 384, 128, 28160, 256, 128, 128, 14336, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2688, 13952, 128, 5248, 7424, 256, 27136, 512, 25856, 256, 19328, 256, 2432, 1152, 25472, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[5120, 128, 896, 512, 256, 32512, 128, 6272, 3712, 384, 0, 384, 384, 128, 19840, 32256, 18560, 10624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 256, 256, 128, 128, 768, 32256, 3328, 128, 32640, 128, 31616, 640, 128, 128, 128, 15360, 14080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4019, 4, 24, 0, 12, 8, 10, 548, 4094, 15, 53, 0, 0, 0, 4046, 275, 40, 1304, 3270, 1, 324, 4095, 4074, 3, 0, 62, 0, 0, 37, 115, 536, 2517, 0, 0, 1, 38, 3525, 4052, 16, 0, 4094, 315, 0, 0, 0, 1, 2, 0, 0, 0, 85, 0, 4096, 0, 4064, 3966, 2, 8, 4064, 1, 3, 3961, 0, 0, 9, 1572, 5, 21, 0, 4096, 56, 0, 36, 0, 28, 0, 0, 34, 3314, 3633, 4085, 0, 0, 22, 347, 0, 0, 3688, 4075, 0, 18, 0, 4091, 3079, 3749, 4029, 0, 4096, 0, 0, 4074, 20, 0, 25, 0, 0, 0, 224, 4051, 0, 4028, 0, 0, 1, 384, 0, 0, 292, 0, 0, 0, 9, 3940, 4096, 32, 4033, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 3976, 1636, 28, 0, 0, 1, 0, 219, 4096, 0, 20, 7, 5, 0, 1, 0, 0, 3561, 245, 0, 3739, 10, 4096, 45, 0, 0, 0, 85, 54, 3446, 142, 2597, 7, 0, 0, 0, 4086, 4096, 0, 0, 3469, 2610, 60, 0, 2, 0, 3, 0, 16, 8, 0, 783, 4083, 0, 4012, 4085, 0, 0, 4091, 27, 3799, 500, 0, 4096, 0, 4038, 0, 0, 5, 4041, 0, 0, 27, 0, 0, 1, 119, 52, 0, 4096, 4046, 8, 235, 0, 375, 0, 267, 4096, 4057, 0, 1, 11, 4096, 2175, 3009, 0, 0, 3781, 0, 0, 4093, 0, 8, 0, 0, 0, 0, 859, 3286, 0, 4069, 0, 456, 0, 3626, 0, 0, 108, 15, 0, 0, 0, 28, 4069, 1, 4063, 13] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[8, 4096, 2441, 0, 10, 0, 0, 0, 3524, 4052, 0, 0, 3, 2, 1429, 51, 3, 0, 1929, 2850, 0, 540, 533, 3989, 0, 0, 1, 2, 0, 47, 824, 551, 1431, 2, 11, 0, 162, 4095, 4077, 2, 25, 3274, 31, 134, 0, 75, 0, 4, 0, 1, 5, 3, 0, 3988, 51, 4076, 3934, 0, 0, 3515, 2, 3798, 467, 0, 4088, 0, 4093, 2, 17, 0, 3921, 0, 25, 0, 30, 0, 66, 0, 0, 768, 4024, 4016, 0, 0, 4086, 0, 0, 64, 4095, 4092, 0, 0, 7, 4058, 0, 4012, 1565, 18, 3140, 116, 0, 4094, 0, 15, 0, 0, 2, 0, 0, 4049, 48, 3998, 17, 243, 3, 3600, 0, 36, 232, 0, 9, 1, 257, 5, 4035, 0, 4077, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 745, 0, 366, 0, 0, 101, 0, 4039, 56, 1, 0, 0, 18, 2, 63, 0, 1316, 3395, 250, 3520, 3913, 4096, 0, 0, 0, 143, 0, 0, 28, 0, 0, 0, 0, 0, 1379, 3684, 4096, 142, 0, 3783, 2, 21, 694, 0, 0, 0, 981, 0, 0, 0, 3185, 4094, 216, 4085, 4096, 4, 0, 4096, 401, 126, 0, 8, 0, 39, 3422, 0, 844, 0, 3687, 0, 0, 0, 0, 0, 32, 0, 0, 3851, 3914, 4028, 461, 0, 0, 14, 0, 0, 3495, 4015, 14, 0, 0, 3982, 2504, 4096, 4096, 0, 3166, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 269, 3530, 0, 3913, 0, 0, 0, 1, 0, 0, 0, 0, 4, 1, 119, 4095, 4096, 0, 4062, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 3371, 7, 0, 533, 0, 2, 192, 0, 4095, 20, 250, 20, 0, 0, 4084, 9, 2, 22, 0, 0, 3953, 3, 3850, 0, 1, 0, 0, 87, 1662, 0, 2959, 2265, 0, 0, 0, 1664, 3972, 3142, 0, 2, 2212, 0, 1421, 0, 15, 4084, 0, 71, 10, 4081, 0, 0, 2907, 0, 3530, 3488, 0, 25, 2211, 356, 0, 145, 0, 1520, 3, 26, 15, 8, 696, 3083, 0, 850, 0, 761, 0, 3378, 0, 1749, 3691, 4096, 4010, 139, 5, 0, 0, 41, 15, 4068, 1631, 6, 0, 0, 4020, 3438, 4040, 3827, 0, 3476, 14, 2, 3171, 335, 269, 4, 10, 0, 3, 1228, 3507, 0, 3963, 0, 3719, 0, 99, 0, 408, 4, 0, 830, 0, 122, 0, 2706, 12, 1351, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 308, 0, 2, 0, 8, 0, 0, 3999, 1713, 0, 2699, 0, 0, 4032, 1, 0, 0, 1531, 4, 4081, 2284, 4094, 0, 0, 10, 0, 0, 0, 726, 0, 374, 97, 17, 0, 48, 3623, 3506, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 26, 0, 84, 1, 0, 4095, 10, 4094, 4096, 32, 0, 3158, 246, 4088, 1166, 1, 0, 0, 924, 0, 0, 1684, 2537, 0, 1, 0, 0, 0, 0, 84, 0, 3044, 3974, 4047, 274, 10, 4082, 159, 0, 0, 3815, 4005, 35, 0, 0, 4095, 1, 180, 1011, 0, 4085, 116, 28, 4018, 277, 0, 0, 3, 0, 0, 0, 4096, 0, 4059, 0, 1295, 0, 3312, 0, 1, 828, 0, 0, 5, 385, 4041, 4049, 0, 4065, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 3769, 0, 6, 0, 0, 0, 380, 4081, 0, 12, 10, 0, 177, 2, 9, 0, 256, 2605, 0, 0, 4096, 4096, 0, 0, 2, 0, 1, 16, 1, 23, 4096, 0, 0, 0, 218, 4096, 2377, 0, 0, 4026, 202, 0, 2234, 0, 0, 0, 3, 0, 0, 0, 3, 4050, 511, 4096, 4096, 0, 0, 4096, 0, 509, 4095, 0, 3, 0, 1437, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 69, 0, 1185, 4078, 4094, 2960, 0, 125, 69, 0, 0, 4096, 4036, 0, 0, 0, 4096, 3807, 4095, 2783, 0, 4059, 2, 0, 4096, 0, 0, 0, 189, 0, 0, 101, 4069, 0, 4072, 0, 6, 0, 91, 0, 0, 0, 0, 0, 0, 1821, 1005, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[23, 4096, 1585, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 72, 0, 84, 242, 67, 0, 0, 1832, 0, 4053, 0, 4075, 0, 0, 1, 0, 0, 28, 2924, 4088, 92, 3, 0, 1, 6, 4055, 2947, 0, 0, 3201, 413, 3, 0, 0, 1, 0, 0, 0, 47, 7, 2937, 3875, 0, 4096, 4038, 0, 0, 4095, 0, 6, 4089, 0, 4089, 0, 3355, 1, 0, 964, 4082, 0, 3, 91, 0, 0, 25, 25, 0, 120, 3834, 3945, 4095, 0, 3, 47, 0, 0, 4086, 4029, 43, 2, 8, 4060, 201, 807, 4074, 0, 4096, 383, 0, 4093, 1, 0, 0, 1917, 0, 0, 0, 2913, 0, 4003, 0, 163, 19, 0, 0, 0, 0, 3, 3, 0, 0, 2120, 4090, 11, 4092, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4019, 4, 24, 0, 12, 8, 10, 548, 4094, 15, 53, 0, 0, 0, 4046, 2995, 932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 40, 1304, 3270, 1, 324, 4095, 4074, 3, 0, 62, 0, 0, 37, 115, 536, 1579, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2517, 0, 0, 1, 38, 3525, 4052, 16, 0, 4094, 315, 0, 0, 0, 1, 2, 1344, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 4096, 0, 0, 3688, 2, 8, 4064, 1, 3, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1572, 5, 21, 0, 4096, 56, 0, 36, 0, 28, 0, 0, 34, 3314, 4091, 1081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1431, 0, 0, 0, 22, 347, 0, 0, 3688, 4075, 0, 18, 0, 0, 3079, 3749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 4096, 0, 0, 4074, 20, 0, 25, 0, 0, 0, 224, 4051, 0, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 384, 0, 0, 292, 0, 0, 0, 9, 3940, 4096, 32, 4033, 0, 3004, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16760, 16573, 16183, 15908, 14343, 16409, 18036, 16860] avg=16384.0000 max=18036 max_rank=6 min=14343 min_rank=4 imbalance_ratio=1.100830 max_load_reduction=113036 max_load_reduction_pct=0.862396 moved_to_echo=65648 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3976, 1636, 28, 0, 0, 1, 0, 219, 4096, 0, 20, 7, 5, 0, 1, 2957, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3561, 245, 0, 3739, 10, 4096, 45, 0, 0, 0, 85, 54, 3446, 142, 1595, 968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2597, 7, 0, 0, 0, 4086, 4096, 0, 0, 3469, 2610, 60, 0, 2, 0, 3, 1516, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 8, 0, 783, 4083, 0, 0, 3799, 0, 0, 4091, 27, 3799, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4038, 0, 0, 5, 4041, 0, 0, 27, 0, 0, 1, 119, 52, 0, 4096, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1612, 0, 8, 235, 0, 375, 0, 267, 4096, 4057, 0, 1, 11, 0, 2175, 3009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3781, 0, 0, 4093, 0, 8, 0, 0, 0, 0, 859, 3286, 0, 1533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 0, 3626, 0, 0, 108, 15, 0, 0, 0, 28, 4069, 1, 4063, 13, 2974, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13887, 17986, 18732, 17106, 17547, 15846, 13560, 16408] avg=16384.0000 max=18732 max_rank=2 min=13560 min_rank=6 imbalance_ratio=1.143311 max_load_reduction=112340 max_load_reduction_pct=0.857086 moved_to_echo=63361 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[8, 4096, 2441, 0, 10, 0, 0, 0, 3524, 4052, 0, 0, 3, 2, 1429, 51, 3003, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1929, 2850, 0, 540, 533, 3989, 0, 0, 1, 2, 0, 47, 824, 551, 1567, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1431, 2, 11, 0, 162, 4095, 4077, 2, 25, 3274, 31, 134, 0, 75, 0, 4, 1489, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 3, 0, 3988, 51, 0, 3658, 0, 0, 3515, 2, 3798, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4093, 2, 17, 0, 3921, 0, 25, 0, 30, 0, 66, 0, 0, 768, 4058, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1584, 0, 0, 0, 4086, 0, 0, 64, 4095, 4092, 0, 0, 7, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1565, 18, 3140, 116, 0, 4094, 0, 15, 0, 0, 2, 0, 0, 4049, 48, 1506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 243, 3, 3600, 0, 36, 232, 0, 9, 1, 257, 5, 4035, 0, 4077, 0, 2953, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19544, 13787, 15088, 15488, 18131, 17940, 14553, 16541] avg=16384.0000 max=19544 max_rank=0 min=13787 min_rank=1 imbalance_ratio=1.192871 max_load_reduction=111528 max_load_reduction_pct=0.850891 moved_to_echo=67165 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 745, 0, 366, 0, 0, 101, 0, 4039, 56, 1, 0, 0, 18, 2, 3010, 905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 1316, 3395, 250, 3520, 3913, 4096, 0, 0, 0, 143, 0, 0, 28, 0, 1534, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1379, 3684, 4096, 142, 0, 3783, 2, 21, 694, 0, 0, 0, 1448, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 981, 0, 0, 0, 3185, 4094, 216, 0, 3809, 4, 0, 4096, 401, 126, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 3422, 0, 844, 0, 3687, 0, 0, 0, 0, 0, 32, 0, 0, 3851, 3982, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1541, 0, 461, 0, 0, 14, 0, 0, 3495, 4015, 14, 0, 0, 0, 2504, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3166, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 269, 3530, 0, 1474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 1, 119, 4095, 4096, 0, 4062, 5, 2962, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13339, 19183, 15536, 16920, 16923, 16140, 16611, 16420] avg=16384.0000 max=19183 max_rank=1 min=13339 min_rank=0 imbalance_ratio=1.170837 max_load_reduction=111889 max_load_reduction_pct=0.853645 moved_to_echo=66094 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3371, 7, 0, 533, 0, 2, 192, 0, 4095, 20, 250, 20, 0, 0, 4084, 2602, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 22, 0, 0, 3953, 3, 3850, 0, 1, 0, 0, 87, 1662, 0, 2959, 1556, 971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2265, 0, 0, 0, 1664, 3972, 3142, 0, 2, 2212, 0, 1421, 0, 15, 4084, 0, 1520, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 10, 4081, 0, 0, 2907, 0, 0, 3241, 0, 25, 2211, 356, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1520, 3, 26, 15, 8, 696, 3083, 0, 850, 0, 761, 0, 3378, 0, 1749, 3691, 4020, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1605, 0, 139, 5, 0, 0, 41, 15, 4068, 1631, 6, 0, 0, 0, 3438, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3827, 0, 3476, 14, 2, 3171, 335, 269, 4, 10, 0, 3, 1228, 3507, 0, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3719, 0, 99, 0, 408, 4, 0, 830, 0, 122, 0, 2706, 12, 1351, 0, 2948, 928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16098, 15075, 20544, 13047, 20862, 14988, 17331, 13127] avg=16384.0000 max=20862 max_rank=4 min=13047 min_rank=3 imbalance_ratio=1.273315 max_load_reduction=110210 max_load_reduction_pct=0.840836 moved_to_echo=66308 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 3769, 0, 6, 0, 0, 0, 380, 4081, 0, 12, 10, 0, 177, 2, 3019, 942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 256, 2605, 0, 0, 4096, 4096, 0, 0, 2, 0, 1, 16, 1, 23, 1596, 963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 218, 4096, 2377, 0, 0, 4026, 202, 0, 2234, 0, 0, 0, 1509, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 4050, 511, 0, 3809, 0, 0, 4096, 0, 509, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1437, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 69, 0, 1185, 4096, 1085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1606, 0, 2960, 0, 125, 69, 0, 0, 4096, 4036, 0, 0, 0, 0, 3807, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2783, 0, 4059, 2, 0, 4096, 0, 0, 0, 189, 0, 0, 101, 4069, 0, 1534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 91, 0, 0, 0, 0, 0, 0, 1821, 1005, 4096, 0, 4096, 0, 3009, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16494, 13664, 19045, 17076, 11965, 20794, 16833, 15201] avg=16384.0000 max=20794 max_rank=5 min=11965 min_rank=4 imbalance_ratio=1.269165 max_load_reduction=110278 max_load_reduction_pct=0.841354 moved_to_echo=64793 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 308, 0, 2, 0, 8, 0, 0, 3999, 1713, 0, 2699, 0, 0, 4032, 3018, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1531, 4, 4081, 2284, 4094, 0, 0, 10, 0, 0, 0, 726, 0, 1591, 939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 97, 17, 0, 48, 3623, 3506, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 1471, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 84, 1, 0, 4095, 10, 0, 3809, 32, 0, 3158, 246, 4088, 1166, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 924, 0, 0, 1684, 2537, 0, 1, 0, 0, 0, 0, 84, 0, 3044, 4095, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1564, 0, 274, 10, 4082, 159, 0, 0, 3815, 4005, 35, 0, 0, 0, 1, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1011, 0, 4085, 116, 28, 4018, 277, 0, 0, 3, 0, 0, 0, 4096, 0, 1529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1295, 0, 3312, 0, 1, 828, 0, 0, 5, 385, 4041, 4049, 0, 4065, 0, 2975, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20814, 15261, 13520, 16716, 13441, 14125, 15163, 22032] avg=16384.0000 max=22032 max_rank=7 min=13441 min_rank=4 imbalance_ratio=1.344727 max_load_reduction=109040 max_load_reduction_pct=0.831909 moved_to_echo=64761 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[23, 4096, 1585, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 72, 0, 84, 242, 3018, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 1832, 0, 4053, 0, 4075, 0, 0, 1, 0, 0, 28, 2924, 4088, 1569, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 3, 0, 1, 6, 4055, 2947, 0, 0, 3201, 413, 3, 0, 0, 1, 0, 1419, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 7, 2937, 3875, 0, 0, 3755, 0, 0, 4095, 0, 6, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 3355, 1, 0, 964, 4082, 0, 3, 91, 0, 0, 25, 25, 0, 120, 4060, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1509, 0, 4095, 0, 3, 47, 0, 0, 4086, 4029, 43, 2, 8, 0, 201, 807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 4096, 383, 0, 4093, 1, 0, 0, 1917, 0, 0, 0, 2913, 0, 1508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 19, 0, 0, 0, 0, 3, 3, 0, 0, 2120, 4090, 11, 4092, 4, 2900, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14136, 19543, 12424, 18811, 17860, 14830, 18985, 14483] avg=16384.0000 max=19543 max_rank=1 min=12424 min_rank=2 imbalance_ratio=1.192810 max_load_reduction=111529 max_load_reduction_pct=0.850899 moved_to_echo=66158 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[21504, 128, 29952, 640, 128, 31744, 640, 384, 128, 2176, 128, 128, 2688, 29568, 128, 12160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[13824, 128, 18944, 128, 896, 3456, 29568, 128, 896, 256, 896, 128, 3584, 384, 1920, 16000, 32512, 8576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 31872, 10496, 128, 1024, 128, 128, 384, 4736, 32640, 1920, 384, 2816, 128, 1792, 12544, 23680, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[13440, 128, 128, 128, 3584, 31232, 28416, 256, 128, 28160, 3584, 1664, 2944, 128, 4096, 128, 11776, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1152, 128, 4352, 128, 6912, 31232, 896, 0, 29568, 128, 128, 29440, 1152, 12416, 14464, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 5888, 128, 11136, 0, 512, 1536, 128, 896, 128, 2816, 15360, 31360, 128, 29952, 128, 23808, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[512, 128, 8448, 15744, 256, 20224, 14976, 32384, 128, 128, 128, 256, 256, 1920, 8064, 8320, 12672, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[12544, 0, 8064, 256, 8320, 1024, 128, 384, 31488, 29952, 128, 128, 128, 0, 15232, 24064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 4086, 0, 0, 0, 4031, 0, 0, 0, 0, 0, 4087, 3663, 4096, 43, 263, 969, 16, 129, 0, 0, 2, 156, 0, 0, 1, 4007, 159, 0, 13, 3, 600, 4067, 1, 3979, 0, 17, 0, 0, 27, 4096, 2659, 3596, 0, 44, 2, 0, 9, 4068, 92, 0, 2677, 4048, 5, 4065, 2, 4096, 91, 1, 1338, 2, 4021, 3653, 21, 182, 0, 0, 0, 4055, 4018, 4089, 3, 4038, 2, 12, 0, 4, 13, 81, 0, 0, 0, 4093, 2743, 0, 3996, 786, 0, 0, 3941, 0, 0, 0, 8, 23, 0, 0, 4073, 0, 4074, 0, 2, 0, 15, 0, 0, 0, 0, 4, 1, 0, 0, 3138, 0, 106, 435, 122, 4096, 0, 4091, 962, 4096, 0, 0, 36, 733, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 4089, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 4058, 4096, 72, 2852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2855, 3848, 0, 4075, 0, 791, 0, 0, 0, 4096, 4096, 392, 3, 0, 0, 0, 0, 3098, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4026, 0, 167, 0, 3088, 4069, 0, 18, 25, 0, 0, 3149, 4087, 4093, 2069, 4096, 0, 0, 0, 0, 0, 948, 0, 69, 2, 4096, 4088, 24, 3476, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 9, 1599, 0, 42, 2, 27, 171, 0, 4096, 0, 4096, 0, 0, 7, 42, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 4091, 0, 47, 3, 0, 9, 1, 5, 0, 0, 4032, 4091, 4040, 3397, 2554, 118, 0, 50, 1, 0, 0, 75, 27, 7, 0, 4017, 1, 0, 155, 0, 4, 3841, 1, 4085, 0, 0, 1, 0, 3, 3993, 3504, 4041, 3253, 11, 0, 0, 2, 4073, 0, 0, 3313, 4084, 2, 4003, 0, 4035, 879, 0, 128, 0, 4092, 735, 0, 0, 26, 0, 2, 4054, 1603, 3887, 12, 4076, 35, 0, 0, 1, 0, 3995, 30, 0, 0, 4095, 199, 99, 4056, 3864, 7, 0, 4025, 12, 0, 0, 0, 0, 0, 47, 4095, 0, 3939, 4, 53, 0, 0, 272, 6, 0, 2047, 0, 14, 210, 65, 0, 0, 23, 0, 0, 2498, 330, 4090, 376, 4088, 1, 0, 0, 0, 10, 22] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[16, 4004, 0, 0, 1, 0, 15, 0, 0, 0, 43, 3997, 4076, 4083, 555, 0, 2876, 0, 58, 0, 0, 0, 0, 20, 0, 0, 3492, 0, 0, 0, 49, 0, 3993, 63, 3895, 112, 22, 0, 0, 0, 4096, 4096, 2242, 1, 0, 0, 0, 0, 9, 0, 0, 722, 4096, 53, 4096, 120, 4081, 4096, 132, 3426, 32, 4053, 4085, 0, 0, 32, 8, 0, 2162, 4096, 4096, 1148, 4001, 0, 41, 0, 0, 0, 0, 0, 0, 391, 3972, 869, 0, 4095, 1349, 0, 126, 3906, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 4061, 0, 0, 235, 14, 0, 223, 19, 0, 4092, 0, 1134, 51, 3901, 136, 9, 0, 0, 3809, 0, 4061, 1, 4027, 0, 82, 0, 0, 8, 78] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4, 4095, 4, 0, 0, 3395, 1, 0, 15, 1, 0, 4041, 4093, 4095, 1635, 111, 1654, 9, 95, 0, 1, 0, 25, 17, 0, 0, 3883, 0, 0, 0, 0, 0, 3977, 0, 4094, 0, 68, 0, 0, 0, 3783, 2712, 0, 78, 0, 1, 0, 0, 0, 0, 0, 3793, 4091, 0, 4081, 19, 4079, 3782, 4021, 0, 5, 3994, 1458, 0, 10, 0, 0, 0, 2914, 3686, 4094, 37, 4001, 0, 0, 106, 0, 0, 973, 15, 15, 0, 4096, 1, 0, 4092, 4029, 0, 0, 4020, 0, 0, 0, 3, 0, 0, 0, 4087, 7, 4010, 0, 0, 0, 2, 17, 92, 0, 838, 2511, 17, 59, 1, 99, 0, 4094, 19, 0, 1958, 9, 4094, 3, 4087, 0, 0, 1024, 301, 441, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 0, 22, 0, 0, 0, 0, 0, 4060, 4094, 4090, 0, 490, 4, 0, 0, 0, 0, 0, 674, 0, 448, 0, 4087, 14, 0, 0, 0, 0, 3975, 1, 3882, 0, 3, 1, 0, 34, 4083, 4009, 2835, 29, 5, 0, 1, 0, 2651, 12, 0, 4096, 3985, 0, 4096, 0, 4095, 23, 1114, 1584, 0, 4096, 4073, 0, 3961, 0, 0, 108, 3963, 2, 4070, 0, 4093, 0, 0, 7, 0, 0, 0, 0, 0, 157, 4074, 509, 6, 4076, 4087, 0, 24, 4092, 0, 58, 24, 0, 0, 0, 4, 3924, 28, 3881, 1, 0, 0, 0, 0, 0, 0, 46, 3967, 0, 4, 425, 697, 14, 3950, 34, 19, 760, 86, 4071, 0, 4011, 7, 0, 25, 1005, 40, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[349, 3201, 92, 0, 4, 4094, 0, 0, 2578, 0, 1, 2439, 4096, 3875, 39, 4080, 8, 0, 28, 16, 0, 0, 533, 3, 0, 0, 3749, 0, 0, 0, 0, 0, 4096, 1, 1822, 82, 8, 0, 0, 6, 4077, 0, 0, 0, 1242, 3, 0, 2, 3375, 0, 19, 43, 3693, 0, 3368, 336, 3731, 0, 3480, 2136, 0, 4075, 875, 0, 14, 703, 0, 0, 2799, 4017, 3241, 3876, 4096, 0, 1, 0, 0, 0, 0, 2, 0, 0, 3732, 0, 2031, 4096, 4093, 0, 252, 1271, 25, 253, 0, 0, 1194, 23, 0, 4089, 0, 3465, 0, 0, 187, 32, 13, 0, 0, 3, 2, 0, 0, 0, 0, 112, 0, 4049, 27, 4024, 234, 4052, 4, 3848, 4, 807, 1106, 3640, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[2, 4095, 0, 0, 2, 521, 0, 1, 70, 3, 0, 3807, 4087, 4096, 4096, 57, 16, 0, 0, 8, 0, 0, 703, 0, 0, 0, 4089, 0, 6, 0, 0, 0, 4096, 1, 4054, 0, 2787, 0, 1, 0, 129, 1506, 4047, 4096, 0, 2, 0, 0, 4040, 3, 6, 4091, 4052, 6, 4095, 2, 4096, 1, 3852, 3560, 0, 2568, 3990, 0, 1450, 5, 51, 0, 156, 782, 4085, 0, 4096, 0, 42, 16, 0, 0, 0, 0, 0, 11, 4087, 8, 0, 3541, 3842, 8, 0, 4093, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 3797, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 264, 3956, 0, 6, 25, 67, 1990, 8, 4092, 0, 4078, 31, 0, 393, 1253, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4086, 0, 0, 0, 4031, 0, 0, 0, 0, 0, 4087, 3663, 116, 43, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 969, 16, 129, 0, 0, 2, 156, 0, 0, 1, 4007, 159, 0, 13, 3, 600, 4096, 4048, 2878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2588, 1, 3979, 0, 17, 0, 0, 27, 4096, 2659, 3596, 0, 44, 2, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 92, 0, 2677, 0, 5, 0, 2, 0, 91, 1, 1338, 2, 3408, 3653, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 4055, 4018, 4089, 3, 4038, 2, 12, 0, 4, 13, 81, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3910, 2743, 0, 3996, 786, 0, 0, 3941, 0, 0, 0, 8, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 4074, 0, 2, 0, 15, 0, 0, 0, 0, 4, 1, 0, 0, 3980, 1192, 1187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3138, 0, 106, 435, 122, 4096, 0, 4091, 962, 4096, 0, 0, 36, 733, 0, 0, 613, 287, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16289, 17077, 17018, 15358, 16663, 15407, 14528, 18732] avg=16384.0000 max=18732 max_rank=7 min=14528 min_rank=6 imbalance_ratio=1.143311 max_load_reduction=112340 max_load_reduction_pct=0.857086 moved_to_echo=65330 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 0, 47, 3, 0, 9, 1, 5, 0, 0, 4032, 4091, 115, 3397, 2554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 50, 1, 0, 0, 75, 27, 7, 0, 4017, 1, 0, 155, 0, 4, 4035, 4084, 2834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2444, 1, 4085, 0, 0, 1, 0, 3, 3993, 3504, 4041, 3253, 11, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 3313, 0, 2, 0, 0, 0, 879, 0, 128, 0, 3469, 735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 2, 4054, 1603, 3887, 12, 4076, 35, 0, 0, 1, 0, 3995, 30, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 199, 99, 4056, 3864, 7, 0, 4025, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4095, 0, 3939, 4, 53, 0, 0, 272, 6, 0, 2047, 0, 14, 210, 65, 3925, 1126, 1169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 2498, 330, 4090, 376, 4088, 1, 0, 0, 0, 10, 22, 623, 271, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18345, 15408, 21338, 12599, 17887, 16174, 16972, 12349] avg=16384.0000 max=21338 max_rank=2 min=12349 min_rank=7 imbalance_ratio=1.302368 max_load_reduction=109734 max_load_reduction_pct=0.837204 moved_to_echo=63382 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 0, 22, 0, 0, 0, 0, 0, 4060, 4094, 116, 0, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 674, 0, 448, 0, 4087, 14, 0, 0, 0, 0, 4095, 3985, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2529, 1, 3882, 0, 3, 1, 0, 34, 4083, 4009, 2835, 29, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2651, 12, 0, 4096, 0, 0, 0, 0, 0, 23, 1114, 1584, 0, 3472, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 0, 0, 108, 3963, 2, 4070, 0, 4093, 0, 0, 7, 0, 0, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 3892, 509, 6, 4076, 4087, 0, 24, 4092, 0, 58, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3924, 28, 3881, 1, 0, 0, 0, 0, 0, 0, 46, 3967, 0, 4, 425, 3974, 1165, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 697, 14, 3950, 34, 19, 760, 86, 4071, 0, 4011, 7, 0, 25, 1005, 40, 0, 624, 281, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12878, 16208, 17412, 17025, 16369, 16925, 18614, 15641] avg=16384.0000 max=18614 max_rank=6 min=12878 min_rank=0 imbalance_ratio=1.136108 max_load_reduction=112458 max_load_reduction_pct=0.857986 moved_to_echo=67549 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[16, 4004, 0, 0, 1, 0, 15, 0, 0, 0, 43, 3997, 4076, 116, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2876, 0, 58, 0, 0, 0, 0, 20, 0, 0, 3492, 0, 0, 0, 49, 0, 4081, 4096, 2900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2541, 63, 3895, 112, 22, 0, 0, 0, 4096, 4096, 2242, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 722, 0, 53, 0, 120, 0, 4096, 132, 3426, 32, 3436, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 8, 0, 2162, 4096, 4096, 1148, 4001, 0, 41, 0, 0, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 3794, 869, 0, 4095, 1349, 0, 126, 3906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 4061, 0, 0, 235, 14, 0, 223, 19, 0, 4092, 0, 1134, 51, 3967, 1170, 1196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 136, 9, 0, 0, 3809, 0, 4061, 1, 4027, 0, 82, 0, 0, 8, 78, 617, 282, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12823, 17572, 17068, 16111, 15745, 14530, 20195, 17028] avg=16384.0000 max=20195 max_rank=6 min=12823 min_rank=0 imbalance_ratio=1.232605 max_load_reduction=110877 max_load_reduction_pct=0.845924 moved_to_echo=67498 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 4058, 116, 72, 2852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2855, 4096, 4096, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2448, 0, 4075, 0, 791, 0, 0, 0, 4096, 4096, 392, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3098, 0, 0, 4096, 0, 0, 0, 0, 0, 4026, 0, 167, 0, 2618, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 25, 0, 0, 3149, 4087, 4093, 2069, 4096, 0, 0, 0, 0, 0, 948, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 2, 3913, 4088, 24, 3476, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 9, 3980, 1128, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1599, 0, 42, 2, 27, 171, 0, 4096, 0, 4096, 0, 0, 7, 42, 0, 0, 470, 272, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15289, 18044, 15901, 18074, 18651, 19765, 14507, 10841] avg=16384.0000 max=19765 max_rank=5 min=10841 min_rank=7 imbalance_ratio=1.206360 max_load_reduction=111307 max_load_reduction_pct=0.849205 moved_to_echo=63764 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[349, 3201, 92, 0, 4, 4094, 0, 0, 2578, 0, 1, 2439, 4096, 107, 39, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 28, 16, 0, 0, 533, 3, 0, 0, 3749, 0, 0, 0, 0, 0, 3731, 3693, 2386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2597, 1, 1822, 82, 8, 0, 0, 6, 4077, 0, 0, 0, 1242, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3375, 0, 19, 43, 0, 0, 0, 336, 0, 0, 3480, 2136, 0, 3449, 875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 703, 0, 0, 2799, 4017, 3241, 3876, 4096, 0, 1, 0, 0, 0, 0, 2, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3553, 0, 2031, 4096, 4093, 0, 252, 1271, 25, 253, 0, 0, 1194, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 3465, 0, 0, 187, 32, 13, 0, 0, 3, 2, 0, 0, 0, 3768, 1204, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 4049, 27, 4024, 234, 4052, 4, 3848, 4, 807, 1106, 3640, 0, 0, 626, 295, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21080, 14147, 9840, 13713, 18906, 16791, 13745, 22850] avg=16384.0000 max=22850 max_rank=7 min=9840 min_rank=2 imbalance_ratio=1.394653 max_load_reduction=108222 max_load_reduction_pct=0.825668 moved_to_echo=72292 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 4095, 4, 0, 0, 3395, 1, 0, 15, 1, 0, 4041, 4093, 116, 1635, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1654, 9, 95, 0, 1, 0, 25, 17, 0, 0, 3883, 0, 0, 0, 0, 0, 4079, 4091, 2891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2530, 0, 4094, 0, 68, 0, 0, 0, 3783, 2712, 0, 78, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3793, 0, 0, 0, 19, 0, 3782, 4021, 0, 5, 3386, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 2914, 3686, 4094, 37, 4001, 0, 0, 106, 0, 0, 973, 15, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3913, 1, 0, 4092, 4029, 0, 0, 4020, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 7, 4010, 0, 0, 0, 2, 17, 92, 0, 838, 2511, 17, 59, 1, 3979, 1166, 1190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 4094, 19, 0, 1958, 9, 4094, 3, 4087, 0, 0, 1024, 301, 441, 0, 608, 281, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17511, 16745, 13266, 16464, 16002, 16073, 17976, 17035] avg=16384.0000 max=17976 max_rank=6 min=13266 min_rank=2 imbalance_ratio=1.097168 max_load_reduction=113096 max_load_reduction_pct=0.862854 moved_to_echo=67086 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 4095, 0, 0, 2, 521, 0, 1, 70, 3, 0, 3807, 4087, 116, 4096, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 8, 0, 0, 703, 0, 0, 0, 4089, 0, 6, 0, 0, 0, 4096, 4052, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2606, 1, 4054, 0, 2787, 0, 1, 0, 129, 1506, 4047, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 3, 6, 4091, 0, 6, 0, 2, 0, 1, 3852, 3560, 0, 2177, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1450, 5, 51, 0, 156, 782, 4085, 0, 4096, 0, 42, 16, 0, 0, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3904, 8, 0, 3541, 3842, 8, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 3797, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 264, 3980, 1201, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 6, 25, 67, 1990, 8, 4092, 0, 4078, 31, 0, 393, 1253, 0, 0, 391, 289, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16857, 15871, 19229, 21728, 10849, 15407, 14535, 16596] avg=16384.0000 max=21728 max_rank=3 min=10849 min_rank=4 imbalance_ratio=1.326172 max_load_reduction=109344 max_load_reduction_pct=0.834229 moved_to_echo=57387 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32512, 128, 31360, 128, 128, 512, 128, 384, 384, 128, 2944, 10624, 128, 1408, 896, 31616, 9472, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[384, 31872, 128, 128, 128, 12160, 128, 128, 2688, 128, 128, 30592, 32384, 1024, 9856, 10496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[5760, 896, 128, 128, 23296, 22400, 31744, 7168, 32512, 128, 128, 256, 128, 128, 6016, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 640, 30848, 8448, 2176, 31488, 26240, 128, 512, 29568, 128, 384, 128, 128, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[13440, 384, 8320, 4608, 384, 19328, 768, 32768, 1408, 32384, 128, 896, 2688, 7040, 512, 128, 4608, 2304, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[5760, 128, 384, 128, 128, 128, 2176, 128, 512, 128, 31488, 256, 128, 256, 128, 3584, 32384, 32256, 22656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[20352, 128, 29952, 256, 3712, 128, 128, 128, 28416, 22656, 17280, 7552, 1408, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[21376, 128, 128, 22912, 0, 128, 0, 512, 0, 12928, 12672, 12416, 128, 25472, 23040, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1, 74, 1, 0, 24, 0, 26, 4057, 4094, 2732, 0, 1, 0, 0, 0, 1871, 0, 0, 8, 4088, 0, 0, 0, 4086, 0, 5, 2208, 4063, 0, 0, 0, 4072, 0, 0, 8, 0, 0, 2, 6, 4083, 0, 3976, 4, 18, 4083, 10, 0, 20, 1, 3515, 1968, 7, 85, 2872, 0, 41, 4062, 0, 0, 0, 0, 0, 106, 0, 0, 0, 12, 116, 0, 3415, 118, 21, 0, 888, 3879, 0, 0, 0, 100, 4096, 96, 3, 199, 4038, 2, 0, 4096, 0, 3992, 4089, 4, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 9, 4087, 0, 3320, 6, 4084, 0, 50, 219, 0, 1443, 0, 4, 3789, 0, 2, 4035, 0, 4073, 4096, 12, 3999, 4095, 10, 31, 87, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 11, 1131, 3962, 4096, 2128, 0, 19, 8, 0, 499, 0, 0, 0, 72, 3418, 2875, 292, 0, 4076, 1, 0, 1465, 3999, 25, 117, 1314, 3688, 0, 0, 0, 0, 0, 117, 0, 3823, 94, 2815, 66, 286, 4000, 0, 344, 0, 0, 4011, 1731, 0, 0, 572, 0, 0, 4014, 1, 0, 8, 0, 0, 0, 0, 0, 306, 2, 0, 49, 3320, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4085, 0, 120, 0, 4096, 0, 0, 4040, 0, 4088, 3811, 0, 4096, 520, 743, 0, 1989, 0, 157, 0, 749, 0, 3781, 0, 2516, 0, 3995, 0, 35, 0, 0, 4096, 0, 0, 4095, 723, 0, 4096, 0, 3866, 3531, 0, 4096, 4064, 0, 833, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 397, 0, 4, 4034, 3760, 2819, 0, 968, 0, 1730, 36, 0, 993, 1, 2, 20, 4, 0, 0, 4096, 0, 1165, 4038, 328, 1, 4, 1, 4061, 0, 4096, 0, 6, 0, 149, 1, 0, 0, 4088, 84, 0, 4092, 235, 870, 3, 4001, 1279, 1935, 33, 0, 0, 0, 0, 3898, 0, 0, 3, 753, 0, 0, 2, 0, 0, 0, 0, 6, 2719, 0, 4, 3500, 3525, 11, 2376, 45, 8, 2201, 4091, 0, 19, 0, 4095, 196, 2778, 4096, 0, 1603, 4096, 0, 4066, 4096, 4094, 0, 148, 0, 1, 0, 3427, 0, 3939, 0, 4096, 0, 535, 0, 0, 0, 0, 89, 3, 0, 1302, 0, 0, 485, 0, 3653, 3291, 0, 3976, 4026, 2, 0, 473, 9, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 1, 3, 5, 0, 3560, 4095, 4092, 791, 0, 327, 7, 1, 0, 1073, 0, 0, 153, 0, 4096, 37, 0, 4096, 0, 1, 5, 15, 0, 2915, 615, 2375, 0, 979, 4, 655, 0, 8, 0, 0, 2, 23, 0, 0, 4094, 0, 47, 713, 0, 4069, 1317, 0, 0, 15, 4, 0, 4089, 11, 7, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 3849, 0, 0, 0, 3921, 4083, 39, 0, 0, 0, 4095, 0, 4096, 0, 1524, 0, 5, 4092, 3, 3731, 4095, 0, 4058, 4056, 105, 0, 0, 1, 0, 0, 4069, 0, 4094, 70, 4089, 0, 4096, 0, 0, 10, 0, 4096, 0, 0, 2946, 0, 0, 2570, 0, 2790, 4016, 0, 4085, 3980, 0, 4089, 16, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[86, 39, 298, 0, 0, 0, 537, 4096, 4089, 2876, 0, 2731, 48, 0, 2, 4090, 0, 0, 0, 1496, 3525, 0, 109, 3889, 0, 0, 460, 0, 75, 1062, 67, 4066, 3, 2232, 0, 0, 0, 0, 0, 82, 10, 4068, 686, 10, 4086, 0, 18, 977, 0, 603, 0, 46, 28, 4026, 0, 0, 4075, 0, 0, 2, 0, 7, 1, 0, 0, 254, 0, 447, 0, 3904, 8, 26, 3, 0, 4087, 0, 12, 75, 0, 4096, 31, 3342, 76, 3773, 0, 0, 4076, 0, 101, 4016, 0, 4049, 2449, 10, 0, 0, 135, 0, 0, 4059, 0, 4072, 3, 4080, 0, 4094, 2, 0, 1, 21, 4091, 0, 0, 3472, 301, 0, 43, 0, 3908, 4083, 23, 4052, 3926, 13, 697, 555, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2, 26, 0, 3, 0, 4096, 4089, 8, 0, 11, 0, 768, 0, 2097, 10, 0, 0, 4094, 3808, 0, 0, 3614, 0, 0, 3941, 2057, 0, 2545, 1, 4096, 0, 1, 0, 0, 4, 0, 0, 4095, 0, 4096, 108, 0, 4096, 0, 0, 1, 0, 3809, 248, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4091, 0, 0, 0, 72, 4096, 0, 0, 0, 0, 4096, 0, 5, 12, 4087, 0, 0, 4096, 1, 2747, 3908, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 345, 0, 4096, 0, 4095, 0, 4096, 0, 197, 0, 0, 4094, 0, 0, 4078, 0, 0, 0, 0, 4074, 4096, 0, 4096, 4086, 0, 2473, 0, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[20, 4, 27, 0, 1, 0, 1669, 4096, 4074, 4072, 0, 4, 2, 742, 0, 4078, 73, 4, 0, 3466, 49, 46, 0, 3131, 0, 10, 0, 0, 2, 1010, 0, 4076, 0, 3742, 25, 88, 1, 0, 0, 4080, 0, 267, 288, 1, 4073, 0, 44, 1012, 839, 4091, 91, 0, 0, 4056, 0, 0, 4082, 0, 1, 0, 0, 15, 0, 0, 1, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4086, 0, 3523, 671, 1316, 0, 25, 4068, 1, 730, 4087, 28, 4076, 4033, 3996, 0, 74, 0, 0, 0, 1576, 41, 4045, 0, 4012, 0, 4062, 0, 0, 0, 34, 0, 0, 57, 3902, 5, 0, 154, 2, 4036, 3997, 0, 3971, 4064, 0, 0, 743, 9, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 0, 0, 11, 3938, 4095, 3771, 0, 481, 0, 80, 0, 2592, 0, 0, 0, 2599, 3889, 8, 38, 3887, 0, 1, 1812, 0, 32, 45, 0, 3782, 0, 1137, 11, 1, 1, 0, 0, 2516, 9, 3943, 0, 0, 4085, 0, 0, 3934, 0, 4090, 220, 0, 0, 102, 0, 18, 4069, 22, 0, 26, 0, 0, 6, 0, 0, 1, 0, 65, 1, 4092, 65, 3, 1, 16, 4093, 9, 0, 0, 0, 4089, 0, 4090, 0, 10, 0, 136, 4082, 0, 0, 4007, 7, 4096, 4089, 121, 0, 0, 0, 0, 0, 3038, 1, 4096, 29, 4096, 0, 3527, 5, 0, 0, 0, 4068, 0, 5, 4096, 14, 0, 4065, 0, 1934, 4029, 4, 4095, 4080, 0, 0, 1521, 40, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 74, 1, 0, 24, 0, 26, 4057, 4094, 2732, 0, 1, 0, 0, 0, 1871, 1264, 621, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4088, 0, 0, 0, 3465, 0, 5, 2208, 4063, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 2, 6, 4083, 0, 3976, 4, 18, 4083, 10, 0, 20, 2105, 2045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3515, 1968, 7, 85, 2872, 0, 41, 4062, 0, 0, 0, 0, 0, 106, 0, 4096, 1894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 116, 0, 3415, 118, 21, 0, 888, 3879, 0, 0, 0, 100, 4096, 1942, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 3, 199, 4038, 2, 0, 0, 0, 3992, 4089, 4, 109, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4062, 0, 3320, 6, 4084, 0, 50, 219, 0, 1443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3789, 0, 2, 4035, 0, 4073, 4096, 12, 0, 1991, 10, 31, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14791, 17909, 16360, 18647, 15427, 16615, 13193, 18130] avg=16384.0000 max=18647 max_rank=3 min=13193 min_rank=6 imbalance_ratio=1.138123 max_load_reduction=112425 max_load_reduction_pct=0.857735 moved_to_echo=63365 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 11, 1131, 3962, 4096, 2128, 0, 19, 8, 0, 499, 0, 1255, 620, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 3418, 2875, 292, 0, 3456, 1, 0, 1465, 3999, 25, 117, 1314, 3688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 3823, 94, 2815, 66, 286, 4000, 0, 344, 0, 2156, 2045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 1731, 0, 0, 572, 0, 0, 4014, 1, 0, 8, 0, 0, 0, 0, 4040, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 2, 0, 49, 3320, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4085, 1942, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 4096, 0, 0, 0, 0, 4088, 3811, 0, 109, 520, 743, 0, 1989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 749, 0, 3758, 0, 2516, 0, 3995, 0, 35, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 723, 0, 4096, 0, 3866, 3531, 0, 0, 1976, 0, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13752, 20722, 15746, 16317, 14633, 15476, 15306, 19120] avg=16384.0000 max=20722 max_rank=1 min=13752 min_rank=0 imbalance_ratio=1.264771 max_load_reduction=110350 max_load_reduction_pct=0.841904 moved_to_echo=64535 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 397, 0, 4, 4034, 3760, 2819, 0, 968, 0, 1730, 36, 0, 1246, 626, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 993, 1, 2, 20, 4, 0, 0, 3470, 0, 1165, 4038, 328, 1, 4, 1, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 149, 1, 0, 0, 4088, 84, 0, 4092, 235, 870, 3, 2094, 2034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 1279, 1935, 33, 0, 0, 0, 0, 3898, 0, 0, 3, 753, 0, 0, 2, 4096, 1882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2719, 0, 4, 3500, 3525, 11, 2376, 45, 8, 2201, 4091, 1930, 829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4095, 196, 2778, 0, 0, 1603, 4096, 0, 102, 4096, 4094, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3427, 0, 3911, 0, 4096, 0, 535, 0, 0, 0, 0, 89, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1302, 0, 0, 485, 0, 3653, 3291, 0, 0, 1951, 2, 0, 473, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15650, 14088, 17752, 17882, 21245, 21227, 12062, 11166] avg=16384.0000 max=21245 max_rank=4 min=11166 min_rank=7 imbalance_ratio=1.296692 max_load_reduction=109827 max_load_reduction_pct=0.837914 moved_to_echo=65700 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[20, 4, 27, 0, 1, 0, 1669, 4096, 4074, 4072, 0, 4, 2, 742, 0, 4078, 1255, 476, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 4, 0, 3466, 49, 46, 0, 2655, 0, 10, 0, 0, 2, 1010, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3742, 25, 88, 1, 0, 0, 4080, 0, 267, 288, 1, 4073, 0, 44, 1012, 2090, 2035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 839, 4091, 91, 0, 0, 4056, 0, 0, 4082, 0, 1, 0, 0, 15, 0, 0, 4068, 1881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4086, 1932, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3523, 671, 1316, 0, 25, 0, 1, 730, 4087, 28, 109, 4033, 3996, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1576, 41, 4020, 0, 4012, 0, 4062, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 3902, 5, 0, 154, 2, 4036, 3997, 0, 0, 1976, 0, 0, 743, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20545, 11391, 17746, 19124, 15047, 18593, 13745, 14881] avg=16384.0000 max=20545 max_rank=0 min=11391 min_rank=1 imbalance_ratio=1.253967 max_load_reduction=110527 max_load_reduction_pct=0.843254 moved_to_echo=62266 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 3, 5, 0, 3560, 4095, 4092, 791, 0, 327, 7, 1, 0, 1073, 1229, 623, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 4096, 37, 0, 3473, 0, 1, 5, 15, 0, 2915, 615, 2375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 979, 4, 655, 0, 8, 0, 0, 2, 23, 0, 0, 4094, 0, 47, 713, 2151, 2026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 1317, 0, 0, 15, 4, 0, 4089, 11, 7, 0, 0, 2, 1, 0, 4092, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 0, 0, 0, 3921, 4083, 39, 0, 0, 0, 4095, 1924, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1524, 0, 5, 0, 3, 3731, 4095, 0, 108, 4056, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4069, 0, 4069, 70, 4089, 0, 4096, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2946, 0, 0, 2570, 0, 2790, 4016, 0, 0, 1935, 0, 4089, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15832, 13685, 10702, 15541, 18727, 17723, 20500, 18362] avg=16384.0000 max=20500 max_rank=6 min=10702 min_rank=2 imbalance_ratio=1.251221 max_load_reduction=110572 max_load_reduction_pct=0.843597 moved_to_echo=75312 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 26, 0, 3, 0, 4096, 4089, 8, 0, 11, 0, 768, 0, 2097, 1262, 549, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4094, 3808, 0, 0, 3065, 0, 0, 3941, 2057, 0, 2545, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 4095, 0, 4096, 108, 0, 4096, 0, 0, 1, 2156, 2045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3809, 248, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4091, 0, 0, 0, 72, 4096, 0, 0, 0, 0, 4096, 1942, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 12, 4087, 0, 0, 0, 1, 2747, 3908, 0, 109, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 0, 4071, 0, 4095, 0, 4096, 0, 197, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 4074, 4096, 0, 0, 1986, 0, 2473, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12936, 23617, 16602, 14203, 15143, 14963, 16898, 16710] avg=16384.0000 max=23617 max_rank=1 min=12936 min_rank=0 imbalance_ratio=1.441467 max_load_reduction=107455 max_load_reduction_pct=0.819817 moved_to_echo=63714 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[86, 39, 298, 0, 0, 0, 537, 4096, 4089, 2876, 0, 2731, 48, 0, 2, 4090, 1212, 591, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1496, 3525, 0, 109, 3298, 0, 0, 460, 0, 75, 1062, 67, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2232, 0, 0, 0, 0, 0, 82, 10, 4068, 686, 10, 4086, 0, 18, 977, 2133, 2022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 603, 0, 46, 28, 4026, 0, 0, 4075, 0, 0, 2, 0, 7, 1, 0, 4076, 1919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 447, 0, 3904, 8, 26, 3, 0, 4087, 0, 12, 75, 0, 4096, 1919, 805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 3342, 76, 3773, 0, 0, 0, 0, 101, 4016, 0, 108, 2449, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 4059, 0, 4047, 3, 4080, 0, 4094, 2, 0, 1, 21, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3472, 301, 0, 43, 0, 3908, 4083, 23, 0, 1909, 13, 697, 555, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20720, 14158, 16327, 14783, 15636, 13906, 20533, 15009] avg=16384.0000 max=20720 max_rank=0 min=13906 min_rank=5 imbalance_ratio=1.264648 max_load_reduction=110352 max_load_reduction_pct=0.841919 moved_to_echo=65084 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 0, 0, 11, 3938, 4095, 3771, 0, 481, 0, 80, 0, 2592, 1260, 591, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2599, 3889, 8, 38, 3296, 0, 1, 1812, 0, 32, 45, 0, 3782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1137, 11, 1, 1, 0, 0, 2516, 9, 3943, 0, 0, 4085, 0, 0, 3934, 2155, 2045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 220, 0, 0, 102, 0, 18, 4069, 22, 0, 26, 0, 0, 6, 0, 4082, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 65, 1, 4092, 65, 3, 1, 16, 4093, 9, 0, 0, 0, 4089, 1942, 837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 10, 0, 136, 0, 0, 0, 4007, 7, 109, 4089, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3038, 1, 4071, 29, 4096, 0, 3527, 5, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 14, 0, 4065, 0, 1934, 4029, 4, 0, 1983, 0, 0, 1521, 40, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16846, 15502, 19837, 14575, 15214, 12569, 18835, 17694] avg=16384.0000 max=19837 max_rank=2 min=12569 min_rank=5 imbalance_ratio=1.210754 max_load_reduction=111235 max_load_reduction_pct=0.848656 moved_to_echo=64312 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 27776, 1152, 128, 15488, 128, 28416, 31232, 128, 0, 15744, 128, 8192, 3456, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1152, 128, 256, 19200, 18304, 384, 256, 26240, 128, 1280, 13952, 10496, 256, 7808, 2048, 30336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 256, 0, 17280, 128, 32128, 128, 30336, 128, 28544, 128, 384, 256, 128, 22016, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[4864, 25472, 7552, 128, 128, 11776, 128, 128, 32512, 128, 128, 128, 768, 128, 128, 128, 32768, 15360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 12288, 128, 768, 128, 384, 128, 18688, 128, 23296, 1280, 384, 32640, 256, 1408, 6784, 17152, 16384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 15232, 1024, 23040, 256, 2944, 0, 128, 17024, 32128, 128, 896, 27520, 9088, 0, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 384, 128, 512, 128, 7040, 32384, 32512, 19200, 0, 4608, 128, 3328, 640, 15872, 9984, 4736, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 640, 128, 640, 128, 29568, 256, 128, 3584, 8448, 28544, 2432, 128, 128, 2304, 32768, 15488, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[3970, 0, 0, 4087, 4096, 3916, 2, 1631, 0, 4096, 165, 622, 264, 120, 0, 0, 4096, 0, 0, 0, 4054, 5, 0, 3, 0, 4095, 3860, 0, 0, 93, 51, 4095, 0, 0, 0, 2, 0, 0, 0, 2469, 41, 4080, 3149, 81, 0, 0, 73, 0, 0, 43, 0, 36, 0, 0, 0, 0, 568, 0, 0, 8, 87, 9, 0, 3967, 4096, 0, 0, 3997, 126, 0, 1, 0, 3655, 0, 958, 122, 4080, 50, 0, 3685, 0, 2, 678, 4033, 0, 3955, 2, 19, 2585, 0, 0, 10, 0, 2360, 0, 2, 0, 3951, 3642, 4096, 0, 4093, 4029, 0, 0, 0, 307, 3, 5, 0, 1845, 0, 0, 0, 3972, 348, 0, 25, 619, 149, 4036, 4094, 1869, 0, 0, 48, 0, 1591] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[3999, 0, 6, 4096, 3743, 3964, 0, 4094, 0, 4096, 4, 272, 0, 0, 0, 0, 2318, 0, 0, 0, 3974, 189, 0, 14, 2, 4096, 4095, 0, 5, 0, 87, 0, 0, 0, 0, 3955, 7, 0, 13, 318, 0, 3676, 4096, 0, 5, 0, 0, 0, 0, 25, 0, 38, 0, 0, 0, 3, 0, 0, 0, 1, 2089, 2022, 0, 3343, 4088, 0, 0, 4051, 0, 0, 63, 0, 4096, 31, 63, 3, 3909, 4094, 927, 4062, 0, 21, 38, 4082, 884, 4096, 2, 0, 4078, 0, 0, 0, 25, 32, 0, 0, 0, 4096, 3957, 4096, 4, 4095, 2, 0, 0, 0, 732, 0, 0, 19, 4070, 0, 0, 0, 4096, 188, 0, 0, 1, 0, 2267, 214, 1496, 94, 0, 288, 0, 4067] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[3789, 0, 0, 4022, 4096, 12, 186, 3953, 328, 4084, 28, 450, 28, 15, 0, 25, 4076, 0, 0, 20, 3070, 0, 0, 0, 0, 3864, 4094, 74, 11, 0, 98, 112, 0, 0, 0, 61, 0, 20, 0, 2972, 0, 3982, 3966, 0, 0, 14, 0, 6, 63, 0, 0, 3891, 703, 0, 0, 0, 20, 32, 0, 0, 847, 0, 12, 4088, 4079, 0, 314, 160, 6, 0, 0, 17, 4063, 0, 61, 0, 4088, 0, 0, 3873, 0, 10, 2, 4022, 3814, 4096, 0, 158, 4067, 0, 0, 1114, 0, 0, 0, 0, 13, 4081, 3794, 4056, 23, 4074, 1101, 3, 24, 0, 0, 0, 20, 0, 805, 0, 0, 5, 4078, 0, 0, 30, 0, 114, 4065, 4073, 3681, 0, 185, 2089, 0, 3702] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4001, 3, 0, 2447, 4091, 21, 0, 4096, 0, 4096, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 6, 322, 0, 4063, 4096, 0, 173, 0, 0, 2547, 0, 0, 0, 4000, 0, 0, 0, 3908, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4084, 0, 4093, 4053, 0, 0, 4017, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 134, 364, 4096, 0, 0, 1895, 4096, 124, 4096, 0, 5, 4096, 0, 0, 0, 15, 2160, 0, 0, 0, 4096, 4096, 2875, 0, 4096, 6, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 3657, 1490, 0, 0, 0, 3986, 0, 2846] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4013, 0, 0, 3560, 3748, 2355, 0, 4096, 0, 4072, 3719, 1, 2, 0, 0, 0, 4008, 0, 0, 0, 3918, 3, 30, 0, 0, 4086, 4096, 1, 0, 0, 0, 59, 20, 0, 0, 239, 3, 0, 0, 2322, 0, 3860, 4057, 0, 46, 20, 0, 0, 0, 34, 0, 5, 0, 7, 0, 0, 2, 0, 0, 0, 85, 4035, 15, 4080, 4070, 0, 1, 3, 47, 5, 5, 3, 4083, 16, 0, 1, 4090, 4096, 0, 4059, 35, 1, 0, 3971, 1300, 4068, 0, 0, 4089, 47, 10, 0, 558, 1319, 9, 0, 0, 4042, 4090, 3924, 0, 4053, 158, 4, 0, 8, 332, 0, 0, 58, 0, 0, 0, 0, 4091, 94, 0, 0, 364, 0, 3883, 4025, 0, 10, 0, 4096, 79, 1278] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4094, 108, 0, 3506, 4070, 0, 161, 3028, 0, 3663, 3907, 746, 0, 87, 0, 2, 3397, 0, 29, 0, 4077, 110, 12, 10, 19, 3949, 4088, 178, 0, 7, 0, 0, 703, 39, 6, 4096, 158, 0, 0, 0, 0, 4092, 2736, 0, 10, 374, 0, 0, 0, 0, 0, 773, 0, 0, 0, 0, 5, 0, 3, 302, 3330, 4043, 0, 2615, 4096, 241, 0, 3560, 783, 22, 0, 610, 3639, 380, 892, 0, 3361, 98, 176, 3541, 3, 0, 0, 4066, 2945, 908, 0, 0, 215, 0, 6, 531, 1455, 3471, 0, 0, 54, 3953, 230, 4062, 0, 4090, 4026, 0, 0, 0, 0, 0, 2, 79, 1816, 0, 0, 0, 3740, 50, 119, 0, 0, 642, 0, 4091, 0, 1, 2434, 3309, 327, 515] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[462, 1, 0, 3704, 4094, 0, 0, 4085, 0, 4090, 27, 1, 83, 0, 0, 0, 4003, 0, 0, 0, 4043, 0, 480, 3, 1, 4086, 4096, 59, 392, 0, 0, 543, 1, 15, 6, 1941, 505, 0, 0, 3932, 0, 3087, 3931, 0, 31, 0, 17, 0, 0, 0, 8, 0, 0, 0, 1, 0, 5, 1, 12, 0, 0, 2699, 1, 4087, 4091, 0, 0, 2177, 255, 2, 0, 0, 4094, 0, 0, 0, 4096, 2137, 2, 3172, 0, 0, 59, 4086, 0, 4083, 0, 0, 5, 0, 0, 2754, 0, 1, 0, 0, 1, 4085, 4096, 4079, 35, 4096, 4096, 4077, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4, 4072, 0, 0, 0, 4, 0, 2145, 4093, 2518, 0, 0, 4094, 0, 4023] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4027, 4, 0, 4039, 4035, 0, 0, 4091, 0, 4090, 3505, 0, 14, 1, 46, 0, 1382, 75, 0, 0, 4075, 0, 64, 0, 0, 4053, 4083, 0, 39, 0, 1, 0, 2, 216, 45, 0, 8, 0, 0, 1236, 0, 3765, 4070, 0, 0, 0, 0, 2, 0, 0, 10, 0, 0, 0, 4, 0, 127, 3, 0, 5, 4095, 80, 0, 3431, 4094, 0, 4, 0, 0, 21, 0, 0, 4022, 312, 0, 0, 4092, 19, 13, 4090, 8, 0, 1727, 4093, 4, 4092, 0, 0, 4080, 0, 0, 85, 821, 4095, 0, 1, 0, 4087, 4060, 3848, 0, 4090, 6, 44, 0, 0, 2, 0, 2, 51, 4090, 0, 1, 1, 4096, 1, 3, 0, 0, 0, 4080, 4045, 1, 1, 0, 4072, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3999, 0, 6, 4096, 2222, 3964, 0, 4094, 0, 0, 4, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2318, 0, 0, 0, 3974, 189, 0, 14, 2, 4096, 3789, 0, 5, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 7, 0, 13, 318, 0, 3676, 4096, 0, 5, 0, 0, 0, 2675, 1521, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 38, 0, 0, 0, 3, 0, 0, 0, 1, 2089, 2022, 0, 3343, 4096, 3901, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 0, 4051, 0, 0, 63, 0, 4096, 31, 63, 3, 3909, 4094, 927, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 38, 4082, 884, 4096, 2, 0, 4078, 0, 0, 0, 25, 32, 0, 0, 1138, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3957, 4096, 4, 965, 2, 0, 0, 0, 732, 0, 0, 19, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2699, 188, 0, 0, 1, 0, 2267, 214, 1496, 94, 0, 288, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18657, 14474, 16525, 15973, 21486, 14702, 17941, 11314] avg=16384.0000 max=21486 max_rank=4 min=11314 min_rank=7 imbalance_ratio=1.311401 max_load_reduction=109586 max_load_reduction_pct=0.836075 moved_to_echo=65443 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4013, 0, 0, 3560, 2225, 2355, 0, 4096, 0, 0, 3719, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 0, 3918, 3, 30, 0, 0, 4086, 3790, 1, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 239, 3, 0, 0, 2322, 0, 3860, 4057, 0, 46, 20, 0, 0, 2648, 1523, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 5, 0, 7, 0, 0, 2, 0, 0, 0, 85, 4035, 15, 4080, 4072, 3883, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 1, 3, 47, 5, 5, 3, 4083, 16, 0, 1, 4090, 4096, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 1, 0, 3971, 1300, 4068, 0, 0, 4089, 47, 10, 0, 558, 1319, 9, 0, 1137, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 4090, 3924, 0, 955, 158, 4, 0, 8, 332, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2696, 94, 0, 0, 364, 0, 3883, 4025, 0, 10, 0, 4096, 79, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19971, 15895, 14996, 16668, 16596, 16850, 13571, 16525] avg=16384.0000 max=19971 max_rank=0 min=13571 min_rank=6 imbalance_ratio=1.218933 max_load_reduction=111101 max_load_reduction_pct=0.847633 moved_to_echo=63542 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3970, 0, 0, 4087, 2431, 3916, 2, 1631, 0, 0, 165, 622, 264, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4054, 5, 0, 3, 0, 4095, 3572, 0, 0, 93, 51, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2469, 41, 4080, 3149, 81, 0, 0, 73, 0, 2674, 1665, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 36, 0, 0, 0, 0, 568, 0, 0, 8, 87, 9, 0, 3967, 4096, 3908, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 3997, 126, 0, 1, 0, 3655, 0, 958, 122, 4080, 50, 0, 3685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 678, 4033, 0, 3955, 2, 19, 2585, 0, 0, 10, 0, 2360, 0, 2, 1104, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 3642, 4096, 0, 964, 4029, 0, 0, 0, 307, 3, 5, 0, 1845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2617, 348, 0, 25, 619, 149, 4036, 4094, 1869, 0, 0, 48, 0, 1591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17208, 20064, 14485, 13177, 16862, 15038, 18842, 15396] avg=16384.0000 max=20064 max_rank=1 min=13177 min_rank=3 imbalance_ratio=1.224609 max_load_reduction=111008 max_load_reduction_pct=0.846924 moved_to_echo=66138 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[3789, 0, 0, 4022, 2431, 12, 186, 3953, 328, 0, 28, 450, 28, 15, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 20, 3070, 0, 0, 0, 0, 3864, 3788, 74, 11, 0, 98, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 20, 0, 2972, 0, 3982, 3966, 0, 0, 14, 0, 6, 2662, 1665, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 3891, 703, 0, 0, 0, 20, 32, 0, 0, 847, 0, 12, 4088, 4084, 3892, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 314, 160, 6, 0, 0, 17, 4063, 0, 61, 0, 4088, 0, 0, 3873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2, 4022, 3814, 4096, 0, 158, 4067, 0, 0, 1114, 0, 0, 0, 0, 1133, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4081, 3794, 4056, 23, 959, 1101, 3, 24, 0, 0, 0, 20, 0, 805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2688, 0, 0, 30, 0, 114, 4065, 4073, 3681, 0, 185, 2089, 0, 3702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15267, 15113, 15605, 18085, 12769, 18722, 14879, 20632] avg=16384.0000 max=20632 max_rank=7 min=12769 min_rank=4 imbalance_ratio=1.259277 max_load_reduction=110440 max_load_reduction_pct=0.842590 moved_to_echo=67002 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4001, 3, 0, 2447, 2428, 21, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 6, 322, 0, 4063, 3790, 0, 173, 0, 0, 2547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 0, 3908, 0, 4096, 4095, 0, 0, 0, 0, 0, 2676, 1663, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4084, 0, 4093, 4096, 3867, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 0, 4017, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 134, 364, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1895, 4096, 124, 4096, 0, 5, 4096, 0, 0, 0, 15, 2160, 0, 0, 1138, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2875, 0, 965, 6, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2699, 0, 19, 0, 0, 0, 3657, 1490, 0, 0, 0, 3986, 0, 2846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13002, 19093, 20697, 16597, 16989, 17931, 12066, 14697] avg=16384.0000 max=20697 max_rank=2 min=12066 min_rank=6 imbalance_ratio=1.263245 max_load_reduction=110375 max_load_reduction_pct=0.842094 moved_to_echo=61683 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4094, 108, 0, 3506, 2413, 0, 161, 3028, 0, 0, 3907, 746, 0, 87, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3397, 0, 29, 0, 4077, 110, 12, 10, 19, 3949, 3778, 178, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 703, 39, 6, 4096, 158, 0, 0, 0, 0, 4092, 2736, 0, 10, 374, 0, 0, 2676, 1657, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 773, 0, 0, 0, 0, 5, 0, 3, 302, 3330, 4043, 0, 2615, 3663, 3913, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 241, 0, 3560, 783, 22, 0, 610, 3639, 380, 892, 0, 3361, 98, 176, 3541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4066, 2945, 908, 0, 0, 215, 0, 6, 531, 1455, 3471, 0, 0, 1044, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 3953, 230, 4062, 0, 956, 4026, 0, 0, 0, 0, 0, 2, 79, 1816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2457, 50, 119, 0, 0, 642, 0, 4091, 0, 1, 2434, 3309, 327, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18052, 15566, 16786, 19105, 17486, 14954, 15178, 13945] avg=16384.0000 max=19105 max_rank=3 min=13945 min_rank=7 imbalance_ratio=1.166077 max_load_reduction=111967 max_load_reduction_pct=0.854240 moved_to_echo=61563 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[462, 1, 0, 3704, 2430, 0, 0, 4085, 0, 0, 27, 1, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 0, 4043, 0, 480, 3, 1, 4086, 3790, 59, 392, 0, 0, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 15, 6, 1941, 505, 0, 0, 3932, 0, 3087, 3931, 0, 31, 0, 17, 0, 2676, 1664, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1, 0, 5, 1, 12, 0, 0, 2699, 1, 4087, 4090, 3903, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 2177, 255, 2, 0, 0, 4094, 0, 0, 0, 4096, 2137, 2, 3172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 4086, 0, 4083, 0, 0, 5, 0, 0, 2754, 0, 1, 0, 0, 1132, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 4096, 4079, 35, 965, 4096, 4077, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2683, 0, 0, 0, 4, 0, 2145, 4093, 2518, 0, 0, 4094, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10793, 17400, 18063, 15262, 16123, 12426, 21441, 19564] avg=16384.0000 max=21441 max_rank=6 min=10793 min_rank=0 imbalance_ratio=1.308655 max_load_reduction=109631 max_load_reduction_pct=0.836418 moved_to_echo=69554 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4027, 4, 0, 4039, 2395, 0, 0, 4091, 0, 0, 3505, 0, 14, 1, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1382, 75, 0, 0, 4075, 0, 64, 0, 0, 4053, 3778, 0, 39, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 216, 45, 0, 8, 0, 0, 1236, 0, 3765, 4070, 0, 0, 0, 0, 2, 2672, 1640, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4, 0, 127, 3, 0, 5, 4095, 80, 0, 3431, 4090, 3906, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 4, 0, 0, 21, 0, 0, 4022, 312, 0, 0, 4092, 19, 13, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1727, 4093, 4, 4092, 0, 0, 4080, 0, 0, 85, 821, 4095, 0, 1, 1138, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4060, 3848, 0, 964, 6, 44, 0, 0, 2, 0, 2, 51, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2699, 1, 3, 0, 0, 0, 4080, 4045, 1, 1, 0, 4072, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18122, 13467, 13915, 16205, 12761, 20449, 17154, 18999] avg=16384.0000 max=20449 max_rank=5 min=12761 min_rank=4 imbalance_ratio=1.248108 max_load_reduction=110623 max_load_reduction_pct=0.843987 moved_to_echo=69363 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[27392, 128, 128, 128, 31360, 384, 640, 384, 128, 32384, 30080, 384, 640, 128, 256, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[28416, 128, 128, 29568, 19072, 10368, 384, 29184, 384, 0, 11392, 2176, 512, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 4480, 32512, 9088, 29440, 128, 256, 23296, 128, 128, 4608, 2944, 13440, 128, 128, 9088, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1536, 256, 384, 18048, 1280, 128, 128, 640, 31872, 768, 2048, 128, 31872, 10752, 1536, 30592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32512, 28032, 31104, 128, 7808, 13440, 4224, 128, 128, 1408, 128, 128, 256, 12672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[768, 384, 128, 14336, 768, 128, 128, 17280, 128, 30720, 30208, 128, 128, 512, 128, 128, 21376, 13056, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 21248, 768, 256, 128, 1024, 1024, 24192, 26240, 9600, 128, 2688, 22016, 512, 22144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 4864, 768, 128, 128, 128, 768, 128, 128, 384, 10624, 17024, 128, 29824, 32384, 31232, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1, 0, 0, 3986, 11, 2594, 12, 3869, 0, 0, 0, 0, 0, 11, 0, 4081, 0, 3739, 4068, 4078, 0, 0, 0, 4090, 0, 2332, 0, 3915, 17, 55, 0, 0, 0, 5, 25, 893, 87, 0, 11, 4078, 3928, 4096, 0, 120, 0, 0, 372, 134, 0, 0, 1, 4068, 4051, 252, 0, 0, 3727, 0, 0, 27, 22, 19, 2, 0, 0, 0, 4075, 156, 0, 0, 4094, 0, 0, 4065, 0, 0, 4065, 4069, 4096, 0, 0, 4093, 4, 4091, 2972, 0, 0, 57, 0, 1105, 0, 2543, 0, 1990, 0, 0, 44, 0, 0, 13, 0, 3979, 0, 0, 13, 52, 2, 4094, 923, 210, 30, 3956, 0, 3944, 3482, 0, 0, 7, 417, 25, 4, 2, 31, 4096, 0, 1492, 4, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[40, 0, 276, 4095, 966, 3458, 0, 4006, 0, 0, 0, 68, 0, 0, 0, 3904, 0, 3874, 0, 4092, 660, 0, 0, 4096, 1751, 0, 0, 4056, 0, 10, 515, 0, 0, 0, 4090, 2939, 0, 6, 0, 3969, 4026, 4077, 1, 6, 0, 0, 8, 0, 0, 34, 0, 4033, 4094, 9, 13, 652, 4094, 18, 102, 45, 0, 5, 0, 25, 0, 0, 4047, 0, 0, 0, 2687, 13, 1430, 4082, 90, 0, 4065, 4087, 4088, 0, 0, 4073, 0, 3599, 4096, 37, 0, 0, 14, 2, 1325, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 3655, 0, 0, 19, 0, 4096, 0, 218, 4096, 0, 4, 1, 4089, 5, 8, 21, 19, 2662, 0, 2199, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[61, 0, 0, 4079, 304, 2778, 0, 2689, 0, 1, 21, 1012, 1, 0, 0, 4095, 0, 4032, 76, 3978, 0, 348, 0, 3881, 103, 380, 16, 3752, 2, 0, 0, 4070, 51, 0, 4077, 4002, 262, 104, 0, 4078, 4054, 4094, 15, 0, 0, 6, 0, 1, 172, 0, 0, 3994, 3802, 0, 2, 0, 4045, 6, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 24, 0, 2, 3414, 0, 0, 4090, 57, 56, 3943, 4078, 4090, 0, 0, 4040, 14, 3987, 3397, 0, 24, 1492, 0, 0, 102, 774, 0, 4033, 23, 0, 0, 0, 28, 19, 0, 4014, 0, 0, 0, 10, 1415, 257, 172, 0, 2, 3214, 6, 0, 3916, 117, 0, 1, 0, 90, 0, 0, 24, 4072, 0, 3534, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 221, 1432, 3685, 1, 2520, 0, 4096, 0, 5, 326, 0, 400, 0, 45, 3884, 0, 4043, 1, 3538, 0, 0, 0, 2695, 176, 3558, 2034, 4094, 0, 0, 2689, 0, 0, 0, 3885, 41, 0, 0, 20, 3746, 3290, 1776, 1, 0, 0, 0, 696, 773, 0, 993, 8, 3944, 4096, 0, 1226, 0, 3613, 0, 1, 1914, 0, 17, 360, 0, 0, 5, 2722, 0, 0, 0, 0, 0, 4030, 3592, 198, 0, 1894, 2802, 4096, 342, 0, 3506, 0, 4096, 57, 4003, 0, 0, 2659, 515, 2232, 0, 0, 3559, 2, 0, 12, 35, 0, 473, 620, 3992, 0, 0, 1, 104, 5, 21, 156, 2800, 0, 3288, 0, 66, 1972, 158, 248, 0, 175, 1969, 10, 679, 0, 3770, 0, 365, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 2107, 0, 4080, 0, 0, 1260, 1, 0, 0, 1, 4096, 0, 4090, 860, 607, 176, 0, 0, 4087, 319, 3922, 0, 4094, 0, 0, 0, 1441, 167, 0, 3009, 3, 0, 0, 22, 4096, 2482, 4096, 0, 0, 0, 0, 0, 0, 0, 437, 0, 4096, 3700, 0, 0, 2134, 3796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4094, 0, 1, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 3506, 4096, 0, 0, 3252, 0, 1, 0, 3925, 0, 17, 0, 0, 0, 0, 0, 0, 37, 4096, 0, 0, 0, 0, 14, 0, 1, 12, 0, 4086, 0, 1, 4096, 4091, 0, 0, 37, 0, 0, 0, 0, 4096, 0, 3761, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 390, 204, 4078, 0, 1353, 1175, 4007, 0, 1, 1391, 6, 0, 7, 0, 3986, 0, 4039, 65, 3875, 1, 0, 0, 4062, 402, 40, 8, 207, 47, 0, 15, 1926, 1, 0, 4072, 4021, 0, 2, 0, 4075, 4096, 4086, 0, 1, 0, 4, 9, 543, 0, 0, 0, 4048, 4095, 0, 0, 306, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 15, 0, 0, 112, 0, 1, 4096, 0, 0, 4090, 4095, 4096, 0, 27, 4065, 0, 4056, 3265, 26, 0, 4042, 3540, 4094, 394, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 14, 1, 0, 4067, 27, 0, 0, 4096, 0, 3647, 4096, 375, 0, 0, 1909, 0, 0, 432, 0, 3552, 0, 4, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 70, 0, 3968, 8, 3627, 4043, 4075, 0, 18, 1284, 1, 0, 1, 0, 4087, 0, 3942, 161, 2401, 132, 0, 0, 4039, 0, 3691, 0, 3921, 0, 0, 0, 22, 0, 0, 4084, 0, 0, 0, 0, 3897, 3463, 4096, 75, 0, 1, 3716, 0, 2, 0, 0, 0, 4060, 1895, 2, 0, 0, 4032, 0, 0, 0, 0, 5, 0, 244, 12, 0, 2, 1, 354, 29, 4082, 0, 1, 3851, 408, 0, 4095, 4058, 4083, 6, 0, 4063, 0, 3330, 4041, 0, 0, 2058, 0, 0, 0, 0, 166, 0, 0, 0, 0, 4, 6, 1, 416, 3, 36, 52, 45, 0, 0, 2120, 1, 729, 12, 3747, 240, 1871, 4095, 3092, 0, 69, 4066, 0, 0, 550, 0, 2159, 0, 3951, 0, 104] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[22, 0, 0, 4095, 0, 13, 3642, 4092, 0, 0, 68, 0, 6, 0, 1671, 3790, 0, 4077, 6, 4063, 534, 1, 37, 4096, 19, 3548, 1, 3470, 0, 0, 2329, 780, 0, 1, 2727, 2, 2, 0, 0, 4052, 3762, 3674, 0, 14, 0, 0, 0, 0, 0, 113, 23, 4091, 4096, 0, 0, 38, 4096, 0, 0, 0, 1, 0, 0, 20, 0, 556, 4094, 0, 0, 0, 4030, 0, 160, 4096, 0, 6, 4091, 4078, 4090, 4, 0, 4096, 0, 4082, 601, 225, 1, 0, 0, 1000, 117, 0, 0, 4055, 3, 0, 0, 0, 0, 5, 141, 3708, 0, 0, 0, 0, 2721, 119, 13, 18, 0, 4094, 0, 1169, 4096, 3, 0, 0, 4089, 29, 0, 1, 0, 947, 0, 3448, 0, 14] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3641, 11, 2594, 12, 3869, 0, 0, 0, 0, 0, 11, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 4068, 4078, 0, 0, 0, 4090, 0, 2332, 0, 3915, 17, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 25, 893, 87, 0, 11, 3425, 3928, 4096, 0, 120, 0, 0, 372, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4068, 4051, 252, 0, 0, 3727, 0, 0, 27, 22, 19, 2, 0, 2580, 841, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 156, 0, 0, 4094, 0, 0, 1485, 0, 0, 4065, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2589, 4, 4091, 2972, 0, 0, 57, 0, 1105, 0, 2543, 0, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 13, 0, 3979, 0, 0, 13, 52, 2, 4094, 923, 210, 30, 3956, 4096, 2598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3944, 3482, 0, 0, 7, 417, 25, 4, 2, 31, 4096, 0, 1492, 4, 0, 1415, 653, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14220, 18855, 13096, 15679, 17944, 15351, 20010, 15917] avg=16384.0000 max=20010 max_rank=6 min=13096 min_rank=2 imbalance_ratio=1.221313 max_load_reduction=111062 max_load_reduction_pct=0.847336 moved_to_echo=69222 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[40, 0, 276, 3740, 966, 3458, 0, 4006, 0, 0, 0, 68, 0, 0, 0, 3904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 4092, 660, 0, 0, 4096, 1751, 0, 0, 4056, 0, 10, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2939, 0, 6, 0, 3334, 4026, 4077, 1, 6, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4033, 4094, 9, 13, 652, 4094, 18, 102, 45, 0, 5, 0, 25, 2591, 872, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 2687, 13, 1430, 1491, 90, 0, 4065, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2576, 0, 3599, 4096, 37, 0, 0, 14, 2, 1325, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 3655, 0, 0, 19, 0, 4096, 4088, 2692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 4096, 0, 4, 1, 4089, 5, 8, 21, 19, 2662, 0, 2199, 0, 0, 1408, 635, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16458, 15490, 18487, 16676, 17910, 15745, 14586, 15720] avg=16384.0000 max=18487 max_rank=2 min=14586 min_rank=6 imbalance_ratio=1.128357 max_load_reduction=112585 max_load_reduction_pct=0.858955 moved_to_echo=63961 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 221, 1432, 3363, 1, 2520, 0, 4096, 0, 5, 326, 0, 400, 0, 45, 3884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 1, 3538, 0, 0, 0, 2695, 176, 3558, 2034, 4094, 0, 0, 2689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3885, 41, 0, 0, 20, 3144, 3290, 1776, 1, 0, 0, 0, 696, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 993, 8, 3944, 4096, 0, 1226, 0, 3613, 0, 1, 1914, 0, 17, 360, 0, 2282, 913, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2722, 0, 0, 0, 0, 0, 4030, 1310, 198, 0, 1894, 2802, 0, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2210, 0, 4096, 57, 4003, 0, 0, 2659, 515, 2232, 0, 0, 3559, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 35, 0, 473, 620, 3992, 0, 0, 1, 104, 5, 21, 156, 2800, 0, 3288, 4096, 2815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 1972, 158, 248, 0, 175, 1969, 10, 679, 0, 3770, 0, 365, 0, 0, 1216, 602, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16293, 19100, 13626, 19447, 13303, 19333, 18418, 11552] avg=16384.0000 max=19447 max_rank=3 min=11552 min_rank=7 imbalance_ratio=1.186951 max_load_reduction=111625 max_load_reduction_pct=0.851631 moved_to_echo=62606 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[61, 0, 0, 3726, 304, 2778, 0, 2689, 0, 1, 21, 1012, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 76, 3978, 0, 348, 0, 3881, 103, 380, 16, 3752, 2, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 4077, 4002, 262, 104, 0, 3425, 4054, 4094, 15, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 3994, 3802, 0, 2, 0, 4045, 6, 0, 0, 0, 0, 0, 0, 2596, 907, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 24, 0, 2, 3414, 0, 0, 1494, 57, 56, 3943, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2555, 14, 3987, 3397, 0, 24, 1492, 0, 0, 102, 774, 0, 4033, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 19, 0, 4014, 0, 0, 0, 10, 1415, 257, 172, 0, 2, 3214, 4090, 2802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3916, 117, 0, 1, 0, 90, 0, 0, 24, 4072, 0, 3534, 0, 0, 1397, 653, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14688, 16929, 20091, 15612, 17165, 16401, 16023, 14163] avg=16384.0000 max=20091 max_rank=2 min=14163 min_rank=7 imbalance_ratio=1.226257 max_load_reduction=110981 max_load_reduction_pct=0.846718 moved_to_echo=63752 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 70, 0, 3624, 8, 3627, 4043, 4075, 0, 18, 1284, 1, 0, 1, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 161, 2401, 132, 0, 0, 4039, 0, 3691, 0, 3921, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 3273, 3463, 4096, 75, 0, 1, 3716, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 1895, 2, 0, 0, 4032, 0, 0, 0, 0, 5, 0, 244, 2444, 887, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 2, 1, 354, 29, 4082, 0, 1, 1407, 408, 0, 4095, 4058, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2569, 0, 3330, 4041, 0, 0, 2058, 0, 0, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 1, 416, 3, 36, 52, 45, 0, 0, 2120, 1, 729, 12, 3747, 4083, 2740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 1871, 4095, 3092, 0, 69, 4066, 0, 0, 550, 0, 2159, 0, 3951, 0, 104, 1405, 624, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20838, 14682, 18710, 13658, 14455, 12164, 13995, 22570] avg=16384.0000 max=22570 max_rank=7 min=12164 min_rank=5 imbalance_ratio=1.377563 max_load_reduction=108502 max_load_reduction_pct=0.827805 moved_to_echo=63184 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3741, 0, 2107, 0, 4080, 0, 0, 1260, 1, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 860, 607, 176, 0, 0, 4087, 319, 3922, 0, 4094, 0, 0, 0, 1441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 3009, 3, 0, 0, 22, 3440, 2482, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 0, 4096, 3700, 0, 0, 2134, 3796, 0, 0, 0, 0, 0, 0, 0, 2600, 920, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4094, 0, 1, 1496, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2590, 0, 3506, 4096, 0, 0, 3252, 0, 1, 0, 3925, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4096, 0, 0, 0, 0, 14, 0, 1, 12, 0, 4086, 4096, 2842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4091, 0, 0, 37, 0, 0, 0, 0, 4096, 0, 3761, 0, 0, 1416, 656, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15286, 15834, 13219, 17773, 17880, 17387, 15184, 18509] avg=16384.0000 max=18509 max_rank=7 min=13219 min_rank=2 imbalance_ratio=1.129700 max_load_reduction=112563 max_load_reduction_pct=0.858788 moved_to_echo=68960 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 390, 204, 3725, 0, 1353, 1175, 4007, 0, 1, 1391, 6, 0, 7, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 65, 3875, 1, 0, 0, 4062, 402, 40, 8, 207, 47, 0, 15, 1926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4072, 4021, 0, 2, 0, 3423, 4096, 4086, 0, 1, 0, 4, 9, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 4095, 0, 0, 306, 4095, 0, 0, 0, 0, 0, 0, 0, 2600, 909, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 15, 0, 0, 112, 0, 1, 1496, 0, 0, 4090, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2571, 0, 4056, 3265, 26, 0, 4042, 3540, 4094, 394, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 14, 1, 0, 4067, 27, 0, 0, 4096, 4096, 2807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3647, 4096, 375, 0, 0, 1909, 0, 0, 432, 0, 3552, 0, 4, 0, 0, 1405, 652, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16245, 10971, 20258, 16142, 13900, 22017, 15114, 16425] avg=16384.0000 max=22017 max_rank=5 min=10971 min_rank=1 imbalance_ratio=1.343811 max_load_reduction=109055 max_load_reduction_pct=0.832024 moved_to_echo=67456 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 3740, 0, 13, 3642, 4092, 0, 0, 68, 0, 6, 0, 1671, 3790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 6, 4063, 534, 1, 37, 4096, 19, 3548, 1, 3470, 0, 0, 2329, 780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2727, 2, 2, 0, 0, 3403, 3762, 3674, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 23, 4091, 4096, 0, 0, 38, 4096, 0, 0, 0, 1, 0, 0, 20, 2600, 917, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 556, 4094, 0, 0, 0, 4030, 0, 160, 1496, 0, 6, 4091, 4078, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2590, 0, 4082, 601, 225, 1, 0, 0, 1000, 117, 0, 0, 4055, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 141, 3708, 0, 0, 0, 0, 2721, 119, 13, 18, 0, 4094, 4090, 2833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1169, 4096, 3, 0, 0, 4089, 29, 0, 1, 0, 947, 0, 3448, 0, 14, 1416, 649, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17044, 19211, 13585, 16085, 18515, 12674, 17742, 16216] avg=16384.0000 max=19211 max_rank=1 min=12674 min_rank=5 imbalance_ratio=1.172546 max_load_reduction=111861 max_load_reduction_pct=0.853432 moved_to_echo=65147 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 768, 1920, 29312, 1408, 18560, 8960, 30976, 0, 128, 4352, 1152, 512, 128, 1792, 32000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 11008, 29952, 7936, 256, 128, 14848, 2176, 128, 1792, 128, 25472, 0, 18816, 128, 128, 11136, 5248, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 2560, 5248, 26752, 1536, 384, 128, 31104, 2816, 17536, 2176, 27520, 128, 128, 5632, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 20352, 128, 30848, 22528, 4352, 128, 11008, 6272, 6784, 4224, 7296, 256, 17792, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 640, 27264, 256, 384, 128, 22528, 128, 5632, 11776, 768, 128, 30464, 31488, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 512, 1280, 19840, 128, 128, 128, 256, 7936, 10752, 1408, 3840, 128, 30592, 32768, 22144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 128, 25984, 11904, 384, 128, 128, 26880, 29184, 30080, 128, 256, 128, 3840, 1152, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[256, 1664, 128, 32384, 29952, 384, 1280, 3200, 31616, 128, 128, 2048, 128, 128, 384, 384, 20352, 7168, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[9, 70, 109, 0, 54, 0, 8, 0, 0, 0, 4092, 0, 0, 0, 41, 24, 0, 4066, 23, 123, 3982, 553, 4066, 19, 0, 3510, 0, 0, 876, 0, 4096, 35, 0, 3518, 4041, 0, 6, 1, 6, 4058, 1308, 0, 0, 3022, 54, 270, 223, 68, 3887, 0, 4004, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4087, 4056, 0, 7, 4096, 0, 0, 4096, 0, 0, 1, 4063, 0, 0, 0, 3747, 1, 0, 0, 0, 1, 2994, 52, 0, 4090, 0, 761, 6, 0, 0, 0, 4044, 4087, 0, 0, 1, 0, 0, 4, 0, 4091, 0, 0, 31, 80, 4067, 0, 0, 0, 4092, 0, 2444, 0, 0, 13, 88, 80, 4073, 3899, 639, 3376, 4096, 9, 37, 4014, 0, 1422, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 979, 3625, 0, 0, 385, 36, 83, 0, 3033, 96, 0, 853, 7, 14, 398, 3362, 35, 0, 3526, 0, 3901, 0, 0, 4032, 0, 0, 1444, 0, 3517, 1901, 0, 0, 4026, 0, 0, 1, 289, 4096, 0, 0, 0, 4096, 0, 241, 76, 3738, 3913, 0, 3231, 0, 0, 660, 0, 8, 0, 59, 0, 0, 3922, 2649, 4046, 0, 3670, 0, 3791, 2081, 423, 1, 0, 953, 0, 13, 0, 286, 0, 856, 0, 0, 0, 4092, 0, 4085, 61, 0, 875, 0, 0, 505, 26, 3721, 251, 4, 1, 61, 1, 169, 4057, 161, 3879, 0, 0, 0, 830, 3784, 49, 0, 13, 4094, 57, 4048, 3, 8, 1098, 0, 175, 0, 0, 3946, 1234, 3253, 0, 0, 4095, 0, 2100, 1984] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[4, 355, 0, 0, 0, 37, 26, 3, 0, 0, 4096, 0, 50, 0, 75, 1, 0, 2, 86, 26, 4096, 2, 4094, 0, 0, 0, 0, 0, 914, 262, 4093, 0, 0, 4096, 4075, 0, 3, 4013, 165, 4096, 71, 0, 0, 4088, 0, 0, 2770, 41, 4085, 0, 4096, 0, 0, 0, 6, 0, 0, 1133, 17, 0, 3981, 3798, 0, 0, 4021, 0, 1190, 4045, 0, 0, 0, 3922, 0, 105, 0, 0, 2, 0, 7, 0, 0, 3732, 0, 1207, 4083, 0, 4, 0, 0, 6, 1, 3058, 4095, 5, 187, 0, 16, 5, 680, 0, 4093, 4076, 25, 0, 0, 4009, 16, 0, 0, 3759, 52, 0, 83, 0, 0, 127, 0, 2512, 2762, 4095, 0, 4096, 0, 37, 4096, 0, 4096, 4011] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 56, 0, 0, 0, 72, 38, 4053, 1, 0, 0, 12, 340, 11, 2925, 24, 0, 4096, 0, 4041, 0, 0, 0, 70, 75, 201, 66, 4074, 0, 0, 3765, 4095, 1, 0, 4013, 0, 4010, 178, 19, 0, 3755, 0, 0, 4094, 0, 4051, 7, 4007, 0, 97, 0, 0, 3, 100, 0, 12, 9, 2793, 3981, 0, 4, 3431, 0, 986, 3692, 0, 0, 0, 4078, 2, 13, 98, 352, 0, 0, 4, 16, 0, 505, 277, 0, 3946, 207, 764, 0, 8, 19, 5, 4096, 4096, 54, 133, 0, 44, 0, 20, 0, 4090, 4086, 0, 29, 0, 3957, 0, 0, 0, 1370, 0, 1, 1362, 6, 0, 4092, 45, 46, 2522, 3813, 3281, 4011, 0, 216, 4021, 0, 4023, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 975, 0, 899, 0, 0, 0, 0, 321, 0, 4096, 0, 10, 0, 0, 0, 1209, 3953, 0, 0, 4029, 272, 4055, 0, 0, 0, 10, 0, 0, 87, 4055, 571, 0, 4081, 3795, 85, 409, 3, 3321, 3929, 0, 0, 2, 418, 0, 0, 3692, 4096, 3735, 0, 4085, 359, 0, 0, 73, 1, 2, 0, 0, 0, 3920, 4084, 0, 0, 4012, 47, 176, 3943, 0, 0, 0, 2935, 705, 0, 7, 575, 0, 0, 0, 0, 14, 63, 3, 50, 300, 0, 3775, 9, 0, 1, 0, 4096, 3775, 100, 1, 0, 27, 0, 0, 0, 3980, 0, 2, 0, 0, 3620, 0, 1, 0, 4052, 37, 0, 9, 0, 252, 1451, 883, 3846, 4096, 3616, 4095, 3200, 0, 0, 4096, 0, 1051, 3539] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4095, 87, 0, 0, 0, 0, 1337, 2576, 0, 1, 4096, 0, 4096, 0, 0, 1322, 0, 0, 57, 0, 4094, 0, 0, 4096, 4096, 0, 0, 897, 16, 4096, 0, 0, 0, 2846, 0, 0, 239, 0, 4096, 0, 4093, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4084, 4096, 0, 0, 4090, 0, 15, 4096, 0, 0, 0, 4095, 20, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4025, 0, 34, 1, 0, 0, 0, 3756, 4094, 1, 120, 0, 0, 0, 63, 0, 4096, 3990, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4079, 1, 0, 3234, 4096, 4079, 4096, 0, 0, 4096, 0, 4086, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 54, 529, 0, 0, 1, 0, 105, 11, 4095, 29, 1, 0, 0, 0, 4, 150, 0, 69, 4095, 0, 3525, 0, 20, 0, 0, 0, 380, 0, 4095, 0, 0, 4013, 4088, 1, 45, 3372, 6, 4096, 0, 0, 0, 171, 0, 0, 0, 0, 3547, 3, 4095, 0, 0, 0, 0, 0, 0, 4040, 0, 1, 4096, 4096, 627, 0, 4049, 0, 0, 4086, 0, 1, 0, 2887, 0, 0, 6, 1471, 0, 1, 0, 0, 2, 107, 0, 1332, 4025, 92, 4057, 0, 0, 0, 0, 4093, 4095, 43, 1, 0, 3, 0, 4096, 0, 4096, 3961, 0, 0, 0, 4096, 1, 0, 0, 4096, 1, 1887, 0, 0, 9, 3983, 0, 4096, 4083, 577, 13, 4052, 0, 100, 4094, 0, 44, 3976] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 2, 0, 31, 0, 0, 0, 0, 0, 0, 3917, 0, 0, 96, 5, 0, 7, 4074, 30, 36, 4096, 431, 4085, 0, 2, 173, 0, 63, 574, 208, 4044, 0, 0, 3161, 4096, 1, 0, 3674, 709, 4096, 1, 3, 0, 1279, 1, 0, 3380, 24, 4096, 14, 4095, 0, 60, 1, 0, 0, 4, 1852, 18, 1, 2232, 4096, 101, 0, 4090, 0, 94, 4077, 0, 9, 0, 4076, 2, 30, 0, 172, 0, 8, 0, 0, 0, 17, 0, 4092, 4014, 0, 0, 0, 0, 0, 0, 3987, 3778, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 4091, 2, 929, 0, 0, 138, 7, 205, 4066, 3826, 4096, 2122, 4023, 0, 1, 4090, 5, 4041, 3923] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[9, 70, 109, 0, 54, 0, 8, 0, 0, 0, 4092, 0, 0, 0, 41, 24, 4014, 4096, 3190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 23, 123, 3982, 553, 4066, 19, 0, 3510, 0, 0, 876, 0, 3135, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3518, 4041, 0, 6, 1, 6, 842, 1308, 0, 0, 3022, 54, 270, 223, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 0, 3329, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4087, 4056, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 4063, 0, 0, 0, 3747, 1, 0, 0, 0, 3342, 356, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2994, 52, 0, 4090, 0, 761, 6, 0, 0, 0, 4044, 4087, 0, 0, 1, 605, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4091, 0, 0, 31, 80, 4067, 0, 0, 0, 4092, 0, 2444, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 88, 80, 4073, 3899, 639, 3376, 0, 9, 37, 0, 0, 1422, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15707, 20388, 13359, 15375, 19728, 17155, 14970, 14390] avg=16384.0000 max=20388 max_rank=1 min=13359 min_rank=2 imbalance_ratio=1.244385 max_load_reduction=110684 max_load_reduction_pct=0.844452 moved_to_echo=66243 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 979, 3625, 0, 0, 385, 36, 83, 0, 3033, 96, 0, 853, 7, 14, 4095, 3253, 3224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 3362, 35, 0, 3526, 0, 3901, 0, 0, 4032, 0, 0, 1444, 0, 2683, 1901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 1, 289, 840, 0, 0, 0, 4096, 0, 241, 76, 3738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 2678, 0, 0, 660, 0, 8, 0, 59, 0, 0, 3922, 2649, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3670, 0, 3791, 2081, 423, 1, 0, 953, 0, 13, 0, 286, 0, 856, 0, 0, 1622, 309, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4085, 61, 0, 875, 0, 0, 505, 26, 3721, 251, 4, 1, 61, 525, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 169, 4057, 161, 3879, 0, 0, 0, 830, 3784, 49, 0, 13, 4094, 57, 4048, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 1098, 0, 175, 0, 0, 3946, 1234, 0, 0, 0, 0, 0, 2100, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19683, 21282, 13307, 17935, 14037, 14626, 21276, 8926] avg=16384.0000 max=21282 max_rank=1 min=8926 min_rank=7 imbalance_ratio=1.298950 max_load_reduction=109790 max_load_reduction_pct=0.837631 moved_to_echo=58865 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4, 355, 0, 0, 0, 37, 26, 3, 0, 0, 4096, 0, 50, 0, 75, 1, 4096, 4096, 3220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 86, 26, 4096, 2, 4094, 0, 0, 0, 0, 0, 914, 262, 3133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4075, 0, 3, 4013, 165, 850, 71, 0, 0, 4088, 0, 0, 2770, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 3405, 0, 0, 0, 6, 0, 0, 1133, 17, 0, 3981, 3798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 1190, 4045, 0, 0, 0, 3922, 0, 105, 0, 0, 2, 0, 7, 0, 3273, 355, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3732, 0, 1207, 4083, 0, 4, 0, 0, 6, 1, 3058, 4095, 5, 187, 0, 605, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 5, 680, 0, 4093, 4076, 25, 0, 0, 4009, 16, 0, 0, 3759, 52, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 127, 0, 2512, 2762, 4095, 0, 0, 0, 37, 0, 0, 4096, 738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16059, 12615, 20172, 16425, 16946, 17509, 16896, 14450] avg=16384.0000 max=20172 max_rank=2 min=12615 min_rank=1 imbalance_ratio=1.231201 max_load_reduction=110900 max_load_reduction_pct=0.846100 moved_to_echo=65801 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 975, 0, 899, 0, 0, 0, 0, 321, 0, 4096, 0, 10, 0, 0, 0, 4096, 3200, 3089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1209, 3953, 0, 0, 4029, 272, 4055, 0, 0, 0, 10, 0, 0, 87, 3104, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 3795, 85, 409, 3, 3321, 815, 0, 0, 2, 418, 0, 0, 3692, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3735, 0, 3396, 359, 0, 0, 73, 1, 2, 0, 0, 0, 3920, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 47, 176, 3943, 0, 0, 0, 2935, 705, 0, 7, 575, 0, 0, 0, 0, 2887, 352, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 63, 3, 50, 300, 0, 3775, 9, 0, 1, 0, 4096, 3775, 100, 1, 0, 599, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 3980, 0, 2, 0, 0, 3620, 0, 1, 0, 4052, 37, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 252, 1451, 883, 3846, 4096, 3616, 4095, 0, 0, 0, 0, 0, 1051, 652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16686, 17290, 20717, 15570, 15664, 13310, 11884, 19951] avg=16384.0000 max=20717 max_rank=2 min=11884 min_rank=6 imbalance_ratio=1.264465 max_load_reduction=110355 max_load_reduction_pct=0.841942 moved_to_echo=60809 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 0, 56, 0, 0, 0, 72, 38, 4053, 1, 0, 0, 12, 340, 4021, 4011, 3152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2925, 24, 0, 4096, 0, 4041, 0, 0, 0, 70, 75, 201, 66, 3118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3765, 4095, 1, 0, 4013, 0, 832, 178, 19, 0, 3755, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 7, 3331, 0, 97, 0, 0, 3, 100, 0, 12, 9, 2793, 3981, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3431, 0, 986, 3692, 0, 0, 0, 4078, 2, 13, 98, 352, 0, 0, 4, 16, 3342, 354, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 277, 0, 3946, 207, 764, 0, 8, 19, 5, 4096, 4096, 54, 133, 0, 602, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 20, 0, 4090, 4086, 0, 29, 0, 3957, 0, 0, 0, 1370, 0, 1, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1362, 6, 0, 4092, 45, 46, 2522, 3813, 3281, 0, 0, 216, 0, 0, 4023, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15766, 14627, 20752, 14388, 16394, 15226, 13759, 20160] avg=16384.0000 max=20752 max_rank=2 min=13759 min_rank=6 imbalance_ratio=1.266602 max_load_reduction=110320 max_load_reduction_pct=0.841675 moved_to_echo=65539 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 54, 529, 0, 0, 1, 0, 105, 11, 4095, 29, 1, 0, 0, 0, 4094, 4052, 3220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 150, 0, 69, 4095, 0, 3525, 0, 20, 0, 0, 0, 380, 0, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 4088, 1, 45, 3372, 6, 850, 0, 0, 0, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3547, 3, 3405, 0, 0, 0, 0, 0, 0, 4040, 0, 1, 4096, 4096, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 4086, 0, 1, 0, 2887, 0, 0, 6, 1471, 0, 1, 0, 0, 3244, 356, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 107, 0, 1332, 4025, 92, 4057, 0, 0, 0, 0, 4093, 4095, 43, 1, 0, 605, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 3961, 0, 0, 0, 4096, 1, 0, 0, 4096, 1, 1887, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3983, 0, 4096, 4083, 577, 13, 0, 0, 100, 0, 0, 44, 732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16191, 11377, 12546, 19815, 16127, 18977, 22402, 13637] avg=16384.0000 max=22402 max_rank=6 min=11377 min_rank=1 imbalance_ratio=1.367310 max_load_reduction=108670 max_load_reduction_pct=0.829086 moved_to_echo=71143 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4095, 87, 0, 0, 0, 0, 4096, 4096, 3220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1337, 2576, 0, 1, 4096, 0, 4096, 0, 0, 1322, 0, 0, 57, 0, 3134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 897, 16, 850, 0, 0, 0, 2846, 0, 0, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3403, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 15, 4096, 0, 0, 0, 4095, 20, 1, 0, 0, 0, 0, 0, 0, 3342, 355, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4025, 0, 34, 1, 0, 0, 0, 3756, 4094, 1, 120, 0, 605, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 4096, 3990, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 1, 0, 3234, 4096, 4079, 0, 0, 0, 0, 0, 4086, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15596, 16619, 13040, 15684, 16040, 17258, 16506, 20329] avg=16384.0000 max=20329 max_rank=7 min=13040 min_rank=2 imbalance_ratio=1.240784 max_load_reduction=110743 max_load_reduction_pct=0.844902 moved_to_echo=70133 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 31, 0, 0, 0, 0, 0, 0, 3917, 0, 0, 96, 5, 0, 4090, 4023, 3220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4074, 30, 36, 4096, 431, 4085, 0, 2, 173, 0, 63, 574, 208, 3095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3161, 4096, 1, 0, 3674, 709, 850, 1, 3, 0, 1279, 1, 0, 3380, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 3405, 0, 60, 1, 0, 0, 4, 1852, 18, 1, 2232, 4096, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 94, 4077, 0, 9, 0, 4076, 2, 30, 0, 172, 0, 8, 0, 0, 3201, 351, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 4092, 4014, 0, 0, 0, 0, 0, 0, 3987, 3778, 0, 0, 0, 598, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 4091, 2, 929, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 7, 205, 4066, 3826, 4096, 2122, 0, 0, 1, 0, 5, 4041, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15384, 16874, 17179, 15880, 16136, 17011, 13379, 19229] avg=16384.0000 max=19229 max_rank=7 min=13379 min_rank=6 imbalance_ratio=1.173645 max_load_reduction=111843 max_load_reduction_pct=0.853294 moved_to_echo=65755 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[3072, 21120, 256, 256, 32128, 1280, 31872, 128, 128, 9088, 128, 256, 4480, 640, 24576, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 256, 8960, 256, 32512, 16128, 128, 128, 1024, 31744, 128, 128, 128, 29696, 256, 9344, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[31488, 128, 6272, 30208, 512, 128, 128, 27136, 768, 256, 128, 6656, 128, 896, 128, 128, 24320, 2816, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 26752, 32384, 128, 512, 16000, 4608, 6784, 1664, 128, 128, 19712, 128, 512, 14592, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[31488, 128, 26368, 384, 256, 768, 128, 128, 128, 7168, 128, 128, 29184, 30976, 4864, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 1408, 1152, 5120, 128, 128, 512, 128, 640, 128, 31488, 256, 128, 1024, 256, 384, 32640, 30848, 25600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 15616, 384, 10880, 24576, 384, 10368, 128, 128, 640, 128, 30976, 28288, 256, 512, 128, 4864, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1536, 128, 1536, 13952, 1408, 18688, 24448, 24960, 18304, 0, 128, 512, 0, 128, 20864, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 419, 4066, 0, 923, 4005, 0, 0, 0, 338, 1563, 3994, 0, 0, 0, 41, 24, 0, 6, 4051, 0, 26, 0, 0, 4050, 0, 0, 4023, 3973, 0, 4096, 0, 91, 4053, 0, 1185, 0, 4094, 0, 86, 0, 0, 0, 2, 3863, 1751, 4018, 4096, 0, 0, 0, 0, 3782, 9, 122, 0, 0, 706, 238, 5, 0, 3385, 0, 0, 3860, 0, 0, 0, 87, 0, 0, 1796, 0, 0, 0, 0, 0, 4018, 0, 0, 71, 3953, 0, 0, 19, 3742, 444, 2591, 0, 4031, 0, 4096, 4096, 18, 0, 0, 49, 0, 15, 4076, 1974, 59, 0, 0, 39, 595, 0, 0, 458, 3626, 3928, 4096, 75, 0, 0, 188, 3763, 83, 4035, 0, 0, 3954, 0, 0, 153, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[41, 36, 19, 0, 4053, 73, 0, 4096, 128, 0, 62, 222, 1, 3934, 0, 2, 12, 0, 15, 32, 0, 3966, 0, 0, 0, 0, 4090, 0, 2836, 4061, 4096, 0, 4094, 0, 0, 4079, 73, 0, 0, 391, 0, 0, 933, 0, 27, 0, 4073, 1, 3957, 4092, 0, 0, 0, 0, 2421, 0, 0, 15, 0, 0, 0, 0, 0, 4026, 0, 0, 4020, 0, 0, 0, 1, 93, 0, 0, 1413, 0, 0, 7, 0, 3036, 0, 0, 0, 4052, 0, 0, 6, 4064, 9, 4031, 0, 2036, 0, 4043, 4082, 3735, 64, 0, 0, 0, 15, 4059, 3708, 114, 0, 281, 3, 20, 0, 1308, 4094, 4069, 4022, 4064, 0, 36, 0, 0, 2668, 52, 28, 0, 3785, 4096, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[674, 0, 0, 3, 4096, 309, 4, 4085, 4067, 0, 64, 0, 3975, 4024, 810, 0, 1799, 220, 0, 35, 4022, 3579, 13, 132, 0, 227, 4096, 0, 0, 3176, 2331, 2510, 3643, 0, 0, 3813, 0, 0, 0, 53, 0, 0, 3548, 1, 0, 0, 4096, 0, 3086, 2548, 477, 0, 0, 305, 0, 0, 0, 1, 2447, 614, 2, 0, 1322, 3233, 0, 3, 0, 2, 57, 0, 135, 0, 0, 99, 67, 1, 0, 114, 0, 3921, 29, 0, 0, 4091, 0, 1999, 0, 1380, 3, 4088, 0, 3978, 0, 1259, 3447, 0, 639, 0, 0, 0, 13, 4096, 4091, 32, 0, 46, 0, 4096, 0, 0, 4042, 3265, 4086, 4094, 0, 16, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 209, 136] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[13, 0, 0, 27, 4096, 0, 5, 3774, 0, 0, 0, 102, 41, 4070, 0, 0, 1033, 0, 0, 0, 1442, 3282, 15, 0, 11, 0, 4096, 0, 42, 4080, 4091, 1517, 4034, 0, 0, 4083, 109, 3, 0, 0, 397, 0, 25, 0, 18, 0, 4070, 0, 4052, 4083, 0, 9, 6, 0, 98, 5, 0, 6, 47, 2383, 0, 0, 1, 4041, 0, 0, 4080, 6, 0, 1, 0, 23, 0, 268, 4087, 90, 0, 149, 0, 3948, 5, 0, 0, 3976, 1, 2, 10, 4032, 0, 4096, 0, 4090, 0, 3031, 4025, 0, 0, 0, 20, 3, 0, 3860, 2238, 56, 0, 0, 0, 4087, 0, 0, 3917, 4008, 4062, 4093, 0, 1, 950, 32, 4091, 92, 62, 0, 202, 4062, 0, 0, 0, 109] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 134, 1, 66, 3626, 53, 130, 733, 0, 0, 443, 0, 0, 3908, 3, 3, 0, 0, 24, 5, 2162, 4039, 28, 1, 3, 0, 4080, 0, 2, 4007, 2982, 1908, 4024, 0, 0, 4067, 0, 221, 0, 1908, 95, 0, 1040, 0, 0, 0, 4096, 1, 2532, 3163, 3918, 14, 0, 44, 3, 246, 0, 1, 0, 0, 0, 74, 21, 4091, 37, 0, 4096, 0, 0, 0, 0, 4, 0, 20, 3953, 0, 137, 53, 0, 3941, 8, 47, 0, 4092, 0, 0, 13, 4060, 8, 4069, 31, 4038, 49, 4066, 4096, 0, 70, 255, 0, 0, 26, 3642, 3922, 0, 1, 0, 811, 0, 1, 0, 3533, 4077, 3852, 4094, 0, 14, 0, 0, 4082, 170, 2471, 0, 1344, 3989, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[38, 1, 0, 68, 4029, 0, 0, 3962, 48, 0, 0, 0, 1, 4075, 0, 0, 0, 4, 0, 0, 673, 4049, 0, 0, 0, 0, 4063, 0, 566, 3653, 4095, 110, 3342, 0, 0, 4092, 0, 91, 0, 0, 1, 15, 0, 0, 52, 0, 4096, 0, 4096, 4096, 0, 3, 0, 0, 9, 0, 0, 3, 0, 1, 0, 0, 5, 4090, 1, 4, 3864, 0, 19, 0, 0, 0, 1, 3783, 4087, 185, 0, 203, 0, 4096, 6, 0, 0, 4094, 0, 0, 0, 4096, 0, 3937, 0, 3878, 0, 3954, 4095, 4, 0, 0, 0, 791, 293, 3849, 4093, 0, 30, 0, 0, 0, 0, 2, 3989, 4072, 4095, 4089, 0, 2, 0, 0, 4096, 0, 3824, 5, 6, 4091, 2, 9, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 2853, 0, 0, 0, 1588, 331, 4096, 0, 0, 695, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 835, 4096, 4089, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 31, 1, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 3513, 2216, 285, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 4096, 0, 4096, 0, 2216, 4013, 3774, 270, 0, 0, 51, 17, 3934, 4096, 0, 0, 0, 1, 0, 0, 14, 4055, 4096, 4096, 4096, 0, 0, 20, 2, 3269, 0, 1, 0, 2897, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3961, 0, 0, 3600, 0, 0, 569, 0, 42, 4040, 0, 0, 0, 0, 0, 7, 4096, 4096, 38, 0, 0, 0, 4096, 0, 25, 4094, 4091, 0, 4093, 0, 0, 4094, 0, 5, 0, 0, 31, 3, 8, 5, 265, 113, 4088, 295, 4066, 4090, 1, 0, 0, 0, 0, 15, 3, 0, 0, 0, 0, 1, 0, 4050, 0, 0, 4095, 0, 0, 12, 0, 0, 0, 38, 3714, 1293, 0, 8, 0, 4067, 34, 0, 0, 4084, 0, 0, 3, 4096, 0, 4094, 1, 2438, 0, 4079, 4096, 0, 3, 0, 0, 59, 84, 4052, 4055, 0, 0, 0, 1, 0, 1, 0, 4089, 4094, 4091, 4096, 0, 0, 31, 0, 4096, 0, 4095, 0, 0, 4075, 16, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[41, 36, 19, 0, 4053, 73, 0, 4096, 128, 0, 62, 222, 1, 3934, 0, 2, 2524, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 15, 32, 0, 3966, 0, 0, 0, 0, 1476, 0, 2836, 4061, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4079, 73, 0, 0, 391, 0, 0, 933, 0, 27, 0, 4073, 1, 1908, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3957, 4092, 0, 0, 0, 0, 2421, 0, 0, 15, 0, 0, 0, 0, 0, 4026, 2288, 272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 0, 1, 93, 0, 0, 1413, 0, 0, 7, 0, 3036, 4052, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4064, 9, 4031, 0, 2036, 0, 4043, 620, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 15, 3739, 3708, 114, 0, 281, 3, 20, 0, 1308, 4094, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 1884, 0, 36, 0, 0, 2668, 52, 28, 0, 3785, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15517, 16494, 15899, 17071, 13560, 18544, 17415, 16572] avg=16384.0000 max=18544 max_rank=5 min=13560 min_rank=4 imbalance_ratio=1.131836 max_load_reduction=112528 max_load_reduction_pct=0.858521 moved_to_echo=66091 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 419, 4066, 0, 923, 4005, 0, 0, 0, 338, 1563, 3994, 0, 0, 2532, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 24, 0, 6, 4051, 0, 26, 0, 0, 1463, 0, 0, 4023, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 91, 4053, 0, 1185, 0, 4094, 0, 86, 0, 0, 0, 2, 3863, 1751, 1923, 321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 4096, 0, 0, 0, 0, 3782, 9, 122, 0, 0, 706, 238, 5, 0, 3385, 2265, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3860, 0, 0, 0, 87, 0, 0, 1796, 0, 0, 0, 0, 0, 4018, 3953, 942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 19, 3742, 444, 2591, 0, 4031, 0, 4096, 622, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 15, 3755, 1974, 59, 0, 0, 39, 595, 0, 0, 458, 3626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3928, 1899, 75, 0, 0, 188, 3763, 83, 4035, 0, 0, 3954, 0, 0, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18162, 13607, 21465, 18900, 14656, 15634, 10570, 18078] avg=16384.0000 max=21465 max_rank=2 min=10570 min_rank=6 imbalance_ratio=1.310120 max_load_reduction=109607 max_load_reduction_pct=0.836235 moved_to_echo=58938 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 0, 27, 4096, 0, 5, 3774, 0, 0, 0, 102, 41, 4070, 0, 0, 2488, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 0, 0, 0, 1442, 3282, 15, 0, 11, 0, 1479, 0, 42, 4080, 4091, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 4083, 109, 3, 0, 0, 397, 0, 25, 0, 18, 0, 4070, 0, 1922, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 4083, 0, 9, 6, 0, 98, 5, 0, 6, 47, 2383, 0, 0, 1, 4041, 2291, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 6, 0, 1, 0, 23, 0, 268, 4087, 90, 0, 149, 0, 3948, 3976, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 2, 10, 4032, 0, 4096, 0, 4090, 0, 3031, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 3, 0, 3556, 2238, 56, 0, 0, 0, 4087, 0, 0, 3917, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 1897, 0, 1, 950, 32, 4091, 92, 62, 0, 202, 4062, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14942, 16992, 14965, 17296, 17553, 15879, 17885, 15560] avg=16384.0000 max=17885 max_rank=6 min=14942 min_rank=0 imbalance_ratio=1.091614 max_load_reduction=113187 max_load_reduction_pct=0.863548 moved_to_echo=66877 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[674, 0, 0, 3, 4096, 309, 4, 4085, 4067, 0, 64, 0, 3975, 4024, 810, 0, 2135, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1799, 220, 0, 35, 4022, 3579, 13, 132, 0, 227, 1471, 0, 0, 3176, 2331, 2510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3643, 0, 0, 3813, 0, 0, 0, 53, 0, 0, 3548, 1, 0, 0, 4096, 0, 1925, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3086, 2548, 477, 0, 0, 305, 0, 0, 0, 1, 2447, 614, 2, 0, 1322, 3233, 2295, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 57, 0, 135, 0, 0, 99, 67, 1, 0, 114, 0, 3921, 4091, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 1999, 0, 1380, 3, 4088, 0, 3978, 0, 1259, 516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 639, 0, 0, 0, 13, 3768, 4091, 32, 0, 46, 0, 4096, 0, 0, 4042, 3265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1889, 0, 16, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 209, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[24576, 19515, 17407, 16610, 9286, 13252, 19992, 10434] avg=16384.0000 max=24576 max_rank=0 min=9286 min_rank=4 imbalance_ratio=1.500000 max_load_reduction=106496 max_load_reduction_pct=0.812500 moved_to_echo=52964 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 134, 1, 66, 3626, 53, 130, 733, 0, 0, 443, 0, 0, 3908, 3, 3, 2532, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 5, 2162, 4039, 28, 1, 3, 0, 1473, 0, 2, 4007, 2982, 1908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 4067, 0, 221, 0, 1908, 95, 0, 1040, 0, 0, 0, 4096, 1, 1922, 287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2532, 3163, 3918, 14, 0, 44, 3, 246, 0, 1, 0, 0, 0, 74, 21, 4091, 2282, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 4096, 0, 0, 0, 0, 4, 0, 20, 3953, 0, 137, 53, 0, 3941, 4092, 942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 47, 0, 0, 0, 0, 13, 4060, 8, 4069, 31, 4038, 49, 4066, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 255, 0, 0, 26, 3355, 3922, 0, 1, 0, 811, 0, 1, 0, 3533, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3852, 1898, 0, 14, 0, 0, 4082, 170, 2471, 0, 1344, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11957, 16634, 17661, 16663, 17275, 17011, 16051, 17820] avg=16384.0000 max=17820 max_rank=7 min=11957 min_rank=0 imbalance_ratio=1.087646 max_load_reduction=113252 max_load_reduction_pct=0.864044 moved_to_echo=68157 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[38, 1, 0, 68, 4029, 0, 0, 3962, 48, 0, 0, 0, 1, 4075, 0, 0, 2532, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 673, 4049, 0, 0, 0, 0, 1466, 0, 566, 3653, 4095, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3342, 0, 0, 4092, 0, 91, 0, 0, 1, 15, 0, 0, 52, 0, 4096, 0, 1920, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 9, 0, 0, 3, 0, 1, 0, 0, 5, 4090, 2273, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 3864, 0, 19, 0, 0, 0, 1, 3783, 4087, 185, 0, 203, 0, 4096, 4094, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 3937, 0, 3878, 0, 3954, 622, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 791, 293, 3546, 4093, 0, 30, 0, 0, 0, 0, 2, 3989, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1895, 0, 2, 0, 0, 4096, 0, 3824, 5, 6, 4091, 2, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15078, 14616, 13912, 14850, 21278, 16497, 16816, 18025] avg=16384.0000 max=21278 max_rank=4 min=13912 min_rank=2 imbalance_ratio=1.298706 max_load_reduction=109794 max_load_reduction_pct=0.837662 moved_to_echo=72616 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 2853, 0, 0, 0, 1588, 331, 4096, 0, 0, 2481, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 695, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1479, 0, 835, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1923, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 31, 1, 0, 0, 0, 7, 0, 0, 0, 4096, 2291, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 3513, 2216, 285, 0, 0, 0, 4096, 4096, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 4096, 0, 2216, 609, 3774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 0, 0, 51, 17, 3624, 4096, 0, 0, 0, 1, 0, 0, 14, 4055, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1899, 0, 0, 20, 2, 3269, 0, 1, 0, 2897, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15769, 15288, 14521, 14891, 19211, 18888, 16224, 16280] avg=16384.0000 max=19211 max_rank=4 min=14521 min_rank=2 imbalance_ratio=1.172546 max_load_reduction=111861 max_load_reduction_pct=0.853432 moved_to_echo=70603 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3961, 0, 0, 3600, 0, 0, 569, 0, 42, 4040, 0, 0, 2532, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 4096, 38, 0, 0, 0, 1479, 0, 25, 4094, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4094, 0, 5, 0, 0, 31, 3, 8, 5, 265, 113, 4088, 295, 1923, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4090, 1, 0, 0, 0, 0, 15, 3, 0, 0, 0, 0, 1, 0, 4050, 2291, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 12, 0, 0, 0, 38, 3714, 1293, 0, 8, 0, 4067, 4084, 942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 3, 4096, 0, 4094, 1, 2438, 0, 4079, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 59, 84, 3733, 4055, 0, 0, 0, 1, 0, 1, 0, 4089, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1899, 0, 0, 31, 0, 4096, 0, 4095, 0, 0, 4075, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15071, 17926, 15242, 14791, 18253, 15367, 16119, 18303] avg=16384.0000 max=18303 max_rank=7 min=14791 min_rank=3 imbalance_ratio=1.117126 max_load_reduction=112769 max_load_reduction_pct=0.860359 moved_to_echo=68042 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[31488, 0, 128, 32384, 256, 1536, 0, 6528, 640, 128, 5632, 128, 384, 128, 32512, 2048, 15488, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1152, 256, 128, 1024, 512, 29184, 28288, 384, 128, 384, 896, 8832, 128, 1408, 28288, 31360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[3584, 384, 128, 128, 12416, 31232, 128, 256, 128, 256, 11904, 0, 4352, 31232, 29824, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[29952, 30336, 4480, 128, 128, 384, 6400, 384, 128, 128, 2560, 3712, 256, 128, 1408, 31104, 18304, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32256, 15232, 128, 128, 1024, 256, 26112, 512, 14592, 128, 8320, 32512, 128, 128, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 2048, 128, 29568, 512, 31104, 128, 28672, 128, 26752, 4864, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 28160, 128, 128, 128, 256, 128, 128, 9600, 19584, 1920, 256, 640, 0, 31232, 32512, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[768, 256, 128, 640, 32128, 512, 1152, 27136, 4352, 0, 1152, 2304, 6016, 32256, 896, 128, 19840, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 39, 0, 4096, 356, 411, 3865, 0, 0, 321, 0, 0, 34, 4094, 4096, 0, 0, 0, 8, 4093, 0, 0, 0, 0, 0, 95, 0, 4073, 0, 3983, 1551, 4096, 0, 0, 0, 7, 0, 32, 0, 19, 0, 4096, 33, 3635, 0, 1836, 3961, 11, 0, 16, 0, 138, 4096, 3236, 1103, 2, 13, 295, 0, 9, 0, 0, 0, 4095, 4071, 0, 0, 0, 0, 4067, 117, 0, 0, 0, 4072, 4094, 16, 0, 94, 0, 0, 0, 0, 0, 4080, 0, 0, 3546, 0, 0, 16, 998, 3612, 11, 2, 0, 0, 14, 4076, 4045, 10, 4022, 15, 3930, 7, 0, 0, 0, 0, 150, 0, 0, 1, 0, 0, 0, 3901, 0, 4007, 3947, 65, 3962, 99, 4096, 1, 0, 0, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 3737, 0, 0, 0, 1, 1, 0, 4089, 4069, 0, 0, 25, 0, 4090, 0, 472, 249, 28, 44, 0, 2652, 4089, 37, 4038, 1701, 4055, 0, 69, 0, 0, 7, 0, 0, 0, 529, 3669, 0, 4062, 6, 4092, 4096, 3995, 424, 0, 0, 0, 3883, 4038, 0, 2869, 230, 133, 0, 97, 1, 1, 0, 4096, 4018, 0, 3584, 0, 1, 4096, 0, 0, 0, 0, 0, 3182, 0, 3744, 235, 0, 0, 1400, 0, 0, 0, 0, 0, 0, 7, 923, 712, 495, 0, 0, 3855, 0, 0, 0, 4086, 2854, 3, 4093, 0, 4096, 185, 56, 38, 0, 0, 0, 0, 14, 4, 81, 0, 0, 4, 0, 4096, 3037, 0, 4088, 0, 3787, 1, 20, 441, 66] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[12, 4, 0, 4079, 12, 22, 4001, 147, 0, 12, 123, 0, 0, 4075, 4058, 0, 4, 0, 0, 2979, 0, 0, 0, 1, 16, 0, 0, 4095, 11, 3746, 4096, 4030, 0, 0, 1, 0, 0, 43, 0, 0, 9, 4096, 0, 0, 0, 1786, 3808, 3828, 0, 0, 0, 0, 4092, 4080, 0, 0, 1977, 0, 19, 206, 3, 253, 0, 4066, 4040, 0, 4096, 3, 0, 4095, 0, 0, 0, 16, 4096, 3740, 0, 1847, 182, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 267, 3, 0, 4062, 4096, 0, 4080, 0, 15, 0, 4048, 4096, 0, 2389, 1, 3910, 139, 0, 1, 0, 1, 574, 0, 0, 0, 2, 9, 1, 1352, 0, 4096, 418, 0, 4096, 0, 3361, 7, 47, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 5, 0, 4096, 0, 45, 4096, 42, 0, 3992, 40, 0, 0, 4095, 4095, 0, 0, 1, 3448, 0, 0, 0, 5, 3622, 0, 0, 1024, 4086, 0, 4094, 919, 3757, 0, 125, 0, 0, 0, 65, 0, 8, 1, 4084, 0, 80, 0, 4007, 4073, 4053, 0, 0, 0, 0, 4090, 4035, 0, 459, 4096, 30, 0, 0, 1265, 2, 0, 4095, 3881, 0, 0, 0, 0, 4083, 0, 1, 0, 253, 4094, 4095, 0, 0, 398, 0, 0, 0, 30, 0, 0, 185, 232, 4002, 0, 0, 13, 0, 5, 0, 383, 8, 0, 2, 4082, 3995, 0, 4090, 0, 4049, 0, 0, 0, 0, 0, 0, 117, 0, 13, 0, 37, 11, 3741, 0, 4087, 965, 5, 4079, 1, 4096, 6, 0, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[6, 94, 63, 4072, 0, 0, 3900, 0, 19, 65, 0, 0, 72, 3501, 4096, 0, 0, 21, 4068, 898, 0, 0, 0, 8, 3, 39, 2616, 4073, 0, 2190, 4074, 3945, 3, 5, 4, 0, 11, 237, 68, 0, 43, 0, 13, 0, 0, 4096, 4091, 4026, 50, 1, 305, 7, 3955, 4020, 0, 4071, 1959, 0, 103, 0, 91, 0, 0, 4094, 4081, 0, 0, 14, 0, 3804, 0, 0, 354, 73, 4096, 3717, 0, 3488, 0, 1, 233, 0, 0, 0, 0, 2, 705, 27, 0, 0, 0, 4037, 611, 1, 3622, 26, 0, 79, 4096, 4085, 2, 3917, 0, 3864, 0, 7, 0, 1, 172, 0, 0, 0, 0, 0, 115, 21, 15, 3, 4080, 2864, 0, 3996, 10, 4040, 30, 343, 0, 1364] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[52, 0, 0, 4096, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 4093, 2734, 0, 0, 0, 22, 0, 0, 11, 4096, 0, 4096, 4094, 4096, 0, 1607, 0, 0, 0, 4075, 0, 0, 0, 4095, 0, 0, 0, 282, 4096, 4089, 0, 0, 0, 0, 4096, 4096, 0, 1, 4020, 742, 0, 0, 0, 2, 0, 4096, 4093, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 3997, 4095, 0, 1296, 0, 0, 0, 0, 0, 0, 1, 0, 58, 229, 0, 79, 0, 4022, 0, 0, 1140, 0, 0, 776, 4096, 4096, 0, 4096, 0, 3843, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 3805, 0, 4075, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 28, 0, 4096, 4, 0, 4096, 0, 12, 1280, 294, 0, 0, 4089, 4096, 55, 9, 0, 4075, 0, 0, 0, 3, 0, 31, 0, 7, 3998, 0, 3942, 4096, 4076, 2, 3396, 0, 0, 0, 1706, 3, 0, 0, 4077, 0, 179, 103, 2957, 4096, 2349, 0, 0, 0, 0, 4068, 4090, 0, 950, 0, 2218, 0, 673, 0, 4, 0, 4093, 4017, 0, 64, 0, 0, 4043, 1, 0, 0, 0, 3453, 3932, 26, 386, 177, 0, 0, 66, 0, 0, 3314, 0, 0, 0, 0, 0, 5, 2819, 2683, 12, 0, 0, 97, 0, 4050, 4088, 0, 4093, 153, 4073, 0, 0, 0, 0, 8, 246, 0, 0, 0, 0, 0, 0, 0, 62, 4055, 4050, 8, 3721, 14, 4096, 0, 6, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 29, 0, 4093, 36, 1, 4092, 1, 0, 3178, 102, 0, 0, 4093, 4092, 1, 0, 0, 18, 10, 6, 17, 0, 3106, 0, 0, 0, 4096, 3, 4090, 902, 4051, 0, 0, 0, 3, 0, 3792, 20, 96, 0, 4020, 0, 0, 0, 462, 4066, 4013, 0, 96, 0, 0, 4094, 4092, 0, 4096, 4083, 0, 0, 0, 13, 0, 0, 4096, 4095, 0, 4096, 0, 5, 4085, 6, 0, 0, 0, 1022, 2044, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7, 1902, 0, 0, 4096, 0, 0, 1, 4080, 3947, 0, 4091, 0, 4037, 0, 1413, 0, 0, 0, 1, 1, 0, 41, 0, 19, 4, 4096, 0, 4091, 1020, 0, 4085, 0, 4096, 260, 0, 3, 1359] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 39, 0, 2924, 356, 411, 3865, 0, 0, 321, 0, 0, 34, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4093, 0, 0, 0, 0, 0, 95, 0, 332, 0, 3983, 1551, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 32, 0, 19, 0, 4096, 33, 3635, 0, 1836, 3961, 11, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 138, 4096, 3236, 1103, 2, 13, 295, 0, 9, 0, 0, 0, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 2572, 117, 0, 0, 0, 4072, 4094, 16, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 3546, 0, 0, 16, 998, 3612, 11, 2, 0, 3741, 1495, 1172, 883, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4076, 4045, 10, 4022, 15, 3930, 7, 0, 0, 0, 0, 150, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3901, 0, 3124, 3947, 65, 3962, 99, 4096, 1, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16140, 14158, 13713, 12648, 15036, 19786, 16311, 23280] avg=16384.0000 max=23280 max_rank=7 min=12648 min_rank=3 imbalance_ratio=1.420898 max_load_reduction=107792 max_load_reduction_pct=0.822388 moved_to_echo=74413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 28, 0, 2924, 4, 0, 4096, 0, 12, 1280, 294, 0, 0, 4089, 4096, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4075, 0, 0, 0, 3, 0, 31, 0, 7, 326, 0, 3942, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3396, 0, 0, 0, 1706, 3, 0, 0, 4077, 0, 179, 103, 2957, 4096, 2349, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 4090, 0, 950, 0, 2218, 0, 673, 0, 4, 0, 3738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 64, 0, 0, 2557, 1, 0, 0, 0, 3453, 3932, 26, 386, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 3314, 0, 0, 0, 0, 0, 5, 2819, 2683, 12, 0, 0, 3672, 1486, 1172, 893, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 4050, 4088, 0, 4093, 153, 4073, 0, 0, 0, 0, 8, 246, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 3162, 4050, 8, 3721, 14, 4096, 0, 6, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16878, 16565, 18951, 15741, 14613, 16352, 16850, 15122] avg=16384.0000 max=18951 max_rank=2 min=14613 min_rank=4 imbalance_ratio=1.156677 max_load_reduction=112121 max_load_reduction_pct=0.855415 moved_to_echo=62937 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[12, 4, 0, 2912, 12, 22, 4001, 147, 0, 12, 123, 0, 0, 4075, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2979, 0, 0, 0, 1, 16, 0, 0, 334, 11, 3746, 4096, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 43, 0, 0, 9, 4096, 0, 0, 0, 1786, 3808, 3828, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4080, 0, 0, 1977, 0, 19, 206, 3, 253, 0, 3714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 4096, 3, 0, 2590, 0, 0, 0, 16, 4096, 3740, 0, 1847, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 267, 3, 0, 4062, 4096, 0, 4080, 0, 3761, 1505, 1167, 902, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4048, 4096, 0, 2389, 1, 3910, 139, 0, 1, 0, 1, 574, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 1, 1352, 0, 3194, 418, 0, 4096, 0, 3361, 7, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15378, 15217, 13654, 14344, 20610, 24167, 15215, 12487] avg=16384.0000 max=24167 max_rank=5 min=12487 min_rank=7 imbalance_ratio=1.475037 max_load_reduction=106905 max_load_reduction_pct=0.815620 moved_to_echo=72479 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2919, 0, 0, 3737, 0, 0, 0, 1, 1, 0, 4089, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 4090, 0, 472, 249, 28, 44, 0, 2652, 327, 37, 4038, 1701, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 7, 0, 0, 0, 529, 3669, 0, 4062, 6, 4092, 4096, 3995, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 424, 0, 0, 0, 3883, 4038, 0, 2869, 230, 133, 0, 97, 1, 1, 0, 3732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 3584, 0, 1, 2585, 0, 0, 0, 0, 0, 3182, 0, 3744, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1400, 0, 0, 0, 0, 0, 0, 7, 923, 712, 495, 0, 0, 3855, 0, 3762, 1511, 1177, 906, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 2854, 3, 4093, 0, 4096, 185, 56, 38, 0, 0, 0, 0, 14, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 81, 0, 0, 4, 0, 3190, 3037, 0, 4088, 0, 3787, 1, 20, 441, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14816, 17718, 20613, 15408, 17349, 14981, 15468, 14719] avg=16384.0000 max=20613 max_rank=2 min=14719 min_rank=7 imbalance_ratio=1.258118 max_load_reduction=110459 max_load_reduction_pct=0.842735 moved_to_echo=62517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[52, 0, 0, 2924, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2734, 0, 0, 0, 22, 0, 0, 11, 334, 0, 4096, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1607, 0, 0, 0, 4075, 0, 0, 0, 4095, 0, 0, 0, 282, 4096, 4089, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 4020, 742, 0, 0, 0, 2, 0, 3741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 2590, 0, 0, 1, 0, 3997, 4095, 0, 1296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 58, 229, 0, 79, 0, 4022, 0, 0, 1140, 0, 3762, 1506, 1172, 902, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 776, 4096, 4096, 0, 4096, 0, 3843, 0, 24, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3194, 4091, 0, 3805, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15256, 19480, 18327, 16698, 16072, 13101, 16973, 15165] avg=16384.0000 max=19480 max_rank=1 min=13101 min_rank=5 imbalance_ratio=1.188965 max_load_reduction=111592 max_load_reduction_pct=0.851379 moved_to_echo=61311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 2924, 0, 45, 4096, 42, 0, 3992, 40, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3448, 0, 0, 0, 5, 3622, 0, 0, 1024, 333, 0, 4094, 919, 3757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 65, 0, 8, 1, 4084, 0, 80, 0, 4007, 4073, 4053, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4035, 0, 459, 4096, 30, 0, 0, 1265, 2, 0, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 0, 0, 0, 0, 2582, 0, 1, 0, 253, 4094, 4095, 0, 0, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 185, 232, 4002, 0, 0, 13, 0, 5, 0, 383, 8, 3753, 1501, 1172, 900, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4082, 3995, 0, 4090, 0, 4049, 0, 0, 0, 0, 0, 0, 117, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 37, 11, 3741, 0, 3187, 965, 5, 4079, 1, 4096, 6, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19334, 17203, 16579, 17717, 15304, 12414, 16377, 16144] avg=16384.0000 max=19334 max_rank=0 min=12414 min_rank=5 imbalance_ratio=1.180054 max_load_reduction=111738 max_load_reduction_pct=0.852493 moved_to_echo=60239 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[6, 94, 63, 2907, 0, 0, 3900, 0, 19, 65, 0, 0, 72, 3501, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4068, 898, 0, 0, 0, 8, 3, 39, 2616, 332, 0, 2190, 4074, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 4, 0, 11, 237, 68, 0, 43, 0, 13, 0, 0, 4096, 4091, 4026, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 1, 305, 7, 3955, 4020, 0, 4071, 1959, 0, 103, 0, 91, 0, 0, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 14, 0, 2406, 0, 0, 354, 73, 4096, 3717, 0, 3488, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 0, 2, 705, 27, 0, 0, 0, 4037, 611, 1, 3622, 26, 3741, 1398, 1165, 899, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 4096, 4085, 2, 3917, 0, 3864, 0, 7, 0, 1, 172, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 21, 15, 3, 3181, 2864, 0, 3996, 10, 4040, 30, 343, 0, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14723, 18194, 12680, 18301, 18230, 16697, 16265, 15982] avg=16384.0000 max=18301 max_rank=3 min=12680 min_rank=2 imbalance_ratio=1.117004 max_load_reduction=112771 max_load_reduction_pct=0.860374 moved_to_echo=67174 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 29, 0, 2922, 36, 1, 4092, 1, 0, 3178, 102, 0, 0, 4093, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 10, 6, 17, 0, 3106, 0, 0, 0, 334, 3, 4090, 902, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3792, 20, 96, 0, 4020, 0, 0, 0, 462, 4066, 4013, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 4094, 4092, 0, 4096, 4083, 0, 0, 0, 13, 0, 0, 3741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 5, 2583, 6, 0, 0, 0, 1022, 2044, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7, 1902, 0, 0, 4096, 0, 3762, 1502, 1171, 901, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4080, 3947, 0, 4091, 0, 4037, 0, 1413, 0, 0, 0, 1, 1, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 19, 4, 4096, 0, 3190, 1020, 0, 4085, 0, 4096, 260, 0, 3, 1359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18547, 12537, 16555, 20215, 13858, 13574, 17613, 18173] avg=16384.0000 max=20215 max_rank=3 min=12537 min_rank=1 imbalance_ratio=1.233826 max_load_reduction=110857 max_load_reduction_pct=0.845772 moved_to_echo=63218 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 896, 32640, 31232, 128, 30848, 256, 31872, 384, 1536, 128, 128, 256, 1024, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 256, 128, 23424, 512, 512, 32000, 256, 128, 8960, 640, 128, 128, 32256, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32384, 0, 11904, 128, 128, 20480, 128, 128, 384, 384, 24832, 28928, 128, 10880, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 5248, 128, 128, 128, 9984, 128, 128, 640, 28160, 128, 8064, 128, 19584, 32384, 26368, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 15744, 14848, 128, 512, 384, 6912, 128, 256, 6400, 2688, 128, 30208, 21504, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[512, 128, 384, 256, 32384, 31744, 1152, 12544, 16384, 3456, 128, 1024, 1408, 384, 0, 29952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 1536, 128, 0, 11520, 256, 1024, 7808, 384, 1024, 768, 18432, 11008, 128, 17280, 128, 30080, 11904, 9472, 7296, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 256, 128, 13184, 128, 25472, 20480, 128, 31872, 128, 31744, 384, 512, 512, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1, 0, 4040, 38, 1, 0, 42, 0, 4096, 0, 13, 1699, 112, 0, 4096, 10, 0, 0, 325, 0, 0, 0, 0, 125, 4096, 0, 0, 34, 1, 964, 261, 0, 18, 0, 3238, 2334, 4096, 25, 39, 0, 2280, 4096, 0, 0, 67, 517, 2217, 0, 1261, 253, 0, 23, 0, 4089, 0, 4055, 0, 5, 0, 0, 0, 4000, 4017, 0, 4095, 903, 0, 0, 0, 0, 0, 4059, 0, 0, 707, 0, 1769, 3681, 3376, 0, 0, 4070, 3951, 870, 388, 0, 0, 0, 0, 3638, 0, 0, 8, 3717, 3894, 0, 3797, 0, 3973, 3498, 0, 66, 1873, 0, 201, 4007, 0, 0, 130, 0, 0, 455, 3972, 0, 24, 5, 4096, 1152, 0, 0, 0, 0, 0, 2, 0, 4078, 7, 4026] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 1, 22, 0, 0, 4094, 2, 33, 4096, 0, 0, 4056, 2, 3, 0, 0, 0, 26, 0, 90, 0, 4087, 0, 0, 15, 0, 19, 0, 0, 0, 0, 4079, 3781, 4019, 0, 356, 0, 1704, 4096, 261, 8, 2, 43, 554, 0, 4096, 0, 0, 0, 113, 4085, 0, 4094, 10, 176, 0, 0, 4, 4096, 0, 0, 4094, 4095, 0, 0, 0, 3, 1, 4091, 0, 0, 0, 653, 0, 3197, 3823, 0, 0, 4096, 4055, 3967, 17, 1, 0, 0, 0, 4095, 0, 0, 0, 4085, 4053, 0, 175, 4, 4006, 3941, 0, 0, 0, 2, 26, 4096, 3, 3, 0, 0, 0, 1, 3722, 155, 0, 5, 3796, 0, 2037, 0, 273, 86, 0, 0, 4096, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4, 26, 263, 0, 0, 0, 0, 0, 4044, 0, 0, 8, 0, 0, 3644, 4094, 27, 0, 300, 11, 172, 0, 1, 0, 3741, 0, 0, 22, 0, 1953, 59, 0, 0, 2244, 4096, 6, 0, 0, 0, 0, 4032, 4054, 46, 0, 0, 0, 0, 0, 64, 0, 2, 243, 0, 3948, 30, 4096, 0, 3387, 0, 77, 0, 4063, 427, 0, 4095, 0, 1, 19, 0, 0, 275, 4089, 272, 4091, 0, 0, 0, 2910, 598, 1287, 2, 4096, 3925, 2165, 262, 2, 0, 125, 0, 4096, 110, 0, 0, 3985, 2131, 0, 4065, 3688, 4096, 2369, 2, 0, 3734, 0, 0, 4096, 23, 0, 4, 0, 0, 0, 4074, 0, 7, 0, 0, 3951, 1476, 14, 0, 0, 329, 0, 3708, 3769, 0, 3947] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 34, 4079, 0, 0, 0, 4, 2, 4096, 24, 204, 22, 0, 1045, 0, 0, 0, 537, 4096, 46, 3082, 0, 0, 0, 0, 66, 0, 0, 4096, 1603, 4091, 39, 0, 20, 2695, 4074, 0, 0, 4058, 0, 0, 396, 4067, 0, 1, 29, 0, 4076, 2, 4092, 0, 11, 0, 1, 4095, 3537, 0, 1, 4047, 4043, 120, 0, 0, 0, 0, 3573, 0, 4, 0, 3863, 0, 0, 1209, 50, 0, 4066, 4083, 4096, 2, 0, 0, 3, 0, 4065, 0, 10, 10, 4073, 3784, 2, 3951, 617, 4079, 4065, 0, 0, 0, 0, 0, 4063, 20, 0, 0, 0, 18, 0, 4036, 0, 0, 0, 107, 0, 0, 0, 22, 21, 0, 0, 2581, 3948, 28, 4088] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[151, 1, 3368, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 0, 4096, 332, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 545, 4096, 0, 0, 0, 4074, 4096, 0, 0, 22, 0, 4, 0, 4096, 2861, 0, 0, 0, 4096, 0, 4096, 0, 481, 0, 0, 9, 4093, 0, 0, 4096, 703, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 43, 0, 0, 1195, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 4096, 1, 4091, 2, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4075, 2562, 3851, 378, 1160, 0, 16, 0, 2836, 4094, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[56, 2, 23, 3, 0, 0, 12, 3, 4013, 0, 2, 148, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4059, 0, 0, 0, 0, 8, 38, 0, 0, 29, 4096, 4077, 4092, 0, 0, 0, 4094, 4096, 0, 0, 3194, 0, 0, 178, 3227, 0, 0, 0, 0, 4055, 0, 4091, 0, 0, 0, 0, 135, 4088, 1, 5, 4091, 526, 0, 2, 0, 46, 0, 4093, 0, 25, 3, 3827, 154, 2693, 305, 0, 0, 4075, 4072, 4096, 17, 0, 181, 1, 665, 4096, 0, 0, 1, 4094, 4096, 0, 3553, 65, 4096, 3978, 0, 0, 0, 0, 0, 4090, 0, 3, 0, 0, 0, 0, 4091, 4, 0, 0, 2682, 1150, 0, 0, 3718, 0, 2, 0, 2366, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 6, 1, 0, 38, 4, 0, 0, 3961, 2, 149, 4046, 0, 59, 3865, 929, 1, 4, 1, 43, 0, 0, 0, 0, 4095, 0, 435, 131, 0, 0, 0, 243, 0, 0, 3683, 4086, 3748, 100, 12, 0, 4005, 4023, 0, 0, 0, 47, 230, 135, 3757, 1, 0, 57, 0, 4078, 0, 3927, 0, 0, 23, 73, 0, 4090, 438, 0, 4096, 3310, 1, 0, 0, 0, 0, 3842, 10, 0, 0, 813, 0, 0, 4094, 274, 165, 4096, 4031, 3978, 0, 3, 717, 14, 0, 2920, 91, 0, 0, 3990, 3712, 0, 4085, 0, 3973, 4096, 8, 0, 169, 14, 4, 4069, 0, 9, 119, 17, 1, 0, 4006, 0, 0, 0, 4085, 0, 109, 0, 0, 0, 0, 0, 3623, 3918, 0, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 75, 0, 0, 0, 10, 0, 50, 4093, 0, 5, 0, 0, 0, 4096, 2, 720, 2, 5, 2632, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 520, 0, 8, 0, 4096, 3630, 4094, 0, 0, 0, 4093, 4077, 0, 2, 4079, 3, 0, 1, 4079, 0, 0, 0, 0, 4085, 8, 4082, 0, 0, 1, 0, 0, 3554, 0, 8, 4096, 4045, 0, 50, 0, 3, 11, 4096, 1, 676, 12, 1084, 0, 0, 3971, 1, 14, 4083, 4072, 4091, 44, 0, 0, 0, 0, 4086, 33, 4, 0, 4088, 4073, 66, 1414, 1852, 4094, 3188, 0, 0, 20, 0, 25, 4095, 0, 0, 0, 0, 14, 16, 4096, 0, 1, 0, 2, 17, 14, 929, 205, 12, 2, 0, 4063, 4076, 1, 4016] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 1, 22, 0, 0, 4094, 2, 33, 4096, 0, 0, 4056, 2, 3088, 899, 856, 300, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 26, 0, 90, 0, 4087, 0, 0, 15, 0, 19, 0, 0, 4096, 3961, 2565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 3781, 4019, 0, 356, 0, 1704, 675, 261, 8, 2, 43, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 113, 4085, 0, 3195, 10, 176, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3794, 4095, 0, 0, 0, 3, 1, 4091, 0, 0, 0, 653, 0, 3197, 3823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 967, 3967, 17, 1, 0, 0, 0, 4095, 0, 0, 0, 4085, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 4, 4006, 3941, 0, 0, 0, 2, 26, 3905, 3, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3722, 155, 0, 5, 3796, 0, 2037, 0, 273, 86, 0, 0, 4096, 4096, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17644, 14862, 15482, 15775, 19657, 17185, 12066, 18401] avg=16384.0000 max=19657 max_rank=4 min=12066 min_rank=6 imbalance_ratio=1.199768 max_load_reduction=111415 max_load_reduction_pct=0.850029 moved_to_echo=67309 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4040, 38, 1, 0, 42, 0, 4096, 0, 13, 1699, 112, 0, 4096, 10, 3009, 891, 856, 300, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 0, 0, 125, 4096, 0, 0, 34, 1, 964, 261, 0, 4070, 3894, 2565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 3238, 2334, 4096, 25, 39, 0, 2280, 675, 0, 0, 67, 517, 2217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1261, 253, 0, 23, 0, 4089, 0, 3164, 0, 5, 0, 0, 0, 4000, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3795, 903, 0, 0, 0, 0, 0, 4059, 0, 0, 707, 0, 1769, 3681, 3376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 942, 870, 388, 0, 0, 0, 0, 3638, 0, 0, 8, 3717, 3894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3797, 0, 3973, 3498, 0, 66, 1873, 0, 201, 3820, 0, 0, 130, 0, 0, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 0, 24, 5, 4096, 1152, 0, 0, 0, 0, 0, 2, 0, 4078, 7, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19391, 16335, 15506, 16812, 18290, 13457, 17813, 13468] avg=16384.0000 max=19391 max_rank=0 min=13457 min_rank=5 imbalance_ratio=1.183533 max_load_reduction=111681 max_load_reduction_pct=0.852058 moved_to_echo=63028 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4, 26, 263, 0, 0, 0, 0, 0, 4044, 0, 0, 8, 0, 0, 3644, 4094, 2992, 903, 852, 304, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 300, 11, 172, 0, 1, 0, 3741, 0, 0, 22, 0, 1953, 59, 0, 4096, 3821, 2544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2244, 4096, 6, 0, 0, 0, 0, 4032, 658, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 2, 243, 0, 3948, 30, 3193, 0, 3387, 0, 77, 0, 4063, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3791, 0, 1, 19, 0, 0, 275, 4089, 272, 4091, 0, 0, 0, 2910, 598, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 933, 2165, 262, 2, 0, 125, 0, 4096, 110, 0, 0, 3985, 2131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 3688, 4096, 2369, 2, 0, 3734, 0, 0, 3903, 23, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 7, 0, 0, 3951, 1476, 14, 0, 0, 329, 0, 3708, 3769, 0, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17327, 16747, 11082, 15434, 17333, 13811, 21884, 17454] avg=16384.0000 max=21884 max_rank=6 min=11082 min_rank=2 imbalance_ratio=1.335693 max_load_reduction=109188 max_load_reduction_pct=0.833038 moved_to_echo=70482 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 34, 4079, 0, 0, 0, 4, 2, 4096, 24, 3109, 899, 851, 297, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 22, 0, 1045, 0, 0, 0, 537, 4096, 46, 3082, 0, 0, 0, 0, 66, 4066, 3954, 2552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1603, 4091, 39, 0, 20, 2695, 671, 0, 0, 4058, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 1, 29, 0, 4076, 2, 3193, 0, 11, 0, 1, 4095, 3537, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3750, 4043, 120, 0, 0, 0, 0, 3573, 0, 4, 0, 3863, 0, 0, 1209, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 974, 4096, 2, 0, 0, 3, 0, 4065, 0, 10, 10, 4073, 3784, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 617, 4079, 4065, 0, 0, 0, 0, 0, 3874, 20, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 107, 0, 0, 0, 22, 21, 0, 0, 2581, 3948, 28, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13588, 19670, 17669, 19013, 16612, 17019, 16624, 10877] avg=16384.0000 max=19670 max_rank=1 min=10877 min_rank=7 imbalance_ratio=1.200562 max_load_reduction=111402 max_load_reduction_pct=0.849930 moved_to_echo=61132 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 1, 0, 38, 4, 0, 0, 3961, 2, 149, 4046, 0, 59, 3865, 929, 3069, 862, 841, 300, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 1, 43, 0, 0, 0, 0, 4095, 0, 435, 131, 0, 0, 0, 243, 4096, 3950, 2520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3683, 4086, 3748, 100, 12, 0, 4005, 662, 0, 0, 0, 47, 230, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3757, 1, 0, 57, 0, 4078, 0, 3065, 0, 0, 23, 73, 0, 4090, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 3310, 1, 0, 0, 0, 0, 3842, 10, 0, 0, 813, 0, 0, 4094, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 962, 3978, 0, 3, 717, 14, 0, 2920, 91, 0, 0, 3990, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 3973, 4096, 8, 0, 169, 14, 4, 3879, 0, 9, 119, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 4085, 0, 109, 0, 0, 0, 0, 0, 3623, 3918, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18322, 15519, 16708, 15582, 16140, 16552, 16374, 15875] avg=16384.0000 max=18322 max_rank=0 min=15519 min_rank=1 imbalance_ratio=1.118286 max_load_reduction=112750 max_load_reduction_pct=0.860214 moved_to_echo=64941 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[151, 1, 3368, 0, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 0, 4096, 332, 3119, 900, 856, 300, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3961, 2565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 545, 4096, 0, 0, 0, 4074, 675, 0, 0, 22, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2861, 0, 0, 0, 4096, 0, 3196, 0, 481, 0, 0, 9, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 703, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 43, 0, 0, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 977, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 4096, 4096, 0, 0, 0, 0, 0, 3905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4075, 2562, 3851, 378, 1160, 0, 16, 0, 2836, 4094, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17424, 14722, 13512, 18832, 9829, 17360, 16190, 23203] avg=16384.0000 max=23203 max_rank=7 min=9829 min_rank=4 imbalance_ratio=1.416199 max_load_reduction=107869 max_load_reduction_pct=0.822975 moved_to_echo=66582 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[56, 2, 23, 3, 0, 0, 12, 3, 4013, 0, 2, 148, 0, 0, 4096, 0, 3101, 898, 856, 300, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4059, 0, 0, 0, 0, 8, 38, 0, 4075, 3961, 2565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4096, 4077, 4092, 0, 0, 0, 4094, 675, 0, 0, 3194, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3227, 0, 0, 0, 0, 4055, 0, 3193, 0, 0, 0, 0, 135, 4088, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3791, 526, 0, 2, 0, 46, 0, 4093, 0, 25, 3, 3827, 154, 2693, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 971, 4096, 17, 0, 181, 1, 665, 4096, 0, 0, 1, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3553, 65, 4096, 3978, 0, 0, 0, 0, 0, 3899, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 0, 2682, 1150, 0, 0, 3718, 0, 2, 0, 2366, 4096, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13704, 14708, 20435, 14704, 15465, 18218, 15594, 18244] avg=16384.0000 max=20435 max_rank=2 min=13704 min_rank=0 imbalance_ratio=1.247253 max_load_reduction=110637 max_load_reduction_pct=0.844093 moved_to_echo=67521 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 75, 0, 0, 0, 10, 0, 50, 4093, 0, 5, 0, 0, 0, 4096, 2, 3101, 897, 852, 300, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 720, 2, 5, 2632, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 520, 0, 4083, 3884, 2553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 3630, 4094, 0, 0, 0, 4093, 672, 0, 2, 4079, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 4085, 8, 3185, 0, 0, 1, 0, 0, 3554, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 4045, 0, 50, 0, 3, 11, 4096, 1, 676, 12, 1084, 0, 0, 3971, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 971, 4091, 44, 0, 0, 0, 0, 4086, 33, 4, 0, 4088, 4073, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1414, 1852, 4094, 3188, 0, 0, 20, 0, 25, 3904, 0, 0, 0, 0, 14, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 2, 17, 14, 929, 205, 12, 2, 0, 4063, 4076, 1, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13672, 18509, 20678, 14920, 17746, 17470, 14527, 13550] avg=16384.0000 max=20678 max_rank=2 min=13550 min_rank=7 imbalance_ratio=1.262085 max_load_reduction=110394 max_load_reduction_pct=0.842239 moved_to_echo=63293 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32128, 256, 128, 128, 18944, 8832, 7552, 1408, 5504, 128, 384, 128, 23296, 32128, 128, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 2304, 31488, 20096, 28288, 256, 512, 128, 27008, 5376, 384, 128, 11520, 640, 3072, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[24704, 3200, 128, 384, 128, 32512, 128, 25472, 128, 4096, 128, 256, 4352, 31616, 4992, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[25216, 6272, 32512, 29312, 128, 128, 5888, 128, 256, 31104, 128, 128, 256, 128, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 0, 7808, 27392, 768, 128, 1024, 256, 768, 31104, 256, 128, 128, 32128, 29952, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[30336, 17664, 128, 128, 0, 128, 384, 32000, 384, 4864, 768, 10368, 2048, 12544, 18688, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[1024, 128, 640, 3840, 256, 128, 128, 768, 32384, 128, 3584, 256, 128, 3072, 896, 384, 32768, 31488, 20480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[256, 128, 7808, 128, 128, 128, 128, 128, 32512, 128, 256, 10112, 128, 128, 32128, 5504, 24704, 7168, 6912, 2432, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 3801, 324, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 4096, 2, 13, 23, 3299, 72, 0, 3307, 4096, 1, 0, 2802, 0, 3733, 823, 3237, 0, 0, 67, 0, 0, 0, 62, 18, 0, 0, 947, 4012, 10, 0, 4, 0, 0, 0, 0, 39, 2, 3798, 0, 3193, 548, 73, 77, 4, 7, 13, 0, 3784, 0, 0, 4096, 0, 0, 4062, 97, 0, 1217, 0, 0, 86, 4080, 4026, 0, 0, 2, 4089, 1736, 0, 0, 705, 0, 0, 0, 1202, 2194, 3, 118, 32, 3694, 3510, 0, 0, 2625, 3997, 0, 22, 0, 4090, 0, 0, 0, 34, 397, 68, 0, 408, 4096, 0, 0, 3932, 614, 0, 4032, 0, 997, 135, 4096, 3919, 0, 0, 0, 0, 0, 4087, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4067, 17, 0, 4066, 5, 4096, 0, 0, 0, 3922, 0, 3877, 0, 4096, 1994, 72, 4090, 0, 4061, 4031, 0, 32, 0, 0, 3938, 0, 1876, 4, 0, 184, 0, 27, 0, 0, 0, 0, 0, 3909, 4096, 15, 0, 0, 0, 0, 12, 0, 20, 0, 4070, 59, 4072, 92, 2, 10, 0, 0, 0, 263, 168, 0, 33, 4034, 0, 405, 4078, 2, 1082, 4054, 1, 0, 542, 3058, 201, 0, 0, 16, 4096, 0, 0, 0, 0, 47, 0, 0, 4085, 3547, 0, 10, 0, 4095, 4096, 0, 0, 576, 0, 0, 0, 42, 4075, 36, 0, 2, 3166, 0, 0, 10, 290, 4090, 0, 0, 4053, 0, 0, 3992, 0, 0, 0, 4008, 3742, 35, 121, 0, 0, 0, 18, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4068, 4, 0, 3979, 0, 4092, 0, 872, 228, 4096, 0, 4096, 0, 3766, 64, 0, 2770, 66, 3293, 3814, 0, 41, 0, 0, 4096, 0, 4038, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 1, 0, 49, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 0, 37, 0, 0, 680, 0, 767, 0, 0, 0, 0, 0, 1027, 1821, 93, 0, 0, 0, 0, 4081, 3045, 0, 0, 0, 0, 3287, 0, 325, 640, 0, 3496, 110, 1, 4096, 4003, 59, 0, 68, 4094, 4017, 0, 0, 2934, 4090, 0, 943, 26, 3957, 0, 0, 1983, 3983, 0, 5, 0, 1319, 4096, 0, 0, 4082, 0, 27, 4038, 0, 0, 0, 3886, 4044, 88, 0, 0, 10, 0, 131, 55, 3997] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4050, 5, 173, 4086, 0, 3993, 0, 22, 136, 3748, 220, 4044, 0, 4084, 0, 0, 4070, 0, 3559, 4005, 0, 21, 0, 0, 4002, 0, 3952, 0, 68, 0, 0, 13, 3, 94, 72, 547, 117, 15, 3972, 0, 0, 95, 0, 0, 0, 0, 74, 0, 4067, 0, 3882, 0, 0, 0, 18, 115, 0, 0, 0, 44, 0, 3820, 0, 108, 4025, 0, 0, 0, 0, 0, 7, 0, 2191, 32, 0, 261, 4090, 0, 0, 13, 1, 2298, 0, 0, 4095, 3292, 0, 12, 1, 3987, 4000, 0, 0, 3935, 0, 0, 0, 0, 4016, 1, 0, 0, 4096, 0, 4, 0, 4096, 4092, 10, 12, 4082, 186, 0, 4096, 38, 0, 876, 2, 4045, 0, 0, 0, 325, 1, 3421, 45, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[26, 3629, 0, 0, 4096, 0, 4078, 0, 0, 0, 4056, 20, 4096, 0, 3816, 69, 0, 4073, 0, 4096, 4085, 0, 0, 798, 30, 4080, 0, 4096, 0, 0, 248, 0, 5, 0, 1, 0, 56, 0, 2, 94, 8, 0, 588, 44, 7, 6, 0, 0, 239, 4063, 0, 3949, 0, 0, 23, 0, 0, 0, 3505, 0, 2, 0, 4032, 0, 0, 4096, 0, 0, 330, 0, 0, 0, 5, 3, 0, 0, 82, 4095, 0, 0, 0, 0, 4096, 11, 37, 1889, 3977, 1, 0, 0, 4096, 4093, 0, 88, 4015, 4086, 0, 12, 0, 4094, 11, 0, 58, 5, 0, 0, 0, 2840, 4069, 0, 0, 4090, 0, 0, 4095, 0, 12, 0, 3602, 4054, 35, 0, 0, 0, 0, 2998, 0, 4081] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[1, 4096, 0, 1, 4096, 0, 4074, 0, 0, 0, 1253, 93, 4079, 0, 4068, 375, 1, 3690, 0, 4033, 4084, 0, 0, 24, 1, 4079, 14, 3900, 0, 0, 446, 0, 1, 0, 0, 8, 0, 0, 1523, 39, 0, 0, 0, 1, 0, 859, 2, 0, 0, 2928, 0, 4096, 0, 27, 0, 0, 0, 0, 0, 254, 0, 16, 4093, 0, 4, 3790, 0, 0, 3681, 0, 0, 3646, 4068, 353, 0, 0, 7, 4095, 1, 0, 0, 0, 893, 0, 97, 3921, 3986, 4, 3, 0, 3973, 4096, 0, 0, 4096, 0, 0, 209, 0, 4093, 3, 0, 15, 529, 0, 0, 0, 4093, 4096, 0, 10, 4096, 0, 0, 4095, 51, 0, 0, 578, 4050, 0, 0, 0, 0, 0, 4094, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 8, 0, 4096, 0, 0, 2, 0, 4067, 0, 4096, 4096, 0, 0, 0, 0, 3809, 0, 4096, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 3909, 5, 0, 0, 0, 0, 0, 0, 0, 63, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 68, 4091, 0, 1919, 3753, 0, 0, 323, 0, 1, 2, 0, 0, 4096, 0, 0, 0, 36, 47, 0, 0, 4096, 4095, 0, 0, 0, 4096, 4085, 0, 0, 4091, 3884, 0, 0, 0, 4096, 0, 0, 0, 2452, 0, 0, 0, 4033, 4096, 0, 169, 4096, 0, 0, 4096, 0, 0, 209, 4033, 4096, 0, 0, 0, 0, 0, 4079, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4073, 1, 139, 4091, 0, 4095, 0, 4, 0, 1, 4, 4095, 0, 1, 90, 6, 4015, 0, 4089, 4091, 0, 0, 464, 16, 4066, 0, 2985, 1, 0, 46, 7, 0, 0, 0, 7, 9, 0, 0, 21, 0, 0, 2, 0, 0, 0, 0, 71, 0, 4081, 0, 4086, 0, 0, 0, 0, 0, 0, 1129, 32, 0, 0, 4092, 0, 0, 4096, 0, 0, 267, 7, 0, 286, 0, 3215, 0, 0, 48, 4096, 0, 0, 1, 15, 4095, 25, 0, 4096, 4096, 0, 11, 0, 4061, 4080, 0, 10, 3633, 4060, 0, 5, 5, 4096, 0, 0, 343, 4030, 1, 0, 0, 4089, 4096, 17, 0, 4086, 0, 0, 4086, 0, 0, 1, 4096, 3981, 0, 0, 0, 0, 13, 3931, 0, 4088] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4067, 17, 0, 3102, 5, 0, 0, 0, 0, 3922, 0, 3877, 0, 4096, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 4090, 0, 4061, 514, 0, 32, 0, 0, 3938, 0, 1876, 4, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 3909, 4096, 15, 0, 0, 0, 0, 12, 0, 20, 4096, 4095, 3517, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 59, 4072, 92, 2, 10, 0, 0, 0, 263, 168, 0, 33, 4034, 0, 2125, 1036, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 4078, 2, 1082, 4054, 1, 0, 542, 3058, 201, 0, 0, 16, 4096, 0, 0, 964, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 4085, 3547, 0, 10, 0, 0, 1448, 0, 0, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4075, 36, 0, 2, 3166, 0, 0, 10, 290, 2769, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 0, 0, 0, 4008, 3742, 35, 121, 0, 0, 0, 18, 0, 1966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[21080, 14771, 21399, 16349, 19435, 9713, 14443, 13882] avg=16384.0000 max=21399 max_rank=2 min=9713 min_rank=5 imbalance_ratio=1.306091 max_load_reduction=109673 max_load_reduction_pct=0.836739 moved_to_echo=57473 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3801, 324, 0, 3120, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 13, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3299, 72, 0, 3307, 518, 1, 0, 2802, 0, 3733, 823, 3237, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 18, 0, 0, 947, 4012, 10, 0, 4, 0, 0, 0, 0, 39, 4096, 3694, 3578, 1385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3798, 0, 3193, 548, 73, 77, 4, 7, 13, 0, 3784, 0, 0, 4096, 0, 2132, 893, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 97, 0, 1217, 0, 0, 86, 4080, 4026, 0, 0, 2, 4089, 1736, 0, 975, 940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 705, 0, 0, 0, 1202, 2194, 3, 118, 32, 0, 1232, 0, 0, 2625, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4090, 0, 0, 0, 34, 397, 68, 0, 408, 2765, 0, 0, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 614, 0, 4032, 0, 997, 135, 4096, 3919, 0, 0, 0, 0, 0, 4087, 0, 1964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11379, 17859, 17845, 19011, 21310, 12108, 11716, 19844] avg=16384.0000 max=21310 max_rank=4 min=11379 min_rank=0 imbalance_ratio=1.300659 max_load_reduction=109762 max_load_reduction_pct=0.837418 moved_to_echo=64978 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4068, 4, 0, 3036, 0, 0, 0, 872, 228, 4096, 0, 4096, 0, 3766, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2770, 66, 3293, 486, 0, 41, 0, 0, 4096, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 1, 0, 49, 0, 0, 0, 0, 0, 0, 4092, 4094, 3328, 1581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 0, 37, 0, 0, 680, 0, 767, 0, 0, 0, 0, 0, 2076, 1016, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1027, 1821, 93, 0, 0, 0, 0, 4081, 3045, 0, 0, 0, 0, 3287, 0, 325, 943, 937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 640, 0, 3496, 110, 1, 4096, 4003, 59, 0, 68, 0, 1420, 0, 0, 2934, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 943, 26, 3957, 0, 0, 1983, 3983, 0, 5, 0, 1319, 2774, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4038, 0, 0, 0, 3886, 4044, 88, 0, 0, 10, 0, 131, 55, 1921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20230, 14790, 13171, 13133, 15559, 20917, 19072, 14200] avg=16384.0000 max=20917 max_rank=5 min=13133 min_rank=3 imbalance_ratio=1.276672 max_load_reduction=110155 max_load_reduction_pct=0.840416 moved_to_echo=69748 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[26, 3629, 0, 0, 3125, 0, 0, 0, 0, 0, 4056, 20, 4096, 0, 3816, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 4096, 521, 0, 0, 798, 30, 4080, 0, 4096, 0, 0, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 56, 0, 2, 94, 8, 0, 588, 44, 7, 6, 0, 0, 4078, 4096, 3564, 1610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 4063, 0, 3949, 0, 0, 23, 0, 0, 0, 3505, 0, 2, 0, 4032, 0, 2120, 1036, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 330, 0, 0, 0, 5, 3, 0, 0, 82, 4095, 0, 0, 971, 931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 37, 1889, 3977, 1, 0, 0, 0, 1447, 0, 88, 4015, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4094, 11, 0, 58, 5, 0, 0, 0, 2840, 2755, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 12, 0, 3602, 4054, 35, 0, 0, 0, 0, 2998, 0, 1961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18837, 17942, 14159, 19352, 10513, 19647, 13865, 16757] avg=16384.0000 max=19647 max_rank=5 min=10513 min_rank=4 imbalance_ratio=1.199158 max_load_reduction=111425 max_load_reduction_pct=0.850105 moved_to_echo=60782 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4050, 5, 173, 3117, 0, 0, 0, 22, 136, 3748, 220, 4044, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 3559, 510, 0, 21, 0, 0, 4002, 0, 3952, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3, 94, 72, 547, 117, 15, 3972, 0, 0, 95, 0, 0, 0, 0, 74, 3993, 3987, 3495, 1574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 3882, 0, 0, 0, 18, 115, 0, 0, 0, 44, 0, 3820, 0, 2127, 1012, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 4025, 0, 0, 0, 0, 0, 7, 0, 2191, 32, 0, 261, 4090, 0, 0, 969, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 2298, 0, 0, 4095, 3292, 0, 12, 1, 0, 1414, 0, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 1, 0, 0, 4096, 0, 4, 0, 4096, 2771, 10, 12, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 4096, 38, 0, 876, 2, 4045, 0, 0, 0, 325, 1, 3421, 45, 1967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19599, 16182, 18051, 15470, 12619, 15061, 19088, 15002] avg=16384.0000 max=19599 max_rank=0 min=12619 min_rank=4 imbalance_ratio=1.196228 max_load_reduction=111473 max_load_reduction_pct=0.850471 moved_to_echo=61770 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[1, 4096, 0, 1, 3125, 0, 0, 0, 0, 0, 1253, 93, 4079, 0, 4068, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3690, 0, 4033, 520, 0, 0, 24, 1, 4079, 14, 3900, 0, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 1523, 39, 0, 0, 0, 1, 0, 859, 2, 0, 4074, 3973, 3564, 1612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2928, 0, 4096, 0, 27, 0, 0, 0, 0, 0, 254, 0, 16, 4093, 0, 2125, 1036, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3790, 0, 0, 3681, 0, 0, 3646, 4068, 353, 0, 0, 7, 4095, 1, 0, 971, 937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 893, 0, 97, 3921, 3986, 4, 3, 0, 0, 1448, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 4093, 3, 0, 15, 529, 0, 0, 0, 4093, 2774, 0, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 51, 0, 0, 578, 4050, 0, 0, 0, 0, 0, 4094, 0, 1966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17091, 16708, 15656, 14960, 21553, 14448, 15822, 14834] avg=16384.0000 max=21553 max_rank=4 min=14448 min_rank=5 imbalance_ratio=1.315491 max_load_reduction=109519 max_load_reduction_pct=0.835564 moved_to_echo=66657 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 3125, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 4096, 522, 0, 0, 0, 0, 3809, 0, 4096, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3909, 5, 0, 0, 0, 0, 0, 0, 0, 63, 4096, 4096, 3574, 1607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 2128, 1034, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 4091, 0, 1919, 3753, 0, 0, 323, 0, 1, 2, 0, 0, 4096, 0, 0, 971, 937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 47, 0, 0, 4096, 4095, 0, 0, 0, 0, 1444, 0, 0, 4091, 3884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2452, 0, 0, 0, 4033, 2774, 0, 169, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 209, 4033, 4096, 0, 0, 0, 0, 0, 4079, 0, 1968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11327, 16604, 17350, 15836, 16161, 17693, 17620, 18481] avg=16384.0000 max=18481 max_rank=7 min=11327 min_rank=0 imbalance_ratio=1.127991 max_load_reduction=112591 max_load_reduction_pct=0.859001 moved_to_echo=69955 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4073, 1, 139, 3121, 0, 0, 0, 4, 0, 1, 4, 4095, 0, 1, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4015, 0, 4089, 521, 0, 0, 464, 16, 4066, 0, 2985, 1, 0, 46, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 0, 0, 21, 0, 0, 2, 0, 0, 0, 0, 71, 4095, 4061, 3570, 1605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 4086, 0, 0, 0, 0, 0, 0, 1129, 32, 0, 0, 4092, 0, 2124, 1032, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 267, 7, 0, 286, 0, 3215, 0, 0, 48, 4096, 0, 0, 970, 937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 15, 4095, 25, 0, 4096, 4096, 0, 11, 0, 0, 1443, 0, 10, 3633, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 4096, 0, 0, 343, 4030, 1, 0, 0, 4089, 2774, 17, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 1, 4096, 3981, 0, 0, 0, 0, 13, 3931, 0, 1964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11529, 16216, 13441, 16961, 13922, 21485, 19446, 18072] avg=16384.0000 max=21485 max_rank=5 min=11529 min_rank=0 imbalance_ratio=1.311340 max_load_reduction=109587 max_load_reduction_pct=0.836082 moved_to_echo=72925 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 1280, 128, 32640, 128, 0, 2432, 18304, 512, 128, 128, 21248, 22272, 128, 256, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[768, 768, 14976, 256, 256, 27520, 29312, 128, 256, 128, 0, 11392, 0, 128, 25984, 20224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 31232, 128, 31488, 640, 256, 128, 128, 896, 128, 5760, 4352, 128, 128, 28288, 0, 17024, 8192, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32000, 384, 384, 24960, 128, 0, 0, 1024, 384, 17152, 384, 32512, 128, 19968, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 256, 128, 640, 128, 10368, 12288, 128, 128, 768, 128, 128, 896, 128, 384, 32640, 32128, 28288, 12672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[896, 128, 32640, 128, 1024, 1280, 24320, 32000, 256, 128, 0, 384, 128, 22784, 128, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1664, 30080, 256, 3072, 13312, 128, 0, 9088, 14336, 10112, 128, 0, 512, 32000, 1792, 384, 7808, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3456, 26880, 128, 30592, 4224, 128, 128, 4096, 128, 31872, 896, 28288, 128, 128, 1024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4067, 0, 3991, 0, 29, 4073, 104, 0, 0, 2943, 3, 0, 0, 42, 127, 89, 0, 0, 0, 0, 0, 57, 179, 4015, 259, 0, 77, 0, 2808, 0, 0, 4044, 0, 3687, 4096, 406, 0, 4050, 0, 23, 0, 0, 0, 0, 80, 0, 4035, 33, 0, 1072, 0, 0, 5, 0, 4024, 0, 0, 2, 17, 4082, 0, 0, 0, 3915, 4007, 0, 0, 0, 32, 4064, 0, 10, 0, 0, 0, 0, 0, 4093, 0, 0, 3851, 4, 4030, 0, 0, 4095, 0, 5, 3973, 0, 4096, 4096, 1380, 0, 0, 4000, 0, 21, 4096, 4082, 0, 0, 3996, 4096, 0, 0, 71, 0, 10, 65, 0, 31, 7, 3811, 0, 1, 1, 4096, 56, 232, 0, 5, 25, 0, 36, 4095, 0, 4069] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[2752, 63, 4046, 0, 0, 720, 0, 40, 305, 3762, 2, 0, 0, 33, 0, 0, 343, 0, 0, 0, 0, 0, 12, 4096, 0, 28, 0, 0, 3854, 0, 0, 4096, 0, 4093, 3758, 0, 0, 3966, 0, 0, 0, 763, 0, 0, 0, 0, 107, 0, 0, 0, 0, 1494, 0, 0, 3976, 1, 0, 0, 0, 3700, 0, 184, 1, 2360, 3587, 292, 26, 77, 0, 3465, 448, 0, 3918, 0, 131, 33, 0, 3913, 64, 17, 3162, 0, 0, 1863, 0, 4096, 0, 4096, 4096, 0, 4069, 4003, 0, 60, 0, 3372, 19, 0, 4055, 4089, 0, 0, 4025, 3241, 0, 297, 0, 0, 0, 1194, 0, 101, 0, 4040, 791, 0, 645, 4073, 243, 126, 0, 0, 0, 1191, 0, 4096, 1606, 3897] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[3662, 0, 3948, 4, 24, 4072, 0, 0, 4, 0, 0, 0, 0, 0, 108, 145, 0, 0, 0, 0, 14, 0, 9, 3838, 48, 0, 0, 0, 4045, 0, 0, 4096, 0, 0, 4096, 0, 38, 4086, 16, 260, 1, 4038, 0, 0, 0, 0, 3837, 0, 4, 3944, 0, 0, 299, 21, 3985, 0, 86, 0, 0, 4096, 1, 0, 0, 1275, 4036, 230, 0, 27, 0, 4096, 200, 0, 0, 463, 0, 14, 150, 4058, 0, 0, 4096, 33, 0, 145, 0, 2621, 363, 85, 4091, 0, 4066, 4085, 1, 0, 234, 3839, 0, 25, 4096, 4096, 0, 0, 4078, 3759, 1, 0, 62, 0, 8, 13, 3837, 0, 0, 4025, 11, 24, 6, 4052, 2, 0, 14, 0, 0, 0, 65, 4058, 3781, 4026] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[2415, 17, 4047, 6, 0, 4080, 79, 0, 0, 40, 5, 0, 0, 1, 96, 0, 143, 13, 0, 662, 0, 0, 0, 3431, 44, 0, 20, 81, 4054, 102, 0, 4050, 0, 4095, 4042, 0, 0, 4035, 0, 0, 0, 0, 187, 30, 0, 34, 4081, 0, 0, 3926, 22, 0, 1909, 0, 4075, 33, 842, 0, 0, 4064, 0, 78, 0, 5, 4005, 0, 0, 41, 0, 4037, 0, 0, 13, 0, 57, 11, 0, 4023, 0, 0, 4090, 0, 3933, 385, 0, 4028, 9, 4, 3938, 21, 3718, 4092, 0, 0, 0, 568, 0, 0, 3971, 3997, 76, 0, 4079, 2490, 62, 0, 0, 43, 0, 0, 0, 0, 0, 4092, 554, 1, 4031, 4022, 0, 0, 0, 0, 0, 95, 0, 4082, 3608, 4052] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4086, 0, 4096, 92, 0, 4095, 0, 0, 0, 2839, 0, 0, 0, 0, 0, 0, 0, 0, 993, 25, 0, 0, 0, 4019, 0, 0, 0, 0, 3680, 0, 0, 4096, 0, 3785, 4096, 0, 0, 4096, 0, 0, 0, 2409, 0, 0, 0, 0, 629, 0, 0, 7, 0, 0, 0, 1, 4096, 0, 432, 0, 0, 4096, 0, 0, 0, 3188, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 5, 11, 0, 4, 3874, 0, 0, 4096, 0, 4082, 4096, 0, 0, 0, 2593, 0, 0, 4096, 4095, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 1348, 0, 0, 4095, 0, 0, 2882, 4096, 0, 0, 0, 0, 0, 3, 0, 4096, 4081, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4077, 0, 1496, 0, 12, 4096, 0, 0, 0, 3804, 38, 960, 512, 0, 3707, 0, 0, 0, 0, 0, 0, 0, 1965, 3633, 0, 0, 0, 0, 1771, 1, 0, 4096, 230, 332, 4094, 44, 0, 4096, 9, 1103, 34, 0, 0, 0, 115, 0, 0, 4, 0, 153, 0, 4049, 10, 20, 3598, 0, 3531, 0, 0, 4096, 630, 93, 0, 0, 3403, 0, 0, 0, 0, 4096, 0, 0, 0, 131, 0, 0, 4096, 596, 0, 0, 4096, 2224, 0, 0, 7, 0, 0, 466, 4010, 0, 0, 1810, 205, 0, 0, 0, 514, 3996, 3811, 4096, 0, 0, 4093, 4096, 0, 0, 0, 3601, 284, 0, 1, 57, 0, 2628, 3767, 0, 15, 3867, 115, 18, 0, 1913, 404, 0, 0, 4093, 4096, 4058] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[3914, 4, 4086, 0, 0, 3679, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 0, 0, 0, 149, 131, 0, 0, 0, 3901, 0, 0, 0, 0, 4081, 0, 5, 4096, 11, 3902, 4082, 4, 0, 4096, 165, 0, 0, 2489, 0, 0, 2, 0, 4021, 109, 1, 0, 0, 0, 0, 19, 4090, 0, 0, 1, 0, 4077, 0, 0, 0, 4063, 4086, 0, 0, 0, 0, 3950, 0, 0, 322, 0, 383, 0, 534, 3806, 1, 0, 4096, 0, 0, 15, 0, 3946, 0, 3686, 4043, 71, 4086, 4096, 58, 0, 0, 4, 22, 0, 4094, 3382, 0, 0, 4076, 3637, 24, 0, 0, 0, 0, 0, 1, 0, 0, 1010, 0, 3, 1435, 4096, 1, 1, 0, 530, 3, 0, 0, 4095, 4091, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4046, 0, 4082, 0, 630, 3430, 0, 0, 0, 4092, 13, 0, 0, 16, 36, 0, 4, 55, 2, 509, 0, 1, 4, 3573, 0, 0, 0, 22, 4096, 4, 0, 4094, 2, 3746, 4096, 0, 2, 4091, 0, 0, 0, 44, 14, 0, 110, 0, 244, 0, 0, 0, 0, 0, 0, 460, 4054, 48, 0, 0, 0, 4096, 0, 0, 0, 4093, 3871, 0, 0, 0, 0, 4051, 0, 0, 1, 2, 0, 0, 258, 2888, 0, 135, 4096, 0, 3413, 586, 0, 980, 0, 0, 4096, 0, 4058, 4072, 0, 0, 0, 3182, 0, 0, 4094, 4096, 5, 0, 4096, 4089, 1, 0, 0, 12, 2, 0, 3885, 0, 0, 4016, 0, 0, 1426, 4095, 0, 0, 0, 0, 0, 0, 0, 4003, 3672, 4082] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4067, 0, 3991, 0, 29, 4073, 104, 0, 0, 2943, 3, 0, 0, 42, 127, 89, 1121, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 179, 4015, 259, 0, 77, 0, 2808, 0, 0, 4044, 4095, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3687, 4096, 406, 0, 4050, 0, 23, 0, 0, 0, 0, 80, 0, 4035, 33, 888, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 0, 0, 5, 0, 4024, 0, 0, 2, 17, 4082, 0, 0, 0, 3915, 2645, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 0, 32, 4064, 0, 10, 0, 0, 0, 0, 0, 4093, 0, 0, 3506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 4, 4030, 0, 0, 4095, 0, 5, 0, 0, 4096, 4096, 1380, 0, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 4082, 0, 0, 2168, 4096, 0, 0, 71, 0, 10, 65, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3811, 0, 1, 1, 3208, 56, 232, 0, 5, 25, 0, 36, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17313, 15648, 18005, 16115, 15712, 22188, 14640, 11451] avg=16384.0000 max=22188 max_rank=5 min=11451 min_rank=7 imbalance_ratio=1.354248 max_load_reduction=108884 max_load_reduction_pct=0.830719 moved_to_echo=63991 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3662, 0, 3948, 4, 24, 4072, 0, 0, 4, 0, 0, 0, 0, 0, 108, 145, 1144, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 9, 3838, 48, 0, 0, 0, 4045, 0, 0, 4096, 4058, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 38, 4086, 16, 260, 1, 4038, 0, 0, 0, 0, 3837, 0, 879, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3944, 0, 0, 299, 21, 3985, 0, 86, 0, 0, 4096, 1, 0, 0, 1275, 2813, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 230, 0, 27, 0, 4096, 200, 0, 0, 463, 0, 14, 150, 4058, 0, 0, 3609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 33, 0, 145, 0, 2621, 363, 85, 0, 0, 4066, 4085, 1, 0, 234, 3839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 4096, 0, 0, 2212, 3759, 1, 0, 62, 0, 8, 13, 3837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 11, 24, 6, 3173, 2, 0, 14, 0, 0, 0, 65, 0, 3781, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13881, 16226, 17973, 16888, 16883, 15985, 18109, 15127] avg=16384.0000 max=18109 max_rank=6 min=13881 min_rank=0 imbalance_ratio=1.105286 max_load_reduction=112963 max_load_reduction_pct=0.861839 moved_to_echo=66104 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2752, 63, 4046, 0, 0, 720, 0, 40, 305, 3762, 2, 0, 0, 33, 0, 0, 1129, 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 0, 0, 0, 0, 0, 12, 4096, 0, 28, 0, 0, 3854, 0, 0, 4096, 4096, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3758, 0, 0, 3966, 0, 0, 0, 763, 0, 0, 0, 0, 107, 0, 883, 712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 0, 0, 3976, 1, 0, 0, 0, 3700, 0, 184, 1, 2360, 2172, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3587, 292, 26, 77, 0, 3465, 448, 0, 3918, 0, 131, 33, 0, 3913, 64, 17, 3615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 0, 1863, 0, 4096, 0, 4096, 0, 0, 4069, 4003, 0, 60, 0, 3372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4055, 4089, 0, 0, 2184, 3241, 0, 297, 0, 0, 0, 1194, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 791, 0, 645, 3190, 243, 126, 0, 0, 0, 1191, 0, 0, 1606, 3897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13446, 16642, 14282, 14252, 19586, 21955, 15180, 15729] avg=16384.0000 max=21955 max_rank=5 min=13446 min_rank=0 imbalance_ratio=1.340027 max_load_reduction=109117 max_load_reduction_pct=0.832497 moved_to_echo=72450 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4077, 0, 1496, 0, 12, 4096, 0, 0, 0, 3804, 38, 960, 512, 0, 3707, 0, 1154, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1965, 3633, 0, 0, 0, 0, 1771, 1, 0, 4096, 4093, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 332, 4094, 44, 0, 4096, 9, 1103, 34, 0, 0, 0, 115, 0, 0, 4, 845, 728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 4049, 10, 20, 3598, 0, 3531, 0, 0, 4096, 630, 93, 0, 0, 2817, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3403, 0, 0, 0, 0, 4096, 0, 0, 0, 131, 0, 0, 4096, 596, 0, 0, 3537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 2224, 0, 0, 7, 0, 0, 466, 0, 0, 0, 1810, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 3996, 3811, 4096, 0, 0, 2211, 4096, 0, 0, 0, 3601, 284, 0, 1, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2628, 3767, 0, 15, 3022, 115, 18, 0, 1913, 404, 0, 0, 0, 4096, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20630, 15676, 11634, 19353, 15859, 5217, 22667, 20036] avg=16384.0000 max=22667 max_rank=6 min=5217 min_rank=5 imbalance_ratio=1.383484 max_load_reduction=108405 max_load_reduction_pct=0.827065 moved_to_echo=63779 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2415, 17, 4047, 6, 0, 4080, 79, 0, 0, 40, 5, 0, 0, 1, 96, 0, 1145, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 13, 0, 662, 0, 0, 0, 3431, 44, 0, 20, 81, 4054, 102, 0, 4050, 4082, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4042, 0, 0, 4035, 0, 0, 0, 0, 187, 30, 0, 34, 4081, 0, 872, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3926, 22, 0, 1909, 0, 4075, 33, 842, 0, 0, 4064, 0, 78, 0, 5, 2809, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 41, 0, 4037, 0, 0, 13, 0, 57, 11, 0, 4023, 0, 0, 3475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 0, 3933, 385, 0, 4028, 9, 4, 0, 21, 3718, 4092, 0, 0, 0, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 3997, 76, 0, 2212, 2490, 62, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 554, 1, 4031, 3150, 0, 0, 0, 0, 0, 95, 0, 0, 3608, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12700, 16795, 18098, 18113, 15662, 17270, 12851, 19583] avg=16384.0000 max=19583 max_rank=7 min=12700 min_rank=0 imbalance_ratio=1.195251 max_load_reduction=111489 max_load_reduction_pct=0.850594 moved_to_echo=65366 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4086, 0, 4096, 92, 0, 4095, 0, 0, 0, 2839, 0, 0, 0, 0, 0, 0, 1149, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 993, 25, 0, 0, 0, 4019, 0, 0, 0, 0, 3680, 0, 0, 4096, 4096, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3785, 4096, 0, 0, 4096, 0, 0, 0, 2409, 0, 0, 0, 0, 629, 0, 888, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 4096, 0, 432, 0, 0, 4096, 0, 0, 0, 3188, 2813, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 5, 11, 0, 4, 3874, 0, 0, 0, 0, 4082, 4096, 0, 0, 0, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 2222, 4087, 0, 0, 0, 0, 0, 0, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2882, 3208, 0, 0, 0, 0, 0, 3, 0, 0, 4081, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17127, 17026, 16628, 14999, 15901, 15178, 15848, 18365] avg=16384.0000 max=18365 max_rank=7 min=14999 min_rank=3 imbalance_ratio=1.120911 max_load_reduction=112707 max_load_reduction_pct=0.859886 moved_to_echo=65292 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3914, 4, 4086, 0, 0, 3679, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 0, 1144, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 131, 0, 0, 0, 3901, 0, 0, 0, 0, 4081, 0, 5, 4096, 4095, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3902, 4082, 4, 0, 4096, 165, 0, 0, 2489, 0, 0, 2, 0, 4021, 109, 888, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 19, 4090, 0, 0, 1, 0, 4077, 0, 0, 0, 4063, 2813, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 3950, 0, 0, 322, 0, 383, 0, 534, 3806, 1, 0, 3567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 0, 0, 15, 0, 3946, 0, 3686, 0, 71, 4086, 4096, 58, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4094, 3382, 0, 0, 2211, 3637, 24, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 0, 3, 1435, 3208, 1, 1, 0, 530, 3, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17711, 16575, 20490, 15423, 16649, 16475, 13371, 14378] avg=16384.0000 max=20490 max_rank=2 min=13371 min_rank=6 imbalance_ratio=1.250610 max_load_reduction=110582 max_load_reduction_pct=0.843674 moved_to_echo=60873 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4046, 0, 4082, 0, 630, 3430, 0, 0, 0, 4092, 13, 0, 0, 16, 36, 0, 1149, 770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 55, 2, 509, 0, 1, 4, 3573, 0, 0, 0, 22, 4096, 4, 0, 4094, 4003, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3746, 4096, 0, 2, 4091, 0, 0, 0, 44, 14, 0, 110, 0, 244, 0, 888, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 4054, 48, 0, 0, 0, 4096, 0, 0, 0, 4093, 2813, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3871, 0, 0, 0, 0, 4051, 0, 0, 1, 2, 0, 0, 258, 2888, 0, 135, 3614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 0, 3413, 586, 0, 980, 0, 0, 0, 0, 4058, 4072, 0, 0, 0, 3182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 5, 0, 2222, 4089, 1, 0, 0, 12, 2, 0, 3885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 0, 1426, 3207, 0, 0, 0, 0, 0, 0, 0, 0, 3672, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18264, 16484, 13962, 15929, 14820, 16804, 18406, 16403] avg=16384.0000 max=18406 max_rank=6 min=13962 min_rank=2 imbalance_ratio=1.123413 max_load_reduction=112666 max_load_reduction_pct=0.859573 moved_to_echo=66433 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[3968, 2304, 11392, 3072, 128, 23680, 384, 8448, 0, 128, 28288, 30464, 1664, 128, 256, 17664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[31104, 640, 128, 256, 128, 31872, 768, 128, 4352, 640, 640, 128, 5120, 27520, 128, 256, 28544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 23680, 32384, 512, 128, 32640, 256, 1408, 128, 9856, 256, 128, 384, 128, 17024, 256, 7040, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[29056, 128, 29824, 128, 768, 28288, 256, 128, 384, 21632, 128, 1024, 640, 128, 4096, 256, 9216, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 27776, 5248, 128, 10496, 25472, 512, 384, 128, 2560, 512, 1408, 128, 0, 24960, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[512, 128, 1152, 1408, 128, 128, 2176, 30592, 384, 128, 128, 128, 28416, 128, 128, 32768, 32640, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 9216, 128, 5632, 2304, 640, 32000, 128, 4992, 128, 128, 32384, 640, 384, 128, 18944, 21760, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[640, 4096, 32384, 32000, 128, 0, 17664, 29568, 128, 384, 256, 3712, 384, 1280, 9088, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True + [2026-08-30 05:03:33] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 798.8 | throughput per GPU (TFLOP/s/GPU): 417.3 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.061058E+01 | loss scale: 1.0 | grad norm: 216.873 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[52, 1382, 2016, 18, 2186, 12, 815, 32, 595, 116, 2484, 256, 584, 637, 10, 348, 186, 1818, 1300, 1786, 2959, 523, 882, 228, 9, 2656, 167, 702, 961, 295, 822, 1921, 3488, 10, 172, 678, 662, 138, 930, 1228, 2292, 3162, 898, 672, 166, 1744, 1526, 1338, 381, 211, 1008, 397, 251, 180, 2824, 116, 32, 998, 2357, 2394, 114, 303, 533, 58, 907, 3109, 82, 1399, 335, 187, 1121, 1934, 374, 741, 513, 106, 773, 2295, 2673, 1579, 1429, 646, 991, 3485, 401, 987, 1372, 3326, 90, 285, 1168, 890, 1639, 561, 907, 69, 1547, 318, 2750, 442, 80, 878, 1480, 1574, 2967, 2091, 298, 1035, 272, 647, 529, 2099, 208, 1190, 903, 818, 50, 1813, 747, 1657, 941, 1503, 2200, 1417, 1511, 25, 660, 29] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[62, 1586, 1788, 11, 2596, 11, 1079, 49, 1066, 61, 2539, 589, 710, 481, 28, 314, 279, 1385, 883, 1441, 3108, 809, 663, 366, 49, 2509, 230, 377, 870, 123, 893, 2127, 3010, 42, 52, 1020, 964, 148, 417, 1182, 2520, 2444, 724, 291, 245, 1339, 1842, 455, 509, 199, 795, 488, 308, 428, 3206, 138, 30, 998, 3244, 2138, 364, 383, 255, 73, 666, 2960, 97, 1528, 497, 278, 1495, 2860, 433, 1301, 783, 37, 744, 1963, 1602, 1292, 1603, 694, 976, 2958, 530, 733, 1605, 3403, 536, 325, 703, 538, 1710, 440, 571, 174, 1340, 430, 2558, 492, 42, 672, 1962, 2385, 2771, 2650, 305, 819, 716, 992, 717, 1831, 217, 1340, 440, 688, 54, 1528, 1115, 1883, 652, 1133, 2034, 1459, 1552, 60, 816, 51] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[39, 1697, 1764, 11, 2319, 7, 696, 120, 968, 75, 2112, 364, 851, 651, 7, 419, 262, 1633, 1738, 1516, 2763, 374, 705, 292, 1, 2714, 222, 479, 982, 159, 994, 2072, 3436, 56, 59, 810, 1234, 40, 1071, 924, 2439, 3158, 662, 382, 201, 968, 1997, 868, 588, 329, 841, 226, 81, 247, 3065, 266, 26, 1076, 3324, 2460, 470, 261, 480, 33, 1060, 3259, 57, 1326, 459, 243, 1936, 2680, 278, 928, 388, 60, 862, 2378, 2108, 1517, 1358, 418, 1381, 3184, 568, 745, 1211, 3519, 124, 283, 701, 885, 1547, 352, 514, 52, 1328, 234, 3045, 941, 60, 1343, 1069, 1901, 2680, 2079, 323, 1461, 797, 653, 339, 1850, 182, 1350, 654, 734, 60, 1137, 705, 1637, 652, 877, 1990, 1040, 1454, 62, 908, 62] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[21, 1007, 2059, 50, 2421, 11, 678, 59, 801, 106, 2183, 410, 791, 403, 14, 330, 148, 1892, 1809, 1174, 2771, 532, 970, 303, 12, 2414, 176, 571, 1060, 95, 948, 1972, 3475, 26, 77, 1193, 898, 44, 782, 877, 2282, 3036, 531, 381, 182, 1494, 1120, 655, 552, 101, 1081, 400, 235, 443, 3645, 134, 8, 1272, 3082, 2065, 223, 231, 365, 42, 778, 3613, 85, 2310, 508, 225, 1752, 2685, 156, 1108, 442, 57, 998, 2468, 2868, 1279, 1445, 642, 1148, 2958, 537, 685, 1601, 3358, 64, 440, 672, 638, 1697, 277, 675, 47, 1125, 292, 2842, 602, 65, 1097, 1532, 2130, 2615, 2467, 531, 1255, 327, 629, 570, 1979, 193, 1677, 366, 661, 30, 891, 664, 1223, 871, 1224, 2307, 1082, 1578, 21, 747, 140] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[9, 1890, 2087, 33, 2681, 3, 732, 53, 723, 92, 2024, 283, 684, 416, 10, 265, 237, 1790, 1429, 1255, 3238, 801, 672, 213, 27, 2447, 208, 600, 1119, 338, 904, 2103, 3452, 20, 137, 1263, 357, 46, 1090, 1152, 2598, 2934, 1117, 434, 255, 1427, 1823, 1772, 304, 182, 1310, 174, 129, 261, 2882, 327, 12, 844, 2378, 3029, 179, 253, 442, 17, 887, 3484, 55, 1081, 458, 248, 1635, 2384, 159, 857, 328, 241, 907, 2750, 2117, 1391, 2204, 799, 1103, 3252, 580, 555, 1465, 3320, 57, 241, 1178, 831, 1476, 472, 553, 53, 1556, 322, 3443, 432, 55, 855, 1119, 1916, 2866, 1534, 373, 963, 266, 659, 453, 1724, 199, 1064, 415, 829, 68, 1563, 878, 1638, 784, 1127, 2045, 529, 1515, 15, 728, 52] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[20, 1484, 2421, 61, 1938, 7, 710, 50, 876, 144, 2340, 272, 896, 485, 27, 302, 210, 1743, 1387, 1266, 2920, 446, 651, 170, 4, 2241, 176, 645, 1286, 224, 1124, 2348, 3323, 35, 44, 1360, 843, 44, 869, 1570, 2630, 3221, 794, 460, 342, 1284, 1158, 894, 436, 175, 581, 152, 120, 266, 3130, 125, 32, 1096, 3165, 1795, 251, 182, 493, 16, 920, 3125, 80, 1742, 306, 248, 1530, 2386, 175, 839, 741, 125, 894, 2091, 2420, 790, 1864, 613, 890, 3381, 616, 1313, 1180, 3056, 81, 360, 575, 658, 1715, 375, 581, 64, 1092, 415, 2959, 605, 30, 1357, 1735, 2163, 2551, 2552, 444, 1499, 445, 563, 609, 2008, 203, 1511, 421, 864, 31, 1348, 396, 1466, 953, 1032, 2619, 1090, 1757, 24, 675, 161] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[22, 780, 1561, 9, 2326, 6, 763, 51, 872, 69, 2179, 413, 745, 546, 16, 338, 162, 1820, 1675, 1765, 3002, 466, 678, 136, 6, 2480, 179, 341, 901, 88, 997, 2229, 3548, 51, 74, 720, 528, 88, 830, 1261, 2505, 3193, 739, 238, 239, 1613, 1724, 973, 481, 229, 853, 376, 211, 270, 3374, 124, 10, 1132, 3107, 2039, 119, 433, 309, 78, 685, 3400, 67, 1265, 583, 243, 1993, 2826, 477, 1090, 508, 57, 755, 2157, 1926, 1512, 1782, 646, 1179, 3107, 252, 670, 1874, 3450, 111, 247, 714, 797, 1639, 228, 1125, 82, 943, 220, 3362, 528, 10, 1173, 1583, 1628, 3172, 2035, 326, 1326, 626, 666, 482, 2163, 161, 1195, 464, 667, 50, 1992, 579, 1983, 894, 1029, 2320, 1054, 1298, 35, 520, 51] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[15, 1034, 2456, 7, 2789, 5, 786, 21, 624, 108, 1557, 136, 1087, 288, 1, 434, 119, 2439, 1296, 1951, 2764, 272, 993, 162, 1, 2068, 160, 454, 1289, 46, 838, 2253, 3674, 188, 26, 950, 1119, 20, 1141, 1210, 2728, 2941, 645, 241, 149, 1268, 1602, 1546, 318, 130, 1179, 168, 135, 93, 3221, 195, 64, 1032, 2784, 2434, 130, 233, 270, 17, 1268, 3371, 33, 1851, 429, 290, 2012, 2645, 493, 1019, 531, 76, 1244, 2432, 1901, 875, 1633, 383, 875, 3362, 637, 513, 1284, 3746, 55, 294, 560, 631, 1708, 248, 551, 54, 847, 264, 3063, 584, 24, 1341, 1131, 2103, 2868, 1470, 368, 1554, 424, 687, 409, 2358, 192, 1485, 383, 976, 37, 1447, 482, 1697, 1028, 1134, 2192, 774, 1704, 12, 700, 26] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[39, 1697, 1764, 11, 2319, 7, 696, 120, 968, 75, 2112, 364, 851, 651, 7, 419, 3045, 1613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 1633, 1738, 1516, 2263, 374, 705, 292, 1, 2714, 222, 479, 982, 159, 994, 2072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1355, 56, 59, 810, 1234, 40, 1071, 924, 2439, 3158, 662, 382, 201, 968, 1997, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 329, 841, 226, 81, 247, 3065, 266, 26, 1076, 3324, 2460, 470, 261, 480, 33, 2078, 1342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 304, 57, 1326, 459, 243, 1936, 2680, 278, 928, 388, 60, 862, 2378, 2108, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1358, 418, 1381, 3184, 568, 745, 1211, 2445, 124, 283, 701, 885, 1547, 352, 514, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1328, 234, 0, 941, 60, 1343, 1069, 1901, 2197, 2079, 323, 1461, 797, 653, 339, 1850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 1350, 654, 734, 60, 1137, 705, 1637, 652, 877, 1990, 1040, 1454, 62, 908, 62, 1074, 500, 483, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16758, 16406, 16224, 17193, 16584, 15768, 16575, 15564] avg=16384.0000 max=17193 max_rank=3 min=15564 min_rank=7 imbalance_ratio=1.049377 max_load_reduction=113879 max_load_reduction_pct=0.868828 moved_to_echo=64491 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[62, 1586, 1788, 11, 2596, 11, 1079, 49, 1066, 61, 2539, 589, 710, 481, 28, 314, 2558, 1465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 1385, 883, 1441, 2546, 809, 663, 366, 49, 2509, 230, 377, 870, 123, 893, 2127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1187, 42, 52, 1020, 964, 148, 417, 1182, 2520, 2444, 724, 291, 245, 1339, 1842, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 199, 795, 488, 308, 428, 3206, 138, 30, 998, 3244, 2138, 364, 383, 255, 73, 1821, 1219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 666, 276, 97, 1528, 497, 278, 1495, 2860, 433, 1301, 783, 37, 744, 1963, 1602, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1603, 694, 976, 2958, 530, 733, 1605, 2365, 536, 325, 703, 538, 1710, 440, 571, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1340, 430, 0, 492, 42, 672, 1962, 2385, 2272, 2650, 305, 819, 716, 992, 717, 1831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, 1340, 440, 688, 54, 1528, 1115, 1883, 652, 1133, 2034, 1459, 1552, 60, 816, 51, 1038, 562, 499, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16993, 15550, 14872, 16596, 15852, 16461, 17625, 17123] avg=16384.0000 max=17625 max_rank=6 min=14872 min_rank=2 imbalance_ratio=1.075745 max_load_reduction=113447 max_load_reduction_pct=0.865532 moved_to_echo=67061 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[52, 1382, 2016, 18, 2186, 12, 815, 32, 595, 116, 2484, 256, 584, 637, 10, 348, 2750, 1539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 1818, 1300, 1786, 2424, 523, 882, 228, 9, 2656, 167, 702, 961, 295, 822, 1921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1375, 10, 172, 678, 662, 138, 930, 1228, 2292, 3162, 898, 672, 166, 1744, 1526, 1338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 381, 211, 1008, 397, 251, 180, 2824, 116, 32, 998, 2357, 2394, 114, 303, 533, 58, 2110, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 907, 290, 82, 1399, 335, 187, 1121, 1934, 374, 741, 513, 106, 773, 2295, 2673, 1579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1429, 646, 991, 3485, 401, 987, 1372, 2311, 90, 285, 1168, 890, 1639, 561, 907, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1547, 318, 0, 442, 80, 878, 1480, 1574, 2432, 2091, 298, 1035, 272, 647, 529, 2099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 1190, 903, 818, 50, 1813, 747, 1657, 941, 1503, 2200, 1417, 1511, 25, 660, 29, 1015, 535, 535, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15832, 16680, 16991, 15547, 15309, 17231, 15722, 17760] avg=16384.0000 max=17760 max_rank=7 min=15309 min_rank=4 imbalance_ratio=1.083984 max_load_reduction=113312 max_load_reduction_pct=0.864502 moved_to_echo=66022 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[21, 1007, 2059, 50, 2421, 11, 678, 59, 801, 106, 2183, 410, 791, 403, 14, 330, 2842, 1788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 1892, 1809, 1174, 2270, 532, 970, 303, 12, 2414, 176, 571, 1060, 95, 948, 1972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1370, 26, 77, 1193, 898, 44, 782, 877, 2282, 3036, 531, 381, 182, 1494, 1120, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 101, 1081, 400, 235, 443, 3645, 134, 8, 1272, 3082, 2065, 223, 231, 365, 42, 2102, 1487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 778, 338, 85, 2310, 508, 225, 1752, 2685, 156, 1108, 442, 57, 998, 2468, 2868, 1279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1445, 642, 1148, 2958, 537, 685, 1601, 2333, 64, 440, 672, 638, 1697, 277, 675, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1125, 292, 0, 602, 65, 1097, 1532, 2130, 2144, 2467, 531, 1255, 327, 629, 570, 1979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 1677, 366, 661, 30, 891, 664, 1223, 871, 1224, 2307, 1082, 1578, 21, 747, 140, 1025, 501, 471, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15974, 16346, 14948, 17468, 18057, 15859, 16745, 15675] avg=16384.0000 max=18057 max_rank=4 min=14948 min_rank=2 imbalance_ratio=1.102112 max_load_reduction=113015 max_load_reduction_pct=0.862236 moved_to_echo=66336 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[20, 1484, 2421, 61, 1938, 7, 710, 50, 876, 144, 2340, 272, 896, 485, 27, 302, 2959, 1551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 1743, 1387, 1266, 2387, 446, 651, 170, 4, 2241, 176, 645, 1286, 224, 1124, 2348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1305, 35, 44, 1360, 843, 44, 869, 1570, 2630, 3221, 794, 460, 342, 1284, 1158, 894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 436, 175, 581, 152, 120, 266, 3130, 125, 32, 1096, 3165, 1795, 251, 182, 493, 16, 2014, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 920, 284, 80, 1742, 306, 248, 1530, 2386, 175, 839, 741, 125, 894, 2091, 2420, 790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1864, 613, 890, 3381, 616, 1313, 1180, 2120, 81, 360, 575, 658, 1715, 375, 581, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 415, 0, 605, 30, 1357, 1735, 2163, 2088, 2552, 444, 1499, 445, 563, 609, 2008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 1511, 421, 864, 31, 1348, 396, 1466, 953, 1032, 2619, 1090, 1757, 24, 675, 161, 936, 533, 463, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16543, 16308, 16853, 15319, 15571, 16386, 17605, 16487] avg=16384.0000 max=17605 max_rank=6 min=15319 min_rank=3 imbalance_ratio=1.074524 max_load_reduction=113467 max_load_reduction_pct=0.865685 moved_to_echo=66049 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[9, 1890, 2087, 33, 2681, 3, 732, 53, 723, 92, 2024, 283, 684, 416, 10, 265, 3443, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 1790, 1429, 1255, 2652, 801, 672, 213, 27, 2447, 208, 600, 1119, 338, 904, 2103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1361, 20, 137, 1263, 357, 46, 1090, 1152, 2598, 2934, 1117, 434, 255, 1427, 1823, 1772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 182, 1310, 174, 129, 261, 2882, 327, 12, 844, 2378, 3029, 179, 253, 442, 17, 2088, 1434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 887, 326, 55, 1081, 458, 248, 1635, 2384, 159, 857, 328, 241, 907, 2750, 2117, 1391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2204, 799, 1103, 3252, 580, 555, 1465, 2307, 57, 241, 1178, 831, 1476, 472, 553, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1556, 322, 0, 432, 55, 855, 1119, 1916, 2349, 1534, 373, 963, 266, 659, 453, 1724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 1064, 415, 829, 68, 1563, 878, 1638, 784, 1127, 2045, 529, 1515, 15, 728, 52, 1013, 586, 517, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17152, 16795, 17786, 16245, 15824, 17126, 14576, 15568] avg=16384.0000 max=17786 max_rank=2 min=14576 min_rank=6 imbalance_ratio=1.085571 max_load_reduction=113286 max_load_reduction_pct=0.864304 moved_to_echo=63094 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[22, 780, 1561, 9, 2326, 6, 763, 51, 872, 69, 2179, 413, 745, 546, 16, 338, 3362, 1683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 1820, 1675, 1765, 2459, 466, 678, 136, 6, 2480, 179, 341, 901, 88, 997, 2229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1399, 51, 74, 720, 528, 88, 830, 1261, 2505, 3193, 739, 238, 239, 1613, 1724, 973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 229, 853, 376, 211, 270, 3374, 124, 10, 1132, 3107, 2039, 119, 433, 309, 78, 2146, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 685, 317, 67, 1265, 583, 243, 1993, 2826, 477, 1090, 508, 57, 755, 2157, 1926, 1512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1782, 646, 1179, 3107, 252, 670, 1874, 2397, 111, 247, 714, 797, 1639, 228, 1125, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 943, 220, 0, 528, 10, 1173, 1583, 1628, 2600, 2035, 326, 1326, 626, 666, 482, 2163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 1195, 464, 667, 50, 1992, 579, 1983, 894, 1029, 2320, 1054, 1298, 35, 520, 51, 1053, 543, 572, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15741, 16382, 16175, 16691, 16461, 16850, 16309, 16463] avg=16384.0000 max=16850 max_rank=5 min=15741 min_rank=0 imbalance_ratio=1.028442 max_load_reduction=114222 max_load_reduction_pct=0.871445 moved_to_echo=66083 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[15, 1034, 2456, 7, 2789, 5, 786, 21, 624, 108, 1557, 136, 1087, 288, 1, 434, 3063, 1668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 2439, 1296, 1951, 2264, 272, 993, 162, 1, 2068, 160, 454, 1289, 46, 838, 2253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1449, 188, 26, 950, 1119, 20, 1141, 1210, 2728, 2941, 645, 241, 149, 1268, 1602, 1546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 130, 1179, 168, 135, 93, 3221, 195, 64, 1032, 2784, 2434, 130, 233, 270, 17, 2222, 1388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1268, 315, 33, 1851, 429, 290, 2012, 2645, 493, 1019, 531, 76, 1244, 2432, 1901, 875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1633, 383, 875, 3362, 637, 513, 1284, 2603, 55, 294, 560, 631, 1708, 248, 551, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 847, 264, 0, 584, 24, 1341, 1131, 2103, 2351, 1470, 368, 1554, 424, 687, 409, 2358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 1485, 383, 976, 37, 1447, 482, 1697, 1028, 1134, 2192, 774, 1704, 12, 700, 26, 1143, 500, 517, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16079, 16605, 17223, 16013, 17414, 15391, 15915, 16432] avg=16384.0000 max=17414 max_rank=4 min=15391 min_rank=5 imbalance_ratio=1.062866 max_load_reduction=113658 max_load_reduction_pct=0.867142 moved_to_echo=65152 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[1664, 14592, 11520, 12160, 19328, 4224, 6272, 1920, 128, 19584, 1536, 4224, 8576, 1408, 7552, 17152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 10880, 16256, 256, 19328, 128, 6272, 512, 6528, 896, 17536, 2816, 6400, 3968, 128, 2816, 24064, 13056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[13440, 4864, 8576, 25728, 4224, 6272, 11648, 18944, 1152, 2560, 6272, 5888, 13184, 3072, 5504, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[7296, 2560, 640, 12544, 3584, 2048, 13568, 20480, 2560, 7936, 4352, 768, 7296, 18560, 17664, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9856, 2560, 0, 4736, 384, 8832, 11648, 15872, 18560, 16896, 3072, 9984, 3968, 5504, 4224, 16128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[10880, 512, 768, 8064, 6656, 640, 7168, 9472, 20096, 24192, 6144, 3200, 1792, 11264, 12800, 8576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3584, 1664, 7680, 2432, 1536, 2304, 25472, 1536, 256, 8448, 23552, 18432, 1920, 2304, 3200, 384, 16640, 10880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1664, 10880, 4096, 6272, 384, 11776, 5632, 13184, 6784, 9088, 17792, 8448, 12416, 256, 5760, 640, 8320, 4352, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1609, 6, 3, 0, 2356, 0, 0, 8, 0, 20, 0, 4096, 3548, 0, 1666, 9, 1, 2729, 3546, 3912, 2, 3, 8, 4, 4073, 1, 0, 3997, 2, 1, 1, 0, 4, 0, 2892, 0, 12, 0, 0, 3645, 3969, 0, 0, 0, 0, 0, 41, 3448, 4082, 0, 1, 1, 0, 0, 4096, 0, 18, 0, 2730, 0, 0, 0, 3894, 3391, 0, 4079, 0, 3979, 7, 3431, 62, 4087, 0, 0, 1, 3979, 4096, 0, 0, 0, 10, 9, 4091, 4093, 2557, 11, 4089, 0, 1, 1, 8, 49, 4088, 25, 2, 4096, 4092, 2, 13, 9, 0, 0, 0, 2, 1, 13, 0, 14, 0, 0, 3558, 0, 0, 0, 10, 27, 320, 1, 0, 4092, 66, 4085, 1, 4088, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4001, 64, 35, 0, 2348, 416, 0, 192, 0, 1, 18, 4092, 201, 0, 3725, 0, 0, 1150, 3232, 3216, 4, 0, 8, 3, 4062, 0, 1, 3871, 8, 1, 6, 0, 70, 0, 3664, 0, 22, 8, 2, 1244, 3741, 10, 12, 0, 2, 2, 0, 4069, 4096, 0, 0, 0, 0, 119, 4090, 0, 20, 2, 4064, 0, 0, 71, 3619, 2683, 0, 4090, 0, 3958, 3, 458, 8, 4096, 4, 153, 0, 3893, 4046, 1, 0, 13, 1, 45, 4089, 4093, 4012, 1, 4008, 1, 0, 5, 0, 35, 4043, 0, 0, 4084, 4092, 13, 2703, 69, 0, 0, 4, 34, 4, 0, 0, 67, 0, 0, 3122, 7, 0, 0, 3, 0, 475, 0, 11, 4095, 677, 4094, 9, 4089, 0, 99] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[4, 0, 1183, 9, 980, 2, 48, 53, 29, 1795, 12, 111, 0, 4071, 2903, 0, 673, 1, 18, 3654, 3603, 3427, 0, 0, 0, 0, 4083, 0, 1, 4082, 33, 80, 2, 2, 0, 0, 3975, 24, 0, 2, 77, 3147, 3996, 21, 0, 36, 10, 144, 0, 3903, 3594, 0, 0, 12, 15, 35, 4089, 8, 0, 0, 3532, 0, 0, 281, 1804, 3836, 0, 4045, 2, 4083, 0, 227, 23, 3762, 1, 2009, 13, 4067, 4013, 10, 18, 963, 36, 27, 4082, 4086, 634, 0, 3244, 6, 6, 41, 0, 23, 4086, 119, 0, 4088, 4086, 0, 1226, 0, 0, 0, 0, 3, 12, 7, 0, 2, 0, 0, 3611, 30, 0, 0, 0, 161, 2546, 3, 0, 4058, 42, 4059, 7, 4073, 0, 32] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 528, 7, 3, 0, 2173, 0, 0, 163, 0, 144, 0, 4092, 3862, 2, 3990, 0, 14, 1071, 3381, 1504, 1, 2, 70, 0, 4077, 1, 0, 4093, 1, 0, 3, 0, 0, 0, 4066, 0, 11, 6, 0, 1505, 3468, 3, 1, 1, 0, 0, 0, 3690, 4028, 0, 0, 2, 0, 0, 4095, 0, 1, 0, 3886, 3, 1, 209, 2403, 1750, 2, 4092, 0, 3470, 1, 3492, 3, 3996, 53, 64, 0, 3911, 4058, 1, 0, 362, 0, 220, 4093, 4096, 3582, 3, 4094, 0, 2, 0, 56, 0, 4094, 0, 76, 4095, 4092, 0, 1241, 66, 2, 0, 0, 2, 0, 0, 4, 1, 0, 0, 4069, 1, 2, 1, 5, 0, 3186, 0, 2, 4091, 4, 3979, 1, 4094, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 616, 53, 162, 11, 941, 4, 119, 422, 0, 9, 0, 4052, 3311, 2, 1984, 1, 3, 1087, 3994, 3928, 0, 1, 385, 0, 4013, 0, 0, 4093, 36, 0, 691, 0, 0, 0, 3513, 167, 99, 0, 0, 2576, 3983, 27, 0, 3, 101, 0, 0, 4091, 4096, 0, 33, 8, 0, 25, 4047, 0, 73, 0, 3206, 43, 0, 44, 3082, 3056, 0, 4087, 0, 4091, 0, 1998, 19, 4080, 5, 435, 1, 2905, 4039, 12, 0, 0, 0, 54, 4035, 4087, 3050, 6, 4090, 2, 18, 15, 55, 2, 3820, 0, 0, 3971, 4083, 3, 426, 51, 0, 0, 1, 2, 0, 0, 4, 7, 3, 0, 3983, 12, 5, 0, 0, 17, 3182, 2, 162, 3962, 0, 4003, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 6, 762, 1, 32, 0, 974, 1, 1, 24, 0, 11, 131, 3981, 2325, 0, 3259, 4, 19, 2190, 2849, 2973, 0, 0, 114, 0, 4094, 1, 0, 4096, 18, 198, 5, 13, 0, 0, 3421, 589, 89, 0, 2, 3275, 4033, 0, 11, 0, 0, 80, 0, 4040, 3896, 0, 0, 0, 0, 4, 4083, 0, 8, 0, 3488, 104, 57, 7, 3174, 3222, 0, 4029, 1, 4050, 0, 3211, 1, 4054, 3, 53, 0, 3998, 4069, 0, 0, 18, 1, 0, 4096, 4076, 379, 84, 3994, 0, 0, 0, 46, 13, 4091, 8, 644, 4094, 4096, 0, 1033, 3, 1, 0, 0, 132, 0, 1, 181, 21, 0, 0, 4044, 0, 19, 0, 50, 6, 1548, 0, 278, 3968, 2, 4088, 0, 4075, 0, 848] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2, 9, 2456, 46, 43, 0, 282, 3, 0, 438, 0, 6, 46, 4096, 3184, 0, 3639, 2, 1, 2578, 4089, 3918, 0, 0, 0, 9, 4002, 0, 1, 4085, 0, 15, 180, 0, 10, 0, 3271, 0, 0, 0, 8, 1993, 3861, 12, 0, 0, 0, 13, 0, 3980, 4078, 0, 0, 0, 2, 39, 4088, 12, 0, 0, 3881, 0, 37, 0, 4064, 1785, 0, 4069, 0, 4052, 8, 761, 13, 4096, 10, 8, 1, 3902, 3983, 0, 6, 169, 0, 18, 4096, 4084, 1375, 64, 3898, 3, 0, 0, 12, 3, 4096, 7, 39, 4088, 4084, 0, 1294, 34, 0, 0, 1, 22, 0, 0, 10, 0, 0, 18, 3955, 0, 0, 0, 0, 6, 2151, 1, 79, 4028, 0, 4088, 1, 4094, 0, 81] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1117, 12, 433, 1, 2115, 0, 2, 20, 5, 0, 0, 4082, 2334, 0, 1785, 0, 0, 2428, 4065, 4004, 37, 0, 0, 0, 4089, 0, 0, 4085, 24, 4, 774, 20, 1, 0, 4095, 52, 1, 8, 5, 1961, 4090, 2, 26, 0, 3, 0, 0, 4075, 4092, 5, 0, 0, 2, 0, 4092, 0, 2, 0, 3619, 32, 8, 0, 2613, 2770, 0, 4089, 7, 4093, 0, 4037, 1, 3908, 8, 3, 0, 2286, 4008, 0, 0, 6, 5, 14, 4096, 4082, 4076, 15, 4096, 0, 0, 3, 0, 0, 4068, 3, 3, 4091, 4096, 3, 330, 4, 0, 0, 3, 17, 2, 0, 5, 0, 0, 2, 3253, 2, 1, 0, 0, 0, 999, 1, 11, 4093, 0, 4083, 7, 4094, 2, 76] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1609, 6, 3, 0, 2356, 0, 0, 8, 0, 20, 0, 4096, 3548, 0, 2580, 2325, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1666, 9, 1, 2729, 3546, 3912, 2, 3, 8, 4, 537, 1, 0, 3997, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 2892, 0, 12, 0, 0, 3645, 3969, 0, 0, 0, 0, 0, 3757, 1211, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 3448, 4082, 0, 1, 1, 0, 0, 4096, 0, 18, 0, 2730, 0, 0, 0, 667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3894, 3391, 0, 0, 0, 3979, 7, 3431, 62, 330, 0, 0, 1, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 9, 4091, 1513, 2557, 11, 4089, 0, 1, 1, 8, 49, 4088, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4092, 2, 13, 9, 0, 0, 0, 2, 1, 13, 0, 14, 0, 0, 4079, 2951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3558, 0, 0, 0, 10, 27, 320, 1, 0, 4092, 66, 4085, 1, 2418, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17554, 16418, 16636, 15084, 19074, 16452, 15274, 14580] avg=16384.0000 max=19074 max_rank=4 min=14580 min_rank=7 imbalance_ratio=1.164185 max_load_reduction=111998 max_load_reduction_pct=0.854477 moved_to_echo=65380 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4001, 64, 35, 0, 2348, 416, 0, 192, 0, 1, 18, 4092, 201, 0, 2580, 2319, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3725, 0, 0, 1150, 3232, 3216, 4, 0, 8, 3, 535, 0, 1, 3871, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 70, 0, 3664, 0, 22, 8, 2, 1244, 3741, 10, 12, 0, 2, 2, 3766, 1208, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4096, 0, 0, 0, 0, 119, 4090, 0, 20, 2, 4064, 0, 0, 71, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3619, 2683, 0, 0, 0, 3958, 3, 458, 8, 330, 4, 153, 0, 3893, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 45, 4089, 1513, 4012, 1, 4008, 1, 0, 5, 0, 35, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4092, 13, 2703, 69, 0, 0, 4, 34, 4, 0, 0, 67, 0, 0, 4090, 2915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3122, 7, 0, 0, 3, 0, 475, 0, 11, 4095, 677, 4094, 9, 2417, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17271, 15754, 14888, 17199, 15110, 17766, 18075, 15009] avg=16384.0000 max=18075 max_rank=6 min=14888 min_rank=2 imbalance_ratio=1.103210 max_load_reduction=112997 max_load_reduction_pct=0.862099 moved_to_echo=65960 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 528, 7, 3, 0, 2173, 0, 0, 163, 0, 144, 0, 4092, 3862, 2, 2582, 2328, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 0, 14, 1071, 3381, 1504, 1, 2, 70, 0, 537, 1, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4066, 0, 11, 6, 0, 1505, 3468, 3, 1, 1, 0, 0, 3674, 1212, 1136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3690, 4028, 0, 0, 2, 0, 0, 4095, 0, 1, 0, 3886, 3, 1, 209, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2403, 1750, 2, 0, 0, 3470, 1, 3492, 3, 322, 53, 64, 0, 3911, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 220, 4093, 1514, 3582, 3, 4094, 0, 2, 0, 56, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 4095, 4092, 0, 1241, 66, 2, 0, 0, 2, 0, 0, 4, 1, 0, 0, 4092, 2922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 1, 2, 1, 5, 0, 3186, 0, 2, 4091, 4, 3979, 1, 2421, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16890, 14665, 15086, 16583, 15472, 18020, 16593, 17763] avg=16384.0000 max=18020 max_rank=5 min=14665 min_rank=1 imbalance_ratio=1.099854 max_load_reduction=113052 max_load_reduction_pct=0.862518 moved_to_echo=67848 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 9, 2456, 46, 43, 0, 282, 3, 0, 438, 0, 6, 46, 4096, 3184, 0, 2578, 2290, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3639, 2, 1, 2578, 4089, 3918, 0, 0, 0, 9, 517, 0, 1, 4085, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 10, 0, 3271, 0, 0, 0, 8, 1993, 3861, 12, 0, 0, 0, 13, 3769, 1195, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 4078, 0, 0, 0, 2, 39, 4088, 12, 0, 0, 3881, 0, 37, 0, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 1785, 0, 0, 0, 4052, 8, 761, 13, 327, 10, 8, 1, 3902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 169, 0, 18, 4096, 1506, 1375, 64, 3898, 3, 0, 0, 12, 3, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4088, 4084, 0, 1294, 34, 0, 0, 1, 22, 0, 0, 10, 0, 0, 18, 4069, 2868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 0, 0, 0, 0, 6, 2151, 1, 79, 4028, 0, 4088, 1, 2413, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16487, 18854, 15427, 16790, 14931, 15253, 16527, 16803] avg=16384.0000 max=18854 max_rank=1 min=14931 min_rank=4 imbalance_ratio=1.150757 max_load_reduction=112218 max_load_reduction_pct=0.856155 moved_to_echo=63514 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 762, 1, 32, 0, 974, 1, 1, 24, 0, 11, 131, 3981, 2325, 0, 2569, 2337, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3259, 4, 19, 2190, 2849, 2973, 0, 0, 114, 0, 539, 1, 0, 4096, 18, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 13, 0, 0, 3421, 589, 89, 0, 2, 3275, 4033, 0, 11, 0, 0, 80, 3727, 1218, 1139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 3896, 0, 0, 0, 0, 4, 4083, 0, 8, 0, 3488, 104, 57, 7, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3174, 3222, 0, 0, 1, 4050, 0, 3211, 1, 327, 3, 53, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1, 0, 4096, 1507, 379, 84, 3994, 0, 0, 0, 46, 13, 4091, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 644, 4094, 4096, 0, 1033, 3, 1, 0, 0, 132, 0, 1, 181, 21, 0, 0, 4029, 2930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 19, 0, 50, 6, 1548, 0, 278, 3968, 2, 4088, 0, 2410, 0, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14155, 16260, 17602, 16352, 18040, 14237, 17165, 17261] avg=16384.0000 max=18040 max_rank=4 min=14155 min_rank=0 imbalance_ratio=1.101074 max_load_reduction=113032 max_load_reduction_pct=0.862366 moved_to_echo=66703 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 616, 53, 162, 11, 941, 4, 119, 422, 0, 9, 0, 4052, 3311, 2, 2576, 2291, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1984, 1, 3, 1087, 3994, 3928, 0, 1, 385, 0, 529, 0, 0, 4093, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 691, 0, 0, 0, 3513, 167, 99, 0, 0, 2576, 3983, 27, 0, 3, 101, 0, 3751, 1193, 1130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 33, 8, 0, 25, 4047, 0, 73, 0, 3206, 43, 0, 44, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3082, 3056, 0, 0, 0, 4091, 0, 1998, 19, 329, 5, 435, 1, 2905, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 4035, 1511, 3050, 6, 4090, 2, 18, 15, 55, 2, 3820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 4083, 3, 426, 51, 0, 0, 1, 2, 0, 0, 4, 7, 3, 0, 4087, 2909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 12, 5, 0, 0, 17, 3182, 2, 162, 3962, 0, 4003, 0, 2422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15574, 16041, 17234, 16335, 15933, 16658, 15547, 17750] avg=16384.0000 max=17750 max_rank=7 min=15547 min_rank=6 imbalance_ratio=1.083374 max_load_reduction=113322 max_load_reduction_pct=0.864578 moved_to_echo=65888 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 1183, 9, 980, 2, 48, 53, 29, 1795, 12, 111, 0, 4071, 2903, 0, 2576, 2331, 1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 673, 1, 18, 3654, 3603, 3427, 0, 0, 0, 0, 538, 0, 1, 4082, 33, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 3975, 24, 0, 2, 77, 3147, 3996, 21, 0, 36, 10, 144, 3458, 1214, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3903, 3594, 0, 0, 12, 15, 35, 4089, 8, 0, 0, 3532, 0, 0, 281, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1804, 3836, 0, 0, 2, 4083, 0, 227, 23, 304, 1, 2009, 13, 4067, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 963, 36, 27, 4082, 1510, 634, 0, 3244, 6, 6, 41, 0, 23, 4086, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4086, 0, 1226, 0, 0, 0, 0, 3, 12, 7, 0, 2, 0, 0, 4045, 2891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3611, 30, 0, 0, 0, 161, 2546, 3, 0, 4058, 42, 4059, 7, 2408, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17107, 16110, 17230, 16134, 16379, 14795, 16360, 16957] avg=16384.0000 max=17230 max_rank=2 min=14795 min_rank=5 imbalance_ratio=1.051636 max_load_reduction=113842 max_load_reduction_pct=0.868546 moved_to_echo=64491 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1117, 12, 433, 1, 2115, 0, 2, 20, 5, 0, 0, 4082, 2334, 0, 2573, 2335, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1785, 0, 0, 2428, 4065, 4004, 37, 0, 0, 0, 538, 0, 0, 4085, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 774, 20, 1, 0, 4095, 52, 1, 8, 5, 1961, 4090, 2, 26, 0, 3, 0, 3593, 1216, 1122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4092, 5, 0, 0, 2, 0, 4092, 0, 2, 0, 3619, 32, 8, 0, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2613, 2770, 0, 0, 7, 4093, 0, 4037, 1, 315, 8, 3, 0, 2286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 14, 4096, 1509, 4076, 15, 4096, 0, 0, 3, 0, 0, 4068, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4091, 4096, 3, 330, 4, 0, 0, 3, 17, 2, 0, 5, 0, 0, 2, 4089, 2886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3253, 2, 1, 0, 0, 0, 999, 1, 11, 4093, 0, 4083, 7, 2421, 2, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16034, 16970, 16969, 16595, 16133, 17891, 15531, 14949] avg=16384.0000 max=17891 max_rank=5 min=14949 min_rank=7 imbalance_ratio=1.091980 max_load_reduction=113181 max_load_reduction_pct=0.863503 moved_to_echo=64504 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[768, 32640, 32768, 128, 8320, 256, 128, 0, 128, 256, 128, 128, 256, 128, 128, 128, 32640, 23296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 31360, 32000, 128, 128, 128, 128, 256, 32768, 128, 128, 128, 28416, 256, 128, 640, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[29696, 128, 128, 128, 128, 256, 14464, 128, 640, 32512, 896, 32512, 128, 19456, 128, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1664, 128, 128, 0, 28928, 896, 256, 128, 128, 19456, 31232, 128, 128, 128, 128, 256, 29568, 9728, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 12288, 256, 1792, 128, 11264, 512, 256, 3072, 128, 384, 256, 32640, 21760, 128, 20736, 18560, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[24704, 22528, 128, 0, 128, 31872, 128, 17664, 256, 2688, 128, 2816, 128, 29056, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[20736, 128, 128, 16896, 28800, 27008, 128, 128, 640, 128, 4352, 128, 128, 32512, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 1536, 128, 512, 32768, 12160, 19712, 256, 31616, 128, 128, 128, 256, 128, 32512, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[374, 1357, 1, 3242, 0, 0, 778, 0, 36, 17, 2, 2861, 4039, 0, 0, 4073, 4027, 4082, 0, 195, 3939, 2210, 1, 0, 0, 4089, 0, 0, 4, 2617, 4088, 0, 0, 4083, 2402, 4095, 0, 0, 2949, 4070, 1, 51, 1197, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 17, 9, 14, 4089, 0, 625, 3267, 0, 4092, 14, 9, 4058, 0, 0, 10, 0, 0, 21, 4079, 1, 0, 4060, 417, 4078, 1, 0, 7, 9, 9, 567, 4096, 0, 4, 0, 64, 0, 15, 18, 4096, 0, 4093, 981, 0, 2595, 238, 10, 0, 0, 2972, 3774, 42, 60, 4093, 0, 0, 4083, 3, 0, 0, 6, 1294, 4095, 0, 15, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[115, 1003, 0, 2597, 0, 0, 1420, 0, 0, 0, 9, 2716, 4095, 0, 0, 4080, 4096, 3940, 0, 1894, 4052, 3966, 0, 0, 0, 4080, 0, 0, 1, 469, 4084, 0, 0, 4095, 1795, 4092, 0, 12, 1203, 4096, 0, 0, 1170, 0, 16, 0, 0, 0, 0, 54, 0, 342, 2254, 0, 0, 10, 0, 4076, 0, 444, 3047, 3, 4096, 0, 2, 3513, 0, 0, 0, 0, 0, 834, 3998, 170, 0, 4096, 1189, 4096, 0, 2, 202, 11, 0, 2454, 4095, 52, 0, 0, 222, 0, 0, 0, 4096, 0, 4089, 4, 0, 3988, 19, 0, 0, 9, 3984, 4096, 0, 29, 4089, 0, 0, 4025, 6, 0, 0, 0, 173, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 3, 1034, 14, 0, 2498, 3, 0, 0, 46, 4094, 4096, 8, 0, 4079, 4089, 4096, 5, 404, 4096, 4040, 0, 4, 0, 4005, 10, 0, 23, 1211, 4083, 0, 0, 4096, 11, 4093, 0, 0, 4084, 4064, 0, 196, 15, 0, 0, 0, 0, 4, 0, 1, 1, 5, 2023, 0, 0, 0, 0, 4095, 0, 16, 3964, 0, 4087, 0, 2, 4041, 0, 0, 12, 0, 1, 97, 4084, 1146, 0, 4096, 1442, 4089, 9, 3, 145, 0, 0, 267, 4077, 0, 0, 2, 10, 3, 9, 0, 4083, 1, 4088, 0, 0, 4095, 1151, 0, 19, 0, 4038, 4082, 0, 0, 4093, 0, 0, 4096, 0, 0, 16, 0, 992, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[96, 1, 4, 608, 2, 0, 30, 1, 15, 2, 0, 4094, 3993, 0, 0, 4027, 4016, 4095, 1, 992, 4079, 3383, 8, 0, 0, 2496, 0, 0, 0, 2275, 4095, 0, 0, 4090, 32, 4081, 1, 0, 2522, 4017, 4, 979, 3499, 0, 1680, 0, 12, 7, 0, 74, 0, 8, 3879, 0, 0, 0, 6, 4092, 1, 539, 3606, 0, 4087, 0, 4, 4088, 9, 0, 0, 2, 2, 207, 4088, 17, 0, 4089, 4093, 4095, 0, 0, 2, 0, 3, 31, 4096, 18, 4, 0, 0, 2, 0, 0, 4096, 0, 4091, 960, 1, 3870, 0, 0, 7, 2, 254, 3757, 0, 0, 4093, 0, 0, 4096, 0, 6, 0, 0, 1353, 4096, 0, 0, 1, 0, 0, 0, 3, 0, 3, 4, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[47, 61, 0, 2921, 18, 0, 3857, 3, 5, 0, 17, 1901, 4059, 0, 0, 4095, 4064, 4096, 0, 134, 4066, 3566, 9, 4, 0, 4056, 0, 0, 0, 2836, 4042, 0, 0, 4087, 4, 4096, 0, 0, 4070, 4068, 0, 1, 2004, 0, 0, 0, 0, 3, 10, 51, 0, 66, 3705, 1, 0, 28, 30, 4096, 11, 545, 291, 0, 4087, 23, 31, 4076, 0, 4, 0, 10, 0, 0, 4083, 105, 42, 4066, 572, 4074, 0, 0, 386, 1, 1, 30, 4079, 16, 0, 0, 6, 3, 1091, 11, 4076, 0, 4086, 310, 0, 3891, 12, 0, 0, 0, 3769, 4050, 1, 18, 4066, 3, 3, 3931, 0, 22, 3, 22, 838, 4067, 0, 9, 0, 10, 0, 0, 0, 0, 2, 3, 48, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[55, 274, 0, 3543, 0, 0, 102, 6, 1, 0, 0, 1862, 4036, 0, 0, 4094, 4096, 4092, 0, 1457, 4066, 2200, 0, 0, 2, 4096, 0, 0, 0, 1909, 3972, 0, 0, 4077, 3321, 4013, 4, 0, 4078, 4095, 0, 1, 2279, 1, 103, 0, 6, 0, 0, 106, 0, 27, 4096, 0, 0, 3, 0, 4084, 13, 210, 4065, 0, 4073, 0, 0, 4096, 1, 0, 0, 12, 0, 1927, 4095, 2, 0, 4094, 1248, 4031, 0, 0, 63, 1, 0, 886, 4095, 0, 1, 0, 638, 0, 0, 2, 4069, 0, 4005, 0, 17, 298, 378, 0, 0, 2, 1654, 3231, 0, 0, 4094, 0, 0, 4087, 2, 30, 331, 0, 1153, 3985, 0, 1, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[51, 0, 0, 2189, 0, 0, 9, 1, 7, 8, 215, 3131, 4014, 0, 0, 4094, 4095, 4095, 57, 1414, 4079, 4090, 195, 0, 2, 3697, 8, 1, 0, 3154, 4095, 0, 0, 4096, 1224, 4072, 14, 0, 1356, 4013, 0, 387, 3628, 3, 0, 1, 42, 0, 4, 0, 3, 6, 4056, 0, 2, 0, 0, 4094, 2, 300, 4002, 0, 4087, 3, 104, 3945, 0, 0, 5, 0, 57, 1294, 4077, 0, 10, 4093, 2458, 4096, 11, 22, 3, 4, 0, 50, 3694, 0, 2, 3, 264, 0, 150, 0, 4090, 0, 4068, 1482, 0, 314, 6, 0, 10, 0, 83, 4079, 0, 0, 4092, 0, 0, 4085, 2, 1, 0, 0, 2075, 4094, 0, 27, 0, 1, 24, 0, 0, 101, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[6, 379, 0, 2343, 0, 13, 317, 6, 0, 0, 9, 2611, 4082, 0, 0, 4013, 4095, 4040, 10, 3127, 3932, 3366, 0, 0, 0, 4083, 0, 3, 1, 4080, 4091, 0, 0, 4087, 1628, 4080, 0, 0, 1784, 4092, 2, 87, 210, 0, 1205, 1, 0, 0, 0, 11, 0, 0, 3362, 0, 8, 0, 0, 4088, 0, 12, 4090, 0, 4096, 1, 326, 4070, 2, 0, 0, 0, 12, 470, 4091, 29, 0, 4096, 2829, 4093, 1, 0, 251, 0, 0, 9, 3947, 0, 0, 0, 11, 0, 72, 1, 3798, 0, 4087, 209, 0, 2757, 1, 5, 0, 12, 1857, 4075, 0, 5, 4096, 0, 0, 4081, 0, 0, 0, 6, 216, 4094, 0, 0, 0, 0, 0, 7, 2, 3, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3, 1034, 14, 0, 2498, 3, 0, 0, 46, 4094, 4096, 8, 0, 4079, 1384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1616, 5, 404, 4096, 4040, 0, 4, 0, 4005, 10, 0, 23, 1211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1834, 11, 4093, 0, 0, 4084, 4064, 0, 196, 15, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5, 2023, 0, 0, 0, 0, 4095, 0, 16, 3964, 0, 4087, 0, 978, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4041, 0, 0, 12, 0, 1, 97, 4084, 1146, 0, 1586, 1442, 4089, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 267, 4077, 0, 0, 2, 10, 3, 9, 0, 4083, 1, 4088, 0, 1914, 831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1151, 0, 19, 0, 4038, 4082, 0, 0, 3115, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 992, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4083, 2510, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17260, 15414, 14301, 15217, 16512, 15430, 20596, 16342] avg=16384.0000 max=20596 max_rank=6 min=14301 min_rank=2 imbalance_ratio=1.257080 max_load_reduction=110476 max_load_reduction_pct=0.842865 moved_to_echo=68880 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[374, 1357, 1, 3242, 0, 0, 778, 0, 36, 17, 2, 2861, 4039, 0, 0, 4073, 1383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1602, 0, 195, 3939, 2210, 1, 0, 0, 4089, 0, 0, 4, 2617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1819, 2402, 4095, 0, 0, 2949, 4070, 1, 51, 1197, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 17, 9, 14, 4089, 0, 625, 3267, 0, 4092, 14, 983, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4058, 0, 0, 10, 0, 0, 21, 4079, 1, 0, 1567, 417, 4078, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 9, 567, 4096, 0, 4, 0, 64, 0, 15, 18, 4096, 0, 4093, 981, 1912, 832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2595, 238, 10, 0, 0, 2972, 3774, 42, 60, 3110, 0, 0, 4083, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1294, 4095, 0, 15, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4027, 4088, 2493, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18163, 14657, 16588, 17245, 14241, 16703, 16887, 16588] avg=16384.0000 max=18163 max_rank=0 min=14241 min_rank=4 imbalance_ratio=1.108582 max_load_reduction=112909 max_load_reduction_pct=0.861427 moved_to_echo=64419 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[115, 1003, 0, 2597, 0, 0, 1420, 0, 0, 0, 9, 2716, 4095, 0, 0, 4080, 1383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1553, 0, 1894, 4052, 3966, 0, 0, 0, 4080, 0, 0, 1, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1835, 1795, 4092, 0, 12, 1203, 4096, 0, 0, 1170, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 342, 2254, 0, 0, 10, 0, 4076, 0, 444, 3047, 3, 4096, 0, 977, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3513, 0, 0, 0, 0, 0, 834, 3998, 170, 0, 1586, 1189, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 11, 0, 2454, 4095, 52, 0, 0, 222, 0, 0, 0, 4096, 0, 4089, 4, 1842, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 19, 0, 0, 9, 3984, 4096, 0, 29, 3112, 0, 0, 4025, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 4084, 2510, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17418, 16015, 14219, 15350, 15390, 17897, 19268, 15515] avg=16384.0000 max=19268 max_rank=6 min=14219 min_rank=2 imbalance_ratio=1.176025 max_load_reduction=111804 max_load_reduction_pct=0.852997 moved_to_echo=68070 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[96, 1, 4, 608, 2, 0, 30, 1, 15, 2, 0, 4094, 3993, 0, 0, 4027, 1382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1615, 1, 992, 4079, 3383, 8, 0, 0, 2496, 0, 0, 0, 2275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1832, 32, 4081, 1, 0, 2522, 4017, 4, 979, 3499, 0, 1680, 0, 12, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 8, 3879, 0, 0, 0, 6, 4092, 1, 539, 3606, 0, 4087, 0, 978, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4088, 9, 0, 0, 2, 2, 207, 4088, 17, 0, 1583, 4093, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 31, 4096, 18, 4, 0, 0, 2, 0, 0, 4096, 0, 4091, 960, 1914, 829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3870, 0, 0, 7, 2, 254, 3757, 0, 0, 3115, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1353, 4096, 0, 0, 1, 0, 0, 0, 3, 0, 3, 4, 0, 0, 4016, 4095, 2506, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14255, 14849, 18666, 17317, 18188, 16046, 15108, 16643] avg=16384.0000 max=18666 max_rank=2 min=14255 min_rank=0 imbalance_ratio=1.139282 max_load_reduction=112406 max_load_reduction_pct=0.857590 moved_to_echo=65985 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[55, 274, 0, 3543, 0, 0, 102, 6, 1, 0, 0, 1862, 4036, 0, 0, 4094, 1377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1613, 0, 1457, 4066, 2200, 0, 0, 2, 4096, 0, 0, 0, 1909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1826, 3321, 4013, 4, 0, 4078, 4095, 0, 1, 2279, 1, 103, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 27, 4096, 0, 0, 3, 0, 4084, 13, 210, 4065, 0, 4073, 0, 978, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 12, 0, 1927, 4095, 2, 0, 1585, 1248, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 1, 0, 886, 4095, 0, 1, 0, 638, 0, 0, 2, 4069, 0, 4005, 0, 1913, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 298, 378, 0, 0, 2, 1654, 3231, 0, 0, 3116, 0, 0, 4087, 2, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 331, 0, 1153, 3985, 0, 1, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 4096, 3972, 2509, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15350, 15343, 19727, 17702, 16997, 16500, 12815, 16638] avg=16384.0000 max=19727 max_rank=2 min=12815 min_rank=6 imbalance_ratio=1.204041 max_load_reduction=111345 max_load_reduction_pct=0.849495 moved_to_echo=62950 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[47, 61, 0, 2921, 18, 0, 3857, 3, 5, 0, 17, 1901, 4059, 0, 0, 4095, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1616, 0, 134, 4066, 3566, 9, 4, 0, 4056, 0, 0, 0, 2836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1831, 4, 4096, 0, 0, 4070, 4068, 0, 1, 2004, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 51, 0, 66, 3705, 1, 0, 28, 30, 4096, 11, 545, 291, 0, 4087, 23, 972, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4076, 0, 4, 0, 10, 0, 0, 4083, 105, 42, 1574, 572, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 386, 1, 1, 30, 4079, 16, 0, 0, 6, 3, 1091, 11, 4076, 0, 4086, 310, 1914, 829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3891, 12, 0, 0, 0, 3769, 4050, 1, 18, 3094, 3, 3, 3931, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 22, 838, 4067, 0, 9, 0, 10, 0, 0, 0, 0, 2, 3, 48, 11, 4064, 4042, 2492, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18364, 16287, 16077, 13963, 14571, 16839, 18794, 16177] avg=16384.0000 max=18794 max_rank=6 min=13963 min_rank=3 imbalance_ratio=1.147095 max_load_reduction=112278 max_load_reduction_pct=0.856613 moved_to_echo=66381 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[51, 0, 0, 2189, 0, 0, 9, 1, 7, 8, 215, 3131, 4014, 0, 0, 4094, 1384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1615, 57, 1414, 4079, 4090, 195, 0, 2, 3697, 8, 1, 0, 3154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1834, 1224, 4072, 14, 0, 1356, 4013, 0, 387, 3628, 3, 0, 1, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 6, 4056, 0, 2, 0, 0, 4094, 2, 300, 4002, 0, 4087, 3, 978, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 3945, 0, 0, 5, 0, 57, 1294, 4077, 0, 10, 1585, 2458, 4096, 11, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 50, 3694, 0, 2, 3, 264, 0, 150, 0, 4090, 0, 4068, 1482, 1914, 831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 6, 0, 10, 0, 83, 4079, 0, 0, 3114, 0, 0, 4085, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2075, 4094, 0, 27, 0, 1, 24, 0, 0, 101, 0, 0, 0, 0, 4095, 4095, 2508, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15103, 18312, 16574, 17584, 17664, 16555, 11694, 17586] avg=16384.0000 max=18312 max_rank=1 min=11694 min_rank=6 imbalance_ratio=1.117676 max_load_reduction=112760 max_load_reduction_pct=0.860291 moved_to_echo=63499 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[6, 379, 0, 2343, 0, 13, 317, 6, 0, 0, 9, 2611, 4082, 0, 0, 4013, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1593, 10, 3127, 3932, 3366, 0, 0, 0, 4083, 0, 3, 1, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1831, 1628, 4080, 0, 0, 1784, 4092, 2, 87, 210, 0, 1205, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 3362, 0, 8, 0, 0, 4088, 0, 12, 4090, 0, 4096, 1, 979, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 4070, 2, 0, 0, 0, 12, 470, 4091, 29, 0, 1586, 2829, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 9, 3947, 0, 0, 0, 11, 0, 72, 1, 3798, 0, 4087, 209, 1888, 829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2757, 1, 5, 0, 12, 1857, 4075, 0, 5, 3117, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 216, 4094, 0, 0, 0, 0, 0, 7, 2, 3, 0, 0, 0, 0, 4095, 4091, 2510, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15159, 20195, 14920, 16694, 17509, 15102, 15910, 15583] avg=16384.0000 max=20195 max_rank=1 min=14920 min_rank=2 imbalance_ratio=1.232605 max_load_reduction=110877 max_load_reduction_pct=0.845924 moved_to_echo=64104 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[768, 3200, 128, 18560, 128, 128, 9088, 128, 128, 128, 384, 23296, 32512, 128, 0, 32640, 11136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 21888, 1920, 128, 128, 128, 18688, 31232, 128, 128, 24960, 128, 128, 32512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 384, 128, 512, 27520, 128, 128, 128, 128, 32768, 128, 2816, 26368, 128, 32768, 128, 7936, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 12928, 128, 9728, 32384, 26880, 256, 128, 128, 30720, 128, 128, 128, 18560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[384, 128, 8192, 32640, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32640, 32640, 20096, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[512, 32000, 128, 128, 128, 128, 128, 4864, 32640, 1536, 128, 12672, 14336, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[1152, 128, 128, 4352, 32256, 128, 128, 128, 1280, 128, 1408, 128, 32512, 128, 32640, 3968, 15232, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 14720, 10496, 32640, 128, 128, 22144, 32640, 128, 1792, 14080, 128, 3072, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4090, 5, 0, 0, 4096, 0, 0, 1971, 1, 0, 4, 0, 0, 0, 0, 24, 2945, 3636, 0, 8, 0, 2, 185, 0, 0, 11, 42, 0, 35, 8, 0, 4088, 1, 0, 1, 0, 0, 3625, 4093, 0, 118, 0, 4089, 4046, 4094, 0, 4096, 3905, 0, 19, 2, 0, 0, 86, 0, 4092, 1, 0, 0, 6, 0, 3863, 0, 4, 4096, 8, 2, 4072, 7, 0, 4095, 0, 6, 1419, 0, 1, 4061, 3, 0, 4, 1, 4094, 4092, 0, 1, 3796, 0, 0, 2, 2, 0, 6, 4090, 18, 99, 2, 771, 0, 152, 0, 4077, 9, 0, 4096, 4092, 4092, 20, 1, 4066, 0, 3, 0, 4084, 0, 0, 1, 18, 0, 4088, 1431, 0, 261, 0, 0, 4094, 449, 0, 3998] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4075, 0, 9, 0, 4095, 0, 0, 3677, 0, 21, 1, 0, 0, 0, 0, 0, 465, 4088, 4, 0, 5, 0, 1, 2, 0, 2582, 0, 141, 218, 0, 0, 4081, 6, 12, 0, 0, 40, 2723, 4094, 0, 15, 1, 4096, 4070, 4045, 0, 4096, 4091, 6, 0, 0, 12, 0, 15, 0, 4036, 0, 0, 0, 24, 0, 681, 25, 1, 4081, 12, 3, 3963, 0, 0, 4095, 0, 0, 1011, 0, 0, 2733, 268, 10, 1, 0, 4095, 4094, 13, 0, 3795, 0, 0, 0, 0, 0, 3692, 4077, 0, 0, 0, 67, 48, 15, 9, 3661, 0, 0, 4080, 4069, 4084, 325, 7, 4096, 67, 13, 0, 4070, 0, 0, 0, 1, 20, 4049, 70, 1, 1, 0, 0, 4091, 2257, 379, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4091, 159, 0, 0, 4038, 4, 0, 4095, 1, 0, 1, 0, 3, 0, 0, 0, 2044, 4068, 0, 0, 0, 0, 12, 1, 0, 708, 3, 113, 610, 0, 0, 4096, 0, 1, 8, 0, 6, 2824, 4092, 0, 37, 0, 4068, 4096, 4091, 0, 4089, 4090, 0, 7, 0, 0, 0, 2, 0, 4096, 0, 0, 3, 1, 21, 864, 0, 0, 4076, 0, 0, 4042, 1, 0, 3969, 0, 5, 3613, 0, 0, 4078, 1019, 5, 0, 5, 4085, 4094, 2, 0, 4082, 0, 0, 0, 18, 0, 0, 4089, 0, 40, 0, 112, 0, 0, 1, 4082, 0, 0, 4094, 3980, 4096, 0, 0, 3964, 0, 0, 5, 4087, 0, 1, 0, 0, 0, 4079, 955, 51, 7, 0, 0, 3768, 12, 20, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4089, 8, 0, 0, 4096, 0, 250, 4044, 0, 25, 1, 0, 0, 0, 0, 0, 3813, 4088, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 50, 0, 0, 4096, 0, 0, 160, 0, 4, 90, 4096, 24, 41, 0, 3979, 4089, 4095, 0, 4094, 4069, 1, 0, 0, 0, 0, 32, 0, 4081, 0, 0, 0, 0, 0, 2025, 0, 0, 4064, 0, 0, 3732, 0, 0, 4070, 21, 0, 4072, 0, 0, 3941, 3679, 17, 0, 0, 4096, 4096, 0, 0, 4052, 29, 0, 0, 0, 0, 0, 4096, 7, 11, 0, 74, 0, 212, 0, 4095, 0, 0, 4096, 4063, 4096, 2, 0, 3617, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 6, 15, 9, 0, 0, 2237, 904, 2, 4030] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 4072, 2, 4, 4094, 0, 2, 6, 0, 0, 1, 0, 1, 2608, 1648, 0, 2, 1, 3, 0, 1, 1, 2010, 65, 20, 9, 0, 0, 4095, 9, 0, 35, 0, 0, 4095, 4089, 0, 0, 2, 3308, 4096, 4096, 0, 4094, 3969, 32, 0, 0, 2, 2, 0, 0, 4040, 8, 0, 6, 0, 0, 847, 132, 0, 4093, 0, 0, 4093, 1, 0, 4066, 1, 0, 205, 0, 0, 4086, 163, 29, 1, 0, 4091, 4094, 0, 0, 3944, 0, 0, 0, 0, 0, 3726, 4091, 0, 0, 0, 771, 0, 0, 0, 4091, 12, 0, 4094, 801, 4096, 0, 3, 4094, 173, 0, 0, 4096, 4, 2, 0, 80, 1, 4083, 3595, 0, 1022, 0, 0, 3621, 52, 1, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4080, 285, 7, 0, 4018, 0, 154, 3688, 0, 22, 3, 10, 0, 32, 0, 0, 2454, 4089, 1, 0, 0, 0, 18, 0, 0, 413, 22, 0, 207, 9, 1, 4077, 0, 11, 58, 0, 0, 4074, 4083, 62, 0, 10, 4050, 3707, 4036, 0, 4082, 3343, 0, 1, 0, 4, 0, 0, 0, 3856, 0, 0, 0, 53, 169, 13, 0, 0, 4096, 0, 0, 4096, 7, 1, 3869, 93, 5, 3557, 6, 0, 3590, 3330, 50, 7, 0, 4060, 4075, 7, 0, 4054, 15, 0, 0, 0, 16, 0, 4080, 12, 18, 3, 1, 3, 43, 1, 4088, 0, 9, 3984, 4085, 4096, 53, 0, 4069, 15, 0, 35, 4081, 20, 5, 8, 0, 0, 4005, 1, 377, 246, 0, 0, 3603, 27, 0, 3968] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4041, 42, 0, 0, 4055, 0, 0, 4093, 7, 89, 3, 0, 0, 0, 0, 0, 3835, 4095, 0, 2, 3, 0, 0, 1, 0, 1907, 0, 12, 2948, 5, 15, 4096, 9, 0, 538, 0, 268, 3364, 4091, 2, 4, 2, 4096, 4072, 4093, 0, 4096, 3737, 1, 0, 0, 0, 0, 38, 3, 4012, 79, 14, 0, 4, 0, 35, 0, 6, 4095, 12, 5, 4096, 2, 0, 4019, 0, 0, 203, 0, 0, 3656, 87, 0, 0, 1, 4090, 4096, 0, 0, 2617, 8, 0, 0, 7, 0, 1854, 4087, 0, 48, 0, 2, 2, 14, 0, 1821, 3, 0, 3850, 4094, 4096, 1, 0, 3550, 121, 2, 0, 4090, 0, 2, 4, 0, 2, 4086, 1708, 60, 7, 29, 0, 4091, 148, 443, 4050] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4096, 26, 4, 8, 4063, 0, 575, 3943, 4, 0, 0, 0, 3, 0, 0, 3, 3691, 3669, 4, 0, 0, 11, 0, 5, 2, 10, 0, 28, 2332, 0, 1, 4096, 0, 10, 0, 0, 0, 810, 4089, 0, 0, 0, 4020, 4096, 4087, 0, 4096, 3971, 23, 0, 0, 0, 0, 0, 4, 4079, 3, 0, 0, 13, 0, 1109, 23, 0, 4037, 35, 8, 3592, 0, 0, 3574, 6, 2, 4090, 3, 0, 3683, 763, 415, 6, 0, 4084, 4086, 0, 12, 3907, 77, 0, 7, 0, 0, 185, 4088, 1, 16, 1, 1029, 19, 565, 0, 3910, 3, 1, 2986, 4085, 4068, 0, 0, 4093, 130, 0, 0, 4086, 1, 0, 6, 0, 0, 4091, 116, 1, 1, 15, 1, 3964, 155, 0, 4061] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4075, 0, 9, 0, 4095, 0, 0, 3677, 0, 21, 1, 0, 0, 0, 0, 0, 2893, 726, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 4088, 4, 0, 5, 0, 1, 2, 0, 2582, 0, 141, 218, 0, 0, 4081, 1737, 1379, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 12, 0, 0, 40, 2723, 0, 0, 15, 1, 4096, 4070, 1582, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 12, 0, 15, 0, 4036, 0, 0, 0, 24, 0, 681, 25, 1, 4096, 4094, 2874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1188, 12, 3, 3963, 0, 0, 4095, 0, 0, 1011, 0, 0, 2733, 268, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3183, 13, 0, 3795, 0, 0, 0, 0, 0, 3692, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 48, 15, 9, 3661, 0, 0, 4080, 4069, 516, 325, 7, 4096, 67, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2691, 0, 0, 0, 1, 20, 4049, 70, 1, 1, 0, 0, 4091, 2257, 379, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16191, 15614, 16636, 15864, 13284, 18855, 16973, 17655] avg=16384.0000 max=18855 max_rank=5 min=13284 min_rank=4 imbalance_ratio=1.150818 max_load_reduction=112217 max_load_reduction_pct=0.856148 moved_to_echo=66767 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4090, 5, 0, 0, 4096, 0, 0, 1971, 1, 0, 4, 0, 0, 0, 0, 24, 2901, 731, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2945, 3636, 0, 8, 0, 2, 185, 0, 0, 11, 42, 0, 35, 8, 0, 4088, 1754, 1381, 907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 3625, 0, 0, 118, 0, 4089, 4046, 1609, 0, 0, 3905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2, 0, 0, 86, 0, 4092, 1, 0, 0, 6, 0, 3863, 0, 4, 4096, 4093, 2874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1195, 8, 2, 4072, 7, 0, 4095, 0, 6, 1419, 0, 1, 4061, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 3185, 0, 1, 3796, 0, 0, 2, 2, 0, 6, 4090, 18, 99, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 771, 0, 152, 0, 4077, 9, 0, 4096, 4092, 528, 20, 1, 4066, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2703, 0, 0, 1, 18, 0, 4088, 1431, 0, 261, 0, 0, 4094, 449, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14513, 15002, 17394, 19136, 14873, 15296, 17815, 17043] avg=16384.0000 max=19136 max_rank=3 min=14513 min_rank=0 imbalance_ratio=1.167969 max_load_reduction=111936 max_load_reduction_pct=0.854004 moved_to_echo=65027 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 4072, 2, 4, 4094, 0, 2, 6, 0, 0, 1, 0, 1, 2899, 731, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2608, 1648, 0, 2, 1, 3, 0, 1, 1, 2010, 65, 20, 9, 0, 0, 4095, 1755, 1385, 907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 35, 0, 0, 4095, 0, 0, 0, 2, 3308, 4096, 1610, 0, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 2, 2, 0, 0, 4040, 8, 0, 6, 0, 0, 847, 132, 0, 4094, 4089, 2876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1194, 0, 0, 4093, 1, 0, 4066, 1, 0, 205, 0, 0, 4086, 163, 29, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3187, 0, 0, 3944, 0, 0, 0, 0, 0, 3726, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 771, 0, 0, 0, 4091, 12, 0, 4094, 801, 530, 0, 3, 4094, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2711, 4, 2, 0, 80, 1, 4083, 3595, 0, 1022, 0, 0, 3621, 52, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16596, 14510, 17124, 16128, 13839, 19039, 14569, 19267] avg=16384.0000 max=19267 max_rank=7 min=13839 min_rank=4 imbalance_ratio=1.175964 max_load_reduction=111805 max_load_reduction_pct=0.853004 moved_to_echo=66714 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4091, 159, 0, 0, 4038, 4, 0, 4095, 1, 0, 1, 0, 3, 0, 0, 0, 2886, 730, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2044, 4068, 0, 0, 0, 0, 12, 1, 0, 708, 3, 113, 610, 0, 0, 4096, 1753, 1382, 907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 6, 2824, 0, 0, 37, 0, 4068, 4096, 1608, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2, 0, 4096, 0, 0, 3, 1, 21, 864, 0, 0, 4089, 4092, 2876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1190, 0, 0, 4042, 1, 0, 3969, 0, 5, 3613, 0, 0, 4078, 1019, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4085, 3187, 2, 0, 4082, 0, 0, 0, 18, 0, 0, 4089, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 1, 4082, 0, 0, 4094, 3980, 530, 0, 0, 3964, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2705, 0, 1, 0, 0, 0, 4079, 955, 51, 7, 0, 0, 3768, 12, 20, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16698, 15697, 16738, 16051, 17922, 15508, 16768, 15690] avg=16384.0000 max=17922 max_rank=4 min=15508 min_rank=5 imbalance_ratio=1.093872 max_load_reduction=113150 max_load_reduction_pct=0.863266 moved_to_echo=65888 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4041, 42, 0, 0, 4055, 0, 0, 4093, 7, 89, 3, 0, 0, 0, 0, 0, 2900, 731, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3835, 4095, 0, 2, 3, 0, 0, 1, 0, 1907, 0, 12, 2948, 5, 15, 4096, 1754, 1383, 908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 538, 0, 268, 3364, 0, 2, 4, 2, 4096, 4072, 1608, 0, 0, 3737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 38, 3, 4012, 79, 14, 0, 4, 0, 35, 0, 6, 4096, 4091, 2876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1195, 12, 5, 4096, 2, 0, 4019, 0, 0, 203, 0, 0, 3656, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 3188, 0, 0, 2617, 8, 0, 0, 7, 0, 1854, 4087, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 14, 0, 1821, 3, 0, 3850, 4094, 530, 1, 0, 3550, 121, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2707, 0, 2, 4, 0, 2, 4086, 1708, 60, 7, 29, 0, 4091, 148, 443, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16651, 20964, 17700, 15255, 13275, 15900, 13990, 17337] avg=16384.0000 max=20964 max_rank=1 min=13275 min_rank=4 imbalance_ratio=1.279541 max_load_reduction=110108 max_load_reduction_pct=0.840057 moved_to_echo=60502 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4080, 285, 7, 0, 4018, 0, 154, 3688, 0, 22, 3, 10, 0, 32, 0, 0, 2901, 721, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2454, 4089, 1, 0, 0, 0, 18, 0, 0, 413, 22, 0, 207, 9, 1, 4077, 1729, 1380, 903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 58, 0, 0, 4074, 0, 62, 0, 10, 4050, 3707, 1586, 0, 0, 3343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 3856, 0, 0, 0, 53, 169, 13, 0, 0, 4082, 4083, 2876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1195, 0, 0, 4096, 7, 1, 3869, 93, 5, 3557, 6, 0, 3590, 3330, 50, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 3172, 7, 0, 4054, 15, 0, 0, 0, 16, 0, 4080, 12, 18, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 43, 1, 4088, 0, 9, 3984, 4085, 530, 53, 0, 4069, 15, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2701, 20, 5, 8, 0, 0, 4005, 1, 377, 246, 0, 0, 3603, 27, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16611, 15303, 16901, 15137, 19806, 15437, 16916, 14961] avg=16384.0000 max=19806 max_rank=4 min=14961 min_rank=7 imbalance_ratio=1.208862 max_load_reduction=111266 max_load_reduction_pct=0.848892 moved_to_echo=67120 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4089, 8, 0, 0, 4096, 0, 250, 4044, 0, 25, 1, 0, 0, 0, 0, 0, 2878, 731, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3813, 4088, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 50, 0, 0, 4096, 1754, 1385, 908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 4, 90, 0, 24, 41, 0, 3979, 4089, 1610, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 32, 0, 4081, 0, 0, 0, 0, 0, 2025, 0, 0, 4094, 4096, 2876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1186, 0, 0, 3732, 0, 0, 4070, 21, 0, 4072, 0, 0, 3941, 3679, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3188, 0, 0, 4052, 29, 0, 0, 0, 0, 0, 4096, 7, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 212, 0, 4095, 0, 0, 4096, 4063, 530, 2, 0, 3617, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2711, 0, 0, 0, 0, 0, 4088, 6, 15, 9, 0, 0, 2237, 904, 2, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16812, 16095, 14066, 17205, 20718, 15479, 16695, 14002] avg=16384.0000 max=20718 max_rank=4 min=14002 min_rank=7 imbalance_ratio=1.264526 max_load_reduction=110354 max_load_reduction_pct=0.841934 moved_to_echo=66894 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 26, 4, 8, 4063, 0, 575, 3943, 4, 0, 0, 0, 3, 0, 0, 3, 2859, 730, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3691, 3669, 4, 0, 0, 11, 0, 5, 2, 10, 0, 28, 2332, 0, 1, 4096, 1751, 1381, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 810, 0, 0, 0, 0, 4020, 4096, 1606, 0, 0, 3971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 4, 4079, 3, 0, 0, 13, 0, 1109, 23, 0, 4096, 4089, 2857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1178, 35, 8, 3592, 0, 0, 3574, 6, 2, 4090, 3, 0, 3683, 763, 415, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 3180, 0, 12, 3907, 77, 0, 7, 0, 0, 185, 4088, 1, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1029, 19, 565, 0, 3910, 3, 1, 2986, 4085, 525, 0, 0, 4093, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2705, 1, 0, 6, 0, 0, 4091, 116, 1, 1, 15, 1, 3964, 155, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17000, 17887, 14513, 16296, 17355, 15558, 17346, 15117] avg=16384.0000 max=17887 max_rank=1 min=14513 min_rank=2 imbalance_ratio=1.091736 max_load_reduction=113185 max_load_reduction_pct=0.863533 moved_to_echo=65376 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2944, 128, 1024, 128, 29952, 128, 128, 31360, 29312, 4224, 512, 128, 31616, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[32768, 640, 128, 128, 32640, 128, 1024, 29696, 128, 256, 128, 128, 128, 128, 0, 128, 23168, 5888, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 25472, 128, 128, 30336, 256, 0, 128, 128, 128, 9472, 32768, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 896, 0, 384, 21632, 0, 128, 256, 128, 31744, 32384, 12928, 0, 0, 31232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[21888, 29440, 128, 128, 128, 128, 256, 128, 128, 7680, 256, 384, 6528, 128, 128, 32768, 14080, 11136, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[21760, 128, 128, 128, 128, 128, 32640, 7936, 512, 1664, 128, 128, 29568, 4096, 896, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 256, 128, 32384, 128, 128, 128, 128, 256, 9472, 256, 128, 32768, 32768, 23040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[9600, 128, 128, 31744, 128, 128, 31872, 128, 128, 18176, 128, 128, 29952, 9344, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[108, 1, 0, 4092, 1, 0, 7, 4067, 3078, 262, 12, 1, 0, 0, 0, 4096, 4088, 3440, 0, 4088, 2, 4091, 28, 0, 693, 2119, 2, 4, 4062, 0, 3639, 0, 0, 1, 4087, 0, 15, 4, 8, 0, 4053, 0, 4096, 4095, 0, 0, 2, 0, 4, 0, 0, 3815, 0, 0, 4092, 4091, 4093, 0, 3190, 0, 15, 3796, 0, 0, 2, 1, 2, 0, 4057, 4085, 5, 4087, 4096, 13, 0, 4, 0, 0, 1859, 1, 0, 0, 0, 0, 23, 4, 1, 0, 0, 0, 0, 370, 5, 6, 0, 7, 9, 0, 0, 1, 0, 3, 2, 3113, 32, 0, 3483, 4033, 1, 2, 95, 0, 4073, 0, 0, 0, 4070, 865, 4085, 1, 2, 4091, 0, 3035, 10, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 25, 4045, 6, 0, 0, 4094, 1331, 3476, 0, 0, 21, 10, 592, 4051, 4042, 3571, 0, 4071, 6, 4082, 1, 10, 2429, 1271, 24, 0, 4095, 0, 2369, 6, 1, 79, 3994, 4, 208, 0, 26, 2, 4013, 0, 4089, 4092, 50, 1, 0, 0, 12, 8, 0, 4092, 0, 0, 3749, 4093, 4042, 20, 4080, 0, 46, 3537, 11, 0, 869, 40, 0, 0, 4087, 4085, 0, 4081, 4094, 13, 2, 0, 0, 0, 42, 0, 0, 0, 7, 0, 6, 2, 0, 0, 398, 0, 0, 905, 0, 0, 6, 0, 0, 0, 1, 0, 12, 0, 0, 3802, 0, 0, 712, 4056, 0, 485, 32, 35, 4059, 1, 0, 34, 4059, 3115, 4095, 9, 10, 4004, 0, 16, 0, 14, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[9, 0, 18, 3995, 0, 19, 55, 4054, 4054, 2, 7, 0, 0, 0, 1, 4089, 4046, 937, 9, 4091, 0, 4081, 0, 0, 1425, 0, 0, 1, 4093, 13, 3888, 0, 0, 2, 4088, 0, 5, 10, 0, 0, 4020, 0, 4050, 4088, 17, 1, 2, 1, 0, 0, 0, 4096, 7, 0, 4095, 4079, 4086, 5, 2692, 26, 4, 4096, 498, 180, 0, 0, 0, 0, 4090, 4096, 0, 4065, 3881, 1, 0, 5, 0, 39, 3965, 0, 140, 0, 4, 0, 1, 25, 0, 3, 0, 1, 0, 899, 0, 3, 0, 0, 3, 5, 0, 5, 0, 0, 17, 4093, 70, 0, 1442, 4095, 14, 615, 71, 0, 4082, 14, 4, 13, 4088, 1, 4076, 0, 0, 3992, 23, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 36, 4094, 3064, 121, 0, 0, 2, 0, 0, 4072, 4067, 969, 0, 4096, 1, 4096, 128, 0, 2482, 3292, 1, 0, 4096, 0, 2814, 0, 0, 1, 4096, 1, 2, 0, 0, 0, 3937, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3941, 0, 3, 4069, 4094, 4096, 0, 3975, 0, 0, 3886, 0, 0, 32, 0, 0, 0, 4079, 4096, 0, 4096, 4006, 314, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 1, 0, 33, 22, 0, 150, 0, 7, 0, 21, 0, 0, 9, 0, 0, 0, 0, 2698, 1, 0, 4093, 4096, 0, 7, 6, 3, 4096, 0, 0, 0, 4096, 3685, 4095, 1, 0, 4096, 0, 1281, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 114, 4095, 0, 0, 0, 4083, 1910, 1255, 5, 0, 0, 0, 243, 4089, 4084, 3401, 2, 4096, 0, 4096, 3, 1, 1419, 478, 0, 0, 4096, 0, 4032, 0, 0, 3, 4081, 7, 1833, 0, 0, 4, 4096, 0, 4095, 4083, 0, 0, 10, 0, 1, 0, 1, 4096, 0, 0, 4096, 4096, 4093, 0, 3625, 0, 0, 2793, 8, 0, 74, 0, 0, 4, 4083, 4096, 0, 4093, 4046, 1, 0, 0, 0, 0, 592, 0, 0, 0, 0, 2, 17, 0, 0, 1, 6, 0, 0, 1, 0, 1, 0, 13, 0, 0, 0, 0, 4, 0, 2, 4065, 17, 0, 2966, 4093, 0, 11, 0, 0, 4091, 4, 0, 0, 4020, 2874, 4090, 5, 0, 4051, 0, 1219, 5, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 23, 4076, 0, 0, 2, 4080, 4048, 2065, 0, 0, 0, 0, 0, 4094, 4090, 1571, 0, 4077, 0, 4096, 0, 0, 2330, 3032, 0, 0, 3985, 0, 3658, 1, 1, 8, 4089, 14, 67, 1, 0, 0, 4096, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 3932, 0, 0, 4066, 4096, 4096, 8, 2264, 0, 162, 3889, 0, 0, 287, 47, 0, 0, 3882, 4096, 0, 4092, 4055, 0, 2, 0, 0, 0, 16, 0, 7, 4, 1, 22, 1, 6, 0, 7, 22, 0, 0, 17, 0, 0, 0, 0, 0, 0, 1, 69, 10, 0, 3, 4087, 61, 0, 1418, 4096, 0, 0, 152, 3, 4096, 0, 0, 1, 4034, 10, 4096, 30, 22, 4054, 0, 4054, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 82, 4059, 0, 0, 0, 4095, 4004, 3806, 0, 0, 333, 1, 30, 4096, 4096, 3494, 0, 3995, 1, 4093, 13, 1744, 4, 0, 0, 1, 4095, 0, 2644, 0, 7, 0, 4094, 0, 0, 1, 0, 0, 4067, 0, 4094, 4096, 0, 13, 15, 0, 2, 0, 0, 4029, 3, 0, 4091, 4095, 4090, 4, 2777, 1, 0, 3940, 585, 0, 0, 0, 4, 0, 4092, 4095, 0, 4094, 4029, 7, 0, 2, 0, 0, 202, 3, 51, 173, 0, 0, 2, 0, 1, 0, 34, 154, 2, 376, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 2627, 0, 0, 3889, 4089, 0, 94, 0, 8, 4095, 0, 2, 1, 4093, 529, 4090, 8, 4, 4051, 2, 1476, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 0, 4092, 0, 0, 1, 4094, 4080, 2415, 7, 0, 0, 0, 17, 4075, 4096, 2201, 5, 4093, 0, 4078, 0, 0, 3862, 8, 25, 0, 4061, 3, 4083, 12, 0, 36, 4069, 0, 3, 0, 0, 6, 4056, 0, 4093, 4092, 0, 0, 0, 0, 6, 0, 0, 4094, 14, 0, 4055, 4096, 4081, 10, 3567, 9, 0, 1490, 19, 0, 0, 17, 14, 18, 4067, 4087, 0, 4094, 4086, 0, 0, 2, 0, 0, 2087, 0, 0, 0, 0, 0, 1, 0, 2, 0, 44, 1, 0, 60, 4, 0, 4, 1, 0, 27, 0, 1, 0, 2, 5, 4096, 0, 0, 4088, 4096, 4, 2, 0, 42, 4090, 28, 0, 0, 4084, 921, 4089, 0, 0, 3571, 5, 222, 0, 0, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 114, 4095, 0, 0, 0, 4083, 1910, 1255, 5, 0, 0, 0, 243, 3143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3401, 2, 3127, 0, 0, 3, 1, 1419, 478, 0, 0, 4096, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4081, 7, 1833, 0, 0, 4, 4096, 0, 4095, 3861, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4096, 0, 0, 4096, 0, 1428, 0, 3625, 0, 0, 2793, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 4, 4083, 2868, 0, 4093, 4046, 1, 0, 0, 0, 0, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 17, 0, 0, 1, 6, 0, 0, 1, 0, 1, 0, 13, 4096, 4096, 4084, 3305, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 4065, 17, 0, 2966, 4093, 0, 11, 0, 0, 2355, 1228, 969, 946, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 0, 0, 4020, 2874, 785, 5, 0, 4051, 0, 1219, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14849, 16559, 17990, 16048, 15761, 15932, 16878, 17055] avg=16384.0000 max=17990 max_rank=2 min=14849 min_rank=0 imbalance_ratio=1.098022 max_load_reduction=113082 max_load_reduction_pct=0.862747 moved_to_echo=65626 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[108, 1, 0, 4092, 1, 0, 7, 4067, 3078, 262, 12, 1, 0, 0, 0, 3148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3440, 0, 3121, 2, 0, 28, 0, 693, 2119, 2, 4, 4062, 0, 3639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 15, 4, 8, 0, 4053, 0, 4096, 3872, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3815, 0, 0, 4092, 0, 1428, 0, 3190, 0, 15, 3796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 0, 4057, 2860, 5, 4087, 4096, 13, 0, 4, 0, 0, 1859, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4, 1, 0, 0, 0, 0, 370, 5, 6, 0, 7, 4091, 4091, 4088, 3301, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 0, 3, 2, 3113, 32, 0, 3483, 4033, 1, 2, 95, 0, 2355, 1225, 967, 948, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 4070, 865, 784, 1, 2, 4091, 0, 3035, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14777, 17110, 16138, 16340, 16987, 16297, 16492, 16931] avg=16384.0000 max=17110 max_rank=1 min=14777 min_rank=0 imbalance_ratio=1.044312 max_load_reduction=113962 max_load_reduction_pct=0.869461 moved_to_echo=66707 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 25, 4045, 6, 0, 0, 4094, 1331, 3476, 0, 0, 21, 10, 592, 3114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3571, 0, 3108, 6, 0, 1, 10, 2429, 1271, 24, 0, 4095, 0, 2369, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 79, 3994, 4, 208, 0, 26, 2, 4013, 0, 4089, 3870, 50, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 8, 0, 4092, 0, 0, 3749, 0, 1411, 20, 4080, 0, 46, 3537, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 869, 40, 0, 0, 4087, 2860, 0, 4081, 4094, 13, 2, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 6, 2, 0, 0, 398, 0, 0, 905, 0, 0, 6, 0, 4093, 4082, 4042, 3309, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 3802, 0, 0, 712, 4056, 0, 485, 32, 35, 2325, 1225, 963, 937, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 1, 0, 34, 4059, 3115, 786, 9, 10, 4004, 0, 16, 0, 14, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16715, 16890, 16337, 16966, 16088, 17156, 14807, 16113] avg=16384.0000 max=17156 max_rank=5 min=14807 min_rank=6 imbalance_ratio=1.047119 max_load_reduction=113916 max_load_reduction_pct=0.869110 moved_to_echo=64164 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 18, 3995, 0, 19, 55, 4054, 4054, 2, 7, 0, 0, 0, 1, 3138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 937, 9, 3118, 0, 0, 0, 0, 1425, 0, 0, 1, 4093, 13, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4088, 0, 5, 10, 0, 0, 4020, 0, 4050, 3862, 17, 1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 4095, 0, 1419, 5, 2692, 26, 4, 4096, 498, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2864, 0, 4065, 3881, 1, 0, 5, 0, 39, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 4, 0, 1, 25, 0, 3, 0, 1, 0, 899, 0, 3, 0, 0, 4079, 4081, 4046, 3300, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 0, 5, 0, 0, 17, 4093, 70, 0, 1442, 4095, 14, 615, 71, 0, 2354, 1232, 973, 951, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 14, 4, 13, 4088, 1, 776, 0, 0, 3992, 23, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15352, 13484, 16058, 17118, 18910, 16895, 16166, 17089] avg=16384.0000 max=18910 max_rank=4 min=13484 min_rank=1 imbalance_ratio=1.154175 max_load_reduction=112162 max_load_reduction_pct=0.855728 moved_to_echo=69060 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 23, 4076, 0, 0, 2, 4080, 4048, 2065, 0, 0, 0, 0, 0, 3147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1571, 0, 3112, 0, 0, 0, 0, 2330, 3032, 0, 0, 3985, 0, 3658, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 4089, 14, 67, 1, 0, 0, 4096, 0, 4096, 3873, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3932, 0, 0, 4066, 0, 1429, 8, 2264, 0, 162, 3889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 47, 0, 0, 3882, 2868, 0, 4092, 4055, 0, 2, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 1, 22, 1, 6, 0, 7, 22, 0, 0, 17, 0, 0, 0, 0, 4096, 4096, 4090, 3310, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 69, 10, 0, 3, 4087, 61, 0, 1418, 4096, 0, 0, 152, 3, 2357, 1228, 965, 947, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4034, 10, 786, 30, 22, 4054, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17442, 17689, 16246, 15750, 15249, 15989, 15620, 17087] avg=16384.0000 max=17689 max_rank=1 min=15249 min_rank=4 imbalance_ratio=1.079651 max_load_reduction=113383 max_load_reduction_pct=0.865044 moved_to_echo=63945 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 82, 4059, 0, 0, 0, 4095, 4004, 3806, 0, 0, 333, 1, 30, 3148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3494, 0, 3050, 1, 0, 13, 1744, 4, 0, 0, 1, 4095, 0, 2644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4094, 0, 0, 1, 0, 0, 4067, 0, 4094, 3873, 0, 13, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4029, 3, 0, 4091, 0, 1427, 4, 2777, 1, 0, 3940, 585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4092, 2867, 0, 4094, 4029, 7, 0, 2, 0, 0, 202, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 173, 0, 0, 2, 0, 1, 0, 34, 154, 2, 376, 0, 0, 0, 0, 4095, 4093, 4096, 3305, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 2627, 0, 0, 3889, 4089, 0, 94, 0, 8, 2353, 1228, 945, 948, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 1, 4093, 529, 785, 8, 4, 4051, 2, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19558, 15046, 16164, 16859, 15300, 16692, 16407, 15046] avg=16384.0000 max=19558 max_rank=0 min=15046 min_rank=1 imbalance_ratio=1.193726 max_load_reduction=111514 max_load_reduction_pct=0.850784 moved_to_echo=63445 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 36, 4094, 3064, 121, 0, 0, 2, 0, 0, 3130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 969, 0, 3127, 1, 0, 128, 0, 2482, 3292, 1, 0, 4096, 0, 2814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 1, 2, 0, 0, 0, 3937, 0, 4096, 3873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 0, 3, 4069, 0, 1429, 0, 3975, 0, 0, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 4079, 2868, 0, 4096, 4006, 314, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 33, 22, 0, 150, 0, 7, 0, 21, 4094, 4096, 4067, 3309, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 2698, 1, 0, 4093, 4096, 0, 7, 6, 3, 2357, 1228, 969, 942, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3685, 786, 1, 0, 4096, 0, 1281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14540, 16910, 16006, 17303, 15530, 16110, 16632, 18041] avg=16384.0000 max=18041 max_rank=7 min=14540 min_rank=0 imbalance_ratio=1.101135 max_load_reduction=113031 max_load_reduction_pct=0.862358 moved_to_echo=66313 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4092, 0, 0, 1, 4094, 4080, 2415, 7, 0, 0, 0, 17, 3132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2201, 5, 3124, 0, 0, 0, 0, 3862, 8, 25, 0, 4061, 3, 4083, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 4069, 0, 3, 0, 0, 6, 4056, 0, 4093, 3870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4094, 14, 0, 4055, 0, 1424, 10, 3567, 9, 0, 1490, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 14, 18, 4067, 2862, 0, 4094, 4086, 0, 0, 2, 0, 0, 2087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 44, 1, 0, 60, 4, 0, 4, 1, 4096, 4078, 4096, 3305, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 1, 0, 2, 5, 4096, 0, 0, 4088, 4096, 4, 2, 0, 42, 2348, 1225, 969, 943, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 28, 0, 0, 4084, 921, 784, 0, 0, 3571, 5, 222, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17839, 17384, 16133, 14688, 17247, 16001, 18070, 13710] avg=16384.0000 max=18070 max_rank=6 min=13710 min_rank=7 imbalance_ratio=1.102905 max_load_reduction=113002 max_load_reduction_pct=0.862137 moved_to_echo=65028 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 19584, 128, 24960, 128, 0, 256, 1792, 14720, 10240, 128, 128, 32640, 128, 27136, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 384, 32640, 128, 128, 128, 32768, 25600, 13440, 128, 128, 384, 128, 896, 25216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 128, 32640, 12032, 6272, 128, 128, 32000, 128, 15488, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 256, 32640, 128, 2176, 128, 128, 128, 32384, 0, 32768, 30976, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 28672, 256, 0, 22144, 32768, 128, 1280, 384, 128, 18816, 9856, 7808, 7680, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 32128, 128, 128, 32384, 0, 11520, 128, 26240, 128, 256, 27520, 1152, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[256, 256, 128, 128, 128, 128, 128, 128, 640, 256, 128, 2816, 128, 128, 128, 128, 32768, 32768, 32640, 26496, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1280, 128, 128, 128, 32512, 23040, 128, 32768, 32384, 384, 128, 128, 0, 128, 8960, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 3213, 1154, 0, 1, 0, 0, 4069, 5, 0, 3, 6, 4096, 4010, 4095, 0, 4006, 19, 4096, 4089, 6, 0, 2628, 0, 1, 0, 0, 0, 0, 0, 58, 4096, 0, 0, 0, 4095, 0, 907, 0, 0, 25, 2, 0, 0, 0, 0, 27, 0, 0, 3, 335, 134, 4093, 3, 0, 4096, 18, 3, 3, 0, 0, 0, 4095, 1, 3, 0, 4093, 0, 4077, 4062, 0, 0, 940, 19, 4096, 0, 0, 5, 0, 0, 1, 0, 0, 1, 0, 4054, 0, 4095, 4095, 0, 0, 0, 0, 4086, 0, 4095, 0, 0, 4094, 4096, 0, 4095, 4065, 65, 0, 2, 0, 0, 3, 3, 0, 4094, 68, 0, 4091, 4079, 0, 0, 4, 1, 0, 0, 3006, 4092, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1, 822, 1561, 7, 9, 0, 0, 3831, 0, 107, 2, 0, 4087, 3345, 4096, 104, 4045, 12, 4093, 4096, 0, 42, 163, 16, 2, 0, 7, 5, 1, 0, 0, 4093, 1, 0, 0, 4089, 0, 916, 0, 0, 0, 59, 15, 11, 4, 53, 0, 0, 44, 2, 4031, 0, 4094, 0, 0, 4065, 0, 0, 0, 1, 0, 2, 4091, 0, 0, 0, 4090, 0, 4074, 3997, 0, 0, 768, 663, 4073, 0, 0, 0, 0, 38, 0, 11, 85, 0, 0, 4096, 0, 4096, 3924, 0, 0, 0, 8, 3975, 5, 3573, 4, 0, 4096, 4067, 0, 4096, 4095, 179, 0, 0, 0, 0, 519, 5, 0, 4096, 0, 0, 4076, 4096, 2, 0, 54, 0, 0, 4, 4090, 4087, 0, 0, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 68, 1396, 0, 0, 0, 0, 3103, 1, 241, 0, 0, 3817, 3962, 4095, 0, 4074, 0, 4094, 4096, 68, 0, 2010, 0, 0, 0, 0, 0, 0, 4, 945, 4096, 0, 0, 0, 4091, 43, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3696, 43, 4081, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 4051, 0, 0, 0, 4096, 0, 4096, 4023, 30, 0, 975, 0, 4096, 0, 24, 0, 0, 1, 0, 0, 0, 0, 1, 4087, 0, 4096, 4096, 1, 0, 0, 0, 4037, 0, 4096, 0, 0, 4091, 4043, 0, 4096, 4095, 238, 30, 0, 0, 0, 16, 3, 0, 4096, 4, 0, 4080, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 0, 99, 1, 3, 0, 6, 4095, 0, 23, 0, 0, 4095, 4081, 4090, 0, 4089, 227, 4048, 4093, 0, 0, 0, 0, 0, 6, 0, 0, 0, 14, 0, 3496, 0, 58, 0, 4096, 2, 2, 1, 2, 0, 0, 0, 0, 0, 0, 0, 2, 58, 0, 4083, 2, 4090, 0, 0, 4094, 0, 1, 3, 0, 1, 0, 4069, 0, 6, 4, 4083, 4, 4093, 4060, 0, 0, 4052, 0, 4092, 1, 14, 0, 6, 0, 0, 0, 22, 1, 0, 4094, 0, 4096, 4075, 0, 1, 3, 0, 4025, 0, 4094, 2, 1, 4088, 4082, 0, 4089, 4091, 331, 0, 38, 5, 1, 2, 2, 0, 4091, 0, 0, 4093, 4059, 0, 1, 2, 0, 49, 0, 4095, 4089, 0, 5, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1, 2327, 2965, 0, 0, 0, 7, 3785, 0, 0, 0, 0, 4092, 263, 4053, 12, 4026, 459, 4077, 4092, 0, 0, 3690, 63, 12, 0, 0, 10, 0, 8, 3, 3880, 4, 0, 0, 4039, 0, 323, 0, 0, 5, 240, 0, 0, 3, 4, 0, 0, 2, 1, 4093, 103, 4096, 0, 0, 4083, 0, 0, 0, 8, 0, 6, 4084, 0, 0, 0, 4046, 0, 4095, 3981, 10, 10, 0, 0, 4081, 34, 2, 0, 0, 0, 0, 9, 0, 0, 204, 4096, 0, 4096, 4072, 0, 0, 0, 5, 4082, 0, 4061, 0, 0, 4094, 3979, 0, 4094, 4096, 0, 0, 0, 14, 0, 118, 43, 0, 4096, 6, 1, 4041, 3901, 0, 0, 1, 0, 0, 0, 2340, 4096, 216, 68, 56, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 3873, 1798, 29, 15, 0, 0, 4075, 0, 4, 0, 0, 4056, 2864, 4042, 0, 4007, 0, 4080, 4081, 13, 67, 3170, 0, 0, 0, 0, 1, 16, 0, 0, 3867, 37, 8, 9, 4080, 21, 514, 0, 0, 2, 0, 1, 9, 65, 0, 18, 0, 0, 8, 298, 5, 4070, 3, 0, 4067, 29, 0, 55, 1161, 4, 220, 4073, 19, 22, 5, 4095, 37, 4078, 982, 0, 1, 3912, 44, 4087, 10, 1186, 0, 40, 0, 0, 2, 0, 1, 1, 4048, 2, 3920, 4054, 0, 0, 3, 0, 3083, 3, 4058, 0, 0, 4052, 4096, 1, 4081, 4082, 117, 0, 2, 2, 2, 242, 0, 41, 4090, 0, 22, 3961, 4014, 0, 0, 31, 9, 0, 2, 1701, 3906, 0, 10, 74, 26] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[95, 1911, 317, 0, 0, 0, 6, 4040, 0, 1, 0, 23, 2522, 2652, 4095, 11, 4094, 57, 4088, 4096, 0, 21, 2760, 0, 102, 7, 0, 0, 0, 39, 0, 4096, 0, 0, 0, 3924, 0, 189, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 2, 0, 3883, 9, 4083, 0, 0, 4096, 12, 0, 0, 19, 0, 0, 4018, 1, 0, 3, 4096, 7, 4083, 4066, 0, 0, 24, 0, 4096, 0, 0, 0, 0, 0, 4, 2, 197, 0, 0, 4095, 0, 3889, 4096, 0, 0, 0, 0, 3923, 0, 4096, 0, 0, 4096, 4081, 0, 4096, 4096, 182, 55, 34, 0, 0, 328, 2145, 0, 4093, 0, 0, 3995, 4096, 0, 0, 35, 0, 0, 0, 3738, 4052, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[2, 3967, 3780, 6, 17, 7, 0, 4096, 0, 73, 0, 0, 4095, 1814, 4094, 0, 4085, 13, 4096, 4090, 0, 0, 12, 2, 0, 0, 0, 0, 5, 62, 145, 4084, 0, 0, 0, 4082, 6, 1641, 1, 0, 60, 169, 0, 0, 0, 0, 0, 0, 294, 0, 12, 154, 4072, 0, 0, 4089, 0, 0, 1, 37, 6, 0, 4017, 13, 0, 7, 4096, 1, 4095, 3943, 23, 1, 2798, 0, 4090, 0, 0, 0, 7, 0, 3, 0, 0, 0, 0, 4096, 1, 4087, 4084, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 4091, 3939, 8, 4085, 3974, 0, 0, 0, 8, 0, 0, 0, 0, 4094, 0, 1, 2768, 4081, 11, 0, 0, 3, 10, 12, 3176, 4093, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3213, 1154, 0, 1, 0, 0, 4069, 5, 0, 3, 6, 4096, 4010, 2013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 19, 4096, 2162, 6, 0, 2628, 0, 1, 0, 0, 0, 0, 0, 58, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 907, 0, 0, 25, 2, 0, 0, 0, 0, 27, 0, 4094, 3786, 2082, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 335, 134, 4093, 3, 0, 4096, 18, 3, 3, 0, 0, 0, 4095, 1, 1003, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 4077, 4062, 0, 0, 940, 19, 2610, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 268, 0, 4095, 4095, 0, 0, 0, 0, 4086, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 3579, 4065, 65, 0, 2, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 4091, 4079, 0, 0, 4, 1, 0, 0, 3006, 4092, 0, 1, 0, 0, 516, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18570, 17072, 16663, 14069, 15809, 16641, 15907, 16341] avg=16384.0000 max=18570 max_rank=0 min=14069 min_rank=3 imbalance_ratio=1.133423 max_load_reduction=112502 max_load_reduction_pct=0.858322 moved_to_echo=64698 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[95, 1911, 317, 0, 0, 0, 6, 4040, 0, 1, 0, 23, 2522, 2652, 2013, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 57, 4088, 2165, 0, 21, 2760, 0, 102, 7, 0, 0, 0, 39, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3924, 0, 189, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 4093, 3825, 2082, 1648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3883, 9, 4083, 0, 0, 4096, 12, 0, 0, 19, 0, 0, 4018, 1, 1003, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 7, 4083, 4066, 0, 0, 24, 0, 2610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 197, 0, 0, 270, 0, 3889, 4096, 0, 0, 0, 0, 3923, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 3580, 4096, 182, 55, 34, 0, 0, 328, 2145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 4096, 0, 0, 35, 0, 0, 0, 3738, 4052, 0, 0, 0, 0, 516, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13591, 17429, 15765, 17409, 14889, 16477, 18597, 16915] avg=16384.0000 max=18597 max_rank=6 min=13591 min_rank=0 imbalance_ratio=1.135071 max_load_reduction=112475 max_load_reduction_pct=0.858116 moved_to_echo=66878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 2327, 2965, 0, 0, 0, 7, 3785, 0, 0, 0, 0, 4092, 263, 1987, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 459, 4077, 2155, 0, 0, 3690, 63, 12, 0, 0, 10, 0, 8, 3, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4039, 0, 323, 0, 0, 5, 240, 0, 0, 3, 4, 0, 0, 4096, 3828, 2066, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4093, 103, 4096, 0, 0, 4083, 0, 0, 0, 8, 0, 6, 4084, 0, 1004, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 4095, 3981, 10, 10, 0, 0, 2592, 34, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 204, 268, 0, 4096, 4072, 0, 0, 0, 5, 4082, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3979, 0, 3572, 4096, 0, 0, 0, 14, 0, 118, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 4041, 3901, 0, 0, 1, 0, 0, 0, 2340, 4096, 216, 68, 56, 9, 522, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15439, 18383, 16260, 17765, 14770, 16797, 15916, 15742] avg=16384.0000 max=18383 max_rank=1 min=14770 min_rank=4 imbalance_ratio=1.122009 max_load_reduction=112689 max_load_reduction_pct=0.859749 moved_to_echo=63225 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 3873, 1798, 29, 15, 0, 0, 4075, 0, 4, 0, 0, 4056, 2864, 1986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 4080, 2157, 13, 67, 3170, 0, 0, 0, 0, 1, 16, 0, 0, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 8, 9, 4080, 21, 514, 0, 0, 2, 0, 1, 9, 65, 0, 18, 0, 4090, 3781, 2056, 1642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 298, 5, 4070, 3, 0, 4067, 29, 0, 55, 1161, 4, 220, 4073, 19, 1001, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 5, 4095, 37, 4078, 982, 0, 1, 3912, 44, 2604, 10, 1186, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 267, 2, 3920, 4054, 0, 0, 3, 0, 3083, 3, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 4096, 1, 3567, 4082, 117, 0, 2, 2, 2, 242, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 3961, 4014, 0, 0, 31, 9, 0, 2, 1701, 3906, 0, 10, 74, 26, 514, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18700, 17378, 16333, 15295, 17016, 15394, 16204, 14752] avg=16384.0000 max=18700 max_rank=0 min=14752 min_rank=7 imbalance_ratio=1.141357 max_load_reduction=112372 max_load_reduction_pct=0.857330 moved_to_echo=63366 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1, 822, 1561, 7, 9, 0, 0, 3831, 0, 107, 2, 0, 4087, 3345, 2013, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 12, 4093, 2165, 0, 42, 163, 16, 2, 0, 7, 5, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 0, 916, 0, 0, 0, 59, 15, 11, 4, 53, 0, 0, 4096, 3826, 2083, 1648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 2, 4031, 0, 4094, 0, 0, 4065, 0, 0, 0, 1, 0, 2, 4091, 0, 998, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4074, 3997, 0, 0, 768, 663, 2595, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 85, 0, 0, 270, 0, 4096, 3924, 0, 0, 0, 8, 3975, 5, 3573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 4067, 0, 3580, 4095, 179, 0, 0, 0, 0, 519, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 2, 0, 54, 0, 0, 4, 4090, 4087, 0, 0, 5, 0, 516, 480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15889, 14644, 16801, 17611, 16225, 15947, 16545, 17410] avg=16384.0000 max=17611 max_rank=3 min=14644 min_rank=1 imbalance_ratio=1.074890 max_load_reduction=113461 max_load_reduction_pct=0.865639 moved_to_echo=66127 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 99, 1, 3, 0, 6, 4095, 0, 23, 0, 0, 4095, 4081, 2010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 227, 4048, 2164, 0, 0, 0, 0, 0, 6, 0, 0, 0, 14, 0, 3496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 4096, 2, 2, 1, 2, 0, 0, 0, 0, 0, 0, 0, 2, 4091, 3824, 2080, 1647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 4083, 2, 4090, 0, 0, 4094, 0, 1, 3, 0, 1, 0, 4069, 0, 1002, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 4083, 4, 4093, 4060, 0, 0, 4052, 0, 2608, 1, 14, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 1, 0, 270, 0, 4096, 4075, 0, 1, 3, 0, 4025, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4088, 4082, 0, 3574, 4091, 331, 0, 38, 5, 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4059, 0, 1, 2, 0, 49, 0, 4095, 4089, 0, 5, 0, 0, 515, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14413, 14044, 15805, 17685, 18931, 16587, 16217, 17390] avg=16384.0000 max=18931 max_rank=4 min=14044 min_rank=1 imbalance_ratio=1.155457 max_load_reduction=112141 max_load_reduction_pct=0.855568 moved_to_echo=69125 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 68, 1396, 0, 0, 0, 0, 3103, 1, 241, 0, 0, 3817, 3962, 2013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 4094, 2165, 68, 0, 2010, 0, 0, 0, 0, 0, 0, 4, 945, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 43, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3817, 2082, 1648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3696, 43, 4081, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 4051, 0, 1003, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4023, 30, 0, 975, 0, 2610, 0, 24, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 270, 0, 4096, 4096, 1, 0, 0, 0, 4037, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4043, 0, 3580, 4095, 238, 30, 0, 0, 0, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4080, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 516, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14601, 17456, 15838, 17259, 15855, 16597, 16096, 17370] avg=16384.0000 max=17456 max_rank=1 min=14601 min_rank=0 imbalance_ratio=1.065430 max_load_reduction=113616 max_load_reduction_pct=0.866821 moved_to_echo=65918 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 3967, 3780, 6, 17, 7, 0, 4096, 0, 73, 0, 0, 4095, 1814, 2012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 13, 4096, 2162, 0, 0, 12, 2, 0, 0, 0, 0, 5, 62, 145, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 6, 1641, 1, 0, 60, 169, 0, 0, 0, 0, 0, 0, 4094, 3826, 2082, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 12, 154, 4072, 0, 0, 4089, 0, 0, 1, 37, 6, 0, 4017, 13, 1002, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 1, 4095, 3943, 23, 1, 2798, 0, 2606, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 270, 1, 4087, 4084, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3939, 8, 3570, 3974, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2768, 4081, 11, 0, 0, 3, 10, 12, 3176, 4093, 0, 0, 0, 0, 515, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19869, 14666, 17607, 13979, 17577, 16632, 15590, 15152] avg=16384.0000 max=19869 max_rank=0 min=13979 min_rank=3 imbalance_ratio=1.212708 max_load_reduction=111203 max_load_reduction_pct=0.848412 moved_to_echo=64951 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 16256, 13184, 128, 128, 128, 128, 31104, 128, 512, 128, 128, 30976, 23040, 16128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32512, 896, 32768, 17408, 128, 256, 14464, 128, 128, 128, 128, 128, 128, 128, 1152, 31744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 384, 128, 256, 2176, 128, 32384, 32512, 128, 128, 128, 128, 31360, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 32768, 128, 32768, 29184, 128, 128, 13568, 768, 20864, 128, 1280, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[512, 128, 20480, 512, 32768, 128, 0, 32768, 128, 128, 128, 1280, 128, 256, 32512, 128, 8064, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 31232, 32512, 128, 128, 128, 128, 128, 128, 26368, 32512, 256, 128, 256, 128, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 32384, 128, 28672, 32640, 1152, 128, 128, 128, 128, 1280, 2304, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 32512, 128, 4608, 128, 128, 128, 512, 128, 128, 128, 128, 128, 128, 32768, 30592, 16640, 13184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1526, 0, 2981, 0, 4083, 6, 0, 4, 0, 0, 0, 0, 582, 785, 1, 0, 5, 0, 0, 4082, 0, 4096, 0, 178, 4094, 0, 1, 0, 0, 19, 0, 4096, 4052, 4095, 4096, 0, 2890, 0, 4077, 0, 2, 0, 1, 4096, 0, 1, 0, 0, 0, 3, 837, 36, 4090, 3985, 0, 0, 12, 26, 7, 0, 0, 3, 0, 0, 1, 4093, 0, 6, 1, 0, 6, 3621, 4096, 0, 4090, 2, 4062, 0, 4059, 3812, 0, 0, 0, 62, 0, 0, 4096, 1, 0, 0, 280, 0, 0, 0, 4090, 0, 0, 0, 3869, 4, 0, 0, 0, 0, 4094, 12, 0, 0, 1, 0, 0, 3850, 20, 0, 0, 0, 0, 0, 4093, 3524, 4096, 4096, 0, 0, 0, 0, 4091, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3006, 0, 3932, 0, 4034, 14, 0, 240, 4, 12, 0, 0, 2913, 60, 29, 0, 0, 0, 0, 4093, 5, 4064, 0, 0, 4090, 0, 1, 0, 0, 128, 0, 4096, 4096, 4036, 4093, 0, 4051, 0, 3923, 0, 36, 0, 5, 3849, 0, 0, 0, 0, 159, 0, 83, 60, 4093, 4040, 101, 0, 1, 0, 4, 0, 0, 0, 0, 0, 20, 4096, 0, 0, 2716, 0, 47, 1575, 4096, 12, 4096, 0, 3452, 0, 544, 1846, 33, 0, 0, 57, 0, 755, 4092, 6, 0, 0, 1859, 0, 3, 0, 4084, 4, 0, 0, 4093, 1, 0, 0, 0, 537, 3952, 0, 0, 0, 16, 0, 0, 3527, 0, 0, 0, 0, 0, 13, 4093, 1717, 4070, 4095, 20, 0, 0, 35, 4087, 4072] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[956, 0, 4086, 0, 4096, 26, 0, 0, 0, 0, 6, 11, 2442, 29, 33, 0, 10, 0, 0, 4019, 4, 4042, 0, 0, 4092, 2, 0, 13, 0, 18, 6, 4082, 4089, 3552, 4081, 0, 3113, 1, 4095, 11, 2, 9, 0, 4080, 9, 2, 0, 0, 20, 2, 549, 0, 4096, 4095, 775, 0, 0, 134, 1, 20, 4, 9, 0, 0, 0, 4073, 0, 143, 727, 30, 0, 2436, 4079, 0, 4088, 2, 2661, 0, 3214, 5, 52, 0, 13, 55, 0, 211, 4018, 0, 0, 0, 3592, 0, 0, 0, 4057, 21, 6, 0, 3969, 15, 4, 0, 0, 5, 4074, 0, 0, 17, 0, 35, 0, 4074, 140, 5, 2, 4, 0, 0, 4091, 4090, 4090, 4081, 0, 0, 0, 0, 4087, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1404, 0, 3816, 0, 4096, 0, 0, 70, 0, 5, 0, 1, 1927, 1481, 0, 0, 0, 0, 86, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 4096, 4012, 4091, 4096, 0, 4095, 0, 3011, 0, 0, 0, 0, 3670, 0, 0, 0, 0, 0, 0, 4061, 23, 4050, 4007, 67, 0, 0, 0, 546, 0, 0, 0, 21, 0, 0, 4096, 0, 0, 959, 0, 0, 4061, 4096, 0, 4095, 0, 4096, 0, 0, 57, 0, 0, 1, 0, 11, 65, 4026, 0, 0, 0, 1875, 0, 0, 0, 4096, 0, 0, 0, 4096, 8, 0, 9, 0, 36, 4096, 0, 0, 32, 61, 0, 0, 4057, 1, 0, 43, 0, 0, 0, 4096, 1599, 4096, 4096, 0, 0, 0, 0, 4096, 4089] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[4084, 0, 3983, 0, 4017, 0, 13, 9, 0, 0, 17, 50, 6, 0, 0, 8, 0, 8, 394, 4095, 5, 4030, 0, 36, 4070, 8, 0, 0, 0, 0, 7, 4063, 4095, 4088, 3992, 0, 3687, 0, 3743, 1, 0, 22, 7, 4086, 2, 11, 14, 13, 0, 0, 961, 8, 4035, 3765, 85, 35, 0, 0, 999, 6, 0, 6, 0, 5, 0, 4047, 0, 0, 671, 0, 23, 53, 4061, 0, 4049, 0, 4092, 0, 1360, 2495, 96, 0, 0, 0, 24, 786, 4047, 0, 60, 4, 2423, 34, 0, 64, 3963, 30, 24, 0, 4093, 0, 0, 27, 0, 0, 4075, 1, 0, 0, 3001, 1, 0, 4020, 137, 42, 0, 22, 0, 0, 4087, 554, 4078, 4038, 1, 0, 0, 0, 3878, 4042] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[3393, 8, 4095, 0, 4072, 0, 0, 288, 3, 4, 0, 1, 1467, 1, 7, 0, 0, 0, 175, 4096, 0, 4082, 19, 771, 4076, 0, 1, 0, 0, 9, 0, 4096, 3686, 3699, 4096, 0, 4095, 0, 3237, 0, 56, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3190, 0, 4051, 4095, 3, 0, 0, 12, 574, 0, 0, 0, 18, 0, 0, 4096, 0, 0, 221, 0, 0, 2634, 4091, 0, 3923, 6, 4037, 0, 31, 447, 606, 0, 0, 225, 492, 3, 4008, 0, 2, 0, 2143, 0, 54, 19, 4094, 0, 2, 0, 4082, 0, 1, 0, 0, 0, 4096, 0, 0, 107, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4028, 1581, 4096, 4094, 0, 0, 2, 5, 4087, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[4073, 4, 1097, 0, 4094, 60, 8, 289, 0, 0, 5, 0, 18, 2, 0, 0, 0, 3, 0, 4088, 4, 4093, 10, 3244, 4082, 0, 0, 1, 9, 0, 0, 4094, 4093, 4092, 4085, 87, 3982, 0, 4013, 0, 0, 12, 2, 4092, 0, 0, 0, 0, 0, 1, 4032, 0, 4085, 4084, 0, 0, 0, 0, 254, 0, 3, 0, 3, 2, 0, 3934, 0, 0, 0, 0, 11, 3975, 4061, 319, 4083, 10, 4093, 0, 136, 3226, 2, 0, 0, 0, 0, 20, 4053, 0, 0, 0, 0, 0, 0, 0, 4082, 1, 1, 0, 4086, 0, 2, 0, 4, 1, 4094, 0, 3, 4, 1, 0, 0, 3877, 0, 328, 0, 5, 0, 16, 4092, 17, 4096, 4057, 0, 0, 2, 0, 4093, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[253, 0, 3657, 0, 4096, 0, 0, 0, 0, 0, 12, 920, 3981, 0, 0, 0, 0, 0, 22, 4089, 0, 4094, 0, 26, 3998, 0, 1, 7, 22, 4, 20, 4057, 3764, 2815, 4071, 0, 3851, 6, 3844, 0, 0, 0, 0, 4085, 0, 0, 6, 0, 2, 0, 3212, 0, 4023, 3863, 0, 0, 0, 737, 0, 12, 2, 2, 7, 8, 0, 4083, 14, 46, 64, 0, 0, 2517, 4068, 71, 4088, 6, 1231, 0, 3743, 2138, 34, 6, 0, 344, 54, 0, 4080, 2, 23, 12, 1, 2, 6, 0, 4085, 5, 2, 0, 3924, 34, 0, 0, 0, 0, 4096, 0, 1, 154, 0, 0, 0, 4088, 0, 66, 0, 0, 0, 0, 4071, 4079, 4048, 4078, 0, 0, 0, 0, 4057, 4082] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1526, 0, 2981, 0, 4083, 6, 0, 4, 0, 0, 0, 0, 582, 785, 1, 0, 2985, 931, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4082, 0, 4096, 0, 178, 4094, 0, 1, 0, 0, 19, 0, 3501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 4095, 0, 0, 2890, 0, 4077, 0, 2, 0, 1, 1111, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 837, 36, 4090, 3985, 0, 0, 12, 26, 7, 0, 0, 3, 0, 0, 3658, 1975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2105, 0, 6, 1, 0, 6, 3621, 0, 0, 4090, 2, 4062, 0, 4059, 3812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 4096, 1, 0, 0, 280, 0, 0, 0, 4090, 0, 4096, 2121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3869, 4, 0, 0, 0, 0, 4094, 12, 0, 0, 1, 0, 0, 3850, 1988, 1250, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 1912, 3524, 0, 4096, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14322, 15976, 16229, 14632, 21765, 14746, 15663, 17739] avg=16384.0000 max=21765 max_rank=4 min=14322 min_rank=0 imbalance_ratio=1.328430 max_load_reduction=109307 max_load_reduction_pct=0.833946 moved_to_echo=69913 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[956, 0, 4086, 0, 4096, 26, 0, 0, 0, 0, 6, 11, 2442, 29, 33, 0, 2976, 933, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4019, 4, 4042, 0, 0, 4092, 2, 0, 13, 0, 18, 6, 3486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3552, 0, 0, 3113, 1, 4095, 11, 2, 9, 0, 1104, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2, 549, 0, 4096, 4095, 775, 0, 0, 134, 1, 20, 4, 9, 0, 0, 3644, 1968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2090, 0, 143, 727, 30, 0, 2436, 0, 0, 4088, 2, 2661, 0, 3214, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 13, 55, 0, 211, 4018, 0, 0, 0, 3592, 0, 0, 0, 4057, 21, 4090, 2111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3969, 15, 4, 0, 0, 5, 4074, 0, 0, 17, 0, 35, 0, 4074, 1983, 1252, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 5, 2, 4, 0, 0, 1906, 4090, 0, 4081, 0, 0, 0, 0, 4087, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16031, 15692, 15987, 15317, 15396, 18220, 16030, 18399] avg=16384.0000 max=18399 max_rank=7 min=15317 min_rank=3 imbalance_ratio=1.122986 max_load_reduction=112673 max_load_reduction_pct=0.859627 moved_to_echo=68045 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3393, 8, 4095, 0, 4072, 0, 0, 288, 3, 4, 0, 1, 1467, 1, 7, 0, 2985, 916, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 4096, 0, 4082, 19, 771, 4076, 0, 1, 0, 0, 9, 0, 3501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3686, 3699, 0, 0, 4095, 0, 3237, 0, 56, 0, 0, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3190, 0, 4051, 4095, 3, 0, 0, 12, 574, 0, 0, 0, 18, 0, 3657, 1974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2106, 0, 0, 221, 0, 0, 2634, 0, 0, 3923, 6, 4037, 0, 31, 447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 606, 0, 0, 225, 492, 3, 4008, 0, 2, 0, 2143, 0, 54, 19, 4094, 0, 4096, 2117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4082, 0, 1, 0, 0, 0, 4096, 0, 0, 107, 0, 0, 0, 4096, 1990, 1230, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1882, 1581, 0, 4094, 0, 0, 2, 5, 4087, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17679, 16730, 15884, 17574, 13405, 17859, 16199, 15742] avg=16384.0000 max=17859 max_rank=5 min=13405 min_rank=4 imbalance_ratio=1.090027 max_load_reduction=113213 max_load_reduction_pct=0.863747 moved_to_echo=63205 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3006, 0, 3932, 0, 4034, 14, 0, 240, 4, 12, 0, 0, 2913, 60, 29, 0, 2805, 931, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 4064, 0, 0, 4090, 0, 1, 0, 0, 128, 0, 3501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4036, 0, 0, 4051, 0, 3923, 0, 36, 0, 5, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 83, 60, 4093, 4040, 101, 0, 1, 0, 4, 0, 0, 0, 0, 0, 3656, 1975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2106, 0, 0, 2716, 0, 47, 1575, 0, 12, 4096, 0, 3452, 0, 544, 1846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 57, 0, 755, 4092, 6, 0, 0, 1859, 0, 3, 0, 4084, 4, 4070, 2121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 537, 3952, 0, 0, 0, 16, 0, 0, 3527, 1990, 1250, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1912, 1717, 0, 4095, 20, 0, 0, 35, 4087, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18417, 15882, 17191, 14172, 16414, 17084, 15961, 15951] avg=16384.0000 max=18417 max_rank=0 min=14172 min_rank=3 imbalance_ratio=1.124084 max_load_reduction=112655 max_load_reduction_pct=0.859489 moved_to_echo=65410 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4073, 4, 1097, 0, 4094, 60, 8, 289, 0, 0, 5, 0, 18, 2, 0, 0, 2982, 931, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4088, 4, 4093, 10, 3244, 4082, 0, 0, 1, 9, 0, 0, 3499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4092, 0, 87, 3982, 0, 4013, 0, 0, 12, 2, 1110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4032, 0, 4085, 4084, 0, 0, 0, 0, 254, 0, 3, 0, 3, 2, 3648, 1960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2023, 0, 0, 0, 0, 11, 3975, 0, 319, 4083, 10, 4093, 0, 136, 3226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 20, 4053, 0, 0, 0, 0, 0, 0, 0, 4082, 1, 4096, 2101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4086, 0, 2, 0, 4, 1, 4094, 0, 3, 4, 1, 0, 0, 3877, 1911, 1249, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 0, 5, 0, 16, 1912, 17, 0, 4057, 0, 0, 2, 0, 4093, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14000, 19033, 17391, 18072, 17876, 14355, 15828, 14517] avg=16384.0000 max=19033 max_rank=1 min=14000 min_rank=0 imbalance_ratio=1.161682 max_load_reduction=112039 max_load_reduction_pct=0.854790 moved_to_echo=62576 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 3983, 0, 4017, 0, 13, 9, 0, 0, 17, 50, 6, 0, 0, 8, 2978, 930, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 394, 4095, 5, 4030, 0, 36, 4070, 8, 0, 0, 0, 0, 7, 3473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4088, 0, 0, 3687, 0, 3743, 1, 0, 22, 7, 1108, 2, 11, 14, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 961, 8, 4035, 3765, 85, 35, 0, 0, 999, 6, 0, 6, 0, 5, 3566, 1959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2081, 0, 0, 671, 0, 23, 53, 0, 0, 4049, 0, 4092, 0, 1360, 2495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 24, 786, 4047, 0, 60, 4, 2423, 34, 0, 64, 3963, 30, 4078, 2102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 4093, 0, 0, 27, 0, 0, 4075, 1, 0, 0, 3001, 1, 0, 4020, 1966, 1248, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 42, 0, 22, 0, 0, 1909, 554, 0, 4038, 1, 0, 0, 0, 3878, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16521, 16126, 16791, 15430, 14824, 17711, 19046, 14623] avg=16384.0000 max=19046 max_rank=6 min=14623 min_rank=7 imbalance_ratio=1.162476 max_load_reduction=112026 max_load_reduction_pct=0.854691 moved_to_echo=66204 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1404, 0, 3816, 0, 4096, 0, 0, 70, 0, 5, 0, 1, 1927, 1481, 0, 0, 2674, 932, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 3501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 4091, 0, 0, 4095, 0, 3011, 0, 0, 0, 0, 996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 23, 4050, 4007, 67, 0, 0, 0, 546, 0, 0, 0, 21, 0, 3657, 1976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2106, 0, 0, 959, 0, 0, 4061, 0, 0, 4095, 0, 4096, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 65, 4026, 0, 0, 0, 1875, 0, 0, 0, 4096, 0, 4096, 2120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 9, 0, 36, 4096, 0, 0, 32, 61, 0, 0, 4057, 1990, 1251, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 43, 0, 0, 0, 1913, 1599, 0, 4096, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16845, 15882, 16205, 18408, 15374, 16290, 16231, 15837] avg=16384.0000 max=18408 max_rank=3 min=15374 min_rank=4 imbalance_ratio=1.123535 max_load_reduction=112664 max_load_reduction_pct=0.859558 moved_to_echo=63732 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[253, 0, 3657, 0, 4096, 0, 0, 0, 0, 0, 12, 920, 3981, 0, 0, 0, 2977, 926, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4089, 0, 4094, 0, 26, 3998, 0, 1, 7, 22, 4, 20, 3468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3764, 2815, 0, 0, 3851, 6, 3844, 0, 0, 0, 0, 1108, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3212, 0, 4023, 3863, 0, 0, 0, 737, 0, 12, 2, 2, 7, 8, 3636, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2100, 14, 46, 64, 0, 0, 2517, 0, 71, 4088, 6, 1231, 0, 3743, 2138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 6, 0, 344, 54, 0, 4080, 2, 23, 12, 1, 2, 6, 0, 4085, 5, 4048, 2105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3924, 34, 0, 0, 0, 0, 4096, 0, 1, 154, 0, 0, 0, 4088, 1983, 1243, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 1902, 4079, 0, 4078, 0, 0, 0, 0, 4057, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17257, 15751, 15394, 17467, 16018, 14807, 16114, 18264] avg=16384.0000 max=18264 max_rank=7 min=14807 min_rank=5 imbalance_ratio=1.114746 max_load_reduction=112808 max_load_reduction_pct=0.860657 moved_to_echo=65203 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[18816, 128, 27648, 0, 32640, 128, 128, 1024, 128, 128, 128, 1024, 13440, 2432, 128, 128, 23424, 7552, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 32256, 128, 128, 128, 128, 640, 32640, 128, 128, 384, 3200, 128, 0, 31616, 15872, 9984, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 16768, 128, 256, 5376, 128, 128, 20992, 0, 512, 32512, 128, 27776, 0, 13184, 14080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 512, 128, 128, 0, 128, 15360, 17280, 0, 32640, 128, 0, 128, 128, 32512, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[896, 128, 128, 768, 640, 1920, 32512, 128, 128, 128, 12288, 128, 128, 128, 32640, 128, 32768, 17024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 768, 32768, 128, 32640, 128, 4352, 32640, 128, 128, 128, 128, 256, 128, 28032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32000, 30592, 0, 128, 29824, 128, 29952, 128, 128, 128, 128, 8704, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[256, 128, 17024, 128, 32640, 32000, 1152, 128, 128, 1024, 2432, 128, 128, 128, 128, 128, 29184, 15872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 0, 3, 0, 4095, 0, 4095, 0, 0, 4092, 0, 4, 0, 0, 8, 2544, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4096, 157, 4093, 4096, 0, 4096, 0, 0, 4088, 752, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 10, 0, 4096, 0, 2, 0, 4090, 0, 4079, 4096, 0, 0, 3187, 0, 3113, 4, 0, 0, 0, 0, 2, 0, 4085, 0, 0, 4052, 0, 0, 0, 0, 1, 4093, 4094, 4, 0, 1, 0, 1, 1164, 0, 53, 0, 4092, 0, 4095, 1374, 0, 4096, 0, 0, 0, 0, 4088, 1, 4096, 17, 4096, 0, 4060, 0, 0, 4096, 0, 17, 0, 0, 33, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 4096, 1, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 11, 0, 4054, 38, 1, 0, 4090, 22, 4080, 0, 0, 4073, 0, 13, 0, 65, 90, 3617, 0, 0, 0, 0, 3794, 26, 9, 0, 0, 4077, 0, 4050, 4083, 0, 4087, 16, 0, 3978, 1579, 14, 35, 0, 73, 0, 0, 3891, 2737, 0, 0, 44, 4071, 0, 8, 48, 3351, 3, 3937, 4079, 70, 19, 3087, 6, 3085, 16, 3, 20, 20, 1, 0, 0, 984, 1, 0, 3501, 0, 1, 8, 0, 0, 2890, 4082, 0, 2, 6, 0, 2, 2, 0, 295, 0, 4079, 0, 4087, 3817, 2, 4045, 0, 0, 3, 1, 3668, 0, 4095, 1, 4087, 0, 4056, 6, 0, 4096, 11, 1, 0, 13, 2, 530, 0, 0, 12, 0, 4096, 0, 0, 0, 4017, 0, 0, 102, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 9, 4078, 0, 0, 0, 4071, 8, 4034, 214, 0, 4094, 1, 0, 787, 11, 8, 4092, 0, 0, 0, 0, 4096, 1, 0, 11, 0, 4085, 0, 4089, 4095, 6, 4085, 6, 7, 4085, 2929, 49, 3, 0, 26, 18, 1, 2206, 311, 11, 0, 0, 3999, 13, 5, 4, 4034, 0, 4046, 4095, 0, 0, 264, 10, 3699, 3429, 14, 0, 0, 5, 0, 0, 3797, 0, 0, 3981, 0, 46, 0, 2, 0, 1942, 4055, 0, 0, 0, 52, 0, 31, 0, 0, 291, 4091, 12, 4082, 1163, 0, 4050, 0, 0, 4, 0, 4092, 6, 4084, 0, 4081, 0, 4096, 0, 0, 3614, 0, 122, 0, 2, 5, 0, 0, 0, 0, 34, 4095, 0, 0, 0, 3934, 9, 155, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4072, 0, 3, 0, 4093, 0, 4076, 0, 0, 3843, 2, 0, 15, 5, 0, 3900, 0, 0, 0, 3, 4094, 0, 0, 0, 0, 4096, 12, 4093, 4091, 3, 4090, 0, 0, 4092, 683, 0, 0, 0, 0, 5, 0, 327, 99, 1, 19, 0, 4082, 0, 0, 0, 3938, 0, 4063, 4093, 0, 10, 0, 0, 3906, 3936, 0, 0, 0, 1, 3, 0, 500, 0, 0, 4004, 0, 6, 3, 0, 0, 3796, 4093, 2, 0, 3, 0, 1, 21, 0, 19, 24, 3940, 2, 4096, 3498, 478, 4091, 1, 0, 0, 0, 4096, 3, 4090, 50, 4073, 0, 4078, 0, 5, 4096, 0, 26, 0, 0, 275, 210, 0, 0, 0, 36, 4094, 0, 2, 0, 4096, 1, 3514, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[3, 2, 0, 4088, 2, 0, 10, 4076, 5, 4095, 0, 0, 4094, 1, 9, 0, 4, 0, 375, 0, 0, 2, 8, 4090, 0, 0, 0, 18, 4093, 0, 4033, 4083, 0, 4091, 0, 0, 4082, 4009, 0, 0, 1, 19, 6, 104, 0, 3928, 0, 0, 3, 4091, 0, 0, 0, 3273, 10, 4094, 3841, 4, 0, 3788, 2, 4077, 4076, 1257, 6, 0, 0, 0, 2, 3908, 0, 3, 4094, 4, 0, 0, 20, 1, 307, 4090, 0, 2, 0, 157, 2, 420, 5, 0, 8, 3871, 21, 4083, 0, 0, 4083, 0, 4, 44, 0, 4057, 0, 4061, 0, 4087, 1, 4089, 13, 0, 3170, 0, 0, 0, 0, 0, 0, 0, 1, 126, 51, 3718, 113, 0, 0, 4002, 7, 410, 161, 23] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2, 4092, 0, 4, 5, 4035, 0, 4088, 0, 0, 4090, 0, 0, 0, 0, 386, 4096, 0, 0, 6, 0, 3959, 0, 0, 0, 0, 4096, 590, 4079, 4096, 0, 4090, 0, 43, 4064, 3726, 0, 1, 0, 0, 0, 0, 1154, 1630, 0, 0, 0, 4096, 0, 0, 20, 854, 212, 3732, 3752, 0, 0, 752, 0, 1552, 3687, 0, 0, 0, 0, 39, 0, 1320, 0, 0, 4054, 8, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 2, 59, 0, 3, 4, 4077, 1, 4093, 3579, 2, 4096, 4, 0, 7, 0, 4096, 0, 4094, 124, 4096, 0, 4081, 0, 0, 4096, 0, 0, 0, 0, 259, 1, 0, 0, 0, 0, 4096, 3, 0, 0, 4095, 0, 1419, 81, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4, 0, 4096, 0, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 4096, 5, 4096, 4096, 0, 4066, 0, 0, 3822, 2, 0, 0, 0, 0, 13, 250, 3879, 551, 0, 0, 0, 4084, 0, 0, 0, 4085, 77, 4096, 4087, 0, 0, 3, 0, 4014, 3278, 0, 0, 0, 0, 396, 18, 979, 0, 1, 3208, 0, 0, 0, 5, 0, 4021, 4096, 4, 0, 0, 0, 0, 208, 0, 7, 0, 4096, 0, 4096, 0, 0, 4096, 63, 0, 0, 0, 4085, 0, 4096, 0, 4083, 0, 4096, 0, 2, 4096, 0, 10, 0, 0, 0, 1397, 0, 0, 0, 1, 2519, 736, 0, 0, 4090, 0, 3358, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[7, 0, 0, 4065, 1, 0, 37, 4090, 0, 4096, 54, 0, 4089, 6, 0, 61, 0, 0, 4011, 0, 0, 0, 23, 4096, 5, 0, 0, 4, 4082, 1, 4081, 4090, 0, 4086, 28, 0, 4032, 331, 8, 1, 0, 0, 0, 0, 4086, 443, 2, 0, 0, 3802, 2, 13, 0, 3763, 0, 4083, 4065, 0, 0, 10, 0, 4013, 2930, 22, 22, 15, 0, 0, 52, 3497, 0, 4, 4090, 0, 108, 12, 0, 5, 4022, 4053, 0, 0, 22, 78, 0, 136, 0, 0, 0, 4071, 0, 4096, 65, 0, 4088, 44, 0, 6, 7, 4096, 0, 4088, 0, 4075, 0, 4047, 9, 0, 4087, 0, 143, 0, 3, 0, 1, 7, 0, 0, 0, 3945, 5, 7, 0, 4065, 0, 1373, 0, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4092, 0, 3, 0, 4095, 0, 4009, 0, 0, 4092, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2544, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 4096, 157, 4093, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 752, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 10, 1895, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4090, 0, 1244, 4096, 0, 0, 3187, 0, 3113, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4085, 0, 0, 4052, 0, 0, 0, 0, 1, 4093, 4094, 1550, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 1, 1164, 0, 53, 0, 4092, 0, 4095, 1374, 0, 4096, 0, 1964, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 191, 17, 4096, 0, 4060, 0, 0, 4096, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 4096, 1, 0, 1, 0, 4096, 2658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16295, 15652, 15807, 15736, 19124, 16930, 16566, 14962] avg=16384.0000 max=19124 max_rank=4 min=14962 min_rank=7 imbalance_ratio=1.167236 max_load_reduction=111948 max_load_reduction_pct=0.854095 moved_to_echo=67582 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 4078, 0, 0, 0, 4071, 8, 3947, 214, 0, 4094, 1, 0, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 8, 4092, 0, 0, 0, 0, 4096, 1, 0, 11, 0, 4085, 0, 4089, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4085, 6, 7, 4085, 2929, 49, 3, 0, 26, 18, 1, 2206, 311, 11, 0, 1899, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 5, 4, 4034, 0, 1227, 4095, 0, 0, 264, 10, 3699, 3429, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3797, 0, 0, 3981, 0, 46, 0, 2, 0, 1942, 4055, 1553, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 31, 0, 0, 291, 4091, 12, 4082, 1163, 0, 4050, 0, 1952, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4092, 6, 184, 0, 4081, 0, 4096, 0, 0, 3614, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 34, 4095, 0, 0, 0, 3934, 9, 155, 0, 0, 3999, 2653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17209, 17036, 16509, 16794, 16628, 15811, 16199, 14886] avg=16384.0000 max=17209 max_rank=0 min=14886 min_rank=7 imbalance_ratio=1.050354 max_load_reduction=113863 max_load_reduction_pct=0.868706 moved_to_echo=63524 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 0, 4054, 38, 1, 0, 4090, 22, 3995, 0, 0, 4073, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 90, 3617, 0, 0, 0, 0, 3794, 26, 9, 0, 0, 4077, 0, 4050, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 16, 0, 3978, 1579, 14, 35, 0, 73, 0, 0, 3891, 2737, 0, 0, 1889, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 8, 48, 3351, 3, 1200, 4079, 70, 19, 3087, 6, 3085, 16, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 20, 1, 0, 0, 984, 1, 0, 3501, 0, 1, 8, 0, 0, 2890, 4082, 1545, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 2, 2, 0, 295, 0, 4079, 0, 4087, 3817, 2, 4045, 0, 1896, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3668, 0, 191, 1, 4087, 0, 4056, 6, 0, 4096, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2, 530, 0, 0, 12, 0, 4096, 0, 0, 0, 4017, 0, 0, 102, 0, 4071, 2657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16297, 16377, 19140, 15019, 14300, 18318, 16121, 15500] avg=16384.0000 max=19140 max_rank=2 min=14300 min_rank=4 imbalance_ratio=1.168213 max_load_reduction=111932 max_load_reduction_pct=0.853973 moved_to_echo=64239 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 2, 0, 4088, 2, 0, 10, 4076, 5, 4009, 0, 0, 4094, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 375, 0, 0, 2, 8, 4090, 0, 0, 0, 18, 4093, 0, 4033, 649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4082, 4009, 0, 0, 1, 19, 6, 104, 0, 3928, 0, 0, 1889, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3273, 10, 1248, 3841, 4, 0, 3788, 2, 4077, 4076, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2, 3908, 0, 3, 4094, 4, 0, 0, 20, 1, 307, 4090, 1545, 1237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 157, 2, 420, 5, 0, 8, 3871, 21, 4083, 0, 0, 4083, 0, 1972, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 44, 0, 4057, 0, 189, 0, 4087, 1, 4089, 13, 0, 3170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 126, 51, 3718, 113, 0, 0, 4002, 7, 410, 161, 23, 4091, 2635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16299, 13272, 19003, 21579, 15217, 14710, 15654, 15338] avg=16384.0000 max=21579 max_rank=3 min=13272 min_rank=1 imbalance_ratio=1.317078 max_load_reduction=109493 max_load_reduction_pct=0.835365 moved_to_echo=60919 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4072, 0, 3, 0, 4093, 0, 3991, 0, 0, 3843, 2, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3900, 0, 0, 0, 3, 4094, 0, 0, 0, 0, 4096, 12, 4093, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4090, 0, 0, 4092, 683, 0, 0, 0, 0, 5, 0, 327, 99, 1, 19, 1893, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3938, 0, 1239, 4093, 0, 10, 0, 0, 3906, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 500, 0, 0, 4004, 0, 6, 3, 0, 0, 3796, 4093, 1548, 1245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 1, 21, 0, 19, 24, 3940, 2, 4096, 3498, 478, 4091, 1, 1957, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 191, 50, 4073, 0, 4078, 0, 5, 4096, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 210, 0, 0, 0, 36, 4094, 0, 2, 0, 4096, 1, 3514, 0, 0, 4082, 2654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16019, 16853, 12079, 17122, 15199, 18218, 16618, 18964] avg=16384.0000 max=18964 max_rank=7 min=12079 min_rank=2 imbalance_ratio=1.157471 max_load_reduction=112108 max_load_reduction_pct=0.855316 moved_to_echo=68999 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 4092, 0, 4, 5, 4035, 0, 4003, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 386, 4096, 0, 0, 6, 0, 3959, 0, 0, 0, 0, 4096, 590, 4079, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 43, 4064, 3726, 0, 1, 0, 0, 0, 0, 1154, 1630, 0, 0, 1895, 797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 854, 212, 1138, 3752, 0, 0, 752, 0, 1552, 3687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 1320, 0, 0, 4054, 8, 0, 0, 0, 0, 4096, 4096, 1550, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 59, 0, 3, 4, 4077, 1, 4093, 3579, 2, 4096, 4, 1797, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 191, 124, 4096, 0, 4081, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 1, 0, 0, 0, 0, 4096, 3, 0, 0, 4095, 0, 1419, 81, 0, 4096, 2656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16231, 17863, 17400, 11967, 16410, 17804, 16691, 16706] avg=16384.0000 max=17863 max_rank=1 min=11967 min_rank=3 imbalance_ratio=1.090271 max_load_reduction=113209 max_load_reduction_pct=0.863716 moved_to_echo=67611 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4010, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 4096, 5, 4096, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 3822, 2, 0, 0, 0, 0, 13, 250, 3879, 551, 0, 0, 1895, 875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 77, 1248, 4087, 0, 0, 3, 0, 4014, 3278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 18, 979, 0, 1, 3208, 0, 0, 0, 5, 0, 4021, 4096, 1550, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 208, 0, 7, 0, 4096, 0, 4096, 0, 0, 4096, 63, 1973, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 191, 0, 4083, 0, 4096, 0, 2, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1397, 0, 0, 0, 1, 2519, 736, 0, 0, 4090, 0, 3358, 3, 0, 4084, 2658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16302, 17066, 15353, 16792, 15521, 14629, 16563, 18846] avg=16384.0000 max=18846 max_rank=7 min=14629 min_rank=5 imbalance_ratio=1.150269 max_load_reduction=112226 max_load_reduction_pct=0.856216 moved_to_echo=65559 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 4065, 1, 0, 37, 4090, 0, 4010, 54, 0, 4089, 6, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 0, 23, 4096, 5, 0, 0, 4, 4082, 1, 4081, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 28, 0, 4032, 331, 8, 1, 0, 0, 0, 0, 4086, 443, 2, 0, 1892, 872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 13, 0, 3763, 0, 1245, 4065, 0, 0, 10, 0, 4013, 2930, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 15, 0, 0, 52, 3497, 0, 4, 4090, 0, 108, 12, 0, 5, 4022, 4053, 1548, 1245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 78, 0, 136, 0, 0, 0, 4071, 0, 4096, 65, 0, 4088, 44, 1966, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 4096, 0, 190, 0, 4075, 0, 4047, 9, 0, 4087, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 7, 0, 0, 0, 3945, 5, 7, 0, 4065, 0, 1373, 0, 9, 3802, 2653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16420, 16953, 15781, 16063, 18673, 14652, 16660, 15870] avg=16384.0000 max=18673 max_rank=4 min=14652 min_rank=5 imbalance_ratio=1.139709 max_load_reduction=112399 max_load_reduction_pct=0.857536 moved_to_echo=65855 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 32640, 128, 128, 128, 32768, 128, 32000, 384, 0, 32512, 128, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 512, 26752, 0, 0, 128, 128, 32384, 128, 128, 128, 128, 32768, 768, 32640, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 32384, 128, 1536, 256, 32768, 128, 32640, 128, 128, 31360, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 128, 27392, 384, 9856, 32128, 128, 128, 11136, 128, 27520, 21376, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 640, 2176, 128, 128, 256, 128, 30720, 896, 128, 0, 32512, 128, 10240, 384, 128, 32384, 21248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 128, 32256, 14080, 128, 128, 128, 128, 128, 384, 19712, 9728, 128, 128, 15232, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 512, 128, 19072, 128, 128, 31104, 128, 256, 128, 128, 128, 25216, 32768, 12416, 9984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 384, 128, 2048, 128, 384, 384, 32384, 128, 32768, 13568, 512, 32768, 128, 15488, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 1, 4091, 2, 0, 0, 4091, 0, 0, 201, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 2, 4090, 7, 0, 4094, 0, 0, 4096, 0, 15, 0, 4077, 2, 1, 0, 23, 0, 429, 9, 4088, 0, 0, 4096, 0, 4096, 0, 0, 4095, 0, 0, 3891, 0, 4089, 4096, 4096, 0, 4094, 3086, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3, 3971, 5, 0, 0, 3869, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4069, 0, 1884, 4094, 0, 0, 4083, 0, 301, 4084, 0, 4090, 0, 0, 108, 0, 4093, 1, 4095, 0, 2, 0, 0, 0, 0, 0, 2748, 0, 0, 4091, 4, 4091, 0, 0, 12, 29, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4093, 12, 0, 4096, 0, 0, 0, 3731, 0, 8, 368, 0, 0, 11, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 27, 0, 4084, 0, 128, 3361, 0, 0, 4096, 1660, 0, 0, 4093, 0, 0, 3, 0, 0, 168, 30, 4064, 3, 0, 4089, 0, 4092, 0, 0, 4096, 0, 1892, 4075, 1, 3861, 4096, 4053, 8, 4096, 3723, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 4092, 0, 4, 0, 1722, 0, 0, 0, 0, 0, 0, 4096, 612, 0, 0, 0, 4096, 0, 1, 4093, 0, 1, 4096, 0, 0, 4096, 5, 4096, 0, 0, 0, 0, 4086, 44, 4090, 0, 0, 645, 0, 19, 3, 0, 2679, 0, 1, 4079, 0, 4091, 6, 3, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 63, 0, 4084, 0, 0, 0, 2398, 3, 0, 969, 0, 1, 4, 0, 0, 2, 4096, 14, 483, 0, 0, 0, 112, 0, 3999, 0, 0, 4024, 0, 0, 4096, 169, 0, 0, 4096, 7, 0, 1, 0, 0, 4046, 3, 4074, 0, 10, 4096, 0, 4093, 0, 2, 3428, 7, 1789, 3739, 0, 3858, 4096, 4096, 0, 4096, 4033, 0, 0, 4092, 1, 1, 0, 0, 0, 0, 1, 4055, 1, 0, 0, 3569, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 3420, 0, 25, 4094, 0, 0, 4087, 1, 0, 4090, 0, 3458, 5, 0, 8, 0, 4091, 0, 4087, 0, 0, 19, 0, 0, 0, 0, 1225, 0, 0, 4095, 0, 4096, 0, 0, 269, 0, 0, 1, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4050, 0, 0, 4095, 0, 0, 0, 4096, 9, 0, 3856, 0, 0, 3, 42, 0, 0, 4066, 2, 0, 0, 0, 0, 939, 4, 4086, 17, 0, 4096, 0, 2, 4089, 36, 81, 3, 4090, 0, 1, 0, 12, 0, 274, 0, 4045, 0, 11, 4071, 0, 4015, 25, 37, 4075, 2, 0, 3518, 0, 4092, 4088, 4064, 0, 4096, 3102, 0, 5, 4071, 2, 0, 0, 6, 0, 0, 0, 4072, 0, 0, 0, 650, 24, 0, 3, 0, 0, 6, 4051, 84, 3, 13, 4, 4066, 6, 1, 4090, 0, 0, 4047, 0, 1271, 4096, 29, 3979, 21, 48, 17, 0, 3973, 0, 4085, 0, 0, 0, 0, 0, 1, 0, 2188, 0, 0, 4096, 2, 4093, 14, 0, 0, 765, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 198, 0, 4096, 0, 0, 0, 1388, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4016, 0, 113, 4033, 0, 0, 4096, 190, 0, 0, 4096, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 4096, 0, 4096, 0, 1, 4095, 0, 3395, 4096, 0, 3995, 4096, 4096, 0, 4095, 3999, 0, 0, 4096, 0, 17, 0, 0, 0, 0, 0, 2078, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 4096, 0, 2714, 4096, 0, 0, 4096, 0, 6, 4094, 0, 3968, 0, 0, 35, 0, 4096, 90, 4096, 0, 187, 0, 0, 20, 0, 0, 1410, 0, 0, 4096, 0, 4096, 0, 0, 1, 909, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4095, 0, 0, 3831, 0, 1, 0, 3144, 51, 45, 1718, 13, 0, 98, 26, 0, 0, 4037, 20, 0, 23, 0, 0, 197, 0, 4040, 56, 0, 3354, 0, 0, 4084, 0, 0, 6, 4014, 0, 10, 0, 72, 12, 1975, 1, 3911, 0, 0, 4080, 1, 4037, 17, 35, 4007, 57, 1380, 4045, 25, 2375, 4052, 3956, 0, 4083, 4087, 1, 14, 4079, 23, 0, 1, 6, 67, 17, 0, 1311, 0, 18, 1, 4036, 0, 0, 0, 0, 0, 0, 4082, 601, 6, 1, 0, 4078, 0, 107, 4076, 0, 1, 4020, 0, 0, 4093, 0, 4088, 0, 0, 1, 0, 4023, 64, 4050, 1, 0, 128, 8, 0, 50, 16, 3774, 0, 0, 4044, 0, 4070, 0, 2, 0, 1021, 0, 22, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4089, 0, 6, 1877, 0, 0, 0, 705, 0, 0, 0, 1, 0, 0, 0, 359, 0, 4090, 281, 1509, 0, 0, 0, 4, 2, 4089, 0, 0, 4096, 1, 0, 4091, 711, 51, 10, 4082, 0, 5, 1, 0, 5, 0, 13, 4065, 0, 0, 4092, 12, 4095, 14, 32, 3089, 0, 4091, 4082, 0, 4091, 4089, 4073, 2, 4093, 3875, 0, 2, 4094, 0, 0, 0, 5, 21, 2, 0, 74, 0, 0, 0, 880, 0, 4, 0, 0, 0, 1, 2733, 23, 0, 0, 0, 4045, 4, 3365, 4088, 5, 0, 4090, 0, 4081, 4062, 0, 3736, 0, 3, 71, 0, 4085, 0, 4094, 0, 33, 2636, 0, 0, 1, 1, 6, 0, 0, 4089, 2, 4082, 0, 1, 2, 2741, 0, 27, 2, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4090, 0, 0, 4076, 0, 11, 0, 4058, 0, 0, 20, 10, 0, 52, 17, 21, 0, 4047, 6, 0, 0, 0, 0, 6, 0, 4083, 0, 0, 4059, 0, 0, 4080, 0, 0, 0, 4058, 0, 6, 10, 0, 13, 3779, 0, 4056, 0, 0, 4094, 84, 4084, 0, 88, 4076, 0, 33, 3379, 23, 4072, 4076, 4042, 6, 4088, 312, 1, 0, 4081, 16, 39, 0, 0, 40, 7, 0, 4065, 0, 6, 13, 4079, 0, 1, 1, 11, 5, 0, 4011, 28, 0, 23, 0, 4009, 17, 209, 4083, 0, 0, 4086, 0, 0, 4047, 22, 4018, 0, 40, 0, 0, 4093, 0, 4090, 8, 0, 0, 14, 32, 6, 0, 3816, 0, 60, 4096, 0, 4087, 0, 0, 0, 743, 0, 52, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 1, 4091, 2, 0, 0, 4091, 0, 0, 201, 0, 0, 2, 0, 0, 3870, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 2, 4090, 7, 0, 4094, 0, 0, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4077, 2, 1, 0, 23, 0, 429, 9, 4088, 0, 0, 4096, 0, 1036, 891, 231, 226, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 3891, 0, 4089, 0, 0, 0, 0, 3086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3, 3971, 5, 0, 0, 3869, 0, 0, 0, 4094, 2337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4069, 0, 1884, 3203, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 301, 4084, 0, 4090, 0, 0, 108, 0, 4093, 1, 3059, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2748, 0, 0, 4091, 4, 4091, 0, 0, 12, 29, 0, 1, 0, 0, 3394, 1759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17056, 16157, 15170, 15115, 18372, 17335, 15738, 16129] avg=16384.0000 max=18372 max_rank=4 min=15115 min_rank=3 imbalance_ratio=1.121338 max_load_reduction=112700 max_load_reduction_pct=0.859833 moved_to_echo=67574 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 63, 0, 4084, 0, 0, 0, 2398, 3, 0, 969, 0, 1, 4, 0, 0, 3871, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 14, 483, 0, 0, 0, 112, 0, 3999, 0, 0, 4024, 0, 0, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 4096, 7, 0, 1, 0, 0, 4046, 3, 4074, 0, 10, 4096, 0, 1034, 891, 231, 225, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3390, 7, 1789, 3739, 0, 3858, 0, 0, 0, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1, 0, 0, 0, 0, 1, 4055, 1, 0, 0, 3569, 0, 0, 0, 4096, 2338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 3420, 0, 25, 3203, 0, 0, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3458, 5, 0, 8, 0, 4091, 0, 3053, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1225, 0, 0, 4095, 0, 4096, 0, 0, 269, 0, 0, 1, 0, 1, 3392, 1758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16190, 16595, 18921, 16818, 18154, 14833, 14724, 14837] avg=16384.0000 max=18921 max_rank=2 min=14724 min_rank=6 imbalance_ratio=1.154846 max_load_reduction=112151 max_load_reduction_pct=0.855644 moved_to_echo=62548 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4093, 12, 0, 4096, 0, 0, 0, 3731, 0, 8, 368, 0, 0, 11, 0, 0, 3830, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 27, 0, 4084, 0, 128, 3361, 0, 0, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1660, 0, 0, 4093, 0, 0, 3, 0, 0, 168, 30, 4064, 3, 0, 4089, 0, 1035, 891, 231, 223, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 1892, 4075, 1, 3861, 0, 0, 8, 0, 3723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 4092, 0, 4, 0, 1722, 0, 0, 0, 4096, 2337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 612, 0, 0, 0, 4096, 0, 1, 3202, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 4096, 0, 0, 0, 0, 4086, 44, 3055, 0, 0, 645, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2679, 0, 1, 4079, 0, 4091, 6, 3, 0, 0, 0, 0, 0, 0, 3391, 1759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16850, 15564, 16536, 17610, 16350, 16104, 16046, 16012] avg=16384.0000 max=17610 max_rank=3 min=15564 min_rank=1 imbalance_ratio=1.074829 max_load_reduction=113462 max_load_reduction_pct=0.865646 moved_to_echo=64512 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4050, 0, 0, 4095, 0, 0, 0, 4096, 9, 0, 3856, 0, 0, 3, 42, 0, 3841, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 2, 0, 0, 0, 0, 939, 4, 4086, 17, 0, 4096, 0, 2, 3855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 81, 3, 4090, 0, 1, 0, 12, 0, 274, 0, 4045, 0, 11, 4071, 0, 1036, 895, 234, 223, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 37, 4026, 2, 0, 3518, 0, 4092, 0, 0, 0, 0, 3102, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 2, 0, 0, 6, 0, 0, 0, 4072, 0, 0, 0, 650, 24, 0, 3, 4096, 2333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4051, 84, 3, 13, 4, 4066, 6, 1, 3195, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1271, 4096, 29, 3979, 21, 48, 17, 0, 3973, 0, 3049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2188, 0, 0, 4096, 2, 4093, 14, 0, 0, 765, 0, 0, 0, 0, 3326, 1755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20681, 17067, 15061, 14807, 15257, 15476, 16483, 16240] avg=16384.0000 max=20681 max_rank=0 min=14807 min_rank=3 imbalance_ratio=1.262268 max_load_reduction=110391 max_load_reduction_pct=0.842216 moved_to_echo=63456 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4089, 0, 6, 1877, 0, 0, 0, 705, 0, 0, 0, 1, 0, 0, 0, 359, 3850, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 281, 1509, 0, 0, 0, 4, 2, 4089, 0, 0, 4096, 1, 0, 3860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 711, 51, 10, 4082, 0, 5, 1, 0, 5, 0, 13, 4065, 0, 0, 4092, 12, 1036, 890, 231, 223, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 32, 3055, 0, 4091, 4082, 0, 4091, 0, 0, 2, 0, 3875, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 5, 21, 2, 0, 74, 0, 0, 0, 880, 0, 4, 0, 4093, 2334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2733, 23, 0, 0, 0, 4045, 4, 3365, 3198, 5, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4062, 0, 3736, 0, 3, 71, 0, 4085, 0, 3058, 0, 33, 2636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 6, 0, 0, 4089, 2, 4082, 0, 1, 2, 2741, 0, 27, 2, 8, 3393, 1755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11589, 17932, 15461, 19244, 11507, 17464, 21765, 16110] avg=16384.0000 max=21765 max_rank=6 min=11507 min_rank=4 imbalance_ratio=1.328430 max_load_reduction=109307 max_load_reduction_pct=0.833946 moved_to_echo=66846 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 3831, 0, 1, 0, 3144, 51, 45, 1718, 13, 0, 98, 26, 0, 3738, 692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 20, 0, 23, 0, 0, 197, 0, 4040, 56, 0, 3354, 0, 0, 3854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4014, 0, 10, 0, 72, 12, 1975, 1, 3911, 0, 0, 4080, 1, 1025, 887, 230, 218, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 35, 3962, 57, 1380, 4045, 25, 2375, 0, 0, 0, 0, 4087, 1, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 23, 0, 1, 6, 67, 17, 0, 1311, 0, 18, 1, 4036, 0, 0, 0, 4083, 2312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 601, 6, 1, 0, 4078, 0, 107, 3189, 0, 1, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4088, 0, 0, 1, 0, 4023, 64, 3025, 1, 0, 128, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 16, 3774, 0, 0, 4044, 0, 4070, 0, 2, 0, 1021, 0, 22, 0, 0, 3345, 1740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17452, 15581, 16487, 15998, 15954, 16085, 15431, 18084] avg=16384.0000 max=18084 max_rank=7 min=15431 min_rank=6 imbalance_ratio=1.103760 max_load_reduction=112988 max_load_reduction_pct=0.862030 moved_to_echo=65554 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 198, 0, 4096, 0, 0, 0, 1388, 0, 0, 0, 0, 0, 0, 0, 27, 3871, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4016, 0, 113, 4033, 0, 0, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 0, 0, 4096, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 4096, 0, 1036, 892, 231, 225, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4049, 0, 3395, 4096, 0, 3995, 0, 0, 0, 0, 3999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 17, 0, 0, 0, 0, 0, 2078, 0, 0, 0, 4093, 0, 0, 0, 4095, 2338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 4096, 0, 2714, 3204, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4094, 0, 3968, 0, 0, 35, 0, 4096, 90, 3060, 0, 187, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1410, 0, 0, 4096, 0, 4096, 0, 0, 1, 909, 0, 0, 0, 0, 3393, 1758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14379, 16125, 14915, 19535, 16717, 18182, 15556, 15663] avg=16384.0000 max=19535 max_rank=3 min=14379 min_rank=0 imbalance_ratio=1.192322 max_load_reduction=111537 max_load_reduction_pct=0.850960 moved_to_echo=66118 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 4076, 0, 11, 0, 4058, 0, 0, 20, 10, 0, 52, 17, 21, 3820, 700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 6, 0, 0, 0, 0, 6, 0, 4083, 0, 0, 4059, 0, 0, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 6, 10, 0, 13, 3779, 0, 4056, 0, 0, 4094, 84, 1035, 889, 230, 222, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 4031, 0, 33, 3379, 23, 4072, 0, 0, 6, 0, 312, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 16, 39, 0, 0, 40, 7, 0, 4065, 0, 6, 13, 4079, 0, 1, 1, 4088, 2325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 5, 0, 4011, 28, 0, 23, 0, 4009, 17, 209, 3194, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 22, 4018, 0, 40, 0, 0, 4093, 0, 3055, 8, 0, 0, 14, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3816, 0, 60, 4096, 0, 4087, 0, 0, 0, 743, 0, 52, 0, 2, 3384, 1751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16875, 16051, 18521, 11945, 18761, 15593, 15329, 17997] avg=16384.0000 max=18761 max_rank=4 min=11945 min_rank=3 imbalance_ratio=1.145081 max_load_reduction=112311 max_load_reduction_pct=0.856865 moved_to_echo=67680 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32640, 384, 2048, 128, 128, 0, 1408, 128, 32512, 128, 256, 31232, 128, 128, 30976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 17920, 0, 128, 32768, 128, 32768, 128, 128, 384, 6272, 0, 128, 128, 128, 27136, 14080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 31360, 1408, 128, 128, 128, 32000, 128, 8320, 25600, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 384, 128, 30336, 128, 128, 0, 23680, 128, 128, 7168, 128, 128, 256, 128, 512, 30720, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[5760, 32768, 128, 31488, 128, 128, 256, 0, 32640, 256, 24448, 128, 256, 3456, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 256, 30720, 128, 12672, 30848, 128, 30464, 0, 0, 128, 0, 26240, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 128, 23808, 128, 128, 128, 22912, 128, 128, 128, 32768, 18688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[2816, 256, 128, 32640, 128, 128, 128, 128, 128, 10752, 128, 32512, 128, 128, 32768, 128, 8320, 7168, 1920, 1792, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 17, 0, 4087, 0, 859, 755, 0, 4082, 0, 4095, 390, 0, 0, 1, 1646, 0, 240, 4096, 0, 0, 0, 0, 0, 4095, 0, 3148, 0, 0, 0, 3458, 1145, 0, 4096, 4081, 0, 0, 0, 3479, 4095, 4096, 0, 1551, 0, 0, 0, 0, 1, 0, 4096, 4044, 4096, 0, 0, 0, 9, 0, 0, 0, 2, 0, 0, 0, 4081, 1, 0, 0, 4096, 4093, 4094, 4084, 0, 0, 0, 68, 14, 4092, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 2, 13, 0, 1, 3823, 0, 4096, 16, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 4096, 0, 4060, 4096, 4, 0, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 2074, 2362, 0, 4096, 90, 4096, 721, 0, 0, 0, 23, 0, 0, 3866, 0, 0, 791, 0, 1, 3989, 0, 3742, 0, 0, 0, 59, 2027, 0, 4041, 4094, 0, 0, 0, 4086, 4094, 4053, 0, 2238, 0, 0, 0, 0, 42, 86, 3488, 4096, 4096, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4039, 4096, 0, 0, 0, 0, 0, 4063, 0, 9, 0, 4096, 0, 0, 3937, 0, 0, 0, 0, 0, 0, 2902, 0, 4096, 0, 0, 0, 4096, 0, 111, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 289, 0, 0, 0, 40] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4074, 0, 1955, 1060, 0, 4068, 0, 4096, 1010, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 26, 0, 261, 4096, 0, 3362, 5, 3, 0, 2296, 706, 0, 4088, 4084, 5, 0, 0, 2869, 4096, 4094, 18, 4094, 0, 0, 2, 0, 275, 0, 4096, 3982, 4092, 12, 0, 4, 0, 0, 0, 13, 8, 4, 0, 0, 4080, 0, 0, 4, 4071, 4090, 4065, 4096, 0, 0, 46, 0, 0, 4096, 14, 30, 0, 4093, 3, 0, 3979, 0, 1, 0, 0, 3, 58, 2782, 0, 4083, 0, 0, 4, 4042, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4068, 4088, 0, 4096, 4095, 0, 0, 30, 0, 3, 0, 0, 0, 0, 11, 1, 1, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[10, 0, 0, 4038, 16, 1, 0, 0, 4069, 0, 4090, 0, 22, 0, 0, 33, 3, 0, 3985, 8, 23, 0, 1, 93, 4053, 0, 4024, 5, 0, 0, 4091, 4, 15, 4080, 4080, 0, 0, 0, 3976, 4085, 4095, 1, 326, 38, 0, 0, 0, 3767, 0, 4062, 4054, 4076, 0, 67, 31, 0, 98, 5, 0, 10, 1, 0, 0, 4086, 0, 0, 2, 4055, 4089, 4042, 4080, 30, 4, 156, 92, 0, 4076, 0, 0, 3, 4058, 15, 14, 3710, 2, 14, 46, 0, 0, 786, 4068, 1, 4007, 0, 0, 0, 4014, 7, 0, 26, 0, 4082, 9, 0, 6, 1, 4, 3307, 4041, 4, 4081, 4074, 1, 1, 0, 1, 426, 0, 1, 2, 0, 0, 28, 3, 0, 32, 48, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[2, 0, 1, 4081, 0, 4051, 2445, 0, 4092, 0, 4092, 0, 4, 2, 0, 15, 0, 0, 4055, 0, 3, 15, 0, 2, 4091, 0, 3685, 0, 0, 2, 0, 1, 0, 4049, 4039, 0, 2, 0, 4082, 3827, 4094, 0, 1361, 4, 11, 0, 0, 23, 630, 3443, 4094, 4093, 0, 0, 0, 0, 0, 6, 0, 0, 0, 477, 1, 4073, 36, 0, 0, 4093, 4094, 4084, 4094, 16, 2, 0, 0, 233, 3973, 0, 0, 347, 4091, 0, 4, 4094, 5, 0, 0, 0, 0, 0, 4042, 0, 4087, 0, 9, 8, 4094, 0, 102, 2, 0, 4094, 3, 0, 2, 1, 7, 3998, 4093, 10, 4088, 4091, 58, 3, 0, 4, 0, 59, 0, 8, 0, 4, 2, 0, 3, 0, 0, 86] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[7, 1, 0, 4092, 8, 1812, 16, 0, 4070, 86, 4074, 3116, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 122, 4096, 0, 26, 0, 0, 24, 1031, 2519, 3, 4096, 4040, 0, 0, 15, 3684, 4091, 4096, 14, 1652, 0, 0, 2, 0, 167, 0, 4096, 3955, 4083, 0, 33, 0, 11, 242, 0, 82, 0, 0, 57, 0, 4083, 1, 14, 0, 4080, 4074, 3946, 4093, 0, 0, 0, 0, 0, 4096, 3957, 160, 0, 4096, 0, 0, 3980, 2, 11, 0, 126, 0, 0, 1861, 0, 4095, 0, 0, 0, 4092, 0, 0, 0, 0, 4086, 0, 0, 1, 0, 27, 3973, 4084, 3, 4081, 4080, 3, 1, 93, 0, 221, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[3, 0, 0, 4090, 0, 4096, 71, 0, 4093, 0, 4095, 0, 0, 0, 0, 894, 0, 0, 4094, 0, 0, 25, 0, 3, 4085, 0, 3555, 0, 8, 4, 0, 461, 0, 4096, 4096, 0, 0, 0, 4096, 4090, 4096, 0, 3778, 0, 0, 2, 0, 0, 0, 3538, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 4096, 4080, 4096, 4096, 168, 1, 0, 0, 363, 4081, 0, 2, 373, 4096, 0, 12, 3845, 0, 0, 0, 0, 0, 0, 3069, 0, 4092, 35, 0, 0, 4079, 1, 192, 0, 0, 4090, 0, 3, 0, 0, 73, 4032, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 7, 16, 0, 0, 0, 9, 119, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[2, 0, 0, 4084, 0, 4048, 1099, 0, 4075, 2, 4060, 0, 9, 26, 0, 538, 0, 0, 4036, 1, 9, 0, 0, 0, 4051, 148, 2100, 43, 0, 0, 36, 0, 0, 4072, 4077, 0, 0, 11, 4084, 4089, 4070, 0, 3890, 7, 0, 0, 0, 1986, 26, 4012, 4096, 4096, 45, 0, 0, 0, 0, 33, 0, 2, 0, 2, 7, 4083, 10, 0, 0, 4094, 4094, 4096, 4090, 0, 0, 0, 0, 16, 4042, 3007, 50, 2, 4074, 0, 0, 3992, 0, 0, 0, 18, 0, 22, 1091, 0, 4074, 0, 20, 29, 3901, 1, 8, 12, 0, 4082, 0, 0, 0, 0, 126, 2633, 4094, 0, 4091, 4078, 0, 1, 0, 0, 0, 3, 0, 19, 1, 0, 16, 3, 17, 0, 22, 88] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 17, 0, 4087, 0, 859, 755, 0, 4082, 0, 3806, 390, 0, 0, 1, 1646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 4096, 0, 0, 0, 0, 0, 4095, 0, 3148, 0, 0, 0, 3458, 1145, 2700, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1302, 4081, 0, 0, 0, 3479, 4095, 0, 0, 1551, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4044, 4096, 0, 0, 0, 9, 0, 0, 0, 2, 0, 0, 0, 4081, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 3026, 4094, 0, 0, 0, 0, 68, 14, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 2, 13, 0, 1, 3823, 0, 4096, 16, 0, 0, 902, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 4060, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 4096, 3801, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15643, 19047, 14509, 16370, 15391, 17294, 16632, 16186] avg=16384.0000 max=19047 max_rank=1 min=14509 min_rank=2 imbalance_ratio=1.162537 max_load_reduction=112025 max_load_reduction_pct=0.854683 moved_to_echo=65503 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4074, 0, 1955, 1060, 0, 4068, 0, 3807, 1010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 26, 0, 261, 4096, 0, 3362, 5, 3, 0, 2296, 706, 2695, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1299, 4084, 5, 0, 0, 2869, 4096, 0, 18, 4094, 0, 0, 2, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3982, 4092, 12, 0, 4, 0, 0, 0, 13, 8, 4, 0, 0, 4080, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4071, 3024, 4065, 0, 0, 0, 46, 0, 0, 4096, 14, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 3979, 0, 1, 0, 0, 3, 58, 2782, 0, 4083, 0, 0, 4, 901, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 4088, 0, 4096, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 3, 0, 0, 0, 0, 11, 1, 1, 0, 0, 0, 0, 4096, 4091, 4094, 3801, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15974, 17709, 16742, 16333, 15350, 16154, 16588, 16222] avg=16384.0000 max=17709 max_rank=1 min=15350 min_rank=4 imbalance_ratio=1.080872 max_load_reduction=113363 max_load_reduction_pct=0.864891 moved_to_echo=64314 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 2074, 2362, 0, 4096, 90, 3807, 721, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3866, 0, 0, 791, 0, 1, 3989, 0, 3742, 0, 0, 0, 59, 2027, 2664, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1284, 4094, 0, 0, 0, 4086, 4094, 0, 0, 2238, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 3488, 4096, 4096, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 4096, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3028, 4039, 0, 0, 0, 0, 0, 0, 4063, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3937, 0, 0, 0, 0, 0, 0, 2902, 0, 4096, 0, 0, 0, 903, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 289, 0, 0, 0, 40, 4096, 4096, 4053, 3801, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17269, 17304, 15838, 15982, 15235, 16181, 16789, 16474] avg=16384.0000 max=17304 max_rank=1 min=15235 min_rank=4 imbalance_ratio=1.056152 max_load_reduction=113768 max_load_reduction_pct=0.867981 moved_to_echo=64679 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 4090, 0, 4096, 71, 0, 4093, 0, 3806, 0, 0, 0, 0, 894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 25, 0, 3, 4085, 0, 3555, 0, 8, 4, 0, 461, 2700, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1302, 4096, 0, 0, 0, 4096, 4090, 0, 0, 3778, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3538, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 3017, 4096, 0, 168, 1, 0, 0, 363, 4081, 0, 2, 373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 3845, 0, 0, 0, 0, 0, 0, 3069, 0, 4092, 35, 0, 0, 899, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 1, 192, 0, 0, 0, 0, 3, 0, 0, 73, 4032, 4096, 0, 4096, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 16, 0, 0, 0, 9, 119, 0, 0, 0, 0, 4096, 4090, 4096, 3801, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17053, 15099, 17364, 15866, 16200, 16295, 16867, 16328] avg=16384.0000 max=17364 max_rank=2 min=15099 min_rank=1 imbalance_ratio=1.059814 max_load_reduction=113708 max_load_reduction_pct=0.867523 moved_to_echo=65690 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 4038, 16, 1, 0, 0, 4069, 0, 3801, 0, 22, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3985, 8, 23, 0, 1, 93, 4053, 0, 4024, 5, 0, 0, 4091, 4, 2690, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1296, 4080, 0, 0, 0, 3976, 4085, 0, 1, 326, 38, 0, 0, 0, 3767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4054, 4076, 0, 67, 31, 0, 98, 5, 0, 10, 1, 0, 0, 4086, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4055, 3023, 4042, 0, 30, 4, 156, 92, 0, 4076, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 15, 14, 3710, 2, 14, 46, 0, 0, 786, 4068, 1, 4007, 0, 0, 0, 901, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 7, 0, 26, 0, 0, 9, 0, 6, 1, 4, 3307, 4041, 4, 4081, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 426, 0, 1, 2, 0, 0, 28, 3, 0, 32, 48, 1, 4080, 4082, 4095, 3781, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[11990, 19145, 17584, 16532, 15483, 17869, 15793, 16676] avg=16384.0000 max=19145 max_rank=1 min=11990 min_rank=0 imbalance_ratio=1.168518 max_load_reduction=111927 max_load_reduction_pct=0.853935 moved_to_echo=65821 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 1, 4081, 0, 4051, 2445, 0, 4092, 0, 3803, 0, 4, 2, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 3, 15, 0, 2, 4091, 0, 3685, 0, 0, 2, 0, 1, 2669, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1287, 4039, 0, 2, 0, 4082, 3827, 0, 0, 1361, 4, 11, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 630, 3443, 4094, 4093, 0, 0, 0, 0, 0, 6, 0, 0, 0, 477, 1, 4073, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 4093, 3027, 4084, 0, 16, 2, 0, 0, 233, 3973, 0, 0, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4, 4094, 5, 0, 0, 0, 0, 0, 4042, 0, 4087, 0, 9, 8, 902, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 102, 2, 0, 0, 3, 0, 2, 1, 7, 3998, 4093, 10, 4088, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 3, 0, 4, 0, 59, 0, 8, 0, 4, 2, 0, 3, 0, 0, 86, 4094, 4094, 4094, 3797, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18496, 14688, 14636, 16859, 15811, 17489, 16694, 16399] avg=16384.0000 max=18496 max_rank=0 min=14636 min_rank=2 imbalance_ratio=1.128906 max_load_reduction=112576 max_load_reduction_pct=0.858887 moved_to_echo=66393 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[7, 1, 0, 4092, 8, 1812, 16, 0, 4070, 86, 3779, 3116, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 122, 4096, 0, 26, 0, 0, 24, 1031, 2519, 2704, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1293, 4040, 0, 0, 15, 3684, 4091, 0, 14, 1652, 0, 0, 2, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3955, 4083, 0, 33, 0, 11, 242, 0, 82, 0, 0, 57, 0, 4083, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 0, 4080, 3004, 3946, 0, 0, 0, 0, 0, 0, 4096, 3957, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3980, 2, 11, 0, 126, 0, 0, 1861, 0, 4095, 0, 0, 0, 902, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 1, 0, 27, 3973, 4084, 3, 4081, 289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 93, 0, 221, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4086, 4096, 3791, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16990, 14807, 14961, 16687, 19258, 15323, 16550, 16496] avg=16384.0000 max=19258 max_rank=4 min=14807 min_rank=1 imbalance_ratio=1.175415 max_load_reduction=111814 max_load_reduction_pct=0.853073 moved_to_echo=67627 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 4084, 0, 4048, 1099, 0, 4075, 2, 3774, 0, 9, 26, 0, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 1, 9, 0, 0, 0, 4051, 148, 2100, 43, 0, 0, 36, 0, 2684, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1294, 4077, 0, 0, 11, 4084, 4089, 0, 0, 3890, 7, 0, 0, 0, 1986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4012, 4096, 4096, 45, 0, 0, 0, 0, 33, 0, 2, 0, 2, 7, 4083, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4094, 3027, 4096, 0, 0, 0, 0, 0, 16, 4042, 3007, 50, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 3992, 0, 0, 0, 18, 0, 22, 1091, 0, 4074, 0, 20, 29, 902, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 1, 8, 12, 0, 0, 0, 0, 0, 0, 126, 2633, 4094, 0, 4091, 293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 19, 1, 0, 16, 3, 17, 0, 22, 88, 4090, 4082, 4070, 3785, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17657, 13273, 19438, 16443, 18344, 14467, 15159, 16291] avg=16384.0000 max=19438 max_rank=2 min=13273 min_rank=1 imbalance_ratio=1.186401 max_load_reduction=111634 max_load_reduction_pct=0.851700 moved_to_echo=64261 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 128, 31744, 128, 128, 128, 256, 128, 896, 23680, 128, 32640, 128, 128, 128, 7296, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32512, 128, 512, 128, 0, 0, 128, 128, 128, 128, 256, 30208, 32768, 128, 32768, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 256, 32384, 128, 128, 896, 128, 512, 32640, 256, 23680, 128, 128, 128, 11008, 6912, 21632, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32768, 24192, 32512, 0, 256, 128, 256, 256, 640, 32640, 7040, 256, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[768, 30848, 32512, 32768, 128, 128, 128, 128, 384, 128, 128, 128, 128, 640, 128, 32768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 32768, 128, 18944, 7808, 0, 32768, 256, 30464, 5248, 128, 128, 128, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 10368, 32640, 128, 128, 128, 30464, 32512, 0, 128, 18944, 128, 128, 128, 0, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 768, 128, 128, 128, 128, 128, 128, 512, 128, 128, 128, 256, 32768, 32768, 32768, 30464, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 0, 4096, 0, 4094, 0, 4093, 4085, 3824, 0, 16, 0, 24, 0, 0, 1, 4095, 28, 4048, 4096, 2, 0, 4089, 4039, 4096, 0, 192, 1, 3985, 4096, 35, 23, 0, 17, 0, 38, 0, 3547, 61, 4096, 4, 0, 0, 0, 0, 4095, 1, 11, 0, 4090, 3999, 1, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 29, 0, 0, 0, 0, 54, 0, 2, 4096, 0, 2, 26, 2167, 0, 0, 0, 4061, 0, 4096, 4090, 0, 0, 0, 0, 4066, 0, 196, 0, 0, 1, 1871, 3968, 1341, 4, 19, 9, 0, 0, 0, 0, 0, 9, 4069, 0, 0, 0, 4084, 0, 4083, 0, 613, 4096, 0, 15, 4093, 2737, 5, 51, 0, 4093, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 529, 4083, 0, 3959, 16, 4071, 4095, 3886, 0, 0, 17, 0, 0, 0, 40, 4096, 0, 4006, 4067, 0, 0, 4057, 3828, 4073, 73, 0, 0, 4096, 4077, 0, 1158, 0, 0, 12, 0, 3, 4090, 197, 4085, 69, 2, 0, 0, 0, 4086, 0, 29, 0, 4096, 4095, 0, 0, 22, 0, 133, 1, 16, 43, 1015, 0, 0, 0, 0, 3, 8, 0, 90, 0, 5, 4094, 0, 1, 15, 46, 0, 0, 0, 4093, 49, 4050, 4072, 0, 0, 10, 37, 0, 2, 11, 14, 0, 0, 434, 3764, 960, 1, 5, 7, 27, 0, 0, 10, 52, 220, 4003, 0, 0, 7, 4046, 1, 3998, 0, 4064, 3974, 0, 0, 4084, 4067, 0, 0, 13, 4096, 0, 418, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4095, 198, 4088, 4080, 4042, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 3577, 4096, 0, 0, 4096, 4094, 4092, 0, 0, 0, 3742, 4094, 20, 2263, 3, 0, 0, 1, 0, 4093, 2046, 4096, 0, 0, 2, 0, 0, 4090, 1, 0, 0, 4068, 695, 0, 0, 0, 0, 1, 0, 0, 0, 3560, 0, 0, 0, 3, 26, 0, 7, 0, 0, 0, 4094, 0, 0, 0, 2541, 0, 0, 0, 4096, 0, 4096, 4093, 0, 17, 0, 0, 1, 0, 0, 0, 0, 0, 16, 4087, 3344, 0, 4, 36, 0, 1, 0, 0, 0, 3, 4095, 0, 15, 0, 4088, 0, 4080, 0, 66, 4083, 4, 13, 4084, 2561, 0, 0, 0, 4088, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[6, 0, 20, 0, 0, 4091, 2, 4087, 0, 4093, 4093, 4079, 0, 0, 6, 28, 0, 4, 0, 4094, 2, 4092, 4089, 0, 13, 4092, 4095, 4094, 19, 0, 3, 4000, 4086, 0, 14, 0, 0, 3, 1, 2, 4093, 470, 4096, 0, 2, 1, 2, 4, 4093, 1, 0, 0, 4091, 2993, 0, 0, 31, 10, 29, 22, 0, 0, 25, 5, 0, 872, 0, 0, 0, 8, 0, 6, 3, 4050, 1, 3, 0, 0, 0, 3989, 3, 4094, 0, 4083, 4091, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 828, 4047, 0, 0, 0, 0, 0, 345, 3, 4, 0, 3407, 4088, 0, 10, 13, 4060, 0, 4086, 198, 0, 3912, 0, 0, 3544, 4094, 0, 107, 10, 3935, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 95, 4096, 0, 4096, 0, 4096, 4096, 4093, 0, 0, 0, 441, 0, 0, 0, 4096, 0, 4012, 4096, 0, 0, 4096, 4096, 4096, 0, 454, 0, 4095, 4096, 0, 77, 0, 0, 0, 0, 0, 4025, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3548, 3098, 0, 0, 0, 0, 1, 0, 0, 0, 3562, 0, 0, 5, 0, 0, 0, 0, 50, 0, 0, 4096, 0, 0, 3, 46, 0, 80, 0, 4096, 0, 4094, 4094, 0, 97, 0, 0, 312, 0, 1, 0, 0, 0, 0, 4074, 1328, 0, 0, 47, 0, 90, 0, 0, 0, 3417, 4096, 0, 2, 0, 4096, 0, 3962, 0, 53, 4096, 0, 0, 4096, 4095, 0, 3, 0, 3991, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4093, 39, 4069, 0, 4075, 4039, 4056, 59, 0, 0, 189, 31, 48, 38, 4066, 319, 4089, 4048, 60, 1, 4077, 4050, 4069, 84, 0, 38, 3954, 4032, 2, 1, 0, 0, 43, 0, 54, 4092, 16, 3921, 1, 20, 0, 0, 0, 4092, 0, 117, 0, 4079, 6, 0, 0, 0, 34, 0, 3, 18, 0, 2431, 10, 113, 0, 7, 180, 14, 0, 0, 0, 0, 4082, 3, 27, 0, 3773, 0, 0, 0, 4024, 0, 4060, 4001, 0, 0, 9, 0, 1878, 37, 0, 0, 0, 66, 10, 3912, 1685, 0, 17, 170, 11, 0, 7, 11, 50, 1968, 4060, 0, 0, 0, 3880, 11, 4028, 0, 122, 4042, 0, 2, 4086, 4003, 0, 0, 0, 4085, 72, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096, 48, 4096, 4095, 4018, 0, 8, 0, 674, 0, 0, 0, 4096, 1, 4028, 4096, 0, 59, 4095, 4094, 4096, 102, 46, 0, 3998, 4096, 6, 43, 0, 0, 0, 0, 0, 4092, 779, 3929, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4060, 1768, 0, 0, 88, 0, 5, 0, 0, 1, 1241, 0, 15, 19, 3, 5, 0, 0, 0, 0, 0, 3720, 0, 0, 0, 4086, 0, 226, 0, 4096, 0, 4093, 4090, 4, 0, 0, 14, 2351, 0, 1, 0, 0, 0, 52, 4084, 303, 0, 0, 1, 0, 0, 0, 0, 1, 4090, 4087, 0, 41, 0, 4096, 0, 112, 0, 1156, 4082, 0, 0, 4096, 4003, 8, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[2, 0, 11, 9, 0, 4077, 1, 4067, 28, 4084, 4075, 4096, 16, 0, 0, 0, 18, 17, 0, 4086, 0, 2804, 4071, 2, 0, 4091, 3974, 4091, 0, 22, 0, 4096, 4096, 0, 4087, 14, 0, 5, 0, 0, 4093, 1212, 4025, 19, 2, 2, 0, 11, 4077, 30, 0, 0, 4072, 3644, 0, 0, 4, 10, 9, 5, 0, 45, 4068, 4, 0, 0, 0, 0, 0, 0, 1, 0, 5, 4007, 3, 0, 0, 1696, 11, 43, 0, 4083, 0, 4056, 4008, 0, 80, 0, 0, 0, 19, 1, 25, 51, 0, 46, 4074, 4, 0, 0, 1, 0, 0, 19, 1, 0, 1, 4075, 0, 0, 0, 4089, 3, 2421, 0, 655, 4031, 0, 0, 4074, 4096, 0, 1, 27, 4059, 32, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4095, 198, 3910, 4080, 4042, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3577, 4096, 0, 0, 227, 4094, 0, 0, 0, 0, 3742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 20, 2263, 3, 0, 0, 1, 0, 4093, 2046, 4096, 0, 0, 2, 0, 0, 2641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 4068, 695, 0, 0, 0, 0, 1, 0, 0, 0, 3560, 0, 2458, 1209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 26, 0, 7, 0, 0, 0, 4094, 0, 0, 0, 2541, 0, 0, 4096, 4096, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4093, 0, 17, 0, 0, 1, 0, 0, 0, 0, 0, 16, 2582, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 3344, 0, 4, 36, 0, 1, 0, 0, 0, 3, 4095, 0, 15, 0, 4088, 1197, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 66, 4083, 4, 13, 4084, 2561, 0, 0, 0, 309, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16328, 15736, 19259, 16082, 16497, 14920, 17048, 15202] avg=16384.0000 max=19259 max_rank=2 min=14920 min_rank=5 imbalance_ratio=1.175476 max_load_reduction=111813 max_load_reduction_pct=0.853065 moved_to_echo=63667 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 0, 0, 0, 4094, 0, 3915, 4085, 3824, 0, 16, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 28, 4048, 4096, 2, 0, 226, 4039, 0, 0, 192, 1, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 35, 23, 0, 17, 0, 38, 0, 3547, 61, 4096, 4, 0, 0, 0, 0, 2637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 11, 0, 4090, 3999, 1, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2460, 1207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 54, 0, 2, 4096, 0, 2, 26, 2167, 0, 0, 4096, 4095, 1636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 4096, 4090, 0, 0, 0, 0, 4066, 0, 196, 0, 0, 1, 1871, 2586, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 1341, 4, 19, 9, 0, 0, 0, 0, 0, 9, 4069, 0, 0, 0, 4084, 1198, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 613, 4096, 0, 15, 4093, 2737, 5, 51, 0, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15961, 16618, 14554, 15869, 16203, 20986, 14879, 16002] avg=16384.0000 max=20986 max_rank=5 min=14554 min_rank=2 imbalance_ratio=1.280884 max_load_reduction=110086 max_load_reduction_pct=0.839890 moved_to_echo=68070 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 529, 0, 0, 3959, 16, 3889, 4095, 3886, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 4006, 4067, 0, 0, 216, 3828, 0, 73, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 1158, 0, 0, 12, 0, 3, 4090, 197, 4085, 69, 2, 0, 0, 0, 2620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 29, 0, 4096, 4095, 0, 0, 22, 0, 133, 1, 16, 43, 1015, 0, 2447, 1201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 0, 90, 0, 5, 4094, 0, 1, 15, 46, 0, 0, 4083, 4096, 1626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 49, 4050, 4072, 0, 0, 10, 37, 0, 2, 11, 14, 0, 0, 434, 2591, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3764, 960, 1, 5, 7, 27, 0, 0, 10, 52, 220, 4003, 0, 0, 7, 4046, 1204, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3998, 0, 4064, 3974, 0, 0, 4084, 4067, 0, 0, 13, 301, 0, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16391, 16326, 16313, 17184, 14067, 15383, 14488, 20920] avg=16384.0000 max=20920 max_rank=7 min=14067 min_rank=4 imbalance_ratio=1.276855 max_load_reduction=110152 max_load_reduction_pct=0.840393 moved_to_echo=64858 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 20, 0, 0, 0, 2, 4087, 0, 3915, 4093, 4079, 0, 0, 6, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 4092, 4089, 0, 13, 227, 4095, 0, 19, 0, 3, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 14, 0, 0, 3, 1, 2, 4093, 470, 4096, 0, 2, 1, 2, 4, 2638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 4091, 2993, 0, 0, 31, 10, 29, 22, 0, 0, 25, 5, 2460, 1208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 872, 0, 0, 0, 8, 0, 6, 3, 4050, 1, 3, 0, 0, 0, 3989, 4091, 4094, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 4083, 4091, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 828, 2486, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 345, 3, 4, 0, 3407, 4088, 0, 10, 13, 4060, 1152, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 198, 0, 3912, 0, 0, 3544, 4094, 0, 107, 10, 297, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16236, 16544, 15412, 14968, 18751, 15605, 17307, 16249] avg=16384.0000 max=18751 max_rank=4 min=14968 min_rank=3 imbalance_ratio=1.144470 max_load_reduction=112321 max_load_reduction_pct=0.856941 moved_to_echo=67912 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 48, 3918, 4095, 4018, 0, 8, 0, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4028, 4096, 0, 59, 227, 4094, 0, 102, 46, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 43, 0, 0, 0, 0, 0, 4092, 779, 3929, 0, 0, 0, 0, 0, 2640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4060, 1768, 0, 0, 88, 0, 5, 0, 0, 1, 1241, 0, 2461, 1209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 19, 3, 5, 0, 0, 0, 0, 0, 3720, 0, 0, 0, 4086, 0, 226, 4096, 4096, 1635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 4090, 4, 0, 0, 14, 2351, 0, 1, 0, 0, 0, 52, 2588, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 303, 0, 0, 1, 0, 0, 0, 0, 1, 4090, 4087, 0, 41, 0, 4096, 1199, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 1156, 4082, 0, 0, 4096, 4003, 8, 0, 0, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16857, 16651, 15585, 14924, 17901, 17308, 18080, 13766] avg=16384.0000 max=18080 max_rank=6 min=13766 min_rank=7 imbalance_ratio=1.103516 max_load_reduction=112992 max_load_reduction_pct=0.862061 moved_to_echo=67055 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 39, 4069, 0, 3898, 4039, 4056, 59, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 48, 38, 0, 319, 4089, 4048, 60, 1, 227, 4050, 0, 84, 0, 38, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 2, 1, 0, 0, 43, 0, 54, 4092, 16, 3921, 1, 20, 0, 0, 0, 2629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 117, 0, 4079, 6, 0, 0, 0, 34, 0, 3, 18, 0, 2431, 10, 2444, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 7, 180, 14, 0, 0, 0, 0, 4082, 3, 27, 0, 3773, 0, 0, 4093, 4066, 1625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 4060, 4001, 0, 0, 9, 0, 1878, 37, 0, 0, 0, 66, 10, 2581, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 1685, 0, 17, 170, 11, 0, 7, 11, 50, 1968, 4060, 0, 0, 0, 3880, 1196, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4028, 0, 122, 4042, 0, 2, 4086, 4003, 0, 0, 0, 308, 72, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16349, 16987, 14811, 14437, 17983, 16684, 17144, 16677] avg=16384.0000 max=17983 max_rank=4 min=14437 min_rank=3 imbalance_ratio=1.097595 max_load_reduction=113089 max_load_reduction_pct=0.862801 moved_to_echo=68488 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 95, 0, 0, 4096, 0, 3918, 4096, 4093, 0, 0, 0, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 4096, 0, 0, 227, 4096, 0, 0, 454, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 77, 0, 0, 0, 0, 0, 4025, 0, 4095, 0, 0, 0, 0, 0, 2641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3548, 3098, 0, 0, 0, 0, 1, 0, 0, 0, 3562, 0, 2461, 1209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 50, 0, 0, 4096, 0, 0, 3, 46, 0, 80, 4096, 4096, 1635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 4094, 0, 97, 0, 0, 312, 0, 1, 0, 0, 0, 0, 2521, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 1328, 0, 0, 47, 0, 90, 0, 0, 0, 3417, 4096, 0, 2, 0, 4096, 1168, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 0, 53, 4096, 0, 0, 4096, 4095, 0, 3, 0, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16739, 16980, 14934, 17975, 14107, 15234, 18496, 16607] avg=16384.0000 max=18496 max_rank=6 min=14107 min_rank=4 imbalance_ratio=1.128906 max_load_reduction=112576 max_load_reduction_pct=0.858887 moved_to_echo=64444 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 11, 9, 0, 0, 1, 4067, 28, 3906, 4075, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 17, 0, 0, 0, 2804, 4071, 2, 0, 226, 3974, 0, 0, 22, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4087, 14, 0, 5, 0, 0, 4093, 1212, 4025, 19, 2, 2, 0, 11, 2638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 30, 0, 0, 4072, 3644, 0, 0, 4, 10, 9, 5, 0, 45, 4068, 4, 2457, 1208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 4007, 3, 0, 0, 1696, 11, 43, 4077, 4086, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4056, 4008, 0, 80, 0, 0, 0, 19, 1, 25, 51, 0, 46, 2564, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4, 0, 0, 1, 0, 0, 19, 1, 0, 1, 4075, 0, 0, 0, 4089, 1188, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2421, 0, 655, 4031, 0, 0, 4074, 4096, 0, 1, 27, 307, 32, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16211, 15230, 20204, 19633, 15563, 14952, 13630, 15649] avg=16384.0000 max=20204 max_rank=2 min=13630 min_rank=6 imbalance_ratio=1.233154 max_load_reduction=110868 max_load_reduction_pct=0.845856 moved_to_echo=59794 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32128, 9088, 128, 128, 384, 128, 512, 128, 128, 128, 13184, 32640, 0, 128, 128, 32512, 9600, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 384, 30720, 32768, 128, 128, 1920, 32384, 0, 384, 768, 128, 32000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 7680, 128, 128, 128, 128, 128, 32128, 4864, 32384, 128, 128, 128, 128, 128, 21120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 640, 0, 128, 32640, 384, 31360, 32768, 32128, 128, 128, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 256, 0, 32128, 20352, 128, 0, 256, 128, 256, 128, 128, 128, 16000, 128, 19712, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 128, 32640, 32640, 128, 256, 128, 128, 8704, 128, 256, 128, 128, 128, 3328, 20608, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 26880, 256, 6784, 32384, 128, 128, 32256, 29696, 128, 256, 128, 2560, 128, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 1024, 128, 256, 128, 128, 256, 128, 128, 32256, 128, 128, 128, 14464, 128, 4352, 32768, 32768, 13184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[428, 4096, 0, 19, 0, 0, 0, 4078, 4096, 3671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 31, 4096, 4096, 4087, 0, 0, 0, 4089, 0, 0, 1, 11, 0, 0, 0, 0, 3, 0, 0, 0, 4087, 4045, 4096, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3807, 4091, 4076, 0, 0, 0, 0, 0, 0, 4092, 4093, 0, 0, 0, 4084, 0, 0, 0, 0, 4092, 360, 4095, 0, 9, 0, 0, 6, 0, 0, 4096, 0, 4068, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4, 1, 0, 0, 0, 53, 0, 0, 0, 4096, 0, 19, 4030, 4086, 0, 0, 3, 5, 4096, 58, 0, 0, 0, 4096, 0, 0, 4074, 4049, 3, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[2, 4083, 3, 0, 117, 6, 0, 3282, 4044, 3462, 0, 0, 36, 13, 0, 0, 22, 6, 24, 0, 1091, 4092, 4064, 4071, 0, 85, 118, 4041, 10, 0, 0, 0, 13, 0, 27, 32, 33, 10, 3, 696, 4055, 4064, 4078, 4067, 1, 40, 92, 0, 6, 0, 34, 6, 0, 0, 656, 4074, 2920, 0, 10, 19, 2, 0, 70, 4091, 4010, 9, 0, 8, 4048, 61, 4, 0, 0, 4077, 311, 4082, 45, 0, 11, 0, 0, 0, 1, 4086, 2021, 4088, 0, 5, 0, 0, 0, 17, 4093, 0, 4090, 0, 0, 11, 0, 0, 28, 0, 31, 0, 4039, 0, 344, 1091, 4027, 0, 0, 0, 0, 4092, 3798, 0, 0, 15, 4072, 0, 0, 4058, 4001, 17, 3729, 0, 881, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[619, 4096, 0, 0, 0, 0, 0, 4076, 4096, 3110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 3874, 4096, 3859, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4076, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 4095, 4063, 0, 0, 0, 0, 19, 0, 4092, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 4052, 0, 38, 0, 0, 0, 0, 0, 3550, 533, 4039, 0, 0, 0, 0, 218, 0, 4096, 0, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 0, 4096, 0, 0, 1057, 4096, 0, 0, 0, 0, 4033, 3762, 0, 1, 0, 4096, 11, 0, 4096, 4096, 0, 4088, 41, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1312, 4087, 0, 0, 96, 0, 0, 3906, 4081, 2722, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 61, 4095, 4092, 4096, 0, 0, 0, 4077, 0, 0, 199, 0, 1, 0, 0, 0, 0, 0, 0, 31, 4096, 3974, 4095, 4096, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 3767, 4095, 4022, 0, 3, 2, 0, 0, 0, 4061, 4076, 5, 0, 0, 4089, 1, 0, 3, 0, 4096, 72, 4012, 0, 0, 0, 3, 0, 0, 0, 4090, 736, 4093, 0, 0, 0, 9, 0, 1, 3985, 0, 4096, 16, 18, 0, 0, 19, 11, 6, 0, 0, 4096, 0, 128, 3961, 4095, 1, 0, 11, 0, 4096, 1, 0, 1, 0, 4092, 1, 0, 4096, 4096, 10, 3872, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[173, 4096, 0, 6, 0, 2, 0, 4090, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2213, 4095, 4075, 3872, 0, 0, 13, 4087, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4079, 4025, 4096, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3685, 4075, 4055, 0, 42, 0, 0, 0, 7, 4096, 4073, 0, 0, 0, 4030, 20, 0, 0, 0, 4062, 75, 4093, 8, 0, 0, 0, 531, 0, 0, 2777, 0, 4082, 0, 0, 0, 3, 0, 0, 4096, 0, 4096, 0, 20, 0, 2, 0, 0, 3, 0, 0, 4096, 0, 0, 450, 4096, 0, 8, 1, 1, 4096, 2766, 0, 5, 1, 4096, 0, 6, 4089, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1035, 4081, 0, 103, 2, 17, 3, 3826, 4094, 3883, 0, 0, 0, 10, 4, 0, 8, 0, 14, 0, 0, 4089, 4095, 4096, 11, 0, 102, 4083, 1, 0, 53, 0, 1, 317, 0, 10, 0, 0, 1, 0, 4090, 4095, 4096, 4094, 0, 14, 0, 0, 10, 0, 33, 0, 0, 0, 3406, 4092, 3754, 1, 0, 0, 0, 0, 2, 4095, 4053, 1, 29, 2, 3984, 5, 0, 0, 0, 4089, 0, 4060, 0, 0, 16, 0, 74, 0, 0, 4078, 197, 4067, 0, 0, 0, 0, 0, 0, 4066, 0, 4056, 22, 0, 0, 9, 250, 22, 0, 8, 25, 4076, 0, 151, 3182, 4029, 12, 6, 5, 0, 4070, 3, 0, 0, 0, 4082, 0, 464, 4075, 4092, 0, 3955, 0, 36, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[353, 4095, 24, 0, 14, 0, 0, 4065, 4096, 4020, 2, 0, 0, 0, 0, 0, 3, 0, 14, 0, 692, 3287, 4094, 4054, 0, 5, 0, 4093, 0, 69, 0, 5, 3, 0, 150, 11, 3, 5, 2, 40, 4026, 4091, 3913, 4090, 0, 3, 0, 0, 0, 13, 0, 2, 0, 0, 4092, 4091, 3890, 0, 0, 0, 147, 134, 0, 3986, 4080, 0, 0, 5, 4093, 11, 0, 1, 0, 4040, 0, 4091, 16, 0, 0, 0, 0, 2, 4, 0, 3126, 4094, 0, 0, 98, 0, 815, 4, 4095, 1, 3939, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4091, 0, 0, 266, 4084, 4, 0, 7, 0, 3993, 4058, 6, 0, 0, 4090, 1, 9, 4088, 4085, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[2632, 4073, 0, 0, 241, 0, 0, 3431, 4063, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 4082, 4089, 4086, 19, 2, 99, 4086, 0, 146, 2, 0, 9, 0, 26, 0, 0, 1, 0, 0, 4021, 3886, 4067, 4075, 0, 0, 7, 0, 14, 0, 25, 10, 4, 6, 4093, 3950, 2308, 0, 0, 9, 79, 55, 37, 4073, 4068, 47, 0, 12, 4064, 42, 2, 13, 13, 3928, 0, 4053, 2, 35, 0, 0, 1, 0, 8, 4003, 20, 4087, 0, 0, 0, 0, 39, 1, 4056, 16, 4041, 12, 0, 0, 24, 99, 0, 0, 0, 0, 4071, 0, 0, 3878, 4086, 0, 0, 0, 0, 4049, 18, 1, 0, 0, 4081, 158, 0, 3972, 4084, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[428, 3925, 0, 19, 0, 0, 0, 4078, 4096, 3671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 31, 4096, 3668, 4087, 0, 0, 0, 4089, 0, 0, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4087, 4045, 4096, 4020, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3807, 4091, 4076, 0, 0, 0, 0, 0, 0, 4092, 428, 231, 171, 117, 75, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 4084, 0, 0, 0, 0, 4092, 360, 4095, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 3837, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 53, 0, 0, 0, 4096, 0, 19, 4030, 4086, 0, 0, 3, 4096, 1717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 58, 0, 0, 0, 0, 0, 0, 4074, 2215, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16217, 15998, 16254, 17123, 16698, 16134, 18101, 14547] avg=16384.0000 max=18101 max_rank=6 min=14547 min_rank=7 imbalance_ratio=1.104797 max_load_reduction=112971 max_load_reduction_pct=0.861900 moved_to_echo=65480 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1312, 3917, 0, 0, 96, 0, 0, 3906, 4081, 2722, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 4095, 3665, 4096, 0, 0, 0, 4077, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 31, 4096, 3974, 4095, 4021, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3767, 4095, 4022, 0, 3, 2, 0, 0, 0, 4061, 427, 233, 170, 118, 75, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 5, 0, 0, 4089, 1, 0, 3, 0, 4096, 72, 4012, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 736, 3860, 0, 0, 0, 9, 0, 1, 3985, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 19, 11, 6, 0, 0, 4096, 0, 128, 3961, 4095, 1, 0, 11, 4092, 1737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 1, 0, 0, 1, 0, 4096, 2241, 10, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16036, 16193, 16227, 17008, 16322, 16793, 18175, 14318] avg=16384.0000 max=18175 max_rank=6 min=14318 min_rank=7 imbalance_ratio=1.109314 max_load_reduction=112897 max_load_reduction_pct=0.861336 moved_to_echo=65608 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2, 3913, 3, 0, 117, 6, 0, 3282, 4044, 3462, 0, 0, 36, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 6, 24, 0, 1091, 4092, 3640, 4071, 0, 85, 118, 4041, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 27, 32, 33, 10, 3, 696, 4055, 4064, 4078, 3992, 1, 40, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 34, 6, 0, 0, 656, 4074, 2920, 0, 10, 19, 2, 0, 70, 4091, 424, 232, 170, 115, 75, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 9, 0, 8, 4048, 61, 4, 0, 0, 4077, 311, 4082, 45, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 2021, 3856, 0, 5, 0, 0, 0, 17, 4093, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 28, 0, 31, 0, 4039, 0, 344, 1091, 4027, 0, 0, 0, 4072, 1697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3798, 0, 0, 15, 0, 0, 0, 4058, 2189, 17, 3729, 0, 881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14878, 17200, 17136, 12938, 16632, 18169, 15340, 18779] avg=16384.0000 max=18779 max_rank=7 min=12938 min_rank=3 imbalance_ratio=1.146179 max_load_reduction=112293 max_load_reduction_pct=0.856728 moved_to_echo=68920 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[173, 3925, 0, 6, 0, 2, 0, 4090, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2213, 4095, 3650, 3872, 0, 0, 13, 4087, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4079, 4025, 4096, 4021, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3685, 4075, 4055, 0, 42, 0, 0, 0, 7, 4096, 425, 232, 171, 118, 75, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 4030, 20, 0, 0, 0, 4062, 75, 4093, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 0, 0, 2777, 0, 3850, 0, 0, 0, 3, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 2, 0, 0, 3, 0, 0, 4096, 0, 0, 450, 4096, 0, 8, 1, 4096, 1737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 2766, 0, 5, 1, 0, 0, 6, 4089, 2241, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 17937, 16246, 17016, 16326, 15353, 14509, 17301] avg=16384.0000 max=17937 max_rank=1 min=14509 min_rank=6 imbalance_ratio=1.094788 max_load_reduction=113135 max_load_reduction_pct=0.863152 moved_to_echo=63489 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[353, 3924, 24, 0, 14, 0, 0, 4065, 4096, 4020, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 14, 0, 692, 3287, 3667, 4054, 0, 5, 0, 4093, 0, 69, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 150, 11, 3, 5, 2, 40, 4026, 4091, 3913, 4015, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 2, 0, 0, 4092, 4091, 3890, 0, 0, 0, 147, 134, 0, 3986, 427, 233, 171, 118, 75, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 5, 4093, 11, 0, 1, 0, 4040, 0, 4091, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 3126, 3861, 0, 0, 98, 0, 815, 4, 4095, 1, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4091, 0, 0, 266, 4084, 4, 0, 7, 4090, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 4058, 6, 0, 0, 0, 1, 9, 4088, 2235, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16498, 15889, 16262, 17414, 16302, 15945, 14276, 18486] avg=16384.0000 max=18486 max_rank=7 min=14276 min_rank=6 imbalance_ratio=1.128296 max_load_reduction=112586 max_load_reduction_pct=0.858963 moved_to_echo=65009 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1035, 3907, 0, 103, 2, 17, 3, 3826, 4094, 3883, 0, 0, 0, 10, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 14, 0, 0, 4089, 3664, 4096, 11, 0, 102, 4083, 1, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 317, 0, 10, 0, 0, 1, 0, 4090, 4095, 4096, 4015, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 33, 0, 0, 0, 3406, 4092, 3754, 1, 0, 0, 0, 0, 2, 4095, 431, 236, 174, 120, 79, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 1, 29, 2, 3984, 5, 0, 0, 0, 4089, 0, 4060, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 4078, 197, 3831, 0, 0, 0, 0, 0, 0, 4066, 0, 4056, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 250, 22, 0, 8, 25, 4076, 0, 151, 3182, 4029, 12, 6, 5, 4082, 1738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 3, 0, 0, 0, 0, 0, 464, 4075, 2234, 0, 3955, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16884, 16121, 16639, 16470, 16202, 16324, 17595, 14837] avg=16384.0000 max=17595 max_rank=6 min=14837 min_rank=7 imbalance_ratio=1.073914 max_load_reduction=113477 max_load_reduction_pct=0.865761 moved_to_echo=64958 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[619, 3925, 0, 0, 0, 0, 0, 4076, 4096, 3110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 3874, 3668, 3859, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4076, 4096, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 4095, 4063, 0, 0, 0, 0, 19, 0, 4092, 428, 230, 171, 118, 75, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 4052, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3550, 533, 3809, 0, 0, 0, 0, 218, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 4096, 0, 0, 1057, 4096, 0, 0, 0, 4096, 1737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 3762, 0, 1, 0, 0, 11, 0, 4096, 2241, 0, 4088, 41, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15826, 15543, 16291, 17384, 16345, 16302, 15105, 18276] avg=16384.0000 max=18276 max_rank=7 min=15105 min_rank=6 imbalance_ratio=1.115479 max_load_reduction=112796 max_load_reduction_pct=0.860565 moved_to_echo=66028 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[2632, 3903, 0, 0, 241, 0, 0, 3431, 4063, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 4082, 3662, 4086, 19, 2, 99, 4086, 0, 146, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 26, 0, 0, 1, 0, 0, 4021, 3886, 4067, 4000, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 25, 10, 4, 6, 4093, 3950, 2308, 0, 0, 9, 79, 55, 37, 4073, 427, 232, 170, 118, 75, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 47, 0, 12, 4064, 42, 2, 13, 13, 3928, 0, 4053, 2, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 4003, 20, 3855, 0, 0, 0, 0, 39, 1, 4056, 16, 4041, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 99, 0, 0, 0, 0, 4071, 0, 0, 3878, 4086, 0, 0, 0, 4081, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 18, 1, 0, 0, 0, 158, 0, 3972, 2234, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18349, 16191, 16017, 15719, 16245, 16052, 17971, 14528] avg=16384.0000 max=18349 max_rank=0 min=14528 min_rank=7 imbalance_ratio=1.119934 max_load_reduction=112723 max_load_reduction_pct=0.860008 moved_to_echo=64796 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[640, 128, 128, 26752, 6656, 30848, 0, 128, 128, 128, 1152, 128, 32640, 128, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 384, 128, 128, 128, 128, 32768, 0, 768, 17920, 32640, 128, 128, 128, 32768, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 4224, 31744, 29312, 32256, 128, 128, 384, 32768, 128, 256, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32384, 128, 128, 128, 32512, 256, 128, 128, 128, 32512, 896, 32640, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[6656, 31360, 128, 128, 512, 128, 128, 30848, 32768, 29056, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 384, 256, 128, 128, 128, 128, 896, 32640, 32256, 32640, 32128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32640, 14464, 128, 128, 128, 0, 256, 512, 32640, 17920, 128, 32128, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 128, 27648, 32640, 29184, 128, 128, 128, 256, 256, 128, 32640, 3456, 1920, 1408, 1024, 640, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 66, 0, 4095, 0, 0, 0, 0, 0, 4092, 4092, 0, 0, 5, 19, 4095, 2, 4089, 0, 3205, 0, 14, 0, 0, 0, 4077, 0, 122, 4090, 1, 3756, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 6, 3220, 3039, 0, 0, 0, 0, 0, 0, 6, 0, 6, 4095, 0, 3527, 5, 0, 0, 4083, 4096, 4, 0, 0, 2942, 0, 0, 4079, 0, 0, 0, 0, 13, 0, 0, 0, 4095, 4036, 8, 4080, 4095, 49, 0, 0, 0, 0, 3, 4093, 1, 0, 0, 0, 4096, 3, 0, 0, 24, 2, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 84, 4063, 4090, 0, 0, 14, 0, 4092, 304, 4, 4086, 0, 4096, 0, 0, 0, 231] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 1, 0, 4095, 0, 0, 3, 0, 8, 3760, 4083, 1, 0, 2, 0, 4096, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 4065, 0, 2659, 4095, 3, 3747, 15, 0, 0, 0, 0, 4086, 4094, 0, 0, 0, 0, 4092, 33, 0, 0, 1, 0, 0, 12, 0, 0, 0, 4096, 0, 3920, 0, 0, 0, 4094, 4096, 2, 0, 0, 1811, 0, 0, 4058, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 3639, 0, 4091, 4081, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4092, 4095, 0, 0, 21, 0, 4096, 406, 0, 4096, 0, 4096, 0, 0, 6, 542] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 76, 0, 2660, 0, 0, 0, 2, 2, 3246, 4096, 0, 1, 0, 0, 4060, 0, 4093, 0, 2895, 6, 0, 0, 0, 0, 4089, 0, 30, 4096, 0, 4063, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 679, 0, 3989, 0, 0, 0, 0, 2, 0, 45, 0, 0, 0, 4096, 0, 3651, 0, 0, 0, 4013, 4083, 21, 17, 0, 886, 0, 0, 3922, 21, 0, 0, 0, 0, 1, 0, 0, 4095, 4096, 0, 4091, 4091, 0, 0, 0, 49, 0, 0, 4094, 1, 8, 100, 0, 4072, 18, 26, 0, 1, 0, 6, 0, 0, 0, 0, 4, 0, 4092, 0, 4095, 0, 4096, 0, 0, 4093, 4096, 0, 0, 2842, 0, 4096, 0, 0, 3718, 0, 4096, 8, 0, 236, 3818] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[2, 0, 24, 4075, 0, 0, 0, 1, 12, 3818, 4079, 1, 0, 4, 0, 4094, 0, 4084, 0, 3542, 14, 2, 0, 33, 1, 3951, 0, 12, 4081, 0, 4090, 0, 4, 0, 0, 0, 4053, 4092, 0, 0, 0, 12, 2886, 0, 0, 0, 23, 1, 0, 0, 0, 0, 260, 4080, 1, 101, 0, 8, 0, 4060, 4095, 0, 2, 0, 4070, 0, 2, 4096, 9, 0, 0, 0, 52, 12, 3, 1, 4074, 4085, 1, 4066, 4082, 0, 38, 0, 1373, 0, 2, 4096, 0, 22, 0, 1, 4067, 5, 0, 0, 0, 0, 18, 30, 0, 0, 0, 14, 0, 4079, 0, 4092, 0, 4096, 0, 0, 4096, 3941, 22, 2, 0, 15, 4086, 4071, 12, 4092, 0, 4081, 0, 555, 0, 15] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[28, 0, 15, 2, 0, 0, 5, 0, 78, 3861, 4090, 0, 0, 1, 0, 4078, 0, 4093, 0, 4094, 6, 0, 0, 0, 0, 3987, 6, 0, 4092, 0, 4073, 13, 0, 8, 664, 199, 4090, 4054, 0, 3, 2762, 341, 4092, 0, 3, 0, 0, 0, 3, 59, 0, 0, 0, 4080, 1, 4096, 0, 0, 3, 4094, 4093, 0, 70, 11, 1798, 0, 5, 4092, 0, 0, 17, 7, 0, 18, 1, 0, 4093, 2808, 0, 4096, 4091, 0, 0, 5, 0, 0, 3, 4091, 0, 0, 10, 0, 3753, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4089, 4, 4080, 0, 4067, 2, 104, 4093, 3919, 0, 8, 3954, 3, 4034, 0, 0, 4094, 4, 4086, 1, 0, 6, 383] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 896, 0, 0, 81, 24, 0, 3704, 4065, 21, 0, 7, 0, 4034, 9, 4066, 0, 4096, 3, 0, 28, 88, 0, 4054, 12, 3965, 4069, 1, 3995, 0, 0, 0, 24, 0, 4031, 4081, 0, 0, 0, 0, 3058, 0, 19, 0, 0, 108, 0, 0, 47, 2, 0, 4094, 33, 3500, 203, 90, 0, 3992, 4074, 0, 0, 41, 1409, 0, 17, 3633, 7, 0, 19, 12, 11, 2, 11, 36, 4061, 4084, 0, 4067, 3966, 35, 0, 1, 2279, 26, 1, 4077, 0, 44, 0, 0, 4059, 6, 0, 0, 1, 0, 0, 6, 0, 0, 21, 29, 0, 4087, 14, 4055, 0, 4071, 0, 663, 4050, 4074, 8, 19, 36, 29, 4079, 746, 50, 4025, 0, 4072, 0, 3, 0, 456] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 2335, 0, 0, 0, 0, 15, 2125, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 4025, 0, 0, 0, 0, 1, 4095, 0, 63, 4096, 0, 4079, 0, 0, 0, 60, 17, 4096, 3993, 0, 0, 126, 76, 4093, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 4096, 3, 0, 0, 1646, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3950, 0, 4096, 4096, 0, 0, 0, 3541, 0, 0, 4085, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 1916, 0, 4096, 0, 0, 4006, 0, 4096, 0, 0, 294, 405] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[16, 0, 45, 4004, 0, 0, 0, 0, 41, 4062, 4067, 0, 22, 10, 0, 4077, 63, 4074, 4, 2139, 0, 0, 0, 0, 0, 4051, 0, 323, 4087, 2, 3850, 50, 6, 0, 30, 7, 4081, 4044, 0, 0, 0, 0, 3016, 0, 15, 35, 0, 0, 0, 15, 26, 0, 0, 4064, 9, 2927, 0, 0, 0, 4095, 4076, 0, 0, 0, 3447, 70, 7, 4089, 0, 0, 51, 1, 0, 1, 0, 0, 4076, 4088, 1, 4096, 4059, 0, 2, 0, 1265, 0, 7, 4084, 0, 33, 0, 0, 4026, 12, 0, 0, 16, 11, 0, 0, 0, 0, 0, 0, 0, 4077, 18, 4053, 0, 4064, 4, 400, 4090, 4072, 0, 10, 1, 10, 4085, 47, 0, 4087, 4, 4085, 30, 0, 0, 3060] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 66, 0, 4095, 0, 0, 0, 0, 0, 4092, 4092, 0, 0, 5, 19, 4095, 1319, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3929, 0, 3205, 0, 14, 0, 0, 0, 4077, 0, 122, 0, 1, 3756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 6, 3220, 3039, 0, 0, 0, 0, 3483, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 4095, 0, 3527, 5, 0, 0, 4083, 4096, 4, 0, 0, 630, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2942, 0, 0, 4079, 0, 0, 0, 0, 13, 0, 0, 0, 2146, 4036, 8, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 49, 0, 0, 0, 0, 3, 4093, 1, 0, 0, 0, 4096, 3, 0, 0, 2447, 607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 2, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 84, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1147, 4090, 0, 0, 14, 0, 4092, 304, 4, 4086, 0, 0, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18252, 15106, 18095, 16612, 17304, 15394, 16341, 13968] avg=16384.0000 max=18252 max_rank=0 min=13968 min_rank=7 imbalance_ratio=1.114014 max_load_reduction=112820 max_load_reduction_pct=0.860748 moved_to_echo=63007 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4095, 0, 0, 3, 0, 8, 3760, 4083, 1, 0, 2, 0, 4096, 1319, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 4094, 0, 0, 0, 0, 0, 4065, 0, 2659, 0, 3, 3747, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4094, 0, 0, 0, 0, 4092, 33, 0, 0, 1, 0, 3487, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 3920, 0, 0, 0, 4094, 4096, 2, 0, 0, 630, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1811, 0, 0, 4058, 0, 1, 0, 0, 0, 0, 0, 0, 2146, 3639, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4096, 0, 10, 0, 2464, 608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 0, 4096, 0, 0, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 4095, 0, 0, 21, 0, 4096, 406, 0, 4096, 0, 0, 0, 0, 6, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17841, 18519, 15948, 17010, 15746, 15361, 16230, 14417] avg=16384.0000 max=18519 max_rank=1 min=14417 min_rank=7 imbalance_ratio=1.130310 max_load_reduction=112553 max_load_reduction_pct=0.858711 moved_to_echo=61754 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 76, 0, 2660, 0, 0, 0, 2, 2, 3246, 4096, 0, 1, 0, 0, 4060, 1319, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 0, 2895, 6, 0, 0, 0, 0, 4089, 0, 30, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 679, 0, 3989, 0, 0, 0, 0, 2, 3489, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 4096, 0, 3651, 0, 0, 0, 4013, 4083, 21, 17, 0, 630, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 886, 0, 0, 3922, 21, 0, 0, 0, 0, 1, 0, 0, 2146, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 49, 0, 0, 4094, 1, 8, 100, 0, 4072, 18, 26, 0, 2465, 607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 4, 0, 4092, 0, 4095, 0, 4096, 0, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 4096, 0, 0, 2842, 0, 4096, 0, 0, 3718, 0, 0, 8, 0, 236, 3818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15935, 15016, 16508, 16716, 15163, 15531, 16234, 19969] avg=16384.0000 max=19969 max_rank=7 min=15016 min_rank=1 imbalance_ratio=1.218811 max_load_reduction=111103 max_load_reduction_pct=0.847649 moved_to_echo=66897 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 24, 4075, 0, 0, 0, 1, 12, 3818, 4079, 1, 0, 4, 0, 4094, 1314, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3920, 0, 3542, 14, 2, 0, 33, 1, 3951, 0, 12, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4053, 4092, 0, 0, 0, 12, 2886, 0, 0, 0, 23, 1, 3476, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 4080, 1, 101, 0, 8, 0, 4060, 4095, 0, 2, 0, 629, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 2, 4096, 9, 0, 0, 0, 52, 12, 3, 1, 2131, 4085, 1, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 38, 0, 1373, 0, 2, 4096, 0, 22, 0, 1, 4067, 5, 0, 0, 2472, 605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 30, 0, 0, 0, 14, 0, 4079, 0, 4092, 0, 4096, 0, 0, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1148, 3941, 22, 2, 0, 15, 4086, 4071, 12, 4092, 0, 0, 0, 555, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17900, 15565, 14702, 13400, 18528, 16763, 16255, 17959] avg=16384.0000 max=18528 max_rank=4 min=13400 min_rank=3 imbalance_ratio=1.130859 max_load_reduction=112544 max_load_reduction_pct=0.858643 moved_to_echo=69505 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[28, 0, 15, 2, 0, 0, 5, 0, 78, 3861, 4090, 0, 0, 1, 0, 4078, 1318, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 0, 4094, 6, 0, 0, 0, 0, 3987, 6, 0, 0, 0, 4073, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 664, 199, 4090, 4054, 0, 3, 2762, 341, 4092, 0, 3, 0, 0, 0, 3486, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 59, 0, 0, 0, 4080, 1, 4096, 0, 0, 3, 4094, 4093, 0, 70, 11, 630, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1798, 0, 5, 4092, 0, 0, 17, 7, 0, 18, 1, 0, 2145, 2808, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 5, 0, 0, 3, 4091, 0, 0, 10, 0, 3753, 0, 0, 0, 2465, 606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4089, 4, 4080, 0, 4067, 2, 104, 3931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 3919, 0, 8, 3954, 3, 4034, 0, 0, 4094, 4, 0, 1, 0, 6, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13949, 16112, 19857, 17300, 14987, 15024, 16282, 17561] avg=16384.0000 max=19857 max_rank=2 min=13949 min_rank=0 imbalance_ratio=1.211975 max_load_reduction=111215 max_load_reduction_pct=0.848503 moved_to_echo=63854 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 896, 0, 0, 81, 24, 0, 3704, 4065, 21, 0, 7, 0, 4034, 1308, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3907, 0, 4096, 3, 0, 28, 88, 0, 4054, 12, 3965, 0, 1, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 4031, 4081, 0, 0, 0, 0, 3058, 0, 19, 0, 0, 108, 3465, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 2, 0, 4094, 33, 3500, 203, 90, 0, 3992, 4074, 0, 0, 41, 625, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1409, 0, 17, 3633, 7, 0, 19, 12, 11, 2, 11, 36, 2128, 4084, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 35, 0, 1, 2279, 26, 1, 4077, 0, 44, 0, 0, 4059, 6, 0, 0, 2439, 604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 21, 29, 0, 4087, 14, 4055, 0, 4071, 0, 663, 3918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1143, 4074, 8, 19, 36, 29, 4079, 746, 50, 4025, 0, 0, 0, 3, 0, 456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14608, 20158, 14940, 16860, 15436, 17537, 16865, 14668] avg=16384.0000 max=20158 max_rank=1 min=14608 min_rank=0 imbalance_ratio=1.230347 max_load_reduction=110914 max_load_reduction_pct=0.846207 moved_to_echo=64506 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2335, 0, 0, 0, 0, 15, 2125, 4096, 0, 0, 0, 0, 4095, 1319, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 4025, 0, 0, 0, 0, 1, 4095, 0, 63, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 17, 4096, 3993, 0, 0, 126, 76, 4093, 48, 0, 0, 0, 0, 3489, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 4096, 3, 0, 0, 630, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1646, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 2147, 3950, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3541, 0, 0, 4085, 0, 0, 0, 0, 4096, 0, 12, 0, 2467, 607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1156, 4096, 0, 0, 1916, 0, 4096, 0, 0, 4006, 0, 0, 0, 0, 294, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14458, 16199, 16154, 17176, 15921, 18904, 16291, 15969] avg=16384.0000 max=18904 max_rank=5 min=14458 min_rank=0 imbalance_ratio=1.153809 max_load_reduction=112168 max_load_reduction_pct=0.855774 moved_to_echo=67085 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 45, 4004, 0, 0, 0, 0, 41, 4062, 4067, 0, 22, 10, 0, 4077, 1313, 472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 3915, 4, 2139, 0, 0, 0, 0, 0, 4051, 0, 323, 0, 2, 3850, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 30, 7, 4081, 4044, 0, 0, 0, 0, 3016, 0, 15, 35, 0, 0, 3480, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 26, 0, 0, 4064, 9, 2927, 0, 0, 0, 4095, 4076, 0, 0, 0, 627, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3447, 70, 7, 4089, 0, 0, 51, 1, 0, 1, 0, 0, 2136, 4088, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 2, 0, 1265, 0, 7, 4084, 0, 33, 0, 0, 4026, 12, 0, 0, 2463, 607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 11, 0, 0, 0, 0, 0, 0, 0, 4077, 18, 4053, 0, 4064, 4, 400, 3931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 4072, 0, 10, 1, 10, 4085, 47, 0, 4087, 4, 0, 30, 0, 0, 3060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18129, 14397, 14868, 15998, 17987, 16558, 16574, 16561] avg=16384.0000 max=18129 max_rank=0 min=14397 min_rank=1 imbalance_ratio=1.106506 max_load_reduction=112943 max_load_reduction_pct=0.861687 moved_to_echo=67680 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 256, 128, 22272, 0, 0, 128, 128, 256, 28672, 32768, 128, 128, 128, 128, 32640, 10624, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[18048, 128, 128, 32128, 128, 128, 128, 128, 128, 128, 128, 128, 17152, 30848, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32640, 128, 128, 128, 8576, 128, 128, 32768, 128, 128, 128, 128, 32384, 128, 128, 0, 19712, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 31488, 128, 28160, 128, 128, 128, 128, 128, 32384, 128, 7296, 0, 128, 31744, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 128, 0, 32768, 128, 32768, 0, 32768, 128, 1280, 31488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[9216, 32384, 128, 128, 8832, 128, 32768, 5632, 128, 32256, 128, 0, 128, 640, 640, 8960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 896, 256, 32640, 32640, 128, 128, 3584, 512, 28544, 3200, 128, 128, 128, 128, 27904, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 256, 128, 128, 384, 32768, 128, 25856, 256, 128, 128, 32640, 32768, 128, 128, 128, 5120, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3396, 135, 0, 66, 33, 0, 0, 4096, 4096, 4060, 0, 0, 0, 0, 1254, 8, 0, 0, 7, 4096, 0, 0, 161, 0, 0, 1, 145, 4085, 3999, 4096, 16, 0, 4095, 0, 0, 0, 0, 0, 0, 4092, 1566, 197, 0, 767, 0, 0, 4096, 0, 0, 0, 127, 0, 0, 4085, 4096, 0, 0, 4096, 0, 0, 6, 4096, 0, 0, 4096, 0, 0, 1, 0, 2, 4096, 0, 0, 0, 4092, 4093, 0, 4079, 0, 0, 4096, 881, 0, 3784, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 4, 4087, 5, 0, 1, 0, 302, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 3, 0, 3979, 4093, 0, 0, 0, 0, 3864, 0, 0, 4030, 30] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4066, 0, 0, 3187, 0, 27, 10, 1672, 0, 21, 3810, 4001, 3811, 19, 0, 16, 1, 3219, 0, 18, 5, 0, 4085, 59, 0, 619, 0, 0, 0, 0, 4060, 1845, 4045, 0, 34, 4038, 0, 0, 7, 82, 0, 42, 4036, 1, 0, 14, 1746, 0, 0, 4093, 0, 2, 0, 325, 1, 51, 3988, 4078, 0, 0, 4091, 0, 0, 511, 1794, 70, 0, 3886, 21, 2, 0, 30, 0, 4076, 0, 1, 0, 4096, 4079, 0, 3604, 0, 0, 4082, 2287, 0, 3612, 0, 0, 1, 0, 0, 0, 0, 4086, 0, 46, 3982, 4, 41, 0, 0, 33, 0, 6, 102, 0, 0, 3956, 26, 4064, 6, 103, 0, 272, 0, 14, 3760, 4081, 168, 0, 5, 2, 4087, 9, 36, 4082, 724] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 7, 4095, 421, 0, 75, 895, 9, 0, 4096, 4096, 4096, 0, 17, 0, 0, 3418, 2537, 0, 0, 21, 4096, 0, 1, 148, 0, 0, 9, 0, 4096, 2055, 4069, 0, 1, 4089, 0, 5, 0, 0, 1, 0, 4096, 2290, 1617, 0, 245, 0, 0, 4062, 4, 0, 0, 1217, 6, 0, 4081, 4096, 0, 0, 3533, 0, 0, 398, 3783, 0, 0, 4073, 2, 0, 0, 0, 0, 4087, 0, 3, 0, 4087, 4091, 0, 3930, 0, 0, 4093, 3, 0, 2554, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 2, 4095, 0, 0, 0, 0, 0, 4, 0, 31, 2, 0, 4071, 5, 4096, 0, 0, 0, 0, 0, 0, 3727, 4024, 0, 0, 0, 0, 0, 4, 0, 4093, 23] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4081, 0, 0, 133, 95, 4, 5, 4033, 4096, 3801, 0, 0, 285, 0, 4043, 1831, 0, 0, 3, 4096, 0, 2, 70, 0, 29, 21, 51, 4096, 2081, 4089, 0, 0, 4096, 0, 0, 0, 0, 14, 0, 4085, 631, 0, 0, 3589, 2, 0, 3950, 0, 0, 0, 286, 0, 6, 4096, 3608, 0, 11, 76, 0, 0, 971, 3667, 0, 0, 4091, 10, 0, 1390, 0, 0, 4096, 0, 1, 0, 4022, 4093, 0, 3992, 0, 0, 4096, 4058, 0, 3704, 0, 0, 0, 0, 0, 0, 30, 4091, 0, 33, 4078, 5, 0, 14, 0, 0, 0, 0, 8, 0, 0, 4095, 0, 4093, 0, 0, 0, 0, 824, 0, 4058, 4084, 0, 0, 0, 0, 1926, 0, 0, 48, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4087, 9, 0, 1568, 0, 27, 0, 0, 0, 2, 4051, 4093, 3986, 4, 0, 1680, 0, 4084, 893, 1, 0, 0, 4075, 9, 0, 0, 5, 19, 50, 3760, 4095, 4091, 4094, 13, 10, 3754, 0, 0, 0, 0, 0, 3, 4085, 0, 0, 956, 0, 0, 2, 4017, 0, 2, 0, 0, 146, 0, 3874, 3155, 25, 0, 0, 0, 15, 0, 4068, 0, 0, 3972, 11, 46, 3543, 2, 0, 4094, 190, 0, 40, 4088, 4082, 3, 4096, 0, 3, 4068, 0, 4, 989, 0, 0, 8, 317, 3, 0, 1, 3424, 0, 0, 4078, 0, 13, 0, 40, 3922, 0, 1, 0, 0, 0, 4087, 0, 4094, 0, 6, 0, 0, 109, 0, 4070, 4094, 0, 1, 2, 0, 4093, 0, 0, 670, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4072, 20, 1, 380, 811, 0, 12, 4014, 4078, 3992, 1, 9, 80, 9, 0, 3642, 0, 0, 44, 4096, 0, 0, 0, 0, 0, 0, 36, 3920, 1474, 4085, 0, 0, 4093, 34, 0, 3, 0, 0, 0, 4089, 4095, 768, 0, 2588, 0, 0, 4096, 0, 2, 6, 0, 0, 0, 3951, 4095, 7, 2, 4049, 26, 10, 477, 4089, 11, 24, 4093, 2, 0, 0, 21, 0, 4085, 0, 21, 0, 4025, 4087, 18, 4096, 0, 0, 4079, 0, 0, 2886, 0, 1, 0, 0, 0, 1, 0, 4096, 19, 13, 4072, 0, 37, 645, 0, 2, 0, 0, 10, 0, 6, 4002, 16, 4078, 0, 32, 0, 31, 0, 0, 2814, 4052, 0, 0, 2, 1, 164, 0, 0, 4000, 178] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 29, 3841, 792, 0, 0, 0, 0, 0, 4095, 4064, 4096, 0, 0, 401, 0, 2854, 3971, 0, 0, 0, 4096, 0, 0, 0, 0, 13, 68, 26, 3989, 3849, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 4, 2438, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 3981, 4066, 0, 0, 1898, 0, 0, 61, 3675, 0, 0, 4089, 0, 0, 216, 0, 0, 4096, 56, 0, 0, 4087, 4096, 0, 4096, 88, 0, 4096, 17, 0, 3965, 0, 0, 0, 5, 0, 12, 0, 4056, 0, 0, 4096, 0, 0, 0, 83, 154, 0, 0, 0, 0, 0, 4096, 0, 4041, 0, 0, 0, 0, 115, 0, 4096, 4096, 0, 0, 0, 0, 548, 0, 0, 4067, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4016, 0, 0, 1629, 237, 63, 0, 4083, 0, 0, 4060, 4070, 4084, 23, 0, 637, 0, 598, 3030, 0, 14, 0, 4069, 10, 0, 0, 0, 0, 271, 298, 3994, 4090, 4074, 0, 0, 4079, 0, 0, 25, 0, 0, 0, 4037, 1966, 3769, 138, 2353, 16, 4, 3940, 0, 12, 0, 0, 18, 0, 4070, 4081, 6, 40, 3566, 0, 37, 22, 4013, 0, 0, 4068, 17, 23, 91, 0, 0, 4085, 53, 0, 27, 4083, 4033, 28, 4096, 20, 11, 4068, 1667, 0, 3244, 0, 0, 10, 0, 18, 0, 0, 4073, 0, 0, 4077, 28, 0, 0, 3, 76, 0, 0, 0, 4, 0, 4038, 2, 3385, 0, 0, 0, 0, 205, 0, 684, 4096, 0, 19, 0, 4, 589, 0, 0, 605, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3396, 135, 0, 66, 33, 0, 0, 4096, 3491, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 8, 0, 0, 7, 4096, 0, 0, 161, 0, 0, 1, 145, 4085, 3999, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 4095, 0, 0, 0, 0, 0, 0, 4092, 1566, 197, 0, 767, 0, 144, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 127, 0, 0, 4085, 4096, 0, 0, 4096, 0, 0, 6, 689, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 2, 0, 0, 0, 0, 4092, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 4096, 881, 0, 3784, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4087, 5, 0, 1, 0, 302, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 3407, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3979, 4093, 0, 0, 0, 0, 3864, 0, 0, 4030, 30, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15277, 13901, 15017, 17656, 16170, 16966, 16145, 19940] avg=16384.0000 max=19940 max_rank=7 min=13901 min_rank=1 imbalance_ratio=1.217041 max_load_reduction=111132 max_load_reduction_pct=0.847870 moved_to_echo=69221 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4081, 0, 0, 133, 95, 4, 5, 4033, 3491, 3801, 0, 0, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 1831, 0, 0, 3, 4096, 0, 2, 70, 0, 29, 21, 51, 4096, 2081, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 14, 0, 4085, 631, 0, 0, 3589, 2, 144, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3950, 0, 0, 0, 286, 0, 6, 4096, 3608, 0, 11, 76, 0, 0, 971, 690, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3667, 0, 0, 4091, 10, 0, 1390, 0, 0, 0, 0, 1, 0, 4022, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 0, 0, 4096, 4058, 0, 3704, 0, 0, 0, 0, 0, 0, 30, 4091, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4078, 5, 0, 14, 0, 0, 0, 0, 8, 0, 0, 4095, 0, 4093, 0, 3406, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 824, 0, 4058, 4084, 0, 0, 0, 0, 1926, 0, 0, 48, 3, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15928, 16468, 16694, 14155, 17064, 19992, 15888, 14883] avg=16384.0000 max=19992 max_rank=5 min=14155 min_rank=3 imbalance_ratio=1.220215 max_load_reduction=111080 max_load_reduction_pct=0.847473 moved_to_echo=67827 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3187, 0, 27, 10, 1672, 0, 21, 3810, 3411, 3811, 19, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3219, 0, 18, 5, 0, 4085, 59, 0, 619, 0, 0, 0, 0, 4060, 1845, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 34, 4038, 0, 0, 7, 82, 0, 42, 4036, 1, 0, 14, 1746, 0, 140, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2, 0, 325, 1, 51, 3988, 4078, 0, 0, 4091, 0, 0, 511, 684, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1794, 70, 0, 3886, 21, 2, 0, 30, 0, 0, 0, 1, 0, 4096, 3870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3604, 0, 0, 4082, 2287, 0, 3612, 0, 0, 1, 0, 0, 0, 0, 4086, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 3982, 4, 41, 0, 0, 33, 0, 6, 102, 0, 0, 3956, 26, 4064, 6, 3382, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 272, 0, 14, 3760, 4081, 168, 0, 5, 2, 4087, 9, 36, 4082, 724, 3922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15984, 14055, 14229, 18274, 13770, 17693, 15802, 21265] avg=16384.0000 max=21265 max_rank=7 min=13770 min_rank=4 imbalance_ratio=1.297913 max_load_reduction=109807 max_load_reduction_pct=0.837761 moved_to_echo=68530 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 7, 4095, 421, 0, 75, 895, 9, 0, 4096, 3491, 4096, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3418, 2537, 0, 0, 21, 4096, 0, 1, 148, 0, 0, 9, 0, 4096, 2055, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 1, 4089, 0, 5, 0, 0, 1, 0, 4096, 2290, 1617, 0, 245, 0, 144, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4, 0, 0, 1217, 6, 0, 4081, 4096, 0, 0, 3533, 0, 0, 398, 689, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3783, 0, 0, 4073, 2, 0, 0, 0, 0, 0, 0, 3, 0, 4087, 3881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 0, 0, 4093, 3, 0, 2554, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 4, 0, 31, 2, 0, 4071, 5, 4096, 0, 3406, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3727, 4024, 0, 0, 0, 0, 0, 4, 0, 4093, 23, 3932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17202, 16526, 16601, 18547, 15829, 14697, 15867, 15803] avg=16384.0000 max=18547 max_rank=3 min=14697 min_rank=5 imbalance_ratio=1.132019 max_load_reduction=112525 max_load_reduction_pct=0.858498 moved_to_echo=62196 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 0, 1568, 0, 27, 0, 0, 0, 2, 4051, 3489, 3986, 4, 0, 1680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 893, 1, 0, 0, 4075, 9, 0, 0, 5, 19, 50, 3760, 4095, 4091, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 13, 10, 3754, 0, 0, 0, 0, 0, 3, 4085, 0, 0, 956, 0, 0, 143, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4017, 0, 2, 0, 0, 146, 0, 3874, 3155, 25, 0, 0, 0, 15, 0, 689, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 3972, 11, 46, 3543, 2, 0, 0, 190, 0, 40, 4088, 3873, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 4068, 0, 4, 989, 0, 0, 8, 317, 3, 0, 1, 3424, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 13, 0, 40, 3922, 0, 1, 0, 0, 0, 4087, 0, 4094, 0, 3398, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 109, 0, 4070, 4094, 0, 1, 2, 0, 4093, 0, 0, 670, 0, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14816, 21226, 13102, 12386, 19836, 12934, 19788, 16984] avg=16384.0000 max=21226 max_rank=1 min=12386 min_rank=3 imbalance_ratio=1.295532 max_load_reduction=109846 max_load_reduction_pct=0.838058 moved_to_echo=69542 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4072, 20, 1, 380, 811, 0, 12, 4014, 3469, 3992, 1, 9, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3642, 0, 0, 44, 4096, 0, 0, 0, 0, 0, 0, 36, 3920, 1474, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4093, 34, 0, 3, 0, 0, 0, 4089, 4095, 768, 0, 2588, 0, 146, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 6, 0, 0, 0, 3951, 4095, 7, 2, 4049, 26, 10, 477, 690, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 11, 24, 4093, 2, 0, 0, 21, 0, 0, 0, 21, 0, 4025, 3862, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4079, 0, 0, 2886, 0, 1, 0, 0, 0, 1, 0, 4096, 19, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4072, 0, 37, 645, 0, 2, 0, 0, 10, 0, 6, 4002, 16, 4078, 0, 3406, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 31, 0, 0, 2814, 4052, 0, 0, 2, 1, 164, 0, 0, 4000, 178, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16861, 13368, 19951, 17874, 16166, 15206, 16443, 15203] avg=16384.0000 max=19951 max_rank=2 min=13368 min_rank=1 imbalance_ratio=1.217712 max_load_reduction=111121 max_load_reduction_pct=0.847786 moved_to_echo=63018 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 29, 3841, 792, 0, 0, 0, 0, 0, 4095, 3465, 4096, 0, 0, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2854, 3971, 0, 0, 0, 4096, 0, 0, 0, 0, 13, 68, 26, 3989, 3849, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 4, 2438, 6, 142, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3981, 4066, 0, 0, 1898, 0, 0, 61, 690, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3675, 0, 0, 4089, 0, 0, 216, 0, 0, 0, 56, 0, 0, 4087, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 88, 0, 4096, 17, 0, 3965, 0, 0, 0, 5, 0, 12, 0, 4056, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 83, 154, 0, 0, 0, 0, 0, 4096, 0, 4041, 0, 3406, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 4096, 4096, 0, 0, 0, 0, 548, 0, 0, 4067, 0, 3940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16719, 19011, 14834, 15249, 16009, 16356, 16032, 16862] avg=16384.0000 max=19011 max_rank=1 min=14834 min_rank=2 imbalance_ratio=1.160339 max_load_reduction=112061 max_load_reduction_pct=0.854958 moved_to_echo=65259 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1629, 237, 63, 0, 4083, 0, 0, 4060, 3469, 4084, 23, 0, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 598, 3030, 0, 14, 0, 4069, 10, 0, 0, 0, 0, 271, 298, 3994, 4090, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 4079, 0, 0, 25, 0, 0, 0, 4037, 1966, 3769, 138, 2353, 16, 143, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3940, 0, 12, 0, 0, 18, 0, 4070, 4081, 6, 40, 3566, 0, 37, 22, 677, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 4068, 17, 23, 91, 0, 0, 0, 53, 0, 27, 4083, 3825, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 20, 11, 4068, 1667, 0, 3244, 0, 0, 10, 0, 18, 0, 0, 4073, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 28, 0, 0, 3, 76, 0, 0, 0, 4, 0, 4038, 2, 3385, 0, 3339, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 684, 4096, 0, 19, 0, 4, 589, 0, 0, 605, 0, 3930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18285, 16517, 20644, 16931, 16228, 17228, 15107, 10132] avg=16384.0000 max=20644 max_rank=2 min=10132 min_rank=7 imbalance_ratio=1.260010 max_load_reduction=110428 max_load_reduction_pct=0.842499 moved_to_echo=58695 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[29312, 128, 128, 32384, 128, 128, 5248, 128, 128, 0, 384, 128, 128, 32640, 30976, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 19584, 16000, 128, 128, 128, 32768, 128, 128, 1024, 128, 128, 512, 4352, 32384, 23552, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32640, 128, 128, 768, 128, 4608, 128, 128, 256, 128, 128, 32512, 128, 32000, 128, 27264, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32000, 128, 128, 32768, 8960, 128, 24832, 0, 128, 128, 384, 128, 128, 128, 32128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 32384, 128, 128, 128, 128, 128, 128, 32640, 10624, 6400, 1152, 13824, 128, 1152, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32384, 128, 128, 128, 2048, 256, 128, 32128, 31360, 128, 128, 21376, 128, 128, 2560, 5504, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 25984, 1664, 128, 768, 7680, 128, 128, 32256, 27776, 32000, 128, 128, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 0, 384, 1280, 128, 27264, 32640, 256, 128, 128, 128, 15360, 128, 128, 21632, 1024, 31488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 34, 666, 0, 19, 3968, 0, 0, 3, 0, 7, 4076, 1067, 4087, 39, 0, 3564, 326, 25, 0, 2, 88, 83, 0, 0, 0, 9, 4092, 0, 7, 0, 1, 0, 0, 4090, 23, 13, 4093, 4096, 0, 0, 0, 4074, 182, 464, 4096, 4095, 4096, 0, 0, 4088, 0, 0, 4096, 0, 0, 4092, 4096, 4066, 0, 0, 6, 0, 1, 3529, 0, 0, 3607, 0, 219, 0, 1, 4096, 3199, 0, 0, 4086, 199, 0, 8, 0, 4096, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 58, 3, 4080, 4077, 0, 0, 402, 4062, 4096, 0, 0, 0, 0, 3742, 0, 0, 61, 0, 0, 0, 12, 0, 10, 3, 0, 1, 4034, 3290, 1, 17, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 18, 4069, 0, 0, 1449, 0, 0, 0, 0, 0, 4096, 4092, 4096, 0, 0, 1219, 713, 0, 0, 0, 38, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 12, 0, 4096, 0, 69, 4077, 4057, 0, 0, 0, 2234, 0, 2827, 4096, 4096, 4083, 0, 3, 4091, 5, 0, 4096, 0, 0, 4029, 4063, 4084, 0, 5, 4, 0, 0, 718, 0, 0, 4096, 61, 212, 0, 2, 4094, 4042, 0, 0, 4096, 790, 28, 0, 0, 4084, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4047, 4096, 0, 0, 242, 4096, 4096, 0, 0, 0, 0, 967, 7, 0, 14, 0, 0, 13, 0, 0, 993, 0, 0, 0, 4078, 4096, 0, 2, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 147, 18, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 2830, 4096, 0, 0, 3941, 0, 16, 0, 0, 117, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4059, 4096, 0, 0, 428, 4086, 0, 0, 4096, 4096, 3934, 0, 0, 4096, 54, 0, 4096, 8, 0, 4096, 3443, 4087, 27, 0, 0, 1, 1, 4029, 105, 4, 4059, 60, 1, 0, 660, 4096, 4085, 0, 5, 4096, 4043, 0, 0, 0, 4053, 0, 4078, 0, 0, 5, 0, 0, 0, 0, 2, 0, 1493, 4089, 0, 0, 67, 3515, 4096, 0, 0, 0, 0, 3963, 0, 0, 68, 0, 0, 0, 0, 6, 0, 8, 21, 0, 4001, 4042, 0, 0, 68, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4077, 0, 211, 2773, 11, 55, 2066, 0, 10, 1, 18, 31, 4018, 4088, 4071, 34, 26, 0, 208, 9, 0, 0, 93, 0, 0, 4, 0, 0, 3908, 0, 0, 0, 1, 0, 0, 4059, 0, 351, 4095, 3999, 0, 17, 0, 2075, 155, 3376, 4049, 4074, 3928, 0, 51, 4066, 0, 13, 4091, 0, 0, 4045, 3849, 4032, 0, 15, 2, 6, 52, 2522, 8, 36, 3844, 97, 75, 2, 37, 4062, 3968, 0, 0, 4082, 1999, 783, 0, 14, 4059, 0, 4079, 34, 0, 0, 0, 11, 0, 24, 0, 0, 4056, 4078, 1, 16, 628, 4061, 4079, 16, 0, 0, 0, 0, 0, 8, 0, 4, 0, 34, 288, 83, 3058, 0, 0, 35, 4088, 688, 0, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4070, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4017, 4096, 0, 0, 8, 3305, 0, 3460, 4096, 4096, 4095, 0, 44, 4096, 16, 0, 4095, 0, 0, 4062, 2656, 4059, 0, 0, 0, 0, 0, 3418, 44, 0, 4010, 3255, 0, 0, 6, 4091, 4096, 0, 0, 4096, 3956, 122, 0, 0, 4096, 0, 4096, 60, 0, 0, 0, 0, 0, 0, 0, 0, 104, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 0, 0, 4096, 4033, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[15, 2965, 589, 0, 0, 4, 0, 193, 0, 0, 0, 0, 0, 4088, 0, 4086, 0, 0, 4094, 0, 0, 2, 0, 0, 2879, 0, 3, 0, 0, 4063, 0, 12, 11, 4, 0, 601, 4092, 0, 0, 3758, 4028, 0, 2, 0, 3279, 0, 2801, 4076, 4073, 4074, 0, 56, 4087, 0, 5, 3770, 0, 7, 3336, 4094, 4072, 0, 0, 0, 0, 2, 554, 0, 0, 3486, 4093, 0, 0, 1207, 3889, 4096, 9, 0, 4088, 4096, 0, 9, 0, 4096, 0, 4094, 32, 27, 0, 0, 3, 0, 0, 0, 0, 0, 4092, 1, 1, 0, 2889, 4091, 0, 5, 8, 34, 4083, 0, 3, 0, 0, 21, 0, 0, 5, 696, 2, 39, 2, 4091, 3961, 0, 0, 26, 13, 5, 4, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4066, 11, 67, 456, 0, 0, 82, 24, 0, 0, 3, 0, 3935, 4014, 4051, 0, 0, 159, 3425, 34, 11, 1, 0, 15, 365, 0, 10, 0, 4096, 111, 2, 8, 1, 13, 0, 4053, 12, 1010, 4082, 4060, 0, 15, 6, 3122, 0, 4059, 3971, 4096, 4096, 0, 0, 4075, 0, 0, 4092, 0, 0, 4088, 4096, 4030, 7, 0, 0, 1, 40, 223, 0, 0, 3391, 0, 1471, 5, 3, 4091, 2415, 8, 42, 4080, 0, 5, 0, 0, 3921, 0, 4081, 0, 0, 0, 0, 0, 0, 6, 2, 0, 4013, 4090, 0, 8, 4086, 4071, 4074, 0, 0, 0, 2, 646, 44, 0, 2, 0, 0, 0, 273, 0, 2373, 0, 0, 0, 4036, 1526, 0, 0, 19, 25, 0, 0, 69] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[30, 4048, 1, 0, 4056, 9, 0, 123, 82, 3, 0, 0, 0, 4074, 1340, 4075, 0, 0, 967, 3736, 0, 7, 0, 0, 423, 2785, 15, 0, 23, 4095, 0, 0, 0, 13, 0, 5, 4080, 0, 0, 4072, 4070, 0, 5, 0, 0, 0, 1178, 4064, 4084, 4091, 24, 45, 4059, 0, 8, 4048, 0, 24, 3975, 4087, 4073, 0, 0, 0, 0, 0, 3658, 10, 0, 2502, 46, 0, 0, 0, 4072, 3750, 14, 0, 4066, 1003, 139, 33, 14, 4096, 21, 4086, 44, 1, 5, 11, 2, 0, 0, 0, 14, 3935, 4072, 4, 0, 1191, 4017, 4062, 8, 0, 1, 0, 1961, 0, 0, 0, 10, 5, 3, 0, 0, 0, 29, 15, 24, 4048, 4074, 11, 0, 116, 0, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 147, 18, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 2830, 3035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 0, 16, 0, 0, 117, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4096, 0, 0, 428, 4086, 0, 0, 2897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 0, 0, 0, 54, 0, 4096, 8, 0, 4096, 3443, 473, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4029, 105, 4, 4059, 60, 1, 0, 660, 3251, 4085, 0, 5, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 4078, 0, 0, 5, 0, 0, 0, 0, 2, 0, 1493, 3614, 1199, 334, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 67, 3515, 4096, 0, 0, 0, 0, 3963, 0, 0, 68, 0, 0, 845, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8, 21, 0, 4001, 4042, 0, 0, 68, 0, 0, 0, 0, 4096, 3882, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14227, 16288, 15566, 16131, 20304, 14992, 17370, 16194] avg=16384.0000 max=20304 max_rank=4 min=14227 min_rank=0 imbalance_ratio=1.239258 max_load_reduction=110768 max_load_reduction_pct=0.845093 moved_to_echo=68860 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 34, 666, 0, 19, 3968, 0, 0, 3, 0, 7, 4076, 1067, 3029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 3564, 326, 25, 0, 2, 88, 83, 0, 0, 0, 9, 4092, 0, 7, 4095, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 23, 13, 4093, 4096, 0, 0, 0, 4074, 182, 464, 2897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 4096, 470, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3529, 0, 0, 3607, 0, 219, 0, 1, 3251, 3199, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 58, 3, 4080, 3596, 1199, 333, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 402, 4062, 4096, 0, 0, 0, 0, 3742, 0, 0, 61, 0, 0, 845, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 10, 3, 0, 1, 4034, 3290, 1, 17, 0, 0, 0, 0, 0, 4090, 3875, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16965, 16345, 15843, 16856, 14006, 17643, 18010, 15404] avg=16384.0000 max=18010 max_rank=6 min=14006 min_rank=4 imbalance_ratio=1.099243 max_load_reduction=113062 max_load_reduction_pct=0.862595 moved_to_echo=65063 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4077, 0, 211, 2773, 11, 55, 2066, 0, 10, 1, 18, 31, 4018, 4088, 3016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 26, 0, 208, 9, 0, 0, 93, 0, 0, 4, 0, 0, 3908, 0, 0, 4074, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 351, 4095, 3999, 0, 17, 0, 2075, 155, 3376, 2863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3928, 0, 51, 0, 0, 13, 4091, 0, 0, 4045, 3849, 466, 0, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 52, 2522, 8, 36, 3844, 97, 75, 2, 37, 3224, 3968, 0, 0, 0, 1999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 783, 0, 14, 4059, 0, 4079, 34, 0, 0, 0, 11, 0, 24, 0, 0, 4056, 3566, 1186, 332, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 1, 16, 628, 4061, 4079, 16, 0, 0, 0, 0, 0, 8, 0, 4, 0, 838, 723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 288, 83, 3058, 0, 0, 35, 4088, 688, 0, 0, 0, 0, 0, 2, 0, 4059, 3854, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20375, 12367, 16932, 16460, 15870, 18356, 14452, 16260] avg=16384.0000 max=20375 max_rank=0 min=12367 min_rank=1 imbalance_ratio=1.243591 max_load_reduction=110697 max_load_reduction_pct=0.844551 moved_to_echo=64938 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4066, 11, 67, 456, 0, 0, 82, 24, 0, 0, 3, 0, 3935, 4014, 2992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 3425, 34, 11, 1, 0, 15, 365, 0, 10, 0, 4096, 111, 2, 4096, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 13, 0, 0, 12, 1010, 4082, 4060, 0, 15, 6, 3122, 0, 4059, 2803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 4088, 4096, 463, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 40, 223, 0, 0, 3391, 0, 1471, 5, 3, 3245, 2415, 8, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3921, 0, 4081, 0, 0, 0, 0, 0, 0, 6, 2, 0, 4013, 3567, 1168, 335, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 8, 4086, 4071, 4074, 0, 0, 0, 2, 646, 44, 0, 2, 0, 0, 846, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 2373, 0, 0, 0, 4036, 1526, 0, 0, 19, 25, 0, 0, 69, 4053, 3862, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15650, 16335, 19191, 16842, 10844, 17311, 18593, 16306] avg=16384.0000 max=19191 max_rank=2 min=10844 min_rank=4 imbalance_ratio=1.171326 max_load_reduction=111881 max_load_reduction_pct=0.853584 moved_to_echo=63054 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[15, 2965, 589, 0, 0, 4, 0, 193, 0, 0, 0, 0, 0, 4088, 0, 3028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 2879, 0, 3, 0, 0, 4063, 0, 12, 4073, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4, 0, 601, 0, 0, 0, 3758, 4028, 0, 2, 0, 3279, 0, 2801, 2883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 56, 0, 0, 5, 3770, 0, 7, 3336, 4094, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 554, 0, 0, 3486, 4093, 0, 0, 1207, 3087, 4096, 9, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 4094, 32, 27, 0, 0, 3, 0, 0, 0, 0, 0, 3601, 1193, 333, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1, 0, 2889, 4091, 0, 5, 8, 34, 4083, 0, 3, 0, 0, 21, 802, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 696, 2, 39, 2, 4091, 3961, 0, 0, 26, 13, 5, 4, 0, 4092, 3873, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10882, 19143, 17367, 15813, 20630, 13602, 16755, 16880] avg=16384.0000 max=20630 max_rank=4 min=10882 min_rank=0 imbalance_ratio=1.259155 max_load_reduction=110442 max_load_reduction_pct=0.842606 moved_to_echo=67867 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4086, 0, 0, 4096, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 4096, 0, 0, 8, 3305, 0, 3460, 2897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 44, 0, 16, 0, 4095, 0, 0, 4062, 2656, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3418, 44, 0, 4010, 3255, 0, 0, 6, 3247, 4096, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 4096, 0, 4096, 60, 0, 0, 0, 0, 0, 0, 0, 0, 104, 3590, 1199, 334, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 844, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 0, 0, 4096, 4033, 0, 0, 0, 0, 0, 0, 0, 4096, 3882, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15332, 16283, 17783, 15437, 22032, 13815, 13909, 16481] avg=16384.0000 max=22032 max_rank=4 min=13815 min_rank=5 imbalance_ratio=1.344727 max_load_reduction=109040 max_load_reduction_pct=0.831909 moved_to_echo=66237 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[30, 4048, 1, 0, 4056, 9, 0, 123, 82, 3, 0, 0, 0, 4074, 1340, 3020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 967, 3736, 0, 7, 0, 0, 423, 2785, 15, 0, 23, 4095, 0, 0, 4084, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 5, 0, 0, 0, 4072, 4070, 0, 5, 0, 0, 0, 1178, 2874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 24, 45, 0, 0, 8, 4048, 0, 24, 3975, 4087, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3658, 10, 0, 2502, 46, 0, 0, 0, 3232, 3750, 14, 0, 0, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 33, 14, 4096, 21, 4086, 44, 1, 5, 11, 2, 0, 0, 0, 14, 3935, 3602, 1190, 332, 212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 4, 0, 1191, 4017, 4062, 8, 0, 1, 0, 1961, 0, 0, 0, 10, 5, 840, 723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 29, 15, 24, 4048, 4074, 11, 0, 116, 0, 0, 2, 0, 4080, 3847, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16786, 20130, 12217, 16773, 14215, 17737, 16894, 16320] avg=16384.0000 max=20130 max_rank=1 min=12217 min_rank=2 imbalance_ratio=1.228638 max_load_reduction=110942 max_load_reduction_pct=0.846420 moved_to_echo=65166 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 18, 4069, 0, 0, 1449, 0, 0, 0, 0, 0, 4096, 4092, 3035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1219, 713, 0, 0, 0, 38, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 69, 4077, 4057, 0, 0, 0, 2234, 0, 2827, 2897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 3, 0, 5, 0, 4096, 0, 0, 4029, 4063, 472, 0, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 718, 0, 0, 4096, 61, 212, 0, 2, 3250, 4042, 0, 0, 0, 790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 4084, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4047, 3612, 1199, 334, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 242, 4096, 4096, 0, 0, 0, 0, 967, 7, 0, 14, 0, 0, 844, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 993, 0, 0, 0, 4078, 4096, 0, 2, 0, 0, 0, 0, 0, 4096, 3878, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20855, 14181, 16173, 16760, 13171, 17616, 15089, 17227] avg=16384.0000 max=20855 max_rank=0 min=13171 min_rank=4 imbalance_ratio=1.272888 max_load_reduction=110217 max_load_reduction_pct=0.840889 moved_to_echo=63103 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 128, 128, 6656, 30848, 32768, 128, 128, 128, 128, 15488, 128, 128, 256, 128, 128, 6784, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 31616, 768, 384, 12032, 128, 128, 7936, 128, 128, 128, 128, 128, 32512, 21632, 24192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 640, 128, 7552, 128, 128, 128, 32512, 25728, 128, 128, 256, 128, 128, 128, 128, 32768, 30976, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[1152, 128, 128, 32512, 128, 32768, 256, 128, 128, 128, 128, 0, 128, 128, 128, 21760, 28800, 9600, 2688, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 128, 640, 0, 128, 1536, 32256, 32512, 0, 128, 512, 22272, 384, 18176, 23040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 18048, 8448, 128, 128, 128, 384, 3456, 3200, 128, 128, 128, 32640, 128, 128, 32768, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32512, 128, 256, 0, 128, 128, 32384, 128, 128, 31744, 30464, 3840, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 18688, 256, 128, 29056, 7680, 2048, 128, 1920, 25856, 29696, 128, 128, 0, 16128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 0, 4096, 4046, 0, 43, 0, 3588, 4082, 4096, 0, 0, 4096, 0, 19, 0, 0, 16, 0, 0, 6, 0, 0, 4096, 0, 0, 4096, 0, 4063, 0, 0, 0, 0, 540, 0, 0, 0, 865, 0, 4096, 0, 0, 3, 4069, 0, 3281, 17, 4060, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4096, 4072, 0, 0, 4080, 4096, 0, 17, 33, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 39, 4096, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4052, 0, 4091, 4096, 4081, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[3, 0, 37, 10, 0, 25, 482, 0, 4096, 4075, 0, 0, 0, 242, 4078, 4083, 0, 0, 4082, 8, 0, 0, 15, 0, 17, 0, 0, 22, 7, 3, 3, 52, 4061, 0, 4067, 0, 20, 50, 1282, 3859, 0, 0, 0, 3, 126, 4062, 1, 0, 0, 4059, 0, 3938, 4079, 4048, 171, 4087, 9, 0, 28, 40, 0, 0, 0, 47, 94, 15, 4089, 4006, 4083, 241, 0, 2639, 4072, 0, 0, 50, 34, 13, 0, 4066, 17, 46, 55, 42, 56, 17, 0, 3909, 14, 2, 3, 0, 4037, 4029, 0, 0, 0, 0, 11, 0, 3994, 0, 17, 16, 0, 29, 0, 0, 0, 11, 1, 4064, 0, 34, 0, 4093, 5, 1, 4045, 4044, 4020, 8, 4072, 4079, 3522, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 16, 0, 0, 606, 0, 4087, 4094, 82, 0, 0, 164, 4086, 4085, 0, 0, 3689, 0, 11, 12, 0, 23, 59, 0, 0, 0, 5, 3384, 23, 0, 4062, 0, 4083, 0, 0, 0, 216, 4096, 10, 0, 7, 0, 0, 4078, 0, 0, 0, 4092, 3, 4060, 4086, 4046, 0, 4090, 12, 4, 7, 34, 0, 0, 2, 16, 0, 0, 3483, 4095, 4082, 546, 9, 4009, 4085, 0, 10, 2, 4, 0, 0, 4042, 0, 0, 0, 0, 288, 35, 0, 4077, 0, 0, 0, 38, 4089, 4096, 0, 4, 0, 0, 0, 0, 4096, 2, 0, 0, 5, 49, 0, 4, 4, 0, 0, 4071, 0, 10, 9, 3834, 1, 0, 4093, 4075, 505, 0, 4096, 4094, 3699, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 43, 1, 43, 0, 0, 0, 4096, 4096, 0, 16, 0, 669, 4096, 4096, 0, 5, 3979, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 2, 105, 0, 4004, 0, 2, 0, 101, 0, 4089, 113, 0, 0, 4079, 0, 4096, 4095, 4084, 0, 4096, 0, 9, 0, 1, 0, 69, 0, 0, 27, 0, 449, 4057, 4093, 0, 0, 3838, 4091, 0, 10, 3099, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 5, 0, 1, 4096, 4095, 6, 0, 0, 0, 0, 0, 4008, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 4003, 4092, 4096, 0, 4089, 4093, 4037, 0, 66, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 344, 0, 50, 0, 4096, 4054, 0, 140, 0, 4057, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 13, 0, 0, 5, 0, 0, 3900, 0, 0, 4096, 0, 4096, 0, 0, 3, 36, 22, 3, 18, 0, 285, 59, 4090, 2, 0, 69, 3780, 0, 3987, 4078, 4096, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 7, 0, 75, 4079, 4046, 0, 0, 4081, 4089, 0, 2, 1686, 0, 0, 0, 4096, 0, 0, 0, 4, 66, 0, 0, 4089, 3, 11, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 4030, 0, 0, 0, 4096, 0, 0, 4095, 4096, 2221, 0, 4021, 4096, 4003, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 13, 5, 0, 0, 4063, 4078, 0, 0, 0, 4066, 4075, 4070, 0, 3, 4095, 4, 296, 0, 0, 0, 0, 3, 1, 16, 0, 4072, 0, 0, 4089, 0, 3869, 1, 0, 0, 29, 30, 1, 0, 0, 3991, 0, 4086, 0, 2, 233, 3823, 0, 90, 1450, 4094, 15, 4090, 0, 0, 6, 0, 31, 319, 5, 9, 0, 0, 4092, 4081, 4091, 1, 4, 3989, 4093, 0, 0, 1202, 5, 0, 2, 2722, 3, 0, 0, 0, 286, 0, 0, 4095, 2, 44, 0, 6, 4086, 4082, 0, 0, 57, 5, 2, 0, 4084, 0, 4, 4, 0, 40, 0, 3, 20, 3, 0, 4093, 13, 0, 30, 4057, 0, 8, 3909, 4093, 2345, 0, 4093, 4078, 4096, 17, 4, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 18, 60, 0, 0, 5, 0, 1, 4079, 4065, 0, 0, 7, 2712, 4082, 4067, 43, 0, 4090, 18, 32, 0, 1, 0, 7, 18, 0, 0, 0, 3966, 20, 0, 4069, 0, 4057, 0, 0, 0, 109, 3288, 0, 0, 10, 100, 6, 4086, 0, 0, 10, 4039, 5, 3937, 4062, 4032, 11, 4079, 0, 0, 0, 0, 1, 103, 0, 0, 24, 0, 2320, 4090, 4073, 0, 0, 3891, 4080, 0, 12, 0, 0, 0, 0, 4013, 0, 0, 4, 0, 20, 0, 156, 4087, 8, 43, 0, 0, 4052, 4077, 0, 7, 0, 4, 7, 27, 3964, 20, 10, 18, 36, 47, 4, 16, 0, 11, 18, 4096, 3, 0, 0, 4065, 0, 0, 4062, 4059, 56, 0, 4096, 4082, 4096, 12, 0, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 1, 0, 4041, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 2550, 0, 0, 4096, 0, 4096, 0, 0, 75, 53, 4075, 0, 0, 0, 2, 4, 4095, 0, 0, 0, 4093, 0, 4096, 3791, 4096, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 849, 4093, 0, 0, 1003, 0, 1, 0, 4091, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 1, 1, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4088, 4091, 4096, 0, 4095, 4096, 4025, 0, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 16, 0, 0, 606, 0, 1936, 4094, 82, 0, 0, 164, 4086, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3689, 0, 11, 12, 0, 23, 59, 0, 0, 0, 5, 3384, 23, 0, 4094, 4085, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 4083, 0, 0, 0, 216, 4096, 10, 0, 7, 0, 0, 4078, 0, 0, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 4060, 4086, 4046, 0, 1500, 12, 4, 7, 34, 0, 0, 2, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3483, 4095, 1126, 546, 9, 4009, 0, 0, 10, 2, 4, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 35, 0, 4077, 0, 0, 0, 38, 4089, 4096, 0, 4, 2449, 1438, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 5, 49, 0, 4, 4, 0, 0, 4071, 2956, 2760, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 9, 3834, 1, 0, 4093, 1626, 505, 0, 0, 0, 3699, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15069, 16721, 17265, 17862, 17326, 16621, 16430, 13778] avg=16384.0000 max=17862 max_rank=3 min=13778 min_rank=7 imbalance_ratio=1.090210 max_load_reduction=113210 max_load_reduction_pct=0.863724 moved_to_echo=64155 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 37, 10, 0, 25, 482, 0, 1946, 4075, 0, 0, 0, 242, 4078, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 8, 0, 0, 15, 0, 17, 0, 0, 22, 7, 3, 3, 52, 4079, 4072, 1327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 4067, 0, 20, 50, 1282, 3859, 0, 0, 0, 3, 126, 4062, 1, 0, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 3938, 4079, 4048, 171, 1507, 9, 0, 28, 40, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 15, 4089, 4006, 1131, 241, 0, 2639, 0, 0, 0, 50, 34, 13, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 46, 55, 42, 56, 17, 0, 3909, 14, 2, 3, 0, 4037, 4029, 0, 0, 2426, 1437, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 3994, 0, 17, 16, 0, 29, 0, 0, 0, 11, 1, 4064, 2952, 2745, 2477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4093, 5, 1, 4045, 1618, 4020, 8, 0, 0, 3522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14981, 13687, 18244, 17926, 16378, 16193, 16317, 17346] avg=16384.0000 max=18244 max_rank=2 min=13687 min_rank=1 imbalance_ratio=1.113525 max_load_reduction=112828 max_load_reduction_pct=0.860809 moved_to_echo=66234 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 0, 1946, 4046, 0, 43, 0, 3588, 4082, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 16, 0, 0, 6, 0, 0, 4096, 0, 0, 4096, 4096, 1334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4063, 0, 0, 0, 0, 540, 0, 0, 0, 865, 0, 4096, 0, 0, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4069, 0, 3281, 17, 4060, 0, 1510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4096, 1128, 0, 0, 4080, 0, 0, 17, 33, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 39, 4096, 0, 0, 0, 0, 4096, 4096, 1, 0, 2457, 1437, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2944, 2757, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1639, 4052, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17804, 17759, 14373, 12940, 17529, 16327, 16376, 17964] avg=16384.0000 max=17964 max_rank=7 min=12940 min_rank=3 imbalance_ratio=1.096436 max_load_reduction=113108 max_load_reduction_pct=0.862946 moved_to_echo=68196 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 13, 5, 0, 0, 1931, 4078, 0, 0, 0, 4066, 4075, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 4, 296, 0, 0, 0, 0, 3, 1, 16, 0, 4072, 0, 0, 4078, 4093, 1335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 3869, 1, 0, 0, 29, 30, 1, 0, 0, 3991, 0, 4086, 0, 2, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 3823, 0, 90, 1450, 4094, 15, 1508, 0, 0, 6, 0, 31, 319, 5, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4081, 1133, 1, 4, 3989, 0, 0, 0, 1202, 5, 0, 2, 2722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 286, 0, 0, 4095, 2, 44, 0, 6, 4086, 4082, 0, 0, 2455, 1425, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 5, 2, 0, 4084, 0, 4, 4, 0, 40, 0, 3, 20, 3, 0, 4093, 2958, 2758, 2479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 30, 4057, 0, 8, 3909, 1638, 2345, 0, 0, 0, 4096, 17, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18239, 17996, 16805, 11583, 17231, 16587, 16510, 16121] avg=16384.0000 max=18239 max_rank=0 min=11583 min_rank=3 imbalance_ratio=1.113220 max_load_reduction=112833 max_load_reduction_pct=0.860847 moved_to_echo=66449 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 1946, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4041, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 2550, 0, 0, 4096, 4093, 1335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 75, 53, 4075, 0, 0, 0, 2, 4, 4095, 0, 0, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 3791, 4096, 0, 1510, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 1135, 0, 0, 849, 0, 0, 0, 1003, 0, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 1, 1, 0, 4094, 4096, 0, 0, 2454, 1437, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4095, 2961, 2760, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 1637, 4096, 0, 0, 0, 4025, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14235, 16137, 17209, 17591, 15270, 16283, 16402, 17945] avg=16384.0000 max=17945 max_rank=7 min=14235 min_rank=0 imbalance_ratio=1.095276 max_load_reduction=113127 max_load_reduction_pct=0.863091 moved_to_echo=65900 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 43, 1, 43, 0, 0, 0, 1946, 4096, 0, 16, 0, 669, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3979, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4091, 1333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 2, 105, 0, 4004, 0, 2, 0, 101, 0, 4089, 113, 0, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 4096, 4095, 4084, 0, 1510, 0, 9, 0, 1, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 449, 4057, 1134, 0, 0, 3838, 0, 0, 10, 3099, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 5, 0, 1, 4096, 4095, 6, 0, 2455, 1437, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 4094, 2959, 2756, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4003, 1637, 4096, 0, 0, 0, 4037, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15006, 13569, 17318, 17943, 16713, 16284, 16304, 17935] avg=16384.0000 max=17943 max_rank=3 min=13569 min_rank=1 imbalance_ratio=1.095154 max_load_reduction=113129 max_load_reduction_pct=0.863106 moved_to_echo=67236 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 344, 0, 50, 0, 1946, 4054, 0, 140, 0, 4057, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 13, 0, 0, 5, 0, 0, 3900, 0, 0, 4096, 4089, 1312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3, 36, 22, 3, 18, 0, 285, 59, 4090, 2, 0, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 3780, 0, 3987, 4078, 4096, 0, 1510, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 75, 4079, 1121, 0, 0, 4081, 0, 0, 2, 1686, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 66, 0, 0, 4089, 3, 11, 0, 0, 4088, 4096, 0, 0, 2457, 1437, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 4030, 2925, 2709, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 1639, 2221, 0, 0, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18783, 17511, 13423, 17524, 15147, 16354, 16276, 16054] avg=16384.0000 max=18783 max_rank=0 min=13423 min_rank=2 imbalance_ratio=1.146423 max_load_reduction=112289 max_load_reduction_pct=0.856697 moved_to_echo=63831 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 60, 0, 0, 5, 0, 1, 1938, 4065, 0, 0, 7, 2712, 4082, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 4090, 18, 32, 0, 1, 0, 7, 18, 0, 0, 0, 3966, 20, 0, 4082, 4080, 1335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 4057, 0, 0, 0, 109, 3288, 0, 0, 10, 100, 6, 4086, 0, 0, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4039, 5, 3937, 4062, 4032, 11, 1503, 0, 0, 0, 0, 1, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 2320, 4090, 1128, 0, 0, 3891, 0, 0, 12, 0, 0, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 20, 0, 156, 4087, 8, 43, 0, 0, 4052, 4077, 0, 7, 2435, 1431, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 27, 3964, 20, 10, 18, 36, 47, 4, 16, 0, 11, 18, 4096, 2945, 2761, 2473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4065, 0, 0, 4062, 1624, 56, 0, 0, 0, 4096, 12, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16955, 17692, 16435, 17703, 15478, 16423, 16457, 13929] avg=16384.0000 max=17703 max_rank=3 min=13929 min_rank=7 imbalance_ratio=1.080505 max_load_reduction=113369 max_load_reduction_pct=0.864937 moved_to_echo=62287 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 128, 22784, 32640, 9088, 896, 128, 27392, 0, 0, 128, 7168, 128, 128, 128, 31232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 256, 128, 512, 128, 1152, 128, 15616, 32640, 128, 256, 128, 15616, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 0, 32512, 128, 128, 256, 1792, 19968, 128, 128, 128, 5376, 256, 32768, 128, 128, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 32512, 128, 128, 32512, 13184, 21504, 128, 0, 0, 31616, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 32256, 128, 384, 128, 128, 256, 128, 128, 128, 128, 128, 22016, 128, 128, 32768, 32768, 10752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[384, 32128, 128, 27520, 25728, 32640, 256, 12160, 128, 128, 128, 128, 128, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 32512, 128, 128, 128, 128, 256, 128, 128, 128, 128, 128, 32640, 23680, 22016, 19968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 768, 128, 256, 32640, 128, 128, 128, 128, 32640, 32768, 128, 128, 19712, 11520, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[24, 1559, 2285, 5, 2134, 0, 719, 15, 760, 68, 2285, 305, 790, 587, 7, 270, 113, 1715, 1101, 1505, 3134, 406, 962, 224, 13, 2972, 168, 573, 866, 170, 784, 2265, 3456, 11, 68, 407, 821, 69, 576, 916, 2703, 3106, 540, 330, 227, 785, 1535, 1558, 340, 233, 1000, 429, 190, 408, 2912, 119, 30, 1055, 3084, 2767, 155, 338, 247, 54, 1103, 3201, 20, 962, 361, 224, 1993, 2162, 433, 783, 444, 67, 826, 2622, 2317, 1084, 1425, 593, 1046, 3440, 520, 1076, 1191, 3713, 79, 264, 903, 1057, 1698, 401, 659, 54, 1499, 240, 2858, 559, 11, 1254, 1671, 2000, 2845, 2447, 399, 1091, 474, 719, 454, 2014, 128, 1165, 521, 856, 70, 1943, 900, 1675, 767, 1264, 2121, 1086, 1246, 18, 780, 23] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[16, 1228, 1792, 18, 2190, 1, 850, 26, 1297, 145, 2534, 380, 838, 700, 12, 303, 129, 1088, 1629, 1053, 3065, 887, 658, 516, 3, 2468, 175, 508, 991, 88, 1240, 1920, 3246, 30, 64, 1171, 966, 54, 446, 1159, 2492, 3427, 977, 506, 210, 1353, 1095, 422, 342, 160, 1143, 558, 510, 280, 3080, 199, 6, 1618, 3308, 2056, 538, 117, 455, 34, 666, 3245, 123, 1351, 442, 248, 1230, 1858, 307, 1324, 472, 85, 696, 2713, 2016, 1137, 1464, 818, 872, 2810, 409, 931, 1337, 3629, 179, 266, 511, 665, 1725, 423, 402, 111, 1738, 643, 2795, 657, 36, 872, 2066, 2358, 3109, 1902, 335, 1220, 566, 1280, 369, 1809, 160, 1621, 370, 1054, 43, 1199, 1178, 1714, 524, 940, 1927, 899, 1419, 27, 970, 37] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[5, 996, 1826, 5, 2290, 0, 821, 25, 459, 146, 1867, 509, 936, 370, 17, 289, 193, 1453, 1064, 1509, 2989, 686, 655, 212, 2, 2547, 214, 386, 1104, 114, 1013, 2226, 3604, 24, 72, 777, 606, 90, 826, 880, 2694, 2900, 649, 239, 253, 1312, 1737, 1333, 294, 131, 904, 414, 148, 128, 3539, 180, 3, 1235, 2850, 2243, 179, 325, 509, 19, 952, 3325, 54, 1481, 592, 91, 1649, 2275, 258, 666, 306, 106, 722, 2307, 2779, 1350, 1156, 829, 1049, 3625, 556, 666, 1396, 3483, 93, 194, 603, 1175, 1977, 486, 1377, 50, 1741, 426, 3291, 585, 11, 1065, 1236, 1632, 2900, 1782, 364, 967, 263, 741, 466, 2430, 179, 1778, 476, 856, 28, 2021, 563, 1536, 1009, 945, 2323, 1077, 1661, 20, 1044, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[19, 1484, 2004, 17, 2498, 0, 1044, 15, 983, 141, 2307, 246, 647, 481, 5, 229, 139, 1965, 759, 1120, 3174, 612, 866, 191, 12, 2733, 146, 463, 449, 66, 957, 2349, 3372, 48, 47, 1280, 497, 81, 695, 941, 2692, 3035, 496, 250, 225, 1237, 1987, 592, 281, 281, 1266, 293, 341, 490, 2908, 84, 59, 1088, 3412, 2858, 151, 293, 164, 69, 609, 3472, 37, 1236, 715, 183, 1524, 2618, 515, 1139, 420, 40, 899, 2639, 2426, 1723, 1859, 531, 1068, 2947, 494, 587, 1181, 3628, 126, 256, 734, 646, 2397, 680, 686, 48, 1223, 228, 3302, 744, 11, 949, 1485, 2086, 2659, 2050, 372, 1011, 596, 561, 607, 2585, 329, 972, 508, 1052, 23, 794, 543, 1483, 903, 1124, 2127, 796, 1392, 43, 809, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[25, 1260, 1960, 13, 2171, 22, 853, 31, 486, 177, 2176, 430, 669, 409, 28, 271, 153, 1609, 1215, 1422, 2909, 500, 644, 296, 21, 2178, 178, 494, 1187, 173, 922, 2760, 3410, 93, 188, 1180, 1009, 107, 466, 1416, 2696, 2761, 595, 281, 316, 1515, 1836, 1076, 341, 378, 757, 338, 151, 327, 2961, 86, 26, 1080, 2725, 2306, 241, 372, 629, 85, 1078, 3037, 77, 1303, 510, 175, 964, 2439, 199, 920, 596, 106, 1002, 2215, 2484, 810, 1957, 573, 1046, 3032, 544, 764, 1748, 3325, 124, 411, 815, 838, 1637, 589, 534, 37, 1598, 418, 3134, 426, 34, 1337, 1280, 1987, 2832, 1624, 458, 942, 783, 993, 585, 2836, 315, 1477, 343, 747, 48, 1426, 564, 1124, 970, 1327, 2348, 1148, 1423, 12, 1215, 39] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[18, 1308, 2383, 9, 2447, 5, 746, 41, 453, 155, 1901, 473, 903, 399, 34, 297, 179, 1640, 1097, 1487, 2889, 416, 658, 136, 47, 2340, 299, 456, 858, 204, 622, 2384, 3575, 24, 71, 878, 920, 94, 858, 1187, 2752, 2626, 845, 209, 179, 1357, 2274, 990, 378, 303, 792, 225, 173, 538, 3314, 157, 13, 1086, 2734, 2530, 206, 500, 173, 50, 613, 3566, 52, 1011, 549, 256, 1264, 2343, 232, 844, 753, 109, 892, 1674, 2329, 1229, 1572, 475, 1101, 3029, 419, 672, 1949, 3711, 304, 261, 753, 891, 1919, 387, 656, 94, 879, 396, 3554, 712, 77, 1461, 1592, 1758, 3143, 1615, 398, 1336, 852, 618, 515, 2508, 268, 1324, 611, 768, 94, 1459, 485, 1839, 1057, 1357, 2371, 692, 1124, 19, 976, 10] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[15, 1644, 1736, 19, 2437, 0, 751, 30, 926, 56, 2221, 409, 1346, 349, 6, 348, 291, 1322, 1175, 1323, 2910, 812, 712, 251, 20, 2641, 170, 337, 799, 62, 891, 1522, 3328, 9, 78, 709, 716, 62, 720, 1051, 2630, 2863, 567, 283, 240, 1282, 1226, 782, 304, 285, 1270, 155, 243, 250, 3278, 118, 41, 1082, 3262, 2264, 183, 240, 191, 42, 943, 3483, 55, 1830, 571, 201, 1700, 2649, 327, 1040, 187, 46, 828, 2570, 2165, 1575, 2052, 792, 610, 3065, 568, 657, 1386, 3697, 189, 408, 1004, 547, 2213, 346, 954, 46, 1262, 387, 3560, 715, 20, 943, 1461, 2213, 2973, 2091, 477, 834, 657, 831, 482, 2102, 298, 1152, 238, 997, 11, 1051, 417, 2187, 719, 1096, 2033, 1448, 1480, 4, 843, 101] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[11, 1292, 1817, 63, 2473, 2, 911, 21, 862, 43, 1894, 169, 682, 413, 13, 262, 191, 1623, 984, 1306, 2988, 570, 709, 408, 7, 2500, 214, 353, 272, 56, 889, 2218, 3504, 127, 108, 983, 350, 138, 753, 1071, 2809, 2316, 442, 89, 266, 1268, 1865, 1505, 396, 155, 1062, 262, 97, 253, 3185, 151, 38, 819, 3455, 2767, 203, 223, 357, 40, 632, 3382, 91, 1929, 811, 111, 2293, 2442, 389, 1219, 354, 42, 1023, 2430, 2125, 1419, 1123, 615, 1429, 2653, 434, 472, 1410, 3513, 121, 423, 992, 977, 1899, 522, 754, 37, 1311, 350, 3284, 596, 33, 1322, 2011, 1970, 2945, 1921, 275, 1158, 549, 777, 408, 2840, 371, 1233, 711, 624, 26, 1156, 671, 1461, 556, 1169, 2629, 748, 1361, 32, 1204, 31] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[24, 1559, 2285, 5, 2134, 0, 719, 15, 760, 68, 2285, 305, 790, 587, 7, 270, 2858, 1521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 1715, 1101, 1505, 3134, 406, 962, 224, 13, 2972, 168, 573, 866, 170, 784, 2265, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2068, 11, 68, 407, 821, 69, 576, 916, 2703, 3106, 540, 330, 227, 785, 1535, 1558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 233, 1000, 429, 190, 408, 2912, 119, 30, 1055, 3084, 2767, 155, 338, 247, 54, 1585, 1000, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1103, 680, 20, 962, 361, 224, 1993, 2162, 433, 783, 444, 67, 826, 2622, 2317, 1084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1425, 593, 1046, 3440, 520, 1076, 1191, 2128, 79, 264, 903, 1057, 1698, 401, 659, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1499, 240, 0, 559, 11, 1254, 1671, 2000, 1479, 2447, 399, 1091, 474, 719, 454, 2014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 1165, 521, 856, 70, 1943, 900, 1675, 767, 1264, 2121, 1086, 1246, 18, 780, 23, 1085, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16192, 17274, 15720, 16300, 16081, 16534, 16311, 16660] avg=16384.0000 max=17274 max_rank=1 min=15720 min_rank=2 imbalance_ratio=1.054321 max_load_reduction=113798 max_load_reduction_pct=0.868210 moved_to_echo=65586 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[19, 1484, 2004, 17, 2498, 0, 1044, 15, 983, 141, 2307, 246, 647, 481, 5, 229, 3302, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 1965, 759, 1120, 3174, 612, 866, 191, 12, 2733, 146, 463, 449, 66, 957, 2349, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2018, 48, 47, 1280, 497, 81, 695, 941, 2692, 3035, 496, 250, 225, 1237, 1987, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 281, 1266, 293, 341, 490, 2908, 84, 59, 1088, 3412, 2858, 151, 293, 164, 69, 1549, 1085, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 609, 737, 37, 1236, 715, 183, 1524, 2618, 515, 1139, 420, 40, 899, 2639, 2426, 1723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1859, 531, 1068, 2947, 494, 587, 1181, 2079, 126, 256, 734, 646, 2397, 680, 686, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1223, 228, 0, 744, 11, 949, 1485, 2086, 1382, 2050, 372, 1011, 596, 561, 607, 2585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 972, 508, 1052, 23, 794, 543, 1483, 903, 1124, 2127, 796, 1392, 43, 809, 8, 1058, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17072, 16297, 16121, 17003, 17460, 16319, 15890, 14910] avg=16384.0000 max=17460 max_rank=4 min=14910 min_rank=7 imbalance_ratio=1.065674 max_load_reduction=113612 max_load_reduction_pct=0.866791 moved_to_echo=64579 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[16, 1228, 1792, 18, 2190, 1, 850, 26, 1297, 145, 2534, 380, 838, 700, 12, 303, 2795, 1542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 1088, 1629, 1053, 3065, 887, 658, 516, 3, 2468, 175, 508, 991, 88, 1240, 1920, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1942, 30, 64, 1171, 966, 54, 446, 1159, 2492, 3427, 977, 506, 210, 1353, 1095, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 342, 160, 1143, 558, 510, 280, 3080, 199, 6, 1618, 3308, 2056, 538, 117, 455, 34, 1549, 1014, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 666, 689, 123, 1351, 442, 248, 1230, 1858, 307, 1324, 472, 85, 696, 2713, 2016, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1464, 818, 872, 2810, 409, 931, 1337, 2080, 179, 266, 511, 665, 1725, 423, 402, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1738, 643, 0, 657, 36, 872, 2066, 2358, 1616, 1902, 335, 1220, 566, 1280, 369, 1809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 1621, 370, 1054, 43, 1199, 1178, 1714, 524, 940, 1927, 899, 1419, 27, 970, 37, 1019, 1106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16667, 16703, 16314, 17354, 15357, 15003, 17467, 16207] avg=16384.0000 max=17467 max_rank=6 min=15003 min_rank=5 imbalance_ratio=1.066101 max_load_reduction=113605 max_load_reduction_pct=0.866737 moved_to_echo=64034 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 996, 1826, 5, 2290, 0, 821, 25, 459, 146, 1867, 509, 936, 370, 17, 289, 3291, 1580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 1453, 1064, 1509, 2989, 686, 655, 212, 2, 2547, 214, 386, 1104, 114, 1013, 2226, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2157, 24, 72, 777, 606, 90, 826, 880, 2694, 2900, 649, 239, 253, 1312, 1737, 1333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 131, 904, 414, 148, 128, 3539, 180, 3, 1235, 2850, 2243, 179, 325, 509, 19, 1487, 1039, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 952, 706, 54, 1481, 592, 91, 1649, 2275, 258, 666, 306, 106, 722, 2307, 2779, 1350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1156, 829, 1049, 3625, 556, 666, 1396, 1996, 93, 194, 603, 1175, 1977, 486, 1377, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1741, 426, 0, 585, 11, 1065, 1236, 1632, 1507, 1782, 364, 967, 263, 741, 466, 2430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 1778, 476, 856, 28, 2021, 563, 1536, 1009, 945, 2323, 1077, 1661, 20, 1044, 3, 1131, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15432, 16683, 16549, 15988, 16294, 17228, 15216, 17682] avg=16384.0000 max=17682 max_rank=7 min=15216 min_rank=6 imbalance_ratio=1.079224 max_load_reduction=113390 max_load_reduction_pct=0.865097 moved_to_echo=66420 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[25, 1260, 1960, 13, 2171, 22, 853, 31, 486, 177, 2176, 430, 669, 409, 28, 271, 3134, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 1609, 1215, 1422, 2909, 500, 644, 296, 21, 2178, 178, 494, 1187, 173, 922, 2760, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2033, 93, 188, 1180, 1009, 107, 466, 1416, 2696, 2761, 595, 281, 316, 1515, 1836, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 378, 757, 338, 151, 327, 2961, 86, 26, 1080, 2725, 2306, 241, 372, 629, 85, 1425, 952, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 638, 77, 1303, 510, 175, 964, 2439, 199, 920, 596, 106, 1002, 2215, 2484, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1957, 573, 1046, 3032, 544, 764, 1748, 1900, 124, 411, 815, 838, 1637, 589, 534, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1598, 418, 0, 426, 34, 1337, 1280, 1987, 1465, 1624, 458, 942, 783, 993, 585, 2836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 1477, 343, 747, 48, 1426, 564, 1124, 970, 1327, 2348, 1148, 1423, 12, 1215, 39, 1074, 1011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15562, 16964, 17568, 15536, 15516, 16549, 16766, 16611] avg=16384.0000 max=17568 max_rank=2 min=15516 min_rank=4 imbalance_ratio=1.072266 max_load_reduction=113504 max_load_reduction_pct=0.865967 moved_to_echo=65442 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[15, 1644, 1736, 19, 2437, 0, 751, 30, 926, 56, 2221, 409, 1346, 349, 6, 348, 3560, 1655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 1322, 1175, 1323, 2910, 812, 712, 251, 20, 2641, 170, 337, 799, 62, 891, 1522, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1992, 9, 78, 709, 716, 62, 720, 1051, 2630, 2863, 567, 283, 240, 1282, 1226, 782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 285, 1270, 155, 243, 250, 3278, 118, 41, 1082, 3262, 2264, 183, 240, 191, 42, 1578, 1088, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 943, 740, 55, 1830, 571, 201, 1700, 2649, 327, 1040, 187, 46, 828, 2570, 2165, 1575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2052, 792, 610, 3065, 568, 657, 1386, 2119, 189, 408, 1004, 547, 2213, 346, 954, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 387, 0, 715, 20, 943, 1461, 2213, 1545, 2091, 477, 834, 657, 831, 482, 2102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 1152, 238, 997, 11, 1051, 417, 2187, 719, 1096, 2033, 1448, 1480, 4, 843, 101, 1044, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17508, 15530, 15210, 16244, 17427, 16956, 16020, 16177] avg=16384.0000 max=17508 max_rank=0 min=15210 min_rank=2 imbalance_ratio=1.068604 max_load_reduction=113564 max_load_reduction_pct=0.866425 moved_to_echo=66580 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[18, 1308, 2383, 9, 2447, 5, 746, 41, 453, 155, 1901, 473, 903, 399, 34, 297, 3554, 1694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 1640, 1097, 1487, 2889, 416, 658, 136, 47, 2340, 299, 456, 858, 204, 622, 2384, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2139, 24, 71, 878, 920, 94, 858, 1187, 2752, 2626, 845, 209, 179, 1357, 2274, 990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 303, 792, 225, 173, 538, 3314, 157, 13, 1086, 2734, 2530, 206, 500, 173, 50, 1584, 1114, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 613, 758, 52, 1011, 549, 256, 1264, 2343, 232, 844, 753, 109, 892, 1674, 2329, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1572, 475, 1101, 3029, 419, 672, 1949, 2127, 304, 261, 753, 891, 1919, 387, 656, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 879, 396, 0, 712, 77, 1461, 1592, 1758, 1634, 1615, 398, 1336, 852, 618, 515, 2508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 268, 1324, 611, 768, 94, 1459, 485, 1839, 1057, 1357, 2371, 692, 1124, 19, 976, 10, 1122, 1118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16820, 16026, 17403, 16261, 14908, 16609, 16351, 16694] avg=16384.0000 max=17403 max_rank=2 min=14908 min_rank=4 imbalance_ratio=1.062195 max_load_reduction=113669 max_load_reduction_pct=0.867226 moved_to_echo=64562 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 1292, 1817, 63, 2473, 2, 911, 21, 862, 43, 1894, 169, 682, 413, 13, 262, 3284, 1607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 1623, 984, 1306, 2988, 570, 709, 408, 7, 2500, 214, 353, 272, 56, 889, 2218, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2097, 127, 108, 983, 350, 138, 753, 1071, 2809, 2316, 442, 89, 266, 1268, 1865, 1505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 155, 1062, 262, 97, 253, 3185, 151, 38, 819, 3455, 2767, 203, 223, 357, 40, 1500, 1056, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 632, 719, 91, 1929, 811, 111, 2293, 2442, 389, 1219, 354, 42, 1023, 2430, 2125, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 615, 1429, 2653, 434, 472, 1410, 2013, 121, 423, 992, 977, 1899, 522, 754, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1311, 350, 0, 596, 33, 1322, 2011, 1970, 1530, 1921, 275, 1158, 549, 777, 408, 2840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 1233, 711, 624, 26, 1156, 671, 1461, 556, 1169, 2629, 748, 1361, 32, 1204, 31, 1100, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15819, 15595, 16187, 16386, 18029, 15874, 17051, 16131] avg=16384.0000 max=18029 max_rank=4 min=15595 min_rank=1 imbalance_ratio=1.100403 max_load_reduction=113043 max_load_reduction_pct=0.862450 moved_to_echo=67085 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 10880, 15872, 256, 18688, 128, 6784, 256, 6272, 1024, 17280, 2944, 6912, 3712, 128, 2304, 25856, 12800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[6656, 5760, 512, 11136, 4608, 1536, 12672, 18816, 2688, 7936, 3584, 640, 6912, 19200, 18688, 10368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12672, 5248, 8320, 24704, 3968, 5888, 11648, 16512, 1280, 2560, 6400, 6912, 15488, 3840, 6144, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[11264, 3200, 0, 5120, 256, 9216, 12928, 16128, 12160, 15488, 3200, 8576, 4864, 6528, 3968, 19200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[1408, 12416, 9088, 10752, 24064, 4992, 5888, 2304, 128, 20480, 1664, 3584, 6528, 1024, 7424, 17664, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2048, 10752, 3840, 7040, 384, 11136, 5376, 13056, 6528, 9344, 17920, 7936, 11136, 256, 7936, 256, 8704, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[16512, 384, 768, 7424, 5888, 768, 5376, 8704, 21504, 23040, 5120, 2304, 1920, 10112, 13568, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2688, 2048, 8320, 2688, 1920, 2688, 25216, 1152, 256, 9088, 24832, 19840, 1920, 2432, 2816, 512, 12288, 8448, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1472, 1, 976, 2, 3246, 792, 26, 864, 0, 1260, 38, 4096, 400, 0, 679, 0, 2, 1738, 4009, 1381, 0, 0, 202, 0, 4080, 0, 0, 4086, 11, 2, 29, 1, 0, 0, 4079, 94, 0, 0, 0, 612, 3699, 0, 8, 0, 3, 9, 0, 4062, 4032, 0, 0, 13, 0, 0, 4096, 0, 0, 0, 2792, 4, 1, 274, 3013, 3693, 0, 4096, 0, 3333, 0, 2483, 0, 4077, 54, 1396, 0, 3771, 4096, 0, 0, 5, 2, 0, 4095, 4093, 3758, 19, 4050, 0, 0, 1, 25, 0, 3973, 0, 0, 4084, 4074, 1, 2785, 96, 0, 1, 0, 154, 10, 0, 0, 0, 0, 0, 3755, 0, 0, 3, 0, 37, 594, 0, 1, 4082, 1, 4084, 0, 4093, 0, 13] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3264, 0, 66, 1, 798, 248, 165, 682, 0, 109, 0, 3925, 525, 7, 96, 0, 0, 1119, 3948, 3367, 0, 0, 10, 0, 4090, 0, 0, 4093, 0, 218, 516, 0, 0, 0, 3961, 52, 345, 0, 83, 2969, 4029, 0, 0, 5, 0, 148, 1, 4095, 4094, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 3781, 0, 1, 45, 2928, 2964, 0, 4096, 0, 4073, 0, 3681, 4, 4077, 42, 2, 0, 3509, 4089, 0, 0, 129, 0, 0, 4096, 4096, 3669, 0, 3926, 0, 0, 0, 0, 0, 4096, 0, 26, 4096, 4096, 0, 1104, 5, 0, 0, 0, 354, 0, 0, 8, 0, 0, 18, 3754, 0, 0, 0, 0, 0, 351, 0, 27, 4092, 2, 4096, 0, 4096, 0, 510] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[2, 58, 806, 2, 1, 6, 30, 46, 0, 1092, 0, 6, 0, 4061, 1121, 24, 942, 31, 17, 282, 4084, 4057, 78, 7, 19, 0, 4085, 0, 4, 3663, 3, 1, 287, 17, 96, 0, 3710, 10, 0, 3, 40, 3131, 3767, 0, 9, 33, 2, 35, 0, 3905, 3992, 13, 0, 1, 49, 0, 4086, 0, 0, 13, 4035, 0, 13, 12, 3943, 2178, 0, 4058, 0, 4052, 28, 3941, 18, 4075, 385, 681, 6, 4078, 3840, 26, 0, 42, 0, 204, 4074, 4069, 2679, 21, 4043, 2, 0, 4, 0, 8, 4078, 1, 49, 4074, 4087, 0, 2170, 383, 0, 0, 0, 157, 0, 54, 156, 0, 0, 0, 4089, 16, 3, 0, 19, 259, 902, 27, 0, 4038, 10, 4075, 0, 4095, 10, 78] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 2, 85, 0, 0, 0, 638, 0, 0, 80, 0, 98, 0, 4085, 257, 5, 3838, 1, 0, 3433, 3809, 3722, 0, 1, 88, 1, 4092, 0, 0, 3661, 1, 1, 200, 0, 6, 0, 3947, 1, 1, 1, 0, 3238, 4055, 1, 0, 0, 0, 0, 3, 4093, 4087, 1, 0, 0, 0, 1, 4093, 1, 0, 0, 3040, 0, 0, 1, 3990, 3569, 0, 4092, 0, 4093, 0, 3057, 19, 4096, 0, 6, 0, 4009, 4087, 0, 5, 2, 20, 128, 4096, 4093, 2751, 0, 4096, 0, 0, 2, 0, 1, 4094, 0, 0, 4093, 4094, 0, 474, 71, 0, 1, 2, 22, 3, 7, 111, 0, 3, 1, 4084, 0, 0, 0, 0, 78, 53, 0, 86, 4095, 490, 4096, 0, 4093, 0, 141] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 423, 4, 92, 0, 3547, 54, 1, 0, 0, 62, 0, 3940, 3748, 0, 1080, 0, 7, 743, 2946, 875, 4, 1, 738, 0, 3955, 2, 0, 4077, 0, 0, 39, 0, 2, 0, 3779, 13, 36, 1, 0, 2700, 3492, 0, 0, 0, 0, 1, 0, 3954, 4035, 0, 2, 1, 0, 0, 4088, 27, 190, 0, 3850, 8, 12, 0, 2279, 3774, 0, 4095, 0, 3956, 0, 3832, 31, 3811, 99, 25, 0, 4084, 4034, 0, 0, 179, 1, 6, 4095, 4090, 3799, 0, 4093, 0, 0, 0, 0, 0, 4089, 2, 3, 4096, 4094, 0, 858, 0, 0, 0, 0, 78, 0, 0, 227, 0, 0, 25, 3420, 0, 0, 1, 0, 8, 3067, 0, 0, 4016, 83, 4096, 0, 4096, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 2, 3714, 7, 9, 1, 1526, 38, 55, 154, 0, 9, 0, 4062, 414, 2, 234, 2, 6, 2688, 3533, 3204, 16, 0, 200, 0, 4083, 0, 0, 4086, 6, 0, 2, 0, 1, 0, 4087, 0, 0, 0, 10, 3887, 3586, 0, 0, 14, 0, 16, 1, 3471, 4088, 1, 0, 0, 2, 0, 4091, 26, 0, 6, 4093, 0, 0, 0, 2939, 2472, 0, 4094, 5, 4077, 0, 2910, 30, 3987, 7, 1001, 0, 4065, 4066, 0, 3, 0, 2, 11, 4096, 4091, 2281, 0, 4055, 0, 3, 6, 16, 0, 4069, 2, 4, 4095, 4090, 0, 963, 125, 0, 0, 0, 4, 0, 26, 0, 0, 1, 0, 4092, 0, 0, 6, 0, 2, 1694, 0, 0, 4037, 15, 4082, 0, 4096, 0, 19] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 367, 0, 5, 0, 2866, 6, 14, 393, 0, 192, 18, 4064, 191, 0, 3150, 3, 1, 3328, 3612, 4078, 13, 9, 32, 0, 4083, 13, 0, 4064, 0, 0, 234, 0, 3, 3, 4071, 9, 221, 0, 4, 4020, 3986, 57, 0, 31, 0, 5, 0, 3941, 4089, 3, 0, 20, 14, 70, 4062, 0, 4, 0, 3734, 0, 3, 25, 2133, 3070, 0, 4000, 0, 3588, 0, 1141, 117, 4087, 59, 75, 3, 3505, 4084, 0, 10, 11, 228, 17, 4094, 4084, 1376, 0, 4082, 0, 45, 1, 2, 2, 4045, 3, 0, 4082, 4071, 1, 1071, 8, 0, 0, 5, 494, 0, 111, 7, 6, 0, 1, 2804, 31, 0, 0, 0, 7, 3158, 1, 3, 4057, 0, 4084, 1, 4095, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1673, 0, 22, 0, 2557, 1, 1, 127, 0, 97, 0, 3511, 2452, 35, 3297, 0, 0, 125, 3209, 4040, 0, 0, 54, 0, 4045, 0, 0, 4012, 0, 1, 9, 4, 0, 0, 4084, 191, 0, 0, 25, 4050, 4093, 0, 0, 23, 0, 3, 3, 3977, 4092, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 2329, 0, 1, 615, 3802, 3905, 0, 4096, 0, 3252, 0, 3872, 14, 4074, 958, 122, 0, 4045, 4095, 0, 0, 2, 0, 12, 4096, 4096, 845, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 4096, 0, 115, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3981, 0, 0, 0, 0, 0, 128, 0, 0, 4048, 0, 4096, 0, 4096, 0, 81] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3264, 0, 66, 1, 798, 248, 165, 682, 0, 109, 0, 3925, 525, 7, 4096, 2842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 1119, 3948, 3367, 0, 0, 10, 0, 2124, 0, 0, 4093, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 0, 0, 0, 3961, 52, 345, 0, 83, 2969, 4029, 0, 0, 5, 0, 148, 1966, 1265, 1322, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4094, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 3781, 0, 1, 45, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2928, 2964, 0, 0, 0, 4073, 0, 3681, 4, 0, 42, 2, 0, 2244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 1381, 4096, 3669, 0, 3926, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 4096, 0, 1104, 5, 0, 0, 0, 354, 0, 0, 8, 0, 0, 18, 4077, 1393, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3754, 0, 0, 0, 0, 0, 351, 0, 27, 4092, 2, 4096, 0, 2990, 0, 510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16728, 14975, 17116, 16772, 15938, 17297, 16424, 15822] avg=16384.0000 max=17297 max_rank=5 min=14975 min_rank=1 imbalance_ratio=1.055725 max_load_reduction=113775 max_load_reduction_pct=0.868034 moved_to_echo=65481 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1472, 1, 976, 2, 3246, 792, 26, 864, 0, 1260, 38, 4096, 400, 0, 4096, 2847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 679, 0, 2, 1738, 4009, 1381, 0, 0, 202, 0, 2118, 0, 0, 4086, 11, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 1, 0, 0, 4079, 94, 0, 0, 0, 612, 3699, 0, 8, 0, 3, 9, 1962, 1359, 1321, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4032, 0, 0, 13, 0, 0, 4096, 0, 0, 0, 2792, 4, 1, 274, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3013, 3693, 0, 0, 0, 3333, 0, 2483, 0, 0, 54, 1396, 0, 2412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 1381, 4093, 3758, 19, 4050, 0, 0, 1, 25, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4074, 1, 2785, 96, 0, 1, 0, 154, 10, 0, 0, 0, 0, 0, 4077, 1393, 1249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 3, 0, 37, 594, 0, 1, 4082, 1, 4084, 0, 2987, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20116, 14228, 13631, 15925, 16384, 17307, 17924, 15557] avg=16384.0000 max=20116 max_rank=0 min=13631 min_rank=2 imbalance_ratio=1.227783 max_load_reduction=110956 max_load_reduction_pct=0.846527 moved_to_echo=67172 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 58, 806, 2, 1, 6, 30, 46, 0, 1092, 0, 6, 0, 4061, 1121, 24, 4058, 2670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 942, 31, 17, 282, 4084, 4057, 78, 7, 19, 0, 2117, 0, 4, 3663, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 17, 96, 0, 3710, 10, 0, 3, 40, 3131, 3767, 0, 9, 33, 2, 35, 1968, 1473, 1317, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 3992, 13, 0, 1, 49, 0, 4086, 0, 0, 13, 4035, 0, 13, 12, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 2178, 0, 0, 0, 4052, 28, 3941, 18, 0, 385, 681, 6, 2605, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 204, 1369, 4069, 2679, 21, 4043, 2, 0, 4, 0, 8, 4078, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 4074, 4087, 0, 2170, 383, 0, 0, 0, 157, 0, 54, 156, 0, 0, 0, 4075, 1388, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 16, 3, 0, 19, 259, 902, 27, 0, 4038, 10, 4075, 0, 2979, 10, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13983, 15305, 16358, 16775, 17863, 16520, 17763, 16505] avg=16384.0000 max=17863 max_rank=4 min=13983 min_rank=0 imbalance_ratio=1.090271 max_load_reduction=113209 max_load_reduction_pct=0.863716 moved_to_echo=68651 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 423, 4, 92, 0, 3547, 54, 1, 0, 0, 62, 0, 3940, 3748, 0, 4095, 2805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1080, 0, 7, 743, 2946, 875, 4, 1, 738, 0, 2054, 2, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 2, 0, 3779, 13, 36, 1, 0, 2700, 3492, 0, 0, 0, 0, 1, 1901, 1472, 1321, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 4035, 0, 2, 1, 0, 0, 4088, 27, 190, 0, 3850, 8, 12, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2279, 3774, 0, 0, 0, 3956, 0, 3832, 31, 0, 99, 25, 0, 2612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 1, 6, 1381, 4090, 3799, 0, 4093, 0, 0, 0, 0, 0, 4089, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4094, 0, 858, 0, 0, 0, 0, 78, 0, 0, 227, 0, 0, 25, 3811, 1393, 1229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3420, 0, 0, 1, 0, 8, 3067, 0, 0, 4016, 83, 4096, 0, 2990, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18771, 12527, 15212, 16818, 16608, 17640, 15814, 17682] avg=16384.0000 max=18771 max_rank=0 min=12527 min_rank=1 imbalance_ratio=1.145691 max_load_reduction=112301 max_load_reduction_pct=0.856789 moved_to_echo=67744 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 85, 0, 0, 0, 638, 0, 0, 80, 0, 98, 0, 4085, 257, 5, 4092, 2841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 1, 0, 3433, 3809, 3722, 0, 1, 88, 1, 2125, 0, 0, 3661, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 6, 0, 3947, 1, 1, 1, 0, 3238, 4055, 1, 0, 0, 0, 0, 1967, 1445, 1322, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 4087, 1, 0, 0, 0, 1, 4093, 1, 0, 0, 3040, 0, 0, 1, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 3569, 0, 0, 0, 4093, 0, 3057, 19, 0, 0, 6, 0, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 20, 128, 1381, 4093, 2751, 0, 4096, 0, 0, 2, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4094, 0, 474, 71, 0, 1, 2, 22, 3, 7, 111, 0, 3, 1, 4096, 1393, 1246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 78, 53, 0, 86, 4095, 490, 4096, 0, 2987, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12183, 20681, 16639, 15971, 17298, 16573, 15617, 16110] avg=16384.0000 max=20681 max_rank=1 min=12183 min_rank=0 imbalance_ratio=1.262268 max_load_reduction=110391 max_load_reduction_pct=0.842216 moved_to_echo=65598 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 3714, 7, 9, 1, 1526, 38, 55, 154, 0, 9, 0, 4062, 414, 2, 4094, 2827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 2, 6, 2688, 3533, 3204, 16, 0, 200, 0, 2120, 0, 0, 4086, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 4087, 0, 0, 0, 10, 3887, 3586, 0, 0, 14, 0, 16, 1963, 1465, 1322, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3471, 4088, 1, 0, 0, 2, 0, 4091, 26, 0, 6, 4093, 0, 0, 0, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2939, 2472, 0, 0, 5, 4077, 0, 2910, 30, 0, 7, 1001, 0, 2600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 11, 1381, 4091, 2281, 0, 4055, 0, 3, 6, 16, 0, 4069, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 4090, 0, 963, 125, 0, 0, 0, 4, 0, 26, 0, 0, 1, 0, 3987, 1393, 1239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 6, 0, 2, 1694, 0, 0, 4037, 15, 4082, 0, 2990, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16914, 16095, 16808, 16430, 16041, 15920, 15927, 16937] avg=16384.0000 max=16937 max_rank=7 min=15920 min_rank=5 imbalance_ratio=1.033752 max_load_reduction=114135 max_load_reduction_pct=0.870781 moved_to_echo=64825 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 367, 0, 5, 0, 2866, 6, 14, 393, 0, 192, 18, 4064, 191, 0, 4000, 2840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3150, 3, 1, 3328, 3612, 4078, 13, 9, 32, 0, 2120, 13, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 3, 3, 4071, 9, 221, 0, 4, 4020, 3986, 57, 0, 31, 0, 5, 1963, 1263, 1321, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 4089, 3, 0, 20, 14, 70, 4062, 0, 4, 0, 3734, 0, 3, 25, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2133, 3070, 0, 0, 0, 3588, 0, 1141, 117, 0, 59, 75, 3, 2242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 11, 228, 17, 1381, 4084, 1376, 0, 4082, 0, 45, 1, 2, 2, 4045, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4071, 1, 1071, 8, 0, 0, 5, 494, 0, 111, 7, 6, 0, 1, 4087, 1392, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2804, 31, 0, 0, 0, 7, 3158, 1, 3, 4057, 0, 4084, 1, 2989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14957, 20423, 17646, 16616, 12428, 15287, 16580, 17135] avg=16384.0000 max=20423 max_rank=1 min=12428 min_rank=4 imbalance_ratio=1.246521 max_load_reduction=110649 max_load_reduction_pct=0.844185 moved_to_echo=61430 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1673, 0, 22, 0, 2557, 1, 1, 127, 0, 97, 0, 3511, 2452, 35, 4096, 2848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3297, 0, 0, 125, 3209, 4040, 0, 0, 54, 0, 2100, 0, 0, 4012, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4, 0, 0, 4084, 191, 0, 0, 25, 4050, 4093, 0, 0, 23, 0, 3, 1945, 1458, 1322, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3977, 4092, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 2329, 0, 1, 615, 651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3802, 3905, 0, 0, 0, 3252, 0, 3872, 14, 0, 958, 122, 0, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 12, 1381, 4096, 845, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 115, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4074, 1393, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 0, 0, 0, 0, 0, 128, 0, 0, 4048, 0, 4096, 0, 2990, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17420, 16838, 17662, 15765, 18512, 14528, 15023, 15324] avg=16384.0000 max=18512 max_rank=4 min=14528 min_rank=5 imbalance_ratio=1.129883 max_load_reduction=112560 max_load_reduction_pct=0.858765 moved_to_echo=63387 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[13440, 128, 128, 13568, 29184, 24832, 128, 128, 1408, 128, 16896, 128, 128, 31744, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[30080, 128, 128, 128, 128, 512, 9984, 128, 128, 32512, 640, 32768, 128, 23936, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 11904, 128, 1280, 128, 15232, 1280, 384, 3456, 0, 1920, 128, 31744, 9216, 128, 32640, 22528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 384, 256, 384, 11136, 32768, 21248, 128, 32512, 128, 128, 128, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 32768, 128, 9600, 768, 0, 128, 128, 1280, 128, 256, 512, 128, 128, 128, 32384, 11264, 9984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 31616, 32512, 128, 128, 128, 128, 128, 32768, 128, 256, 128, 27776, 128, 128, 1024, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[1408, 128, 128, 128, 31744, 384, 640, 128, 256, 24704, 30720, 128, 128, 128, 128, 256, 15744, 11264, 10624, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[25088, 25728, 0, 0, 128, 30464, 128, 24960, 256, 0, 1664, 3328, 128, 19968, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 449, 0, 0, 837, 0, 0, 0, 179, 4095, 4096, 0, 0, 4096, 4096, 4096, 12, 3643, 4096, 3696, 0, 0, 0, 4096, 0, 0, 0, 1893, 4096, 0, 0, 4096, 4, 4050, 0, 0, 2568, 4052, 0, 61, 23, 0, 481, 0, 0, 0, 0, 1, 0, 8, 4091, 0, 0, 0, 1, 4096, 0, 4, 2079, 0, 4096, 0, 0, 4079, 0, 0, 94, 0, 0, 18, 4096, 22, 1, 4096, 3780, 4096, 0, 0, 25, 0, 0, 134, 3990, 0, 0, 0, 0, 0, 353, 0, 4095, 0, 4096, 44, 0, 4073, 1651, 0, 0, 0, 1060, 3986, 0, 31, 4096, 0, 0, 4096, 0, 0, 20, 0, 1755, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[2, 1, 21, 1848, 173, 0, 478, 0, 36, 6, 518, 4006, 4078, 16, 13, 4068, 4093, 4072, 14, 0, 4092, 621, 9, 9, 0, 4078, 0, 0, 0, 2292, 4075, 57, 19, 4069, 2379, 4031, 16, 11, 4096, 4094, 34, 0, 1104, 0, 143, 12, 0, 1, 0, 59, 7, 396, 3784, 0, 0, 0, 134, 4081, 24, 0, 3100, 0, 3953, 32, 14, 4082, 10, 0, 0, 4, 0, 994, 4076, 0, 0, 4081, 3005, 4082, 0, 0, 56, 1, 14, 262, 4082, 5, 0, 2, 2006, 1, 0, 0, 4084, 0, 4063, 1, 2, 3561, 20, 0, 0, 8, 182, 3904, 0, 0, 4085, 4, 3, 4050, 0, 26, 0, 20, 1907, 4082, 0, 0, 0, 10, 0, 2, 0, 0, 0, 2, 24, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 522, 365, 0, 3574, 0, 0, 0, 102, 4096, 4017, 0, 0, 4086, 4096, 4095, 0, 4, 4095, 1869, 0, 1, 0, 4041, 0, 0, 0, 263, 4091, 0, 2, 4087, 678, 4078, 0, 0, 3374, 4023, 0, 105, 0, 0, 575, 0, 0, 1, 1, 0, 0, 557, 691, 0, 4, 0, 1, 4095, 0, 6, 3600, 59, 4089, 0, 0, 4025, 0, 5, 130, 0, 2, 1612, 4094, 15, 29, 4092, 2583, 4093, 0, 2, 88, 8, 1, 343, 4069, 0, 28, 2, 0, 0, 0, 0, 3968, 2, 4075, 6, 9, 3530, 2518, 0, 2, 0, 3416, 3914, 3, 100, 4087, 2, 0, 4096, 0, 0, 11, 0, 2648, 4091, 0, 1, 0, 0, 0, 0, 0, 1, 24, 0, 2, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 1354, 0, 500, 44, 0, 3, 0, 0, 10, 251, 2750, 3996, 0, 0, 4096, 4094, 4034, 0, 2790, 4095, 2440, 340, 1, 1, 3865, 0, 0, 0, 1542, 4096, 48, 0, 4095, 92, 4031, 3, 0, 840, 3999, 2, 137, 201, 0, 4, 0, 0, 0, 0, 4, 0, 13, 1210, 2, 0, 0, 0, 4093, 0, 0, 3961, 0, 4068, 0, 0, 4094, 0, 0, 0, 0, 0, 3776, 4096, 5, 0, 4093, 3925, 4095, 0, 57, 3, 15, 0, 994, 3772, 0, 0, 4, 7, 0, 9, 0, 4096, 0, 4096, 424, 0, 4078, 1539, 0, 0, 0, 2622, 4058, 0, 0, 4096, 0, 0, 4090, 0, 24, 0, 0, 1797, 4094, 0, 0, 7, 0, 0, 1, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[17, 1413, 0, 1144, 0, 0, 6, 0, 0, 2, 5, 2609, 4087, 0, 0, 4096, 4092, 4096, 0, 2102, 4084, 1440, 0, 0, 0, 3995, 78, 0, 0, 1379, 4096, 175, 0, 4096, 2118, 4096, 0, 0, 4031, 4096, 0, 17, 2105, 0, 140, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 4088, 0, 1270, 2686, 0, 4096, 0, 2, 4096, 0, 0, 0, 0, 0, 280, 4096, 8, 0, 4096, 3214, 4062, 1, 0, 17, 0, 0, 62, 4093, 3, 0, 0, 8, 0, 875, 0, 4053, 0, 4096, 707, 0, 2954, 1032, 0, 0, 0, 1253, 3736, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 220, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4, 0, 0, 2506, 38, 0, 3824, 0, 0, 0, 92, 3392, 4094, 0, 0, 4094, 4089, 4096, 28, 272, 4093, 3827, 3, 0, 0, 3982, 18, 0, 1, 3060, 4093, 0, 0, 4088, 1173, 4096, 1, 1, 3461, 3899, 0, 23, 760, 0, 107, 0, 1, 1, 4, 0, 0, 2, 3224, 0, 0, 0, 0, 4090, 0, 1570, 438, 0, 4094, 0, 12, 4096, 0, 0, 2, 0, 0, 418, 4095, 0, 12, 4096, 3119, 4096, 4, 0, 407, 0, 0, 265, 4054, 0, 0, 0, 998, 0, 18, 0, 4085, 10, 4081, 31, 0, 3488, 27, 0, 0, 0, 474, 4089, 0, 0, 4095, 0, 0, 4096, 0, 15, 0, 0, 148, 4093, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 4, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 1, 3361, 4, 4, 445, 1, 11, 3, 163, 4085, 4090, 0, 0, 4095, 4094, 4092, 0, 52, 4028, 4057, 6, 0, 0, 4073, 0, 0, 0, 2727, 4093, 4, 1, 4094, 26, 4089, 0, 3, 2280, 3983, 0, 0, 231, 0, 18, 1, 3, 0, 2, 0, 0, 845, 3016, 2, 0, 0, 0, 4093, 0, 11, 477, 1, 4093, 3, 0, 4091, 3, 0, 0, 0, 3, 2811, 4096, 3, 7, 4094, 4078, 4095, 0, 0, 33, 0, 2, 116, 4092, 2, 0, 0, 16, 2, 33, 0, 4088, 0, 4079, 58, 0, 3785, 41, 2, 2, 0, 3927, 4076, 0, 0, 4093, 0, 0, 4089, 0, 0, 2, 0, 363, 4092, 0, 1, 0, 2, 0, 4, 2, 24, 0, 2, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 3486, 0, 0, 9, 0, 0, 0, 1580, 4083, 4096, 0, 0, 4096, 4096, 4096, 0, 2, 4096, 4096, 0, 0, 0, 4085, 0, 0, 0, 115, 4096, 0, 0, 4096, 2365, 3695, 0, 0, 3468, 3976, 0, 0, 173, 0, 4, 0, 0, 0, 0, 1, 0, 5, 4068, 0, 0, 0, 1, 4096, 0, 352, 570, 0, 4096, 0, 0, 4074, 0, 0, 0, 0, 0, 1160, 4096, 6, 1, 4096, 3675, 4096, 0, 0, 0, 0, 0, 895, 4082, 0, 0, 0, 1, 0, 1807, 0, 4079, 0, 4096, 167, 0, 2641, 2195, 0, 0, 0, 614, 3899, 0, 21, 4096, 0, 0, 4096, 0, 0, 22, 0, 22, 4096, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 30, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 449, 0, 0, 837, 0, 0, 0, 179, 4095, 4096, 0, 0, 4096, 677, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 3643, 2657, 3696, 0, 0, 0, 4096, 0, 0, 0, 1893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3419, 4, 4050, 0, 0, 2568, 4052, 0, 61, 23, 0, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 4091, 0, 0, 0, 1, 4096, 0, 4, 2079, 0, 4096, 0, 1439, 511, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 94, 0, 0, 18, 0, 22, 1, 3356, 3780, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 134, 3990, 0, 0, 0, 0, 0, 353, 0, 4095, 0, 4096, 44, 1655, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 1651, 0, 0, 0, 1060, 3986, 0, 31, 1973, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 1755, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4096, 4096, 2970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14658, 20093, 14658, 16794, 15446, 15518, 16870, 17035] avg=16384.0000 max=20093 max_rank=1 min=14658 min_rank=0 imbalance_ratio=1.226379 max_load_reduction=110979 max_load_reduction_pct=0.846703 moved_to_echo=64869 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 21, 1848, 173, 0, 478, 0, 36, 6, 518, 4006, 4078, 16, 13, 4068, 678, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 14, 0, 2650, 621, 9, 9, 0, 4078, 0, 0, 0, 2292, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 3391, 2379, 4031, 16, 11, 4096, 4094, 34, 0, 1104, 0, 143, 12, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 7, 396, 3784, 0, 0, 0, 134, 4081, 24, 0, 3100, 0, 3953, 32, 1442, 512, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4082, 10, 0, 0, 4, 0, 994, 0, 0, 0, 3334, 3005, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 1, 14, 262, 4082, 5, 0, 2, 2006, 1, 0, 0, 4084, 0, 4063, 1, 1656, 1121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3561, 20, 0, 0, 8, 182, 3904, 0, 0, 1960, 4, 3, 4050, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 1907, 4082, 0, 0, 0, 10, 0, 2, 0, 0, 0, 2, 24, 0, 4093, 4076, 2954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16177, 13802, 19331, 17993, 15525, 17354, 13720, 17170] avg=16384.0000 max=19331 max_rank=2 min=13720 min_rank=6 imbalance_ratio=1.179871 max_load_reduction=111741 max_load_reduction_pct=0.852516 moved_to_echo=63769 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1354, 0, 500, 44, 0, 3, 0, 0, 10, 251, 2750, 3996, 0, 0, 4096, 677, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 2790, 2656, 2440, 340, 1, 1, 3865, 0, 0, 0, 1542, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3418, 92, 4031, 3, 0, 840, 3999, 2, 137, 201, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 13, 1210, 2, 0, 0, 0, 4093, 0, 0, 3961, 0, 4068, 0, 1439, 511, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 3776, 0, 5, 0, 3353, 3925, 4095, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 15, 0, 994, 3772, 0, 0, 4, 7, 0, 9, 0, 4096, 0, 4096, 424, 1655, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 1539, 0, 0, 0, 2622, 4058, 0, 0, 1973, 0, 0, 4090, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1797, 4094, 0, 0, 7, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 4096, 2970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13910, 17717, 12727, 15769, 19305, 16201, 18384, 17059] avg=16384.0000 max=19305 max_rank=4 min=12727 min_rank=2 imbalance_ratio=1.178284 max_load_reduction=111767 max_load_reduction_pct=0.852715 moved_to_echo=70949 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[17, 1413, 0, 1144, 0, 0, 6, 0, 0, 2, 5, 2609, 4087, 0, 0, 4096, 677, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2102, 2649, 1440, 0, 0, 0, 3995, 78, 0, 0, 1379, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3419, 2118, 4096, 0, 0, 4031, 4096, 0, 17, 2105, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 4088, 0, 1270, 2686, 0, 4096, 0, 1435, 511, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 280, 0, 8, 0, 3356, 3214, 4062, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 62, 4093, 3, 0, 0, 8, 0, 875, 0, 4053, 0, 4096, 707, 1655, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2954, 1032, 0, 0, 0, 1253, 3736, 1, 0, 1973, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4092, 4096, 2970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14285, 15914, 20022, 18616, 15019, 16695, 15045, 15476] avg=16384.0000 max=20022 max_rank=2 min=14285 min_rank=0 imbalance_ratio=1.222046 max_load_reduction=111050 max_load_reduction_pct=0.847244 moved_to_echo=62235 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 2506, 38, 0, 3824, 0, 0, 0, 92, 3392, 4094, 0, 0, 4094, 676, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 28, 272, 2655, 3827, 3, 0, 0, 3982, 18, 0, 1, 3060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3412, 1173, 4096, 1, 1, 3461, 3899, 0, 23, 760, 0, 107, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2, 3224, 0, 0, 0, 0, 4090, 0, 1570, 438, 0, 4094, 0, 1438, 511, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 2, 0, 0, 418, 0, 0, 12, 3356, 3119, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 0, 265, 4054, 0, 0, 0, 998, 0, 18, 0, 4085, 10, 4081, 31, 1655, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3488, 27, 0, 0, 0, 474, 4089, 0, 0, 1973, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 4093, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 4, 0, 4089, 4095, 2968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18949, 17942, 16935, 15838, 15115, 16729, 14162, 15402] avg=16384.0000 max=18949 max_rank=0 min=14162 min_rank=6 imbalance_ratio=1.156555 max_load_reduction=112123 max_load_reduction_pct=0.855431 moved_to_echo=61408 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 522, 365, 0, 3574, 0, 0, 0, 102, 4096, 4017, 0, 0, 4086, 676, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 2656, 1869, 0, 1, 0, 4041, 0, 0, 0, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3411, 678, 4078, 0, 0, 3374, 4023, 0, 105, 0, 0, 575, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 557, 691, 0, 4, 0, 1, 4095, 0, 6, 3600, 59, 4089, 0, 1439, 510, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 5, 130, 0, 2, 1612, 0, 15, 29, 3353, 2583, 4093, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 8, 1, 343, 4069, 0, 28, 2, 0, 0, 0, 0, 3968, 2, 4075, 6, 1652, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3530, 2518, 0, 2, 0, 3416, 3914, 3, 100, 1968, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 2648, 4091, 0, 1, 0, 0, 0, 0, 0, 1, 24, 0, 2, 2, 4096, 4094, 2966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17667, 12929, 16247, 15519, 15849, 15367, 19558, 17936] avg=16384.0000 max=19558 max_rank=6 min=12929 min_rank=1 imbalance_ratio=1.193726 max_load_reduction=111514 max_load_reduction_pct=0.850784 moved_to_echo=68710 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 3361, 4, 4, 445, 1, 11, 3, 163, 4085, 4090, 0, 0, 4095, 677, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 52, 2613, 4057, 6, 0, 0, 4073, 0, 0, 0, 2727, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3417, 26, 4089, 0, 3, 2280, 3983, 0, 0, 231, 0, 18, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 845, 3016, 2, 0, 0, 0, 4093, 0, 11, 477, 1, 4093, 3, 1415, 511, 467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 3, 2811, 0, 3, 7, 3354, 4078, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 2, 116, 4092, 2, 0, 0, 16, 2, 33, 0, 4088, 0, 4079, 58, 1654, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3785, 41, 2, 2, 0, 3927, 4076, 0, 0, 1972, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 363, 4092, 0, 1, 0, 2, 0, 4, 2, 24, 0, 2, 1, 0, 4094, 4096, 2968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17170, 17624, 14052, 14936, 18445, 15300, 17894, 15651] avg=16384.0000 max=18445 max_rank=4 min=14052 min_rank=2 imbalance_ratio=1.125793 max_load_reduction=112627 max_load_reduction_pct=0.859276 moved_to_echo=67290 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3486, 0, 0, 9, 0, 0, 0, 1580, 4083, 4096, 0, 0, 4096, 677, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 2657, 4096, 0, 0, 0, 4085, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3419, 2365, 3695, 0, 0, 3468, 3976, 0, 0, 173, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 4068, 0, 0, 0, 1, 4096, 0, 352, 570, 0, 4096, 0, 1439, 511, 468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 1160, 0, 6, 1, 3356, 3675, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 895, 4082, 0, 0, 0, 1, 0, 1807, 0, 4079, 0, 4096, 167, 1655, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2641, 2195, 0, 0, 0, 614, 3899, 0, 21, 1973, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 22, 4096, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 30, 0, 4096, 4096, 2970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18256, 15051, 17100, 15607, 16368, 17908, 15439, 15343] avg=16384.0000 max=18256 max_rank=0 min=15051 min_rank=1 imbalance_ratio=1.114258 max_load_reduction=112816 max_load_reduction_pct=0.860718 moved_to_echo=65058 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 28160, 9088, 128, 128, 128, 13568, 31744, 128, 256, 15872, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[640, 128, 128, 3072, 32256, 128, 128, 128, 3072, 128, 3200, 0, 32640, 128, 32768, 1536, 13312, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 27392, 8960, 32256, 128, 128, 24192, 32128, 128, 384, 4608, 0, 1536, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32768, 128, 8960, 21248, 22144, 384, 128, 128, 32256, 128, 0, 128, 13312, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 2816, 128, 13824, 640, 128, 9216, 128, 128, 128, 2944, 29184, 32640, 128, 128, 32768, 5504, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 8960, 32768, 0, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 23808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 1920, 24192, 128, 128, 0, 256, 32768, 128, 3328, 17024, 128, 32640, 128, 11520, 4096, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32640, 128, 128, 256, 128, 128, 11136, 0, 128, 128, 26880, 27392, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 2798, 0, 0, 1, 0, 1, 0, 0, 1, 3724, 3404, 0, 4, 0, 0, 0, 3, 0, 1764, 0, 1, 264, 0, 0, 4096, 0, 1, 0, 0, 1, 1640, 4096, 0, 0, 0, 4089, 3798, 4096, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 3592, 0, 0, 0, 0, 36, 18, 1, 0, 4095, 0, 0, 4046, 0, 0, 4080, 0, 0, 1627, 0, 0, 3867, 384, 0, 0, 1, 4096, 4096, 0, 0, 4074, 76, 0, 0, 0, 0, 3597, 4096, 0, 2, 0, 10, 0, 121, 0, 4076, 0, 0, 4074, 4026, 4096, 20, 0, 3633, 17, 0, 0, 4091, 0, 0, 0, 49, 2, 4091, 2692, 0, 0, 0, 0, 3923, 102, 8, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 46, 1661, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 4003, 0, 0, 0, 0, 0, 0, 0, 297, 1, 280, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 3581, 4096, 8, 28, 0, 4096, 4096, 4096, 0, 4096, 4054, 0, 0, 0, 0, 0, 0, 0, 4054, 241, 0, 0, 0, 3, 1437, 53, 0, 4096, 1, 0, 4091, 0, 0, 4045, 0, 0, 2743, 0, 0, 3975, 56, 96, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 10, 1, 3796, 4096, 0, 0, 0, 848, 0, 32, 0, 3961, 1, 0, 4059, 4074, 4096, 204, 0, 1488, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4095, 318, 0, 53, 0, 0, 3817, 17, 15, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 4096, 0, 15, 3343, 0, 1, 0, 0, 0, 0, 0, 0, 710, 4092, 0, 0, 0, 0, 232, 0, 0, 350, 3, 84, 2369, 0, 0, 4090, 0, 0, 19, 0, 0, 3000, 4096, 108, 17, 0, 3930, 4042, 4096, 0, 4095, 4094, 0, 0, 1, 0, 15, 0, 0, 4073, 0, 0, 0, 1, 0, 233, 0, 0, 4094, 0, 1, 4096, 2, 0, 4096, 0, 2, 3821, 0, 0, 4086, 3923, 0, 0, 0, 4096, 4096, 1, 0, 4082, 0, 6, 0, 0, 0, 417, 4096, 1, 1, 0, 87, 0, 257, 0, 4081, 6, 0, 4096, 4093, 4096, 0, 0, 3941, 0, 1, 25, 4096, 0, 1, 0, 23, 0, 4090, 519, 1, 0, 0, 0, 493, 701, 156, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4081, 1, 0, 0, 4096, 0, 4, 3953, 0, 0, 6, 0, 0, 0, 0, 0, 2582, 4096, 0, 0, 0, 0, 8, 0, 0, 54, 0, 425, 231, 0, 0, 4090, 0, 1, 1, 8, 42, 3323, 4096, 0, 0, 0, 4095, 4058, 4094, 0, 4096, 3880, 0, 0, 0, 0, 0, 4, 0, 4042, 1, 0, 0, 1, 4, 3094, 0, 1, 4086, 0, 0, 3494, 0, 0, 3820, 0, 0, 1181, 0, 0, 4072, 42, 120, 0, 11, 4096, 4088, 0, 2, 4054, 0, 1, 0, 23, 0, 669, 4096, 1, 2, 0, 1711, 0, 33, 0, 4071, 0, 0, 4088, 4096, 4079, 5, 0, 4072, 7, 0, 10, 4096, 0, 0, 0, 0, 0, 4096, 33, 0, 98, 0, 0, 4068, 3, 92, 4088] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4092, 0, 3, 0, 4096, 0, 12, 3937, 5, 0, 0, 0, 7, 0, 0, 0, 301, 3624, 0, 2, 0, 2, 0, 0, 0, 490, 0, 1406, 0, 0, 0, 4093, 0, 0, 0, 5, 0, 4054, 4094, 0, 0, 19, 4095, 4096, 4096, 8, 4096, 2714, 2, 5, 4, 0, 0, 1, 0, 4082, 0, 5, 0, 0, 10, 1361, 21, 0, 4075, 0, 6, 3978, 3, 3, 4046, 0, 0, 3805, 2, 9, 4081, 2678, 0, 1, 0, 4090, 4094, 4, 0, 4032, 0, 0, 0, 4, 0, 22, 4091, 0, 15, 0, 2255, 0, 0, 2, 4096, 8, 0, 4090, 3379, 4092, 1, 0, 4072, 0, 0, 1, 4090, 0, 0, 0, 1, 0, 4085, 0, 4, 7, 0, 0, 2861, 57, 3, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4092, 54, 1, 0, 4096, 1, 0, 1359, 0, 0, 41, 0, 0, 6, 0, 1, 2286, 3108, 6, 0, 0, 0, 1, 0, 0, 0, 0, 3417, 120, 21, 2, 4087, 0, 0, 129, 0, 122, 3771, 4081, 70, 0, 0, 4074, 4095, 4083, 1, 4043, 4087, 0, 20, 0, 0, 0, 109, 11, 3754, 13, 1, 0, 0, 0, 36, 2, 0, 4096, 0, 3, 4092, 39, 1, 4080, 0, 0, 4066, 0, 0, 3713, 0, 0, 0, 0, 4069, 4095, 6, 0, 4054, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 1, 3446, 4, 0, 0, 4042, 0, 0, 3644, 3421, 4079, 40, 0, 3947, 0, 0, 0, 4080, 1, 0, 0, 0, 0, 4095, 2, 233, 440, 8, 0, 3870, 31, 0, 4072] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4084, 0, 0, 0, 3779, 0, 15, 4042, 9, 15, 1, 0, 0, 0, 0, 0, 3, 3662, 26, 5, 2, 0, 0, 0, 0, 34, 6, 422, 2, 6, 18, 4041, 39, 6, 1, 11, 0, 3756, 4066, 79, 135, 48, 4086, 4077, 4005, 0, 4062, 4044, 1, 62, 0, 0, 143, 0, 8, 4090, 0, 0, 0, 30, 0, 1871, 0, 25, 4061, 47, 22, 4023, 3, 0, 4096, 0, 14, 3953, 14, 1, 3909, 0, 4, 14, 0, 4079, 4080, 34, 3, 4087, 3, 0, 0, 1, 14, 40, 4047, 2, 0, 0, 20, 488, 0, 0, 4087, 0, 18, 4090, 4063, 3996, 12, 6, 3555, 5, 0, 0, 4055, 5, 5, 5, 0, 24, 4046, 3166, 22, 248, 0, 37, 3708, 2, 92, 3949] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1550, 2050, 0, 0, 0, 0, 0, 0, 0, 3128, 1, 413, 8, 0, 0, 4096, 0, 0, 3, 0, 0, 3326, 4096, 1, 1, 0, 4095, 4096, 4096, 0, 4096, 3986, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 0, 0, 0, 5, 190, 0, 0, 4096, 1, 0, 4087, 0, 0, 3801, 0, 0, 2933, 0, 0, 4096, 41, 54, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 27, 4096, 0, 0, 0, 3378, 0, 1, 0, 4085, 4, 0, 4096, 4095, 4096, 7, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 441, 0, 113, 0, 0, 3107, 18, 121, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 46, 1661, 0, 0, 0, 0, 0, 0, 0, 0, 3646, 980, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 4003, 0, 0, 0, 0, 0, 0, 0, 297, 1, 280, 1, 0, 0, 4096, 2670, 1060, 828, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3581, 0, 8, 28, 0, 4096, 4096, 983, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 241, 0, 0, 0, 3, 1437, 53, 0, 4096, 4096, 3116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 1, 0, 4091, 0, 0, 4045, 0, 0, 2743, 0, 0, 3975, 56, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3036, 0, 0, 4096, 0, 0, 0, 10, 1, 3796, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 848, 0, 32, 0, 3133, 1, 0, 4059, 4074, 0, 204, 0, 1488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3756, 0, 0, 0, 1, 0, 4095, 318, 0, 53, 0, 0, 3817, 17, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14968, 17567, 16846, 17096, 15457, 19131, 13839, 16168] avg=16384.0000 max=19131 max_rank=5 min=13839 min_rank=6 imbalance_ratio=1.167664 max_load_reduction=111941 max_load_reduction_pct=0.854042 moved_to_echo=64595 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 4096, 0, 15, 3343, 0, 1, 0, 0, 0, 0, 0, 0, 3645, 981, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 710, 4092, 0, 0, 0, 0, 232, 0, 0, 350, 3, 84, 2369, 0, 0, 4090, 2670, 1060, 853, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 3000, 0, 108, 17, 0, 3930, 4042, 983, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 15, 0, 0, 4073, 0, 0, 0, 1, 0, 233, 0, 0, 4096, 4095, 3115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 0, 1, 4096, 2, 0, 4096, 0, 2, 3821, 0, 0, 4086, 3923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3036, 1, 0, 4082, 0, 6, 0, 0, 0, 417, 4096, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 257, 0, 3228, 6, 0, 4096, 4093, 0, 0, 0, 3941, 0, 1, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3756, 0, 1, 0, 23, 0, 4090, 519, 1, 0, 0, 0, 493, 701, 156, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16615, 16853, 16193, 15629, 20476, 15736, 15734, 13836] avg=16384.0000 max=20476 max_rank=4 min=13836 min_rank=7 imbalance_ratio=1.249756 max_load_reduction=110596 max_load_reduction_pct=0.843781 moved_to_echo=65782 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4081, 1, 0, 0, 4096, 0, 4, 3953, 0, 0, 6, 0, 0, 0, 0, 0, 3637, 977, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2582, 4096, 0, 0, 0, 0, 8, 0, 0, 54, 0, 425, 231, 0, 0, 4090, 2668, 1058, 851, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 8, 42, 3323, 0, 0, 0, 0, 4095, 4058, 983, 0, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4042, 1, 0, 0, 1, 4, 3094, 0, 1, 4096, 4096, 3102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 0, 0, 3494, 0, 0, 3820, 0, 0, 1181, 0, 0, 4072, 42, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 3030, 0, 2, 4054, 0, 1, 0, 23, 0, 669, 4096, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1711, 0, 33, 0, 3220, 0, 0, 4088, 4096, 0, 5, 0, 4072, 7, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3756, 0, 0, 0, 0, 0, 4096, 33, 0, 98, 0, 0, 4068, 3, 92, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17198, 16403, 16391, 18441, 13178, 15985, 17242, 16234] avg=16384.0000 max=18441 max_rank=3 min=13178 min_rank=4 imbalance_ratio=1.125549 max_load_reduction=112631 max_load_reduction_pct=0.859306 moved_to_echo=62639 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 2798, 0, 0, 1, 0, 1, 0, 0, 1, 3645, 981, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3724, 3404, 0, 4, 0, 0, 0, 3, 0, 1764, 0, 1, 264, 0, 0, 4096, 2670, 1060, 852, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1640, 0, 0, 0, 0, 4089, 3798, 983, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3592, 0, 0, 0, 0, 36, 18, 1, 0, 4096, 4092, 3115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 0, 0, 4046, 0, 0, 4080, 0, 0, 1627, 0, 0, 3867, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 3036, 0, 0, 4074, 76, 0, 0, 0, 0, 3597, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 121, 0, 3224, 0, 0, 4074, 4026, 0, 20, 0, 3633, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3751, 0, 0, 0, 49, 2, 4091, 2692, 0, 0, 0, 0, 3923, 102, 8, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16062, 18182, 14608, 14950, 14454, 18978, 15125, 18713] avg=16384.0000 max=18978 max_rank=5 min=14454 min_rank=4 imbalance_ratio=1.158325 max_load_reduction=112094 max_load_reduction_pct=0.855209 moved_to_echo=67270 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4084, 0, 0, 0, 3779, 0, 15, 4042, 9, 15, 1, 0, 0, 0, 0, 0, 3620, 957, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3662, 26, 5, 2, 0, 0, 0, 0, 34, 6, 422, 2, 6, 18, 4041, 2612, 1059, 860, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 6, 1, 11, 0, 3756, 0, 79, 135, 48, 4086, 4077, 957, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 62, 0, 0, 143, 0, 8, 4090, 0, 0, 0, 30, 0, 1871, 0, 25, 4066, 4062, 3039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 441, 47, 22, 4023, 3, 0, 4096, 0, 14, 3953, 14, 1, 3909, 0, 4, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 3021, 34, 3, 4087, 3, 0, 0, 1, 14, 40, 4047, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 488, 0, 0, 3227, 0, 18, 4090, 4063, 0, 12, 6, 3555, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3713, 5, 5, 5, 0, 24, 4046, 3166, 22, 248, 0, 37, 3708, 2, 92, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16958, 13100, 17239, 17397, 16541, 15331, 15484, 19022] avg=16384.0000 max=19022 max_rank=7 min=13100 min_rank=1 imbalance_ratio=1.161011 max_load_reduction=112050 max_load_reduction_pct=0.854874 moved_to_echo=66378 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 3, 0, 4096, 0, 12, 3937, 5, 0, 0, 0, 7, 0, 0, 0, 3628, 979, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 301, 3624, 0, 2, 0, 2, 0, 0, 0, 490, 0, 1406, 0, 0, 0, 4093, 2670, 1059, 857, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4054, 0, 0, 0, 19, 4095, 4096, 983, 8, 0, 2714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 4, 0, 0, 1, 0, 4082, 0, 5, 0, 0, 10, 1361, 21, 0, 4094, 4096, 3113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 447, 0, 6, 3978, 3, 3, 4046, 0, 0, 3805, 2, 9, 4081, 2678, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3035, 4, 0, 4032, 0, 0, 0, 4, 0, 22, 4091, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2255, 0, 0, 2, 3239, 8, 0, 4090, 3379, 0, 1, 0, 4072, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3750, 0, 0, 0, 1, 0, 4085, 0, 4, 7, 0, 0, 2861, 57, 3, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17202, 14844, 15974, 16794, 19059, 15293, 17047, 14859] avg=16384.0000 max=19059 max_rank=4 min=14844 min_rank=1 imbalance_ratio=1.163269 max_load_reduction=112013 max_load_reduction_pct=0.854591 moved_to_echo=66258 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4092, 54, 1, 0, 4096, 1, 0, 1359, 0, 0, 41, 0, 0, 6, 0, 1, 3646, 977, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2286, 3108, 6, 0, 0, 0, 1, 0, 0, 0, 0, 3417, 120, 21, 2, 4087, 2661, 1060, 845, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 122, 3771, 0, 70, 0, 0, 4074, 4095, 981, 1, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 109, 11, 3754, 13, 1, 0, 0, 0, 36, 2, 0, 4081, 4043, 3102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 0, 3, 4092, 39, 1, 4080, 0, 0, 4066, 0, 0, 3713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 3035, 6, 0, 4054, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3446, 4, 0, 0, 3197, 0, 0, 3644, 3421, 0, 40, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3741, 1, 0, 0, 0, 0, 4095, 2, 233, 440, 8, 0, 3870, 31, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14715, 17953, 17330, 15172, 16444, 15266, 17699, 16493] avg=16384.0000 max=17953 max_rank=1 min=14715 min_rank=0 imbalance_ratio=1.095764 max_load_reduction=113119 max_load_reduction_pct=0.863029 moved_to_echo=65902 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 3646, 981, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1550, 2050, 0, 0, 0, 0, 0, 0, 0, 3128, 1, 413, 8, 0, 0, 4096, 2670, 1060, 854, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3326, 0, 1, 1, 0, 4095, 4096, 983, 0, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 0, 0, 0, 5, 190, 0, 0, 4096, 4096, 3115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 450, 1, 0, 4087, 0, 0, 3801, 0, 0, 2933, 0, 0, 4096, 41, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3036, 0, 0, 4096, 0, 0, 0, 0, 1, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3378, 0, 1, 0, 3231, 4, 0, 4096, 4095, 0, 7, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3756, 0, 0, 0, 0, 0, 4095, 441, 0, 113, 0, 0, 3107, 18, 121, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17354, 16170, 16491, 15593, 15463, 15352, 18902, 15747] avg=16384.0000 max=18902 max_rank=6 min=15352 min_rank=5 imbalance_ratio=1.153687 max_load_reduction=112170 max_load_reduction_pct=0.855789 moved_to_echo=65464 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 0, 32512, 128, 128, 25216, 128, 128, 128, 0, 128, 128, 0, 128, 29184, 7936, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 256, 128, 256, 26496, 0, 384, 256, 128, 32640, 32384, 7936, 128, 0, 30976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[11776, 512, 512, 128, 25728, 128, 128, 32256, 31360, 0, 384, 128, 28800, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[30080, 128, 128, 128, 128, 128, 32768, 7296, 384, 1024, 128, 128, 25856, 1024, 512, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 24320, 128, 128, 32640, 128, 128, 0, 128, 128, 8576, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 256, 128, 128, 31872, 256, 128, 0, 128, 128, 8320, 128, 128, 32768, 32768, 24832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[3712, 128, 128, 32000, 128, 128, 32128, 0, 128, 24192, 128, 128, 31872, 7168, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[15232, 28160, 128, 128, 128, 128, 256, 128, 0, 6144, 128, 6528, 3072, 128, 128, 32768, 21376, 8576, 6912, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 13, 4096, 0, 1, 0, 4096, 2107, 4035, 0, 0, 1, 0, 362, 4091, 4094, 2108, 0, 4096, 0, 4096, 1, 0, 41, 6, 0, 0, 4096, 0, 3623, 0, 1, 0, 4096, 0, 9, 0, 0, 0, 4085, 0, 4096, 4095, 0, 1, 0, 0, 0, 0, 0, 4076, 0, 0, 4069, 4096, 4096, 0, 3796, 1, 120, 3986, 0, 0, 39, 0, 0, 0, 4096, 4096, 0, 4096, 4076, 0, 0, 0, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 0, 0, 0, 4086, 0, 0, 2946, 4096, 0, 22, 6, 0, 4096, 0, 0, 0, 4095, 1369, 4096, 7, 0, 4096, 0, 198, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 7, 4094, 4093, 1129, 0, 0, 0, 0, 79, 4096, 4077, 3952, 0, 4096, 0, 4096, 0, 65, 30, 18, 0, 0, 4096, 0, 1831, 0, 0, 0, 4088, 1, 11, 0, 0, 0, 4096, 0, 4096, 4093, 2, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4090, 4096, 4096, 0, 4089, 0, 273, 3944, 0, 28, 1696, 0, 0, 0, 4094, 4095, 0, 4095, 4096, 0, 0, 0, 0, 1, 767, 0, 0, 0, 2, 0, 0, 1, 0, 0, 109, 0, 0, 1, 0, 8, 0, 0, 0, 0, 45, 3, 0, 0, 0, 4096, 9, 0, 2680, 4096, 0, 0, 7, 0, 4096, 0, 0, 0, 4096, 181, 4096, 0, 0, 4095, 0, 3659, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4090, 0, 0, 6, 4095, 4083, 1161, 0, 0, 0, 0, 0, 4096, 4081, 335, 0, 4091, 16, 4092, 0, 7, 191, 3226, 0, 0, 4096, 0, 2271, 0, 0, 0, 4096, 0, 241, 0, 0, 0, 4076, 11, 4094, 4088, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 4093, 4093, 0, 3973, 50, 0, 1711, 0, 0, 1328, 0, 0, 0, 4096, 4096, 0, 4096, 4056, 11, 0, 0, 0, 3, 2109, 0, 15, 1, 0, 0, 0, 0, 0, 9, 0, 3, 0, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 4096, 14, 0, 3888, 4096, 0, 0, 39, 3, 4084, 0, 0, 0, 4093, 3, 4096, 16, 0, 4096, 6, 4051, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 3, 4092, 2788, 2329, 0, 0, 29, 7, 3847, 4096, 3930, 3643, 0, 4085, 0, 4092, 11, 0, 448, 108, 7, 1, 4094, 0, 3756, 0, 87, 0, 4095, 0, 1093, 0, 0, 0, 3850, 0, 4092, 4090, 21, 0, 4, 0, 0, 2, 0, 4070, 0, 0, 4095, 4090, 4093, 0, 4070, 17, 11, 4094, 5, 2, 0, 0, 0, 2, 4087, 4094, 0, 4093, 4096, 0, 0, 0, 2, 0, 1054, 3, 8, 6, 1, 2, 0, 0, 0, 0, 0, 1, 0, 4, 0, 2, 0, 0, 0, 2, 0, 4, 0, 0, 1, 4096, 0, 0, 56, 4085, 4, 2, 22, 5, 4091, 0, 4, 0, 4085, 1508, 4092, 0, 0, 4077, 0, 136, 2, 0, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[5, 11, 473, 4078, 0, 0, 0, 4073, 3909, 42, 80, 0, 9, 2, 77, 4077, 4038, 2782, 0, 4064, 45, 3995, 12, 0, 1513, 356, 0, 1, 4021, 13, 854, 0, 8, 4, 4051, 0, 127, 0, 17, 0, 4062, 0, 4040, 4095, 0, 85, 0, 0, 23, 11, 0, 3986, 53, 2, 4017, 4036, 4056, 42, 3096, 0, 83, 2941, 0, 0, 422, 9, 29, 0, 4081, 4089, 12, 4085, 4058, 0, 0, 2, 40, 12, 3712, 4, 16, 0, 1, 25, 0, 5, 3, 0, 0, 0, 0, 48, 6, 0, 0, 4, 161, 34, 31, 0, 70, 3, 0, 4006, 0, 0, 2477, 4080, 19, 0, 0, 27, 4033, 50, 0, 36, 4040, 1731, 4049, 27, 48, 4080, 5, 4005, 0, 21, 6, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[70, 6, 0, 4095, 1, 0, 9, 4071, 4057, 256, 2, 0, 0, 0, 12, 4095, 4047, 530, 0, 4095, 0, 4091, 6, 0, 3547, 4035, 0, 1, 4096, 3, 4057, 0, 0, 33, 4077, 0, 1, 2, 64, 44, 4047, 0, 4090, 4086, 26, 0, 1, 0, 1, 0, 11, 4093, 0, 0, 3982, 4095, 4093, 0, 1396, 0, 0, 2226, 12, 0, 0, 0, 0, 0, 4091, 4094, 0, 4095, 4086, 22, 3, 0, 26, 0, 2017, 1, 36, 89, 0, 0, 2, 13, 0, 47, 0, 85, 3, 1164, 8, 0, 29, 0, 0, 0, 273, 0, 0, 0, 0, 4096, 0, 0, 811, 3973, 6, 0, 0, 0, 4073, 1, 0, 2, 4065, 3994, 4042, 3, 10, 4091, 0, 159, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 1, 4036, 0, 0, 4, 4096, 4094, 3369, 0, 0, 0, 0, 205, 4094, 4094, 1176, 0, 4096, 6, 4096, 10, 0, 49, 16, 0, 4, 4096, 0, 3587, 0, 0, 0, 4056, 0, 52, 1, 2, 1, 4096, 0, 4096, 4076, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4092, 4084, 4096, 0, 4078, 36, 14, 4074, 12, 0, 0, 1, 0, 0, 4093, 4071, 0, 4096, 4094, 0, 0, 0, 0, 0, 2591, 0, 0, 0, 1, 0, 0, 0, 0, 16, 10, 0, 2, 0, 0, 3, 0, 0, 0, 0, 27, 0, 0, 2, 0, 4094, 10, 0, 1855, 4086, 0, 0, 0, 0, 4085, 0, 0, 0, 4095, 1337, 4088, 1, 0, 4067, 1, 2253, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 224, 4096, 0, 5, 0, 4096, 4095, 1811, 0, 0, 0, 0, 145, 4048, 4095, 3990, 0, 4096, 0, 4096, 1, 0, 54, 241, 0, 0, 4096, 0, 4073, 0, 1, 553, 4096, 0, 102, 0, 0, 0, 4093, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4095, 4096, 4096, 0, 2334, 7, 39, 4096, 2, 0, 53, 0, 0, 0, 4096, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 3852, 0, 0, 3730, 4096, 0, 100, 65, 0, 4096, 0, 0, 0, 4096, 1083, 4096, 0, 0, 4028, 0, 1963, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 13, 4096, 0, 1, 0, 2084, 2107, 4035, 0, 0, 1, 0, 362, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2108, 0, 0, 0, 4096, 1, 0, 41, 6, 0, 0, 1166, 0, 3623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 9, 0, 0, 0, 4085, 0, 4096, 3844, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4069, 1402, 0, 0, 3796, 1, 120, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 4096, 4096, 0, 1612, 4076, 0, 0, 0, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3437, 2930, 1619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 0, 0, 0, 4086, 0, 0, 2946, 4096, 0, 22, 6, 0, 2484, 2012, 1075, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 658, 1369, 4096, 7, 0, 4096, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16790, 15135, 16132, 17450, 17861, 16188, 16996, 14520] avg=16384.0000 max=17861 max_rank=4 min=14520 min_rank=7 imbalance_ratio=1.090149 max_load_reduction=113211 max_load_reduction_pct=0.863731 moved_to_echo=65565 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 7, 2083, 4093, 1129, 0, 0, 0, 0, 79, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3952, 0, 0, 0, 4096, 0, 65, 30, 18, 0, 0, 1166, 0, 1831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 11, 0, 0, 0, 4096, 0, 4096, 3842, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4090, 1402, 0, 0, 4089, 0, 273, 3944, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1696, 0, 0, 0, 4094, 4095, 0, 1612, 4096, 0, 0, 0, 0, 1, 767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 109, 0, 0, 1, 0, 8, 0, 0, 4096, 4096, 3437, 2930, 1619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 3, 0, 0, 0, 4096, 9, 0, 2680, 4096, 0, 0, 7, 0, 2483, 2011, 1075, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 659, 181, 4096, 0, 0, 4095, 0, 3659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15583, 15235, 16136, 17916, 16361, 16299, 16756, 16786] avg=16384.0000 max=17916 max_rank=3 min=15235 min_rank=1 imbalance_ratio=1.093506 max_load_reduction=113156 max_load_reduction_pct=0.863312 moved_to_echo=66202 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4090, 0, 0, 6, 2084, 4083, 1161, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 335, 0, 0, 16, 4092, 0, 7, 191, 3226, 0, 0, 1166, 0, 2271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 241, 0, 0, 0, 4076, 11, 4094, 3837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 1400, 0, 0, 3973, 50, 0, 1711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1328, 0, 0, 0, 4096, 4096, 0, 1612, 4056, 11, 0, 0, 0, 3, 2109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 0, 9, 0, 3, 0, 7, 0, 0, 0, 0, 4093, 4091, 3435, 2930, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 4096, 14, 0, 3888, 4096, 0, 0, 39, 3, 2484, 2011, 1075, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 658, 3, 4096, 16, 0, 4096, 6, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15520, 15385, 16355, 15328, 17311, 16202, 17961, 17010] avg=16384.0000 max=17961 max_rank=6 min=15328 min_rank=3 imbalance_ratio=1.096252 max_load_reduction=113111 max_load_reduction_pct=0.862968 moved_to_echo=68484 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[5, 11, 473, 4078, 0, 0, 0, 2070, 3909, 42, 80, 0, 9, 2, 77, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 2782, 0, 0, 45, 3995, 12, 0, 1513, 356, 0, 1, 1143, 13, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 4051, 0, 127, 0, 17, 0, 4062, 0, 4040, 3838, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 11, 0, 3986, 53, 2, 4017, 1373, 0, 42, 3096, 0, 83, 2941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 9, 29, 0, 4081, 4089, 12, 1603, 4058, 0, 0, 2, 40, 12, 3712, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 1, 25, 0, 5, 3, 0, 0, 0, 0, 48, 6, 0, 0, 4, 4056, 4064, 3395, 2878, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 34, 31, 0, 70, 3, 0, 4006, 0, 0, 2477, 4080, 19, 0, 0, 27, 2482, 2003, 1064, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 50, 0, 36, 645, 1731, 4049, 27, 48, 4080, 5, 4005, 0, 21, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14833, 14752, 16232, 15627, 18073, 16100, 16714, 18741] avg=16384.0000 max=18741 max_rank=7 min=14752 min_rank=1 imbalance_ratio=1.143860 max_load_reduction=112331 max_load_reduction_pct=0.857018 moved_to_echo=69628 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 3, 2082, 2788, 2329, 0, 0, 29, 7, 3847, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 3643, 0, 0, 0, 4092, 11, 0, 448, 108, 7, 1, 1166, 0, 3756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 4095, 0, 1093, 0, 0, 0, 3850, 0, 4092, 3839, 21, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4070, 0, 0, 4095, 1400, 0, 0, 4070, 17, 11, 4094, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4087, 4094, 0, 1611, 4096, 0, 0, 0, 2, 0, 1054, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 1, 2, 0, 0, 0, 0, 0, 1, 0, 4, 0, 2, 0, 0, 4093, 4085, 3428, 2928, 1616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 1, 4096, 0, 0, 56, 4085, 4, 2, 22, 5, 2482, 2010, 1074, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4, 0, 657, 1508, 4092, 0, 0, 4077, 0, 136, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19276, 17162, 17081, 17766, 14949, 16174, 14094, 14570] avg=16384.0000 max=19276 max_rank=0 min=14094 min_rank=6 imbalance_ratio=1.176514 max_load_reduction=111796 max_load_reduction_pct=0.852936 moved_to_echo=59787 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 4036, 0, 0, 4, 2084, 4094, 3369, 0, 0, 0, 0, 205, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1176, 0, 0, 6, 4096, 10, 0, 49, 16, 0, 4, 1166, 0, 3587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 52, 1, 2, 1, 4096, 0, 4096, 3826, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 1398, 0, 0, 4078, 36, 14, 4074, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 4071, 0, 1612, 4094, 0, 0, 0, 0, 0, 2591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 16, 10, 0, 2, 0, 0, 3, 0, 0, 4096, 4096, 3437, 2930, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 2, 0, 4094, 10, 0, 1855, 4086, 0, 0, 0, 0, 2484, 2012, 1072, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 658, 1337, 4088, 1, 0, 4067, 1, 2253, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17887, 14204, 16131, 17800, 16462, 16205, 15892, 16491] avg=16384.0000 max=17887 max_rank=0 min=14204 min_rank=1 imbalance_ratio=1.091736 max_load_reduction=113185 max_load_reduction_pct=0.863533 moved_to_echo=65050 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[70, 6, 0, 4095, 1, 0, 9, 2072, 4057, 256, 2, 0, 0, 0, 12, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 530, 0, 0, 0, 4091, 6, 0, 3547, 4035, 0, 1, 1166, 3, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4077, 0, 1, 2, 64, 44, 4047, 0, 4090, 3835, 26, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 4093, 0, 0, 3982, 1402, 0, 0, 1396, 0, 0, 2226, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4094, 0, 1612, 4086, 22, 3, 0, 26, 0, 2017, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 89, 0, 0, 2, 13, 0, 47, 0, 85, 3, 1164, 8, 0, 29, 0, 4093, 4095, 3411, 2930, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 0, 0, 0, 4096, 0, 0, 811, 3973, 6, 0, 0, 0, 2483, 1999, 1075, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 1, 0, 2, 654, 3994, 4042, 3, 10, 4091, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14675, 21483, 16220, 13123, 15952, 17623, 14967, 17029] avg=16384.0000 max=21483 max_rank=1 min=13123 min_rank=3 imbalance_ratio=1.311218 max_load_reduction=109589 max_load_reduction_pct=0.836098 moved_to_echo=65571 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 224, 4096, 0, 5, 0, 2084, 4095, 1811, 0, 0, 0, 0, 145, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3990, 0, 0, 0, 4096, 1, 0, 54, 241, 0, 0, 1166, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 553, 4096, 0, 102, 0, 0, 0, 4093, 0, 4096, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4095, 1402, 0, 0, 2334, 7, 39, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 4096, 4096, 0, 1612, 4095, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 4096, 4096, 3437, 2930, 1619, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 3852, 0, 0, 3730, 4096, 0, 100, 65, 0, 2484, 2012, 1075, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 659, 1083, 4096, 0, 0, 4028, 0, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16508, 17716, 16785, 16062, 14103, 16281, 17692, 15925] avg=16384.0000 max=17716 max_rank=1 min=14103 min_rank=4 imbalance_ratio=1.081299 max_load_reduction=113356 max_load_reduction_pct=0.864838 moved_to_echo=64001 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32512, 18560, 0, 0, 128, 32768, 128, 128, 5888, 8064, 128, 128, 9344, 128, 24064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 32640, 128, 128, 32640, 11264, 0, 128, 26880, 128, 640, 27136, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 768, 32768, 128, 128, 128, 16768, 29312, 14208, 128, 0, 128, 128, 4736, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 128, 5248, 11264, 32768, 128, 128, 32640, 128, 16512, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 640, 32768, 128, 1664, 128, 128, 128, 32512, 128, 32768, 30720, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[3584, 128, 128, 128, 32768, 32768, 128, 12928, 32768, 128, 128, 128, 128, 128, 16384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[256, 128, 512, 128, 128, 128, 128, 32512, 128, 0, 18560, 32640, 128, 128, 256, 128, 19968, 16128, 8704, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 256, 128, 128, 1408, 128, 128, 128, 128, 32768, 32768, 27520, 23424, 12928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 148, 4061, 0, 0, 0, 0, 4085, 0, 291, 0, 31, 4063, 2971, 4096, 1, 4093, 2, 4096, 4096, 144, 0, 360, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4095, 0, 231, 1, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 2, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 3343, 0, 4096, 4084, 0, 0, 92, 86, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 0, 4062, 4096, 0, 0, 0, 0, 4096, 0, 3793, 0, 0, 4096, 3806, 1, 4096, 4096, 0, 0, 0, 0, 0, 9, 1187, 0, 4096, 0, 0, 4056, 4095, 0, 0, 15, 0, 0, 0, 4088, 4094, 0, 0, 14, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[8, 1004, 2296, 0, 0, 0, 3, 4023, 0, 0, 0, 0, 4034, 1246, 4089, 0, 4093, 0, 4090, 4094, 1, 0, 188, 8, 0, 2, 3, 0, 3, 0, 11, 4096, 8, 0, 0, 4096, 0, 3597, 0, 0, 0, 13, 4, 0, 0, 2, 0, 0, 147, 0, 3936, 291, 4058, 0, 0, 4095, 3, 2, 0, 44, 0, 0, 4096, 4, 0, 0, 4095, 3, 4091, 4084, 1, 0, 1140, 0, 4091, 0, 23, 1, 0, 10, 0, 0, 28, 0, 6, 4087, 0, 4093, 4094, 5, 5, 0, 0, 4094, 1, 4090, 1, 0, 4095, 4093, 0, 4095, 4090, 60, 0, 0, 0, 0, 68, 0, 10, 4095, 0, 0, 4059, 4052, 0, 4, 5, 0, 0, 0, 2503, 4073, 1, 0, 38, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 51, 173, 0, 1, 0, 0, 4084, 1, 3, 0, 4, 4096, 3319, 4096, 0, 4086, 0, 4096, 4096, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4090, 0, 656, 0, 0, 0, 3179, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 1, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4093, 0, 0, 907, 0, 4096, 0, 1, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4064, 0, 4092, 0, 0, 4096, 4080, 0, 4096, 4093, 1171, 35, 11, 0, 0, 299, 19, 0, 4096, 0, 0, 4096, 4033, 0, 0, 0, 0, 0, 0, 2741, 4089, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 37, 1282, 20, 2, 0, 29, 4052, 1, 185, 0, 0, 4026, 4069, 4051, 0, 4077, 64, 4039, 4061, 0, 5, 143, 96, 0, 0, 0, 0, 21, 2, 0, 4072, 33, 59, 13, 4048, 0, 1576, 0, 12, 138, 330, 0, 0, 18, 39, 54, 0, 0, 7, 4076, 22, 4078, 28, 17, 4084, 0, 10, 1, 158, 0, 0, 4078, 0, 0, 0, 4095, 0, 3918, 4023, 17, 0, 933, 86, 4072, 14, 0, 0, 0, 2, 6, 0, 2, 49, 8, 4039, 0, 4082, 3846, 0, 0, 17, 8, 3997, 0, 3886, 25, 10, 4079, 4043, 0, 4084, 4043, 903, 13, 0, 21, 3, 43, 404, 3, 4036, 10, 12, 3850, 3907, 1, 0, 0, 24, 0, 18, 2863, 3996, 0, 165, 201, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 4078, 0, 0, 0, 0, 3946, 1, 101, 0, 431, 4096, 3903, 4096, 15, 4087, 1, 4093, 4096, 0, 0, 1, 20, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 566, 0, 0, 0, 2, 0, 0, 0, 0, 4076, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 4085, 0, 4095, 3301, 1, 0, 47, 3, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 1, 4092, 4096, 11, 0, 0, 0, 4006, 0, 4090, 0, 0, 4096, 4087, 0, 4095, 4096, 0, 0, 2, 0, 0, 0, 103, 0, 4088, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4080, 4096, 0, 18, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 1, 3963, 0, 0, 0, 0, 4087, 0, 4, 3, 0, 4088, 2318, 4096, 0, 4081, 49, 4096, 4096, 40, 0, 295, 1, 108, 0, 0, 0, 0, 0, 0, 3580, 0, 0, 10, 4094, 8, 3330, 7, 2, 18, 54, 1, 0, 0, 2, 0, 0, 0, 1, 4029, 0, 4096, 0, 0, 4094, 0, 0, 0, 8, 0, 11, 4091, 0, 0, 0, 4096, 2, 4095, 3760, 0, 0, 1707, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4061, 4092, 0, 0, 0, 0, 4093, 0, 3514, 0, 0, 4091, 4057, 0, 4094, 4069, 347, 2, 27, 0, 0, 0, 0, 0, 4091, 1, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 1621, 4096, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 2925, 199, 0, 10, 0, 0, 4095, 1, 124, 0, 0, 4052, 1666, 4096, 0, 4082, 48, 4090, 4090, 2, 0, 2210, 0, 65, 0, 5, 0, 1, 22, 844, 4079, 6, 0, 0, 3815, 0, 1169, 0, 1, 0, 731, 0, 4, 0, 0, 0, 0, 1, 0, 3845, 0, 4072, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 3971, 4, 36, 0, 4058, 0, 4094, 4095, 0, 0, 49, 0, 4094, 0, 1, 0, 2, 28, 0, 0, 0, 0, 128, 4092, 0, 4095, 4087, 3, 2, 0, 1, 4092, 0, 4085, 3, 0, 4041, 4089, 0, 4091, 4088, 358, 0, 0, 4, 0, 0, 0, 316, 4083, 1, 0, 4096, 4096, 1, 3, 0, 0, 0, 2, 2422, 3915, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 339, 4025, 0, 1, 0, 0, 4096, 5, 22, 0, 0, 4096, 2682, 4096, 0, 4095, 6, 4096, 4096, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 22, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 3890, 3, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 3968, 0, 0, 1784, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4080, 0, 4096, 4086, 383, 0, 1, 0, 0, 69, 21, 14, 4096, 1, 0, 4094, 4037, 0, 0, 0, 0, 0, 0, 3201, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 51, 173, 0, 1, 0, 0, 4084, 1, 3, 0, 4, 4096, 3319, 2254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 3584, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 656, 0, 0, 0, 3179, 0, 0, 0, 0, 0, 0, 4096, 3917, 1842, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 1, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 4096, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4093, 0, 0, 907, 0, 3518, 0, 1, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 4095, 4096, 0, 0, 0, 0, 4064, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3440, 4080, 0, 0, 4093, 1171, 35, 11, 0, 0, 299, 19, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4033, 0, 0, 0, 0, 0, 0, 2741, 4089, 0, 0, 0, 0, 578, 456, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13986, 15898, 18256, 16267, 16722, 16526, 17244, 16173] avg=16384.0000 max=18256 max_rank=2 min=13986 min_rank=0 imbalance_ratio=1.114258 max_load_reduction=112816 max_load_reduction_pct=0.860718 moved_to_echo=66665 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 37, 1282, 20, 2, 0, 29, 4052, 1, 185, 0, 0, 4026, 4069, 2224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 64, 4039, 3544, 0, 5, 143, 96, 0, 0, 0, 0, 21, 2, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 59, 13, 4048, 0, 1576, 0, 12, 138, 330, 0, 0, 18, 39, 54, 0, 4084, 3866, 1827, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4076, 22, 4078, 28, 17, 4084, 0, 10, 1, 158, 0, 0, 4078, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3918, 4023, 17, 0, 933, 86, 3491, 14, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 49, 8, 173, 0, 4082, 3846, 0, 0, 17, 8, 3997, 0, 3886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 10, 3419, 4043, 0, 0, 4043, 903, 13, 0, 21, 3, 43, 404, 3, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 12, 3850, 3907, 1, 0, 0, 24, 0, 18, 2863, 3996, 0, 165, 201, 2, 581, 454, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15927, 16063, 16575, 16622, 16579, 16074, 16966, 16266] avg=16384.0000 max=16966 max_rank=6 min=15927 min_rank=0 imbalance_ratio=1.035522 max_load_reduction=114106 max_load_reduction_pct=0.870560 moved_to_echo=65885 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 148, 4061, 0, 0, 0, 0, 4085, 0, 291, 0, 31, 4063, 2971, 2254, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 4096, 3584, 144, 0, 360, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 231, 1, 0, 0, 123, 0, 0, 0, 0, 0, 0, 4096, 3916, 1842, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 2, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4077, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3343, 0, 4096, 4084, 0, 0, 92, 86, 3518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 179, 0, 4062, 4096, 0, 0, 0, 0, 4096, 0, 3793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3440, 3806, 1, 0, 4096, 0, 0, 0, 0, 0, 9, 1187, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 4095, 0, 0, 15, 0, 0, 0, 4088, 4094, 0, 0, 14, 0, 578, 456, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17905, 16371, 14780, 16358, 15219, 16228, 16635, 17576] avg=16384.0000 max=17905 max_rank=0 min=14780 min_rank=2 imbalance_ratio=1.092834 max_load_reduction=113167 max_load_reduction_pct=0.863396 moved_to_echo=65658 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4078, 0, 0, 0, 0, 3946, 1, 101, 0, 431, 4096, 3903, 2254, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 4093, 3584, 0, 0, 1, 20, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 566, 0, 0, 0, 2, 0, 0, 4095, 3917, 1842, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 4096, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4095, 3301, 1, 0, 47, 3, 3518, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 1, 4092, 4096, 11, 0, 0, 0, 4006, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3440, 4087, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 103, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4080, 4096, 0, 18, 0, 0, 578, 456, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18826, 15882, 14994, 16422, 15059, 16475, 15816, 17598] avg=16384.0000 max=18826 max_rank=0 min=14994 min_rank=2 imbalance_ratio=1.149048 max_load_reduction=112246 max_load_reduction_pct=0.856369 moved_to_echo=64948 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[8, 1004, 2296, 0, 0, 0, 3, 4023, 0, 0, 0, 0, 4034, 1246, 2250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4090, 3583, 1, 0, 188, 8, 0, 2, 3, 0, 3, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 3597, 0, 0, 0, 13, 4, 0, 0, 2, 0, 0, 4095, 3908, 1839, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 3936, 291, 4058, 0, 0, 4095, 3, 2, 0, 44, 0, 0, 4096, 4, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 4091, 4084, 1, 0, 1140, 0, 3513, 0, 23, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 6, 179, 0, 4093, 4094, 5, 5, 0, 0, 4094, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3439, 4093, 0, 0, 4090, 60, 0, 0, 0, 0, 68, 0, 10, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4052, 0, 4, 5, 0, 0, 0, 2503, 4073, 1, 0, 38, 0, 578, 455, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14864, 16078, 18038, 16732, 16961, 16595, 15856, 15948] avg=16384.0000 max=18038 max_rank=2 min=14864 min_rank=0 imbalance_ratio=1.100952 max_load_reduction=113034 max_load_reduction_pct=0.862381 moved_to_echo=65360 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3963, 0, 0, 0, 0, 4087, 0, 4, 3, 0, 4088, 2318, 2254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 49, 4096, 3584, 40, 0, 295, 1, 108, 0, 0, 0, 0, 0, 0, 3580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4094, 8, 3330, 7, 2, 18, 54, 1, 0, 0, 2, 0, 0, 4094, 3917, 1842, 475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4029, 0, 4096, 0, 0, 4094, 0, 0, 0, 8, 0, 11, 4091, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4095, 3760, 0, 0, 1707, 11, 3518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 4061, 4092, 0, 0, 0, 0, 4093, 0, 3514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3436, 4057, 0, 0, 4069, 347, 2, 27, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 1621, 4096, 1, 0, 0, 0, 578, 456, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16718, 15834, 17854, 16386, 17189, 15939, 16029, 15123] avg=16384.0000 max=17854 max_rank=2 min=15123 min_rank=7 imbalance_ratio=1.089722 max_load_reduction=113218 max_load_reduction_pct=0.863785 moved_to_echo=64280 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2925, 199, 0, 10, 0, 0, 4095, 1, 124, 0, 0, 4052, 1666, 2254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 48, 4090, 3579, 2, 0, 2210, 0, 65, 0, 5, 0, 1, 22, 844, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3815, 0, 1169, 0, 1, 0, 731, 0, 4, 0, 0, 0, 0, 4091, 3913, 1842, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3845, 0, 4072, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 3971, 4, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 4058, 0, 4094, 4095, 0, 0, 49, 0, 3516, 0, 1, 0, 2, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 179, 0, 4095, 4087, 3, 2, 0, 1, 4092, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3395, 4089, 0, 0, 4088, 358, 0, 0, 4, 0, 0, 0, 316, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 1, 3, 0, 0, 0, 2, 2422, 3915, 0, 0, 0, 0, 578, 455, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15326, 19027, 16041, 16045, 15879, 16672, 16336, 15746] avg=16384.0000 max=19027 max_rank=1 min=15326 min_rank=0 imbalance_ratio=1.161316 max_load_reduction=112045 max_load_reduction_pct=0.854836 moved_to_echo=64633 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 339, 4025, 0, 1, 0, 0, 4096, 5, 22, 0, 0, 4096, 2682, 2254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 4096, 3584, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 4096, 3917, 1842, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3890, 3, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 4096, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3968, 0, 0, 1784, 0, 3518, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3440, 4080, 0, 0, 4086, 383, 0, 1, 0, 0, 69, 21, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 4037, 0, 0, 0, 0, 0, 0, 3201, 4095, 0, 0, 0, 0, 578, 456, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17520, 15919, 14534, 16240, 17464, 16563, 16190, 16642] avg=16384.0000 max=17520 max_rank=0 min=14534 min_rank=2 imbalance_ratio=1.069336 max_load_reduction=113552 max_load_reduction_pct=0.866333 moved_to_echo=66859 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 256, 32768, 28672, 256, 128, 3328, 128, 256, 128, 128, 0, 128, 128, 896, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 27520, 32384, 128, 0, 32768, 3328, 128, 128, 128, 128, 512, 1792, 384, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 256, 1536, 128, 32768, 32512, 128, 128, 128, 128, 32640, 128, 31744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 4608, 20096, 128, 128, 0, 128, 32512, 128, 768, 128, 512, 32640, 22272, 18048, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 32512, 128, 10624, 128, 128, 256, 5120, 128, 128, 128, 128, 128, 0, 32768, 31360, 14720, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 32512, 32512, 128, 128, 128, 128, 0, 128, 23552, 32512, 128, 256, 256, 128, 4736, 3712, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[256, 128, 31872, 384, 32768, 128, 128, 32768, 128, 128, 128, 256, 128, 128, 32640, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 32000, 128, 32640, 31488, 128, 0, 6784, 256, 28160, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[2567, 0, 3111, 0, 4096, 0, 0, 4, 0, 289, 0, 1, 3531, 62, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4091, 4096, 0, 4077, 0, 4077, 0, 0, 0, 0, 3831, 0, 0, 0, 0, 0, 0, 830, 0, 4096, 4095, 9, 0, 0, 0, 549, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1660, 0, 0, 107, 4096, 2, 4069, 0, 4077, 0, 20, 4094, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 457, 0, 0, 0, 4096, 0, 0, 0, 4083, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 9, 0, 0, 4077, 68, 0, 0, 0, 0, 1, 4096, 3491, 4096, 4095, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[3817, 0, 2669, 0, 4090, 14, 0, 516, 0, 550, 0, 3, 1556, 21, 0, 0, 0, 0, 2, 4086, 0, 4087, 9, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 3476, 4020, 4096, 0, 4090, 0, 3810, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2230, 0, 4096, 4023, 41, 0, 0, 6, 3892, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 874, 0, 0, 35, 4095, 0, 4092, 0, 3551, 0, 194, 4085, 0, 0, 0, 3, 69, 0, 4095, 0, 0, 0, 1191, 0, 0, 0, 4096, 0, 0, 0, 4091, 3, 0, 0, 0, 41, 4067, 0, 0, 115, 121, 0, 0, 4067, 2, 1, 5, 0, 0, 0, 4096, 109, 4096, 4096, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[4039, 0, 4069, 0, 4096, 23, 1, 1, 2, 127, 0, 0, 3607, 0, 0, 0, 4, 1, 0, 3789, 4, 4095, 2, 0, 4095, 3, 4, 0, 0, 0, 0, 4086, 4096, 4085, 4092, 0, 4017, 1, 4096, 87, 36, 8, 2, 4094, 7, 0, 0, 0, 0, 1, 3919, 0, 4083, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 4092, 0, 0, 0, 0, 2, 1270, 4056, 156, 4096, 0, 3958, 0, 206, 4084, 5, 0, 0, 0, 0, 303, 4085, 0, 0, 0, 4, 1, 0, 0, 4090, 0, 1, 2, 4027, 1, 0, 0, 3, 240, 4084, 10, 0, 32, 3, 0, 0, 2671, 0, 0, 0, 0, 0, 28, 4096, 348, 4038, 4095, 13, 0, 1, 0, 4077, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1396, 17, 3966, 0, 3946, 143, 8, 0, 24, 0, 29, 801, 468, 0, 0, 5, 0, 20, 3, 4068, 7, 3886, 0, 61, 4006, 0, 6, 14, 0, 0, 0, 3923, 4088, 3794, 4028, 0, 4092, 19, 4074, 0, 32, 20, 22, 4052, 29, 25, 12, 48, 113, 0, 3741, 0, 4065, 4030, 0, 11, 40, 0, 0, 46, 0, 0, 0, 0, 0, 4083, 7, 0, 1929, 0, 12, 709, 4065, 53, 4006, 34, 2205, 0, 2243, 2725, 0, 0, 144, 596, 0, 1, 3941, 50, 4, 2, 1497, 0, 0, 79, 4037, 3, 37, 32, 3037, 0, 0, 0, 2, 76, 4083, 0, 12, 0, 11, 0, 0, 3731, 191, 0, 0, 0, 12, 38, 4054, 3825, 4081, 4040, 1, 7, 55, 0, 4077, 4067] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[782, 0, 4096, 0, 4096, 19, 0, 1, 3, 11, 0, 12, 429, 0, 0, 1, 3, 0, 0, 4065, 0, 4029, 0, 0, 4095, 0, 11, 0, 0, 19, 2, 4091, 4078, 4095, 4096, 0, 4096, 0, 4093, 0, 1, 0, 0, 4096, 2, 0, 0, 0, 29, 0, 2530, 4, 4096, 4093, 269, 0, 0, 0, 0, 4, 2, 0, 2, 0, 0, 4086, 2, 0, 21, 0, 0, 3408, 4093, 13, 4085, 0, 3925, 0, 1403, 3217, 3, 1, 2, 35, 0, 0, 4089, 1, 5, 1, 140, 0, 0, 0, 4075, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 661, 0, 0, 4096, 0, 32, 3, 0, 0, 0, 4094, 3715, 4092, 4093, 7, 0, 0, 0, 4074, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[3124, 0, 4089, 0, 4085, 1, 0, 79, 0, 135, 0, 72, 1558, 12, 0, 0, 0, 0, 6, 4096, 0, 4096, 5, 10, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 4090, 5, 4092, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 3507, 0, 4085, 4078, 5, 0, 0, 0, 2110, 0, 0, 0, 0, 6, 0, 4090, 0, 2, 696, 0, 0, 95, 4094, 4, 4056, 0, 3712, 0, 324, 4093, 0, 39, 0, 94, 43, 0, 4093, 0, 0, 0, 460, 0, 0, 0, 4074, 0, 0, 0, 4095, 0, 2, 0, 0, 7, 4068, 0, 0, 0, 2, 36, 1, 3935, 0, 5, 0, 0, 0, 0, 4086, 391, 4096, 4074, 0, 0, 0, 198, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1200, 3, 3599, 0, 4095, 12, 0, 390, 0, 1347, 0, 101, 308, 1905, 0, 0, 0, 0, 0, 4090, 0, 4078, 7, 89, 4095, 0, 0, 0, 0, 73, 0, 4096, 4094, 4019, 4096, 0, 4096, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 2556, 0, 4096, 4096, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 35, 1, 0, 1834, 4096, 0, 4089, 0, 4001, 0, 1725, 4024, 0, 1, 0, 194, 41, 21, 4096, 0, 0, 0, 77, 0, 0, 0, 4096, 0, 0, 0, 4043, 0, 0, 0, 0, 123, 4059, 0, 0, 1, 0, 0, 0, 3582, 0, 0, 0, 0, 0, 85, 4096, 1493, 4093, 4087, 0, 0, 2, 40, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[3123, 0, 4082, 0, 4096, 0, 0, 1, 0, 18, 0, 1, 22, 282, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 4096, 0, 4090, 0, 4065, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3809, 0, 4096, 4095, 9, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 3675, 4096, 1, 4094, 0, 2576, 0, 1, 3769, 0, 0, 0, 4, 2, 0, 4096, 0, 0, 0, 1068, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 69, 0, 0, 4093, 0, 0, 0, 0, 0, 3, 4096, 2107, 4077, 4095, 0, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2567, 0, 3111, 0, 4096, 0, 0, 4, 0, 289, 0, 1, 3531, 62, 0, 0, 1971, 1812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 181, 0, 4077, 0, 0, 0, 0, 3831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 830, 0, 4096, 4095, 9, 0, 0, 0, 549, 0, 0, 0, 0, 0, 3649, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1660, 0, 0, 107, 2209, 2, 4069, 0, 4077, 0, 20, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 457, 0, 0, 0, 4096, 0, 4096, 3309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 9, 0, 0, 4077, 3896, 447, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 1, 2098, 3491, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17444, 16459, 16276, 14015, 16238, 16056, 16639, 17945] avg=16384.0000 max=17945 max_rank=7 min=14015 min_rank=3 imbalance_ratio=1.095276 max_load_reduction=113127 max_load_reduction_pct=0.863091 moved_to_echo=66878 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3817, 0, 2669, 0, 4090, 14, 0, 516, 0, 550, 0, 3, 1556, 21, 0, 0, 1971, 1812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4086, 0, 4087, 9, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3476, 4020, 0, 0, 182, 0, 3810, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2230, 0, 4096, 4023, 41, 0, 0, 6, 3892, 0, 0, 0, 0, 0, 3649, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 0, 0, 35, 2208, 0, 4092, 0, 3551, 0, 194, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 69, 0, 4095, 0, 0, 0, 1191, 0, 0, 0, 4096, 0, 4096, 3309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 41, 4067, 0, 0, 115, 121, 0, 0, 4067, 3908, 447, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 5, 0, 0, 0, 2098, 109, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17019, 16453, 15588, 18724, 15039, 16859, 16887, 14503] avg=16384.0000 max=18724 max_rank=3 min=14503 min_rank=7 imbalance_ratio=1.142822 max_load_reduction=112348 max_load_reduction_pct=0.857147 moved_to_echo=63288 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1396, 17, 3966, 0, 3946, 143, 8, 0, 24, 0, 29, 801, 468, 0, 0, 5, 1957, 1803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 3, 4068, 7, 3886, 0, 61, 4006, 0, 6, 14, 0, 0, 0, 3923, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3794, 0, 0, 179, 19, 4074, 0, 32, 20, 22, 4052, 29, 25, 12, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 3741, 0, 4065, 4030, 0, 11, 40, 0, 0, 46, 0, 0, 0, 0, 3589, 786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1929, 0, 12, 709, 2185, 53, 4006, 34, 2205, 0, 2243, 2725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 596, 0, 1, 3941, 50, 4, 2, 1497, 0, 0, 79, 4037, 3, 4067, 3297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 32, 3037, 0, 0, 0, 2, 76, 4083, 0, 12, 0, 11, 0, 0, 3731, 3913, 439, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 0, 0, 12, 38, 2064, 3825, 4081, 4040, 1, 7, 55, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14563, 16071, 16394, 16421, 16108, 17718, 15406, 18391] avg=16384.0000 max=18391 max_rank=7 min=14563 min_rank=0 imbalance_ratio=1.122498 max_load_reduction=112681 max_load_reduction_pct=0.859688 moved_to_echo=67623 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3124, 0, 4089, 0, 4085, 1, 0, 79, 0, 135, 0, 72, 1558, 12, 0, 0, 1967, 1811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 4096, 5, 10, 4096, 0, 0, 0, 0, 0, 0, 4096, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 182, 5, 4092, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3507, 0, 4085, 4078, 5, 0, 0, 0, 2110, 0, 0, 0, 0, 6, 3650, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 696, 0, 0, 95, 2208, 4, 4056, 0, 3712, 0, 324, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 94, 43, 0, 4093, 0, 0, 0, 460, 0, 0, 0, 4074, 0, 4096, 3303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 7, 4068, 0, 0, 0, 2, 36, 1, 3935, 3908, 446, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 2092, 391, 4096, 4074, 0, 0, 0, 198, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16933, 16480, 16560, 18228, 15190, 16202, 16527, 14952] avg=16384.0000 max=18228 max_rank=3 min=14952 min_rank=7 imbalance_ratio=1.112549 max_load_reduction=112844 max_load_reduction_pct=0.860931 moved_to_echo=62871 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1200, 3, 3599, 0, 4095, 12, 0, 390, 0, 1347, 0, 101, 308, 1905, 0, 0, 1971, 1812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4078, 7, 89, 4095, 0, 0, 0, 0, 73, 0, 4096, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4019, 0, 0, 182, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2556, 0, 4096, 4096, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 3650, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 35, 1, 0, 1834, 2209, 0, 4089, 0, 4001, 0, 1725, 4024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 194, 41, 21, 4096, 0, 0, 0, 77, 0, 0, 0, 4096, 0, 4096, 3308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 123, 4059, 0, 0, 1, 0, 0, 0, 3582, 3914, 446, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 2098, 1493, 4093, 4087, 0, 0, 2, 40, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16743, 16603, 16482, 15205, 17920, 15930, 16195, 15994] avg=16384.0000 max=17920 max_rank=4 min=15205 min_rank=3 imbalance_ratio=1.093750 max_load_reduction=113152 max_load_reduction_pct=0.863281 moved_to_echo=66039 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[782, 0, 4096, 0, 4096, 19, 0, 1, 3, 11, 0, 12, 429, 0, 0, 1, 1970, 1811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4065, 0, 4029, 0, 0, 4095, 0, 11, 0, 0, 19, 2, 4091, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4095, 0, 0, 182, 0, 4093, 0, 1, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 2530, 4, 4096, 4093, 269, 0, 0, 0, 0, 4, 2, 0, 2, 0, 3649, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 21, 0, 0, 3408, 2207, 13, 4085, 0, 3925, 0, 1403, 3217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 2, 35, 0, 0, 4089, 1, 5, 1, 140, 0, 0, 0, 4075, 0, 4095, 3301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 661, 0, 0, 4096, 3914, 447, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 3, 0, 0, 0, 2097, 3715, 4092, 4093, 7, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13231, 16390, 16547, 15463, 18281, 15748, 17299, 18113] avg=16384.0000 max=18281 max_rank=4 min=13231 min_rank=0 imbalance_ratio=1.115784 max_load_reduction=112791 max_load_reduction_pct=0.860527 moved_to_echo=69441 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4039, 0, 4069, 0, 4096, 23, 1, 1, 2, 127, 0, 0, 3607, 0, 0, 0, 1971, 1795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 3789, 4, 4095, 2, 0, 4095, 3, 4, 0, 0, 0, 0, 4086, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 179, 1, 4096, 87, 36, 8, 2, 4094, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3919, 0, 4083, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3646, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 2, 1270, 2187, 156, 4096, 0, 3958, 0, 206, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 303, 4085, 0, 0, 0, 4, 1, 0, 0, 4090, 0, 4096, 3305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4027, 1, 0, 0, 3, 240, 4084, 10, 0, 32, 3, 0, 0, 2671, 3838, 446, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 2098, 348, 4038, 4095, 13, 0, 1, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19731, 16157, 16691, 16556, 15965, 15889, 15385, 14698] avg=16384.0000 max=19731 max_rank=0 min=14698 min_rank=7 imbalance_ratio=1.204285 max_load_reduction=111341 max_load_reduction_pct=0.849464 moved_to_echo=61937 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3123, 0, 4082, 0, 4096, 0, 0, 1, 0, 18, 0, 1, 22, 282, 0, 0, 1971, 1812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 182, 0, 4065, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3809, 0, 4096, 4095, 9, 0, 0, 0, 14, 0, 0, 0, 0, 0, 3649, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3675, 2209, 1, 4094, 0, 2576, 0, 1, 3769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 4096, 0, 0, 0, 1068, 0, 0, 0, 4096, 0, 4096, 3308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 69, 0, 0, 4093, 3908, 447, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2098, 2107, 4077, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15408, 16459, 16534, 16460, 16331, 16670, 16734, 16476] avg=16384.0000 max=16734 max_rank=6 min=15408 min_rank=0 imbalance_ratio=1.021362 max_load_reduction=114338 max_load_reduction_pct=0.872330 moved_to_echo=66211 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[20096, 128, 29696, 0, 32640, 256, 128, 1024, 128, 2560, 128, 1024, 11520, 2304, 0, 128, 15872, 14592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[256, 128, 23168, 128, 32768, 32640, 384, 128, 128, 128, 6656, 128, 128, 0, 128, 128, 29184, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 31616, 128, 128, 0, 128, 512, 32768, 128, 128, 256, 896, 128, 128, 30336, 31232, 3584, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32128, 32384, 0, 0, 1536, 128, 32512, 128, 128, 128, 128, 32512, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 128, 128, 0, 128, 256, 16768, 15488, 32768, 32768, 128, 128, 128, 256, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 32512, 128, 32512, 128, 256, 32768, 128, 128, 128, 0, 128, 128, 32640, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 5248, 128, 128, 11136, 17664, 256, 32640, 128, 28032, 0, 6144, 30208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 256, 1024, 256, 384, 32640, 128, 128, 128, 4992, 128, 0, 128, 32768, 128, 32768, 26496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 27, 0, 0, 2232, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 7, 4096, 4096, 0, 4096, 0, 0, 4096, 3923, 0, 0, 0, 0, 0, 85, 1532, 485, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4009, 0, 0, 2, 0, 4059, 3968, 0, 0, 0, 0, 0, 0, 2307, 0, 0, 4080, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 10, 4096, 0, 4096, 1954, 1, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4091, 0, 4096, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 42, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 11, 4096, 0, 4096, 0, 0, 4034, 0, 0, 92, 0, 6, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 35, 4081, 4093, 0, 4089, 0, 20, 4096, 13, 0, 0, 0, 19, 0, 0, 2684, 137, 0, 0, 0, 4074, 0, 0, 0, 4096, 1, 4096, 4093, 0, 0, 3, 0, 3986, 2834, 0, 0, 0, 0, 0, 14, 4023, 0, 55, 4047, 0, 0, 0, 0, 0, 4092, 4095, 0, 0, 3, 0, 0, 0, 0, 0, 2, 4096, 0, 4096, 1095, 0, 4095, 0, 0, 2, 0, 4095, 0, 4096, 27, 4068, 0, 4096, 0, 0, 4016, 0, 12, 0, 0, 5, 4, 0, 0, 0, 0, 4081, 0, 0, 0, 4096, 0, 1600, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4094, 6, 0, 4076, 0, 0, 4, 0, 15, 4096, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 4096, 19, 4096, 4096, 0, 4096, 0, 0, 4091, 1318, 0, 1, 0, 0, 0, 3, 5, 3998, 0, 2, 0, 4095, 0, 0, 0, 4080, 0, 4092, 4096, 0, 0, 5, 0, 3746, 3980, 0, 0, 0, 0, 0, 0, 297, 0, 0, 4084, 2, 0, 0, 1, 0, 4045, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 9, 4096, 0, 4096, 896, 6, 4096, 0, 0, 0, 0, 4046, 0, 4096, 0, 4096, 0, 4096, 0, 0, 2697, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 3576, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4084, 1, 4094, 0, 1, 4096, 0, 0, 0, 0, 0, 2585, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 173, 4096, 4096, 0, 4092, 0, 1, 4096, 22, 0, 0, 0, 0, 0, 0, 3776, 1556, 0, 0, 0, 4093, 11, 0, 0, 4080, 0, 4095, 4096, 0, 0, 3476, 0, 4046, 1502, 2, 0, 7, 0, 13, 0, 0, 0, 0, 3025, 0, 0, 0, 0, 0, 3842, 4093, 4, 0, 0, 0, 0, 17, 0, 3, 0, 4095, 0, 4096, 2812, 939, 4096, 0, 0, 1, 0, 4095, 0, 4096, 4, 3973, 0, 4083, 0, 0, 4096, 1, 68, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 173, 0, 0, 4096, 0, 752, 38, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 10, 40, 3986, 11, 16, 51, 3984, 29, 4022, 55, 0, 3967, 38, 1, 0, 32, 0, 3905, 0, 0, 1, 29, 4057, 10, 0, 0, 160, 4021, 0, 4075, 4073, 24, 4070, 8, 2, 4039, 2084, 0, 0, 0, 0, 0, 0, 3472, 1, 3, 109, 4, 4033, 0, 61, 19, 4006, 0, 3917, 3977, 0, 0, 3863, 10, 1387, 244, 0, 7, 69, 12, 37, 128, 568, 1, 1, 3887, 3, 0, 12, 0, 0, 2630, 4069, 0, 1, 11, 0, 34, 67, 35, 0, 0, 4082, 0, 4071, 3998, 0, 3948, 0, 0, 125, 133, 4064, 0, 4075, 0, 4058, 4, 4054, 25, 0, 3953, 0, 0, 0, 37, 65, 0, 29, 8, 0, 233, 4087, 0, 59, 77, 3576, 1, 2739, 92, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 0, 4086, 0, 0, 0, 4042, 5, 4086, 0, 1, 4094, 0, 0, 1, 0, 0, 4083, 2, 1, 0, 0, 4096, 0, 0, 86, 0, 4086, 31, 4085, 4095, 3, 4095, 7, 0, 4029, 423, 0, 28, 0, 19, 0, 31, 3894, 0, 0, 25, 3, 4096, 22, 0, 0, 3039, 0, 4028, 3969, 0, 0, 1475, 0, 4096, 193, 56, 2, 6, 0, 0, 61, 3884, 6, 0, 3569, 1, 0, 19, 0, 15, 4085, 4092, 3, 4, 0, 2, 0, 0, 5, 0, 316, 4094, 0, 4082, 1993, 3, 4090, 36, 1, 6, 17, 4093, 0, 4062, 2, 4096, 0, 4076, 0, 0, 4029, 0, 137, 0, 0, 250, 0, 0, 0, 14, 12, 4093, 1, 1, 0, 4093, 2, 1288, 19, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4, 4096, 0, 0, 0, 4093, 3, 4094, 63, 0, 4063, 0, 0, 0, 0, 0, 897, 0, 0, 0, 9, 4095, 0, 0, 0, 1, 4095, 3, 4096, 4094, 0, 4092, 0, 3, 4095, 31, 0, 0, 0, 1, 0, 0, 3658, 0, 0, 0, 0, 4092, 0, 0, 13, 4096, 0, 4093, 4090, 0, 2, 3167, 4, 4034, 1058, 0, 0, 2, 0, 0, 2, 4092, 0, 0, 4093, 0, 15, 0, 1, 0, 3958, 4093, 0, 0, 12, 0, 0, 0, 0, 0, 51, 4086, 0, 4087, 0, 34, 4077, 0, 4, 11, 7, 4095, 0, 4095, 5, 4093, 0, 4086, 2, 0, 3759, 0, 3, 0, 0, 6, 0, 1, 3, 2, 0, 4095, 18, 0, 1, 3963, 0, 3985, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 235, 0, 0, 3988, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 148, 4096, 4096, 0, 4096, 0, 0, 4096, 2311, 0, 0, 0, 0, 0, 2, 3880, 447, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1886, 0, 2122, 3910, 0, 0, 0, 0, 0, 0, 3038, 0, 0, 4096, 0, 1, 0, 0, 0, 1865, 4096, 0, 0, 0, 0, 0, 74, 0, 0, 1, 4096, 0, 4096, 1287, 0, 4077, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 3484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4070, 0, 13, 25, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4088, 0, 0, 4096, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2232, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 7, 4096, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3923, 0, 0, 0, 0, 0, 85, 1532, 485, 0, 0, 1719, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 984, 4009, 0, 0, 2, 0, 4059, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2307, 0, 0, 4080, 0, 0, 0, 0, 0, 4094, 4096, 1585, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 10, 4096, 0, 4096, 1954, 1, 4096, 0, 2171, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 309, 0, 4091, 0, 4096, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 42, 0, 4096, 2295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16403, 15319, 17428, 17118, 17103, 16439, 16637, 14625] avg=16384.0000 max=17428 max_rank=2 min=14625 min_rank=7 imbalance_ratio=1.063721 max_load_reduction=113644 max_load_reduction_pct=0.867035 moved_to_echo=64804 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 11, 4096, 0, 4088, 0, 0, 4034, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 35, 4081, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 20, 4096, 13, 0, 0, 0, 19, 0, 0, 2684, 137, 0, 0, 1718, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 984, 4093, 0, 0, 3, 0, 3986, 2834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4023, 0, 55, 4047, 0, 0, 0, 0, 0, 4092, 4095, 1584, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 4096, 0, 4096, 1095, 0, 4095, 0, 2171, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 309, 27, 4068, 0, 4096, 0, 0, 4016, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 0, 0, 0, 4081, 0, 0, 0, 4096, 0, 1600, 3, 0, 4074, 2295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16415, 17192, 14268, 15997, 18851, 15566, 16625, 16158] avg=16384.0000 max=18851 max_rank=4 min=14268 min_rank=2 imbalance_ratio=1.150574 max_load_reduction=112221 max_load_reduction_pct=0.856178 moved_to_echo=67200 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4084, 1, 4086, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2585, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 173, 4096, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 4096, 22, 0, 0, 0, 0, 0, 0, 3776, 1556, 0, 0, 1719, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4080, 0, 984, 4096, 0, 0, 3476, 0, 4046, 1502, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 13, 0, 0, 0, 0, 3025, 0, 0, 0, 0, 0, 3842, 4093, 1585, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 17, 0, 3, 0, 4095, 0, 4096, 2812, 939, 4096, 0, 2170, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 309, 4, 3973, 0, 4083, 0, 0, 4096, 1, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 173, 0, 0, 4096, 0, 752, 38, 0, 4093, 2295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16364, 15838, 16754, 18197, 13506, 18240, 16630, 15543] avg=16384.0000 max=18240 max_rank=5 min=13506 min_rank=4 imbalance_ratio=1.113281 max_load_reduction=112832 max_load_reduction_pct=0.860840 moved_to_echo=63919 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4086, 6, 0, 4076, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 4096, 19, 4096, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 1318, 0, 1, 0, 0, 0, 3, 5, 3998, 0, 2, 1719, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 983, 4096, 0, 0, 5, 0, 3746, 3980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 4084, 2, 0, 0, 1, 0, 4045, 4096, 1585, 940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 9, 4096, 0, 4096, 896, 6, 4096, 0, 2169, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 309, 0, 4096, 0, 4096, 0, 0, 2697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 3576, 3, 0, 4095, 2295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16364, 17190, 16725, 16890, 15050, 15386, 15244, 18223] avg=16384.0000 max=18223 max_rank=7 min=15050 min_rank=4 imbalance_ratio=1.112244 max_load_reduction=112849 max_load_reduction_pct=0.860970 moved_to_echo=63903 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 10, 40, 3986, 11, 16, 51, 3984, 29, 4011, 55, 0, 3967, 38, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 3905, 0, 0, 1, 29, 4057, 10, 0, 0, 160, 4021, 0, 4075, 778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4070, 8, 2, 4039, 2084, 0, 0, 0, 0, 0, 0, 3472, 1, 3, 109, 1713, 1487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 61, 19, 4006, 0, 936, 3977, 0, 0, 3863, 10, 1387, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 69, 12, 37, 128, 568, 1, 1, 3887, 3, 0, 12, 0, 0, 2630, 4069, 1582, 903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 11, 0, 34, 67, 35, 0, 0, 4082, 0, 4071, 3998, 0, 3948, 0, 2078, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 133, 4064, 0, 302, 0, 4058, 4, 4054, 25, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 65, 0, 29, 8, 0, 233, 4087, 0, 59, 77, 3576, 1, 2739, 92, 1, 4033, 2286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16199, 17068, 17012, 14507, 13909, 18336, 16718, 17323] avg=16384.0000 max=18336 max_rank=5 min=13909 min_rank=4 imbalance_ratio=1.119141 max_load_reduction=112736 max_load_reduction_pct=0.860107 moved_to_echo=66286 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4086, 0, 0, 0, 4042, 5, 4078, 0, 1, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 2, 1, 0, 0, 4096, 0, 0, 86, 0, 4086, 31, 4085, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 7, 0, 4029, 423, 0, 28, 0, 19, 0, 31, 3894, 0, 0, 25, 1718, 1480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 22, 0, 0, 3039, 0, 968, 3969, 0, 0, 1475, 0, 4096, 193, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 61, 3884, 6, 0, 3569, 1, 0, 19, 0, 15, 4085, 4092, 1585, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 2, 0, 0, 5, 0, 316, 4094, 0, 4082, 1993, 3, 4090, 36, 2135, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 17, 4093, 0, 306, 2, 4096, 0, 4076, 0, 0, 4029, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 0, 0, 0, 14, 12, 4093, 1, 1, 0, 4093, 2, 1288, 19, 0, 4096, 2276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16308, 17262, 15752, 13821, 18250, 16771, 16763, 16145] avg=16384.0000 max=18250 max_rank=4 min=13821 min_rank=3 imbalance_ratio=1.113892 max_load_reduction=112822 max_load_reduction_pct=0.860764 moved_to_echo=67929 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4, 4096, 0, 0, 0, 4093, 3, 4086, 63, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 897, 0, 0, 0, 9, 4095, 0, 0, 0, 1, 4095, 3, 4096, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3, 4095, 31, 0, 0, 0, 1, 0, 0, 3658, 0, 0, 0, 1718, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 0, 984, 4090, 0, 2, 3167, 4, 4034, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 4092, 0, 0, 4093, 0, 15, 0, 1, 0, 3958, 4093, 1585, 940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 51, 4086, 0, 4087, 0, 34, 4077, 0, 2169, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 11, 7, 4095, 0, 309, 5, 4093, 0, 4086, 2, 0, 3759, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 3, 2, 0, 4095, 18, 0, 1, 3963, 0, 3985, 0, 0, 4092, 2294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16408, 13987, 15090, 17448, 18781, 14524, 16374, 18460] avg=16384.0000 max=18781 max_rank=4 min=13987 min_rank=1 imbalance_ratio=1.146301 max_load_reduction=112291 max_load_reduction_pct=0.856712 moved_to_echo=68139 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4088, 0, 0, 4096, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 148, 4096, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2311, 0, 0, 0, 0, 0, 2, 3880, 447, 0, 0, 1719, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 984, 4096, 0, 0, 1886, 0, 2122, 3910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3038, 0, 0, 4096, 0, 1, 0, 0, 0, 1865, 4096, 1585, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 1, 4096, 0, 4096, 1287, 0, 4077, 0, 2171, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 309, 0, 4096, 0, 4096, 0, 0, 3484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4070, 0, 13, 25, 0, 4096, 2295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16611, 17216, 18043, 17094, 15622, 15810, 16081, 14595] avg=16384.0000 max=18043 max_rank=2 min=14595 min_rank=7 imbalance_ratio=1.101257 max_load_reduction=113029 max_load_reduction_pct=0.862343 moved_to_echo=62108 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 128, 32640, 10240, 0, 128, 0, 128, 0, 128, 22912, 6656, 128, 256, 13824, 12032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 256, 18304, 128, 128, 30976, 128, 128, 128, 128, 128, 28672, 32768, 12800, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 25984, 128, 128, 128, 128, 32768, 128, 0, 128, 256, 32768, 512, 32768, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 128, 32640, 128, 32640, 128, 128, 32640, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 256, 256, 32768, 0, 2560, 128, 32640, 128, 32768, 128, 0, 30080, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 256, 128, 128, 512, 32768, 0, 32768, 14080, 1024, 32640, 128, 17280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 31616, 128, 7808, 32512, 0, 128, 13952, 128, 27520, 17792, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 512, 128, 128, 128, 128, 256, 32768, 256, 128, 128, 32128, 128, 14080, 256, 128, 32768, 18432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3101, 1, 0, 0, 1101, 0, 0, 380, 0, 0, 0, 0, 0, 0, 4096, 0, 54, 0, 0, 0, 74, 0, 4096, 0, 0, 3838, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 2109, 19, 4085, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 3686, 4046, 0, 4038, 4096, 4096, 0, 4096, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2031, 0, 0, 0, 2892, 0, 0, 0, 0, 0, 0, 4094, 4, 0, 0, 0, 4094, 0, 2131, 4096, 0, 0, 4095, 0, 1004, 4096, 0, 4096, 0, 0, 12, 0, 4096, 0, 4096, 0, 2, 1503, 0, 0, 0, 0, 240, 0, 0, 4096, 0, 4096, 0, 0, 0, 520, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4095, 163, 0, 4096, 0, 0, 0, 4005, 0, 0, 9, 0, 0, 0, 0, 0, 1, 4095, 0, 37, 0, 0, 5, 47, 0, 4091, 1, 1, 4079, 0, 0, 4095, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 502, 0, 4092, 0, 0, 4096, 36, 4094, 1, 0, 4096, 0, 3636, 4035, 0, 4081, 4096, 4096, 0, 4095, 4086, 0, 0, 4096, 6, 1, 0, 0, 0, 0, 0, 56, 0, 0, 0, 1295, 0, 0, 0, 0, 2, 0, 4092, 1, 0, 4, 0, 4038, 0, 2000, 4096, 8, 0, 4096, 0, 399, 4095, 0, 4096, 0, 0, 0, 6, 4096, 0, 4095, 0, 0, 4, 0, 0, 0, 0, 3637, 0, 0, 4096, 0, 4095, 0, 0, 20, 673, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 15, 0, 4090, 0, 0, 0, 2782, 0, 0, 1, 0, 0, 0, 0, 137, 0, 4096, 0, 714, 0, 0, 0, 3, 0, 4095, 0, 40, 1980, 0, 0, 4096, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 2147, 0, 4093, 0, 0, 4096, 0, 4096, 0, 0, 3862, 0, 2798, 4094, 0, 4095, 4096, 4088, 0, 4096, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2349, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 4096, 36, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 4096, 0, 2, 4096, 0, 4065, 0, 0, 1, 0, 4096, 0, 4096, 0, 14, 0, 0, 0, 0, 0, 3538, 0, 0, 4096, 0, 4096, 0, 0, 0, 129, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4061, 232, 6, 4012, 2, 26, 0, 4022, 0, 63, 0, 0, 0, 97, 149, 0, 0, 4035, 0, 0, 0, 1, 39, 931, 0, 4067, 110, 9, 4078, 17, 43, 4064, 0, 151, 0, 4055, 50, 0, 19, 0, 30, 3, 0, 4034, 1, 0, 4043, 0, 3964, 16, 0, 3613, 0, 4070, 4015, 2, 3941, 4071, 3912, 0, 4076, 2424, 2, 62, 4080, 0, 25, 19, 74, 58, 22, 0, 4087, 0, 0, 3, 3867, 3, 0, 0, 0, 12, 19, 4072, 334, 48, 89, 10, 3966, 50, 0, 4084, 0, 0, 4018, 1, 481, 4041, 54, 3794, 15, 0, 0, 0, 3840, 0, 4048, 0, 0, 10, 0, 0, 1, 0, 1042, 12, 25, 4030, 0, 3933, 26, 66, 0, 0, 0, 0, 45, 50] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4094, 140, 0, 4090, 0, 6, 0, 4094, 0, 0, 0, 0, 0, 1, 0, 63, 14, 4080, 0, 64, 1, 1, 0, 206, 0, 4095, 0, 0, 4022, 1, 2, 4093, 17, 0, 0, 4090, 2, 0, 0, 0, 2, 36, 17, 4078, 0, 0, 4089, 0, 4093, 0, 0, 3954, 0, 17, 4076, 0, 4095, 4095, 4031, 0, 4096, 3892, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 3933, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4016, 3, 0, 6, 0, 4087, 0, 36, 4096, 0, 1, 4095, 0, 181, 4094, 5, 4096, 0, 19, 12, 8, 4078, 0, 4091, 0, 0, 0, 0, 0, 0, 2, 4088, 0, 0, 4071, 0, 4084, 3, 9, 0, 9, 0, 13, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4084, 113, 5, 4075, 7, 4, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 56, 380, 0, 26, 0, 0, 1, 4094, 0, 68, 4038, 0, 6, 4096, 939, 34, 0, 4092, 2, 11, 0, 1, 3, 4018, 80, 4094, 47, 0, 4054, 1, 4071, 0, 0, 3948, 0, 61, 4096, 0, 3903, 4083, 4011, 28, 4096, 4065, 0, 3, 4095, 0, 0, 0, 0, 13, 1, 1, 3768, 0, 0, 0, 4047, 2, 0, 1, 7, 0, 0, 3519, 3, 2, 0, 5, 3303, 0, 206, 4092, 1, 0, 4091, 0, 642, 4033, 0, 4091, 0, 4, 0, 11, 4086, 0, 4090, 5, 0, 90, 0, 0, 2, 0, 3751, 0, 0, 4083, 0, 4093, 0, 0, 0, 0, 0, 0, 14, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4075, 0, 0, 3553, 0, 0, 0, 3043, 0, 0, 1055, 0, 0, 0, 0, 0, 18, 4094, 0, 206, 0, 0, 0, 0, 0, 4096, 0, 0, 3817, 0, 0, 4096, 2768, 0, 0, 4093, 0, 0, 0, 0, 0, 1145, 0, 4090, 0, 0, 4096, 2, 4096, 0, 0, 4096, 0, 26, 4094, 0, 4095, 4096, 4096, 0, 4096, 4081, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 1715, 1, 0, 0, 3770, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 4044, 0, 2, 4096, 5, 5, 4095, 0, 526, 4096, 0, 4087, 0, 0, 16, 0, 4095, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 2997, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 11, 0, 4044, 0, 0, 0, 3926, 0, 0, 123, 0, 0, 0, 0, 0, 0, 4096, 0, 104, 0, 0, 0, 10, 0, 4095, 0, 55, 4092, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 3907, 0, 3970, 4096, 0, 4095, 4096, 4069, 0, 4096, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4089, 33, 0, 0, 0, 4096, 0, 184, 4096, 0, 0, 4096, 0, 7, 4096, 0, 4070, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 3101, 1, 0, 0, 1101, 0, 0, 380, 0, 0, 0, 0, 0, 3505, 1699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 54, 0, 0, 0, 74, 0, 4096, 0, 0, 3838, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 2109, 19, 4085, 0, 0, 4096, 0, 1371, 525, 427, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3686, 0, 0, 4038, 0, 2271, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2031, 0, 0, 0, 2892, 0, 0, 0, 4096, 2397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4, 0, 0, 0, 4094, 0, 2131, 3669, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 4096, 0, 4096, 0, 0, 12, 0, 4096, 0, 3571, 0, 2, 1503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 4096, 0, 4096, 0, 0, 0, 520, 0, 0, 0, 0, 4046, 591, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13883, 16211, 16772, 18184, 15512, 18087, 18380, 14043] avg=16384.0000 max=18380 max_rank=6 min=13883 min_rank=0 imbalance_ratio=1.121826 max_load_reduction=112692 max_load_reduction_pct=0.859772 moved_to_echo=66022 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4095, 163, 0, 4096, 0, 0, 0, 4005, 0, 0, 9, 0, 0, 0, 0, 0, 3503, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 37, 0, 0, 5, 47, 0, 4091, 1, 1, 4079, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 502, 0, 4092, 0, 0, 4096, 36, 1371, 525, 427, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 3636, 0, 0, 4081, 0, 2271, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 1, 0, 0, 0, 0, 0, 56, 0, 0, 0, 1295, 0, 0, 0, 4095, 2396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 1, 0, 4, 0, 4038, 0, 2000, 3669, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 4095, 0, 4096, 0, 0, 0, 6, 4096, 0, 3570, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3637, 0, 0, 4096, 0, 4095, 0, 0, 20, 673, 0, 0, 0, 0, 4035, 591, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17571, 16409, 15199, 18171, 11945, 17910, 16266, 17601] avg=16384.0000 max=18171 max_rank=3 min=11945 min_rank=4 imbalance_ratio=1.109070 max_load_reduction=112901 max_load_reduction_pct=0.861366 moved_to_echo=63722 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 15, 0, 4090, 0, 0, 0, 2782, 0, 0, 1, 0, 0, 0, 0, 137, 3505, 1699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 714, 0, 0, 0, 3, 0, 4095, 0, 40, 1980, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 2147, 0, 4093, 0, 0, 4096, 0, 1368, 525, 427, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3862, 0, 2798, 0, 0, 4095, 0, 2267, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2349, 0, 0, 0, 4076, 0, 0, 0, 4096, 2397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 36, 0, 0, 0, 4090, 0, 0, 3669, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 4065, 0, 0, 1, 0, 4096, 0, 3571, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3538, 0, 0, 4096, 0, 4096, 0, 0, 0, 129, 0, 0, 0, 0, 4094, 591, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16325, 14981, 16811, 17112, 17014, 15987, 15845, 16997] avg=16384.0000 max=17112 max_rank=3 min=14981 min_rank=1 imbalance_ratio=1.044434 max_load_reduction=113960 max_load_reduction_pct=0.869446 moved_to_echo=65843 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4061, 232, 6, 4012, 2, 26, 0, 4022, 0, 63, 0, 0, 0, 97, 149, 0, 3393, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 1, 39, 931, 0, 4067, 110, 9, 4078, 17, 43, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 4055, 50, 0, 19, 0, 30, 3, 0, 4034, 1, 0, 4043, 0, 1314, 524, 428, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3613, 0, 4070, 0, 2, 3941, 0, 2162, 0, 0, 2424, 2, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 25, 19, 74, 58, 22, 0, 4087, 0, 0, 3, 3867, 3, 0, 0, 4076, 2382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 19, 4072, 334, 48, 89, 10, 3966, 50, 0, 3656, 0, 0, 4018, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 4041, 54, 3794, 15, 0, 0, 0, 3840, 0, 3524, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1042, 12, 25, 4030, 0, 3933, 26, 66, 0, 0, 0, 0, 45, 50, 4015, 571, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17752, 17349, 14697, 16292, 18696, 16275, 15759, 14252] avg=16384.0000 max=18696 max_rank=4 min=14252 min_rank=7 imbalance_ratio=1.141113 max_load_reduction=112376 max_load_reduction_pct=0.857361 moved_to_echo=64982 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4075, 0, 0, 3553, 0, 0, 0, 3043, 0, 0, 1055, 0, 0, 0, 0, 0, 3506, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4094, 0, 206, 0, 0, 0, 0, 0, 4096, 0, 0, 3817, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2768, 0, 0, 4093, 0, 0, 0, 0, 0, 1145, 0, 4090, 0, 0, 4096, 2, 1371, 525, 427, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 26, 0, 0, 4095, 0, 2271, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 1715, 1, 0, 0, 3770, 0, 0, 0, 4096, 2396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 4044, 0, 2, 3669, 5, 5, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 526, 4096, 0, 4087, 0, 0, 16, 0, 4095, 0, 3571, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2997, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 590, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16932, 16284, 18560, 14569, 16075, 15930, 16396, 16326] avg=16384.0000 max=18560 max_rank=2 min=14569 min_rank=3 imbalance_ratio=1.132812 max_load_reduction=112512 max_load_reduction_pct=0.858398 moved_to_echo=64727 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4084, 113, 5, 4075, 7, 4, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 3484, 1694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 56, 380, 0, 26, 0, 0, 1, 4094, 0, 68, 4038, 0, 6, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 939, 34, 0, 4092, 2, 11, 0, 1, 3, 4018, 80, 4094, 47, 0, 4054, 1, 1343, 525, 426, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 0, 61, 0, 0, 3903, 0, 2224, 28, 0, 4065, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 13, 1, 1, 3768, 0, 0, 0, 4047, 2, 0, 1, 4096, 2389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3519, 3, 2, 0, 5, 3303, 0, 206, 3666, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 642, 4033, 0, 4091, 0, 4, 0, 11, 4086, 0, 3565, 5, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3751, 0, 0, 4083, 0, 4093, 0, 0, 0, 0, 0, 0, 14, 0, 4096, 587, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13517, 16797, 19713, 14232, 18413, 14803, 16527, 17070] avg=16384.0000 max=19713 max_rank=2 min=13517 min_rank=0 imbalance_ratio=1.203186 max_load_reduction=111359 max_load_reduction_pct=0.849602 moved_to_echo=66813 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4094, 140, 0, 4090, 0, 6, 0, 4094, 0, 0, 0, 0, 0, 1, 0, 63, 3502, 1698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4080, 0, 64, 1, 1, 0, 206, 0, 4095, 0, 0, 4022, 1, 2, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 4090, 2, 0, 0, 0, 2, 36, 17, 4078, 0, 0, 4089, 0, 1349, 525, 427, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3954, 0, 17, 0, 0, 4095, 0, 2236, 0, 0, 3892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 3933, 0, 0, 0, 4086, 0, 0, 0, 4096, 2397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 3, 0, 6, 0, 4087, 0, 36, 3669, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 4094, 5, 4096, 0, 19, 12, 8, 4078, 0, 3566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4088, 0, 0, 4071, 0, 4084, 3, 9, 0, 9, 0, 13, 0, 6, 4076, 591, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17688, 16536, 14675, 14194, 18609, 15913, 16059, 17398] avg=16384.0000 max=18609 max_rank=4 min=14194 min_rank=3 imbalance_ratio=1.135803 max_load_reduction=112463 max_load_reduction_pct=0.858025 moved_to_echo=67979 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 11, 0, 4044, 0, 0, 0, 3926, 0, 0, 123, 0, 0, 0, 0, 0, 3505, 1699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 104, 0, 0, 0, 10, 0, 4095, 0, 55, 4092, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 4096, 0, 1362, 525, 427, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 0, 3970, 0, 0, 4095, 0, 2256, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 4096, 0, 0, 0, 4096, 2397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 33, 0, 0, 0, 4096, 0, 184, 3669, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 4070, 0, 0, 0, 0, 4096, 0, 3571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 591, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17404, 16505, 14645, 18318, 14808, 16167, 15840, 17385] avg=16384.0000 max=18318 max_rank=3 min=14645 min_rank=2 imbalance_ratio=1.118042 max_load_reduction=112754 max_load_reduction_pct=0.860245 moved_to_echo=64200 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 768, 128, 31104, 128, 128, 0, 23040, 0, 128, 1664, 0, 0, 128, 256, 256, 27904, 13696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 23424, 128, 128, 32768, 0, 32640, 128, 128, 128, 1408, 0, 128, 128, 128, 32640, 4736, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 128, 18176, 128, 0, 128, 28160, 128, 0, 128, 32768, 19200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3328, 32768, 128, 32512, 128, 128, 128, 128, 32512, 0, 28544, 128, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 1664, 128, 128, 128, 1280, 128, 32768, 128, 256, 29952, 128, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 32128, 512, 128, 128, 128, 31744, 128, 4608, 29440, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 0, 31616, 0, 18304, 0, 128, 32384, 0, 18048, 128, 0, 30848, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[3840, 256, 0, 32768, 128, 128, 128, 128, 128, 9984, 128, 32768, 128, 0, 32768, 128, 10880, 4224, 3456, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 3016, 566, 0, 4096, 73, 4096, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3333, 0, 0, 0, 2864, 2, 0, 4096, 4096, 0, 0, 0, 4095, 4068, 4096, 0, 1976, 0, 0, 0, 0, 1, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4085, 4096, 0, 0, 0, 81, 60, 4096, 0, 0, 0, 4096, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 3530, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 935, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 2727, 2347, 0, 4096, 434, 4096, 0, 0, 0, 0, 1, 0, 7, 4093, 0, 0, 0, 0, 2, 4095, 2, 2126, 0, 0, 0, 54, 7, 0, 4096, 4096, 1, 0, 0, 4096, 4094, 4096, 0, 4056, 0, 0, 0, 0, 625, 0, 4096, 4034, 4096, 1, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 4094, 0, 0, 0, 4095, 4096, 4091, 4096, 0, 0, 32, 90, 0, 4096, 28, 7, 0, 4096, 0, 0, 4060, 0, 0, 0, 2, 0, 0, 3974, 0, 4096, 9, 0, 0, 4079, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4085, 4096, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4089, 5, 25, 1575, 6, 4085, 42, 4096, 108, 0, 0, 3, 22, 0, 0, 4092, 0, 3, 0, 4, 1, 4090, 0, 3487, 0, 0, 0, 3816, 336, 4, 4096, 4094, 0, 0, 0, 4095, 3899, 4091, 0, 2724, 0, 1, 0, 0, 10, 3, 4093, 4087, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 20, 0, 4096, 4, 0, 0, 4093, 4095, 4077, 4095, 0, 0, 2, 0, 0, 4095, 328, 51, 0, 4095, 0, 0, 4095, 0, 7, 0, 0, 2, 0, 4061, 4, 4080, 0, 11, 0, 4093, 6, 0, 2, 2, 4096, 0, 0, 0, 0, 0, 4095, 4095, 2, 4091, 4094, 0, 2, 0, 2, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 55, 22, 4079, 42, 2135, 0, 0, 3976, 0, 3985, 1, 15, 2, 23, 81, 0, 157, 4058, 0, 10, 0, 0, 76, 4031, 0, 0, 13, 0, 48, 2967, 3144, 35, 4068, 3791, 69, 0, 1, 3363, 4050, 4075, 10, 3360, 13, 0, 0, 14, 1161, 0, 4076, 3914, 4073, 0, 0, 15, 0, 158, 3, 114, 30, 0, 0, 0, 4080, 0, 71, 18, 4037, 4069, 3704, 4082, 0, 3, 51, 26, 0, 4058, 300, 286, 0, 4045, 40, 0, 4090, 13, 0, 1, 0, 12, 28, 2947, 27, 3958, 0, 3, 8, 4079, 1, 0, 20, 0, 4036, 12, 0, 0, 0, 136, 4085, 4072, 0, 3970, 4055, 37, 1, 0, 0, 0, 0, 0, 2, 0, 0, 3, 1276, 15, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 145, 0, 4096, 0, 3, 219, 0, 4096, 66, 4096, 1, 0, 0, 0, 0, 0, 8, 4095, 0, 0, 14, 0, 0, 4096, 12, 3936, 0, 2, 0, 4042, 0, 0, 4096, 4096, 0, 0, 0, 3825, 4018, 4083, 0, 4070, 0, 0, 4, 0, 112, 0, 4096, 4096, 4096, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4072, 0, 0, 0, 0, 0, 4088, 38, 0, 1, 4095, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 0, 4054, 4096, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 3, 0, 4096, 0, 3850, 0, 0, 4094, 0, 4096, 1144, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 33, 3, 0, 4089, 11, 3980, 0, 0, 0, 160, 138, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 4089, 0, 3771, 0, 0, 0, 0, 94, 0, 4060, 4092, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4096, 0, 0, 0, 4095, 4093, 4096, 4096, 0, 0, 0, 13, 39, 4096, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 3200, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[6, 0, 2, 4094, 0, 1788, 2532, 0, 4073, 0, 4080, 1, 0, 0, 1, 1495, 0, 0, 4095, 0, 0, 7, 3, 25, 4013, 0, 4079, 1, 105, 5, 8, 1657, 0, 4054, 4070, 0, 0, 0, 3943, 4062, 4096, 5, 3056, 0, 0, 0, 0, 0, 0, 4096, 4096, 4086, 0, 0, 2, 0, 179, 0, 17, 3, 1, 77, 2, 4081, 1, 38, 21, 4095, 4086, 2207, 4094, 0, 0, 0, 468, 3, 4090, 12, 0, 118, 4092, 0, 0, 4096, 0, 2, 0, 61, 0, 0, 3035, 0, 4076, 0, 0, 1, 4093, 0, 0, 0, 0, 4095, 0, 0, 4, 3, 0, 3994, 4090, 0, 4091, 4095, 0, 0, 0, 2, 0, 0, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 91, 80, 0, 4096, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3914, 0, 0, 0, 723, 3940, 0, 4096, 4096, 0, 0, 0, 4090, 4086, 4096, 0, 2961, 0, 0, 0, 0, 451, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 12, 0, 4096, 87, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 3016, 566, 0, 4096, 73, 4096, 4, 0, 0, 0, 0, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3333, 0, 0, 0, 2864, 2, 1840, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 763, 4096, 0, 0, 0, 4095, 4068, 0, 0, 1976, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4085, 24, 0, 0, 0, 81, 60, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 3530, 0, 4096, 0, 0, 0, 452, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 935, 0, 0, 0, 0, 4096, 4096, 4072, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16988, 16454, 14999, 16349, 16538, 16288, 16411, 17045] avg=16384.0000 max=17045 max_rank=7 min=14999 min_rank=2 imbalance_ratio=1.040344 max_load_reduction=114027 max_load_reduction_pct=0.869957 moved_to_echo=66282 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 2727, 2347, 0, 4096, 434, 4096, 0, 0, 0, 0, 1, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4093, 0, 0, 0, 0, 2, 4095, 2, 2126, 0, 0, 0, 54, 7, 1840, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 763, 4096, 1, 0, 0, 4096, 4094, 0, 0, 4056, 0, 0, 0, 0, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4034, 4096, 1, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4091, 24, 0, 0, 32, 90, 0, 4096, 28, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4060, 0, 0, 0, 2, 0, 0, 3974, 0, 4096, 9, 0, 0, 452, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4085, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4072, 3830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18838, 12448, 17731, 16291, 16559, 16724, 16387, 16094] avg=16384.0000 max=18838 max_rank=0 min=12448 min_rank=1 imbalance_ratio=1.149780 max_load_reduction=112234 max_load_reduction_pct=0.856277 moved_to_echo=65764 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 55, 22, 4079, 42, 2135, 0, 0, 3976, 0, 3985, 1, 15, 2, 23, 81, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 4058, 0, 10, 0, 0, 76, 4031, 0, 0, 13, 0, 48, 2967, 3144, 1833, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 743, 3791, 69, 0, 1, 3363, 4050, 0, 10, 3360, 13, 0, 0, 14, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 3914, 4073, 0, 0, 15, 0, 158, 3, 114, 30, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 18, 4037, 4069, 3704, 18, 0, 3, 51, 26, 0, 4058, 300, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 40, 0, 4090, 13, 0, 1, 0, 12, 28, 2947, 27, 3958, 0, 3, 8, 456, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 1, 0, 20, 0, 4036, 12, 0, 0, 0, 136, 4085, 0, 0, 3970, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 0, 2, 0, 0, 3, 1276, 15, 0, 2, 0, 4072, 4075, 4064, 3834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15452, 16563, 16610, 16427, 16641, 15664, 16334, 17381] avg=16384.0000 max=17381 max_rank=7 min=15452 min_rank=0 imbalance_ratio=1.060852 max_load_reduction=113691 max_load_reduction_pct=0.867393 moved_to_echo=66020 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 145, 0, 4096, 0, 3, 219, 0, 4096, 66, 4096, 1, 0, 0, 0, 0, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4095, 0, 0, 14, 0, 0, 4096, 12, 3936, 0, 2, 0, 4042, 0, 1840, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 763, 4096, 0, 0, 0, 3825, 4018, 0, 0, 4070, 0, 0, 4, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 24, 0, 0, 0, 0, 0, 4088, 38, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 452, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 0, 4054, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 4048, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13763, 18268, 16888, 16380, 16439, 16867, 16394, 16073] avg=16384.0000 max=18268 max_rank=1 min=13763 min_rank=0 imbalance_ratio=1.114990 max_load_reduction=112804 max_load_reduction_pct=0.860626 moved_to_echo=65773 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4089, 5, 25, 1575, 6, 4085, 42, 4096, 108, 0, 0, 3, 22, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3, 0, 4, 1, 4090, 0, 3487, 0, 0, 0, 3816, 336, 1840, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 763, 4094, 0, 0, 0, 4095, 3899, 0, 0, 2724, 0, 1, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 4087, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 20, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4093, 4095, 4077, 24, 0, 0, 2, 0, 0, 4095, 328, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 7, 0, 0, 2, 0, 4061, 4, 4080, 0, 11, 0, 452, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 6, 0, 2, 2, 4096, 0, 0, 0, 0, 0, 4095, 0, 2, 4091, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 4095, 4091, 4071, 3843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15097, 17892, 15590, 16360, 16769, 16842, 16415, 16107] avg=16384.0000 max=17892 max_rank=1 min=15097 min_rank=0 imbalance_ratio=1.092041 max_load_reduction=113180 max_load_reduction_pct=0.863495 moved_to_echo=66133 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 4096, 0, 3850, 0, 0, 4094, 0, 4096, 1144, 0, 0, 0, 0, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 33, 3, 0, 4089, 11, 3980, 0, 0, 0, 160, 138, 1840, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 763, 4096, 0, 0, 0, 4096, 4096, 0, 0, 3771, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 4092, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4093, 4096, 24, 0, 0, 0, 13, 39, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 3200, 0, 4096, 1, 0, 0, 452, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4089, 4072, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18324, 14573, 16916, 16310, 16456, 15972, 16411, 16110] avg=16384.0000 max=18324 max_rank=0 min=14573 min_rank=1 imbalance_ratio=1.118408 max_load_reduction=112748 max_load_reduction_pct=0.860199 moved_to_echo=64949 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 2, 4094, 0, 1788, 2532, 0, 4073, 0, 4080, 1, 0, 0, 1, 1495, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 7, 3, 25, 4013, 0, 4079, 1, 105, 5, 8, 1657, 1821, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 755, 4070, 0, 0, 0, 3943, 4062, 0, 5, 3056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4086, 0, 0, 2, 0, 179, 0, 17, 3, 1, 77, 2, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 38, 21, 4095, 4086, 2207, 24, 0, 0, 0, 468, 3, 4090, 12, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 2, 0, 61, 0, 0, 3035, 0, 4076, 0, 0, 1, 448, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 4095, 0, 0, 4, 3, 0, 3994, 0, 0, 4091, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 4070, 3843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19102, 16042, 15891, 16606, 15163, 15845, 16309, 16114] avg=16384.0000 max=19102 max_rank=0 min=15163 min_rank=4 imbalance_ratio=1.165894 max_load_reduction=111970 max_load_reduction_pct=0.854263 moved_to_echo=63431 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 91, 80, 0, 4096, 0, 4096, 8, 0, 0, 0, 0, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3914, 0, 0, 0, 723, 3940, 1840, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 763, 4096, 0, 0, 0, 4090, 4086, 0, 0, 2961, 0, 0, 0, 0, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 24, 0, 0, 0, 12, 0, 4096, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 452, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 4096, 4096, 4072, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13508, 18832, 16447, 16349, 16507, 16870, 16411, 16148] avg=16384.0000 max=18832 max_rank=1 min=13508 min_rank=0 imbalance_ratio=1.149414 max_load_reduction=112240 max_load_reduction_pct=0.856323 moved_to_echo=65936 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 256, 32768, 0, 128, 128, 128, 128, 32640, 128, 24960, 128, 128, 128, 14720, 9344, 14720, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 32512, 32768, 128, 0, 128, 128, 384, 128, 256, 128, 128, 128, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 128, 0, 32768, 128, 128, 128, 128, 128, 128, 29056, 128, 32640, 128, 128, 128, 3712, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 256, 32640, 0, 128, 32640, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 256, 128, 32768, 128, 13696, 7424, 128, 32640, 640, 32768, 1280, 128, 128, 128, 1664, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 6144, 32512, 128, 0, 128, 31616, 32384, 0, 128, 25984, 128, 128, 128, 128, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 128, 32768, 32768, 30464, 256, 0, 128, 128, 768, 128, 32768, 896, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 0, 128, 0, 0, 128, 128, 128, 128, 128, 2304, 128, 0, 128, 128, 32768, 32768, 32640, 30848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[21, 0, 33, 1, 0, 4050, 14, 4004, 2209, 4032, 4074, 3805, 30, 5, 82, 0, 47, 55, 73, 4058, 0, 4043, 4074, 118, 0, 4062, 2951, 4000, 0, 0, 24, 4060, 3959, 0, 650, 0, 0, 63, 3, 68, 4066, 82, 4018, 0, 22, 3, 62, 47, 4060, 2, 0, 0, 3797, 1, 0, 38, 0, 10, 0, 1, 97, 176, 68, 30, 62, 0, 0, 266, 1, 0, 0, 58, 6, 4041, 1, 7, 11, 1906, 2, 0, 0, 4074, 0, 4062, 4049, 1, 73, 8, 0, 0, 13, 0, 28, 77, 0, 3938, 3989, 0, 0, 35, 0, 3, 0, 0, 21, 283, 3977, 4033, 0, 719, 5, 4011, 0, 3998, 0, 21, 3900, 3, 0, 3909, 4089, 41, 0, 0, 4066, 0, 0, 37] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 77, 4096, 0, 4096, 0, 4094, 4096, 4089, 0, 3, 0, 415, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4090, 4095, 4096, 0, 133, 0, 3823, 4096, 1, 0, 0, 0, 0, 0, 0, 3943, 16, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4074, 3325, 2, 0, 1, 0, 0, 0, 0, 0, 2193, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 743, 0, 174, 0, 4096, 0, 4096, 4093, 0, 0, 0, 0, 2285, 0, 0, 0, 0, 2, 371, 4090, 127, 0, 0, 1, 0, 8, 0, 0, 0, 2450, 4096, 0, 82, 1, 4096, 0, 3908, 0, 51, 4096, 0, 0, 4096, 4062, 0, 3, 0, 4086, 0, 505, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096, 2, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 4096, 0, 0, 4096, 3116, 4096, 3, 10, 0, 4091, 4096, 0, 2, 0, 0, 0, 0, 0, 4092, 0, 4096, 28, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 4004, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 14, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 221, 4082, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 4096, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 1131, 4095, 0, 1, 0, 4095, 0, 4089, 0, 0, 4096, 0, 0, 4096, 3719, 0, 0, 0, 4096, 0, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 4096, 0, 4096, 0, 4084, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4077, 4087, 0, 0, 4075, 3731, 4095, 0, 15, 0, 3948, 4096, 0, 138, 2, 9, 0, 0, 0, 3867, 0, 4080, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4041, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 234, 0, 0, 0, 0, 13, 0, 0, 4063, 2, 0, 58, 1084, 0, 465, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 174, 0, 0, 0, 0, 0, 3377, 4096, 132, 0, 0, 0, 0, 9, 0, 0, 11, 3717, 4092, 2, 10, 1, 4095, 0, 4096, 0, 2, 4087, 0, 0, 4089, 3680, 0, 34, 0, 4096, 13, 13, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 35, 4096, 0, 4096, 581, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 4096, 0, 0, 4096, 3381, 4096, 11, 8, 0, 4061, 4096, 16, 3974, 0, 0, 0, 0, 0, 4089, 0, 4092, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1, 0, 0, 135, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 232, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4, 0, 0, 77, 0, 0, 0, 0, 0, 1, 4023, 1208, 0, 0, 0, 0, 0, 0, 0, 1, 2906, 4091, 0, 77, 0, 4096, 0, 4005, 0, 33, 4093, 0, 0, 4096, 3984, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 17, 4093, 1, 4096, 0, 4094, 4090, 4066, 0, 2, 0, 0, 0, 0, 1, 4094, 0, 4054, 4094, 0, 0, 4096, 3035, 4090, 0, 13, 0, 3988, 4093, 0, 14, 0, 0, 0, 13, 12, 4079, 1419, 4096, 3, 0, 0, 0, 7, 4093, 0, 7, 0, 4084, 877, 0, 2, 0, 0, 0, 6, 3, 78, 3930, 1, 4, 112, 1, 0, 0, 5, 19, 5, 0, 4096, 0, 1, 0, 184, 0, 0, 0, 4095, 0, 4096, 4090, 0, 0, 0, 1, 0, 0, 19, 0, 0, 0, 1477, 3744, 1786, 2, 0, 0, 0, 0, 0, 0, 42, 3847, 4079, 0, 0, 0, 4096, 1, 4090, 0, 21, 4095, 0, 5, 4090, 4096, 0, 0, 0, 4091, 1, 37, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4088, 1, 4083, 5, 4093, 4096, 4031, 0, 0, 4, 65, 0, 0, 15, 4093, 0, 4095, 4089, 0, 2, 4096, 4090, 4084, 0, 287, 0, 4094, 4089, 0, 184, 0, 0, 0, 34, 12, 4081, 20, 4096, 190, 1, 3, 0, 0, 4095, 1, 0, 7, 4094, 3777, 0, 0, 0, 0, 64, 5, 0, 2, 2095, 1, 0, 378, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 106, 1, 0, 0, 4093, 7, 4095, 4064, 0, 0, 0, 0, 0, 30, 5, 1, 0, 3, 3431, 4011, 3741, 0, 0, 0, 0, 101, 0, 3, 9, 1, 4093, 0, 0, 11, 4090, 0, 2212, 44, 232, 4094, 1, 0, 4091, 3797, 0, 0, 0, 4094, 0, 1, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 4096, 19, 4096, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 4096, 0, 0, 4096, 4045, 4096, 7, 306, 0, 4094, 4096, 0, 349, 0, 0, 0, 0, 0, 4094, 3, 4096, 22, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 3695, 0, 0, 0, 0, 0, 0, 0, 1, 419, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 217, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3408, 4096, 3891, 0, 0, 0, 0, 0, 0, 0, 0, 207, 4095, 0, 76, 0, 4095, 0, 4096, 0, 30, 4096, 0, 0, 4096, 3815, 0, 0, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3724, 0, 4096, 2, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 509, 3116, 4096, 3, 10, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4092, 0, 4096, 28, 0, 0, 0, 0, 2347, 908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 4004, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 2802, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 221, 4082, 0, 0, 4096, 4096, 2781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 1240, 372, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 1131, 3964, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 4096, 3719, 0, 0, 0, 386, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16012, 15917, 15569, 16309, 19387, 14205, 17284, 16389] avg=16384.0000 max=19387 max_rank=4 min=14205 min_rank=5 imbalance_ratio=1.183289 max_load_reduction=111685 max_load_reduction_pct=0.852089 moved_to_echo=67265 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[21, 0, 33, 1, 0, 3679, 14, 4004, 2209, 4032, 0, 3805, 30, 5, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 55, 73, 0, 0, 4043, 0, 118, 0, 499, 2951, 4000, 0, 0, 24, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 650, 0, 0, 63, 3, 68, 4066, 82, 4018, 0, 22, 3, 62, 47, 2329, 904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 2, 0, 0, 3797, 1, 0, 38, 0, 10, 0, 1, 97, 176, 68, 30, 2786, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 266, 1, 0, 0, 58, 6, 4041, 1, 7, 11, 1906, 2, 0, 4074, 4074, 2754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 4062, 4049, 1, 73, 8, 0, 0, 13, 0, 28, 77, 0, 3938, 1234, 371, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 0, 35, 0, 3, 0, 0, 21, 283, 3977, 3898, 0, 719, 5, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 21, 3900, 3, 0, 3909, 4089, 41, 0, 0, 376, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17915, 15870, 16276, 12370, 17263, 18063, 16941, 16374] avg=16384.0000 max=18063 max_rank=5 min=12370 min_rank=3 imbalance_ratio=1.102478 max_load_reduction=113009 max_load_reduction_pct=0.862190 moved_to_echo=68641 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 77, 3724, 0, 4096, 0, 4094, 0, 4089, 0, 3, 0, 415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 509, 4095, 4096, 0, 133, 0, 3823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 3943, 16, 4096, 0, 0, 0, 0, 0, 2343, 905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4074, 3325, 2, 0, 1, 0, 0, 0, 0, 0, 2193, 0, 2795, 1316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 743, 0, 174, 4096, 4096, 2780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4093, 0, 0, 0, 0, 2285, 0, 0, 0, 0, 2, 371, 1238, 372, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 127, 0, 0, 1, 0, 8, 0, 0, 0, 2450, 3964, 0, 82, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3908, 0, 51, 4096, 0, 0, 4096, 4062, 0, 3, 0, 386, 0, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16498, 16752, 15400, 17802, 16009, 16685, 14819, 17107] avg=16384.0000 max=17802 max_rank=3 min=14819 min_rank=6 imbalance_ratio=1.086548 max_load_reduction=113270 max_load_reduction_pct=0.864182 moved_to_echo=64620 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 0, 0, 3724, 0, 4096, 0, 4084, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4077, 0, 0, 0, 506, 3731, 4095, 0, 15, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 138, 2, 9, 0, 0, 0, 3867, 0, 4080, 0, 0, 0, 0, 0, 2335, 908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4041, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 2802, 1316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 13, 0, 0, 4063, 2, 0, 58, 1084, 0, 465, 4095, 4087, 2780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 174, 0, 0, 0, 0, 0, 3377, 1234, 372, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 132, 0, 0, 0, 0, 9, 0, 0, 11, 3717, 3961, 2, 10, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4087, 0, 0, 4089, 3680, 0, 34, 0, 386, 13, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16003, 16376, 15435, 16368, 16881, 17575, 16034, 16400] avg=16384.0000 max=17575 max_rank=5 min=15435 min_rank=2 imbalance_ratio=1.072693 max_load_reduction=113497 max_load_reduction_pct=0.865913 moved_to_echo=66890 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 35, 3724, 0, 4096, 581, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 509, 3381, 4096, 11, 8, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 3974, 0, 0, 0, 0, 0, 4089, 0, 4092, 2, 0, 0, 0, 0, 2347, 908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2802, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 135, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 232, 0, 0, 4096, 4096, 2781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4, 0, 0, 77, 0, 0, 0, 0, 0, 1, 1240, 372, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 1208, 0, 0, 0, 0, 0, 0, 0, 1, 2906, 3960, 0, 77, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 33, 4093, 0, 0, 4096, 3984, 0, 0, 0, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16627, 16148, 19524, 16355, 15437, 14113, 16271, 16597] avg=16384.0000 max=19524 max_rank=2 min=14113 min_rank=5 imbalance_ratio=1.191650 max_load_reduction=111548 max_load_reduction_pct=0.851044 moved_to_echo=62418 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 17, 3721, 1, 4096, 0, 4094, 0, 4066, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4054, 0, 0, 0, 509, 3035, 4090, 0, 13, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 14, 0, 0, 0, 13, 12, 4079, 1419, 4096, 3, 0, 0, 0, 7, 2347, 906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 7, 0, 4084, 877, 0, 2, 0, 0, 0, 6, 3, 78, 3930, 1, 2799, 1314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 112, 1, 0, 0, 5, 19, 5, 0, 4096, 0, 1, 0, 184, 0, 0, 4090, 4094, 2780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4090, 0, 0, 0, 1, 0, 0, 19, 0, 0, 0, 1477, 1240, 372, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3744, 1786, 2, 0, 0, 0, 0, 0, 0, 42, 3847, 3948, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 21, 4095, 0, 5, 4090, 4096, 0, 0, 0, 386, 1, 37, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15997, 15690, 16989, 17194, 15391, 15521, 17465, 16825] avg=16384.0000 max=17465 max_rank=6 min=15391 min_rank=4 imbalance_ratio=1.065979 max_load_reduction=113607 max_load_reduction_pct=0.866753 moved_to_echo=65202 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3717, 1, 4083, 5, 4093, 0, 4031, 0, 0, 4, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4095, 0, 0, 2, 509, 4090, 4084, 0, 287, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 184, 0, 0, 0, 34, 12, 4081, 20, 4096, 190, 1, 3, 0, 0, 2347, 907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 7, 4094, 3777, 0, 0, 0, 0, 64, 5, 0, 2, 2095, 1, 2801, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 0, 0, 0, 0, 0, 3967, 0, 0, 0, 106, 1, 0, 4096, 4089, 2778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 7, 4095, 4064, 0, 0, 0, 0, 0, 30, 5, 1, 0, 3, 3431, 1240, 371, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 3741, 0, 0, 0, 0, 101, 0, 3, 9, 1, 3962, 0, 0, 11, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2212, 44, 232, 4094, 1, 0, 4091, 3797, 0, 0, 0, 386, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15999, 17176, 15964, 18257, 15415, 17471, 15929, 14861] avg=16384.0000 max=18257 max_rank=3 min=14861 min_rank=7 imbalance_ratio=1.114319 max_load_reduction=112815 max_load_reduction_pct=0.860710 moved_to_echo=63676 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 3724, 0, 4096, 19, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 509, 4045, 4096, 7, 306, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 349, 0, 0, 0, 0, 0, 4094, 3, 4096, 22, 0, 0, 0, 0, 2347, 908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 3695, 0, 0, 0, 0, 0, 0, 0, 1, 419, 0, 2802, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 217, 0, 0, 4096, 4096, 2781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3408, 1240, 372, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3891, 0, 0, 0, 0, 0, 0, 0, 0, 207, 3964, 0, 76, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 30, 4096, 0, 0, 4096, 3815, 0, 0, 0, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16021, 17143, 15915, 16417, 15289, 17439, 16329, 16519] avg=16384.0000 max=17439 max_rank=5 min=15289 min_rank=4 imbalance_ratio=1.064392 max_load_reduction=113633 max_load_reduction_pct=0.866951 moved_to_echo=65576 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 256, 29824, 128, 32768, 2816, 32768, 0, 32384, 128, 128, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 0, 128, 32640, 0, 128, 128, 4096, 28544, 32768, 128, 896, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32256, 14976, 128, 128, 128, 128, 128, 0, 128, 384, 18304, 31744, 128, 1024, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 128, 128, 128, 32384, 23936, 128, 128, 128, 128, 128, 128, 128, 384, 8832, 128, 22400, 10624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 30592, 128, 512, 32640, 128, 128, 32640, 31360, 128, 128, 0, 3200, 128, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32640, 128, 5376, 128, 128, 128, 128, 128, 32384, 1664, 32768, 256, 128, 128, 128, 128, 18816, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 32768, 128, 32768, 32768, 128, 128, 128, 128, 2560, 128, 128, 128, 128, 128, 16256, 9984, 3072, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 768, 128, 512, 128, 128, 128, 128, 128, 32640, 128, 128, 384, 8576, 128, 640, 32768, 32768, 22272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[724, 4058, 0, 0, 730, 6, 0, 4078, 4069, 4070, 0, 22, 36, 22, 2, 0, 51, 0, 54, 0, 24, 4070, 4056, 4067, 0, 62, 0, 4047, 27, 0, 338, 0, 26, 0, 0, 118, 16, 0, 30, 282, 4043, 3404, 4073, 4034, 4, 32, 45, 0, 1, 0, 96, 8, 0, 0, 3692, 4023, 3947, 0, 2, 23, 0, 0, 45, 4069, 3401, 0, 69, 10, 3925, 0, 0, 6, 7, 4061, 18, 4057, 0, 0, 39, 4, 150, 24, 12, 4057, 0, 4035, 0, 35, 0, 41, 0, 18, 3932, 38, 4051, 3, 0, 0, 0, 0, 0, 39, 26, 3, 4023, 0, 32, 4069, 3974, 0, 0, 47, 0, 4047, 0, 36, 0, 0, 4056, 0, 185, 4036, 3993, 71, 3816, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[896, 4096, 0, 0, 0, 2, 0, 3446, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3422, 4091, 4096, 4091, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4096, 4095, 0, 0, 0, 0, 5, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3934, 263, 4093, 0, 0, 0, 0, 4, 0, 4082, 0, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 16, 13, 4096, 0, 0, 0, 1, 4096, 325, 0, 16, 3, 4096, 0, 0, 4095, 4096, 0, 4089, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[166, 4096, 0, 0, 37, 0, 0, 4083, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2860, 4096, 4096, 4089, 0, 0, 4, 4096, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3299, 4096, 4028, 0, 0, 0, 0, 0, 0, 4087, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 2, 4096, 0, 0, 0, 0, 31, 0, 0, 3608, 0, 4096, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 0, 0, 0, 0, 37, 1, 0, 0, 1, 4096, 0, 0, 2359, 4096, 0, 0, 0, 5, 4096, 0, 0, 4, 0, 4096, 0, 2, 4052, 4095, 0, 4073, 0, 42, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1269, 4096, 0, 0, 0, 0, 0, 4094, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 4059, 4096, 3969, 0, 0, 1, 4096, 0, 0, 11, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3897, 4096, 4087, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 3227, 497, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 3330, 4096, 0, 0, 0, 0, 4096, 33, 0, 0, 0, 4096, 0, 0, 4096, 4093, 0, 4096, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 35, 0, 0, 4073, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 114, 4092, 4096, 3755, 0, 0, 0, 4094, 0, 0, 9, 9, 0, 13, 0, 0, 0, 0, 0, 0, 4096, 4096, 4072, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4057, 4095, 4096, 0, 0, 0, 0, 8, 0, 4096, 4096, 0, 0, 0, 4095, 130, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 137, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 4087, 0, 0, 0, 4, 7, 0, 0, 0, 0, 4096, 0, 0, 612, 4096, 0, 0, 0, 50, 4085, 3382, 0, 5, 0, 4096, 8, 0, 4096, 4096, 0, 4093, 1, 28, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 132, 8, 0, 4095, 4096, 4096, 0, 0, 0, 1, 2, 0, 0, 0, 0, 2, 0, 4093, 4092, 4096, 0, 0, 0, 4096, 0, 0, 55, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4086, 4096, 4093, 0, 0, 0, 17, 0, 7, 2, 0, 0, 0, 4072, 4095, 4072, 4, 0, 0, 2, 0, 0, 4084, 4096, 0, 11, 0, 4066, 0, 0, 14, 0, 4082, 0, 4096, 0, 0, 1, 11, 2, 0, 0, 4096, 1, 4096, 3, 1, 0, 0, 0, 0, 4084, 0, 4092, 0, 0, 0, 0, 0, 4, 0, 4, 0, 4092, 12, 0, 4090, 4082, 0, 0, 0, 55, 4094, 0, 0, 0, 0, 4094, 3, 16, 4064, 4037, 0, 3979, 2, 12, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4094, 21, 252, 0, 0, 16, 4095, 4094, 4091, 0, 0, 0, 0, 0, 0, 1, 33, 5, 0, 3817, 4093, 4088, 4095, 0, 2, 0, 4079, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1, 0, 4095, 3937, 4094, 4093, 10, 0, 0, 0, 0, 0, 0, 0, 2, 12, 4094, 4052, 4091, 0, 0, 0, 68, 0, 0, 3971, 4089, 0, 0, 1, 4034, 157, 1, 0, 2, 4090, 0, 4084, 0, 0, 0, 0, 154, 7, 0, 3609, 368, 3975, 4, 0, 0, 0, 128, 0, 4096, 1, 4089, 2, 0, 3, 1, 0, 1, 7, 0, 0, 4086, 5, 0, 139, 4081, 0, 0, 9, 0, 4062, 16, 7, 0, 0, 4092, 4, 0, 4096, 4071, 5, 4041, 43, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 4096, 4063, 0, 0, 9, 4096, 0, 0, 4, 0, 0, 3, 0, 0, 0, 0, 0, 1, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4087, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4040, 4096, 0, 0, 0, 0, 4096, 108, 0, 0, 0, 4096, 15, 0, 4096, 4095, 0, 4096, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[896, 4096, 0, 0, 0, 2, 0, 3446, 3660, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3422, 4091, 2791, 4091, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 4096, 4096, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4096, 4095, 0, 0, 0, 0, 5, 0, 4096, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 263, 4093, 0, 0, 0, 0, 4, 0, 4082, 0, 4096, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 16, 13, 4096, 0, 0, 0, 4096, 1305, 448, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3648, 325, 0, 16, 3, 0, 0, 0, 4095, 4096, 0, 4089, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16196, 18493, 16431, 16584, 16456, 16544, 14093, 16275] avg=16384.0000 max=18493 max_rank=1 min=14093 min_rank=6 imbalance_ratio=1.128723 max_load_reduction=112579 max_load_reduction_pct=0.858910 moved_to_echo=63368 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[724, 4058, 0, 0, 730, 6, 0, 4078, 3617, 4070, 0, 22, 36, 22, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 54, 0, 24, 4070, 2761, 4067, 0, 62, 0, 4047, 27, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 118, 16, 0, 30, 282, 4043, 3404, 4073, 4034, 4, 32, 45, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 96, 8, 0, 0, 3692, 4023, 3947, 0, 2, 23, 0, 0, 45, 4069, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3401, 0, 69, 10, 3925, 0, 0, 6, 7, 4061, 18, 4057, 0, 0, 39, 4, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 150, 24, 12, 4057, 0, 4035, 0, 35, 0, 41, 0, 18, 3932, 38, 4051, 3, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 26, 3, 4023, 0, 32, 4069, 3974, 0, 0, 47, 4056, 1295, 447, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3600, 0, 36, 0, 0, 0, 0, 185, 4036, 3993, 71, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17365, 15501, 16163, 16131, 15674, 16474, 18027, 15737] avg=16384.0000 max=18027 max_rank=6 min=15501 min_rank=1 imbalance_ratio=1.100281 max_load_reduction=113045 max_load_reduction_pct=0.862465 moved_to_echo=65912 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1269, 4096, 0, 0, 0, 0, 0, 4094, 3664, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 4059, 2791, 3969, 0, 0, 1, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3897, 4096, 4087, 0, 0, 0, 0, 0, 0, 4096, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 12, 4096, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3227, 497, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 3330, 4096, 0, 0, 0, 4096, 1305, 448, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3648, 33, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17210, 15115, 16443, 16395, 16468, 16084, 17389, 15968] avg=16384.0000 max=17389 max_rank=6 min=15115 min_rank=1 imbalance_ratio=1.061340 max_load_reduction=113683 max_load_reduction_pct=0.867332 moved_to_echo=65909 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[166, 4096, 0, 0, 37, 0, 0, 4083, 3664, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2860, 4096, 2791, 4089, 0, 0, 4, 4096, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 4096, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3299, 4096, 4028, 0, 0, 0, 0, 0, 0, 4087, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 2, 4096, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 3608, 0, 4096, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1, 0, 0, 1, 4096, 0, 0, 2359, 4096, 0, 0, 0, 4096, 1305, 448, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3648, 0, 0, 4, 0, 0, 0, 2, 4052, 4095, 0, 4073, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16142, 17950, 16436, 15729, 16453, 15986, 16455, 15921] avg=16384.0000 max=17950 max_rank=1 min=15729 min_rank=3 imbalance_ratio=1.095581 max_load_reduction=113122 max_load_reduction_pct=0.863052 moved_to_echo=64815 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 35, 0, 0, 4073, 3664, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 114, 4092, 2791, 3755, 0, 0, 0, 4094, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4096, 4096, 4072, 4096, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4057, 4095, 4096, 0, 0, 0, 0, 8, 0, 4096, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 130, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 137, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 4087, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 0, 0, 0, 0, 4096, 0, 0, 612, 4096, 0, 0, 0, 4096, 1305, 447, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 3638, 3382, 0, 5, 0, 0, 8, 0, 4096, 4096, 0, 4093, 1, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15946, 14888, 16426, 16573, 16585, 16578, 14679, 19397] avg=16384.0000 max=19397 max_rank=7 min=14679 min_rank=6 imbalance_ratio=1.183899 max_load_reduction=111675 max_load_reduction_pct=0.852013 moved_to_echo=67239 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 132, 8, 0, 4095, 3664, 4096, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 2788, 4096, 0, 0, 0, 4096, 0, 0, 55, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4086, 4096, 4093, 0, 0, 0, 17, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 0, 0, 4072, 4095, 4072, 4, 0, 0, 2, 0, 0, 4084, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 4066, 0, 0, 14, 0, 4082, 0, 4096, 0, 0, 1, 11, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 1, 4096, 3, 1, 0, 0, 0, 0, 4084, 0, 4092, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 4092, 12, 0, 4090, 4082, 0, 0, 0, 4094, 1304, 448, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 3646, 0, 0, 0, 0, 0, 3, 16, 4064, 4037, 0, 3979, 2, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16092, 15134, 16433, 16557, 16449, 16447, 18146, 15814] avg=16384.0000 max=18146 max_rank=6 min=15134 min_rank=1 imbalance_ratio=1.107544 max_load_reduction=112926 max_load_reduction_pct=0.861557 moved_to_echo=66856 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 21, 252, 0, 0, 16, 4095, 3662, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 33, 5, 0, 3817, 4093, 2786, 4095, 0, 2, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1, 0, 4095, 3937, 4094, 4093, 10, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 12, 4094, 4052, 4091, 0, 0, 0, 68, 0, 0, 3971, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 1, 4034, 157, 1, 0, 2, 4090, 0, 4084, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 7, 0, 3609, 368, 3975, 4, 0, 0, 0, 128, 0, 4096, 1, 4089, 2, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 1, 7, 0, 0, 4086, 5, 0, 139, 4081, 0, 0, 9, 4092, 1302, 444, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3618, 16, 7, 0, 0, 0, 4, 0, 4096, 4071, 5, 4041, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16231, 18911, 16299, 16509, 16530, 16505, 14186, 15901] avg=16384.0000 max=18911 max_rank=1 min=14186 min_rank=6 imbalance_ratio=1.154236 max_load_reduction=112161 max_load_reduction_pct=0.855721 moved_to_echo=63122 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 3664, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 2791, 4063, 0, 0, 9, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4087, 0, 0, 0, 0, 0, 0, 4096, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4040, 4096, 0, 0, 0, 4096, 1305, 448, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3648, 108, 0, 0, 0, 0, 15, 0, 4096, 4095, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15890, 15080, 16441, 16594, 16457, 16454, 18097, 16059] avg=16384.0000 max=18097 max_rank=6 min=15080 min_rank=1 imbalance_ratio=1.104553 max_load_reduction=112975 max_load_reduction_pct=0.861931 moved_to_echo=67067 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32128, 0, 128, 128, 32512, 384, 128, 128, 128, 32768, 128, 32768, 0, 0, 128, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 31360, 32768, 32512, 128, 128, 128, 128, 128, 128, 32640, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 18688, 32640, 0, 0, 128, 32768, 10496, 3584, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[384, 128, 128, 30720, 1280, 32640, 128, 128, 0, 128, 256, 128, 32640, 128, 32768, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 10496, 32768, 22400, 32256, 128, 128, 128, 32768, 128, 0, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3072, 32768, 128, 256, 1024, 128, 128, 32128, 29312, 32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 29184, 3968, 128, 128, 128, 0, 128, 256, 32640, 32640, 128, 32384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 384, 32768, 32000, 32768, 32768, 128, 128, 128, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 14, 0, 4096, 0, 0, 0, 0, 0, 3841, 4091, 0, 0, 5, 7, 4081, 0, 4088, 0, 4096, 0, 4, 0, 0, 0, 4085, 1, 18, 4096, 0, 4086, 0, 0, 0, 0, 0, 4085, 4093, 0, 0, 0, 0, 4068, 11, 7, 0, 0, 0, 0, 0, 0, 4, 0, 4095, 0, 3744, 0, 0, 0, 4072, 4095, 0, 1, 19, 1123, 0, 0, 4077, 2, 1, 10, 0, 0, 1, 0, 0, 4093, 4088, 14, 4089, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 3, 0, 0, 4094, 0, 188, 0, 6, 0, 27, 8, 0, 0, 0, 0, 8, 4092, 2, 4096, 0, 4096, 0, 0, 4094, 4061, 0, 0, 53, 3, 4094, 1, 0, 4089, 0, 4096, 0, 1, 0, 3366] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 92, 0, 4094, 0, 0, 0, 12, 0, 4095, 4055, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4082, 0, 1, 4096, 0, 3404, 0, 0, 2, 28, 0, 4096, 4065, 0, 0, 7, 0, 3979, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4026, 0, 0, 0, 4096, 4096, 0, 1, 0, 299, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 32, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 553, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4096, 0, 4096, 0, 4096, 0, 0, 4090, 3698, 0, 0, 4094, 0, 4096, 24, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 1036, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 2773, 0, 0, 0, 0, 0, 4095, 0, 10, 4096, 0, 1843, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 389, 37, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3720, 0, 0, 0, 4096, 4096, 0, 0, 0, 3496, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3910, 0, 4096, 4096, 0, 0, 0, 1826, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 71, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 2, 4096, 4095, 0, 0, 4066, 0, 4096, 12, 0, 4092, 0, 4096, 0, 5, 0, 1373] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 1, 0, 2149, 4096, 0, 14, 0, 0, 4096, 0, 4096, 0, 4096, 0, 1, 0, 0, 0, 4095, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 4094, 4096, 0, 2, 969, 0, 3447, 326, 5, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 0, 1341, 2, 0, 0, 4096, 4094, 1, 16, 0, 3631, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3804, 6, 4096, 4096, 0, 0, 0, 3845, 0, 0, 4089, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 114, 4096, 4093, 0, 0, 351, 0, 4094, 123, 0, 4061, 0, 4096, 0, 6, 15, 344] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 72, 4075, 66, 1, 192, 0, 1, 4076, 4029, 230, 0, 45, 0, 4067, 19, 4070, 34, 3125, 131, 0, 0, 2, 1, 4004, 27, 204, 4046, 6, 3866, 0, 1, 0, 0, 0, 4056, 4073, 0, 8, 0, 0, 3775, 162, 62, 0, 94, 0, 33, 0, 23, 7, 1022, 4060, 26, 125, 0, 0, 0, 4028, 4062, 0, 0, 0, 3879, 0, 33, 3730, 42, 0, 62, 29, 0, 5, 34, 28, 4038, 4063, 25, 4036, 3912, 0, 0, 0, 1, 0, 24, 4041, 17, 92, 0, 0, 4066, 2, 0, 0, 2, 34, 1, 0, 0, 58, 12, 123, 30, 4054, 40, 4007, 21, 4033, 2, 471, 4055, 3780, 9, 32, 3146, 23, 4074, 705, 57, 4058, 0, 4076, 0, 4, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 32, 0, 2571, 0, 0, 0, 1, 0, 3425, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 3918, 0, 0, 0, 0, 0, 3971, 0, 160, 4096, 0, 3948, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 3703, 1, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 0, 4049, 0, 0, 0, 4095, 4096, 3, 1, 0, 2627, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 5, 4096, 4096, 0, 0, 42, 0, 4096, 110, 0, 4096, 0, 4096, 0, 172, 0, 4039] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 3361, 0, 0, 0, 0, 48, 4092, 4096, 2, 0, 0, 0, 4089, 0, 4090, 0, 4053, 13, 0, 0, 0, 13, 3797, 0, 2, 4091, 0, 3486, 0, 23, 1, 0, 0, 4083, 4096, 0, 0, 21, 186, 4096, 521, 7, 0, 0, 0, 0, 0, 0, 14, 0, 4085, 0, 3805, 0, 0, 7, 4093, 4094, 0, 0, 0, 4054, 0, 2, 3796, 0, 10, 1, 1, 3, 10, 0, 0, 4095, 4095, 0, 4096, 4096, 231, 0, 0, 3, 0, 0, 4092, 0, 0, 0, 0, 4056, 0, 116, 0, 0, 0, 0, 9, 0, 5, 5, 0, 0, 4089, 3, 4078, 4, 4095, 1, 33, 4072, 4083, 3, 5, 455, 0, 4091, 6, 0, 4074, 0, 4085, 0, 0, 1, 753] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 0, 0, 4048, 0, 0, 0, 0, 0, 3397, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 20, 4096, 0, 4093, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4075, 0, 0, 0, 4096, 4096, 0, 0, 0, 2828, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 38, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 12, 4096, 4037, 0, 0, 1360, 0, 4096, 0, 0, 4096, 0, 4096, 0, 90, 0, 387] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 1036, 4096, 0, 0, 0, 0, 4096, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 801, 0, 2773, 0, 0, 0, 0, 0, 4095, 0, 10, 4096, 0, 1843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 389, 37, 4094, 1, 0, 0, 0, 0, 3295, 486, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3720, 0, 0, 0, 4096, 4096, 0, 0, 0, 856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3496, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1474, 3910, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1826, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 71, 0, 2541, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 2, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4066, 0, 0, 12, 0, 4092, 0, 1069, 0, 5, 0, 1373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14365, 13618, 16594, 16864, 17072, 17451, 16300, 18808] avg=16384.0000 max=18808 max_rank=7 min=13618 min_rank=1 imbalance_ratio=1.147949 max_load_reduction=112264 max_load_reduction_pct=0.856506 moved_to_echo=69631 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 92, 0, 4094, 0, 0, 0, 12, 0, 4095, 4055, 0, 0, 0, 0, 4096, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 801, 0, 4096, 0, 0, 0, 0, 0, 4082, 0, 1, 4096, 0, 3404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 28, 0, 4096, 4065, 0, 0, 7, 0, 3979, 326, 0, 0, 0, 0, 3295, 486, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4026, 0, 0, 0, 4096, 4096, 0, 1, 0, 856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1474, 4096, 32, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 553, 0, 2541, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4096, 0, 4096, 0, 4096, 0, 0, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3698, 0, 0, 4094, 0, 0, 24, 0, 4096, 0, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17485, 16480, 16385, 17169, 14092, 16104, 16286, 17071] avg=16384.0000 max=17485 max_rank=0 min=14092 min_rank=4 imbalance_ratio=1.067200 max_load_reduction=113587 max_load_reduction_pct=0.866600 moved_to_echo=63553 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 1, 0, 2149, 4096, 0, 14, 0, 0, 4096, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 801, 0, 4096, 0, 1, 0, 0, 0, 4095, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 2, 969, 0, 3447, 326, 5, 0, 0, 0, 3295, 486, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 0, 1341, 2, 0, 0, 4096, 4094, 1, 16, 0, 856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3631, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1474, 3804, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3845, 0, 0, 4089, 0, 0, 0, 0, 4090, 0, 0, 0, 2541, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 114, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 351, 0, 0, 123, 0, 4061, 0, 1069, 0, 6, 15, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15492, 17183, 16821, 14528, 17107, 19385, 16398, 14158] avg=16384.0000 max=19385 max_rank=5 min=14158 min_rank=7 imbalance_ratio=1.183167 max_load_reduction=111687 max_load_reduction_pct=0.852104 moved_to_echo=67048 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 72, 4075, 66, 1, 192, 0, 1, 4076, 4029, 230, 0, 45, 0, 4067, 1031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 792, 34, 3125, 131, 0, 0, 2, 1, 4004, 27, 204, 4046, 6, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4056, 4073, 0, 8, 0, 0, 3775, 162, 62, 0, 94, 0, 3278, 489, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 23, 7, 1022, 4060, 26, 125, 0, 0, 0, 4028, 4062, 0, 0, 0, 846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3879, 0, 33, 3730, 42, 0, 62, 29, 0, 5, 34, 28, 1442, 4063, 25, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 0, 0, 0, 1, 0, 24, 4041, 17, 92, 0, 0, 4066, 2, 0, 0, 2531, 719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 34, 1, 0, 0, 58, 12, 123, 30, 4054, 40, 4007, 21, 4033, 2, 471, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 3780, 9, 32, 3146, 23, 0, 705, 57, 4058, 0, 1056, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17885, 16257, 16098, 14232, 17408, 15405, 16862, 16925] avg=16384.0000 max=17885 max_rank=0 min=14232 min_rank=3 imbalance_ratio=1.091614 max_load_reduction=113187 max_load_reduction_pct=0.863548 moved_to_echo=66600 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 14, 0, 4096, 0, 0, 0, 0, 0, 3841, 4091, 0, 0, 5, 7, 4081, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 799, 0, 4096, 0, 4, 0, 0, 0, 4085, 1, 18, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4093, 0, 0, 0, 0, 4068, 11, 7, 0, 0, 0, 3289, 486, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4095, 0, 3744, 0, 0, 0, 4072, 4095, 0, 1, 19, 855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 0, 0, 4077, 2, 1, 10, 0, 0, 1, 0, 0, 1473, 4088, 14, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 3, 0, 0, 4094, 0, 188, 0, 2541, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 27, 8, 0, 0, 0, 0, 8, 4092, 2, 4096, 0, 4096, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4061, 0, 0, 53, 3, 0, 1, 0, 4089, 0, 1069, 0, 1, 0, 3366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17176, 17185, 16139, 16885, 14878, 15743, 16329, 16737] avg=16384.0000 max=17185 max_rank=1 min=14878 min_rank=4 imbalance_ratio=1.048889 max_load_reduction=113887 max_load_reduction_pct=0.868889 moved_to_echo=63687 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 32, 0, 2571, 0, 0, 0, 1, 0, 3425, 4096, 0, 0, 0, 0, 4096, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 801, 0, 3918, 0, 0, 0, 0, 0, 3971, 0, 160, 4096, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 3703, 1, 0, 0, 0, 0, 3295, 486, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 0, 4049, 0, 0, 0, 4095, 4096, 3, 1, 0, 856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2627, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1474, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 2541, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 5, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 42, 0, 0, 110, 0, 4096, 0, 1069, 0, 172, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15262, 16894, 15765, 17221, 16383, 15538, 16289, 17720] avg=16384.0000 max=17720 max_rank=7 min=15262 min_rank=0 imbalance_ratio=1.081543 max_load_reduction=113352 max_load_reduction_pct=0.864807 moved_to_echo=65930 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3361, 0, 0, 0, 0, 48, 4092, 4096, 2, 0, 0, 0, 4089, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 800, 0, 4053, 13, 0, 0, 0, 13, 3797, 0, 2, 4091, 0, 3486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1, 0, 0, 4083, 4096, 0, 0, 21, 186, 4096, 521, 7, 0, 0, 0, 3290, 485, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4085, 0, 3805, 0, 0, 7, 4093, 4094, 0, 0, 0, 856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 2, 3796, 0, 10, 1, 1, 3, 10, 0, 0, 1474, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 231, 0, 0, 3, 0, 0, 4092, 0, 0, 0, 0, 4056, 0, 116, 0, 2534, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 5, 5, 0, 0, 4089, 3, 4078, 4, 4095, 1, 33, 3990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 4083, 3, 5, 455, 0, 0, 6, 0, 4074, 0, 1066, 0, 0, 1, 753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16729, 16255, 16910, 16954, 17542, 15852, 16312, 14518] avg=16384.0000 max=17542 max_rank=4 min=14518 min_rank=7 imbalance_ratio=1.070679 max_load_reduction=113530 max_load_reduction_pct=0.866165 moved_to_echo=64224 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4048, 0, 0, 0, 0, 0, 3397, 4096, 0, 0, 0, 0, 4096, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 801, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 20, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 193, 0, 0, 0, 0, 3295, 486, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4075, 0, 0, 0, 4096, 4096, 0, 0, 0, 856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2828, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1474, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 38, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2, 0, 2541, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 12, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4037, 0, 0, 1360, 0, 0, 0, 0, 4096, 0, 1069, 0, 90, 0, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16678, 17200, 16360, 17219, 16590, 15594, 16296, 15135] avg=16384.0000 max=17219 max_rank=3 min=15135 min_rank=7 imbalance_ratio=1.050964 max_load_reduction=113853 max_load_reduction_pct=0.868629 moved_to_echo=63615 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 256, 128, 30464, 128, 128, 256, 128, 128, 26112, 32768, 256, 128, 128, 128, 32768, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 128, 1152, 32768, 128, 24960, 128, 0, 128, 32768, 32768, 128, 128, 128, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 0, 32768, 32768, 0, 128, 1408, 256, 31360, 1664, 128, 0, 128, 0, 26368, 3968, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[22016, 0, 128, 32128, 128, 128, 128, 128, 128, 128, 128, 128, 11776, 32256, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 6400, 128, 30336, 256, 128, 0, 128, 128, 32256, 128, 512, 32768, 128, 28928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32000, 128, 128, 13568, 128, 0, 1024, 128, 32768, 0, 8576, 0, 384, 128, 10368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32640, 256, 0, 128, 5760, 0, 128, 32768, 128, 128, 0, 0, 32768, 128, 1024, 0, 20352, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 32768, 128, 32768, 128, 640, 32000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 2755, 86, 0, 0, 2, 0, 0, 4095, 4096, 4093, 0, 0, 0, 0, 4082, 3810, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 2, 75, 4096, 3960, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 201, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4048, 4096, 0, 0, 4096, 0, 0, 0, 3307, 0, 0, 4096, 16, 0, 445, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4088, 1, 0, 4096, 4025, 0, 3507, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 31, 0, 14, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 2409, 0, 0, 6, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 3890, 56, 0, 76, 68, 0, 0, 4080, 4096, 4046, 0, 0, 0, 0, 3974, 2968, 0, 0, 0, 4096, 0, 0, 34, 0, 0, 8, 0, 4096, 3516, 4095, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 3950, 49, 0, 612, 21, 0, 4096, 0, 0, 0, 52, 0, 0, 4096, 4096, 0, 0, 3844, 1, 0, 165, 3716, 0, 0, 4096, 0, 0, 1061, 0, 0, 4096, 0, 0, 0, 4088, 4096, 0, 3924, 0, 0, 4096, 3, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 5, 0, 338, 0, 0, 0, 0, 11, 0, 0, 4066, 0, 4044, 1, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 527, 0, 0, 86, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 1531, 116, 0, 0, 266, 0, 0, 4096, 4096, 4092, 0, 0, 1, 0, 3998, 2246, 0, 0, 0, 4096, 0, 8, 0, 0, 10, 126, 2461, 4093, 4030, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 985, 76, 0, 196, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3900, 4089, 0, 8, 1812, 0, 0, 0, 3968, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 11, 0, 3985, 0, 0, 0, 61, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 105, 40, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 3962, 4096, 0, 0, 0, 0, 3700, 0, 0, 3378, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4057, 18, 0, 4079, 0, 69, 1497, 2880, 0, 3, 3952, 3990, 4025, 10, 61, 0, 18, 401, 2386, 14, 0, 105, 4069, 0, 0, 40, 6, 104, 0, 1, 4055, 1946, 4011, 9, 39, 4054, 5, 0, 1, 0, 0, 41, 4034, 3893, 2462, 0, 4058, 0, 23, 3988, 27, 34, 0, 218, 0, 0, 3990, 4074, 2, 0, 3650, 0, 0, 0, 4080, 34, 0, 4045, 0, 0, 0, 48, 55, 4048, 0, 89, 0, 4001, 4067, 0, 4088, 126, 22, 4062, 666, 28, 1473, 0, 26, 29, 0, 0, 1, 46, 4069, 2, 0, 3980, 76, 28, 0, 0, 0, 45, 2, 0, 41, 6, 3964, 1, 4067, 0, 120, 0, 0, 2, 26, 316, 4082, 91, 0, 7, 1, 0, 37, 78, 2084, 514] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4095, 0, 0, 4094, 990, 6, 0, 2011, 3, 0, 4095, 4096, 4096, 1, 18, 1, 2, 2596, 1, 4, 10, 0, 4091, 9, 0, 2684, 7, 0, 0, 62, 4094, 1694, 4090, 0, 0, 4046, 0, 0, 0, 6, 0, 0, 4094, 0, 0, 4, 3996, 123, 0, 4096, 1, 4, 0, 844, 0, 0, 3836, 4085, 0, 0, 3764, 1, 42, 578, 3469, 0, 0, 4090, 0, 2, 4, 0, 0, 4092, 88, 11, 0, 4091, 4090, 0, 4096, 0, 0, 4095, 1893, 0, 4090, 0, 0, 1, 0, 9, 0, 0, 3922, 0, 0, 3987, 0, 11, 293, 0, 0, 0, 0, 0, 2, 5, 4087, 6, 4096, 0, 0, 0, 0, 77, 0, 3899, 4037, 0, 0, 6, 0, 1, 0, 0, 152, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 1492, 370, 0, 2, 0, 0, 0, 4070, 4096, 4095, 0, 0, 5, 0, 4090, 3493, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 38, 4096, 3210, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4050, 85, 117, 0, 0, 38, 0, 4081, 0, 0, 0, 9, 0, 0, 4093, 4096, 0, 0, 4096, 0, 0, 116, 3930, 0, 0, 3923, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4065, 4082, 0, 4094, 0, 0, 4096, 3951, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 4096, 18, 0, 4096, 0, 0, 24, 0, 26, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 31, 0, 4096, 4092, 0, 0, 0, 12, 63, 0, 0, 3772, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 6, 0, 4046, 420, 0, 0, 581, 0, 0, 4006, 4096, 4072, 0, 15, 0, 0, 4088, 134, 0, 0, 0, 4092, 0, 59, 0, 4, 0, 0, 18, 4096, 3326, 4093, 0, 3, 4092, 0, 0, 0, 5, 0, 4, 4094, 545, 3281, 0, 148, 0, 0, 4086, 6, 3, 0, 0, 0, 0, 4084, 4094, 0, 0, 3222, 0, 0, 0, 4096, 0, 0, 4093, 2, 0, 1298, 0, 0, 4092, 0, 0, 0, 4079, 4094, 0, 4054, 0, 0, 4094, 62, 2, 3379, 3, 0, 1, 0, 1, 5, 0, 4095, 0, 17, 4047, 0, 11, 0, 4, 17, 0, 1, 0, 5, 1, 4096, 0, 4094, 0, 0, 0, 0, 0, 1, 4086, 4085, 26, 2, 0, 0, 69, 1, 0, 4086, 58] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 2803, 192, 0, 102, 32, 0, 0, 4095, 4096, 4096, 0, 0, 9, 0, 3916, 3937, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 29, 4096, 1041, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 1, 10, 0, 3202, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 3629, 0, 0, 4096, 0, 0, 1954, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4091, 2, 0, 4096, 3983, 0, 235, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3584, 0, 0, 18, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2755, 86, 0, 0, 2, 0, 0, 4095, 3926, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 3810, 0, 0, 0, 2949, 0, 0, 1, 0, 0, 2, 75, 4096, 3960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 201, 0, 0, 0, 0, 541, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4048, 4096, 0, 0, 4096, 0, 0, 0, 187, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3307, 0, 0, 4096, 16, 0, 445, 0, 0, 3835, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 3276, 4025, 0, 3507, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 31, 0, 14, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 2596, 1147, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 2409, 0, 0, 6, 0, 4096, 1500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14957, 18975, 13103, 16693, 15795, 18993, 16353, 16203] avg=16384.0000 max=18993 max_rank=5 min=13103 min_rank=2 imbalance_ratio=1.159241 max_load_reduction=112079 max_load_reduction_pct=0.855095 moved_to_echo=67344 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1531, 116, 0, 0, 266, 0, 0, 4096, 3926, 4092, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 2246, 0, 0, 0, 2949, 0, 8, 0, 0, 10, 126, 2461, 4093, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 985, 76, 0, 196, 0, 541, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3900, 4089, 0, 8, 1812, 0, 0, 0, 187, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 4096, 0, 0, 1, 0, 0, 3835, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3276, 11, 0, 3985, 0, 0, 0, 61, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 105, 40, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 2597, 1147, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 4096, 0, 0, 0, 0, 3700, 0, 0, 3378, 2, 4096, 1499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14028, 19921, 14160, 14262, 15996, 15519, 16453, 20733] avg=16384.0000 max=20733 max_rank=7 min=14028 min_rank=0 imbalance_ratio=1.265442 max_load_reduction=110339 max_load_reduction_pct=0.841820 moved_to_echo=68701 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3890, 56, 0, 76, 68, 0, 0, 4080, 3926, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 2968, 0, 0, 0, 2949, 0, 0, 34, 0, 0, 8, 0, 4096, 3516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 3950, 49, 0, 612, 21, 541, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 52, 0, 0, 4096, 4096, 0, 0, 3844, 1, 0, 165, 187, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3716, 0, 0, 4096, 0, 0, 1061, 0, 0, 3835, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3924, 0, 0, 3276, 3, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 5, 0, 338, 0, 0, 0, 0, 11, 0, 0, 4066, 0, 4044, 1, 2596, 1147, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 527, 0, 0, 86, 11, 4096, 1500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16142, 17545, 17532, 16707, 16796, 15356, 16581, 14413] avg=16384.0000 max=17545 max_rank=1 min=14413 min_rank=7 imbalance_ratio=1.070862 max_load_reduction=113527 max_load_reduction_pct=0.866142 moved_to_echo=63146 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 18, 0, 4079, 0, 69, 1497, 2880, 0, 3, 3952, 3823, 4025, 10, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 401, 2386, 14, 0, 105, 2926, 0, 0, 40, 6, 104, 0, 1, 4055, 1946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 9, 39, 4054, 5, 0, 1, 0, 0, 41, 4034, 3893, 2462, 0, 4058, 0, 539, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3988, 27, 34, 0, 218, 0, 0, 3990, 4074, 2, 0, 3650, 0, 0, 0, 188, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 34, 0, 4045, 0, 0, 0, 48, 55, 3785, 0, 89, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 126, 22, 3244, 666, 28, 1473, 0, 26, 29, 0, 0, 1, 46, 4069, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 76, 28, 0, 0, 0, 45, 2, 0, 41, 6, 3964, 1, 4067, 0, 2579, 1143, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 2, 26, 316, 4082, 91, 0, 7, 1, 0, 37, 78, 2084, 514, 4057, 1488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[20417, 12002, 23221, 16361, 16137, 13820, 16211, 12903] avg=16384.0000 max=23221 max_rank=2 min=12002 min_rank=1 imbalance_ratio=1.417297 max_load_reduction=107851 max_load_reduction_pct=0.822838 moved_to_echo=59071 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1492, 370, 0, 2, 0, 0, 0, 4070, 3926, 4095, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3493, 0, 0, 0, 2949, 0, 5, 0, 0, 0, 0, 38, 4096, 3210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4050, 85, 117, 0, 0, 38, 541, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 9, 0, 0, 4093, 4096, 0, 0, 4096, 0, 0, 116, 187, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3930, 0, 0, 3923, 0, 0, 0, 0, 0, 3835, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3276, 3951, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 24, 0, 26, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 2587, 1147, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 4096, 4092, 0, 0, 0, 12, 63, 0, 0, 3772, 0, 4096, 1495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13960, 17881, 13088, 16848, 15753, 19536, 16349, 17657] avg=16384.0000 max=19536 max_rank=5 min=13088 min_rank=2 imbalance_ratio=1.192383 max_load_reduction=111536 max_load_reduction_pct=0.850952 moved_to_echo=69295 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 4046, 420, 0, 0, 581, 0, 0, 4006, 3926, 4072, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 134, 0, 0, 0, 2946, 0, 59, 0, 4, 0, 0, 18, 4096, 3326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 4092, 0, 0, 0, 5, 0, 4, 4094, 545, 3281, 0, 148, 0, 540, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 6, 3, 0, 0, 0, 0, 4084, 4094, 0, 0, 3222, 0, 0, 0, 187, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 2, 0, 1298, 0, 0, 3831, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 3276, 62, 2, 3379, 3, 0, 1, 0, 1, 5, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4047, 0, 11, 0, 4, 17, 0, 1, 0, 5, 1, 4096, 0, 4094, 0, 2595, 1146, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 4085, 26, 2, 0, 0, 69, 1, 0, 4086, 58, 4096, 1499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17072, 14671, 16879, 15852, 17399, 14878, 16312, 18009] avg=16384.0000 max=18009 max_rank=7 min=14671 min_rank=1 imbalance_ratio=1.099182 max_load_reduction=113063 max_load_reduction_pct=0.862602 moved_to_echo=66598 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 990, 6, 0, 2011, 3, 0, 4095, 3926, 4096, 1, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2596, 1, 4, 10, 0, 2946, 9, 0, 2684, 7, 0, 0, 62, 4094, 1694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4046, 0, 0, 0, 6, 0, 0, 4094, 0, 0, 4, 3996, 123, 541, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4, 0, 844, 0, 0, 3836, 4085, 0, 0, 3764, 1, 42, 578, 187, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3469, 0, 0, 4090, 0, 2, 4, 0, 0, 3831, 88, 11, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3276, 1893, 0, 4090, 0, 0, 1, 0, 9, 0, 0, 3922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 11, 293, 0, 0, 0, 0, 0, 2, 5, 4087, 6, 4096, 0, 2593, 1145, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 3899, 4037, 0, 0, 6, 0, 1, 0, 0, 152, 0, 4095, 1497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19241, 14109, 16974, 17608, 15586, 17287, 16503, 13764] avg=16384.0000 max=19241 max_rank=0 min=13764 min_rank=7 imbalance_ratio=1.174377 max_load_reduction=111831 max_load_reduction_pct=0.853203 moved_to_echo=63140 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2803, 192, 0, 102, 32, 0, 0, 4095, 3926, 4096, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3916, 3937, 0, 0, 0, 2949, 0, 0, 0, 0, 0, 0, 29, 4096, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 1, 10, 0, 3202, 0, 541, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 187, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3629, 0, 0, 4096, 0, 0, 1954, 0, 0, 3835, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 0, 3276, 3983, 0, 235, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2596, 1147, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3584, 0, 0, 18, 0, 4096, 1500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15255, 15968, 16115, 16741, 17610, 15683, 16310, 17390] avg=16384.0000 max=17610 max_rank=4 min=15255 min_rank=0 imbalance_ratio=1.074829 max_load_reduction=113462 max_load_reduction_pct=0.865646 moved_to_echo=66993 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32640, 256, 128, 26240, 14720, 128, 24832, 128, 128, 128, 128, 128, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 32768, 128, 0, 128, 128, 0, 128, 32768, 9728, 6016, 128, 12288, 256, 4352, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 24704, 2304, 128, 1792, 5888, 128, 128, 32512, 31360, 32640, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32640, 128, 128, 0, 1152, 0, 0, 32256, 32768, 128, 128, 28672, 128, 128, 896, 1536, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 32512, 128, 128, 896, 128, 128, 128, 128, 128, 128, 128, 32640, 128, 32768, 128, 20864, 9216, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[30208, 128, 0, 32640, 128, 128, 4864, 128, 128, 30720, 128, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 27264, 19072, 128, 128, 128, 23680, 128, 128, 2816, 128, 128, 256, 2688, 32768, 22784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 28672, 32768, 128, 128, 128, 128, 10368, 128, 128, 13696, 640, 32768, 12032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 9, 0, 4028, 0, 0, 1533, 0, 0, 0, 0, 0, 4096, 3754, 4096, 0, 0, 4050, 1, 0, 0, 0, 0, 14, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 282, 0, 3580, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4087, 4096, 0, 0, 0, 0, 0, 3358, 0, 0, 3046, 4, 7, 0, 1, 4096, 4096, 0, 0, 4096, 3962, 0, 0, 0, 3451, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 4096, 0, 0, 4, 4096, 4096, 0, 0, 0, 0, 1959, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 3758, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4095, 97, 3, 36, 0, 1, 3196, 0, 0, 0, 0, 0, 4089, 499, 4096, 0, 0, 3956, 296, 0, 0, 0, 0, 0, 1, 0, 3, 0, 4095, 3, 0, 0, 0, 1, 0, 4096, 0, 9, 4096, 4096, 0, 0, 0, 4033, 1, 3418, 4096, 4096, 4075, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4052, 4096, 134, 0, 0, 0, 0, 3216, 0, 0, 3993, 1349, 194, 0, 11, 4096, 3906, 0, 0, 4096, 2104, 0, 0, 0, 4096, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 12, 0, 2006, 4092, 0, 0, 124, 4096, 4096, 0, 0, 0, 0, 1, 0, 0, 91, 0, 2, 0, 0, 0, 152, 0, 0, 0, 4096, 4081, 0, 38, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4092, 0, 0, 4094, 0, 9, 0, 15, 1, 2, 0, 4, 4090, 4096, 4087, 0, 4, 0, 252, 0, 2, 0, 45, 0, 759, 0, 0, 0, 4093, 0, 0, 0, 0, 1, 0, 4084, 17, 66, 4093, 4096, 0, 1, 1, 1556, 4, 4087, 4094, 4053, 4094, 0, 1, 4093, 0, 0, 4081, 0, 0, 4096, 4095, 4096, 0, 0, 0, 2, 0, 3191, 0, 0, 4096, 43, 4, 0, 52, 4058, 4092, 2, 0, 4084, 1156, 0, 0, 1, 4088, 0, 4095, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4026, 4081, 10, 0, 910, 4038, 4082, 0, 12, 0, 0, 142, 0, 0, 17, 0, 51, 0, 0, 0, 549, 0, 0, 2, 4071, 3759, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4073, 0, 0, 1529, 0, 1, 136, 0, 0, 0, 0, 0, 4096, 908, 4096, 0, 0, 4096, 389, 0, 0, 0, 0, 307, 62, 0, 47, 0, 4063, 39, 0, 1, 0, 0, 6, 4096, 3, 0, 4080, 4096, 0, 0, 22, 4095, 21, 4084, 4096, 4096, 4063, 0, 0, 4095, 0, 0, 4095, 0, 0, 4096, 4069, 4096, 11, 3, 0, 0, 0, 1461, 0, 0, 4041, 47, 0, 0, 3, 4096, 4029, 0, 0, 4096, 0, 0, 0, 0, 3839, 0, 4019, 0, 10, 0, 0, 0, 0, 0, 0, 0, 3543, 4096, 0, 0, 0, 4092, 4081, 0, 0, 0, 0, 764, 0, 0, 51, 0, 0, 0, 0, 0, 3618, 0, 2, 0, 4093, 3962, 0, 63, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4076, 237, 0, 4057, 23, 543, 771, 10, 17, 200, 0, 24, 4023, 4090, 4050, 0, 42, 2152, 1973, 79, 1, 8, 0, 0, 0, 21, 0, 72, 4011, 5, 18, 4, 22, 0, 0, 3982, 0, 159, 4083, 4061, 0, 39, 0, 1379, 168, 3963, 4023, 4062, 3876, 13, 58, 4069, 0, 1, 4075, 0, 20, 4053, 4080, 3815, 0, 0, 0, 4, 154, 2890, 125, 0, 3383, 0, 1427, 63, 0, 4068, 3344, 12, 38, 4059, 0, 138, 3, 45, 2881, 1, 4087, 0, 79, 0, 6, 33, 0, 8, 0, 0, 4043, 3940, 0, 0, 1345, 3871, 3961, 15, 24, 14, 0, 1339, 0, 41, 0, 7, 1, 28, 391, 0, 0, 0, 0, 26, 4071, 465, 0, 0, 46, 0, 1, 7, 80] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 131, 2, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3445, 4091, 0, 0, 3482, 2, 0, 0, 0, 0, 260, 83, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 4086, 4096, 0, 0, 5, 1740, 17, 2688, 4096, 4096, 4096, 0, 0, 4096, 2, 0, 4095, 0, 0, 4091, 4088, 4067, 12, 0, 0, 0, 0, 3545, 7, 0, 315, 3105, 0, 0, 1, 3793, 4095, 0, 0, 4096, 655, 3, 0, 0, 4001, 0, 4093, 0, 0, 2, 0, 0, 0, 0, 1, 0, 606, 4094, 0, 0, 17, 4087, 4075, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 583, 0, 702, 0, 11, 0, 4096, 3851, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 3922, 0, 0, 4086, 2, 0, 260, 0, 1, 0, 1, 4, 4088, 1860, 4092, 0, 1, 1921, 17, 0, 0, 0, 94, 7, 3605, 5, 5, 0, 3959, 0, 0, 7, 0, 5, 0, 4084, 0, 0, 4095, 4055, 34, 11, 0, 24, 0, 12, 4093, 4095, 4086, 77, 0, 4086, 0, 3, 4092, 45, 8, 3953, 4096, 4067, 205, 0, 2, 0, 4, 3147, 0, 0, 4096, 316, 0, 0, 4, 4041, 3951, 0, 0, 4076, 3754, 0, 0, 0, 4085, 1, 4031, 0, 0, 0, 0, 0, 0, 0, 37, 16, 1772, 4070, 0, 3, 0, 3781, 4095, 0, 3, 0, 59, 4017, 0, 1, 170, 0, 89, 0, 0, 0, 38, 0, 49, 6, 4091, 4096, 3, 0, 0, 5, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 2737, 0, 0, 657, 0, 0, 0, 0, 0, 4096, 1756, 4096, 0, 0, 3150, 2, 0, 0, 0, 0, 157, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 2300, 0, 4096, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 3367, 0, 0, 4089, 200, 11, 0, 3, 4096, 4096, 0, 0, 4096, 522, 0, 0, 0, 4008, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1926, 0, 0, 4, 0, 0, 0, 0, 0, 256, 0, 0, 0, 4096, 3627, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 9, 0, 4028, 0, 0, 1533, 0, 0, 0, 0, 0, 4096, 3754, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 1, 0, 0, 0, 0, 14, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2980, 0, 0, 0, 282, 0, 3580, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3358, 0, 0, 3046, 4, 7, 0, 1, 3658, 4096, 0, 0, 0, 3962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3451, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 4017, 1942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 4096, 4096, 0, 0, 0, 0, 1959, 0, 0, 0, 0, 0, 1116, 520, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 3758, 0, 0, 0, 0, 0, 0, 0, 4096, 3893, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19150, 16556, 15034, 16417, 18132, 13576, 16325, 15882] avg=16384.0000 max=19150 max_rank=0 min=13576 min_rank=5 imbalance_ratio=1.168823 max_load_reduction=111922 max_load_reduction_pct=0.853897 moved_to_echo=63915 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 97, 3, 36, 0, 1, 3196, 0, 0, 0, 0, 0, 4089, 499, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 296, 0, 0, 0, 0, 0, 1, 0, 3, 0, 4095, 3, 0, 4096, 4096, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 9, 0, 2980, 0, 0, 0, 4033, 1, 3418, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4052, 4096, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3216, 0, 0, 3993, 1349, 194, 0, 11, 3658, 3906, 0, 0, 0, 2104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 12, 0, 2006, 4017, 1942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 124, 4096, 4096, 0, 0, 0, 0, 1, 0, 0, 91, 0, 2, 1116, 520, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 4096, 4081, 0, 38, 0, 0, 0, 0, 0, 4096, 3893, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13650, 16749, 18634, 16495, 18431, 16144, 14576, 16393] avg=16384.0000 max=18634 max_rank=2 min=13650 min_rank=0 imbalance_ratio=1.137329 max_load_reduction=112438 max_load_reduction_pct=0.857834 moved_to_echo=65544 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 0, 4094, 0, 9, 0, 15, 1, 2, 0, 4, 4090, 4096, 1631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 252, 0, 2, 0, 45, 0, 759, 0, 0, 0, 4093, 0, 0, 4093, 4081, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4084, 17, 66, 0, 2980, 0, 1, 1, 1556, 4, 4087, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3191, 0, 0, 4096, 43, 4, 0, 52, 3624, 4092, 2, 0, 0, 1156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 0, 4095, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4026, 3975, 1938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 10, 0, 910, 4038, 4082, 0, 12, 0, 0, 142, 0, 0, 17, 0, 51, 1116, 518, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 0, 0, 2, 4071, 3759, 0, 1, 0, 0, 0, 0, 0, 4084, 3891, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18034, 13531, 16891, 16423, 16262, 18126, 15411, 16394] avg=16384.0000 max=18126 max_rank=5 min=13531 min_rank=1 imbalance_ratio=1.106323 max_load_reduction=112946 max_load_reduction_pct=0.861710 moved_to_echo=66193 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4073, 0, 0, 1529, 0, 1, 136, 0, 0, 0, 0, 0, 4096, 908, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 389, 0, 0, 0, 0, 307, 62, 0, 47, 0, 4063, 39, 0, 4095, 4095, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 4096, 3, 0, 0, 2980, 0, 0, 22, 4095, 21, 4084, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4069, 4096, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1461, 0, 0, 4041, 47, 0, 0, 3, 3658, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3839, 0, 4019, 0, 10, 0, 0, 0, 0, 0, 0, 0, 3543, 4017, 1942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 4081, 0, 0, 0, 0, 764, 0, 0, 51, 0, 0, 1116, 520, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3618, 0, 2, 0, 4093, 3962, 0, 63, 0, 0, 0, 0, 0, 4096, 3878, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[12377, 17395, 19404, 16380, 13239, 17370, 15158, 19749] avg=16384.0000 max=19749 max_rank=7 min=12377 min_rank=0 imbalance_ratio=1.205383 max_load_reduction=111323 max_load_reduction_pct=0.849327 moved_to_echo=65516 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4076, 237, 0, 4057, 23, 543, 771, 10, 17, 200, 0, 24, 4023, 4090, 1608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 2152, 1973, 79, 1, 8, 0, 0, 0, 21, 0, 72, 4011, 5, 18, 4069, 4075, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 22, 0, 0, 3982, 0, 159, 0, 2949, 0, 39, 0, 1379, 168, 3963, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 3876, 13, 58, 0, 0, 1, 0, 0, 20, 4053, 4080, 3815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 154, 2890, 125, 0, 3383, 0, 1427, 63, 0, 3630, 3344, 12, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 3, 45, 2881, 1, 4087, 0, 79, 0, 6, 33, 0, 8, 0, 0, 4043, 3986, 1926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 0, 1345, 3871, 3961, 15, 24, 14, 0, 1339, 0, 41, 0, 7, 1, 1112, 516, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 391, 0, 0, 0, 0, 26, 4071, 465, 0, 0, 46, 0, 1, 7, 80, 4059, 3880, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19679, 16729, 16688, 15949, 15070, 17236, 16624, 13097] avg=16384.0000 max=19679 max_rank=0 min=13097 min_rank=7 imbalance_ratio=1.201111 max_load_reduction=111393 max_load_reduction_pct=0.849861 moved_to_echo=62027 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 131, 2, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3445, 1632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3482, 2, 0, 0, 0, 0, 260, 83, 0, 0, 0, 4096, 0, 0, 4096, 4095, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 2980, 0, 0, 5, 1740, 17, 2688, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4091, 4088, 4067, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3545, 7, 0, 315, 3105, 0, 0, 1, 3388, 4095, 0, 0, 0, 655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4001, 0, 4093, 0, 0, 2, 0, 0, 0, 0, 1, 0, 606, 4017, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 17, 4087, 4075, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 1116, 519, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 583, 0, 702, 0, 11, 0, 4096, 3851, 0, 0, 0, 0, 0, 0, 0, 4096, 3883, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17349, 16317, 15633, 16398, 15111, 14663, 18342, 17259] avg=16384.0000 max=18342 max_rank=6 min=14663 min_rank=5 imbalance_ratio=1.119507 max_load_reduction=112730 max_load_reduction_pct=0.860062 moved_to_echo=65375 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3922, 0, 0, 4086, 2, 0, 260, 0, 1, 0, 1, 4, 4088, 1860, 1632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1921, 17, 0, 0, 0, 94, 7, 3605, 5, 5, 0, 3959, 0, 0, 4086, 4092, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 5, 0, 4084, 0, 0, 0, 2950, 34, 11, 0, 24, 0, 12, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4086, 77, 0, 0, 0, 3, 0, 45, 8, 3953, 4096, 4067, 205, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3147, 0, 0, 4096, 316, 0, 0, 4, 3609, 3951, 0, 0, 0, 3754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 4031, 0, 0, 0, 0, 0, 0, 0, 37, 16, 1772, 4016, 1941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 3, 0, 3781, 4095, 0, 3, 0, 59, 4017, 0, 1, 170, 0, 89, 1105, 519, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 49, 6, 4091, 4096, 3, 0, 0, 5, 0, 0, 0, 4076, 3892, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15856, 17995, 11220, 16584, 18881, 15899, 18344, 16293] avg=16384.0000 max=18881 max_rank=4 min=11220 min_rank=2 imbalance_ratio=1.152405 max_load_reduction=112191 max_load_reduction_pct=0.855949 moved_to_echo=69417 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 1, 0, 2737, 0, 0, 657, 0, 0, 0, 0, 0, 4096, 1756, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3150, 2, 0, 0, 0, 0, 157, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2980, 0, 0, 0, 2300, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3367, 0, 0, 4089, 200, 11, 0, 3, 3658, 4096, 0, 0, 0, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 4017, 1942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1926, 0, 0, 4, 0, 0, 1116, 520, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 4096, 3627, 0, 0, 0, 0, 0, 0, 0, 4096, 3893, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14977, 15800, 17568, 16426, 15946, 18058, 16292, 16005] avg=16384.0000 max=18058 max_rank=5 min=14977 min_rank=0 imbalance_ratio=1.102173 max_load_reduction=113014 max_load_reduction_pct=0.862228 moved_to_echo=66301 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32640, 512, 128, 24576, 128, 640, 6656, 128, 128, 256, 128, 128, 32768, 20480, 13056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32640, 128, 128, 2432, 32256, 32640, 128, 128, 128, 128, 14208, 0, 128, 384, 128, 256, 8960, 4224, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 22912, 2944, 128, 128, 128, 256, 768, 4608, 128, 128, 128, 32512, 128, 128, 32768, 32768, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 32512, 128, 128, 0, 128, 128, 0, 128, 128, 32640, 32768, 32512, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 1024, 0, 5376, 0, 128, 128, 32768, 27648, 128, 128, 128, 128, 128, 128, 128, 32768, 31104, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 256, 0, 23808, 128, 128, 128, 15488, 256, 25984, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[256, 128, 128, 30464, 128, 32640, 0, 128, 128, 128, 128, 0, 128, 128, 128, 20096, 32128, 15616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 256, 24192, 256, 0, 27136, 5120, 1664, 128, 128, 28928, 31616, 128, 128, 0, 12160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 4069, 4096, 4096, 0, 0, 4076, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4078, 0, 0, 4096, 0, 4096, 0, 0, 8, 116, 3681, 0, 0, 0, 2, 21, 4096, 1, 0, 0, 4096, 0, 4096, 46, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 384, 4060, 4095, 0, 0, 4096, 4096, 0, 0, 31, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4091, 0, 0, 4096, 4096, 4092, 0, 4096, 4096, 3987, 0, 13, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 0, 4096, 4096, 0, 0, 0, 213, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2674, 0, 0, 4096, 0, 4096, 0, 0, 2, 0, 2344, 0, 0, 0, 283, 0, 4096, 0, 0, 0, 4096, 0, 4094, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2849, 4096, 4096, 1, 0, 3854, 4096, 0, 0, 574, 0, 0, 0, 3733, 0, 0, 0, 0, 75, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4096, 4096, 3879, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 0, 0, 4092, 4096, 0, 0, 0, 3745, 4096, 4090, 1, 0, 4028, 0, 0, 0, 11, 0, 0, 2, 0, 0, 0, 2035, 0, 3, 4092, 14, 4096, 0, 0, 0, 3, 4095, 0, 0, 0, 130, 3, 4094, 0, 1, 0, 4096, 0, 4096, 4048, 4093, 0, 4095, 0, 0, 2, 1, 0, 12, 0, 2, 0, 1, 4003, 4082, 4096, 0, 0, 913, 4093, 0, 0, 8, 5, 3, 0, 4096, 0, 1, 2, 0, 4, 0, 5, 4085, 0, 0, 0, 21, 4090, 4093, 0, 0, 0, 2, 1, 3, 4085, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4089, 0, 0, 58, 4002, 0, 0, 4094, 4096, 1565, 2, 4090, 4094, 4095, 4, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 18, 0, 4095, 4094, 0, 12, 2, 1022, 4077, 4078, 1, 0, 4086, 1, 0, 0, 1, 55, 45, 7, 0, 2, 0, 11, 3, 0, 4094, 0, 4082, 0, 0, 0, 0, 4094, 0, 0, 7, 3386, 0, 4092, 0, 0, 95, 4089, 4, 3486, 4059, 4089, 0, 4092, 8, 0, 0, 0, 54, 0, 0, 0, 2, 0, 4077, 4015, 4090, 0, 2, 8, 4048, 2, 26, 97, 0, 7, 5, 4039, 6, 0, 18, 15, 216, 0, 0, 4086, 0, 0, 0, 10, 4091, 4096, 0, 0, 2, 4, 2, 2, 4054, 0, 3, 0, 0, 4, 0, 0, 13, 2, 0, 4095, 1, 0, 0, 4095, 0, 11, 4094, 4089, 4096, 0, 4037, 4070, 4096, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[9, 0, 2, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 1, 4096, 4096, 0, 0, 3765, 0, 2, 0, 0, 7, 0, 0, 3, 0, 0, 4091, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 4046, 0, 0, 0, 8, 1, 4096, 0, 0, 0, 4096, 0, 4096, 329, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 55, 0, 0, 0, 4091, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4071, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 4094, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[189, 0, 97, 20, 202, 68, 630, 0, 4059, 4068, 0, 0, 0, 385, 4062, 4067, 0, 0, 4076, 26, 0, 11, 7, 0, 23, 5, 0, 0, 23, 311, 4, 79, 3974, 0, 4085, 1, 31, 357, 351, 4084, 17, 0, 22, 0, 810, 3959, 21, 0, 0, 4073, 2, 3875, 3987, 4011, 77, 4050, 0, 0, 22, 0, 0, 0, 50, 364, 0, 0, 2849, 3733, 4041, 800, 5, 2078, 4038, 0, 0, 967, 21, 40, 2, 4078, 25, 12, 1, 3, 0, 11, 50, 4013, 28, 0, 0, 120, 4065, 4020, 0, 13, 0, 5, 25, 23, 3711, 0, 37, 20, 0, 96, 18, 14, 0, 0, 35, 4065, 12, 0, 505, 3559, 0, 48, 4057, 4059, 3639, 6, 4000, 4064, 3380, 36, 0, 10] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 0, 0, 0, 0, 4047, 4096, 0, 7, 0, 3332, 4096, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2871, 0, 0, 4096, 0, 4095, 0, 0, 0, 51, 4075, 3, 0, 0, 2460, 3, 4096, 0, 0, 1, 4088, 0, 4096, 1476, 4096, 0, 4094, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1514, 4096, 4096, 0, 0, 2098, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 84, 0, 0, 4096, 0, 6, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 4084, 0, 0, 4093, 4096, 2530, 0, 4096, 4096, 4096, 0, 13, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 9, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3868, 0, 0, 4096, 0, 4096, 0, 0, 6, 0, 2415, 0, 0, 0, 3602, 0, 4096, 0, 0, 0, 4096, 0, 3479, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 4096, 4096, 2, 0, 3985, 4096, 0, 0, 372, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 4096, 4096, 2665, 0, 4096, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 0, 4096, 2364, 0, 0, 0, 213, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2674, 0, 0, 4096, 4096, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3159, 0, 0, 2, 0, 2344, 0, 0, 0, 283, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1444, 0, 4094, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2849, 4096, 0, 1, 0, 3854, 2818, 0, 0, 574, 0, 0, 0, 3733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 1732, 1278, 937, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3201, 2562, 2462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3879, 0, 4096, 895, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14868, 16596, 13980, 17825, 17925, 16400, 16417, 17061] avg=16384.0000 max=17925 max_rank=4 min=13980 min_rank=2 imbalance_ratio=1.094055 max_load_reduction=113147 max_load_reduction_pct=0.863243 moved_to_echo=67803 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 0, 0, 0, 0, 0, 4096, 2364, 1, 0, 0, 4069, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4078, 0, 0, 4096, 4096, 1633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3159, 0, 0, 8, 116, 3681, 0, 0, 0, 2, 21, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1444, 0, 4096, 46, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 384, 4060, 0, 0, 0, 4096, 2818, 0, 0, 31, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 6, 0, 1732, 1278, 937, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3201, 2562, 2462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4092, 0, 4096, 895, 3987, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18724, 17981, 15180, 13778, 15492, 16331, 16412, 17174] avg=16384.0000 max=18724 max_rank=0 min=13778 min_rank=3 imbalance_ratio=1.142822 max_load_reduction=112348 max_load_reduction_pct=0.857147 moved_to_echo=65409 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 2, 0, 0, 0, 0, 0, 4096, 2364, 0, 1, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3765, 0, 2, 0, 0, 7, 0, 0, 3, 0, 0, 4091, 0, 0, 4096, 4096, 1635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3159, 0, 0, 0, 1, 4046, 0, 0, 0, 8, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1444, 0, 4096, 329, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4096, 2818, 0, 0, 55, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 1732, 1278, 937, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3201, 2562, 2461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 4096, 0, 4096, 895, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14665, 17695, 15407, 14057, 19251, 16329, 16416, 17252] avg=16384.0000 max=19251 max_rank=4 min=14057 min_rank=3 imbalance_ratio=1.174988 max_load_reduction=111821 max_load_reduction_pct=0.853127 moved_to_echo=69248 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 0, 0, 0, 4092, 2364, 0, 0, 0, 3745, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4028, 0, 0, 0, 11, 0, 0, 2, 0, 0, 0, 2035, 0, 3, 4096, 4094, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 14, 3159, 0, 0, 0, 3, 4095, 0, 0, 0, 130, 3, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1444, 0, 4096, 4048, 4093, 0, 4095, 0, 0, 2, 1, 0, 12, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4003, 4082, 0, 0, 0, 913, 2816, 0, 0, 8, 5, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 4, 0, 5, 4085, 0, 0, 0, 21, 4090, 4093, 0, 0, 1732, 1277, 937, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 3, 4085, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4089, 3200, 2562, 2462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 4002, 0, 0, 0, 0, 1565, 2, 4090, 894, 4095, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18393, 15904, 15591, 17793, 15927, 16337, 16415, 14712] avg=16384.0000 max=18393 max_rank=0 min=14712 min_rank=7 imbalance_ratio=1.122620 max_load_reduction=112679 max_load_reduction_pct=0.859673 moved_to_echo=63391 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[189, 0, 97, 20, 202, 68, 630, 0, 4059, 2345, 0, 0, 0, 385, 4062, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 26, 0, 11, 7, 0, 23, 5, 0, 0, 23, 311, 4, 79, 4059, 4057, 1613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 3148, 1, 31, 357, 351, 4084, 17, 0, 22, 0, 810, 3959, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1430, 2, 3875, 3987, 4011, 77, 4050, 0, 0, 22, 0, 0, 0, 50, 364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2849, 3733, 0, 800, 5, 2078, 2772, 0, 0, 967, 21, 40, 2, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 12, 1, 3, 0, 11, 50, 4013, 28, 0, 0, 120, 4065, 4020, 0, 13, 1723, 1266, 937, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 25, 23, 3711, 0, 37, 20, 0, 96, 18, 14, 0, 0, 35, 4065, 3181, 2549, 2428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 505, 3559, 0, 48, 0, 0, 3639, 6, 4000, 883, 3380, 36, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16124, 14294, 16775, 17868, 17345, 16381, 16207, 16078] avg=16384.0000 max=17868 max_rank=3 min=14294 min_rank=1 imbalance_ratio=1.090576 max_load_reduction=113204 max_load_reduction_pct=0.863678 moved_to_echo=66011 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[25, 0, 0, 0, 0, 0, 0, 0, 4047, 2364, 0, 7, 0, 3332, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2871, 0, 0, 4096, 4093, 1635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3159, 0, 0, 0, 51, 4075, 3, 0, 0, 2460, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1441, 0, 4096, 1476, 4096, 0, 4094, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1514, 4096, 0, 0, 0, 2098, 2818, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 4096, 0, 6, 0, 0, 4096, 4096, 0, 0, 1732, 1278, 936, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3201, 2557, 2461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4084, 0, 0, 0, 0, 2530, 0, 4096, 895, 4096, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17967, 16785, 17943, 15212, 14622, 16414, 16411, 15718] avg=16384.0000 max=17967 max_rank=0 min=14622 min_rank=4 imbalance_ratio=1.096619 max_load_reduction=113105 max_load_reduction_pct=0.862923 moved_to_echo=63165 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 0, 0, 18, 0, 4095, 2363, 0, 12, 2, 1022, 4077, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4086, 1, 0, 0, 1, 55, 45, 7, 0, 2, 0, 11, 3, 0, 4089, 4094, 1632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3149, 0, 0, 0, 0, 4094, 0, 0, 7, 3386, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 1442, 4, 3486, 4059, 4089, 0, 4092, 8, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4077, 4015, 0, 0, 2, 8, 2785, 2, 26, 97, 0, 7, 5, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 18, 15, 216, 0, 0, 4086, 0, 0, 0, 10, 4091, 4096, 0, 0, 1731, 1263, 933, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 2, 2, 4054, 0, 3, 0, 0, 4, 0, 0, 13, 2, 0, 4095, 3181, 2557, 2458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 11, 0, 0, 4096, 0, 4037, 889, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15670, 14027, 18822, 17329, 15065, 16555, 16377, 17227] avg=16384.0000 max=18822 max_rank=2 min=14027 min_rank=1 imbalance_ratio=1.148804 max_load_reduction=112250 max_load_reduction_pct=0.856400 moved_to_echo=65224 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 2364, 0, 0, 0, 9, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3868, 0, 0, 4096, 4096, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3159, 0, 0, 6, 0, 2415, 0, 0, 0, 3602, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1444, 0, 3479, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 4096, 0, 2, 0, 3985, 2818, 0, 0, 372, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 1732, 1278, 937, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3201, 2562, 2462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 2665, 0, 4096, 895, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14661, 17790, 17374, 17210, 15445, 16325, 16417, 15850] avg=16384.0000 max=17790 max_rank=1 min=14661 min_rank=0 imbalance_ratio=1.085815 max_load_reduction=113282 max_load_reduction_pct=0.864273 moved_to_echo=64037 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32640, 128, 25344, 128, 128, 384, 640, 28928, 128, 0, 128, 9984, 896, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 0, 128, 128, 256, 128, 768, 0, 32768, 18944, 128, 128, 128, 12800, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 19968, 32384, 0, 896, 128, 21248, 22528, 128, 128, 2176, 128, 128, 128, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 32384, 0, 128, 128, 0, 128, 128, 128, 128, 128, 128, 32768, 25600, 20480, 19712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 32384, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 19968, 128, 128, 32768, 32768, 13056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 640, 32128, 0, 128, 0, 0, 26624, 128, 32640, 7168, 32000, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 384, 128, 128, 32768, 128, 128, 0, 256, 32768, 32768, 128, 128, 13952, 10240, 7552, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 11648, 128, 31360, 22144, 32768, 128, 32768, 128, 0, 128, 128, 128, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True + [2026-08-30 05:03:34] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 792.4 | throughput per GPU (TFLOP/s/GPU): 420.7 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.300473E+01 | loss scale: 1.0 | grad norm: 46.616 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[14, 555, 2105, 0, 1480, 0, 724, 14, 1391, 94, 882, 969, 1774, 90, 5, 338, 282, 1407, 976, 1049, 3806, 1147, 471, 142, 6, 2177, 284, 255, 572, 64, 689, 2042, 3783, 0, 19, 973, 1610, 27, 465, 1169, 3259, 2815, 581, 371, 24, 566, 736, 1008, 571, 473, 581, 104, 17, 37, 3712, 104, 0, 2765, 3707, 1933, 148, 266, 438, 22, 512, 3813, 3, 1334, 879, 81, 2266, 2896, 422, 1044, 495, 10, 158, 1538, 1163, 454, 1633, 611, 64, 3694, 149, 726, 2964, 4032, 52, 121, 614, 1048, 2534, 457, 1032, 32, 926, 677, 3675, 282, 5, 1409, 1475, 1555, 3756, 2224, 19, 914, 1411, 1727, 110, 2493, 175, 1096, 428, 863, 19, 898, 796, 1809, 547, 581, 2625, 1865, 636, 7, 1144, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[21, 465, 1722, 3, 1413, 1, 769, 39, 1567, 190, 1641, 647, 949, 347, 10, 464, 307, 1185, 1007, 1363, 3805, 1149, 332, 136, 1, 2426, 192, 145, 1615, 89, 1058, 1611, 3808, 4, 35, 1235, 1643, 30, 923, 1231, 3207, 2604, 658, 178, 0, 797, 501, 566, 548, 334, 559, 200, 103, 34, 3863, 165, 2, 2061, 3204, 2354, 328, 250, 845, 0, 990, 4004, 13, 1291, 196, 97, 2582, 2032, 571, 695, 324, 8, 85, 1722, 941, 713, 1362, 968, 94, 3711, 343, 861, 2927, 3770, 64, 103, 567, 490, 2567, 321, 1660, 6, 985, 900, 3745, 308, 3, 1764, 1019, 2157, 3726, 2144, 67, 1612, 853, 1763, 166, 1448, 39, 1033, 1387, 631, 34, 1515, 941, 1706, 551, 636, 1931, 742, 671, 15, 1556, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[22, 789, 2408, 2, 1918, 7, 1039, 27, 1417, 76, 1005, 709, 1301, 286, 2, 422, 156, 1993, 519, 1518, 3677, 952, 336, 35, 2, 2511, 159, 275, 884, 19, 763, 1950, 4034, 2, 18, 812, 1731, 25, 782, 1409, 2642, 2710, 285, 193, 5, 1263, 1051, 971, 542, 413, 491, 101, 27, 33, 3844, 140, 2, 2785, 3835, 2731, 197, 345, 266, 1, 758, 3984, 7, 1133, 175, 34, 2765, 2608, 861, 589, 184, 22, 191, 1797, 1465, 727, 2024, 615, 78, 3905, 133, 1049, 1838, 4043, 39, 52, 720, 806, 3243, 367, 1038, 10, 634, 423, 3916, 289, 37, 1717, 757, 1955, 3464, 2123, 4, 1512, 1098, 1654, 255, 1613, 24, 717, 1015, 228, 5, 570, 156, 1783, 792, 886, 2417, 1147, 559, 10, 1174, 13] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[16, 481, 1925, 1, 1516, 4, 771, 9, 1639, 117, 1561, 603, 1120, 294, 0, 332, 304, 1670, 765, 1117, 3824, 993, 282, 79, 1, 2833, 186, 120, 879, 27, 825, 1755, 4003, 1, 13, 950, 2040, 40, 707, 1106, 2832, 3035, 610, 200, 1, 1049, 794, 1014, 542, 373, 647, 98, 40, 12, 3904, 187, 0, 2067, 3687, 2733, 218, 377, 452, 2, 848, 4038, 0, 1150, 180, 68, 2896, 2377, 716, 575, 198, 2, 202, 1621, 1271, 827, 2112, 517, 50, 3932, 127, 1095, 2257, 3936, 35, 29, 625, 1004, 2656, 267, 1398, 2, 1159, 357, 3944, 213, 3, 1884, 880, 1662, 3958, 2185, 40, 1379, 578, 1303, 178, 1406, 27, 1299, 1771, 444, 7, 1229, 318, 1654, 714, 795, 2359, 1111, 361, 10, 1041, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[10, 610, 1810, 4, 1877, 0, 629, 15, 1386, 108, 1791, 892, 1239, 240, 5, 424, 248, 1163, 871, 1387, 3792, 999, 405, 83, 3, 2551, 178, 402, 693, 22, 866, 1573, 3927, 2, 9, 962, 1732, 19, 549, 1325, 3103, 2673, 414, 179, 28, 1121, 947, 1312, 449, 561, 807, 105, 31, 22, 3949, 130, 4, 2545, 3617, 2555, 322, 459, 290, 2, 1072, 3983, 0, 1715, 383, 119, 2863, 2391, 702, 661, 300, 6, 75, 1058, 1182, 783, 2001, 1179, 61, 3760, 125, 846, 2806, 3986, 52, 27, 715, 1047, 2361, 216, 1732, 7, 757, 458, 3686, 219, 2, 1290, 1229, 2485, 3861, 1784, 46, 1191, 1042, 1171, 220, 1887, 41, 792, 835, 347, 17, 570, 260, 1644, 757, 831, 2101, 1146, 404, 1, 1355, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[14, 297, 2101, 0, 1470, 0, 918, 5, 1367, 89, 1525, 663, 664, 347, 1, 391, 299, 1792, 605, 1530, 3872, 771, 520, 40, 0, 2838, 206, 124, 735, 15, 594, 2056, 3945, 14, 18, 997, 2364, 19, 399, 1604, 3224, 2589, 166, 224, 8, 1186, 1049, 791, 505, 400, 860, 129, 12, 4, 3878, 90, 0, 2580, 3559, 2617, 216, 301, 280, 1, 645, 4033, 1, 1440, 171, 47, 3116, 2377, 413, 820, 216, 18, 128, 1800, 1373, 815, 2320, 593, 56, 3808, 142, 785, 2518, 4055, 101, 64, 672, 512, 3341, 264, 1143, 4, 715, 532, 4013, 220, 0, 1945, 830, 2500, 3787, 1775, 5, 1183, 713, 1827, 105, 2047, 137, 681, 795, 307, 7, 970, 284, 1244, 579, 951, 1920, 1486, 497, 6, 1342, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[19, 483, 1986, 4, 1328, 0, 660, 9, 1527, 275, 1479, 848, 1313, 525, 0, 408, 236, 1879, 1302, 1331, 3724, 1275, 375, 126, 2, 2203, 185, 279, 1118, 25, 880, 2087, 3882, 1, 24, 1606, 2155, 24, 372, 1597, 2942, 2590, 247, 130, 15, 957, 957, 691, 528, 545, 678, 141, 51, 22, 3767, 202, 1, 2600, 3612, 2549, 352, 326, 392, 8, 975, 3913, 8, 1252, 173, 162, 2012, 2460, 292, 984, 252, 5, 111, 1766, 1185, 504, 1242, 868, 75, 3807, 187, 927, 2195, 4008, 24, 70, 360, 912, 2949, 434, 830, 0, 609, 471, 3692, 435, 22, 2125, 1293, 2651, 3646, 2172, 31, 1364, 660, 2192, 108, 1763, 185, 891, 1136, 686, 8, 525, 439, 1559, 755, 589, 1991, 660, 756, 4, 851, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[10, 366, 2260, 0, 2009, 0, 664, 9, 908, 123, 1021, 690, 1263, 279, 0, 301, 205, 1854, 708, 1647, 3509, 928, 335, 87, 1, 2552, 172, 246, 576, 23, 621, 1977, 4011, 6, 12, 1011, 2440, 20, 633, 1529, 2496, 1988, 348, 152, 2, 880, 1087, 734, 593, 621, 719, 156, 23, 30, 3973, 288, 0, 2331, 3656, 2158, 133, 274, 448, 5, 692, 4031, 0, 1701, 258, 36, 3053, 2805, 502, 826, 308, 5, 300, 1929, 980, 719, 2251, 261, 86, 3893, 142, 588, 2510, 4049, 90, 59, 1023, 876, 2641, 226, 1253, 1, 871, 392, 3991, 264, 2, 1998, 809, 1755, 3795, 1636, 17, 1391, 813, 2276, 217, 2104, 148, 915, 1290, 512, 23, 969, 233, 1646, 814, 729, 2298, 925, 442, 8, 1593, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[21, 465, 1722, 3, 1413, 1, 769, 39, 1567, 190, 1641, 647, 949, 347, 10, 464, 3745, 1921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 307, 1185, 1007, 1363, 3805, 1149, 332, 136, 1, 2426, 192, 145, 1615, 89, 1058, 1611, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2291, 4, 35, 1235, 1643, 30, 923, 1231, 3207, 2604, 658, 178, 0, 797, 501, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 334, 559, 200, 103, 34, 3863, 165, 2, 2061, 3204, 2354, 328, 250, 845, 0, 791, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 990, 3321, 13, 1291, 196, 97, 2582, 2032, 571, 695, 324, 8, 85, 1722, 941, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1362, 968, 94, 3711, 343, 861, 2927, 384, 64, 103, 567, 490, 2567, 321, 1660, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 985, 900, 0, 308, 3, 1764, 1019, 2157, 1832, 2144, 67, 1612, 853, 1763, 166, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1033, 1387, 631, 34, 1515, 941, 1706, 551, 636, 1931, 742, 671, 15, 1556, 7, 1894, 1465, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15914, 16946, 15903, 15799, 15581, 16428, 17021, 17480] avg=16384.0000 max=17480 max_rank=7 min=15581 min_rank=4 imbalance_ratio=1.066895 max_load_reduction=113592 max_load_reduction_pct=0.866638 moved_to_echo=66510 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[14, 555, 2105, 0, 1480, 0, 724, 14, 1391, 94, 882, 969, 1774, 90, 5, 338, 3675, 2055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 1407, 976, 1049, 3806, 1147, 471, 142, 6, 2177, 284, 255, 572, 64, 689, 2042, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2275, 0, 19, 973, 1610, 27, 465, 1169, 3259, 2815, 581, 371, 24, 566, 736, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 473, 581, 104, 17, 37, 3712, 104, 0, 2765, 3707, 1933, 148, 266, 438, 22, 786, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 3162, 3, 1334, 879, 81, 2266, 2896, 422, 1044, 495, 10, 158, 1538, 1163, 454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1633, 611, 64, 3694, 149, 726, 2964, 410, 52, 121, 614, 1048, 2534, 457, 1032, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 926, 677, 0, 282, 5, 1409, 1475, 1555, 1846, 2224, 19, 914, 1411, 1727, 110, 2493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 1096, 428, 863, 19, 898, 796, 1809, 547, 581, 2625, 1865, 636, 7, 1144, 6, 1910, 1567, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16165, 15869, 15898, 15815, 16417, 16141, 17073, 17694] avg=16384.0000 max=17694 max_rank=7 min=15815 min_rank=3 imbalance_ratio=1.079956 max_load_reduction=113378 max_load_reduction_pct=0.865005 moved_to_echo=67325 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[22, 789, 2408, 2, 1918, 7, 1039, 27, 1417, 76, 1005, 709, 1301, 286, 2, 422, 3916, 2060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 1993, 519, 1518, 3677, 952, 336, 35, 2, 2511, 159, 275, 884, 19, 763, 1950, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2427, 2, 18, 812, 1731, 25, 782, 1409, 2642, 2710, 285, 193, 5, 1263, 1051, 971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 413, 491, 101, 27, 33, 3844, 140, 2, 2785, 3835, 2731, 197, 345, 266, 1, 838, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 758, 3304, 7, 1133, 175, 34, 2765, 2608, 861, 589, 184, 22, 191, 1797, 1465, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2024, 615, 78, 3905, 133, 1049, 1838, 412, 39, 52, 720, 806, 3243, 367, 1038, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 634, 423, 0, 289, 37, 1717, 757, 1955, 1703, 2123, 4, 1512, 1098, 1654, 255, 1613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 717, 1015, 228, 5, 570, 156, 1783, 792, 886, 2417, 1147, 559, 10, 1174, 13, 1761, 1571, 769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17406, 16271, 16326, 16749, 16620, 16329, 15774, 15597] avg=16384.0000 max=17406 max_rank=0 min=15597 min_rank=7 imbalance_ratio=1.062378 max_load_reduction=113666 max_load_reduction_pct=0.867203 moved_to_echo=64320 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[16, 481, 1925, 1, 1516, 4, 771, 9, 1639, 117, 1561, 603, 1120, 294, 0, 332, 3944, 2006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 1670, 765, 1117, 3824, 993, 282, 79, 1, 2833, 186, 120, 879, 27, 825, 1755, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2407, 1, 13, 950, 2040, 40, 707, 1106, 2832, 3035, 610, 200, 1, 1049, 794, 1014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 542, 373, 647, 98, 40, 12, 3904, 187, 0, 2067, 3687, 2733, 218, 377, 452, 2, 832, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 848, 3349, 0, 1150, 180, 68, 2896, 2377, 716, 575, 198, 2, 202, 1621, 1271, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2112, 517, 50, 3932, 127, 1095, 2257, 400, 35, 29, 625, 1004, 2656, 267, 1398, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1159, 357, 0, 213, 3, 1884, 880, 1662, 1946, 2185, 40, 1379, 578, 1303, 178, 1406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 1299, 1771, 444, 7, 1229, 318, 1654, 714, 795, 2359, 1111, 361, 10, 1041, 9, 2012, 1530, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16339, 16189, 16799, 16331, 16280, 16506, 15173, 17455] avg=16384.0000 max=17455 max_rank=7 min=15173 min_rank=6 imbalance_ratio=1.065369 max_load_reduction=113617 max_load_reduction_pct=0.866829 moved_to_echo=65414 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[10, 610, 1810, 4, 1877, 0, 629, 15, 1386, 108, 1791, 892, 1239, 240, 5, 424, 3686, 2031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 1163, 871, 1387, 3792, 999, 405, 83, 3, 2551, 178, 402, 693, 22, 866, 1573, 522, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2362, 2, 9, 962, 1732, 19, 549, 1325, 3103, 2673, 414, 179, 28, 1121, 947, 1312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 561, 807, 105, 31, 22, 3949, 130, 4, 2545, 3617, 2555, 322, 459, 290, 2, 816, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1072, 3303, 0, 1715, 383, 119, 2863, 2391, 702, 661, 300, 6, 75, 1058, 1182, 783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2001, 1179, 61, 3760, 125, 846, 2806, 406, 52, 27, 715, 1047, 2361, 216, 1732, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 458, 0, 219, 2, 1290, 1229, 2485, 1898, 1784, 46, 1191, 1042, 1171, 220, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 792, 835, 347, 17, 570, 260, 1644, 757, 831, 2101, 1146, 404, 1, 1355, 3, 1963, 1549, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16757, 15758, 16737, 16822, 16613, 17341, 15679, 15365] avg=16384.0000 max=17341 max_rank=5 min=15365 min_rank=7 imbalance_ratio=1.058411 max_load_reduction=113731 max_load_reduction_pct=0.867699 moved_to_echo=64998 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[14, 297, 2101, 0, 1470, 0, 918, 5, 1367, 89, 1525, 663, 664, 347, 1, 391, 4013, 2066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 299, 1792, 605, 1530, 3872, 771, 520, 40, 0, 2838, 206, 124, 735, 15, 594, 2056, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2373, 14, 18, 997, 2364, 19, 399, 1604, 3224, 2589, 166, 224, 8, 1186, 1049, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 505, 400, 860, 129, 12, 4, 3878, 90, 0, 2580, 3559, 2617, 216, 301, 280, 1, 820, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 645, 3344, 1, 1440, 171, 47, 3116, 2377, 413, 820, 216, 18, 128, 1800, 1373, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2320, 593, 56, 3808, 142, 785, 2518, 413, 101, 64, 672, 512, 3341, 264, 1143, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 715, 532, 0, 220, 0, 1945, 830, 2500, 1861, 1775, 5, 1183, 713, 1827, 105, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 681, 795, 307, 7, 970, 284, 1244, 579, 951, 1920, 1486, 497, 6, 1342, 0, 1926, 1576, 752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15931, 16526, 17025, 16412, 16724, 16736, 16258, 15460] avg=16384.0000 max=17025 max_rank=2 min=15460 min_rank=7 imbalance_ratio=1.039124 max_load_reduction=114047 max_load_reduction_pct=0.870110 moved_to_echo=65178 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[19, 483, 1986, 4, 1328, 0, 660, 9, 1527, 275, 1479, 848, 1313, 525, 0, 408, 3692, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 1879, 1302, 1331, 3724, 1275, 375, 126, 2, 2203, 185, 279, 1118, 25, 880, 2087, 517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2326, 1, 24, 1606, 2155, 24, 372, 1597, 2942, 2590, 247, 130, 15, 957, 957, 691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 545, 678, 141, 51, 22, 3767, 202, 1, 2600, 3612, 2549, 352, 326, 392, 8, 811, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 975, 3238, 8, 1252, 173, 162, 2012, 2460, 292, 984, 252, 5, 111, 1766, 1185, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1242, 868, 75, 3807, 187, 927, 2195, 399, 24, 70, 360, 912, 2949, 434, 830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 609, 471, 0, 435, 22, 2125, 1293, 2651, 1788, 2172, 31, 1364, 660, 2192, 108, 1763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 891, 1136, 686, 8, 525, 439, 1559, 755, 589, 1991, 660, 756, 4, 851, 6, 1858, 1562, 745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16603, 17544, 16634, 16743, 15379, 15279, 17684, 15206] avg=16384.0000 max=17684 max_rank=6 min=15206 min_rank=7 imbalance_ratio=1.079346 max_load_reduction=113388 max_load_reduction_pct=0.865082 moved_to_echo=63548 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[10, 366, 2260, 0, 2009, 0, 664, 9, 908, 123, 1021, 690, 1263, 279, 0, 301, 3991, 2063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 1854, 708, 1647, 3509, 928, 335, 87, 1, 2552, 172, 246, 576, 23, 621, 1977, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2412, 6, 12, 1011, 2440, 20, 633, 1529, 2496, 1988, 348, 152, 2, 880, 1087, 734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 593, 621, 719, 156, 23, 30, 3973, 288, 0, 2331, 3656, 2158, 133, 274, 448, 5, 834, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 692, 3344, 0, 1701, 258, 36, 3053, 2805, 502, 826, 308, 5, 300, 1929, 980, 719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2251, 261, 86, 3893, 142, 588, 2510, 412, 90, 59, 1023, 876, 2641, 226, 1253, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 871, 392, 0, 264, 2, 1998, 809, 1755, 1865, 1636, 17, 1391, 813, 2276, 217, 2104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 915, 1290, 512, 23, 969, 233, 1646, 814, 729, 2298, 925, 442, 8, 1593, 1, 1930, 1574, 765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15957, 15969, 15750, 16401, 17458, 16312, 16410, 16815] avg=16384.0000 max=17458 max_rank=4 min=15750 min_rank=2 imbalance_ratio=1.065552 max_load_reduction=113614 max_load_reduction_pct=0.866806 moved_to_echo=66995 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[6528, 26368, 128, 11136, 2432, 768, 21632, 19968, 4480, 6272, 2304, 128, 1280, 13312, 9600, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 4096, 16384, 128, 13056, 128, 6272, 128, 11264, 1152, 11008, 6144, 9728, 2432, 128, 3200, 30720, 16256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[18944, 128, 256, 8576, 15744, 256, 4864, 11008, 23808, 21120, 3328, 1664, 128, 7936, 7168, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[896, 7424, 8704, 4096, 128, 7296, 3456, 13056, 5632, 6016, 17664, 9088, 4352, 128, 10112, 128, 15360, 12416, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[6656, 4224, 0, 2304, 128, 14208, 8320, 16768, 14848, 16128, 256, 10624, 7168, 13952, 1408, 14848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[14976, 5632, 640, 30592, 1408, 6912, 20096, 3328, 512, 640, 5376, 6784, 22400, 2560, 10112, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[4352, 3840, 5376, 1152, 384, 256, 30976, 1408, 128, 19840, 28928, 19712, 1920, 2688, 3456, 128, 6528, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[2048, 13056, 6784, 11008, 30080, 8320, 3072, 768, 128, 20096, 1664, 1920, 7168, 384, 6400, 15104, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 4095, 1343, 4096, 0, 0, 10, 369, 4095, 2, 1053, 0, 0, 4045, 0, 0, 0, 0, 1, 4095, 4084, 0, 2, 265, 0, 3, 3, 0, 73, 0, 0, 4096, 0, 2010, 0, 0, 0, 0, 0, 8, 0, 672, 88, 0, 0, 0, 24, 3541, 2461, 4078, 0, 0, 0, 1, 0, 4094, 0, 0, 0, 4088, 0, 2, 0, 3950, 0, 0, 4093, 3, 25, 0, 4091, 4083, 4094, 19, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4094, 4095, 0, 3982, 0, 0, 4078, 1, 0, 79, 0, 2, 4096, 4096, 0, 0, 1, 0, 0, 0, 4, 0, 1, 24, 0, 4068, 0, 56, 7, 0, 0, 0, 4051, 4093, 3959, 0, 4092, 23, 4096, 1, 4089, 0, 730] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[2, 30, 4086, 70, 4071, 2, 0, 5, 200, 4044, 21, 1380, 9, 0, 4007, 0, 0, 13, 37, 1, 4084, 4036, 2, 3, 8, 9, 0, 0, 4, 567, 4, 0, 4049, 0, 3243, 0, 57, 0, 14, 0, 0, 129, 3303, 699, 45, 0, 2, 4, 1763, 1624, 4014, 3, 0, 1, 62, 5, 4086, 2, 0, 0, 4078, 1, 5, 0, 1745, 0, 0, 3919, 0, 2358, 0, 4082, 3584, 4052, 0, 0, 2, 0, 10, 0, 0, 99, 65, 20, 4091, 4095, 4044, 77, 3477, 5, 0, 4076, 9, 13, 153, 18, 0, 4073, 4017, 0, 56, 6, 0, 0, 0, 33, 1, 102, 4, 3, 4043, 4, 49, 0, 11, 4, 7, 4088, 4089, 4080, 0, 3706, 2, 4063, 2, 4055, 10, 686] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 5, 4091, 745, 4090, 1, 1, 0, 353, 4094, 0, 150, 0, 0, 3353, 0, 0, 20, 0, 0, 4095, 4090, 0, 1, 36, 0, 12, 2, 0, 175, 3, 0, 4094, 1, 1695, 0, 27, 0, 2, 1, 0, 2, 2294, 0, 4, 0, 0, 0, 4072, 2735, 4091, 2, 0, 0, 1, 0, 4095, 0, 0, 0, 4085, 0, 0, 1, 3207, 0, 0, 4094, 0, 1703, 0, 4095, 3838, 4090, 0, 0, 1, 2, 13, 0, 5, 3, 9, 20, 4094, 4096, 4084, 0, 3737, 0, 0, 4094, 0, 3, 365, 5, 0, 4094, 4096, 0, 14, 6, 0, 0, 2, 0, 3, 23, 6, 0, 4081, 0, 15, 0, 0, 3, 0, 4036, 4095, 4045, 0, 4095, 2, 4062, 0, 4024, 2, 291] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4096, 786, 4093, 0, 0, 4, 3304, 4096, 0, 197, 0, 0, 4083, 0, 0, 0, 0, 0, 4094, 4091, 0, 0, 171, 0, 0, 0, 0, 279, 0, 0, 4096, 0, 1894, 0, 0, 5, 0, 0, 0, 9, 2733, 3, 0, 0, 0, 4, 909, 420, 4094, 0, 0, 0, 52, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 1711, 0, 0, 4094, 1, 707, 0, 4088, 3940, 4095, 219, 0, 0, 0, 0, 0, 0, 62, 0, 0, 4096, 4096, 4096, 0, 4046, 0, 0, 4017, 0, 1, 555, 0, 0, 4096, 4096, 0, 77, 0, 0, 0, 0, 1, 0, 0, 108, 0, 4096, 0, 2, 0, 0, 0, 0, 4053, 4096, 3748, 1, 4093, 0, 4096, 0, 4096, 0, 2885] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4096, 2849, 4096, 0, 0, 1, 1175, 4095, 24, 1120, 0, 0, 4072, 0, 0, 0, 0, 0, 4096, 4075, 0, 0, 164, 0, 1, 0, 0, 1021, 0, 0, 4096, 0, 1503, 0, 0, 0, 1, 0, 12, 0, 1297, 8, 0, 0, 0, 183, 3688, 1428, 4096, 0, 0, 0, 5, 0, 4095, 0, 7, 0, 4096, 0, 0, 0, 1599, 0, 0, 4018, 0, 841, 0, 4096, 4071, 4093, 90, 0, 0, 0, 4, 0, 0, 90, 0, 17, 4096, 4096, 4096, 0, 4041, 0, 0, 4078, 0, 0, 191, 0, 0, 4095, 4096, 0, 1, 0, 0, 0, 0, 30, 0, 0, 24, 0, 4079, 0, 11, 1, 0, 0, 0, 3425, 4093, 3955, 0, 4068, 0, 4096, 0, 4096, 0, 85] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 4, 4095, 1947, 4093, 1, 0, 0, 309, 4095, 2, 557, 0, 19, 4071, 6, 0, 13, 6, 0, 4092, 4084, 2, 2, 1609, 1, 0, 0, 3, 261, 4, 1, 4091, 0, 1241, 0, 19, 12, 0, 0, 0, 4, 2791, 0, 0, 0, 0, 387, 2841, 1445, 4091, 1, 2, 0, 72, 2, 4093, 3, 1, 0, 4084, 0, 0, 0, 2880, 11, 0, 4018, 0, 771, 0, 4036, 3404, 4049, 2, 7, 2, 4, 1, 2, 0, 0, 2, 0, 4093, 4094, 4051, 0, 3719, 2, 2, 4069, 0, 0, 91, 1, 0, 4094, 4094, 0, 1, 1, 0, 2, 0, 1, 0, 0, 59, 0, 4091, 0, 113, 0, 0, 0, 3, 3957, 4092, 4057, 3, 4091, 1, 4045, 0, 4094, 0, 505] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4077, 260, 4089, 0, 2, 18, 840, 4042, 0, 2086, 0, 0, 4028, 0, 0, 1, 0, 0, 4095, 4082, 0, 6, 102, 0, 7, 29, 0, 104, 0, 0, 4096, 2, 2171, 0, 0, 0, 4, 0, 0, 2, 3330, 55, 0, 0, 0, 0, 3353, 2412, 4096, 0, 0, 0, 99, 0, 4096, 0, 0, 0, 4094, 0, 2, 0, 2190, 9, 0, 4030, 0, 549, 0, 3993, 4087, 4030, 6, 0, 0, 0, 20, 0, 0, 0, 0, 0, 4096, 4096, 4073, 0, 3969, 0, 0, 4021, 0, 0, 1, 0, 0, 4096, 4092, 0, 210, 0, 0, 0, 0, 15, 0, 63, 81, 0, 3936, 1, 294, 0, 0, 0, 0, 3895, 4091, 3154, 7, 4022, 1, 4062, 0, 4094, 0, 108] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4091, 1253, 4094, 0, 0, 0, 1209, 4096, 0, 401, 1, 0, 3422, 0, 0, 12, 0, 0, 4094, 4095, 2, 1, 62, 0, 0, 0, 0, 1359, 0, 0, 4094, 0, 1393, 0, 6, 7, 0, 0, 0, 1, 3132, 0, 0, 0, 0, 0, 2064, 769, 4093, 0, 0, 1, 1, 1, 4095, 0, 0, 0, 4096, 2, 0, 0, 3125, 0, 0, 4088, 0, 1363, 0, 4096, 3926, 4096, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 4095, 4095, 4095, 0, 4093, 1, 0, 4094, 0, 1, 100, 0, 0, 4096, 4096, 0, 1, 1, 1, 0, 0, 0, 0, 5, 3, 0, 4092, 0, 6, 0, 0, 0, 0, 4094, 4095, 3849, 2, 4069, 0, 4088, 0, 4093, 0, 1260] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 30, 1310, 70, 4071, 2, 0, 5, 200, 4044, 21, 1380, 9, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 37, 1, 4084, 4036, 2, 3, 8, 9, 0, 0, 4, 567, 4, 0, 4055, 2849, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 3243, 0, 57, 0, 14, 0, 0, 129, 3303, 699, 45, 0, 2, 4, 4092, 3632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1763, 1624, 4014, 3, 0, 1, 62, 5, 3724, 2, 0, 0, 4078, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1745, 0, 0, 3919, 0, 2358, 0, 4082, 3584, 753, 0, 0, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 65, 20, 4091, 3, 4044, 77, 3477, 5, 0, 4076, 9, 13, 153, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4017, 0, 56, 6, 0, 0, 0, 33, 1, 102, 4, 3, 4043, 4, 2776, 450, 424, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 11, 4, 7, 4088, 0, 4080, 0, 3706, 2, 3639, 2, 0, 10, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15151, 16129, 19269, 15282, 16453, 16150, 16354, 16284] avg=16384.0000 max=19269 max_rank=2 min=15151 min_rank=0 imbalance_ratio=1.176086 max_load_reduction=111803 max_load_reduction_pct=0.852989 moved_to_echo=65241 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1312, 1343, 4096, 0, 0, 10, 369, 4095, 2, 1053, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4084, 0, 2, 265, 0, 3, 3, 0, 73, 0, 0, 4089, 2878, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2010, 0, 0, 0, 0, 0, 8, 0, 672, 88, 0, 0, 0, 24, 4091, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3541, 2461, 4078, 0, 0, 0, 1, 0, 3731, 0, 0, 0, 4088, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3950, 0, 0, 4093, 3, 25, 0, 4091, 4083, 761, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 3, 4095, 0, 3982, 0, 0, 4078, 1, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 1, 0, 0, 0, 4, 0, 1, 24, 0, 4068, 0, 2783, 455, 427, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 7, 0, 0, 0, 4051, 0, 3959, 0, 4092, 23, 3669, 1, 0, 0, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16326, 15951, 14624, 17902, 17025, 16336, 16320, 16588] avg=16384.0000 max=17902 max_rank=3 min=14624 min_rank=2 imbalance_ratio=1.092651 max_load_reduction=113170 max_load_reduction_pct=0.863419 moved_to_echo=66269 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1313, 2849, 4096, 0, 0, 1, 1175, 4095, 24, 1120, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4075, 0, 0, 164, 0, 1, 0, 0, 1021, 0, 0, 4096, 2877, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1503, 0, 0, 0, 1, 0, 12, 0, 1297, 8, 0, 0, 0, 183, 4093, 3636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3688, 1428, 4096, 0, 0, 0, 5, 0, 3732, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1599, 0, 0, 4018, 0, 841, 0, 4096, 4071, 761, 90, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 17, 4096, 3, 4096, 0, 4041, 0, 0, 4078, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 1, 0, 0, 0, 0, 30, 0, 0, 24, 0, 4079, 0, 2783, 455, 427, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 0, 3425, 0, 3955, 0, 4068, 0, 3669, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18745, 16787, 14829, 17052, 15480, 16612, 16353, 15214] avg=16384.0000 max=18745 max_rank=0 min=14829 min_rank=2 imbalance_ratio=1.144104 max_load_reduction=112327 max_load_reduction_pct=0.856987 moved_to_echo=63659 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 1311, 745, 4090, 1, 1, 0, 353, 4094, 0, 150, 0, 0, 3353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 4095, 4090, 0, 1, 36, 0, 12, 2, 0, 175, 3, 0, 4024, 2875, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1695, 0, 27, 0, 2, 1, 0, 2, 2294, 0, 4, 0, 0, 0, 4093, 3637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 2735, 4091, 2, 0, 0, 1, 0, 3732, 0, 0, 0, 4085, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3207, 0, 0, 4094, 0, 1703, 0, 4095, 3838, 761, 0, 0, 1, 2, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 9, 20, 4094, 3, 4084, 0, 3737, 0, 0, 4094, 0, 3, 365, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 14, 6, 0, 0, 2, 0, 3, 23, 6, 0, 4081, 0, 2780, 454, 424, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 3, 0, 4036, 0, 4045, 0, 4095, 2, 3638, 0, 0, 2, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14103, 15791, 15850, 18719, 17714, 16422, 16346, 16127] avg=16384.0000 max=18719 max_rank=3 min=14103 min_rank=0 imbalance_ratio=1.142517 max_load_reduction=112353 max_load_reduction_pct=0.857185 moved_to_echo=66609 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1313, 786, 4093, 0, 0, 4, 3304, 4096, 0, 197, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4091, 0, 0, 171, 0, 0, 0, 0, 279, 0, 0, 4096, 2879, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1894, 0, 0, 5, 0, 0, 0, 9, 2733, 3, 0, 0, 0, 4, 4093, 3639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 909, 420, 4094, 0, 0, 0, 52, 0, 3733, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1711, 0, 0, 4094, 1, 707, 0, 4088, 3940, 761, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 4096, 3, 4096, 0, 4046, 0, 0, 4017, 0, 1, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 77, 0, 0, 0, 0, 1, 0, 0, 108, 0, 4096, 0, 2783, 455, 427, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4053, 0, 3748, 1, 4093, 0, 3669, 0, 0, 0, 2885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17876, 16067, 16476, 13303, 15521, 16876, 16502, 18451] avg=16384.0000 max=18451 max_rank=7 min=13303 min_rank=3 imbalance_ratio=1.126160 max_load_reduction=112621 max_load_reduction_pct=0.859230 moved_to_echo=67350 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 1308, 1947, 4093, 1, 0, 0, 309, 4095, 2, 557, 0, 19, 4071, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 6, 0, 4092, 4084, 2, 2, 1609, 1, 0, 0, 3, 261, 4, 1, 4094, 2851, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1241, 0, 19, 12, 0, 0, 0, 4, 2791, 0, 0, 0, 0, 387, 4094, 3634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2841, 1445, 4091, 1, 2, 0, 72, 2, 3724, 3, 1, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2880, 11, 0, 4018, 0, 771, 0, 4036, 3404, 746, 2, 7, 2, 4, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4093, 0, 4051, 0, 3719, 2, 2, 4069, 0, 0, 91, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 1, 1, 0, 2, 0, 1, 0, 0, 59, 0, 4091, 0, 2787, 452, 427, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 3, 3957, 0, 4057, 3, 4091, 1, 3618, 0, 0, 0, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 17481, 16273, 16266, 15884, 16030, 16378, 16348] avg=16384.0000 max=17481 max_rank=1 min=15884 min_rank=4 imbalance_ratio=1.066956 max_load_reduction=113591 max_load_reduction_pct=0.866631 moved_to_echo=64640 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1307, 260, 4089, 0, 2, 18, 840, 4042, 0, 2086, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 4082, 0, 6, 102, 0, 7, 29, 0, 104, 0, 0, 4094, 2833, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 2171, 0, 0, 0, 4, 0, 0, 2, 3330, 55, 0, 0, 0, 0, 4096, 3633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3353, 2412, 4096, 0, 0, 0, 99, 0, 3733, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2190, 9, 0, 4030, 0, 549, 0, 3993, 4087, 749, 6, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4073, 0, 3969, 0, 0, 4021, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 210, 0, 0, 0, 0, 15, 0, 63, 81, 0, 3936, 1, 2770, 448, 424, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 0, 0, 0, 3895, 0, 3154, 7, 4022, 1, 3638, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16672, 15811, 17389, 17789, 15633, 16160, 16499, 15119] avg=16384.0000 max=17789 max_rank=3 min=15119 min_rank=7 imbalance_ratio=1.085754 max_load_reduction=113283 max_load_reduction_pct=0.864281 moved_to_echo=63411 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1311, 1253, 4094, 0, 0, 0, 1209, 4096, 0, 401, 1, 0, 3422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4094, 4095, 2, 1, 62, 0, 0, 0, 0, 1359, 0, 0, 4093, 2879, 458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1393, 0, 6, 7, 0, 0, 0, 1, 3132, 0, 0, 0, 0, 0, 4092, 3637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2064, 769, 4093, 0, 0, 1, 1, 1, 3732, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3125, 0, 0, 4088, 0, 1363, 0, 4096, 3926, 762, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 4095, 3, 4095, 0, 4093, 1, 0, 4094, 0, 1, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 1, 1, 0, 0, 0, 0, 5, 3, 0, 4092, 0, 2780, 455, 427, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4094, 0, 3849, 2, 4069, 0, 3661, 0, 0, 0, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15787, 17055, 16362, 14759, 17362, 16486, 16320, 16941] avg=16384.0000 max=17362 max_rank=4 min=14759 min_rank=3 imbalance_ratio=1.059692 max_load_reduction=113710 max_load_reduction_pct=0.867538 moved_to_echo=67109 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 10496, 9344, 32768, 128, 128, 128, 7808, 32768, 128, 7040, 128, 128, 31104, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[20480, 128, 0, 32384, 128, 8320, 0, 32640, 30976, 6144, 384, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 15232, 0, 128, 128, 128, 128, 128, 256, 19584, 896, 128, 0, 128, 640, 32768, 29184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 32768, 0, 384, 128, 128, 128, 128, 128, 128, 256, 384, 128, 32512, 128, 22272, 3712, 3456, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[22272, 13312, 32768, 128, 128, 128, 384, 128, 29952, 128, 128, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[640, 128, 128, 128, 128, 31616, 0, 30848, 128, 32256, 128, 29312, 128, 0, 128, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 32768, 32640, 128, 128, 2432, 128, 128, 128, 128, 3840, 128, 128, 32768, 23040, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 256, 128, 128, 32768, 128, 32640, 128, 31104, 128, 128, 32640, 128, 128, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4077, 0, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4, 1757, 4096, 0, 3034, 520, 1052, 0, 0, 0, 4096, 0, 0, 4046, 1817, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3346, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 4096, 0, 0, 4096, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 41, 9, 0, 0, 4096, 0, 4095, 3213, 0, 396, 4074, 0, 0, 0, 4096, 0, 343, 0, 4096, 3332, 0, 4096, 0, 17, 4096, 0, 1679, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4071, 0, 0, 4096, 90, 0, 0, 0, 4091, 0, 0, 4096, 4095, 0, 0, 2366, 1359, 4095, 0, 3981, 1136, 3140, 0, 0, 0, 4095, 0, 0, 2516, 785, 3, 4066, 9, 4095, 0, 0, 1, 0, 0, 4092, 0, 1, 3, 0, 4096, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 4091, 0, 4096, 0, 0, 4, 1, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 1, 4096, 0, 3, 0, 0, 0, 4096, 0, 0, 1, 0, 121, 0, 2, 1, 4096, 0, 4090, 346, 19, 895, 3977, 0, 1, 0, 4054, 3, 4, 0, 4095, 3501, 0, 4094, 0, 25, 4089, 0, 513, 4096, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 96, 0, 0, 0, 4096, 0, 0, 4091, 4050, 0, 0, 679, 1272, 4095, 0, 4061, 3, 3918, 0, 0, 0, 4096, 0, 0, 3447, 914, 0, 4091, 0, 4096, 1, 158, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 1652, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 30, 4096, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 2, 4096, 3, 4096, 0, 0, 4096, 0, 2, 0, 3, 0, 4096, 0, 0, 0, 0, 45, 21, 0, 0, 4096, 0, 4096, 3446, 0, 1294, 4061, 0, 0, 0, 4030, 0, 156, 0, 4096, 3332, 0, 4096, 0, 2, 4093, 0, 180, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4093, 0, 0, 4094, 0, 1, 2, 2, 4088, 0, 7, 4089, 4094, 0, 0, 200, 1160, 4076, 0, 3844, 0, 4039, 0, 1, 1, 4091, 1, 0, 3984, 1211, 1, 4088, 1, 4092, 17, 24, 0, 0, 0, 4094, 0, 1, 1, 2, 4092, 2, 1, 0, 0, 2232, 0, 0, 3, 0, 0, 0, 4094, 0, 7, 2, 4094, 2, 4092, 3, 3, 0, 1, 2, 2, 1, 5, 1, 4093, 1, 4070, 1, 0, 4091, 0, 1, 0, 3, 0, 4093, 4, 1, 0, 1, 197, 13, 4, 0, 4094, 0, 4085, 3704, 0, 1494, 4093, 2, 0, 2, 3846, 0, 1009, 4, 4088, 1521, 0, 4094, 2, 8, 4086, 2, 169, 4091, 0, 2, 0, 2, 0, 1, 5, 0, 40, 0, 2, 15] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4095, 0, 0, 4096, 15, 0, 0, 0, 4096, 0, 0, 4096, 4057, 0, 0, 18, 777, 4095, 0, 3666, 1, 3054, 0, 0, 0, 4096, 0, 0, 4092, 1007, 0, 4092, 0, 4096, 46, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2835, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 7, 4094, 2, 0, 4096, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 172, 5, 0, 0, 4093, 0, 4096, 3114, 0, 1563, 4095, 0, 0, 0, 4096, 0, 54, 0, 4096, 3551, 0, 4096, 0, 29, 4073, 0, 634, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4086, 0, 0, 4094, 25, 0, 0, 0, 4093, 0, 1, 4094, 4013, 1, 0, 88, 590, 4093, 1, 4024, 349, 3897, 0, 0, 0, 4094, 2, 2, 3945, 302, 0, 4096, 0, 4096, 10, 9, 2, 0, 0, 4094, 0, 4, 0, 0, 4094, 0, 0, 1, 0, 2709, 2, 0, 0, 0, 0, 1, 4094, 1, 0, 0, 4092, 0, 4094, 0, 0, 0, 0, 0, 0, 3, 0, 20, 4096, 0, 4091, 0, 0, 4094, 0, 3, 0, 2, 0, 4096, 0, 0, 1, 0, 6, 0, 2, 3, 4093, 0, 4095, 3401, 0, 986, 4092, 2, 0, 0, 4077, 0, 589, 2, 4095, 3566, 0, 4096, 0, 36, 4067, 0, 165, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[4033, 1, 13, 4081, 59, 13, 9, 0, 4060, 0, 1, 4085, 4055, 0, 0, 387, 3276, 4084, 6, 3972, 15, 1948, 0, 13, 0, 4064, 0, 14, 4093, 2171, 0, 4080, 0, 4030, 11, 0, 2, 19, 15, 4083, 9, 10, 0, 3, 4083, 3, 1, 0, 34, 362, 2, 33, 0, 3, 11, 34, 4071, 26, 15, 13, 4070, 8, 4071, 0, 0, 0, 21, 11, 3, 14, 0, 1, 4070, 19, 4081, 13, 0, 4079, 0, 11, 0, 0, 2, 4063, 9, 10, 0, 6, 873, 10, 2, 0, 4092, 20, 4082, 3646, 7, 497, 4083, 17, 0, 0, 4073, 0, 559, 0, 4077, 2729, 0, 4088, 2, 44, 4055, 7, 1, 4078, 8, 1, 14, 10, 24, 4, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4083, 0, 0, 4096, 6, 0, 0, 0, 4095, 0, 0, 4091, 4092, 0, 0, 33, 2439, 4096, 0, 4095, 0, 3811, 1, 0, 0, 4095, 0, 0, 3391, 1964, 0, 3835, 0, 4096, 1, 0, 0, 1, 0, 4095, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 2687, 0, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 4096, 0, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 1, 4095, 12, 4094, 0, 0, 4095, 0, 3, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 40, 0, 0, 4085, 0, 4093, 3284, 0, 19, 4095, 0, 0, 0, 4096, 0, 123, 0, 4096, 2857, 0, 4096, 0, 14, 4090, 0, 93, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 1, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 96, 0, 0, 0, 4096, 0, 0, 3631, 4050, 0, 0, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1272, 0, 0, 4061, 3, 3918, 0, 0, 0, 0, 0, 0, 3447, 914, 0, 2669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 158, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 3633, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1652, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 30, 4096, 0, 4096, 0, 1150, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 2, 4096, 3, 4096, 0, 0, 4096, 0, 2, 3661, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 45, 21, 0, 0, 4096, 0, 4096, 3446, 460, 272, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1294, 4061, 0, 0, 0, 4030, 0, 156, 0, 3945, 3332, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 180, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 4096, 3667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16648, 16284, 16515, 15583, 16392, 16686, 16820, 16144] avg=16384.0000 max=16820 max_rank=6 min=15583 min_rank=3 imbalance_ratio=1.026611 max_load_reduction=114252 max_load_reduction_pct=0.871674 moved_to_echo=66042 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4071, 0, 0, 0, 90, 0, 0, 0, 4091, 0, 0, 3635, 4095, 0, 0, 2366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1359, 0, 0, 3981, 1136, 3140, 0, 0, 0, 0, 0, 0, 2516, 785, 3, 2652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4095, 0, 0, 1, 0, 0, 4092, 0, 1, 3, 0, 4096, 0, 0, 0, 3633, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 4091, 0, 4096, 0, 1143, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 1, 4096, 0, 3, 3660, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 121, 0, 2, 1, 4096, 0, 4090, 346, 461, 271, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 895, 3977, 0, 1, 0, 4054, 3, 4, 0, 3944, 3501, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 513, 4096, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 3667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18348, 15572, 16365, 17880, 16382, 13636, 16423, 16466] avg=16384.0000 max=18348 max_rank=0 min=13636 min_rank=5 imbalance_ratio=1.119873 max_load_reduction=112724 max_load_reduction_pct=0.860016 moved_to_echo=62907 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4077, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 3635, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1757, 0, 0, 3034, 520, 1052, 0, 0, 0, 0, 0, 0, 4046, 1817, 0, 2669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3633, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3346, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 1150, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 4096, 0, 0, 4096, 0, 0, 3662, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 41, 9, 0, 0, 4096, 0, 4095, 3213, 461, 272, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 4074, 0, 0, 0, 4096, 0, 343, 0, 3945, 3332, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1679, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 3668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15912, 14895, 16355, 17247, 16382, 16442, 16203, 17636] avg=16384.0000 max=17636 max_rank=7 min=14895 min_rank=1 imbalance_ratio=1.076416 max_load_reduction=113436 max_load_reduction_pct=0.865448 moved_to_echo=66663 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4086, 0, 0, 0, 25, 0, 0, 0, 4093, 0, 1, 3633, 4013, 1, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 590, 0, 1, 4024, 349, 3897, 0, 0, 0, 0, 2, 2, 3945, 302, 0, 2671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 9, 2, 0, 0, 4094, 0, 4, 0, 0, 4094, 0, 0, 1, 3633, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2709, 2, 0, 0, 0, 0, 1, 4094, 1, 0, 0, 4092, 0, 4094, 0, 1152, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 20, 4096, 0, 4091, 0, 0, 4094, 0, 3, 3658, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 1, 0, 6, 0, 2, 3, 4093, 0, 4095, 3401, 461, 273, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 986, 4092, 2, 0, 0, 4077, 0, 589, 2, 3944, 3566, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 165, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4, 4096, 3665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15940, 15783, 16378, 16606, 16394, 16584, 17294, 16093] avg=16384.0000 max=17294 max_rank=6 min=15783 min_rank=1 imbalance_ratio=1.055542 max_load_reduction=113778 max_load_reduction_pct=0.868057 moved_to_echo=66365 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 0, 1, 2, 2, 4088, 0, 7, 3625, 4094, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 0, 0, 3844, 0, 4039, 0, 1, 1, 0, 1, 0, 3984, 1211, 1, 2658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 17, 24, 0, 0, 0, 4094, 0, 1, 1, 2, 4092, 2, 1, 0, 3630, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2232, 0, 0, 3, 0, 0, 0, 4094, 0, 7, 2, 4094, 2, 4092, 3, 1154, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 2, 2, 1, 5, 1, 4093, 1, 4070, 1, 0, 4091, 0, 1, 3643, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 4, 1, 0, 1, 197, 13, 4, 0, 4094, 0, 4085, 3704, 464, 276, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 4093, 2, 0, 2, 3846, 0, 1009, 4, 3932, 1521, 0, 0, 2, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 2, 169, 4091, 0, 2, 0, 2, 0, 1, 5, 0, 40, 0, 2, 15, 4094, 3665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16112, 16900, 16390, 16144, 16344, 17095, 15913, 16174] avg=16384.0000 max=17095 max_rank=5 min=15913 min_rank=6 imbalance_ratio=1.043396 max_load_reduction=113977 max_load_reduction_pct=0.869576 moved_to_echo=65526 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4033, 1, 13, 0, 59, 13, 9, 0, 4060, 0, 1, 3625, 4055, 0, 0, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3276, 0, 6, 3972, 15, 1948, 0, 13, 0, 0, 0, 14, 4093, 2171, 0, 2661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 11, 0, 2, 19, 15, 4083, 9, 10, 0, 3, 4083, 3, 1, 0, 3605, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 362, 2, 33, 0, 3, 11, 34, 4071, 26, 15, 13, 4070, 8, 4071, 0, 1147, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 11, 3, 14, 0, 1, 4070, 19, 4081, 13, 0, 4079, 0, 11, 3650, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4063, 9, 10, 0, 6, 873, 10, 2, 0, 4092, 20, 4082, 3646, 460, 272, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 497, 4083, 17, 0, 0, 4073, 0, 559, 0, 3926, 2729, 0, 0, 2, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 7, 1, 4078, 8, 1, 14, 10, 24, 4, 0, 0, 0, 1, 0, 0, 4088, 3653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16256, 18169, 16308, 14359, 16401, 17698, 15937, 15944] avg=16384.0000 max=18169 max_rank=1 min=14359 min_rank=3 imbalance_ratio=1.108948 max_load_reduction=112903 max_load_reduction_pct=0.861382 moved_to_echo=65980 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 15, 0, 0, 0, 4096, 0, 0, 3635, 4057, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 777, 0, 0, 3666, 1, 3054, 0, 0, 0, 0, 0, 0, 4092, 1007, 0, 2669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 46, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3634, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2835, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 1151, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 7, 4094, 2, 0, 4096, 0, 0, 3660, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 172, 5, 0, 0, 4093, 0, 4096, 3114, 461, 272, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1563, 4095, 0, 0, 0, 4096, 0, 54, 0, 3945, 3551, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 634, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15916, 15266, 16403, 16737, 16385, 16466, 17333, 16566] avg=16384.0000 max=17333 max_rank=6 min=15266 min_rank=1 imbalance_ratio=1.057922 max_load_reduction=113739 max_load_reduction_pct=0.867760 moved_to_echo=66750 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4083, 0, 0, 0, 6, 0, 0, 0, 4095, 0, 0, 3631, 4092, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2439, 0, 0, 4095, 0, 3811, 1, 0, 0, 0, 0, 0, 3391, 1964, 0, 2502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 1, 0, 4095, 0, 1, 0, 0, 4096, 0, 0, 0, 3633, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2687, 0, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 4096, 0, 4095, 0, 1078, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 4095, 12, 4094, 0, 0, 4095, 0, 3, 3661, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 40, 0, 0, 4085, 0, 4093, 3284, 460, 255, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4095, 0, 0, 0, 4096, 0, 123, 0, 3945, 2857, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 93, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 1, 2, 4096, 3667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15940, 18203, 16358, 16516, 16392, 16465, 15149, 16049] avg=16384.0000 max=18203 max_rank=1 min=15149 min_rank=6 imbalance_ratio=1.111023 max_load_reduction=112869 max_load_reduction_pct=0.861122 moved_to_echo=64055 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 19840, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 32768, 128, 32768, 128, 9216, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 7168, 32640, 128, 128, 128, 32384, 128, 2944, 128, 31616, 24448, 0, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[32768, 128, 3456, 32768, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 29440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[12672, 0, 128, 30720, 2048, 24960, 128, 128, 128, 0, 128, 128, 29568, 10240, 128, 21248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 32768, 128, 256, 128, 128, 128, 32768, 128, 128, 128, 128, 32768, 128, 128, 128, 29056, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32640, 128, 128, 0, 384, 128, 128, 128, 32768, 0, 128, 29056, 32640, 128, 0, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 128, 1536, 128, 128, 128, 32768, 128, 32768, 24192, 3712, 2176, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 32768, 128, 128, 32768, 0, 128, 29312, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4083, 27, 20, 19, 3065, 5, 4074, 273, 0, 0, 0, 0, 58, 1, 21, 13, 0, 26, 8, 4, 3, 15, 0, 0, 9, 4068, 0, 4075, 4087, 9, 17, 4034, 0, 18, 1, 0, 12, 10, 4080, 0, 1869, 0, 4023, 38, 4058, 0, 51, 4082, 4, 0, 42, 17, 0, 0, 6, 4079, 32, 5, 17, 0, 4082, 0, 80, 0, 195, 4076, 25, 0, 0, 3, 402, 19, 8, 3740, 5, 10, 4090, 4048, 0, 7, 16, 4082, 4085, 1, 0, 32, 0, 4075, 2, 33, 4, 69, 3226, 2, 4056, 1, 4041, 4052, 4054, 1, 4084, 0, 7, 4073, 0, 4051, 0, 0, 5, 19, 12, 0, 4063, 14, 6, 6, 0, 0, 0, 7, 46, 14, 3392, 3, 0, 30, 4083, 4077] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1559, 0, 4096, 182, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 4096, 0, 0, 4000, 0, 0, 0, 0, 0, 0, 4096, 2, 2476, 0, 4087, 0, 4093, 0, 17, 4096, 0, 0, 0, 0, 4, 0, 0, 4096, 4, 0, 0, 55, 4096, 17, 0, 0, 481, 4096, 0, 0, 0, 0, 380, 0, 0, 3777, 0, 0, 3971, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4061, 0, 0, 0, 210, 4095, 0, 4074, 0, 3821, 4087, 4096, 0, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3704, 0, 0, 1, 4096, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 689, 0, 4093, 111, 0, 0, 3, 3, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 4094, 1, 4093, 4093, 1, 1, 4091, 0, 0, 0, 0, 0, 3, 4092, 0, 1192, 0, 3841, 2, 4093, 1, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 6, 4094, 0, 0, 0, 2856, 4087, 0, 0, 2, 0, 47, 0, 0, 3951, 1, 2, 4093, 4096, 0, 0, 1, 4096, 4095, 0, 4, 0, 1, 4026, 2, 0, 1, 110, 4093, 0, 4096, 0, 4050, 4041, 4094, 0, 4096, 0, 1, 4095, 0, 4096, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 14, 2, 0, 0, 7, 0, 3740, 0, 0, 0, 4094, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 1323, 0, 4096, 535, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 4095, 0, 0, 4086, 79, 0, 1, 0, 0, 3, 4096, 0, 1175, 0, 4095, 0, 4096, 0, 5, 4085, 1, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 14, 0, 270, 4096, 0, 0, 0, 0, 2621, 1, 0, 3341, 0, 0, 3943, 4096, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 3332, 0, 2, 0, 225, 4096, 0, 4059, 0, 3875, 3921, 4096, 0, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 81, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4064, 0, 0, 0, 4096, 4015] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4089, 0, 1, 3, 1669, 3, 4089, 251, 6, 0, 16, 0, 0, 1, 0, 3, 0, 0, 0, 0, 3, 6, 0, 1, 0, 4093, 2, 4093, 4094, 0, 0, 4038, 21, 4, 0, 0, 5, 2, 4093, 0, 3558, 2, 4063, 0, 4094, 2, 5, 4094, 0, 0, 0, 1, 2, 0, 5, 4089, 0, 3, 4, 104, 4092, 0, 0, 0, 240, 4085, 0, 0, 0, 2, 101, 2, 2, 3646, 2, 2, 4094, 4091, 0, 1, 0, 4091, 4092, 0, 0, 0, 6, 3886, 1, 0, 0, 149, 4069, 4, 4082, 0, 3895, 3713, 4094, 0, 4092, 0, 5, 4093, 0, 4093, 0, 0, 3, 3, 0, 43, 4091, 3, 1, 2, 0, 0, 1, 4, 0, 6, 3642, 2, 0, 0, 4092, 3842] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 242, 0, 4096, 604, 0, 0, 48, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 6, 682, 0, 4092, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 49, 4096, 29, 0, 0, 1969, 4095, 0, 0, 0, 0, 1080, 0, 0, 4089, 0, 0, 4028, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4042, 0, 0, 0, 526, 4096, 0, 4096, 0, 3901, 3991, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3331, 0, 0, 2, 4096, 4078] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1121, 0, 4096, 1266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 411, 0, 4096, 1, 4096, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 4096, 127, 0, 0, 11, 4096, 0, 1, 0, 374, 4037, 0, 0, 0, 0, 3371, 0, 0, 3422, 0, 0, 4092, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3602, 0, 0, 0, 3, 4095, 0, 4096, 0, 3575, 3761, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3648, 0, 0, 0, 4096, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 980, 0, 4096, 2725, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 547, 0, 4082, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 17, 4096, 0, 0, 0, 724, 4079, 1, 1, 0, 0, 159, 0, 0, 3987, 0, 0, 4095, 4095, 0, 0, 0, 4096, 4096, 0, 2, 1, 0, 4095, 0, 0, 0, 1, 4096, 0, 4091, 0, 4094, 4095, 4095, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3176, 1, 2, 0, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1559, 0, 4096, 182, 0, 0, 6, 0, 0, 0, 0, 0, 3693, 1504, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 4096, 0, 0, 4000, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2637, 2, 2476, 0, 4087, 0, 4093, 0, 17, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 4, 0, 0, 55, 4096, 17, 0, 0, 4096, 3936, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 4096, 0, 0, 0, 0, 380, 0, 0, 3777, 0, 0, 3971, 2592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 160, 0, 0, 0, 0, 4061, 0, 0, 0, 210, 4095, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3821, 4087, 4096, 0, 0, 0, 0, 318, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3704, 0, 0, 1, 4096, 4087, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16027, 16311, 17408, 16389, 15297, 16696, 16417, 16527] avg=16384.0000 max=17408 max_rank=2 min=15297 min_rank=4 imbalance_ratio=1.062500 max_load_reduction=113664 max_load_reduction_pct=0.867188 moved_to_echo=64937 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4083, 27, 20, 19, 3065, 5, 4074, 273, 0, 0, 0, 0, 58, 1, 21, 13, 3672, 1487, 887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 8, 4, 3, 15, 0, 0, 9, 4068, 0, 4075, 4087, 9, 17, 4034, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 1, 0, 12, 10, 2627, 0, 1869, 0, 4023, 38, 4058, 0, 51, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 42, 17, 0, 0, 6, 4079, 32, 5, 17, 0, 4082, 0, 80, 0, 4084, 3925, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 4076, 25, 0, 0, 3, 402, 19, 8, 3740, 5, 10, 4090, 2561, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4082, 160, 1, 0, 32, 0, 4075, 2, 33, 4, 69, 3226, 2, 4056, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 4052, 4054, 1, 0, 0, 7, 317, 0, 4051, 0, 0, 5, 19, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 14, 6, 6, 0, 0, 0, 7, 46, 14, 3392, 3, 0, 30, 4083, 4077, 541, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17705, 16380, 16789, 16457, 15141, 15759, 16559, 16282] avg=16384.0000 max=17705 max_rank=0 min=15141 min_rank=4 imbalance_ratio=1.080627 max_load_reduction=113367 max_load_reduction_pct=0.864922 moved_to_echo=63741 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 1323, 0, 4096, 535, 0, 0, 3, 0, 0, 0, 1, 0, 3693, 1504, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 4095, 0, 0, 4086, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 1, 0, 0, 3, 2637, 0, 1175, 0, 4095, 0, 4096, 0, 5, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 14, 0, 4096, 3936, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 4096, 0, 0, 0, 0, 2621, 1, 0, 3341, 0, 0, 3943, 2592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 160, 0, 0, 1, 0, 3332, 0, 2, 0, 225, 4096, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3875, 3921, 4096, 0, 0, 0, 0, 318, 0, 4095, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4064, 0, 0, 0, 4096, 4015, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16140, 16393, 16176, 16336, 16864, 15971, 16386, 16806] avg=16384.0000 max=16864 max_rank=4 min=15971 min_rank=5 imbalance_ratio=1.029297 max_load_reduction=114208 max_load_reduction_pct=0.871338 moved_to_echo=66027 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 0, 689, 0, 4093, 111, 0, 0, 3, 3, 0, 0, 1, 0, 3692, 1504, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 4094, 1, 4093, 4093, 1, 1, 4091, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2634, 0, 1192, 0, 3841, 2, 4093, 1, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 6, 4094, 0, 0, 0, 4096, 3935, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2856, 4087, 0, 0, 2, 0, 47, 0, 0, 3951, 1, 2, 4093, 2592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 160, 0, 4, 0, 1, 4026, 2, 0, 1, 110, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 4041, 4094, 0, 0, 0, 1, 318, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 14, 2, 0, 0, 7, 0, 3740, 0, 0, 0, 4094, 4096, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15079, 16403, 15862, 16310, 17631, 16590, 16603, 16594] avg=16384.0000 max=17631 max_rank=4 min=15079 min_rank=0 imbalance_ratio=1.076111 max_load_reduction=113441 max_load_reduction_pct=0.865486 moved_to_echo=67418 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1121, 0, 4096, 1266, 0, 0, 0, 0, 0, 0, 0, 0, 3693, 1504, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4095, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2637, 0, 411, 0, 4096, 1, 4096, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 127, 0, 0, 11, 4096, 0, 1, 0, 4096, 3936, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 4037, 0, 0, 0, 0, 3371, 0, 0, 3422, 0, 0, 4092, 2592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 160, 0, 0, 0, 0, 3602, 0, 0, 0, 3, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3575, 3761, 4096, 0, 0, 0, 0, 318, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3648, 0, 0, 0, 4096, 4084, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16667, 16408, 15296, 16448, 17888, 16052, 15846, 16467] avg=16384.0000 max=17888 max_rank=4 min=15296 min_rank=2 imbalance_ratio=1.091797 max_load_reduction=113184 max_load_reduction_pct=0.863525 moved_to_echo=66253 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4089, 0, 1, 3, 1669, 3, 4089, 251, 6, 0, 16, 0, 0, 1, 0, 3, 3695, 1507, 894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6, 0, 1, 0, 4093, 2, 4093, 4094, 0, 0, 4038, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4, 0, 0, 5, 2, 2624, 0, 3558, 2, 4063, 0, 4094, 2, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 5, 4089, 0, 3, 4, 104, 4092, 0, 0, 0, 4092, 3934, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 4085, 0, 0, 0, 2, 101, 2, 2, 3646, 2, 2, 4094, 2584, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 158, 0, 0, 0, 6, 3886, 1, 0, 0, 149, 4069, 4, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3895, 3713, 4094, 0, 0, 0, 5, 312, 0, 4093, 0, 0, 3, 3, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 1, 2, 0, 0, 1, 4, 0, 6, 3642, 2, 0, 0, 4092, 3842, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16227, 16357, 18474, 16412, 14761, 16446, 16161, 16234] avg=16384.0000 max=18474 max_rank=2 min=14761 min_rank=4 imbalance_ratio=1.127563 max_load_reduction=112598 max_load_reduction_pct=0.859055 moved_to_echo=63602 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 242, 0, 4096, 604, 0, 0, 48, 0, 64, 0, 0, 0, 3693, 1504, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2637, 6, 682, 0, 4092, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 49, 4096, 29, 0, 0, 4096, 3936, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1969, 4095, 0, 0, 0, 0, 1080, 0, 0, 4089, 0, 0, 4028, 2592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 160, 0, 0, 0, 0, 4042, 0, 0, 0, 526, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3901, 3991, 4096, 0, 0, 0, 0, 318, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3331, 0, 0, 2, 4096, 4078, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15237, 16409, 15610, 16394, 17853, 17016, 16402, 16151] avg=16384.0000 max=17853 max_rank=4 min=15237 min_rank=0 imbalance_ratio=1.089661 max_load_reduction=113219 max_load_reduction_pct=0.863792 moved_to_echo=67422 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 980, 0, 4096, 2725, 0, 0, 0, 0, 7, 0, 0, 0, 3693, 1504, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 4087, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2637, 0, 547, 0, 4082, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 17, 4096, 0, 0, 0, 4096, 3936, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 724, 4079, 1, 1, 0, 0, 159, 0, 0, 3987, 0, 0, 4095, 2591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 160, 0, 2, 1, 0, 4095, 0, 0, 0, 1, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 4095, 0, 0, 0, 0, 318, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3176, 1, 2, 0, 4096, 4096, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17990, 16411, 15457, 16326, 15637, 16542, 16698, 16011] avg=16384.0000 max=17990 max_rank=0 min=15457 min_rank=2 imbalance_ratio=1.098022 max_load_reduction=113082 max_load_reduction_pct=0.862747 moved_to_echo=64888 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[31360, 31744, 32768, 128, 0, 0, 128, 2560, 0, 32768, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 128, 128, 128, 28800, 128, 128, 128, 32768, 32384, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 1280, 128, 128, 128, 128, 31232, 128, 128, 128, 1408, 31872, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 0, 128, 32768, 256, 128, 128, 256, 32768, 128, 128, 0, 32768, 31488, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[7168, 32768, 128, 128, 128, 128, 8192, 128, 128, 30080, 128, 128, 32512, 20736, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 128, 128, 10752, 128, 32768, 6016, 128, 0, 128, 128, 256, 128, 128, 128, 29568, 12032, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 128, 128, 21120, 128, 12032, 128, 32384, 128, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 128, 32768, 128, 32768, 32768, 128, 128, 32640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4009, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 3647, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 804, 4096, 4095, 4096, 0, 0, 0, 1, 0, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 3593, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 36, 0, 4089, 3877, 0, 0, 0, 0, 0, 19, 0, 4046, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 33, 0, 4096, 298, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 2818, 0, 4096, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 391, 0, 3479, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 752, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 11, 0, 4096, 756, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4085, 0, 4093, 3882, 0, 0, 0, 4087, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4093, 2, 4093, 2601, 0, 4095, 0, 0, 2, 4093, 2, 4093, 2, 0, 2, 1, 4, 8, 0, 4093, 0, 4094, 3, 4095, 5, 0, 0, 3, 0, 0, 2, 0, 135, 0, 3192, 4091, 4091, 4095, 0, 3, 0, 0, 0, 0, 0, 0, 4092, 8, 3, 9, 4094, 0, 0, 3850, 0, 0, 0, 2, 4090, 4092, 0, 3, 0, 4045, 0, 0, 43, 0, 4087, 3933, 2, 0, 0, 0, 0, 2, 0, 2554, 0, 2, 4094, 0, 0, 0, 4094, 16, 2, 0, 4, 0, 0, 0, 0, 4096, 328, 0, 0, 1, 1, 4096, 0, 0, 4096, 0, 4095, 1, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4055, 2, 4077, 3425, 7, 38, 0, 4068, 0, 17, 0, 0, 0, 18, 0, 4076, 0, 19, 0, 4075, 0, 4078, 798, 22, 4084, 0, 2, 1, 4068, 0, 4076, 27, 0, 65, 0, 1, 2, 0, 4053, 32, 4078, 8, 4078, 0, 67, 30, 0, 0, 18, 11, 0, 3150, 15, 3730, 4068, 4025, 4057, 20, 0, 17, 11, 0, 0, 0, 0, 4076, 0, 31, 0, 3999, 0, 1, 3751, 0, 19, 13, 3, 4085, 4080, 0, 0, 9, 4058, 7, 0, 1, 14, 4039, 4073, 0, 18, 0, 11, 0, 0, 41, 588, 16, 24, 4073, 23, 20, 16, 4072, 0, 18, 13, 0, 14, 0, 0, 28, 4066, 990, 5, 9, 35, 0, 4072, 0, 10, 3966, 20, 4076, 0, 0, 20, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4090, 0, 4095, 4053, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 4096, 2128, 0, 4096, 0, 0, 0, 4096, 2, 4096, 0, 0, 127, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 829, 0, 1014, 4095, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4094, 3, 40, 3877, 0, 0, 0, 0, 4093, 4093, 4, 0, 0, 4090, 0, 0, 0, 0, 4093, 3803, 0, 0, 0, 0, 0, 4, 0, 3801, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 2, 0, 0, 18, 0, 4096, 828, 0, 0, 0, 0, 4096, 0, 0, 4063, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4095, 3528, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 2461, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 286, 0, 2620, 4096, 4096, 4096, 0, 160, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 3, 0, 4094, 4087, 0, 0, 0, 0, 0, 0, 0, 2492, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 21, 0, 4096, 725, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4084, 2, 4093, 4093, 0, 8, 0, 4088, 0, 0, 1, 0, 2, 3, 4, 4093, 1, 4, 0, 4090, 0, 4092, 3277, 0, 4089, 8, 0, 1, 4092, 9, 4092, 5, 0, 109, 4, 0, 0, 0, 4093, 2, 4091, 0, 4093, 0, 6, 0, 0, 25, 5, 3, 0, 596, 0, 2537, 4087, 4077, 4091, 3, 0, 0, 51, 0, 0, 0, 0, 4091, 18, 2, 0, 4092, 0, 0, 4074, 2, 0, 3, 4, 4089, 4090, 0, 7, 3, 4083, 3, 0, 0, 5, 4028, 4091, 0, 4, 0, 3, 0, 0, 0, 1689, 3, 5, 4093, 0, 1, 0, 4088, 7, 3, 6, 0, 3, 0, 0, 7, 4085, 0, 3, 5, 0, 0, 4091, 0, 1, 4089, 4, 4089, 0, 1, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4090, 0, 4095, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 3230, 0, 4096, 3, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 3300, 4096, 4095, 4096, 0, 1, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4034, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 6, 0, 4095, 4096, 0, 0, 0, 0, 1, 3, 0, 1329, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 211, 0, 0, 1, 0, 4096, 0, 0, 4095, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4009, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1475, 3647, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4086, 0, 0, 0, 0, 0, 2337, 1711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 804, 4096, 4095, 4096, 0, 0, 0, 1, 0, 0, 0, 498, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 3593, 0, 0, 0, 0, 176, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 36, 0, 4089, 3877, 0, 0, 0, 0, 0, 19, 0, 2178, 2010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 33, 0, 4096, 1910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 298, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 3598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16504, 17410, 16326, 14006, 16058, 16305, 18280, 16183] avg=16384.0000 max=18280 max_rank=6 min=14006 min_rank=3 imbalance_ratio=1.115723 max_load_reduction=112792 max_load_reduction_pct=0.860535 moved_to_echo=66826 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 4095, 4053, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 1, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1475, 2128, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 5, 0, 0, 0, 2337, 1711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 829, 0, 1014, 4095, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 498, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4094, 3, 40, 3877, 0, 0, 0, 0, 176, 4093, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4093, 3803, 0, 0, 0, 0, 0, 4, 0, 2178, 2008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 2, 0, 0, 18, 0, 4096, 1909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 828, 0, 0, 0, 0, 4096, 0, 0, 4063, 0, 3598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16538, 15893, 16468, 14911, 16385, 16176, 18020, 16681] avg=16384.0000 max=18020 max_rank=6 min=14911 min_rank=3 imbalance_ratio=1.099854 max_load_reduction=113052 max_load_reduction_pct=0.862518 moved_to_echo=67262 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4085, 0, 4093, 3882, 0, 0, 0, 4087, 0, 1, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 1474, 2601, 0, 0, 0, 0, 2, 4093, 2, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4, 8, 0, 4093, 0, 4094, 3, 4095, 5, 0, 0, 3, 0, 2335, 1710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 135, 0, 3192, 4091, 4091, 4095, 0, 3, 0, 0, 0, 0, 0, 498, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 8, 3, 9, 4094, 0, 0, 3850, 0, 0, 0, 2, 175, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4045, 0, 0, 43, 0, 4087, 3933, 2, 0, 0, 0, 0, 2, 0, 2177, 2007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2554, 0, 2, 4094, 0, 0, 0, 4094, 16, 2, 0, 4, 0, 0, 0, 0, 4095, 1908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 328, 0, 0, 1, 1, 4096, 0, 0, 4096, 0, 3597, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16355, 16362, 16353, 16391, 16325, 16299, 16769, 16218] avg=16384.0000 max=16769 max_rank=6 min=16218 min_rank=7 imbalance_ratio=1.023499 max_load_reduction=114303 max_load_reduction_pct=0.872063 moved_to_echo=65611 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1475, 2818, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2337, 1711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 0, 3479, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 498, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 176, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 2178, 2010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 752, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 11, 0, 4096, 1910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 756, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16581, 16582, 16336, 16940, 16554, 16476, 14961, 16642] avg=16384.0000 max=16940 max_rank=3 min=14961 min_rank=6 imbalance_ratio=1.033936 max_load_reduction=114132 max_load_reduction_pct=0.870758 moved_to_echo=64633 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4055, 2, 4077, 3425, 7, 38, 0, 4068, 0, 17, 0, 0, 0, 18, 0, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4075, 0, 1469, 798, 22, 0, 0, 2, 1, 4068, 0, 4076, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 1, 2, 0, 4053, 32, 4078, 8, 4078, 0, 67, 30, 0, 0, 2327, 1702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 11, 0, 3150, 15, 3730, 4068, 4025, 4057, 20, 0, 17, 11, 0, 0, 0, 496, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 31, 0, 3999, 0, 1, 3751, 0, 19, 13, 3, 175, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4058, 7, 0, 1, 14, 4039, 4073, 0, 18, 0, 11, 0, 0, 41, 2168, 2005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 16, 24, 4073, 23, 20, 16, 4072, 0, 18, 13, 0, 14, 0, 0, 28, 4084, 1905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 990, 5, 9, 35, 0, 4072, 0, 10, 3966, 20, 3580, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15913, 14557, 16443, 19900, 16148, 16444, 14894, 16773] avg=16384.0000 max=19900 max_rank=3 min=14557 min_rank=1 imbalance_ratio=1.214600 max_load_reduction=111172 max_load_reduction_pct=0.848175 moved_to_echo=64259 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4084, 2, 4093, 4093, 0, 8, 0, 4088, 0, 0, 1, 0, 2, 3, 4, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4090, 0, 1469, 3277, 0, 0, 8, 0, 1, 4092, 9, 4092, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 4, 0, 0, 0, 4093, 2, 4091, 0, 4093, 0, 6, 0, 0, 25, 2337, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 0, 596, 0, 2537, 4087, 4077, 4091, 3, 0, 0, 51, 0, 0, 0, 504, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 18, 2, 0, 4092, 0, 0, 4074, 2, 0, 3, 4, 166, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 4083, 3, 0, 0, 5, 4028, 4091, 0, 4, 0, 3, 0, 0, 0, 2182, 2010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1689, 3, 5, 4093, 0, 1, 0, 4088, 7, 3, 6, 0, 3, 0, 0, 7, 4089, 1913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 3, 5, 0, 0, 4091, 0, 1, 4089, 4, 3585, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16576, 17048, 16473, 16240, 16542, 16419, 15907, 15867] avg=16384.0000 max=17048 max_rank=1 min=15867 min_rank=7 imbalance_ratio=1.040527 max_load_reduction=114024 max_load_reduction_pct=0.869934 moved_to_echo=64735 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4095, 3528, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1475, 2461, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2337, 1711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 286, 0, 2620, 4096, 4096, 4096, 0, 160, 0, 0, 0, 0, 0, 498, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 176, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 4094, 4087, 0, 0, 0, 0, 0, 0, 0, 2178, 2010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2492, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 21, 0, 4096, 1910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 725, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16021, 16224, 16336, 16136, 16564, 16468, 16712, 16611] avg=16384.0000 max=16712 max_rank=6 min=16021 min_rank=0 imbalance_ratio=1.020020 max_load_reduction=114360 max_load_reduction_pct=0.872498 moved_to_echo=66355 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 4095, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1475, 3230, 0, 0, 3, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 2337, 1711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 3300, 4096, 4095, 4096, 0, 1, 0, 0, 1, 0, 0, 498, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4034, 0, 0, 0, 0, 176, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 0, 4095, 4096, 0, 0, 0, 0, 1, 3, 0, 2178, 2010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1329, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 1910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 211, 0, 0, 1, 0, 4096, 0, 0, 4095, 0, 3598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16584, 16996, 16337, 16548, 16496, 16485, 15529, 16097] avg=16384.0000 max=16996 max_rank=1 min=15529 min_rank=6 imbalance_ratio=1.037354 max_load_reduction=114076 max_load_reduction_pct=0.870331 moved_to_echo=64607 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 32768, 31232, 128, 128, 0, 32768, 0, 128, 128, 0, 128, 128, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 128, 128, 128, 32768, 128, 128, 31488, 128, 128, 128, 128, 1408, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 32768, 128, 11904, 20992, 128, 0, 128, 128, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 4224, 128, 128, 128, 128, 32768, 0, 128, 32640, 128, 28800, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 32768, 128, 0, 128, 128, 32640, 32128, 128, 128, 0, 128, 128, 128, 128, 17536, 16128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[17280, 128, 128, 32768, 128, 128, 128, 32768, 128, 128, 128, 128, 128, 0, 128, 128, 32768, 15360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 384, 128, 128, 128, 0, 32768, 128, 32768, 128, 32768, 128, 128, 128, 128, 128, 18688, 13696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 5760, 128, 20736, 32768, 32768, 32768, 128, 256, 128, 128, 128, 0, 0, 4096, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 1, 4041, 0, 50, 1370, 4096, 0, 4096, 4096, 4092, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 0, 0, 0, 4068, 0, 0, 3273, 0, 111, 0, 0, 3, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2913, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4018, 644, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4068, 4096, 0, 4092, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4068, 4082, 22, 24, 2, 31, 4075, 0, 4072, 0, 0, 4032, 19, 0, 1498, 4059, 0, 4084, 4050, 4056, 0, 4071, 4084, 0, 14, 0, 1, 20, 1, 4045, 6, 27, 30, 0, 0, 0, 4002, 4072, 19, 29, 0, 0, 88, 0, 57, 4075, 0, 26, 0, 0, 0, 3989, 0, 19, 672, 0, 18, 25, 4076, 13, 4, 3779, 17, 231, 11, 55, 206, 0, 12, 0, 4, 13, 25, 4067, 0, 12, 0, 20, 21, 17, 2693, 47, 20, 12, 3975, 36, 0, 4066, 0, 0, 23, 8, 4045, 0, 0, 27, 20, 4071, 4070, 6, 4063, 0, 0, 0, 57, 16, 0, 4003, 31, 0, 4081, 0, 40, 0, 0, 6, 16, 4052, 14, 0, 21, 0, 3244, 4062, 0, 4081, 19] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4083, 4093, 6, 6, 2, 7, 4087, 0, 4088, 0, 0, 4090, 3, 29, 3721, 4092, 0, 4089, 4091, 4090, 0, 4086, 4093, 0, 2, 1, 2, 5, 2, 4087, 0, 10, 8, 0, 4, 0, 4077, 4076, 4, 0, 11, 0, 6, 0, 0, 4089, 10, 8, 0, 0, 16, 4080, 0, 0, 0, 0, 5, 4, 3969, 0, 2, 943, 4, 63, 3, 12, 86, 3, 2, 0, 2, 8, 4, 4088, 0, 2, 0, 7, 0, 0, 979, 86, 0, 0, 4073, 51, 0, 4090, 0, 0, 6, 1, 4086, 2, 6, 9, 5, 4090, 4091, 3, 4091, 0, 0, 0, 50, 5, 0, 4024, 3420, 5, 4091, 0, 6, 1, 0, 0, 1, 4079, 3, 0, 5, 0, 3070, 4084, 0, 4093, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 10, 4070, 4096, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 3329, 0, 0, 1348, 0, 4, 0, 0, 754, 0, 577, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3371, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 1, 8, 0, 0, 4096, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2906, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4093, 4094, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 4096, 0, 100, 3880, 4094, 2, 4096, 4096, 4095, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 49, 4096, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 1504, 0, 3, 0, 0, 24, 1, 192, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2498, 2, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 4096, 0, 4096, 2, 0, 30, 0, 0, 0, 4096, 442, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 3600, 4096, 0, 4090, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4091, 4090, 2, 0, 0, 0, 4093, 0, 4096, 0, 0, 4088, 0, 3, 3403, 4086, 23, 4094, 4094, 4096, 0, 4094, 4094, 0, 2, 0, 0, 2, 0, 4093, 0, 0, 1, 0, 0, 0, 4094, 4076, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 1, 0, 0, 3, 4095, 4, 0, 86, 10, 2, 0, 4051, 2, 1, 1388, 0, 0, 0, 0, 580, 0, 46, 0, 2, 0, 4, 4096, 0, 0, 0, 2, 0, 0, 3994, 0, 0, 0, 4088, 6, 0, 4095, 0, 0, 1, 0, 4094, 2, 0, 0, 1, 4094, 4093, 0, 4093, 0, 0, 0, 6, 0, 0, 4060, 189, 0, 4093, 1, 3, 3, 0, 0, 0, 4096, 0, 0, 0, 3, 2712, 4094, 0, 4073, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 4095, 0, 324, 2687, 4096, 0, 4096, 4096, 4094, 0, 4096, 4051, 0, 0, 0, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 3581, 0, 87, 0, 0, 36, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3468, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 2, 0, 0, 4096, 722, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1399, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4095, 0, 0, 0, 0, 4095, 0, 4096, 0, 2, 4096, 0, 1, 1906, 4090, 2, 4096, 4096, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 115, 4096, 0, 0, 12, 0, 0, 0, 3981, 0, 0, 3358, 0, 0, 0, 0, 136, 0, 12, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3379, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 1, 0, 0, 1, 0, 0, 4096, 698, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2793, 4096, 0, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4095, 0, 1228, 0, 1, 4041, 0, 50, 1370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2771, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 0, 0, 0, 4068, 0, 0, 3273, 0, 4096, 1308, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 3, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2913, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3344, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4018, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4068, 4096, 0, 4092, 0, 752, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14881, 16401, 16378, 17269, 16091, 16378, 16198, 17476] avg=16384.0000 max=17476 max_rank=7 min=14881 min_rank=0 imbalance_ratio=1.066650 max_load_reduction=113596 max_load_reduction_pct=0.866669 moved_to_echo=66143 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4083, 0, 6, 6, 2, 7, 4087, 0, 1220, 0, 0, 4090, 3, 29, 3721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 9, 4090, 0, 0, 4093, 0, 2, 1, 2, 5, 2, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8, 0, 4, 0, 4077, 4076, 4, 0, 11, 0, 6, 0, 0, 4089, 10, 2770, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 16, 4080, 0, 0, 0, 0, 5, 4, 3969, 0, 2, 943, 4, 4086, 1312, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 3, 12, 86, 3, 2, 0, 2, 8, 4, 4088, 0, 2, 0, 7, 0, 4089, 4091, 3683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 979, 86, 0, 0, 4073, 51, 0, 4090, 0, 0, 6, 1, 4086, 2, 6, 1175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 5, 3333, 4091, 3, 4091, 0, 0, 0, 50, 5, 0, 4024, 3420, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 1, 4079, 3, 0, 5, 0, 3070, 4084, 0, 4093, 5, 757, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17254, 16383, 16383, 14839, 16143, 14555, 19036, 16479] avg=16384.0000 max=19036 max_rank=6 min=14555 min_rank=5 imbalance_ratio=1.161865 max_load_reduction=112036 max_load_reduction_pct=0.854767 moved_to_echo=66213 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4068, 0, 22, 24, 2, 31, 4075, 0, 1221, 0, 0, 4032, 19, 0, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 17, 4056, 0, 0, 4084, 0, 14, 0, 1, 20, 1, 4045, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 30, 0, 0, 0, 4002, 4072, 19, 29, 0, 0, 88, 0, 57, 4075, 0, 2740, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 3989, 0, 19, 672, 0, 18, 25, 4076, 13, 4, 3779, 17, 4071, 1293, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 11, 55, 206, 0, 12, 0, 4, 13, 25, 4067, 0, 12, 0, 20, 21, 4084, 4081, 3674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2693, 47, 20, 12, 3975, 36, 0, 4066, 0, 0, 23, 8, 4045, 0, 0, 1170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 20, 3323, 4070, 6, 4063, 0, 0, 0, 57, 16, 0, 4003, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 6, 16, 4052, 14, 0, 21, 0, 3244, 4062, 0, 4081, 19, 748, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14992, 16303, 16450, 18410, 16516, 16112, 15616, 16673] avg=16384.0000 max=18410 max_rank=3 min=14992 min_rank=0 imbalance_ratio=1.123657 max_load_reduction=112662 max_load_reduction_pct=0.859543 moved_to_echo=64917 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 0, 4093, 0, 1228, 0, 0, 4096, 0, 100, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 17, 4095, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2771, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 4096, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 1504, 0, 4096, 1308, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 24, 1, 192, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2498, 2, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3344, 4096, 0, 4096, 2, 0, 30, 0, 0, 0, 4096, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 3600, 4096, 0, 4090, 0, 752, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17490, 16402, 16377, 15543, 16192, 15957, 16106, 17005] avg=16384.0000 max=17490 max_rank=0 min=15543 min_rank=3 imbalance_ratio=1.067505 max_load_reduction=113582 max_load_reduction_pct=0.866562 moved_to_echo=65260 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 0, 2, 0, 0, 0, 4093, 0, 1228, 0, 0, 4088, 0, 3, 3403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 23, 0, 18, 4096, 0, 0, 4094, 0, 2, 0, 0, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 4076, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 2769, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 4095, 4, 0, 86, 10, 2, 0, 4051, 2, 1, 1388, 0, 4094, 1307, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 580, 0, 46, 0, 2, 0, 4, 4096, 0, 0, 0, 2, 0, 4094, 4093, 3680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 4088, 6, 0, 4095, 0, 0, 1, 0, 4094, 2, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3342, 4093, 0, 4093, 0, 0, 0, 6, 0, 0, 4060, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 3, 0, 0, 0, 4096, 0, 0, 0, 3, 2712, 4094, 0, 4073, 0, 752, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16908, 16414, 16349, 15454, 16597, 17457, 15784, 16109] avg=16384.0000 max=17457 max_rank=5 min=15454 min_rank=3 imbalance_ratio=1.065491 max_load_reduction=113615 max_load_reduction_pct=0.866814 moved_to_echo=65947 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1228, 0, 0, 4096, 0, 10, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2771, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 3329, 0, 0, 1348, 0, 4096, 1308, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 754, 0, 577, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3371, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3344, 4096, 0, 4096, 0, 0, 1, 8, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2906, 4096, 0, 4096, 0, 752, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17596, 16401, 16378, 14589, 17310, 16836, 15644, 16318] avg=16384.0000 max=17596 max_rank=0 min=14589 min_rank=3 imbalance_ratio=1.073975 max_load_reduction=113476 max_load_reduction_pct=0.865753 moved_to_echo=66108 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1228, 0, 1, 4095, 0, 324, 2687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 4094, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2771, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 3581, 0, 4096, 1308, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 36, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3468, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3344, 4096, 0, 4096, 0, 0, 0, 2, 0, 0, 4096, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1399, 4096, 0, 4096, 0, 752, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16527, 16373, 16378, 17592, 16102, 16933, 16356, 14811] avg=16384.0000 max=17592 max_rank=3 min=14811 min_rank=7 imbalance_ratio=1.073730 max_load_reduction=113480 max_load_reduction_pct=0.865784 moved_to_echo=64202 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 4095, 0, 1228, 0, 2, 4096, 0, 1, 1906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 17, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 2771, 1319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 4096, 0, 0, 12, 0, 0, 0, 3981, 0, 0, 3358, 0, 4096, 1308, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 12, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3379, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3344, 4096, 0, 4096, 1, 0, 0, 1, 0, 0, 4096, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2793, 4096, 0, 4094, 0, 752, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15424, 16395, 16379, 17376, 16121, 16844, 16332, 16201] avg=16384.0000 max=17376 max_rank=3 min=15424 min_rank=0 imbalance_ratio=1.060547 max_load_reduction=113696 max_load_reduction_pct=0.867432 moved_to_echo=65498 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 0, 256, 32768, 0, 0, 32768, 0, 128, 128, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32768, 0, 128, 128, 128, 128, 32768, 0, 9856, 0, 128, 32640, 128, 640, 22656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 128, 32768, 32768, 128, 128, 128, 0, 128, 0, 128, 32768, 128, 22144, 10624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 23296, 256, 128, 128, 32640, 128, 0, 32768, 0, 0, 128, 128, 32768, 128, 128, 9472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 26752, 32768, 128, 32768, 128, 0, 128, 128, 128, 0, 32512, 6272, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 0, 256, 32768, 128, 128, 896, 128, 128, 128, 31744, 128, 128, 19200, 128, 32768, 10496, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 128, 0, 128, 128, 23808, 32768, 0, 32768, 128, 6144, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[512, 128, 128, 1920, 128, 896, 0, 128, 128, 128, 32768, 0, 128, 0, 128, 128, 32768, 32768, 29568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 13, 0, 4094, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 83, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 4096, 0, 65, 58, 13, 0, 0, 4059, 0, 0, 4095, 4096, 4094, 6, 0, 4, 0, 3956, 4096, 1, 7, 0, 2, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 4092, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 4093, 3, 4079] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1218, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 4096, 0, 1260, 0, 5, 0, 0, 3991, 0, 0, 4096, 4094, 4096, 0, 0, 3, 0, 2879, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 4096, 0, 0, 0, 0, 0, 4059, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 2973, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 148, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 186, 0, 7, 0, 0, 4064, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 3827, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4094, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 4059, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[8, 1, 4072, 8, 18, 0, 26, 4, 12, 0, 30, 4076, 4074, 4010, 4075, 0, 28, 10, 6, 0, 24, 1, 0, 26, 0, 24, 8, 35, 3, 36, 24, 4049, 0, 40, 4061, 16, 3, 22, 0, 0, 4049, 21, 25, 4074, 20, 0, 47, 42, 35, 0, 70, 0, 4050, 21, 4070, 0, 3890, 4050, 0, 7, 18, 4060, 0, 32, 0, 0, 22, 0, 4073, 0, 301, 0, 1211, 44, 4, 4093, 44, 14, 4067, 4038, 4060, 0, 0, 0, 0, 639, 4068, 0, 62, 0, 0, 0, 0, 4, 4070, 23, 0, 22, 0, 0, 29, 3, 0, 4017, 4073, 4063, 14, 0, 90, 0, 29, 4068, 4064, 0, 0, 0, 2, 4065, 4071, 4074, 0, 4065, 1, 25, 9, 2125, 0, 3915] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 875, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 168, 8, 32, 0, 0, 4041, 0, 0, 4096, 4096, 4096, 0, 0, 12, 0, 3168, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 4054, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 4078, 0, 4003] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[2, 3, 4090, 11, 4, 2, 10, 0, 8, 0, 6, 4087, 4091, 4088, 4090, 0, 6, 0, 2, 0, 5, 3, 0, 2, 0, 5, 0, 5, 3, 6, 8, 4089, 0, 0, 4087, 4, 0, 4, 16, 0, 4085, 4, 9, 4089, 6, 0, 6, 7, 0, 0, 13, 0, 4089, 0, 4089, 0, 1196, 4084, 19, 1, 1, 4087, 0, 7, 0, 0, 5, 0, 4088, 0, 3389, 0, 0, 11, 1, 4093, 10, 5, 4089, 4081, 4085, 0, 0, 0, 0, 1807, 4086, 29, 0, 1, 0, 3, 0, 3, 4061, 2, 0, 7, 0, 0, 10, 4, 0, 3983, 4092, 4088, 4, 0, 5, 0, 0, 4094, 4090, 0, 2, 2, 1, 4091, 4091, 4091, 0, 4091, 1, 6, 1, 1924, 0, 4016] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[2, 0, 4092, 0, 0, 1, 0, 1, 0, 0, 2, 4088, 4095, 4095, 4093, 0, 0, 1, 0, 0, 2, 0, 0, 1, 0, 1, 0, 0, 0, 0, 2, 4096, 2, 0, 4094, 0, 16, 0, 0, 3, 4076, 0, 0, 4095, 0, 0, 2, 1, 0, 2, 4, 0, 4095, 0, 4093, 2, 1890, 4094, 0, 0, 2, 4092, 0, 0, 2, 0, 0, 2, 4094, 5, 464, 56, 877, 0, 0, 4073, 0, 0, 4094, 4094, 4094, 0, 0, 0, 0, 1689, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 2, 2, 1, 0, 0, 0, 1, 4093, 4094, 4093, 0, 0, 0, 0, 0, 4086, 4094, 0, 8, 3, 2, 4094, 4095, 4096, 2, 4094, 0, 1, 3, 3238, 3, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2, 0, 0, 0, 4095, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3760, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 324, 3, 23, 0, 0, 4087, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1761, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 2334, 1, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4094, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 13, 0, 4094, 0, 0, 4091, 0, 0, 0, 0, 3054, 1008, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2460, 0, 83, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3070, 0, 65, 58, 13, 0, 0, 4059, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 4, 0, 3956, 4096, 1, 7, 0, 2, 0, 0, 0, 4073, 0, 1004, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 4092, 0, 0, 0, 0, 0, 4088, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1042, 0, 0, 0, 4096, 0, 0, 0, 4093, 3, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16365, 16374, 14821, 15460, 17871, 16391, 17409] avg=16384.0000 max=17871 max_rank=5 min=14821 min_rank=3 imbalance_ratio=1.090759 max_load_reduction=113201 max_load_reduction_pct=0.863655 moved_to_echo=67131 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 3054, 1008, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2460, 0, 1218, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3070, 0, 1260, 0, 5, 0, 0, 3991, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 2879, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1004, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 4096, 0, 0, 0, 0, 0, 4059, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1042, 0, 0, 0, 4096, 0, 0, 0, 2973, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16369, 16366, 15966, 16520, 16797, 16368, 16303] avg=16384.0000 max=16797 max_rank=5 min=15966 min_rank=3 imbalance_ratio=1.025208 max_load_reduction=114275 max_load_reduction_pct=0.871849 moved_to_echo=65988 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[8, 1, 4072, 8, 18, 0, 26, 4, 12, 0, 30, 4076, 0, 4010, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 10, 6, 0, 24, 1, 0, 26, 0, 24, 8, 35, 3, 36, 24, 4049, 4074, 4074, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 4061, 16, 3, 22, 0, 0, 4049, 21, 25, 4074, 20, 0, 47, 42, 3031, 1002, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 70, 0, 4050, 21, 2444, 0, 3890, 4050, 0, 7, 18, 4060, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 3052, 0, 301, 0, 1211, 44, 4, 4093, 44, 14, 4067, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 639, 4068, 0, 62, 0, 0, 0, 0, 4, 4070, 23, 999, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 29, 3, 0, 4017, 4073, 4063, 14, 0, 90, 0, 29, 4068, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 2, 1034, 0, 0, 0, 4065, 1, 25, 9, 2125, 0, 3915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16339, 16476, 16470, 18677, 16890, 14549, 16431, 15240] avg=16384.0000 max=18677 max_rank=3 min=14549 min_rank=5 imbalance_ratio=1.139954 max_load_reduction=112395 max_load_reduction_pct=0.857506 moved_to_echo=63110 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3054, 1008, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2460, 0, 148, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3070, 0, 186, 0, 7, 0, 0, 4064, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3827, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1004, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4094, 0, 0, 0, 0, 0, 4096, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1042, 0, 0, 0, 4096, 0, 0, 0, 4059, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16366, 16368, 14896, 15519, 17747, 16404, 17389] avg=16384.0000 max=17747 max_rank=5 min=14896 min_rank=3 imbalance_ratio=1.083191 max_load_reduction=113325 max_load_reduction_pct=0.864601 moved_to_echo=67059 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 3, 4090, 11, 4, 2, 10, 0, 8, 0, 6, 4087, 0, 4088, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 5, 3, 0, 2, 0, 5, 0, 5, 3, 6, 8, 4089, 4091, 4091, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4, 0, 4, 16, 0, 4085, 4, 9, 4089, 6, 0, 6, 7, 3056, 1012, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4089, 0, 2445, 0, 1196, 4084, 19, 1, 1, 4087, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3054, 0, 3389, 0, 0, 11, 1, 4093, 10, 5, 4089, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 1807, 4086, 29, 0, 1, 0, 3, 0, 3, 4061, 2, 1008, 632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 10, 4, 0, 3983, 4092, 4088, 4, 0, 5, 0, 0, 4094, 26, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 2, 2, 1, 1035, 0, 0, 0, 4091, 1, 6, 1, 1924, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16399, 16390, 16402, 15942, 18738, 15717, 16315, 15169] avg=16384.0000 max=18738 max_rank=4 min=15169 min_rank=7 imbalance_ratio=1.143677 max_load_reduction=112334 max_load_reduction_pct=0.857040 moved_to_echo=65939 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4092, 0, 0, 1, 0, 1, 0, 0, 2, 4088, 0, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 1, 0, 1, 0, 0, 0, 0, 2, 4096, 4096, 4095, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 16, 0, 0, 3, 4076, 0, 0, 4095, 0, 0, 2, 1, 3053, 1008, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 4095, 0, 2457, 2, 1890, 4094, 0, 0, 2, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 3068, 5, 464, 56, 877, 0, 0, 4073, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1689, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 1004, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 1, 0, 0, 0, 1, 4093, 4094, 4093, 0, 0, 0, 0, 0, 4086, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 8, 3, 2, 1041, 0, 0, 2, 4094, 0, 1, 3, 3238, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 16372, 16367, 16638, 16735, 15609, 16395, 16583] avg=16384.0000 max=16735 max_rank=4 min=15609 min_rank=5 imbalance_ratio=1.021423 max_load_reduction=114337 max_load_reduction_pct=0.872322 moved_to_echo=65322 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 3054, 1008, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2460, 0, 875, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3070, 0, 168, 8, 32, 0, 0, 4041, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 0, 3168, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1004, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 4054, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1042, 0, 0, 0, 4096, 0, 0, 0, 4078, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16431, 16367, 16360, 15624, 15511, 17100, 16364, 17315] avg=16384.0000 max=17315 max_rank=7 min=15511 min_rank=4 imbalance_ratio=1.056824 max_load_reduction=113757 max_load_reduction_pct=0.867897 moved_to_echo=66290 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4095, 0, 0, 4093, 0, 0, 0, 0, 3054, 1008, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2460, 0, 3760, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3070, 0, 324, 3, 23, 0, 0, 4087, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1761, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 1004, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 4094, 22, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1042, 0, 0, 0, 4096, 0, 0, 0, 2334, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16367, 16365, 18508, 15699, 15682, 16404, 15664] avg=16384.0000 max=18508 max_rank=3 min=15664 min_rank=7 imbalance_ratio=1.129639 max_load_reduction=112564 max_load_reduction_pct=0.858795 moved_to_echo=63449 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 128, 128, 128, 0, 128, 32768, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 0, 128, 128, 24448, 8064, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 32640, 32768, 32768, 128, 0, 128, 0, 128, 32640, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 0, 128, 128, 128, 8320, 0, 0, 128, 32768, 128, 128, 128, 24832, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 19712, 128, 13184, 32768, 128, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 0, 128, 0, 19840, 32768, 128, 128, 128, 128, 128, 0, 128, 32768, 128, 8064, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 32768, 32768, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 24576, 128, 6272, 128, 2176, 128, 128, 32512, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 3991, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4093, 0, 763, 4096, 4096, 4096, 4096, 0, 2, 3427, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 15, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 0, 4096, 4082, 0, 4096, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 3512, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4093, 0, 2809, 4096, 4096, 4096, 4096, 0, 336, 289, 9, 47, 0, 0, 0, 0, 4096, 0, 4072, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4092, 0, 0, 3, 0, 4096, 0, 0, 1283, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[21, 26, 0, 0, 8, 0, 0, 3960, 27, 26, 4071, 8, 31, 22, 0, 4076, 0, 0, 4064, 0, 228, 14, 0, 4066, 31, 0, 3, 32, 4065, 0, 4056, 4058, 0, 4067, 29, 31, 3949, 0, 16, 0, 36, 0, 48, 0, 4069, 28, 12, 4062, 30, 0, 0, 0, 33, 4053, 0, 49, 4021, 24, 0, 0, 30, 6, 0, 31, 36, 0, 15, 0, 30, 4081, 0, 1, 107, 0, 0, 16, 0, 21, 0, 4067, 0, 4004, 42, 4060, 0, 1363, 4070, 4061, 4063, 4068, 2, 86, 2476, 0, 0, 0, 25, 6, 2, 4074, 0, 4079, 0, 4066, 10, 0, 22, 0, 3994, 28, 4060, 7, 29, 4048, 4069, 38, 34, 24, 4068, 0, 22, 0, 0, 4030, 0, 0, 141, 10] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[6, 5, 1, 0, 0, 0, 0, 4062, 4, 3, 4089, 0, 3, 3, 0, 4092, 0, 0, 4087, 0, 8, 2, 7, 4089, 4, 3, 8, 3, 4083, 0, 4088, 3978, 0, 4091, 9, 0, 4090, 0, 11, 0, 5, 0, 0, 0, 4090, 13, 0, 4087, 7, 25, 0, 4, 7, 4053, 0, 0, 4076, 7, 0, 0, 4, 4, 0, 7, 3, 5, 4, 1, 3, 4093, 2, 0, 21, 0, 0, 0, 0, 4, 0, 4090, 0, 4090, 0, 4045, 3, 3851, 4092, 4089, 4090, 4090, 19, 32, 86, 0, 56, 0, 3, 7, 2, 4089, 0, 4092, 0, 4089, 0, 1, 4, 0, 4057, 5, 4084, 2, 8, 4087, 4074, 6, 8, 6, 4092, 0, 3, 0, 0, 4088, 0, 0, 276, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4077, 0, 4096, 4087, 0, 4096, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4055, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 3775, 0, 3927, 4096, 4096, 4096, 4096, 0, 53, 346, 0, 88, 0, 0, 0, 0, 4095, 0, 4081, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 112, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3, 4096, 4082, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 20, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4080, 0, 3921, 4096, 4096, 4096, 4096, 0, 101, 73, 2, 10, 0, 0, 0, 0, 4095, 0, 4082, 4, 4096, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 4096, 4093, 0, 0, 0, 4096, 0, 0, 17, 0, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 2, 2, 0, 4094, 1, 0, 4094, 1, 0, 1, 6, 4094, 1, 0, 4095, 3, 81, 0, 0, 4090, 1, 2, 0, 4, 4095, 0, 4095, 4094, 0, 4092, 0, 0, 4095, 15, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 4094, 0, 0, 0, 0, 1, 3778, 0, 0, 4095, 2, 0, 0, 2, 1, 4, 0, 0, 3, 1, 0, 2, 4071, 0, 0, 0, 1, 0, 0, 0, 0, 1, 4095, 5, 4055, 0, 4096, 0, 3669, 4091, 4095, 4095, 4094, 4, 0, 704, 0, 0, 1, 0, 0, 1, 4094, 0, 4094, 0, 4094, 0, 0, 2, 0, 4094, 0, 4094, 0, 0, 4094, 4090, 2, 0, 0, 4095, 0, 1, 0, 2, 4079, 2, 0, 20, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 4066, 0, 4096, 4090, 0, 4096, 0, 0, 4080, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 2516, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4093, 0, 3963, 0, 1923, 4096, 4096, 4096, 4096, 0, 89, 3657, 1, 2, 0, 0, 0, 0, 4096, 0, 4095, 3, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 4090, 1, 0, 0, 0, 4096, 0, 0, 192, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 3769, 250, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3512, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 3786, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 2809, 0, 4003, 0, 0, 0, 336, 289, 9, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4072, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3842, 0, 0, 3, 0, 4096, 0, 0, 1283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16366, 16357, 16110, 16369, 15682, 16383, 17416] avg=16384.0000 max=17416 max_rank=7 min=15682 min_rank=5 imbalance_ratio=1.062988 max_load_reduction=113656 max_load_reduction_pct=0.867126 moved_to_echo=65850 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 3769, 250, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 3786, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4093, 0, 763, 0, 4003, 0, 0, 0, 2, 3427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3846, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16288, 16395, 16691, 16370, 16383, 16407, 16149] avg=16384.0000 max=16691 max_rank=3 min=16149 min_rank=7 imbalance_ratio=1.018738 max_load_reduction=114381 max_load_reduction_pct=0.872658 moved_to_echo=65309 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[21, 26, 0, 0, 8, 0, 0, 3960, 27, 26, 4071, 8, 31, 22, 0, 4076, 3749, 249, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 228, 14, 0, 4066, 31, 0, 3, 32, 4065, 0, 4056, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 29, 31, 3949, 0, 16, 0, 36, 0, 48, 0, 4069, 28, 12, 4062, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 33, 4053, 0, 49, 4021, 24, 0, 0, 30, 6, 0, 31, 4070, 4068, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 15, 0, 30, 4081, 0, 1, 107, 0, 0, 16, 0, 21, 0, 4067, 4056, 3755, 302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 42, 4060, 0, 1363, 0, 3969, 0, 0, 2, 86, 2476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 6, 2, 23, 0, 4079, 0, 4066, 10, 0, 22, 0, 3994, 28, 4060, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4048, 4069, 38, 34, 24, 3819, 0, 22, 0, 0, 4030, 0, 0, 141, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16356, 16561, 16357, 16723, 16487, 16002, 16322, 16264] avg=16384.0000 max=16723 max_rank=3 min=16002 min_rank=5 imbalance_ratio=1.020691 max_load_reduction=114349 max_load_reduction_pct=0.872414 moved_to_echo=65075 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[6, 5, 1, 0, 0, 0, 0, 4062, 4, 3, 4089, 0, 3, 3, 0, 4092, 3767, 255, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8, 2, 7, 4089, 4, 3, 8, 3, 4083, 0, 4088, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 9, 0, 4090, 0, 11, 0, 5, 0, 0, 0, 4090, 13, 0, 4087, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 25, 0, 4, 7, 4053, 0, 0, 4076, 7, 0, 0, 4, 4, 0, 7, 4092, 4090, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 4, 1, 3, 4093, 2, 0, 21, 0, 0, 0, 0, 4, 0, 4090, 4080, 3779, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4045, 3, 3851, 0, 3988, 0, 0, 19, 32, 86, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 2, 16, 0, 4092, 0, 4089, 0, 1, 4, 0, 4057, 5, 4084, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4087, 4074, 6, 8, 6, 3837, 0, 3, 0, 0, 4088, 0, 0, 276, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16280, 16408, 16687, 16391, 16170, 16362, 16395] avg=16384.0000 max=16687 max_rank=3 min=16170 min_rank=5 imbalance_ratio=1.018494 max_load_reduction=114385 max_load_reduction_pct=0.872688 moved_to_echo=65318 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4091, 3769, 250, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 3785, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4080, 0, 3921, 0, 4003, 0, 0, 0, 101, 73, 2, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4082, 4, 4096, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 3846, 0, 0, 17, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16381, 16395, 16715, 16366, 16286, 16397, 16147] avg=16384.0000 max=16715 max_rank=3 min=16147 min_rank=7 imbalance_ratio=1.020203 max_load_reduction=114357 max_load_reduction_pct=0.872475 moved_to_echo=65196 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 2, 2, 0, 4094, 1, 0, 4094, 1, 0, 1, 6, 4094, 3768, 250, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 3, 81, 0, 0, 4090, 1, 2, 0, 4, 4095, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4095, 15, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 4094, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3778, 0, 0, 4095, 2, 0, 0, 2, 1, 4, 0, 4091, 4094, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 2, 4071, 0, 0, 0, 1, 0, 0, 0, 0, 1, 4095, 4087, 3785, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4055, 0, 4096, 0, 3669, 0, 4002, 0, 0, 4, 0, 704, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 22, 0, 4094, 0, 4094, 0, 0, 2, 0, 4094, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4090, 2, 0, 0, 3845, 0, 1, 0, 2, 4079, 2, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 16474, 16401, 16378, 16350, 16536, 16401, 16135] avg=16384.0000 max=16536 max_rank=5 min=16135 min_rank=7 imbalance_ratio=1.009277 max_load_reduction=114536 max_load_reduction_pct=0.873840 moved_to_echo=65422 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 3769, 250, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4077, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 3785, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3775, 0, 3927, 0, 4003, 0, 0, 0, 53, 346, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4081, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3846, 0, 0, 0, 0, 4096, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16364, 16377, 16654, 16369, 16282, 16391, 16246] avg=16384.0000 max=16654 max_rank=3 min=16246 min_rank=7 imbalance_ratio=1.016479 max_load_reduction=114418 max_load_reduction_pct=0.872940 moved_to_echo=65288 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 3769, 250, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 4066, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2516, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4088, 3786, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3963, 0, 1923, 0, 4003, 0, 0, 0, 89, 3657, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 4095, 3, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 3840, 1, 0, 0, 0, 4096, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16358, 16382, 15114, 16370, 17731, 16409, 16320] avg=16384.0000 max=17731 max_rank=5 min=15114 min_rank=3 imbalance_ratio=1.082214 max_load_reduction=113341 max_load_reduction_pct=0.864723 moved_to_echo=66830 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32768, 128, 128, 32640, 128, 128, 0, 128, 0, 128, 0, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 128, 30208, 0, 128, 32768, 128, 0, 0, 128, 128, 128, 128, 32768, 32768, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 32640, 128, 128, 32768, 128, 128, 128, 128, 32768, 30208, 2048, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 256, 0, 32768, 128, 32768, 128, 128, 128, 128, 32640, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32640, 128, 32256, 128, 22272, 0, 32000, 0, 0, 128, 768, 11136, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 30848, 128, 128, 128, 128, 32768, 128, 0, 2048, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 128, 384, 128, 128, 32768, 128, 128, 128, 128, 32768, 128, 32768, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 128, 128, 128, 128, 0, 128, 0, 128, 128, 32768, 32768, 30336, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[1169, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 1, 0, 4096, 4096, 0, 4096, 0, 3693, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 14, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 144, 4096, 0, 4096, 0, 0, 3172, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[2, 4096, 0, 4079, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4080, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 39, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3933, 0, 0, 4096, 0, 0, 0, 0, 1575, 4096, 0, 4096, 0, 0, 2671, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[774, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4080, 6, 0, 0, 11, 0, 4096, 59, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 85, 4096, 0, 4096, 0, 0, 3172, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[2041, 3934, 44, 3986, 0, 0, 26, 4048, 0, 4041, 0, 0, 0, 23, 4073, 25, 4073, 0, 0, 0, 14, 0, 4042, 4065, 25, 4037, 4061, 18, 30, 0, 3996, 4060, 0, 4065, 45, 3459, 0, 30, 32, 399, 27, 4032, 6, 0, 0, 0, 3986, 35, 0, 37, 62, 4059, 0, 33, 0, 0, 71, 2, 0, 0, 4069, 4038, 0, 0, 4055, 23, 36, 0, 0, 20, 35, 0, 4041, 4063, 13, 27, 73, 0, 8, 0, 5, 1, 22, 4047, 0, 14, 59, 8, 2722, 29, 123, 4060, 15, 31, 5, 25, 2719, 4065, 45, 4050, 31, 0, 877, 50, 24, 0, 4024, 18, 0, 0, 0, 17, 0, 0, 4067, 133, 0, 16, 0, 52, 26, 0, 0, 4064, 4061, 22, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[174, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4092, 5, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 3, 4096, 4096, 0, 4095, 4093, 0, 0, 0, 4096, 4096, 0, 4096, 0, 3967, 4, 0, 0, 207, 0, 4096, 4, 1, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 150, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 1139, 4096, 0, 4096, 0, 0, 2561, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[377, 4092, 1, 4096, 4, 0, 2, 4094, 0, 4095, 0, 2, 2, 0, 4093, 0, 4096, 0, 0, 0, 0, 1, 4094, 4095, 2, 4089, 4091, 0, 0, 0, 4093, 4094, 4, 4094, 0, 4078, 0, 0, 1, 78, 0, 4096, 214, 0, 0, 0, 4065, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 4, 3, 0, 4, 4094, 4096, 2, 3, 4096, 0, 0, 0, 0, 0, 1, 0, 4093, 4084, 0, 0, 11, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3897, 0, 0, 4094, 2, 2, 0, 2, 1601, 4094, 0, 4096, 1, 0, 2049, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 2, 2, 24, 0, 0, 0, 4094, 4090, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[51, 4064, 7, 4073, 0, 0, 0, 4086, 0, 4089, 0, 0, 0, 29, 4090, 4, 4091, 0, 2, 4, 5, 0, 4079, 4090, 3, 4078, 4087, 0, 20, 4, 4055, 4087, 2, 4090, 10, 3902, 0, 8, 7, 198, 6, 4082, 8, 0, 0, 0, 4068, 9, 0, 7, 13, 4086, 0, 5, 0, 0, 12, 1, 0, 2, 4091, 4085, 0, 0, 4092, 6, 3, 1, 8, 6, 9, 0, 4087, 4089, 6, 5, 4, 0, 5, 1, 0, 0, 3, 4088, 0, 7, 0, 5, 3585, 7, 6, 4087, 4, 8, 0, 2, 743, 4089, 9, 4080, 5, 0, 3830, 0, 5, 0, 4063, 4, 0, 0, 0, 5, 0, 0, 4090, 0, 0, 0, 1, 17, 4, 27, 0, 4090, 4085, 9, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[133, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 2, 4096, 4096, 0, 4096, 4095, 0, 0, 0, 4094, 4096, 1, 4096, 0, 4090, 0, 0, 0, 1, 0, 4096, 47, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3955, 0, 0, 4096, 0, 0, 0, 0, 50, 4096, 0, 4096, 0, 0, 4012, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 4079, 0, 0, 0, 3544, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4080, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 4096, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3827, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 39, 0, 0, 3831, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3933, 0, 0, 4096, 0, 0, 0, 0, 4096, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1575, 4096, 0, 4096, 0, 0, 2671, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3838, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15814, 16413, 16408, 16373, 16383, 16490, 16829, 16362] avg=16384.0000 max=16829 max_rank=6 min=15814 min_rank=0 imbalance_ratio=1.027161 max_load_reduction=114243 max_load_reduction_pct=0.871605 moved_to_echo=66064 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1169, 4096, 0, 4096, 0, 0, 0, 3544, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 1, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3693, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3827, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 14, 0, 0, 3831, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 4096, 0, 4096, 0, 0, 3172, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3838, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17001, 16414, 16010, 16373, 16361, 16652, 15899, 16362] avg=16384.0000 max=17001 max_rank=0 min=15899 min_rank=6 imbalance_ratio=1.037659 max_load_reduction=114071 max_load_reduction_pct=0.870293 moved_to_echo=65274 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2041, 3934, 44, 3986, 0, 0, 26, 3502, 0, 4041, 0, 0, 0, 23, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4042, 0, 25, 4037, 4061, 18, 30, 0, 3996, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 45, 3459, 0, 30, 32, 399, 27, 4032, 6, 0, 0, 0, 3986, 35, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 62, 4059, 0, 33, 0, 0, 71, 2, 0, 0, 4069, 4038, 0, 0, 3807, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 23, 36, 0, 0, 20, 35, 0, 4041, 4063, 13, 27, 73, 0, 8, 0, 3798, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 22, 4047, 0, 14, 59, 8, 2722, 29, 123, 4060, 15, 31, 5, 25, 4073, 266, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2719, 4065, 45, 4050, 31, 0, 877, 50, 24, 0, 4024, 18, 0, 0, 0, 17, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 133, 0, 16, 0, 52, 26, 0, 0, 4064, 4061, 22, 2, 0, 3809, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17622, 16251, 16145, 16434, 16417, 15505, 16212, 16486] avg=16384.0000 max=17622 max_rank=0 min=15505 min_rank=5 imbalance_ratio=1.075562 max_load_reduction=113450 max_load_reduction_pct=0.865555 moved_to_echo=64620 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[174, 4096, 0, 4096, 0, 0, 0, 3544, 0, 4092, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 4096, 0, 0, 4095, 4093, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3967, 4, 0, 0, 207, 0, 4096, 4, 1, 0, 0, 4093, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 3828, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 150, 0, 0, 3831, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 4096, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1139, 4096, 0, 4096, 0, 0, 2561, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3837, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16007, 16414, 16497, 16374, 16487, 16649, 16283, 16361] avg=16384.0000 max=16649 max_rank=5 min=16007 min_rank=0 imbalance_ratio=1.016174 max_load_reduction=114423 max_load_reduction_pct=0.872978 moved_to_echo=65780 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[774, 4096, 0, 4096, 0, 0, 0, 3544, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4080, 6, 0, 0, 11, 0, 4096, 59, 0, 0, 0, 4096, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3828, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 7, 0, 0, 3831, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 4096, 0, 4096, 0, 0, 3172, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3837, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16606, 16413, 16473, 16375, 16353, 16651, 15840, 16361] avg=16384.0000 max=16651 max_rank=5 min=15840 min_rank=6 imbalance_ratio=1.016296 max_load_reduction=114421 max_load_reduction_pct=0.872963 moved_to_echo=65205 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[51, 4064, 7, 4073, 0, 0, 0, 3526, 0, 4089, 0, 0, 0, 29, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 5, 0, 4079, 0, 3, 4078, 4087, 0, 20, 4, 4055, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4090, 10, 3902, 0, 8, 7, 198, 6, 4082, 8, 0, 0, 0, 4068, 9, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 13, 4086, 0, 5, 0, 0, 12, 1, 0, 2, 4091, 4085, 0, 0, 3823, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 6, 3, 1, 8, 6, 9, 0, 4087, 4089, 6, 5, 4, 0, 5, 1, 3828, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4088, 0, 7, 0, 5, 3585, 7, 6, 4087, 4, 8, 0, 2, 4091, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 743, 4089, 9, 4080, 5, 0, 3830, 0, 5, 0, 4063, 4, 0, 0, 0, 5, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 1, 17, 4, 27, 0, 4090, 4085, 9, 2, 0, 3831, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15843, 16357, 16423, 16384, 16379, 16160, 17131, 16395] avg=16384.0000 max=17131 max_rank=6 min=15843 min_rank=0 imbalance_ratio=1.045593 max_load_reduction=113941 max_load_reduction_pct=0.869301 moved_to_echo=66065 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[377, 4092, 1, 4096, 4, 0, 2, 3542, 0, 4095, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 2, 4089, 4091, 0, 0, 0, 4093, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 0, 4078, 0, 0, 1, 78, 0, 4096, 214, 0, 0, 0, 4065, 1, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4, 3, 0, 4, 4094, 4096, 2, 3, 3826, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4093, 4084, 0, 0, 11, 0, 2, 0, 3830, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3897, 0, 0, 4094, 2, 2, 0, 2, 4096, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1601, 4094, 0, 4096, 1, 0, 2049, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 2, 2, 24, 0, 0, 0, 4094, 4090, 0, 0, 0, 3836, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16213, 16398, 16660, 16385, 16345, 16454, 16234, 16383] avg=16384.0000 max=16660 max_rank=2 min=16213 min_rank=0 imbalance_ratio=1.016846 max_load_reduction=114412 max_load_reduction_pct=0.872894 moved_to_echo=65416 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[133, 4096, 0, 4096, 0, 0, 0, 3544, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 4095, 0, 0, 0, 4094, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4090, 0, 0, 0, 1, 0, 4096, 47, 0, 0, 0, 4096, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3828, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3831, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3955, 0, 0, 4096, 0, 0, 0, 0, 4096, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 4096, 0, 4096, 0, 0, 4012, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3838, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15966, 16412, 16456, 16374, 16347, 16511, 16644, 16362] avg=16384.0000 max=16644 max_rank=6 min=15966 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=114428 max_load_reduction_pct=0.873016 moved_to_echo=65864 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 31360, 128, 128, 128, 1024, 128, 32768, 384, 128, 0, 0, 32640, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 32768, 0, 128, 128, 128, 30464, 128, 256, 32768, 128, 128, 128, 128, 32768, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[4736, 32640, 128, 32640, 128, 0, 128, 28416, 0, 32768, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 32768, 256, 0, 128, 128, 128, 128, 128, 0, 32768, 32768, 128, 128, 0, 30720, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[8064, 32768, 128, 32768, 128, 0, 22400, 128, 128, 128, 32768, 128, 0, 0, 0, 128, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 128, 32768, 0, 128, 32768, 32768, 128, 128, 128, 32640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 0, 32768, 32768, 128, 128, 128, 256, 128, 128, 30720, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 32768, 0, 128, 0, 0, 128, 128, 0, 128, 32768, 32768, 128, 128, 30720, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 3781, 6, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 63, 0, 0, 4025, 4075, 4093, 0, 4096, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 10, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4058, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 3364, 17, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 56, 0, 0, 4025, 4093, 4077, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 727, 0, 0, 16, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4095, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4063, 41, 4060, 48, 0, 4028, 0, 0, 0, 0, 4040, 13, 31, 23, 4055, 0, 4047, 4052, 0, 37, 0, 0, 4071, 4053, 5, 0, 16, 0, 0, 0, 0, 0, 4062, 3944, 4052, 43, 4068, 0, 4073, 4066, 0, 0, 18, 134, 0, 0, 0, 27, 4016, 33, 0, 0, 0, 0, 4025, 4070, 38, 0, 0, 0, 9, 25, 37, 19, 5, 15, 0, 4060, 0, 4063, 9, 19, 4056, 0, 45, 4058, 4048, 4065, 20, 4013, 0, 28, 4065, 40, 59, 0, 0, 22, 0, 71, 0, 0, 0, 0, 41, 406, 0, 34, 41, 0, 4071, 23, 0, 28, 22, 3882, 3770, 4036, 118, 0, 4051, 56, 0, 4067, 0, 0, 0, 0, 55, 0, 40, 14, 0, 36, 1, 46, 31] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 2819, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 151, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 3, 0, 4096, 0, 0, 0, 0, 77, 0, 0, 0, 0, 3, 0, 0, 1040, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4025, 4096, 4096, 1, 69, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4084, 0, 4089, 8, 0, 4087, 4, 0, 0, 0, 4082, 0, 7, 5, 4084, 0, 4077, 4086, 0, 5, 0, 0, 4088, 4087, 5, 0, 2, 0, 0, 0, 0, 0, 4062, 4080, 4089, 9, 4088, 2, 4091, 4090, 0, 0, 6, 0, 0, 0, 0, 7, 4087, 7, 0, 23, 0, 4, 4087, 4090, 7, 0, 0, 0, 1, 5, 0, 6, 29, 3, 0, 4087, 12, 4090, 0, 1, 4088, 0, 8, 4086, 4086, 4088, 3, 4079, 0, 3, 4090, 10, 10, 0, 0, 95, 0, 17, 0, 0, 0, 1, 7, 0, 0, 7, 9, 0, 4090, 6, 0, 6, 4, 4001, 4056, 4082, 9, 0, 4089, 12, 13, 4089, 13, 0, 0, 0, 10, 1, 11, 3, 0, 7, 6, 8, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4095, 2, 4094, 2, 0, 3983, 0, 0, 4, 0, 4094, 0, 0, 0, 4094, 0, 4093, 4095, 0, 1, 7, 0, 4086, 4093, 2, 0, 2, 0, 5, 0, 0, 0, 4096, 4093, 4075, 0, 4091, 2, 4094, 4094, 0, 0, 0, 10, 0, 0, 0, 1, 4094, 1, 3, 3, 0, 0, 4091, 4093, 2, 0, 0, 0, 1, 2, 2, 0, 0, 0, 0, 4093, 1, 4094, 0, 0, 4095, 0, 1, 4095, 4095, 4094, 0, 4095, 0, 0, 4094, 1, 0, 0, 0, 39, 0, 2, 0, 0, 85, 1, 0, 186, 1, 1, 1, 2, 4094, 0, 0, 0, 0, 3910, 4094, 4095, 2, 0, 4095, 1, 0, 4093, 0, 0, 0, 3, 0, 1, 1, 0, 0, 2, 3, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 3356, 17, 0, 2, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 17, 0, 0, 4093, 4091, 4073, 0, 4096, 0, 4095, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 46, 0, 4092, 0, 0, 0, 0, 89, 3, 0, 0, 0, 133, 0, 0, 1631, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2921, 4094, 4096, 0, 9, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 4053, 15, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 21, 0, 0, 4096, 4096, 4094, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 404, 0, 0, 3781, 6, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 4075, 4093, 0, 4096, 0, 0, 43, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3692, 218, 162, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3891, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 4096, 4096, 0, 0, 3878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16479, 16445, 16333, 16382, 16385, 16214, 16479, 16355] avg=16384.0000 max=16479 max_rank=0 min=16214 min_rank=5 imbalance_ratio=1.005798 max_load_reduction=114593 max_load_reduction_pct=0.874275 moved_to_echo=65433 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4063, 41, 401, 48, 0, 4028, 0, 0, 0, 0, 4040, 13, 31, 23, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 4052, 0, 37, 0, 0, 4069, 4053, 5, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 3944, 4052, 43, 4068, 0, 0, 44, 0, 0, 18, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4016, 33, 0, 0, 0, 0, 4025, 4070, 38, 0, 0, 0, 9, 25, 37, 3659, 216, 160, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 5, 15, 0, 4060, 0, 0, 9, 19, 4056, 0, 45, 4058, 4048, 1, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 28, 4065, 40, 59, 0, 0, 22, 0, 71, 0, 0, 0, 0, 41, 4064, 3862, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 406, 0, 34, 41, 0, 0, 23, 0, 28, 22, 3882, 3770, 4036, 118, 0, 3835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 4067, 0, 0, 0, 0, 55, 0, 40, 14, 0, 36, 1, 46, 31, 4073, 4071, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16743, 16279, 16365, 16317, 16355, 16294, 16195, 16524] avg=16384.0000 max=16743 max_rank=0 min=16195 min_rank=6 imbalance_ratio=1.021912 max_load_reduction=114329 max_load_reduction_pct=0.872261 moved_to_echo=65368 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 2, 404, 2, 0, 3983, 0, 0, 4, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4095, 0, 1, 7, 0, 4084, 4093, 2, 0, 2, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 4075, 0, 4091, 2, 0, 44, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 3, 3, 0, 0, 4091, 4093, 2, 0, 0, 0, 1, 2, 2, 3690, 218, 161, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 4095, 0, 1, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 1, 0, 0, 0, 39, 0, 2, 0, 0, 85, 1, 0, 4094, 3889, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 1, 1, 1, 2, 0, 0, 0, 0, 0, 3910, 4094, 4095, 2, 0, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 0, 0, 3, 0, 1, 1, 0, 0, 2, 3, 1, 1, 4094, 4094, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16678, 16382, 16411, 16364, 16380, 16329, 16169, 16359] avg=16384.0000 max=16678 max_rank=0 min=16169 min_rank=6 imbalance_ratio=1.017944 max_load_reduction=114394 max_load_reduction_pct=0.872757 moved_to_echo=65237 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4084, 0, 402, 8, 0, 4087, 4, 0, 0, 0, 4082, 0, 7, 5, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4086, 0, 5, 0, 0, 4083, 4087, 5, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4080, 4089, 9, 4088, 2, 0, 37, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4087, 7, 0, 23, 0, 4, 4087, 4090, 7, 0, 0, 0, 1, 5, 0, 3687, 224, 164, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 29, 3, 0, 4087, 12, 0, 0, 1, 4088, 0, 8, 4086, 4086, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 3, 4090, 10, 10, 0, 0, 95, 0, 17, 0, 0, 0, 1, 7, 4088, 3889, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 0, 0, 6, 0, 6, 4, 4001, 4056, 4082, 9, 0, 3865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 4089, 13, 0, 0, 0, 10, 1, 11, 3, 0, 7, 6, 8, 6, 4091, 4090, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16763, 16345, 16373, 16398, 16409, 16319, 16045, 16420] avg=16384.0000 max=16763 max_rank=0 min=16045 min_rank=6 imbalance_ratio=1.023132 max_load_reduction=114309 max_load_reduction_pct=0.872108 moved_to_echo=65193 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 404, 0, 0, 3364, 17, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 4093, 4077, 0, 4096, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3692, 218, 162, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 727, 0, 0, 4096, 3891, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 4096, 0, 0, 3878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16073, 16438, 16334, 16370, 16384, 16939, 16180, 16354] avg=16384.0000 max=16939 max_rank=5 min=16073 min_rank=0 imbalance_ratio=1.033875 max_load_reduction=114133 max_load_reduction_pct=0.870766 moved_to_echo=65857 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 404, 0, 0, 3356, 17, 0, 2, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4091, 4073, 0, 4096, 0, 0, 43, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3692, 218, 162, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 46, 0, 4092, 0, 0, 0, 0, 89, 3, 0, 0, 0, 133, 0, 0, 4096, 3891, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2921, 4094, 4096, 0, 9, 3878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16067, 16398, 16400, 16366, 16385, 16476, 16629, 16351] avg=16384.0000 max=16629 max_rank=6 min=16067 min_rank=0 imbalance_ratio=1.014954 max_load_reduction=114443 max_load_reduction_pct=0.873131 moved_to_echo=65841 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 404, 0, 0, 2819, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 151, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3692, 218, 162, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 4096, 0, 0, 0, 0, 77, 0, 0, 0, 0, 3, 0, 0, 4095, 3891, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 4096, 4096, 1, 69, 3878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15511, 16382, 16427, 16513, 16390, 16290, 17205, 16354] avg=16384.0000 max=17205 max_rank=6 min=15511 min_rank=0 imbalance_ratio=1.050110 max_load_reduction=113867 max_load_reduction_pct=0.868736 moved_to_echo=66239 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 404, 0, 0, 4053, 15, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4094, 0, 4096, 0, 0, 43, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3692, 218, 162, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 3891, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 4096, 0, 0, 3878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16758, 16403, 16429, 16362, 16384, 16211, 16170, 16355] avg=16384.0000 max=16758 max_rank=0 min=16170 min_rank=6 imbalance_ratio=1.022827 max_load_reduction=114314 max_load_reduction_pct=0.872147 moved_to_echo=65120 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 32768, 128, 128, 0, 0, 384, 128, 128, 0, 0, 1024, 128, 128, 32768, 31104, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32768, 128, 3328, 128, 0, 29568, 128, 0, 128, 0, 32768, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 32768, 0, 128, 128, 0, 32768, 32768, 128, 0, 128, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 128, 128, 32768, 0, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 128, 128, 128, 128, 128, 0, 128, 128, 128, 128, 32768, 32768, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 128, 256, 0, 128, 32768, 32768, 128, 0, 0, 0, 128, 128, 128, 29568, 1792, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32640, 32640, 32768, 128, 32768, 128, 0, 384, 128, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[3712, 128, 128, 128, 128, 0, 128, 0, 128, 128, 31104, 32512, 32768, 256, 128, 30976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 1363, 4096, 0, 0, 0, 0, 0, 4, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4, 1688, 0, 0, 4096, 4096, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 0, 4073, 1069, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4041, 4096, 4096, 4096, 0, 55, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[2, 0, 1, 0, 0, 4094, 1, 0, 4094, 4095, 233, 4094, 8, 0, 0, 0, 0, 0, 0, 4094, 4095, 2, 4094, 0, 1, 0, 1, 0, 4095, 0, 1804, 0, 2, 4094, 4094, 4079, 0, 0, 29, 4094, 423, 0, 0, 2, 0, 0, 1, 2, 4094, 0, 4093, 1, 0, 4094, 1, 0, 0, 0, 1, 1, 0, 4092, 4095, 1, 73, 0, 0, 4094, 0, 0, 2, 2, 0, 0, 1, 3, 0, 4078, 1, 0, 2, 4094, 0, 4068, 1361, 0, 4093, 2, 0, 0, 2, 0, 0, 4, 0, 0, 1, 0, 4096, 1, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 4095, 0, 4095, 0, 0, 1, 45, 0, 4085, 4095, 4094, 4096, 3, 191, 0, 4095, 1, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[8, 8, 5, 11, 0, 4078, 1, 0, 4088, 4084, 756, 4090, 0, 0, 7, 0, 0, 0, 0, 4088, 4090, 18, 4085, 5, 6, 0, 0, 0, 4086, 0, 3033, 0, 0, 4090, 4089, 4063, 0, 3, 0, 4088, 44, 0, 0, 7, 0, 4, 0, 12, 4091, 6, 4057, 5, 12, 4084, 1, 8, 3, 11, 9, 6, 0, 4087, 4089, 6, 0, 0, 0, 4082, 4, 0, 0, 13, 0, 0, 0, 6, 0, 4006, 9, 5, 1, 4084, 0, 4089, 51, 5, 4089, 10, 0, 0, 0, 9, 9, 2, 0, 0, 0, 0, 4092, 0, 6, 22, 0, 4, 4, 4090, 0, 5, 0, 4077, 5, 4081, 0, 7, 0, 0, 0, 4064, 4086, 4083, 4089, 3, 344, 11, 4088, 3, 0, 12] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 109, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 88, 1915, 0, 0, 4096, 4096, 4096, 0, 0, 27, 4096, 56, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4052, 0, 0, 0, 4096, 0, 4095, 1425, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1, 4096, 4096, 4096, 4096, 0, 518, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 339, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 4096, 136, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 4096, 4096, 0, 3627, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[38, 40, 32, 0, 52, 4056, 1, 4, 4051, 4046, 637, 4074, 0, 0, 38, 0, 0, 18, 0, 4000, 4060, 63, 4049, 0, 23, 0, 3, 0, 4043, 0, 2777, 24, 0, 4061, 4046, 4039, 0, 28, 0, 3994, 104, 0, 9, 35, 27, 25, 0, 57, 4069, 20, 3931, 30, 0, 4049, 5, 0, 14, 23, 36, 15, 0, 4035, 4062, 1, 0, 0, 0, 4036, 16, 0, 20, 48, 0, 0, 12, 2, 11, 4058, 35, 35, 2, 4029, 0, 4010, 36, 17, 4062, 50, 0, 0, 47, 2, 59, 76, 0, 1, 0, 22, 4045, 0, 57, 138, 0, 26, 31, 4061, 27, 8, 0, 3941, 0, 3945, 24, 29, 34, 96, 0, 3609, 4019, 4055, 4067, 0, 1187, 38, 4047, 34, 0, 24] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 28, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 5, 2453, 0, 0, 4096, 4096, 4096, 0, 0, 10, 4095, 18, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 4096, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 18, 0, 4096, 4096, 4096, 4096, 0, 1605, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 129, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 3776, 0, 0, 4096, 4096, 4096, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 4096, 185, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1, 4092, 4096, 4096, 4096, 0, 6, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 1363, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 1872, 0, 4096, 0, 0, 0, 0, 0, 4096, 4, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 4096, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4096, 4074, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4073, 1069, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2194, 892, 412, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 4096, 3204, 0, 0, 55, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17335, 15856, 16310, 16406, 16381, 16897, 16395, 15492] avg=16384.0000 max=17335 max_rank=0 min=15492 min_rank=7 imbalance_ratio=1.058044 max_load_reduction=113737 max_load_reduction_pct=0.867744 moved_to_echo=65165 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 1, 0, 0, 4094, 1, 0, 4094, 4095, 233, 3682, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1872, 2, 4094, 0, 1, 0, 1, 0, 4095, 0, 1804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4018, 4094, 4079, 0, 0, 29, 4094, 423, 0, 0, 2, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4093, 1, 0, 4094, 1, 0, 0, 0, 1, 1, 0, 4092, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 4094, 0, 0, 2, 2, 0, 0, 1, 3, 0, 4078, 1, 0, 4096, 4072, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 4068, 1361, 0, 4093, 2, 0, 0, 2, 0, 0, 4, 0, 0, 2193, 891, 412, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 1, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 4095, 0, 4095, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 45, 0, 4085, 4095, 3203, 0, 3, 191, 0, 4095, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16210, 15963, 16744, 16401, 16452, 17187, 16395, 15720] avg=16384.0000 max=17187 max_rank=5 min=15720 min_rank=7 imbalance_ratio=1.049011 max_load_reduction=113885 max_load_reduction_pct=0.868874 moved_to_echo=65754 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[8, 8, 5, 11, 0, 4078, 1, 0, 4088, 4084, 756, 3672, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1860, 18, 4085, 5, 6, 0, 0, 0, 4086, 0, 3033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 4089, 4063, 0, 3, 0, 4088, 44, 0, 0, 7, 0, 4, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 6, 4057, 5, 12, 4084, 1, 8, 3, 11, 9, 6, 0, 4087, 4089, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4, 0, 0, 13, 0, 0, 0, 6, 0, 4006, 9, 5, 4089, 4076, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4084, 0, 4089, 51, 5, 4089, 10, 0, 0, 0, 9, 9, 2, 0, 0, 2193, 893, 418, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 6, 22, 0, 4, 4, 4090, 0, 5, 0, 4077, 5, 4081, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4064, 4086, 3190, 0, 3, 344, 11, 4088, 3, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16718, 17181, 16319, 16399, 16327, 15921, 16399, 15808] avg=16384.0000 max=17181 max_rank=1 min=15808 min_rank=7 imbalance_ratio=1.048645 max_load_reduction=113891 max_load_reduction_pct=0.868919 moved_to_echo=64455 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 109, 3684, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1872, 0, 4096, 0, 0, 0, 0, 0, 4096, 88, 1915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 4096, 4096, 0, 0, 27, 4096, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4052, 0, 0, 4096, 4074, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 1425, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2194, 892, 412, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 3204, 0, 0, 518, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16081, 16163, 16391, 16406, 16357, 17275, 16388, 16011] avg=16384.0000 max=17275 max_rank=5 min=16011 min_rank=7 imbalance_ratio=1.054382 max_load_reduction=113797 max_load_reduction_pct=0.868202 moved_to_echo=66031 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[38, 40, 32, 0, 52, 4056, 1, 4, 4051, 4046, 637, 3664, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4000, 1856, 63, 4049, 0, 23, 0, 3, 0, 4043, 0, 2777, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 4046, 4039, 0, 28, 0, 3994, 104, 0, 9, 35, 27, 25, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 20, 3931, 30, 0, 4049, 5, 0, 14, 23, 36, 15, 0, 4035, 4062, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 16, 0, 20, 48, 0, 0, 12, 2, 11, 4058, 35, 35, 4067, 4048, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4029, 0, 4010, 36, 17, 4062, 50, 0, 0, 47, 2, 59, 76, 0, 1, 2174, 883, 410, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4045, 0, 57, 138, 0, 26, 31, 4061, 27, 8, 0, 3941, 0, 3945, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 29, 34, 96, 0, 3609, 4019, 3172, 0, 0, 1187, 38, 4047, 34, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16659, 16856, 16350, 16242, 16418, 15922, 16312, 16313] avg=16384.0000 max=16856 max_rank=1 min=15922 min_rank=5 imbalance_ratio=1.028809 max_load_reduction=114216 max_load_reduction_pct=0.871399 moved_to_echo=64965 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4092, 0, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1872, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 4096, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4074, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 136, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2194, 892, 412, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 3204, 0, 0, 3627, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15968, 14499, 16309, 16405, 16391, 15987, 16395, 19118] avg=16384.0000 max=19118 max_rank=7 min=14499 min_rank=1 imbalance_ratio=1.166870 max_load_reduction=111954 max_load_reduction_pct=0.854141 moved_to_echo=67891 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 28, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1872, 0, 4096, 0, 0, 0, 0, 0, 4096, 5, 2453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 4096, 4096, 0, 0, 10, 4095, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 4096, 4074, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2194, 892, 412, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 4096, 4096, 3204, 0, 0, 1605, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16000, 16618, 16335, 16406, 16357, 15846, 16395, 17115] avg=16384.0000 max=17115 max_rank=7 min=15846 min_rank=5 imbalance_ratio=1.044617 max_load_reduction=113957 max_load_reduction_pct=0.869423 moved_to_echo=65713 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 129, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1872, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 3776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 4096, 4096, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 4074, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 185, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2194, 892, 412, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 4096, 3204, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16101, 17936, 16314, 16407, 16389, 16037, 16393, 15495] avg=16384.0000 max=17936 max_rank=1 min=15495 min_rank=7 imbalance_ratio=1.094727 max_load_reduction=113136 max_load_reduction_pct=0.863159 moved_to_echo=64314 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 32768, 128, 128, 256, 0, 128, 128, 32768, 128, 128, 0, 32640, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[256, 128, 32640, 128, 128, 32768, 128, 128, 128, 128, 128, 128, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 128, 128, 32768, 32768, 3328, 29440, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 32768, 14976, 128, 32768, 128, 128, 0, 128, 0, 32768, 128, 17792, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 128, 128, 0, 128, 128, 128, 128, 32640, 128, 128, 32768, 32640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32128, 32768, 32768, 0, 128, 128, 32768, 768, 0, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 0, 32640, 4352, 128, 32768, 128, 128, 0, 128, 128, 128, 128, 0, 128, 17536, 7168, 3328, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 256, 128, 32256, 32768, 25600, 0, 128, 7552, 128, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4058, 2, 0, 4059, 0, 0, 3, 0, 4016, 45, 45, 0, 0, 0, 0, 39, 44, 0, 4053, 0, 4053, 4051, 169, 27, 4030, 4072, 0, 0, 6, 4025, 4065, 37, 4062, 46, 4058, 15, 53, 34, 0, 3965, 0, 4041, 4023, 30, 0, 4060, 0, 0, 0, 28, 44, 48, 43, 42, 4049, 0, 0, 0, 0, 37, 15, 0, 4044, 11, 772, 4052, 36, 0, 53, 26, 0, 26, 0, 3192, 4065, 48, 22, 0, 4051, 0, 13, 6, 4052, 0, 0, 31, 0, 41, 0, 43, 45, 4, 45, 4060, 0, 0, 11, 35, 4059, 4045, 48, 98, 21, 0, 0, 0, 9, 0, 44, 26, 3945, 4041, 4048, 21, 42, 0, 0, 4052, 7, 4063, 25, 0, 4045, 57, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 1615, 0, 4096, 2625, 8, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3957, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 621, 0, 4096, 3463, 5, 0, 0, 4096, 4094, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 4077, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4081, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4088, 0, 0, 4089, 0, 0, 0, 0, 3910, 2, 0, 0, 0, 0, 0, 6, 8, 0, 4087, 4, 4087, 4086, 384, 4, 4083, 4091, 0, 0, 0, 4086, 4088, 7, 4089, 10, 4088, 8, 11, 7, 0, 4063, 0, 4084, 4078, 7, 0, 4089, 0, 0, 0, 7, 7, 8, 8, 16, 4085, 0, 0, 0, 0, 4, 6, 0, 4084, 3, 153, 4088, 8, 0, 7, 6, 0, 5, 0, 3969, 4090, 10, 0, 0, 4082, 34, 9, 0, 4087, 0, 0, 5, 0, 10, 0, 7, 6, 9, 9, 4089, 0, 0, 0, 6, 4087, 4081, 4, 0, 6, 0, 0, 0, 9, 0, 12, 9, 3826, 4085, 4085, 5, 9, 0, 2, 4087, 8, 4089, 4, 0, 4090, 13, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 543, 0, 4096, 4031, 0, 0, 0, 4096, 4061, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 29, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4079, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3653, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4084, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 438, 0, 4096, 3662, 17, 0, 0, 4096, 4089, 0, 4081, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4078, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4094, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4095, 0, 2, 4094, 0, 0, 0, 14, 4056, 2, 1, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 4094, 4093, 54, 0, 4094, 3989, 65, 1, 0, 4094, 4052, 0, 4088, 0, 4094, 0, 1, 1, 3, 4094, 0, 4094, 4095, 1, 0, 4094, 0, 0, 0, 0, 2, 1, 2, 0, 4094, 0, 0, 0, 0, 2, 0, 3, 4094, 0, 62, 4094, 1, 0, 0, 0, 0, 2, 0, 4093, 4094, 1, 0, 2, 4088, 8, 0, 2, 4095, 2, 0, 2, 0, 2, 0, 1, 0, 0, 1, 4095, 0, 0, 2, 1, 4094, 4094, 0, 0, 1, 9, 0, 31, 0, 3, 2, 0, 4095, 4095, 4095, 1, 0, 0, 0, 4094, 0, 4095, 0, 0, 4059, 0, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 50, 0, 4096, 4053, 8, 0, 0, 4096, 4086, 0, 4094, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 43, 4096, 0, 0, 0, 0, 0, 0, 0, 4081, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 4072, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3994, 129, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 3967, 3463, 5, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4080, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4068, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 4077, 4082, 87, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16427, 16246, 16399, 16371, 16394, 16369, 16483, 16383] avg=16384.0000 max=16483 max_rank=6 min=16246 min_rank=1 imbalance_ratio=1.006042 max_load_reduction=114589 max_load_reduction_pct=0.874245 moved_to_echo=65629 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4058, 2, 0, 4059, 0, 0, 3, 0, 4016, 45, 45, 0, 0, 0, 0, 3957, 127, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 44, 0, 0, 0, 0, 0, 169, 27, 3903, 4072, 0, 0, 6, 4025, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4062, 46, 15, 15, 53, 34, 0, 3965, 0, 4041, 4023, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 44, 48, 43, 42, 4049, 0, 0, 0, 0, 37, 15, 0, 4044, 4060, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 772, 4052, 36, 0, 53, 26, 0, 26, 0, 3192, 4050, 48, 22, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 6, 4052, 0, 0, 31, 0, 41, 0, 43, 45, 4, 45, 4060, 0, 4053, 4023, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 35, 4059, 4045, 48, 98, 21, 0, 0, 0, 9, 0, 44, 26, 3945, 4049, 86, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 4048, 21, 42, 0, 0, 4052, 7, 14, 25, 0, 4045, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16327, 16350, 16321, 16450, 16339, 16429, 16504, 16352] avg=16384.0000 max=16504 max_rank=6 min=16321 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=114568 max_load_reduction_pct=0.874084 moved_to_echo=65624 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 3994, 129, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1615, 0, 3967, 2625, 8, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 4080, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4068, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3957, 4082, 87, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16418, 16407, 16399, 16371, 16363, 16369, 16347, 16398] avg=16384.0000 max=16418 max_rank=0 min=16347 min_rank=6 imbalance_ratio=1.002075 max_load_reduction=114654 max_load_reduction_pct=0.874741 moved_to_echo=65477 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4088, 0, 0, 4089, 0, 0, 0, 0, 3910, 2, 0, 0, 0, 0, 0, 3987, 133, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8, 0, 0, 4, 0, 0, 384, 4, 3950, 4091, 0, 0, 0, 4086, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 10, 13, 8, 11, 7, 0, 4063, 0, 4084, 4078, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 8, 8, 16, 4085, 0, 0, 0, 0, 4, 6, 0, 4084, 4089, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 153, 4088, 8, 0, 7, 6, 0, 5, 0, 3969, 4073, 10, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 9, 0, 4087, 0, 0, 5, 0, 10, 0, 7, 6, 9, 9, 4089, 0, 4087, 4059, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4087, 4081, 4, 0, 6, 0, 0, 0, 9, 0, 12, 9, 3826, 4079, 88, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4085, 5, 9, 0, 2, 4087, 8, 10, 4, 0, 4090, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16226, 16621, 16377, 16389, 16404, 16423, 16234, 16398] avg=16384.0000 max=16621 max_rank=1 min=16226 min_rank=0 imbalance_ratio=1.014465 max_load_reduction=114451 max_load_reduction_pct=0.873192 moved_to_echo=65459 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3994, 129, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 0, 3967, 4031, 0, 0, 0, 4096, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4069, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3653, 4082, 87, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16427, 16698, 16399, 16372, 16380, 16369, 16041, 16386] avg=16384.0000 max=16698 max_rank=1 min=16041 min_rank=6 imbalance_ratio=1.019165 max_load_reduction=114374 max_load_reduction_pct=0.872604 moved_to_echo=65176 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 3994, 129, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 0, 3967, 3662, 17, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 15, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4079, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4069, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4078, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4094, 4082, 87, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16428, 16269, 16387, 16372, 16376, 16369, 16473, 16398] avg=16384.0000 max=16473 max_rank=6 min=16269 min_rank=1 imbalance_ratio=1.005432 max_load_reduction=114599 max_load_reduction_pct=0.874321 moved_to_echo=65616 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 2, 4094, 0, 0, 0, 14, 4056, 2, 1, 0, 0, 0, 0, 3992, 129, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 54, 0, 3965, 3989, 65, 1, 0, 4094, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 16, 0, 1, 1, 3, 4094, 0, 4094, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 0, 4094, 0, 0, 0, 0, 2, 0, 3, 4094, 4094, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 4094, 1, 0, 0, 0, 0, 2, 0, 4093, 4078, 1, 0, 2, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 4095, 2, 0, 2, 0, 2, 0, 1, 0, 0, 1, 4095, 0, 4090, 4066, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4094, 4094, 0, 0, 1, 9, 0, 31, 0, 3, 2, 0, 4095, 4081, 86, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 1, 0, 0, 0, 4094, 0, 14, 0, 0, 4059, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16401, 16221, 16393, 16374, 16421, 16376, 16526, 16360] avg=16384.0000 max=16526 max_rank=6 min=16221 min_rank=1 imbalance_ratio=1.008667 max_load_reduction=114546 max_load_reduction_pct=0.873917 moved_to_echo=65683 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 3994, 129, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 3967, 4053, 8, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 15, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 4096, 0, 0, 0, 0, 0, 0, 0, 4081, 4079, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4068, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 4072, 4082, 87, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16418, 16260, 16397, 16373, 16395, 16368, 16464, 16397] avg=16384.0000 max=16464 max_rank=6 min=16260 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=114608 max_load_reduction_pct=0.874390 moved_to_echo=65624 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 128, 128, 32768, 0, 0, 128, 128, 32512, 128, 128, 128, 0, 0, 0, 32000, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 128, 0, 128, 32768, 128, 128, 128, 0, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 1152, 32768, 128, 0, 128, 128, 0, 128, 0, 31744, 32640, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 32768, 32768, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 31744, 32640, 768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 0, 128, 0, 0, 3968, 128, 31744, 30080, 128, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 128, 0, 128, 0, 128, 128, 128, 128, 32768, 0, 32768, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 128, 128, 128, 32768, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 128, 32640, 0, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 24, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 10, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4058, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4087, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4096, 4093, 0, 0, 4096, 0, 2, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4080, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 5, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 4096, 0, 0, 0, 4096, 4075, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[1, 4095, 4094, 4095, 0, 7, 4094, 0, 2, 4094, 4094, 0, 2, 1, 18, 5, 0, 4096, 0, 0, 0, 1, 3, 1, 2, 12, 0, 4093, 0, 7, 0, 0, 0, 0, 2, 0, 0, 4, 4094, 0, 1, 1, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 1, 4094, 4092, 0, 0, 4088, 1, 0, 2, 4094, 0, 0, 4093, 4094, 2, 0, 0, 1, 0, 1, 0, 4094, 0, 3, 3, 2, 4094, 4095, 0, 3, 1, 0, 0, 4092, 0, 0, 4090, 4094, 4093, 0, 2, 4094, 1, 30, 2, 0, 0, 4095, 4057, 0, 0, 0, 1, 0, 4082, 2, 0, 0, 4094, 1, 0, 0, 0, 0, 1, 0, 0, 4094, 4091, 1, 0, 2, 4094, 4085, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[46, 4043, 4059, 4049, 0, 80, 4004, 26, 28, 4030, 3960, 0, 14, 0, 0, 0, 0, 4034, 11, 2, 0, 28, 0, 6, 4, 1, 50, 4055, 58, 0, 0, 0, 0, 0, 42, 33, 0, 0, 4036, 30, 34, 13, 1, 26, 55, 4053, 0, 0, 0, 37, 36, 37, 4043, 4038, 0, 207, 4045, 21, 31, 54, 4050, 0, 37, 3997, 4009, 41, 0, 31, 43, 0, 35, 47, 4043, 0, 62, 0, 0, 4046, 4056, 2, 81, 25, 28, 39, 4060, 64, 39, 4032, 4022, 4063, 0, 12, 3995, 55, 0, 25, 0, 32, 3991, 4057, 0, 0, 0, 0, 0, 3902, 65, 0, 69, 4049, 6, 0, 0, 21, 0, 45, 38, 0, 4048, 4028, 0, 78, 36, 4029, 4062, 0, 17, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[11, 4082, 4089, 4086, 0, 1, 4080, 0, 6, 4072, 4067, 0, 2, 0, 0, 0, 0, 4088, 1, 0, 0, 8, 0, 0, 7, 12, 10, 4087, 10, 0, 0, 0, 7, 0, 10, 6, 0, 0, 4078, 7, 5, 11, 9, 3, 4, 4077, 0, 0, 0, 12, 0, 8, 4085, 4086, 0, 75, 4085, 0, 5, 16, 4088, 0, 2, 4084, 4083, 7, 0, 6, 12, 13, 0, 9, 4089, 0, 2, 0, 14, 4086, 4089, 4, 14, 3, 3, 6, 4088, 15, 3, 4083, 4082, 4087, 0, 6, 4073, 9, 0, 5, 1, 0, 4031, 4087, 0, 0, 0, 0, 0, 4077, 16, 0, 8, 4087, 2, 0, 0, 4, 0, 10, 0, 9, 4085, 4084, 0, 10, 4, 4079, 4089, 11, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 59, 0, 0, 0, 0, 4096, 4096, 0, 3, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 5, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 0, 4096, 4017, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 4091, 0, 0, 0, 2, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 4096, 0, 0, 4096, 0, 3, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 4094, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 23, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 24, 4096, 4072, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4078, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 33, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4058, 0, 0, 0, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16381, 16391, 16351, 16376, 16376, 16427, 16363] avg=16384.0000 max=16427 max_rank=6 min=16351 min_rank=3 imbalance_ratio=1.002625 max_load_reduction=114645 max_load_reduction_pct=0.874672 moved_to_echo=65542 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 23, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 3, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4072, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4075, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 33, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4080, 0, 0, 0, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16408, 16395, 16367, 16351, 16376, 16364, 16426, 16385] avg=16384.0000 max=16426 max_rank=6 min=16351 min_rank=3 imbalance_ratio=1.002563 max_load_reduction=114646 max_load_reduction_pct=0.874680 moved_to_echo=65551 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[46, 4043, 0, 23, 0, 80, 4004, 26, 28, 4030, 3960, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 11, 2, 0, 28, 0, 6, 4, 1, 50, 4055, 58, 0, 0, 0, 4060, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 33, 0, 0, 4036, 30, 34, 13, 1, 26, 55, 4053, 0, 0, 4050, 4025, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 36, 37, 4011, 4038, 0, 207, 4045, 21, 31, 54, 0, 0, 37, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 41, 0, 31, 43, 0, 35, 47, 4043, 0, 62, 0, 0, 4046, 4048, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 25, 28, 39, 0, 64, 39, 4032, 4022, 4046, 0, 12, 3995, 55, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 3991, 4057, 0, 0, 0, 0, 0, 3902, 65, 0, 69, 4049, 6, 0, 32, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 45, 38, 0, 4048, 4028, 0, 78, 36, 4029, 4062, 0, 17, 0, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16254, 16361, 16405, 16551, 16407, 16463, 16213, 16418] avg=16384.0000 max=16551 max_rank=3 min=16213 min_rank=6 imbalance_ratio=1.010193 max_load_reduction=114521 max_load_reduction_pct=0.873726 moved_to_echo=65501 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[1, 4095, 0, 23, 0, 7, 4094, 0, 2, 4094, 4094, 0, 2, 1, 18, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 3, 1, 2, 12, 0, 4093, 0, 7, 0, 0, 4092, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 4094, 0, 1, 1, 0, 0, 0, 4096, 0, 0, 4094, 4071, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4061, 4092, 0, 0, 4088, 1, 0, 2, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 1, 0, 1, 0, 4094, 0, 3, 3, 2, 4094, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 4090, 4094, 4075, 0, 2, 4094, 1, 30, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4057, 0, 0, 0, 1, 0, 4082, 2, 0, 0, 4094, 1, 0, 33, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 4091, 1, 0, 2, 4094, 4085, 0, 0, 0, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16436, 16394, 16370, 16339, 16381, 16392, 16375, 16385] avg=16384.0000 max=16436 max_rank=0 min=16339 min_rank=3 imbalance_ratio=1.003174 max_load_reduction=114636 max_load_reduction_pct=0.874603 moved_to_echo=65533 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 23, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4072, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4078, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 33, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 4096, 0, 0, 0, 4096, 4075, 0, 0, 0, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16386, 16367, 16351, 16376, 16371, 16427, 16387] avg=16384.0000 max=16427 max_rank=6 min=16351 min_rank=3 imbalance_ratio=1.002625 max_load_reduction=114645 max_load_reduction_pct=0.874672 moved_to_echo=65561 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[11, 4082, 0, 16, 0, 1, 4080, 0, 6, 4072, 4067, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 8, 0, 0, 7, 12, 10, 4087, 10, 0, 0, 0, 4088, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 10, 6, 0, 0, 4078, 7, 5, 11, 9, 3, 4, 4077, 0, 0, 4088, 4068, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 8, 4051, 4086, 0, 75, 4085, 0, 5, 16, 0, 0, 2, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 7, 0, 6, 12, 13, 0, 9, 4089, 0, 2, 0, 14, 4086, 4079, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3, 3, 6, 0, 15, 3, 4083, 4082, 4065, 0, 6, 4073, 9, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4031, 4087, 0, 0, 0, 0, 0, 4077, 16, 0, 8, 4087, 2, 0, 34, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 10, 0, 9, 4085, 4084, 0, 10, 4, 4079, 4089, 11, 5, 0, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16337, 16394, 16379, 16424, 16404, 16367, 16357, 16410] avg=16384.0000 max=16424 max_rank=3 min=16337 min_rank=0 imbalance_ratio=1.002441 max_load_reduction=114648 max_load_reduction_pct=0.874695 moved_to_echo=65538 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 23, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 59, 4096, 4072, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4096, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4078, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 33, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 0, 4096, 4017, 0, 0, 0, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16419, 16377, 16426, 16354, 16376, 16371, 16425, 16324] avg=16384.0000 max=16426 max_rank=2 min=16324 min_rank=7 imbalance_ratio=1.002563 max_load_reduction=114646 max_load_reduction_pct=0.874680 moved_to_echo=65496 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 23, 0, 0, 4096, 0, 0, 4096, 4091, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4072, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4078, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 33, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 4094, 0, 0, 0, 9, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16404, 16384, 16367, 16351, 16376, 16368, 16422, 16400] avg=16384.0000 max=16422 max_rank=6 min=16351 min_rank=3 imbalance_ratio=1.002319 max_load_reduction=114650 max_load_reduction_pct=0.874710 moved_to_echo=65566 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 0, 256, 0, 128, 32768, 128, 128, 32768, 32640, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 32512, 32768, 0, 384, 32768, 128, 128, 128, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32640, 32768, 0, 0, 0, 128, 0, 32640, 128, 0, 128, 32768, 128, 0, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 0, 128, 128, 128, 128, 128, 32768, 0, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 128, 128, 32768, 32768, 128, 128, 128, 32768, 32640, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 128, 0, 128, 32768, 128, 128, 128, 128, 128, 128, 32768, 0, 128, 32768, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 32768, 32768, 32640, 0, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4021, 1, 21, 52, 4025, 71, 0, 4042, 0, 42, 0, 4062, 148, 31, 191, 10, 0, 37, 0, 16, 0, 103, 4056, 0, 4030, 0, 3, 3, 0, 48, 4040, 29, 34, 54, 0, 4053, 0, 0, 53, 75, 0, 3363, 0, 0, 4039, 10, 4027, 0, 0, 84, 0, 42, 7, 3905, 33, 20, 4023, 4025, 36, 27, 4042, 29, 68, 4008, 4047, 3960, 0, 0, 0, 46, 13, 0, 17, 4036, 41, 4052, 40, 4058, 0, 52, 60, 4050, 82, 4044, 3266, 0, 0, 0, 0, 37, 1, 30, 0, 0, 4037, 0, 0, 3, 4031, 42, 116, 0, 42, 0, 15, 4047, 49, 0, 0, 30, 114, 1295, 3931, 52, 0, 0, 3905, 0, 4064, 0, 23, 39, 0, 60, 0, 4060, 0, 4046] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4083, 0, 0, 9, 4083, 12, 0, 4085, 0, 9, 0, 3923, 40, 8, 29, 8, 0, 8, 0, 3, 0, 23, 4087, 0, 4079, 0, 0, 0, 0, 6, 4086, 6, 6, 14, 0, 4047, 0, 0, 5, 14, 0, 4050, 0, 0, 4086, 0, 4084, 0, 0, 7, 7, 10, 0, 4054, 5, 0, 3840, 4082, 8, 2, 4087, 0, 13, 4081, 4083, 3843, 2, 0, 10, 10, 5, 0, 3, 4081, 9, 4088, 7, 4088, 0, 12, 14, 4087, 11, 4084, 4027, 0, 4, 0, 0, 8, 0, 5, 0, 0, 4085, 25, 0, 0, 4077, 7, 197, 0, 0, 0, 4, 4086, 17, 4, 0, 8, 17, 518, 4084, 0, 0, 3, 4074, 3, 4089, 0, 10, 9, 0, 11, 0, 4089, 0, 4085] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 318, 4096, 0, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 10, 0, 0, 4096, 0, 206, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 248, 3754, 0, 0, 0, 4096, 0, 3791, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4093, 0, 2, 1, 4094, 1, 80, 4094, 0, 0, 0, 4095, 4, 1, 4, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 4095, 0, 2, 0, 1, 0, 4095, 1, 0, 0, 0, 4094, 0, 2, 1, 1, 0, 4070, 2, 0, 4094, 1, 4093, 0, 2, 0, 0, 1, 0, 4094, 0, 2, 4086, 4094, 1, 0, 4094, 0, 3, 4093, 4094, 4083, 2, 1, 0, 1, 0, 0, 0, 4094, 1, 4061, 1, 4096, 0, 0, 2, 4094, 0, 4094, 4085, 0, 0, 3, 0, 0, 0, 1, 1, 0, 4094, 0, 0, 0, 4092, 2, 21, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 3, 63, 4014, 0, 0, 2, 4094, 0, 4095, 0, 2, 0, 2, 2, 2, 4095, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 78, 4096, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4050, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 88, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 105, 4012, 0, 0, 0, 4096, 0, 4006, 0, 0, 0, 0, 0, 0, 4090, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 786, 4096, 0, 0, 0, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3995, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 114, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 275, 4083, 0, 0, 0, 4096, 0, 3529, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 910, 4096, 0, 0, 0, 3214, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 4096, 0, 0, 0, 4096, 0, 4096, 3509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 4096, 0, 442, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1055, 3197, 0, 0, 0, 4096, 0, 4073, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 26, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4093, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 3, 0, 0, 4096, 0, 49, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 4033, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4021, 1, 21, 52, 4025, 71, 0, 3927, 0, 42, 0, 4062, 148, 31, 191, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 16, 0, 103, 4056, 0, 4030, 0, 3, 3, 0, 48, 4040, 29, 3561, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 54, 0, 4053, 0, 0, 53, 75, 0, 3363, 0, 0, 4039, 10, 4027, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 42, 7, 3905, 33, 20, 4023, 4025, 36, 27, 68, 29, 68, 4008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 3960, 0, 0, 0, 46, 13, 0, 17, 4036, 41, 4052, 40, 47, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 4050, 82, 4044, 3266, 0, 0, 0, 0, 37, 1, 30, 0, 0, 4037, 0, 158, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4031, 42, 116, 0, 42, 0, 15, 4047, 49, 0, 0, 30, 114, 1295, 4011, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3931, 52, 0, 0, 3905, 0, 4064, 0, 23, 39, 0, 60, 0, 341, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16602, 16417, 15805, 16375, 16351, 15783, 17278, 16461] avg=16384.0000 max=17278 max_rank=6 min=15783 min_rank=5 imbalance_ratio=1.054565 max_load_reduction=113794 max_load_reduction_pct=0.868179 moved_to_echo=65873 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 318, 3980, 0, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 3592, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 4096, 0, 0, 68, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 10, 159, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 206, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 248, 4049, 3530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3754, 0, 0, 0, 4096, 0, 3791, 0, 0, 0, 0, 0, 0, 345, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16466, 16381, 16479, 16436, 16430, 16573, 16225, 16082] avg=16384.0000 max=16573 max_rank=5 min=16082 min_rank=7 imbalance_ratio=1.011536 max_load_reduction=114499 max_load_reduction_pct=0.873558 moved_to_echo=65310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4083, 0, 0, 9, 4083, 12, 0, 3962, 0, 9, 0, 3923, 40, 8, 29, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 3, 0, 23, 4087, 0, 4079, 0, 0, 0, 0, 6, 4086, 6, 3591, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 14, 0, 4047, 0, 0, 5, 14, 0, 4050, 0, 0, 4086, 0, 4084, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 10, 0, 4054, 5, 0, 3840, 4082, 8, 2, 62, 0, 13, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 3843, 2, 0, 10, 10, 5, 0, 3, 4081, 9, 4088, 7, 43, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4087, 11, 4084, 4027, 0, 4, 0, 0, 8, 0, 5, 0, 0, 4085, 25, 162, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 7, 197, 0, 0, 0, 4, 4086, 17, 4, 0, 8, 17, 518, 4045, 3524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 3, 4074, 3, 4089, 0, 10, 9, 0, 11, 0, 336, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16166, 16390, 16407, 16171, 16196, 16534, 16504, 16704] avg=16384.0000 max=16704 max_rank=7 min=16166 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=114368 max_load_reduction_pct=0.872559 moved_to_echo=65938 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 2, 1, 4094, 1, 80, 3978, 0, 0, 0, 4095, 4, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 0, 4095, 0, 2, 0, 1, 0, 4095, 1, 3591, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 1, 1, 0, 4070, 2, 0, 4094, 1, 4093, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 4094, 0, 2, 4086, 4094, 1, 0, 68, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4083, 2, 1, 0, 1, 0, 0, 0, 4094, 1, 4061, 1, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 4094, 4085, 0, 0, 3, 0, 0, 0, 1, 1, 0, 4094, 0, 159, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 21, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 3, 63, 4049, 3528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 0, 2, 4094, 0, 4095, 0, 2, 0, 2, 2, 2, 345, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16353, 16377, 16456, 16444, 16385, 16551, 15853, 16653] avg=16384.0000 max=16653 max_rank=7 min=15853 min_rank=6 imbalance_ratio=1.016418 max_load_reduction=114419 max_load_reduction_pct=0.872948 moved_to_echo=65442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 78, 3980, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3587, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4050, 4096, 0, 0, 68, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 159, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 88, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 105, 4049, 3530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 4096, 0, 4006, 0, 0, 0, 0, 0, 0, 344, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16311, 16373, 16481, 16406, 16431, 16552, 15964, 16554] avg=16384.0000 max=16554 max_rank=7 min=15964 min_rank=6 imbalance_ratio=1.010376 max_load_reduction=114518 max_load_reduction_pct=0.873703 moved_to_echo=65501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 910, 3980, 0, 0, 0, 3214, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3592, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 4096, 0, 0, 68, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 159, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 442, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1055, 4049, 3530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3197, 0, 0, 0, 4096, 0, 4073, 0, 0, 0, 0, 0, 0, 345, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16297, 16378, 16480, 16441, 16421, 15980, 17268, 15807] avg=16384.0000 max=17268 max_rank=6 min=15807 min_rank=7 imbalance_ratio=1.053955 max_load_reduction=113804 max_load_reduction_pct=0.868256 moved_to_echo=65476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 786, 3980, 0, 0, 0, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3592, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3995, 4096, 0, 0, 68, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 159, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 114, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 275, 4049, 3530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 4096, 0, 3529, 0, 0, 0, 0, 0, 0, 345, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16587, 16378, 16482, 16351, 16431, 16534, 16160, 16149] avg=16384.0000 max=16587 max_rank=0 min=16149 min_rank=7 imbalance_ratio=1.012390 max_load_reduction=114485 max_load_reduction_pct=0.873451 moved_to_echo=65274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 26, 3980, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3592, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 4096, 0, 0, 68, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 3, 159, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 49, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4049, 3530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 4033, 0, 0, 0, 0, 0, 0, 345, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16290, 16378, 16482, 16448, 16427, 16565, 15820, 16662] avg=16384.0000 max=16662 max_rank=7 min=15820 min_rank=6 imbalance_ratio=1.016968 max_load_reduction=114410 max_load_reduction_pct=0.872879 moved_to_echo=65474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 128, 32768, 128, 2304, 31872, 0, 128, 0, 31104, 256, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 32768, 128, 1280, 128, 128, 0, 128, 32768, 128, 128, 0, 128, 256, 3584, 32384, 28288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[31232, 128, 0, 128, 32640, 128, 31744, 0, 128, 128, 128, 128, 128, 2816, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 0, 128, 32768, 0, 32768, 128, 128, 128, 128, 128, 32768, 128, 28800, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 32384, 128, 128, 128, 128, 128, 0, 128, 32768, 128, 32768, 128, 384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 32768, 0, 128, 128, 128, 0, 32000, 128, 0, 32768, 128, 32768, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 32640, 128, 128, 32256, 32768, 128, 128, 640, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 32768, 31360, 0, 128, 128, 0, 128, 128, 128, 128, 128, 32768, 128, 1280, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4075, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 21, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4054, 0, 0, 17, 0, 0, 0, 0, 4096, 10, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 2, 0, 0, 4096, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4094, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4078, 0, 1, 4096, 0, 0, 0, 23, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4084, 7, 0, 4081, 12, 4083, 4080, 17, 0, 0, 4, 0, 0, 0, 4079, 17, 0, 9, 4084, 4073, 9, 3, 4078, 245, 0, 10, 30, 5, 0, 13, 0, 0, 11, 0, 0, 0, 0, 4083, 4088, 0, 10, 0, 2, 6, 4081, 12, 0, 4087, 4079, 6, 13, 0, 0, 19, 4068, 0, 11, 4084, 4088, 0, 0, 0, 0, 0, 4087, 0, 0, 4089, 5, 10, 4081, 0, 10, 0, 12, 0, 4087, 4082, 14, 6, 3, 4, 0, 15, 16, 0, 10, 2, 8, 11, 3710, 0, 0, 0, 4082, 4073, 5, 11, 4083, 4082, 0, 14, 12, 7, 9, 0, 4050, 0, 3, 4074, 15, 0, 0, 192, 0, 8, 10, 4079, 4038, 11, 8, 0, 2, 4084, 0, 3, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4037, 37, 0, 4053, 45, 4042, 3644, 17, 2, 3, 0, 0, 0, 26, 4034, 181, 0, 50, 4032, 3958, 48, 18, 3984, 945, 18, 49, 101, 27, 16, 418, 0, 0, 63, 0, 19, 0, 0, 4033, 4052, 0, 41, 0, 12, 33, 4032, 63, 0, 4058, 4039, 82, 23, 0, 0, 199, 3702, 0, 52, 4046, 4016, 0, 0, 0, 0, 0, 4055, 48, 18, 4050, 21, 44, 3907, 0, 38, 0, 50, 0, 4052, 4033, 61, 22, 0, 4, 2, 54, 63, 0, 53, 24, 26, 47, 3548, 0, 0, 0, 4019, 4058, 19, 41, 4023, 4029, 0, 68, 68, 44, 43, 25, 3968, 46, 1, 4039, 0, 0, 2, 182, 0, 4, 57, 3558, 4048, 19, 27, 0, 28, 4022, 0, 20, 4044] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4094, 1, 0, 4094, 1, 4094, 4095, 0, 2, 13, 3, 0, 0, 0, 4094, 41, 0, 0, 4094, 4093, 1, 0, 4091, 15, 6, 2, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 0, 2, 4094, 0, 0, 4093, 4094, 1, 0, 4, 0, 4, 4086, 5, 2, 4094, 4090, 0, 0, 0, 0, 0, 4093, 0, 1, 4093, 0, 1, 4085, 0, 1, 0, 2, 0, 4092, 4095, 0, 8, 1, 0, 1, 2, 3, 0, 3, 0, 1, 2, 4044, 0, 0, 0, 4093, 4095, 2, 2, 4094, 4094, 0, 1, 2, 2, 1, 3, 4081, 0, 0, 4094, 0, 1, 0, 6, 0, 1, 0, 4095, 4094, 1, 0, 3, 0, 4090, 0, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 70, 0, 4096, 4075, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3949, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4093, 4085, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16434, 16148, 16381, 16416, 16395, 16485, 16386, 16427] avg=16384.0000 max=16485 max_rank=5 min=16148 min_rank=1 imbalance_ratio=1.006165 max_load_reduction=114587 max_load_reduction_pct=0.874229 moved_to_echo=65693 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 70, 0, 4096, 4054, 0, 0, 17, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 27, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 3949, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4091, 4085, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4078, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 4096, 0, 0, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16429, 16185, 16380, 16418, 16381, 16482, 16370, 16427] avg=16384.0000 max=16482 max_rank=5 min=16185 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=114590 max_load_reduction_pct=0.874252 moved_to_echo=65660 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 70, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3949, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4093, 4085, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16454, 16149, 16381, 16416, 16395, 16485, 16386, 16406] avg=16384.0000 max=16485 max_rank=5 min=16149 min_rank=1 imbalance_ratio=1.006165 max_load_reduction=114587 max_load_reduction_pct=0.874229 moved_to_echo=65672 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4084, 7, 0, 59, 12, 4083, 4080, 17, 0, 0, 4, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 9, 4084, 4073, 9, 3, 4078, 245, 0, 10, 30, 5, 0, 13, 0, 3849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4083, 4088, 0, 10, 0, 2, 6, 4081, 12, 0, 3946, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4079, 6, 13, 0, 0, 19, 4068, 0, 11, 4084, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4089, 5, 10, 4081, 0, 10, 0, 12, 0, 4087, 4082, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 4, 0, 15, 16, 0, 10, 2, 8, 11, 3710, 0, 0, 0, 4082, 4073, 4080, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 11, 4083, 4082, 0, 14, 12, 7, 9, 0, 4050, 0, 3, 4074, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 8, 10, 4079, 4038, 11, 8, 0, 2, 4084, 0, 3, 4084, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16425, 16425, 16388, 16387, 16397, 16142, 16365, 16543] avg=16384.0000 max=16543 max_rank=7 min=16142 min_rank=5 imbalance_ratio=1.009705 max_load_reduction=114529 max_load_reduction_pct=0.873787 moved_to_echo=65447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4037, 37, 0, 69, 45, 4042, 3644, 17, 2, 3, 0, 0, 0, 26, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 50, 4032, 3958, 48, 18, 3984, 945, 18, 49, 101, 27, 16, 418, 0, 3819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 19, 0, 0, 4033, 4052, 0, 41, 0, 12, 33, 4032, 63, 0, 3913, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4039, 82, 23, 0, 0, 199, 3702, 0, 52, 4046, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 48, 18, 4050, 21, 44, 3907, 0, 38, 0, 50, 0, 4052, 4033, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4, 2, 54, 63, 0, 53, 24, 26, 47, 3548, 0, 0, 0, 4019, 4055, 4044, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 19, 41, 4023, 4029, 0, 68, 68, 44, 43, 25, 3968, 46, 1, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 182, 0, 4, 57, 3558, 4048, 19, 27, 0, 28, 4022, 0, 20, 4044, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15956, 17664, 16404, 16190, 16333, 16075, 16417, 16033] avg=16384.0000 max=17664 max_rank=1 min=15956 min_rank=0 imbalance_ratio=1.078125 max_load_reduction=113408 max_load_reduction_pct=0.865234 moved_to_echo=64858 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 1, 0, 70, 1, 4094, 4095, 0, 2, 13, 3, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 4094, 4093, 1, 0, 4091, 15, 6, 2, 0, 1, 1, 0, 0, 3858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 0, 2, 4094, 0, 0, 3946, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4094, 1, 0, 4, 0, 4, 4086, 5, 2, 4094, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1, 4093, 0, 1, 4085, 0, 1, 0, 2, 0, 4092, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 1, 2, 3, 0, 3, 0, 1, 2, 4044, 0, 0, 0, 4093, 4092, 4082, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 2, 4094, 4094, 0, 1, 2, 2, 1, 3, 4081, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 1, 0, 4095, 4094, 1, 0, 3, 0, 4090, 0, 0, 4094, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16467, 16203, 16376, 16412, 16381, 16447, 16379, 16407] avg=16384.0000 max=16467 max_rank=0 min=16203 min_rank=1 imbalance_ratio=1.005066 max_load_reduction=114605 max_load_reduction_pct=0.874367 moved_to_echo=65614 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 70, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3949, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 4093, 4085, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16453, 16149, 16381, 16416, 16395, 16478, 16384, 16416] avg=16384.0000 max=16478 max_rank=5 min=16149 min_rank=1 imbalance_ratio=1.005737 max_load_reduction=114594 max_load_reduction_pct=0.874283 moved_to_echo=65673 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 70, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3949, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 4093, 4085, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 0, 0, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16454, 16149, 16381, 16417, 16395, 16478, 16385, 16413] avg=16384.0000 max=16478 max_rank=5 min=16149 min_rank=1 imbalance_ratio=1.005737 max_load_reduction=114594 max_load_reduction_pct=0.874283 moved_to_echo=65671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 32768, 32768, 0, 128, 128, 128, 128, 128, 32640, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 512, 0, 128, 128, 32256, 32768, 128, 128, 128, 128, 32768, 0, 128, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 0, 128, 32768, 32640, 128, 128, 32640, 1280, 128, 128, 256, 128, 128, 512, 0, 30848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32768, 128, 0, 640, 128, 32768, 32256, 128, 128, 128, 128, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 128, 32640, 0, 128, 0, 128, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 128, 0, 0, 32768, 32768, 0, 128, 0, 128, 128, 32768, 128, 0, 31616, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 128, 31872, 0, 0, 0, 32768, 32768, 32640, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[256, 32768, 128, 128, 128, 0, 256, 32384, 128, 128, 32768, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 3930, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3970, 210, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3892, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4094, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[2995, 30, 0, 1, 4043, 13, 4028, 0, 4037, 4043, 0, 0, 41, 0, 4044, 4013, 0, 0, 4060, 51, 39, 32, 0, 0, 0, 52, 51, 45, 4040, 0, 4049, 967, 0, 0, 4044, 0, 37, 4020, 0, 4022, 0, 23, 0, 58, 0, 3886, 3981, 44, 35, 4009, 39, 73, 0, 0, 71, 4041, 1, 4046, 0, 4037, 0, 72, 0, 4035, 0, 0, 0, 56, 0, 0, 42, 0, 4052, 3, 0, 6, 0, 0, 0, 3913, 0, 0, 3787, 4058, 0, 0, 94, 0, 7, 0, 0, 4054, 3995, 54, 0, 0, 13, 36, 77, 85, 0, 61, 61, 63, 4, 38, 0, 57, 47, 48, 17, 522, 53, 0, 4038, 0, 29, 4049, 4045, 4033, 21, 4059, 0, 4042, 54, 57, 39, 55] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4095, 0, 23, 0, 4091, 0, 4096, 0, 4096, 4096, 1, 0, 0, 0, 4096, 4096, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 139, 4, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3811, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4069, 2, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[3978, 8, 33, 1, 4016, 0, 4084, 0, 4085, 4085, 0, 0, 10, 6, 4087, 4081, 0, 1, 4088, 11, 7, 5, 0, 0, 0, 11, 11, 5, 4085, 3, 4076, 152, 0, 0, 4086, 0, 6, 4080, 0, 4081, 0, 3, 0, 10, 0, 661, 4071, 4, 8, 4079, 11, 3253, 0, 0, 15, 4086, 0, 4083, 0, 4078, 0, 13, 3, 4084, 0, 0, 0, 12, 0, 0, 8, 0, 4084, 0, 0, 2, 0, 0, 0, 4059, 0, 0, 4042, 4087, 0, 5, 30, 19, 1, 0, 0, 4088, 4043, 13, 0, 4, 2, 9, 11, 9, 0, 12, 15, 12, 0, 12, 0, 10, 10, 8, 9, 233, 12, 0, 4084, 0, 7, 4087, 4079, 4086, 3, 4089, 0, 4086, 13, 13, 8, 12] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4021, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 454, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 3633, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4, 0, 4088, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3696, 973, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 52, 4096, 0, 0, 4096, 0, 3314, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 3788, 0, 4094, 0, 4095, 4094, 3, 0, 0, 0, 4094, 4094, 0, 0, 4096, 2, 1, 2, 0, 3, 0, 1, 1, 2, 4093, 1, 4095, 50, 0, 3, 4093, 0, 0, 4093, 0, 4093, 0, 1, 0, 2, 0, 589, 4092, 0, 1, 4093, 0, 3809, 0, 2, 3, 4094, 0, 4095, 0, 4092, 1, 1, 0, 4094, 0, 0, 0, 2, 0, 0, 2, 0, 4094, 0, 1, 2, 1, 0, 3, 4091, 4, 0, 4093, 4090, 0, 0, 11, 2, 1, 0, 0, 4094, 4094, 2, 0, 0, 0, 2, 0, 5, 0, 1, 3, 0, 0, 0, 2, 2, 1, 2, 0, 0, 2, 1, 4093, 0, 0, 4094, 4094, 4094, 0, 4037, 1, 4094, 1, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4, 0, 3810, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 38, 1, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 604, 4096, 0, 0, 4096, 0, 3734, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4094, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 1, 0, 3930, 0, 4096, 0, 277, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3970, 210, 3126, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3397, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3892, 0, 0, 0, 907, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3819, 245, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 45, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 4096, 4096, 4096, 3851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16496, 16150, 15685, 17087, 16426, 16420, 16433, 16375] avg=16384.0000 max=17087 max_rank=3 min=15685 min_rank=2 imbalance_ratio=1.042908 max_load_reduction=113985 max_load_reduction_pct=0.869637 moved_to_echo=65654 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 23, 0, 4091, 0, 4096, 0, 277, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 139, 3126, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3397, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3811, 0, 0, 0, 907, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3819, 246, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4069, 2, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 45, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 4096, 4096, 4096, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16679, 16180, 15689, 17006, 16416, 16408, 16317, 16377] avg=16384.0000 max=17006 max_rank=3 min=15689 min_rank=2 imbalance_ratio=1.037964 max_load_reduction=114066 max_load_reduction_pct=0.870255 moved_to_echo=65518 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2995, 30, 0, 1, 4043, 13, 4028, 0, 273, 0, 0, 0, 41, 0, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 51, 39, 32, 0, 0, 0, 52, 51, 45, 4040, 0, 4049, 967, 3084, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3353, 0, 37, 4020, 0, 4022, 0, 23, 0, 58, 0, 3886, 3981, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4009, 39, 73, 0, 0, 71, 895, 1, 0, 0, 4037, 0, 72, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 42, 0, 4052, 3, 0, 6, 0, 0, 0, 3913, 4042, 3764, 242, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3787, 4058, 0, 0, 94, 0, 7, 0, 0, 4054, 3995, 54, 0, 0, 45, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 36, 77, 85, 0, 61, 61, 63, 4, 38, 0, 57, 47, 48, 17, 522, 4049, 4044, 4046, 3801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 4038, 0, 29, 0, 4038, 4033, 21, 4059, 0, 0, 54, 57, 39, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15437, 17116, 19424, 13267, 16182, 16101, 17069, 16476] avg=16384.0000 max=19424 max_rank=2 min=13267 min_rank=3 imbalance_ratio=1.185547 max_load_reduction=111648 max_load_reduction_pct=0.851807 moved_to_echo=65828 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3978, 8, 33, 1, 4016, 0, 4084, 0, 271, 0, 0, 0, 10, 6, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 11, 7, 5, 0, 0, 0, 11, 11, 5, 4085, 3, 4076, 152, 3123, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3378, 0, 6, 4080, 0, 4081, 0, 3, 0, 10, 0, 661, 4071, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4079, 11, 3253, 0, 0, 15, 899, 0, 0, 0, 4078, 0, 13, 3, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8, 0, 4084, 0, 0, 2, 0, 0, 0, 4059, 4086, 3814, 246, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 4087, 0, 5, 30, 19, 1, 0, 0, 4088, 4043, 13, 0, 4, 52, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 11, 9, 0, 12, 15, 12, 0, 12, 0, 10, 10, 8, 9, 233, 4087, 4087, 4083, 3839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4084, 0, 7, 0, 4070, 4086, 3, 4089, 0, 0, 13, 13, 8, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16488, 16234, 16294, 16443, 16375, 16393, 16448, 16397] avg=16384.0000 max=16488 max_rank=0 min=16234 min_rank=1 imbalance_ratio=1.006348 max_load_reduction=114584 max_load_reduction_pct=0.874207 moved_to_echo=65613 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 3788, 0, 4094, 0, 277, 0, 3, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 1, 2, 0, 3, 0, 1, 1, 2, 4093, 1, 4095, 50, 3124, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3394, 0, 0, 4093, 0, 4093, 0, 1, 0, 2, 0, 589, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 3809, 0, 2, 3, 907, 0, 0, 0, 4092, 1, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 4094, 0, 1, 2, 1, 0, 3, 4091, 4094, 3818, 246, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4093, 4090, 0, 0, 11, 2, 1, 0, 0, 4094, 4094, 2, 0, 0, 45, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 1, 3, 0, 0, 0, 2, 2, 1, 2, 0, 0, 4094, 4094, 4095, 3848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4093, 0, 0, 0, 4087, 4094, 0, 4037, 1, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16350, 16125, 16267, 17003, 16417, 16443, 16149, 16318] avg=16384.0000 max=17003 max_rank=3 min=16125 min_rank=1 imbalance_ratio=1.037781 max_load_reduction=114069 max_load_reduction_pct=0.870277 moved_to_echo=65327 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4021, 0, 4096, 0, 277, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 454, 3126, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3397, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3633, 0, 0, 0, 907, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3819, 245, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 45, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 4096, 4096, 4096, 3851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16586, 16522, 15686, 16828, 16419, 16436, 16218, 16377] avg=16384.0000 max=16828 max_rank=3 min=15686 min_rank=2 imbalance_ratio=1.027100 max_load_reduction=114244 max_load_reduction_pct=0.871613 moved_to_echo=65450 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4, 0, 4088, 0, 4096, 0, 277, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3696, 973, 3126, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3397, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 52, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3314, 0, 0, 0, 907, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3819, 246, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 45, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 4096, 4096, 4096, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16657, 16641, 15737, 16509, 16416, 16437, 16298, 16377] avg=16384.0000 max=16657 max_rank=0 min=15737 min_rank=2 imbalance_ratio=1.016663 max_load_reduction=114415 max_load_reduction_pct=0.872917 moved_to_echo=65528 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4, 0, 3810, 0, 4096, 0, 277, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 38, 3126, 654, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3397, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 604, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3734, 0, 0, 0, 907, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3819, 246, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 45, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 4096, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16104, 16290, 16929, 16421, 16434, 16140, 16375] avg=16384.0000 max=16929 max_rank=3 min=16104 min_rank=1 imbalance_ratio=1.033264 max_load_reduction=114143 max_load_reduction_pct=0.870842 moved_to_echo=65370 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 25600, 0, 128, 128, 7296, 128, 0, 0, 32768, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[31616, 128, 128, 128, 31872, 128, 32768, 0, 2304, 0, 128, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 32512, 32768, 128, 128, 256, 128, 128, 0, 0, 32768, 32640, 128, 0, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 32640, 32768, 128, 32768, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 0, 128, 0, 128, 128, 128, 32768, 128, 32256, 3072, 25088, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 27136, 0, 128, 32768, 0, 32768, 0, 128, 0, 128, 0, 5888, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 1536, 32768, 32768, 32768, 30848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 0, 128, 0, 128, 128, 0, 32768, 128, 128, 128, 128, 0, 128, 32640, 32768, 30592, 2048, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2, 393, 2450, 4, 1029, 0, 827, 3, 1724, 78, 1708, 1038, 1229, 234, 0, 318, 140, 1913, 422, 1439, 3867, 590, 290, 53, 0, 2558, 155, 165, 560, 16, 526, 1857, 4017, 15, 15, 692, 2011, 27, 172, 1318, 2734, 2458, 162, 172, 8, 1025, 862, 731, 355, 370, 609, 114, 62, 35, 3679, 137, 4, 2477, 3556, 2625, 187, 425, 219, 1, 788, 4016, 1, 1547, 213, 39, 3361, 2802, 613, 596, 215, 5, 101, 1237, 1544, 964, 2247, 472, 48, 3852, 90, 925, 2369, 4042, 76, 12, 858, 923, 3226, 266, 897, 3, 650, 450, 3984, 215, 2, 1921, 830, 1815, 3782, 2296, 22, 1355, 754, 1681, 280, 2258, 205, 1203, 1075, 176, 11, 1039, 391, 1987, 823, 1081, 2508, 1275, 379, 5, 1406, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2, 564, 2264, 10, 1801, 2, 827, 4, 1756, 91, 1443, 694, 1410, 187, 0, 648, 253, 1958, 693, 1081, 3745, 736, 321, 142, 0, 2670, 193, 391, 421, 24, 613, 1825, 3847, 1, 11, 1069, 1759, 21, 570, 1631, 2768, 2675, 369, 231, 7, 801, 933, 587, 405, 468, 788, 79, 25, 57, 3829, 142, 0, 2775, 3649, 2979, 128, 327, 206, 3, 621, 4065, 1, 984, 185, 56, 2922, 2750, 872, 649, 142, 20, 275, 1824, 1117, 923, 1983, 935, 50, 3974, 101, 638, 2574, 4027, 44, 40, 612, 788, 2868, 224, 1198, 0, 955, 316, 3974, 269, 0, 2051, 1157, 1903, 3989, 1868, 2, 1020, 832, 1160, 140, 2331, 95, 766, 933, 574, 4, 610, 245, 2056, 600, 688, 2295, 1115, 579, 8, 1166, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[3, 365, 2239, 0, 1096, 0, 710, 2, 1611, 97, 1102, 665, 1336, 259, 0, 584, 202, 1845, 836, 1572, 3581, 844, 418, 42, 0, 2673, 244, 247, 803, 36, 589, 2003, 4057, 0, 21, 1097, 2684, 26, 501, 983, 3319, 2585, 180, 148, 2, 737, 1524, 759, 467, 250, 1025, 124, 34, 11, 4011, 281, 1, 2492, 3686, 2159, 176, 294, 304, 0, 803, 4035, 0, 1234, 212, 91, 3200, 2592, 714, 1000, 292, 4, 407, 1277, 1135, 739, 1630, 754, 37, 3891, 131, 1012, 2342, 3999, 48, 34, 445, 804, 2684, 80, 1182, 0, 518, 583, 4042, 199, 2, 1889, 823, 2447, 3915, 1993, 6, 1831, 689, 1958, 152, 1840, 55, 991, 998, 337, 12, 908, 283, 1686, 758, 535, 1858, 1055, 478, 5, 1475, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2, 829, 1702, 1, 1323, 0, 911, 9, 2144, 66, 2053, 1233, 1507, 433, 1, 439, 177, 1815, 456, 695, 3899, 1110, 197, 54, 2, 3051, 133, 87, 376, 12, 704, 1949, 3894, 6, 9, 1475, 1684, 19, 304, 1075, 3393, 2582, 168, 95, 6, 775, 967, 686, 336, 565, 885, 71, 47, 16, 3813, 110, 2, 3082, 3764, 2490, 273, 147, 224, 0, 410, 4027, 0, 1502, 210, 20, 2997, 2820, 443, 838, 180, 0, 154, 1959, 910, 1385, 2155, 346, 40, 3958, 123, 1165, 2113, 4034, 39, 53, 857, 448, 3089, 157, 801, 3, 809, 399, 4019, 215, 1, 1632, 1158, 2711, 3852, 1808, 0, 995, 788, 2534, 305, 1496, 109, 488, 870, 223, 4, 689, 349, 1840, 482, 545, 2111, 1128, 446, 4, 1493, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[10, 602, 2428, 2, 991, 0, 891, 8, 1519, 249, 1739, 749, 1511, 313, 0, 453, 219, 1766, 947, 1275, 3847, 1109, 242, 36, 0, 2816, 168, 227, 595, 29, 705, 1979, 3872, 1, 12, 1063, 1680, 13, 433, 1581, 3142, 2542, 351, 353, 4, 854, 992, 1040, 373, 373, 912, 119, 97, 26, 3837, 120, 4, 2341, 3788, 2289, 197, 126, 200, 0, 783, 4005, 2, 1289, 104, 129, 2975, 2292, 590, 706, 239, 0, 85, 1754, 1189, 1240, 2234, 467, 48, 3888, 251, 983, 2685, 3920, 35, 49, 692, 832, 2603, 477, 1286, 1, 986, 461, 3896, 279, 1, 1726, 1227, 2197, 3897, 1964, 50, 1124, 647, 1476, 135, 1346, 204, 646, 771, 447, 8, 964, 458, 1572, 586, 941, 2562, 1135, 405, 5, 958, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[6, 756, 2348, 0, 975, 1, 426, 39, 1534, 183, 1163, 729, 1142, 160, 0, 407, 219, 1428, 1232, 1233, 3578, 793, 360, 90, 0, 2337, 183, 322, 913, 25, 754, 2083, 3956, 1, 55, 878, 2446, 16, 645, 1493, 2688, 3198, 458, 192, 7, 759, 1028, 886, 666, 695, 667, 157, 23, 22, 3660, 223, 0, 2434, 3365, 2613, 454, 214, 424, 8, 1486, 4035, 0, 1012, 147, 124, 3097, 2460, 398, 601, 362, 20, 213, 1629, 983, 803, 1876, 806, 57, 3673, 228, 1048, 2332, 3968, 40, 41, 519, 513, 2656, 87, 1072, 7, 752, 540, 3884, 239, 5, 2109, 1241, 2247, 3790, 1914, 20, 1406, 1123, 1378, 83, 2013, 126, 1369, 1106, 578, 17, 878, 394, 1788, 680, 597, 1942, 1439, 348, 8, 1012, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[23, 320, 1531, 58, 1487, 0, 624, 41, 1462, 523, 1380, 335, 896, 341, 5, 590, 262, 1522, 667, 1198, 3792, 1671, 399, 133, 30, 2377, 179, 455, 592, 22, 755, 1944, 3675, 34, 66, 1082, 1837, 30, 629, 1065, 2565, 2566, 396, 153, 57, 1036, 1274, 1447, 452, 204, 1143, 292, 127, 112, 3618, 186, 1, 2635, 3809, 2836, 294, 241, 385, 38, 807, 3809, 35, 1393, 498, 92, 3360, 2425, 757, 585, 212, 16, 443, 1396, 1353, 694, 1469, 1221, 250, 3834, 180, 547, 1895, 3957, 97, 44, 927, 955, 2445, 234, 1638, 22, 1107, 509, 3836, 222, 6, 1351, 1148, 2240, 3374, 1853, 27, 1563, 828, 1490, 179, 2004, 82, 635, 1095, 323, 18, 654, 294, 1896, 921, 618, 2334, 1232, 799, 18, 913, 14] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[3, 221, 1844, 0, 1850, 2, 927, 2, 1489, 118, 1067, 798, 868, 269, 1, 279, 142, 2207, 563, 1203, 3729, 1134, 206, 84, 0, 2439, 173, 65, 889, 15, 993, 2490, 4038, 8, 7, 851, 2094, 34, 528, 2091, 2909, 2928, 517, 167, 6, 646, 1003, 1005, 402, 442, 828, 81, 5, 12, 3653, 130, 6, 2734, 3496, 2318, 198, 280, 387, 0, 703, 4058, 1, 1405, 222, 50, 3095, 2540, 336, 549, 187, 8, 454, 2062, 1243, 686, 1914, 487, 42, 3961, 116, 666, 2853, 4033, 115, 28, 779, 938, 3226, 148, 1074, 0, 1358, 408, 4028, 301, 0, 1884, 839, 1949, 3641, 1485, 16, 839, 490, 1334, 140, 2375, 53, 1021, 1103, 254, 4, 1161, 327, 1454, 754, 695, 2110, 1034, 647, 0, 1514, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 393, 2450, 4, 1029, 0, 827, 3, 1724, 78, 1708, 1038, 1229, 234, 0, 318, 3984, 1675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 1913, 422, 1439, 3867, 590, 290, 53, 0, 2558, 155, 165, 560, 16, 526, 1857, 872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2510, 15, 15, 692, 2011, 27, 172, 1318, 2734, 2458, 162, 172, 8, 1025, 862, 731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 370, 609, 114, 62, 35, 3679, 137, 4, 2477, 3556, 2625, 187, 425, 219, 1, 540, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 788, 2740, 1, 1547, 213, 39, 3361, 2802, 613, 596, 215, 5, 101, 1237, 1544, 964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2247, 472, 48, 3852, 90, 925, 2369, 604, 76, 12, 858, 923, 3226, 266, 897, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 650, 450, 0, 215, 2, 1921, 830, 1815, 2017, 2296, 22, 1355, 754, 1681, 280, 2258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 1203, 1075, 176, 11, 1039, 391, 1987, 823, 1081, 2508, 1275, 379, 5, 1406, 3, 1765, 1763, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16696, 15423, 14912, 15799, 16766, 16868, 16546, 18062] avg=16384.0000 max=18062 max_rank=7 min=14912 min_rank=2 imbalance_ratio=1.102417 max_load_reduction=113010 max_load_reduction_pct=0.862198 moved_to_echo=68242 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 564, 2264, 10, 1801, 2, 827, 4, 1756, 91, 1443, 694, 1410, 187, 0, 648, 3974, 1669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 253, 1958, 693, 1081, 3745, 736, 321, 142, 0, 2670, 193, 391, 421, 24, 613, 1825, 883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2404, 1, 11, 1069, 1759, 21, 570, 1631, 2768, 2675, 369, 231, 7, 801, 933, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 468, 788, 79, 25, 57, 3829, 142, 0, 2775, 3649, 2979, 128, 327, 206, 3, 517, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 2773, 1, 984, 185, 56, 2922, 2750, 872, 649, 142, 20, 275, 1824, 1117, 923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1983, 935, 50, 3974, 101, 638, 2574, 601, 44, 40, 612, 788, 2868, 224, 1198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 955, 316, 0, 269, 0, 2051, 1157, 1903, 2127, 1868, 2, 1020, 832, 1160, 140, 2331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 766, 933, 574, 4, 610, 245, 2056, 600, 688, 2295, 1115, 579, 8, 1166, 0, 1862, 1757, 926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17346, 15949, 15837, 16786, 16114, 16630, 16131, 16279] avg=16384.0000 max=17346 max_rank=0 min=15837 min_rank=2 imbalance_ratio=1.058716 max_load_reduction=113726 max_load_reduction_pct=0.867661 moved_to_echo=65154 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 829, 1702, 1, 1323, 0, 911, 9, 2144, 66, 2053, 1233, 1507, 433, 1, 439, 4019, 1672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 1815, 456, 695, 3899, 1110, 197, 54, 2, 3051, 133, 87, 376, 12, 704, 1949, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2433, 6, 9, 1475, 1684, 19, 304, 1075, 3393, 2582, 168, 95, 6, 775, 967, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 565, 885, 71, 47, 16, 3813, 110, 2, 3082, 3764, 2490, 273, 147, 224, 0, 523, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 2748, 0, 1502, 210, 20, 2997, 2820, 443, 838, 180, 0, 154, 1959, 910, 1385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2155, 346, 40, 3958, 123, 1165, 2113, 602, 39, 53, 857, 448, 3089, 157, 801, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 809, 399, 0, 215, 1, 1632, 1158, 2711, 2054, 1808, 0, 995, 788, 2534, 305, 1496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 488, 870, 223, 4, 689, 349, 1840, 482, 545, 2111, 1128, 446, 4, 1493, 0, 1798, 1760, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18344, 15591, 15677, 16753, 16576, 15949, 16905, 15277] avg=16384.0000 max=18344 max_rank=0 min=15277 min_rank=7 imbalance_ratio=1.119629 max_load_reduction=112728 max_load_reduction_pct=0.860046 moved_to_echo=64707 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[10, 602, 2428, 2, 991, 0, 891, 8, 1519, 249, 1739, 749, 1511, 313, 0, 453, 3896, 1625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 1766, 947, 1275, 3847, 1109, 242, 36, 0, 2816, 168, 227, 595, 29, 705, 1979, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2420, 1, 12, 1063, 1680, 13, 433, 1581, 3142, 2542, 351, 353, 4, 854, 992, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 373, 912, 119, 97, 26, 3837, 120, 4, 2341, 3788, 2289, 197, 126, 200, 0, 520, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 783, 2732, 2, 1289, 104, 129, 2975, 2292, 590, 706, 239, 0, 85, 1754, 1189, 1240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2234, 467, 48, 3888, 251, 983, 2685, 585, 35, 49, 692, 832, 2603, 477, 1286, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 986, 461, 0, 279, 1, 1726, 1227, 2197, 2078, 1964, 50, 1124, 647, 1476, 135, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 646, 771, 447, 8, 964, 458, 1572, 586, 941, 2562, 1135, 405, 5, 958, 5, 1819, 1710, 932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16986, 16830, 16481, 15725, 16109, 17116, 15697, 16128] avg=16384.0000 max=17116 max_rank=5 min=15697 min_rank=6 imbalance_ratio=1.044678 max_load_reduction=113956 max_load_reduction_pct=0.869415 moved_to_echo=65050 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 365, 2239, 0, 1096, 0, 710, 2, 1611, 97, 1102, 665, 1336, 259, 0, 584, 4042, 1661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 1845, 836, 1572, 3581, 844, 418, 42, 0, 2673, 244, 247, 803, 36, 589, 2003, 880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2526, 0, 21, 1097, 2684, 26, 501, 983, 3319, 2585, 180, 148, 2, 737, 1524, 759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 250, 1025, 124, 34, 11, 4011, 281, 1, 2492, 3686, 2159, 176, 294, 304, 0, 549, 411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 803, 2744, 0, 1234, 212, 91, 3200, 2592, 714, 1000, 292, 4, 407, 1277, 1135, 739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1630, 754, 37, 3891, 131, 1012, 2342, 590, 48, 34, 445, 804, 2684, 80, 1182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 583, 0, 199, 2, 1889, 823, 2447, 2085, 1993, 6, 1831, 689, 1958, 152, 1840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 991, 998, 337, 12, 908, 283, 1686, 758, 535, 1858, 1055, 478, 5, 1475, 1, 1830, 1748, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15772, 16815, 17092, 16275, 16444, 15664, 17015, 15995] avg=16384.0000 max=17092 max_rank=2 min=15664 min_rank=5 imbalance_ratio=1.043213 max_load_reduction=113980 max_load_reduction_pct=0.869598 moved_to_echo=65118 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[6, 756, 2348, 0, 975, 1, 426, 39, 1534, 183, 1163, 729, 1142, 160, 0, 407, 3884, 1645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 1428, 1232, 1233, 3578, 793, 360, 90, 0, 2337, 183, 322, 913, 25, 754, 2083, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2472, 1, 55, 878, 2446, 16, 645, 1493, 2688, 3198, 458, 192, 7, 759, 1028, 886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 666, 695, 667, 157, 23, 22, 3660, 223, 0, 2434, 3365, 2613, 454, 214, 424, 8, 531, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1486, 2753, 0, 1012, 147, 124, 3097, 2460, 398, 601, 362, 20, 213, 1629, 983, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1876, 806, 57, 3673, 228, 1048, 2332, 592, 40, 41, 519, 513, 2656, 87, 1072, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 752, 540, 0, 239, 5, 2109, 1241, 2247, 2021, 1914, 20, 1406, 1123, 1378, 83, 2013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 1369, 1106, 578, 17, 878, 394, 1788, 680, 597, 1942, 1439, 348, 8, 1012, 3, 1769, 1731, 953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15398, 16426, 17222, 16562, 16088, 15547, 17091, 16738] avg=16384.0000 max=17222 max_rank=2 min=15398 min_rank=0 imbalance_ratio=1.051147 max_load_reduction=113850 max_load_reduction_pct=0.868607 moved_to_echo=65464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[23, 320, 1531, 58, 1487, 0, 624, 41, 1462, 523, 1380, 335, 896, 341, 5, 590, 3836, 1640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 1522, 667, 1198, 3792, 1671, 399, 133, 30, 2377, 179, 455, 592, 22, 755, 1944, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2296, 34, 66, 1082, 1837, 30, 629, 1065, 2565, 2566, 396, 153, 57, 1036, 1274, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 204, 1143, 292, 127, 112, 3618, 186, 1, 2635, 3809, 2836, 294, 241, 385, 38, 494, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 807, 2599, 35, 1393, 498, 92, 3360, 2425, 757, 585, 212, 16, 443, 1396, 1353, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1469, 1221, 250, 3834, 180, 547, 1895, 591, 97, 44, 927, 955, 2445, 234, 1638, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 509, 0, 222, 6, 1351, 1148, 2240, 1800, 1853, 27, 1563, 828, 1490, 179, 2004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 635, 1095, 323, 18, 654, 294, 1896, 921, 618, 2334, 1232, 799, 18, 913, 14, 1574, 1726, 885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15092, 16825, 16533, 17250, 16665, 16349, 16327, 16031] avg=16384.0000 max=17250 max_rank=3 min=15092 min_rank=0 imbalance_ratio=1.052856 max_load_reduction=113822 max_load_reduction_pct=0.868393 moved_to_echo=65372 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 221, 1844, 0, 1850, 2, 927, 2, 1489, 118, 1067, 798, 868, 269, 1, 279, 4028, 1672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 2207, 563, 1203, 3729, 1134, 206, 84, 0, 2439, 173, 65, 889, 15, 993, 2490, 881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2524, 8, 7, 851, 2094, 34, 528, 2091, 2909, 2928, 517, 167, 6, 646, 1003, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 442, 828, 81, 5, 12, 3653, 130, 6, 2734, 3496, 2318, 198, 280, 387, 0, 542, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 703, 2769, 1, 1405, 222, 50, 3095, 2540, 336, 549, 187, 8, 454, 2062, 1243, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1914, 487, 42, 3961, 116, 666, 2853, 602, 115, 28, 779, 938, 3226, 148, 1074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1358, 408, 0, 301, 0, 1884, 839, 1949, 1942, 1485, 16, 839, 490, 1334, 140, 2375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 1021, 1103, 254, 4, 1161, 327, 1454, 754, 695, 2110, 1034, 647, 0, 1514, 1, 1699, 1759, 972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15438, 17213, 17318, 15922, 16310, 16949, 15360, 16562] avg=16384.0000 max=17318 max_rank=2 min=15360 min_rank=6 imbalance_ratio=1.057007 max_load_reduction=113754 max_load_reduction_pct=0.867874 moved_to_echo=65181 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[15616, 5504, 640, 31104, 1280, 7040, 19200, 4864, 512, 384, 5760, 6272, 22912, 1792, 9216, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1664, 14464, 5888, 9728, 30080, 8064, 2560, 640, 128, 20992, 1536, 2048, 5248, 256, 5760, 16256, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 4096, 16896, 128, 10624, 128, 6144, 128, 13312, 1408, 11776, 6272, 9984, 2304, 128, 3840, 31744, 13312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[19712, 128, 256, 8320, 16256, 256, 3840, 11264, 23552, 21632, 2688, 1536, 128, 6656, 8704, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[7168, 3712, 0, 2048, 128, 14592, 8448, 17536, 16128, 15232, 256, 10240, 6272, 13056, 1536, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1024, 7168, 8064, 2944, 128, 6912, 2816, 14336, 5632, 5760, 17792, 9472, 4096, 128, 9984, 128, 14208, 14080, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[3456, 3456, 6912, 1152, 512, 384, 30208, 1408, 128, 20992, 29184, 20352, 1920, 2176, 2432, 128, 4224, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[6528, 21888, 128, 10368, 1792, 640, 25088, 20736, 4736, 5632, 1920, 128, 2176, 13184, 9600, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 8, 4090, 951, 4073, 1, 0, 1, 1211, 4094, 0, 248, 0, 7, 3709, 0, 1, 0, 8, 0, 4085, 4090, 20, 7, 69, 0, 0, 0, 0, 1142, 0, 0, 4091, 0, 2145, 0, 71, 4, 2, 0, 1, 0, 3498, 7, 0, 0, 0, 0, 2615, 2663, 4083, 0, 0, 26, 21, 14, 4096, 0, 0, 1, 4074, 0, 0, 0, 1802, 0, 0, 4062, 6, 1278, 0, 4066, 3799, 4058, 1, 1, 1, 0, 0, 0, 4, 0, 21, 7, 4096, 4090, 4093, 6, 4080, 0, 0, 3930, 0, 0, 0, 0, 4, 4096, 4080, 17, 2, 0, 3, 0, 3, 7, 6, 10, 20, 0, 3965, 0, 22, 10, 0, 0, 0, 4054, 4086, 3640, 0, 4024, 0, 4086, 0, 4080, 0, 229] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4095, 1844, 4094, 0, 0, 5, 573, 4094, 0, 2341, 1, 0, 3998, 0, 0, 71, 0, 0, 4092, 4095, 0, 0, 91, 1, 0, 0, 1, 744, 0, 0, 4092, 0, 2088, 1, 0, 11, 0, 0, 2, 0, 534, 5, 0, 0, 2, 17, 3685, 1722, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 4, 0, 888, 1, 0, 4089, 4, 2091, 0, 4075, 4079, 4095, 0, 0, 0, 0, 2, 1, 1, 5, 0, 0, 4096, 4094, 4095, 0, 3996, 0, 0, 4086, 0, 0, 37, 0, 2, 4093, 4096, 0, 0, 0, 0, 0, 3, 6, 0, 1, 3, 0, 4095, 0, 9, 0, 1, 0, 0, 3933, 4095, 4025, 0, 4095, 0, 4096, 0, 4094, 0, 106] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4095, 1024, 4095, 0, 0, 0, 90, 4095, 0, 545, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 300, 0, 0, 4096, 0, 1040, 0, 0, 2, 0, 0, 0, 0, 3094, 45, 0, 0, 0, 2, 3921, 253, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3849, 5, 0, 4058, 0, 1632, 0, 4096, 3975, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4096, 4096, 4096, 0, 4076, 0, 0, 4072, 0, 3, 67, 0, 0, 4096, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 1, 5, 0, 4089, 0, 16, 0, 0, 0, 0, 4095, 4096, 3981, 0, 4094, 0, 4096, 0, 4096, 0, 817] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4093, 65, 4096, 2, 8, 2, 70, 4088, 0, 2376, 0, 1, 3739, 0, 0, 0, 1, 0, 4096, 4093, 0, 0, 17, 0, 0, 0, 0, 2285, 9, 0, 4081, 0, 1520, 0, 1, 0, 0, 1, 0, 97, 3378, 0, 0, 1, 0, 0, 1397, 1897, 4082, 0, 3, 1, 116, 11, 4096, 0, 0, 26, 4095, 0, 0, 8, 3495, 1, 0, 4062, 1, 19, 0, 4096, 3980, 4096, 3, 0, 0, 1, 3, 0, 0, 62, 6, 19, 4096, 4095, 4064, 4, 4047, 0, 0, 3982, 0, 0, 0, 15, 0, 4095, 4089, 0, 40, 2, 1, 0, 0, 13, 0, 0, 44, 0, 4081, 1, 256, 42, 1, 0, 0, 3972, 4089, 4090, 4, 4063, 8, 4084, 0, 4069, 0, 29] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 4095, 2746, 4071, 22, 0, 0, 61, 3973, 0, 744, 0, 13, 3851, 5, 0, 26, 57, 5, 4054, 4091, 0, 16, 118, 1, 4, 6, 0, 557, 0, 0, 4088, 16, 738, 0, 80, 12, 0, 0, 38, 1, 2334, 0, 7, 5, 0, 0, 3204, 2258, 4061, 12, 5, 0, 2, 55, 4090, 0, 0, 0, 4095, 0, 0, 8, 3324, 160, 1, 3810, 0, 743, 0, 4052, 3777, 4070, 0, 0, 0, 32, 133, 0, 0, 38, 3, 25, 4045, 4076, 4095, 37, 4048, 0, 1, 3892, 1, 0, 147, 32, 0, 4090, 4092, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4071, 0, 103, 6, 1, 15, 0, 4016, 4068, 3992, 5, 4037, 16, 4096, 0, 4094, 6, 175] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 4092, 170, 4093, 0, 0, 31, 1593, 4093, 0, 14, 0, 0, 4023, 0, 0, 0, 0, 0, 4089, 4076, 1, 0, 12, 0, 2, 0, 0, 3133, 0, 0, 4095, 0, 1833, 1, 0, 107, 0, 0, 0, 59, 1779, 8, 2, 4, 0, 24, 3453, 439, 4093, 0, 1, 0, 18, 2, 4095, 0, 0, 0, 4086, 0, 0, 0, 3171, 0, 0, 4068, 0, 2232, 0, 4029, 3753, 4095, 2, 0, 0, 2, 2, 0, 2, 0, 0, 4, 4096, 4094, 4096, 0, 3946, 0, 2, 4053, 0, 2, 3, 0, 2, 4096, 4095, 0, 6, 0, 0, 0, 0, 2, 0, 0, 1, 0, 4095, 0, 0, 1, 0, 0, 1, 2936, 4095, 4018, 0, 4067, 1, 4095, 1, 4091, 0, 295] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4074, 39, 4091, 41, 10, 1, 3723, 4062, 0, 369, 0, 59, 2307, 0, 0, 37, 2, 0, 4012, 4074, 96, 17, 0, 0, 0, 1, 0, 91, 0, 0, 3964, 11, 2575, 2, 11, 4, 1, 0, 0, 0, 3382, 148, 27, 24, 0, 9, 1227, 42, 4083, 0, 0, 0, 8, 0, 4074, 0, 3, 2, 4092, 0, 20, 0, 2383, 1, 1, 4048, 0, 2364, 2, 4096, 3952, 4058, 132, 0, 0, 131, 0, 0, 0, 0, 0, 22, 4072, 4096, 4072, 1, 3666, 0, 0, 4069, 1, 0, 1360, 60, 11, 4061, 4096, 0, 46, 0, 0, 1, 0, 46, 0, 1, 0, 0, 4092, 0, 69, 1, 1, 0, 0, 4088, 4086, 3976, 2, 3597, 0, 4072, 5, 4089, 0, 1330] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4096, 1165, 4078, 0, 53, 9, 109, 4096, 0, 154, 0, 0, 4040, 0, 0, 0, 0, 0, 4096, 4059, 0, 0, 258, 0, 0, 24, 0, 181, 0, 0, 4096, 0, 2124, 0, 0, 0, 20, 0, 10, 0, 3174, 0, 0, 2, 0, 58, 3721, 917, 4073, 0, 0, 0, 11, 0, 4095, 0, 2, 0, 4096, 0, 3, 0, 3061, 1, 0, 4060, 0, 1414, 0, 4091, 4096, 4089, 45, 0, 0, 0, 0, 0, 0, 2, 0, 10, 4096, 4096, 4096, 0, 4095, 0, 0, 4093, 0, 3, 19, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 1, 5, 0, 0, 16, 0, 4069, 0, 294, 15, 0, 0, 0, 4044, 4095, 4041, 0, 3920, 0, 4095, 0, 4096, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1692, 1844, 4094, 0, 0, 5, 573, 4094, 0, 2341, 1, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 4092, 4095, 0, 0, 91, 1, 0, 0, 1, 744, 0, 0, 4095, 2566, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2088, 1, 0, 11, 0, 0, 2, 0, 534, 5, 0, 0, 2, 17, 4094, 3697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3685, 1722, 4092, 0, 0, 0, 0, 0, 3990, 0, 0, 0, 4093, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 888, 1, 0, 4089, 4, 2091, 0, 4075, 4079, 49, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 4096, 0, 4034, 0, 3996, 0, 0, 4086, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 4096, 0, 0, 0, 0, 0, 3, 6, 0, 1, 3, 0, 4095, 0, 2403, 1480, 106, 61, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 1, 0, 0, 3933, 0, 4025, 0, 4095, 0, 0, 0, 4053, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18642, 16155, 14543, 17586, 15279, 16255, 16390, 16222] avg=16384.0000 max=18642 max_rank=0 min=14543 min_rank=2 imbalance_ratio=1.137817 max_load_reduction=112430 max_load_reduction_pct=0.857773 moved_to_echo=64146 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1687, 1024, 4095, 0, 0, 0, 90, 4095, 0, 545, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 300, 0, 0, 4096, 2570, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1040, 0, 0, 2, 0, 0, 0, 0, 3094, 45, 0, 0, 0, 2, 4096, 3697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 253, 4096, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 5, 0, 4058, 0, 1632, 0, 4096, 3975, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4096, 0, 4029, 0, 4076, 0, 0, 4072, 0, 3, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 1, 5, 0, 4089, 0, 2408, 1485, 111, 67, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 4095, 0, 3981, 0, 4094, 0, 0, 0, 4051, 0, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15630, 15559, 16072, 16351, 17656, 16346, 16404, 17054] avg=16384.0000 max=17656 max_rank=4 min=15559 min_rank=1 imbalance_ratio=1.077637 max_load_reduction=113416 max_load_reduction_pct=0.865295 moved_to_echo=67460 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 1690, 951, 4073, 1, 0, 1, 1211, 4094, 0, 248, 0, 7, 3709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 0, 4085, 4090, 20, 7, 69, 0, 0, 0, 0, 1142, 0, 0, 4086, 2543, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2145, 0, 71, 4, 2, 0, 1, 0, 3498, 7, 0, 0, 0, 0, 4090, 3688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2615, 2663, 4083, 0, 0, 26, 21, 14, 3990, 0, 0, 1, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1802, 0, 0, 4062, 6, 1278, 0, 4066, 3799, 48, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 21, 7, 4096, 0, 4032, 6, 4080, 0, 0, 3930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4080, 17, 2, 0, 3, 0, 3, 7, 6, 10, 20, 0, 3965, 0, 2400, 1467, 106, 61, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 10, 0, 0, 0, 4054, 0, 3640, 0, 4024, 0, 0, 0, 4039, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15993, 16449, 17597, 17487, 15064, 16176, 16288, 16018] avg=16384.0000 max=17597 max_rank=2 min=15064 min_rank=4 imbalance_ratio=1.074036 max_load_reduction=113475 max_load_reduction_pct=0.865746 moved_to_echo=63546 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1692, 2746, 4071, 22, 0, 0, 61, 3973, 0, 744, 0, 13, 3851, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 57, 5, 4054, 4091, 0, 16, 118, 1, 4, 6, 0, 557, 0, 0, 4068, 2550, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 16, 738, 0, 80, 12, 0, 0, 38, 1, 2334, 0, 7, 5, 0, 0, 4076, 3697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3204, 2258, 4061, 12, 5, 0, 2, 55, 3984, 0, 0, 0, 4095, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3324, 160, 1, 3810, 0, 743, 0, 4052, 3777, 49, 0, 0, 0, 32, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 3, 25, 4045, 0, 4034, 37, 4048, 0, 1, 3892, 1, 0, 147, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4092, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4071, 0, 2403, 1471, 106, 61, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 6, 1, 15, 0, 4016, 0, 3992, 5, 4037, 16, 0, 0, 4053, 6, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17179, 15952, 15092, 17684, 16081, 16303, 16356, 16425] avg=16384.0000 max=17684 max_rank=3 min=15092 min_rank=2 imbalance_ratio=1.079346 max_load_reduction=113388 max_load_reduction_pct=0.865082 moved_to_echo=65165 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1691, 65, 4096, 2, 8, 2, 70, 4088, 0, 2376, 0, 1, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4093, 0, 0, 17, 0, 0, 0, 0, 2285, 9, 0, 4089, 2567, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 1520, 0, 1, 0, 0, 1, 0, 97, 3378, 0, 0, 1, 0, 0, 4095, 3686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1397, 1897, 4082, 0, 3, 1, 116, 11, 3990, 0, 0, 26, 4095, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3495, 1, 0, 4062, 1, 19, 0, 4096, 3980, 48, 3, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 6, 19, 4096, 0, 4003, 4, 4047, 0, 0, 3982, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4089, 0, 40, 2, 1, 0, 0, 13, 0, 0, 44, 0, 4081, 1, 2402, 1481, 106, 61, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 42, 1, 0, 0, 3972, 0, 4090, 4, 4063, 8, 0, 0, 4028, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16138, 17555, 16860, 15626, 15709, 16234, 16457, 16493] avg=16384.0000 max=17555 max_rank=1 min=15626 min_rank=3 imbalance_ratio=1.071472 max_load_reduction=113517 max_load_reduction_pct=0.866066 moved_to_echo=64893 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1691, 170, 4093, 0, 0, 31, 1593, 4093, 0, 14, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4076, 1, 0, 12, 0, 2, 0, 0, 3133, 0, 0, 4095, 2566, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1833, 1, 0, 107, 0, 0, 0, 59, 1779, 8, 2, 4, 0, 24, 4094, 3697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3453, 439, 4093, 0, 1, 0, 18, 2, 3989, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3171, 0, 0, 4068, 0, 2232, 0, 4029, 3753, 49, 2, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 4096, 0, 4035, 0, 3946, 0, 2, 4053, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4095, 0, 6, 0, 0, 0, 0, 2, 0, 0, 1, 0, 4095, 0, 2401, 1480, 106, 61, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 2936, 0, 4018, 0, 4067, 1, 0, 1, 4050, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15709, 18372, 15703, 16081, 17308, 16143, 16386, 15370] avg=16384.0000 max=18372 max_rank=1 min=15370 min_rank=7 imbalance_ratio=1.121338 max_load_reduction=112700 max_load_reduction_pct=0.859833 moved_to_echo=65207 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1683, 39, 4091, 41, 10, 1, 3723, 4062, 0, 369, 0, 59, 2307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 2, 0, 4012, 4074, 96, 17, 0, 0, 0, 1, 0, 91, 0, 0, 4086, 2543, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 11, 2575, 2, 11, 4, 1, 0, 0, 0, 3382, 148, 27, 24, 0, 9, 4096, 3675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 42, 4083, 0, 0, 0, 8, 0, 3968, 0, 3, 2, 4092, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2383, 1, 1, 4048, 0, 2364, 2, 4096, 3952, 48, 132, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4072, 0, 4011, 1, 3666, 0, 0, 4069, 1, 0, 1360, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4061, 4096, 0, 46, 0, 0, 1, 0, 46, 0, 1, 0, 0, 4092, 0, 2391, 1467, 106, 61, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 1, 1, 0, 0, 4088, 0, 3976, 2, 3597, 0, 0, 5, 4048, 0, 1330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 15356, 17929, 13445, 17158, 17262, 16420, 17117] avg=16384.0000 max=17929 max_rank=2 min=13445 min_rank=3 imbalance_ratio=1.094299 max_load_reduction=113143 max_load_reduction_pct=0.863213 moved_to_echo=67957 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1692, 1165, 4078, 0, 53, 9, 109, 4096, 0, 154, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4059, 0, 0, 258, 0, 0, 24, 0, 181, 0, 0, 4095, 2562, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2124, 0, 0, 0, 20, 0, 10, 0, 3174, 0, 0, 2, 0, 58, 4096, 3696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3721, 917, 4073, 0, 0, 0, 11, 0, 3989, 0, 2, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3061, 1, 0, 4060, 0, 1414, 0, 4091, 4096, 49, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 10, 4096, 0, 4035, 0, 4095, 0, 0, 4093, 0, 3, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 1, 5, 0, 0, 16, 0, 4069, 0, 2404, 1478, 106, 61, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 15, 0, 0, 0, 4044, 0, 4041, 0, 3920, 0, 0, 0, 4055, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15396, 15674, 17276, 16812, 16817, 16353, 16371, 16373] avg=16384.0000 max=17276 max_rank=2 min=15396 min_rank=0 imbalance_ratio=1.054443 max_load_reduction=113796 max_load_reduction_pct=0.868195 moved_to_echo=65914 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32640, 128, 14080, 128, 256, 256, 128, 128, 128, 256, 21248, 256, 128, 128, 128, 128, 32768, 29568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 13568, 8064, 32768, 128, 128, 128, 7552, 32640, 0, 6912, 128, 128, 29824, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[23296, 10240, 32768, 128, 128, 128, 256, 128, 32000, 0, 128, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32640, 128, 19328, 11904, 896, 512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 32256, 128, 32000, 0, 128, 32256, 128, 128, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[896, 128, 128, 128, 128, 31232, 0, 31872, 128, 32000, 128, 0, 128, 32384, 128, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 256, 128, 128, 32640, 32768, 128, 128, 640, 128, 128, 128, 128, 8448, 128, 0, 32768, 20480, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[22016, 256, 128, 32384, 128, 11776, 128, 32640, 31488, 384, 256, 128, 128, 256, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[3591, 0, 0, 4096, 0, 0, 1, 0, 4093, 0, 1, 4095, 4092, 0, 0, 315, 1907, 4095, 0, 3990, 501, 3368, 0, 1, 0, 4095, 0, 0, 3934, 1738, 0, 4096, 1, 4095, 3, 1, 1, 0, 3, 4095, 1, 0, 0, 0, 4095, 0, 2, 0, 2, 2613, 0, 5, 0, 1, 0, 0, 4096, 0, 0, 5, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 4092, 0, 1, 4093, 0, 4, 79, 3, 0, 4096, 0, 2, 1, 0, 21, 221, 0, 0, 4096, 1, 4092, 2944, 0, 166, 4095, 1, 0, 0, 4096, 1, 2, 0, 4095, 2089, 0, 4096, 1, 4, 4095, 0, 1168, 4093, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4096, 1, 2, 4095, 200, 0, 0, 1, 4094, 1, 0, 4095, 4090, 0, 0, 1547, 1769, 4094, 3, 3960, 8, 1664, 0, 0, 0, 4096, 0, 0, 3994, 147, 0, 4087, 0, 4096, 2, 6, 0, 0, 0, 4093, 0, 3, 2, 0, 4096, 0, 0, 0, 0, 3020, 0, 0, 0, 0, 0, 3, 4094, 1, 0, 0, 4096, 0, 4092, 0, 3, 2, 2, 1, 2, 0, 0, 0, 4093, 2, 4090, 0, 4, 4096, 1, 3, 54, 0, 0, 4093, 0, 0, 0, 0, 31, 3, 0, 0, 4087, 0, 4091, 2093, 0, 45, 4086, 0, 1, 0, 4074, 1, 2116, 0, 4096, 2564, 0, 4096, 2, 8, 4087, 0, 1373, 4095, 0, 0, 0, 0, 0, 2, 0, 0, 27, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4078, 0, 0, 4096, 12, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 28, 2321, 4096, 0, 4096, 131, 3646, 0, 0, 0, 4096, 0, 0, 3965, 1002, 0, 4089, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 1799, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 4096, 0, 0, 453, 0, 0, 4096, 0, 0, 0, 0, 24, 53, 0, 0, 4096, 0, 4085, 2421, 0, 758, 4086, 0, 0, 0, 4090, 0, 282, 0, 4096, 3361, 0, 4096, 0, 11, 4096, 0, 260, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[3600, 0, 2, 4095, 2, 0, 0, 0, 4096, 0, 0, 4093, 4096, 0, 0, 11, 2877, 4096, 1, 3988, 165, 3426, 0, 0, 1, 4087, 0, 0, 3803, 373, 2, 4066, 4, 4093, 5, 6, 2, 0, 0, 4093, 0, 0, 8, 1, 4096, 0, 0, 0, 0, 483, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 2, 0, 4096, 7, 4093, 0, 0, 4096, 7, 0, 96, 4, 0, 4095, 1, 3, 5, 0, 1, 209, 0, 0, 4095, 0, 4096, 3616, 0, 2406, 4096, 0, 0, 0, 4006, 0, 227, 0, 4096, 1571, 0, 4095, 0, 52, 4096, 0, 1861, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4090, 0, 0, 4096, 115, 0, 0, 0, 4091, 0, 0, 4096, 4047, 0, 0, 1351, 2515, 4095, 0, 4082, 114, 2161, 0, 0, 0, 4092, 0, 0, 3695, 801, 0, 4080, 0, 4096, 34, 3, 0, 0, 0, 4096, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 3788, 0, 1, 2, 1, 0, 0, 4096, 0, 0, 5, 4095, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 9, 4096, 0, 1, 4095, 0, 0, 8, 0, 0, 4096, 1, 4, 0, 0, 186, 0, 0, 0, 4092, 0, 4085, 3106, 0, 141, 4091, 0, 1, 0, 4087, 0, 117, 0, 4093, 2225, 0, 4096, 0, 0, 4073, 0, 229, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[3879, 0, 0, 4096, 0, 0, 0, 2, 4096, 3, 0, 4075, 4071, 9, 0, 0, 1732, 4088, 0, 3895, 81, 4044, 0, 0, 0, 4096, 0, 0, 4059, 2034, 15, 4094, 3, 4072, 13, 0, 18, 31, 0, 4095, 0, 0, 1, 0, 4081, 0, 15, 0, 0, 1414, 0, 23, 5, 0, 0, 0, 4086, 0, 26, 0, 4065, 4, 4081, 0, 1, 0, 1, 0, 0, 8, 8, 3, 4096, 14, 4029, 62, 0, 4093, 0, 0, 29, 0, 0, 4086, 0, 29, 24, 0, 71, 107, 0, 0, 4086, 0, 4045, 2787, 8, 1099, 4095, 8, 0, 8, 3972, 2, 225, 4, 4096, 3141, 1, 4087, 0, 81, 4023, 7, 147, 4081, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4067, 0, 1, 4096, 512, 0, 0, 2, 4069, 0, 0, 4095, 4034, 0, 0, 1, 649, 4095, 0, 3178, 136, 3863, 0, 0, 7, 4096, 0, 0, 3646, 1949, 1, 3638, 0, 4096, 413, 3, 0, 1, 0, 4094, 0, 0, 18, 0, 4071, 0, 0, 0, 0, 956, 0, 3, 0, 0, 0, 0, 4096, 14, 0, 0, 4068, 0, 4073, 0, 3, 0, 0, 1, 0, 1, 0, 18, 4096, 22, 4080, 1, 0, 4094, 1, 0, 0, 0, 0, 4094, 2, 0, 31, 2, 327, 10, 0, 19, 4012, 0, 4084, 1801, 0, 2586, 4082, 0, 0, 0, 4096, 0, 0, 0, 4096, 3879, 0, 4093, 11, 0, 4039, 0, 1301, 4095, 0, 0, 0, 0, 0, 0, 0, 14, 1, 0, 39, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[3852, 0, 0, 4096, 20, 0, 0, 0, 4096, 0, 0, 4088, 4094, 0, 0, 71, 2905, 4085, 0, 4070, 9, 3997, 0, 0, 0, 4096, 0, 0, 2746, 606, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2502, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 4096, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 4096, 0, 1, 1, 1, 0, 4096, 0, 0, 0, 0, 11, 18, 0, 0, 4096, 0, 4096, 4058, 0, 52, 4096, 0, 0, 0, 4096, 0, 58, 0, 4096, 3634, 0, 4096, 0, 7, 4079, 0, 74, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 8, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3591, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 1, 3786, 4092, 0, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1907, 0, 0, 3990, 501, 3368, 0, 1, 0, 0, 0, 0, 3934, 1738, 0, 2199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 3, 1, 1, 0, 3, 4095, 1, 0, 0, 0, 4095, 0, 2, 0, 3229, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2613, 0, 5, 0, 1, 0, 0, 4096, 0, 0, 5, 4096, 0, 4096, 1, 1194, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 4092, 0, 1, 4093, 0, 4, 3295, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 3, 0, 4096, 0, 2, 1, 0, 21, 221, 0, 0, 4096, 1, 4092, 2944, 703, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 4095, 1, 0, 0, 4096, 1, 2, 0, 15, 2089, 0, 4096, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1168, 4093, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0, 4096, 3304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15879, 17638, 16326, 16960, 16372, 16568, 14566, 16763] avg=16384.0000 max=17638 max_rank=1 min=14566 min_rank=6 imbalance_ratio=1.076538 max_load_reduction=113434 max_load_reduction_pct=0.865433 moved_to_echo=64269 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 1, 2, 0, 200, 0, 0, 1, 4094, 1, 0, 3786, 4090, 0, 0, 1547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1769, 0, 3, 3960, 8, 1664, 0, 0, 0, 0, 0, 0, 3994, 147, 0, 2194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 6, 0, 0, 0, 4093, 0, 3, 2, 0, 4096, 0, 0, 0, 3230, 799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3020, 0, 0, 0, 0, 0, 3, 4094, 1, 0, 0, 4096, 0, 4092, 0, 1192, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 2, 1, 2, 0, 0, 0, 4093, 2, 4090, 0, 4, 4096, 1, 3, 3295, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 4093, 0, 0, 0, 0, 31, 3, 0, 0, 4087, 0, 4091, 2093, 701, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 4086, 0, 1, 0, 4074, 1, 2116, 0, 15, 2564, 0, 4096, 2, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 1373, 4095, 0, 0, 0, 0, 0, 2, 0, 0, 27, 0, 0, 0, 4095, 3305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17818, 13739, 16327, 17349, 16385, 15462, 17008, 16984] avg=16384.0000 max=17818 max_rank=0 min=13739 min_rank=1 imbalance_ratio=1.087524 max_load_reduction=113254 max_load_reduction_pct=0.864059 moved_to_echo=65839 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4078, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 0, 3782, 4096, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2321, 0, 0, 4096, 131, 3646, 0, 0, 0, 0, 0, 0, 3965, 1002, 0, 2188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 3232, 801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1799, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 1195, 854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 4096, 0, 0, 3295, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 453, 0, 0, 4096, 0, 0, 0, 0, 24, 53, 0, 0, 4096, 0, 4085, 2421, 706, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 758, 4086, 0, 0, 0, 4090, 0, 282, 0, 10, 3361, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 260, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 3304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16092, 17349, 16323, 16136, 16377, 16248, 16694, 15853] avg=16384.0000 max=17349 max_rank=1 min=15853 min_rank=7 imbalance_ratio=1.058899 max_load_reduction=113723 max_load_reduction_pct=0.867638 moved_to_echo=65172 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 115, 0, 0, 0, 4091, 0, 0, 3787, 4047, 0, 0, 1351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2515, 0, 0, 4082, 114, 2161, 0, 0, 0, 0, 0, 0, 3695, 801, 0, 2190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 34, 3, 0, 0, 0, 4096, 0, 0, 5, 0, 4096, 0, 0, 0, 3227, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3788, 0, 1, 2, 1, 0, 0, 4096, 0, 0, 5, 4095, 0, 4091, 0, 1190, 850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 9, 4096, 0, 1, 4095, 0, 0, 3295, 790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 1, 4, 0, 0, 186, 0, 0, 0, 4092, 0, 4085, 3106, 700, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 4091, 0, 1, 0, 4087, 0, 117, 0, 16, 2225, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 229, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 9, 4096, 3302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17481, 15558, 16357, 18119, 16382, 16587, 14774, 15814] avg=16384.0000 max=18119 max_rank=3 min=14774 min_rank=6 imbalance_ratio=1.105896 max_load_reduction=112953 max_load_reduction_pct=0.861763 moved_to_echo=63557 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3879, 0, 0, 0, 0, 0, 0, 2, 4096, 3, 0, 3767, 4071, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1732, 0, 0, 3895, 81, 4044, 0, 0, 0, 0, 0, 0, 4059, 2034, 15, 2197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4072, 13, 0, 18, 31, 0, 4095, 0, 0, 1, 0, 4081, 0, 15, 0, 3230, 799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1414, 0, 23, 5, 0, 0, 0, 4086, 0, 26, 0, 4065, 4, 4081, 0, 1194, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8, 8, 3, 4096, 14, 4029, 62, 0, 4093, 0, 0, 3289, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 4086, 0, 29, 24, 0, 71, 107, 0, 0, 4086, 0, 4045, 2787, 703, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1099, 4095, 8, 0, 8, 3972, 2, 225, 4, 15, 3141, 1, 4087, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 7, 147, 4081, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15827, 18057, 16358, 15749, 16395, 16275, 16746, 15665] avg=16384.0000 max=18057 max_rank=1 min=15665 min_rank=7 imbalance_ratio=1.102112 max_load_reduction=113015 max_load_reduction_pct=0.862236 moved_to_echo=65081 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4067, 0, 1, 0, 512, 0, 0, 2, 4069, 0, 0, 3786, 4034, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 0, 0, 3178, 136, 3863, 0, 0, 7, 0, 0, 0, 3646, 1949, 1, 1953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 413, 3, 0, 1, 0, 4094, 0, 0, 18, 0, 4071, 0, 0, 0, 3230, 801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 956, 0, 3, 0, 0, 0, 0, 4096, 14, 0, 0, 4068, 0, 4073, 0, 1061, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 1, 0, 18, 4096, 22, 4080, 1, 0, 4094, 1, 0, 3294, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 31, 2, 327, 10, 0, 19, 4012, 0, 4084, 1801, 624, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2586, 4082, 0, 0, 0, 4096, 0, 0, 0, 15, 3879, 0, 4093, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 1301, 4095, 0, 0, 0, 0, 0, 0, 0, 14, 1, 0, 39, 0, 4096, 3304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16472, 15382, 16727, 15122, 16403, 15315, 18762, 16889] avg=16384.0000 max=18762 max_rank=6 min=15122 min_rank=3 imbalance_ratio=1.145142 max_load_reduction=112310 max_load_reduction_pct=0.856857 moved_to_echo=67369 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3600, 0, 2, 0, 2, 0, 0, 0, 4096, 0, 0, 3784, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2877, 0, 1, 3988, 165, 3426, 0, 0, 1, 0, 0, 0, 3803, 373, 2, 2183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4093, 5, 6, 2, 0, 0, 4093, 0, 0, 8, 1, 4096, 0, 0, 0, 3230, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4089, 0, 4096, 0, 1185, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 4096, 7, 4093, 0, 0, 4096, 7, 0, 3296, 789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 4, 0, 4095, 1, 3, 5, 0, 1, 209, 0, 0, 4095, 0, 4096, 3616, 698, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2406, 4096, 0, 0, 0, 4006, 0, 227, 0, 15, 1571, 0, 4095, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1861, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15591, 16819, 16338, 14799, 16387, 17228, 16468, 17442] avg=16384.0000 max=17442 max_rank=7 min=14799 min_rank=3 imbalance_ratio=1.064575 max_load_reduction=113630 max_load_reduction_pct=0.866928 moved_to_echo=67525 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3852, 0, 0, 0, 20, 0, 0, 0, 4096, 0, 0, 3779, 4094, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2905, 0, 0, 4070, 9, 3997, 0, 0, 0, 0, 0, 0, 2746, 606, 0, 2197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3230, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2502, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 4096, 0, 4096, 0, 1194, 851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 4096, 0, 1, 3287, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4096, 0, 0, 0, 0, 11, 18, 0, 0, 4096, 0, 4096, 4058, 703, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 4096, 0, 0, 0, 4096, 0, 58, 0, 15, 3634, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 74, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 8, 0, 0, 1, 4096, 3305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15912, 16530, 16316, 16838, 16371, 17389, 16054, 15662] avg=16384.0000 max=17389 max_rank=5 min=15662 min_rank=7 imbalance_ratio=1.061340 max_load_reduction=113683 max_load_reduction_pct=0.867332 moved_to_echo=65476 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[31360, 128, 128, 0, 896, 0, 128, 128, 32768, 128, 128, 30336, 32640, 128, 0, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32640, 128, 6528, 32768, 0, 0, 128, 128, 0, 128, 128, 128, 128, 0, 128, 128, 32768, 26496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 512, 128, 128, 128, 128, 32768, 128, 128, 128, 128, 32768, 0, 128, 0, 25856, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 7296, 32768, 128, 128, 128, 32640, 128, 3072, 128, 128, 22528, 128, 32768, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 16640, 0, 128, 128, 128, 0, 128, 32768, 128, 128, 128, 32768, 128, 32768, 128, 9472, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[16768, 0, 128, 31360, 1152, 26240, 0, 128, 128, 0, 0, 0, 29952, 8704, 128, 17408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 32768, 128, 128, 32768, 128, 128, 26368, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[768, 128, 0, 32768, 128, 128, 128, 128, 768, 640, 0, 128, 32768, 128, 32768, 22912, 5632, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[4092, 1, 0, 0, 423, 2, 4093, 2039, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4, 1, 0, 0, 0, 0, 0, 1, 4092, 0, 4093, 4093, 2, 0, 4095, 0, 1, 0, 0, 0, 0, 4095, 2, 2017, 0, 4088, 1, 4095, 0, 1, 4096, 1, 0, 3, 0, 13, 0, 0, 4096, 0, 0, 0, 0, 4094, 4, 3, 0, 419, 4093, 3, 2, 0, 1, 357, 0, 0, 3096, 0, 0, 4047, 4095, 2, 0, 0, 4095, 4095, 1, 0, 0, 0, 4095, 1, 0, 1, 330, 4078, 0, 4064, 1, 3827, 3962, 4095, 0, 4095, 0, 0, 4094, 0, 4093, 0, 0, 1, 1, 0, 0, 4096, 0, 0, 1, 5, 0, 0, 0, 0, 1, 4085, 0, 0, 8, 4094, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 102, 0, 4096, 2099, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4096, 0, 949, 0, 4010, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 4, 0, 0, 534, 4096, 0, 0, 0, 0, 646, 0, 0, 4069, 0, 0, 4095, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 29, 4096, 0, 4066, 0, 4091, 4095, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[4096, 2, 0, 0, 449, 0, 4096, 207, 1, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 4096, 1, 4096, 4087, 0, 0, 4068, 26, 0, 0, 0, 0, 0, 4096, 0, 2610, 0, 3923, 0, 4096, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 4096, 34, 0, 0, 22, 4096, 6, 5, 0, 884, 4077, 0, 0, 0, 0, 92, 0, 0, 4080, 0, 0, 4088, 4095, 2, 0, 14, 4096, 4096, 1, 15, 0, 0, 4011, 0, 21, 0, 130, 4094, 0, 4091, 0, 4063, 4038, 4095, 0, 4095, 1, 0, 4096, 0, 4093, 1, 0, 0, 0, 0, 42, 4094, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4049, 0, 0, 1, 4096, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[4072, 4, 0, 0, 573, 0, 4088, 3883, 0, 0, 393, 0, 16, 0, 0, 0, 0, 1, 0, 15, 0, 1, 3, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4095, 394, 0, 0, 0, 0, 0, 4096, 0, 1978, 0, 4094, 1, 4077, 0, 1, 4096, 0, 0, 13, 0, 0, 0, 0, 4095, 10, 0, 0, 124, 4096, 32, 0, 0, 83, 3368, 0, 1, 0, 0, 224, 0, 1, 3945, 0, 0, 3982, 4084, 0, 0, 9, 4075, 4096, 0, 0, 0, 0, 3998, 0, 0, 0, 1110, 4063, 0, 4096, 0, 3267, 2547, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 12, 0, 0, 0, 0, 4, 4093, 0, 0, 0, 1, 0, 0, 0, 153, 0, 2935, 0, 0, 0, 4096, 3907] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 1767, 0, 4092, 38, 4, 0, 4, 0, 2, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 4094, 4095, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 4096, 0, 1417, 2, 4037, 4, 4094, 0, 0, 4089, 0, 0, 0, 1, 0, 1, 0, 4094, 0, 0, 0, 0, 4096, 2, 0, 0, 649, 4063, 0, 0, 0, 0, 1698, 0, 0, 3675, 0, 0, 3861, 4092, 0, 0, 3, 4096, 4096, 0, 4, 5, 0, 3881, 0, 0, 0, 157, 4096, 0, 4062, 0, 3967, 3622, 4095, 0, 4096, 0, 0, 4096, 2, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 0, 0, 2, 4094, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[4057, 42, 4, 0, 334, 0, 4069, 213, 10, 0, 93, 0, 12, 0, 0, 1, 0, 8, 0, 110, 0, 0, 0, 0, 0, 4079, 0, 4084, 4087, 0, 0, 3957, 0, 0, 8, 0, 7, 0, 4093, 35, 2230, 0, 3574, 0, 4096, 0, 0, 4047, 0, 0, 0, 0, 0, 5, 0, 4081, 0, 0, 0, 141, 4081, 0, 0, 22, 2619, 4096, 15, 15, 3, 9, 38, 0, 1, 3997, 0, 5, 4091, 4096, 0, 0, 6, 4095, 4087, 1, 1, 0, 0, 3665, 0, 1, 0, 69, 4095, 2, 4017, 0, 4081, 4086, 4083, 0, 4084, 15, 0, 4093, 12, 4096, 12, 14, 0, 0, 0, 0, 4091, 0, 4, 0, 0, 6, 0, 0, 0, 0, 3659, 0, 0, 0, 4096, 4036] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 16, 0, 4096, 1458, 0, 4, 54, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 4096, 0, 4076, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 1044, 4, 4084, 0, 4091, 0, 23, 4072, 1, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 7, 4096, 22, 8, 0, 3352, 4096, 0, 0, 0, 0, 157, 0, 0, 2910, 0, 0, 4017, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4000, 0, 1, 0, 474, 4095, 0, 4088, 0, 4077, 4047, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 38, 0, 7, 0, 0, 0, 2987, 0, 0, 1, 4096, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 169, 0, 4096, 736, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4053, 7, 0, 0, 0, 0, 0, 4096, 3, 753, 0, 4092, 0, 4088, 0, 4, 4096, 0, 0, 0, 0, 26, 0, 0, 4096, 1, 0, 0, 67, 4096, 13, 0, 0, 3236, 4096, 0, 0, 0, 0, 164, 0, 0, 3251, 0, 0, 4071, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4067, 0, 0, 0, 41, 4096, 0, 4080, 0, 4058, 4080, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 0, 0, 0, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4092, 1, 0, 0, 423, 2, 4093, 2039, 0, 0, 1, 0, 0, 0, 0, 1, 3650, 1563, 1024, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 1, 4092, 0, 4093, 4093, 2, 0, 4095, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2532, 2, 2017, 0, 4088, 1, 4095, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 13, 0, 0, 4096, 0, 0, 0, 0, 4094, 4, 3, 0, 4094, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 419, 4093, 3, 2, 0, 1, 357, 0, 0, 3096, 0, 0, 4047, 2827, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 0, 1, 0, 0, 0, 4095, 1, 0, 1, 330, 4078, 0, 4064, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3827, 3962, 4095, 0, 4095, 0, 0, 0, 0, 443, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 5, 0, 0, 0, 0, 1, 4085, 0, 0, 8, 4094, 4095, 244, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16957, 16394, 16833, 16335, 14847, 16518, 16424, 16764] avg=16384.0000 max=16957 max_rank=0 min=14847 min_rank=4 imbalance_ratio=1.034973 max_load_reduction=114115 max_load_reduction_pct=0.870628 moved_to_echo=64553 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 102, 0, 4096, 2099, 0, 0, 0, 0, 1, 0, 0, 0, 3657, 1568, 1028, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4083, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2528, 0, 949, 0, 4010, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 4, 0, 0, 4096, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 4096, 0, 0, 0, 0, 646, 0, 0, 4069, 0, 0, 4095, 2822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 29, 4096, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 4096, 0, 4096, 0, 0, 0, 0, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 0, 4096, 4096, 246, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16716, 16386, 15679, 16320, 16262, 16227, 16817, 16665] avg=16384.0000 max=16817 max_rank=6 min=15679 min_rank=2 imbalance_ratio=1.026428 max_load_reduction=114255 max_load_reduction_pct=0.871696 moved_to_echo=65971 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 2, 0, 0, 449, 0, 4096, 207, 1, 3, 0, 0, 0, 0, 1, 0, 3650, 1563, 1024, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 4096, 1, 4096, 4087, 0, 0, 4068, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 2533, 0, 2610, 0, 3923, 0, 4096, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 34, 0, 0, 22, 4096, 6, 5, 0, 4096, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 884, 4077, 0, 0, 0, 0, 92, 0, 0, 4080, 0, 0, 4088, 2827, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3948, 0, 1, 15, 0, 0, 4011, 0, 21, 0, 130, 4094, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4038, 4095, 0, 4095, 1, 0, 0, 0, 443, 1, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4049, 0, 0, 1, 4096, 4087, 244, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15160, 16408, 17255, 16383, 16050, 16325, 16778, 16713] avg=16384.0000 max=17255 max_rank=2 min=15160 min_rank=0 imbalance_ratio=1.053162 max_load_reduction=113817 max_load_reduction_pct=0.868355 moved_to_echo=65866 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4057, 42, 4, 0, 334, 0, 4069, 213, 10, 0, 93, 0, 12, 0, 0, 1, 3653, 1562, 1025, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 110, 0, 0, 0, 0, 0, 4079, 0, 4084, 4087, 0, 0, 3957, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 7, 0, 2531, 35, 2230, 0, 3574, 0, 4096, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4081, 0, 0, 0, 141, 4081, 0, 0, 22, 4093, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2619, 4096, 15, 15, 3, 9, 38, 0, 1, 3997, 0, 5, 4091, 2827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3947, 0, 1, 1, 0, 0, 3665, 0, 1, 0, 69, 4095, 2, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4086, 4083, 0, 4084, 15, 0, 0, 12, 443, 12, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4, 0, 0, 6, 0, 0, 0, 0, 3659, 0, 0, 0, 4096, 4036, 244, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15143, 16338, 16528, 16442, 17716, 15804, 16830, 16271] avg=16384.0000 max=17716 max_rank=4 min=15143 min_rank=0 imbalance_ratio=1.081299 max_load_reduction=113356 max_load_reduction_pct=0.864838 moved_to_echo=66621 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 16, 0, 4096, 1458, 0, 4, 54, 0, 0, 0, 0, 0, 3653, 1563, 1025, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 4096, 0, 4076, 4092, 0, 0, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2533, 0, 1044, 4, 4084, 0, 4091, 0, 23, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 7, 4096, 22, 8, 0, 4096, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3352, 4096, 0, 0, 0, 0, 157, 0, 0, 2910, 0, 0, 4017, 2827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 0, 0, 0, 0, 0, 4000, 0, 1, 0, 474, 4095, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4047, 4096, 0, 4096, 0, 0, 0, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 38, 0, 7, 0, 0, 0, 2987, 0, 0, 1, 4096, 4095, 244, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16032, 16408, 15851, 16358, 17359, 16606, 16759, 15699] avg=16384.0000 max=17359 max_rank=4 min=15699 min_rank=7 imbalance_ratio=1.059509 max_load_reduction=113713 max_load_reduction_pct=0.867561 moved_to_echo=66423 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 1767, 0, 4092, 38, 4, 0, 4, 0, 2, 0, 0, 0, 3653, 1563, 1024, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 4094, 4095, 0, 0, 4064, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2533, 0, 1417, 2, 4037, 4, 4094, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4094, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 4063, 0, 0, 0, 0, 1698, 0, 0, 3675, 0, 0, 3861, 2824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3948, 0, 0, 4, 5, 0, 3881, 0, 0, 0, 157, 4096, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3967, 3622, 4095, 0, 4096, 0, 0, 0, 2, 443, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 0, 0, 2, 4094, 4084, 244, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16311, 16370, 16176, 16318, 16770, 16156, 16226, 16745] avg=16384.0000 max=16770 max_rank=4 min=16156 min_rank=5 imbalance_ratio=1.023560 max_load_reduction=114302 max_load_reduction_pct=0.872055 moved_to_echo=65897 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4072, 4, 0, 0, 573, 0, 4088, 3883, 0, 0, 393, 0, 16, 0, 0, 0, 3653, 1563, 1022, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 15, 0, 1, 3, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4095, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 394, 0, 0, 0, 0, 0, 2533, 0, 1978, 0, 4094, 1, 4077, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4095, 10, 0, 0, 124, 4096, 32, 0, 0, 4096, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 3368, 0, 1, 0, 0, 224, 0, 1, 3945, 0, 0, 3982, 2819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3928, 0, 0, 0, 0, 0, 3998, 0, 0, 0, 1110, 4063, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3267, 2547, 4096, 0, 4096, 0, 0, 0, 0, 443, 12, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 0, 153, 0, 2935, 0, 0, 0, 4096, 3907, 243, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19336, 16415, 17174, 16493, 14423, 17204, 14465, 15562] avg=16384.0000 max=19336 max_rank=0 min=14423 min_rank=4 imbalance_ratio=1.180176 max_load_reduction=111736 max_load_reduction_pct=0.852478 moved_to_echo=61654 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 169, 0, 4096, 736, 0, 0, 11, 0, 0, 0, 0, 0, 3653, 1563, 1025, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4053, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 2533, 3, 753, 0, 4092, 0, 4088, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 4096, 1, 0, 0, 67, 4096, 13, 0, 0, 4096, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3236, 4096, 0, 0, 0, 0, 164, 0, 0, 3251, 0, 0, 4071, 2827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 41, 4096, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 4080, 4096, 0, 4096, 0, 0, 0, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 0, 0, 0, 4096, 4096, 244, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15417, 16353, 15576, 16423, 17645, 16232, 16773, 16653] avg=16384.0000 max=17645 max_rank=4 min=15417 min_rank=0 imbalance_ratio=1.076965 max_load_reduction=113427 max_load_reduction_pct=0.865379 moved_to_echo=67303 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 128, 128, 128, 3840, 128, 32768, 10752, 128, 128, 640, 0, 128, 0, 128, 128, 29312, 12544, 8320, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 256, 0, 128, 128, 0, 128, 32768, 128, 32768, 32768, 128, 0, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[512, 128, 128, 0, 128, 0, 20352, 128, 13056, 128, 32000, 128, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 31616, 0, 128, 128, 128, 0, 31872, 128, 128, 128, 2432, 32768, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 128, 128, 256, 128, 29824, 0, 0, 128, 32768, 32512, 2048, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[31488, 30592, 32768, 0, 32768, 128, 0, 0, 128, 3584, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[11776, 32000, 128, 128, 128, 128, 3456, 0, 128, 29056, 0, 128, 32256, 22656, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 128, 128, 0, 32768, 128, 0, 0, 384, 32768, 128, 128, 128, 32768, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4088, 4092, 0, 0, 0, 4096, 3, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 5, 4096, 0, 4096, 680, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 1, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 1104, 0, 3943, 4096, 4094, 4096, 0, 1, 6, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 4096, 6, 0, 3860, 0, 0, 0, 0, 4091, 4092, 0, 0, 0, 4058, 0, 0, 60, 0, 4073, 3915, 0, 0, 0, 0, 5, 12, 0, 2665, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 35, 0, 4096, 186, 0, 0, 0, 0, 4096, 0, 0, 4079, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4077, 1, 4084, 4096, 0, 0, 4, 4059, 11, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 9, 4094, 0, 4085, 1391, 1, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 8, 0, 4094, 1, 4096, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 12, 165, 0, 3515, 4096, 4086, 4096, 0, 2, 4, 9, 1, 0, 0, 0, 4089, 1, 0, 14, 4082, 4, 0, 3589, 26, 6, 0, 0, 4081, 4090, 3, 0, 0, 4065, 0, 0, 16, 0, 4096, 4095, 0, 0, 4, 0, 0, 53, 24, 3472, 0, 0, 4082, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 38, 0, 4096, 0, 0, 0, 12, 0, 4096, 56, 0, 4086, 0, 4096, 19, 0, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4095, 1, 4093, 4092, 0, 0, 1, 4095, 0, 0, 1, 2, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 4095, 3283, 1, 4096, 0, 0, 0, 4095, 1, 4095, 0, 0, 0, 1, 0, 0, 0, 4095, 1, 4095, 0, 4093, 1, 1, 2, 0, 1, 1, 0, 0, 229, 0, 1172, 4096, 4095, 4095, 1, 5, 1, 0, 0, 0, 3, 0, 4095, 0, 1, 0, 4094, 0, 0, 4022, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 4095, 1, 0, 13, 0, 4085, 4052, 0, 0, 1, 0, 0, 5, 0, 3556, 1, 0, 4094, 0, 0, 1, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 62, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4094, 0, 4096, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 3153, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 889, 0, 3317, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3290, 0, 0, 0, 0, 4096, 4096, 0, 2, 0, 4096, 0, 0, 3, 0, 4093, 4003, 0, 0, 0, 0, 0, 0, 0, 1718, 0, 0, 4096, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4094, 4049, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 4094, 0, 4095, 592, 0, 4095, 3, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 4095, 0, 0, 5, 0, 0, 0, 0, 0, 1117, 0, 2820, 4095, 4094, 4095, 0, 2, 0, 1, 1, 0, 0, 1, 4096, 0, 1, 1, 4096, 0, 0, 3925, 2, 0, 0, 0, 4094, 4094, 2, 0, 0, 4083, 0, 2, 0, 0, 4092, 4093, 0, 0, 2, 0, 2, 0, 0, 3651, 0, 2, 4094, 0, 0, 0, 4094, 24, 0, 0, 0, 0, 0, 13, 0, 4095, 202, 0, 0, 0, 1, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4078, 3975, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 19, 4096, 0, 4096, 3873, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 100, 22, 0, 151, 4096, 4095, 4096, 0, 7, 0, 1, 4, 0, 0, 7, 4096, 0, 0, 1, 4096, 0, 0, 3967, 0, 0, 0, 0, 4080, 4031, 0, 0, 0, 4073, 0, 0, 120, 0, 4094, 4087, 0, 0, 19, 0, 0, 8, 0, 1251, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 192, 0, 4096, 2816, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 2, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 3811, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 4096, 0, 4096, 2082, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 0, 4093, 0, 0, 1, 0, 0, 0, 0, 4, 300, 0, 3308, 4096, 4093, 4096, 0, 55, 0, 0, 0, 0, 0, 2, 4096, 46, 0, 0, 4096, 0, 0, 3966, 0, 0, 0, 0, 4088, 4095, 0, 0, 0, 4096, 0, 0, 47, 0, 4096, 3652, 0, 0, 0, 0, 0, 0, 0, 2635, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 48, 0, 4096, 554, 0, 0, 0, 0, 4096, 0, 0, 4076, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4033, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 3638, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 1087, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 4, 3786, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 10, 0, 4063, 4000, 0, 0, 0, 0, 0, 5, 0, 3670, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 1, 0, 0, 50, 0, 4096, 95, 0, 0, 0, 0, 4096, 0, 0, 4073, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4077, 1, 4084, 4096, 0, 0, 4, 4059, 11, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4094, 0, 4085, 1391, 1, 4096, 0, 0, 0, 0, 0, 1757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4094, 1, 4096, 0, 4096, 0, 0, 0, 0, 3, 2417, 1674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 165, 0, 3515, 4096, 4086, 4096, 0, 2, 4, 9, 1, 0, 0, 665, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 1, 0, 14, 4082, 4, 0, 3589, 26, 6, 0, 0, 4081, 4090, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 16, 0, 4096, 4095, 0, 0, 4, 0, 0, 53, 24, 2793, 1375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3472, 0, 0, 4082, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 0, 38, 0, 4096, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 12, 0, 3603, 56, 0, 4086, 0, 4096, 19, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 15433, 16389, 17144, 16193, 16521, 17009, 15971] avg=16384.0000 max=17144 max_rank=3 min=15433 min_rank=1 imbalance_ratio=1.046387 max_load_reduction=113928 max_load_reduction_pct=0.869202 moved_to_echo=65694 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4095, 1, 4093, 4092, 0, 0, 1, 4095, 0, 0, 1, 2, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 3283, 1, 4096, 0, 0, 0, 0, 1, 1756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 1, 4095, 0, 4093, 1, 1, 2, 0, 1, 2420, 1674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 229, 0, 1172, 4096, 4095, 4095, 1, 5, 1, 0, 0, 0, 3, 665, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 1, 0, 4094, 0, 0, 4022, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 13, 0, 4085, 4052, 0, 0, 1, 0, 0, 5, 0, 2798, 1377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3556, 1, 0, 4094, 0, 0, 1, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 1217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 62, 1, 0, 0, 0, 3603, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16460, 17328, 16384, 14856, 16604, 16427, 17061, 15952] avg=16384.0000 max=17328 max_rank=1 min=14856 min_rank=3 imbalance_ratio=1.057617 max_load_reduction=113744 max_load_reduction_pct=0.867798 moved_to_echo=66044 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4088, 4092, 0, 0, 0, 4096, 3, 0, 0, 1, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 4096, 680, 0, 4095, 0, 0, 0, 0, 0, 1757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 2421, 1674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1104, 0, 3943, 4096, 4094, 4096, 0, 1, 6, 0, 0, 0, 0, 665, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 297, 0, 0, 0, 4096, 6, 0, 3860, 0, 0, 0, 0, 4091, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 60, 0, 4073, 3915, 0, 0, 0, 0, 5, 12, 0, 2797, 1378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2665, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 35, 0, 4096, 1217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 186, 0, 0, 0, 0, 3603, 0, 0, 4079, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16455, 14729, 16380, 18498, 16445, 16298, 16207, 16060] avg=16384.0000 max=18498 max_rank=3 min=14729 min_rank=1 imbalance_ratio=1.129028 max_load_reduction=112574 max_load_reduction_pct=0.858871 moved_to_echo=65010 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 4096, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3153, 0, 4096, 0, 0, 0, 0, 0, 1749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2423, 1680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 889, 0, 3317, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 667, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 4096, 0, 0, 3290, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 3, 0, 4093, 4003, 0, 0, 0, 0, 0, 0, 0, 2803, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1718, 0, 0, 4096, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 4096, 1222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 3602, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16446, 17190, 16391, 17655, 15871, 16380, 15239, 15900] avg=16384.0000 max=17655 max_rank=3 min=15239 min_rank=6 imbalance_ratio=1.077576 max_load_reduction=113417 max_load_reduction_pct=0.865303 moved_to_echo=63390 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 3811, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2082, 0, 4096, 0, 0, 0, 0, 0, 1757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 0, 4093, 0, 0, 1, 0, 0, 2421, 1674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 300, 0, 3308, 4096, 4093, 4096, 0, 55, 0, 0, 0, 0, 0, 665, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 297, 46, 0, 0, 4096, 0, 0, 3966, 0, 0, 0, 0, 4088, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 47, 0, 4096, 3652, 0, 0, 0, 0, 0, 0, 0, 2798, 1378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2635, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 48, 0, 4096, 1217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 554, 0, 0, 0, 0, 3603, 0, 0, 4076, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16180, 16127, 16384, 17110, 16590, 16067, 16189, 16425] avg=16384.0000 max=17110 max_rank=3 min=16067 min_rank=5 imbalance_ratio=1.044312 max_load_reduction=113962 max_load_reduction_pct=0.869461 moved_to_echo=65271 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4094, 4049, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 4095, 592, 0, 4095, 3, 0, 0, 0, 0, 1757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 4095, 0, 0, 5, 0, 0, 2421, 1674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1117, 0, 2820, 4095, 4094, 4095, 0, 2, 0, 1, 1, 0, 0, 665, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 297, 0, 1, 1, 4096, 0, 0, 3925, 2, 0, 0, 0, 4094, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 2, 0, 0, 4092, 4093, 0, 0, 2, 0, 2, 0, 0, 2798, 1378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3651, 0, 2, 4094, 0, 0, 0, 4094, 24, 0, 0, 0, 0, 0, 13, 0, 4096, 1217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 202, 0, 0, 0, 1, 3602, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16416, 14638, 16389, 17383, 16513, 16450, 17191, 16092] avg=16384.0000 max=17383 max_rank=3 min=14638 min_rank=1 imbalance_ratio=1.060974 max_load_reduction=113689 max_load_reduction_pct=0.867378 moved_to_echo=66246 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4078, 3975, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 19, 4096, 0, 4096, 3873, 0, 4095, 0, 0, 0, 0, 0, 1757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 2421, 1674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 22, 0, 151, 4096, 4095, 4096, 0, 7, 0, 1, 4, 0, 0, 665, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 297, 0, 0, 1, 4096, 0, 0, 3967, 0, 0, 0, 0, 4080, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 120, 0, 4094, 4087, 0, 0, 19, 0, 0, 8, 0, 2799, 1378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1251, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 192, 0, 4096, 1217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2816, 0, 0, 0, 0, 3603, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16301, 17944, 16383, 13730, 16479, 16578, 14948, 18709] avg=16384.0000 max=18709 max_rank=7 min=13730 min_rank=3 imbalance_ratio=1.141907 max_load_reduction=112363 max_load_reduction_pct=0.857262 moved_to_echo=66714 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 4033, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3638, 0, 4096, 0, 0, 0, 0, 0, 1757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4085, 0, 0, 0, 0, 0, 2421, 1674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 1087, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 665, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 297, 0, 0, 0, 4096, 0, 4, 3786, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 4063, 4000, 0, 0, 0, 0, 0, 5, 0, 2799, 1378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3670, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 1, 0, 0, 50, 0, 4096, 1217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 95, 0, 0, 0, 0, 3603, 0, 0, 4073, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16402, 17683, 16372, 14696, 16377, 16351, 17228, 15963] avg=16384.0000 max=17683 max_rank=1 min=14696 min_rank=3 imbalance_ratio=1.079285 max_load_reduction=113389 max_load_reduction_pct=0.865089 moved_to_echo=65919 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[22656, 128, 128, 32768, 0, 0, 128, 32768, 128, 0, 0, 128, 128, 0, 384, 0, 32768, 9856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 128, 32768, 0, 32768, 18816, 128, 32768, 128, 0, 0, 0, 128, 14080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 32768, 32256, 0, 0, 128, 32768, 128, 0, 128, 128, 128, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 128, 0, 32768, 128, 32768, 0, 32768, 128, 128, 128, 0, 128, 19456, 13440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 3968, 128, 0, 128, 128, 28928, 128, 0, 32768, 0, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 32768, 128, 128, 384, 0, 32768, 32000, 0, 0, 128, 0, 128, 128, 128, 22400, 11136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 2432, 128, 128, 128, 32768, 128, 128, 30464, 128, 128, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 4096, 0, 19328, 32768, 32768, 32768, 128, 128, 128, 128, 128, 0, 128, 5376, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 4094, 0, 1, 1, 0, 4092, 0, 4095, 1, 1, 4091, 1, 65, 1247, 4083, 1, 4096, 4095, 4095, 0, 4095, 4093, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4091, 4091, 1, 1, 0, 5, 0, 1, 0, 4075, 2, 0, 1, 3, 1, 4090, 0, 1, 27, 0, 0, 0, 4069, 1, 0, 3677, 0, 4, 1, 0, 108, 5, 1, 1, 0, 0, 2, 4092, 0, 0, 3, 0, 2, 0, 471, 11, 0, 1, 4095, 1, 0, 4095, 0, 0, 1, 1, 4089, 0, 0, 0, 0, 4093, 4095, 1, 4095, 0, 0, 5, 1, 1, 0, 4095, 2811, 1, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 2, 3931, 4095, 1, 4095, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4092, 0, 0, 0, 0, 4094, 0, 4096, 0, 17, 4086, 0, 47, 3360, 4096, 0, 4086, 4096, 4096, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 10, 0, 0, 0, 0, 3982, 0, 0, 0, 1, 1, 4096, 0, 0, 2, 0, 0, 0, 4093, 0, 0, 3480, 0, 0, 0, 1, 894, 0, 175, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1476, 27, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 9, 1, 0, 0, 0, 4092, 12, 0, 4096, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 3002, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4094, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4091, 0, 143, 3078, 4087, 0, 4096, 4096, 4092, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 13, 4094, 0, 0, 8, 0, 0, 0, 3927, 0, 0, 1487, 0, 4, 0, 0, 0, 3, 80, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 3454, 0, 0, 0, 4096, 0, 3, 4096, 2, 2, 0, 0, 4093, 0, 1, 0, 0, 4096, 4096, 0, 4096, 2, 4, 1, 0, 0, 0, 4096, 541, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3661, 4096, 0, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 1894, 4096, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 5, 4096, 0, 0, 103, 0, 0, 0, 4078, 0, 0, 3704, 0, 8, 0, 0, 5, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3335, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 478, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2781, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 5, 4051, 0, 31, 1068, 4089, 0, 4096, 4096, 4095, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 3, 0, 0, 0, 4085, 0, 0, 0, 0, 7, 4096, 0, 0, 12, 0, 0, 0, 4040, 0, 0, 3147, 0, 65, 0, 0, 170, 0, 497, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3122, 39, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 2, 0, 0, 4072, 318, 0, 4096, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 3995, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[14, 4096, 4089, 0, 0, 0, 0, 4084, 0, 4094, 0, 0, 4096, 0, 239, 3080, 4082, 0, 4096, 4096, 4092, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 4087, 93, 0, 0, 1, 0, 0, 4096, 4009, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 0, 0, 13, 48, 4075, 17, 0, 86, 0, 0, 0, 3883, 0, 0, 4092, 0, 340, 0, 0, 1807, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 230, 0, 0, 0, 4096, 0, 8, 4086, 0, 0, 0, 0, 4077, 0, 0, 1, 0, 4096, 4096, 0, 4083, 0, 13, 9, 0, 0, 0, 4073, 982, 0, 4096, 0, 0, 7, 0, 0, 0, 4086, 0, 0, 0, 12, 1635, 4087, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4064, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 4095, 0, 253, 4053, 4096, 27, 4096, 4096, 4096, 0, 4096, 3987, 0, 0, 0, 0, 0, 0, 4082, 12, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4, 27, 4088, 0, 0, 0, 0, 4, 4096, 0, 0, 3, 0, 0, 0, 4025, 0, 0, 688, 0, 888, 0, 81, 223, 1, 503, 0, 0, 0, 1, 4094, 2, 0, 0, 0, 0, 0, 2813, 52, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 4033, 0, 9, 0, 0, 4096, 4096, 0, 4096, 0, 0, 12, 0, 0, 0, 3947, 137, 0, 4095, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 3086, 4096, 0, 4058, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4094, 0, 4096, 0, 1, 4093, 0, 4, 2113, 4096, 0, 4091, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 2132, 0, 77, 0, 0, 955, 0, 70, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3814, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 225, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2909, 4096, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 1, 1, 0, 4092, 0, 1524, 1, 1, 4091, 1, 65, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 1, 4096, 0, 10, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4091, 1, 1, 0, 5, 0, 1, 0, 4075, 2, 3634, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 1, 4090, 0, 1, 27, 0, 0, 0, 4069, 1, 0, 3677, 0, 4095, 871, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 108, 5, 1, 1, 0, 0, 2, 4092, 0, 0, 3, 0, 2, 4095, 4095, 3224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 11, 0, 1, 4095, 1, 0, 4095, 0, 0, 1, 1, 4089, 0, 0, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 4095, 0, 0, 5, 1, 1, 0, 4095, 2811, 1, 3429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 2, 3931, 4095, 1, 4095, 1, 666, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15118, 16378, 16386, 17287, 15633, 14526, 18532, 17212] avg=16384.0000 max=18532 max_rank=6 min=14526 min_rank=5 imbalance_ratio=1.131104 max_load_reduction=112540 max_load_reduction_pct=0.858612 moved_to_echo=65903 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4092, 0, 0, 0, 0, 4094, 0, 1524, 0, 17, 4086, 0, 47, 3360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 10, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 10, 0, 0, 0, 0, 3982, 0, 3635, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4096, 0, 0, 2, 0, 0, 0, 4093, 0, 0, 3480, 0, 4096, 872, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 894, 0, 175, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1476, 27, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4042, 0, 0, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 9, 1, 0, 0, 0, 4092, 12, 0, 3430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 3002, 4096, 0, 4096, 0, 666, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17220, 16374, 16304, 17092, 16582, 15498, 15736, 16266] avg=16384.0000 max=17220 max_rank=0 min=15498 min_rank=5 imbalance_ratio=1.051025 max_load_reduction=113852 max_load_reduction_pct=0.868622 moved_to_echo=64082 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1515, 0, 0, 4096, 0, 0, 1894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3637, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 103, 0, 0, 0, 4078, 0, 0, 3704, 0, 4096, 872, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 5, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3335, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 478, 0, 3423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2781, 4096, 0, 4096, 0, 673, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15697, 16388, 16409, 17409, 15526, 17389, 16184, 16070] avg=16384.0000 max=17409 max_rank=3 min=15526 min_rank=4 imbalance_ratio=1.062561 max_load_reduction=113663 max_load_reduction_pct=0.867180 moved_to_echo=65169 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1524, 0, 5, 4051, 0, 31, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 0, 10, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 3, 0, 0, 0, 4085, 0, 3634, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 12, 0, 0, 0, 4040, 0, 0, 3147, 0, 4096, 872, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 170, 0, 497, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3122, 39, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4063, 0, 0, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 4072, 318, 0, 3430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 3995, 4096, 0, 4096, 0, 666, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14871, 16386, 16397, 16721, 16244, 17177, 16014, 17262] avg=16384.0000 max=17262 max_rank=7 min=14871 min_rank=0 imbalance_ratio=1.053589 max_load_reduction=113810 max_load_reduction_pct=0.868301 moved_to_echo=66697 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1524, 0, 0, 4091, 0, 143, 3078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 0, 11, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3631, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4094, 0, 0, 8, 0, 0, 0, 3927, 0, 0, 1487, 0, 4094, 871, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3, 80, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 4096, 4096, 3225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3454, 0, 0, 0, 4096, 0, 3, 4096, 2, 2, 0, 0, 4093, 0, 1, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 4, 1, 0, 0, 0, 4096, 541, 0, 3429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3661, 4096, 0, 4093, 0, 666, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17028, 16388, 16402, 14944, 15599, 17508, 16265, 16938] avg=16384.0000 max=17508 max_rank=5 min=14944 min_rank=3 imbalance_ratio=1.068604 max_load_reduction=113564 max_load_reduction_pct=0.866425 moved_to_echo=66310 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 4089, 0, 0, 0, 0, 4084, 0, 1525, 0, 0, 4096, 0, 239, 3080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4096, 0, 11, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4087, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4009, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 3631, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 48, 4075, 17, 0, 86, 0, 0, 0, 3883, 0, 0, 4092, 0, 4096, 872, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 340, 0, 0, 1807, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 4096, 4096, 3224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 0, 4096, 0, 8, 4086, 0, 0, 0, 0, 4077, 0, 0, 1760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4083, 0, 13, 9, 0, 0, 0, 4073, 982, 0, 3430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4086, 0, 0, 0, 12, 1635, 4087, 0, 4095, 0, 666, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17127, 16456, 16335, 17632, 17665, 14257, 16687, 14913] avg=16384.0000 max=17665 max_rank=4 min=14257 min_rank=5 imbalance_ratio=1.078186 max_load_reduction=113407 max_load_reduction_pct=0.865227 moved_to_echo=63522 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4064, 0, 0, 0, 0, 4096, 0, 1524, 0, 1, 4095, 0, 253, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 27, 4096, 0, 10, 0, 0, 3987, 0, 0, 0, 0, 0, 0, 4082, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4, 27, 4088, 0, 3635, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 3, 0, 0, 0, 4025, 0, 0, 688, 0, 4096, 872, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 888, 0, 81, 223, 1, 503, 0, 0, 0, 1, 4094, 2, 0, 0, 0, 0, 4096, 4096, 3224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2813, 52, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 4033, 0, 9, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 3947, 137, 0, 3429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 3086, 4096, 0, 4058, 0, 666, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18086, 16310, 16431, 14235, 17209, 16861, 15717, 16223] avg=16384.0000 max=18086 max_rank=0 min=14235 min_rank=3 imbalance_ratio=1.103882 max_load_reduction=112986 max_load_reduction_pct=0.862015 moved_to_echo=66010 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 4094, 0, 1524, 0, 1, 4093, 0, 4, 2113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 10, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3635, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 2132, 0, 4096, 872, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 955, 0, 70, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3814, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 225, 0, 3430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2909, 4096, 0, 4095, 0, 666, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15925, 16392, 16408, 15752, 16614, 17856, 15937, 16188] avg=16384.0000 max=17856 max_rank=5 min=15752 min_rank=3 imbalance_ratio=1.089844 max_load_reduction=113216 max_load_reduction_pct=0.863770 moved_to_echo=66595 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 32768, 0, 128, 32768, 128, 128, 128, 128, 128, 0, 32640, 5504, 128, 27520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 32768, 0, 128, 128, 0, 32768, 0, 12288, 128, 128, 32768, 128, 896, 19968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 128, 256, 0, 0, 0, 32256, 128, 0, 22528, 0, 32768, 7040, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 18816, 256, 0, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 32640, 0, 128, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[32768, 128, 32768, 0, 128, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1408, 128, 128, 4224, 128, 1408, 128, 0, 0, 128, 32768, 128, 0, 128, 0, 128, 32768, 32768, 25856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 0, 0, 32768, 32768, 128, 128, 128, 128, 0, 128, 128, 32640, 128, 29184, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 0, 0, 32640, 0, 128, 0, 128, 25088, 32768, 128, 32768, 128, 5376, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 4096, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 1, 0, 4096, 0, 0, 4084, 0, 1, 0, 0, 0, 0, 0, 2, 4096, 0, 4096, 0, 2806, 4096, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 745, 0, 202, 0, 0, 3893, 0, 0, 4096, 4096, 4096, 0, 0, 5, 0, 2441, 4096, 0, 0, 0, 1, 0, 0, 0, 4091, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4094, 4085, 0, 0, 0, 0, 0, 4074, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 2238, 2, 4088] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4095, 1, 0, 0, 1, 0, 0, 1, 0, 4095, 4096, 4095, 4095, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 2, 0, 0, 1, 1, 4093, 0, 1, 4095, 0, 10, 0, 2, 0, 4094, 1, 1, 4094, 0, 1, 2, 1, 0, 1, 0, 0, 4095, 0, 4096, 0, 2609, 4095, 0, 0, 1, 4091, 0, 1, 0, 0, 1, 0, 4095, 0, 1217, 11, 19, 0, 0, 3956, 0, 1, 4094, 4092, 4094, 0, 0, 0, 3, 1288, 4095, 9, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 4093, 4079, 0, 0, 2, 0, 1, 4095, 4095, 0, 0, 0, 1, 4094, 4095, 4095, 0, 4096, 0, 0, 0, 3193, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 5, 0, 0, 0, 4095, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 4096, 0, 2323, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 12, 1, 35, 0, 0, 4037, 0, 0, 4096, 4088, 4096, 0, 0, 2, 0, 2387, 4096, 0, 0, 0, 3, 0, 1, 0, 4094, 0, 0, 1, 0, 0, 0, 0, 2, 4093, 4095, 4096, 0, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 0, 4096, 4096, 4095, 0, 4096, 0, 0, 0, 3504, 2, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2288, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 582, 1, 16, 0, 0, 4096, 0, 0, 4096, 4091, 4096, 0, 0, 0, 0, 1985, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 3327, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4086, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 4096, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 20, 0, 0, 0, 4079, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 7, 4096, 0, 4096, 0, 468, 4085, 58, 0, 0, 4089, 0, 0, 0, 0, 0, 61, 4095, 27, 164, 111, 414, 0, 0, 3719, 0, 0, 4091, 4070, 4096, 1, 0, 0, 0, 3460, 4096, 0, 1, 0, 4, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4077, 4094, 0, 1, 0, 0, 0, 3804, 4095, 50, 0, 0, 0, 4091, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 16, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 4079, 4096, 4096, 4096, 2, 15, 0, 0, 10, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 9, 4096, 0, 32, 0, 0, 0, 4042, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 16, 0, 4095, 0, 4096, 0, 386, 4096, 0, 2, 0, 4083, 3, 0, 2, 0, 0, 4, 4095, 0, 2301, 350, 0, 0, 0, 3920, 0, 0, 4081, 4096, 4075, 10, 0, 0, 13, 2294, 4096, 17, 4, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4054, 4086, 0, 0, 0, 0, 0, 3999, 4096, 1, 0, 0, 0, 4096, 4096, 4086, 0, 4096, 0, 0, 0, 3146, 31, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 12, 0, 5, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2531, 4094, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 873, 6, 349, 0, 0, 3879, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 615, 4096, 4, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 4094, 0, 0, 0, 0, 0, 4063, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 4063, 9, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1470, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 8, 4096, 0, 1594, 0, 615, 0, 0, 4058, 0, 0, 4096, 4084, 4096, 0, 0, 16, 0, 1582, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4076, 0, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 2991, 11, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 1, 0, 0, 1, 0, 0, 1, 0, 4095, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 2, 0, 0, 1, 1, 4093, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 10, 0, 2, 0, 4094, 1, 1, 4094, 0, 1, 2, 1, 3330, 762, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 2231, 0, 2609, 4095, 0, 0, 1, 4091, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3042, 0, 1217, 11, 19, 0, 0, 3956, 0, 1, 4094, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 3, 1288, 4095, 9, 0, 0, 0, 0, 0, 1, 4095, 0, 1103, 980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 4093, 4079, 0, 0, 2, 0, 1, 4095, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 765, 0, 0, 0, 1, 4094, 0, 4095, 0, 0, 0, 0, 0, 3193, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16383, 16398, 17124, 16433, 15668, 16440, 16243] avg=16384.0000 max=17124 max_rank=3 min=15668 min_rank=5 imbalance_ratio=1.045166 max_load_reduction=113948 max_load_reduction_pct=0.869354 moved_to_echo=64784 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 1, 0, 4096, 0, 0, 4084, 0, 1, 0, 0, 3331, 762, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 2231, 0, 2806, 4096, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3042, 0, 745, 0, 202, 0, 0, 3893, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 2441, 4096, 0, 0, 0, 1, 0, 0, 0, 4091, 0, 1103, 981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4094, 4085, 0, 0, 0, 0, 0, 4074, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 765, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2238, 2, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 16385, 16376, 17318, 16074, 16814, 16423, 15285] avg=16384.0000 max=17318 max_rank=3 min=15285 min_rank=7 imbalance_ratio=1.057007 max_load_reduction=113754 max_load_reduction_pct=0.867874 moved_to_echo=64596 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 3332, 767, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2225, 0, 2288, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3038, 0, 582, 1, 16, 0, 0, 4096, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1985, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1104, 984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 0, 4096, 74, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 764, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3327, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16380, 16391, 16801, 15920, 16360, 16464, 16379] avg=16384.0000 max=16801 max_rank=3 min=15920 min_rank=4 imbalance_ratio=1.025452 max_load_reduction=114271 max_load_reduction_pct=0.871819 moved_to_echo=65123 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 4079, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 10, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 32, 0, 0, 0, 4042, 0, 0, 4082, 0, 0, 0, 0, 3331, 762, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 4095, 0, 2231, 0, 386, 4096, 0, 2, 0, 4083, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4, 3042, 0, 2301, 350, 0, 0, 0, 3920, 0, 0, 4081, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 10, 0, 0, 13, 2294, 4096, 17, 4, 0, 7, 0, 0, 0, 4096, 0, 1103, 980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4054, 4086, 0, 0, 0, 0, 0, 3999, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 765, 1, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 0, 0, 3146, 31, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 16404, 16358, 14912, 17796, 16695, 16308, 16221] avg=16384.0000 max=17796 max_rank=4 min=14912 min_rank=3 imbalance_ratio=1.086182 max_load_reduction=113276 max_load_reduction_pct=0.864227 moved_to_echo=67020 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 4095, 0, 0, 4086, 0, 0, 0, 0, 3330, 762, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 2231, 0, 2323, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3042, 0, 12, 1, 35, 0, 0, 4037, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 2387, 4096, 0, 0, 0, 3, 0, 1, 0, 4094, 0, 1103, 981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 4093, 4095, 4096, 0, 0, 0, 0, 0, 4094, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 765, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 3504, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16380, 16379, 16841, 15312, 16763, 16454, 16555] avg=16384.0000 max=16841 max_rank=3 min=15312 min_rank=4 imbalance_ratio=1.027893 max_load_reduction=114231 max_load_reduction_pct=0.871513 moved_to_echo=65084 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 5, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 3331, 762, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2231, 0, 2531, 4094, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3042, 0, 873, 6, 349, 0, 0, 3879, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 615, 4096, 4, 0, 0, 0, 0, 0, 0, 4085, 0, 1103, 981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 4094, 0, 0, 0, 0, 0, 4063, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 765, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4063, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16380, 16399, 17051, 16341, 14980, 16415, 17125] avg=16384.0000 max=17125 max_rank=7 min=14980 min_rank=5 imbalance_ratio=1.045227 max_load_reduction=113947 max_load_reduction_pct=0.869347 moved_to_echo=64861 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4086, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 0, 4079, 0, 0, 4096, 0, 0, 0, 0, 3330, 762, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 4096, 0, 2231, 0, 468, 4085, 58, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 3042, 27, 164, 111, 414, 0, 0, 3719, 0, 0, 4091, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3460, 4096, 0, 1, 0, 4, 0, 0, 0, 4089, 0, 1103, 980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4077, 4094, 0, 1, 0, 0, 0, 3804, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 765, 50, 0, 0, 0, 4091, 0, 4095, 0, 0, 0, 0, 0, 4096, 16, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16380, 16387, 15037, 15699, 17830, 16144, 17209] avg=16384.0000 max=17830 max_rank=5 min=15037 min_rank=3 imbalance_ratio=1.088257 max_load_reduction=113242 max_load_reduction_pct=0.863968 moved_to_echo=66882 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 3331, 762, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2231, 0, 1470, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3042, 0, 1594, 0, 615, 0, 0, 4058, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 16, 0, 1582, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 1103, 981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4076, 0, 0, 0, 0, 0, 4083, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 765, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2991, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16380, 16384, 15988, 17497, 15962, 16424, 16055] avg=16384.0000 max=17497 max_rank=4 min=15962 min_rank=5 imbalance_ratio=1.067932 max_load_reduction=113575 max_load_reduction_pct=0.866508 moved_to_echo=65938 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 128, 128, 0, 128, 0, 32768, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 0, 128, 128, 16128, 32768, 128, 128, 0, 128, 0, 128, 128, 32768, 0, 8832, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[6144, 128, 0, 0, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 26624, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 17920, 0, 14976, 32768, 128, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 32768, 0, 128, 0, 128, 0, 32768, 128, 128, 32768, 0, 128, 128, 128, 26752, 6144, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 0, 0, 128, 32768, 32768, 32768, 0, 128, 128, 0, 128, 32384, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 128, 128, 128, 0, 0, 0, 128, 0, 0, 128, 128, 32768, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 24448, 128, 7552, 512, 1664, 0, 0, 31616, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4081, 0, 0, 0, 6, 4072, 0, 0, 4096, 0, 81, 0, 0, 4096, 0, 0, 0, 0, 4082, 22, 4096, 4095, 0, 4094, 0, 0, 3930, 1, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 4096, 0, 0, 1, 3, 0, 2199, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 0, 4004, 0, 4096, 0, 2574, 4089, 4096, 4096, 4096, 0, 131, 2983, 0, 25, 0, 0, 0, 0, 4096, 0, 4059, 44, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4094, 4079, 0, 0, 0, 4073, 11, 0, 23, 0, 4096, 0, 0, 466, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 1, 0, 1, 3, 4095, 0, 1, 4095, 0, 4, 0, 1, 4095, 0, 0, 0, 0, 4087, 3, 4095, 4076, 0, 4094, 1, 0, 4095, 0, 0, 0, 1, 0, 2, 3, 4095, 0, 1, 4095, 1, 0, 0, 2, 0, 3977, 0, 0, 4093, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 4096, 0, 4084, 0, 3835, 4095, 4095, 4095, 4093, 0, 3, 188, 0, 9, 0, 1, 0, 1, 4095, 0, 4095, 0, 4095, 1, 0, 1, 2, 4095, 1, 4094, 0, 0, 4094, 4095, 1, 1, 0, 4078, 0, 1, 19, 0, 4094, 0, 0, 210, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 8, 4096, 4070, 0, 4090, 0, 0, 4096, 1, 0, 0, 0, 5, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 1587, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 4096, 0, 4046, 0, 2754, 4096, 4096, 4096, 4096, 0, 53, 3847, 0, 19, 0, 0, 0, 0, 4095, 0, 4078, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4079, 2, 0, 0, 0, 4096, 0, 0, 30, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 1, 0, 0, 4091, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3, 4096, 4082, 0, 4096, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 3997, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4084, 1, 4086, 0, 1529, 4096, 4096, 4096, 4096, 2, 7, 2302, 6, 9, 0, 0, 0, 0, 4087, 0, 4093, 13, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4095, 6, 0, 4, 0, 4091, 0, 0, 378, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 4096, 4092, 0, 4096, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 3976, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 0, 2820, 4096, 4096, 4096, 4096, 0, 370, 827, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 230, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3990, 0, 0, 4095, 0, 0, 0, 0, 4078, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4059, 17, 4096, 4022, 0, 4096, 0, 0, 4065, 14, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 3779, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4096, 0, 4088, 0, 4073, 0, 855, 4096, 4096, 4096, 4096, 0, 11, 2813, 1, 88, 0, 0, 0, 0, 4089, 0, 3965, 111, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 4096, 4015, 0, 0, 0, 4096, 0, 0, 10, 0, 4090, 0, 0, 1067, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4082, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4093, 0, 0, 4033, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 2312, 4096, 4096, 4096, 4096, 0, 50, 693, 11, 7, 0, 0, 0, 0, 4095, 0, 4070, 37, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 1108, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 4096, 4095, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 2372, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 4054, 0, 4096, 0, 3925, 4096, 4096, 4096, 4096, 0, 113, 1692, 0, 51, 0, 0, 0, 0, 4096, 0, 4076, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 4093, 0, 0, 3, 0, 4096, 0, 0, 110, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4081, 0, 0, 0, 6, 4072, 3228, 485, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 81, 0, 0, 9, 0, 0, 0, 0, 4082, 22, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3930, 1, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 2199, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 4090, 3248, 859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 4096, 0, 2574, 3604, 0, 0, 0, 0, 131, 2983, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4059, 44, 4096, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3659, 4079, 0, 0, 0, 4073, 11, 0, 23, 0, 4096, 0, 0, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16371, 16481, 16257, 15339, 16403, 17417, 16397, 16407] avg=16384.0000 max=17417 max_rank=5 min=15339 min_rank=3 imbalance_ratio=1.063049 max_load_reduction=113655 max_load_reduction_pct=0.867119 moved_to_echo=66624 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 1, 0, 1, 3, 4095, 3227, 486, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 4, 0, 1, 9, 0, 0, 0, 0, 4087, 3, 4095, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 4095, 0, 0, 0, 1, 0, 2, 3, 4095, 0, 1, 4095, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 3977, 0, 0, 4093, 0, 0, 0, 0, 0, 1, 1, 4095, 4095, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 4089, 3246, 859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4084, 0, 3835, 3609, 0, 0, 0, 0, 3, 188, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 4095, 0, 4095, 1, 0, 1, 2, 6, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3659, 4095, 1, 1, 0, 4078, 0, 1, 19, 0, 4094, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16409, 16371, 16419, 17112, 16387, 15824, 16392, 16158] avg=16384.0000 max=17112 max_rank=3 min=15824 min_rank=5 imbalance_ratio=1.044434 max_load_reduction=113960 max_load_reduction_pct=0.869446 moved_to_echo=64761 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 3228, 486, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4095, 8, 4096, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 1, 0, 0, 0, 5, 1, 0, 4096, 0, 0, 4096, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1587, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 4090, 3248, 859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4046, 0, 2754, 3610, 0, 0, 0, 0, 53, 3847, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4078, 0, 4096, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3660, 4096, 0, 0, 0, 4079, 2, 0, 0, 0, 4096, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16404, 16374, 16418, 14723, 16394, 18425, 16371, 15963] avg=16384.0000 max=18425 max_rank=5 min=14723 min_rank=3 imbalance_ratio=1.124573 max_load_reduction=112647 max_load_reduction_pct=0.859428 moved_to_echo=67153 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 3230, 489, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 3, 0, 0, 0, 0, 4094, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 3248, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 2820, 3607, 0, 0, 0, 0, 370, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3656, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 16383, 16393, 17112, 16394, 15815, 16389, 16174] avg=16384.0000 max=17112 max_rank=3 min=15815 min_rank=5 imbalance_ratio=1.044434 max_load_reduction=113960 max_load_reduction_pct=0.869446 moved_to_echo=64772 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4082, 3228, 486, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 9, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4033, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 3248, 859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2312, 3610, 0, 0, 0, 0, 50, 693, 11, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4070, 37, 4096, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3660, 4096, 0, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 1108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16391, 16394, 16353, 17228, 16387, 14875, 16400, 17044] avg=16384.0000 max=17228 max_rank=3 min=14875 min_rank=5 imbalance_ratio=1.051514 max_load_reduction=113844 max_load_reduction_pct=0.868561 moved_to_echo=64706 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 1, 0, 0, 4091, 3228, 486, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4096, 3, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 3249, 859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1, 4086, 0, 1529, 3610, 0, 0, 0, 2, 7, 2302, 6, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4093, 13, 4095, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3660, 4096, 0, 0, 0, 4095, 6, 0, 4, 0, 4091, 0, 0, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16400, 16381, 16418, 17132, 16385, 15636, 16390, 16330] avg=16384.0000 max=17132 max_rank=3 min=15636 min_rank=5 imbalance_ratio=1.045654 max_load_reduction=113940 max_load_reduction_pct=0.869293 moved_to_echo=64741 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3990, 0, 0, 4095, 0, 0, 0, 0, 4078, 3228, 486, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4059, 17, 4096, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4065, 14, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3779, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4096, 4089, 3248, 859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4073, 0, 855, 3610, 0, 0, 0, 0, 11, 2813, 1, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 3965, 111, 4096, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3660, 4015, 0, 0, 0, 4096, 0, 0, 10, 0, 4090, 0, 0, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16280, 16299, 16401, 16918, 16334, 15539, 16363, 16938] avg=16384.0000 max=16938 max_rank=7 min=15539 min_rank=5 imbalance_ratio=1.033813 max_load_reduction=114134 max_load_reduction_pct=0.870773 moved_to_echo=65174 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 3228, 486, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4093, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2372, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4090, 3248, 859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 4096, 0, 3925, 3610, 0, 0, 0, 0, 113, 1692, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 0, 4096, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3660, 4096, 0, 0, 0, 4093, 0, 0, 3, 0, 4096, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16405, 16389, 16413, 15508, 16388, 17541, 16370, 16058] avg=16384.0000 max=17541 max_rank=5 min=15508 min_rank=3 imbalance_ratio=1.070618 max_load_reduction=113531 max_load_reduction_pct=0.866173 moved_to_echo=66357 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32768, 128, 0, 32512, 128, 0, 0, 128, 128, 128, 128, 32768, 0, 128, 32768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 32768, 0, 32640, 256, 32768, 128, 0, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 29312, 32768, 128, 128, 0, 32768, 128, 128, 128, 0, 32768, 0, 0, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 128, 128, 0, 25984, 0, 0, 32768, 0, 0, 0, 0, 0, 128, 128, 32768, 32768, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 0, 128, 0, 128, 128, 0, 0, 0, 0, 32768, 128, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 0, 0, 0, 0, 32768, 0, 0, 32768, 128, 128, 128, 128, 32768, 25856, 3968, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 128, 128, 32768, 32768, 25984, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 32640, 128, 32768, 0, 20608, 28928, 0, 0, 0, 128, 768, 15360, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[117, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4086, 11, 0, 0, 0, 4096, 0, 4096, 7, 0, 0, 0, 9, 4096, 4096, 0, 4087, 4094, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4087, 12, 0, 0, 48, 0, 4096, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4022, 0, 0, 4096, 0, 0, 0, 0, 482, 4096, 0, 4096, 0, 0, 3508, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[369, 4096, 0, 4090, 0, 0, 0, 4096, 0, 4095, 4, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 16, 4095, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 3, 4096, 0, 4048, 0, 0, 0, 576, 0, 4096, 5, 0, 0, 0, 4082, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 4095, 4093, 0, 0, 0, 144, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3791, 0, 1, 4096, 0, 0, 0, 0, 27, 4096, 0, 4096, 0, 0, 3326, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[138, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 3995, 0, 0, 0, 26, 0, 4096, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4050, 0, 0, 4096, 0, 0, 0, 0, 263, 4096, 0, 4096, 0, 0, 3813, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4, 4095, 1, 4088, 0, 0, 1, 4095, 0, 4096, 0, 0, 0, 0, 4095, 1, 4095, 0, 1, 0, 1, 0, 4095, 4095, 0, 4093, 4095, 2, 1, 0, 4079, 4094, 0, 4095, 1, 3855, 0, 0, 1, 6, 1, 4093, 14, 0, 0, 0, 4095, 1, 0, 1, 3, 4095, 0, 0, 1, 0, 1, 0, 0, 0, 4095, 4095, 2, 1, 4089, 1, 0, 1, 2, 0, 0, 1, 4095, 4095, 0, 0, 3, 0, 1, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 4042, 0, 0, 4095, 1, 0, 0, 0, 309, 4095, 0, 4093, 1, 0, 4073, 0, 0, 1, 4094, 1, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 1, 2, 8, 1, 0, 0, 4095, 4095, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[2, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 11, 4096, 0, 4095, 0, 0, 0, 0, 27, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4072, 6, 0, 0, 156, 0, 4096, 0, 2, 0, 0, 4053, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 236, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 418, 4096, 0, 4096, 0, 0, 3309, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[2533, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 59, 0, 0, 0, 4084, 0, 4096, 18, 0, 0, 0, 106, 4096, 4096, 0, 4076, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4090, 88, 0, 0, 743, 0, 4096, 0, 0, 0, 0, 4021, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 829, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2609, 0, 0, 4096, 0, 0, 0, 0, 427, 4096, 0, 4096, 0, 0, 920, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[209, 4096, 0, 4096, 18, 0, 0, 4096, 2, 4096, 17, 0, 0, 0, 4087, 0, 4096, 0, 0, 0, 0, 61, 4096, 4096, 0, 4057, 4037, 0, 0, 0, 4096, 4096, 0, 4095, 0, 3781, 48, 0, 0, 17, 0, 4096, 0, 32, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 3816, 0, 38, 4096, 0, 0, 0, 0, 1919, 4096, 0, 4096, 0, 0, 2451, 0, 0, 0, 4082, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4091, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 7, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 131, 0, 4096, 31, 0, 0, 0, 4077, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 44, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4056, 0, 0, 4096, 0, 0, 0, 0, 2014, 4096, 0, 4096, 0, 0, 1948, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[117, 3667, 0, 4096, 0, 0, 0, 0, 0, 4086, 11, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 9, 4083, 0, 0, 4087, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 4087, 12, 0, 0, 48, 0, 4096, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3816, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 9, 0, 0, 3818, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4022, 0, 0, 4096, 0, 0, 0, 0, 4096, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 4096, 0, 4096, 0, 0, 3508, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 302, 130, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3817, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16073, 16376, 16307, 16383, 16237, 16590, 16723, 16383] avg=16384.0000 max=16723 max_rank=6 min=16073 min_rank=0 imbalance_ratio=1.020691 max_load_reduction=114349 max_load_reduction_pct=0.872414 moved_to_echo=65933 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4, 3666, 1, 4088, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4082, 0, 0, 4093, 4095, 2, 1, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 1, 3855, 0, 0, 1, 6, 1, 4093, 14, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 4095, 0, 0, 1, 0, 1, 0, 0, 0, 4095, 4095, 2, 1, 3815, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 1, 2, 0, 0, 1, 4095, 4095, 0, 0, 3, 0, 1, 0, 3816, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 4042, 0, 0, 4095, 1, 0, 0, 0, 4095, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 4095, 0, 4093, 1, 0, 4073, 0, 0, 1, 4094, 1, 0, 0, 0, 0, 302, 130, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 1, 2, 8, 1, 0, 0, 4095, 4095, 0, 0, 0, 3817, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15952, 16354, 16032, 16387, 16231, 16608, 17112, 16396] avg=16384.0000 max=17112 max_rank=6 min=15952 min_rank=0 imbalance_ratio=1.044434 max_load_reduction=113960 max_load_reduction_pct=0.869446 moved_to_echo=66347 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[369, 3667, 0, 4090, 0, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 16, 4082, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3966, 0, 4048, 0, 0, 0, 576, 0, 4096, 5, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3816, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 2, 0, 0, 0, 4095, 4093, 0, 0, 0, 144, 0, 0, 3818, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3791, 0, 1, 4096, 0, 0, 0, 0, 4096, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4096, 0, 4096, 0, 0, 3326, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 302, 130, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 3818, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16321, 16387, 16776, 16384, 16373, 16360, 16086, 16385] avg=16384.0000 max=16776 max_rank=2 min=16086 min_rank=6 imbalance_ratio=1.023926 max_load_reduction=114296 max_load_reduction_pct=0.872009 moved_to_echo=65204 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[138, 3657, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 3995, 0, 0, 0, 26, 0, 4096, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3817, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3819, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4050, 0, 0, 4096, 0, 0, 0, 0, 4096, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 263, 4096, 0, 4096, 0, 0, 3813, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 308, 137, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3817, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16083, 16370, 16175, 16384, 16238, 16617, 16823, 16382] avg=16384.0000 max=16823 max_rank=6 min=16083 min_rank=0 imbalance_ratio=1.026794 max_load_reduction=114249 max_load_reduction_pct=0.871651 moved_to_echo=66060 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[209, 3667, 0, 4096, 18, 0, 0, 0, 2, 4096, 17, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 4083, 0, 0, 4057, 4037, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 0, 3781, 48, 0, 0, 17, 0, 4096, 0, 32, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3816, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4, 0, 0, 3819, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 3816, 0, 38, 4096, 0, 0, 0, 0, 4096, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1919, 4096, 0, 4096, 0, 0, 2451, 0, 0, 0, 4082, 0, 5, 0, 0, 0, 302, 130, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 3817, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16192, 16334, 16030, 16385, 16224, 16432, 17094, 16381] avg=16384.0000 max=17094 max_rank=6 min=16030 min_rank=2 imbalance_ratio=1.043335 max_load_reduction=113978 max_load_reduction_pct=0.869583 moved_to_echo=66131 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2, 3667, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 4083, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 4072, 6, 0, 0, 156, 0, 4096, 0, 2, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3816, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 236, 0, 0, 3819, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 4096, 0, 4096, 0, 0, 3309, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 302, 130, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3816, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15968, 16398, 16351, 16383, 16467, 16664, 16460, 16381] avg=16384.0000 max=16664 max_rank=5 min=15968 min_rank=0 imbalance_ratio=1.017090 max_load_reduction=114408 max_load_reduction_pct=0.872864 moved_to_echo=65972 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[2533, 3667, 0, 4096, 0, 0, 0, 0, 0, 4096, 59, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 106, 4083, 0, 0, 4076, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 4090, 88, 0, 0, 743, 0, 4096, 0, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3816, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 829, 0, 0, 3819, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2609, 0, 0, 4096, 0, 0, 0, 0, 4096, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 427, 4096, 0, 4096, 0, 0, 920, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 302, 130, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3817, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18535, 16475, 17004, 16383, 17037, 15177, 14079, 16382] avg=16384.0000 max=18535 max_rank=0 min=14079 min_rank=6 imbalance_ratio=1.131287 max_load_reduction=112537 max_load_reduction_pct=0.858589 moved_to_echo=62675 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 3667, 0, 4091, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4083, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 4096, 0, 0, 0, 131, 0, 4096, 31, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3816, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 44, 0, 0, 3819, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4056, 0, 0, 4096, 0, 0, 0, 0, 4096, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2014, 4096, 0, 4096, 0, 0, 1948, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 302, 130, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3817, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15948, 16378, 16397, 16383, 16265, 16624, 16695, 16382] avg=16384.0000 max=16695 max_rank=6 min=15948 min_rank=0 imbalance_ratio=1.018982 max_load_reduction=114377 max_load_reduction_pct=0.872627 moved_to_echo=65966 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[3456, 29440, 128, 32768, 128, 0, 128, 0, 128, 32768, 128, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 31744, 128, 32128, 256, 0, 128, 1792, 128, 32768, 128, 128, 0, 0, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 128, 128, 128, 0, 0, 32768, 32768, 0, 0, 0, 30592, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 0, 128, 32768, 32768, 0, 0, 128, 1280, 128, 0, 30592, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 0, 32768, 128, 128, 0, 0, 30592, 0, 128, 32768, 128, 0, 0, 0, 32768, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[5888, 32768, 0, 32768, 128, 0, 23424, 0, 0, 128, 32768, 128, 128, 0, 0, 0, 2432, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 32768, 128, 0, 128, 0, 128, 0, 0, 0, 32768, 32768, 128, 128, 30592, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 256, 32768, 0, 0, 32768, 32768, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 3328, 22, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 4079, 4096, 0, 0, 0, 0, 0, 17, 0, 0, 4090, 4092, 4085, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 9, 0, 4077, 0, 0, 0, 0, 31, 1, 0, 0, 0, 97, 0, 0, 729, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4014, 4095, 4096, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4095, 1, 0, 3631, 0, 2, 2, 0, 4095, 0, 1, 1, 4095, 0, 4095, 4095, 0, 1, 0, 0, 4094, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 4091, 4093, 1, 4095, 0, 4095, 4095, 1, 0, 1, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 4089, 4096, 1, 0, 0, 0, 0, 1, 2, 0, 0, 1, 0, 4094, 4, 4095, 1, 1, 4095, 0, 0, 4095, 4095, 4095, 0, 4094, 0, 1, 4095, 0, 0, 0, 3, 2, 1, 0, 1, 0, 40, 0, 0, 1790, 0, 1, 1, 0, 4095, 1, 0, 1, 1, 2725, 4095, 4095, 6, 0, 4095, 1, 0, 4095, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 2, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 3298, 39, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 219, 0, 0, 3818, 4091, 4083, 0, 4096, 0, 4094, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 512, 0, 0, 4059, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 46, 0, 4038, 0, 0, 0, 0, 1446, 120, 0, 0, 3, 139, 0, 0, 411, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2400, 4096, 4096, 0, 10, 4096, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 2528, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4087, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 26, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 12, 0, 4084, 0, 0, 0, 0, 48, 6, 0, 0, 0, 74, 0, 0, 1499, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4074, 4091, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 3893, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4, 0, 0, 206, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 2755, 631, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 2, 8, 0, 0, 4095, 4094, 4080, 0, 4096, 0, 4093, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 90, 0, 0, 0, 0, 6, 0, 0, 656, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 4075, 4095, 4096, 0, 1, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 3716, 12, 0, 3, 0, 4096, 0, 0, 0, 4070, 0, 4096, 4094, 0, 0, 13, 0, 4066, 4096, 0, 0, 0, 0, 7, 7, 0, 0, 4054, 4096, 4089, 0, 4096, 0, 4085, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 39, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 3, 0, 4096, 4096, 4096, 0, 4096, 30, 0, 4080, 0, 0, 0, 0, 93, 2, 0, 0, 0, 0, 0, 0, 394, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 3944, 4096, 4096, 0, 9, 4096, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 3536, 8, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 18, 0, 0, 4066, 4079, 4088, 0, 4096, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 9, 0, 4094, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4072, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 674, 0, 4095, 1, 0, 3631, 0, 2, 2, 0, 4095, 0, 1, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4095, 0, 1, 0, 0, 4094, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4091, 4093, 1, 0, 0, 4095, 66, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 4089, 4096, 1, 0, 0, 0, 0, 1, 2, 3421, 359, 216, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 4, 4095, 1, 1, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 4095, 0, 0, 0, 3, 2, 1, 0, 1, 0, 40, 0, 0, 4095, 3813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1790, 0, 1, 1, 0, 4095, 1, 0, 1, 1, 2725, 4095, 0, 6, 0, 3736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 2, 1, 1, 4095, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16597, 16352, 16442, 16309, 16386, 16145, 16452, 16389] avg=16384.0000 max=16597 max_rank=0 min=16145 min_rank=5 imbalance_ratio=1.013000 max_load_reduction=114475 max_load_reduction_pct=0.873375 moved_to_echo=65372 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 674, 0, 4096, 0, 0, 3328, 22, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4096, 0, 0, 0, 0, 4079, 4096, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4092, 4085, 0, 0, 0, 4096, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3422, 359, 216, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 4077, 0, 0, 0, 0, 31, 1, 0, 0, 0, 97, 0, 0, 4096, 3814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 729, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4014, 4095, 0, 0, 2, 3737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16311, 16355, 16429, 16314, 16384, 16221, 16673, 16385] avg=16384.0000 max=16673 max_rank=6 min=16221 min_rank=5 imbalance_ratio=1.017639 max_load_reduction=114399 max_load_reduction_pct=0.872795 moved_to_echo=65663 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 672, 0, 4096, 0, 0, 3893, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3424, 360, 223, 34, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4, 0, 0, 4096, 3820, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 3736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16853, 16350, 16437, 16334, 16379, 16119, 16217, 16383] avg=16384.0000 max=16853 max_rank=0 min=16119 min_rank=5 imbalance_ratio=1.028625 max_load_reduction=114219 max_load_reduction_pct=0.871422 moved_to_echo=65098 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 674, 0, 4096, 0, 0, 3716, 12, 0, 3, 0, 4096, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4094, 0, 0, 13, 0, 4066, 4096, 0, 0, 0, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 4096, 4089, 0, 0, 0, 4085, 66, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 39, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3422, 359, 216, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 3, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 30, 0, 4080, 0, 0, 0, 0, 93, 2, 0, 0, 0, 0, 0, 0, 4096, 3814, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 394, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 3944, 4096, 0, 0, 9, 3737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16667, 16350, 16391, 16353, 16387, 16212, 16280, 16432] avg=16384.0000 max=16667 max_rank=0 min=16212 min_rank=5 imbalance_ratio=1.017273 max_load_reduction=114405 max_load_reduction_pct=0.872841 moved_to_echo=65311 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 674, 0, 4096, 0, 0, 2528, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4096, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4087, 4096, 0, 0, 0, 4096, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 26, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 3422, 359, 216, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 4084, 0, 0, 0, 0, 48, 6, 0, 0, 0, 74, 0, 0, 4096, 3814, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1499, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4074, 4091, 0, 0, 0, 3737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15490, 16351, 16396, 16331, 16384, 16231, 17497, 16392] avg=16384.0000 max=17497 max_rank=6 min=15490 min_rank=0 imbalance_ratio=1.067932 max_load_reduction=113575 max_load_reduction_pct=0.866508 moved_to_echo=66504 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 674, 0, 4096, 0, 0, 2755, 631, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 2, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4094, 4080, 0, 0, 0, 4093, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 3422, 359, 216, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 90, 0, 0, 0, 0, 6, 0, 0, 4096, 3812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 656, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 4075, 4095, 0, 0, 1, 3737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16349, 16366, 16428, 16309, 16384, 16195, 16659, 16382] avg=16384.0000 max=16659 max_rank=6 min=16195 min_rank=5 imbalance_ratio=1.016785 max_load_reduction=114413 max_load_reduction_pct=0.872902 moved_to_echo=65620 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 674, 0, 4096, 0, 0, 3298, 39, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3818, 4091, 4083, 0, 0, 0, 4094, 66, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 512, 0, 0, 4059, 4096, 0, 0, 0, 0, 0, 0, 0, 3422, 359, 216, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 46, 0, 4038, 0, 0, 0, 0, 1446, 120, 0, 0, 3, 139, 0, 0, 4096, 3814, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 411, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2400, 4096, 0, 0, 10, 3737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16299, 16575, 16153, 16789, 16384, 17799, 14750, 16323] avg=16384.0000 max=17799 max_rank=5 min=14750 min_rank=6 imbalance_ratio=1.086365 max_load_reduction=113273 max_load_reduction_pct=0.864204 moved_to_echo=65256 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 674, 0, 4096, 0, 0, 3536, 8, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4079, 4088, 0, 0, 0, 4096, 66, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3422, 359, 216, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 4094, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 4096, 3814, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4072, 4096, 0, 0, 0, 3737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16506, 16373, 16396, 16333, 16384, 16150, 16544, 16386] avg=16384.0000 max=16544 max_rank=6 min=16150 min_rank=5 imbalance_ratio=1.009766 max_load_reduction=114528 max_load_reduction_pct=0.873779 moved_to_echo=65464 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 32768, 0, 0, 0, 128, 1792, 256, 0, 128, 128, 384, 0, 0, 32768, 30592, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 5504, 0, 32768, 128, 0, 26752, 768, 128, 128, 128, 32768, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32640, 32768, 0, 128, 128, 0, 32768, 32768, 128, 0, 0, 0, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[6272, 128, 128, 128, 0, 32768, 128, 128, 128, 128, 29440, 32768, 0, 128, 128, 29952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32384, 32768, 32768, 128, 0, 0, 32768, 640, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 32768, 128, 32768, 128, 128, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 32768, 0, 0, 0, 0, 128, 0, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32768, 0, 0, 640, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 128, 128, 27392, 2944, 1792, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 2, 4094, 0, 0, 4094, 4096, 145, 4095, 4, 0, 1, 0, 0, 3, 0, 4092, 4095, 0, 4095, 0, 1, 0, 1, 0, 4094, 0, 2461, 0, 0, 4095, 4095, 4091, 0, 0, 0, 4093, 4, 0, 1, 0, 0, 1, 0, 1, 4093, 0, 4094, 1, 16, 4095, 0, 0, 0, 1, 1, 1, 0, 4095, 4092, 1, 0, 0, 0, 4094, 1, 0, 0, 2, 0, 4, 1, 0, 0, 4093, 0, 1, 1, 4095, 15, 4088, 962, 0, 4095, 1, 0, 0, 1, 0, 2, 0, 0, 0, 0, 2, 4076, 0, 1, 2, 0, 1, 0, 4095, 1, 1, 0, 4089, 0, 4094, 2, 0, 0, 13, 0, 4075, 4091, 4095, 4095, 0, 541, 1, 4094, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4086, 3, 4084, 2, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3743, 0, 0, 4096, 4096, 4057, 0, 0, 81, 4074, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4089, 0, 6, 0, 0, 4096, 0, 2, 0, 0, 0, 70, 0, 0, 0, 4096, 0, 0, 0, 4096, 54, 4061, 0, 0, 4096, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 33, 4035, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 30, 4096, 0, 4096, 0, 0, 0, 0, 53, 3836, 4096, 4096, 4080, 0, 448, 0, 4096, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 230, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3393, 0, 0, 4096, 4096, 4094, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4096, 0, 4096, 51, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2, 0, 4092, 4096, 4096, 4096, 0, 433, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 10, 3810, 0, 0, 4096, 4096, 4096, 0, 0, 48, 4090, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4084, 2, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 55, 1, 4094, 4096, 4096, 4096, 0, 168, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 149, 4094, 2, 0, 0, 0, 0, 2, 2, 4093, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 557, 0, 0, 4096, 4096, 4094, 0, 0, 18, 4096, 167, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 4078, 4, 0, 4096, 0, 0, 4, 0, 8, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 44, 3, 4095, 4096, 4096, 4095, 0, 3168, 0, 4096, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4086, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 10, 1550, 1, 0, 4096, 4096, 4087, 0, 0, 95, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 20, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4061, 0, 0, 0, 4096, 0, 4096, 3, 0, 4096, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 147, 0, 4096, 4096, 4096, 4096, 0, 2302, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 7, 4092, 9, 0, 0, 0, 0, 4, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 34, 349, 0, 0, 4096, 4096, 4096, 0, 0, 192, 4096, 61, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 86, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 4096, 0, 4074, 1, 0, 4096, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 37, 3966, 1, 0, 38, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 170, 6, 4089, 4096, 4096, 4096, 0, 3334, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 23, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3515, 0, 0, 4096, 4096, 4096, 0, 0, 20, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 4096, 19, 4073, 25, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 483, 0, 4095, 4096, 4096, 4096, 0, 52, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 2, 4024, 0, 0, 4094, 4096, 145, 4095, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4092, 4095, 0, 1666, 0, 1, 0, 1, 0, 4094, 0, 2461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 4095, 4091, 0, 0, 0, 4093, 4, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4094, 1, 16, 0, 0, 0, 0, 1, 1, 1, 0, 4095, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 2, 0, 4, 1, 0, 0, 4093, 0, 1, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 15, 4088, 962, 0, 4095, 1, 0, 0, 1, 0, 2, 0, 0, 0, 2429, 1388, 74, 63, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4076, 0, 1, 2, 0, 1, 0, 4095, 1, 1, 0, 4089, 0, 4094, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 13, 0, 4075, 4091, 0, 4095, 0, 541, 1, 2706, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16463, 16413, 16307, 16395, 16383, 17217, 16369, 15525] avg=16384.0000 max=17217 max_rank=5 min=15525 min_rank=7 imbalance_ratio=1.050842 max_load_reduction=113855 max_load_reduction_pct=0.868645 moved_to_echo=65494 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4025, 0, 0, 4096, 4086, 3, 4084, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 1667, 0, 0, 0, 0, 0, 4096, 0, 3743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 4096, 4057, 0, 0, 81, 4074, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 2, 0, 0, 0, 70, 0, 0, 0, 4096, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 54, 4061, 0, 0, 4096, 0, 0, 13, 0, 0, 0, 0, 0, 0, 2429, 1388, 74, 64, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4035, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 30, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 3836, 4096, 0, 4080, 0, 448, 0, 2708, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16296, 17697, 16333, 16377, 16458, 16279, 16409, 15223] avg=16384.0000 max=17697 max_rank=1 min=15223 min_rank=7 imbalance_ratio=1.080139 max_load_reduction=113375 max_load_reduction_pct=0.864983 moved_to_echo=64369 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4019, 0, 0, 4096, 4096, 230, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1662, 0, 0, 0, 0, 0, 4096, 0, 3393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 4096, 4094, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 51, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2434, 1394, 80, 65, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 4096, 0, 4096, 0, 433, 0, 2702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16537, 17343, 16303, 16379, 16372, 16316, 16401, 15421] avg=16384.0000 max=17343 max_rank=1 min=15421 min_rank=7 imbalance_ratio=1.058533 max_load_reduction=113729 max_load_reduction_pct=0.867683 moved_to_echo=64510 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4025, 0, 0, 4096, 4086, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1667, 0, 0, 0, 0, 0, 4096, 10, 1550, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 4096, 4087, 0, 0, 95, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4061, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3, 0, 4096, 0, 7, 3, 0, 0, 0, 0, 0, 0, 2429, 1388, 74, 64, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 4096, 4096, 0, 4096, 0, 2302, 0, 2708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16330, 15516, 16385, 16385, 16366, 16260, 16385, 17445] avg=16384.0000 max=17445 max_rank=7 min=15516 min_rank=1 imbalance_ratio=1.064758 max_load_reduction=113627 max_load_reduction_pct=0.866905 moved_to_echo=66456 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4025, 0, 0, 4096, 4094, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1667, 0, 0, 0, 0, 0, 4096, 10, 3810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 4096, 4096, 0, 0, 48, 4090, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4084, 2, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2429, 1388, 74, 64, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 1, 4094, 4096, 0, 4096, 0, 168, 0, 2708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16323, 17775, 16356, 16384, 16384, 16242, 16390, 15218] avg=16384.0000 max=17775 max_rank=1 min=15218 min_rank=7 imbalance_ratio=1.084900 max_load_reduction=113297 max_load_reduction_pct=0.864388 moved_to_echo=64234 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4025, 0, 0, 4096, 4096, 149, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4093, 4096, 0, 1667, 0, 0, 0, 0, 0, 4096, 1, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 4096, 4094, 0, 0, 18, 4096, 167, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4078, 4, 0, 4096, 0, 0, 4, 0, 8, 0, 0, 0, 0, 2429, 1388, 74, 64, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 3, 4095, 4096, 0, 4095, 0, 3168, 0, 2708, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16462, 14514, 16494, 16383, 16374, 16246, 16388, 18211] avg=16384.0000 max=18211 max_rank=7 min=14514 min_rank=1 imbalance_ratio=1.111511 max_load_reduction=112861 max_load_reduction_pct=0.861061 moved_to_echo=67219 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4025, 0, 0, 4096, 4096, 7, 4092, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 4096, 0, 1667, 0, 0, 0, 0, 0, 4096, 34, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 4096, 4096, 0, 0, 192, 4096, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4074, 1, 0, 4096, 0, 0, 18, 0, 0, 0, 0, 0, 0, 2429, 1388, 74, 64, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 3966, 1, 0, 38, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 6, 4089, 4096, 0, 4096, 0, 3334, 0, 2708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16325, 14342, 16563, 16384, 16379, 16244, 16336, 18499] avg=16384.0000 max=18499 max_rank=7 min=14342 min_rank=1 imbalance_ratio=1.129089 max_load_reduction=112573 max_load_reduction_pct=0.858864 moved_to_echo=67458 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4025, 0, 0, 4096, 4096, 23, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 1667, 0, 0, 0, 0, 0, 4096, 0, 3515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 4096, 4096, 0, 0, 20, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 19, 4073, 25, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2429, 1388, 74, 64, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 0, 4095, 4096, 0, 4096, 0, 52, 0, 2708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16336, 17472, 16331, 16385, 16356, 16268, 16394, 15530] avg=16384.0000 max=17472 max_rank=1 min=15530 min_rank=7 imbalance_ratio=1.066406 max_load_reduction=113600 max_load_reduction_pct=0.866699 moved_to_echo=64548 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 32768, 128, 128, 0, 0, 0, 0, 128, 128, 128, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 32640, 128, 128, 128, 0, 128, 0, 32768, 128, 128, 128, 32768, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 0, 1024, 128, 32512, 32768, 0, 32768, 0, 10496, 128, 21760, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 128, 32768, 32768, 0, 13440, 0, 128, 0, 128, 0, 32768, 128, 19456, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32256, 32768, 32768, 0, 0, 512, 32768, 256, 128, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 128, 32256, 0, 0, 32768, 32768, 640, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 0, 32768, 128, 128, 0, 128, 0, 128, 128, 0, 0, 32640, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 32768, 1152, 0, 32768, 128, 128, 128, 128, 128, 128, 0, 0, 0, 19456, 11136, 640, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 4095, 0, 0, 0, 1, 4094, 0, 1, 0, 0, 0, 0, 0, 1, 0, 4090, 1, 4095, 4095, 521, 1, 4092, 3758, 57, 0, 0, 4094, 4095, 1, 4095, 1, 4095, 2, 1, 1, 0, 4095, 0, 4095, 4094, 1, 0, 4095, 1, 0, 0, 0, 1, 1, 1, 0, 4093, 0, 0, 0, 0, 0, 1, 0, 4094, 1, 347, 4095, 1, 0, 0, 1, 0, 1, 3, 4053, 4095, 1, 1, 0, 4089, 1, 1, 0, 4095, 0, 0, 1, 0, 1, 2, 1, 1, 0, 1, 4095, 4, 2, 0, 1, 4095, 4094, 1, 0, 0, 8, 0, 0, 0, 0, 0, 1, 3547, 4095, 4095, 0, 1, 0, 0, 4095, 0, 4095, 0, 0, 4092, 4, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 386, 0, 4092, 4030, 24, 0, 0, 4096, 4095, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 95, 4096, 0, 0, 0, 0, 0, 0, 7, 4093, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3701, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4066, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 209, 0, 4096, 3952, 8, 0, 0, 4096, 4095, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 4027, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 5, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 4096, 268, 0, 4096, 3847, 25, 0, 0, 4096, 4054, 0, 4073, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 22, 4096, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 13, 4096, 4091, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4082, 0, 11, 0, 59, 0, 7, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 4032, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 596, 0, 4096, 3689, 12, 0, 0, 4096, 4089, 0, 4094, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4094, 0, 88, 4096, 0, 0, 0, 0, 0, 0, 5, 3836, 4095, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 2, 2, 0, 0, 4096, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4092, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 4096, 915, 0, 4096, 3455, 44, 0, 0, 4096, 4096, 0, 4089, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 3822, 4096, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 6, 0, 13, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4052, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 2556, 0, 4096, 1801, 60, 0, 0, 4096, 3992, 0, 3962, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 1, 4096, 0, 171, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 46, 0, 82, 0, 0, 0, 0, 3831, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4059, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 484, 0, 4096, 3802, 1, 0, 0, 4096, 4094, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 5, 0, 0, 0, 6, 0, 0, 0, 0, 3906, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4087, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 4095, 0, 0, 0, 1, 4094, 0, 1, 0, 0, 0, 0, 3904, 193, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 1, 0, 0, 521, 1, 3899, 3758, 57, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 22, 2, 1, 1, 0, 4095, 0, 0, 4094, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 4093, 0, 0, 0, 0, 0, 1, 0, 4094, 4095, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 347, 4086, 1, 0, 0, 1, 0, 1, 3, 4053, 4095, 1, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 0, 0, 1, 0, 1, 2, 1, 1, 0, 1, 4095, 4, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4095, 4094, 1, 0, 0, 8, 0, 0, 0, 0, 0, 1, 3547, 4073, 166, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4095, 0, 1, 0, 0, 4095, 0, 4095, 0, 0, 4092, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 16423, 16409, 16377, 16679, 16392, 15992, 16408] avg=16384.0000 max=16679 max_rank=4 min=15992 min_rank=6 imbalance_ratio=1.018005 max_load_reduction=114393 max_load_reduction_pct=0.872749 moved_to_echo=65471 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 3905, 193, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 386, 0, 3899, 4030, 24, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 4087, 0, 0, 0, 0, 0, 0, 7, 4093, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3701, 4074, 166, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16530, 16406, 16381, 16474, 16383, 16144, 16379] avg=16384.0000 max=16530 max_rank=1 min=16144 min_rank=6 imbalance_ratio=1.008911 max_load_reduction=114542 max_load_reduction_pct=0.873886 moved_to_echo=65380 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3907, 200, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 209, 0, 3896, 3952, 8, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4086, 0, 0, 0, 0, 0, 0, 0, 4027, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4077, 172, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16405, 16256, 16403, 16383, 16314, 16382, 16528, 16401] avg=16384.0000 max=16528 max_rank=6 min=16256 min_rank=1 imbalance_ratio=1.008789 max_load_reduction=114544 max_load_reduction_pct=0.873901 moved_to_echo=65625 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 5, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3905, 193, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 268, 0, 3903, 3847, 25, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 22, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 22, 4096, 4096, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4087, 0, 0, 0, 0, 0, 0, 13, 4096, 4091, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 11, 0, 59, 0, 7, 0, 0, 0, 0, 4096, 4074, 166, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16400, 16190, 16374, 16404, 16385, 16382, 16595, 16342] avg=16384.0000 max=16595 max_rank=6 min=16190 min_rank=1 imbalance_ratio=1.012878 max_load_reduction=114477 max_load_reduction_pct=0.873390 moved_to_echo=65704 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3905, 193, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 915, 0, 3903, 3455, 44, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 22, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 3822, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 6, 0, 13, 0, 0, 0, 0, 4096, 4074, 166, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16395, 16502, 16399, 16382, 16097, 16382, 16550, 16365] avg=16384.0000 max=16550 max_rank=6 min=16097 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=114522 max_load_reduction_pct=0.873734 moved_to_echo=65394 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 3905, 193, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 596, 0, 3903, 3689, 12, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 22, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4094, 4096, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 4087, 0, 0, 0, 0, 0, 0, 5, 3836, 4095, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 4074, 166, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16396, 16385, 16404, 16379, 16202, 16386, 16515, 16405] avg=16384.0000 max=16515 max_rank=6 min=16202 min_rank=4 imbalance_ratio=1.007996 max_load_reduction=114557 max_load_reduction_pct=0.874001 moved_to_echo=65508 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 3905, 193, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2556, 0, 3903, 1801, 60, 0, 0, 0, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 0, 22, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 1, 4096, 4096, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 4087, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 46, 0, 82, 0, 0, 0, 0, 3831, 4074, 166, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16313, 16408, 16271, 16385, 16546, 16382, 16395, 16372] avg=16384.0000 max=16546 max_rank=4 min=16271 min_rank=2 imbalance_ratio=1.009888 max_load_reduction=114526 max_load_reduction_pct=0.873764 moved_to_echo=65695 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 3905, 193, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 484, 0, 3903, 3802, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 5, 0, 0, 0, 6, 0, 0, 0, 0, 3906, 4074, 166, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16396, 16378, 16406, 16381, 16375, 16383, 16353, 16400] avg=16384.0000 max=16406 max_rank=2 min=16353 min_rank=6 imbalance_ratio=1.001343 max_load_reduction=114666 max_load_reduction_pct=0.874832 moved_to_echo=65511 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 0, 128, 32768, 0, 0, 0, 128, 32768, 0, 128, 0, 128, 0, 0, 31360, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 32768, 32768, 128, 128, 0, 128, 0, 128, 0, 0, 0, 128, 31360, 32640, 1408, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[256, 32768, 0, 128, 0, 0, 32768, 0, 32768, 0, 0, 32640, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 128, 128, 128, 128, 128, 128, 0, 128, 32768, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32640, 128, 256, 128, 128, 128, 0, 32768, 0, 0, 32768, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 768, 32768, 128, 0, 0, 128, 0, 128, 128, 32128, 32768, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 128, 0, 32768, 128, 0, 0, 6016, 128, 31232, 28416, 256, 0, 0, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 128, 128, 128, 0, 32768, 128, 0, 128, 0, 0, 128, 128, 32768, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 4095, 4095, 4094, 0, 0, 4094, 0, 1, 4094, 4095, 0, 0, 0, 0, 4, 0, 4093, 0, 0, 0, 1, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 4095, 0, 1, 1, 0, 18, 0, 4095, 0, 0, 0, 0, 3, 1, 4095, 4093, 0, 4, 4091, 0, 1, 2, 4095, 0, 1, 4094, 4095, 1, 0, 2, 1, 0, 0, 1, 4095, 1, 0, 1, 0, 4095, 4095, 0, 2, 1, 1, 0, 4095, 1, 0, 4095, 4095, 4095, 0, 0, 4095, 1, 0, 1, 0, 0, 4094, 4087, 0, 0, 0, 0, 0, 4080, 1, 0, 0, 4095, 0, 1, 4, 0, 0, 1, 0, 0, 4095, 4094, 0, 2, 1, 4094, 4095, 0, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4084, 4096, 0, 0, 4096, 0, 0, 4096, 4090, 0, 0, 0, 65, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 32, 0, 4096, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4093, 4064, 0, 0, 4096, 0, 12, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4026, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 12, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4082, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 4093, 0, 0, 0, 2, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 4090, 0, 0, 4096, 0, 93, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4003, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4069, 4096, 0, 0, 4096, 0, 0, 4096, 4073, 2, 0, 0, 100, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 26, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 26, 0, 0, 0, 0, 4096, 4096, 0, 8, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 2, 0, 4096, 0, 11, 0, 0, 64, 4096, 4033, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3981, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 3989, 4096, 0, 0, 4096, 0, 0, 4096, 3974, 0, 0, 0, 98, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 396, 0, 0, 0, 0, 4096, 4096, 14, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4067, 0, 0, 4096, 0, 141, 0, 0, 0, 4095, 3693, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4059, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4090, 4096, 0, 0, 4096, 0, 0, 4096, 4067, 0, 0, 0, 96, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4085, 0, 0, 4096, 0, 33, 0, 0, 1, 4096, 4088, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 4096, 0, 0, 0, 4096, 3995, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 34, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0, 0, 4096, 0, 35, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4022, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 4094, 0, 1, 4091, 4095, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 1, 4095, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4095, 0, 1, 1, 0, 18, 0, 4095, 0, 0, 4095, 4015, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 4093, 0, 4, 4091, 0, 1, 2, 4093, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 2, 1, 0, 0, 1, 4095, 1, 0, 1, 0, 4095, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 4063, 1, 0, 0, 4095, 4095, 0, 0, 4095, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4087, 0, 0, 0, 0, 0, 4080, 1, 0, 0, 4095, 0, 1, 32, 17, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 4095, 4094, 0, 2, 1, 4094, 4095, 0, 1, 1, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16368, 16337, 16383, 16388, 16355, 16411, 16450] avg=16384.0000 max=16450 max_rank=7 min=16337 min_rank=2 imbalance_ratio=1.004028 max_load_reduction=114622 max_load_reduction_pct=0.874496 moved_to_echo=65604 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4089, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 4018, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 35, 17, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4082, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 16370, 16321, 16375, 16390, 16361, 16446, 16431] avg=16384.0000 max=16446 max_rank=6 min=16321 min_rank=2 imbalance_ratio=1.003784 max_load_reduction=114626 max_load_reduction_pct=0.874527 moved_to_echo=65628 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4084, 0, 0, 0, 4096, 0, 0, 4093, 4090, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 32, 0, 4096, 0, 6, 4096, 4017, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 4093, 4064, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 32, 17, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4026, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16428, 16377, 16358, 16382, 16385, 16329, 16437, 16376] avg=16384.0000 max=16437 max_rank=6 min=16329 min_rank=5 imbalance_ratio=1.003235 max_load_reduction=114635 max_load_reduction_pct=0.874596 moved_to_echo=65527 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4069, 0, 0, 0, 4096, 0, 0, 4093, 4073, 2, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 26, 4, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 26, 4096, 4017, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 4096, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 4096, 4096, 2, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 4096, 4033, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 32, 18, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3981, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16433, 16403, 16340, 16390, 16371, 16365, 16439, 16331] avg=16384.0000 max=16439 max_rank=6 min=16331 min_rank=7 imbalance_ratio=1.003357 max_load_reduction=114633 max_load_reduction_pct=0.874580 moved_to_echo=65506 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4093, 4093, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 6, 4096, 4017, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 4095, 4090, 0, 0, 4096, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 32, 17, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4003, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16377, 16322, 16382, 16383, 16438, 16437, 16353] avg=16384.0000 max=16438 max_rank=5 min=16322 min_rank=2 imbalance_ratio=1.003296 max_load_reduction=114634 max_load_reduction_pct=0.874588 moved_to_echo=65611 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 4096, 0, 0, 4093, 4067, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 4096, 4017, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 4096, 4085, 0, 0, 4096, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4088, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 32, 17, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 4096, 0, 0, 0, 4096, 3995, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16442, 16382, 16326, 16382, 16385, 16374, 16430, 16351] avg=16384.0000 max=16442 max_rank=0 min=16326 min_rank=2 imbalance_ratio=1.003540 max_load_reduction=114630 max_load_reduction_pct=0.874557 moved_to_echo=65540 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3989, 0, 0, 0, 4096, 0, 0, 4093, 3974, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 48, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 396, 4096, 4017, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 4096, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 4096, 4067, 0, 0, 4096, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3693, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 32, 18, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4059, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16250, 16420, 16714, 16396, 16385, 16464, 16035, 16408] avg=16384.0000 max=16714 max_rank=2 min=16035 min_rank=6 imbalance_ratio=1.020142 max_load_reduction=114358 max_load_reduction_pct=0.872482 moved_to_echo=65292 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 1, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 34, 4096, 4017, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 32, 17, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4022, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16375, 16354, 16382, 16385, 16386, 16437, 16372] avg=16384.0000 max=16437 max_rank=6 min=16354 min_rank=2 imbalance_ratio=1.003235 max_load_reduction=114635 max_load_reduction_pct=0.874596 moved_to_echo=65580 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 32768, 32384, 0, 0, 0, 128, 0, 32768, 128, 0, 0, 32768, 0, 128, 384, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 32640, 0, 0, 0, 32768, 0, 128, 32768, 32640, 128, 0, 0, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 32768, 128, 128, 32768, 0, 128, 128, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 0, 0, 32768, 0, 128, 128, 0, 128, 0, 32768, 0, 512, 32768, 32256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 0, 128, 32768, 128, 0, 128, 0, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 0, 32768, 32768, 0, 128, 128, 32768, 32384, 0, 128, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 32512, 128, 0, 0, 32768, 32768, 128, 0, 32768, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 0, 0, 0, 128, 0, 0, 128, 128, 128, 32768, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 212, 4096, 0, 0, 0, 3970, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 145, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 65, 3895, 0, 0, 0, 4096, 0, 4017, 0, 0, 0, 0, 0, 0, 4093, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 20, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[4094, 0, 0, 1, 4095, 1, 69, 4095, 0, 1, 0, 4089, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 1, 4095, 0, 0, 1, 1, 1, 4093, 1, 0, 0, 0, 4087, 0, 0, 0, 1, 0, 4076, 3, 1, 4095, 0, 4095, 0, 0, 1, 0, 1, 0, 4094, 0, 1, 4095, 4095, 1, 1, 4094, 4, 0, 4095, 4094, 4091, 1, 0, 0, 1, 1, 4, 0, 4092, 1, 4095, 1, 4095, 0, 1, 1, 4095, 1, 4095, 3715, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 5, 0, 1, 4095, 0, 341, 0, 0, 0, 0, 4094, 0, 1, 2, 1, 1, 60, 4032, 0, 0, 0, 4094, 0, 4093, 0, 2, 1, 1, 1, 2, 4095, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 399, 4096, 0, 0, 0, 3688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4043, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4095, 1, 0, 0, 4096, 0, 4096, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4095, 0, 27, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 103, 4088, 0, 0, 0, 4096, 0, 4062, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 154, 4096, 0, 0, 0, 3783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4065, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4061, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4084, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 0, 0, 4096, 0, 312, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 143, 3923, 0, 0, 0, 4096, 0, 4036, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 256, 4096, 0, 0, 0, 3987, 0, 0, 0, 0, 2, 0, 22, 0, 0, 0, 4096, 0, 4096, 19, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 45, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4021, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4086, 0, 0, 0, 4096, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 135, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4078, 0, 0, 0, 4096, 0, 3860, 0, 0, 0, 0, 0, 0, 4070, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 1129, 4096, 0, 0, 0, 3228, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4094, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3244, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4029, 0, 4096, 0, 0, 0, 4094, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 0, 705, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 367, 3739, 0, 0, 0, 4096, 0, 4060, 0, 0, 0, 0, 0, 0, 4072, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 111, 4096, 0, 0, 0, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4065, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 4096, 0, 115, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 46, 4095, 0, 0, 0, 4096, 0, 4043, 0, 0, 0, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 1, 4095, 1, 69, 4049, 0, 1, 0, 4089, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 1, 4095, 0, 0, 1, 1, 1, 4093, 1, 3910, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 1, 0, 4076, 3, 1, 4095, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4094, 0, 1, 4095, 134, 1, 1, 4094, 4, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4091, 1, 0, 0, 1, 1, 4, 0, 4092, 1, 4095, 1, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 4095, 3715, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 5, 44, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 341, 0, 0, 0, 0, 4094, 0, 1, 2, 1, 1, 60, 4082, 3783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 4094, 0, 4093, 0, 2, 1, 1, 1, 2, 4095, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16400, 16376, 16360, 16521, 16395, 16065, 16461, 16494] avg=16384.0000 max=16521 max_rank=3 min=16065 min_rank=5 imbalance_ratio=1.008362 max_load_reduction=114551 max_load_reduction_pct=0.873955 moved_to_echo=65415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 212, 4050, 0, 0, 0, 3970, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 3911, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 134, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 44, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 145, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 65, 4084, 3784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3895, 0, 0, 0, 4096, 0, 4017, 0, 0, 0, 0, 0, 0, 4093, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16431, 16378, 16386, 16498, 16396, 16439, 16270, 16274] avg=16384.0000 max=16498 max_rank=3 min=16270 min_rank=6 imbalance_ratio=1.006958 max_load_reduction=114574 max_load_reduction_pct=0.874130 moved_to_echo=65379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 2, 4045, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3914, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 125, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 48, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4089, 3786, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16335, 16387, 16386, 16509, 16390, 16433, 16087, 16545] avg=16384.0000 max=16545 max_rank=7 min=16087 min_rank=6 imbalance_ratio=1.009827 max_load_reduction=114527 max_load_reduction_pct=0.873772 moved_to_echo=65455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 399, 4050, 0, 0, 0, 3688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3911, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4043, 134, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 44, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 27, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 103, 4084, 3784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4096, 0, 4062, 0, 0, 0, 0, 0, 0, 4096, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16329, 16377, 16386, 16465, 16396, 16415, 16189, 16515] avg=16384.0000 max=16515 max_rank=7 min=16189 min_rank=6 imbalance_ratio=1.007996 max_load_reduction=114557 max_load_reduction_pct=0.874001 moved_to_echo=65515 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 256, 4050, 0, 0, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 22, 0, 0, 0, 4096, 0, 4096, 19, 0, 0, 0, 0, 4096, 0, 3911, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 45, 4093, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4021, 134, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 44, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 135, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4084, 3784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 4096, 0, 3860, 0, 0, 0, 0, 0, 0, 4070, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16485, 16420, 16429, 16443, 16386, 16435, 16197, 16277] avg=16384.0000 max=16485 max_rank=0 min=16197 min_rank=6 imbalance_ratio=1.006165 max_load_reduction=114587 max_load_reduction_pct=0.874229 moved_to_echo=65295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 154, 4050, 0, 0, 0, 3783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 3911, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4065, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4061, 134, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4084, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 44, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 312, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 143, 4084, 3784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 0, 0, 4096, 0, 4036, 0, 0, 0, 0, 0, 0, 4096, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16179, 16379, 16355, 16483, 16384, 16453, 16515, 16324] avg=16384.0000 max=16515 max_rank=6 min=16179 min_rank=0 imbalance_ratio=1.007996 max_load_reduction=114557 max_load_reduction_pct=0.874001 moved_to_echo=65676 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 1129, 4050, 0, 0, 0, 3228, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 3911, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4094, 1, 0, 4096, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3244, 134, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4029, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 44, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 705, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 367, 4084, 3784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3739, 0, 0, 0, 4096, 0, 4060, 0, 0, 0, 0, 0, 0, 4072, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16601, 16378, 16384, 15666, 16329, 16442, 17132, 16140] avg=16384.0000 max=17132 max_rank=6 min=15666 min_rank=3 imbalance_ratio=1.045654 max_load_reduction=113940 max_load_reduction_pct=0.869293 moved_to_echo=66043 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 111, 4050, 0, 0, 0, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3911, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4065, 134, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 44, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 115, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 46, 4084, 3784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 4043, 0, 0, 0, 0, 0, 0, 4096, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16312, 16377, 16386, 16487, 16396, 16390, 16221, 16503] avg=16384.0000 max=16503 max_rank=7 min=16221 min_rank=6 imbalance_ratio=1.007263 max_load_reduction=114569 max_load_reduction_pct=0.874092 moved_to_echo=65510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 128, 0, 32768, 0, 128, 31744, 1152, 128, 128, 32768, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 0, 0, 128, 128, 128, 0, 32768, 128, 32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 32768, 0, 1920, 0, 0, 0, 0, 32768, 0, 128, 128, 128, 128, 896, 32768, 30336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 0, 0, 128, 32768, 128, 2432, 32512, 0, 128, 0, 30848, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 128, 32768, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32000, 0, 0, 0, 32768, 0, 32384, 0, 128, 128, 128, 128, 128, 32768, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 0, 128, 0, 32768, 128, 128, 32768, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 0, 0, 32768, 128, 32768, 128, 0, 128, 128, 128, 32768, 128, 31360, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4094, 1, 0, 4095, 1, 4095, 4030, 0, 1, 0, 0, 0, 0, 1, 4095, 0, 0, 1, 4094, 4094, 1, 0, 4093, 12, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 4095, 4095, 0, 1, 0, 0, 1, 4095, 0, 0, 4094, 4095, 2, 1, 0, 0, 3, 4082, 0, 1, 4095, 4093, 0, 2, 0, 0, 0, 4095, 1, 2, 4092, 0, 1, 4092, 0, 0, 0, 1, 1, 4093, 4094, 2, 2, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 4046, 0, 1, 0, 4094, 4095, 0, 1, 4095, 4094, 0, 0, 0, 1, 1, 1, 4081, 0, 0, 4094, 4, 0, 2, 130, 0, 1, 2, 4094, 4095, 1, 1, 0, 1, 4095, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 2, 4094, 0, 4096, 4079, 0, 0, 46, 0, 2, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 4096, 0, 0, 0, 32, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 31, 0, 0, 4096, 3, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4079, 0, 0, 6, 0, 0, 4096, 0, 0, 4084, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 19, 0, 0, 0, 4065, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4090, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 4063, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 46, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 3901, 0, 0, 178, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4092, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 25, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4061, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4096, 0, 0, 0, 104, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 4096, 4096, 0, 0, 4096, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 1, 0, 4095, 1, 9, 4030, 0, 1, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 4094, 1, 0, 4093, 12, 0, 1, 0, 1, 0, 0, 0, 4053, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 4095, 4095, 0, 1, 0, 0, 1, 4095, 0, 0, 4064, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 2, 1, 0, 0, 3, 4082, 0, 1, 4095, 4093, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 2, 4092, 0, 1, 4092, 0, 0, 0, 1, 1, 4093, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 4046, 0, 1, 0, 4094, 4092, 4089, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 12, 4094, 0, 0, 0, 1, 1, 1, 4081, 0, 0, 4094, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 130, 0, 1, 2, 4094, 4095, 1, 1, 0, 1, 4059, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16327, 16386, 16384, 16379, 16382, 16349, 16384, 16481] avg=16384.0000 max=16481 max_rank=7 min=16327 min_rank=0 imbalance_ratio=1.005920 max_load_reduction=114591 max_load_reduction_pct=0.874260 moved_to_echo=65596 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 9, 4095, 0, 0, 0, 0, 31, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12, 4054, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4065, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4079, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4084, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4093, 4090, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4065, 4096, 0, 0, 0, 0, 4059, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16423, 16394, 16383, 16369, 16375, 16397, 16396, 16335] avg=16384.0000 max=16423 max_rank=0 min=16335 min_rank=7 imbalance_ratio=1.002380 max_load_reduction=114649 max_load_reduction_pct=0.874702 moved_to_echo=65503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 2, 4094, 0, 9, 4079, 0, 0, 46, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4065, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 4096, 4093, 4090, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4059, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16424, 16379, 16383, 16385, 16385, 16371, 16396, 16349] avg=16384.0000 max=16424 max_rank=0 min=16349 min_rank=7 imbalance_ratio=1.002441 max_load_reduction=114648 max_load_reduction_pct=0.874695 moved_to_echo=65501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4070, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 4096, 4096, 4096, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 4058, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16349, 16382, 16390, 16382, 16384, 16369, 16372, 16444] avg=16384.0000 max=16444 max_rank=7 min=16349 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=114628 max_load_reduction_pct=0.874542 moved_to_echo=65569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 9, 3901, 0, 0, 178, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4065, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4093, 4090, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 4059, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16376, 16380, 16383, 16387, 16387, 16397, 16391, 16371] avg=16384.0000 max=16397 max_rank=5 min=16371 min_rank=7 imbalance_ratio=1.000793 max_load_reduction=114675 max_load_reduction_pct=0.874901 moved_to_echo=65546 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 9, 4090, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 4054, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4065, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 4095, 4090, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 4059, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16382, 16383, 16390, 16385, 16395, 16360, 16387] avg=16384.0000 max=16395 max_rank=5 min=16360 min_rank=6 imbalance_ratio=1.000671 max_load_reduction=114677 max_load_reduction_pct=0.874916 moved_to_echo=65527 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 9, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4065, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4093, 4090, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 4059, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16383, 16383, 16390, 16387, 16398, 16377, 16364] avg=16384.0000 max=16398 max_rank=5 min=16364 min_rank=7 imbalance_ratio=1.000854 max_load_reduction=114674 max_load_reduction_pct=0.874893 moved_to_echo=65526 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 4096, 0, 0, 4096, 1, 0, 0, 1, 0, 0, 0, 0, 4054, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4065, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4093, 4090, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 4059, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16393, 16386, 16383, 16390, 16387, 16396, 16396, 16341] avg=16384.0000 max=16396 max_rank=5 min=16341 min_rank=7 imbalance_ratio=1.000732 max_load_reduction=114676 max_load_reduction_pct=0.874908 moved_to_echo=65520 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 32768, 32768, 128, 0, 32768, 128, 0, 128, 128, 128, 0, 0, 128, 32512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 128, 128, 32768, 128, 128, 32512, 0, 128, 256, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 128, 128, 32768, 0, 0, 0, 128, 128, 128, 32768, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 384, 0, 128, 128, 32768, 32768, 128, 128, 0, 128, 32512, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 128, 32768, 0, 128, 32768, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 0, 32768, 32768, 0, 128, 0, 0, 128, 32768, 0, 0, 32640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 128, 32768, 0, 128, 0, 32768, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 128, 32768, 0, 0, 0, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4076, 1, 9, 0, 3965, 0, 4095, 0, 4094, 4095, 0, 0, 1, 2, 4095, 4095, 0, 0, 4087, 1, 1, 0, 0, 0, 3, 1, 1, 0, 4095, 2, 4093, 271, 0, 0, 4095, 0, 1, 4094, 0, 4094, 0, 1, 0, 1, 0, 61, 4093, 0, 1, 4094, 0, 3810, 2, 4, 0, 4095, 0, 4093, 0, 4093, 1, 0, 0, 4095, 0, 0, 0, 1, 0, 2, 1, 0, 4095, 0, 6, 0, 0, 0, 0, 4094, 0, 0, 4088, 4095, 0, 0, 0, 2, 0, 0, 0, 4093, 4092, 1, 0, 0, 1, 1, 2, 1, 0, 1, 1, 1, 0, 2, 0, 1, 1, 1, 0, 99, 1, 2, 4095, 0, 0, 4095, 4095, 4094, 1, 4095, 1, 4094, 1, 3, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 10, 0, 4086, 0, 4096, 0, 4096, 4096, 1, 0, 0, 0, 4096, 4096, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 10, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 2, 4, 4096, 0, 0, 4096, 0, 4092, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 4096, 1, 1, 17, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 4095, 4096, 0, 4087, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 16, 0, 4088, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 33, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 206, 4096, 0, 0, 4096, 0, 3898, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 4094, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 2, 0, 4055, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4071, 162, 6, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4094, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3970, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4095, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 73, 0, 4031, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3978, 11, 33, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 7, 122, 4096, 0, 0, 4096, 0, 4030, 0, 1, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 60, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4037, 4096, 0, 0, 45, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4065, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4095, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3948, 1, 29, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 3, 1, 4096, 0, 0, 4096, 0, 4076, 0, 23, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 101, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 4096, 2, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 4073, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 4096, 0, 0, 4096, 4093, 4096, 0, 4069, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3868, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 196, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 90, 4096, 0, 0, 4096, 0, 4042, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 3, 0, 4010, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 5, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 4096, 0, 4089, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4094, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4076, 1, 9, 0, 3965, 0, 0, 0, 4094, 0, 0, 0, 1, 2, 4095, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 1, 0, 0, 0, 3, 1, 1, 0, 4095, 2, 4093, 271, 4018, 51, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 1, 4094, 0, 4094, 0, 1, 0, 1, 0, 61, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 3810, 2, 4, 0, 0, 0, 4093, 0, 4093, 1, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 0, 4095, 0, 6, 0, 0, 0, 0, 4094, 4085, 4027, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4095, 0, 0, 0, 2, 0, 0, 0, 4093, 4092, 1, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 1, 0, 1, 1, 1, 0, 2, 0, 1, 1, 1, 0, 99, 4095, 4095, 4095, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 4095, 4095, 4094, 1, 4095, 1, 9, 1, 3, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16303, 16632, 16366, 16175, 16364, 16394, 16439, 16399] avg=16384.0000 max=16632 max_rank=1 min=16175 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=114440 max_load_reduction_pct=0.873108 moved_to_echo=65596 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 16, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 4019, 51, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 4022, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 206, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3898, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4087, 4028, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4094, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16452, 16325, 16549, 16263, 16361, 16397, 16332, 16393] avg=16384.0000 max=16549 max_rank=2 min=16263 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=114523 max_load_reduction_pct=0.873741 moved_to_echo=65483 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 10, 0, 4086, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 10, 4019, 51, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 2, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4087, 4028, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 1, 1, 17, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 4096, 0, 4087, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16445, 16355, 16316, 16457, 16360, 16427, 16331, 16381] avg=16384.0000 max=16457 max_rank=3 min=16316 min_rank=2 imbalance_ratio=1.004456 max_load_reduction=114615 max_load_reduction_pct=0.874443 moved_to_echo=65499 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3868, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 196, 4022, 53, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 90, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4042, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4089, 4031, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 4096, 4096, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16211, 16569, 16395, 16404, 16367, 16393, 16342, 16391] avg=16384.0000 max=16569 max_rank=1 min=16211 min_rank=0 imbalance_ratio=1.011292 max_load_reduction=114503 max_load_reduction_pct=0.873589 moved_to_echo=65493 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3948, 1, 4019, 51, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 4022, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 3, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 0, 23, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 4096, 4087, 4028, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 4073, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 4096, 4096, 4096, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 4096, 0, 4069, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16443, 16097, 16343, 16464, 16465, 16387, 16510, 16363] avg=16384.0000 max=16510 max_rank=6 min=16097 min_rank=1 imbalance_ratio=1.007690 max_load_reduction=114562 max_load_reduction_pct=0.874039 moved_to_echo=65725 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 2, 0, 4055, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4071, 162, 4019, 51, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4022, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4087, 4028, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3970, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4096, 4096, 4096, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4095, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16405, 16501, 16316, 16459, 16363, 16274, 16362, 16392] avg=16384.0000 max=16501 max_rank=1 min=16274 min_rank=5 imbalance_ratio=1.007141 max_load_reduction=114571 max_load_reduction_pct=0.874107 moved_to_echo=65391 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 73, 0, 4031, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3978, 11, 4019, 51, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 4022, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 7, 122, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4030, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4087, 4028, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 4096, 0, 0, 45, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 4096, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4065, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16452, 16245, 16472, 16396, 16421, 16393, 16331, 16362] avg=16384.0000 max=16472 max_rank=2 min=16245 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=114600 max_load_reduction_pct=0.874329 moved_to_echo=65507 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 3, 0, 4010, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 5, 4019, 51, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4087, 4028, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 4096, 4096, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4094, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16361, 16348, 16315, 16454, 16371, 16407, 16425, 16391] avg=16384.0000 max=16454 max_rank=3 min=16315 min_rank=2 imbalance_ratio=1.004272 max_load_reduction=114618 max_load_reduction_pct=0.874466 moved_to_echo=65594 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 128, 128, 0, 32256, 0, 0, 0, 32768, 0, 128, 0, 128, 128, 32768, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 32768, 32640, 128, 128, 128, 128, 0, 0, 0, 32768, 32768, 128, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 32768, 0, 32256, 128, 128, 0, 0, 0, 32768, 0, 32768, 128, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 128, 0, 128, 0, 128, 128, 128, 0, 512, 32768, 32768, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 32256, 0, 128, 32768, 0, 32768, 0, 128, 0, 128, 128, 512, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 32640, 128, 128, 0, 0, 0, 128, 128, 128, 0, 32768, 128, 32512, 768, 32256, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 0, 0, 32768, 32768, 32768, 128, 32768, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 0, 128, 0, 256, 128, 0, 32768, 0, 128, 0, 0, 0, 0, 32768, 32768, 32256, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True + [2026-08-30 05:03:34] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 789.1 | throughput per GPU (TFLOP/s/GPU): 422.5 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.033579E+01 | loss scale: 1.0 | grad norm: 17.795 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2, 179, 818, 0, 390, 0, 908, 0, 1896, 82, 186, 2259, 785, 789, 0, 1444, 771, 536, 344, 593, 3966, 2857, 148, 49, 1, 1843, 201, 215, 880, 0, 1003, 1282, 4096, 5, 14, 729, 2928, 14, 625, 2522, 2682, 975, 231, 115, 0, 634, 305, 1029, 1073, 1569, 1384, 3, 0, 0, 3663, 161, 0, 3224, 3339, 2406, 181, 298, 421, 0, 843, 4050, 0, 932, 108, 11, 2168, 1673, 1466, 464, 67, 0, 83, 721, 757, 403, 1937, 1070, 1, 3901, 112, 2717, 2422, 3980, 47, 2, 997, 984, 3140, 316, 2761, 0, 1435, 1299, 3066, 214, 0, 2015, 608, 2343, 3973, 1454, 4, 754, 1163, 1431, 92, 818, 61, 462, 2760, 830, 0, 821, 200, 1137, 803, 1106, 1746, 1574, 95, 0, 1621, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[4, 440, 1026, 1, 428, 4, 700, 17, 2308, 38, 269, 2030, 1196, 520, 0, 1618, 804, 753, 887, 1272, 3702, 2166, 217, 69, 11, 1688, 339, 171, 753, 14, 901, 1465, 4040, 0, 24, 773, 2550, 18, 1228, 1910, 3127, 1487, 277, 169, 1, 749, 509, 861, 811, 1152, 770, 128, 8, 4, 3250, 240, 0, 2489, 3231, 1969, 549, 362, 819, 8, 1092, 4008, 1, 546, 293, 40, 1577, 2321, 1542, 704, 219, 0, 187, 459, 417, 703, 1389, 1274, 15, 3767, 62, 1999, 2986, 3889, 134, 11, 714, 1274, 2770, 276, 2292, 6, 1058, 1516, 3805, 104, 0, 1954, 776, 1691, 3422, 1633, 1, 1411, 1604, 1566, 176, 594, 19, 443, 2626, 581, 40, 915, 120, 1906, 930, 1000, 1482, 1552, 54, 1, 1801, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[8, 410, 1161, 0, 338, 0, 1011, 6, 2162, 150, 452, 2557, 1236, 525, 2, 1041, 447, 1046, 706, 743, 3856, 1938, 158, 44, 6, 1504, 236, 345, 1299, 8, 923, 1283, 4087, 0, 24, 1382, 2977, 18, 599, 2865, 2539, 1425, 309, 402, 0, 437, 459, 663, 1024, 1468, 1114, 53, 3, 6, 3393, 465, 2, 3015, 3163, 2582, 651, 354, 755, 0, 1169, 4061, 0, 1231, 103, 22, 1853, 1707, 1024, 549, 127, 2, 62, 623, 284, 485, 1904, 2070, 5, 3732, 30, 2073, 2312, 3855, 47, 14, 700, 1309, 2840, 356, 2641, 1, 675, 2034, 3274, 179, 3, 2162, 688, 1821, 3537, 1237, 6, 797, 954, 1321, 96, 725, 53, 867, 2100, 821, 5, 499, 294, 1246, 861, 677, 1474, 1387, 94, 12, 2145, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[0, 170, 832, 3, 289, 0, 641, 0, 2377, 169, 231, 2500, 907, 569, 0, 1710, 778, 956, 619, 821, 3930, 2611, 148, 13, 0, 1645, 242, 126, 987, 7, 797, 1434, 4074, 4, 7, 1033, 3156, 22, 353, 2451, 2279, 1267, 111, 131, 0, 772, 490, 506, 855, 1520, 1028, 25, 5, 5, 3836, 289, 0, 2998, 3466, 2606, 513, 264, 657, 6, 1053, 4078, 0, 870, 212, 12, 1549, 2390, 1605, 722, 158, 1, 59, 414, 369, 368, 1861, 983, 6, 3885, 32, 2229, 1978, 4035, 32, 7, 455, 2013, 3229, 313, 2941, 4, 841, 1222, 3736, 202, 0, 2376, 519, 2204, 3598, 1205, 4, 811, 1264, 1882, 165, 814, 74, 426, 2183, 758, 13, 590, 285, 1349, 756, 923, 1567, 1182, 38, 3, 1972, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[18, 263, 1054, 0, 337, 2, 677, 1, 2446, 47, 258, 2064, 1125, 314, 2, 1281, 813, 450, 906, 622, 3885, 1773, 181, 35, 1, 1519, 199, 312, 1428, 17, 749, 1346, 4009, 1, 27, 1070, 3450, 20, 635, 2647, 3060, 1341, 221, 217, 6, 662, 460, 1008, 1008, 1732, 986, 68, 8, 8, 3563, 193, 0, 2938, 3434, 2288, 709, 268, 797, 0, 1231, 4036, 0, 859, 172, 137, 1756, 1534, 1455, 506, 196, 13, 50, 370, 261, 360, 1351, 1940, 0, 3678, 105, 2307, 2346, 4000, 59, 6, 577, 1156, 2806, 133, 2229, 0, 1129, 1518, 3526, 184, 1, 2243, 755, 2230, 3883, 1453, 1, 1011, 1038, 1836, 44, 924, 69, 595, 2449, 650, 9, 547, 218, 1132, 841, 1201, 1323, 1568, 107, 1, 1996, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[4, 266, 695, 0, 375, 0, 875, 1, 2527, 103, 408, 2191, 1364, 551, 0, 1273, 421, 479, 457, 938, 3980, 2768, 156, 35, 1, 1642, 234, 58, 1090, 1, 1150, 1216, 4085, 0, 13, 1452, 2883, 16, 569, 2289, 2621, 1032, 292, 148, 0, 624, 255, 375, 963, 1388, 1543, 3, 2, 1, 3669, 270, 0, 2876, 3737, 2358, 579, 219, 710, 0, 710, 4089, 0, 1348, 113, 13, 1676, 1341, 1299, 651, 131, 1, 39, 504, 216, 1240, 2019, 1674, 6, 3870, 42, 1792, 2705, 3968, 53, 9, 888, 814, 3259, 584, 2200, 0, 873, 1748, 3902, 185, 0, 2272, 757, 2120, 3851, 1221, 0, 591, 1222, 2377, 73, 520, 92, 479, 2755, 605, 0, 426, 168, 1502, 470, 860, 849, 1509, 75, 0, 2085, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[14, 292, 791, 0, 317, 0, 571, 4, 2492, 68, 202, 2521, 1580, 191, 4, 1011, 569, 349, 517, 672, 4013, 2759, 131, 23, 1, 1694, 243, 72, 671, 7, 1077, 1346, 4094, 5, 14, 947, 3246, 19, 506, 2055, 2880, 1202, 160, 180, 6, 605, 410, 946, 828, 1386, 1172, 60, 0, 3, 3552, 256, 0, 3178, 3563, 2485, 292, 546, 444, 0, 930, 4061, 0, 956, 146, 24, 2886, 2155, 1777, 565, 68, 1, 43, 329, 466, 314, 1947, 1471, 3, 3995, 37, 2617, 2016, 3994, 49, 7, 578, 1468, 3168, 364, 2552, 0, 775, 1540, 3867, 102, 1, 1741, 445, 1825, 3265, 1798, 1, 838, 1546, 1712, 171, 791, 46, 721, 2209, 342, 6, 524, 186, 1177, 981, 1081, 1488, 1745, 52, 0, 1865, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[12, 238, 559, 0, 267, 0, 854, 1, 2375, 90, 452, 2504, 892, 293, 1, 878, 710, 519, 465, 588, 3932, 2168, 149, 13, 0, 1728, 311, 284, 1338, 10, 998, 1360, 4082, 1, 7, 1720, 2863, 13, 472, 2482, 3258, 1091, 232, 137, 4, 552, 381, 1032, 1013, 1317, 1222, 5, 6, 1, 3174, 197, 0, 2916, 3320, 2128, 516, 262, 901, 1, 541, 4006, 2, 953, 180, 43, 1738, 2317, 1531, 589, 176, 8, 47, 539, 357, 510, 1853, 991, 9, 3743, 37, 2816, 2298, 3991, 48, 30, 702, 1116, 2974, 433, 2082, 0, 951, 1576, 3720, 197, 0, 2533, 611, 2364, 3647, 1600, 3, 1122, 1253, 1449, 146, 702, 105, 592, 2946, 666, 20, 747, 271, 867, 1108, 1046, 1512, 1398, 57, 0, 1907, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 179, 818, 0, 390, 0, 908, 0, 1896, 82, 186, 2259, 785, 789, 0, 1444, 3980, 2392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 771, 536, 344, 593, 3966, 2857, 148, 49, 1, 1843, 201, 215, 880, 0, 1003, 1282, 834, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2904, 5, 14, 729, 2928, 14, 625, 2522, 2682, 975, 231, 115, 0, 634, 305, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1073, 1569, 1384, 3, 0, 0, 2409, 161, 0, 3224, 3339, 2406, 181, 298, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 843, 4050, 0, 932, 108, 11, 2168, 1673, 1466, 464, 67, 0, 83, 721, 757, 403, 1254, 1078, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1937, 1070, 1, 544, 112, 2717, 2422, 0, 47, 2, 997, 984, 3140, 316, 2761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1435, 1299, 2232, 214, 0, 2015, 608, 2343, 0, 1454, 4, 754, 1163, 1431, 92, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 462, 2760, 830, 0, 821, 200, 1137, 803, 1106, 1746, 1574, 95, 0, 1621, 1, 2279, 1581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16110, 16286, 15712, 16468, 16507, 17050, 15862, 17077] avg=16384.0000 max=17077 max_rank=7 min=15712 min_rank=2 imbalance_ratio=1.042297 max_load_reduction=113995 max_load_reduction_pct=0.869713 moved_to_echo=66496 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 440, 1026, 1, 428, 4, 700, 17, 2308, 38, 269, 2030, 1196, 520, 0, 1618, 3889, 2061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 753, 887, 1272, 3702, 2166, 217, 69, 11, 1688, 339, 171, 753, 14, 901, 1465, 1035, 753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2864, 0, 24, 773, 2550, 18, 1228, 1910, 3127, 1487, 277, 169, 1, 749, 509, 861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 811, 1152, 770, 128, 8, 4, 2138, 240, 0, 2489, 3231, 1969, 549, 362, 819, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1092, 4008, 1, 546, 293, 40, 1577, 2321, 1542, 704, 219, 0, 187, 459, 417, 703, 1112, 1041, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1389, 1274, 15, 525, 62, 1999, 2986, 0, 134, 11, 714, 1274, 2770, 276, 2292, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 1516, 2770, 104, 0, 1954, 776, 1691, 0, 1633, 1, 1411, 1604, 1566, 176, 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 443, 2626, 581, 40, 915, 120, 1906, 930, 1000, 1482, 1552, 54, 1, 1801, 0, 2201, 1361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16549, 17000, 16547, 14678, 16685, 15727, 16854, 17032] avg=16384.0000 max=17032 max_rank=7 min=14678 min_rank=3 imbalance_ratio=1.039551 max_load_reduction=114040 max_load_reduction_pct=0.870056 moved_to_echo=66298 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[8, 410, 1161, 0, 338, 0, 1011, 6, 2162, 150, 452, 2557, 1236, 525, 2, 1041, 3855, 2130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 447, 1046, 706, 743, 3856, 1938, 158, 44, 6, 1504, 236, 345, 1299, 8, 923, 1283, 891, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2897, 0, 24, 1382, 2977, 18, 599, 2865, 2539, 1425, 309, 402, 0, 437, 459, 663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1468, 1114, 53, 3, 6, 2232, 465, 2, 3015, 3163, 2582, 651, 354, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1169, 4061, 0, 1231, 103, 22, 1853, 1707, 1024, 549, 127, 2, 62, 623, 284, 485, 1161, 1031, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1904, 2070, 5, 520, 30, 2073, 2312, 0, 47, 14, 700, 1309, 2840, 356, 2641, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 675, 2034, 2383, 179, 3, 2162, 688, 1821, 0, 1237, 6, 797, 954, 1321, 96, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 867, 2100, 821, 5, 499, 294, 1246, 861, 677, 1474, 1387, 94, 12, 2145, 2, 2181, 1407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17044, 16195, 16996, 16887, 15922, 16822, 15081, 16125] avg=16384.0000 max=17044 max_rank=0 min=15081 min_rank=6 imbalance_ratio=1.040283 max_load_reduction=114028 max_load_reduction_pct=0.869965 moved_to_echo=63950 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[0, 170, 832, 3, 289, 0, 641, 0, 2377, 169, 231, 2500, 907, 569, 0, 1710, 4035, 2167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 778, 956, 619, 821, 3930, 2611, 148, 13, 0, 1645, 242, 126, 987, 7, 797, 1434, 1020, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2882, 4, 7, 1033, 3156, 22, 353, 2451, 2279, 1267, 111, 131, 0, 772, 490, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 855, 1520, 1028, 25, 5, 5, 2518, 289, 0, 2998, 3466, 2606, 513, 264, 657, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 4078, 0, 870, 212, 12, 1549, 2390, 1605, 722, 158, 1, 59, 414, 369, 368, 1318, 1079, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1861, 983, 6, 532, 32, 2229, 1978, 0, 32, 7, 455, 2013, 3229, 313, 2941, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 841, 1222, 2716, 202, 0, 2376, 519, 2204, 0, 1205, 4, 811, 1264, 1882, 165, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 426, 2183, 758, 13, 590, 285, 1349, 756, 923, 1567, 1182, 38, 3, 1972, 6, 2274, 1431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16600, 16896, 15464, 16755, 16687, 16615, 16225, 15830] avg=16384.0000 max=16896 max_rank=1 min=15464 min_rank=2 imbalance_ratio=1.031250 max_load_reduction=114176 max_load_reduction_pct=0.871094 moved_to_echo=65357 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[4, 266, 695, 0, 375, 0, 875, 1, 2527, 103, 408, 2191, 1364, 551, 0, 1273, 3968, 2319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 479, 457, 938, 3980, 2768, 156, 35, 1, 1642, 234, 58, 1090, 1, 1150, 1216, 1061, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2896, 0, 13, 1452, 2883, 16, 569, 2289, 2621, 1032, 292, 148, 0, 624, 255, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 963, 1388, 1543, 3, 2, 1, 2413, 270, 0, 2876, 3737, 2358, 579, 219, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 710, 4089, 0, 1348, 113, 13, 1676, 1341, 1299, 651, 131, 1, 39, 504, 216, 1240, 1256, 1069, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2019, 1674, 6, 540, 42, 1792, 2705, 0, 53, 9, 888, 814, 3259, 584, 2200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 873, 1748, 2841, 185, 0, 2272, 757, 2120, 0, 1221, 0, 591, 1222, 2377, 73, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 479, 2755, 605, 0, 426, 168, 1502, 470, 860, 849, 1509, 75, 0, 2085, 0, 2261, 1532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16920, 16448, 15465, 17062, 16124, 16585, 16800, 15668] avg=16384.0000 max=17062 max_rank=3 min=15465 min_rank=2 imbalance_ratio=1.041382 max_load_reduction=114010 max_load_reduction_pct=0.869827 moved_to_echo=65177 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[18, 263, 1054, 0, 337, 2, 677, 1, 2446, 47, 258, 2064, 1125, 314, 2, 1281, 4000, 2339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 813, 450, 906, 622, 3885, 1773, 181, 35, 1, 1519, 199, 312, 1428, 17, 749, 1346, 959, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2842, 1, 27, 1070, 3450, 20, 635, 2647, 3060, 1341, 221, 217, 6, 662, 460, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 1732, 986, 68, 8, 8, 2343, 193, 0, 2938, 3434, 2288, 709, 268, 797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1231, 4036, 0, 859, 172, 137, 1756, 1534, 1455, 506, 196, 13, 50, 370, 261, 360, 1220, 1016, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1351, 1940, 0, 513, 105, 2307, 2346, 0, 59, 6, 577, 1156, 2806, 133, 2229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1129, 1518, 2567, 184, 1, 2243, 755, 2230, 0, 1453, 1, 1011, 1038, 1836, 44, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 595, 2449, 650, 9, 547, 218, 1132, 841, 1201, 1323, 1568, 107, 1, 1996, 2, 2149, 1544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16228, 15942, 17667, 16780, 15592, 15528, 16934, 16401] avg=16384.0000 max=17667 max_rank=2 min=15528 min_rank=5 imbalance_ratio=1.078308 max_load_reduction=113405 max_load_reduction_pct=0.865211 moved_to_echo=64455 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[14, 292, 791, 0, 317, 0, 571, 4, 2492, 68, 202, 2521, 1580, 191, 4, 1011, 3994, 1966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 569, 349, 517, 672, 4013, 2759, 131, 23, 1, 1694, 243, 72, 671, 7, 1077, 1346, 1052, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2902, 5, 14, 947, 3246, 19, 506, 2055, 2880, 1202, 160, 180, 6, 605, 410, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 1386, 1172, 60, 0, 3, 2336, 256, 0, 3178, 3563, 2485, 292, 546, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 930, 4061, 0, 956, 146, 24, 2886, 2155, 1777, 565, 68, 1, 43, 329, 466, 314, 1216, 1104, 429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1947, 1471, 3, 557, 37, 2617, 2016, 0, 49, 7, 578, 1468, 3168, 364, 2552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 775, 1540, 2815, 102, 1, 1741, 445, 1825, 0, 1798, 1, 838, 1546, 1712, 171, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 721, 2209, 342, 6, 524, 186, 1177, 981, 1081, 1488, 1745, 52, 0, 1865, 2, 2334, 1299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16018, 15959, 16083, 16549, 17470, 16834, 16101, 16058] avg=16384.0000 max=17470 max_rank=4 min=15959 min_rank=1 imbalance_ratio=1.066284 max_load_reduction=113602 max_load_reduction_pct=0.866714 moved_to_echo=66463 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[12, 238, 559, 0, 267, 0, 854, 1, 2375, 90, 452, 2504, 892, 293, 1, 878, 3991, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 710, 519, 465, 588, 3932, 2168, 149, 13, 0, 1728, 311, 284, 1338, 10, 998, 1360, 1012, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2893, 1, 7, 1720, 2863, 13, 472, 2482, 3258, 1091, 232, 137, 4, 552, 381, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 1317, 1222, 5, 6, 1, 2088, 197, 0, 2916, 3320, 2128, 516, 262, 901, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 4006, 2, 953, 180, 43, 1738, 2317, 1531, 589, 176, 8, 47, 539, 357, 510, 1086, 1034, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 991, 9, 522, 37, 2816, 2298, 0, 48, 30, 702, 1116, 2974, 433, 2082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 1576, 2708, 197, 0, 2533, 611, 2364, 0, 1600, 3, 1122, 1253, 1449, 146, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 592, 2946, 666, 20, 747, 271, 867, 1108, 1046, 1512, 1398, 57, 0, 1907, 1, 2187, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15603, 16346, 17138, 15893, 16085, 15911, 17215, 16881] avg=16384.0000 max=17215 max_rank=6 min=15603 min_rank=0 imbalance_ratio=1.050720 max_load_reduction=113857 max_load_reduction_pct=0.868660 moved_to_echo=66092 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[23168, 128, 256, 9216, 24064, 256, 4992, 19328, 22528, 9856, 1920, 1536, 128, 5120, 3328, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[7808, 12544, 21120, 1408, 128, 17408, 5248, 16640, 0, 11648, 128, 7424, 10112, 13696, 1024, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 2304, 7040, 128, 2816, 128, 6272, 128, 18688, 768, 2560, 18688, 9088, 3840, 128, 10368, 31744, 17664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[640, 4608, 20096, 5376, 128, 5120, 1792, 10368, 6784, 7936, 11520, 12032, 640, 128, 15488, 128, 17920, 11648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[7680, 32512, 128, 7808, 1408, 384, 15232, 15488, 11776, 4864, 1152, 128, 640, 3968, 3200, 4480, 9728, 8576, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[5376, 5120, 4992, 6272, 31360, 19072, 1408, 384, 128, 13312, 2048, 1664, 8448, 128, 7680, 10752, 7936, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[14336, 11520, 128, 4352, 512, 18560, 19072, 0, 512, 128, 5632, 10240, 24192, 2816, 19712, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[7680, 11648, 9344, 384, 128, 128, 18560, 2176, 128, 23680, 27264, 18944, 4096, 2688, 5504, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 1079, 0, 0, 322, 3917, 4096, 2787, 4094, 0, 0, 89, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4092, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3994, 0, 0, 3867, 0, 0, 0, 4069, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 653, 12, 0, 0, 0, 0, 0, 0, 0, 3868, 4089, 0, 1109, 4096, 4096, 0, 0, 0, 0, 1735, 0, 0, 1, 48, 0, 4075, 292, 0, 4053, 0, 0, 0, 0, 0, 0, 4024, 4095, 0, 4096, 0, 1814, 1, 0, 0, 0, 4094, 4096, 0, 1794, 13, 4, 0, 0, 1448, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4093, 228, 0, 0, 34, 3833, 4096, 3604, 4096, 0, 0, 282, 0, 0, 4082, 0, 0, 19, 33, 0, 0, 4079, 0, 3, 4083, 0, 2, 0, 0, 4096, 0, 4094, 0, 1, 4084, 0, 0, 3, 26, 4088, 4096, 0, 4, 0, 3574, 2, 13, 4062, 0, 0, 0, 4091, 1, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1854, 0, 2, 0, 0, 0, 0, 0, 0, 4055, 4082, 0, 1545, 4096, 4095, 1, 0, 0, 0, 270, 0, 3, 10, 8, 0, 4084, 115, 0, 4044, 0, 0, 0, 0, 3, 0, 4095, 4041, 0, 4096, 0, 160, 1, 0, 12, 0, 3946, 4096, 0, 3026, 110, 0, 0, 0, 2128, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 1388, 0, 0, 199, 3999, 4096, 3621, 4096, 0, 0, 83, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 1, 4090, 0, 0, 0, 0, 4093, 0, 4096, 0, 2, 4094, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4093, 0, 0, 3851, 0, 0, 2, 4038, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2822, 7, 0, 0, 0, 0, 0, 0, 0, 3500, 4095, 0, 2520, 4096, 4096, 0, 0, 0, 0, 1254, 0, 0, 238, 3, 0, 4096, 148, 0, 4049, 0, 0, 0, 0, 0, 0, 3940, 3961, 0, 4096, 0, 260, 8, 0, 0, 0, 4082, 4096, 4, 399, 41, 7, 0, 0, 643, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 3, 4092, 4091, 1702, 0, 20, 1, 4075, 4093, 3790, 4069, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 4092, 0, 5, 4089, 1, 5, 0, 0, 4096, 0, 4092, 0, 0, 4074, 29, 0, 0, 0, 4091, 4078, 0, 0, 0, 4027, 4, 0, 4086, 0, 0, 1, 4085, 0, 4094, 0, 0, 1, 4090, 5, 4, 0, 0, 0, 0, 0, 1, 0, 0, 39, 279, 0, 0, 0, 4, 0, 7, 3, 0, 3850, 4085, 5, 2419, 4092, 4096, 6, 0, 1, 0, 1091, 0, 0, 648, 0, 0, 4096, 234, 12, 3651, 4, 4, 0, 0, 24, 4, 3981, 4049, 0, 4087, 0, 158, 7, 0, 0, 0, 4084, 4092, 10, 2244, 11, 1, 25, 2, 583, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 4080, 4093, 316, 0, 0, 46, 4092, 4094, 4025, 4089, 0, 0, 241, 0, 0, 4084, 0, 1, 30, 0, 0, 0, 4093, 1, 1, 4089, 0, 1, 6, 1, 4092, 0, 4092, 3, 0, 4085, 0, 0, 2, 0, 4079, 4091, 0, 0, 0, 4007, 0, 26, 3979, 0, 2, 5, 4087, 4, 4095, 1, 1, 0, 4096, 1, 0, 0, 0, 1, 1, 0, 0, 3, 1, 0, 2221, 0, 0, 1, 0, 0, 0, 0, 0, 4078, 4096, 0, 250, 4095, 4092, 1, 0, 3, 2, 188, 4, 5, 193, 1, 1, 4049, 1337, 0, 4041, 11, 0, 0, 0, 0, 1, 3872, 3975, 4, 4095, 1, 124, 0, 4, 8, 0, 4080, 4095, 0, 2641, 43, 33, 3, 0, 1285, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4095, 4078, 1285, 4, 28, 1, 4027, 4066, 3246, 4092, 0, 6, 141, 3, 0, 4078, 1, 0, 1, 6, 0, 0, 4092, 2, 0, 4088, 1, 0, 1, 0, 4095, 0, 4085, 0, 0, 4021, 0, 2, 0, 52, 4067, 4089, 0, 0, 3, 3881, 0, 27, 3922, 0, 0, 0, 4094, 0, 4092, 0, 2, 0, 4095, 3, 13, 2, 0, 0, 0, 0, 1, 0, 1, 0, 1772, 50, 1, 4, 3, 1, 0, 0, 0, 3829, 4075, 0, 3058, 4091, 4086, 1, 0, 4, 3, 294, 0, 0, 825, 0, 0, 4086, 376, 2, 3968, 9, 4, 3, 0, 30, 7, 3970, 4041, 1, 4093, 0, 10, 1, 0, 0, 0, 4088, 4096, 0, 1602, 152, 0, 18, 14, 279, 3, 233] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 4095, 4093, 171, 1, 13, 37, 3752, 4094, 3936, 4095, 0, 0, 148, 0, 1, 4085, 0, 0, 0, 0, 0, 6, 4095, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 4093, 0, 0, 4056, 1, 0, 7, 2, 4075, 4090, 0, 2, 0, 4020, 3, 0, 4024, 1, 0, 0, 4083, 0, 4095, 1, 3, 0, 4096, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 3, 3811, 11, 0, 0, 0, 1, 0, 3, 1, 3952, 4087, 3, 640, 4093, 4092, 0, 0, 0, 0, 581, 0, 0, 874, 0, 1, 4065, 85, 0, 3580, 1, 0, 1, 0, 2, 0, 4080, 4076, 0, 4095, 0, 73, 8, 0, 1, 3, 4074, 4095, 0, 1776, 166, 23, 0, 0, 1237, 0, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 3, 4096, 4088, 787, 0, 4, 458, 3980, 4096, 3338, 4091, 0, 0, 534, 0, 0, 4096, 1, 0, 9, 0, 0, 3, 4096, 0, 0, 4094, 0, 0, 0, 0, 4094, 2, 4094, 0, 4, 4096, 0, 2, 6, 4, 4096, 4074, 0, 0, 0, 4082, 0, 0, 4070, 0, 0, 0, 4087, 1, 4093, 0, 0, 1, 4095, 2, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 3261, 1, 0, 0, 0, 0, 3, 0, 0, 3241, 3954, 0, 475, 4096, 4095, 7, 9, 0, 0, 277, 0, 0, 165, 0, 0, 3937, 265, 0, 4050, 1, 0, 0, 1, 29, 0, 4073, 4079, 3, 4094, 0, 51, 4, 0, 14, 0, 3883, 4095, 0, 1770, 1, 6, 38, 0, 2365, 0, 43] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3586, 316, 0, 0, 46, 4092, 0, 4025, 4089, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 1, 30, 0, 0, 0, 4093, 1, 1, 4089, 0, 1, 6, 1, 2268, 1406, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 3, 0, 4085, 0, 0, 2, 0, 4079, 4091, 0, 0, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 3979, 0, 2, 5, 4087, 4, 4095, 1, 1, 0, 4096, 1, 0, 0, 88, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 3, 1, 0, 2221, 0, 0, 1, 0, 0, 0, 0, 4092, 4095, 4080, 1997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 0, 250, 1827, 4092, 1, 0, 3, 2, 188, 4, 5, 193, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4022, 1337, 0, 4041, 11, 0, 0, 0, 0, 1, 3872, 3975, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 4, 8, 0, 4080, 4095, 0, 2641, 43, 33, 3, 0, 1285, 1, 0, 2531, 2097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16396, 16400, 16422, 16412, 16492, 14740, 17265, 16945] avg=16384.0000 max=17265 max_rank=6 min=14740 min_rank=5 imbalance_ratio=1.053772 max_load_reduction=113807 max_load_reduction_pct=0.868279 moved_to_echo=65442 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3589, 1079, 0, 0, 322, 3917, 0, 2787, 4094, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4092, 0, 0, 1, 0, 2269, 1407, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3867, 0, 0, 0, 4069, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 88, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 653, 12, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3868, 4089, 0, 1109, 1827, 4096, 0, 0, 0, 0, 1735, 0, 0, 1, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 292, 0, 4053, 0, 0, 0, 0, 0, 0, 4024, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1814, 1, 0, 0, 0, 4094, 4096, 0, 1794, 13, 4, 0, 0, 1448, 0, 0, 2533, 2098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15877, 16382, 16438, 16243, 14952, 16773, 16512, 17895] avg=16384.0000 max=17895 max_rank=7 min=14952 min_rank=4 imbalance_ratio=1.092224 max_load_reduction=113177 max_load_reduction_pct=0.863472 moved_to_echo=66132 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3586, 228, 0, 0, 34, 3833, 0, 3604, 4096, 0, 0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 19, 33, 0, 0, 4079, 0, 3, 4083, 0, 2, 0, 0, 2269, 1407, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 1, 4084, 0, 0, 3, 26, 4088, 4096, 0, 4, 0, 3574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 13, 4062, 0, 0, 0, 4091, 1, 4096, 0, 0, 2, 4096, 0, 0, 0, 88, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1854, 0, 2, 0, 0, 0, 0, 0, 4096, 4096, 4096, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 4082, 0, 1545, 1827, 4095, 1, 0, 0, 0, 270, 0, 3, 10, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 115, 0, 4044, 0, 0, 0, 0, 3, 0, 4095, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 1, 0, 12, 0, 3946, 4096, 0, 3026, 110, 0, 0, 0, 2128, 0, 0, 2532, 2098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15663, 16396, 16031, 16478, 16144, 15896, 16355, 18109] avg=16384.0000 max=18109 max_rank=7 min=15663 min_rank=0 imbalance_ratio=1.105286 max_load_reduction=112963 max_load_reduction_pct=0.861839 moved_to_echo=66504 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3589, 1388, 0, 0, 199, 3999, 0, 3621, 4096, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 0, 1, 4090, 0, 0, 0, 0, 2269, 1407, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 2, 4094, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 0, 0, 2, 4038, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 88, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2822, 7, 0, 0, 0, 0, 0, 0, 4093, 4096, 4096, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3500, 4095, 0, 2520, 1827, 4096, 0, 0, 0, 0, 1254, 0, 0, 238, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 148, 0, 4049, 0, 0, 0, 0, 0, 0, 3940, 3961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 8, 0, 0, 0, 4082, 4096, 4, 399, 41, 7, 0, 0, 643, 0, 1, 2533, 2098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16975, 16379, 16536, 16198, 17112, 17533, 16167, 14172] avg=16384.0000 max=17533 max_rank=5 min=14172 min_rank=7 imbalance_ratio=1.070129 max_load_reduction=113539 max_load_reduction_pct=0.866234 moved_to_echo=64984 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 3584, 1702, 0, 20, 1, 4075, 0, 3790, 4069, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4092, 0, 5, 4089, 1, 5, 0, 0, 2266, 1406, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 4074, 29, 0, 0, 0, 4091, 4078, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4086, 0, 0, 1, 4085, 0, 4094, 0, 0, 1, 4090, 5, 4, 0, 88, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 39, 279, 0, 0, 0, 4, 0, 7, 3, 4096, 4087, 4092, 1996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3850, 4085, 5, 2419, 1826, 4096, 6, 0, 1, 0, 1091, 0, 0, 648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 234, 12, 3651, 4, 4, 0, 0, 24, 4, 3981, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 7, 0, 0, 0, 4084, 4092, 10, 2244, 11, 1, 25, 2, 583, 0, 2, 2531, 2097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17246, 16377, 16454, 16485, 14604, 18027, 16032, 15847] avg=16384.0000 max=18027 max_rank=5 min=14604 min_rank=4 imbalance_ratio=1.100281 max_load_reduction=113045 max_load_reduction_pct=0.862465 moved_to_echo=64510 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3566, 1285, 4, 28, 1, 4027, 0, 3246, 4092, 0, 6, 141, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 1, 0, 1, 6, 0, 0, 4092, 2, 0, 4088, 1, 0, 1, 0, 2269, 1408, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 4021, 0, 2, 0, 52, 4067, 4089, 0, 0, 3, 3881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 3922, 0, 0, 0, 4094, 0, 4092, 0, 2, 0, 4095, 3, 13, 2, 90, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1772, 50, 1, 4, 3, 1, 0, 0, 4095, 4093, 4095, 1984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3829, 4075, 0, 3058, 1822, 4086, 1, 0, 4, 3, 294, 0, 0, 825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 376, 2, 3968, 9, 4, 3, 0, 30, 7, 3970, 4041, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 4088, 4096, 0, 1602, 152, 0, 18, 14, 279, 3, 233, 2531, 2082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16399, 16369, 16261, 16372, 16100, 17997, 16465, 15109] avg=16384.0000 max=17997 max_rank=5 min=15109 min_rank=7 imbalance_ratio=1.098450 max_load_reduction=113075 max_load_reduction_pct=0.862694 moved_to_echo=65671 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3586, 171, 1, 13, 37, 3752, 0, 3936, 4095, 0, 0, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 0, 0, 0, 0, 0, 6, 4095, 0, 0, 4096, 0, 0, 0, 0, 2267, 1406, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 0, 4056, 1, 0, 7, 2, 4075, 4090, 0, 2, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4024, 1, 0, 0, 4083, 0, 4095, 1, 3, 0, 4096, 0, 0, 0, 88, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 3, 3811, 11, 0, 0, 0, 1, 0, 3, 4093, 4095, 4095, 1997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3952, 4087, 3, 640, 1826, 4092, 0, 0, 0, 0, 581, 0, 0, 874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4038, 85, 0, 3580, 1, 0, 1, 0, 2, 0, 4080, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 8, 0, 1, 3, 4074, 4095, 0, 1776, 166, 23, 0, 0, 1237, 0, 11, 2531, 2097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15740, 16375, 16409, 16421, 18112, 16056, 15864, 16095] avg=16384.0000 max=18112 max_rank=4 min=15740 min_rank=0 imbalance_ratio=1.105469 max_load_reduction=112960 max_load_reduction_pct=0.861816 moved_to_echo=66127 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 3581, 787, 0, 4, 458, 3980, 0, 3338, 4091, 0, 0, 534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 9, 0, 0, 3, 4096, 0, 0, 4094, 0, 0, 0, 0, 2269, 1407, 419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 155, 0, 4, 4096, 0, 2, 6, 4, 4096, 4074, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 4087, 1, 4093, 0, 0, 1, 4095, 2, 0, 0, 88, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 3261, 1, 0, 0, 0, 0, 3, 0, 4094, 4094, 4096, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3241, 3954, 0, 475, 1827, 4095, 7, 9, 0, 0, 277, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3911, 265, 0, 4050, 1, 0, 0, 1, 29, 0, 4073, 4079, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 4, 0, 14, 0, 3883, 4095, 0, 1770, 1, 6, 38, 0, 2365, 0, 43, 2532, 2098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16776, 16394, 16521, 16463, 17556, 14050, 16412, 16900] avg=16384.0000 max=17556 max_rank=4 min=14050 min_rank=5 imbalance_ratio=1.071533 max_load_reduction=113516 max_load_reduction_pct=0.866058 moved_to_echo=64918 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32384, 2944, 128, 31488, 128, 128, 128, 128, 128, 128, 32128, 32384, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 28672, 7040, 128, 128, 1152, 31744, 0, 28416, 32768, 0, 128, 1536, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 31872, 128, 128, 128, 32640, 128, 32768, 128, 128, 128, 32768, 128, 128, 128, 768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[2688, 128, 128, 128, 128, 32384, 32768, 128, 15360, 640, 128, 128, 128, 9984, 128, 384, 20352, 16768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 1280, 128, 128, 32640, 128, 128, 128, 128, 32768, 32768, 0, 128, 128, 31744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 16768, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 32768, 16000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 18176, 11264, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 30464, 32640, 128, 12032, 14720, 32768, 128, 128, 128, 128, 5760, 128, 128, 3072, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 412, 0, 0, 0, 0, 0, 4096, 0, 17, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 110, 4096, 0, 0, 3823, 0, 0, 3860, 0, 3809, 0, 0, 4096, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 3768, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 4096, 0, 0, 0, 0, 0, 768, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4095, 3950, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 416, 0, 0, 0, 0, 0, 4096, 0, 211, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 4096, 94, 4096, 0, 0, 4058, 0, 0, 3281, 0, 3925, 0, 0, 4096, 0, 0, 0, 0, 154, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3219, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4096, 0, 0, 3, 0, 0, 1057, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4094, 4088, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 0, 4092, 4079, 3, 3, 4, 4094, 5, 1, 619, 0, 0, 0, 2, 0, 4090, 3, 430, 0, 4, 1, 0, 4, 4093, 0, 2, 4093, 0, 0, 4084, 1661, 4092, 4, 0, 3554, 0, 0, 3911, 4, 2703, 0, 0, 4093, 3, 0, 0, 0, 1935, 4, 0, 0, 0, 1, 0, 4090, 0, 4072, 2944, 4093, 1, 2, 6, 1, 5, 5, 1, 0, 0, 0, 4088, 4091, 2, 4075, 4, 1, 6, 6, 9, 1211, 0, 1, 4090, 5, 0, 0, 1, 4083, 4092, 0, 0, 4093, 0, 1, 4094, 4084, 4093, 19, 3, 2, 1, 2, 0, 4095, 0, 0, 4092, 3, 4094, 3, 4090, 0, 4, 0, 1544, 0, 1, 4, 3, 2, 0, 0, 0, 4088, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[24, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2720, 0, 0, 0, 1, 0, 4096, 0, 19, 8, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 229, 4096, 3, 11, 4045, 0, 0, 4019, 0, 3822, 0, 0, 4096, 0, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 4095, 0, 4085, 1212, 4095, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 5, 4083, 0, 0, 0, 0, 0, 37, 0, 0, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 0, 2, 4074, 4095, 4096, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 4096, 0, 4090, 0, 4096, 0, 0, 0, 3727, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 61] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[20, 3, 1, 4095, 4089, 0, 1, 1, 4095, 0, 1, 2657, 0, 1, 3, 3, 0, 4092, 0, 136, 30, 0, 40, 1, 0, 4095, 0, 4, 4093, 0, 6, 4092, 730, 4095, 2, 0, 3905, 0, 1, 4092, 1, 3722, 0, 1, 4094, 1, 0, 7, 1, 408, 0, 0, 5, 1, 0, 0, 4090, 0, 4084, 1850, 4057, 0, 0, 1, 1, 0, 1, 0, 1, 1, 0, 4093, 4094, 2, 4079, 0, 2, 7, 0, 0, 61, 2, 0, 4094, 1, 0, 0, 0, 4095, 4094, 2, 0, 4088, 0, 0, 4072, 4060, 4094, 0, 1, 0, 1, 5, 0, 4092, 1, 2, 4094, 0, 4090, 0, 4090, 0, 1, 5, 2882, 4, 1, 9, 1, 0, 3, 0, 0, 4095, 0, 0, 44] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[19, 1, 0, 4095, 4083, 0, 0, 0, 4096, 0, 0, 1898, 0, 0, 0, 1, 9, 4096, 0, 31, 0, 0, 7, 1, 0, 4095, 0, 0, 4094, 2, 1, 4092, 126, 4096, 1, 0, 3854, 2, 4, 4042, 0, 3943, 0, 0, 4095, 0, 0, 0, 0, 138, 0, 0, 0, 0, 1, 6, 4090, 0, 4092, 3174, 4096, 0, 0, 1, 0, 4, 3, 1, 0, 0, 1, 4095, 4095, 0, 4095, 3, 4, 0, 0, 0, 689, 0, 0, 4096, 0, 0, 0, 0, 4092, 4096, 0, 0, 4096, 0, 1, 4085, 4082, 4096, 7, 0, 0, 0, 5, 0, 4095, 0, 0, 4095, 0, 4090, 7, 4088, 0, 0, 0, 2565, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[10, 0, 0, 4096, 4096, 0, 0, 0, 4091, 0, 0, 507, 0, 0, 0, 2, 1, 4096, 0, 26, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 4086, 303, 4096, 3, 0, 4050, 0, 0, 3865, 0, 3519, 0, 0, 4096, 0, 0, 0, 0, 1441, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 2409, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 1, 4074, 0, 0, 18, 0, 0, 143, 0, 0, 4096, 0, 0, 5, 0, 4096, 4096, 2, 0, 4096, 0, 5, 4095, 4095, 4092, 0, 0, 0, 0, 7, 0, 4090, 0, 1, 4096, 0, 4096, 3, 4096, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 131] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4095, 0, 0, 1, 4095, 0, 0, 1790, 0, 0, 0, 0, 1, 4095, 1, 33, 2, 2, 0, 0, 29, 4095, 0, 1, 4094, 0, 7, 4095, 185, 4093, 2, 0, 3954, 2, 2, 3757, 1, 3181, 0, 0, 4095, 1, 0, 0, 0, 931, 0, 1, 0, 1, 1, 1, 4096, 0, 4094, 1699, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4095, 3, 4096, 0, 0, 0, 0, 4, 914, 0, 0, 4095, 0, 0, 0, 0, 4095, 4095, 0, 4, 4095, 0, 0, 4089, 4090, 4081, 0, 0, 0, 1, 2, 0, 4094, 1, 0, 4093, 0, 4091, 0, 4094, 0, 1, 0, 4007, 1, 1, 0, 1, 0, 1, 0, 1, 4095, 1, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 416, 0, 0, 0, 0, 2711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 211, 0, 0, 0, 0, 0, 3970, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 4096, 0, 0, 4058, 0, 0, 3281, 0, 3925, 0, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3219, 4084, 0, 0, 0, 600, 126, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4096, 0, 0, 3, 0, 0, 3353, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 0, 0, 3496, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 4096, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15417, 16470, 15972, 16498, 16386, 16839, 16404, 17086] avg=16384.0000 max=17086 max_rank=7 min=15417 min_rank=0 imbalance_ratio=1.042847 max_load_reduction=113986 max_load_reduction_pct=0.869644 moved_to_echo=66715 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[20, 3, 1, 4095, 4089, 0, 1, 1, 4095, 0, 1, 2657, 0, 1, 3, 3, 2710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 136, 30, 0, 40, 1, 0, 3969, 0, 4, 4093, 0, 6, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 4095, 2, 0, 3905, 0, 1, 4092, 1, 3722, 0, 1, 517, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 408, 0, 0, 5, 1, 0, 0, 4090, 0, 4084, 1850, 4057, 0, 0, 1, 600, 126, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 4093, 4094, 2, 4079, 0, 2, 7, 0, 0, 3350, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 2, 0, 3494, 1, 0, 0, 0, 4095, 0, 2, 0, 4088, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 4094, 0, 1, 0, 1, 5, 0, 28, 1, 2, 0, 0, 4090, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 2882, 4, 1, 9, 1, 0, 3, 0, 0, 4095, 0, 0, 44, 4094, 4094, 714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17680, 16463, 17074, 15346, 16375, 15815, 16372, 15947] avg=16384.0000 max=17680 max_rank=0 min=15346 min_rank=3 imbalance_ratio=1.079102 max_load_reduction=113392 max_load_reduction_pct=0.865112 moved_to_echo=64509 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 412, 0, 0, 0, 0, 2711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 17, 0, 0, 1, 0, 0, 3970, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 4096, 0, 0, 3823, 0, 0, 3860, 0, 3809, 0, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 3768, 4093, 0, 1, 0, 600, 126, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 4096, 0, 0, 0, 0, 0, 3353, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 768, 0, 0, 3496, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3950, 4096, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 4096, 4096, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15412, 16276, 16216, 17082, 16382, 16551, 16266, 16887] avg=16384.0000 max=17082 max_rank=3 min=15412 min_rank=0 imbalance_ratio=1.042603 max_load_reduction=113990 max_load_reduction_pct=0.869675 moved_to_echo=66086 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4092, 4079, 3, 3, 4, 4094, 5, 1, 619, 0, 0, 0, 2, 2714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 430, 0, 4, 1, 0, 4, 3960, 0, 2, 4093, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1661, 4092, 4, 0, 3554, 0, 0, 3911, 4, 2703, 0, 0, 499, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1935, 4, 0, 0, 0, 1, 0, 4090, 0, 4072, 2944, 4093, 1, 2, 6, 603, 133, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 5, 1, 0, 0, 0, 4088, 4091, 2, 4075, 4, 1, 6, 6, 9, 3357, 750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1211, 0, 1, 3487, 5, 0, 0, 1, 4083, 0, 0, 0, 4093, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4093, 19, 3, 2, 1, 2, 0, 20, 0, 0, 0, 3, 4094, 3, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1544, 0, 1, 4, 3, 2, 0, 0, 0, 4088, 0, 0, 0, 4092, 4092, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15617, 16671, 16431, 18014, 16401, 16976, 16414, 14548] avg=16384.0000 max=18014 max_rank=3 min=14548 min_rank=7 imbalance_ratio=1.099487 max_load_reduction=113058 max_load_reduction_pct=0.862564 moved_to_echo=64339 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[24, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2720, 0, 0, 0, 1, 2711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 8, 0, 0, 0, 0, 3970, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 4096, 3, 11, 4045, 0, 0, 4019, 0, 3822, 0, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 0, 0, 0, 0, 0, 0, 4095, 0, 4085, 1212, 4095, 0, 28, 0, 600, 126, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 5, 4083, 0, 0, 0, 0, 0, 3352, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 3496, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 2, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 1, 0, 28, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3727, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 61, 4096, 4096, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17744, 16285, 16743, 14924, 16375, 15802, 16406, 16793] avg=16384.0000 max=17744 max_rank=0 min=14924 min_rank=3 imbalance_ratio=1.083008 max_load_reduction=113328 max_load_reduction_pct=0.864624 moved_to_echo=65376 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[19, 1, 0, 4095, 4083, 0, 0, 0, 4096, 0, 0, 1898, 0, 0, 0, 1, 2710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 31, 0, 0, 7, 1, 0, 3969, 0, 0, 4094, 2, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 4096, 1, 0, 3854, 2, 4, 4042, 0, 3943, 0, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 1, 6, 4090, 0, 4092, 3174, 4096, 0, 0, 1, 600, 126, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 1, 0, 0, 1, 4095, 4095, 0, 4095, 3, 4, 0, 0, 0, 3352, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, 0, 0, 3496, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 1, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 7, 0, 0, 0, 5, 0, 28, 0, 0, 0, 0, 4090, 7, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2565, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 8, 4096, 4095, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16903, 16302, 16586, 16447, 16397, 16459, 16403, 15575] avg=16384.0000 max=16903 max_rank=0 min=15575 min_rank=7 imbalance_ratio=1.031677 max_load_reduction=114169 max_load_reduction_pct=0.871040 moved_to_echo=64834 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 0, 4096, 4096, 0, 0, 0, 4091, 0, 0, 507, 0, 0, 0, 2, 2711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 26, 0, 0, 0, 0, 1, 3970, 0, 0, 4096, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 4096, 3, 0, 4050, 0, 0, 3865, 0, 3519, 0, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1441, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 2409, 4096, 0, 3, 0, 600, 126, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 1, 4074, 0, 0, 18, 0, 0, 3348, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 3496, 0, 0, 5, 0, 4096, 0, 2, 0, 4096, 0, 5, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4092, 0, 0, 0, 0, 7, 0, 28, 0, 1, 0, 0, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 131, 4096, 4096, 714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15513, 16276, 16354, 16980, 16374, 15938, 16418, 17219] avg=16384.0000 max=17219 max_rank=7 min=15513 min_rank=0 imbalance_ratio=1.050964 max_load_reduction=113853 max_load_reduction_pct=0.868629 moved_to_echo=65949 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 4095, 0, 0, 1, 4095, 0, 0, 1790, 0, 0, 0, 0, 2710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 33, 2, 2, 0, 0, 29, 3969, 0, 1, 4094, 0, 7, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 4093, 2, 0, 3954, 2, 2, 3757, 1, 3181, 0, 0, 518, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 931, 0, 1, 0, 1, 1, 1, 4096, 0, 4094, 1699, 4096, 0, 12, 0, 600, 126, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4095, 3, 4096, 0, 0, 0, 0, 4, 3351, 744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 914, 0, 0, 3495, 0, 0, 0, 0, 4095, 0, 0, 4, 4095, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4081, 0, 0, 0, 1, 2, 0, 29, 1, 0, 0, 0, 4091, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4007, 1, 1, 0, 1, 0, 1, 0, 1, 4095, 1, 0, 6, 4095, 4093, 714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16786, 16329, 15696, 15781, 16382, 16692, 16389, 17017] avg=16384.0000 max=17017 max_rank=7 min=15696 min_rank=2 imbalance_ratio=1.038635 max_load_reduction=114055 max_load_reduction_pct=0.870171 moved_to_echo=66480 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32640, 32768, 128, 128, 128, 128, 128, 0, 256, 128, 128, 0, 128, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[3456, 32768, 128, 128, 31360, 128, 128, 30848, 128, 28672, 0, 128, 4224, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 1024, 128, 128, 128, 128, 128, 31872, 0, 128, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 32768, 32768, 128, 128, 128, 32768, 128, 128, 11136, 0, 128, 128, 128, 21760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[4992, 128, 128, 28032, 128, 0, 128, 128, 32768, 0, 128, 128, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[0, 128, 128, 26880, 128, 128, 128, 128, 128, 128, 0, 128, 32768, 128, 128, 384, 32768, 32768, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 5760, 128, 128, 128, 128, 128, 128, 32768, 0, 32768, 20352, 32768, 128, 128, 128, 4864, 1024, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 32768, 32768, 128, 32768, 128, 128, 128, 128, 128, 26880, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2267, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 1317, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 542, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 3989, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4069, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3977, 0, 4096, 4084, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 153, 117, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 17, 4096, 4096, 0, 0, 4079, 0, 3925, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 3997, 0, 0, 0, 0, 0, 73, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 14] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 6, 8, 4, 1622, 7, 4090, 4087, 0, 13, 12, 0, 2, 0, 0, 3, 2, 8, 4, 4089, 5, 0, 0, 5, 2, 4088, 1, 4089, 0, 0, 8, 0, 0, 0, 0, 0, 4, 0, 4092, 0, 4092, 0, 0, 9, 5, 0, 0, 0, 7, 0, 0, 2, 32, 0, 1, 4092, 4092, 4, 3, 4085, 4089, 4090, 4092, 9, 2486, 0, 0, 3, 0, 4, 4089, 0, 7, 4091, 4, 4, 48, 4092, 0, 2, 0, 4092, 4092, 0, 0, 4082, 2, 4059, 0, 4077, 0, 4091, 4, 1, 0, 3, 5, 4096, 4084, 4, 4087, 2, 3, 4091, 0, 3753, 0, 4, 2, 0, 3, 251, 23, 4, 0, 3, 4091, 0, 0, 5, 0, 4, 4083, 2, 0, 4092, 4090, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3658, 0, 4095, 4096, 0, 0, 22, 0, 0, 0, 4, 0, 0, 135, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 21, 3, 0, 1, 0, 0, 0, 4096, 0, 4086, 2, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4, 0, 4096, 4096, 0, 0, 4095, 4096, 4094, 4093, 9, 454, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 3, 4084, 0, 3976, 0, 4096, 0, 4094, 0, 0, 4, 0, 0, 4094, 4096, 0, 4091, 0, 0, 4096, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2923, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 154, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 1081, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 49, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4093, 0, 4009, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4072, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1, 5, 0, 1, 3626, 1, 4093, 4083, 2, 0, 17, 0, 0, 1, 0, 1, 0, 0, 2, 4092, 1, 0, 0, 0, 0, 4085, 0, 4093, 0, 0, 1, 7, 6, 0, 0, 0, 1, 0, 4094, 0, 4088, 0, 0, 7, 9, 0, 4, 0, 1, 0, 0, 2, 14, 10, 0, 4094, 4095, 1, 1, 4092, 4095, 4093, 4092, 0, 493, 7, 0, 11, 0, 2, 4095, 0, 1, 4091, 2, 2, 3, 4094, 0, 1, 3, 4092, 4092, 1, 0, 4080, 2, 3935, 0, 4088, 0, 4094, 2, 0, 0, 1, 0, 4095, 4090, 1, 4093, 1, 2, 4093, 2, 4069, 0, 4, 1, 1, 1, 122, 0, 2, 0, 1, 4095, 0, 0, 4, 1, 3, 4091, 0, 3, 4092, 4094, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3944, 0, 4096, 4085, 0, 0, 7, 0, 0, 0, 1, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 161, 0, 0, 5, 0, 0, 0, 4096, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 36, 6, 0, 4096, 4096, 0, 0, 4095, 4096, 4092, 4096, 4, 181, 2, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 61, 4096, 4096, 0, 0, 4096, 0, 4024, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4091, 4096, 0, 4093, 0, 0, 4096, 0, 3764, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 4096, 55] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 1, 1, 3547, 2, 4095, 4091, 0, 8, 1, 1, 1, 1, 0, 0, 1, 3, 0, 4094, 1, 0, 0, 0, 0, 4094, 0, 4054, 0, 0, 0, 83, 3, 1, 2, 0, 0, 1, 4095, 0, 4095, 0, 0, 70, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 4095, 1, 1, 4094, 4093, 4095, 4095, 3, 513, 121, 2, 0, 0, 1, 4095, 1, 1, 4095, 0, 0, 0, 4089, 0, 0, 0, 4095, 4095, 0, 0, 4088, 0, 3947, 0, 4092, 2, 4095, 1, 0, 0, 1, 2, 4094, 4095, 0, 4095, 0, 1, 4095, 1, 3960, 0, 1, 1, 0, 0, 81, 0, 1, 0, 1, 4096, 0, 0, 0, 0, 0, 4095, 1, 0, 4095, 4095, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2267, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 876, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 6, 3165, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 4096, 4077, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1317, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 542, 4096, 0, 0, 3123, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 4095, 0, 3989, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 55, 0, 4069, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15445, 16337, 16372, 16382, 17295, 16391, 16471, 16379] avg=16384.0000 max=17295 max_rank=4 min=15445 min_rank=0 imbalance_ratio=1.055603 max_load_reduction=113777 max_load_reduction_pct=0.868050 moved_to_echo=66536 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 8, 4, 1622, 7, 4090, 4087, 0, 13, 12, 0, 2, 0, 0, 3, 4092, 879, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8, 4, 4089, 5, 0, 0, 5, 2, 4088, 1, 4089, 0, 0, 8, 0, 3168, 864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4092, 0, 4092, 0, 0, 9, 5, 0, 0, 0, 4092, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2, 32, 0, 1, 0, 0, 4, 3, 4085, 4063, 4090, 4092, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2486, 0, 0, 3, 0, 4, 4089, 0, 7, 4091, 4, 4, 48, 4092, 0, 2, 3124, 26, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 4082, 2, 4059, 0, 4077, 0, 4091, 4, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4084, 4, 4087, 2, 3, 44, 0, 3753, 0, 4, 2, 0, 3, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4, 0, 3, 4091, 0, 0, 5, 0, 4, 4083, 2, 0, 4085, 4090, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14840, 16333, 16371, 16388, 17987, 16423, 16338, 16392] avg=16384.0000 max=17987 max_rank=4 min=14840 min_rank=0 imbalance_ratio=1.097839 max_load_reduction=113085 max_load_reduction_pct=0.862770 moved_to_echo=67140 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3977, 0, 4096, 4084, 0, 0, 16, 0, 0, 0, 0, 0, 4096, 876, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 30, 3165, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4077, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 117, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3123, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 115, 0, 0, 4079, 0, 3925, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 55, 0, 3997, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17159, 16338, 16370, 16366, 15706, 16328, 16413, 16392] avg=16384.0000 max=17159 max_rank=0 min=15706 min_rank=4 imbalance_ratio=1.047302 max_load_reduction=113913 max_load_reduction_pct=0.869087 moved_to_echo=64839 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 5, 0, 1, 3626, 1, 4093, 4083, 2, 0, 17, 0, 0, 1, 0, 1, 4092, 875, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4092, 1, 0, 0, 0, 0, 4085, 0, 4093, 0, 0, 1, 7, 3163, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1, 0, 4094, 0, 4088, 0, 0, 7, 9, 0, 4, 0, 4095, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 14, 10, 0, 0, 0, 1, 1, 4092, 4076, 4093, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 7, 0, 11, 0, 2, 4095, 0, 1, 4091, 2, 2, 3, 4094, 0, 1, 3120, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 114, 1, 0, 4080, 2, 3935, 0, 4088, 0, 4094, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4090, 1, 4093, 1, 2, 55, 2, 4069, 0, 4, 1, 1, 1, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4095, 0, 0, 4, 1, 3, 4091, 0, 3, 4086, 4094, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16812, 16302, 16384, 16382, 15947, 16320, 16537, 16388] avg=16384.0000 max=16812 max_rank=0 min=15947 min_rank=4 imbalance_ratio=1.026123 max_load_reduction=114260 max_load_reduction_pct=0.871735 moved_to_echo=65192 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3658, 0, 4095, 4096, 0, 0, 22, 0, 0, 0, 4, 0, 4096, 876, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 21, 3165, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 4096, 0, 4086, 2, 7, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4, 0, 0, 0, 0, 0, 4095, 4077, 4094, 4093, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 454, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 3123, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 3, 4084, 0, 3976, 0, 4096, 0, 4094, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4091, 0, 0, 55, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16861, 16465, 16373, 16407, 15886, 16372, 16327, 16381] avg=16384.0000 max=16861 max_rank=0 min=15886 min_rank=4 imbalance_ratio=1.029114 max_load_reduction=114211 max_load_reduction_pct=0.871361 moved_to_echo=64966 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2923, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 876, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 7, 3165, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 10, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 4077, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1081, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 49, 4096, 0, 0, 3123, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 4093, 0, 4009, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 55, 0, 4072, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16102, 16472, 16379, 16377, 16566, 16409, 16437, 16330] avg=16384.0000 max=16566 max_rank=4 min=16102 min_rank=0 imbalance_ratio=1.011108 max_load_reduction=114506 max_load_reduction_pct=0.873611 moved_to_echo=65742 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3944, 0, 4096, 4085, 0, 0, 7, 0, 0, 0, 1, 0, 4096, 876, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 161, 3165, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 6, 0, 0, 0, 0, 0, 4095, 4077, 4092, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 2, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3123, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 115, 0, 0, 4096, 0, 4024, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 4093, 0, 0, 55, 0, 3764, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4093, 0, 0, 4090, 4096, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17119, 16472, 16379, 16406, 15620, 16488, 16164, 16424] avg=16384.0000 max=17119 max_rank=0 min=15620 min_rank=4 imbalance_ratio=1.044861 max_load_reduction=113953 max_load_reduction_pct=0.869392 moved_to_echo=64696 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 1, 3547, 2, 4095, 4091, 0, 8, 1, 1, 1, 1, 0, 0, 4095, 876, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 4094, 1, 0, 0, 0, 0, 4094, 0, 4054, 0, 0, 0, 83, 3165, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 2, 0, 0, 1, 4095, 0, 4095, 0, 0, 70, 0, 1, 0, 0, 4095, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4094, 4074, 4095, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 121, 2, 0, 0, 1, 4095, 1, 1, 4095, 0, 0, 0, 4089, 0, 0, 3122, 19, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 0, 0, 4088, 0, 3947, 0, 4092, 2, 4095, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 4095, 0, 4095, 0, 1, 54, 1, 3960, 0, 1, 1, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4096, 0, 0, 0, 0, 0, 4095, 1, 0, 4089, 4095, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16734, 16353, 16444, 16364, 16065, 16341, 16385, 16386] avg=16384.0000 max=16734 max_rank=0 min=16065 min_rank=4 imbalance_ratio=1.021362 max_load_reduction=114338 max_load_reduction_pct=0.872330 moved_to_echo=65177 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 25600, 128, 32768, 32768, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 7040, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 0, 32768, 128, 128, 128, 128, 128, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 384, 128, 32768, 128, 0, 0, 128, 128, 32768, 128, 32768, 0, 0, 128, 384, 25344, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[6784, 256, 128, 128, 0, 128, 32768, 128, 128, 32768, 128, 128, 768, 32768, 0, 128, 25088, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 128, 128, 128, 32768, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 32768, 128, 32768, 128, 128, 512, 128, 31744, 0, 128, 128, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 1024, 128, 128, 32768, 128, 31872, 0, 32768, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 256, 128, 128, 0, 0, 128, 128, 32768, 32640, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2268, 0, 4096, 0, 0, 4096, 4095, 0, 0, 4096, 4090, 4096, 0, 0, 22, 0, 0, 4096, 0, 4096, 0, 4096, 0, 76, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 23, 1080, 4096, 0, 0, 0, 2950, 0, 0, 0, 0, 4096, 4070, 0, 0, 0, 3292, 0, 4095, 406, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 1945, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 266, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[1, 6, 4091, 0, 4, 2, 0, 0, 0, 5, 0, 0, 0, 5, 0, 4091, 0, 5, 0, 3724, 0, 4090, 1, 6, 4086, 4032, 5, 2, 4083, 4091, 4090, 7, 0, 4, 4, 0, 4016, 2, 4091, 0, 4091, 0, 61, 2, 4092, 0, 0, 0, 6, 5, 4090, 4091, 6, 4089, 6, 1352, 4090, 4, 0, 0, 426, 0, 0, 0, 0, 4090, 4094, 0, 0, 16, 2902, 0, 4081, 1378, 0, 5, 0, 4092, 4092, 0, 4092, 3, 3, 4, 3, 3971, 4, 0, 2511, 0, 5, 3, 5, 0, 0, 4088, 3, 4, 7, 4091, 0, 5, 0, 4090, 114, 5, 5, 0, 0, 3, 0, 8, 4087, 24, 1, 0, 6, 34, 4090, 0, 5, 0, 4, 4089, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4084, 0, 4096, 0, 0, 4096, 4090, 0, 0, 4096, 4091, 4096, 0, 0, 0, 0, 1, 4091, 0, 4096, 0, 4096, 0, 693, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 13, 206, 4096, 0, 0, 0, 1433, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 3719, 0, 4096, 160, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 983, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 1021, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4, 0, 4088, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 4092, 0, 0, 0, 3563, 0, 4095, 0, 0, 4096, 4065, 0, 0, 4096, 4076, 4096, 0, 0, 1, 0, 0, 4091, 0, 4096, 0, 4096, 4, 8, 0, 4090, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 62, 92, 4092, 0, 0, 0, 1113, 2, 0, 0, 1, 4096, 3718, 0, 0, 0, 3713, 0, 4096, 21, 0, 0, 0, 4092, 4093, 4, 4090, 0, 0, 0, 0, 4093, 0, 7, 3844, 0, 0, 3, 0, 1, 8, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 292, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 1, 4096, 1, 0, 0, 0, 4096, 6, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 4089, 0, 1, 4, 0, 4, 4, 2, 0, 0, 0, 0, 0, 4089, 0, 4, 0, 3618, 0, 4084, 0, 2, 4094, 3971, 0, 0, 4091, 4094, 4093, 3, 0, 34, 3, 31, 4095, 0, 4094, 2, 4093, 0, 29, 2, 4090, 0, 0, 0, 0, 2, 4093, 4094, 3, 4093, 37, 794, 4092, 1, 0, 0, 1414, 0, 0, 0, 0, 4093, 3886, 0, 0, 0, 2799, 0, 4074, 300, 0, 2, 1, 4095, 4091, 0, 4094, 3, 4, 3, 1, 4085, 2, 13, 3460, 0, 2, 0, 2, 0, 1, 4088, 1, 2, 4, 4092, 0, 2, 0, 4093, 111, 2, 2, 0, 2, 0, 0, 2, 4094, 0, 0, 2, 0, 11, 4094, 0, 1, 5, 3, 4093, 13, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3120, 0, 4096, 0, 0, 4096, 3915, 0, 0, 4096, 4080, 4096, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 4096, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 384, 601, 4095, 0, 0, 0, 2410, 0, 0, 0, 0, 4096, 3965, 0, 0, 0, 3058, 0, 4083, 198, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3895, 0, 0, 2880, 0, 0, 2, 0, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 135, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3828, 0, 4095, 0, 0, 4096, 4042, 0, 0, 4096, 4082, 4096, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 4096, 0, 3, 0, 4090, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 327, 663, 4096, 0, 0, 0, 1082, 0, 0, 0, 15, 4096, 3853, 0, 0, 0, 3446, 0, 4096, 65, 0, 0, 0, 4092, 4096, 0, 4095, 0, 0, 0, 0, 4094, 0, 5, 3120, 0, 0, 0, 0, 5, 10, 4085, 0, 0, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 1, 0, 4096, 37, 0, 0, 0, 8, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4095, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 4095, 0, 1, 0, 4041, 0, 4096, 0, 0, 4095, 4001, 0, 0, 4095, 4081, 4095, 1, 0, 51, 0, 0, 4079, 0, 4094, 1, 4095, 0, 11, 1, 4095, 1, 0, 0, 1, 1, 4095, 4095, 1, 4095, 202, 304, 4095, 1, 0, 3, 3178, 0, 2, 0, 0, 4095, 4037, 0, 0, 0, 3188, 0, 4054, 84, 0, 1, 1, 4090, 4095, 1, 4095, 0, 0, 0, 0, 4094, 1, 0, 1068, 0, 1, 2, 1, 0, 3, 4092, 0, 1, 1, 4095, 1, 1, 0, 4095, 382, 0, 1, 0, 1, 0, 0, 1, 4094, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 1, 4094, 1, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2268, 0, 4096, 0, 0, 0, 4095, 0, 0, 644, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 4096, 0, 3983, 0, 4096, 0, 76, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1578, 0, 4096, 23, 1080, 4096, 0, 0, 0, 2950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 3292, 0, 4095, 406, 0, 0, 0, 4096, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 1945, 0, 0, 0, 0, 0, 0, 4096, 1541, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 266, 0, 0, 0, 0, 0, 0, 0, 4096, 3452, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3099, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 15193, 16369, 17919, 16606, 15879, 16356, 16366] avg=16384.0000 max=17919 max_rank=3 min=15193 min_rank=1 imbalance_ratio=1.093689 max_load_reduction=113153 max_load_reduction_pct=0.863289 moved_to_echo=65207 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 6, 4091, 0, 4, 2, 0, 0, 0, 5, 0, 0, 0, 5, 0, 4091, 4090, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3724, 0, 4090, 1, 6, 0, 4032, 5, 2, 638, 4091, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 4016, 2, 3973, 0, 4091, 0, 61, 2, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 4090, 1569, 6, 4089, 6, 1352, 4090, 4, 0, 0, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 16, 2902, 0, 4081, 1378, 0, 5, 0, 4092, 637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 3, 4, 3, 3971, 4, 0, 2511, 0, 5, 3, 5, 0, 0, 4088, 1543, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 7, 4091, 0, 5, 0, 4090, 114, 5, 5, 0, 0, 3, 0, 8, 4085, 3445, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 24, 1, 0, 6, 34, 4090, 0, 5, 0, 4, 4089, 0, 0, 0, 0, 3102, 979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16601, 16245, 15643, 17205, 16353, 16218, 16421] avg=16384.0000 max=17205 max_rank=4 min=15643 min_rank=3 imbalance_ratio=1.050110 max_load_reduction=113867 max_load_reduction_pct=0.868736 moved_to_echo=66197 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 0, 0, 4090, 0, 0, 644, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 3983, 0, 4096, 0, 693, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1578, 0, 4096, 13, 206, 4096, 0, 0, 0, 1433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 3719, 0, 4096, 160, 0, 0, 0, 4096, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 983, 0, 0, 0, 0, 0, 2, 4096, 1541, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1021, 0, 0, 0, 0, 0, 0, 0, 4096, 3452, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3099, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 17005, 16960, 15518, 16802, 14927, 17111, 16365] avg=16384.0000 max=17111 max_rank=6 min=14927 min_rank=5 imbalance_ratio=1.044373 max_load_reduction=113961 max_load_reduction_pct=0.869453 moved_to_echo=65205 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4089, 0, 1, 4, 0, 4, 4, 2, 0, 0, 0, 0, 0, 4089, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3618, 0, 4084, 0, 2, 0, 3971, 0, 0, 643, 4094, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 3, 31, 4095, 0, 3981, 2, 4093, 0, 29, 2, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 1578, 3, 4093, 37, 794, 4092, 1, 0, 0, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3886, 0, 0, 0, 2799, 0, 4074, 300, 0, 2, 1, 4095, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 4, 3, 1, 4085, 2, 13, 3460, 0, 2, 0, 2, 0, 1, 4088, 1540, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4, 4092, 0, 2, 0, 4093, 111, 2, 2, 0, 2, 0, 0, 2, 4094, 3448, 349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 11, 4094, 0, 1, 5, 3, 4093, 13, 0, 0, 0, 3095, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16419, 16360, 16107, 15804, 17411, 16204, 16387] avg=16384.0000 max=17411 max_rank=5 min=15804 min_rank=4 imbalance_ratio=1.062683 max_load_reduction=113661 max_load_reduction_pct=0.867165 moved_to_echo=65806 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 4088, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 4092, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3563, 0, 4095, 0, 0, 0, 4065, 0, 0, 644, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 0, 3983, 0, 4096, 4, 8, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1578, 0, 4096, 62, 92, 4092, 0, 0, 0, 1113, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3718, 0, 0, 0, 3713, 0, 4096, 21, 0, 0, 0, 4092, 646, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4093, 0, 7, 3844, 0, 0, 3, 0, 1, 8, 4096, 1541, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 292, 0, 0, 0, 0, 0, 5, 0, 4096, 3452, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 1, 0, 0, 0, 4096, 6, 0, 0, 0, 3097, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16443, 16273, 15131, 16291, 17796, 16387, 16370] avg=16384.0000 max=17796 max_rank=5 min=15131 min_rank=3 imbalance_ratio=1.086182 max_load_reduction=113276 max_load_reduction_pct=0.864227 moved_to_echo=66844 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3120, 0, 4096, 0, 0, 0, 3915, 0, 0, 644, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 3983, 0, 4096, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1578, 0, 4096, 384, 601, 4095, 0, 0, 0, 2410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 0, 0, 0, 3058, 0, 4083, 198, 0, 0, 0, 4096, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3895, 0, 0, 2880, 0, 0, 2, 0, 0, 4, 4096, 1541, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 135, 0, 0, 0, 0, 0, 0, 0, 4096, 3452, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3099, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 15855, 16281, 17260, 16047, 16627, 16225, 16393] avg=16384.0000 max=17260 max_rank=3 min=15855 min_rank=1 imbalance_ratio=1.053467 max_load_reduction=113812 max_load_reduction_pct=0.868317 moved_to_echo=65292 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4091, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3828, 0, 4095, 0, 0, 0, 4042, 0, 0, 644, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 3983, 0, 4096, 0, 3, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1578, 0, 4096, 327, 663, 4096, 0, 0, 0, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3853, 0, 0, 0, 3446, 0, 4096, 65, 0, 0, 0, 4092, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4094, 0, 5, 3120, 0, 0, 0, 0, 5, 10, 4085, 1541, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 1, 0, 4096, 3452, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 37, 0, 0, 0, 8, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3099, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16691, 16269, 15938, 16214, 17068, 16096, 16409] avg=16384.0000 max=17068 max_rank=5 min=15938 min_rank=3 imbalance_ratio=1.041748 max_load_reduction=114004 max_load_reduction_pct=0.869781 moved_to_echo=65787 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 4095, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4041, 0, 4096, 0, 0, 0, 4001, 0, 0, 644, 4081, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 4079, 0, 3981, 1, 4095, 0, 11, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 1578, 1, 4095, 202, 304, 4095, 1, 0, 3, 3178, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 3188, 0, 4054, 84, 0, 1, 1, 4090, 647, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4094, 1, 0, 1068, 0, 1, 2, 1, 0, 3, 4092, 1541, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 1, 1, 0, 4095, 382, 0, 1, 0, 1, 0, 0, 1, 4095, 3451, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 1, 4094, 1, 0, 1, 0, 3098, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16865, 16315, 17556, 16103, 15011, 16475, 16361] avg=16384.0000 max=17556 max_rank=3 min=15011 min_rank=5 imbalance_ratio=1.071533 max_load_reduction=113516 max_load_reduction_pct=0.866058 moved_to_echo=63950 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 31744, 0, 0, 128, 26240, 0, 32768, 2688, 0, 128, 128, 32768, 5248, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 12672, 128, 32768, 1152, 5120, 32768, 128, 0, 128, 14080, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 128, 32768, 128, 0, 128, 0, 24832, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 31872, 128, 32768, 128, 896, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 28288, 0, 32768, 128, 128, 0, 32256, 128, 128, 5248, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 0, 32768, 2432, 128, 128, 0, 128, 128, 128, 128, 32768, 27648, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 0, 128, 128, 128, 0, 128, 128, 128, 128, 32768, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 128, 128, 128, 128, 32512, 128, 128, 19840, 0, 128, 128, 128, 128, 128, 32768, 12416, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 4096, 4090, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3572, 0, 0, 0, 0, 19, 22, 0, 0, 0, 4096, 4096, 0, 4096, 4092, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 602, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 26, 0, 4093, 0, 0, 0, 373, 3865, 0, 0, 0, 0, 0, 0, 4096, 2173, 0, 0, 0, 4096, 0, 0, 1241, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 406, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 18, 0, 0, 2510, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4091, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 531, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 4095, 0, 0, 0, 511, 3988, 0, 0, 0, 0, 0, 0, 4096, 1429, 0, 0, 0, 4096, 0, 0, 2304, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 976, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4090, 4088, 6, 6, 3, 9, 4091, 0, 4090, 0, 4089, 0, 7, 0, 0, 0, 4074, 4080, 4087, 4087, 4, 4088, 4090, 0, 4, 0, 0, 4, 18, 0, 11, 3000, 7, 0, 0, 0, 0, 0, 5, 0, 0, 4087, 4091, 0, 4090, 4089, 7, 4092, 2, 0, 4091, 0, 0, 4, 0, 1012, 6, 6, 0, 6, 1, 8, 0, 4087, 4, 0, 1, 5, 4052, 0, 4, 4, 337, 3831, 4, 5, 0, 6, 4, 8, 4085, 1251, 5, 0, 0, 4092, 3, 0, 2828, 0, 6, 4, 6, 0, 4090, 5, 5, 4055, 4089, 3, 4089, 0, 0, 4075, 45, 5, 1, 0, 4088, 8, 41, 0, 1, 0, 7, 2, 5, 5, 5, 0, 5, 0, 0, 4089, 1, 4086, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 32, 0, 0, 2584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4074, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1378, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 175, 0, 4095, 0, 0, 0, 1048, 4076, 0, 0, 0, 0, 0, 0, 4096, 801, 0, 0, 0, 4095, 0, 0, 1349, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 698, 0, 0, 0, 4096, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4091, 0, 0, 0, 0, 4090, 0, 4095, 0, 4096, 0, 0, 4, 1, 0, 4096, 4084, 4096, 4096, 0, 4096, 4096, 0, 0, 1, 0, 0, 12, 0, 4, 3396, 0, 3, 0, 0, 32, 30, 0, 0, 0, 4096, 4091, 0, 4096, 4072, 0, 4091, 0, 0, 4096, 7, 3, 0, 1, 744, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 4095, 0, 0, 0, 152, 4039, 0, 0, 3, 0, 0, 0, 4096, 389, 0, 0, 9, 4071, 0, 0, 3461, 1, 0, 0, 0, 0, 4092, 0, 0, 4096, 4096, 0, 4096, 0, 7, 4096, 82, 0, 0, 0, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[0, 4095, 4095, 1, 1, 0, 0, 4095, 1, 4095, 0, 4094, 1, 1, 0, 0, 0, 4095, 4095, 4094, 4081, 0, 4095, 4095, 0, 1, 0, 0, 0, 1, 0, 2, 3593, 0, 0, 0, 0, 29, 13, 1, 0, 0, 4095, 4095, 0, 4095, 4095, 1, 4095, 0, 0, 4094, 0, 0, 1, 0, 11, 1, 1, 0, 1, 0, 1, 0, 4095, 1, 0, 1, 0, 4082, 1, 1, 0, 164, 3951, 0, 0, 0, 1, 0, 0, 4093, 1525, 1, 0, 0, 4094, 0, 3, 1368, 0, 1, 1, 1, 1, 4071, 2, 1, 4093, 4094, 1, 4095, 0, 0, 4095, 1680, 1, 1, 0, 4095, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 4095, 0, 4095, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4095, 0, 0, 0, 0, 4093, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 4096, 4089, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3006, 0, 0, 0, 0, 26, 16, 0, 0, 0, 4096, 4096, 0, 4096, 4095, 0, 4096, 0, 0, 4093, 0, 2, 0, 0, 1352, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 4052, 0, 0, 0, 544, 4036, 0, 0, 0, 0, 0, 0, 4096, 2105, 0, 0, 0, 4096, 0, 7, 1016, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 231, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4092, 4092, 3, 3, 0, 5, 4094, 0, 4089, 0, 4094, 1, 0, 0, 0, 0, 4090, 4092, 4091, 4092, 3, 4091, 4093, 0, 2, 0, 0, 2, 6, 0, 0, 3322, 4, 0, 2, 0, 0, 17, 2, 4, 0, 4090, 4092, 0, 4093, 3930, 4, 4094, 3, 0, 4092, 0, 0, 2, 0, 1088, 4, 3, 0, 4, 3, 0, 0, 4083, 1, 0, 1, 3, 3573, 0, 2, 0, 1050, 4055, 1, 3, 1, 2, 0, 0, 4090, 1689, 0, 4, 1, 4094, 0, 0, 1740, 0, 3, 2, 5, 0, 4094, 7, 2, 4086, 4091, 1, 4093, 0, 0, 4093, 10, 1, 1, 0, 4094, 3, 0, 0, 4, 0, 3, 0, 2, 0, 3, 0, 2, 6, 0, 4091, 0, 4092, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 976, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1378, 0, 0, 0, 0, 0, 0, 0, 4096, 3251, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 175, 0, 4095, 0, 0, 0, 1048, 4076, 0, 0, 0, 0, 0, 2894, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 801, 0, 0, 0, 4095, 0, 0, 1349, 0, 0, 0, 0, 0, 4096, 537, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3559, 0, 0, 0, 0, 4096, 698, 0, 0, 0, 4096, 0, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16416, 15826, 16919, 17226, 15200, 16717, 16382] avg=16384.0000 max=17226 max_rank=4 min=15200 min_rank=5 imbalance_ratio=1.051392 max_load_reduction=113846 max_load_reduction_pct=0.868576 moved_to_echo=65525 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2510, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 4096, 0, 976, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 531, 0, 0, 0, 0, 0, 0, 0, 4096, 3251, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 4095, 0, 0, 0, 511, 3988, 0, 0, 0, 0, 0, 2894, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1429, 0, 0, 0, 4096, 0, 0, 2304, 2, 0, 0, 0, 0, 4096, 537, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3559, 0, 0, 0, 0, 4096, 976, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16402, 15773, 16071, 16445, 16786, 16827, 16382] avg=16384.0000 max=16827 max_rank=6 min=15773 min_rank=2 imbalance_ratio=1.027039 max_load_reduction=114245 max_load_reduction_pct=0.871620 moved_to_echo=66440 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3572, 0, 0, 0, 0, 19, 22, 0, 0, 0, 4096, 4096, 0, 976, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 602, 0, 0, 0, 0, 0, 0, 0, 4096, 3251, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 26, 0, 4093, 0, 0, 0, 373, 3865, 0, 0, 0, 0, 0, 2894, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2173, 0, 0, 0, 4096, 0, 0, 1241, 2, 0, 0, 0, 0, 4096, 537, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3559, 0, 0, 0, 0, 4096, 406, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16377, 16873, 16143, 16191, 16467, 16254, 16382] avg=16384.0000 max=16873 max_rank=2 min=16143 min_rank=3 imbalance_ratio=1.029846 max_load_reduction=114199 max_load_reduction_pct=0.871269 moved_to_echo=65294 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4088, 6, 6, 3, 9, 4091, 0, 4090, 0, 4089, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4080, 4083, 4087, 4, 0, 0, 0, 4, 0, 0, 4, 18, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3000, 7, 0, 0, 0, 0, 0, 5, 0, 0, 4087, 4091, 0, 966, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 4091, 0, 0, 4, 0, 1012, 6, 6, 0, 6, 1, 8, 0, 4088, 3248, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4, 0, 1, 5, 4052, 0, 4, 4, 337, 3831, 4, 5, 0, 6, 4, 2891, 842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4085, 1251, 5, 0, 0, 4092, 3, 0, 2828, 0, 6, 4, 6, 0, 4090, 538, 233, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 4055, 3551, 3, 0, 0, 0, 4075, 45, 5, 1, 0, 4088, 8, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 2, 5, 5, 5, 0, 5, 0, 0, 4089, 1, 4086, 5, 4090, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16365, 16252, 16567, 16077, 17153, 15882, 16387] avg=16384.0000 max=17153 max_rank=5 min=15882 min_rank=6 imbalance_ratio=1.046936 max_load_reduction=113919 max_load_reduction_pct=0.869133 moved_to_echo=65499 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4091, 0, 0, 0, 0, 4090, 0, 4095, 0, 4096, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 12, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3396, 0, 3, 0, 0, 32, 30, 0, 0, 0, 4096, 4091, 0, 976, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 7, 3, 0, 1, 744, 0, 0, 0, 0, 0, 0, 0, 4096, 3251, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 4095, 0, 0, 0, 152, 4039, 0, 0, 3, 0, 0, 2894, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 389, 0, 0, 9, 4071, 0, 0, 3461, 1, 0, 0, 0, 0, 4092, 537, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3559, 0, 0, 0, 7, 4096, 82, 0, 0, 0, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16389, 16696, 16291, 16122, 16882, 15933, 16380] avg=16384.0000 max=16882 max_rank=5 min=15933 min_rank=6 imbalance_ratio=1.030396 max_load_reduction=114190 max_load_reduction_pct=0.871201 moved_to_echo=65317 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4092, 3, 3, 0, 5, 4094, 0, 4089, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4092, 4091, 4092, 3, 0, 0, 0, 2, 0, 0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3322, 4, 0, 2, 0, 0, 17, 2, 4, 0, 4090, 4092, 0, 976, 3930, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 4092, 0, 0, 2, 0, 1088, 4, 3, 0, 4, 3, 0, 0, 4091, 3248, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 1, 0, 1, 3, 3573, 0, 2, 0, 1050, 4055, 1, 3, 1, 2, 0, 2891, 844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1689, 0, 4, 1, 4094, 0, 0, 1740, 0, 3, 2, 5, 0, 4094, 536, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 4086, 3555, 1, 0, 0, 0, 4093, 10, 1, 1, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 0, 2, 0, 3, 0, 2, 6, 0, 4091, 0, 4092, 2, 4093, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16378, 16443, 16634, 16510, 16484, 15853, 16389] avg=16384.0000 max=16634 max_rank=3 min=15853 min_rank=6 imbalance_ratio=1.015259 max_load_reduction=114438 max_load_reduction_pct=0.873093 moved_to_echo=65236 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 0, 0, 0, 0, 4093, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3006, 0, 0, 0, 0, 26, 16, 0, 0, 0, 4096, 4096, 0, 976, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 2, 0, 0, 1352, 0, 0, 0, 0, 0, 0, 0, 4096, 3253, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 4052, 0, 0, 0, 544, 4036, 0, 0, 0, 0, 0, 2894, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2105, 0, 0, 0, 4096, 0, 7, 1016, 0, 0, 0, 0, 0, 4096, 537, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3559, 0, 0, 0, 0, 4096, 231, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 4096, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16376, 16311, 16895, 16468, 16179, 16078, 16385] avg=16384.0000 max=16895 max_rank=3 min=16078 min_rank=6 imbalance_ratio=1.031189 max_load_reduction=114177 max_load_reduction_pct=0.871101 moved_to_echo=65110 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4095, 1, 1, 0, 0, 4095, 1, 4095, 0, 4094, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 4094, 4081, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3593, 0, 0, 0, 0, 29, 13, 1, 0, 0, 4095, 4095, 0, 976, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 0, 0, 1, 0, 11, 1, 1, 0, 1, 0, 1, 0, 4095, 3250, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 4082, 1, 1, 0, 164, 3951, 0, 0, 0, 1, 0, 2893, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1525, 1, 0, 0, 4094, 0, 3, 1368, 0, 1, 1, 1, 1, 4071, 536, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4093, 3558, 1, 0, 0, 0, 4095, 1680, 1, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 4095, 0, 4095, 1, 4095, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16369, 16898, 15552, 16033, 15921, 17528, 16385] avg=16384.0000 max=17528 max_rank=6 min=15552 min_rank=3 imbalance_ratio=1.069824 max_load_reduction=113544 max_load_reduction_pct=0.866272 moved_to_echo=65867 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 128, 32768, 128, 32768, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 0, 32768, 128, 128, 128, 128, 6784, 128, 128, 0, 128, 128, 128, 0, 32768, 26112, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 28544, 128, 0, 0, 128, 32768, 4224, 128, 128, 0, 32768, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[25088, 128, 128, 128, 0, 128, 128, 128, 128, 0, 32768, 32768, 0, 7808, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 0, 32768, 128, 32768, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 0, 256, 128, 32256, 128, 128, 128, 4224, 31872, 128, 128, 128, 128, 128, 23168, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 32768, 11392, 128, 128, 128, 32768, 128, 128, 15360, 128, 128, 128, 128, 128, 32768, 4352, 1920, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 32768, 32768, 32768, 128, 0, 0, 0, 128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 524, 4096, 0, 362, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4094, 0, 0, 515, 4096, 4096, 0, 4015, 0, 0, 0, 0, 0, 4096, 4094, 4096, 4096, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 4096, 209, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2911, 0, 0, 0, 0, 0, 0, 3786, 4096, 4096, 0, 0, 0, 1, 4096, 4096, 0, 4096, 0, 0, 0, 4095, 0, 4082] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 180, 4096, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 4096, 0, 0, 446, 4096, 4096, 0, 3985, 0, 0, 0, 0, 0, 4096, 4083, 4096, 4096, 4093, 1, 0, 0, 4091, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 4096, 5, 4096, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 0, 0, 0, 0, 3923, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 3996] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[4, 0, 4089, 5, 7, 22, 6, 0, 0, 4065, 6, 0, 4091, 0, 4088, 4085, 4, 3, 5, 0, 6, 5, 1, 8, 4, 7, 0, 7, 3, 0, 7, 4064, 0, 1037, 4089, 0, 102, 5, 0, 0, 4089, 9, 15, 0, 8, 0, 9, 9, 0, 2, 0, 0, 4086, 6, 4084, 0, 57, 630, 4090, 4072, 0, 4091, 0, 6, 0, 0, 6, 4085, 4085, 4089, 4088, 4072, 0, 23, 8, 4093, 1, 3, 0, 0, 4088, 4087, 0, 0, 0, 13, 4089, 107, 4070, 0, 0, 0, 0, 5, 0, 6, 0, 2, 4, 27, 11, 0, 0, 0, 3161, 0, 5, 7, 0, 10, 0, 3294, 4087, 4034, 0, 0, 4, 0, 4088, 4091, 7, 4089, 0, 8, 2, 4085, 0, 3950] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4011, 0, 6, 4092, 1, 4096, 4096, 0, 0, 0, 5, 0, 0, 10, 0, 0, 0, 9, 0, 0, 0, 0, 4088, 0, 156, 4096, 0, 560, 0, 0, 0, 4087, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4055, 0, 4096, 0, 0, 2231, 4096, 4091, 0, 4095, 0, 0, 0, 0, 0, 4096, 4095, 4096, 4091, 4096, 10, 0, 0, 4072, 0, 0, 8, 2, 4088, 4096, 0, 3, 0, 13, 4096, 404, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1331, 2, 0, 0, 0, 0, 0, 3710, 4096, 4089, 0, 0, 0, 0, 4096, 4093, 0, 4092, 0, 0, 0, 4096, 0, 4026] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 549, 4096, 0, 29, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 0, 7, 1403, 4096, 4090, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 4082, 0, 0, 0, 0, 4096, 4096, 0, 12, 0, 9, 4096, 847, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1698, 0, 0, 0, 0, 0, 3, 3693, 4096, 4094, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4060] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[2, 0, 4093, 4, 3, 1, 3, 0, 0, 4086, 3, 6, 4093, 0, 4092, 4090, 0, 0, 2, 0, 3, 4, 10, 4, 0, 3, 0, 4, 1, 5, 3, 4088, 0, 658, 4093, 0, 565, 3, 3, 0, 4091, 2, 0, 0, 2, 0, 5, 4, 3, 0, 0, 0, 4092, 3, 4092, 0, 0, 887, 4093, 4083, 0, 4092, 0, 3, 0, 0, 3, 4085, 4092, 4093, 4087, 4092, 13, 3, 7, 4094, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 2, 4093, 1527, 4089, 0, 0, 0, 0, 2, 0, 3, 4, 3, 4, 37, 4, 0, 0, 0, 604, 0, 2, 3, 4, 0, 0, 4074, 4093, 4057, 0, 1, 2, 0, 4092, 4088, 3, 4093, 2, 8, 2, 4090, 0, 3954] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 4094, 0, 1652, 4096, 0, 676, 0, 0, 0, 4095, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 1030, 4096, 4096, 0, 4096, 5, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 4086, 0, 0, 2, 0, 4094, 4096, 0, 12, 0, 9, 4096, 274, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1409, 0, 0, 0, 0, 0, 2, 3492, 4096, 4095, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 3716] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 4095, 1, 1, 4, 1, 0, 0, 4086, 1, 10, 4095, 0, 4095, 4094, 0, 0, 1, 2, 1, 1, 13, 1, 0, 1, 1, 1, 0, 0, 1, 4085, 0, 1007, 4094, 0, 44, 1, 1, 0, 4089, 1, 0, 3, 1, 0, 0, 1, 0, 0, 0, 0, 4094, 1, 4094, 0, 0, 590, 4095, 4095, 0, 4096, 0, 2, 0, 0, 1, 4094, 4094, 4095, 4094, 4095, 2, 1, 0, 4095, 0, 0, 8, 0, 4092, 4093, 0, 0, 0, 4, 4095, 119, 4092, 1, 0, 2, 0, 1, 0, 1, 0, 0, 1, 0, 2, 1, 0, 0, 3178, 0, 1, 1, 0, 0, 0, 3282, 4095, 4087, 1, 0, 1, 0, 4095, 4095, 1, 4095, 0, 0, 2, 4094, 0, 4060] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 15, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 4096, 0, 362, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3966, 1831, 936, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3158, 0, 0, 515, 0, 4096, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 6, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 3656, 209, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2911, 0, 0, 0, 0, 0, 0, 3786, 4096, 4090, 2250, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 130, 0, 0, 0, 4095, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16375, 16251, 15879, 16389, 16153, 17142, 16500] avg=16384.0000 max=17142 max_rank=6 min=15879 min_rank=3 imbalance_ratio=1.046265 max_load_reduction=113930 max_load_reduction_pct=0.869217 moved_to_echo=66184 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 7, 5, 7, 22, 6, 0, 0, 4065, 6, 0, 4091, 0, 4088, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 5, 0, 6, 5, 1, 8, 4, 7, 0, 7, 3, 0, 7, 4064, 4090, 4089, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 4089, 0, 102, 5, 0, 0, 4089, 9, 15, 0, 8, 0, 9, 9, 3963, 1831, 937, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4086, 6, 3147, 0, 57, 630, 0, 4072, 0, 4091, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4085, 4085, 0, 3, 4072, 0, 23, 8, 4093, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4087, 0, 0, 0, 13, 3646, 107, 4070, 0, 0, 0, 0, 5, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 27, 11, 0, 0, 0, 3161, 0, 5, 7, 0, 10, 0, 3294, 4088, 4085, 2251, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 4, 0, 0, 4091, 7, 126, 0, 8, 2, 4085, 0, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16382, 16546, 16097, 16379, 16022, 16953, 16307] avg=16384.0000 max=16953 max_rank=6 min=16022 min_rank=5 imbalance_ratio=1.034729 max_load_reduction=114119 max_load_reduction_pct=0.870659 moved_to_echo=65661 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 15, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 4096, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3966, 1831, 936, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 3160, 0, 0, 446, 0, 4096, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 6, 4093, 1, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 3656, 5, 4096, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 0, 0, 0, 0, 3923, 4096, 4090, 2250, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 130, 0, 0, 0, 4096, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16393, 16375, 15551, 15783, 16370, 15954, 18232, 16414] avg=16384.0000 max=18232 max_rank=6 min=15551 min_rank=2 imbalance_ratio=1.112793 max_load_reduction=112840 max_load_reduction_pct=0.860901 moved_to_echo=66970 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 16, 4, 3, 1, 3, 0, 0, 4086, 3, 6, 4093, 0, 4092, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 4, 10, 4, 0, 3, 0, 4, 1, 5, 3, 4088, 4093, 4093, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 658, 4093, 0, 565, 3, 3, 0, 4091, 2, 0, 0, 2, 0, 5, 4, 3963, 1829, 935, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4092, 3, 3157, 0, 0, 887, 0, 4083, 0, 4092, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4085, 4092, 0, 6, 4092, 13, 3, 7, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 2, 3653, 1527, 4089, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 4, 37, 4, 0, 0, 0, 604, 0, 2, 3, 4, 0, 0, 4074, 4092, 4081, 2248, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 1, 2, 0, 0, 4088, 3, 130, 2, 8, 2, 4090, 0, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16399, 16398, 16593, 16320, 16395, 17462, 15168, 16337] avg=16384.0000 max=17462 max_rank=5 min=15168 min_rank=6 imbalance_ratio=1.065796 max_load_reduction=113610 max_load_reduction_pct=0.866776 moved_to_echo=65362 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 15, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 549, 4096, 0, 29, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3966, 1831, 936, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 3160, 0, 7, 1403, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 6, 4096, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 12, 0, 9, 3656, 847, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1698, 0, 0, 0, 0, 0, 3, 3693, 4096, 4090, 2250, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 130, 0, 0, 0, 4096, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16399, 16376, 15943, 16843, 16376, 16820, 15839, 16476] avg=16384.0000 max=16843 max_rank=3 min=15839 min_rank=6 imbalance_ratio=1.028015 max_load_reduction=114229 max_load_reduction_pct=0.871498 moved_to_echo=65511 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 15, 0, 0, 0, 0, 0, 0, 4011, 0, 6, 4092, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 10, 0, 0, 0, 9, 0, 0, 0, 0, 4088, 4096, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 4096, 0, 560, 0, 0, 0, 4087, 0, 0, 0, 0, 4, 0, 0, 3962, 1831, 936, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 3160, 0, 0, 2231, 0, 4091, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 6, 4096, 10, 0, 0, 4072, 0, 0, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 3, 0, 13, 3656, 404, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1331, 2, 0, 0, 0, 0, 0, 3710, 4096, 4085, 2250, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4093, 0, 130, 0, 0, 0, 4096, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16317, 16391, 16072, 17632, 16385, 16357, 15484, 16434] avg=16384.0000 max=17632 max_rank=3 min=15484 min_rank=6 imbalance_ratio=1.076172 max_load_reduction=113440 max_load_reduction_pct=0.865479 moved_to_echo=64660 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 15, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 4094, 4096, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1652, 4096, 0, 676, 0, 0, 0, 4095, 0, 0, 0, 0, 11, 0, 0, 3966, 1831, 936, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3160, 0, 0, 1030, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 6, 4096, 0, 0, 0, 4086, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 12, 0, 9, 3656, 274, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1409, 0, 0, 0, 0, 0, 2, 3492, 4096, 4090, 2250, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 130, 0, 0, 0, 4096, 0, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16391, 17703, 16482, 16382, 16244, 15347, 16133] avg=16384.0000 max=17703 max_rank=2 min=15347 min_rank=6 imbalance_ratio=1.080505 max_load_reduction=113369 max_load_reduction_pct=0.864937 moved_to_echo=64106 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 16, 1, 1, 4, 1, 0, 0, 4086, 1, 10, 4095, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 1, 13, 1, 0, 1, 1, 1, 0, 0, 1, 4085, 4095, 4095, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 4094, 0, 44, 1, 1, 0, 4089, 1, 0, 3, 1, 0, 0, 1, 3965, 1830, 936, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 3158, 0, 0, 590, 0, 4095, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 4094, 0, 6, 4095, 2, 1, 0, 4095, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4093, 0, 0, 0, 4, 3655, 119, 4092, 1, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 1, 0, 0, 3178, 0, 1, 1, 0, 0, 0, 3282, 4095, 4088, 2249, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 0, 1, 0, 0, 4095, 1, 130, 0, 0, 2, 4094, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16405, 16384, 16413, 16036, 16396, 16060, 16907, 16471] avg=16384.0000 max=16907 max_rank=6 min=16036 min_rank=3 imbalance_ratio=1.031921 max_load_reduction=114165 max_load_reduction_pct=0.871010 moved_to_echo=65834 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 0, 0, 32640, 128, 128, 32768, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 0, 32768, 128, 1152, 128, 128, 128, 32768, 0, 31872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 32768, 32768, 0, 128, 32768, 128, 128, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 25344, 0, 128, 7808, 0, 32768, 0, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 32768, 0, 128, 0, 128, 29312, 3584, 32768, 128, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 0, 18176, 128, 128, 128, 128, 128, 128, 29312, 32768, 32768, 18048, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 5888, 32768, 0, 2432, 128, 128, 0, 32768, 128, 128, 128, 128, 128, 128, 128, 31744, 14720, 7552, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132992 observed_loss=-1920 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132992 slack_slots=129152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 78, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 121, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 2906, 0, 0, 0, 5, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3460, 0, 0, 1522, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 40, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[8, 7, 4087, 0, 0, 5, 0, 226, 7, 5, 3, 6, 6, 6, 0, 4090, 0, 0, 0, 0, 4088, 4090, 2, 4076, 3, 3, 4088, 7, 0, 0, 4087, 0, 4087, 9, 0, 4089, 9, 0, 2570, 0, 5, 0, 0, 0, 4089, 9, 0, 4076, 8, 4090, 4090, 4027, 3, 0, 0, 0, 0, 8, 0, 0, 5, 0, 0, 8, 7, 0, 5, 0, 6, 3392, 8, 0, 2138, 0, 0, 3, 0, 7, 0, 4089, 0, 3982, 0, 4088, 5, 4084, 4090, 4090, 4087, 6, 4087, 1, 0, 0, 4084, 4089, 6, 5, 7, 39, 0, 7, 0, 4086, 3, 0, 6, 0, 4087, 6, 4084, 0, 7, 4081, 3, 0, 7, 7, 4089, 0, 6, 4087, 0, 2, 0, 0, 17, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4083, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4093, 0, 0, 4096, 0, 0, 3446, 0, 0, 0, 4, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3319, 0, 0, 1376, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 5, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 4, 98, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 314, 0, 4096, 4093, 0, 4096, 0, 0, 4075, 0, 0, 0, 4096, 3, 4069, 0, 0, 4089, 0, 5, 3524, 0, 0, 0, 5, 0, 4096, 0, 0, 4096, 0, 4091, 4096, 4090, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2807, 0, 0, 1340, 0, 0, 0, 0, 0, 29, 4096, 0, 4095, 107, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 6, 0, 26, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 5, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 139, 0, 4096, 4088, 0, 4096, 0, 0, 4092, 0, 8, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 2415, 0, 0, 0, 12, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3828, 0, 0, 1593, 0, 0, 0, 0, 0, 27, 4096, 0, 4096, 77, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 19, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 79, 0, 4096, 4089, 0, 4096, 0, 0, 4092, 0, 364, 0, 4096, 2, 4096, 0, 0, 4096, 0, 0, 2497, 0, 0, 0, 16, 0, 4096, 0, 0, 4096, 0, 4096, 4095, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2314, 0, 0, 2724, 0, 0, 0, 0, 0, 52, 4096, 0, 4096, 87, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 7, 4096, 4086, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 8, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[3, 2, 4093, 0, 0, 1, 0, 196, 3, 1, 0, 2, 2, 4, 1, 4092, 0, 0, 1, 0, 4092, 4093, 0, 4091, 1, 4, 4092, 3, 47, 0, 4090, 0, 4093, 4, 0, 4093, 0, 0, 3615, 3, 0, 0, 4, 0, 4093, 4, 0, 4088, 4, 4092, 4093, 4066, 4, 0, 0, 3, 0, 4, 0, 0, 2, 0, 0, 3, 3, 0, 3, 0, 3, 3235, 0, 0, 555, 0, 0, 0, 0, 3, 3, 4092, 0, 4086, 536, 4092, 1, 4090, 4092, 4093, 4092, 2, 4092, 0, 0, 0, 4091, 4079, 3, 2, 4, 51, 0, 4, 0, 4091, 4, 2, 3, 0, 4092, 2, 4089, 1, 4, 4085, 3, 1, 3, 3, 4091, 0, 3, 4092, 0, 9, 0, 0, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 4016, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 4095, 0, 0, 3, 0, 4095, 4092, 1, 4095, 0, 0, 4091, 1, 53, 0, 4094, 0, 4093, 0, 0, 4095, 0, 0, 2586, 0, 1, 0, 23, 0, 4095, 1, 0, 4093, 1, 4095, 4095, 4072, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 3757, 2, 0, 1740, 1, 0, 0, 0, 1, 26, 4095, 0, 4046, 157, 4095, 1, 4095, 4095, 4094, 4095, 0, 4095, 0, 0, 0, 4094, 4087, 1, 1, 1, 13, 0, 0, 0, 4094, 0, 0, 1, 0, 4095, 1, 4095, 0, 1, 4094, 0, 1, 1, 1, 4095, 1, 1, 4095, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 121, 0, 3955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2906, 0, 0, 0, 5, 0, 1149, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3205, 901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3460, 0, 0, 1522, 0, 0, 0, 0, 0, 0, 4096, 4096, 3195, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 40, 0, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 2947, 894, 141, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3202, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16356, 16442, 16348, 16394, 16460, 16322, 16369, 16381] avg=16384.0000 max=16460 max_rank=4 min=16322 min_rank=5 imbalance_ratio=1.004639 max_load_reduction=114612 max_load_reduction_pct=0.874420 moved_to_echo=65532 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[8, 7, 4087, 0, 0, 5, 0, 226, 7, 5, 3, 6, 6, 6, 0, 4090, 4090, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 4076, 3, 3, 4088, 7, 0, 0, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 9, 0, 4089, 9, 0, 2570, 0, 5, 0, 0, 0, 1142, 9, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4090, 4090, 4027, 3, 0, 0, 0, 0, 8, 0, 0, 5, 0, 0, 8, 3199, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 5, 0, 6, 3392, 8, 0, 2138, 0, 0, 3, 0, 7, 0, 4089, 4088, 3188, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 0, 0, 5, 3982, 0, 0, 0, 6, 0, 1, 0, 0, 4084, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 7, 39, 0, 7, 0, 4086, 3, 0, 6, 0, 4087, 6, 4084, 0, 2947, 893, 142, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4081, 3, 0, 7, 7, 4089, 0, 6, 4087, 0, 2, 0, 0, 17, 0, 3194, 888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16546, 16214, 15996, 16338, 17021, 16149, 16420, 16388] avg=16384.0000 max=17021 max_rank=4 min=15996 min_rank=2 imbalance_ratio=1.038879 max_load_reduction=114051 max_load_reduction_pct=0.870140 moved_to_echo=65978 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4083, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 3955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 3446, 0, 0, 0, 4, 0, 1149, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3319, 0, 0, 1376, 0, 0, 0, 0, 0, 0, 4096, 4096, 3195, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 0, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 2947, 894, 141, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3202, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16329, 16254, 16884, 16395, 16173, 16288, 16369, 16380] avg=16384.0000 max=16884 max_rank=2 min=16173 min_rank=4 imbalance_ratio=1.030518 max_load_reduction=114188 max_load_reduction_pct=0.871185 moved_to_echo=65210 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 4088, 0, 4096, 0, 0, 4092, 0, 8, 0, 3955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2415, 0, 0, 0, 12, 0, 1149, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3828, 0, 0, 1593, 0, 0, 0, 0, 0, 27, 4096, 4096, 3194, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 77, 0, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 2947, 894, 141, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3202, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16394, 16378, 15864, 16402, 16924, 16361, 16369, 16380] avg=16384.0000 max=16924 max_rank=4 min=15864 min_rank=2 imbalance_ratio=1.032959 max_load_reduction=114148 max_load_reduction_pct=0.870880 moved_to_echo=66034 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4090, 0, 0, 0, 4, 98, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 4093, 0, 4096, 0, 0, 4075, 0, 0, 0, 3955, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 4089, 0, 5, 3524, 0, 0, 0, 5, 0, 1149, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 4090, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3205, 901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2807, 0, 0, 1340, 0, 0, 0, 0, 0, 29, 4096, 4096, 3195, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 107, 0, 0, 3997, 0, 0, 0, 0, 0, 6, 0, 26, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 2947, 894, 141, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 3202, 891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16536, 16937, 16390, 15654, 16416, 16369, 16386] avg=16384.0000 max=16937 max_rank=2 min=15654 min_rank=4 imbalance_ratio=1.033752 max_load_reduction=114135 max_load_reduction_pct=0.870781 moved_to_echo=64825 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 2, 4093, 0, 0, 1, 0, 196, 3, 1, 0, 2, 2, 4, 1, 4092, 4092, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 4091, 1, 4, 4092, 3, 47, 0, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 0, 4093, 0, 0, 3615, 3, 0, 0, 4, 0, 1148, 4, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 4093, 4066, 4, 0, 0, 3, 0, 4, 0, 0, 2, 0, 0, 3, 3203, 901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 3, 3235, 0, 0, 555, 0, 0, 0, 0, 3, 3, 4092, 4092, 3191, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 536, 0, 1, 3991, 0, 0, 0, 2, 0, 0, 0, 0, 4091, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 4, 51, 0, 4, 0, 4091, 4, 2, 3, 0, 4092, 2, 4089, 1, 2945, 893, 140, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4085, 3, 1, 3, 3, 4091, 0, 3, 4092, 0, 9, 0, 0, 5, 0, 3199, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16495, 16282, 17052, 16375, 15270, 16786, 16425, 16387] avg=16384.0000 max=17052 max_rank=2 min=15270 min_rank=4 imbalance_ratio=1.040771 max_load_reduction=114020 max_load_reduction_pct=0.869904 moved_to_echo=64868 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 4089, 0, 4096, 0, 0, 4092, 0, 364, 0, 3955, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2497, 0, 0, 0, 16, 0, 1149, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2314, 0, 0, 2724, 0, 0, 0, 0, 0, 52, 4096, 4096, 3194, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 87, 0, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 2947, 894, 141, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 3202, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16350, 16677, 15950, 16402, 16566, 16369, 16370, 16388] avg=16384.0000 max=16677 max_rank=1 min=15950 min_rank=2 imbalance_ratio=1.017883 max_load_reduction=114395 max_load_reduction_pct=0.872765 moved_to_echo=65693 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 4016, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 4095, 4095, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4092, 1, 4095, 0, 0, 4091, 1, 53, 0, 3953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4095, 0, 0, 2586, 0, 1, 0, 23, 0, 1149, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4095, 4072, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 3206, 902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 3757, 2, 0, 1740, 1, 0, 0, 0, 1, 26, 4095, 4095, 3193, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 157, 0, 1, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 13, 0, 0, 0, 4094, 0, 0, 1, 0, 4095, 1, 4095, 0, 2946, 893, 141, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 1, 1, 1, 4095, 1, 1, 4095, 0, 1, 0, 0, 0, 0, 3202, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16218, 16289, 16041, 16376, 17004, 16381, 16381, 16382] avg=16384.0000 max=17004 max_rank=4 min=16041 min_rank=2 imbalance_ratio=1.037842 max_load_reduction=114068 max_load_reduction_pct=0.870270 moved_to_echo=66148 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 0, 32768, 128, 128, 23680, 128, 128, 0, 128, 0, 9216, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 32768, 32640, 128, 128, 0, 128, 0, 128, 0, 0, 128, 128, 0, 128, 25728, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 640, 0, 0, 32768, 128, 32768, 128, 128, 32768, 128, 640, 0, 31744, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 32640, 1024, 0, 128, 32000, 0, 0, 0, 128, 0, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 32768, 128, 128, 32768, 0, 128, 128, 0, 128, 0, 25728, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 0, 128, 26112, 128, 0, 13056, 128, 0, 128, 0, 128, 256, 32768, 32768, 25600, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 32768, 0, 0, 128, 128, 896, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 0, 128, 0, 32768, 128, 128, 128, 0, 32768, 128, 32768, 128, 23680, 7168, 1152, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 2746, 0, 4096, 0, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 4094, 0, 4096, 0, 4081, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 917, 4094, 4096, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 454, 0, 4096, 0, 4096, 4, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 939, 0, 4096, 0, 0, 0, 42, 4096, 0, 4096, 0, 0, 0, 0, 4090, 0, 4096, 0, 4086, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 2232, 4096, 4096, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 903, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4089, 0, 8, 0, 4088, 10, 7, 436, 0, 4088, 0, 0, 2, 156, 4089, 7, 4088, 0, 5, 0, 7, 3753, 0, 4089, 6, 4055, 7, 0, 4088, 1, 4066, 4087, 1, 4086, 9, 0, 0, 8, 7, 4090, 7, 4085, 3975, 0, 0, 0, 0, 9, 0, 8, 14, 4086, 0, 0, 0, 2, 7, 3, 0, 10, 137, 4081, 0, 0, 0, 4, 6, 0, 0, 0, 7, 5, 4083, 4082, 6, 8, 31, 4086, 0, 0, 0, 0, 7, 4075, 0, 6, 0, 4, 0, 0, 4, 4087, 4, 11, 0, 7, 4089, 90, 4091, 4081, 0, 0, 19, 3954, 7, 0, 4074, 0, 0, 7, 0, 3, 0, 0, 4089, 4090, 0, 9, 0, 3852, 0, 4087, 0, 4089, 46, 8, 7, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 3034, 0, 4096, 0, 0, 0, 8, 4096, 0, 4096, 0, 0, 0, 0, 4093, 0, 4096, 0, 4085, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3832, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 154, 4093, 4096, 0, 0, 0, 4058, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1020, 0, 4096, 0, 4096, 151, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4059, 0, 4, 0, 4091, 0, 3, 3794, 0, 4092, 0, 0, 6, 28, 4093, 3, 4092, 0, 4, 0, 3, 4087, 1, 4092, 3, 4070, 1, 0, 4091, 0, 4084, 4092, 0, 4093, 4, 0, 0, 4, 4, 4093, 3, 4088, 4064, 0, 1, 0, 0, 4, 0, 4, 5, 4090, 0, 0, 0, 5, 0, 2, 0, 0, 0, 4092, 0, 0, 0, 3, 4, 9, 0, 0, 3, 1, 4092, 4091, 2, 4, 8, 4092, 1, 4, 0, 0, 3, 4090, 0, 3, 0, 2, 0, 0, 0, 4089, 3, 5, 0, 2, 4092, 354, 4094, 4090, 0, 0, 7, 3902, 4, 0, 4083, 0, 0, 6, 0, 1, 3, 0, 4092, 4093, 0, 4, 4, 167, 0, 4092, 0, 4092, 25, 4, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4091, 5, 0, 1, 4096, 0, 0, 1744, 0, 4096, 4, 3, 0, 5, 4096, 0, 4096, 9, 0, 0, 0, 3934, 1, 4096, 0, 4073, 0, 0, 4095, 0, 4096, 4096, 0, 4096, 0, 0, 4, 0, 0, 4091, 0, 4096, 4078, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 32, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 438, 4084, 4096, 0, 0, 0, 4009, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 2127, 0, 4096, 0, 4096, 7, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 2627, 0, 4096, 0, 0, 0, 127, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4060, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 31, 0, 0, 0, 0, 44, 4096, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 727, 4096, 4096, 0, 0, 0, 4023, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 616, 0, 4096, 0, 4096, 49, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4092, 0, 1, 0, 4094, 0, 1, 3410, 0, 4095, 0, 0, 0, 7, 4095, 1, 4095, 0, 1, 0, 1, 4094, 0, 4095, 1, 4095, 2, 1, 4094, 1, 4094, 4095, 0, 4094, 1, 0, 0, 1, 1, 4095, 1, 4092, 4045, 0, 0, 0, 0, 1, 0, 0, 2, 4094, 0, 0, 0, 0, 2, 0, 0, 1, 7, 4095, 0, 0, 4, 1, 1, 13, 0, 0, 1, 1, 4095, 4095, 1, 1, 3, 4094, 0, 2, 0, 0, 1, 4093, 2, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 1, 4095, 686, 4095, 4094, 1, 1, 2, 4055, 1, 0, 4090, 1, 0, 0, 0, 0, 0, 1, 4095, 4095, 0, 1, 0, 16, 0, 4093, 0, 4095, 51, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 2746, 0, 4096, 0, 0, 0, 2, 1711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4081, 0, 0, 4096, 0, 4096, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4009, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2337, 1185, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 917, 4094, 3476, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2911, 4096, 0, 0, 0, 454, 0, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16747, 16454, 16444, 16347, 16370, 16385, 16668, 15657] avg=16384.0000 max=16747 max_rank=0 min=15657 min_rank=7 imbalance_ratio=1.022156 max_load_reduction=114325 max_load_reduction_pct=0.872231 moved_to_echo=65080 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 939, 0, 4096, 0, 0, 0, 42, 1711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4086, 0, 0, 4096, 0, 4096, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4009, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2337, 1185, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2232, 4096, 3476, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2911, 4096, 0, 0, 0, 903, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14980, 16455, 16444, 16344, 16370, 16385, 17992, 16102] avg=16384.0000 max=17992 max_rank=6 min=14980 min_rank=0 imbalance_ratio=1.098145 max_load_reduction=113080 max_load_reduction_pct=0.862732 moved_to_echo=66849 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4089, 0, 8, 0, 4088, 10, 7, 436, 0, 4088, 0, 0, 2, 156, 1702, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7, 3753, 0, 0, 6, 4055, 7, 0, 4088, 1, 4066, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 9, 0, 0, 8, 7, 4090, 7, 4085, 3975, 0, 0, 0, 0, 9, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 14, 4086, 0, 0, 0, 2, 7, 3, 0, 10, 137, 4081, 0, 0, 4088, 4001, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 0, 0, 0, 7, 5, 4083, 4082, 6, 8, 31, 4086, 0, 0, 2336, 1185, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4075, 0, 6, 0, 4, 0, 0, 4, 4087, 4, 11, 0, 7, 4089, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 4091, 3456, 0, 0, 19, 3954, 7, 0, 4074, 0, 0, 7, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2904, 4090, 0, 9, 0, 3852, 0, 4087, 0, 4089, 46, 8, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14593, 16074, 16338, 16488, 16403, 16383, 15701, 19092] avg=16384.0000 max=19092 max_rank=7 min=14593 min_rank=0 imbalance_ratio=1.165283 max_load_reduction=111980 max_load_reduction_pct=0.854340 moved_to_echo=67579 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4091, 5, 0, 1, 4096, 0, 0, 1744, 0, 4096, 4, 3, 0, 5, 1711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 3934, 1, 0, 0, 4073, 0, 0, 4095, 0, 4096, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4091, 0, 4096, 4078, 5, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 32, 0, 0, 0, 0, 12, 4096, 0, 0, 4095, 4009, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 4096, 0, 0, 2337, 1185, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 438, 4084, 3476, 0, 0, 0, 4009, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2911, 4092, 0, 0, 0, 2127, 0, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15756, 16295, 16430, 16392, 16387, 16385, 16098, 17329] avg=16384.0000 max=17329 max_rank=7 min=15756 min_rank=0 imbalance_ratio=1.057678 max_load_reduction=113743 max_load_reduction_pct=0.867790 moved_to_echo=66199 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 2627, 0, 4096, 0, 0, 0, 127, 1711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4060, 0, 0, 4096, 0, 4096, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4079, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 31, 0, 0, 0, 0, 44, 4096, 0, 0, 4096, 4009, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2337, 1185, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 727, 4096, 3476, 0, 0, 0, 4023, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2911, 4096, 0, 0, 0, 616, 0, 4096, 0, 4096, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16753, 16435, 16427, 16420, 16379, 16384, 16410, 15864] avg=16384.0000 max=16753 max_rank=0 min=15864 min_rank=7 imbalance_ratio=1.022522 max_load_reduction=114319 max_load_reduction_pct=0.872185 moved_to_echo=65037 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4059, 0, 4, 0, 4091, 0, 3, 3794, 0, 4092, 0, 0, 6, 28, 1710, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 4087, 1, 0, 3, 4070, 1, 0, 4091, 0, 4084, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 0, 0, 4, 4, 4093, 3, 4088, 4064, 0, 1, 0, 0, 4, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 4090, 0, 0, 0, 5, 0, 2, 0, 0, 0, 4092, 0, 0, 4092, 4005, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 9, 0, 0, 3, 1, 4092, 4091, 2, 4, 8, 4092, 1, 4, 2335, 1184, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4090, 0, 3, 0, 2, 0, 0, 0, 4089, 3, 5, 0, 2, 4092, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 4094, 3471, 0, 0, 7, 3902, 4, 0, 4083, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2908, 4093, 0, 4, 4, 167, 0, 4092, 0, 4092, 25, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17790, 16431, 16418, 16343, 16392, 16381, 15922, 15395] avg=16384.0000 max=17790 max_rank=0 min=15395 min_rank=7 imbalance_ratio=1.085815 max_load_reduction=113282 max_load_reduction_pct=0.864273 moved_to_echo=64090 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 3034, 0, 4096, 0, 0, 0, 8, 1711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4085, 0, 0, 4096, 0, 4096, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3832, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 23, 0, 0, 0, 0, 19, 4096, 0, 0, 4096, 4009, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 4096, 0, 0, 2337, 1185, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 4093, 3476, 0, 0, 0, 4058, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2911, 4096, 0, 0, 0, 1020, 0, 4096, 0, 4096, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17041, 16457, 16180, 16387, 16378, 16384, 15875, 16370] avg=16384.0000 max=17041 max_rank=0 min=15875 min_rank=6 imbalance_ratio=1.040100 max_load_reduction=114031 max_load_reduction_pct=0.869987 moved_to_echo=65007 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 1, 0, 4094, 0, 1, 3410, 0, 4095, 0, 0, 0, 7, 1711, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4094, 0, 0, 1, 4095, 2, 1, 4094, 1, 4094, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 1, 1, 4095, 1, 4092, 4045, 0, 0, 0, 0, 1, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 0, 0, 2, 0, 0, 1, 7, 4095, 0, 0, 4094, 4008, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 1, 13, 0, 0, 1, 1, 4095, 4095, 1, 1, 3, 4094, 0, 2, 2336, 1185, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 2, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 1, 4095, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 686, 4095, 3474, 1, 1, 2, 4055, 1, 0, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2910, 4095, 0, 1, 0, 16, 0, 4093, 0, 4095, 51, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17412, 16471, 16391, 16351, 16393, 16385, 16406, 15263] avg=16384.0000 max=17412 max_rank=0 min=15263 min_rank=7 imbalance_ratio=1.062744 max_load_reduction=113660 max_load_reduction_pct=0.867157 moved_to_echo=64447 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 128, 32768, 128, 128, 18816, 0, 32768, 128, 128, 128, 384, 13696, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 0, 128, 32256, 128, 0, 128, 32640, 128, 128, 32768, 128, 32768, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 32768, 128, 128, 0, 128, 0, 0, 128, 32768, 128, 128, 0, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 32768, 128, 0, 0, 128, 128, 128, 0, 128, 256, 32768, 0, 0, 32768, 32128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 5632, 32768, 27904, 128, 128, 128, 32256, 128, 0, 32768, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 128, 32768, 128, 32768, 32384, 128, 128, 0, 0, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 23296, 32768, 0, 128, 128, 9216, 0, 32768, 0, 32768, 384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 128, 32768, 32768, 128, 128, 128, 32768, 128, 128, 18816, 9600, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 4027, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4095, 32, 4096, 0, 5, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4091, 0, 4096, 4096, 4, 4059, 0, 0, 1, 0, 0, 4096, 74, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4080, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4011, 3, 4091, 2, 0, 3, 4086, 3, 0, 0, 0, 0, 3, 3, 0, 0, 0, 4091, 6, 3, 3, 5, 4093, 4092, 4, 4, 4, 4090, 72, 4088, 0, 0, 0, 3, 1, 4, 4092, 2, 4093, 4092, 0, 1, 4, 0, 0, 3, 6, 4, 5, 4, 0, 4070, 0, 0, 4092, 4092, 4, 0, 0, 0, 2, 3, 1, 0, 4090, 1, 0, 4091, 4092, 4092, 3, 4, 8, 4092, 5, 4091, 4091, 4092, 3, 0, 4082, 2, 0, 4, 4091, 1, 4092, 4084, 14, 4093, 0, 0, 0, 0, 4, 4091, 0, 4, 4, 1, 4090, 2, 0, 4, 1, 0, 0, 4089, 4092, 0, 4092, 0, 0, 6, 2, 0, 2, 0, 6, 0, 15, 0, 0, 4, 0, 3, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4062, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 0, 4096, 36, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 3997, 3, 4087, 6, 0, 22, 4080, 2, 0, 0, 0, 0, 6, 7, 0, 2, 0, 4088, 8, 6, 0, 0, 4089, 4086, 7, 8, 10, 4084, 51, 4081, 0, 0, 0, 8, 0, 10, 4088, 8, 4090, 4088, 0, 0, 7, 0, 0, 4, 0, 7, 0, 8, 0, 4073, 0, 0, 4087, 4089, 8, 0, 0, 0, 6, 6, 1, 7, 4088, 2, 0, 4086, 4086, 4088, 0, 7, 8, 4086, 10, 4086, 4086, 4087, 5, 0, 4046, 7, 12, 10, 4089, 0, 4084, 4085, 3, 4089, 0, 0, 0, 0, 9, 4072, 10, 7, 10, 0, 4087, 5, 0, 7, 3, 0, 0, 4083, 4090, 0, 4087, 0, 1, 25, 2, 0, 3, 0, 11, 0, 52, 0, 0, 9, 9, 8, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4060, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4093, 27, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4095, 0, 4096, 4096, 0, 4092, 0, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4044, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4085, 47, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 3, 4072, 0, 0, 0, 0, 0, 4096, 37, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4059, 0, 4095, 1, 0, 1, 4094, 5, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4095, 0, 1, 0, 6, 4094, 4095, 1, 1, 0, 4090, 20, 4094, 0, 0, 0, 0, 0, 1, 4095, 0, 4095, 4095, 0, 0, 1, 1, 0, 0, 0, 1, 6, 1, 0, 4094, 0, 1, 4095, 4095, 1, 0, 0, 0, 1, 1, 0, 0, 4095, 1, 0, 4095, 4095, 4095, 0, 1, 3, 4093, 1, 4095, 4095, 4095, 1, 0, 4085, 1, 0, 1, 4095, 0, 4094, 4089, 0, 4095, 0, 0, 0, 0, 1, 4093, 18, 1, 1, 1, 4095, 0, 0, 1, 1, 0, 1, 4095, 4095, 0, 4095, 2, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 1, 2, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4062, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 35, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4041, 30, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4061, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 16, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 36, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16401, 16358, 16380, 16383, 16384, 16399, 16399, 16368] avg=16384.0000 max=16401 max_rank=0 min=16358 min_rank=1 imbalance_ratio=1.001038 max_load_reduction=114671 max_load_reduction_pct=0.874870 moved_to_echo=65550 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4027, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 35, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 32, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4041, 30, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4061, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 16, 4096, 4, 4059, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 74, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16369, 16385, 16385, 16388, 16384, 16363, 16429, 16369] avg=16384.0000 max=16429 max_rank=6 min=16363 min_rank=5 imbalance_ratio=1.002747 max_load_reduction=114643 max_load_reduction_pct=0.874657 moved_to_echo=65545 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4080, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 35, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4041, 30, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4061, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 16, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16419, 16357, 16380, 16385, 16384, 16400, 16379, 16368] avg=16384.0000 max=16419 max_rank=0 min=16357 min_rank=1 imbalance_ratio=1.002136 max_load_reduction=114653 max_load_reduction_pct=0.874733 moved_to_echo=65531 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4011, 3, 4091, 2, 0, 3, 4086, 3, 0, 0, 0, 0, 3, 3, 0, 4090, 36, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 6, 3, 3, 5, 0, 4092, 4, 4, 4, 4090, 72, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 4, 4092, 2, 4093, 4092, 0, 1, 4, 0, 0, 3, 6, 4037, 30, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 4, 0, 4070, 0, 0, 4092, 4092, 4, 0, 0, 0, 2, 3, 1, 4056, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 3, 4, 8, 4092, 5, 4091, 4091, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 2, 0, 4, 4091, 1, 17, 4084, 14, 4093, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4, 4, 1, 4090, 2, 0, 4, 1, 0, 0, 4089, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 2, 0, 6, 0, 15, 0, 0, 4, 0, 3, 4, 4093, 4092, 4092, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16347, 16432, 16389, 16371, 16389, 16392, 16357, 16395] avg=16384.0000 max=16432 max_rank=1 min=16347 min_rank=0 imbalance_ratio=1.002930 max_load_reduction=114640 max_load_reduction_pct=0.874634 moved_to_echo=65533 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 3997, 3, 4087, 6, 0, 22, 4080, 2, 0, 0, 0, 0, 6, 7, 0, 4088, 36, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4056, 8, 6, 0, 0, 0, 4086, 7, 8, 10, 4084, 51, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 10, 4088, 8, 4090, 4088, 0, 0, 7, 0, 0, 4, 0, 4033, 32, 27, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8, 0, 4073, 0, 0, 4087, 4089, 8, 0, 0, 0, 6, 6, 1, 4050, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2, 0, 4086, 0, 0, 0, 7, 8, 4086, 10, 4081, 4086, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 7, 12, 10, 4089, 0, 11, 4085, 3, 4089, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 10, 7, 10, 0, 4087, 5, 0, 7, 3, 0, 0, 4083, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 25, 2, 0, 3, 0, 11, 0, 52, 0, 0, 9, 9, 8, 7, 4089, 4088, 4087, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16350, 16399, 16400, 16375, 16378, 16361, 16347, 16462] avg=16384.0000 max=16462 max_rank=7 min=16347 min_rank=6 imbalance_ratio=1.004761 max_load_reduction=114610 max_load_reduction_pct=0.874405 moved_to_echo=65548 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4060, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 35, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4041, 30, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4061, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 16, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16399, 16378, 16380, 16388, 16384, 16395, 16380, 16368] avg=16384.0000 max=16399 max_rank=0 min=16368 min_rank=7 imbalance_ratio=1.000916 max_load_reduction=114673 max_load_reduction_pct=0.874886 moved_to_echo=65527 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4044, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 36, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4085, 47, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4041, 30, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4060, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 16, 4096, 3, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 37, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16390, 16380, 16387, 16384, 16379, 16400, 16368] avg=16384.0000 max=16400 max_rank=6 min=16368 min_rank=7 imbalance_ratio=1.000977 max_load_reduction=114672 max_load_reduction_pct=0.874878 moved_to_echo=65531 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4059, 0, 4095, 1, 0, 1, 4094, 5, 0, 0, 0, 0, 1, 1, 0, 4095, 35, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 1, 0, 6, 0, 4095, 1, 1, 0, 4090, 20, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 4095, 4095, 0, 0, 1, 1, 0, 0, 0, 4039, 30, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 1, 0, 4094, 0, 1, 4095, 4095, 1, 0, 0, 0, 1, 1, 0, 4060, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 1, 3, 4093, 1, 4095, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 1, 4095, 0, 16, 4089, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 18, 1, 1, 1, 4095, 0, 0, 1, 1, 0, 1, 4095, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 1, 2, 1, 1, 4094, 4095, 4095, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16403, 16373, 16378, 16395, 16385, 16383, 16381, 16374] avg=16384.0000 max=16403 max_rank=0 min=16373 min_rank=1 imbalance_ratio=1.001160 max_load_reduction=114669 max_load_reduction_pct=0.874855 moved_to_echo=65523 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32384, 128, 32768, 128, 0, 128, 32768, 128, 0, 0, 128, 0, 128, 128, 0, 32768, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 256, 128, 128, 128, 32768, 128, 0, 128, 128, 0, 128, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 32768, 0, 0, 128, 128, 128, 32768, 128, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32768, 128, 128, 128, 32768, 128, 128, 32768, 128, 32768, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 32512, 128, 128, 128, 128, 0, 32768, 128, 128, 128, 32768, 256, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 128, 128, 32768, 128, 32768, 32768, 0, 128, 128, 128, 0, 128, 128, 32384, 256, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 128, 32768, 32768, 128, 0, 0, 0, 128, 128, 128, 32512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 128, 0, 128, 0, 0, 128, 128, 128, 128, 32768, 32768, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 41, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 33, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 15, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 25, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 56, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[9, 0, 7, 10, 0, 0, 7, 1, 4087, 4085, 6, 4091, 0, 0, 9, 0, 0, 4084, 4064, 2, 4088, 14, 4085, 4084, 8, 0, 5, 0, 4086, 0, 4089, 7, 9, 4089, 4086, 0, 0, 4, 0, 0, 30, 0, 4, 6, 0, 0, 0, 3925, 4088, 7, 0, 7, 3, 4084, 6, 0, 4, 171, 9, 7, 3, 1, 4083, 0, 4084, 0, 0, 0, 6, 4045, 0, 4089, 0, 0, 0, 0, 0, 13, 9, 7, 5, 4081, 4085, 5, 2, 7, 4087, 10, 4080, 4, 0, 4087, 11, 8, 0, 5, 3, 0, 9, 2, 4087, 25, 0, 6, 7, 4086, 0, 7, 0, 0, 8, 4076, 0, 8, 0, 4083, 0, 12, 0, 4087, 4089, 1, 4088, 12, 4085, 6, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 4093, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3893, 4096, 0, 5, 0, 0, 4096, 0, 0, 0, 65, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 82, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 20, 2, 38, 4096, 4090, 0, 4096, 0, 4096, 0, 4, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[5, 0, 3, 0, 0, 4, 5, 1, 4092, 4091, 0, 4094, 0, 0, 4, 2, 0, 4090, 4077, 0, 4092, 6, 4091, 4091, 4, 0, 1, 0, 4091, 0, 4092, 0, 5, 4093, 4092, 2, 0, 1, 0, 0, 12, 0, 5, 0, 5, 0, 0, 3246, 4092, 4, 0, 3, 0, 4091, 4, 0, 1, 870, 4, 4, 0, 13, 4089, 0, 4092, 0, 0, 0, 3, 4038, 0, 4092, 0, 0, 0, 0, 0, 0, 4, 4, 3, 4083, 4090, 4, 7, 3, 4092, 5, 4086, 2, 4, 4091, 4, 4, 0, 0, 4, 0, 7, 25, 4091, 17, 0, 2, 4, 4091, 0, 3, 0, 0, 4, 4068, 0, 4, 0, 4092, 0, 5, 0, 4092, 4092, 4, 4091, 0, 4091, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3897, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 115, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4091, 61, 0, 28, 4096, 4096, 0, 4093, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3676, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 252, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 146, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 4, 0, 32, 4096, 4096, 0, 4093, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 1, 0, 2, 0, 1, 0, 4095, 4095, 0, 4095, 0, 0, 1, 0, 0, 4095, 4092, 0, 4095, 2, 4094, 4095, 1, 0, 0, 0, 4095, 0, 4095, 0, 1, 4095, 4094, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 3677, 4095, 1, 0, 1, 2, 4095, 1, 0, 0, 325, 1, 1, 0, 0, 4095, 0, 4094, 2, 0, 0, 1, 4073, 1, 4095, 0, 0, 0, 1, 0, 0, 1, 1, 0, 4094, 4095, 1, 1, 1, 4094, 1, 4095, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 113, 4094, 0, 0, 1, 1, 4093, 0, 1, 0, 0, 1, 4095, 0, 1, 0, 4095, 0, 0, 3, 4095, 4095, 0, 4095, 2, 4095, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3783, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 4096, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 25, 0, 0, 0, 0, 4096, 0, 3843, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3786, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 56, 0, 4096, 0, 4096, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16392, 16592, 16185, 16402, 16385, 16325, 16411] avg=16384.0000 max=16592 max_rank=2 min=16185 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=114480 max_load_reduction_pct=0.873413 moved_to_echo=65523 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3784, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 4096, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 41, 0, 0, 0, 0, 4096, 0, 3843, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3801, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3787, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 4096, 0, 4096, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16392, 16590, 16201, 16401, 16384, 16353, 16370] avg=16384.0000 max=16590 max_rank=2 min=16201 min_rank=3 imbalance_ratio=1.012573 max_load_reduction=114482 max_load_reduction_pct=0.873428 moved_to_echo=65508 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 3, 0, 0, 4, 5, 1, 4092, 4091, 0, 4094, 0, 0, 4, 2, 3781, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4077, 0, 0, 6, 4091, 4091, 4, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4093, 4092, 2, 0, 1, 0, 0, 12, 0, 5, 0, 5, 0, 0, 3246, 4092, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 0, 3, 0, 4091, 4, 0, 1, 870, 4, 4, 0, 13, 4089, 0, 3838, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 3, 4038, 0, 4092, 0, 0, 0, 0, 0, 0, 4, 4, 3796, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4083, 4090, 4, 7, 3, 0, 5, 4086, 2, 4, 4091, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7, 25, 4091, 17, 0, 2, 4, 4091, 0, 3, 0, 0, 4, 4068, 3782, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4092, 0, 5, 0, 0, 4092, 4, 4091, 0, 4062, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 16368, 15849, 17042, 16340, 16386, 16343, 16352] avg=16384.0000 max=17042 max_rank=3 min=15849 min_rank=2 imbalance_ratio=1.040161 max_load_reduction=114030 max_load_reduction_pct=0.869980 moved_to_echo=65421 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 7, 10, 0, 0, 7, 1, 4087, 4085, 6, 4091, 0, 0, 9, 0, 3776, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4064, 2, 0, 14, 4085, 4084, 8, 0, 5, 0, 4, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4089, 4086, 0, 0, 4, 0, 0, 30, 0, 4, 6, 0, 0, 0, 3925, 4089, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 7, 3, 4084, 6, 0, 4, 171, 9, 7, 3, 1, 4083, 0, 3835, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 6, 4045, 0, 4089, 0, 0, 0, 0, 0, 13, 9, 7, 3793, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4081, 4085, 5, 2, 7, 0, 10, 4080, 4, 0, 4087, 11, 8, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 9, 2, 4087, 25, 0, 6, 7, 4086, 0, 7, 0, 0, 8, 4076, 3778, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4083, 0, 12, 0, 0, 4089, 1, 4088, 12, 4049, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 16357, 16537, 16344, 16357, 16390, 16341, 16349] avg=16384.0000 max=16537 max_rank=2 min=16341 min_rank=6 imbalance_ratio=1.009338 max_load_reduction=114535 max_load_reduction_pct=0.873833 moved_to_echo=65437 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 4093, 0, 0, 0, 0, 3783, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3893, 4096, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 4096, 0, 0, 0, 65, 0, 0, 0, 0, 4096, 0, 3843, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 1, 0, 0, 0, 0, 3801, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3786, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 20, 2, 38, 0, 4090, 0, 4096, 0, 4067, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16392, 16494, 16230, 16394, 16381, 16398, 16408] avg=16384.0000 max=16494 max_rank=2 min=16230 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=114578 max_load_reduction_pct=0.874161 moved_to_echo=65581 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3784, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3676, 4096, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 252, 0, 0, 0, 0, 4096, 0, 3843, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3800, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 3787, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 32, 0, 4096, 0, 4093, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16392, 16260, 16412, 16400, 16379, 16460, 16388] avg=16384.0000 max=16460 max_rank=6 min=16260 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=114612 max_load_reduction_pct=0.874420 moved_to_echo=65627 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3784, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3897, 4096, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 3843, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3800, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 3786, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 61, 0, 28, 0, 4096, 0, 4093, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16392, 16487, 16170, 16398, 16384, 16423, 16436] avg=16384.0000 max=16487 max_rank=2 min=16170 min_rank=3 imbalance_ratio=1.006287 max_load_reduction=114585 max_load_reduction_pct=0.874214 moved_to_echo=65641 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 0, 2, 0, 1, 0, 4095, 4095, 0, 4095, 0, 0, 1, 0, 3784, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4092, 0, 0, 2, 4094, 4095, 1, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4094, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 3677, 4095, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 2, 4095, 1, 0, 0, 325, 1, 1, 0, 0, 4095, 0, 3842, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 1, 4073, 1, 4095, 0, 0, 0, 1, 0, 0, 1, 1, 3800, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 1, 1, 1, 0, 1, 4095, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 4094, 0, 0, 1, 1, 4093, 0, 1, 0, 0, 1, 4095, 3785, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 3, 0, 4095, 0, 4095, 2, 4066, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16387, 16263, 16488, 16380, 16383, 16429, 16358] avg=16384.0000 max=16488 max_rank=3 min=16263 min_rank=2 imbalance_ratio=1.006348 max_load_reduction=114584 max_load_reduction_pct=0.874207 moved_to_echo=65550 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 128, 128, 32768, 128, 0, 128, 1792, 128, 128, 128, 128, 32768, 0, 30848, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 32768, 128, 0, 128, 0, 128, 128, 128, 128, 128, 128, 0, 128, 30336, 32768, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 640, 32768, 128, 0, 128, 128, 32768, 0, 128, 0, 0, 128, 32768, 30336, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 32768, 128, 128, 256, 0, 32768, 128, 32768, 128, 32512, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 128, 32768, 32768, 128, 32768, 0, 0, 128, 128, 30336, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 32768, 128, 0, 128, 32768, 32768, 128, 0, 128, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 0, 0, 128, 32640, 128, 32768, 0, 0, 0, 128, 0, 128, 128, 128, 30464, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 4096, 1927, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3780, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 473, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 56, 0, 0, 4096, 0, 4096, 0, 4094, 0, 0, 1959, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4087, 3, 0, 4088, 4058, 0, 0, 2, 0, 7, 2, 4083, 0, 0, 0, 9, 9, 0, 4087, 0, 4085, 4086, 1, 0, 4079, 4089, 1004, 0, 0, 4085, 4087, 8, 9, 9, 4084, 3, 11, 8, 0, 0, 0, 4066, 0, 8, 4085, 4088, 0, 0, 0, 10, 9, 9, 9, 10, 0, 5, 0, 4085, 0, 8, 8, 0, 4085, 6, 4086, 4086, 8, 0, 0, 8, 2, 0, 0, 78, 4088, 10, 0, 0, 4087, 4086, 5, 0, 4086, 6, 0, 9, 0, 9, 20, 8, 4088, 11, 6, 4088, 0, 0, 5, 7, 226, 0, 8, 4081, 0, 4083, 0, 4076, 2, 13, 2793, 0, 0, 4080, 11, 8, 8, 0, 2, 4087, 11, 4088, 8, 3, 10, 12, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4092, 1, 0, 4092, 4069, 0, 0, 0, 0, 5, 4, 4090, 0, 3, 0, 3, 5, 0, 4011, 0, 4091, 4091, 0, 0, 4089, 4092, 144, 0, 0, 4091, 4091, 3, 3, 4, 4091, 4, 5, 4, 0, 0, 0, 4076, 0, 4, 4090, 4092, 0, 0, 0, 5, 4, 5, 4, 0, 0, 1, 0, 4090, 0, 4, 4, 0, 4091, 3, 4092, 4090, 5, 0, 4, 4, 2, 0, 0, 3, 4092, 4, 0, 0, 4092, 4091, 0, 0, 4091, 3, 0, 4, 0, 5, 0, 4, 4092, 6, 3, 4092, 0, 0, 1, 4, 125, 7, 6, 4085, 0, 4089, 2, 4075, 2, 0, 3926, 0, 0, 4089, 22, 3, 5, 0, 0, 4091, 5, 4091, 4, 2, 3, 5, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 5, 4088, 0, 4096, 4096, 35, 0, 4096, 4091, 1927, 0, 0, 4096, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4090, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 11, 501, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 4054, 0, 0, 4096, 0, 5, 0, 0, 104, 0, 0, 4096, 0, 4094, 0, 4057, 0, 0, 1645, 0, 0, 4096, 5, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 4096, 1, 0, 4096, 4096, 421, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 8, 177, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 4096, 0, 0, 0, 0, 49, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 3515, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 20, 0, 4096, 4094, 431, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 298, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 4096, 0, 3, 0, 0, 15, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 3410, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 8, 0, 4096, 4096, 2127, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 278, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 49, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 1744, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 4095, 4090, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 1, 0, 4088, 0, 4095, 4095, 0, 0, 4094, 4095, 454, 0, 0, 4094, 4095, 1, 1, 1, 4095, 0, 1, 1, 0, 0, 0, 4005, 0, 1, 4095, 4095, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 4094, 0, 1, 1, 0, 4095, 1, 4095, 4095, 1, 0, 0, 1, 0, 1, 0, 547, 4095, 1, 0, 0, 4095, 4095, 1, 0, 4095, 1, 1, 1, 0, 1, 7, 1, 4095, 1, 2, 4094, 0, 0, 0, 1, 65, 0, 1, 4095, 0, 4095, 0, 4093, 0, 1, 3111, 1, 0, 4094, 14, 1, 1, 0, 0, 4095, 1, 4095, 1, 1, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 1927, 0, 0, 3052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3780, 0, 0, 4096, 4096, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 473, 3802, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 4096, 0, 4096, 0, 4094, 0, 0, 1959, 0, 0, 1044, 197, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16399, 17267, 16140, 16379, 16563, 16393, 15556, 16375] avg=16384.0000 max=17267 max_rank=1 min=15556 min_rank=6 imbalance_ratio=1.053894 max_load_reduction=113805 max_load_reduction_pct=0.868263 moved_to_echo=64887 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4087, 3, 0, 4088, 4058, 0, 0, 2, 0, 7, 2, 4083, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 0, 4087, 0, 0, 0, 1, 0, 4079, 4089, 1004, 0, 0, 3040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 9, 9, 4084, 3, 11, 8, 0, 0, 0, 4066, 0, 8, 4085, 4088, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 9, 9, 9, 10, 0, 5, 0, 4085, 0, 8, 8, 0, 4085, 4087, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4086, 4086, 8, 0, 0, 8, 2, 0, 0, 78, 3770, 10, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 5, 0, 4086, 6, 0, 9, 0, 9, 20, 8, 4088, 11, 6, 4088, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7, 226, 0, 8, 4081, 0, 4083, 0, 4076, 2, 13, 2793, 0, 0, 1045, 201, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 11, 8, 8, 0, 2, 4087, 11, 4088, 8, 3, 10, 12, 0, 1, 0, 4071, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16352, 16318, 16458, 16407, 16141, 16438, 16554, 16404] avg=16384.0000 max=16554 max_rank=6 min=16141 min_rank=4 imbalance_ratio=1.010376 max_load_reduction=114518 max_load_reduction_pct=0.873703 moved_to_echo=65537 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 1, 0, 4092, 4069, 0, 0, 0, 0, 5, 4, 4090, 0, 3, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 0, 4011, 0, 0, 0, 0, 0, 4089, 4092, 144, 0, 0, 3048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 4, 4091, 4, 5, 4, 0, 0, 0, 4076, 0, 4, 4090, 4092, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 5, 4, 0, 0, 1, 0, 4090, 0, 4, 4, 0, 4091, 4091, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4092, 4090, 5, 0, 4, 4, 2, 0, 0, 3, 3798, 4, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4091, 3, 0, 4, 0, 5, 0, 4, 4092, 6, 3, 4092, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 125, 7, 6, 4085, 0, 4089, 2, 4075, 2, 0, 3926, 0, 0, 1043, 197, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 22, 3, 5, 0, 0, 4091, 5, 4091, 4, 2, 3, 5, 0, 0, 0, 4077, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 15392, 16448, 16386, 16097, 16400, 17576, 16401] avg=16384.0000 max=17576 max_rank=6 min=15392 min_rank=1 imbalance_ratio=1.072754 max_load_reduction=113496 max_load_reduction_pct=0.865906 moved_to_echo=66474 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 1, 0, 4096, 4096, 421, 0, 0, 3052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 4096, 4096, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 8, 177, 3802, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 3515, 0, 0, 1044, 197, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16400, 15757, 16422, 16380, 16275, 16357, 17107, 16374] avg=16384.0000 max=17107 max_rank=6 min=15757 min_rank=1 imbalance_ratio=1.044128 max_load_reduction=113965 max_load_reduction_pct=0.869484 moved_to_echo=66113 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4088, 0, 0, 0, 35, 0, 4096, 4091, 1927, 0, 0, 3052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4090, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 11, 501, 3802, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 4054, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 104, 0, 0, 4096, 0, 4094, 0, 4057, 0, 0, 1645, 0, 0, 1044, 197, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4083, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16360, 17294, 16444, 16380, 16602, 16357, 15255, 16380] avg=16384.0000 max=17294 max_rank=1 min=15255 min_rank=6 imbalance_ratio=1.055542 max_load_reduction=113778 max_load_reduction_pct=0.868057 moved_to_echo=64594 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 4096, 4096, 2127, 0, 0, 3052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 4096, 4096, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 278, 3802, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 1744, 0, 0, 1044, 197, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16400, 17475, 16353, 16379, 16368, 16387, 15335, 16375] avg=16384.0000 max=17475 max_rank=1 min=15335 min_rank=6 imbalance_ratio=1.066589 max_load_reduction=113597 max_load_reduction_pct=0.866676 moved_to_echo=64465 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 20, 0, 4096, 4094, 431, 0, 0, 3052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 4096, 4096, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 298, 3802, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 15, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 3410, 0, 0, 1044, 197, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 15787, 16439, 16380, 16388, 16336, 16971, 16374] avg=16384.0000 max=16971 max_rank=6 min=15787 min_rank=1 imbalance_ratio=1.035828 max_load_reduction=114101 max_load_reduction_pct=0.870522 moved_to_echo=66069 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 4095, 4090, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4088, 0, 0, 0, 0, 0, 4094, 4095, 454, 0, 0, 3050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4095, 0, 1, 1, 0, 0, 0, 4005, 0, 1, 4095, 4095, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 4094, 0, 1, 1, 0, 4095, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4095, 1, 0, 0, 1, 0, 1, 0, 547, 3801, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 4095, 1, 1, 1, 0, 1, 7, 1, 4095, 1, 2, 4094, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 65, 0, 1, 4095, 0, 4095, 0, 4093, 0, 1, 3111, 1, 0, 1044, 197, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 14, 1, 1, 0, 0, 4095, 1, 4095, 1, 1, 0, 1, 0, 0, 0, 4081, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 15782, 16368, 16381, 16638, 16404, 16718, 16389] avg=16384.0000 max=16718 max_rank=6 min=15782 min_rank=1 imbalance_ratio=1.020386 max_load_reduction=114354 max_load_reduction_pct=0.872452 moved_to_echo=66149 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 128, 0, 32768, 32768, 0, 0, 128, 0, 128, 128, 32768, 0, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 32768, 128, 0, 128, 128, 128, 128, 128, 2432, 30464, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 0, 128, 0, 32768, 0, 128, 128, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 0, 32768, 128, 128, 128, 0, 128, 128, 128, 32640, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 128, 0, 128, 32768, 128, 32768, 128, 128, 128, 128, 0, 128, 0, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 32256, 0, 128, 32768, 32768, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 128, 128, 32768, 0, 0, 0, 128, 0, 32768, 32768, 8448, 0, 0, 24448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 768, 128, 128, 32768, 0, 32768, 128, 32768, 128, 128, 22144, 128, 0, 8448, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 2, 4096, 0, 0, 0, 4096, 0, 0, 4093, 4096, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 4081, 0, 0, 4096, 0, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 3, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4089, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 100, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4, 4096, 0, 4, 0, 4096, 0, 0, 4074, 4096, 0, 0, 0, 0, 15, 0, 0, 4091, 0, 4091, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3, 3996, 0, 0, 4096, 0, 22, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 4076, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 4082, 1, 4087, 0, 4088, 4023, 1, 8, 0, 5, 0, 0, 0, 4074, 0, 0, 4086, 3, 0, 0, 10, 0, 6, 10, 3, 4, 4087, 11, 0, 0, 0, 0, 0, 8, 8, 9, 0, 0, 9, 8, 12, 10, 4085, 2, 9, 7, 4083, 0, 9, 0, 8, 4084, 0, 3907, 331, 4084, 0, 0, 46, 4086, 0, 8, 3901, 4085, 9, 0, 10, 12, 65, 0, 11, 4087, 0, 4088, 3, 0, 4086, 4088, 7, 35, 5, 7, 8, 4087, 16, 8, 4085, 3, 4082, 0, 7, 4081, 14, 0, 5, 4080, 0, 0, 3, 0, 4084, 0, 4079, 3, 0, 14, 4, 4090, 4085, 0, 0, 0, 8, 0, 12, 4057, 0, 4083, 0, 0, 0, 0, 4080, 4087, 0, 6, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 4089, 0, 4091, 0, 4092, 4013, 0, 5, 0, 6, 0, 0, 0, 4087, 0, 0, 4092, 1, 0, 0, 5, 0, 3, 5, 0, 0, 4091, 5, 3, 0, 0, 0, 0, 4, 4, 5, 0, 3, 5, 4, 4, 3, 4091, 3, 7, 4, 4076, 0, 5, 0, 4, 4090, 0, 3859, 233, 4091, 0, 0, 8, 4091, 0, 5, 4081, 4089, 6, 0, 2, 5, 22, 0, 5, 4092, 0, 4092, 4, 0, 4091, 4091, 3, 13, 5, 4, 4, 4091, 6, 4, 4091, 61, 4084, 0, 3, 4088, 9, 0, 3, 4090, 0, 0, 1, 0, 4091, 0, 4090, 1, 0, 6, 4, 4092, 4090, 0, 0, 1, 4, 0, 5, 4086, 0, 4091, 0, 0, 0, 1, 4090, 4091, 0, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 25, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4039, 0, 0, 4096, 0, 14, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1, 4096, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4046, 0, 0, 4096, 0, 7, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 4094, 1, 4094, 0, 4095, 4083, 0, 1, 0, 0, 0, 3, 0, 4094, 0, 0, 4095, 0, 0, 0, 1, 0, 1, 1, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 2, 0, 4095, 0, 0, 1, 4093, 0, 1, 1, 1, 4094, 0, 4084, 71, 4094, 0, 0, 2, 4095, 0, 1, 4095, 4094, 1, 0, 0, 1, 10, 0, 1, 4095, 0, 4095, 0, 0, 4095, 4095, 1, 2, 1, 1, 1, 4094, 2, 1, 4095, 3, 4037, 0, 1, 4095, 2, 0, 1, 4094, 0, 1, 1, 0, 4093, 0, 4094, 0, 0, 1, 1, 4095, 4095, 0, 2, 0, 1, 0, 1, 4089, 0, 4094, 0, 0, 1, 0, 4094, 4095, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 4085, 4069, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 2, 4093, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 4096, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 4081, 0, 0, 4096, 0, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 16365, 16378, 16377, 16372, 16382, 16395, 16392] avg=16384.0000 max=16411 max_rank=0 min=16365 min_rank=1 imbalance_ratio=1.001648 max_load_reduction=114661 max_load_reduction_pct=0.874794 moved_to_echo=65541 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4085, 4069, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 3, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 4096, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4089, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 16356, 16378, 16381, 16372, 16387, 16395, 16392] avg=16384.0000 max=16411 max_rank=0 min=16356 min_rank=1 imbalance_ratio=1.001648 max_load_reduction=114661 max_load_reduction_pct=0.874794 moved_to_echo=65546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 0, 4091, 0, 27, 4013, 0, 5, 0, 6, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 5, 0, 3, 5, 0, 0, 4091, 5, 3, 0, 0, 4079, 4065, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 5, 0, 3, 5, 4, 4, 3, 4091, 3, 7, 4, 4076, 4091, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4, 4090, 0, 3859, 233, 4088, 0, 0, 8, 0, 0, 5, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 6, 0, 2, 5, 22, 0, 5, 4092, 0, 4092, 4, 0, 4073, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 5, 4, 4, 4091, 6, 4, 4091, 61, 4084, 0, 3, 4088, 9, 0, 3, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 1, 0, 4091, 0, 4090, 1, 0, 6, 4, 4092, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 5, 4086, 0, 4091, 0, 0, 0, 1, 4090, 4091, 0, 1, 1, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16318, 16359, 16388, 16373, 16393, 16476, 16386, 16379] avg=16384.0000 max=16476 max_rank=5 min=16318 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=114596 max_load_reduction_pct=0.874298 moved_to_echo=65634 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4082, 1, 4087, 0, 21, 4023, 1, 8, 0, 5, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 3, 0, 0, 10, 0, 6, 10, 3, 4, 4087, 11, 0, 0, 0, 4081, 4067, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 9, 0, 0, 9, 8, 12, 10, 4085, 2, 9, 7, 4083, 4088, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8, 4084, 0, 3907, 331, 4078, 0, 0, 46, 0, 0, 8, 3901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 9, 0, 10, 12, 65, 0, 11, 4087, 0, 4088, 3, 0, 4050, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 5, 7, 8, 4087, 16, 8, 4085, 3, 4082, 0, 7, 4081, 14, 0, 5, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 3, 0, 4084, 0, 4079, 3, 0, 14, 4, 4090, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 12, 4057, 0, 4083, 0, 0, 0, 0, 4080, 4087, 0, 6, 0, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16302, 16383, 16417, 16372, 16427, 16460, 16361, 16350] avg=16384.0000 max=16460 max_rank=5 min=16302 min_rank=0 imbalance_ratio=1.004639 max_load_reduction=114612 max_load_reduction_pct=0.874420 moved_to_echo=65598 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 100, 2, 0, 4085, 4069, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4, 4093, 0, 4, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 15, 0, 0, 4091, 0, 4091, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3, 3996, 0, 0, 4096, 0, 22, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 4076, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 16457, 16378, 16367, 16371, 16319, 16377, 16392] avg=16384.0000 max=16457 max_rank=1 min=16319 min_rank=5 imbalance_ratio=1.004456 max_load_reduction=114615 max_load_reduction_pct=0.874443 moved_to_echo=65459 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 50, 0, 0, 4085, 4069, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 4093, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 4096, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4046, 0, 0, 4096, 0, 7, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4084, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 16406, 16377, 16381, 16371, 16351, 16383, 16392] avg=16384.0000 max=16411 max_rank=0 min=16351 min_rank=5 imbalance_ratio=1.001648 max_load_reduction=114661 max_load_reduction_pct=0.874794 moved_to_echo=65497 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 27, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 33, 0, 0, 4085, 4069, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 25, 0, 0, 4096, 0, 4096, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4039, 0, 0, 4096, 0, 14, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4083, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 16389, 16377, 16380, 16391, 16351, 16382, 16391] avg=16384.0000 max=16411 max_rank=0 min=16351 min_rank=5 imbalance_ratio=1.001648 max_load_reduction=114661 max_load_reduction_pct=0.874794 moved_to_echo=65515 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 1, 4094, 0, 27, 4083, 0, 1, 0, 0, 0, 3, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 1, 1, 1, 0, 4095, 1, 0, 0, 0, 4084, 4068, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 2, 0, 4095, 0, 0, 1, 4093, 4095, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4094, 0, 4084, 71, 4091, 0, 0, 2, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 1, 10, 0, 1, 4095, 0, 4095, 0, 0, 4077, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 1, 4094, 2, 1, 4095, 3, 4037, 0, 1, 4095, 2, 0, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 1, 0, 4093, 0, 4094, 0, 0, 1, 1, 4095, 11, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4089, 0, 4094, 0, 0, 1, 0, 4094, 4095, 0, 1, 0, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 16357, 16379, 16441, 16375, 16346, 16393, 16384] avg=16384.0000 max=16441 max_rank=3 min=16346 min_rank=5 imbalance_ratio=1.003479 max_load_reduction=114631 max_load_reduction_pct=0.874565 moved_to_echo=65498 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 128, 32768, 0, 256, 32640, 128, 128, 0, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 128, 128, 0, 32768, 0, 32768, 128, 128, 128, 128, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 0, 128, 128, 128, 0, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 32768, 0, 0, 128, 128, 32768, 32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 32768, 128, 32512, 0, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 32768, 128, 0, 0, 128, 0, 128, 128, 128, 128, 32768, 128, 256, 128, 0, 32768, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 32384, 768, 32768, 0, 128, 128, 0, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 128, 128, 256, 0, 128, 32768, 0, 32768, 128, 0, 32640, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[516, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2312, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 1212, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4073, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 78, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[443, 0, 0, 0, 4096, 0, 5, 4096, 0, 0, 0, 2477, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 1161, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 34, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[669, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2863, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 618, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 33, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[273, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1181, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 2378, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 264, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[137, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1483, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 1828, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 689, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[11, 10, 0, 9, 4086, 13, 0, 4086, 0, 9, 0, 3394, 22, 10, 4086, 5, 0, 9, 8, 0, 7, 4085, 12, 4, 9, 4047, 6, 10, 12, 0, 4078, 6, 6, 13, 0, 4083, 4084, 0, 8, 10, 4048, 4075, 9, 4080, 4084, 11, 4082, 0, 0, 0, 9, 8, 8, 0, 0, 0, 4086, 4082, 7, 4, 4086, 15, 8, 4014, 0, 0, 5, 4077, 0, 10, 0, 0, 5, 0, 9, 4084, 8, 4087, 3, 6, 13, 31, 49, 4085, 4082, 0, 0, 4083, 0, 0, 0, 2, 0, 4078, 4083, 4081, 0, 0, 4071, 0, 0, 0, 0, 0, 5, 4084, 6, 8, 7, 8, 7, 64, 0, 7, 0, 0, 4069, 0, 704, 4074, 11, 10, 0, 14, 0, 3, 0, 4085] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[393, 5, 0, 5, 4090, 7, 0, 4090, 0, 5, 0, 2363, 12, 5, 4091, 1, 0, 5, 2, 0, 0, 4087, 5, 0, 0, 4067, 0, 4, 5, 0, 4088, 3, 2, 6, 0, 4090, 4090, 0, 10, 2, 4074, 4085, 5, 4087, 4090, 6, 4088, 1, 0, 5, 5, 4, 6, 514, 2, 3, 4091, 4089, 3, 3, 4090, 0, 4, 3964, 6, 0, 1, 4086, 0, 5, 0, 0, 3, 0, 5, 4091, 5, 4092, 0, 0, 5, 7, 18, 4091, 4091, 0, 0, 4090, 0, 0, 0, 1, 0, 4089, 4091, 4091, 0, 0, 4089, 0, 0, 0, 0, 0, 3, 4090, 4, 1, 2, 4, 4, 27, 0, 4, 0, 0, 4082, 2, 950, 4083, 6, 5, 0, 6, 0, 0, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[752, 2, 0, 1, 4095, 1, 0, 4095, 0, 1, 0, 2561, 0, 1, 4095, 1, 0, 2, 0, 0, 0, 4095, 2, 1, 0, 4094, 0, 1, 1, 0, 4094, 1, 1, 2, 0, 4092, 4095, 0, 0, 1, 4090, 4093, 2, 4094, 4095, 1, 4094, 0, 0, 0, 1, 1, 1, 703, 0, 1, 4094, 4093, 1, 1, 4095, 4, 1, 4065, 0, 0, 0, 4094, 0, 1, 0, 0, 1, 0, 1, 4095, 1, 4095, 2, 1, 1, 1, 5, 4095, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 4094, 4095, 0, 1, 4089, 0, 0, 1, 0, 0, 1, 4095, 2, 1, 1, 1, 1, 3, 0, 0, 0, 0, 4093, 0, 111, 4093, 1, 1, 0, 1, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[516, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2312, 0, 0, 4096, 0, 1027, 325, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4082, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1212, 0, 0, 3069, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4073, 0, 4096, 0, 0, 4096, 14, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 78, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16473, 16387, 16394, 16569, 16384, 16379, 16376, 16110] avg=16384.0000 max=16569 max_rank=3 min=16110 min_rank=7 imbalance_ratio=1.011292 max_load_reduction=114503 max_load_reduction_pct=0.873589 moved_to_echo=65249 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[443, 0, 0, 0, 4096, 0, 5, 4096, 0, 0, 0, 2477, 0, 0, 4096, 0, 1027, 325, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4082, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1161, 0, 0, 3069, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 4096, 0, 0, 4096, 14, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 34, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16570, 16387, 16394, 16518, 16386, 16379, 16372, 16066] avg=16384.0000 max=16570 max_rank=0 min=16066 min_rank=7 imbalance_ratio=1.011353 max_load_reduction=114502 max_load_reduction_pct=0.873581 moved_to_echo=65203 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[273, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1181, 0, 0, 4096, 0, 1027, 325, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4082, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2378, 0, 0, 3069, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 14, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 264, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15099, 16387, 16394, 17735, 16406, 16379, 16376, 16296] avg=16384.0000 max=17735 max_rank=3 min=15099 min_rank=0 imbalance_ratio=1.082458 max_load_reduction=113337 max_load_reduction_pct=0.864693 moved_to_echo=65457 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[393, 5, 0, 5, 4090, 7, 0, 4090, 0, 5, 0, 2363, 12, 5, 4091, 1, 1026, 325, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 4087, 5, 0, 0, 4067, 0, 4, 5, 0, 4088, 3, 4078, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 0, 10, 0, 0, 10, 2, 4074, 4085, 5, 4087, 0, 6, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 4, 6, 514, 2, 3, 3065, 4089, 3, 3, 4090, 0, 4, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 4086, 0, 5, 0, 0, 3, 0, 5, 4091, 5, 4092, 0, 0, 4090, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7, 18, 0, 4091, 0, 0, 4090, 0, 0, 0, 1, 0, 4089, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 3, 4090, 4, 1, 2, 4, 4, 27, 4091, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4082, 2, 950, 4083, 6, 5, 0, 6, 0, 0, 0, 4090, 3738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16423, 16361, 16376, 15757, 16404, 16387, 16398, 16966] avg=16384.0000 max=16966 max_rank=7 min=15757 min_rank=3 imbalance_ratio=1.035522 max_load_reduction=114106 max_load_reduction_pct=0.870560 moved_to_echo=66155 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[669, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2863, 0, 0, 4096, 0, 1027, 325, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4082, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 618, 0, 0, 3069, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 4096, 0, 0, 4096, 14, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 33, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17177, 16387, 16394, 15975, 16319, 16379, 16376, 16065] avg=16384.0000 max=17177 max_rank=0 min=15975 min_rank=3 imbalance_ratio=1.048401 max_load_reduction=113895 max_load_reduction_pct=0.868950 moved_to_echo=65139 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[11, 10, 0, 9, 4086, 13, 0, 4086, 0, 9, 0, 3394, 22, 10, 4086, 5, 1028, 327, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 0, 7, 4085, 12, 4, 9, 4047, 6, 10, 12, 0, 4078, 6, 4070, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 13, 0, 0, 0, 0, 8, 10, 4048, 4075, 9, 4080, 0, 11, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 8, 0, 0, 0, 3058, 4082, 7, 4, 4086, 15, 8, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4077, 0, 10, 0, 0, 5, 0, 9, 4084, 8, 4087, 3, 6, 4084, 13, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 31, 49, 0, 4082, 0, 0, 4083, 0, 0, 0, 2, 0, 4078, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 5, 4084, 6, 8, 7, 8, 7, 64, 4085, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4069, 0, 704, 4074, 11, 10, 0, 14, 0, 3, 0, 4085, 3732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17104, 16387, 16342, 15299, 16399, 16411, 16421, 16709] avg=16384.0000 max=17104 max_rank=0 min=15299 min_rank=3 imbalance_ratio=1.043945 max_load_reduction=113968 max_load_reduction_pct=0.869507 moved_to_echo=65940 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[137, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1483, 0, 0, 4096, 0, 1027, 325, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4083, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1828, 0, 0, 3069, 4096, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 4096, 0, 0, 4096, 13, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 689, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15265, 16388, 16390, 17184, 16366, 16379, 16375, 16725] avg=16384.0000 max=17184 max_rank=3 min=15265 min_rank=0 imbalance_ratio=1.048828 max_load_reduction=113888 max_load_reduction_pct=0.868896 moved_to_echo=65845 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[752, 2, 0, 1, 4095, 1, 0, 4095, 0, 1, 0, 2561, 0, 1, 4095, 1, 1026, 325, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 2, 1, 0, 4094, 0, 1, 1, 0, 4094, 1, 4080, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 10, 0, 0, 0, 1, 4090, 4093, 2, 4094, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 703, 0, 1, 3068, 4093, 1, 1, 4095, 4, 1, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 1, 0, 1, 4095, 1, 4095, 2, 1, 4095, 14, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5, 0, 4094, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 1, 0, 0, 1, 4095, 2, 1, 1, 1, 1, 3, 4095, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 111, 4093, 1, 1, 0, 1, 0, 0, 0, 4095, 3740, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16961, 16388, 16388, 16035, 16408, 16379, 16378, 16135] avg=16384.0000 max=16961 max_rank=0 min=16035 min_rank=3 imbalance_ratio=1.035217 max_load_reduction=114111 max_load_reduction_pct=0.870598 moved_to_echo=65300 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 0, 32768, 0, 0, 0, 128, 0, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 32768, 0, 128, 0, 0, 128, 128, 128, 32640, 128, 32768, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 32768, 128, 2944, 32768, 128, 128, 0, 128, 0, 128, 0, 32768, 29952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 32768, 0, 0, 128, 0, 0, 128, 32768, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 128, 8448, 128, 128, 24576, 32768, 128, 128, 32768, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[3200, 128, 0, 128, 32768, 128, 128, 32768, 0, 128, 0, 18688, 128, 128, 32768, 128, 8320, 2688, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 0, 0, 128, 128, 32768, 32768, 128, 32768, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 0, 128, 32768, 128, 128, 128, 32768, 128, 128, 128, 0, 32768, 128, 32640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 2, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 9, 4096, 0, 4096, 0, 0, 0, 261, 4096, 0, 0, 4096, 0, 0, 2364, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 653, 0, 0, 0, 4096, 0, 0, 0, 816, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4088, 4096, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 6, 4096, 0, 4096, 0, 0, 0, 671, 4096, 0, 0, 4096, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 765, 0, 0, 0, 4096, 0, 0, 0, 2455, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4077, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 130, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 5, 0, 0, 4096, 92, 4096, 0, 4096, 0, 0, 0, 1736, 4096, 0, 0, 4096, 0, 0, 748, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 961, 0, 0, 0, 4096, 0, 0, 0, 496, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4092, 4096, 4092, 4091, 4085, 0, 0, 0, 0, 0, 0, 0, 4096, 4050, 0, 0, 4096, 0, 0, 4083, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4090, 4, 424, 4091, 0, 4089, 4049, 4092, 1, 0, 4087, 0, 0, 2, 4091, 311, 4082, 4, 4089, 0, 5, 0, 2250, 4092, 21, 4, 4091, 5, 5, 71, 0, 0, 6, 0, 5, 0, 4, 4088, 4, 4, 0, 0, 2, 5, 0, 7, 0, 4092, 0, 8, 0, 0, 3, 3, 0, 1, 5, 4091, 2, 0, 7, 0, 5, 0, 4091, 5, 7, 14, 4, 5, 0, 0, 6, 0, 5, 0, 1, 0, 38, 0, 4089, 0, 4091, 6, 0, 4090, 54, 2, 1, 2, 4087, 0, 0, 0, 992, 0, 0, 5, 4087, 4090, 0, 12, 4, 0, 5, 4089, 4082, 4092, 4091, 4087, 6, 0, 0, 0, 4, 4, 4, 4089, 4090, 0, 3, 4080, 0, 0, 4072, 3, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4083, 9, 142, 4087, 11, 4079, 4042, 4086, 0, 0, 4078, 0, 0, 1, 4085, 789, 4053, 10, 4082, 0, 11, 0, 360, 4087, 63, 8, 4084, 10, 8, 1069, 0, 0, 12, 0, 7, 0, 9, 4073, 18, 8, 0, 0, 4, 9, 0, 15, 0, 4088, 0, 16, 1, 0, 7, 10, 0, 0, 12, 4084, 12, 0, 0, 0, 10, 0, 4085, 12, 15, 31, 9, 11, 0, 0, 15, 0, 13, 0, 9, 0, 55, 0, 4081, 0, 4085, 13, 0, 4083, 454, 0, 3, 7, 4082, 0, 0, 0, 1264, 0, 0, 11, 4081, 4084, 0, 29, 12, 0, 10, 4082, 4073, 4083, 4080, 4081, 6, 0, 0, 0, 8, 12, 10, 4067, 4086, 0, 6, 4067, 0, 16, 4019, 5, 4085] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 8, 4096, 0, 4096, 4096, 4096, 0, 1, 4096, 0, 0, 0, 4096, 1, 4096, 0, 4096, 0, 0, 0, 661, 4096, 0, 0, 4096, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 7, 0, 0, 0, 4096, 0, 0, 0, 3441, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 4096, 4048, 0, 0, 4096, 0, 0, 4095, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 13, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 68, 4096, 0, 4096, 0, 0, 0, 1409, 4096, 0, 0, 4096, 0, 0, 394, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 39, 0, 0, 0, 4096, 0, 0, 0, 2213, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 4096, 0, 0, 4082, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4095, 1, 0, 4095, 0, 4094, 4056, 4095, 0, 0, 4092, 0, 0, 1, 4095, 77, 4095, 1, 4094, 0, 1, 0, 1473, 4095, 0, 2, 4094, 1, 1, 1377, 0, 0, 1, 0, 0, 0, 1, 4093, 0, 1, 0, 0, 1, 1, 0, 2, 0, 4095, 0, 2, 1, 0, 1, 1, 0, 12, 1, 4095, 1, 0, 1, 0, 1, 0, 4094, 1, 1, 0, 1, 1, 0, 0, 2, 0, 2, 0, 0, 0, 3, 0, 4094, 0, 4095, 2, 0, 4094, 7, 1, 0, 1, 4094, 0, 9, 0, 1198, 0, 0, 0, 4094, 4095, 0, 0, 2, 0, 1, 4089, 4094, 4095, 4095, 4092, 0, 0, 0, 3, 1, 1, 1, 4091, 4094, 0, 1, 4094, 0, 4, 4083, 1, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4029, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 4096, 0, 0, 0, 671, 2033, 0, 0, 4096, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4062, 2063, 1972, 67, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4066, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2124, 0, 0, 4096, 765, 0, 0, 0, 4096, 0, 0, 0, 2455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4077, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16317, 15216, 16378, 16373, 16360, 17632, 16378, 16418] avg=16384.0000 max=17632 max_rank=5 min=15216 min_rank=1 imbalance_ratio=1.076172 max_load_reduction=113440 max_load_reduction_pct=0.865479 moved_to_echo=66788 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4029, 0, 2, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 4096, 0, 0, 0, 261, 2033, 0, 0, 4096, 0, 0, 2364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4062, 2063, 1972, 67, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4066, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2124, 0, 0, 4096, 653, 0, 0, 0, 4096, 0, 0, 0, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16319, 16955, 16378, 16373, 16360, 15881, 16388, 16418] avg=16384.0000 max=16955 max_rank=1 min=15881 min_rank=5 imbalance_ratio=1.034851 max_load_reduction=114117 max_load_reduction_pct=0.870644 moved_to_echo=65047 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4023, 4, 424, 0, 0, 4089, 4049, 0, 1, 0, 4087, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 311, 4082, 4, 4089, 0, 5, 0, 2250, 2031, 21, 4, 4091, 5, 5, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 5, 0, 4, 4088, 4, 4, 0, 0, 2, 5, 0, 7, 0, 4092, 4091, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 8, 0, 0, 3, 3, 0, 1, 5, 4091, 2, 0, 7, 0, 5, 4058, 2061, 1970, 67, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 5, 7, 14, 4, 5, 0, 0, 6, 0, 5, 0, 1, 0, 38, 4090, 4092, 4057, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 2121, 6, 0, 4090, 54, 2, 1, 2, 4087, 0, 0, 0, 992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 13, 0, 0, 12, 4, 0, 5, 4089, 4082, 0, 4091, 4087, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4089, 4090, 0, 3, 4080, 0, 0, 4072, 3, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16679, 16969, 16393, 16390, 16421, 15444, 16394, 16382] avg=16384.0000 max=16969 max_rank=1 min=15444 min_rank=5 imbalance_ratio=1.035706 max_load_reduction=114103 max_load_reduction_pct=0.870537 moved_to_echo=64641 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4029, 0, 13, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 4096, 0, 4096, 0, 0, 0, 1409, 2033, 0, 0, 4096, 0, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4062, 2063, 1972, 67, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4066, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2124, 0, 0, 4096, 39, 0, 0, 0, 4096, 0, 0, 0, 2213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 4096, 0, 0, 4082, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16330, 16192, 16378, 16373, 16360, 16664, 16380, 16395] avg=16384.0000 max=16664 max_rank=5 min=16192 min_rank=1 imbalance_ratio=1.017090 max_load_reduction=114408 max_load_reduction_pct=0.872864 moved_to_echo=65799 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4015, 9, 142, 0, 11, 4079, 4042, 0, 0, 0, 4078, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 789, 4053, 10, 4082, 0, 11, 0, 360, 2025, 63, 8, 4084, 10, 8, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 7, 0, 9, 4073, 18, 8, 0, 0, 4, 9, 0, 15, 0, 4086, 4087, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 16, 1, 0, 7, 10, 0, 0, 12, 4084, 12, 0, 0, 0, 10, 4058, 2062, 1973, 68, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 12, 15, 31, 9, 11, 0, 0, 15, 0, 13, 0, 9, 0, 55, 4084, 4083, 4058, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 2112, 13, 0, 4083, 454, 0, 3, 7, 4082, 0, 0, 0, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 29, 12, 0, 10, 4082, 4073, 0, 4080, 4081, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 10, 4067, 4086, 0, 6, 4067, 0, 16, 4019, 5, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16572, 16408, 16423, 16485, 16099, 16385, 16323] avg=16384.0000 max=16572 max_rank=1 min=16099 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=114500 max_load_reduction_pct=0.873566 moved_to_echo=65292 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4029, 0, 130, 0, 0, 4096, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 4096, 0, 4096, 0, 0, 0, 1736, 2033, 0, 0, 4096, 0, 0, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 4062, 2063, 1972, 67, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 4092, 4066, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2124, 0, 0, 4096, 961, 0, 0, 0, 4096, 0, 0, 0, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 4091, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4050, 0, 0, 4096, 0, 0, 4083, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16452, 16897, 16379, 16377, 16362, 15869, 16377, 16359] avg=16384.0000 max=16897 max_rank=1 min=15869 min_rank=5 imbalance_ratio=1.031311 max_load_reduction=114175 max_load_reduction_pct=0.871086 moved_to_echo=64967 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4029, 0, 8, 0, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 661, 2033, 0, 0, 4096, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4062, 2063, 1972, 67, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4066, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2124, 0, 0, 4096, 7, 0, 0, 0, 4096, 0, 0, 0, 3441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4048, 0, 0, 4096, 0, 0, 4095, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16326, 15022, 16379, 16373, 16359, 17860, 16384, 16369] avg=16384.0000 max=17860 max_rank=5 min=15022 min_rank=1 imbalance_ratio=1.090088 max_load_reduction=113212 max_load_reduction_pct=0.863739 moved_to_echo=66972 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4028, 1, 0, 0, 0, 4094, 4056, 0, 0, 0, 4092, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 4095, 1, 4094, 0, 1, 0, 1473, 2033, 0, 2, 4094, 1, 1, 1377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4093, 0, 1, 0, 0, 1, 1, 0, 2, 0, 4095, 4095, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 1, 0, 1, 1, 0, 12, 1, 4095, 1, 0, 1, 0, 1, 4061, 2062, 1972, 67, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 1, 1, 0, 0, 2, 0, 2, 0, 0, 0, 3, 4095, 4095, 4064, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2123, 2, 0, 4094, 7, 1, 0, 1, 4094, 0, 9, 0, 1198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 2, 0, 1, 4089, 4094, 0, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 1, 1, 4091, 4094, 0, 1, 4094, 0, 4, 4083, 1, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16272, 17249, 16379, 16390, 16365, 15623, 16386, 16408] avg=16384.0000 max=17249 max_rank=1 min=15623 min_rank=5 imbalance_ratio=1.052795 max_load_reduction=113823 max_load_reduction_pct=0.868401 moved_to_echo=64782 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32256, 128, 768, 0, 128, 32768, 32640, 0, 128, 128, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 128, 128, 0, 0, 128, 128, 0, 128, 32768, 32768, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 0, 17024, 128, 0, 32768, 2944, 128, 128, 128, 32768, 0, 128, 0, 12928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1408, 32768, 128, 32768, 0, 128, 0, 8832, 16256, 128, 128, 32768, 128, 128, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 128, 0, 0, 128, 0, 128, 128, 128, 0, 128, 32768, 32768, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 0, 128, 128, 0, 128, 128, 128, 128, 128, 32768, 128, 0, 128, 0, 128, 32512, 16512, 15872, 640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 128, 128, 32768, 32768, 0, 128, 32768, 0, 128, 32640, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 128, 32768, 128, 128, 0, 0, 128, 128, 0, 128, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 4078, 0, 4096, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 4095, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4096, 0, 0, 0, 0, 1, 0, 26, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 69, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4088, 5, 4090, 0, 1, 0, 5, 2, 5, 4091, 0, 0, 2, 1, 4091, 4082, 0, 1, 4091, 4, 5, 5, 0, 2, 188, 5, 5, 0, 4090, 3, 4090, 4091, 0, 5, 10, 2, 0, 0, 0, 4084, 0, 0, 5, 6, 0, 4092, 4063, 4, 5, 5, 0, 0, 0, 0, 6, 4089, 0, 4091, 0, 4087, 2, 5, 3, 4090, 0, 0, 1, 7, 0, 0, 4, 0, 3983, 0, 4090, 5, 2, 0, 5, 0, 0, 0, 4084, 6, 4010, 4086, 4087, 4091, 0, 0, 0, 4090, 0, 4, 0, 0, 1, 5, 4092, 0, 0, 5, 4091, 5, 0, 4, 0, 6, 6, 6, 0, 4, 6, 4090, 4089, 0, 0, 4091, 4091, 4090, 4, 32, 0, 4090, 5, 2, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 3, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 5, 0, 6, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 3, 0, 0, 0, 4096, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 1, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 0, 4096, 0, 0, 0, 0, 18, 0, 109, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4072, 11, 4081, 0, 0, 0, 4, 0, 21, 4086, 0, 0, 10, 0, 4086, 4063, 0, 0, 4087, 4, 9, 9, 0, 8, 286, 11, 12, 0, 4083, 7, 4083, 4086, 0, 6, 29, 1, 2, 0, 0, 4065, 5, 0, 10, 11, 0, 4088, 4041, 11, 9, 9, 4, 0, 0, 0, 6, 4083, 0, 4085, 0, 4080, 0, 0, 5, 4082, 0, 0, 5, 13, 0, 0, 9, 0, 3931, 0, 4084, 10, 0, 0, 13, 0, 0, 0, 4070, 8, 3946, 4078, 4083, 4085, 0, 0, 0, 4085, 0, 13, 0, 0, 4, 11, 4088, 8, 0, 11, 4085, 6, 0, 8, 0, 11, 13, 8, 8, 0, 12, 4085, 4084, 0, 0, 4084, 4085, 4082, 8, 53, 0, 4085, 14, 18, 5, 12] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4093, 1, 4094, 0, 0, 0, 4, 0, 0, 4095, 0, 0, 0, 0, 4095, 4091, 0, 0, 4095, 1, 1, 1, 0, 0, 28, 1, 1, 0, 4094, 1, 4095, 4095, 0, 1, 1, 0, 0, 0, 0, 4094, 1, 0, 2, 1, 0, 4095, 4088, 1, 1, 0, 1, 0, 0, 0, 0, 4095, 0, 4095, 1, 4094, 0, 0, 1, 4094, 0, 0, 1, 2, 0, 0, 1, 0, 4092, 0, 4095, 1, 0, 0, 1, 0, 0, 0, 4092, 1, 4091, 4094, 4095, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 1, 4075, 1, 0, 2, 4094, 2, 0, 1, 0, 1, 1, 1, 2, 2, 2, 4094, 4095, 0, 0, 4095, 4094, 4095, 1, 4, 0, 4094, 1, 3, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4088, 5, 4090, 0, 1, 0, 5, 2, 5, 7, 0, 0, 2, 1, 4091, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4015, 4, 5, 5, 0, 2, 188, 5, 5, 0, 4090, 3, 4090, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 10, 2, 0, 0, 0, 4084, 0, 0, 5, 6, 0, 4092, 4063, 4, 4007, 76, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 6, 4089, 0, 4091, 0, 4087, 2, 5, 3, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 4, 0, 3983, 0, 4090, 5, 2, 0, 5, 0, 4091, 4091, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 6, 4010, 4086, 4087, 0, 0, 0, 0, 20, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 4092, 0, 0, 5, 4091, 5, 0, 4, 0, 6, 6, 6, 0, 4, 4084, 4035, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4077, 4089, 0, 0, 0, 0, 4090, 4, 32, 0, 4090, 5, 2, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16504, 16367, 16383, 16335, 16297, 16407, 16400] avg=16384.0000 max=16504 max_rank=1 min=16297 min_rank=5 imbalance_ratio=1.007324 max_load_reduction=114568 max_load_reduction_pct=0.874084 moved_to_echo=65439 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4072, 11, 4081, 0, 0, 0, 4, 0, 21, 5, 0, 0, 10, 0, 4086, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4010, 4, 9, 9, 0, 8, 286, 11, 12, 0, 4083, 7, 4083, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 29, 1, 2, 0, 0, 4065, 5, 0, 10, 11, 0, 4088, 4041, 11, 4008, 77, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 4, 0, 0, 0, 6, 4083, 0, 4083, 0, 4080, 0, 0, 5, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 13, 0, 0, 9, 0, 3931, 0, 4084, 10, 0, 0, 13, 0, 4085, 4084, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 8, 3946, 4078, 4083, 0, 0, 0, 0, 12, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 11, 4088, 8, 0, 11, 4085, 6, 0, 8, 0, 11, 13, 8, 8, 0, 4081, 4029, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4066, 4084, 0, 0, 0, 0, 4082, 8, 53, 0, 4085, 14, 18, 5, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16353, 16608, 16375, 16361, 16290, 16210, 16436, 16439] avg=16384.0000 max=16608 max_rank=1 min=16210 min_rank=5 imbalance_ratio=1.013672 max_load_reduction=114464 max_load_reduction_pct=0.873291 moved_to_echo=65375 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4012, 76, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4078, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 4095, 4096, 4096, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4041, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16391, 16308, 16389, 16390, 16422, 16415, 16386, 16371] avg=16384.0000 max=16422 max_rank=4 min=16308 min_rank=1 imbalance_ratio=1.002319 max_load_reduction=114650 max_load_reduction_pct=0.874710 moved_to_echo=65594 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 0, 3, 0, 3, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4012, 76, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 5, 4096, 4096, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4040, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16391, 16314, 16389, 16384, 16442, 16410, 16371, 16371] avg=16384.0000 max=16442 max_rank=4 min=16314 min_rank=1 imbalance_ratio=1.003540 max_load_reduction=114630 max_load_reduction_pct=0.874557 moved_to_echo=65594 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4012, 76, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4096, 0, 0, 0, 0, 1, 4096, 4096, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4041, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16391, 16311, 16389, 16384, 16421, 16430, 16375, 16371] avg=16384.0000 max=16430 max_rank=5 min=16311 min_rank=1 imbalance_ratio=1.002808 max_load_reduction=114642 max_load_reduction_pct=0.874649 moved_to_echo=65597 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 0, 2, 0, 3, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 4096, 0, 4012, 76, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 1, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 0, 4096, 0, 0, 0, 0, 18, 4096, 4096, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4041, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16391, 16313, 16390, 16404, 16325, 16513, 16365, 16371] avg=16384.0000 max=16513 max_rank=5 min=16313 min_rank=1 imbalance_ratio=1.007874 max_load_reduction=114559 max_load_reduction_pct=0.874016 moved_to_echo=65574 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 0, 0, 0, 0, 0, 69, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4012, 76, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 4096, 0, 0, 0, 0, 9, 4096, 4096, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4040, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16391, 16377, 16389, 16384, 16398, 16404, 16358, 16371] avg=16384.0000 max=16404 max_rank=5 min=16358 min_rank=6 imbalance_ratio=1.001221 max_load_reduction=114668 max_load_reduction_pct=0.874847 moved_to_echo=65531 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4093, 1, 4094, 0, 0, 0, 4, 0, 0, 7, 0, 0, 0, 0, 4095, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 1, 1, 1, 0, 0, 28, 1, 1, 0, 4094, 1, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4094, 1, 0, 2, 1, 0, 4095, 4088, 1, 4011, 76, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 4095, 0, 4095, 1, 4094, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 1, 0, 4092, 0, 4095, 1, 0, 0, 1, 0, 4095, 4095, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 4091, 4094, 4095, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4075, 1, 0, 2, 4094, 2, 0, 1, 0, 1, 1, 1, 2, 2, 4088, 4038, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4081, 4095, 0, 0, 0, 0, 4095, 1, 4, 0, 4094, 1, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16337, 16384, 16382, 16439, 16393, 16374, 16378] avg=16384.0000 max=16439 max_rank=4 min=16337 min_rank=1 imbalance_ratio=1.003357 max_load_reduction=114633 max_load_reduction_pct=0.874580 moved_to_echo=65584 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 32768, 0, 128, 0, 128, 128, 128, 128, 0, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 32640, 32768, 0, 0, 0, 0, 32768, 128, 128, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 32768, 128, 0, 128, 32768, 128, 0, 128, 0, 128, 128, 128, 128, 128, 32768, 32384, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 32768, 128, 32768, 128, 32768, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 0, 128, 128, 0, 32384, 0, 32768, 128, 128, 0, 128, 128, 32768, 32768, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 256, 32768, 128, 32640, 32768, 32768, 0, 0, 0, 0, 256, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 32256, 128, 128, 128, 128, 128, 640, 128, 128, 0, 32768, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 32768, 128, 0, 128, 128, 0, 32768, 32768, 128, 32128, 640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[7, 134, 1375, 0, 349, 0, 911, 3, 2540, 92, 255, 2640, 1008, 299, 0, 1341, 790, 495, 654, 957, 3880, 2122, 195, 46, 3, 1732, 206, 176, 1145, 12, 1109, 1581, 4051, 1, 11, 1322, 2739, 36, 666, 1797, 2467, 1249, 465, 322, 0, 427, 541, 650, 1578, 1574, 895, 108, 1, 6, 2997, 194, 0, 3127, 3409, 1864, 293, 244, 854, 1, 767, 4005, 0, 896, 174, 55, 1551, 1873, 1139, 659, 131, 2, 64, 581, 416, 343, 1704, 1462, 14, 3717, 51, 2866, 2822, 3836, 56, 5, 552, 1288, 2954, 437, 2390, 2, 1691, 1268, 3165, 213, 3, 2214, 897, 2171, 3911, 1436, 6, 801, 1163, 1253, 92, 359, 13, 815, 2573, 940, 24, 784, 268, 1488, 1008, 902, 1186, 1750, 56, 1, 1859, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[3, 312, 1252, 1, 352, 0, 829, 2, 2163, 49, 231, 2217, 828, 468, 0, 904, 828, 982, 414, 542, 3917, 2033, 173, 44, 2, 1747, 179, 210, 1427, 7, 1011, 1370, 4076, 0, 10, 1290, 2544, 15, 483, 2637, 2385, 1425, 158, 316, 4, 598, 352, 921, 1036, 1659, 1073, 13, 0, 1, 3526, 266, 2, 3218, 3612, 2444, 317, 298, 873, 0, 869, 4074, 0, 1178, 131, 8, 1702, 1675, 983, 667, 42, 0, 66, 844, 520, 497, 2112, 1375, 2, 3789, 35, 2439, 2402, 4050, 73, 18, 937, 989, 3282, 450, 2046, 0, 1009, 1294, 3582, 133, 0, 1891, 438, 2195, 3686, 1616, 0, 1053, 899, 1836, 80, 564, 29, 824, 2422, 505, 5, 580, 208, 1168, 983, 1148, 1444, 1980, 58, 0, 2138, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[5, 306, 1095, 0, 389, 0, 700, 3, 2245, 79, 288, 1993, 1216, 451, 0, 1195, 757, 639, 363, 952, 3940, 1963, 178, 31, 0, 1586, 317, 242, 803, 21, 1089, 1422, 4086, 0, 11, 1058, 3075, 11, 1576, 2297, 2931, 1556, 137, 147, 4, 1041, 457, 716, 700, 1331, 1018, 85, 1, 9, 3308, 228, 0, 3132, 3014, 2490, 287, 510, 416, 0, 739, 4057, 0, 774, 179, 27, 2453, 1974, 1583, 655, 138, 0, 86, 504, 356, 264, 1970, 1319, 10, 3656, 38, 2056, 2355, 4059, 84, 29, 1029, 1381, 3167, 480, 2710, 0, 1002, 1654, 3718, 119, 0, 2311, 588, 1770, 3505, 1599, 2, 927, 1120, 1628, 83, 943, 126, 216, 1845, 456, 15, 562, 206, 1333, 1335, 1055, 1514, 1422, 75, 2, 1909, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[8, 214, 1176, 0, 387, 0, 650, 5, 1714, 87, 298, 2597, 1032, 361, 0, 1033, 1001, 679, 934, 990, 3911, 2402, 169, 52, 1, 1612, 244, 172, 889, 3, 587, 1205, 4065, 0, 12, 1287, 2293, 18, 841, 2300, 2801, 1175, 160, 100, 0, 568, 544, 1247, 954, 1548, 980, 8, 1, 0, 3671, 553, 1, 2515, 3655, 2514, 452, 405, 810, 0, 976, 4062, 0, 876, 200, 65, 2404, 1649, 1276, 754, 156, 1, 126, 477, 524, 500, 1728, 1146, 5, 3838, 42, 2295, 2214, 4014, 78, 4, 987, 1916, 3120, 343, 2286, 0, 710, 1217, 3847, 165, 1, 2064, 474, 2061, 3616, 1508, 2, 1047, 1042, 1377, 146, 865, 58, 691, 2317, 526, 40, 605, 149, 1035, 1057, 1410, 1705, 1468, 51, 0, 1865, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2, 140, 1028, 0, 292, 0, 976, 0, 2418, 144, 306, 2263, 1216, 328, 0, 1120, 649, 887, 545, 610, 3936, 2036, 128, 26, 0, 1954, 216, 49, 1308, 2, 1174, 1351, 4085, 1, 7, 1447, 2985, 18, 351, 2534, 2842, 806, 150, 217, 6, 720, 443, 936, 668, 1399, 917, 17, 2, 0, 3666, 254, 0, 3229, 3622, 2622, 347, 471, 548, 0, 771, 4047, 0, 831, 148, 28, 1631, 2199, 1391, 788, 88, 9, 50, 447, 503, 260, 1743, 1467, 3, 3908, 43, 2386, 2236, 4039, 55, 9, 815, 1056, 3375, 220, 2666, 7, 1045, 1465, 3793, 133, 0, 2235, 548, 2213, 3684, 1637, 1, 696, 1425, 1648, 73, 587, 45, 803, 2259, 400, 3, 609, 199, 1032, 767, 1011, 1069, 1822, 88, 1, 2148, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2, 263, 1085, 3, 364, 1, 578, 11, 2110, 82, 129, 1876, 1017, 368, 0, 1352, 937, 894, 930, 544, 3840, 2339, 164, 35, 0, 820, 216, 161, 1299, 11, 987, 1427, 4082, 0, 7, 1233, 3484, 22, 777, 2385, 2715, 1171, 177, 166, 0, 545, 465, 972, 823, 1072, 750, 64, 3, 2, 3654, 321, 1, 3098, 3595, 2310, 433, 244, 983, 17, 1307, 4064, 0, 1330, 197, 32, 2606, 1449, 1410, 1007, 215, 4, 405, 395, 401, 338, 1388, 1271, 14, 3864, 48, 2280, 2309, 4043, 56, 9, 459, 1265, 2972, 363, 1941, 1, 550, 1391, 3753, 247, 0, 2340, 705, 2285, 3319, 1308, 12, 1462, 1456, 2267, 97, 673, 74, 863, 2181, 778, 9, 509, 179, 1118, 851, 760, 1549, 1325, 107, 16, 2028, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[5, 342, 1297, 2, 358, 1, 723, 31, 2396, 109, 226, 2626, 1111, 610, 3, 1370, 1210, 635, 742, 926, 3947, 2221, 137, 26, 5, 1476, 309, 191, 1390, 2, 1030, 1439, 4089, 1, 9, 1033, 3420, 21, 768, 2320, 2497, 1325, 227, 273, 0, 645, 419, 699, 908, 1090, 1171, 22, 32, 6, 3694, 369, 0, 3232, 3075, 1649, 367, 391, 477, 0, 992, 4045, 0, 1563, 179, 15, 2109, 1878, 1737, 764, 95, 4, 120, 456, 415, 303, 1259, 1511, 7, 3797, 67, 2508, 2152, 3967, 51, 20, 582, 821, 3271, 367, 2344, 1, 944, 1610, 3650, 85, 0, 2120, 803, 1998, 3720, 1914, 0, 1110, 835, 2268, 109, 564, 142, 532, 1647, 744, 7, 496, 141, 1145, 1085, 837, 934, 1307, 69, 1, 1729, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1, 180, 1078, 3, 328, 2, 860, 2, 2129, 84, 178, 2031, 901, 378, 0, 1626, 793, 394, 1128, 828, 3784, 2048, 167, 38, 0, 1266, 200, 193, 958, 2, 978, 1568, 4093, 0, 9, 1139, 3476, 25, 683, 2420, 2438, 1505, 266, 126, 1, 636, 692, 1017, 1165, 1565, 1061, 36, 4, 5, 3313, 245, 1, 2894, 3534, 2384, 617, 237, 599, 0, 1008, 4062, 0, 525, 74, 39, 2692, 2093, 1603, 776, 217, 2, 199, 456, 272, 481, 1935, 786, 5, 3655, 21, 2061, 2039, 4017, 32, 9, 678, 1135, 2975, 321, 1916, 0, 709, 1228, 3792, 216, 0, 2211, 585, 2239, 3926, 1485, 0, 1112, 1530, 1344, 34, 953, 85, 673, 2359, 630, 10, 444, 181, 1296, 987, 1070, 1657, 1592, 57, 1, 2263, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[7, 134, 1375, 0, 349, 0, 911, 3, 2540, 92, 255, 2640, 1008, 299, 0, 1341, 3836, 2463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 790, 495, 654, 957, 3880, 2122, 195, 46, 3, 1732, 206, 176, 1145, 12, 1109, 1581, 835, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2633, 1, 11, 1322, 2739, 36, 666, 1797, 2467, 1249, 465, 322, 0, 427, 541, 650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1578, 1574, 895, 108, 1, 6, 2106, 194, 0, 3127, 3409, 1864, 293, 244, 854, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 767, 4005, 0, 896, 174, 55, 1551, 1873, 1139, 659, 131, 2, 64, 581, 416, 343, 1418, 588, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1704, 1462, 14, 762, 51, 2866, 2822, 0, 56, 5, 552, 1288, 2954, 437, 2390, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1691, 1268, 2330, 213, 3, 2214, 897, 2171, 0, 1436, 6, 801, 1163, 1253, 92, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 815, 2573, 940, 24, 784, 268, 1488, 1008, 902, 1186, 1750, 56, 1, 1859, 4, 2367, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17253, 16443, 15326, 16254, 15048, 17365, 15897, 17486] avg=16384.0000 max=17486 max_rank=7 min=15048 min_rank=4 imbalance_ratio=1.067261 max_load_reduction=113586 max_load_reduction_pct=0.866592 moved_to_echo=65796 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 306, 1095, 0, 389, 0, 700, 3, 2245, 79, 288, 1993, 1216, 451, 0, 1195, 4059, 2206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 639, 363, 952, 3940, 1963, 178, 31, 0, 1586, 317, 242, 803, 21, 1089, 1422, 981, 557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2656, 0, 11, 1058, 3075, 11, 1576, 2297, 2931, 1556, 137, 147, 4, 1041, 457, 716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 700, 1331, 1018, 85, 1, 9, 2325, 228, 0, 3132, 3014, 2490, 287, 510, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 739, 4057, 0, 774, 179, 27, 2453, 1974, 1583, 655, 138, 0, 86, 504, 356, 264, 1430, 579, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1970, 1319, 10, 749, 38, 2056, 2355, 0, 84, 29, 1029, 1381, 3167, 480, 2710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 1654, 2737, 119, 0, 2311, 588, 1770, 0, 1599, 2, 927, 1120, 1628, 83, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 216, 1845, 456, 15, 562, 206, 1333, 1335, 1055, 1514, 1422, 75, 2, 1909, 0, 2328, 1299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16230, 15841, 17673, 15546, 16224, 17377, 16483, 15698] avg=16384.0000 max=17673 max_rank=2 min=15546 min_rank=3 imbalance_ratio=1.078674 max_load_reduction=113399 max_load_reduction_pct=0.865166 moved_to_echo=65782 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[3, 312, 1252, 1, 352, 0, 829, 2, 2163, 49, 231, 2217, 828, 468, 0, 904, 4050, 2321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 982, 414, 542, 3917, 2033, 173, 44, 2, 1747, 179, 210, 1427, 7, 1011, 1370, 945, 594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2650, 0, 10, 1290, 2544, 15, 483, 2637, 2385, 1425, 158, 316, 4, 598, 352, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1036, 1659, 1073, 13, 0, 1, 2477, 266, 2, 3218, 3612, 2444, 317, 298, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 869, 4074, 0, 1178, 131, 8, 1702, 1675, 983, 667, 42, 0, 66, 844, 520, 497, 1426, 600, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2112, 1375, 2, 776, 35, 2439, 2402, 0, 73, 18, 937, 989, 3282, 450, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1294, 2637, 133, 0, 1891, 438, 2195, 0, 1616, 0, 1053, 899, 1836, 80, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 824, 2422, 505, 5, 580, 208, 1168, 983, 1148, 1444, 1980, 58, 0, 2138, 0, 2413, 1365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15982, 16425, 15788, 17289, 15737, 16936, 15645, 17270] avg=16384.0000 max=17289 max_rank=3 min=15645 min_rank=6 imbalance_ratio=1.055237 max_load_reduction=113783 max_load_reduction_pct=0.868095 moved_to_echo=65588 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[8, 214, 1176, 0, 387, 0, 650, 5, 1714, 87, 298, 2597, 1032, 361, 0, 1033, 4014, 2276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 679, 934, 990, 3911, 2402, 169, 52, 1, 1612, 244, 172, 889, 3, 587, 1205, 1015, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2643, 0, 12, 1287, 2293, 18, 841, 2300, 2801, 1175, 160, 100, 0, 568, 544, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 954, 1548, 980, 8, 1, 0, 2580, 553, 1, 2515, 3655, 2514, 452, 405, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 976, 4062, 0, 876, 200, 65, 2404, 1649, 1276, 754, 156, 1, 126, 477, 524, 500, 1422, 607, 473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1728, 1146, 5, 787, 42, 2295, 2214, 0, 78, 4, 987, 1916, 3120, 343, 2286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 710, 1217, 2832, 165, 1, 2064, 474, 2061, 0, 1508, 2, 1047, 1042, 1377, 146, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 691, 2317, 526, 40, 605, 149, 1035, 1057, 1410, 1705, 1468, 51, 0, 1865, 0, 2444, 1340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15852, 16484, 15989, 16976, 16548, 16951, 15511, 16761] avg=16384.0000 max=16976 max_rank=3 min=15511 min_rank=6 imbalance_ratio=1.036133 max_load_reduction=114096 max_load_reduction_pct=0.870483 moved_to_echo=65771 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 140, 1028, 0, 292, 0, 976, 0, 2418, 144, 306, 2263, 1216, 328, 0, 1120, 4039, 2320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 649, 887, 545, 610, 3936, 2036, 128, 26, 0, 1954, 216, 49, 1308, 2, 1174, 1351, 1004, 621, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2651, 1, 7, 1447, 2985, 18, 351, 2534, 2842, 806, 150, 217, 6, 720, 443, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 1399, 917, 17, 2, 0, 2567, 254, 0, 3229, 3622, 2622, 347, 471, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 771, 4047, 0, 831, 148, 28, 1631, 2199, 1391, 788, 88, 9, 50, 447, 503, 260, 1434, 623, 478, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1743, 1467, 3, 793, 43, 2386, 2236, 0, 55, 9, 815, 1056, 3375, 220, 2666, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 1465, 2789, 133, 0, 2235, 548, 2213, 0, 1637, 1, 696, 1425, 1648, 73, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 803, 2259, 400, 3, 609, 199, 1032, 767, 1011, 1069, 1822, 88, 1, 2148, 0, 2492, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16592, 16496, 16114, 16663, 15726, 16874, 16495, 16112] avg=16384.0000 max=16874 max_rank=5 min=15726 min_rank=4 imbalance_ratio=1.029907 max_load_reduction=114198 max_load_reduction_pct=0.871262 moved_to_echo=65207 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2, 263, 1085, 3, 364, 1, 578, 11, 2110, 82, 129, 1876, 1017, 368, 0, 1352, 4043, 2089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 937, 894, 930, 544, 3840, 2339, 164, 35, 0, 820, 216, 161, 1299, 11, 987, 1427, 990, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2654, 0, 7, 1233, 3484, 22, 777, 2385, 2715, 1171, 177, 166, 0, 545, 465, 972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 823, 1072, 750, 64, 3, 2, 2568, 321, 1, 3098, 3595, 2310, 433, 244, 983, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1307, 4064, 0, 1330, 197, 32, 2606, 1449, 1410, 1007, 215, 4, 405, 395, 401, 338, 1428, 611, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1388, 1271, 14, 792, 48, 2280, 2309, 0, 56, 9, 459, 1265, 2972, 363, 1941, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 550, 1391, 2763, 247, 0, 2340, 705, 2285, 0, 1308, 12, 1462, 1456, 2267, 97, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 863, 2181, 778, 9, 509, 179, 1118, 851, 760, 1549, 1325, 107, 16, 2028, 1, 2461, 1230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15373, 16209, 16773, 16284, 17670, 15168, 17556, 16039] avg=16384.0000 max=17670 max_rank=4 min=15168 min_rank=5 imbalance_ratio=1.078491 max_load_reduction=113402 max_load_reduction_pct=0.865189 moved_to_echo=66433 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1, 180, 1078, 3, 328, 2, 860, 2, 2129, 84, 178, 2031, 901, 378, 0, 1626, 4017, 2472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 793, 394, 1128, 828, 3784, 2048, 167, 38, 0, 1266, 200, 193, 958, 2, 978, 1568, 1000, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2661, 0, 9, 1139, 3476, 25, 683, 2420, 2438, 1505, 266, 126, 1, 636, 692, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1165, 1565, 1061, 36, 4, 5, 2328, 245, 1, 2894, 3534, 2384, 617, 237, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1008, 4062, 0, 525, 74, 39, 2692, 2093, 1603, 776, 217, 2, 199, 456, 272, 481, 1432, 578, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1935, 786, 5, 749, 21, 2061, 2039, 0, 32, 9, 678, 1135, 2975, 321, 1916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 709, 1228, 2792, 216, 0, 2211, 585, 2239, 0, 1485, 0, 1112, 1530, 1344, 34, 953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 673, 2359, 630, 10, 444, 181, 1296, 987, 1070, 1657, 1592, 57, 1, 2263, 7, 2328, 1454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16270, 15903, 17094, 16675, 16936, 14662, 16438, 17094] avg=16384.0000 max=17094 max_rank=2 min=14662 min_rank=5 imbalance_ratio=1.043335 max_load_reduction=113978 max_load_reduction_pct=0.869583 moved_to_echo=65130 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 342, 1297, 2, 358, 1, 723, 31, 2396, 109, 226, 2626, 1111, 610, 3, 1370, 3967, 2343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1210, 635, 742, 926, 3947, 2221, 137, 26, 5, 1476, 309, 191, 1390, 2, 1030, 1439, 963, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2658, 1, 9, 1033, 3420, 21, 768, 2320, 2497, 1325, 227, 273, 0, 645, 419, 699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 908, 1090, 1171, 22, 32, 6, 2596, 369, 0, 3232, 3075, 1649, 367, 391, 477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 992, 4045, 0, 1563, 179, 15, 2109, 1878, 1737, 764, 95, 4, 120, 456, 415, 303, 1431, 601, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1259, 1511, 7, 778, 67, 2508, 2152, 0, 51, 20, 582, 821, 3271, 367, 2344, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 944, 1610, 2687, 85, 0, 2120, 803, 1998, 0, 1914, 0, 1110, 835, 2268, 109, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 532, 1647, 744, 7, 496, 141, 1145, 1085, 837, 934, 1307, 69, 1, 1729, 1, 2418, 1377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17520, 17271, 16315, 15385, 17183, 15739, 17047, 14612] avg=16384.0000 max=17520 max_rank=0 min=14612 min_rank=7 imbalance_ratio=1.069336 max_load_reduction=113552 max_load_reduction_pct=0.866333 moved_to_echo=64581 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[13952, 10368, 128, 6272, 384, 18944, 18560, 0, 512, 128, 6144, 9856, 25216, 3072, 18304, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[7040, 5632, 5760, 6400, 31232, 17280, 1408, 384, 128, 12288, 1920, 1408, 9344, 128, 8064, 11392, 7808, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[21248, 128, 128, 9856, 24064, 256, 6272, 18816, 21120, 10240, 1792, 1792, 128, 5248, 3968, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 1920, 9472, 128, 2944, 128, 6272, 128, 17792, 768, 1920, 18304, 8448, 3328, 128, 9984, 32128, 18560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[7680, 11136, 21632, 1408, 128, 17408, 5120, 17024, 0, 12544, 128, 8320, 9472, 13696, 768, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[7936, 11264, 7936, 384, 128, 128, 19584, 2432, 128, 24448, 27520, 18304, 3200, 2816, 5632, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[640, 5504, 17664, 4992, 128, 4608, 1536, 9728, 8192, 8320, 11136, 12672, 640, 128, 16000, 128, 19328, 10880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[7552, 32512, 0, 8064, 1408, 384, 17152, 14848, 11136, 6144, 1152, 128, 1152, 4224, 3456, 3072, 11520, 4864, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4095, 4096, 575, 0, 4, 213, 4087, 4096, 3992, 4096, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 4065, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 4085, 0, 1, 4066, 0, 0, 0, 4094, 0, 4096, 0, 0, 10, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2451, 3, 0, 0, 0, 0, 0, 0, 0, 3979, 4070, 0, 952, 4096, 4096, 0, 0, 0, 0, 143, 0, 0, 404, 0, 0, 4096, 160, 0, 4068, 0, 0, 0, 0, 5, 0, 4093, 3483, 0, 4096, 0, 62, 0, 0, 0, 0, 4060, 4096, 0, 2513, 1, 3, 1, 0, 1701, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 1298, 0, 3, 25, 3902, 4070, 4060, 4096, 0, 0, 81, 0, 0, 4068, 0, 0, 3, 0, 0, 1, 4094, 0, 0, 4096, 0, 0, 0, 0, 4081, 0, 4096, 0, 8, 4069, 20, 0, 2, 0, 4091, 4066, 0, 4, 0, 4082, 0, 5, 3957, 0, 0, 0, 4095, 24, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 970, 4, 35, 0, 0, 0, 0, 0, 0, 3925, 4033, 0, 2083, 4096, 4086, 0, 0, 0, 0, 97, 0, 0, 1163, 4, 0, 4083, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 4077, 4085, 0, 4096, 0, 222, 12, 0, 0, 0, 3939, 4096, 0, 1813, 607, 21, 0, 0, 777, 0, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 284, 0, 0, 18, 3981, 4091, 3983, 4095, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 4095, 0, 0, 5, 0, 4096, 0, 4096, 0, 0, 4093, 0, 0, 0, 3, 4096, 4080, 0, 0, 0, 4086, 0, 1, 4094, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2213, 0, 0, 0, 0, 0, 0, 0, 0, 3677, 3991, 0, 183, 4096, 4096, 0, 0, 0, 0, 289, 0, 0, 2228, 7, 0, 4095, 17, 0, 4096, 0, 0, 0, 0, 1, 0, 3990, 4055, 0, 4096, 0, 141, 0, 0, 0, 0, 4070, 4096, 0, 2552, 93, 5, 1, 0, 978, 0, 123] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4095, 4096, 1729, 0, 8, 40, 3886, 4090, 4024, 4095, 0, 0, 237, 0, 0, 4094, 0, 0, 0, 0, 6, 5, 4096, 0, 0, 4095, 0, 0, 8, 1, 4090, 1, 4096, 0, 0, 4089, 2, 0, 0, 17, 4091, 4088, 0, 5, 0, 3900, 0, 0, 3944, 0, 0, 3, 4091, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 6, 3, 0, 0, 0, 0, 0, 6, 1609, 0, 0, 0, 0, 0, 2, 0, 0, 3990, 4069, 3, 1118, 4092, 4091, 0, 6, 1, 0, 250, 1, 0, 43, 90, 0, 4095, 237, 1, 4042, 0, 0, 0, 0, 3, 0, 4081, 4082, 0, 4091, 3, 34, 8, 0, 1, 0, 4036, 4090, 7, 2382, 187, 31, 0, 3, 1060, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 1735, 0, 0, 35, 3915, 4094, 3751, 4095, 0, 0, 2, 0, 0, 4096, 0, 0, 4, 4, 0, 3, 4096, 0, 0, 4090, 0, 1, 0, 0, 4096, 0, 4095, 0, 0, 4082, 0, 0, 0, 0, 4081, 4096, 0, 0, 0, 4068, 0, 9, 4018, 0, 0, 0, 4092, 0, 4096, 0, 3, 0, 4096, 0, 0, 0, 1, 4, 0, 0, 0, 4, 0, 0, 1439, 14, 0, 0, 0, 0, 0, 0, 0, 3894, 4092, 0, 1990, 4096, 4096, 0, 0, 0, 0, 327, 0, 0, 797, 41, 0, 4086, 11, 0, 4075, 0, 0, 0, 0, 25, 0, 3846, 4072, 0, 4085, 0, 8, 4, 0, 0, 0, 4096, 4096, 0, 1083, 20, 0, 2, 0, 1817, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4088, 4095, 749, 0, 0, 25, 4005, 4094, 3965, 4096, 1, 0, 30, 2, 0, 4096, 7, 0, 0, 13, 0, 0, 4095, 0, 3, 4092, 7, 0, 9, 0, 4090, 0, 4092, 3, 0, 4078, 0, 3, 0, 19, 4093, 4075, 0, 0, 0, 4096, 16, 14, 4077, 0, 1, 0, 3922, 0, 4092, 0, 0, 0, 4087, 2, 11, 0, 1, 0, 0, 4, 1, 0, 0, 9, 3606, 0, 0, 0, 0, 0, 0, 0, 0, 3880, 4094, 0, 526, 4096, 4092, 0, 0, 0, 0, 127, 1, 0, 2533, 125, 0, 3891, 88, 1, 4067, 4, 0, 0, 0, 0, 0, 4072, 3977, 0, 4090, 0, 9, 0, 0, 0, 0, 4068, 4094, 0, 557, 109, 13, 1, 0, 532, 0, 161] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4095, 4096, 2165, 0, 7, 4, 3973, 4096, 3645, 4096, 0, 3, 148, 0, 1, 4089, 3, 0, 0, 38, 0, 0, 4095, 1, 2, 4092, 0, 0, 1, 0, 4094, 0, 4091, 5, 13, 4096, 0, 2, 8, 0, 4095, 4096, 0, 4, 1, 4056, 0, 1, 3795, 0, 0, 1, 4087, 20, 4094, 0, 0, 0, 4089, 0, 0, 1, 11, 0, 1, 1, 1, 28, 0, 0, 481, 41, 0, 3, 0, 0, 3, 0, 0, 3878, 4091, 0, 1947, 4089, 4090, 0, 6, 1, 1, 353, 0, 0, 151, 18, 0, 4087, 581, 3, 3912, 0, 0, 0, 0, 0, 0, 4075, 3929, 0, 4096, 0, 9, 16, 1, 0, 0, 4086, 4091, 0, 2746, 9, 0, 0, 0, 937, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 4084, 4092, 628, 4, 0, 18, 4053, 4093, 3302, 4094, 4, 5, 10, 1, 0, 4093, 10, 1, 1, 27, 1, 0, 4096, 0, 0, 4073, 0, 8, 0, 0, 4085, 2, 4095, 0, 1, 4088, 0, 0, 0, 4, 4089, 4093, 1, 0, 4, 3850, 0, 19, 4089, 2, 0, 5, 3982, 1, 4093, 0, 1, 0, 4091, 0, 10, 0, 1, 1, 0, 0, 0, 0, 0, 5, 1634, 131, 0, 2, 2, 2, 28, 0, 1, 4093, 4092, 4, 1826, 4094, 4090, 7, 0, 0, 0, 977, 2, 0, 275, 0, 4, 4094, 124, 0, 4008, 1, 0, 2, 0, 0, 1, 4074, 3910, 0, 4088, 8, 309, 1, 1, 0, 0, 4054, 4090, 1, 2777, 4, 1, 2, 0, 941, 4, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3219, 0, 1298, 0, 3, 25, 3902, 4070, 4060, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 3, 0, 0, 1, 4094, 0, 0, 4096, 0, 0, 0, 0, 2422, 899, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 8, 4069, 20, 0, 2, 0, 81, 4066, 0, 4, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3957, 0, 0, 0, 4095, 24, 4096, 0, 0, 0, 4096, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 970, 4, 35, 0, 0, 0, 0, 0, 4096, 4096, 4096, 2253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3925, 4033, 0, 2083, 1674, 4086, 0, 0, 0, 0, 97, 0, 0, 1163, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 4077, 4085, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 12, 0, 0, 0, 3939, 4096, 0, 1813, 607, 21, 0, 0, 777, 0, 11, 3054, 1757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16658, 16347, 16413, 16386, 15550, 17065, 16344, 16309] avg=16384.0000 max=17065 max_rank=5 min=15550 min_rank=4 imbalance_ratio=1.041565 max_load_reduction=114007 max_load_reduction_pct=0.869804 moved_to_echo=65268 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3218, 0, 575, 0, 4, 213, 4087, 4096, 3992, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 1, 0, 0, 2422, 899, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 81, 4095, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4066, 0, 0, 0, 4094, 0, 4096, 0, 0, 10, 4095, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2451, 3, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 2255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 4070, 0, 952, 1674, 4096, 0, 0, 0, 0, 143, 0, 0, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 160, 0, 4068, 0, 0, 0, 0, 5, 0, 4093, 3483, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 4060, 4096, 0, 2513, 1, 3, 1, 0, 1701, 0, 1, 3054, 1759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16188, 16373, 16422, 16475, 16997, 15318, 16048, 17251] avg=16384.0000 max=17251 max_rank=7 min=15318 min_rank=5 imbalance_ratio=1.052917 max_load_reduction=113821 max_load_reduction_pct=0.868385 moved_to_echo=65614 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3219, 0, 284, 0, 0, 18, 3981, 4091, 3983, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 4095, 0, 0, 5, 0, 2422, 899, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 3, 82, 4080, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2213, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 2255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3677, 3991, 0, 183, 1674, 4096, 0, 0, 0, 0, 289, 0, 0, 2228, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 17, 0, 4096, 0, 0, 0, 0, 1, 0, 3990, 4055, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 0, 0, 4070, 4096, 0, 2552, 93, 5, 1, 0, 978, 0, 123, 3054, 1759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15583, 16385, 16440, 16494, 16756, 16145, 16397, 16872] avg=16384.0000 max=16872 max_rank=7 min=15583 min_rank=0 imbalance_ratio=1.029785 max_load_reduction=114200 max_load_reduction_pct=0.871277 moved_to_echo=66170 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3218, 0, 1729, 0, 8, 40, 3886, 4090, 4024, 0, 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 6, 5, 4096, 0, 0, 4095, 0, 0, 8, 1, 2419, 898, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 4089, 2, 0, 0, 17, 81, 4088, 0, 5, 0, 3900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3944, 0, 0, 3, 4091, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 0, 0, 0, 6, 1609, 0, 0, 0, 0, 0, 2, 0, 4095, 4096, 4096, 2253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 4069, 3, 1118, 1673, 4091, 0, 6, 1, 0, 250, 1, 0, 43, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 237, 1, 4042, 0, 0, 0, 0, 3, 0, 4081, 4082, 0, 143, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 8, 0, 1, 0, 4036, 4090, 7, 2382, 187, 31, 0, 3, 1060, 0, 5, 3050, 1757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17232, 16386, 16273, 16342, 16166, 15335, 16687, 16651] avg=16384.0000 max=17232 max_rank=0 min=15335 min_rank=5 imbalance_ratio=1.051758 max_load_reduction=113840 max_load_reduction_pct=0.868530 moved_to_echo=64839 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3210, 0, 1735, 0, 0, 35, 3915, 4094, 3751, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 4, 0, 3, 4096, 0, 0, 4090, 0, 1, 0, 0, 2425, 899, 767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 72, 4096, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4018, 0, 0, 0, 4092, 0, 4096, 0, 3, 0, 4096, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 4, 0, 0, 1439, 14, 0, 0, 0, 0, 0, 0, 4095, 4096, 4095, 2250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3894, 4092, 0, 1990, 1671, 4096, 0, 0, 0, 0, 327, 0, 0, 797, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 11, 0, 4075, 0, 0, 0, 0, 25, 0, 3846, 4072, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 0, 0, 0, 4096, 4096, 0, 1083, 20, 0, 2, 0, 1817, 0, 6, 3047, 1759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16742, 16385, 16414, 16433, 15998, 16908, 16254, 15938] avg=16384.0000 max=16908 max_rank=5 min=15938 min_rank=7 imbalance_ratio=1.031982 max_load_reduction=114164 max_load_reduction_pct=0.871002 moved_to_echo=65098 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3212, 0, 749, 0, 0, 25, 4005, 4094, 3965, 0, 1, 0, 30, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 13, 0, 0, 4095, 0, 3, 4092, 7, 0, 9, 0, 2422, 898, 763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 3, 0, 4078, 0, 3, 0, 19, 81, 4075, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 14, 4077, 0, 1, 0, 3922, 0, 4092, 0, 0, 0, 4087, 2, 11, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 1, 0, 0, 9, 3606, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4092, 2254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3880, 4094, 0, 526, 1674, 4092, 0, 0, 0, 0, 127, 1, 0, 2533, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3891, 88, 1, 4067, 4, 0, 0, 0, 0, 0, 4072, 3977, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4068, 4094, 0, 557, 109, 13, 1, 0, 532, 0, 161, 3049, 1758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16083, 16405, 16445, 16335, 18158, 17052, 16243, 14351] avg=16384.0000 max=18158 max_rank=4 min=14351 min_rank=7 imbalance_ratio=1.108276 max_load_reduction=112914 max_load_reduction_pct=0.861465 moved_to_echo=65804 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3209, 0, 628, 4, 0, 18, 4053, 4093, 3302, 0, 4, 5, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 10, 1, 1, 27, 1, 0, 4096, 0, 0, 4073, 0, 8, 0, 0, 2420, 897, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 2, 0, 0, 1, 4088, 0, 0, 0, 4, 81, 4093, 1, 0, 4, 3850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4089, 2, 0, 5, 3982, 1, 4093, 0, 1, 0, 4091, 0, 10, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 5, 1634, 131, 0, 2, 2, 2, 28, 0, 4094, 4092, 4095, 2252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 4092, 4, 1826, 1674, 4090, 7, 0, 0, 0, 977, 2, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 124, 0, 4008, 1, 0, 2, 0, 0, 1, 4074, 3910, 0, 143, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 1, 1, 0, 0, 4054, 4090, 1, 2777, 4, 1, 2, 0, 941, 4, 3, 3048, 1756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15327, 16389, 16209, 16406, 16339, 17041, 16369, 16992] avg=16384.0000 max=17041 max_rank=5 min=15327 min_rank=0 imbalance_ratio=1.040100 max_load_reduction=114031 max_load_reduction_pct=0.869987 moved_to_echo=66741 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3218, 0, 2165, 0, 7, 4, 3973, 4096, 3645, 0, 0, 3, 148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 3, 0, 0, 38, 0, 0, 4095, 1, 2, 4092, 0, 0, 1, 0, 2417, 899, 764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 5, 13, 4096, 0, 2, 8, 0, 81, 4096, 0, 4, 1, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3795, 0, 0, 1, 4087, 20, 4094, 0, 0, 0, 4089, 0, 0, 1, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1, 1, 1, 28, 0, 0, 481, 41, 0, 3, 0, 0, 3, 0, 4096, 4096, 4091, 2255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3878, 4091, 0, 1947, 1672, 4090, 0, 6, 1, 1, 353, 0, 0, 151, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 581, 3, 3912, 0, 0, 0, 0, 0, 0, 4075, 3929, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 16, 1, 0, 0, 4086, 4091, 0, 2746, 9, 0, 0, 0, 937, 0, 0, 3054, 1759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17259, 16402, 16456, 16201, 15108, 16208, 16730, 16708] avg=16384.0000 max=17259 max_rank=0 min=15108 min_rank=4 imbalance_ratio=1.053406 max_load_reduction=113813 max_load_reduction_pct=0.868324 moved_to_echo=64754 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 640, 32768, 128, 128, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32640, 1280, 128, 32256, 128, 0, 128, 0, 128, 128, 32384, 31616, 0, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 32128, 128, 128, 128, 32512, 128, 32768, 0, 128, 128, 32768, 128, 128, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 25728, 0, 9216, 128, 128, 384, 31872, 32768, 30848, 0, 128, 128, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 31360, 32640, 128, 10752, 13440, 32768, 128, 128, 128, 128, 2688, 128, 0, 7680, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 19456, 7296, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 128, 14464, 256, 128, 128, 128, 128, 128, 0, 32768, 32768, 32768, 18048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[896, 128, 128, 128, 0, 32512, 32768, 128, 16512, 1152, 128, 128, 128, 8832, 128, 384, 24448, 14080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2361, 0, 0, 0, 0, 0, 4096, 0, 97, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 1103, 4096, 0, 0, 3909, 0, 0, 4083, 0, 3324, 0, 0, 4096, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 1901, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 4096, 0, 0, 22, 0, 0, 496, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4095, 3994, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 3126, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1507, 0, 0, 0, 0, 0, 4096, 0, 77, 3, 0, 0, 0, 6, 4096, 0, 0, 4096, 0, 0, 4096, 654, 4096, 0, 0, 3057, 0, 0, 3825, 0, 3382, 0, 0, 4096, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3184, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4096, 0, 4096, 0, 0, 151, 0, 1, 921, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4093, 4096, 4094, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 4096, 0, 4095, 0, 4095, 0, 0, 0, 3680, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 0, 41] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[16, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 783, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 194, 4096, 1, 3, 3881, 0, 0, 3949, 0, 3660, 0, 0, 4096, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 4096, 0, 4080, 2581, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 4075, 0, 0, 1, 0, 0, 1658, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4093, 4081, 4096, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4094, 0, 0, 0, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 28] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[39, 0, 1, 4096, 4095, 0, 0, 0, 4096, 0, 5, 1428, 1, 0, 1, 0, 0, 4094, 0, 8, 24, 1, 0, 0, 0, 4095, 0, 0, 4094, 0, 0, 4096, 88, 4096, 22, 0, 4035, 0, 0, 3909, 0, 4086, 0, 0, 4096, 0, 0, 0, 0, 108, 0, 0, 0, 0, 1, 1, 4096, 0, 4083, 3294, 4094, 0, 11, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4095, 2, 4096, 0, 0, 0, 0, 0, 769, 0, 0, 4095, 0, 0, 5, 0, 4088, 4096, 0, 0, 4096, 0, 1, 4081, 4068, 4096, 0, 0, 0, 0, 8, 0, 4092, 0, 0, 4095, 0, 4096, 6, 4096, 0, 0, 2, 2697, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 2, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[32, 0, 0, 4096, 4096, 0, 0, 0, 4091, 0, 0, 658, 0, 0, 0, 0, 0, 4096, 0, 77, 32, 0, 1, 0, 0, 4096, 0, 0, 4089, 0, 0, 4095, 146, 4096, 5, 0, 4072, 1, 0, 3419, 0, 4002, 0, 0, 4096, 0, 0, 0, 0, 908, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2052, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4088, 0, 0, 6, 0, 0, 1312, 0, 0, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4075, 4069, 4094, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 4096, 0, 4093, 0, 4096, 0, 0, 5, 3805, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[6, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 3147, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 529, 4096, 17, 4, 4004, 0, 0, 4016, 0, 3991, 0, 0, 4096, 0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 4095, 0, 4088, 1260, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 0, 0, 36, 0, 0, 930, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4090, 4086, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 1, 2259, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 14] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[64, 0, 0, 4092, 4092, 1, 3, 0, 4092, 2, 0, 2491, 0, 0, 4, 0, 0, 4092, 1, 126, 0, 0, 1, 1, 0, 4094, 0, 3, 4092, 0, 5, 4092, 1093, 4090, 32, 13, 4009, 6, 0, 4095, 0, 3296, 2, 4, 4094, 3, 3, 0, 3, 406, 1, 0, 0, 0, 0, 2, 4093, 3, 4091, 2375, 4094, 4, 14, 0, 4, 1, 0, 0, 5, 3, 0, 4093, 4091, 0, 4053, 1, 0, 0, 0, 3, 143, 0, 2, 4093, 0, 7, 7, 1, 4092, 4093, 0, 19, 4092, 0, 1, 4076, 4071, 4089, 5, 1, 5, 0, 1, 0, 4094, 1, 0, 4092, 0, 4091, 8, 4087, 0, 4, 0, 2366, 0, 2, 0, 4, 0, 0, 3, 0, 4090, 0, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 4083, 0, 0, 0, 4092, 0, 0, 1206, 2, 0, 0, 0, 3, 4096, 0, 122, 1, 0, 4, 0, 0, 4093, 1, 0, 4092, 1, 0, 4096, 82, 4096, 9, 6, 3869, 0, 0, 4070, 0, 3434, 0, 0, 4093, 0, 1, 2, 0, 667, 0, 1, 0, 0, 1, 0, 4095, 0, 4096, 2612, 4096, 1, 5, 0, 1, 0, 0, 0, 0, 0, 0, 4080, 4096, 0, 4090, 0, 0, 0, 2, 0, 1330, 4, 0, 4095, 0, 0, 1, 0, 4092, 4094, 1, 3, 4091, 0, 0, 4096, 4096, 4096, 0, 0, 4, 0, 0, 0, 4091, 0, 0, 4096, 0, 4093, 0, 4071, 0, 0, 0, 3120, 1, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 3, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1507, 0, 0, 0, 0, 2381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 77, 3, 0, 0, 0, 6, 4096, 0, 0, 4096, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 654, 4096, 0, 0, 3057, 0, 0, 3825, 0, 3382, 0, 0, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3184, 4082, 0, 0, 0, 939, 346, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4096, 0, 4096, 0, 0, 151, 0, 1, 3078, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 921, 0, 0, 0, 0, 0, 0, 0, 4096, 3157, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 32, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3680, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 0, 41, 4096, 4096, 984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16176, 16397, 15379, 16866, 16488, 16363, 16398, 17005] avg=16384.0000 max=17005 max_rank=7 min=15379 min_rank=2 imbalance_ratio=1.037903 max_load_reduction=114067 max_load_reduction_pct=0.870262 moved_to_echo=66254 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[16, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 783, 0, 0, 0, 0, 2384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 194, 4096, 1, 3, 3881, 0, 0, 3949, 0, 3660, 0, 0, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 4096, 0, 4080, 2581, 4091, 0, 0, 0, 942, 352, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 4075, 0, 0, 1, 0, 0, 3082, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1658, 0, 0, 0, 0, 0, 0, 0, 4096, 3154, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 24, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3684, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 28, 4096, 4096, 990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15471, 16311, 16139, 16312, 16357, 17097, 16395, 16990] avg=16384.0000 max=17097 max_rank=5 min=15471 min_rank=0 imbalance_ratio=1.043518 max_load_reduction=113975 max_load_reduction_pct=0.869560 moved_to_echo=66839 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2361, 0, 0, 0, 0, 2381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 97, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1103, 4096, 0, 0, 3909, 0, 0, 4083, 0, 3324, 0, 0, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 1901, 4096, 0, 0, 0, 939, 346, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 4096, 0, 0, 22, 0, 0, 3080, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 0, 0, 0, 0, 0, 0, 0, 4096, 3157, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 31, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3126, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 4096, 4096, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17030, 16409, 16880, 15703, 16391, 15940, 16313, 16406] avg=16384.0000 max=17030 max_rank=0 min=15703 min_rank=3 imbalance_ratio=1.039429 max_load_reduction=114042 max_load_reduction_pct=0.870071 moved_to_echo=65050 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 0, 4096, 4096, 0, 0, 0, 4091, 0, 0, 658, 0, 0, 0, 0, 2381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 77, 32, 0, 1, 0, 0, 4096, 0, 0, 4089, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 4096, 5, 0, 4072, 1, 0, 3419, 0, 4002, 0, 0, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 908, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2052, 4096, 0, 0, 0, 939, 346, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4088, 0, 0, 6, 0, 0, 3078, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1312, 0, 0, 0, 0, 0, 1, 0, 4096, 3157, 0, 0, 4096, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 32, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3805, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 4096, 4096, 984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15354, 16413, 16106, 16606, 16372, 16737, 16382, 17102] avg=16384.0000 max=17102 max_rank=7 min=15354 min_rank=0 imbalance_ratio=1.043823 max_load_reduction=113970 max_load_reduction_pct=0.869522 moved_to_echo=66593 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 3147, 0, 0, 0, 0, 2381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 529, 4096, 17, 4, 4004, 0, 0, 4016, 0, 3991, 0, 0, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 4095, 0, 4088, 1260, 4096, 0, 0, 0, 939, 346, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 0, 0, 36, 0, 0, 3080, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 930, 0, 0, 0, 0, 0, 0, 0, 4096, 3157, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 31, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2259, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 14, 4096, 4096, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17822, 16333, 17022, 15165, 16407, 16369, 16405, 15549] avg=16384.0000 max=17822 max_rank=0 min=15165 min_rank=3 imbalance_ratio=1.087769 max_load_reduction=113250 max_load_reduction_pct=0.864029 moved_to_echo=64730 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[39, 0, 1, 4096, 4095, 0, 0, 0, 4096, 0, 5, 1428, 1, 0, 1, 0, 2381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 8, 24, 1, 0, 0, 0, 4095, 0, 0, 4094, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 4096, 22, 0, 4035, 0, 0, 3909, 0, 4086, 0, 0, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 1, 1, 4096, 0, 4083, 3294, 4094, 0, 11, 0, 939, 346, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4095, 2, 4096, 0, 0, 0, 0, 0, 3080, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 769, 0, 0, 0, 0, 0, 5, 0, 4088, 3157, 0, 0, 4096, 0, 1, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 31, 0, 0, 0, 0, 8, 0, 4092, 0, 0, 0, 0, 4096, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2697, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 2, 2, 4095, 4095, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16143, 16339, 16601, 17046, 16374, 16197, 16397, 15975] avg=16384.0000 max=17046 max_rank=3 min=15975 min_rank=7 imbalance_ratio=1.040405 max_load_reduction=114026 max_load_reduction_pct=0.869949 moved_to_echo=64943 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[64, 0, 0, 4092, 4092, 1, 3, 0, 4092, 2, 0, 2491, 0, 0, 4, 0, 2380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 126, 0, 0, 1, 1, 0, 4094, 0, 3, 4092, 0, 5, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 4090, 32, 13, 4009, 6, 0, 4095, 0, 3296, 2, 4, 365, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 406, 1, 0, 0, 0, 0, 2, 4093, 3, 4091, 2375, 4094, 4, 14, 0, 938, 346, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 5, 3, 0, 4093, 4091, 0, 4053, 1, 0, 0, 0, 3, 3074, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 2, 0, 0, 7, 7, 1, 4092, 3155, 0, 19, 4092, 0, 1, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 32, 5, 1, 5, 0, 1, 0, 4094, 1, 0, 0, 0, 4091, 8, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2366, 0, 2, 0, 4, 0, 0, 3, 0, 4090, 0, 0, 4, 4092, 4093, 983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17221, 16434, 17011, 16443, 16331, 15595, 16396, 15641] avg=16384.0000 max=17221 max_rank=0 min=15595 min_rank=5 imbalance_ratio=1.051086 max_load_reduction=113851 max_load_reduction_pct=0.868614 moved_to_echo=63963 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 4083, 0, 0, 0, 4092, 0, 0, 1206, 2, 0, 0, 0, 2379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 122, 1, 0, 4, 0, 0, 4093, 1, 0, 4092, 1, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 4096, 9, 6, 3869, 0, 0, 4070, 0, 3434, 0, 0, 365, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, 0, 1, 0, 0, 1, 0, 4095, 0, 4096, 2612, 4096, 1, 5, 0, 938, 346, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4080, 4096, 0, 4090, 0, 0, 0, 2, 0, 3080, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1330, 4, 0, 0, 0, 0, 1, 0, 4092, 3156, 1, 3, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 31, 0, 0, 4, 0, 0, 0, 4091, 0, 0, 0, 0, 4093, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3120, 1, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 3, 11, 4096, 4095, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15855, 16436, 15934, 16931, 16352, 16774, 16386, 16404] avg=16384.0000 max=16931 max_rank=3 min=15855 min_rank=0 imbalance_ratio=1.033386 max_load_reduction=114141 max_load_reduction_pct=0.870827 moved_to_echo=65916 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32640, 256, 128, 128, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 0, 128, 32768, 32768, 128, 128, 0, 32768, 128, 128, 13696, 128, 0, 128, 0, 19072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[7680, 128, 128, 0, 0, 128, 128, 128, 32768, 25344, 128, 128, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[3968, 32768, 128, 128, 30848, 128, 0, 31488, 0, 29184, 128, 128, 2944, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 24832, 128, 128, 0, 128, 128, 0, 128, 128, 32768, 128, 128, 128, 32768, 32768, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 0, 128, 128, 0, 32768, 32768, 128, 32768, 128, 0, 256, 128, 128, 24704, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 128, 640, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 2688, 128, 128, 0, 0, 128, 128, 32768, 128, 32768, 19328, 32768, 128, 128, 0, 7552, 2816, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3486, 0, 4096, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 665, 6, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 4047, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 3981, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3549, 0, 4096, 4096, 0, 0, 20, 0, 0, 0, 1, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4050, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 34, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 0, 274, 49, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 186, 4096, 0, 0, 3, 4096, 4096, 0, 0, 4095, 0, 4061, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4044, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 71] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3591, 0, 4096, 4094, 0, 0, 18, 0, 0, 0, 2, 0, 0, 52, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 13, 1, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 11, 5, 0, 0, 2, 0, 0, 0, 5, 0, 1, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4095, 0, 1042, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3, 4096, 4096, 0, 5, 4076, 0, 3414, 0, 4096, 0, 4096, 0, 0, 2, 0, 0, 4088, 4093, 0, 4096, 0, 0, 4096, 0, 4007, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 4096, 132] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3394, 0, 4096, 4084, 0, 0, 6, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4084, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4095, 4096, 0, 690, 51, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 7, 4096, 0, 0, 4, 4096, 4096, 0, 0, 4083, 0, 4071, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4053, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 18] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[6, 2, 3, 4, 2317, 0, 4083, 4082, 0, 7, 22, 4, 0, 7, 0, 2, 0, 5, 4, 4090, 5, 6, 0, 5, 0, 4082, 0, 4084, 0, 0, 0, 0, 0, 4, 0, 5, 5, 0, 4092, 0, 4085, 0, 0, 18, 0, 0, 0, 4, 6, 0, 5, 4, 4, 0, 5, 4091, 4092, 4, 3, 4090, 4091, 4091, 4092, 0, 2055, 7, 3, 0, 4, 4, 4088, 0, 5, 4093, 4, 4, 1, 4087, 1, 0, 0, 4089, 4092, 0, 0, 4070, 0, 3770, 0, 4090, 0, 4091, 12, 2, 0, 5, 1, 4092, 4092, 0, 4092, 0, 4, 4088, 0, 4089, 3, 0, 4, 2, 0, 42, 7, 4, 2, 0, 4072, 0, 0, 4, 1, 5, 4089, 0, 8, 4092, 4091, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4042, 0, 4096, 4086, 0, 0, 6, 0, 0, 0, 1, 0, 0, 43, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 1, 53, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 24, 4096, 4096, 0, 0, 4093, 0, 4050, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 4095, 0, 0, 4096, 0, 3974, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3809, 0, 4096, 4094, 0, 0, 8, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 4096, 0, 4091, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 4096, 4094, 4095, 4, 352, 24, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 24, 4096, 4096, 0, 0, 4094, 0, 3989, 0, 4096, 0, 4096, 0, 0, 3, 0, 0, 4091, 4096, 0, 4091, 0, 0, 4096, 0, 4024, 0, 0, 0, 0, 0, 31, 6, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4089, 0, 0, 4094, 4096, 12] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3746, 0, 4093, 4092, 0, 0, 19, 0, 0, 0, 0, 0, 4, 71, 0, 4095, 0, 0, 0, 0, 0, 4091, 0, 4091, 0, 2, 0, 7, 5, 0, 3, 0, 0, 0, 4095, 0, 4086, 0, 0, 80, 0, 0, 0, 0, 0, 7, 0, 0, 30, 0, 0, 4096, 4094, 0, 0, 4093, 4093, 4093, 4096, 0, 367, 0, 3, 1, 0, 0, 4095, 0, 0, 4089, 0, 0, 14, 4091, 0, 1, 5, 4093, 4096, 0, 0, 4085, 0, 3981, 3, 4096, 0, 4096, 0, 0, 2, 0, 1, 4094, 4095, 0, 4091, 2, 0, 4096, 0, 4096, 0, 0, 0, 0, 3, 3, 5, 0, 0, 0, 3993, 0, 0, 0, 5, 0, 4096, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3549, 0, 4096, 4096, 0, 0, 20, 0, 0, 0, 1, 0, 4096, 562, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4050, 0, 0, 0, 4, 3500, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 34, 1, 0, 0, 0, 0, 4096, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 49, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 186, 4096, 0, 0, 3347, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 4095, 0, 4061, 0, 4096, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 34, 0, 4044, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4084, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16452, 16327, 16382, 16378, 16162, 16524, 16406, 16441] avg=16384.0000 max=16524 max_rank=5 min=16162 min_rank=4 imbalance_ratio=1.008545 max_load_reduction=114548 max_load_reduction_pct=0.873932 moved_to_echo=65533 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3486, 0, 4096, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 562, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 3500, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 101, 0, 0, 0, 0, 4096, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 665, 6, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3347, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 4047, 0, 4096, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 34, 0, 3981, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16368, 16367, 16453, 16378, 16324, 16507, 16320, 16355] avg=16384.0000 max=16507 max_rank=5 min=16320 min_rank=6 imbalance_ratio=1.007507 max_load_reduction=114565 max_load_reduction_pct=0.874062 moved_to_echo=65506 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3394, 0, 4096, 4084, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 565, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4084, 0, 0, 0, 2, 3506, 579, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 1, 0, 0, 0, 0, 4096, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4084, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 690, 51, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 7, 4096, 0, 0, 3348, 16, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 4083, 0, 4071, 0, 4096, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 25, 0, 4053, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4080, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16274, 16372, 16353, 16379, 16412, 16519, 16376, 16387] avg=16384.0000 max=16519 max_rank=5 min=16274 min_rank=0 imbalance_ratio=1.008240 max_load_reduction=114553 max_load_reduction_pct=0.873970 moved_to_echo=65694 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4042, 0, 4096, 4086, 0, 0, 6, 0, 0, 0, 1, 0, 4096, 562, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 19, 3500, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 1, 0, 0, 1, 0, 4096, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3347, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4096, 0, 0, 0, 4093, 0, 4050, 0, 4096, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4095, 0, 0, 34, 0, 3974, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4084, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16922, 16425, 16351, 16379, 15706, 16532, 16383, 16374] avg=16384.0000 max=16922 max_rank=0 min=15706 min_rank=4 imbalance_ratio=1.032837 max_load_reduction=114150 max_load_reduction_pct=0.870895 moved_to_echo=64995 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3809, 0, 4096, 4094, 0, 0, 8, 0, 0, 0, 5, 0, 4095, 562, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2, 3500, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4091, 0, 29, 0, 0, 0, 0, 0, 4096, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4090, 0, 0, 4096, 4096, 4094, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 352, 24, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 3347, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4096, 0, 0, 0, 4094, 0, 3989, 0, 4096, 0, 173, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 4091, 0, 0, 34, 0, 4024, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4089, 0, 0, 4094, 4084, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16701, 16366, 16377, 16381, 16030, 16475, 16367, 16375] avg=16384.0000 max=16701 max_rank=0 min=16030 min_rank=4 imbalance_ratio=1.019348 max_load_reduction=114371 max_load_reduction_pct=0.872581 moved_to_echo=65247 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3591, 0, 4096, 4094, 0, 0, 18, 0, 0, 0, 2, 0, 4095, 562, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 13, 3500, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 11, 5, 0, 0, 2, 0, 4096, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 0, 4090, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1042, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3347, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 5, 4076, 0, 3414, 0, 4096, 0, 173, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4093, 0, 4096, 0, 0, 34, 0, 4007, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 4084, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16491, 16428, 16363, 16384, 16696, 15865, 16347, 16498] avg=16384.0000 max=16696 max_rank=4 min=15865 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=114376 max_load_reduction_pct=0.872620 moved_to_echo=65406 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[6, 2, 3, 4, 2317, 0, 4083, 4082, 0, 7, 22, 4, 0, 7, 0, 2, 4092, 561, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 4090, 5, 6, 0, 5, 0, 4082, 0, 4084, 0, 0, 0, 0, 3494, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 5, 0, 4092, 0, 4085, 0, 0, 18, 0, 0, 0, 4, 4092, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 5, 4, 4, 0, 5, 0, 4086, 4, 3, 4090, 4091, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2055, 7, 3, 0, 4, 4, 4088, 0, 5, 4093, 4, 4, 1, 4087, 1, 0, 3342, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4070, 0, 3770, 0, 4090, 0, 174, 12, 2, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 4092, 0, 4092, 0, 4, 33, 0, 4089, 3, 0, 4, 2, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 2, 0, 4072, 0, 0, 4, 1, 5, 4089, 0, 8, 4092, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15224, 16350, 16364, 16389, 17716, 16212, 16454, 16363] avg=16384.0000 max=17716 max_rank=4 min=15224 min_rank=0 imbalance_ratio=1.081299 max_load_reduction=113356 max_load_reduction_pct=0.864838 moved_to_echo=66745 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3746, 0, 4093, 4092, 0, 0, 19, 0, 0, 0, 0, 0, 4096, 562, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 71, 0, 4095, 0, 0, 0, 0, 0, 4091, 0, 4091, 0, 2, 0, 7, 3500, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3, 0, 0, 0, 4095, 0, 4086, 0, 0, 80, 0, 0, 0, 0, 4096, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 30, 0, 0, 0, 4088, 0, 0, 4093, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 3, 1, 0, 0, 4095, 0, 0, 4089, 0, 0, 14, 4091, 0, 1, 3347, 12, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4093, 0, 0, 0, 4085, 0, 3981, 3, 4096, 0, 173, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 4095, 0, 4091, 2, 0, 34, 0, 4096, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3993, 0, 0, 0, 5, 0, 4096, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16640, 16437, 16429, 16404, 16026, 16438, 16419, 16279] avg=16384.0000 max=16640 max_rank=0 min=16026 min_rank=4 imbalance_ratio=1.015625 max_load_reduction=114432 max_load_reduction_pct=0.873047 moved_to_echo=65162 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 0, 32768, 128, 128, 256, 0, 0, 128, 128, 32768, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 256, 128, 32768, 128, 128, 0, 128, 0, 32768, 0, 32768, 0, 128, 0, 128, 28032, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 28032, 0, 32768, 32768, 0, 128, 128, 128, 0, 128, 128, 128, 32768, 4608, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 32768, 0, 32768, 128, 128, 384, 0, 32384, 128, 0, 128, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 0, 0, 128, 32768, 0, 31488, 128, 32768, 0, 1408, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[5504, 256, 128, 128, 128, 128, 32768, 0, 128, 32768, 128, 128, 256, 32768, 128, 128, 26880, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 32640, 0, 0, 128, 128, 128, 32768, 0, 128, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 0, 32768, 128, 128, 32768, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4005, 0, 4096, 0, 0, 4096, 4044, 0, 0, 4096, 4072, 4096, 0, 0, 37, 0, 0, 4089, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 50, 19, 4096, 0, 0, 0, 2111, 0, 0, 0, 0, 4096, 4095, 0, 0, 11, 3794, 0, 4088, 341, 0, 0, 0, 4093, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1636, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 379, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3903, 0, 4096, 0, 0, 4096, 3965, 0, 0, 4096, 4096, 4096, 0, 0, 2, 0, 0, 4095, 0, 4096, 0, 4096, 0, 13, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 178, 4096, 0, 0, 0, 381, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 3554, 0, 4093, 23, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3664, 0, 0, 3528, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 1279, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4034, 0, 4096, 0, 0, 4096, 4086, 0, 0, 4096, 4087, 4096, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 4096, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 125, 228, 4096, 0, 0, 0, 2769, 0, 0, 0, 0, 4096, 4069, 0, 0, 0, 3932, 0, 4092, 388, 0, 0, 0, 4093, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 609, 0, 0, 6, 0, 5, 9, 4092, 0, 0, 0, 4096, 0, 0, 0, 4096, 221, 0, 0, 0, 0, 0, 3, 0, 4096, 2, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4096, 7, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3976, 0, 4087, 0, 0, 4096, 4032, 0, 0, 4096, 4061, 4096, 0, 0, 32, 0, 0, 4096, 0, 4096, 0, 4096, 0, 19, 0, 4096, 0, 0, 0, 0, 0, 4096, 4092, 0, 4096, 112, 164, 4096, 0, 0, 1, 1871, 0, 0, 0, 1, 4096, 3721, 0, 0, 3, 2241, 0, 4096, 0, 0, 0, 0, 4085, 4096, 6, 4094, 0, 0, 0, 0, 4096, 0, 10, 3963, 0, 0, 2, 0, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 1, 4096, 374, 0, 0, 0, 0, 0, 1, 0, 4096, 4, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3887, 0, 4096, 0, 0, 4096, 3994, 0, 0, 4096, 4073, 4096, 0, 0, 0, 0, 5, 4096, 0, 4096, 0, 4096, 0, 25, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 47, 163, 4096, 0, 0, 0, 3046, 0, 0, 0, 0, 4096, 4054, 0, 0, 0, 945, 0, 4096, 1, 0, 0, 0, 4090, 4096, 0, 4096, 0, 0, 0, 0, 4071, 0, 5, 4059, 0, 0, 0, 0, 0, 4, 4055, 0, 0, 0, 4096, 0, 0, 0, 4096, 333, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3717, 0, 4096, 0, 0, 4096, 4088, 0, 0, 4096, 4088, 4096, 0, 0, 2, 0, 5, 4013, 0, 4096, 0, 4096, 0, 28, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 86, 688, 4096, 0, 0, 0, 390, 0, 0, 0, 0, 4096, 4076, 0, 0, 2, 3928, 0, 4096, 511, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 1000, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 4096, 2052, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 4091, 0, 5, 6, 0, 0, 0, 5, 0, 0, 0, 4, 0, 4091, 0, 7, 0, 3803, 0, 4077, 0, 3, 4091, 3897, 1, 5, 4047, 4069, 4089, 5, 0, 0, 4, 4, 3951, 1, 4090, 6, 4090, 0, 253, 7, 4091, 0, 0, 0, 7, 6, 4078, 4091, 0, 4088, 121, 94, 4091, 6, 0, 0, 387, 0, 0, 0, 0, 4090, 4073, 0, 2, 12, 3942, 0, 4092, 0, 3, 5, 3, 4092, 4091, 0, 4091, 1, 4, 5, 0, 4079, 5, 0, 4045, 1, 7, 1, 5, 0, 0, 4071, 2, 5, 6, 4091, 0, 4, 0, 4090, 67, 5, 8, 0, 5, 0, 0, 3, 4090, 16, 5, 1, 0, 0, 4090, 3, 1, 0, 6, 4089, 0, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[3, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4054, 0, 4096, 0, 0, 4096, 4061, 0, 0, 4092, 4054, 4096, 0, 0, 3, 0, 0, 4088, 0, 4096, 0, 4096, 0, 12, 0, 4093, 2, 0, 1, 0, 0, 4096, 4090, 0, 4095, 52, 85, 4092, 0, 0, 1, 2143, 0, 0, 0, 10, 4096, 3902, 0, 0, 0, 3069, 0, 4093, 53, 0, 0, 0, 4088, 4091, 0, 4096, 0, 0, 0, 0, 4096, 0, 5, 2849, 0, 0, 4, 0, 3, 11, 4096, 1, 0, 0, 4096, 0, 0, 3, 4096, 217, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 1, 4096, 6, 1, 0, 0, 4096, 5, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 275, 0, 0, 0, 4044, 0, 0, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 4089, 0, 4071, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2187, 50, 19, 4096, 0, 0, 0, 2111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 11, 3794, 0, 4088, 341, 0, 0, 0, 4093, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1636, 0, 0, 0, 0, 0, 0, 4096, 1418, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 379, 0, 0, 0, 0, 0, 0, 0, 4096, 3478, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3253, 491, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16492, 16389, 16655, 17265, 15367, 16146, 16375] avg=16384.0000 max=17265 max_rank=4 min=15367 min_rank=5 imbalance_ratio=1.053772 max_load_reduction=113807 max_load_reduction_pct=0.868279 moved_to_echo=65153 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3887, 0, 275, 0, 0, 0, 3994, 0, 0, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 0, 4071, 0, 4096, 0, 25, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2187, 47, 163, 4096, 0, 0, 0, 3046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 0, 945, 0, 4096, 1, 0, 0, 0, 4090, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4071, 0, 5, 4059, 0, 0, 0, 0, 0, 4, 4055, 1418, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 333, 0, 0, 0, 0, 0, 0, 0, 4096, 3478, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 3253, 491, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16325, 16389, 17731, 14029, 17733, 16100, 16382] avg=16384.0000 max=17733 max_rank=5 min=14029 min_rank=4 imbalance_ratio=1.082336 max_load_reduction=113339 max_load_reduction_pct=0.864708 moved_to_echo=64244 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3903, 0, 275, 0, 0, 0, 3965, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4095, 0, 4071, 0, 4096, 0, 13, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 2187, 0, 178, 4096, 0, 0, 0, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 3554, 0, 4093, 23, 0, 0, 0, 4096, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3664, 0, 0, 3528, 0, 0, 0, 0, 0, 0, 4096, 1418, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1279, 0, 0, 0, 0, 0, 0, 0, 4096, 3478, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 3253, 491, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16335, 16373, 15033, 16696, 16827, 17046, 16376] avg=16384.0000 max=17046 max_rank=6 min=15033 min_rank=3 imbalance_ratio=1.040405 max_load_reduction=114026 max_load_reduction_pct=0.869949 moved_to_echo=66945 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 275, 0, 0, 0, 4086, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4071, 0, 4096, 0, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2187, 125, 228, 4096, 0, 0, 0, 2769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 3932, 0, 4092, 388, 0, 0, 0, 4093, 843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 609, 0, 0, 6, 0, 5, 9, 4092, 1418, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 221, 0, 0, 0, 0, 0, 3, 0, 4096, 3478, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4096, 7, 0, 0, 0, 3253, 491, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16578, 16362, 17597, 17417, 14357, 15991, 16386] avg=16384.0000 max=17597 max_rank=3 min=14357 min_rank=5 imbalance_ratio=1.074036 max_load_reduction=113475 max_load_reduction_pct=0.865746 moved_to_echo=64151 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3717, 0, 265, 0, 0, 0, 4088, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 4013, 0, 4070, 0, 4096, 0, 28, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2177, 86, 688, 4096, 0, 0, 0, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 2, 3928, 0, 4096, 511, 0, 0, 0, 4095, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 1000, 0, 0, 0, 0, 0, 0, 4087, 1424, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2052, 0, 0, 0, 0, 0, 0, 0, 4096, 3483, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3258, 495, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 16254, 16310, 15629, 17546, 14722, 17824, 16395] avg=16384.0000 max=17824 max_rank=6 min=14722 min_rank=5 imbalance_ratio=1.087891 max_load_reduction=113248 max_load_reduction_pct=0.864014 moved_to_echo=66487 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 0, 274, 0, 0, 0, 4032, 0, 0, 4096, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 4096, 0, 4071, 0, 4096, 0, 19, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 2187, 112, 164, 4096, 0, 0, 1, 1871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3721, 0, 0, 3, 2241, 0, 4096, 0, 0, 0, 0, 4085, 843, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 10, 3963, 0, 0, 2, 0, 0, 4, 4096, 1418, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 374, 0, 0, 0, 0, 0, 1, 0, 4096, 3471, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3253, 491, 342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 16439, 16410, 16619, 14996, 17708, 16136, 16391] avg=16384.0000 max=17708 max_rank=5 min=14996 min_rank=4 imbalance_ratio=1.080811 max_load_reduction=113364 max_load_reduction_pct=0.864899 moved_to_echo=65231 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4091, 0, 5, 6, 0, 0, 0, 5, 0, 0, 0, 4, 0, 4091, 4091, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3803, 0, 274, 0, 3, 0, 3897, 1, 5, 4047, 4069, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 3951, 1, 4066, 6, 4090, 0, 253, 7, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 4078, 4091, 0, 2182, 121, 94, 4091, 6, 0, 0, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 2, 12, 3942, 0, 4092, 0, 3, 5, 3, 4092, 842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 4, 5, 0, 4079, 5, 0, 4045, 1, 7, 1, 5, 0, 0, 4071, 1416, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 6, 4091, 0, 4, 0, 4090, 67, 5, 8, 0, 5, 0, 0, 3, 4089, 3462, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 16, 5, 1, 0, 0, 4090, 3, 1, 0, 6, 4089, 0, 0, 2, 0, 3249, 490, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16111, 16473, 15063, 17066, 17755, 15838, 16383] avg=16384.0000 max=17755 max_rank=5 min=15063 min_rank=3 imbalance_ratio=1.083679 max_load_reduction=113317 max_load_reduction_pct=0.864540 moved_to_echo=67042 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4054, 0, 275, 0, 0, 0, 4061, 0, 0, 4092, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4088, 0, 4071, 0, 4096, 0, 12, 0, 4093, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 2186, 52, 85, 4092, 0, 0, 1, 2143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3902, 0, 0, 0, 3069, 0, 4093, 53, 0, 0, 0, 4088, 842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 5, 2849, 0, 0, 4, 0, 3, 11, 4096, 1418, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 3, 4096, 217, 0, 0, 0, 0, 0, 3, 0, 4096, 3478, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 6, 1, 0, 0, 4096, 5, 0, 0, 0, 3249, 491, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16538, 16366, 16745, 16057, 16603, 15991, 16384] avg=16384.0000 max=16745 max_rank=3 min=15991 min_rank=6 imbalance_ratio=1.022034 max_load_reduction=114327 max_load_reduction_pct=0.872246 moved_to_echo=65035 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 128, 0, 128, 32768, 128, 128, 0, 128, 32768, 128, 0, 128, 0, 26112, 3968, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 32768, 0, 128, 128, 32768, 4992, 128, 128, 0, 128, 0, 128, 128, 32768, 27904, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 0, 0, 128, 128, 0, 0, 0, 128, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 128, 128, 128, 0, 32384, 128, 128, 21760, 128, 128, 128, 128, 128, 128, 32768, 11392, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 32640, 128, 32640, 128, 32768, 0, 384, 128, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 32128, 0, 128, 128, 25472, 0, 32768, 1408, 128, 128, 128, 32768, 6784, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 32768, 0, 17536, 640, 1664, 32768, 128, 0, 128, 13184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 31488, 0, 2304, 0, 128, 0, 32256, 128, 128, 32768, 32640, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 2, 4096, 4077, 4096, 4085, 0, 4096, 4096, 0, 0, 0, 0, 0, 32, 0, 0, 3720, 0, 0, 0, 0, 25, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4094, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1284, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 4083, 0, 0, 0, 51, 3961, 0, 0, 0, 0, 0, 0, 4096, 906, 0, 0, 11, 4096, 0, 0, 2344, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 2530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4095, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4063, 0, 0, 0, 73, 3907, 0, 0, 0, 0, 0, 0, 4095, 1935, 0, 0, 0, 4096, 0, 0, 3747, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 39, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2562, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 989, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 4085, 0, 0, 0, 5, 4077, 0, 0, 2, 0, 0, 0, 4096, 256, 0, 0, 7, 4096, 0, 0, 3077, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 1312, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[0, 4089, 4087, 6, 7, 4, 8, 4091, 0, 4090, 0, 4087, 0, 4, 0, 0, 0, 4084, 4091, 4081, 4090, 18, 4088, 4089, 0, 5, 0, 0, 6, 0, 0, 10, 2304, 4, 0, 0, 0, 0, 0, 6, 9, 0, 4078, 4091, 0, 4091, 4090, 0, 4092, 0, 0, 4034, 0, 0, 6, 0, 1184, 6, 7, 0, 3, 8, 0, 1, 4079, 3, 3, 76, 7, 4066, 2, 5, 4, 270, 4090, 5, 5, 0, 7, 1, 1, 4089, 2630, 1, 0, 0, 4092, 0, 0, 1844, 1, 7, 4, 0, 0, 4070, 7, 7, 4012, 4088, 1, 4089, 0, 0, 4080, 37, 0, 3, 0, 4090, 0, 30, 0, 1, 0, 7, 0, 4, 8, 6, 0, 6, 3, 0, 4088, 0, 4088, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4094, 0, 4096, 0, 4096, 0, 0, 0, 0, 3, 4096, 4093, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 15, 1, 0, 0, 1, 4096, 4095, 0, 4096, 4089, 0, 4096, 0, 0, 4091, 1, 1, 0, 0, 860, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 12, 0, 4088, 0, 0, 0, 40, 3949, 0, 0, 2, 0, 0, 0, 4096, 456, 0, 0, 0, 4096, 0, 2, 2096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 795, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4093, 0, 0, 0, 0, 4092, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4091, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1281, 0, 0, 0, 0, 33, 11, 0, 0, 0, 4096, 4094, 0, 4096, 4079, 0, 4093, 0, 0, 4096, 0, 2, 0, 0, 1588, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 258, 0, 4096, 0, 0, 0, 771, 3475, 0, 0, 6, 0, 0, 0, 4096, 1311, 0, 0, 44, 4082, 0, 0, 2757, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 798, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4093, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3442, 0, 0, 0, 0, 6, 7, 0, 0, 2, 4096, 4096, 0, 4096, 4091, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 722, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 4034, 0, 0, 0, 1363, 3950, 0, 0, 0, 0, 0, 0, 4096, 629, 0, 0, 0, 4094, 0, 0, 1954, 0, 0, 0, 0, 0, 4086, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 284, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[2, 4096, 4093, 0, 0, 0, 0, 4095, 0, 4096, 0, 4096, 0, 0, 3, 0, 5, 4096, 4087, 4096, 4092, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 6, 2740, 0, 0, 0, 0, 35, 1, 0, 0, 2, 4096, 4090, 0, 4096, 4069, 0, 4094, 0, 0, 4096, 2, 30, 0, 0, 517, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4073, 0, 0, 0, 13, 4014, 0, 0, 0, 0, 0, 0, 4096, 1861, 0, 0, 10, 4095, 0, 0, 3110, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 13, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4077, 4096, 4085, 0, 1, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3720, 0, 0, 0, 0, 25, 0, 0, 0, 0, 4096, 4096, 0, 1255, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1284, 0, 0, 0, 0, 0, 0, 0, 4095, 3192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 4083, 0, 0, 0, 51, 3961, 0, 0, 0, 0, 0, 2841, 901, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 906, 0, 0, 11, 4096, 0, 0, 2344, 0, 0, 0, 0, 0, 4089, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 3694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16389, 17286, 16765, 16105, 15772, 15986, 16382] avg=16384.0000 max=17286 max_rank=2 min=15772 min_rank=5 imbalance_ratio=1.055054 max_load_reduction=113786 max_load_reduction_pct=0.868118 moved_to_echo=64245 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1255, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 4095, 3192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4063, 0, 0, 0, 73, 3907, 0, 0, 0, 0, 0, 2841, 901, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1935, 0, 0, 0, 4096, 0, 0, 3747, 0, 0, 0, 0, 0, 4095, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 39, 0, 0, 0, 3694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16386, 16072, 15576, 16050, 18198, 16021, 16382] avg=16384.0000 max=18198 max_rank=5 min=15576 min_rank=3 imbalance_ratio=1.110718 max_load_reduction=112874 max_load_reduction_pct=0.861160 moved_to_echo=66651 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2562, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 4096, 0, 1251, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 989, 0, 0, 0, 0, 0, 0, 0, 4096, 3195, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 4085, 0, 0, 0, 5, 4077, 0, 0, 2, 0, 0, 2845, 901, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 256, 0, 0, 7, 4096, 0, 0, 3077, 1, 0, 0, 0, 0, 4096, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 1312, 0, 0, 0, 3690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16383, 16114, 16470, 16188, 15861, 17290, 16382] avg=16384.0000 max=17290 max_rank=6 min=15861 min_rank=5 imbalance_ratio=1.055298 max_load_reduction=113782 max_load_reduction_pct=0.868088 moved_to_echo=65721 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 4087, 6, 7, 4, 8, 4091, 0, 2, 0, 4087, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4091, 4081, 4090, 18, 1, 0, 0, 5, 0, 0, 6, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2304, 4, 0, 0, 0, 0, 0, 6, 9, 0, 4078, 4091, 0, 1254, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4034, 0, 0, 6, 0, 1184, 6, 7, 0, 3, 8, 0, 1, 4087, 3188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 3, 3, 76, 7, 4066, 2, 5, 4, 270, 4090, 5, 5, 0, 7, 1, 2837, 900, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 2630, 1, 0, 0, 4092, 0, 0, 1844, 1, 7, 4, 0, 0, 4070, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 4012, 4088, 1, 0, 0, 0, 4080, 37, 0, 3, 0, 3690, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 4, 8, 6, 0, 6, 3, 0, 4088, 0, 4088, 6, 4089, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16386, 15836, 16617, 16531, 16968, 15955, 16394] avg=16384.0000 max=16968 max_rank=5 min=15836 min_rank=2 imbalance_ratio=1.035645 max_load_reduction=114104 max_load_reduction_pct=0.870544 moved_to_echo=65848 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4093, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 15, 1, 0, 0, 1, 4096, 4095, 0, 1255, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 1, 1, 0, 0, 860, 0, 0, 0, 0, 0, 0, 0, 4096, 3192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 12, 0, 4088, 0, 0, 0, 40, 3949, 0, 0, 2, 0, 0, 2841, 904, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 456, 0, 0, 0, 4096, 0, 2, 2096, 0, 0, 0, 0, 0, 4096, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 795, 0, 0, 0, 3694, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16384, 17632, 16339, 16103, 15072, 16778, 16382] avg=16384.0000 max=17632 max_rank=2 min=15072 min_rank=5 imbalance_ratio=1.076172 max_load_reduction=113440 max_load_reduction_pct=0.865479 moved_to_echo=64335 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3442, 0, 0, 0, 0, 6, 7, 0, 0, 2, 4096, 4096, 0, 1255, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 722, 0, 0, 0, 0, 0, 0, 0, 4096, 3193, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 11, 0, 4034, 0, 0, 0, 1363, 3950, 0, 0, 0, 0, 0, 2841, 903, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 629, 0, 0, 0, 4094, 0, 0, 1954, 0, 0, 0, 0, 0, 4086, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 284, 0, 0, 0, 3694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16381, 16995, 16205, 17370, 15089, 16266, 16382] avg=16384.0000 max=17370 max_rank=4 min=15089 min_rank=5 imbalance_ratio=1.060181 max_load_reduction=113702 max_load_reduction_pct=0.867477 moved_to_echo=65107 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 4093, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1281, 0, 0, 0, 0, 33, 11, 0, 0, 0, 4096, 4094, 0, 1255, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 2, 0, 0, 1588, 0, 0, 0, 0, 0, 0, 0, 4096, 3195, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 258, 0, 4096, 0, 0, 0, 771, 3475, 0, 0, 6, 0, 0, 2841, 901, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1311, 0, 0, 44, 4082, 0, 0, 2757, 0, 0, 0, 0, 0, 4095, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 798, 0, 0, 0, 3694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16379, 14849, 17072, 16613, 16615, 16780, 16387] avg=16384.0000 max=17072 max_rank=3 min=14849 min_rank=2 imbalance_ratio=1.041992 max_load_reduction=114000 max_load_reduction_pct=0.869751 moved_to_echo=66395 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 4093, 0, 0, 0, 0, 4095, 0, 1, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4087, 4096, 4092, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2740, 0, 0, 0, 0, 35, 1, 0, 0, 2, 4096, 4090, 0, 1255, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 2, 30, 0, 0, 517, 0, 0, 0, 0, 0, 0, 0, 4095, 3192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4073, 0, 0, 0, 13, 4014, 0, 0, 0, 0, 0, 2841, 903, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1861, 0, 0, 10, 4095, 0, 0, 3110, 0, 0, 0, 0, 0, 4095, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 13, 0, 0, 0, 3693, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4094, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16384, 16288, 16028, 16112, 17497, 15996, 16381] avg=16384.0000 max=17497 max_rank=5 min=15996 min_rank=6 imbalance_ratio=1.067932 max_load_reduction=113575 max_load_reduction_pct=0.866508 moved_to_echo=65986 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 32768, 128, 0, 0, 0, 32768, 3328, 0, 128, 0, 29568, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 128, 32768, 0, 128, 0, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 0, 0, 32768, 128, 128, 128, 0, 7296, 128, 128, 0, 128, 128, 0, 128, 32768, 25600, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[32768, 128, 128, 384, 128, 32640, 128, 128, 128, 2688, 31488, 128, 128, 128, 128, 128, 22784, 7296, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[22784, 128, 0, 0, 0, 128, 128, 128, 128, 128, 32768, 32768, 0, 10112, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 9984, 128, 0, 128, 32768, 0, 128, 20992, 128, 128, 128, 0, 0, 32768, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 32768, 32768, 32768, 128, 128, 0, 128, 128, 0, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 0, 128, 128, 128, 0, 128, 128, 128, 32768, 0, 32768, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 197, 4096, 0, 180, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 4095, 0, 0, 645, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4090, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 2176, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2873, 0, 0, 0, 0, 0, 0, 3805, 4096, 4096, 0, 0, 0, 1, 4096, 4096, 0, 4096, 0, 0, 0, 4095, 0, 2449] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 248, 4096, 0, 843, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 752, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 14, 4096, 831, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2317, 0, 0, 0, 0, 0, 0, 3971, 4096, 4095, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 3315] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4088, 0, 443, 4096, 0, 135, 0, 0, 0, 4071, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 4094, 0, 4096, 0, 0, 2254, 4096, 4083, 0, 4083, 0, 0, 0, 0, 0, 4096, 4096, 4094, 4096, 4096, 0, 0, 0, 4081, 0, 0, 7, 0, 4096, 4096, 0, 0, 0, 21, 4096, 731, 4096, 0, 9, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2340, 0, 0, 0, 0, 0, 19, 3769, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4093, 0, 0, 0, 4096, 0, 2605] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4085, 0, 1, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4092, 0, 403, 4096, 0, 669, 0, 0, 0, 4092, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 0, 8, 1126, 4096, 4096, 0, 4096, 5, 0, 0, 0, 0, 4096, 4096, 4096, 4094, 4063, 0, 0, 0, 4088, 0, 0, 1, 0, 4096, 4096, 0, 4, 0, 21, 4096, 8, 4095, 0, 10, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3588, 0, 0, 0, 0, 0, 0, 2441, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4064] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 47, 4096, 0, 1253, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 4096, 0, 21, 552, 4096, 4096, 0, 4092, 13, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4095, 0, 0, 0, 4085, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 4096, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2945, 0, 0, 0, 0, 0, 0, 3981, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 3472] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[5, 0, 4086, 2, 6, 10, 8, 0, 6, 4061, 6, 0, 4090, 0, 4088, 4085, 5, 0, 6, 0, 7, 7, 0, 9, 5, 7, 0, 5, 4, 4, 7, 4035, 0, 224, 4089, 0, 1241, 7, 0, 0, 4090, 3, 19, 0, 7, 0, 10, 9, 0, 0, 0, 0, 4088, 5, 4087, 0, 0, 489, 4089, 4085, 0, 4089, 0, 8, 0, 0, 6, 4087, 4083, 4086, 4084, 4083, 0, 35, 0, 4093, 1, 3, 0, 0, 4089, 4078, 0, 0, 0, 55, 4089, 739, 4071, 0, 0, 0, 0, 3, 23, 5, 4, 9, 4, 15, 12, 4, 0, 0, 2184, 0, 5, 8, 8, 2, 0, 3733, 4090, 4078, 0, 8, 4, 2, 4090, 4091, 7, 4089, 1, 14, 0, 4079, 0, 3625] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 456, 4096, 0, 518, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 4096, 0, 0, 592, 4096, 4093, 0, 4095, 21, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4094, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 4096, 0, 2, 0, 2, 4096, 891, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2152, 0, 0, 0, 0, 0, 5, 3611, 4096, 4092, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4058] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 4096, 0, 0, 0, 0, 0, 0, 4042, 0, 6, 4096, 2, 4095, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 65, 4096, 0, 50, 0, 0, 0, 4091, 0, 0, 6, 0, 7, 0, 0, 0, 0, 17, 1, 4096, 0, 4096, 0, 0, 1387, 4096, 4091, 0, 3886, 1, 0, 0, 0, 0, 4096, 4096, 4095, 4096, 4083, 0, 0, 0, 4077, 0, 0, 0, 0, 4093, 4096, 0, 6, 0, 9, 4096, 12, 4096, 0, 12, 0, 1, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2898, 0, 0, 0, 0, 0, 9, 4047, 4096, 4088, 1, 0, 0, 0, 4095, 4089, 0, 4093, 0, 0, 0, 4096, 0, 4077] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 4096, 0, 843, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3454, 2225, 944, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3152, 0, 1, 752, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 4096, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 14, 3402, 831, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2317, 0, 0, 0, 0, 0, 0, 3971, 4096, 4086, 1856, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 642, 0, 4096, 0, 0, 0, 4096, 0, 3315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16394, 16384, 16600, 16193, 16395, 16534, 16328, 16244] avg=16384.0000 max=16600 max_rank=2 min=16193 min_rank=3 imbalance_ratio=1.013184 max_load_reduction=114472 max_load_reduction_pct=0.873352 moved_to_echo=65501 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 10, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 4096, 0, 180, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3454, 2225, 944, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 3151, 0, 0, 645, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 4096, 4090, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3402, 2176, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2873, 0, 0, 0, 0, 0, 0, 3805, 4096, 4086, 1856, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 642, 0, 4096, 0, 0, 0, 4095, 0, 2449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16382, 15886, 16059, 16394, 17866, 16718, 15379] avg=16384.0000 max=17866 max_rank=5 min=15379 min_rank=7 imbalance_ratio=1.090454 max_load_reduction=113206 max_load_reduction_pct=0.863693 moved_to_echo=66357 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4085, 0, 1, 10, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4092, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 403, 4096, 0, 669, 0, 0, 0, 4092, 0, 0, 0, 0, 1, 0, 0, 3454, 2225, 944, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 3152, 0, 8, 1126, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 4094, 4063, 0, 0, 0, 4088, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4, 0, 21, 3402, 8, 4095, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3588, 0, 0, 0, 0, 0, 0, 2441, 4096, 4086, 1856, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 642, 0, 4096, 0, 0, 0, 4096, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16384, 16578, 16572, 16357, 15732, 16071, 16994] avg=16384.0000 max=16994 max_rank=7 min=15732 min_rank=5 imbalance_ratio=1.037231 max_load_reduction=114078 max_load_reduction_pct=0.870346 moved_to_echo=65154 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 6, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4096, 0, 1253, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3461, 2229, 948, 697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 3148, 0, 21, 552, 0, 4096, 0, 4092, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 4096, 4095, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 3399, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2945, 0, 0, 0, 0, 0, 0, 3981, 4096, 4090, 1862, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 635, 0, 4096, 0, 0, 0, 4096, 0, 3472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16392, 16827, 16018, 16377, 15699, 16975, 16395] avg=16384.0000 max=16975 max_rank=6 min=15699 min_rank=5 imbalance_ratio=1.036072 max_load_reduction=114097 max_load_reduction_pct=0.870491 moved_to_echo=65446 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 4086, 2, 6, 10, 8, 0, 6, 4061, 6, 0, 10, 0, 4088, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 0, 7, 7, 0, 9, 5, 7, 0, 5, 4, 4, 7, 4035, 4090, 4089, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 4089, 0, 1241, 7, 0, 0, 4090, 3, 19, 0, 7, 0, 10, 9, 3450, 2220, 942, 693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 5, 3145, 0, 0, 489, 0, 4085, 0, 4089, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4083, 14, 4084, 4083, 0, 35, 0, 4093, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4078, 0, 0, 0, 55, 3396, 739, 4071, 0, 0, 0, 0, 3, 23, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 9, 4, 15, 12, 4, 0, 0, 2184, 0, 5, 8, 8, 2, 0, 3733, 4087, 4080, 1852, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 8, 4, 2, 0, 641, 7, 4089, 1, 14, 0, 4079, 0, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 16368, 17004, 15909, 16402, 16459, 16009, 16548] avg=16384.0000 max=17004 max_rank=2 min=15909 min_rank=3 imbalance_ratio=1.037842 max_load_reduction=114068 max_load_reduction_pct=0.870270 moved_to_echo=65418 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 10, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 4096, 0, 518, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3454, 2225, 944, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 3152, 0, 0, 592, 0, 4093, 0, 4095, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 4096, 4094, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2, 0, 2, 3402, 891, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2152, 0, 0, 0, 0, 0, 5, 3611, 4096, 4086, 1856, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 642, 0, 4096, 0, 0, 0, 4096, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16405, 16389, 16483, 16023, 16390, 16590, 15808, 16984] avg=16384.0000 max=16984 max_rank=7 min=15808 min_rank=6 imbalance_ratio=1.036621 max_load_reduction=114088 max_load_reduction_pct=0.870422 moved_to_echo=65772 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 10, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4088, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 443, 4096, 0, 135, 0, 0, 0, 4071, 0, 0, 0, 0, 8, 0, 0, 3454, 2224, 944, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 3152, 0, 0, 2254, 0, 4083, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 4096, 4096, 0, 0, 0, 4081, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 21, 3402, 731, 4096, 0, 9, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2340, 0, 0, 0, 0, 0, 19, 3769, 4096, 4086, 1856, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 642, 0, 4093, 0, 0, 0, 4096, 0, 2605, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16391, 16394, 16069, 17667, 16390, 16461, 16168, 15532] avg=16384.0000 max=17667 max_rank=3 min=15532 min_rank=7 imbalance_ratio=1.078308 max_load_reduction=113405 max_load_reduction_pct=0.865211 moved_to_echo=64551 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4096, 0, 0, 0, 0, 0, 0, 4042, 0, 6, 10, 2, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 4096, 0, 50, 0, 0, 0, 4091, 0, 0, 6, 0, 7, 0, 0, 3448, 2224, 944, 694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 4096, 0, 3152, 0, 0, 1387, 0, 4091, 0, 3886, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 4096, 4083, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 6, 0, 9, 3402, 12, 4096, 0, 12, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2898, 0, 0, 0, 0, 0, 9, 4047, 4096, 4086, 1856, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 0, 0, 0, 641, 0, 4093, 0, 0, 0, 4096, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16348, 16379, 15625, 16631, 16367, 15731, 16995, 16996] avg=16384.0000 max=16996 max_rank=7 min=15625 min_rank=2 imbalance_ratio=1.037354 max_load_reduction=114076 max_load_reduction_pct=0.870331 moved_to_echo=66089 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 128, 0, 128, 32768, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 32768, 0, 128, 0, 128, 27264, 5504, 32768, 0, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 2176, 32768, 0, 4992, 128, 0, 0, 32768, 128, 128, 128, 128, 128, 128, 128, 27648, 17920, 7680, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 25216, 0, 128, 7808, 0, 32768, 0, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 0, 5248, 128, 32768, 128, 128, 0, 32768, 0, 27776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 32768, 128, 32768, 32768, 128, 128, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 0, 21376, 0, 128, 128, 128, 128, 128, 29440, 32768, 32768, 14976, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4054, 0, 0, 0, 0, 43, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 26, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 5, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 1707, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 0, 0, 2421, 0, 0, 0, 0, 0, 27, 4096, 0, 4095, 15, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 26, 4096, 4096, 0, 0, 0, 10, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4084, 0, 0, 0, 0, 4, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 2, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 3653, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3404, 0, 0, 1134, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 6, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 4094, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4016, 0, 0, 0, 0, 546, 0, 0, 0, 0, 0, 0, 0, 4084, 2, 0, 0, 0, 4096, 4091, 0, 4096, 0, 0, 4088, 0, 0, 0, 4096, 3, 4095, 0, 0, 4091, 0, 0, 4054, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 0, 0, 2250, 0, 0, 0, 0, 0, 14, 4096, 0, 4095, 149, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 1, 0, 25, 4096, 4081, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 7, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[9, 6, 4045, 0, 0, 5, 0, 0, 7, 8, 4, 6, 7, 5, 0, 4090, 0, 0, 1, 0, 4088, 4089, 0, 4078, 4, 1, 4088, 8, 0, 0, 4087, 0, 4087, 12, 0, 4090, 5, 0, 3782, 0, 8, 0, 85, 0, 4088, 8, 0, 4082, 8, 4088, 4089, 4049, 6, 0, 0, 2, 0, 8, 0, 0, 5, 0, 0, 8, 9, 0, 7, 0, 5, 1390, 0, 0, 3036, 0, 0, 0, 0, 7, 1, 4089, 0, 4006, 0, 4086, 3, 4086, 4088, 4088, 4089, 7, 4089, 0, 0, 0, 4073, 4089, 5, 8, 7, 67, 0, 15, 0, 4087, 8, 1, 6, 0, 4080, 6, 4082, 0, 8, 4084, 14, 0, 4, 8, 4088, 0, 6, 4080, 0, 1, 0, 0, 13, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 2994, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 35, 0, 4096, 4096, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 3259, 0, 0, 0, 26, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3567, 0, 0, 2241, 0, 0, 0, 0, 0, 5, 4096, 0, 4095, 101, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 12, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4092, 0, 0, 0, 0, 284, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 4096, 4092, 0, 4096, 0, 0, 4095, 0, 51, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 2302, 0, 0, 0, 14, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3063, 0, 0, 2456, 0, 0, 0, 0, 0, 7, 4096, 0, 4084, 19, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 2, 0, 16, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4092, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 43, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 1, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4013, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 619, 0, 0, 3053, 0, 0, 0, 0, 0, 16, 4096, 0, 4096, 446, 4096, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 8, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 4081, 0, 0, 0, 0, 209, 0, 0, 0, 0, 0, 0, 0, 4085, 5, 0, 18, 0, 4096, 4088, 0, 4096, 0, 0, 4090, 0, 13, 0, 4096, 2, 4096, 0, 0, 4092, 0, 1, 1891, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4095, 4096, 0, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3105, 0, 0, 2858, 0, 0, 0, 0, 0, 13, 4096, 1, 4083, 122, 4096, 0, 4096, 4096, 4094, 4096, 0, 4094, 0, 0, 16, 4096, 4077, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 2, 0, 4096, 0, 0, 8, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4054, 0, 0, 0, 0, 43, 0, 0, 2, 0, 0, 0, 0, 4096, 4096, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 5, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1707, 0, 0, 0, 1, 0, 999, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3225, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 0, 0, 2421, 0, 0, 0, 0, 0, 27, 4096, 4079, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 15, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 3097, 873, 94, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3223, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16404, 16397, 14995, 16386, 17798, 16330, 16380, 16382] avg=16384.0000 max=17798 max_rank=4 min=14995 min_rank=2 imbalance_ratio=1.086304 max_load_reduction=113274 max_load_reduction_pct=0.864212 moved_to_echo=66890 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4084, 0, 0, 0, 0, 4, 0, 0, 2, 0, 0, 0, 0, 4096, 4096, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3653, 0, 0, 0, 0, 0, 999, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3225, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3404, 0, 0, 1134, 0, 0, 0, 0, 0, 0, 4096, 4079, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 3097, 873, 94, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 3223, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16395, 16368, 16940, 16386, 15936, 16292, 16371, 16384] avg=16384.0000 max=16940 max_rank=2 min=15936 min_rank=4 imbalance_ratio=1.033936 max_load_reduction=114132 max_load_reduction_pct=0.870758 moved_to_echo=64983 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[9, 6, 4045, 0, 0, 5, 0, 0, 7, 8, 4, 6, 7, 5, 0, 4090, 4089, 4088, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 0, 4078, 4, 1, 4088, 8, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 12, 0, 4090, 5, 0, 3782, 0, 8, 0, 85, 0, 997, 8, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4088, 4089, 4049, 6, 0, 0, 2, 0, 8, 0, 0, 5, 0, 0, 8, 3217, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 7, 0, 5, 1390, 0, 0, 3036, 0, 0, 0, 0, 7, 1, 4089, 4071, 3218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 3993, 3, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4073, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 8, 7, 67, 0, 15, 0, 4087, 8, 1, 6, 0, 4080, 6, 4082, 0, 3091, 872, 93, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4084, 14, 0, 4, 8, 4088, 0, 6, 4080, 0, 1, 0, 0, 13, 0, 3216, 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16338, 17156, 16351, 15833, 16171, 16446, 16391] avg=16384.0000 max=17156 max_rank=2 min=15833 min_rank=4 imbalance_ratio=1.047119 max_load_reduction=113916 max_load_reduction_pct=0.869110 moved_to_echo=64841 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2994, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 4096, 0, 4096, 0, 0, 4095, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3259, 0, 0, 0, 26, 0, 994, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3226, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3567, 0, 0, 2241, 0, 0, 0, 0, 0, 5, 4096, 4078, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 101, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 3102, 874, 95, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3222, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15354, 16398, 16567, 16387, 17210, 16397, 16379, 16380] avg=16384.0000 max=17210 max_rank=4 min=15354 min_rank=0 imbalance_ratio=1.050415 max_load_reduction=113862 max_load_reduction_pct=0.868698 moved_to_echo=66366 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4016, 0, 0, 0, 0, 546, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 4088, 0, 0, 0, 4078, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4091, 0, 0, 4054, 0, 0, 0, 0, 0, 999, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3226, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 0, 0, 2250, 0, 0, 0, 0, 0, 14, 4096, 4078, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 149, 4002, 0, 0, 0, 0, 0, 0, 0, 1, 0, 25, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 7, 0, 0, 4096, 0, 4096, 0, 3097, 874, 94, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 3222, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16856, 16358, 17335, 16396, 14915, 16449, 16380, 16383] avg=16384.0000 max=17335 max_rank=2 min=14915 min_rank=4 imbalance_ratio=1.058044 max_load_reduction=113737 max_load_reduction_pct=0.867744 moved_to_echo=64127 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4092, 0, 0, 0, 0, 284, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4092, 0, 4096, 0, 0, 4095, 0, 51, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2302, 0, 0, 0, 14, 0, 999, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3225, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3063, 0, 0, 2456, 0, 0, 0, 0, 0, 7, 4096, 4078, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 19, 4002, 0, 0, 0, 0, 0, 0, 0, 2, 0, 16, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 3097, 874, 94, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 3222, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16678, 16413, 15603, 16389, 16923, 16312, 16371, 16383] avg=16384.0000 max=16923 max_rank=4 min=15603 min_rank=2 imbalance_ratio=1.032898 max_load_reduction=114149 max_load_reduction_pct=0.870888 moved_to_echo=65989 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4092, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 1, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4013, 0, 0, 0, 1, 0, 999, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3225, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 619, 0, 0, 3053, 0, 0, 0, 0, 0, 16, 4096, 4078, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 446, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 3097, 874, 94, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3222, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16414, 16410, 17301, 16386, 15085, 16724, 16371, 16381] avg=16384.0000 max=17301 max_rank=2 min=15085 min_rank=4 imbalance_ratio=1.055969 max_load_reduction=113771 max_load_reduction_pct=0.868004 moved_to_echo=64561 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4081, 0, 0, 0, 0, 209, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 18, 0, 0, 4088, 0, 4096, 0, 0, 4090, 0, 13, 0, 4078, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 1, 1891, 0, 0, 0, 0, 0, 999, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3225, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3105, 0, 0, 2858, 0, 0, 0, 0, 0, 13, 4096, 4078, 3221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4083, 122, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 3097, 874, 94, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 2, 0, 4096, 0, 0, 8, 0, 0, 0, 3220, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16585, 16390, 15175, 16391, 17372, 16397, 16374, 16388] avg=16384.0000 max=17372 max_rank=4 min=15175 min_rank=2 imbalance_ratio=1.060303 max_load_reduction=113700 max_load_reduction_pct=0.867462 moved_to_echo=66531 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32768, 896, 32128, 128, 0, 0, 0, 0, 128, 0, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 32768, 128, 128, 32768, 0, 128, 128, 0, 128, 0, 25856, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 0, 32768, 128, 128, 24704, 0, 128, 0, 128, 0, 8064, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 0, 32768, 0, 32768, 128, 128, 32768, 128, 128, 0, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 32768, 32768, 128, 128, 0, 128, 128, 128, 0, 0, 128, 0, 0, 128, 25856, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 128, 20480, 0, 0, 19456, 0, 0, 0, 0, 128, 128, 32768, 32640, 25856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 32768, 128, 128, 128, 0, 32768, 128, 32768, 0, 24832, 7040, 768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 31488, 0, 0, 128, 0, 1280, 128, 128, 128, 128, 128, 128, 0, 32768, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 3359, 0, 4096, 0, 0, 0, 6, 4096, 0, 4096, 0, 0, 0, 0, 4094, 0, 4096, 0, 4093, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 435, 4096, 4096, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 307, 0, 4096, 0, 4096, 3, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4096, 0, 0, 2710, 0, 4096, 0, 0, 0, 8, 4096, 0, 4096, 8, 0, 0, 0, 4095, 0, 4096, 0, 4053, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 1, 0, 0, 4095, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 30, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 971, 4075, 4096, 0, 0, 0, 3998, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 511, 0, 4096, 0, 4096, 21, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 3230, 0, 4096, 0, 0, 0, 24, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4086, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4072, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 501, 4086, 4096, 0, 0, 0, 4026, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 416, 6, 4096, 0, 4096, 6, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4096, 0, 0, 3212, 0, 4096, 0, 0, 0, 30, 4096, 0, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 4092, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 15, 4096, 4096, 0, 0, 0, 3906, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1032, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 2881, 0, 4096, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 4057, 0, 4096, 0, 4073, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1160, 4094, 4096, 0, 0, 0, 4077, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 97, 0, 4096, 0, 4096, 31, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 3006, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4083, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1061, 4093, 4096, 0, 0, 0, 4070, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 56, 0, 4096, 0, 4096, 14, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4086, 0, 8, 0, 4084, 0, 8, 2384, 0, 4088, 0, 0, 1, 231, 4088, 7, 4089, 0, 4, 0, 7, 3511, 0, 4088, 6, 4080, 5, 8, 4088, 3, 4070, 4087, 10, 4080, 10, 0, 0, 7, 8, 4089, 7, 4082, 4075, 0, 0, 0, 0, 9, 0, 9, 14, 4085, 0, 0, 0, 0, 3, 3, 0, 0, 92, 4083, 0, 0, 8, 6, 8, 31, 0, 0, 8, 5, 4083, 4087, 6, 8, 19, 4081, 0, 0, 0, 0, 7, 4082, 5, 1, 0, 8, 0, 0, 6, 4087, 6, 11, 0, 8, 4087, 601, 4090, 4083, 0, 0, 8, 3240, 8, 0, 4058, 6, 0, 5, 0, 2, 0, 0, 4089, 4090, 0, 7, 6, 2230, 5, 4087, 0, 4088, 0, 8, 6, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 3, 0, 0, 4096, 0, 0, 1752, 0, 4096, 2, 0, 0, 2, 4095, 0, 4096, 5, 0, 0, 0, 4090, 0, 4096, 0, 4093, 0, 0, 4094, 0, 4096, 4096, 0, 4096, 0, 0, 3, 0, 0, 4093, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 57, 4085, 4096, 0, 0, 0, 3629, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 2, 2742, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4096, 0, 0, 3212, 0, 1243, 0, 0, 0, 30, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4092, 0, 0, 4096, 0, 4096, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4080, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4003, 66, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2787, 934, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 3723, 0, 0, 0, 3906, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3162, 4096, 0, 0, 0, 1032, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16772, 16472, 16380, 16378, 16370, 16382, 15836, 16482] avg=16384.0000 max=16772 max_rank=0 min=15836 min_rank=6 imbalance_ratio=1.023682 max_load_reduction=114300 max_load_reduction_pct=0.872040 moved_to_echo=65070 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 3359, 0, 1243, 0, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4093, 0, 0, 4096, 0, 4096, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4090, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4003, 66, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2787, 934, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 435, 4096, 3723, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3162, 4096, 0, 0, 0, 307, 0, 4096, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16896, 16472, 16390, 16360, 16370, 16382, 16442, 15760] avg=16384.0000 max=16896 max_rank=0 min=15760 min_rank=7 imbalance_ratio=1.031250 max_load_reduction=114176 max_load_reduction_pct=0.871094 moved_to_echo=64954 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 3230, 0, 1243, 0, 0, 0, 24, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 4096, 0, 4096, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4072, 1, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4003, 66, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2787, 934, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 4086, 3723, 0, 0, 0, 4026, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3162, 4095, 0, 0, 0, 416, 6, 4096, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16785, 16467, 16373, 16381, 16376, 16382, 16431, 15877] avg=16384.0000 max=16785 max_rank=0 min=15877 min_rank=7 imbalance_ratio=1.024475 max_load_reduction=114287 max_load_reduction_pct=0.871941 moved_to_echo=65066 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4096, 0, 0, 2710, 0, 1243, 0, 0, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4095, 0, 0, 0, 4053, 0, 0, 4096, 0, 4096, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4095, 0, 4096, 4083, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 30, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4003, 66, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2787, 934, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 971, 4075, 3723, 0, 0, 0, 3998, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3162, 4094, 0, 0, 0, 511, 0, 4096, 0, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16248, 16441, 16383, 16391, 16384, 16382, 16863, 15980] avg=16384.0000 max=16863 max_rank=6 min=15980 min_rank=7 imbalance_ratio=1.029236 max_load_reduction=114209 max_load_reduction_pct=0.871346 moved_to_echo=65609 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 2881, 0, 1237, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 4073, 0, 0, 4096, 0, 4096, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4077, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4008, 71, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2788, 935, 367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 4094, 3713, 0, 0, 0, 4077, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3161, 4096, 0, 0, 0, 97, 0, 4096, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16410, 16381, 16386, 16389, 16382, 17140, 15577] avg=16384.0000 max=17140 max_rank=6 min=15577 min_rank=7 imbalance_ratio=1.046143 max_load_reduction=113932 max_load_reduction_pct=0.869232 moved_to_echo=65488 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4086, 0, 8, 0, 4084, 0, 8, 2384, 0, 1241, 0, 0, 1, 231, 4088, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7, 3511, 0, 0, 6, 4080, 5, 8, 4088, 3, 4070, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4080, 10, 0, 0, 7, 8, 4089, 7, 4082, 4075, 0, 0, 0, 0, 9, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 14, 4085, 0, 0, 0, 0, 3, 3, 0, 0, 92, 4083, 0, 0, 4087, 3994, 66, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 6, 8, 31, 0, 0, 8, 5, 4083, 4087, 6, 8, 19, 4081, 0, 0, 2781, 932, 360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4082, 5, 1, 0, 8, 0, 0, 6, 4087, 6, 11, 0, 8, 4089, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 4090, 3711, 0, 0, 8, 3240, 8, 0, 4058, 6, 0, 5, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3157, 4090, 0, 7, 6, 2230, 5, 4087, 0, 4088, 0, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16138, 15875, 16389, 16437, 16423, 16397, 15729, 17684] avg=16384.0000 max=17684 max_rank=7 min=15729 min_rank=6 imbalance_ratio=1.079346 max_load_reduction=113388 max_load_reduction_pct=0.865082 moved_to_echo=66233 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 3006, 0, 1243, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4083, 0, 0, 4096, 0, 4096, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4086, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4003, 66, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2787, 934, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 4093, 3723, 0, 0, 0, 4070, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3162, 4096, 0, 0, 0, 56, 0, 4096, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16537, 16464, 16386, 16361, 16379, 16382, 17043, 15520] avg=16384.0000 max=17043 max_rank=6 min=15520 min_rank=7 imbalance_ratio=1.040222 max_load_reduction=114029 max_load_reduction_pct=0.869972 moved_to_echo=65324 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 3, 0, 0, 4096, 0, 0, 1752, 0, 1243, 2, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4090, 0, 0, 0, 4093, 0, 0, 4094, 0, 4096, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 4093, 0, 4096, 4090, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4003, 66, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 2787, 934, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4085, 3723, 0, 0, 0, 3629, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3162, 4094, 0, 0, 2, 2742, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15289, 16471, 16390, 16378, 16381, 16383, 15588, 18192] avg=16384.0000 max=18192 max_rank=7 min=15289 min_rank=0 imbalance_ratio=1.110352 max_load_reduction=112880 max_load_reduction_pct=0.861206 moved_to_echo=66544 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 0, 32768, 0, 128, 22656, 0, 9984, 128, 0, 128, 384, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 32768, 128, 0, 0, 128, 128, 128, 0, 0, 128, 32768, 0, 0, 32768, 32128, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 0, 128, 0, 0, 128, 32768, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 128, 32768, 128, 32768, 32768, 128, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 4864, 32768, 29824, 0, 0, 128, 31104, 128, 0, 32768, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 0, 128, 32256, 0, 0, 128, 32768, 128, 128, 32768, 128, 32768, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 25344, 32768, 0, 128, 128, 7424, 128, 32768, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 128, 32768, 32768, 128, 128, 128, 32768, 0, 0, 22400, 7552, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 4026, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 70, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 4091, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 4060, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 36, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 4030, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4091, 25, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 2, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 4096, 0, 4075, 0, 0, 0, 0, 0, 4096, 62, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 4079, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4083, 0, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[2, 4082, 9, 4086, 8, 0, 0, 4083, 9, 0, 0, 0, 0, 6, 7, 0, 0, 0, 4087, 5, 7, 0, 0, 4088, 4085, 9, 8, 0, 4089, 3, 4080, 0, 0, 5, 6, 3, 11, 4087, 0, 4089, 4088, 0, 0, 7, 0, 0, 6, 9, 8, 11, 8, 0, 4047, 0, 5, 4087, 4089, 9, 0, 0, 0, 4, 7, 0, 0, 4086, 1, 7, 4086, 4084, 4088, 0, 8, 29, 4085, 11, 4087, 4085, 4087, 7, 0, 4044, 6, 2, 11, 4089, 0, 4085, 4071, 0, 4088, 0, 0, 0, 0, 9, 4083, 0, 8, 10, 4, 4088, 5, 0, 8, 5, 0, 0, 4081, 4090, 0, 4084, 7, 0, 0, 3, 0, 0, 0, 14, 0, 58, 0, 0, 9, 4, 8, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 4072, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4094, 5, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4094, 0, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 4096, 24, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 4038, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 4075, 22, 4096, 0, 4, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4091, 0, 4096, 4096, 4, 4096, 0, 0, 1, 0, 0, 4096, 57, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 4096, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4026, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 4096, 70, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4055, 25, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4071, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 14, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4071, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16348, 16444, 16380, 16386, 16382, 16398, 16359, 16375] avg=16384.0000 max=16444 max_rank=1 min=16348 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=114628 max_load_reduction_pct=0.874542 moved_to_echo=65514 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4086, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4055, 25, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4071, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 14, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4071, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16413, 16373, 16380, 16386, 16382, 16398, 16365, 16375] avg=16384.0000 max=16413 max_rank=0 min=16365 min_rank=6 imbalance_ratio=1.001770 max_load_reduction=114659 max_load_reduction_pct=0.874779 moved_to_echo=65520 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4060, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4055, 25, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4071, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 14, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4071, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16374, 16380, 16386, 16382, 16398, 16395, 16374] avg=16384.0000 max=16398 max_rank=5 min=16374 min_rank=1 imbalance_ratio=1.000854 max_load_reduction=114674 max_load_reduction_pct=0.874893 moved_to_echo=65549 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4079, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 0, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4059, 29, 14, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4071, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 5, 4096, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4067, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16402, 16374, 16397, 16391, 16377, 16376, 16381, 16374] avg=16384.0000 max=16402 max_rank=0 min=16374 min_rank=1 imbalance_ratio=1.001099 max_load_reduction=114670 max_load_reduction_pct=0.874863 moved_to_echo=65508 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4072, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 25, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 4094, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4055, 25, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4071, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 14, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4071, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16395, 16377, 16380, 16386, 16382, 16395, 16383, 16374] avg=16384.0000 max=16395 max_rank=0 min=16374 min_rank=7 imbalance_ratio=1.000671 max_load_reduction=114677 max_load_reduction_pct=0.874916 moved_to_echo=65534 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4030, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 25, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4086, 0, 0, 0, 4091, 25, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4055, 25, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4071, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 14, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4071, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16358, 16393, 16380, 16386, 16384, 16375, 16422, 16374] avg=16384.0000 max=16422 max_rank=6 min=16358 min_rank=0 imbalance_ratio=1.002319 max_load_reduction=114650 max_load_reduction_pct=0.874710 moved_to_echo=65555 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2, 4082, 9, 4086, 8, 0, 0, 4083, 9, 0, 0, 0, 0, 6, 7, 0, 4088, 24, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7, 0, 0, 4088, 4075, 9, 8, 0, 4089, 3, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 3, 11, 4087, 0, 4089, 4088, 0, 0, 7, 0, 0, 6, 9, 4044, 25, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 11, 8, 0, 4047, 0, 5, 4087, 4089, 9, 0, 0, 0, 4, 7, 0, 4062, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1, 7, 0, 4084, 0, 0, 8, 29, 4085, 11, 0, 4085, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 6, 2, 11, 4089, 0, 14, 4071, 0, 4088, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 8, 10, 4, 0, 5, 0, 8, 5, 0, 0, 4081, 4065, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3, 0, 0, 0, 14, 0, 58, 0, 0, 9, 4, 8, 8, 4088, 4087, 4087, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16413, 16364, 16392, 16364, 16401, 16334, 16353, 16451] avg=16384.0000 max=16451 max_rank=7 min=16334 min_rank=5 imbalance_ratio=1.004089 max_load_reduction=114621 max_load_reduction_pct=0.874489 moved_to_echo=65539 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 4038, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 25, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4086, 0, 0, 0, 4075, 22, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 4055, 25, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4071, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 14, 4096, 4, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 57, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4068, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16360, 16373, 16383, 16387, 16382, 16398, 16414, 16375] avg=16384.0000 max=16414 max_rank=6 min=16360 min_rank=0 imbalance_ratio=1.001831 max_load_reduction=114658 max_load_reduction_pct=0.874771 moved_to_echo=65569 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32512, 128, 32768, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 128, 128, 128, 32768, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 0, 32768, 32768, 0, 0, 128, 0, 0, 128, 128, 32512, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 256, 128, 128, 128, 0, 128, 128, 128, 128, 128, 0, 32768, 32640, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 128, 128, 0, 32768, 0, 0, 128, 128, 32768, 128, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 128, 0, 128, 32768, 32768, 128, 128, 0, 32768, 256, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 128, 128, 128, 32768, 0, 128, 32768, 128, 32768, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 128, 32768, 32768, 128, 0, 0, 0, 128, 128, 0, 32640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 0, 0, 0, 128, 0, 128, 0, 128, 128, 128, 128, 128, 32768, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3505, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 588, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4095, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3645, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 415, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 35, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3401, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 608, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 85, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 1, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3328, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 523, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 245, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4090, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3758, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 35, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 303, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 13, 4, 8, 4096, 4091, 0, 4082, 0, 4096, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[10, 0, 7, 5, 0, 0, 5, 0, 4086, 4087, 0, 4090, 0, 0, 9, 0, 0, 4083, 4064, 1, 4088, 10, 4085, 4085, 8, 0, 5, 0, 4084, 1, 4087, 0, 9, 4090, 4086, 0, 0, 8, 0, 0, 26, 0, 10, 7, 3, 0, 0, 3969, 4088, 7, 0, 7, 0, 4086, 7, 0, 5, 132, 10, 7, 0, 28, 4082, 0, 4086, 0, 0, 0, 7, 4032, 3, 4088, 0, 0, 0, 0, 0, 8, 10, 8, 7, 4082, 4084, 8, 16, 6, 4083, 11, 4073, 4, 3, 4087, 12, 7, 0, 12, 0, 0, 11, 0, 4086, 68, 0, 6, 6, 4082, 0, 7, 0, 0, 8, 4032, 0, 9, 0, 4089, 0, 0, 4, 4086, 4088, 0, 4088, 7, 4086, 5, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4008, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 20, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 68, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 6, 0, 3, 4096, 4091, 0, 4092, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4089, 0, 0, 0, 0, 3, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3950, 4096, 0, 4, 0, 0, 4096, 0, 0, 0, 58, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4092, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 75, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 21, 2, 10, 4096, 4093, 0, 4087, 0, 4096, 0, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3694, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3505, 4096, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 588, 0, 0, 0, 0, 4096, 0, 3789, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3703, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3693, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16390, 16186, 16669, 16393, 16382, 16286, 16381] avg=16384.0000 max=16669 max_rank=3 min=16186 min_rank=2 imbalance_ratio=1.017395 max_load_reduction=114403 max_load_reduction_pct=0.872826 moved_to_echo=65442 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3694, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3328, 4096, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 523, 0, 0, 0, 0, 4096, 0, 3789, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3703, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3693, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16390, 16009, 16604, 16393, 16382, 16527, 16382] avg=16384.0000 max=16604 max_rank=3 min=16009 min_rank=2 imbalance_ratio=1.013428 max_load_reduction=114468 max_load_reduction_pct=0.873322 moved_to_echo=65684 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3695, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3401, 4096, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 608, 0, 0, 0, 0, 4096, 0, 3789, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3703, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 3693, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16390, 16086, 16689, 16391, 16381, 16366, 16383] avg=16384.0000 max=16689 max_rank=3 min=16086 min_rank=2 imbalance_ratio=1.018616 max_load_reduction=114383 max_load_reduction_pct=0.872673 moved_to_echo=65521 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3695, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3645, 4096, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 415, 0, 0, 0, 0, 4096, 0, 3791, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3702, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3692, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16384, 16327, 16507, 16391, 16379, 16320, 16377] avg=16384.0000 max=16507 max_rank=3 min=16320 min_rank=6 imbalance_ratio=1.007507 max_load_reduction=114565 max_load_reduction_pct=0.874062 moved_to_echo=65467 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 3695, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4008, 4096, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 20, 0, 0, 0, 0, 4096, 0, 3789, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3703, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3693, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 3, 0, 4091, 0, 4092, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16390, 16692, 16101, 16389, 16382, 16350, 16382] avg=16384.0000 max=16692 max_rank=2 min=16101 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=114380 max_load_reduction_pct=0.872650 moved_to_echo=65503 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4090, 0, 0, 0, 0, 3695, 403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3758, 4096, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 35, 0, 0, 0, 0, 4096, 0, 3789, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3702, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3692, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 13, 4, 8, 0, 4091, 0, 4082, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16388, 16446, 16117, 16388, 16382, 16586, 16385] avg=16384.0000 max=16586 max_rank=6 min=16117 min_rank=3 imbalance_ratio=1.012329 max_load_reduction=114486 max_load_reduction_pct=0.873459 moved_to_echo=65741 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 7, 5, 0, 0, 5, 0, 4086, 4087, 0, 4090, 0, 0, 9, 0, 3685, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4064, 1, 0, 10, 4085, 6, 8, 0, 5, 0, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4090, 4086, 0, 0, 8, 0, 0, 26, 0, 10, 7, 3, 0, 0, 3969, 4088, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 7, 0, 7, 0, 4086, 7, 0, 5, 132, 10, 7, 0, 28, 4082, 0, 3779, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 7, 4032, 3, 4088, 0, 0, 0, 0, 0, 8, 10, 8, 3695, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4082, 4082, 8, 16, 6, 4083, 11, 4073, 4, 3, 0, 12, 7, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4086, 68, 0, 6, 6, 4082, 0, 7, 0, 0, 8, 4032, 3685, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4089, 0, 0, 4, 0, 4088, 0, 4088, 7, 4084, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16347, 16688, 16242, 16338, 16406, 16291, 16374] avg=16384.0000 max=16688 max_rank=2 min=16242 min_rank=3 imbalance_ratio=1.018555 max_load_reduction=114384 max_load_reduction_pct=0.872681 moved_to_echo=65409 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4089, 0, 0, 0, 0, 3694, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3950, 4096, 393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 4096, 0, 0, 0, 58, 0, 0, 0, 0, 4096, 0, 3789, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 1, 0, 0, 0, 0, 3703, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4085, 3693, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 21, 2, 10, 0, 4093, 0, 4087, 0, 4094, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16393, 16638, 16143, 16389, 16378, 16346, 16408] avg=16384.0000 max=16638 max_rank=2 min=16143 min_rank=3 imbalance_ratio=1.015503 max_load_reduction=114434 max_load_reduction_pct=0.873062 moved_to_echo=65521 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32768, 32768, 128, 0, 128, 32768, 128, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 128, 128, 0, 0, 128, 0, 32768, 32768, 0, 32768, 0, 0, 128, 0, 29568, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 128, 0, 32768, 128, 0, 128, 2432, 128, 128, 0, 128, 32768, 0, 30336, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 0, 0, 128, 32768, 128, 32768, 0, 0, 0, 128, 0, 128, 128, 128, 29696, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 896, 32768, 128, 0, 128, 128, 32768, 0, 128, 0, 0, 128, 32768, 29568, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32768, 32768, 0, 0, 128, 0, 0, 128, 0, 128, 128, 128, 0, 128, 29568, 32768, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 32768, 128, 32768, 128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 32768, 128, 128, 128, 0, 32768, 0, 32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 4096, 1512, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 379, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 62, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 2217, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4042, 0, 4096, 4096, 19, 0, 4096, 4096, 161, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3951, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 311, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 0, 0, 0, 107, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 3702, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 7, 0, 4096, 4096, 683, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 155, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 19, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 3257, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 6, 4094, 0, 4096, 4096, 17, 0, 4096, 4090, 1072, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 7, 474, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 4067, 0, 0, 4096, 0, 0, 0, 0, 37, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 2554, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 4066, 0, 4096, 4096, 18, 0, 4096, 4096, 2388, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 76, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 1756, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4086, 3, 0, 4088, 4067, 0, 0, 6, 0, 2, 0, 4081, 0, 0, 0, 9, 10, 0, 4088, 0, 4086, 4084, 0, 0, 4071, 4088, 724, 0, 0, 4085, 4084, 8, 7, 10, 4084, 2, 12, 8, 0, 0, 0, 4052, 0, 8, 4082, 4088, 0, 0, 0, 11, 9, 9, 9, 13, 0, 4, 0, 4084, 0, 9, 10, 0, 4083, 6, 4086, 4085, 9, 0, 16, 8, 0, 4, 0, 83, 4088, 11, 0, 0, 4087, 4085, 1, 0, 4085, 5, 0, 8, 0, 10, 11, 9, 4087, 11, 9, 4088, 0, 0, 7, 7, 191, 0, 14, 4079, 0, 4083, 0, 4067, 4, 5, 3123, 6, 0, 4078, 0, 8, 9, 0, 0, 4086, 13, 4087, 8, 5, 24, 12, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4053, 0, 4096, 4096, 26, 0, 4096, 4096, 1317, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 8, 61, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 4096, 0, 0, 0, 0, 64, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 2706, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 12, 0, 4096, 4096, 301, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4, 58, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 4060, 0, 0, 4096, 0, 6, 0, 0, 95, 0, 0, 4096, 0, 4092, 0, 4095, 0, 0, 3682, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 3085, 19, 0, 4096, 4096, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3951, 0, 0, 4096, 4096, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 311, 3894, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 3702, 0, 0, 1011, 131, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16395, 15499, 16284, 16380, 16493, 16388, 17247, 16386] avg=16384.0000 max=17247 max_rank=6 min=15499 min_rank=1 imbalance_ratio=1.052673 max_load_reduction=113825 max_load_reduction_pct=0.868416 moved_to_echo=66514 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3085, 0, 0, 4096, 4096, 1512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 4096, 4096, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 379, 3894, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 2217, 0, 0, 1011, 131, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 16885, 16356, 16380, 16561, 16397, 15716, 16380] avg=16384.0000 max=16885 max_rank=1 min=15716 min_rank=6 imbalance_ratio=1.030579 max_load_reduction=114187 max_load_reduction_pct=0.871178 moved_to_echo=65054 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3085, 7, 0, 4096, 4096, 683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 4096, 4096, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 155, 3894, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 3257, 0, 0, 1011, 131, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 16063, 16411, 16381, 16337, 16390, 16714, 16379] avg=16384.0000 max=16714 max_rank=6 min=16063 min_rank=1 imbalance_ratio=1.020142 max_load_reduction=114358 max_load_reduction_pct=0.872482 moved_to_echo=65820 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 3078, 18, 0, 4096, 4096, 2388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 4096, 4096, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 76, 3878, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 4096, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 1756, 0, 0, 1018, 137, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 17742, 16375, 16381, 16242, 16354, 15210, 16379] avg=16384.0000 max=17742 max_rank=1 min=15210 min_rank=6 imbalance_ratio=1.082886 max_load_reduction=113330 max_load_reduction_pct=0.864639 moved_to_echo=64185 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 3085, 26, 0, 4096, 4096, 1317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 4096, 4096, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 8, 61, 3894, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 2706, 0, 0, 1011, 131, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16673, 16410, 16381, 16251, 16381, 16208, 16379] avg=16384.0000 max=16673 max_rank=1 min=16208 min_rank=6 imbalance_ratio=1.017639 max_load_reduction=114399 max_load_reduction_pct=0.872795 moved_to_echo=65219 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4086, 3, 0, 4088, 4067, 0, 0, 6, 0, 2, 0, 4081, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 10, 0, 4088, 0, 0, 3076, 0, 0, 4071, 4088, 724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 7, 10, 4084, 2, 12, 8, 0, 0, 0, 4052, 0, 8, 4082, 4088, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 9, 9, 9, 13, 0, 4, 0, 4084, 0, 9, 10, 0, 4083, 4086, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4086, 4085, 9, 0, 16, 8, 0, 4, 0, 83, 3886, 11, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 4085, 5, 0, 8, 0, 10, 11, 9, 4087, 11, 9, 4088, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 191, 0, 14, 4079, 0, 4083, 0, 4067, 4, 5, 3123, 6, 0, 1008, 131, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 8, 9, 0, 0, 4086, 13, 4087, 8, 5, 24, 12, 0, 0, 0, 4077, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16346, 16066, 16406, 16408, 16281, 16422, 16732, 16411] avg=16384.0000 max=16732 max_rank=6 min=16066 min_rank=1 imbalance_ratio=1.021240 max_load_reduction=114340 max_load_reduction_pct=0.872345 moved_to_echo=65846 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4094, 0, 0, 3085, 17, 0, 4096, 4090, 1072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 4096, 4092, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 7, 474, 3894, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 4067, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 2554, 0, 0, 1011, 131, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16460, 16405, 16381, 16663, 16374, 16029, 16379] avg=16384.0000 max=16663 max_rank=4 min=16029 min_rank=6 imbalance_ratio=1.017029 max_load_reduction=114409 max_load_reduction_pct=0.872871 moved_to_echo=65445 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3085, 12, 0, 4096, 4096, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 4096, 4093, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4, 58, 3894, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 4060, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 95, 0, 0, 4096, 0, 4092, 0, 4095, 0, 0, 3682, 0, 0, 1011, 131, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 15684, 16425, 16380, 16244, 16366, 17216, 16379] avg=16384.0000 max=17216 max_rank=6 min=15684 min_rank=1 imbalance_ratio=1.050781 max_load_reduction=113856 max_load_reduction_pct=0.868652 moved_to_echo=66205 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 128, 0, 0, 0, 32512, 128, 128, 32768, 32768, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 32768, 128, 0, 128, 128, 0, 128, 128, 1664, 31232, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 32768, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 0, 32768, 128, 128, 128, 0, 128, 128, 128, 32640, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 0, 128, 0, 32768, 0, 128, 128, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 640, 0, 128, 32768, 0, 32768, 0, 32768, 128, 128, 23040, 128, 0, 8192, 1152, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 0, 32768, 32768, 0, 0, 128, 0, 128, 0, 32768, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 128, 128, 32640, 0, 0, 24704, 128, 0, 32768, 32768, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 10, 4096, 0, 0, 0, 4096, 0, 0, 4094, 4096, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4032, 0, 0, 4096, 0, 53, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4069, 0, 0, 4096, 0, 27, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 1, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 16, 4096, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 0, 0, 0, 8, 0, 0, 4089, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3933, 0, 0, 4096, 0, 67, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4076, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 5, 4096, 0, 0, 0, 4096, 0, 0, 4092, 4096, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4054, 0, 0, 4096, 0, 30, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4082, 6, 4085, 0, 4085, 3941, 0, 8, 0, 9, 0, 0, 0, 4072, 0, 0, 4085, 4, 0, 0, 11, 0, 5, 11, 7, 0, 4086, 12, 0, 0, 0, 0, 0, 10, 7, 9, 3, 0, 9, 9, 9, 8, 4086, 1, 9, 7, 4079, 0, 11, 0, 9, 4083, 2, 3640, 489, 4080, 0, 0, 40, 4086, 0, 8, 4003, 4081, 10, 0, 11, 11, 144, 0, 10, 4087, 0, 4087, 5, 0, 4086, 4086, 7, 24, 16, 9, 7, 4087, 16, 8, 4085, 18, 4069, 0, 10, 4079, 15, 0, 6, 4082, 0, 0, 0, 0, 4076, 0, 4078, 2, 0, 16, 5, 4090, 4084, 0, 0, 0, 9, 0, 11, 4086, 0, 4084, 0, 0, 2, 0, 4077, 4086, 0, 4, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 2, 4096, 0, 0, 0, 4096, 0, 0, 4094, 4096, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4087, 0, 0, 4096, 0, 7, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 60, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 0, 4096, 0, 0, 0, 4096, 0, 0, 4090, 4096, 0, 0, 0, 0, 4, 0, 0, 4092, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4044, 0, 0, 4096, 0, 12, 0, 4096, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 23, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4085, 4073, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 10, 4096, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4032, 0, 0, 4096, 0, 53, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4094, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16362, 16379, 16393, 16367, 16384, 16393, 16387] avg=16384.0000 max=16407 max_rank=0 min=16362 min_rank=1 imbalance_ratio=1.001404 max_load_reduction=114665 max_load_reduction_pct=0.874825 moved_to_echo=65531 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 23, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4085, 4073, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4069, 0, 0, 4096, 0, 27, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4087, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16361, 16380, 16384, 16372, 16395, 16386, 16387] avg=16384.0000 max=16407 max_rank=0 min=16361 min_rank=1 imbalance_ratio=1.001404 max_load_reduction=114665 max_load_reduction_pct=0.874825 moved_to_echo=65540 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 23, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4085, 4073, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 4096, 0, 0, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16406, 16361, 16380, 16384, 16363, 16396, 16395, 16387] avg=16384.0000 max=16406 max_rank=0 min=16361 min_rank=1 imbalance_ratio=1.001343 max_load_reduction=114666 max_load_reduction_pct=0.874832 moved_to_echo=65541 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 19, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 4077, 13, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 5, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 4096, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4054, 0, 0, 4096, 0, 30, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16403, 16378, 16379, 16385, 16357, 16390, 16389, 16391] avg=16384.0000 max=16403 max_rank=0 min=16357 min_rank=4 imbalance_ratio=1.001160 max_load_reduction=114669 max_load_reduction_pct=0.874855 moved_to_echo=65527 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 23, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4085, 4073, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4087, 0, 0, 4096, 0, 7, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4092, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16361, 16380, 16384, 16369, 16393, 16391, 16387] avg=16384.0000 max=16407 max_rank=0 min=16361 min_rank=1 imbalance_ratio=1.001404 max_load_reduction=114665 max_load_reduction_pct=0.874825 moved_to_echo=65540 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 23, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 100, 0, 0, 4085, 4073, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 16, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 4089, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3933, 0, 0, 4096, 0, 67, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4076, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16461, 16380, 16399, 16364, 16299, 16375, 16387] avg=16384.0000 max=16461 max_rank=1 min=16299 min_rank=5 imbalance_ratio=1.004700 max_load_reduction=114611 max_load_reduction_pct=0.874413 moved_to_echo=65425 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4082, 6, 4085, 0, 23, 3941, 0, 8, 0, 9, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4, 0, 0, 11, 0, 5, 11, 7, 0, 4086, 12, 0, 0, 0, 4073, 4062, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 7, 9, 3, 0, 9, 9, 9, 8, 4086, 1, 9, 7, 4079, 4087, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 9, 4083, 2, 3640, 489, 4080, 0, 0, 40, 2, 0, 8, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 10, 0, 11, 11, 144, 0, 10, 4087, 0, 0, 5, 0, 4086, 4065, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 16, 9, 7, 4087, 16, 8, 4085, 18, 4069, 0, 10, 4079, 15, 0, 6, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4076, 0, 4078, 2, 0, 16, 5, 4090, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 11, 4086, 0, 4084, 0, 0, 2, 0, 4077, 4086, 0, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16226, 16366, 16414, 16367, 16517, 16460, 16360, 16362] avg=16384.0000 max=16517 max_rank=4 min=16226 min_rank=0 imbalance_ratio=1.008118 max_load_reduction=114555 max_load_reduction_pct=0.873985 moved_to_echo=65699 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 23, 4096, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 60, 1, 0, 4085, 4073, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 4092, 0, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4044, 0, 0, 4096, 0, 12, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4083, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16409, 16422, 16380, 16376, 16363, 16355, 16383, 16384] avg=16384.0000 max=16422 max_rank=1 min=16355 min_rank=5 imbalance_ratio=1.002319 max_load_reduction=114650 max_load_reduction_pct=0.874710 moved_to_echo=65485 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 32768, 0, 256, 32640, 0, 128, 128, 128, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 128, 128, 256, 0, 128, 32768, 0, 0, 128, 0, 32768, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 0, 0, 32768, 0, 32768, 128, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 32384, 640, 32768, 0, 0, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 32768, 0, 32768, 0, 0, 128, 0, 32768, 32768, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 128, 32768, 128, 32384, 0, 128, 32768, 128, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32768, 128, 0, 0, 128, 0, 128, 128, 128, 0, 32768, 128, 256, 128, 0, 32768, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 0, 128, 128, 128, 128, 0, 128, 128, 128, 128, 32768, 128, 128, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[253, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2763, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 1137, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 26, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[44, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 3558, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 127, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 405, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[27, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3465, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4095, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 548, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 69, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[72, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3456, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 561, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 31, 4096, 0, 0, 0, 0, 1, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[384, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 2023, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 1371, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 390, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[576, 11, 0, 10, 4085, 16, 0, 4083, 0, 9, 0, 2420, 30, 10, 4086, 10, 0, 10, 2, 0, 6, 4079, 16, 0, 0, 4058, 10, 8, 11, 0, 4080, 7, 9, 13, 0, 4085, 4082, 0, 13, 12, 3995, 4074, 9, 4079, 4085, 11, 4073, 0, 0, 5, 10, 9, 4, 91, 5, 5, 4081, 4085, 7, 6, 4085, 12, 10, 4034, 0, 0, 5, 4078, 0, 11, 0, 1, 6, 0, 10, 4075, 10, 4087, 0, 1, 16, 15, 32, 4085, 4081, 0, 0, 4084, 0, 0, 0, 0, 0, 4080, 4082, 4084, 0, 0, 4076, 0, 0, 0, 0, 0, 5, 4084, 10, 0, 6, 9, 9, 38, 0, 3, 0, 0, 4069, 0, 1116, 4061, 11, 10, 0, 12, 0, 4, 0, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[414, 0, 0, 0, 4096, 0, 7, 4096, 0, 0, 0, 3096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 653, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[200, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 3824, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 82, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4085, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[253, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2763, 0, 0, 4096, 0, 565, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4057, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1137, 0, 0, 4096, 3531, 0, 0, 4096, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 4096, 0, 0, 4096, 39, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 26, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16068, 16385, 16397, 16936, 16362, 16384, 16382, 16158] avg=16384.0000 max=16936 max_rank=3 min=16068 min_rank=0 imbalance_ratio=1.033691 max_load_reduction=114136 max_load_reduction_pct=0.870789 moved_to_echo=65286 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[27, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3465, 0, 0, 4096, 0, 565, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4058, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 0, 0, 4096, 3531, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 0, 4096, 38, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 69, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16544, 16386, 16396, 16367, 16412, 16384, 16382, 16201] avg=16384.0000 max=16544 max_rank=0 min=16201 min_rank=7 imbalance_ratio=1.009766 max_load_reduction=114528 max_load_reduction_pct=0.873779 moved_to_echo=65379 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[72, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3456, 0, 0, 4096, 0, 565, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4058, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 0, 0, 4096, 3531, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 4096, 0, 0, 4096, 38, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 31, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16580, 16386, 16397, 16379, 16401, 16384, 16381, 16164] avg=16384.0000 max=16580 max_rank=0 min=16164 min_rank=7 imbalance_ratio=1.011963 max_load_reduction=114492 max_load_reduction_pct=0.873505 moved_to_echo=65330 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[44, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 3558, 0, 0, 4096, 0, 567, 201, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4058, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 4096, 3529, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 4096, 0, 0, 4096, 38, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 405, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16665, 16387, 16390, 15941, 16389, 16378, 16381, 16541] avg=16384.0000 max=16665 max_rank=0 min=15941 min_rank=3 imbalance_ratio=1.017151 max_load_reduction=114407 max_load_reduction_pct=0.872856 moved_to_echo=65689 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[384, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 2023, 0, 0, 4096, 0, 565, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4058, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1371, 0, 0, 4096, 3531, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 4096, 0, 0, 4096, 38, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 390, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15460, 16386, 16397, 17190, 16353, 16384, 16381, 16521] avg=16384.0000 max=17190 max_rank=3 min=15460 min_rank=0 imbalance_ratio=1.049194 max_load_reduction=113882 max_load_reduction_pct=0.868851 moved_to_echo=65639 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[576, 11, 0, 10, 4085, 16, 0, 4083, 0, 9, 0, 2420, 30, 10, 4086, 10, 563, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2, 0, 6, 4079, 16, 0, 0, 4058, 10, 8, 11, 0, 4080, 7, 4043, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 13, 0, 13, 0, 0, 13, 12, 3995, 4074, 9, 4079, 0, 11, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 10, 9, 4, 91, 5, 5, 4081, 3522, 7, 6, 4085, 12, 10, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4078, 0, 11, 0, 1, 6, 0, 10, 4075, 10, 4087, 0, 1, 4084, 39, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 15, 32, 0, 4081, 0, 0, 4084, 0, 0, 0, 0, 0, 4080, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 5, 4084, 10, 0, 6, 9, 9, 38, 4085, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4069, 0, 1116, 4061, 11, 10, 0, 12, 0, 4, 0, 4084, 3834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16107, 16370, 16301, 15886, 16409, 16390, 16405, 17204] avg=16384.0000 max=17204 max_rank=7 min=15886 min_rank=3 imbalance_ratio=1.050049 max_load_reduction=113868 max_load_reduction_pct=0.868744 moved_to_echo=66408 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[414, 0, 0, 0, 4096, 0, 7, 4096, 0, 0, 0, 3096, 0, 0, 4096, 0, 565, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4058, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 653, 0, 0, 4096, 3531, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 4096, 0, 0, 4096, 38, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16569, 16386, 16397, 16472, 16331, 16384, 16380, 16153] avg=16384.0000 max=16569 max_rank=0 min=16153 min_rank=7 imbalance_ratio=1.011292 max_load_reduction=114503 max_load_reduction_pct=0.873589 moved_to_echo=65248 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[200, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 3824, 0, 0, 4096, 0, 565, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4058, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 4096, 3531, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4085, 0, 4096, 0, 0, 4096, 38, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17079, 16386, 16397, 15901, 16415, 16384, 16380, 16130] avg=16384.0000 max=17079 max_rank=0 min=15901 min_rank=3 imbalance_ratio=1.042419 max_load_reduction=113993 max_load_reduction_pct=0.869698 moved_to_echo=65309 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 0, 0, 128, 32768, 128, 0, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[2048, 128, 0, 128, 32768, 128, 128, 32768, 0, 128, 0, 24704, 128, 128, 32768, 128, 4608, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 0, 0, 128, 128, 32768, 32768, 128, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 0, 32768, 0, 2176, 32768, 128, 128, 0, 128, 128, 128, 0, 32768, 30848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 128, 4608, 128, 128, 32768, 28288, 128, 128, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 32768, 0, 128, 0, 128, 128, 128, 128, 32512, 128, 32768, 0, 128, 32768, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 0, 128, 32768, 128, 0, 0, 32768, 128, 128, 128, 0, 32768, 128, 32512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 1, 4096, 0, 4096, 4092, 4096, 0, 0, 4096, 0, 0, 0, 4096, 34, 4096, 0, 4096, 0, 0, 0, 1977, 4096, 0, 0, 4096, 0, 0, 994, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 159, 0, 0, 0, 4096, 0, 0, 0, 993, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4047, 0, 0, 4096, 0, 0, 4087, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 12, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 68, 4096, 0, 4096, 0, 0, 0, 2074, 4096, 0, 0, 4096, 0, 0, 936, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 197, 0, 0, 0, 4096, 0, 0, 0, 839, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 4096, 4078, 0, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 514, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 55, 4096, 0, 4096, 0, 0, 0, 955, 4096, 0, 0, 4096, 0, 0, 757, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 1388, 0, 0, 0, 4096, 0, 0, 0, 442, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4081, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 10, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 9, 4096, 0, 4096, 0, 0, 0, 675, 4096, 0, 0, 4096, 0, 0, 1004, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 269, 0, 0, 0, 4096, 0, 0, 0, 2165, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 4096, 4072, 0, 0, 4096, 0, 0, 4094, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 641, 4096, 0, 4096, 4096, 4092, 0, 1, 4094, 1, 0, 0, 4096, 36, 4096, 0, 4096, 0, 0, 0, 566, 4096, 0, 0, 4096, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 1937, 0, 0, 0, 4096, 0, 0, 0, 857, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 4096, 4095, 4096, 4034, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4083, 9, 13, 4085, 5, 4082, 4033, 4085, 1, 0, 4082, 0, 0, 10, 4084, 929, 4065, 9, 4081, 0, 11, 0, 24, 4086, 55, 8, 4084, 10, 12, 788, 0, 0, 11, 0, 0, 0, 10, 4068, 13, 8, 0, 0, 5, 10, 0, 15, 0, 4087, 0, 14, 0, 0, 8, 12, 0, 12, 12, 4084, 10, 0, 5, 0, 10, 0, 4083, 12, 15, 34, 8, 12, 0, 0, 26, 0, 14, 0, 3, 0, 31, 0, 4080, 0, 4084, 14, 0, 4080, 2180, 3, 0, 5, 4077, 0, 0, 0, 181, 7, 0, 9, 4079, 4082, 0, 25, 13, 0, 11, 4084, 4074, 4082, 4062, 4080, 5, 0, 0, 0, 7, 12, 11, 4064, 4086, 0, 6, 4054, 0, 0, 4061, 4, 4084] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 434, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 19, 4096, 0, 4096, 0, 0, 0, 2835, 4096, 0, 0, 4096, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 101, 0, 0, 0, 4096, 0, 0, 0, 636, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4008, 4096, 4096, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 255, 4096, 0, 4096, 4096, 4096, 0, 5, 4096, 0, 0, 0, 4096, 9, 4096, 0, 4096, 0, 0, 0, 408, 4096, 0, 0, 4096, 0, 0, 749, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 1179, 0, 0, 0, 4096, 0, 0, 0, 1545, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4091, 4096, 4092, 4095, 4084, 0, 0, 0, 0, 0, 0, 0, 4096, 4073, 0, 0, 4096, 0, 0, 4078, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 3871, 4092, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4096, 0, 4096, 0, 0, 0, 1977, 2065, 0, 0, 4096, 0, 0, 994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4049, 2031, 1879, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4067, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2217, 0, 0, 4096, 159, 0, 0, 0, 4096, 0, 0, 0, 993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 29, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4047, 0, 0, 4096, 0, 0, 4087, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16156, 17358, 16383, 16376, 16371, 15657, 16413, 16358] avg=16384.0000 max=17358 max_rank=1 min=15657 min_rank=5 imbalance_ratio=1.059448 max_load_reduction=113714 max_load_reduction_pct=0.867569 moved_to_echo=64799 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 514, 0, 0, 3871, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 4096, 0, 4096, 0, 0, 0, 955, 2065, 0, 0, 4096, 0, 0, 757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4049, 2031, 1879, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4067, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2217, 0, 0, 4096, 1388, 0, 0, 0, 4096, 0, 0, 0, 442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4096, 29, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16673, 16120, 16383, 16376, 16371, 16335, 16398, 16416] avg=16384.0000 max=16673 max_rank=0 min=16120 min_rank=1 imbalance_ratio=1.017639 max_load_reduction=114399 max_load_reduction_pct=0.872795 moved_to_echo=65520 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 12, 0, 0, 3870, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 4096, 0, 4096, 0, 0, 0, 2074, 2058, 0, 0, 4096, 0, 0, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4055, 2038, 1880, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4070, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2216, 0, 0, 4096, 197, 0, 0, 0, 4096, 0, 0, 0, 839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 26, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4078, 0, 0, 4096, 0, 0, 4096, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16170, 17424, 16383, 16391, 16379, 15540, 16398, 16387] avg=16384.0000 max=17424 max_rank=1 min=15540 min_rank=5 imbalance_ratio=1.063477 max_load_reduction=113648 max_load_reduction_pct=0.867065 moved_to_echo=64704 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 0, 0, 3871, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 4096, 0, 0, 0, 675, 2065, 0, 0, 4096, 0, 0, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4049, 2031, 1879, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4067, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2217, 0, 0, 4096, 269, 0, 0, 0, 4096, 0, 0, 0, 2165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 29, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4072, 0, 0, 4096, 0, 0, 4094, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16169, 16041, 16383, 16376, 16371, 16939, 16403, 16390] avg=16384.0000 max=16939 max_rank=5 min=16041 min_rank=1 imbalance_ratio=1.033875 max_load_reduction=114133 max_load_reduction_pct=0.870766 moved_to_echo=66103 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 434, 0, 0, 3871, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 0, 4096, 0, 0, 0, 2835, 2065, 0, 0, 4096, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4049, 2031, 1879, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4067, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2217, 0, 0, 4096, 101, 0, 0, 0, 4096, 0, 0, 0, 636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 4096, 29, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 4096, 0, 0, 4096, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16593, 17382, 16383, 16376, 16371, 15242, 16316, 16409] avg=16384.0000 max=17382 max_rank=1 min=15242 min_rank=5 imbalance_ratio=1.060913 max_load_reduction=113690 max_load_reduction_pct=0.867386 moved_to_echo=64338 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 9, 13, 0, 5, 3858, 4033, 4085, 1, 0, 4082, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 929, 4065, 9, 4081, 0, 11, 0, 24, 2059, 55, 8, 4084, 10, 12, 788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 10, 4068, 13, 8, 0, 0, 5, 10, 0, 15, 0, 4085, 4084, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 14, 0, 0, 8, 12, 0, 12, 12, 4084, 10, 0, 5, 0, 10, 4037, 2027, 1873, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 12, 15, 34, 8, 12, 0, 0, 26, 0, 14, 0, 3, 0, 31, 4082, 4079, 4053, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 2211, 14, 0, 4080, 2180, 3, 0, 5, 4077, 0, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 0, 0, 0, 25, 13, 0, 11, 4084, 4074, 29, 4062, 4080, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 12, 11, 4064, 4086, 0, 6, 4054, 0, 0, 4061, 4, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16096, 16135, 16391, 16415, 16468, 16831, 16399, 16337] avg=16384.0000 max=16831 max_rank=5 min=16096 min_rank=0 imbalance_ratio=1.027283 max_load_reduction=114241 max_load_reduction_pct=0.871590 moved_to_echo=66035 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 641, 0, 0, 3871, 4096, 4092, 0, 1, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 4096, 0, 4096, 0, 0, 0, 566, 2065, 0, 0, 4096, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 2, 0, 4096, 0, 0, 0, 0, 0, 4049, 2031, 1879, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4066, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2217, 0, 0, 4096, 1937, 0, 0, 0, 4096, 0, 0, 0, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 29, 4096, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 4096, 0, 0, 4096, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16796, 15093, 16383, 16381, 16370, 17299, 16350, 16400] avg=16384.0000 max=17299 max_rank=5 min=15093 min_rank=1 imbalance_ratio=1.055847 max_load_reduction=113773 max_load_reduction_pct=0.868019 moved_to_echo=66419 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 255, 0, 0, 3871, 4096, 4096, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 4096, 0, 0, 0, 408, 2065, 0, 0, 4096, 0, 0, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 1, 0, 4049, 2031, 1879, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 4096, 4096, 4063, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2217, 0, 0, 4096, 1179, 0, 0, 0, 4096, 0, 0, 0, 1545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 29, 4095, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4073, 0, 0, 4096, 0, 0, 4078, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16419, 15519, 16383, 16381, 16371, 17229, 16395, 16375] avg=16384.0000 max=17229 max_rank=5 min=15519 min_rank=1 imbalance_ratio=1.051575 max_load_reduction=113843 max_load_reduction_pct=0.868553 moved_to_echo=66370 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 0, 0, 128, 32768, 128, 128, 0, 0, 128, 128, 0, 128, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 0, 128, 0, 0, 128, 128, 128, 128, 128, 32768, 128, 0, 128, 128, 128, 32512, 16256, 15104, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1280, 32768, 128, 32768, 0, 128, 0, 9600, 16512, 128, 128, 32768, 128, 128, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 32768, 0, 17792, 128, 0, 32768, 7424, 128, 0, 128, 32768, 0, 0, 0, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 1920, 0, 128, 30976, 32768, 32768, 128, 128, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 128, 128, 32768, 32640, 0, 128, 32768, 0, 0, 32768, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 128, 0, 128, 128, 0, 128, 128, 128, 0, 128, 32768, 32768, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 0, 0, 0, 128, 128, 0, 128, 32768, 32768, 256, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 3, 0, 15, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4095, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 0, 15, 0, 20, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4073, 11, 4081, 0, 0, 0, 14, 0, 0, 4086, 0, 0, 9, 0, 4086, 4068, 0, 0, 4087, 9, 10, 11, 0, 0, 223, 11, 9, 0, 4084, 7, 4083, 4085, 0, 1, 26, 8, 5, 0, 0, 4028, 5, 0, 12, 14, 0, 4087, 4068, 10, 9, 10, 5, 0, 0, 0, 9, 4082, 0, 4083, 0, 4079, 0, 9, 5, 4080, 0, 0, 4, 15, 0, 0, 9, 0, 4076, 0, 4081, 10, 0, 0, 13, 0, 0, 0, 4063, 8, 3872, 4081, 4082, 4084, 1, 0, 0, 4085, 0, 9, 0, 0, 2, 11, 4089, 1, 0, 15, 4086, 8, 0, 10, 0, 12, 14, 6, 6, 0, 13, 4083, 4085, 0, 5, 4084, 4085, 4083, 8, 62, 0, 4084, 13, 20, 1, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 105, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 10, 0, 1, 0, 0, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 2, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 4045, 0, 4096, 0, 0, 0, 0, 13, 0, 48, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4041, 48, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 4096, 4096, 4096, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4080, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16354, 16387, 16385, 16399, 16413, 16370, 16374] avg=16384.0000 max=16413 max_rank=5 min=16354 min_rank=1 imbalance_ratio=1.001770 max_load_reduction=114659 max_load_reduction_pct=0.874779 moved_to_echo=65556 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4041, 48, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4095, 4096, 4096, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4080, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16336, 16387, 16384, 16400, 16407, 16394, 16374] avg=16384.0000 max=16407 max_rank=5 min=16336 min_rank=1 imbalance_ratio=1.001404 max_load_reduction=114665 max_load_reduction_pct=0.874825 moved_to_echo=65575 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4041, 48, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 3, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 4096, 4096, 4096, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4080, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16336, 16387, 16384, 16403, 16422, 16376, 16374] avg=16384.0000 max=16422 max_rank=5 min=16336 min_rank=1 imbalance_ratio=1.002319 max_load_reduction=114650 max_load_reduction_pct=0.874710 moved_to_echo=65575 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4046, 53, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4096, 0, 0, 0, 0, 4, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4083, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16354, 16398, 16386, 16393, 16398, 16388, 16371] avg=16384.0000 max=16398 max_rank=2 min=16354 min_rank=1 imbalance_ratio=1.000854 max_load_reduction=114674 max_load_reduction_pct=0.874893 moved_to_echo=65550 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4073, 11, 4081, 0, 0, 0, 14, 0, 0, 6, 0, 0, 9, 0, 4086, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 9, 10, 11, 0, 0, 223, 11, 9, 0, 4084, 7, 4083, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 26, 8, 5, 0, 0, 4028, 5, 0, 12, 14, 0, 4087, 4068, 10, 4029, 48, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 10, 5, 0, 0, 0, 9, 4082, 0, 4083, 0, 4079, 0, 9, 5, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 15, 0, 0, 9, 0, 4076, 0, 4081, 10, 0, 0, 13, 0, 4085, 4085, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 8, 3872, 4081, 4082, 23, 1, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 11, 4089, 1, 0, 15, 4086, 8, 0, 10, 0, 12, 14, 6, 6, 0, 4080, 4070, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4083, 0, 0, 5, 4074, 0, 4083, 8, 62, 0, 4084, 13, 20, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16348, 16571, 16351, 16371, 16393, 16139, 16442, 16457] avg=16384.0000 max=16571 max_rank=1 min=16139 min_rank=5 imbalance_ratio=1.011414 max_load_reduction=114501 max_load_reduction_pct=0.873573 moved_to_echo=65431 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 1, 0, 105, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4041, 48, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 4096, 4096, 4096, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4080, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16442, 16387, 16384, 16316, 16411, 16368, 16374] avg=16384.0000 max=16442 max_rank=1 min=16316 min_rank=4 imbalance_ratio=1.003540 max_load_reduction=114630 max_load_reduction_pct=0.874557 moved_to_echo=65469 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4041, 48, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 0, 15, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 0, 4096, 4096, 4096, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4080, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16335, 16392, 16384, 16405, 16427, 16365, 16374] avg=16384.0000 max=16427 max_rank=5 min=16335 min_rank=1 imbalance_ratio=1.002625 max_load_reduction=114645 max_load_reduction_pct=0.874672 moved_to_echo=65571 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 0, 10, 0, 1, 0, 0, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4092, 4096, 0, 4041, 48, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 2, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4045, 0, 4096, 0, 0, 0, 0, 13, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 4096, 0, 4096, 4096, 4096, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4080, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16344, 16383, 16394, 16363, 16455, 16369, 16374] avg=16384.0000 max=16455 max_rank=5 min=16344 min_rank=1 imbalance_ratio=1.004333 max_load_reduction=114617 max_load_reduction_pct=0.874458 moved_to_echo=65561 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 128, 32768, 0, 0, 0, 128, 0, 0, 128, 0, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 32640, 128, 0, 128, 32768, 128, 0, 128, 0, 128, 128, 128, 128, 0, 32768, 32640, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 32384, 128, 128, 128, 128, 0, 384, 128, 128, 0, 32768, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 0, 0, 128, 32768, 0, 32768, 128, 128, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 128, 128, 0, 32640, 0, 32768, 128, 0, 0, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 128, 32640, 32768, 32768, 256, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 0, 0, 32768, 128, 0, 128, 128, 0, 32768, 32768, 128, 32384, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 32768, 128, 32768, 0, 32768, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True + [2026-08-30 05:03:35] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 799.2 | throughput per GPU (TFLOP/s/GPU): 417.1 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.016021E+01 | loss scale: 1.0 | grad norm: 16.535 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[34, 197, 319, 2, 37, 0, 1139, 7, 1033, 204, 41, 2206, 1156, 1224, 2, 3847, 2873, 402, 1344, 1341, 2812, 2830, 3, 81, 3, 15, 1317, 448, 2156, 2, 1611, 571, 4050, 0, 9, 2595, 1777, 35, 2475, 1257, 925, 4, 338, 695, 1, 1224, 218, 1252, 2853, 3492, 1709, 10, 3, 3, 723, 1705, 0, 1400, 926, 1515, 2306, 1251, 1868, 3, 1783, 2983, 0, 594, 396, 106, 260, 726, 3870, 1387, 471, 0, 175, 0, 40, 529, 1549, 2973, 0, 1110, 33, 2026, 499, 2529, 490, 5, 1591, 2820, 959, 311, 1804, 2, 1738, 3190, 825, 148, 1, 2050, 814, 2163, 1106, 513, 2, 1071, 2393, 768, 287, 171, 289, 641, 2060, 2391, 33, 319, 315, 846, 1811, 1342, 417, 3154, 39, 3, 2272, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[7, 152, 426, 0, 15, 0, 1274, 4, 914, 226, 94, 1804, 1035, 892, 1, 3900, 3297, 505, 1646, 1193, 2960, 2280, 11, 36, 0, 36, 1388, 421, 2308, 0, 1737, 707, 4073, 1, 19, 2437, 1710, 18, 2054, 823, 828, 4, 319, 624, 0, 1128, 246, 2140, 2645, 3704, 1902, 28, 0, 1, 568, 1260, 0, 1862, 728, 1071, 1957, 1019, 2354, 3, 2741, 2986, 0, 617, 387, 70, 291, 771, 3906, 1633, 261, 1, 106, 3, 32, 343, 1643, 2483, 0, 1250, 53, 1916, 621, 2412, 332, 0, 1748, 2477, 675, 603, 1606, 0, 1631, 3081, 701, 182, 2, 2054, 515, 1452, 1527, 531, 0, 1302, 2390, 1230, 215, 354, 388, 775, 2031, 2666, 65, 179, 224, 782, 2572, 1551, 591, 2887, 35, 5, 2392, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[18, 254, 669, 0, 13, 0, 1316, 3, 1268, 235, 141, 2483, 1133, 1412, 0, 3749, 1942, 255, 1668, 981, 3124, 2276, 12, 71, 4, 29, 1297, 435, 2711, 1, 1792, 426, 4049, 0, 19, 2829, 2671, 27, 1902, 899, 1003, 15, 414, 858, 3, 730, 136, 1444, 3049, 3516, 2155, 1, 1, 2, 529, 1406, 0, 2065, 605, 1624, 2086, 1465, 2477, 1, 2974, 3301, 0, 902, 261, 117, 249, 1023, 3754, 1180, 243, 0, 101, 29, 29, 362, 1834, 2667, 3, 961, 93, 1932, 405, 2510, 138, 4, 1753, 2472, 951, 538, 1455, 1, 1785, 3280, 375, 232, 3, 2312, 581, 1381, 1294, 301, 4, 788, 2022, 770, 166, 410, 359, 762, 1784, 2453, 28, 183, 455, 647, 2612, 1111, 332, 2893, 18, 3, 1773, 14] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[27, 253, 381, 0, 29, 1, 1084, 7, 1114, 177, 102, 2060, 1174, 1096, 3, 3929, 3125, 245, 1585, 842, 2701, 2076, 64, 90, 11, 37, 1260, 637, 2069, 0, 1533, 376, 4008, 1, 11, 2137, 1856, 35, 2256, 1386, 1001, 19, 492, 879, 3, 1727, 116, 1887, 3240, 3139, 2271, 38, 2, 1, 854, 1089, 1, 1317, 750, 1368, 2018, 1584, 2457, 0, 2996, 3064, 4, 669, 495, 86, 169, 653, 3646, 1705, 247, 7, 107, 8, 118, 557, 1932, 2482, 6, 1340, 71, 2010, 285, 2484, 240, 8, 1868, 2715, 1166, 797, 1420, 1, 1367, 3097, 450, 180, 0, 1746, 556, 1671, 1333, 749, 1, 1473, 2434, 970, 250, 340, 310, 1042, 2056, 1864, 17, 308, 234, 798, 1977, 1522, 414, 2770, 20, 0, 1735, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[33, 367, 657, 2, 24, 3, 1309, 2, 736, 119, 77, 1773, 1727, 1120, 2, 3908, 2715, 207, 1612, 1896, 2468, 1900, 109, 49, 1, 21, 1288, 691, 2187, 0, 1549, 422, 4076, 6, 9, 2296, 1504, 57, 2706, 1680, 994, 7, 555, 646, 5, 1128, 352, 2100, 2895, 3419, 2113, 53, 1, 0, 756, 997, 0, 1652, 735, 749, 2194, 1197, 2318, 2, 2210, 3041, 6, 595, 253, 41, 360, 689, 3756, 1484, 394, 4, 94, 3, 69, 429, 1591, 2946, 2, 1289, 41, 1985, 1090, 2767, 257, 10, 1963, 2954, 649, 473, 1673, 4, 1147, 3012, 1186, 241, 21, 2069, 941, 1110, 978, 588, 10, 976, 2436, 877, 170, 269, 178, 826, 1719, 2227, 33, 546, 285, 1138, 2303, 1140, 579, 2750, 25, 0, 1983, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[94, 574, 435, 0, 208, 0, 1173, 33, 1427, 137, 58, 1198, 1454, 987, 0, 3874, 3159, 503, 2329, 845, 2120, 1985, 147, 77, 6, 111, 1537, 488, 1991, 4, 1825, 462, 3955, 1, 111, 2346, 1289, 75, 2743, 1290, 1425, 28, 899, 1512, 0, 1098, 338, 2283, 2752, 3696, 2388, 195, 3, 2, 394, 1280, 2, 1096, 893, 1422, 2423, 1160, 2472, 19, 2013, 3256, 0, 503, 243, 298, 625, 675, 3683, 1230, 323, 1, 53, 7, 52, 1090, 1622, 2334, 4, 711, 56, 1660, 1377, 1823, 762, 8, 2051, 2377, 357, 296, 1218, 3, 1968, 2592, 332, 262, 6, 1837, 831, 1539, 1284, 1040, 5, 1663, 2231, 351, 190, 300, 260, 955, 1932, 2555, 91, 197, 417, 622, 1937, 1140, 705, 2416, 18, 4, 1847, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[32, 273, 565, 0, 46, 0, 950, 1, 598, 232, 53, 1964, 1678, 1360, 0, 3965, 3415, 322, 1658, 1136, 2667, 2319, 110, 100, 1, 13, 1360, 552, 2571, 3, 1646, 383, 4072, 1, 18, 2227, 2060, 29, 2713, 1988, 2193, 0, 366, 818, 0, 818, 177, 2331, 2331, 3585, 1821, 53, 1, 1, 748, 1695, 0, 1560, 668, 783, 2046, 1123, 2588, 1, 2699, 2818, 0, 759, 588, 38, 315, 791, 3774, 1730, 226, 5, 71, 1, 85, 442, 1699, 2888, 0, 1366, 92, 1850, 530, 1424, 479, 7, 1449, 3207, 1055, 465, 1028, 0, 1627, 3198, 415, 224, 1, 1696, 479, 1411, 1038, 513, 3, 2001, 2096, 584, 233, 72, 215, 916, 1965, 2092, 54, 381, 177, 617, 1793, 1380, 414, 2883, 45, 0, 1880, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[24, 269, 256, 2, 14, 2, 1151, 8, 967, 119, 136, 1971, 1460, 1029, 0, 3925, 2133, 444, 1369, 825, 2116, 2265, 27, 189, 40, 10, 858, 556, 2547, 2, 1360, 476, 4037, 2, 10, 1712, 1513, 27, 2507, 1936, 993, 38, 533, 700, 10, 1285, 216, 1920, 2813, 3572, 2440, 45, 14, 11, 919, 1446, 1, 2058, 1052, 1076, 1954, 1033, 2064, 1, 2545, 3297, 0, 982, 535, 38, 286, 452, 3728, 2068, 313, 56, 81, 5, 81, 522, 1484, 2981, 7, 1234, 40, 1864, 774, 2626, 716, 14, 1883, 2829, 933, 496, 1610, 17, 1742, 3152, 715, 225, 5, 1750, 779, 1776, 1392, 552, 6, 903, 2355, 628, 144, 463, 250, 771, 1667, 2194, 17, 196, 370, 596, 2092, 1279, 755, 2967, 23, 6, 2317, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[18, 254, 669, 0, 13, 0, 1316, 3, 1268, 235, 141, 2483, 1133, 1412, 0, 3749, 3461, 1187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1333, 255, 1668, 981, 3124, 2276, 12, 71, 4, 29, 1297, 435, 2711, 1, 1792, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2573, 0, 19, 2829, 2671, 27, 1902, 899, 1003, 15, 414, 858, 3, 730, 136, 1444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3049, 55, 2155, 1, 1, 2, 529, 1406, 0, 2065, 605, 1624, 2086, 1465, 2477, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2974, 3301, 0, 902, 261, 117, 249, 1023, 3754, 1180, 243, 0, 101, 29, 29, 362, 1476, 553, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1834, 2667, 3, 961, 93, 1932, 405, 2510, 138, 4, 1753, 732, 951, 538, 1455, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1785, 3241, 375, 232, 3, 2312, 581, 1381, 1294, 301, 4, 788, 2022, 770, 166, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 359, 762, 1784, 2453, 28, 183, 455, 647, 2612, 1111, 332, 2893, 18, 3, 1773, 14, 518, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17342, 16415, 15523, 17521, 16645, 15977, 15665, 15984] avg=16384.0000 max=17521 max_rank=3 min=15523 min_rank=2 imbalance_ratio=1.069397 max_load_reduction=113551 max_load_reduction_pct=0.866325 moved_to_echo=64271 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[7, 152, 426, 0, 15, 0, 1274, 4, 914, 226, 94, 1804, 1035, 892, 1, 3900, 3646, 1190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2263, 505, 1646, 1193, 2960, 2280, 11, 36, 0, 36, 1388, 421, 2308, 0, 1737, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2588, 1, 19, 2437, 1710, 18, 2054, 823, 828, 4, 319, 624, 0, 1128, 246, 2140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2645, 58, 1902, 28, 0, 1, 568, 1260, 0, 1862, 728, 1071, 1957, 1019, 2354, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2741, 2986, 0, 617, 387, 70, 291, 771, 3906, 1633, 261, 1, 106, 3, 32, 343, 1485, 554, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1643, 2483, 0, 1250, 53, 1916, 621, 2412, 332, 0, 1748, 733, 675, 603, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1631, 3044, 701, 182, 2, 2054, 515, 1452, 1527, 531, 0, 1302, 2390, 1230, 215, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 388, 775, 2031, 2666, 65, 179, 224, 782, 2572, 1551, 591, 2887, 35, 5, 2392, 0, 880, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15580, 17491, 14939, 15456, 16341, 16075, 17130, 18060] avg=16384.0000 max=18060 max_rank=7 min=14939 min_rank=2 imbalance_ratio=1.102295 max_load_reduction=113012 max_load_reduction_pct=0.862213 moved_to_echo=67606 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[34, 197, 319, 2, 37, 0, 1139, 7, 1033, 204, 41, 2206, 1156, 1224, 2, 3847, 3438, 1354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1972, 402, 1344, 1341, 2812, 2830, 3, 81, 3, 15, 1317, 448, 2156, 2, 1611, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2574, 0, 9, 2595, 1777, 35, 2475, 1257, 925, 4, 338, 695, 1, 1224, 218, 1252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2853, 54, 1709, 10, 3, 3, 723, 1705, 0, 1400, 926, 1515, 2306, 1251, 1868, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1783, 2983, 0, 594, 396, 106, 260, 726, 3870, 1387, 471, 0, 175, 0, 40, 529, 1476, 631, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1549, 2973, 0, 1110, 33, 2026, 499, 2529, 490, 5, 1591, 835, 959, 311, 1804, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1738, 3152, 825, 148, 1, 2050, 814, 2163, 1106, 513, 2, 1071, 2393, 768, 287, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 289, 641, 2060, 2391, 33, 319, 315, 846, 1811, 1342, 417, 3154, 39, 3, 2272, 0, 767, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16240, 16908, 15379, 16329, 15561, 16716, 17202, 16737] avg=16384.0000 max=17202 max_rank=6 min=15379 min_rank=2 imbalance_ratio=1.049927 max_load_reduction=113870 max_load_reduction_pct=0.868759 moved_to_echo=66216 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[33, 367, 657, 2, 24, 3, 1309, 2, 736, 119, 77, 1773, 1727, 1120, 2, 3908, 3366, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1863, 207, 1612, 1896, 2468, 1900, 109, 49, 1, 21, 1288, 691, 2187, 0, 1549, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2590, 6, 9, 2296, 1504, 57, 2706, 1680, 994, 7, 555, 646, 5, 1128, 352, 2100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2895, 53, 2113, 53, 1, 0, 756, 997, 0, 1652, 735, 749, 2194, 1197, 2318, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2210, 3041, 6, 595, 253, 41, 360, 689, 3756, 1484, 394, 4, 94, 3, 69, 429, 1486, 661, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1591, 2946, 2, 1289, 41, 1985, 1090, 2767, 257, 10, 1963, 874, 649, 473, 1673, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1147, 2976, 1186, 241, 21, 2069, 941, 1110, 978, 588, 10, 976, 2436, 877, 170, 269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 826, 1719, 2227, 33, 546, 285, 1138, 2303, 1140, 579, 2750, 25, 0, 1983, 11, 725, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16644, 16263, 16635, 15715, 15702, 17614, 15995, 16504] avg=16384.0000 max=17614 max_rank=5 min=15702 min_rank=4 imbalance_ratio=1.075073 max_load_reduction=113458 max_load_reduction_pct=0.865616 moved_to_echo=65815 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[27, 253, 381, 0, 29, 1, 1084, 7, 1114, 177, 102, 2060, 1174, 1096, 3, 3929, 3090, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2145, 245, 1585, 842, 2701, 2076, 64, 90, 11, 37, 1260, 637, 2069, 0, 1533, 376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2547, 1, 11, 2137, 1856, 35, 2256, 1386, 1001, 19, 492, 879, 3, 1727, 116, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3240, 49, 2271, 38, 2, 1, 854, 1089, 1, 1317, 750, 1368, 2018, 1584, 2457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2996, 3064, 4, 669, 495, 86, 169, 653, 3646, 1705, 247, 7, 107, 8, 118, 557, 1461, 607, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1932, 2482, 6, 1340, 71, 2010, 285, 2484, 240, 8, 1868, 804, 1166, 797, 1420, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1367, 3060, 450, 180, 0, 1746, 556, 1671, 1333, 749, 1, 1473, 2434, 970, 250, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 1042, 2056, 1864, 17, 308, 234, 798, 1977, 1522, 414, 2770, 20, 0, 1735, 1, 834, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15831, 15671, 16353, 17039, 16745, 16914, 16580, 15939] avg=16384.0000 max=17039 max_rank=3 min=15671 min_rank=1 imbalance_ratio=1.039978 max_load_reduction=114033 max_load_reduction_pct=0.870003 moved_to_echo=66178 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[94, 574, 435, 0, 208, 0, 1173, 33, 1427, 137, 58, 1198, 1454, 987, 0, 3874, 3643, 1146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2161, 503, 2329, 845, 2120, 1985, 147, 77, 6, 111, 1537, 488, 1991, 4, 1825, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2510, 1, 111, 2346, 1289, 75, 2743, 1290, 1425, 28, 899, 1512, 0, 1098, 338, 2283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2752, 53, 2388, 195, 3, 2, 394, 1280, 2, 1096, 893, 1422, 2423, 1160, 2472, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2013, 3256, 0, 503, 243, 298, 625, 675, 3683, 1230, 323, 1, 53, 7, 52, 1090, 1445, 535, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1622, 2334, 4, 711, 56, 1660, 1377, 1823, 762, 8, 2051, 696, 357, 296, 1218, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1968, 2558, 332, 262, 6, 1837, 831, 1539, 1284, 1040, 5, 1663, 2231, 351, 190, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 260, 955, 1932, 2555, 91, 197, 417, 622, 1937, 1140, 705, 2416, 18, 4, 1847, 3, 847, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16441, 16591, 17948, 16554, 16183, 14978, 16397, 15980] avg=16384.0000 max=17948 max_rank=2 min=14978 min_rank=5 imbalance_ratio=1.095459 max_load_reduction=113124 max_load_reduction_pct=0.863068 moved_to_echo=63538 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[32, 273, 565, 0, 46, 0, 950, 1, 598, 232, 53, 1964, 1678, 1360, 0, 3965, 3529, 1540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2343, 322, 1658, 1136, 2667, 2319, 110, 100, 1, 13, 1360, 552, 2571, 3, 1646, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2588, 1, 18, 2227, 2060, 29, 2713, 1988, 2193, 0, 366, 818, 0, 818, 177, 2331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2331, 56, 1821, 53, 1, 1, 748, 1695, 0, 1560, 668, 783, 2046, 1123, 2588, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2699, 2818, 0, 759, 588, 38, 315, 791, 3774, 1730, 226, 5, 71, 1, 85, 442, 1484, 717, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1699, 2888, 0, 1366, 92, 1850, 530, 1424, 479, 7, 1449, 950, 1055, 465, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1627, 3160, 415, 224, 1, 1696, 479, 1411, 1038, 513, 3, 2001, 2096, 584, 233, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 215, 916, 1965, 2092, 54, 381, 177, 617, 1793, 1380, 414, 2883, 45, 0, 1880, 0, 912, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16786, 17184, 18327, 15475, 16703, 15282, 15553, 15762] avg=16384.0000 max=18327 max_rank=2 min=15282 min_rank=5 imbalance_ratio=1.118591 max_load_reduction=112745 max_load_reduction_pct=0.860176 moved_to_echo=63300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[24, 269, 256, 2, 14, 2, 1151, 8, 967, 119, 136, 1971, 1460, 1029, 0, 3925, 3516, 1359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1465, 444, 1369, 825, 2116, 2265, 27, 189, 40, 10, 858, 556, 2547, 2, 1360, 476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2566, 2, 10, 1712, 1513, 27, 2507, 1936, 993, 38, 533, 700, 10, 1285, 216, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2813, 56, 2440, 45, 14, 11, 919, 1446, 1, 2058, 1052, 1076, 1954, 1033, 2064, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2545, 3297, 0, 982, 535, 38, 286, 452, 3728, 2068, 313, 56, 81, 5, 81, 522, 1471, 633, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1484, 2981, 7, 1234, 40, 1864, 774, 2626, 716, 14, 1883, 837, 933, 496, 1610, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 3115, 715, 225, 5, 1750, 779, 1776, 1392, 552, 6, 903, 2355, 628, 144, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 771, 1667, 2194, 17, 196, 370, 596, 2092, 1279, 755, 2967, 23, 6, 2317, 0, 569, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16208, 14549, 15968, 16983, 17192, 17516, 16550, 16106] avg=16384.0000 max=17516 max_rank=5 min=14549 min_rank=1 imbalance_ratio=1.069092 max_load_reduction=113556 max_load_reduction_pct=0.866364 moved_to_echo=67364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[13440, 21760, 128, 9344, 512, 15360, 5632, 18688, 3456, 128, 14336, 6528, 6784, 4096, 11904, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[15616, 2944, 13312, 9088, 20992, 18048, 512, 768, 128, 384, 10368, 4352, 18560, 128, 13056, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[13056, 24320, 5120, 1792, 128, 15616, 5504, 12544, 9984, 4864, 128, 10240, 18432, 6272, 1664, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[20608, 128, 256, 18688, 14464, 384, 19456, 11264, 9472, 128, 3968, 6784, 128, 9216, 1920, 15360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[19968, 24832, 128, 5632, 3200, 896, 2560, 5888, 30208, 12544, 2560, 128, 896, 128, 512, 4352, 11904, 4992, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[384, 2432, 3712, 128, 512, 128, 9472, 128, 8064, 1536, 768, 15488, 10880, 9216, 128, 31104, 27776, 10624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[22656, 512, 16896, 512, 128, 128, 5504, 10880, 128, 13056, 6400, 9728, 17024, 9856, 18688, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2304, 6784, 15232, 18560, 384, 2432, 2560, 6144, 17152, 10496, 4224, 22784, 256, 128, 16256, 128, 6144, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 7, 4, 4088, 1, 1, 4094, 4094, 0, 6, 49, 0, 0, 0, 3829, 5, 1, 4091, 7, 3, 1, 1, 4086, 3972, 4090, 0, 1, 4096, 6, 10, 4095, 0, 5, 326, 49, 0, 5, 9, 4088, 0, 0, 4073, 4077, 48, 0, 4076, 2, 15, 11, 10, 4, 1, 0, 4096, 1, 4088, 4084, 0, 0, 4082, 2657, 0, 146, 0, 3, 0, 0, 0, 1, 15, 5, 0, 10, 1, 1, 1, 3, 2, 0, 0, 5, 0, 4093, 0, 0, 4069, 23, 0, 0, 8, 5, 3838, 0, 4, 1531, 4095, 7, 3, 0, 4096, 5, 4092, 0, 3, 4054, 15, 0, 10, 3, 0, 4075, 0, 4092, 4082, 0, 4086, 0, 19, 4091, 0, 0, 0, 4021, 0, 0, 2, 3, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 2, 4096, 12, 0, 4096, 4095, 4, 0, 1128, 0, 0, 0, 4057, 0, 0, 4091, 8, 0, 2, 0, 4089, 2279, 4092, 0, 0, 4093, 0, 6, 4095, 0, 0, 128, 87, 0, 1, 0, 4090, 0, 1, 4095, 4095, 4, 0, 4091, 0, 0, 1, 0, 0, 0, 0, 4096, 29, 4096, 4096, 0, 0, 4086, 1146, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4, 4090, 0, 0, 4087, 15, 0, 0, 0, 0, 3756, 0, 7, 3675, 4095, 0, 0, 6, 4095, 4, 4092, 0, 0, 4082, 44, 0, 7, 3, 0, 4095, 0, 4095, 4082, 0, 4091, 0, 23, 4094, 0, 26, 0, 3963, 3, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 2, 1, 4092, 4096, 3, 0, 135, 7, 1, 0, 4081, 2, 0, 4096, 1, 0, 0, 5, 4093, 3097, 4092, 0, 0, 4090, 4, 0, 4092, 1, 0, 343, 131, 0, 8, 0, 4093, 1, 2, 4091, 4052, 33, 1, 4090, 0, 4, 0, 3, 0, 0, 0, 4096, 10, 4095, 4095, 0, 2, 4091, 858, 1, 593, 4, 1, 0, 0, 0, 2, 57, 2, 0, 0, 1, 0, 2, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4095, 14, 0, 0, 0, 0, 3734, 0, 0, 3167, 4093, 0, 0, 0, 4095, 8, 4096, 0, 0, 4058, 173, 0, 0, 1, 0, 4091, 1, 4095, 4096, 1, 4095, 0, 11, 4093, 1, 4, 6, 4095, 2, 0, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 2, 4093, 0, 1, 4094, 4096, 2, 1, 124, 1, 0, 0, 3422, 0, 0, 4095, 10, 0, 1, 1, 4095, 3437, 4094, 0, 0, 4096, 2, 1, 4096, 0, 0, 374, 8, 1, 0, 0, 4093, 0, 1, 4095, 4095, 46, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4094, 101, 4095, 4093, 0, 0, 4093, 1528, 2, 21, 0, 1, 2, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 2, 0, 0, 1, 1, 4096, 0, 1, 4092, 0, 3, 0, 0, 0, 3580, 0, 0, 3734, 4094, 0, 0, 0, 4094, 23, 4095, 0, 2, 4065, 18, 0, 7, 0, 1, 4091, 2, 4094, 4093, 0, 4096, 0, 0, 4094, 0, 0, 0, 4086, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 4095, 4095, 0, 15, 140, 0, 1, 0, 3176, 1, 0, 4096, 17, 1, 0, 1, 4095, 3265, 4095, 1, 1, 4096, 0, 0, 4095, 0, 0, 285, 12, 0, 0, 7, 4096, 0, 0, 4096, 4067, 1, 0, 4084, 1, 0, 0, 1, 0, 0, 0, 4091, 7, 4096, 4096, 0, 0, 4094, 2495, 0, 11, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 3, 4094, 1, 0, 4093, 2, 0, 1, 0, 1, 3669, 0, 0, 3380, 4096, 0, 0, 2, 4095, 0, 4095, 0, 0, 4089, 26, 0, 2, 0, 0, 4090, 0, 4096, 4096, 0, 4095, 0, 9, 4095, 0, 0, 1, 4014, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 7, 0, 4094, 4091, 1, 4, 277, 0, 0, 4, 3439, 7, 0, 4092, 0, 0, 3, 0, 4090, 3309, 4090, 0, 0, 4089, 6, 0, 4091, 4, 1, 377, 0, 0, 2, 3, 4091, 0, 0, 4091, 4088, 0, 0, 4086, 0, 0, 0, 1, 0, 1, 0, 4093, 7, 4095, 4096, 0, 0, 4085, 1787, 0, 19, 4, 0, 0, 1, 3, 3, 1, 0, 6, 0, 0, 0, 0, 4, 3, 0, 1, 1, 6, 4086, 0, 5, 4094, 13, 0, 0, 5, 0, 3278, 1, 2, 3789, 4088, 0, 0, 0, 4094, 18, 4096, 3, 2, 4088, 13, 0, 5, 8, 2, 4085, 2, 4090, 4090, 0, 4092, 1, 10, 4094, 4, 2, 0, 4085, 75, 0, 1, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 2, 4091, 2, 3, 4094, 4094, 1, 33, 347, 5, 0, 3, 3919, 0, 5, 4091, 3, 0, 5, 1, 4083, 2144, 4086, 1, 0, 4088, 0, 0, 4090, 0, 0, 51, 5, 7, 26, 2, 4093, 3, 4, 4094, 4085, 33, 0, 4088, 2, 0, 0, 0, 0, 1, 2, 4091, 3, 4092, 4092, 0, 7, 4096, 2348, 1, 150, 0, 0, 1, 0, 0, 3, 0, 0, 2, 0, 3, 0, 0, 0, 0, 4, 0, 0, 4, 4088, 4, 1, 4086, 0, 0, 0, 2, 6, 3569, 0, 0, 3707, 4095, 0, 0, 0, 4096, 3, 4087, 2, 1, 4076, 25, 0, 0, 0, 0, 4092, 2, 4092, 4096, 0, 4096, 0, 53, 4095, 0, 0, 11, 4096, 6, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 2, 4081, 0, 0, 4050, 4096, 1, 27, 4, 0, 0, 0, 1734, 0, 0, 4090, 0, 0, 6, 86, 4096, 3614, 4069, 0, 3, 4096, 51, 0, 4072, 0, 0, 1267, 9, 0, 0, 0, 4083, 0, 5, 4095, 3992, 14, 0, 4071, 0, 0, 47, 0, 0, 0, 0, 4053, 4, 4090, 4071, 0, 5, 4077, 3429, 0, 142, 0, 0, 3, 1, 0, 0, 63, 0, 5, 12, 0, 8, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 3997, 120, 0, 0, 0, 0, 3159, 0, 5, 2965, 4096, 1, 0, 0, 4093, 15, 4096, 0, 0, 4094, 96, 0, 63, 0, 1, 4028, 0, 4093, 4096, 0, 4093, 0, 1, 4081, 7, 53, 0, 3959, 0, 0, 0, 0, 25] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 4, 4088, 1, 1, 4094, 4094, 0, 6, 49, 0, 0, 0, 3829, 5, 310, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 941, 7, 3, 1, 1, 4086, 3972, 4090, 0, 1, 0, 6, 10, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 326, 49, 0, 5, 9, 4088, 0, 0, 3656, 4077, 48, 0, 4076, 2, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 10, 4, 1, 0, 4096, 1, 1897, 4084, 0, 0, 4082, 2657, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 15, 5, 0, 10, 1, 1, 1, 3, 2, 0, 0, 4096, 4071, 3150, 2732, 2191, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4093, 0, 0, 4069, 23, 0, 0, 8, 5, 3838, 0, 4, 1531, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 0, 4056, 5, 4092, 0, 3, 4054, 15, 0, 10, 3, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4082, 0, 4086, 0, 19, 4091, 0, 0, 0, 4021, 0, 0, 2, 3, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16528, 17214, 16356, 16989, 16389, 14939, 16323, 16334] avg=16384.0000 max=17214 max_rank=1 min=14939 min_rank=5 imbalance_ratio=1.050659 max_load_reduction=113858 max_load_reduction_pct=0.868668 moved_to_echo=63985 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 4096, 12, 0, 4096, 4095, 4, 0, 1128, 0, 0, 0, 4057, 0, 316, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 937, 8, 0, 2, 0, 4089, 2279, 4092, 0, 0, 0, 0, 6, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 87, 0, 1, 0, 4090, 0, 1, 3668, 4095, 4, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 29, 1896, 4096, 0, 0, 4086, 1146, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 4076, 3154, 2737, 2200, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 4090, 0, 0, 4087, 15, 0, 0, 0, 0, 3756, 0, 7, 3675, 1358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4049, 4, 4092, 0, 0, 4082, 44, 0, 7, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4082, 0, 4091, 0, 23, 4094, 0, 26, 0, 3963, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17852, 15508, 16165, 15495, 16375, 16993, 16382, 16302] avg=16384.0000 max=17852 max_rank=0 min=15495 min_rank=3 imbalance_ratio=1.089600 max_load_reduction=113220 max_load_reduction_pct=0.863800 moved_to_echo=66052 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 7, 0, 4094, 4091, 1, 4, 277, 0, 0, 4, 3439, 7, 312, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 941, 0, 0, 3, 0, 4090, 3309, 4090, 0, 0, 0, 6, 0, 4091, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 377, 0, 0, 2, 3, 4091, 0, 0, 3671, 4088, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4093, 7, 1900, 4096, 0, 0, 4085, 1787, 0, 19, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 3, 1, 0, 6, 0, 0, 0, 0, 4, 3, 0, 1, 4089, 4069, 3151, 2727, 2195, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 4086, 0, 5, 4094, 13, 0, 0, 5, 0, 3278, 1, 2, 3789, 1361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 18, 4096, 3, 2, 4088, 13, 0, 5, 8, 2, 4085, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4090, 0, 4092, 1, 10, 4094, 4, 2, 0, 4085, 75, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 16534, 16319, 15993, 16361, 16641, 16376, 16476] avg=16384.0000 max=16641 max_rank=5 min=15993 min_rank=3 imbalance_ratio=1.015686 max_load_reduction=114431 max_load_reduction_pct=0.873039 moved_to_echo=65854 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 2, 1, 4092, 4096, 3, 0, 135, 7, 1, 0, 4081, 2, 312, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 942, 1, 0, 0, 5, 4093, 3097, 4092, 0, 0, 0, 4, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 131, 0, 8, 0, 4093, 1, 2, 3671, 4052, 33, 1, 4090, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 10, 1900, 4095, 0, 2, 4091, 858, 1, 593, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 57, 2, 0, 0, 1, 0, 2, 0, 0, 0, 1, 4090, 4074, 3154, 2730, 2195, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 14, 0, 0, 0, 0, 3734, 0, 0, 3167, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 8, 4096, 0, 0, 4058, 173, 0, 0, 1, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 1, 4095, 0, 11, 4093, 1, 4, 6, 4095, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16867, 16327, 16429, 15653, 16417, 16469, 16483, 16427] avg=16384.0000 max=16867 max_rank=0 min=15653 min_rank=3 imbalance_ratio=1.029480 max_load_reduction=114205 max_load_reduction_pct=0.871315 moved_to_echo=65796 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2, 4091, 2, 3, 4094, 4094, 1, 33, 347, 5, 0, 3, 3919, 0, 312, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 941, 3, 0, 5, 1, 4083, 2144, 4086, 1, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 5, 7, 26, 2, 4093, 3, 4, 3674, 4085, 33, 0, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4091, 3, 1898, 4092, 0, 7, 4096, 2348, 1, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 2, 0, 3, 0, 0, 0, 0, 4, 0, 4088, 4071, 3150, 2732, 2194, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4088, 4, 1, 4086, 0, 0, 0, 2, 6, 3569, 0, 0, 3707, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 3, 4087, 2, 1, 4076, 25, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 0, 4096, 0, 53, 4095, 0, 0, 11, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16947, 15359, 16073, 16689, 16356, 16830, 16344, 16474] avg=16384.0000 max=16947 max_rank=0 min=15359 min_rank=1 imbalance_ratio=1.034363 max_load_reduction=114125 max_load_reduction_pct=0.870705 moved_to_echo=66004 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2, 4093, 0, 1, 4094, 4096, 2, 1, 124, 1, 0, 0, 3422, 0, 312, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 942, 10, 0, 1, 1, 4095, 3437, 4094, 0, 0, 0, 2, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 8, 1, 0, 0, 4093, 0, 1, 3675, 4095, 46, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 101, 1900, 4093, 0, 0, 4093, 1528, 2, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 2, 0, 0, 4096, 4073, 3153, 2731, 2195, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4096, 0, 1, 4092, 0, 3, 0, 0, 0, 3580, 0, 0, 3734, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 23, 4095, 0, 2, 4065, 18, 0, 7, 0, 1, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4093, 0, 4096, 0, 0, 4094, 0, 0, 0, 4086, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16189, 16679, 16385, 15832, 16367, 16871, 16358, 16391] avg=16384.0000 max=16871 max_rank=5 min=15832 min_rank=3 imbalance_ratio=1.029724 max_load_reduction=114201 max_load_reduction_pct=0.871284 moved_to_echo=65987 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 4095, 4095, 0, 15, 140, 0, 1, 0, 3176, 1, 312, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 942, 17, 1, 0, 1, 4095, 3265, 4095, 1, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 12, 0, 0, 7, 4096, 0, 0, 3676, 4067, 1, 0, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4091, 7, 1900, 4096, 0, 0, 4094, 2495, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 4096, 4075, 3154, 2732, 2196, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 1, 0, 4093, 2, 0, 1, 0, 1, 3669, 0, 0, 3380, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4055, 0, 4095, 0, 0, 4089, 26, 0, 2, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 0, 4095, 0, 9, 4095, 0, 0, 1, 4014, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15970, 16513, 16229, 16695, 16366, 16608, 16359, 16332] avg=16384.0000 max=16695 max_rank=3 min=15970 min_rank=0 imbalance_ratio=1.018982 max_load_reduction=114377 max_load_reduction_pct=0.872627 moved_to_echo=65665 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2, 4081, 0, 0, 4050, 4096, 1, 27, 4, 0, 0, 0, 1734, 0, 312, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 941, 0, 0, 6, 86, 4096, 3614, 4069, 0, 3, 0, 51, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1267, 9, 0, 0, 0, 4083, 0, 5, 3675, 3992, 14, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 4053, 4, 1898, 4071, 0, 5, 4077, 3429, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 63, 0, 5, 12, 0, 8, 0, 0, 0, 0, 0, 4096, 4072, 3149, 2732, 2192, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 3997, 120, 0, 0, 0, 0, 3159, 0, 5, 2965, 1364, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4053, 15, 4096, 0, 0, 4094, 96, 0, 63, 0, 1, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 0, 4093, 0, 1, 4081, 7, 53, 0, 3959, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14347, 16938, 17116, 17726, 16441, 15721, 16447, 16336] avg=16384.0000 max=17726 max_rank=3 min=14347 min_rank=0 imbalance_ratio=1.081909 max_load_reduction=113346 max_load_reduction_pct=0.864761 moved_to_echo=64945 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 32512, 128, 32768, 128, 128, 32640, 512, 0, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 3200, 384, 128, 128, 128, 32768, 128, 128, 29440, 32640, 256, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 32768, 128, 32768, 128, 128, 32768, 128, 128, 128, 32384, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 7552, 128, 128, 128, 128, 32768, 25216, 32768, 128, 128, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 256, 15232, 32768, 0, 128, 32768, 16256, 128, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 32768, 128, 128, 32640, 256, 128, 128, 128, 128, 28672, 128, 128, 25984, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 32768, 32768, 128, 128, 2304, 128, 128, 128, 27776, 128, 2560, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=133248 observed_loss=-2176 tokens_per_expert=[128, 128, 128, 128, 128, 256, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 32640, 25216, 21888, 17664, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=133248 slack_slots=128896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4091, 4096, 3761, 4096, 9, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 4091, 0, 4095, 0, 0, 948, 0, 0, 0, 18, 4096, 0, 0, 4092, 0, 0, 0, 4096, 4096, 1575, 0, 4095, 0, 0, 0, 0, 4096, 3, 0, 310, 0, 0, 3, 0, 3739, 0, 0, 0, 0, 0, 0, 4072, 0, 4086, 0, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 54, 0, 0, 0, 0, 0, 4002, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4088, 0, 0, 0, 4091, 2, 0, 0, 0, 0, 9, 0, 4088, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2467, 0, 3642, 0, 0, 4, 4093, 4, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4096, 4094, 3892, 4096, 31, 0, 0, 0, 4093, 0, 4095, 4, 0, 0, 0, 3997, 0, 4060, 0, 0, 3622, 0, 1, 0, 2, 4096, 0, 0, 4095, 0, 0, 0, 4096, 4096, 1411, 7, 4002, 0, 103, 0, 0, 4084, 0, 0, 1077, 0, 0, 0, 0, 3076, 0, 0, 0, 0, 0, 0, 4094, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 4096, 137, 0, 0, 0, 0, 0, 4076, 4046, 0, 0, 4094, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4084, 0, 0, 0, 3907, 161, 0, 0, 0, 0, 18, 0, 4068, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 433, 0, 2963, 0, 0, 10, 4096, 3, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4065, 4093, 3918, 4095, 1, 0, 1, 1, 4089, 0, 4089, 2, 0, 0, 0, 4096, 1, 4092, 0, 0, 2617, 1, 40, 0, 20, 4095, 3, 0, 4096, 0, 2, 0, 4095, 4096, 135, 3, 4077, 2, 1, 2, 1, 4094, 21, 1, 615, 1, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 2, 4092, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4095, 3, 0, 1, 2, 0, 0, 4092, 4094, 0, 1, 4095, 0, 0, 3761, 1, 4095, 4095, 0, 1, 4095, 0, 0, 0, 4095, 2, 0, 0, 0, 1, 9, 0, 4087, 0, 0, 0, 0, 0, 4090, 1, 1, 0, 0, 0, 0, 0, 1273, 1, 4076, 0, 0, 0, 4093, 2, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4031, 4090, 4063, 4090, 0, 1, 3, 0, 4094, 3, 4091, 2, 2, 15, 0, 4093, 0, 4095, 1, 0, 2927, 2, 24, 3, 6, 4092, 0, 3, 4091, 6, 5, 6, 4092, 4090, 23, 0, 4074, 0, 0, 0, 1, 4092, 0, 5, 213, 3, 0, 0, 2, 3186, 4, 2, 0, 0, 0, 0, 4089, 1, 4095, 0, 0, 4, 3, 4, 0, 0, 0, 0, 3, 2, 3, 0, 4089, 5, 0, 0, 5, 0, 0, 4085, 4092, 2, 4, 4089, 1, 0, 4066, 5, 4092, 4092, 0, 0, 4095, 5, 0, 0, 4062, 3, 0, 4, 1, 0, 0, 0, 4094, 4, 0, 0, 4, 0, 4065, 4, 4, 4, 0, 0, 3, 2, 2301, 4, 3773, 0, 2, 0, 4091, 0, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4089, 4090, 3822, 4091, 0, 1, 3, 0, 4055, 5, 4083, 8, 1, 0, 0, 4091, 0, 4095, 0, 5, 2800, 0, 1, 1, 32, 4088, 0, 0, 4093, 1, 0, 0, 4094, 4092, 2190, 14, 4064, 0, 53, 1, 5, 4082, 0, 0, 285, 3, 5, 1, 0, 2651, 4, 0, 0, 6, 0, 13, 4024, 2, 4083, 0, 8, 0, 4, 0, 0, 0, 0, 6, 0, 0, 0, 7, 4090, 1, 0, 0, 1, 0, 3, 4062, 4077, 0, 0, 4094, 1, 0, 4093, 0, 4093, 4096, 2, 0, 4095, 8, 0, 0, 4085, 11, 11, 0, 0, 0, 25, 7, 4090, 0, 0, 14, 2, 2, 4081, 0, 0, 3, 0, 1, 0, 0, 2647, 3, 2032, 0, 4, 1, 4091, 0, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[4090, 4090, 4064, 4094, 6, 1, 0, 0, 4095, 0, 4095, 4, 1, 0, 0, 4089, 2, 4089, 0, 0, 1465, 0, 0, 1, 3, 4094, 0, 0, 4093, 1, 0, 0, 4091, 4096, 10, 3, 4093, 1, 0, 0, 0, 4091, 19, 0, 175, 1, 1, 2, 0, 3964, 0, 0, 1, 4, 0, 2, 4092, 0, 4088, 15, 1, 1, 6, 2, 0, 0, 7, 2, 0, 0, 0, 0, 4087, 0, 0, 0, 4, 1, 3, 4092, 4092, 0, 1, 4096, 0, 2, 4085, 0, 4086, 4095, 0, 0, 4091, 0, 0, 0, 4076, 0, 0, 0, 0, 1, 11, 0, 4092, 2, 0, 8, 0, 0, 4081, 0, 0, 1, 5, 0, 1, 0, 2662, 1, 4050, 0, 8, 2, 4095, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4083, 4094, 4072, 4095, 0, 2, 1, 0, 4093, 1, 4095, 0, 0, 0, 2, 4095, 0, 4094, 0, 0, 1774, 1, 5, 0, 25, 4096, 2, 0, 4094, 0, 0, 0, 4094, 4095, 103, 1, 4090, 0, 3, 2, 0, 4096, 1, 0, 520, 1, 0, 0, 0, 3783, 0, 2, 0, 1, 0, 0, 4092, 0, 4093, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 4094, 35, 0, 0, 0, 0, 0, 4095, 4094, 0, 1, 4094, 0, 0, 3916, 0, 4096, 4096, 0, 2, 4094, 0, 2, 0, 4088, 0, 2, 0, 3, 1, 0, 0, 4095, 0, 1, 0, 0, 2, 4088, 2, 0, 1, 0, 0, 0, 0, 2772, 0, 3500, 1, 0, 0, 4095, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4077, 4095, 3353, 4096, 2, 0, 0, 0, 4096, 0, 4068, 1, 0, 0, 0, 4085, 0, 4094, 0, 0, 2246, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 1, 0, 4096, 4096, 174, 0, 4075, 0, 0, 0, 0, 4095, 0, 0, 668, 0, 0, 0, 0, 3280, 0, 0, 1, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4096, 11, 0, 0, 0, 1, 0, 4088, 4095, 0, 0, 4096, 0, 0, 4083, 0, 4095, 4096, 0, 0, 4095, 1, 0, 0, 4084, 2, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 1, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 2712, 0, 4072, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4089, 0, 3822, 0, 0, 1, 3, 0, 4055, 5, 257, 8, 1, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 5, 2800, 0, 1, 1, 32, 4088, 0, 0, 4093, 1, 0, 0, 1186, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2906, 2190, 14, 4064, 0, 53, 1, 5, 4082, 0, 0, 285, 3, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2651, 4, 0, 0, 6, 0, 13, 4024, 2, 4083, 0, 8, 0, 4, 0, 4090, 631, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 7, 4090, 1, 0, 0, 1, 0, 3, 4062, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 84, 1, 0, 4093, 0, 4093, 0, 2, 0, 4095, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 11, 11, 0, 0, 0, 25, 7, 4090, 0, 0, 14, 2, 2, 4081, 0, 3379, 728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 2647, 3, 2032, 0, 4, 1, 4091, 0, 0, 4094, 2508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16332, 16892, 17703, 15528, 16345, 16453, 16435, 15384] avg=16384.0000 max=17703 max_rank=2 min=15384 min_rank=7 imbalance_ratio=1.080505 max_load_reduction=113369 max_load_reduction_pct=0.864937 moved_to_echo=64617 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4065, 0, 3918, 0, 1, 0, 1, 1, 4089, 0, 257, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 0, 2617, 1, 40, 0, 20, 4095, 3, 0, 4096, 0, 2, 0, 1187, 591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2909, 135, 3, 4077, 2, 1, 2, 1, 4094, 21, 1, 615, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 2, 4092, 0, 4090, 0, 0, 0, 0, 0, 4093, 631, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4095, 3, 0, 1, 2, 0, 0, 4092, 4095, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 85, 0, 0, 3761, 1, 4095, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 1, 9, 0, 4087, 0, 0, 0, 0, 0, 4090, 1, 3379, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1273, 1, 4076, 0, 0, 0, 4093, 2, 0, 4095, 2512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16430, 16745, 15957, 16986, 16375, 16133, 16393, 16053] avg=16384.0000 max=16986 max_rank=3 min=15957 min_rank=2 imbalance_ratio=1.036743 max_load_reduction=114086 max_load_reduction_pct=0.870407 moved_to_echo=64954 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4031, 0, 4063, 0, 0, 1, 3, 0, 4094, 3, 258, 2, 2, 15, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 2927, 2, 24, 3, 6, 4092, 0, 3, 4091, 6, 5, 6, 1185, 591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2905, 23, 0, 4074, 0, 0, 0, 1, 4092, 0, 5, 213, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3186, 4, 2, 0, 0, 0, 0, 4089, 1, 4095, 0, 0, 4, 3, 4, 4090, 631, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 3, 0, 4089, 5, 0, 0, 5, 0, 0, 4085, 4092, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 4, 84, 1, 0, 4066, 5, 4092, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 3, 0, 4, 1, 0, 0, 0, 4094, 4, 0, 0, 4, 0, 4065, 4, 3374, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 3, 2, 2301, 4, 3773, 0, 2, 0, 4091, 0, 0, 4090, 2513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16565, 17037, 15408, 16124, 16361, 16446, 16344, 16787] avg=16384.0000 max=17037 max_rank=1 min=15408 min_rank=2 imbalance_ratio=1.039856 max_load_reduction=114035 max_load_reduction_pct=0.870018 moved_to_echo=65938 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 3761, 0, 9, 0, 0, 0, 4092, 0, 245, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 948, 0, 0, 0, 18, 4096, 0, 0, 4092, 0, 0, 0, 1191, 596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2905, 1575, 0, 4095, 0, 0, 0, 0, 4096, 3, 0, 310, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3739, 0, 0, 0, 0, 0, 0, 4072, 0, 4086, 0, 0, 0, 3, 0, 4096, 636, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 54, 0, 0, 0, 0, 0, 4002, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 76, 0, 0, 4096, 0, 4096, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 0, 0, 0, 0, 9, 0, 4088, 0, 0, 3, 0, 0, 4096, 0, 3384, 735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2467, 0, 3642, 0, 0, 4, 4093, 4, 0, 4096, 2520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16289, 15036, 17083, 16645, 16333, 16452, 16408, 16826] avg=16384.0000 max=17083 max_rank=2 min=15036 min_rank=1 imbalance_ratio=1.042664 max_load_reduction=113989 max_load_reduction_pct=0.869667 moved_to_echo=66019 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 4064, 0, 6, 1, 0, 0, 4095, 0, 258, 4, 1, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4089, 0, 0, 1465, 0, 0, 1, 3, 4094, 0, 0, 4093, 1, 0, 0, 1187, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2909, 10, 3, 4093, 1, 0, 0, 0, 4091, 19, 0, 175, 1, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 0, 0, 1, 4, 0, 2, 4092, 0, 4088, 15, 1, 1, 6, 2, 4090, 632, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2, 0, 0, 0, 0, 4087, 0, 0, 0, 4, 1, 3, 4092, 4095, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 84, 0, 2, 4085, 0, 4086, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 1, 11, 0, 4092, 2, 0, 8, 0, 0, 4081, 0, 3380, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 1, 0, 2662, 1, 4050, 0, 8, 2, 4095, 0, 0, 4096, 2515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16608, 15527, 15396, 16911, 16372, 16441, 16381, 17436] avg=16384.0000 max=17436 max_rank=7 min=15396 min_rank=2 imbalance_ratio=1.064209 max_load_reduction=113636 max_load_reduction_pct=0.866974 moved_to_echo=66630 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 3892, 0, 31, 0, 0, 0, 4093, 0, 258, 4, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 3622, 0, 1, 0, 2, 4096, 0, 0, 4095, 0, 0, 0, 1187, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2909, 1411, 7, 4002, 0, 103, 0, 0, 4084, 0, 0, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3076, 0, 0, 0, 0, 0, 0, 4094, 0, 4051, 0, 0, 0, 0, 0, 4094, 631, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 4096, 137, 0, 0, 0, 0, 0, 4076, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 84, 0, 0, 4095, 0, 4096, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 161, 0, 0, 0, 0, 18, 0, 4068, 0, 0, 1, 0, 0, 4096, 0, 3379, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 433, 0, 2963, 0, 0, 10, 4096, 3, 0, 4096, 2515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16371, 17655, 17689, 15959, 16502, 16405, 16360, 14131] avg=16384.0000 max=17689 max_rank=2 min=14131 min_rank=7 imbalance_ratio=1.079651 max_load_reduction=113383 max_load_reduction_pct=0.865044 moved_to_echo=63398 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4077, 0, 3353, 0, 2, 0, 0, 0, 4096, 0, 256, 1, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2246, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 1, 0, 1187, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2909, 174, 0, 4075, 0, 0, 0, 0, 4095, 0, 0, 668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3280, 0, 0, 1, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 4095, 632, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4096, 11, 0, 0, 0, 1, 0, 4088, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 84, 0, 0, 4083, 0, 4095, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 2, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 1, 0, 0, 4072, 0, 3380, 725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2712, 0, 4072, 0, 0, 0, 4096, 0, 0, 4096, 2499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15870, 16310, 16017, 16210, 16377, 16453, 16360, 17475] avg=16384.0000 max=17475 max_rank=7 min=15870 min_rank=0 imbalance_ratio=1.066589 max_load_reduction=113597 max_load_reduction_pct=0.866676 moved_to_echo=66665 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4083, 0, 4072, 0, 0, 2, 1, 0, 4093, 1, 258, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1774, 1, 5, 0, 25, 4096, 2, 0, 4094, 0, 0, 0, 1187, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2908, 103, 1, 4090, 0, 3, 2, 0, 4096, 1, 0, 520, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3783, 0, 2, 0, 1, 0, 0, 4092, 0, 4093, 0, 0, 1, 0, 0, 4094, 631, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 4094, 35, 0, 0, 0, 0, 0, 4095, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 84, 0, 0, 3916, 0, 4096, 0, 0, 2, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 2, 0, 3, 1, 0, 0, 4095, 0, 1, 0, 0, 2, 4088, 2, 3379, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2772, 0, 3500, 1, 0, 0, 4095, 0, 0, 4096, 2515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16607, 15870, 15819, 16709, 16407, 16289, 16391, 16980] avg=16384.0000 max=16980 max_rank=7 min=15819 min_rank=2 imbalance_ratio=1.036377 max_load_reduction=114092 max_load_reduction_pct=0.870453 moved_to_echo=66067 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32640, 0, 30976, 0, 128, 128, 128, 128, 32768, 128, 2048, 128, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[32512, 256, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 128, 128, 32768, 128, 27136, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 768, 128, 128, 32256, 128, 32768, 0, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 27776, 128, 128, 128, 128, 0, 128, 32768, 128, 32768, 128, 128, 128, 128, 128, 32768, 5120, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 128, 128, 18432, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 128, 128, 9600, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 23296, 5632, 128, 32640, 128, 256, 128, 128, 32768, 128, 128, 3968, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 17280, 128, 28160, 128, 128, 128, 32768, 128, 0, 32768, 20224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 128, 32768, 256, 0, 128, 128, 128, 128, 32640, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4087, 0, 4092, 6, 3784, 0, 0, 1027, 0, 4092, 4092, 0, 4096, 0, 0, 2, 0, 0, 1, 1, 0, 0, 0, 0, 1, 14, 0, 4, 0, 0, 0, 4095, 3, 0, 4095, 1, 4091, 0, 0, 0, 0, 0, 18, 5, 0, 4096, 4, 0, 4096, 4096, 0, 0, 1884, 0, 4093, 4095, 25, 3896, 0, 0, 4096, 0, 0, 4091, 4092, 0, 4075, 0, 0, 0, 4, 0, 0, 4096, 4092, 4093, 0, 1, 3998, 0, 0, 7, 4093, 0, 4096, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2152, 2, 0, 0, 1, 91, 0, 0, 0, 4096, 0, 4095, 0, 0, 4068, 4096, 3641] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 6, 4, 1, 4094, 0, 4092, 1, 4094, 0, 0, 568, 0, 4079, 4080, 0, 4092, 0, 2, 2, 1, 0, 0, 1, 0, 4, 0, 0, 0, 1, 0, 0, 4, 3, 4, 4092, 3, 20, 4092, 1, 4089, 0, 0, 1, 3, 6, 41, 1, 0, 4093, 18, 6, 4092, 4090, 3, 2, 1485, 0, 4093, 4087, 58, 3830, 0, 0, 4093, 1, 0, 3835, 4091, 0, 4067, 1, 1, 0, 3, 0, 2, 4096, 4084, 4092, 1, 0, 3717, 0, 2, 12, 4094, 0, 4093, 1, 0, 9, 0, 1, 0, 0, 2, 0, 0, 3, 4090, 0, 0, 0, 4, 1, 3, 0, 2778, 3, 0, 0, 1, 229, 0, 0, 3, 4089, 1, 4090, 0, 0, 4083, 4096, 3959] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 1, 1, 2, 0, 0, 4093, 0, 4095, 1, 2271, 0, 1, 2858, 1, 4094, 4094, 0, 4094, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4, 0, 0, 0, 0, 0, 4095, 0, 5, 4095, 5, 4094, 1, 7, 10, 1, 1, 2, 0, 0, 4095, 10, 0, 4094, 4091, 1, 1, 2153, 0, 4094, 4095, 12, 3781, 0, 0, 4095, 0, 0, 4079, 4093, 0, 4095, 1, 1, 0, 0, 0, 0, 4091, 4095, 4096, 1, 0, 3634, 1, 0, 0, 4095, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 2, 0, 2, 1, 1910, 2, 1, 2, 0, 33, 0, 1, 1, 4094, 0, 4095, 0, 0, 4080, 4095, 3824] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 1, 1, 0, 1, 0, 4089, 0, 4095, 0, 2056, 0, 0, 1656, 0, 4096, 4095, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 6, 4095, 0, 12, 9, 0, 0, 0, 0, 1, 4091, 25, 0, 4095, 4088, 1, 1, 3408, 3, 4095, 4095, 23, 3001, 0, 1, 4095, 0, 1, 4093, 4093, 0, 4070, 1, 1, 0, 0, 0, 0, 4095, 4095, 4095, 0, 0, 3226, 1, 1, 0, 4095, 0, 4095, 0, 0, 0, 1, 1, 2, 0, 0, 1, 0, 0, 4095, 1, 1, 5, 0, 1, 0, 0, 3049, 0, 0, 0, 0, 4, 0, 0, 1, 4095, 1, 4095, 1, 0, 4096, 4095, 4039] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 4, 4023, 0, 0, 823, 0, 4084, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 11, 0, 4078, 18, 0, 4096, 4096, 0, 0, 1573, 4, 4096, 4096, 21, 3613, 0, 0, 4096, 0, 0, 4066, 4093, 0, 4086, 0, 0, 0, 8, 0, 0, 4096, 4095, 4095, 0, 0, 4018, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2744, 2, 0, 0, 0, 44, 0, 0, 0, 4096, 0, 4096, 0, 0, 4084, 4096, 3663] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[4, 0, 5, 5, 0, 3, 0, 4092, 0, 4092, 0, 3790, 0, 6, 617, 4, 4092, 4091, 4, 4089, 5, 7, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 6, 5, 0, 4092, 0, 0, 4085, 0, 4091, 0, 21, 0, 1, 0, 122, 0, 4, 4091, 1, 0, 4091, 4091, 5, 5, 2343, 0, 4091, 4092, 40, 3942, 0, 0, 4089, 3, 4, 4090, 4090, 1, 4066, 4, 5, 0, 0, 5, 0, 4085, 4091, 4091, 2, 0, 3987, 7, 0, 2, 4090, 7, 4090, 0, 3, 0, 4, 7, 12, 0, 0, 7, 0, 5, 4091, 1, 0, 0, 0, 4, 5, 0, 1570, 0, 5, 0, 3, 31, 0, 1, 4, 4091, 3, 4091, 2, 0, 4092, 4091, 4010] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4094, 0, 4095, 1, 3843, 0, 0, 2572, 0, 4090, 4095, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 0, 26, 0, 0, 4093, 4, 0, 4096, 4095, 0, 0, 1530, 1, 4096, 4096, 20, 3851, 0, 0, 4096, 0, 0, 4093, 4095, 0, 4092, 0, 0, 0, 2, 0, 0, 4095, 4096, 4095, 0, 0, 3609, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 906, 0, 0, 0, 0, 178, 0, 0, 0, 4096, 0, 4096, 0, 0, 4084, 4096, 3967] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 21, 4058, 0, 4096, 26, 4062, 0, 0, 1274, 0, 4024, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 48, 4096, 0, 0, 44, 0, 0, 30, 0, 0, 4033, 155, 0, 4096, 4096, 0, 0, 933, 24, 4096, 4096, 47, 3823, 0, 0, 4096, 0, 0, 4095, 4071, 0, 4059, 0, 0, 0, 32, 0, 0, 4096, 4095, 4096, 0, 0, 4008, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2108, 0, 0, 0, 0, 300, 0, 0, 0, 4094, 0, 4094, 0, 0, 4068, 4096, 3802] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 6, 4, 1, 4094, 0, 4092, 1, 4094, 0, 0, 568, 0, 1994, 1277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4080, 0, 4092, 0, 2, 2, 1, 0, 0, 1, 0, 4, 0, 0, 0, 2086, 2024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 3, 4, 4092, 3, 20, 4092, 1, 4089, 0, 0, 1, 3, 2379, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 41, 1, 0, 4093, 18, 6, 0, 4090, 3, 2, 1485, 0, 2099, 4087, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3830, 0, 0, 437, 1, 0, 3835, 4091, 0, 4067, 1, 1, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 4092, 1, 0, 3717, 0, 2, 12, 320, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 3, 4090, 0, 0, 0, 4, 1, 3, 0, 2778, 4093, 4092, 1939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 229, 0, 0, 3, 4089, 1, 4090, 0, 0, 4083, 133, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16133, 16371, 16380, 15989, 16268, 16334, 17006, 16591] avg=16384.0000 max=17006 max_rank=6 min=15989 min_rank=3 imbalance_ratio=1.037964 max_load_reduction=114066 max_load_reduction_pct=0.870255 moved_to_echo=66199 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4087, 0, 4092, 6, 3784, 0, 0, 1027, 0, 1994, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4092, 0, 4096, 0, 0, 2, 0, 0, 1, 1, 0, 0, 0, 0, 1, 2085, 2024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4, 0, 0, 0, 4095, 3, 0, 4095, 1, 4091, 0, 0, 0, 0, 2381, 1687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 5, 0, 4096, 4, 0, 0, 4096, 0, 0, 1884, 0, 2099, 4095, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 0, 0, 437, 0, 0, 4091, 4092, 0, 4075, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 4093, 0, 1, 3998, 0, 0, 7, 321, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2152, 4096, 4096, 1939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 91, 0, 0, 0, 4096, 0, 4095, 0, 0, 4068, 133, 3641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16269, 16394, 16371, 16322, 16595, 16611, 16383, 16127] avg=16384.0000 max=16611 max_rank=5 min=16127 min_rank=7 imbalance_ratio=1.013855 max_load_reduction=114461 max_load_reduction_pct=0.873268 moved_to_echo=65716 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 4, 4023, 0, 0, 823, 0, 1999, 1284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2090, 2027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 2385, 1692, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4078, 18, 0, 0, 4096, 0, 0, 1573, 4, 2097, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3613, 0, 0, 427, 0, 0, 4066, 4093, 0, 4086, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4095, 0, 0, 4018, 0, 0, 0, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2744, 4096, 4096, 1942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 44, 0, 0, 0, 4096, 0, 4096, 0, 0, 4084, 127, 3663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16320, 16392, 16369, 15994, 16293, 16618, 16974, 16112] avg=16384.0000 max=16974 max_rank=6 min=15994 min_rank=3 imbalance_ratio=1.036011 max_load_reduction=114098 max_load_reduction_pct=0.870499 moved_to_echo=65997 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 1, 0, 1, 0, 4089, 0, 4095, 0, 2056, 0, 0, 1656, 0, 1995, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2086, 2024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 6, 4095, 0, 12, 9, 0, 2381, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 25, 0, 0, 4088, 1, 1, 3408, 3, 2100, 4095, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3001, 0, 1, 436, 0, 1, 4093, 4093, 0, 4070, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 4095, 0, 0, 3226, 1, 1, 0, 321, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 1, 0, 0, 4095, 1, 1, 5, 0, 1, 0, 0, 3049, 4095, 4095, 1938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 4095, 1, 4095, 1, 0, 4096, 133, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15174, 16400, 16381, 17836, 15697, 15835, 17284, 16465] avg=16384.0000 max=17836 max_rank=3 min=15174 min_rank=0 imbalance_ratio=1.088623 max_load_reduction=113236 max_load_reduction_pct=0.863922 moved_to_echo=65281 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 5, 5, 0, 3, 0, 4092, 0, 4092, 0, 3790, 0, 6, 617, 4, 1993, 1276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4091, 4, 4089, 5, 7, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 2084, 2022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 5, 0, 4092, 0, 0, 4085, 0, 4091, 0, 21, 0, 1, 2377, 1686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 4, 4091, 1, 0, 0, 4091, 5, 5, 2343, 0, 2098, 4092, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3942, 0, 0, 436, 3, 4, 4090, 4090, 1, 4066, 4, 5, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4091, 4091, 2, 0, 3987, 7, 0, 2, 320, 7, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 12, 0, 0, 7, 0, 5, 4091, 1, 0, 0, 0, 4, 5, 0, 1570, 4090, 4091, 1936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3, 31, 0, 1, 4, 4091, 3, 4091, 2, 0, 4092, 133, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15887, 16399, 16364, 16892, 16646, 16599, 15819, 16466] avg=16384.0000 max=16892 max_rank=3 min=15819 min_rank=6 imbalance_ratio=1.031006 max_load_reduction=114180 max_load_reduction_pct=0.871124 moved_to_echo=65530 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 1, 2, 0, 0, 4093, 0, 4095, 1, 2271, 0, 1, 2858, 1, 1995, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 4094, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2086, 2024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4095, 0, 5, 4095, 5, 4094, 1, 7, 10, 1, 2381, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 4095, 10, 0, 0, 4091, 1, 1, 2153, 0, 2099, 4095, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3781, 0, 0, 436, 0, 0, 4079, 4093, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 4096, 1, 0, 3634, 1, 0, 0, 321, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 2, 0, 2, 1, 1910, 4095, 4094, 1938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 2, 0, 33, 0, 1, 1, 4094, 0, 4095, 0, 0, 4080, 133, 3824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16599, 16396, 16386, 16560, 16486, 16241, 16138, 16266] avg=16384.0000 max=16599 max_rank=0 min=16138 min_rank=6 imbalance_ratio=1.013123 max_load_reduction=114473 max_load_reduction_pct=0.873360 moved_to_echo=65131 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4094, 0, 4095, 1, 3843, 0, 0, 2572, 0, 1996, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4095, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2087, 2024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 2381, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 4093, 4, 0, 0, 4095, 0, 0, 1530, 1, 2100, 4096, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 0, 0, 437, 0, 0, 4093, 4095, 0, 4092, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4095, 0, 0, 3609, 0, 0, 2, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 906, 4096, 4096, 1939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 4096, 0, 4096, 0, 0, 4084, 133, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17879, 16393, 16361, 15965, 16570, 16217, 15133, 16554] avg=16384.0000 max=17879 max_rank=0 min=15133 min_rank=6 imbalance_ratio=1.091248 max_load_reduction=113193 max_load_reduction_pct=0.863594 moved_to_echo=64474 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 21, 4058, 0, 4096, 26, 4062, 0, 0, 1274, 0, 1996, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2087, 2024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 48, 4096, 0, 0, 44, 0, 2381, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 4033, 155, 0, 0, 4096, 0, 0, 933, 24, 2100, 4096, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3823, 0, 0, 437, 0, 0, 4095, 4071, 0, 4059, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 4008, 0, 0, 2, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2108, 4096, 4096, 1939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 300, 0, 0, 0, 4094, 0, 4094, 0, 0, 4068, 133, 3802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16811, 16327, 16460, 15514, 16517, 16617, 16335, 16491] avg=16384.0000 max=16811 max_rank=0 min=15514 min_rank=3 imbalance_ratio=1.026062 max_load_reduction=114261 max_load_reduction_pct=0.871742 moved_to_echo=65960 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 32768, 0, 32768, 128, 28032, 0, 128, 11520, 128, 16000, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 256, 128, 128, 32768, 256, 128, 0, 32768, 128, 128, 15360, 128, 16896, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 1024, 0, 128, 128, 32768, 128, 32768, 128, 0, 32768, 1152, 30976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 32768, 128, 128, 128, 128, 19072, 13568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 32768, 128, 32768, 128, 128, 128, 128, 0, 128, 128, 0, 128, 0, 128, 128, 16768, 16256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[29824, 0, 128, 3584, 128, 128, 32512, 32768, 128, 32640, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 32768, 128, 128, 128, 128, 128, 128, 128, 17280, 32768, 32768, 15616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 32768, 32768, 128, 128, 30208, 128, 128, 128, 2560, 128, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 3, 8, 0, 4095, 3, 4, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4091, 4, 4089, 0, 4095, 4096, 0, 0, 4095, 0, 2, 0, 0, 0, 4093, 4087, 2, 0, 204, 0, 4094, 0, 4095, 4, 4094, 0, 0, 4076, 7, 4096, 0, 0, 0, 0, 0, 0, 4077, 4090, 0, 1, 4, 4094, 4093, 0, 4090, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 1, 0, 4092, 0, 3, 4094, 1, 0, 4093, 0, 2, 0, 4096, 4087, 0, 0, 0, 0, 1, 27, 0, 1, 0, 0, 3889, 0, 4096, 4094, 16, 0, 0, 1, 4096, 4096, 3, 1, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 0, 1, 0, 4095, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 4094, 4087, 0, 4095, 1, 4095, 4094, 0, 0, 4093, 0, 0, 2, 0, 1, 4095, 4092, 0, 0, 264, 3, 4091, 1, 4094, 5, 4095, 0, 0, 4092, 9, 4095, 0, 0, 0, 0, 0, 0, 4063, 4095, 1, 1, 0, 4095, 4095, 0, 4095, 0, 1, 1, 1, 0, 1, 0, 2, 1, 1, 0, 1, 4091, 0, 0, 4096, 1, 0, 4095, 0, 0, 1, 4095, 4095, 1, 1, 0, 0, 0, 1, 1, 1, 4, 1, 3782, 0, 4095, 4095, 86, 0, 0, 1, 4095, 4094, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 4089, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 811, 0, 4067, 0, 4096, 7, 4096, 0, 4, 4096, 11, 4096, 0, 0, 0, 0, 0, 0, 4077, 4096, 0, 0, 0, 4090, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 25, 0, 5, 0, 0, 3285, 0, 4096, 4096, 17, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4085, 0, 4095, 0, 4096, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 56, 0, 4058, 0, 4096, 5, 4096, 0, 0, 4096, 43, 4096, 0, 0, 0, 0, 1, 0, 4013, 4096, 0, 0, 0, 4090, 4089, 7, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 3, 11, 4093, 0, 0, 4096, 0, 0, 1, 4096, 4093, 0, 0, 0, 0, 0, 30, 0, 6, 0, 0, 3967, 0, 4096, 4096, 128, 0, 0, 0, 4096, 4096, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1, 1, 0, 0, 4091, 1, 0, 4092, 0, 0, 0, 1, 1, 1, 0, 0, 2, 0, 1, 1, 4094, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 4095, 4088, 0, 4095, 1, 4095, 4095, 0, 1, 4095, 0, 0, 1, 0, 1, 4095, 4095, 1, 0, 144, 0, 4033, 1, 4092, 0, 4095, 0, 42, 4095, 4, 4094, 0, 1, 0, 0, 0, 0, 4018, 4095, 0, 1, 0, 4090, 4094, 0, 4095, 0, 0, 1, 1, 0, 1, 0, 3, 0, 1, 4, 1, 4095, 0, 0, 4092, 1, 2, 4095, 0, 0, 0, 4094, 4095, 0, 1, 0, 1, 1, 0, 1, 8, 3, 1, 3933, 0, 4095, 4095, 118, 0, 1, 1, 4095, 4095, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 4, 12, 6, 5, 0, 0, 0, 4087, 5, 0, 4090, 2, 6, 0, 24, 0, 5, 0, 7, 0, 8, 5, 4, 4090, 2, 0, 3, 0, 4, 4090, 4, 0, 0, 1, 4091, 4072, 0, 4090, 5, 4089, 4091, 0, 0, 4091, 1, 0, 0, 3, 6, 4090, 4091, 5, 0, 9, 5, 4090, 5, 4087, 0, 4090, 0, 18, 4064, 3, 4090, 34, 0, 5, 0, 0, 2, 4042, 4090, 0, 5, 0, 4091, 4091, 0, 4090, 0, 4, 5, 4, 0, 6, 0, 0, 0, 6, 0, 5, 4084, 0, 0, 4091, 5, 3, 4091, 0, 3, 0, 4086, 4091, 3, 5, 0, 6, 0, 5, 6, 0, 18, 4, 4076, 0, 4087, 4088, 5, 0, 0, 5, 4087, 4091, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 27, 0, 4087, 0, 4096, 1, 4096, 0, 0, 4096, 17, 4096, 0, 0, 0, 0, 0, 0, 4079, 4096, 0, 0, 0, 4095, 4096, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 1, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 3884, 0, 4096, 4096, 192, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4074, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 482, 0, 3951, 0, 4096, 153, 4096, 0, 5, 4096, 38, 4096, 0, 0, 0, 0, 0, 0, 4046, 4096, 0, 0, 0, 4091, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4096, 2, 0, 4069, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 66, 0, 49, 0, 0, 3614, 0, 4096, 4096, 40, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 3, 8, 0, 4095, 3, 4, 4096, 0, 0, 0, 0, 4087, 4082, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 4095, 4094, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4, 4089, 0, 4095, 14, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 2, 0, 204, 0, 4094, 0, 3860, 4, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 29, 0, 0, 0, 0, 0, 0, 4077, 4090, 0, 1, 4, 4094, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 1, 0, 4092, 0, 3, 4044, 3890, 235, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 4079, 0, 2, 0, 4096, 4087, 0, 0, 0, 0, 1, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3889, 0, 4096, 4094, 16, 0, 0, 1, 4096, 206, 3, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16403, 16391, 16390, 16327, 16395, 16372, 16387, 16407] avg=16384.0000 max=16407 max_rank=7 min=16327 min_rank=3 imbalance_ratio=1.001404 max_load_reduction=114665 max_load_reduction_pct=0.874825 moved_to_echo=65561 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 0, 0, 4090, 4082, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4095, 0, 4096, 14, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 56, 0, 4058, 0, 3861, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 29, 0, 0, 0, 0, 1, 0, 4013, 4096, 0, 0, 0, 4090, 4089, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 3, 11, 4044, 3890, 235, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4082, 0, 0, 1, 4096, 4093, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3967, 0, 4096, 4096, 128, 0, 0, 0, 4096, 206, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16388, 16379, 16167, 16368, 16398, 16395, 16597] avg=16384.0000 max=16597 max_rank=7 min=16167 min_rank=3 imbalance_ratio=1.013000 max_load_reduction=114475 max_load_reduction_pct=0.873375 moved_to_echo=65758 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1, 1, 0, 0, 4091, 1, 0, 4092, 0, 0, 0, 1, 4089, 4081, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 2, 0, 1, 1, 4094, 0, 0, 0, 0, 0, 4095, 0, 4095, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4088, 0, 4095, 1, 4095, 14, 0, 1, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 144, 0, 4033, 1, 3858, 0, 0, 0, 42, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 29, 0, 1, 0, 0, 0, 0, 4018, 4095, 0, 1, 0, 4090, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 1, 0, 1, 0, 3, 0, 1, 4, 1, 4095, 0, 0, 4042, 3889, 234, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 2, 4081, 0, 0, 0, 4094, 4095, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 1, 3933, 0, 4095, 4095, 118, 0, 1, 1, 4095, 206, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16391, 16391, 16270, 16332, 16381, 16369, 16557] avg=16384.0000 max=16557 max_rank=7 min=16270 min_rank=3 imbalance_ratio=1.010559 max_load_reduction=114515 max_load_reduction_pct=0.873680 moved_to_echo=65639 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 4091, 4086, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 0, 4096, 0, 4096, 10, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 811, 0, 4067, 0, 3859, 7, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 21, 0, 0, 0, 0, 0, 0, 4077, 4096, 0, 0, 0, 4090, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 4048, 3892, 237, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4077, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3285, 0, 4096, 4096, 17, 0, 0, 0, 4096, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 16385, 16388, 16940, 16389, 16394, 16385, 15799] avg=16384.0000 max=16940 max_rank=3 min=15799 min_rank=7 imbalance_ratio=1.033936 max_load_reduction=114132 max_load_reduction_pct=0.870758 moved_to_echo=64967 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 1, 0, 1, 0, 4095, 1, 0, 4095, 0, 1, 0, 0, 4089, 4080, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 1, 0, 0, 4095, 0, 4094, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4095, 1, 4095, 14, 0, 0, 4093, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 0, 0, 264, 3, 4091, 1, 3859, 5, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 29, 0, 0, 0, 0, 0, 0, 4063, 4095, 1, 1, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 0, 1, 0, 2, 1, 1, 0, 1, 4091, 0, 0, 4043, 3888, 235, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4081, 0, 0, 1, 4095, 4095, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 1, 3782, 0, 4095, 4095, 86, 0, 0, 1, 4095, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16388, 16387, 16408, 16388, 16375, 16373, 16366] avg=16384.0000 max=16408 max_rank=3 min=16366 min_rank=7 imbalance_ratio=1.001465 max_load_reduction=114664 max_load_reduction_pct=0.874817 moved_to_echo=65502 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 12, 6, 5, 0, 0, 0, 4087, 5, 0, 4090, 2, 6, 0, 24, 4084, 4077, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7, 0, 8, 5, 4, 4090, 2, 0, 3, 0, 4, 4090, 4, 4091, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4072, 0, 4090, 5, 4089, 14, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6, 0, 4091, 5, 0, 9, 5, 4090, 5, 3853, 0, 0, 0, 18, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 29, 34, 0, 5, 0, 0, 2, 4042, 4090, 0, 5, 0, 4091, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4, 5, 4, 0, 6, 0, 0, 0, 6, 0, 5, 4084, 0, 0, 4038, 3885, 234, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 5, 3, 4077, 0, 3, 0, 4086, 4091, 3, 5, 0, 6, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4, 4076, 0, 4087, 4088, 5, 0, 0, 5, 4087, 206, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16425, 16409, 16363, 16149, 16392, 16375, 16381, 16578] avg=16384.0000 max=16578 max_rank=7 min=16149 min_rank=3 imbalance_ratio=1.011841 max_load_reduction=114494 max_load_reduction_pct=0.873520 moved_to_echo=65726 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 0, 4093, 0, 1, 4096, 0, 0, 0, 0, 4090, 4081, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 27, 0, 4087, 0, 3861, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 29, 0, 0, 0, 0, 0, 0, 4079, 4096, 0, 0, 0, 4095, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 1, 0, 4044, 3890, 235, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4082, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3884, 0, 4096, 4096, 192, 0, 0, 0, 4096, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16386, 16398, 16167, 16413, 16379, 16373, 16571] avg=16384.0000 max=16571 max_rank=7 min=16167 min_rank=3 imbalance_ratio=1.011414 max_load_reduction=114501 max_load_reduction_pct=0.873573 moved_to_echo=65736 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 4096, 0, 0, 0, 0, 4090, 4082, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 4096, 0, 4096, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 482, 0, 3951, 0, 3861, 153, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 29, 0, 0, 0, 0, 0, 0, 4046, 4096, 0, 0, 0, 4091, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 4096, 2, 0, 4044, 3890, 235, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 4082, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 3614, 0, 4096, 4096, 40, 0, 0, 0, 4096, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16319, 16334, 16376, 16644, 16395, 16398, 16409, 16197] avg=16384.0000 max=16644 max_rank=3 min=16197 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=114428 max_load_reduction_pct=0.873016 moved_to_echo=65399 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 32768, 0, 128, 128, 32768, 32768, 128, 128, 0, 128, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 0, 32768, 128, 128, 32768, 128, 128, 0, 128, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 32768, 128, 128, 32384, 31232, 1920, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 30464, 0, 32768, 32768, 640, 0, 128, 128, 32768, 1664, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 0, 128, 32768, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 2048, 128, 32512, 128, 30976, 256, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[256, 256, 128, 128, 128, 0, 128, 128, 32512, 32768, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 32768, 128, 32768, 128, 0, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[510, 4096, 4082, 0, 0, 0, 0, 0, 289, 0, 5, 111, 0, 0, 1, 0, 0, 1493, 0, 3, 0, 2751, 0, 4092, 5, 0, 23, 0, 0, 4089, 0, 4096, 0, 3776, 4096, 0, 4096, 3, 7, 0, 0, 4096, 0, 4090, 0, 4096, 0, 4092, 4091, 4096, 2, 0, 0, 4096, 0, 3, 4011, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4084, 0, 12, 0, 0, 0, 4092, 0, 0, 0, 1, 0, 4083, 0, 14, 4095, 0, 0, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 4094, 0, 10, 3775, 0, 4096, 3, 0, 0, 2, 3, 1, 0, 0, 0, 0, 4096, 3742, 12, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 2, 0, 4084, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[577, 4095, 4094, 1, 1, 0, 0, 0, 29, 0, 0, 34, 0, 0, 0, 0, 0, 2424, 0, 0, 0, 2049, 0, 4095, 0, 1, 3, 0, 1, 4093, 1, 4095, 0, 3792, 4095, 0, 4095, 1, 0, 1, 0, 4094, 0, 4094, 0, 4095, 0, 4094, 4095, 4094, 0, 1, 0, 4095, 1, 0, 4045, 1, 1, 0, 1, 1, 2, 4094, 0, 1, 0, 4078, 1, 0, 1, 1, 2, 4095, 1, 1, 1, 2, 1, 4095, 1, 1, 4095, 0, 1, 0, 2, 0, 1, 4095, 4095, 1, 0, 1, 0, 4094, 4093, 1, 0, 4081, 0, 4095, 1, 0, 2, 4, 0, 0, 0, 0, 2, 0, 4095, 3449, 0, 1, 2, 1, 1, 1, 0, 0, 4095, 0, 1, 1, 4093, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[926, 4096, 4088, 0, 0, 0, 0, 0, 74, 0, 0, 55, 0, 0, 0, 0, 0, 2219, 0, 5, 0, 1543, 0, 4096, 4, 0, 9, 0, 0, 4096, 0, 4096, 0, 3733, 4096, 0, 4096, 4, 4, 0, 0, 4096, 22, 4092, 0, 4096, 0, 4095, 4095, 4096, 6, 0, 0, 4096, 0, 0, 3908, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 0, 4086, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 1, 4057, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3928, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4092, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[553, 4096, 4089, 0, 0, 0, 0, 0, 10, 0, 0, 65, 0, 0, 0, 0, 0, 1078, 0, 12, 0, 3466, 0, 4096, 0, 0, 7, 0, 0, 4096, 0, 4096, 0, 3500, 4096, 0, 4096, 4, 5, 0, 0, 4096, 0, 4094, 0, 4096, 0, 4096, 4096, 4096, 4, 0, 0, 4096, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4081, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 19, 0, 4095, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4081, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3624, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4089, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[2326, 4095, 4092, 1, 1, 0, 0, 3, 25, 0, 0, 267, 0, 0, 0, 0, 0, 2024, 1, 0, 2, 1041, 0, 4095, 1, 1, 0, 0, 1, 4095, 0, 4095, 5, 3425, 4095, 0, 4095, 1, 0, 1, 0, 4095, 0, 4095, 0, 4095, 0, 4095, 4095, 4092, 1, 0, 0, 4093, 1, 0, 3916, 1, 1, 0, 1, 1, 0, 4091, 0, 0, 0, 4094, 1, 10, 1, 1, 0, 4095, 1, 1, 1, 0, 1, 4095, 1, 1, 4095, 0, 0, 0, 5, 0, 0, 4095, 4095, 1, 0, 1, 0, 4095, 4095, 1, 0, 4071, 1, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 4095, 3351, 0, 1, 0, 1, 12, 1, 0, 0, 4095, 0, 4, 0, 4092, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[711, 4096, 4095, 0, 0, 0, 0, 0, 44, 0, 0, 19, 0, 0, 0, 0, 0, 3110, 0, 0, 0, 1898, 0, 4096, 0, 0, 4, 0, 0, 4096, 0, 4096, 0, 3937, 4096, 0, 4096, 1, 1, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 3, 0, 0, 4096, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4083, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2638, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[112, 4089, 4091, 6, 7, 4, 14, 4, 40, 6, 0, 0, 0, 0, 0, 0, 1, 1778, 5, 0, 0, 2518, 0, 4089, 0, 5, 0, 0, 5, 4090, 6, 4090, 1, 3846, 4085, 4, 4089, 0, 0, 6, 7, 4089, 0, 4090, 1, 4088, 0, 4090, 4090, 4090, 0, 0, 0, 4081, 5, 0, 4076, 6, 7, 0, 5, 5, 0, 4088, 0, 0, 0, 4092, 6, 0, 4, 5, 2, 4088, 0, 4, 3, 0, 7, 4091, 4, 0, 4088, 3, 0, 0, 5, 0, 0, 4090, 4090, 6, 0, 6, 0, 4090, 4090, 6, 0, 4051, 3, 4089, 0, 0, 0, 7, 0, 0, 6, 1, 7, 3, 4089, 4020, 0, 7, 0, 5, 1, 5, 0, 4, 4089, 0, 0, 0, 4091, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1533, 4096, 4052, 0, 0, 0, 0, 0, 98, 0, 0, 268, 0, 0, 0, 0, 0, 1755, 0, 39, 0, 3035, 0, 4096, 0, 0, 88, 0, 0, 4096, 0, 4096, 0, 3566, 4096, 0, 4096, 32, 1, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 104, 0, 0, 4096, 0, 0, 3071, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 46, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 2, 3986, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2988, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4063, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[510, 4096, 4082, 0, 0, 0, 0, 0, 289, 0, 5, 111, 0, 0, 1, 0, 4096, 3018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1493, 0, 3, 0, 2751, 0, 4092, 5, 0, 23, 0, 0, 4089, 0, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3776, 4096, 0, 0, 3, 7, 0, 0, 392, 0, 4090, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 193, 2, 0, 0, 4096, 0, 3, 4011, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 12, 0, 0, 0, 4092, 0, 0, 0, 1, 0, 4083, 3285, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4095, 0, 0, 1, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 10, 3775, 0, 4096, 3, 0, 0, 2, 3, 1, 0, 0, 0, 0, 3086, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3742, 12, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 2, 0, 4084, 0, 419, 201, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16208, 16351, 16456, 16487, 16374, 16388, 16148, 16660] avg=16384.0000 max=16660 max_rank=7 min=16148 min_rank=6 imbalance_ratio=1.016846 max_load_reduction=114412 max_load_reduction_pct=0.872894 moved_to_echo=65570 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[2326, 4095, 4092, 1, 1, 0, 0, 3, 25, 0, 0, 267, 0, 0, 0, 0, 4095, 3018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2024, 1, 0, 2, 1041, 0, 4095, 1, 1, 0, 0, 1, 4095, 0, 3894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3425, 4095, 0, 0, 1, 0, 1, 0, 392, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 192, 1, 0, 0, 4093, 1, 0, 3916, 1, 1, 0, 1, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 10, 1, 1, 0, 4095, 1, 1, 1, 0, 1, 4095, 3284, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 5, 0, 0, 4095, 4087, 1, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 4071, 1, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 3083, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3351, 0, 1, 0, 1, 12, 1, 0, 0, 4095, 0, 4, 0, 4092, 1, 419, 201, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17923, 15155, 16109, 16393, 16402, 16381, 16428, 16281] avg=16384.0000 max=17923 max_rank=0 min=15155 min_rank=1 imbalance_ratio=1.093933 max_load_reduction=113149 max_load_reduction_pct=0.863258 moved_to_echo=65492 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[926, 4096, 4088, 0, 0, 0, 0, 0, 74, 0, 0, 55, 0, 0, 0, 0, 4096, 3018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2219, 0, 5, 0, 1543, 0, 4096, 4, 0, 9, 0, 0, 4096, 0, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3733, 4096, 0, 0, 4, 4, 0, 0, 392, 22, 4092, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 193, 6, 0, 0, 4096, 0, 0, 3908, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 21, 0, 4086, 3285, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4057, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3086, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3928, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4092, 0, 419, 201, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16353, 15867, 16438, 16389, 16381, 16378, 16415, 16851] avg=16384.0000 max=16851 max_rank=7 min=15867 min_rank=1 imbalance_ratio=1.028503 max_load_reduction=114221 max_load_reduction_pct=0.871437 moved_to_echo=66025 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[553, 4096, 4089, 0, 0, 0, 0, 0, 10, 0, 0, 65, 0, 0, 0, 0, 4096, 3019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1078, 0, 12, 0, 3466, 0, 4096, 0, 0, 7, 0, 0, 4096, 0, 3893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3500, 4096, 0, 0, 4, 5, 0, 0, 386, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 184, 4, 0, 0, 4096, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 19, 0, 4095, 3289, 822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4081, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3090, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3624, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4089, 0, 421, 203, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15928, 16648, 16181, 16555, 16403, 16374, 16440, 16543] avg=16384.0000 max=16648 max_rank=1 min=15928 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=114424 max_load_reduction_pct=0.872986 moved_to_echo=65760 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[112, 4089, 4091, 6, 7, 4, 14, 4, 40, 6, 0, 0, 0, 0, 0, 0, 4089, 3013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1778, 5, 0, 0, 2518, 0, 4089, 0, 5, 0, 0, 5, 4090, 6, 3890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3846, 4085, 4, 0, 0, 0, 6, 7, 391, 0, 4090, 1, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 193, 0, 0, 0, 4081, 5, 0, 4076, 6, 7, 0, 5, 5, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 6, 0, 4, 5, 2, 4088, 0, 4, 3, 0, 7, 4091, 3280, 816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4088, 3, 0, 0, 5, 0, 0, 4090, 4082, 6, 0, 6, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 6, 0, 4051, 3, 4089, 0, 0, 0, 7, 0, 0, 6, 1, 7, 3, 3081, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4020, 0, 7, 0, 5, 1, 5, 0, 4, 4089, 0, 0, 0, 4091, 5, 418, 200, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15475, 16387, 16521, 16556, 16398, 16374, 16419, 16942] avg=16384.0000 max=16942 max_rank=7 min=15475 min_rank=0 imbalance_ratio=1.034058 max_load_reduction=114130 max_load_reduction_pct=0.870743 moved_to_echo=66133 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[577, 4095, 4094, 1, 1, 0, 0, 0, 29, 0, 0, 34, 0, 0, 0, 0, 4095, 3017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2424, 0, 0, 0, 2049, 0, 4095, 0, 1, 3, 0, 1, 4093, 1, 3894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3792, 4095, 0, 0, 1, 0, 1, 0, 392, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 193, 0, 1, 0, 4095, 1, 0, 4045, 1, 1, 0, 1, 1, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4078, 1, 0, 1, 1, 2, 4095, 1, 1, 1, 2, 1, 4095, 3284, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 1, 0, 2, 0, 1, 4095, 4087, 1, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 4081, 0, 4095, 1, 0, 2, 4, 0, 0, 0, 0, 2, 0, 3084, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3449, 0, 1, 2, 1, 1, 1, 0, 0, 4095, 0, 1, 1, 4093, 1, 418, 201, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15943, 16561, 16469, 16530, 16381, 16379, 16441, 16368] avg=16384.0000 max=16561 max_rank=1 min=15943 min_rank=0 imbalance_ratio=1.010803 max_load_reduction=114511 max_load_reduction_pct=0.873650 moved_to_echo=65569 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[711, 4096, 4095, 0, 0, 0, 0, 0, 44, 0, 0, 19, 0, 0, 0, 0, 4096, 3019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3110, 0, 0, 0, 1898, 0, 4096, 0, 0, 4, 0, 0, 4096, 0, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3937, 4096, 0, 0, 1, 1, 0, 0, 392, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 193, 3, 0, 0, 4096, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4094, 3285, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4083, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3086, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2638, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 419, 201, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16080, 17099, 16619, 16518, 16388, 16376, 16438, 15554] avg=16384.0000 max=17099 max_rank=1 min=15554 min_rank=7 imbalance_ratio=1.043640 max_load_reduction=113973 max_load_reduction_pct=0.869545 moved_to_echo=64756 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1533, 4096, 4052, 0, 0, 0, 0, 0, 98, 0, 0, 268, 0, 0, 0, 0, 4096, 3019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1755, 0, 39, 0, 3035, 0, 4096, 0, 0, 88, 0, 0, 4096, 0, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3566, 4096, 0, 0, 32, 1, 0, 0, 392, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 193, 104, 0, 0, 4096, 0, 0, 3071, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3285, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 4096, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 3986, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3086, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2988, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4063, 0, 419, 201, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17162, 17004, 16279, 15644, 16345, 16422, 16343, 15873] avg=16384.0000 max=17162 max_rank=0 min=15644 min_rank=3 imbalance_ratio=1.047485 max_load_reduction=113910 max_load_reduction_pct=0.869064 moved_to_echo=64983 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 27776, 128, 128, 128, 128, 128, 128, 0, 128, 32768, 128, 128, 128, 32768, 128, 3456, 1664, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 29696, 32768, 128, 0, 128, 128, 128, 128, 3200, 128, 32768, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 1536, 128, 128, 0, 32768, 128, 128, 31232, 128, 128, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[7296, 32768, 32768, 128, 128, 128, 128, 128, 640, 128, 128, 896, 0, 0, 128, 0, 32768, 24192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[32768, 128, 128, 32256, 128, 32768, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 24704, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 16000, 128, 128, 128, 18304, 0, 32768, 128, 128, 256, 0, 128, 32768, 128, 31232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 128, 0, 128, 32768, 32768, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[0, 128, 0, 32768, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 128, 128, 32768, 26368, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4081, 412, 0, 4091, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 1, 54, 0, 0, 0, 4088, 4096, 5, 4093, 0, 918, 0, 0, 0, 4081, 0, 0, 4094, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 4091, 0, 27, 4096, 0, 4096, 0, 4094, 2912, 3, 4052, 4086, 12, 0, 0, 6, 16, 0, 1, 39, 4095, 15, 7, 78, 4096, 4096, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 4093, 4091, 5, 3, 0, 0, 0, 0, 0, 0, 0, 1, 4005, 0, 0, 4094, 0, 0, 0, 0, 4083, 0, 4069, 0, 3917, 0, 12, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 74, 0, 4092, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 0, 370, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 3, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 4095, 0, 25, 4096, 0, 4096, 0, 4096, 3859, 3, 4093, 4092, 4, 0, 0, 0, 0, 0, 0, 16, 4094, 9, 0, 55, 4096, 4093, 7, 0, 4096, 0, 2, 0, 0, 0, 4, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 4096, 0, 0, 0, 0, 4089, 0, 4084, 0, 3832, 0, 6, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4095, 588, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 0, 4095, 4096, 0, 4096, 0, 65, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 14, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 4096, 0, 38, 4096, 0, 4096, 0, 4096, 3463, 0, 4088, 4096, 4, 0, 0, 0, 3, 0, 0, 13, 4096, 6, 0, 97, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4078, 0, 3955, 0, 3, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[5, 0, 4089, 1330, 6, 4090, 7, 0, 15, 0, 6, 0, 4090, 0, 0, 4061, 0, 0, 5, 0, 6, 6, 4088, 6, 6, 7, 4084, 0, 2, 0, 7, 4, 0, 4076, 4083, 0, 4088, 5, 680, 0, 7, 5, 4090, 0, 6, 4079, 8, 8, 0, 0, 4063, 0, 0, 5, 0, 0, 6, 0, 4089, 4089, 4, 0, 4087, 8, 0, 4090, 4, 4084, 0, 4090, 2144, 0, 4088, 4091, 3, 13, 7, 12, 0, 0, 0, 0, 4066, 0, 0, 0, 4090, 4088, 0, 4, 4086, 0, 0, 5, 0, 6, 0, 2, 6, 4090, 4091, 0, 1, 2, 2, 0, 5, 6, 0, 0, 0, 4091, 0, 0, 4089, 5, 4, 7, 14, 4091, 5, 4091, 5, 4088, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 4090, 937, 1, 4095, 1, 0, 0, 1, 1, 0, 4095, 0, 0, 4093, 0, 0, 1, 0, 1, 1, 4095, 1, 0, 1, 4095, 1, 0, 0, 1, 0, 0, 4089, 4095, 0, 4094, 1, 1103, 0, 0, 2, 4095, 0, 1, 4095, 2, 1, 1, 0, 4045, 1, 0, 1, 0, 1, 0, 0, 4095, 4095, 0, 0, 4093, 1, 0, 4095, 1, 4095, 0, 4094, 2140, 0, 4093, 4095, 1, 1, 2, 0, 2, 0, 0, 0, 4095, 0, 0, 5, 4094, 4095, 0, 0, 4094, 1, 0, 2, 0, 1, 0, 0, 1, 4095, 4094, 0, 0, 1, 1, 0, 1, 1, 0, 0, 4, 4094, 0, 0, 4095, 1, 1, 1, 3, 4095, 1, 4095, 0, 4058, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 551, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4073, 4096, 0, 4096, 0, 1331, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 12, 4096, 0, 4096, 0, 4096, 2563, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 4096, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4092, 0, 3758, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 4095, 258, 1, 4095, 1, 0, 0, 0, 1, 0, 4095, 0, 0, 4094, 0, 0, 1, 0, 1, 1, 4095, 1, 0, 1, 4095, 1, 0, 13, 1, 0, 0, 4095, 4094, 0, 4094, 1, 570, 0, 1, 0, 4095, 0, 1, 4095, 1, 1, 8, 1, 4087, 0, 0, 1, 0, 0, 0, 0, 4095, 4095, 1, 0, 4094, 1, 8, 4095, 1, 4095, 1, 4095, 3519, 0, 4032, 4095, 5, 2, 1, 0, 0, 0, 0, 0, 4090, 0, 0, 55, 4095, 4095, 0, 0, 4094, 0, 0, 1, 0, 1, 0, 0, 1, 4095, 4095, 4, 0, 1, 0, 0, 1, 1, 0, 1, 0, 4086, 0, 0, 4095, 1, 1, 1, 1, 4095, 1, 4095, 1, 3834, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4084, 1251, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 4095, 4096, 0, 4096, 0, 241, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 70, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 54, 4096, 0, 4096, 0, 4096, 2544, 0, 4015, 4096, 59, 0, 0, 0, 2, 0, 0, 47, 4096, 8, 5, 202, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4082, 0, 3879, 0, 12, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4081, 412, 0, 4091, 0, 0, 0, 0, 0, 0, 3425, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 1, 54, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3446, 5, 0, 0, 918, 0, 0, 0, 4081, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 4091, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 4096, 0, 1193, 2912, 3, 4052, 4086, 12, 0, 0, 6, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 39, 4095, 15, 7, 78, 4019, 4096, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4091, 5, 3, 0, 0, 0, 0, 0, 0, 0, 1, 4005, 2901, 666, 542, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4083, 0, 4069, 0, 3917, 0, 12, 2, 0, 108, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16105, 16421, 16632, 16379, 16403, 16451, 16320, 16361] avg=16384.0000 max=16632 max_rank=2 min=16105 min_rank=0 imbalance_ratio=1.015137 max_load_reduction=114440 max_load_reduction_pct=0.873108 moved_to_echo=65535 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 74, 0, 4092, 0, 0, 0, 0, 0, 0, 3427, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3446, 1, 0, 0, 370, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 4096, 0, 1193, 3859, 3, 4093, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4094, 9, 0, 55, 4019, 4093, 7, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 2903, 666, 542, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 0, 4084, 0, 3832, 0, 6, 0, 0, 108, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15785, 16374, 16099, 16385, 17365, 16391, 16382, 16291] avg=16384.0000 max=17365 max_rank=4 min=15785 min_rank=0 imbalance_ratio=1.059875 max_load_reduction=113707 max_load_reduction_pct=0.867516 moved_to_echo=66429 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 588, 0, 4094, 0, 0, 0, 0, 0, 0, 3425, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3442, 0, 0, 0, 65, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 4096, 0, 1190, 3463, 0, 4088, 4096, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 6, 0, 97, 4010, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 2906, 671, 544, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4078, 0, 3955, 0, 3, 0, 0, 110, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16298, 16381, 15792, 16401, 16978, 16414, 16390, 16418] avg=16384.0000 max=16978 max_rank=4 min=15792 min_rank=2 imbalance_ratio=1.036255 max_load_reduction=114094 max_load_reduction_pct=0.870468 moved_to_echo=66200 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 4089, 1330, 6, 4090, 7, 0, 15, 0, 6, 0, 3425, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 6, 4088, 6, 6, 7, 4084, 0, 2, 0, 7, 4, 4090, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 3436, 0, 0, 5, 680, 0, 7, 5, 4090, 0, 6, 4079, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 5, 0, 0, 6, 0, 4089, 4089, 4, 0, 4087, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4084, 0, 1192, 2144, 0, 4088, 4091, 3, 13, 7, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 4013, 4088, 0, 4, 4086, 0, 0, 5, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 4090, 4091, 0, 1, 2, 2, 0, 5, 6, 0, 0, 0, 4091, 2898, 665, 540, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 5, 4, 7, 14, 4091, 5, 4091, 5, 4088, 0, 0, 0, 0, 107, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17034, 16386, 16400, 16352, 15638, 16268, 16412, 16582] avg=16384.0000 max=17034 max_rank=0 min=15638 min_rank=4 imbalance_ratio=1.039673 max_load_reduction=114038 max_load_reduction_pct=0.870041 moved_to_echo=64900 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4095, 258, 1, 4095, 1, 0, 0, 0, 1, 0, 3429, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 4095, 1, 0, 1, 4095, 1, 0, 13, 1, 0, 4095, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3445, 0, 0, 1, 570, 0, 1, 0, 4095, 0, 1, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 4087, 0, 0, 1, 0, 0, 0, 0, 4095, 4095, 1, 0, 4094, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 4095, 1, 1193, 3519, 0, 4032, 4095, 5, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 55, 4018, 4095, 0, 0, 4094, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4095, 4, 0, 1, 0, 0, 1, 1, 0, 1, 0, 4086, 2902, 666, 541, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 1, 1, 4095, 1, 4095, 1, 3834, 0, 0, 0, 0, 108, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15975, 16387, 16305, 16384, 16952, 16354, 16406, 16309] avg=16384.0000 max=16952 max_rank=4 min=15975 min_rank=0 imbalance_ratio=1.034668 max_load_reduction=114120 max_load_reduction_pct=0.870667 moved_to_echo=66021 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4090, 937, 1, 4095, 1, 0, 0, 1, 1, 0, 3429, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 4095, 1, 0, 1, 4095, 1, 0, 0, 1, 0, 4095, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3445, 0, 0, 1, 1103, 0, 0, 2, 4095, 0, 1, 4095, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4045, 1, 0, 1, 0, 1, 0, 0, 4095, 4095, 0, 0, 4093, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1193, 2140, 0, 4093, 4095, 1, 1, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 5, 4017, 4095, 0, 0, 4094, 1, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4094, 0, 0, 1, 1, 0, 1, 1, 0, 0, 4, 4094, 2901, 666, 542, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 1, 3, 4095, 1, 4095, 0, 4058, 1, 0, 0, 0, 108, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16649, 16373, 16834, 16334, 15623, 16310, 16414, 16535] avg=16384.0000 max=16834 max_rank=2 min=15623 min_rank=4 imbalance_ratio=1.027466 max_load_reduction=114238 max_load_reduction_pct=0.871567 moved_to_echo=64882 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 551, 0, 4096, 0, 0, 0, 0, 0, 0, 3430, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 3446, 0, 0, 0, 1331, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 1193, 2563, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 4019, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2903, 666, 542, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4092, 0, 3758, 0, 1, 0, 0, 108, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16269, 16373, 17041, 16385, 16054, 16315, 16408, 16227] avg=16384.0000 max=17041 max_rank=2 min=16054 min_rank=4 imbalance_ratio=1.040100 max_load_reduction=114031 max_load_reduction_pct=0.869987 moved_to_echo=65004 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4084, 1251, 0, 4096, 0, 0, 0, 0, 0, 0, 3430, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3446, 0, 0, 0, 241, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 4096, 0, 1193, 2544, 0, 4015, 4096, 59, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4096, 8, 5, 202, 4019, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 2903, 666, 542, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4082, 0, 3879, 0, 12, 0, 0, 108, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16957, 16377, 15969, 16452, 16059, 16569, 16340, 16349] avg=16384.0000 max=16957 max_rank=0 min=15969 min_rank=2 imbalance_ratio=1.034973 max_load_reduction=114115 max_load_reduction_pct=0.870628 moved_to_echo=65317 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 32768, 5504, 128, 32768, 128, 0, 128, 128, 128, 0, 27520, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 27648, 128, 0, 128, 5376, 0, 128, 128, 32768, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 0, 128, 128, 128, 128, 32768, 128, 128, 128, 32768, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 128, 0, 128, 0, 128, 128, 0, 32768, 32768, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 32768, 32768, 128, 128, 128, 128, 0, 128, 128, 0, 128, 128, 32640, 23296, 5376, 4352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 32768, 128, 128, 128, 128, 32768, 128, 32768, 128, 31360, 128, 128, 128, 0, 896, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 0, 128, 32768, 128, 9600, 23168, 128, 32640, 32768, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 32768, 128, 128, 512, 32256, 32768, 128, 128, 32768, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1301, 0, 4096, 0, 0, 0, 3970, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 3913, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 1202, 0, 285, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4095, 53, 4096, 1564, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4093, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1832, 0, 4096, 0, 0, 0, 4076, 12, 4090, 0, 0, 0, 4094, 0, 4096, 0, 4096, 1, 4096, 0, 0, 4089, 0, 0, 2846, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 4095, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4091, 0, 0, 3024, 0, 40, 0, 0, 0, 0, 0, 4096, 4096, 5, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 164, 4096, 296, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 708, 0, 4096, 0, 0, 0, 4075, 516, 4096, 0, 0, 0, 4092, 0, 4096, 0, 4096, 3, 4091, 0, 5, 4088, 0, 0, 3929, 0, 0, 4091, 0, 0, 4096, 0, 0, 2, 0, 4093, 4090, 41, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 4091, 0, 4093, 0, 0, 1998, 0, 606, 0, 1, 3, 0, 0, 4096, 4094, 153, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4094, 0, 4094, 100, 4096, 159, 0, 0, 0, 0, 8, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 8, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 1, 0, 4094, 1, 1, 0, 1, 0, 1, 1424, 0, 4095, 0, 0, 0, 4057, 0, 4094, 0, 0, 1, 4095, 1, 4095, 0, 4094, 0, 4095, 1, 0, 4095, 0, 0, 2857, 0, 1, 4095, 0, 0, 4095, 2, 0, 0, 1, 4095, 4095, 5, 1, 1, 4094, 0, 0, 1, 0, 0, 1, 4092, 1, 1, 1, 4095, 1, 4093, 1, 1, 3247, 0, 574, 1, 0, 0, 1, 1, 4095, 4095, 0, 0, 4095, 4095, 1, 4095, 0, 0, 4094, 1, 4095, 0, 4090, 14, 4095, 143, 1, 1, 1, 2, 1, 0, 0, 0, 1, 1, 1, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4066, 1, 4094, 0, 0, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[6, 7, 0, 0, 0, 5, 0, 4086, 6, 8, 3, 5, 7, 5, 499, 19, 3927, 0, 0, 0, 3953, 45, 4089, 0, 4, 3, 4090, 7, 4089, 0, 4084, 0, 4088, 10, 0, 4090, 0, 0, 3159, 0, 7, 4089, 6, 0, 4090, 4, 0, 0, 7, 4089, 4090, 0, 6, 0, 4084, 0, 4, 4, 0, 0, 6, 4089, 0, 7, 7, 4081, 6, 4089, 5, 11, 2680, 1, 1192, 0, 0, 0, 0, 6, 4088, 4089, 1, 0, 4089, 4088, 5, 4088, 6, 4, 4088, 7, 4089, 0, 4078, 0, 4070, 950, 6, 7, 4, 0, 0, 14, 0, 0, 7, 0, 6, 0, 4085, 0, 0, 2, 6, 0, 0, 1, 6, 4, 4089, 4048, 3, 4085, 0, 1, 0, 0, 4, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 1, 0, 4095, 1, 1, 0, 1, 2, 1, 928, 0, 4095, 0, 1, 0, 4079, 4, 4095, 0, 0, 0, 4094, 1, 4095, 0, 4095, 1, 4095, 0, 1, 4095, 1, 0, 3843, 1, 1, 4094, 0, 0, 4095, 2, 0, 1, 1, 4094, 4092, 7, 1, 0, 4093, 0, 0, 1, 0, 0, 1, 4094, 2, 1, 1, 4095, 1, 4094, 0, 1, 2084, 0, 707, 0, 1, 0, 0, 1, 4095, 4095, 2, 0, 4094, 4095, 1, 4094, 1, 0, 4095, 1, 4094, 0, 4087, 1, 4090, 646, 1, 1, 1, 0, 0, 1, 0, 1, 2, 1, 1, 0, 4094, 0, 0, 0, 1, 0, 1, 0, 1, 1, 4094, 4088, 0, 4094, 0, 0, 0, 0, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 1772, 0, 4096, 0, 0, 0, 3520, 170, 4095, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4095, 0, 0, 3372, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4095, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 2501, 0, 436, 0, 0, 0, 0, 0, 4096, 4096, 116, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 36, 4096, 363, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 600, 0, 4096, 0, 0, 0, 4067, 103, 4095, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 3850, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 2030, 0, 1060, 0, 0, 0, 0, 0, 4096, 4096, 6, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 252, 4096, 321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 4096, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 708, 0, 4096, 4058, 2988, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4075, 516, 4096, 0, 0, 0, 4092, 0, 0, 0, 38, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 5, 4088, 0, 0, 3929, 0, 0, 4091, 0, 0, 627, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4090, 41, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 4090, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4093, 0, 0, 1998, 0, 606, 0, 1, 3, 0, 0, 1139, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 0, 0, 3436, 0, 4094, 0, 4094, 100, 4096, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 8, 0, 0, 0, 2957, 660, 481, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15952, 16916, 16833, 16417, 16025, 16132, 16398, 16399] avg=16384.0000 max=16916 max_rank=1 min=15952 min_rank=0 imbalance_ratio=1.032471 max_load_reduction=114156 max_load_reduction_pct=0.870941 moved_to_echo=64954 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1301, 0, 4096, 4060, 2990, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3970, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3913, 0, 0, 4096, 0, 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1202, 0, 285, 0, 0, 0, 0, 0, 1133, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3433, 0, 4096, 0, 4095, 53, 4096, 1564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 2963, 663, 486, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16550, 16294, 16821, 16375, 14908, 17337, 16380, 16407] avg=16384.0000 max=17337 max_rank=5 min=14908 min_rank=4 imbalance_ratio=1.058167 max_load_reduction=113735 max_load_reduction_pct=0.867729 moved_to_echo=65032 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1832, 0, 4096, 4058, 2988, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4076, 12, 4090, 0, 0, 0, 4094, 0, 0, 0, 38, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 0, 0, 2846, 0, 0, 4096, 0, 0, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 3024, 0, 40, 0, 0, 0, 0, 0, 1139, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 3436, 0, 4096, 0, 4096, 164, 4096, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 1, 0, 0, 0, 2957, 660, 481, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17076, 16407, 15754, 16377, 16486, 16189, 16385, 16398] avg=16384.0000 max=17076 max_rank=0 min=15754 min_rank=2 imbalance_ratio=1.042236 max_load_reduction=113996 max_load_reduction_pct=0.869720 moved_to_echo=65458 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 1, 0, 4094, 1, 1, 0, 1, 0, 1, 1424, 0, 4095, 4056, 2987, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4057, 0, 4094, 0, 0, 1, 4095, 1, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 4095, 0, 0, 2857, 0, 1, 4095, 0, 0, 627, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4095, 5, 1, 1, 4094, 0, 0, 1, 0, 0, 1, 4087, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 4093, 1, 1, 3247, 0, 574, 1, 0, 0, 1, 1, 1139, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3435, 1, 4095, 0, 4090, 14, 4095, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 1, 0, 0, 0, 1, 1, 1, 1, 4094, 0, 0, 0, 4095, 4095, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4066, 1, 4094, 0, 0, 0, 0, 1, 0, 2956, 659, 481, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16669, 16381, 15773, 16383, 17250, 15874, 16382, 16360] avg=16384.0000 max=17250 max_rank=4 min=15773 min_rank=2 imbalance_ratio=1.052856 max_load_reduction=113822 max_load_reduction_pct=0.868393 moved_to_echo=65866 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[6, 7, 0, 0, 0, 5, 0, 4086, 6, 8, 3, 5, 7, 5, 499, 19, 4088, 4046, 2983, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 0, 0, 0, 3953, 45, 4089, 0, 4, 3, 4090, 7, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 10, 0, 4090, 0, 0, 3159, 0, 7, 4089, 6, 0, 626, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 4090, 0, 6, 0, 4084, 0, 4, 4, 0, 0, 6, 4084, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4081, 6, 4089, 5, 11, 2680, 1, 1192, 0, 0, 0, 0, 6, 1137, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 6, 4, 3430, 7, 4089, 0, 4078, 0, 4070, 950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 4, 0, 0, 14, 0, 0, 7, 0, 6, 0, 4085, 0, 0, 2, 4089, 4089, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 1, 6, 4, 4089, 4048, 3, 4085, 0, 1, 0, 0, 4, 0, 2951, 658, 481, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15780, 16156, 16079, 16381, 17304, 16640, 16390, 16342] avg=16384.0000 max=17304 max_rank=4 min=15780 min_rank=0 imbalance_ratio=1.056152 max_load_reduction=113768 max_load_reduction_pct=0.867981 moved_to_echo=66676 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 1, 0, 4095, 1, 1, 0, 1, 2, 1, 928, 0, 4094, 4057, 2987, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 4079, 4, 4095, 0, 0, 0, 4094, 1, 0, 0, 38, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 4095, 1, 0, 3843, 1, 1, 4094, 0, 0, 627, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 4092, 7, 1, 0, 4093, 0, 0, 1, 0, 0, 1, 4089, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 4094, 0, 1, 2084, 0, 707, 0, 1, 0, 0, 1, 1139, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1, 0, 3435, 1, 4094, 0, 4087, 1, 4090, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 2, 1, 1, 0, 4094, 0, 0, 0, 4095, 4094, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 4094, 4088, 0, 4094, 0, 0, 0, 0, 5, 0, 2956, 660, 481, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16176, 16408, 16761, 16382, 16219, 16358, 16381, 16387] avg=16384.0000 max=16761 max_rank=2 min=16176 min_rank=0 imbalance_ratio=1.023010 max_load_reduction=114311 max_load_reduction_pct=0.872124 moved_to_echo=65345 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 1772, 0, 4096, 4058, 2988, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3520, 170, 4095, 0, 0, 0, 4096, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 3372, 0, 0, 4096, 0, 0, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 2501, 0, 436, 0, 0, 0, 0, 0, 1139, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 3436, 0, 4096, 0, 4096, 36, 4096, 363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 1, 0, 0, 0, 2957, 660, 481, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17018, 16015, 16286, 16380, 16363, 16239, 16378, 16393] avg=16384.0000 max=17018 max_rank=0 min=16015 min_rank=1 imbalance_ratio=1.038696 max_load_reduction=114054 max_load_reduction_pct=0.870163 moved_to_echo=65373 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 600, 0, 4096, 4058, 2988, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4067, 103, 4095, 0, 0, 0, 4096, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3850, 0, 0, 4096, 0, 0, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2030, 0, 1060, 0, 0, 0, 0, 0, 1139, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 3436, 0, 4096, 0, 4096, 252, 4096, 321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 4096, 0, 0, 1, 0, 0, 0, 2957, 660, 481, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15851, 16495, 16765, 16377, 16517, 16303, 16378, 16386] avg=16384.0000 max=16765 max_rank=2 min=15851 min_rank=0 imbalance_ratio=1.023254 max_load_reduction=114307 max_load_reduction_pct=0.872093 moved_to_echo=65584 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 128, 32768, 128, 128, 18816, 128, 4992, 128, 128, 128, 128, 128, 9216, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 32768, 128, 0, 27776, 128, 128, 32768, 128, 0, 5120, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[384, 0, 0, 0, 128, 0, 128, 128, 27520, 128, 32768, 0, 32768, 640, 32768, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 32768, 0, 128, 128, 0, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32640, 0, 128, 0, 31872, 896, 32768, 0, 128, 128, 32768, 128, 0, 0, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 0, 0, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 0, 0, 128, 128, 32768, 128, 128, 128, 128, 128, 128, 9088, 128, 32768, 32512, 23936, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 128, 32768, 32768, 128, 32768, 0, 128, 128, 0, 128, 0, 23680, 5376, 3968, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 2, 0, 4096, 4096, 0, 4096, 340, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 4096, 4087, 0, 4094, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3757, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 4093, 4096, 0, 4096, 4089, 4096, 0, 4096, 4093, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 10, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4093, 4, 0, 2, 1, 0, 0, 0, 1, 0, 3, 5, 0, 4095, 4091, 0, 4096, 526, 0, 0, 0, 6, 0, 4093, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4089, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 0, 0, 0, 0, 1, 4096, 1, 0, 4096, 4084, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3570, 0, 0, 0, 4096, 0, 0, 4, 8, 0, 5, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 4096, 0, 4090, 4096, 0, 4096, 4088, 4096, 0, 4096, 4092, 1, 0, 0, 4, 1, 0, 0, 4096, 0, 3, 0, 4096, 4093, 4096, 4095, 13, 4092, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 3, 1, 1, 0, 4, 0, 1, 0, 4094, 4095, 1, 4094, 438, 1, 2, 1, 0, 1, 4095, 1, 1, 0, 1, 4095, 0, 4058, 4095, 1, 4094, 1, 0, 0, 1, 1, 8, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 4089, 2, 0, 4095, 4094, 0, 4094, 1, 0, 0, 0, 1, 0, 0, 1, 4094, 0, 0, 0, 1, 4094, 3671, 1, 1, 17, 4095, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 4093, 0, 0, 2, 0, 1, 4095, 0, 4095, 4094, 0, 4094, 4095, 4092, 1, 4095, 4092, 0, 0, 2, 0, 0, 0, 1, 4094, 4, 0, 1, 4095, 4095, 4095, 4095, 0, 4095, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 2, 1, 4095, 4095, 1, 4095, 19, 1, 0, 1, 0, 0, 4095, 1, 1, 0, 1, 4095, 0, 4061, 4095, 1, 4094, 1, 1, 0, 1, 1, 7, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4094, 0, 0, 4094, 4094, 0, 4094, 1, 1, 1, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 1, 4094, 4093, 1, 1, 16, 4094, 0, 3, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 4093, 0, 0, 2, 0, 1, 4095, 0, 4095, 4094, 0, 4093, 4096, 4092, 1, 4095, 4088, 0, 0, 2, 0, 0, 0, 0, 4095, 1, 0, 1, 4095, 4095, 4095, 4095, 0, 4095, 0, 1, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 502, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4091, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3599, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 4092, 4096, 0, 4096, 4092, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4089, 0, 6, 0, 0, 6, 7, 0, 0, 3, 0, 0, 4, 4088, 4089, 7, 4089, 3, 6, 0, 6, 0, 2, 4090, 6, 7, 0, 6, 4089, 3, 3808, 4089, 2, 4087, 10, 4, 0, 8, 7, 59, 6, 0, 9, 0, 0, 0, 0, 9, 0, 7, 44, 0, 4030, 0, 0, 4083, 4090, 0, 4087, 5, 0, 0, 0, 0, 2, 1, 6, 4089, 5, 1, 0, 4, 4086, 4088, 6, 7, 11, 4086, 0, 1, 0, 0, 7, 0, 1, 0, 7, 6, 5, 0, 4089, 0, 0, 8, 1, 6, 4089, 0, 4090, 4079, 0, 4078, 4092, 4075, 7, 4081, 4078, 0, 0, 4, 0, 0, 0, 0, 4089, 12, 0, 7, 4088, 4089, 4089, 4087, 0, 4089, 5, 266, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 155, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4092, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3943, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 4096, 4095, 4096, 0, 4096, 4094, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 4, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 802, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3294, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 4095, 4096, 0, 4096, 4067, 0, 0, 0, 27, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4093, 4, 0, 2, 1, 0, 0, 0, 1, 0, 3, 5, 0, 4095, 4091, 0, 4081, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 526, 0, 0, 0, 6, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 3792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3, 0, 4096, 4096, 4079, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 1, 0, 4096, 4084, 0, 4092, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3570, 0, 0, 0, 4096, 0, 0, 304, 29, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, 0, 5, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4096, 4088, 4096, 0, 0, 4092, 1, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 3, 0, 0, 4093, 0, 4095, 13, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16609, 16374, 16376, 16193, 16385, 16391, 16357] avg=16384.0000 max=16609 max_rank=1 min=16193 min_rank=4 imbalance_ratio=1.013733 max_load_reduction=114463 max_load_reduction_pct=0.873283 moved_to_echo=65326 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4088, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 502, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 3785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4084, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 3, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3599, 0, 0, 1, 4096, 0, 0, 311, 31, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 4096, 0, 0, 4092, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16391, 16570, 16374, 16400, 16238, 16378, 16381, 16340] avg=16384.0000 max=16570 max_rank=1 min=16238 min_rank=4 imbalance_ratio=1.011353 max_load_reduction=114502 max_load_reduction_pct=0.873581 moved_to_echo=65337 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 2, 0, 4096, 4096, 0, 4081, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 340, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 3792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 4082, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 4096, 4087, 0, 4094, 0, 0, 3, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3757, 0, 0, 1, 4096, 0, 0, 304, 29, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4096, 4089, 4096, 0, 0, 4093, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 4096, 0, 4096, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16419, 16364, 16391, 16381, 16380, 16391, 16359] avg=16384.0000 max=16419 max_rank=1 min=16359 min_rank=7 imbalance_ratio=1.002136 max_load_reduction=114653 max_load_reduction_pct=0.874733 moved_to_echo=65511 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 2, 1, 4095, 4095, 1, 4079, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 19, 1, 0, 1, 0, 0, 4095, 1, 1, 0, 1, 0, 0, 4061, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 1, 0, 1, 1, 7, 1, 0, 1, 0, 0, 0, 0, 1, 4095, 4095, 4083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 0, 0, 4094, 4094, 0, 4094, 1, 1, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 1, 4094, 4093, 1, 1, 16, 4094, 0, 3, 304, 29, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 4093, 0, 0, 2, 0, 1, 4095, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4093, 4096, 4092, 1, 0, 4088, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 1, 0, 1, 0, 4095, 0, 4095, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16066, 16383, 16386, 16735, 16385, 16385, 16349] avg=16384.0000 max=16735 max_rank=4 min=16066 min_rank=1 imbalance_ratio=1.021423 max_load_reduction=114337 max_load_reduction_pct=0.872322 moved_to_echo=65854 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4089, 0, 6, 0, 0, 6, 7, 0, 0, 3, 0, 0, 4, 4088, 4089, 7, 4064, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 6, 0, 6, 0, 2, 4090, 6, 7, 0, 6, 0, 3, 3808, 3785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4087, 10, 4, 0, 8, 7, 59, 6, 0, 9, 0, 0, 0, 0, 9, 4088, 4081, 4078, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 44, 0, 4030, 0, 0, 4083, 4090, 0, 4087, 5, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 6, 4089, 5, 1, 0, 4, 4086, 4088, 6, 7, 11, 4086, 0, 1, 304, 29, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 0, 7, 6, 5, 0, 4089, 0, 0, 8, 1, 6, 4089, 4089, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4078, 4092, 4075, 7, 0, 4078, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 12, 0, 7, 0, 4089, 0, 4087, 0, 4089, 5, 266, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 15811, 16449, 16352, 16728, 16396, 16347, 16614] avg=16384.0000 max=16728 max_rank=4 min=15811 min_rank=1 imbalance_ratio=1.020996 max_load_reduction=114344 max_load_reduction_pct=0.872375 moved_to_echo=66085 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 3, 1, 1, 0, 4, 0, 1, 0, 4094, 4095, 1, 4079, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 438, 1, 2, 1, 0, 1, 4095, 1, 1, 0, 1, 0, 0, 4058, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 0, 0, 1, 1, 8, 1, 0, 0, 0, 0, 1, 0, 1, 4095, 4095, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4089, 2, 0, 4095, 4094, 0, 4094, 1, 0, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 1, 4094, 3671, 1, 1, 17, 4095, 0, 1, 304, 29, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 4093, 0, 0, 2, 0, 1, 4095, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4094, 4095, 4092, 1, 0, 4092, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4059, 4, 0, 1, 0, 4095, 0, 4095, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16484, 16384, 16384, 16311, 16383, 16389, 16351] avg=16384.0000 max=16484 max_rank=1 min=16311 min_rank=4 imbalance_ratio=1.006104 max_load_reduction=114588 max_load_reduction_pct=0.874237 moved_to_echo=65434 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4081, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 155, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4092, 3792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4083, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 4095, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3943, 0, 0, 1, 4096, 0, 0, 304, 29, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4096, 4095, 4096, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 4096, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16231, 16371, 16391, 16568, 16382, 16394, 16353] avg=16384.0000 max=16568 max_rank=4 min=16231 min_rank=1 imbalance_ratio=1.011230 max_load_reduction=114504 max_load_reduction_pct=0.873596 moved_to_echo=65697 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4081, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 802, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4084, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3294, 0, 0, 1, 4096, 0, 0, 304, 29, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4096, 4095, 4096, 0, 0, 4067, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16882, 16373, 16392, 15918, 16383, 16394, 16349] avg=16384.0000 max=16882 max_rank=1 min=15918 min_rank=4 imbalance_ratio=1.030396 max_load_reduction=114190 max_load_reduction_pct=0.871201 moved_to_echo=65044 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 0, 32768, 32768, 32768, 128, 0, 32768, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 0, 128, 32768, 30080, 128, 128, 128, 32768, 0, 128, 2560, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 128, 128, 0, 128, 0, 0, 128, 128, 128, 32768, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 2816, 128, 128, 128, 128, 128, 32768, 128, 128, 0, 128, 0, 128, 32512, 30336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 32512, 128, 128, 128, 0, 32768, 0, 32768, 128, 32768, 128, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 128, 128, 0, 32768, 0, 0, 128, 128, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 32768, 32768, 0, 32768, 128, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4094, 4096, 0, 0, 0, 4096, 4096, 0, 0, 12, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 41, 4096, 4096, 2, 1, 0, 0, 0, 0, 4096, 4094, 4088, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 1, 4088, 0, 0, 0, 0, 4094, 0, 0, 1, 5, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4084, 0, 4095, 4095, 4055, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 9, 0, 0, 4082, 0, 14, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4089, 4087, 9, 0, 14, 4069, 4088, 0, 0, 0, 0, 6, 7, 0, 0, 0, 4089, 14, 7, 0, 4088, 4088, 0, 7, 4, 0, 1012, 4082, 4077, 0, 0, 1, 5, 2, 9, 4088, 4088, 4090, 4090, 1, 0, 7, 8, 0, 6, 8, 7, 0, 7, 0, 0, 4, 4, 4089, 32, 7, 0, 0, 4044, 6, 6, 0, 0, 4086, 1, 0, 0, 0, 4089, 0, 6, 4, 4086, 10, 4082, 4084, 4084, 6, 5, 0, 6, 6, 8, 4089, 0, 4085, 4080, 3545, 4088, 3, 6, 0, 0, 2, 4071, 4083, 7, 8, 0, 40, 0, 0, 6, 2, 0, 0, 0, 4090, 0, 9, 0, 4088, 2, 10, 0, 0, 12, 11, 8, 4089, 0, 3597, 8, 9, 9, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4095, 4094, 1, 0, 2, 4090, 4094, 0, 0, 1, 1, 1, 1, 0, 0, 0, 4094, 3, 1, 0, 4095, 4094, 0, 1, 1, 0, 84, 4094, 4093, 1, 0, 0, 0, 0, 0, 4095, 4095, 4095, 4095, 0, 0, 1, 0, 1, 1, 2, 1, 0, 1, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 4036, 1, 1, 0, 0, 4095, 1, 1, 0, 4, 4095, 1, 1, 0, 4095, 1, 4094, 4094, 4095, 1, 1, 0, 1, 0, 0, 4094, 0, 4093, 4095, 4071, 4095, 0, 1, 0, 1, 0, 4094, 4094, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 4095, 0, 1, 0, 4094, 1, 1, 0, 0, 5, 0, 1, 4095, 0, 4089, 1, 1, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4077, 4096, 0, 0, 0, 4096, 4096, 0, 0, 12, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 425, 4096, 4096, 8, 7, 0, 0, 0, 0, 4096, 4088, 4083, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 4, 8, 4061, 0, 0, 1, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 4088, 0, 4095, 4092, 3683, 4090, 0, 0, 0, 9, 0, 4096, 4096, 0, 0, 0, 0, 0, 12, 0, 0, 21, 0, 0, 4080, 0, 31, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4095, 4094, 1, 1, 2, 4093, 4095, 0, 0, 0, 0, 1, 1, 0, 2, 0, 4095, 0, 1, 0, 4095, 4095, 0, 1, 1, 0, 65, 4095, 4095, 0, 0, 0, 0, 1, 1, 4095, 4095, 4095, 4095, 0, 0, 1, 1, 0, 1, 1, 1, 4, 1, 0, 0, 0, 1, 4095, 1, 1, 0, 0, 4053, 1, 1, 0, 0, 4095, 1, 0, 1, 0, 4095, 0, 1, 2, 4095, 1, 4094, 4095, 4092, 1, 1, 0, 1, 0, 1, 4095, 0, 4092, 4094, 4074, 4094, 0, 1, 0, 0, 0, 4094, 4095, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4095, 0, 19, 0, 4092, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 4073, 1, 2, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 6, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 50, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4049, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4089, 0, 19, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4094, 4096, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 509, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 4095, 4096, 3586, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 4094, 0, 11, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 138, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 4030, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 4096, 3958, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 4078, 0, 86, 0, 4090, 0, 0, 17, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4077, 4096, 0, 0, 0, 4096, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3806, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 425, 4096, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 4088, 4083, 4085, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 4, 8, 4061, 0, 0, 1, 4096, 4096, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 12, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4095, 4092, 3683, 224, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 12, 0, 0, 21, 0, 0, 4080, 0, 31, 3771, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 4057, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16524, 16361, 16395, 16391, 16191, 16392, 16441] avg=16384.0000 max=16524 max_rank=1 min=16191 min_rank=5 imbalance_ratio=1.008545 max_load_reduction=114548 max_load_reduction_pct=0.873932 moved_to_echo=65415 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 4096, 0, 0, 0, 4096, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3809, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 41, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4094, 4088, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 1, 4088, 0, 0, 0, 4096, 4096, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 5, 0, 0, 0, 0, 12, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4095, 4095, 4055, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 9, 0, 0, 4082, 0, 14, 3776, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4057, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16394, 16144, 16375, 16404, 16400, 16554, 16361, 16440] avg=16384.0000 max=16554 max_rank=5 min=16144 min_rank=1 imbalance_ratio=1.010376 max_load_reduction=114518 max_load_reduction_pct=0.873703 moved_to_echo=65755 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 4087, 9, 0, 14, 4069, 4088, 0, 0, 0, 0, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3802, 14, 7, 0, 0, 4088, 0, 7, 4, 0, 1012, 4082, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 2, 9, 4088, 4088, 4090, 4090, 1, 0, 7, 8, 0, 6, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 0, 0, 4, 4, 4089, 32, 7, 0, 0, 4044, 6, 6, 0, 4089, 4088, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 0, 0, 0, 0, 0, 6, 4, 11, 10, 4082, 4084, 4084, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 6, 8, 4089, 0, 4085, 4080, 3545, 224, 3, 6, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 4083, 7, 8, 0, 40, 0, 0, 6, 2, 0, 0, 0, 4090, 0, 9, 3769, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 2, 10, 0, 0, 12, 11, 8, 4089, 0, 3597, 8, 9, 9, 6, 4048, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16369, 17093, 16405, 16410, 16374, 16059, 16370, 15992] avg=16384.0000 max=17093 max_rank=1 min=15992 min_rank=7 imbalance_ratio=1.043274 max_load_reduction=113979 max_load_reduction_pct=0.869591 moved_to_echo=64795 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 6, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3789, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 50, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 4074, 0, 0, 0, 4096, 4096, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4049, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4089, 0, 19, 3781, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4062, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 16128, 16391, 16398, 16382, 16550, 16378, 16448] avg=16384.0000 max=16550 max_rank=5 min=16128 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=114522 max_load_reduction_pct=0.873734 moved_to_echo=65758 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 4094, 1, 1, 2, 4093, 4095, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3808, 0, 1, 0, 0, 4095, 0, 1, 1, 0, 65, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4095, 4095, 4095, 0, 0, 1, 1, 0, 1, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 1, 0, 0, 0, 1, 4095, 1, 1, 0, 0, 4053, 1, 1, 0, 4095, 4095, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 0, 0, 1, 2, 12, 1, 4094, 4095, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 4095, 0, 4092, 4094, 4074, 224, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4095, 0, 19, 3775, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 1, 0, 1, 4095, 0, 4073, 1, 2, 0, 1, 4056, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16163, 16388, 16376, 16395, 16583, 16367, 16417] avg=16384.0000 max=16583 max_rank=5 min=16163 min_rank=1 imbalance_ratio=1.012146 max_load_reduction=114489 max_load_reduction_pct=0.873482 moved_to_echo=65762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 4094, 1, 0, 2, 4090, 4094, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3807, 3, 1, 0, 0, 4094, 0, 1, 1, 0, 84, 4094, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 4095, 4095, 0, 0, 1, 0, 1, 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 4036, 1, 1, 0, 4095, 4095, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 4, 0, 1, 1, 0, 12, 1, 4094, 4094, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4094, 0, 4093, 4095, 4071, 224, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 4095, 0, 1, 3776, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 0, 5, 0, 1, 4095, 0, 4089, 1, 1, 1, 1, 4056, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16179, 16387, 16353, 16400, 16581, 16351, 16441] avg=16384.0000 max=16581 max_rank=5 min=16179 min_rank=1 imbalance_ratio=1.012024 max_load_reduction=114491 max_load_reduction_pct=0.873497 moved_to_echo=65773 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 4096, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3809, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 509, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 4096, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 10, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4095, 4096, 3586, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 4094, 0, 11, 3776, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4059, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16606, 16385, 16406, 16395, 16093, 16361, 16442] avg=16384.0000 max=16606 max_rank=1 min=16093 min_rank=5 imbalance_ratio=1.013550 max_load_reduction=114466 max_load_reduction_pct=0.873306 moved_to_echo=65291 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3809, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 138, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 4001, 0, 0, 0, 4096, 4096, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 0, 0, 5, 0, 0, 0, 0, 12, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 4096, 3958, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 4078, 0, 86, 3776, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 17, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4057, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16235, 16380, 16330, 16335, 16461, 16492, 16451] avg=16384.0000 max=16492 max_rank=6 min=16235 min_rank=1 imbalance_ratio=1.006592 max_load_reduction=114580 max_load_reduction_pct=0.874176 moved_to_echo=65739 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 32768, 128, 128, 0, 128, 0, 128, 128, 128, 128, 128, 128, 32768, 0, 256, 30208, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 30464, 128, 128, 0, 0, 32768, 0, 128, 128, 0, 2432, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 32768, 32768, 128, 128, 128, 32768, 32768, 0, 0, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 32768, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 32768, 128, 128, 128, 128, 32512, 128, 128, 128, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 32768, 128, 128, 128, 0, 128, 128, 128, 32768, 128, 32256, 128, 128, 128, 128, 32512, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 32768, 0, 32768, 32768, 31104, 1792, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 32768, 32768, 32768, 128, 0, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4052, 0, 0, 11, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4090, 0, 4092, 0, 0, 4092, 0, 276, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 184, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 19, 0, 0, 0, 0, 9, 4095, 0, 0, 0, 0, 2016, 1827, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4011, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1, 4094, 1, 2, 0, 0, 0, 0, 4094, 4094, 1, 2, 0, 0, 1, 0, 0, 4094, 4078, 0, 4095, 2, 4093, 4095, 2, 0, 1, 0, 4094, 0, 3, 4, 1, 24, 0, 2, 4093, 1, 0, 0, 13, 0, 1, 4095, 2, 0, 0, 1, 4095, 1, 4093, 1, 1, 4095, 1, 4094, 0, 4094, 1, 1, 4093, 0, 45, 0, 4095, 0, 0, 1, 1, 4075, 1, 4095, 4094, 0, 188, 4091, 4094, 1, 2, 1, 1, 4085, 4094, 1, 34, 0, 2, 1, 0, 1, 2, 0, 4095, 2, 12, 1, 0, 2, 1, 1, 4095, 1, 0, 1, 1, 3892, 916, 1, 0, 0, 1, 0, 0, 1, 0, 4095, 3110, 4088, 0, 2, 1, 0, 0, 4094, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 6, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 3971, 0, 4, 21, 0, 4090, 0, 4096, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 4085, 0, 1, 0, 4091, 0, 0, 0, 0, 4096, 0, 4092, 4096, 0, 485, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 9, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 2368, 1585, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 3955, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 2, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 4096, 0, 66, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 480, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4089, 0, 4, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 3088, 540, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4045, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 7, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 4096, 0, 2, 0, 4096, 0, 0, 0, 0, 4090, 0, 4096, 4096, 0, 35, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 3, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 2958, 1236, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4067, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1, 4093, 1, 0, 0, 0, 1, 1, 4095, 4094, 0, 2, 0, 1, 1, 0, 0, 4094, 4074, 2, 4095, 2, 4094, 4095, 1, 0, 1, 0, 4095, 0, 3, 8, 1, 24, 0, 1, 4093, 1, 0, 0, 19, 0, 2, 4095, 1, 0, 0, 1, 4095, 1, 4092, 1, 0, 4094, 1, 4091, 0, 4095, 1, 1, 4095, 4, 2, 0, 4095, 0, 0, 1, 1, 3966, 0, 4095, 4094, 0, 97, 4094, 4094, 0, 0, 1, 0, 4089, 4095, 1, 145, 0, 1, 1, 0, 1, 0, 0, 4039, 3, 8, 0, 1, 1, 1, 0, 4094, 1, 0, 1, 1, 2237, 2948, 1, 1, 0, 1, 0, 0, 1, 0, 4095, 2911, 4090, 0, 0, 1, 5, 0, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[8, 4083, 6, 7, 9, 0, 5, 7, 4086, 4088, 7, 35, 0, 5, 8, 0, 0, 4081, 3899, 19, 4089, 5, 4085, 4086, 7, 0, 5, 0, 4085, 0, 7, 22, 7, 174, 0, 8, 4080, 4, 0, 0, 132, 0, 8, 4087, 8, 0, 0, 130, 4090, 7, 4054, 7, 1, 4080, 6, 4083, 0, 4088, 7, 7, 4087, 0, 408, 0, 4086, 0, 0, 0, 6, 3803, 0, 4089, 4076, 0, 1543, 4064, 4085, 3, 10, 6, 4, 3988, 4087, 7, 252, 0, 0, 9, 0, 5, 0, 7, 4089, 3, 0, 0, 3, 6, 7, 0, 4088, 1, 0, 6, 5, 2582, 1364, 6, 2, 0, 7, 0, 0, 8, 0, 4088, 2174, 4004, 0, 69, 13, 18, 5, 4087, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 93, 0, 4096, 0, 4096, 0, 0, 4095, 0, 4092, 0, 4096, 0, 0, 4096, 0, 1, 0, 4096, 0, 0, 0, 0, 4088, 0, 4096, 4096, 0, 28, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3575, 0, 0, 0, 0, 0, 0, 23, 4096, 0, 0, 0, 0, 3589, 1590, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 3501, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 24, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 3856, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 0, 0, 0, 4096, 21, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4052, 0, 0, 11, 0, 4096, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4006, 0, 4090, 0, 0, 0, 0, 4092, 0, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3779, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 184, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 3987, 0, 19, 0, 3861, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4095, 0, 0, 0, 0, 2016, 1827, 0, 0, 0, 0, 0, 4096, 3802, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4011, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 317, 219, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 16414, 16316, 16560, 16251, 16358, 15881, 16919] avg=16384.0000 max=16919 max_rank=7 min=15881 min_rank=6 imbalance_ratio=1.032654 max_load_reduction=114153 max_load_reduction_pct=0.870918 moved_to_echo=65409 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 3856, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4096, 0, 0, 0, 4096, 21, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 3971, 0, 4, 21, 0, 4096, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4006, 0, 4095, 0, 0, 0, 0, 4085, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3775, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 485, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 9, 0, 3861, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 2368, 1585, 0, 0, 0, 0, 0, 4090, 3806, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 3955, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 19, 0, 316, 219, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16363, 16422, 16249, 16283, 16548, 16343, 15995, 16869] avg=16384.0000 max=16869 max_rank=7 min=15995 min_rank=6 imbalance_ratio=1.029602 max_load_reduction=114203 max_load_reduction_pct=0.871300 moved_to_echo=65755 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 4094, 1, 2, 0, 0, 0, 0, 4094, 4094, 1, 2, 0, 0, 1, 0, 3855, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4078, 0, 0, 2, 4093, 21, 2, 0, 1, 0, 4094, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 24, 0, 2, 4093, 1, 0, 0, 13, 0, 1, 4095, 2, 0, 0, 1, 4095, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 1, 1, 4005, 1, 4094, 0, 0, 1, 1, 4093, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 0, 0, 1, 1, 4075, 1, 0, 4094, 0, 188, 4091, 0, 1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 4094, 1, 34, 0, 2, 1, 0, 1, 2, 0, 4095, 2, 12, 1, 3859, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 4095, 1, 0, 1, 1, 3892, 916, 1, 0, 0, 1, 0, 4095, 3804, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 3110, 4088, 0, 2, 1, 0, 0, 4094, 0, 0, 0, 0, 317, 219, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16392, 16387, 16336, 16233, 16480, 16847, 16017] avg=16384.0000 max=16847 max_rank=6 min=16017 min_rank=7 imbalance_ratio=1.028259 max_load_reduction=114225 max_load_reduction_pct=0.871468 moved_to_echo=65577 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 4093, 1, 0, 0, 0, 1, 1, 4095, 4094, 0, 2, 0, 1, 1, 0, 3855, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4074, 2, 0, 2, 4094, 21, 1, 0, 1, 0, 4095, 0, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 24, 0, 1, 4093, 1, 0, 0, 19, 0, 2, 4095, 1, 0, 0, 1, 4095, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 1, 0, 4004, 1, 4091, 0, 0, 1, 1, 4095, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 0, 0, 1, 1, 3966, 0, 0, 4094, 0, 97, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4095, 1, 145, 0, 1, 1, 0, 1, 0, 0, 4039, 3, 8, 0, 3858, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 4094, 1, 0, 1, 1, 2237, 2948, 1, 1, 0, 1, 0, 4095, 3804, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 2911, 4090, 0, 0, 1, 5, 0, 4095, 0, 0, 0, 0, 317, 219, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16395, 16392, 16293, 16032, 16532, 17223, 15824] avg=16384.0000 max=17223 max_rank=6 min=15824 min_rank=7 imbalance_ratio=1.051208 max_load_reduction=113849 max_load_reduction_pct=0.868599 moved_to_echo=65611 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[8, 4083, 6, 7, 9, 0, 5, 7, 4086, 4088, 7, 35, 0, 5, 8, 0, 3846, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 3899, 19, 0, 5, 4085, 22, 7, 0, 5, 0, 4085, 0, 7, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 174, 0, 8, 4080, 4, 0, 0, 132, 0, 8, 4087, 8, 0, 0, 130, 4089, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4054, 7, 1, 3991, 6, 4083, 0, 0, 7, 7, 4087, 0, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3770, 0, 0, 0, 6, 3803, 0, 0, 4076, 0, 1543, 4064, 0, 3, 10, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3988, 4087, 7, 252, 0, 0, 9, 0, 5, 0, 7, 4089, 3, 0, 0, 3851, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6, 7, 0, 4088, 1, 0, 6, 5, 2582, 1364, 6, 2, 0, 7, 0, 4090, 3797, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4088, 2174, 4004, 0, 69, 13, 18, 5, 4087, 0, 0, 0, 0, 316, 218, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16434, 16237, 16780, 16658, 17281, 16593, 16000, 15089] avg=16384.0000 max=17281 max_rank=4 min=15089 min_rank=7 imbalance_ratio=1.054749 max_load_reduction=113791 max_load_reduction_pct=0.868156 moved_to_echo=64963 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3860, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 7, 0, 4096, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4004, 0, 4095, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3776, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 35, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 3, 0, 3860, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 2958, 1236, 0, 0, 0, 0, 0, 4096, 3805, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4067, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 320, 220, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16401, 16281, 16293, 16093, 16408, 16229, 16985] avg=16384.0000 max=16985 max_rank=7 min=16093 min_rank=4 imbalance_ratio=1.036682 max_load_reduction=114087 max_load_reduction_pct=0.870415 moved_to_echo=65715 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3856, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 21, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 2, 0, 4096, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4006, 0, 4095, 0, 0, 0, 0, 4096, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3779, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 480, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4089, 0, 4, 0, 3860, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 3088, 540, 0, 0, 0, 0, 0, 4096, 3805, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4045, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 317, 219, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16406, 16322, 16359, 16547, 16439, 15662, 16957] avg=16384.0000 max=16957 max_rank=7 min=15662 min_rank=6 imbalance_ratio=1.034973 max_load_reduction=114115 max_load_reduction_pct=0.870628 moved_to_echo=65605 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3856, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 21, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 93, 0, 4096, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4005, 0, 4092, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3779, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 28, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3575, 0, 0, 0, 3861, 291, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4096, 0, 0, 0, 0, 3589, 1590, 0, 0, 0, 0, 0, 4096, 3805, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 3501, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 24, 0, 317, 219, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16405, 16345, 16290, 16087, 15919, 17235, 16412] avg=16384.0000 max=17235 max_rank=6 min=15919 min_rank=5 imbalance_ratio=1.051941 max_load_reduction=113837 max_load_reduction_pct=0.868507 moved_to_echo=65653 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 128, 128, 22784, 12032, 128, 128, 0, 128, 0, 32768, 30464, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 32768, 128, 128, 32128, 128, 32768, 0, 0, 128, 128, 32768, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 32640, 128, 0, 128, 32768, 256, 128, 0, 128, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 32768, 32768, 128, 512, 0, 128, 128, 128, 128, 128, 128, 32000, 128, 128, 128, 30976, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 128, 128, 32768, 32768, 128, 128, 0, 128, 128, 0, 30848, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 256, 0, 128, 32768, 128, 128, 0, 256, 0, 128, 32512, 128, 128, 256, 256, 32768, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[30336, 0, 0, 128, 128, 32384, 128, 0, 32768, 0, 3072, 32768, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 32768, 27776, 32768, 0, 128, 128, 128, 128, 32768, 0, 0, 128, 0, 2560, 1792, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 6, 43, 0, 4094, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 1418, 0, 0, 0, 1, 0, 0, 2245, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 4096, 4089, 0, 7, 4082, 0, 0, 0, 4096, 40, 4094, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 4, 0, 4095, 0, 0, 0, 4, 4096, 0, 12, 0, 0, 4, 4096, 0, 572, 0, 4085, 2, 0, 0, 3998, 4096, 4057, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4089, 0, 1, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4091, 0, 0, 4096, 4096, 8, 57, 0, 4095, 4092, 0, 14, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 1863, 0, 0, 0, 0, 0, 0, 600, 3, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 4, 0, 4096, 4083, 0, 9, 4076, 0, 0, 0, 4096, 35, 4087, 0, 4096, 0, 4093, 4096, 0, 0, 4096, 0, 4096, 0, 9, 0, 4088, 0, 0, 0, 9, 4092, 0, 17, 0, 0, 1, 4096, 0, 1750, 0, 4073, 1, 0, 0, 3972, 4095, 4049, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 1, 4, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2083, 0, 0, 0, 0, 0, 0, 1421, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 4096, 4095, 0, 0, 4093, 0, 0, 0, 4096, 4, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4, 4096, 0, 4, 0, 0, 1, 4096, 0, 620, 0, 4093, 1, 0, 0, 4066, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 7, 2, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 1518, 0, 0, 0, 0, 0, 0, 1408, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 4087, 0, 0, 0, 4096, 23, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 17, 4096, 0, 10, 0, 0, 17, 4096, 0, 1191, 0, 4079, 35, 0, 0, 4064, 4096, 4069, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4095, 0, 1, 6, 3957, 0, 0, 0, 0, 1, 0, 4, 4069, 1, 2, 0, 1, 4094, 2, 0, 4095, 4095, 0, 0, 0, 4095, 4089, 0, 0, 4095, 4094, 1, 1, 1, 0, 10, 1, 1, 4083, 0, 2154, 1, 0, 1, 0, 14, 3, 974, 0, 1, 1, 1, 1, 0, 4095, 1, 0, 4095, 0, 1, 1, 0, 0, 1, 4095, 4095, 1, 0, 4090, 1, 1, 0, 4094, 21, 4095, 1, 4093, 0, 4095, 4094, 1, 0, 4095, 0, 4095, 1, 0, 0, 4095, 1, 0, 2, 6, 4095, 0, 0, 0, 0, 35, 4094, 1, 1067, 0, 4094, 61, 1, 0, 4085, 4095, 4032, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 2, 1, 1, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 4094, 0, 1, 6, 3941, 0, 0, 0, 0, 2, 0, 0, 4057, 1, 3, 1, 2, 4095, 2, 1, 4094, 4094, 0, 0, 0, 4095, 4089, 0, 0, 4095, 4094, 1, 2, 2, 0, 16, 1, 1, 4087, 0, 579, 1, 0, 1, 0, 33, 2, 1765, 0, 1, 2, 1, 1, 1, 4092, 2, 0, 4094, 0, 2, 1, 0, 1, 1, 4094, 4093, 2, 0, 4092, 1, 0, 0, 4094, 6, 4094, 1, 4090, 0, 4094, 4094, 1, 0, 4095, 1, 4095, 1, 0, 0, 4095, 1, 0, 2, 3, 4095, 0, 0, 0, 0, 27, 4093, 5, 1861, 0, 4094, 28, 0, 0, 4095, 4094, 4069, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 2, 1, 1, 2, 1, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1, 4086, 0, 8, 63, 3726, 0, 0, 0, 3, 2, 0, 0, 3900, 8, 0, 10, 8, 4079, 8, 0, 4086, 4082, 0, 0, 0, 4089, 4031, 0, 0, 4084, 4086, 7, 10, 7, 0, 379, 10, 7, 4056, 0, 1730, 6, 0, 7, 0, 227, 0, 573, 0, 10, 6, 7, 7, 4, 4051, 6, 0, 4087, 10, 8, 7, 0, 1, 8, 4088, 4089, 8, 0, 4088, 8, 0, 0, 3992, 0, 4089, 10, 4068, 0, 4088, 4086, 0, 0, 4087, 0, 4081, 8, 0, 0, 4086, 7, 9, 11, 0, 4088, 0, 0, 0, 3, 115, 4083, 40, 1616, 112, 4087, 8, 0, 6, 4088, 4086, 4082, 0, 0, 0, 0, 4, 0, 5, 4087, 9, 10, 8, 5, 22, 12, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 31, 6, 0, 4096, 4093, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 2270, 0, 0, 0, 0, 0, 0, 999, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4060, 0, 0, 4079, 0, 0, 0, 4096, 131, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 61, 4096, 0, 55, 0, 0, 78, 4096, 0, 929, 0, 4052, 144, 0, 0, 3919, 4096, 3955, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4089, 0, 1, 0, 0, 0, 0, 0, 4089, 0, 0, 3022, 1182, 12, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 8, 57, 0, 0, 4092, 0, 14, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 1863, 0, 0, 0, 0, 0, 0, 4096, 4096, 2315, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 600, 3, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 4, 4095, 1778, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 9, 4076, 0, 0, 0, 4096, 35, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 9, 0, 4088, 0, 0, 0, 9, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 1, 2914, 0, 1750, 0, 4073, 1, 0, 0, 3972, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16502, 16445, 16458, 15738, 16393, 16378, 16777, 16381] avg=16384.0000 max=16777 max_rank=6 min=15738 min_rank=3 imbalance_ratio=1.023987 max_load_reduction=114295 max_load_reduction_pct=0.872002 moved_to_echo=65929 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3027, 1182, 12, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 6, 43, 0, 0, 4095, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 1418, 0, 0, 0, 1, 0, 0, 4096, 4096, 2317, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2245, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 7, 4096, 1779, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 7, 4082, 0, 0, 0, 4096, 40, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4, 0, 4095, 0, 0, 0, 4, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4, 2914, 0, 572, 0, 4085, 2, 0, 0, 3998, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16440, 16422, 16019, 17384, 16408, 16379, 15644, 16376] avg=16384.0000 max=17384 max_rank=3 min=15644 min_rank=6 imbalance_ratio=1.061035 max_load_reduction=113688 max_load_reduction_pct=0.867371 moved_to_echo=64807 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 1, 6, 3957, 0, 0, 0, 0, 1, 0, 4, 4069, 1, 2, 3029, 1182, 12, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 2, 0, 0, 4095, 0, 0, 0, 0, 4089, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 10, 1, 1, 4083, 0, 2154, 1, 0, 1, 0, 14, 3, 4095, 4095, 2317, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 974, 0, 1, 1, 1, 1, 0, 4095, 1, 0, 4095, 0, 1, 1, 0, 0, 4095, 1778, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 4090, 1, 1, 0, 4094, 21, 0, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 4095, 1, 0, 0, 4095, 1, 0, 2, 6, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 2912, 1, 1067, 0, 4094, 61, 1, 0, 4085, 0, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 2, 1, 1, 0, 1, 0, 0, 0, 4095, 4095, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16369, 16367, 16785, 16110, 16396, 16378, 16288, 16379] avg=16384.0000 max=16785 max_rank=2 min=16110 min_rank=3 imbalance_ratio=1.024475 max_load_reduction=114287 max_load_reduction_pct=0.871941 moved_to_echo=65441 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 3029, 1187, 13, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 7, 2, 0, 0, 4096, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 1518, 0, 0, 0, 0, 0, 0, 4096, 4096, 2318, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1408, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1778, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4087, 0, 0, 0, 4096, 23, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 17, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 17, 2909, 0, 1191, 0, 4079, 35, 0, 0, 4064, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16505, 16381, 16120, 16541, 16386, 16388, 16374, 16377] avg=16384.0000 max=16541 max_rank=3 min=16120 min_rank=2 imbalance_ratio=1.009583 max_load_reduction=114531 max_load_reduction_pct=0.873802 moved_to_echo=65525 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 1, 6, 3941, 0, 0, 0, 0, 2, 0, 0, 4057, 1, 3, 3027, 1181, 12, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4095, 2, 1, 0, 4094, 0, 0, 0, 0, 4089, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 16, 1, 1, 4087, 0, 579, 1, 0, 1, 0, 33, 2, 4094, 4095, 2316, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1765, 0, 1, 2, 1, 1, 1, 4092, 2, 0, 4094, 0, 2, 1, 0, 1, 4094, 1778, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 2, 0, 4092, 1, 0, 0, 4094, 6, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 1, 4095, 1, 0, 0, 4095, 1, 0, 2, 3, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2912, 5, 1861, 0, 4094, 28, 0, 0, 4095, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 2, 1, 1, 2, 1, 0, 2, 0, 4095, 4094, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16335, 16370, 15237, 16902, 16378, 16376, 17091, 16383] avg=16384.0000 max=17091 max_rank=6 min=15237 min_rank=2 imbalance_ratio=1.043152 max_load_reduction=113981 max_load_reduction_pct=0.869606 moved_to_echo=66228 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 4086, 0, 8, 63, 3726, 0, 0, 0, 3, 2, 0, 0, 3900, 8, 0, 3024, 1179, 12, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8, 4079, 8, 0, 0, 4082, 0, 0, 0, 0, 4031, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 10, 7, 0, 379, 10, 7, 4056, 0, 1730, 6, 0, 7, 0, 227, 0, 4086, 4084, 2313, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 0, 10, 6, 7, 7, 4, 4051, 6, 0, 4087, 10, 8, 7, 0, 1, 4086, 1775, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4089, 8, 0, 4088, 8, 0, 0, 3992, 0, 0, 10, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4081, 8, 0, 0, 4086, 7, 9, 11, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 115, 2904, 40, 1616, 112, 4087, 8, 0, 6, 4088, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 4087, 9, 10, 8, 5, 22, 12, 0, 2, 0, 4087, 4088, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16022, 16296, 16936, 15703, 16269, 16364, 17061, 16421] avg=16384.0000 max=17061 max_rank=6 min=15703 min_rank=3 imbalance_ratio=1.041321 max_load_reduction=114011 max_load_reduction_pct=0.869835 moved_to_echo=66115 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 3029, 1182, 12, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 4, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2083, 0, 0, 0, 0, 0, 0, 4096, 4096, 2318, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1421, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 8, 4095, 1778, 1067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4093, 0, 0, 0, 4096, 4, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 2914, 0, 620, 0, 4093, 1, 0, 0, 4066, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16510, 16380, 16695, 16561, 16382, 16376, 15791, 16377] avg=16384.0000 max=16695 max_rank=2 min=15791 min_rank=6 imbalance_ratio=1.018982 max_load_reduction=114377 max_load_reduction_pct=0.872627 moved_to_echo=64926 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3028, 1182, 12, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 31, 6, 0, 0, 4093, 0, 2, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 2270, 0, 0, 0, 0, 0, 0, 4096, 4096, 2318, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 999, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1778, 1068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 4079, 0, 0, 0, 4096, 131, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 61, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 78, 2914, 0, 929, 0, 4052, 144, 0, 0, 3919, 0, 3955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16411, 16822, 16133, 16460, 16433, 16046, 16378] avg=16384.0000 max=16822 max_rank=2 min=16046 min_rank=6 imbalance_ratio=1.026733 max_load_reduction=114250 max_load_reduction_pct=0.871658 moved_to_echo=65317 active_clone_links=15 active_echo_slots_local=15 active_home_experts_offloading=12 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 0, 128, 128, 31872, 0, 128, 0, 128, 128, 0, 128, 32512, 128, 128, 24320, 9472, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 32768, 128, 128, 0, 0, 32768, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 384, 23296, 128, 9728, 128, 32768, 384, 128, 128, 32384, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 0, 128, 0, 128, 32768, 128, 128, 128, 128, 128, 128, 0, 128, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 0, 0, 128, 32768, 128, 128, 0, 32768, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 32768, 128, 128, 32768, 128, 128, 0, 32768, 384, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 0, 512, 128, 128, 32640, 0, 13696, 128, 0, 128, 128, 384, 128, 32768, 32768, 18560, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[10112, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 128, 128, 0, 128, 32768, 14336, 8576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4084, 0, 0, 0, 4094, 0, 4094, 0, 0, 1, 1, 4094, 2, 0, 0, 0, 4095, 0, 4094, 0, 0, 0, 2, 2, 3, 4094, 4094, 1, 4094, 0, 0, 0, 2, 2, 1, 2, 0, 0, 1, 2, 4, 4095, 4094, 1, 2, 1, 4093, 0, 1, 0, 2, 4094, 2, 0, 4094, 0, 0, 0, 4095, 0, 0, 1, 0, 4090, 1, 0, 2, 2, 4095, 4093, 0, 1, 0, 4095, 1, 0, 4094, 4094, 1, 3846, 0, 2, 1, 0, 2, 1, 4094, 2945, 0, 1, 0, 0, 4095, 0, 1, 4092, 0, 2, 0, 4090, 4094, 0, 0, 0, 4, 0, 1, 1408, 4094, 1, 0, 1, 1, 0, 2, 0, 2, 4094, 0, 0, 0, 1, 4094, 4095, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 8, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 38, 0, 21, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4, 0, 4096, 0, 0, 4096, 6, 0, 4, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4091, 4096, 0, 3206, 0, 0, 0, 0, 0, 0, 4096, 3086, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1833, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 28, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 25, 0, 19, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 4095, 10, 0, 4, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4095, 0, 0, 4080, 4096, 0, 4015, 0, 0, 0, 0, 0, 0, 4096, 3572, 0, 0, 0, 0, 4082, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 549, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 3968, 0, 11, 0, 4088, 0, 4086, 0, 0, 11, 0, 4085, 7, 0, 0, 0, 4086, 5, 4081, 0, 5, 0, 8, 10, 9, 4088, 4088, 10, 4080, 0, 0, 1, 0, 10, 5, 9, 6, 0, 8, 9, 92, 4087, 4086, 3, 4, 5, 4074, 0, 3, 0, 7, 4083, 10, 0, 4089, 0, 0, 0, 4087, 0, 0, 7, 0, 4061, 11, 0, 5, 9, 4086, 4082, 0, 12, 0, 4087, 6, 8, 4088, 4088, 7, 4071, 0, 3, 7, 7, 22, 7, 4084, 1265, 26, 6, 2, 0, 4089, 0, 6, 4083, 0, 6, 7, 3928, 4068, 0, 0, 0, 5, 20, 6, 3028, 4085, 5, 0, 6, 8, 0, 9, 5, 1, 4079, 0, 0, 0, 0, 4080, 4087, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 161, 4096, 4, 4096, 0, 3, 0, 0, 4096, 0, 32, 0, 48, 4090, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 4092, 33, 0, 10, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4091, 0, 0, 4067, 4096, 0, 3615, 0, 0, 0, 0, 0, 0, 4096, 2977, 0, 0, 0, 0, 4059, 0, 0, 4091, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 1391, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 8, 0, 11, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4094, 4096, 0, 3947, 0, 0, 0, 0, 0, 0, 4096, 1453, 0, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 2773, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 4094, 0, 2, 0, 4095, 0, 4095, 0, 0, 1, 0, 4095, 1, 1, 0, 0, 4095, 0, 4091, 0, 1, 0, 1, 1, 3, 4095, 4095, 1, 4095, 0, 0, 0, 2, 2, 1, 1, 0, 0, 1, 1, 0, 4095, 4094, 0, 1, 1, 4092, 0, 0, 2, 1, 4094, 2, 0, 4095, 0, 0, 0, 4095, 1, 2, 1, 0, 4093, 2, 0, 0, 1, 4094, 4094, 1, 1, 0, 4095, 1, 1, 4092, 4095, 1, 3860, 2, 1, 1, 4, 1, 1, 4095, 3659, 0, 0, 0, 0, 4095, 0, 1, 4093, 0, 0, 0, 4089, 4094, 0, 0, 0, 5, 2, 1, 669, 4094, 1, 0, 1, 1, 0, 1, 0, 1, 4094, 0, 0, 0, 0, 4094, 4095, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 39, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 140, 0, 39, 4096, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 111, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4058, 4096, 0, 3675, 0, 0, 0, 0, 0, 0, 4096, 3567, 0, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 19, 0, 0, 660, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2, 4096, 0, 0, 161, 4053, 4, 4096, 0, 3, 0, 0, 4096, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 4090, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 4092, 33, 0, 10, 4096, 0, 0, 0, 0, 4062, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 4091, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3615, 0, 0, 0, 0, 0, 0, 4096, 2977, 0, 0, 0, 0, 4059, 0, 0, 1518, 43, 19, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 1391, 2578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16543, 16412, 16366, 16404, 16365, 16341, 16254, 16387] avg=16384.0000 max=16543 max_rank=0 min=16254 min_rank=6 imbalance_ratio=1.009705 max_load_reduction=114529 max_load_reduction_pct=0.873787 moved_to_echo=65347 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 28, 4053, 0, 4096, 0, 0, 0, 0, 4096, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4096, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4095, 10, 0, 4, 4096, 0, 0, 0, 0, 4062, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 4095, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 4096, 3572, 0, 0, 0, 0, 4082, 0, 0, 1518, 43, 19, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 549, 2578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16394, 16390, 16370, 16375, 16382, 17359, 15415, 16387] avg=16384.0000 max=17359 max_rank=5 min=15415 min_rank=6 imbalance_ratio=1.059509 max_load_reduction=113713 max_load_reduction_pct=0.867561 moved_to_echo=65543 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 8, 4050, 0, 4096, 0, 0, 0, 0, 4096, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 4096, 6, 0, 4, 4096, 0, 0, 0, 0, 4068, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3206, 0, 0, 0, 0, 0, 0, 4096, 3086, 0, 0, 0, 0, 4091, 0, 0, 1524, 46, 23, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1833, 2572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16381, 16371, 16381, 16384, 16092, 16693, 16386] avg=16384.0000 max=16693 max_rank=6 min=16092 min_rank=5 imbalance_ratio=1.018860 max_load_reduction=114379 max_load_reduction_pct=0.872643 moved_to_echo=65555 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4084, 0, 0, 0, 4051, 0, 4094, 0, 0, 1, 1, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4094, 0, 0, 0, 2, 2, 3, 0, 4080, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 1, 2, 0, 0, 1, 2, 4, 4095, 4094, 1, 2, 1, 4093, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 4094, 2, 0, 4094, 0, 0, 0, 4095, 0, 0, 1, 0, 4061, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 2, 2, 15, 4093, 0, 1, 0, 4095, 1, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 0, 2, 1, 0, 2, 1, 4094, 2945, 0, 1, 0, 0, 4095, 0, 1, 1518, 43, 19, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 0, 4090, 4094, 0, 0, 0, 4, 0, 1, 1408, 2576, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 2, 0, 2, 4094, 0, 0, 0, 1, 4094, 4095, 1, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16327, 16371, 16380, 16361, 16395, 16582, 16268, 16388] avg=16384.0000 max=16582 max_rank=5 min=16268 min_rank=6 imbalance_ratio=1.012085 max_load_reduction=114490 max_load_reduction_pct=0.873489 moved_to_echo=65633 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 2, 0, 4052, 0, 4095, 0, 0, 1, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4091, 0, 1, 0, 1, 1, 3, 0, 4081, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 1, 1, 0, 0, 1, 1, 0, 4095, 4094, 0, 1, 1, 4092, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4094, 2, 0, 4095, 0, 0, 0, 4095, 1, 2, 1, 0, 4060, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 1, 15, 4094, 1, 1, 0, 4095, 1, 1, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3860, 2, 1, 1, 4, 1, 1, 4095, 3659, 0, 0, 0, 0, 4095, 0, 1, 1518, 43, 19, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4089, 4094, 0, 0, 0, 5, 2, 1, 669, 2576, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 1, 4094, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16341, 16369, 16372, 16364, 16397, 17314, 15530, 16385] avg=16384.0000 max=17314 max_rank=5 min=15530 min_rank=6 imbalance_ratio=1.056763 max_load_reduction=113758 max_load_reduction_pct=0.867905 moved_to_echo=65626 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 3968, 0, 11, 0, 4045, 0, 4086, 0, 0, 11, 0, 4085, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 5, 4081, 0, 5, 0, 8, 10, 9, 0, 4074, 10, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 10, 5, 9, 6, 0, 8, 9, 92, 4087, 4086, 3, 4, 5, 4074, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 4083, 10, 0, 4089, 0, 0, 0, 4087, 0, 0, 7, 0, 4052, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 11, 0, 5, 9, 15, 4082, 0, 12, 0, 4087, 6, 8, 4088, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 3, 7, 7, 22, 7, 4084, 1265, 26, 6, 2, 0, 4089, 0, 6, 1514, 43, 19, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 6, 7, 3928, 4068, 0, 0, 0, 5, 20, 6, 3028, 2571, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8, 0, 9, 5, 1, 4079, 0, 0, 0, 0, 4080, 4087, 0, 0, 0, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16213, 16368, 16473, 16349, 16391, 15185, 17727, 16366] avg=16384.0000 max=17727 max_rank=6 min=15185 min_rank=5 imbalance_ratio=1.081970 max_load_reduction=113345 max_load_reduction_pct=0.864754 moved_to_echo=65669 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 4053, 0, 4096, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4, 0, 0, 4096, 0, 0, 0, 0, 4062, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 4096, 1453, 0, 0, 0, 0, 4092, 0, 0, 1518, 43, 19, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 2773, 2578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16350, 16381, 16370, 16366, 16397, 15182, 17640, 16386] avg=16384.0000 max=17640 max_rank=6 min=15182 min_rank=5 imbalance_ratio=1.076660 max_load_reduction=113432 max_load_reduction_pct=0.865417 moved_to_echo=65605 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 39, 4053, 0, 4096, 0, 0, 0, 0, 4096, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4096, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 111, 0, 0, 4096, 0, 0, 0, 0, 4062, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 4096, 0, 0, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3675, 0, 0, 0, 0, 0, 0, 4096, 3567, 0, 0, 0, 0, 4085, 0, 0, 1518, 43, 19, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 19, 0, 0, 660, 2578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16520, 16400, 16370, 16472, 16361, 17017, 15545, 16387] avg=16384.0000 max=17017 max_rank=5 min=15545 min_rank=6 imbalance_ratio=1.038635 max_load_reduction=114055 max_load_reduction_pct=0.870171 moved_to_echo=65310 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32640, 0, 128, 256, 32512, 128, 32768, 0, 128, 128, 128, 32768, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 32768, 256, 0, 128, 32768, 128, 128, 128, 0, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 0, 128, 128, 128, 32768, 128, 128, 0, 32768, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 32768, 32768, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 32768, 128, 32768, 0, 128, 0, 128, 128, 128, 0, 32640, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[30336, 128, 128, 128, 128, 128, 128, 32768, 22528, 128, 128, 128, 0, 32768, 0, 128, 12160, 384, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 0, 128, 128, 32640, 32768, 0, 0, 0, 128, 128, 128, 12416, 20608, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 128, 128, 32768, 0, 0, 0, 128, 32768, 32768, 128, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4080, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 4088, 0, 0, 214, 0, 0, 4096, 0, 0, 0, 0, 3870, 0, 0, 0, 4096, 0, 0, 4096, 4092, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 4096, 0, 4072, 1, 0, 24, 0, 0, 0, 4096, 4096, 11, 1, 0, 0, 4086, 5, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3824, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 4094, 0, 0, 72, 0, 0, 4096, 0, 0, 0, 0, 4063, 0, 0, 0, 4096, 0, 0, 4096, 4089, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 4096, 4096, 0, 4081, 0, 0, 122, 0, 0, 0, 4091, 4096, 14, 5, 0, 0, 4090, 4, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 11, 4071, 9, 3851, 4, 0, 0, 0, 8, 5, 10, 19, 8, 4088, 13, 0, 9, 184, 0, 0, 4081, 0, 0, 0, 0, 10, 7, 10, 4083, 4028, 6, 8, 13, 0, 4021, 4083, 4061, 4088, 9, 0, 0, 13, 0, 4084, 8, 0, 108, 0, 4079, 7, 8, 0, 4038, 5, 2, 4085, 4083, 8, 0, 4086, 4086, 10, 0, 4078, 0, 0, 4078, 0, 9, 0, 10, 7, 0, 9, 3, 8, 82, 1, 25, 15, 21, 864, 4087, 4083, 4, 3946, 201, 6, 0, 0, 0, 0, 4053, 4081, 19, 0, 1, 0, 4068, 0, 0, 3, 0, 0, 4085, 6, 0, 4, 7, 14, 74, 3729, 8, 0, 0, 0, 9, 4087, 4069, 11, 9, 4077, 11, 3517, 0, 0, 4085] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3346, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 37, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 91, 0, 0, 4096, 0, 0, 0, 0, 4032, 0, 0, 0, 4096, 0, 0, 4096, 4091, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 685, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 4096, 2, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 1, 4093, 1, 4035, 1, 2, 2, 0, 1, 1, 0, 2, 1, 4095, 0, 0, 1, 63, 0, 1, 4094, 1, 0, 1, 0, 0, 1, 2, 4094, 4087, 1, 1, 2, 0, 4081, 4094, 4093, 4095, 1, 0, 2, 2, 0, 4095, 2, 0, 0, 0, 4012, 1, 1, 1, 4089, 1, 0, 4094, 4091, 1, 0, 4095, 4095, 1, 0, 4093, 0, 0, 4092, 0, 1, 0, 1, 1, 0, 1, 1, 1, 4, 0, 0, 1, 3, 226, 4095, 4094, 1, 4040, 14, 0, 0, 0, 0, 0, 4092, 4094, 6, 3, 0, 0, 4090, 0, 0, 1, 0, 0, 4094, 2, 0, 0, 1, 1, 16, 3993, 1, 0, 0, 0, 1, 4095, 4094, 2, 1, 4093, 2, 4095, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1, 2, 4094, 2, 4061, 1, 2, 0, 0, 1, 1, 1, 2, 1, 4095, 0, 0, 2, 89, 0, 0, 4095, 1, 0, 0, 0, 0, 1, 2, 4094, 4087, 1, 1, 2, 0, 4094, 4094, 4093, 4094, 0, 0, 0, 2, 0, 4094, 2, 0, 10, 0, 3872, 1, 1, 1, 4087, 1, 0, 4094, 4092, 1, 0, 4094, 4094, 2, 0, 4091, 0, 0, 4093, 0, 2, 0, 1, 1, 0, 1, 2, 2, 16, 0, 5, 1, 3, 308, 4094, 4094, 1, 4057, 8, 0, 1, 0, 0, 0, 4080, 4094, 14, 0, 0, 0, 4087, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 13, 3981, 1, 0, 0, 0, 1, 4095, 4093, 1, 2, 4093, 3, 4079, 1, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4011, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 4040, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 4096, 4096, 0, 4095, 0, 0, 94, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 112, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 4096, 4096, 4096, 0, 0, 440, 0, 0, 4096, 0, 0, 0, 0, 3846, 0, 0, 0, 4096, 0, 0, 4096, 4094, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4096, 4096, 0, 4096, 0, 0, 35, 0, 0, 0, 4095, 4096, 5, 70, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4080, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4091, 0, 3967, 315, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3626, 313, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3991, 4096, 4088, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3870, 0, 0, 0, 4096, 0, 0, 4096, 4092, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 3783, 0, 4072, 1, 0, 24, 0, 0, 0, 4096, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4086, 5, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4063, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16585, 16331, 16485, 16283, 16368, 16102, 16371, 16547] avg=16384.0000 max=16585 max_rank=0 min=16102 min_rank=5 imbalance_ratio=1.012268 max_load_reduction=114487 max_load_reduction_pct=0.873466 moved_to_echo=65388 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3824, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3967, 315, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3626, 313, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3991, 4096, 4094, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 4096, 0, 0, 4096, 4089, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 3783, 0, 4081, 0, 0, 122, 0, 0, 0, 4091, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4090, 4, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4063, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16335, 16332, 16349, 16473, 16369, 16300, 16373, 16541] avg=16384.0000 max=16541 max_rank=7 min=16300 min_rank=5 imbalance_ratio=1.009583 max_load_reduction=114531 max_load_reduction_pct=0.873802 moved_to_echo=65583 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3346, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3969, 320, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3631, 315, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3989, 4096, 4096, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 0, 0, 4096, 0, 0, 4096, 4091, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 685, 0, 3781, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4062, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15866, 16378, 16368, 16442, 16369, 16751, 16369, 16529] avg=16384.0000 max=16751 max_rank=5 min=15866 min_rank=0 imbalance_ratio=1.022400 max_load_reduction=114321 max_load_reduction_pct=0.872200 moved_to_echo=66018 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 11, 4071, 9, 3851, 4, 0, 0, 0, 8, 5, 10, 19, 8, 4088, 13, 3957, 314, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 184, 0, 0, 4081, 0, 0, 0, 0, 10, 7, 10, 4083, 4028, 6, 3617, 312, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 13, 0, 4021, 3979, 4061, 4088, 9, 0, 0, 13, 0, 0, 8, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 7, 8, 0, 4038, 5, 2, 4085, 4083, 8, 0, 0, 129, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 4078, 0, 9, 0, 10, 7, 0, 9, 3, 8, 82, 1, 25, 4087, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 21, 864, 0, 3771, 4, 3946, 201, 6, 0, 0, 0, 0, 4053, 4081, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4068, 0, 0, 3, 0, 0, 4085, 6, 0, 4, 7, 14, 74, 4086, 4050, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3729, 8, 0, 0, 0, 9, 0, 4069, 11, 9, 4077, 11, 3517, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16402, 16451, 16308, 16454, 16470, 16981, 16412, 15594] avg=16384.0000 max=16981 max_rank=5 min=15594 min_rank=7 imbalance_ratio=1.036438 max_load_reduction=114091 max_load_reduction_pct=0.870445 moved_to_echo=65457 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4093, 1, 4035, 1, 2, 2, 0, 1, 1, 0, 2, 1, 4095, 0, 3966, 315, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 63, 0, 1, 4094, 1, 0, 1, 0, 0, 1, 2, 4094, 4087, 1, 3626, 312, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 4081, 3989, 4093, 4095, 1, 0, 2, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 1, 1, 1, 4089, 1, 0, 4094, 4091, 1, 0, 0, 129, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4092, 0, 1, 0, 1, 1, 0, 1, 1, 1, 4, 0, 0, 4095, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 226, 0, 3782, 1, 4040, 14, 0, 0, 0, 0, 0, 4092, 4094, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4090, 0, 0, 1, 0, 0, 4094, 2, 0, 0, 1, 1, 16, 4095, 4061, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 1, 0, 0, 0, 1, 0, 4094, 2, 1, 4093, 2, 4095, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16550, 16389, 16268, 16421, 16370, 16259, 16379, 16436] avg=16384.0000 max=16550 max_rank=0 min=16259 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=114522 max_load_reduction_pct=0.873734 moved_to_echo=65444 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 4094, 2, 4061, 1, 2, 0, 0, 1, 1, 1, 2, 1, 4095, 0, 3965, 315, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 89, 0, 0, 4095, 1, 0, 0, 0, 0, 1, 2, 4094, 4087, 1, 3626, 312, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 4094, 3989, 4093, 4094, 0, 0, 0, 2, 0, 0, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3872, 1, 1, 1, 4087, 1, 0, 4094, 4092, 1, 0, 0, 129, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4093, 0, 2, 0, 1, 1, 0, 1, 2, 2, 16, 0, 5, 4095, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 308, 0, 3782, 1, 4057, 8, 0, 1, 0, 0, 0, 4080, 4094, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 13, 4094, 4061, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 1, 0, 0, 0, 1, 0, 4093, 1, 2, 4093, 3, 4079, 1, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16577, 16415, 16287, 16281, 16388, 16349, 16366, 16409] avg=16384.0000 max=16577 max_rank=0 min=16281 min_rank=3 imbalance_ratio=1.011780 max_load_reduction=114495 max_load_reduction_pct=0.873528 moved_to_echo=65512 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4011, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3967, 315, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3626, 313, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3991, 4096, 4096, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 129, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 3783, 0, 4095, 0, 0, 94, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4062, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16520, 16332, 16290, 16457, 16369, 16199, 16366, 16539] avg=16384.0000 max=16539 max_rank=7 min=16199 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=114533 max_load_reduction_pct=0.873817 moved_to_echo=65473 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3967, 315, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3626, 313, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3991, 4096, 4096, 0, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 0, 0, 0, 4096, 0, 0, 4096, 4094, 0, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 3783, 0, 4096, 0, 0, 35, 0, 0, 0, 4095, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4063, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16237, 16444, 16717, 16261, 16369, 16131, 16436, 16477] avg=16384.0000 max=16717 max_rank=2 min=16131 min_rank=5 imbalance_ratio=1.020325 max_load_reduction=114355 max_load_reduction_pct=0.872459 moved_to_echo=65413 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 0, 32768, 0, 128, 0, 128, 128, 0, 128, 128, 128, 128, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 32768, 128, 30976, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 31744, 2560, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 128, 128, 0, 32768, 128, 0, 128, 128, 128, 128, 32768, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32128, 128, 0, 0, 128, 128, 0, 32768, 128, 128, 32768, 128, 32256, 128, 0, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 2304, 0, 30336, 128, 32512, 256, 128, 384, 0, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 31872, 128, 128, 128, 32768, 128, 128, 32768, 32768, 128, 0, 0, 1152, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 512, 0, 128, 32768, 128, 0, 128, 0, 128, 128, 128, 32768, 32768, 128, 29056, 2560, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 0, 32768, 32000, 32768, 32768, 128, 0, 896, 128, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4092, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4085, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 1, 0, 4096, 11, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4094, 1, 4093, 4094, 0, 0, 0, 4094, 0, 4090, 4094, 0, 0, 2, 4095, 9, 4092, 0, 4094, 0, 2, 0, 0, 4095, 4095, 1, 4094, 2, 2, 4094, 0, 0, 2, 0, 2, 0, 1, 0, 0, 1, 0, 0, 1, 2, 1, 2, 0, 4095, 0, 2, 0, 0, 1, 2, 4087, 5, 2, 4094, 2, 4093, 8, 0, 2, 0, 4094, 2, 3, 1, 1, 1, 0, 0, 4, 0, 2, 4083, 0, 1, 4095, 0, 4093, 0, 8, 2, 0, 4093, 0, 0, 0, 0, 4093, 0, 4090, 0, 0, 2, 0, 2, 0, 18, 0, 4094, 4, 0, 1, 4094, 0, 4094, 4076, 4, 1, 0, 2, 0, 2, 0, 0, 0, 0, 4089, 1, 4094, 0, 1, 4090, 1, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 4096, 0, 0, 0, 4088, 0, 4074, 4096, 2, 0, 0, 4096, 3, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4092, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 12, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 4, 0, 0, 4094, 0, 4, 0, 4096, 35, 4093, 4085, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4093, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 4096, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4084, 8, 4081, 4086, 0, 0, 0, 4088, 1, 4080, 4085, 0, 0, 15, 4084, 52, 4072, 6, 4082, 0, 10, 2, 0, 4088, 4088, 8, 4079, 9, 9, 4087, 0, 0, 10, 0, 0, 0, 10, 0, 9, 7, 0, 0, 5, 8, 0, 17, 0, 4088, 0, 16, 0, 5, 7, 17, 4029, 0, 11, 4086, 8, 4064, 29, 0, 9, 0, 4086, 10, 16, 20, 8, 7, 0, 0, 26, 0, 9, 3977, 0, 4, 4088, 0, 4076, 0, 17, 7, 0, 4079, 143, 0, 3, 4, 4078, 0, 4070, 0, 0, 11, 0, 9, 0, 106, 0, 4087, 35, 0, 9, 4085, 0, 4086, 3940, 26, 2, 0, 3, 0, 6, 4, 0, 0, 1, 4064, 7, 4069, 0, 6, 4056, 7, 4086] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4094, 1, 4093, 4094, 0, 0, 0, 4095, 0, 4091, 4094, 0, 0, 1, 4095, 4, 4093, 1, 4094, 0, 1, 0, 0, 4095, 4095, 1, 4095, 1, 1, 4095, 0, 0, 1, 0, 1, 0, 2, 0, 1, 1, 0, 0, 1, 1, 1, 2, 0, 4095, 0, 2, 1, 0, 1, 2, 4091, 0, 1, 4095, 1, 4094, 5, 0, 1, 0, 4095, 1, 2, 2, 1, 1, 0, 0, 2, 0, 2, 4088, 0, 1, 4095, 3, 4093, 0, 11, 2, 0, 4093, 0, 0, 0, 0, 4094, 0, 4093, 0, 0, 2, 0, 1, 0, 1, 0, 4095, 7, 0, 1, 4094, 0, 4095, 4087, 1, 2, 0, 1, 0, 1, 0, 0, 0, 1, 4089, 1, 4094, 0, 0, 4090, 1, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4093, 0, 4096, 4096, 0, 0, 0, 4081, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4095, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 16, 4096, 4094, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4088, 0, 4074, 13, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 4096, 4083, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4080, 0, 4, 0, 4096, 35, 4093, 4085, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4095, 8, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16369, 16377, 16376, 16379, 16397, 16380, 16399, 16395] avg=16384.0000 max=16399 max_rank=6 min=16369 min_rank=0 imbalance_ratio=1.000916 max_load_reduction=114673 max_load_reduction_pct=0.874886 moved_to_echo=65571 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 4096, 4083, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4082, 0, 1, 0, 4096, 11, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4091, 8, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16393, 16379, 16379, 16379, 16393, 16371, 16387, 16391] avg=16384.0000 max=16393 max_rank=0 min=16371 min_rank=5 imbalance_ratio=1.000549 max_load_reduction=114679 max_load_reduction_pct=0.874931 moved_to_echo=65542 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 1, 4093, 0, 0, 0, 0, 4094, 0, 4090, 13, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4092, 0, 4091, 0, 2, 0, 0, 0, 0, 1, 4094, 2, 2, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 1, 0, 0, 1, 0, 0, 1, 2, 1, 2, 0, 4095, 4095, 4094, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 1, 2, 4087, 5, 2, 4094, 2, 4093, 8, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 3, 1, 1, 1, 0, 0, 4, 0, 2, 4083, 0, 1, 4095, 4081, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 8, 2, 0, 4085, 0, 0, 0, 0, 4093, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 18, 0, 4080, 4, 0, 1, 4094, 0, 4094, 4076, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 4089, 1, 4094, 0, 1, 4090, 1, 4094, 8, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16387, 16386, 16394, 16385, 16371, 16376, 16386] avg=16384.0000 max=16394 max_rank=3 min=16371 min_rank=5 imbalance_ratio=1.000610 max_load_reduction=114678 max_load_reduction_pct=0.874924 moved_to_echo=65518 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4086, 16, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 4096, 2, 4096, 4093, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 9, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16391, 16372, 16380, 16392, 16394, 16375, 16366, 16402] avg=16384.0000 max=16402 max_rank=7 min=16366 min_rank=6 imbalance_ratio=1.001099 max_load_reduction=114670 max_load_reduction_pct=0.874863 moved_to_echo=65537 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 1, 4093, 0, 0, 0, 0, 4095, 0, 4091, 13, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4093, 1, 4091, 0, 1, 0, 0, 0, 0, 1, 4095, 1, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 1, 1, 0, 0, 1, 1, 1, 2, 0, 4095, 4095, 4094, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 1, 0, 1, 2, 4091, 0, 1, 4095, 1, 4094, 5, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 2, 2, 1, 1, 0, 0, 2, 0, 2, 4088, 0, 1, 4095, 4081, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 11, 2, 0, 4085, 0, 0, 0, 0, 4094, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 1, 0, 4081, 7, 0, 1, 4094, 0, 4095, 4087, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 4089, 1, 4094, 0, 0, 4090, 1, 4095, 8, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16383, 16385, 16390, 16388, 16381, 16372, 16385] avg=16384.0000 max=16390 max_rank=3 min=16372 min_rank=6 imbalance_ratio=1.000366 max_load_reduction=114682 max_load_reduction_pct=0.874954 moved_to_echo=65526 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4083, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4096, 2, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 8, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16395, 16380, 16380, 16385, 16387, 16376, 16373, 16396] avg=16384.0000 max=16396 max_rank=7 min=16373 min_rank=6 imbalance_ratio=1.000732 max_load_reduction=114676 max_load_reduction_pct=0.874908 moved_to_echo=65532 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4084, 8, 4081, 0, 0, 0, 0, 4088, 1, 4080, 13, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 4072, 6, 4079, 0, 10, 2, 0, 0, 0, 8, 4079, 9, 9, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 10, 0, 9, 7, 0, 0, 5, 8, 0, 17, 0, 4088, 4084, 4086, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 16, 0, 5, 7, 17, 4029, 0, 11, 4086, 8, 4064, 29, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 10, 16, 20, 8, 7, 0, 0, 26, 0, 9, 3977, 0, 4, 4088, 4072, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 17, 7, 0, 4071, 143, 0, 3, 4, 4078, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 9, 0, 106, 0, 4073, 35, 0, 9, 4085, 0, 4086, 3940, 26, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 4, 0, 0, 1, 4064, 7, 4069, 0, 6, 4056, 7, 4086, 8, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16370, 16413, 16407, 16370, 16340, 16469, 16382, 16321] avg=16384.0000 max=16469 max_rank=5 min=16321 min_rank=7 imbalance_ratio=1.005188 max_load_reduction=114603 max_load_reduction_pct=0.874352 moved_to_echo=65512 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 4096, 0, 0, 0, 0, 4081, 0, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4095, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4083, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4096, 16, 4096, 4094, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 8, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16381, 16379, 16383, 16388, 16349, 16417, 16396] avg=16384.0000 max=16417 max_rank=6 min=16349 min_rank=5 imbalance_ratio=1.002014 max_load_reduction=114655 max_load_reduction_pct=0.874748 moved_to_echo=65550 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 32768, 0, 0, 0, 0, 32768, 128, 32768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 128, 0, 128, 128, 0, 128, 32640, 0, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 32768, 0, 128, 128, 0, 0, 0, 128, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 128, 128, 0, 128, 128, 0, 0, 128, 128, 128, 128, 0, 32768, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 32768, 256, 0, 128, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 128, 0, 32640, 128, 128, 128, 32768, 128, 32768, 32640, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 128, 32768, 128, 32768, 0, 128, 32768, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4095, 0, 9, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 4068, 4096, 4096, 0, 0, 0, 0, 4091, 0, 4096, 52, 4085, 0, 0, 0, 2, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 16, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4030, 4090, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 51, 4092, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4034, 4094, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4067, 2, 4095, 0, 1, 4090, 0, 0, 1, 4094, 0, 0, 0, 4093, 4095, 0, 0, 0, 4095, 0, 1, 1, 4092, 4095, 4095, 0, 1, 0, 4094, 1, 0, 4095, 0, 0, 3, 0, 1, 1, 0, 0, 0, 0, 1, 2, 0, 4094, 0, 1, 1, 2, 2, 0, 1, 0, 4094, 4094, 4093, 2, 1, 0, 4, 4094, 1, 4087, 1, 4085, 0, 1, 1, 0, 1, 0, 2, 0, 2, 1, 0, 0, 2, 0, 0, 4092, 0, 2, 0, 4093, 6, 4094, 1, 0, 4094, 4095, 0, 0, 0, 0, 0, 1, 1, 0, 4094, 1, 4095, 1, 0, 1, 0, 1, 2, 2, 0, 0, 2, 4094, 4094, 0, 0, 4094, 4094, 4093, 1, 14, 1, 5, 2, 4094, 25, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[3898, 10, 4078, 4, 0, 4075, 8, 0, 9, 4086, 0, 0, 0, 4082, 4088, 0, 4, 0, 4087, 0, 10, 8, 4050, 4083, 4085, 1, 12, 0, 4086, 9, 36, 4087, 0, 0, 28, 8, 1, 0, 0, 5, 0, 0, 11, 12, 0, 4086, 0, 11, 8, 10, 10, 0, 7, 0, 4084, 4084, 4073, 13, 0, 20, 0, 4085, 5, 4060, 0, 3986, 0, 9, 7, 0, 9, 0, 2, 0, 0, 9, 0, 0, 10, 0, 0, 4069, 0, 2, 0, 4078, 0, 4083, 5, 0, 4080, 4086, 0, 0, 0, 0, 0, 9, 19, 9, 4077, 9, 4087, 7, 0, 8, 0, 10, 11, 11, 0, 0, 13, 4083, 4084, 0, 0, 4086, 4085, 4087, 6, 137, 6, 16, 12, 4087, 159, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 57, 4084, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4039, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 9, 4090, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4087, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[4062, 2, 4094, 0, 1, 4090, 1, 0, 1, 4094, 1, 0, 0, 4093, 4094, 0, 0, 0, 4094, 0, 2, 1, 4092, 4094, 4094, 2, 1, 0, 4094, 1, 3, 4094, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 2, 2, 0, 4094, 0, 1, 1, 2, 2, 0, 2, 0, 4093, 4094, 4092, 5, 0, 1, 0, 4094, 1, 4089, 0, 4079, 0, 0, 2, 0, 1, 0, 1, 0, 2, 1, 0, 0, 2, 0, 0, 4091, 0, 2, 0, 4093, 1, 4094, 1, 0, 4094, 4094, 1, 2, 0, 0, 0, 2, 4, 0, 4094, 0, 4094, 1, 0, 1, 0, 2, 2, 2, 0, 0, 2, 4094, 4093, 1, 0, 4094, 4094, 4094, 1, 16, 1, 5, 2, 4094, 32, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 211, 4034, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 3928, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 32, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4084, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 4096, 4055, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4064, 41, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 4039, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16416, 16384, 16379, 16389, 16405, 16384, 16396, 16319] avg=16384.0000 max=16416 max_rank=0 min=16319 min_rank=7 imbalance_ratio=1.001953 max_load_reduction=114656 max_load_reduction_pct=0.874756 moved_to_echo=65504 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3898, 10, 4078, 4, 0, 4075, 8, 0, 9, 36, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 10, 8, 4050, 4083, 4085, 1, 12, 0, 4086, 9, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 8, 1, 0, 0, 5, 0, 0, 11, 12, 0, 4086, 0, 11, 4088, 4087, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 10, 10, 0, 7, 0, 4084, 11, 4073, 13, 0, 20, 0, 4085, 5, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 0, 9, 7, 0, 9, 0, 2, 0, 0, 9, 0, 0, 10, 0, 4086, 4087, 4044, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 2, 0, 4078, 0, 4083, 5, 0, 4080, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 19, 9, 4077, 9, 4087, 7, 0, 8, 0, 10, 11, 11, 0, 0, 4073, 4050, 35, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4080, 4084, 0, 0, 0, 4085, 4087, 6, 137, 6, 16, 12, 0, 159, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16200, 16384, 16418, 16386, 16255, 16324, 16418, 16687] avg=16384.0000 max=16687 max_rank=7 min=16200 min_rank=0 imbalance_ratio=1.018494 max_load_reduction=114385 max_load_reduction_pct=0.872688 moved_to_echo=65684 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 36, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4068, 11, 4096, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 4085, 0, 0, 0, 2, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 4096, 4095, 4054, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 16, 4096, 0, 4096, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4059, 35, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 4030, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16429, 16385, 16378, 16363, 16427, 16407, 16374, 16309] avg=16384.0000 max=16429 max_rank=0 min=16309 min_rank=7 imbalance_ratio=1.002747 max_load_reduction=114643 max_load_reduction_pct=0.874657 moved_to_echo=65517 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4067, 2, 4095, 0, 1, 4090, 0, 0, 1, 36, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4092, 4095, 4095, 0, 1, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 1, 0, 0, 0, 0, 1, 2, 0, 4094, 0, 1, 4095, 4095, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 1, 0, 4094, 11, 4093, 2, 1, 0, 4, 4094, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 0, 1, 1, 0, 1, 0, 2, 0, 2, 1, 0, 0, 2, 0, 4094, 4095, 4053, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 0, 4093, 6, 4094, 1, 0, 4094, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4094, 1, 4095, 1, 0, 1, 0, 1, 2, 2, 0, 0, 4083, 4058, 35, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4091, 4094, 0, 0, 0, 4094, 4093, 1, 14, 1, 5, 2, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16380, 16382, 16393, 16343, 16389, 16378, 16422] avg=16384.0000 max=16422 max_rank=7 min=16343 min_rank=4 imbalance_ratio=1.002319 max_load_reduction=114650 max_load_reduction_pct=0.874710 moved_to_echo=65532 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 36, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 11, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 4092, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 4096, 4095, 4054, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 4096, 0, 4096, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4060, 35, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 4034, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16421, 16385, 16378, 16383, 16417, 16396, 16375, 16317] avg=16384.0000 max=16421 max_rank=0 min=16317 min_rank=7 imbalance_ratio=1.002258 max_load_reduction=114651 max_load_reduction_pct=0.874718 moved_to_echo=65505 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4062, 2, 4094, 0, 1, 4090, 1, 0, 1, 36, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4092, 4094, 4094, 2, 1, 0, 4094, 1, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 2, 2, 0, 4094, 0, 1, 4094, 4094, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 2, 0, 4093, 11, 4092, 5, 0, 1, 0, 4094, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 2, 0, 1, 0, 1, 0, 2, 1, 0, 0, 2, 0, 4094, 4093, 4052, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2, 0, 4093, 1, 4094, 1, 0, 4094, 7, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 4094, 0, 4094, 1, 0, 1, 0, 2, 2, 2, 0, 0, 4083, 4058, 35, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4091, 4093, 1, 0, 0, 4094, 4094, 1, 16, 1, 5, 2, 0, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16385, 16380, 16393, 16333, 16386, 16381, 16433] avg=16384.0000 max=16433 max_rank=7 min=16333 min_rank=4 imbalance_ratio=1.002991 max_load_reduction=114639 max_load_reduction_pct=0.874626 moved_to_echo=65533 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 36, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 11, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4090, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 4096, 4054, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 4096, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4060, 35, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16420, 16384, 16379, 16394, 16356, 16392, 16375, 16372] avg=16384.0000 max=16420 max_rank=0 min=16356 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=114652 max_load_reduction_pct=0.874725 moved_to_echo=65495 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 36, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 11, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 4034, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 4096, 4095, 4054, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 4096, 0, 4096, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4060, 35, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 3928, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16420, 16385, 16378, 16371, 16536, 16394, 16375, 16213] avg=16384.0000 max=16536 max_rank=4 min=16213 min_rank=7 imbalance_ratio=1.009277 max_load_reduction=114536 max_load_reduction_pct=0.873840 moved_to_echo=65518 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32512, 128, 32768, 128, 128, 32768, 128, 0, 128, 384, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 0, 128, 128, 32768, 128, 32768, 128, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 128, 128, 0, 128, 0, 0, 128, 128, 0, 32768, 0, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 32768, 128, 0, 128, 0, 128, 128, 128, 0, 0, 32768, 32512, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 128, 32768, 128, 128, 128, 128, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 0, 128, 128, 32768, 32768, 32768, 128, 128, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 32768, 128, 0, 0, 32512, 32768, 128, 256, 128, 128, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[384, 32640, 0, 128, 128, 128, 128, 0, 128, 128, 128, 128, 0, 0, 128, 0, 32768, 32768, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[107, 502, 424, 0, 45, 1, 1163, 4, 1770, 192, 158, 2225, 1397, 1722, 0, 3943, 3109, 308, 1189, 700, 3021, 2232, 83, 98, 2, 67, 968, 516, 1979, 0, 1892, 618, 3780, 2, 18, 2039, 2017, 87, 1801, 1211, 834, 14, 429, 793, 0, 1704, 171, 1045, 2791, 3297, 2001, 85, 4, 6, 475, 1106, 0, 1997, 1223, 1540, 2072, 768, 2222, 12, 1965, 3251, 0, 811, 277, 96, 220, 611, 3731, 1524, 276, 9, 124, 5, 41, 1257, 1412, 2411, 0, 1090, 52, 2170, 847, 1925, 508, 0, 1418, 2844, 590, 1006, 1329, 4, 1757, 3371, 629, 298, 6, 2303, 983, 1082, 974, 600, 3, 1319, 2279, 780, 203, 477, 270, 788, 2396, 2834, 60, 418, 285, 803, 2049, 1388, 413, 2992, 38, 0, 1483, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[26, 198, 462, 0, 88, 0, 1310, 5, 919, 157, 61, 1732, 939, 1055, 0, 3954, 3420, 357, 1379, 1048, 3105, 2199, 11, 53, 4, 40, 1027, 424, 2126, 0, 1473, 487, 4081, 0, 11, 2123, 2253, 40, 2294, 1989, 1134, 4, 321, 498, 0, 1073, 230, 2004, 2555, 3834, 2191, 14, 5, 0, 558, 1560, 0, 1525, 690, 1581, 1908, 1149, 2246, 1, 2034, 3078, 0, 981, 394, 37, 345, 474, 3962, 1479, 210, 11, 117, 0, 32, 634, 2225, 2408, 0, 1157, 42, 2175, 805, 2159, 463, 5, 1657, 2738, 950, 603, 1360, 8, 1980, 2513, 769, 123, 6, 2374, 648, 1179, 903, 534, 4, 1784, 2349, 806, 113, 484, 243, 1008, 2530, 2124, 17, 246, 159, 636, 2035, 1542, 511, 2795, 27, 5, 2150, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[5, 218, 392, 0, 14, 1, 1335, 6, 645, 137, 84, 1609, 961, 992, 1, 3804, 3188, 387, 825, 1339, 2428, 2634, 4, 60, 4, 63, 1573, 392, 2337, 1, 1454, 556, 4083, 0, 7, 2256, 1495, 37, 2495, 1653, 1526, 0, 237, 775, 0, 1106, 383, 1951, 2478, 3374, 2012, 40, 0, 0, 610, 1662, 0, 1445, 724, 1391, 1497, 1354, 2459, 1, 2632, 3392, 1, 772, 390, 65, 299, 666, 3941, 1235, 300, 2, 130, 3, 52, 938, 1824, 2380, 0, 1279, 73, 1964, 458, 2293, 438, 2, 1578, 3173, 835, 674, 2018, 1, 1433, 3199, 671, 210, 2, 1807, 668, 1497, 1211, 470, 0, 867, 2320, 1074, 206, 332, 364, 874, 2408, 1764, 24, 283, 300, 781, 2169, 1862, 842, 2673, 30, 0, 2423, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[59, 393, 580, 0, 87, 0, 874, 7, 842, 151, 28, 1623, 1815, 825, 0, 3946, 3377, 275, 1673, 1861, 2474, 1891, 8, 63, 0, 9, 1496, 512, 2292, 0, 1404, 307, 4044, 1, 9, 1900, 1965, 28, 2964, 1343, 947, 9, 458, 681, 0, 1027, 188, 2119, 3083, 3506, 2130, 29, 0, 17, 848, 1719, 0, 1562, 598, 886, 2164, 836, 2277, 0, 2776, 3281, 0, 613, 339, 117, 474, 921, 4015, 1662, 280, 1, 184, 2, 85, 366, 1748, 2969, 1, 1551, 59, 2246, 480, 2759, 187, 6, 1491, 2775, 1044, 451, 1808, 0, 781, 2968, 720, 190, 0, 1767, 704, 1833, 1044, 527, 1, 1522, 2266, 720, 125, 240, 218, 859, 1482, 2472, 48, 112, 157, 881, 2450, 938, 643, 2350, 45, 0, 2108, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[30, 232, 334, 2, 14, 2, 1045, 3, 1018, 161, 94, 1848, 1095, 1455, 0, 4017, 3336, 465, 1419, 1030, 2655, 2200, 78, 149, 0, 46, 1412, 489, 1820, 4, 1735, 500, 3995, 1, 8, 2055, 1486, 39, 2260, 1620, 735, 5, 544, 596, 0, 1333, 318, 1765, 3239, 3653, 2369, 28, 0, 7, 844, 1625, 0, 1366, 756, 1329, 2197, 1390, 2008, 2, 1849, 3097, 0, 924, 316, 99, 266, 636, 3810, 1465, 452, 2, 169, 13, 56, 697, 1546, 2569, 0, 1078, 27, 1994, 886, 2184, 437, 9, 1764, 2952, 1048, 784, 1452, 0, 1623, 2737, 990, 230, 0, 2127, 518, 1098, 923, 456, 0, 1033, 2447, 866, 412, 371, 344, 780, 2356, 2503, 22, 454, 657, 672, 1999, 1250, 892, 2723, 11, 1, 1735, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[11, 207, 673, 0, 16, 0, 802, 0, 1072, 151, 72, 1993, 1365, 1312, 0, 3880, 3004, 273, 2261, 1261, 2643, 1958, 6, 59, 0, 16, 1540, 678, 2261, 1, 1800, 993, 4051, 0, 15, 2315, 2040, 21, 2411, 1498, 871, 5, 432, 895, 0, 1464, 168, 1990, 2939, 3299, 2301, 30, 0, 0, 832, 1453, 0, 1393, 900, 1218, 2004, 1757, 2423, 0, 2860, 3239, 0, 489, 306, 94, 517, 906, 3791, 2252, 314, 3, 83, 0, 52, 411, 2009, 2551, 0, 1485, 32, 1512, 545, 2269, 256, 4, 1630, 3043, 971, 363, 1152, 0, 832, 2647, 600, 126, 0, 2079, 690, 1085, 1125, 371, 1, 1617, 1849, 554, 85, 204, 297, 845, 1916, 2058, 28, 212, 237, 678, 2410, 1251, 461, 2767, 23, 0, 1852, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[35, 137, 489, 1, 17, 0, 1200, 3, 1435, 188, 84, 2130, 1299, 1126, 0, 3744, 2679, 536, 2301, 1929, 2378, 3034, 17, 95, 2, 49, 1072, 535, 3105, 0, 2198, 503, 4074, 1, 48, 2473, 2482, 71, 2268, 1467, 1116, 15, 530, 1000, 1, 1037, 108, 1623, 3018, 3609, 2362, 31, 0, 0, 805, 1457, 0, 1629, 727, 1198, 2510, 1000, 2357, 0, 2357, 3277, 2, 556, 253, 108, 235, 661, 3659, 1198, 308, 4, 118, 9, 74, 275, 1541, 2972, 0, 826, 65, 2117, 1299, 1916, 146, 8, 1189, 2094, 436, 534, 1473, 1, 1253, 3100, 208, 174, 1, 2181, 416, 1728, 835, 336, 1, 1109, 2295, 577, 171, 136, 172, 1639, 2114, 2487, 42, 186, 342, 663, 2403, 1024, 486, 2206, 13, 1, 1722, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[32, 337, 460, 0, 54, 0, 1296, 4, 875, 222, 45, 1937, 1461, 1937, 0, 3767, 2532, 280, 1317, 1447, 2786, 2250, 42, 77, 0, 70, 1333, 824, 2604, 0, 1877, 413, 4039, 0, 15, 2401, 1990, 43, 2305, 1630, 1488, 6, 545, 486, 0, 1471, 141, 1751, 2902, 3665, 2072, 16, 1, 2, 416, 1036, 0, 1474, 578, 1225, 2211, 913, 2229, 9, 3242, 3194, 0, 826, 246, 231, 302, 817, 3875, 1827, 325, 3, 117, 6, 96, 299, 1801, 2721, 1, 1229, 106, 2404, 906, 2106, 186, 12, 1576, 2549, 870, 675, 1054, 3, 1507, 3270, 540, 143, 0, 2047, 616, 1598, 884, 381, 0, 1616, 2403, 582, 199, 339, 216, 794, 2128, 1520, 65, 164, 212, 790, 2477, 1022, 373, 2358, 40, 1, 1842, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[107, 502, 424, 0, 45, 1, 1163, 4, 1770, 192, 158, 2225, 1397, 1722, 0, 3943, 3252, 1282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1580, 308, 1189, 700, 3021, 2232, 83, 98, 2, 67, 968, 516, 1979, 0, 1892, 618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2585, 2, 18, 2039, 2017, 87, 1801, 1211, 834, 14, 429, 793, 0, 1704, 171, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2791, 45, 2001, 85, 4, 6, 475, 1106, 0, 1997, 1223, 1540, 2072, 768, 2222, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1965, 3251, 0, 811, 277, 96, 220, 611, 3731, 1524, 276, 9, 124, 5, 41, 1257, 1294, 669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1412, 2411, 0, 1090, 52, 2170, 847, 1925, 508, 0, 1418, 893, 590, 1006, 1329, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1757, 3371, 629, 298, 6, 2303, 983, 1082, 974, 600, 3, 1319, 2279, 780, 203, 477, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 270, 788, 2396, 2834, 60, 418, 285, 803, 2049, 1388, 413, 2992, 38, 0, 1483, 8, 523, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18187, 15253, 14750, 16347, 16161, 15655, 17736, 16983] avg=16384.0000 max=18187 max_rank=0 min=14750 min_rank=2 imbalance_ratio=1.110046 max_load_reduction=112885 max_load_reduction_pct=0.861244 moved_to_echo=66535 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[26, 198, 462, 0, 88, 0, 1310, 5, 919, 157, 61, 1732, 939, 1055, 0, 3954, 3782, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1738, 357, 1379, 1048, 3105, 2199, 11, 53, 4, 40, 1027, 424, 2126, 0, 1473, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2790, 0, 11, 2123, 2253, 40, 2294, 1989, 1134, 4, 321, 498, 0, 1073, 230, 2004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2555, 52, 2191, 14, 5, 0, 558, 1560, 0, 1525, 690, 1581, 1908, 1149, 2246, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2034, 3078, 0, 981, 394, 37, 345, 474, 3962, 1479, 210, 11, 117, 0, 32, 634, 1424, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2225, 2408, 0, 1157, 42, 2175, 805, 2159, 463, 5, 1657, 859, 950, 603, 1360, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1980, 2513, 769, 123, 6, 2374, 648, 1179, 903, 534, 4, 1784, 2349, 806, 113, 484, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 1008, 2530, 2124, 17, 246, 159, 636, 2035, 1542, 511, 2795, 27, 5, 2150, 1, 565, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15923, 15471, 16764, 16035, 15856, 16876, 17295, 16852] avg=16384.0000 max=17295 max_rank=6 min=15471 min_rank=1 imbalance_ratio=1.055603 max_load_reduction=113777 max_load_reduction_pct=0.868050 moved_to_echo=66879 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[5, 218, 392, 0, 14, 1, 1335, 6, 645, 137, 84, 1609, 961, 992, 1, 3804, 3328, 1431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1620, 387, 825, 1339, 2428, 2634, 4, 60, 4, 63, 1573, 392, 2337, 1, 1454, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2792, 0, 7, 2256, 1495, 37, 2495, 1653, 1526, 0, 237, 775, 0, 1106, 383, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2478, 46, 2012, 40, 0, 0, 610, 1662, 0, 1445, 724, 1391, 1497, 1354, 2459, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2632, 3392, 1, 772, 390, 65, 299, 666, 3941, 1235, 300, 2, 130, 3, 52, 938, 1327, 747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1824, 2380, 0, 1279, 73, 1964, 458, 2293, 438, 2, 1578, 995, 835, 674, 2018, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1433, 3199, 671, 210, 2, 1807, 668, 1497, 1211, 470, 0, 867, 2320, 1074, 206, 332, 726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 874, 2408, 1764, 24, 283, 300, 781, 2169, 1862, 842, 2673, 30, 0, 2423, 0, 565, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14963, 15677, 16713, 15719, 16892, 16812, 16693, 17603] avg=16384.0000 max=17603 max_rank=7 min=14963 min_rank=0 imbalance_ratio=1.074402 max_load_reduction=113469 max_load_reduction_pct=0.865700 moved_to_echo=68000 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[59, 393, 580, 0, 87, 0, 874, 7, 842, 151, 28, 1623, 1815, 825, 0, 3946, 3459, 1251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1716, 275, 1673, 1861, 2474, 1891, 8, 63, 0, 9, 1496, 512, 2292, 0, 1404, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2765, 1, 9, 1900, 1965, 28, 2964, 1343, 947, 9, 458, 681, 0, 1027, 188, 2119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3083, 47, 2130, 29, 0, 17, 848, 1719, 0, 1562, 598, 886, 2164, 836, 2277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2776, 3281, 0, 613, 339, 117, 474, 921, 4015, 1662, 280, 1, 184, 2, 85, 366, 1406, 653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1748, 2969, 1, 1551, 59, 2246, 480, 2759, 187, 6, 1491, 871, 1044, 451, 1808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 781, 2968, 720, 190, 0, 1767, 704, 1833, 1044, 527, 1, 1522, 2266, 720, 125, 240, 719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 218, 859, 1482, 2472, 48, 112, 157, 881, 2450, 938, 643, 2350, 45, 0, 2108, 0, 560, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15940, 15981, 16404, 16196, 17175, 17671, 16127, 15578] avg=16384.0000 max=17671 max_rank=5 min=15578 min_rank=7 imbalance_ratio=1.078552 max_load_reduction=113401 max_load_reduction_pct=0.865181 moved_to_echo=66551 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[30, 232, 334, 2, 14, 2, 1045, 3, 1018, 161, 94, 1848, 1095, 1455, 0, 4017, 3604, 1331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1695, 465, 1419, 1030, 2655, 2200, 78, 149, 0, 46, 1412, 489, 1820, 4, 1735, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2731, 1, 8, 2055, 1486, 39, 2260, 1620, 735, 5, 544, 596, 0, 1333, 318, 1765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3239, 49, 2369, 28, 0, 7, 844, 1625, 0, 1366, 756, 1329, 2197, 1390, 2008, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1849, 3097, 0, 924, 316, 99, 266, 636, 3810, 1465, 452, 2, 169, 13, 56, 697, 1389, 695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1546, 2569, 0, 1078, 27, 1994, 886, 2184, 437, 9, 1764, 926, 1048, 784, 1452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1623, 2737, 990, 230, 0, 2127, 518, 1098, 923, 456, 0, 1033, 2447, 866, 412, 371, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 780, 2356, 2503, 22, 454, 657, 672, 1999, 1250, 892, 2723, 11, 1, 1735, 0, 553, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16285, 15697, 15496, 17209, 15935, 16704, 16542, 17204] avg=16384.0000 max=17209 max_rank=3 min=15496 min_rank=2 imbalance_ratio=1.050354 max_load_reduction=113863 max_load_reduction_pct=0.868706 moved_to_echo=66385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[35, 137, 489, 1, 17, 0, 1200, 3, 1435, 188, 84, 2130, 1299, 1126, 0, 3744, 3565, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1355, 536, 2301, 1929, 2378, 3034, 17, 95, 2, 49, 1072, 535, 3105, 0, 2198, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2777, 1, 48, 2473, 2482, 71, 2268, 1467, 1116, 15, 530, 1000, 1, 1037, 108, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3018, 44, 2362, 31, 0, 0, 805, 1457, 0, 1629, 727, 1198, 2510, 1000, 2357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2357, 3277, 2, 556, 253, 108, 235, 661, 3659, 1198, 308, 4, 118, 9, 74, 275, 1118, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1541, 2972, 0, 826, 65, 2117, 1299, 1916, 146, 8, 1189, 647, 436, 534, 1473, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1253, 3100, 208, 174, 1, 2181, 416, 1728, 835, 336, 1, 1109, 2295, 577, 171, 136, 729, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 1639, 2114, 2487, 42, 186, 342, 663, 2403, 1024, 486, 2206, 13, 1, 1722, 2, 568, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16402, 19109, 17017, 17138, 14710, 15170, 15250, 16276] avg=16384.0000 max=19109 max_rank=1 min=14710 min_rank=4 imbalance_ratio=1.166321 max_load_reduction=111963 max_load_reduction_pct=0.854210 moved_to_echo=61406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[11, 207, 673, 0, 16, 0, 802, 0, 1072, 151, 72, 1993, 1365, 1312, 0, 3880, 3254, 1372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1527, 273, 2261, 1261, 2643, 1958, 6, 59, 0, 16, 1540, 678, 2261, 1, 1800, 993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2769, 0, 15, 2315, 2040, 21, 2411, 1498, 871, 5, 432, 895, 0, 1464, 168, 1990, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2939, 45, 2301, 30, 0, 0, 832, 1453, 0, 1393, 900, 1218, 2004, 1757, 2423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2860, 3239, 0, 489, 306, 94, 517, 906, 3791, 2252, 314, 3, 83, 0, 52, 411, 1250, 716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2009, 2551, 0, 1485, 32, 1512, 545, 2269, 256, 4, 1630, 955, 971, 363, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 832, 2647, 600, 126, 0, 2079, 690, 1085, 1125, 371, 1, 1617, 1849, 554, 85, 204, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 845, 1916, 2058, 28, 212, 237, 678, 2410, 1251, 461, 2767, 23, 0, 1852, 0, 561, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16180, 17277, 16894, 17295, 17283, 15734, 14586, 15823] avg=16384.0000 max=17295 max_rank=3 min=14586 min_rank=6 imbalance_ratio=1.055603 max_load_reduction=113777 max_load_reduction_pct=0.868050 moved_to_echo=63426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[32, 337, 460, 0, 54, 0, 1296, 4, 875, 222, 45, 1937, 1461, 1937, 0, 3767, 3616, 1149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1287, 280, 1317, 1447, 2786, 2250, 42, 77, 0, 70, 1333, 824, 2604, 0, 1877, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2762, 0, 15, 2401, 1990, 43, 2305, 1630, 1488, 6, 545, 486, 0, 1471, 141, 1751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2902, 49, 2072, 16, 1, 2, 416, 1036, 0, 1474, 578, 1225, 2211, 913, 2229, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3242, 3194, 0, 826, 246, 231, 302, 817, 3875, 1827, 325, 3, 117, 6, 96, 299, 1054, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1801, 2721, 1, 1229, 106, 2404, 906, 2106, 186, 12, 1576, 800, 870, 675, 1054, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1507, 3270, 540, 143, 0, 2047, 616, 1598, 884, 381, 0, 1616, 2403, 582, 199, 339, 718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 794, 2128, 1520, 65, 164, 212, 790, 2477, 1022, 373, 2358, 40, 1, 1842, 1, 559, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17192, 16607, 17034, 15133, 17060, 16450, 16843, 14753] avg=16384.0000 max=17192 max_rank=0 min=14753 min_rank=7 imbalance_ratio=1.049316 max_load_reduction=113880 max_load_reduction_pct=0.868835 moved_to_echo=65106 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[19840, 25856, 128, 6016, 2560, 896, 2688, 5760, 30848, 12672, 2560, 128, 1152, 128, 512, 4992, 10368, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[23040, 384, 17536, 384, 128, 128, 5504, 11648, 0, 12416, 6272, 10368, 16640, 9216, 18304, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 2304, 3840, 128, 384, 128, 9088, 128, 8576, 1408, 640, 15104, 10368, 10496, 128, 31104, 27904, 10112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[11264, 23808, 5248, 1536, 128, 16768, 5248, 11136, 7936, 3712, 128, 10880, 18304, 6016, 1536, 2688, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[12544, 2944, 12416, 10624, 21504, 18432, 256, 768, 128, 384, 10496, 4480, 18560, 128, 13952, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[14208, 20992, 128, 9728, 512, 16640, 6272, 17664, 2688, 128, 12416, 7040, 6784, 5120, 11648, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[22016, 128, 256, 17664, 15744, 384, 18816, 12416, 8704, 128, 3584, 5760, 128, 10240, 1792, 14336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2176, 7680, 17408, 17792, 384, 2176, 2432, 6016, 18048, 10368, 4736, 20864, 256, 128, 15360, 128, 4480, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 3, 1, 4093, 4093, 0, 0, 190, 1, 1, 0, 3377, 0, 1, 4094, 9, 1, 0, 3, 4090, 3716, 4095, 0, 0, 4092, 4, 2, 4095, 0, 0, 708, 8, 0, 0, 2, 4093, 1, 0, 4095, 4088, 3, 1, 4088, 2, 1, 0, 0, 0, 1, 0, 4092, 9, 4095, 4094, 0, 2, 4093, 2164, 1, 43, 1, 0, 0, 1, 1, 0, 1, 1, 1, 4, 1, 0, 1, 1, 0, 2, 0, 0, 2, 4095, 0, 1, 4093, 8, 0, 1, 1, 1, 2499, 1, 0, 3466, 4094, 0, 1, 0, 4095, 0, 4095, 0, 1, 4029, 242, 1, 4, 0, 1, 4093, 0, 4093, 4095, 2, 4093, 1, 7, 4095, 1, 0, 0, 4095, 9, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 4095, 2, 1, 45, 0, 0, 0, 3684, 0, 0, 4096, 0, 0, 0, 0, 4095, 2778, 4096, 0, 0, 4091, 0, 0, 4096, 0, 2, 280, 143, 0, 0, 0, 4096, 0, 0, 4095, 4029, 106, 0, 4094, 0, 0, 0, 1, 5, 0, 0, 4096, 31, 4096, 4096, 0, 0, 4096, 2615, 0, 116, 0, 0, 4, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 46, 0, 0, 0, 0, 3085, 0, 0, 3475, 4096, 0, 0, 0, 4083, 45, 4084, 0, 0, 4055, 12, 0, 58, 0, 0, 4092, 0, 4095, 4094, 0, 4095, 0, 28, 4096, 0, 0, 0, 4038, 29, 0, 0, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 6, 4096, 5, 0, 4096, 4096, 0, 26, 13, 0, 0, 0, 2135, 0, 0, 4095, 4, 0, 0, 0, 4096, 3764, 4096, 0, 1, 4096, 0, 0, 4096, 0, 0, 475, 28, 0, 5, 0, 4096, 0, 0, 4096, 4077, 22, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4091, 3490, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4083, 0, 0, 4090, 1, 0, 0, 0, 0, 3858, 0, 0, 2559, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 4079, 13, 0, 0, 0, 0, 4072, 0, 4096, 4096, 0, 4096, 0, 14, 4096, 0, 0, 0, 4096, 7, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 4095, 0, 0, 4096, 4096, 0, 0, 109, 0, 0, 0, 3976, 1, 0, 4095, 8, 0, 0, 1, 4096, 3580, 4096, 0, 1, 4096, 0, 0, 4096, 0, 1, 409, 10, 0, 4, 0, 4095, 0, 0, 4095, 4089, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4093, 1253, 0, 22, 0, 1, 0, 0, 0, 0, 8, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4093, 2, 0, 0, 0, 1, 3846, 0, 0, 3174, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 4046, 39, 0, 5, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 1, 4095, 0, 0, 0, 4094, 3, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[2, 0, 0, 4093, 3, 1, 4094, 4093, 0, 4, 406, 0, 0, 0, 3135, 0, 2, 4094, 3, 1, 0, 0, 4095, 3608, 4094, 2, 2, 4094, 0, 2, 4094, 2, 2, 773, 11, 0, 0, 0, 4090, 0, 1, 4094, 4088, 0, 2, 4090, 1, 1, 2, 0, 4, 1, 0, 4092, 16, 4094, 4094, 0, 2, 4091, 1692, 2, 46, 0, 0, 4, 1, 0, 0, 2, 2, 2, 0, 1, 0, 0, 1, 0, 2, 0, 1, 0, 4095, 0, 0, 4090, 0, 0, 2, 0, 0, 3131, 2, 0, 3495, 4094, 2, 0, 1, 4092, 0, 4094, 0, 2, 4065, 104, 2, 1, 0, 2, 4093, 0, 4092, 4090, 1, 4094, 1, 9, 4092, 0, 1, 0, 4088, 5, 0, 1, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 5, 150, 0, 0, 0, 4084, 0, 0, 4096, 6, 0, 0, 0, 4096, 3423, 4095, 0, 0, 4090, 0, 1, 4096, 0, 0, 201, 4, 0, 8, 0, 4096, 0, 0, 4096, 4089, 2, 0, 4095, 0, 0, 2, 0, 0, 0, 1, 4095, 16, 4096, 4096, 0, 2, 4094, 889, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4092, 9, 0, 1, 0, 2, 3967, 0, 0, 3666, 4092, 0, 0, 2, 4094, 1, 4094, 0, 0, 4014, 20, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4091, 0, 1, 4095, 0, 0, 10, 4094, 1, 1, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 4082, 4090, 7, 10, 241, 1, 1, 0, 3290, 0, 5, 4096, 5, 0, 0, 0, 4082, 3077, 4096, 1, 0, 4096, 7, 0, 4091, 0, 120, 285, 302, 0, 0, 0, 4090, 0, 0, 4079, 4043, 40, 0, 4089, 0, 0, 20, 0, 0, 0, 1, 4070, 11, 4090, 4093, 3, 0, 4096, 1670, 0, 430, 3, 0, 64, 0, 1, 0, 2, 2, 7, 10, 2, 0, 0, 0, 0, 0, 0, 3, 5, 4096, 0, 0, 4085, 26, 0, 0, 0, 0, 3467, 0, 1, 3560, 4095, 0, 0, 0, 4096, 6, 4096, 0, 0, 4093, 0, 1, 9, 1, 1, 4030, 0, 4093, 4096, 2, 4089, 0, 27, 4091, 0, 5, 0, 3985, 18, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 25, 86, 0, 0, 0, 3894, 0, 0, 4096, 3, 0, 0, 1, 4096, 3112, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 146, 44, 0, 0, 0, 4096, 0, 0, 4096, 4094, 3, 0, 4093, 0, 0, 1, 0, 1, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4095, 1791, 0, 440, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 4066, 14, 0, 0, 0, 0, 3294, 0, 0, 3532, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4095, 22, 0, 0, 0, 0, 4087, 0, 4096, 4096, 0, 4096, 0, 2, 4096, 0, 16, 0, 4096, 1, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 4096, 5, 0, 4096, 4096, 0, 26, 13, 0, 0, 0, 2135, 0, 465, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 710, 4, 0, 0, 0, 4096, 3764, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 475, 28, 0, 5, 0, 3607, 0, 0, 4096, 4077, 22, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1996, 0, 0, 4091, 3490, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4094, 3385, 2667, 2100, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4083, 0, 0, 4090, 1, 0, 0, 0, 0, 3858, 0, 0, 2559, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 0, 0, 4079, 13, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 14, 4096, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14957, 16767, 16404, 17813, 16367, 16021, 16336, 16407] avg=16384.0000 max=17813 max_rank=3 min=14957 min_rank=0 imbalance_ratio=1.087219 max_load_reduction=113259 max_load_reduction_pct=0.864098 moved_to_echo=65131 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 3, 1, 4093, 4093, 0, 0, 190, 1, 1, 0, 3377, 0, 465, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 710, 9, 1, 0, 3, 4090, 3716, 0, 0, 0, 4092, 4, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 708, 8, 0, 0, 2, 3605, 1, 0, 4095, 4088, 3, 1, 4088, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 9, 4095, 1995, 0, 2, 4093, 2164, 1, 43, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 4, 1, 0, 1, 1, 0, 2, 0, 4095, 4093, 3384, 2666, 2099, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 1, 4093, 8, 0, 1, 1, 1, 2499, 1, 0, 3466, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 4076, 0, 1, 4029, 242, 1, 4, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 2, 4093, 1, 7, 4095, 1, 0, 0, 4095, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16339, 16723, 16602, 16496, 16374, 15596, 16543, 16399] avg=16384.0000 max=16723 max_rank=1 min=15596 min_rank=5 imbalance_ratio=1.020691 max_load_reduction=114349 max_load_reduction_pct=0.872414 moved_to_echo=64912 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4096, 4095, 2, 1, 45, 0, 0, 0, 3684, 0, 465, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 710, 0, 0, 0, 0, 4095, 2778, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 280, 143, 0, 0, 0, 3607, 0, 0, 4095, 4029, 106, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 4096, 31, 4096, 1996, 0, 0, 4096, 2615, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 3386, 2667, 2100, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 46, 0, 0, 0, 0, 3085, 0, 0, 3475, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 45, 4065, 0, 0, 4055, 12, 0, 58, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 4095, 0, 28, 4096, 0, 0, 0, 4038, 29, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16503, 15770, 16356, 17052, 16376, 16219, 16410, 16386] avg=16384.0000 max=17052 max_rank=3 min=15770 min_rank=1 imbalance_ratio=1.040771 max_load_reduction=114020 max_load_reduction_pct=0.869904 moved_to_echo=65391 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 4093, 3, 1, 4094, 4093, 0, 4, 406, 0, 0, 0, 3135, 0, 465, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 710, 3, 1, 0, 0, 4095, 3608, 0, 2, 2, 4094, 0, 2, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 773, 11, 0, 0, 0, 3602, 0, 1, 4094, 4088, 0, 2, 4090, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 1, 0, 4092, 16, 4094, 1995, 0, 2, 4091, 1692, 2, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 2, 2, 2, 0, 1, 0, 0, 1, 0, 2, 0, 4094, 4088, 3384, 2666, 2099, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 4090, 0, 0, 2, 0, 0, 3131, 2, 0, 3495, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4092, 0, 4075, 0, 2, 4065, 104, 2, 1, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 1, 4094, 1, 9, 4092, 0, 1, 0, 4088, 5, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16315, 16615, 16665, 16037, 16369, 16244, 16439, 16388] avg=16384.0000 max=16665 max_rank=2 min=16037 min_rank=3 imbalance_ratio=1.017151 max_load_reduction=114407 max_load_reduction_pct=0.872856 moved_to_echo=65440 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4094, 0, 0, 4082, 4090, 7, 10, 241, 1, 1, 0, 3290, 0, 465, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 710, 5, 0, 0, 0, 4082, 3077, 0, 1, 0, 4096, 7, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 285, 302, 0, 0, 0, 3602, 0, 0, 4079, 4043, 40, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 1, 4070, 11, 4090, 1994, 3, 0, 4096, 1670, 0, 430, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 1, 0, 2, 2, 7, 10, 2, 0, 0, 0, 0, 0, 0, 4096, 4094, 3386, 2666, 2099, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 4096, 0, 0, 4085, 26, 0, 0, 0, 0, 3467, 0, 1, 3560, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 4077, 0, 0, 4093, 0, 1, 9, 1, 1, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 2, 4089, 0, 27, 4091, 0, 5, 0, 3985, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16300, 16074, 16560, 16388, 16452, 16672, 16314, 16312] avg=16384.0000 max=16672 max_rank=5 min=16074 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=114400 max_load_reduction_pct=0.872803 moved_to_echo=65750 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 5, 150, 0, 0, 0, 4084, 0, 470, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 708, 6, 0, 0, 0, 4096, 3423, 0, 0, 0, 4090, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 4, 0, 8, 0, 3599, 0, 0, 4096, 4089, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 4095, 16, 4096, 1989, 0, 2, 4094, 889, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 3388, 2667, 2107, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4092, 9, 0, 1, 0, 2, 3967, 0, 0, 3666, 1425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 1, 4068, 0, 0, 4014, 20, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 1, 4095, 0, 0, 10, 4094, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17024, 16420, 16094, 15214, 16383, 17252, 16295, 16390] avg=16384.0000 max=17252 max_rank=5 min=15214 min_rank=3 imbalance_ratio=1.052979 max_load_reduction=113820 max_load_reduction_pct=0.868378 moved_to_echo=66320 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 4095, 0, 0, 4096, 4096, 0, 0, 109, 0, 0, 0, 3976, 1, 465, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 710, 8, 0, 0, 1, 4096, 3580, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 409, 10, 0, 4, 0, 3606, 0, 0, 4095, 4089, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1996, 0, 0, 4093, 1253, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3385, 2667, 2100, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4093, 2, 0, 0, 0, 1, 3846, 0, 0, 3174, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 0, 0, 4046, 39, 0, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1, 4095, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16858, 16588, 16308, 15556, 16380, 16639, 16358, 16385] avg=16384.0000 max=16858 max_rank=0 min=15556 min_rank=3 imbalance_ratio=1.028931 max_load_reduction=114214 max_load_reduction_pct=0.871384 moved_to_echo=65762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 25, 86, 0, 0, 0, 3894, 0, 465, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 710, 3, 0, 0, 1, 4096, 3112, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 44, 0, 0, 0, 3607, 0, 0, 4096, 4094, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4096, 0, 4096, 1996, 0, 0, 4095, 1791, 0, 440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4094, 3386, 2667, 2100, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 0, 4066, 14, 0, 0, 0, 0, 3294, 0, 0, 3532, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4077, 0, 0, 4095, 22, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 2, 4096, 0, 16, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16776, 16115, 16083, 16516, 16371, 16429, 16377, 16405] avg=16384.0000 max=16776 max_rank=0 min=16083 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=114296 max_load_reduction_pct=0.872009 moved_to_echo=65582 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 32768, 128, 32640, 0, 128, 32512, 512, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 3328, 640, 0, 128, 128, 28928, 128, 128, 32768, 32640, 256, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 32768, 128, 128, 32768, 128, 128, 128, 32640, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 32768, 0, 128, 32768, 128, 0, 128, 128, 128, 27264, 128, 128, 27008, 11520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 32768, 32768, 128, 128, 1280, 128, 128, 0, 27648, 128, 3840, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 5760, 128, 128, 0, 128, 32768, 27136, 0, 128, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 128, 32768, 16000, 128, 128, 32768, 15616, 128, 1280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 0, 128, 128, 0, 128, 0, 32768, 32768, 27136, 21376, 16896, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[4082, 4095, 4071, 4094, 0, 2, 1, 0, 4088, 0, 4093, 0, 1, 0, 0, 4093, 2, 4094, 1, 0, 1808, 0, 26, 1, 0, 4095, 0, 0, 4094, 0, 5, 1, 4094, 4092, 6, 0, 4093, 0, 4, 3, 1, 4095, 0, 2, 421, 1, 0, 0, 2, 4075, 2, 1, 0, 0, 0, 1, 4095, 1, 4094, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 2, 0, 4094, 18, 0, 2, 0, 0, 0, 4095, 4090, 2, 1, 4093, 1, 0, 3526, 2, 4094, 4093, 1, 2, 4095, 2, 0, 0, 4059, 0, 0, 2, 1, 1, 0, 0, 4095, 1, 0, 0, 1, 0, 4095, 0, 2, 1, 0, 0, 0, 1, 2995, 1, 3581, 2, 2, 0, 4091, 0, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[4092, 4092, 3814, 4096, 6, 0, 0, 0, 4096, 0, 4096, 3, 0, 0, 0, 4090, 0, 4081, 0, 0, 1260, 0, 15, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 242, 12, 4083, 0, 0, 0, 0, 4093, 0, 0, 1173, 0, 0, 0, 0, 4009, 0, 0, 0, 2, 0, 4, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 23, 0, 0, 0, 0, 6, 4004, 4060, 0, 0, 4092, 0, 0, 4041, 0, 4096, 4096, 0, 0, 4091, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 23, 0, 4087, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 2700, 0, 3338, 0, 5, 0, 4096, 3, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[4082, 4094, 4035, 4093, 0, 2, 2, 0, 4093, 0, 4093, 0, 3, 0, 1, 4092, 0, 4094, 2, 0, 3150, 0, 31, 2, 0, 4094, 0, 2, 4094, 0, 0, 3, 4094, 4093, 339, 0, 4077, 1, 0, 5, 2, 4093, 1, 3, 463, 2, 0, 0, 4, 3404, 2, 2, 0, 0, 0, 2, 4094, 2, 4095, 6, 1, 3, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 4094, 27, 2, 0, 0, 0, 0, 4093, 4092, 3, 2, 4093, 3, 0, 4059, 3, 4093, 4092, 1, 3, 4093, 4, 1, 0, 4082, 1, 0, 2, 0, 0, 0, 0, 4094, 1, 1, 0, 0, 0, 4090, 7, 2, 2, 0, 0, 1, 1, 993, 2, 3989, 2, 3, 0, 4094, 0, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[4092, 4096, 3941, 4096, 0, 0, 0, 0, 4096, 0, 4045, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 2732, 0, 5, 0, 16, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 956, 0, 4090, 0, 0, 0, 0, 4096, 3, 0, 397, 0, 0, 0, 0, 2699, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4063, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2755, 0, 2971, 0, 1, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[4095, 4096, 3993, 4096, 10, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 4075, 0, 4095, 0, 0, 1336, 0, 0, 0, 10, 4096, 0, 0, 4096, 0, 0, 0, 4095, 4096, 459, 2, 4095, 0, 0, 0, 0, 4096, 4, 0, 377, 0, 0, 0, 0, 3877, 0, 0, 0, 0, 0, 0, 4083, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2546, 0, 3813, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 3965, 4096, 8, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 4074, 0, 4096, 0, 0, 3125, 0, 0, 0, 58, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 137, 14, 4087, 0, 1, 0, 0, 4095, 2, 0, 219, 0, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 4090, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4040, 4096, 0, 0, 4096, 0, 0, 4090, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 969, 0, 4008, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 4088, 4096, 3, 0, 0, 0, 4096, 0, 3962, 0, 0, 0, 0, 4092, 0, 4095, 0, 0, 1890, 0, 0, 0, 19, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 234, 0, 4093, 0, 0, 0, 0, 4095, 0, 0, 218, 0, 0, 0, 0, 3839, 0, 0, 0, 0, 0, 0, 4089, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 4095, 0, 0, 4078, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 1, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 2934, 0, 3327, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[4096, 4096, 3863, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 2732, 0, 0, 0, 23, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 185, 0, 4062, 0, 2, 0, 0, 4096, 0, 0, 493, 0, 0, 0, 0, 3261, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 45, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 4092, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2551, 0, 3339, 0, 0, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 3941, 0, 0, 0, 0, 0, 159, 0, 4045, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2732, 0, 5, 0, 16, 4096, 0, 0, 4096, 0, 0, 0, 1033, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3312, 4096, 956, 0, 4090, 0, 0, 0, 0, 4096, 3, 0, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2699, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4094, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 4096, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 92, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 3547, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2755, 0, 2971, 0, 1, 0, 4096, 0, 0, 4096, 2339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16329, 16858, 16950, 15442, 16390, 16476, 16369, 16258] avg=16384.0000 max=16950 max_rank=2 min=15442 min_rank=3 imbalance_ratio=1.034546 max_load_reduction=114122 max_load_reduction_pct=0.870682 moved_to_echo=65493 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 3814, 0, 6, 0, 0, 0, 159, 0, 4096, 3, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 1260, 0, 15, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1033, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3312, 4096, 242, 12, 4083, 0, 0, 0, 0, 4093, 0, 0, 1173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 2, 0, 4, 4096, 0, 4075, 0, 0, 0, 0, 0, 4091, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 23, 0, 0, 0, 0, 6, 4004, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 4092, 0, 0, 4041, 0, 0, 92, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 23, 0, 4087, 0, 0, 8, 0, 0, 4096, 0, 3547, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2700, 0, 3338, 0, 5, 0, 4096, 3, 0, 4096, 2339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16260, 15365, 17011, 16734, 16330, 16376, 16418, 16578] avg=16384.0000 max=17011 max_rank=2 min=15365 min_rank=1 imbalance_ratio=1.038269 max_load_reduction=114061 max_load_reduction_pct=0.870216 moved_to_echo=65702 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4082, 0, 4071, 0, 0, 2, 1, 0, 158, 0, 4093, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 1, 0, 1808, 0, 26, 1, 0, 4095, 0, 0, 4094, 0, 5, 1, 1031, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3310, 4092, 6, 0, 4093, 0, 4, 3, 1, 4095, 0, 2, 421, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4075, 2, 1, 0, 0, 0, 1, 4095, 1, 4094, 0, 0, 1, 0, 0, 4093, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 2, 0, 4094, 18, 0, 2, 0, 0, 0, 4095, 4094, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 1, 4093, 1, 0, 3526, 2, 0, 92, 1, 2, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 2, 1, 1, 0, 0, 4095, 1, 0, 0, 1, 0, 4095, 0, 3544, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 1, 2995, 1, 3581, 2, 2, 0, 4091, 0, 0, 4094, 2335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16501, 15942, 16028, 16822, 16404, 15907, 16363, 17105] avg=16384.0000 max=17105 max_rank=7 min=15907 min_rank=5 imbalance_ratio=1.044006 max_load_reduction=113967 max_load_reduction_pct=0.869499 moved_to_echo=65779 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 3993, 0, 10, 0, 0, 0, 143, 0, 4096, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1336, 0, 0, 0, 10, 4096, 0, 0, 4096, 0, 0, 0, 1039, 787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3308, 4096, 459, 2, 4095, 0, 0, 0, 0, 4096, 4, 0, 377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 0, 0, 0, 0, 0, 4083, 0, 4095, 0, 0, 0, 0, 0, 4095, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 4096, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 84, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 1, 0, 0, 4096, 0, 3550, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2546, 0, 3813, 0, 0, 0, 4096, 0, 0, 4096, 2343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 15459, 16437, 16612, 16390, 16466, 16402, 16894] avg=16384.0000 max=16894 max_rank=7 min=15459 min_rank=1 imbalance_ratio=1.031128 max_load_reduction=114178 max_load_reduction_pct=0.871109 moved_to_echo=66152 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4082, 0, 4035, 0, 0, 2, 2, 0, 157, 0, 4093, 0, 3, 0, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 3150, 0, 31, 2, 0, 4094, 0, 2, 4094, 0, 0, 3, 1033, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3310, 4093, 339, 0, 4077, 1, 0, 5, 2, 4093, 1, 3, 463, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3404, 2, 2, 0, 0, 0, 2, 4094, 2, 4095, 6, 1, 3, 0, 0, 4092, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 4094, 27, 2, 0, 0, 0, 0, 4093, 4093, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3, 2, 4093, 3, 0, 4059, 3, 0, 92, 1, 3, 4093, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 1, 0, 2, 0, 0, 0, 0, 4094, 1, 1, 0, 0, 0, 4090, 7, 3543, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 1, 1, 993, 2, 3989, 2, 3, 0, 4094, 0, 0, 4093, 2338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16467, 17289, 16389, 16164, 16408, 16449, 16386, 15520] avg=16384.0000 max=17289 max_rank=1 min=15520 min_rank=7 imbalance_ratio=1.055237 max_load_reduction=113783 max_load_reduction_pct=0.868095 moved_to_echo=64763 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 3965, 0, 8, 0, 0, 0, 158, 0, 4096, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3125, 0, 0, 0, 58, 4096, 0, 0, 4096, 0, 0, 0, 1031, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3312, 4096, 137, 14, 4087, 0, 1, 0, 0, 4095, 2, 0, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 4090, 0, 4093, 0, 0, 0, 0, 0, 4095, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4040, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4090, 0, 0, 92, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 3547, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 969, 0, 4008, 0, 0, 0, 4096, 0, 0, 4096, 2335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 17286, 15963, 16719, 16331, 16470, 16402, 15504] avg=16384.0000 max=17286 max_rank=1 min=15504 min_rank=7 imbalance_ratio=1.055054 max_load_reduction=113786 max_load_reduction_pct=0.868118 moved_to_echo=64707 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4088, 0, 3, 0, 0, 0, 159, 0, 3962, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1890, 0, 0, 0, 19, 4096, 0, 0, 4096, 0, 0, 0, 1033, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3312, 4096, 234, 0, 4093, 0, 0, 0, 0, 4095, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3839, 0, 0, 0, 0, 0, 0, 4089, 0, 4095, 0, 0, 0, 0, 0, 4095, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 4089, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 4078, 0, 0, 92, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 1, 0, 0, 4091, 1, 3547, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2934, 0, 3327, 0, 0, 0, 4096, 0, 0, 4096, 2339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16400, 16013, 16048, 16575, 16390, 16456, 16398, 16792] avg=16384.0000 max=16792 max_rank=7 min=16013 min_rank=1 imbalance_ratio=1.024902 max_load_reduction=114280 max_load_reduction_pct=0.871887 moved_to_echo=66036 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 3863, 0, 0, 0, 0, 0, 159, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2732, 0, 0, 0, 23, 4096, 0, 0, 4096, 0, 0, 0, 1033, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3312, 4096, 185, 0, 4062, 0, 2, 0, 0, 4096, 0, 0, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3261, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 4095, 457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 45, 0, 0, 0, 0, 0, 4095, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4092, 0, 0, 92, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3547, 565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2551, 0, 3339, 0, 0, 0, 4096, 0, 0, 4096, 2339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16306, 16860, 16246, 16004, 16429, 16472, 16334, 16421] avg=16384.0000 max=16860 max_rank=1 min=16004 min_rank=3 imbalance_ratio=1.029053 max_load_reduction=114212 max_load_reduction_pct=0.871368 moved_to_echo=65656 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 31872, 0, 128, 128, 128, 0, 1280, 0, 32640, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32640, 128, 0, 128, 128, 128, 128, 0, 32768, 128, 128, 128, 128, 0, 32768, 128, 28416, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 128, 18560, 128, 28416, 128, 128, 0, 32768, 128, 0, 32768, 18816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[26496, 32768, 2560, 128, 32768, 128, 128, 128, 128, 32768, 128, 128, 3840, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 0, 18048, 0, 128, 128, 128, 32768, 0, 128, 32768, 0, 128, 128, 8320, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 32768, 128, 0, 32128, 128, 0, 768, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 29184, 128, 128, 0, 128, 0, 128, 32768, 128, 32768, 128, 128, 128, 0, 0, 32768, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 0, 128, 128, 0, 32768, 256, 128, 128, 0, 0, 128, 32640, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 1, 3787, 0, 0, 1363, 0, 4094, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 12, 6, 0, 0, 1, 0, 0, 4091, 14, 0, 4096, 4096, 0, 0, 2844, 2, 4096, 4096, 47, 3802, 0, 0, 4096, 0, 0, 4092, 4095, 0, 4083, 0, 0, 0, 3, 0, 0, 4096, 4096, 4096, 0, 0, 3947, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1138, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 4079, 4096, 3551] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[2, 0, 0, 2, 1, 2, 0, 4090, 0, 4094, 0, 3778, 0, 2, 1295, 2, 4094, 4094, 2, 4094, 2, 0, 0, 0, 1, 0, 1, 0, 1, 0, 2, 0, 0, 0, 0, 3, 2, 2, 4094, 0, 12, 4094, 0, 4094, 0, 8, 12, 2, 0, 6, 0, 1, 4092, 7, 0, 4093, 4092, 2, 2, 1547, 0, 4094, 4095, 10, 4023, 0, 0, 4092, 0, 2, 4057, 4094, 0, 4089, 2, 2, 0, 1, 1, 1, 4087, 4094, 4095, 0, 0, 3496, 3, 0, 0, 4093, 0, 4093, 1, 0, 0, 2, 1, 1, 0, 0, 1, 0, 2, 4091, 1, 0, 0, 1, 2, 2, 1, 3163, 0, 2, 1, 1, 17, 0, 0, 2, 4094, 2, 4094, 1, 0, 4093, 4094, 3154] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 4095, 0, 4096, 0, 4057, 0, 0, 676, 0, 4076, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 4087, 28, 0, 4096, 4096, 0, 0, 2247, 0, 4096, 4096, 41, 2815, 0, 0, 4096, 0, 0, 4082, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 3724, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3095, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 0, 4096, 0, 0, 4051, 4096, 3852] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 307, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 5, 0, 0, 113, 0, 0, 4096, 1, 0, 4096, 4096, 0, 0, 1898, 0, 4096, 4096, 28, 3947, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4053, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1873, 0, 0, 0, 0, 129, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4038] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4, 4078, 0, 4093, 10, 4079, 0, 0, 298, 0, 4078, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 9, 4095, 0, 0, 4, 0, 0, 83, 19, 0, 4049, 20, 0, 4096, 4090, 0, 0, 2077, 9, 4096, 4096, 24, 3446, 0, 0, 4096, 0, 0, 4096, 4089, 0, 4067, 0, 0, 0, 12, 0, 0, 4096, 4096, 4096, 0, 0, 3588, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 2959, 0, 0, 0, 0, 335, 0, 0, 0, 4096, 0, 4089, 0, 0, 4053, 4096, 3685] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[3, 0, 3, 3, 0, 2, 0, 4093, 0, 4094, 1, 3607, 0, 3, 1762, 3, 4093, 4092, 2, 4093, 3, 0, 0, 1, 0, 1, 0, 0, 0, 0, 3, 0, 3, 0, 0, 1, 3, 1, 4094, 0, 4, 4091, 0, 4093, 0, 5, 11, 1, 0, 6, 1, 2, 4092, 0, 0, 4093, 4080, 3, 2, 302, 0, 4092, 4094, 40, 4042, 0, 3, 4092, 0, 2, 4078, 4093, 3, 4092, 3, 3, 1, 0, 1, 1, 4090, 4093, 4094, 3, 0, 3616, 4, 3, 0, 4092, 0, 4091, 0, 0, 1, 3, 2, 1, 0, 0, 0, 0, 3, 4087, 2, 1, 0, 0, 3, 2, 0, 3452, 0, 3, 2, 2, 90, 0, 0, 2, 4094, 2, 4093, 0, 0, 4093, 4093, 3565] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 3966, 0, 0, 1195, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 4096, 0, 0, 10, 0, 0, 1, 0, 0, 4089, 5, 0, 4096, 4096, 0, 0, 1301, 0, 4096, 4096, 42, 3973, 0, 0, 4096, 0, 0, 4015, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4019, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2139, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 0, 4096, 0, 0, 4088, 4096, 3907] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 4092, 0, 0, 875, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 2, 4, 0, 4094, 1, 0, 4096, 4096, 0, 0, 767, 0, 4096, 4096, 7, 3605, 0, 0, 4096, 0, 0, 4092, 4096, 0, 4091, 0, 0, 0, 2, 0, 0, 4096, 4096, 4096, 0, 0, 4077, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3026, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 4096, 0, 0, 4084, 4096, 4030] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 307, 0, 1680, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2324, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 5, 0, 2074, 2007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 4096, 1, 0, 4096, 4096, 0, 0, 1898, 0, 2416, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 0, 0, 416, 0, 0, 4096, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 283, 0, 0, 0, 4053, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1873, 4096, 4096, 1489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 312, 0, 4096, 0, 0, 4096, 4096, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15880, 16389, 16374, 16744, 16643, 16624, 15651, 16767] avg=16384.0000 max=16767 max_rank=7 min=15651 min_rank=6 imbalance_ratio=1.023376 max_load_reduction=114305 max_load_reduction_pct=0.872078 moved_to_echo=65685 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 1, 3787, 0, 0, 1363, 0, 1684, 1610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2327, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 12, 6, 0, 2081, 2013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 14, 0, 4096, 4096, 0, 0, 2844, 2, 2412, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3802, 0, 0, 405, 0, 0, 4092, 4095, 0, 4083, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 275, 0, 0, 0, 3947, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1138, 4096, 4096, 1494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 302, 0, 4096, 0, 0, 4079, 4096, 3551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16638, 16394, 16401, 17603, 16480, 16511, 14920, 16125] avg=16384.0000 max=17603 max_rank=3 min=14920 min_rank=6 imbalance_ratio=1.074402 max_load_reduction=113469 max_load_reduction_pct=0.865700 moved_to_echo=64036 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3, 4095, 0, 4096, 0, 4057, 0, 0, 676, 0, 1680, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2324, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 12, 0, 2074, 2007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 28, 0, 4096, 4096, 0, 0, 2247, 0, 2416, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2815, 0, 0, 416, 0, 0, 4082, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 283, 0, 0, 0, 3724, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3095, 4096, 4096, 1489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 312, 0, 4096, 0, 0, 4051, 4096, 3852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16213, 16369, 16381, 17011, 15501, 16295, 16872, 16430] avg=16384.0000 max=17011 max_rank=3 min=15501 min_rank=4 imbalance_ratio=1.038269 max_load_reduction=114061 max_load_reduction_pct=0.870216 moved_to_echo=65098 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 2, 1, 2, 0, 4090, 0, 4094, 0, 3778, 0, 2, 1295, 2, 1679, 1604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 2, 4094, 2, 0, 0, 0, 1, 0, 1, 0, 1, 0, 2, 0, 2323, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 2, 4094, 0, 12, 4094, 0, 4094, 0, 8, 12, 2, 2072, 2006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 4092, 7, 0, 4093, 4092, 2, 2, 1547, 0, 2415, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 416, 0, 2, 4057, 4094, 0, 4089, 2, 2, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 283, 0, 0, 0, 3496, 3, 0, 0, 4093, 0, 4093, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 2, 4091, 1, 0, 0, 1, 2, 2, 1, 3163, 4095, 4095, 1488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 17, 0, 0, 2, 312, 2, 4094, 1, 0, 4093, 4094, 3154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16551, 16390, 16401, 16267, 16688, 16058, 16944, 15773] avg=16384.0000 max=16944 max_rank=6 min=15773 min_rank=7 imbalance_ratio=1.034180 max_load_reduction=114128 max_load_reduction_pct=0.870728 moved_to_echo=65463 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4, 4078, 0, 4093, 10, 4079, 0, 0, 298, 0, 1680, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4092, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2324, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 9, 4095, 0, 0, 4, 0, 2074, 2007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 19, 0, 4049, 20, 0, 4096, 4090, 0, 0, 2077, 9, 2416, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3446, 0, 0, 416, 0, 0, 4096, 4089, 0, 4067, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 283, 0, 0, 0, 3588, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 2959, 4096, 4096, 1489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 0, 0, 312, 0, 4089, 0, 0, 4053, 4096, 3685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15848, 16367, 16381, 16883, 16126, 16160, 16737, 16570] avg=16384.0000 max=16883 max_rank=3 min=15848 min_rank=0 imbalance_ratio=1.030457 max_load_reduction=114189 max_load_reduction_pct=0.871193 moved_to_echo=65593 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3, 3, 0, 2, 0, 4093, 0, 4094, 1, 3607, 0, 3, 1762, 3, 1678, 1604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4092, 2, 4093, 3, 0, 0, 1, 0, 1, 0, 0, 0, 0, 3, 0, 2322, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 3, 1, 4094, 0, 4, 4091, 0, 4093, 0, 5, 11, 1, 2072, 2006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 2, 4092, 0, 0, 4093, 4080, 3, 2, 302, 0, 2414, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 3, 416, 0, 2, 4078, 4093, 3, 4092, 3, 3, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 283, 0, 3, 0, 3616, 4, 3, 0, 4092, 0, 4091, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 3, 4087, 2, 1, 0, 0, 3, 2, 0, 3452, 4094, 4094, 1488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 2, 90, 0, 0, 2, 312, 2, 4093, 0, 0, 4093, 4093, 3565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16856, 16386, 16385, 15035, 16738, 16186, 17229, 16257] avg=16384.0000 max=17229 max_rank=6 min=15035 min_rank=3 imbalance_ratio=1.051575 max_load_reduction=113843 max_load_reduction_pct=0.868553 moved_to_echo=66410 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 3966, 0, 0, 1195, 0, 1680, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2324, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 4096, 0, 0, 10, 0, 2074, 2007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 5, 0, 4096, 4096, 0, 0, 1301, 0, 2416, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3973, 0, 0, 416, 0, 0, 4015, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 283, 0, 0, 0, 4019, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2139, 4096, 4096, 1489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 312, 0, 4096, 0, 0, 4088, 4096, 3907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16640, 16388, 16380, 16046, 16594, 16590, 15916, 16518] avg=16384.0000 max=16640 max_rank=0 min=15916 min_rank=6 imbalance_ratio=1.015625 max_load_reduction=114432 max_load_reduction_pct=0.873047 moved_to_echo=65618 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 4092, 0, 0, 875, 0, 1680, 1606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2324, 1777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 2074, 2007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 4094, 1, 0, 4096, 4096, 0, 0, 767, 0, 2416, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3605, 0, 0, 416, 0, 0, 4092, 4096, 0, 4091, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 283, 0, 0, 0, 4077, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3026, 4096, 4096, 1489, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 312, 0, 4096, 0, 0, 4084, 4096, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16446, 16389, 16369, 15483, 16302, 16648, 16803, 16632] avg=16384.0000 max=16803 max_rank=6 min=15483 min_rank=3 imbalance_ratio=1.025574 max_load_reduction=114269 max_load_reduction_pct=0.871803 moved_to_echo=66385 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 32768, 128, 32768, 128, 0, 0, 128, 128, 128, 128, 0, 128, 0, 128, 0, 18688, 14336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[29696, 0, 128, 3328, 0, 128, 32640, 32768, 128, 32768, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 128, 128, 32768, 0, 128, 32768, 128, 32768, 0, 128, 128, 128, 16640, 16128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 32768, 128, 128, 128, 128, 128, 128, 128, 20864, 32768, 32768, 12032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 640, 0, 0, 128, 2560, 128, 32768, 128, 0, 32640, 32768, 29824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 2304, 0, 128, 0, 30592, 128, 128, 128, 32768, 0, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 32768, 0, 32768, 128, 31488, 0, 128, 7808, 128, 13568, 12928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 256, 128, 128, 32768, 128, 0, 32768, 32768, 128, 128, 13056, 128, 19328, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 4094, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 266, 0, 4073, 0, 4096, 0, 4096, 0, 1, 4096, 14, 4096, 0, 0, 0, 0, 0, 0, 4060, 4096, 0, 0, 0, 4093, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 20, 0, 5, 0, 0, 3844, 0, 4096, 4096, 8, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 282, 0, 3983, 0, 4096, 28, 4096, 0, 0, 4096, 24, 4096, 0, 0, 0, 0, 0, 0, 4088, 4095, 0, 0, 0, 4094, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 35, 0, 7, 0, 0, 3817, 0, 4096, 4096, 45, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 77, 0, 4079, 0, 4096, 0, 4096, 0, 0, 4096, 20, 4096, 0, 0, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4013, 0, 4096, 4096, 10, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 11, 0, 4088, 0, 4096, 0, 4096, 0, 2, 4096, 18, 4096, 0, 0, 0, 0, 0, 0, 4071, 4096, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4056, 0, 4096, 4096, 40, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 4096, 0, 4096, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 524, 0, 4052, 0, 4096, 20, 4096, 0, 9, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4064, 4096, 0, 0, 0, 4085, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4096, 3, 0, 4093, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 32, 0, 42, 0, 0, 3572, 0, 4096, 4096, 9, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 2, 0, 0, 3, 4094, 2, 0, 4093, 2, 2, 0, 10, 0, 2, 0, 0, 0, 0, 2, 2, 4092, 0, 1, 1, 0, 19, 4094, 2, 0, 0, 1, 4094, 4075, 0, 4094, 2, 4093, 4094, 1, 2, 4094, 0, 0, 2, 1, 2, 4093, 4094, 2, 0, 11, 3, 4094, 2, 4094, 0, 4094, 0, 0, 4085, 22, 4092, 0, 0, 0, 0, 0, 1, 4033, 4093, 0, 2, 0, 4094, 4094, 0, 4094, 0, 3, 2, 1, 0, 2, 0, 2, 1, 2, 0, 2, 4088, 0, 0, 4094, 2, 2, 4093, 0, 3, 0, 4092, 4094, 2, 3, 0, 1, 0, 0, 3, 0, 0, 2, 4093, 0, 4094, 4093, 31, 0, 0, 3, 4092, 4094, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 289, 0, 4065, 0, 4096, 0, 4096, 0, 0, 4096, 18, 4096, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3800, 0, 4096, 4096, 28, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 1, 2, 2, 2, 0, 3, 4091, 3, 0, 4091, 1, 4, 0, 0, 0, 3, 0, 0, 0, 0, 3, 3, 4092, 0, 1, 2, 0, 16, 4092, 1, 2, 0, 2, 4093, 4078, 0, 4093, 2, 4092, 4092, 6, 3, 4093, 0, 0, 0, 3, 3, 4092, 4093, 3, 0, 405, 0, 4094, 2, 4092, 2, 4093, 0, 1, 4089, 0, 4091, 0, 0, 0, 0, 0, 2, 4015, 4093, 0, 3, 0, 4094, 4093, 0, 4093, 0, 3, 3, 3, 0, 3, 0, 0, 1, 3, 0, 3, 4087, 0, 4, 4093, 3, 2, 4093, 0, 2, 2, 4093, 4093, 1, 4, 0, 3, 0, 0, 5, 0, 0, 2, 3755, 0, 4092, 4093, 12, 0, 1, 3, 4090, 4092, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 4094, 4095, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4094, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 266, 0, 4073, 0, 3881, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 4060, 14, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 4074, 3901, 215, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3844, 0, 4096, 195, 8, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16389, 16384, 16413, 16373, 16400, 16389, 16340] avg=16384.0000 max=16413 max_rank=3 min=16340 min_rank=7 imbalance_ratio=1.001770 max_load_reduction=114659 max_load_reduction_pct=0.874779 moved_to_echo=65502 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 4094, 4090, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 282, 0, 3983, 0, 3887, 28, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4096, 0, 0, 0, 0, 0, 0, 4088, 25, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4066, 3897, 209, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 3817, 0, 4096, 199, 45, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 16376, 16390, 16372, 16423, 16377, 16406, 16356] avg=16384.0000 max=16423 max_rank=4 min=16356 min_rank=7 imbalance_ratio=1.002380 max_load_reduction=114649 max_load_reduction_pct=0.874702 moved_to_echo=65562 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4095, 4090, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 77, 0, 4079, 0, 3887, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 0, 0, 0, 0, 0, 0, 4083, 24, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4067, 3897, 209, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 4096, 199, 10, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16382, 16390, 16235, 16415, 16378, 16377, 16510] avg=16384.0000 max=16510 max_rank=7 min=16235 min_rank=3 imbalance_ratio=1.007690 max_load_reduction=114562 max_load_reduction_pct=0.874039 moved_to_echo=65680 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 2, 0, 0, 3, 4094, 2, 0, 4093, 2, 2, 0, 10, 4094, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 2, 4092, 0, 1, 1, 0, 19, 4094, 2, 4092, 4088, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 4075, 0, 0, 2, 4093, 4094, 1, 2, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4093, 0, 2, 0, 11, 3, 4094, 2, 3885, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4092, 0, 0, 0, 0, 0, 1, 4033, 25, 0, 2, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3, 2, 1, 0, 2, 0, 2, 1, 2, 0, 2, 4088, 0, 0, 4064, 3894, 209, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 2, 4093, 0, 3, 0, 4092, 4081, 2, 3, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 4094, 199, 31, 0, 0, 3, 4092, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16400, 16399, 16370, 16178, 16363, 16377, 16376, 16609] avg=16384.0000 max=16609 max_rank=7 min=16178 min_rank=3 imbalance_ratio=1.013733 max_load_reduction=114463 max_load_reduction_pct=0.873283 moved_to_echo=65725 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 3, 4096, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 4096, 0, 4094, 4090, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4094, 0, 0, 0, 4096, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 524, 0, 4052, 0, 3887, 20, 0, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4064, 24, 0, 0, 0, 4085, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4096, 3, 0, 4067, 3897, 209, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 3572, 0, 4096, 199, 9, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16369, 16367, 16384, 16684, 16358, 16400, 16400, 16110] avg=16384.0000 max=16684 max_rank=3 min=16110 min_rank=7 imbalance_ratio=1.018311 max_load_reduction=114388 max_load_reduction_pct=0.872711 moved_to_echo=65268 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 2, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 4094, 4090, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4094, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 4088, 0, 3887, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 4071, 24, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4067, 3897, 209, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4056, 0, 4096, 199, 40, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16382, 16388, 16180, 16400, 16379, 16372, 16584] avg=16384.0000 max=16584 max_rank=7 min=16180 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=114488 max_load_reduction_pct=0.873474 moved_to_echo=65735 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 1, 2, 2, 2, 0, 3, 4091, 3, 0, 4091, 1, 4, 0, 0, 4093, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 3, 4092, 0, 1, 2, 0, 16, 4092, 1, 4092, 4087, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 6, 4078, 0, 0, 2, 4092, 4092, 6, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 4092, 0, 3, 0, 405, 0, 4094, 2, 3883, 2, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 2, 4015, 25, 0, 3, 0, 4094, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 3, 3, 0, 3, 0, 0, 1, 3, 0, 3, 4087, 0, 4, 4064, 3894, 209, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 2, 4093, 0, 2, 2, 4093, 4080, 1, 4, 0, 3, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3755, 0, 4092, 199, 12, 0, 1, 3, 4090, 4092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16396, 16376, 16577, 16323, 16383, 16381, 16247] avg=16384.0000 max=16577 max_rank=3 min=16247 min_rank=7 imbalance_ratio=1.011780 max_load_reduction=114495 max_load_reduction_pct=0.873528 moved_to_echo=65334 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4094, 4090, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 289, 0, 4065, 0, 3887, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4096, 0, 0, 0, 0, 0, 0, 4087, 24, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4067, 3897, 209, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3800, 0, 4096, 199, 28, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16381, 16390, 16433, 16417, 16378, 16371, 16316] avg=16384.0000 max=16433 max_rank=3 min=16316 min_rank=7 imbalance_ratio=1.002991 max_load_reduction=114639 max_load_reduction_pct=0.874626 moved_to_echo=65482 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 32768, 128, 128, 32768, 128, 128, 0, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 0, 0, 0, 0, 128, 32512, 256, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 32768, 0, 128, 0, 1920, 128, 32640, 128, 31104, 128, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 128, 32768, 32768, 128, 128, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 32768, 0, 128, 128, 32768, 32768, 128, 128, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 30976, 0, 32768, 1664, 256, 0, 128, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 0, 0, 128, 128, 128, 32768, 0, 128, 128, 0, 128, 32768, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 0, 128, 128, 128, 0, 128, 0, 128, 128, 128, 128, 128, 32768, 128, 128, 32640, 31232, 1792, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1016, 4093, 4094, 2, 2, 2, 6, 1, 194, 1, 0, 0, 0, 0, 0, 1, 0, 1384, 3, 0, 4, 2790, 0, 4094, 0, 2, 0, 0, 2, 4092, 0, 4093, 2, 4084, 4092, 1, 4093, 0, 0, 3, 0, 4093, 0, 4094, 0, 4094, 0, 4094, 4093, 4094, 0, 0, 0, 4092, 2, 1, 3936, 3, 2, 0, 2, 2, 0, 4094, 0, 1, 0, 4094, 2, 0, 2, 2, 1, 4094, 0, 2, 2, 0, 3, 4094, 3, 0, 4094, 2, 0, 0, 1, 0, 0, 4093, 4094, 2, 0, 2, 0, 4091, 4093, 2, 0, 4078, 1, 4093, 0, 0, 0, 13, 0, 1, 1, 0, 1, 0, 4093, 2973, 0, 3, 1, 2, 1, 2, 0, 2, 4093, 0, 0, 0, 4091, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[281, 4096, 4096, 0, 0, 0, 0, 0, 26, 0, 0, 21, 0, 0, 0, 0, 0, 2463, 0, 0, 0, 1923, 0, 4096, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 3808, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4079, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3792, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[348, 4096, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 47, 0, 0, 0, 0, 0, 2140, 0, 7, 0, 2682, 0, 4096, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 3356, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 3, 0, 0, 4096, 0, 0, 3906, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 18, 0, 4096, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 3954, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4021, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1088, 4096, 4077, 0, 0, 0, 0, 0, 463, 0, 0, 53, 0, 0, 0, 0, 0, 2475, 0, 16, 0, 1284, 0, 4096, 0, 0, 12, 0, 0, 4095, 0, 4096, 1, 3805, 4096, 0, 4096, 10, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 32, 0, 0, 4096, 0, 0, 3716, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 4082, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 0, 4093, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4072, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 3467, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[578, 4092, 4093, 4, 4, 2, 2, 3, 22, 3, 0, 0, 0, 0, 0, 0, 0, 1484, 5, 0, 4, 2434, 0, 4093, 0, 3, 0, 0, 3, 4091, 3, 4092, 0, 4023, 4091, 0, 4091, 0, 0, 4, 2, 4092, 0, 4093, 0, 4091, 0, 4093, 4093, 4092, 0, 0, 0, 4090, 3, 0, 3797, 3, 4, 0, 3, 3, 0, 4093, 0, 1, 0, 4093, 3, 0, 2, 3, 1, 4093, 1, 3, 3, 0, 4, 4094, 3, 0, 4088, 1, 0, 0, 5, 0, 0, 4092, 4092, 3, 0, 3, 0, 4088, 4093, 3, 0, 4085, 2, 4093, 0, 0, 0, 0, 0, 0, 4, 7, 1, 0, 4092, 4035, 0, 4, 0, 3, 4, 3, 0, 3, 4092, 0, 0, 0, 4093, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[953, 4096, 4096, 0, 0, 0, 0, 0, 320, 0, 0, 167, 0, 0, 0, 0, 0, 2703, 0, 10, 0, 1317, 0, 4096, 0, 0, 1, 0, 0, 4096, 0, 4096, 1, 3820, 4096, 0, 4096, 0, 1, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 20, 0, 0, 4095, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4073, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[615, 4096, 4095, 0, 0, 0, 0, 0, 14, 0, 0, 20, 0, 0, 0, 0, 0, 2632, 0, 1, 0, 1941, 0, 4096, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 3330, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 1, 0, 0, 4096, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4084, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3804, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[479, 4096, 4096, 0, 0, 0, 0, 0, 90, 0, 0, 32, 0, 0, 0, 0, 0, 1646, 0, 1, 0, 2502, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 3519, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 1, 0, 0, 4096, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[281, 4096, 4096, 0, 0, 0, 0, 0, 26, 0, 0, 21, 0, 0, 0, 0, 4096, 3243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2463, 0, 0, 0, 1923, 0, 3960, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3808, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 4096, 0, 0, 0, 4096, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 4096, 3410, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4079, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3271, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3792, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 307, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15859, 16539, 16475, 16511, 16386, 16384, 16395, 16523] avg=16384.0000 max=16539 max_rank=1 min=15859 min_rank=0 imbalance_ratio=1.009460 max_load_reduction=114533 max_load_reduction_pct=0.873817 moved_to_echo=65688 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1016, 4093, 4094, 2, 2, 2, 6, 1, 194, 1, 0, 0, 0, 0, 0, 1, 4094, 3242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1384, 3, 0, 4, 2790, 0, 3958, 0, 2, 0, 0, 2, 4092, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4084, 4092, 1, 4093, 0, 0, 3, 0, 4093, 0, 0, 0, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 4094, 0, 0, 0, 4092, 2, 1, 3936, 3, 2, 0, 2, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 2, 0, 2, 2, 1, 4094, 0, 2, 2, 0, 3, 4094, 3408, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4094, 2, 0, 0, 1, 0, 0, 4093, 4094, 2, 0, 2, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 4078, 1, 4093, 0, 0, 0, 13, 0, 1, 1, 0, 1, 0, 3269, 852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2973, 0, 3, 1, 2, 1, 2, 0, 2, 4093, 0, 0, 0, 4091, 2, 307, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16748, 16328, 16747, 16375, 16382, 16382, 16404, 15706] avg=16384.0000 max=16748 max_rank=0 min=15706 min_rank=7 imbalance_ratio=1.022217 max_load_reduction=114324 max_load_reduction_pct=0.872223 moved_to_echo=64874 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[348, 4096, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 47, 0, 0, 0, 0, 4096, 3244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2140, 0, 7, 0, 2682, 0, 3958, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3356, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 4096, 3, 0, 0, 4096, 0, 0, 3906, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 18, 0, 4096, 3412, 682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3954, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3276, 852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4021, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 314, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15926, 16980, 16014, 16335, 16396, 16386, 16274, 16761] avg=16384.0000 max=16980 max_rank=1 min=15926 min_rank=0 imbalance_ratio=1.036377 max_load_reduction=114092 max_load_reduction_pct=0.870453 moved_to_echo=65817 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1088, 4096, 4077, 0, 0, 0, 0, 0, 463, 0, 0, 53, 0, 0, 0, 0, 4096, 3244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2475, 0, 16, 0, 1284, 0, 3960, 0, 0, 12, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3805, 4096, 0, 4096, 10, 0, 0, 0, 4096, 0, 0, 0, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 4096, 32, 0, 0, 4096, 0, 0, 3716, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4082, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 0, 4093, 3410, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4072, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3271, 852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3467, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 307, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17117, 15938, 16483, 16180, 16373, 16391, 16393, 16197] avg=16384.0000 max=17117 max_rank=0 min=15938 min_rank=1 imbalance_ratio=1.044739 max_load_reduction=113955 max_load_reduction_pct=0.869408 moved_to_echo=65354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[953, 4096, 4096, 0, 0, 0, 0, 0, 320, 0, 0, 167, 0, 0, 0, 0, 4096, 3244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2703, 0, 10, 0, 1317, 0, 3960, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3820, 4096, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 4096, 20, 0, 0, 4095, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 4096, 3410, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4073, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3271, 852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 307, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16972, 16183, 16489, 16456, 16379, 16384, 16388, 15821] avg=16384.0000 max=16972 max_rank=0 min=15821 min_rank=7 imbalance_ratio=1.035889 max_load_reduction=114100 max_load_reduction_pct=0.870514 moved_to_echo=64972 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[578, 4092, 4093, 4, 4, 2, 2, 3, 22, 3, 0, 0, 0, 0, 0, 0, 4093, 3241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1484, 5, 0, 4, 2434, 0, 3957, 0, 3, 0, 0, 3, 4091, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 4091, 0, 4091, 0, 0, 4, 2, 4092, 0, 0, 0, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 4092, 0, 0, 0, 4090, 3, 0, 3797, 3, 4, 0, 3, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 3, 0, 2, 3, 1, 4093, 1, 3, 3, 0, 4, 4094, 3406, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4088, 1, 0, 0, 5, 0, 0, 4092, 4092, 3, 0, 3, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 4085, 2, 4093, 0, 0, 0, 0, 0, 0, 4, 7, 1, 0, 3269, 852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4035, 0, 4, 0, 3, 4, 3, 0, 3, 4092, 0, 0, 0, 4093, 3, 307, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16137, 16076, 16681, 16235, 16384, 16375, 16409, 16775] avg=16384.0000 max=16775 max_rank=7 min=16076 min_rank=1 imbalance_ratio=1.023865 max_load_reduction=114297 max_load_reduction_pct=0.872017 moved_to_echo=65943 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[615, 4096, 4095, 0, 0, 0, 0, 0, 14, 0, 0, 20, 0, 0, 0, 0, 4096, 3244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2632, 0, 1, 0, 1941, 0, 3960, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3330, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 4096, 1, 0, 0, 4096, 0, 0, 4024, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 4096, 3410, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3271, 852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3804, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 307, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16180, 16727, 15997, 16461, 16386, 16385, 16399, 16537] avg=16384.0000 max=16727 max_rank=1 min=15997 min_rank=2 imbalance_ratio=1.020935 max_load_reduction=114345 max_load_reduction_pct=0.872383 moved_to_echo=65707 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[479, 4096, 4096, 0, 0, 0, 0, 0, 90, 0, 0, 32, 0, 0, 0, 0, 4096, 3244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1646, 0, 1, 0, 2502, 0, 3960, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3519, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 4096, 1, 0, 0, 4096, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 0, 4096, 3410, 677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3271, 852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 307, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16133, 16301, 16186, 16519, 16386, 16385, 16410, 16752] avg=16384.0000 max=16752 max_rank=7 min=16133 min_rank=0 imbalance_ratio=1.022461 max_load_reduction=114320 max_load_reduction_pct=0.872192 moved_to_echo=65933 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 29312, 128, 128, 128, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 32768, 128, 2560, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 128, 0, 0, 32768, 32768, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 17024, 128, 128, 128, 16896, 0, 31744, 0, 128, 128, 0, 128, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[5376, 32768, 32768, 128, 128, 128, 128, 128, 1152, 128, 0, 384, 0, 0, 0, 128, 32768, 25984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 0, 32640, 128, 32768, 0, 0, 0, 128, 0, 128, 128, 128, 128, 0, 26240, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1280, 32768, 128, 0, 0, 32768, 128, 128, 31616, 128, 128, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 29824, 32768, 128, 32768, 128, 128, 128, 128, 32768, 0, 0, 0, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 128, 128, 32768, 27392, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 836, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 4093, 4096, 0, 4096, 0, 138, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 9, 4096, 0, 4096, 0, 4096, 3147, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 4055, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[2, 0, 4092, 660, 3, 4093, 4, 0, 0, 0, 4, 0, 4092, 0, 0, 4081, 0, 1, 3, 0, 3, 2, 4091, 5, 4, 3, 4090, 0, 2, 0, 4, 0, 1, 4063, 4086, 0, 4092, 3, 922, 0, 2, 1, 4093, 1, 4, 4090, 6, 5, 0, 1, 4044, 0, 2, 3, 0, 0, 1, 0, 4092, 4092, 0, 3, 4088, 5, 0, 4092, 2, 4089, 0, 4092, 2595, 0, 4091, 4093, 0, 6, 3, 8, 0, 0, 0, 7, 4083, 0, 0, 0, 4092, 4092, 0, 3, 4091, 2, 0, 3, 0, 3, 6, 2, 3, 4093, 4093, 0, 1, 2, 0, 0, 3, 4, 0, 0, 0, 4094, 0, 0, 4092, 4, 2, 4, 9, 4092, 4, 4094, 3, 4081, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 18, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 696, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 4096, 0, 1, 4096, 0, 4096, 0, 4096, 3583, 0, 4094, 4096, 5, 0, 0, 0, 0, 0, 0, 6, 4096, 3, 0, 22, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4070, 0, 3890, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[2, 0, 4089, 928, 2, 4094, 3, 0, 0, 0, 2, 0, 4094, 0, 0, 4081, 1, 0, 2, 0, 2, 2, 4093, 3, 3, 2, 4093, 0, 1, 0, 3, 0, 1, 4054, 4088, 0, 4093, 2, 504, 0, 2, 2, 4094, 1, 1, 4091, 6, 3, 0, 2, 4089, 0, 0, 2, 1, 0, 0, 0, 4093, 4094, 0, 3, 4091, 2, 0, 4093, 1, 4089, 0, 4093, 2717, 0, 4093, 4094, 0, 4, 2, 1, 0, 1, 0, 12, 4092, 0, 0, 0, 4093, 4094, 0, 1, 4092, 2, 0, 2, 1, 2, 0, 1, 2, 4094, 4094, 0, 1, 2, 2, 0, 2, 2, 0, 0, 0, 4094, 0, 0, 4092, 1, 1, 3, 7, 4094, 3, 4094, 2, 4082, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 621, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 4096, 0, 4096, 0, 758, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 2, 4096, 0, 4096, 0, 4096, 2731, 0, 4095, 4096, 2, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 8, 4096, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 4071, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 556, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 4096, 4096, 0, 4096, 0, 359, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 34, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 53, 4096, 0, 4096, 0, 4096, 3796, 0, 4057, 4096, 2, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 79, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4043, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4085, 0, 3399, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 623, 0, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 27, 0, 0, 0, 4096, 4096, 0, 4096, 0, 1154, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 20, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 4096, 0, 72, 4096, 0, 4096, 0, 4096, 2599, 0, 4027, 4094, 18, 0, 0, 0, 7, 0, 0, 34, 4096, 6, 0, 92, 4096, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4077, 0, 3673, 0, 7, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 4096, 587, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 455, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 17, 4096, 0, 4096, 0, 4096, 3686, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 3445, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 836, 0, 3498, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 6, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 138, 0, 0, 0, 4096, 0, 0, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 981, 0, 4096, 0, 0, 3147, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 4064, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3115, 613, 369, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 4055, 0, 0, 0, 0, 229, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16622, 16382, 15906, 16384, 16425, 16357, 16393, 16603] avg=16384.0000 max=16622 max_rank=0 min=15906 min_rank=2 imbalance_ratio=1.014526 max_load_reduction=114450 max_load_reduction_pct=0.873184 moved_to_echo=65778 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4092, 660, 3, 3495, 4, 0, 0, 0, 4, 0, 4092, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 3, 2, 4091, 5, 4, 3, 4090, 0, 2, 0, 4, 0, 4092, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4063, 4086, 0, 0, 3, 922, 0, 2, 1, 4093, 1, 4, 3478, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4044, 0, 2, 3, 0, 0, 1, 0, 4092, 4092, 0, 3, 4088, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 980, 2, 4089, 0, 0, 2595, 0, 4091, 4093, 0, 6, 3, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4083, 0, 0, 0, 4060, 4092, 0, 3, 4091, 2, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 3, 4093, 4093, 0, 1, 2, 0, 0, 3, 4, 0, 0, 0, 4094, 3112, 612, 369, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 2, 4, 9, 4092, 4, 4094, 3, 4081, 0, 0, 0, 0, 229, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16433, 16385, 16665, 16331, 15867, 16344, 16401, 16646] avg=16384.0000 max=16665 max_rank=2 min=15867 min_rank=4 imbalance_ratio=1.017151 max_load_reduction=114407 max_load_reduction_pct=0.872856 moved_to_echo=65258 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 18, 0, 3488, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 696, 0, 0, 0, 4096, 0, 0, 3477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 976, 0, 4096, 0, 0, 3583, 0, 4094, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 3, 0, 22, 4054, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3120, 619, 374, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4070, 0, 3890, 0, 0, 0, 0, 234, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15794, 16377, 16461, 16390, 16851, 16373, 16405, 16421] avg=16384.0000 max=16851 max_rank=4 min=15794 min_rank=0 imbalance_ratio=1.028503 max_load_reduction=114221 max_load_reduction_pct=0.871437 moved_to_echo=66050 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 556, 0, 3498, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 359, 0, 0, 0, 4096, 0, 0, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 981, 0, 4096, 0, 0, 3796, 0, 4057, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 79, 4064, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4043, 3115, 613, 369, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4085, 0, 3399, 0, 0, 0, 0, 229, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16342, 16384, 16130, 16417, 17081, 16441, 16340, 15937] avg=16384.0000 max=17081 max_rank=4 min=15937 min_rank=7 imbalance_ratio=1.042542 max_load_reduction=113991 max_load_reduction_pct=0.869682 moved_to_echo=65799 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4089, 928, 2, 3496, 3, 0, 0, 0, 2, 0, 4094, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 2, 2, 4093, 3, 3, 2, 4093, 0, 1, 0, 3, 0, 4093, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4054, 4088, 0, 0, 2, 504, 0, 2, 2, 4094, 1, 1, 3479, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4089, 0, 0, 2, 1, 0, 0, 0, 4093, 4094, 0, 3, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 980, 1, 4089, 0, 0, 2717, 0, 4093, 4094, 0, 4, 2, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4092, 0, 0, 0, 4061, 4094, 0, 1, 4092, 2, 0, 2, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4094, 4094, 0, 1, 2, 2, 0, 2, 2, 0, 0, 0, 4094, 3113, 612, 369, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 1, 3, 7, 4094, 3, 4094, 2, 4082, 0, 0, 1, 0, 229, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16697, 16385, 16237, 16377, 15982, 16359, 16394, 16641] avg=16384.0000 max=16697 max_rank=0 min=15982 min_rank=4 imbalance_ratio=1.019104 max_load_reduction=114375 max_load_reduction_pct=0.872612 moved_to_echo=65376 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 623, 0, 3498, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 27, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1154, 0, 0, 0, 4090, 0, 0, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 981, 0, 4096, 0, 0, 2599, 0, 4027, 4094, 18, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4096, 6, 0, 92, 4064, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 3115, 613, 369, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4077, 0, 3673, 0, 7, 0, 0, 229, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16404, 16404, 16919, 16396, 15894, 16481, 16364, 16210] avg=16384.0000 max=16919 max_rank=2 min=15894 min_rank=4 imbalance_ratio=1.032654 max_load_reduction=114153 max_load_reduction_pct=0.870918 moved_to_echo=64949 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 621, 0, 3498, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 758, 0, 0, 0, 4095, 0, 0, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 981, 0, 4096, 0, 0, 2731, 0, 4095, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 8, 4064, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3115, 613, 369, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 4071, 0, 1, 0, 0, 229, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16378, 16528, 16385, 16003, 16362, 16389, 16620] avg=16384.0000 max=16620 max_rank=7 min=16003 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=114452 max_load_reduction_pct=0.873199 moved_to_echo=65374 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 587, 0, 3498, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 455, 0, 0, 0, 4096, 0, 0, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 981, 0, 4096, 0, 0, 3686, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 4064, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3115, 613, 369, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 3445, 0, 0, 0, 0, 229, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 16377, 16226, 16392, 16969, 16355, 16386, 15994] avg=16384.0000 max=16969 max_rank=4 min=15994 min_rank=7 imbalance_ratio=1.035706 max_load_reduction=114103 max_load_reduction_pct=0.870537 moved_to_echo=65704 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 128, 128, 128, 32768, 0, 128, 128, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 32768, 0, 0, 128, 4992, 0, 128, 128, 32768, 128, 128, 27904, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 32768, 4864, 128, 28032, 128, 0, 0, 0, 128, 0, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 32768, 128, 128, 128, 128, 32768, 128, 32768, 128, 30720, 0, 128, 128, 0, 1920, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 128, 0, 256, 32512, 32768, 0, 128, 32768, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 128, 0, 128, 0, 32768, 32768, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 7936, 128, 32768, 0, 0, 24960, 0, 32768, 32768, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 32768, 32768, 0, 128, 128, 128, 0, 128, 128, 0, 0, 128, 32768, 24960, 4992, 3072, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 948, 0, 4096, 0, 0, 0, 4083, 2, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 3475, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 2259, 0, 1141, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 42, 4096, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1, 3, 0, 0, 0, 2, 0, 4092, 2, 1, 0, 2, 2, 2, 744, 4, 4093, 1, 0, 0, 4027, 11, 4093, 0, 1, 1, 4094, 3, 4092, 0, 4093, 0, 4093, 3, 0, 4094, 1, 0, 3101, 0, 1, 4093, 3, 1, 4094, 2, 0, 0, 3, 4093, 4094, 0, 2, 0, 4091, 0, 1, 2, 0, 0, 2, 4093, 1, 3, 2, 4090, 2, 4092, 2, 4, 1912, 1, 1518, 1, 1, 0, 2, 3, 4091, 4093, 10, 0, 4093, 4093, 2, 4092, 0, 2, 4094, 2, 4093, 0, 4084, 0, 4085, 985, 2, 3, 3, 0, 0, 4, 0, 0, 3, 0, 2, 0, 4091, 0, 0, 1, 1, 0, 1, 0, 2, 1, 4093, 4072, 3, 4093, 0, 4, 0, 0, 9, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2704, 0, 4096, 0, 0, 0, 3736, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 2941, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 2284, 0, 47, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 151, 4096, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4069, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4, 4, 0, 0, 0, 3, 0, 4090, 4, 4, 2, 3, 4, 3, 359, 6, 4088, 0, 0, 0, 3983, 12, 4092, 0, 2, 2, 4092, 4, 4089, 0, 4090, 0, 4092, 5, 0, 4093, 0, 0, 3727, 0, 4, 4092, 7, 0, 4092, 2, 0, 0, 4, 4092, 4092, 1, 3, 0, 4089, 0, 2, 0, 0, 0, 3, 4092, 0, 4, 4, 4088, 3, 4092, 3, 4, 1576, 0, 1171, 1, 0, 0, 3, 4, 4088, 4091, 0, 0, 4092, 4091, 3, 4091, 3, 3, 4092, 4, 4092, 0, 4077, 0, 4079, 1480, 3, 4, 3, 2, 0, 7, 0, 0, 4, 0, 4, 0, 4089, 1, 0, 2, 2, 0, 0, 0, 4, 0, 4092, 4084, 2, 4091, 0, 3, 0, 0, 4, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1732, 0, 4096, 0, 0, 0, 3892, 5, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 3405, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 2641, 0, 445, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 56, 4096, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 85, 0, 4096, 0, 0, 0, 4085, 318, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4025, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 2526, 0, 832, 0, 0, 0, 0, 0, 4096, 4096, 15, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 11, 4096, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 1854, 0, 4096, 0, 0, 0, 4093, 43, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 3663, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 1458, 0, 735, 0, 0, 0, 0, 0, 4096, 4096, 12, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 98, 4096, 334, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 4096, 0, 0, 5, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2059, 0, 4096, 0, 0, 0, 3984, 4, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 3002, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 2716, 0, 311, 0, 0, 0, 0, 0, 4096, 4096, 5, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 47, 4096, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 948, 0, 4096, 4032, 2842, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4083, 2, 64, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 679, 0, 0, 3475, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1150, 0, 0, 2259, 0, 1141, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3516, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 42, 4096, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 2946, 580, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16016, 16437, 16442, 16384, 16838, 16187, 16382, 16386] avg=16384.0000 max=16838 max_rank=4 min=16016 min_rank=0 imbalance_ratio=1.027710 max_load_reduction=114234 max_load_reduction_pct=0.871536 moved_to_echo=65793 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2704, 0, 4096, 4032, 2842, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3736, 0, 64, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 679, 0, 0, 2941, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1150, 0, 0, 2284, 0, 47, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3516, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 151, 4096, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4069, 0, 4096, 0, 0, 0, 0, 0, 0, 2946, 580, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17772, 16088, 15908, 16383, 15769, 16407, 16382, 16363] avg=16384.0000 max=17772 max_rank=0 min=15769 min_rank=4 imbalance_ratio=1.084717 max_load_reduction=113300 max_load_reduction_pct=0.864410 moved_to_echo=64921 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 3, 0, 0, 0, 2, 0, 4092, 2, 1, 0, 2, 2, 2, 744, 4, 4093, 4029, 2841, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 4027, 11, 64, 0, 1, 1, 0, 3, 4092, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 678, 1, 0, 3101, 0, 1, 4093, 3, 1, 4094, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 4094, 0, 2, 0, 4091, 0, 1, 2, 0, 0, 2, 4093, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4090, 2, 1149, 2, 4, 1912, 1, 1518, 1, 1, 0, 2, 3, 4091, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 3513, 2, 4092, 0, 2, 0, 2, 0, 0, 4084, 0, 4085, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 3, 0, 0, 4, 0, 0, 3, 0, 2, 0, 4091, 0, 0, 1, 4094, 4094, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 1, 4093, 4072, 3, 4093, 0, 4, 0, 0, 9, 0, 2943, 580, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15820, 16386, 16070, 16385, 16871, 16775, 16388, 16377] avg=16384.0000 max=16871 max_rank=4 min=15820 min_rank=0 imbalance_ratio=1.029724 max_load_reduction=114201 max_load_reduction_pct=0.871284 moved_to_echo=66411 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4, 4, 0, 0, 0, 3, 0, 4090, 4, 4, 2, 3, 4, 3, 359, 6, 4092, 4028, 2840, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 3983, 12, 64, 0, 2, 2, 0, 4, 4089, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 5, 0, 678, 0, 0, 3727, 0, 4, 4092, 7, 0, 4092, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 4092, 1, 3, 0, 4089, 0, 2, 0, 0, 0, 3, 4092, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4088, 3, 1149, 3, 4, 1576, 0, 1171, 1, 0, 0, 3, 4, 4088, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3511, 3, 4091, 3, 3, 0, 4, 0, 0, 4077, 0, 4079, 1480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 3, 2, 0, 7, 0, 0, 4, 0, 4, 0, 4089, 1, 0, 2, 4092, 4092, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 4092, 4084, 2, 4091, 0, 3, 0, 0, 4, 0, 2943, 580, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15448, 16334, 16699, 16382, 16185, 17251, 16393, 16380] avg=16384.0000 max=17251 max_rank=5 min=15448 min_rank=0 imbalance_ratio=1.052917 max_load_reduction=113821 max_load_reduction_pct=0.868385 moved_to_echo=66209 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1732, 0, 4096, 4037, 2847, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3892, 5, 59, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 669, 0, 0, 3405, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1144, 0, 0, 2641, 0, 445, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3506, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 56, 4096, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 2952, 585, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16811, 16244, 16362, 16389, 16517, 15966, 16381, 16402] avg=16384.0000 max=16811 max_rank=0 min=15966 min_rank=5 imbalance_ratio=1.026062 max_load_reduction=114261 max_load_reduction_pct=0.871742 moved_to_echo=65266 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 1854, 0, 4096, 4032, 2842, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 43, 64, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 679, 0, 0, 3663, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1150, 0, 0, 1458, 0, 735, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 3516, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 98, 4096, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 4096, 0, 0, 5, 0, 0, 0, 2946, 580, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16923, 16488, 16630, 16382, 15631, 16248, 16383, 16387] avg=16384.0000 max=16923 max_rank=0 min=15631 min_rank=4 imbalance_ratio=1.032898 max_load_reduction=114149 max_load_reduction_pct=0.870888 moved_to_echo=64649 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 85, 0, 4096, 4032, 2842, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4085, 318, 64, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 679, 0, 0, 4025, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1150, 0, 0, 2526, 0, 832, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 3516, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 11, 4096, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 2946, 580, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15155, 16755, 16992, 16383, 16796, 16222, 16381, 16388] avg=16384.0000 max=16992 max_rank=2 min=15155 min_rank=0 imbalance_ratio=1.037109 max_load_reduction=114080 max_load_reduction_pct=0.870361 moved_to_echo=65787 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2059, 0, 4096, 4032, 2842, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3984, 4, 64, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 679, 0, 0, 3002, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1150, 0, 0, 2716, 0, 311, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 3516, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 47, 4096, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 2946, 580, 575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17127, 16340, 15969, 16384, 16465, 16016, 16382, 16389] avg=16384.0000 max=17127 max_rank=0 min=15969 min_rank=2 imbalance_ratio=1.045349 max_load_reduction=113945 max_load_reduction_pct=0.869331 moved_to_echo=65252 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 0, 0, 32000, 512, 512, 0, 128, 128, 0, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 0, 0, 128, 128, 32768, 128, 128, 128, 128, 128, 128, 10496, 128, 32768, 32256, 22784, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 32768, 128, 128, 0, 32768, 0, 128, 128, 0, 0, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 0, 28160, 128, 32768, 128, 128, 0, 128, 0, 0, 32768, 512, 32768, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 5504, 128, 0, 27392, 0, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 9216, 128, 128, 17408, 128, 6272, 128, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 0, 128, 0, 128, 0, 32768, 128, 0, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 32768, 128, 32768, 0, 128, 128, 0, 128, 0, 23680, 4736, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[4093, 0, 3, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 4093, 4093, 3, 4094, 24, 2, 2, 2, 0, 2, 4094, 2, 3, 0, 2, 4093, 1, 4029, 4093, 3, 4085, 4, 2, 0, 3, 3, 24, 3, 0, 1, 0, 0, 0, 0, 4, 0, 1, 16, 0, 4070, 0, 0, 4092, 4094, 1, 4091, 3, 0, 0, 0, 0, 0, 1, 2, 4093, 1, 1, 0, 2, 4092, 4071, 2, 2, 16, 4092, 0, 0, 0, 0, 2, 2, 0, 0, 3, 3, 2, 0, 4093, 0, 0, 3, 0, 3, 4093, 3, 4093, 4091, 0, 4092, 4094, 4087, 3, 4090, 4086, 0, 0, 0, 0, 0, 1, 0, 4094, 5, 0, 2, 4093, 4093, 4093, 4092, 0, 4093, 2, 52, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 391, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4065, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3736, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 4096, 4094, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[4091, 0, 4, 0, 0, 0, 4, 0, 0, 3, 0, 0, 0, 4092, 4092, 4, 4092, 18, 4, 2, 4, 0, 2, 4093, 3, 4, 0, 4, 4092, 1, 3958, 4092, 1, 4089, 6, 3, 0, 4, 4, 46, 4, 0, 0, 0, 0, 0, 0, 6, 0, 2, 33, 0, 4060, 0, 0, 4089, 4092, 0, 4091, 1, 1, 0, 0, 0, 0, 2, 3, 4092, 2, 0, 0, 2, 4089, 4076, 4, 4, 22, 4091, 0, 0, 0, 0, 4, 4, 0, 0, 5, 3, 4, 0, 4092, 0, 0, 5, 0, 4, 4092, 4, 4092, 4087, 0, 4080, 4093, 4071, 4, 4088, 4087, 0, 0, 0, 0, 2, 0, 0, 4092, 9, 0, 4, 4091, 4092, 4092, 4090, 0, 4092, 4, 103, 4, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 175, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4090, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3923, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 4, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 158, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4083, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3951, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 4096, 4094, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 42, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4054, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 4096, 4095, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 2, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 4096, 938, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 4096, 4091, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3161, 0, 0, 8, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 4096, 4090, 4096, 0, 4096, 4089, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 461, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4088, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3643, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4085, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4085, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3923, 0, 0, 0, 4096, 0, 0, 245, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4077, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16308, 16375, 16387, 16472, 16382, 16395, 16369] avg=16384.0000 max=16472 max_rank=4 min=16308 min_rank=1 imbalance_ratio=1.005371 max_load_reduction=114600 max_load_reduction_pct=0.874329 moved_to_echo=65618 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 3, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 4093, 4093, 3, 4080, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 24, 2, 2, 2, 0, 2, 0, 2, 3, 0, 2, 4093, 1, 4029, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4085, 4, 2, 0, 3, 3, 24, 3, 0, 1, 0, 0, 0, 0, 4, 4093, 4093, 4083, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 16, 0, 4070, 0, 0, 4092, 4094, 1, 4091, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4093, 1, 1, 0, 2, 4092, 4071, 2, 2, 16, 4092, 0, 0, 245, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 3, 3, 2, 0, 4093, 0, 0, 3, 0, 3, 4094, 4094, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4091, 0, 4092, 0, 4087, 3, 10, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 0, 2, 4093, 4093, 4074, 4092, 0, 0, 2, 52, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16104, 16403, 16371, 16636, 16390, 16372, 16416] avg=16384.0000 max=16636 max_rank=4 min=16104 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=114436 max_load_reduction_pct=0.873077 moved_to_echo=65814 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4085, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4065, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4085, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3736, 0, 0, 0, 4096, 0, 0, 245, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 11, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4077, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16499, 16376, 16387, 16285, 16382, 16394, 16365] avg=16384.0000 max=16499 max_rank=1 min=16285 min_rank=4 imbalance_ratio=1.007019 max_load_reduction=114573 max_load_reduction_pct=0.874123 moved_to_echo=65426 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 4, 0, 0, 0, 4, 0, 0, 3, 0, 0, 0, 4092, 4092, 4, 4078, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3848, 18, 4, 2, 4, 0, 2, 0, 3, 4, 0, 4, 4092, 1, 3958, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 6, 3, 0, 4, 4, 46, 4, 0, 0, 0, 0, 0, 0, 6, 4092, 4092, 4082, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 33, 0, 4060, 0, 0, 4089, 4092, 0, 4091, 1, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4092, 2, 0, 0, 2, 4089, 4076, 4, 4, 22, 4091, 0, 0, 244, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 5, 3, 4, 0, 4092, 0, 0, 5, 0, 4, 4093, 4092, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4087, 0, 4080, 0, 4071, 4, 10, 4087, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4, 4091, 4092, 4073, 4090, 0, 0, 4, 103, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16032, 16431, 16372, 16647, 16396, 16345, 16470] avg=16384.0000 max=16647 max_rank=4 min=16032 min_rank=1 imbalance_ratio=1.016052 max_load_reduction=114425 max_load_reduction_pct=0.872993 moved_to_echo=65858 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4091, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3849, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4084, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3951, 0, 0, 0, 4096, 0, 0, 247, 23, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 1, 4094, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4067, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16391, 16282, 16375, 16390, 16513, 16381, 16385, 16355] avg=16384.0000 max=16513 max_rank=4 min=16282 min_rank=1 imbalance_ratio=1.007874 max_load_reduction=114559 max_load_reduction_pct=0.874016 moved_to_echo=65634 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 4085, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4079, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 4096, 4091, 0, 4096, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3161, 0, 0, 8, 4096, 0, 6, 245, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 11, 4089, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4077, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 17074, 16362, 16391, 15724, 16379, 16391, 16365] avg=16384.0000 max=17074 max_rank=1 min=15724 min_rank=4 imbalance_ratio=1.042114 max_load_reduction=113998 max_load_reduction_pct=0.869736 moved_to_echo=64859 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4085, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4085, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4054, 0, 0, 0, 4096, 0, 0, 245, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 11, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4077, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16181, 16374, 16387, 16603, 16381, 16395, 16367] avg=16384.0000 max=16603 max_rank=4 min=16181 min_rank=1 imbalance_ratio=1.013367 max_load_reduction=114469 max_load_reduction_pct=0.873329 moved_to_echo=65746 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4085, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4085, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3643, 0, 0, 0, 4096, 0, 0, 245, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4077, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16592, 16376, 16387, 16192, 16381, 16395, 16365] avg=16384.0000 max=16592 max_rank=1 min=16192 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=114480 max_load_reduction_pct=0.873413 moved_to_echo=65333 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[30848, 2304, 128, 128, 128, 0, 128, 0, 128, 128, 0, 128, 32768, 128, 32512, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 128, 0, 0, 128, 0, 128, 0, 32768, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 128, 32768, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 0, 128, 32768, 30720, 128, 128, 128, 32768, 0, 128, 2048, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 0, 128, 128, 128, 0, 32768, 0, 0, 128, 0, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 32768, 0, 0, 32768, 32768, 128, 32768, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 128, 0, 32768, 0, 32768, 0, 32768, 128, 128, 32768, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 0, 128, 0, 128, 32768, 32768, 32640, 32768, 128, 0, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[0, 1, 4093, 4092, 2, 1, 4, 4087, 4093, 0, 1, 0, 0, 3, 3, 0, 0, 0, 4093, 3, 3, 0, 4093, 4092, 0, 3, 2, 0, 696, 4091, 4092, 0, 0, 0, 1, 2, 2, 4093, 4093, 4094, 4093, 0, 0, 3, 3, 0, 2, 3, 3, 0, 3, 0, 0, 1, 2, 4093, 9, 3, 0, 0, 4059, 2, 2, 0, 0, 4093, 2, 0, 0, 1, 4093, 0, 2, 3, 4092, 4, 4091, 4093, 4081, 2, 2, 0, 3, 0, 2, 4093, 0, 4090, 4088, 3635, 4093, 1, 2, 0, 0, 0, 4090, 4092, 3, 3, 0, 6, 0, 0, 2, 1, 0, 2, 0, 4094, 0, 26, 3, 4093, 0, 2, 0, 0, 8, 4, 3, 4093, 0, 3858, 3, 2, 2, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[0, 2, 4091, 4091, 5, 0, 6, 4082, 4091, 0, 0, 0, 0, 4, 4, 0, 0, 0, 4091, 5, 4, 0, 4092, 4091, 0, 4, 2, 0, 939, 4091, 4088, 0, 0, 0, 2, 5, 2, 4092, 4092, 4093, 4092, 0, 0, 4, 1, 0, 3, 5, 4, 0, 4, 2, 0, 2, 2, 4092, 16, 4, 0, 0, 3984, 3, 4, 0, 0, 4091, 3, 0, 0, 0, 4092, 0, 3, 2, 4089, 5, 4089, 4091, 4086, 3, 3, 0, 3, 0, 5, 4091, 0, 4090, 4090, 3451, 4091, 0, 4, 0, 0, 3, 4078, 4090, 4, 5, 0, 16, 0, 0, 4, 2, 0, 2, 0, 4093, 0, 25, 4, 4092, 0, 2, 0, 0, 3, 8, 4, 4092, 0, 3855, 5, 3, 5, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 93, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4003, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 23, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 4073, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4095, 0, 46, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4095, 4096, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 124, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 3980, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 4, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 36, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 4096, 4060, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4095, 0, 22, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 53, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 4033, 0, 0, 0, 0, 4089, 0, 0, 0, 4, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 4096, 4044, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 4076, 0, 60, 0, 4093, 0, 0, 8, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 50, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4032, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 32, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4093, 4092, 2, 1, 4, 4087, 4093, 0, 1, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 3, 3, 0, 0, 4092, 0, 3, 2, 0, 696, 4091, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 4093, 4093, 4094, 4093, 0, 0, 3, 3, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 1, 2, 4093, 9, 3, 0, 0, 4059, 2, 2, 0, 4093, 4093, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 1, 0, 0, 2, 3, 4092, 4, 4091, 8, 4081, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 2, 4093, 0, 4090, 4088, 3635, 189, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4092, 3, 3, 0, 6, 0, 0, 2, 1, 0, 2, 0, 4094, 0, 26, 3813, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 2, 0, 0, 8, 4, 3, 4093, 0, 3858, 3, 2, 2, 2, 4056, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16825, 16389, 16392, 16379, 16105, 16382, 16220] avg=16384.0000 max=16825 max_rank=1 min=16105 min_rank=5 imbalance_ratio=1.026917 max_load_reduction=114247 max_load_reduction_pct=0.871635 moved_to_echo=65086 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 93, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4003, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 3816, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4059, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16227, 16384, 16413, 16389, 16480, 16357, 16438] avg=16384.0000 max=16480 max_rank=5 min=16227 min_rank=1 imbalance_ratio=1.005859 max_load_reduction=114592 max_load_reduction_pct=0.874268 moved_to_echo=65664 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4091, 4091, 5, 0, 6, 4082, 4091, 0, 0, 0, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 5, 4, 0, 0, 4091, 0, 4, 2, 0, 939, 4091, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 2, 4092, 4092, 4093, 4092, 0, 0, 4, 1, 0, 3, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 2, 0, 2, 2, 4092, 16, 4, 0, 0, 3984, 3, 4, 0, 4092, 4092, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 0, 0, 0, 0, 0, 3, 2, 4089, 5, 4089, 8, 4086, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 5, 4091, 0, 4090, 4090, 3451, 189, 0, 4, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4090, 4, 5, 0, 16, 0, 0, 4, 2, 0, 2, 0, 4093, 0, 25, 3811, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 0, 2, 0, 0, 3, 8, 4, 4092, 0, 3855, 5, 3, 5, 4, 4054, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16376, 17066, 16391, 16330, 16379, 15929, 16379, 16222] avg=16384.0000 max=17066 max_rank=1 min=15929 min_rank=5 imbalance_ratio=1.041626 max_load_reduction=114006 max_load_reduction_pct=0.869797 moved_to_echo=64909 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3844, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 23, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4050, 0, 0, 0, 4096, 4096, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 4073, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4095, 0, 46, 3820, 252, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4061, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 16155, 16379, 16374, 16380, 16542, 16408, 16442] avg=16384.0000 max=16542 max_rank=5 min=16155 min_rank=1 imbalance_ratio=1.009644 max_load_reduction=114530 max_load_reduction_pct=0.873795 moved_to_echo=65772 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 53, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 4033, 0, 0, 0, 4096, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4096, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 4096, 4044, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 4076, 0, 60, 3816, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 8, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4059, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16187, 16377, 16356, 16389, 16516, 16416, 16443] avg=16384.0000 max=16516 max_rank=5 min=16187 min_rank=1 imbalance_ratio=1.008057 max_load_reduction=114556 max_load_reduction_pct=0.873993 moved_to_echo=65764 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 36, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 4096, 4060, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4095, 0, 22, 3816, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4059, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16170, 16384, 16403, 16375, 16534, 16384, 16438] avg=16384.0000 max=16534 max_rank=5 min=16170 min_rank=1 imbalance_ratio=1.009155 max_load_reduction=114538 max_load_reduction_pct=0.873856 moved_to_echo=65731 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4095, 4096, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 124, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 3980, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 4, 3816, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 4061, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16258, 16384, 16410, 16390, 16457, 16358, 16431] avg=16384.0000 max=16457 max_rank=5 min=16258 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=114615 max_load_reduction_pct=0.874443 moved_to_echo=65636 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 50, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4077, 0, 0, 0, 4096, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 4032, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 32, 3816, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4059, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16184, 16384, 16394, 16391, 16509, 16388, 16438] avg=16384.0000 max=16509 max_rank=5 min=16184 min_rank=1 imbalance_ratio=1.007629 max_load_reduction=114563 max_load_reduction_pct=0.874046 moved_to_echo=65726 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 32768, 128, 128, 0, 128, 0, 0, 128, 128, 128, 128, 0, 32768, 0, 256, 30592, 2048, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 30848, 128, 128, 0, 0, 32768, 0, 128, 128, 0, 2048, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 128, 128, 128, 32768, 0, 32384, 128, 128, 128, 128, 32512, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 32768, 32768, 128, 128, 128, 32768, 32768, 0, 128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 128, 0, 0, 128, 0, 0, 128, 128, 32768, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 32768, 128, 128, 0, 0, 32512, 128, 128, 0, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 32768, 32768, 32768, 32768, 0, 0, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 128, 32768, 0, 32768, 32768, 31360, 1536, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 4094, 0, 4096, 4096, 0, 64, 4096, 4096, 0, 0, 0, 0, 4095, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 3, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 3215, 1957, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2994, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 16, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 875, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3661, 216, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3433, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[4, 4091, 3, 1, 4, 5, 2, 3, 4091, 4090, 2, 20, 0, 3, 5, 0, 0, 4089, 3998, 6, 4092, 4, 4090, 4090, 4, 0, 4, 0, 4089, 0, 7, 23, 4, 99, 0, 5, 4088, 1, 0, 0, 61, 0, 6, 4092, 4, 0, 0, 51, 4092, 4, 4074, 4, 1, 4086, 4, 4091, 0, 4092, 4, 4, 4091, 0, 268, 0, 4091, 0, 0, 4, 4, 3961, 0, 4092, 4086, 0, 519, 4084, 4088, 9, 5, 3, 2, 4068, 4091, 4, 131, 2, 0, 5, 0, 3, 0, 0, 4091, 1, 0, 2, 1, 1, 5, 0, 4090, 2, 0, 3, 3, 3379, 1156, 4, 1, 0, 4, 0, 0, 4, 0, 4092, 2785, 4044, 0, 1, 6, 1, 7, 4091, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2, 0, 4096, 0, 0, 0, 0, 4074, 0, 4096, 4096, 0, 342, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3169, 823, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3919, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 16, 0, 4096, 0, 4096, 0, 0, 4094, 0, 4096, 0, 4095, 0, 0, 4096, 0, 9, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 215, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 1, 0, 0, 0, 3939, 0, 10, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 3931, 223, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 3979, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[3, 4092, 2, 2, 7, 2, 1, 2, 4093, 4093, 2, 14, 0, 3, 3, 0, 0, 4091, 4032, 5, 4093, 4, 4092, 4092, 3, 0, 2, 0, 4092, 1, 0, 4, 3, 74, 0, 3, 4091, 1, 0, 0, 49, 0, 4, 4093, 3, 0, 0, 27, 4093, 3, 4082, 3, 1, 4089, 3, 4092, 0, 4093, 3, 3, 4093, 0, 110, 0, 4092, 0, 0, 2, 2, 3977, 0, 4093, 4088, 1, 446, 4083, 4091, 3, 4, 2, 1, 4085, 4092, 3, 142, 1, 0, 4, 0, 2, 0, 1, 4092, 2, 4, 2, 1, 2, 1, 0, 4092, 0, 0, 2, 2, 3694, 681, 2, 0, 0, 3, 0, 0, 3, 0, 4093, 3163, 4070, 0, 1, 6, 0, 1, 4093, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 15, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 13, 0, 4096, 0, 0, 0, 0, 3938, 0, 4096, 4096, 0, 227, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2788, 2329, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3269, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 8, 0, 4096, 0, 0, 0, 0, 4055, 0, 4096, 4096, 0, 172, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2670, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3766, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3734, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 16, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 0, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4048, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 4096, 0, 342, 4096, 3800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 3741, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3169, 823, 0, 0, 0, 0, 0, 4096, 3714, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3919, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 346, 296, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16400, 16321, 16338, 16408, 16409, 15922, 16897] avg=16384.0000 max=16897 max_rank=7 min=15922 min_rank=6 imbalance_ratio=1.031311 max_load_reduction=114175 max_load_reduction_pct=0.871086 moved_to_echo=65636 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3735, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 1, 0, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4094, 0, 0, 4096, 0, 64, 4096, 3798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 3, 0, 3741, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 3215, 1957, 0, 0, 0, 0, 0, 4096, 3713, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2994, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 350, 298, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 16396, 16332, 16329, 16150, 16403, 17102, 15982] avg=16384.0000 max=17102 max_rank=6 min=15982 min_rank=7 imbalance_ratio=1.043823 max_load_reduction=113970 max_load_reduction_pct=0.869522 moved_to_echo=65637 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3734, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4096, 0, 0, 0, 16, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 16, 0, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 4096, 0, 4047, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 215, 4096, 3800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 1, 0, 0, 0, 3939, 0, 10, 0, 3741, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 3931, 223, 0, 0, 0, 0, 0, 4096, 3714, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 3979, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 346, 296, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16376, 16403, 16333, 16342, 16303, 16266, 16098, 16951] avg=16384.0000 max=16951 max_rank=7 min=16098 min_rank=6 imbalance_ratio=1.034607 max_load_reduction=114121 max_load_reduction_pct=0.870674 moved_to_echo=65618 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[4, 4091, 3, 1, 4, 5, 2, 3, 4091, 4090, 2, 20, 0, 3, 5, 0, 3728, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3998, 6, 0, 4, 17, 4090, 4, 0, 4, 0, 4089, 0, 7, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 99, 0, 5, 4088, 1, 0, 0, 61, 0, 6, 4092, 4, 0, 0, 51, 4092, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4074, 4, 1, 4086, 4, 4091, 0, 4044, 4, 4, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 3961, 0, 0, 4086, 0, 519, 4084, 3793, 9, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4068, 4091, 4, 131, 2, 0, 5, 0, 3, 0, 0, 4091, 1, 0, 2, 3737, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5, 0, 4090, 2, 0, 3, 3, 3379, 1156, 4, 1, 0, 4, 0, 4092, 3710, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4092, 2785, 4044, 0, 1, 6, 1, 7, 4091, 0, 0, 0, 0, 345, 295, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16406, 16331, 16571, 16584, 16468, 16518, 16475, 15719] avg=16384.0000 max=16584 max_rank=3 min=15719 min_rank=7 imbalance_ratio=1.012207 max_load_reduction=114488 max_load_reduction_pct=0.873474 moved_to_echo=65180 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 4092, 2, 2, 7, 2, 1, 2, 4093, 4093, 2, 14, 0, 3, 3, 0, 3730, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4032, 5, 0, 4, 16, 4092, 3, 0, 2, 0, 4092, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 74, 0, 3, 4091, 1, 0, 0, 49, 0, 4, 4093, 3, 0, 0, 27, 4093, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4082, 3, 1, 4089, 3, 4092, 0, 4045, 3, 3, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 3977, 0, 0, 4088, 1, 446, 4083, 3796, 3, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 4092, 3, 142, 1, 0, 4, 0, 2, 0, 1, 4092, 2, 4, 2, 3737, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 0, 4092, 0, 0, 2, 2, 3694, 681, 2, 0, 0, 3, 0, 4093, 3712, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 3163, 4070, 0, 1, 6, 0, 1, 4093, 1, 0, 0, 0, 346, 295, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16404, 16342, 16510, 16434, 16404, 16549, 16309, 16120] avg=16384.0000 max=16549 max_rank=5 min=16120 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=114523 max_load_reduction_pct=0.873741 moved_to_echo=65382 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3734, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 16, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 1, 0, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4048, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 875, 4096, 3800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 2, 0, 3741, 383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3661, 216, 0, 0, 0, 0, 0, 4096, 3713, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3433, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 1, 0, 346, 296, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16400, 16353, 16352, 16963, 16409, 15806, 16412] avg=16384.0000 max=16963 max_rank=4 min=15806 min_rank=6 imbalance_ratio=1.035339 max_load_reduction=114109 max_load_reduction_pct=0.870583 moved_to_echo=65590 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3734, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 16, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 15, 0, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4048, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3938, 0, 0, 4096, 0, 227, 4096, 3800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3860, 0, 0, 0, 3741, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2788, 2329, 0, 0, 0, 0, 0, 4096, 3714, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3269, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 346, 296, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16400, 16320, 16349, 16157, 16175, 17047, 16247] avg=16384.0000 max=17047 max_rank=6 min=16157 min_rank=4 imbalance_ratio=1.040466 max_load_reduction=114025 max_load_reduction_pct=0.869942 moved_to_echo=65626 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3734, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 16, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 1, 0, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4048, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 4096, 0, 172, 4096, 3800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 3741, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2670, 1713, 0, 0, 0, 0, 0, 4096, 3714, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3766, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 346, 296, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16400, 16332, 16344, 16219, 16343, 16313, 16744] avg=16384.0000 max=16744 max_rank=7 min=16219 min_rank=4 imbalance_ratio=1.021973 max_load_reduction=114328 max_load_reduction_pct=0.872253 moved_to_echo=65619 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 128, 128, 26624, 9216, 128, 128, 0, 128, 0, 32768, 29824, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 32640, 128, 0, 128, 128, 32768, 128, 0, 128, 0, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 256, 0, 128, 32768, 128, 0, 0, 128, 0, 128, 32640, 128, 0, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 128, 128, 32768, 32768, 128, 128, 0, 128, 128, 0, 29952, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 32768, 128, 384, 128, 0, 128, 128, 128, 0, 128, 32384, 128, 128, 128, 29952, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 32768, 27392, 32768, 0, 128, 128, 128, 128, 32768, 128, 0, 128, 0, 2816, 2432, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 32384, 0, 0, 32768, 128, 2944, 32768, 30464, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 128, 128, 32768, 128, 32768, 0, 32384, 128, 128, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 40, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 2156, 0, 0, 0, 0, 0, 0, 832, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 4096, 0, 1079, 0, 4096, 4, 0, 0, 4084, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 2, 2, 0, 4096, 4095, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2336, 0, 0, 0, 0, 0, 0, 1395, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 4093, 0, 0, 0, 4096, 47, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4, 4096, 0, 3, 0, 0, 6, 4096, 0, 394, 0, 4074, 22, 0, 0, 4065, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 2, 0, 4096, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1961, 0, 0, 0, 0, 0, 0, 1155, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 4096, 0, 0, 0, 4096, 109, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 15, 4096, 0, 0, 0, 0, 32, 4096, 0, 975, 0, 4084, 63, 0, 0, 4024, 4096, 4059, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 4092, 1, 3, 17, 3866, 0, 0, 0, 2, 2, 0, 1, 4019, 3, 0, 3, 3, 4091, 3, 0, 4092, 4091, 0, 0, 0, 4093, 4075, 0, 0, 4092, 4092, 2, 4, 3, 1, 135, 4, 3, 4084, 0, 3118, 3, 0, 3, 0, 115, 1, 661, 0, 4, 3, 3, 3, 2, 4091, 2, 0, 4092, 2, 3, 3, 0, 0, 3, 4092, 4093, 3, 0, 4093, 3, 0, 0, 4067, 0, 4093, 4, 4087, 0, 4093, 4092, 2, 0, 4092, 0, 4091, 3, 0, 0, 4092, 3, 2, 4, 0, 4093, 0, 0, 0, 2, 55, 4089, 20, 325, 4, 4093, 0, 0, 0, 4093, 4092, 4093, 0, 0, 0, 0, 2, 1, 0, 4093, 2, 4, 3, 2, 5, 3, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 1, 0, 0, 4096, 4095, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 763, 0, 0, 0, 0, 0, 0, 2854, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 25, 0, 4096, 4095, 0, 0, 4095, 0, 0, 0, 4096, 1, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 2, 0, 0, 7, 4096, 0, 487, 0, 4096, 1, 0, 0, 4084, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1, 4091, 2, 4, 30, 3832, 0, 0, 8, 2, 3, 0, 13, 4008, 5, 0, 7, 4, 4085, 6, 0, 4090, 4089, 0, 0, 0, 4092, 4063, 0, 0, 4090, 4090, 4, 5, 4, 0, 224, 5, 4, 4071, 0, 1958, 4, 0, 4, 0, 113, 0, 876, 0, 6, 4, 4, 5, 3, 4082, 3, 0, 4090, 4, 4, 5, 0, 2, 4, 4091, 4092, 4, 0, 4091, 4, 0, 0, 4060, 0, 4092, 6, 4085, 0, 4092, 4090, 2, 0, 4091, 0, 4089, 4, 0, 0, 4092, 4, 5, 6, 0, 4092, 0, 0, 0, 0, 75, 4088, 21, 1212, 6, 4091, 0, 0, 0, 4092, 4090, 4072, 0, 0, 0, 1, 4, 0, 3, 4091, 4, 6, 4, 3, 10, 4, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 7, 0, 0, 4096, 4096, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 875, 0, 0, 0, 0, 0, 0, 2488, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 30, 0, 4096, 4080, 0, 2, 4082, 0, 0, 0, 4096, 48, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 37, 4096, 0, 17, 0, 0, 16, 4096, 0, 741, 0, 4090, 56, 0, 0, 4035, 4096, 4045, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 1, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 1811, 0, 0, 0, 0, 0, 0, 1483, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 14, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 1, 0, 0, 0, 4096, 0, 813, 0, 4093, 14, 0, 0, 4074, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3422, 755, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 40, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 2156, 0, 0, 0, 0, 0, 0, 4096, 4096, 2150, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 832, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4, 4096, 1946, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3341, 0, 1079, 0, 4096, 4, 0, 0, 4084, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16478, 16426, 16599, 15741, 16372, 16378, 16698, 16380] avg=16384.0000 max=16698 max_rank=6 min=15741 min_rank=3 imbalance_ratio=1.019165 max_load_reduction=114374 max_load_reduction_pct=0.872604 moved_to_echo=65828 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 3424, 758, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 2, 0, 0, 4095, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2336, 0, 0, 0, 0, 0, 0, 4096, 4096, 2150, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1395, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1946, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4093, 0, 0, 0, 4096, 47, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4083, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 6, 3338, 0, 394, 0, 4074, 22, 0, 0, 4065, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16453, 16389, 16775, 16301, 16411, 16375, 15989, 16379] avg=16384.0000 max=16775 max_rank=2 min=15989 min_rank=6 imbalance_ratio=1.023865 max_load_reduction=114297 max_load_reduction_pct=0.872017 moved_to_echo=65154 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 3422, 755, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 2, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1961, 0, 0, 0, 0, 0, 0, 4096, 4096, 2150, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1946, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 0, 4096, 109, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 3341, 0, 975, 0, 4084, 63, 0, 0, 4024, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16401, 16385, 16402, 16063, 16470, 16393, 16578, 16380] avg=16384.0000 max=16578 max_rank=6 min=16063 min_rank=3 imbalance_ratio=1.011841 max_load_reduction=114494 max_load_reduction_pct=0.873520 moved_to_echo=65821 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 3422, 755, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 7, 0, 0, 0, 4096, 0, 1, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 875, 0, 0, 0, 0, 0, 0, 4096, 4096, 2150, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2488, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 30, 4096, 1946, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 2, 4082, 0, 0, 0, 4096, 48, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 16, 3341, 0, 741, 0, 4090, 56, 0, 0, 4035, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16430, 16394, 15297, 17424, 16391, 16415, 16341, 16380] avg=16384.0000 max=17424 max_rank=3 min=15297 min_rank=2 imbalance_ratio=1.063477 max_load_reduction=113648 max_load_reduction_pct=0.867065 moved_to_echo=65527 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 1, 3, 17, 3866, 0, 0, 0, 2, 2, 0, 1, 4019, 3, 0, 3419, 754, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 4091, 3, 0, 0, 4091, 0, 0, 0, 0, 4075, 0, 0, 2, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 3, 1, 135, 4, 3, 4084, 0, 3118, 3, 0, 3, 0, 115, 1, 4093, 4092, 2148, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 661, 0, 4, 3, 3, 3, 2, 4091, 2, 0, 4092, 2, 3, 3, 0, 0, 4092, 1944, 671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 3, 0, 4093, 3, 0, 0, 4067, 0, 0, 4, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 4092, 0, 4091, 3, 0, 0, 4086, 3, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 55, 3335, 20, 325, 4, 4093, 0, 0, 0, 4093, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 4093, 2, 4, 3, 2, 5, 3, 0, 0, 0, 4093, 4093, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16198, 16360, 17812, 15576, 16340, 16375, 16020, 16391] avg=16384.0000 max=17812 max_rank=2 min=15576 min_rank=3 imbalance_ratio=1.087158 max_load_reduction=113260 max_load_reduction_pct=0.864105 moved_to_echo=65126 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 4091, 2, 4, 30, 3832, 0, 0, 8, 2, 3, 0, 13, 4008, 5, 0, 3417, 753, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 4085, 6, 0, 0, 4089, 0, 0, 0, 0, 4063, 0, 0, 2, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 4, 0, 224, 5, 4, 4071, 0, 1958, 4, 0, 4, 0, 113, 0, 4092, 4091, 2146, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 876, 0, 6, 4, 4, 5, 3, 4082, 3, 0, 4090, 4, 4, 5, 0, 2, 4090, 1944, 671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4092, 4, 0, 4091, 4, 0, 0, 4060, 0, 0, 6, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 0, 4091, 0, 4089, 4, 0, 0, 4086, 4, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 3335, 21, 1212, 6, 4091, 0, 0, 0, 4092, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 3, 4091, 4, 6, 4, 3, 10, 4, 0, 0, 0, 4092, 4092, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16188, 16346, 16728, 15793, 16333, 16377, 16904, 16403] avg=16384.0000 max=16904 max_rank=6 min=15793 min_rank=3 imbalance_ratio=1.031738 max_load_reduction=114168 max_load_reduction_pct=0.871033 moved_to_echo=66017 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 3424, 755, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 4095, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 763, 0, 0, 0, 0, 0, 0, 4096, 4096, 2150, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2854, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 25, 4096, 1946, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 4096, 1, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7, 3341, 0, 487, 0, 4096, 1, 0, 0, 4084, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16458, 16385, 15205, 17785, 16370, 16379, 16111, 16379] avg=16384.0000 max=17785 max_rank=3 min=15205 min_rank=2 imbalance_ratio=1.085510 max_load_reduction=113287 max_load_reduction_pct=0.864311 moved_to_echo=65239 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 3422, 755, 13, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 1811, 0, 0, 0, 0, 0, 0, 4096, 4096, 2150, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1483, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1946, 672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 14, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3341, 0, 813, 0, 4093, 14, 0, 0, 4074, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16466, 16387, 16254, 16389, 16385, 16380, 16431, 16380] avg=16384.0000 max=16466 max_rank=0 min=16254 min_rank=2 imbalance_ratio=1.005005 max_load_reduction=114606 max_load_reduction_pct=0.874374 moved_to_echo=65576 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 128, 128, 32128, 0, 0, 128, 128, 128, 0, 128, 32640, 128, 0, 27392, 6144, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 0, 32768, 0, 32768, 128, 0, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 256, 26752, 128, 6144, 128, 32768, 256, 0, 0, 32640, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 0, 0, 32768, 128, 128, 0, 0, 32768, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[11776, 0, 128, 128, 128, 128, 128, 32768, 128, 0, 32768, 128, 128, 128, 0, 128, 32768, 15616, 5376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 384, 128, 128, 32768, 0, 15104, 128, 0, 128, 0, 256, 128, 32768, 32768, 17280, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 32768, 128, 128, 32768, 128, 0, 0, 32768, 256, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 128, 128, 128, 32768, 128, 128, 128, 128, 128, 128, 0, 0, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 5, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 3890, 0, 0, 0, 0, 0, 0, 4096, 1978, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2317, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 59, 0, 20, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4053, 0, 0, 0, 0, 0, 0, 4096, 3021, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1037, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 19, 0, 14, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4094, 4096, 0, 3585, 0, 0, 0, 0, 0, 0, 4096, 1596, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2978, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[0, 4035, 5, 5, 0, 4091, 0, 4091, 0, 0, 6, 0, 4090, 4, 0, 1, 0, 4090, 2, 4087, 0, 7, 0, 4, 5, 8, 4091, 4091, 6, 4087, 0, 0, 0, 0, 6, 3, 5, 0, 0, 4, 5, 30, 4091, 4091, 3, 3, 3, 4086, 0, 0, 0, 4, 4089, 6, 0, 4092, 0, 0, 0, 4091, 3, 0, 4, 0, 4082, 5, 0, 4, 5, 4091, 4087, 0, 10, 0, 4090, 4, 3, 4091, 4092, 4, 4062, 0, 3, 4, 0, 11, 4, 4089, 2676, 9, 1, 1, 0, 4092, 0, 3, 4090, 0, 2, 0, 4024, 4078, 2, 0, 2, 4, 8, 3, 1540, 4089, 3, 0, 5, 4, 0, 5, 0, 0, 4088, 0, 0, 0, 0, 4086, 4091, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 2, 0, 6, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 3520, 0, 0, 0, 0, 0, 0, 4096, 2153, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 2511, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[0, 4038, 2, 3, 0, 4092, 0, 4092, 0, 0, 4, 0, 4092, 3, 0, 0, 0, 4092, 2, 4090, 0, 4, 0, 3, 4, 6, 4093, 4093, 4, 4090, 0, 0, 0, 0, 4, 2, 3, 0, 0, 3, 3, 27, 4092, 4092, 2, 2, 2, 4089, 0, 0, 0, 3, 4091, 4, 0, 4093, 0, 0, 0, 4093, 2, 0, 3, 0, 4084, 4, 0, 2, 3, 4092, 4091, 0, 5, 0, 4092, 3, 1, 4092, 4093, 3, 3982, 0, 3, 2, 0, 5, 3, 4091, 2823, 14, 1, 0, 0, 4093, 0, 2, 4090, 0, 3, 2, 4027, 4080, 2, 0, 1, 7, 5, 2, 1475, 4092, 2, 0, 3, 3, 0, 4, 0, 1, 4091, 0, 0, 0, 0, 4089, 4092, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 90, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 47, 0, 19, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 4096, 28, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4077, 4096, 0, 3994, 0, 0, 0, 0, 0, 0, 4096, 3033, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 6, 0, 0, 1003, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 12, 0, 4, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 3769, 0, 0, 0, 0, 0, 0, 4096, 2800, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1607, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4081, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3890, 0, 0, 0, 0, 0, 0, 4096, 1978, 0, 0, 0, 0, 4096, 0, 0, 1791, 15, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2317, 2305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16374, 16377, 16375, 16381, 16389, 15879, 16910, 16387] avg=16384.0000 max=16910 max_rank=6 min=15879 min_rank=5 imbalance_ratio=1.032104 max_load_reduction=114162 max_load_reduction_pct=0.870987 moved_to_echo=65565 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 1, 4096, 0, 4081, 0, 0, 0, 0, 4096, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 4096, 3021, 0, 0, 0, 0, 4096, 0, 0, 1791, 15, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1037, 2305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16429, 16396, 16375, 16382, 16386, 17088, 15630, 16386] avg=16384.0000 max=17088 max_rank=5 min=15630 min_rank=6 imbalance_ratio=1.042969 max_load_reduction=113984 max_load_reduction_pct=0.869629 moved_to_echo=65490 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4035, 5, 5, 0, 4091, 0, 4076, 0, 0, 6, 0, 4090, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 4087, 0, 7, 0, 4, 5, 8, 0, 4083, 6, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 5, 0, 0, 4, 5, 30, 4091, 4091, 3, 3, 3, 4086, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4089, 6, 0, 4092, 0, 0, 0, 4091, 3, 0, 4, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 5, 0, 4, 5, 4, 4087, 0, 10, 0, 4090, 4, 3, 4091, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 3, 4, 0, 11, 4, 4089, 2676, 9, 1, 1, 0, 4092, 0, 3, 1788, 15, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 2, 0, 4024, 4078, 2, 0, 2, 4, 8, 3, 1540, 2301, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 5, 0, 0, 4088, 0, 0, 0, 0, 4086, 4091, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16313, 16379, 16412, 16376, 16389, 16771, 16057, 16375] avg=16384.0000 max=16771 max_rank=5 min=16057 min_rank=6 imbalance_ratio=1.023621 max_load_reduction=114301 max_load_reduction_pct=0.872047 moved_to_echo=65592 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4080, 0, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4091, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 1, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3585, 0, 0, 0, 0, 0, 0, 4096, 1596, 0, 0, 0, 0, 4096, 0, 0, 1793, 16, 12, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2978, 2303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16386, 16381, 16382, 16382, 15199, 17569, 16386] avg=16384.0000 max=17569 max_rank=6 min=15199 min_rank=5 imbalance_ratio=1.072327 max_load_reduction=113503 max_load_reduction_pct=0.865959 moved_to_echo=65536 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4038, 2, 3, 0, 4092, 0, 4077, 0, 0, 4, 0, 4092, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 4090, 0, 4, 0, 3, 4, 6, 0, 4085, 4, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 3, 0, 0, 3, 3, 27, 4092, 4092, 2, 2, 2, 4089, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4091, 4, 0, 4093, 0, 0, 0, 4093, 2, 0, 3, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4, 0, 2, 3, 4, 4091, 0, 5, 0, 4092, 3, 1, 4092, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 0, 3, 2, 0, 5, 3, 4091, 2823, 14, 1, 0, 0, 4093, 0, 2, 1789, 15, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3, 2, 4027, 4080, 2, 0, 1, 7, 5, 2, 1475, 2303, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 4, 0, 1, 4091, 0, 0, 0, 0, 4089, 4092, 0, 0, 0, 4093, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16311, 16380, 16404, 16378, 16384, 16836, 15999, 16380] avg=16384.0000 max=16836 max_rank=5 min=15999 min_rank=6 imbalance_ratio=1.027588 max_load_reduction=114236 max_load_reduction_pct=0.871552 moved_to_echo=65599 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 90, 4096, 0, 4081, 0, 0, 0, 0, 4096, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 4096, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 28, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 4096, 3033, 0, 0, 0, 0, 4091, 0, 0, 1791, 15, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 6, 0, 0, 1003, 2305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16506, 16392, 16373, 16410, 16370, 17033, 15602, 16386] avg=16384.0000 max=17033 max_rank=5 min=15602 min_rank=6 imbalance_ratio=1.039612 max_load_reduction=114039 max_load_reduction_pct=0.870049 moved_to_echo=65391 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4081, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4087, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3520, 0, 0, 0, 0, 0, 0, 4096, 2153, 0, 0, 0, 0, 4094, 0, 0, 1791, 15, 8, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 2511, 2305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16371, 16382, 16377, 16382, 16383, 15686, 17105, 16386] avg=16384.0000 max=17105 max_rank=6 min=15686 min_rank=5 imbalance_ratio=1.044006 max_load_reduction=113967 max_load_reduction_pct=0.869499 moved_to_echo=65560 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 4096, 0, 4081, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 4086, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3769, 0, 0, 0, 0, 0, 0, 4096, 2800, 0, 0, 0, 0, 4096, 0, 0, 1791, 15, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1607, 2305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16380, 16375, 16381, 16389, 16580, 16200, 16386] avg=16384.0000 max=16580 max_rank=5 min=16200 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=114492 max_load_reduction_pct=0.873505 moved_to_echo=65555 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 128, 128, 128, 32768, 0, 32640, 0, 0, 128, 0, 32768, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 32768, 0, 128, 0, 128, 128, 128, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 0, 128, 128, 128, 32768, 0, 128, 0, 32768, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 0, 128, 128, 32640, 32768, 128, 0, 128, 128, 128, 128, 14592, 18432, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 32768, 32768, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[30976, 0, 128, 128, 0, 128, 128, 32768, 20096, 128, 128, 128, 0, 32768, 0, 128, 14336, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 0, 32768, 128, 0, 128, 32768, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 128, 128, 0, 0, 128, 128, 128, 32768, 32768, 128, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 5, 4088, 4, 3974, 4, 0, 0, 0, 3, 3, 1, 6, 3, 4093, 4, 0, 4, 65, 1, 0, 4089, 2, 1, 0, 0, 3, 3, 4, 4091, 4074, 2, 3, 5, 0, 4082, 4091, 4079, 4093, 4, 0, 0, 5, 0, 4091, 4, 0, 10, 0, 3988, 3, 3, 0, 4068, 2, 0, 4091, 4081, 3, 0, 4092, 4092, 5, 0, 4089, 0, 0, 4087, 0, 4, 0, 3, 3, 0, 3, 3, 3, 30, 0, 9, 3, 7, 464, 4092, 4091, 2, 3983, 93, 7, 0, 0, 0, 0, 4066, 4091, 15, 0, 1, 0, 4086, 0, 0, 0, 0, 1, 4092, 2, 0, 0, 3, 7, 29, 3914, 3, 0, 0, 0, 3, 4093, 4082, 5, 4, 4088, 3, 3995, 0, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 6, 4082, 5, 3982, 6, 0, 0, 0, 5, 4, 6, 11, 4, 4091, 5, 0, 5, 212, 1, 0, 4086, 0, 0, 0, 0, 4, 5, 6, 4088, 4059, 3, 4, 7, 0, 4028, 4089, 4077, 4091, 6, 0, 0, 9, 0, 4090, 6, 0, 24, 0, 3974, 4, 5, 0, 4065, 2, 0, 4089, 4086, 4, 0, 4090, 4090, 5, 0, 4087, 0, 0, 4082, 0, 5, 0, 6, 4, 0, 4, 4, 5, 35, 0, 18, 6, 11, 585, 4091, 4089, 2, 3954, 137, 9, 0, 0, 0, 0, 4051, 4089, 19, 0, 2, 0, 4086, 0, 0, 5, 0, 1, 4090, 1, 0, 0, 4, 12, 56, 3776, 4, 0, 0, 0, 4, 4091, 4073, 6, 5, 4085, 4, 3863, 0, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4009, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 89, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 4096, 4096, 4096, 0, 0, 89, 0, 0, 4096, 0, 0, 0, 0, 3992, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4096, 0, 4096, 0, 0, 6, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 32, 0, 0, 4096, 0, 0, 1, 0, 4034, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 4096, 4096, 0, 4096, 0, 0, 8, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4094, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3998, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 104, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4087, 4096, 4096, 4096, 0, 0, 130, 0, 0, 4096, 0, 0, 0, 0, 3956, 0, 0, 0, 4096, 0, 0, 4096, 4093, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 4076, 0, 0, 24, 0, 0, 0, 4096, 4096, 8, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4094, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 39, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 4096, 4096, 4096, 0, 0, 46, 0, 0, 4096, 0, 0, 0, 0, 4058, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 4096, 4096, 0, 4096, 0, 0, 22, 0, 0, 0, 4096, 4096, 4, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3734, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 7, 0, 0, 4096, 0, 0, 27, 0, 4071, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 4096, 4096, 0, 4096, 0, 0, 23, 0, 0, 0, 4096, 4096, 1, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4090, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 0, 0, 56, 0, 0, 4096, 0, 0, 0, 0, 4068, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 0, 4096, 0, 0, 7, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 4088, 4, 3974, 4, 0, 0, 0, 3, 3, 1, 6, 3, 4093, 4, 4005, 187, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 65, 1, 0, 4089, 2, 1, 0, 0, 3, 3, 4, 4091, 4074, 2, 3794, 198, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 0, 4082, 4091, 4079, 0, 4, 0, 0, 5, 0, 4051, 4, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 3, 3, 0, 4068, 2, 0, 4091, 4081, 3, 0, 0, 87, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4087, 0, 4, 0, 3, 3, 0, 3, 3, 3, 30, 0, 9, 4093, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 464, 0, 4091, 2, 3983, 93, 7, 0, 0, 0, 0, 4066, 3893, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4086, 0, 0, 0, 0, 1, 4092, 2, 0, 0, 3, 7, 29, 4092, 4070, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 3, 0, 0, 0, 3, 0, 4082, 5, 4, 4088, 3, 3995, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16402, 16371, 16334, 16331, 16409, 16624, 16394, 16207] avg=16384.0000 max=16624 max_rank=5 min=16207 min_rank=7 imbalance_ratio=1.014648 max_load_reduction=114448 max_load_reduction_pct=0.873169 moved_to_echo=65634 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4009, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4009, 187, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3799, 198, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 89, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 4096, 0, 0, 6, 0, 0, 0, 4096, 3898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4074, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16420, 16414, 16432, 16367, 16374, 16198, 16373, 16494] avg=16384.0000 max=16494 max_rank=7 min=16198 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=114578 max_load_reduction_pct=0.874161 moved_to_echo=65439 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4009, 187, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3799, 198, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 32, 0, 0, 4056, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 4096, 0, 4096, 0, 0, 8, 0, 0, 0, 4096, 3898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4074, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4094, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16440, 16327, 16377, 16409, 16373, 16281, 16373, 16492] avg=16384.0000 max=16492 max_rank=7 min=16281 min_rank=5 imbalance_ratio=1.006592 max_load_reduction=114580 max_load_reduction_pct=0.874176 moved_to_echo=65519 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 4082, 5, 3982, 6, 0, 0, 0, 5, 4, 6, 11, 4, 4091, 5, 4003, 187, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 212, 1, 0, 4086, 0, 0, 0, 0, 4, 5, 6, 4088, 4059, 3, 3793, 198, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 7, 0, 4028, 4089, 4077, 0, 6, 0, 0, 9, 0, 4050, 6, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 4, 5, 0, 4065, 2, 0, 4089, 4086, 4, 0, 0, 87, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4082, 0, 5, 0, 6, 4, 0, 4, 4, 5, 35, 0, 18, 4091, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 11, 585, 0, 4089, 2, 3954, 137, 9, 0, 0, 0, 0, 4051, 3891, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4086, 0, 0, 5, 0, 1, 4090, 1, 0, 0, 4, 12, 56, 4091, 4069, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3776, 4, 0, 0, 0, 4, 0, 4073, 6, 5, 4085, 4, 3863, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16418, 16500, 16300, 16321, 16421, 16754, 16428, 15930] avg=16384.0000 max=16754 max_rank=5 min=15930 min_rank=7 imbalance_ratio=1.022583 max_load_reduction=114318 max_load_reduction_pct=0.872177 moved_to_echo=65533 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3998, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4009, 187, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3799, 198, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 4096, 0, 0, 0, 130, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 4096, 0, 0, 4096, 4093, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 4076, 0, 0, 24, 0, 0, 0, 4096, 3898, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4074, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4094, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 16429, 16465, 16328, 16374, 16202, 16384, 16479] avg=16384.0000 max=16479 max_rank=7 min=16202 min_rank=5 imbalance_ratio=1.005798 max_load_reduction=114593 max_load_reduction_pct=0.874275 moved_to_echo=65439 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4015, 189, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3800, 201, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 46, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 4096, 0, 4096, 0, 0, 22, 0, 0, 0, 4096, 3895, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4074, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16371, 16386, 16427, 16374, 16269, 16374, 16489] avg=16384.0000 max=16489 max_rank=7 min=16269 min_rank=5 imbalance_ratio=1.006409 max_load_reduction=114583 max_load_reduction_pct=0.874199 moved_to_echo=65506 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 3734, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4009, 187, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3799, 198, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 7, 0, 0, 4056, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 334, 0, 4096, 0, 4096, 0, 0, 23, 0, 0, 0, 4096, 3898, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4074, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4090, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16145, 16325, 16378, 16446, 16374, 16544, 16373, 16487] avg=16384.0000 max=16544 max_rank=5 min=16145 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=114528 max_load_reduction_pct=0.873779 moved_to_echo=65778 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4009, 187, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3799, 198, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 56, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 4096, 0, 0, 7, 0, 0, 0, 4096, 3898, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4074, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16454, 16335, 16400, 16443, 16373, 16200, 16373, 16494] avg=16384.0000 max=16494 max_rank=7 min=16200 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=114578 max_load_reduction_pct=0.874161 moved_to_echo=65440 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32256, 128, 0, 0, 0, 128, 0, 32768, 128, 128, 32768, 128, 32512, 0, 0, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 0, 32768, 0, 128, 0, 128, 128, 0, 128, 128, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 32768, 128, 31744, 128, 128, 0, 0, 128, 128, 128, 128, 128, 32768, 128, 32128, 1536, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 32768, 0, 0, 128, 0, 128, 32768, 128, 0, 0, 128, 128, 128, 32768, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 1664, 0, 32768, 128, 32512, 256, 128, 128, 0, 0, 0, 32768, 31232, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 0, 32768, 32768, 32768, 0, 128, 0, 384, 128, 0, 32512, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32256, 128, 128, 0, 32768, 128, 0, 32768, 32768, 128, 0, 0, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 640, 128, 0, 32768, 128, 128, 0, 0, 128, 128, 128, 32768, 32768, 128, 30464, 1664, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 4090, 3, 4090, 4092, 0, 0, 0, 4092, 1, 4086, 4091, 0, 0, 5, 4091, 47, 4086, 2, 4090, 1, 4, 1, 0, 4093, 4093, 2, 4087, 4, 3, 4093, 0, 0, 4, 0, 0, 0, 4, 0, 3, 3, 0, 0, 2, 3, 2, 6, 1, 4093, 0, 6, 0, 0, 2, 5, 4073, 1, 4, 4092, 4, 4078, 12, 0, 2, 0, 4092, 4, 4, 5, 3, 3, 1, 0, 15, 0, 3, 4057, 0, 1, 4093, 0, 4089, 0, 17, 1, 0, 4089, 57, 0, 1, 2, 4088, 0, 4079, 0, 0, 4, 1, 4, 0, 20, 0, 4092, 15, 0, 4, 4091, 0, 4092, 4024, 7, 0, 0, 2, 0, 3, 3, 6, 0, 8, 4077, 3, 4087, 0, 2, 4081, 3, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 4089, 5, 4088, 4090, 0, 0, 0, 4091, 2, 4080, 4089, 0, 0, 11, 4089, 1, 4082, 3, 4088, 1, 6, 1, 0, 4092, 4091, 3, 4085, 5, 5, 4091, 0, 0, 6, 0, 0, 0, 6, 0, 6, 4, 0, 0, 3, 5, 3, 10, 2, 4092, 0, 8, 0, 0, 4, 7, 4067, 0, 6, 4091, 6, 4078, 23, 0, 4, 0, 4090, 7, 7, 8, 5, 5, 1, 0, 20, 0, 6, 4019, 0, 3, 4092, 0, 4083, 0, 18, 4, 0, 4087, 83, 0, 2, 1, 4084, 0, 4086, 0, 0, 5, 0, 3, 0, 73, 0, 4091, 14, 0, 5, 4090, 0, 4090, 4010, 10, 0, 0, 0, 0, 4, 0, 9, 0, 3, 4075, 4, 4087, 0, 3, 4072, 4, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 4096, 0, 0, 0, 4093, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 4094, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 4095, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 44, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4090, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16382, 16379, 16382, 16389, 16371, 16390, 16389] avg=16384.0000 max=16390 max_rank=0 min=16371 min_rank=5 imbalance_ratio=1.000366 max_load_reduction=114682 max_load_reduction_pct=0.874954 moved_to_echo=65539 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4090, 3, 4090, 0, 0, 0, 0, 4092, 1, 4086, 0, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4086, 2, 4090, 1, 4, 1, 0, 0, 0, 2, 4087, 4, 3, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 0, 3, 3, 0, 0, 2, 3, 2, 6, 1, 4093, 4093, 4092, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 6, 0, 0, 2, 5, 4073, 1, 4, 4092, 4, 4078, 12, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 4, 5, 3, 3, 1, 0, 15, 0, 3, 4057, 0, 1, 4093, 4085, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 17, 1, 0, 4076, 57, 0, 1, 2, 4088, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 4, 0, 20, 0, 4092, 15, 0, 4, 4091, 0, 4092, 4024, 7, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 3, 6, 0, 8, 4077, 3, 4087, 0, 2, 4081, 3, 4092, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 16418, 16392, 16369, 16377, 16410, 16361, 16372] avg=16384.0000 max=16418 max_rank=1 min=16361 min_rank=6 imbalance_ratio=1.002075 max_load_reduction=114654 max_load_reduction_pct=0.874741 moved_to_echo=65520 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4090, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16382, 16380, 16382, 16388, 16371, 16390, 16389] avg=16384.0000 max=16390 max_rank=0 min=16371 min_rank=5 imbalance_ratio=1.000366 max_load_reduction=114682 max_load_reduction_pct=0.874954 moved_to_echo=65538 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4089, 5, 4088, 0, 0, 0, 0, 4091, 2, 4080, 0, 0, 0, 11, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4082, 3, 4088, 1, 6, 1, 0, 0, 0, 3, 4085, 5, 5, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 6, 0, 6, 4, 0, 0, 3, 5, 3, 10, 2, 4092, 4091, 4090, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 8, 0, 0, 4, 7, 4067, 0, 6, 4091, 6, 4078, 23, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 7, 7, 8, 5, 5, 1, 0, 20, 0, 6, 4019, 0, 3, 4092, 4083, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 18, 4, 0, 4074, 83, 0, 2, 1, 4084, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3, 0, 73, 0, 4091, 14, 0, 5, 4090, 0, 4090, 4010, 10, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 9, 0, 3, 4075, 4, 4087, 0, 3, 4072, 4, 4090, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 16369, 16402, 16383, 16357, 16435, 16398, 16356] avg=16384.0000 max=16435 max_rank=5 min=16356 min_rank=7 imbalance_ratio=1.003113 max_load_reduction=114637 max_load_reduction_pct=0.874611 moved_to_echo=65546 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 4096, 4095, 0, 0, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16379, 16380, 16371, 16395, 16364, 16402, 16399] avg=16384.0000 max=16402 max_rank=6 min=16364 min_rank=5 imbalance_ratio=1.001099 max_load_reduction=114670 max_load_reduction_pct=0.874863 moved_to_echo=65560 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4090, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3, 4096, 4096, 0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16378, 16380, 16379, 16388, 16374, 16397, 16389] avg=16384.0000 max=16397 max_rank=6 min=16374 min_rank=5 imbalance_ratio=1.000793 max_load_reduction=114675 max_load_reduction_pct=0.874901 moved_to_echo=65548 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4091, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 2, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16382, 16380, 16381, 16389, 16370, 16393, 16389] avg=16384.0000 max=16393 max_rank=6 min=16370 min_rank=5 imbalance_ratio=1.000549 max_load_reduction=114679 max_load_reduction_pct=0.874931 moved_to_echo=65541 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4090, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4046, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16382, 16379, 16425, 16389, 16377, 16341, 16389] avg=16384.0000 max=16425 max_rank=3 min=16341 min_rank=6 imbalance_ratio=1.002502 max_load_reduction=114647 max_load_reduction_pct=0.874687 moved_to_echo=65496 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 128, 128, 128, 0, 32768, 256, 0, 128, 128, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 32768, 128, 32768, 0, 0, 0, 0, 32768, 128, 32768, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 0, 128, 0, 0, 128, 128, 32768, 128, 128, 32768, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 32768, 128, 128, 128, 0, 0, 0, 128, 32768, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 128, 0, 32768, 128, 0, 128, 32768, 128, 32768, 32640, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 128, 128, 0, 0, 128, 128, 128, 128, 128, 32768, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 128, 128, 0, 128, 0, 128, 32768, 0, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 128, 128, 0, 128, 32768, 128, 32768, 0, 128, 32768, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[4006, 4, 4090, 1, 0, 4087, 3, 0, 2, 4092, 0, 0, 0, 4091, 4093, 0, 0, 0, 4092, 0, 4, 3, 4080, 4091, 4092, 3, 3, 0, 4092, 6, 17, 4092, 0, 0, 11, 0, 1, 1, 6, 0, 0, 0, 5, 4, 0, 4093, 0, 4, 3, 4, 0, 0, 3, 0, 4091, 4091, 4086, 1, 4, 0, 5, 4090, 2, 4076, 0, 4058, 0, 2, 3, 0, 4, 0, 2, 0, 0, 3, 0, 1, 3, 0, 0, 4085, 0, 0, 0, 4087, 2, 4092, 2, 0, 4089, 4092, 0, 0, 0, 2, 1, 4, 8, 0, 4089, 3, 4093, 3, 0, 3, 0, 4, 5, 4, 0, 0, 5, 4092, 4092, 0, 0, 4093, 4092, 4091, 2, 60, 3, 13, 5, 4092, 63, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4092, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[3987, 6, 4088, 2, 0, 4081, 6, 0, 4, 4090, 0, 0, 0, 4088, 4091, 0, 0, 0, 4091, 0, 5, 5, 4075, 4089, 4090, 2, 7, 0, 4090, 8, 26, 4091, 0, 0, 16, 7, 2, 2, 0, 3, 0, 0, 6, 7, 0, 4091, 0, 6, 4, 6, 2, 0, 4, 0, 4087, 4089, 4083, 9, 0, 13, 0, 4089, 3, 4064, 0, 4040, 0, 6, 5, 0, 5, 0, 1, 0, 2, 5, 0, 2, 6, 0, 0, 4081, 0, 0, 0, 4085, 0, 4089, 3, 0, 4086, 4090, 0, 0, 0, 0, 0, 5, 13, 0, 4087, 4, 4091, 1, 0, 5, 0, 6, 7, 6, 2, 0, 7, 4089, 4090, 1, 0, 4091, 4086, 4089, 2, 59, 4, 9, 6, 4090, 101, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 35, 4094, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4063, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 28, 4084, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4065, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 16, 4085, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4083, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4093, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4006, 4, 4090, 1, 0, 4087, 3, 0, 2, 28, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 4080, 4091, 4092, 3, 3, 0, 4090, 6, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1, 1, 6, 0, 0, 0, 5, 4, 0, 4093, 0, 4, 4092, 4092, 4080, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 3, 0, 4091, 9, 4086, 1, 4, 0, 5, 4090, 2, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 2, 3, 0, 4, 0, 2, 0, 0, 3, 0, 1, 3, 0, 4093, 4093, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4087, 2, 4092, 2, 0, 4089, 9, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 8, 0, 4089, 3, 4093, 3, 0, 3, 0, 4, 5, 4, 0, 0, 4082, 4064, 25, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4092, 4067, 0, 0, 0, 4092, 4091, 2, 60, 3, 13, 5, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16312, 16389, 16394, 16374, 16349, 16368, 16393, 16493] avg=16384.0000 max=16493 max_rank=7 min=16312 min_rank=0 imbalance_ratio=1.006653 max_load_reduction=114579 max_load_reduction_pct=0.874168 moved_to_echo=65603 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4084, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4068, 26, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4070, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 16382, 16377, 16393, 16380, 16393, 16378, 16357] avg=16384.0000 max=16412 max_rank=0 min=16357 min_rank=7 imbalance_ratio=1.001709 max_load_reduction=114660 max_load_reduction_pct=0.874786 moved_to_echo=65508 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3987, 6, 4088, 2, 0, 4081, 6, 0, 4, 28, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 4075, 4089, 4090, 2, 7, 0, 4088, 8, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 7, 2, 2, 0, 3, 0, 0, 6, 7, 0, 4091, 0, 6, 4091, 4090, 4078, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 2, 0, 4, 0, 4087, 9, 4083, 9, 0, 13, 0, 4089, 3, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 0, 6, 5, 0, 5, 0, 1, 0, 2, 5, 0, 2, 6, 0, 4091, 4091, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 4085, 0, 4089, 3, 0, 4086, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 13, 0, 4087, 4, 4091, 1, 0, 5, 0, 6, 7, 6, 2, 0, 4080, 4062, 25, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4089, 4065, 1, 0, 0, 4086, 4089, 2, 59, 4, 9, 6, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16290, 16395, 16404, 16373, 16340, 16353, 16399, 16518] avg=16384.0000 max=16518 max_rank=7 min=16290 min_rank=0 imbalance_ratio=1.008179 max_load_reduction=114554 max_load_reduction_pct=0.873978 moved_to_echo=65610 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4084, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 9, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4084, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4068, 26, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4070, 0, 0, 0, 4065, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 16382, 16378, 16390, 16412, 16393, 16378, 16327] avg=16384.0000 max=16412 max_rank=0 min=16327 min_rank=7 imbalance_ratio=1.001709 max_load_reduction=114660 max_load_reduction_pct=0.874786 moved_to_echo=65510 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 26, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4091, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 6, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 4094, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4070, 28, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4068, 0, 0, 0, 4063, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16410, 16378, 16385, 16377, 16424, 16385, 16390, 16323] avg=16384.0000 max=16424 max_rank=4 min=16323 min_rank=7 imbalance_ratio=1.002441 max_load_reduction=114648 max_load_reduction_pct=0.874695 moved_to_echo=65522 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4084, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4068, 26, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4070, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 16382, 16378, 16393, 16382, 16393, 16378, 16354] avg=16384.0000 max=16412 max_rank=0 min=16354 min_rank=7 imbalance_ratio=1.001709 max_load_reduction=114660 max_load_reduction_pct=0.874786 moved_to_echo=65507 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4084, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 9, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4085, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 4096, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4068, 26, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4070, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 16382, 16378, 16379, 16404, 16394, 16378, 16345] avg=16384.0000 max=16412 max_rank=0 min=16345 min_rank=7 imbalance_ratio=1.001709 max_load_reduction=114660 max_load_reduction_pct=0.874786 moved_to_echo=65521 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4084, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4068, 26, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4070, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 16382, 16378, 16393, 16381, 16393, 16378, 16355] avg=16384.0000 max=16412 max_rank=0 min=16355 min_rank=7 imbalance_ratio=1.001709 max_load_reduction=114660 max_load_reduction_pct=0.874786 moved_to_echo=65507 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 32768, 128, 0, 128, 0, 128, 128, 128, 128, 0, 32768, 32640, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32640, 128, 32768, 128, 0, 32768, 128, 0, 128, 256, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 128, 0, 128, 128, 128, 128, 0, 128, 128, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 32640, 128, 0, 0, 32768, 32768, 128, 128, 128, 128, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 128, 32768, 32768, 32768, 128, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 32768, 0, 0, 128, 32768, 128, 32768, 128, 0, 32768, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 128, 0, 32768, 128, 32768, 128, 128, 128, 128, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 128, 0, 0, 128, 128, 0, 32768, 0, 128, 32768, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True + [2026-08-30 05:03:36] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 799.3 | throughput per GPU (TFLOP/s/GPU): 417.1 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.890817E+00 | loss scale: 1.0 | grad norm: 6.160 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[620, 1519, 1414, 5, 45, 1, 3135, 65, 536, 1742, 219, 310, 2342, 3547, 0, 2519, 1484, 1289, 2061, 2349, 345, 1025, 155, 1319, 10, 18, 3626, 2208, 1431, 24, 2306, 706, 2330, 0, 287, 422, 387, 436, 2184, 194, 263, 3, 2068, 3181, 0, 2948, 356, 1386, 1502, 2272, 1335, 170, 0, 17, 9, 2547, 1, 210, 98, 442, 2060, 2967, 1774, 11, 1057, 278, 0, 1369, 1352, 821, 306, 969, 2626, 2353, 1510, 66, 764, 8, 272, 1868, 943, 1098, 0, 62, 727, 468, 649, 81, 1741, 15, 2003, 1460, 209, 3021, 255, 0, 2249, 1639, 28, 874, 3, 472, 1360, 958, 393, 1143, 3, 1114, 382, 2320, 1457, 1076, 1660, 2174, 341, 2154, 643, 677, 1134, 1136, 1557, 2973, 689, 1007, 177, 6, 686, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[242, 1012, 1795, 0, 51, 0, 2979, 44, 422, 1429, 200, 753, 2603, 3291, 0, 2535, 1542, 1352, 2433, 2645, 118, 1025, 122, 1074, 1, 13, 3741, 2192, 1954, 10, 2384, 997, 3339, 0, 257, 570, 1011, 610, 1635, 186, 141, 0, 1512, 3388, 0, 3224, 562, 1097, 1584, 2498, 832, 99, 0, 1, 34, 3212, 2, 145, 40, 153, 1947, 3525, 1833, 1, 1744, 221, 0, 1809, 1399, 462, 430, 803, 2306, 2618, 1567, 10, 886, 0, 203, 1260, 700, 858, 0, 35, 876, 627, 410, 64, 1757, 11, 1626, 1536, 168, 2557, 356, 0, 1514, 1243, 30, 939, 0, 821, 1463, 736, 272, 731, 7, 1051, 614, 2555, 1152, 1143, 1859, 2311, 337, 2479, 713, 479, 1498, 1036, 1512, 2609, 712, 902, 101, 6, 546, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[287, 1095, 1330, 0, 65, 3, 2873, 33, 933, 1504, 355, 820, 2329, 3066, 0, 2929, 2036, 1750, 2143, 2216, 61, 1138, 77, 1233, 16, 18, 3713, 1868, 1302, 10, 2105, 455, 2201, 0, 293, 526, 434, 445, 1731, 142, 164, 6, 1858, 2996, 0, 3020, 454, 838, 1892, 2642, 1258, 144, 0, 4, 58, 2638, 0, 353, 106, 333, 2029, 3486, 1434, 23, 1215, 234, 0, 2003, 2365, 262, 317, 773, 2532, 2333, 1444, 35, 629, 7, 238, 1845, 978, 1094, 1, 66, 464, 489, 630, 112, 2353, 13, 1639, 1838, 207, 2441, 290, 0, 1885, 1491, 24, 1297, 0, 467, 1703, 874, 227, 911, 10, 641, 367, 2081, 1003, 1690, 1876, 2409, 388, 2200, 703, 380, 1418, 1648, 1603, 2834, 497, 1467, 250, 7, 598, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[454, 1249, 1538, 0, 90, 0, 2222, 85, 592, 1188, 412, 421, 2438, 3557, 0, 2792, 2166, 1410, 2315, 2589, 88, 1252, 7, 896, 16, 17, 3340, 2240, 1449, 8, 2380, 715, 2916, 1, 345, 555, 932, 577, 1802, 116, 125, 1, 1393, 3249, 0, 2854, 510, 1130, 1982, 2176, 910, 104, 1, 7, 42, 2722, 0, 130, 83, 143, 1837, 3879, 1585, 8, 1195, 240, 0, 1309, 1821, 234, 354, 633, 3203, 2838, 1822, 21, 1070, 5, 308, 2091, 298, 1163, 0, 65, 938, 1055, 335, 86, 1711, 10, 1382, 1398, 139, 2219, 470, 0, 978, 1143, 10, 1102, 0, 580, 1275, 1264, 132, 1238, 27, 1515, 495, 2887, 1631, 1168, 1729, 2675, 513, 2120, 616, 440, 1146, 1328, 1377, 2475, 1055, 820, 420, 0, 448, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[393, 1123, 1781, 2, 227, 0, 2592, 9, 557, 1300, 86, 683, 2732, 2979, 0, 2758, 2013, 1337, 2399, 2120, 76, 864, 124, 1155, 3, 4, 3362, 2214, 1562, 6, 2046, 943, 3178, 2, 400, 593, 639, 537, 1769, 115, 160, 3, 1263, 2922, 0, 3097, 463, 1556, 1974, 1516, 639, 186, 0, 6, 195, 2961, 1, 311, 324, 142, 2088, 3395, 1830, 19, 2377, 322, 0, 1565, 2089, 401, 377, 858, 2785, 2788, 1651, 72, 870, 6, 427, 1157, 364, 1363, 0, 76, 868, 1428, 360, 198, 1481, 24, 1061, 1628, 228, 2274, 356, 3, 799, 1242, 13, 1502, 0, 905, 1527, 887, 215, 947, 27, 1647, 1026, 2639, 724, 862, 1997, 2474, 310, 1853, 923, 349, 1020, 1191, 1607, 2142, 1001, 1143, 395, 9, 505, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[220, 1303, 992, 0, 372, 0, 2646, 42, 795, 772, 189, 300, 2429, 2337, 0, 2825, 2058, 1082, 2089, 2380, 69, 958, 152, 1135, 62, 15, 3556, 2003, 1080, 6, 1790, 983, 3387, 0, 413, 758, 493, 594, 1770, 156, 167, 11, 1824, 2838, 0, 3173, 706, 1376, 1552, 1878, 998, 274, 0, 0, 68, 2646, 0, 325, 345, 304, 1657, 3270, 1861, 47, 1632, 251, 0, 1217, 2795, 498, 925, 835, 2588, 2755, 1942, 20, 862, 0, 198, 1468, 724, 811, 0, 77, 898, 785, 922, 106, 2475, 16, 1583, 1455, 306, 1682, 475, 0, 1234, 1028, 20, 913, 7, 560, 1635, 988, 303, 1116, 39, 1632, 642, 2408, 967, 1203, 2076, 1855, 520, 2416, 786, 531, 1595, 1682, 1558, 2249, 1038, 1135, 271, 35, 796, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[381, 1097, 1185, 0, 73, 0, 2877, 24, 229, 1342, 178, 610, 2203, 3495, 0, 2916, 2144, 1197, 2144, 2086, 173, 1163, 93, 1064, 11, 7, 3744, 2106, 1360, 32, 1897, 905, 2626, 0, 300, 442, 464, 441, 1973, 247, 147, 8, 2014, 3044, 0, 3107, 331, 1008, 1505, 2388, 1196, 167, 0, 5, 46, 3109, 0, 148, 86, 274, 1968, 3702, 1805, 27, 930, 245, 0, 1610, 1861, 409, 328, 528, 2815, 2593, 1689, 7, 1162, 10, 224, 2070, 543, 756, 0, 67, 670, 971, 501, 42, 2046, 11, 1455, 1358, 210, 2642, 476, 1, 1578, 1535, 37, 1253, 1, 847, 1289, 811, 281, 1010, 42, 1068, 429, 2307, 1393, 915, 1447, 2811, 577, 2218, 839, 690, 1176, 1427, 1376, 3030, 1107, 935, 270, 0, 829, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[370, 1045, 1266, 4, 60, 0, 2854, 92, 559, 1572, 266, 303, 1960, 3379, 0, 2962, 1033, 1812, 2157, 2808, 98, 950, 68, 1107, 2, 16, 3615, 2256, 1438, 17, 2530, 814, 2564, 1, 341, 591, 458, 645, 1670, 291, 96, 3, 2051, 3177, 0, 2866, 341, 1285, 1860, 3200, 1335, 123, 0, 82, 63, 3072, 1, 165, 23, 271, 1936, 3420, 1322, 10, 1334, 342, 0, 1306, 1450, 819, 363, 725, 2664, 2637, 1663, 8, 982, 9, 314, 2034, 613, 766, 0, 10, 513, 633, 633, 76, 1729, 45, 1681, 1173, 133, 3001, 242, 0, 1854, 1321, 34, 892, 1, 690, 1104, 655, 237, 870, 3, 826, 555, 2039, 919, 1404, 1740, 2676, 604, 2727, 1058, 407, 1183, 1280, 1706, 3048, 811, 1072, 204, 0, 578, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[620, 1519, 1414, 5, 45, 1, 3135, 65, 536, 1742, 219, 310, 2342, 2896, 0, 2519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1484, 1289, 2061, 1895, 345, 1025, 155, 1319, 10, 18, 0, 2208, 1431, 24, 2306, 706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2330, 0, 287, 422, 387, 436, 2184, 194, 263, 3, 2068, 3000, 0, 2948, 356, 1386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1502, 2272, 1335, 170, 0, 17, 9, 2547, 1, 210, 98, 442, 2060, 2967, 1774, 11, 269, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 278, 0, 1369, 1352, 821, 306, 969, 2321, 2353, 1510, 66, 764, 8, 272, 1868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 943, 1098, 0, 62, 727, 468, 649, 81, 1741, 15, 2003, 1460, 209, 3021, 255, 0, 3626, 995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2249, 1639, 28, 874, 3, 472, 1360, 958, 393, 1143, 3, 1114, 382, 2320, 1457, 1076, 651, 645, 454, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1660, 2174, 341, 2154, 643, 677, 1134, 1136, 1557, 1333, 689, 1007, 177, 6, 686, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17368, 16276, 16264, 15865, 15314, 17353, 17257, 15375] avg=16384.0000 max=17368 max_rank=0 min=15314 min_rank=4 imbalance_ratio=1.060059 max_load_reduction=113704 max_load_reduction_pct=0.867493 moved_to_echo=65299 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[287, 1095, 1330, 0, 65, 3, 2873, 33, 933, 1504, 355, 820, 2329, 2504, 0, 2929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2036, 1750, 2143, 1788, 61, 1138, 77, 1233, 16, 18, 0, 1868, 1302, 10, 2105, 455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2201, 0, 293, 526, 434, 445, 1731, 142, 164, 6, 1858, 2826, 0, 3020, 454, 838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1892, 2642, 1258, 144, 0, 4, 58, 2638, 0, 353, 106, 333, 2029, 3486, 1434, 23, 259, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1215, 234, 0, 2003, 2365, 262, 317, 773, 2239, 2333, 1444, 35, 629, 7, 238, 1845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1094, 1, 66, 464, 489, 630, 112, 2353, 13, 1639, 1838, 207, 2441, 290, 0, 3713, 949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1885, 1491, 24, 1297, 0, 467, 1703, 874, 227, 911, 10, 641, 367, 2081, 1003, 1690, 562, 615, 428, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1876, 2409, 388, 2200, 703, 380, 1418, 1648, 1603, 1270, 497, 1467, 250, 7, 598, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17060, 16000, 14938, 16829, 15939, 17277, 16310, 16719] avg=16384.0000 max=17277 max_rank=5 min=14938 min_rank=2 imbalance_ratio=1.054504 max_load_reduction=113795 max_load_reduction_pct=0.868187 moved_to_echo=66245 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[242, 1012, 1795, 0, 51, 0, 2979, 44, 422, 1429, 200, 753, 2603, 2687, 0, 2535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1542, 1352, 2433, 2134, 118, 1025, 122, 1074, 1, 13, 0, 2192, 1954, 10, 2384, 997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3339, 0, 257, 570, 1011, 610, 1635, 186, 141, 0, 1512, 3195, 0, 3224, 562, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1584, 2498, 832, 99, 0, 1, 34, 3212, 2, 145, 40, 153, 1947, 3525, 1833, 1, 236, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1744, 221, 0, 1809, 1399, 462, 430, 803, 2039, 2618, 1567, 10, 886, 0, 203, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 700, 858, 0, 35, 876, 627, 410, 64, 1757, 11, 1626, 1536, 168, 2557, 356, 0, 3741, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1514, 1243, 30, 939, 0, 821, 1463, 736, 272, 731, 7, 1051, 614, 2555, 1152, 1143, 604, 566, 511, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1859, 2311, 337, 2479, 713, 479, 1498, 1036, 1512, 1170, 712, 902, 101, 6, 546, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16752, 17351, 17339, 16335, 15451, 16195, 15983, 15666] avg=16384.0000 max=17351 max_rank=1 min=15451 min_rank=4 imbalance_ratio=1.059021 max_load_reduction=113721 max_load_reduction_pct=0.867622 moved_to_echo=63295 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[220, 1303, 992, 0, 372, 0, 2646, 42, 795, 772, 189, 300, 2429, 1908, 0, 2825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2058, 1082, 2089, 1920, 69, 958, 152, 1135, 62, 15, 0, 2003, 1080, 6, 1790, 983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3387, 0, 413, 758, 493, 594, 1770, 156, 167, 11, 1824, 2677, 0, 3173, 706, 1376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1552, 1878, 998, 274, 0, 0, 68, 2646, 0, 325, 345, 304, 1657, 3270, 1861, 47, 265, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1632, 251, 0, 1217, 2795, 498, 925, 835, 2288, 2755, 1942, 20, 862, 0, 198, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 724, 811, 0, 77, 898, 785, 922, 106, 2475, 16, 1583, 1455, 306, 1682, 475, 0, 3556, 753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 1028, 20, 913, 7, 560, 1635, 988, 303, 1116, 39, 1632, 642, 2408, 967, 1203, 429, 488, 460, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2076, 1855, 520, 2416, 786, 531, 1595, 1682, 1558, 1008, 1038, 1135, 271, 35, 796, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14793, 15402, 17505, 15651, 17686, 16624, 16107, 17304] avg=16384.0000 max=17686 max_rank=4 min=14793 min_rank=0 imbalance_ratio=1.079468 max_load_reduction=113386 max_load_reduction_pct=0.865067 moved_to_echo=67721 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[393, 1123, 1781, 2, 227, 0, 2592, 9, 557, 1300, 86, 683, 2732, 2433, 0, 2758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2013, 1337, 2399, 1710, 76, 864, 124, 1155, 3, 4, 0, 2214, 1562, 6, 2046, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3178, 2, 400, 593, 639, 537, 1769, 115, 160, 3, 1263, 2756, 0, 3097, 463, 1556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1974, 1516, 639, 186, 0, 6, 195, 2961, 1, 311, 324, 142, 2088, 3395, 1830, 19, 285, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2377, 322, 0, 1565, 2089, 401, 377, 858, 2462, 2788, 1651, 72, 870, 6, 427, 1157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 364, 1363, 0, 76, 868, 1428, 360, 198, 1481, 24, 1061, 1628, 228, 2274, 356, 3, 3362, 717, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 799, 1242, 13, 1502, 0, 905, 1527, 887, 215, 947, 27, 1647, 1026, 2639, 724, 862, 546, 465, 410, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1997, 2474, 310, 1853, 923, 349, 1020, 1191, 1607, 960, 1001, 1143, 395, 9, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16676, 16456, 16531, 16038, 17422, 15791, 16421, 15737] avg=16384.0000 max=17422 max_rank=4 min=15737 min_rank=7 imbalance_ratio=1.063354 max_load_reduction=113650 max_load_reduction_pct=0.867081 moved_to_echo=65371 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[454, 1249, 1538, 0, 90, 0, 2222, 85, 592, 1188, 412, 421, 2438, 2905, 0, 2792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2166, 1410, 2315, 2089, 88, 1252, 7, 896, 16, 17, 0, 2240, 1449, 8, 2380, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2916, 1, 345, 555, 932, 577, 1802, 116, 125, 1, 1393, 3064, 0, 2854, 510, 1130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1982, 2176, 910, 104, 1, 7, 42, 2722, 0, 130, 83, 143, 1837, 3879, 1585, 8, 328, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1195, 240, 0, 1309, 1821, 234, 354, 633, 2831, 2838, 1822, 21, 1070, 5, 308, 2091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 1163, 0, 65, 938, 1055, 335, 86, 1711, 10, 1382, 1398, 139, 2219, 470, 0, 3340, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 978, 1143, 10, 1102, 0, 580, 1275, 1264, 132, 1238, 27, 1515, 495, 2887, 1631, 1168, 652, 537, 500, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1729, 2675, 513, 2120, 616, 440, 1146, 1328, 1377, 1110, 1055, 820, 420, 0, 448, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 17048, 16321, 16122, 16772, 15437, 17178, 15808] avg=16384.0000 max=17178 max_rank=6 min=15437 min_rank=5 imbalance_ratio=1.048462 max_load_reduction=113894 max_load_reduction_pct=0.868942 moved_to_echo=65195 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[381, 1097, 1185, 0, 73, 0, 2877, 24, 229, 1342, 178, 610, 2203, 2850, 0, 2916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2144, 1197, 2144, 1680, 173, 1163, 93, 1064, 11, 7, 0, 2106, 1360, 32, 1897, 905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2626, 0, 300, 442, 464, 441, 1973, 247, 147, 8, 2014, 2867, 0, 3107, 331, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1505, 2388, 1196, 167, 0, 5, 46, 3109, 0, 148, 86, 274, 1968, 3702, 1805, 27, 292, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 930, 245, 0, 1610, 1861, 409, 328, 528, 2481, 2593, 1689, 7, 1162, 10, 224, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 756, 0, 67, 670, 971, 501, 42, 2046, 11, 1455, 1358, 210, 2642, 476, 1, 3744, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1578, 1535, 37, 1253, 1, 847, 1289, 811, 281, 1010, 42, 1068, 429, 2307, 1393, 915, 645, 662, 406, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1447, 2811, 577, 2218, 839, 690, 1176, 1427, 1376, 1349, 1107, 935, 270, 0, 829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15965, 15976, 15975, 16895, 16147, 16512, 16551, 17051] avg=16384.0000 max=17051 max_rank=7 min=15965 min_rank=0 imbalance_ratio=1.040710 max_load_reduction=114021 max_load_reduction_pct=0.869911 moved_to_echo=66261 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[370, 1045, 1266, 4, 60, 0, 2854, 92, 559, 1572, 266, 303, 1960, 2759, 0, 2962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 1812, 2157, 2265, 98, 950, 68, 1107, 2, 16, 0, 2256, 1438, 17, 2530, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2564, 1, 341, 591, 458, 645, 1670, 291, 96, 3, 2051, 2996, 0, 2866, 341, 1285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1860, 3200, 1335, 123, 0, 82, 63, 3072, 1, 165, 23, 271, 1936, 3420, 1322, 10, 273, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1334, 342, 0, 1306, 1450, 819, 363, 725, 2355, 2637, 1663, 8, 982, 9, 314, 2034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 613, 766, 0, 10, 513, 633, 633, 76, 1729, 45, 1681, 1173, 133, 3001, 242, 0, 3615, 1020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1854, 1321, 34, 892, 1, 690, 1104, 655, 237, 870, 3, 826, 555, 2039, 919, 1404, 620, 662, 543, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1740, 2676, 604, 2727, 1058, 407, 1183, 1280, 1706, 1366, 811, 1072, 204, 0, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16072, 16563, 16199, 17337, 16341, 15883, 15265, 17412] avg=16384.0000 max=17412 max_rank=7 min=15265 min_rank=6 imbalance_ratio=1.062744 max_load_reduction=113660 max_load_reduction_pct=0.867157 moved_to_echo=64901 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[14592, 11264, 17792, 15488, 1152, 8448, 896, 9088, 128, 128, 0, 17152, 11648, 128, 17536, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[22656, 128, 2688, 4480, 4864, 4352, 14592, 1536, 1280, 128, 14080, 23424, 0, 24320, 3840, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[5248, 7936, 128, 512, 6016, 6528, 4480, 768, 15360, 256, 12544, 11904, 1664, 19840, 2944, 128, 28800, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[3072, 9472, 11392, 128, 1024, 128, 22272, 512, 4736, 10880, 1920, 4224, 19072, 20992, 0, 22272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[13952, 18688, 8576, 1280, 128, 128, 640, 22912, 128, 1792, 1152, 2176, 15616, 27648, 13568, 256, 2304, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[12160, 10752, 256, 8832, 128, 5376, 11392, 7296, 2176, 8064, 256, 9600, 4608, 19328, 9344, 9472, 4736, 4736, 3712, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[14464, 19456, 3712, 18176, 6400, 3968, 10240, 10752, 12416, 9600, 6912, 8576, 2176, 128, 4992, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[11520, 2176, 0, 12288, 15232, 3968, 3456, 6144, 19072, 20992, 13312, 256, 7296, 128, 2304, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 395, 16, 0, 4096, 4085, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 179, 0, 4094, 0, 4096, 1, 4087, 0, 0, 4096, 1, 0, 4093, 0, 4095, 0, 0, 1, 0, 0, 0, 4035, 0, 0, 1301, 4096, 0, 0, 1713, 0, 0, 4096, 3824, 0, 3, 1222, 0, 0, 4096, 0, 0, 4094, 0, 0, 3638, 2376, 0, 138, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3992, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 2340, 0, 0, 0, 4096, 4079, 3745, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 873, 4096, 0, 0, 4096, 0, 1, 1186, 0, 4095, 0, 0, 1835, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 391, 1, 0, 4096, 4064, 0, 0, 4093, 1, 0, 0, 1, 5, 0, 0, 4096, 0, 0, 4072, 4091, 0, 84, 0, 4092, 0, 4096, 3, 4083, 0, 0, 4096, 1, 0, 4096, 1, 4093, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 1144, 4094, 0, 0, 226, 19, 2, 4096, 4053, 0, 2, 1017, 0, 0, 4096, 0, 0, 4092, 0, 0, 3909, 3074, 0, 253, 0, 1, 6, 0, 0, 0, 0, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2249, 0, 0, 0, 4096, 4096, 4035, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 1002, 4096, 0, 0, 4093, 0, 2, 2417, 0, 4087, 7, 1, 927, 0, 0, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1366, 23, 0, 4092, 4068, 0, 0, 4095, 0, 0, 0, 0, 53, 0, 0, 4096, 0, 0, 4096, 4086, 5, 74, 0, 4090, 0, 4096, 38, 4082, 0, 0, 4096, 0, 0, 4096, 0, 4093, 0, 0, 2, 0, 0, 0, 4076, 0, 2, 535, 4096, 1, 0, 1317, 9, 0, 4096, 3670, 0, 1, 268, 0, 0, 4092, 0, 0, 4071, 0, 0, 3503, 2515, 0, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 4096, 0, 2169, 0, 0, 0, 4096, 4090, 3671, 0, 0, 1, 4096, 0, 1, 0, 0, 0, 2050, 4096, 0, 0, 4096, 0, 11, 2873, 0, 4089, 0, 0, 878, 0, 0, 0, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3, 523, 0, 0, 4096, 4095, 0, 0, 4095, 2, 0, 0, 1, 2, 0, 0, 4095, 0, 0, 4092, 4095, 0, 43, 0, 4094, 0, 4096, 32, 4091, 1, 0, 4096, 2, 0, 4092, 2, 4095, 0, 0, 1, 0, 0, 0, 4084, 0, 0, 2040, 4096, 0, 0, 845, 9, 0, 4096, 2188, 0, 0, 1707, 0, 0, 4095, 0, 0, 4090, 1, 0, 3669, 3679, 0, 161, 0, 2, 5, 0, 1, 0, 1, 0, 1, 3959, 0, 0, 1, 1, 0, 3, 0, 0, 1, 0, 0, 4095, 0, 1463, 0, 0, 0, 4092, 4092, 4064, 0, 0, 5, 4095, 0, 8, 0, 0, 0, 464, 4096, 0, 1, 4095, 1, 6, 996, 1, 4091, 2, 1, 2840, 0, 0, 0, 4085] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 209, 0, 0, 4076, 4025, 0, 9, 4093, 0, 0, 0, 1, 6, 0, 0, 4096, 0, 0, 4096, 4091, 0, 23, 0, 4081, 0, 4096, 84, 4091, 0, 2, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4015, 0, 15, 1175, 4091, 0, 0, 604, 0, 0, 4096, 3821, 0, 0, 335, 0, 0, 4096, 0, 0, 4094, 0, 0, 3600, 3603, 0, 56, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3687, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 2601, 0, 0, 0, 4096, 4055, 4069, 0, 0, 2, 4096, 0, 63, 0, 0, 0, 1575, 4096, 0, 0, 4096, 0, 0, 1173, 0, 4083, 0, 0, 2240, 0, 0, 0, 4071] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 199, 0, 0, 4096, 3318, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4086, 0, 2, 0, 4095, 0, 4093, 3, 4071, 0, 0, 4096, 1, 0, 4096, 1, 4095, 0, 0, 0, 0, 0, 0, 4017, 0, 0, 2773, 4095, 0, 0, 924, 0, 0, 4096, 4026, 0, 5, 1208, 0, 0, 4095, 0, 0, 4093, 0, 0, 3890, 3654, 0, 103, 0, 1, 1, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 2520, 0, 0, 0, 4096, 4025, 4091, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 186, 4096, 0, 0, 4096, 0, 3, 138, 0, 4095, 1, 1, 1887, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 80, 0, 0, 4091, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4096, 1, 163, 0, 4095, 5, 4096, 568, 4033, 0, 0, 4096, 0, 2, 4091, 0, 4086, 0, 0, 1, 0, 24, 0, 4094, 0, 2, 308, 4096, 0, 0, 849, 0, 2, 4096, 2300, 0, 0, 1586, 0, 0, 4096, 0, 0, 4092, 0, 0, 3929, 2832, 0, 481, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 1872, 0, 0, 0, 4096, 4095, 3786, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 702, 4096, 0, 0, 4096, 0, 9, 2603, 0, 4095, 0, 0, 2654, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1, 2, 1, 60, 0, 2, 4089, 3918, 4, 4, 4088, 0, 0, 1, 6, 11, 2, 1, 4094, 0, 0, 4093, 4092, 4, 65, 0, 4086, 0, 4094, 31, 4096, 0, 0, 4093, 0, 0, 4092, 0, 4095, 0, 0, 0, 1, 2, 1, 4044, 3, 1, 1917, 4089, 3, 1, 175, 10, 0, 4093, 4062, 1, 0, 735, 0, 0, 4090, 0, 1, 4089, 0, 0, 4036, 3363, 0, 271, 1, 9, 14, 1, 1, 0, 1, 1, 3, 3837, 8, 1, 0, 3, 0, 0, 3, 0, 2, 0, 1, 4094, 0, 3321, 0, 0, 0, 4093, 4061, 4083, 0, 2, 0, 4085, 3, 27, 1, 1, 3, 506, 4093, 6, 1, 4093, 0, 17, 303, 4, 4096, 1, 0, 2092, 0, 0, 0, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 395, 16, 0, 4096, 4085, 0, 0, 4095, 0, 0, 0, 0, 0, 2646, 759, 311, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 4095, 0, 179, 0, 4094, 0, 0, 1, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4093, 0, 4095, 0, 0, 1, 0, 0, 0, 4035, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1301, 4096, 0, 0, 1713, 0, 0, 1450, 3824, 0, 3, 1222, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3638, 2376, 0, 138, 0, 1, 0, 0, 0, 0, 0, 0, 3073, 2099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 2340, 4096, 1997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3337, 4079, 3745, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 712, 0, 0, 4096, 0, 1, 1186, 0, 4095, 0, 0, 1835, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16519, 16397, 16360, 17705, 15419, 16522, 16130, 16020] avg=16384.0000 max=17705 max_rank=3 min=15419 min_rank=4 imbalance_ratio=1.080627 max_load_reduction=113367 max_load_reduction_pct=0.864922 moved_to_echo=64091 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 523, 0, 0, 4096, 4095, 0, 0, 4095, 2, 0, 0, 1, 2, 2642, 757, 305, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 4095, 0, 43, 0, 4094, 0, 0, 32, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4092, 2, 4095, 0, 0, 1, 0, 0, 0, 4084, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2040, 4096, 0, 0, 845, 9, 0, 1454, 2188, 0, 0, 1707, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 3669, 3679, 0, 161, 0, 2, 5, 0, 1, 0, 1, 0, 3071, 2098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3959, 0, 0, 1, 1, 0, 3, 0, 0, 1, 0, 0, 4095, 0, 1463, 4095, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3335, 4092, 4064, 0, 0, 5, 4095, 0, 8, 0, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 720, 0, 1, 4095, 1, 6, 996, 1, 4091, 2, 1, 2840, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16632, 16303, 16406, 16434, 16778, 15617, 16063, 16839] avg=16384.0000 max=16839 max_rank=7 min=15617 min_rank=5 imbalance_ratio=1.027771 max_load_reduction=114233 max_load_reduction_pct=0.871529 moved_to_echo=65297 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 199, 0, 0, 4096, 3318, 0, 0, 4095, 0, 0, 0, 0, 1, 2642, 758, 305, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 4086, 0, 2, 0, 4095, 0, 0, 3, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 1, 4095, 0, 0, 0, 0, 0, 0, 4017, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2773, 4095, 0, 0, 924, 0, 0, 1454, 4026, 0, 5, 1208, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3890, 3654, 0, 103, 0, 1, 1, 0, 0, 0, 0, 0, 3071, 2096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 2520, 4096, 1997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 4025, 4091, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 720, 0, 0, 4096, 0, 3, 138, 0, 4095, 1, 1, 1887, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15525, 16208, 16340, 18580, 16909, 16734, 15740, 15036] avg=16384.0000 max=18580 max_rank=3 min=15036 min_rank=7 imbalance_ratio=1.134033 max_load_reduction=112492 max_load_reduction_pct=0.858246 moved_to_echo=64419 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1366, 23, 0, 4092, 4068, 0, 0, 4095, 0, 0, 0, 0, 53, 2642, 758, 305, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 4086, 5, 74, 0, 4090, 0, 0, 38, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4093, 0, 0, 2, 0, 0, 0, 4076, 0, 2, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 4096, 1, 0, 1317, 9, 0, 1454, 3670, 0, 1, 268, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 3503, 2515, 0, 223, 0, 0, 0, 0, 0, 0, 0, 0, 3071, 2098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 4096, 0, 2169, 4096, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 4090, 3671, 0, 0, 1, 4096, 0, 1, 0, 0, 0, 2050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 720, 0, 0, 4096, 0, 11, 2873, 0, 4089, 0, 0, 878, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17513, 16326, 16399, 15443, 15481, 15904, 17247, 16759] avg=16384.0000 max=17513 max_rank=0 min=15443 min_rank=3 imbalance_ratio=1.068909 max_load_reduction=113559 max_load_reduction_pct=0.866386 moved_to_echo=65391 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 209, 0, 0, 4076, 4025, 0, 9, 4093, 0, 0, 0, 1, 6, 2642, 758, 305, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 4091, 0, 23, 0, 4081, 0, 0, 84, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4015, 0, 15, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1175, 4091, 0, 0, 604, 0, 0, 1454, 3821, 0, 0, 335, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3600, 3603, 0, 56, 0, 0, 1, 0, 0, 0, 0, 0, 3071, 2098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3687, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 2601, 4096, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 4055, 4069, 0, 0, 2, 4096, 0, 63, 0, 0, 0, 1575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 720, 0, 0, 4096, 0, 0, 1173, 0, 4083, 0, 0, 2240, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16235, 16321, 16353, 15576, 16523, 16483, 17198, 16383] avg=16384.0000 max=17198 max_rank=6 min=15576 min_rank=3 imbalance_ratio=1.049683 max_load_reduction=113874 max_load_reduction_pct=0.868790 moved_to_echo=66587 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 80, 0, 0, 4091, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 2642, 758, 305, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 4096, 1, 163, 0, 4095, 5, 0, 568, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4091, 0, 4086, 0, 0, 1, 0, 24, 0, 4094, 0, 2, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 308, 4096, 0, 0, 849, 0, 2, 1454, 2300, 0, 0, 1586, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 3929, 2832, 0, 481, 0, 0, 0, 0, 0, 0, 0, 0, 3071, 2098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 1872, 4096, 1998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 4095, 3786, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 720, 0, 0, 4096, 0, 9, 2603, 0, 4095, 0, 0, 2654, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16180, 16912, 16430, 14691, 16503, 16067, 16017, 18272] avg=16384.0000 max=18272 max_rank=7 min=14691 min_rank=3 imbalance_ratio=1.115234 max_load_reduction=112800 max_load_reduction_pct=0.860596 moved_to_echo=66859 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 391, 1, 0, 4096, 4064, 0, 0, 4093, 1, 0, 0, 1, 5, 2642, 758, 305, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3927, 4091, 0, 84, 0, 4092, 0, 0, 3, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 1, 4093, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1144, 4094, 0, 0, 226, 19, 2, 1454, 4053, 0, 2, 1017, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 3909, 3074, 0, 253, 0, 1, 6, 0, 0, 0, 0, 0, 3071, 2099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2249, 4096, 1997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3338, 4096, 4035, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 720, 0, 0, 4093, 0, 2, 2417, 0, 4087, 7, 1, 927, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16468, 16280, 16418, 16107, 16505, 16381, 16569, 16344] avg=16384.0000 max=16569 max_rank=6 min=16107 min_rank=3 imbalance_ratio=1.011292 max_load_reduction=114503 max_load_reduction_pct=0.873589 moved_to_echo=65799 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 1, 60, 0, 2, 4089, 3918, 4, 4, 4088, 0, 0, 1, 6, 11, 2640, 757, 305, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 3948, 4092, 4, 65, 0, 4086, 0, 0, 31, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4092, 0, 4095, 0, 0, 0, 1, 2, 1, 4044, 3, 1, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1917, 4089, 3, 1, 175, 10, 0, 1453, 4062, 1, 0, 735, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 4036, 3363, 0, 271, 1, 9, 14, 1, 1, 0, 1, 1, 3069, 2097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3837, 8, 1, 0, 3, 0, 0, 3, 0, 2, 0, 1, 4094, 0, 3321, 4094, 1997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3336, 4061, 4083, 0, 2, 0, 4085, 3, 27, 1, 1, 3, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 719, 6, 1, 4093, 0, 17, 303, 4, 4096, 1, 0, 2092, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16000, 16325, 16366, 16536, 16954, 17364, 16108, 15419] avg=16384.0000 max=17364 max_rank=5 min=15419 min_rank=7 imbalance_ratio=1.059814 max_load_reduction=113708 max_load_reduction_pct=0.867523 moved_to_echo=65845 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 128, 0, 30208, 25216, 0, 1792, 128, 128, 128, 128, 128, 0, 128, 128, 24576, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 128, 128, 32768, 128, 32768, 0, 128, 128, 128, 128, 128, 32512, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[5760, 128, 128, 32768, 128, 128, 11776, 128, 32768, 128, 128, 15360, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 0, 26752, 32640, 31616, 0, 128, 128, 32768, 128, 128, 128, 128, 128, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[11264, 32768, 128, 128, 6656, 128, 128, 11648, 28032, 128, 128, 8192, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 3328, 128, 128, 32768, 31744, 128, 128, 32768, 128, 0, 128, 128, 128, 21248, 6144, 2560, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 0, 0, 0, 31616, 32768, 128, 640, 0, 32768, 128, 0, 768, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 31104, 128, 128, 128, 128, 128, 128, 128, 0, 128, 0, 128, 32768, 0, 18560, 32768, 16000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[2, 4018, 0, 4, 4062, 0, 0, 3135, 0, 20, 5, 0, 0, 4044, 0, 4096, 3, 0, 0, 4074, 4088, 0, 4094, 0, 4096, 4095, 2, 2, 4096, 4090, 980, 1, 2111, 1, 4096, 0, 0, 5, 4093, 0, 0, 3416, 4095, 0, 7, 0, 0, 4089, 0, 23, 0, 2, 0, 1, 2478, 1569, 3, 0, 4094, 4093, 0, 0, 3, 2, 0, 1, 4091, 1, 1, 0, 0, 0, 5, 4094, 0, 0, 4096, 1, 0, 16, 4094, 0, 0, 705, 0, 1, 210, 0, 4093, 5, 0, 0, 0, 4021, 0, 4089, 0, 17, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 3, 0, 1, 4093, 7, 0, 0, 3, 1, 0, 1, 4094, 0, 0, 0, 178, 0, 2509, 0, 0, 3360] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[3, 4080, 0, 0, 4096, 0, 0, 3515, 0, 0, 0, 0, 0, 3987, 0, 4096, 0, 0, 0, 4094, 4095, 0, 4096, 0, 4096, 4096, 0, 0, 4094, 4096, 2204, 0, 3274, 0, 4096, 0, 0, 15, 4079, 0, 0, 2230, 4096, 0, 4, 0, 0, 4096, 0, 883, 0, 0, 0, 0, 505, 675, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 125, 4096, 0, 0, 224, 0, 0, 6, 0, 4096, 10, 0, 0, 0, 3835, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 52, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 307, 0, 3681, 0, 0, 3187] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 4065, 6, 0, 4090, 0, 0, 2160, 0, 0, 1, 0, 0, 3955, 0, 4096, 0, 0, 0, 4095, 4088, 0, 4096, 0, 4096, 4096, 0, 0, 4095, 4096, 1845, 0, 3299, 0, 4096, 0, 0, 10, 4093, 0, 0, 3588, 4096, 0, 1, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 2301, 1726, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 3, 4096, 0, 0, 58, 0, 0, 15, 0, 4096, 2, 0, 0, 0, 4023, 0, 4090, 0, 3, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 35, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 55, 0, 3008, 0, 0, 2638] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 3905, 0, 0, 4096, 0, 0, 3104, 0, 4, 117, 0, 0, 3871, 0, 4096, 0, 0, 0, 4084, 4092, 0, 4096, 0, 4096, 4096, 0, 0, 4095, 4096, 1511, 0, 3239, 0, 4096, 0, 0, 0, 4096, 0, 0, 1354, 4096, 0, 0, 0, 0, 4096, 0, 484, 0, 0, 0, 0, 2437, 631, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 74, 4096, 0, 0, 213, 0, 0, 73, 0, 4096, 16, 0, 0, 0, 3823, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 660, 0, 3764, 0, 0, 3504] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 4092, 4, 0, 4096, 0, 0, 2811, 0, 0, 19, 0, 0, 3919, 0, 4096, 0, 0, 0, 4077, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4092, 4093, 2224, 0, 3869, 9, 4096, 2, 0, 0, 4096, 0, 0, 3043, 4096, 0, 2, 0, 0, 4096, 1, 390, 0, 0, 0, 0, 2356, 527, 0, 0, 4088, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 1, 7, 4095, 0, 0, 100, 0, 0, 79, 0, 4096, 18, 0, 0, 0, 4042, 0, 4075, 0, 11, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 32, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 95, 5, 2178, 0, 0, 2991] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 4075, 1, 0, 4096, 0, 0, 2362, 0, 4, 35, 0, 0, 4009, 0, 4096, 0, 1, 0, 4093, 4095, 0, 4096, 0, 4096, 4096, 0, 0, 4095, 4096, 2369, 0, 3240, 0, 4096, 0, 3, 23, 4087, 0, 0, 2972, 4096, 0, 0, 0, 0, 4095, 0, 191, 0, 0, 0, 0, 841, 748, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 21, 4096, 0, 0, 452, 0, 0, 1, 0, 4089, 3, 0, 0, 0, 3787, 0, 4092, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 67, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 259, 0, 3554, 0, 0, 3779] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 4068, 1, 0, 4096, 0, 0, 2545, 0, 15, 53, 0, 0, 4037, 0, 4096, 0, 0, 0, 4082, 4094, 0, 4096, 0, 4096, 4096, 0, 0, 4093, 4096, 940, 0, 3230, 0, 4096, 0, 0, 1, 4096, 0, 0, 2073, 4096, 0, 6, 0, 0, 4096, 0, 1010, 0, 0, 0, 0, 2450, 258, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 4096, 0, 0, 54, 4090, 0, 0, 367, 0, 0, 166, 0, 4096, 11, 0, 0, 0, 4016, 0, 4092, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 235, 0, 3842, 0, 0, 3412] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 4094, 9, 0, 4095, 1, 1, 2985, 0, 0, 110, 0, 0, 3935, 0, 4096, 0, 0, 0, 4080, 4091, 0, 4095, 1, 4095, 4095, 0, 1, 4095, 4096, 686, 0, 3326, 0, 4095, 2, 1, 6, 4092, 0, 0, 2541, 4095, 1, 0, 1, 0, 4094, 0, 372, 1, 0, 0, 0, 2808, 227, 0, 0, 4092, 4092, 0, 1, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 1, 0, 25, 4091, 0, 1, 391, 1, 0, 10, 0, 4095, 8, 0, 0, 0, 3959, 1, 4096, 0, 3, 0, 0, 1, 1, 4095, 1, 0, 0, 0, 1, 0, 1, 4095, 5, 1, 0, 0, 1, 0, 1, 4095, 1, 0, 1, 262, 0, 3801, 0, 0, 3233] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 4018, 0, 4, 4062, 0, 0, 3135, 0, 20, 5, 0, 0, 4044, 0, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4074, 4088, 0, 0, 0, 0, 0, 2, 2, 4096, 2514, 980, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2111, 1, 0, 0, 0, 5, 4093, 0, 0, 3416, 2335, 0, 7, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 2, 0, 1, 2478, 1569, 3, 0, 4094, 4093, 0, 0, 3, 2, 2744, 2212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 1, 1, 0, 0, 0, 5, 4094, 0, 0, 4096, 1, 0, 16, 1760, 1576, 488, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 705, 0, 1, 210, 0, 3867, 5, 0, 0, 0, 4021, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 3, 0, 1, 4093, 7, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 1, 4094, 0, 0, 0, 178, 0, 2509, 0, 0, 3360, 4095, 1350, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16686, 15760, 16057, 17224, 16356, 16992, 16376, 15621] avg=16384.0000 max=17224 max_rank=3 min=15621 min_rank=7 imbalance_ratio=1.051270 max_load_reduction=113848 max_load_reduction_pct=0.868591 moved_to_echo=65345 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3, 4080, 0, 0, 4096, 0, 0, 3515, 0, 0, 0, 0, 0, 3987, 0, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 2517, 2204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3274, 0, 0, 0, 0, 15, 4079, 0, 0, 2230, 2335, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 883, 0, 0, 0, 0, 505, 675, 0, 0, 4091, 4096, 0, 0, 0, 0, 2746, 2212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 125, 1761, 1579, 488, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 224, 0, 0, 6, 0, 3870, 10, 0, 0, 0, 3835, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 52, 4096, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 307, 0, 3681, 0, 0, 3187, 4096, 1350, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17077, 17004, 16033, 15208, 16467, 16130, 16406, 16747] avg=16384.0000 max=17077 max_rank=0 min=15208 min_rank=3 imbalance_ratio=1.042297 max_load_reduction=113995 max_load_reduction_pct=0.869713 moved_to_echo=65750 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4065, 6, 0, 4090, 0, 0, 2160, 0, 0, 1, 0, 0, 3955, 0, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4088, 0, 0, 0, 0, 0, 0, 0, 4095, 2517, 1845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3299, 0, 0, 0, 0, 10, 4093, 0, 0, 3588, 2335, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2301, 1726, 0, 0, 4096, 4096, 0, 0, 0, 0, 2746, 2212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 3, 1761, 1579, 488, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 58, 0, 0, 15, 0, 3870, 2, 0, 0, 0, 4023, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 35, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 55, 0, 3008, 0, 0, 2638, 4096, 1350, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15673, 16640, 17421, 17178, 16343, 16154, 16391, 15272] avg=16384.0000 max=17421 max_rank=2 min=15272 min_rank=7 imbalance_ratio=1.063293 max_load_reduction=113651 max_load_reduction_pct=0.867088 moved_to_echo=64160 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3905, 0, 0, 4096, 0, 0, 3104, 0, 4, 117, 0, 0, 3871, 0, 1387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4092, 0, 0, 0, 0, 0, 0, 0, 4095, 2515, 1511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3239, 0, 0, 0, 0, 0, 4096, 0, 0, 1354, 2333, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 484, 0, 0, 0, 0, 2437, 631, 0, 0, 4096, 4096, 0, 0, 0, 0, 2747, 2216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 74, 1763, 1581, 493, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 213, 0, 0, 73, 0, 3867, 16, 0, 0, 0, 3823, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 660, 0, 3764, 0, 0, 3504, 4096, 1349, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16484, 16297, 15118, 16707, 16428, 16183, 16357, 17498] avg=16384.0000 max=17498 max_rank=7 min=15118 min_rank=2 imbalance_ratio=1.067993 max_load_reduction=113574 max_load_reduction_pct=0.866501 moved_to_echo=66466 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 4, 0, 4096, 0, 0, 2811, 0, 0, 19, 0, 0, 3919, 0, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 2515, 2224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3869, 9, 0, 2, 0, 0, 4096, 0, 0, 3043, 2335, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 390, 0, 0, 0, 0, 2356, 527, 0, 0, 4088, 4092, 0, 0, 0, 0, 2746, 2212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 1, 7, 1761, 1578, 488, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 100, 0, 0, 79, 0, 3870, 18, 0, 0, 0, 4042, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 32, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 95, 5, 2178, 0, 0, 2991, 4096, 1350, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16337, 17004, 17452, 16412, 16350, 16279, 16398, 14840] avg=16384.0000 max=17452 max_rank=2 min=14840 min_rank=7 imbalance_ratio=1.065186 max_load_reduction=113620 max_load_reduction_pct=0.866852 moved_to_echo=63867 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4075, 1, 0, 4096, 0, 0, 2362, 0, 4, 35, 0, 0, 4009, 0, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 2517, 2369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3240, 0, 0, 0, 3, 23, 4087, 0, 0, 2972, 2335, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 0, 0, 0, 841, 748, 0, 0, 4095, 4096, 0, 0, 0, 0, 2746, 2212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 21, 1761, 1579, 488, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 452, 0, 0, 1, 0, 3864, 3, 0, 0, 0, 3787, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 67, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 259, 0, 3554, 0, 0, 3779, 4096, 1350, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15978, 17170, 16755, 14929, 16362, 16295, 16420, 17163] avg=16384.0000 max=17170 max_rank=1 min=14929 min_rank=3 imbalance_ratio=1.047974 max_load_reduction=113902 max_load_reduction_pct=0.869003 moved_to_echo=66240 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4068, 1, 0, 4096, 0, 0, 2545, 0, 15, 53, 0, 0, 4037, 0, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4094, 0, 0, 0, 0, 0, 0, 0, 4093, 2517, 940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3230, 0, 0, 0, 0, 1, 4096, 0, 0, 2073, 2335, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1010, 0, 0, 0, 0, 2450, 258, 0, 0, 4094, 4094, 0, 0, 0, 0, 2746, 2212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 4096, 0, 0, 4096, 0, 0, 54, 1761, 1579, 488, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 367, 0, 0, 166, 0, 3870, 11, 0, 0, 0, 4016, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 235, 0, 3842, 0, 0, 3412, 4096, 1350, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16211, 15726, 15837, 16864, 16401, 16612, 16361, 17060] avg=16384.0000 max=17060 max_rank=7 min=15726 min_rank=1 imbalance_ratio=1.041260 max_load_reduction=114012 max_load_reduction_pct=0.869843 moved_to_echo=66434 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 9, 0, 4095, 1, 1, 2985, 0, 0, 110, 0, 0, 3935, 0, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4091, 0, 0, 1, 0, 0, 0, 1, 4095, 2517, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3326, 0, 0, 2, 1, 6, 4092, 0, 0, 2541, 2335, 1, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 1, 0, 0, 0, 2808, 227, 0, 0, 4092, 4092, 0, 1, 0, 0, 2745, 2212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 1, 0, 25, 1760, 1579, 488, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1, 391, 1, 0, 10, 0, 3869, 8, 0, 0, 0, 3959, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 1, 4095, 1, 0, 0, 0, 1, 0, 1, 4095, 5, 4095, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 4095, 1, 0, 1, 262, 0, 3801, 0, 0, 3233, 4095, 1350, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16626, 15471, 16399, 16550, 16365, 16427, 16363, 16871] avg=16384.0000 max=16871 max_rank=7 min=15471 min_rank=1 imbalance_ratio=1.029724 max_load_reduction=114201 max_load_reduction_pct=0.871284 moved_to_echo=66026 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32512, 128, 128, 32768, 128, 128, 22656, 0, 128, 384, 0, 0, 31872, 0, 11264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 32768, 32768, 0, 0, 128, 0, 0, 128, 128, 32768, 20224, 12800, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[25600, 128, 0, 128, 128, 128, 32768, 0, 0, 21248, 18688, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 128, 128, 32768, 128, 0, 0, 0, 128, 0, 128, 32768, 256, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 32768, 128, 128, 0, 0, 0, 128, 32768, 0, 0, 32768, 128, 128, 384, 14208, 12672, 3968, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 0, 128, 2560, 128, 128, 640, 0, 30976, 128, 0, 0, 0, 31616, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 0, 128, 32768, 128, 0, 128, 2176, 128, 26368, 0, 0, 26112, 32768, 10880, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 3456, 128, 128, 0, 128, 16256, 6400, 128, 0, 32768, 32768, 0, 128, 128, 128, 22016, 17792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 61, 2259, 4094, 3838, 0, 0, 0, 4096, 0, 4084, 1, 0, 4096, 0, 0, 4, 1385, 0, 3, 5, 0, 0, 120, 0, 0, 4096, 0, 1, 0, 0, 0, 4094, 0, 0, 4096, 4093, 4095, 0, 4033, 4095, 0, 4096, 4096, 4095, 0, 3, 4096, 0, 3523, 0, 0, 0, 3, 3, 2, 0, 4096, 0, 0, 0, 4096, 165, 0, 0, 4095, 4071, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4093, 4091, 1, 0, 0, 0, 4096, 3630, 0, 2261, 1, 0, 0, 0, 0, 0, 3, 4019, 0, 0, 0, 3, 0, 0, 14, 0, 0, 0, 0, 14, 7, 0, 0, 0, 3375, 0, 0, 0, 0, 0, 4094, 4091, 0, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 3663, 4096, 1985, 0, 0, 0, 4096, 0, 4096, 6, 0, 4096, 0, 0, 0, 1165, 0, 0, 0, 0, 0, 628, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4084, 4096, 0, 4069, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 3491, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 325, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4059, 0, 2546, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2651, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 250, 3646, 4096, 3289, 0, 0, 0, 4096, 0, 4096, 1, 0, 4096, 0, 0, 0, 301, 0, 0, 0, 0, 0, 379, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4095, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 2898, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 4096, 55, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3445, 0, 3798, 0, 0, 0, 0, 0, 0, 0, 3937, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2490, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 0, 0, 0, 325, 2310, 4095, 2328, 0, 0, 0, 4095, 1, 4091, 0, 1, 4095, 0, 1, 0, 301, 0, 0, 0, 1, 0, 1925, 0, 0, 4095, 0, 0, 0, 1, 0, 4095, 0, 0, 4095, 4095, 4095, 2, 4095, 4095, 1, 4095, 4095, 4095, 1, 5, 4095, 0, 3280, 0, 1, 1, 0, 0, 0, 0, 4095, 1, 0, 1, 4095, 254, 1, 0, 4094, 4078, 1, 1, 1, 1, 0, 0, 1, 4095, 0, 4095, 4095, 0, 0, 0, 0, 4095, 3508, 0, 3434, 0, 2, 0, 0, 1, 0, 0, 4062, 0, 0, 0, 0, 1, 1, 17, 0, 1, 1, 0, 11, 0, 1, 1, 1, 2834, 0, 1, 0, 0, 1, 4095, 4091, 0, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 74, 2171, 4096, 3717, 0, 0, 0, 4096, 0, 4096, 3, 0, 4095, 0, 0, 0, 221, 0, 0, 0, 0, 0, 257, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 4096, 4096, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 3129, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 683, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4079, 0, 3236, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 1, 0, 0, 18, 0, 0, 0, 0, 11, 0, 0, 0, 0, 2904, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 574, 3886, 4096, 3305, 0, 0, 0, 4096, 0, 4096, 3, 0, 4096, 0, 0, 0, 702, 0, 0, 0, 0, 0, 578, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4093, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 3002, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 139, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3942, 0, 2890, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 12, 0, 0, 0, 1477, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 9, 3836, 4096, 2841, 0, 0, 0, 4096, 0, 4083, 0, 0, 4096, 0, 0, 0, 154, 0, 0, 0, 0, 0, 430, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4089, 4096, 0, 4090, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 3806, 0, 0, 0, 2, 2, 0, 0, 4096, 0, 0, 0, 4096, 155, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4091, 0, 0, 0, 0, 4096, 3813, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 3, 0, 0, 30, 0, 0, 0, 0, 28, 20, 0, 0, 0, 1460, 0, 0, 0, 0, 0, 4096, 4094, 1, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 73, 4010, 4096, 3751, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 684, 0, 0, 0, 0, 0, 739, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4083, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 2353, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 71, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3779, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 3862, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 8, 25, 0, 0, 0, 1194, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 61, 2259, 4094, 3838, 0, 0, 0, 4096, 0, 1665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1, 0, 4096, 0, 0, 4, 1385, 0, 3, 5, 0, 0, 120, 0, 0, 4047, 2897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 4093, 4095, 0, 4033, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 909, 4095, 0, 3, 0, 0, 3523, 0, 0, 0, 3, 3, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 165, 0, 0, 4095, 4071, 0, 0, 0, 0, 0, 0, 0, 2695, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1128, 0, 4093, 4091, 1, 0, 0, 0, 0, 3630, 0, 2261, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4019, 0, 0, 0, 3, 0, 0, 14, 0, 0, 0, 0, 14, 4096, 4096, 4096, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 3375, 0, 0, 0, 0, 0, 4094, 4091, 0, 0, 4095, 0, 1302, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16013, 16642, 16334, 16730, 16302, 15205, 16390, 17456] avg=16384.0000 max=17456 max_rank=7 min=15205 min_rank=5 imbalance_ratio=1.065430 max_load_reduction=113616 max_load_reduction_pct=0.866821 moved_to_echo=65353 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 3663, 4096, 1985, 0, 0, 0, 4096, 0, 1665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 4096, 0, 0, 0, 1165, 0, 0, 0, 0, 0, 628, 0, 0, 4048, 2898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4084, 4096, 0, 4069, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 909, 4096, 0, 0, 0, 0, 3491, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 325, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 2695, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1128, 0, 4096, 4096, 0, 0, 0, 0, 0, 4059, 0, 2546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 4096, 4096, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2651, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 1302, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15506, 16937, 16363, 16688, 16484, 15925, 16437, 16732] avg=16384.0000 max=16937 max_rank=1 min=15506 min_rank=0 imbalance_ratio=1.033752 max_load_reduction=114135 max_load_reduction_pct=0.870781 moved_to_echo=65578 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 250, 3646, 4096, 3289, 0, 0, 0, 4096, 0, 1666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 301, 0, 0, 0, 0, 0, 379, 0, 0, 4047, 2898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4095, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 909, 4096, 0, 0, 0, 0, 2898, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 55, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 2695, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1128, 0, 4096, 4096, 0, 0, 0, 0, 0, 3445, 0, 3798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3937, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 4096, 4096, 4096, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2490, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 4096, 0, 1302, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17044, 15818, 16401, 16098, 16206, 16563, 16374, 16568] avg=16384.0000 max=17044 max_rank=0 min=15818 min_rank=1 imbalance_ratio=1.040283 max_load_reduction=114028 max_load_reduction_pct=0.869965 moved_to_echo=65711 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 574, 3886, 4096, 3305, 0, 0, 0, 4096, 0, 1671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 4096, 0, 0, 0, 702, 0, 0, 0, 0, 0, 578, 0, 0, 4048, 2904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4093, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 901, 4096, 0, 0, 0, 0, 3002, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 139, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 2702, 1182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1120, 0, 4096, 4096, 0, 0, 0, 0, 0, 3942, 0, 2890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 4096, 4096, 4096, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 1477, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 4096, 0, 1305, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17628, 16427, 16391, 16191, 16311, 16144, 16416, 15564] avg=16384.0000 max=17628 max_rank=0 min=15564 min_rank=7 imbalance_ratio=1.075928 max_load_reduction=113444 max_load_reduction_pct=0.865509 moved_to_echo=64435 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 9, 3836, 4096, 2841, 0, 0, 0, 4096, 0, 1666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 4096, 0, 0, 0, 154, 0, 0, 0, 0, 0, 430, 0, 0, 4048, 2898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 4096, 0, 4090, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 909, 4096, 0, 0, 0, 0, 3806, 0, 0, 0, 2, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 155, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 2695, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1128, 0, 4096, 4091, 0, 0, 0, 0, 0, 3813, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 3, 0, 0, 30, 0, 0, 0, 0, 28, 4096, 4096, 4096, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 1460, 0, 0, 0, 0, 0, 4096, 4094, 1, 0, 4096, 0, 1302, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16548, 15709, 16389, 17007, 16302, 17196, 16360, 15561] avg=16384.0000 max=17196 max_rank=5 min=15561 min_rank=7 imbalance_ratio=1.049561 max_load_reduction=113876 max_load_reduction_pct=0.868805 moved_to_echo=65419 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 0, 0, 0, 325, 2310, 4095, 2328, 0, 0, 0, 4095, 1, 1665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1, 4095, 0, 1, 0, 301, 0, 0, 0, 1, 0, 1925, 0, 0, 4048, 2897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 4095, 4095, 2, 4095, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 909, 4095, 1, 5, 0, 0, 3280, 0, 1, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 254, 1, 0, 4094, 4078, 1, 1, 1, 1, 0, 0, 1, 2695, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1128, 0, 4095, 4095, 0, 0, 0, 0, 0, 3508, 0, 3434, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4062, 0, 0, 0, 0, 1, 1, 17, 0, 1, 1, 0, 11, 4095, 4095, 4095, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2834, 0, 1, 0, 0, 1, 4095, 4091, 0, 0, 4095, 0, 1302, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14822, 17360, 16402, 16482, 16404, 16262, 16427, 16913] avg=16384.0000 max=17360 max_rank=1 min=14822 min_rank=0 imbalance_ratio=1.059570 max_load_reduction=113712 max_load_reduction_pct=0.867554 moved_to_echo=66006 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 74, 2171, 4096, 3717, 0, 0, 0, 4096, 0, 1665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 4095, 0, 0, 0, 221, 0, 0, 0, 0, 0, 257, 0, 0, 4048, 2898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 4096, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 909, 4096, 0, 0, 0, 0, 3129, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 683, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 2695, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1128, 0, 4096, 4096, 0, 0, 0, 0, 0, 4079, 0, 3236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 1, 0, 0, 18, 0, 0, 0, 0, 11, 4096, 4096, 4096, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2904, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 4096, 0, 1302, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15819, 15618, 16403, 16326, 16843, 16635, 16445, 16983] avg=16384.0000 max=16983 max_rank=7 min=15618 min_rank=1 imbalance_ratio=1.036560 max_load_reduction=114089 max_load_reduction_pct=0.870430 moved_to_echo=66906 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 73, 4010, 4096, 3751, 0, 0, 0, 4096, 0, 1666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 684, 0, 0, 0, 0, 0, 739, 0, 0, 4048, 2898, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4083, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 909, 4096, 0, 0, 0, 0, 2353, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 71, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 2695, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1128, 0, 4096, 4096, 0, 0, 0, 0, 0, 3779, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3862, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 8, 4096, 4096, 4096, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 1194, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 4096, 0, 1302, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17692, 16561, 16389, 15550, 16220, 17142, 16223, 15295] avg=16384.0000 max=17692 max_rank=0 min=15295 min_rank=7 imbalance_ratio=1.079834 max_load_reduction=113380 max_load_reduction_pct=0.865021 moved_to_echo=64880 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 7296, 32768, 128, 128, 0, 0, 25600, 0, 128, 128, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 32128, 0, 0, 0, 128, 128, 128, 256, 0, 128, 128, 0, 128, 32768, 32768, 32768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 32768, 0, 128, 128, 4992, 0, 128, 128, 128, 0, 5120, 0, 0, 32384, 23296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 128, 0, 32768, 0, 128, 0, 32768, 32768, 128, 32768, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 32768, 1920, 128, 0, 32768, 32768, 128, 128, 128, 128, 0, 0, 128, 21632, 9472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[9088, 0, 32768, 32768, 128, 0, 0, 0, 0, 30336, 0, 26368, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 128, 0, 0, 1408, 25856, 32768, 25088, 0, 0, 0, 32768, 128, 13440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 18432, 0, 128, 0, 0, 128, 32768, 32768, 128, 0, 32768, 0, 10496, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 6, 125, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 4090, 0, 40, 0, 4096, 0, 0, 0, 0, 3021, 4092, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 21, 0, 0, 4096, 4096, 4096, 0, 4094, 5, 4096, 4096, 4068, 4096, 0, 0, 0, 0, 0, 3939, 0, 0, 0, 2654, 0, 4096, 4076, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 26, 4095, 4078, 0, 0, 4096, 0, 0, 2, 2, 4093, 0, 0, 0, 0, 0, 4096, 0, 19, 2509, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4088, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 280, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 0, 4095, 4095, 4092, 0, 0, 0, 4095, 0, 0, 0, 1, 2900, 4095, 1, 0, 1, 0, 3, 0, 1, 1, 4095, 1, 0, 0, 4095, 14, 0, 1, 4092, 4095, 4095, 0, 4096, 3, 4095, 4095, 4094, 4094, 0, 1, 0, 1, 0, 3993, 0, 1, 0, 2485, 1, 4090, 4095, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 4095, 1, 0, 0, 4092, 4086, 1, 2, 4095, 0, 0, 2, 0, 4095, 0, 1, 0, 1, 0, 4095, 1, 7, 2629, 0, 1, 0, 4095, 0, 4095, 0, 0, 1, 4095, 4095, 0, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 4094, 0, 1, 0, 4096, 0, 0, 0, 0, 3168, 4096, 0, 0, 2, 0, 10, 0, 0, 0, 4096, 0, 0, 0, 4096, 53, 0, 0, 4096, 4096, 4096, 0, 4096, 7, 4096, 4096, 4088, 4096, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 1260, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 4090, 4085, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3, 2553, 0, 0, 0, 4096, 6, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4, 0, 4096, 0, 0, 0, 0, 3127, 4096, 1, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 13, 0, 0, 4096, 4096, 4096, 0, 4095, 0, 4096, 4096, 4096, 4094, 0, 0, 0, 0, 0, 3989, 0, 0, 0, 2348, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 2414, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 889, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4096, 4090, 0, 10, 0, 4096, 0, 0, 0, 0, 3537, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 0, 4096, 6, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 4096, 4064, 4096, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 1006, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 38, 4096, 4093, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 9, 2761, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2494, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 119, 0, 0, 4096, 4096, 4096, 0, 4095, 0, 4096, 4096, 4095, 4096, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 3145, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 22, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1658, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2902, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 94, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 3776, 0, 4067, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1268, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 4095, 0, 9, 0, 4096, 0, 0, 0, 0, 3445, 4081, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 0, 0, 0, 4096, 10, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 4096, 4083, 4096, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 2317, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 51, 4096, 4094, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 8, 1250, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 6, 125, 0, 0, 0, 0, 4096, 4096, 3474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 4090, 3707, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 4096, 0, 0, 0, 0, 3021, 4092, 0, 0, 0, 0, 1, 0, 4096, 622, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 0, 4096, 0, 4055, 0, 4094, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 1794, 0, 0, 0, 0, 0, 3939, 0, 0, 0, 2654, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 26, 4095, 2136, 1906, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 4096, 0, 0, 2, 2, 4093, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 2509, 0, 0, 0, 4096, 0, 1960, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15896, 16378, 16357, 16367, 16551, 16376, 16371, 16776] avg=16384.0000 max=16776 max_rank=7 min=15896 min_rank=0 imbalance_ratio=1.023926 max_load_reduction=114296 max_load_reduction_pct=0.872009 moved_to_echo=66074 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4088, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 280, 0, 1, 1, 1, 4095, 4095, 3473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 0, 4095, 4095, 4092, 3706, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 2900, 4095, 1, 0, 1, 0, 3, 0, 4095, 622, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 4095, 14, 0, 1, 4092, 0, 4054, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1793, 0, 1, 0, 1, 0, 3993, 0, 1, 0, 2485, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 4095, 1, 0, 0, 4092, 2135, 1905, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 2, 4095, 0, 0, 2, 0, 4095, 0, 1, 0, 1, 0, 4095, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2629, 0, 1, 0, 4095, 0, 1960, 0, 0, 1, 0, 4095, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16038, 16390, 16202, 16358, 16459, 16363, 16383, 16879] avg=16384.0000 max=16879 max_rank=7 min=16038 min_rank=0 imbalance_ratio=1.030212 max_load_reduction=114193 max_load_reduction_pct=0.871223 moved_to_echo=66084 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1158, 0, 0, 0, 0, 4096, 4096, 3474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 4094, 3707, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 3168, 4096, 0, 0, 2, 0, 10, 0, 4096, 622, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 53, 0, 0, 4096, 0, 4055, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1794, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 1260, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 4090, 2136, 1906, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2553, 0, 0, 0, 4096, 6, 1960, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16915, 16383, 16480, 16403, 15325, 16379, 16377, 16810] avg=16384.0000 max=16915 max_rank=0 min=15325 min_rank=4 imbalance_ratio=1.032410 max_load_reduction=114157 max_load_reduction_pct=0.870949 moved_to_echo=64891 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 889, 0, 0, 0, 0, 4096, 4096, 3475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4096, 4090, 3707, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 3537, 4096, 0, 0, 0, 0, 9, 0, 4096, 621, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 4096, 0, 4055, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 1794, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 1006, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 38, 4096, 2136, 1906, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2761, 0, 0, 0, 4096, 0, 1960, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16650, 16366, 16854, 16349, 15045, 16405, 16385, 17018] avg=16384.0000 max=17018 max_rank=7 min=15045 min_rank=4 imbalance_ratio=1.038696 max_load_reduction=114054 max_load_reduction_pct=0.870163 moved_to_echo=64853 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 775, 0, 0, 0, 0, 4096, 4096, 3475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 3708, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2494, 4096, 0, 0, 0, 0, 0, 0, 4096, 621, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 119, 0, 0, 4096, 0, 4045, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1787, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 3145, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 22, 4096, 2137, 1908, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1658, 0, 0, 0, 4096, 0, 1959, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16537, 16397, 15791, 16451, 17201, 16397, 16393, 15905] avg=16384.0000 max=17201 max_rank=4 min=15791 min_rank=2 imbalance_ratio=1.049866 max_load_reduction=113871 max_load_reduction_pct=0.868767 moved_to_echo=65896 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 0, 0, 0, 1, 4096, 4096, 3474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 3707, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2902, 4096, 0, 0, 0, 0, 0, 0, 4096, 622, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 94, 0, 0, 4096, 0, 4055, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1794, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 3776, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2136, 1906, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1268, 0, 0, 0, 4096, 0, 1960, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15950, 16390, 16201, 16436, 17822, 16368, 16389, 15516] avg=16384.0000 max=17822 max_rank=4 min=15516 min_rank=7 imbalance_ratio=1.087769 max_load_reduction=113250 max_load_reduction_pct=0.864029 moved_to_echo=66095 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 395, 0, 0, 0, 0, 4096, 4096, 3475, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 3708, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 3127, 4096, 1, 0, 1, 0, 0, 0, 4096, 621, 388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 0, 0, 4096, 0, 4055, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1793, 0, 0, 0, 0, 0, 3989, 0, 0, 0, 2348, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2136, 1905, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2414, 0, 0, 0, 4095, 0, 1960, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16158, 16392, 16430, 16355, 16322, 16366, 16388, 16661] avg=16384.0000 max=16661 max_rank=7 min=16158 min_rank=0 imbalance_ratio=1.016907 max_load_reduction=114411 max_load_reduction_pct=0.872887 moved_to_echo=65737 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1172, 0, 0, 0, 0, 4096, 4096, 3474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 4095, 3707, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 3445, 4081, 0, 0, 0, 0, 19, 0, 4096, 622, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 4096, 0, 4055, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 1794, 0, 0, 0, 0, 0, 4057, 0, 0, 0, 2317, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 51, 4096, 2136, 1906, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1250, 0, 0, 0, 4096, 1, 1960, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16928, 16376, 16757, 16353, 16347, 16418, 16386, 15507] avg=16384.0000 max=16928 max_rank=0 min=15507 min_rank=7 imbalance_ratio=1.033203 max_load_reduction=114144 max_load_reduction_pct=0.870850 moved_to_echo=64658 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 32768, 0, 0, 128, 128, 32768, 0, 128, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 17152, 0, 128, 0, 32768, 128, 15744, 0, 0, 128, 0, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 0, 128, 0, 128, 128, 0, 0, 0, 128, 32768, 128, 128, 256, 32768, 17152, 15360, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 0, 0, 0, 0, 128, 0, 0, 128, 128, 5120, 0, 128, 128, 128, 32768, 32768, 27904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 32768, 14464, 0, 128, 0, 128, 0, 32384, 0, 128, 0, 19072, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 0, 0, 32768, 384, 0, 128, 32768, 0, 32512, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 0, 128, 24704, 32768, 128, 0, 128, 0, 128, 0, 32768, 4992, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 0, 0, 128, 128, 0, 128, 128, 0, 0, 32768, 32768, 32768, 29696, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4096, 22, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4068, 0, 4096, 4096, 4096, 4096, 0, 4013, 0, 0, 0, 0, 4096, 3957, 899, 0, 0, 0, 4096, 4096, 4096, 13, 1263, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 332, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1564, 0, 0, 9, 282, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 4096, 0, 4091, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4095, 70, 1, 1, 1, 0, 4095, 2, 4095, 1, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 1, 4095, 0, 1, 4084, 1, 4095, 4095, 4095, 4095, 0, 3846, 0, 0, 1, 0, 4095, 4001, 840, 0, 2, 0, 4095, 4095, 4094, 0, 1771, 0, 4024, 1, 0, 0, 1, 1, 0, 0, 1, 0, 4095, 0, 0, 573, 1, 1, 34, 1, 1, 0, 4095, 0, 1, 1, 4095, 1194, 3, 0, 0, 23, 0, 0, 0, 4095, 0, 1, 4095, 4095, 1, 0, 1, 0, 4090, 4091, 1, 0, 1, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 4095, 0, 4087, 1, 0, 1, 5, 0, 0, 0, 4095, 0, 0, 4095, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4096, 19, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 4094, 4096, 4096, 4096, 0, 3739, 0, 0, 0, 0, 4096, 3881, 134, 0, 0, 0, 4096, 4096, 4096, 0, 1757, 0, 3753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 219, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2866, 0, 0, 0, 80, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4026, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 74, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 15, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4093, 0, 4096, 4096, 4096, 4096, 0, 3858, 0, 0, 0, 0, 4096, 3730, 999, 0, 1, 0, 4096, 4096, 4096, 21, 1427, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 571, 0, 0, 5, 0, 0, 0, 4092, 0, 0, 0, 4096, 1705, 0, 0, 6, 121, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 2, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 3954, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 305, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 4096, 4096, 4096, 4096, 0, 3181, 0, 0, 0, 0, 4096, 4071, 724, 0, 0, 0, 4096, 4096, 4096, 3, 1384, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 37, 0, 0, 8, 0, 0, 0, 4093, 0, 0, 0, 4096, 2522, 0, 0, 4, 91, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4092, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 107, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 4076, 0, 0, 0, 0, 4096, 1878, 439, 0, 0, 0, 4096, 4096, 4096, 0, 3131, 0, 2963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2870, 0, 0, 0, 47, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 3878, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 134, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4, 0, 0, 0, 4096, 4, 0, 4096, 8, 0, 4082, 0, 4096, 4092, 4094, 4096, 0, 3686, 6, 1, 0, 0, 4096, 2524, 477, 0, 7, 0, 4095, 4096, 4096, 4, 2530, 0, 3401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 987, 0, 0, 1, 0, 0, 0, 4091, 0, 0, 0, 4096, 2292, 0, 0, 3, 471, 0, 0, 0, 4089, 0, 0, 4096, 4096, 0, 0, 0, 28, 4096, 4083, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 1, 0, 4094, 0, 0, 4096, 0, 3962, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 106, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 57, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 4074, 0, 4095, 4096, 4096, 4096, 0, 4025, 0, 0, 0, 0, 4096, 3633, 346, 0, 0, 0, 4096, 4096, 4096, 2, 726, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 668, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2386, 0, 0, 0, 446, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 4034, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 22, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 22, 0, 0, 0, 0, 2416, 1449, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4068, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4013, 0, 0, 0, 0, 0, 3957, 899, 0, 0, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2647, 4096, 13, 1263, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1564, 0, 4096, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 282, 0, 0, 0, 4095, 0, 0, 4096, 3884, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 2739, 1365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16316, 16364, 17376, 16157, 15541, 16462, 16391, 16465] avg=16384.0000 max=17376 max_rank=2 min=15541 min_rank=4 imbalance_ratio=1.060547 max_load_reduction=113696 max_load_reduction_pct=0.867432 moved_to_echo=64859 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4095, 70, 1, 1, 1, 0, 4095, 2, 4095, 1, 0, 24, 0, 0, 0, 0, 2412, 1447, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 1, 4095, 0, 1, 4084, 1, 4095, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 3846, 0, 0, 1, 0, 0, 4001, 840, 0, 2, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2648, 4094, 0, 1771, 0, 4024, 1, 0, 0, 1, 1, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 1, 1, 34, 1, 1, 0, 4095, 0, 1, 1, 4095, 1194, 3, 4095, 1356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4095, 0, 1, 4095, 3897, 1, 0, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 1, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 2739, 1361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4087, 1, 0, 1, 5, 0, 0, 0, 4095, 0, 0, 4095, 1, 1, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16356, 16380, 17202, 16636, 15451, 16203, 16384, 16460] avg=16384.0000 max=17202 max_rank=2 min=15451 min_rank=4 imbalance_ratio=1.049927 max_load_reduction=113870 max_load_reduction_pct=0.868759 moved_to_echo=64498 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 74, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 15, 0, 0, 0, 0, 2412, 1448, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4093, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3858, 0, 0, 0, 0, 0, 3730, 999, 0, 1, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2648, 4096, 21, 1427, 0, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 0, 0, 5, 0, 0, 0, 4092, 0, 0, 0, 4096, 1705, 0, 4096, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 121, 0, 0, 0, 4096, 0, 0, 4096, 3898, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 2739, 1362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3954, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16349, 16389, 17102, 16300, 15922, 16315, 16374, 16321] avg=16384.0000 max=17102 max_rank=2 min=15922 min_rank=4 imbalance_ratio=1.043823 max_load_reduction=113970 max_load_reduction_pct=0.869522 moved_to_echo=64932 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 134, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2412, 1448, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4, 0, 0, 0, 4096, 4, 0, 4096, 8, 0, 4082, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4094, 0, 0, 3686, 6, 1, 0, 0, 0, 2524, 477, 0, 7, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2648, 4096, 4, 2530, 0, 3401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 987, 0, 0, 1, 0, 0, 0, 4091, 0, 0, 0, 4096, 2292, 0, 4096, 1356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 471, 0, 0, 0, 4089, 0, 0, 4096, 3898, 0, 0, 0, 28, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 1, 0, 4094, 0, 0, 2740, 1361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3962, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16391, 16400, 15209, 16772, 16919, 16681, 16375, 16325] avg=16384.0000 max=16919 max_rank=4 min=15209 min_rank=2 imbalance_ratio=1.032654 max_load_reduction=114153 max_load_reduction_pct=0.870918 moved_to_echo=66300 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 19, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 5, 0, 0, 0, 0, 2412, 1448, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 4094, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3739, 0, 0, 0, 0, 0, 3881, 134, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2648, 4096, 0, 1757, 0, 3753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2866, 0, 4096, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 4096, 0, 0, 4096, 3898, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 4096, 0, 0, 2739, 1362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4026, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16284, 16388, 16268, 16350, 16730, 16266, 16393, 16393] avg=16384.0000 max=16730 max_rank=4 min=16266 min_rank=5 imbalance_ratio=1.021118 max_load_reduction=114342 max_load_reduction_pct=0.872360 moved_to_echo=65782 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 107, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 2412, 1448, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4076, 0, 0, 0, 0, 0, 1878, 439, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2648, 4096, 0, 3131, 0, 2963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2870, 0, 4096, 1356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 4095, 0, 0, 4096, 3898, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 2740, 1362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3878, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16368, 16391, 14907, 16934, 17603, 16232, 16391, 16246] avg=16384.0000 max=17603 max_rank=4 min=14907 min_rank=2 imbalance_ratio=1.074402 max_load_reduction=113469 max_load_reduction_pct=0.865700 moved_to_echo=66472 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 305, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 16, 0, 0, 0, 0, 2412, 1448, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3181, 0, 0, 0, 0, 0, 4071, 724, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2648, 4096, 3, 1384, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 8, 0, 0, 0, 4093, 0, 0, 0, 4096, 2522, 0, 4096, 1356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 91, 0, 0, 0, 4096, 0, 0, 4096, 3898, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 2740, 1362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16581, 16389, 16490, 16284, 16208, 16281, 16379, 16460] avg=16384.0000 max=16581 max_rank=0 min=16208 min_rank=4 imbalance_ratio=1.012024 max_load_reduction=114491 max_load_reduction_pct=0.873497 moved_to_echo=65328 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 106, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 57, 0, 4, 0, 0, 2412, 1448, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 4074, 0, 4095, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4025, 0, 0, 0, 0, 0, 3633, 346, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2648, 4096, 2, 726, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 668, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2386, 0, 4096, 1356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 446, 0, 0, 0, 4096, 0, 0, 4096, 3898, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 2740, 1362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4034, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16427, 16371, 16518, 15639, 16698, 16632, 16385, 16402] avg=16384.0000 max=16698 max_rank=4 min=15639 min_rank=3 imbalance_ratio=1.019165 max_load_reduction=114374 max_load_reduction_pct=0.872604 moved_to_echo=66117 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 0, 128, 0, 32768, 0, 128, 0, 32768, 0, 0, 22016, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 0, 32128, 128, 0, 128, 128, 0, 0, 0, 32768, 0, 0, 32768, 128, 128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 32768, 0, 0, 30464, 128, 128, 128, 0, 0, 27776, 4864, 0, 128, 0, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[21248, 32768, 128, 14080, 0, 30336, 128, 0, 0, 128, 128, 0, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 896, 128, 128, 128, 0, 32768, 128, 32768, 128, 0, 256, 0, 128, 0, 0, 19328, 11648, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 32768, 128, 128, 32768, 128, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 4480, 128, 128, 128, 128, 128, 0, 32768, 0, 128, 128, 32768, 17408, 128, 32768, 10880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 1664, 0, 0, 0, 32768, 0, 128, 32768, 31232, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3861, 0, 0, 4095, 0, 0, 0, 0, 1, 6, 0, 0, 4094, 0, 0, 0, 4089, 1, 0, 4091, 4093, 0, 4096, 4082, 4096, 3, 0, 0, 0, 1, 0, 4013, 4096, 12, 4095, 0, 0, 0, 4096, 0, 0, 3, 0, 4093, 0, 2, 0, 3759, 774, 0, 0, 4093, 99, 0, 84, 4096, 0, 4, 0, 4096, 4096, 4095, 3798, 0, 4096, 0, 3, 0, 4093, 0, 123, 0, 0, 4047, 3961, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 16, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3694, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4089, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 128, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3860, 452, 0, 0, 4095, 6, 0, 66, 4096, 0, 0, 1, 4096, 4096, 4095, 3465, 0, 4096, 0, 0, 0, 4096, 0, 627, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4050, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 4095, 4096, 0, 4096, 4088, 4096, 0, 0, 0, 0, 0, 0, 4064, 4096, 49, 4096, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3697, 1996, 0, 0, 4094, 24, 0, 44, 4096, 0, 5, 3, 4096, 4096, 4096, 2148, 0, 4096, 0, 0, 0, 4096, 0, 425, 0, 0, 4067, 3970, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 1, 1, 1, 1, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 4095, 1, 1, 1, 3669, 0, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 4095, 1, 0, 4092, 4095, 1, 4095, 4085, 4094, 0, 0, 1, 0, 0, 0, 4085, 4095, 342, 4095, 0, 1, 1, 4095, 0, 1, 0, 0, 4095, 0, 0, 0, 3791, 1920, 2, 1, 4095, 0, 0, 15, 4095, 0, 0, 0, 4095, 4095, 4095, 2168, 1, 4095, 0, 0, 1, 4095, 1, 402, 0, 1, 4095, 4096, 4095, 0, 1, 0, 0, 1, 1, 0, 0, 4, 0, 0, 4089, 4095, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4038, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 4069, 4096, 63, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3677, 925, 0, 0, 4096, 9, 0, 27, 4096, 0, 0, 0, 4096, 4096, 4096, 2707, 0, 4096, 0, 0, 0, 4096, 0, 881, 0, 0, 4096, 4048, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3964, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4094, 4096, 0, 4096, 4094, 4096, 0, 0, 0, 0, 0, 0, 4092, 4096, 85, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3865, 677, 0, 0, 4096, 7, 0, 91, 4096, 0, 0, 0, 4096, 4096, 4096, 3443, 0, 4096, 0, 0, 0, 4096, 0, 169, 0, 0, 4096, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3900, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4052, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 47, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4021, 1456, 0, 0, 4096, 24, 0, 3, 4096, 0, 0, 0, 4096, 4096, 4096, 2860, 0, 4096, 0, 0, 0, 4096, 0, 52, 0, 0, 4095, 4065, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3964, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 4094, 4096, 0, 0, 0, 0, 0, 0, 3960, 4096, 138, 4096, 0, 7, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3789, 1557, 0, 0, 4096, 5, 0, 5, 4096, 0, 0, 0, 4096, 4096, 4096, 1527, 0, 4096, 0, 0, 0, 4096, 0, 1458, 0, 0, 4085, 4049, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4093, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 60, 4094, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3861, 0, 0, 4095, 0, 0, 144, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 4094, 0, 0, 0, 4089, 1, 0, 4091, 4093, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4029, 3, 0, 0, 0, 1, 0, 4013, 0, 12, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3145, 0, 0, 3, 0, 4093, 0, 2, 0, 3759, 774, 0, 0, 4093, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 1289, 0, 4, 0, 0, 0, 4095, 3798, 0, 4096, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 4047, 3961, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 16, 0, 2807, 807, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16409, 16350, 16384, 16235, 15968, 17462, 15879, 16385] avg=16384.0000 max=17462 max_rank=5 min=15879 min_rank=6 imbalance_ratio=1.065796 max_load_reduction=113610 max_load_reduction_pct=0.866776 moved_to_echo=65694 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 1, 1, 1, 1, 4095, 0, 1, 0, 0, 0, 0, 0, 4093, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 4095, 1, 1, 1, 3669, 0, 0, 4095, 1, 1, 144, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 4095, 1, 0, 4092, 4095, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4028, 0, 0, 1, 0, 0, 0, 4085, 0, 342, 4095, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3144, 0, 1, 0, 0, 4095, 0, 0, 0, 3791, 1920, 2, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 1288, 0, 0, 0, 0, 0, 4095, 2168, 1, 4095, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 1, 4095, 4096, 4095, 0, 1, 0, 0, 1, 1, 0, 0, 4, 0, 2807, 807, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4095, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16365, 16164, 16389, 16638, 17049, 15759, 16329, 16379] avg=16384.0000 max=17049 max_rank=4 min=15759 min_rank=5 imbalance_ratio=1.040588 max_load_reduction=114023 max_load_reduction_pct=0.869926 moved_to_echo=65516 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 48, 4095, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4050, 0, 0, 4096, 0, 0, 144, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 4095, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4029, 0, 0, 0, 0, 0, 0, 4064, 0, 49, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3145, 0, 0, 0, 0, 4096, 0, 0, 0, 3697, 1996, 0, 0, 4094, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 1289, 0, 5, 3, 0, 0, 4096, 2148, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 425, 0, 0, 4067, 3970, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2807, 807, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 16540, 16387, 16327, 17052, 15777, 16193, 16384] avg=16384.0000 max=17052 max_rank=4 min=15777 min_rank=5 imbalance_ratio=1.040771 max_load_reduction=114020 max_load_reduction_pct=0.869904 moved_to_echo=65406 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 24, 4095, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4038, 0, 0, 4096, 0, 0, 151, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4027, 0, 0, 0, 0, 0, 0, 4069, 0, 63, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3134, 0, 0, 0, 0, 4096, 0, 0, 0, 3677, 925, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 1284, 0, 0, 0, 0, 0, 4096, 2707, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 881, 0, 0, 4096, 4048, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 2812, 811, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16536, 16393, 16350, 15937, 16306, 16776, 16385] avg=16384.0000 max=16776 max_rank=6 min=15937 min_rank=4 imbalance_ratio=1.023926 max_load_reduction=114296 max_load_reduction_pct=0.872009 moved_to_echo=65404 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3694, 0, 0, 4096, 0, 0, 144, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4089, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4029, 0, 0, 0, 0, 0, 0, 4095, 0, 128, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3145, 0, 0, 0, 0, 4096, 0, 0, 0, 3860, 452, 0, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 1289, 0, 0, 1, 0, 0, 4095, 3465, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 627, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2807, 807, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16365, 16184, 16386, 16444, 15654, 17108, 16546, 16385] avg=16384.0000 max=17108 max_rank=5 min=15654 min_rank=4 imbalance_ratio=1.044189 max_load_reduction=113964 max_load_reduction_pct=0.869476 moved_to_echo=65693 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 4094, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3964, 0, 0, 4096, 0, 0, 144, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4094, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4029, 0, 0, 0, 0, 0, 0, 4092, 0, 85, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3145, 0, 0, 0, 0, 4096, 0, 0, 0, 3865, 677, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 1289, 0, 0, 0, 0, 0, 4096, 3443, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 4096, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2807, 807, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16368, 16454, 16391, 16390, 15886, 17111, 16086, 16386] avg=16384.0000 max=17111 max_rank=5 min=15886 min_rank=4 imbalance_ratio=1.044373 max_load_reduction=113961 max_load_reduction_pct=0.869453 moved_to_echo=65469 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 35, 4095, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3964, 0, 0, 4096, 0, 0, 144, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4029, 0, 0, 0, 0, 0, 0, 3960, 0, 138, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3145, 0, 0, 0, 0, 4096, 0, 0, 0, 3789, 1557, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1289, 0, 0, 0, 0, 0, 4096, 1527, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1458, 0, 0, 4085, 4049, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2807, 807, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16399, 16454, 16393, 16324, 16688, 15109, 17322, 16383] avg=16384.0000 max=17322 max_rank=6 min=15109 min_rank=5 imbalance_ratio=1.057251 max_load_reduction=113750 max_load_reduction_pct=0.867844 moved_to_echo=65502 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3900, 0, 0, 4096, 0, 0, 144, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4052, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4029, 0, 0, 0, 0, 0, 0, 4096, 0, 47, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3145, 0, 0, 0, 0, 4096, 0, 0, 0, 4021, 1456, 0, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1289, 0, 0, 0, 0, 0, 4096, 2860, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 4095, 4065, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2807, 807, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16365, 16390, 16349, 16364, 16838, 16440, 15941, 16385] avg=16384.0000 max=16838 max_rank=4 min=15941 min_rank=6 imbalance_ratio=1.027710 max_load_reduction=114234 max_load_reduction_pct=0.871536 moved_to_echo=65604 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 0, 32768, 128, 128, 128, 128, 32768, 0, 128, 0, 0, 0, 0, 256, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 32768, 32768, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 128, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 32256, 128, 0, 128, 0, 128, 0, 32512, 0, 896, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[4224, 0, 128, 32768, 32384, 32768, 0, 128, 0, 128, 128, 128, 0, 0, 128, 0, 22528, 6528, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 0, 128, 32768, 0, 128, 0, 32768, 128, 0, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[384, 10368, 0, 128, 128, 0, 0, 32768, 22144, 128, 32768, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 32768, 128, 128, 32768, 128, 128, 128, 31232, 0, 0, 32768, 128, 128, 1280, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[25216, 0, 128, 128, 0, 32768, 0, 128, 0, 30464, 9856, 128, 128, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4081, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 2, 0, 4088, 4063, 0, 0, 0, 0, 4096, 0, 4094, 729, 1, 4093, 0, 1, 1, 0, 4092, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 4093, 0, 4096, 3, 0, 0, 0, 1815, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 27, 0, 417, 0, 0, 4096, 6, 4092, 0, 0, 0, 0, 3, 0, 34, 0, 0, 1, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4069, 4096, 1147, 4091, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[3999, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4039, 0, 0, 0, 0, 4096, 0, 4096, 694, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4061, 0, 4090, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 2731, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 250, 0, 539, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3980, 4096, 170, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4029, 1, 0, 0, 0, 1, 4094, 0, 1, 1, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 4095, 23, 0, 0, 0, 4095, 1, 3, 0, 0, 0, 0, 4095, 3864, 3, 0, 0, 0, 4094, 0, 4095, 997, 0, 4095, 1, 1, 0, 1, 4095, 0, 0, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 4095, 4095, 1, 1, 4095, 1, 4072, 1, 1, 0, 0, 4095, 1, 4095, 0, 1, 1, 0, 2379, 4095, 0, 4095, 4095, 1, 4095, 1, 1, 25, 1, 877, 0, 0, 4095, 0, 4095, 1, 1, 1, 2, 0, 2, 2, 0, 1, 0, 1, 4086, 0, 4095, 1, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 4094, 0, 4091, 4094, 119, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3982, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3549, 0, 0, 0, 0, 4096, 0, 4096, 563, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4072, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 3233, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 99, 0, 269, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4086, 4096, 626, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4076, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3550, 0, 0, 0, 0, 4096, 0, 4096, 525, 0, 4095, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 2205, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 12, 0, 1458, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4068, 4096, 490, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[3984, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3999, 0, 0, 0, 0, 4096, 0, 4096, 669, 0, 4096, 0, 2, 0, 0, 4079, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 3063, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 31, 0, 513, 0, 0, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3996, 4096, 124, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4078, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4058, 0, 0, 0, 0, 4096, 0, 4096, 274, 0, 4093, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 2651, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 70, 0, 553, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4088, 4096, 610, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4049, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3565, 0, 0, 0, 0, 4096, 0, 4096, 740, 0, 4095, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4088, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 2715, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 125, 0, 1023, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4076, 4096, 76, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4081, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 2586, 786, 387, 376, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4093, 0, 3, 0, 0, 2, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4063, 0, 0, 0, 0, 0, 0, 3707, 729, 1, 4093, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 4093, 0, 1510, 3, 0, 0, 0, 1815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3310, 0, 4096, 0, 0, 0, 0, 0, 27, 0, 417, 0, 0, 4096, 6, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 34, 0, 0, 1, 0, 4096, 0, 4095, 0, 0, 4096, 4076, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4069, 3717, 1147, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16432, 16389, 16683, 16387, 15607, 16044, 16410, 17120] avg=16384.0000 max=17120 max_rank=7 min=15607 min_rank=4 imbalance_ratio=1.044922 max_load_reduction=113952 max_load_reduction_pct=0.869385 moved_to_echo=65181 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4029, 1, 0, 0, 0, 1, 4094, 0, 1, 1, 0, 1, 1, 0, 4095, 0, 2585, 786, 387, 375, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 23, 0, 0, 0, 4095, 1, 3, 0, 0, 0, 4095, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3864, 3, 0, 0, 0, 0, 0, 3708, 997, 0, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 4095, 4095, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 4072, 1, 1, 0, 0, 4095, 1, 1510, 0, 1, 1, 0, 2379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3309, 0, 4095, 0, 1, 0, 1, 1, 25, 1, 877, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 0, 2, 2, 0, 1, 0, 1, 4086, 0, 4095, 1, 0, 4095, 4075, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 4091, 3716, 119, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16376, 16399, 16764, 16388, 16158, 16500, 16371, 16116] avg=16384.0000 max=16764 max_rank=2 min=16116 min_rank=7 imbalance_ratio=1.023193 max_load_reduction=114308 max_load_reduction_pct=0.872101 moved_to_echo=65145 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3999, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2586, 786, 387, 376, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4039, 0, 0, 0, 0, 0, 0, 3709, 694, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 4090, 0, 0, 0, 0, 4096, 0, 1510, 2, 0, 0, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3310, 0, 4096, 0, 0, 0, 0, 0, 250, 0, 539, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4076, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3980, 3717, 170, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16346, 16391, 16634, 16387, 16490, 16387, 16378, 16059] avg=16384.0000 max=16634 max_rank=2 min=16059 min_rank=7 imbalance_ratio=1.015259 max_load_reduction=114438 max_load_reduction_pct=0.873093 moved_to_echo=65314 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3984, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2586, 786, 387, 376, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3999, 0, 0, 0, 0, 0, 0, 3709, 669, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 4096, 0, 1510, 0, 0, 0, 0, 3063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3310, 0, 4096, 0, 0, 0, 0, 0, 31, 0, 513, 0, 0, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4076, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3996, 3717, 124, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16331, 16376, 16571, 16370, 16858, 16144, 16394, 16028] avg=16384.0000 max=16858 max_rank=4 min=16028 min_rank=7 imbalance_ratio=1.028931 max_load_reduction=114214 max_load_reduction_pct=0.871384 moved_to_echo=65424 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3982, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2589, 790, 390, 378, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3549, 0, 0, 0, 0, 0, 0, 3706, 563, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 4096, 0, 0, 0, 0, 4096, 0, 1507, 0, 0, 0, 0, 3233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3306, 0, 4096, 0, 0, 0, 0, 0, 99, 0, 269, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4075, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 3710, 626, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16342, 16376, 16010, 16386, 17004, 15962, 16378, 16614] avg=16384.0000 max=17004 max_rank=4 min=15962 min_rank=5 imbalance_ratio=1.037842 max_load_reduction=114068 max_load_reduction_pct=0.870270 moved_to_echo=65958 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4076, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2586, 786, 387, 376, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3550, 0, 0, 0, 0, 0, 0, 3709, 525, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 1510, 0, 0, 0, 0, 2205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3310, 0, 4096, 0, 0, 0, 0, 0, 12, 0, 1458, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4076, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4068, 3717, 490, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16423, 16376, 15975, 16385, 16002, 17068, 16376, 16467] avg=16384.0000 max=17068 max_rank=5 min=15975 min_rank=2 imbalance_ratio=1.041748 max_load_reduction=114004 max_load_reduction_pct=0.869781 moved_to_echo=65913 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4049, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2586, 786, 387, 376, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3565, 0, 0, 0, 0, 0, 0, 3709, 740, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4095, 0, 0, 0, 0, 4096, 0, 1510, 0, 0, 0, 0, 2715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3310, 0, 4096, 0, 0, 0, 0, 0, 125, 0, 1023, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4076, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 3717, 76, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16396, 16386, 16205, 16383, 16504, 16746, 16391, 16061] avg=16384.0000 max=16746 max_rank=5 min=16061 min_rank=7 imbalance_ratio=1.022095 max_load_reduction=114326 max_load_reduction_pct=0.872238 moved_to_echo=65702 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4078, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 2586, 786, 387, 376, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4058, 0, 0, 0, 0, 0, 0, 3709, 274, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 4096, 0, 1510, 1, 0, 0, 0, 2651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3310, 0, 4096, 0, 0, 0, 0, 0, 70, 0, 553, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4076, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 3717, 610, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16426, 16379, 16230, 16386, 16449, 16221, 16374, 16607] avg=16384.0000 max=16607 max_rank=7 min=16221 min_rank=5 imbalance_ratio=1.013611 max_load_reduction=114465 max_load_reduction_pct=0.873299 moved_to_echo=65651 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[26496, 0, 32768, 0, 128, 0, 128, 128, 640, 128, 5760, 0, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 30720, 128, 0, 0, 0, 0, 0, 29696, 5248, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 32768, 0, 32768, 128, 0, 32768, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32768, 0, 0, 128, 128, 32768, 0, 128, 0, 0, 0, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 32768, 128, 0, 0, 0, 32768, 128, 128, 0, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 32768, 128, 128, 0, 0, 32768, 128, 12160, 128, 128, 128, 0, 20864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 32768, 0, 32512, 29824, 3456, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32384, 128, 0, 0, 0, 128, 32768, 128, 128, 128, 128, 128, 128, 0, 32768, 0, 20736, 6400, 3200, 3072, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1097, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 422, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 3899, 0, 4096, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4091, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 2791, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1603, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 893, 0, 4076, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2676, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 3045, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3, 21, 0, 2388, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 4096, 0, 0, 110, 0, 3689, 0, 0, 2, 0, 0, 4026, 0, 0, 4093, 0, 1, 5, 0, 0, 0, 0, 4073, 0, 0, 3633, 0, 4096, 4092, 0, 0, 4085, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 4096, 3, 4094, 0, 6, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 4096, 0, 0, 0, 4096, 0, 4090, 4096, 8, 4, 4065, 0, 0, 4096, 0, 5, 4096, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 4096, 4092, 4096, 6, 4096, 2524, 0, 4087, 0, 13] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 25, 0, 1503, 2, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 250, 0, 4060, 0, 0, 0, 0, 0, 4082, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4069, 0, 0, 3722, 0, 4096, 4096, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4060, 0, 0, 4096, 0, 11, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 1, 4096, 2792, 0, 4093, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 8, 0, 1809, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 739, 0, 4081, 0, 0, 0, 0, 0, 4011, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 3543, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4044, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 2239, 0, 4094, 0, 15] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 1727, 5, 1, 4095, 4094, 1, 0, 1, 0, 0, 4095, 1, 1, 0, 1, 4095, 0, 0, 490, 1, 4050, 1, 0, 0, 1, 1, 4095, 1, 0, 4095, 0, 0, 0, 0, 1, 0, 1, 4096, 0, 0, 3154, 0, 4095, 4095, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 1, 0, 0, 1, 1, 4095, 4094, 0, 4095, 0, 0, 1, 0, 0, 0, 1, 1, 0, 6, 0, 0, 4090, 1, 1, 1, 4095, 0, 4095, 3919, 0, 0, 4096, 0, 1, 4095, 0, 0, 4093, 1, 4095, 0, 0, 0, 4095, 0, 0, 1, 4095, 4095, 4095, 0, 4095, 3037, 0, 4095, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1004, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 158, 0, 3795, 0, 0, 0, 0, 0, 4040, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4011, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 3372, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 4, 0, 2558, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 56, 0, 3960, 0, 0, 0, 0, 0, 4005, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 3246, 0, 4096, 4096, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4090, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 2548, 0, 4095, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1097, 0, 0, 4096, 4062, 2409, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3704, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 2800, 1075, 392, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 3899, 0, 612, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1296, 2791, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15769, 16414, 16557, 16783, 16378, 16383, 16413, 16375] avg=16384.0000 max=16783 max_rank=3 min=15769 min_rank=0 imbalance_ratio=1.024353 max_load_reduction=114289 max_load_reduction_pct=0.871956 moved_to_echo=65549 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1603, 0, 0, 4096, 4060, 2403, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3707, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 2795, 1069, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2676, 0, 624, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1301, 3045, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16266, 16888, 16516, 15588, 16384, 16376, 16420, 16634] avg=16384.0000 max=16888 max_rank=1 min=15588 min_rank=3 imbalance_ratio=1.030762 max_load_reduction=114184 max_load_reduction_pct=0.871155 moved_to_echo=65814 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 3, 21, 0, 2388, 0, 0, 4096, 4060, 2403, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3707, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 1, 0, 4096, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3689, 0, 0, 2, 0, 0, 4026, 0, 0, 4093, 0, 1, 5, 0, 0, 2795, 1069, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 3633, 0, 624, 4092, 0, 0, 4085, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 4096, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 8, 4, 4065, 0, 0, 0, 0, 5, 4096, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4092, 4096, 6, 1301, 2524, 0, 4087, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17072, 16102, 16069, 16508, 16390, 16410, 16400, 16121] avg=16384.0000 max=17072 max_rank=0 min=16069 min_rank=2 imbalance_ratio=1.041992 max_load_reduction=114000 max_load_reduction_pct=0.869751 moved_to_echo=65321 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 25, 0, 1503, 2, 0, 4096, 4060, 2403, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3707, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 4082, 0, 0, 4096, 0, 2, 0, 0, 0, 2795, 1069, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 3722, 0, 624, 4096, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4060, 0, 0, 0, 0, 11, 4096, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 1301, 2792, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16194, 16245, 16493, 16608, 16383, 16375, 16395, 16379] avg=16384.0000 max=16608 max_rank=3 min=16194 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=114464 max_load_reduction_pct=0.873291 moved_to_echo=65532 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 8, 0, 1809, 0, 0, 4096, 4060, 2403, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3707, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 4011, 0, 0, 4096, 0, 0, 0, 0, 0, 2795, 1069, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 3543, 0, 624, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4044, 0, 0, 0, 0, 0, 4096, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1301, 2239, 0, 4094, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16480, 16734, 16441, 16439, 16384, 16385, 16368, 15841] avg=16384.0000 max=16734 max_rank=1 min=15841 min_rank=7 imbalance_ratio=1.021362 max_load_reduction=114338 max_load_reduction_pct=0.872330 moved_to_echo=64978 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 1727, 5, 1, 4095, 4059, 2403, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3706, 4094, 1, 0, 1, 0, 0, 4095, 1, 1, 0, 1, 4095, 0, 0, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4050, 1, 0, 0, 1, 1, 4095, 1, 0, 4095, 0, 0, 0, 0, 1, 2794, 1069, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 3154, 0, 623, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 1, 0, 0, 1, 1, 4095, 4094, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 6, 0, 0, 4090, 1, 1, 1, 4095, 4095, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3919, 0, 0, 4096, 0, 1, 0, 0, 0, 4093, 1, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 4095, 0, 1301, 3037, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16395, 16485, 16498, 16064, 16385, 16379, 16241, 16625] avg=16384.0000 max=16625 max_rank=7 min=16064 min_rank=3 imbalance_ratio=1.014709 max_load_reduction=114447 max_load_reduction_pct=0.873161 moved_to_echo=65630 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1004, 0, 0, 4096, 4060, 2403, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3707, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3795, 0, 0, 0, 0, 0, 4040, 0, 0, 4096, 0, 0, 0, 0, 0, 2795, 1069, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4011, 0, 624, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1301, 3372, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15668, 16153, 16184, 16923, 16384, 16378, 16421, 16961] avg=16384.0000 max=16961 max_rank=7 min=15668 min_rank=0 imbalance_ratio=1.035217 max_load_reduction=114111 max_load_reduction_pct=0.870598 moved_to_echo=66144 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 4, 0, 2558, 0, 0, 4096, 4060, 2403, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3707, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 0, 0, 0, 0, 0, 4005, 0, 0, 4096, 0, 0, 0, 0, 0, 2795, 1069, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 3246, 0, 624, 4096, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1301, 2548, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17228, 16051, 16314, 16159, 16384, 16386, 16414, 16136] avg=16384.0000 max=17228 max_rank=0 min=16051 min_rank=1 imbalance_ratio=1.051514 max_load_reduction=113844 max_load_reduction_pct=0.868561 moved_to_echo=65320 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[29696, 32768, 128, 0, 128, 0, 0, 32768, 128, 128, 128, 128, 32768, 0, 0, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 0, 128, 128, 0, 128, 128, 32768, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 32768, 32640, 128, 128, 32640, 128, 128, 0, 0, 128, 32768, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 128, 32768, 0, 0, 27904, 0, 4992, 32768, 0, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 0, 128, 32768, 128, 0, 0, 0, 128, 128, 0, 13696, 128, 128, 32768, 32512, 19328, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 31872, 128, 0, 128, 128, 128, 32512, 128, 0, 32768, 0, 128, 128, 0, 128, 22400, 8576, 3200, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 0, 0, 0, 128, 128, 0, 128, 0, 0, 32768, 128, 128, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 0, 0, 0, 128, 0, 32768, 32768, 128, 10496, 22400, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[3636, 0, 0, 4096, 0, 0, 0, 4096, 2984, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 862, 4096, 0, 0, 0, 0, 0, 4093, 4096, 2, 4, 4096, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4084, 4096, 0, 6, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 4096, 4095, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 719, 0, 4093, 0, 0, 0, 5, 4096, 0, 10, 4096, 0, 0, 0, 0, 4096, 4, 0, 0, 4096, 0, 0, 0, 4096, 0, 4083, 4096, 0, 0, 4096, 0, 4093, 0, 0, 4093, 1, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[2303, 0, 0, 4096, 0, 0, 0, 4096, 3178, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1588, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1128, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4092, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[3079, 0, 7, 4095, 1, 0, 1, 4055, 3134, 0, 1, 4093, 1, 1, 1, 0, 0, 0, 3, 16, 1, 326, 4095, 2, 0, 1, 0, 0, 4095, 4089, 0, 0, 4094, 0, 0, 0, 4095, 4095, 0, 1, 1, 1, 0, 1, 6, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 4095, 4095, 1, 0, 4085, 0, 1, 1, 4095, 0, 0, 0, 0, 0, 4094, 4095, 0, 1, 0, 0, 1, 4093, 0, 0, 1, 1, 0, 1, 4095, 4095, 1, 0, 1968, 0, 3785, 0, 0, 1, 0, 4094, 0, 0, 4095, 1, 1, 0, 0, 4092, 0, 1, 0, 4095, 1, 0, 4, 4094, 0, 4095, 4095, 4, 24, 4095, 0, 4095, 0, 1, 4095, 0, 4085, 1, 1, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[3739, 0, 0, 4096, 0, 0, 0, 4096, 982, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1783, 4096, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1693, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[2924, 0, 0, 4096, 0, 0, 0, 4096, 2433, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1446, 0, 4096, 0, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4083, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[2583, 0, 0, 4096, 0, 0, 0, 4093, 1771, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1801, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2040, 0, 4091, 3, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[3996, 0, 0, 4096, 0, 0, 0, 4095, 3021, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 835, 0, 4044, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[2532, 0, 0, 4096, 0, 0, 0, 4096, 1858, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1137, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2684, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4075, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3636, 0, 0, 2678, 0, 0, 0, 4096, 2984, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 862, 4096, 0, 0, 0, 0, 0, 4093, 4096, 2, 4, 1511, 1418, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4084, 4096, 0, 6, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4095, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2579, 4091, 0, 0, 719, 0, 4093, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4094, 0, 0, 0, 0, 4096, 4, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 6, 0, 0, 0, 0, 4093, 0, 0, 4093, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17490, 16083, 16377, 16382, 16389, 15583, 16396, 16372] avg=16384.0000 max=17490 max_rank=0 min=15583 min_rank=5 imbalance_ratio=1.067505 max_load_reduction=113582 max_load_reduction_pct=0.866562 moved_to_echo=64740 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2303, 0, 0, 2671, 0, 0, 0, 4096, 3178, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1588, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1516, 1425, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2571, 4096, 0, 0, 1128, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16344, 16819, 16389, 16390, 16386, 15986, 16376, 16382] avg=16384.0000 max=16819 max_rank=1 min=15986 min_rank=5 imbalance_ratio=1.026550 max_load_reduction=114253 max_load_reduction_pct=0.871681 moved_to_echo=65130 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3079, 0, 7, 2677, 1, 0, 1, 4055, 3134, 0, 1, 4093, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 16, 1, 326, 4095, 2, 0, 1, 0, 0, 4095, 4089, 0, 0, 1511, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4095, 4095, 0, 1, 1, 1, 0, 1, 6, 0, 1, 1, 4088, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 4095, 4095, 1, 0, 4085, 0, 1, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 0, 1, 0, 0, 1, 4093, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 2578, 4095, 1, 0, 1968, 0, 3785, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 0, 4092, 0, 1, 0, 4095, 1, 0, 4, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 4, 24, 0, 0, 4095, 0, 1, 4095, 0, 4085, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17051, 15557, 16390, 16377, 16381, 16524, 16383, 16409] avg=16384.0000 max=17051 max_rank=0 min=15557 min_rank=1 imbalance_ratio=1.040710 max_load_reduction=114021 max_load_reduction_pct=0.869911 moved_to_echo=65697 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2924, 0, 0, 2678, 0, 0, 0, 4096, 2433, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1511, 1418, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2579, 4096, 0, 0, 1446, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 6, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16227, 16620, 16383, 16382, 16384, 16313, 16386, 16377] avg=16384.0000 max=16620 max_rank=1 min=16227 min_rank=0 imbalance_ratio=1.014404 max_load_reduction=114452 max_load_reduction_pct=0.873199 moved_to_echo=65460 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3996, 0, 0, 2678, 0, 0, 0, 4095, 3021, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 1511, 1418, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4089, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 1, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2579, 4096, 0, 0, 835, 0, 4044, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17886, 15617, 16384, 16386, 16377, 15650, 16382, 16390] avg=16384.0000 max=17886 max_rank=0 min=15617 min_rank=1 imbalance_ratio=1.091675 max_load_reduction=113186 max_load_reduction_pct=0.863541 moved_to_echo=64799 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2532, 0, 0, 2678, 0, 0, 0, 4096, 1858, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1137, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1511, 1418, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2579, 4096, 0, 0, 2684, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 6, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15262, 16355, 16383, 16385, 16385, 17551, 16382, 16369] avg=16384.0000 max=17551 max_rank=5 min=15262 min_rank=0 imbalance_ratio=1.071228 max_load_reduction=113521 max_load_reduction_pct=0.866096 moved_to_echo=66687 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3739, 0, 0, 2678, 0, 0, 0, 4096, 982, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1783, 4096, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 1511, 1418, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2579, 4096, 0, 0, 1693, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 6, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15591, 17002, 16383, 16385, 16385, 16560, 16383, 16383] avg=16384.0000 max=17002 max_rank=1 min=15591 min_rank=0 imbalance_ratio=1.037720 max_load_reduction=114070 max_load_reduction_pct=0.870285 moved_to_echo=65711 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2583, 0, 0, 2678, 0, 0, 0, 4093, 1771, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1801, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1511, 1418, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2579, 4096, 0, 0, 2040, 0, 4091, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15221, 17019, 16383, 16385, 16385, 16905, 16384, 16390] avg=16384.0000 max=17019 max_rank=1 min=15221 min_rank=0 imbalance_ratio=1.038757 max_load_reduction=114053 max_load_reduction_pct=0.870155 moved_to_echo=66064 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 0, 0, 32768, 32768, 0, 128, 128, 128, 0, 128, 128, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 128, 32768, 32768, 0, 128, 0, 0, 128, 32768, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 32768, 128, 128, 0, 32768, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[24832, 0, 128, 21504, 128, 0, 128, 32768, 19456, 0, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 0, 128, 128, 0, 32768, 32768, 128, 128, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 20736, 32768, 128, 0, 12544, 0, 32512, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 32768, 0, 128, 32768, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 128, 9344, 32768, 128, 0, 128, 0, 0, 32768, 32768, 128, 128, 12160, 11392, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 3, 4094, 4096, 0, 4096, 5, 0, 0, 0, 1502, 4096, 0, 0, 4094, 0, 0, 0, 0, 2352, 0, 3, 0, 4091, 0, 0, 0, 0, 1, 11, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 4096, 4020, 0, 2, 0, 0, 0, 0, 0, 3, 0, 4095, 4096, 0, 4095, 28, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 4095, 0, 0, 0, 2085, 4096, 2327, 4087, 0, 0, 0, 3, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4068, 0, 4096, 0, 2, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1, 4091, 0, 0, 6, 0, 0, 1, 4095, 2, 1, 5, 4083, 1, 1, 0, 4093, 0, 0, 4095, 4095, 1, 4091, 0, 1, 0, 1, 313, 4094, 0, 0, 4095, 1, 8, 0, 1, 2678, 0, 0, 0, 4095, 0, 1, 1, 1, 0, 0, 22, 4095, 1, 4064, 1, 4095, 0, 1, 0, 0, 4095, 1, 1, 4093, 2500, 0, 0, 0, 2, 0, 1, 1, 0, 1, 4095, 4094, 0, 4094, 0, 0, 0, 0, 1, 1, 0, 4095, 1, 4095, 0, 0, 1, 4095, 1, 0, 205, 3636, 4095, 2962, 4095, 0, 0, 1, 0, 4095, 3, 0, 1, 0, 0, 4095, 1, 0, 0, 1, 1, 4092, 1, 4093, 3, 0, 0, 0, 0, 1, 4095, 2, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 1, 0, 0, 0, 568, 4096, 0, 0, 4096, 0, 0, 0, 0, 2552, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3670, 4096, 2056, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4081, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 1190, 4096, 0, 0, 4096, 0, 0, 0, 0, 4046, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 2649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3017, 4096, 1388, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4086, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 2, 0, 0, 0, 969, 4096, 0, 0, 4096, 0, 0, 0, 0, 3592, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 2516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 26, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2242, 4096, 2952, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 1108, 4096, 0, 0, 4096, 0, 0, 0, 0, 4042, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 2962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 10, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3148, 4096, 1023, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 1, 0, 0, 0, 1332, 4096, 0, 0, 4096, 0, 0, 0, 0, 3085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 12, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 3563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3033, 4096, 1275, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 1, 0, 0, 0, 1546, 4096, 0, 0, 4096, 0, 0, 0, 0, 3439, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 2465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3130, 4096, 1707, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3231, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 3, 4094, 0, 0, 4096, 5, 0, 0, 0, 1502, 0, 0, 0, 3029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2352, 0, 3, 0, 4091, 0, 0, 0, 0, 1, 11, 0, 4096, 4096, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 4020, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 4096, 0, 4095, 28, 0, 0, 0, 0, 2144, 1065, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 3214, 0, 0, 0, 2085, 0, 2327, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4091, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3231, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 4096, 0, 2, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 5, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16826, 15515, 17392, 16402, 15805, 16382, 16367] avg=16384.0000 max=17392 max_rank=3 min=15515 min_rank=2 imbalance_ratio=1.061523 max_load_reduction=113680 max_load_reduction_pct=0.867310 moved_to_echo=64956 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3231, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 568, 0, 0, 0, 3030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2552, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 1, 0, 0, 0, 0, 2144, 1066, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3215, 0, 0, 0, 3670, 0, 2056, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3231, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 15887, 15705, 16828, 16375, 17133, 16384, 16374] avg=16384.0000 max=17133 max_rank=5 min=15705 min_rank=2 imbalance_ratio=1.045715 max_load_reduction=113939 max_load_reduction_pct=0.869286 moved_to_echo=66266 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3232, 871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1190, 0, 0, 0, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 2148, 1070, 878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3210, 0, 0, 0, 3017, 0, 1388, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3230, 864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16391, 16504, 17200, 16014, 16384, 15807, 16382, 16390] avg=16384.0000 max=17200 max_rank=2 min=15807 min_rank=5 imbalance_ratio=1.049805 max_load_reduction=113872 max_load_reduction_pct=0.868774 moved_to_echo=64963 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 4091, 0, 0, 6, 0, 0, 1, 4095, 2, 1, 5, 4083, 1, 1, 0, 3229, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4095, 0, 1, 4091, 0, 1, 0, 1, 313, 0, 0, 0, 3029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 1, 2678, 0, 0, 0, 4095, 0, 1, 1, 1, 0, 0, 22, 4095, 4095, 864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 1, 4064, 1, 0, 0, 1, 0, 0, 4095, 1, 1, 4093, 2500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 0, 1, 4095, 4094, 0, 4094, 0, 0, 0, 0, 1, 2144, 1066, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 1, 3214, 1, 0, 205, 3636, 0, 2962, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 3, 0, 1, 0, 0, 4095, 1, 0, 0, 1, 1, 4095, 3231, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 4093, 3, 0, 0, 0, 0, 1, 4095, 2, 4095, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 15624, 15862, 15841, 16375, 18211, 16389, 16387] avg=16384.0000 max=18211 max_rank=5 min=15624 min_rank=1 imbalance_ratio=1.111511 max_load_reduction=112861 max_load_reduction_pct=0.861061 moved_to_echo=67362 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3231, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 1332, 0, 0, 0, 3030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 12, 0, 4096, 4096, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4, 0, 0, 0, 0, 2144, 1066, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3215, 0, 0, 0, 3033, 0, 1275, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3231, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16651, 16250, 16936, 16378, 15703, 16384, 16389] avg=16384.0000 max=16936 max_rank=3 min=15703 min_rank=5 imbalance_ratio=1.033691 max_load_reduction=114136 max_load_reduction_pct=0.870789 moved_to_echo=64854 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3231, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 1546, 0, 0, 0, 3030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 4096, 4096, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 2144, 1066, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3215, 0, 0, 0, 3130, 0, 1707, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3231, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16865, 16596, 15838, 16374, 16240, 16384, 16389] avg=16384.0000 max=16865 max_rank=1 min=15838 min_rank=3 imbalance_ratio=1.029358 max_load_reduction=114207 max_load_reduction_pct=0.871330 moved_to_echo=65387 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3231, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1108, 0, 0, 0, 3030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 10, 0, 0, 0, 0, 2144, 1066, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3215, 0, 0, 0, 3148, 0, 1023, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3230, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16426, 17196, 16334, 16384, 15576, 16383, 16387] avg=16384.0000 max=17196 max_rank=2 min=15576 min_rank=5 imbalance_ratio=1.049561 max_load_reduction=113876 max_load_reduction_pct=0.868805 moved_to_echo=64730 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4086, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3231, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 0, 969, 0, 0, 0, 3030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3592, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 4096, 4096, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 26, 0, 0, 0, 0, 2144, 1066, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3215, 0, 0, 0, 2242, 0, 2952, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3231, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16376, 16289, 16748, 15889, 16400, 16597, 16384, 16389] avg=16384.0000 max=16748 max_rank=2 min=15889 min_rank=3 imbalance_ratio=1.022217 max_load_reduction=114324 max_load_reduction_pct=0.872223 moved_to_echo=65770 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 32768, 0, 128, 32768, 128, 128, 0, 128, 8576, 0, 0, 0, 24320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 32768, 32768, 0, 32768, 128, 0, 0, 0, 128, 17280, 8576, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 0, 0, 128, 0, 0, 128, 32768, 128, 128, 128, 32768, 128, 128, 0, 25856, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 25856, 0, 128, 0, 32768, 0, 128, 128, 128, 128, 128, 128, 32768, 32768, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 128, 25728, 128, 0, 256, 24064, 0, 15744, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[8704, 128, 32768, 128, 0, 0, 128, 0, 0, 32768, 128, 128, 32768, 24192, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 128, 32768, 128, 0, 128, 0, 0, 32768, 128, 0, 0, 128, 128, 32768, 25856, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 32768, 128, 128, 0, 0, 0, 128, 32768, 128, 32768, 0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4088, 0, 0, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 72, 4096, 0, 1, 0, 0, 4096, 4095, 4078, 0, 0, 0, 0, 4096, 0, 0, 1956, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1901, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 4057, 0, 0, 0, 4096, 0, 4096, 231, 0, 0, 0, 0, 4096, 4096, 4094, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 52, 0, 0, 0, 4095, 0, 1, 0, 4095, 4049, 1, 492, 0, 1, 0, 1, 0, 1, 0, 158, 4086, 0, 176, 0, 0, 4094, 4095, 4095, 0, 1, 1, 0, 4095, 1, 1, 670, 0, 4, 1, 0, 1, 0, 21, 0, 4091, 4075, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 1, 1, 1, 0, 0, 1, 4095, 0, 1, 4088, 0, 1, 0, 4091, 2498, 4092, 5, 1, 0, 0, 6, 4095, 4095, 0, 4095, 0, 4081, 1, 4089, 0, 3954, 1, 0, 1, 4095, 12, 4089, 251, 0, 0, 23, 0, 4095, 4090, 4095, 6, 4095, 0, 1, 3, 0, 4086, 0, 0, 0, 0, 1, 0, 4079, 4095, 1, 1, 1, 1, 3, 4, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 4096, 0, 1, 0, 0, 4096, 4096, 4059, 0, 0, 0, 0, 4096, 0, 0, 805, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3158, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 3990, 0, 0, 0, 4096, 0, 4096, 216, 0, 0, 1, 0, 4096, 4096, 4094, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3, 4064, 0, 0, 0, 4096, 4096, 0, 16, 0, 0, 0, 0, 0, 0, 0, 78, 4096, 0, 83, 0, 0, 4096, 4096, 4062, 0, 0, 0, 0, 4096, 0, 0, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2160, 4096, 1, 0, 0, 0, 0, 4096, 4093, 0, 4096, 0, 4096, 0, 4096, 0, 4039, 0, 0, 0, 4096, 0, 4096, 807, 0, 0, 3, 0, 4090, 4096, 4080, 0, 4096, 0, 0, 0, 0, 4093, 0, 1, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 0, 55, 4096, 0, 38, 0, 0, 4096, 4096, 4063, 0, 0, 0, 0, 4096, 0, 0, 3063, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 584, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 4085, 0, 0, 0, 4096, 0, 4096, 392, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 0, 4094, 0, 0, 0, 4096, 4096, 0, 15, 0, 0, 0, 0, 0, 0, 0, 275, 4096, 0, 52, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 2720, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 959, 4096, 1, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 4053, 0, 0, 0, 4096, 0, 4096, 116, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 44, 0, 0, 0, 0, 0, 0, 0, 180, 4096, 0, 4, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 4096, 0, 0, 613, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3390, 4096, 1, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 3854, 0, 0, 0, 4096, 0, 4096, 112, 0, 0, 5, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4096, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 379, 4096, 0, 0, 0, 0, 4096, 4096, 4088, 0, 0, 0, 0, 4096, 0, 0, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2806, 4096, 3, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 3941, 0, 0, 0, 4096, 0, 4096, 61, 0, 0, 0, 0, 4096, 4096, 4080, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4088, 0, 0, 0, 4096, 4096, 0, 2, 2185, 1396, 276, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 4096, 0, 1, 0, 0, 4096, 3902, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1956, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1911, 0, 0, 4096, 0, 0, 0, 4096, 1901, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 4096, 0, 4096, 0, 4057, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 0, 4096, 3820, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2605, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16328, 16245, 16842, 16389, 16100, 16440, 16337, 16391] avg=16384.0000 max=16842 max_rank=2 min=16100 min_rank=4 imbalance_ratio=1.027954 max_load_reduction=114230 max_load_reduction_pct=0.871506 moved_to_echo=65268 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 3, 4064, 0, 0, 0, 4096, 4096, 0, 16, 2182, 1393, 269, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 4096, 0, 83, 0, 0, 4096, 3910, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1914, 0, 0, 4096, 0, 0, 0, 4096, 2160, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 4096, 0, 4096, 0, 4039, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 807, 0, 0, 3, 0, 4090, 3827, 4080, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 2598, 1499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16303, 16325, 15984, 16386, 16363, 16429, 16900, 16382] avg=16384.0000 max=16900 max_rank=6 min=15984 min_rank=2 imbalance_ratio=1.031494 max_load_reduction=114172 max_load_reduction_pct=0.871063 moved_to_echo=66074 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 52, 0, 0, 0, 4095, 0, 1, 0, 4095, 4049, 1, 492, 2181, 1393, 269, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 158, 4086, 0, 176, 0, 0, 4094, 3909, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 1, 1, 670, 0, 4, 1, 0, 1, 0, 21, 0, 4095, 4095, 2596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4075, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 1, 1, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1914, 0, 1, 4088, 0, 1, 0, 4091, 2498, 4092, 5, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 4081, 1, 4089, 0, 3954, 1, 0, 1, 0, 12, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 23, 0, 4095, 3821, 4095, 6, 0, 0, 1, 3, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4079, 4095, 1, 1, 1, 1, 3, 4, 0, 1, 0, 4095, 2600, 1499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16814, 16521, 15582, 16365, 16698, 16330, 16381, 16381] avg=16384.0000 max=16814 max_rank=0 min=15582 min_rank=2 imbalance_ratio=1.026245 max_load_reduction=114258 max_load_reduction_pct=0.871719 moved_to_echo=65790 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 4096, 4096, 0, 0, 2182, 1394, 269, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 4096, 0, 1, 0, 0, 4096, 3910, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 805, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1914, 0, 0, 4096, 0, 0, 0, 4096, 3158, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 4096, 0, 4096, 0, 3990, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 1, 0, 4096, 3827, 4094, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2600, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16299, 16250, 15691, 16386, 17360, 16380, 16328, 16378] avg=16384.0000 max=17360 max_rank=4 min=15691 min_rank=2 imbalance_ratio=1.059570 max_load_reduction=113712 max_load_reduction_pct=0.867554 moved_to_echo=66446 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 7, 2182, 1394, 269, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 4096, 0, 38, 0, 0, 4096, 3910, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3063, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1914, 0, 0, 4096, 0, 0, 0, 4096, 584, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 4096, 0, 4096, 0, 4085, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 392, 0, 0, 0, 0, 4096, 3827, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2600, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16325, 16258, 17949, 16387, 14785, 16475, 16507, 16386] avg=16384.0000 max=17949 max_rank=2 min=14785 min_rank=4 imbalance_ratio=1.095520 max_load_reduction=113123 max_load_reduction_pct=0.863060 moved_to_echo=64153 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4096, 4096, 0, 4, 2182, 1394, 269, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 4096, 0, 0, 0, 0, 4096, 3910, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1035, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1914, 0, 0, 4096, 0, 0, 0, 4096, 2806, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 4096, 0, 4096, 0, 3941, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 4096, 3827, 4080, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2600, 1499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16309, 16569, 15920, 16386, 17011, 16331, 16160, 16386] avg=16384.0000 max=17011 max_rank=4 min=15920 min_rank=2 imbalance_ratio=1.038269 max_load_reduction=114061 max_load_reduction_pct=0.870216 moved_to_echo=65888 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 44, 2182, 1394, 269, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 4096, 0, 4, 0, 0, 4096, 3910, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 613, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1914, 0, 0, 4096, 0, 0, 0, 4096, 3390, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 4096, 0, 4096, 0, 3854, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 5, 0, 4096, 3827, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2600, 1499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16361, 16380, 15498, 16386, 17593, 16244, 16228, 16382] avg=16384.0000 max=17593 max_rank=4 min=15498 min_rank=2 imbalance_ratio=1.073792 max_load_reduction=113479 max_load_reduction_pct=0.865776 moved_to_echo=66447 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 0, 0, 0, 4094, 0, 0, 0, 4096, 4096, 0, 15, 2182, 1394, 269, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 4096, 0, 52, 0, 0, 4095, 3910, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2720, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1914, 0, 0, 4096, 0, 0, 0, 4096, 959, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 4096, 0, 4096, 0, 4053, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 4096, 3827, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2600, 1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16333, 16524, 17606, 16387, 15162, 16443, 16231, 16386] avg=16384.0000 max=17606 max_rank=2 min=15162 min_rank=4 imbalance_ratio=1.074585 max_load_reduction=113466 max_load_reduction_pct=0.865677 moved_to_echo=64222 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 896, 0, 0, 0, 32768, 128, 32768, 0, 32000, 128, 0, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 0, 128, 0, 32768, 32768, 128, 128, 128, 128, 128, 128, 128, 128, 0, 32768, 20864, 12032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 32768, 128, 128, 128, 128, 32768, 128, 0, 0, 32768, 128, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 15360, 0, 128, 32768, 0, 128, 0, 32768, 17536, 32768, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 128, 0, 0, 128, 32768, 0, 128, 0, 32768, 32768, 128, 640, 17536, 11264, 2176, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 128, 12032, 0, 128, 128, 0, 128, 0, 128, 0, 32768, 32768, 20864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[2304, 0, 0, 128, 0, 32768, 30720, 32768, 128, 0, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 128, 0, 128, 0, 128, 0, 1408, 32768, 0, 384, 0, 0, 32768, 31360, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 986, 0, 4096, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 10, 0, 2845, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 3, 433, 0, 4, 0, 4096, 0, 0, 4092, 2, 0, 6, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4, 4096, 0, 0, 3, 3986, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 82, 4093, 0, 0, 0, 0, 0, 0, 3957, 0, 0, 0, 0, 4096, 4091, 0, 0, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 1901, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 937, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 627, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 7, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4069, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 677, 4096, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 4093, 4095, 0, 4095, 0, 4070, 1, 0, 4095, 1, 1134, 0, 4093, 1, 0, 4011, 0, 0, 0, 1, 1, 112, 4095, 174, 1, 4095, 4077, 0, 0, 441, 1, 0, 1, 1, 0, 1, 1, 4095, 15, 4095, 0, 1, 1, 0, 2166, 0, 0, 1, 4094, 1, 0, 4095, 0, 0, 0, 4095, 0, 1, 1, 0, 4048, 1, 4078, 2, 1, 2, 4094, 0, 2, 0, 4056, 1, 1, 0, 4095, 1, 31, 0, 0, 1, 0, 7, 1, 4094, 501, 4095, 1, 0, 0, 1, 0, 1, 3791, 0, 1, 31, 0, 4087, 4094, 0, 0, 22, 4095, 1, 9, 0, 1, 0, 1, 1, 0, 1, 2, 0, 4051, 0, 0, 4095, 4094, 4093, 39, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 959, 0, 4096, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 2640, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 819, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3676, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 202, 4096, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 922, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 4096, 0, 0, 2104, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1119, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4035, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 50, 4096, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 2, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2359, 0, 4096, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 11, 0, 1256, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 577, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3906, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 242, 4096, 0, 0, 0, 0, 0, 0, 3948, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 1198, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 1907, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1040, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4082, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 1772, 0, 4096, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 1707, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 601, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3698, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 451, 4096, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2696, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2845, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 3, 4096, 2083, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 0, 4, 0, 4096, 0, 0, 4092, 2, 0, 6, 4096, 0, 0, 0, 0, 1847, 1323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4, 166, 0, 0, 3, 3986, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 82, 4093, 0, 0, 0, 0, 0, 0, 4096, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3957, 0, 0, 0, 0, 4096, 4091, 0, 0, 3, 4092, 0, 0, 0, 0, 0, 77, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15970, 16367, 17496, 15899, 16441, 16183, 16332, 16384] avg=16384.0000 max=17496 max_rank=2 min=15899 min_rank=3 imbalance_ratio=1.067871 max_load_reduction=113576 max_load_reduction_pct=0.866516 moved_to_echo=65340 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2696, 0, 0, 1, 4093, 0, 0, 0, 0, 4070, 1, 0, 4095, 1, 1134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 4011, 0, 0, 0, 1, 1, 112, 0, 174, 1, 4079, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 441, 1, 0, 1, 1, 0, 1, 1, 4095, 15, 4095, 0, 1, 1, 0, 4095, 2082, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2166, 0, 0, 1, 4094, 1, 0, 4095, 0, 0, 0, 4095, 0, 1, 1, 0, 1846, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 1, 4078, 2, 1, 2, 166, 0, 2, 0, 4056, 1, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 1, 0, 7, 1, 4094, 501, 4095, 1, 0, 0, 1, 0, 1, 4095, 3815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3791, 0, 1, 31, 0, 4087, 4094, 0, 0, 22, 4095, 1, 9, 0, 1, 0, 77, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 2, 0, 4051, 0, 0, 4095, 4094, 4093, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16091, 16550, 15110, 17622, 16454, 16643, 16225, 16377] avg=16384.0000 max=17622 max_rank=3 min=15110 min_rank=2 imbalance_ratio=1.075562 max_load_reduction=113450 max_load_reduction_pct=0.865555 moved_to_echo=65699 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2692, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2640, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2090, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 819, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1850, 1326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 156, 0, 0, 0, 3676, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 202, 4096, 0, 0, 0, 0, 0, 0, 4096, 3815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 78, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15939, 16345, 17299, 16283, 16120, 16305, 16390, 16391] avg=16384.0000 max=17299 max_rank=2 min=15939 min_rank=0 imbalance_ratio=1.055847 max_load_reduction=113773 max_load_reduction_pct=0.868019 moved_to_echo=65206 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2696, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 2359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1256, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2083, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 577, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3, 4096, 0, 0, 0, 0, 1847, 1323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 166, 0, 0, 0, 3906, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 242, 4096, 0, 0, 0, 0, 0, 0, 4096, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 77, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17343, 16368, 15907, 16038, 16359, 16346, 16327, 16384] avg=16384.0000 max=17343 max_rank=0 min=15907 min_rank=2 imbalance_ratio=1.058533 max_load_reduction=113729 max_load_reduction_pct=0.867683 moved_to_echo=65416 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2696, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 937, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2083, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 627, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 7, 4096, 0, 0, 0, 0, 1847, 1323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 166, 0, 0, 0, 4069, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 677, 4096, 0, 0, 0, 0, 0, 0, 4096, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 77, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16885, 16364, 15588, 16092, 16521, 16781, 16457, 16384] avg=16384.0000 max=16885 max_rank=0 min=15588 min_rank=2 imbalance_ratio=1.030579 max_load_reduction=114187 max_load_reduction_pct=0.871178 moved_to_echo=66143 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2696, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1907, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2083, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1847, 1323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 166, 0, 0, 0, 4082, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 77, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16182, 16368, 16558, 16498, 16536, 16106, 16440, 16384] avg=16384.0000 max=16558 max_rank=2 min=16106 min_rank=5 imbalance_ratio=1.010620 max_load_reduction=114514 max_load_reduction_pct=0.873672 moved_to_echo=65466 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2696, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2104, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2083, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1119, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1847, 1323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 166, 0, 0, 0, 4035, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 50, 4096, 0, 0, 0, 0, 0, 0, 4096, 3815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 2, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 77, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15906, 16370, 16756, 16577, 16489, 16153, 16437, 16384] avg=16384.0000 max=16756 max_rank=2 min=15906 min_rank=0 imbalance_ratio=1.022705 max_load_reduction=114316 max_load_reduction_pct=0.872162 moved_to_echo=65463 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[2696, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1707, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2083, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 601, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4, 4096, 0, 0, 0, 0, 1847, 1323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 166, 0, 0, 0, 3698, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 451, 4096, 0, 0, 0, 0, 0, 0, 4096, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 77, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16756, 16340, 16358, 16063, 16152, 16555, 16464, 16384] avg=16384.0000 max=16756 max_rank=0 min=16063 min_rank=3 imbalance_ratio=1.022705 max_load_reduction=114316 max_load_reduction_pct=0.872162 moved_to_echo=65555 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32000, 0, 128, 128, 0, 32768, 32768, 0, 0, 128, 32768, 128, 128, 0, 128, 0, 640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 0, 128, 128, 32768, 2304, 32768, 128, 0, 0, 128, 0, 128, 32768, 30592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[7424, 0, 128, 128, 32768, 128, 0, 32768, 128, 0, 128, 32768, 0, 128, 128, 0, 14848, 10624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 0, 32640, 0, 0, 0, 128, 128, 128, 0, 256, 128, 32640, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[21632, 0, 0, 128, 32768, 0, 0, 0, 0, 32768, 128, 0, 32768, 128, 11264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 32768, 128, 128, 128, 1408, 0, 128, 128, 31616, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 0, 32768, 0, 0, 32768, 32768, 32768, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 13952, 128, 0, 128, 128, 0, 128, 128, 32768, 128, 32768, 0, 128, 128, 128, 32768, 16768, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4096, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 4096, 3, 4088, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4093, 4096, 0, 0, 4096, 1, 0, 4094, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4090, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 6, 0, 0, 15, 0, 4096, 0, 0, 0, 0, 6, 4035, 0, 0, 0, 0, 4, 0, 4089, 7, 0, 0, 4096, 0, 4096, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 1, 4093, 1, 1, 1, 0, 0, 4094, 1, 1, 1, 3, 1, 63, 1, 0, 1, 4095, 0, 0, 0, 1, 28, 0, 0, 1, 1, 1, 4095, 4084, 1, 1, 1, 0, 0, 4094, 0, 4095, 0, 0, 4082, 3, 0, 4094, 1, 0, 4, 0, 0, 0, 0, 0, 4090, 0, 4094, 4095, 0, 1, 0, 4095, 4095, 3, 0, 3573, 0, 0, 4056, 0, 1, 0, 1, 1, 4089, 1, 1, 1, 7, 0, 4095, 2, 3, 4095, 4095, 4094, 1, 0, 4095, 4095, 4091, 0, 0, 0, 0, 0, 4095, 4069, 1, 1, 4092, 0, 0, 6, 0, 0, 4095, 3, 0, 0, 1, 596, 4031, 0, 1, 0, 0, 0, 1, 4059, 0, 2, 1, 4094, 13, 4093, 0, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4081, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4079, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4096, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 2, 4038, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4096, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4070, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 63, 4038, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 3, 0, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4096, 0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4083, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4048, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 32, 3954, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4096, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 4092, 4096, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4088, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4094, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4095, 52, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 6, 0, 0, 15, 0, 4063, 0, 0, 0, 0, 6, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4089, 7, 0, 0, 4096, 0, 4096, 0, 0, 2, 4094, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16376, 16378, 16384, 16383, 16458, 16381, 16315, 16397] avg=16384.0000 max=16458 max_rank=4 min=16315 min_rank=6 imbalance_ratio=1.004517 max_load_reduction=114614 max_load_reduction_pct=0.874435 moved_to_echo=65551 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4093, 1, 1, 1, 0, 0, 4094, 1, 1, 1, 3, 1, 63, 1, 4095, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 1, 28, 0, 0, 1, 1, 1, 4095, 4084, 1, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4094, 0, 4095, 0, 0, 4082, 3, 0, 4094, 1, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4094, 2, 0, 1, 0, 4095, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3573, 0, 0, 4056, 0, 1, 0, 1, 1, 4089, 1, 1, 1, 7, 0, 4095, 52, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 3, 4094, 1, 0, 0, 4095, 4091, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 1, 1, 4092, 0, 0, 6, 0, 0, 4062, 3, 0, 0, 1, 596, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4059, 0, 2, 1, 4094, 13, 4093, 0, 0, 3, 4093, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16443, 16348, 16378, 16380, 15908, 16384, 16862, 16369] avg=16384.0000 max=16862 max_rank=6 min=15908 min_rank=4 imbalance_ratio=1.029175 max_load_reduction=114210 max_load_reduction_pct=0.871353 moved_to_echo=65523 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4096, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4096, 4096, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 52, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 63, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16354, 16387, 16385, 16440, 16384, 16356, 16391] avg=16384.0000 max=16440 max_rank=4 min=16354 min_rank=1 imbalance_ratio=1.003418 max_load_reduction=114632 max_load_reduction_pct=0.874573 moved_to_echo=65571 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4096, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 4096, 4096, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 53, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 0, 4063, 0, 0, 0, 0, 2, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16396, 16387, 16384, 16463, 16383, 16301, 16383] avg=16384.0000 max=16463 max_rank=4 min=16301 min_rank=6 imbalance_ratio=1.004822 max_load_reduction=114609 max_load_reduction_pct=0.874397 moved_to_echo=65530 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 4096, 4096, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 52, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16376, 16388, 16393, 16384, 16453, 16384, 16307, 16387] avg=16384.0000 max=16453 max_rank=4 min=16307 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=114619 max_load_reduction_pct=0.874474 moved_to_echo=65531 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 4096, 4096, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 52, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 4, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4094, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16350, 16382, 16386, 16466, 16387, 16340, 16386] avg=16384.0000 max=16466 max_rank=4 min=16340 min_rank=6 imbalance_ratio=1.005005 max_load_reduction=114606 max_load_reduction_pct=0.874374 moved_to_echo=65579 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 4096, 4096, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 52, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16349, 16387, 16384, 16466, 16384, 16350, 16375] avg=16384.0000 max=16466 max_rank=4 min=16349 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=114606 max_load_reduction_pct=0.874374 moved_to_echo=65575 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4096, 0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 4096, 4096, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 52, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4063, 0, 0, 0, 0, 32, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4094, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16509, 16374, 16386, 16418, 16385, 16241, 16384] avg=16384.0000 max=16509 max_rank=1 min=16241 min_rank=6 imbalance_ratio=1.007629 max_load_reduction=114563 max_load_reduction_pct=0.874046 moved_to_echo=65428 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 32768, 128, 0, 128, 128, 0, 32512, 128, 0, 0, 128, 768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 128, 0, 32768, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32256, 128, 0, 32768, 0, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 0, 32768, 512, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 0, 128, 128, 32768, 128, 128, 128, 32768, 128, 32768, 0, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 32768, 0, 32768, 128, 0, 128, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 0, 0, 0, 128, 384, 0, 0, 128, 128, 128, 32768, 32768, 128, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 32768, 0, 128, 32768, 128, 0, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2434, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 27, 0, 0, 2248, 0, 0, 0, 12, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4077, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 73, 0, 4096, 0, 4096, 0, 4092, 4096, 0, 0, 0, 0, 0, 4034, 0, 4096, 3474, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2462, 4096, 0, 4096, 0, 0, 0, 0, 4094, 4096, 0, 24, 0, 0, 3244, 0, 0, 0, 25, 6, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 205, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 12, 4090, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4019, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 162, 0, 4096, 4, 4096, 0, 4086, 4096, 0, 0, 0, 0, 0, 4078, 0, 4092, 2161, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4095, 0, 0, 0, 3, 0, 4093, 4094, 0, 0, 62, 4095, 3869, 4092, 0, 4094, 0, 0, 0, 0, 4095, 4095, 1, 199, 1, 1, 465, 0, 0, 1, 0, 14, 0, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 2, 0, 7, 0, 2, 0, 1, 1, 3, 4005, 0, 1, 4095, 1, 0, 2, 53, 0, 0, 4095, 2, 2, 3, 1, 1, 1, 0, 4095, 0, 1, 4084, 4092, 1, 4095, 4089, 4093, 4092, 15, 5, 0, 4079, 4095, 0, 1, 0, 4094, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 4095, 4095, 0, 1, 4095, 0, 4094, 0, 4095, 4094, 0, 1, 0, 1, 11, 4094, 0, 4095, 3615, 1, 0, 0, 1, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4096, 915, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 25, 0, 0, 3352, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 52, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 89, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4057, 0, 4096, 3807, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3242, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 41, 0, 0, 1691, 0, 0, 0, 29, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 354, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4051, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 86, 0, 4096, 0, 4096, 0, 4072, 4096, 0, 0, 0, 0, 0, 4086, 0, 4096, 2819, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2405, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 18, 0, 0, 1939, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 112, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4060, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 147, 0, 4096, 0, 4096, 0, 4091, 4096, 0, 0, 0, 0, 0, 4095, 0, 4096, 3605, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2958, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 5, 0, 0, 707, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 807, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3708, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 2925, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 35, 0, 0, 1871, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4058, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 213, 0, 4096, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 4076, 0, 4096, 3188, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2462, 4096, 0, 3556, 0, 0, 0, 0, 4094, 0, 0, 24, 0, 0, 3244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 6, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 205, 0, 4096, 3890, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 12, 4090, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 30, 4096, 0, 0, 0, 4096, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 4005, 4, 4096, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4092, 2161, 0, 0, 0, 0, 0, 0, 0, 4066, 540, 206, 91, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 17476, 16400, 16403, 16389, 16337, 16449, 15241] avg=16384.0000 max=17476 max_rank=1 min=15241 min_rank=7 imbalance_ratio=1.066650 max_load_reduction=113596 max_load_reduction_pct=0.866669 moved_to_echo=64416 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4088, 0, 0, 0, 3, 0, 4093, 4094, 0, 0, 62, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3869, 4092, 0, 3554, 0, 0, 0, 0, 4095, 0, 1, 199, 1, 1, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 14, 0, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 2, 0, 4095, 4095, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 2, 0, 1, 1, 3, 4005, 0, 1, 4095, 1, 0, 2, 53, 0, 4095, 3889, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 2, 3, 1, 1, 1, 0, 4095, 0, 1, 4084, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 30, 4092, 15, 5, 0, 4079, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 0, 0, 0, 1, 4004, 0, 4094, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 11, 4094, 0, 4095, 3615, 1, 0, 0, 1, 0, 1, 0, 4063, 540, 205, 91, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16436, 16277, 16385, 16172, 16378, 16406, 16292, 16726] avg=16384.0000 max=16726 max_rank=7 min=16172 min_rank=3 imbalance_ratio=1.020874 max_load_reduction=114346 max_load_reduction_pct=0.872391 moved_to_echo=65802 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2434, 4096, 0, 3556, 0, 0, 0, 0, 4096, 0, 0, 27, 0, 0, 2248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 3889, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 30, 4096, 0, 0, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 4005, 0, 4096, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 4096, 3474, 0, 0, 0, 0, 0, 0, 0, 4066, 540, 207, 91, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16457, 16382, 16196, 16388, 16395, 16362, 16515] avg=16384.0000 max=16515 max_rank=7 min=16196 min_rank=3 imbalance_ratio=1.007996 max_load_reduction=114557 max_load_reduction_pct=0.874001 moved_to_echo=65660 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3242, 4096, 0, 3556, 0, 0, 0, 0, 4096, 0, 0, 41, 0, 0, 1691, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 354, 0, 4096, 3890, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 30, 4096, 0, 0, 0, 4096, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 4005, 0, 4096, 0, 4072, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 2819, 0, 0, 0, 0, 0, 0, 0, 4066, 540, 206, 91, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16722, 16406, 16549, 16383, 16369, 16355, 15911] avg=16384.0000 max=16722 max_rank=1 min=15911 min_rank=7 imbalance_ratio=1.020630 max_load_reduction=114350 max_load_reduction_pct=0.872421 moved_to_echo=65018 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2405, 4096, 0, 3549, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 1939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 112, 0, 4096, 3890, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 23, 4096, 0, 0, 0, 4096, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 4000, 0, 4096, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 3605, 0, 0, 0, 0, 0, 0, 0, 4073, 547, 206, 96, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16103, 16376, 16306, 16382, 16371, 16430, 16729] avg=16384.0000 max=16729 max_rank=7 min=16103 min_rank=1 imbalance_ratio=1.021057 max_load_reduction=114343 max_load_reduction_pct=0.872368 moved_to_echo=65912 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2958, 4096, 0, 3556, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 807, 0, 4096, 3890, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 30, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4005, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3708, 0, 0, 0, 0, 0, 0, 0, 4066, 540, 206, 91, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 15418, 16369, 17001, 16384, 16414, 16299, 16810] avg=16384.0000 max=17001 max_rank=3 min=15418 min_rank=1 imbalance_ratio=1.037659 max_load_reduction=114071 max_load_reduction_pct=0.870293 moved_to_echo=65907 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 915, 4096, 0, 3556, 0, 0, 0, 0, 4096, 0, 0, 25, 0, 0, 3352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 52, 0, 4096, 3890, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 30, 4096, 0, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 4005, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 4096, 3807, 0, 0, 0, 0, 0, 0, 0, 4066, 540, 206, 91, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16376, 16040, 16369, 16247, 16384, 16404, 16382, 16870] avg=16384.0000 max=16870 max_rank=7 min=16040 min_rank=1 imbalance_ratio=1.029663 max_load_reduction=114202 max_load_reduction_pct=0.871292 moved_to_echo=66040 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2925, 4096, 0, 3556, 0, 0, 0, 0, 4096, 0, 0, 35, 0, 0, 1871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 4096, 3890, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 30, 4096, 0, 0, 0, 4096, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 0, 4005, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 4096, 3188, 0, 0, 0, 0, 0, 0, 0, 4066, 540, 206, 91, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16579, 16385, 16198, 16384, 16376, 16503, 16270] avg=16384.0000 max=16579 max_rank=1 min=16198 min_rank=3 imbalance_ratio=1.011902 max_load_reduction=114493 max_load_reduction_pct=0.873512 moved_to_echo=65533 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 32768, 0, 0, 0, 128, 0, 32768, 32768, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[21248, 32768, 0, 28544, 0, 0, 0, 0, 32768, 0, 128, 384, 128, 128, 15616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 128, 0, 32768, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 896, 128, 32128, 128, 32768, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 256, 32768, 128, 128, 0, 32768, 32640, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 128, 128, 128, 32768, 0, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 32768, 128, 128, 32768, 128, 0, 128, 1664, 0, 32768, 31232, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 128, 32640, 0, 32768, 26496, 128, 0, 0, 128, 0, 128, 0, 32640, 4352, 1664, 768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 4094, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 3891, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 578, 3800, 0, 0, 4096, 3880, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4076, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 545, 3526, 0, 0, 4096, 3470, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4090, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4033, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 265, 3833, 0, 0, 4096, 4025, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4092, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 4096, 0, 4096, 4095, 0, 0, 1, 4096, 3, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 4096, 0, 3909, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1563, 2776, 0, 0, 4096, 3779, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 12, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4092, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4071, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 92, 4032, 0, 0, 4096, 4081, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4091, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4087, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 205, 3499, 0, 0, 4096, 4049, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 2, 0, 0, 21, 0, 4073, 0, 4094, 4095, 0, 0, 0, 4095, 0, 1, 0, 4041, 4095, 17, 0, 2, 0, 58, 1, 4095, 4095, 0, 4093, 5, 3678, 1, 1, 4092, 1, 0, 0, 1, 1, 0, 230, 0, 1, 1, 1, 2, 0, 1, 0, 4, 4094, 4093, 6, 0, 2, 1658, 3173, 1, 0, 4094, 3774, 0, 1, 1, 4015, 1, 0, 0, 4088, 0, 1, 0, 0, 1, 1, 4090, 4092, 0, 0, 0, 0, 0, 1, 0, 1, 4094, 4095, 1, 0, 0, 4091, 0, 1, 2, 4095, 3867, 1, 4095, 0, 0, 1, 4093, 1, 1, 1, 0, 3, 1, 4095, 4095, 0, 0, 4095, 1, 0, 1, 62, 1, 2, 0, 4095, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 13, 0, 0, 0, 0, 0, 4092, 4096, 0, 4096, 0, 3909, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 685, 3735, 0, 0, 4096, 3925, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4091, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 4096, 0, 4096, 4095, 0, 3994, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3909, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 195, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1563, 2776, 0, 0, 3843, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3779, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 20, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16351, 16379, 16387, 16476, 16300, 16397, 16407, 16375] avg=16384.0000 max=16476 max_rank=3 min=16300 min_rank=4 imbalance_ratio=1.005615 max_load_reduction=114596 max_load_reduction_pct=0.874298 moved_to_echo=65479 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 4094, 0, 4096, 4096, 0, 3995, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3891, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 83, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 578, 3800, 0, 0, 3843, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3880, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 18, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16337, 16385, 16257, 16514, 16403, 16387, 16412, 16377] avg=16384.0000 max=16514 max_rank=3 min=16257 min_rank=2 imbalance_ratio=1.007935 max_load_reduction=114558 max_load_reduction_pct=0.874008 moved_to_echo=65579 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 0, 4096, 0, 4096, 4096, 0, 3995, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4033, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 265, 3833, 0, 0, 3845, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4025, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 18, 13, 6, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16415, 16378, 16326, 16236, 16545, 16377, 16424, 16371] avg=16384.0000 max=16545 max_rank=4 min=16236 min_rank=3 imbalance_ratio=1.009827 max_load_reduction=114527 max_load_reduction_pct=0.873772 moved_to_echo=65717 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 303, 0, 4096, 0, 4096, 4096, 0, 3995, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 374, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 545, 3526, 0, 0, 3843, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3470, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 18, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16591, 16380, 16733, 16207, 15993, 16384, 16407, 16377] avg=16384.0000 max=16733 max_rank=2 min=15993 min_rank=4 imbalance_ratio=1.021301 max_load_reduction=114339 max_load_reduction_pct=0.872337 moved_to_echo=65161 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4096, 0, 4096, 4096, 0, 3994, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4071, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 92, 4032, 0, 0, 3843, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 20, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16299, 16380, 16355, 16261, 16602, 16388, 16410, 16377] avg=16384.0000 max=16602 max_rank=4 min=16261 min_rank=3 imbalance_ratio=1.013306 max_load_reduction=114470 max_load_reduction_pct=0.873337 moved_to_echo=65777 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 4096, 0, 4096, 4096, 0, 3995, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4087, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 237, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 205, 3499, 0, 0, 3844, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4049, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 18, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16501, 16380, 16606, 15841, 16571, 16384, 16412, 16377] avg=16384.0000 max=16606 max_rank=2 min=15841 min_rank=3 imbalance_ratio=1.013550 max_load_reduction=114466 max_load_reduction_pct=0.873306 moved_to_echo=65744 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 2, 0, 0, 21, 0, 4073, 0, 4094, 4095, 0, 3993, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1, 0, 4041, 0, 17, 0, 2, 0, 58, 1, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 5, 3678, 1, 1, 4092, 1, 0, 0, 1, 1, 0, 230, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 0, 1, 0, 4, 4094, 4093, 6, 0, 2, 1658, 3173, 1, 0, 3842, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3774, 0, 1, 1, 4015, 1, 0, 0, 4088, 0, 1, 0, 0, 1, 1, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4092, 0, 0, 0, 0, 0, 1, 0, 1, 4094, 0, 1, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4095, 3867, 1, 4095, 0, 0, 1, 4093, 1, 1, 1, 0, 3, 20, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4088, 0, 0, 4095, 1, 0, 1, 62, 1, 2, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16284, 16401, 16194, 16980, 16210, 16370, 16192, 16441] avg=16384.0000 max=16980 max_rank=3 min=16192 min_rank=6 imbalance_ratio=1.036377 max_load_reduction=114092 max_load_reduction_pct=0.870453 moved_to_echo=65213 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 4096, 4096, 0, 3994, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3909, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 22, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 685, 3735, 0, 0, 3843, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3925, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 18, 7, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16294, 16389, 16214, 16557, 16448, 16385, 16408, 16377] avg=16384.0000 max=16557 max_rank=3 min=16214 min_rank=2 imbalance_ratio=1.010559 max_load_reduction=114515 max_load_reduction_pct=0.873680 moved_to_echo=65618 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 128, 31744, 128, 128, 32768, 128, 0, 0, 128, 128, 0, 1280, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 32768, 128, 0, 128, 0, 0, 128, 0, 128, 32768, 0, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 32768, 32640, 128, 32768, 0, 0, 128, 32768, 128, 128, 128, 0, 128, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 128, 128, 0, 32768, 0, 128, 0, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 128, 0, 0, 0, 0, 128, 0, 0, 896, 0, 32768, 0, 32768, 32768, 0, 32000, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 31104, 0, 128, 128, 32768, 128, 0, 0, 32768, 0, 128, 0, 0, 128, 128, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 32768, 0, 0, 32768, 128, 0, 128, 128, 128, 128, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 32768, 32768, 128, 0, 128, 5632, 28416, 128, 0, 30848, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[383, 1393, 1520, 0, 138, 0, 2616, 48, 606, 1309, 213, 516, 2647, 3062, 0, 2549, 2212, 1404, 2033, 2450, 89, 860, 21, 1042, 10, 12, 3684, 2137, 1543, 12, 2153, 707, 2633, 0, 244, 506, 332, 476, 2046, 159, 149, 11, 1816, 3121, 0, 2988, 342, 800, 1798, 2281, 1008, 189, 0, 5, 10, 2936, 0, 648, 36, 384, 1924, 2931, 2080, 11, 1061, 342, 0, 1597, 2458, 492, 227, 525, 2833, 2462, 1855, 13, 707, 2, 243, 2273, 386, 923, 0, 20, 609, 819, 921, 74, 2193, 11, 1715, 1309, 102, 2423, 394, 0, 1424, 1393, 50, 971, 0, 858, 1481, 567, 116, 962, 2, 887, 640, 2514, 904, 815, 1336, 2528, 391, 2735, 1505, 501, 1695, 1641, 1696, 2550, 777, 1030, 224, 0, 657, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[514, 1341, 1068, 3, 122, 3, 2975, 27, 427, 1065, 195, 561, 2056, 3241, 4, 2523, 1720, 1406, 1741, 2840, 41, 945, 109, 974, 18, 11, 3460, 1808, 1162, 3, 2157, 904, 3167, 0, 242, 444, 245, 583, 1859, 133, 166, 0, 1819, 3088, 5, 3124, 344, 927, 1694, 2107, 1205, 122, 2, 7, 34, 2839, 2, 502, 95, 322, 1263, 3623, 1911, 58, 1179, 386, 0, 1362, 2362, 163, 575, 843, 3169, 2588, 1568, 15, 955, 0, 246, 2198, 987, 980, 3, 64, 612, 1037, 638, 55, 2210, 16, 2428, 1829, 122, 2354, 311, 0, 1563, 746, 73, 812, 2, 447, 1373, 526, 292, 1025, 5, 1069, 696, 2386, 1211, 1532, 1906, 2540, 493, 2008, 796, 525, 1666, 1206, 1593, 3005, 932, 1390, 172, 3, 467, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[454, 1650, 1321, 0, 30, 0, 3396, 53, 361, 1912, 238, 276, 2028, 3379, 0, 2601, 1974, 1535, 1859, 2313, 98, 858, 144, 856, 6, 46, 3706, 2037, 1978, 26, 2391, 1044, 2589, 0, 119, 783, 455, 411, 1901, 132, 91, 4, 1915, 3168, 2, 2817, 472, 898, 2175, 2420, 944, 62, 1, 5, 36, 2392, 0, 535, 10, 273, 1659, 3640, 2031, 8, 1436, 246, 0, 1528, 2022, 260, 211, 681, 2279, 2185, 1813, 35, 743, 3, 550, 1594, 999, 1643, 1, 25, 709, 632, 403, 46, 1859, 11, 2372, 1730, 161, 2220, 315, 0, 1385, 1810, 15, 718, 0, 831, 1525, 703, 213, 1253, 10, 1251, 357, 2068, 1406, 1464, 1752, 2707, 284, 1487, 618, 571, 1500, 1025, 1525, 2641, 962, 1091, 118, 3, 549, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[670, 1090, 1813, 1, 150, 3, 3014, 33, 787, 1356, 228, 512, 1931, 3097, 3, 3200, 1645, 1498, 2604, 2617, 80, 864, 167, 1563, 10, 5, 3491, 1928, 1188, 14, 2076, 610, 2637, 0, 174, 822, 302, 669, 1838, 59, 107, 23, 1674, 3119, 0, 3327, 468, 497, 1792, 2620, 1097, 156, 0, 3, 29, 2433, 0, 458, 111, 284, 2071, 3414, 1241, 42, 968, 212, 0, 1625, 1946, 203, 271, 608, 2547, 2546, 1628, 2, 516, 5, 331, 1827, 843, 1172, 1, 42, 549, 473, 833, 147, 2120, 11, 1646, 1447, 262, 2100, 427, 1, 1409, 1177, 8, 1259, 5, 763, 1808, 851, 314, 1230, 6, 867, 562, 2757, 1530, 1452, 1694, 2119, 365, 2335, 826, 431, 1845, 1162, 1374, 2664, 823, 1633, 188, 18, 571, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[567, 1201, 1908, 1, 26, 0, 3201, 76, 708, 1460, 251, 496, 2658, 3068, 0, 2840, 1257, 1414, 2047, 2294, 177, 1517, 144, 821, 13, 25, 3693, 2152, 1765, 17, 2145, 670, 2622, 12, 235, 1021, 152, 404, 1768, 138, 92, 0, 2161, 3032, 2, 2739, 507, 1190, 1803, 2067, 1380, 75, 2, 5, 43, 2717, 0, 297, 91, 260, 1697, 3391, 1814, 39, 964, 257, 1, 1623, 1623, 439, 281, 1077, 2644, 2312, 2083, 8, 835, 3, 174, 2084, 971, 1120, 5, 99, 534, 298, 374, 70, 2117, 24, 1927, 1568, 157, 2138, 464, 4, 1538, 1182, 35, 649, 8, 384, 1285, 421, 219, 1063, 35, 756, 354, 2379, 1287, 1224, 1539, 2399, 416, 1625, 726, 592, 2262, 1537, 1618, 3196, 938, 1793, 135, 3, 822, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[662, 1170, 1051, 1, 125, 0, 3301, 108, 837, 1110, 263, 741, 2263, 2972, 0, 2341, 1016, 1254, 1909, 2484, 160, 901, 115, 1013, 43, 18, 3606, 1952, 2061, 8, 2133, 553, 2240, 12, 347, 702, 529, 390, 1509, 228, 151, 5, 1987, 3039, 10, 2852, 378, 792, 1973, 2585, 1446, 165, 9, 19, 17, 2460, 0, 309, 122, 462, 2019, 3116, 1250, 28, 1269, 366, 0, 2263, 1925, 1041, 312, 951, 2333, 2651, 1691, 31, 267, 16, 244, 1947, 1101, 2107, 0, 42, 505, 308, 723, 199, 2429, 68, 2231, 1115, 200, 2261, 427, 0, 1885, 1356, 29, 1092, 7, 457, 1320, 1235, 348, 924, 5, 748, 601, 2125, 682, 1277, 2005, 2309, 207, 2313, 567, 313, 1493, 1459, 1643, 3157, 519, 1733, 250, 3, 654, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[306, 1438, 1482, 0, 75, 0, 3070, 44, 848, 1861, 202, 330, 2797, 3516, 0, 2576, 1351, 1673, 1688, 2231, 72, 953, 140, 1226, 6, 11, 3707, 2381, 1956, 26, 2473, 1016, 2597, 0, 156, 888, 407, 470, 1817, 121, 147, 1, 1913, 3012, 0, 2802, 529, 905, 1536, 2718, 923, 76, 0, 7, 26, 2479, 0, 430, 33, 268, 2211, 3158, 2112, 13, 1217, 308, 0, 1637, 1994, 425, 226, 470, 2343, 1869, 1887, 7, 760, 6, 384, 1860, 625, 1075, 0, 33, 797, 651, 423, 51, 2164, 20, 1784, 1060, 182, 2340, 202, 0, 1779, 1455, 16, 925, 6, 565, 1643, 739, 303, 807, 7, 1058, 426, 2323, 1081, 1338, 1657, 2640, 502, 2008, 656, 593, 2055, 1220, 1468, 3021, 746, 955, 211, 1, 856, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[580, 1791, 726, 0, 21, 1, 3294, 59, 683, 2078, 221, 278, 2075, 3381, 0, 2743, 1424, 983, 2053, 2402, 75, 739, 119, 1057, 1, 5, 3665, 1897, 2178, 17, 2028, 662, 2226, 0, 377, 932, 650, 514, 1835, 63, 371, 0, 1714, 3285, 0, 3117, 311, 921, 2210, 2528, 997, 123, 0, 0, 10, 2682, 1, 286, 20, 247, 1903, 3155, 2183, 0, 1225, 245, 1, 1484, 1820, 523, 137, 725, 2619, 2108, 2062, 16, 817, 1, 294, 1641, 1045, 805, 0, 7, 899, 591, 402, 27, 1981, 23, 1677, 1488, 154, 2292, 322, 0, 1823, 2037, 12, 737, 1, 1011, 1345, 758, 242, 1161, 3, 772, 434, 2851, 1522, 782, 1878, 2804, 300, 1744, 580, 443, 1888, 1219, 1201, 3516, 860, 946, 129, 0, 745, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[383, 1393, 1520, 0, 138, 0, 2616, 48, 606, 1309, 213, 516, 2647, 1931, 0, 2549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2212, 1404, 2033, 2400, 89, 860, 21, 1042, 10, 12, 0, 2137, 1543, 12, 2153, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2633, 0, 244, 506, 332, 476, 2046, 159, 149, 11, 1816, 3121, 0, 2988, 342, 800, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1798, 2281, 1008, 189, 0, 5, 10, 2936, 0, 648, 36, 384, 1924, 2931, 2080, 11, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1061, 342, 0, 1597, 2458, 492, 227, 525, 2833, 2462, 1855, 13, 707, 2, 243, 2273, 50, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 386, 923, 0, 20, 609, 819, 921, 74, 2193, 11, 1715, 1309, 102, 2423, 394, 0, 3684, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1424, 1393, 50, 971, 0, 858, 1481, 567, 116, 962, 2, 887, 640, 2514, 904, 815, 1530, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1336, 2528, 391, 2735, 1505, 501, 1695, 1641, 1696, 740, 777, 1030, 224, 0, 657, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15869, 16635, 16159, 16610, 17177, 15863, 15303, 17456] avg=16384.0000 max=17456 max_rank=7 min=15303 min_rank=6 imbalance_ratio=1.065430 max_load_reduction=113616 max_load_reduction_pct=0.866821 moved_to_echo=65799 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[454, 1650, 1321, 0, 30, 0, 3396, 53, 361, 1912, 238, 276, 2028, 2130, 0, 2601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1974, 1535, 1859, 2266, 98, 858, 144, 856, 6, 46, 0, 2037, 1978, 26, 2391, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2589, 0, 119, 783, 455, 411, 1901, 132, 91, 4, 1915, 3168, 2, 2817, 472, 898, 591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2175, 2420, 944, 62, 1, 5, 36, 2392, 0, 535, 10, 273, 1659, 3640, 2031, 8, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1436, 246, 0, 1528, 2022, 260, 211, 681, 2279, 2185, 1813, 35, 743, 3, 550, 1594, 47, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 999, 1643, 1, 25, 709, 632, 403, 46, 1859, 11, 2372, 1730, 161, 2220, 315, 0, 3706, 290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1385, 1810, 15, 718, 0, 831, 1525, 703, 213, 1253, 10, 1251, 357, 2068, 1406, 1464, 1585, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1752, 2707, 284, 1487, 618, 571, 1500, 1025, 1525, 766, 962, 1091, 118, 3, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16450, 17118, 16348, 16599, 15674, 17122, 16803, 14958] avg=16384.0000 max=17122 max_rank=5 min=14958 min_rank=7 imbalance_ratio=1.045044 max_load_reduction=113950 max_load_reduction_pct=0.869370 moved_to_echo=64557 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[514, 1341, 1068, 3, 122, 3, 2975, 27, 427, 1065, 195, 561, 2056, 2043, 4, 2523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1720, 1406, 1741, 2782, 41, 945, 109, 974, 18, 11, 0, 1808, 1162, 3, 2157, 904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3167, 0, 242, 444, 245, 583, 1859, 133, 166, 0, 1819, 3088, 5, 3124, 344, 927, 567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1694, 2107, 1205, 122, 2, 7, 34, 2839, 2, 502, 95, 322, 1263, 3623, 1911, 58, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1179, 386, 0, 1362, 2362, 163, 575, 843, 3169, 2588, 1568, 15, 955, 0, 246, 2198, 58, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 987, 980, 3, 64, 612, 1037, 638, 55, 2210, 16, 2428, 1829, 122, 2354, 311, 0, 3460, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1563, 746, 73, 812, 2, 447, 1373, 526, 292, 1025, 5, 1069, 696, 2386, 1211, 1532, 1804, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1906, 2540, 493, 2008, 796, 525, 1666, 1206, 1593, 871, 932, 1390, 172, 3, 467, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14927, 15781, 16713, 16177, 17707, 17436, 15762, 16569] avg=16384.0000 max=17707 max_rank=4 min=14927 min_rank=0 imbalance_ratio=1.080750 max_load_reduction=113365 max_load_reduction_pct=0.864906 moved_to_echo=67474 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[670, 1090, 1813, 1, 150, 3, 3014, 33, 787, 1356, 228, 512, 1931, 1952, 3, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1645, 1498, 2604, 2563, 80, 864, 167, 1563, 10, 5, 0, 1928, 1188, 14, 2076, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2637, 0, 174, 822, 302, 669, 1838, 59, 107, 23, 1674, 3119, 0, 3327, 468, 497, 542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1792, 2620, 1097, 156, 0, 3, 29, 2433, 0, 458, 111, 284, 2071, 3414, 1241, 42, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 968, 212, 0, 1625, 1946, 203, 271, 608, 2547, 2546, 1628, 2, 516, 5, 331, 1827, 54, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 843, 1172, 1, 42, 549, 473, 833, 147, 2120, 11, 1646, 1447, 262, 2100, 427, 1, 3491, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1409, 1177, 8, 1259, 5, 763, 1808, 851, 314, 1230, 6, 867, 562, 2757, 1530, 1452, 1599, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1694, 2119, 365, 2335, 826, 431, 1845, 1162, 1374, 773, 823, 1633, 188, 18, 571, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16743, 16815, 16258, 16125, 15327, 15857, 17788, 16159] avg=16384.0000 max=17788 max_rank=6 min=15327 min_rank=4 imbalance_ratio=1.085693 max_load_reduction=113284 max_load_reduction_pct=0.864288 moved_to_echo=65131 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[567, 1201, 1908, 1, 26, 0, 3201, 76, 708, 1460, 251, 496, 2658, 1917, 0, 2840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1257, 1414, 2047, 2242, 177, 1517, 144, 821, 13, 25, 0, 2152, 1765, 17, 2145, 670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2622, 12, 235, 1021, 152, 404, 1768, 138, 92, 0, 2161, 3032, 2, 2739, 507, 1190, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1803, 2067, 1380, 75, 2, 5, 43, 2717, 0, 297, 91, 260, 1697, 3391, 1814, 39, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 964, 257, 1, 1623, 1623, 439, 281, 1077, 2644, 2312, 2083, 8, 835, 3, 174, 2084, 52, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 971, 1120, 5, 99, 534, 298, 374, 70, 2117, 24, 1927, 1568, 157, 2138, 464, 4, 3693, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1538, 1182, 35, 649, 8, 384, 1285, 421, 219, 1063, 35, 756, 354, 2379, 1287, 1224, 1922, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1539, 2399, 416, 1625, 726, 592, 2262, 1537, 1618, 920, 938, 1793, 135, 3, 822, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17310, 16406, 16615, 16055, 16503, 15917, 14935, 17331] avg=16384.0000 max=17331 max_rank=7 min=14935 min_rank=6 imbalance_ratio=1.057800 max_load_reduction=113741 max_load_reduction_pct=0.867775 moved_to_echo=64686 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[306, 1438, 1482, 0, 75, 0, 3070, 44, 848, 1861, 202, 330, 2797, 2217, 0, 2576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1351, 1673, 1688, 2185, 72, 953, 140, 1226, 6, 11, 0, 2381, 1956, 26, 2473, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2597, 0, 156, 888, 407, 470, 1817, 121, 147, 1, 1913, 3012, 0, 2802, 529, 905, 615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1536, 2718, 923, 76, 0, 7, 26, 2479, 0, 430, 33, 268, 2211, 3158, 2112, 13, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1217, 308, 0, 1637, 1994, 425, 226, 470, 2343, 1869, 1887, 7, 760, 6, 384, 1860, 46, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 625, 1075, 0, 33, 797, 651, 423, 51, 2164, 20, 1784, 1060, 182, 2340, 202, 0, 3707, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1779, 1455, 16, 925, 6, 565, 1643, 739, 303, 807, 7, 1058, 426, 2323, 1081, 1338, 1813, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1657, 2640, 502, 2008, 656, 593, 2055, 1220, 1468, 877, 746, 955, 211, 1, 856, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17246, 17157, 16380, 16414, 15482, 15445, 16501, 16447] avg=16384.0000 max=17246 max_rank=0 min=15445 min_rank=5 imbalance_ratio=1.052612 max_load_reduction=113826 max_load_reduction_pct=0.868423 moved_to_echo=63875 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[662, 1170, 1051, 1, 125, 0, 3301, 108, 837, 1110, 263, 741, 2263, 1873, 0, 2341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1016, 1254, 1909, 2433, 160, 901, 115, 1013, 43, 18, 0, 1952, 2061, 8, 2133, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2240, 12, 347, 702, 529, 390, 1509, 228, 151, 5, 1987, 3039, 10, 2852, 378, 792, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1973, 2585, 1446, 165, 9, 19, 17, 2460, 0, 309, 122, 462, 2019, 3116, 1250, 28, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 366, 0, 2263, 1925, 1041, 312, 951, 2333, 2651, 1691, 31, 267, 16, 244, 1947, 51, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1101, 2107, 0, 42, 505, 308, 723, 199, 2429, 68, 2231, 1115, 200, 2261, 427, 0, 3606, 346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1885, 1356, 29, 1092, 7, 457, 1320, 1235, 348, 924, 5, 748, 601, 2125, 682, 1277, 1895, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2005, 2309, 207, 2313, 567, 313, 1493, 1459, 1643, 916, 519, 1733, 250, 3, 654, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15846, 15569, 15691, 16339, 17394, 17668, 16170, 16395] avg=16384.0000 max=17668 max_rank=5 min=15569 min_rank=1 imbalance_ratio=1.078369 max_load_reduction=113404 max_load_reduction_pct=0.865204 moved_to_echo=67627 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[580, 1791, 726, 0, 21, 1, 3294, 59, 683, 2078, 221, 278, 2075, 2131, 0, 2743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1424, 983, 2053, 2353, 75, 739, 119, 1057, 1, 5, 0, 1897, 2178, 17, 2028, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2226, 0, 377, 932, 650, 514, 1835, 63, 371, 0, 1714, 3285, 0, 3117, 311, 921, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2210, 2528, 997, 123, 0, 0, 10, 2682, 1, 286, 20, 247, 1903, 3155, 2183, 0, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1225, 245, 1, 1484, 1820, 523, 137, 725, 2619, 2108, 2062, 16, 817, 1, 294, 1641, 49, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1045, 805, 0, 7, 899, 591, 402, 27, 1981, 23, 1677, 1488, 154, 2292, 322, 0, 3665, 386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1823, 2037, 12, 737, 1, 1011, 1345, 758, 242, 1161, 3, 772, 434, 2851, 1522, 782, 2110, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1878, 2804, 300, 1744, 580, 443, 1888, 1219, 1201, 1020, 860, 946, 129, 0, 745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16681, 15591, 16908, 16753, 15808, 15764, 17810, 15757] avg=16384.0000 max=17810 max_rank=6 min=15591 min_rank=1 imbalance_ratio=1.087036 max_load_reduction=113262 max_load_reduction_pct=0.864120 moved_to_echo=65139 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[12672, 11264, 16000, 19328, 896, 7680, 1024, 8576, 128, 256, 0, 16384, 13952, 128, 17664, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[13824, 20096, 3072, 16256, 6400, 4096, 14464, 10496, 12160, 6912, 6656, 10624, 1536, 128, 5376, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[4224, 11136, 11008, 128, 768, 128, 24960, 512, 5376, 12160, 1920, 3712, 18560, 16256, 128, 21376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[12928, 11264, 256, 7168, 128, 5376, 11904, 5888, 2048, 8448, 128, 7424, 4096, 19456, 9728, 9984, 14336, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9344, 2432, 128, 13184, 16256, 3584, 2304, 5888, 20864, 18816, 14592, 128, 5632, 128, 2560, 15488, 512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[15104, 19328, 9088, 1024, 128, 128, 256, 20992, 128, 3584, 640, 2560, 14848, 26496, 14720, 256, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[20736, 128, 1920, 6144, 3072, 3968, 14592, 1152, 1280, 128, 15104, 24960, 128, 23808, 3456, 7040, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[7040, 9856, 128, 384, 5248, 4864, 4736, 768, 17152, 256, 15872, 11648, 1408, 18176, 2944, 128, 29056, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 479, 0, 2, 4091, 3776, 0, 3, 4079, 0, 0, 0, 0, 2, 0, 0, 4095, 0, 0, 4096, 4088, 19, 19, 0, 4074, 0, 4094, 1, 4083, 0, 0, 4090, 8, 4, 4095, 8, 4094, 0, 0, 1, 0, 1, 1, 4042, 0, 3, 840, 4084, 0, 0, 2004, 20, 48, 4091, 3038, 0, 10, 721, 0, 0, 4095, 5, 0, 4089, 0, 5, 3644, 3825, 1, 714, 5, 11, 9, 1, 0, 0, 7, 0, 0, 3952, 1, 4, 0, 6, 0, 0, 0, 0, 3, 3, 0, 4074, 0, 1007, 0, 0, 6, 4093, 4007, 4066, 0, 0, 0, 4091, 1, 9, 0, 0, 0, 1865, 4094, 22, 0, 4095, 0, 0, 579, 0, 4096, 0, 0, 2289, 0, 4, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1, 1, 0, 1545, 0, 0, 4094, 4090, 0, 0, 4094, 3, 0, 1, 0, 14, 0, 0, 4095, 1, 2, 4095, 4096, 4, 8, 1, 4073, 0, 4095, 64, 4090, 0, 0, 4095, 0, 5, 4095, 5, 4094, 1, 11, 0, 0, 0, 0, 4087, 0, 0, 1022, 4090, 0, 0, 1488, 9, 2, 4095, 2959, 0, 4, 813, 1, 0, 4095, 1, 0, 4070, 1, 1, 2739, 2872, 1, 300, 1, 1, 17, 0, 1, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 4095, 0, 929, 0, 1, 0, 4094, 4096, 3925, 0, 1, 0, 4093, 0, 8, 0, 1, 0, 837, 4095, 0, 1, 4093, 0, 0, 3441, 0, 4066, 0, 0, 1697, 1, 2, 1, 4053] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1465, 0, 0, 4096, 4095, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4093, 2, 80, 0, 4077, 0, 4094, 1, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 0, 0, 4084, 0, 0, 868, 4096, 0, 0, 1618, 1, 0, 4096, 2720, 0, 1, 301, 0, 0, 4096, 0, 0, 4091, 0, 0, 3096, 3631, 0, 807, 0, 2, 21, 0, 0, 0, 0, 0, 0, 3858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1744, 0, 0, 0, 4096, 4096, 3975, 0, 0, 0, 4096, 0, 41, 1, 0, 0, 665, 4096, 2, 0, 4087, 0, 0, 2031, 0, 4085, 7, 0, 1810, 0, 0, 0, 4082] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 589, 7, 0, 4096, 4096, 0, 0, 4094, 1, 0, 0, 1, 12, 0, 0, 4096, 0, 0, 4083, 4094, 0, 128, 0, 4094, 0, 4096, 58, 4096, 0, 0, 4096, 2, 0, 4096, 1, 4094, 0, 3, 0, 0, 0, 0, 4089, 0, 0, 690, 4095, 0, 0, 401, 11, 0, 4096, 3720, 0, 2, 427, 0, 0, 4096, 0, 0, 4091, 0, 0, 4011, 3610, 0, 14, 0, 2, 1, 0, 0, 0, 0, 0, 0, 3587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2454, 0, 0, 0, 4096, 4096, 4034, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 1213, 4096, 0, 0, 4096, 0, 2, 1145, 0, 4093, 0, 1, 2579, 0, 0, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 542, 0, 0, 4090, 4089, 0, 8, 4083, 1, 0, 0, 0, 2, 0, 6, 4092, 0, 0, 4089, 4081, 2, 74, 0, 4093, 4, 4096, 4, 4091, 0, 2, 4091, 1, 8, 4091, 2, 4095, 0, 3, 20, 0, 5, 0, 4003, 0, 1, 997, 4095, 0, 0, 378, 0, 0, 4095, 3810, 0, 11, 215, 0, 0, 4095, 0, 0, 4093, 0, 0, 3832, 3253, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3750, 2, 0, 1, 2, 0, 1, 0, 0, 0, 0, 0, 4096, 2, 2841, 4, 0, 0, 4096, 4059, 4090, 0, 0, 5, 4094, 0, 20, 0, 0, 0, 506, 4091, 1, 0, 4093, 0, 0, 2625, 0, 4096, 0, 0, 1716, 0, 0, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 415, 2, 0, 4095, 4073, 0, 0, 4090, 0, 0, 0, 1, 2, 1, 1, 4095, 0, 0, 4094, 4093, 0, 78, 0, 4096, 0, 4095, 377, 4096, 1, 6, 4095, 5, 1, 4096, 0, 4088, 0, 1, 0, 0, 5, 0, 4067, 0, 0, 1325, 4095, 0, 1, 211, 20, 1, 4096, 2939, 0, 0, 479, 1, 0, 4095, 1, 0, 4093, 0, 0, 3038, 3424, 0, 99, 0, 0, 16, 0, 1, 0, 0, 0, 0, 4007, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4094, 1, 1633, 1, 1, 0, 4095, 4090, 3795, 0, 0, 0, 4096, 0, 101, 0, 0, 0, 866, 4096, 0, 0, 4093, 0, 1, 2263, 0, 4093, 0, 0, 3666, 0, 1, 0, 4072] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[3, 11, 0, 531, 0, 1, 4072, 3929, 18, 2, 4082, 0, 5, 3, 0, 0, 0, 0, 4079, 3, 0, 4092, 4090, 0, 19, 1, 4040, 0, 4095, 44, 4085, 4, 1, 4083, 18, 4, 4074, 10, 4094, 0, 0, 0, 3, 14, 0, 4062, 8, 3, 1862, 4091, 0, 3, 1334, 0, 3, 4086, 3574, 1, 20, 2268, 0, 1, 4096, 1, 0, 4094, 2, 10, 3431, 3222, 10, 89, 5, 4, 8, 0, 2, 2, 0, 2, 11, 3770, 0, 0, 10, 0, 9, 29, 0, 8, 0, 0, 2, 4084, 15, 1507, 0, 4, 0, 4093, 4093, 4087, 0, 3, 16, 4078, 1, 57, 0, 2, 0, 527, 4081, 0, 4, 4083, 0, 0, 530, 10, 4093, 0, 0, 1977, 0, 9, 3, 4072] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 741, 0, 0, 4096, 4093, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4081, 4095, 1, 6, 0, 4095, 0, 4072, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 0, 0, 4086, 0, 0, 1393, 4096, 0, 0, 643, 63, 0, 4096, 3234, 0, 2, 481, 0, 0, 4096, 0, 0, 4095, 0, 0, 3766, 3616, 0, 39, 0, 1, 70, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 1717, 0, 0, 0, 4096, 4096, 4080, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 446, 4096, 1, 0, 4066, 0, 0, 2101, 0, 4096, 10, 0, 2424, 0, 0, 0, 4039] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 479, 0, 2, 4091, 3776, 0, 3, 4079, 0, 0, 0, 0, 2, 2027, 795, 479, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 19, 19, 0, 4074, 0, 0, 1, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 8, 4, 4095, 8, 4094, 0, 0, 1, 0, 1, 1, 4042, 0, 3, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 840, 4084, 0, 0, 2004, 20, 48, 4091, 3038, 0, 10, 721, 0, 0, 2068, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 5, 3644, 3825, 1, 714, 5, 11, 9, 1, 0, 0, 7, 0, 3610, 1511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3952, 1, 4, 0, 6, 0, 0, 0, 0, 3, 3, 0, 4074, 0, 1007, 4095, 2583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3298, 4007, 4066, 0, 0, 0, 4091, 1, 9, 0, 0, 0, 1865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 22, 0, 4095, 0, 0, 579, 0, 4096, 0, 0, 2289, 0, 4, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15818, 16273, 16369, 16929, 17432, 15728, 17343, 15180] avg=16384.0000 max=17432 max_rank=4 min=15180 min_rank=7 imbalance_ratio=1.063965 max_load_reduction=113640 max_load_reduction_pct=0.867004 moved_to_echo=65683 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 589, 7, 0, 4096, 4096, 0, 0, 4094, 1, 0, 0, 1, 12, 2025, 792, 475, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4094, 0, 128, 0, 4094, 0, 0, 58, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 1, 4094, 0, 3, 0, 0, 0, 0, 4089, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 690, 4095, 0, 0, 401, 11, 0, 4096, 3720, 0, 2, 427, 0, 0, 2071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4011, 3610, 0, 14, 0, 2, 1, 0, 0, 0, 0, 0, 3607, 1511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2454, 4096, 2585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3304, 4096, 4034, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 0, 2, 1145, 0, 4093, 0, 1, 2579, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16266, 16458, 16398, 15513, 16847, 16818, 16751, 16021] avg=16384.0000 max=16847 max_rank=4 min=15513 min_rank=3 imbalance_ratio=1.028259 max_load_reduction=114225 max_load_reduction_pct=0.871468 moved_to_echo=66437 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 1545, 0, 0, 4094, 4090, 0, 0, 4094, 3, 0, 1, 0, 14, 2025, 791, 475, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4095, 4096, 4, 8, 1, 4073, 0, 0, 64, 3995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 5, 4095, 5, 4094, 1, 11, 0, 0, 0, 0, 4087, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1022, 4090, 0, 0, 1488, 9, 2, 4095, 2959, 0, 4, 813, 1, 0, 2070, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 1, 1, 2739, 2872, 1, 300, 1, 1, 17, 0, 1, 0, 0, 0, 3606, 1510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 4095, 0, 929, 4095, 2585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3303, 4096, 3925, 0, 1, 0, 4093, 0, 8, 0, 1, 0, 837, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 1, 4093, 0, 0, 3441, 0, 4066, 0, 0, 1697, 1, 2, 1, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17212, 16339, 16410, 16554, 15120, 15803, 16265, 17369] avg=16384.0000 max=17369 max_rank=7 min=15120 min_rank=4 imbalance_ratio=1.060120 max_load_reduction=113703 max_load_reduction_pct=0.867485 moved_to_echo=64557 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1465, 0, 0, 4096, 4095, 0, 0, 4095, 2, 0, 0, 0, 0, 2025, 792, 475, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4093, 2, 80, 0, 4077, 0, 0, 1, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 0, 0, 4084, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 868, 4096, 0, 0, 1618, 1, 0, 4096, 2720, 0, 1, 301, 0, 0, 2071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 3096, 3631, 0, 807, 0, 2, 21, 0, 0, 0, 0, 0, 3607, 1510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1744, 4096, 2584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3304, 4096, 3975, 0, 0, 0, 4096, 0, 41, 1, 0, 0, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2, 0, 4087, 0, 0, 2031, 0, 4085, 7, 0, 1810, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17123, 16349, 16391, 15772, 16765, 16376, 16178, 16118] avg=16384.0000 max=17123 max_rank=0 min=15772 min_rank=3 imbalance_ratio=1.045105 max_load_reduction=113949 max_load_reduction_pct=0.869362 moved_to_echo=65437 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 415, 2, 0, 4095, 4073, 0, 0, 4090, 0, 0, 0, 1, 2, 2025, 792, 475, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4094, 4093, 0, 78, 0, 4096, 0, 0, 377, 4001, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4095, 5, 1, 4096, 0, 4088, 0, 1, 0, 0, 5, 0, 4067, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 4095, 0, 1, 211, 20, 1, 4096, 2939, 0, 0, 479, 1, 0, 2070, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3038, 3424, 0, 99, 0, 0, 16, 0, 1, 0, 0, 0, 3607, 1510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4094, 1, 1633, 4095, 2585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 3303, 4090, 3795, 0, 0, 0, 4096, 0, 101, 0, 0, 0, 866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4093, 0, 1, 2263, 0, 4093, 0, 0, 3666, 0, 1, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16048, 16742, 16381, 15239, 15788, 16418, 16253, 18203] avg=16384.0000 max=18203 max_rank=7 min=15239 min_rank=3 imbalance_ratio=1.111023 max_load_reduction=112869 max_load_reduction_pct=0.861122 moved_to_echo=66662 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 741, 0, 0, 4096, 4093, 0, 0, 4095, 1, 0, 0, 0, 0, 2025, 792, 475, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4095, 1, 6, 0, 4095, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 2, 0, 0, 0, 0, 4086, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1393, 4096, 0, 0, 643, 63, 0, 4096, 3234, 0, 2, 481, 0, 0, 2071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3766, 3616, 0, 39, 0, 1, 70, 0, 0, 0, 0, 0, 3607, 1502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 1717, 4096, 2570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3304, 4096, 4080, 0, 0, 0, 4096, 0, 0, 8, 0, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 0, 4066, 0, 0, 2101, 0, 4096, 10, 0, 2424, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16396, 16279, 16393, 16079, 16696, 16448, 16030, 16751] avg=16384.0000 max=16751 max_rank=7 min=16030 min_rank=6 imbalance_ratio=1.022400 max_load_reduction=114321 max_load_reduction_pct=0.872200 moved_to_echo=65925 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 11, 0, 531, 0, 1, 4072, 3929, 18, 2, 4082, 0, 5, 3, 0, 0, 2025, 791, 473, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4092, 4090, 0, 19, 1, 4040, 0, 0, 44, 3990, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4083, 18, 4, 4074, 10, 4094, 0, 0, 0, 3, 14, 0, 4062, 8, 3, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1862, 4091, 0, 3, 1334, 0, 3, 4086, 3574, 1, 20, 2268, 0, 1, 2071, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 10, 3431, 3222, 10, 89, 5, 4, 8, 0, 2, 2, 0, 2, 3594, 1511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3770, 0, 0, 10, 0, 9, 29, 0, 8, 0, 0, 2, 4084, 15, 1507, 4079, 2584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3302, 4093, 4087, 0, 3, 16, 4078, 1, 57, 0, 2, 0, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4, 4083, 0, 0, 530, 10, 4093, 0, 0, 1977, 0, 9, 3, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16024, 16283, 16391, 19315, 15986, 16108, 16170, 14795] avg=16384.0000 max=19315 max_rank=3 min=14795 min_rank=7 imbalance_ratio=1.178894 max_load_reduction=111757 max_load_reduction_pct=0.852638 moved_to_echo=63059 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 542, 0, 0, 4090, 4089, 0, 8, 4083, 1, 0, 0, 0, 2, 2025, 792, 475, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4089, 4081, 2, 74, 0, 4093, 4, 0, 4, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4091, 1, 8, 4091, 2, 4095, 0, 3, 20, 0, 5, 0, 4003, 0, 1, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 997, 4095, 0, 0, 378, 0, 0, 4095, 3810, 0, 11, 215, 0, 0, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3832, 3253, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 3603, 1511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3750, 2, 0, 1, 2, 0, 1, 0, 0, 0, 0, 0, 4096, 2, 2841, 4092, 2585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3304, 4059, 4090, 0, 0, 5, 4094, 0, 20, 0, 0, 0, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 0, 4093, 0, 0, 2625, 0, 4096, 0, 0, 1716, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16185, 16349, 16339, 15671, 16438, 17373, 16082, 16635] avg=16384.0000 max=17373 max_rank=5 min=15671 min_rank=3 imbalance_ratio=1.060364 max_load_reduction=113699 max_load_reduction_pct=0.867455 moved_to_echo=66528 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 26496, 32640, 32128, 0, 128, 128, 32768, 128, 256, 128, 128, 0, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[0, 32768, 128, 128, 27648, 27520, 128, 2304, 128, 128, 256, 128, 128, 128, 128, 128, 28928, 12160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 32768, 0, 128, 14720, 128, 32768, 128, 128, 18176, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 0, 6400, 128, 128, 32768, 32256, 128, 128, 32768, 128, 128, 128, 128, 128, 16256, 6400, 3840, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 32768, 128, 128, 32768, 128, 32768, 128, 128, 128, 128, 128, 128, 32640, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[9088, 32768, 0, 128, 8192, 128, 128, 32768, 26112, 128, 128, 5760, 128, 128, 16640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132992 observed_loss=-1920 tokens_per_expert=[128, 31104, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 13952, 32768, 20736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 128, 32768, 32768, 128, 512, 128, 32768, 128, 0, 640, 32000, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132992 slack_slots=129152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 4093, 0, 2, 4095, 0, 0, 3211, 0, 0, 4, 0, 0, 3846, 0, 4095, 1, 0, 1, 4092, 4066, 0, 4096, 0, 4096, 4095, 0, 0, 4095, 4094, 952, 0, 3256, 0, 4096, 0, 0, 43, 4094, 1, 1, 2525, 4096, 0, 0, 0, 0, 4095, 0, 1791, 0, 0, 0, 0, 2100, 465, 0, 0, 4094, 4094, 1, 0, 1, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 4096, 1, 0, 98, 4096, 0, 1, 157, 0, 0, 33, 0, 4095, 10, 0, 0, 0, 4086, 1, 4079, 2, 0, 0, 0, 1, 0, 4095, 1, 0, 0, 1, 1, 0, 0, 4096, 38, 0, 0, 2, 0, 0, 0, 4095, 0, 0, 0, 133, 1, 2735, 0, 1, 3237] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 4021, 8, 0, 4092, 1, 1, 2859, 0, 0, 2, 0, 1, 4003, 0, 4094, 1, 1, 1, 4076, 4085, 0, 4095, 1, 4095, 4095, 0, 0, 4061, 4094, 1634, 0, 3453, 0, 4095, 1, 0, 144, 4070, 1, 1, 3524, 4095, 1, 2, 1, 0, 4095, 0, 1346, 1, 0, 0, 0, 1746, 617, 0, 1, 4089, 4095, 0, 1, 0, 0, 0, 1, 4095, 0, 0, 1, 0, 0, 2, 4095, 1, 1, 4094, 0, 0, 62, 4095, 0, 1, 61, 1, 0, 2, 1, 4095, 7, 0, 0, 0, 4084, 1, 4075, 0, 0, 0, 1, 1, 0, 4088, 0, 1, 1, 1, 0, 0, 1, 4094, 135, 1, 1, 0, 0, 1, 1, 4093, 1, 1, 1, 72, 0, 2503, 0, 0, 2599] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 4054, 6, 0, 4096, 0, 0, 1791, 0, 0, 3, 0, 0, 4067, 0, 4096, 0, 1, 0, 4094, 4094, 0, 4096, 0, 4096, 4096, 0, 0, 4093, 4096, 2100, 0, 3098, 0, 4096, 0, 1, 13, 4080, 0, 0, 2051, 4096, 0, 0, 0, 0, 4095, 0, 1024, 0, 0, 0, 0, 1471, 907, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 8, 4096, 0, 0, 142, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 4087, 0, 4094, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 12, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 781, 0, 3689, 0, 0, 3494] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1, 4088, 98, 2, 4078, 3, 4, 1263, 12, 1, 18, 6, 4, 4033, 0, 4090, 5, 5, 2, 4053, 4024, 3, 4085, 7, 4089, 4091, 0, 0, 4093, 4080, 1267, 1, 3826, 11, 4085, 0, 6, 0, 4092, 4, 4, 2671, 4087, 5, 0, 4, 2, 4079, 8, 64, 4, 7, 0, 0, 2276, 1610, 0, 6, 4080, 4093, 0, 6, 0, 0, 0, 5, 4087, 8, 0, 2, 0, 0, 11, 4090, 0, 2, 4092, 0, 2, 31, 4070, 0, 3, 222, 7, 0, 31, 5, 4090, 0, 0, 1, 6, 4087, 6, 4056, 0, 16, 6, 5, 0, 0, 4082, 0, 9, 8, 2, 0, 0, 4, 4091, 2, 7, 5, 12, 0, 3, 8, 4088, 4, 0, 0, 659, 0, 3786, 0, 0, 2825] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[5, 4012, 2, 1, 4096, 0, 0, 1292, 0, 0, 300, 6, 0, 3802, 0, 4095, 0, 0, 0, 4063, 4064, 0, 4089, 0, 4089, 4095, 0, 0, 4081, 4090, 2107, 0, 3776, 12, 4095, 8, 0, 2, 4095, 0, 0, 2332, 4096, 0, 14, 0, 0, 4083, 0, 886, 0, 0, 0, 1, 1860, 392, 0, 0, 4095, 4085, 0, 0, 6, 0, 0, 0, 4089, 6, 2, 0, 0, 0, 23, 4096, 0, 0, 4087, 0, 6, 97, 4090, 1, 0, 324, 0, 4, 186, 0, 4095, 5, 0, 0, 0, 4055, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 6, 1, 0, 4096, 10, 0, 0, 9, 0, 0, 0, 4092, 0, 1, 0, 160, 7, 3328, 0, 0, 3883] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 4078, 1, 0, 4096, 0, 0, 2725, 0, 1, 15, 0, 0, 3886, 0, 4096, 0, 0, 0, 4091, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 4095, 4096, 1566, 0, 2569, 0, 4096, 0, 1, 33, 4078, 0, 0, 2742, 4096, 0, 0, 0, 0, 4096, 0, 1729, 0, 0, 0, 0, 2318, 221, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 11, 4096, 0, 0, 302, 0, 0, 5, 0, 4095, 1, 0, 0, 0, 3964, 0, 4091, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 20, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 74, 0, 3776, 0, 0, 2760] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[1, 4091, 0, 0, 4091, 0, 0, 1017, 1, 0, 23, 0, 0, 3768, 0, 4095, 0, 0, 0, 4084, 4048, 0, 4091, 0, 4096, 4096, 0, 0, 4089, 4093, 2281, 0, 3461, 0, 4096, 3, 0, 2, 4091, 0, 0, 3362, 4096, 0, 8, 0, 0, 4094, 0, 373, 0, 0, 0, 0, 1858, 721, 2, 0, 4087, 4082, 0, 0, 5, 0, 0, 0, 4096, 7, 0, 0, 0, 2, 10, 4096, 0, 0, 4096, 0, 7, 33, 4086, 0, 0, 532, 0, 0, 34, 0, 4094, 13, 0, 0, 0, 4053, 0, 4082, 3, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 230, 6, 3294, 0, 0, 3680] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 4087, 1, 0, 4096, 0, 0, 1680, 0, 0, 58, 0, 0, 3241, 0, 4096, 0, 0, 0, 4080, 4048, 0, 4096, 0, 4096, 4096, 0, 0, 4095, 4096, 2093, 0, 3759, 0, 4096, 0, 1, 5, 4089, 0, 0, 2492, 4096, 0, 0, 0, 0, 4096, 0, 70, 0, 0, 0, 0, 2739, 361, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 23, 4096, 0, 0, 96, 0, 0, 85, 0, 4095, 11, 0, 0, 0, 4025, 0, 4067, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 193, 0, 3867, 0, 0, 3987] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4093, 0, 2, 4095, 0, 0, 3211, 0, 0, 4, 0, 0, 3846, 0, 2342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4092, 4066, 0, 0, 0, 0, 0, 0, 0, 4095, 2399, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3256, 0, 2054, 0, 0, 43, 4094, 1, 1, 2525, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1791, 0, 0, 0, 0, 2100, 465, 0, 0, 4094, 4094, 1, 0, 1, 0, 2908, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 4096, 1, 0, 98, 1753, 1695, 372, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 157, 0, 0, 33, 0, 3873, 10, 0, 0, 0, 4086, 1, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 4095, 1, 0, 0, 1, 1, 0, 0, 4096, 38, 4095, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 0, 0, 0, 133, 1, 2735, 0, 1, 3237, 4096, 1188, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17593, 15606, 16069, 17124, 16430, 16336, 16395, 15519] avg=16384.0000 max=17593 max_rank=0 min=15519 min_rank=7 imbalance_ratio=1.073792 max_load_reduction=113479 max_load_reduction_pct=0.865776 moved_to_echo=64680 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4021, 8, 0, 4092, 1, 1, 2859, 0, 0, 2, 0, 1, 4003, 0, 2342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4076, 4085, 0, 0, 1, 0, 0, 0, 0, 4061, 2399, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3453, 0, 2054, 1, 0, 144, 4070, 1, 1, 3524, 0, 1, 2, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1346, 1, 0, 0, 0, 1746, 617, 0, 1, 4089, 4095, 0, 1, 0, 0, 2907, 1669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 1, 0, 0, 2, 4095, 1, 1, 4094, 0, 0, 62, 1752, 1695, 372, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 61, 1, 0, 2, 1, 3873, 7, 0, 0, 0, 4084, 1, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4088, 0, 1, 1, 1, 0, 0, 1, 4094, 135, 4095, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 4093, 1, 1, 1, 72, 0, 2503, 0, 0, 2599, 4095, 1188, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17330, 16259, 17347, 16472, 16393, 16201, 16482, 14588] avg=16384.0000 max=17347 max_rank=2 min=14588 min_rank=7 imbalance_ratio=1.058777 max_load_reduction=113725 max_load_reduction_pct=0.867653 moved_to_echo=63664 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4054, 6, 0, 4096, 0, 0, 1791, 0, 0, 3, 0, 0, 4067, 0, 2343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 4093, 2400, 2100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3098, 0, 2054, 0, 1, 13, 4080, 0, 0, 2051, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 0, 0, 0, 0, 1471, 907, 0, 0, 4095, 4096, 0, 0, 0, 0, 2909, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 8, 1753, 1696, 372, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 142, 0, 0, 0, 0, 3872, 2, 0, 0, 0, 4087, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 12, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 781, 0, 3689, 0, 0, 3494, 4096, 1187, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16360, 16782, 15392, 16172, 16339, 16293, 16360, 17374] avg=16384.0000 max=17374 max_rank=7 min=15392 min_rank=2 imbalance_ratio=1.060425 max_load_reduction=113698 max_load_reduction_pct=0.867447 moved_to_echo=66366 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[5, 4012, 2, 1, 4096, 0, 0, 1292, 0, 0, 300, 6, 0, 3802, 0, 2338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4064, 0, 0, 0, 0, 0, 0, 0, 4081, 2394, 2107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3776, 12, 2045, 8, 0, 2, 4095, 0, 0, 2332, 0, 0, 14, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 886, 0, 0, 0, 1, 1860, 392, 0, 0, 4095, 4085, 0, 0, 6, 0, 2904, 1672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 6, 2, 0, 0, 0, 23, 4096, 0, 0, 4087, 0, 6, 97, 1757, 1696, 378, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 324, 0, 4, 186, 0, 3869, 5, 0, 0, 0, 4055, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 6, 1, 0, 4096, 10, 4095, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4092, 0, 1, 0, 160, 7, 3328, 0, 0, 3883, 4089, 1185, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15854, 16709, 16367, 15901, 16463, 16623, 16370, 16785] avg=16384.0000 max=16785 max_rank=7 min=15854 min_rank=0 imbalance_ratio=1.024475 max_load_reduction=114287 max_load_reduction_pct=0.871941 moved_to_echo=66241 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 4088, 98, 2, 4078, 3, 4, 1263, 12, 1, 18, 6, 4, 4033, 0, 2339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 2, 4053, 4024, 3, 0, 7, 0, 0, 0, 0, 4093, 2391, 1267, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3826, 11, 2049, 0, 6, 0, 4092, 4, 4, 2671, 0, 5, 0, 4, 2, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 64, 4, 7, 0, 0, 2276, 1610, 0, 6, 4080, 4093, 0, 6, 0, 0, 2901, 1665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4087, 8, 0, 2, 0, 0, 11, 4090, 0, 2, 4092, 0, 2, 31, 1751, 1689, 371, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 3, 222, 7, 0, 31, 5, 3868, 0, 0, 1, 6, 4087, 6, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 6, 5, 0, 0, 4082, 0, 9, 8, 2, 0, 0, 4, 4091, 2, 4091, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 12, 0, 3, 8, 4088, 4, 0, 0, 659, 0, 3786, 0, 0, 2825, 4089, 1184, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15950, 15851, 16753, 16720, 16363, 16362, 16373, 16700] avg=16384.0000 max=16753 max_rank=2 min=15851 min_rank=1 imbalance_ratio=1.022522 max_load_reduction=114319 max_load_reduction_pct=0.872185 moved_to_echo=65798 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4078, 1, 0, 4096, 0, 0, 2725, 0, 1, 15, 0, 0, 3886, 0, 2343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 2400, 1566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2569, 0, 2054, 0, 1, 33, 4078, 0, 0, 2742, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1729, 0, 0, 0, 0, 2318, 221, 0, 0, 4096, 4096, 0, 0, 0, 0, 2908, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 11, 1753, 1696, 372, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 302, 0, 0, 5, 0, 3873, 1, 0, 0, 0, 3964, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 20, 4096, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 74, 0, 3776, 0, 0, 2760, 4096, 1188, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17145, 16248, 15573, 17038, 16342, 16332, 16370, 16024] avg=16384.0000 max=17145 max_rank=0 min=15573 min_rank=2 imbalance_ratio=1.046448 max_load_reduction=113927 max_load_reduction_pct=0.869194 moved_to_echo=65068 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 4087, 1, 0, 4096, 0, 0, 1680, 0, 0, 58, 0, 0, 3241, 0, 2343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4048, 0, 0, 0, 0, 0, 0, 0, 4095, 2400, 2093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3759, 0, 2054, 0, 1, 5, 4089, 0, 0, 2492, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 2739, 361, 0, 0, 4096, 4096, 0, 0, 0, 0, 2909, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 23, 1753, 1696, 372, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 96, 0, 0, 85, 0, 3873, 11, 0, 0, 0, 4025, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 193, 0, 3867, 0, 0, 3987, 4096, 1187, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15506, 16716, 16496, 15941, 16354, 16253, 16345, 17461] avg=16384.0000 max=17461 max_rank=7 min=15506 min_rank=0 imbalance_ratio=1.065735 max_load_reduction=113611 max_load_reduction_pct=0.866783 moved_to_echo=66413 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 4091, 0, 0, 4091, 0, 0, 1017, 1, 0, 23, 0, 0, 3768, 0, 2342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4048, 0, 0, 0, 0, 0, 0, 0, 4089, 2399, 2281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3461, 0, 2054, 3, 0, 2, 4091, 0, 0, 3362, 0, 0, 8, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 373, 0, 0, 0, 0, 1858, 721, 2, 0, 4087, 4082, 0, 0, 5, 0, 2906, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 2, 10, 4096, 0, 0, 4096, 0, 7, 33, 1753, 1694, 372, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 532, 0, 0, 34, 0, 3872, 13, 0, 0, 0, 4053, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 16, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 230, 6, 3294, 0, 0, 3680, 4096, 1185, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15334, 16901, 17075, 15704, 16388, 16672, 16377, 16621] avg=16384.0000 max=17075 max_rank=2 min=15334 min_rank=0 imbalance_ratio=1.042175 max_load_reduction=113997 max_load_reduction_pct=0.869728 moved_to_echo=66058 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32640, 128, 128, 32768, 128, 128, 15872, 128, 128, 512, 128, 128, 30720, 0, 18816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32640, 32640, 128, 0, 128, 0, 0, 0, 0, 32768, 19200, 14080, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[27264, 128, 16512, 128, 128, 256, 32768, 128, 128, 21760, 0, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 1920, 128, 128, 384, 128, 30976, 128, 0, 128, 128, 32512, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 0, 32768, 128, 128, 128, 128, 128, 128, 128, 32768, 256, 32768, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 0, 128, 128, 32768, 128, 128, 32768, 128, 128, 384, 14080, 13568, 3072, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 7296, 128, 128, 0, 128, 16384, 5376, 128, 128, 32768, 32768, 128, 128, 128, 0, 23296, 13440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 128, 128, 128, 2304, 128, 27008, 0, 128, 26496, 32768, 9600, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 1, 205, 3703, 4095, 3222, 0, 0, 0, 4095, 0, 4095, 10, 0, 4096, 0, 0, 1, 803, 0, 0, 0, 1, 0, 1994, 1, 0, 4094, 1, 0, 1, 0, 0, 4095, 0, 0, 4095, 4094, 4094, 0, 4096, 4096, 0, 4095, 4095, 4095, 0, 1, 4096, 0, 1480, 0, 1, 0, 0, 0, 3, 0, 4095, 0, 0, 0, 4095, 66, 1, 1, 4095, 4093, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 4095, 0, 0, 0, 0, 4096, 2626, 1, 3966, 0, 1, 3, 0, 0, 0, 0, 3975, 0, 1, 0, 0, 1, 0, 57, 1, 0, 0, 0, 8, 7, 0, 1, 0, 2461, 1, 0, 0, 0, 1, 4096, 4084, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 0, 110, 3678, 4095, 3397, 0, 0, 0, 4096, 0, 4078, 0, 0, 4096, 0, 0, 1, 531, 0, 0, 6, 0, 0, 764, 0, 0, 4096, 0, 5, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4069, 4096, 0, 4091, 4096, 4096, 0, 6, 4096, 0, 1927, 0, 0, 0, 12, 5, 1, 0, 4096, 0, 0, 0, 4096, 84, 0, 1, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 4091, 0, 0, 0, 0, 4095, 2358, 0, 3756, 0, 0, 0, 0, 0, 0, 4, 3953, 0, 0, 0, 14, 0, 0, 73, 0, 0, 0, 0, 45, 19, 0, 0, 0, 3917, 0, 0, 0, 1, 0, 4094, 4081, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 291, 3459, 4096, 3340, 0, 0, 0, 4096, 0, 4096, 7, 0, 4095, 0, 0, 0, 268, 0, 0, 0, 0, 0, 1757, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 4096, 4096, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 3371, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 20, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3852, 0, 3115, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 1, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 990, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 357, 4022, 4096, 993, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 360, 0, 0, 0, 0, 0, 2020, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 2885, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 243, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 3230, 0, 3895, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 2489, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 0, 1, 0, 1034, 3931, 4095, 1803, 0, 0, 1, 4094, 1, 4095, 25, 1, 4095, 1, 0, 0, 351, 0, 0, 0, 0, 1, 2712, 1, 0, 4095, 0, 0, 0, 1, 1, 4095, 0, 1, 4095, 4095, 4095, 0, 4093, 4095, 1, 4095, 4095, 4095, 1, 0, 4095, 0, 631, 0, 1, 1, 0, 1, 0, 2, 4095, 1, 0, 0, 4095, 266, 1, 0, 4095, 4086, 1, 1, 1, 0, 0, 0, 0, 4094, 0, 4095, 4095, 0, 0, 1, 0, 4095, 2244, 0, 3797, 0, 0, 0, 0, 1, 0, 0, 4071, 1, 0, 0, 0, 1, 1, 11, 0, 1, 1, 1, 18, 0, 1, 0, 1, 3701, 0, 0, 1, 0, 1, 4095, 4078, 0, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 0, 0, 15, 3741, 4096, 3152, 0, 0, 0, 4096, 0, 4079, 0, 0, 4096, 0, 0, 0, 256, 0, 0, 3, 0, 0, 137, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4082, 4096, 0, 4094, 4096, 4095, 0, 0, 4096, 0, 3664, 0, 0, 0, 3, 8, 12, 1, 4096, 0, 0, 0, 4096, 325, 0, 0, 4096, 4067, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 4095, 4090, 0, 0, 0, 0, 4096, 3788, 0, 3862, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 7, 0, 0, 41, 0, 0, 0, 0, 21, 0, 0, 0, 0, 1566, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[7, 0, 7, 8, 0, 3, 0, 293, 3855, 4090, 2978, 0, 1, 1, 4084, 7, 4077, 0, 7, 4087, 6, 0, 0, 726, 0, 0, 4, 1, 2, 1250, 4, 0, 4088, 0, 0, 7, 9, 1, 4090, 6, 0, 4089, 4089, 4089, 4, 4085, 4089, 8, 4088, 4086, 4086, 6, 0, 4089, 0, 1849, 0, 7, 4, 0, 11, 0, 14, 4085, 11, 0, 5, 4089, 76, 7, 0, 4089, 4076, 6, 7, 7, 0, 5, 0, 5, 4087, 0, 4092, 4089, 1, 0, 5, 1, 4088, 3601, 2, 2070, 4, 0, 0, 6, 5, 5, 0, 4041, 9, 0, 6, 0, 3, 5, 1, 0, 6, 4, 0, 16, 0, 7, 5, 2, 3772, 0, 0, 7, 0, 8, 4089, 4088, 0, 3, 4084, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 695, 3883, 4096, 2101, 0, 0, 0, 4096, 0, 4089, 0, 0, 4096, 0, 0, 0, 118, 0, 0, 0, 0, 0, 372, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 4095, 4096, 0, 4096, 4096, 4096, 0, 0, 4096, 0, 3390, 0, 0, 0, 0, 0, 16, 0, 4096, 0, 0, 0, 4096, 104, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 2992, 0, 3877, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 3, 0, 0, 2, 0, 0, 0, 0, 37, 4, 0, 0, 0, 2950, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 1, 205, 3703, 4095, 3222, 0, 0, 0, 4095, 0, 1039, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 10, 0, 4096, 0, 0, 1, 803, 0, 0, 0, 1, 0, 1994, 1, 0, 4090, 2299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 1, 0, 0, 0, 0, 0, 6, 4094, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1688, 4095, 0, 1, 0, 0, 1480, 0, 1, 0, 0, 0, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 66, 1, 1, 4095, 4093, 0, 0, 0, 0, 0, 0, 0, 2164, 1790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1565, 0, 4095, 4095, 0, 0, 0, 0, 0, 2626, 1, 3966, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 1, 0, 0, 1, 0, 57, 1, 0, 0, 0, 8, 4096, 4096, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 0, 2461, 1, 0, 0, 0, 1, 4096, 4084, 0, 0, 4096, 0, 1368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16730, 17390, 16386, 15458, 16305, 16352, 16336, 16115] avg=16384.0000 max=17390 max_rank=1 min=15458 min_rank=3 imbalance_ratio=1.061401 max_load_reduction=113682 max_load_reduction_pct=0.867325 moved_to_echo=65108 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 291, 3459, 4096, 3340, 0, 0, 0, 4096, 0, 1039, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 4095, 0, 0, 0, 268, 0, 0, 0, 0, 0, 1757, 0, 0, 4090, 2300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 6, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1689, 4096, 0, 0, 0, 0, 3371, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 20, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 2164, 1790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1565, 0, 4096, 4096, 0, 0, 0, 0, 0, 3852, 0, 3115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 1, 0, 0, 27, 0, 0, 0, 0, 0, 4096, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 990, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 4096, 0, 1368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16687, 16613, 16391, 17348, 16262, 16724, 16407, 14640] avg=16384.0000 max=17348 max_rank=3 min=14640 min_rank=7 imbalance_ratio=1.058838 max_load_reduction=113724 max_load_reduction_pct=0.867645 moved_to_echo=64033 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 0, 0, 110, 3678, 4095, 3397, 0, 0, 0, 4096, 0, 1043, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 4096, 0, 0, 1, 531, 0, 0, 6, 0, 0, 764, 0, 0, 4089, 2305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1679, 4096, 0, 6, 0, 0, 1927, 0, 0, 0, 12, 5, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 84, 0, 1, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 2169, 1791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1559, 0, 4090, 4091, 0, 0, 0, 0, 0, 2358, 0, 3756, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3953, 0, 0, 0, 14, 0, 0, 73, 0, 0, 0, 0, 45, 4096, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 3917, 0, 0, 0, 1, 0, 4094, 4081, 0, 0, 4096, 0, 1374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16792, 15870, 16362, 15913, 16316, 15854, 16383, 17582] avg=16384.0000 max=17582 max_rank=7 min=15854 min_rank=5 imbalance_ratio=1.073120 max_load_reduction=113490 max_load_reduction_pct=0.865860 moved_to_echo=66135 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 357, 4022, 4096, 993, 0, 0, 0, 4096, 0, 1039, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 360, 0, 0, 0, 0, 0, 2020, 0, 0, 4089, 2300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1689, 4096, 0, 0, 0, 0, 2885, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 243, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 2164, 1790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1566, 0, 4096, 4096, 0, 0, 0, 0, 0, 3230, 0, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 4, 4096, 4096, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2489, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 4096, 0, 1368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14969, 16961, 16390, 16864, 16481, 16883, 16381, 16143] avg=16384.0000 max=16961 max_rank=1 min=14969 min_rank=0 imbalance_ratio=1.035217 max_load_reduction=114111 max_load_reduction_pct=0.870598 moved_to_echo=65888 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 7, 8, 0, 3, 0, 293, 3855, 4090, 2978, 0, 1, 1, 4084, 7, 1037, 365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 7, 4087, 6, 0, 0, 726, 0, 0, 4, 1, 2, 1250, 4, 0, 4082, 2296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 7, 9, 1, 0, 6, 0, 6, 4089, 4089, 4, 4085, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1684, 4086, 6, 0, 0, 0, 1849, 0, 7, 4, 0, 11, 0, 14, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 5, 4089, 76, 7, 0, 4089, 4076, 6, 7, 7, 0, 5, 0, 5, 2159, 1787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1563, 0, 4092, 4089, 1, 0, 5, 1, 0, 3601, 2, 2070, 4, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 4041, 9, 0, 6, 0, 3, 5, 1, 0, 6, 4, 0, 16, 4089, 4089, 4090, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 5, 2, 3772, 0, 0, 7, 0, 8, 4089, 4088, 0, 3, 4084, 0, 1365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16736, 16542, 16392, 15834, 16329, 15434, 16375, 17430] avg=16384.0000 max=17430 max_rank=7 min=15434 min_rank=5 imbalance_ratio=1.063843 max_load_reduction=113642 max_load_reduction_pct=0.867020 moved_to_echo=65568 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 695, 3883, 4096, 2101, 0, 0, 0, 4096, 0, 1039, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 118, 0, 0, 0, 0, 0, 372, 0, 0, 4090, 2300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1689, 4096, 0, 0, 0, 0, 3390, 0, 0, 0, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 104, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 2164, 1790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1566, 0, 4096, 4096, 0, 0, 0, 0, 0, 2992, 0, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 3, 0, 0, 2, 0, 0, 0, 0, 37, 4096, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2950, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 1368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16276, 15065, 16389, 17383, 16336, 16627, 16387, 16609] avg=16384.0000 max=17383 max_rank=3 min=15065 min_rank=1 imbalance_ratio=1.060974 max_load_reduction=113689 max_load_reduction_pct=0.867378 moved_to_echo=65959 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 0, 1, 0, 1034, 3931, 4095, 1803, 0, 0, 1, 4094, 1, 1039, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 25, 1, 4095, 1, 0, 0, 351, 0, 0, 0, 0, 1, 2712, 1, 0, 4089, 2299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 1, 0, 0, 1, 6, 4095, 4095, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1688, 4095, 1, 0, 0, 0, 631, 0, 1, 1, 0, 1, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 266, 1, 0, 4095, 4086, 1, 1, 1, 0, 0, 0, 0, 2163, 1790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1565, 0, 4095, 4095, 0, 0, 1, 0, 0, 2244, 0, 3797, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4071, 1, 0, 0, 0, 1, 1, 11, 0, 1, 1, 1, 18, 4095, 4095, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3701, 0, 0, 1, 0, 1, 4095, 4078, 0, 0, 4095, 0, 1368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16368, 17670, 16388, 14610, 16500, 15797, 16398, 17341] avg=16384.0000 max=17670 max_rank=1 min=14610 min_rank=3 imbalance_ratio=1.078491 max_load_reduction=113402 max_load_reduction_pct=0.865189 moved_to_echo=66036 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 0, 0, 15, 3741, 4096, 3152, 0, 0, 0, 4096, 0, 1039, 366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 4096, 0, 0, 0, 256, 0, 0, 3, 0, 0, 137, 0, 0, 4090, 2300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1689, 4095, 0, 0, 0, 0, 3664, 0, 0, 0, 3, 8, 12, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 325, 0, 0, 4096, 4067, 0, 0, 0, 0, 3, 0, 0, 2164, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1566, 0, 4095, 4090, 0, 0, 0, 0, 0, 3788, 0, 3862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 7, 0, 0, 41, 0, 0, 0, 0, 21, 4096, 4096, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1566, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 4096, 0, 1368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16514, 14961, 16374, 17662, 16543, 17401, 16405, 15212] avg=16384.0000 max=17662 max_rank=3 min=14961 min_rank=1 imbalance_ratio=1.078003 max_load_reduction=113410 max_load_reduction_pct=0.865250 moved_to_echo=65561 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 32768, 128, 0, 128, 3456, 0, 0, 128, 128, 128, 11008, 128, 0, 32768, 18432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132992 observed_loss=-1920 tokens_per_expert=[128, 128, 128, 32384, 128, 128, 128, 128, 128, 128, 256, 128, 128, 128, 128, 256, 32768, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 128, 128, 128, 32768, 32768, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 13568, 32768, 128, 128, 0, 0, 19200, 0, 128, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 21888, 128, 0, 128, 128, 128, 32768, 32768, 0, 128, 32768, 0, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[12544, 0, 32768, 32768, 128, 0, 128, 128, 0, 24704, 128, 28416, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 32768, 1280, 128, 128, 32768, 32768, 128, 128, 128, 0, 128, 0, 128, 17408, 14336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 128, 128, 128, 3072, 30336, 32768, 20992, 0, 128, 128, 32768, 128, 8320, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132992 slack_slots=129152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 4096, 0, 4, 0, 4096, 0, 0, 0, 0, 3609, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 4096, 12, 0, 0, 4096, 4096, 4096, 0, 4096, 1, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 933, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 2157, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 13, 0, 0, 0, 10, 821, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 4084, 0, 10, 0, 4096, 0, 0, 0, 0, 3136, 4090, 0, 0, 0, 1, 19, 0, 0, 0, 4096, 8, 0, 0, 4096, 28, 0, 0, 4096, 4096, 4096, 0, 4095, 7, 4096, 4096, 4082, 4090, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1409, 0, 4096, 4072, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 6, 36, 4090, 4060, 0, 0, 4096, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 4096, 0, 8, 2801, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 1, 0, 0, 0, 0, 1, 2, 1503, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 4095, 4095, 4095, 0, 1, 1, 4095, 0, 0, 0, 1, 2485, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 0, 4095, 70, 1, 1, 4095, 4095, 4095, 0, 4095, 0, 4095, 4090, 4094, 4089, 0, 0, 0, 1, 0, 4086, 0, 1, 0, 2142, 0, 4093, 4088, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 4095, 1, 0, 0, 4095, 4095, 1, 1, 4095, 0, 1, 0, 0, 4095, 1, 1, 0, 1, 0, 4084, 1, 3, 2024, 1, 0, 0, 4095, 0, 4095, 0, 0, 2, 4094, 4095, 1, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 727, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4096, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 2574, 4096, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 0, 4096, 33, 0, 0, 4092, 4096, 4096, 0, 4096, 7, 4096, 4096, 4058, 4096, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 1403, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 43, 4085, 4066, 0, 0, 4096, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 4096, 0, 18, 3490, 0, 0, 0, 4096, 5, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4, 0, 4096, 0, 0, 0, 0, 2352, 4096, 1, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 33, 0, 0, 4096, 4096, 4096, 0, 4095, 0, 4096, 4095, 4096, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2778, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1909, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4096, 4093, 0, 11, 0, 4096, 0, 0, 0, 0, 3558, 4096, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 4096, 8, 0, 0, 4096, 4096, 4096, 0, 4096, 5, 4096, 4096, 4096, 4090, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 921, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 4096, 4082, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 7, 2021, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[3974, 2, 0, 0, 5, 0, 0, 6, 0, 6, 6, 174, 1, 7, 4, 0, 3, 8, 0, 7, 0, 0, 8, 7, 0, 9, 8, 4, 0, 4088, 4087, 4091, 0, 12, 4, 4088, 0, 0, 0, 9, 2592, 4085, 1, 10, 5, 0, 5, 0, 0, 8, 4083, 10, 5, 0, 4088, 92, 13, 8, 4078, 4088, 4088, 0, 4086, 0, 4085, 4052, 4070, 4084, 2, 3, 0, 4, 4, 4072, 0, 8, 0, 3078, 1, 4078, 4076, 0, 7, 8, 5, 0, 8, 0, 0, 5, 8, 4087, 8, 1, 15, 4078, 4087, 7, 7, 4088, 0, 3, 1, 0, 4089, 4, 10, 0, 6, 0, 4084, 6, 16, 2429, 4, 1, 0, 4088, 0, 4082, 0, 0, 6, 4086, 4087, 1, 1, 4069] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1947, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 4096, 4096, 1, 0, 0, 4096, 3, 0, 1, 0, 2843, 4095, 1, 0, 0, 0, 20, 0, 0, 0, 4095, 1, 0, 0, 4095, 19, 0, 0, 4095, 4095, 4096, 0, 4094, 6, 4095, 4095, 4082, 4088, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 798, 0, 4096, 4091, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 1, 16, 4096, 4096, 0, 0, 4095, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 4096, 0, 5, 2577, 0, 0, 0, 4096, 0, 4096, 0, 1, 0, 4096, 4096, 0, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 13, 0, 0, 0, 10, 821, 0, 0, 0, 0, 4096, 4096, 2926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 4084, 4079, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 3136, 4090, 0, 0, 0, 1, 19, 0, 4096, 1170, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 4047, 28, 0, 0, 4096, 0, 0, 0, 4095, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4082, 2434, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 6, 36, 4090, 2405, 1656, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 4096, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 2801, 0, 0, 0, 1661, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16058, 16371, 16635, 16377, 16114, 16396, 16363, 16758] avg=16384.0000 max=16758 max_rank=7 min=16058 min_rank=0 imbalance_ratio=1.022827 max_load_reduction=114314 max_load_reduction_pct=0.872147 moved_to_echo=65631 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 1, 0, 0, 0, 0, 1, 2, 1503, 1, 0, 0, 0, 4095, 4095, 2924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 4095, 4095, 4095, 4075, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 1, 2485, 4095, 0, 0, 0, 0, 0, 0, 4095, 1171, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 0, 4057, 70, 1, 1, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4094, 2437, 0, 0, 0, 1, 0, 4086, 0, 1, 0, 2142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 4095, 1, 0, 0, 4095, 2400, 1652, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 4095, 0, 1, 0, 0, 4095, 1, 1, 0, 1, 0, 4084, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2024, 1, 0, 0, 1668, 0, 4095, 0, 0, 2, 4094, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16716, 16392, 15962, 16417, 16851, 16367, 16385, 15982] avg=16384.0000 max=16851 max_rank=4 min=15962 min_rank=2 imbalance_ratio=1.028503 max_load_reduction=114221 max_load_reduction_pct=0.871437 moved_to_echo=65585 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1116, 0, 0, 0, 0, 4096, 4096, 2926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4078, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 2352, 4096, 1, 0, 1, 0, 0, 0, 4096, 1170, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4058, 33, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 2439, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 4096, 2399, 1654, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1909, 0, 0, 0, 1668, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16330, 16393, 15834, 16378, 17504, 16375, 16393, 15865] avg=16384.0000 max=17504 max_rank=4 min=15834 min_rank=2 imbalance_ratio=1.068359 max_load_reduction=113568 max_load_reduction_pct=0.866455 moved_to_echo=66137 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1681, 0, 0, 0, 0, 4096, 4096, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4096, 4093, 4078, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4096, 0, 0, 0, 0, 3558, 4096, 0, 0, 0, 0, 11, 0, 4096, 1171, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4058, 8, 0, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2438, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 4096, 2400, 1652, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 2021, 0, 0, 0, 1669, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16894, 16368, 17057, 16359, 15638, 16392, 16379, 15985] avg=16384.0000 max=17057 max_rank=2 min=15638 min_rank=4 imbalance_ratio=1.041077 max_load_reduction=114015 max_load_reduction_pct=0.869865 moved_to_echo=64394 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1472, 0, 0, 0, 0, 4096, 4096, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 4096, 4078, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 3609, 4096, 0, 0, 0, 0, 6, 0, 4096, 1171, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4058, 12, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2441, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 4096, 2400, 1655, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2157, 0, 0, 0, 1669, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16685, 16388, 17096, 16359, 15662, 16375, 16393, 16114] avg=16384.0000 max=17096 max_rank=2 min=15662 min_rank=4 imbalance_ratio=1.043457 max_load_reduction=113976 max_load_reduction_pct=0.869568 moved_to_echo=64544 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1947, 0, 0, 0, 0, 4096, 4095, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 4096, 4096, 4078, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 3, 0, 1, 0, 2843, 4095, 1, 0, 0, 0, 20, 0, 4095, 1171, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4057, 19, 0, 0, 4095, 0, 0, 0, 4094, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4082, 2436, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 798, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 1, 16, 4096, 2400, 1652, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2577, 0, 0, 0, 1669, 0, 4096, 0, 1, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17161, 16389, 16344, 16367, 15498, 16383, 16392, 16538] avg=16384.0000 max=17161 max_rank=0 min=15498 min_rank=4 imbalance_ratio=1.047424 max_load_reduction=113911 max_load_reduction_pct=0.869072 moved_to_echo=64811 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3974, 2, 0, 0, 5, 0, 0, 6, 0, 6, 6, 174, 1, 7, 4, 0, 4088, 4088, 2920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 0, 7, 0, 0, 8, 7, 0, 9, 8, 4, 0, 4088, 4087, 4091, 4060, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4, 4088, 0, 0, 0, 9, 2592, 4085, 1, 10, 5, 0, 5, 0, 4085, 1167, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4083, 10, 5, 0, 4050, 92, 13, 8, 4078, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 4070, 2434, 2, 3, 0, 4, 4, 4072, 0, 8, 0, 3078, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 7, 8, 5, 0, 8, 0, 0, 5, 8, 4087, 8, 1, 15, 4078, 2396, 1650, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 7, 7, 4088, 0, 3, 1, 0, 4089, 4, 10, 0, 6, 0, 4084, 6, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2429, 4, 1, 0, 1665, 0, 4082, 0, 0, 6, 4086, 0, 1, 1, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15281, 16407, 16081, 16433, 17728, 16381, 16401, 16360] avg=16384.0000 max=17728 max_rank=4 min=15281 min_rank=0 imbalance_ratio=1.082031 max_load_reduction=113344 max_load_reduction_pct=0.864746 moved_to_echo=66870 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 727, 0, 0, 0, 0, 4096, 4096, 2926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4096, 4086, 4078, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2574, 4096, 0, 0, 0, 0, 13, 0, 4096, 1170, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4058, 33, 0, 0, 4092, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4058, 2441, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 1403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 43, 4085, 2400, 1655, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 4096, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 3490, 0, 0, 0, 1669, 5, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15947, 16364, 16063, 16382, 16077, 16403, 16366, 17470] avg=16384.0000 max=17470 max_rank=7 min=15947 min_rank=0 imbalance_ratio=1.066284 max_load_reduction=113602 max_load_reduction_pct=0.866714 moved_to_echo=66316 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 32768, 19584, 128, 128, 0, 128, 128, 32768, 0, 128, 0, 13568, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 0, 0, 128, 0, 128, 128, 0, 128, 128, 9472, 128, 128, 128, 0, 32768, 32768, 23424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 32768, 0, 128, 128, 128, 32768, 128, 128, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 32768, 128, 0, 128, 128, 23168, 32768, 128, 128, 128, 128, 128, 0, 32768, 9472, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 128, 0, 128, 128, 128, 32768, 128, 128, 256, 32768, 19200, 13312, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 19456, 128, 128, 0, 13440, 128, 32768, 0, 128, 128, 32768, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 128, 128, 0, 32512, 384, 128, 128, 32768, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 128, 0, 128, 128, 128, 0, 32768, 32768, 32768, 32640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 98, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 3919, 0, 0, 0, 0, 4096, 4078, 698, 0, 0, 0, 4096, 4096, 4096, 0, 2159, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 183, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 1113, 0, 0, 0, 26, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4095, 65, 0, 1, 1, 1, 4095, 1, 4095, 1, 0, 6, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4095, 0, 1, 4095, 0, 1, 4093, 0, 4095, 4095, 4095, 4095, 0, 3806, 0, 0, 1, 0, 4095, 4075, 1743, 0, 2, 0, 4094, 4095, 4095, 0, 409, 1, 4091, 1, 0, 0, 1, 1, 0, 0, 1, 0, 4095, 0, 0, 402, 0, 1, 13, 1, 1, 0, 4095, 1, 1, 1, 4095, 1779, 3, 1, 0, 11, 0, 0, 0, 4095, 0, 1, 4095, 4095, 1, 0, 1, 0, 4095, 4094, 1, 0, 0, 1, 0, 0, 5, 0, 4095, 0, 0, 1, 4094, 0, 0, 4095, 1, 4068, 1, 0, 1, 1, 1, 0, 0, 4095, 1, 0, 4093, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4096, 35, 0, 0, 0, 0, 4090, 0, 4094, 0, 0, 7, 0, 0, 0, 0, 0, 9, 0, 2, 0, 0, 0, 4096, 4, 0, 4096, 27, 0, 4071, 0, 4095, 4096, 4096, 4096, 0, 4085, 7, 3, 0, 0, 4096, 3667, 575, 0, 7, 0, 4094, 4096, 4096, 9, 1091, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 1838, 0, 0, 21, 0, 0, 0, 4071, 0, 0, 0, 4096, 977, 0, 0, 6, 314, 0, 0, 0, 4088, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4085, 0, 0, 4096, 0, 3938, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4091, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 158, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4095, 4096, 4096, 4096, 0, 3988, 0, 0, 0, 0, 4096, 4068, 524, 0, 0, 0, 4096, 4096, 4096, 0, 1378, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 236, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 1946, 0, 0, 0, 90, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 3989, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4096, 33, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 16, 0, 0, 0, 0, 0, 7, 0, 8, 0, 0, 0, 4096, 4, 0, 4096, 0, 0, 4082, 0, 4096, 4096, 4090, 4096, 0, 3856, 5, 0, 0, 0, 4096, 3584, 1489, 0, 6, 0, 4096, 4096, 4096, 14, 791, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 528, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 2019, 0, 0, 7, 108, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4091, 0, 0, 4096, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4086, 71, 5, 9, 9, 8, 4085, 4, 4087, 9, 0, 45, 1, 0, 0, 0, 0, 0, 7, 0, 9, 3, 0, 4087, 0, 8, 4081, 0, 7, 4060, 0, 4088, 4088, 4084, 4087, 0, 4060, 0, 0, 9, 0, 4086, 3676, 426, 0, 9, 5, 4087, 4086, 4087, 8, 2494, 0, 3867, 8, 0, 0, 9, 9, 2, 6, 9, 0, 4082, 0, 0, 610, 5, 8, 31, 7, 8, 0, 4086, 5, 6, 7, 4078, 1241, 48, 5, 0, 73, 0, 3, 0, 4089, 0, 4, 4086, 4087, 9, 0, 7, 0, 4086, 4086, 8, 0, 8, 3, 0, 0, 0, 0, 4085, 0, 0, 6, 4088, 5, 0, 4087, 3, 3827, 9, 0, 8, 12, 9, 0, 3, 4084, 3, 0, 4087, 2, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 114, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 24, 0, 3, 0, 0, 0, 3, 0, 1, 0, 0, 0, 4095, 1, 0, 4096, 0, 0, 4087, 0, 4096, 4096, 4096, 4096, 0, 3922, 0, 1, 0, 0, 4095, 3992, 1625, 0, 2, 1, 4095, 4095, 4095, 5, 1205, 0, 4060, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 246, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 0, 4096, 1138, 0, 0, 0, 76, 0, 0, 1, 4095, 1, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4094, 0, 1, 0, 1, 1, 0, 1, 0, 4095, 0, 0, 0, 4093, 0, 0, 4095, 0, 4082, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 1, 4094, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 116, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 48, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4096, 0, 4096, 4096, 4096, 4096, 0, 4035, 0, 0, 0, 0, 4096, 3359, 300, 0, 0, 0, 4096, 4096, 4096, 13, 2407, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1617, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 4096, 450, 0, 0, 0, 206, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 4094, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4095, 65, 0, 1, 1, 1, 4095, 1, 4095, 1, 0, 6, 1, 0, 0, 0, 2042, 1416, 504, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4095, 0, 1, 4095, 0, 1, 4093, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3806, 0, 0, 1, 0, 0, 4075, 1743, 0, 2, 0, 3590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2679, 0, 409, 1, 4091, 1, 0, 0, 1, 1, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 1, 13, 1, 1, 0, 4095, 1, 1, 1, 4095, 1779, 3, 4095, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 0, 4095, 0, 1, 4095, 3983, 1, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 1, 0, 0, 5, 0, 4095, 0, 0, 1, 4094, 0, 0, 2053, 2049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4068, 1, 0, 1, 1, 1, 0, 0, 4095, 1, 0, 4093, 0, 1, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16347, 16384, 17312, 15374, 16534, 16283, 16393, 16445] avg=16384.0000 max=17312 max_rank=2 min=15374 min_rank=3 imbalance_ratio=1.056641 max_load_reduction=113760 max_load_reduction_pct=0.867920 moved_to_echo=65655 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 35, 0, 0, 0, 0, 4090, 0, 4094, 0, 0, 7, 0, 0, 0, 0, 2044, 1423, 508, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2, 0, 0, 0, 4096, 4, 0, 4096, 27, 0, 4071, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 7, 3, 0, 0, 0, 3667, 575, 0, 7, 0, 3586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2673, 9, 1091, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1838, 0, 0, 21, 0, 0, 0, 4071, 0, 0, 0, 4096, 977, 0, 4096, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 314, 0, 0, 0, 4088, 0, 0, 4096, 3978, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 4085, 0, 0, 2052, 2050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3938, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4091, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16321, 16406, 16026, 15920, 17145, 16577, 16367, 16310] avg=16384.0000 max=17145 max_rank=4 min=15920 min_rank=3 imbalance_ratio=1.046448 max_load_reduction=113927 max_load_reduction_pct=0.869194 moved_to_echo=66399 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 158, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 8, 0, 0, 0, 0, 2043, 1416, 505, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3988, 0, 0, 0, 0, 0, 4068, 524, 0, 0, 0, 3591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2680, 0, 1378, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 1946, 0, 4096, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 4096, 0, 0, 4096, 3984, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 4096, 0, 0, 2053, 2050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3989, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16441, 16385, 16267, 16341, 16517, 16362, 16394, 16365] avg=16384.0000 max=16517 max_rank=4 min=16267 min_rank=2 imbalance_ratio=1.008118 max_load_reduction=114555 max_load_reduction_pct=0.873985 moved_to_echo=65638 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 98, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 22, 0, 0, 0, 0, 2043, 1416, 505, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3919, 0, 0, 0, 0, 0, 4078, 698, 0, 0, 0, 3591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2680, 0, 2159, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 1113, 0, 4096, 2045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 4096, 0, 0, 4096, 3984, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 0, 0, 2053, 2051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16395, 16386, 16382, 17121, 15628, 16298, 16394, 16468] avg=16384.0000 max=17121 max_rank=3 min=15628 min_rank=4 imbalance_ratio=1.044983 max_load_reduction=113951 max_load_reduction_pct=0.869377 moved_to_echo=64788 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 114, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 24, 0, 3, 0, 0, 2042, 1416, 504, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 4095, 1, 0, 4096, 0, 0, 4087, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3922, 0, 1, 0, 0, 0, 3992, 1625, 0, 2, 1, 3591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2679, 5, 1205, 0, 4060, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 1, 0, 0, 0, 4093, 0, 0, 0, 4096, 1138, 0, 4096, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 1, 4095, 1, 0, 4096, 3984, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 1, 1, 0, 1, 0, 4095, 0, 0, 0, 4093, 0, 0, 2053, 2050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4082, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 1, 4094, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 16381, 17230, 16139, 15716, 16349, 16389, 16456] avg=16384.0000 max=17230 max_rank=2 min=15716 min_rank=4 imbalance_ratio=1.051636 max_load_reduction=113842 max_load_reduction_pct=0.868546 moved_to_echo=64910 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4086, 71, 5, 9, 9, 8, 4085, 4, 4087, 9, 0, 45, 1, 0, 0, 0, 2038, 1413, 503, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 9, 3, 0, 4087, 0, 8, 4081, 0, 7, 4060, 0, 4088, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 4060, 0, 0, 9, 0, 0, 3676, 426, 0, 9, 5, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 2674, 8, 2494, 0, 3867, 8, 0, 0, 9, 9, 2, 6, 9, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 5, 8, 31, 7, 8, 0, 4086, 5, 6, 7, 4078, 1241, 48, 4088, 2042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 73, 0, 3, 0, 4089, 0, 4, 4086, 3976, 9, 0, 7, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 8, 0, 8, 3, 0, 0, 0, 0, 4085, 0, 0, 6, 4088, 5, 0, 2048, 2045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 3, 3827, 9, 0, 8, 12, 9, 0, 3, 4084, 3, 0, 4087, 2, 7, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16396, 16352, 15853, 17254, 16270, 16338, 16382, 16227] avg=16384.0000 max=17254 max_rank=3 min=15853 min_rank=2 imbalance_ratio=1.053101 max_load_reduction=113818 max_load_reduction_pct=0.868362 moved_to_echo=65217 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 116, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 48, 0, 1, 0, 0, 2043, 1416, 505, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4035, 0, 0, 0, 0, 0, 3359, 300, 0, 0, 0, 3591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2680, 13, 2407, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1617, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 4096, 450, 0, 4096, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 0, 0, 4096, 0, 0, 4096, 3984, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 2053, 2050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16440, 16387, 15381, 17226, 16382, 16478, 16386, 16392] avg=16384.0000 max=17226 max_rank=3 min=15381 min_rank=2 imbalance_ratio=1.051392 max_load_reduction=113846 max_load_reduction_pct=0.868576 moved_to_echo=65638 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 33, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 16, 0, 0, 0, 0, 2044, 1416, 505, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8, 0, 0, 0, 4096, 4, 0, 4096, 0, 0, 4082, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 3856, 5, 0, 0, 0, 0, 3584, 1489, 0, 6, 0, 3591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2680, 14, 791, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 2019, 0, 4096, 2046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 108, 0, 0, 0, 4096, 0, 0, 4096, 3984, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4091, 0, 0, 2052, 2050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16320, 16391, 16621, 15697, 16880, 16387, 16367, 16409] avg=16384.0000 max=16880 max_rank=4 min=15697 min_rank=3 imbalance_ratio=1.030273 max_load_reduction=114192 max_load_reduction_pct=0.871216 moved_to_echo=66043 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 0, 128, 0, 32768, 0, 0, 128, 32768, 128, 0, 16512, 16512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 21504, 128, 12032, 128, 32128, 128, 0, 0, 128, 128, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 0, 0, 31744, 128, 128, 128, 0, 0, 30592, 7424, 0, 128, 128, 28800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 768, 128, 128, 128, 128, 32768, 128, 32768, 128, 0, 256, 128, 128, 0, 0, 16384, 11392, 4096, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 32768, 128, 128, 32768, 128, 128, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 32128, 128, 0, 128, 128, 128, 0, 128, 32768, 128, 128, 32768, 128, 128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 5760, 128, 128, 128, 128, 128, 0, 32768, 128, 128, 128, 32768, 10752, 128, 32768, 16384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 1024, 0, 128, 128, 32768, 128, 128, 32768, 31872, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3576, 0, 1, 4096, 0, 0, 0, 0, 1, 2, 1, 0, 4096, 0, 0, 0, 4094, 0, 0, 4096, 4088, 0, 4096, 4096, 4096, 1, 0, 0, 0, 0, 0, 4091, 4096, 371, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 1, 0, 3248, 2918, 0, 0, 4096, 11, 0, 284, 4096, 0, 5, 0, 4096, 4095, 4095, 1783, 0, 4096, 0, 1, 0, 4095, 0, 136, 0, 0, 4094, 4055, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[8, 0, 0, 4087, 9, 0, 9, 0, 4087, 0, 9, 0, 7, 3, 2, 0, 1, 0, 7, 4086, 8, 9, 4084, 9, 9, 9, 3937, 0, 1, 4086, 9, 1, 9, 0, 0, 0, 0, 8, 4085, 0, 9, 2, 4084, 0, 9, 3684, 4089, 9, 4087, 4087, 4082, 0, 0, 10, 0, 0, 6, 4084, 4084, 155, 4087, 36, 0, 0, 4085, 0, 0, 0, 8, 4086, 0, 0, 0, 4039, 1962, 9, 11, 4075, 0, 2, 12, 4082, 0, 0, 0, 4086, 4086, 4087, 1295, 4, 4079, 7, 0, 7, 4078, 6, 1177, 1, 7, 4088, 4088, 4084, 2, 7, 0, 0, 7, 9, 8, 0, 61, 9, 3, 4082, 4084, 3, 4, 9, 4, 5, 8, 11, 7, 15, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3940, 0, 0, 4096, 0, 0, 0, 0, 2, 9, 0, 0, 4092, 0, 0, 0, 4085, 0, 0, 4096, 4092, 0, 4096, 4088, 4096, 0, 0, 0, 0, 0, 0, 4056, 4096, 219, 4087, 0, 0, 0, 4096, 0, 0, 7, 0, 4094, 0, 0, 0, 3482, 2217, 0, 0, 4089, 90, 0, 18, 4096, 0, 8, 0, 4096, 4096, 4091, 1343, 0, 4096, 0, 6, 0, 4096, 0, 1126, 0, 0, 4027, 3994, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 10, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 1, 3, 1, 0, 4095, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 4095, 1, 1, 4095, 1, 1, 1, 3747, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 4095, 0, 1, 4079, 4095, 1, 4095, 4093, 4094, 0, 0, 1, 1, 0, 0, 4095, 4095, 517, 4094, 3, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 3953, 2241, 0, 1, 4095, 1, 1, 25, 4095, 0, 0, 0, 4095, 4095, 4095, 1451, 1, 4095, 0, 0, 1, 4093, 1, 359, 0, 1, 4095, 4095, 4095, 1, 1, 0, 0, 1, 1, 0, 0, 5, 2, 0, 4095, 4095, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3916, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 4094, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4091, 4096, 1130, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3587, 1757, 0, 0, 4096, 5, 0, 38, 4096, 0, 0, 0, 4096, 4096, 4096, 1708, 0, 4096, 0, 0, 0, 4096, 0, 158, 0, 0, 4094, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3718, 2, 0, 4096, 0, 0, 0, 0, 5, 2, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 4095, 4092, 0, 4096, 4084, 4096, 0, 0, 0, 0, 0, 0, 4088, 4096, 160, 4091, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 3527, 1515, 0, 0, 4089, 66, 0, 22, 4096, 0, 6, 6, 4096, 4096, 4096, 2183, 0, 4096, 0, 2, 0, 4096, 0, 1169, 0, 0, 4040, 4056, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 24, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3618, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4094, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 157, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4018, 1221, 0, 0, 4095, 1, 0, 15, 4096, 0, 0, 1, 4096, 4096, 4095, 2391, 0, 4096, 0, 0, 0, 4096, 0, 870, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3466, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 4088, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4090, 4096, 287, 4096, 0, 4, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 2966, 3061, 0, 0, 4088, 12, 0, 33, 4096, 0, 0, 0, 4096, 4096, 4096, 2212, 0, 4096, 0, 0, 0, 4096, 0, 285, 0, 0, 4072, 4040, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 20, 4095, 4071, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3576, 0, 1, 4096, 0, 0, 308, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 0, 4096, 0, 0, 0, 4094, 0, 0, 4096, 4088, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3733, 1, 0, 0, 0, 0, 0, 4091, 0, 371, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2459, 0, 0, 0, 0, 4095, 0, 1, 0, 3248, 2918, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 284, 2245, 0, 5, 0, 0, 0, 4095, 1783, 0, 4096, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 4094, 4055, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1851, 1594, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16376, 16216, 16433, 16387, 16828, 16604, 15852, 16376] avg=16384.0000 max=16828 max_rank=4 min=15852 min_rank=6 imbalance_ratio=1.027100 max_load_reduction=114244 max_load_reduction_pct=0.871613 moved_to_echo=65660 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 64, 4096, 4072, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3940, 0, 0, 4096, 0, 0, 309, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 0, 0, 4092, 0, 0, 0, 4085, 0, 0, 4096, 4092, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3729, 0, 0, 0, 0, 0, 0, 4056, 0, 219, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2451, 0, 0, 7, 0, 4094, 0, 0, 0, 3482, 2217, 0, 0, 4089, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 2242, 0, 8, 0, 0, 0, 4091, 1343, 0, 4096, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1126, 0, 0, 4027, 3994, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 10, 0, 1854, 1597, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16424, 16585, 16434, 16179, 16430, 15900, 16734, 16386] avg=16384.0000 max=16734 max_rank=6 min=15900 min_rank=5 imbalance_ratio=1.021362 max_load_reduction=114338 max_load_reduction_pct=0.872330 moved_to_echo=65450 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[8, 0, 0, 4087, 9, 0, 9, 0, 4087, 0, 9, 0, 7, 3, 2, 0, 4086, 4060, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 4086, 8, 9, 4084, 9, 9, 9, 3937, 0, 1, 4086, 9, 1, 307, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8, 4085, 0, 9, 2, 4084, 0, 9, 3684, 4089, 9, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 3720, 0, 0, 10, 0, 0, 6, 4084, 0, 155, 4087, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2452, 0, 0, 0, 8, 4086, 0, 0, 0, 4039, 1962, 9, 11, 4075, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2238, 0, 0, 0, 0, 0, 4087, 1295, 4, 4079, 7, 0, 7, 4078, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1177, 1, 7, 4088, 4088, 4084, 2, 7, 0, 0, 7, 9, 8, 0, 61, 9, 1844, 1590, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4082, 4084, 3, 4, 9, 4, 5, 8, 11, 7, 15, 0, 0, 0, 0, 4087, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16370, 16606, 16043, 16185, 16644, 15813, 17006, 16405] avg=16384.0000 max=17006 max_rank=6 min=15813 min_rank=5 imbalance_ratio=1.037964 max_load_reduction=114066 max_load_reduction_pct=0.870255 moved_to_echo=65868 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 1, 3, 1, 0, 4095, 0, 1, 0, 0, 1, 0, 0, 4095, 4070, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 4095, 1, 1, 1, 3747, 0, 0, 4094, 1, 0, 308, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 4095, 0, 1, 4079, 4095, 1, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3731, 0, 0, 1, 1, 0, 0, 4095, 0, 517, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2458, 0, 0, 0, 0, 4095, 0, 0, 0, 3953, 2241, 0, 1, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 2245, 0, 0, 0, 0, 0, 4095, 1451, 1, 4095, 0, 0, 1, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 359, 0, 1, 4095, 4095, 4095, 1, 1, 0, 0, 1, 1, 0, 0, 5, 2, 1850, 1594, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16366, 16389, 16423, 16535, 16845, 16007, 16125, 16382] avg=16384.0000 max=16845 max_rank=4 min=16007 min_rank=5 imbalance_ratio=1.028137 max_load_reduction=114227 max_load_reduction_pct=0.871483 moved_to_echo=65359 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4072, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3618, 0, 0, 4096, 0, 0, 308, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4094, 4096, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3733, 0, 0, 0, 0, 0, 0, 4096, 0, 157, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2459, 0, 0, 0, 0, 4096, 0, 0, 0, 4018, 1221, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2245, 0, 0, 1, 0, 0, 4095, 2391, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 870, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1851, 1594, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16363, 16257, 16437, 16178, 15890, 16939, 16627, 16381] avg=16384.0000 max=16939 max_rank=5 min=15890 min_rank=4 imbalance_ratio=1.033875 max_load_reduction=114133 max_load_reduction_pct=0.870766 moved_to_echo=65837 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 63, 4096, 4071, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3466, 0, 0, 4096, 0, 0, 308, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 4088, 4096, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3733, 0, 0, 0, 0, 0, 0, 4090, 0, 287, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2459, 0, 0, 0, 0, 4096, 0, 0, 0, 2966, 3061, 0, 0, 4088, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 2245, 0, 0, 0, 0, 0, 4096, 2212, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 0, 0, 4072, 4040, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1851, 1594, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16425, 16105, 16432, 16306, 16682, 16778, 15963, 16381] avg=16384.0000 max=16778 max_rank=5 min=15963 min_rank=6 imbalance_ratio=1.024048 max_load_reduction=114294 max_load_reduction_pct=0.871994 moved_to_echo=65804 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 4096, 4071, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3916, 0, 0, 4096, 0, 0, 308, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 4094, 4096, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3733, 0, 0, 0, 0, 0, 0, 4091, 0, 1130, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2459, 0, 0, 0, 0, 4096, 0, 0, 0, 3587, 1757, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 2245, 0, 0, 0, 0, 0, 4096, 1708, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 0, 0, 4094, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1851, 1594, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16365, 16555, 16438, 17146, 16000, 16279, 15907, 16382] avg=16384.0000 max=17146 max_rank=3 min=15907 min_rank=6 imbalance_ratio=1.046509 max_load_reduction=113926 max_load_reduction_pct=0.869186 moved_to_echo=64568 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 36, 4096, 4072, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3718, 2, 0, 4096, 0, 0, 308, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 4095, 4092, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 3733, 0, 0, 0, 0, 0, 0, 4088, 0, 160, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2459, 0, 0, 1, 0, 4096, 0, 0, 0, 3527, 1515, 0, 0, 4089, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 2245, 0, 6, 6, 0, 0, 4096, 2183, 0, 4096, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1169, 0, 0, 4040, 4056, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 24, 0, 1851, 1594, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16359, 16432, 16156, 15753, 16752, 16858, 16379] avg=16384.0000 max=16858 max_rank=6 min=15753 min_rank=4 imbalance_ratio=1.028931 max_load_reduction=114214 max_load_reduction_pct=0.871384 moved_to_echo=65742 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 32768, 128, 128, 128, 128, 128, 128, 128, 128, 128, 0, 128, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32768, 29952, 128, 0, 128, 128, 0, 128, 32768, 0, 3072, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 0, 32768, 128, 128, 128, 0, 32768, 0, 128, 0, 128, 128, 128, 256, 32768, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[5376, 128, 128, 32640, 32640, 32768, 128, 128, 0, 128, 128, 128, 128, 0, 128, 128, 14848, 12800, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 128, 128, 32768, 0, 128, 128, 32768, 0, 128, 32384, 32768, 128, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[19712, 0, 0, 128, 128, 32768, 0, 128, 0, 28928, 16896, 128, 128, 32768, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[512, 18048, 0, 128, 128, 0, 0, 32768, 14464, 128, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 32768, 128, 128, 128, 29952, 128, 128, 32768, 128, 128, 2560, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[3963, 0, 0, 1, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 4095, 3425, 0, 0, 0, 0, 4096, 0, 4096, 1532, 1, 4093, 0, 0, 0, 0, 4078, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4055, 0, 4093, 0, 0, 0, 0, 4095, 0, 4096, 1, 0, 0, 0, 3038, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 294, 0, 93, 0, 1, 4096, 3, 4095, 0, 0, 0, 0, 1, 0, 11, 0, 0, 0, 0, 4095, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3950, 4096, 132, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3995, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4090, 0, 5, 0, 0, 0, 0, 4090, 3606, 0, 0, 0, 0, 4096, 0, 4093, 931, 0, 4095, 0, 8, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 4086, 0, 0, 0, 0, 4094, 0, 4096, 6, 0, 0, 1, 2212, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 7, 0, 300, 0, 0, 4096, 9, 4096, 0, 0, 0, 0, 3, 0, 62, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4059, 4096, 1213, 4087, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4095, 1, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 1, 1, 4095, 0, 7, 0, 0, 0, 0, 4095, 22, 0, 1, 0, 4095, 1, 1, 0, 0, 0, 2, 4095, 3027, 3, 0, 0, 0, 4095, 0, 4095, 1583, 0, 4095, 0, 0, 0, 1, 4095, 0, 0, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 4095, 4095, 1, 0, 3913, 1, 4079, 1, 1, 0, 0, 4095, 1, 4095, 0, 1, 1, 0, 3405, 4095, 1, 4095, 4095, 1, 4094, 1, 0, 115, 1, 24, 0, 0, 4095, 0, 4095, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 4061, 0, 4095, 1, 1, 0, 0, 0, 0, 1, 0, 4095, 0, 1, 4094, 0, 4094, 4095, 235, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4063, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 3873, 0, 0, 0, 0, 4096, 0, 4096, 1419, 0, 4095, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4076, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 2358, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 138, 0, 256, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4030, 4096, 266, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4009, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4093, 0, 9, 0, 0, 5, 0, 4089, 3827, 0, 0, 0, 0, 4096, 0, 4096, 641, 0, 4096, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4088, 0, 4091, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 3034, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 33, 0, 422, 0, 0, 4096, 7, 4096, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4036, 4096, 353, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4083, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4051, 0, 0, 0, 0, 4096, 0, 4096, 1060, 0, 4093, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4093, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 1661, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 93, 0, 1304, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4081, 4096, 49, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4086, 9, 0, 0, 0, 7, 4077, 0, 8, 11, 0, 9, 8, 4, 4086, 10, 0, 4, 0, 0, 0, 4088, 5, 0, 4, 0, 4085, 10, 0, 0, 0, 0, 13, 4087, 3575, 43, 3, 4, 0, 4082, 4, 4085, 391, 0, 4087, 7, 0, 0, 10, 4086, 0, 0, 10, 1, 4084, 0, 7, 8, 0, 0, 8, 4085, 4085, 10, 0, 4084, 9, 4045, 7, 11, 0, 0, 4088, 3, 4084, 0, 9, 10, 6, 3370, 4086, 3, 4085, 4083, 6, 4081, 11, 5, 7, 10, 597, 0, 0, 4084, 0, 4086, 8, 10, 10, 7, 0, 11, 0, 0, 7, 0, 9, 3986, 0, 4085, 8, 4, 2, 0, 8, 0, 10, 4, 4086, 0, 6, 4071, 0, 4080, 4079, 336, 4089, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4023, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4006, 0, 0, 0, 0, 4096, 0, 4096, 718, 0, 4095, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 2413, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 78, 0, 279, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4081, 4096, 778, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3963, 0, 0, 1, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 2647, 616, 505, 366, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4092, 4, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4095, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3425, 0, 0, 0, 0, 0, 0, 3480, 1532, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 4093, 0, 0, 0, 0, 1448, 0, 4096, 1, 0, 0, 0, 3038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 4096, 0, 4096, 0, 0, 294, 0, 93, 0, 1, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 0, 0, 4095, 1, 4096, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3950, 3725, 132, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16302, 16378, 16626, 16372, 16731, 16270, 16395, 15998] avg=16384.0000 max=16731 max_rank=4 min=15998 min_rank=7 imbalance_ratio=1.021179 max_load_reduction=114341 max_load_reduction_pct=0.872353 moved_to_echo=65394 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3995, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 2650, 617, 509, 368, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 4090, 0, 5, 0, 0, 0, 4096, 4084, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3606, 0, 0, 0, 0, 0, 0, 3476, 931, 0, 4095, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4086, 0, 0, 0, 0, 1444, 0, 4096, 6, 0, 0, 1, 2212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3587, 4096, 0, 4096, 0, 0, 7, 0, 300, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 62, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4059, 3718, 1213, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16348, 16382, 16206, 16391, 15939, 16191, 16442, 17173] avg=16384.0000 max=17173 max_rank=7 min=15939 min_rank=4 imbalance_ratio=1.048157 max_load_reduction=113899 max_load_reduction_pct=0.868980 moved_to_echo=65745 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4095, 1, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 1, 1, 4095, 0, 2647, 616, 505, 365, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4095, 22, 0, 1, 0, 4095, 1, 1, 0, 0, 0, 4095, 4084, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 3027, 3, 0, 0, 0, 0, 0, 3479, 1583, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 4095, 4095, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 1, 4079, 1, 1, 0, 0, 1448, 1, 4095, 0, 1, 1, 0, 3405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3590, 4095, 1, 4094, 1, 0, 115, 1, 24, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 4061, 0, 4095, 1, 1, 4095, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4094, 0, 4094, 3725, 235, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16435, 16403, 16284, 16390, 16946, 16017, 16352, 16245] avg=16384.0000 max=16946 max_rank=4 min=16017 min_rank=5 imbalance_ratio=1.034302 max_load_reduction=114126 max_load_reduction_pct=0.870712 moved_to_echo=65560 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4063, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2648, 616, 505, 366, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3873, 0, 0, 0, 0, 0, 0, 3480, 1419, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 4096, 0, 0, 0, 0, 1448, 0, 4096, 0, 0, 0, 0, 2358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 4096, 0, 4096, 0, 0, 138, 0, 256, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4030, 3725, 266, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16401, 16378, 16963, 16387, 16074, 16273, 16383, 16213] avg=16384.0000 max=16963 max_rank=2 min=16074 min_rank=4 imbalance_ratio=1.035339 max_load_reduction=114109 max_load_reduction_pct=0.870583 moved_to_echo=64943 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4083, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 2648, 616, 505, 366, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4085, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4051, 0, 0, 0, 0, 0, 0, 3480, 1060, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4095, 0, 0, 0, 0, 1448, 0, 4096, 1, 0, 0, 0, 1661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 4096, 0, 4096, 0, 0, 93, 0, 1304, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4081, 3725, 49, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16422, 16383, 16780, 16387, 15394, 17276, 16383, 16047] avg=16384.0000 max=17276 max_rank=5 min=15394 min_rank=4 imbalance_ratio=1.054443 max_load_reduction=113796 max_load_reduction_pct=0.868195 moved_to_echo=65100 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4086, 9, 0, 0, 0, 7, 4077, 0, 8, 11, 0, 9, 8, 4, 4086, 10, 2643, 614, 504, 364, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4088, 5, 0, 4, 0, 4085, 10, 0, 0, 0, 0, 4086, 4071, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4087, 3575, 43, 3, 4, 0, 0, 4, 3471, 391, 0, 4087, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4086, 0, 0, 10, 1, 4084, 0, 7, 8, 0, 0, 8, 4085, 4085, 10, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 9, 4045, 7, 11, 0, 0, 1445, 3, 4084, 0, 9, 10, 6, 3370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3581, 4083, 6, 4081, 11, 5, 7, 10, 597, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 10, 10, 7, 0, 11, 0, 0, 7, 0, 9, 3986, 0, 4085, 8, 4, 4086, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8, 0, 10, 4, 0, 0, 6, 4071, 0, 4080, 3710, 336, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16451, 16355, 15685, 16399, 17083, 16468, 16315, 16316] avg=16384.0000 max=17083 max_rank=4 min=15685 min_rank=2 imbalance_ratio=1.042664 max_load_reduction=113989 max_load_reduction_pct=0.869667 moved_to_echo=66182 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4009, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 2648, 616, 505, 366, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4093, 0, 9, 0, 0, 5, 4096, 4084, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3827, 0, 0, 0, 0, 0, 0, 3480, 641, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4091, 0, 0, 0, 0, 1448, 0, 4096, 0, 0, 0, 0, 3034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 4096, 0, 4096, 0, 0, 33, 0, 422, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4036, 3725, 353, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16352, 16400, 16133, 16365, 16757, 16341, 16420, 16304] avg=16384.0000 max=16757 max_rank=4 min=16133 min_rank=2 imbalance_ratio=1.022766 max_load_reduction=114315 max_load_reduction_pct=0.872154 moved_to_echo=65822 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4023, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2648, 616, 505, 366, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4085, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4006, 0, 0, 0, 0, 0, 0, 3480, 718, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 1448, 0, 4096, 0, 0, 0, 0, 2413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 4096, 0, 4096, 0, 0, 78, 0, 279, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4081, 3725, 778, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16361, 16393, 16395, 16381, 16148, 16236, 16382, 16776] avg=16384.0000 max=16776 max_rank=7 min=16148 min_rank=4 imbalance_ratio=1.023926 max_load_reduction=114296 max_load_reduction_pct=0.872009 moved_to_echo=65542 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 28800, 32768, 128, 32768, 128, 128, 768, 128, 3328, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 29440, 128, 128, 128, 0, 0, 128, 27904, 8320, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 128, 128, 0, 0, 128, 32768, 0, 32512, 29824, 3456, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 0, 0, 32768, 128, 0, 128, 0, 32768, 128, 128, 0, 0, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32512, 128, 32768, 128, 128, 0, 0, 11648, 128, 32768, 128, 128, 128, 128, 21504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 0, 0, 128, 128, 32768, 0, 128, 128, 0, 0, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 0, 128, 0, 128, 32640, 128, 32768, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32384, 128, 0, 128, 0, 128, 32768, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 21248, 4992, 4096, 2944, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2659, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 321, 0, 4091, 0, 0, 0, 0, 0, 4077, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3857, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 1378, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 24, 0, 489, 4, 0, 4096, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 1075, 0, 4094, 0, 0, 0, 0, 0, 4080, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 4068, 0, 0, 3496, 0, 4096, 4096, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 4096, 2, 0, 4060, 0, 0, 4096, 0, 15, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 7, 4096, 3156, 0, 4084, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 4095, 1, 1, 0, 0, 0, 2, 0, 2525, 3, 1, 4095, 4095, 1, 0, 1, 0, 1, 4095, 1, 1, 0, 1, 4095, 0, 0, 242, 1, 4095, 1, 1, 0, 1, 1, 4094, 1, 0, 4095, 0, 1, 0, 0, 1, 0, 0, 4095, 0, 0, 2906, 0, 4095, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 1, 0, 0, 1, 1, 4095, 4095, 0, 4095, 0, 0, 1, 0, 0, 0, 1, 1, 1, 3, 0, 0, 4091, 1, 0, 1, 4095, 0, 4095, 4028, 0, 0, 4095, 0, 1, 4093, 0, 0, 4095, 0, 4094, 1, 0, 0, 4095, 2, 0, 1, 4095, 4095, 4095, 0, 4095, 2584, 0, 4093, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 40, 0, 1461, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 131, 0, 4094, 0, 0, 2, 0, 0, 4065, 0, 0, 4092, 0, 12, 5, 0, 0, 0, 0, 4056, 0, 0, 3469, 0, 4096, 4091, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 4095, 4096, 0, 4096, 2, 5, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4096, 0, 0, 0, 4096, 0, 4088, 4096, 8, 4, 4031, 0, 0, 4096, 0, 31, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4089, 4096, 12, 4096, 3130, 0, 4083, 0, 15] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[10, 0, 10, 0, 2, 4086, 11, 10, 2, 0, 0, 0, 7, 2453, 35, 10, 4086, 4080, 10, 6, 10, 0, 2, 4087, 8, 10, 0, 10, 4086, 0, 0, 814, 13, 3998, 13, 9, 0, 9, 10, 4087, 9, 0, 4077, 0, 0, 0, 0, 12, 0, 2, 4094, 0, 0, 2413, 0, 4081, 4085, 7, 0, 4087, 1, 0, 0, 0, 1, 0, 7, 4086, 0, 0, 0, 9, 0, 0, 9, 9, 4086, 4078, 0, 4086, 0, 0, 9, 0, 0, 0, 9, 9, 8, 34, 9, 0, 4051, 12, 1, 10, 4086, 2, 4086, 3390, 0, 0, 4093, 0, 10, 4081, 0, 0, 4079, 0, 4084, 3, 0, 0, 4086, 15, 0, 11, 4085, 4086, 4086, 0, 4083, 3237, 3, 4083, 8, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 9, 0, 1730, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 528, 0, 4073, 0, 0, 1, 0, 0, 4086, 0, 0, 4095, 1, 0, 2, 0, 0, 0, 0, 4086, 0, 0, 3911, 0, 4096, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 4096, 1, 4096, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 4096, 3, 1, 4065, 0, 0, 4096, 1, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 4096, 4095, 4096, 0, 4096, 2072, 0, 4095, 0, 10] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1864, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 166, 0, 4093, 0, 0, 0, 0, 0, 4055, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3852, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4089, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 2340, 0, 4096, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 1004, 4, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1351, 0, 4095, 0, 0, 0, 0, 0, 4077, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 3278, 0, 4096, 4096, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 2, 0, 4070, 0, 0, 4096, 0, 30, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 6, 4096, 2563, 0, 4093, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2659, 0, 0, 4096, 3987, 2392, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3518, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4077, 0, 0, 4096, 0, 0, 0, 0, 0, 2559, 983, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3857, 0, 721, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1537, 4096, 0, 0, 4096, 1378, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17250, 16127, 16384, 16866, 16384, 16364, 16494, 15203] avg=16384.0000 max=17250 max_rank=0 min=15203 min_rank=7 imbalance_ratio=1.052856 max_load_reduction=113822 max_load_reduction_pct=0.868393 moved_to_echo=64445 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 9, 0, 1730, 0, 0, 4096, 3987, 2392, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3518, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 1, 0, 0, 4086, 0, 0, 4095, 1, 0, 2, 0, 0, 2559, 983, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 3911, 0, 721, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4093, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 3, 1, 4065, 0, 0, 4096, 1, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1537, 4095, 0, 0, 4096, 2072, 0, 4095, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16332, 16333, 16378, 16908, 16383, 16366, 16466, 15906] avg=16384.0000 max=16908 max_rank=3 min=15906 min_rank=7 imbalance_ratio=1.031982 max_load_reduction=114164 max_load_reduction_pct=0.871002 moved_to_echo=65121 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 0, 4095, 1, 1, 0, 0, 0, 2, 0, 2525, 3, 1, 4094, 3986, 2391, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3517, 4095, 1, 0, 1, 0, 1, 4095, 1, 1, 0, 1, 4095, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 0, 1, 1, 4094, 1, 0, 4095, 0, 1, 0, 0, 1, 2559, 983, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2906, 0, 721, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 1, 0, 0, 1, 1, 4095, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 3, 0, 0, 4091, 1, 0, 1, 4095, 4095, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4028, 0, 0, 4095, 0, 1, 4093, 0, 0, 109, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1536, 4095, 0, 0, 4095, 2584, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17120, 16050, 16412, 15912, 16384, 16365, 16422, 16407] avg=16384.0000 max=17120 max_rank=0 min=15912 min_rank=3 imbalance_ratio=1.044922 max_load_reduction=113952 max_load_reduction_pct=0.869385 moved_to_echo=65578 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 40, 0, 1461, 0, 0, 4096, 3991, 2396, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3513, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 4065, 0, 0, 4092, 0, 12, 5, 0, 0, 2565, 987, 581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 3469, 0, 713, 4091, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 4095, 4096, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 8, 4, 4031, 0, 0, 4096, 0, 31, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1531, 4089, 0, 12, 4096, 3130, 0, 4083, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16106, 15926, 16403, 16414, 16393, 16415, 16459, 16956] avg=16384.0000 max=16956 max_rank=7 min=15926 min_rank=1 imbalance_ratio=1.034912 max_load_reduction=114116 max_load_reduction_pct=0.870636 moved_to_echo=66223 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[10, 0, 10, 0, 2, 4086, 11, 10, 2, 0, 0, 0, 7, 2453, 35, 10, 4084, 3971, 2383, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3509, 4080, 10, 6, 10, 0, 2, 4087, 8, 10, 0, 10, 4086, 0, 0, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3998, 13, 9, 0, 9, 10, 4087, 9, 0, 4077, 0, 0, 0, 0, 12, 2553, 979, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 2413, 0, 719, 4085, 7, 0, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 4086, 0, 0, 0, 9, 0, 0, 9, 9, 4086, 4078, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 9, 9, 8, 34, 9, 0, 4051, 12, 1, 10, 4086, 4086, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4086, 3390, 0, 0, 4093, 0, 10, 4081, 0, 0, 108, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 11, 1532, 4086, 0, 0, 4083, 3237, 3, 4083, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17094, 16632, 16346, 15408, 16371, 16390, 15773, 17058] avg=16384.0000 max=17094 max_rank=0 min=15408 min_rank=3 imbalance_ratio=1.043335 max_load_reduction=113978 max_load_reduction_pct=0.869583 moved_to_echo=65592 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 24, 0, 489, 4, 0, 4096, 3987, 2392, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3518, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4080, 0, 0, 4096, 0, 7, 0, 0, 0, 2559, 983, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 3496, 0, 721, 4096, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 2, 0, 4060, 0, 0, 4096, 0, 15, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1537, 4096, 0, 7, 4096, 3156, 0, 4084, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15110, 16875, 16397, 16461, 16389, 16388, 16473, 16979] avg=16384.0000 max=16979 max_rank=7 min=15110 min_rank=0 imbalance_ratio=1.036316 max_load_reduction=114093 max_load_reduction_pct=0.870461 moved_to_echo=66229 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1864, 0, 0, 4096, 3987, 2392, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3518, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4055, 0, 0, 4096, 0, 0, 0, 0, 0, 2559, 983, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3852, 0, 721, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4089, 0, 0, 4096, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1537, 4096, 0, 0, 4096, 2340, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16455, 15972, 16364, 16861, 16384, 16383, 16486, 16167] avg=16384.0000 max=16861 max_rank=3 min=15972 min_rank=1 imbalance_ratio=1.029114 max_load_reduction=114211 max_load_reduction_pct=0.871361 moved_to_echo=65420 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 1004, 4, 0, 4096, 3987, 2392, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3518, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4077, 0, 0, 4096, 0, 0, 0, 0, 0, 2559, 983, 578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 3278, 0, 721, 4096, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 4070, 0, 0, 4096, 0, 30, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1537, 4096, 0, 6, 4096, 2563, 0, 4093, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15605, 17157, 16388, 16242, 16384, 16401, 16499, 16396] avg=16384.0000 max=17157 max_rank=1 min=15605 min_rank=0 imbalance_ratio=1.047180 max_load_reduction=113915 max_load_reduction_pct=0.869102 moved_to_echo=65680 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[28160, 32768, 128, 128, 128, 0, 128, 32768, 128, 128, 0, 128, 32768, 0, 0, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 0, 128, 32768, 128, 128, 128, 0, 128, 128, 128, 14208, 128, 128, 32768, 32000, 19200, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 32768, 32000, 128, 128, 32640, 128, 128, 32768, 128, 128, 896, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 0, 128, 128, 0, 128, 128, 32768, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 128, 32768, 0, 0, 27264, 0, 5760, 32768, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32640, 128, 128, 128, 128, 128, 32640, 128, 0, 32768, 128, 128, 128, 0, 128, 20480, 7936, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 0, 0, 128, 128, 128, 256, 128, 0, 32768, 128, 128, 128, 32768, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 0, 128, 128, 128, 12288, 32768, 0, 128, 32768, 20480, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[2267, 0, 0, 4096, 0, 0, 0, 4096, 3298, 0, 0, 4096, 0, 0, 0, 1, 1, 1, 0, 0, 0, 2150, 4096, 0, 0, 0, 0, 0, 4094, 4096, 0, 1, 4096, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4092, 4095, 0, 2, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 481, 0, 4096, 1, 0, 0, 1, 4096, 0, 10, 4096, 0, 0, 1, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 4084, 4095, 0, 0, 4096, 0, 4094, 0, 0, 4095, 1, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[3984, 4, 43, 4083, 12, 3, 122, 3858, 2864, 0, 6, 4017, 1, 6, 10, 0, 0, 0, 27, 166, 10, 339, 4083, 3, 0, 11, 0, 0, 4086, 4049, 0, 0, 4067, 0, 4, 32, 4087, 4086, 0, 20, 6, 7, 0, 9, 30, 0, 7, 12, 10, 0, 9, 0, 0, 7, 5, 4085, 4087, 10, 0, 4062, 0, 8, 10, 4084, 2, 0, 0, 0, 0, 4085, 4086, 1, 10, 0, 0, 13, 3957, 0, 0, 10, 10, 0, 10, 4084, 4086, 4, 0, 2066, 0, 2974, 3, 0, 8, 0, 4070, 4, 0, 4076, 10, 11, 0, 6, 4048, 0, 10, 4, 4081, 9, 0, 62, 4080, 0, 4086, 4084, 32, 171, 4081, 0, 4083, 10, 11, 4086, 0, 3976, 11, 12, 8, 10] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[3426, 0, 5, 4095, 1, 0, 0, 3998, 2242, 0, 1, 4093, 0, 1, 1, 0, 0, 0, 4, 22, 1, 1629, 4095, 0, 0, 1, 0, 0, 4095, 4089, 0, 0, 4094, 0, 0, 1, 4095, 4095, 0, 2, 0, 1, 0, 1, 10, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 4095, 4095, 1, 0, 4089, 0, 1, 1, 4095, 1, 1, 0, 0, 0, 4095, 4095, 0, 1, 0, 0, 1, 4081, 0, 0, 1, 1, 0, 1, 4095, 4095, 0, 0, 987, 0, 4071, 0, 0, 1, 0, 4094, 1, 0, 4095, 1, 1, 0, 1, 4093, 0, 1, 1, 4095, 1, 0, 17, 4093, 0, 4095, 4095, 6, 11, 4094, 0, 4095, 0, 1, 4095, 0, 4076, 1, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[2182, 0, 0, 4096, 0, 0, 0, 4096, 3067, 0, 0, 4096, 0, 0, 0, 6, 4, 0, 0, 0, 0, 1356, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4041, 4096, 0, 6, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 1622, 0, 4096, 0, 0, 0, 0, 4096, 0, 16, 4096, 0, 0, 0, 0, 4096, 7, 0, 0, 4096, 0, 0, 0, 4096, 0, 4085, 4096, 0, 0, 4096, 0, 4092, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[2863, 0, 0, 4096, 0, 0, 0, 4092, 2170, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1353, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1807, 0, 4095, 1, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 4096, 4096, 0, 1, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3768, 0, 1, 4096, 0, 0, 0, 4096, 2627, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 713, 4096, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4090, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 4092, 0, 0, 1092, 0, 4096, 0, 0, 0, 5, 4096, 0, 8, 4096, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 4079, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[3003, 0, 0, 4096, 0, 0, 0, 4096, 1764, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2397, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1027, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[726, 0, 0, 4096, 0, 0, 0, 4096, 3540, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1175, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2751, 0, 4096, 1, 0, 0, 0, 4096, 0, 14, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4080, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2267, 0, 0, 2745, 0, 0, 0, 4096, 3298, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 2150, 4096, 0, 0, 0, 0, 0, 4094, 4096, 0, 1, 1340, 1334, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4092, 4095, 0, 2, 4096, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4095, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2761, 4095, 0, 0, 481, 0, 4096, 1, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 1, 0, 4096, 2, 0, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4091, 0, 4094, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16503, 17123, 16371, 16389, 16402, 15531, 16387, 16366] avg=16384.0000 max=17123 max_rank=1 min=15531 min_rank=5 imbalance_ratio=1.045105 max_load_reduction=113949 max_load_reduction_pct=0.869362 moved_to_echo=64686 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3984, 4, 43, 2736, 12, 3, 122, 3858, 2864, 0, 6, 4017, 1, 6, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 166, 10, 339, 4083, 3, 0, 11, 0, 0, 4086, 4049, 0, 0, 1336, 1330, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 4, 32, 4087, 4086, 0, 20, 6, 7, 0, 9, 30, 0, 7, 12, 4066, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 9, 0, 0, 7, 5, 4085, 4087, 10, 0, 4062, 0, 8, 10, 4084, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4085, 4086, 1, 10, 0, 0, 13, 3957, 0, 0, 10, 4086, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 10, 2754, 4086, 4, 0, 2066, 0, 2974, 3, 0, 8, 0, 4070, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 10, 11, 0, 6, 4048, 0, 10, 4, 4071, 9, 0, 62, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 32, 171, 4076, 0, 4083, 10, 11, 0, 0, 3976, 11, 12, 8, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17666, 15450, 16444, 16382, 16268, 15989, 16387, 16486] avg=16384.0000 max=17666 max_rank=0 min=15450 min_rank=1 imbalance_ratio=1.078247 max_load_reduction=113406 max_load_reduction_pct=0.865219 moved_to_echo=65130 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3426, 0, 5, 2744, 1, 0, 0, 3998, 2242, 0, 1, 4093, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 22, 1, 1629, 4095, 0, 0, 1, 0, 0, 4095, 4089, 0, 0, 1340, 1334, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 4095, 4095, 0, 2, 0, 1, 0, 1, 10, 0, 1, 1, 4076, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 4095, 4095, 1, 0, 4089, 0, 1, 1, 4095, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4095, 4095, 0, 1, 0, 0, 1, 4081, 0, 0, 1, 4095, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 2761, 4095, 0, 0, 987, 0, 4071, 0, 0, 1, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 1, 4093, 0, 1, 1, 4085, 1, 0, 17, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 6, 11, 4089, 0, 4095, 0, 1, 0, 0, 4076, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16512, 16620, 16388, 16386, 16390, 16012, 16389, 16375] avg=16384.0000 max=16620 max_rank=1 min=16012 min_rank=5 imbalance_ratio=1.014404 max_load_reduction=114452 max_load_reduction_pct=0.873199 moved_to_echo=65166 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2863, 0, 0, 2745, 0, 0, 0, 4092, 2170, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1353, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1340, 1334, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2762, 4096, 0, 0, 1807, 0, 4095, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4091, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15966, 16325, 16377, 16389, 16402, 16857, 16376, 16380] avg=16384.0000 max=16857 max_rank=5 min=15966 min_rank=0 imbalance_ratio=1.028870 max_load_reduction=114215 max_load_reduction_pct=0.871391 moved_to_echo=66015 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2182, 0, 0, 2737, 0, 0, 0, 4096, 3067, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1356, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1347, 1339, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4041, 4096, 0, 6, 4096, 0, 0, 0, 4096, 7, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2757, 4089, 0, 0, 1622, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 0, 0, 0, 0, 4096, 7, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4089, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16184, 16349, 16370, 16358, 16402, 16660, 16387, 16362] avg=16384.0000 max=16660 max_rank=5 min=16184 min_rank=0 imbalance_ratio=1.016846 max_load_reduction=114412 max_load_reduction_pct=0.872894 moved_to_echo=65811 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[726, 0, 0, 2745, 0, 0, 0, 4096, 3540, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1175, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 1340, 1334, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2762, 4096, 0, 0, 2751, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15203, 16148, 16376, 16389, 16403, 17802, 16388, 16363] avg=16384.0000 max=17802 max_rank=5 min=15203 min_rank=0 imbalance_ratio=1.086548 max_load_reduction=113270 max_load_reduction_pct=0.864182 moved_to_echo=66956 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3768, 0, 1, 2745, 0, 0, 0, 4096, 2627, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 713, 4096, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 1340, 1334, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4090, 4096, 0, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2762, 4092, 0, 0, 1092, 0, 4096, 0, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 0, 0, 4096, 2, 0, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17334, 15687, 16370, 16390, 16402, 16143, 16384, 16362] avg=16384.0000 max=17334 max_rank=0 min=15687 min_rank=1 imbalance_ratio=1.057983 max_load_reduction=113738 max_load_reduction_pct=0.867752 moved_to_echo=65291 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3003, 0, 0, 2745, 0, 0, 0, 4096, 1764, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2397, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 1340, 1334, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2762, 4096, 0, 0, 1027, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15704, 17370, 16376, 16389, 16403, 16078, 16374, 16378] avg=16384.0000 max=17370 max_rank=1 min=15704 min_rank=0 imbalance_ratio=1.060181 max_load_reduction=113702 max_load_reduction_pct=0.867477 moved_to_echo=65233 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 32768, 128, 128, 128, 32768, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[22272, 128, 128, 22016, 128, 128, 128, 32512, 21632, 0, 128, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 0, 0, 32768, 32768, 128, 128, 0, 0, 128, 32640, 128, 128, 128, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 22144, 32768, 128, 0, 11904, 0, 31744, 128, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 0, 128, 128, 32768, 32768, 0, 128, 128, 128, 0, 128, 128, 0, 128, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 256, 128, 11136, 32768, 128, 0, 128, 0, 0, 32768, 32768, 0, 128, 10752, 10752, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 0, 128, 256, 32768, 0, 32768, 128, 128, 0, 128, 32640, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 32768, 32768, 128, 128, 32768, 0, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 4095, 4095, 0, 4096, 1, 0, 0, 0, 1960, 4096, 0, 0, 4095, 0, 0, 0, 0, 3113, 0, 1, 0, 4095, 0, 0, 0, 0, 1, 6, 0, 4096, 0, 4096, 0, 4095, 1, 0, 0, 0, 4096, 0, 0, 4096, 1021, 0, 1, 0, 0, 0, 0, 0, 1, 0, 4095, 4096, 0, 4096, 60, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4095, 0, 3, 0, 2681, 4095, 3457, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 1, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1, 4092, 1, 0, 12, 1, 1, 0, 4095, 15, 1, 4, 4087, 1, 1, 0, 4092, 0, 0, 4095, 4095, 2, 4094, 2, 1, 0, 1, 644, 4093, 0, 0, 4095, 1, 7, 1, 1, 2704, 0, 0, 0, 4095, 0, 1, 0, 1, 0, 0, 49, 4095, 1, 4059, 1, 4095, 0, 1, 0, 0, 4095, 1, 1, 4094, 2539, 0, 0, 0, 3, 0, 1, 1, 0, 0, 4095, 4093, 0, 4095, 15, 0, 0, 1, 1, 1, 0, 4095, 1, 4095, 0, 0, 1, 4095, 1, 0, 2, 3263, 4095, 3081, 4095, 0, 0, 0, 0, 4095, 2, 0, 1, 1, 0, 4095, 1, 0, 0, 1, 1, 4088, 1, 4095, 0, 0, 0, 0, 0, 1, 4095, 2, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 2, 0, 0, 0, 1268, 4096, 0, 0, 4096, 0, 0, 0, 0, 2967, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 51, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2796, 4096, 2689, 4085, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 4076, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 4096, 4096, 0, 4095, 0, 0, 0, 0, 1659, 4096, 0, 0, 4096, 0, 0, 0, 0, 3871, 0, 4, 0, 4091, 0, 0, 0, 0, 0, 25, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1459, 0, 1, 0, 4, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 12, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 4083, 0, 0, 0, 2830, 4096, 2486, 4083, 0, 0, 0, 5, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 1, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 6, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 1, 0, 0, 0, 2054, 4096, 0, 0, 4096, 0, 0, 0, 0, 3245, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 2336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2936, 4096, 1712, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[11, 4069, 5, 0, 138, 4, 3, 9, 4085, 48, 11, 34, 3938, 10, 11, 0, 4058, 0, 0, 4086, 4086, 18, 4052, 0, 11, 0, 8, 258, 4065, 0, 10, 4086, 11, 74, 1, 13, 1815, 0, 0, 0, 4086, 0, 14, 2, 11, 0, 0, 376, 4085, 10, 3778, 9, 4084, 0, 10, 0, 0, 4085, 11, 9, 4074, 3791, 0, 0, 0, 0, 0, 10, 9, 0, 5, 4085, 4062, 1, 4080, 0, 0, 4, 12, 9, 7, 0, 4081, 11, 4085, 0, 0, 12, 4079, 7, 0, 1265, 3319, 4085, 1725, 4084, 0, 3, 10, 0, 4081, 24, 0, 8, 2, 0, 4080, 10, 0, 0, 10, 9, 4017, 11, 4082, 6, 0, 0, 2, 0, 20, 4083, 27, 4082, 5, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 4083, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 775, 4096, 0, 0, 4096, 0, 0, 0, 0, 3577, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 18, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 3307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4092, 0, 2, 0, 3802, 4096, 824, 4089, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 1, 0, 0, 0, 1541, 4096, 0, 0, 4096, 0, 0, 0, 0, 3531, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 84, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3128, 4096, 2210, 4080, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3177, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 1960, 4096, 0, 0, 2835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3113, 0, 1, 0, 4095, 0, 0, 0, 0, 1, 6, 0, 4095, 4095, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 1, 0, 0, 0, 1792, 0, 0, 4096, 1021, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 4096, 0, 4096, 60, 0, 0, 0, 0, 1420, 1393, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2675, 0, 0, 0, 4095, 0, 3, 0, 2681, 0, 3457, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 3175, 919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 1, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 17085, 16327, 15102, 16414, 16999, 16381, 16392] avg=16384.0000 max=17085 max_rank=1 min=15102 min_rank=3 imbalance_ratio=1.042786 max_load_reduction=113987 max_load_reduction_pct=0.869652 moved_to_echo=66186 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 4092, 1, 0, 12, 1, 1, 0, 4095, 15, 1, 4, 4087, 1, 1, 0, 3176, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 2, 4094, 2, 1, 0, 1, 644, 4093, 0, 0, 2835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 1, 1, 2704, 0, 0, 0, 4095, 0, 1, 0, 1, 0, 0, 49, 4095, 4095, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4059, 1, 4095, 0, 1, 0, 0, 1791, 1, 1, 4094, 2539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 1, 0, 0, 4095, 4093, 0, 4095, 15, 0, 0, 1, 1, 1420, 1393, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 2675, 0, 0, 1, 4095, 1, 0, 2, 3263, 0, 3081, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 1, 1, 0, 4095, 1, 0, 0, 1, 1, 4095, 3174, 919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 4095, 0, 0, 0, 0, 0, 1, 4095, 2, 4095, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16399, 15764, 15971, 16583, 16371, 17215, 16385, 16384] avg=16384.0000 max=17215 max_rank=5 min=15764 min_rank=1 imbalance_ratio=1.050720 max_load_reduction=113857 max_load_reduction_pct=0.868660 moved_to_echo=66355 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4076, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3178, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 1659, 4096, 0, 0, 2828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3871, 0, 4, 0, 4091, 0, 0, 0, 0, 0, 25, 0, 4096, 4096, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1780, 0, 0, 4096, 1459, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 12, 0, 0, 0, 0, 1423, 1398, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2667, 0, 0, 0, 4083, 0, 0, 0, 2830, 0, 2486, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3174, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 1, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 6, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16364, 16775, 17105, 15528, 16384, 16149, 16379, 16388] avg=16384.0000 max=17105 max_rank=2 min=15528 min_rank=3 imbalance_ratio=1.044006 max_load_reduction=113967 max_load_reduction_pct=0.869499 moved_to_echo=65300 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3177, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 2054, 4096, 0, 0, 2836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3245, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 1792, 0, 0, 4096, 2336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 7, 0, 0, 0, 0, 1420, 1393, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2676, 0, 0, 0, 4096, 0, 0, 0, 2936, 0, 1712, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3174, 919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16374, 17179, 16455, 16415, 16361, 15516, 16381, 16391] avg=16384.0000 max=17179 max_rank=1 min=15516 min_rank=5 imbalance_ratio=1.048523 max_load_reduction=113893 max_load_reduction_pct=0.868935 moved_to_echo=64649 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3177, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 1268, 4096, 0, 0, 2836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2967, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 1792, 0, 0, 4096, 2528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 51, 0, 0, 0, 0, 1420, 1393, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2676, 0, 0, 0, 4096, 0, 0, 0, 2796, 0, 2689, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3175, 919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16394, 16176, 16607, 16405, 16342, 16382, 16391] avg=16384.0000 max=16607 max_rank=3 min=16176 min_rank=2 imbalance_ratio=1.013611 max_load_reduction=114465 max_load_reduction_pct=0.873299 moved_to_echo=65520 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[11, 4069, 5, 0, 138, 4, 3, 9, 4085, 48, 11, 34, 3938, 10, 11, 0, 3165, 909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 0, 18, 4052, 0, 11, 0, 8, 258, 4065, 0, 10, 2830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 74, 1, 13, 1815, 0, 0, 0, 4086, 0, 14, 2, 11, 0, 0, 376, 4086, 4086, 918, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 3778, 9, 4084, 0, 10, 0, 0, 1786, 11, 9, 4074, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 9, 0, 5, 4085, 4062, 1, 4080, 0, 0, 4, 12, 9, 1417, 1390, 1250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 11, 2668, 0, 0, 12, 4079, 7, 0, 1265, 3319, 0, 1725, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 10, 0, 4081, 24, 0, 8, 2, 0, 4080, 10, 0, 0, 10, 9, 4085, 3167, 916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4017, 11, 4082, 6, 0, 0, 2, 0, 20, 4083, 27, 4082, 5, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16450, 15310, 15493, 17562, 16334, 17177, 16405, 16341] avg=16384.0000 max=17562 max_rank=3 min=15310 min_rank=1 imbalance_ratio=1.071899 max_load_reduction=113510 max_load_reduction_pct=0.866013 moved_to_echo=66257 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3177, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 1541, 4096, 0, 0, 2836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3531, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 1792, 0, 0, 4096, 1812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 84, 0, 0, 0, 0, 1420, 1393, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2676, 0, 0, 0, 4096, 0, 0, 0, 3128, 0, 2210, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3175, 919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16666, 16740, 15890, 16438, 16190, 16382, 16391] avg=16384.0000 max=16740 max_rank=2 min=15890 min_rank=3 imbalance_ratio=1.021729 max_load_reduction=114332 max_load_reduction_pct=0.872284 moved_to_echo=65401 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 4083, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3177, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 775, 4096, 0, 0, 2836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3577, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 18, 0, 4096, 4096, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 1791, 0, 0, 4096, 3307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 11, 0, 0, 0, 0, 1420, 1393, 1253, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2675, 0, 0, 0, 4092, 0, 2, 0, 3802, 0, 824, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3174, 919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16363, 15899, 16805, 17385, 16365, 15484, 16377, 16394] avg=16384.0000 max=17385 max_rank=3 min=15484 min_rank=5 imbalance_ratio=1.061096 max_load_reduction=113687 max_load_reduction_pct=0.867363 moved_to_echo=64620 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 0, 0, 128, 32768, 128, 128, 0, 128, 10240, 32768, 0, 128, 22784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 24832, 0, 128, 0, 32768, 0, 128, 128, 128, 128, 128, 512, 32768, 32768, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 32768, 128, 0, 128, 128, 0, 32768, 128, 0, 0, 128, 128, 32768, 25472, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 0, 256, 128, 128, 128, 32768, 128, 128, 128, 32640, 128, 128, 0, 25472, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 32768, 128, 128, 128, 128, 0, 128, 32768, 128, 32768, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 21504, 0, 0, 128, 32768, 128, 128, 1280, 24832, 0, 18304, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 32512, 128, 32768, 128, 128, 0, 0, 14336, 128, 128, 32768, 18816, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 32768, 32768, 128, 32768, 256, 0, 128, 128, 128, 11392, 11264, 10112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4076, 0, 0, 0, 4096, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 1, 165, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 3848, 4096, 0, 0, 0, 0, 0, 4096, 4095, 1, 4096, 0, 4096, 0, 4096, 0, 3924, 0, 0, 0, 4096, 0, 4096, 173, 0, 0, 0, 0, 4094, 4096, 4087, 0, 4096, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 1, 1, 80, 0, 0, 0, 4095, 0, 0, 0, 4095, 4037, 1, 399, 1, 1, 0, 1, 1, 0, 0, 23, 4092, 0, 208, 0, 0, 4094, 4095, 4095, 0, 1, 1, 0, 4095, 1, 1, 1479, 0, 2, 1, 0, 1, 0, 46, 0, 4092, 4081, 1, 1, 1, 1, 4095, 0, 1, 0, 4095, 3, 1, 1, 0, 0, 1, 4095, 0, 1, 4059, 0, 1, 0, 4092, 2048, 4087, 10, 1, 0, 0, 10, 4094, 4095, 0, 4095, 0, 4083, 1, 4084, 0, 3937, 1, 0, 2, 4095, 14, 4092, 73, 0, 1, 19, 0, 4095, 4093, 4095, 12, 4095, 0, 0, 4, 0, 4070, 0, 0, 1, 0, 1, 0, 4091, 4095, 1, 1, 1, 1, 0, 3, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 0, 4094, 0, 0, 0, 4096, 4096, 0, 10, 0, 0, 1, 0, 0, 0, 0, 12, 4096, 0, 34, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2781, 4095, 15, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 4076, 0, 0, 0, 4096, 0, 4096, 50, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 25, 0, 0, 0, 0, 0, 0, 0, 344, 4096, 0, 0, 0, 0, 4096, 4096, 4086, 0, 0, 0, 0, 4096, 0, 0, 1388, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2260, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 3931, 0, 0, 0, 4096, 0, 4096, 291, 0, 0, 8, 0, 4088, 4096, 4076, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4081, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4023, 0, 0, 0, 4096, 4096, 0, 16, 0, 0, 7, 0, 0, 0, 0, 185, 4096, 0, 2, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 1740, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2045, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 3892, 0, 0, 0, 4096, 0, 4096, 402, 0, 0, 8, 0, 4089, 4096, 4086, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4082, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4088, 0, 0, 0, 4096, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 69, 4096, 0, 29, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2621, 4096, 70, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 4027, 0, 0, 0, 4096, 0, 4096, 220, 0, 0, 0, 0, 4096, 4096, 4095, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[6, 11, 2, 13, 334, 0, 0, 0, 4085, 3, 12, 0, 4080, 3766, 11, 1639, 14, 11, 0, 15, 0, 0, 0, 47, 4019, 0, 1470, 0, 0, 4072, 4076, 4081, 0, 12, 9, 0, 4084, 14, 10, 1232, 0, 31, 10, 0, 10, 0, 240, 1, 4051, 3954, 14, 11, 11, 12, 4085, 0, 8, 0, 4082, 16, 11, 11, 0, 0, 11, 4084, 0, 10, 3847, 8, 11, 0, 3934, 278, 4016, 124, 14, 0, 0, 76, 4081, 4082, 0, 4083, 0, 3984, 12, 4025, 0, 3705, 10, 5, 14, 4082, 105, 3991, 1, 0, 1, 106, 0, 4086, 4018, 4080, 68, 4084, 0, 6, 35, 0, 4038, 0, 0, 0, 0, 4, 1, 4069, 4084, 15, 13, 10, 10, 24, 28, 0, 13, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4031, 0, 0, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 531, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 903, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2375, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 0, 3993, 0, 0, 0, 4096, 0, 4096, 459, 0, 0, 0, 0, 4096, 4096, 4089, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4076, 0, 0, 0, 4096, 4096, 0, 9, 2258, 1018, 377, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 165, 4096, 0, 0, 0, 0, 4096, 3719, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1837, 0, 0, 4096, 0, 0, 0, 4096, 3848, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 1, 0, 0, 4096, 0, 4096, 0, 3924, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 0, 3887, 4096, 4087, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 2897, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16123, 16173, 15290, 16400, 17973, 16394, 16338, 16381] avg=16384.0000 max=17973 max_rank=4 min=15290 min_rank=2 imbalance_ratio=1.096985 max_load_reduction=113099 max_load_reduction_pct=0.862877 moved_to_echo=67086 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 1, 80, 0, 0, 0, 4095, 0, 0, 0, 4095, 4037, 1, 399, 2258, 1018, 377, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 23, 4092, 0, 208, 0, 0, 4094, 3718, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 1, 1, 1479, 0, 2, 1, 0, 1, 0, 46, 0, 4095, 4095, 2900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4081, 1, 1, 1, 1, 4095, 0, 1, 0, 4095, 3, 1, 1, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1837, 0, 1, 4059, 0, 1, 0, 4092, 2048, 4087, 10, 1, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 0, 4083, 1, 4084, 0, 3937, 1, 0, 2, 0, 14, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 1, 19, 0, 3888, 4093, 4095, 12, 0, 0, 0, 4, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4091, 4095, 1, 1, 1, 1, 0, 3, 0, 1, 0, 4095, 2896, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16554, 16234, 16718, 16388, 16147, 16394, 16255, 16382] avg=16384.0000 max=16718 max_rank=2 min=16147 min_rank=4 imbalance_ratio=1.020386 max_load_reduction=114354 max_load_reduction_pct=0.872452 moved_to_echo=65178 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4, 0, 0, 0, 4094, 0, 0, 0, 4096, 4096, 0, 10, 2258, 1018, 377, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 12, 4096, 0, 34, 0, 0, 4095, 3719, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1838, 0, 0, 4096, 0, 0, 0, 4096, 2781, 4095, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 4096, 0, 4096, 0, 4076, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 3889, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2897, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16145, 16053, 16402, 16400, 16921, 16545, 16227, 16379] avg=16384.0000 max=16921 max_rank=4 min=16053 min_rank=1 imbalance_ratio=1.032776 max_load_reduction=114151 max_load_reduction_pct=0.870903 moved_to_echo=66072 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 1, 4023, 0, 0, 0, 4096, 4096, 0, 16, 2263, 1024, 380, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 185, 4096, 0, 2, 0, 0, 4096, 3716, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1740, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1833, 0, 0, 4096, 0, 0, 0, 4096, 2045, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 0, 0, 4096, 0, 4096, 0, 3892, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 0, 8, 0, 3872, 4096, 4086, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 2901, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16096, 16198, 16930, 16406, 16166, 16351, 16553, 16372] avg=16384.0000 max=16930 max_rank=2 min=16096 min_rank=0 imbalance_ratio=1.033325 max_load_reduction=114142 max_load_reduction_pct=0.870834 moved_to_echo=65442 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[6, 11, 2, 13, 334, 0, 0, 0, 4085, 3, 12, 0, 4080, 3766, 11, 1639, 2252, 1015, 375, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 11, 0, 15, 0, 0, 0, 47, 4019, 0, 1470, 0, 0, 4072, 3701, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 9, 0, 4084, 14, 10, 1232, 0, 31, 10, 0, 10, 0, 240, 1, 4084, 4083, 2892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 3954, 14, 11, 11, 12, 4085, 0, 8, 0, 4082, 16, 11, 11, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1832, 0, 10, 3847, 8, 11, 0, 3934, 278, 4016, 124, 14, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 3984, 12, 4025, 0, 3705, 10, 5, 14, 0, 105, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 106, 0, 3879, 4018, 4080, 68, 0, 0, 6, 35, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 4069, 4084, 15, 13, 10, 10, 24, 28, 0, 13, 0, 4082, 2887, 1190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17796, 17430, 16712, 16281, 14161, 16030, 16232, 16430] avg=16384.0000 max=17796 max_rank=0 min=14161 min_rank=4 imbalance_ratio=1.086182 max_load_reduction=113276 max_load_reduction_pct=0.864227 moved_to_echo=62853 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4031, 0, 0, 0, 4096, 4096, 0, 2, 2258, 1018, 377, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 4096, 0, 0, 0, 0, 4096, 3719, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 903, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1838, 0, 0, 4096, 0, 0, 0, 4096, 2375, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 4096, 0, 4096, 0, 3993, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 459, 0, 0, 0, 0, 3889, 4096, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2897, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16071, 16538, 16093, 16399, 16501, 16462, 16629, 16379] avg=16384.0000 max=16629 max_rank=6 min=16071 min_rank=0 imbalance_ratio=1.014954 max_load_reduction=114443 max_load_reduction_pct=0.873131 moved_to_echo=65971 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 25, 2258, 1018, 377, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 4096, 0, 0, 0, 0, 4096, 3719, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1388, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1838, 0, 0, 4096, 0, 0, 0, 4096, 2260, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 4096, 0, 4096, 0, 3931, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 0, 0, 8, 0, 3881, 4096, 4076, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 2897, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16157, 16341, 16578, 16399, 16386, 16400, 16442, 16369] avg=16384.0000 max=16578 max_rank=2 min=16157 min_rank=0 imbalance_ratio=1.011841 max_load_reduction=114494 max_load_reduction_pct=0.873520 moved_to_echo=65597 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 0, 0, 0, 4088, 0, 0, 0, 4096, 4096, 0, 4, 2258, 1018, 377, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 4096, 0, 29, 0, 0, 4096, 3719, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1838, 0, 0, 4096, 0, 0, 0, 4096, 2621, 4096, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 4096, 0, 4096, 0, 4027, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 3889, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2897, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16130, 16105, 16349, 16399, 16817, 16496, 16396, 16380] avg=16384.0000 max=16817 max_rank=4 min=16105 min_rank=1 imbalance_ratio=1.026428 max_load_reduction=114255 max_load_reduction_pct=0.871696 moved_to_echo=66089 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[1792, 0, 128, 256, 0, 31104, 32768, 32768, 128, 0, 0, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 0, 128, 128, 32768, 32768, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 23296, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 128, 9216, 0, 128, 128, 0, 128, 0, 384, 128, 32768, 32768, 23296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 32640, 128, 128, 128, 128, 32768, 128, 128, 0, 32768, 128, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 14720, 0, 128, 32512, 128, 128, 0, 32640, 18304, 32768, 256, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 1408, 32768, 0, 1792, 0, 0, 32768, 29824, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1536, 0, 128, 0, 0, 32768, 128, 32768, 0, 31488, 128, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 512, 0, 0, 128, 32640, 128, 128, 0, 32768, 32384, 128, 2176, 18176, 8192, 3072, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 1, 0, 0, 4096, 4096, 1, 4095, 0, 4096, 0, 0, 4096, 0, 1890, 0, 4096, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 6, 0, 2205, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 1084, 0, 6, 0, 4096, 0, 0, 4093, 1, 0, 11, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3136, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 64, 4096, 0, 0, 0, 0, 1, 0, 3912, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 2066, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 4096, 4096, 0, 0, 958, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 930, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4033, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 209, 4096, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 4094, 4095, 0, 4095, 0, 4068, 1, 0, 4095, 1, 432, 0, 4093, 1, 1, 4049, 0, 1, 0, 1, 1, 139, 4095, 223, 1, 4095, 4082, 0, 0, 616, 1, 1, 1, 0, 0, 1, 1, 4095, 15, 4095, 0, 1, 1, 0, 2264, 0, 0, 1, 4094, 1, 0, 4095, 0, 0, 0, 4095, 0, 1, 1, 0, 4007, 2, 4082, 3, 1, 4, 4094, 0, 1, 0, 3946, 1, 1, 1, 4095, 1, 38, 0, 1, 1, 2, 8, 1, 4094, 912, 4095, 1, 0, 0, 0, 0, 1, 3818, 0, 1, 54, 0, 4091, 4095, 0, 0, 26, 4095, 1, 13, 0, 1, 0, 1, 1, 0, 1, 0, 0, 4048, 0, 0, 4094, 4080, 4092, 28, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 1804, 0, 4096, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 17, 0, 2071, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 1181, 0, 6, 0, 4096, 0, 0, 4096, 1, 0, 22, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 3407, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 3835, 0, 0, 0, 0, 4096, 4089, 0, 0, 8, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 1040, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 4096, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 2786, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3980, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 98, 4096, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 3, 0, 4096, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4078, 0, 0, 21, 4069, 4084, 0, 4083, 0, 3905, 14, 0, 4081, 11, 11, 0, 4067, 0, 5, 4008, 0, 2, 0, 12, 13, 722, 4082, 1222, 14, 4080, 3955, 0, 0, 336, 16, 7, 13, 3, 0, 11, 12, 4085, 152, 4083, 0, 11, 7, 0, 916, 0, 0, 11, 4075, 13, 0, 4086, 0, 0, 0, 4084, 8, 12, 12, 0, 3456, 18, 3927, 16, 14, 23, 4069, 0, 14, 0, 3609, 11, 10, 0, 4085, 11, 266, 7, 3, 10, 18, 99, 8, 3983, 2173, 4082, 9, 1, 0, 17, 0, 9, 3721, 0, 10, 230, 0, 3975, 4083, 0, 0, 117, 4085, 8, 137, 0, 5, 0, 9, 11, 0, 13, 23, 0, 3719, 0, 0, 4083, 4062, 4060, 281, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 1611, 0, 4096, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 8, 0, 703, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 4096, 0, 0, 0, 0, 1852, 0, 4, 0, 4096, 0, 0, 4096, 0, 0, 16, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 4039, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 89, 4091, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 0, 0, 4096, 4090, 0, 0, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 1614, 0, 4096, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 3, 0, 2310, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 968, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3647, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 4096, 0, 0, 0, 0, 0, 0, 3773, 0, 0, 3, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 1, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 2816, 0, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2205, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 2395, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 0, 6, 0, 4096, 0, 0, 4093, 1, 0, 11, 4096, 0, 0, 0, 0, 1280, 1225, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 476, 0, 0, 0, 3136, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 64, 4096, 0, 0, 0, 0, 1, 0, 4096, 3610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16996, 16135, 17372, 15977, 15894, 15963, 16332, 16403] avg=16384.0000 max=17372 max_rank=2 min=15894 min_rank=4 imbalance_ratio=1.060303 max_load_reduction=113700 max_load_reduction_pct=0.867462 moved_to_echo=64592 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 2816, 0, 2066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3853, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 958, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2395, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 930, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1280, 1225, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 476, 0, 0, 0, 4033, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 209, 4096, 0, 0, 0, 0, 0, 0, 4096, 3610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17170, 16143, 16127, 15808, 16797, 16107, 16516, 16404] avg=16384.0000 max=17170 max_rank=0 min=15808 min_rank=3 imbalance_ratio=1.047974 max_load_reduction=113902 max_load_reduction_pct=0.869003 moved_to_echo=65824 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 2814, 0, 1804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3847, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2071, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2398, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 0, 6, 0, 4096, 0, 0, 4096, 1, 0, 22, 4096, 0, 0, 0, 0, 1282, 1227, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 471, 0, 0, 0, 3407, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 3610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3835, 0, 0, 0, 0, 4096, 4089, 0, 0, 8, 4093, 0, 0, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16906, 16122, 17243, 16094, 16148, 15892, 16260, 16407] avg=16384.0000 max=17243 max_rank=2 min=15892 min_rank=5 imbalance_ratio=1.052429 max_load_reduction=113829 max_load_reduction_pct=0.868446 moved_to_echo=64707 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 4094, 0, 0, 0, 0, 4068, 1, 0, 2816, 1, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 1, 4049, 0, 1, 0, 1, 1, 139, 0, 223, 1, 3852, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 616, 1, 1, 1, 0, 0, 1, 1, 4095, 15, 4095, 0, 1, 1, 0, 4095, 2394, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2264, 0, 0, 1, 4094, 1, 0, 4095, 0, 0, 0, 4095, 0, 1, 1, 0, 1279, 1224, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 2, 4082, 3, 1, 4, 476, 0, 1, 0, 3946, 1, 1, 1, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 1, 1, 2, 8, 1, 4094, 912, 4095, 1, 0, 0, 0, 0, 1, 4095, 3610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3818, 0, 1, 54, 0, 4091, 4095, 0, 0, 26, 4095, 1, 13, 0, 1, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 4048, 0, 0, 4094, 4080, 4092, 28, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15508, 16444, 15802, 17140, 16621, 16859, 16333, 16365] avg=16384.0000 max=17140 max_rank=3 min=15508 min_rank=0 imbalance_ratio=1.046143 max_load_reduction=113932 max_load_reduction_pct=0.869232 moved_to_echo=66178 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4078, 0, 0, 21, 4069, 0, 0, 0, 0, 3905, 14, 0, 2806, 11, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 5, 4008, 0, 2, 0, 12, 13, 722, 0, 1222, 14, 3839, 3955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 336, 16, 7, 13, 3, 0, 11, 12, 4085, 152, 4083, 0, 11, 7, 0, 4082, 2379, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 916, 0, 0, 11, 4075, 13, 0, 4086, 0, 0, 0, 4084, 8, 12, 12, 0, 1275, 1217, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3456, 18, 3927, 16, 14, 23, 473, 0, 14, 0, 3609, 11, 10, 0, 4085, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 7, 3, 10, 18, 99, 8, 3983, 2173, 4082, 9, 1, 0, 17, 0, 9, 4084, 3599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3721, 0, 10, 230, 0, 3975, 4083, 0, 0, 117, 4085, 8, 137, 0, 5, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 11, 0, 13, 23, 0, 3719, 0, 0, 4083, 4062, 4060, 281, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14915, 17859, 15681, 15793, 15667, 18368, 16508, 16281] avg=16384.0000 max=18368 max_rank=5 min=14915 min_rank=0 imbalance_ratio=1.121094 max_load_reduction=112704 max_load_reduction_pct=0.859863 moved_to_echo=66824 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 2816, 0, 1040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3853, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2395, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2786, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1280, 1225, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 476, 0, 0, 0, 3980, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 98, 4096, 0, 0, 0, 0, 0, 0, 4096, 3611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 3, 0, 4096, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16144, 16135, 15498, 17664, 16744, 15997, 16486, 16404] avg=16384.0000 max=17664 max_rank=3 min=15498 min_rank=2 imbalance_ratio=1.078125 max_load_reduction=113408 max_load_reduction_pct=0.865234 moved_to_echo=65631 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 2816, 0, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2310, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2395, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 968, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1280, 1225, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 476, 0, 0, 0, 3647, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 4096, 0, 0, 0, 0, 0, 0, 4096, 3611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3773, 0, 0, 3, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16718, 16108, 17478, 15846, 16411, 15904, 16203, 16404] avg=16384.0000 max=17478 max_rank=2 min=15846 min_rank=3 imbalance_ratio=1.066772 max_load_reduction=113594 max_load_reduction_pct=0.866653 moved_to_echo=64922 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 2816, 0, 1611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 703, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 4096, 0, 0, 0, 0, 4096, 2395, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1852, 0, 4, 0, 4096, 0, 0, 4096, 0, 0, 16, 4096, 0, 0, 0, 0, 1280, 1225, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 476, 0, 0, 0, 4039, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 89, 4091, 0, 0, 0, 0, 0, 0, 4096, 3610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 0, 0, 4096, 4090, 0, 0, 3, 4094, 0, 0, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16715, 16126, 15871, 16750, 16790, 15982, 16434, 16404] avg=16384.0000 max=16790 max_rank=4 min=15871 min_rank=2 imbalance_ratio=1.024780 max_load_reduction=114282 max_load_reduction_pct=0.871902 moved_to_echo=65610 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 0, 128, 32768, 0, 128, 0, 0, 32640, 128, 0, 22528, 128, 10496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32128, 128, 32640, 128, 128, 128, 3840, 0, 128, 0, 29824, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 32640, 0, 128, 0, 128, 128, 896, 0, 1536, 128, 30848, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 9600, 128, 128, 128, 128, 0, 128, 128, 32768, 256, 32768, 0, 128, 128, 0, 32768, 19200, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[31232, 0, 128, 384, 0, 32768, 32768, 0, 0, 256, 32768, 128, 256, 0, 128, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[12032, 0, 128, 128, 32768, 128, 0, 32768, 128, 0, 128, 32768, 128, 128, 128, 0, 10240, 9856, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 128, 0, 32384, 0, 0, 32768, 32768, 32768, 384, 0, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[384, 128, 128, 128, 128, 128, 128, 32768, 3584, 32768, 128, 128, 0, 128, 128, 128, 32768, 28928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 61, 0, 4096, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 329, 3670, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[3, 14, 4060, 13, 0, 13, 0, 0, 4071, 11, 11, 13, 35, 11, 384, 21, 0, 13, 4084, 4, 0, 0, 0, 3, 0, 0, 0, 12, 13, 4083, 4013, 9, 12, 18, 0, 0, 4074, 0, 4085, 7, 0, 3914, 32, 1, 4080, 12, 0, 29, 0, 25, 9, 0, 5, 4041, 5, 4082, 4074, 0, 11, 0, 4080, 4084, 28, 0, 2108, 0, 0, 3757, 0, 12, 0, 13, 10, 4051, 12, 8, 13, 53, 20, 4084, 10, 34, 4084, 4084, 4078, 7, 0, 4083, 4081, 4048, 0, 0, 0, 0, 0, 4082, 3986, 8, 1, 4033, 0, 0, 122, 0, 2, 4082, 31, 0, 0, 7, 2015, 4073, 0, 12, 0, 0, 0, 15, 4065, 0, 18, 12, 4053, 89, 4053, 0, 4, 57] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4096, 0, 4082, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4091, 4096, 0, 0, 4068, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4091, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 7, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 44, 4037, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 134, 0, 4096, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 4096, 4, 4078, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4090, 4096, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4091, 4088, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 7, 0, 0, 15, 0, 4096, 0, 0, 0, 0, 77, 3846, 0, 0, 0, 0, 5, 0, 4091, 4, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 94, 0, 4096, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 4096, 1, 4090, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 1, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 4096, 0, 0, 4095, 4095, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 2, 0, 0, 6, 0, 4095, 0, 0, 0, 0, 28, 3959, 1, 0, 0, 0, 1, 0, 4092, 2, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 24, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 0, 11, 4062, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 2, 4093, 1, 0, 0, 0, 0, 4094, 1, 1, 0, 3, 1, 67, 2, 24, 1, 4095, 1, 0, 0, 1, 4, 0, 0, 3, 1, 1, 4094, 4084, 1, 1, 2, 0, 0, 4094, 0, 4095, 0, 0, 4076, 3, 0, 4094, 1, 0, 3, 0, 1, 0, 0, 0, 4090, 1, 4094, 4094, 0, 1, 0, 4094, 4095, 18, 0, 3681, 0, 0, 4044, 0, 1, 0, 1, 1, 4091, 1, 2, 1, 6, 3, 4095, 0, 3, 4095, 4095, 4094, 1, 0, 4095, 4095, 4091, 0, 0, 0, 0, 0, 4095, 4074, 1, 0, 4088, 0, 0, 8, 0, 1, 4095, 2, 0, 0, 1, 562, 3927, 0, 1, 0, 0, 0, 1, 4089, 0, 2, 1, 4092, 11, 4092, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 24, 0, 4096, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 168, 3872, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 4096, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 4096, 4096, 0, 3962, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 218, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 3856, 0, 0, 0, 0, 329, 3670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4022, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16319, 16437, 16406, 16388, 16725, 16394, 16046, 16357] avg=16384.0000 max=16725 max_rank=4 min=16046 min_rank=6 imbalance_ratio=1.020813 max_load_reduction=114347 max_load_reduction_pct=0.872398 moved_to_echo=65522 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[3, 14, 4060, 13, 0, 13, 0, 0, 4071, 11, 11, 13, 35, 11, 384, 21, 4084, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4084, 4, 0, 0, 0, 3, 0, 0, 0, 12, 13, 4083, 4013, 9, 3949, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 18, 0, 0, 4074, 0, 4085, 7, 0, 3914, 32, 1, 4080, 12, 0, 29, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 9, 0, 5, 4041, 5, 4082, 4074, 0, 11, 0, 4080, 4, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2108, 0, 0, 3757, 0, 12, 0, 13, 10, 4051, 12, 8, 13, 53, 20, 4084, 218, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 34, 0, 0, 4078, 7, 0, 4083, 4081, 4048, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 8, 1, 4033, 0, 0, 122, 0, 2, 3842, 31, 0, 0, 7, 2015, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 15, 4065, 0, 18, 12, 4053, 89, 4053, 0, 4, 57, 4010, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16763, 16253, 16286, 16364, 14483, 16350, 18120, 16453] avg=16384.0000 max=18120 max_rank=6 min=14483 min_rank=4 imbalance_ratio=1.105957 max_load_reduction=112952 max_load_reduction_pct=0.861755 moved_to_echo=65406 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 4096, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 4095, 4096, 0, 3962, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4090, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 218, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4095, 4096, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 6, 0, 3855, 0, 0, 0, 0, 28, 3959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 4092, 2, 0, 0, 4096, 0, 4096, 0, 0, 0, 4022, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16319, 16428, 16401, 16389, 16725, 16392, 16042, 16376] avg=16384.0000 max=16725 max_rank=4 min=16042 min_rank=6 imbalance_ratio=1.020813 max_load_reduction=114347 max_load_reduction_pct=0.872398 moved_to_echo=65535 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 4096, 0, 3962, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 218, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 10, 0, 3856, 0, 0, 0, 0, 11, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4022, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16318, 16347, 16396, 16388, 16726, 16394, 16131, 16372] avg=16384.0000 max=16726 max_rank=4 min=16131 min_rank=6 imbalance_ratio=1.020874 max_load_reduction=114346 max_load_reduction_pct=0.872391 moved_to_echo=65623 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 4096, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 4093, 4096, 0, 3966, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4078, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 225, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 7, 0, 0, 15, 0, 3842, 0, 0, 0, 0, 77, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4091, 4, 0, 0, 4096, 0, 4096, 0, 0, 0, 4023, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16324, 16514, 16399, 16378, 16726, 16371, 15979, 16381] avg=16384.0000 max=16726 max_rank=4 min=15979 min_rank=6 imbalance_ratio=1.020874 max_load_reduction=114346 max_load_reduction_pct=0.872391 moved_to_echo=65457 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4093, 1, 0, 0, 0, 0, 4094, 1, 1, 0, 3, 1, 67, 2, 4095, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 1, 4095, 1, 0, 0, 1, 4, 0, 0, 3, 1, 1, 4094, 4084, 1, 3961, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 4094, 0, 4095, 0, 0, 4076, 3, 0, 4094, 1, 0, 3, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4090, 1, 4094, 4094, 0, 1, 0, 4094, 4, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3681, 0, 0, 4044, 0, 1, 0, 1, 1, 4091, 1, 2, 1, 6, 3, 4095, 218, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4094, 1, 0, 4095, 4095, 4091, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 1, 0, 4088, 0, 0, 8, 0, 1, 3855, 2, 0, 0, 1, 562, 3927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4089, 0, 2, 1, 4092, 11, 4092, 0, 1, 0, 4021, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16389, 16341, 16391, 16397, 16269, 16389, 16519, 16377] avg=16384.0000 max=16519 max_rank=6 min=16269 min_rank=4 imbalance_ratio=1.008240 max_load_reduction=114553 max_load_reduction_pct=0.873970 moved_to_echo=65554 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 4096, 4095, 0, 3962, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4096, 218, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4091, 4096, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 7, 0, 0, 10, 0, 3856, 0, 0, 0, 0, 44, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 4096, 0, 4096, 0, 0, 0, 4025, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16323, 16372, 16392, 16380, 16692, 16388, 16146, 16379] avg=16384.0000 max=16692 max_rank=4 min=16146 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=114380 max_load_reduction_pct=0.872650 moved_to_echo=65605 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 4096, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 4096, 4096, 0, 3962, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 218, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 3856, 0, 0, 0, 0, 168, 3872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4022, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16317, 16380, 16401, 16388, 16726, 16394, 16089, 16377] avg=16384.0000 max=16726 max_rank=4 min=16089 min_rank=6 imbalance_ratio=1.020874 max_load_reduction=114346 max_load_reduction_pct=0.872391 moved_to_echo=65586 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 32768, 128, 0, 128, 128, 0, 32768, 128, 128, 128, 128, 128, 512, 128, 32768, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32640, 128, 128, 32768, 128, 0, 256, 128, 128, 30848, 128, 0, 0, 128, 3328, 31488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[30336, 128, 0, 32384, 0, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 128, 32768, 1792, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 0, 128, 128, 32768, 128, 128, 128, 32768, 128, 32768, 0, 128, 128, 32256, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 32768, 128, 0, 32640, 128, 128, 32768, 128, 0, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 128, 32768, 128, 0, 0, 128, 256, 0, 0, 128, 128, 128, 32768, 32768, 128, 31744, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 128, 32768, 128, 32768, 32768, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 32768, 128, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4094, 0, 0, 1, 11, 0, 4092, 4094, 0, 0, 54, 4095, 3474, 4092, 0, 4094, 1, 0, 1, 0, 4095, 4095, 0, 214, 1, 1, 1327, 0, 0, 1, 0, 6, 0, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 2, 1, 12, 0, 2, 0, 2, 1, 1, 4003, 2, 1, 4095, 1, 0, 2, 12, 0, 0, 4095, 0, 1, 3, 1, 1, 1, 0, 4095, 1, 0, 4088, 4092, 1, 4095, 4076, 4093, 4092, 17, 2, 0, 4082, 4095, 1, 1, 0, 4093, 0, 0, 0, 0, 1, 0, 1, 0, 3, 0, 4095, 4095, 0, 1, 4095, 0, 4094, 0, 4095, 4095, 0, 1, 0, 1, 16, 4095, 0, 4095, 3179, 1, 0, 0, 1, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3957, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 37, 0, 0, 169, 0, 0, 0, 42, 9, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 17, 4088, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 3975, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 224, 0, 4096, 5, 4096, 0, 4079, 4096, 0, 0, 0, 0, 0, 4063, 0, 4089, 3906, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 4081, 0, 0, 7, 83, 1, 4057, 4076, 0, 0, 531, 4082, 3924, 4053, 0, 4073, 1, 2, 6, 0, 4086, 4085, 2, 74, 13, 11, 1213, 0, 0, 14, 0, 22, 0, 14, 7, 4, 11, 4073, 0, 10, 12, 10, 23, 2, 76, 0, 24, 0, 25, 10, 21, 3574, 0, 15, 4082, 12, 0, 28, 171, 6, 0, 4082, 14, 17, 32, 11, 11, 10, 0, 4085, 5, 13, 3956, 4049, 10, 4086, 3980, 4060, 4058, 121, 26, 0, 3982, 4082, 0, 8, 0, 4063, 0, 0, 0, 0, 17, 0, 12, 0, 23, 0, 4084, 4084, 0, 13, 4083, 0, 4078, 0, 4083, 4077, 0, 12, 0, 13, 131, 4084, 0, 4081, 2689, 10, 0, 0, 7, 0, 11, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 4096, 3671, 4096, 0, 4096, 0, 0, 0, 0, 4095, 4096, 0, 44, 0, 0, 463, 0, 0, 0, 23, 2, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 8, 4095, 0, 0, 4096, 4096, 0, 4095, 4096, 4096, 4096, 0, 0, 0, 4096, 4049, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 123, 0, 4096, 1, 4096, 0, 4089, 4096, 0, 0, 0, 0, 0, 4085, 0, 4095, 3909, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3989, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 70, 0, 0, 138, 0, 0, 0, 41, 11, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 36, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4004, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 193, 0, 4096, 0, 4096, 0, 4087, 4096, 0, 0, 0, 0, 0, 4049, 0, 4096, 3862, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3591, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 57, 0, 0, 1750, 0, 0, 0, 23, 13, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 186, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 14, 4089, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4044, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 133, 0, 4096, 0, 4096, 0, 4083, 4096, 0, 0, 0, 0, 0, 4080, 0, 4095, 2505, 0, 0, 0, 0, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4096, 2970, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 6, 0, 0, 1381, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 163, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 18, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4077, 0, 4096, 3677, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 3229, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 15, 0, 0, 563, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 362, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 4096, 0, 0, 0, 4096, 4071, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 47, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 4080, 0, 4096, 4005, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 4094, 0, 0, 1, 11, 0, 4092, 4094, 0, 0, 54, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3474, 4092, 0, 4094, 1, 0, 1, 0, 3656, 0, 0, 214, 1, 1, 1327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 2, 1, 4095, 4095, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 2, 0, 2, 1, 1, 4003, 2, 1, 4095, 1, 0, 2, 12, 0, 4093, 3996, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 3, 1, 1, 1, 0, 4095, 1, 0, 4088, 4092, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 61, 4092, 17, 2, 0, 4082, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 3, 0, 0, 4008, 0, 1, 0, 0, 4094, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 16, 4095, 0, 4095, 3179, 1, 0, 0, 1, 0, 1, 0, 4032, 439, 89, 87, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16367, 16861, 16361, 16257, 16389, 16427, 16298, 16112] avg=16384.0000 max=16861 max_rank=1 min=16112 min_rank=7 imbalance_ratio=1.029114 max_load_reduction=114211 max_load_reduction_pct=0.871361 moved_to_echo=65226 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3957, 4096, 0, 4096, 0, 0, 0, 0, 3655, 0, 0, 37, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 9, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 6, 0, 4096, 4003, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 17, 4088, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 59, 4096, 0, 0, 0, 4096, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 224, 0, 0, 5, 4096, 0, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 4089, 3906, 0, 0, 0, 0, 0, 0, 0, 4037, 441, 92, 92, 79, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16305, 16010, 16407, 16331, 16394, 16322, 16504, 16799] avg=16384.0000 max=16799 max_rank=7 min=16010 min_rank=1 imbalance_ratio=1.025330 max_load_reduction=114273 max_load_reduction_pct=0.871834 moved_to_echo=66019 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 4096, 0, 4096, 0, 0, 0, 0, 3657, 0, 0, 70, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 11, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 36, 0, 4096, 3997, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 61, 4096, 0, 0, 0, 4096, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 193, 0, 0, 0, 4096, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 4096, 3862, 0, 0, 0, 0, 0, 0, 0, 4035, 439, 89, 87, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16309, 16046, 16402, 16355, 16394, 16353, 16481, 16732] avg=16384.0000 max=16732 max_rank=7 min=16046 min_rank=1 imbalance_ratio=1.021240 max_load_reduction=114340 max_load_reduction_pct=0.872345 moved_to_echo=65960 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3671, 4096, 0, 4096, 0, 0, 0, 0, 3656, 0, 0, 44, 0, 0, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 2, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 15, 0, 4096, 3996, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 8, 4095, 0, 0, 4096, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 61, 4096, 0, 0, 0, 4096, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 123, 0, 0, 1, 4096, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4095, 3909, 0, 0, 0, 0, 1, 0, 0, 4035, 439, 89, 87, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16310, 16026, 16376, 16333, 16401, 16398, 16413, 16815] avg=16384.0000 max=16815 max_rank=7 min=16026 min_rank=1 imbalance_ratio=1.026306 max_load_reduction=114257 max_load_reduction_pct=0.871712 moved_to_echo=66027 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 12, 0, 4081, 0, 0, 7, 83, 1, 4057, 4076, 0, 0, 531, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3924, 4053, 0, 4073, 1, 2, 6, 0, 3648, 0, 2, 74, 13, 11, 1213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 22, 0, 14, 7, 4, 11, 4073, 0, 10, 12, 10, 23, 2, 4085, 4084, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 24, 0, 25, 10, 21, 3574, 0, 15, 4082, 12, 0, 28, 171, 6, 4063, 3987, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 14, 17, 32, 11, 11, 10, 0, 4085, 5, 13, 3956, 4049, 10, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 61, 4058, 121, 26, 0, 3982, 4082, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 12, 0, 23, 0, 0, 3997, 0, 13, 0, 0, 4078, 0, 4083, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 13, 131, 4084, 0, 4081, 2689, 10, 0, 0, 7, 0, 11, 0, 3999, 438, 89, 87, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16855, 17020, 16420, 16128, 16305, 16318, 16300, 15726] avg=16384.0000 max=17020 max_rank=1 min=15726 min_rank=7 imbalance_ratio=1.038818 max_load_reduction=114052 max_load_reduction_pct=0.870148 moved_to_echo=64649 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2970, 4096, 0, 4096, 0, 0, 0, 0, 3657, 0, 0, 6, 0, 0, 1381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 163, 0, 4096, 3997, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 61, 4096, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 18, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4096, 3677, 0, 0, 0, 0, 0, 0, 0, 4035, 439, 89, 87, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16308, 16206, 16351, 16482, 16394, 16441, 16315, 16575] avg=16384.0000 max=16575 max_rank=7 min=16206 min_rank=1 imbalance_ratio=1.011658 max_load_reduction=114497 max_load_reduction_pct=0.873543 moved_to_echo=65725 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 4096, 0, 4096, 0, 0, 0, 0, 3657, 0, 0, 57, 0, 0, 1750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 13, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 186, 0, 4096, 3997, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 14, 4089, 0, 0, 4096, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 61, 4096, 0, 0, 0, 4096, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 133, 0, 0, 0, 4096, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 4095, 2505, 0, 0, 0, 0, 0, 0, 2, 4035, 439, 89, 87, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16309, 17247, 16393, 16505, 16401, 16393, 16417, 15407] avg=16384.0000 max=17247 max_rank=1 min=15407 min_rank=7 imbalance_ratio=1.052673 max_load_reduction=113825 max_load_reduction_pct=0.868416 moved_to_echo=64618 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3229, 4096, 0, 4096, 0, 0, 0, 0, 3657, 0, 0, 15, 0, 0, 563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 362, 0, 4096, 3997, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 61, 4096, 0, 0, 0, 4096, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 47, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 4096, 4005, 0, 0, 0, 0, 0, 0, 0, 4035, 439, 89, 87, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16309, 15656, 16362, 16681, 16394, 16420, 16344, 16906] avg=16384.0000 max=16906 max_rank=7 min=15656 min_rank=1 imbalance_ratio=1.031860 max_load_reduction=114166 max_load_reduction_pct=0.871017 moved_to_echo=66064 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32768, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 32640, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32640, 512, 32768, 256, 128, 0, 32640, 32512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 32768, 0, 0, 128, 128, 128, 32768, 32768, 128, 0, 640, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[28928, 32768, 0, 32768, 128, 128, 128, 0, 29312, 0, 128, 640, 128, 128, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 256, 128, 128, 128, 128, 128, 128, 32768, 0, 128, 128, 128, 128, 128, 32768, 32768, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 256, 32640, 0, 32768, 27776, 128, 0, 0, 128, 128, 128, 128, 32256, 3584, 768, 768, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 32256, 128, 128, 32768, 128, 0, 128, 1024, 128, 32768, 32000, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 32128, 768, 128, 0, 128, 32768, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[3, 0, 1, 0, 0, 1, 0, 0, 0, 114, 0, 4096, 0, 4096, 4095, 0, 0, 1, 4096, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 4091, 4096, 0, 4096, 0, 3813, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 844, 3637, 0, 0, 4096, 3859, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 13, 0, 6, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 0, 0, 2, 0, 0, 318, 0, 4071, 0, 4094, 4095, 0, 0, 0, 4095, 0, 1, 1, 4047, 4095, 8, 0, 1, 0, 64, 1, 4095, 4095, 0, 4093, 6, 3229, 1, 1, 4086, 1, 0, 1, 1, 1, 0, 187, 0, 1, 1, 1, 0, 0, 1, 0, 17, 4094, 4092, 3, 0, 2, 1313, 3528, 1, 0, 4094, 3932, 0, 1, 1, 3948, 1, 0, 0, 4087, 2, 1, 0, 0, 1, 1, 4088, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 1, 0, 0, 4093, 1, 1, 3, 4094, 3953, 1, 4095, 0, 0, 1, 4094, 1, 2, 1, 0, 2, 2, 4094, 4095, 0, 0, 4095, 0, 1, 1, 63, 1, 2, 0, 4095, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4086, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 470, 3604, 0, 0, 4096, 4065, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 4096, 0, 3984, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 818, 3016, 0, 0, 4096, 4027, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 18, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4000, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 325, 3457, 0, 0, 4096, 4092, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4090, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 4096, 0, 3925, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 5, 0, 960, 2952, 0, 0, 4096, 3996, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 14, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 15, 0, 0, 0, 0, 17, 0, 3, 111, 0, 3887, 0, 4077, 4085, 0, 0, 0, 4083, 0, 14, 3, 3744, 4080, 166, 0, 17, 0, 438, 9, 4079, 4083, 0, 4077, 56, 3845, 8, 9, 4067, 11, 0, 0, 14, 16, 0, 1434, 0, 14, 11, 15, 1, 0, 12, 0, 37, 4076, 4033, 25, 0, 31, 3921, 2362, 6, 0, 4049, 2463, 0, 19, 12, 3423, 11, 0, 13, 4001, 3, 13, 0, 1, 18, 7, 4041, 4059, 0, 0, 0, 0, 0, 31, 1, 1, 4066, 4084, 12, 0, 0, 4061, 1, 12, 36, 4079, 2470, 14, 4082, 0, 0, 12, 4075, 13, 17, 14, 0, 27, 17, 4080, 4078, 0, 0, 4084, 10, 0, 10, 350, 10, 24, 11, 4081, 4084, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 577, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 3830, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2242, 1599, 0, 0, 4096, 4035, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4091, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 1, 0, 0, 1, 0, 0, 0, 114, 0, 4096, 0, 4096, 4095, 0, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 4091, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3813, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 844, 3637, 0, 0, 3568, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3859, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 13, 0, 6, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 123, 54, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16233, 16349, 16137, 16476, 16543, 16412, 16592, 16330] avg=16384.0000 max=16592 max_rank=6 min=16137 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=114480 max_load_reduction_pct=0.873413 moved_to_echo=65877 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 4096, 0, 4096, 4096, 0, 3828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 30, 0, 0, 4096, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 470, 3604, 0, 0, 3568, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4065, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 123, 54, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16148, 16352, 16438, 16068, 16749, 16394, 16593, 16330] avg=16384.0000 max=16749 max_rank=4 min=16068 min_rank=3 imbalance_ratio=1.022278 max_load_reduction=114323 max_load_reduction_pct=0.872215 moved_to_echo=66066 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 0, 0, 2, 0, 0, 318, 0, 4071, 0, 4094, 4095, 0, 3827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4047, 4095, 8, 0, 1, 0, 64, 1, 4095, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 6, 3229, 1, 1, 4086, 1, 0, 1, 1, 1, 0, 187, 0, 1, 4095, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 17, 4094, 4092, 3, 0, 2, 1313, 3528, 1, 0, 3567, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3932, 0, 1, 1, 3948, 1, 0, 0, 4087, 2, 1, 0, 0, 1, 1, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 9, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 4094, 3953, 1, 4095, 0, 0, 1, 4094, 1, 2, 1, 0, 2, 123, 54, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 4095, 0, 0, 4095, 0, 1, 1, 63, 1, 2, 0, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16409, 16376, 15737, 16854, 16465, 16378, 16458, 16395] avg=16384.0000 max=16854 max_rank=3 min=15737 min_rank=2 imbalance_ratio=1.028687 max_load_reduction=114218 max_load_reduction_pct=0.871414 moved_to_echo=65696 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 4096, 0, 4096, 4096, 0, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4089, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3925, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 43, 0, 0, 4096, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 5, 0, 960, 2952, 0, 0, 3571, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3996, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 14, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 126, 59, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16519, 16343, 16288, 15914, 16683, 16398, 16601, 16326] avg=16384.0000 max=16683 max_rank=4 min=15914 min_rank=3 imbalance_ratio=1.018250 max_load_reduction=114389 max_load_reduction_pct=0.872719 moved_to_echo=66008 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 15, 0, 0, 0, 0, 17, 0, 3, 111, 0, 3887, 0, 4077, 4085, 0, 3817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3, 3744, 4080, 166, 0, 17, 0, 438, 9, 4079, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 56, 3845, 8, 9, 4067, 11, 0, 0, 14, 16, 0, 1434, 0, 14, 4083, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 15, 1, 0, 12, 0, 37, 4076, 4033, 25, 0, 31, 3921, 2362, 6, 0, 3557, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 2463, 0, 19, 12, 3423, 11, 0, 13, 4001, 3, 13, 0, 1, 18, 7, 395, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 4059, 0, 0, 0, 0, 0, 31, 1, 1, 4066, 10, 12, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 36, 4079, 2470, 14, 4082, 0, 0, 12, 4075, 13, 17, 14, 0, 27, 122, 54, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4080, 4078, 0, 0, 4084, 10, 0, 10, 350, 10, 24, 11, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16012, 16601, 17668, 18320, 14428, 16282, 15060, 16701] avg=16384.0000 max=18320 max_rank=3 min=14428 min_rank=4 imbalance_ratio=1.118164 max_load_reduction=112752 max_load_reduction_pct=0.860229 moved_to_echo=62471 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 4096, 0, 4096, 4096, 0, 3829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4089, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3984, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 23, 0, 0, 4096, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 818, 3016, 0, 0, 3568, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4027, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 18, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 123, 54, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16525, 16346, 16328, 15828, 16711, 16411, 16593, 16330] avg=16384.0000 max=16711 max_rank=4 min=15828 min_rank=3 imbalance_ratio=1.019958 max_load_reduction=114361 max_load_reduction_pct=0.872505 moved_to_echo=66045 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 577, 0, 4096, 0, 4096, 4096, 0, 3828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3830, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2242, 1599, 0, 0, 3568, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4035, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 123, 54, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16693, 16352, 16153, 15835, 16718, 16403, 16588, 16330] avg=16384.0000 max=16718 max_rank=4 min=15835 min_rank=3 imbalance_ratio=1.020386 max_load_reduction=114354 max_load_reduction_pct=0.872452 moved_to_echo=66039 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 0, 4096, 0, 4096, 4096, 0, 3828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 4096, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4000, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 325, 3457, 0, 0, 3568, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 123, 54, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16533, 16353, 16323, 15777, 16775, 16394, 16587, 16330] avg=16384.0000 max=16775 max_rank=4 min=15777 min_rank=3 imbalance_ratio=1.023865 max_load_reduction=114297 max_load_reduction_pct=0.872017 moved_to_echo=66086 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 128, 128, 128, 32384, 32768, 256, 0, 128, 0, 512, 128, 32768, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 128, 30720, 128, 128, 32768, 128, 0, 128, 128, 128, 0, 1792, 0, 128, 32768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 32768, 128, 0, 128, 0, 0, 128, 128, 128, 32768, 128, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 30592, 0, 128, 128, 32000, 128, 0, 128, 32768, 128, 128, 0, 128, 128, 128, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 32768, 32768, 128, 128, 128, 11008, 24192, 128, 0, 28544, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 0, 128, 128, 0, 128, 2432, 0, 32640, 0, 32768, 32768, 0, 30720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 32768, 31104, 128, 32768, 0, 0, 128, 32768, 128, 128, 128, 0, 128, 1024, 512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 32768, 0, 0, 32768, 128, 128, 128, 512, 128, 128, 128, 32384, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True + [2026-08-30 05:03:37] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 790.8 | throughput per GPU (TFLOP/s/GPU): 421.6 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.308431E+00 | loss scale: 1.0 | grad norm: 5.510 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1957, 2131, 2034, 3, 196, 0, 1945, 527, 1136, 2844, 507, 335, 1719, 2452, 0, 850, 720, 1546, 1091, 1686, 12, 813, 191, 2110, 108, 24, 3480, 2063, 1556, 84, 2330, 1355, 386, 8, 1526, 942, 361, 1075, 1272, 200, 281, 13, 2287, 2730, 13, 2184, 759, 460, 1412, 780, 574, 628, 11, 8, 6, 2422, 0, 447, 90, 646, 2110, 2194, 1391, 252, 1000, 31, 0, 2012, 1977, 952, 615, 1043, 1218, 1308, 2144, 125, 1666, 12, 683, 1759, 595, 372, 7, 2, 1362, 384, 1641, 76, 2300, 44, 878, 469, 214, 1977, 375, 2, 1855, 1133, 37, 1863, 10, 938, 1805, 780, 267, 1343, 58, 1428, 722, 2385, 2297, 1135, 1774, 2245, 196, 1713, 2258, 940, 3019, 1837, 1201, 2189, 1032, 685, 819, 8, 542, 42] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1825, 1802, 1697, 1, 239, 1, 2520, 208, 562, 3172, 1299, 496, 1284, 3068, 0, 1652, 696, 2130, 1457, 1509, 11, 861, 162, 1727, 34, 70, 3737, 2563, 1080, 40, 1410, 870, 835, 2, 1263, 627, 146, 1873, 1436, 206, 108, 1, 2088, 2860, 0, 2373, 713, 997, 1529, 1057, 484, 423, 2, 8, 13, 2488, 0, 275, 29, 383, 1276, 3285, 1335, 70, 772, 14, 1, 1582, 2680, 774, 323, 706, 1264, 1758, 1891, 129, 1488, 6, 1044, 2462, 784, 488, 0, 18, 1470, 543, 665, 36, 2290, 63, 1544, 1115, 136, 2733, 408, 0, 1074, 1054, 8, 1424, 2, 520, 1442, 805, 141, 1609, 44, 1144, 166, 2320, 2738, 2074, 2422, 1786, 376, 1590, 2234, 859, 2162, 938, 823, 2327, 1375, 493, 1029, 4, 526, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1956, 2522, 2070, 0, 50, 0, 2588, 374, 948, 3054, 915, 352, 1724, 3240, 0, 1437, 838, 1667, 1357, 1851, 51, 828, 180, 1846, 99, 18, 3528, 2226, 1952, 86, 2168, 804, 519, 3, 929, 558, 113, 1363, 1117, 311, 55, 2, 2383, 3174, 0, 2408, 618, 1044, 715, 826, 684, 215, 0, 22, 15, 1933, 0, 317, 22, 327, 1953, 3238, 1445, 62, 960, 5, 1, 1867, 2031, 1275, 578, 985, 1245, 1681, 2141, 69, 1514, 12, 979, 2039, 620, 561, 5, 41, 1909, 402, 1170, 25, 2047, 136, 1435, 571, 162, 2049, 383, 1, 1568, 1145, 8, 1420, 4, 460, 1186, 653, 178, 1354, 22, 1339, 218, 1811, 2831, 1096, 1984, 2292, 206, 1327, 2071, 974, 2548, 1681, 979, 2501, 1001, 816, 796, 6, 589, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1728, 2332, 1425, 0, 358, 0, 2709, 640, 1104, 2972, 492, 678, 1642, 2506, 0, 1214, 1168, 1226, 1231, 1573, 29, 599, 167, 1772, 143, 44, 3305, 1857, 1636, 41, 1953, 1090, 581, 1, 1161, 693, 194, 1818, 1405, 143, 120, 16, 2352, 2882, 6, 2445, 804, 652, 1237, 897, 643, 384, 9, 21, 47, 2317, 2, 252, 41, 424, 1577, 2741, 1414, 129, 991, 12, 0, 1763, 2368, 1300, 555, 834, 1469, 1674, 2170, 118, 1705, 14, 753, 2058, 663, 383, 1, 20, 1961, 538, 1298, 18, 2426, 37, 1453, 863, 153, 1667, 455, 3, 1731, 793, 8, 1699, 16, 599, 1037, 851, 149, 1342, 105, 1037, 381, 2468, 2671, 1355, 1619, 2101, 471, 1696, 1948, 946, 2846, 1714, 1075, 2699, 1166, 536, 530, 13, 656, 49] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1938, 2130, 1793, 0, 203, 1, 1679, 480, 951, 2253, 548, 263, 1221, 2859, 1, 1678, 1024, 1740, 1555, 1542, 23, 612, 169, 1972, 150, 23, 3536, 1849, 1083, 37, 2185, 1250, 724, 2, 1029, 250, 465, 1714, 2103, 137, 143, 4, 2718, 2925, 1, 2378, 634, 1107, 1146, 698, 455, 701, 3, 13, 21, 2375, 0, 169, 23, 273, 1913, 3051, 1416, 140, 1243, 16, 1, 1544, 2529, 1507, 641, 720, 1675, 2307, 2287, 219, 1409, 7, 760, 2064, 527, 477, 10, 12, 1821, 350, 1165, 23, 2746, 29, 925, 697, 113, 1949, 321, 2, 1211, 897, 4, 1898, 18, 428, 1787, 682, 147, 1625, 53, 1767, 543, 1919, 2354, 1585, 1838, 1983, 270, 1238, 2631, 1013, 1762, 1666, 1553, 1700, 1217, 564, 664, 13, 461, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1324, 1992, 2123, 2, 256, 0, 2182, 214, 338, 2470, 352, 623, 1879, 2405, 0, 1781, 1397, 1708, 1562, 1751, 29, 587, 156, 2063, 169, 16, 3525, 2121, 1300, 39, 1767, 1382, 911, 4, 1241, 532, 160, 1137, 1520, 138, 333, 5, 2082, 3099, 0, 2054, 967, 792, 1056, 869, 487, 453, 2, 13, 15, 2805, 0, 119, 15, 609, 1660, 2999, 1405, 167, 1084, 15, 0, 1709, 2611, 908, 485, 917, 1625, 1993, 1900, 90, 1417, 5, 649, 1643, 628, 315, 3, 22, 1866, 529, 709, 9, 2211, 75, 1284, 970, 418, 1815, 397, 1, 1399, 1299, 6, 2206, 3, 620, 1297, 781, 216, 1400, 73, 1674, 329, 2124, 2423, 1946, 2331, 2181, 372, 1376, 2322, 752, 1889, 1769, 1226, 2163, 1272, 1119, 545, 1, 486, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1603, 2230, 1962, 1, 251, 1, 2390, 264, 816, 2874, 620, 283, 1089, 3147, 0, 1385, 1111, 1832, 1556, 1422, 11, 724, 165, 1805, 92, 21, 3593, 2172, 1603, 44, 1842, 1036, 429, 2, 1367, 566, 129, 1438, 1260, 151, 183, 1, 2055, 3034, 1, 2487, 848, 778, 1243, 868, 751, 242, 7, 29, 22, 1803, 1, 686, 1, 338, 1646, 2932, 1374, 85, 1084, 32, 0, 1475, 1817, 1064, 469, 1011, 1165, 2032, 2347, 128, 1164, 25, 1221, 2070, 788, 535, 7, 15, 1312, 567, 1007, 22, 1967, 111, 1165, 960, 144, 2531, 466, 2, 1384, 1082, 9, 2205, 1, 689, 1823, 834, 258, 1347, 76, 1059, 288, 2121, 2666, 1848, 2390, 2427, 322, 1149, 2051, 695, 2825, 1526, 1140, 2384, 1057, 724, 837, 5, 444, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1843, 2180, 1765, 0, 295, 0, 2496, 205, 706, 2427, 596, 632, 1777, 2601, 0, 1248, 793, 1552, 1938, 1384, 24, 1073, 143, 1794, 55, 42, 3493, 2034, 1353, 68, 1802, 1517, 688, 0, 1419, 264, 147, 1657, 1447, 188, 45, 1, 1926, 2765, 0, 2132, 845, 696, 1144, 1248, 465, 322, 5, 8, 3, 2165, 0, 105, 8, 281, 1868, 3002, 998, 111, 749, 34, 0, 1367, 2444, 1051, 436, 1009, 1262, 1959, 2232, 92, 1822, 2, 589, 1974, 617, 278, 2, 13, 1217, 446, 747, 29, 2720, 19, 1855, 1056, 230, 2517, 287, 0, 1504, 1060, 5, 1727, 1, 587, 1985, 1028, 132, 1638, 26, 1331, 571, 2044, 2412, 1995, 2346, 2010, 268, 886, 2797, 751, 2426, 1752, 1134, 2284, 1374, 802, 838, 5, 506, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1957, 2131, 2034, 3, 196, 0, 1945, 527, 1136, 2844, 507, 335, 1719, 26, 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 720, 1546, 1091, 1686, 12, 813, 191, 2110, 108, 24, 988, 2063, 1556, 84, 2330, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 386, 8, 1526, 942, 361, 1075, 1272, 200, 281, 13, 2287, 2730, 13, 2184, 759, 460, 623, 664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1412, 780, 574, 628, 11, 8, 6, 2422, 0, 447, 90, 646, 2110, 2194, 1391, 252, 3019, 818, 453, 235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 31, 0, 2012, 1159, 952, 615, 1043, 1218, 1308, 2144, 125, 1666, 12, 683, 1759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 595, 372, 7, 2, 1362, 384, 1641, 76, 2300, 44, 878, 469, 214, 1977, 375, 2, 2426, 2039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1855, 1133, 37, 1863, 10, 938, 1805, 780, 267, 1343, 58, 1428, 722, 2385, 1674, 1135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1774, 2245, 196, 1713, 1359, 940, 0, 1837, 1201, 2189, 1032, 685, 819, 8, 542, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16210, 16677, 15784, 17496, 15727, 15163, 17433, 16582] avg=16384.0000 max=17496 max_rank=3 min=15163 min_rank=5 imbalance_ratio=1.067871 max_load_reduction=113576 max_load_reduction_pct=0.866516 moved_to_echo=64905 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1938, 2130, 1793, 0, 203, 1, 1679, 480, 951, 2253, 548, 263, 1221, 30, 1, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 1740, 1555, 1542, 23, 612, 169, 1972, 150, 23, 1004, 1849, 1083, 37, 2185, 1250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 724, 2, 1029, 250, 465, 1714, 2103, 137, 143, 4, 2718, 2925, 1, 2378, 634, 1107, 638, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1146, 698, 455, 701, 3, 13, 21, 2375, 0, 169, 23, 273, 1913, 3051, 1416, 140, 1762, 1046, 460, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1243, 16, 1, 1544, 1483, 1507, 641, 720, 1675, 2307, 2287, 219, 1409, 7, 760, 2064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 477, 10, 12, 1821, 350, 1165, 23, 2746, 29, 925, 697, 113, 1949, 321, 2, 2829, 2072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1211, 897, 4, 1898, 18, 428, 1787, 682, 147, 1625, 53, 1767, 543, 1919, 1716, 1585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1838, 1983, 270, 1238, 1584, 1013, 0, 1666, 1553, 1700, 1217, 564, 664, 13, 461, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15169, 16218, 17745, 15939, 17883, 16068, 16280, 15770] avg=16384.0000 max=17883 max_rank=4 min=15169 min_rank=0 imbalance_ratio=1.091492 max_load_reduction=113189 max_load_reduction_pct=0.863564 moved_to_echo=66001 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1956, 2522, 2070, 0, 50, 0, 2588, 374, 948, 3054, 915, 352, 1724, 34, 0, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 838, 1667, 1357, 1851, 51, 828, 180, 1846, 99, 18, 1002, 2226, 1952, 86, 2168, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 3, 929, 558, 113, 1363, 1117, 311, 55, 2, 2383, 3174, 0, 2408, 618, 1044, 768, 609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 715, 826, 684, 215, 0, 22, 15, 1933, 0, 317, 22, 327, 1953, 3238, 1445, 62, 2548, 840, 459, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 960, 5, 1, 1867, 1191, 1275, 578, 985, 1245, 1681, 2141, 69, 1514, 12, 979, 2039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 620, 561, 5, 41, 1909, 402, 1170, 25, 2047, 136, 1435, 571, 162, 2049, 383, 1, 3206, 2067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1568, 1145, 8, 1420, 4, 460, 1186, 653, 178, 1354, 22, 1339, 218, 1811, 2063, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1984, 2292, 206, 1327, 1246, 974, 0, 1681, 979, 2501, 1001, 816, 796, 6, 589, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18024, 16973, 15974, 15837, 16542, 16790, 14525, 16407] avg=16384.0000 max=18024 max_rank=0 min=14525 min_rank=6 imbalance_ratio=1.100098 max_load_reduction=113048 max_load_reduction_pct=0.862488 moved_to_echo=64264 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1728, 2332, 1425, 0, 358, 0, 2709, 640, 1104, 2972, 492, 678, 1642, 27, 0, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1168, 1226, 1231, 1573, 29, 599, 167, 1772, 143, 44, 938, 1857, 1636, 41, 1953, 1090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 1, 1161, 693, 194, 1818, 1405, 143, 120, 16, 2352, 2882, 6, 2445, 804, 652, 724, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1237, 897, 643, 384, 9, 21, 47, 2317, 2, 252, 41, 424, 1577, 2741, 1414, 129, 2846, 980, 430, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 991, 12, 0, 1763, 1388, 1300, 555, 834, 1469, 1674, 2170, 118, 1705, 14, 753, 2058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 663, 383, 1, 20, 1961, 538, 1298, 18, 2426, 37, 1453, 863, 153, 1667, 455, 3, 2479, 1937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1731, 793, 8, 1699, 16, 599, 1037, 851, 149, 1342, 105, 1037, 381, 2468, 1947, 1355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1619, 2101, 471, 1696, 1173, 946, 0, 1714, 1075, 2699, 1166, 536, 530, 13, 656, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17321, 15467, 16569, 16594, 16804, 16355, 15518, 16444] avg=16384.0000 max=17321 max_rank=0 min=15467 min_rank=1 imbalance_ratio=1.057190 max_load_reduction=113751 max_load_reduction_pct=0.867851 moved_to_echo=65121 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1825, 1802, 1697, 1, 239, 1, 2520, 208, 562, 3172, 1299, 496, 1284, 32, 0, 1652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 696, 2130, 1457, 1509, 11, 861, 162, 1727, 34, 70, 1060, 2563, 1080, 40, 1410, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 835, 2, 1263, 627, 146, 1873, 1436, 206, 108, 1, 2088, 2860, 0, 2373, 713, 997, 743, 656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1529, 1057, 484, 423, 2, 8, 13, 2488, 0, 275, 29, 383, 1276, 3285, 1335, 70, 2162, 1109, 487, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 772, 14, 1, 1582, 1571, 774, 323, 706, 1264, 1758, 1891, 129, 1488, 6, 1044, 2462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 784, 488, 0, 18, 1470, 543, 665, 36, 2290, 63, 1544, 1115, 136, 2733, 408, 0, 3036, 2190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 1054, 8, 1424, 2, 520, 1442, 805, 141, 1609, 44, 1144, 166, 2320, 1995, 2074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2422, 1786, 376, 1590, 1345, 859, 0, 938, 823, 2327, 1375, 493, 1029, 4, 526, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16790, 15680, 16927, 16648, 15785, 17519, 15822, 15901] avg=16384.0000 max=17519 max_rank=5 min=15680 min_rank=1 imbalance_ratio=1.069275 max_load_reduction=113553 max_load_reduction_pct=0.866341 moved_to_echo=65027 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1324, 1992, 2123, 2, 256, 0, 2182, 214, 338, 2470, 352, 623, 1879, 22, 0, 1781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1397, 1708, 1562, 1751, 29, 587, 156, 2063, 169, 16, 992, 2121, 1300, 39, 1767, 1382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 911, 4, 1241, 532, 160, 1137, 1520, 138, 333, 5, 2082, 3099, 0, 2054, 967, 792, 661, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1056, 869, 487, 453, 2, 13, 15, 2805, 0, 119, 15, 609, 1660, 2999, 1405, 167, 1889, 1083, 463, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 15, 0, 1709, 1528, 908, 485, 917, 1625, 1993, 1900, 90, 1417, 5, 649, 1643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 628, 315, 3, 22, 1866, 529, 709, 9, 2211, 75, 1284, 970, 418, 1815, 397, 1, 2383, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1399, 1299, 6, 2206, 3, 620, 1297, 781, 216, 1400, 73, 1674, 329, 2124, 1762, 1946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2331, 2181, 372, 1376, 1391, 752, 0, 1769, 1226, 2163, 1272, 1119, 545, 1, 486, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15558, 17039, 16322, 16354, 15968, 15705, 17135, 16991] avg=16384.0000 max=17135 max_rank=6 min=15558 min_rank=0 imbalance_ratio=1.045837 max_load_reduction=113937 max_load_reduction_pct=0.869270 moved_to_echo=65799 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1603, 2230, 1962, 1, 251, 1, 2390, 264, 816, 2874, 620, 283, 1089, 33, 0, 1385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1111, 1832, 1556, 1422, 11, 724, 165, 1805, 92, 21, 1019, 2172, 1603, 44, 1842, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 429, 2, 1367, 566, 129, 1438, 1260, 151, 183, 1, 2055, 3034, 1, 2487, 848, 778, 723, 603, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1243, 868, 751, 242, 7, 29, 22, 1803, 1, 686, 1, 338, 1646, 2932, 1374, 85, 2825, 752, 468, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1084, 32, 0, 1475, 1065, 1064, 469, 1011, 1165, 2032, 2347, 128, 1164, 25, 1221, 2070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 788, 535, 7, 15, 1312, 567, 1007, 22, 1967, 111, 1165, 960, 144, 2531, 466, 2, 3114, 2106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1384, 1082, 9, 2205, 1, 689, 1823, 834, 258, 1347, 76, 1059, 288, 2121, 1943, 1848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2390, 2427, 322, 1149, 1234, 695, 0, 1526, 1140, 2384, 1057, 724, 837, 5, 444, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15802, 16455, 16055, 16287, 16352, 16819, 16967, 16335] avg=16384.0000 max=16967 max_rank=6 min=15802 min_rank=0 imbalance_ratio=1.035583 max_load_reduction=114105 max_load_reduction_pct=0.870552 moved_to_echo=66473 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1843, 2180, 1765, 0, 295, 0, 2496, 205, 706, 2427, 596, 632, 1777, 28, 0, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 793, 1552, 1938, 1384, 24, 1073, 143, 1794, 55, 42, 991, 2034, 1353, 68, 1802, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 688, 0, 1419, 264, 147, 1657, 1447, 188, 45, 1, 1926, 2765, 0, 2132, 845, 696, 654, 822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1144, 1248, 465, 322, 5, 8, 3, 2165, 0, 105, 8, 281, 1868, 3002, 998, 111, 2426, 1011, 455, 292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 749, 34, 0, 1367, 1433, 1051, 436, 1009, 1262, 1959, 2232, 92, 1822, 2, 589, 1974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 617, 278, 2, 13, 1217, 446, 747, 29, 2720, 19, 1855, 1056, 230, 2517, 287, 0, 2573, 2047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1504, 1060, 5, 1727, 1, 587, 1985, 1028, 132, 1638, 26, 1331, 571, 2044, 1758, 1995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2346, 2010, 268, 886, 1683, 751, 0, 1752, 1134, 2284, 1374, 802, 838, 5, 506, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16198, 16563, 15696, 15917, 16011, 16653, 17392, 16642] avg=16384.0000 max=17392 max_rank=6 min=15696 min_rank=2 imbalance_ratio=1.061523 max_load_reduction=113680 max_load_reduction_pct=0.867310 moved_to_echo=66698 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[14208, 17408, 14976, 128, 1920, 128, 18560, 2944, 6656, 22144, 5376, 3712, 12416, 256, 128, 11264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[7936, 256, 128, 13440, 10880, 8832, 4224, 7296, 11008, 14720, 17152, 1024, 12288, 128, 6784, 16128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[16768, 17152, 2560, 11008, 11136, 7040, 0, 12928, 9216, 18304, 9600, 5760, 6144, 128, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[7808, 13440, 11776, 12800, 256, 6144, 1408, 15104, 896, 384, 8064, 16896, 11648, 512, 15488, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[5120, 128, 9984, 4480, 1792, 12160, 11648, 1536, 1280, 128, 17920, 23552, 128, 18560, 6272, 6528, 5632, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[5248, 3456, 128, 256, 12928, 3840, 8448, 256, 18816, 640, 10624, 6784, 1664, 17280, 3200, 128, 22144, 16640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[11776, 8576, 128, 14464, 128, 4864, 12416, 6528, 1536, 11776, 512, 10880, 3328, 17280, 14976, 13056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[9600, 7296, 4608, 3456, 128, 128, 256, 18432, 128, 2432, 256, 3328, 14080, 23552, 10880, 1024, 19584, 7680, 3712, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 4067, 0, 3184, 7, 0, 0, 3659, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 14, 2, 0, 4083, 0, 4095, 4096, 13, 0, 0, 4095, 1, 1, 4095, 7, 233, 0, 20, 0, 0, 0, 0, 309, 0, 0, 3992, 4092, 0, 0, 4096, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 4077, 0, 0, 3813, 1, 0, 4096, 0, 59, 4096, 0, 0, 0, 0, 0, 0, 3949, 0, 0, 0, 0, 0, 3807, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 4095, 4096, 0, 0, 1412, 0, 26, 2, 0, 4081, 22, 1, 128, 0, 0, 0, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4095, 0, 896, 31, 0, 1, 4078, 0, 0, 0, 1, 4096, 1, 0, 4095, 1, 0, 4092, 0, 18, 2, 0, 4087, 0, 4095, 4096, 15, 1, 0, 4093, 0, 18, 4095, 3, 310, 0, 10, 0, 1, 1, 0, 234, 0, 0, 4088, 4091, 0, 1, 4095, 5, 2, 4095, 1, 1, 4095, 16, 0, 0, 4096, 0, 0, 4054, 0, 0, 3987, 5, 1, 4094, 0, 16, 4093, 0, 1, 0, 0, 0, 0, 3610, 0, 3, 1, 0, 2, 3965, 2, 1, 0, 0, 1, 4095, 1, 1, 0, 1, 0, 4095, 4096, 0, 0, 1, 1, 4095, 1, 4094, 0, 0, 1, 4092, 4095, 0, 1, 3052, 0, 1, 11, 0, 4083, 0, 0, 277, 0, 0, 0, 4089] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4081, 0, 2430, 4, 0, 0, 3940, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 4092, 4, 35, 0, 0, 4076, 0, 4087, 4089, 61, 0, 0, 4096, 0, 19, 4065, 1, 109, 0, 2, 0, 0, 0, 1, 683, 0, 5, 4088, 4096, 0, 0, 4096, 0, 84, 4096, 0, 0, 4095, 0, 1, 0, 4088, 0, 7, 4065, 0, 0, 3845, 0, 0, 4096, 0, 10, 4063, 0, 0, 0, 3, 0, 0, 4008, 1, 0, 0, 0, 0, 3821, 0, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 4096, 4083, 12, 3, 0, 11, 4089, 0, 4096, 0, 0, 1, 4096, 4091, 0, 0, 1125, 1, 34, 0, 0, 4096, 1, 0, 407, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 3955, 0, 1969, 2, 0, 0, 3691, 0, 0, 0, 0, 4067, 0, 0, 4096, 0, 0, 4059, 99, 9, 4, 0, 4035, 8, 4091, 4096, 2, 0, 2, 4096, 0, 76, 4046, 0, 36, 0, 1, 31, 0, 62, 0, 864, 0, 0, 4030, 4096, 0, 0, 4091, 19, 16, 4095, 0, 0, 4053, 32, 17, 0, 4050, 0, 2, 4095, 0, 0, 3884, 10, 0, 4096, 0, 9, 4095, 0, 0, 0, 0, 0, 0, 3697, 0, 0, 0, 0, 0, 3573, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4095, 71, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 4096, 4096, 0, 0, 1046, 0, 100, 1, 0, 4094, 1, 0, 1684, 0, 0, 0, 4053] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 3988, 2, 3079, 0, 0, 1, 3766, 4, 0, 0, 0, 4096, 1, 0, 4095, 1, 0, 4095, 6, 62, 0, 0, 4075, 0, 4086, 4092, 325, 0, 0, 4095, 0, 13, 4096, 1, 31, 0, 8, 0, 0, 0, 0, 365, 0, 0, 4087, 4083, 2, 0, 4095, 8, 1, 4093, 0, 0, 4092, 51, 3, 0, 4086, 0, 0, 4092, 0, 1, 3824, 9, 0, 4093, 0, 16, 4096, 0, 0, 3, 0, 0, 0, 3748, 4, 0, 0, 0, 0, 4070, 0, 0, 0, 1, 0, 4092, 0, 0, 0, 0, 0, 4096, 4094, 173, 0, 0, 0, 4095, 0, 4094, 0, 0, 4, 4093, 4096, 2, 0, 574, 0, 49, 1, 0, 4095, 0, 0, 478, 0, 2, 0, 4089] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4050, 0, 3874, 3, 0, 1, 3824, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4, 0, 0, 0, 4094, 2, 4088, 4096, 30, 0, 0, 4095, 0, 0, 4096, 6, 31, 0, 14, 0, 1, 3, 1, 943, 0, 0, 4096, 4094, 0, 0, 4096, 2, 7, 4094, 0, 0, 4093, 1, 3, 0, 4096, 0, 1, 4044, 0, 0, 3634, 19, 0, 4093, 0, 46, 4093, 0, 0, 0, 0, 0, 0, 3968, 0, 4, 0, 0, 0, 3828, 0, 0, 0, 0, 0, 4095, 2, 1, 0, 0, 3, 4091, 4040, 6, 1, 0, 2, 4096, 1, 4089, 0, 0, 0, 4096, 4095, 0, 0, 257, 0, 55, 0, 0, 4084, 2, 0, 104, 0, 0, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 4053, 0, 3360, 2, 0, 0, 3945, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4077, 0, 49, 1, 0, 4089, 0, 4095, 4093, 7, 0, 2, 4096, 0, 10, 4091, 0, 118, 0, 5, 0, 0, 3, 0, 1311, 0, 2, 4066, 4094, 0, 0, 4096, 0, 30, 4096, 0, 0, 4096, 7, 0, 0, 4094, 0, 0, 4078, 0, 0, 3434, 0, 0, 4096, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 3753, 0, 3, 0, 0, 0, 3609, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3, 10, 0, 4, 4096, 0, 4095, 6, 0, 0, 4096, 4096, 0, 0, 831, 0, 20, 0, 0, 4090, 2, 0, 85, 0, 0, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 3, 0, 48, 4088, 0, 4014, 0, 0, 0, 3966, 0, 0, 0, 0, 4094, 0, 2, 4094, 0, 0, 4095, 5, 59, 4, 0, 4066, 0, 4095, 4096, 101, 0, 2, 4095, 0, 5, 4088, 6, 155, 0, 11, 0, 1, 0, 0, 1510, 0, 0, 4059, 4088, 0, 0, 4093, 0, 13, 4096, 1, 0, 4093, 0, 0, 0, 4091, 0, 8, 4014, 0, 0, 3110, 0, 0, 4096, 0, 109, 4091, 0, 0, 0, 0, 0, 0, 3873, 0, 0, 0, 0, 1, 3265, 0, 0, 0, 0, 0, 4096, 2, 3, 0, 0, 0, 4095, 4087, 22, 0, 0, 9, 4094, 0, 4089, 0, 0, 0, 4096, 4090, 0, 0, 69, 0, 23, 0, 0, 4077, 72, 0, 264, 0, 0, 0, 4080] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 4067, 0, 3184, 7, 0, 0, 3659, 2, 0, 0, 0, 4096, 1519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 14, 2, 0, 4083, 0, 4095, 4013, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 4095, 7, 233, 0, 20, 0, 0, 0, 0, 309, 0, 0, 4096, 3160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 4092, 0, 0, 13, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4077, 0, 0, 3813, 1, 0, 4096, 0, 59, 4096, 0, 0, 0, 0, 0, 289, 83, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 0, 0, 0, 0, 3807, 0, 0, 0, 0, 0, 4096, 0, 0, 3781, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1412, 0, 26, 2, 0, 4081, 22, 1, 128, 0, 0, 0, 4092, 2275, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16538, 16314, 16017, 16291, 16542, 16569, 16351, 16450] avg=16384.0000 max=16569 max_rank=5 min=16017 min_rank=2 imbalance_ratio=1.011292 max_load_reduction=114503 max_load_reduction_pct=0.873589 moved_to_echo=65912 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 4095, 0, 896, 31, 0, 1, 4078, 0, 0, 0, 1, 4096, 1518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4092, 0, 18, 2, 0, 4087, 0, 4095, 4013, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 18, 4095, 3, 310, 0, 10, 0, 1, 1, 0, 234, 0, 0, 4095, 3160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4091, 0, 1, 13, 5, 2, 0, 1, 1, 4095, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 0, 3987, 5, 1, 4094, 0, 16, 4093, 0, 1, 0, 0, 0, 289, 83, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3610, 0, 3, 1, 0, 2, 3965, 2, 1, 0, 0, 1, 4095, 1, 1, 3781, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 1, 1, 4095, 1, 4094, 0, 0, 1, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 3052, 0, 1, 11, 0, 4083, 0, 0, 277, 0, 0, 0, 4089, 2275, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14717, 16325, 16020, 16409, 16649, 16398, 16356, 18198] avg=16384.0000 max=18198 max_rank=7 min=14717 min_rank=0 imbalance_ratio=1.110718 max_load_reduction=112874 max_load_reduction_pct=0.861160 moved_to_echo=67601 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 3988, 2, 3079, 0, 0, 1, 3766, 4, 0, 0, 0, 4096, 1518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4095, 6, 62, 0, 0, 4075, 0, 4086, 4009, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 13, 4096, 1, 31, 0, 8, 0, 0, 0, 0, 365, 0, 0, 4095, 3159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4083, 2, 0, 13, 8, 1, 0, 0, 0, 4092, 51, 3, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 3824, 9, 0, 4093, 0, 16, 4096, 0, 0, 3, 0, 0, 289, 83, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 4, 0, 0, 0, 0, 4070, 0, 0, 0, 1, 0, 4092, 0, 0, 3782, 934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 173, 0, 0, 0, 4095, 0, 4094, 0, 0, 4, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 574, 0, 49, 1, 0, 4095, 0, 0, 478, 0, 2, 0, 4089, 2275, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16458, 16660, 15863, 16426, 16532, 16631, 16527, 15975] avg=16384.0000 max=16660 max_rank=1 min=15863 min_rank=2 imbalance_ratio=1.016846 max_load_reduction=114412 max_load_reduction_pct=0.872894 moved_to_echo=65665 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4081, 0, 2430, 4, 0, 0, 3940, 0, 0, 0, 0, 4096, 1523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4, 35, 0, 0, 4076, 0, 4087, 4003, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 4065, 1, 109, 0, 2, 0, 0, 0, 1, 683, 0, 5, 4095, 3161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 3, 0, 84, 0, 0, 0, 4095, 0, 1, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4065, 0, 0, 3845, 0, 0, 4096, 0, 10, 4063, 0, 0, 0, 3, 0, 291, 86, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 1, 0, 0, 0, 0, 3821, 0, 0, 0, 0, 0, 4096, 0, 2, 3782, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4083, 12, 3, 0, 11, 4089, 0, 4096, 0, 0, 1, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 1125, 1, 34, 0, 0, 4096, 1, 0, 407, 0, 0, 0, 4095, 2279, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16076, 16358, 16237, 16455, 16495, 16645, 16363, 16443] avg=16384.0000 max=16645 max_rank=5 min=16076 min_rank=0 imbalance_ratio=1.015930 max_load_reduction=114427 max_load_reduction_pct=0.873009 moved_to_echo=65946 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 3955, 0, 1969, 2, 0, 0, 3691, 0, 0, 0, 0, 4067, 1517, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 99, 9, 4, 0, 4035, 8, 4091, 4013, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 76, 4046, 0, 36, 0, 1, 31, 0, 62, 0, 864, 0, 0, 4096, 3159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 4096, 0, 0, 14, 19, 16, 0, 0, 0, 4053, 32, 17, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 3884, 10, 0, 4096, 0, 9, 4095, 0, 0, 0, 0, 0, 288, 83, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3697, 0, 0, 0, 0, 0, 3573, 0, 0, 0, 0, 0, 4096, 0, 0, 3780, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 71, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1046, 0, 100, 1, 0, 4094, 1, 0, 1684, 0, 0, 0, 4053, 2272, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15204, 16320, 16469, 16327, 16589, 16082, 16419, 17662] avg=16384.0000 max=17662 max_rank=7 min=15204 min_rank=0 imbalance_ratio=1.078003 max_load_reduction=113410 max_load_reduction_pct=0.865250 moved_to_echo=66752 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4050, 0, 3874, 3, 0, 1, 3824, 0, 0, 0, 0, 4096, 1519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4094, 2, 4088, 4013, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 6, 31, 0, 14, 0, 1, 3, 1, 943, 0, 0, 4096, 3158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 13, 2, 7, 0, 0, 0, 4093, 1, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4044, 0, 0, 3634, 19, 0, 4093, 0, 46, 4093, 0, 0, 0, 0, 0, 289, 83, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 4, 0, 0, 0, 3828, 0, 0, 0, 0, 0, 4095, 2, 1, 3778, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4040, 6, 1, 0, 2, 4096, 1, 4089, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 257, 0, 55, 0, 0, 4084, 2, 0, 104, 0, 0, 0, 4094, 2275, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17369, 16327, 16444, 16405, 16329, 16612, 16307, 15279] avg=16384.0000 max=17369 max_rank=0 min=15279 min_rank=7 imbalance_ratio=1.060120 max_load_reduction=113703 max_load_reduction_pct=0.867485 moved_to_echo=64527 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 0, 48, 4088, 0, 4014, 0, 0, 0, 3966, 0, 0, 0, 0, 4094, 1518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 5, 59, 4, 0, 4066, 0, 4095, 4013, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 5, 4088, 6, 155, 0, 11, 0, 1, 0, 0, 1510, 0, 0, 4094, 3160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4088, 0, 0, 14, 0, 13, 0, 1, 0, 4093, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4014, 0, 0, 3110, 0, 0, 4096, 0, 109, 4091, 0, 0, 0, 0, 0, 288, 83, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3873, 0, 0, 0, 0, 1, 3265, 0, 0, 0, 0, 0, 4096, 2, 3, 3781, 936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 22, 0, 0, 9, 4094, 0, 4089, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 69, 0, 23, 0, 0, 4077, 72, 0, 264, 0, 0, 0, 4080, 2273, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17731, 16440, 17127, 16359, 15826, 15957, 16370, 15262] avg=16384.0000 max=17731 max_rank=0 min=15262 min_rank=7 imbalance_ratio=1.082214 max_load_reduction=113341 max_load_reduction_pct=0.864723 moved_to_echo=63415 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 4053, 0, 3360, 2, 0, 0, 3945, 0, 0, 0, 0, 4096, 1519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 49, 1, 0, 4089, 0, 4095, 4010, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 10, 4091, 0, 118, 0, 5, 0, 0, 3, 0, 1311, 0, 2, 4096, 3161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4094, 0, 0, 13, 0, 30, 0, 0, 0, 4096, 7, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 3434, 0, 0, 4096, 0, 7, 4096, 0, 0, 0, 0, 0, 289, 83, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3753, 0, 3, 0, 0, 0, 3609, 0, 0, 0, 0, 0, 4096, 0, 0, 3782, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 10, 0, 4, 4096, 0, 4095, 6, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 831, 0, 20, 0, 0, 4090, 2, 0, 85, 0, 0, 0, 4090, 2275, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16979, 16328, 16895, 16400, 16110, 16178, 16379, 15803] avg=16384.0000 max=16979 max_rank=0 min=15803 min_rank=7 imbalance_ratio=1.036316 max_load_reduction=114093 max_load_reduction_pct=0.870461 moved_to_echo=64470 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 32768, 384, 128, 128, 128, 32768, 128, 32768, 128, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32512, 32768, 128, 128, 128, 128, 256, 0, 128, 128, 32768, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 256, 32768, 128, 1024, 0, 128, 128, 128, 128, 128, 6272, 0, 128, 32768, 25344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 32384, 128, 22912, 128, 0, 128, 30976, 128, 0, 0, 128, 32768, 12160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32640, 0, 128, 29568, 128, 128, 32768, 0, 384, 32768, 0, 128, 128, 128, 0, 2432, 768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 8448, 128, 384, 128, 0, 32768, 128, 128, 3456, 0, 128, 0, 32768, 18304, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 0, 32768, 128, 256, 128, 0, 32640, 128, 32768, 32128, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 30720, 128, 128, 128, 0, 128, 29952, 128, 128, 0, 128, 128, 32768, 128, 128, 30336, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 2462, 0, 355, 0, 0, 2435, 0, 4092, 153, 0, 0, 0, 0, 4091, 0, 0, 0, 3012, 0, 0, 4096, 0, 4096, 4096, 0, 0, 3, 4096, 3858, 0, 41, 0, 4096, 0, 0, 4096, 844, 0, 0, 0, 4096, 0, 47, 0, 0, 4096, 4069, 2534, 0, 0, 1, 0, 4096, 4087, 0, 0, 4010, 4050, 0, 0, 1, 2154, 0, 0, 4096, 44, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 1, 3809, 4013, 0, 0, 70, 3988, 0, 371, 0, 2688, 0, 0, 4089, 0, 0, 0, 3985, 0, 3, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 160, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4083, 13, 0, 6, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 1219, 0, 218, 1, 1, 3695, 1, 4092, 2, 0, 1, 0, 1, 4095, 0, 1, 0, 3931, 0, 1, 4095, 1, 4095, 4095, 0, 0, 2, 4095, 4023, 0, 0, 0, 4095, 1, 0, 4095, 36, 1, 1, 0, 4095, 1, 1, 1, 0, 4093, 4090, 3259, 1, 0, 1, 0, 4093, 4094, 1, 1, 4089, 4094, 0, 1, 1, 2246, 0, 1, 4095, 2, 0, 1, 0, 0, 0, 4095, 0, 0, 4095, 0, 2, 3899, 4095, 0, 1, 13, 4054, 0, 189, 1, 1829, 1, 0, 4096, 0, 2, 0, 4092, 0, 2, 0, 1, 8, 0, 4094, 0, 1, 1, 0, 0, 0, 0, 51, 4095, 1, 1, 0, 0, 0, 0, 4095, 1, 0, 0, 4095, 3, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2441, 0, 706, 0, 0, 3823, 0, 4096, 10, 0, 0, 0, 0, 4096, 0, 0, 0, 3448, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 3757, 0, 0, 0, 4096, 1, 2, 4096, 21, 0, 0, 0, 4096, 0, 14, 0, 0, 4096, 4094, 3659, 0, 0, 0, 0, 4096, 4089, 0, 0, 4084, 4082, 0, 0, 0, 2312, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 11, 4072, 4019, 0, 0, 44, 3998, 0, 100, 0, 350, 0, 0, 4060, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 48, 4096, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4090, 11, 0, 4, 0, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1073, 1, 1643, 0, 0, 3562, 0, 4089, 6, 0, 0, 0, 0, 4091, 0, 0, 0, 3352, 0, 0, 4089, 0, 4096, 4095, 0, 0, 0, 4096, 4054, 0, 1, 0, 4096, 7, 0, 4096, 5, 0, 0, 0, 4096, 0, 14, 0, 0, 4095, 4096, 3769, 0, 0, 0, 0, 4095, 4096, 0, 0, 4065, 4082, 0, 0, 0, 2194, 0, 0, 4096, 10, 0, 0, 0, 0, 6, 4096, 0, 0, 4096, 0, 15, 4068, 4085, 0, 0, 0, 4018, 0, 27, 0, 945, 0, 0, 4094, 0, 0, 0, 4034, 1, 23, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 2, 0, 0, 0, 4090, 0, 1, 0, 4086, 3, 0, 6, 0, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1656, 0, 528, 0, 0, 3581, 0, 4087, 60, 0, 0, 0, 0, 4096, 0, 0, 0, 3579, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 4096, 3861, 0, 7, 0, 4096, 13, 0, 4096, 93, 0, 0, 0, 4096, 0, 6, 0, 0, 4096, 4088, 3447, 0, 0, 6, 0, 4094, 4096, 0, 0, 4038, 4036, 0, 0, 3, 2512, 0, 0, 4092, 24, 0, 0, 0, 0, 2, 4096, 0, 0, 4093, 0, 14, 3972, 4063, 0, 0, 32, 3862, 0, 62, 0, 1582, 4, 0, 4028, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 83, 4094, 0, 0, 4, 0, 0, 0, 4073, 0, 0, 0, 4096, 16, 0, 19, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2160, 0, 255, 0, 0, 3375, 0, 4087, 9, 3, 0, 0, 0, 4091, 1, 0, 0, 3161, 0, 0, 4092, 0, 4096, 4096, 0, 0, 0, 4092, 4031, 0, 10, 0, 4096, 0, 0, 4096, 55, 0, 0, 0, 4096, 0, 16, 0, 0, 4095, 4088, 3706, 0, 1, 0, 2, 4096, 4091, 0, 0, 4090, 4086, 0, 0, 3, 1275, 0, 0, 4095, 7, 0, 0, 0, 0, 4, 4096, 0, 0, 4093, 4, 7, 3948, 4085, 0, 0, 0, 4044, 0, 229, 0, 2231, 4, 0, 4062, 0, 0, 0, 4087, 0, 11, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 209, 4094, 0, 0, 5, 0, 0, 0, 4092, 0, 0, 0, 4091, 8, 3, 3, 0, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1425, 0, 605, 0, 0, 3937, 0, 4091, 6, 0, 0, 0, 0, 4093, 0, 0, 0, 3848, 0, 0, 4092, 0, 4096, 4093, 0, 0, 0, 4093, 3913, 0, 4, 4, 4096, 7, 0, 4095, 19, 0, 0, 0, 4096, 0, 11, 0, 0, 4095, 4087, 3413, 0, 0, 6, 1, 4091, 4093, 0, 0, 4082, 4088, 0, 0, 2, 2174, 0, 0, 4096, 4, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 6, 5, 4055, 4093, 0, 0, 0, 4029, 0, 37, 0, 1077, 3, 0, 4079, 0, 0, 0, 4086, 2, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4, 0, 0, 150, 4093, 0, 0, 4, 0, 0, 0, 4091, 0, 1, 0, 4095, 10, 0, 3, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3121, 0, 317, 0, 0, 3182, 0, 4095, 2, 0, 0, 1, 0, 4096, 0, 0, 0, 3823, 3, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 3871, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4095, 4083, 3396, 0, 0, 9, 0, 4095, 4096, 0, 0, 4095, 4096, 0, 0, 0, 2631, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 3998, 4096, 0, 0, 0, 4049, 0, 0, 0, 264, 1, 0, 4092, 0, 0, 0, 4069, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 40, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 1219, 0, 218, 1, 1, 3695, 1, 4092, 2, 0, 1, 0, 1, 4095, 1346, 793, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3931, 0, 1, 4095, 1, 0, 754, 0, 0, 2, 4095, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3937, 1, 0, 4095, 36, 1, 1, 0, 4095, 1, 1, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2651, 3259, 1, 0, 1, 0, 0, 0, 1, 1, 4089, 4094, 0, 1, 1, 2246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 2, 0, 1, 0, 0, 0, 4095, 0, 0, 4095, 0, 2, 3899, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 13, 4054, 0, 189, 1, 1829, 1, 0, 2750, 0, 2, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 8, 0, 4094, 0, 1, 1, 0, 0, 0, 0, 51, 4095, 4094, 3341, 646, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4095, 1, 0, 0, 4095, 3, 1, 0, 0, 0, 4095, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15529, 16903, 16262, 16345, 16287, 17027, 16344, 16375] avg=16384.0000 max=17027 max_rank=5 min=15529 min_rank=0 imbalance_ratio=1.039246 max_load_reduction=114045 max_load_reduction_pct=0.870094 moved_to_echo=66033 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1073, 1, 1643, 0, 0, 3562, 0, 4089, 6, 0, 0, 0, 0, 4091, 1345, 794, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3352, 0, 0, 4089, 0, 0, 754, 0, 0, 0, 4096, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3937, 7, 0, 4096, 5, 0, 0, 0, 4096, 0, 14, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2655, 3769, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4082, 0, 0, 0, 2194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 0, 0, 0, 6, 4096, 0, 0, 4096, 0, 15, 4068, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4018, 0, 27, 0, 945, 0, 0, 2749, 0, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 23, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 4096, 3341, 647, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4090, 0, 1, 0, 4086, 3, 0, 6, 0, 9, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16666, 16345, 16251, 16765, 16484, 15858, 16324, 16379] avg=16384.0000 max=16765 max_rank=3 min=15858 min_rank=5 imbalance_ratio=1.023254 max_load_reduction=114307 max_load_reduction_pct=0.872093 moved_to_echo=65045 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2462, 0, 355, 0, 0, 2435, 0, 4092, 153, 0, 0, 0, 0, 4091, 1344, 789, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3012, 0, 0, 4096, 0, 0, 754, 0, 0, 3, 4096, 3858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 3937, 0, 0, 4096, 844, 0, 0, 0, 4096, 0, 47, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2637, 2534, 0, 0, 1, 0, 0, 0, 0, 0, 4010, 4050, 0, 0, 1, 2154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 44, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 1, 3809, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 70, 3988, 0, 371, 0, 2688, 0, 0, 2745, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 160, 4096, 4087, 3342, 643, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4083, 13, 0, 6, 0, 2, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15784, 15819, 17157, 15387, 16240, 17860, 16441, 16384] avg=16384.0000 max=17860 max_rank=5 min=15387 min_rank=3 imbalance_ratio=1.090088 max_load_reduction=113212 max_load_reduction_pct=0.863739 moved_to_echo=66925 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3121, 0, 317, 0, 0, 3182, 0, 4095, 2, 0, 0, 1, 0, 4096, 1345, 792, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3823, 3, 0, 4096, 0, 0, 754, 0, 0, 0, 4096, 3871, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3937, 0, 1, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2646, 3396, 0, 0, 9, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 2631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 3998, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4049, 0, 0, 0, 264, 1, 0, 2747, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 40, 4096, 4096, 3342, 645, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 1, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17013, 16643, 16226, 16873, 16384, 15226, 16330, 16377] avg=16384.0000 max=17013 max_rank=0 min=15226 min_rank=5 imbalance_ratio=1.038391 max_load_reduction=114059 max_load_reduction_pct=0.870201 moved_to_echo=64317 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1425, 0, 605, 0, 0, 3937, 0, 4091, 6, 0, 0, 0, 0, 4093, 1340, 793, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3848, 0, 0, 4092, 0, 0, 754, 0, 0, 0, 4093, 3913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 3937, 7, 0, 4095, 19, 0, 0, 0, 4096, 0, 11, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2649, 3413, 0, 0, 6, 1, 0, 0, 0, 0, 4082, 4088, 0, 0, 2, 2174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4, 4096, 0, 0, 4096, 6, 5, 4055, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4029, 0, 37, 0, 1077, 3, 0, 2739, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4, 0, 0, 150, 4093, 4093, 3339, 645, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4091, 0, 1, 0, 4095, 10, 0, 3, 0, 2, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16352, 16700, 16268, 16415, 16459, 16064, 16430, 16384] avg=16384.0000 max=16700 max_rank=1 min=16064 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=114372 max_load_reduction_pct=0.872589 moved_to_echo=65337 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1656, 0, 528, 0, 0, 3581, 0, 4087, 60, 0, 0, 0, 0, 4096, 1328, 797, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3579, 0, 0, 4094, 0, 0, 750, 0, 0, 0, 4096, 3861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3933, 13, 0, 4096, 93, 0, 0, 0, 4096, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2643, 3447, 0, 0, 6, 0, 0, 0, 0, 0, 4038, 4036, 0, 0, 3, 2512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 24, 0, 0, 0, 0, 2, 4096, 0, 0, 4093, 0, 14, 3972, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 0, 0, 32, 3862, 0, 62, 0, 1582, 4, 0, 2700, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 83, 4094, 4096, 3346, 648, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4073, 0, 0, 0, 4096, 16, 0, 19, 0, 3, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16196, 16380, 16340, 16685, 16393, 16314, 16373, 16391] avg=16384.0000 max=16685 max_rank=3 min=16196 min_rank=0 imbalance_ratio=1.018372 max_load_reduction=114387 max_load_reduction_pct=0.872704 moved_to_echo=65471 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2160, 0, 255, 0, 0, 3375, 0, 4087, 9, 3, 0, 0, 0, 4091, 1335, 793, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3161, 0, 0, 4092, 0, 0, 754, 0, 0, 0, 4092, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3937, 0, 0, 4096, 55, 0, 0, 0, 4096, 0, 16, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2649, 3706, 0, 1, 0, 2, 0, 0, 0, 0, 4090, 4086, 0, 0, 3, 1275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 0, 0, 0, 0, 4, 4096, 0, 0, 4093, 4, 7, 3948, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4044, 0, 229, 0, 2231, 4, 0, 2727, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 0, 209, 4094, 4091, 3342, 646, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4092, 0, 0, 0, 4091, 8, 3, 3, 0, 9, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16170, 16131, 16305, 15812, 16351, 17407, 16502, 16394] avg=16384.0000 max=17407 max_rank=5 min=15812 min_rank=3 imbalance_ratio=1.062439 max_load_reduction=113665 max_load_reduction_pct=0.867195 moved_to_echo=66654 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2441, 0, 706, 0, 0, 3823, 0, 4096, 10, 0, 0, 0, 0, 4096, 1334, 794, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3448, 0, 0, 4096, 0, 0, 754, 0, 0, 0, 4096, 3757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3937, 1, 2, 4096, 21, 0, 0, 0, 4096, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2653, 3659, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4082, 0, 0, 0, 2312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 11, 4072, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 44, 3998, 0, 100, 0, 350, 0, 0, 2726, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 48, 4096, 4089, 3342, 647, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4090, 11, 0, 4, 0, 7, 4096, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17362, 16151, 16263, 16790, 16474, 15316, 16328, 16388] avg=16384.0000 max=17362 max_rank=0 min=15316 min_rank=5 imbalance_ratio=1.059692 max_load_reduction=113710 max_load_reduction_pct=0.867538 moved_to_echo=64506 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32640, 0, 128, 256, 32128, 0, 1024, 128, 11008, 128, 0, 21888, 0, 128, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 0, 0, 128, 32768, 0, 0, 32768, 128, 128, 31872, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 31616, 128, 128, 32768, 1152, 128, 128, 128, 32768, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 28160, 128, 128, 32768, 128, 0, 6144, 0, 0, 128, 32768, 31488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 15616, 128, 4736, 128, 128, 27648, 128, 32768, 256, 128, 128, 128, 128, 32768, 10752, 6400, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 0, 0, 0, 32768, 128, 128, 0, 32768, 128, 128, 128, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[21248, 27264, 128, 128, 128, 128, 0, 0, 128, 128, 32640, 32640, 0, 128, 128, 17536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 128, 0, 32768, 0, 128, 128, 0, 128, 0, 0, 768, 32768, 32768, 26752, 5248, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 34, 4085, 2, 0, 8, 4088, 4096, 0, 0, 0, 4096, 0, 4024, 0, 0, 4096, 0, 0, 0, 4082, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 5, 4096, 3260, 4078, 0, 0, 3331, 0, 4096, 4096, 4096, 0, 0, 4096, 9, 2, 0, 0, 0, 3, 24, 7, 0, 4096, 0, 0, 0, 9, 4095, 0, 0, 4096, 4055, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 4091, 0, 0, 0, 0, 4096, 0, 0, 56, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 1719, 4096, 0, 0, 0, 296, 0, 0, 0, 0, 0, 4096, 3679, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 46, 4096, 0, 0, 177, 4096, 4096, 0, 0, 0, 4096, 0, 4012, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 74, 4096, 3623, 4094, 0, 0, 2080, 0, 4096, 4096, 4096, 0, 0, 4096, 37, 0, 0, 0, 0, 0, 41, 0, 0, 4096, 0, 0, 0, 1, 4066, 0, 0, 4096, 4030, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4080, 4096, 0, 0, 0, 0, 4096, 0, 0, 66, 0, 0, 0, 0, 0, 0, 31, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1930, 4096, 0, 0, 0, 426, 0, 0, 0, 0, 0, 4096, 3952, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4094, 0, 0, 46, 4094, 4096, 0, 0, 0, 4096, 0, 4051, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2819, 4089, 0, 0, 3442, 0, 4096, 4096, 4096, 0, 0, 4096, 75, 2, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 17, 4087, 0, 0, 4095, 4079, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 4095, 0, 0, 14, 0, 0, 1, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2294, 4096, 0, 0, 0, 57, 0, 0, 0, 0, 0, 4096, 3586, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 19, 4096, 0, 0, 471, 4096, 4096, 0, 0, 0, 4096, 0, 4052, 0, 0, 4095, 0, 0, 8, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3556, 4075, 0, 0, 2590, 0, 4096, 4096, 4090, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 1, 4087, 0, 0, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 4096, 0, 0, 0, 0, 4094, 0, 0, 27, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2684, 4096, 0, 0, 0, 27, 0, 0, 0, 0, 0, 4096, 2977, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 4096, 1, 0, 28, 4093, 4096, 0, 0, 0, 4096, 0, 4066, 0, 0, 4096, 0, 0, 1, 4091, 0, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 10, 4096, 2699, 4080, 0, 0, 3259, 0, 4096, 4096, 4096, 0, 0, 4096, 9, 0, 0, 0, 0, 2, 6, 1, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 4095, 4087, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 4091, 4096, 0, 0, 0, 0, 4095, 0, 0, 25, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 4096, 4, 0, 0, 0, 2423, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 3875, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 4090, 2, 0, 17, 4094, 4096, 0, 0, 0, 4096, 0, 4062, 0, 0, 4096, 0, 0, 1, 4091, 0, 1, 4096, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 3, 0, 4096, 3667, 4093, 0, 0, 3744, 0, 4096, 4096, 4093, 0, 0, 4096, 39, 7, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 6, 4087, 0, 0, 4096, 4087, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4091, 4094, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 5, 0, 0, 4096, 1, 0, 0, 0, 1292, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 3519, 1, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 1, 1, 5, 4095, 0, 0, 70, 4095, 4095, 0, 0, 1, 4093, 1, 4094, 2, 1, 4092, 1, 0, 0, 4095, 0, 0, 4088, 0, 0, 4094, 0, 0, 4095, 1, 0, 2, 1, 0, 4095, 0, 61, 4095, 3254, 4094, 0, 5, 2871, 1, 4095, 4095, 4095, 1, 1, 4095, 48, 0, 0, 1, 1, 0, 1, 1, 0, 4095, 1, 0, 0, 18, 4095, 1, 0, 4095, 4092, 1, 1, 1, 0, 0, 0, 1, 4092, 0, 4095, 4095, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 1, 0, 0, 1, 0, 7, 0, 1, 0, 1, 0, 1, 1, 4095, 0, 1, 1, 0, 1875, 4091, 1, 0, 0, 36, 0, 0, 1, 0, 1, 4095, 4052, 0, 1, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4095, 0, 0, 127, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 6, 0, 4093, 0, 0, 0, 4096, 0, 0, 4084, 0, 0, 4096, 0, 0, 4095, 1, 0, 1, 0, 0, 4096, 0, 7, 4096, 3397, 4096, 0, 7, 3240, 0, 4096, 4096, 4082, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1612, 4095, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 3904, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4094, 0, 0, 46, 4094, 3968, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 45, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1825, 0, 0, 0, 2819, 4089, 0, 0, 3442, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4056, 4096, 0, 0, 0, 75, 2, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4087, 0, 0, 4095, 4079, 0, 0, 0, 0, 0, 0, 0, 2161, 1956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 4078, 0, 0, 14, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2294, 4096, 4096, 2095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 57, 0, 0, 0, 0, 0, 4096, 3586, 0, 0, 4096, 0, 128, 110, 40, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16299, 16381, 16271, 16429, 16395, 16378, 16693, 16226] avg=16384.0000 max=16693 max_rank=6 min=16226 min_rank=7 imbalance_ratio=1.018860 max_load_reduction=114379 max_load_reduction_pct=0.872643 moved_to_echo=65692 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 34, 4085, 2, 0, 8, 4088, 3961, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 33, 0, 0, 0, 4082, 0, 1, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1820, 1, 5, 0, 3260, 4078, 0, 0, 3331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4055, 4096, 0, 0, 0, 9, 2, 0, 0, 0, 3, 24, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4095, 0, 0, 4096, 4055, 0, 0, 0, 0, 0, 0, 0, 2165, 1962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 4091, 0, 0, 0, 0, 4078, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 1719, 4096, 4096, 2101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 296, 0, 0, 0, 0, 0, 4096, 3679, 0, 0, 4096, 0, 135, 111, 41, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16274, 16332, 16591, 16388, 16382, 16409, 16128, 16568] avg=16384.0000 max=16591 max_rank=2 min=16128 min_rank=6 imbalance_ratio=1.012634 max_load_reduction=114481 max_load_reduction_pct=0.873421 moved_to_echo=65487 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 4090, 2, 0, 17, 4094, 3968, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 45, 0, 0, 1, 4091, 0, 1, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1825, 3, 0, 0, 3667, 4093, 0, 0, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4056, 4093, 0, 0, 0, 39, 7, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4087, 0, 0, 4096, 4087, 0, 0, 0, 0, 2, 0, 0, 2160, 1956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 4094, 0, 0, 0, 0, 4079, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 5, 0, 0, 4096, 1, 0, 0, 0, 1292, 4096, 4096, 2095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 3519, 1, 0, 4096, 0, 128, 110, 40, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16276, 16388, 17428, 16390, 16394, 16369, 15716, 16111] avg=16384.0000 max=17428 max_rank=2 min=15716 min_rank=6 imbalance_ratio=1.063721 max_load_reduction=113644 max_load_reduction_pct=0.867035 moved_to_echo=64590 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 4096, 1, 0, 28, 4093, 3968, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 45, 0, 0, 1, 4091, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1825, 0, 10, 0, 2699, 4080, 0, 0, 3259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4056, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 2, 6, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4095, 4087, 0, 0, 0, 0, 3, 0, 0, 2161, 1956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 4096, 0, 0, 0, 0, 4078, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 4096, 4, 0, 0, 0, 2423, 4096, 4096, 2095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 3875, 0, 0, 4096, 0, 128, 110, 40, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16285, 16394, 15969, 16362, 16391, 16386, 16822, 16463] avg=16384.0000 max=16822 max_rank=6 min=15969 min_rank=2 imbalance_ratio=1.026733 max_load_reduction=114250 max_load_reduction_pct=0.871658 moved_to_echo=66062 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 1, 5, 4095, 0, 0, 70, 4095, 3967, 0, 0, 1, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 1, 44, 1, 0, 0, 4095, 0, 0, 4088, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 2, 1, 0, 1825, 0, 61, 0, 3254, 4094, 0, 5, 2871, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4055, 4095, 1, 1, 0, 48, 0, 0, 1, 1, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 18, 4095, 1, 0, 4095, 4092, 1, 1, 1, 0, 0, 0, 1, 2160, 1955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4095, 4095, 0, 0, 0, 0, 4078, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 1, 0, 1, 0, 1, 1, 4095, 0, 1, 1, 0, 1875, 4095, 4095, 2093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 36, 0, 0, 1, 0, 1, 4095, 4052, 0, 1, 4095, 0, 128, 110, 40, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16331, 16419, 16210, 16394, 16421, 16362, 16267, 16668] avg=16384.0000 max=16668 max_rank=7 min=16210 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=114404 max_load_reduction_pct=0.872833 moved_to_echo=65718 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4095, 0, 0, 127, 4096, 3968, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 44, 0, 0, 0, 4096, 0, 0, 4084, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 0, 1825, 0, 7, 0, 3397, 4096, 0, 7, 3240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4056, 4082, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 2161, 1955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 1, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1612, 4096, 4096, 2094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 3904, 0, 0, 4096, 0, 128, 110, 40, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16422, 16669, 16344, 16405, 16366, 15996, 16488] avg=16384.0000 max=16669 max_rank=2 min=15996 min_rank=6 imbalance_ratio=1.017395 max_load_reduction=114403 max_load_reduction_pct=0.872826 moved_to_echo=65255 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 19, 4096, 0, 0, 471, 4096, 3968, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 44, 0, 0, 8, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1825, 0, 0, 0, 3556, 4075, 0, 0, 2590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4056, 4090, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4087, 0, 0, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 2161, 1956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4089, 4096, 0, 0, 0, 0, 4077, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2684, 4096, 4096, 2095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 27, 0, 0, 0, 0, 0, 4096, 2977, 0, 0, 4096, 0, 128, 110, 40, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16746, 16391, 16142, 16343, 16374, 16385, 17104, 15587] avg=16384.0000 max=17104 max_rank=6 min=15587 min_rank=7 imbalance_ratio=1.043945 max_load_reduction=113968 max_load_reduction_pct=0.869507 moved_to_echo=65450 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 46, 4096, 0, 0, 177, 4096, 3968, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 45, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1825, 0, 74, 0, 3623, 4094, 0, 0, 2080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4056, 4096, 0, 0, 0, 37, 0, 0, 0, 0, 0, 41, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4066, 0, 0, 4096, 4030, 0, 0, 0, 0, 0, 0, 0, 2161, 1956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4080, 4096, 0, 0, 0, 0, 4079, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1930, 4096, 4096, 2095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 426, 0, 0, 0, 0, 0, 4096, 3952, 0, 0, 4096, 0, 128, 110, 40, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16479, 16345, 15792, 16422, 16310, 16417, 16346, 16961] avg=16384.0000 max=16961 max_rank=7 min=15792 min_rank=2 imbalance_ratio=1.035217 max_load_reduction=114111 max_load_reduction_pct=0.870598 moved_to_echo=66034 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32512, 128, 128, 384, 128, 0, 128, 32768, 0, 128, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 0, 0, 896, 0, 0, 128, 0, 128, 32768, 29568, 128, 128, 32768, 0, 1152, 896, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32512, 32768, 128, 128, 0, 256, 128, 0, 128, 128, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 0, 128, 128, 0, 14720, 128, 256, 0, 26368, 32768, 0, 128, 24576, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 128, 32768, 128, 0, 1024, 32768, 31744, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 256, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 0, 15872, 32768, 32768, 16768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 0, 32768, 32768, 0, 0, 128, 0, 32640, 0, 0, 256, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 32768, 32640, 128, 128, 128, 0, 128, 0, 128, 17408, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[3632, 0, 4073, 0, 0, 4060, 3262, 0, 0, 0, 21, 17, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 4088, 0, 4096, 0, 4096, 0, 0, 0, 0, 9, 154, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 4068, 4096, 0, 0, 4, 158, 4096, 9, 4077, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4086, 0, 0, 1500, 4096, 4076, 0, 4033, 0, 4094, 0, 0, 8, 0, 0, 271, 0, 4072, 4093, 0, 0, 0, 25, 0, 4096, 0, 0, 3392, 45, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[2921, 0, 4088, 0, 0, 3997, 1030, 0, 0, 0, 69, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 4096, 4095, 0, 4096, 0, 4096, 33, 0, 0, 0, 0, 113, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 3963, 4096, 0, 0, 68, 35, 4096, 44, 4013, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3985, 0, 0, 2794, 4096, 4070, 0, 4056, 0, 4096, 0, 0, 65, 0, 0, 2242, 0, 3646, 4059, 0, 0, 0, 402, 0, 4096, 0, 0, 3368, 105, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[2741, 1, 4086, 0, 0, 4083, 1359, 1, 0, 0, 5, 2, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 4095, 4095, 4095, 1, 4095, 1, 4095, 16, 0, 0, 1, 0, 25, 0, 0, 0, 0, 4095, 0, 6, 1, 4092, 0, 0, 0, 4094, 4095, 1, 1, 4087, 4095, 0, 0, 90, 0, 4095, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 4095, 0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 4095, 1, 0, 4095, 4095, 0, 1, 4010, 4095, 4095, 1, 4095, 0, 4095, 1, 1, 0, 1, 0, 109, 1, 4016, 4095, 0, 0, 0, 5, 0, 4095, 0, 0, 4029, 0, 4095, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[3142, 0, 4085, 0, 0, 4034, 3351, 0, 0, 0, 35, 18, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 4096, 4094, 0, 4096, 0, 4096, 53, 0, 0, 0, 9, 105, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4092, 0, 0, 4085, 4096, 0, 0, 101, 48, 4096, 19, 4022, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 1964, 4096, 4082, 0, 4070, 0, 4096, 0, 0, 5, 0, 0, 156, 0, 3757, 4053, 0, 0, 0, 75, 0, 4096, 0, 0, 3721, 74, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[3409, 0, 4086, 0, 0, 4014, 2378, 0, 0, 0, 16, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 4096, 0, 4096, 0, 4096, 39, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4088, 4096, 0, 0, 4012, 4096, 0, 0, 49, 50, 4096, 8, 4069, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4093, 0, 0, 3223, 4096, 4089, 0, 4087, 0, 4096, 0, 0, 14, 0, 0, 215, 0, 3368, 4096, 0, 0, 0, 84, 0, 4096, 0, 0, 3701, 52, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[2579, 0, 4076, 0, 0, 4065, 1732, 0, 0, 0, 15, 28, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4096, 4091, 0, 4096, 0, 4096, 3, 0, 0, 0, 9, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4094, 4096, 0, 0, 4090, 4096, 0, 0, 67, 75, 4096, 8, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4017, 0, 0, 3800, 4096, 4082, 0, 4089, 2, 4095, 0, 0, 4, 0, 0, 86, 0, 3870, 4088, 0, 0, 0, 266, 0, 4096, 0, 0, 3990, 26, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[3106, 0, 4086, 0, 0, 4095, 1242, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 4096, 0, 4096, 259, 0, 0, 0, 0, 28, 6, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 0, 0, 4089, 4096, 0, 0, 4094, 4096, 0, 0, 5, 37, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 3886, 4096, 4096, 0, 4075, 0, 4096, 0, 0, 0, 0, 0, 102, 0, 3706, 4096, 0, 1, 0, 11, 0, 4096, 0, 0, 4026, 3, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[3382, 0, 4065, 0, 0, 4005, 1954, 0, 0, 0, 12, 14, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 4092, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 18, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4085, 4092, 0, 0, 4077, 4096, 0, 0, 255, 69, 4096, 5, 4081, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4069, 0, 0, 2829, 4096, 4089, 0, 4053, 0, 4096, 0, 0, 8, 0, 0, 202, 0, 4052, 4091, 0, 0, 1, 55, 0, 4096, 0, 0, 3759, 23, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2741, 1, 4086, 0, 0, 4083, 1359, 1, 0, 0, 5, 2, 1, 1, 1, 0, 2984, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 4095, 4095, 4095, 4095, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 4095, 16, 0, 0, 1, 0, 25, 0, 0, 0, 0, 4095, 0, 3568, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 3995, 0, 0, 0, 4094, 4095, 1, 1, 4087, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 4095, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 4095, 1, 0, 4095, 4095, 4073, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4010, 697, 4095, 1, 4095, 0, 4095, 1, 1, 0, 1, 0, 109, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 4095, 0, 0, 0, 5, 0, 490, 0, 0, 4029, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15345, 16407, 16311, 16370, 16399, 16403, 17107, 16730] avg=16384.0000 max=17107 max_rank=6 min=15345 min_rank=0 imbalance_ratio=1.044128 max_load_reduction=113965 max_load_reduction_pct=0.869484 moved_to_echo=66639 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3632, 0, 4073, 0, 0, 4060, 3262, 0, 0, 0, 21, 17, 0, 0, 0, 0, 2985, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 4088, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 9, 154, 0, 0, 0, 0, 4096, 0, 3569, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 4096, 4096, 0, 0, 4068, 0, 0, 0, 4, 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 4077, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4086, 4075, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1500, 697, 4076, 0, 4033, 0, 4094, 0, 0, 8, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 4093, 0, 0, 0, 25, 0, 490, 0, 0, 3392, 45, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18130, 16382, 16434, 16421, 16391, 16422, 14679, 16213] avg=16384.0000 max=18130 max_rank=0 min=14679 min_rank=6 imbalance_ratio=1.106567 max_load_reduction=112942 max_load_reduction_pct=0.861679 moved_to_echo=63705 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2921, 0, 4088, 0, 0, 3997, 1030, 0, 0, 0, 69, 54, 0, 0, 0, 0, 2985, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 4096, 4095, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 33, 0, 0, 0, 0, 113, 0, 0, 0, 0, 4096, 0, 3569, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 4096, 4096, 0, 0, 3963, 0, 0, 0, 68, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 44, 4013, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3985, 4075, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2794, 697, 4070, 0, 4056, 0, 4096, 0, 0, 65, 0, 0, 2242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3646, 4059, 0, 0, 0, 402, 0, 490, 0, 0, 3368, 105, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15224, 16337, 16417, 16257, 16362, 16289, 18020, 16166] avg=16384.0000 max=18020 max_rank=6 min=15224 min_rank=0 imbalance_ratio=1.099854 max_load_reduction=113052 max_load_reduction_pct=0.862518 moved_to_echo=66837 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3106, 0, 4086, 0, 0, 4095, 1242, 0, 0, 0, 1, 2, 0, 0, 0, 0, 2985, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 259, 0, 0, 0, 0, 28, 6, 0, 0, 0, 4096, 0, 3569, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3999, 0, 0, 0, 4089, 4096, 0, 0, 4094, 0, 0, 0, 5, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4075, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3886, 697, 4096, 0, 4075, 0, 4096, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3706, 4096, 0, 1, 0, 11, 0, 490, 0, 0, 4026, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15597, 16404, 16564, 16321, 16400, 16405, 16952, 16429] avg=16384.0000 max=16952 max_rank=6 min=15597 min_rank=0 imbalance_ratio=1.034668 max_load_reduction=114120 max_load_reduction_pct=0.870667 moved_to_echo=66186 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3142, 0, 4085, 0, 0, 4034, 3351, 0, 0, 0, 35, 18, 0, 0, 0, 0, 2988, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 4096, 4094, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 53, 0, 0, 0, 9, 105, 0, 0, 0, 0, 4096, 0, 3575, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3989, 0, 0, 0, 4096, 4092, 0, 0, 4085, 0, 0, 0, 101, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 19, 4022, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4074, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1964, 691, 4082, 0, 4070, 0, 4096, 0, 0, 5, 0, 0, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3757, 4053, 0, 0, 0, 75, 0, 478, 0, 0, 3721, 74, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17736, 16373, 16447, 16411, 16353, 16434, 15064, 16254] avg=16384.0000 max=17736 max_rank=0 min=15064 min_rank=6 imbalance_ratio=1.082520 max_load_reduction=113336 max_load_reduction_pct=0.864685 moved_to_echo=64105 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2579, 0, 4076, 0, 0, 4065, 1732, 0, 0, 0, 15, 28, 0, 0, 0, 0, 2985, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4096, 4091, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 3, 0, 0, 0, 9, 7, 0, 0, 0, 0, 4096, 0, 3569, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 4094, 4096, 0, 0, 4090, 0, 0, 0, 67, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4017, 4074, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3800, 697, 4082, 0, 4089, 2, 4095, 0, 0, 4, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3870, 4088, 0, 0, 0, 266, 0, 490, 0, 0, 3990, 26, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15560, 16387, 16290, 16421, 16396, 16337, 16855, 16826] avg=16384.0000 max=16855 max_rank=6 min=15560 min_rank=0 imbalance_ratio=1.028748 max_load_reduction=114217 max_load_reduction_pct=0.871407 moved_to_echo=66414 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3382, 0, 4065, 0, 0, 4005, 1954, 0, 0, 0, 12, 14, 0, 0, 0, 0, 2985, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 4092, 4096, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 5, 18, 0, 0, 0, 0, 4096, 0, 3569, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 4085, 4092, 0, 0, 4077, 0, 0, 0, 255, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 4081, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4069, 4075, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2829, 697, 4089, 0, 4053, 0, 4096, 0, 0, 8, 0, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 4091, 0, 0, 1, 55, 0, 490, 0, 0, 3759, 23, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16497, 16392, 16294, 16577, 16389, 16382, 15974, 16567] avg=16384.0000 max=16577 max_rank=3 min=15974 min_rank=6 imbalance_ratio=1.011780 max_load_reduction=114495 max_load_reduction_pct=0.873528 moved_to_echo=65312 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3409, 0, 4086, 0, 0, 4014, 2378, 0, 0, 0, 16, 15, 0, 0, 0, 0, 2985, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 4096, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 39, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 3569, 414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 4088, 4096, 0, 0, 4012, 0, 0, 0, 49, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 4069, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4093, 4074, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3223, 697, 4089, 0, 4087, 0, 4096, 0, 0, 14, 0, 0, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3368, 4096, 0, 0, 0, 84, 0, 490, 0, 0, 3701, 52, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16983, 16390, 16315, 16294, 16382, 16400, 16421, 15887] avg=16384.0000 max=16983 max_rank=0 min=15887 min_rank=7 imbalance_ratio=1.036560 max_load_reduction=114089 max_load_reduction_pct=0.870430 moved_to_echo=65090 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[24960, 128, 32768, 0, 0, 32384, 16384, 128, 0, 0, 256, 256, 128, 128, 128, 0, 23936, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 128, 32640, 32768, 0, 0, 0, 0, 0, 0, 0, 128, 0, 128, 128, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 32768, 128, 32768, 512, 0, 0, 128, 128, 512, 128, 0, 0, 0, 32768, 0, 28672, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[30592, 32768, 0, 128, 128, 1024, 0, 3968, 0, 0, 30080, 384, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 128, 128, 128, 0, 0, 128, 128, 32768, 128, 0, 32768, 32640, 32640, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 24064, 5632, 32768, 128, 32640, 128, 32768, 128, 128, 128, 128, 0, 3456, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 128, 32000, 0, 0, 0, 32768, 32768, 128, 128, 32512, 0, 0, 0, 640, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 128, 128, 0, 0, 0, 128, 0, 0, 0, 128, 128, 32640, 32768, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 1, 1, 4095, 4095, 4095, 1, 0, 102, 0, 4078, 0, 0, 0, 0, 1, 0, 1, 1, 1, 4094, 0, 1, 4095, 4094, 1, 0, 0, 1, 4095, 4095, 4095, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 4095, 4095, 4095, 3, 0, 0, 0, 1, 0, 0, 1, 1, 3723, 0, 1, 0, 4093, 0, 0, 4095, 0, 1, 4095, 1, 1, 0, 0, 0, 0, 1, 4094, 4093, 0, 1, 1042, 4092, 0, 0, 0, 3344, 4095, 1, 4095, 4095, 1, 0, 1, 4095, 4095, 0, 1, 0, 1, 1, 0, 0, 4094, 0, 4095, 0, 0, 0, 4086, 0, 0, 6, 0, 4, 1, 0, 1, 1, 1, 0, 1, 4094, 1, 0, 1, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 661, 0, 4093, 0, 0, 0, 0, 0, 52, 0, 0, 0, 4075, 13, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 4096, 4096, 130, 0, 0, 0, 0, 0, 0, 0, 0, 3546, 0, 0, 0, 4055, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4024, 0, 0, 683, 4096, 0, 0, 0, 3356, 4078, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 4096, 0, 0, 0, 4052, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4089, 4094, 0, 0, 110, 0, 4096, 0, 0, 0, 3, 0, 9, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4095, 23, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4080, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4076, 0, 0, 59, 4096, 0, 0, 0, 3944, 4072, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4096, 0, 0, 0, 4092, 0, 0, 14, 0, 25, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4088, 4096, 0, 0, 217, 0, 4095, 0, 0, 0, 18, 0, 32, 0, 0, 0, 4094, 4, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 4096, 4096, 27, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 4064, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 4058, 0, 0, 251, 4096, 0, 0, 0, 3689, 4083, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 4096, 0, 0, 0, 4079, 0, 0, 13, 0, 1, 0, 0, 0, 35, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4088, 4091, 0, 0, 130, 0, 4096, 0, 0, 0, 8, 0, 11, 0, 0, 0, 4088, 8, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 4096, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 4096, 4083, 0, 0, 85, 4096, 0, 0, 0, 3917, 4084, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 4088, 0, 0, 3, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4089, 4092, 0, 0, 393, 0, 4016, 0, 0, 0, 9, 0, 5, 0, 0, 0, 4093, 4, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4095, 4096, 0, 2, 5, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 4096, 4096, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 4079, 0, 0, 528, 4096, 0, 0, 0, 3423, 4089, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 4077, 0, 0, 2, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 118, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 9, 0, 0, 0, 0, 4096, 4096, 4095, 1, 0, 0, 1, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 220, 4095, 0, 0, 0, 3694, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 2, 0, 6, 0, 0, 0, 80, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4094, 4092, 4092, 0, 0, 411, 0, 4078, 0, 0, 0, 5, 0, 8, 0, 0, 0, 4096, 3, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 1, 5, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 4096, 4094, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 0, 436, 4096, 0, 0, 0, 3335, 4088, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4095, 0, 0, 0, 4090, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1, 1, 3852, 4095, 4095, 1, 0, 102, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 4094, 0, 1, 4095, 4094, 1, 0, 0, 1, 3988, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4093, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 3, 0, 0, 0, 1, 0, 0, 1, 1, 3723, 0, 1, 0, 4093, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 4095, 1, 1, 0, 0, 0, 0, 1, 4094, 4093, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1042, 4092, 0, 0, 0, 3344, 4095, 1, 0, 0, 1, 0, 1, 4095, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 4094, 0, 4095, 0, 0, 0, 4086, 0, 0, 4095, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4, 1, 0, 1, 1, 1, 0, 1, 4094, 1, 0, 1, 1, 1, 4095, 4095, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16226, 16364, 16379, 16151, 16402, 16779, 16402, 16369] avg=16384.0000 max=16779 max_rank=5 min=16151 min_rank=3 imbalance_ratio=1.024109 max_load_reduction=114293 max_load_reduction_pct=0.871986 moved_to_echo=65952 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3853, 4096, 4096, 0, 0, 661, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 4075, 13, 0, 4096, 4096, 0, 0, 0, 0, 3989, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 130, 0, 0, 0, 0, 0, 0, 0, 0, 3546, 0, 0, 0, 4055, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4024, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 4096, 0, 0, 0, 3356, 4078, 0, 0, 0, 0, 0, 0, 4096, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 4096, 0, 0, 0, 4052, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16799, 16407, 16375, 16061, 16333, 16416, 16319, 16362] avg=16384.0000 max=16799 max_rank=0 min=16061 min_rank=3 imbalance_ratio=1.025330 max_load_reduction=114273 max_load_reduction_pct=0.871834 moved_to_echo=65430 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3843, 4088, 4096, 0, 0, 217, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 32, 0, 0, 0, 4094, 4, 0, 4096, 4096, 0, 0, 0, 0, 3994, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 27, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 4064, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 4058, 0, 20, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 4096, 0, 0, 0, 3689, 4083, 0, 0, 0, 0, 0, 0, 4096, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 4096, 0, 0, 0, 4079, 0, 0, 4096, 29, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 1, 0, 0, 0, 35, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16339, 16425, 16381, 16474, 16372, 16313, 16364, 16404] avg=16384.0000 max=16474 max_rank=3 min=16313 min_rank=5 imbalance_ratio=1.005493 max_load_reduction=114598 max_load_reduction_pct=0.874313 moved_to_echo=65453 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3853, 4096, 4096, 0, 0, 118, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 0, 3989, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4095, 9, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 1, 0, 0, 1, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4096, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 4095, 0, 0, 0, 3694, 4096, 0, 0, 0, 0, 0, 0, 4096, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6, 0, 0, 0, 80, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16241, 16362, 16390, 16511, 16404, 16308, 16412, 16444] avg=16384.0000 max=16511 max_rank=3 min=16241 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=114561 max_load_reduction_pct=0.874031 moved_to_echo=65568 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3853, 4088, 4091, 0, 0, 130, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 11, 0, 0, 0, 4088, 8, 0, 4096, 4096, 0, 0, 0, 0, 3989, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 4085, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 4096, 4083, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 4096, 0, 0, 0, 3917, 4084, 0, 0, 0, 0, 0, 0, 4096, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 4088, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16258, 16382, 16384, 16500, 16396, 16385, 16396, 16371] avg=16384.0000 max=16500 max_rank=3 min=16258 min_rank=0 imbalance_ratio=1.007080 max_load_reduction=114572 max_load_reduction_pct=0.874115 moved_to_echo=65548 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3851, 4092, 4092, 0, 0, 411, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8, 0, 0, 0, 4096, 3, 0, 4096, 4096, 0, 0, 0, 0, 3989, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 1, 5, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 4088, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 436, 4096, 0, 0, 0, 3335, 4088, 0, 0, 0, 0, 0, 0, 4096, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4095, 0, 0, 0, 4090, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16524, 16379, 16390, 16465, 16395, 16158, 16401, 16360] avg=16384.0000 max=16524 max_rank=0 min=16158 min_rank=5 imbalance_ratio=1.008545 max_load_reduction=114548 max_load_reduction_pct=0.873932 moved_to_echo=65314 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3853, 4089, 4092, 0, 0, 393, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 5, 0, 0, 0, 4093, 4, 0, 4096, 4096, 0, 0, 0, 0, 3989, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4094, 0, 2, 5, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 0, 0, 0, 4085, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 4079, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 4096, 0, 0, 0, 3423, 4089, 0, 0, 0, 0, 0, 0, 4096, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 4077, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16443, 16378, 16391, 16385, 16385, 16339, 16387, 16364] avg=16384.0000 max=16443 max_rank=0 min=16339 min_rank=5 imbalance_ratio=1.003601 max_load_reduction=114629 max_load_reduction_pct=0.874550 moved_to_echo=65475 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 3853, 4089, 4094, 0, 0, 110, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 9, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 3989, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 23, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4080, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4076, 0, 19, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 4096, 0, 0, 0, 3944, 4072, 0, 0, 0, 0, 0, 0, 4096, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4096, 0, 0, 0, 4092, 0, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 25, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 4096, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16242, 16375, 16382, 16525, 16385, 16374, 16391, 16398] avg=16384.0000 max=16525 max_rank=3 min=16242 min_rank=0 imbalance_ratio=1.008606 max_load_reduction=114547 max_load_reduction_pct=0.873924 moved_to_echo=65548 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 32768, 32768, 0, 128, 128, 0, 128, 0, 32768, 128, 0, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 3328, 32768, 0, 0, 0, 28800, 32768, 128, 0, 0, 128, 0, 128, 32768, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 128, 0, 128, 128, 30848, 32768, 32768, 128, 0, 2176, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 32768, 256, 0, 0, 128, 128, 0, 0, 128, 128, 31616, 0, 128, 0, 32768, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 32768, 0, 128, 32768, 128, 128, 0, 128, 0, 0, 128, 32768, 32640, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 32768, 0, 32768, 0, 0, 0, 32768, 0, 0, 32768, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 0, 128, 128, 128, 0, 128, 32768, 128, 0, 128, 128, 128, 32768, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 32768, 128, 0, 0, 128, 32000, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 11, 4096, 0, 777, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4064, 0, 0, 10, 4085, 0, 541, 4096, 4096, 0, 0, 0, 0, 0, 4096, 2704, 4096, 0, 3396, 3936, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 812, 0, 0, 0, 4096, 0, 4096, 4096, 0, 30, 289, 4096, 4096, 4078, 3994, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 24, 0, 0, 0, 0, 4096, 0, 0, 4004, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 9, 4096, 0, 679, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 4087, 0, 0, 9, 4094, 0, 766, 4083, 4096, 0, 0, 0, 0, 0, 4090, 3429, 4096, 0, 3024, 4083, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 224, 0, 0, 0, 4096, 0, 4096, 4096, 0, 11, 134, 4096, 4096, 4092, 4083, 0, 4096, 0, 4, 0, 4095, 0, 4096, 2, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3992, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 9, 4096, 0, 172, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 0, 0, 0, 4044, 0, 0, 4, 4094, 0, 294, 4075, 4096, 0, 0, 0, 0, 0, 4096, 3768, 4096, 0, 3898, 4082, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 163, 0, 0, 0, 4096, 0, 4096, 4096, 0, 18, 137, 4096, 4096, 4087, 4027, 0, 4096, 0, 8, 0, 4094, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 22, 0, 0, 0, 0, 4096, 0, 0, 4044, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 12, 4096, 0, 112, 0, 0, 0, 0, 0, 4094, 0, 0, 4089, 0, 0, 0, 4083, 0, 0, 9, 4096, 0, 221, 4089, 4096, 0, 0, 0, 0, 0, 4082, 3992, 4096, 0, 3969, 4084, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 81, 0, 0, 0, 4096, 0, 4096, 4096, 0, 10, 346, 4096, 4096, 4096, 3599, 0, 4096, 0, 3, 0, 4091, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 4096, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 1, 0, 1, 0, 4094, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 4095, 1, 1, 4093, 1, 1, 1, 0, 4095, 0, 457, 1, 0, 0, 0, 0, 4094, 0, 1, 4095, 5, 1, 0, 4094, 0, 0, 0, 4095, 0, 130, 4094, 4095, 0, 0, 1, 1, 0, 4095, 3767, 4095, 0, 3409, 4095, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 412, 1, 1, 0, 4095, 0, 4095, 4093, 0, 0, 99, 4095, 4095, 4095, 4026, 1, 4085, 0, 0, 1, 4092, 0, 4095, 1, 1, 0, 1, 4095, 0, 1, 3, 0, 1, 1, 0, 0, 4095, 0, 0, 4090, 4095, 0, 1, 1, 2, 1, 1, 0, 1, 1, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 341, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4080, 0, 0, 5, 4096, 0, 554, 4096, 4096, 0, 0, 0, 0, 0, 4096, 3259, 4096, 0, 3753, 4073, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 145, 0, 0, 0, 4096, 0, 4096, 4096, 0, 11, 318, 4096, 4096, 4096, 3966, 0, 4096, 0, 0, 0, 4087, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 0, 0, 0, 0, 4096, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 4096, 0, 103, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 4084, 0, 0, 10, 4095, 0, 62, 4088, 4096, 0, 0, 0, 0, 0, 4095, 3556, 4096, 0, 3957, 4087, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 105, 0, 0, 0, 4096, 0, 4096, 4096, 0, 8, 459, 4096, 4096, 4091, 4074, 0, 4096, 0, 4, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 478, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 168, 4096, 4096, 0, 0, 0, 0, 0, 4096, 3256, 4096, 0, 3936, 4086, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 148, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 277, 4096, 4096, 4095, 4034, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 16, 4096, 4071, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 9, 4096, 0, 679, 0, 0, 3338, 233, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 4087, 0, 0, 9, 4094, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 766, 4083, 0, 0, 0, 0, 0, 0, 752, 3429, 0, 0, 3024, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 224, 0, 0, 0, 4096, 0, 3836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 134, 0, 0, 4092, 4083, 0, 3955, 0, 4, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 0, 4057, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16374, 16672, 16393, 16137, 16348, 16374, 16380, 16394] avg=16384.0000 max=16672 max_rank=1 min=16137 min_rank=3 imbalance_ratio=1.017578 max_load_reduction=114400 max_load_reduction_pct=0.872803 moved_to_echo=65496 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 14, 4096, 4071, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 12, 4096, 0, 112, 0, 0, 3331, 233, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4089, 0, 0, 0, 4083, 0, 0, 9, 4096, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 4089, 0, 0, 0, 0, 0, 0, 751, 3992, 0, 0, 3969, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 81, 0, 0, 0, 4096, 0, 3835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 346, 0, 0, 4096, 3599, 0, 3955, 0, 3, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 4096, 0, 4058, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16395, 16101, 16387, 17106, 16204, 16100, 16383, 16396] avg=16384.0000 max=17106 max_rank=3 min=16100 min_rank=5 imbalance_ratio=1.044067 max_load_reduction=113966 max_load_reduction_pct=0.869492 moved_to_echo=65083 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 92, 4096, 4071, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 11, 4096, 0, 777, 0, 0, 3343, 233, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4064, 0, 0, 10, 4085, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 4096, 0, 0, 0, 0, 0, 0, 753, 2704, 0, 0, 3396, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 812, 0, 0, 0, 4096, 0, 3835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 289, 0, 0, 4078, 3994, 0, 3955, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 24, 0, 0, 0, 0, 4096, 0, 4058, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16410, 16777, 16367, 15426, 16935, 16442, 16398, 16317] avg=16384.0000 max=16935 max_rank=4 min=15426 min_rank=3 imbalance_ratio=1.033630 max_load_reduction=114137 max_load_reduction_pct=0.870796 moved_to_echo=66092 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3992, 0, 0, 0, 0, 0, 0, 33, 4096, 4072, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 9, 4096, 0, 172, 0, 0, 3346, 234, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 0, 0, 0, 4044, 0, 0, 4, 4094, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 294, 4075, 0, 0, 0, 0, 0, 0, 750, 3768, 0, 0, 3898, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 163, 0, 0, 0, 4096, 0, 3835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 137, 0, 0, 4087, 4027, 0, 3951, 0, 8, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 22, 0, 0, 0, 0, 4096, 0, 4057, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16328, 16177, 16341, 16867, 16286, 16322, 16395, 16356] avg=16384.0000 max=16867 max_rank=3 min=16177 min_rank=1 imbalance_ratio=1.029480 max_load_reduction=114205 max_load_reduction_pct=0.871315 moved_to_echo=65359 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4071, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 478, 0, 0, 3343, 233, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 4096, 0, 0, 0, 0, 0, 0, 753, 3256, 0, 0, 3936, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 148, 0, 0, 0, 4096, 0, 3835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 4095, 4034, 0, 3954, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 4058, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 16468, 16402, 16295, 16271, 16456, 16374, 16409] avg=16384.0000 max=16468 max_rank=1 min=16271 min_rank=4 imbalance_ratio=1.005127 max_load_reduction=114604 max_load_reduction_pct=0.874359 moved_to_echo=65510 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4095, 1, 0, 1, 0, 4094, 0, 1, 0, 0, 1, 0, 0, 4095, 4070, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 4093, 1, 1, 1, 0, 4095, 0, 457, 1, 0, 3342, 234, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 4095, 5, 1, 0, 4094, 0, 0, 0, 4095, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 4094, 0, 0, 0, 1, 1, 0, 753, 3767, 0, 0, 3409, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 412, 1, 1, 0, 4095, 0, 3832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 4095, 4026, 1, 3945, 0, 0, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 1, 4095, 0, 1, 3, 0, 1, 1, 0, 0, 4095, 0, 4056, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4095, 0, 1, 1, 2, 1, 1, 0, 1, 1, 0, 0, 0, 0, 4095, 4095, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16398, 16447, 16401, 16250, 16532, 16259, 16377, 16408] avg=16384.0000 max=16532 max_rank=4 min=16250 min_rank=3 imbalance_ratio=1.009033 max_load_reduction=114540 max_load_reduction_pct=0.873871 moved_to_echo=65576 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 15, 4096, 4071, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 4096, 0, 103, 0, 0, 3342, 233, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 4084, 0, 0, 10, 4095, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 4088, 0, 0, 0, 0, 0, 0, 753, 3556, 0, 0, 3957, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 105, 0, 0, 0, 4096, 0, 3835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 459, 0, 0, 4091, 4074, 0, 3955, 0, 4, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 4058, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16097, 16392, 16503, 16228, 16686, 16379, 16399] avg=16384.0000 max=16686 max_rank=5 min=16097 min_rank=1 imbalance_ratio=1.018433 max_load_reduction=114386 max_load_reduction_pct=0.872696 moved_to_echo=65692 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 8, 4096, 4071, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 341, 0, 0, 3343, 234, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4080, 0, 0, 5, 4096, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 554, 4096, 0, 0, 0, 0, 0, 0, 753, 3259, 0, 0, 3753, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 145, 0, 0, 0, 4096, 0, 3835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 318, 0, 0, 4096, 3966, 0, 3955, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 0, 0, 0, 0, 4096, 0, 4057, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16333, 16389, 16488, 16268, 16433, 16386, 16393] avg=16384.0000 max=16488 max_rank=3 min=16268 min_rank=4 imbalance_ratio=1.006348 max_load_reduction=114584 max_load_reduction_pct=0.874207 moved_to_echo=65480 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 0, 0, 32768, 0, 0, 0, 128, 2176, 128, 128, 0, 32768, 0, 30720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32640, 32768, 0, 128, 128, 128, 128, 128, 0, 128, 128, 0, 128, 0, 0, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 128, 128, 0, 0, 32768, 0, 32512, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 32768, 0, 3200, 128, 0, 26752, 1920, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 0, 32768, 128, 0, 128, 0, 32512, 0, 128, 0, 0, 128, 0, 256, 32768, 32640, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 0, 32768, 0, 128, 32768, 128, 128, 0, 32640, 0, 0, 128, 32768, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[2816, 32768, 0, 0, 0, 128, 128, 0, 6144, 27776, 0, 0, 29440, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 2176, 0, 0, 32768, 31872, 128, 31744, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[4083, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 1254, 0, 0, 0, 0, 4096, 1079, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1115, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 849, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3848, 4096, 0, 0, 0, 0, 39, 0, 4096, 0, 0, 4096, 0, 5, 0, 0, 3, 4096, 0, 0, 4096, 0, 0, 4096, 4095, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[3983, 0, 0, 19, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1463, 0, 0, 0, 0, 4087, 768, 0, 0, 0, 4081, 0, 0, 0, 0, 1, 0, 4096, 1357, 0, 4096, 0, 0, 4096, 0, 4077, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 665, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 3715, 4096, 0, 0, 0, 0, 345, 0, 4096, 0, 0, 4085, 0, 0, 7, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4035, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2015, 0, 0, 0, 0, 4093, 679, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4093, 167, 0, 4095, 0, 0, 4096, 0, 4092, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1479, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4096, 0, 0, 0, 4, 0, 3664, 4096, 0, 0, 0, 0, 249, 0, 4096, 0, 0, 4089, 0, 0, 4, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4092, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4095, 1, 0, 1, 1, 1, 4094, 0, 1, 1, 0, 1, 1, 0, 4095, 0, 942, 0, 0, 0, 0, 4095, 1305, 0, 1, 0, 4095, 1, 0, 0, 0, 0, 0, 4095, 147, 2, 4095, 0, 0, 4095, 0, 4095, 4095, 0, 0, 0, 4083, 0, 1, 1, 0, 0, 1, 0, 4095, 0, 1, 1, 0, 0, 1, 4093, 4095, 1, 0, 0, 1, 0, 1, 1, 0, 0, 4095, 0, 4095, 4093, 1, 1, 0, 0, 4095, 1, 4095, 1685, 1, 4095, 1, 1, 0, 1, 0, 0, 0, 4095, 0, 4095, 1, 1, 1, 5, 0, 4034, 4094, 0, 0, 0, 1, 46, 0, 4095, 1, 1, 4095, 0, 8, 0, 1, 33, 4095, 0, 0, 4095, 0, 3, 4095, 4091, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4052, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1494, 0, 0, 0, 0, 4092, 577, 0, 0, 0, 4092, 0, 2, 0, 0, 0, 0, 4096, 204, 5, 4096, 0, 0, 4096, 0, 4091, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1806, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 3774, 4096, 0, 0, 0, 0, 339, 0, 4096, 0, 0, 4088, 0, 14, 5, 0, 19, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4091, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[3935, 0, 0, 22, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2725, 0, 0, 0, 0, 4084, 780, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 160, 0, 4096, 0, 0, 4096, 0, 4085, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 16, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 529, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3264, 4096, 0, 0, 0, 0, 875, 0, 4096, 0, 0, 4074, 0, 4, 7, 0, 16, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4031, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2277, 0, 0, 0, 0, 4093, 861, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 367, 0, 4096, 0, 0, 4096, 0, 4092, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 659, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3855, 4096, 0, 0, 0, 0, 237, 0, 4096, 0, 0, 4093, 0, 0, 1, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[4057, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 982, 0, 0, 0, 0, 4091, 1407, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 1327, 0, 4095, 0, 0, 4096, 0, 4089, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 576, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3757, 4096, 0, 0, 0, 0, 147, 0, 4096, 0, 0, 4086, 0, 22, 6, 0, 20, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4083, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 2496, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 0, 0, 0, 0, 4096, 1079, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1115, 0, 3499, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2446, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2446, 1032, 597, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3064, 849, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3848, 4096, 0, 0, 0, 0, 39, 0, 4096, 0, 0, 2568, 1600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4084, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16435, 16149, 16901, 16384, 16375, 16202, 16247, 16379] avg=16384.0000 max=16901 max_rank=2 min=16149 min_rank=1 imbalance_ratio=1.031555 max_load_reduction=114171 max_load_reduction_pct=0.871056 moved_to_echo=65203 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4035, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2497, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2015, 0, 0, 0, 0, 4093, 679, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 167, 0, 3498, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2446, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2446, 1032, 597, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3064, 1479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3664, 4096, 0, 0, 0, 0, 249, 0, 4096, 0, 0, 2568, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 16503, 15946, 16386, 16384, 16834, 16276, 16365] avg=16384.0000 max=16834 max_rank=5 min=15946 min_rank=2 imbalance_ratio=1.027466 max_load_reduction=114238 max_load_reduction_pct=0.871567 moved_to_echo=65859 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3983, 0, 0, 19, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2497, 1651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1463, 0, 0, 0, 0, 4087, 768, 0, 0, 0, 4081, 0, 0, 0, 0, 1, 4096, 1529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1357, 0, 3495, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2445, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2446, 1034, 601, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3062, 665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3715, 4096, 0, 0, 0, 0, 345, 0, 4096, 0, 0, 2567, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 4082, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16342, 16025, 17121, 16383, 16401, 16016, 16418, 16366] avg=16384.0000 max=17121 max_rank=2 min=16016 min_rank=5 imbalance_ratio=1.044983 max_load_reduction=113951 max_load_reduction_pct=0.869377 moved_to_echo=65201 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4095, 1, 0, 1, 1, 1, 4094, 0, 1, 1, 0, 1, 1, 0, 4095, 0, 2496, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 942, 0, 0, 0, 0, 4095, 1305, 0, 1, 0, 4095, 1, 0, 0, 0, 0, 4095, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 147, 2, 3498, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 4095, 0, 1, 1, 0, 0, 1, 4093, 4095, 1, 2445, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 4095, 0, 4095, 4093, 1, 1, 0, 0, 2445, 1032, 597, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 3063, 1685, 1, 0, 1, 1, 0, 1, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 5, 0, 4034, 4094, 0, 0, 0, 1, 46, 0, 4095, 1, 1, 2567, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 8, 0, 1, 33, 0, 0, 0, 0, 0, 3, 4083, 4091, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16438, 16062, 15920, 16385, 16374, 17038, 16446, 16409] avg=16384.0000 max=17038 max_rank=5 min=15920 min_rank=2 imbalance_ratio=1.039917 max_load_reduction=114034 max_load_reduction_pct=0.870010 moved_to_echo=66267 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3935, 0, 0, 22, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2496, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2725, 0, 0, 0, 0, 4084, 780, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 160, 0, 3499, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2446, 1649, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2447, 1032, 597, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3064, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3264, 4096, 0, 0, 0, 0, 875, 0, 4096, 0, 0, 2568, 1600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 4, 7, 0, 16, 0, 0, 0, 0, 0, 0, 4084, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16295, 17309, 15936, 16383, 16392, 15881, 16499, 16377] avg=16384.0000 max=17309 max_rank=1 min=15881 min_rank=5 imbalance_ratio=1.056458 max_load_reduction=113763 max_load_reduction_pct=0.867943 moved_to_echo=65149 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4031, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2497, 1651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2277, 0, 0, 0, 0, 4093, 861, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 1529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 367, 0, 3499, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2445, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2446, 1032, 597, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3064, 659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3855, 4096, 0, 0, 0, 0, 237, 0, 4096, 0, 0, 2567, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 4084, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16949, 16150, 16383, 16379, 16011, 16450, 16371] avg=16384.0000 max=16949 max_rank=1 min=16011 min_rank=5 imbalance_ratio=1.034485 max_load_reduction=114123 max_load_reduction_pct=0.870689 moved_to_echo=65211 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4057, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2496, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 982, 0, 0, 0, 0, 4091, 1407, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1327, 0, 3498, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2446, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2446, 1032, 597, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3064, 576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3757, 4096, 0, 0, 0, 0, 147, 0, 4096, 0, 0, 2568, 1600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 22, 6, 0, 20, 0, 0, 0, 0, 0, 0, 4084, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16403, 16194, 17107, 16384, 16384, 15928, 16264, 16408] avg=16384.0000 max=17107 max_rank=2 min=15928 min_rank=5 imbalance_ratio=1.044128 max_load_reduction=113965 max_load_reduction_pct=0.869484 moved_to_echo=64984 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4052, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2497, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1494, 0, 0, 0, 0, 4092, 577, 0, 0, 0, 4092, 0, 2, 0, 0, 0, 4096, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 204, 5, 3499, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 2446, 1650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 2446, 1032, 597, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3064, 1806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3774, 4096, 0, 0, 0, 0, 339, 0, 4096, 0, 0, 2568, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 14, 5, 0, 19, 0, 0, 0, 0, 0, 0, 4084, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16402, 15881, 15991, 16384, 16383, 17162, 16472, 16397] avg=16384.0000 max=17162 max_rank=5 min=15881 min_rank=1 imbalance_ratio=1.047485 max_load_reduction=113910 max_load_reduction_pct=0.869064 moved_to_echo=66414 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 32768, 4864, 128, 28032, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 24576, 8320, 128, 0, 128, 128, 128, 128, 0, 0, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 0, 29952, 32768, 0, 0, 0, 128, 2304, 0, 32768, 128, 128, 20608, 12800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32384, 128, 0, 128, 128, 128, 32768, 0, 128, 128, 128, 128, 128, 0, 32768, 0, 20096, 13312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[13184, 0, 0, 0, 0, 32768, 7552, 0, 128, 0, 32768, 128, 128, 0, 0, 128, 32768, 12288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 128, 128, 0, 0, 32768, 0, 32768, 32768, 128, 128, 0, 0, 19584, 8320, 4864, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 0, 128, 128, 32768, 0, 128, 128, 0, 0, 128, 32768, 32768, 128, 19584, 13312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3552, 4096, 0, 0, 0, 0, 0, 4096, 0, 51, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 3953, 0, 0, 4095, 0, 101, 0, 930, 0, 0, 0, 2736, 0, 0, 0, 0, 3930, 4096, 1, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4077, 0, 1031, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2828, 4096, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4088, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 882, 0, 0, 0, 0, 0, 4058, 0, 0, 4096, 0, 17, 2, 120, 0, 0, 0, 3023, 0, 0, 0, 0, 4055, 4096, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 7, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4081, 0, 1389, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4094, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3881, 4095, 1, 0, 0, 0, 0, 4093, 0, 5, 1, 1, 4095, 4094, 1, 0, 1, 0, 0, 4095, 1, 1, 0, 1, 4095, 0, 0, 0, 1, 190, 1, 1, 0, 1, 1, 4095, 1, 0, 4095, 0, 3, 0, 1467, 1, 0, 0, 4062, 0, 0, 15, 0, 2486, 4095, 0, 0, 4095, 4087, 0, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 0, 0, 12, 1, 1, 61, 4094, 0, 4095, 4065, 0, 1, 0, 0, 0, 1, 1, 1, 4095, 0, 0, 4092, 1, 1, 1, 4095, 0, 4095, 0, 169, 0, 17, 0, 1, 4026, 0, 4092, 4094, 0, 4095, 0, 0, 0, 4095, 3, 0, 1, 4095, 4095, 4095, 0, 4095, 9, 0, 24, 1, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3408, 4096, 0, 0, 0, 0, 0, 4096, 0, 41, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4085, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 197, 0, 0, 0, 0, 0, 4048, 0, 0, 4090, 0, 30, 5, 596, 0, 0, 0, 3541, 0, 0, 0, 0, 4069, 4094, 1, 0, 4093, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 1, 4096, 7, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4091, 0, 473, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 4096, 4084, 4096, 4, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1751, 4096, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4029, 0, 0, 4074, 0, 38, 0, 137, 0, 0, 0, 3605, 0, 0, 4, 0, 4093, 4096, 4, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 2705, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3947, 4096, 0, 0, 0, 0, 0, 4096, 0, 35, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 4043, 0, 0, 4080, 0, 53, 4, 367, 0, 0, 0, 3125, 0, 0, 0, 0, 4006, 4096, 14, 0, 4094, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4065, 0, 812, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 1, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3439, 4096, 0, 0, 0, 0, 0, 4094, 0, 5, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 0, 0, 139, 0, 0, 0, 3787, 0, 0, 14, 0, 4087, 4096, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 805, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 4096, 0, 4096, 1, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2810, 4096, 0, 0, 0, 0, 0, 4096, 0, 32, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 4052, 0, 0, 4092, 0, 10, 0, 259, 0, 0, 0, 3679, 0, 0, 4, 0, 4095, 4096, 3, 0, 4094, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 5, 4096, 2, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 0, 1320, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 4096, 4096, 2, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3552, 4096, 0, 0, 0, 0, 0, 4096, 0, 51, 0, 0, 4096, 681, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 3953, 0, 0, 4095, 0, 101, 0, 930, 0, 4096, 3415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2736, 0, 0, 0, 0, 3930, 896, 1, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 4096, 3011, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3911, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 1031, 0, 0, 0, 0, 4096, 0, 4096, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16757, 16353, 16745, 15752, 16370, 16388, 16326, 16381] avg=16384.0000 max=16757 max_rank=0 min=15752 min_rank=3 imbalance_ratio=1.022766 max_load_reduction=114315 max_load_reduction_pct=0.872154 moved_to_echo=65465 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2828, 4096, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 4096, 682, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4088, 0, 0, 4, 0, 4096, 0, 0, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 882, 0, 0, 0, 0, 0, 4058, 0, 0, 4096, 0, 17, 2, 120, 0, 4096, 3414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3023, 0, 0, 0, 0, 4055, 896, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 7, 4096, 3011, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3910, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 1389, 0, 0, 0, 0, 4096, 0, 4096, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16007, 16390, 16685, 16154, 16382, 16387, 16688, 16379] avg=16384.0000 max=16688 max_rank=6 min=16007 min_rank=0 imbalance_ratio=1.018555 max_load_reduction=114384 max_load_reduction_pct=0.872681 moved_to_echo=65836 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3881, 4095, 1, 0, 0, 0, 0, 4093, 0, 5, 1, 1, 4095, 682, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4094, 1, 0, 1, 0, 0, 4095, 1, 1, 0, 1, 4095, 0, 0, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 190, 1, 1, 0, 1, 1, 4095, 1, 0, 4095, 0, 3, 0, 1467, 1, 4095, 3413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 15, 0, 2486, 896, 0, 0, 4095, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 1, 0, 0, 0, 0, 12, 1, 1, 61, 4094, 0, 4095, 3010, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 1, 0, 0, 0, 1, 1, 1, 4095, 0, 0, 4092, 1, 1, 1, 3909, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 169, 0, 17, 0, 1, 4026, 0, 4092, 3024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 1, 0, 4095, 0, 0, 4092, 9, 0, 24, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17041, 16394, 17365, 15641, 16435, 16357, 15424, 16415] avg=16384.0000 max=17365 max_rank=2 min=15424 min_rank=6 imbalance_ratio=1.059875 max_load_reduction=113707 max_load_reduction_pct=0.867516 moved_to_echo=64631 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3439, 4096, 0, 0, 0, 0, 0, 4094, 0, 5, 0, 0, 4096, 681, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 4087, 0, 0, 4096, 0, 0, 0, 139, 0, 4096, 3415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3787, 0, 0, 14, 0, 4087, 896, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 3011, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3910, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 805, 0, 0, 0, 0, 4096, 0, 4096, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4093, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16597, 16394, 15856, 16975, 16363, 16387, 16119, 16381] avg=16384.0000 max=16975 max_rank=3 min=15856 min_rank=2 imbalance_ratio=1.036072 max_load_reduction=114097 max_load_reduction_pct=0.870491 moved_to_echo=65250 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3947, 4096, 0, 0, 0, 0, 0, 4096, 0, 35, 0, 0, 4096, 682, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 4043, 0, 0, 4080, 0, 53, 4, 367, 0, 4096, 3414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3125, 0, 0, 0, 0, 4006, 887, 14, 0, 4094, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4096, 0, 4096, 3013, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3910, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 812, 0, 0, 0, 0, 4096, 0, 4096, 3018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17138, 16379, 16083, 16217, 16398, 16394, 16087, 16376] avg=16384.0000 max=17138 max_rank=0 min=16083 min_rank=2 imbalance_ratio=1.046021 max_load_reduction=113934 max_load_reduction_pct=0.869247 moved_to_echo=65255 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3408, 4096, 0, 0, 0, 0, 0, 4096, 0, 41, 0, 0, 4096, 682, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4085, 0, 0, 4, 0, 4096, 0, 0, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 0, 0, 0, 0, 0, 4048, 0, 0, 4090, 0, 30, 5, 596, 0, 4096, 3414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3541, 0, 0, 0, 0, 4069, 896, 1, 0, 4093, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 1, 4096, 7, 4096, 3009, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3910, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 473, 0, 0, 0, 0, 4096, 0, 4096, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4084, 0, 4, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16605, 16387, 16476, 16691, 16369, 16387, 15782, 16375] avg=16384.0000 max=16691 max_rank=3 min=15782 min_rank=6 imbalance_ratio=1.018738 max_load_reduction=114381 max_load_reduction_pct=0.872658 moved_to_echo=64913 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2810, 4096, 0, 0, 0, 0, 0, 4096, 0, 32, 0, 0, 4096, 682, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 4052, 0, 0, 4092, 0, 10, 0, 259, 0, 4096, 3414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3679, 0, 0, 4, 0, 4095, 896, 3, 0, 4094, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 5, 4096, 2, 4096, 3011, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3910, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 1320, 0, 0, 0, 0, 4096, 0, 4096, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 2, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15998, 16387, 16057, 16863, 16371, 16385, 16627, 16384] avg=16384.0000 max=16863 max_rank=3 min=15998 min_rank=0 imbalance_ratio=1.029236 max_load_reduction=114209 max_load_reduction_pct=0.871346 moved_to_echo=65767 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1751, 4096, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 4096, 682, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4029, 0, 0, 4074, 0, 38, 0, 137, 0, 4096, 3414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3605, 0, 0, 4, 0, 4093, 896, 4, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4096, 0, 4096, 3011, 1063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3910, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2705, 0, 0, 0, 0, 4096, 0, 4096, 3026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14929, 16388, 15805, 16779, 16384, 16387, 18019, 16381] avg=16384.0000 max=18019 max_rank=6 min=14929 min_rank=0 imbalance_ratio=1.099792 max_load_reduction=113053 max_load_reduction_pct=0.862526 moved_to_echo=67171 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 0, 128, 0, 0, 32768, 128, 128, 128, 128, 32768, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 1664, 128, 128, 0, 128, 128, 32384, 128, 0, 32768, 0, 256, 128, 4096, 128, 32768, 27392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 128, 128, 128, 32768, 0, 0, 32768, 128, 128, 128, 31360, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 128, 0, 25728, 32768, 128, 0, 0, 0, 0, 32768, 0, 256, 128, 128, 32768, 5504, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 0, 32768, 0, 128, 32768, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 32768, 0, 8704, 0, 128, 0, 128, 32768, 0, 32768, 24320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 27648, 0, 0, 128, 0, 30848, 7168, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 32768, 128, 0, 128, 0, 0, 128, 128, 128, 256, 32768, 128, 32768, 24192, 8576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4033, 11, 0, 0, 0, 78, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4089, 3025, 0, 0, 4096, 0, 0, 4049, 4096, 4096, 0, 0, 0, 0, 0, 0, 3901, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1248, 4096, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4082, 4067, 4096, 4096, 4096, 0, 4080, 0, 0, 4096, 0, 57, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 3956, 9, 0, 0, 0, 98, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4079, 3699, 0, 0, 4096, 0, 0, 4015, 4096, 4096, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 578, 4094, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4080, 4095, 4096, 4090, 4096, 0, 4079, 0, 0, 4096, 0, 26, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 3962, 75, 0, 0, 0, 85, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4072, 3799, 0, 0, 4096, 0, 0, 3966, 4096, 4096, 0, 0, 0, 0, 0, 0, 3669, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 817, 4096, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 73, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4037, 4010, 4096, 4096, 4096, 0, 4030, 0, 0, 4096, 0, 109, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 3998, 63, 0, 0, 0, 164, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4069, 3397, 0, 0, 4096, 0, 0, 4021, 4096, 4096, 0, 0, 0, 0, 0, 0, 3735, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1024, 4093, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 15, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4070, 4023, 4096, 4096, 4096, 0, 4082, 0, 0, 4096, 0, 54, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4056, 4, 0, 0, 0, 123, 0, 0, 0, 4096, 0, 5, 0, 1, 0, 4096, 4094, 0, 0, 0, 0, 0, 4090, 3185, 0, 0, 4096, 0, 0, 4075, 4096, 4096, 0, 0, 0, 0, 0, 0, 3883, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1040, 4092, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 4096, 0, 4, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4087, 4053, 4096, 4096, 4096, 0, 4084, 0, 0, 4096, 0, 44, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4095, 1, 1, 4095, 1, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 1, 4093, 0, 0, 1, 127, 1, 4095, 4094, 0, 0, 1, 0, 0, 4095, 3238, 0, 0, 4094, 0, 0, 4095, 4095, 4095, 0, 0, 2, 1, 0, 1, 3993, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 4095, 1, 0, 4069, 0, 1, 1, 4095, 1, 0, 0, 0, 0, 864, 4095, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 4095, 0, 0, 4095, 1, 1, 0, 0, 4088, 0, 1, 1, 4095, 1, 0, 8, 4094, 0, 4095, 4093, 4095, 4095, 4095, 0, 4095, 0, 1, 4095, 0, 0, 1, 3, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4054, 3, 0, 0, 0, 34, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 4091, 3443, 0, 0, 4096, 0, 0, 4076, 4096, 4096, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 748, 4093, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 4096, 0, 5, 4096, 0, 0, 2, 0, 4096, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 4086, 4083, 4096, 4096, 4096, 0, 4089, 0, 0, 4096, 0, 25, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4093, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 4094, 3923, 0, 0, 4096, 0, 0, 4080, 4096, 4096, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 216, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4075, 4095, 4096, 4096, 0, 4096, 0, 0, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4067, 0, 0, 3956, 9, 0, 0, 0, 98, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4079, 3699, 0, 0, 577, 29, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4015, 4096, 233, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 4096, 3286, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 4094, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 4096, 3265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4095, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4095, 29, 4090, 0, 0, 4079, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16322, 16596, 16462, 16387, 16140, 16384, 16382, 16399] avg=16384.0000 max=16596 max_rank=1 min=16140 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=114476 max_load_reduction_pct=0.873383 moved_to_echo=65305 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4065, 0, 0, 3998, 63, 0, 0, 0, 164, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4069, 3397, 0, 0, 583, 31, 20, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4021, 4096, 225, 0, 0, 0, 0, 0, 0, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 4096, 3288, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1024, 4093, 0, 0, 0, 0, 0, 26, 0, 0, 0, 4096, 4096, 3268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 0, 0, 0, 0, 15, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 4023, 19, 4096, 0, 0, 4082, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16482, 16295, 16173, 16415, 16603, 16379, 16381, 16344] avg=16384.0000 max=16603 max_rank=4 min=16173 min_rank=2 imbalance_ratio=1.013367 max_load_reduction=114469 max_load_reduction_pct=0.873329 moved_to_echo=65707 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4067, 0, 0, 3962, 75, 0, 0, 0, 85, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4072, 3799, 0, 0, 577, 29, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3966, 4096, 233, 0, 0, 0, 0, 0, 0, 3669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3286, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 817, 4096, 0, 0, 0, 0, 0, 51, 0, 0, 0, 4096, 4096, 3265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 0, 73, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 4010, 29, 4096, 0, 0, 4030, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16689, 16060, 16389, 16421, 16439, 16382, 16311] avg=16384.0000 max=16689 max_rank=1 min=16060 min_rank=2 imbalance_ratio=1.018616 max_load_reduction=114383 max_load_reduction_pct=0.872673 moved_to_echo=65553 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4095, 1, 1, 4066, 1, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 127, 1, 4095, 4094, 0, 0, 1, 0, 0, 4095, 3238, 0, 0, 577, 29, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4095, 4095, 233, 0, 0, 2, 1, 0, 1, 3993, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 4095, 1, 0, 4069, 0, 1, 1, 4095, 3285, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 864, 4095, 1, 1, 0, 0, 1, 0, 0, 0, 1, 4095, 4095, 3264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4077, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 0, 0, 4088, 0, 1, 1, 4095, 1, 0, 8, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4093, 29, 4095, 0, 0, 4095, 0, 1, 0, 0, 0, 1, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16354, 16277, 16516, 16354, 16418, 16356, 16384, 16413] avg=16384.0000 max=16516 max_rank=2 min=16277 min_rank=1 imbalance_ratio=1.008057 max_load_reduction=114556 max_load_reduction_pct=0.873993 moved_to_echo=65571 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4067, 0, 0, 4056, 4, 0, 0, 0, 123, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 4096, 4094, 0, 0, 0, 0, 0, 4090, 3185, 0, 0, 577, 29, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4075, 4096, 233, 0, 0, 0, 0, 0, 0, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 4096, 3286, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1040, 4092, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 4096, 3265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4091, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4053, 29, 4096, 0, 0, 4084, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16442, 16097, 16383, 16388, 16599, 16383, 16387, 16393] avg=16384.0000 max=16599 max_rank=4 min=16097 min_rank=1 imbalance_ratio=1.013123 max_load_reduction=114473 max_load_reduction_pct=0.873360 moved_to_echo=65762 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4093, 0, 0, 4067, 0, 0, 4090, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 4094, 3923, 0, 0, 577, 29, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 4096, 233, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3286, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4075, 29, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16360, 16836, 16591, 16376, 15768, 16366, 16382, 16393] avg=16384.0000 max=16836 max_rank=1 min=15768 min_rank=4 imbalance_ratio=1.027588 max_load_reduction=114236 max_load_reduction_pct=0.871552 moved_to_echo=64909 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4067, 0, 0, 4054, 3, 0, 0, 0, 34, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 4091, 3443, 0, 0, 577, 29, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 4096, 233, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3286, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 748, 4093, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 4096, 3265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4093, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4094, 0, 0, 2, 0, 4096, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4083, 29, 4096, 0, 0, 4089, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16350, 16350, 16512, 16382, 16306, 16372, 16392, 16408] avg=16384.0000 max=16512 max_rank=2 min=16306 min_rank=4 imbalance_ratio=1.007812 max_load_reduction=114560 max_load_reduction_pct=0.874023 moved_to_echo=65478 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4067, 0, 0, 4033, 11, 0, 0, 0, 78, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4089, 3025, 0, 0, 577, 29, 18, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4049, 4096, 233, 0, 0, 0, 0, 0, 0, 3901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3286, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1248, 4096, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 4096, 3265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4067, 29, 4096, 0, 0, 4080, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 15932, 16375, 16381, 16817, 16393, 16382, 16411] avg=16384.0000 max=16817 max_rank=4 min=15932 min_rank=1 imbalance_ratio=1.026428 max_load_reduction=114255 max_load_reduction_pct=0.871696 moved_to_echo=66003 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 128, 32640, 128, 0, 32256, 256, 0, 0, 128, 640, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 0, 0, 0, 6656, 32768, 128, 128, 0, 0, 128, 128, 0, 0, 128, 32768, 32768, 26240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32640, 32512, 256, 32768, 0, 0, 32640, 0, 128, 0, 128, 384, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 0, 32384, 32768, 1920, 0, 0, 128, 128, 0, 128, 31360, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 0, 128, 32640, 32768, 0, 0, 0, 0, 256, 0, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 32768, 128, 128, 128, 32768, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 256, 128, 32768, 32768, 0, 0, 128, 0, 0, 32768, 27776, 0, 0, 4736, 256, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 128, 32768, 128, 128, 32768, 0, 128, 128, 32768, 26368, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3118, 0, 0, 0, 42, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 51, 103, 3993, 4096, 0, 244, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 3974, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4, 0, 0, 0, 4096, 4096, 794, 4096, 79, 4067, 0, 0, 0, 0, 3990, 20, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3538, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 32, 24, 4073, 4096, 0, 28, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 9, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4093, 4096, 0, 4089, 4041, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 4090, 543, 4096, 30, 4086, 0, 0, 0, 0, 4067, 9, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2914, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 49, 34, 4067, 4096, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 7, 0, 0, 0, 0, 0, 4096, 4096, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2, 0, 4095, 4096, 1128, 4096, 17, 4083, 0, 0, 0, 0, 4070, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3663, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 55, 54, 4042, 4096, 0, 22, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 29, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4094, 3860, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4092, 622, 4096, 1, 4085, 0, 0, 0, 0, 4030, 12, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 3588, 2, 0, 1, 0, 1, 0, 4, 4089, 1, 1, 0, 4092, 0, 0, 4095, 4095, 11, 0, 0, 1, 0, 1, 4092, 4086, 0, 0, 4095, 1, 9, 1, 1, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 0, 0, 4095, 4095, 1, 115, 1, 4095, 0, 1, 0, 0, 4095, 1, 1, 4094, 0, 0, 4037, 0, 2, 0, 1, 1, 0, 1, 4091, 4092, 0, 4095, 3179, 0, 5, 1, 1, 1, 0, 4095, 1, 4095, 0, 0, 1, 39, 1, 4037, 4095, 770, 4095, 0, 4094, 0, 0, 0, 0, 4095, 590, 0, 1, 0, 0, 4093, 1, 0, 0, 1, 1, 4095, 1, 4095, 2, 0, 0, 0, 0, 4, 4095, 26, 4095, 0, 0, 0, 4086] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3757, 0, 0, 0, 10, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 3, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 30, 45, 4064, 4096, 0, 25, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 4096, 4096, 0, 4092, 3802, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 4096, 4090, 593, 4096, 19, 4089, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3767, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 6, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 39, 17, 4074, 4096, 0, 5, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 4095, 4096, 0, 4092, 3457, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4091, 4092, 911, 4096, 15, 4088, 0, 0, 0, 0, 4058, 40, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3676, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 15, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4094, 3975, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 0, 4093, 4096, 514, 4096, 17, 4095, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3118, 0, 0, 0, 42, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 51, 103, 3993, 4096, 3514, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 244, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 3974, 0, 0, 0, 0, 239, 54, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3356, 0, 4, 0, 0, 0, 4096, 4096, 794, 0, 79, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3990, 20, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3551, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16030, 16385, 16398, 16574, 16558, 16492, 16254, 16381] avg=16384.0000 max=16574 max_rank=3 min=16030 min_rank=0 imbalance_ratio=1.011597 max_load_reduction=114498 max_load_reduction_pct=0.873550 moved_to_echo=65685 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3538, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 32, 24, 4073, 4096, 3514, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 28, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4093, 4096, 0, 4089, 4041, 0, 0, 0, 0, 239, 54, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3356, 0, 0, 0, 0, 0, 4095, 4090, 543, 0, 30, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 9, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3550, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16417, 16384, 16386, 16367, 16616, 16200, 16319, 16383] avg=16384.0000 max=16616 max_rank=4 min=16200 min_rank=5 imbalance_ratio=1.014160 max_load_reduction=114456 max_load_reduction_pct=0.873230 moved_to_echo=65518 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3663, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 55, 54, 4042, 4096, 3513, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 22, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 29, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4094, 3860, 0, 0, 0, 0, 245, 58, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3347, 0, 0, 0, 0, 0, 4096, 4092, 622, 0, 1, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 12, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3550, 504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16541, 16380, 16402, 16377, 16460, 16243, 16288, 16381] avg=16384.0000 max=16541 max_rank=0 min=16243 min_rank=5 imbalance_ratio=1.009583 max_load_reduction=114531 max_load_reduction_pct=0.873802 moved_to_echo=65372 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3676, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 4096, 3514, 545, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 15, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4094, 3975, 0, 0, 0, 0, 239, 54, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3356, 0, 1, 0, 0, 0, 4093, 4096, 514, 0, 17, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3551, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16546, 16384, 16349, 16346, 16557, 16172, 16337, 16381] avg=16384.0000 max=16557 max_rank=4 min=16172 min_rank=5 imbalance_ratio=1.010559 max_load_reduction=114515 max_load_reduction_pct=0.873680 moved_to_echo=65447 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3767, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 39, 17, 4074, 4096, 3514, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4095, 4096, 0, 4092, 3457, 0, 0, 0, 0, 239, 54, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3356, 0, 0, 0, 0, 0, 4091, 4092, 911, 0, 15, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 40, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3550, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16641, 16390, 16381, 16335, 16047, 16553, 16341, 16384] avg=16384.0000 max=16641 max_rank=0 min=16047 min_rank=4 imbalance_ratio=1.015686 max_load_reduction=114431 max_load_reduction_pct=0.873039 moved_to_echo=65325 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 3588, 2, 0, 1, 0, 1, 0, 4, 4089, 1, 1, 0, 4095, 4095, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 11, 0, 0, 1, 0, 1, 4092, 4086, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 1, 1, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 0, 0, 4095, 4095, 3513, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 1, 115, 1, 0, 0, 1, 0, 0, 4095, 1, 1, 4094, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 0, 1, 4091, 4092, 0, 4095, 3179, 0, 5, 1, 1, 239, 54, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 3355, 0, 0, 1, 39, 1, 4037, 4095, 770, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 590, 0, 1, 0, 0, 4093, 1, 0, 0, 1, 1, 4095, 3549, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 4095, 2, 0, 0, 0, 0, 4, 4095, 26, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16461, 16378, 16359, 16387, 15765, 16394, 16927, 16401] avg=16384.0000 max=16927 max_rank=6 min=15765 min_rank=4 imbalance_ratio=1.033142 max_load_reduction=114145 max_load_reduction_pct=0.870857 moved_to_echo=65487 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3757, 0, 0, 0, 10, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 582, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 30, 45, 4064, 4096, 3514, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 25, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 4096, 0, 4092, 3802, 0, 0, 0, 0, 239, 54, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3354, 0, 0, 0, 0, 0, 4096, 4090, 593, 0, 19, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3550, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16637, 16387, 16396, 16355, 16387, 16241, 16289, 16380] avg=16384.0000 max=16637 max_rank=0 min=16241 min_rank=5 imbalance_ratio=1.015442 max_load_reduction=114435 max_load_reduction_pct=0.873070 moved_to_echo=65297 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2914, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 49, 34, 4067, 4096, 3513, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 4096, 0, 4091, 4096, 0, 0, 0, 0, 239, 54, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3356, 0, 0, 0, 2, 0, 4095, 4096, 1128, 0, 17, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3550, 501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15799, 16384, 16401, 16331, 16682, 16777, 16317, 16381] avg=16384.0000 max=16777 max_rank=5 min=15799 min_rank=0 imbalance_ratio=1.023987 max_load_reduction=114295 max_load_reduction_pct=0.872002 moved_to_echo=66157 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 28032, 128, 0, 128, 128, 128, 0, 128, 32768, 128, 128, 0, 32768, 32768, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 128, 32768, 128, 0, 0, 0, 0, 128, 32768, 128, 0, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 0, 0, 32512, 768, 0, 128, 0, 0, 32768, 128, 0, 0, 128, 128, 32768, 28416, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32384, 128, 512, 128, 0, 0, 128, 0, 0, 32768, 128, 128, 32768, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 26880, 0, 128, 128, 128, 128, 32768, 32768, 5888, 0, 256, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 0, 0, 0, 0, 128, 0, 0, 128, 128, 128, 32768, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 32768, 32768, 0, 32768, 30464, 0, 128, 128, 128, 1920, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 0, 32768, 0, 128, 128, 128, 384, 384, 32512, 32768, 28160, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 0, 1, 3, 0, 0, 4096, 4096, 4096, 0, 4096, 110, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1806, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 828, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1623, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 3832, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 22, 0, 4094, 4096, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4064, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4095, 0, 0, 0, 59, 1, 0, 4094, 4095, 4071, 1, 4095, 304, 1, 0, 1, 0, 0, 0, 4095, 4065, 0, 13, 0, 0, 4093, 1, 7, 0, 1, 1, 0, 4095, 1, 1, 0, 0, 26, 1, 1, 1, 0, 3627, 0, 4087, 4080, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 9, 1, 1, 0, 326, 1, 4095, 0, 1, 3963, 0, 1, 0, 4093, 500, 3995, 0, 1, 0, 0, 2, 4094, 4095, 0, 3694, 0, 0, 1, 4089, 0, 0, 1, 0, 1, 4095, 1, 4091, 4073, 0, 1, 75, 0, 4095, 4076, 28, 16, 4095, 0, 1, 2, 4, 3986, 0, 0, 0, 0, 1, 0, 4047, 4095, 0, 1, 1, 1, 4, 16, 4093, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 315, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2836, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 509, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 555, 4094, 0, 0, 0, 0, 0, 4096, 4096, 0, 4019, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 43, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 4032, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 35, 0, 8, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3109, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 321, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1570, 4082, 0, 0, 0, 0, 0, 4096, 4096, 0, 3174, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 12, 0, 4086, 4096, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4085, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 40, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1528, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 713, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2011, 4088, 0, 0, 0, 0, 0, 4096, 4096, 0, 3928, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 13, 0, 4088, 4096, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4070, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 69, 0, 7, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3056, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 301, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 840, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 3934, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 16, 0, 4092, 4096, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4073, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 3, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2772, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 530, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1151, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 3732, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 7, 0, 4092, 4096, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4073, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 7, 0, 1, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2773, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 732, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 936, 4085, 0, 0, 0, 0, 0, 4096, 4096, 0, 3755, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 0, 1, 3, 0, 0, 4096, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 2333, 1120, 532, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1806, 0, 4096, 4096, 1402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3564, 0, 0, 0, 4096, 0, 0, 0, 0, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2976, 0, 0, 4096, 0, 0, 0, 4096, 1623, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3832, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 334, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 22, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2694, 1429, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16399, 15503, 16680, 16886, 16455, 16395, 16379] avg=16384.0000 max=16886 max_rank=4 min=15503 min_rank=2 imbalance_ratio=1.030640 max_load_reduction=114186 max_load_reduction_pct=0.871170 moved_to_echo=66115 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4095, 0, 0, 0, 59, 1, 0, 4094, 0, 4071, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 1, 0, 1, 0, 0, 0, 4095, 4065, 0, 13, 0, 0, 4093, 1, 7, 2328, 1114, 530, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 1, 1, 0, 0, 26, 1, 1, 1, 0, 3627, 0, 4095, 4095, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4080, 1, 1, 1, 1, 3565, 0, 0, 0, 4095, 9, 1, 1, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2981, 0, 1, 3963, 0, 1, 0, 4093, 500, 3995, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3694, 0, 0, 1, 4089, 0, 0, 1, 0, 1, 340, 1, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 1, 75, 0, 4095, 4076, 28, 16, 0, 0, 1, 2, 4, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4047, 4095, 0, 1, 1, 1, 4, 16, 4093, 1, 0, 2695, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16414, 16556, 17345, 16168, 15538, 16312, 16357, 16382] avg=16384.0000 max=17345 max_rank=2 min=15538 min_rank=4 imbalance_ratio=1.058655 max_load_reduction=113727 max_load_reduction_pct=0.867668 moved_to_echo=64589 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 2329, 1114, 531, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2836, 0, 4096, 4096, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3565, 0, 0, 0, 4096, 0, 0, 0, 0, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2982, 0, 0, 4096, 0, 0, 0, 4096, 555, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4019, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 339, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 43, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2693, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16581, 16526, 16362, 15823, 16646, 16409, 16345] avg=16384.0000 max=16646 max_rank=5 min=15823 min_rank=4 imbalance_ratio=1.015991 max_load_reduction=114426 max_load_reduction_pct=0.873001 moved_to_echo=65223 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 3, 0, 0, 0, 4096, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 2329, 1114, 531, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2772, 0, 4096, 4096, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3565, 0, 0, 0, 4096, 0, 0, 0, 0, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2982, 0, 0, 4096, 0, 0, 0, 4096, 1151, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3732, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 339, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 4092, 4096, 0, 0, 2, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2693, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16297, 16464, 16383, 16421, 16359, 16381, 16386] avg=16384.0000 max=16464 max_rank=2 min=16297 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=114608 max_load_reduction_pct=0.874390 moved_to_echo=65547 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4095, 0, 0, 2329, 1114, 531, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2773, 0, 4096, 4096, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3565, 0, 0, 0, 4096, 0, 0, 0, 0, 732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2982, 0, 0, 4096, 0, 0, 0, 4096, 936, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3755, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 339, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2693, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16272, 16462, 16585, 16195, 16382, 16387, 16409] avg=16384.0000 max=16585 max_rank=3 min=16195 min_rank=4 imbalance_ratio=1.012268 max_load_reduction=114487 max_load_reduction_pct=0.873466 moved_to_echo=65373 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4, 0, 0, 0, 4096, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 8, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 2329, 1114, 531, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3109, 0, 4096, 4096, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3565, 0, 0, 0, 4096, 0, 0, 0, 0, 321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2982, 0, 0, 4096, 0, 0, 0, 4096, 1570, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3174, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 339, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 12, 0, 4086, 4096, 0, 0, 2, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2693, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16309, 16803, 16174, 16826, 15802, 16380, 16398] avg=16384.0000 max=16826 max_rank=4 min=15802 min_rank=5 imbalance_ratio=1.026978 max_load_reduction=114246 max_load_reduction_pct=0.871628 moved_to_echo=65406 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4, 0, 0, 0, 4096, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 7, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 2329, 1114, 531, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3056, 0, 4096, 4096, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3565, 0, 0, 0, 4096, 0, 0, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2982, 0, 0, 4096, 0, 0, 0, 4096, 840, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3934, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 339, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 16, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2695, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16342, 16748, 16154, 16110, 16561, 16387, 16388] avg=16384.0000 max=16748 max_rank=2 min=16110 min_rank=4 imbalance_ratio=1.022217 max_load_reduction=114324 max_load_reduction_pct=0.872223 moved_to_echo=65446 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 10, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 2329, 1114, 531, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1528, 0, 4096, 4096, 1401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3565, 0, 0, 0, 4096, 0, 0, 0, 0, 713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2982, 0, 0, 4096, 0, 0, 0, 4096, 2011, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3928, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 339, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 13, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2695, 1428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16316, 15221, 16566, 17273, 16555, 16376, 16385] avg=16384.0000 max=17273 max_rank=4 min=15221 min_rank=2 imbalance_ratio=1.054260 max_load_reduction=113799 max_load_reduction_pct=0.868217 moved_to_echo=66589 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 0, 30080, 0, 128, 128, 32768, 0, 0, 128, 0, 128, 2816, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 32768, 0, 0, 128, 128, 128, 0, 32768, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1024, 128, 128, 128, 0, 0, 0, 32768, 32768, 0, 128, 0, 0, 32768, 128, 128, 18688, 8960, 4352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 128, 128, 128, 28544, 0, 0, 0, 32768, 128, 128, 128, 0, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 128, 0, 0, 128, 128, 128, 128, 128, 21632, 0, 32768, 32768, 11264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 23936, 0, 128, 32640, 0, 128, 0, 32768, 9216, 32640, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 128, 256, 0, 32768, 32768, 128, 128, 128, 0, 128, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 0, 128, 0, 32640, 32768, 0, 128, 128, 128, 128, 128, 32768, 128, 0, 21632, 11520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4078, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1348, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 35, 0, 4096, 0, 0, 0, 4096, 2643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1469, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4093, 4083, 0, 4095, 0, 4075, 1, 0, 4095, 1, 3983, 1, 4094, 0, 1, 0, 0, 1, 0, 1, 1, 4095, 4095, 0, 1, 4095, 4086, 4045, 0, 3460, 1, 1, 1, 0, 0, 1, 1, 4095, 17, 22, 0, 3, 1, 0, 4059, 0, 0, 1, 4094, 1, 3985, 63, 0, 0, 4087, 4095, 1, 1, 1, 0, 0, 1, 4072, 9, 1, 2, 4094, 2169, 2, 0, 0, 1, 1, 0, 5, 1, 164, 0, 0, 1, 1, 63, 1, 0, 4094, 4095, 1, 0, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 4076, 4095, 0, 0, 26, 4095, 1, 36, 0, 0, 0, 1, 1, 0, 1, 2578, 0, 0, 0, 0, 4095, 4090, 4066, 8, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 2275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1829, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4084, 0, 4095, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 4096, 2292, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1835, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4089, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 4096, 3668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 4096, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4089, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 4096, 2360, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4094, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1835, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 25, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 2275, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4091, 3970, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1829, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 2631, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16408, 16393, 16501, 16388, 15883, 16366, 16378, 16755] avg=16384.0000 max=16755 max_rank=7 min=15883 min_rank=4 imbalance_ratio=1.022644 max_load_reduction=114317 max_load_reduction_pct=0.872169 moved_to_echo=65382 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4078, 0, 25, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 4096, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 4096, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2865, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 1, 0, 4091, 3970, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 4096, 0, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1348, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 2631, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16391, 16392, 16401, 16397, 16472, 16359, 16386, 16274] avg=16384.0000 max=16472 max_rank=4 min=16274 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=114600 max_load_reduction_pct=0.874329 moved_to_echo=65491 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 4083, 0, 25, 0, 4075, 1, 0, 0, 1, 3983, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 1, 0, 1, 1, 8, 0, 0, 1, 4095, 4086, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3460, 1, 1, 1, 0, 0, 1, 1, 4095, 17, 22, 0, 3, 1, 0, 4095, 4095, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 1, 4094, 1, 3985, 63, 0, 0, 4087, 4, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4072, 9, 1, 2, 4094, 2169, 2, 0, 0, 1, 1, 0, 5, 1, 3974, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 1, 1, 63, 1, 0, 4094, 4095, 1, 0, 0, 0, 0, 1, 4091, 3969, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 0, 4076, 4095, 0, 0, 26, 4095, 1, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 2578, 0, 0, 0, 0, 4095, 4090, 4066, 8, 0, 0, 0, 2630, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16262, 16333, 15919, 16297, 15772, 16594, 16418, 17477] avg=16384.0000 max=17477 max_rank=7 min=15772 min_rank=4 imbalance_ratio=1.066711 max_load_reduction=113595 max_load_reduction_pct=0.866661 moved_to_echo=66261 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 25, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 4096, 4096, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 4096, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 4096, 0, 0, 0, 4096, 2643, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4096, 0, 0, 0, 0, 0, 0, 4091, 3971, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1469, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 2631, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16409, 16392, 16473, 16408, 16285, 16333, 16377, 16395] avg=16384.0000 max=16473 max_rank=2 min=16285 min_rank=4 imbalance_ratio=1.005432 max_load_reduction=114599 max_load_reduction_pct=0.874321 moved_to_echo=65390 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4084, 0, 25, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 4096, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 4096, 2292, 0, 2, 0, 0, 0, 0, 0, 0, 3975, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4092, 0, 0, 0, 0, 0, 0, 4091, 3970, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 4094, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1835, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 2630, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 16392, 16487, 16394, 15908, 16352, 16382, 16760] avg=16384.0000 max=16760 max_rank=7 min=15908 min_rank=4 imbalance_ratio=1.022949 max_load_reduction=114312 max_load_reduction_pct=0.872131 moved_to_echo=65402 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4089, 0, 17, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 4096, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 19, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 0, 4096, 3668, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 1444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4095, 0, 0, 0, 0, 0, 0, 4096, 3970, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 4096, 0, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 2635, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16394, 16386, 16440, 16403, 17292, 16346, 16379, 15432] avg=16384.0000 max=17292 max_rank=4 min=15432 min_rank=7 imbalance_ratio=1.055420 max_load_reduction=113780 max_load_reduction_pct=0.868073 moved_to_echo=65449 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4089, 0, 25, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 4096, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 4096, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 4096, 2360, 0, 2, 0, 0, 0, 0, 0, 0, 3975, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4094, 0, 0, 0, 0, 2, 0, 4091, 3970, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1835, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 2631, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16402, 16392, 16410, 16393, 15977, 16362, 16375, 16761] avg=16384.0000 max=16761 max_rank=7 min=15977 min_rank=4 imbalance_ratio=1.023010 max_load_reduction=114311 max_load_reduction_pct=0.872124 moved_to_echo=65475 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 25, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 4096, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 4096, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 4093, 3970, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 2631, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16409, 16392, 16441, 16392, 17483, 16360, 16377, 15218] avg=16384.0000 max=17483 max_rank=4 min=15218 min_rank=7 imbalance_ratio=1.067078 max_load_reduction=113589 max_load_reduction_pct=0.866615 moved_to_echo=65438 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 32768, 32768, 0, 256, 0, 32768, 128, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 0, 128, 0, 128, 128, 0, 128, 128, 128, 0, 0, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 32768, 22144, 128, 128, 0, 128, 128, 0, 128, 128, 31872, 11648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 128, 32768, 0, 32768, 32768, 0, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 128, 128, 32768, 128, 32768, 128, 0, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[256, 0, 0, 128, 128, 128, 128, 0, 32768, 32768, 128, 0, 0, 0, 128, 128, 32768, 31872, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 31872, 128, 128, 128, 0, 0, 128, 128, 32768, 128, 128, 0, 128, 128, 0, 32768, 32768, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 11776, 0, 0, 0, 0, 32768, 32768, 32768, 128, 0, 0, 0, 21120, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4008, 0, 0, 1493, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1372, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 168, 4096, 4093, 0, 992, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4091, 4096, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 4096, 217, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4083, 0, 0, 635, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1813, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 500, 4096, 4089, 0, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4090, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4089, 4096, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 4096, 56, 0, 0, 4096, 0, 4096, 89, 0, 0, 0, 4096, 3950, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 1, 4083, 1, 0, 0, 0, 0, 4094, 1, 1, 5, 3, 1, 14, 2, 4090, 1, 4095, 0, 0, 0, 0, 4095, 0, 1, 4095, 1, 1, 762, 4082, 1, 1, 1, 0, 0, 4094, 0, 18, 0, 1040, 4062, 3, 0, 4092, 1, 0, 5, 0, 0, 0, 0, 0, 4090, 0, 4095, 4094, 0, 1, 0, 945, 4095, 4095, 0, 1383, 0, 0, 0, 0, 1, 0, 1, 1, 4092, 1, 1, 1, 2, 4095, 4095, 4, 5, 4095, 12, 4095, 1, 0, 4095, 4095, 4089, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 4077, 0, 0, 14, 3939, 0, 4095, 24, 0, 0, 1, 4095, 4070, 0, 1, 0, 0, 0, 1, 6, 0, 2, 1, 4084, 148, 4090, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4071, 0, 0, 189, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1537, 4096, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 895, 4096, 4090, 0, 1511, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4092, 4096, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 4096, 91, 0, 0, 4096, 0, 4093, 6, 0, 0, 0, 4096, 3989, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4080, 0, 0, 282, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1541, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 373, 4096, 4096, 0, 1675, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 253, 0, 0, 4096, 0, 4096, 5, 0, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4049, 0, 0, 1568, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 355, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 479, 4096, 4088, 0, 1654, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4087, 4096, 0, 0, 0, 0, 0, 4096, 18, 0, 0, 4096, 154, 0, 0, 4096, 0, 4095, 5, 0, 0, 0, 4096, 4018, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 0, 289, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1947, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 725, 4096, 4092, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4092, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4092, 4096, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 4096, 127, 0, 0, 4096, 0, 4094, 16, 0, 0, 0, 4096, 3947, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 472, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1724, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 704, 4096, 4096, 0, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 4096, 32, 0, 0, 0, 4096, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3412, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 1493, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1372, 4096, 0, 0, 4096, 0, 0, 0, 1360, 684, 594, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 168, 3502, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 992, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 4096, 2733, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 217, 0, 0, 4096, 0, 4055, 2, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4072, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 17105, 16339, 15955, 16027, 16382, 16520, 16364] avg=16384.0000 max=17105 max_rank=1 min=15955 min_rank=3 imbalance_ratio=1.044006 max_load_reduction=113967 max_load_reduction_pct=0.869499 moved_to_echo=65293 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3412, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 635, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1813, 4096, 0, 0, 4096, 0, 0, 0, 1360, 684, 594, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 500, 3502, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4090, 4096, 2733, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 56, 0, 0, 4096, 0, 4055, 89, 0, 0, 0, 0, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4072, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16322, 16780, 16283, 16197, 16380, 16349, 16373] avg=16384.0000 max=16780 max_rank=2 min=16197 min_rank=4 imbalance_ratio=1.024170 max_load_reduction=114292 max_load_reduction_pct=0.871979 moved_to_echo=65299 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4083, 1, 0, 0, 0, 0, 4094, 1, 1, 5, 3, 1, 14, 2, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 3411, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 762, 4082, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4094, 0, 18, 0, 1040, 4062, 3, 0, 4092, 1, 0, 5, 1360, 684, 594, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4094, 0, 1, 0, 945, 3501, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1383, 0, 0, 0, 0, 1, 0, 1, 1, 4092, 1, 1, 1, 2, 4095, 4095, 2734, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 4095, 12, 1, 1, 0, 0, 4095, 4089, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4077, 0, 0, 14, 3939, 0, 4054, 24, 0, 0, 1, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 6, 0, 2, 1, 4084, 148, 4090, 0, 1, 0, 4095, 4070, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16393, 16445, 15996, 16726, 16432, 16397, 16181, 16502] avg=16384.0000 max=16726 max_rank=3 min=15996 min_rank=2 imbalance_ratio=1.020874 max_load_reduction=114346 max_load_reduction_pct=0.872391 moved_to_echo=65512 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3412, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 189, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1537, 4096, 0, 0, 4096, 0, 0, 0, 1360, 684, 594, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 895, 3502, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1511, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4095, 4096, 2733, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 91, 0, 0, 4096, 0, 4052, 6, 0, 0, 0, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4071, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 15864, 16504, 16682, 16552, 16383, 16334, 16368] avg=16384.0000 max=16682 max_rank=3 min=15864 min_rank=1 imbalance_ratio=1.018188 max_load_reduction=114390 max_load_reduction_pct=0.872726 moved_to_echo=65637 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3409, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 1568, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 355, 4096, 0, 0, 4096, 0, 0, 0, 1363, 687, 599, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 479, 3497, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1654, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 4096, 2733, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4096, 154, 0, 0, 4096, 0, 4050, 5, 0, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4071, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 17218, 15337, 16256, 16697, 16375, 16437, 16371] avg=16384.0000 max=17218 max_rank=1 min=15337 min_rank=2 imbalance_ratio=1.050903 max_load_reduction=113854 max_load_reduction_pct=0.868637 moved_to_echo=65880 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3412, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 472, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1724, 4096, 0, 0, 4096, 0, 0, 0, 1360, 684, 594, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 704, 3502, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 2733, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 4055, 32, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4072, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16172, 16691, 16494, 16257, 16388, 16326, 16364] avg=16384.0000 max=16691 max_rank=2 min=16172 min_rank=1 imbalance_ratio=1.018738 max_load_reduction=114381 max_load_reduction_pct=0.872658 moved_to_echo=65335 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3412, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 282, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1541, 4096, 0, 0, 4096, 0, 0, 0, 1363, 684, 594, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 373, 3502, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1675, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 4096, 2733, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 253, 0, 0, 4096, 0, 4055, 5, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4071, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 15966, 16511, 16163, 16719, 16384, 16586, 16362] avg=16384.0000 max=16719 max_rank=4 min=15966 min_rank=1 imbalance_ratio=1.020447 max_load_reduction=114353 max_load_reduction_pct=0.872444 moved_to_echo=66051 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3412, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 289, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1947, 4096, 0, 0, 4096, 0, 0, 0, 1360, 684, 594, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 725, 3502, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4092, 4096, 2733, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 127, 0, 0, 4096, 0, 4053, 16, 0, 0, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4071, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 15980, 16914, 16513, 16191, 16383, 16339, 16368] avg=16384.0000 max=16914 max_rank=2 min=15980 min_rank=1 imbalance_ratio=1.032349 max_load_reduction=114158 max_load_reduction_pct=0.870956 moved_to_echo=65281 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 128, 0, 32768, 128, 128, 128, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 27392, 0, 0, 0, 0, 0, 0, 128, 32640, 128, 128, 5760, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32768, 1024, 0, 128, 32640, 0, 32512, 256, 0, 0, 128, 0, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[10752, 0, 0, 0, 0, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 32768, 32768, 21888, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 0, 32768, 0, 0, 32768, 0, 128, 0, 4864, 28032, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 0, 128, 128, 128, 0, 128, 128, 32768, 256, 32768, 0, 128, 0, 32768, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 0, 0, 32768, 32768, 0, 0, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 0, 32768, 0, 128, 0, 11392, 32768, 128, 0, 32768, 128, 0, 128, 11008, 5504, 4864, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 4052, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 3, 0, 0, 0, 0, 0, 0, 41, 4096, 0, 0, 4096, 4096, 0, 4095, 13, 4096, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 0, 1, 6, 0, 4090, 4031, 0, 0, 4095, 4095, 0, 4092, 0, 4094, 0, 1, 1, 0, 36, 4095, 0, 0, 1, 1, 0, 0, 0, 1, 4094, 4095, 4082, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 2, 0, 4, 4094, 2, 0, 3, 1, 1, 0, 0, 1, 4095, 1, 0, 0, 4057, 0, 0, 72, 0, 1, 5, 1, 1, 1, 0, 4095, 1, 1, 4054, 4091, 1, 4095, 0, 4088, 4092, 28, 4095, 0, 0, 39, 1, 1, 0, 4094, 0, 0, 0, 0, 1, 0, 1, 0, 4, 0, 4095, 4095, 4059, 1, 4095, 0, 4093, 0, 4095, 4095, 0, 1, 0, 1, 4095, 17, 0, 4095, 0, 1, 0, 0, 2, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4067, 0, 0, 3989, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 4096, 0, 0, 4096, 4096, 0, 4096, 52, 4096, 4096, 0, 4096, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4088, 4096, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4072, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4096, 0, 0, 4096, 4096, 0, 4096, 12, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4059, 0, 0, 3957, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4065, 0, 0, 3, 0, 0, 0, 0, 0, 0, 139, 4096, 0, 0, 4096, 4096, 0, 4096, 57, 4096, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 4053, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4093, 0, 0, 4096, 4096, 0, 4093, 26, 4096, 4096, 0, 4096, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4092, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4087, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 8, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 4096, 0, 4096, 4, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4077, 0, 0, 4078, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 4, 0, 0, 0, 0, 0, 0, 22, 4092, 0, 0, 4096, 4096, 0, 4096, 16, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4092, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 4052, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 4006, 50, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 4089, 19, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 41, 4096, 0, 0, 4096, 4046, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 4096, 0, 4064, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16402, 16381, 16385, 16391, 16377, 16366, 16390, 16380] avg=16384.0000 max=16402 max_rank=0 min=16366 min_rank=5 imbalance_ratio=1.001099 max_load_reduction=114670 max_load_reduction_pct=0.874863 moved_to_echo=65513 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 0, 1, 6, 0, 4090, 4031, 0, 0, 4095, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4094, 0, 1, 1, 0, 36, 4095, 0, 0, 1, 1, 0, 0, 4006, 50, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 4095, 4082, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 2, 0, 3, 1, 1, 0, 0, 1, 4095, 1, 0, 0, 4057, 0, 4088, 18, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 1, 5, 1, 1, 1, 0, 4095, 1, 1, 4054, 4041, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4092, 28, 4063, 0, 0, 39, 1, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4, 0, 0, 0, 4059, 1, 7, 0, 4093, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 17, 0, 4095, 0, 1, 0, 0, 2, 0, 1, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16408, 16375, 16370, 16369, 16406, 16356, 16398] avg=16384.0000 max=16408 max_rank=1 min=16356 min_rank=6 imbalance_ratio=1.001465 max_load_reduction=114664 max_load_reduction_pct=0.874817 moved_to_echo=65529 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4077, 0, 0, 4078, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4006, 50, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 4089, 19, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 22, 4092, 0, 0, 4096, 4046, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 4096, 0, 4064, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 4096, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16418, 16379, 16385, 16391, 16356, 16372, 16387, 16384] avg=16384.0000 max=16418 max_rank=0 min=16356 min_rank=4 imbalance_ratio=1.002075 max_load_reduction=114654 max_load_reduction_pct=0.874741 moved_to_echo=65499 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4067, 0, 0, 3989, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 4013, 53, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4067, 0, 4093, 20, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 4096, 0, 0, 4096, 4043, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 4096, 4096, 0, 4059, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4096, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16311, 16398, 16389, 16378, 16421, 16417, 16379, 16379] avg=16384.0000 max=16421 max_rank=4 min=16311 min_rank=0 imbalance_ratio=1.002258 max_load_reduction=114651 max_load_reduction_pct=0.874718 moved_to_echo=65596 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4087, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4006, 50, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 4089, 19, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 4096, 4046, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 4063, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16445, 16375, 16384, 16397, 16345, 16355, 16391, 16380] avg=16384.0000 max=16445 max_rank=0 min=16345 min_rank=4 imbalance_ratio=1.003723 max_load_reduction=114627 max_load_reduction_pct=0.874535 moved_to_echo=65471 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4059, 0, 0, 3957, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 4006, 50, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4065, 0, 4089, 19, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 139, 4096, 0, 0, 4096, 4046, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4096, 4096, 0, 4064, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16279, 16377, 16384, 16370, 16476, 16410, 16391, 16385] avg=16384.0000 max=16476 max_rank=4 min=16279 min_rank=0 imbalance_ratio=1.005615 max_load_reduction=114596 max_load_reduction_pct=0.874298 moved_to_echo=65662 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4072, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4006, 50, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4078, 0, 4089, 19, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4096, 0, 0, 4096, 4046, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 4096, 0, 4064, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16430, 16375, 16385, 16383, 16362, 16364, 16391, 16382] avg=16384.0000 max=16430 max_rank=0 min=16362 min_rank=4 imbalance_ratio=1.002808 max_load_reduction=114642 max_load_reduction_pct=0.874649 moved_to_echo=65499 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4081, 0, 0, 4053, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4006, 50, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4085, 0, 4089, 19, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4093, 0, 0, 4096, 4046, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4096, 4096, 0, 4064, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 4096, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 16379, 16385, 16392, 16366, 16382, 16387, 16384] avg=16384.0000 max=16397 max_rank=0 min=16366 min_rank=4 imbalance_ratio=1.000793 max_load_reduction=114675 max_load_reduction_pct=0.874901 moved_to_echo=65519 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 32768, 128, 128, 0, 32768, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 0, 32768, 0, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 0, 0, 32128, 512, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 32768, 32768, 32768, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 0, 32768, 0, 128, 0, 0, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[256, 32768, 32768, 128, 32512, 0, 128, 128, 128, 128, 0, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 128, 128, 0, 32768, 32640, 0, 0, 32384, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 128, 0, 128, 128, 32768, 128, 0, 0, 32640, 0, 32768, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 128, 384, 32768, 128, 128, 32768, 32384, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 49, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4086, 4096, 0, 4093, 0, 4021, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 13, 4096, 4091, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4053, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 30, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 4072, 4096, 0, 4096, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 36, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4065, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 33, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 158, 0, 4095, 81, 1, 0, 0, 4095, 4093, 1, 0, 0, 4095, 0, 0, 1, 3, 9, 1, 4095, 4095, 0, 4092, 6, 4095, 1, 1, 2, 1, 0, 0, 0, 1, 0, 10, 0, 1, 1, 1, 1, 0, 1, 0, 4, 4094, 4085, 3, 4095, 4094, 4095, 0, 0, 0, 3851, 0, 0, 3, 1, 3869, 1, 0, 0, 4087, 2, 1, 0, 0, 1, 1, 4086, 4093, 4077, 0, 4047, 0, 32, 1, 0, 1, 4094, 62, 1, 0, 0, 4093, 1, 1, 4095, 4095, 0, 1, 4095, 0, 0, 1, 4094, 1, 1, 1, 0, 12, 1, 4095, 4094, 0, 0, 4095, 0, 0, 1, 204, 1, 1, 0, 4095, 4095, 4078] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 4096, 0, 4072, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 11, 4096, 4087, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4081, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 26, 0, 0, 0, 4096, 3, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 4096, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 4096, 4090, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 19, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4080, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4094, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 4096, 1, 0, 0, 0, 4096, 4096, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 4096, 0, 4086, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 4096, 4092, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4080, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 12, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 4096, 4096, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 4072, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4069, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 36, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4065, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 27, 22, 18, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4078, 4096, 0, 4096, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16347, 16369, 16373, 16414, 16425, 16399, 16383, 16362] avg=16384.0000 max=16425 max_rank=4 min=16347 min_rank=0 imbalance_ratio=1.002502 max_load_reduction=114647 max_load_reduction_pct=0.874687 moved_to_echo=65569 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 49, 0, 4096, 0, 0, 4096, 4096, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4021, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4070, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 13, 0, 4091, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4053, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 26, 23, 19, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4077, 4096, 0, 4096, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 16376, 16360, 16385, 16423, 16395, 16377, 16359] avg=16384.0000 max=16423 max_rank=4 min=16359 min_rank=7 imbalance_ratio=1.002380 max_load_reduction=114649 max_load_reduction_pct=0.874702 moved_to_echo=65554 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 4096, 4096, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4069, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 27, 22, 18, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4078, 4096, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16348, 16387, 16392, 16378, 16456, 16365, 16383, 16363] avg=16384.0000 max=16456 max_rank=4 min=16348 min_rank=0 imbalance_ratio=1.004395 max_load_reduction=114616 max_load_reduction_pct=0.874451 moved_to_echo=65567 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 158, 0, 4095, 81, 1, 4095, 4095, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4093, 1, 0, 0, 2, 0, 0, 1, 3, 9, 1, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 6, 4095, 1, 1, 2, 1, 0, 0, 0, 1, 0, 10, 0, 1, 4094, 4067, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 4, 4094, 4085, 3, 0, 4094, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3851, 0, 0, 3, 1, 3869, 1, 0, 0, 4087, 2, 1, 0, 0, 1, 1, 4093, 26, 21, 18, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4075, 4077, 0, 4047, 0, 32, 1, 0, 1, 0, 62, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4094, 0, 1, 4095, 0, 0, 1, 4094, 1, 1, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4073, 0, 0, 4095, 0, 0, 1, 204, 1, 1, 0, 0, 4095, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16584, 16395, 16410, 16374, 15981, 16382, 16397, 16549] avg=16384.0000 max=16584 max_rank=0 min=15981 min_rank=4 imbalance_ratio=1.012207 max_load_reduction=114488 max_load_reduction_pct=0.873474 moved_to_echo=65309 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 9, 2, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4072, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4069, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 11, 0, 4087, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4081, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 27, 22, 18, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4078, 4096, 0, 4096, 0, 26, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16345, 16386, 16382, 16380, 16441, 16395, 16383, 16360] avg=16384.0000 max=16441 max_rank=4 min=16345 min_rank=0 imbalance_ratio=1.003479 max_load_reduction=114631 max_load_reduction_pct=0.874565 moved_to_echo=65579 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 4096, 1, 0, 4096, 4096, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4069, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 0, 4092, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4080, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 27, 22, 18, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4078, 4096, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16357, 16384, 16392, 16380, 16440, 16378, 16383, 16358] avg=16384.0000 max=16440 max_rank=4 min=16357 min_rank=0 imbalance_ratio=1.003418 max_load_reduction=114632 max_load_reduction_pct=0.874573 moved_to_echo=65559 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 4096, 4096, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4069, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 0, 4090, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 27, 22, 18, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4078, 4096, 0, 4096, 0, 19, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16349, 16385, 16377, 16378, 16455, 16386, 16383, 16359] avg=16384.0000 max=16455 max_rank=4 min=16349 min_rank=0 imbalance_ratio=1.004333 max_load_reduction=114617 max_load_reduction_pct=0.874458 moved_to_echo=65583 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4080, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4070, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 5, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 26, 22, 18, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4078, 4096, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16345, 16390, 16386, 16383, 16451, 16372, 16383, 16362] avg=16384.0000 max=16451 max_rank=4 min=16345 min_rank=0 imbalance_ratio=1.004089 max_load_reduction=114621 max_load_reduction_pct=0.874489 moved_to_echo=65568 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 128, 32640, 128, 128, 128, 128, 0, 0, 0, 128, 0, 128, 0, 128, 32768, 32640, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 32640, 0, 0, 32768, 0, 0, 128, 256, 128, 128, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32640, 0, 0, 128, 128, 32512, 128, 0, 0, 32768, 128, 128, 0, 0, 128, 128, 32768, 256, 256, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 32768, 32768, 128, 0, 128, 128, 0, 0, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 0, 128, 0, 0, 0, 128, 256, 0, 32768, 128, 128, 32768, 32768, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 32640, 32768, 0, 32768, 0, 256, 128, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 32768, 0, 0, 128, 32768, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 32768, 32768, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[953, 2565, 1818, 0, 276, 9, 1836, 501, 1065, 2755, 599, 261, 1199, 2071, 0, 1243, 857, 2261, 662, 1881, 51, 536, 170, 1929, 120, 49, 3439, 2044, 2028, 66, 1828, 1178, 541, 2, 1176, 405, 175, 1005, 1564, 178, 276, 2, 2440, 2905, 1, 2160, 1005, 1078, 1062, 1213, 836, 347, 16, 2, 25, 2405, 1, 279, 11, 656, 1595, 3485, 1495, 172, 762, 8, 0, 1407, 2461, 967, 914, 1250, 1976, 1454, 2397, 215, 1900, 13, 949, 1801, 933, 290, 39, 17, 1643, 538, 512, 17, 2298, 70, 1284, 771, 270, 2194, 625, 5, 1659, 553, 6, 1907, 22, 602, 1599, 711, 428, 1315, 246, 1604, 329, 1905, 2226, 1805, 1474, 2140, 586, 1158, 1910, 821, 2359, 1599, 1334, 2074, 1470, 884, 495, 9, 1097, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1861, 2174, 1863, 0, 274, 0, 2644, 385, 786, 2617, 462, 541, 1481, 2985, 0, 1190, 966, 1477, 1286, 1778, 81, 648, 167, 1750, 128, 13, 3462, 2335, 1495, 49, 1969, 991, 680, 3, 1083, 389, 206, 1090, 1704, 263, 157, 47, 1979, 2847, 1, 2927, 717, 857, 1028, 614, 656, 543, 1, 9, 14, 1721, 0, 276, 72, 270, 1648, 2616, 1468, 102, 1180, 14, 0, 1662, 2466, 1135, 558, 801, 1658, 1769, 2213, 103, 1681, 11, 846, 2356, 599, 497, 20, 22, 1867, 393, 899, 38, 2188, 183, 1063, 721, 253, 1700, 559, 0, 1328, 885, 9, 2311, 4, 396, 1596, 974, 255, 1244, 69, 1271, 489, 2248, 2474, 1520, 1926, 2182, 249, 1358, 2193, 951, 2003, 1681, 1386, 2526, 1532, 1355, 616, 3, 724, 13] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1243, 1740, 2019, 6, 427, 4, 2059, 146, 969, 2510, 559, 416, 2193, 2989, 0, 1392, 689, 1690, 1321, 1709, 49, 653, 196, 2274, 66, 11, 3422, 2307, 1465, 55, 2150, 1186, 738, 2, 959, 211, 169, 1334, 1033, 379, 228, 0, 1714, 2739, 2, 2430, 801, 677, 1232, 723, 318, 627, 5, 35, 7, 2253, 1, 176, 135, 256, 1777, 3054, 856, 173, 1170, 15, 0, 2222, 2010, 1295, 622, 1152, 1384, 2141, 2174, 74, 1393, 1, 843, 1420, 512, 530, 8, 19, 1499, 584, 892, 34, 2377, 27, 703, 1063, 317, 2414, 357, 0, 1475, 1142, 4, 1582, 2, 838, 1695, 971, 176, 1548, 68, 1888, 287, 2480, 2535, 2007, 1992, 1854, 152, 1630, 2342, 652, 2038, 1950, 1305, 2042, 1856, 705, 1194, 3, 420, 28] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1382, 2122, 1557, 0, 206, 0, 2637, 444, 680, 2925, 904, 758, 1814, 2835, 0, 1458, 924, 1800, 1132, 1595, 17, 643, 160, 1879, 77, 65, 3546, 1946, 1184, 40, 1597, 1558, 804, 0, 984, 539, 68, 1404, 1168, 136, 57, 4, 2011, 2843, 0, 2770, 981, 718, 1176, 856, 592, 643, 0, 28, 3, 2244, 0, 194, 18, 343, 1676, 3464, 1293, 126, 879, 8, 0, 1726, 2142, 869, 386, 1035, 1159, 1692, 2421, 76, 1793, 87, 989, 1991, 871, 226, 5, 12, 1417, 459, 900, 3, 2305, 35, 1233, 853, 244, 1988, 556, 0, 1691, 1120, 3, 1752, 3, 301, 1507, 780, 158, 1690, 54, 1285, 319, 1920, 2590, 1690, 2019, 2251, 419, 789, 2430, 1087, 2818, 1604, 949, 3056, 1600, 656, 466, 2, 674, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1722, 1697, 1501, 1, 300, 0, 2495, 281, 1149, 2706, 923, 380, 1251, 2724, 0, 1383, 703, 2022, 1723, 1989, 40, 1011, 155, 2247, 91, 34, 3429, 1867, 1582, 37, 2022, 983, 445, 0, 1449, 668, 122, 1655, 1503, 174, 194, 5, 2590, 3066, 0, 1941, 897, 777, 1229, 1224, 621, 390, 2, 20, 3, 2076, 0, 172, 59, 275, 1841, 2938, 1630, 122, 747, 14, 0, 1503, 1934, 1325, 398, 885, 1293, 1770, 2207, 130, 1640, 4, 802, 2526, 626, 297, 4, 0, 1531, 420, 1019, 27, 2091, 72, 2021, 723, 116, 1919, 210, 0, 1296, 1068, 5, 2162, 1, 227, 1562, 891, 183, 1533, 44, 1202, 453, 1875, 2367, 1446, 2194, 2282, 562, 1451, 2028, 839, 1980, 1587, 1499, 2469, 1118, 599, 691, 6, 655, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1729, 2132, 1723, 0, 213, 0, 2675, 432, 1041, 2798, 895, 177, 1576, 3029, 1, 1414, 874, 1397, 1447, 1797, 19, 832, 150, 1739, 89, 22, 3719, 2049, 1549, 27, 1822, 925, 681, 0, 1307, 480, 96, 1439, 1482, 162, 196, 6, 1961, 2848, 2, 2326, 797, 689, 1309, 752, 546, 529, 0, 5, 6, 1850, 1, 193, 61, 277, 1577, 2858, 1113, 146, 843, 7, 0, 1721, 2596, 849, 395, 880, 1642, 1492, 2327, 100, 1480, 3, 431, 2453, 520, 284, 2, 4, 1555, 554, 925, 8, 2837, 28, 1170, 449, 184, 1482, 351, 0, 1495, 899, 13, 2141, 1, 456, 2267, 708, 223, 1594, 44, 1306, 495, 2712, 2935, 1344, 2177, 2160, 439, 1508, 2838, 1141, 2233, 1649, 1138, 2699, 1108, 696, 511, 18, 562, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1642, 2320, 2362, 1, 236, 0, 2167, 575, 654, 2482, 534, 643, 1953, 2514, 0, 1931, 675, 1209, 2030, 2029, 14, 497, 167, 1734, 79, 43, 3435, 2229, 1547, 18, 1315, 1093, 728, 0, 1143, 430, 410, 1391, 1426, 150, 199, 0, 2173, 2981, 0, 1613, 782, 908, 1370, 865, 395, 861, 8, 34, 32, 2504, 0, 269, 14, 220, 2032, 2977, 1564, 118, 1225, 5, 0, 1370, 1840, 1436, 471, 1205, 1871, 1997, 2473, 336, 1367, 5, 605, 1884, 559, 595, 5, 33, 1930, 861, 604, 30, 2142, 148, 677, 903, 212, 1771, 519, 1, 759, 1232, 5, 2058, 17, 451, 1544, 1092, 213, 1417, 103, 2161, 571, 1864, 2025, 1300, 1586, 2490, 353, 1415, 2477, 954, 1857, 1703, 1012, 1985, 1166, 885, 721, 8, 819, 24] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1795, 2100, 1853, 0, 250, 0, 2039, 269, 815, 2674, 723, 271, 881, 3178, 0, 1613, 1186, 1767, 1439, 1431, 38, 577, 170, 1509, 73, 41, 3522, 1951, 843, 42, 1322, 1117, 997, 0, 1128, 371, 152, 1870, 1923, 256, 159, 2, 2151, 2815, 0, 2544, 882, 904, 868, 1072, 1093, 403, 0, 23, 15, 2526, 0, 241, 22, 629, 1619, 3073, 1722, 72, 996, 15, 0, 1243, 2568, 1460, 715, 1027, 1908, 2296, 2343, 466, 1443, 3, 679, 2461, 675, 648, 17, 25, 981, 395, 521, 39, 2466, 71, 1517, 935, 84, 2472, 425, 0, 1294, 665, 18, 1813, 2, 394, 1790, 699, 197, 1471, 73, 1500, 257, 1954, 2699, 1940, 2241, 1923, 236, 1849, 1989, 863, 1434, 1202, 1207, 2146, 1501, 520, 735, 3, 539, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[953, 2565, 1818, 0, 276, 9, 1836, 501, 1065, 2755, 599, 261, 1199, 164, 0, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 857, 2261, 662, 1881, 51, 536, 170, 1929, 120, 49, 1281, 2044, 2028, 66, 1828, 1178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 2, 1176, 405, 175, 1005, 1564, 178, 276, 2, 2440, 2905, 1, 2160, 1005, 1078, 785, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 1213, 836, 347, 16, 2, 25, 2405, 1, 279, 11, 656, 1595, 3485, 1495, 172, 1805, 1696, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 762, 8, 0, 1407, 2461, 967, 914, 1250, 1976, 1454, 701, 215, 1900, 13, 949, 1801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 933, 290, 39, 17, 1643, 538, 512, 17, 2298, 70, 1284, 771, 270, 2194, 625, 5, 1907, 2074, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1659, 553, 6, 1907, 22, 602, 1599, 711, 428, 1315, 246, 1604, 329, 1905, 1441, 1805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1474, 2140, 586, 1158, 891, 821, 2359, 1599, 1334, 0, 1470, 884, 495, 9, 1097, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15244, 16941, 16312, 17506, 16778, 15840, 16132, 16319] avg=16384.0000 max=17506 max_rank=3 min=15244 min_rank=0 imbalance_ratio=1.068481 max_load_reduction=113566 max_load_reduction_pct=0.866440 moved_to_echo=65069 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1243, 1740, 2019, 6, 427, 4, 2059, 146, 969, 2510, 559, 416, 2193, 237, 0, 1392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, 1690, 1321, 1709, 49, 653, 196, 2274, 66, 11, 1275, 2307, 1465, 55, 2150, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 738, 2, 959, 211, 169, 1334, 1033, 379, 228, 0, 1714, 2739, 2, 2430, 801, 677, 894, 753, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1232, 723, 318, 627, 5, 35, 7, 2253, 1, 176, 135, 256, 1777, 3054, 856, 173, 1796, 1538, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 15, 0, 2222, 2010, 1295, 622, 1152, 1384, 2141, 636, 74, 1393, 1, 843, 1420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 530, 8, 19, 1499, 584, 892, 34, 2377, 27, 703, 1063, 317, 2414, 357, 0, 2752, 2042, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1475, 1142, 4, 1582, 2, 838, 1695, 971, 176, 1548, 68, 1888, 287, 2480, 1641, 2007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1992, 1854, 152, 1630, 1093, 652, 2038, 1950, 1305, 0, 1856, 705, 1194, 3, 420, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15920, 17096, 15063, 15458, 16378, 16481, 17804, 16872] avg=16384.0000 max=17804 max_rank=6 min=15063 min_rank=2 imbalance_ratio=1.086670 max_load_reduction=113268 max_load_reduction_pct=0.864166 moved_to_echo=67535 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1861, 2174, 1863, 0, 274, 0, 2644, 385, 786, 2617, 462, 541, 1481, 237, 0, 1190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 966, 1477, 1286, 1778, 81, 648, 167, 1750, 128, 13, 1290, 2335, 1495, 49, 1969, 991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 680, 3, 1083, 389, 206, 1090, 1704, 263, 157, 47, 1979, 2847, 1, 2927, 717, 857, 873, 705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 614, 656, 543, 1, 9, 14, 1721, 0, 276, 72, 270, 1648, 2616, 1468, 102, 1817, 1566, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1180, 14, 0, 1662, 2466, 1135, 558, 801, 1658, 1769, 647, 103, 1681, 11, 846, 2356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 599, 497, 20, 22, 1867, 393, 899, 38, 2188, 183, 1063, 721, 253, 1700, 559, 0, 2748, 2526, 355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1328, 885, 9, 2311, 4, 396, 1596, 974, 255, 1244, 69, 1271, 489, 2248, 1601, 1520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1926, 2182, 249, 1358, 1023, 951, 2003, 1681, 1386, 0, 1532, 1355, 616, 3, 724, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16515, 16423, 16528, 14886, 16887, 16631, 16200, 17002] avg=16384.0000 max=17002 max_rank=7 min=14886 min_rank=3 imbalance_ratio=1.037720 max_load_reduction=114070 max_load_reduction_pct=0.870285 moved_to_echo=66720 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1382, 2122, 1557, 0, 206, 0, 2637, 444, 680, 2925, 904, 758, 1814, 221, 0, 1458, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 924, 1800, 1132, 1595, 17, 643, 160, 1879, 77, 65, 1313, 1946, 1184, 40, 1597, 1558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 0, 984, 539, 68, 1404, 1168, 136, 57, 4, 2011, 2843, 0, 2770, 981, 718, 917, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1176, 856, 592, 643, 0, 28, 3, 2244, 0, 194, 18, 343, 1676, 3464, 1293, 126, 1865, 1717, 518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 879, 8, 0, 1726, 2142, 869, 386, 1035, 1159, 1692, 704, 76, 1793, 87, 989, 1991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 871, 226, 5, 12, 1417, 459, 900, 3, 2305, 35, 1233, 853, 244, 1988, 556, 0, 2614, 3056, 368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1691, 1120, 3, 1752, 3, 301, 1507, 780, 158, 1690, 54, 1285, 319, 1920, 1673, 1690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2019, 2251, 419, 789, 1128, 1087, 2818, 1604, 949, 0, 1600, 656, 466, 2, 674, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17108, 15930, 16188, 16756, 15536, 17145, 15946, 16463] avg=16384.0000 max=17145 max_rank=5 min=15536 min_rank=4 imbalance_ratio=1.046448 max_load_reduction=113927 max_load_reduction_pct=0.869194 moved_to_echo=65090 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1722, 1697, 1501, 1, 300, 0, 2495, 281, 1149, 2706, 923, 380, 1251, 216, 0, 1383, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 703, 2022, 1723, 1989, 40, 1011, 155, 2247, 91, 34, 1277, 1867, 1582, 37, 2022, 983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 0, 1449, 668, 122, 1655, 1503, 174, 194, 5, 2590, 3066, 0, 1941, 897, 777, 835, 652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1229, 1224, 621, 390, 2, 20, 3, 2076, 0, 172, 59, 275, 1841, 2938, 1630, 122, 1800, 1562, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 747, 14, 0, 1503, 1934, 1325, 398, 885, 1293, 1770, 645, 130, 1640, 4, 802, 2526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 626, 297, 4, 0, 1531, 420, 1019, 27, 2091, 72, 2021, 723, 116, 1919, 210, 0, 2508, 2469, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1296, 1068, 5, 2162, 1, 227, 1562, 891, 183, 1533, 44, 1202, 453, 1875, 1532, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2194, 2282, 562, 1451, 946, 839, 1980, 1587, 1499, 0, 1118, 599, 691, 6, 655, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16005, 17783, 16973, 16394, 15616, 16405, 15480, 16416] avg=16384.0000 max=17783 max_rank=1 min=15480 min_rank=6 imbalance_ratio=1.085388 max_load_reduction=113289 max_load_reduction_pct=0.864326 moved_to_echo=63917 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1795, 2100, 1853, 0, 250, 0, 2039, 269, 815, 2674, 723, 271, 881, 252, 0, 1613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1186, 1767, 1439, 1431, 38, 577, 170, 1509, 73, 41, 1312, 1951, 843, 42, 1322, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 997, 0, 1128, 371, 152, 1870, 1923, 256, 159, 2, 2151, 2815, 0, 2544, 882, 904, 952, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 868, 1072, 1093, 403, 0, 23, 15, 2526, 0, 241, 22, 629, 1619, 3073, 1722, 72, 1849, 1658, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 996, 15, 0, 1243, 2568, 1460, 715, 1027, 1908, 2296, 685, 466, 1443, 3, 679, 2461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 675, 648, 17, 25, 981, 395, 521, 39, 2466, 71, 1517, 935, 84, 2472, 425, 0, 2926, 2146, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1294, 665, 18, 1813, 2, 394, 1790, 699, 197, 1471, 73, 1500, 257, 1954, 1747, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2241, 1923, 236, 1849, 928, 863, 1434, 1202, 1207, 0, 1501, 520, 735, 3, 539, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15535, 14818, 17745, 17307, 17965, 16704, 15814, 15184] avg=16384.0000 max=17965 max_rank=4 min=14818 min_rank=1 imbalance_ratio=1.096497 max_load_reduction=113107 max_load_reduction_pct=0.862938 moved_to_echo=65667 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1729, 2132, 1723, 0, 213, 0, 2675, 432, 1041, 2798, 895, 177, 1576, 240, 1, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 874, 1397, 1447, 1797, 19, 832, 150, 1739, 89, 22, 1385, 2049, 1549, 27, 1822, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 681, 0, 1307, 480, 96, 1439, 1482, 162, 196, 6, 1961, 2848, 2, 2326, 797, 689, 1035, 912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1309, 752, 546, 529, 0, 5, 6, 1850, 1, 193, 61, 277, 1577, 2858, 1113, 146, 1952, 1647, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 843, 7, 0, 1721, 2596, 849, 395, 880, 1642, 1492, 680, 100, 1480, 3, 431, 2453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 520, 284, 2, 4, 1555, 554, 925, 8, 2837, 28, 1170, 449, 184, 1482, 351, 0, 2789, 2699, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1495, 899, 13, 2141, 1, 456, 2267, 708, 223, 1594, 44, 1306, 495, 2712, 1900, 1344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2177, 2160, 439, 1508, 1324, 1141, 2233, 1649, 1138, 0, 1108, 696, 511, 18, 562, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17046, 16123, 16419, 15424, 15572, 16223, 17598, 16667] avg=16384.0000 max=17598 max_rank=6 min=15424 min_rank=3 imbalance_ratio=1.074097 max_load_reduction=113474 max_load_reduction_pct=0.865738 moved_to_echo=66060 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1642, 2320, 2362, 1, 236, 0, 2167, 575, 654, 2482, 534, 643, 1953, 199, 0, 1931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 675, 1209, 2030, 2029, 14, 497, 167, 1734, 79, 43, 1279, 2229, 1547, 18, 1315, 1093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 728, 0, 1143, 430, 410, 1391, 1426, 150, 199, 0, 2173, 2981, 0, 1613, 782, 908, 714, 796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1370, 865, 395, 861, 8, 34, 32, 2504, 0, 269, 14, 220, 2032, 2977, 1564, 118, 1803, 1750, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1225, 5, 0, 1370, 1840, 1436, 471, 1205, 1871, 1997, 723, 336, 1367, 5, 605, 1884, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 595, 5, 33, 1930, 861, 604, 30, 2142, 148, 677, 903, 212, 1771, 519, 1, 2315, 1985, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 759, 1232, 5, 2058, 17, 451, 1544, 1092, 213, 1417, 103, 2161, 571, 1864, 1311, 1300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1586, 2490, 353, 1415, 1156, 954, 1857, 1703, 1012, 0, 1166, 885, 721, 8, 819, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17699, 15958, 15844, 17341, 16340, 15643, 16098, 16149] avg=16384.0000 max=17699 max_rank=0 min=15643 min_rank=5 imbalance_ratio=1.080261 max_load_reduction=113373 max_load_reduction_pct=0.864967 moved_to_echo=64230 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[15616, 17408, 3072, 11264, 8576, 7424, 16768, 13056, 9856, 0, 11392, 6400, 5504, 128, 5504, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[9344, 7424, 5120, 4352, 128, 256, 128, 17664, 128, 1920, 512, 2944, 13824, 24576, 11264, 1152, 14720, 13184, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[11008, 7680, 128, 15744, 128, 3712, 13568, 6912, 1920, 11904, 768, 12288, 3200, 17024, 12928, 13056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[5376, 3456, 128, 256, 12544, 4224, 6272, 256, 18816, 640, 9728, 6528, 1792, 16000, 3712, 128, 20608, 19072, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[12416, 16896, 14720, 128, 2304, 128, 18560, 3072, 7168, 21504, 5632, 3456, 12416, 1792, 128, 11648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[6912, 13696, 11136, 14336, 384, 5504, 1408, 15104, 768, 384, 10496, 16768, 11776, 384, 14080, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[5632, 128, 9344, 3584, 1408, 11264, 11904, 1792, 1536, 128, 17024, 23168, 128, 18816, 6912, 6656, 7040, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[7808, 128, 0, 12928, 18048, 9344, 4480, 8320, 12928, 14720, 5504, 1536, 12800, 128, 6144, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 4095, 0, 1469, 16, 0, 9, 3975, 10, 0, 0, 7, 4089, 0, 0, 4093, 0, 0, 4096, 5, 197, 1, 0, 4096, 1, 4083, 4093, 42, 0, 3, 4092, 1, 26, 4081, 5, 431, 0, 32, 0, 0, 38, 0, 847, 0, 0, 4086, 4090, 0, 0, 4094, 0, 89, 4096, 0, 0, 4015, 16, 0, 0, 4093, 0, 28, 4068, 1, 0, 3985, 106, 0, 4083, 0, 9, 4086, 0, 0, 0, 8, 0, 0, 3624, 0, 0, 0, 0, 0, 3732, 3, 0, 7, 4, 0, 4082, 0, 0, 0, 0, 0, 4095, 3777, 4, 0, 0, 0, 4096, 0, 4061, 0, 0, 1, 4095, 4093, 7, 0, 2032, 0, 108, 0, 0, 4085, 0, 0, 184, 0, 0, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 25, 4083, 0, 2970, 3, 0, 0, 3779, 5, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 4094, 48, 17, 0, 0, 4077, 0, 4087, 4086, 36, 0, 0, 4096, 0, 6, 4093, 0, 18, 0, 5, 0, 0, 0, 0, 774, 0, 1, 4092, 4087, 0, 0, 4096, 15, 1, 4096, 0, 0, 4096, 8, 0, 0, 4080, 0, 2, 4052, 0, 0, 4016, 19, 0, 4096, 0, 24, 4096, 0, 0, 0, 0, 0, 0, 4043, 0, 5, 0, 0, 0, 3987, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 4091, 66, 15, 0, 3, 4096, 0, 4085, 6, 0, 0, 4096, 4096, 1, 0, 657, 0, 55, 1, 0, 4081, 15, 0, 22, 0, 0, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4044, 0, 2527, 1, 0, 0, 3999, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4091, 6, 372, 0, 0, 4093, 0, 4095, 4096, 48, 0, 0, 4096, 1, 7, 4096, 3, 139, 0, 11, 0, 0, 0, 0, 576, 0, 0, 4068, 4093, 0, 0, 4096, 2, 1, 4096, 0, 0, 4096, 15, 0, 0, 4096, 0, 0, 4061, 0, 0, 3892, 0, 0, 4096, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 3638, 0, 2, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 4093, 5, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 862, 0, 39, 3, 0, 4090, 3, 0, 399, 0, 0, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 1, 4095, 0, 3266, 7, 0, 0, 3780, 1, 0, 1, 0, 4095, 0, 0, 4095, 0, 0, 4091, 3, 52, 0, 0, 4094, 1, 4096, 4095, 115, 0, 0, 4096, 0, 1, 4095, 3, 46, 0, 27, 0, 0, 19, 0, 570, 0, 1, 4064, 4093, 0, 0, 4094, 3, 16, 4095, 0, 0, 4077, 3, 1, 0, 4096, 0, 1, 4021, 0, 0, 3495, 0, 0, 4096, 0, 353, 4095, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 1, 1, 1, 3831, 0, 0, 0, 0, 1, 4096, 1, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 690, 2, 40, 0, 0, 4078, 7, 1, 336, 0, 0, 0, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 4, 0, 2, 4089, 0, 3782, 0, 0, 0, 3965, 0, 0, 0, 0, 4090, 0, 4, 4093, 0, 0, 4086, 14, 4, 0, 2, 4091, 0, 4088, 4091, 17, 0, 1, 4091, 4, 0, 4096, 0, 39, 0, 0, 0, 0, 11, 0, 461, 1, 2, 4094, 4088, 1, 0, 4092, 0, 0, 4094, 0, 0, 4092, 10, 0, 0, 4094, 0, 7, 3899, 4, 0, 3622, 1, 0, 4095, 0, 289, 4092, 0, 0, 4, 0, 1, 4, 3914, 5, 9, 0, 0, 0, 3702, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 4092, 4058, 3, 5, 0, 3, 4096, 0, 4090, 3, 1, 4, 4095, 4096, 0, 0, 505, 4, 301, 1, 0, 4084, 3, 0, 127, 0, 1, 0, 4064] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 21, 0, 0, 3557, 15, 1775, 1, 1, 5, 3882, 0, 0, 0, 0, 4089, 1, 26, 4095, 0, 0, 4096, 9, 3, 0, 0, 3947, 0, 4096, 4081, 244, 0, 0, 4089, 15, 142, 3901, 0, 72, 0, 0, 2, 0, 0, 0, 304, 0, 60, 4096, 4069, 34, 0, 4089, 0, 59, 4080, 0, 4, 4093, 19, 0, 4, 4095, 0, 0, 4076, 0, 12, 3729, 136, 16, 4082, 4, 191, 4056, 0, 1, 12, 0, 4, 0, 4096, 0, 0, 0, 1, 0, 4028, 0, 0, 14, 1, 1, 4093, 12, 0, 25, 0, 3, 4071, 3919, 202, 0, 1, 27, 4068, 4, 4091, 0, 4, 0, 4015, 4055, 22, 1, 1909, 0, 0, 0, 2, 4096, 0, 0, 712, 0, 14, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4040, 0, 3397, 9, 0, 0, 3796, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4088, 2, 7, 3, 0, 4066, 0, 4096, 4096, 33, 0, 0, 4095, 1, 114, 4096, 1, 135, 0, 11, 0, 0, 3, 0, 1238, 0, 9, 4071, 4094, 0, 0, 4096, 0, 1, 4096, 0, 0, 4095, 100, 0, 0, 4096, 0, 0, 3975, 0, 0, 3563, 2, 0, 4096, 0, 15, 4096, 0, 0, 0, 0, 0, 0, 3714, 0, 0, 0, 0, 0, 3129, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 4096, 0, 4096, 0, 0, 0, 4095, 4096, 0, 0, 728, 0, 39, 2, 0, 4086, 0, 1, 679, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 4022, 0, 1973, 1, 0, 0, 3744, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 1, 20, 1, 0, 4077, 0, 4096, 4096, 16, 0, 8, 4095, 0, 47, 4079, 0, 51, 0, 1, 0, 0, 2, 0, 1087, 0, 0, 4095, 4096, 1, 0, 4096, 0, 1, 4096, 1, 0, 4096, 3, 0, 0, 4094, 0, 1, 4086, 0, 0, 3984, 21, 0, 4096, 0, 3, 4092, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 4096, 4096, 20, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 4095, 0, 0, 1046, 0, 0, 0, 0, 4096, 0, 0, 502, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 21, 0, 0, 3557, 15, 1775, 1, 1, 5, 3882, 0, 0, 0, 0, 4089, 1748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 26, 0, 0, 0, 3968, 9, 3, 0, 0, 3947, 0, 4096, 4081, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 15, 142, 3901, 0, 72, 0, 0, 2, 0, 0, 0, 304, 0, 60, 4095, 3197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4069, 34, 0, 0, 0, 59, 0, 0, 4, 4093, 19, 0, 4, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 12, 3729, 136, 16, 4082, 4, 191, 4056, 0, 1, 12, 0, 4, 128, 83, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4028, 0, 0, 14, 1, 1, 4093, 12, 0, 3684, 892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 3, 4071, 3919, 202, 0, 1, 27, 39, 4, 4091, 0, 4, 0, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 22, 1, 1909, 0, 0, 0, 2, 4096, 0, 0, 712, 0, 14, 0, 4095, 2198, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15094, 16375, 15877, 16459, 16544, 16822, 16401, 17500] avg=16384.0000 max=17500 max_rank=7 min=15094 min_rank=0 imbalance_ratio=1.068115 max_load_reduction=113572 max_load_reduction_pct=0.866486 moved_to_echo=67267 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3, 4095, 0, 1469, 16, 0, 9, 3975, 10, 0, 0, 7, 4089, 1760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 5, 197, 1, 0, 4096, 1, 4083, 4093, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4092, 1, 26, 4081, 5, 431, 0, 32, 0, 0, 38, 0, 847, 0, 0, 4093, 3202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4090, 0, 0, 0, 0, 89, 0, 0, 0, 4015, 16, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4068, 1, 0, 3985, 106, 0, 4083, 0, 9, 4086, 0, 0, 0, 8, 0, 128, 84, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3624, 0, 0, 0, 0, 0, 3732, 3, 0, 7, 4, 0, 4082, 0, 0, 3699, 892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3777, 4, 0, 0, 0, 39, 0, 4061, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 7, 0, 2032, 0, 108, 0, 0, 4085, 0, 0, 184, 0, 0, 0, 4093, 2213, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15433, 16486, 16851, 16375, 16600, 16043, 16072, 17212] avg=16384.0000 max=17212 max_rank=7 min=15433 min_rank=0 imbalance_ratio=1.050537 max_load_reduction=113860 max_load_reduction_pct=0.868683 moved_to_echo=65927 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 1, 4095, 0, 3266, 7, 0, 0, 3780, 1, 0, 1, 0, 4095, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 3, 52, 0, 0, 4094, 1, 4096, 4095, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4095, 3, 46, 0, 27, 0, 0, 19, 0, 570, 0, 1, 4095, 3202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 4093, 0, 0, 0, 3, 16, 0, 0, 0, 4077, 3, 1, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4021, 0, 0, 3495, 0, 0, 4096, 0, 353, 4095, 0, 0, 0, 0, 0, 133, 85, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 1, 1, 1, 3831, 0, 0, 0, 0, 1, 4096, 1, 2, 3698, 892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 32, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 690, 2, 40, 0, 0, 4078, 7, 1, 336, 0, 0, 0, 4092, 2217, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17008, 16414, 16155, 16335, 16297, 16492, 16416, 15955] avg=16384.0000 max=17008 max_rank=0 min=15955 min_rank=7 imbalance_ratio=1.038086 max_load_reduction=114064 max_load_reduction_pct=0.870239 moved_to_echo=65160 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 25, 4083, 0, 2970, 3, 0, 0, 3779, 5, 0, 0, 0, 4092, 1760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 48, 17, 0, 0, 4077, 0, 4087, 4086, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 4093, 0, 18, 0, 5, 0, 0, 0, 0, 774, 0, 1, 4096, 3203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4087, 0, 0, 0, 15, 1, 0, 0, 0, 4096, 8, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4052, 0, 0, 4016, 19, 0, 4096, 0, 24, 4096, 0, 0, 0, 0, 0, 128, 84, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 5, 0, 0, 0, 3987, 0, 0, 0, 0, 0, 4094, 0, 0, 3699, 893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 66, 15, 0, 3, 39, 0, 4085, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 657, 0, 55, 1, 0, 4081, 15, 0, 22, 0, 0, 0, 4094, 2213, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16717, 16317, 16292, 16365, 16531, 16721, 16497, 15632] avg=16384.0000 max=16721 max_rank=5 min=15632 min_rank=7 imbalance_ratio=1.020569 max_load_reduction=114351 max_load_reduction_pct=0.872429 moved_to_echo=65381 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 2, 4089, 0, 3782, 0, 0, 0, 3965, 0, 0, 0, 0, 4090, 1760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3958, 14, 4, 0, 2, 4091, 0, 4088, 4091, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 4, 0, 4096, 0, 39, 0, 0, 0, 0, 11, 0, 461, 1, 2, 4093, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 4092, 10, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3899, 4, 0, 3622, 1, 0, 4095, 0, 289, 4092, 0, 0, 4, 0, 1, 128, 84, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3914, 5, 9, 0, 0, 0, 3702, 0, 0, 0, 0, 0, 4094, 0, 0, 3697, 892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4092, 4058, 3, 5, 0, 3, 39, 0, 4090, 3, 1, 4, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 505, 4, 301, 1, 0, 4084, 3, 0, 127, 0, 1, 0, 4064, 2213, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17692, 16269, 15999, 16365, 16240, 16317, 16394, 15796] avg=16384.0000 max=17692 max_rank=0 min=15796 min_rank=7 imbalance_ratio=1.079834 max_load_reduction=113380 max_load_reduction_pct=0.865021 moved_to_echo=64747 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4044, 0, 2527, 1, 0, 0, 3999, 0, 0, 0, 0, 4096, 1760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 6, 372, 0, 0, 4093, 0, 4095, 4096, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 7, 4096, 3, 139, 0, 11, 0, 0, 0, 0, 576, 0, 0, 4096, 3202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 4093, 0, 0, 0, 2, 1, 0, 0, 0, 4096, 15, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 3892, 0, 0, 4096, 0, 7, 4096, 0, 0, 0, 0, 0, 128, 84, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3638, 0, 2, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 4096, 0, 1, 3698, 894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 5, 0, 0, 0, 39, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 862, 0, 39, 3, 0, 4090, 3, 0, 399, 0, 0, 0, 4093, 2213, 398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16427, 16673, 16227, 16357, 16378, 16389, 16425, 16196] avg=16384.0000 max=16673 max_rank=1 min=16196 min_rank=7 imbalance_ratio=1.017639 max_load_reduction=114399 max_load_reduction_pct=0.872795 moved_to_echo=65388 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4040, 0, 3397, 9, 0, 0, 3796, 1, 0, 0, 0, 4096, 1760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 2, 7, 3, 0, 4066, 0, 4096, 4096, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 114, 4096, 1, 135, 0, 11, 0, 0, 3, 0, 1238, 0, 9, 4096, 3203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 4094, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 100, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 0, 3563, 2, 0, 4096, 0, 15, 4096, 0, 0, 0, 0, 0, 128, 84, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3714, 0, 0, 0, 0, 0, 3129, 0, 0, 0, 0, 0, 4096, 0, 0, 3699, 893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3, 39, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 728, 0, 39, 2, 0, 4086, 0, 1, 679, 0, 0, 0, 4095, 2213, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17099, 16263, 17002, 16443, 15973, 15531, 16425, 16336] avg=16384.0000 max=17099 max_rank=0 min=15531 min_rank=5 imbalance_ratio=1.043640 max_load_reduction=113973 max_load_reduction_pct=0.869545 moved_to_echo=64265 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 4022, 0, 1973, 1, 0, 0, 3744, 0, 0, 0, 0, 4096, 1760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 1, 20, 1, 0, 4077, 0, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4095, 0, 47, 4079, 0, 51, 0, 1, 0, 0, 2, 0, 1087, 0, 0, 4096, 3203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 1, 0, 0, 0, 1, 0, 1, 0, 4096, 3, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 0, 0, 3984, 21, 0, 4096, 0, 3, 4092, 0, 0, 0, 0, 0, 128, 84, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 4096, 0, 4, 3699, 893, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 20, 0, 0, 0, 39, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1046, 0, 0, 0, 0, 4096, 0, 0, 502, 0, 0, 0, 4096, 2213, 397, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15602, 16275, 16669, 16373, 16509, 16757, 16442, 16445] avg=16384.0000 max=16757 max_rank=5 min=15602 min_rank=0 imbalance_ratio=1.022766 max_load_reduction=114315 max_load_reduction_pct=0.872154 moved_to_echo=66153 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 31104, 128, 128, 128, 128, 128, 30592, 128, 0, 128, 128, 128, 32768, 128, 128, 29696, 7168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 32768, 128, 0, 0, 128, 256, 0, 128, 128, 32768, 256, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 32128, 128, 21248, 128, 128, 128, 30976, 128, 0, 128, 128, 32768, 14080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 8448, 128, 640, 128, 128, 32768, 128, 128, 3072, 0, 128, 0, 32768, 17792, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 32768, 32256, 384, 128, 128, 128, 384, 128, 32768, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 384, 32640, 128, 1024, 0, 128, 128, 0, 128, 0, 5888, 128, 128, 32768, 25728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 0, 0, 31744, 128, 768, 128, 128, 32640, 128, 32768, 32768, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32256, 128, 128, 30336, 384, 128, 32768, 128, 896, 32768, 0, 128, 128, 128, 128, 1152, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2021, 0, 105, 0, 0, 3646, 0, 4088, 177, 0, 0, 0, 0, 4089, 1, 0, 0, 2869, 1, 0, 4096, 0, 4095, 4091, 0, 0, 0, 4089, 3816, 0, 10, 7, 4096, 20, 0, 4093, 562, 0, 0, 0, 4096, 0, 3, 0, 0, 4096, 4032, 3373, 0, 0, 14, 0, 4096, 4088, 0, 0, 4079, 4072, 0, 0, 10, 1658, 0, 0, 4090, 19, 0, 0, 0, 0, 2, 4096, 0, 0, 4095, 0, 11, 3995, 4096, 1, 0, 126, 3855, 0, 476, 0, 1969, 3, 0, 4055, 0, 0, 0, 4047, 2, 13, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 11, 0, 0, 108, 4082, 0, 0, 7, 0, 0, 0, 4087, 0, 0, 0, 4094, 8, 0, 19, 2, 20] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 2010, 0, 329, 0, 0, 3775, 0, 4096, 22, 0, 0, 0, 0, 4096, 0, 0, 0, 3467, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 3977, 0, 0, 0, 4096, 11, 1, 4096, 108, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 2829, 0, 0, 7, 0, 4095, 4096, 0, 0, 4077, 4075, 0, 0, 0, 1812, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 6, 4033, 4082, 0, 0, 7, 4018, 0, 32, 0, 2192, 0, 0, 4086, 0, 0, 0, 4068, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 116, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4095, 0, 0, 2, 0, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[23, 11, 3046, 0, 78, 1, 1, 3469, 1, 4053, 213, 7, 1, 0, 0, 4075, 0, 3, 2, 3292, 0, 0, 4073, 3, 4091, 4096, 0, 0, 17, 4051, 3723, 1, 9, 0, 4095, 42, 0, 4095, 223, 0, 1, 0, 4091, 2, 19, 2, 0, 4069, 4096, 2170, 3, 0, 0, 0, 4095, 4084, 0, 1, 4008, 4033, 1, 1, 0, 1736, 5, 0, 4070, 29, 0, 0, 0, 0, 11, 4093, 0, 0, 4088, 0, 76, 3842, 3913, 2, 0, 73, 4036, 2, 179, 2, 2639, 19, 0, 4089, 1, 0, 1, 4085, 0, 58, 3, 0, 0, 0, 4089, 0, 4, 0, 4, 0, 6, 4, 140, 4067, 3, 1, 24, 0, 0, 4, 4092, 1, 0, 0, 4080, 25, 0, 5, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2581, 0, 925, 0, 0, 3837, 0, 4095, 3, 0, 0, 0, 0, 4095, 1, 0, 0, 3378, 0, 0, 4095, 0, 4096, 4095, 0, 0, 0, 4096, 3909, 0, 3, 1, 4096, 0, 0, 4095, 6, 0, 0, 0, 4096, 0, 1, 0, 0, 4094, 4094, 3597, 0, 0, 2, 1, 4095, 4096, 0, 0, 4091, 4095, 0, 0, 0, 1776, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 1, 4035, 4094, 0, 0, 3, 4050, 1, 1, 0, 515, 2, 0, 4079, 0, 0, 0, 4072, 0, 2, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 79, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 4095, 0, 0, 1, 0, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1104, 0, 1320, 0, 0, 3762, 0, 4096, 28, 0, 0, 0, 0, 4096, 0, 0, 0, 3608, 3, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4019, 0, 0, 0, 4096, 0, 0, 4096, 19, 0, 0, 0, 4096, 0, 14, 0, 0, 4096, 4093, 2668, 0, 0, 0, 0, 4096, 4079, 0, 0, 4089, 4063, 0, 0, 3, 2209, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 2, 4062, 4096, 0, 0, 78, 3889, 0, 108, 0, 1949, 0, 0, 4049, 0, 0, 0, 3934, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 76, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 6, 0, 4096, 1, 0, 1, 0, 16] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1674, 0, 1030, 0, 0, 3720, 0, 4095, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 3873, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4020, 0, 0, 0, 4096, 0, 2, 4096, 4, 0, 0, 1, 4096, 0, 0, 0, 0, 4095, 4094, 3843, 0, 0, 9, 0, 4095, 4096, 0, 0, 4094, 4096, 0, 0, 0, 1776, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 4003, 4096, 0, 0, 2, 3942, 0, 24, 0, 960, 1, 0, 4009, 0, 0, 0, 3957, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 3317, 0, 316, 0, 0, 2166, 0, 4096, 2, 0, 0, 2, 0, 4096, 0, 0, 0, 3977, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 4096, 3966, 0, 0, 0, 4096, 0, 0, 4096, 10, 0, 0, 1, 4096, 0, 2, 0, 0, 4094, 4094, 3234, 0, 0, 1, 0, 4096, 4096, 0, 0, 4095, 4096, 0, 0, 0, 1601, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4057, 4096, 0, 0, 0, 4092, 0, 23, 0, 1773, 1, 0, 4089, 0, 0, 0, 4068, 0, 1, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 164, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 1626, 4, 1749, 0, 1, 3678, 0, 4092, 13, 0, 0, 0, 2, 4091, 4, 0, 0, 3276, 0, 0, 4091, 0, 4095, 4094, 0, 0, 0, 4096, 3907, 0, 7, 4, 4096, 6, 0, 4096, 1, 0, 0, 1, 4095, 1, 10, 0, 0, 4079, 4087, 3591, 0, 0, 0, 0, 4092, 4092, 0, 0, 4087, 4087, 0, 0, 2, 2557, 0, 0, 4090, 0, 1, 0, 0, 0, 1, 4096, 1, 0, 4096, 6, 0, 3987, 4093, 0, 1, 9, 4044, 0, 12, 0, 317, 1, 0, 4090, 0, 0, 0, 3976, 4, 11, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 5, 0, 1, 27, 4091, 1, 0, 3, 0, 0, 1, 4095, 0, 0, 0, 4090, 7, 2, 2, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2021, 0, 105, 0, 0, 3646, 0, 4088, 177, 0, 0, 0, 0, 4089, 904, 761, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2869, 1, 0, 4096, 0, 0, 813, 0, 0, 0, 4089, 3816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 7, 3966, 20, 0, 4093, 562, 0, 0, 0, 4096, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3128, 3373, 0, 0, 14, 0, 0, 0, 0, 0, 4079, 4072, 0, 0, 10, 1658, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 19, 0, 0, 0, 0, 2, 4096, 0, 0, 4095, 0, 11, 3995, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2628, 1, 0, 126, 3855, 0, 476, 0, 1969, 3, 0, 4055, 0, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 13, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 11, 0, 0, 108, 4082, 4088, 3278, 707, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4087, 0, 0, 0, 4094, 8, 0, 19, 2, 20, 4095, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15845, 15685, 16853, 16334, 16384, 17160, 16400, 16411] avg=16384.0000 max=17160 max_rank=5 min=15685 min_rank=1 imbalance_ratio=1.047363 max_load_reduction=113912 max_load_reduction_pct=0.869080 moved_to_echo=66355 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[23, 11, 3046, 0, 78, 1, 1, 3469, 1, 4053, 213, 7, 1, 0, 0, 4075, 919, 727, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 3292, 0, 0, 4073, 3, 0, 814, 0, 0, 17, 4051, 3723, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3965, 42, 0, 4095, 223, 0, 1, 0, 4091, 2, 19, 2, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3177, 2170, 3, 0, 0, 0, 0, 0, 0, 1, 4008, 4033, 1, 1, 0, 1736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4070, 29, 0, 0, 0, 0, 11, 4093, 0, 0, 4088, 0, 76, 3842, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2511, 2, 0, 73, 4036, 2, 179, 2, 2639, 19, 0, 4089, 1, 0, 1, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 3, 0, 0, 0, 4089, 0, 4, 0, 4, 0, 6, 4, 140, 4067, 4084, 3282, 675, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 24, 0, 0, 4, 4092, 1, 0, 0, 4080, 25, 0, 5, 0, 0, 4091, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16679, 15979, 16518, 15130, 16290, 17639, 16433, 16404] avg=16384.0000 max=17639 max_rank=5 min=15130 min_rank=3 imbalance_ratio=1.076599 max_load_reduction=113433 max_load_reduction_pct=0.865425 moved_to_echo=66766 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2581, 0, 925, 0, 0, 3837, 0, 4095, 3, 0, 0, 0, 0, 4095, 922, 765, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3378, 0, 0, 4095, 0, 0, 807, 0, 0, 0, 4096, 3909, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3953, 0, 0, 4095, 6, 0, 0, 0, 4096, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3172, 3597, 0, 0, 2, 1, 0, 0, 0, 0, 4091, 4095, 0, 0, 0, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 1, 4035, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2618, 0, 0, 3, 4050, 1, 1, 0, 515, 2, 0, 4079, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 79, 4096, 4096, 3288, 711, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 4095, 0, 0, 1, 0, 7, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17283, 16286, 16249, 16734, 16409, 15341, 16392, 16378] avg=16384.0000 max=17283 max_rank=0 min=15341 min_rank=5 imbalance_ratio=1.054871 max_load_reduction=113789 max_load_reduction_pct=0.868141 moved_to_echo=64520 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3317, 0, 316, 0, 0, 2166, 0, 4096, 2, 0, 0, 2, 0, 4096, 918, 761, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3977, 0, 0, 4096, 0, 0, 814, 0, 0, 1, 4096, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 4096, 10, 0, 0, 1, 4096, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3176, 3234, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 1601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4057, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2628, 0, 0, 0, 4092, 0, 23, 0, 1773, 1, 0, 4089, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 164, 4096, 4096, 3282, 707, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15729, 16950, 16265, 16203, 16421, 16674, 16462, 16368] avg=16384.0000 max=16950 max_rank=1 min=15729 min_rank=0 imbalance_ratio=1.034546 max_load_reduction=114122 max_load_reduction_pct=0.870682 moved_to_echo=65925 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2010, 0, 329, 0, 0, 3775, 0, 4096, 22, 0, 0, 0, 0, 4096, 918, 758, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3467, 0, 0, 4096, 0, 0, 814, 0, 0, 0, 4096, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 11, 1, 4096, 108, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3176, 2829, 0, 0, 7, 0, 0, 0, 0, 0, 4077, 4075, 0, 0, 0, 1812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 6, 4033, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2620, 0, 0, 7, 4018, 0, 32, 0, 2192, 0, 0, 4086, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 116, 4096, 4096, 3282, 704, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4095, 0, 0, 2, 0, 7, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16059, 16450, 16374, 15976, 16405, 17023, 16410, 16375] avg=16384.0000 max=17023 max_rank=5 min=15976 min_rank=3 imbalance_ratio=1.039001 max_load_reduction=114049 max_load_reduction_pct=0.870125 moved_to_echo=66213 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1104, 0, 1320, 0, 0, 3762, 0, 4096, 28, 0, 0, 0, 0, 4096, 918, 761, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3608, 3, 0, 4096, 0, 0, 814, 0, 0, 0, 4096, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 0, 4096, 19, 0, 0, 0, 4096, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3175, 2668, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4063, 0, 0, 3, 2209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 2, 4062, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2628, 0, 0, 78, 3889, 0, 108, 0, 1949, 0, 0, 4049, 0, 0, 0, 3934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 76, 4096, 4079, 3282, 707, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 6, 0, 4096, 1, 0, 1, 0, 16, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16139, 16636, 16287, 16207, 16440, 16635, 16340, 16388] avg=16384.0000 max=16636 max_rank=1 min=16139 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=114436 max_load_reduction_pct=0.873077 moved_to_echo=65803 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1626, 4, 1749, 0, 1, 3678, 0, 4092, 13, 0, 0, 0, 2, 4091, 917, 760, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 3276, 0, 0, 4091, 0, 0, 813, 0, 0, 0, 4096, 3907, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 3966, 6, 0, 4096, 1, 0, 0, 1, 4095, 1, 10, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3170, 3591, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4087, 0, 0, 2, 2557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 1, 0, 0, 0, 1, 4096, 1, 0, 4096, 6, 0, 3987, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2627, 0, 1, 9, 4044, 0, 12, 0, 317, 1, 0, 4090, 0, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 11, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 5, 0, 1, 27, 4091, 4092, 3281, 706, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 1, 4095, 0, 0, 0, 4090, 7, 2, 2, 0, 5, 4095, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16988, 16187, 16266, 17494, 16354, 15077, 16330, 16376] avg=16384.0000 max=17494 max_rank=3 min=15077 min_rank=5 imbalance_ratio=1.067749 max_load_reduction=113578 max_load_reduction_pct=0.866531 moved_to_echo=64137 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1674, 0, 1030, 0, 0, 3720, 0, 4095, 2, 0, 0, 0, 0, 4096, 918, 761, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3873, 0, 0, 4096, 0, 0, 814, 0, 0, 0, 4096, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 2, 4096, 4, 0, 0, 1, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3176, 3843, 0, 0, 9, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 1776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 4003, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2628, 0, 0, 2, 3942, 0, 24, 0, 960, 1, 0, 4009, 0, 0, 0, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 4096, 3282, 707, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 5, 4096, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16350, 16899, 16260, 16994, 16369, 15523, 16305, 16372] avg=16384.0000 max=16994 max_rank=3 min=15523 min_rank=5 imbalance_ratio=1.037231 max_load_reduction=114078 max_load_reduction_pct=0.870346 moved_to_echo=64569 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 27776, 128, 0, 32768, 128, 0, 6528, 0, 0, 128, 32768, 31360, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[25472, 25344, 128, 0, 128, 128, 0, 0, 0, 128, 32640, 32640, 128, 128, 128, 15232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 32768, 128, 128, 0, 0, 0, 128, 32768, 128, 0, 32768, 128, 128, 32128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 17408, 128, 5888, 128, 128, 28160, 128, 32768, 512, 128, 128, 128, 128, 32768, 7424, 6144, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 0, 128, 32768, 128, 128, 0, 32768, 128, 128, 128, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 31744, 128, 128, 32768, 1024, 0, 128, 128, 32768, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[20992, 128, 128, 384, 32000, 128, 896, 128, 12416, 128, 0, 32640, 128, 0, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 0, 32768, 0, 128, 0, 128, 128, 128, 128, 768, 32768, 32768, 26368, 5632, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 13, 4091, 2, 0, 594, 4092, 4096, 0, 0, 0, 4096, 0, 4081, 4, 0, 4096, 0, 0, 5, 4093, 0, 3, 4096, 0, 0, 4094, 0, 0, 4096, 1, 1, 0, 0, 0, 4092, 3, 0, 4096, 3698, 4091, 0, 0, 2506, 0, 4096, 4096, 4092, 0, 0, 4095, 35, 7, 0, 0, 0, 0, 3, 5, 0, 4096, 0, 1, 0, 0, 4089, 0, 0, 4092, 4084, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 4091, 4092, 0, 0, 0, 0, 4096, 1, 0, 21, 0, 0, 0, 1, 0, 0, 24, 2, 0, 0, 0, 5, 0, 0, 4096, 4, 0, 0, 0, 1801, 4094, 0, 0, 0, 41, 0, 0, 0, 0, 0, 4094, 3585, 0, 0, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4095, 0, 0, 534, 4096, 4096, 0, 0, 0, 4096, 0, 4095, 3, 0, 4094, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 4096, 0, 0, 4095, 1, 0, 1, 0, 0, 4096, 0, 21, 4096, 3688, 4096, 0, 8, 1248, 0, 4096, 4096, 4087, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2936, 4094, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 3845, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 219, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 6, 0, 4096, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 4096, 0, 0, 4095, 0, 0, 1, 0, 0, 4096, 0, 14, 4095, 3412, 4096, 0, 0, 2040, 0, 4096, 4096, 4094, 0, 0, 4096, 109, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 7, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2425, 4094, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4096, 4045, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[3, 0, 3, 2, 55, 4081, 2, 0, 63, 4095, 4093, 1, 1, 3, 4091, 3, 3929, 0, 2, 4093, 2, 1, 2, 4056, 0, 3, 4094, 1, 0, 4085, 1, 0, 4093, 2, 8, 3, 1, 0, 4093, 9, 7, 4094, 2248, 4067, 2, 0, 3545, 2, 4093, 4093, 4088, 1, 0, 4093, 8, 17, 1, 3, 1, 10, 67, 0, 0, 4092, 4, 0, 4, 74, 4070, 2, 0, 4094, 4029, 3, 3, 3, 0, 0, 0, 0, 4093, 0, 4095, 4075, 2, 0, 3, 0, 4052, 0, 0, 45, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 3, 4093, 14, 2, 3, 0, 2694, 4092, 2, 0, 1, 853, 0, 0, 1, 0, 3, 4084, 3021, 0, 2, 4042, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 11, 4096, 0, 0, 19, 4095, 4096, 0, 0, 0, 4096, 0, 4084, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 15, 4096, 3316, 4082, 0, 0, 3405, 0, 4096, 4096, 4096, 0, 0, 4096, 54, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 6, 4091, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 878, 4096, 0, 0, 0, 727, 0, 0, 0, 0, 0, 4096, 3877, 4, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 47, 4096, 4096, 0, 0, 0, 4096, 0, 4007, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 4090, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 4096, 3863, 4075, 0, 0, 3271, 0, 4096, 4096, 4093, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 18, 6, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 4096, 0, 0, 0, 0, 4096, 0, 0, 63, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 1433, 4096, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4096, 3671, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 4088, 6, 0, 35, 4090, 4096, 0, 0, 0, 4096, 0, 3994, 0, 0, 4096, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 4089, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 100, 4096, 3989, 4095, 0, 0, 2403, 0, 4096, 4096, 4088, 0, 0, 4096, 19, 13, 0, 0, 0, 7, 10, 10, 0, 4096, 0, 0, 0, 210, 4084, 0, 0, 4086, 4075, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 4084, 4089, 0, 0, 0, 0, 4096, 0, 0, 49, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 13, 0, 0, 4096, 0, 0, 0, 0, 1263, 4096, 0, 0, 0, 428, 0, 0, 0, 0, 0, 4096, 3914, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4093, 0, 0, 522, 4095, 4096, 0, 0, 0, 4096, 0, 4094, 1, 0, 4095, 0, 0, 1, 4096, 0, 1, 4087, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 2796, 4095, 0, 0, 2494, 0, 4096, 4096, 4090, 0, 0, 4096, 30, 1, 0, 0, 0, 2, 1, 0, 0, 4096, 0, 0, 0, 16, 4094, 0, 0, 4095, 4092, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4096, 1, 0, 6, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 2, 0, 0, 4095, 1, 0, 0, 0, 2367, 4096, 0, 0, 0, 62, 0, 0, 0, 0, 0, 4095, 4008, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 13, 4091, 2, 0, 594, 4092, 3835, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4, 0, 57, 0, 0, 5, 4093, 0, 3, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2187, 1, 1, 0, 0, 0, 4092, 3, 0, 0, 3698, 4091, 0, 0, 2506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 4092, 0, 0, 4095, 35, 7, 0, 0, 0, 0, 3, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4089, 0, 0, 4092, 4084, 0, 0, 0, 0, 4, 0, 0, 1929, 1909, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4091, 4092, 0, 0, 0, 0, 4096, 1, 0, 21, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 2, 0, 0, 0, 5, 0, 0, 4096, 4, 0, 0, 0, 1801, 4096, 4096, 2110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 41, 0, 0, 0, 0, 0, 4094, 3585, 0, 0, 4093, 0, 54, 23, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16723, 16433, 16579, 16375, 16346, 16386, 16234, 15996] avg=16384.0000 max=16723 max_rank=0 min=15996 min_rank=7 imbalance_ratio=1.020691 max_load_reduction=114349 max_load_reduction_pct=0.872414 moved_to_echo=64962 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 219, 4096, 3835, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 57, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2186, 0, 0, 1, 0, 0, 4096, 0, 14, 0, 3412, 4096, 0, 0, 2040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 4094, 0, 0, 4096, 109, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 1929, 1909, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2425, 4096, 4095, 2110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4096, 4045, 0, 0, 4096, 0, 54, 23, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16342, 16445, 15845, 16438, 16371, 16370, 16824, 16437] avg=16384.0000 max=16824 max_rank=6 min=15845 min_rank=2 imbalance_ratio=1.026855 max_load_reduction=114248 max_load_reduction_pct=0.871643 moved_to_echo=66002 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3, 2, 55, 4081, 2, 0, 63, 4095, 3833, 1, 1, 3, 4091, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3929, 0, 2, 57, 2, 1, 2, 4056, 0, 3, 4094, 1, 0, 4085, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2185, 2, 8, 3, 1, 0, 4093, 9, 7, 0, 2248, 4067, 2, 0, 3545, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4040, 4088, 1, 0, 4093, 8, 17, 1, 3, 1, 10, 67, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 74, 4070, 2, 0, 4094, 4029, 3, 3, 3, 0, 0, 0, 0, 1928, 1908, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 4095, 4075, 2, 0, 3, 0, 4052, 0, 0, 45, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 3, 4093, 14, 2, 3, 0, 2694, 4093, 4094, 2108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2, 0, 1, 853, 0, 0, 1, 0, 3, 4084, 3021, 0, 2, 4042, 0, 53, 23, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16236, 16233, 16172, 16421, 16359, 16354, 17108, 16189] avg=16384.0000 max=17108 max_rank=6 min=16172 min_rank=2 imbalance_ratio=1.044189 max_load_reduction=113964 max_load_reduction_pct=0.869476 moved_to_echo=66010 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4093, 0, 0, 522, 4095, 3828, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 50, 0, 0, 1, 4096, 0, 1, 4087, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 2796, 4095, 0, 0, 2494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 4090, 0, 0, 4096, 30, 1, 0, 0, 0, 2, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4094, 0, 0, 4095, 4092, 0, 0, 0, 0, 1, 0, 0, 1934, 1915, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 4095, 4096, 0, 0, 0, 0, 4096, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 2, 0, 0, 4095, 1, 0, 0, 0, 2367, 4096, 4096, 2111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 62, 0, 0, 0, 0, 0, 4095, 4008, 0, 0, 4096, 0, 55, 29, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16635, 16425, 15663, 16357, 16386, 16373, 16775, 16458] avg=16384.0000 max=16775 max_rank=6 min=15663 min_rank=2 imbalance_ratio=1.023865 max_load_reduction=114297 max_load_reduction_pct=0.872017 moved_to_echo=65992 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 4096, 0, 0, 47, 4096, 3835, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 0, 0, 57, 0, 0, 0, 4093, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2187, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 3863, 4075, 0, 0, 3271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 4093, 0, 0, 4096, 28, 0, 0, 0, 0, 0, 18, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 1929, 1909, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4092, 4096, 0, 0, 0, 0, 4096, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 1433, 4096, 4096, 2110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4096, 3671, 0, 0, 4096, 0, 54, 23, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16170, 16343, 17496, 16379, 16349, 16431, 15839, 16065] avg=16384.0000 max=17496 max_rank=2 min=15839 min_rank=6 imbalance_ratio=1.067871 max_load_reduction=113576 max_load_reduction_pct=0.866516 moved_to_echo=64684 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4095, 0, 0, 534, 4096, 3835, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 58, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2186, 1, 0, 1, 0, 0, 4096, 0, 21, 0, 3688, 4096, 0, 8, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 4087, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 1928, 1909, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2936, 4096, 4096, 2108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 3845, 0, 0, 4096, 0, 54, 23, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16657, 16440, 15345, 16336, 16364, 16372, 17335, 16223] avg=16384.0000 max=17335 max_rank=6 min=15345 min_rank=2 imbalance_ratio=1.058044 max_load_reduction=113737 max_load_reduction_pct=0.867744 moved_to_echo=66294 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 11, 4096, 0, 0, 19, 4095, 3835, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 57, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2187, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 3316, 4082, 0, 0, 3405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 4096, 0, 0, 4096, 54, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4091, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 1929, 1909, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 878, 4096, 4096, 2110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 727, 0, 0, 0, 0, 0, 4096, 3877, 4, 0, 4096, 0, 54, 23, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16152, 16429, 17101, 16385, 16360, 16384, 15276, 16985] avg=16384.0000 max=17101 max_rank=2 min=15276 min_rank=6 imbalance_ratio=1.043762 max_load_reduction=113971 max_load_reduction_pct=0.869530 moved_to_echo=65005 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 7, 4088, 6, 0, 35, 4090, 3835, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 57, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2187, 0, 1, 0, 0, 0, 4096, 0, 100, 0, 3989, 4095, 0, 0, 2403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 4088, 0, 0, 4096, 19, 13, 0, 0, 0, 7, 10, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 4084, 0, 0, 4086, 4075, 0, 0, 0, 0, 6, 0, 0, 1929, 1909, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4084, 4089, 0, 0, 0, 0, 4096, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 13, 0, 0, 4096, 0, 0, 0, 0, 1263, 4096, 4096, 2110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 428, 0, 0, 0, 0, 0, 4096, 3914, 0, 0, 4096, 0, 54, 23, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16157, 16324, 16871, 16381, 16537, 16402, 15681, 16719] avg=16384.0000 max=16871 max_rank=2 min=15681 min_rank=6 imbalance_ratio=1.029724 max_load_reduction=114201 max_load_reduction_pct=0.871284 moved_to_echo=65339 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32384, 128, 128, 512, 128, 128, 128, 32768, 0, 128, 32768, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 0, 128, 2176, 0, 0, 128, 0, 128, 32768, 30080, 128, 128, 32768, 0, 512, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32384, 32768, 128, 0, 32768, 384, 128, 128, 128, 128, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 0, 15872, 32768, 32768, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[17536, 128, 128, 128, 128, 0, 32768, 128, 256, 0, 27136, 32768, 128, 128, 20992, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 128, 32768, 128, 0, 2048, 32768, 30720, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 384, 32768, 128, 0, 32768, 32768, 128, 128, 128, 0, 128, 0, 0, 15488, 15360, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 0, 32768, 32768, 128, 0, 128, 0, 32768, 128, 0, 256, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[3252, 0, 4086, 0, 0, 4088, 2092, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 0, 4096, 0, 4096, 52, 0, 0, 0, 0, 5, 6, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 0, 0, 3898, 4096, 0, 0, 65, 122, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 3326, 4096, 4096, 0, 4085, 0, 4096, 0, 0, 0, 0, 0, 140, 0, 3509, 4096, 0, 0, 0, 23, 0, 4096, 0, 0, 4009, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[3106, 0, 4076, 0, 0, 4080, 2111, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 4094, 0, 4096, 0, 4096, 55, 0, 0, 0, 1, 19, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 0, 0, 4095, 4096, 0, 0, 4048, 4096, 0, 0, 5, 52, 4096, 1, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 4094, 0, 0, 3120, 4095, 4093, 0, 4081, 0, 4095, 0, 0, 1, 0, 0, 141, 0, 3983, 4095, 0, 0, 0, 8, 0, 4096, 0, 0, 3859, 24, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[3514, 1, 3995, 0, 0, 3670, 2237, 6, 0, 3, 80, 21, 0, 2, 1, 0, 0, 3, 9, 3, 0, 0, 2, 3, 0, 3, 3, 1, 0, 4067, 4092, 4078, 1, 4093, 1, 4093, 0, 0, 0, 3, 55, 12, 5, 1, 0, 0, 4089, 4, 0, 3, 4090, 2, 2, 0, 4093, 4094, 4, 3, 3959, 4093, 3, 0, 442, 38, 4092, 17, 3952, 4093, 0, 1, 0, 1, 8, 2, 0, 3, 0, 5, 0, 4092, 0, 0, 3, 2, 3, 2, 3, 0, 16, 0, 3, 4092, 3, 0, 4092, 4032, 0, 3, 2379, 4093, 4036, 0, 4079, 0, 4094, 2, 3, 32, 1, 1, 317, 1, 4007, 4095, 0, 0, 6, 550, 1, 4092, 0, 0, 3477, 128, 4093, 1, 0, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[3579, 0, 4073, 0, 0, 3973, 2583, 0, 0, 0, 26, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 3, 16, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4093, 4096, 0, 0, 4062, 4096, 0, 0, 273, 68, 4096, 1, 4051, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 2082, 4096, 4084, 0, 4021, 0, 4096, 0, 0, 0, 0, 0, 469, 0, 3540, 4085, 0, 0, 0, 193, 0, 4096, 0, 0, 3806, 87, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[3440, 0, 4076, 0, 0, 4089, 2793, 0, 0, 0, 16, 6, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4096, 4091, 0, 4096, 0, 4096, 27, 0, 0, 0, 4, 41, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 4069, 4096, 0, 0, 4084, 4096, 0, 1, 15, 22, 4096, 8, 4073, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 45, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4067, 0, 0, 2376, 4096, 4080, 0, 4061, 1, 4092, 0, 0, 0, 0, 0, 61, 0, 3622, 4088, 0, 0, 0, 63, 0, 4096, 0, 0, 4018, 23, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[3319, 0, 4049, 0, 0, 3968, 1549, 0, 0, 0, 23, 37, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 4096, 4085, 0, 4096, 0, 4096, 1, 0, 0, 0, 11, 32, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 0, 4094, 4089, 0, 0, 3945, 4096, 0, 0, 105, 5, 4096, 11, 4022, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 3670, 4096, 4079, 0, 4079, 0, 4091, 0, 0, 0, 0, 0, 502, 0, 3657, 4085, 0, 0, 0, 104, 0, 4096, 0, 0, 3735, 93, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[3670, 0, 4093, 0, 0, 4096, 1573, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 9, 0, 0, 0, 0, 7, 3, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 0, 0, 4095, 4096, 0, 0, 4066, 4096, 0, 0, 25, 33, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 3082, 4096, 4096, 0, 4046, 0, 4096, 0, 0, 0, 0, 0, 197, 0, 3986, 4096, 0, 1, 0, 28, 0, 4096, 0, 0, 3834, 10, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[3947, 0, 4085, 0, 0, 4033, 581, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 4096, 0, 4096, 0, 4096, 9, 0, 0, 0, 0, 176, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4089, 4096, 0, 0, 4090, 4096, 0, 0, 105, 12, 4096, 8, 4084, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4091, 0, 0, 3689, 4096, 4088, 0, 4094, 0, 4096, 0, 0, 6, 0, 0, 232, 0, 3947, 4095, 0, 0, 0, 47, 0, 4096, 0, 0, 3715, 18, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3514, 1, 3995, 0, 0, 3670, 2237, 6, 0, 3, 80, 21, 0, 2, 1, 0, 2804, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 3, 0, 0, 2, 3, 0, 3, 3, 1, 0, 4067, 4092, 4078, 4093, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 1, 4093, 0, 0, 0, 3, 55, 12, 5, 1, 0, 0, 4089, 4, 3644, 378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4000, 2, 2, 0, 4093, 4094, 4, 3, 3959, 0, 3, 0, 442, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 17, 3952, 4093, 0, 1, 0, 1, 8, 2, 0, 3, 0, 5, 0, 4092, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 3, 2, 3, 0, 16, 0, 3, 4092, 3, 0, 4092, 4032, 4077, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2379, 911, 4036, 0, 4079, 0, 4094, 2, 3, 32, 1, 1, 317, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 4095, 0, 0, 6, 550, 1, 434, 0, 0, 3477, 128, 0, 1, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16394, 16373, 16379, 16643, 16296, 16342, 15859, 16786] avg=16384.0000 max=16786 max_rank=7 min=15859 min_rank=6 imbalance_ratio=1.024536 max_load_reduction=114286 max_load_reduction_pct=0.871933 moved_to_echo=65283 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3440, 0, 4076, 0, 0, 4089, 2793, 0, 0, 0, 16, 6, 0, 0, 0, 0, 2806, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 4096, 4091, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 27, 0, 0, 0, 4, 41, 0, 0, 0, 0, 4096, 0, 3648, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 0, 0, 4069, 4096, 0, 0, 4084, 0, 0, 1, 15, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 4073, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 45, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4067, 4079, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2376, 911, 4080, 0, 4061, 1, 4092, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3622, 4088, 0, 0, 0, 63, 0, 434, 0, 0, 4018, 23, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17286, 16383, 16387, 16292, 16400, 16398, 15582, 16344] avg=16384.0000 max=17286 max_rank=0 min=15582 min_rank=6 imbalance_ratio=1.055054 max_load_reduction=113786 max_load_reduction_pct=0.868118 moved_to_echo=64724 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3106, 0, 4076, 0, 0, 4080, 2111, 0, 0, 0, 5, 0, 0, 0, 0, 0, 2807, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 4094, 4096, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 55, 0, 0, 0, 1, 19, 0, 0, 0, 0, 4096, 0, 3652, 381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3996, 0, 0, 0, 4095, 4096, 0, 0, 4048, 0, 0, 0, 5, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 4094, 4078, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3120, 907, 4093, 0, 4081, 0, 4095, 0, 0, 1, 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 4095, 0, 0, 0, 8, 0, 428, 0, 0, 3859, 24, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16250, 16396, 16396, 16293, 16416, 16390, 16438, 16493] avg=16384.0000 max=16493 max_rank=7 min=16250 min_rank=0 imbalance_ratio=1.006653 max_load_reduction=114579 max_load_reduction_pct=0.874168 moved_to_echo=65737 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3252, 0, 4086, 0, 0, 4088, 2092, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2806, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 4096, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 52, 0, 0, 0, 0, 5, 6, 0, 0, 0, 4096, 0, 3648, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 4095, 4096, 0, 0, 3898, 0, 0, 0, 65, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4079, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3326, 911, 4096, 0, 4085, 0, 4096, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3509, 4096, 0, 0, 0, 23, 0, 434, 0, 0, 4009, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16400, 16378, 16282, 16416, 16390, 16654, 16167] avg=16384.0000 max=16654 max_rank=6 min=16167 min_rank=7 imbalance_ratio=1.016479 max_load_reduction=114418 max_load_reduction_pct=0.872940 moved_to_echo=65627 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3579, 0, 4073, 0, 0, 3973, 2583, 0, 0, 0, 26, 21, 0, 0, 0, 0, 2806, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 4096, 4096, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 3, 16, 0, 0, 0, 0, 4096, 0, 3648, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 4093, 4096, 0, 0, 4062, 0, 0, 0, 273, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4051, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4079, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2082, 911, 4084, 0, 4021, 0, 4096, 0, 0, 0, 0, 0, 469, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3540, 4085, 0, 0, 0, 193, 0, 434, 0, 0, 3806, 87, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17121, 16362, 16334, 16598, 16370, 16383, 15663, 16241] avg=16384.0000 max=17121 max_rank=0 min=15663 min_rank=6 imbalance_ratio=1.044983 max_load_reduction=113951 max_load_reduction_pct=0.869377 moved_to_echo=64657 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3947, 0, 4085, 0, 0, 4033, 581, 0, 0, 0, 7, 2, 0, 0, 0, 0, 2806, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 4096, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 9, 0, 0, 0, 0, 176, 0, 0, 0, 0, 4096, 0, 3648, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 4089, 4096, 0, 0, 4090, 0, 0, 0, 105, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 4084, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4091, 4079, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3689, 911, 4088, 0, 4094, 0, 4096, 0, 0, 6, 0, 0, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 4095, 0, 0, 0, 47, 0, 434, 0, 0, 3715, 18, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15521, 16394, 16500, 16398, 16410, 16381, 17116, 16352] avg=16384.0000 max=17116 max_rank=6 min=15521 min_rank=0 imbalance_ratio=1.044678 max_load_reduction=113956 max_load_reduction_pct=0.869415 moved_to_echo=66259 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3319, 0, 4049, 0, 0, 3968, 1549, 0, 0, 0, 23, 37, 0, 0, 0, 0, 2806, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 4096, 4085, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 11, 32, 0, 0, 0, 5, 4096, 0, 3648, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 0, 4094, 4089, 0, 0, 3945, 0, 0, 0, 105, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 4022, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4079, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3670, 911, 4079, 0, 4079, 0, 4091, 0, 0, 0, 0, 0, 502, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3657, 4085, 0, 0, 0, 104, 0, 434, 0, 0, 3735, 93, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15811, 16363, 16364, 16244, 16351, 16403, 17332, 16204] avg=16384.0000 max=17332 max_rank=6 min=15811 min_rank=0 imbalance_ratio=1.057861 max_load_reduction=113740 max_load_reduction_pct=0.867767 moved_to_echo=66290 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3670, 0, 4093, 0, 0, 4096, 1573, 0, 0, 0, 1, 5, 0, 0, 0, 0, 2806, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 9, 0, 0, 0, 0, 7, 3, 0, 0, 0, 4096, 0, 3648, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4006, 0, 0, 0, 4095, 4096, 0, 0, 4066, 0, 0, 0, 25, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4079, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3082, 911, 4096, 0, 4046, 0, 4096, 0, 0, 0, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 4096, 0, 1, 0, 28, 0, 434, 0, 0, 3834, 10, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16304, 16401, 16334, 16322, 16413, 16385, 16428, 16485] avg=16384.0000 max=16485 max_rank=7 min=16304 min_rank=0 imbalance_ratio=1.006165 max_load_reduction=114587 max_load_reduction_pct=0.874229 moved_to_echo=65711 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[30336, 32768, 0, 128, 128, 1024, 128, 3584, 0, 0, 30464, 384, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 32768, 256, 0, 0, 128, 128, 384, 128, 128, 0, 128, 32768, 128, 29184, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[27904, 128, 32640, 0, 0, 32000, 15616, 128, 0, 128, 256, 128, 0, 128, 128, 128, 22528, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 32512, 32768, 0, 128, 0, 128, 128, 128, 0, 128, 0, 128, 0, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 0, 128, 0, 128, 32768, 128, 128, 32768, 32768, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 32128, 128, 128, 0, 32768, 32768, 128, 128, 32256, 0, 128, 128, 1152, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 0, 0, 128, 128, 0, 128, 128, 128, 0, 32640, 32768, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 23808, 7296, 32768, 0, 32640, 128, 32768, 128, 128, 128, 128, 128, 2176, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4085, 4093, 0, 0, 66, 0, 4092, 0, 0, 0, 20, 0, 15, 0, 0, 0, 4094, 10, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4090, 4096, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 4096, 4096, 77, 0, 0, 2, 0, 0, 0, 0, 0, 3975, 0, 0, 0, 4074, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 4046, 0, 0, 916, 4096, 0, 0, 0, 3321, 4074, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 4096, 0, 0, 0, 4011, 0, 0, 1, 0, 6, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 55, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 4096, 4094, 2, 0, 0, 1, 0, 0, 0, 0, 0, 3915, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1166, 4096, 0, 0, 0, 3053, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 0, 0, 3, 0, 4, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4093, 0, 3, 3, 3, 0, 4092, 4068, 4093, 3, 0, 92, 1, 4078, 1, 0, 0, 21, 3, 0, 3, 4, 1, 4093, 0, 3, 4093, 4093, 2, 0, 0, 9, 4093, 4093, 4093, 0, 5, 22, 0, 3, 0, 4093, 0, 2, 0, 9, 3, 4093, 4088, 4071, 103, 10, 0, 0, 3, 0, 0, 3, 3, 4036, 1, 3, 0, 4050, 0, 0, 4093, 2, 3, 4093, 1, 3, 0, 24, 3, 0, 3, 4078, 4038, 3, 3, 1382, 4093, 0, 4, 0, 2624, 4080, 1, 4092, 4093, 2, 0, 3, 4093, 4089, 0, 3, 0, 3, 1, 0, 0, 4093, 0, 4093, 0, 0, 1, 4005, 0, 0, 65, 1, 0, 3, 0, 3, 62, 0, 0, 0, 4093, 0, 0, 6, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4094, 4096, 0, 0, 233, 0, 4078, 0, 0, 0, 3, 0, 1, 0, 0, 0, 4095, 1, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 1, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4095, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 214, 4095, 0, 0, 0, 3668, 4093, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 3, 0, 3, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4093, 4089, 4096, 0, 0, 345, 0, 4046, 0, 0, 0, 8, 0, 6, 0, 0, 0, 4096, 4, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4092, 4096, 0, 0, 4, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4094, 4095, 4092, 13, 0, 0, 16, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 0, 0, 108, 4096, 0, 0, 0, 3736, 4088, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 1, 0, 0, 4083, 1, 0, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 99, 0, 4077, 0, 0, 0, 1, 0, 27, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 4073, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4073, 0, 0, 704, 4096, 0, 0, 0, 3423, 4058, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4061, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 112, 0, 4090, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 83, 4096, 0, 0, 0, 3983, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4, 0, 1, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4093, 4096, 0, 0, 221, 0, 4071, 0, 0, 0, 3, 0, 1, 0, 0, 0, 4084, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4095, 53, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 402, 4095, 0, 0, 0, 3500, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 4088, 0, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 3, 3, 0, 4092, 4068, 3968, 3, 0, 92, 1, 4078, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 3, 0, 3, 4, 1, 4093, 0, 3, 4093, 4093, 2, 0, 0, 9, 4011, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4089, 0, 5, 22, 0, 3, 0, 4093, 0, 2, 0, 9, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4071, 103, 10, 0, 0, 3, 0, 0, 3, 3, 4036, 1, 3, 0, 4050, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 3, 4093, 1, 3, 0, 24, 3, 0, 3, 4078, 4038, 3, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1382, 4093, 0, 4, 0, 2624, 4080, 1, 73, 0, 2, 0, 3, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 1, 0, 0, 4093, 0, 4093, 0, 0, 1, 4005, 0, 0, 4093, 24, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 1, 0, 3, 0, 3, 62, 0, 0, 0, 4093, 0, 0, 6, 0, 2, 4093, 4093, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16312, 16401, 16412, 16418, 16361, 16354, 16324, 16490] avg=16384.0000 max=16490 max_rank=7 min=16312 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=114582 max_load_reduction_pct=0.874191 moved_to_echo=65529 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4093, 4089, 3971, 0, 0, 345, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 6, 0, 0, 0, 4096, 4, 0, 4096, 4096, 0, 0, 0, 0, 4015, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4092, 0, 0, 4, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4092, 13, 0, 0, 16, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 4086, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 0, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 4096, 0, 0, 0, 3736, 4088, 0, 72, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 1, 0, 0, 4083, 1, 0, 4096, 25, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 2, 0, 0, 4096, 4096, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16544, 16386, 16379, 16409, 16383, 16196, 16403, 16372] avg=16384.0000 max=16544 max_rank=0 min=16196 min_rank=5 imbalance_ratio=1.009766 max_load_reduction=114528 max_load_reduction_pct=0.873779 moved_to_echo=65354 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 3971, 0, 0, 55, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4015, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 2, 0, 0, 1, 0, 0, 0, 0, 0, 3915, 0, 0, 0, 4096, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1166, 4096, 0, 0, 0, 3053, 4096, 0, 72, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 0, 0, 4096, 24, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16300, 16368, 16383, 16251, 16401, 16579, 16416, 16374] avg=16384.0000 max=16579 max_rank=5 min=16251 min_rank=3 imbalance_ratio=1.011902 max_load_reduction=114493 max_load_reduction_pct=0.873512 moved_to_echo=65770 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4095, 4085, 3968, 0, 0, 66, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 15, 0, 0, 0, 4094, 10, 0, 4096, 4096, 0, 0, 0, 0, 4015, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4092, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 77, 0, 0, 2, 0, 0, 0, 0, 0, 3975, 0, 0, 0, 4074, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 4046, 0, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 916, 4096, 0, 0, 0, 3321, 4074, 0, 72, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 4096, 0, 0, 0, 4011, 0, 0, 4096, 24, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16306, 16411, 16383, 16367, 16348, 16575, 16312, 16370] avg=16384.0000 max=16575 max_rank=5 min=16306 min_rank=0 imbalance_ratio=1.011658 max_load_reduction=114497 max_load_reduction_pct=0.873543 moved_to_echo=65605 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4094, 3953, 0, 0, 233, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 4095, 1, 0, 4096, 4096, 0, 0, 0, 0, 4020, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4085, 0, 0, 1, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 4095, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 20, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 4095, 0, 0, 0, 3668, 4093, 0, 66, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 25, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16454, 16382, 16374, 16429, 16411, 16232, 16418, 16372] avg=16384.0000 max=16454 max_rank=0 min=16232 min_rank=5 imbalance_ratio=1.004272 max_load_reduction=114618 max_load_reduction_pct=0.874466 moved_to_echo=65433 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 3971, 0, 0, 99, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 27, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4015, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 4073, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4073, 0, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 704, 4096, 0, 0, 0, 3423, 4058, 0, 72, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4061, 0, 0, 4096, 24, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16339, 16396, 16380, 16386, 16378, 16449, 16382, 16362] avg=16384.0000 max=16449 max_rank=5 min=16339 min_rank=0 imbalance_ratio=1.003967 max_load_reduction=114623 max_load_reduction_pct=0.874504 moved_to_echo=65571 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4093, 3971, 0, 0, 221, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 4084, 0, 0, 4096, 4096, 0, 0, 0, 0, 4015, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 53, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 4089, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 4095, 0, 0, 0, 3500, 4096, 0, 72, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 0, 0, 0, 4088, 0, 0, 4096, 24, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16452, 16360, 16380, 16464, 16391, 16261, 16399, 16365] avg=16384.0000 max=16464 max_rank=3 min=16261 min_rank=5 imbalance_ratio=1.004883 max_load_reduction=114608 max_load_reduction_pct=0.874390 moved_to_echo=65416 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 4096, 4096, 3971, 0, 0, 112, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 4095, 0, 0, 0, 0, 4015, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 4096, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 13, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 4096, 0, 0, 0, 3983, 4096, 0, 72, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 25, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16365, 16368, 16381, 16348, 16399, 16426, 16418, 16367] avg=16384.0000 max=16426 max_rank=5 min=16348 min_rank=3 imbalance_ratio=1.002563 max_load_reduction=114646 max_load_reduction_pct=0.874680 moved_to_echo=65610 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 32768, 0, 128, 128, 0, 128, 0, 32768, 128, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 32768, 128, 128, 32768, 128, 128, 0, 128, 128, 0, 128, 32768, 32640, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 32768, 384, 128, 0, 128, 128, 0, 0, 128, 128, 32256, 128, 128, 0, 32768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 32768, 128, 0, 0, 128, 32128, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 128, 0, 32768, 32768, 31744, 128, 0, 1280, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 32768, 0, 32768, 128, 0, 128, 32640, 128, 0, 32768, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 0, 128, 128, 0, 0, 0, 32768, 0, 128, 128, 0, 128, 32768, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 4992, 32768, 0, 128, 0, 27392, 32768, 128, 640, 0, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 16, 4096, 0, 1532, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 4050, 1, 0, 18, 4093, 0, 163, 4080, 4096, 1, 0, 0, 0, 0, 4094, 2329, 4096, 0, 3653, 4064, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 474, 0, 0, 0, 4096, 0, 4096, 4096, 0, 22, 168, 4096, 4096, 4086, 4030, 0, 4096, 0, 8, 0, 4090, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 0, 4067, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[3, 0, 0, 4093, 3, 0, 3, 0, 4059, 0, 3, 1, 0, 2, 0, 70, 0, 0, 3, 4093, 3, 3, 4093, 3, 3, 3, 35, 4093, 0, 149, 3, 0, 8, 0, 0, 3992, 0, 3, 4093, 6, 3, 0, 4063, 0, 0, 3, 4094, 2, 1288, 4069, 4091, 15, 0, 4, 1, 0, 4093, 2913, 4089, 0, 3216, 4093, 0, 0, 4092, 0, 0, 0, 0, 4093, 0, 0, 0, 2, 619, 4, 4, 0, 4092, 0, 4093, 4092, 2, 0, 93, 4093, 4091, 4093, 4089, 0, 4091, 2, 14, 2, 4052, 3, 4091, 1, 2, 0, 3, 4093, 0, 3, 0, 63, 2, 3, 0, 1, 4092, 5, 1, 4026, 4093, 0, 1, 3, 0, 2, 3, 5, 3, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 159, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 2, 0, 0, 4094, 0, 0, 5, 4095, 0, 280, 4095, 4096, 0, 0, 0, 0, 0, 4095, 3735, 4096, 0, 3922, 4081, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 137, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4, 319, 4096, 4096, 4094, 3741, 0, 4096, 0, 1, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 12, 4095, 0, 350, 0, 0, 0, 0, 0, 4096, 1, 0, 4095, 0, 0, 0, 4082, 0, 0, 10, 4090, 0, 774, 4092, 4096, 3, 0, 0, 0, 0, 4094, 3171, 4096, 0, 3831, 4049, 0, 0, 4096, 0, 0, 1, 0, 4094, 3, 1, 0, 0, 148, 0, 0, 0, 4096, 0, 4096, 4096, 0, 7, 199, 4096, 4093, 4093, 3866, 0, 4096, 0, 6, 0, 4089, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 141, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 4095, 0, 0, 0, 4096, 0, 154, 4096, 4096, 0, 0, 0, 0, 0, 4096, 3874, 4096, 0, 3942, 4088, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 37, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 56, 4096, 4096, 4096, 4092, 0, 4094, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 304, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 779, 4096, 4096, 0, 0, 0, 0, 0, 4096, 2822, 4096, 0, 4072, 4092, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 45, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 186, 4096, 4096, 4095, 4086, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 360, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 4085, 0, 0, 12, 4096, 0, 859, 4096, 4096, 0, 0, 0, 0, 0, 4096, 2072, 4096, 0, 3977, 4066, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 335, 0, 0, 0, 4096, 0, 4096, 4096, 0, 3, 684, 4096, 4096, 4095, 4025, 0, 4096, 0, 0, 0, 4079, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 4096, 0, 0, 4066, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 335, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 429, 4092, 4096, 0, 0, 0, 0, 0, 4095, 3001, 4096, 0, 4014, 4037, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 360, 0, 0, 0, 4096, 0, 4096, 4096, 0, 1, 213, 4096, 4096, 4096, 4017, 0, 4096, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 4096, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 4093, 3, 0, 3, 0, 4059, 0, 3, 1, 0, 2, 0, 70, 4091, 4075, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 3, 3, 4093, 3, 3, 3, 35, 4093, 0, 149, 3, 0, 3281, 274, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 3992, 0, 3, 4093, 6, 3, 0, 4063, 0, 0, 3, 4094, 2, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1288, 4069, 0, 15, 0, 4, 1, 0, 812, 2913, 0, 0, 3216, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4093, 0, 0, 0, 2, 619, 4, 4, 0, 4092, 0, 3817, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 93, 0, 3960, 4093, 4089, 0, 0, 2, 14, 2, 4052, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 2, 0, 3, 4093, 0, 3, 0, 63, 2, 3, 0, 1, 4092, 5, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4026, 4093, 0, 1, 3, 0, 2, 3, 5, 3, 0, 0, 0, 0, 0, 4093, 4093, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16410, 16150, 16287, 16411, 16729, 16310, 16435, 16340] avg=16384.0000 max=16729 max_rank=4 min=16150 min_rank=1 imbalance_ratio=1.021057 max_load_reduction=114343 max_load_reduction_pct=0.872368 moved_to_echo=65814 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 66, 4096, 4080, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 16, 4096, 0, 1532, 0, 0, 3282, 275, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 0, 0, 4050, 1, 0, 18, 4093, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 4080, 0, 1, 0, 0, 0, 0, 812, 2329, 0, 0, 3653, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 474, 0, 0, 0, 4096, 0, 3821, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 168, 0, 3963, 4086, 4030, 0, 0, 0, 8, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 0, 0, 0, 4096, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 17505, 16366, 15102, 16588, 16367, 16385, 16376] avg=16384.0000 max=17505 max_rank=1 min=15102 min_rank=3 imbalance_ratio=1.068420 max_load_reduction=113567 max_load_reduction_pct=0.866447 moved_to_echo=65716 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 12, 4096, 4080, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 159, 0, 0, 3287, 275, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 2, 0, 0, 4094, 0, 0, 5, 4095, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 4095, 0, 0, 0, 0, 0, 0, 808, 3735, 0, 0, 3922, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 137, 0, 0, 0, 4096, 0, 3821, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 319, 0, 3959, 4094, 3741, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16125, 16410, 16921, 16253, 16213, 16371, 16395] avg=16384.0000 max=16921 max_rank=3 min=16125 min_rank=1 imbalance_ratio=1.032776 max_load_reduction=114151 max_load_reduction_pct=0.870903 moved_to_echo=65232 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 12, 4096, 4080, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 12, 4095, 0, 350, 0, 0, 3282, 275, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4095, 0, 0, 0, 4082, 0, 0, 10, 4090, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 774, 4092, 0, 3, 0, 0, 0, 0, 812, 3171, 0, 0, 3831, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4094, 3, 1, 0, 0, 148, 0, 0, 0, 4096, 0, 3821, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 199, 0, 3962, 4093, 3866, 0, 0, 0, 6, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16371, 16317, 16394, 16732, 16266, 16222, 16375, 16395] avg=16384.0000 max=16732 max_rank=3 min=16222 min_rank=5 imbalance_ratio=1.021240 max_load_reduction=114340 max_load_reduction_pct=0.872345 moved_to_echo=65258 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4079, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 141, 0, 0, 3284, 274, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 4095, 0, 0, 0, 4096, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 154, 4096, 0, 0, 0, 0, 0, 0, 812, 3874, 0, 0, 3942, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 37, 0, 0, 0, 4096, 0, 3820, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 3963, 4096, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16374, 16099, 16407, 16966, 16151, 16303, 16371, 16401] avg=16384.0000 max=16966 max_rank=3 min=16099 min_rank=1 imbalance_ratio=1.035522 max_load_reduction=114106 max_load_reduction_pct=0.870560 moved_to_echo=65226 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 11, 4096, 4080, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 335, 0, 0, 3283, 275, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 429, 4092, 0, 0, 0, 0, 0, 0, 812, 3001, 0, 0, 4014, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 360, 0, 0, 0, 4096, 0, 3821, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 213, 0, 3963, 4096, 4017, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 4096, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16367, 16293, 16395, 16385, 16475, 16382, 16380, 16395] avg=16384.0000 max=16475 max_rank=4 min=16293 min_rank=1 imbalance_ratio=1.005554 max_load_reduction=114597 max_load_reduction_pct=0.874306 moved_to_echo=65632 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 33, 4096, 4080, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 360, 0, 0, 3284, 275, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 4085, 0, 0, 12, 4096, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 859, 4096, 0, 0, 0, 0, 0, 0, 812, 2072, 0, 0, 3977, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 335, 0, 0, 0, 4096, 0, 3821, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 684, 0, 3963, 4095, 4025, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 4096, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16409, 16319, 16407, 15882, 16451, 16849, 16385, 16370] avg=16384.0000 max=16849 max_rank=5 min=15882 min_rank=3 imbalance_ratio=1.028381 max_load_reduction=114223 max_load_reduction_pct=0.871452 moved_to_echo=66055 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 304, 0, 0, 3284, 275, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 779, 4096, 0, 0, 0, 0, 0, 0, 812, 2822, 0, 0, 4072, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 45, 0, 0, 0, 4096, 0, 3820, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 3963, 4095, 4086, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16374, 16264, 16406, 16673, 16159, 16426, 16370, 16400] avg=16384.0000 max=16673 max_rank=3 min=16159 min_rank=4 imbalance_ratio=1.017639 max_load_reduction=114399 max_load_reduction_pct=0.872795 moved_to_echo=65355 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 128, 128, 0, 128, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 32768, 0, 3456, 128, 0, 26368, 2304, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 32768, 128, 0, 128, 0, 32640, 0, 128, 128, 0, 128, 0, 256, 32768, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 128, 1920, 0, 31744, 32768, 32000, 0, 0, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32640, 32768, 0, 128, 128, 0, 128, 128, 128, 128, 0, 0, 128, 0, 0, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 0, 0, 128, 0, 32768, 128, 128, 0, 128, 2176, 128, 128, 0, 32768, 0, 30592, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[4736, 32768, 0, 128, 0, 128, 128, 0, 6528, 23936, 0, 0, 30720, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 0, 32768, 128, 128, 32768, 128, 128, 0, 32768, 128, 0, 128, 32768, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[3991, 3, 0, 0, 0, 3, 4092, 0, 3, 3, 0, 3, 3, 0, 4093, 3, 910, 4, 0, 0, 0, 4072, 1063, 0, 1, 0, 4080, 3, 0, 0, 0, 0, 1, 4075, 784, 0, 4093, 1, 4, 4093, 3, 4079, 4092, 1, 0, 3, 4033, 2, 3, 9, 1, 0, 3, 14, 4092, 0, 2, 4, 0, 0, 3, 4061, 4093, 4, 1, 34, 3, 0, 2, 3, 0, 1, 4093, 0, 4091, 4089, 3, 3, 0, 13, 4092, 1, 4093, 2228, 1, 4092, 3, 3, 0, 3, 0, 0, 0, 4092, 17, 4092, 2, 3, 4, 3, 0, 3037, 4092, 0, 0, 1, 3, 100, 0, 4092, 3, 1, 4092, 0, 168, 0, 3, 71, 4093, 0, 0, 4092, 0, 0, 4092, 4090, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[4092, 0, 0, 35, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 1629, 0, 0, 0, 0, 4096, 895, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 195, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1578, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3686, 4096, 0, 0, 0, 0, 174, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[4069, 0, 0, 17, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 991, 0, 0, 0, 0, 4095, 1686, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 4095, 120, 0, 4095, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4095, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1482, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 3831, 4096, 0, 0, 0, 0, 80, 0, 4095, 0, 0, 4095, 0, 0, 1, 0, 13, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[4036, 0, 0, 50, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 907, 0, 0, 0, 0, 4089, 846, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 890, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1980, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3330, 4096, 0, 0, 0, 0, 264, 0, 4096, 0, 0, 4079, 0, 1, 0, 0, 2, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[3951, 0, 0, 27, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2703, 0, 0, 0, 0, 4081, 422, 0, 0, 0, 4086, 0, 0, 0, 0, 2, 0, 4094, 468, 0, 4090, 0, 0, 4096, 0, 4088, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 15, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 887, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 4096, 0, 0, 0, 4, 0, 3632, 4096, 0, 0, 0, 0, 207, 0, 4096, 0, 0, 4078, 0, 0, 10, 0, 17, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4091, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[4080, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1131, 0, 0, 0, 0, 4096, 503, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1281, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1294, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3956, 4096, 0, 0, 0, 0, 11, 0, 4096, 0, 0, 4096, 0, 9, 0, 0, 24, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[4028, 0, 0, 8, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 786, 0, 0, 0, 0, 4090, 543, 0, 0, 0, 4092, 0, 0, 0, 0, 5, 0, 4091, 1481, 0, 4093, 0, 0, 4096, 0, 4091, 4094, 1, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 2, 0, 0, 4096, 4096, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1339, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 5, 4094, 0, 0, 0, 0, 3, 3897, 4096, 0, 0, 1, 0, 152, 0, 4094, 0, 0, 4091, 0, 1, 4, 0, 61, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4088, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4033, 0, 0, 10, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 609, 0, 0, 0, 0, 4096, 105, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 925, 0, 4096, 0, 0, 4096, 0, 4093, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 2105, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 3631, 4096, 0, 0, 0, 0, 865, 0, 4096, 0, 0, 4092, 0, 1, 1, 0, 9, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3991, 3, 0, 0, 0, 3, 4092, 0, 3, 3, 0, 3, 3, 0, 4093, 3, 2414, 1718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 910, 4, 0, 0, 0, 4072, 1063, 0, 1, 0, 4080, 3, 0, 0, 0, 0, 4093, 2138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4075, 784, 0, 3341, 1, 4, 0, 3, 4079, 0, 1, 0, 3, 4033, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 1, 0, 3, 14, 4092, 0, 2, 4, 0, 0, 3, 4061, 4093, 4, 2414, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 34, 3, 0, 2, 3, 0, 1, 4093, 0, 4091, 4089, 3, 3, 0, 13, 1678, 1613, 752, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2479, 1, 0, 2228, 1, 4092, 3, 3, 0, 3, 0, 0, 0, 4092, 17, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 4, 3, 0, 3037, 4092, 0, 0, 1, 3, 100, 0, 4092, 3, 1, 2375, 1955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 168, 0, 3, 71, 0, 0, 0, 4052, 0, 0, 0, 4090, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16329, 16364, 16327, 16381, 16419, 17011, 15671, 16570] avg=16384.0000 max=17011 max_rank=5 min=15671 min_rank=6 imbalance_ratio=1.038269 max_load_reduction=114061 max_load_reduction_pct=0.870216 moved_to_echo=65671 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4069, 0, 0, 17, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 2418, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 991, 0, 0, 0, 0, 4095, 1686, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4096, 2139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 120, 0, 3337, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4095, 4096, 0, 2417, 1678, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 1679, 1616, 758, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2480, 0, 0, 1482, 0, 4096, 0, 0, 2, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3831, 4096, 0, 0, 0, 0, 80, 0, 4095, 0, 0, 2376, 1957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 13, 0, 0, 0, 4054, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16417, 17103, 15742, 16384, 16384, 16253, 16435, 16354] avg=16384.0000 max=17103 max_rank=1 min=15742 min_rank=2 imbalance_ratio=1.043884 max_load_reduction=113969 max_load_reduction_pct=0.869514 moved_to_echo=65426 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4036, 0, 0, 50, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 2417, 1719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 907, 0, 0, 0, 0, 4089, 846, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 890, 0, 3344, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2417, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 1679, 1615, 752, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2481, 0, 0, 1980, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3330, 4096, 0, 0, 0, 0, 264, 0, 4096, 0, 0, 2377, 1957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 1, 0, 0, 2, 0, 0, 0, 4056, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16415, 16173, 16522, 16384, 16379, 16749, 16120, 16330] avg=16384.0000 max=16749 max_rank=5 min=16120 min_rank=6 imbalance_ratio=1.022278 max_load_reduction=114323 max_load_reduction_pct=0.872215 moved_to_echo=65578 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3951, 0, 0, 27, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2417, 1719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2703, 0, 0, 0, 0, 4081, 422, 0, 0, 0, 4086, 0, 0, 0, 0, 2, 4096, 2139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 468, 0, 3339, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 2416, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 1680, 1615, 751, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2481, 0, 0, 887, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3632, 4096, 0, 0, 0, 0, 207, 0, 4096, 0, 0, 2377, 1957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 10, 0, 17, 0, 0, 0, 4056, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16306, 17529, 16085, 16384, 16389, 15662, 16369, 16348] avg=16384.0000 max=17529 max_rank=1 min=15662 min_rank=5 imbalance_ratio=1.069885 max_load_reduction=113543 max_load_reduction_pct=0.866264 moved_to_echo=64768 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4028, 0, 0, 8, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 2417, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 786, 0, 0, 0, 0, 4090, 543, 0, 0, 0, 4092, 0, 0, 0, 0, 5, 4096, 2139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1481, 0, 3341, 0, 0, 0, 0, 4091, 0, 1, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 2, 0, 0, 4096, 4096, 0, 2416, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 1678, 1615, 752, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2481, 0, 0, 1339, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4096, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3897, 4096, 0, 0, 1, 0, 152, 0, 4094, 0, 0, 2376, 1957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1, 4, 0, 61, 0, 0, 0, 4056, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16367, 15751, 17104, 16389, 16376, 16112, 16576, 16397] avg=16384.0000 max=17104 max_rank=2 min=15751 min_rank=1 imbalance_ratio=1.043945 max_load_reduction=113968 max_load_reduction_pct=0.869507 moved_to_echo=65461 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 35, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 2417, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1629, 0, 0, 0, 0, 4096, 895, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 195, 0, 3344, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2417, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 1679, 1615, 752, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2481, 0, 0, 1578, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3686, 4096, 0, 0, 0, 0, 174, 0, 4096, 0, 0, 2376, 1957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4056, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16459, 16951, 15826, 16384, 16374, 16348, 16385, 16345] avg=16384.0000 max=16951 max_rank=1 min=15826 min_rank=2 imbalance_ratio=1.034607 max_load_reduction=114121 max_load_reduction_pct=0.870674 moved_to_echo=65452 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4080, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2417, 1719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1131, 0, 0, 0, 0, 4096, 503, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1281, 0, 3344, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2416, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 1680, 1615, 752, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2481, 0, 0, 1294, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 4096, 0, 0, 0, 0, 11, 0, 4096, 0, 0, 2377, 1958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 24, 0, 0, 0, 4056, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16408, 16060, 16912, 16383, 16375, 16063, 16494, 16377] avg=16384.0000 max=16912 max_rank=2 min=16060 min_rank=1 imbalance_ratio=1.032227 max_load_reduction=114160 max_load_reduction_pct=0.870972 moved_to_echo=65309 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4033, 0, 0, 10, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2417, 1719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 609, 0, 0, 0, 0, 4096, 105, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 2139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 925, 0, 3344, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 2416, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 1680, 1615, 752, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2481, 0, 0, 2105, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3631, 4096, 0, 0, 0, 0, 865, 0, 4096, 0, 0, 2377, 1957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 1, 1, 0, 9, 0, 0, 0, 4056, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16371, 15141, 16554, 16383, 16376, 16874, 17022, 16351] avg=16384.0000 max=17022 max_rank=6 min=15141 min_rank=1 imbalance_ratio=1.038940 max_load_reduction=114050 max_load_reduction_pct=0.870132 moved_to_echo=66623 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[19968, 128, 0, 12928, 128, 32768, 128, 128, 128, 128, 0, 0, 128, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32384, 128, 0, 256, 0, 128, 32768, 128, 128, 128, 128, 128, 128, 0, 32768, 128, 19456, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 29056, 32768, 0, 0, 128, 128, 1920, 0, 32768, 128, 128, 19072, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 0, 256, 128, 128, 256, 0, 0, 0, 32512, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[9728, 128, 0, 0, 0, 32768, 6144, 0, 128, 0, 32768, 128, 0, 0, 0, 128, 32768, 17152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 128, 32768, 0, 32768, 32768, 128, 128, 0, 128, 13440, 12928, 6144, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 0, 128, 128, 128, 0, 128, 32768, 32768, 128, 19456, 13440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 6144, 0, 26752, 128, 128, 0, 128, 32768, 0, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 3320, 4088, 3, 4, 0, 0, 0, 4093, 10, 69, 17, 3, 4093, 4091, 3, 0, 3, 0, 1, 4094, 2, 3, 0, 3, 4093, 0, 0, 0, 4, 70, 4, 4, 0, 4, 3, 4093, 3, 1, 4092, 0, 161, 0, 2258, 4, 0, 0, 3718, 0, 0, 0, 0, 1533, 4092, 13, 1, 4093, 4065, 0, 0, 0, 3, 0, 3, 4092, 0, 0, 0, 3, 0, 0, 3, 3, 273, 4089, 0, 4092, 3972, 0, 0, 0, 0, 0, 3, 3, 2, 4093, 0, 0, 4079, 4, 0, 3, 4093, 5, 4043, 0, 1214, 0, 5, 0, 3, 3940, 0, 4043, 4082, 0, 4092, 0, 0, 0, 4093, 5, 0, 3, 4093, 4093, 4092, 0, 4091, 55, 0, 0, 4, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3192, 4096, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 247, 0, 0, 0, 2502, 0, 0, 0, 0, 3957, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 2351, 0, 0, 0, 0, 4091, 0, 4095, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 1, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4077, 4096, 0, 0, 0, 0, 0, 4096, 0, 25, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4071, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 4024, 0, 0, 4050, 0, 100, 8, 198, 0, 0, 0, 3324, 0, 0, 8, 0, 4071, 4095, 0, 0, 4087, 4068, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 67, 4096, 5, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4085, 0, 503, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 4090, 4096, 5, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3463, 4096, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 105, 0, 0, 1, 0, 0, 4081, 0, 0, 4088, 0, 1, 1, 144, 0, 0, 0, 3387, 0, 0, 1, 0, 4084, 4095, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 8, 4096, 2, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 4095, 0, 1105, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 4094, 4096, 1, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3623, 4096, 0, 0, 0, 0, 0, 4096, 0, 26, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 165, 0, 0, 0, 0, 0, 4041, 0, 0, 4089, 0, 60, 0, 639, 0, 0, 0, 3235, 0, 0, 0, 0, 4075, 4096, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4087, 0, 561, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3436, 4096, 0, 0, 0, 0, 0, 4095, 0, 3, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 386, 0, 0, 0, 0, 0, 4071, 0, 0, 4095, 0, 4, 0, 36, 0, 0, 0, 3721, 0, 0, 2, 0, 4019, 4096, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 705, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 4096, 0, 4096, 1, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3167, 4095, 0, 0, 0, 0, 1, 4096, 0, 38, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 405, 0, 0, 4, 0, 0, 4055, 0, 0, 4091, 0, 17, 6, 103, 0, 0, 0, 3424, 0, 0, 0, 0, 4072, 4092, 0, 0, 4092, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 1, 0, 0, 0, 0, 4096, 7, 4093, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 4079, 1, 1119, 3, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4095, 0, 1, 0, 4096, 4094, 4096, 3, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3760, 4096, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 4073, 0, 0, 4087, 0, 19, 0, 344, 0, 0, 0, 3634, 0, 0, 0, 0, 4058, 4096, 1, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4092, 0, 398, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 3320, 4088, 3, 4, 0, 0, 0, 4093, 10, 69, 17, 3, 3981, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4091, 3, 0, 3, 0, 1, 4094, 2, 3, 0, 3, 4093, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 70, 4, 4, 0, 4, 3, 4093, 3, 1, 4092, 0, 161, 0, 2258, 4, 4093, 3420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3718, 0, 0, 0, 0, 1533, 1100, 13, 1, 4093, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 4092, 0, 0, 0, 3, 0, 0, 3, 3, 273, 4089, 0, 4092, 2992, 953, 94, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 0, 0, 0, 0, 0, 3, 3, 2, 4093, 0, 0, 4079, 4, 0, 3, 3998, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4043, 0, 1214, 0, 5, 0, 3, 3940, 0, 4043, 3129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 5, 0, 3, 0, 4093, 0, 0, 4091, 55, 0, 0, 4, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16264, 16390, 18214, 14523, 16602, 16268, 16382, 16429] avg=16384.0000 max=18214 max_rank=2 min=14523 min_rank=3 imbalance_ratio=1.111694 max_load_reduction=112858 max_load_reduction_pct=0.861038 moved_to_echo=65681 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3192, 4096, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 3985, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 247, 0, 4096, 3422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2502, 0, 0, 0, 0, 3957, 1101, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2995, 956, 94, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4002, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2351, 0, 0, 0, 0, 4091, 0, 4095, 3140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16053, 16389, 15990, 15752, 16335, 16401, 17773, 16379] avg=16384.0000 max=17773 max_rank=6 min=15752 min_rank=3 imbalance_ratio=1.084778 max_load_reduction=113299 max_load_reduction_pct=0.864403 moved_to_echo=66888 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4077, 4096, 0, 0, 0, 0, 0, 4096, 0, 25, 0, 0, 3985, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4071, 0, 0, 16, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 4024, 0, 0, 4050, 0, 100, 8, 198, 0, 4096, 3422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3324, 0, 0, 8, 0, 4071, 1100, 0, 0, 4087, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 67, 4096, 5, 4096, 2995, 956, 94, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4002, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 503, 0, 0, 0, 0, 4096, 0, 4096, 3140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 0, 0, 0, 4090, 0, 5, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16953, 16379, 15974, 16658, 16408, 16402, 15920, 16378] avg=16384.0000 max=16953 max_rank=0 min=15920 min_rank=6 imbalance_ratio=1.034729 max_load_reduction=114119 max_load_reduction_pct=0.870659 moved_to_echo=65108 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 3167, 4095, 0, 0, 0, 0, 1, 4096, 0, 38, 0, 0, 3986, 675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 1, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 0, 0, 4, 0, 0, 4055, 0, 0, 4091, 0, 17, 6, 103, 0, 4096, 3421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3424, 0, 0, 0, 0, 4072, 1100, 0, 0, 4092, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 1, 0, 0, 0, 0, 4096, 7, 4093, 2992, 956, 95, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4001, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 1, 1119, 3, 0, 0, 0, 4096, 0, 4096, 3140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 1, 0, 0, 4094, 0, 3, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16059, 16382, 16198, 16777, 16340, 16403, 16534, 16379] avg=16384.0000 max=16777 max_rank=3 min=16059 min_rank=0 imbalance_ratio=1.023987 max_load_reduction=114295 max_load_reduction_pct=0.872002 moved_to_echo=65656 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3463, 4096, 0, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 3986, 675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 4096, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 1, 0, 0, 4081, 0, 0, 4088, 0, 1, 1, 144, 0, 4096, 3421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3387, 0, 0, 1, 0, 4084, 1095, 0, 0, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 8, 4096, 2, 4096, 3000, 960, 95, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4001, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1105, 0, 0, 0, 0, 4096, 0, 4096, 3136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 0, 0, 0, 0, 4094, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16329, 16391, 15938, 16756, 16357, 16399, 16528, 16374] avg=16384.0000 max=16756 max_rank=3 min=15938 min_rank=2 imbalance_ratio=1.022705 max_load_reduction=114316 max_load_reduction_pct=0.872162 moved_to_echo=65658 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3623, 4096, 0, 0, 0, 0, 0, 4096, 0, 26, 0, 0, 3986, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 0, 0, 4041, 0, 0, 4089, 0, 60, 0, 639, 0, 4096, 3422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3235, 0, 0, 0, 0, 4075, 1101, 0, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 13, 4096, 0, 4096, 2995, 956, 94, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4002, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 561, 0, 0, 0, 0, 4096, 0, 4096, 3140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16501, 16365, 16512, 16588, 16348, 16400, 15980, 16378] avg=16384.0000 max=16588 max_rank=3 min=15980 min_rank=6 imbalance_ratio=1.012451 max_load_reduction=114484 max_load_reduction_pct=0.873444 moved_to_echo=65106 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3760, 4096, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 3986, 674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 4073, 0, 0, 4087, 0, 19, 0, 344, 0, 4096, 3422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3634, 0, 0, 0, 0, 4058, 1101, 1, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 4096, 2995, 956, 94, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4002, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 398, 0, 0, 0, 0, 4096, 0, 4096, 3140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16622, 16388, 16137, 16984, 16342, 16400, 15822, 16377] avg=16384.0000 max=16984 max_rank=3 min=15822 min_rank=6 imbalance_ratio=1.036621 max_load_reduction=114088 max_load_reduction_pct=0.870422 moved_to_echo=64941 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3436, 4096, 0, 0, 0, 0, 0, 4095, 0, 3, 0, 0, 3986, 675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 386, 0, 0, 0, 0, 0, 4071, 0, 0, 4095, 0, 4, 0, 36, 0, 4096, 3421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3721, 0, 0, 2, 0, 4019, 1101, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 4096, 2995, 956, 95, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4001, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 705, 0, 0, 0, 0, 4096, 0, 4096, 3140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4095, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16291, 16388, 16109, 17034, 16340, 16399, 16133, 16378] avg=16384.0000 max=17034 max_rank=3 min=16109 min_rank=2 imbalance_ratio=1.039673 max_load_reduction=114038 max_load_reduction_pct=0.870041 moved_to_echo=65250 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 0, 32768, 0, 128, 32768, 128, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 0, 28160, 32768, 128, 128, 0, 0, 128, 32768, 128, 256, 128, 128, 32000, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 0, 128, 0, 128, 32768, 128, 128, 128, 128, 32768, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 32768, 128, 8064, 128, 128, 0, 128, 32640, 0, 32768, 25216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 27008, 0, 0, 128, 0, 29952, 8832, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 128, 128, 128, 32768, 0, 0, 32768, 128, 0, 128, 32128, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 1408, 128, 128, 128, 128, 128, 32640, 128, 128, 32768, 0, 384, 128, 4096, 128, 32768, 27392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 32768, 0, 0, 128, 128, 128, 128, 128, 128, 384, 32768, 128, 32768, 24064, 7680, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4000, 104, 0, 0, 0, 101, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 4093, 0, 0, 0, 0, 0, 4079, 3694, 0, 0, 4096, 0, 0, 3988, 4095, 4096, 0, 0, 0, 0, 0, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 4096, 0, 7, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 524, 4089, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 57, 0, 0, 0, 4096, 4096, 0, 4, 4096, 0, 0, 0, 0, 4096, 6, 0, 0, 4096, 0, 0, 0, 4096, 0, 4050, 4047, 4096, 4096, 4096, 0, 4049, 0, 0, 4096, 0, 146, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4094, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 3628, 0, 0, 4096, 0, 0, 4084, 4096, 4096, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 501, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4089, 4096, 4096, 4096, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4092, 3, 13, 4089, 4, 3, 4089, 6, 0, 0, 3, 4, 4, 0, 3, 4089, 10, 0, 18, 18, 1, 4092, 4076, 3, 0, 4, 6, 0, 4093, 3375, 0, 0, 4090, 0, 0, 4087, 4093, 4092, 0, 2, 0, 3, 0, 3, 3311, 0, 0, 4, 3, 0, 3, 0, 33, 3, 2, 4, 4093, 4, 0, 4091, 0, 2, 4, 4092, 4, 0, 0, 1, 0, 1406, 4093, 14, 3, 0, 0, 5, 10, 0, 0, 3, 3, 0, 3, 4092, 4093, 2, 0, 0, 0, 0, 0, 0, 3, 4082, 4092, 0, 0, 4091, 3, 4, 0, 1, 4069, 0, 3, 0, 4092, 2, 0, 6, 4079, 0, 4093, 4086, 4093, 4092, 4092, 0, 4063, 3, 4, 4092, 0, 70, 3, 2, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 0, 4096, 0, 0, 3984, 28, 0, 0, 0, 111, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4074, 3445, 0, 0, 4096, 0, 0, 4005, 4096, 4096, 0, 0, 0, 0, 0, 0, 3866, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 917, 4096, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 26, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4065, 4079, 4096, 4096, 4096, 0, 4081, 0, 0, 4096, 0, 69, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4053, 1, 0, 0, 0, 40, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4091, 3683, 0, 0, 4096, 0, 0, 4035, 4095, 4096, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 507, 4095, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 1, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4094, 4073, 4096, 4096, 4096, 0, 4089, 0, 0, 4095, 0, 6, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4094, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 50, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 0, 0, 0, 4093, 3320, 0, 0, 4096, 0, 0, 4070, 4096, 4096, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 805, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 4093, 4095, 4096, 4096, 0, 4096, 0, 0, 4096, 1, 11, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4040, 11, 0, 0, 0, 44, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 4089, 3283, 0, 3, 4096, 0, 0, 3966, 4092, 4096, 0, 0, 0, 0, 0, 0, 3927, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1097, 4090, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 4096, 0, 6, 4096, 0, 0, 2, 0, 4096, 4, 0, 0, 4096, 0, 0, 0, 4096, 0, 4086, 4087, 4092, 4096, 4096, 0, 4083, 0, 0, 4092, 3, 32, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 0, 4096, 0, 0, 4067, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 3778, 0, 0, 4096, 0, 0, 4050, 4096, 4096, 0, 0, 0, 0, 0, 0, 3643, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 794, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 4089, 4096, 4096, 4096, 0, 4093, 0, 0, 4096, 0, 32, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4092, 3, 13, 4065, 4, 3, 4089, 6, 0, 0, 3, 4, 4, 0, 3, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 18, 18, 1, 4092, 4076, 3, 0, 4, 6, 0, 4093, 3375, 0, 0, 534, 24, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4087, 4093, 304, 0, 2, 0, 3, 0, 3, 3311, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 33, 3, 2, 4, 4093, 4, 0, 4091, 0, 2, 4, 4092, 3254, 826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 1406, 4093, 14, 3, 0, 0, 5, 10, 0, 0, 3, 4092, 4092, 3267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 4082, 4093, 2, 0, 0, 0, 0, 0, 0, 3, 4082, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3, 4, 0, 1, 4069, 0, 3, 0, 4092, 2, 0, 6, 4079, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4086, 0, 0, 0, 0, 4063, 3, 4, 4089, 0, 70, 3, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 16266, 15897, 16414, 16990, 16360, 16351, 16416] avg=16384.0000 max=16990 max_rank=4 min=15897 min_rank=2 imbalance_ratio=1.036987 max_load_reduction=114082 max_load_reduction_pct=0.870377 moved_to_echo=66117 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4072, 0, 0, 4000, 104, 0, 0, 0, 101, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4093, 0, 0, 0, 0, 0, 4079, 3694, 0, 0, 535, 24, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3988, 4095, 304, 0, 0, 0, 0, 0, 0, 3880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 4096, 0, 7, 4096, 0, 0, 0, 4096, 3257, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 4089, 0, 0, 0, 0, 0, 31, 0, 0, 0, 4096, 4096, 3269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4090, 0, 0, 0, 0, 57, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 4096, 6, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 4047, 0, 0, 0, 0, 4049, 0, 0, 4093, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16469, 16534, 16363, 16396, 16105, 16425, 16395, 16385] avg=16384.0000 max=16534 max_rank=1 min=16105 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=114538 max_load_reduction_pct=0.873856 moved_to_echo=65310 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 4072, 0, 0, 4095, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 3628, 0, 0, 535, 24, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 4096, 304, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3257, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 501, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 4094, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16365, 16488, 16658, 16372, 16058, 16374, 16385, 16372] avg=16384.0000 max=16658 max_rank=2 min=16058 min_rank=4 imbalance_ratio=1.016724 max_load_reduction=114414 max_load_reduction_pct=0.872910 moved_to_echo=65189 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4070, 0, 0, 4053, 1, 0, 0, 0, 40, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 4091, 3683, 0, 0, 539, 26, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4035, 4095, 296, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 4096, 3261, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 4095, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4096, 4096, 3268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4095, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 1, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4073, 0, 0, 0, 0, 4089, 0, 0, 4084, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16356, 16550, 16609, 16380, 16064, 16378, 16389, 16346] avg=16384.0000 max=16609 max_rank=2 min=16064 min_rank=4 imbalance_ratio=1.013733 max_load_reduction=114463 max_load_reduction_pct=0.873283 moved_to_echo=65177 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4072, 0, 0, 4040, 11, 0, 0, 0, 44, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 0, 4089, 3283, 0, 3, 535, 24, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3966, 4092, 304, 0, 0, 0, 0, 0, 0, 3927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 5, 4096, 0, 0, 0, 4096, 3257, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 4090, 0, 0, 0, 0, 0, 5, 1, 0, 0, 4096, 4096, 3265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4091, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 0, 0, 2, 0, 4096, 4, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4087, 0, 0, 0, 0, 4083, 0, 0, 4089, 3, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16359, 16138, 16385, 16383, 16650, 16380, 16397, 16380] avg=16384.0000 max=16650 max_rank=4 min=16138 min_rank=1 imbalance_ratio=1.016235 max_load_reduction=114422 max_load_reduction_pct=0.872971 moved_to_echo=65807 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 4072, 0, 0, 3984, 28, 0, 0, 0, 111, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4074, 3445, 0, 0, 535, 24, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4005, 4096, 304, 0, 0, 0, 0, 0, 0, 3866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3257, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 917, 4096, 0, 0, 0, 0, 0, 17, 0, 0, 0, 4096, 4096, 3268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 26, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4079, 0, 0, 0, 0, 4081, 0, 0, 4093, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16282, 16367, 16375, 16490, 16400, 16385, 16387] avg=16384.0000 max=16490 max_rank=4 min=16282 min_rank=1 imbalance_ratio=1.006470 max_load_reduction=114582 max_load_reduction_pct=0.874191 moved_to_echo=65662 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 4072, 0, 0, 4067, 0, 0, 0, 0, 23, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 3778, 0, 0, 535, 24, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4050, 4096, 304, 0, 0, 0, 0, 0, 0, 3643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3257, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 794, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 4096, 3269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4089, 0, 0, 0, 0, 4093, 0, 0, 4093, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16353, 16636, 16189, 16379, 16355, 16379, 16385, 16396] avg=16384.0000 max=16636 max_rank=1 min=16189 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=114436 max_load_reduction_pct=0.873077 moved_to_echo=65515 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 4072, 0, 0, 4094, 0, 0, 0, 0, 50, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 0, 0, 0, 4093, 3320, 0, 0, 535, 24, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4070, 4096, 304, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3257, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 805, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 4096, 0, 0, 4093, 1, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16406, 16178, 16604, 16373, 16360, 16376, 16385, 16390] avg=16384.0000 max=16604 max_rank=2 min=16178 min_rank=1 imbalance_ratio=1.013428 max_load_reduction=114468 max_load_reduction_pct=0.873322 moved_to_echo=65511 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 32640, 128, 128, 32512, 256, 0, 0, 128, 384, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 128, 32768, 32768, 128, 0, 0, 0, 128, 0, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 0, 128, 0, 6656, 32768, 128, 128, 0, 0, 128, 128, 128, 128, 128, 32768, 32768, 26240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 32768, 128, 128, 0, 32768, 128, 0, 128, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 128, 32768, 128, 128, 32768, 0, 128, 128, 32768, 26112, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 0, 0, 32384, 32768, 2432, 0, 128, 0, 128, 0, 128, 30848, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 32768, 0, 0, 0, 0, 32768, 128, 128, 32768, 128, 384, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 128, 128, 32768, 32768, 128, 0, 128, 128, 0, 32768, 28288, 0, 128, 4352, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[3, 19, 0, 1, 3191, 0, 0, 3, 50, 2, 4, 17, 4089, 3, 4, 0, 4091, 1, 0, 4093, 4093, 29, 0, 0, 4, 1, 2, 4086, 4066, 1, 9, 4093, 4, 8, 1, 4, 1, 0, 0, 0, 4093, 0, 4, 3, 4, 0, 22, 4045, 4092, 4, 5, 3, 4092, 0, 3, 0, 0, 4089, 3, 3, 4091, 51, 0, 3817, 0, 0, 0, 3, 3, 0, 4, 4092, 4084, 0, 4077, 2623, 0, 2, 0, 3, 3, 0, 4091, 3, 4092, 0, 0, 3, 124, 2, 4019, 4092, 1629, 4092, 6, 4093, 1, 1, 4, 0, 3996, 982, 0, 3, 0, 0, 4091, 3, 0, 0, 3, 3, 4092, 4, 4092, 5, 0, 0, 0, 0, 4, 4091, 15, 4090, 3, 0, 0, 4052] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3019, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 7, 3, 4088, 4096, 0, 13, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4094, 3689, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 7, 0, 4090, 4094, 1457, 4096, 9, 4094, 0, 0, 0, 1, 4089, 12, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2837, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 29, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 3199, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 0, 4086, 4096, 2119, 4096, 8, 4096, 0, 0, 0, 0, 4095, 11, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2595, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 54, 64, 4045, 4096, 0, 83, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 32, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 3664, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 1789, 4096, 4, 4079, 0, 0, 0, 0, 4038, 29, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3688, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 1, 4096, 4095, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 6, 0, 4092, 0, 0, 0, 0, 9, 25, 4071, 4096, 0, 26, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 4096, 18, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 4089, 3807, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 12, 0, 4095, 4088, 632, 4096, 51, 4084, 0, 0, 0, 4, 4069, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1, 2, 0, 0, 0, 4096, 0, 4096, 0, 0, 4, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3675, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 19, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 110, 56, 4039, 4096, 0, 215, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 15, 0, 0, 0, 0, 0, 4096, 4096, 0, 4092, 3954, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4094, 4087, 345, 4096, 16, 4078, 0, 0, 0, 1, 3952, 9, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 7, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3613, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 4096, 4096, 0, 29, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 4044, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 4093, 4096, 487, 4096, 11, 4095, 0, 0, 0, 0, 4094, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3573, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 7, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 28, 4074, 4096, 0, 123, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4092, 3240, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 1249, 4096, 4, 4094, 0, 0, 0, 0, 4080, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 19, 0, 1, 3191, 0, 0, 3, 50, 2, 4, 17, 4089, 3, 4, 0, 4093, 4093, 807, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 4090, 29, 0, 0, 4, 1, 2, 4086, 4066, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, 1, 4, 1, 0, 0, 0, 4093, 0, 4, 3, 4, 0, 22, 4045, 4092, 3285, 783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 4, 5, 3, 0, 0, 3, 0, 0, 4089, 3, 3, 4091, 51, 0, 3817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 4, 4092, 4084, 0, 4077, 2623, 0, 2, 0, 3, 524, 38, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 2871, 0, 0, 3, 124, 2, 4019, 4092, 1629, 0, 6, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4, 0, 3996, 982, 0, 3, 0, 0, 4091, 3, 0, 0, 3, 3, 4089, 3308, 697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 4092, 5, 0, 0, 0, 0, 4, 4091, 15, 4090, 3, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16380, 16349, 16123, 15456, 16845, 17181, 16359] avg=16384.0000 max=17181 max_rank=6 min=15456 min_rank=4 imbalance_ratio=1.048645 max_load_reduction=113891 max_load_reduction_pct=0.868919 moved_to_echo=65841 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3019, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 7, 3, 4088, 4096, 3287, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 13, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4094, 3689, 0, 0, 0, 0, 529, 40, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2865, 0, 0, 0, 7, 0, 4090, 4094, 1457, 0, 9, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 12, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3311, 701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16118, 16377, 16362, 16357, 16551, 16616, 16306, 16385] avg=16384.0000 max=16616 max_rank=5 min=16118 min_rank=0 imbalance_ratio=1.014160 max_load_reduction=114456 max_load_reduction_pct=0.873230 moved_to_echo=65858 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2837, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3288, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 29, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 3199, 0, 0, 0, 0, 525, 39, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2873, 0, 1, 0, 0, 0, 4086, 4096, 2119, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 11, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 3312, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15933, 16380, 16360, 16374, 16054, 17279, 16305, 16387] avg=16384.0000 max=17279 max_rank=5 min=15933 min_rank=0 imbalance_ratio=1.054626 max_load_reduction=113793 max_load_reduction_pct=0.868172 moved_to_echo=66025 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3573, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 7, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 28, 4074, 4096, 3287, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 123, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4092, 3240, 0, 0, 0, 0, 525, 39, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2873, 0, 0, 0, 0, 0, 4096, 4096, 1249, 0, 4, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 3312, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16672, 16388, 16369, 16469, 16091, 16412, 16284, 16387] avg=16384.0000 max=16672 max_rank=0 min=16091 min_rank=4 imbalance_ratio=1.017578 max_load_reduction=114400 max_load_reduction_pct=0.872803 moved_to_echo=65174 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3688, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 4092, 0, 0, 0, 0, 9, 25, 4071, 4096, 3287, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 26, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 18, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 4089, 3807, 0, 0, 0, 0, 524, 39, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2872, 0, 0, 0, 12, 0, 4095, 4088, 632, 0, 51, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4069, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 3311, 697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 1, 2, 0, 0, 0, 4096, 0, 4096, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16785, 16381, 16372, 16386, 16650, 15834, 16270, 16394] avg=16384.0000 max=16785 max_rank=0 min=15834 min_rank=5 imbalance_ratio=1.024475 max_load_reduction=114287 max_load_reduction_pct=0.871941 moved_to_echo=65148 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3675, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 19, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 110, 56, 4039, 4096, 3287, 784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 215, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 4096, 4096, 0, 4092, 3954, 0, 0, 0, 0, 525, 39, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2873, 0, 0, 0, 0, 0, 4094, 4087, 345, 0, 16, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3952, 9, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 3312, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 7, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16783, 16400, 16466, 16560, 16820, 15493, 16161, 16389] avg=16384.0000 max=16820 max_rank=4 min=15493 min_rank=5 imbalance_ratio=1.026611 max_load_reduction=114252 max_load_reduction_pct=0.871674 moved_to_echo=64863 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2595, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 4, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 54, 64, 4045, 4096, 3287, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 83, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 3664, 0, 0, 0, 0, 525, 39, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2873, 0, 0, 0, 0, 0, 4096, 4096, 1789, 0, 4, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 29, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 3311, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15692, 16385, 16427, 16428, 16551, 16937, 16265, 16387] avg=16384.0000 max=16937 max_rank=5 min=15692 min_rank=0 imbalance_ratio=1.033752 max_load_reduction=114135 max_load_reduction_pct=0.870781 moved_to_echo=66140 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3613, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 4096, 4096, 3287, 785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 29, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 4044, 0, 0, 0, 0, 525, 39, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2873, 0, 0, 0, 1, 0, 4093, 4096, 487, 0, 11, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 5, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3311, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16710, 16381, 16367, 16375, 16899, 15656, 16300, 16384] avg=16384.0000 max=16899 max_rank=4 min=15656 min_rank=5 imbalance_ratio=1.031433 max_load_reduction=114173 max_load_reduction_pct=0.871071 moved_to_echo=65239 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 32512, 1152, 0, 128, 0, 0, 32768, 128, 0, 0, 128, 128, 32768, 26496, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32512, 128, 640, 128, 0, 0, 128, 0, 0, 32768, 128, 128, 32768, 128, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 0, 0, 128, 32768, 128, 32768, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 26240, 0, 0, 128, 128, 128, 128, 128, 32768, 128, 128, 0, 32768, 32768, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 0, 32768, 128, 0, 0, 128, 128, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 0, 32768, 0, 128, 128, 128, 256, 256, 32640, 32768, 26368, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 23040, 0, 128, 128, 256, 128, 32768, 32768, 9728, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 32768, 32768, 0, 32768, 28288, 0, 128, 0, 128, 4224, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 33, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2518, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1233, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1941, 4080, 0, 0, 0, 0, 0, 4096, 4096, 0, 2482, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[4, 7, 0, 4, 4093, 0, 0, 0, 189, 1, 0, 4088, 4091, 4052, 3, 4091, 317, 4, 105, 5, 1, 0, 2, 4092, 3961, 0, 13, 0, 0, 4091, 18, 31, 0, 4, 3, 0, 4090, 4, 3, 0, 0, 6, 3, 3, 3, 0, 4064, 2, 4066, 4046, 4, 3, 4, 4, 4093, 15, 1, 0, 4091, 4, 4, 4, 0, 128, 4, 4092, 0, 3, 3946, 0, 4, 0, 4074, 58, 3743, 2, 3, 0, 0, 3, 4092, 4092, 0, 4069, 0, 0, 4, 4084, 0, 0, 3, 1, 4, 4092, 10, 4077, 4087, 2, 1, 220, 0, 4093, 4089, 0, 48, 4092, 0, 3, 3, 0, 3954, 0, 0, 0, 0, 0, 0, 3873, 4092, 5, 4, 4, 3, 0, 27, 4092, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 2, 0, 1, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3085, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 66, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1352, 4093, 0, 0, 0, 0, 0, 4096, 4096, 0, 3701, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4085, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4079, 0, 0, 5, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 262, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2421, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 864, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 633, 4093, 0, 0, 0, 0, 0, 4096, 4096, 0, 4054, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 20, 0, 4089, 4096, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4053, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2474, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1255, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1083, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 3406, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 22, 0, 4090, 4096, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4059, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 14, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3358, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 872, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 205, 4080, 0, 0, 0, 0, 0, 4096, 4096, 0, 3767, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 2, 0, 4094, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 6, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3115, 0, 4096, 4096, 0, 0, 0, 0, 4095, 1, 0, 0, 4096, 0, 0, 0, 0, 592, 0, 4093, 0, 0, 4096, 0, 0, 0, 4096, 838, 4086, 0, 0, 0, 0, 0, 4096, 4095, 1, 3660, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 9, 0, 4087, 4096, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 213, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3243, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 406, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 723, 4088, 0, 0, 0, 0, 0, 4096, 4096, 0, 3625, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 3, 0, 4090, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[4, 7, 0, 4, 4093, 0, 0, 0, 189, 1, 0, 4088, 0, 4052, 3, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 317, 4, 105, 5, 1, 0, 2, 4092, 3961, 0, 13, 0, 0, 4091, 18, 31, 2495, 782, 670, 17, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 4090, 4, 3, 0, 0, 6, 3, 3, 3, 0, 4064, 2, 4092, 4091, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4046, 4, 3, 4, 4, 3423, 15, 1, 0, 4091, 4, 4, 4, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3310, 0, 3, 3946, 0, 4, 0, 4074, 58, 3743, 2, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4069, 0, 0, 4, 4084, 0, 0, 3, 1, 4, 502, 10, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 2, 1, 220, 0, 4093, 4074, 0, 48, 0, 0, 3, 3, 0, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3873, 4092, 5, 4, 4, 3, 0, 27, 4092, 5, 0, 3035, 1095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16515, 16619, 17425, 15797, 15150, 16846, 16485, 16235] avg=16384.0000 max=17425 max_rank=2 min=15150 min_rank=4 imbalance_ratio=1.063538 max_load_reduction=113647 max_load_reduction_pct=0.867058 moved_to_echo=64716 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4079, 0, 0, 5, 0, 0, 0, 4096, 0, 4096, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 262, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 7, 0, 2497, 783, 671, 17, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2421, 0, 4096, 4096, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3425, 0, 0, 0, 4096, 0, 0, 0, 0, 864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3313, 0, 0, 4096, 0, 0, 0, 4096, 633, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4054, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 502, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 20, 0, 4089, 4081, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3037, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16355, 16540, 15769, 16577, 16231, 16845, 16376, 16379] avg=16384.0000 max=16845 max_rank=5 min=15769 min_rank=2 imbalance_ratio=1.028137 max_load_reduction=114227 max_load_reduction_pct=0.871483 moved_to_echo=65831 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 0, 4096, 0, 0, 2497, 783, 671, 17, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2518, 0, 4096, 4096, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3425, 0, 0, 0, 4096, 0, 0, 0, 0, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3313, 0, 0, 4096, 0, 0, 0, 4096, 1941, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2482, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 502, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3037, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16367, 16305, 15864, 16946, 17526, 15272, 16370, 16422] avg=16384.0000 max=17526 max_rank=4 min=15272 min_rank=5 imbalance_ratio=1.069702 max_load_reduction=113546 max_load_reduction_pct=0.866287 moved_to_echo=65590 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 2497, 783, 671, 17, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2474, 0, 4096, 4096, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3425, 0, 0, 0, 4096, 0, 0, 0, 0, 1255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3313, 0, 0, 4096, 0, 0, 0, 4096, 1083, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3406, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 502, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 22, 0, 4090, 4081, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3037, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16367, 16275, 15821, 16968, 16683, 16196, 16377, 16385] avg=16384.0000 max=16968 max_rank=3 min=15821 min_rank=2 imbalance_ratio=1.035645 max_load_reduction=114104 max_load_reduction_pct=0.870544 moved_to_echo=65641 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 2497, 783, 671, 17, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3243, 0, 4096, 4096, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3425, 0, 0, 0, 4096, 0, 0, 0, 0, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3313, 0, 0, 4096, 0, 0, 0, 4096, 723, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3625, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 502, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 4090, 4081, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3037, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16367, 16484, 16590, 16119, 16316, 16415, 16365, 16416] avg=16384.0000 max=16590 max_rank=2 min=16119 min_rank=3 imbalance_ratio=1.012573 max_load_reduction=114482 max_load_reduction_pct=0.873428 moved_to_echo=65512 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 6, 0, 0, 0, 4096, 0, 4096, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4, 0, 2497, 782, 671, 17, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3115, 0, 4096, 4096, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3424, 1, 0, 0, 4096, 0, 0, 0, 0, 592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3311, 0, 0, 4096, 0, 0, 0, 4096, 838, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 3660, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 502, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 4087, 4081, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3037, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16369, 16278, 16466, 16305, 16427, 16454, 16363, 16410] avg=16384.0000 max=16466 max_rank=2 min=16278 min_rank=1 imbalance_ratio=1.005005 max_load_reduction=114606 max_load_reduction_pct=0.874374 moved_to_echo=65654 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 1, 2501, 784, 674, 19, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3358, 0, 4096, 4096, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3421, 0, 0, 0, 4096, 0, 0, 0, 0, 872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3312, 0, 0, 4096, 0, 0, 0, 4096, 205, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3767, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 497, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4094, 4079, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3037, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16365, 16299, 16705, 16581, 15789, 16553, 16366, 16414] avg=16384.0000 max=16705 max_rank=2 min=15789 min_rank=4 imbalance_ratio=1.019592 max_load_reduction=114367 max_load_reduction_pct=0.872551 moved_to_echo=65122 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 4095, 0, 0, 2497, 783, 671, 17, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3085, 0, 4096, 4096, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3425, 0, 0, 0, 4096, 0, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3313, 0, 0, 4096, 0, 0, 0, 4096, 1352, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3701, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 502, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 4081, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3037, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16367, 16272, 16432, 15779, 16950, 16491, 16370, 16411] avg=16384.0000 max=16950 max_rank=4 min=15779 min_rank=3 imbalance_ratio=1.034546 max_load_reduction=114122 max_load_reduction_pct=0.870682 moved_to_echo=66222 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 384, 0, 32768, 32640, 0, 128, 0, 0, 128, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32768, 128, 28800, 0, 128, 128, 32768, 0, 0, 128, 128, 128, 4096, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 128, 256, 128, 0, 32768, 0, 32768, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 32768, 128, 128, 128, 128, 27520, 128, 128, 0, 32768, 128, 128, 128, 0, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 26624, 0, 128, 32640, 0, 128, 0, 32768, 6912, 32384, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 0, 0, 0, 32512, 32768, 128, 128, 128, 128, 0, 128, 32768, 128, 0, 24320, 8832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 0, 32768, 128, 128, 0, 0, 128, 128, 128, 128, 128, 24320, 128, 32768, 32768, 8576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[896, 128, 128, 128, 128, 0, 128, 32768, 32640, 0, 128, 128, 0, 32768, 128, 128, 20096, 6272, 5376, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4091, 0, 0, 8, 4084, 4092, 0, 4092, 0, 4032, 4, 0, 4091, 4, 3852, 3, 4088, 3, 0, 0, 0, 2, 0, 4, 4, 4093, 4089, 0, 4, 4091, 4087, 4090, 0, 1740, 5, 3, 4, 0, 0, 3, 4, 4093, 48, 23, 0, 12, 3, 0, 4039, 0, 0, 4, 4089, 4, 3829, 151, 0, 0, 4090, 4092, 5, 4, 4, 0, 0, 7, 3993, 1, 4, 24, 4080, 3845, 4, 0, 0, 3, 5, 0, 6, 3, 354, 9, 1, 3, 14, 108, 1, 0, 4073, 4070, 0, 0, 0, 1, 0, 3, 0, 0, 3, 4091, 0, 4075, 4092, 0, 0, 104, 4092, 3, 4, 0, 1, 0, 3, 3, 0, 5, 2662, 0, 0, 0, 0, 4092, 4091, 3977, 3, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4083, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 17, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 1, 4096, 2652, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4091, 0, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1474, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4089, 0, 4094, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2904, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4093, 0, 4095, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 4096, 4096, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 1, 4096, 2920, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4092, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4096, 4092, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1205, 0, 1, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4095, 0, 4095, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 1, 4096, 2485, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1615, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4095, 0, 4095, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 832, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 2842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1393, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 4079, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2387, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 4084, 4092, 0, 45, 0, 4032, 4, 0, 0, 4, 3852, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 0, 0, 0, 2, 0, 4, 4, 0, 4089, 0, 4, 2, 4087, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1740, 5, 3, 4, 0, 0, 3, 4, 4093, 48, 23, 0, 12, 3, 0, 4093, 4091, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 4, 4089, 4, 3829, 151, 0, 0, 4090, 16, 5, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 3993, 1, 4, 24, 4080, 3845, 4, 0, 0, 3, 5, 0, 6, 3, 3806, 1759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 354, 9, 1, 3, 14, 108, 1, 0, 4073, 4070, 0, 0, 0, 1, 0, 3, 4089, 3776, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4081, 0, 4075, 4092, 0, 0, 104, 4092, 3, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 5, 2662, 0, 0, 0, 0, 4092, 4091, 3977, 3, 2, 0, 0, 2288, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16124, 16373, 14437, 16235, 17540, 16772, 16455, 17136] avg=16384.0000 max=17540 max_rank=4 min=14437 min_rank=2 imbalance_ratio=1.070557 max_load_reduction=113532 max_load_reduction_pct=0.866180 moved_to_echo=67903 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4083, 0, 44, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 4096, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 17, 0, 4096, 0, 4096, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 1, 4096, 2652, 0, 5, 1, 0, 0, 0, 0, 0, 3810, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 4094, 0, 0, 0, 0, 0, 0, 4094, 3780, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 4091, 0, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1474, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 2291, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16415, 16386, 16686, 16417, 16423, 16305, 16377, 16063] avg=16384.0000 max=16686 max_rank=2 min=16063 min_rank=7 imbalance_ratio=1.018433 max_load_reduction=114386 max_load_reduction_pct=0.872696 moved_to_echo=65168 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4095, 0, 37, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 1, 4096, 2485, 0, 1, 0, 0, 0, 0, 0, 0, 3815, 1765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 0, 1, 0, 4096, 3779, 276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 4096, 4095, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1615, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 2293, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16420, 16384, 16694, 16390, 16262, 16342, 16371, 16209] avg=16384.0000 max=16694 max_rank=2 min=16209 min_rank=7 imbalance_ratio=1.018921 max_load_reduction=114378 max_load_reduction_pct=0.872635 moved_to_echo=65184 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4089, 0, 44, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1212, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 1760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 3780, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2904, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 2290, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16421, 16387, 16688, 16399, 14974, 16336, 16375, 17492] avg=16384.0000 max=17492 max_rank=7 min=14974 min_rank=4 imbalance_ratio=1.067627 max_load_reduction=113580 max_load_reduction_pct=0.866547 moved_to_echo=65177 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4095, 0, 45, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 4096, 0, 4096, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3274, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 1760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4096, 0, 0, 0, 0, 0, 0, 4094, 3780, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 832, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 2290, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16428, 16386, 16691, 16422, 17036, 16314, 16375, 15420] avg=16384.0000 max=17036 max_rank=4 min=15420 min_rank=7 imbalance_ratio=1.039795 max_load_reduction=114036 max_load_reduction_pct=0.870026 moved_to_echo=65145 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4093, 0, 45, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4095, 4096, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 4096, 0, 4096, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 1, 4096, 2920, 0, 4, 0, 0, 0, 0, 0, 0, 3810, 1760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4092, 0, 0, 0, 0, 4, 0, 4095, 3779, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 4092, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1205, 0, 1, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 2290, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16426, 16385, 16666, 16407, 16690, 16334, 16370, 15794] avg=16384.0000 max=16690 max_rank=4 min=15794 min_rank=7 imbalance_ratio=1.018677 max_load_reduction=114382 max_load_reduction_pct=0.872665 moved_to_echo=65188 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4079, 0, 44, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1778, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 4094, 3780, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2387, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 2291, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 16386, 16649, 16403, 15541, 16332, 16374, 16976] avg=16384.0000 max=16976 max_rank=7 min=15541 min_rank=4 imbalance_ratio=1.036133 max_load_reduction=114096 max_load_reduction_pct=0.870483 moved_to_echo=65223 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 4096, 0, 44, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 2842, 0, 0, 0, 0, 0, 0, 0, 0, 3810, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 3779, 270, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1393, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 2291, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16427, 16385, 16561, 16399, 16606, 16337, 16375, 15982] avg=16384.0000 max=16606 max_rank=4 min=15982 min_rank=7 imbalance_ratio=1.013550 max_load_reduction=114466 max_load_reduction_pct=0.873306 moved_to_echo=65300 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 30208, 128, 128, 128, 0, 0, 128, 128, 32768, 128, 128, 0, 128, 128, 0, 32768, 32768, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[384, 128, 128, 128, 128, 128, 128, 0, 32768, 32768, 0, 0, 0, 128, 128, 128, 32768, 30336, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 14592, 0, 128, 0, 0, 32768, 32768, 32768, 128, 128, 0, 0, 18432, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 0, 128, 32768, 32768, 0, 384, 0, 32768, 128, 0, 0, 128, 32640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 0, 128, 128, 32768, 128, 32512, 256, 0, 0, 32768, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 0, 128, 128, 0, 128, 128, 0, 32768, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132992 observed_loss=-1920 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 32768, 21120, 128, 128, 128, 128, 128, 0, 128, 128, 30592, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 128, 32768, 0, 32768, 32768, 0, 0, 128, 32768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132992 slack_slots=129152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 347, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1439, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 548, 4096, 4095, 0, 1846, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 23, 0, 0, 4096, 0, 4095, 8, 0, 0, 0, 4095, 3982, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[2, 4, 4084, 4, 0, 5, 14, 0, 4086, 4, 4, 0, 10, 3, 57, 6, 4088, 4, 4092, 0, 0, 0, 4, 4091, 0, 1, 3984, 4, 5, 1373, 4050, 3, 4, 6, 0, 0, 4088, 0, 12, 2, 1271, 4080, 7, 0, 4091, 5, 0, 0, 2, 0, 0, 0, 4, 4082, 0, 4092, 4079, 0, 4, 0, 875, 4092, 4092, 0, 575, 0, 0, 0, 1, 4, 0, 4, 3, 4041, 4, 3, 5, 10, 4092, 4091, 2, 2, 4092, 30, 4090, 3, 0, 4092, 4072, 4067, 0, 0, 0, 0, 0, 4092, 29, 3, 0, 4035, 37, 0, 103, 3792, 0, 4089, 25, 0, 0, 3, 4092, 4087, 0, 4, 0, 0, 0, 5, 52, 0, 6, 4, 4070, 300, 4088, 0, 2, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 550, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2908, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 439, 4096, 4096, 0, 293, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 6, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 4096, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4079, 0, 0, 984, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 1175, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4, 574, 4096, 4090, 0, 1378, 3, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4092, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4095, 4090, 4096, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 4096, 49, 0, 0, 4096, 0, 4093, 1, 0, 0, 0, 4092, 4044, 0, 0, 0, 0, 11, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4040, 0, 0, 879, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2199, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 61, 4096, 4086, 0, 745, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4087, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4081, 4096, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 4096, 339, 0, 0, 4096, 0, 4092, 7, 0, 0, 0, 4089, 4036, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 548, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 646, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 1413, 4096, 4096, 0, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4096, 88, 0, 0, 4096, 0, 4096, 6, 0, 0, 0, 4096, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4050, 0, 0, 314, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1802, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 1006, 4096, 4096, 0, 957, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4094, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 149, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 4096, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4081, 0, 0, 918, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 851, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 187, 4096, 4096, 0, 2108, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 4096, 49, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 4, 4084, 4, 0, 5, 14, 0, 4086, 4, 4, 0, 10, 3, 57, 6, 4092, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4, 0, 0, 0, 0, 4, 3389, 0, 1, 3984, 4, 5, 1373, 4050, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 0, 0, 4088, 0, 12, 2, 1271, 4080, 7, 0, 4091, 5, 0, 0, 1210, 702, 633, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 4082, 0, 0, 4079, 0, 4, 0, 875, 3459, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 575, 0, 0, 0, 1, 4, 0, 4, 3, 4041, 4, 3, 5, 10, 4092, 4091, 2875, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 30, 4090, 3, 0, 4092, 4072, 4067, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 3, 0, 4035, 37, 0, 103, 3792, 0, 0, 25, 0, 0, 3, 4082, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 5, 52, 0, 6, 4, 4070, 300, 4088, 0, 2, 1, 4092, 4033, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16456, 16905, 16121, 16597, 15764, 16365, 16196, 16668] avg=16384.0000 max=16905 max_rank=1 min=15764 min_rank=4 imbalance_ratio=1.031799 max_load_reduction=114167 max_load_reduction_pct=0.871025 moved_to_echo=64993 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3389, 0, 0, 4094, 0, 0, 347, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1439, 4096, 0, 0, 4096, 0, 0, 0, 1215, 707, 638, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1, 548, 3458, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1846, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 4096, 2881, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 23, 0, 0, 4096, 0, 0, 8, 0, 0, 0, 4083, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4039, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 16022, 16299, 16295, 17070, 16383, 16289, 16341] avg=16384.0000 max=17070 max_rank=4 min=16022 min_rank=1 imbalance_ratio=1.041870 max_load_reduction=114002 max_load_reduction_pct=0.869766 moved_to_echo=66083 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3393, 0, 0, 4096, 0, 0, 550, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2908, 4096, 0, 0, 4096, 0, 0, 0, 1211, 703, 633, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 439, 3463, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 2878, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4086, 3993, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4041, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16371, 16231, 17753, 16190, 15514, 16392, 16279, 16342] avg=16384.0000 max=17753 max_rank=2 min=15514 min_rank=4 imbalance_ratio=1.083557 max_load_reduction=113319 max_load_reduction_pct=0.864555 moved_to_echo=64527 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3393, 0, 0, 4079, 0, 0, 984, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 1175, 4096, 0, 0, 4096, 0, 0, 0, 1211, 703, 633, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4, 574, 3463, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1378, 3, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4092, 4096, 2878, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4090, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 49, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4082, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 1, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4037, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16648, 16021, 16327, 16595, 16384, 16371, 16349] avg=16384.0000 max=16648 max_rank=1 min=16021 min_rank=2 imbalance_ratio=1.016113 max_load_reduction=114424 max_load_reduction_pct=0.872986 moved_to_echo=65699 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3393, 0, 0, 4040, 0, 0, 879, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2199, 4096, 0, 0, 4096, 0, 0, 0, 1211, 703, 633, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 61, 3463, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4087, 4096, 2878, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4081, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 339, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 4079, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4036, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16504, 17044, 15804, 15955, 16376, 16659, 16351] avg=16384.0000 max=17044 max_rank=2 min=15804 min_rank=3 imbalance_ratio=1.040283 max_load_reduction=114028 max_load_reduction_pct=0.869965 moved_to_echo=65341 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3393, 0, 0, 4050, 0, 0, 314, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1802, 4096, 0, 0, 4096, 0, 0, 0, 1211, 703, 633, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1006, 3463, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 957, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 4096, 2878, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 149, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4086, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4040, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 15949, 16647, 16757, 16176, 16389, 16442, 16340] avg=16384.0000 max=16757 max_rank=3 min=15949 min_rank=1 imbalance_ratio=1.022766 max_load_reduction=114315 max_load_reduction_pct=0.872154 moved_to_echo=65347 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3393, 0, 0, 4081, 0, 0, 918, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 851, 4096, 0, 0, 4096, 0, 0, 0, 1211, 703, 633, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 187, 3463, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2108, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 4096, 2878, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4096, 49, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4086, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4040, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 16584, 15696, 15938, 17328, 16391, 16422, 16341] avg=16384.0000 max=17328 max_rank=4 min=15696 min_rank=2 imbalance_ratio=1.057617 max_load_reduction=113744 max_load_reduction_pct=0.867798 moved_to_echo=66482 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3393, 0, 0, 4096, 0, 0, 548, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 646, 4096, 0, 0, 4096, 0, 0, 0, 1211, 703, 633, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1413, 3463, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1448, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 2878, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 88, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 4086, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4040, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 16229, 15491, 17164, 16670, 16392, 16414, 16340] avg=16384.0000 max=17164 max_rank=3 min=15491 min_rank=2 imbalance_ratio=1.047607 max_load_reduction=113908 max_load_reduction_pct=0.869049 moved_to_echo=65816 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 128, 27136, 0, 128, 32640, 128, 128, 6016, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 128, 32768, 0, 0, 32768, 0, 128, 128, 5120, 27776, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 32768, 128, 0, 128, 128, 0, 32768, 128, 128, 0, 128, 128, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 0, 32768, 768, 0, 128, 32512, 0, 0, 128, 0, 0, 128, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 0, 128, 128, 128, 128, 128, 128, 32768, 384, 32768, 0, 128, 128, 32768, 32384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 32768, 32768, 32768, 0, 0, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[9472, 128, 0, 0, 128, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 32768, 32768, 23040, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 128, 128, 12416, 32768, 128, 0, 32768, 128, 0, 0, 9728, 5632, 5120, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4046, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 2, 0, 0, 0, 0, 0, 0, 46, 4096, 0, 0, 4096, 4096, 0, 4096, 8, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 4077, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4090, 0, 0, 11, 0, 0, 0, 0, 0, 0, 12, 4095, 0, 0, 4096, 4096, 0, 4095, 5, 4096, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 4036, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 4086, 0, 0, 4096, 4096, 0, 4095, 22, 4096, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4091, 4096, 0, 0, 0, 0, 4090, 1, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 4014, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 6, 0, 4096, 0, 0, 0, 4081, 0, 0, 5, 0, 0, 0, 0, 0, 0, 76, 4090, 0, 0, 4096, 4096, 0, 4092, 20, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 2, 4096, 0, 4093, 4096, 0, 0, 0, 0, 4094, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4027, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4071, 0, 0, 5, 0, 0, 0, 0, 0, 0, 69, 4096, 0, 0, 4096, 4096, 0, 4096, 16, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 4, 0, 4089, 0, 0, 3, 8, 0, 4086, 3890, 0, 0, 4092, 4092, 0, 4085, 0, 4090, 0, 0, 3, 0, 187, 4092, 0, 0, 4, 4, 0, 0, 0, 5, 4091, 4092, 4091, 4, 3, 0, 3, 4090, 0, 3, 4, 4, 6, 0, 9, 4069, 11, 0, 24, 3, 4, 0, 15, 4, 4091, 4, 2, 3, 3994, 5, 0, 92, 2, 7, 11, 4, 4, 3, 0, 4093, 2, 4, 4034, 4071, 3, 4093, 0, 4089, 4083, 101, 4092, 0, 0, 0, 0, 3, 0, 4080, 0, 0, 0, 0, 5, 0, 4, 0, 0, 0, 4092, 4092, 4041, 4, 4092, 0, 4089, 0, 4092, 4091, 0, 4, 0, 5, 4092, 40, 0, 4085, 0, 3, 0, 0, 3, 0, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 4056, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4096, 0, 0, 4096, 4096, 0, 4096, 20, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4096, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 4082, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 0, 7, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 4096, 4096, 0, 4096, 1, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 4077, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4011, 50, 22, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4090, 0, 4093, 22, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 12, 4095, 0, 0, 4096, 4096, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 0, 4074, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16424, 16377, 16379, 16405, 16355, 16368, 16386, 16378] avg=16384.0000 max=16424 max_rank=0 min=16355 min_rank=4 imbalance_ratio=1.002441 max_load_reduction=114648 max_load_reduction_pct=0.874695 moved_to_echo=65487 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 4036, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 4004, 44, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4068, 0, 4087, 17, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 4086, 0, 0, 4096, 4096, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4096, 4096, 0, 4075, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 4096, 0, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16368, 16383, 16370, 16399, 16396, 16388, 16385] avg=16384.0000 max=16399 max_rank=4 min=16368 min_rank=1 imbalance_ratio=1.000916 max_load_reduction=114673 max_load_reduction_pct=0.874886 moved_to_echo=65568 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4046, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4004, 44, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 4087, 17, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 46, 4096, 0, 0, 4096, 4096, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16408, 16358, 16383, 16393, 16388, 16371, 16393, 16378] avg=16384.0000 max=16408 max_rank=0 min=16358 min_rank=1 imbalance_ratio=1.001465 max_load_reduction=114664 max_load_reduction_pct=0.874817 moved_to_echo=65530 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4, 0, 4089, 0, 0, 3, 8, 0, 4086, 3890, 0, 0, 4092, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4090, 0, 0, 3, 0, 187, 4092, 0, 0, 4, 4, 0, 0, 4000, 44, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4091, 4091, 4091, 4, 3, 0, 3, 4090, 0, 3, 4, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4069, 11, 0, 24, 3, 4, 0, 15, 4, 4091, 4, 2, 3, 3994, 5, 4083, 17, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 2, 7, 11, 4, 4, 3, 0, 4093, 2, 4, 4034, 4071, 3, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4083, 101, 4072, 0, 0, 0, 0, 3, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4, 0, 0, 0, 0, 0, 4041, 4, 9, 0, 4089, 0, 4092, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5, 4092, 40, 0, 4085, 0, 3, 0, 0, 3, 0, 3, 0, 4092, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16248, 16530, 16395, 16344, 16379, 16428, 16335, 16413] avg=16384.0000 max=16530 max_rank=1 min=16248 min_rank=0 imbalance_ratio=1.008911 max_load_reduction=114542 max_load_reduction_pct=0.873886 moved_to_echo=65555 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 4014, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 4004, 44, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 6, 0, 4096, 0, 0, 0, 4081, 0, 4087, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 76, 4090, 0, 0, 4096, 4096, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 2, 4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16368, 16362, 16383, 16391, 16411, 16383, 16392, 16382] avg=16384.0000 max=16411 max_rank=4 min=16362 min_rank=1 imbalance_ratio=1.001648 max_load_reduction=114661 max_load_reduction_pct=0.874794 moved_to_echo=65568 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 4056, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4004, 44, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4087, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 4096, 0, 0, 4096, 4096, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16408, 16358, 16383, 16399, 16369, 16383, 16392, 16380] avg=16384.0000 max=16408 max_rank=0 min=16358 min_rank=1 imbalance_ratio=1.001465 max_load_reduction=114664 max_load_reduction_pct=0.874817 moved_to_echo=65524 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4027, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 4004, 44, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4071, 0, 4087, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 69, 4096, 0, 0, 4096, 4096, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 16361, 16383, 16374, 16414, 16379, 16393, 16378] avg=16384.0000 max=16414 max_rank=4 min=16361 min_rank=1 imbalance_ratio=1.001831 max_load_reduction=114658 max_load_reduction_pct=0.874771 moved_to_echo=65564 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 4082, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4004, 44, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 4087, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 4096, 4096, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16443, 16358, 16383, 16396, 16357, 16364, 16393, 16378] avg=16384.0000 max=16443 max_rank=0 min=16357 min_rank=4 imbalance_ratio=1.003601 max_load_reduction=114629 max_load_reduction_pct=0.874550 moved_to_echo=65492 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 128, 32768, 0, 128, 0, 0, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 128, 128, 0, 32768, 32640, 0, 0, 32512, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 0, 32768, 0, 0, 128, 0, 256, 32768, 0, 128, 128, 128, 0, 0, 32128, 384, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 32768, 32768, 128, 32640, 0, 128, 0, 0, 128, 0, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 32768, 32768, 32768, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 0, 0, 0, 0, 32768, 128, 128, 128, 32768, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 32768, 128, 0, 128, 128, 128, 128, 128, 128, 32768, 128, 128, 128, 32640, 128, 32768, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 384, 32768, 128, 128, 32768, 32768, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4088, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4095, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 4096, 1, 0, 0, 0, 4096, 4096, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 4085, 4096, 0, 4092, 0, 4082, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 4096, 19, 4096, 4088, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4074, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 10, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 5, 1, 0, 3, 0, 6, 0, 0, 0, 0, 525, 0, 4090, 111, 3, 0, 0, 4091, 4091, 5, 0, 0, 4091, 2, 0, 8, 7, 0, 3, 4080, 4092, 0, 4082, 13, 4016, 1, 3, 39, 3, 0, 0, 2, 4, 0, 39, 1, 5, 3, 5, 2, 0, 4, 0, 4, 4089, 4086, 29, 4074, 4089, 4092, 0, 0, 0, 3974, 0, 0, 6, 4, 3145, 3, 0, 4, 4086, 7, 4, 0, 0, 8, 3, 4082, 4073, 3996, 0, 4054, 0, 11, 0, 0, 1, 4087, 175, 4, 0, 0, 4067, 1, 4, 4087, 4091, 0, 5, 4092, 0, 0, 4, 4089, 4, 6, 5, 0, 34, 5, 4091, 4091, 0, 0, 4092, 4, 0, 3, 464, 4, 8, 6, 4091, 4092, 4006] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 22, 0, 4096, 0, 0, 0, 1, 4096, 4096, 0, 0, 7, 4096, 0, 0, 0, 2, 0, 0, 4089, 4096, 0, 4095, 0, 4058, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 18, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4093, 4096, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 4096, 0, 0, 0, 1, 0, 0, 4093, 4096, 0, 4095, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 6, 0, 0, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4095, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4077, 4096, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 21, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 4096, 0, 4082, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4052, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 1, 0, 3, 0, 6, 0, 0, 0, 0, 525, 0, 4090, 111, 3, 4092, 4092, 3998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4091, 5, 0, 0, 4091, 2, 0, 8, 7, 0, 3, 4080, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 13, 4016, 1, 3, 39, 3, 0, 0, 2, 4, 0, 39, 1, 5, 4087, 3999, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 2, 0, 4, 0, 4, 4087, 4086, 29, 4074, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 6, 4, 3145, 3, 0, 4, 4086, 7, 4, 0, 0, 8, 3, 4089, 83, 47, 7, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 3996, 0, 4054, 0, 11, 0, 0, 1, 0, 175, 4, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 4087, 4091, 0, 5, 4088, 0, 0, 4, 4089, 4, 6, 5, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4044, 4091, 0, 0, 4092, 4, 0, 3, 464, 4, 8, 6, 0, 0, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16926, 16381, 16387, 16383, 15476, 16374, 16418, 16727] avg=16384.0000 max=16926 max_rank=0 min=15476 min_rank=4 imbalance_ratio=1.033081 max_load_reduction=114146 max_load_reduction_pct=0.870865 moved_to_echo=64995 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 4096, 1, 0, 4096, 4096, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 4085, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4082, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4013, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4094, 4096, 19, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4074, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 83, 47, 7, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16302, 16390, 16384, 16396, 16502, 16387, 16380, 16331] avg=16384.0000 max=16502 max_rank=4 min=16302 min_rank=0 imbalance_ratio=1.007202 max_load_reduction=114570 max_load_reduction_pct=0.874100 moved_to_echo=65600 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 4096, 4096, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4013, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 83, 47, 7, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 4095, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16294, 16387, 16387, 16382, 16524, 16381, 16380, 16337] avg=16384.0000 max=16524 max_rank=4 min=16294 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=114548 max_load_reduction_pct=0.873932 moved_to_echo=65622 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 22, 0, 4096, 0, 0, 4096, 4096, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 7, 4096, 0, 0, 0, 2, 0, 0, 4089, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4058, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4014, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 82, 47, 7, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 4096, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 4092, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 4095, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16323, 16390, 16358, 16386, 16513, 16395, 16376, 16331] avg=16384.0000 max=16513 max_rank=4 min=16323 min_rank=0 imbalance_ratio=1.007874 max_load_reduction=114559 max_load_reduction_pct=0.874016 moved_to_echo=65615 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 4096, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 4096, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4014, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 82, 49, 14, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 0, 4096, 0, 6, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16294, 16384, 16393, 16379, 16533, 16377, 16378, 16334] avg=16384.0000 max=16533 max_rank=4 min=16294 min_rank=0 imbalance_ratio=1.009094 max_load_reduction=114539 max_load_reduction_pct=0.873863 moved_to_echo=65622 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 4096, 0, 0, 4096, 4096, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4013, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4052, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 83, 47, 7, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16341, 16385, 16385, 16382, 16480, 16382, 16380, 16337] avg=16384.0000 max=16480 max_rank=4 min=16337 min_rank=7 imbalance_ratio=1.005859 max_load_reduction=114592 max_load_reduction_pct=0.874268 moved_to_echo=65579 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4014, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 82, 47, 7, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16292, 16387, 16392, 16382, 16523, 16378, 16380, 16338] avg=16384.0000 max=16523 max_rank=4 min=16292 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=114549 max_load_reduction_pct=0.873940 moved_to_echo=65619 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 4096, 0, 0, 4096, 4096, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4077, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4014, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4093, 82, 47, 7, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 4096, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16300, 16368, 16386, 16382, 16521, 16398, 16380, 16337] avg=16384.0000 max=16521 max_rank=4 min=16300 min_rank=0 imbalance_ratio=1.008362 max_load_reduction=114551 max_load_reduction_pct=0.873955 moved_to_echo=65636 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 32768, 32768, 128, 0, 128, 32768, 128, 0, 128, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32768, 128, 32640, 128, 128, 128, 128, 0, 0, 128, 128, 0, 128, 128, 128, 32768, 32128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 0, 0, 0, 128, 640, 0, 32768, 128, 128, 32768, 32768, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 32768, 0, 0, 128, 32768, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 32768, 32768, 0, 32768, 0, 128, 0, 0, 128, 0, 256, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 0, 0, 128, 128, 31744, 128, 0, 128, 32768, 128, 128, 0, 0, 128, 128, 32768, 768, 384, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32512, 32768, 0, 0, 32768, 128, 0, 128, 512, 128, 128, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 32768, 32768, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True + [2026-08-30 05:03:38] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 1006.8 | throughput per GPU (TFLOP/s/GPU): 331.1 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.055215E+00 | loss scale: 1.0 | grad norm: 5.380 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1536, 2034, 1961, 2, 490, 0, 1788, 682, 997, 2375, 1066, 798, 1597, 2527, 0, 1243, 710, 1516, 1360, 1755, 21, 866, 159, 2640, 383, 49, 2979, 1853, 1776, 39, 1518, 1529, 547, 42, 1587, 338, 435, 2102, 1506, 215, 319, 9, 2113, 2897, 2, 1824, 1134, 1301, 1116, 434, 616, 676, 5, 26, 35, 2023, 2, 190, 7, 484, 1562, 2951, 1049, 65, 1441, 2, 0, 1268, 2285, 1948, 1031, 1083, 1470, 1796, 2102, 345, 1811, 26, 834, 2495, 646, 384, 17, 38, 2313, 550, 1091, 2, 1595, 378, 763, 896, 313, 1528, 459, 0, 1177, 568, 4, 1643, 6, 541, 1090, 1098, 207, 1244, 127, 1562, 788, 1773, 2378, 992, 2287, 2352, 382, 843, 2639, 771, 2160, 1309, 1470, 1748, 1024, 552, 1025, 19, 396, 126] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2118, 2351, 1597, 0, 638, 0, 2611, 594, 980, 2769, 812, 799, 1014, 2636, 0, 1515, 683, 1419, 1465, 1497, 18, 829, 154, 1980, 111, 46, 3151, 1865, 1495, 73, 1905, 1233, 452, 2, 1574, 625, 364, 1615, 1585, 207, 127, 0, 2303, 2709, 0, 1826, 1194, 748, 1017, 812, 894, 957, 1, 56, 4, 2045, 0, 314, 55, 529, 1774, 2760, 1440, 215, 718, 59, 0, 1130, 1619, 1838, 572, 811, 1279, 1257, 1958, 375, 1642, 19, 572, 2292, 1177, 561, 14, 5, 2089, 575, 1570, 36, 2194, 110, 1096, 771, 137, 2344, 335, 0, 1380, 848, 6, 2117, 10, 453, 1558, 1132, 277, 1221, 99, 793, 624, 1676, 2445, 1951, 2109, 2105, 199, 813, 2611, 767, 2282, 1395, 1168, 1883, 1247, 570, 953, 6, 654, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2275, 2523, 1829, 0, 413, 0, 2775, 700, 369, 3043, 1093, 603, 1477, 2562, 0, 966, 783, 1529, 1067, 1568, 15, 810, 155, 2071, 86, 52, 3470, 1683, 1416, 43, 1283, 1057, 391, 0, 1940, 753, 162, 1554, 1280, 300, 122, 4, 1805, 2732, 0, 1848, 907, 590, 1292, 478, 835, 536, 4, 20, 9, 2399, 0, 142, 16, 364, 1664, 3050, 1178, 134, 1088, 33, 0, 1477, 2149, 1292, 1017, 1333, 1699, 1449, 2448, 321, 2209, 11, 837, 2471, 1190, 575, 3, 39, 1830, 445, 543, 3, 2199, 252, 1152, 1017, 202, 1890, 700, 0, 692, 1419, 12, 2321, 7, 276, 1371, 1033, 139, 1110, 203, 1287, 433, 2092, 2126, 1871, 2280, 1547, 242, 724, 2515, 1196, 2235, 1412, 802, 2102, 1170, 661, 930, 4, 754, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2522, 2116, 1473, 0, 334, 0, 2179, 1035, 1154, 2965, 1535, 971, 1231, 2966, 1, 1137, 944, 1540, 1258, 1123, 18, 1216, 171, 1741, 101, 62, 3494, 1906, 1048, 63, 1467, 857, 255, 0, 1796, 905, 196, 2078, 1555, 95, 140, 0, 1771, 2791, 0, 2388, 878, 1016, 1236, 561, 937, 938, 12, 15, 11, 1994, 0, 410, 38, 439, 1069, 2803, 1385, 444, 811, 6, 0, 1901, 2798, 1085, 536, 903, 1183, 1223, 1627, 122, 1937, 12, 767, 2473, 523, 309, 14, 13, 1980, 515, 1078, 5, 2144, 199, 1777, 579, 256, 1694, 571, 1, 1201, 741, 5, 2006, 8, 373, 1370, 635, 285, 1373, 85, 1328, 338, 1061, 2715, 1854, 2110, 1820, 822, 985, 2523, 1226, 2213, 1543, 1253, 2237, 1141, 678, 712, 12, 588, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2325, 1732, 1587, 1, 598, 1, 2111, 653, 1198, 2284, 1212, 429, 1073, 2615, 0, 1041, 884, 2247, 1072, 1244, 25, 552, 150, 2053, 67, 33, 3060, 2078, 1140, 82, 1283, 849, 524, 0, 1704, 547, 207, 2264, 1540, 267, 292, 4, 2188, 2984, 1, 1633, 1255, 1351, 1018, 719, 500, 491, 1, 18, 26, 1983, 0, 155, 75, 433, 684, 2876, 1162, 179, 886, 22, 0, 1708, 2484, 1837, 893, 945, 1324, 1900, 2205, 331, 1749, 4, 1417, 2001, 968, 596, 8, 14, 1592, 691, 997, 24, 2731, 145, 1918, 686, 311, 1500, 573, 0, 989, 275, 54, 1483, 7, 296, 1153, 982, 193, 1872, 204, 1515, 501, 1868, 2988, 2104, 2032, 2058, 509, 1667, 2404, 880, 1323, 1107, 943, 2627, 1218, 1091, 1027, 5, 450, 27] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2221, 2065, 1714, 0, 594, 0, 1946, 700, 1091, 2655, 1216, 921, 1591, 1781, 0, 1038, 1009, 1486, 1252, 1507, 45, 575, 130, 2157, 148, 26, 2981, 1735, 1251, 49, 1239, 1016, 372, 2, 1610, 415, 268, 1687, 1250, 236, 90, 23, 1939, 2855, 6, 1936, 1008, 873, 920, 779, 603, 1200, 2, 50, 34, 2013, 0, 200, 121, 587, 1373, 2942, 798, 719, 1105, 15, 0, 1249, 2212, 1756, 934, 746, 1357, 1559, 1919, 298, 1655, 15, 851, 2052, 826, 794, 9, 19, 1716, 474, 991, 60, 2326, 133, 1365, 1042, 132, 2046, 856, 0, 956, 522, 18, 2255, 4, 432, 1935, 807, 286, 1962, 202, 1145, 792, 2016, 2338, 1650, 2017, 1960, 377, 1780, 2971, 902, 2017, 1424, 1190, 2026, 1350, 806, 825, 4, 536, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2123, 1804, 1764, 2, 260, 0, 2569, 1098, 1115, 2137, 1141, 836, 988, 1966, 0, 1001, 667, 1626, 1637, 1349, 21, 764, 181, 1853, 387, 52, 3347, 2001, 1590, 59, 1937, 1042, 207, 0, 2003, 997, 274, 1894, 1431, 511, 164, 0, 2486, 2802, 0, 1763, 1037, 814, 918, 766, 836, 469, 16, 39, 9, 1508, 1, 242, 36, 350, 1896, 2474, 1572, 419, 965, 15, 0, 1610, 1927, 2152, 1028, 1169, 884, 1640, 2258, 603, 1219, 7, 1041, 1816, 1093, 750, 11, 30, 2127, 469, 1704, 8, 2064, 171, 1560, 957, 83, 1507, 438, 4, 1076, 638, 2, 1980, 41, 402, 1330, 1172, 208, 1289, 48, 1307, 368, 1136, 1916, 1510, 1901, 2222, 360, 1619, 2302, 664, 2539, 1469, 1210, 2449, 1081, 728, 960, 11, 549, 24] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1742, 2126, 1324, 0, 400, 0, 2468, 713, 1504, 2755, 1428, 571, 1218, 3075, 0, 1141, 986, 1631, 1228, 1318, 9, 602, 159, 2329, 122, 178, 3235, 2191, 1388, 196, 1995, 988, 149, 0, 1924, 612, 225, 1942, 972, 314, 163, 2, 2239, 3067, 0, 2066, 779, 838, 832, 835, 770, 790, 7, 58, 2, 1725, 0, 153, 23, 570, 1401, 2952, 861, 202, 742, 14, 0, 1211, 1932, 1336, 648, 1015, 1325, 1557, 2116, 247, 1729, 6, 1269, 2452, 584, 359, 11, 4, 1645, 481, 907, 16, 2469, 146, 1052, 785, 405, 2262, 621, 0, 1349, 1142, 8, 2123, 5, 458, 1422, 1165, 145, 1734, 95, 1252, 119, 1608, 2707, 2289, 2463, 2119, 449, 751, 2466, 966, 1612, 1309, 992, 2384, 1093, 495, 921, 13, 573, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1536, 2034, 1961, 2, 490, 0, 1788, 682, 997, 0, 1066, 798, 1597, 1601, 0, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 710, 1516, 1360, 1755, 21, 866, 159, 2640, 383, 49, 1752, 1853, 1776, 39, 1518, 1529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 547, 42, 1587, 338, 435, 2102, 1506, 215, 319, 9, 2113, 2897, 2, 1824, 1134, 1301, 590, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1116, 434, 616, 676, 5, 26, 35, 2023, 2, 190, 7, 484, 1562, 2951, 1049, 65, 2375, 2088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1441, 2, 0, 1268, 2285, 1948, 1031, 1083, 1470, 1796, 2102, 345, 1811, 26, 834, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 646, 384, 17, 38, 2313, 550, 1091, 2, 1595, 378, 763, 896, 313, 1528, 459, 0, 2154, 1227, 551, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1177, 568, 4, 1643, 6, 541, 1090, 1098, 207, 1244, 127, 1562, 788, 1773, 2378, 992, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2287, 2352, 382, 843, 0, 771, 2160, 1309, 1470, 1442, 1024, 552, 1025, 19, 396, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15795, 17926, 17101, 15704, 17783, 15241, 15364, 16158] avg=16384.0000 max=17926 max_rank=1 min=15241 min_rank=5 imbalance_ratio=1.094116 max_load_reduction=113146 max_load_reduction_pct=0.863235 moved_to_echo=64546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2118, 2351, 1597, 0, 638, 0, 2611, 594, 980, 0, 812, 799, 1014, 1670, 0, 1515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 683, 1419, 1465, 1497, 18, 829, 154, 1980, 111, 46, 1853, 1865, 1495, 73, 1905, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 452, 2, 1574, 625, 364, 1615, 1585, 207, 127, 0, 2303, 2709, 0, 1826, 1194, 748, 616, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 812, 894, 957, 1, 56, 4, 2045, 0, 314, 55, 529, 1774, 2760, 1440, 215, 2769, 2066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 718, 59, 0, 1130, 1619, 1838, 572, 811, 1279, 1257, 1958, 375, 1642, 19, 572, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1177, 561, 14, 5, 2089, 575, 1570, 36, 2194, 110, 1096, 771, 137, 2344, 335, 0, 1978, 1298, 545, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1380, 848, 6, 2117, 10, 453, 1558, 1132, 277, 1221, 99, 793, 624, 1676, 2445, 1951, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2109, 2105, 199, 813, 0, 767, 2282, 1395, 1168, 1553, 1247, 570, 953, 6, 654, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16699, 16626, 16098, 17708, 14163, 17185, 16769, 15824] avg=16384.0000 max=17708 max_rank=3 min=14163 min_rank=4 imbalance_ratio=1.080811 max_load_reduction=113364 max_load_reduction_pct=0.864899 moved_to_echo=63941 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2325, 1732, 1587, 1, 598, 1, 2111, 653, 1198, 0, 1212, 429, 1073, 1657, 0, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 884, 2247, 1072, 1244, 25, 552, 150, 2053, 67, 33, 1800, 2078, 1140, 82, 1283, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 524, 0, 1704, 547, 207, 2264, 1540, 267, 292, 4, 2188, 2984, 1, 1633, 1255, 1351, 611, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1018, 719, 500, 491, 1, 18, 26, 1983, 0, 155, 75, 433, 684, 2876, 1162, 179, 2284, 1903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 886, 22, 0, 1708, 2484, 1837, 893, 945, 1324, 1900, 2205, 331, 1749, 4, 1417, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 968, 596, 8, 14, 1592, 691, 997, 24, 2731, 145, 1918, 686, 311, 1500, 573, 0, 1727, 1260, 501, 347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 989, 275, 54, 1483, 7, 296, 1153, 982, 193, 1872, 204, 1515, 501, 1868, 2988, 2104, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2032, 2058, 509, 1667, 0, 880, 1323, 1107, 943, 2166, 1218, 1091, 1027, 5, 450, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15618, 15559, 17583, 14507, 17979, 16589, 16734, 16503] avg=16384.0000 max=17979 max_rank=4 min=14507 min_rank=3 imbalance_ratio=1.097351 max_load_reduction=113093 max_load_reduction_pct=0.862831 moved_to_echo=67805 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2275, 2523, 1829, 0, 413, 0, 2775, 700, 369, 0, 1093, 603, 1477, 1623, 0, 966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 783, 1529, 1067, 1568, 15, 810, 155, 2071, 86, 52, 2041, 1683, 1416, 43, 1283, 1057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 0, 1940, 753, 162, 1554, 1280, 300, 122, 4, 1805, 2732, 0, 1848, 907, 590, 599, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1292, 478, 835, 536, 4, 20, 9, 2399, 0, 142, 16, 364, 1664, 3050, 1178, 134, 3043, 1991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1088, 33, 0, 1477, 2149, 1292, 1017, 1333, 1699, 1449, 2448, 321, 2209, 11, 837, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1190, 575, 3, 39, 1830, 445, 543, 3, 2199, 252, 1152, 1017, 202, 1890, 700, 0, 2133, 1429, 524, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 692, 1419, 12, 2321, 7, 276, 1371, 1033, 139, 1110, 203, 1287, 433, 2092, 2126, 1871, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2280, 1547, 242, 724, 0, 1196, 2235, 1412, 802, 1733, 1170, 661, 930, 4, 754, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16646, 15659, 15156, 17155, 17701, 16466, 16592, 15697] avg=16384.0000 max=17701 max_rank=4 min=15156 min_rank=2 imbalance_ratio=1.080383 max_load_reduction=113371 max_load_reduction_pct=0.864952 moved_to_echo=66456 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2123, 1804, 1764, 2, 260, 0, 2569, 1098, 1115, 0, 1141, 836, 988, 1246, 0, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, 1626, 1637, 1349, 21, 764, 181, 1853, 387, 52, 1969, 2001, 1590, 59, 1937, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 2003, 997, 274, 1894, 1431, 511, 164, 0, 2486, 2802, 0, 1763, 1037, 814, 459, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 918, 766, 836, 469, 16, 39, 9, 1508, 1, 242, 36, 350, 1896, 2474, 1572, 419, 2137, 1821, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 965, 15, 0, 1610, 1927, 2152, 1028, 1169, 884, 1640, 2258, 603, 1219, 7, 1041, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1093, 750, 11, 30, 2127, 469, 1704, 8, 2064, 171, 1560, 957, 83, 1507, 438, 4, 1567, 1378, 481, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 638, 2, 1980, 41, 402, 1330, 1172, 208, 1289, 48, 1307, 368, 1136, 1916, 1510, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1901, 2222, 360, 1619, 0, 664, 2539, 1469, 1210, 2019, 1081, 728, 960, 11, 549, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15947, 17135, 17039, 15509, 16767, 16663, 14656, 17356] avg=16384.0000 max=17356 max_rank=7 min=14656 min_rank=6 imbalance_ratio=1.059326 max_load_reduction=113716 max_load_reduction_pct=0.867584 moved_to_echo=65442 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2522, 2116, 1473, 0, 334, 0, 2179, 1035, 1154, 0, 1535, 971, 1231, 1879, 1, 1137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 944, 1540, 1258, 1123, 18, 1216, 171, 1741, 101, 62, 2055, 1906, 1048, 63, 1467, 857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 1796, 905, 196, 2078, 1555, 95, 140, 0, 1771, 2791, 0, 2388, 878, 1016, 693, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1236, 561, 937, 938, 12, 15, 11, 1994, 0, 410, 38, 439, 1069, 2803, 1385, 444, 2965, 1997, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 811, 6, 0, 1901, 2798, 1085, 536, 903, 1183, 1223, 1627, 122, 1937, 12, 767, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 309, 14, 13, 1980, 515, 1078, 5, 2144, 199, 1777, 579, 256, 1694, 571, 1, 2135, 1439, 526, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1201, 741, 5, 2006, 8, 373, 1370, 635, 285, 1373, 85, 1328, 338, 1061, 2715, 1854, 213, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2110, 1820, 822, 985, 0, 1226, 2213, 1543, 1253, 1844, 1141, 678, 712, 12, 588, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17567, 15570, 16737, 17254, 15249, 16152, 15591, 16952] avg=16384.0000 max=17567 max_rank=0 min=15249 min_rank=4 imbalance_ratio=1.072205 max_load_reduction=113505 max_load_reduction_pct=0.865974 moved_to_echo=63944 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2221, 2065, 1714, 0, 594, 0, 1946, 700, 1091, 0, 1216, 921, 1591, 1120, 0, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1009, 1486, 1252, 1507, 45, 575, 130, 2157, 148, 26, 1749, 1735, 1251, 49, 1239, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 2, 1610, 415, 268, 1687, 1250, 236, 90, 23, 1939, 2855, 6, 1936, 1008, 873, 420, 166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 920, 779, 603, 1200, 2, 50, 34, 2013, 0, 200, 121, 587, 1373, 2942, 798, 719, 2655, 2352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 15, 0, 1249, 2212, 1756, 934, 746, 1357, 1559, 1919, 298, 1655, 15, 851, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 794, 9, 19, 1716, 474, 991, 60, 2326, 133, 1365, 1042, 132, 2046, 856, 0, 1775, 1232, 619, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 956, 522, 18, 2255, 4, 432, 1935, 807, 286, 1962, 202, 1145, 792, 2016, 2338, 1650, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2017, 1960, 377, 1780, 0, 902, 2017, 1424, 1190, 1662, 1350, 806, 825, 4, 536, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16217, 15374, 15156, 17348, 15948, 16656, 17518, 16855] avg=16384.0000 max=17518 max_rank=6 min=15156 min_rank=2 imbalance_ratio=1.069214 max_load_reduction=113554 max_load_reduction_pct=0.866348 moved_to_echo=66977 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1742, 2126, 1324, 0, 400, 0, 2468, 713, 1504, 0, 1428, 571, 1218, 1948, 0, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 986, 1631, 1228, 1318, 9, 602, 159, 2329, 122, 178, 1903, 2191, 1388, 196, 1995, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 1924, 612, 225, 1942, 972, 314, 163, 2, 2239, 3067, 0, 2066, 779, 838, 718, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 832, 835, 770, 790, 7, 58, 2, 1725, 0, 153, 23, 570, 1401, 2952, 861, 202, 2755, 1951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 742, 14, 0, 1211, 1932, 1336, 648, 1015, 1325, 1557, 2116, 247, 1729, 6, 1269, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 584, 359, 11, 4, 1645, 481, 907, 16, 2469, 146, 1052, 785, 405, 2262, 621, 0, 2117, 1332, 515, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1349, 1142, 8, 2123, 5, 458, 1422, 1165, 145, 1734, 95, 1252, 119, 1608, 2707, 2289, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2463, 2119, 449, 751, 0, 966, 1612, 1309, 992, 1965, 1093, 495, 921, 13, 573, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16583, 17223, 16202, 15887, 15482, 16120, 17848, 15727] avg=16384.0000 max=17848 max_rank=6 min=15482 min_rank=4 imbalance_ratio=1.089355 max_load_reduction=113224 max_load_reduction_pct=0.863831 moved_to_echo=65177 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[16896, 16768, 13312, 128, 3840, 128, 18560, 6272, 8448, 0, 9600, 6016, 10240, 12800, 128, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[7808, 256, 0, 11648, 17408, 13312, 6784, 8064, 10624, 12416, 16640, 2688, 13952, 128, 7680, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[8832, 6272, 128, 16000, 128, 3328, 11264, 8064, 1792, 11904, 1152, 10240, 3968, 13312, 19712, 14336, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[6784, 13056, 10368, 11392, 256, 6272, 1280, 16896, 1408, 512, 15232, 15360, 11136, 640, 12672, 8576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[2944, 128, 14208, 5248, 2176, 15232, 11136, 2176, 1536, 128, 16896, 22912, 128, 15360, 8192, 7552, 4736, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[8448, 5504, 6016, 6144, 128, 384, 256, 15744, 128, 1920, 384, 3840, 11520, 22912, 9472, 2432, 20992, 16256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[17280, 16256, 3456, 9216, 0, 7424, 16384, 11008, 9088, 14464, 9344, 5632, 7424, 128, 4608, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[7040, 4352, 128, 256, 15360, 4224, 8960, 256, 17792, 1536, 10752, 6784, 1920, 14848, 4608, 128, 15616, 10624, 4352, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 4076, 1, 302, 7, 0, 1, 83, 17, 0, 1, 0, 4096, 0, 0, 4094, 0, 0, 4091, 0, 1988, 2, 1, 4075, 0, 4092, 4094, 53, 0, 1, 4096, 0, 3226, 3942, 33, 0, 1, 1923, 0, 1, 0, 0, 419, 0, 1, 3860, 4081, 0, 0, 4096, 6, 37, 4095, 0, 1, 4096, 42, 0, 0, 4095, 0, 343, 2466, 0, 0, 3338, 3, 1, 4092, 0, 3760, 4095, 0, 0, 0, 1, 0, 0, 2653, 1, 15, 0, 0, 2, 3750, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4059, 0, 6, 0, 89, 4096, 0, 4077, 0, 0, 0, 4090, 4095, 1, 1, 47, 0, 149, 0, 0, 4057, 352, 0, 42, 0, 0, 1, 4068] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4093, 0, 96, 6, 0, 0, 333, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 4093, 0, 1534, 5, 0, 4087, 0, 4096, 4095, 48, 0, 0, 4096, 0, 1850, 4035, 5, 0, 0, 3020, 0, 0, 0, 0, 600, 0, 24, 3953, 4093, 0, 0, 4096, 0, 32, 4096, 0, 0, 4096, 2, 0, 0, 4096, 0, 42, 2887, 0, 0, 3534, 0, 0, 4090, 0, 3497, 4093, 0, 0, 0, 0, 0, 0, 3159, 0, 0, 0, 0, 0, 2242, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3860, 11, 2, 0, 8, 4096, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 113, 0, 578, 2, 0, 4094, 226, 0, 1193, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4084, 0, 16, 0, 0, 0, 147, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 2271, 3, 0, 4067, 0, 4094, 4096, 40, 0, 0, 4096, 0, 2859, 4012, 15, 1, 0, 2617, 0, 0, 3, 0, 539, 0, 0, 3949, 4095, 0, 0, 4096, 0, 134, 4096, 0, 0, 4096, 41, 0, 0, 4093, 0, 95, 2874, 0, 0, 2761, 0, 0, 4096, 0, 3834, 4096, 0, 0, 0, 1, 0, 0, 3752, 0, 4, 0, 0, 0, 1865, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3933, 7, 0, 0, 29, 4096, 0, 4095, 0, 0, 0, 4094, 4096, 0, 0, 87, 0, 368, 1, 0, 4094, 186, 0, 476, 0, 0, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 4096, 0, 23, 0, 0, 0, 424, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 4078, 1, 2369, 0, 0, 4094, 0, 4081, 4086, 0, 0, 0, 4096, 0, 2954, 4052, 2, 0, 0, 2331, 1, 0, 0, 0, 565, 0, 2, 4055, 4095, 0, 0, 4096, 25, 73, 4096, 0, 0, 4092, 305, 1, 0, 4066, 0, 17, 2937, 0, 0, 3604, 6, 0, 4090, 0, 2038, 4096, 0, 0, 0, 0, 0, 0, 2287, 0, 3, 0, 0, 0, 2880, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4035, 1, 97, 0, 182, 4096, 0, 4093, 7, 0, 0, 4093, 4096, 0, 0, 28, 0, 295, 9, 0, 4096, 21, 0, 1339, 0, 0, 0, 4081] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 4060, 0, 732, 0, 0, 0, 92, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 884, 0, 0, 4083, 0, 4095, 4096, 29, 0, 0, 4096, 2, 3512, 4034, 15, 0, 0, 3565, 0, 0, 2, 0, 533, 0, 1, 3969, 4084, 0, 0, 4096, 35, 99, 4096, 0, 0, 4096, 20, 0, 0, 4095, 0, 38, 2676, 0, 0, 2497, 2, 0, 4096, 0, 3650, 4096, 0, 0, 0, 0, 0, 0, 3271, 0, 51, 0, 0, 0, 1537, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 4075, 0, 13, 0, 87, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 3, 0, 245, 1, 0, 4060, 769, 0, 523, 0, 0, 0, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 4073, 0, 4, 0, 0, 2, 224, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 4096, 1, 3431, 1, 0, 4027, 0, 4094, 4095, 41, 0, 0, 4096, 0, 3291, 3982, 4, 4, 0, 1008, 0, 0, 0, 0, 373, 0, 75, 4088, 4083, 0, 0, 4096, 1, 43, 4096, 0, 0, 4093, 188, 0, 0, 4096, 0, 29, 3490, 0, 0, 3636, 3, 0, 4095, 0, 2202, 4095, 0, 0, 0, 0, 0, 0, 3184, 0, 9, 0, 0, 0, 2989, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 3848, 38, 21, 0, 71, 4095, 0, 4062, 0, 0, 0, 4096, 4096, 1, 0, 293, 0, 304, 0, 0, 4087, 18, 2, 131, 0, 0, 0, 4089] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 85, 4060, 0, 286, 1, 1, 0, 173, 10, 0, 0, 0, 4094, 1, 0, 4096, 0, 1, 4091, 2, 2459, 0, 2, 4000, 0, 4095, 4092, 59, 0, 0, 4094, 0, 2474, 4033, 37, 1, 0, 2938, 4, 2, 2, 0, 297, 0, 0, 3998, 4073, 0, 0, 4096, 0, 56, 4094, 0, 0, 4096, 42, 0, 0, 4057, 0, 79, 3231, 0, 0, 2567, 7, 0, 4094, 1, 3654, 4095, 0, 0, 0, 4, 0, 0, 2964, 0, 32, 0, 1, 1, 3049, 0, 0, 0, 0, 0, 4094, 2, 1, 0, 0, 0, 4096, 4091, 23, 3, 0, 11, 4095, 0, 4091, 8, 0, 0, 4096, 4096, 0, 0, 18, 0, 152, 0, 0, 4049, 269, 0, 6, 1, 1, 0, 4088] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 24, 4089, 0, 1, 6, 8, 2, 780, 3, 0, 0, 1, 4096, 0, 0, 4095, 0, 0, 4088, 0, 1965, 2, 0, 4071, 0, 4096, 4093, 5, 1, 0, 4096, 0, 3554, 4031, 10, 4, 0, 2607, 0, 0, 0, 0, 676, 0, 116, 4075, 4092, 0, 0, 4095, 0, 44, 4096, 0, 0, 4096, 101, 0, 0, 4096, 0, 4, 2928, 0, 0, 2606, 4, 0, 4090, 0, 2993, 4078, 1, 0, 0, 0, 0, 0, 2738, 4, 0, 0, 0, 0, 2420, 0, 0, 1, 0, 0, 4095, 1, 1, 0, 0, 1, 4095, 3947, 10, 57, 0, 72, 4096, 1, 4093, 1, 0, 0, 4095, 4095, 0, 0, 68, 0, 159, 1, 0, 4090, 230, 0, 687, 0, 0, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 4076, 1, 302, 7, 0, 1, 83, 17, 0, 1, 0, 4096, 4096, 3623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 1988, 2, 1, 4075, 0, 4092, 1988, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 3226, 3942, 33, 0, 1, 1923, 0, 1, 0, 0, 419, 0, 1, 1122, 1095, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3860, 4081, 0, 0, 0, 6, 37, 0, 0, 1, 4052, 42, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 343, 2466, 0, 0, 3338, 3, 1, 4092, 0, 3760, 3000, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2653, 1, 15, 0, 0, 2, 3750, 0, 1, 0, 0, 0, 4095, 0, 0, 4095, 2114, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4059, 0, 6, 0, 89, 4096, 0, 4077, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 47, 0, 149, 0, 0, 4057, 352, 0, 42, 0, 0, 1, 4068, 1963, 984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16307, 16290, 15904, 16174, 17004, 17197, 16436, 15760] avg=16384.0000 max=17197 max_rank=5 min=15760 min_rank=7 imbalance_ratio=1.049622 max_load_reduction=113875 max_load_reduction_pct=0.868797 moved_to_echo=66397 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 4093, 0, 96, 6, 0, 0, 333, 0, 0, 0, 0, 4093, 4096, 3626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1534, 5, 0, 4087, 0, 4096, 1989, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1850, 4035, 5, 0, 0, 3020, 0, 0, 0, 0, 600, 0, 24, 1122, 1094, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3953, 4093, 0, 0, 0, 0, 32, 0, 0, 0, 4052, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 2887, 0, 0, 3534, 0, 0, 4090, 0, 3497, 2999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3159, 0, 0, 0, 0, 0, 2242, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2114, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 3860, 11, 2, 0, 8, 4096, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 113, 0, 578, 2, 0, 4094, 226, 0, 1193, 0, 0, 0, 4095, 1963, 984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16345, 15852, 15890, 16228, 17049, 16177, 16187, 17344] avg=16384.0000 max=17344 max_rank=7 min=15852 min_rank=1 imbalance_ratio=1.058594 max_load_reduction=113728 max_load_reduction_pct=0.867676 moved_to_echo=66757 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 4060, 0, 732, 0, 0, 0, 92, 0, 0, 0, 0, 4096, 4096, 3626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 884, 0, 0, 4083, 0, 4095, 1990, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 3512, 4034, 15, 0, 0, 3565, 0, 0, 2, 0, 533, 0, 1, 1122, 1095, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3969, 4084, 0, 0, 0, 35, 99, 0, 0, 0, 4052, 20, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 2676, 0, 0, 2497, 2, 0, 4096, 0, 3650, 3001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3271, 0, 51, 0, 0, 0, 1537, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 2114, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4075, 0, 13, 0, 87, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 245, 1, 0, 4060, 769, 0, 523, 0, 0, 0, 4092, 1963, 984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16708, 15175, 18021, 16354, 15960, 15636, 16482, 16736] avg=16384.0000 max=18021 max_rank=2 min=15175 min_rank=1 imbalance_ratio=1.099915 max_load_reduction=113051 max_load_reduction_pct=0.862511 moved_to_echo=64814 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 4073, 0, 4, 0, 0, 2, 224, 0, 0, 0, 0, 4090, 4096, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 3431, 1, 0, 4027, 0, 4094, 1989, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3291, 3982, 4, 4, 0, 1008, 0, 0, 0, 0, 373, 0, 75, 1122, 1095, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4083, 0, 0, 0, 1, 43, 0, 0, 0, 4049, 188, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 3490, 0, 0, 3636, 3, 0, 4095, 0, 2202, 3000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3184, 0, 9, 0, 0, 0, 2989, 0, 0, 1, 0, 0, 4093, 0, 0, 4096, 2114, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 3848, 38, 21, 0, 71, 4095, 0, 4062, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 293, 0, 304, 0, 0, 4087, 18, 2, 131, 0, 0, 0, 4089, 1963, 984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16120, 17680, 15094, 16548, 16455, 16957, 16250, 15968] avg=16384.0000 max=17680 max_rank=1 min=15094 min_rank=2 imbalance_ratio=1.079102 max_load_reduction=113392 max_load_reduction_pct=0.865112 moved_to_echo=65630 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 7, 4096, 0, 23, 0, 0, 0, 424, 0, 0, 0, 0, 4091, 4096, 3626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 1, 2369, 0, 0, 4094, 0, 4081, 1978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2954, 4052, 2, 0, 0, 2331, 1, 0, 0, 0, 565, 0, 2, 1123, 1097, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 4095, 0, 0, 0, 25, 73, 0, 0, 0, 4046, 305, 1, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2937, 0, 0, 3604, 6, 0, 4090, 0, 2038, 2999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2287, 0, 3, 0, 0, 0, 2880, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2115, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4035, 1, 97, 0, 182, 4096, 0, 4093, 7, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 28, 0, 295, 9, 0, 4096, 21, 0, 1339, 0, 0, 0, 4081, 1970, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16363, 16601, 16269, 16666, 15691, 15947, 16615, 16920] avg=16384.0000 max=16920 max_rank=7 min=15691 min_rank=4 imbalance_ratio=1.032715 max_load_reduction=114152 max_load_reduction_pct=0.870911 moved_to_echo=65173 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 4084, 0, 16, 0, 0, 0, 147, 0, 0, 0, 0, 4096, 4096, 3626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2271, 3, 0, 4067, 0, 4094, 1990, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2859, 4012, 15, 1, 0, 2617, 0, 0, 3, 0, 539, 0, 0, 1122, 1095, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 4095, 0, 0, 0, 0, 134, 0, 0, 0, 4052, 41, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 2874, 0, 0, 2761, 0, 0, 4096, 0, 3834, 3001, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3752, 0, 4, 0, 0, 0, 1865, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2114, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 3933, 7, 0, 0, 29, 4096, 0, 4095, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 87, 0, 368, 1, 0, 4094, 186, 0, 476, 0, 0, 0, 4093, 1963, 984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16066, 16559, 16403, 16364, 16662, 16397, 16273, 16348] avg=16384.0000 max=16662 max_rank=4 min=16066 min_rank=0 imbalance_ratio=1.016968 max_load_reduction=114410 max_load_reduction_pct=0.872879 moved_to_echo=65680 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 85, 4060, 0, 286, 1, 1, 0, 173, 10, 0, 0, 0, 4094, 4096, 3626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4091, 2, 2459, 0, 2, 4000, 0, 4095, 1988, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2474, 4033, 37, 1, 0, 2938, 4, 2, 2, 0, 297, 0, 0, 1121, 1095, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 4073, 0, 0, 0, 0, 56, 0, 0, 0, 4052, 42, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 3231, 0, 0, 2567, 7, 0, 4094, 1, 3654, 3000, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2964, 0, 32, 0, 1, 1, 3049, 0, 0, 0, 0, 0, 4094, 2, 1, 4094, 2114, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4091, 23, 3, 0, 11, 4095, 0, 4091, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 152, 0, 0, 4049, 269, 0, 6, 1, 1, 0, 4088, 1963, 983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16432, 16698, 16142, 16278, 16637, 16822, 16437, 15626] avg=16384.0000 max=16822 max_rank=5 min=15626 min_rank=7 imbalance_ratio=1.026733 max_load_reduction=114250 max_load_reduction_pct=0.871658 moved_to_echo=65522 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 24, 4089, 0, 1, 6, 8, 2, 780, 3, 0, 0, 1, 4096, 4095, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 1965, 2, 0, 4071, 0, 4096, 1989, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3554, 4031, 10, 4, 0, 2607, 0, 0, 0, 0, 676, 0, 116, 1121, 1090, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4092, 0, 0, 0, 0, 44, 0, 0, 0, 4052, 101, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2928, 0, 0, 2606, 4, 0, 4090, 0, 2993, 2988, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2738, 4, 0, 0, 0, 0, 2420, 0, 0, 1, 0, 0, 4095, 1, 1, 4096, 2113, 470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 19, 3947, 10, 57, 0, 72, 4096, 1, 4093, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 68, 0, 159, 1, 0, 4090, 230, 0, 687, 0, 0, 0, 4094, 1963, 983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16731, 16217, 17349, 16460, 15614, 15939, 16392, 16370] avg=16384.0000 max=17349 max_rank=2 min=15614 min_rank=4 imbalance_ratio=1.058899 max_load_reduction=113723 max_load_reduction_pct=0.867638 moved_to_echo=64315 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 768, 0, 2304, 128, 0, 32640, 2176, 128, 4480, 128, 128, 128, 32768, 15744, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 23808, 32128, 128, 128, 128, 20096, 128, 128, 128, 0, 4096, 0, 256, 9088, 8832, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 128, 256, 31872, 128, 256, 0, 640, 32768, 128, 32768, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 256, 32640, 128, 1536, 128, 128, 128, 2304, 128, 0, 128, 128, 32768, 32768, 29056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32000, 32768, 0, 0, 0, 128, 640, 0, 0, 128, 32512, 768, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[768, 23552, 0, 0, 24576, 128, 128, 32768, 128, 25728, 24064, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 0, 128, 32768, 128, 17024, 128, 128, 32512, 0, 32768, 16000, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 24064, 128, 128, 0, 128, 128, 20736, 0, 128, 128, 0, 0, 32768, 128, 128, 32768, 17024, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3636, 0, 1103, 0, 0, 582, 0, 4096, 547, 0, 0, 0, 0, 4095, 0, 0, 0, 536, 8, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4094, 1596, 0, 7, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 4094, 4080, 2694, 0, 0, 1865, 0, 4096, 4096, 0, 0, 4076, 4079, 0, 0, 15, 4096, 0, 0, 4093, 28, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 14, 4039, 4090, 0, 0, 457, 3993, 0, 381, 0, 2658, 0, 0, 4038, 0, 0, 0, 3967, 0, 34, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 121, 4092, 0, 0, 9, 0, 0, 0, 4054, 0, 513, 0, 4085, 14, 0, 10, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3871, 0, 820, 0, 0, 855, 0, 4093, 7, 0, 0, 0, 0, 4096, 2, 0, 0, 382, 0, 0, 4093, 0, 4096, 4095, 0, 0, 0, 4095, 2527, 0, 0, 1, 4095, 1, 0, 4095, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 4093, 4086, 3060, 0, 0, 2967, 0, 4096, 4092, 0, 0, 4081, 4090, 0, 0, 0, 4093, 0, 0, 4094, 4, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 10, 8, 4003, 4093, 0, 0, 147, 4009, 0, 193, 0, 2310, 0, 1, 4069, 0, 0, 1, 3135, 1, 14, 1, 0, 18, 0, 4095, 1, 0, 0, 0, 1, 2, 0, 5, 4096, 0, 0, 4, 0, 0, 0, 4093, 0, 386, 0, 4094, 7, 1, 2, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3547, 0, 225, 0, 0, 491, 0, 4094, 107, 0, 0, 0, 0, 4095, 1, 0, 0, 572, 6, 0, 4095, 0, 4096, 4095, 0, 0, 0, 4093, 1615, 0, 2, 0, 4096, 0, 0, 4095, 4, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 4088, 3141, 0, 0, 2650, 0, 4094, 4083, 0, 0, 4080, 4067, 0, 0, 2, 4096, 0, 0, 4092, 5, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 27, 4075, 4095, 0, 0, 989, 4024, 0, 1122, 0, 2223, 1, 0, 4036, 0, 0, 0, 3720, 0, 21, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 2, 0, 0, 34, 4096, 0, 0, 3, 0, 0, 0, 4058, 0, 235, 0, 4093, 7, 0, 4, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3518, 0, 1445, 0, 0, 2253, 0, 4096, 31, 0, 0, 0, 0, 4096, 0, 0, 0, 559, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 1765, 0, 0, 0, 4096, 1, 0, 4096, 10, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 4095, 3690, 0, 0, 2781, 0, 4096, 4096, 0, 0, 4089, 4089, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 1, 4059, 4095, 0, 0, 53, 4037, 0, 70, 0, 880, 0, 0, 4050, 0, 0, 0, 3403, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 158, 0, 4096, 4, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3893, 0, 739, 0, 0, 521, 0, 4093, 383, 0, 0, 0, 0, 4095, 0, 0, 0, 821, 0, 0, 4095, 0, 4096, 4095, 0, 0, 2, 4096, 2638, 0, 5, 1, 4096, 0, 0, 4096, 2, 0, 0, 0, 4095, 0, 0, 0, 0, 4088, 4096, 1787, 0, 1, 1510, 1, 4095, 4092, 0, 0, 4080, 4082, 0, 0, 3, 4096, 0, 0, 4094, 2, 0, 0, 0, 0, 1, 4096, 0, 0, 4095, 0, 11, 4076, 4087, 0, 0, 541, 4063, 0, 514, 0, 2447, 1, 1, 4082, 0, 0, 0, 3924, 1, 86, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 21, 4095, 0, 0, 2, 0, 0, 0, 4095, 0, 741, 0, 4094, 9, 0, 1, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3690, 0, 642, 0, 0, 489, 0, 4096, 130, 0, 0, 0, 0, 4096, 0, 0, 0, 794, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 3332, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 0, 1, 4096, 0, 0, 0, 0, 4093, 4096, 2785, 0, 0, 2428, 0, 4096, 4096, 0, 0, 4090, 4096, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 273, 4065, 0, 113, 0, 1692, 0, 0, 4089, 0, 0, 0, 3925, 0, 1, 0, 0, 18, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 55, 4096, 0, 0, 0, 0, 0, 0, 4092, 0, 160, 0, 4096, 2, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3750, 0, 2794, 0, 0, 1613, 0, 4090, 446, 1, 0, 0, 0, 4094, 0, 0, 0, 94, 0, 0, 4092, 0, 4096, 4093, 0, 0, 0, 4094, 2832, 0, 1, 0, 4096, 4, 0, 4096, 0, 0, 0, 0, 4096, 0, 17, 0, 0, 4095, 4081, 2739, 0, 0, 1297, 1, 4096, 4092, 0, 0, 4063, 4054, 0, 0, 4, 4096, 0, 0, 4096, 13, 0, 0, 0, 0, 2, 4095, 0, 0, 4096, 2, 24, 3993, 4092, 0, 0, 149, 4014, 0, 93, 0, 1325, 1, 0, 4060, 0, 0, 0, 3031, 2, 0, 0, 0, 9, 0, 4078, 0, 0, 0, 0, 2, 0, 0, 15, 4090, 0, 0, 7, 0, 0, 0, 4084, 0, 548, 0, 4095, 15, 1, 3, 0, 20] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3816, 0, 1812, 0, 0, 1773, 0, 4096, 503, 0, 0, 1, 0, 4096, 0, 0, 0, 449, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 1436, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 1, 0, 0, 4088, 4092, 3954, 0, 0, 1428, 0, 4096, 4096, 0, 0, 4094, 4095, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 8, 0, 4040, 4096, 0, 0, 105, 4060, 0, 21, 0, 930, 0, 0, 4039, 0, 0, 0, 3785, 0, 5, 0, 0, 37, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4082, 0, 591, 0, 4096, 0, 0, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3871, 0, 820, 0, 0, 855, 0, 4093, 7, 0, 0, 0, 0, 4096, 993, 945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 382, 0, 0, 4093, 0, 1353, 4095, 0, 0, 0, 4095, 2527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4088, 1, 0, 4095, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3119, 3060, 0, 0, 2967, 0, 0, 0, 0, 0, 4081, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 10, 8, 4003, 22, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2221, 0, 0, 147, 4009, 0, 193, 0, 2310, 0, 1, 4069, 0, 0, 1, 3135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 14, 1, 0, 18, 0, 4095, 1, 0, 0, 0, 1, 2, 0, 5, 4096, 4096, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4093, 0, 386, 0, 4094, 7, 1, 2, 0, 1, 4092, 2743, 879, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15680, 16547, 16378, 17317, 16339, 16086, 16362, 16363] avg=16384.0000 max=17317 max_rank=3 min=15680 min_rank=0 imbalance_ratio=1.056946 max_load_reduction=113755 max_load_reduction_pct=0.867882 moved_to_echo=65150 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3636, 0, 1103, 0, 0, 582, 0, 4096, 547, 0, 0, 0, 0, 4095, 995, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 8, 0, 4096, 0, 1353, 4096, 0, 0, 0, 4094, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4085, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3109, 2694, 0, 0, 1865, 0, 0, 0, 0, 0, 4076, 4079, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 28, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 14, 4039, 24, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2212, 0, 0, 457, 3993, 0, 381, 0, 2658, 0, 0, 4038, 0, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 121, 4092, 4096, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4054, 0, 513, 0, 4085, 14, 0, 10, 0, 0, 4096, 2743, 883, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16001, 15779, 16400, 15838, 16402, 17706, 16479, 16467] avg=16384.0000 max=17706 max_rank=5 min=15779 min_rank=1 imbalance_ratio=1.080688 max_load_reduction=113366 max_load_reduction_pct=0.864914 moved_to_echo=67054 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3893, 0, 739, 0, 0, 521, 0, 4093, 383, 0, 0, 0, 0, 4095, 992, 948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 821, 0, 0, 4095, 0, 1353, 4095, 0, 0, 2, 4096, 2638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 4089, 0, 0, 4096, 2, 0, 0, 0, 4095, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3126, 1787, 0, 1, 1510, 1, 0, 0, 0, 0, 4080, 4082, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 1, 4096, 0, 0, 4095, 0, 11, 4076, 22, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2218, 0, 0, 541, 4063, 0, 514, 0, 2447, 1, 1, 4082, 0, 0, 0, 3924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 86, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 21, 4095, 4095, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4095, 0, 741, 0, 4094, 9, 0, 1, 0, 3, 4092, 2743, 877, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15664, 17100, 16376, 14590, 16404, 17791, 16430, 16717] avg=16384.0000 max=17791 max_rank=5 min=14590 min_rank=3 imbalance_ratio=1.085876 max_load_reduction=113281 max_load_reduction_pct=0.864265 moved_to_echo=67342 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3547, 0, 225, 0, 0, 491, 0, 4094, 107, 0, 0, 0, 0, 4095, 994, 946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 572, 6, 0, 4095, 0, 1353, 4095, 0, 0, 0, 4093, 1615, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4089, 0, 0, 4095, 4, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3120, 3141, 0, 0, 2650, 0, 0, 0, 0, 0, 4080, 4067, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 27, 4075, 22, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2222, 0, 0, 989, 4024, 0, 1122, 0, 2223, 1, 0, 4036, 0, 0, 0, 3720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 2, 0, 0, 34, 4096, 4094, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4058, 0, 235, 0, 4093, 7, 0, 4, 0, 2, 4083, 2743, 879, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14499, 15830, 16383, 17060, 16421, 18337, 16374, 16168] avg=16384.0000 max=18337 max_rank=5 min=14499 min_rank=0 imbalance_ratio=1.119202 max_load_reduction=112735 max_load_reduction_pct=0.860100 moved_to_echo=67300 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3518, 0, 1445, 0, 0, 2253, 0, 4096, 31, 0, 0, 0, 0, 4096, 994, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 4096, 0, 1353, 4096, 0, 0, 0, 4096, 1765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 4096, 10, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3126, 3690, 0, 0, 2781, 0, 0, 0, 0, 0, 4089, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 1, 4059, 22, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2222, 0, 0, 53, 4037, 0, 70, 0, 880, 0, 0, 4050, 0, 0, 0, 3403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 4096, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 158, 0, 4096, 4, 0, 0, 0, 0, 4096, 2743, 879, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17380, 15965, 16389, 17775, 16378, 14715, 16337, 16133] avg=16384.0000 max=17775 max_rank=3 min=14715 min_rank=5 imbalance_ratio=1.084900 max_load_reduction=113297 max_load_reduction_pct=0.864388 moved_to_echo=63563 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3690, 0, 642, 0, 0, 489, 0, 4096, 130, 0, 0, 0, 0, 4096, 994, 948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 794, 0, 0, 4096, 0, 1353, 4096, 0, 0, 0, 4096, 3332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 1, 0, 0, 1, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3126, 2785, 0, 0, 2428, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 22, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 0, 0, 273, 4065, 0, 113, 0, 1692, 0, 0, 4089, 0, 0, 0, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 18, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 55, 4096, 4096, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 160, 0, 4096, 2, 0, 0, 0, 1, 4096, 2743, 879, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15085, 17767, 16376, 16525, 16413, 16380, 16397, 16129] avg=16384.0000 max=17767 max_rank=1 min=15085 min_rank=0 imbalance_ratio=1.084412 max_load_reduction=113305 max_load_reduction_pct=0.864449 moved_to_echo=65319 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3750, 0, 2794, 0, 0, 1613, 0, 4090, 446, 1, 0, 0, 0, 4094, 993, 944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 4092, 0, 1353, 4093, 0, 0, 0, 4094, 2832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4089, 4, 0, 4096, 0, 0, 0, 0, 4096, 0, 17, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3115, 2739, 0, 0, 1297, 1, 0, 0, 0, 0, 4063, 4054, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 0, 0, 0, 0, 2, 4095, 0, 0, 4096, 2, 24, 3993, 22, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2221, 0, 0, 149, 4014, 0, 93, 0, 1325, 1, 0, 4060, 0, 0, 0, 3031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 9, 0, 4078, 0, 0, 0, 0, 2, 0, 0, 15, 4090, 4096, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4084, 0, 548, 0, 4095, 15, 1, 3, 0, 20, 4092, 2743, 878, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18725, 16558, 16398, 15273, 16350, 14894, 16328, 16546] avg=16384.0000 max=18725 max_rank=0 min=14894 min_rank=5 imbalance_ratio=1.142883 max_load_reduction=112347 max_load_reduction_pct=0.857140 moved_to_echo=64118 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3816, 0, 1812, 0, 0, 1773, 0, 4096, 503, 0, 0, 1, 0, 4096, 994, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 0, 0, 4096, 0, 1353, 4096, 0, 0, 0, 4096, 1436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 1, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3123, 3954, 0, 0, 1428, 0, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 8, 0, 4040, 22, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2223, 0, 0, 105, 4060, 0, 21, 0, 930, 0, 0, 4039, 0, 0, 0, 3785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 37, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 591, 0, 4096, 0, 0, 0, 0, 2, 4096, 2743, 879, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18038, 15526, 16372, 16694, 16365, 15163, 16365, 16549] avg=16384.0000 max=18038 max_rank=0 min=15163 min_rank=5 imbalance_ratio=1.100952 max_load_reduction=113034 max_load_reduction_pct=0.862381 moved_to_echo=64442 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[17792, 0, 0, 2816, 32384, 0, 2560, 0, 14592, 128, 128, 32512, 0, 0, 128, 28928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 29824, 0, 9600, 0, 0, 8704, 0, 32768, 2176, 128, 0, 128, 0, 32768, 8064, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 0, 0, 0, 32768, 0, 3456, 0, 32768, 128, 128, 128, 0, 128, 32768, 22016, 7040, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[25088, 23936, 0, 128, 17024, 128, 0, 0, 0, 0, 32768, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 0, 32768, 128, 0, 0, 128, 32768, 0, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 256, 128, 0, 128, 0, 32768, 128, 0, 0, 0, 128, 128, 0, 384, 32768, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 0, 0, 4224, 128, 0, 32768, 0, 10880, 32768, 0, 0, 128, 32768, 17792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 32768, 128, 0, 0, 0, 0, 128, 32768, 0, 0, 32768, 128, 128, 32384, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 48, 4096, 0, 0, 6, 4096, 4096, 0, 0, 0, 4096, 0, 4073, 1, 0, 4092, 0, 0, 0, 4096, 0, 142, 4093, 0, 0, 4096, 0, 0, 4096, 3950, 0, 0, 0, 0, 4096, 0, 858, 4096, 4, 4096, 0, 0, 29, 0, 4096, 4096, 2651, 0, 0, 4096, 4058, 0, 0, 0, 0, 0, 3, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 0, 984, 0, 0, 13, 0, 0, 0, 4073, 0, 0, 2592, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 912, 4096, 0, 0, 0, 185, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 23, 4084, 1, 0, 57, 4095, 4096, 0, 0, 0, 4096, 0, 4084, 3, 0, 4047, 0, 0, 4, 4092, 0, 755, 4096, 0, 0, 4093, 0, 0, 4096, 3315, 1, 0, 0, 0, 4095, 1, 1056, 4096, 12, 4075, 0, 0, 13, 0, 4096, 4096, 3671, 0, 0, 4096, 4072, 0, 0, 0, 0, 1, 10, 4, 0, 4096, 0, 0, 0, 4, 4095, 0, 0, 4093, 4086, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4091, 4095, 0, 0, 0, 0, 253, 0, 0, 11, 0, 0, 10, 4026, 0, 0, 1038, 1, 0, 3, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 2068, 4096, 0, 0, 0, 168, 0, 0, 0, 0, 0, 4096, 5, 13, 0, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 27, 4091, 5, 0, 1, 4087, 4096, 0, 0, 0, 4096, 0, 3838, 0, 0, 3999, 0, 0, 11, 4083, 0, 225, 4096, 0, 0, 4091, 0, 0, 4096, 3071, 0, 0, 0, 0, 4096, 0, 778, 4096, 46, 4020, 0, 0, 0, 0, 4096, 4096, 3023, 0, 0, 4096, 4001, 0, 0, 0, 0, 3, 37, 12, 0, 4096, 0, 0, 0, 1, 4085, 0, 0, 4096, 4070, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4084, 4096, 0, 0, 0, 0, 637, 0, 0, 45, 0, 14, 0, 4043, 0, 0, 1981, 0, 0, 0, 0, 6, 0, 0, 4096, 1, 0, 0, 0, 2851, 4096, 0, 0, 0, 143, 0, 0, 0, 0, 0, 4096, 0, 30, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 4094, 0, 0, 2, 4096, 4096, 0, 0, 0, 4096, 0, 4088, 2, 0, 4095, 0, 0, 0, 4096, 0, 233, 4096, 0, 0, 4096, 0, 0, 4096, 3708, 0, 0, 0, 0, 4096, 0, 1196, 4096, 0, 4089, 0, 3, 7, 0, 4096, 4096, 2929, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 7, 1, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 465, 0, 0, 6, 0, 0, 0, 4087, 0, 0, 1001, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1793, 4096, 0, 0, 0, 964, 0, 0, 0, 0, 0, 4096, 2, 13, 0, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 4092, 0, 0, 6, 4095, 4096, 0, 0, 0, 4096, 0, 3915, 0, 0, 4095, 0, 0, 10, 4093, 0, 349, 4096, 0, 0, 4095, 0, 0, 4096, 3904, 0, 0, 0, 0, 4095, 0, 2052, 4096, 32, 3688, 0, 0, 31, 0, 4096, 4096, 3932, 0, 1, 4096, 4090, 1, 0, 0, 0, 0, 19, 10, 0, 4096, 0, 0, 0, 2, 4094, 0, 0, 4093, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 4095, 0, 0, 0, 0, 154, 0, 0, 39, 0, 1, 0, 4055, 0, 0, 523, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 757, 4096, 0, 0, 0, 392, 0, 0, 0, 0, 0, 4096, 5, 724, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 4096, 0, 0, 39, 4096, 4096, 0, 0, 0, 4095, 0, 4091, 15, 0, 3943, 0, 0, 0, 4096, 0, 98, 4056, 0, 0, 4096, 0, 0, 4094, 3548, 0, 3, 0, 0, 4096, 0, 1009, 4096, 0, 4095, 0, 0, 2, 0, 4096, 4096, 2512, 0, 0, 4096, 4056, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 4096, 0, 0, 2, 0, 763, 0, 0, 0, 0, 0, 2, 4079, 0, 0, 973, 0, 0, 26, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3388, 4094, 0, 0, 0, 163, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 4081, 3, 0, 52, 4091, 4096, 0, 0, 0, 4096, 0, 4014, 0, 0, 3950, 0, 0, 15, 4085, 0, 108, 4096, 0, 0, 4093, 0, 0, 4096, 3679, 0, 0, 0, 0, 4096, 1, 1337, 4096, 40, 4088, 0, 0, 31, 0, 4096, 4096, 2572, 0, 0, 4096, 4069, 0, 0, 0, 0, 2, 13, 7, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 4096, 4075, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 4086, 4094, 0, 0, 0, 0, 509, 0, 0, 25, 0, 2, 0, 4000, 0, 0, 2460, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 1776, 4096, 0, 0, 0, 70, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 30, 4087, 0, 0, 19, 4095, 4096, 0, 0, 0, 4096, 0, 4060, 0, 0, 3971, 0, 0, 6, 4092, 0, 359, 4096, 0, 0, 4095, 0, 0, 4096, 3842, 1, 0, 0, 0, 4095, 2, 1586, 4096, 3, 4018, 0, 2, 0, 0, 4096, 4096, 3467, 0, 0, 4096, 4089, 0, 0, 0, 0, 1, 13, 1, 0, 4096, 0, 0, 0, 1, 4094, 0, 0, 4096, 4084, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4094, 4095, 0, 0, 0, 0, 470, 0, 0, 8, 0, 0, 0, 4054, 0, 0, 1379, 0, 0, 0, 0, 2, 0, 0, 4096, 2, 0, 0, 0, 1339, 4096, 0, 0, 0, 78, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 23, 4084, 1, 0, 57, 4095, 4061, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 3, 0, 4047, 0, 0, 4, 3958, 0, 755, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3374, 3315, 1, 0, 0, 0, 4095, 1, 1056, 0, 12, 4075, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1016, 3671, 0, 0, 0, 4072, 0, 0, 0, 0, 1, 10, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 0, 0, 4093, 4086, 0, 0, 0, 0, 1, 0, 0, 3358, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3594, 0, 4091, 4095, 0, 0, 0, 0, 253, 0, 0, 11, 0, 0, 10, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 1, 0, 3, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 2068, 4096, 4096, 735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 168, 0, 0, 0, 0, 0, 4096, 5, 13, 0, 4095, 0, 2332, 722, 502, 134, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16417, 16947, 15942, 16966, 16385, 16080, 16137, 16198] avg=16384.0000 max=16966 max_rank=3 min=15942 min_rank=2 imbalance_ratio=1.035522 max_load_reduction=114106 max_load_reduction_pct=0.870560 moved_to_echo=64800 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 27, 4091, 5, 0, 1, 4087, 4057, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3838, 0, 0, 3999, 0, 0, 11, 3944, 0, 225, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3369, 3071, 0, 0, 0, 0, 4096, 0, 778, 0, 46, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1007, 3023, 0, 0, 0, 4001, 0, 0, 0, 0, 3, 37, 12, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4085, 0, 0, 4096, 4070, 0, 0, 0, 0, 0, 0, 0, 3357, 752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3590, 0, 4084, 4096, 0, 0, 0, 0, 637, 0, 0, 45, 0, 14, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1981, 0, 0, 0, 0, 6, 0, 0, 4096, 1, 0, 0, 0, 2851, 4096, 4096, 734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 143, 0, 0, 0, 0, 0, 4096, 0, 30, 0, 4096, 0, 2337, 727, 506, 139, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16364, 16113, 15380, 16275, 16361, 16509, 17861, 16209] avg=16384.0000 max=17861 max_rank=6 min=15380 min_rank=2 imbalance_ratio=1.090149 max_load_reduction=113211 max_load_reduction_pct=0.863731 moved_to_echo=66940 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 48, 4096, 0, 0, 6, 4096, 4061, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 1, 0, 4092, 0, 0, 0, 3962, 0, 142, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3374, 3950, 0, 0, 0, 0, 4096, 0, 858, 0, 4, 4096, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1016, 2651, 0, 0, 0, 4058, 0, 0, 0, 0, 0, 3, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3360, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3592, 0, 4096, 4096, 0, 0, 0, 0, 984, 0, 0, 13, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2592, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 912, 4096, 4096, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 185, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4095, 0, 2332, 722, 502, 134, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16403, 16363, 16407, 15922, 16396, 16854, 16529, 16198] avg=16384.0000 max=16854 max_rank=5 min=15922 min_rank=3 imbalance_ratio=1.028687 max_load_reduction=114218 max_load_reduction_pct=0.871414 moved_to_echo=65977 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 12, 4092, 0, 0, 6, 4095, 4061, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3915, 0, 0, 4095, 0, 0, 10, 3959, 0, 349, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3374, 3904, 0, 0, 0, 0, 4095, 0, 2052, 0, 32, 3688, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1016, 3932, 0, 1, 0, 4090, 1, 0, 0, 0, 0, 19, 10, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 0, 0, 4093, 4082, 0, 0, 0, 0, 0, 0, 0, 3359, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3594, 0, 4092, 4095, 0, 0, 0, 0, 154, 0, 0, 39, 0, 1, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 523, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 757, 4096, 4096, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 392, 0, 0, 0, 0, 0, 4096, 5, 724, 0, 4096, 0, 2332, 722, 502, 134, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16362, 16424, 17176, 17261, 16378, 16030, 14307, 17134] avg=16384.0000 max=17261 max_rank=3 min=14307 min_rank=6 imbalance_ratio=1.053528 max_load_reduction=113811 max_load_reduction_pct=0.868309 moved_to_echo=63849 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3, 4094, 0, 0, 2, 4096, 4061, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 2, 0, 4095, 0, 0, 0, 3962, 0, 233, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3374, 3708, 0, 0, 0, 0, 4096, 0, 1196, 0, 0, 4089, 0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1016, 2929, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 7, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 3361, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3594, 0, 4096, 4096, 0, 0, 1, 0, 465, 0, 0, 6, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1793, 4096, 4096, 735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 964, 0, 0, 0, 0, 0, 4096, 2, 13, 0, 4093, 0, 2332, 722, 502, 134, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16352, 16476, 16473, 16220, 16399, 16345, 15818, 16989] avg=16384.0000 max=16989 max_rank=7 min=15818 min_rank=6 imbalance_ratio=1.036926 max_load_reduction=114083 max_load_reduction_pct=0.870384 moved_to_echo=65551 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 6, 4096, 0, 0, 39, 4096, 4061, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 15, 0, 3943, 0, 0, 0, 3962, 0, 98, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3372, 3548, 0, 3, 0, 0, 4096, 0, 1009, 0, 0, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1016, 2512, 0, 0, 0, 4056, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3360, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3591, 0, 4096, 4096, 0, 0, 2, 0, 763, 0, 0, 0, 0, 0, 2, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 973, 0, 0, 26, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3388, 4096, 4096, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 163, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4093, 0, 2332, 722, 502, 134, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16393, 16165, 16125, 15778, 16397, 16629, 17411, 16174] avg=16384.0000 max=17411 max_rank=6 min=15778 min_rank=3 imbalance_ratio=1.062683 max_load_reduction=113661 max_load_reduction_pct=0.867165 moved_to_echo=66611 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 9, 4081, 3, 0, 52, 4091, 4061, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 0, 3950, 0, 0, 15, 3951, 0, 108, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3374, 3679, 0, 0, 0, 0, 4096, 1, 1337, 0, 40, 4088, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1016, 2572, 0, 0, 0, 4069, 0, 0, 0, 0, 2, 13, 7, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4096, 4075, 0, 0, 0, 0, 2, 0, 0, 3359, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3594, 0, 4086, 4094, 0, 0, 0, 0, 509, 0, 0, 25, 0, 2, 0, 4000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2460, 0, 0, 0, 0, 6, 0, 0, 4096, 0, 0, 0, 0, 1776, 4096, 4096, 734, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 70, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 2332, 722, 502, 134, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16393, 16134, 16646, 15871, 16372, 16310, 17264, 16082] avg=16384.0000 max=17264 max_rank=6 min=15871 min_rank=3 imbalance_ratio=1.053711 max_load_reduction=113808 max_load_reduction_pct=0.868286 moved_to_echo=66028 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 30, 4087, 0, 0, 19, 4095, 4061, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 3971, 0, 0, 6, 3958, 0, 359, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3374, 3842, 1, 0, 0, 0, 4095, 2, 1586, 0, 3, 4018, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1016, 3467, 0, 0, 0, 4089, 0, 0, 0, 0, 1, 13, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 4096, 4084, 0, 0, 0, 0, 1, 0, 0, 3360, 748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3594, 0, 4094, 4095, 0, 0, 0, 0, 470, 0, 0, 8, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1379, 0, 0, 0, 0, 2, 0, 0, 4096, 2, 0, 0, 0, 1339, 4096, 4096, 735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 78, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 4093, 0, 2332, 722, 502, 134, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16450, 16923, 16779, 16384, 16315, 15745, 16088] avg=16384.0000 max=16923 max_rank=2 min=15745 min_rank=6 imbalance_ratio=1.032898 max_load_reduction=114149 max_load_reduction_pct=0.870888 moved_to_echo=64532 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 8192, 24832, 0, 128, 0, 32512, 128, 0, 0, 0, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 256, 32768, 128, 0, 256, 32768, 32512, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 12032, 128, 0, 128, 0, 128, 0, 0, 32768, 128, 0, 0, 0, 14976, 32768, 32768, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32256, 128, 0, 32256, 0, 0, 128, 31744, 0, 2304, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[27008, 29056, 128, 128, 0, 0, 32768, 128, 9984, 0, 256, 32256, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[28800, 0, 32768, 32768, 0, 0, 128, 0, 4352, 0, 0, 256, 0, 128, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 0, 0, 0, 2176, 0, 0, 0, 0, 0, 32768, 128, 896, 0, 32768, 0, 18688, 5888, 4096, 1152, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 32768, 0, 0, 32768, 32768, 0, 0, 0, 128, 128, 0, 0, 26880, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[7, 0, 4076, 0, 0, 4056, 4096, 0, 0, 0, 59, 16, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 4096, 4089, 0, 4096, 0, 4096, 3819, 0, 0, 0, 4, 2082, 1706, 0, 0, 0, 4096, 0, 0, 0, 116, 1, 0, 0, 123, 4095, 0, 0, 0, 4096, 0, 0, 8, 3617, 4096, 4, 4070, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4076, 0, 0, 3996, 4096, 4088, 0, 3201, 0, 4095, 0, 0, 244, 0, 0, 40, 0, 1622, 4084, 0, 0, 0, 10, 0, 4096, 0, 0, 4029, 28, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[261, 0, 4091, 0, 0, 3926, 4096, 0, 0, 0, 93, 32, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4096, 4093, 0, 4096, 0, 4096, 3868, 0, 0, 0, 0, 401, 724, 0, 0, 0, 4096, 0, 0, 0, 97, 0, 0, 0, 653, 4096, 0, 0, 0, 4096, 0, 0, 0, 3848, 4096, 10, 4068, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4054, 0, 0, 4044, 4096, 4084, 0, 3393, 0, 4092, 0, 0, 513, 0, 0, 48, 0, 2922, 4082, 0, 0, 0, 33, 0, 4096, 0, 0, 3916, 41, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[277, 0, 4088, 0, 0, 4095, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 4096, 0, 4096, 0, 4096, 4025, 0, 0, 0, 0, 581, 1305, 0, 0, 0, 4096, 0, 0, 0, 32, 0, 0, 0, 81, 4096, 0, 0, 0, 4096, 0, 0, 0, 4057, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4073, 4096, 4096, 0, 3520, 0, 4096, 0, 0, 340, 0, 0, 0, 0, 2201, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4087, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[56, 0, 4074, 0, 0, 4092, 4096, 0, 0, 0, 38, 3, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 4091, 0, 4096, 0, 4095, 3985, 0, 0, 0, 2, 292, 1561, 0, 0, 2, 4096, 0, 0, 0, 69, 0, 0, 0, 166, 4096, 0, 0, 5, 4096, 0, 0, 1, 4020, 4096, 2, 4073, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4092, 0, 0, 4093, 4096, 4091, 0, 3266, 0, 4093, 0, 0, 247, 0, 0, 14, 0, 2836, 4074, 0, 0, 0, 5, 0, 4096, 0, 0, 3993, 16, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[28, 0, 4095, 0, 0, 4037, 4096, 0, 0, 0, 44, 5, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 4094, 0, 4096, 0, 4096, 3800, 0, 0, 0, 0, 928, 1732, 0, 0, 1, 4096, 0, 0, 0, 323, 0, 0, 0, 143, 4096, 0, 0, 0, 4096, 0, 0, 3, 3968, 4096, 2, 4059, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4093, 0, 0, 4003, 4096, 4093, 0, 3659, 0, 4095, 0, 0, 183, 0, 0, 91, 0, 1747, 4089, 0, 0, 0, 0, 0, 4096, 0, 0, 4026, 21, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[68, 0, 4090, 0, 0, 4083, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 4073, 0, 0, 0, 0, 650, 1029, 0, 0, 0, 4096, 0, 0, 0, 37, 0, 0, 0, 218, 4096, 0, 0, 2, 4096, 0, 0, 1, 4041, 4096, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 0, 0, 4095, 4096, 4096, 0, 3488, 0, 4096, 0, 0, 846, 0, 0, 10, 0, 1983, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4055, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[115, 0, 4088, 0, 0, 4093, 4092, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 4075, 0, 0, 0, 0, 713, 2796, 0, 0, 0, 4096, 0, 0, 0, 31, 0, 0, 0, 346, 4096, 0, 0, 0, 4096, 0, 0, 2, 4013, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4086, 0, 0, 4066, 4096, 4096, 0, 1376, 0, 4096, 0, 0, 139, 0, 0, 4, 0, 2842, 4095, 0, 0, 0, 1, 0, 4096, 0, 0, 4082, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[46, 0, 4090, 0, 0, 4090, 4096, 0, 0, 0, 19, 7, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4096, 4094, 0, 4096, 0, 4096, 3969, 0, 0, 0, 1, 714, 1891, 0, 0, 0, 4096, 0, 0, 0, 55, 0, 0, 0, 314, 4095, 0, 0, 0, 4096, 0, 0, 3, 3481, 4096, 2, 4075, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4081, 0, 0, 4076, 4096, 4093, 0, 3260, 0, 4094, 0, 0, 81, 0, 0, 19, 0, 2628, 4082, 0, 0, 0, 11, 0, 4096, 0, 0, 4073, 13, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[56, 0, 4074, 0, 0, 4092, 4096, 0, 0, 0, 38, 3, 0, 0, 0, 0, 3437, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 4091, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 4095, 3985, 0, 0, 0, 2, 292, 1561, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 166, 4096, 0, 0, 5, 4096, 0, 0, 1, 4020, 2216, 1741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4073, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4092, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 644, 4091, 0, 3266, 0, 4093, 0, 0, 247, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2836, 4074, 0, 0, 0, 5, 0, 1799, 0, 0, 3993, 16, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16352, 16385, 15886, 16410, 16378, 16394, 16448, 16819] avg=16384.0000 max=16819 max_rank=7 min=15886 min_rank=2 imbalance_ratio=1.026550 max_load_reduction=114253 max_load_reduction_pct=0.871681 moved_to_echo=66039 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[261, 0, 4091, 0, 0, 3926, 4096, 0, 0, 0, 93, 32, 0, 0, 0, 0, 3439, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4096, 4093, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1837, 0, 4096, 3868, 0, 0, 0, 0, 401, 724, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 653, 4096, 0, 0, 0, 4096, 0, 0, 0, 3848, 2221, 1743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 4068, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4054, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 642, 4084, 0, 3393, 0, 4092, 0, 0, 513, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2922, 4082, 0, 0, 0, 33, 0, 1791, 0, 0, 3916, 41, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16500, 16359, 15022, 16754, 16388, 16352, 16816, 16881] avg=16384.0000 max=16881 max_rank=7 min=15022 min_rank=2 imbalance_ratio=1.030334 max_load_reduction=114191 max_load_reduction_pct=0.871208 moved_to_echo=66437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[277, 0, 4088, 0, 0, 4095, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 4, 3437, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 4096, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 4096, 4025, 0, 0, 0, 0, 581, 1305, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 81, 4096, 0, 0, 0, 4096, 0, 0, 0, 4057, 2216, 1741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 644, 4096, 0, 3520, 0, 4096, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2201, 4096, 0, 0, 0, 0, 0, 1799, 0, 0, 4087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16558, 16395, 15956, 16319, 16400, 16396, 16769, 16279] avg=16384.0000 max=16769 max_rank=6 min=15956 min_rank=2 imbalance_ratio=1.023499 max_load_reduction=114303 max_load_reduction_pct=0.872063 moved_to_echo=65844 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[68, 0, 4090, 0, 0, 4083, 4096, 0, 0, 0, 8, 0, 0, 0, 0, 0, 3437, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 4096, 4073, 0, 0, 0, 0, 650, 1029, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 218, 4096, 0, 0, 2, 4096, 0, 0, 1, 4041, 2216, 1741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4094, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 644, 4096, 0, 3488, 0, 4096, 0, 0, 846, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1983, 4096, 0, 0, 0, 1, 0, 1799, 0, 0, 4055, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16338, 16399, 15797, 16448, 16391, 16394, 17275, 16030] avg=16384.0000 max=17275 max_rank=6 min=15797 min_rank=2 imbalance_ratio=1.054382 max_load_reduction=113797 max_load_reduction_pct=0.868202 moved_to_echo=66090 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 4076, 0, 0, 4056, 4096, 0, 0, 0, 59, 16, 0, 0, 0, 0, 3437, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 4096, 4089, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 4096, 3819, 0, 0, 0, 4, 2082, 1706, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 1, 0, 0, 123, 4095, 0, 0, 0, 4096, 0, 0, 8, 3617, 2216, 1741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4070, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4076, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 644, 4088, 0, 3201, 0, 4095, 0, 0, 244, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1622, 4084, 0, 0, 0, 10, 0, 1799, 0, 0, 4029, 28, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16303, 16373, 17656, 16013, 16377, 16374, 16308, 15668] avg=16384.0000 max=17656 max_rank=2 min=15668 min_rank=7 imbalance_ratio=1.077637 max_load_reduction=113416 max_load_reduction_pct=0.865295 moved_to_echo=64727 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[28, 0, 4095, 0, 0, 4037, 4096, 0, 0, 0, 44, 5, 0, 0, 0, 0, 3437, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 4094, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 4096, 3800, 0, 0, 0, 0, 928, 1732, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 323, 0, 0, 0, 143, 4096, 0, 0, 0, 4096, 0, 0, 3, 3968, 2216, 1741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4059, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4093, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 644, 4093, 0, 3659, 0, 4095, 0, 0, 183, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1747, 4089, 0, 0, 0, 0, 0, 1799, 0, 0, 4026, 21, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16298, 16382, 16506, 16586, 16360, 16394, 16768, 15778] avg=16384.0000 max=16768 max_rank=6 min=15778 min_rank=7 imbalance_ratio=1.023438 max_load_reduction=114304 max_load_reduction_pct=0.872070 moved_to_echo=65300 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[115, 0, 4088, 0, 0, 4093, 4092, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3437, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 4096, 4075, 0, 0, 0, 0, 713, 2796, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 346, 4096, 0, 0, 0, 4096, 0, 0, 2, 4013, 2216, 1741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 4096, 4086, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 644, 4096, 0, 1376, 0, 4096, 0, 0, 139, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2842, 4095, 0, 0, 0, 1, 0, 1799, 0, 0, 4082, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16399, 17629, 16541, 16398, 16387, 14421, 16915] avg=16384.0000 max=17629 max_rank=2 min=14421 min_rank=6 imbalance_ratio=1.075989 max_load_reduction=113443 max_load_reduction_pct=0.865501 moved_to_echo=64121 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[46, 0, 4090, 0, 0, 4090, 4096, 0, 0, 0, 19, 7, 0, 0, 0, 0, 3437, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4096, 4094, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1853, 0, 4096, 3969, 0, 0, 0, 1, 714, 1891, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 314, 4095, 0, 0, 0, 4096, 0, 0, 3, 3481, 2216, 1741, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4075, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4081, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 644, 4093, 0, 3260, 0, 4094, 0, 0, 81, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2628, 4082, 0, 0, 0, 11, 0, 1799, 0, 0, 4073, 13, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16341, 16380, 16620, 16001, 16380, 16381, 16267, 16702] avg=16384.0000 max=16702 max_rank=7 min=16001 min_rank=3 imbalance_ratio=1.019409 max_load_reduction=114370 max_load_reduction_pct=0.872574 moved_to_echo=65730 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[18816, 32768, 0, 0, 0, 128, 0, 14464, 0, 0, 32384, 128, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[32768, 128, 32640, 32768, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 14848, 0, 32768, 31616, 0, 0, 0, 128, 6400, 12800, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[896, 0, 32768, 0, 0, 32512, 32768, 0, 0, 0, 384, 128, 0, 0, 0, 128, 27520, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 768, 128, 0, 0, 2048, 32768, 0, 0, 128, 32768, 0, 0, 128, 31104, 17792, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 32512, 5248, 32768, 0, 25216, 0, 32768, 0, 0, 2688, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 32640, 32768, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 32768, 0, 0, 128, 0, 0, 32768, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4084, 0, 0, 142, 0, 2308, 0, 0, 0, 49, 0, 1512, 0, 0, 0, 1, 10, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 66, 0, 0, 0, 0, 2640, 0, 0, 0, 4028, 0, 0, 0, 4072, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4077, 0, 3601, 4096, 4091, 0, 0, 0, 9, 4086, 0, 4096, 4096, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 4096, 9, 0, 0, 2090, 0, 0, 16, 0, 3, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4094, 4091, 0, 0, 1306, 0, 1293, 0, 0, 0, 17, 0, 1245, 0, 0, 0, 201, 2, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4094, 4096, 0, 0, 4, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4086, 4095, 4096, 46, 0, 0, 2, 0, 843, 0, 0, 0, 4095, 0, 0, 0, 4089, 0, 1, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 4088, 0, 3919, 4060, 4095, 0, 0, 0, 7, 4091, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 69, 0, 0, 3397, 0, 0, 2, 0, 10, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 629, 0, 2852, 0, 0, 0, 7, 0, 566, 0, 0, 0, 28, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4095, 29, 0, 0, 0, 0, 2203, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3877, 4096, 4084, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4051, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 78, 0, 0, 2043, 0, 0, 0, 0, 1, 0, 0, 0, 43, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 416, 0, 1021, 0, 0, 0, 1, 0, 1895, 0, 0, 0, 31, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4094, 4096, 4095, 4, 0, 0, 0, 0, 2065, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 0, 3988, 4090, 4093, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4029, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 177, 0, 0, 2729, 0, 0, 0, 0, 1, 0, 0, 0, 40, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4094, 4096, 4094, 0, 0, 149, 0, 2195, 0, 0, 0, 34, 0, 1769, 0, 0, 0, 56, 5, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4089, 4096, 4096, 14, 0, 0, 0, 0, 1627, 0, 0, 0, 4096, 0, 0, 0, 4079, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 3877, 4096, 4095, 0, 0, 0, 3, 4084, 0, 4096, 4096, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 181, 0, 0, 2433, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4095, 4093, 0, 0, 304, 0, 826, 0, 0, 0, 18, 0, 1772, 0, 0, 0, 0, 3, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 377, 0, 0, 0, 0, 2789, 0, 0, 0, 4095, 0, 0, 0, 4073, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4078, 0, 3890, 4096, 4096, 0, 0, 0, 4, 4084, 0, 4096, 4096, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 226, 0, 0, 2134, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 303, 0, 1743, 0, 0, 0, 3, 0, 2589, 0, 0, 0, 210, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 4096, 4096, 4094, 3, 0, 0, 2, 0, 1335, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3978, 4081, 4091, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 165, 0, 0, 1931, 0, 0, 0, 0, 6, 0, 0, 0, 46, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4095, 4095, 0, 0, 278, 0, 1250, 0, 0, 0, 10, 0, 2172, 0, 0, 0, 0, 3, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 2, 1, 0, 0, 4096, 0, 0, 0, 1, 0, 4063, 4096, 4096, 43, 0, 0, 0, 0, 1389, 0, 0, 0, 4070, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 3945, 4096, 4096, 0, 0, 0, 0, 4091, 0, 4096, 4096, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 134, 0, 0, 3144, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1972, 0, 0, 0, 0, 0, 4096, 4096, 4084, 0, 0, 142, 0, 2308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 1512, 0, 0, 0, 1, 10, 0, 4096, 4096, 0, 0, 0, 0, 4096, 2304, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2180, 4096, 66, 0, 0, 0, 0, 2640, 0, 0, 0, 4028, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4077, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3601, 4096, 230, 0, 0, 0, 9, 4086, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 4096, 9, 0, 0, 2090, 0, 0, 2124, 1906, 1557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16698, 16188, 16393, 17082, 16375, 16097, 15858, 16381] avg=16384.0000 max=17082 max_rank=3 min=15858 min_rank=6 imbalance_ratio=1.042603 max_load_reduction=113990 max_load_reduction_pct=0.869675 moved_to_echo=64711 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1973, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 629, 0, 2852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 566, 0, 0, 0, 28, 0, 0, 4096, 4096, 0, 0, 0, 0, 4092, 2294, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2185, 4095, 29, 0, 0, 0, 0, 2203, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 4096, 240, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 78, 0, 0, 2043, 0, 0, 2123, 1902, 1550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 43, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17742, 15203, 16388, 16700, 16390, 16360, 15885, 16404] avg=16384.0000 max=17742 max_rank=0 min=15203 min_rank=1 imbalance_ratio=1.082886 max_load_reduction=113330 max_load_reduction_pct=0.864639 moved_to_echo=65039 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1973, 0, 0, 0, 0, 0, 4095, 4094, 4091, 0, 0, 1306, 0, 1293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 1245, 0, 0, 0, 201, 2, 0, 4096, 4096, 0, 0, 0, 0, 4092, 2300, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 4096, 0, 0, 4, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2186, 4096, 46, 0, 0, 2, 0, 843, 0, 0, 0, 4095, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 4088, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3919, 4060, 241, 0, 0, 0, 7, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 69, 0, 0, 3397, 0, 0, 2123, 1901, 1554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 10, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16852, 16072, 16381, 15357, 16382, 16414, 17232, 16382] avg=16384.0000 max=17232 max_rank=6 min=15357 min_rank=3 imbalance_ratio=1.051758 max_load_reduction=113840 max_load_reduction_pct=0.868530 moved_to_echo=66410 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1973, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 303, 0, 1743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2589, 0, 0, 0, 210, 0, 0, 4096, 4096, 0, 0, 0, 0, 4092, 2298, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2185, 4094, 3, 0, 0, 2, 0, 1335, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 4081, 241, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 165, 0, 0, 1931, 0, 0, 2123, 1902, 1552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 46, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16307, 17408, 16389, 15808, 16391, 16492, 15865, 16412] avg=16384.0000 max=17408 max_rank=1 min=15808 min_rank=3 imbalance_ratio=1.062500 max_load_reduction=113664 max_load_reduction_pct=0.867188 moved_to_echo=65160 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1973, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 416, 0, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1895, 0, 0, 0, 31, 0, 0, 4096, 4096, 0, 0, 0, 0, 4092, 2299, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2185, 4095, 4, 0, 0, 0, 0, 2065, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 4090, 241, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 177, 0, 0, 2729, 0, 0, 2123, 1902, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 40, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15698, 16534, 16385, 16540, 16394, 16444, 16676, 16401] avg=16384.0000 max=16676 max_rank=6 min=15698 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=114396 max_load_reduction_pct=0.872772 moved_to_echo=65915 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1973, 0, 0, 0, 0, 0, 4096, 4095, 4093, 0, 0, 304, 0, 826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 1772, 0, 0, 0, 0, 3, 0, 4096, 4096, 0, 0, 0, 0, 4092, 2301, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2185, 4096, 377, 0, 0, 0, 0, 2789, 0, 0, 0, 4095, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4078, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3890, 4096, 241, 0, 0, 0, 4, 4084, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 226, 0, 0, 2134, 0, 0, 2123, 1902, 1554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15387, 16401, 16389, 17615, 16375, 16409, 16128, 16368] avg=16384.0000 max=17615 max_rank=3 min=15387 min_rank=0 imbalance_ratio=1.075134 max_load_reduction=113457 max_load_reduction_pct=0.865608 moved_to_echo=65280 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1973, 0, 0, 0, 0, 0, 4094, 4096, 4094, 0, 0, 149, 0, 2195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 1769, 0, 0, 0, 56, 5, 0, 4096, 4096, 0, 0, 0, 0, 4092, 2300, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2185, 4096, 14, 0, 0, 0, 0, 1627, 0, 0, 0, 4096, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3877, 4096, 241, 0, 0, 0, 3, 4084, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 181, 0, 0, 2433, 0, 0, 2123, 1902, 1554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16601, 16472, 16388, 16097, 16380, 16394, 16380, 16360] avg=16384.0000 max=16601 max_rank=0 min=16097 min_rank=3 imbalance_ratio=1.013245 max_load_reduction=114471 max_load_reduction_pct=0.873344 moved_to_echo=65514 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1973, 0, 0, 0, 0, 0, 4096, 4095, 4095, 0, 0, 278, 0, 1250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 2172, 0, 0, 0, 0, 3, 0, 4096, 4096, 0, 0, 0, 0, 4092, 2301, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 4096, 0, 0, 2, 1, 0, 0, 4096, 0, 0, 0, 1, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2185, 4096, 43, 0, 0, 0, 0, 1389, 0, 0, 0, 4070, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 4096, 241, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 134, 0, 0, 3144, 0, 0, 2123, 1902, 1554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15787, 16794, 16359, 15873, 16385, 16462, 17048, 16364] avg=16384.0000 max=17048 max_rank=6 min=15787 min_rank=0 imbalance_ratio=1.040527 max_load_reduction=114024 max_load_reduction_pct=0.869934 moved_to_echo=66259 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[15872, 0, 0, 0, 0, 0, 32768, 32768, 32768, 0, 0, 3584, 0, 13568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[31104, 32768, 1920, 0, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 32768, 32768, 0, 0, 128, 128, 0, 0, 32768, 128, 0, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 0, 0, 32768, 0, 0, 0, 128, 0, 0, 0, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 256, 0, 13568, 0, 0, 0, 640, 128, 0, 32768, 32768, 0, 0, 0, 0, 32768, 18432, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 32768, 0, 32768, 1152, 0, 0, 19968, 0, 0, 17024, 15232, 12544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[17536, 32768, 640, 0, 0, 128, 0, 14976, 0, 0, 0, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[128, 0, 128, 0, 0, 0, 256, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 32768, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 3778, 0, 0, 0, 0, 0, 0, 11, 0, 2, 0, 4096, 0, 0, 4096, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 4091, 4096, 0, 0, 4088, 0, 0, 7, 4095, 0, 15, 4086, 4096, 1, 0, 0, 0, 1, 4094, 697, 62, 0, 10, 4088, 0, 0, 4096, 0, 0, 1, 0, 4090, 0, 0, 0, 0, 929, 0, 0, 0, 4096, 0, 4096, 4096, 0, 15, 4096, 4096, 4095, 4089, 3492, 0, 20, 0, 4, 0, 3694, 0, 4095, 4096, 0, 0, 0, 4096, 0, 0, 1, 3753, 0, 0, 0, 0, 4096, 0, 0, 4030, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 2, 0, 0, 3600, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 1, 4096, 4096, 0, 0, 0, 0, 0, 4096, 2503, 14, 0, 21, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 662, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 3164, 0, 6, 0, 0, 0, 4061, 0, 4096, 4088, 0, 0, 0, 4096, 0, 0, 0, 2347, 0, 0, 0, 0, 4096, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 10, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 1400, 17, 0, 25, 4093, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1535, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 3687, 0, 30, 0, 0, 0, 4017, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 3048, 0, 0, 0, 0, 4096, 0, 0, 4077, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 1450, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3, 4096, 0, 7, 0, 0, 3, 0, 0, 4096, 0, 0, 4094, 4094, 0, 0, 4074, 0, 0, 6, 4094, 0, 17, 4084, 4096, 0, 0, 0, 0, 0, 4096, 2252, 0, 0, 4, 4087, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1782, 0, 0, 0, 4096, 0, 4096, 4096, 0, 10, 4096, 4096, 4096, 4092, 3999, 0, 48, 0, 2, 0, 4064, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 2811, 0, 0, 0, 0, 4096, 0, 0, 4048, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3754, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 4096, 0, 3, 0, 0, 0, 0, 0, 4096, 10, 0, 4091, 4095, 0, 0, 4083, 2, 0, 12, 4094, 0, 33, 4083, 4096, 0, 0, 0, 0, 2, 4093, 1066, 10, 0, 7, 4083, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 299, 0, 0, 0, 4096, 0, 4096, 4095, 0, 10, 4096, 4096, 4096, 4088, 3392, 0, 126, 0, 5, 0, 4083, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 3692, 0, 0, 0, 0, 4096, 0, 0, 4023, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3609, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 11, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 2, 0, 4088, 4084, 0, 0, 4073, 0, 0, 14, 4096, 0, 61, 4066, 4096, 0, 0, 0, 0, 0, 4095, 1962, 8, 0, 35, 4084, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1007, 0, 0, 0, 4096, 0, 4096, 4091, 0, 22, 4096, 4096, 4096, 4086, 3962, 0, 231, 0, 8, 0, 4076, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 1511, 0, 0, 0, 0, 4096, 0, 0, 4014, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 473, 3, 0, 33, 4096, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 194, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 3797, 0, 4, 0, 0, 0, 4092, 0, 4096, 4090, 0, 0, 0, 4096, 0, 0, 7, 3744, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3575, 0, 0, 1, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 23, 0, 0, 0, 0, 0, 4096, 6, 0, 4094, 4096, 0, 0, 4087, 2, 0, 4, 4094, 0, 18, 4067, 4096, 0, 0, 0, 0, 0, 4089, 1201, 20, 0, 132, 4086, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 428, 0, 0, 0, 4096, 0, 4096, 4096, 0, 8, 4096, 4096, 4095, 4094, 4012, 0, 71, 0, 2, 0, 4090, 0, 4096, 4085, 0, 0, 0, 4096, 0, 0, 0, 2876, 0, 0, 0, 0, 4096, 0, 0, 4073, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4095, 0, 0, 0, 0, 3778, 0, 0, 0, 0, 0, 0, 11, 4096, 4096, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 2867, 1219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 0, 4091, 4096, 0, 0, 4088, 0, 0, 7, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4086, 2604, 1, 0, 0, 0, 1, 4094, 697, 62, 0, 10, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4090, 0, 0, 0, 0, 929, 0, 0, 0, 4096, 0, 1750, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 410, 4089, 3492, 0, 20, 0, 4, 0, 3694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 1127, 0, 0, 1, 3753, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 3320, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16852, 16379, 16385, 15658, 16454, 15820, 17168, 16356] avg=16384.0000 max=17168 max_rank=6 min=15658 min_rank=3 imbalance_ratio=1.047852 max_load_reduction=113904 max_load_reduction_pct=0.869019 moved_to_echo=65798 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 2, 0, 0, 3600, 0, 0, 0, 0, 0, 0, 8, 4096, 4096, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 2868, 1219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 2604, 0, 0, 0, 0, 0, 4096, 2503, 14, 0, 21, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 662, 0, 0, 0, 4096, 0, 1750, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 409, 4096, 3164, 0, 6, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 1127, 0, 0, 0, 2347, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3320, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16674, 16384, 16388, 17430, 16192, 15832, 15754, 16418] avg=16384.0000 max=17430 max_rank=3 min=15754 min_rank=6 imbalance_ratio=1.063843 max_load_reduction=113642 max_load_reduction_pct=0.867020 moved_to_echo=64196 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3609, 0, 0, 0, 0, 0, 0, 49, 4096, 4096, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 11, 4096, 0, 1, 0, 0, 2870, 1221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4088, 4084, 0, 0, 4073, 0, 0, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 4066, 2599, 0, 0, 0, 0, 0, 4095, 1962, 8, 0, 35, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 1007, 0, 0, 0, 4096, 0, 1757, 1497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 22, 0, 0, 404, 4086, 3962, 0, 231, 0, 8, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1118, 0, 0, 0, 1511, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3320, 822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16722, 16391, 16357, 16910, 16547, 16880, 14917, 16348] avg=16384.0000 max=16910 max_rank=3 min=14917 min_rank=6 imbalance_ratio=1.032104 max_load_reduction=114162 max_load_reduction_pct=0.870987 moved_to_echo=64692 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 1450, 0, 0, 0, 0, 0, 0, 26, 4096, 4096, 775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3, 4096, 0, 7, 0, 0, 2868, 1219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5, 0, 0, 4094, 4094, 0, 0, 4074, 0, 0, 6, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4084, 2604, 0, 0, 0, 0, 0, 4096, 2252, 0, 0, 4, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1782, 0, 0, 0, 4096, 0, 1750, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 0, 409, 4092, 3999, 0, 48, 0, 2, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1127, 0, 0, 0, 2811, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 3321, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14539, 16385, 16370, 17144, 17312, 16720, 16226, 16376] avg=16384.0000 max=17312 max_rank=4 min=14539 min_rank=0 imbalance_ratio=1.056641 max_load_reduction=113760 max_load_reduction_pct=0.867920 moved_to_echo=66634 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2640, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2868, 1219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 10, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2604, 0, 0, 0, 0, 0, 4096, 1400, 17, 0, 25, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1535, 0, 0, 0, 4096, 0, 1750, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 409, 4096, 3687, 0, 30, 0, 0, 0, 4017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 1127, 0, 0, 0, 3048, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3320, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15704, 16375, 16398, 16331, 17065, 16335, 16461, 16403] avg=16384.0000 max=17065 max_rank=4 min=15704 min_rank=0 imbalance_ratio=1.041565 max_load_reduction=114007 max_load_reduction_pct=0.869804 moved_to_echo=66264 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3575, 0, 0, 1, 0, 0, 0, 8, 4096, 4096, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 23, 0, 0, 2867, 1219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 4094, 4096, 0, 0, 4087, 2, 0, 4, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4067, 2604, 0, 0, 0, 0, 0, 4089, 1201, 20, 0, 132, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 428, 0, 0, 0, 4096, 0, 1750, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 410, 4094, 4012, 0, 71, 0, 2, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4085, 0, 0, 0, 1127, 0, 0, 0, 2876, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 3320, 818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16648, 16399, 16388, 16217, 15958, 16783, 16280, 16399] avg=16384.0000 max=16783 max_rank=5 min=15958 min_rank=4 imbalance_ratio=1.024353 max_load_reduction=114289 max_load_reduction_pct=0.871956 moved_to_echo=65420 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2868, 1219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2604, 0, 0, 0, 0, 0, 4096, 473, 3, 0, 33, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 194, 0, 0, 0, 4096, 0, 1750, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 409, 4096, 3797, 0, 4, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 1127, 0, 0, 7, 3744, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 3320, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17101, 16375, 16394, 15401, 15725, 16494, 17160, 16422] avg=16384.0000 max=17160 max_rank=6 min=15401 min_rank=3 imbalance_ratio=1.047363 max_load_reduction=113912 max_load_reduction_pct=0.869080 moved_to_echo=65801 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3754, 0, 0, 0, 0, 0, 0, 14, 4096, 4096, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 4096, 0, 3, 0, 0, 2868, 1219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 10, 0, 4091, 4095, 0, 0, 4083, 2, 0, 12, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4083, 2604, 0, 0, 0, 0, 2, 4093, 1066, 10, 0, 7, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 299, 0, 0, 0, 4096, 0, 1750, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 10, 0, 0, 409, 4088, 3392, 0, 126, 0, 5, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 1127, 0, 0, 0, 3692, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4091, 3320, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16832, 16384, 16392, 15981, 15819, 16208, 17106, 16350] avg=16384.0000 max=17106 max_rank=6 min=15819 min_rank=4 imbalance_ratio=1.044067 max_load_reduction=113966 max_load_reduction_pct=0.869492 moved_to_echo=65483 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 32512, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 32768, 26624, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[32768, 32768, 0, 0, 0, 9088, 0, 0, 128, 23808, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 128, 0, 32768, 0, 0, 32768, 0, 0, 0, 128, 32768, 0, 128, 0, 0, 23040, 9856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 0, 32768, 0, 128, 0, 0, 26496, 0, 0, 128, 0, 0, 0, 128, 32768, 32768, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 32768, 0, 128, 0, 0, 3328, 32768, 29568, 0, 640, 0, 128, 0, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 0, 128, 128, 0, 32768, 0, 128, 0, 0, 6912, 0, 0, 0, 32768, 0, 14080, 12032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 0, 128, 128, 0, 32768, 32768, 0, 0, 32768, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[256, 32768, 20864, 128, 0, 0, 0, 128, 32768, 11648, 256, 0, 384, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[752, 0, 0, 1159, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 280, 0, 0, 0, 0, 4087, 2919, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4082, 4096, 0, 0, 0, 3815, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 7, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 4087, 0, 3560, 14, 0, 4053, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[981, 0, 0, 1952, 0, 0, 3270, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 867, 0, 0, 0, 0, 4088, 1750, 0, 0, 0, 4090, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 4091, 4096, 0, 0, 0, 3747, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 12, 0, 4096, 0, 0, 4094, 0, 3840, 6, 0, 4067, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[2773, 0, 0, 1165, 0, 0, 2884, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 109, 0, 0, 0, 0, 4088, 1964, 0, 0, 0, 4092, 0, 0, 0, 0, 3, 0, 4094, 0, 0, 4095, 0, 0, 4096, 0, 4090, 4096, 0, 0, 0, 3763, 1, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 1, 0, 0, 4093, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4095, 0, 0, 0, 1, 1, 0, 4096, 0, 0, 0, 0, 6, 0, 4095, 0, 0, 4093, 0, 3760, 5, 0, 4068, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1195, 0, 0, 894, 0, 0, 3932, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 183, 0, 0, 0, 0, 4096, 2663, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 3661, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3870, 0, 0, 4082, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1431, 0, 0, 1648, 0, 0, 2843, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 184, 0, 0, 0, 0, 4087, 2781, 0, 0, 0, 4087, 0, 0, 0, 0, 7, 0, 4094, 0, 0, 4096, 0, 0, 4096, 0, 4087, 4096, 0, 0, 0, 3842, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 3, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 4090, 0, 3674, 10, 0, 4066, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4092, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1283, 0, 0, 1353, 0, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 239, 0, 0, 0, 0, 4096, 1992, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3798, 0, 0, 4074, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[580, 0, 0, 2526, 0, 0, 3765, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 251, 0, 0, 0, 0, 4096, 1804, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 3828, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3627, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1879, 0, 0, 794, 0, 0, 3060, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 157, 0, 0, 0, 0, 4093, 2591, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 4093, 0, 0, 4095, 0, 0, 4096, 0, 4092, 4096, 0, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4094, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 4093, 0, 0, 1, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 4096, 0, 3944, 2, 0, 4088, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[752, 0, 0, 1159, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1956, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 0, 0, 4087, 2919, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 3466, 2140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4082, 270, 0, 0, 0, 3815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3471, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3742, 355, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 7, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 3560, 14, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 354, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16009, 16981, 16359, 16389, 16401, 16384, 16385, 16164] avg=16384.0000 max=16981 max_rank=1 min=16009 min_rank=0 imbalance_ratio=1.036438 max_load_reduction=114091 max_load_reduction_pct=0.870445 moved_to_echo=65334 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2773, 0, 0, 1165, 0, 0, 2884, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1955, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 4088, 1964, 0, 0, 0, 4092, 0, 0, 0, 0, 3, 3466, 2141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4095, 0, 0, 0, 0, 4090, 279, 0, 0, 0, 3763, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 1, 0, 0, 4093, 4096, 0, 3465, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 3738, 352, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4096, 0, 0, 0, 0, 6, 0, 4095, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3760, 5, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 358, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16977, 15863, 16322, 16383, 16380, 16385, 16384, 16378] avg=16384.0000 max=16977 max_rank=0 min=15863 min_rank=1 imbalance_ratio=1.036194 max_load_reduction=114095 max_load_reduction_pct=0.870476 moved_to_echo=65527 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[981, 0, 0, 1952, 0, 0, 3270, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1955, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 867, 0, 0, 0, 0, 4088, 1750, 0, 0, 0, 4090, 0, 1, 0, 0, 0, 3466, 2141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4091, 279, 0, 0, 0, 3747, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 3465, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 3738, 352, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 12, 0, 4096, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3840, 6, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 358, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16357, 16403, 16308, 16383, 16381, 16388, 16392, 16460] avg=16384.0000 max=16460 max_rank=7 min=16308 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=114612 max_load_reduction_pct=0.874420 moved_to_echo=65621 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1195, 0, 0, 894, 0, 0, 3932, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1955, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 183, 0, 0, 0, 0, 4096, 2663, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3466, 2141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 279, 0, 0, 0, 3661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3465, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3738, 352, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3870, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 358, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16175, 16645, 16228, 16383, 16380, 16382, 16377, 16502] avg=16384.0000 max=16645 max_rank=1 min=16175 min_rank=0 imbalance_ratio=1.015930 max_load_reduction=114427 max_load_reduction_pct=0.873009 moved_to_echo=65641 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[580, 0, 0, 2526, 0, 0, 3765, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 1955, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 4096, 1804, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3466, 2141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 4096, 279, 0, 0, 0, 3828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3465, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3738, 352, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3627, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 358, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17026, 15854, 16396, 16383, 16380, 16382, 16378, 16273] avg=16384.0000 max=17026 max_rank=0 min=15854 min_rank=1 imbalance_ratio=1.039185 max_load_reduction=114046 max_load_reduction_pct=0.870102 moved_to_echo=65413 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1283, 0, 0, 1353, 0, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1955, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 0, 0, 0, 4096, 1992, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3466, 2141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 279, 0, 0, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3465, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3738, 352, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3798, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 358, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16562, 16030, 16535, 16383, 16380, 16382, 16378, 16422] avg=16384.0000 max=16562 max_rank=0 min=16030 min_rank=1 imbalance_ratio=1.010864 max_load_reduction=114510 max_load_reduction_pct=0.873642 moved_to_echo=65562 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1431, 0, 0, 1648, 0, 0, 2843, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1956, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 0, 4087, 2781, 0, 0, 0, 4087, 0, 0, 0, 0, 7, 3466, 2140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 4087, 279, 0, 0, 0, 3842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3465, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3738, 352, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3674, 10, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 358, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16078, 16752, 16398, 16385, 16388, 16386, 16395, 16290] avg=16384.0000 max=16752 max_rank=1 min=16078 min_rank=0 imbalance_ratio=1.022461 max_load_reduction=114320 max_load_reduction_pct=0.872192 moved_to_echo=65459 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1879, 0, 0, 794, 0, 0, 3060, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1955, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 4093, 2591, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 3466, 2141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4095, 0, 0, 0, 0, 4092, 279, 0, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4094, 4096, 0, 3465, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 3738, 352, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 4093, 0, 0, 1, 0, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3944, 2, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 358, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15888, 16544, 16526, 16383, 16382, 16383, 16383, 16583] avg=16384.0000 max=16583 max_rank=7 min=15888 min_rank=0 imbalance_ratio=1.012146 max_load_reduction=114489 max_load_reduction_pct=0.873482 moved_to_echo=65731 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[10880, 0, 0, 11520, 0, 0, 27520, 128, 0, 0, 128, 0, 0, 0, 32768, 0, 32768, 15744, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[2304, 0, 0, 0, 0, 32768, 18560, 0, 0, 0, 32768, 0, 128, 0, 0, 128, 27776, 17152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 128, 0, 32768, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 32768, 128, 0, 32768, 0, 0, 128, 0, 0, 128, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 32768, 128, 0, 32768, 0, 0, 0, 0, 32768, 2304, 0, 0, 0, 30592, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[32768, 0, 30080, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 2944, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 0, 128, 32768, 0, 0, 0, 128, 0, 0, 32768, 32768, 0, 27776, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 0, 32768, 0, 32768, 32768, 0, 0, 0, 0, 29952, 2944, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3882, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 1268, 0, 0, 0, 0, 0, 2897, 0, 0, 3937, 0, 179, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4096, 193, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 0, 4093, 0, 0, 0, 0, 4, 0, 4094, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3787, 4096, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 14, 0, 4096, 0, 0, 0, 0, 1921, 0, 0, 2, 0, 0, 3130, 0, 0, 3306, 0, 79, 3, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 64, 4094, 30, 0, 4094, 4088, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4066, 0, 0, 10, 4096, 6, 4095, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4084, 1, 4091, 0, 0, 0, 0, 4, 0, 4096, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 4083, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3299, 4096, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 2029, 0, 0, 0, 0, 0, 3609, 0, 0, 3284, 0, 16, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 60, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4072, 0, 0, 1, 4096, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4094, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3936, 4096, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 959, 0, 0, 0, 0, 0, 3464, 0, 0, 3955, 0, 93, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 36, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3952, 0, 0, 2, 4096, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4091, 0, 4096, 0, 0, 0, 0, 1, 0, 4095, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4093, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3452, 4096, 0, 0, 0, 0, 0, 4096, 0, 28, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4078, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 1258, 0, 0, 2, 0, 0, 2733, 0, 0, 3887, 0, 579, 1, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 98, 4096, 634, 0, 4095, 4084, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3821, 0, 0, 13, 4096, 2, 4096, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4067, 0, 4096, 0, 0, 0, 0, 21, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4079, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4006, 4096, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4076, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 856, 0, 0, 2, 0, 0, 3091, 0, 0, 3996, 0, 465, 1, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 52, 4095, 87, 0, 4095, 4052, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3863, 0, 0, 21, 4096, 2, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4082, 0, 4096, 0, 0, 0, 0, 11, 0, 4094, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 4094, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3152, 4096, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4073, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 1866, 0, 0, 8, 0, 0, 3477, 0, 0, 3728, 1, 119, 6, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 104, 4094, 54, 0, 4094, 4084, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 3913, 0, 0, 27, 4096, 0, 4096, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4084, 0, 4096, 0, 0, 0, 0, 10, 0, 4096, 4094, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 4096, 4050, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3239, 4096, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 1529, 0, 0, 1, 0, 0, 3198, 0, 0, 4021, 0, 177, 2, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 40, 4095, 159, 0, 4095, 4081, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 4096, 3, 4096, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4092, 0, 4096, 0, 0, 0, 0, 11, 0, 4096, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 4094, 4096, 1, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3787, 4096, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 4096, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 14, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1921, 0, 0, 2, 0, 0, 3130, 0, 0, 3306, 0, 79, 3, 4082, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 4094, 30, 0, 4094, 4088, 1, 0, 0, 3606, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4066, 0, 0, 10, 4096, 6, 4095, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3609, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1, 4091, 0, 0, 0, 0, 4, 0, 4096, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4083, 11, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16582, 16394, 16189, 16274, 16488, 16385, 16377, 16383] avg=16384.0000 max=16582 max_rank=0 min=16189 min_rank=2 imbalance_ratio=1.012085 max_load_reduction=114490 max_load_reduction_pct=0.873489 moved_to_echo=65633 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3882, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1268, 0, 0, 0, 0, 0, 2897, 0, 0, 3937, 0, 179, 0, 4094, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4096, 193, 0, 4096, 4092, 0, 0, 0, 3606, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 4096, 0, 4096, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3609, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4093, 0, 0, 0, 0, 4, 0, 4094, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 11, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16657, 16388, 16041, 16408, 16419, 16383, 16381, 16395] avg=16384.0000 max=16657 max_rank=0 min=16041 min_rank=2 imbalance_ratio=1.016663 max_load_reduction=114415 max_load_reduction_pct=0.872917 moved_to_echo=65578 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3299, 4096, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 4096, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2029, 0, 0, 0, 0, 0, 3609, 0, 0, 3284, 0, 16, 0, 4091, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 60, 0, 4096, 4096, 0, 0, 0, 3606, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4072, 0, 0, 1, 4096, 0, 4096, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3608, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 11, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16086, 16391, 16695, 16258, 16480, 16382, 16387, 16393] avg=16384.0000 max=16695 max_rank=2 min=16086 min_rank=0 imbalance_ratio=1.018982 max_load_reduction=114377 max_load_reduction_pct=0.872627 moved_to_echo=65642 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3936, 4096, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 4096, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 959, 0, 0, 0, 0, 0, 3464, 0, 0, 3955, 0, 93, 0, 4083, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 36, 0, 4096, 4096, 0, 0, 0, 3606, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3952, 0, 0, 2, 4096, 0, 4096, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3608, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 0, 0, 0, 0, 1, 0, 4095, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 11, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16715, 16387, 16220, 16230, 16361, 16383, 16384, 16392] avg=16384.0000 max=16715 max_rank=0 min=16220 min_rank=2 imbalance_ratio=1.020203 max_load_reduction=114357 max_load_reduction_pct=0.872475 moved_to_echo=65520 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4006, 4096, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 0, 4096, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4076, 0, 0, 5, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 856, 0, 0, 2, 0, 0, 3091, 0, 0, 3996, 0, 465, 1, 4080, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 4095, 87, 0, 4095, 4052, 0, 0, 0, 3606, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3863, 0, 0, 21, 4096, 2, 4096, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3609, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4096, 0, 0, 0, 0, 11, 0, 4094, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4094, 11, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16804, 16372, 16157, 16284, 16293, 16384, 16384, 16394] avg=16384.0000 max=16804 max_rank=0 min=16157 min_rank=2 imbalance_ratio=1.025635 max_load_reduction=114268 max_load_reduction_pct=0.871796 moved_to_echo=65455 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3452, 4096, 0, 0, 0, 0, 0, 4096, 0, 28, 0, 0, 4096, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4078, 0, 0, 9, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1258, 0, 0, 2, 0, 0, 2733, 0, 0, 3887, 0, 579, 1, 4010, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 4096, 634, 0, 4095, 4084, 0, 0, 0, 3607, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3821, 0, 0, 13, 4096, 2, 4096, 126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3608, 487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 4096, 0, 0, 0, 0, 21, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4079, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16256, 16380, 16136, 16911, 16250, 16392, 16378, 16369] avg=16384.0000 max=16911 max_rank=3 min=16136 min_rank=2 imbalance_ratio=1.032166 max_load_reduction=114161 max_load_reduction_pct=0.870979 moved_to_echo=65389 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3239, 4096, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 4096, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 6, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1529, 0, 0, 1, 0, 0, 3198, 0, 0, 4021, 0, 177, 2, 4086, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 4095, 159, 0, 4095, 4081, 0, 0, 0, 3606, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 4096, 3, 4096, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3608, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 0, 0, 11, 0, 4096, 4096, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4094, 11, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16024, 16390, 16680, 16373, 16432, 16382, 16396, 16395] avg=16384.0000 max=16680 max_rank=2 min=16024 min_rank=0 imbalance_ratio=1.018066 max_load_reduction=114392 max_load_reduction_pct=0.872742 moved_to_echo=65605 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3152, 4096, 0, 0, 0, 0, 0, 4096, 0, 20, 0, 0, 4096, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4073, 0, 0, 6, 0, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1866, 0, 0, 8, 0, 0, 3477, 0, 0, 3728, 1, 119, 6, 4083, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 4094, 54, 0, 4094, 4084, 1, 0, 0, 3606, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 3913, 0, 0, 27, 4096, 0, 4096, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3608, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 0, 0, 0, 10, 0, 4096, 4094, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4050, 11, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15948, 16370, 16954, 16334, 16349, 16381, 16385, 16351] avg=16384.0000 max=16954 max_rank=2 min=15948 min_rank=0 imbalance_ratio=1.034790 max_load_reduction=114118 max_load_reduction_pct=0.870651 moved_to_echo=65466 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 28800, 32768, 0, 0, 0, 0, 0, 32768, 0, 128, 0, 0, 32768, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 32768, 128, 32768, 0, 0, 0, 0, 128, 0, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 0, 32768, 0, 0, 128, 0, 0, 31744, 0, 0, 128, 32768, 128, 32768, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 11776, 0, 0, 128, 0, 0, 25600, 0, 0, 30208, 128, 1792, 128, 32640, 0, 29440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[32768, 32768, 0, 0, 0, 0, 0, 32768, 0, 0, 128, 0, 32768, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 128, 32768, 0, 0, 0, 0, 32768, 128, 128, 32768, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[32768, 128, 0, 0, 128, 0, 0, 0, 0, 32768, 0, 0, 32768, 0, 0, 0, 28928, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 512, 32768, 1280, 0, 32768, 32768, 128, 0, 0, 28928, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131200 slack_slots=130944 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 0, 47, 0, 0, 0, 4096, 0, 0, 0, 4002, 0, 4096, 4096, 0, 0, 0, 4095, 0, 4049, 0, 0, 0, 4096, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 2915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1435, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 0, 3935, 0, 0, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 0, 0, 0, 3941, 0, 4096, 4096, 0, 0, 0, 4074, 0, 3972, 13, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 474, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 3, 4096, 4096, 4096, 0, 4072, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4095, 0, 0, 4071, 33, 0, 0, 0, 53, 0, 0, 0, 4096, 0, 0, 0, 3789, 0, 4096, 4094, 0, 0, 0, 4059, 0, 4074, 20, 0, 0, 4096, 0, 0, 4085, 4096, 4096, 0, 0, 0, 0, 0, 0, 3365, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1099, 4092, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 28, 0, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 3, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4086, 0, 4096, 4096, 4096, 0, 3961, 0, 0, 4096, 0, 22, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4092, 0, 0, 4095, 0, 0, 4081, 5, 0, 0, 0, 70, 0, 0, 0, 4096, 1, 4, 0, 4043, 0, 4096, 4093, 0, 0, 0, 4058, 0, 3842, 11, 0, 0, 4096, 0, 0, 4082, 4095, 4096, 0, 0, 0, 0, 0, 0, 3749, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1136, 4091, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 4, 0, 4096, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 0, 4096, 4095, 4096, 0, 3550, 0, 0, 4094, 18, 12, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4060, 0, 0, 0, 0, 26, 0, 0, 0, 4096, 5, 1, 0, 3680, 0, 4096, 4092, 0, 0, 0, 4065, 0, 4040, 26, 0, 0, 4096, 0, 0, 4074, 4096, 4096, 0, 0, 0, 0, 0, 0, 2791, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1761, 4086, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 44, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 22, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4081, 1, 4096, 4096, 4096, 0, 4003, 0, 0, 4096, 13, 47, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4095, 0, 0, 4068, 7, 0, 0, 0, 42, 0, 0, 0, 4096, 0, 1, 0, 3955, 0, 4096, 4095, 0, 0, 0, 4066, 0, 3999, 6, 0, 0, 4096, 0, 0, 4076, 4096, 4096, 0, 0, 0, 0, 0, 0, 2954, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1570, 4089, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 28, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 18, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4084, 0, 4096, 4096, 4096, 0, 3849, 0, 0, 4095, 15, 22, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4087, 15, 0, 0, 0, 11, 0, 0, 0, 4095, 0, 0, 0, 4053, 0, 4096, 4095, 0, 0, 0, 4093, 0, 4062, 3, 0, 0, 4096, 0, 0, 4082, 4095, 4096, 0, 0, 0, 0, 0, 0, 2996, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1165, 4093, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 7, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4090, 0, 4096, 4096, 4096, 0, 4072, 0, 0, 4095, 2, 15, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4092, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 3433, 0, 4096, 4096, 0, 0, 0, 4086, 0, 3960, 6, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2763, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 0, 3643, 0, 0, 4096, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 0, 47, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 342, 4096, 0, 0, 0, 4095, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4096, 948, 0, 0, 0, 0, 0, 0, 2915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3148, 893, 26, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1435, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3900, 2861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 196, 4096, 0, 0, 3935, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16407, 16584, 16150, 16375, 16388, 16371, 16377, 16420] avg=16384.0000 max=16584 max_rank=1 min=16150 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=114488 max_load_reduction_pct=0.873474 moved_to_echo=65556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 4095, 0, 0, 4081, 5, 0, 0, 0, 70, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 4043, 0, 342, 4093, 0, 0, 0, 4058, 0, 3842, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4082, 4095, 948, 0, 0, 0, 0, 0, 0, 3749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 4096, 3148, 893, 26, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1136, 4091, 0, 0, 0, 0, 0, 4, 0, 1, 0, 4096, 3900, 2861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 16, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4, 0, 4096, 2, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 196, 4095, 0, 0, 3550, 0, 0, 4094, 18, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16413, 16394, 16970, 16383, 16089, 16385, 16384, 16054] avg=16384.0000 max=16970 max_rank=2 min=16054 min_rank=7 imbalance_ratio=1.035767 max_load_reduction=114102 max_load_reduction_pct=0.870529 moved_to_echo=64912 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 0, 342, 4096, 0, 0, 0, 4074, 0, 3972, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 948, 0, 0, 0, 0, 0, 0, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3148, 893, 26, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 474, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3900, 2861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 196, 4096, 0, 0, 4072, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16438, 17147, 16375, 15427, 16371, 16376, 16559] avg=16384.0000 max=17147 max_rank=2 min=15427 min_rank=4 imbalance_ratio=1.046570 max_load_reduction=113925 max_load_reduction_pct=0.869179 moved_to_echo=64733 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4095, 0, 0, 4071, 33, 0, 0, 0, 53, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3789, 0, 342, 4094, 0, 0, 0, 4059, 0, 4074, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 4096, 948, 0, 0, 0, 0, 0, 0, 3365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 4096, 3148, 893, 26, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1099, 4092, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 3900, 2861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 28, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 3, 0, 4096, 1, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 196, 4096, 0, 0, 3961, 0, 0, 4096, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16418, 16378, 16590, 16387, 16062, 16398, 16382, 16457] avg=16384.0000 max=16590 max_rank=2 min=16062 min_rank=4 imbalance_ratio=1.012573 max_load_reduction=114482 max_load_reduction_pct=0.873428 moved_to_echo=65299 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4060, 0, 0, 0, 0, 26, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 3680, 0, 334, 4092, 0, 0, 0, 4065, 0, 4040, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4074, 4096, 948, 0, 0, 0, 0, 0, 0, 2791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 4096, 3148, 896, 31, 16, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1761, 4086, 0, 0, 0, 0, 0, 33, 0, 0, 0, 4096, 3902, 2866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 44, 0, 0, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 22, 0, 4096, 0, 0, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 1, 194, 4096, 0, 0, 4003, 0, 0, 4096, 13, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16343, 16243, 16005, 16404, 16744, 16407, 16395, 16531] avg=16384.0000 max=16744 max_rank=4 min=16005 min_rank=2 imbalance_ratio=1.021973 max_load_reduction=114328 max_load_reduction_pct=0.872253 moved_to_echo=66077 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4087, 15, 0, 0, 0, 11, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 342, 4095, 0, 0, 0, 4093, 0, 4062, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4082, 4095, 948, 0, 0, 0, 0, 0, 0, 2996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 4096, 3148, 893, 26, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1165, 4093, 1, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 3900, 2861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 7, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 7, 0, 4096, 1, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 196, 4096, 0, 0, 4072, 0, 0, 4095, 2, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16374, 16648, 16217, 16384, 16121, 16376, 16386, 16566] avg=16384.0000 max=16648 max_rank=1 min=16121 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=114424 max_load_reduction_pct=0.872986 moved_to_echo=65449 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3433, 0, 342, 4096, 0, 0, 0, 4086, 0, 3960, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 948, 0, 0, 0, 0, 0, 0, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3148, 893, 26, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2763, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3900, 2861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 196, 4096, 0, 0, 3643, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16360, 15923, 15823, 16375, 17716, 16371, 16376, 16128] avg=16384.0000 max=17716 max_rank=4 min=15823 min_rank=2 imbalance_ratio=1.081299 max_load_reduction=113356 max_load_reduction_pct=0.864838 moved_to_echo=66591 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4095, 0, 0, 4068, 7, 0, 0, 0, 42, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3955, 0, 342, 4095, 0, 0, 0, 4066, 0, 3999, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 4096, 948, 0, 0, 0, 0, 0, 0, 2954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 5, 4096, 0, 0, 0, 4096, 3148, 893, 26, 13, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1570, 4089, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 3900, 2861, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 28, 0, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 18, 0, 4096, 1, 0, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 196, 4096, 0, 0, 3849, 0, 0, 4095, 15, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 16464, 16170, 16389, 16525, 16393, 16396, 16357] avg=16384.0000 max=16525 max_rank=4 min=16170 min_rank=2 imbalance_ratio=1.008606 max_load_reduction=114547 max_load_reduction_pct=0.873924 moved_to_echo=65671 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[32768, 0, 0, 32768, 0, 0, 32768, 128, 0, 0, 0, 384, 0, 0, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 0, 32768, 32768, 0, 0, 0, 0, 128, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 128, 0, 30976, 0, 2816, 32768, 0, 0, 0, 32640, 0, 32000, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 0, 0, 0, 11520, 32768, 128, 0, 0, 0, 0, 128, 0, 128, 0, 32768, 31232, 22912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 32768, 0, 128, 32768, 0, 0, 0, 32768, 25216, 7168, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[128, 32768, 0, 0, 128, 0, 32768, 128, 0, 0, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[32768, 128, 1664, 32768, 0, 0, 31104, 0, 0, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 32768, 32768, 7680, 0, 0, 0, 0, 0, 0, 25344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131200 slack_slots=130944 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4067, 0, 4089, 4096, 0, 4, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 56, 217, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4088, 0, 0, 0, 0, 4096, 3102, 4096, 7, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 733, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 6, 0, 4094, 0, 0, 0, 0, 4043, 9, 4070, 4096, 0, 3, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 4090, 4096, 0, 79, 429, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 4081, 0, 0, 0, 9, 4093, 2857, 4096, 24, 4088, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 523, 0, 0, 0, 4096, 0, 4095, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 63, 259, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4036, 0, 0, 0, 0, 4096, 3219, 4096, 3, 4094, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 464, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 259, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4096, 26, 3994, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 16, 0, 0, 0, 0, 0, 4088, 4096, 0, 70, 363, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4066, 0, 0, 0, 4, 4092, 2611, 4096, 13, 4077, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 894, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 691, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 4088, 29, 4064, 4096, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 4096, 4096, 0, 19, 226, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4058, 0, 0, 0, 7, 4095, 1316, 4096, 5, 4081, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1882, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 469, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4096, 29, 4042, 4096, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 4088, 4096, 0, 85, 353, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 4075, 0, 0, 0, 51, 4096, 2817, 4096, 27, 4063, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 370, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4056, 0, 4092, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3, 137, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4087, 0, 0, 0, 1, 4096, 3762, 4096, 2, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 222, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 4094, 0, 0, 0, 0, 4092, 18, 4073, 4096, 0, 4, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 2, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4092, 4096, 0, 4, 353, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4079, 0, 0, 0, 4, 4095, 2621, 4096, 4, 4091, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 996, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 268, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 4094, 0, 0, 0, 0, 4043, 9, 4070, 2769, 1355, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4090, 4096, 0, 79, 429, 0, 0, 0, 0, 4096, 3499, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4081, 0, 0, 0, 9, 1324, 2857, 0, 24, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 523, 0, 0, 0, 4096, 2739, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16396, 16385, 16348, 16385, 16549, 16475, 16148, 16386] avg=16384.0000 max=16549 max_rank=4 min=16148 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=114523 max_load_reduction_pct=0.873741 moved_to_echo=65558 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4067, 0, 4089, 2771, 1355, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 56, 217, 0, 0, 0, 0, 4096, 3499, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 1325, 3102, 0, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 733, 0, 0, 0, 4096, 2739, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16153, 16386, 16379, 16387, 16315, 16714, 16354, 16384] avg=16384.0000 max=16714 max_rank=5 min=16153 min_rank=0 imbalance_ratio=1.020142 max_load_reduction=114358 max_load_reduction_pct=0.872482 moved_to_echo=65767 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4091, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 4096, 2771, 1355, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 63, 259, 0, 0, 0, 0, 4096, 3499, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4036, 0, 0, 0, 0, 1325, 3219, 0, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 464, 0, 0, 0, 4096, 2740, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16294, 16380, 16412, 16383, 16364, 16773, 16082, 16384] avg=16384.0000 max=16773 max_rank=5 min=16082 min_rank=6 imbalance_ratio=1.023743 max_load_reduction=114299 max_load_reduction_pct=0.872032 moved_to_echo=65603 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 691, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 4088, 29, 4064, 2771, 1355, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4096, 4096, 0, 19, 226, 0, 0, 0, 0, 4096, 3498, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4058, 0, 0, 0, 7, 1324, 1316, 0, 5, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 1882, 0, 0, 0, 4096, 2739, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16819, 16386, 16405, 16384, 16299, 14887, 17507, 16385] avg=16384.0000 max=17507 max_rank=6 min=14887 min_rank=5 imbalance_ratio=1.068542 max_load_reduction=113565 max_load_reduction_pct=0.866432 moved_to_echo=65078 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 259, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4096, 26, 3994, 2773, 1357, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 4088, 4096, 0, 70, 363, 0, 0, 0, 0, 4096, 3499, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4066, 0, 0, 0, 4, 1319, 2611, 0, 13, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 894, 0, 0, 0, 4096, 2739, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16384, 16353, 16378, 16483, 16185, 16518, 16383] avg=16384.0000 max=16518 max_rank=6 min=16185 min_rank=5 imbalance_ratio=1.008179 max_load_reduction=114554 max_load_reduction_pct=0.873978 moved_to_echo=65569 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4094, 0, 0, 0, 0, 4092, 18, 4073, 2771, 1355, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4092, 4096, 0, 4, 353, 0, 0, 0, 0, 4096, 3499, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4079, 0, 0, 0, 4, 1324, 2621, 0, 4, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 996, 0, 0, 0, 4096, 2741, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16270, 16384, 16408, 16388, 16397, 16218, 16623, 16384] avg=16384.0000 max=16623 max_rank=6 min=16218 min_rank=5 imbalance_ratio=1.014587 max_load_reduction=114449 max_load_reduction_pct=0.873177 moved_to_echo=65622 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4056, 0, 4092, 2771, 1355, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3, 137, 0, 0, 0, 0, 4096, 3499, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 1, 1325, 3762, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 222, 0, 0, 0, 4096, 2741, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16154, 16384, 16371, 16383, 16182, 17369, 15845, 16384] avg=16384.0000 max=17369 max_rank=5 min=15845 min_rank=6 imbalance_ratio=1.060120 max_load_reduction=113703 max_load_reduction_pct=0.867485 moved_to_echo=65780 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 469, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4096, 29, 4042, 2771, 1356, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4088, 4096, 0, 85, 353, 0, 0, 0, 0, 4096, 3499, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4075, 0, 0, 0, 51, 1325, 2817, 0, 27, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 370, 0, 0, 0, 4096, 2740, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16598, 16383, 16396, 16384, 16483, 16451, 15995, 16382] avg=16384.0000 max=16598 max_rank=0 min=15995 min_rank=6 imbalance_ratio=1.013062 max_load_reduction=114474 max_load_reduction_pct=0.873367 moved_to_echo=65311 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 0, 128, 0, 32768, 0, 0, 0, 0, 32768, 0, 6144, 0, 0, 0, 32768, 22016, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 0, 0, 0, 0, 0, 32768, 0, 128, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[32768, 0, 128, 0, 0, 0, 0, 0, 0, 32768, 0, 0, 32768, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 0, 0, 128, 0, 0, 0, 32768, 0, 32768, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 32640, 0, 0, 0, 128, 10624, 22400, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 128, 0, 0, 0, 0, 0, 32768, 32768, 0, 384, 2432, 0, 0, 0, 0, 32768, 28032, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 32640, 128, 32640, 22272, 10880, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 2048, 0, 0, 0, 0, 0, 0, 0, 32768, 0, 0, 0, 32768, 32768, 30848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131200 slack_slots=130944 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131200 slack_slots=130944 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 8, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4092, 0, 214, 0, 0, 0, 0, 4096, 2164, 0, 0, 2907, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 100, 0, 0, 0, 2, 3247, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 4083, 25, 0, 0, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 2956, 0, 0, 9, 0, 4079, 4096, 0, 0, 1796, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 3006, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4090, 0, 125, 0, 0, 0, 0, 4096, 3530, 0, 0, 3122, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3313, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4094, 0, 0, 0, 4096, 4090, 1, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 2280, 0, 0, 0, 0, 4096, 4096, 0, 0, 294, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3734, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4086, 0, 83, 0, 0, 0, 0, 4096, 1243, 0, 0, 2122, 0, 4096, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 0, 24, 0, 0, 0, 8, 3540, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4094, 120, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3700, 0, 0, 19, 0, 4082, 4096, 0, 0, 2351, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 3216, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 126, 0, 0, 0, 0, 4096, 2498, 0, 0, 3031, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3803, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4075, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 2750, 0, 0, 0, 0, 4096, 4096, 0, 0, 289, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3908, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 9, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4070, 0, 102, 0, 0, 0, 0, 4096, 2201, 0, 0, 2470, 0, 4096, 1, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 3, 3932, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 4068, 26, 0, 0, 0, 0, 0, 4096, 4095, 2, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3549, 0, 0, 5, 0, 4091, 4096, 0, 0, 801, 0, 0, 0, 0, 4091, 0, 1, 0, 0, 0, 0, 3342, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 0, 0, 19, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4062, 0, 217, 0, 0, 0, 0, 4096, 1681, 0, 0, 2543, 0, 4096, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 11, 0, 0, 0, 4, 3443, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4080, 59, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3842, 0, 0, 11, 0, 4088, 4096, 0, 0, 1019, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 3607, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4091, 0, 102, 0, 0, 0, 0, 4095, 2426, 0, 0, 2711, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 31, 0, 0, 0, 0, 3976, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 2919, 0, 0, 5, 0, 4096, 4096, 0, 0, 760, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3461, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 0, 0, 9, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4087, 0, 98, 0, 0, 0, 0, 4096, 2945, 0, 0, 1606, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3754, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 4095, 0, 0, 0, 4096, 4090, 11, 0, 0, 0, 0, 0, 4096, 4096, 1, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3478, 0, 0, 5, 0, 4090, 4096, 0, 0, 1014, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3494, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 8, 0, 0, 0, 0, 4096, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 214, 0, 0, 0, 0, 3171, 2164, 0, 0, 2907, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 100, 0, 0, 0, 2, 3247, 0, 4096, 4096, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4078, 4083, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2956, 0, 0, 9, 0, 4079, 3977, 0, 0, 1796, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3006, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3650, 925, 119, 18, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16644, 16082, 16387, 16377, 16386, 16895, 15916] avg=16384.0000 max=16895 max_rank=6 min=15916 min_rank=7 imbalance_ratio=1.031189 max_load_reduction=114177 max_load_reduction_pct=0.871101 moved_to_echo=65574 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 9, 0, 0, 0, 0, 4096, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 102, 0, 0, 0, 0, 3171, 2201, 0, 0, 2470, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 3, 3932, 0, 4096, 4096, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4078, 4068, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3549, 0, 0, 5, 0, 4091, 3977, 0, 0, 801, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3342, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3650, 925, 119, 18, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16111, 16677, 16383, 16363, 16385, 16514, 16253] avg=16384.0000 max=16677 max_rank=2 min=16111 min_rank=1 imbalance_ratio=1.017883 max_load_reduction=114395 max_load_reduction_pct=0.872765 moved_to_echo=65515 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 126, 0, 0, 0, 0, 3171, 2498, 0, 0, 3031, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3803, 0, 4096, 4096, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4078, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2750, 0, 0, 0, 0, 4096, 3977, 0, 0, 289, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3908, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3650, 925, 119, 18, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 17018, 16534, 16387, 16345, 16383, 15208, 16818] avg=16384.0000 max=17018 max_rank=1 min=15208 min_rank=6 imbalance_ratio=1.038696 max_load_reduction=114054 max_load_reduction_pct=0.870163 moved_to_echo=64754 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 18, 0, 0, 0, 0, 4096, 4096, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 83, 0, 0, 0, 0, 3168, 1243, 0, 0, 2122, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 0, 24, 0, 0, 0, 8, 3540, 0, 4096, 4096, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4078, 4094, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 0, 19, 0, 4082, 3970, 0, 0, 2351, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3216, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3652, 928, 126, 18, 12, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16390, 14798, 16315, 16374, 16484, 16382, 18183, 16146] avg=16384.0000 max=18183 max_rank=6 min=14798 min_rank=1 imbalance_ratio=1.109802 max_load_reduction=112889 max_load_reduction_pct=0.861275 moved_to_echo=67195 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 125, 0, 0, 0, 0, 3171, 3530, 0, 0, 3122, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3313, 0, 4096, 4096, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 4078, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2280, 0, 0, 0, 0, 4096, 3977, 0, 0, 294, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3734, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3650, 925, 119, 18, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 18133, 16044, 16387, 16359, 16383, 14743, 16644] avg=16384.0000 max=18133 max_rank=1 min=14743 min_rank=6 imbalance_ratio=1.106750 max_load_reduction=112939 max_load_reduction_pct=0.861656 moved_to_echo=64129 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 102, 0, 0, 0, 0, 3170, 2426, 0, 0, 2711, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 31, 0, 0, 0, 0, 3976, 0, 4096, 4096, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4078, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2919, 0, 0, 5, 0, 4096, 3977, 0, 0, 760, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3461, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3650, 925, 119, 18, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 16595, 16738, 16387, 16366, 16383, 15853, 16371] avg=16384.0000 max=16738 max_rank=2 min=15853 min_rank=6 imbalance_ratio=1.021606 max_load_reduction=114334 max_load_reduction_pct=0.872299 moved_to_echo=64973 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 0, 0, 9, 0, 0, 0, 0, 4096, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 98, 0, 0, 0, 0, 3171, 2945, 0, 0, 1606, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3754, 0, 4096, 4096, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 4078, 4090, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3478, 0, 0, 5, 0, 4090, 3977, 0, 0, 1014, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3494, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3650, 925, 119, 18, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16382, 16003, 16486, 16386, 16369, 16385, 16657, 16404] avg=16384.0000 max=16657 max_rank=6 min=16003 min_rank=1 imbalance_ratio=1.016663 max_load_reduction=114415 max_load_reduction_pct=0.872917 moved_to_echo=65815 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4090, 0, 0, 19, 0, 0, 0, 0, 4096, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 217, 0, 0, 0, 0, 3171, 1681, 0, 0, 2543, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 0, 11, 0, 0, 0, 4, 3443, 0, 4096, 4096, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4078, 4080, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 0, 0, 11, 0, 4088, 3977, 0, 0, 1019, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3607, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3653, 925, 119, 18, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 15770, 16196, 16381, 16409, 16385, 17019, 16520] avg=16384.0000 max=17019 max_rank=6 min=15770 min_rank=1 imbalance_ratio=1.038757 max_load_reduction=114053 max_load_reduction_pct=0.870155 moved_to_echo=66333 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[25600, 0, 0, 128, 0, 32768, 31872, 0, 0, 8448, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 128, 0, 0, 0, 0, 32768, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[32768, 32768, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 0, 1152, 0, 0, 0, 0, 25472, 18688, 0, 0, 20608, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[128, 0, 0, 0, 32768, 0, 0, 0, 0, 256, 0, 0, 0, 128, 29056, 0, 32768, 32768, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 27776, 32768, 0, 0, 0, 0, 0, 0, 32768, 0, 0, 29312, 7424, 1024, 256, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 32768, 0, 0, 32768, 0, 0, 0, 32640, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[128, 32768, 0, 0, 0, 0, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131200 slack_slots=130944 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131200 slack_slots=130944 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131200 slack_slots=130944 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 12, 0, 2, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 725, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 4096, 4096, 0, 1814, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 4093, 4095, 0, 0, 0, 0, 1, 0, 0, 1593, 0, 4096, 0, 4096, 4095, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 20, 0, 2, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 2922, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 1074, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 4089, 4095, 0, 0, 0, 0, 1, 0, 0, 106, 0, 4096, 0, 4096, 4095, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 2493, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 1398, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 246, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 1386, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 2611, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 134, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 13, 0, 41, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 3000, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 1049, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 0, 2, 0, 0, 0, 0, 0, 76, 0, 4096, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 11, 0, 6, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 1956, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 1834, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 27, 0, 4096, 0, 0, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 4086, 4094, 0, 0, 0, 0, 0, 0, 0, 345, 0, 4096, 0, 4096, 4094, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 1629, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 1907, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 581, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 2502, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 1666, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 4093, 4095, 0, 0, 0, 0, 3, 0, 0, 14, 0, 4096, 0, 4096, 4095, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 12, 0, 2, 0, 4083, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 725, 0, 0, 0, 0, 4095, 0, 0, 0, 2020, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1814, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 4096, 2433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4092, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 2076, 1610, 387, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 4093, 4095, 0, 0, 0, 0, 1, 0, 2486, 1663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1593, 0, 4096, 0, 3709, 4095, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 15032, 16531, 16383, 16379, 16390, 17591, 16381] avg=16384.0000 max=17591 max_rank=6 min=15032 min_rank=1 imbalance_ratio=1.073669 max_load_reduction=113481 max_load_reduction_pct=0.865791 moved_to_echo=66741 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 20, 0, 2, 0, 4083, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2922, 0, 0, 0, 0, 4096, 0, 0, 0, 2020, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1074, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 2433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 4092, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 2076, 1610, 387, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 4089, 4095, 0, 0, 0, 0, 1, 0, 2486, 1663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 4096, 0, 3709, 4095, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16393, 17230, 15793, 16386, 16377, 16405, 16104, 16384] avg=16384.0000 max=17230 max_rank=1 min=15793 min_rank=2 imbalance_ratio=1.051636 max_load_reduction=113842 max_load_reduction_pct=0.868546 moved_to_echo=65270 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 2, 0, 4083, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2493, 0, 0, 0, 0, 4096, 0, 0, 0, 2020, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1398, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 2433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2076, 1610, 387, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2486, 1663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 4096, 0, 3709, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 16801, 16119, 16380, 16375, 16396, 16243, 16385] avg=16384.0000 max=16801 max_rank=1 min=16119 min_rank=2 imbalance_ratio=1.025452 max_load_reduction=114271 max_load_reduction_pct=0.871819 moved_to_echo=65399 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 13, 0, 41, 0, 4076, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3000, 0, 0, 0, 0, 4096, 0, 0, 0, 2016, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 4096, 4096, 2433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 4083, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2080, 1611, 392, 20, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 4085, 4096, 0, 2, 0, 0, 0, 0, 2485, 1663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 4096, 0, 3704, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16418, 17304, 15751, 16389, 16399, 16357, 16069, 16385] avg=16384.0000 max=17304 max_rank=1 min=15751 min_rank=2 imbalance_ratio=1.056152 max_load_reduction=113768 max_load_reduction_pct=0.867981 moved_to_echo=65210 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 1, 0, 4083, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1386, 0, 0, 0, 0, 4096, 0, 0, 0, 2020, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2611, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 2433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2076, 1611, 387, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2485, 1663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 4096, 0, 3709, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 15694, 17332, 16380, 16376, 16400, 16131, 16387] avg=16384.0000 max=17332 max_rank=2 min=15694 min_rank=1 imbalance_ratio=1.057861 max_load_reduction=113740 max_load_reduction_pct=0.867767 moved_to_echo=65294 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1629, 0, 0, 0, 0, 4096, 0, 0, 0, 2020, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1907, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 2433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2076, 1611, 387, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2485, 1663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 0, 4096, 0, 3709, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16371, 15937, 16628, 16380, 16376, 16417, 16578, 16385] avg=16384.0000 max=16628 max_rank=2 min=15937 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=114444 max_load_reduction_pct=0.873138 moved_to_echo=65756 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 0, 1, 0, 4083, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2502, 0, 0, 0, 0, 4096, 0, 0, 0, 2020, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1666, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 4096, 2433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4092, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 2076, 1611, 387, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 4093, 4095, 0, 0, 0, 0, 3, 0, 2485, 1663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4096, 0, 3709, 4095, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 16810, 16379, 16382, 16385, 16348, 16011, 16385] avg=16384.0000 max=16810 max_rank=1 min=16011 min_rank=6 imbalance_ratio=1.026001 max_load_reduction=114262 max_load_reduction_pct=0.871750 moved_to_echo=65129 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 11, 0, 6, 0, 4083, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1956, 0, 0, 0, 0, 4096, 0, 0, 0, 2020, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1834, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 4096, 4096, 2433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 4092, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 4096, 0, 0, 0, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 2076, 1611, 387, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 4086, 4094, 0, 0, 0, 0, 0, 0, 2485, 1663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 0, 4096, 0, 3709, 4094, 0, 0, 5, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16388, 16264, 16539, 16392, 16405, 16359, 16345, 16380] avg=16384.0000 max=16539 max_rank=2 min=16264 min_rank=1 imbalance_ratio=1.009460 max_load_reduction=114533 max_load_reduction_pct=0.873817 moved_to_echo=65489 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[32768, 0, 0, 0, 16640, 0, 0, 0, 0, 32768, 0, 0, 0, 16256, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[32768, 0, 0, 0, 0, 128, 0, 128, 0, 32768, 0, 0, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[32768, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 32768, 0, 0, 128, 32768, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 16640, 12928, 3200, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 3200, 0, 32768, 0, 29696, 32768, 0, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 13440, 0, 0, 0, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 32768, 32768, 19584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0, 0, 32768, 32768, 32768, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[32384, 0, 0, 0, 0, 0, 0, 0, 32768, 32768, 0, 128, 0, 0, 128, 0, 19968, 13312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131200 slack_slots=130944 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131200 slack_slots=130944 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131200 slack_slots=130944 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 5, 0, 4070, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 59, 4096, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4094, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4091, 4096, 0, 0, 14, 0, 0, 4096, 1, 0, 0, 4096, 4046, 0, 0, 4096, 0, 4090, 4096, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 141, 4096, 0, 0, 4095, 0, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 3, 0, 4096, 4095, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4090, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4090, 4096, 0, 0, 42, 0, 0, 4096, 0, 0, 0, 4096, 3923, 0, 0, 4096, 0, 4092, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 347, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 280, 0, 0, 4096, 0, 0, 0, 4096, 3466, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 11, 0, 4073, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 144, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 2, 0, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4074, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4080, 4096, 0, 0, 607, 0, 0, 4096, 0, 0, 0, 4096, 3382, 0, 0, 4096, 0, 4085, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 7, 0, 4074, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 458, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4087, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4091, 4096, 0, 0, 644, 0, 0, 4096, 4, 0, 0, 4096, 3021, 0, 0, 4096, 0, 4089, 4088, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4, 0, 4084, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 41, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 2, 0, 4096, 4092, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4095, 4094, 4096, 0, 0, 143, 0, 0, 4096, 1, 0, 0, 4095, 3924, 0, 0, 4096, 0, 4091, 4093, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 15, 0, 0, 4096, 0, 0, 0, 4096, 4068, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 204, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 166, 0, 0, 4096, 0, 0, 0, 4096, 3726, 0, 0, 4096, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4096, 2, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 141, 4096, 0, 0, 4095, 0, 0, 0, 3684, 234, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 3, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4090, 4096, 4087, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 3862, 0, 0, 42, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 3923, 0, 0, 0, 0, 4092, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16385, 16348, 16390, 16383, 16186, 16612, 16385] avg=16384.0000 max=16612 max_rank=6 min=16186 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=114460 max_load_reduction_pct=0.873260 moved_to_echo=65566 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 347, 4096, 0, 0, 4096, 0, 0, 0, 3684, 234, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4087, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3862, 0, 0, 280, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 3466, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16393, 16554, 16383, 16388, 16430, 16160, 16381] avg=16384.0000 max=16554 max_rank=2 min=16160 min_rank=6 imbalance_ratio=1.010376 max_load_reduction=114518 max_load_reduction_pct=0.873703 moved_to_echo=65359 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4096, 5, 0, 4070, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 59, 4096, 0, 0, 4095, 0, 0, 0, 3684, 234, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4094, 4096, 4087, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 3862, 0, 0, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 407, 4046, 0, 0, 0, 0, 4090, 4096, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16370, 16265, 16386, 16385, 16159, 16738, 16385] avg=16384.0000 max=16738 max_rank=6 min=16159 min_rank=5 imbalance_ratio=1.021606 max_load_reduction=114334 max_load_reduction_pct=0.872299 moved_to_echo=65667 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 4096, 11, 0, 4073, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 144, 4096, 0, 0, 4096, 0, 0, 0, 3689, 238, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 2, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4074, 4096, 4093, 9, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 3858, 0, 0, 607, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 3382, 0, 0, 0, 0, 4085, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16375, 16360, 16385, 16378, 16737, 16056, 16397] avg=16384.0000 max=16737 max_rank=5 min=16056 min_rank=6 imbalance_ratio=1.021545 max_load_reduction=114335 max_load_reduction_pct=0.872307 moved_to_echo=65568 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4096, 4, 0, 4084, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 41, 4096, 0, 0, 4096, 0, 0, 0, 3683, 234, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 2, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4095, 4096, 4087, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4094, 3862, 0, 0, 143, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 407, 3924, 0, 0, 0, 0, 4091, 4093, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16385, 16384, 16248, 16383, 16387, 16290, 16612, 16383] avg=16384.0000 max=16612 max_rank=6 min=16248 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=114460 max_load_reduction_pct=0.873260 moved_to_echo=65672 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 14, 4096, 0, 0, 4096, 0, 0, 0, 3684, 234, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4087, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3862, 0, 0, 15, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 4068, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16393, 16221, 16383, 16388, 16165, 16762, 16377] avg=16384.0000 max=16762 max_rank=6 min=16165 min_rank=5 imbalance_ratio=1.023071 max_load_reduction=114310 max_load_reduction_pct=0.872116 moved_to_echo=65692 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4096, 7, 0, 4074, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 458, 4096, 0, 0, 4096, 0, 0, 0, 3684, 234, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4087, 4096, 4087, 9, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 3862, 0, 0, 644, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 407, 3021, 0, 0, 0, 0, 4089, 4088, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16387, 16378, 16665, 16379, 16375, 16789, 15712, 16387] avg=16384.0000 max=16789 max_rank=5 min=15712 min_rank=6 imbalance_ratio=1.024719 max_load_reduction=114283 max_load_reduction_pct=0.871910 moved_to_echo=65263 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 204, 4096, 0, 0, 4096, 0, 0, 0, 3684, 234, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4087, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3862, 0, 0, 166, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 3726, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16383, 16394, 16411, 16383, 16388, 16316, 16420, 16377] avg=16384.0000 max=16420 max_rank=6 min=16316 min_rank=5 imbalance_ratio=1.002197 max_load_reduction=114652 max_load_reduction_pct=0.874725 moved_to_echo=65501 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 0, 0, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[128, 0, 32768, 0, 0, 0, 0, 32768, 128, 0, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[128, 0, 0, 0, 0, 32768, 0, 0, 32768, 0, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[128, 0, 0, 3328, 29568, 0, 0, 0, 0, 32768, 32768, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 32768, 32768, 30976, 0, 0, 1920, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 128, 0, 0, 0, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 32768, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 128, 0, 32768, 0, 0, 0, 1408, 32768, 0, 0, 32768, 0, 0, 0, 29568, 1920, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 32768, 128, 32768, 0, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131200 slack_slots=130944 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131200 slack_slots=130944 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 105, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3953, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 87, 0, 4095, 0, 0, 0, 1900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2146, 4096, 0, 0, 4096, 4096, 0, 4096, 2, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 16] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 600, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 3882, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 21, 0, 4094, 0, 0, 0, 1689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2001, 4095, 0, 0, 4096, 4096, 0, 4095, 2, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1178, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3824, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 62, 0, 4096, 0, 0, 0, 1602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1487, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 40] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1121, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3485, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 16, 0, 4080, 0, 0, 0, 1758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1821, 4096, 0, 0, 4096, 4096, 0, 4088, 14, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4076, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 22] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 224, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4001, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 76, 0, 4091, 0, 0, 0, 2597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1300, 4094, 0, 0, 4096, 4096, 0, 4093, 6, 4096, 4096, 1, 4096, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 1, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4094, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 277, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3580, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 43, 0, 4096, 0, 0, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3316, 4096, 0, 0, 4096, 4096, 0, 4096, 1, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 236, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 3911, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 116, 0, 4087, 0, 0, 0, 2847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1104, 4093, 0, 0, 4096, 4096, 0, 4093, 6, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4072, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 393, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 3453, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 384, 0, 4088, 0, 0, 0, 2013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1956, 4095, 0, 0, 4096, 4096, 0, 4096, 2, 4096, 4096, 1, 4096, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4095, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1178, 4096, 2025, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2064, 2030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3824, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 62, 0, 4096, 0, 0, 0, 1602, 0, 4096, 2066, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1487, 4096, 0, 0, 2209, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 40, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17044, 16382, 16446, 16032, 15984, 16390, 16384, 16410] avg=16384.0000 max=17044 max_rank=0 min=15984 min_rank=4 imbalance_ratio=1.040283 max_load_reduction=114028 max_load_reduction_pct=0.869965 moved_to_echo=65168 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 105, 4096, 2025, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2064, 2030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3953, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 87, 0, 4095, 0, 0, 0, 1900, 0, 4096, 2066, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2146, 4096, 0, 0, 2209, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15971, 16382, 16575, 16354, 16643, 16377, 16384, 16386] avg=16384.0000 max=16643 max_rank=4 min=15971 min_rank=0 imbalance_ratio=1.015808 max_load_reduction=114429 max_load_reduction_pct=0.873024 moved_to_echo=65790 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 224, 4096, 2025, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 2064, 2030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4001, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 76, 0, 4091, 0, 0, 0, 2597, 0, 4096, 2065, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1300, 4094, 0, 0, 2209, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 7, 1, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16090, 16384, 16623, 17036, 15792, 16388, 16385, 16374] avg=16384.0000 max=17036 max_rank=3 min=15792 min_rank=4 imbalance_ratio=1.039795 max_load_reduction=114036 max_load_reduction_pct=0.870026 moved_to_echo=64939 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 600, 4096, 2025, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2064, 2030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3882, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 21, 0, 4094, 0, 0, 0, 1689, 0, 4096, 2064, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2001, 4095, 0, 0, 2209, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16466, 16383, 16504, 16074, 16496, 16384, 16385, 16380] avg=16384.0000 max=16504 max_rank=2 min=16074 min_rank=3 imbalance_ratio=1.007324 max_load_reduction=114568 max_load_reduction_pct=0.874084 moved_to_echo=65645 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1121, 4096, 2029, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2065, 2032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3485, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 16, 0, 4080, 0, 0, 0, 1758, 0, 4096, 2064, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1821, 4096, 0, 0, 2196, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 22, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16997, 16385, 16114, 16124, 16297, 16380, 16384, 16391] avg=16384.0000 max=16997 max_rank=0 min=16114 min_rank=2 imbalance_ratio=1.037415 max_load_reduction=114075 max_load_reduction_pct=0.870323 moved_to_echo=65452 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 393, 4096, 2025, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 2064, 2032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3453, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 384, 0, 4088, 0, 0, 0, 2013, 0, 4096, 2064, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1956, 4095, 0, 0, 2209, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 7, 1, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16259, 16386, 16075, 16755, 16452, 16392, 16383, 16370] avg=16384.0000 max=16755 max_rank=3 min=16075 min_rank=2 imbalance_ratio=1.022644 max_load_reduction=114317 max_load_reduction_pct=0.872169 moved_to_echo=65597 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 277, 4096, 2025, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2064, 2031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3580, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 43, 0, 4096, 0, 0, 0, 967, 0, 4096, 2065, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3316, 4096, 0, 0, 2209, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16143, 16383, 16202, 15377, 17813, 16388, 16384, 16382] avg=16384.0000 max=17813 max_rank=4 min=15377 min_rank=3 imbalance_ratio=1.087219 max_load_reduction=113259 max_load_reduction_pct=0.864098 moved_to_echo=66967 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 236, 4096, 2025, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 2064, 2032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3911, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 116, 0, 4087, 0, 0, 0, 2847, 0, 4096, 2064, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1104, 4093, 0, 0, 2209, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 4072, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16102, 16387, 16533, 17320, 15595, 16373, 16383, 16379] avg=16384.0000 max=17320 max_rank=3 min=15595 min_rank=4 imbalance_ratio=1.057129 max_load_reduction=113752 max_load_reduction_pct=0.867859 moved_to_echo=64730 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 15232, 32768, 0, 0, 17664, 32768, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 32768, 0, 0, 0, 0, 0, 128, 896, 0, 32768, 0, 0, 0, 15488, 0, 32768, 16640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 32768, 0, 0, 0, 4224, 32768, 16256, 12544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 32768, 0, 0, 0, 128, 32768, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 0, 0, 0, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 32768, 0, 32768, 0, 0, 0, 0, 0, 32768, 0, 128, 0, 0, 0, 0, 16640, 16256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 30208, 32768, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 32768, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131200 slack_slots=130944 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131200 slack_slots=130944 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 1, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 4096, 0, 3288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 4096, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 454, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4095, 0, 449, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 4081, 4096, 0, 4093, 0, 1552, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 11, 4096, 4095, 4096, 0, 0, 0, 30, 0, 0, 0, 0, 553, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 4080, 0, 2034, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 1, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 2996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 61, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4041, 0, 1202, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3979, 2, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 4083, 4096, 0, 4091, 0, 2841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 10, 4096, 4095, 4096, 0, 0, 0, 19, 0, 0, 0, 0, 199, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4090, 0, 1169, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3247, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4050, 4096, 0, 4089, 0, 3599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 4096, 17, 4096, 4096, 4096, 0, 0, 0, 45, 0, 0, 0, 0, 353, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4094, 0, 988, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 4066, 4096, 0, 4087, 0, 3257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 11, 4096, 4095, 4096, 0, 0, 0, 30, 0, 0, 0, 0, 421, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4080, 0, 688, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4093, 4096, 0, 4096, 0, 2757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 4096, 4096, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 417, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4046, 0, 1007, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3723, 2, 0, 4096, 0, 0, 0, 1, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 4081, 4096, 0, 4095, 0, 2813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 12, 4096, 4095, 4096, 0, 0, 0, 12, 0, 0, 0, 0, 371, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4030, 0, 1342, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3979, 2, 0, 4096, 0, 0, 4081, 3254, 1083, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4083, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 2841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 839, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 10, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 199, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 1169, 0, 0, 0, 3013, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16504, 16388, 16334, 16386, 16231, 16464, 16383, 16382] avg=16384.0000 max=16504 max_rank=0 min=16231 min_rank=4 imbalance_ratio=1.007324 max_load_reduction=114568 max_load_reduction_pct=0.874084 moved_to_echo=65460 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 1, 0, 4096, 0, 0, 4081, 3254, 1083, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 839, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 454, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 449, 0, 0, 0, 3013, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16517, 16398, 16787, 16378, 16472, 15749, 16384, 16387] avg=16384.0000 max=16787 max_rank=2 min=15749 min_rank=5 imbalance_ratio=1.024597 max_load_reduction=114285 max_load_reduction_pct=0.871925 moved_to_echo=64992 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 4095, 0, 0, 4081, 3254, 1083, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4081, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1552, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 839, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4089, 11, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 553, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4080, 0, 2034, 0, 0, 0, 3013, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16566, 16385, 15049, 16388, 16596, 17318, 16384, 16386] avg=16384.0000 max=17318 max_rank=5 min=15049 min_rank=2 imbalance_ratio=1.057007 max_load_reduction=113754 max_load_reduction_pct=0.867874 moved_to_echo=66684 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 1, 0, 4096, 0, 0, 4081, 3254, 1083, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 839, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 61, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4041, 0, 1202, 0, 0, 0, 3013, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16506, 16399, 16495, 16377, 16074, 16448, 16384, 16389] avg=16384.0000 max=16506 max_rank=0 min=16074 min_rank=4 imbalance_ratio=1.007446 max_load_reduction=114566 max_load_reduction_pct=0.874069 moved_to_echo=65295 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3247, 0, 0, 4096, 0, 0, 4088, 3257, 1085, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4050, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 3599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 839, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4081, 17, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 353, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 988, 0, 0, 0, 3011, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15787, 16346, 17091, 16389, 16410, 16285, 16385, 16379] avg=16384.0000 max=17091 max_rank=2 min=15787 min_rank=0 imbalance_ratio=1.043152 max_load_reduction=113981 max_load_reduction_pct=0.869606 moved_to_echo=65459 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 4096, 0, 0, 4081, 3254, 1083, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2757, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 839, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 2, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 417, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4046, 0, 1007, 0, 0, 0, 3013, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16580, 16396, 16256, 16379, 16431, 16258, 16384, 16388] avg=16384.0000 max=16580 max_rank=0 min=16256 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=114492 max_load_reduction_pct=0.873505 moved_to_echo=65461 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3723, 2, 0, 4096, 0, 0, 4081, 3254, 1083, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 15, 4096, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4081, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 841, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 12, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 371, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4030, 0, 1342, 0, 0, 0, 3013, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16246, 16386, 16313, 16388, 16395, 16577, 16384, 16383] avg=16384.0000 max=16577 max_rank=5 min=16246 min_rank=0 imbalance_ratio=1.011780 max_load_reduction=114495 max_load_reduction_pct=0.873528 moved_to_echo=65739 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3842, 0, 0, 4096, 0, 0, 4081, 3257, 1083, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4096, 0, 0, 5, 0, 0, 0, 0, 4096, 0, 0, 4066, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 3257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 839, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 11, 4096, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 421, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4080, 0, 688, 0, 0, 0, 3013, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16366, 16374, 16747, 16387, 16463, 15973, 16384, 16378] avg=16384.0000 max=16747 max_rank=2 min=15973 min_rank=5 imbalance_ratio=1.022156 max_load_reduction=114325 max_load_reduction_pct=0.872231 moved_to_echo=65198 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 0, 0, 32768, 0, 0, 0, 128, 0, 0, 0, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 32768, 0, 23168, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32768, 32768, 6784, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[256, 128, 0, 0, 0, 2944, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 32768, 32768, 29824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 0, 32768, 0, 32640, 0, 8960, 0, 0, 0, 24192, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131200 observed_loss=-128 tokens_per_expert=[0, 0, 32768, 32768, 0, 0, 32768, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131328 observed_loss=-256 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 0, 32768, 128, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 30976, 128, 0, 32768, 0, 0, 32768, 26112, 8704, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[0, 128, 128, 32768, 0, 0, 128, 0, 0, 0, 0, 32768, 0, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131200 slack_slots=130944 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131328 slack_slots=130816 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1429, 1940, 1900, 1, 461, 2, 1823, 933, 1105, 1768, 1164, 478, 1599, 2508, 0, 1132, 909, 1583, 1165, 1500, 6, 866, 159, 2185, 228, 28, 2956, 1692, 1343, 42, 1661, 1131, 280, 0, 1611, 430, 239, 1804, 1565, 262, 275, 4, 2207, 2650, 0, 2190, 1221, 1599, 1366, 747, 511, 785, 8, 64, 19, 2041, 0, 328, 36, 567, 1049, 2812, 1343, 262, 1172, 9, 0, 1474, 2032, 1423, 918, 879, 1461, 2002, 2213, 336, 1885, 0, 955, 1942, 705, 220, 7, 15, 2086, 690, 995, 6, 1758, 150, 1170, 935, 201, 1942, 558, 0, 1127, 603, 9, 2328, 5, 419, 1388, 864, 247, 1742, 155, 1387, 259, 1790, 2486, 1508, 1901, 1888, 469, 1746, 2777, 1213, 2046, 1551, 1354, 2089, 1605, 582, 816, 6, 579, 22] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2363, 2043, 1827, 0, 360, 0, 1965, 1109, 1224, 2314, 741, 423, 1384, 1782, 0, 824, 703, 1454, 1045, 1607, 15, 641, 187, 2372, 425, 126, 3569, 2041, 1787, 50, 1484, 1145, 190, 6, 1784, 962, 169, 1620, 1243, 239, 274, 5, 2140, 2525, 0, 1378, 811, 850, 1236, 847, 895, 958, 1, 11, 14, 1609, 0, 701, 99, 416, 1754, 2454, 1401, 658, 1077, 7, 0, 1908, 2037, 2019, 698, 1484, 1211, 1835, 2239, 234, 1139, 20, 678, 1732, 833, 677, 6, 11, 2043, 330, 1647, 20, 2439, 134, 897, 603, 110, 1682, 387, 0, 1407, 1239, 8, 2084, 16, 379, 1687, 1078, 252, 1535, 67, 1101, 407, 1863, 1873, 1602, 2192, 2162, 152, 1752, 2238, 797, 2767, 1581, 1171, 1629, 896, 911, 853, 14, 952, 10] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2174, 2682, 1762, 0, 393, 0, 2055, 916, 1087, 2943, 1003, 703, 1255, 2552, 0, 1247, 1099, 1422, 1058, 1658, 21, 712, 149, 2000, 96, 51, 3247, 2051, 1329, 34, 1339, 1061, 534, 12, 1505, 370, 175, 1564, 1360, 211, 68, 7, 1771, 2671, 0, 2260, 763, 632, 1170, 480, 808, 743, 6, 42, 42, 2656, 0, 272, 27, 485, 1270, 2845, 1035, 163, 777, 18, 0, 1627, 2120, 1695, 677, 1139, 1039, 1995, 2151, 282, 1589, 3, 1231, 1752, 1033, 315, 11, 6, 2065, 616, 917, 37, 2328, 48, 872, 826, 286, 2176, 542, 3, 1330, 674, 1, 1904, 7, 425, 1611, 1032, 326, 1734, 74, 1199, 475, 2035, 3018, 1662, 2270, 1989, 371, 788, 2543, 818, 2299, 1501, 1065, 2185, 1410, 794, 778, 31, 495, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1607, 2143, 1839, 1, 192, 0, 2016, 391, 1098, 2464, 1024, 700, 2163, 3026, 0, 961, 623, 1816, 1031, 1269, 9, 611, 151, 2324, 85, 44, 3108, 2092, 1079, 41, 1549, 1290, 571, 2, 1901, 508, 258, 2067, 739, 492, 243, 0, 1542, 2598, 0, 2066, 1020, 1073, 1192, 683, 459, 639, 3, 91, 6, 2086, 0, 211, 21, 489, 1346, 2988, 1140, 103, 1223, 5, 1, 1485, 1943, 1657, 912, 1116, 1576, 1992, 1921, 171, 1912, 9, 1189, 1624, 954, 201, 12, 6, 2071, 318, 650, 18, 2049, 106, 1001, 1035, 321, 2434, 468, 1, 1742, 904, 12, 1785, 7, 470, 1581, 903, 137, 1641, 74, 1079, 477, 2425, 2638, 1826, 2637, 2099, 444, 956, 2339, 1035, 1409, 1434, 1131, 2377, 1386, 799, 935, 5, 698, 22] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1823, 1736, 1435, 1, 309, 0, 2663, 413, 763, 3053, 1225, 402, 1151, 2985, 2, 1114, 773, 1501, 1432, 1202, 24, 713, 152, 2145, 149, 45, 3399, 1931, 1798, 56, 1612, 1206, 191, 3, 1603, 551, 191, 1604, 1191, 418, 135, 1, 1904, 2801, 0, 2416, 1108, 1424, 1234, 508, 861, 829, 0, 6, 8, 1434, 1, 190, 15, 312, 958, 2742, 1244, 233, 961, 9, 3, 1773, 2095, 1727, 951, 1391, 1081, 2108, 2404, 250, 2145, 14, 926, 1814, 552, 195, 8, 3, 2428, 441, 1088, 18, 2136, 48, 995, 808, 296, 2141, 532, 4, 1038, 826, 0, 2184, 3, 428, 1767, 1294, 275, 1485, 119, 1332, 435, 1604, 2671, 1901, 2254, 1941, 210, 1063, 2047, 971, 2337, 1254, 981, 2730, 1497, 505, 694, 10, 539, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1657, 2157, 2205, 1, 453, 2, 1557, 386, 956, 1654, 922, 714, 1849, 2663, 0, 1289, 934, 1425, 1619, 1575, 6, 877, 148, 2266, 130, 62, 3110, 1514, 993, 31, 1418, 1163, 494, 2, 1813, 246, 451, 1746, 1279, 275, 229, 2, 1618, 2907, 0, 1954, 1094, 888, 1484, 418, 443, 950, 1, 21, 6, 1956, 0, 258, 101, 345, 1244, 2786, 1105, 444, 1221, 4, 1, 1736, 2259, 1750, 1078, 788, 1436, 2098, 1796, 297, 1852, 6, 936, 2015, 888, 307, 12, 61, 2176, 728, 829, 16, 2203, 134, 689, 903, 247, 2150, 590, 3, 828, 624, 4, 2296, 7, 931, 1806, 1114, 218, 1386, 225, 1607, 730, 2111, 1842, 1900, 2311, 2274, 234, 1277, 3118, 1020, 2125, 1572, 952, 1628, 1323, 836, 859, 17, 372, 20] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1647, 2274, 1426, 0, 214, 0, 1825, 517, 926, 2946, 1632, 369, 1148, 3329, 0, 1394, 1034, 2179, 1208, 1578, 45, 893, 145, 2191, 118, 273, 2571, 1869, 1758, 167, 1708, 1207, 188, 0, 2114, 586, 267, 2099, 1253, 700, 211, 2, 1497, 3122, 0, 1955, 929, 954, 1181, 714, 590, 521, 0, 79, 7, 1903, 5, 150, 10, 322, 1461, 3001, 1383, 122, 921, 5, 0, 1533, 1487, 1698, 631, 853, 1570, 1349, 1831, 129, 1642, 6, 1135, 2303, 635, 372, 3, 9, 2118, 482, 925, 7, 2008, 129, 1212, 622, 404, 2094, 416, 0, 1472, 1054, 11, 1857, 9, 642, 1743, 1176, 156, 1794, 43, 1347, 199, 1924, 2816, 1370, 2131, 2131, 462, 1574, 1651, 1347, 1582, 1241, 1020, 2808, 1096, 585, 1027, 3, 344, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1795, 2468, 1383, 0, 637, 3, 1852, 993, 973, 2321, 1091, 307, 1137, 2733, 0, 1164, 954, 1274, 1280, 1357, 9, 1440, 202, 2006, 290, 175, 3312, 1664, 1097, 30, 1417, 1424, 219, 5, 1963, 354, 121, 1852, 1183, 225, 193, 2, 1985, 2824, 0, 2316, 878, 861, 924, 814, 785, 1157, 7, 74, 7, 1937, 0, 242, 61, 674, 1329, 3222, 1071, 189, 1226, 12, 0, 1481, 2244, 1891, 628, 1387, 1245, 1743, 1827, 378, 1147, 6, 1430, 2001, 868, 381, 6, 24, 2070, 579, 977, 16, 2566, 77, 989, 781, 273, 2279, 443, 3, 1366, 787, 8, 1814, 16, 417, 1744, 1014, 237, 1952, 58, 1403, 794, 1270, 2244, 1515, 2178, 1604, 568, 1280, 2335, 644, 2011, 1831, 1021, 2339, 1086, 1025, 790, 6, 459, 16] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2174, 2682, 1762, 0, 393, 0, 2055, 916, 1087, 2884, 1003, 703, 1255, 0, 0, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1099, 1422, 1058, 1658, 21, 712, 149, 2000, 96, 51, 1736, 2051, 1329, 34, 1339, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 12, 1505, 370, 175, 1564, 1360, 211, 68, 7, 1771, 2671, 0, 2260, 763, 632, 969, 306, 138, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 480, 808, 743, 6, 42, 42, 2656, 0, 272, 27, 485, 1270, 2845, 1035, 163, 2151, 1511, 561, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 777, 18, 0, 1627, 1982, 1695, 677, 1139, 1039, 1995, 0, 282, 1589, 3, 1231, 1752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 315, 11, 6, 2065, 616, 917, 37, 2328, 48, 872, 826, 286, 2176, 542, 3, 2552, 2018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1330, 674, 1, 1904, 7, 425, 1611, 1032, 326, 1734, 74, 1199, 475, 2035, 2049, 1662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1403, 1989, 371, 788, 0, 818, 2299, 1501, 1065, 2185, 1410, 794, 778, 31, 495, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18161, 15816, 15375, 16792, 15806, 16651, 16538, 15933] avg=16384.0000 max=18161 max_rank=0 min=15375 min_rank=2 imbalance_ratio=1.108459 max_load_reduction=112911 max_load_reduction_pct=0.861443 moved_to_echo=64928 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2363, 2043, 1827, 0, 360, 0, 1965, 1109, 1224, 2268, 741, 423, 1384, 0, 0, 824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 703, 1454, 1045, 1607, 15, 641, 187, 2372, 425, 126, 1908, 2041, 1787, 50, 1484, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 190, 6, 1784, 962, 169, 1620, 1243, 239, 274, 5, 2140, 2525, 0, 1378, 811, 850, 601, 295, 133, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1236, 847, 895, 958, 1, 11, 14, 1609, 0, 701, 99, 416, 1754, 2454, 1401, 658, 2239, 1661, 541, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 7, 0, 1908, 1904, 2019, 698, 1484, 1211, 1835, 0, 234, 1139, 20, 678, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 677, 6, 11, 2043, 330, 1647, 20, 2439, 134, 897, 603, 110, 1682, 387, 0, 1782, 1775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1407, 1239, 8, 2084, 16, 379, 1687, 1078, 252, 1535, 67, 1101, 407, 1863, 1272, 1602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1356, 2162, 152, 1752, 0, 797, 2767, 1581, 1171, 1629, 896, 911, 853, 14, 952, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16531, 16990, 15271, 17958, 15946, 15376, 15997, 17003] avg=16384.0000 max=17958 max_rank=3 min=15271 min_rank=2 imbalance_ratio=1.096069 max_load_reduction=113114 max_load_reduction_pct=0.862991 moved_to_echo=64322 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1429, 1940, 1900, 1, 461, 2, 1823, 933, 1105, 1733, 1164, 478, 1599, 0, 0, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 909, 1583, 1165, 1500, 6, 866, 159, 2185, 228, 28, 1580, 1692, 1343, 42, 1661, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 1611, 430, 239, 1804, 1565, 262, 275, 4, 2207, 2650, 0, 2190, 1221, 1599, 798, 256, 132, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1366, 747, 511, 785, 8, 64, 19, 2041, 0, 328, 36, 567, 1049, 2812, 1343, 262, 2213, 1376, 469, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1172, 9, 0, 1474, 1900, 1423, 918, 879, 1461, 2002, 0, 336, 1885, 0, 955, 1942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 705, 220, 7, 15, 2086, 690, 995, 6, 1758, 150, 1170, 935, 201, 1942, 558, 0, 2508, 2203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1127, 603, 9, 2328, 5, 419, 1388, 864, 247, 1742, 155, 1387, 259, 1790, 1688, 1508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1176, 1888, 469, 1746, 0, 1213, 2046, 1551, 1354, 2089, 1605, 582, 816, 6, 579, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15700, 16078, 17558, 16570, 16356, 16149, 15519, 17142] avg=16384.0000 max=17558 max_rank=2 min=15519 min_rank=6 imbalance_ratio=1.071655 max_load_reduction=113514 max_load_reduction_pct=0.866043 moved_to_echo=65166 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1607, 2143, 1839, 1, 192, 0, 2016, 391, 1098, 2415, 1024, 700, 2163, 0, 0, 961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 623, 1816, 1031, 1269, 9, 611, 151, 2324, 85, 44, 1661, 2092, 1079, 41, 1549, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 571, 2, 1901, 508, 258, 2067, 739, 492, 243, 0, 1542, 2598, 0, 2066, 1020, 1073, 847, 355, 127, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 683, 459, 639, 3, 91, 6, 2086, 0, 211, 21, 489, 1346, 2988, 1140, 103, 1921, 1447, 651, 483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1223, 5, 1, 1485, 1816, 1657, 912, 1116, 1576, 1992, 0, 171, 1912, 9, 1189, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 954, 201, 12, 6, 2071, 318, 650, 18, 2049, 106, 1001, 1035, 321, 2434, 468, 1, 3026, 1856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 904, 12, 1785, 7, 470, 1581, 903, 137, 1641, 74, 1079, 477, 2425, 1791, 1826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1631, 2099, 444, 956, 0, 1035, 1409, 1434, 1131, 2377, 1386, 799, 935, 5, 698, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16550, 15675, 16458, 15959, 16688, 16527, 16854, 16361] avg=16384.0000 max=16854 max_rank=6 min=15675 min_rank=1 imbalance_ratio=1.028687 max_load_reduction=114218 max_load_reduction_pct=0.871414 moved_to_echo=66430 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1823, 1736, 1435, 1, 309, 0, 2663, 413, 763, 2992, 1225, 402, 1151, 0, 2, 1114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 773, 1501, 1432, 1202, 24, 713, 152, 2145, 149, 45, 1817, 1931, 1798, 56, 1612, 1206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 3, 1603, 551, 191, 1604, 1191, 418, 135, 1, 1904, 2801, 0, 2416, 1108, 1424, 858, 304, 136, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1234, 508, 861, 829, 0, 6, 8, 1434, 1, 190, 15, 312, 958, 2742, 1244, 233, 2404, 1582, 557, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 961, 9, 3, 1773, 1959, 1727, 951, 1391, 1081, 2108, 0, 250, 2145, 14, 926, 1814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 552, 195, 8, 3, 2428, 441, 1088, 18, 2136, 48, 995, 808, 296, 2141, 532, 4, 2985, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 826, 0, 2184, 3, 428, 1767, 1294, 275, 1485, 119, 1332, 435, 1604, 1813, 1901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1393, 1941, 210, 1063, 0, 971, 2337, 1254, 981, 2730, 1497, 505, 694, 10, 539, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16029, 16556, 16900, 15541, 17112, 16302, 16504, 16128] avg=16384.0000 max=17112 max_rank=4 min=15541 min_rank=3 imbalance_ratio=1.044434 max_load_reduction=113960 max_load_reduction_pct=0.869446 moved_to_echo=66046 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1657, 2157, 2205, 1, 453, 2, 1557, 386, 956, 1616, 922, 714, 1849, 0, 0, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 934, 1425, 1619, 1575, 6, 877, 148, 2266, 130, 62, 1658, 1514, 993, 31, 1418, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 494, 2, 1813, 246, 451, 1746, 1279, 275, 229, 2, 1618, 2907, 0, 1954, 1094, 888, 596, 316, 151, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1484, 418, 443, 950, 1, 21, 6, 1956, 0, 258, 101, 345, 1244, 2786, 1105, 444, 1796, 1452, 575, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1221, 4, 1, 1736, 2108, 1750, 1078, 788, 1436, 2098, 0, 297, 1852, 6, 936, 2015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 888, 307, 12, 61, 2176, 728, 829, 16, 2203, 134, 689, 903, 247, 2150, 590, 3, 2663, 2474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 624, 4, 2296, 7, 931, 1806, 1114, 218, 1386, 225, 1607, 730, 2111, 1246, 1900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1420, 2274, 234, 1277, 0, 1020, 2125, 1572, 952, 1628, 1323, 836, 859, 17, 372, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15764, 15819, 16099, 16029, 17326, 17073, 17033, 15929] avg=16384.0000 max=17326 max_rank=4 min=15764 min_rank=0 imbalance_ratio=1.057495 max_load_reduction=113746 max_load_reduction_pct=0.867813 moved_to_echo=67361 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1647, 2274, 1426, 0, 214, 0, 1825, 517, 926, 2887, 1632, 369, 1148, 0, 0, 1394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1034, 2179, 1208, 1578, 45, 893, 145, 2191, 118, 273, 1375, 1869, 1758, 167, 1708, 1207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 2114, 586, 267, 2099, 1253, 700, 211, 2, 1497, 3122, 0, 1955, 929, 954, 904, 287, 97, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 714, 590, 521, 0, 79, 7, 1903, 5, 150, 10, 322, 1461, 3001, 1383, 122, 1831, 1196, 526, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 921, 5, 0, 1533, 1390, 1698, 631, 853, 1570, 1349, 0, 129, 1642, 6, 1135, 2303, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 635, 372, 3, 9, 2118, 482, 925, 7, 2008, 129, 1212, 622, 404, 2094, 416, 0, 3329, 1311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1472, 1054, 11, 1857, 9, 642, 1743, 1176, 156, 1794, 43, 1347, 199, 1924, 1912, 1370, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1318, 2131, 462, 1574, 0, 1347, 1582, 1241, 1020, 2808, 1096, 585, 1027, 3, 344, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16259, 17748, 17224, 15342, 15165, 16076, 16709, 16549] avg=16384.0000 max=17748 max_rank=1 min=15165 min_rank=4 imbalance_ratio=1.083252 max_load_reduction=113324 max_load_reduction_pct=0.864594 moved_to_echo=64499 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1795, 2468, 1383, 0, 637, 3, 1852, 993, 973, 2275, 1091, 307, 1137, 0, 0, 1164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 954, 1274, 1280, 1357, 9, 1440, 202, 2006, 290, 175, 1771, 1664, 1097, 30, 1417, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 5, 1963, 354, 121, 1852, 1183, 225, 193, 2, 1985, 2824, 0, 2316, 878, 861, 721, 293, 146, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 924, 814, 785, 1157, 7, 74, 7, 1937, 0, 242, 61, 674, 1329, 3222, 1071, 189, 1827, 1541, 538, 482, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1226, 12, 0, 1481, 2098, 1891, 628, 1387, 1245, 1743, 0, 378, 1147, 6, 1430, 2001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 868, 381, 6, 24, 2070, 579, 977, 16, 2566, 77, 989, 781, 273, 2279, 443, 3, 2733, 1853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1366, 787, 8, 1814, 16, 417, 1744, 1014, 237, 1952, 58, 1403, 794, 1270, 1523, 1515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1347, 1604, 568, 1280, 0, 644, 2011, 1831, 1021, 2339, 1086, 1025, 790, 6, 459, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16078, 16390, 16187, 16881, 16673, 16918, 15918, 16027] avg=16384.0000 max=16918 max_rank=5 min=15918 min_rank=6 imbalance_ratio=1.032593 max_load_reduction=114154 max_load_reduction_pct=0.870926 moved_to_echo=65536 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[14592, 17536, 13824, 128, 3072, 128, 15872, 5760, 8192, 19072, 8832, 4096, 11776, 0, 128, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9856, 5248, 5376, 6656, 128, 512, 128, 15744, 128, 2432, 384, 3712, 10496, 22912, 9728, 2176, 16384, 11776, 4480, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[7040, 12672, 9856, 11776, 256, 6784, 1408, 17536, 1536, 896, 13568, 14976, 11264, 512, 12288, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[11136, 16128, 2944, 10496, 0, 7936, 16640, 12032, 8704, 17792, 10368, 6144, 6784, 128, 4480, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[2688, 128, 14336, 4096, 1920, 14464, 9856, 2944, 1664, 128, 14720, 22144, 0, 16640, 7936, 8320, 6400, 2432, 1152, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[10368, 6784, 128, 16256, 128, 4224, 13440, 8576, 1920, 13312, 896, 10496, 3840, 15104, 13312, 13312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[8704, 128, 128, 13056, 15232, 13952, 6528, 9088, 10624, 15232, 0, 2176, 13312, 128, 8576, 15232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[6528, 2688, 128, 256, 17152, 4224, 8064, 256, 17536, 896, 7936, 6528, 2176, 17024, 3968, 128, 21632, 15232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 1, 13, 4094, 0, 82, 4, 1, 0, 147, 6, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4067, 3, 2989, 2, 0, 4088, 0, 4090, 4096, 15, 0, 1, 4094, 0, 2116, 4066, 3, 0, 0, 3587, 1, 0, 4, 0, 446, 0, 1, 4019, 4084, 0, 0, 4096, 2, 58, 4096, 0, 0, 4094, 53, 0, 0, 4096, 0, 70, 2112, 0, 0, 1924, 3, 0, 4094, 0, 3916, 4096, 0, 0, 0, 0, 0, 0, 3217, 0, 3, 0, 0, 0, 2608, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 4096, 4050, 0, 4, 0, 6, 4095, 0, 4083, 0, 0, 0, 4096, 4096, 0, 0, 153, 0, 487, 1, 0, 4081, 311, 2, 476, 0, 0, 0, 4088] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 4077, 0, 206, 0, 0, 0, 208, 0, 0, 2, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 2191, 7, 0, 4076, 1, 4093, 4094, 8, 0, 0, 4096, 0, 2519, 3986, 13, 2, 0, 2934, 0, 0, 2, 0, 108, 0, 46, 4043, 4087, 0, 0, 4094, 5, 27, 4096, 0, 0, 4091, 22, 1, 0, 4092, 0, 12, 2025, 0, 0, 3346, 0, 0, 4094, 0, 3867, 4096, 0, 0, 0, 0, 0, 0, 3323, 0, 0, 0, 0, 1, 1778, 0, 0, 1, 0, 0, 4096, 0, 4, 0, 0, 0, 4096, 4066, 0, 30, 0, 28, 4094, 0, 4078, 2, 0, 0, 4096, 4096, 1, 0, 58, 1, 940, 3, 0, 4092, 413, 0, 717, 0, 0, 0, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[3, 0, 0, 110, 4073, 0, 0, 0, 0, 0, 137, 1, 1, 4, 6, 4091, 0, 5, 4091, 0, 0, 4093, 0, 3540, 8, 0, 3758, 2, 4087, 4091, 10, 1, 3, 4090, 0, 2670, 3559, 18, 0, 4, 3311, 0, 5, 0, 5, 268, 5, 242, 4091, 4080, 5, 4, 4091, 0, 9, 4090, 5, 5, 4091, 45, 0, 5, 4091, 0, 10, 2280, 0, 0, 3427, 3, 2, 4091, 7, 1931, 4091, 0, 5, 0, 0, 0, 2, 3520, 0, 0, 1, 0, 0, 3314, 4, 5, 2, 0, 5, 4089, 0, 5, 0, 4, 0, 4091, 3908, 14, 2, 4, 2, 4090, 5, 4083, 0, 1, 0, 4091, 4089, 3, 5, 480, 0, 139, 0, 4, 4084, 0, 0, 194, 1, 0, 0, 4080] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 4088, 0, 199, 0, 0, 0, 306, 5, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 4079, 1, 2379, 0, 0, 4065, 0, 4078, 4096, 10, 0, 0, 4095, 0, 2736, 3629, 0, 4, 0, 3648, 0, 0, 1, 0, 888, 0, 1, 4072, 4079, 0, 0, 4096, 7, 50, 4096, 0, 0, 4094, 8, 0, 0, 4086, 0, 2, 2926, 0, 0, 1721, 1, 0, 4096, 0, 3680, 4096, 0, 0, 0, 0, 0, 0, 3542, 0, 12, 0, 0, 0, 2430, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4030, 0, 23, 0, 4, 4096, 0, 4092, 5, 0, 0, 4096, 4096, 1, 0, 1, 0, 53, 0, 0, 4088, 182, 0, 426, 0, 0, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 296, 4047, 0, 513, 0, 0, 0, 282, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4055, 0, 1698, 0, 0, 4095, 0, 4072, 4096, 22, 0, 0, 4096, 1, 3360, 4029, 7, 1, 0, 2666, 0, 0, 12, 0, 1169, 0, 0, 3742, 4076, 0, 0, 4096, 2, 50, 4096, 0, 0, 4096, 7, 0, 0, 4081, 0, 112, 3449, 0, 0, 3177, 0, 0, 4096, 0, 2737, 4096, 0, 0, 0, 0, 0, 0, 2060, 1, 107, 0, 0, 0, 2452, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 4081, 2, 18, 0, 24, 4096, 0, 4092, 0, 0, 0, 4096, 4096, 0, 0, 21, 0, 426, 1, 0, 4042, 206, 0, 341, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 31, 4071, 3, 414, 0, 0, 0, 498, 9, 0, 0, 0, 4088, 0, 2, 4093, 0, 2, 4090, 2, 2407, 0, 0, 4051, 0, 4094, 4093, 103, 0, 0, 4096, 0, 2831, 3949, 2, 1, 0, 3327, 1, 0, 1, 0, 764, 0, 1, 3793, 4092, 0, 0, 4093, 7, 21, 4096, 2, 0, 4093, 1, 0, 0, 4095, 0, 16, 2310, 0, 2, 2325, 5, 0, 4093, 0, 3897, 4083, 0, 0, 3, 5, 1, 0, 3288, 0, 5, 0, 0, 0, 2138, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 1, 4093, 4047, 17, 12, 0, 5, 4093, 2, 4057, 0, 0, 3, 4094, 4093, 0, 0, 10, 0, 325, 2, 3, 4075, 249, 0, 211, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4079, 1, 26, 2, 0, 0, 66, 0, 0, 0, 1, 4095, 0, 0, 4096, 0, 0, 4096, 0, 2230, 0, 0, 4046, 0, 4096, 4095, 32, 0, 0, 4095, 0, 3295, 3717, 20, 2, 0, 3607, 0, 0, 0, 0, 281, 0, 3, 4061, 4084, 0, 0, 4094, 0, 97, 4096, 0, 0, 4095, 99, 0, 0, 4096, 0, 31, 2982, 0, 1, 3398, 0, 0, 4094, 2, 2795, 4096, 0, 0, 0, 0, 1, 4, 2611, 1, 0, 0, 0, 1, 2627, 0, 0, 3, 1, 0, 4094, 0, 0, 0, 0, 1, 4094, 3956, 26, 34, 0, 60, 4095, 0, 4096, 0, 0, 0, 4096, 4094, 0, 0, 6, 0, 455, 0, 0, 4065, 74, 0, 384, 0, 0, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 28, 4057, 0, 8, 5, 0, 0, 289, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 4096, 0, 1339, 0, 0, 4066, 0, 4092, 4096, 1, 0, 0, 4092, 0, 3248, 2736, 10, 5, 0, 1983, 0, 0, 0, 0, 710, 0, 153, 4061, 4073, 0, 0, 4096, 0, 479, 4096, 0, 0, 4092, 79, 0, 0, 4096, 0, 16, 3678, 0, 1, 3798, 12, 0, 4095, 0, 2467, 4096, 0, 0, 0, 0, 0, 2, 3629, 0, 9, 0, 0, 0, 2670, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 0, 0, 4095, 3865, 180, 5, 0, 63, 4096, 0, 4094, 0, 0, 0, 4094, 4096, 0, 0, 58, 0, 290, 7, 0, 4086, 51, 0, 1069, 0, 2, 0, 4081] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 13, 4094, 0, 82, 4, 1, 0, 147, 6, 0, 0, 0, 4096, 4096, 3630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 3, 2989, 2, 0, 4088, 0, 4090, 1807, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 2116, 4066, 3, 0, 0, 3587, 1, 0, 4, 0, 446, 0, 1, 1322, 615, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 4084, 0, 0, 0, 2, 58, 0, 0, 0, 4082, 53, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 2112, 0, 0, 1924, 3, 0, 3479, 0, 3916, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3217, 0, 3, 0, 0, 0, 2608, 0, 0, 0, 0, 0, 4095, 0, 2, 4096, 2056, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4050, 0, 4, 0, 6, 4095, 0, 4083, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 153, 0, 487, 1, 0, 4081, 311, 2, 476, 0, 0, 0, 4088, 2002, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16170, 17061, 16268, 16394, 15600, 16543, 16372, 16664] avg=16384.0000 max=17061 max_rank=1 min=15600 min_rank=4 imbalance_ratio=1.041321 max_load_reduction=114011 max_load_reduction_pct=0.869835 moved_to_echo=65179 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 18, 4077, 0, 206, 0, 0, 0, 208, 0, 0, 2, 0, 4096, 4096, 3630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2191, 7, 0, 4076, 1, 4093, 1806, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2519, 3986, 13, 2, 0, 2934, 0, 0, 2, 0, 108, 0, 46, 1321, 615, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 4087, 0, 0, 0, 5, 27, 0, 0, 0, 4079, 22, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2025, 0, 0, 3346, 0, 0, 3479, 0, 3867, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3323, 0, 0, 0, 0, 1, 1778, 0, 0, 1, 0, 0, 4096, 0, 4, 4094, 2056, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4066, 0, 30, 0, 28, 4094, 0, 4078, 2, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 58, 1, 940, 3, 0, 4092, 413, 0, 717, 0, 0, 0, 4087, 2002, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16333, 16276, 15654, 16356, 16825, 15819, 16432, 17377] avg=16384.0000 max=17377 max_rank=7 min=15654 min_rank=2 imbalance_ratio=1.060608 max_load_reduction=113695 max_load_reduction_pct=0.867424 moved_to_echo=66453 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 4088, 0, 199, 0, 0, 0, 306, 5, 0, 0, 0, 4092, 4096, 3631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 1, 2379, 0, 0, 4065, 0, 4078, 1807, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2736, 3629, 0, 4, 0, 3648, 0, 0, 1, 0, 888, 0, 1, 1322, 615, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 4079, 0, 0, 0, 7, 50, 0, 0, 0, 4082, 8, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2926, 0, 0, 1721, 1, 0, 3481, 0, 3680, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3542, 0, 12, 0, 0, 0, 2430, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2056, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4030, 0, 23, 0, 4, 4096, 0, 4092, 5, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 1, 0, 53, 0, 0, 4088, 182, 0, 426, 0, 0, 0, 4091, 2002, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16423, 16419, 16951, 16384, 15907, 16697, 16384, 15907] avg=16384.0000 max=16951 max_rank=2 min=15907 min_rank=4 imbalance_ratio=1.034607 max_load_reduction=114121 max_load_reduction_pct=0.870674 moved_to_echo=64895 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 110, 4073, 0, 0, 0, 0, 0, 137, 1, 1, 4, 6, 4091, 4090, 3627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4093, 0, 3540, 8, 0, 3758, 2, 4087, 1798, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4090, 0, 2670, 3559, 18, 0, 4, 3311, 0, 5, 0, 5, 268, 5, 242, 1322, 617, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4080, 5, 4, 0, 0, 9, 0, 5, 5, 4076, 45, 0, 5, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2280, 0, 0, 3427, 3, 2, 3474, 7, 1931, 4091, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3520, 0, 0, 1, 0, 0, 3314, 4, 5, 2, 0, 5, 4089, 0, 5, 4091, 2061, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4091, 3908, 14, 2, 4, 2, 4090, 5, 4083, 0, 1, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 5, 480, 0, 139, 0, 4, 4084, 0, 0, 194, 1, 0, 0, 4080, 2001, 971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16143, 17302, 16134, 16416, 15230, 17563, 16233, 16051] avg=16384.0000 max=17563 max_rank=5 min=15230 min_rank=4 imbalance_ratio=1.071960 max_load_reduction=113509 max_load_reduction_pct=0.866005 moved_to_echo=65077 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 296, 4047, 0, 513, 0, 0, 0, 282, 0, 0, 0, 0, 4096, 4096, 3631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 1698, 0, 0, 4095, 0, 4072, 1807, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 3360, 4029, 7, 1, 0, 2666, 0, 0, 12, 0, 1169, 0, 0, 1322, 615, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3742, 4076, 0, 0, 0, 2, 50, 0, 0, 0, 4084, 7, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 3449, 0, 0, 3177, 0, 0, 3481, 0, 2737, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2060, 1, 107, 0, 0, 0, 2452, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 2056, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 2, 18, 0, 24, 4096, 0, 4092, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 21, 0, 426, 1, 0, 4042, 206, 0, 341, 0, 0, 0, 4095, 2002, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16961, 15749, 17290, 16042, 17052, 15334, 16447, 16197] avg=16384.0000 max=17290 max_rank=2 min=15334 min_rank=5 imbalance_ratio=1.055298 max_load_reduction=113782 max_load_reduction_pct=0.868088 moved_to_echo=65030 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 4079, 1, 26, 2, 0, 0, 66, 0, 0, 0, 1, 4095, 4096, 3631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2230, 0, 0, 4046, 0, 4096, 1807, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3295, 3717, 20, 2, 0, 3607, 0, 0, 0, 0, 281, 0, 3, 1321, 615, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 4084, 0, 0, 0, 0, 97, 0, 0, 0, 4083, 99, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 2982, 0, 1, 3398, 0, 0, 3479, 2, 2795, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2611, 1, 0, 0, 0, 1, 2627, 0, 0, 3, 1, 0, 4094, 0, 0, 4094, 2056, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 3956, 26, 34, 0, 60, 4095, 0, 4096, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 6, 0, 455, 0, 0, 4065, 74, 0, 384, 0, 0, 0, 4090, 2002, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15998, 16307, 16968, 16520, 16785, 15957, 16400, 16137] avg=16384.0000 max=16968 max_rank=2 min=15957 min_rank=5 imbalance_ratio=1.035645 max_load_reduction=114104 max_load_reduction_pct=0.870544 moved_to_echo=65279 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 31, 4071, 3, 414, 0, 0, 0, 498, 9, 0, 0, 0, 4088, 4096, 3629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 4090, 2, 2407, 0, 0, 4051, 0, 4094, 1806, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2831, 3949, 2, 1, 0, 3327, 1, 0, 1, 0, 764, 0, 1, 1321, 615, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3793, 4092, 0, 0, 0, 7, 21, 0, 2, 0, 4081, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2310, 0, 2, 2325, 5, 0, 3478, 0, 3897, 4083, 0, 0, 3, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3288, 0, 5, 0, 0, 0, 2138, 0, 0, 0, 0, 0, 4094, 2, 0, 4093, 2055, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 4047, 17, 12, 0, 5, 4093, 2, 4057, 0, 0, 3, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 10, 0, 325, 2, 3, 4075, 249, 0, 211, 0, 0, 0, 4096, 2001, 966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16839, 16557, 16921, 16092, 16125, 16139, 16368, 16031] avg=16384.0000 max=16921 max_rank=2 min=16031 min_rank=7 imbalance_ratio=1.032776 max_load_reduction=114151 max_load_reduction_pct=0.870903 moved_to_echo=64663 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 28, 4057, 0, 8, 5, 0, 0, 289, 0, 0, 0, 0, 4091, 4096, 3631, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1339, 0, 0, 4066, 0, 4092, 1807, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3248, 2736, 10, 5, 0, 1983, 0, 0, 0, 0, 710, 0, 153, 1322, 615, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 4073, 0, 0, 0, 0, 479, 0, 0, 0, 4080, 79, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 3678, 0, 1, 3798, 12, 0, 3480, 0, 2467, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3629, 0, 9, 0, 0, 0, 2670, 0, 0, 0, 0, 0, 4093, 0, 1, 4096, 2055, 465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3865, 180, 5, 0, 63, 4096, 0, 4094, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 58, 0, 290, 7, 0, 4086, 51, 0, 1069, 0, 2, 0, 4081, 2001, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16205, 15401, 14886, 16868, 17548, 17020, 16436, 16708] avg=16384.0000 max=17548 max_rank=4 min=14886 min_rank=2 imbalance_ratio=1.071045 max_load_reduction=113524 max_load_reduction_pct=0.866119 moved_to_echo=67712 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 128, 896, 128, 3200, 128, 128, 32640, 1536, 128, 3840, 128, 128, 0, 32768, 16128, 7808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 32768, 32128, 256, 128, 128, 256, 32768, 128, 32768, 128, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 25216, 128, 256, 128, 0, 128, 20096, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 16512, 3840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 128, 22784, 29696, 128, 128, 128, 25088, 128, 128, 128, 128, 4736, 128, 512, 10624, 4992, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 512, 32640, 128, 1536, 128, 128, 0, 2048, 128, 128, 128, 128, 32768, 32768, 29056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32000, 32768, 128, 128, 0, 128, 896, 0, 128, 128, 32768, 384, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 0, 128, 32768, 128, 18816, 128, 0, 32256, 128, 32768, 14464, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[384, 21888, 0, 128, 23168, 128, 128, 27904, 128, 25344, 32768, 0, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3973, 0, 1880, 0, 0, 1281, 0, 4093, 119, 0, 0, 0, 0, 4095, 0, 0, 0, 688, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 4095, 2476, 0, 1, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4091, 4093, 3478, 0, 0, 1702, 0, 4096, 4096, 0, 0, 4087, 4095, 0, 0, 1, 4095, 0, 0, 4096, 5, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 3, 2, 4089, 4095, 0, 0, 200, 4068, 0, 2, 0, 934, 2, 0, 4074, 0, 0, 0, 3062, 0, 2, 0, 0, 12, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4, 4095, 0, 0, 1, 0, 0, 0, 4092, 0, 716, 0, 4095, 10, 0, 1, 0, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3424, 0, 1483, 0, 0, 1501, 0, 4096, 525, 0, 0, 0, 0, 4096, 0, 0, 0, 316, 2, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 2013, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4095, 3390, 0, 0, 1571, 0, 4096, 4095, 0, 0, 4096, 4092, 0, 0, 0, 4096, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 6, 1, 4087, 4092, 0, 0, 540, 4058, 0, 92, 0, 1390, 0, 0, 4063, 0, 0, 0, 2999, 0, 4, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 61, 4096, 0, 0, 0, 0, 0, 0, 4093, 0, 1254, 0, 4096, 0, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 4052, 0, 3266, 0, 0, 783, 0, 4094, 146, 0, 0, 0, 0, 4094, 0, 0, 0, 151, 0, 0, 4095, 0, 4096, 4094, 0, 0, 0, 4094, 2120, 0, 2, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 4096, 0, 13, 0, 0, 4095, 4096, 3977, 0, 0, 1818, 1, 4096, 4093, 0, 0, 4080, 4093, 0, 0, 2, 4096, 0, 0, 4094, 7, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 15, 4096, 4087, 0, 0, 211, 4059, 0, 108, 0, 519, 0, 0, 4073, 0, 0, 0, 2938, 0, 3, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 19, 4092, 0, 0, 2, 0, 0, 0, 4096, 0, 413, 0, 4096, 5, 0, 2, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 3, 3727, 0, 182, 5, 5, 781, 2, 4088, 135, 0, 5, 2, 3, 4091, 0, 5, 0, 491, 0, 3, 4091, 5, 4090, 4091, 0, 0, 5, 4089, 2324, 0, 11, 0, 4091, 0, 0, 4091, 2, 6, 5, 4, 4091, 6, 1, 5, 0, 4090, 4092, 1488, 5, 0, 3214, 0, 4091, 4091, 0, 5, 4088, 4092, 0, 5, 0, 4088, 0, 5, 4090, 0, 2, 5, 0, 0, 0, 4091, 0, 5, 4089, 0, 0, 4088, 4071, 0, 5, 655, 4087, 0, 250, 5, 2898, 0, 0, 4049, 5, 3, 0, 3784, 0, 49, 4, 5, 7, 0, 4077, 0, 6, 2, 1, 0, 0, 0, 71, 4091, 5, 5, 4, 0, 0, 6, 4075, 5, 496, 0, 4091, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3715, 0, 1931, 0, 0, 1151, 0, 4092, 265, 3, 0, 0, 0, 4093, 0, 0, 0, 226, 0, 0, 4093, 0, 4095, 4093, 0, 0, 0, 4093, 2084, 0, 1, 1, 4096, 3, 0, 4094, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 4091, 3755, 0, 0, 2006, 3, 4096, 4093, 0, 0, 4080, 4086, 0, 0, 4, 4096, 0, 0, 4092, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 14, 4093, 4096, 0, 0, 452, 3919, 2, 112, 0, 1255, 2, 0, 4025, 0, 0, 0, 2594, 1, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 5, 0, 0, 58, 4092, 0, 0, 4, 0, 0, 0, 4092, 0, 1126, 0, 4091, 5, 0, 5, 0, 11] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3929, 0, 1873, 0, 0, 1226, 0, 4096, 6, 0, 0, 1, 0, 4096, 0, 0, 0, 313, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 1408, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 7, 0, 0, 4091, 4092, 3034, 0, 0, 2299, 0, 4096, 4096, 0, 0, 4093, 4093, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4064, 4096, 0, 0, 521, 4085, 0, 140, 0, 2264, 0, 0, 3944, 0, 0, 0, 3248, 0, 33, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 362, 0, 4094, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 3969, 0, 1172, 0, 0, 1269, 0, 4095, 405, 1, 0, 0, 0, 4094, 0, 0, 0, 168, 0, 0, 4095, 0, 4096, 4095, 0, 0, 1, 4094, 1300, 0, 1, 1, 4096, 6, 0, 4095, 1, 0, 0, 0, 4096, 0, 2, 0, 0, 4091, 4091, 3615, 0, 0, 2296, 1, 4096, 4095, 0, 0, 4066, 4092, 0, 0, 1, 4096, 0, 0, 4095, 4, 0, 0, 0, 1, 2, 4096, 0, 0, 4095, 0, 6, 4076, 4093, 0, 0, 402, 4033, 0, 157, 0, 1821, 0, 0, 4083, 0, 0, 0, 3739, 0, 10, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 4095, 0, 0, 1, 0, 0, 0, 4096, 0, 256, 0, 4095, 2, 0, 1, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 3418, 0, 368, 0, 0, 634, 0, 4096, 303, 0, 0, 0, 0, 4096, 0, 0, 0, 1491, 0, 0, 4091, 0, 4096, 4092, 0, 0, 2, 4091, 1036, 0, 0, 1, 4096, 9, 0, 4096, 28, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 4094, 2728, 0, 0, 2240, 0, 4096, 4090, 0, 0, 4080, 4086, 0, 0, 4, 4095, 0, 0, 4095, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 19, 4068, 4069, 0, 0, 193, 4066, 0, 541, 0, 3267, 1, 0, 4042, 0, 0, 0, 3989, 4, 52, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 17, 4095, 0, 0, 5, 0, 0, 0, 4088, 0, 295, 0, 4092, 12, 0, 5, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3973, 0, 1880, 0, 0, 1281, 0, 4093, 119, 0, 0, 0, 0, 4095, 1214, 357, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 688, 0, 0, 4095, 0, 1775, 4096, 0, 0, 0, 4095, 2476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4086, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2879, 3478, 0, 0, 1702, 0, 0, 0, 0, 0, 4087, 4095, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 3, 2, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2636, 0, 0, 200, 4068, 0, 2, 0, 934, 2, 0, 4074, 0, 0, 0, 3062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 12, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 4, 4095, 4096, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4092, 0, 716, 0, 4095, 10, 0, 1, 0, 9, 4096, 2321, 1102, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17021, 17225, 16373, 16242, 16389, 14978, 16346, 16498] avg=16384.0000 max=17225 max_rank=1 min=14978 min_rank=5 imbalance_ratio=1.051331 max_load_reduction=113847 max_load_reduction_pct=0.868584 moved_to_echo=64211 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4052, 0, 3266, 0, 0, 783, 0, 4094, 146, 0, 0, 0, 0, 4094, 1215, 356, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 4095, 0, 1775, 4094, 0, 0, 0, 4094, 2120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4086, 0, 0, 4096, 3, 0, 0, 0, 4096, 0, 13, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2881, 3977, 0, 0, 1818, 1, 0, 0, 0, 0, 4080, 4093, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 7, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 15, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2631, 0, 0, 211, 4059, 0, 108, 0, 519, 0, 0, 4073, 0, 0, 0, 2938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 19, 4092, 4096, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 413, 0, 4096, 5, 0, 2, 0, 4, 4093, 2321, 1100, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18015, 16329, 16391, 16852, 16406, 14539, 16353, 16187] avg=16384.0000 max=18015 max_rank=0 min=14539 min_rank=5 imbalance_ratio=1.099548 max_load_reduction=113057 max_load_reduction_pct=0.862556 moved_to_echo=63485 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 3727, 0, 182, 5, 5, 781, 2, 4088, 135, 0, 5, 2, 3, 4091, 1218, 358, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 491, 0, 3, 4091, 5, 1767, 4091, 0, 0, 5, 4089, 2324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4073, 0, 0, 4091, 2, 6, 5, 4, 4091, 6, 1, 5, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2874, 1488, 5, 0, 3214, 0, 0, 0, 0, 5, 4088, 4092, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4090, 0, 2, 5, 0, 0, 0, 4091, 0, 5, 4089, 0, 0, 4088, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2613, 0, 5, 655, 4087, 0, 250, 5, 2898, 0, 0, 4049, 5, 3, 0, 3784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 4, 5, 7, 0, 4077, 0, 6, 2, 1, 0, 0, 0, 71, 4091, 4091, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 4, 0, 0, 6, 4075, 5, 496, 0, 4091, 0, 0, 0, 0, 0, 4091, 2323, 1100, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14618, 16871, 16385, 15771, 16380, 18354, 16438, 16255] avg=16384.0000 max=18354 max_rank=5 min=14618 min_rank=0 imbalance_ratio=1.120239 max_load_reduction=112718 max_load_reduction_pct=0.859970 moved_to_echo=67427 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3424, 0, 1483, 0, 0, 1501, 0, 4096, 525, 0, 0, 0, 0, 4096, 1214, 357, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 316, 2, 0, 4096, 0, 1775, 4096, 0, 0, 0, 4096, 2013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2881, 3390, 0, 0, 1571, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 6, 1, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2634, 0, 0, 540, 4058, 0, 92, 0, 1390, 0, 0, 4063, 0, 0, 0, 2999, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 61, 4096, 4096, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1254, 0, 4096, 0, 0, 0, 0, 1, 4095, 2321, 1101, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16705, 16394, 16375, 16030, 16388, 15776, 16388, 17016] avg=16384.0000 max=17016 max_rank=7 min=15776 min_rank=5 imbalance_ratio=1.038574 max_load_reduction=114056 max_load_reduction_pct=0.870178 moved_to_echo=65568 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3929, 0, 1873, 0, 0, 1226, 0, 4096, 6, 0, 0, 1, 0, 4096, 1213, 357, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 313, 0, 0, 4096, 0, 1775, 4096, 0, 0, 0, 4096, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 7, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2879, 3034, 0, 0, 2299, 0, 0, 0, 0, 0, 4093, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4064, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2637, 0, 0, 521, 4085, 0, 140, 0, 2264, 0, 0, 3944, 0, 0, 0, 3248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 19, 4096, 4096, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 362, 0, 4094, 0, 0, 0, 0, 0, 4096, 2321, 1102, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16806, 15784, 16377, 16398, 16354, 16839, 16390, 16124] avg=16384.0000 max=16839 max_rank=5 min=15784 min_rank=1 imbalance_ratio=1.027771 max_load_reduction=114233 max_load_reduction_pct=0.871529 moved_to_echo=65707 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 3969, 0, 1172, 0, 0, 1269, 0, 4095, 405, 1, 0, 0, 0, 4094, 1213, 357, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 4095, 0, 1775, 4095, 0, 0, 1, 4094, 1300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4086, 6, 0, 4095, 1, 0, 0, 0, 4096, 0, 2, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2878, 3615, 0, 0, 2296, 1, 0, 0, 0, 0, 4066, 4092, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 1, 2, 4096, 0, 0, 4095, 0, 6, 4076, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2635, 0, 0, 402, 4033, 0, 157, 0, 1821, 0, 0, 4083, 0, 0, 0, 3739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 10, 4095, 4096, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 256, 0, 4095, 2, 0, 1, 0, 6, 4095, 2321, 1101, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16585, 15528, 16379, 16949, 16376, 16870, 16357, 16028] avg=16384.0000 max=16949 max_rank=3 min=15528 min_rank=1 imbalance_ratio=1.034485 max_load_reduction=114123 max_load_reduction_pct=0.870689 moved_to_echo=65631 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3715, 0, 1931, 0, 0, 1151, 0, 4092, 265, 3, 0, 0, 0, 4093, 1213, 357, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 4093, 0, 1774, 4093, 0, 0, 0, 4093, 2084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4086, 3, 0, 4094, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2878, 3755, 0, 0, 2006, 3, 0, 0, 0, 0, 4080, 4086, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 14, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2637, 0, 0, 452, 3919, 2, 112, 0, 1255, 2, 0, 4025, 0, 0, 0, 2594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 5, 0, 0, 58, 4092, 4096, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4092, 0, 1126, 0, 4091, 5, 0, 5, 0, 11, 4093, 2321, 1102, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16829, 16363, 16379, 16812, 16397, 14998, 16390, 16904] avg=16384.0000 max=16904 max_rank=7 min=14998 min_rank=5 imbalance_ratio=1.031738 max_load_reduction=114168 max_load_reduction_pct=0.871033 moved_to_echo=64689 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3418, 0, 368, 0, 0, 634, 0, 4096, 303, 0, 0, 0, 0, 4096, 1214, 355, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1491, 0, 0, 4091, 0, 1775, 4092, 0, 0, 2, 4091, 1036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4086, 9, 0, 4096, 28, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2880, 2728, 0, 0, 2240, 0, 0, 0, 0, 0, 4080, 4086, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 7, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 19, 4068, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2619, 0, 0, 193, 4066, 0, 541, 0, 3267, 1, 0, 4042, 0, 0, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 52, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 17, 4095, 4096, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4088, 0, 295, 0, 4092, 12, 0, 5, 0, 2, 4090, 2321, 1095, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14493, 16578, 16413, 16018, 16382, 18718, 16410, 16060] avg=16384.0000 max=18718 max_rank=5 min=14493 min_rank=0 imbalance_ratio=1.142456 max_load_reduction=112354 max_load_reduction_pct=0.857193 moved_to_echo=67570 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 256, 128, 128, 128, 0, 32768, 0, 128, 128, 128, 128, 0, 0, 384, 32768, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 0, 128, 32768, 128, 4992, 0, 32768, 128, 0, 128, 0, 128, 32768, 18688, 8832, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 30208, 0, 12160, 128, 128, 8704, 128, 32768, 1920, 128, 128, 128, 128, 32768, 9728, 2944, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[23040, 25472, 128, 0, 17152, 128, 0, 0, 0, 128, 32768, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 32768, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 3968, 128, 128, 32768, 128, 14208, 32768, 0, 0, 128, 32768, 14848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 0, 128, 128, 32768, 0, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[21120, 0, 128, 3200, 32384, 128, 1408, 128, 14464, 128, 0, 32384, 128, 128, 0, 26368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 98, 4096, 0, 0, 3, 4094, 4096, 0, 0, 0, 4096, 0, 4024, 0, 0, 4023, 0, 0, 5, 4090, 0, 99, 4096, 0, 0, 4092, 0, 0, 4095, 3507, 0, 0, 0, 0, 4096, 2, 742, 4096, 11, 3914, 0, 0, 4, 0, 4096, 4096, 3501, 0, 0, 4096, 4072, 0, 0, 0, 0, 2, 6, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 4094, 0, 0, 0, 0, 214, 0, 0, 17, 0, 0, 0, 4065, 0, 0, 2122, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 2238, 4096, 0, 0, 0, 112, 0, 0, 0, 0, 0, 4095, 1, 16, 0, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 11, 4095, 1, 0, 1, 4095, 4096, 0, 0, 0, 4096, 0, 4056, 0, 0, 3903, 0, 0, 1, 4095, 0, 177, 4096, 0, 0, 4096, 0, 0, 4096, 3871, 0, 0, 0, 0, 4095, 0, 1772, 4096, 6, 4081, 0, 0, 23, 0, 4096, 4096, 2888, 0, 1, 4096, 4076, 0, 0, 0, 0, 1, 11, 1, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 4089, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4094, 4095, 0, 0, 0, 0, 246, 0, 0, 4, 0, 0, 0, 4063, 0, 0, 1103, 0, 0, 1, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 2400, 4096, 0, 0, 0, 83, 0, 0, 0, 0, 0, 4095, 2, 3, 0, 4092, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 13, 4095, 0, 0, 1, 4094, 4096, 0, 0, 0, 4096, 0, 4074, 1, 0, 4064, 0, 0, 0, 4095, 0, 22, 4090, 0, 0, 4094, 0, 0, 4096, 3758, 0, 0, 0, 0, 4096, 0, 784, 4096, 2, 4087, 0, 0, 11, 0, 4096, 4096, 2951, 0, 0, 4096, 4073, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 4092, 0, 0, 0, 0, 1, 3, 0, 4096, 0, 4093, 4095, 0, 0, 0, 0, 224, 0, 0, 4, 0, 0, 1, 4074, 0, 0, 2075, 0, 0, 0, 0, 1, 0, 0, 4096, 1, 0, 0, 0, 2453, 4096, 0, 0, 0, 101, 0, 0, 0, 0, 0, 4096, 2, 5, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[5, 6, 6, 5, 10, 4091, 0, 0, 0, 4091, 4091, 0, 1, 1, 4082, 5, 4083, 1, 5, 3979, 5, 1, 0, 4091, 0, 158, 4090, 0, 5, 4091, 0, 0, 4090, 3895, 0, 5, 6, 0, 4091, 0, 2104, 4090, 23, 4054, 1, 0, 39, 5, 4090, 4091, 3811, 5, 1, 4090, 4077, 0, 0, 5, 5, 1, 0, 0, 1, 4090, 5, 0, 0, 13, 4091, 5, 0, 4089, 4077, 5, 5, 5, 1, 0, 0, 0, 4087, 0, 4091, 4091, 0, 1, 6, 3, 37, 0, 0, 0, 0, 0, 0, 4050, 5, 6, 796, 0, 5, 0, 0, 0, 5, 5, 4090, 2, 5, 0, 2, 935, 4088, 5, 0, 5, 170, 6, 0, 5, 0, 5, 4091, 9, 506, 5, 4086, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4096, 0, 0, 5, 4096, 4096, 0, 0, 0, 4096, 0, 4074, 0, 0, 3989, 0, 0, 0, 4096, 0, 279, 4094, 0, 0, 4096, 0, 0, 4096, 3698, 0, 0, 0, 0, 4096, 0, 2134, 4096, 12, 4083, 0, 0, 14, 0, 4096, 4096, 2566, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 3, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 273, 0, 0, 3, 0, 0, 0, 4077, 0, 0, 1062, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 2340, 4096, 0, 0, 0, 78, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 172, 4096, 0, 0, 7, 4096, 4096, 0, 0, 0, 4095, 0, 4063, 11, 0, 4048, 0, 0, 0, 4096, 0, 385, 4048, 0, 0, 4096, 0, 0, 4095, 3863, 0, 1, 0, 0, 4096, 0, 1623, 4096, 8, 4093, 0, 0, 0, 0, 4096, 4096, 2633, 0, 0, 4096, 4032, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 1, 0, 273, 0, 0, 9, 0, 0, 20, 4060, 0, 0, 1338, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1906, 4095, 0, 0, 0, 220, 0, 0, 0, 0, 0, 4096, 3, 52, 0, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 11, 4090, 3, 0, 12, 4094, 4096, 0, 0, 0, 4096, 0, 4027, 0, 0, 4072, 0, 0, 7, 4091, 0, 80, 4095, 0, 0, 4091, 0, 0, 4096, 3760, 0, 0, 0, 0, 4092, 3, 639, 4096, 34, 4073, 0, 0, 2, 0, 4096, 4096, 3440, 0, 0, 4096, 4081, 4, 0, 0, 0, 8, 36, 8, 0, 4096, 0, 0, 0, 2, 4090, 0, 0, 4093, 4075, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 4091, 4093, 0, 0, 0, 0, 179, 0, 0, 28, 0, 0, 0, 4026, 0, 0, 1179, 0, 0, 1, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 2905, 4096, 0, 0, 0, 208, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4091, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 48, 4094, 0, 0, 1, 4096, 4096, 0, 0, 0, 4096, 0, 4067, 0, 0, 4053, 0, 0, 9, 4086, 0, 654, 4096, 0, 0, 4096, 0, 0, 4096, 3303, 0, 0, 0, 0, 4096, 0, 2532, 4096, 11, 4051, 0, 0, 0, 0, 4096, 4096, 3008, 0, 0, 4096, 4080, 0, 0, 0, 0, 14, 12, 5, 0, 4096, 0, 0, 0, 1, 4091, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 4096, 0, 0, 0, 0, 479, 0, 0, 7, 0, 0, 2, 4070, 0, 0, 290, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1177, 4096, 0, 0, 0, 683, 0, 0, 0, 0, 0, 4096, 7, 244, 0, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 13, 4095, 0, 0, 1, 4094, 4049, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 1, 0, 4064, 0, 0, 0, 3986, 0, 22, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2960, 3758, 0, 0, 0, 0, 4096, 0, 784, 0, 2, 4087, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 0, 2951, 0, 0, 4096, 4073, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4092, 0, 0, 0, 0, 1, 3, 0, 3291, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 0, 4093, 4095, 0, 0, 0, 0, 224, 0, 0, 4, 0, 0, 1, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2075, 0, 0, 0, 0, 1, 0, 0, 4096, 1, 0, 0, 0, 2453, 4096, 4096, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 101, 0, 0, 0, 0, 0, 4096, 2, 5, 0, 4096, 0, 2278, 1136, 212, 109, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16348, 16237, 15698, 16226, 16389, 16375, 17621, 16178] avg=16384.0000 max=17621 max_rank=6 min=15698 min_rank=2 imbalance_ratio=1.075500 max_load_reduction=113451 max_load_reduction_pct=0.865562 moved_to_echo=66563 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 98, 4096, 0, 0, 3, 4094, 4049, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 4023, 0, 0, 5, 3981, 0, 99, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2959, 3507, 0, 0, 0, 0, 4096, 2, 742, 0, 11, 3914, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 0, 3501, 0, 0, 4096, 4072, 0, 0, 0, 0, 2, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 0, 3290, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 0, 4094, 4094, 0, 0, 0, 0, 214, 0, 0, 17, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2122, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 2238, 4096, 4096, 802, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 112, 0, 0, 0, 0, 0, 4095, 1, 16, 0, 4093, 0, 2278, 1136, 212, 109, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16436, 16228, 15235, 16780, 16379, 16368, 17451, 16195] avg=16384.0000 max=17451 max_rank=6 min=15235 min_rank=2 imbalance_ratio=1.065125 max_load_reduction=113621 max_load_reduction_pct=0.866859 moved_to_echo=66393 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 11, 4095, 1, 0, 1, 4095, 4049, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 0, 0, 3903, 0, 0, 1, 3986, 0, 177, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2960, 3871, 0, 0, 0, 0, 4095, 0, 1772, 0, 6, 4081, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 0, 2888, 0, 1, 4096, 4076, 0, 0, 0, 0, 1, 11, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4089, 0, 0, 0, 0, 1, 0, 0, 3293, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 0, 4094, 4095, 0, 0, 0, 0, 246, 0, 0, 4, 0, 0, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1103, 0, 0, 1, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 2400, 4096, 4096, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 83, 0, 0, 0, 0, 0, 4095, 2, 3, 0, 4092, 0, 2278, 1136, 212, 109, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16348, 16219, 16808, 16177, 16385, 16386, 16596, 16153] avg=16384.0000 max=16808 max_rank=2 min=16153 min_rank=7 imbalance_ratio=1.025879 max_load_reduction=114264 max_load_reduction_pct=0.871765 moved_to_echo=65520 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 172, 4096, 0, 0, 7, 4096, 4049, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 11, 0, 4048, 0, 0, 0, 3987, 0, 385, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2959, 3863, 0, 1, 0, 0, 4096, 0, 1623, 0, 8, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 0, 2633, 0, 0, 4096, 4032, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 3293, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3882, 0, 4096, 4096, 0, 0, 1, 0, 273, 0, 0, 9, 0, 0, 20, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1338, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1906, 4096, 4096, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 220, 0, 0, 0, 0, 0, 4096, 3, 52, 0, 4093, 0, 2278, 1136, 212, 109, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16515, 16542, 16643, 15865, 16393, 16437, 16336, 16341] avg=16384.0000 max=16643 max_rank=2 min=15865 min_rank=3 imbalance_ratio=1.015808 max_load_reduction=114429 max_load_reduction_pct=0.873024 moved_to_echo=65507 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[5, 6, 6, 5, 10, 4091, 0, 0, 0, 4091, 4040, 0, 1, 1, 4082, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 1, 5, 3979, 5, 1, 0, 3977, 0, 158, 4090, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2952, 3895, 0, 5, 6, 0, 4091, 0, 2104, 0, 23, 4054, 1, 0, 39, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 992, 0, 3811, 5, 1, 4090, 4077, 0, 0, 5, 5, 1, 0, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 13, 4091, 5, 0, 4089, 4077, 5, 5, 5, 1, 0, 0, 0, 3290, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3870, 0, 4091, 4091, 0, 1, 6, 3, 37, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 796, 0, 5, 0, 0, 0, 5, 5, 4090, 2, 5, 0, 2, 935, 4091, 4090, 801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 5, 0, 5, 170, 6, 0, 5, 0, 5, 4091, 9, 506, 5, 4086, 0, 2281, 1138, 217, 114, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16343, 16304, 17175, 17078, 16403, 16149, 14838, 16782] avg=16384.0000 max=17175 max_rank=2 min=14838 min_rank=6 imbalance_ratio=1.048279 max_load_reduction=113897 max_load_reduction_pct=0.868965 moved_to_echo=64172 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1, 4096, 0, 0, 5, 4096, 4049, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 3989, 0, 0, 0, 3987, 0, 279, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2960, 3698, 0, 0, 0, 0, 4096, 0, 2134, 0, 12, 4083, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 0, 2566, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 3, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 3293, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 0, 4095, 4096, 0, 0, 0, 0, 273, 0, 0, 3, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1062, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 2340, 4096, 4096, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 78, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 2278, 1136, 212, 109, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16343, 16423, 16997, 15852, 16386, 16428, 16494, 16149] avg=16384.0000 max=16997 max_rank=2 min=15852 min_rank=3 imbalance_ratio=1.037415 max_load_reduction=114075 max_load_reduction_pct=0.870323 moved_to_echo=65457 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 11, 4090, 3, 0, 12, 4094, 4049, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 4072, 0, 0, 7, 3982, 0, 80, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2960, 3760, 0, 0, 0, 0, 4092, 3, 639, 0, 34, 4073, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 0, 3440, 0, 0, 4096, 4081, 4, 0, 0, 0, 8, 36, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4090, 0, 0, 4093, 4075, 0, 0, 0, 0, 5, 0, 0, 3290, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 0, 4091, 4093, 0, 0, 0, 0, 179, 0, 0, 28, 0, 0, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1179, 0, 0, 1, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 2905, 4096, 4096, 801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 208, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4091, 0, 2278, 1136, 212, 109, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16355, 16263, 15563, 16776, 16366, 16301, 17178, 16270] avg=16384.0000 max=17178 max_rank=6 min=15563 min_rank=2 imbalance_ratio=1.048462 max_load_reduction=113894 max_load_reduction_pct=0.868942 moved_to_echo=66115 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 48, 4094, 0, 0, 1, 4096, 4049, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 4053, 0, 0, 9, 3977, 0, 654, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2960, 3303, 0, 0, 0, 0, 4096, 0, 2532, 0, 11, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1007, 0, 3008, 0, 0, 4096, 4080, 0, 0, 0, 0, 14, 12, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 3293, 811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3884, 0, 4090, 4096, 0, 0, 0, 0, 479, 0, 0, 7, 0, 0, 2, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1177, 4096, 4096, 803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 683, 0, 0, 0, 0, 0, 4096, 7, 244, 0, 4096, 0, 2278, 1136, 212, 109, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16856, 16953, 16318, 16371, 16628, 14558, 17004] avg=16384.0000 max=17004 max_rank=7 min=14558 min_rank=6 imbalance_ratio=1.037842 max_load_reduction=114068 max_load_reduction_pct=0.870270 moved_to_echo=64561 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32512, 128, 128, 32256, 128, 128, 128, 31872, 0, 1920, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 384, 32768, 128, 0, 128, 32768, 32384, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[8064, 0, 24832, 128, 128, 32768, 32640, 128, 0, 128, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 9984, 0, 128, 128, 0, 128, 128, 128, 32768, 128, 128, 0, 128, 16384, 32768, 32768, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[23680, 29696, 0, 128, 128, 0, 32768, 128, 12416, 0, 128, 32512, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[31104, 0, 32768, 32768, 0, 128, 128, 128, 2048, 0, 0, 128, 0, 0, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 0, 128, 1664, 128, 0, 128, 0, 128, 32768, 128, 896, 128, 32768, 0, 18304, 9216, 1792, 896, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 0, 128, 32768, 128, 0, 32768, 32768, 128, 128, 128, 128, 128, 128, 0, 26368, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[127, 0, 4088, 0, 0, 4061, 4096, 0, 0, 0, 25, 3, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 4095, 0, 4096, 0, 4096, 3986, 0, 0, 0, 1, 381, 1820, 0, 0, 0, 4096, 0, 2, 0, 206, 0, 0, 0, 101, 4095, 0, 0, 0, 4096, 0, 0, 2, 3994, 4096, 1, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4090, 0, 0, 4089, 4096, 4095, 0, 3323, 0, 4094, 0, 0, 244, 0, 0, 11, 0, 2287, 4092, 0, 0, 0, 1, 0, 4096, 0, 0, 4037, 9, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[64, 0, 4090, 0, 0, 4054, 4096, 0, 0, 0, 24, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 4096, 0, 4096, 0, 4096, 4046, 0, 0, 0, 0, 637, 1651, 0, 0, 3, 4096, 0, 0, 0, 57, 0, 0, 0, 394, 4096, 0, 0, 0, 4096, 0, 0, 2, 3863, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4091, 0, 0, 3649, 4096, 4096, 0, 2932, 0, 4096, 0, 0, 222, 0, 0, 35, 0, 3093, 4083, 0, 0, 0, 0, 0, 4096, 0, 0, 3971, 14, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[63, 0, 4077, 0, 0, 4065, 4096, 0, 0, 0, 36, 5, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 4093, 0, 4096, 0, 4096, 4032, 0, 0, 0, 1, 385, 424, 0, 0, 1, 4096, 0, 0, 0, 297, 0, 0, 0, 152, 4096, 0, 0, 0, 4096, 0, 0, 4, 4045, 4096, 4, 4069, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3987, 0, 0, 4027, 4096, 4094, 0, 3802, 0, 4093, 0, 0, 249, 0, 0, 74, 0, 3118, 4092, 0, 0, 0, 15, 0, 4096, 0, 0, 4041, 16, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[11, 0, 4089, 0, 0, 4088, 4096, 0, 0, 0, 39, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 4094, 0, 4096, 0, 4096, 3892, 0, 0, 0, 2, 547, 1811, 0, 0, 0, 4096, 0, 0, 0, 222, 0, 0, 0, 98, 4096, 0, 0, 0, 4096, 0, 0, 0, 3810, 4096, 1, 4066, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 4087, 0, 0, 4050, 4096, 4095, 0, 3700, 0, 4094, 0, 0, 77, 0, 0, 40, 0, 2265, 4093, 0, 0, 0, 25, 0, 4096, 0, 0, 4040, 12, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[154, 5, 4091, 1, 0, 4061, 4077, 2, 0, 5, 37, 2, 4, 1, 5, 0, 0, 5, 0, 6, 0, 0, 5, 5, 0, 1, 0, 5, 5, 4091, 4090, 4091, 5, 4091, 5, 4091, 3864, 0, 0, 3, 0, 553, 1711, 1, 0, 2, 4090, 1, 23, 5, 101, 2, 2, 0, 308, 4091, 6, 5, 0, 4090, 0, 0, 34, 3769, 4090, 0, 4091, 4088, 0, 1, 0, 5, 0, 0, 0, 5, 0, 14, 4, 4089, 0, 0, 5, 5, 5, 4078, 5, 0, 17, 3, 5, 4088, 5, 0, 4089, 4076, 1, 5, 4031, 4091, 4091, 6, 3563, 0, 4091, 1, 6, 603, 2, 0, 0, 4, 1650, 4091, 1, 2, 0, 120, 0, 4070, 0, 0, 4094, 0, 4090, 4, 0, 4085] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[9, 0, 4090, 0, 0, 4082, 4096, 0, 0, 0, 10, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4096, 4096, 0, 4096, 0, 4096, 3724, 0, 0, 0, 0, 908, 1603, 0, 0, 0, 4096, 0, 0, 0, 24, 0, 0, 0, 93, 4096, 0, 0, 0, 4096, 0, 0, 3, 4012, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4073, 4096, 4096, 0, 3760, 0, 4096, 0, 0, 101, 0, 0, 27, 0, 2221, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 4041, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[72, 0, 4092, 0, 0, 4039, 4096, 0, 0, 0, 57, 4, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 4090, 0, 4096, 0, 4096, 3958, 0, 0, 0, 4, 429, 1972, 0, 0, 2, 4096, 0, 0, 0, 398, 0, 0, 0, 111, 4094, 0, 0, 4, 4096, 0, 0, 8, 3885, 4096, 4, 4064, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4072, 0, 0, 3914, 4096, 4092, 0, 3066, 0, 4092, 0, 0, 262, 0, 0, 61, 0, 2496, 4085, 0, 0, 0, 2, 0, 4096, 0, 0, 3987, 30, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 4081, 4096, 0, 0, 0, 34, 11, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 4095, 0, 4096, 0, 4096, 4019, 0, 0, 0, 1, 358, 1306, 0, 0, 0, 4096, 0, 0, 0, 84, 0, 0, 0, 476, 4096, 0, 0, 0, 4096, 0, 0, 3, 3805, 4096, 5, 4081, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4039, 0, 0, 4016, 4096, 4089, 0, 3973, 0, 4096, 0, 0, 158, 0, 0, 46, 0, 2355, 4093, 0, 0, 0, 5, 0, 4096, 0, 0, 4024, 20, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[127, 0, 4088, 0, 0, 4061, 4096, 0, 0, 0, 25, 3, 0, 0, 0, 0, 3662, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4096, 4095, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2189, 0, 4096, 3986, 0, 0, 0, 1, 381, 1820, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 206, 0, 0, 0, 101, 4095, 0, 0, 0, 4096, 0, 0, 2, 3994, 2023, 1870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4090, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 422, 4095, 0, 3323, 0, 4094, 0, 0, 244, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2287, 4092, 0, 0, 0, 1, 0, 4096, 0, 0, 4037, 9, 0, 0, 0, 1702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16433, 16387, 16569, 16389, 16388, 16404, 16278, 16224] avg=16384.0000 max=16569 max_rank=2 min=16224 min_rank=7 imbalance_ratio=1.011292 max_load_reduction=114503 max_load_reduction_pct=0.873589 moved_to_echo=65294 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[63, 0, 4077, 0, 0, 4065, 4096, 0, 0, 0, 36, 5, 0, 0, 0, 0, 3662, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 4093, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2189, 0, 4096, 4032, 0, 0, 0, 1, 385, 424, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 0, 152, 4096, 0, 0, 0, 4096, 0, 0, 4, 4045, 2023, 1870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4069, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3987, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 422, 4094, 0, 3802, 0, 4093, 0, 0, 249, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3118, 4092, 0, 0, 0, 15, 0, 4096, 0, 0, 4041, 16, 0, 0, 0, 1702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16376, 15224, 16583, 16372, 16301, 16761, 17080] avg=16384.0000 max=17080 max_rank=7 min=15224 min_rank=2 imbalance_ratio=1.042480 max_load_reduction=113992 max_load_reduction_pct=0.869690 moved_to_echo=66514 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[64, 0, 4090, 0, 0, 4054, 4096, 0, 0, 0, 24, 4, 0, 0, 0, 0, 3662, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4096, 4096, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2189, 0, 4096, 4046, 0, 0, 0, 0, 637, 1651, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 394, 4096, 0, 0, 0, 4096, 0, 0, 2, 3863, 2023, 1870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4091, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3649, 422, 4096, 0, 2932, 0, 4096, 0, 0, 222, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3093, 4083, 0, 0, 0, 0, 0, 4096, 0, 0, 3971, 14, 0, 0, 0, 1702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16365, 16378, 16718, 16401, 16395, 16404, 15452, 16959] avg=16384.0000 max=16959 max_rank=7 min=15452 min_rank=6 imbalance_ratio=1.035095 max_load_reduction=114113 max_load_reduction_pct=0.870613 moved_to_echo=65210 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[154, 5, 4091, 1, 0, 4061, 4077, 2, 0, 5, 37, 2, 4, 1, 5, 0, 3662, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 6, 0, 0, 5, 5, 0, 1, 0, 5, 5, 4091, 4090, 4091, 4090, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2175, 5, 4091, 3864, 0, 0, 3, 0, 553, 1711, 1, 0, 2, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 5, 101, 2, 2, 0, 308, 4091, 6, 5, 0, 4090, 0, 0, 34, 3769, 2020, 1869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4091, 4088, 0, 1, 0, 5, 0, 0, 0, 5, 0, 14, 4, 4089, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 4078, 5, 0, 17, 3, 5, 4088, 5, 0, 4089, 4076, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 4031, 415, 4091, 6, 3563, 0, 4091, 1, 6, 603, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1650, 4091, 1, 2, 0, 120, 0, 4070, 0, 0, 4094, 0, 0, 4, 0, 1693, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16479, 16408, 16501, 16325, 16403, 16412, 16819, 15725] avg=16384.0000 max=16819 max_rank=6 min=15725 min_rank=7 imbalance_ratio=1.026550 max_load_reduction=114253 max_load_reduction_pct=0.871681 moved_to_echo=65359 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[11, 0, 4089, 0, 0, 4088, 4096, 0, 0, 0, 39, 1, 0, 0, 0, 0, 3662, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 4094, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2189, 0, 4096, 3892, 0, 0, 0, 2, 547, 1811, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 0, 0, 98, 4096, 0, 0, 0, 4096, 0, 0, 0, 3810, 2023, 1870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4066, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 4087, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 422, 4095, 0, 3700, 0, 4094, 0, 0, 77, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2265, 4093, 0, 0, 0, 25, 0, 4096, 0, 0, 4040, 12, 0, 0, 0, 1702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16357, 16388, 16633, 16215, 16366, 16402, 16478, 16233] avg=16384.0000 max=16633 max_rank=2 min=16215 min_rank=3 imbalance_ratio=1.015198 max_load_reduction=114439 max_load_reduction_pct=0.873100 moved_to_echo=65479 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 4090, 0, 0, 4082, 4096, 0, 0, 0, 10, 5, 0, 0, 0, 1, 3662, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 4096, 4096, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2189, 0, 4096, 3724, 0, 0, 0, 0, 908, 1603, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 93, 4096, 0, 0, 0, 4096, 0, 0, 3, 4012, 2023, 1870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 422, 4096, 0, 3760, 0, 4096, 0, 0, 101, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2221, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 4041, 0, 0, 0, 0, 1702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16326, 16374, 16616, 16217, 16396, 16410, 16575, 16158] avg=16384.0000 max=16616 max_rank=2 min=16158 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=114456 max_load_reduction_pct=0.873230 moved_to_echo=65539 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[72, 0, 4092, 0, 0, 4039, 4096, 0, 0, 0, 57, 4, 0, 0, 0, 0, 3662, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 4096, 4090, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2189, 0, 4096, 3958, 0, 0, 0, 4, 429, 1972, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 398, 0, 0, 0, 111, 4094, 0, 0, 4, 4096, 0, 0, 8, 3885, 2023, 1870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4064, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4072, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3914, 422, 4092, 0, 3066, 0, 4092, 0, 0, 262, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2496, 4085, 0, 0, 0, 2, 0, 4096, 0, 0, 3987, 30, 0, 0, 0, 1702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16393, 16384, 16746, 16489, 16367, 16386, 15909, 16398] avg=16384.0000 max=16746 max_rank=2 min=15909 min_rank=6 imbalance_ratio=1.022095 max_load_reduction=114326 max_load_reduction_pct=0.872238 moved_to_echo=65060 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 4081, 4096, 0, 0, 0, 34, 11, 0, 0, 0, 0, 3662, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 4095, 4096, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2189, 0, 4096, 4019, 0, 0, 0, 1, 358, 1306, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 476, 4096, 0, 0, 0, 4096, 0, 0, 3, 3805, 2023, 1870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 4081, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4039, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 422, 4089, 0, 3973, 0, 4096, 0, 0, 158, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2355, 4093, 0, 0, 0, 5, 0, 4096, 0, 0, 4024, 20, 0, 0, 0, 1702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16344, 16377, 16065, 16453, 16385, 16353, 16800, 16295] avg=16384.0000 max=16800 max_rank=6 min=16065 min_rank=2 imbalance_ratio=1.025391 max_load_reduction=114272 max_load_reduction_pct=0.871826 moved_to_echo=65833 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[19584, 32768, 128, 128, 0, 256, 0, 32768, 0, 0, 32256, 128, 0, 128, 0, 13696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[512, 128, 32768, 128, 0, 32640, 32768, 128, 0, 128, 384, 128, 128, 128, 128, 128, 29312, 3072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 17536, 128, 32768, 31616, 0, 0, 128, 128, 4224, 12416, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 32768, 32768, 0, 128, 0, 128, 0, 128, 0, 128, 0, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 128, 128, 32768, 128, 0, 128, 128, 128, 32768, 128, 128, 32768, 32640, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 31872, 3456, 32768, 128, 28160, 0, 32768, 128, 128, 1920, 128, 0, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 1408, 128, 128, 0, 1792, 32768, 128, 128, 128, 32768, 0, 0, 128, 31232, 16256, 14976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 0, 0, 128, 128, 0, 128, 0, 128, 128, 32768, 32768, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1426, 0, 1168, 0, 0, 0, 5, 0, 1032, 0, 0, 0, 16, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 39, 0, 0, 0, 0, 763, 0, 0, 0, 4095, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 3981, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 477, 0, 0, 3406, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 4095, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4095, 4093, 0, 0, 726, 0, 1351, 0, 0, 0, 28, 0, 1181, 0, 0, 0, 1, 5, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 4096, 4096, 36, 0, 0, 0, 0, 2447, 0, 0, 0, 4095, 0, 0, 0, 4087, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 3938, 4096, 4096, 0, 0, 0, 0, 4082, 0, 4096, 4096, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 146, 0, 0, 2458, 0, 0, 1, 0, 1, 0, 0, 0, 24, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1005, 0, 1878, 0, 0, 0, 44, 0, 862, 0, 0, 0, 295, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4096, 4096, 4094, 57, 0, 0, 1, 0, 1604, 0, 0, 0, 4082, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3523, 4081, 4065, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 10, 0, 0, 3062, 0, 0, 0, 0, 7, 0, 0, 0, 10, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4095, 4095, 0, 0, 320, 0, 1183, 0, 0, 1, 17, 0, 1915, 0, 0, 0, 29, 1, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4095, 4096, 0, 1, 2, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4095, 4096, 51, 0, 0, 1, 0, 1697, 0, 0, 0, 4091, 0, 0, 0, 4092, 0, 1, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4095, 4091, 0, 3934, 4096, 4096, 0, 0, 0, 1, 4089, 0, 4096, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 47, 0, 0, 3116, 0, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4092, 4092, 0, 0, 257, 0, 2671, 0, 0, 0, 12, 0, 729, 0, 0, 0, 59, 4, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 2, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 4096, 4096, 64, 0, 0, 0, 0, 1372, 0, 0, 0, 4095, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4094, 4083, 0, 3940, 4096, 4096, 0, 0, 0, 0, 4091, 0, 4096, 4096, 0, 0, 0, 4096, 4082, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 261, 0, 0, 2945, 0, 0, 3, 0, 8, 0, 0, 0, 20, 0, 0, 0, 4095, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4090, 0, 7, 2, 6, 4, 4091, 4091, 4090, 6, 0, 290, 5, 1906, 1, 0, 0, 0, 6, 126, 6, 7, 5, 8, 0, 5, 4089, 4090, 5, 0, 0, 2, 4090, 4091, 4089, 0, 0, 0, 0, 5, 0, 4087, 1, 1, 0, 0, 0, 4091, 4090, 4091, 11, 2, 0, 0, 5, 2782, 0, 5, 6, 4085, 0, 5, 0, 4073, 0, 0, 4090, 1, 5, 4090, 5, 5, 1, 4, 5, 0, 5, 4088, 4093, 15, 4087, 4079, 4087, 0, 0, 0, 19, 4076, 3, 4090, 4091, 5, 0, 2, 4088, 4080, 0, 5, 0, 5, 5, 0, 0, 4091, 0, 4091, 17, 0, 0, 2995, 1, 0, 10, 0, 0, 6, 0, 6, 76, 5, 0, 0, 4085, 1, 0, 0, 6, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4095, 4095, 0, 0, 521, 0, 952, 0, 0, 0, 8, 0, 1066, 0, 0, 0, 213, 1, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 4096, 4096, 4096, 35, 0, 0, 0, 0, 2929, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 4005, 4096, 4095, 0, 0, 0, 3, 4094, 0, 4096, 4096, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 239, 0, 0, 2285, 0, 0, 1, 0, 7, 0, 0, 0, 41, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 192, 0, 1169, 0, 0, 0, 9, 0, 2389, 0, 0, 0, 1, 3, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 4096, 96, 0, 0, 0, 0, 2124, 0, 0, 0, 4088, 0, 0, 0, 4082, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 3922, 4096, 4096, 0, 0, 0, 5, 4088, 0, 4096, 4096, 0, 0, 0, 4096, 4012, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 1, 0, 0, 2499, 0, 0, 0, 0, 1, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 1970, 4096, 4096, 0, 0, 1426, 0, 1168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1032, 0, 0, 0, 16, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 2812, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2096, 4096, 39, 0, 0, 0, 0, 763, 0, 0, 0, 4095, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3981, 205, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 477, 0, 0, 3406, 0, 0, 2126, 1994, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16852, 16179, 16386, 15179, 16371, 16473, 17274, 16358] avg=16384.0000 max=17274 max_rank=6 min=15179 min_rank=3 imbalance_ratio=1.054321 max_load_reduction=113798 max_load_reduction_pct=0.868210 moved_to_echo=66476 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 1970, 4096, 4096, 0, 0, 1005, 0, 1878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 862, 0, 0, 0, 295, 0, 0, 4096, 4096, 0, 0, 0, 0, 4094, 2802, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2096, 4094, 57, 0, 0, 1, 0, 1604, 0, 0, 0, 4082, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3523, 204, 4065, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 10, 0, 0, 3062, 0, 0, 2126, 1994, 1075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 10, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17141, 16317, 16386, 16029, 16390, 15977, 16459, 16373] avg=16384.0000 max=17141 max_rank=0 min=15977 min_rank=5 imbalance_ratio=1.046204 max_load_reduction=113931 max_load_reduction_pct=0.869225 moved_to_echo=65199 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 7, 2, 6, 4, 1966, 4091, 4090, 6, 0, 290, 5, 1906, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 126, 6, 7, 5, 8, 0, 5, 4089, 4090, 5, 0, 0, 2, 4090, 2805, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4089, 0, 0, 0, 0, 5, 0, 4087, 1, 1, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2083, 4091, 11, 2, 0, 0, 5, 2782, 0, 5, 6, 4085, 0, 5, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 5, 4090, 5, 5, 1, 4, 5, 0, 5, 4088, 4093, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 194, 4087, 0, 0, 0, 19, 4076, 3, 0, 0, 5, 0, 2, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5, 5, 0, 0, 4091, 0, 4091, 17, 0, 0, 2995, 1, 0, 2125, 1994, 1080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 6, 0, 6, 76, 5, 0, 0, 4085, 1, 0, 0, 6, 5, 4091, 4090, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16464, 15273, 16365, 17148, 16420, 16553, 16409, 16440] avg=16384.0000 max=17148 max_rank=3 min=15273 min_rank=1 imbalance_ratio=1.046631 max_load_reduction=113924 max_load_reduction_pct=0.869171 moved_to_echo=65822 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 1970, 4095, 4095, 0, 0, 320, 0, 1183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 17, 0, 1915, 0, 0, 0, 29, 1, 0, 4096, 4096, 0, 0, 0, 0, 4094, 2812, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 4096, 0, 1, 2, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2096, 4096, 51, 0, 0, 1, 0, 1697, 0, 0, 0, 4091, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4095, 4091, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3934, 205, 4096, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 47, 0, 0, 3116, 0, 0, 2125, 1993, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15759, 17089, 16387, 16124, 16383, 16420, 16549, 16361] avg=16384.0000 max=17089 max_rank=1 min=15759 min_rank=0 imbalance_ratio=1.043030 max_load_reduction=113983 max_load_reduction_pct=0.869621 moved_to_echo=65713 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 1970, 4095, 4093, 0, 0, 726, 0, 1351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 1181, 0, 0, 0, 1, 5, 0, 4096, 4096, 0, 0, 0, 0, 4094, 2812, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2096, 4096, 36, 0, 0, 0, 0, 2447, 0, 0, 0, 4095, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4081, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3938, 205, 4096, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 146, 0, 0, 2458, 0, 0, 2126, 1994, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 24, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16331, 16341, 16386, 16857, 16373, 16412, 15990, 16382] avg=16384.0000 max=16857 max_rank=3 min=15990 min_rank=6 imbalance_ratio=1.028870 max_load_reduction=114215 max_load_reduction_pct=0.871391 moved_to_echo=65157 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 1970, 4095, 4095, 0, 0, 521, 0, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1066, 0, 0, 0, 213, 1, 0, 4096, 4096, 0, 0, 0, 0, 4094, 2812, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 1, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2096, 4096, 35, 0, 0, 0, 0, 2929, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 205, 4095, 0, 0, 0, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 239, 0, 0, 2285, 0, 0, 2126, 1994, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 41, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15729, 16414, 16385, 17344, 16386, 16495, 15914, 16405] avg=16384.0000 max=17344 max_rank=3 min=15729 min_rank=0 imbalance_ratio=1.058594 max_load_reduction=113728 max_load_reduction_pct=0.867676 moved_to_echo=65200 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 1970, 4092, 4092, 0, 0, 257, 0, 2671, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 729, 0, 0, 0, 59, 4, 0, 4096, 4096, 0, 0, 0, 0, 4095, 2812, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 2, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2096, 4096, 64, 0, 0, 0, 0, 1372, 0, 0, 0, 4095, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4094, 4083, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 205, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 261, 0, 0, 2945, 0, 0, 2126, 1994, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8, 0, 0, 0, 20, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17178, 15931, 16391, 15809, 16372, 16414, 16591, 16386] avg=16384.0000 max=17178 max_rank=0 min=15809 min_rank=3 imbalance_ratio=1.048462 max_load_reduction=113894 max_load_reduction_pct=0.868942 moved_to_echo=65763 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 1970, 4096, 4095, 0, 0, 192, 0, 1169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2389, 0, 0, 0, 1, 3, 0, 4096, 4096, 0, 0, 0, 0, 4094, 2812, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2096, 4096, 96, 0, 0, 0, 0, 2124, 0, 0, 0, 4088, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 4086, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3922, 205, 4096, 0, 0, 0, 5, 4088, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4096, 1, 0, 0, 2499, 0, 0, 2126, 1994, 1079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15618, 17528, 16386, 16582, 16377, 16328, 15886, 16367] avg=16384.0000 max=17528 max_rank=1 min=15618 min_rank=0 imbalance_ratio=1.069824 max_load_reduction=113544 max_load_reduction_pct=0.866272 moved_to_echo=64958 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 32768, 32768, 0, 128, 128, 0, 128, 0, 32768, 128, 128, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 128, 0, 128, 32768, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[16768, 32768, 512, 128, 0, 128, 128, 15744, 0, 128, 128, 32768, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[31360, 1664, 32768, 0, 0, 0, 128, 32768, 128, 0, 0, 128, 0, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 15872, 32768, 32768, 128, 0, 4864, 128, 12288, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 32768, 0, 32768, 1280, 0, 0, 22784, 128, 0, 17024, 16000, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 9344, 128, 128, 128, 640, 128, 128, 32768, 32768, 128, 0, 0, 128, 32768, 22528, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 0, 128, 256, 128, 0, 0, 32768, 128, 0, 128, 128, 128, 32768, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3730, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 10, 0, 0, 0, 0, 0, 4096, 76, 0, 4095, 4095, 0, 0, 4092, 1, 0, 5, 4094, 0, 2, 4090, 4096, 0, 0, 0, 0, 0, 4096, 1832, 8, 0, 6, 4090, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 409, 0, 0, 0, 4096, 0, 4096, 4096, 0, 7, 4096, 4096, 4096, 4095, 3809, 0, 29, 0, 1, 0, 4094, 0, 4096, 4090, 0, 0, 0, 4096, 0, 0, 0, 2449, 0, 0, 0, 0, 4096, 0, 0, 4033, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3354, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 15, 0, 4094, 4096, 0, 0, 4076, 0, 0, 7, 4094, 0, 23, 4093, 4096, 0, 0, 0, 0, 0, 4096, 2524, 12, 0, 54, 4080, 1, 0, 4096, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 345, 0, 0, 0, 4096, 0, 4096, 4096, 0, 3, 4096, 4096, 4096, 4096, 3385, 0, 134, 0, 2, 0, 4016, 0, 4096, 4080, 0, 0, 0, 4096, 0, 0, 0, 2594, 0, 0, 0, 0, 4096, 0, 0, 4063, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[5, 0, 0, 4091, 5, 1, 6, 0, 2896, 0, 6, 0, 1, 5, 0, 0, 1, 0, 5, 4090, 5, 5, 4088, 6, 6, 5, 0, 4090, 0, 2, 5, 1, 23, 0, 0, 4060, 40, 5, 4090, 4090, 6, 3, 4091, 5, 0, 0, 4091, 2, 4, 4089, 4090, 0, 0, 6, 5, 0, 4090, 1596, 0, 0, 35, 4091, 0, 23, 4089, 3, 0, 0, 2, 4089, 0, 0, 4, 0, 2099, 4, 6, 0, 4087, 0, 4088, 4081, 0, 0, 4087, 4089, 4090, 4091, 3725, 0, 13, 3, 0, 5, 4030, 5, 4090, 4005, 5, 5, 7, 4090, 0, 5, 130, 2006, 5, 5, 1, 0, 4090, 8, 0, 3948, 4090, 0, 5, 6, 4, 0, 6, 2, 5, 4, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3247, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 13, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 0, 4089, 1, 0, 2, 4095, 0, 9, 4086, 4095, 1, 0, 0, 0, 1, 4091, 1772, 10, 0, 65, 4087, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 426, 0, 0, 0, 4096, 0, 4096, 4096, 0, 5, 4096, 4096, 4095, 4096, 3951, 0, 85, 0, 2, 0, 4082, 0, 4095, 4094, 0, 0, 0, 4096, 0, 0, 2, 2762, 0, 0, 0, 0, 4096, 0, 0, 4076, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3822, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 4096, 0, 4, 0, 0, 0, 0, 0, 4096, 3, 0, 4091, 4087, 0, 0, 4079, 1, 0, 7, 4092, 0, 32, 4093, 4096, 2, 0, 0, 0, 0, 4095, 1244, 4, 0, 78, 4088, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 367, 0, 0, 0, 4096, 0, 4096, 4096, 0, 13, 4096, 4096, 4096, 4092, 3910, 0, 226, 0, 5, 0, 3910, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 2819, 0, 0, 0, 0, 4096, 0, 0, 4069, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 2, 0, 4096, 0, 0, 0, 0, 3735, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 24, 0, 4096, 4096, 0, 1, 4088, 0, 0, 0, 4096, 0, 20, 4096, 4096, 0, 0, 0, 0, 0, 4096, 827, 0, 0, 48, 4091, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 495, 0, 0, 0, 4096, 0, 4096, 4096, 0, 3, 4096, 4096, 4096, 4094, 3943, 0, 190, 0, 0, 0, 4043, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 1, 3117, 0, 0, 0, 0, 4096, 0, 0, 4026, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 1, 0, 4096, 0, 0, 0, 0, 3419, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 3, 4094, 4096, 0, 0, 0, 0, 0, 4096, 1721, 3, 0, 32, 4090, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 952, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4096, 2278, 0, 4, 0, 0, 0, 3968, 0, 4096, 4091, 0, 0, 0, 4096, 0, 0, 0, 4021, 0, 0, 0, 0, 4096, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2379, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 5, 4096, 0, 9, 0, 0, 1, 0, 0, 4096, 5, 0, 4092, 4094, 0, 0, 4083, 1, 0, 6, 4096, 0, 28, 4085, 4096, 0, 0, 0, 0, 0, 4089, 1621, 7, 0, 1, 4086, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 1408, 0, 0, 0, 4096, 0, 4096, 4096, 0, 13, 4096, 4096, 4096, 4096, 4000, 0, 47, 0, 5, 0, 4089, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 2829, 0, 0, 0, 0, 4096, 0, 0, 4069, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3730, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 2, 4096, 0, 10, 0, 0, 2844, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 76, 0, 4095, 4084, 0, 0, 4092, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4090, 2405, 0, 0, 0, 0, 0, 4096, 1832, 8, 0, 6, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 409, 0, 0, 0, 4096, 0, 1691, 1585, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 442, 0, 0, 4095, 3809, 0, 29, 0, 1, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 0, 0, 1268, 0, 0, 0, 2449, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3335, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16784, 16387, 16449, 16529, 15983, 16573, 15999, 16368] avg=16384.0000 max=16784 max_rank=0 min=15983 min_rank=4 imbalance_ratio=1.024414 max_load_reduction=114288 max_load_reduction_pct=0.871948 moved_to_echo=64923 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3354, 0, 0, 0, 0, 0, 0, 18, 4096, 4096, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4, 4096, 0, 1, 0, 0, 2844, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 15, 0, 4094, 4085, 0, 0, 4076, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4093, 2405, 0, 0, 0, 0, 0, 4096, 2524, 12, 0, 54, 4080, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 345, 0, 0, 0, 4096, 0, 1691, 1585, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 442, 0, 0, 4096, 3385, 0, 134, 0, 2, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4080, 0, 0, 0, 1268, 0, 0, 0, 2594, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3335, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16421, 16380, 16373, 17288, 15904, 16174, 16134, 16398] avg=16384.0000 max=17288 max_rank=3 min=15904 min_rank=4 imbalance_ratio=1.055176 max_load_reduction=113784 max_load_reduction_pct=0.868103 moved_to_echo=64610 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3247, 0, 0, 0, 0, 0, 0, 5, 4095, 4096, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 13, 0, 0, 2844, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4085, 0, 0, 4089, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4086, 2405, 1, 0, 0, 0, 1, 4091, 1772, 10, 0, 65, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 426, 0, 0, 0, 4096, 0, 1690, 1585, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 442, 0, 0, 4096, 3951, 0, 85, 0, 2, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4094, 0, 0, 0, 1268, 0, 0, 2, 2762, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 3334, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16301, 16389, 16368, 16527, 15999, 16759, 16317, 16412] avg=16384.0000 max=16759 max_rank=5 min=15999 min_rank=4 imbalance_ratio=1.022888 max_load_reduction=114313 max_load_reduction_pct=0.872139 moved_to_echo=65487 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 4096, 0, 0, 0, 0, 3419, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2844, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 4085, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 2405, 0, 0, 0, 0, 0, 4096, 1721, 3, 0, 32, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 952, 0, 0, 0, 4096, 0, 1691, 1585, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 442, 0, 0, 4096, 2278, 0, 4, 0, 0, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 1268, 0, 0, 0, 4021, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3334, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16470, 16375, 16374, 16444, 16528, 14884, 17572, 16425] avg=16384.0000 max=17572 max_rank=6 min=14884 min_rank=5 imbalance_ratio=1.072510 max_load_reduction=113500 max_load_reduction_pct=0.865936 moved_to_echo=65409 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 4091, 5, 1, 6, 0, 2896, 0, 6, 0, 1, 5, 0, 0, 4090, 4089, 761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 4090, 5, 5, 4088, 6, 6, 5, 0, 4090, 0, 2, 5, 1, 2841, 1242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 4060, 40, 5, 4090, 4073, 6, 3, 4091, 5, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4089, 2399, 0, 0, 6, 5, 0, 4090, 1596, 0, 0, 35, 4091, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3, 0, 0, 2, 4089, 0, 0, 4, 0, 2099, 4, 6, 0, 4087, 0, 1691, 1589, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 436, 0, 0, 4091, 3725, 0, 13, 3, 0, 5, 4030, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4005, 5, 5, 7, 1259, 0, 5, 130, 2006, 5, 5, 1, 0, 4090, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 4090, 0, 5, 6, 4, 0, 6, 2, 5, 4, 0, 0, 0, 0, 4091, 3327, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15956, 16392, 16398, 16338, 17680, 16389, 15621, 16298] avg=16384.0000 max=17680 max_rank=4 min=15621 min_rank=6 imbalance_ratio=1.079102 max_load_reduction=113392 max_load_reduction_pct=0.865112 moved_to_echo=65988 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 4096, 0, 0, 0, 0, 3735, 0, 0, 0, 0, 0, 0, 22, 4096, 4096, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2844, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 24, 0, 4096, 4085, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 4096, 2405, 0, 0, 0, 0, 0, 4096, 827, 0, 0, 48, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 495, 0, 0, 0, 4096, 0, 1691, 1585, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 442, 0, 0, 4094, 3943, 0, 190, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 1268, 0, 0, 1, 3117, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3334, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16809, 16375, 16390, 15583, 16070, 16811, 16672, 16362] avg=16384.0000 max=16811 max_rank=5 min=15583 min_rank=3 imbalance_ratio=1.026062 max_load_reduction=114261 max_load_reduction_pct=0.871742 moved_to_echo=65915 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 3822, 0, 0, 0, 0, 0, 0, 16, 4096, 4096, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 6, 4096, 0, 4, 0, 0, 2844, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 4091, 4076, 0, 0, 4079, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 4093, 2405, 2, 0, 0, 0, 0, 4095, 1244, 4, 0, 78, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 367, 0, 0, 0, 4096, 0, 1691, 1585, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 442, 0, 0, 4092, 3910, 0, 226, 0, 5, 0, 3910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 1268, 0, 0, 0, 2819, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4092, 3334, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16888, 16385, 16353, 16041, 15933, 16694, 16374, 16404] avg=16384.0000 max=16888 max_rank=0 min=15933 min_rank=4 imbalance_ratio=1.030762 max_load_reduction=114184 max_load_reduction_pct=0.871155 moved_to_echo=65405 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 0, 0, 2379, 0, 0, 0, 0, 0, 0, 14, 4096, 4096, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 5, 4096, 0, 9, 0, 0, 2844, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 5, 0, 4092, 4083, 0, 0, 4083, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4085, 2405, 0, 0, 0, 0, 0, 4089, 1621, 7, 0, 1, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 1408, 0, 0, 0, 4096, 0, 1691, 1585, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 13, 442, 0, 0, 4096, 4000, 0, 47, 0, 5, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 1268, 0, 0, 0, 2829, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3334, 810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15443, 16389, 16367, 16322, 16975, 16788, 16383, 16405] avg=16384.0000 max=16975 max_rank=4 min=15443 min_rank=0 imbalance_ratio=1.036072 max_load_reduction=114097 max_load_reduction_pct=0.870491 moved_to_echo=66551 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32384, 32768, 0, 128, 128, 128, 0, 128, 128, 128, 128, 0, 128, 0, 0, 32768, 26752, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 32768, 128, 128, 128, 10240, 0, 128, 256, 22656, 128, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 32768, 128, 128, 128, 0, 26624, 0, 128, 0, 128, 128, 0, 128, 32768, 32768, 6144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 0, 128, 32768, 0, 0, 128, 0, 6528, 128, 128, 0, 32768, 0, 13568, 12800, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 32768, 0, 128, 128, 128, 22784, 9984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 19328, 128, 0, 128, 128, 128, 32768, 13184, 128, 0, 384, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 32768, 256, 128, 32768, 32768, 128, 128, 32768, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 0, 128, 3584, 0, 0, 32768, 29056, 0, 768, 128, 128, 128, 32256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1245, 0, 0, 1323, 0, 0, 3657, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 348, 0, 0, 0, 0, 4094, 2386, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4095, 0, 0, 4096, 0, 4094, 4096, 0, 0, 0, 3521, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4095, 0, 3909, 2, 0, 4091, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[2135, 0, 0, 884, 0, 0, 3512, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 4096, 2674, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 3677, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3485, 0, 0, 4093, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1224, 0, 0, 2676, 0, 0, 3961, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 402, 0, 0, 0, 0, 4094, 993, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 3819, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 0, 3354, 0, 0, 4051, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[1278, 0, 0, 1875, 0, 0, 3732, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 589, 0, 0, 0, 0, 4093, 1610, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 3859, 1, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 4092, 0, 3489, 4, 0, 4046, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[621, 0, 0, 2185, 0, 0, 3529, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 463, 0, 0, 0, 0, 4092, 2098, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 4096, 0, 0, 4096, 0, 4085, 4096, 0, 0, 0, 3820, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 13, 0, 4096, 0, 0, 4091, 0, 3696, 6, 0, 4067, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[2830, 4, 0, 931, 5, 5, 4051, 0, 5, 1, 0, 5, 5, 0, 4090, 2, 207, 1, 0, 0, 0, 4091, 1724, 0, 3, 0, 4091, 6, 0, 0, 0, 0, 0, 4091, 2, 17, 4090, 6, 6, 4083, 6, 4091, 4091, 0, 8, 5, 2594, 0, 6, 4, 0, 0, 6, 0, 4089, 0, 5, 2, 0, 0, 5, 4074, 4090, 7, 0, 0, 5, 0, 5, 6, 0, 0, 4091, 0, 4089, 4075, 6, 6, 0, 2, 4090, 5, 4090, 0, 5, 4085, 6, 6, 0, 6, 0, 0, 0, 4089, 0, 4089, 5, 6, 6, 88, 0, 17, 4061, 0, 0, 0, 5, 0, 0, 4090, 5, 5, 4091, 0, 3971, 0, 6, 4090, 4090, 0, 0, 4064, 0, 24, 4087, 4079, 4091, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[303, 0, 0, 2774, 0, 0, 3643, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 562, 0, 0, 0, 0, 4089, 2367, 0, 0, 0, 4092, 0, 0, 0, 0, 4, 0, 4094, 0, 0, 4095, 0, 0, 4096, 0, 4086, 4096, 0, 0, 0, 3140, 1, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 16, 0, 4096, 0, 0, 4089, 0, 3636, 8, 0, 4050, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4092, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[2321, 0, 0, 427, 0, 0, 3585, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 182, 0, 0, 0, 0, 4091, 2555, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 4089, 4096, 0, 0, 0, 3581, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 4090, 0, 3756, 5, 0, 4079, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1245, 0, 0, 1323, 0, 0, 3657, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3039, 2408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 0, 0, 0, 0, 4094, 2386, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 4096, 1688, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 596, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 3521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3042, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3499, 593, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3909, 2, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15768, 16723, 16403, 16386, 16383, 16385, 16370, 16654] avg=16384.0000 max=16723 max_rank=1 min=15768 min_rank=0 imbalance_ratio=1.020691 max_load_reduction=114349 max_load_reduction_pct=0.872414 moved_to_echo=65792 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1224, 0, 0, 2676, 0, 0, 3961, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 3040, 2408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 402, 0, 0, 0, 0, 4094, 993, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1688, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 596, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 3819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3042, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 3500, 593, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3354, 0, 0, 4051, 0, 0, 0, 4096, 0, 0, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17406, 15385, 16703, 16386, 16381, 16383, 16370, 16058] avg=16384.0000 max=17406 max_rank=0 min=15385 min_rank=1 imbalance_ratio=1.062378 max_load_reduction=113666 max_load_reduction_pct=0.867203 moved_to_echo=65192 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[621, 0, 0, 2185, 0, 0, 3529, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3039, 2408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 463, 0, 0, 0, 0, 4092, 2098, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 4096, 1688, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 596, 0, 0, 4096, 0, 4085, 0, 0, 0, 0, 3820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 3042, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3500, 593, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 13, 0, 4096, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3696, 6, 0, 4067, 0, 0, 0, 4096, 0, 0, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15878, 16549, 16692, 16383, 16384, 16387, 16382, 16417] avg=16384.0000 max=16692 max_rank=2 min=15878 min_rank=0 imbalance_ratio=1.018799 max_load_reduction=114380 max_load_reduction_pct=0.872650 moved_to_echo=65570 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2135, 0, 0, 884, 0, 0, 3512, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 3041, 2409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 4096, 2674, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1687, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 596, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 3677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3042, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3500, 593, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3485, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16078, 16684, 16561, 16385, 16382, 16382, 16369, 16231] avg=16384.0000 max=16684 max_rank=1 min=16078 min_rank=0 imbalance_ratio=1.018311 max_load_reduction=114388 max_load_reduction_pct=0.872711 moved_to_echo=65364 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1278, 0, 0, 1875, 0, 0, 3732, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3040, 2408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 0, 0, 0, 4093, 1610, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4096, 1688, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 596, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 3859, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3042, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3500, 593, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 3489, 4, 0, 4046, 0, 0, 0, 4096, 0, 0, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16429, 16188, 16742, 16385, 16384, 16386, 16370, 16188] avg=16384.0000 max=16742 max_rank=2 min=16188 min_rank=1 imbalance_ratio=1.021851 max_load_reduction=114330 max_load_reduction_pct=0.872269 moved_to_echo=65328 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2830, 4, 0, 931, 5, 5, 4051, 0, 5, 1, 0, 5, 5, 0, 4090, 2, 3037, 2403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 1, 0, 0, 0, 4091, 1724, 0, 3, 0, 4091, 6, 0, 0, 0, 0, 4090, 1684, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 17, 592, 6, 6, 4083, 6, 4091, 0, 0, 8, 5, 2594, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 6, 0, 4089, 0, 5, 2, 0, 0, 5, 4074, 4090, 7, 3033, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 5, 6, 0, 0, 4091, 0, 4089, 4075, 6, 6, 0, 2, 3498, 595, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4086, 0, 5, 4085, 6, 6, 0, 6, 0, 0, 0, 4089, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 6, 88, 0, 17, 4061, 0, 0, 0, 5, 0, 0, 4090, 5, 5, 4091, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 3971, 0, 6, 4090, 0, 0, 0, 4064, 0, 24, 0, 451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17374, 15914, 15501, 16375, 16382, 16377, 16452, 16697] avg=16384.0000 max=17374 max_rank=0 min=15501 min_rank=2 imbalance_ratio=1.060425 max_load_reduction=113698 max_load_reduction_pct=0.867447 moved_to_echo=65908 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[303, 0, 0, 2774, 0, 0, 3643, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3038, 2409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 0, 0, 0, 0, 4089, 2367, 0, 0, 0, 4092, 0, 0, 0, 0, 4, 4096, 1687, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 596, 0, 0, 4096, 0, 4086, 0, 0, 0, 0, 3140, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 3042, 1054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3499, 593, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 16, 0, 4096, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 3636, 8, 0, 4050, 0, 0, 0, 4096, 0, 0, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16263, 16913, 16013, 16387, 16387, 16384, 16385, 16340] avg=16384.0000 max=16913 max_rank=1 min=16013 min_rank=2 imbalance_ratio=1.032288 max_load_reduction=114159 max_load_reduction_pct=0.870964 moved_to_echo=65496 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2321, 0, 0, 427, 0, 0, 3585, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3039, 2408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 0, 0, 0, 0, 4091, 2555, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 1688, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 596, 0, 0, 4095, 0, 4089, 0, 0, 0, 0, 3581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3042, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 3500, 593, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 4096, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3756, 5, 0, 4079, 0, 0, 0, 4096, 0, 0, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15876, 16716, 16457, 16385, 16389, 16388, 16374, 16487] avg=16384.0000 max=16716 max_rank=1 min=15876 min_rank=0 imbalance_ratio=1.020264 max_load_reduction=114356 max_load_reduction_pct=0.872467 moved_to_echo=65638 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[12032, 128, 0, 13184, 128, 128, 29696, 0, 128, 128, 128, 128, 128, 0, 32768, 128, 24320, 19328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 0, 128, 32768, 0, 0, 128, 128, 128, 0, 32768, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 128, 128, 32768, 128, 128, 0, 128, 0, 0, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 128, 128, 4864, 128, 128, 32768, 128, 32768, 0, 0, 128, 128, 28032, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 0, 29312, 128, 128, 32640, 0, 0, 0, 32768, 0, 128, 0, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 0, 128, 128, 32768, 0, 128, 128, 0, 0, 128, 32768, 32768, 128, 24448, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[2816, 128, 0, 0, 0, 32768, 16512, 0, 128, 0, 32768, 128, 0, 0, 0, 128, 32768, 13568, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 0, 128, 128, 0, 0, 32768, 0, 32768, 32768, 128, 128, 0, 128, 28032, 4864, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3435, 4096, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4083, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 2829, 0, 0, 0, 0, 0, 2802, 0, 0, 3488, 0, 87, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4096, 75, 0, 4096, 4060, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3863, 0, 0, 1, 4096, 0, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 8, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 3887, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3569, 4096, 0, 0, 0, 0, 0, 4096, 0, 17, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 2379, 0, 0, 0, 0, 0, 2341, 0, 0, 3855, 0, 45, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 105, 4096, 47, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4036, 0, 0, 1, 4096, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 3647, 4089, 6, 5, 5, 1, 0, 4051, 0, 2, 1, 6, 4090, 4087, 6, 7, 5, 0, 0, 4091, 5, 6, 0, 5, 4090, 0, 0, 7, 8, 1005, 8, 6, 0, 7, 6, 3768, 6, 1, 2736, 0, 143, 0, 3875, 7, 0, 0, 369, 0, 0, 29, 0, 0, 4090, 481, 4, 4090, 4074, 0, 0, 0, 0, 0, 5, 4089, 0, 0, 0, 4, 0, 4037, 5, 1, 157, 4059, 0, 4090, 3968, 0, 6, 0, 574, 0, 6, 6, 5, 4090, 1, 0, 4085, 6, 4, 6, 4090, 18, 4090, 0, 4024, 0, 41, 0, 6, 0, 0, 3919, 4089, 4, 4083, 0, 0, 0, 4090, 15, 0, 6, 4090, 4090, 4090, 0, 4090, 10, 1, 3, 0, 4088] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3947, 4096, 0, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 834, 0, 0, 1, 0, 0, 3393, 0, 0, 3936, 0, 39, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 48, 4095, 119, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4060, 0, 0, 3, 4096, 0, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4089, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3064, 4096, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 1464, 0, 0, 2, 0, 0, 3820, 0, 0, 3623, 0, 241, 2, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 40, 4094, 142, 0, 4094, 4080, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4034, 0, 0, 8, 4096, 1, 4096, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4060, 0, 4096, 0, 0, 0, 0, 7, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4090, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3933, 4096, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4083, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 1100, 0, 0, 4, 0, 0, 3277, 0, 0, 3686, 0, 253, 3, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 85, 4093, 63, 0, 4093, 4084, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 4020, 0, 0, 13, 4096, 4, 4096, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4078, 0, 4096, 0, 0, 0, 0, 10, 0, 4096, 4093, 0, 4096, 0, 0, 3, 4096, 0, 0, 0, 4096, 4063, 4096, 3, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4006, 4096, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 1245, 0, 0, 1, 0, 0, 3044, 0, 0, 3940, 0, 187, 2, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 53, 4095, 100, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 3828, 0, 0, 3, 4096, 3, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4087, 0, 4095, 0, 0, 0, 0, 6, 0, 4096, 4096, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 4078, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3885, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 945, 0, 0, 1, 0, 0, 3127, 0, 0, 3812, 0, 185, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4096, 435, 0, 4096, 4061, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4011, 0, 0, 3, 4096, 5, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 4075, 0, 4094, 0, 0, 0, 0, 6, 0, 4096, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 4090, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3947, 4096, 0, 0, 0, 0, 0, 4096, 0, 14, 0, 0, 4096, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 834, 0, 0, 1, 0, 0, 3393, 0, 0, 3936, 0, 39, 1, 4090, 0, 3722, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 4095, 119, 0, 4096, 4094, 0, 0, 0, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4060, 0, 0, 3, 4096, 0, 4096, 85, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3696, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 34, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16649, 16383, 16160, 16285, 16439, 16327, 16413, 16416] avg=16384.0000 max=16649 max_rank=0 min=16160 min_rank=2 imbalance_ratio=1.016174 max_load_reduction=114423 max_load_reduction_pct=0.872978 moved_to_echo=65595 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3435, 4096, 0, 0, 0, 0, 0, 4096, 0, 18, 0, 0, 4096, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 4083, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2829, 0, 0, 0, 0, 0, 2802, 0, 0, 3488, 0, 87, 0, 4090, 0, 3722, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4096, 75, 0, 4096, 4060, 0, 0, 0, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3863, 0, 0, 1, 4096, 0, 4096, 85, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3695, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 8, 0, 4096, 34, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 3887, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16142, 16371, 17162, 16198, 16240, 16325, 16426, 16208] avg=16384.0000 max=17162 max_rank=2 min=16142 min_rank=0 imbalance_ratio=1.047485 max_load_reduction=113910 max_load_reduction_pct=0.869064 moved_to_echo=65199 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3569, 4096, 0, 0, 0, 0, 0, 4096, 0, 17, 0, 0, 4096, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2379, 0, 0, 0, 0, 0, 2341, 0, 0, 3855, 0, 45, 0, 4084, 0, 3722, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 4096, 47, 0, 4096, 4096, 0, 0, 0, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4036, 0, 0, 1, 4096, 0, 4096, 85, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3695, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 4095, 34, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16275, 16383, 16570, 16273, 16413, 16326, 16415, 16417] avg=16384.0000 max=16570 max_rank=2 min=16273 min_rank=3 imbalance_ratio=1.011353 max_load_reduction=114502 max_load_reduction_pct=0.873581 moved_to_echo=65571 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 6, 0, 3647, 4089, 6, 5, 5, 1, 0, 4051, 0, 2, 1, 6, 4090, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4087, 6, 7, 5, 0, 0, 4091, 5, 6, 0, 5, 4090, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1005, 8, 6, 0, 7, 6, 3768, 6, 1, 2736, 0, 143, 0, 3875, 7, 3719, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 369, 0, 0, 29, 0, 0, 4090, 481, 4, 4090, 4074, 0, 0, 0, 3831, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4089, 0, 0, 0, 4, 0, 4037, 5, 1, 157, 4059, 0, 4090, 89, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 6, 0, 574, 0, 6, 6, 5, 4090, 1, 0, 4085, 6, 4, 6, 3689, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4090, 0, 4024, 0, 41, 0, 6, 0, 0, 3919, 19, 4, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 15, 0, 6, 0, 4090, 4090, 0, 4090, 10, 1, 3, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16310, 16396, 15445, 16968, 16539, 16791, 16204, 16419] avg=16384.0000 max=16968 max_rank=3 min=15445 min_rank=2 imbalance_ratio=1.035645 max_load_reduction=114104 max_load_reduction_pct=0.870544 moved_to_echo=65953 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3064, 4096, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 4096, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1464, 0, 0, 2, 0, 0, 3820, 0, 0, 3623, 0, 241, 2, 4068, 0, 3722, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 4094, 142, 0, 4094, 4080, 0, 0, 0, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4034, 0, 0, 8, 4096, 1, 4096, 85, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3696, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 4096, 0, 0, 0, 0, 7, 0, 4096, 34, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 4090, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15773, 16384, 17086, 16283, 16419, 16327, 16389, 16411] avg=16384.0000 max=17086 max_rank=2 min=15773 min_rank=0 imbalance_ratio=1.042847 max_load_reduction=113986 max_load_reduction_pct=0.869644 moved_to_echo=65546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4006, 4096, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 4096, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1245, 0, 0, 1, 0, 0, 3044, 0, 0, 3940, 0, 187, 2, 4085, 0, 3722, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 4095, 100, 0, 4095, 4090, 0, 0, 0, 3833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 3828, 0, 0, 3, 4096, 3, 4096, 85, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3695, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4095, 0, 0, 0, 0, 6, 0, 4096, 34, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 33, 0, 0, 0, 0, 4078, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16703, 16381, 16370, 16266, 16211, 16327, 16414, 16400] avg=16384.0000 max=16703 max_rank=0 min=16211 min_rank=4 imbalance_ratio=1.019470 max_load_reduction=114369 max_load_reduction_pct=0.872566 moved_to_echo=65352 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3933, 4096, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 4096, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 4083, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1100, 0, 0, 4, 0, 0, 3277, 0, 0, 3686, 0, 253, 3, 4066, 0, 3722, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 4093, 63, 0, 4093, 4084, 0, 0, 0, 3830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 4020, 0, 0, 13, 4096, 4, 4096, 85, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3695, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4096, 0, 0, 0, 0, 10, 0, 4096, 34, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 33, 0, 0, 0, 0, 4063, 4096, 3, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16644, 16384, 16255, 16248, 16416, 16325, 16410, 16390] avg=16384.0000 max=16644 max_rank=0 min=16248 min_rank=3 imbalance_ratio=1.015869 max_load_reduction=114428 max_load_reduction_pct=0.873016 moved_to_echo=65541 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3885, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 945, 0, 0, 1, 0, 0, 3127, 0, 0, 3812, 0, 185, 0, 4088, 0, 3722, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4096, 435, 0, 4096, 4061, 0, 0, 0, 3832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4011, 0, 0, 3, 4096, 5, 4096, 85, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 3695, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4094, 0, 0, 0, 0, 6, 0, 4096, 34, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 4090, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16576, 16390, 16024, 16551, 16395, 16324, 16401, 16411] avg=16384.0000 max=16576 max_rank=0 min=16024 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=114496 max_load_reduction_pct=0.873535 moved_to_echo=65531 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 0, 32768, 0, 128, 0, 128, 128, 0, 32640, 384, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 128, 128, 0, 0, 32768, 128, 128, 128, 128, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 0, 29568, 32768, 128, 128, 128, 128, 0, 32768, 0, 128, 128, 128, 32768, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32640, 128, 128, 0, 640, 0, 128, 128, 128, 32768, 128, 0, 32768, 128, 128, 128, 29568, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 32768, 0, 0, 128, 128, 0, 32000, 128, 128, 256, 32768, 128, 32768, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 384, 128, 0, 128, 0, 32512, 32768, 128, 32768, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 384, 0, 0, 128, 0, 512, 32768, 1536, 128, 32768, 32640, 0, 0, 0, 30720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 11904, 128, 128, 128, 128, 128, 25600, 128, 128, 29184, 0, 1280, 128, 32512, 128, 29824, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4079, 3, 0, 0, 0, 45, 0, 0, 0, 4096, 0, 4, 0, 3648, 0, 4096, 4096, 0, 0, 0, 4089, 0, 3926, 0, 0, 0, 4096, 0, 0, 4086, 4096, 4096, 0, 0, 0, 0, 0, 0, 3404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1510, 4096, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 17, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 5, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4094, 1, 4096, 4096, 4096, 0, 3844, 0, 0, 4096, 2, 3, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4084, 10, 0, 0, 0, 45, 0, 0, 0, 4096, 0, 2, 0, 3776, 0, 4096, 4094, 0, 0, 0, 4081, 0, 4069, 11, 0, 0, 4096, 0, 0, 4076, 4096, 4096, 0, 0, 0, 0, 0, 0, 3159, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1697, 4093, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 14, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4085, 0, 4096, 4094, 4096, 0, 3613, 0, 0, 4096, 2, 20, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4094, 0, 0, 4096, 0, 0, 4082, 1, 0, 0, 0, 19, 0, 0, 0, 4096, 0, 2, 0, 3663, 0, 4096, 4095, 0, 0, 0, 4089, 0, 4037, 4, 0, 0, 4096, 0, 0, 4089, 4096, 4096, 0, 0, 0, 0, 0, 0, 3845, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 896, 4095, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4092, 0, 4096, 4096, 4096, 0, 3929, 0, 0, 4096, 6, 4, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4093, 0, 0, 4077, 37, 0, 0, 0, 108, 0, 0, 0, 4096, 1, 10, 0, 3451, 0, 4096, 4093, 0, 0, 0, 4064, 0, 3892, 13, 0, 0, 4096, 0, 0, 4078, 4096, 4096, 0, 0, 0, 0, 0, 0, 3456, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1471, 4089, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 4096, 0, 3, 4096, 0, 0, 0, 0, 4096, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 4087, 0, 4096, 4096, 4096, 0, 3966, 0, 0, 4092, 7, 18, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4095, 0, 0, 4083, 13, 0, 0, 0, 39, 0, 0, 0, 4096, 0, 1, 0, 3959, 0, 4096, 4095, 0, 0, 0, 4071, 0, 4051, 8, 0, 0, 4096, 0, 0, 4089, 4095, 4096, 0, 0, 0, 0, 0, 0, 3197, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1316, 4094, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 4096, 0, 1, 4096, 0, 0, 2, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4091, 7, 4096, 4095, 4096, 0, 3808, 0, 0, 4094, 5, 10, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 3857, 0, 4096, 4096, 0, 0, 0, 4079, 0, 3957, 34, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 3054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1829, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 4096, 0, 3667, 0, 0, 4096, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4088, 3, 0, 4089, 4, 5, 4090, 0, 0, 0, 5, 0, 0, 0, 6, 4087, 0, 0, 12, 4048, 5, 4088, 4038, 0, 0, 6, 4089, 0, 4091, 0, 0, 0, 4083, 3, 5, 4090, 4090, 4084, 0, 6, 2, 6, 0, 5, 3497, 0, 4, 6, 6, 0, 6, 0, 0, 5, 5, 11, 4090, 6, 0, 3959, 0, 5, 6, 4090, 6, 0, 0, 2, 0, 572, 4091, 0, 5, 0, 0, 8, 0, 0, 0, 6, 6, 0, 6, 4089, 4090, 2, 0, 0, 0, 0, 0, 0, 5, 4063, 4086, 7, 0, 4075, 6, 6, 0, 2, 3799, 0, 5, 5, 4086, 5, 0, 22, 4087, 0, 4090, 902, 4090, 4090, 4086, 0, 3653, 3, 7, 4090, 0, 0, 6, 78, 5, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4079, 0, 0, 0, 0, 45, 0, 0, 0, 4096, 0, 7, 0, 3956, 0, 4096, 4095, 0, 0, 0, 4076, 0, 4087, 20, 0, 0, 4096, 0, 0, 4085, 4096, 4096, 0, 0, 0, 0, 0, 0, 3788, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 437, 4091, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4088, 1, 4096, 4096, 4096, 0, 4047, 0, 0, 4096, 7, 16, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 4096, 0, 0, 4082, 1, 0, 0, 0, 19, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3663, 0, 392, 4095, 0, 0, 0, 4089, 0, 4037, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 680, 4096, 0, 0, 0, 0, 0, 0, 3845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 3, 4096, 0, 0, 0, 4096, 3416, 679, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 896, 4095, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 3946, 3025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 25, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 150, 4096, 0, 3929, 0, 0, 4096, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16357, 16282, 16806, 16393, 16061, 16384, 16416, 16373] avg=16384.0000 max=16806 max_rank=2 min=16061 min_rank=4 imbalance_ratio=1.025757 max_load_reduction=114266 max_load_reduction_pct=0.871780 moved_to_echo=65234 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4084, 10, 0, 0, 0, 45, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3776, 0, 392, 4094, 0, 0, 0, 4081, 0, 4069, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4076, 680, 4096, 0, 0, 0, 0, 0, 0, 3159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 4096, 3416, 679, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1697, 4093, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 3945, 3025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4094, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 14, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 25, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 149, 4096, 0, 3613, 0, 0, 4096, 2, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16396, 16425, 16107, 16399, 16866, 16389, 16429, 16061] avg=16384.0000 max=16866 max_rank=4 min=16061 min_rank=7 imbalance_ratio=1.029419 max_load_reduction=114206 max_load_reduction_pct=0.871323 moved_to_echo=65745 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4079, 3, 0, 0, 0, 45, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3648, 0, 392, 4096, 0, 0, 0, 4089, 0, 3926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4086, 680, 4096, 0, 0, 0, 0, 0, 0, 3404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3416, 679, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1510, 4096, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 3946, 3025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 17, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 25, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 150, 4096, 0, 3844, 0, 0, 4096, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16384, 16155, 16362, 16389, 16681, 16396, 16419, 16286] avg=16384.0000 max=16681 max_rank=4 min=16155 min_rank=1 imbalance_ratio=1.018127 max_load_reduction=114391 max_load_reduction_pct=0.872734 moved_to_echo=65782 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3857, 0, 392, 4096, 0, 0, 0, 4079, 0, 3957, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 680, 4096, 0, 0, 0, 0, 0, 0, 3054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3416, 679, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1829, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3946, 3025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 25, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 150, 4096, 0, 3667, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16353, 16415, 16022, 16389, 16992, 16380, 16416, 16105] avg=16384.0000 max=16992 max_rank=4 min=16022 min_rank=2 imbalance_ratio=1.037109 max_load_reduction=114080 max_load_reduction_pct=0.870361 moved_to_echo=65893 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4095, 0, 0, 4083, 13, 0, 0, 0, 39, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3959, 0, 392, 4095, 0, 0, 0, 4071, 0, 4051, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 680, 4096, 0, 0, 0, 0, 0, 0, 3197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 0, 4096, 3415, 679, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1316, 4094, 1, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 3945, 3025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4095, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 2, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 25, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 7, 0, 150, 4096, 0, 3808, 0, 0, 4094, 5, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16391, 16577, 16158, 16395, 16482, 16390, 16418, 16261] avg=16384.0000 max=16577 max_rank=1 min=16158 min_rank=2 imbalance_ratio=1.011780 max_load_reduction=114495 max_load_reduction_pct=0.873528 moved_to_echo=65551 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4088, 3, 0, 4089, 4, 5, 4090, 0, 0, 0, 5, 0, 0, 0, 6, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4048, 5, 380, 4038, 0, 0, 6, 4089, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 3, 5, 4090, 675, 4084, 0, 6, 2, 6, 0, 5, 3497, 0, 4, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 5, 5, 11, 4090, 6, 0, 3959, 0, 5, 6, 4090, 3415, 683, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 2, 0, 572, 4091, 0, 5, 0, 0, 8, 0, 0, 0, 6, 4090, 3947, 3025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 4079, 4090, 2, 0, 0, 0, 0, 0, 0, 5, 4063, 4086, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 6, 6, 0, 2, 3799, 0, 5, 5, 4086, 5, 0, 22, 4087, 0, 31, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 902, 0, 143, 4086, 0, 3653, 3, 7, 4090, 0, 0, 6, 78, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16334, 16669, 16466, 16299, 15752, 16344, 16139, 17069] avg=16384.0000 max=17069 max_rank=7 min=15752 min_rank=4 imbalance_ratio=1.041809 max_load_reduction=114003 max_load_reduction_pct=0.869774 moved_to_echo=65304 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4093, 0, 0, 4077, 37, 0, 0, 0, 108, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 0, 3451, 0, 392, 4093, 0, 0, 0, 4064, 0, 3892, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4078, 680, 4096, 0, 0, 0, 0, 0, 0, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 4096, 3416, 679, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1471, 4089, 0, 0, 0, 0, 0, 11, 0, 0, 0, 4096, 3946, 3025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4092, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4096, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 25, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 150, 4096, 0, 3966, 0, 0, 4092, 7, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16476, 15916, 16406, 16406, 16638, 16394, 16420, 16416] avg=16384.0000 max=16638 max_rank=4 min=15916 min_rank=1 imbalance_ratio=1.015503 max_load_reduction=114434 max_load_reduction_pct=0.873062 moved_to_echo=65868 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 4096, 0, 0, 4079, 0, 0, 0, 0, 45, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3956, 0, 392, 4095, 0, 0, 0, 4076, 0, 4087, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 680, 4096, 0, 0, 0, 0, 0, 0, 3788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 5, 4096, 0, 0, 0, 4096, 3416, 679, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 437, 4091, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 3946, 3025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 25, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 0, 150, 4096, 0, 4047, 0, 0, 4096, 7, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16381, 16633, 16745, 16402, 15600, 16395, 16415, 16501] avg=16384.0000 max=16745 max_rank=2 min=15600 min_rank=4 imbalance_ratio=1.022034 max_load_reduction=114327 max_load_reduction_pct=0.872246 moved_to_echo=64911 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 32768, 128, 128, 32768, 128, 0, 0, 128, 384, 0, 0, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 32768, 5504, 32768, 0, 128, 128, 128, 0, 128, 27520, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 128, 32768, 32768, 128, 0, 0, 0, 128, 0, 0, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 128, 128, 128, 32512, 128, 128, 128, 32768, 128, 0, 128, 32768, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 128, 30464, 128, 3200, 32768, 0, 0, 128, 32640, 0, 32128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 1024, 0, 1280, 32768, 0, 30592, 128, 128, 32768, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 128, 0, 9728, 32768, 128, 128, 0, 0, 128, 128, 0, 0, 128, 32768, 31616, 24320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 128, 32768, 128, 128, 32640, 0, 128, 128, 32768, 27392, 5504, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 142, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4093, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 3983, 4, 4088, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 4094, 4096, 0, 30, 588, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4093, 0, 0, 0, 2, 4096, 2968, 4096, 6, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 488, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 4082, 18, 4022, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 4095, 4096, 0, 2, 556, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4086, 0, 0, 0, 1, 4095, 3312, 4096, 10, 4084, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 180, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1048, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 5, 4081, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 4096, 4096, 0, 41, 349, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4030, 0, 0, 0, 2, 4096, 2530, 4096, 9, 4088, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 204, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4022, 6, 4083, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 97, 209, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4090, 0, 0, 0, 2, 4096, 3756, 4096, 3, 4063, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 105, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[6, 0, 0, 8, 1594, 12, 2, 9, 1, 1, 0, 39, 3894, 6, 6, 0, 4066, 0, 0, 4090, 4090, 218, 0, 0, 6, 4048, 5, 3894, 3959, 1, 2, 4090, 6, 105, 5, 7, 0, 0, 0, 0, 4090, 0, 8, 6, 5, 1311, 0, 4091, 4090, 6, 0, 6, 4089, 0, 6, 0, 0, 4090, 6, 6, 4082, 0, 0, 3722, 0, 3, 0, 6, 5, 0, 7, 3901, 4072, 0, 1533, 45, 0, 13, 3, 6, 5, 0, 4086, 6, 4090, 0, 3771, 7, 22, 5, 24, 4088, 2535, 4090, 0, 4090, 0, 2, 9, 0, 122, 4090, 0, 5, 0, 0, 4079, 6, 1795, 0, 6, 6, 4088, 6, 4088, 8, 0, 0, 0, 0, 84, 4089, 234, 4087, 6, 0, 0, 4065] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 267, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 4060, 17, 4073, 4096, 0, 8, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 1, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 4092, 4096, 0, 6, 347, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 4056, 0, 0, 0, 1, 4095, 2735, 4096, 25, 4095, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 4096, 0, 792, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 4093, 0, 0, 0, 0, 4047, 18, 4045, 4096, 0, 4, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 5, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 4092, 4096, 0, 3, 458, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4067, 0, 0, 0, 4, 4092, 3225, 4096, 41, 4079, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 4096, 0, 296, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 4081, 12, 4024, 4096, 0, 7, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 5, 0, 0, 0, 0, 0, 4091, 4096, 0, 21, 249, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4072, 0, 0, 0, 3, 4096, 1597, 4096, 5, 4086, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2119, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 142, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4093, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 3983, 4, 4088, 2778, 1673, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4094, 4096, 0, 30, 588, 0, 0, 0, 0, 4096, 3116, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1318, 0, 0, 0, 4093, 0, 0, 0, 2, 4096, 2968, 0, 6, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 488, 0, 0, 0, 4072, 2378, 979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16091, 16405, 16657, 16429, 16458, 16573, 16109, 16350] avg=16384.0000 max=16657 max_rank=2 min=16091 min_rank=0 imbalance_ratio=1.016663 max_load_reduction=114415 max_load_reduction_pct=0.872917 moved_to_echo=65490 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1048, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 5, 4081, 2778, 1673, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 4096, 0, 41, 349, 0, 0, 0, 0, 4096, 3117, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1318, 0, 0, 0, 4030, 0, 0, 0, 2, 4096, 2530, 0, 9, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 204, 0, 0, 0, 4072, 2378, 979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16996, 16408, 16757, 16429, 16235, 16073, 15825, 16349] avg=16384.0000 max=16996 max_rank=0 min=15825 min_rank=6 imbalance_ratio=1.037354 max_load_reduction=114076 max_load_reduction_pct=0.870331 moved_to_echo=64482 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 4082, 18, 4022, 2778, 1673, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 4096, 0, 2, 556, 0, 0, 0, 0, 4096, 3117, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1318, 0, 0, 0, 4086, 0, 0, 0, 1, 4095, 3312, 0, 10, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 180, 0, 0, 0, 4072, 2378, 979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16072, 16408, 16707, 16429, 16399, 16906, 15801, 16350] avg=16384.0000 max=16906 max_rank=5 min=15801 min_rank=6 imbalance_ratio=1.031860 max_load_reduction=114166 max_load_reduction_pct=0.871017 moved_to_echo=65456 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4022, 6, 4083, 2778, 1673, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 97, 209, 0, 0, 0, 0, 4096, 3117, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1318, 0, 0, 0, 4090, 0, 0, 0, 2, 4096, 3756, 0, 3, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 105, 0, 0, 0, 4071, 2378, 979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16002, 16407, 16693, 16429, 16144, 17328, 15720, 16349] avg=16384.0000 max=17328 max_rank=5 min=15720 min_rank=6 imbalance_ratio=1.057617 max_load_reduction=113744 max_load_reduction_pct=0.867798 moved_to_echo=65541 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 8, 1594, 12, 2, 9, 1, 1, 0, 39, 3894, 6, 6, 0, 4090, 4090, 3655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 218, 0, 0, 6, 4048, 5, 3894, 3959, 1, 2, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 105, 5, 7, 0, 0, 0, 0, 4090, 0, 8, 6, 5, 1311, 0, 4091, 2775, 1678, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 6, 0, 6, 4089, 0, 6, 0, 0, 4090, 6, 6, 4082, 0, 0, 3722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 5, 0, 7, 3901, 4072, 0, 1533, 45, 0, 13, 3, 6, 4089, 3113, 435, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1311, 6, 0, 0, 3771, 7, 22, 5, 24, 4088, 2535, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 0, 122, 4090, 0, 5, 0, 0, 4079, 6, 1795, 0, 6, 6, 4069, 2379, 977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 6, 4088, 8, 0, 0, 0, 0, 84, 0, 234, 4087, 6, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17413, 16220, 14127, 16046, 17231, 15864, 17545, 16626] avg=16384.0000 max=17545 max_rank=6 min=14127 min_rank=2 imbalance_ratio=1.070862 max_load_reduction=113527 max_load_reduction_pct=0.866142 moved_to_echo=67266 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 267, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4095, 0, 0, 0, 0, 4060, 17, 4073, 2778, 1673, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 8, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4092, 4096, 0, 6, 347, 0, 0, 0, 0, 4096, 3115, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1318, 0, 0, 0, 4056, 0, 0, 0, 1, 4095, 2735, 0, 25, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 4096, 0, 792, 0, 0, 0, 4072, 2378, 979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16215, 16408, 16733, 16437, 16192, 16325, 16413, 16349] avg=16384.0000 max=16733 max_rank=2 min=16192 min_rank=4 imbalance_ratio=1.021301 max_load_reduction=114339 max_load_reduction_pct=0.872337 moved_to_echo=65279 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4093, 0, 0, 0, 0, 4047, 18, 4045, 2778, 1673, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 4, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4092, 4096, 0, 3, 458, 0, 0, 0, 0, 4096, 3117, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1318, 0, 0, 0, 4067, 0, 0, 0, 4, 4092, 3225, 0, 41, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 4096, 0, 296, 0, 0, 0, 4072, 2378, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16137, 16408, 16693, 16437, 16302, 16826, 15919, 16350] avg=16384.0000 max=16826 max_rank=5 min=15919 min_rank=6 imbalance_ratio=1.026978 max_load_reduction=114246 max_load_reduction_pct=0.871628 moved_to_echo=65397 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 4081, 12, 4024, 2778, 1673, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 7, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4091, 4096, 0, 21, 249, 0, 0, 0, 0, 4096, 3117, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1318, 0, 0, 0, 4072, 0, 0, 0, 3, 4096, 1597, 0, 5, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2119, 0, 0, 0, 4072, 2378, 979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16146, 16408, 16705, 16436, 16111, 15177, 17740, 16349] avg=16384.0000 max=17740 max_rank=6 min=15177 min_rank=5 imbalance_ratio=1.082764 max_load_reduction=113332 max_load_reduction_pct=0.864655 moved_to_echo=65377 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 0, 128, 0, 0, 32768, 128, 6016, 0, 128, 128, 32640, 19072, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[384, 128, 128, 128, 32768, 0, 128, 0, 0, 32768, 128, 128, 32768, 128, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32512, 128, 32768, 128, 0, 0, 0, 0, 128, 0, 256, 32768, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 0, 0, 0, 256, 0, 0, 128, 32768, 128, 32640, 32640, 128, 128, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 0, 128, 3712, 128, 128, 128, 128, 128, 0, 128, 32640, 128, 128, 0, 32768, 32768, 29312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 10624, 128, 0, 0, 32384, 128, 128, 128, 128, 32768, 22784, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 32640, 32768, 0, 1792, 2816, 0, 128, 128, 128, 32768, 24960, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 0, 32768, 0, 128, 128, 128, 29696, 128, 32512, 22272, 13440, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4091, 0, 8, 0, 0, 0, 0, 4096, 3339, 0, 0, 3732, 0, 4095, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3741, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4089, 0, 0, 0, 4096, 4084, 8, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 1574, 0, 0, 6, 0, 4094, 4096, 0, 0, 89, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3912, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 282, 0, 0, 0, 0, 4096, 1068, 0, 0, 3013, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 3, 3691, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4083, 16, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3498, 0, 0, 1, 0, 4096, 4096, 0, 0, 1146, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 3679, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 29, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4089, 0, 194, 0, 0, 0, 0, 4096, 1898, 0, 0, 3429, 0, 4096, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 2, 3548, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 4093, 0, 0, 0, 4096, 4096, 14, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3068, 0, 0, 11, 0, 4089, 4096, 0, 0, 351, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 3854, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 12, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4087, 0, 248, 0, 0, 0, 0, 4096, 1330, 0, 0, 3278, 0, 4096, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 3, 3796, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 4096, 63, 0, 0, 0, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3612, 0, 0, 7, 0, 4078, 4096, 0, 0, 337, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 3726, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 19, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4073, 0, 79, 0, 0, 0, 0, 4096, 1467, 0, 0, 3440, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 1, 3778, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 4080, 35, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3387, 0, 0, 8, 0, 4091, 4096, 0, 0, 351, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 3866, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[5, 1, 5, 7, 4090, 0, 0, 0, 364, 42, 0, 4086, 4088, 3991, 6, 4089, 4089, 6, 1879, 8, 0, 0, 0, 4085, 1217, 0, 78, 3554, 0, 4080, 0, 24, 0, 7, 4, 0, 4089, 7, 6, 0, 0, 393, 6, 6, 6, 0, 4087, 0, 4007, 3985, 8, 6, 6, 7, 4090, 0, 5, 0, 4089, 45, 6, 6, 0, 4050, 6, 4089, 0, 6, 2611, 3, 6, 0, 4040, 3208, 0, 4, 4, 0, 0, 3, 4084, 4089, 0, 101, 0, 0, 7, 4031, 0, 0, 6, 5, 9, 4087, 5, 4066, 3375, 0, 5, 646, 0, 4090, 3966, 84, 121, 1549, 0, 5, 21, 12, 3673, 0, 0, 0, 1, 1, 0, 1897, 4090, 10, 7, 6, 6, 65, 167, 4083, 7, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4074, 0, 538, 0, 0, 0, 0, 4096, 2573, 0, 0, 2136, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 27, 0, 0, 0, 0, 3987, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4093, 0, 0, 0, 4096, 3831, 3, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3103, 0, 0, 7, 0, 4095, 4096, 0, 0, 703, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3597, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4091, 0, 276, 0, 0, 0, 0, 4096, 2726, 0, 0, 2597, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2815, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4034, 59, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 3778, 0, 0, 8, 0, 4081, 4096, 0, 0, 1013, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 3188, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 4049, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 8, 0, 0, 0, 0, 2654, 3339, 0, 0, 3732, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3741, 0, 4096, 4096, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 0, 0, 0, 4096, 4084, 8, 0, 0, 0, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1574, 0, 0, 6, 0, 4094, 4096, 0, 0, 89, 0, 0, 0, 0, 4093, 0, 141, 47, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3912, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3718, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16339, 17919, 16388, 16406, 16696, 16382, 14142, 16800] avg=16384.0000 max=17919 max_rank=1 min=14142 min_rank=6 imbalance_ratio=1.093689 max_load_reduction=113153 max_load_reduction_pct=0.863289 moved_to_echo=64020 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 29, 0, 0, 0, 4096, 4049, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 194, 0, 0, 0, 0, 2654, 1898, 0, 0, 3429, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 0, 0, 0, 2, 3548, 0, 4096, 4096, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 4096, 4096, 14, 0, 0, 0, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3068, 0, 0, 11, 0, 4089, 4096, 0, 0, 351, 0, 0, 0, 0, 4089, 0, 141, 47, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3854, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3718, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16366, 16360, 16206, 16404, 16718, 16382, 15894, 16742] avg=16384.0000 max=16742 max_rank=7 min=15894 min_rank=6 imbalance_ratio=1.021851 max_load_reduction=114330 max_load_reduction_pct=0.872269 moved_to_echo=65736 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4049, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 282, 0, 0, 0, 0, 2654, 1068, 0, 0, 3013, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 3, 3691, 0, 4096, 4096, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4083, 16, 0, 0, 0, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3498, 0, 0, 1, 0, 4096, 4096, 0, 0, 1146, 0, 0, 0, 0, 4095, 0, 141, 47, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3679, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3718, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16337, 15209, 16339, 16406, 16710, 16382, 17122, 16567] avg=16384.0000 max=17122 max_rank=6 min=15209 min_rank=1 imbalance_ratio=1.045044 max_load_reduction=113950 max_load_reduction_pct=0.869370 moved_to_echo=66781 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[5, 1, 5, 7, 4090, 0, 0, 0, 364, 42, 0, 4086, 4039, 3991, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 6, 1879, 8, 0, 0, 0, 2636, 1217, 0, 78, 3554, 0, 4080, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 4089, 7, 6, 0, 0, 393, 6, 6, 6, 0, 4087, 0, 4089, 4089, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 3985, 8, 6, 6, 7, 16, 0, 5, 0, 4089, 45, 6, 6, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4089, 0, 6, 2611, 3, 6, 0, 4040, 3208, 0, 4, 4, 0, 0, 3, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 101, 0, 0, 7, 4031, 0, 0, 6, 5, 9, 4079, 5, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3375, 0, 5, 646, 0, 4090, 3966, 84, 121, 1549, 0, 5, 21, 12, 3673, 0, 143, 49, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1897, 4090, 10, 7, 6, 6, 65, 167, 4083, 7, 0, 3717, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16636, 17571, 17146, 16236, 14308, 16393, 17747, 15035] avg=16384.0000 max=17747 max_rank=6 min=14308 min_rank=4 imbalance_ratio=1.083191 max_load_reduction=113325 max_load_reduction_pct=0.864601 moved_to_echo=63483 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 19, 0, 0, 0, 4096, 4049, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 79, 0, 0, 0, 0, 2654, 1467, 0, 0, 3440, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 0, 0, 0, 1, 3778, 0, 4096, 4096, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 4080, 35, 0, 0, 0, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3387, 0, 0, 8, 0, 4091, 4096, 0, 0, 351, 0, 0, 0, 0, 4092, 0, 141, 47, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3866, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3717, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16355, 15809, 16425, 16406, 16725, 16384, 16215, 16753] avg=16384.0000 max=16753 max_rank=7 min=15809 min_rank=1 imbalance_ratio=1.022522 max_load_reduction=114319 max_load_reduction_pct=0.872185 moved_to_echo=66077 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 4049, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 538, 0, 0, 0, 0, 2654, 2573, 0, 0, 2136, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 27, 0, 0, 0, 0, 3987, 0, 4096, 4096, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 0, 0, 0, 4096, 3831, 3, 0, 0, 0, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3103, 0, 0, 7, 0, 4095, 4096, 0, 0, 703, 0, 0, 0, 0, 4096, 0, 141, 47, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3597, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3718, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16338, 16071, 16658, 16406, 16442, 16382, 16290, 16485] avg=16384.0000 max=16658 max_rank=2 min=16071 min_rank=1 imbalance_ratio=1.016724 max_load_reduction=114414 max_load_reduction_pct=0.872910 moved_to_echo=65599 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 12, 0, 0, 0, 4096, 4049, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 248, 0, 0, 0, 0, 2654, 1330, 0, 0, 3278, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 3, 3796, 0, 4096, 4096, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 4096, 63, 0, 0, 0, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3612, 0, 0, 7, 0, 4078, 4096, 0, 0, 337, 0, 0, 0, 0, 4089, 0, 141, 47, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3726, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3718, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16347, 15693, 16449, 16406, 16769, 16385, 16409, 16614] avg=16384.0000 max=16769 max_rank=4 min=15693 min_rank=1 imbalance_ratio=1.023499 max_load_reduction=114303 max_load_reduction_pct=0.872063 moved_to_echo=66177 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0, 4096, 4049, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 276, 0, 0, 0, 0, 2654, 2726, 0, 0, 2597, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2815, 0, 4096, 4096, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4034, 59, 0, 0, 0, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 0, 0, 8, 0, 4081, 4096, 0, 0, 1013, 0, 0, 0, 0, 4087, 0, 141, 47, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3188, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3718, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16354, 16440, 15461, 16402, 16704, 16382, 17253, 16076] avg=16384.0000 max=17253 max_rank=6 min=15461 min_rank=2 imbalance_ratio=1.053040 max_load_reduction=113819 max_load_reduction_pct=0.868370 moved_to_echo=66415 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 128, 128, 32768, 0, 0, 128, 384, 128, 0, 32768, 32384, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 128, 0, 27776, 32768, 128, 128, 128, 128, 128, 256, 32768, 128, 0, 29824, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 32768, 128, 128, 128, 128, 256, 0, 128, 0, 32768, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[25472, 0, 128, 768, 0, 32768, 32640, 128, 128, 5632, 0, 128, 128, 128, 32384, 0, 1152, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 3584, 128, 0, 0, 0, 21248, 15744, 0, 128, 25216, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 32768, 128, 128, 0, 0, 512, 128, 128, 128, 128, 29568, 0, 32768, 32768, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 128, 128, 0, 128, 128, 32768, 0, 0, 128, 128, 128, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 0, 128, 31360, 128, 128, 0, 32768, 31616, 256, 128, 128, 0, 0, 128, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 1, 0, 6, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 2681, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 1098, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 0, 1, 0, 0, 322, 0, 4096, 0, 4096, 4096, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 12, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 1148, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 2873, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 85, 0, 4096, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 3, 0, 1, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 2296, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 1533, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 2, 0, 0, 304, 0, 4096, 0, 4096, 4095, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 2507, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 558, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 1083, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4086, 0, 0, 13, 4077, 699, 0, 521, 0, 4017, 8, 0, 4088, 6, 3563, 6, 4081, 0, 1, 0, 3637, 11, 0, 6, 8, 4090, 4088, 1, 7, 4088, 4047, 4020, 0, 42, 10, 5, 7, 0, 0, 6, 7, 4090, 61, 19, 2, 33, 5, 0, 3940, 0, 0, 6, 4082, 7, 3840, 168, 0, 0, 4057, 4089, 2, 6, 6, 0, 0, 13, 3919, 54, 7, 8, 4070, 4087, 10, 0, 0, 5, 7, 0, 13, 6, 4090, 2, 4, 6, 1, 875, 4, 0, 4079, 4089, 0, 72, 0, 4, 0, 5, 0, 229, 6, 4089, 0, 3966, 4090, 0, 0, 157, 4090, 5, 327, 0, 0, 0, 7, 6, 0, 7, 4090, 0, 0, 0, 5, 4089, 4070, 2734, 16, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 11, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 3277, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 662, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 24, 0, 4096, 0, 0, 1, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 4092, 4095, 0, 0, 0, 0, 2, 0, 0, 152, 0, 4096, 0, 4096, 4095, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 37, 0, 19, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 1732, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 2046, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4096, 0, 14, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 4088, 4093, 0, 1, 0, 0, 1, 0, 0, 308, 0, 4096, 0, 4096, 4093, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 5, 0, 22, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 3246, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 805, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 1, 0, 0, 0, 0, 0, 196, 0, 4096, 0, 4096, 4096, 0, 0, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4005, 0, 0, 0, 4096, 1, 0, 6, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2681, 0, 0, 0, 0, 0, 1545, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1098, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 4096, 2881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 2888, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 0, 1, 0, 2551, 1177, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 4096, 0, 4096, 4096, 0, 0, 3, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4096, 4094, 0, 0, 0, 0, 91, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16300, 16514, 16263, 16417, 16393, 16304, 16329, 16552] avg=16384.0000 max=16552 max_rank=7 min=16263 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=114520 max_load_reduction_pct=0.873718 moved_to_echo=65578 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4005, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2507, 0, 0, 0, 0, 0, 1545, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 558, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 4095, 0, 0, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2888, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 2551, 1177, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 0, 4096, 0, 4096, 4096, 0, 0, 0, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 91, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16293, 16340, 15728, 16415, 16390, 16264, 17087, 16555] avg=16384.0000 max=17087 max_rank=6 min=15728 min_rank=2 imbalance_ratio=1.042908 max_load_reduction=113985 max_load_reduction_pct=0.869637 moved_to_echo=66296 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4005, 0, 0, 0, 4096, 12, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1148, 0, 0, 0, 0, 0, 1545, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2873, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2888, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2551, 1177, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 4096, 0, 4096, 4096, 0, 0, 1, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 91, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16305, 14981, 18043, 16415, 16390, 16293, 16090, 16555] avg=16384.0000 max=18043 max_rank=2 min=14981 min_rank=1 imbalance_ratio=1.101257 max_load_reduction=113029 max_load_reduction_pct=0.862343 moved_to_echo=65328 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4005, 0, 0, 0, 4096, 3, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2296, 0, 0, 0, 0, 0, 1545, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1533, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 4096, 2881, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 4096, 0, 4096, 0, 0, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 2888, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 0, 0, 0, 0, 2, 0, 2551, 1177, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304, 0, 4096, 0, 4096, 4095, 0, 0, 2, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 91, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16297, 16129, 16695, 16425, 16397, 16266, 16309, 16554] avg=16384.0000 max=16695 max_rank=2 min=16129 min_rank=1 imbalance_ratio=1.018982 max_load_reduction=114377 max_load_reduction_pct=0.872627 moved_to_echo=65526 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[3994, 0, 0, 13, 4077, 699, 0, 521, 0, 4017, 8, 0, 0, 6, 3563, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 1, 0, 3637, 11, 0, 6, 8, 0, 1535, 1, 7, 0, 4047, 4020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 10, 5, 7, 0, 0, 6, 7, 4090, 61, 19, 2, 33, 5, 0, 4090, 4088, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 0, 6, 4082, 7, 3840, 168, 0, 0, 4057, 20, 2, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 3919, 54, 7, 8, 4070, 4087, 10, 0, 0, 5, 7, 0, 13, 6, 2887, 1211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2, 4, 6, 1, 875, 4, 0, 4079, 4089, 0, 72, 0, 4, 0, 5, 2553, 1182, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 6, 4089, 0, 3966, 4090, 0, 0, 157, 3708, 5, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6, 0, 7, 4090, 0, 0, 0, 5, 4089, 4070, 2734, 16, 0, 0, 0, 92, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16904, 17354, 15342, 16134, 16297, 17267, 16577, 15197] avg=16384.0000 max=17354 max_rank=1 min=15197 min_rank=7 imbalance_ratio=1.059204 max_load_reduction=113718 max_load_reduction_pct=0.867599 moved_to_echo=65338 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4005, 0, 0, 0, 4096, 11, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3277, 0, 0, 0, 0, 0, 1545, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 662, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 4096, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 0, 4096, 0, 0, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 4096, 0, 0, 1, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 2888, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 4092, 4095, 0, 0, 0, 0, 2, 0, 2551, 1177, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 4096, 0, 4096, 4095, 0, 0, 4, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 91, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16304, 17110, 15828, 16417, 16416, 16286, 16159, 16552] avg=16384.0000 max=17110 max_rank=1 min=15828 min_rank=2 imbalance_ratio=1.044312 max_load_reduction=113962 max_load_reduction_pct=0.869461 moved_to_echo=65413 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4005, 0, 0, 0, 4096, 37, 0, 19, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1732, 0, 0, 0, 0, 0, 1545, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2046, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 4096, 4096, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 14, 0, 4096, 0, 4096, 0, 0, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 2888, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 4088, 4093, 0, 1, 0, 0, 1, 0, 2551, 1177, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 308, 0, 4096, 0, 4096, 4093, 0, 0, 4, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 91, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16349, 15565, 17203, 16429, 16399, 16262, 16313, 16552] avg=16384.0000 max=17203 max_rank=2 min=15565 min_rank=1 imbalance_ratio=1.049988 max_load_reduction=113869 max_load_reduction_pct=0.868752 moved_to_echo=65526 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4005, 0, 0, 0, 4096, 5, 0, 22, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3246, 0, 0, 0, 0, 0, 1545, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 805, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 4096, 2882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2888, 1214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 0, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 1, 0, 0, 0, 0, 2551, 1177, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 4096, 0, 4096, 4096, 0, 0, 8, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 91, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16320, 17079, 15970, 16420, 16390, 16130, 16208, 16555] avg=16384.0000 max=17079 max_rank=1 min=15970 min_rank=2 imbalance_ratio=1.042419 max_load_reduction=113993 max_load_reduction_pct=0.869698 moved_to_echo=65283 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32128, 0, 0, 128, 32768, 768, 0, 640, 0, 32768, 128, 0, 0, 128, 32256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 128, 0, 20608, 128, 0, 128, 128, 0, 12416, 128, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32640, 0, 128, 128, 32768, 128, 32512, 256, 0, 0, 32768, 256, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 2688, 128, 32768, 0, 32640, 32768, 0, 0, 256, 29824, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 32640, 128, 128, 128, 32768, 32768, 128, 128, 0, 128, 128, 0, 128, 128, 23168, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32384, 128, 128, 128, 128, 896, 128, 0, 32768, 32768, 0, 128, 0, 128, 128, 128, 20480, 9472, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 9728, 128, 128, 128, 0, 0, 128, 128, 32768, 128, 128, 128, 128, 128, 0, 32768, 32768, 23168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 32768, 0, 0, 0, 128, 32768, 32768, 31488, 128, 0, 0, 0, 768, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 2, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 49, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4095, 4096, 0, 0, 117, 0, 0, 4096, 0, 0, 0, 4096, 3932, 0, 0, 4096, 0, 4094, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[6, 8, 3995, 7, 1, 1, 0, 0, 4082, 6, 6, 349, 25, 6, 63, 14, 4069, 7, 4089, 0, 0, 0, 0, 4088, 0, 4, 4080, 6, 7, 88, 4052, 5, 7, 10, 0, 0, 4052, 0, 45, 2, 52, 3727, 28, 5, 4074, 2, 0, 27, 0, 0, 0, 0, 0, 4046, 1, 4086, 4078, 0, 6, 0, 0, 4089, 4089, 0, 0, 0, 3, 0, 0, 7, 0, 7, 6, 4073, 6, 7, 7, 23, 4089, 4089, 104, 46, 4089, 15, 4084, 5, 0, 4090, 4090, 4026, 5, 0, 162, 0, 0, 4089, 0, 5, 1, 3998, 2066, 1, 177, 2866, 3, 4088, 4089, 0, 0, 5, 4090, 1306, 0, 6, 0, 0, 0, 8, 22, 0, 7, 6, 3926, 1728, 4057, 0, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 5, 0, 4070, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 77, 4096, 0, 0, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 6, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4090, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4088, 4096, 0, 0, 241, 0, 0, 4096, 3, 0, 0, 4096, 3803, 0, 0, 4096, 0, 4090, 4093, 0, 0, 0, 4094, 6, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 62, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 101, 0, 0, 4096, 0, 0, 0, 4096, 3932, 0, 0, 4096, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 1, 0, 4081, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 149, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 4096, 4094, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4094, 4095, 0, 0, 4096, 0, 4096, 0, 0, 4095, 4093, 4096, 0, 0, 50, 0, 0, 4096, 1, 0, 0, 4096, 3908, 0, 0, 4096, 0, 4095, 4090, 0, 0, 0, 4095, 9, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 49, 4096, 0, 0, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 2, 0, 4096, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4091, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4095, 4095, 4096, 0, 0, 47, 0, 0, 4096, 0, 0, 0, 4096, 4010, 0, 0, 4096, 0, 4094, 4082, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 9, 0, 4084, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 48, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 109, 0, 0, 4096, 1, 0, 0, 4096, 3944, 0, 0, 4096, 0, 4089, 4095, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 630, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 188, 0, 0, 4096, 0, 0, 0, 4096, 3279, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 4096, 2, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 49, 4096, 0, 0, 4096, 0, 0, 0, 3627, 233, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 11, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 4096, 3852, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3961, 4095, 4096, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3932, 0, 0, 4096, 0, 467, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 3847, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16338, 16389, 16332, 16393, 16381, 16365, 16688, 16186] avg=16384.0000 max=16688 max_rank=6 min=16186 min_rank=7 imbalance_ratio=1.018555 max_load_reduction=114384 max_load_reduction_pct=0.872681 moved_to_echo=65620 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[6, 8, 3995, 7, 1, 1, 0, 0, 4082, 6, 6, 349, 25, 6, 63, 14, 4089, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 7, 0, 0, 0, 0, 0, 4088, 0, 4, 4080, 6, 7, 88, 4052, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 10, 0, 0, 4052, 0, 45, 2, 52, 3727, 28, 5, 4074, 2, 0, 27, 3626, 236, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 1, 4086, 4078, 0, 6, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7, 0, 7, 6, 4073, 6, 7, 7, 23, 4089, 4089, 3853, 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 46, 0, 15, 4084, 5, 0, 3954, 4090, 4026, 5, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 3998, 2066, 1, 177, 2866, 3, 462, 4089, 0, 0, 5, 0, 1306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8, 22, 0, 7, 6, 3926, 1728, 4057, 0, 5, 0, 4090, 3841, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16701, 16406, 16029, 16306, 16418, 16491, 14979, 17742] avg=16384.0000 max=17742 max_rank=7 min=14979 min_rank=6 imbalance_ratio=1.082886 max_load_reduction=113330 max_load_reduction_pct=0.864639 moved_to_echo=65630 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 62, 4096, 0, 0, 4096, 0, 0, 0, 3629, 233, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 3852, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3961, 4096, 4096, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3932, 0, 0, 4096, 0, 467, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 3847, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16338, 16391, 16347, 16395, 16383, 16350, 16685, 16183] avg=16384.0000 max=16685 max_rank=6 min=16183 min_rank=7 imbalance_ratio=1.018372 max_load_reduction=114387 max_load_reduction_pct=0.872704 moved_to_echo=65601 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 4096, 0, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 49, 4096, 0, 0, 4096, 0, 0, 0, 3627, 233, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 2, 0, 11, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4091, 4096, 3852, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3960, 4095, 4096, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4010, 0, 0, 4096, 0, 467, 4082, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 0, 4096, 3847, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16338, 16377, 16332, 16397, 16377, 16294, 16753, 16204] avg=16384.0000 max=16753 max_rank=6 min=16204 min_rank=7 imbalance_ratio=1.022522 max_load_reduction=114319 max_load_reduction_pct=0.872185 moved_to_echo=65628 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 4096, 1, 0, 4081, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 149, 4096, 0, 0, 4096, 0, 0, 0, 3628, 233, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 1, 0, 11, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4094, 4095, 3852, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3960, 4093, 4096, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 3908, 0, 0, 4096, 0, 467, 4090, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 3847, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16339, 16375, 16434, 16396, 16380, 16295, 16667, 16186] avg=16384.0000 max=16667 max_rank=6 min=16186 min_rank=7 imbalance_ratio=1.017273 max_load_reduction=114405 max_load_reduction_pct=0.872841 moved_to_echo=65528 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 4070, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 77, 4096, 0, 0, 4096, 0, 0, 0, 3624, 233, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 6, 0, 11, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4090, 4096, 3852, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3961, 4088, 4096, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 3803, 0, 0, 4096, 0, 466, 4093, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4094, 3853, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16342, 16361, 16357, 16398, 16373, 16482, 16563, 16196] avg=16384.0000 max=16563 max_rank=6 min=16196 min_rank=7 imbalance_ratio=1.010925 max_load_reduction=114509 max_load_reduction_pct=0.873634 moved_to_echo=65614 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 630, 4096, 0, 0, 4096, 0, 0, 0, 3628, 233, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4096, 3852, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3961, 4096, 4096, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3279, 0, 0, 4096, 0, 467, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 3847, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16338, 16386, 16914, 16395, 16382, 16437, 16034, 16186] avg=16384.0000 max=16914 max_rank=2 min=16034 min_rank=6 imbalance_ratio=1.032349 max_load_reduction=114158 max_load_reduction_pct=0.870956 moved_to_echo=65039 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 0, 4096, 9, 0, 4084, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 48, 4096, 0, 0, 4096, 0, 0, 0, 3623, 233, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 11, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 4096, 3852, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3961, 4096, 4096, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 3944, 0, 0, 4096, 0, 466, 4095, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 3847, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16338, 16387, 16327, 16392, 16378, 16358, 16703, 16189] avg=16384.0000 max=16703 max_rank=6 min=16189 min_rank=7 imbalance_ratio=1.019470 max_load_reduction=114369 max_load_reduction_pct=0.872566 moved_to_echo=65628 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 0, 0, 0, 0, 32768, 128, 128, 32768, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 0, 32768, 128, 32768, 32768, 0, 128, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 128, 0, 32768, 128, 128, 384, 128, 128, 128, 128, 32768, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 0, 128, 128, 128, 0, 128, 128, 32640, 1792, 32768, 0, 128, 0, 32768, 30848, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 128, 128, 1152, 32512, 128, 128, 32768, 128, 0, 128, 29056, 1920, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 32768, 28928, 128, 256, 31616, 128, 3840, 32768, 0, 0, 128, 0, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 0, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 32768, 32768, 30848, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 31744, 32768, 32768, 128, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 323, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 3830, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 286, 0, 4093, 0, 0, 0, 755, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2991, 4096, 0, 0, 4096, 4096, 0, 4095, 5, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 500, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3915, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 77, 0, 4096, 0, 0, 0, 2402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1298, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 6, 7, 0, 4088, 0, 0, 5, 54, 0, 4070, 1, 0, 0, 1296, 4086, 0, 4064, 0, 4080, 0, 10, 5, 0, 116, 4090, 0, 0, 7, 6, 0, 0, 0, 6, 4044, 4088, 3960, 8, 5, 1, 6, 4085, 0, 6, 7, 5, 13, 3, 28, 4083, 18, 0, 415, 6, 15, 0, 0, 8, 4072, 6, 3, 11, 3259, 2, 0, 163, 4, 11, 53, 6, 6, 6, 10, 4090, 5, 7, 3869, 4032, 5, 4090, 0, 3960, 4076, 233, 4089, 0, 0, 220, 0, 5, 3642, 4060, 0, 0, 3261, 0, 8, 0, 6, 0, 29, 0, 4089, 4089, 3929, 7, 4089, 0, 4076, 0, 4089, 4082, 0, 6, 0, 7, 4090, 314, 0, 4088, 0, 6, 2, 0, 4, 0, 5, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 350, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 3667, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 304, 0, 4086, 0, 0, 0, 2455, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1428, 4094, 0, 0, 4096, 4096, 0, 4094, 3, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4094, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 88, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 3685, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 612, 0, 4092, 0, 0, 0, 1578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2237, 4093, 0, 0, 4096, 4096, 0, 4091, 2, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 527, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3815, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 159, 0, 4095, 0, 0, 0, 375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3319, 4096, 0, 0, 4096, 4096, 0, 4096, 1, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1340, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 3910, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 336, 0, 4093, 0, 0, 0, 2050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 4095, 0, 0, 4096, 4096, 0, 4095, 1, 4096, 4096, 1, 4096, 0, 1, 0, 0, 0, 4094, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 334, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 3147, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 44, 0, 4086, 0, 0, 0, 2228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2451, 4096, 0, 0, 4096, 4096, 0, 4095, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4082, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 323, 4096, 2075, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2195, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3830, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 286, 0, 4093, 0, 0, 0, 755, 0, 4096, 1880, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2991, 2316, 0, 0, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 131, 0, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16110, 16374, 16474, 15252, 17594, 16513, 16405, 16350] avg=16384.0000 max=17594 max_rank=4 min=15252 min_rank=3 imbalance_ratio=1.073853 max_load_reduction=113478 max_load_reduction_pct=0.865768 moved_to_echo=66862 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 350, 4096, 2075, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 2195, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3667, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 304, 0, 4086, 0, 0, 0, 2455, 0, 4096, 1880, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1428, 2315, 0, 0, 4096, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 131, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16136, 16375, 16311, 16963, 16029, 16510, 16405, 16343] avg=16384.0000 max=16963 max_rank=3 min=16029 min_rank=4 imbalance_ratio=1.035339 max_load_reduction=114109 max_load_reduction_pct=0.870583 moved_to_echo=65287 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 500, 4096, 2075, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2195, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3915, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 77, 0, 4096, 0, 0, 0, 2402, 0, 4096, 1880, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1298, 2316, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 131, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16287, 16373, 16559, 16693, 15902, 16507, 16405, 16346] avg=16384.0000 max=16693 max_rank=3 min=15902 min_rank=4 imbalance_ratio=1.018860 max_load_reduction=114379 max_load_reduction_pct=0.872643 moved_to_echo=65160 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 7, 0, 4088, 0, 0, 5, 54, 0, 4070, 1, 0, 0, 1296, 4086, 2065, 1425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 4080, 0, 10, 5, 0, 116, 4090, 0, 0, 7, 6, 0, 0, 2192, 1883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4044, 4088, 3960, 8, 5, 1, 6, 4085, 0, 6, 7, 5, 13, 3, 358, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 4083, 18, 0, 415, 6, 15, 0, 0, 8, 4072, 6, 3, 11, 3259, 2, 4089, 1882, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 163, 4, 11, 53, 6, 6, 6, 10, 2307, 5, 7, 3869, 4032, 5, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 128, 233, 0, 0, 0, 220, 0, 5, 3642, 4060, 0, 0, 3261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 6, 0, 29, 0, 0, 0, 3929, 7, 15, 0, 4076, 0, 4089, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 7, 4090, 314, 0, 4088, 0, 6, 2, 0, 4, 0, 5, 0, 4089, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17103, 16453, 16595, 17944, 14574, 15509, 16241, 16653] avg=16384.0000 max=17944 max_rank=3 min=14574 min_rank=4 imbalance_ratio=1.095215 max_load_reduction=113128 max_load_reduction_pct=0.863098 moved_to_echo=62977 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 527, 4096, 2075, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2195, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3815, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 159, 0, 4095, 0, 0, 0, 375, 0, 4096, 1880, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3319, 2316, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 131, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16314, 16373, 16459, 14747, 17923, 16508, 16405, 16343] avg=16384.0000 max=17923 max_rank=4 min=14747 min_rank=3 imbalance_ratio=1.093933 max_load_reduction=113149 max_load_reduction_pct=0.863258 moved_to_echo=67179 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1340, 4096, 2075, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2195, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3910, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 336, 0, 4093, 0, 0, 0, 2050, 0, 4096, 1880, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 2315, 0, 0, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 131, 1, 0, 0, 1, 0, 0, 0, 4094, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17127, 16374, 16554, 16597, 15162, 16516, 16404, 16338] avg=16384.0000 max=17127 max_rank=0 min=15162 min_rank=4 imbalance_ratio=1.045349 max_load_reduction=113945 max_load_reduction_pct=0.869331 moved_to_echo=64420 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 88, 4096, 2075, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 2195, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3685, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 612, 0, 4092, 0, 0, 0, 1578, 0, 4096, 1880, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2237, 2314, 0, 0, 4096, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 131, 0, 0, 0, 0, 0, 0, 0, 4087, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15874, 16376, 16329, 16400, 16834, 16508, 16402, 16349] avg=16384.0000 max=16834 max_rank=4 min=15874 min_rank=0 imbalance_ratio=1.027466 max_load_reduction=114238 max_load_reduction_pct=0.871567 moved_to_echo=66093 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 334, 4096, 2075, 1424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2195, 1890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3147, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 44, 0, 4086, 0, 0, 0, 2228, 0, 4096, 1880, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2451, 2316, 0, 0, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 131, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 21, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16121, 16374, 15791, 16476, 17054, 16501, 16405, 16350] avg=16384.0000 max=17054 max_rank=4 min=15791 min_rank=2 imbalance_ratio=1.040894 max_load_reduction=114018 max_load_reduction_pct=0.869888 moved_to_echo=66310 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 0, 32768, 0, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 0, 0, 17664, 15232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 256, 128, 128, 128, 128, 128, 128, 14336, 18560, 128, 128, 32640, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 32768, 384, 128, 32768, 0, 128, 128, 0, 128, 0, 128, 128, 32768, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 128, 128, 0, 32768, 128, 0, 0, 4864, 32768, 16640, 11392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 0, 512, 128, 128, 0, 1920, 128, 32768, 128, 128, 128, 15104, 128, 32768, 15104, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 32640, 128, 256, 0, 32768, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 32768, 30080, 32640, 128, 128, 128, 128, 32768, 0, 128, 128, 128, 128, 128, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 32640, 1152, 256, 0, 0, 128, 256, 0, 128, 32384, 32768, 0, 0, 32000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 2, 0, 4094, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4092, 4096, 0, 4096, 0, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4096, 4096, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 274, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4050, 0, 2038, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 4074, 4096, 0, 4095, 0, 1801, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 4096, 4094, 4096, 0, 0, 0, 34, 5, 0, 0, 0, 1475, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4092, 0, 804, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 1, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4085, 4096, 0, 4096, 0, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4096, 4096, 4096, 0, 0, 0, 7, 1, 0, 0, 0, 508, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 3996, 0, 1673, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3900, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 4083, 4096, 0, 4095, 0, 3016, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 4096, 4095, 4096, 0, 0, 0, 27, 0, 0, 0, 0, 349, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4085, 0, 924, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 0, 11, 0, 2, 0, 2761, 0, 0, 4086, 108, 5, 0, 0, 4089, 4079, 8, 0, 0, 4088, 0, 0, 5, 4054, 67, 10, 4087, 4089, 0, 4085, 42, 2920, 5, 6, 12, 6, 0, 21, 0, 9, 0, 54, 0, 7, 6, 8, 5, 0, 5, 0, 25, 4021, 4012, 43, 3970, 4085, 4089, 0, 0, 0, 0, 0, 0, 9, 6, 1, 6, 0, 0, 4038, 47, 8, 0, 0, 8, 6, 4052, 4068, 4005, 0, 3242, 0, 2117, 9, 0, 5, 4082, 77, 7, 0, 0, 4069, 5, 7, 4089, 4087, 0, 8, 4089, 0, 0, 5, 4087, 7, 9, 7, 0, 144, 9, 4087, 4082, 0, 0, 4090, 3, 0, 6, 1216, 7, 15, 4, 4089, 4090, 3976] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 4093, 0, 0, 0, 0, 4096, 4096, 0, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 4068, 4096, 0, 4090, 0, 1494, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 8, 4096, 4093, 4096, 0, 0, 0, 28, 0, 0, 0, 0, 198, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4091, 0, 2480, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 4096, 4095, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4082, 4096, 0, 4096, 0, 2165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 0, 0, 15, 0, 0, 0, 0, 144, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4072, 0, 1959, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 4, 0, 4094, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 4083, 4096, 0, 4096, 0, 3305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 9, 4096, 4095, 4096, 0, 0, 0, 29, 1, 0, 0, 0, 245, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 679, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 2, 0, 4094, 0, 0, 4073, 2759, 1446, 136, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4092, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1836, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1326, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4096, 4096, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 274, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2650, 4096, 0, 4050, 0, 2038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16610, 16391, 15870, 16411, 16249, 16930, 16364, 16247] avg=16384.0000 max=16930 max_rank=5 min=15870 min_rank=2 imbalance_ratio=1.033325 max_load_reduction=114142 max_load_reduction_pct=0.870834 moved_to_echo=65790 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 4096, 0, 0, 4073, 2759, 1446, 136, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 0, 4074, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1801, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1326, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 6, 4096, 4094, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 5, 0, 0, 0, 1475, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2650, 4096, 0, 4092, 0, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16625, 16375, 15835, 16411, 17478, 15738, 16364, 16246] avg=16384.0000 max=17478 max_rank=4 min=15738 min_rank=5 imbalance_ratio=1.066772 max_load_reduction=113594 max_load_reduction_pct=0.866653 moved_to_echo=65826 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 1, 0, 4096, 0, 0, 4073, 2759, 1446, 136, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4085, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1326, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4, 4096, 4096, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 0, 0, 0, 508, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2650, 4096, 0, 3996, 0, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16489, 16384, 16186, 16411, 16479, 16511, 16364, 16248] avg=16384.0000 max=16511 max_rank=5 min=16186 min_rank=2 imbalance_ratio=1.007751 max_load_reduction=114561 max_load_reduction_pct=0.874031 moved_to_echo=65602 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 7, 0, 0, 0, 0, 11, 0, 2, 0, 2761, 0, 0, 4086, 108, 5, 4073, 2755, 1443, 142, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 8, 0, 0, 4088, 0, 0, 5, 4054, 67, 10, 4087, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 42, 2920, 5, 6, 12, 6, 0, 21, 0, 9, 0, 54, 0, 7, 4082, 4069, 1328, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8, 5, 0, 5, 0, 25, 4021, 4012, 43, 3970, 4085, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 6, 1, 6, 0, 0, 4038, 47, 8, 0, 0, 8, 6, 4089, 4089, 3670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 2625, 4005, 0, 3242, 0, 2117, 9, 0, 5, 0, 77, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7, 4089, 4087, 0, 8, 4065, 0, 0, 5, 4087, 7, 9, 7, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3945, 4082, 0, 0, 4090, 3, 0, 6, 1216, 7, 15, 4, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15417, 16404, 17066, 16196, 15977, 16139, 16520, 17353] avg=16384.0000 max=17353 max_rank=7 min=15417 min_rank=0 imbalance_ratio=1.059143 max_load_reduction=113719 max_load_reduction_pct=0.867607 moved_to_echo=65989 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3900, 0, 0, 4096, 0, 0, 4073, 2759, 1446, 136, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 4083, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3016, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1326, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 4096, 4095, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 349, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2650, 4096, 0, 4085, 0, 924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 4096, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16430, 16383, 17051, 16408, 16338, 15851, 16365, 16246] avg=16384.0000 max=17051 max_rank=2 min=15851 min_rank=5 imbalance_ratio=1.040710 max_load_reduction=114021 max_load_reduction_pct=0.869911 moved_to_echo=64800 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 0, 0, 4096, 0, 0, 4073, 2759, 1446, 136, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4082, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1326, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 144, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2650, 4096, 0, 4072, 0, 1959, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16477, 16381, 16199, 16407, 16123, 16873, 16364, 16248] avg=16384.0000 max=16873 max_rank=5 min=16123 min_rank=4 imbalance_ratio=1.029846 max_load_reduction=114199 max_load_reduction_pct=0.871269 moved_to_echo=65608 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 0, 4093, 0, 0, 4073, 2759, 1446, 136, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 4068, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 1494, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1326, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 8, 4096, 4093, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 198, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2650, 4096, 0, 4091, 0, 2480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 4095, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16552, 16371, 15526, 16412, 16189, 17413, 16367, 16242] avg=16384.0000 max=17413 max_rank=5 min=15526 min_rank=2 imbalance_ratio=1.062805 max_load_reduction=113659 max_load_reduction_pct=0.867149 moved_to_echo=66211 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 4, 0, 4094, 0, 0, 4073, 2759, 1446, 136, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 4083, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1326, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 4096, 9, 4096, 4095, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 1, 0, 0, 0, 245, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 3676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2650, 4096, 0, 4096, 0, 679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4076, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 4096, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16472, 16383, 17339, 16416, 16239, 15617, 16364, 16242] avg=16384.0000 max=17339 max_rank=2 min=15617 min_rank=5 imbalance_ratio=1.058289 max_load_reduction=113733 max_load_reduction_pct=0.867714 moved_to_echo=64462 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 128, 18688, 128, 128, 128, 128, 0, 128, 0, 128, 0, 128, 0, 128, 32768, 32768, 10624, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 32640, 0, 0, 128, 32768, 128, 128, 128, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 31744, 32768, 0, 0, 32768, 128, 0, 128, 1280, 128, 128, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 128, 32768, 0, 0, 128, 32768, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 0, 0, 0, 0, 128, 0, 128, 0, 30720, 128, 0, 32768, 128, 128, 32640, 22144, 11648, 1152, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 21248, 32768, 0, 31744, 0, 12800, 128, 0, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 128, 0, 128, 128, 3200, 128, 0, 0, 32768, 128, 128, 0, 0, 128, 128, 32768, 32768, 29440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 32768, 32768, 128, 32768, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True + [2026-08-30 05:03:39] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 812.4 | throughput per GPU (TFLOP/s/GPU): 410.4 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.902222E+00 | loss scale: 1.0 | grad norm: 5.184 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1946, 1249, 1960, 1, 549, 0, 2238, 1228, 1290, 2799, 1489, 487, 1398, 2380, 1, 1073, 699, 1433, 717, 1367, 11, 758, 169, 1730, 582, 75, 3217, 1669, 1668, 97, 1828, 1117, 149, 7, 2101, 686, 130, 2773, 1471, 1024, 286, 3, 1907, 2247, 0, 1398, 1536, 1094, 1135, 599, 722, 931, 1, 99, 24, 1119, 0, 487, 94, 388, 1423, 2240, 1103, 491, 1303, 9, 0, 1975, 1484, 2031, 1431, 701, 1394, 1550, 2078, 441, 1377, 23, 1132, 1493, 869, 603, 19, 15, 2100, 411, 1163, 4, 1887, 228, 1262, 735, 431, 1931, 923, 0, 1137, 1114, 8, 2312, 22, 444, 1205, 1167, 257, 1356, 119, 1117, 446, 1540, 2248, 1858, 2003, 1811, 459, 954, 2259, 1461, 2120, 1226, 856, 1544, 1161, 920, 1321, 25, 679, 27] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1737, 1359, 1477, 5, 474, 0, 1886, 985, 768, 2205, 789, 1137, 1852, 2214, 0, 1323, 659, 1372, 1596, 1284, 16, 668, 227, 2183, 452, 128, 2860, 1922, 1874, 185, 1328, 1104, 198, 0, 1949, 393, 171, 2294, 1373, 815, 307, 0, 2031, 1990, 2, 1266, 1100, 1422, 1301, 524, 657, 985, 18, 151, 86, 1364, 0, 229, 51, 382, 1669, 2276, 1020, 797, 1375, 7, 3, 1651, 1889, 2259, 1195, 1312, 942, 1665, 1883, 688, 1682, 7, 993, 1712, 1140, 699, 36, 15, 2463, 758, 1247, 19, 1513, 383, 922, 1250, 385, 1631, 995, 3, 1519, 1278, 3, 1968, 40, 601, 1499, 1416, 315, 1463, 442, 1463, 791, 706, 2016, 1269, 1485, 1418, 647, 1298, 2454, 1344, 1830, 1580, 919, 1914, 1292, 658, 1185, 17, 605, 25] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1856, 1475, 1566, 0, 941, 1, 2349, 844, 931, 1984, 1203, 596, 1386, 1761, 0, 737, 536, 1464, 1558, 1250, 19, 657, 213, 1997, 343, 75, 3128, 1991, 1147, 82, 1594, 1552, 301, 2, 1350, 441, 226, 1898, 1056, 1018, 260, 31, 1892, 1952, 0, 1767, 1440, 1012, 1247, 707, 501, 1200, 18, 54, 17, 1463, 1, 341, 60, 622, 1587, 2306, 1248, 723, 1107, 14, 0, 1937, 2083, 1840, 1132, 1451, 1132, 1642, 2045, 722, 1747, 15, 1026, 1287, 826, 401, 43, 47, 1701, 794, 1882, 38, 2097, 237, 680, 1042, 372, 1682, 747, 1, 1085, 983, 12, 1973, 22, 853, 1359, 1423, 340, 1478, 271, 1296, 1075, 1838, 2276, 1842, 2288, 1501, 311, 770, 2305, 1133, 2161, 1352, 1038, 1783, 1776, 942, 1207, 47, 558, 27] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2328, 1735, 1406, 0, 560, 0, 1950, 807, 1507, 1986, 1763, 577, 1079, 2584, 0, 967, 799, 1043, 1136, 803, 48, 1008, 191, 2132, 642, 119, 2763, 1714, 810, 200, 1507, 1091, 94, 0, 2219, 394, 186, 2493, 1081, 825, 441, 17, 1770, 2756, 0, 1857, 892, 911, 1089, 451, 1126, 1548, 40, 146, 6, 1098, 2, 355, 198, 877, 1288, 2569, 1442, 549, 840, 4, 0, 1311, 2093, 1713, 1170, 1058, 1366, 1286, 1525, 327, 1570, 34, 1250, 1929, 949, 697, 9, 17, 1705, 744, 1246, 25, 2533, 235, 1036, 905, 374, 2007, 572, 6, 1331, 975, 1, 1737, 25, 543, 1387, 1250, 318, 1835, 94, 1171, 392, 1357, 2609, 1705, 2017, 1008, 748, 1804, 2103, 1227, 2135, 1496, 1022, 2045, 1286, 750, 1514, 18, 608, 50] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2176, 1872, 1111, 1, 563, 0, 1720, 1467, 1155, 2096, 1055, 697, 1128, 2820, 0, 1190, 877, 1348, 1148, 660, 11, 819, 159, 1897, 314, 85, 3160, 2057, 1073, 208, 1483, 1372, 162, 2, 2346, 738, 489, 2642, 1509, 985, 242, 7, 2077, 2585, 1, 1505, 1335, 1140, 1366, 511, 748, 1161, 21, 229, 45, 2087, 0, 262, 28, 661, 1388, 2427, 1655, 203, 588, 11, 0, 1626, 1944, 1984, 1081, 1231, 1092, 1496, 2258, 345, 1369, 11, 1576, 2024, 514, 543, 48, 19, 1637, 492, 1514, 17, 2055, 193, 929, 1011, 238, 1745, 957, 0, 1095, 1075, 13, 1949, 17, 357, 1419, 1124, 274, 1701, 112, 927, 341, 1753, 2396, 1362, 2136, 1772, 514, 914, 2027, 1547, 1760, 1265, 1011, 1837, 1646, 329, 1020, 19, 498, 35] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1657, 2176, 1569, 10, 522, 5, 1680, 921, 1342, 2186, 1249, 417, 1661, 2117, 0, 1304, 807, 1351, 1312, 1237, 16, 769, 230, 2463, 345, 80, 2922, 1718, 1300, 218, 1625, 1107, 230, 21, 2059, 511, 150, 2265, 978, 777, 329, 16, 1958, 2571, 1, 1257, 1953, 989, 984, 392, 880, 1243, 12, 74, 23, 1995, 8, 503, 120, 657, 1160, 2429, 963, 670, 833, 39, 2, 1579, 2084, 2004, 1373, 1250, 1093, 1964, 2028, 333, 1404, 15, 1080, 1789, 1077, 581, 30, 56, 1670, 653, 1394, 47, 2123, 258, 1364, 886, 385, 1793, 499, 4, 830, 799, 26, 1838, 73, 615, 1963, 1168, 271, 1716, 121, 1134, 213, 1653, 2502, 1726, 2320, 1215, 298, 687, 2095, 928, 2014, 1281, 931, 1878, 1583, 861, 1568, 16, 514, 51] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1848, 1628, 1096, 0, 385, 0, 1777, 957, 1660, 3026, 1722, 290, 994, 2957, 0, 911, 1076, 1908, 483, 1342, 12, 1059, 190, 2625, 335, 117, 2748, 1567, 1158, 152, 1444, 1116, 169, 1, 1936, 591, 166, 2413, 1370, 699, 329, 24, 1625, 2732, 0, 1649, 1146, 1362, 1154, 659, 965, 1020, 4, 114, 4, 1116, 0, 436, 26, 410, 1169, 2475, 1465, 763, 746, 16, 0, 1661, 2393, 1740, 561, 962, 937, 728, 2032, 316, 2373, 16, 1407, 2024, 1170, 432, 6, 5, 2000, 463, 799, 24, 2246, 285, 1391, 647, 437, 1841, 488, 0, 1700, 360, 1, 1652, 9, 676, 1493, 892, 177, 1540, 176, 935, 453, 2567, 2939, 1827, 1904, 1374, 746, 1312, 1872, 899, 2033, 1518, 890, 1860, 1421, 696, 1798, 10, 594, 27] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1443, 1798, 1720, 0, 507, 1, 2141, 607, 1088, 3023, 1497, 792, 1544, 2674, 1, 833, 660, 1345, 847, 940, 17, 539, 194, 2339, 438, 43, 3406, 1884, 1353, 161, 1345, 1220, 417, 3, 1751, 462, 210, 2347, 1430, 1178, 184, 35, 1818, 2189, 0, 1558, 1513, 1079, 978, 535, 680, 923, 11, 98, 7, 1698, 0, 284, 17, 948, 1099, 2591, 1581, 477, 1094, 45, 0, 1187, 1856, 1623, 1520, 1086, 1140, 1493, 2001, 310, 2270, 20, 1394, 1694, 977, 366, 26, 13, 1942, 576, 956, 13, 1779, 271, 1192, 1096, 527, 1989, 743, 0, 1549, 641, 6, 1919, 15, 697, 1184, 854, 230, 1435, 359, 1190, 323, 1582, 2733, 1661, 2377, 1335, 590, 943, 2524, 1680, 1985, 1157, 908, 1773, 1306, 368, 1389, 16, 621, 22] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1737, 1359, 1477, 5, 474, 0, 1886, 985, 768, 2047, 789, 1137, 1852, 0, 0, 1323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 659, 1372, 1596, 1284, 16, 668, 227, 2183, 452, 128, 2179, 1922, 1874, 185, 1328, 1104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 0, 1949, 393, 171, 2294, 1373, 815, 307, 0, 2031, 1990, 2, 1266, 1100, 1422, 158, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1301, 524, 657, 985, 18, 151, 86, 1364, 0, 229, 51, 382, 1669, 2276, 1020, 797, 2214, 2185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1375, 7, 3, 1651, 1702, 2259, 1195, 1312, 942, 1665, 0, 688, 1682, 7, 993, 1712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1140, 699, 36, 15, 2463, 758, 1247, 19, 1513, 383, 922, 1250, 385, 1631, 995, 3, 1883, 681, 333, 209, 269, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1519, 1278, 3, 1968, 40, 601, 1499, 1416, 315, 1463, 442, 1463, 791, 706, 1683, 1269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1276, 1418, 647, 1298, 0, 1344, 1830, 1580, 919, 1914, 1292, 658, 1185, 17, 605, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15839, 17177, 15560, 15909, 17193, 16930, 16456, 16008] avg=16384.0000 max=17193 max_rank=4 min=15560 min_rank=2 imbalance_ratio=1.049377 max_load_reduction=113879 max_load_reduction_pct=0.868828 moved_to_echo=66587 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1946, 1249, 1960, 1, 549, 0, 2238, 1228, 1290, 2599, 1489, 487, 1398, 0, 1, 1073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 699, 1433, 717, 1367, 11, 758, 169, 1730, 582, 75, 2451, 1669, 1668, 97, 1828, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 7, 2101, 686, 130, 2773, 1471, 1024, 286, 3, 1907, 2247, 0, 1398, 1536, 1094, 200, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1135, 599, 722, 931, 1, 99, 24, 1119, 0, 487, 94, 388, 1423, 2240, 1103, 491, 2380, 2011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1303, 9, 0, 1975, 1338, 2031, 1431, 701, 1394, 1550, 0, 441, 1377, 23, 1132, 1493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 869, 603, 19, 15, 2100, 411, 1163, 4, 1887, 228, 1262, 735, 431, 1931, 923, 0, 2078, 766, 371, 282, 248, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1137, 1114, 8, 2312, 22, 444, 1205, 1167, 257, 1356, 119, 1117, 446, 1540, 1877, 1858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1721, 1811, 459, 954, 0, 1461, 2120, 1226, 856, 1544, 1161, 920, 1321, 25, 679, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17508, 16371, 17083, 15247, 16198, 16401, 15979, 16285] avg=16384.0000 max=17508 max_rank=0 min=15247 min_rank=3 imbalance_ratio=1.068604 max_load_reduction=113564 max_load_reduction_pct=0.866425 moved_to_echo=64863 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2328, 1735, 1406, 0, 560, 0, 1950, 807, 1507, 1844, 1763, 577, 1079, 0, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 799, 1043, 1136, 803, 48, 1008, 191, 2132, 642, 119, 2105, 1714, 810, 200, 1507, 1091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 2219, 394, 186, 2493, 1081, 825, 441, 17, 1770, 2756, 0, 1857, 892, 911, 142, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1089, 451, 1126, 1548, 40, 146, 6, 1098, 2, 355, 198, 877, 1288, 2569, 1442, 549, 2584, 1873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 840, 4, 0, 1311, 1886, 1713, 1170, 1058, 1366, 1286, 0, 327, 1570, 34, 1250, 1929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 949, 697, 9, 17, 1705, 744, 1246, 25, 2533, 235, 1036, 905, 374, 2007, 572, 6, 1525, 658, 431, 284, 230, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1331, 975, 1, 1737, 25, 543, 1387, 1250, 318, 1835, 94, 1171, 392, 1357, 2178, 1705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1733, 1008, 748, 1804, 0, 1227, 2135, 1496, 1022, 2045, 1286, 750, 1514, 18, 608, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16523, 15348, 16179, 17241, 15744, 16294, 16299, 17444] avg=16384.0000 max=17444 max_rank=7 min=15348 min_rank=1 imbalance_ratio=1.064697 max_load_reduction=113628 max_load_reduction_pct=0.866913 moved_to_echo=65781 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2176, 1872, 1111, 1, 563, 0, 1720, 1467, 1155, 1946, 1055, 697, 1128, 0, 0, 1190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 877, 1348, 1148, 660, 11, 819, 159, 1897, 314, 85, 2407, 2057, 1073, 208, 1483, 1372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 2, 2346, 738, 489, 2642, 1509, 985, 242, 7, 2077, 2585, 1, 1505, 1335, 1140, 150, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1366, 511, 748, 1161, 21, 229, 45, 2087, 0, 262, 28, 661, 1388, 2427, 1655, 203, 2820, 1806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 11, 0, 1626, 1753, 1984, 1081, 1231, 1092, 1496, 0, 345, 1369, 11, 1576, 2024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 514, 543, 48, 19, 1637, 492, 1514, 17, 2055, 193, 929, 1011, 238, 1745, 957, 0, 2258, 753, 396, 301, 221, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 1075, 13, 1949, 17, 357, 1419, 1124, 274, 1701, 112, 927, 341, 1753, 2000, 1362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1835, 1772, 514, 914, 0, 1547, 1760, 1265, 1011, 1837, 1646, 329, 1020, 19, 498, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16081, 15918, 18008, 17418, 16187, 15939, 15519, 16002] avg=16384.0000 max=18008 max_rank=2 min=15519 min_rank=6 imbalance_ratio=1.099121 max_load_reduction=113064 max_load_reduction_pct=0.862610 moved_to_echo=63647 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1856, 1475, 1566, 0, 941, 1, 2349, 844, 931, 1842, 1203, 596, 1386, 0, 0, 737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 536, 1464, 1558, 1250, 19, 657, 213, 1997, 343, 75, 2383, 1991, 1147, 82, 1594, 1552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 301, 2, 1350, 441, 226, 1898, 1056, 1018, 260, 31, 1892, 1952, 0, 1767, 1440, 1012, 142, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1247, 707, 501, 1200, 18, 54, 17, 1463, 1, 341, 60, 622, 1587, 2306, 1248, 723, 1761, 2053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 14, 0, 1937, 1878, 1840, 1132, 1451, 1132, 1642, 0, 722, 1747, 15, 1026, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 826, 401, 43, 47, 1701, 794, 1882, 38, 2097, 237, 680, 1042, 372, 1682, 747, 1, 2045, 745, 376, 323, 252, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1085, 983, 12, 1973, 22, 853, 1359, 1423, 340, 1478, 271, 1296, 1075, 1838, 1900, 1842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1965, 1501, 311, 770, 0, 1133, 2161, 1352, 1038, 1783, 1776, 942, 1207, 47, 558, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15727, 16861, 14888, 15909, 16930, 16436, 17750, 16571] avg=16384.0000 max=17750 max_rank=6 min=14888 min_rank=2 imbalance_ratio=1.083374 max_load_reduction=113322 max_load_reduction_pct=0.864578 moved_to_echo=67687 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1657, 2176, 1569, 10, 522, 5, 1680, 921, 1342, 2026, 1249, 417, 1661, 0, 0, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 807, 1351, 1312, 1237, 16, 769, 230, 2463, 345, 80, 2221, 1718, 1300, 218, 1625, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 21, 2059, 511, 150, 2265, 978, 777, 329, 16, 1958, 2571, 1, 1257, 1953, 989, 160, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 984, 392, 880, 1243, 12, 74, 23, 1995, 8, 503, 120, 657, 1160, 2429, 963, 670, 2117, 1866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 833, 39, 2, 1579, 1869, 2004, 1373, 1250, 1093, 1964, 0, 333, 1404, 15, 1080, 1789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 581, 30, 56, 1670, 653, 1394, 47, 2123, 258, 1364, 886, 385, 1793, 499, 4, 2028, 701, 417, 332, 229, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 830, 799, 26, 1838, 73, 615, 1963, 1168, 271, 1716, 121, 1134, 213, 1653, 2085, 1726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1988, 1215, 298, 687, 0, 928, 2014, 1281, 931, 1878, 1583, 861, 1568, 16, 514, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16539, 16799, 16330, 16096, 16627, 16637, 16231, 15813] avg=16384.0000 max=16799 max_rank=1 min=15813 min_rank=7 imbalance_ratio=1.025330 max_load_reduction=114273 max_load_reduction_pct=0.871834 moved_to_echo=65308 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1848, 1628, 1096, 0, 385, 0, 1777, 957, 1660, 2809, 1722, 290, 994, 0, 0, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 1908, 483, 1342, 12, 1059, 190, 2625, 335, 117, 2094, 1567, 1158, 152, 1444, 1116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 1, 1936, 591, 166, 2413, 1370, 699, 329, 24, 1625, 2732, 0, 1649, 1146, 1362, 217, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1154, 659, 965, 1020, 4, 114, 4, 1116, 0, 436, 26, 410, 1169, 2475, 1465, 763, 2957, 1667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 746, 16, 0, 1661, 2157, 1740, 561, 962, 937, 728, 0, 316, 2373, 16, 1407, 2024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 432, 6, 5, 2000, 463, 799, 24, 2246, 285, 1391, 647, 437, 1841, 488, 0, 2032, 654, 486, 268, 205, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1700, 360, 1, 1652, 9, 676, 1493, 892, 177, 1540, 176, 935, 453, 2567, 2453, 1827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1636, 1374, 746, 1312, 0, 899, 2033, 1518, 890, 1860, 1421, 696, 1798, 10, 594, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16077, 16678, 16544, 16404, 15644, 16000, 16911, 16814] avg=16384.0000 max=16911 max_rank=6 min=15644 min_rank=4 imbalance_ratio=1.032166 max_load_reduction=114161 max_load_reduction_pct=0.870979 moved_to_echo=65369 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1443, 1798, 1720, 0, 507, 1, 2141, 607, 1088, 2806, 1497, 792, 1544, 0, 1, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 1345, 847, 940, 17, 539, 194, 2339, 438, 43, 2595, 1884, 1353, 161, 1345, 1220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 3, 1751, 462, 210, 2347, 1430, 1178, 184, 35, 1818, 2189, 0, 1558, 1513, 1079, 217, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 978, 535, 680, 923, 11, 98, 7, 1698, 0, 284, 17, 948, 1099, 2591, 1581, 477, 2674, 2247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 45, 0, 1187, 1673, 1623, 1520, 1086, 1140, 1493, 0, 310, 2270, 20, 1394, 1694, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 977, 366, 26, 13, 1942, 576, 956, 13, 1779, 271, 1192, 1096, 527, 1989, 743, 0, 2001, 811, 451, 335, 277, 94, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1549, 641, 6, 1919, 15, 697, 1184, 854, 230, 1435, 359, 1190, 323, 1582, 2282, 1661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2042, 1335, 590, 943, 0, 1680, 1985, 1157, 908, 1773, 1306, 368, 1389, 16, 621, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16778, 15920, 16480, 16848, 16549, 16435, 15927, 16135] avg=16384.0000 max=16848 max_rank=3 min=15920 min_rank=1 imbalance_ratio=1.028320 max_load_reduction=114224 max_load_reduction_pct=0.871460 moved_to_echo=65046 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[6144, 11264, 8832, 8960, 256, 6400, 1664, 17408, 3456, 768, 18560, 14592, 10496, 1408, 12160, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[10368, 7296, 128, 15360, 256, 4864, 11520, 9344, 2304, 12544, 1792, 9344, 4096, 13056, 16512, 13312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9344, 4480, 6400, 9088, 128, 1024, 256, 12032, 128, 2944, 640, 4992, 10880, 19328, 10496, 4736, 19584, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[15104, 13312, 12032, 128, 4608, 128, 15744, 7936, 9856, 17920, 10880, 5120, 11136, 0, 128, 8448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[14208, 11520, 4352, 8704, 0, 10240, 16128, 10880, 7680, 14720, 11520, 5632, 11008, 256, 4736, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[7936, 256, 128, 12928, 14336, 15232, 9472, 9088, 9216, 11904, 0, 3584, 13824, 256, 9984, 13952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1792, 128, 15744, 4224, 1792, 19200, 10368, 7424, 2432, 256, 15104, 19072, 128, 12288, 11008, 9088, 1408, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[7552, 4352, 256, 256, 15232, 4992, 10240, 256, 16256, 2176, 8832, 7680, 3200, 14720, 6016, 128, 15872, 5888, 3328, 2432, 2048, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 230, 3339, 0, 10, 0, 0, 0, 1, 14, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 3964, 1, 3997, 0, 0, 2382, 0, 4096, 4096, 9, 0, 0, 4096, 0, 3725, 210, 970, 0, 0, 4094, 0, 0, 139, 0, 606, 0, 2251, 581, 4080, 0, 0, 4096, 0, 2251, 4096, 0, 0, 3743, 546, 0, 0, 3899, 0, 3486, 0, 0, 0, 115, 0, 0, 4096, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 225, 4, 4062, 0, 0, 0, 52, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 1146, 156, 4096, 0, 194, 4096, 0, 2053, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 1674, 0, 0, 3264, 3542, 0, 449, 0, 0, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 50, 1334, 0, 0, 0, 0, 0, 6, 1, 0, 0, 0, 4081, 0, 0, 4096, 0, 0, 4030, 14, 3408, 2, 0, 2774, 0, 4092, 4056, 8, 0, 0, 4096, 0, 2330, 2, 306, 0, 0, 3718, 0, 0, 225, 0, 530, 0, 3173, 2919, 4090, 0, 0, 4096, 0, 2683, 4096, 0, 0, 3935, 1491, 0, 0, 4094, 0, 3409, 73, 0, 0, 1824, 1, 0, 4081, 0, 3558, 4006, 0, 0, 0, 0, 0, 1, 138, 2, 1088, 0, 0, 0, 1019, 0, 0, 0, 0, 0, 4096, 45, 0, 0, 0, 0, 4095, 225, 1619, 3985, 0, 1093, 4095, 0, 2569, 0, 0, 0, 4096, 4096, 0, 0, 66, 0, 1266, 2, 0, 4091, 1848, 0, 753, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 20, 2823, 0, 5, 0, 0, 7, 1, 5, 0, 0, 3, 4055, 6, 1, 4089, 3, 5, 4076, 7, 4089, 10, 1, 3579, 0, 4077, 4096, 0, 0, 0, 4089, 0, 2494, 122, 1128, 0, 0, 3900, 2, 0, 110, 0, 986, 0, 3299, 689, 4080, 0, 0, 4096, 12, 1866, 4088, 7, 0, 3708, 1933, 9, 2, 4069, 0, 3257, 55, 2, 0, 716, 48, 0, 4096, 0, 3885, 4096, 3, 0, 0, 0, 8, 33, 338, 17, 3052, 0, 0, 0, 346, 10, 0, 0, 0, 5, 4082, 34, 1, 3, 3, 1, 4091, 626, 294, 4053, 0, 440, 4095, 0, 2783, 2, 1, 0, 4094, 4096, 0, 0, 114, 0, 1701, 6, 0, 4078, 667, 1, 107, 6, 0, 0, 4079] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 304, 2812, 0, 0, 12, 0, 0, 5, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4029, 0, 4064, 36, 0, 963, 0, 4095, 4092, 0, 0, 0, 4096, 0, 3682, 0, 1174, 0, 0, 3791, 0, 0, 372, 0, 1436, 0, 3387, 1508, 4071, 0, 0, 4096, 0, 1500, 4096, 0, 0, 3624, 1865, 0, 0, 3791, 0, 3007, 15, 0, 0, 1620, 0, 0, 4096, 0, 3707, 4026, 0, 0, 0, 0, 0, 0, 291, 0, 1538, 0, 0, 3, 761, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 1549, 822, 4092, 0, 328, 4093, 0, 2182, 0, 0, 0, 4096, 4096, 0, 0, 29, 0, 1858, 0, 0, 4079, 833, 0, 567, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 100, 2398, 0, 2, 0, 0, 0, 0, 1, 1, 0, 0, 4095, 1, 1, 4095, 1, 0, 4060, 0, 3951, 0, 1, 2188, 0, 4090, 4086, 10, 1, 0, 4095, 0, 2433, 29, 1880, 2, 1, 4034, 1, 0, 238, 1, 341, 0, 2871, 1037, 4086, 0, 1, 4095, 0, 1454, 4094, 0, 1, 3146, 1571, 0, 1, 4053, 0, 3819, 21, 0, 1, 948, 23, 1, 4095, 1, 3622, 4094, 0, 1, 0, 2, 0, 0, 185, 1, 2480, 0, 1, 0, 1293, 0, 1, 1, 0, 1, 4095, 0, 0, 1, 1, 1, 4095, 1258, 317, 4037, 1, 189, 4090, 1, 3004, 0, 0, 0, 4095, 4095, 0, 1, 119, 0, 2044, 1, 0, 4093, 2130, 0, 170, 0, 0, 1, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 153, 3038, 0, 1, 0, 0, 0, 12, 8, 1, 0, 0, 4095, 0, 1, 4095, 0, 0, 4073, 18, 4083, 0, 1, 3522, 0, 4067, 4095, 41, 1, 0, 4095, 0, 2776, 134, 1119, 1, 1, 3932, 0, 1, 153, 1, 77, 1, 2603, 1594, 4073, 0, 1, 4095, 0, 1095, 4094, 0, 1, 3278, 1818, 0, 1, 3780, 1, 3054, 9, 0, 1, 974, 1, 1, 4095, 0, 3706, 4084, 1, 1, 0, 0, 0, 1, 158, 1, 3175, 1, 1, 1, 1495, 0, 1, 0, 0, 1, 4095, 0, 2, 0, 1, 1, 4095, 161, 1391, 4079, 1, 203, 4083, 1, 2497, 0, 1, 0, 4095, 4095, 0, 1, 71, 0, 1856, 0, 0, 4074, 830, 2, 557, 0, 0, 1, 4086] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 2, 43, 1805, 0, 1, 0, 4, 0, 0, 1, 2, 0, 1, 4096, 0, 0, 4095, 0, 1, 4064, 0, 3901, 3, 0, 3183, 0, 4096, 4090, 0, 2, 0, 4094, 0, 3390, 1, 1557, 0, 0, 3771, 0, 0, 44, 0, 272, 0, 3277, 1514, 4079, 2, 0, 4095, 0, 1410, 4096, 0, 0, 2882, 2128, 2, 0, 4074, 2, 3630, 48, 0, 0, 1204, 4, 0, 4094, 0, 4031, 4011, 0, 1, 0, 0, 0, 3, 344, 0, 2620, 2, 0, 0, 216, 0, 0, 0, 2, 0, 4094, 0, 2, 0, 1, 2, 4094, 542, 779, 4087, 1, 184, 4089, 0, 2876, 0, 1, 0, 4095, 4071, 3, 0, 48, 0, 1845, 0, 0, 4079, 1622, 0, 171, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 66, 2331, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 4081, 12, 3949, 1, 0, 3051, 0, 4093, 4088, 3, 0, 1, 4095, 0, 1584, 0, 1043, 0, 0, 3991, 0, 0, 64, 0, 347, 0, 930, 1015, 4006, 0, 0, 4095, 0, 2437, 4096, 0, 0, 3897, 1685, 0, 0, 4022, 0, 4033, 161, 0, 0, 235, 0, 0, 4096, 0, 4032, 4090, 0, 0, 0, 4, 0, 0, 233, 7, 3351, 0, 0, 1, 593, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4093, 607, 201, 3957, 0, 1745, 4096, 0, 3658, 1, 0, 0, 4095, 4096, 0, 0, 218, 0, 1079, 1, 0, 4087, 2102, 0, 833, 0, 0, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 230, 3339, 0, 10, 0, 0, 0, 1, 14, 0, 0, 0, 4094, 4096, 4095, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 1, 3997, 0, 0, 2382, 0, 1593, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3725, 210, 970, 0, 0, 4094, 0, 0, 139, 0, 606, 0, 2251, 569, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 581, 4065, 0, 0, 0, 0, 2251, 0, 0, 0, 3743, 546, 0, 0, 3899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3486, 0, 0, 0, 115, 0, 0, 4019, 0, 4080, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 4, 4062, 0, 0, 0, 52, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2611, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3532, 1146, 156, 4096, 0, 194, 4096, 0, 2053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1674, 0, 0, 3264, 3542, 0, 449, 0, 0, 0, 4091, 227, 77, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17364, 16042, 16997, 15085, 15796, 17080, 15273, 17435] avg=16384.0000 max=17435 max_rank=7 min=15085 min_rank=3 imbalance_ratio=1.064148 max_load_reduction=113637 max_load_reduction_pct=0.866982 moved_to_echo=65584 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 20, 2823, 0, 5, 0, 0, 7, 1, 5, 0, 0, 3, 4055, 4096, 4094, 1483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 3, 5, 4076, 7, 4089, 10, 1, 3579, 0, 1575, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 2494, 122, 1128, 0, 0, 3900, 2, 0, 110, 0, 986, 0, 3299, 572, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 689, 4063, 0, 0, 0, 12, 1866, 0, 7, 0, 3708, 1933, 9, 2, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3257, 55, 2, 0, 716, 48, 0, 4016, 0, 3885, 4096, 3, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 338, 17, 3052, 0, 0, 0, 346, 10, 0, 0, 0, 5, 4082, 34, 1, 4088, 2606, 1930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 1, 3516, 626, 294, 4053, 0, 440, 4095, 0, 2783, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 114, 0, 1701, 6, 0, 4078, 667, 1, 107, 6, 0, 0, 4079, 232, 80, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16592, 17448, 17045, 16358, 16086, 16542, 15817, 15184] avg=16384.0000 max=17448 max_rank=1 min=15184 min_rank=7 imbalance_ratio=1.064941 max_load_reduction=113624 max_load_reduction_pct=0.866882 moved_to_echo=63629 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 2, 43, 1805, 0, 1, 0, 4, 0, 0, 1, 2, 0, 1, 4096, 4095, 4095, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4064, 0, 3901, 3, 0, 3183, 0, 1593, 4090, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3390, 1, 1557, 0, 0, 3771, 0, 0, 44, 0, 272, 0, 3277, 569, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1514, 4064, 2, 0, 0, 0, 1410, 0, 0, 0, 2882, 2128, 2, 0, 4074, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3630, 48, 0, 0, 1204, 4, 0, 4017, 0, 4031, 4011, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 344, 0, 2620, 2, 0, 0, 216, 0, 0, 0, 2, 0, 4094, 0, 2, 4096, 2611, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3530, 542, 779, 4087, 1, 184, 4089, 0, 2876, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 3, 0, 48, 0, 1845, 0, 0, 4079, 1622, 0, 171, 0, 0, 0, 4095, 227, 77, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15630, 16837, 17312, 16078, 16946, 15924, 16092, 16253] avg=16384.0000 max=17312 max_rank=2 min=15630 min_rank=0 imbalance_ratio=1.056641 max_load_reduction=113760 max_load_reduction_pct=0.867920 moved_to_echo=65215 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 100, 2398, 0, 2, 0, 0, 0, 0, 1, 1, 0, 0, 4095, 4095, 4095, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 4060, 0, 3951, 0, 1, 2188, 0, 1591, 4086, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2433, 29, 1880, 2, 1, 4034, 1, 0, 238, 1, 341, 0, 2871, 568, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 4071, 0, 1, 0, 0, 1454, 0, 0, 1, 3146, 1571, 0, 1, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3819, 21, 0, 1, 948, 23, 1, 4018, 1, 3622, 4094, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 1, 2480, 0, 1, 0, 1293, 0, 1, 1, 0, 1, 4095, 0, 0, 4094, 2610, 1931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 3531, 1258, 317, 4037, 1, 189, 4090, 1, 3004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 119, 0, 2044, 1, 0, 4093, 2130, 0, 170, 0, 0, 1, 4094, 227, 77, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16273, 15891, 16831, 15335, 16551, 16693, 16431, 17067] avg=16384.0000 max=17067 max_rank=7 min=15335 min_rank=3 imbalance_ratio=1.041687 max_load_reduction=114005 max_load_reduction_pct=0.869789 moved_to_echo=66742 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 50, 1334, 0, 0, 0, 0, 0, 6, 1, 0, 0, 0, 4081, 4096, 4096, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 14, 3408, 2, 0, 2774, 0, 1591, 4056, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2330, 2, 306, 0, 0, 3718, 0, 0, 225, 0, 530, 0, 3173, 569, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2919, 4075, 0, 0, 0, 0, 2683, 0, 0, 0, 3935, 1491, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3409, 73, 0, 0, 1824, 1, 0, 4004, 0, 3558, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 138, 2, 1088, 0, 0, 0, 1019, 0, 0, 0, 0, 0, 4096, 45, 0, 4096, 2611, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3531, 225, 1619, 3985, 0, 1093, 4095, 0, 2569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 66, 0, 1266, 2, 0, 4091, 1848, 0, 753, 0, 0, 0, 4096, 227, 77, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15149, 15883, 15286, 19197, 16875, 15028, 17117, 16537] avg=16384.0000 max=19197 max_rank=3 min=15028 min_rank=5 imbalance_ratio=1.171692 max_load_reduction=111875 max_load_reduction_pct=0.853539 moved_to_echo=65557 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 304, 2812, 0, 0, 12, 0, 0, 5, 2, 0, 0, 0, 4096, 4096, 4096, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 4064, 36, 0, 963, 0, 1593, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3682, 0, 1174, 0, 0, 3791, 0, 0, 372, 0, 1436, 0, 3387, 569, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1508, 4056, 0, 0, 0, 0, 1500, 0, 0, 0, 3624, 1865, 0, 0, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3007, 15, 0, 0, 1620, 0, 0, 4019, 0, 3707, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 291, 0, 1538, 0, 0, 3, 761, 0, 0, 0, 0, 0, 4096, 0, 1, 4096, 2610, 1933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3532, 1549, 822, 4092, 0, 328, 4093, 0, 2182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 29, 0, 1858, 0, 0, 4079, 833, 0, 567, 0, 0, 0, 4096, 227, 77, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16909, 14777, 18844, 16344, 16394, 15329, 16598, 15877] avg=16384.0000 max=18844 max_rank=2 min=14777 min_rank=1 imbalance_ratio=1.150146 max_load_reduction=112228 max_load_reduction_pct=0.856232 moved_to_echo=64198 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 153, 3038, 0, 1, 0, 0, 0, 12, 8, 1, 0, 0, 4095, 4095, 4095, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4073, 18, 4083, 0, 1, 3522, 0, 1582, 4095, 41, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2776, 134, 1119, 1, 1, 3932, 0, 1, 153, 1, 77, 1, 2603, 565, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1594, 4058, 0, 1, 0, 0, 1095, 0, 0, 1, 3278, 1818, 0, 1, 3780, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3054, 9, 0, 1, 974, 1, 1, 4018, 0, 3706, 4084, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 158, 1, 3175, 1, 1, 1, 1495, 0, 1, 0, 0, 1, 4095, 0, 2, 4094, 2610, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3531, 161, 1391, 4079, 1, 203, 4083, 1, 2497, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 71, 0, 1856, 0, 0, 4074, 830, 2, 557, 0, 0, 1, 4086, 227, 77, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16984, 17417, 15796, 15627, 15850, 17556, 15950, 15892] avg=16384.0000 max=17556 max_rank=5 min=15627 min_rank=3 imbalance_ratio=1.071533 max_load_reduction=113516 max_load_reduction_pct=0.866058 moved_to_echo=65248 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 66, 2331, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4095, 1485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 12, 3949, 1, 0, 3051, 0, 1592, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1584, 0, 1043, 0, 0, 3991, 0, 0, 64, 0, 347, 0, 930, 569, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 3992, 0, 0, 0, 0, 2437, 0, 0, 0, 3897, 1685, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 161, 0, 0, 235, 0, 0, 4019, 0, 4032, 4090, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 7, 3351, 0, 0, 1, 593, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 2610, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3529, 607, 201, 3957, 0, 1745, 4096, 0, 3658, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 218, 0, 1079, 1, 0, 4087, 2102, 0, 833, 0, 0, 0, 4093, 227, 77, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16171, 16777, 12961, 17048, 16574, 16920, 17794, 16827] avg=16384.0000 max=17794 max_rank=6 min=12961 min_rank=2 imbalance_ratio=1.086060 max_load_reduction=113278 max_load_reduction_pct=0.864243 moved_to_echo=68115 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 0, 22528, 512, 9216, 128, 128, 31232, 128, 128, 1408, 128, 4608, 128, 21888, 4608, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 768, 0, 13440, 128, 0, 31872, 13696, 128, 3712, 128, 0, 128, 32768, 1920, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[10880, 32512, 128, 128, 0, 128, 14720, 0, 128, 128, 28288, 13056, 128, 128, 31872, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 1024, 19968, 0, 128, 128, 128, 128, 128, 128, 128, 0, 128, 32768, 32768, 32768, 11904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 128, 32384, 128, 31488, 128, 128, 21760, 0, 12800, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[27776, 384, 128, 128, 7680, 128, 128, 32256, 128, 30720, 32512, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 28288, 6144, 5632, 32512, 128, 4480, 32768, 128, 21632, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 1920, 128, 21376, 128, 128, 128, 5888, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 20992, 15488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 965, 0, 2071, 0, 0, 13, 0, 4096, 3258, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 172, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 22, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 4092, 1384, 0, 0, 3830, 0, 4096, 4095, 0, 0, 4087, 4067, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 7, 477, 3696, 4096, 0, 0, 3916, 3881, 0, 2188, 0, 507, 0, 0, 1713, 0, 0, 0, 13, 0, 314, 0, 0, 3875, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 519, 0, 4090, 0, 4096, 3, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 961, 0, 14, 0, 0, 14, 0, 4096, 3160, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2, 4, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 9, 0, 0, 0, 4096, 7, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 42, 0, 0, 3956, 0, 4096, 4094, 0, 0, 4060, 3843, 0, 0, 8, 4095, 0, 0, 4096, 30, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4, 1020, 2467, 4083, 0, 0, 4092, 3927, 0, 3282, 0, 3706, 0, 0, 2795, 0, 0, 0, 19, 0, 815, 0, 0, 1013, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 84, 4096, 0, 0, 0, 0, 0, 0, 1689, 0, 4096, 0, 4044, 8, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 993, 0, 247, 0, 0, 12, 0, 4084, 2226, 3, 0, 0, 0, 4085, 9, 0, 0, 0, 0, 13, 4078, 0, 4096, 4085, 0, 0, 0, 4091, 1, 0, 20, 10, 4096, 20, 0, 4095, 21, 0, 0, 0, 4096, 0, 9, 0, 0, 4096, 4079, 271, 0, 0, 3816, 5, 4090, 4051, 0, 0, 4044, 3896, 0, 0, 19, 4080, 0, 0, 4088, 44, 0, 0, 0, 4, 8, 4096, 0, 0, 4093, 1, 1935, 2546, 3886, 0, 0, 4028, 3994, 0, 3578, 0, 1960, 1, 0, 3148, 0, 0, 0, 45, 10, 522, 0, 0, 1835, 0, 4093, 0, 0, 0, 0, 7, 6, 0, 80, 4087, 0, 0, 12, 0, 0, 0, 1891, 0, 4083, 0, 4084, 36, 5, 10, 0, 19] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1800, 1, 346, 0, 0, 17, 1, 4093, 1581, 2, 1, 0, 0, 4094, 2, 1, 0, 1, 2, 8, 4094, 1, 4094, 4094, 0, 0, 2, 4095, 23, 0, 4, 0, 4095, 13, 0, 4094, 60, 0, 1, 0, 4095, 1, 9, 1, 0, 4090, 4096, 405, 0, 0, 4079, 1, 4094, 4095, 0, 1, 4011, 4001, 0, 0, 1, 4096, 0, 0, 4096, 1, 0, 1, 0, 0, 1, 4095, 0, 1, 4094, 0, 367, 3418, 4056, 2, 1, 4076, 4032, 0, 3003, 1, 2440, 0, 2, 2494, 0, 0, 0, 102, 1, 1103, 0, 1, 2980, 0, 4095, 0, 1, 0, 0, 0, 0, 1, 38, 4077, 1, 1, 1, 0, 0, 0, 605, 1, 4082, 0, 4084, 16, 0, 0, 1, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 550, 0, 231, 0, 0, 0, 0, 4096, 3028, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 8, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 5, 0, 1, 0, 4096, 27, 0, 4096, 49, 0, 0, 0, 4096, 0, 6, 0, 0, 4090, 4095, 34, 0, 0, 4092, 0, 4096, 3895, 0, 0, 4027, 4061, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1858, 3078, 3349, 0, 0, 4087, 3927, 0, 3068, 0, 3122, 0, 0, 1571, 0, 0, 0, 461, 0, 775, 0, 0, 1144, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 146, 4096, 0, 0, 0, 0, 0, 0, 2585, 0, 4060, 0, 4096, 10, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1562, 0, 18, 1, 1, 5, 3, 4092, 3354, 0, 1, 0, 0, 4095, 0, 1, 1, 0, 0, 6, 4095, 1, 4095, 4095, 0, 0, 0, 4095, 12, 0, 5, 0, 4095, 4, 0, 4095, 8, 1, 1, 0, 4095, 1, 6, 1, 0, 4089, 4094, 672, 1, 0, 4054, 0, 4094, 4092, 0, 1, 4085, 4082, 1, 1, 3, 4092, 0, 1, 4095, 14, 0, 1, 0, 0, 0, 4095, 0, 1, 4094, 4, 539, 2156, 4014, 0, 1, 3709, 4090, 0, 3308, 1, 1687, 0, 0, 2692, 1, 1, 0, 78, 0, 144, 0, 1, 2582, 0, 4085, 0, 1, 1, 1, 0, 0, 1, 3, 4095, 1, 1, 7, 1, 1, 1, 2179, 1, 4078, 0, 4095, 4, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1811, 0, 66, 1, 1, 31, 0, 4093, 1357, 0, 1, 2, 0, 4095, 0, 1, 0, 1, 0, 1, 4091, 1, 4095, 4095, 1, 0, 0, 4095, 16, 0, 5, 0, 4095, 0, 0, 4095, 0, 1, 1, 1, 4095, 1, 56, 1, 0, 4054, 4094, 38, 1, 0, 3958, 0, 4093, 4095, 0, 1, 4074, 4029, 1, 1, 0, 4093, 0, 1, 4094, 10, 0, 1, 0, 0, 0, 4095, 0, 1, 4094, 1, 826, 2357, 3976, 0, 1, 3790, 4084, 0, 3437, 1, 2345, 0, 0, 2157, 1, 1, 1, 132, 0, 1324, 0, 1, 2060, 0, 4080, 0, 1, 1, 0, 0, 0, 0, 72, 4095, 1, 1, 0, 1, 1, 0, 3177, 1, 4007, 0, 4095, 9, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 625, 0, 429, 0, 0, 6, 1, 4095, 2178, 1, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 20, 4095, 0, 4096, 4095, 1, 0, 0, 4096, 1, 0, 0, 1, 4096, 0, 0, 4095, 3, 0, 0, 0, 4096, 0, 5, 0, 0, 4094, 4092, 810, 0, 1, 4064, 1, 4096, 4093, 0, 0, 4089, 3892, 0, 0, 11, 4096, 0, 0, 4095, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 873, 3454, 4075, 0, 0, 4094, 3981, 0, 3195, 0, 1086, 0, 0, 3422, 0, 0, 0, 21, 1, 548, 0, 0, 2990, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 56, 4095, 0, 0, 1, 0, 0, 0, 1123, 0, 4089, 0, 4091, 3, 0, 12, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 965, 0, 2071, 0, 0, 13, 0, 4096, 3258, 0, 0, 0, 0, 4096, 3278, 664, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 4096, 0, 4060, 4096, 0, 0, 0, 4096, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1384, 0, 0, 3830, 0, 0, 3926, 0, 0, 4087, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 7, 477, 3696, 126, 36, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3916, 603, 0, 2188, 0, 507, 0, 0, 1713, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 3875, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 3428, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 0, 4090, 0, 4096, 3, 0, 0, 0, 1, 4096, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18484, 16542, 16337, 17294, 16667, 13036, 17517, 15195] avg=16384.0000 max=18484 max_rank=0 min=13036 min_rank=5 imbalance_ratio=1.128174 max_load_reduction=112588 max_load_reduction_pct=0.858978 moved_to_echo=62415 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 961, 0, 14, 0, 0, 14, 0, 4096, 3160, 0, 0, 0, 0, 4096, 3317, 665, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 4096, 0, 4060, 4096, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 7, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 3956, 0, 0, 3925, 0, 0, 4060, 3843, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 30, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4, 1020, 2467, 126, 36, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 4092, 610, 0, 3282, 0, 3706, 0, 0, 2795, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 815, 0, 0, 1013, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 84, 4096, 3431, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1689, 0, 4096, 0, 4044, 8, 0, 1, 0, 0, 4096, 2389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16366, 16363, 16357, 15834, 16005, 18587, 15237, 16323] avg=16384.0000 max=18587 max_rank=5 min=15237 min_rank=6 imbalance_ratio=1.134460 max_load_reduction=112485 max_load_reduction_pct=0.858192 moved_to_echo=66152 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1800, 1, 346, 0, 0, 17, 1, 4093, 1581, 2, 1, 0, 0, 4094, 3406, 665, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 1, 2, 8, 4094, 1, 4058, 4094, 0, 0, 2, 4095, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4061, 13, 0, 4094, 60, 0, 1, 0, 4095, 1, 9, 1, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 0, 0, 4079, 1, 0, 3926, 0, 1, 4011, 4001, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 1, 0, 0, 1, 4095, 0, 1, 4094, 0, 367, 3418, 126, 36, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 2, 1, 4076, 626, 0, 3003, 1, 2440, 0, 2, 2494, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1103, 0, 1, 2980, 0, 4095, 0, 1, 0, 0, 0, 0, 1, 38, 4077, 3431, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 605, 1, 4082, 0, 4084, 16, 0, 0, 1, 4, 4094, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16050, 16381, 16429, 16425, 16270, 16803, 17434, 15280] avg=16384.0000 max=17434 max_rank=6 min=15280 min_rank=7 imbalance_ratio=1.064087 max_load_reduction=113638 max_load_reduction_pct=0.866989 moved_to_echo=65787 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 550, 0, 231, 0, 0, 0, 0, 4096, 3028, 0, 0, 0, 0, 4096, 3317, 666, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 4060, 4096, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4062, 27, 0, 4096, 49, 0, 0, 0, 4096, 0, 6, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 4092, 0, 0, 3734, 0, 0, 4027, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 1858, 3078, 120, 36, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3349, 0, 0, 4087, 610, 0, 3068, 0, 3122, 0, 0, 1571, 0, 0, 0, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 775, 0, 0, 1144, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 146, 4096, 3429, 1707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2585, 0, 4060, 0, 4096, 10, 0, 0, 0, 0, 4096, 2389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16025, 16361, 16427, 15948, 17414, 16268, 15393, 17236] avg=16384.0000 max=17414 max_rank=4 min=15393 min_rank=6 imbalance_ratio=1.062866 max_load_reduction=113658 max_load_reduction_pct=0.867142 moved_to_echo=66311 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 993, 0, 247, 0, 0, 12, 0, 4084, 2226, 3, 0, 0, 0, 4085, 3376, 663, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 13, 4078, 0, 4053, 4085, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 10, 4061, 20, 0, 4095, 21, 0, 0, 0, 4096, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 0, 0, 3816, 5, 0, 3876, 0, 0, 4044, 3896, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 44, 0, 0, 0, 4, 8, 4096, 0, 0, 4093, 1, 1935, 2546, 131, 43, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3886, 0, 0, 4028, 618, 0, 3578, 0, 1960, 1, 0, 3148, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 522, 0, 0, 1835, 0, 4093, 0, 0, 0, 0, 7, 6, 0, 80, 4087, 3416, 1701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 1891, 0, 4083, 0, 4084, 36, 5, 10, 0, 19, 4090, 2379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15733, 16330, 16428, 15927, 17024, 17264, 15757, 16609] avg=16384.0000 max=17264 max_rank=5 min=15733 min_rank=0 imbalance_ratio=1.053711 max_load_reduction=113808 max_load_reduction_pct=0.868286 moved_to_echo=66654 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1562, 0, 18, 1, 1, 5, 3, 4092, 3354, 0, 1, 0, 0, 4095, 3455, 666, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 6, 4095, 1, 4059, 4095, 0, 0, 0, 4095, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4061, 4, 0, 4095, 8, 1, 1, 0, 4095, 1, 6, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 1, 0, 4054, 0, 0, 3923, 0, 1, 4085, 4082, 1, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 14, 0, 1, 0, 0, 0, 4095, 0, 1, 4094, 4, 539, 2156, 126, 36, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 0, 1, 3709, 635, 0, 3308, 1, 1687, 0, 0, 2692, 1, 1, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 1, 2582, 0, 4085, 0, 1, 1, 1, 0, 0, 1, 3, 4095, 3428, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 7, 1, 1, 1, 2179, 1, 4078, 0, 4095, 4, 0, 0, 0, 1, 4094, 2386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17296, 16365, 16367, 16823, 15196, 16127, 16048, 16850] avg=16384.0000 max=17296 max_rank=0 min=15196 min_rank=4 imbalance_ratio=1.055664 max_load_reduction=113776 max_load_reduction_pct=0.868042 moved_to_echo=64221 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1811, 0, 66, 1, 1, 31, 0, 4093, 1357, 0, 1, 2, 0, 4095, 3450, 666, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 4091, 1, 4059, 4095, 1, 0, 0, 4095, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4061, 0, 0, 4095, 0, 1, 1, 1, 4095, 1, 56, 1, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 1, 0, 3958, 0, 0, 3926, 0, 1, 4074, 4029, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 10, 0, 1, 0, 0, 0, 4095, 0, 1, 4094, 1, 826, 2357, 126, 36, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 0, 1, 3790, 634, 0, 3437, 1, 2345, 0, 0, 2157, 1, 1, 1, 132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1324, 0, 1, 2060, 0, 4080, 0, 1, 1, 0, 0, 0, 0, 72, 4095, 3428, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 3177, 1, 4007, 0, 4095, 9, 0, 0, 0, 1, 4093, 2387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15617, 16361, 16371, 16029, 15676, 16476, 16768, 17774] avg=16384.0000 max=17774 max_rank=7 min=15617 min_rank=0 imbalance_ratio=1.084839 max_load_reduction=113298 max_load_reduction_pct=0.864395 moved_to_echo=66694 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 625, 0, 429, 0, 0, 6, 1, 4095, 2178, 1, 0, 0, 0, 4096, 3363, 664, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 20, 4095, 0, 4060, 4095, 1, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4062, 0, 0, 4095, 3, 0, 0, 0, 4096, 0, 5, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 810, 0, 1, 4064, 1, 0, 3924, 0, 0, 4089, 3892, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 8, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 873, 3454, 126, 36, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 4094, 618, 0, 3195, 0, 1086, 0, 0, 3422, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 548, 0, 0, 2990, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 56, 4095, 3428, 1706, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1123, 0, 4089, 0, 4091, 3, 0, 12, 0, 0, 4096, 2390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15501, 16369, 16356, 16792, 16820, 16511, 16918, 15805] avg=16384.0000 max=16918 max_rank=6 min=15501 min_rank=0 imbalance_ratio=1.032593 max_load_reduction=114154 max_load_reduction_pct=0.870926 moved_to_echo=66054 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 5632, 0, 128, 18560, 0, 32768, 0, 128, 128, 128, 128, 128, 128, 512, 32768, 27520, 13696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[31616, 128, 128, 31872, 4992, 0, 25088, 128, 16896, 128, 128, 20096, 128, 128, 128, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32512, 128, 0, 32768, 256, 128, 128, 128, 32768, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 9344, 128, 3456, 128, 128, 128, 128, 32768, 20224, 128, 128, 128, 0, 32768, 27008, 5376, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 128, 13824, 128, 32640, 0, 32768, 128, 128, 128, 128, 128, 32768, 19200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 0, 128, 128, 32768, 0, 128, 32768, 128, 7936, 23296, 1024, 384, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 128, 256, 32768, 128, 32512, 32768, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 3712, 128, 128, 31872, 128, 0, 31232, 0, 128, 32512, 31872, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1658, 4096, 0, 0, 0, 4081, 4096, 0, 0, 4017, 3781, 0, 0, 0, 0, 8, 0, 0, 4, 4096, 0, 3954, 3275, 0, 0, 4096, 0, 0, 4095, 4073, 0, 0, 0, 0, 4096, 0, 305, 4096, 0, 2, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 75, 0, 0, 0, 0, 0, 2953, 3, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 4096, 4087, 0, 0, 0, 3762, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 7, 0, 3, 0, 0, 15, 0, 2, 2674, 4090, 0, 0, 3996, 0, 0, 2615, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 54, 4096, 0, 0, 0, 54, 0, 0, 0, 0, 0, 4096, 0, 3694, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 3203, 4095, 0, 0, 1, 4095, 4095, 0, 0, 3920, 3873, 1, 1, 1, 1, 0, 1, 0, 19, 4092, 0, 3898, 3898, 1, 1, 4093, 1, 0, 4094, 4092, 0, 1, 1, 0, 4095, 0, 834, 4094, 0, 3, 0, 0, 3, 1, 4095, 4095, 0, 1, 1, 4095, 155, 0, 0, 1, 1, 2, 820, 2, 0, 4095, 1, 0, 0, 7, 4095, 1, 0, 4095, 4081, 1, 1, 1, 4071, 0, 0, 0, 4094, 1, 4095, 4095, 0, 1, 69, 0, 5, 0, 0, 9, 0, 2, 1915, 4074, 1, 0, 4089, 0, 1, 3255, 1, 0, 1, 1, 4095, 6, 1, 0, 0, 8, 4095, 1, 0, 1, 74, 0, 0, 1, 0, 0, 4095, 0, 2756, 1, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 2610, 4094, 0, 0, 0, 4095, 4095, 0, 0, 3680, 3660, 1, 1, 3, 1, 3, 1, 1, 0, 4095, 0, 3814, 3815, 0, 1, 4092, 0, 0, 4094, 4081, 1, 1, 0, 1, 4095, 0, 221, 4094, 3, 0, 1, 0, 6, 1, 4095, 4095, 5, 0, 1, 4095, 18, 1, 0, 1, 0, 3, 1601, 0, 1, 4095, 1, 0, 0, 3, 4092, 1, 0, 4095, 4076, 1, 1, 1, 4051, 0, 0, 0, 4095, 0, 4095, 4094, 0, 0, 113, 0, 0, 0, 0, 4, 0, 4, 1725, 4065, 1, 1, 3992, 1, 0, 3477, 0, 1, 0, 1, 4095, 18, 1, 0, 0, 55, 4095, 1, 0, 0, 240, 0, 0, 1, 0, 1, 4096, 0, 3802, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1574, 4096, 0, 0, 1, 4096, 4096, 0, 0, 4041, 3787, 0, 12, 0, 0, 0, 0, 0, 16, 4096, 0, 4064, 4079, 0, 0, 4096, 0, 0, 4096, 4092, 0, 0, 0, 0, 4096, 0, 583, 4096, 1, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 4096, 126, 0, 0, 0, 0, 0, 1283, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4080, 0, 0, 0, 4014, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 8, 0, 0, 7, 0, 0, 2723, 4092, 0, 0, 4096, 0, 0, 3006, 0, 0, 0, 0, 4096, 39, 0, 0, 0, 33, 4096, 0, 0, 0, 264, 0, 0, 0, 0, 0, 4096, 0, 3130, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2761, 4083, 14, 0, 0, 4069, 4096, 0, 0, 3588, 3650, 0, 0, 0, 0, 46, 0, 0, 25, 4072, 0, 3838, 4019, 0, 0, 4081, 0, 0, 4096, 4091, 0, 0, 0, 0, 4081, 4, 227, 4096, 18, 0, 0, 0, 5, 0, 4096, 4096, 0, 0, 0, 4096, 804, 12, 0, 0, 0, 22, 1967, 39, 0, 4096, 0, 0, 0, 2, 4079, 0, 0, 4085, 4060, 0, 0, 0, 4028, 9, 0, 0, 4096, 0, 4076, 4091, 0, 0, 70, 0, 23, 0, 0, 29, 0, 15, 2036, 4043, 0, 0, 4096, 0, 0, 2363, 0, 13, 0, 0, 4096, 18, 0, 0, 0, 68, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4089, 0, 3276, 0, 18, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3045, 4096, 0, 0, 12, 4096, 4096, 0, 0, 3894, 4088, 0, 9, 3, 0, 0, 0, 0, 14, 4096, 0, 3651, 4058, 0, 0, 4095, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 72, 4096, 0, 39, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 152, 0, 0, 0, 0, 0, 721, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4090, 0, 0, 0, 4087, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 320, 0, 0, 0, 0, 1, 0, 4, 1825, 4091, 0, 0, 4092, 0, 0, 4091, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 159, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2538, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 2489, 4095, 0, 0, 2, 4095, 4095, 0, 0, 4067, 3584, 1, 0, 8, 1, 2, 1, 0, 11, 4094, 0, 4015, 4060, 1, 1, 4095, 0, 0, 4094, 4095, 0, 50, 1, 0, 4095, 0, 150, 4094, 0, 13, 2, 0, 8, 1, 4095, 4095, 1, 1, 1, 4095, 14, 0, 0, 1, 1, 6, 852, 0, 0, 4095, 1, 0, 0, 21, 4095, 1, 1, 4094, 4063, 1, 1, 1, 4078, 0, 0, 0, 4082, 1, 4095, 4095, 0, 0, 150, 0, 2, 0, 0, 6, 1, 8, 1878, 4080, 1, 0, 4092, 0, 1, 3099, 0, 0, 1, 1, 4095, 24, 1, 0, 0, 11, 4091, 1, 0, 1, 209, 0, 0, 1, 0, 0, 4095, 0, 4011, 1, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2689, 4092, 0, 0, 0, 4080, 4096, 0, 0, 3009, 3796, 0, 0, 0, 0, 62, 0, 0, 16, 4088, 0, 3921, 3921, 0, 0, 4094, 0, 0, 4096, 3923, 0, 0, 0, 0, 4096, 1, 317, 4096, 17, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 126, 0, 0, 0, 0, 2, 2831, 1, 0, 4096, 0, 0, 0, 3, 4093, 0, 0, 4095, 4069, 0, 0, 0, 3728, 1, 0, 0, 4096, 0, 4095, 4095, 0, 0, 94, 0, 8, 0, 0, 32, 0, 8, 1086, 4088, 0, 0, 4092, 0, 0, 3861, 0, 2, 0, 0, 4096, 18, 0, 0, 0, 74, 4095, 0, 0, 0, 151, 0, 0, 0, 0, 0, 4096, 0, 3241, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1658, 2234, 0, 0, 0, 4081, 0, 0, 0, 4017, 3781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4, 4096, 0, 3954, 3275, 0, 0, 4096, 0, 0, 193, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4073, 0, 0, 0, 0, 4096, 0, 305, 3767, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2273, 75, 0, 0, 0, 0, 0, 2953, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4087, 0, 0, 0, 3762, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2010, 0, 0, 7, 0, 3, 0, 0, 15, 0, 2, 2674, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 2615, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 54, 4096, 826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 54, 0, 0, 0, 0, 0, 4096, 0, 3694, 0, 0, 0, 1862, 1637, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15771, 15812, 16338, 17592, 16178, 16993, 15689, 16699] avg=16384.0000 max=17592 max_rank=3 min=15689 min_rank=6 imbalance_ratio=1.073730 max_load_reduction=113480 max_load_reduction_pct=0.865784 moved_to_echo=65559 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 2610, 2233, 0, 0, 0, 4095, 0, 0, 0, 3680, 3660, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 1, 3, 1, 1, 0, 4095, 0, 3814, 3815, 0, 1, 4092, 0, 0, 193, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4081, 1, 1, 0, 1, 4095, 0, 221, 3765, 3, 0, 1, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 5, 0, 1, 2272, 18, 1, 0, 1, 0, 3, 1601, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 4092, 1, 0, 4095, 4076, 1, 1, 1, 4051, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 2009, 0, 0, 113, 0, 0, 0, 0, 4, 0, 4, 1725, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3992, 1, 0, 3477, 0, 1, 0, 1, 4095, 18, 1, 0, 0, 55, 4095, 825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 240, 0, 0, 1, 0, 1, 4096, 0, 3802, 1, 0, 0, 1861, 1637, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16282, 16206, 16270, 16188, 16458, 16110, 16563, 16995] avg=16384.0000 max=16995 max_rank=7 min=16110 min_rank=5 imbalance_ratio=1.037292 max_load_reduction=114077 max_load_reduction_pct=0.870338 moved_to_echo=66126 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3045, 2234, 0, 0, 12, 4096, 0, 0, 0, 3894, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 3, 0, 0, 0, 0, 14, 4096, 0, 3651, 4058, 0, 0, 4095, 0, 0, 193, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 72, 3767, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2273, 152, 0, 0, 0, 0, 0, 721, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4090, 0, 0, 0, 4087, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2010, 0, 0, 320, 0, 0, 0, 0, 1, 0, 4, 1825, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4091, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 159, 4096, 826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2538, 0, 0, 0, 1862, 1637, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17369, 16305, 16166, 15434, 16505, 16443, 17361, 15489] avg=16384.0000 max=17369 max_rank=0 min=15434 min_rank=3 imbalance_ratio=1.060120 max_load_reduction=113703 max_load_reduction_pct=0.867485 moved_to_echo=65798 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1574, 2234, 0, 0, 1, 4096, 0, 0, 0, 4041, 3787, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 16, 4096, 0, 4064, 4079, 0, 0, 4096, 0, 0, 193, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 0, 4096, 0, 583, 3767, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2273, 126, 0, 0, 0, 0, 0, 1283, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4080, 0, 0, 0, 4014, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2010, 0, 0, 0, 0, 8, 0, 0, 7, 0, 0, 2723, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3006, 0, 0, 0, 0, 4096, 39, 0, 0, 0, 33, 4096, 826, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 264, 0, 0, 0, 0, 0, 4096, 0, 3130, 0, 0, 0, 1862, 1637, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15733, 16742, 16636, 15970, 16422, 17032, 16192, 16345] avg=16384.0000 max=17032 max_rank=5 min=15733 min_rank=0 imbalance_ratio=1.039551 max_load_reduction=114040 max_load_reduction_pct=0.870056 moved_to_echo=65991 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 3203, 2234, 0, 0, 1, 4095, 0, 0, 0, 3920, 3873, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 19, 4092, 0, 3898, 3898, 1, 1, 4093, 1, 0, 193, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4092, 0, 1, 1, 0, 4095, 0, 834, 3765, 0, 3, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 1, 1, 2272, 155, 0, 0, 1, 1, 2, 820, 2, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 4095, 1, 0, 4095, 4081, 1, 1, 1, 4071, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 4095, 2010, 0, 1, 69, 0, 5, 0, 0, 9, 0, 2, 1915, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4089, 0, 1, 3255, 1, 0, 1, 1, 4095, 6, 1, 0, 0, 8, 4095, 825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 74, 0, 0, 1, 0, 0, 4095, 0, 2756, 1, 0, 1, 1861, 1637, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17330, 16386, 16889, 15540, 16490, 16275, 16379, 15783] avg=16384.0000 max=17330 max_rank=0 min=15540 min_rank=3 imbalance_ratio=1.057739 max_load_reduction=113742 max_load_reduction_pct=0.867783 moved_to_echo=64927 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2761, 2224, 14, 0, 0, 4069, 0, 0, 0, 3588, 3650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 25, 4072, 0, 3838, 4019, 0, 0, 4081, 0, 0, 199, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 0, 4081, 4, 227, 3759, 18, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2261, 804, 12, 0, 0, 0, 22, 1967, 39, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4079, 0, 0, 4085, 4060, 0, 0, 0, 4028, 9, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4076, 2001, 0, 0, 70, 0, 23, 0, 0, 29, 0, 15, 2036, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2363, 0, 13, 0, 0, 4096, 18, 0, 0, 0, 68, 4096, 828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 0, 0, 0, 0, 4089, 0, 3276, 0, 18, 0, 1859, 1643, 1262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16306, 16472, 16281, 17393, 16401, 16389, 15578, 16252] avg=16384.0000 max=17393 max_rank=3 min=15578 min_rank=6 imbalance_ratio=1.061584 max_load_reduction=113679 max_load_reduction_pct=0.867302 moved_to_echo=64620 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 2489, 2234, 0, 0, 2, 4095, 0, 0, 0, 4067, 3584, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 2, 1, 0, 11, 4094, 0, 4015, 4060, 1, 1, 4095, 0, 0, 193, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 0, 50, 1, 0, 4095, 0, 150, 3765, 0, 13, 2, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 1, 1, 1, 2272, 14, 0, 0, 1, 1, 6, 852, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 21, 4095, 1, 1, 4094, 4063, 1, 1, 1, 4078, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 1, 4095, 2010, 0, 0, 150, 0, 2, 0, 0, 6, 1, 8, 1878, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4092, 0, 1, 3099, 0, 0, 1, 1, 4095, 24, 1, 0, 0, 11, 4095, 825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 1, 209, 0, 0, 1, 0, 0, 4095, 0, 4011, 1, 0, 1, 1861, 1637, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16475, 16668, 16274, 15434, 16493, 16313, 16246, 17169] avg=16384.0000 max=17169 max_rank=7 min=15434 min_rank=3 imbalance_ratio=1.047913 max_load_reduction=113903 max_load_reduction_pct=0.869011 moved_to_echo=66221 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2689, 2232, 0, 0, 0, 4080, 0, 0, 0, 3009, 3796, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 16, 4088, 0, 3921, 3921, 0, 0, 4094, 0, 0, 193, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3923, 0, 0, 0, 0, 4096, 1, 317, 3767, 17, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2273, 126, 0, 0, 0, 0, 2, 2831, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 4095, 4069, 0, 0, 0, 3728, 1, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 2010, 0, 0, 94, 0, 8, 0, 0, 32, 0, 8, 1086, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 3861, 0, 2, 0, 0, 4096, 18, 0, 0, 0, 74, 4096, 825, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 151, 0, 0, 0, 0, 0, 4096, 0, 3241, 0, 0, 0, 1860, 1637, 1260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15806, 16481, 16218, 17521, 16125, 15517, 17064, 16340] avg=16384.0000 max=17521 max_rank=3 min=15517 min_rank=5 imbalance_ratio=1.069397 max_load_reduction=113551 max_load_reduction_pct=0.866325 moved_to_echo=65046 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 32768, 128, 128, 128, 18176, 1536, 128, 0, 128, 128, 128, 13056, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 32640, 128, 128, 25856, 128, 128, 128, 128, 32768, 256, 128, 0, 0, 512, 32768, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 20096, 17920, 128, 0, 128, 32768, 0, 0, 0, 30336, 30336, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 32768, 16128, 0, 128, 896, 0, 128, 0, 0, 128, 128, 128, 15872, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 32640, 128, 128, 128, 128, 32768, 128, 2816, 30208, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 32768, 128, 128, 32768, 32640, 128, 128, 128, 31872, 128, 0, 0, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 32768, 0, 31232, 31232, 128, 128, 32768, 128, 0, 1664, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 0, 128, 1024, 0, 0, 128, 0, 128, 32768, 0, 26496, 128, 128, 128, 14976, 13184, 10112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[5, 0, 4090, 0, 0, 3328, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 4096, 0, 4096, 0, 4096, 4088, 0, 0, 0, 0, 3211, 4095, 0, 0, 0, 4096, 0, 79, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3901, 4096, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 4096, 0, 0, 4096, 2673, 0, 0, 1127, 4096, 4096, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 1, 0, 1754, 4094, 0, 0, 0, 0, 0, 329, 0, 0, 4091, 1, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[6, 0, 4091, 0, 0, 2812, 4093, 1, 0, 1, 4095, 10, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 4094, 4095, 4093, 1, 4095, 0, 4094, 3994, 0, 0, 0, 1, 1937, 4078, 0, 1, 3, 4095, 0, 0, 1, 3, 0, 0, 0, 52, 4095, 1, 1, 0, 4095, 0, 0, 16, 3501, 4095, 1, 3814, 4095, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 4094, 0, 0, 1, 1, 0, 4095, 1, 0, 538, 0, 1, 4095, 1, 1, 4095, 3475, 0, 1, 1438, 4095, 4094, 1, 190, 0, 4092, 0, 1, 4093, 1, 0, 33, 4, 2692, 4095, 1, 0, 3, 43, 1, 60, 0, 0, 4073, 3, 4095, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[6, 0, 4081, 0, 0, 3501, 4096, 0, 0, 0, 4096, 34, 0, 0, 0, 0, 0, 0, 23, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 4096, 4084, 0, 4096, 0, 4096, 3992, 0, 0, 0, 8, 2114, 4058, 0, 0, 19, 4096, 0, 0, 0, 28, 0, 0, 0, 121, 4090, 0, 0, 0, 4096, 0, 0, 0, 3714, 4096, 11, 3845, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 5, 66, 0, 0, 4096, 0, 0, 4096, 3008, 0, 0, 2363, 4091, 4078, 0, 76, 0, 4081, 0, 0, 4096, 0, 0, 34, 0, 1748, 4072, 0, 0, 1, 55, 0, 42, 0, 0, 4009, 30, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[19, 0, 4088, 0, 0, 3363, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4096, 4096, 0, 4096, 0, 4096, 4058, 0, 0, 0, 0, 2436, 4092, 0, 0, 0, 4096, 0, 0, 0, 19, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 0, 3311, 4096, 0, 4028, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 690, 0, 0, 4096, 0, 0, 4096, 2448, 0, 0, 1012, 4096, 4096, 0, 50, 0, 4096, 0, 0, 4096, 0, 0, 46, 4, 3119, 4092, 0, 0, 0, 20, 0, 2, 0, 0, 4083, 2, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 0, 2782, 4096, 0, 0, 0, 4096, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 4096, 4096, 0, 4096, 0, 4096, 4015, 0, 0, 0, 0, 3688, 4047, 0, 0, 3, 4096, 0, 1, 0, 37, 0, 0, 0, 258, 4096, 0, 0, 0, 4096, 0, 0, 36, 3797, 4096, 0, 3911, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 488, 0, 0, 4096, 0, 0, 4096, 2913, 0, 0, 876, 4096, 4096, 0, 92, 0, 4096, 0, 0, 4096, 0, 0, 33, 0, 1668, 4095, 0, 0, 0, 37, 0, 51, 0, 0, 4045, 2, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 1, 4087, 0, 0, 3210, 4092, 1, 0, 0, 4095, 5, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 4093, 4095, 4095, 1, 4095, 1, 4095, 4021, 0, 1, 0, 0, 3045, 4087, 0, 0, 11, 4095, 0, 26, 1, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 4095, 0, 0, 5, 3895, 4095, 0, 4089, 4094, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 4094, 0, 0, 1, 1, 1, 4095, 1, 0, 206, 1, 1, 4095, 1, 0, 4095, 2391, 0, 1, 2465, 4095, 4094, 1, 11, 0, 4095, 0, 1, 4094, 1, 0, 2, 1, 1192, 4095, 0, 0, 0, 21, 0, 3, 0, 0, 4090, 0, 4095, 1, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[3, 1, 4086, 0, 1, 3889, 4089, 0, 0, 1, 4095, 2, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 4095, 4095, 4095, 0, 4095, 1, 4095, 4019, 0, 0, 0, 0, 1391, 4047, 0, 0, 8, 4095, 0, 17, 1, 0, 0, 1, 0, 10, 4095, 1, 1, 0, 4095, 0, 0, 37, 3580, 4095, 0, 4080, 4093, 0, 0, 0, 1, 0, 3, 0, 1, 0, 1, 1, 4094, 0, 0, 1, 1, 1, 4095, 1, 0, 348, 1, 1, 4094, 1, 0, 4095, 3084, 0, 1, 1752, 4095, 4093, 1, 50, 0, 4095, 0, 1, 4094, 1, 0, 4, 2, 2243, 4095, 0, 0, 0, 74, 0, 46, 0, 0, 4093, 0, 4095, 1, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[6, 0, 4091, 0, 0, 3077, 4096, 0, 0, 0, 4096, 7, 0, 0, 0, 1, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 4096, 4095, 0, 4096, 0, 4096, 4059, 0, 0, 0, 1, 2519, 4064, 0, 0, 1, 4096, 0, 1, 0, 77, 0, 0, 0, 69, 4096, 0, 0, 0, 4096, 0, 0, 0, 3939, 4096, 0, 4006, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4095, 0, 0, 91, 0, 0, 4096, 0, 0, 4096, 3339, 0, 0, 897, 4096, 4095, 0, 28, 0, 4095, 0, 0, 4096, 0, 0, 76, 3, 1992, 4078, 0, 0, 0, 27, 0, 462, 0, 0, 4061, 11, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 4090, 0, 0, 3328, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 4096, 4096, 2180, 1931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4088, 0, 0, 0, 0, 3211, 4095, 0, 0, 0, 1638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3901, 4096, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 4096, 0, 0, 4096, 2673, 624, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1127, 4096, 4096, 0, 1, 0, 4096, 0, 0, 2520, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1754, 4094, 0, 0, 0, 0, 0, 329, 0, 0, 4091, 1, 1916, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16456, 16394, 17128, 16553, 16487, 15836, 15937, 16281] avg=16384.0000 max=17128 max_rank=2 min=15836 min_rank=5 imbalance_ratio=1.045410 max_load_reduction=113944 max_load_reduction_pct=0.869324 moved_to_echo=64541 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 4091, 0, 0, 2812, 4093, 1, 0, 1, 4095, 10, 0, 0, 1, 0, 840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 4094, 4095, 4093, 2179, 1931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 3994, 0, 0, 0, 1, 1937, 4078, 0, 1, 3, 1638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 52, 4095, 1, 1, 0, 4095, 0, 0, 16, 3501, 4095, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 3814, 4095, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 4094, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4095, 1, 0, 538, 0, 1, 4095, 1, 1, 4095, 3475, 623, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1438, 4095, 4094, 1, 190, 0, 4092, 0, 1, 2519, 1, 0, 33, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2692, 4095, 1, 0, 3, 43, 1, 60, 0, 0, 4073, 3, 1915, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15950, 16398, 15747, 16138, 16213, 17176, 16469, 16981] avg=16384.0000 max=17176 max_rank=5 min=15747 min_rank=2 imbalance_ratio=1.048340 max_load_reduction=113896 max_load_reduction_pct=0.868958 moved_to_echo=66839 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[19, 0, 4088, 0, 0, 3363, 4096, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4096, 4096, 2180, 1931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4058, 0, 0, 0, 0, 2436, 4092, 0, 0, 0, 1638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 0, 3311, 4096, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4028, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 690, 0, 0, 4096, 0, 0, 4096, 2448, 624, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1012, 4096, 4096, 0, 50, 0, 4096, 0, 0, 2520, 0, 0, 46, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3119, 4092, 0, 0, 0, 20, 0, 2, 0, 0, 4083, 2, 1916, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16506, 16369, 16320, 15901, 16427, 16299, 15920, 17330] avg=16384.0000 max=17330 max_rank=7 min=15901 min_rank=3 imbalance_ratio=1.057739 max_load_reduction=113742 max_load_reduction_pct=0.867783 moved_to_echo=65976 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 4081, 0, 0, 3501, 4096, 0, 0, 0, 4096, 34, 0, 0, 0, 0, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 4096, 4084, 2185, 1936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3992, 0, 0, 0, 8, 2114, 4058, 0, 0, 19, 1626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 121, 4090, 0, 0, 0, 4096, 0, 0, 0, 3714, 4096, 285, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 11, 3845, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 66, 0, 0, 4096, 0, 0, 4096, 3008, 627, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2363, 4091, 4078, 0, 76, 0, 4081, 0, 0, 2506, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1748, 4072, 0, 0, 1, 55, 0, 42, 0, 0, 4009, 30, 1904, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16659, 16373, 15913, 16430, 16261, 16250, 17229, 15957] avg=16384.0000 max=17229 max_rank=6 min=15913 min_rank=2 imbalance_ratio=1.051575 max_load_reduction=113843 max_load_reduction_pct=0.868553 moved_to_echo=65697 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4094, 0, 0, 2782, 4096, 0, 0, 0, 4096, 32, 0, 0, 0, 0, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 4096, 4096, 2180, 1931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4015, 0, 0, 0, 0, 3688, 4047, 0, 0, 3, 1638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 37, 0, 0, 0, 258, 4096, 0, 0, 0, 4096, 0, 0, 36, 3797, 4096, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3911, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 488, 0, 0, 4096, 0, 0, 4096, 2913, 624, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 876, 4096, 4096, 0, 92, 0, 4096, 0, 0, 2520, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1668, 4095, 0, 0, 0, 37, 0, 51, 0, 0, 4045, 2, 1916, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15941, 16357, 17487, 16695, 16311, 16562, 15809, 15910] avg=16384.0000 max=17487 max_rank=2 min=15809 min_rank=6 imbalance_ratio=1.067322 max_load_reduction=113585 max_load_reduction_pct=0.866585 moved_to_echo=64592 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 1, 4086, 0, 1, 3889, 4089, 0, 0, 1, 4095, 2, 1, 0, 1, 1, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 4095, 4095, 4095, 2179, 1931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4019, 0, 0, 0, 0, 1391, 4047, 0, 0, 8, 1638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 0, 0, 1, 0, 10, 4095, 1, 1, 0, 4095, 0, 0, 37, 3580, 4095, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4080, 4093, 0, 0, 0, 1, 0, 3, 0, 1, 0, 1, 1, 4094, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4095, 1, 0, 348, 1, 1, 4094, 1, 0, 4095, 3084, 624, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1752, 4095, 4093, 1, 50, 0, 4095, 0, 1, 2518, 1, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2243, 4095, 0, 0, 0, 74, 0, 46, 0, 0, 4093, 0, 1915, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17011, 16402, 15199, 16211, 16480, 16596, 16613, 16560] avg=16384.0000 max=17011 max_rank=0 min=15199 min_rank=2 imbalance_ratio=1.038269 max_load_reduction=114061 max_load_reduction_pct=0.870216 moved_to_echo=66249 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4087, 0, 0, 3210, 4092, 1, 0, 0, 4095, 5, 1, 0, 1, 0, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 4093, 4095, 4095, 2179, 1931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 4021, 0, 1, 0, 0, 3045, 4087, 0, 0, 11, 1638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 1, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 4095, 0, 0, 5, 3895, 4095, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4089, 4094, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 4094, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4095, 1, 0, 206, 1, 1, 4095, 1, 0, 4095, 2391, 624, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2465, 4095, 4094, 1, 11, 0, 4095, 0, 1, 2518, 1, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 4095, 0, 0, 0, 21, 0, 3, 0, 0, 4090, 0, 1915, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16334, 16401, 16900, 16492, 16488, 15762, 17285, 15410] avg=16384.0000 max=17285 max_rank=6 min=15410 min_rank=7 imbalance_ratio=1.054993 max_load_reduction=113787 max_load_reduction_pct=0.868126 moved_to_echo=64945 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 4091, 0, 0, 3077, 4096, 0, 0, 0, 4096, 7, 0, 0, 0, 1, 841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 4096, 4095, 2180, 1931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4059, 0, 0, 0, 1, 2519, 4064, 0, 0, 1, 1638, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 77, 0, 0, 0, 69, 4096, 0, 0, 0, 4096, 0, 0, 0, 3939, 4096, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4006, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 91, 0, 0, 4096, 0, 0, 4096, 3339, 624, 249, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 897, 4096, 4095, 0, 28, 0, 4095, 0, 0, 2520, 0, 0, 76, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1992, 4078, 0, 0, 0, 27, 0, 462, 0, 0, 4061, 11, 1916, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16215, 16378, 16378, 16652, 16405, 16591, 15810, 16643] avg=16384.0000 max=16652 max_rank=3 min=15810 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=114420 max_load_reduction_pct=0.872955 moved_to_echo=65449 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 0, 128, 25984, 32768, 128, 0, 128, 32768, 128, 128, 0, 128, 128, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 31872, 32768, 128, 128, 0, 128, 0, 128, 0, 128, 0, 128, 128, 32768, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 12032, 32768, 32768, 128, 512, 0, 32768, 0, 128, 20224, 128, 0, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 128, 128, 2432, 128, 128, 32768, 128, 128, 32768, 23424, 5120, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 128, 128, 128, 128, 0, 0, 128, 128, 128, 32640, 32768, 32768, 17536, 15488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[16512, 32768, 128, 0, 128, 384, 128, 1024, 0, 0, 32640, 128, 15360, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 32768, 32256, 0, 128, 0, 128, 20352, 32640, 0, 128, 128, 13184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 256, 0, 128, 0, 640, 32768, 128, 128, 0, 32768, 0, 0, 128, 29696, 32768, 2304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3778, 0, 0, 162, 0, 0, 0, 0, 0, 4080, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 3538, 4092, 2, 0, 0, 0, 0, 7, 4096, 3929, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 2026, 0, 0, 0, 4091, 0, 4061, 4096, 507, 0, 67, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 29, 3876, 4096, 2497, 0, 0, 0, 1, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 110, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 1, 0, 4094, 4094, 3853, 1, 0, 52, 0, 1, 0, 0, 1, 4075, 1, 4095, 1, 1, 1, 0, 1, 1, 4095, 4095, 0, 1, 0, 1, 4094, 4094, 4095, 0, 1, 1, 0, 1, 2735, 4080, 0, 0, 0, 0, 0, 8, 4094, 3900, 64, 3, 0, 0, 1, 4095, 0, 1, 1, 1002, 0, 1, 0, 4068, 0, 4094, 4095, 976, 1, 36, 0, 1, 0, 0, 1, 0, 1, 4094, 4088, 3, 4078, 4094, 3766, 0, 0, 0, 0, 4092, 1, 4095, 4095, 1, 0, 1, 4095, 81, 0, 1, 0, 1, 0, 0, 0, 4091, 0, 4095, 4093, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 4095, 0, 0, 0, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3545, 0, 0, 295, 0, 0, 0, 0, 1, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 3000, 4094, 0, 0, 0, 0, 0, 4, 4096, 2951, 274, 0, 0, 0, 0, 4096, 0, 0, 0, 2040, 0, 0, 0, 4044, 0, 4089, 4096, 407, 0, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 1, 4033, 4096, 3963, 0, 0, 0, 0, 4089, 0, 4096, 4096, 0, 0, 0, 4096, 107, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 1, 1, 4095, 4095, 3961, 1, 0, 65, 1, 0, 0, 0, 0, 4095, 1, 4064, 1, 0, 1, 0, 0, 1, 4093, 4095, 1, 0, 0, 0, 4095, 4095, 4095, 0, 0, 0, 0, 1, 2986, 4093, 0, 0, 0, 0, 0, 5, 4095, 3651, 81, 0, 1, 0, 1, 4095, 0, 1, 1, 2697, 0, 1, 0, 4075, 0, 4092, 4095, 316, 1, 54, 1, 1, 0, 1, 1, 0, 1, 4095, 4091, 8, 4009, 4085, 2642, 1, 0, 0, 4, 4094, 1, 4095, 4095, 1, 0, 0, 4095, 62, 0, 1, 0, 1, 1, 1, 0, 4094, 0, 4095, 4095, 0, 0, 1, 0, 0, 2, 0, 1, 1, 0, 0, 2, 1, 0, 0, 4094, 1, 0, 1, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 1, 1, 4095, 4095, 3839, 1, 0, 5, 1, 0, 1, 0, 0, 4094, 1, 4065, 1, 0, 1, 0, 0, 1, 4092, 4095, 1, 0, 0, 0, 4095, 4095, 4095, 0, 0, 0, 0, 1, 3609, 4094, 0, 0, 0, 0, 0, 1, 4095, 3578, 25, 0, 1, 0, 1, 4095, 0, 1, 1, 931, 0, 1, 0, 4075, 0, 4087, 4095, 569, 1, 87, 1, 1, 0, 1, 1, 0, 1, 4095, 4095, 3, 3906, 4095, 3883, 0, 0, 0, 0, 4095, 1, 4095, 4095, 1, 0, 0, 4095, 84, 0, 1, 0, 1, 1, 1, 0, 4095, 0, 4095, 4095, 0, 0, 6, 0, 0, 6, 0, 5, 1, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 1, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3975, 0, 0, 32, 0, 63, 0, 0, 0, 4071, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 3145, 4095, 0, 0, 0, 0, 0, 0, 4096, 3811, 250, 0, 0, 0, 0, 4096, 0, 0, 0, 515, 0, 0, 0, 4080, 0, 4096, 4096, 718, 0, 9, 0, 0, 0, 0, 0, 0, 0, 4095, 4093, 0, 4023, 4090, 3984, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4096, 4096, 0, 0, 12, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4090, 3932, 0, 0, 170, 0, 37, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 17, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 1, 19, 5, 0, 2647, 4075, 0, 0, 0, 0, 0, 3, 4096, 3957, 123, 0, 0, 0, 0, 4085, 0, 0, 0, 729, 0, 0, 0, 4037, 0, 4090, 4096, 952, 0, 26, 0, 0, 0, 0, 0, 0, 0, 4077, 4071, 3, 3826, 4096, 3850, 0, 0, 0, 1, 4088, 0, 4096, 4096, 0, 0, 0, 4096, 404, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 2, 0, 0, 0, 4095, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3828, 0, 0, 224, 0, 43, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 2046, 4094, 0, 0, 0, 0, 0, 8, 4096, 3958, 108, 0, 0, 0, 0, 4095, 0, 0, 0, 2614, 0, 0, 0, 4023, 0, 4080, 4096, 535, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 3388, 4086, 3815, 0, 0, 0, 0, 4061, 0, 4096, 4096, 0, 0, 0, 4096, 49, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 4096, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3778, 0, 0, 162, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1132, 0, 0, 0, 0, 0, 3538, 4092, 2, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2825, 3929, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 2026, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 3448, 507, 0, 67, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3876, 4096, 2497, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 647, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 3, 0, 0, 2323, 1271, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3449, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16342, 16381, 16963, 16969, 16304, 15323, 16406, 16384] avg=16384.0000 max=16969 max_rank=3 min=15323 min_rank=5 imbalance_ratio=1.035706 max_load_reduction=114103 max_load_reduction_pct=0.870537 moved_to_echo=64417 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 1, 0, 4094, 4094, 3853, 1, 0, 52, 0, 1, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4075, 1, 4095, 1, 1, 1, 0, 1, 1, 4095, 4095, 0, 1, 0, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 1133, 0, 1, 1, 0, 1, 2735, 4080, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2824, 3900, 64, 3, 0, 0, 1, 4095, 0, 1, 1, 1002, 0, 1, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3448, 976, 1, 36, 0, 1, 0, 0, 1, 0, 1, 4094, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 4094, 3766, 0, 0, 0, 0, 4092, 1, 0, 0, 1, 0, 1, 647, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4091, 0, 4095, 4093, 0, 0, 0, 1, 0, 2322, 1270, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 4095, 0, 0, 0, 1, 1, 4095, 4095, 3448, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16305, 16382, 16147, 15960, 16743, 16761, 16395, 16379] avg=16384.0000 max=16761 max_rank=5 min=15960 min_rank=3 imbalance_ratio=1.023010 max_load_reduction=114311 max_load_reduction_pct=0.872124 moved_to_echo=66278 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3545, 0, 0, 295, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1132, 0, 0, 0, 0, 0, 3000, 4094, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2825, 2951, 274, 0, 0, 0, 0, 4096, 0, 0, 0, 2040, 0, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 3448, 407, 0, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 4096, 3963, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 647, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 4096, 4096, 0, 0, 0, 0, 0, 2323, 1271, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3449, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16242, 16393, 16422, 16230, 16153, 16935, 16319, 16378] avg=16384.0000 max=16935 max_rank=5 min=16153 min_rank=4 imbalance_ratio=1.033630 max_load_reduction=114137 max_load_reduction_pct=0.870796 moved_to_echo=65785 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 1, 1, 4095, 4095, 3961, 1, 0, 65, 1, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4064, 1, 0, 1, 0, 0, 1, 4093, 4095, 1, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 1133, 0, 0, 0, 0, 1, 2986, 4093, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2824, 3651, 81, 0, 1, 0, 1, 4095, 0, 1, 1, 2697, 0, 1, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3448, 316, 1, 54, 1, 1, 0, 1, 1, 0, 1, 4095, 4091, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 4085, 2642, 1, 0, 0, 4, 4094, 1, 0, 0, 1, 0, 0, 647, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 4094, 0, 4095, 4095, 0, 0, 1, 0, 0, 2322, 1271, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 0, 0, 2, 1, 0, 0, 4094, 1, 0, 1, 1, 1, 4095, 4095, 3448, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16429, 16365, 16408, 17428, 16110, 15546, 16403, 16383] avg=16384.0000 max=17428 max_rank=3 min=15546 min_rank=5 imbalance_ratio=1.063721 max_load_reduction=113644 max_load_reduction_pct=0.867035 moved_to_echo=64442 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3975, 0, 0, 32, 0, 63, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1132, 0, 0, 0, 0, 0, 3145, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2825, 3811, 250, 0, 0, 0, 0, 4096, 0, 0, 0, 515, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3448, 718, 0, 9, 0, 0, 0, 0, 0, 0, 0, 4095, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4023, 4090, 3984, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 647, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4096, 4096, 0, 0, 12, 0, 0, 2323, 1271, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3449, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16472, 16372, 16564, 15577, 16459, 16842, 16404, 16382] avg=16384.0000 max=16842 max_rank=5 min=15577 min_rank=3 imbalance_ratio=1.027954 max_load_reduction=114230 max_load_reduction_pct=0.871506 moved_to_echo=66087 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4090, 3932, 0, 0, 170, 0, 37, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 17, 0, 4096, 4096, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1125, 0, 1, 19, 5, 0, 2647, 4075, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2821, 3957, 123, 0, 0, 0, 0, 4085, 0, 0, 0, 729, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3439, 952, 0, 26, 0, 0, 0, 0, 0, 0, 0, 4077, 4071, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3826, 4096, 3850, 0, 0, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 642, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 4096, 4096, 3, 0, 0, 0, 0, 2327, 1275, 524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 2, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 3454, 644, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16537, 16418, 16067, 15752, 16658, 16907, 16334, 16399] avg=16384.0000 max=16907 max_rank=5 min=15752 min_rank=3 imbalance_ratio=1.031921 max_load_reduction=114165 max_load_reduction_pct=0.871010 moved_to_echo=66298 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 1, 1, 4095, 4095, 3839, 1, 0, 5, 1, 0, 1, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 4065, 1, 0, 1, 0, 0, 1, 4092, 4095, 1, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 1133, 0, 0, 0, 0, 1, 3609, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2824, 3578, 25, 0, 1, 0, 1, 4095, 0, 1, 1, 931, 0, 1, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 3448, 569, 1, 87, 1, 1, 0, 1, 1, 0, 1, 4095, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3906, 4095, 3883, 0, 0, 0, 0, 4095, 1, 0, 0, 1, 0, 0, 647, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 4095, 0, 4095, 4095, 0, 0, 6, 0, 0, 2322, 1271, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 5, 1, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 1, 1, 1, 4095, 4095, 3448, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16248, 16364, 17028, 15533, 16390, 16712, 16409, 16388] avg=16384.0000 max=17028 max_rank=2 min=15533 min_rank=3 imbalance_ratio=1.039307 max_load_reduction=114044 max_load_reduction_pct=0.870087 moved_to_echo=65899 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3828, 0, 0, 224, 0, 43, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1132, 0, 0, 1, 0, 0, 2046, 4094, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2825, 3958, 108, 0, 0, 0, 0, 4095, 0, 0, 0, 2614, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 3448, 535, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3388, 4086, 3815, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 0, 647, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 4096, 0, 0, 3, 0, 0, 2323, 1271, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 3449, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16497, 16397, 15473, 17623, 16255, 16046, 16402, 16379] avg=16384.0000 max=17623 max_rank=3 min=15473 min_rank=2 imbalance_ratio=1.075623 max_load_reduction=113449 max_load_reduction_pct=0.865547 moved_to_echo=65082 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 32768, 128, 128, 128, 0, 128, 128, 32768, 32768, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 128, 0, 128, 128, 32768, 32768, 30720, 128, 0, 1024, 128, 256, 128, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 27648, 4992, 128, 384, 128, 128, 0, 128, 128, 0, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[22656, 29824, 1024, 128, 128, 0, 128, 32768, 0, 128, 128, 12672, 0, 128, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[31232, 32768, 28416, 128, 0, 0, 128, 32768, 128, 0, 0, 128, 0, 128, 5248, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 128, 0, 32640, 0, 32768, 32768, 128, 0, 128, 128, 0, 18688, 10240, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 32768, 9088, 0, 128, 128, 128, 128, 23808, 32768, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 128, 0, 128, 128, 128, 0, 0, 32768, 128, 0, 128, 128, 128, 32768, 32768, 27648, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 3822, 0, 4096, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4094, 4096, 0, 4096, 4096, 0, 175, 4096, 1396, 0, 0, 4096, 0, 2, 4096, 4096, 0, 0, 0, 0, 0, 4095, 37, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 0, 19, 629, 0, 0, 0, 1114, 0, 0, 0, 4096, 0, 4096, 4096, 0, 7, 4096, 4096, 4096, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4057, 4056, 0, 0, 0, 0, 4096, 0, 0, 38, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[1, 2693, 0, 4095, 1, 704, 1, 0, 5, 0, 1, 0, 0, 1, 0, 2588, 2, 1, 1, 4095, 1, 1, 4095, 1, 1, 1, 1, 4095, 0, 8, 1, 0, 4094, 0, 0, 3923, 4093, 1, 4093, 4095, 1, 166, 4093, 900, 0, 0, 4094, 0, 23, 4067, 4095, 1, 0, 0, 1, 0, 4095, 247, 0, 0, 0, 4093, 0, 1, 4095, 1, 0, 1, 0, 84, 93, 0, 0, 1, 1244, 1, 1, 0, 4095, 0, 4095, 4057, 0, 0, 4079, 4095, 4094, 4095, 26, 0, 0, 0, 1, 1, 0, 0, 4095, 4081, 1, 1, 1, 4095, 0, 1, 3754, 3984, 0, 1, 0, 0, 4095, 0, 0, 133, 4095, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 2706, 0, 4096, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1450, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 3933, 4084, 0, 4096, 4095, 0, 67, 4085, 1032, 0, 0, 4096, 0, 34, 4078, 4096, 0, 0, 0, 0, 0, 4096, 154, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 0, 9, 966, 0, 0, 0, 1936, 0, 0, 0, 4096, 0, 4096, 4096, 0, 4, 4096, 4096, 4096, 4096, 461, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 3082, 4094, 0, 0, 0, 0, 4096, 0, 0, 358, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 3217, 0, 4096, 0, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1976, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4093, 4096, 0, 4096, 4096, 0, 125, 4095, 1099, 0, 2, 4096, 0, 7, 4049, 4096, 0, 0, 0, 0, 0, 4094, 18, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 103, 618, 0, 0, 0, 1333, 0, 0, 0, 4096, 0, 4096, 4096, 0, 6, 4096, 4096, 4096, 4096, 21, 0, 0, 0, 0, 0, 6, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 3398, 4053, 0, 0, 0, 0, 4096, 0, 0, 136, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[1, 3730, 0, 4095, 1, 100, 1, 0, 5, 0, 1, 0, 0, 1, 0, 1418, 0, 0, 1, 4095, 1, 1, 4094, 1, 1, 1, 0, 4095, 0, 0, 1, 0, 4093, 1, 0, 3932, 4091, 1, 4095, 4095, 1, 152, 4093, 971, 0, 0, 4094, 0, 9, 4083, 4095, 0, 0, 1, 1, 0, 4095, 229, 0, 0, 0, 4095, 0, 1, 4095, 1, 2, 0, 0, 89, 322, 0, 1, 0, 1185, 1, 1, 0, 4094, 0, 4095, 4035, 0, 0, 4095, 4095, 4095, 4094, 132, 1, 0, 0, 0, 1, 9, 0, 4095, 4095, 1, 1, 1, 4095, 0, 1, 4058, 4083, 1, 1, 0, 0, 4095, 0, 0, 130, 4095, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 3523, 0, 4096, 0, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2225, 0, 7, 0, 4096, 0, 0, 4096, 0, 0, 0, 12, 4096, 0, 15, 0, 0, 4052, 0, 0, 4094, 4078, 0, 4079, 4094, 0, 27, 4055, 869, 0, 21, 4084, 0, 50, 4056, 4096, 12, 0, 0, 0, 4, 4089, 96, 9, 0, 0, 4072, 0, 0, 4096, 0, 0, 0, 0, 3, 231, 1, 0, 0, 1036, 0, 0, 0, 4096, 0, 4096, 4096, 0, 67, 4096, 4096, 4083, 4084, 176, 0, 23, 0, 16, 0, 1, 0, 4094, 4078, 0, 0, 0, 4096, 0, 0, 3774, 4056, 0, 0, 0, 0, 4096, 0, 0, 69, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 3713, 0, 4095, 1, 606, 1, 0, 2, 0, 1, 0, 0, 1, 0, 1327, 0, 0, 1, 4095, 1, 1, 4094, 1, 1, 1, 0, 4095, 0, 0, 1, 0, 4044, 0, 0, 3991, 4093, 1, 4095, 4095, 1, 142, 4091, 772, 0, 0, 4095, 0, 14, 4057, 4095, 0, 0, 1, 1, 0, 4095, 248, 0, 0, 0, 4095, 0, 1, 4095, 1, 5, 0, 1, 51, 296, 0, 1, 0, 1315, 1, 0, 0, 4095, 0, 4095, 3935, 0, 0, 4095, 4095, 4095, 4093, 4, 1, 0, 0, 0, 1, 8, 0, 4095, 4094, 1, 1, 1, 4094, 0, 1, 3926, 3976, 1, 1, 0, 0, 4095, 1, 0, 337, 4095, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 3338, 0, 4096, 0, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2857, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 7, 0, 0, 4094, 0, 0, 4096, 4094, 0, 4094, 4096, 0, 10, 4086, 359, 0, 3, 4094, 0, 22, 4095, 4096, 1, 0, 0, 0, 1, 4094, 332, 0, 0, 0, 4079, 0, 0, 4096, 0, 0, 0, 0, 1, 57, 1, 0, 0, 1032, 0, 0, 0, 4096, 0, 4096, 4096, 0, 7, 4096, 4096, 4095, 4093, 34, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 3817, 4072, 0, 0, 0, 0, 4096, 0, 0, 238, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3822, 0, 4096, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1023, 4096, 2492, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2906, 933, 161, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4094, 3163, 0, 0, 0, 0, 175, 4096, 1396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 3935, 0, 0, 0, 0, 0, 4095, 37, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 19, 629, 0, 0, 0, 1114, 0, 0, 0, 4096, 0, 4096, 1604, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 4096, 0, 4009, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 416, 0, 0, 4057, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15724, 16374, 17019, 16258, 16428, 16307, 16721, 16241] avg=16384.0000 max=17019 max_rank=2 min=15724 min_rank=0 imbalance_ratio=1.038757 max_load_reduction=114053 max_load_reduction_pct=0.870155 moved_to_echo=65697 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3217, 0, 4096, 0, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1976, 4096, 2493, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2906, 933, 161, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4093, 3163, 0, 0, 0, 0, 125, 4095, 1099, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4049, 3935, 0, 0, 0, 0, 0, 4094, 18, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 103, 618, 0, 0, 0, 1333, 0, 0, 0, 4096, 0, 4096, 1603, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 4096, 0, 4009, 4096, 21, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 416, 0, 0, 3398, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16375, 16673, 16192, 16719, 16330, 16058, 16339] avg=16384.0000 max=16719 max_rank=4 min=16058 min_rank=6 imbalance_ratio=1.020447 max_load_reduction=114353 max_load_reduction_pct=0.872444 moved_to_echo=65446 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 2693, 0, 4095, 1, 704, 1, 0, 5, 0, 1, 0, 0, 1, 0, 2588, 4095, 2491, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 4095, 1, 1, 4095, 1, 1, 1, 1, 4095, 0, 8, 1, 0, 2906, 933, 161, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3923, 3160, 1, 0, 0, 1, 166, 4093, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 4067, 3934, 1, 0, 0, 1, 0, 4095, 247, 0, 0, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 84, 93, 0, 0, 1, 1244, 1, 1, 0, 4095, 0, 4093, 1603, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 4079, 0, 4007, 4095, 26, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4081, 1, 1, 1, 416, 0, 1, 3754, 3984, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 4095, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 4095, 4095, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16857, 16391, 16338, 16462, 16085, 16266, 16335, 16338] avg=16384.0000 max=16857 max_rank=0 min=16085 min_rank=4 imbalance_ratio=1.028870 max_load_reduction=114215 max_load_reduction_pct=0.871391 moved_to_echo=65024 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 2706, 0, 4096, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1450, 4095, 2492, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2906, 931, 161, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3933, 3153, 0, 0, 0, 0, 67, 4085, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4078, 3935, 0, 0, 0, 0, 0, 4096, 154, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 9, 966, 0, 0, 0, 1936, 0, 0, 0, 4096, 0, 4096, 1604, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4, 4096, 0, 4009, 4096, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 416, 0, 0, 3082, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 358, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15262, 16373, 16365, 16388, 17577, 16762, 15784, 16561] avg=16384.0000 max=17577 max_rank=4 min=15262 min_rank=0 imbalance_ratio=1.072815 max_load_reduction=113495 max_load_reduction_pct=0.865898 moved_to_echo=66684 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 3730, 0, 4095, 1, 100, 1, 0, 5, 0, 1, 0, 0, 1, 0, 1418, 4095, 2492, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 4094, 1, 1, 1, 0, 4095, 0, 0, 1, 0, 2906, 932, 161, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 3932, 3159, 1, 0, 0, 1, 152, 4093, 971, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4083, 3934, 0, 0, 1, 1, 0, 4095, 229, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 2, 0, 0, 89, 322, 0, 1, 0, 1185, 1, 1, 0, 4094, 0, 4095, 1602, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 0, 4095, 0, 4008, 4094, 132, 1, 0, 0, 0, 1, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 1, 1, 1, 416, 0, 1, 4058, 4083, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 4095, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4095, 4095, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16121, 16377, 16403, 16448, 16261, 16375, 16753, 16334] avg=16384.0000 max=16753 max_rank=6 min=16121 min_rank=0 imbalance_ratio=1.022522 max_load_reduction=114319 max_load_reduction_pct=0.872185 moved_to_echo=65723 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3523, 0, 4096, 0, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2225, 4094, 2486, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 4096, 0, 0, 0, 12, 4096, 0, 15, 0, 0, 2911, 933, 165, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4052, 0, 0, 4094, 3145, 0, 0, 0, 0, 27, 4055, 869, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 4056, 3931, 12, 0, 0, 0, 4, 4089, 96, 9, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 231, 1, 0, 0, 1036, 0, 0, 0, 4096, 0, 4079, 1598, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 67, 4096, 0, 3992, 4084, 176, 0, 23, 0, 16, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4078, 0, 0, 0, 408, 0, 0, 3774, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 4096, 4096, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16913, 16422, 16263, 16319, 15917, 16551, 16410, 16277] avg=16384.0000 max=16913 max_rank=0 min=15917 min_rank=4 imbalance_ratio=1.032288 max_load_reduction=114159 max_load_reduction_pct=0.870964 moved_to_echo=65155 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 3713, 0, 4095, 1, 606, 1, 0, 2, 0, 1, 0, 0, 1, 0, 1327, 4095, 2493, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 4094, 1, 1, 1, 0, 4095, 0, 0, 1, 0, 2905, 933, 161, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 3991, 3160, 1, 0, 0, 1, 142, 4091, 772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4057, 3934, 0, 0, 1, 1, 0, 4095, 248, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 5, 0, 1, 51, 296, 0, 1, 0, 1315, 1, 0, 0, 4095, 0, 4095, 1602, 773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 0, 0, 4095, 0, 4008, 4093, 4, 1, 0, 0, 0, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4094, 1, 1, 1, 416, 0, 1, 3926, 3976, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 4095, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4095, 4095, 3914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16517, 16377, 16202, 16446, 16331, 16145, 16514, 16540] avg=16384.0000 max=16540 max_rank=7 min=16145 min_rank=5 imbalance_ratio=1.009521 max_load_reduction=114532 max_load_reduction_pct=0.873810 moved_to_echo=65530 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3338, 0, 4096, 0, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2857, 4096, 2491, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 7, 0, 0, 2906, 933, 161, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 3161, 0, 0, 0, 0, 10, 4086, 359, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 4095, 3935, 1, 0, 0, 0, 1, 4094, 332, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 57, 1, 0, 0, 1032, 0, 0, 0, 4096, 0, 4094, 1603, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 7, 4096, 0, 4008, 4093, 34, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 416, 0, 0, 3817, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 3915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17292, 16383, 15809, 16559, 15754, 16336, 16497, 16442] avg=16384.0000 max=17292 max_rank=0 min=15754 min_rank=4 imbalance_ratio=1.055420 max_load_reduction=113780 max_load_reduction_pct=0.868073 moved_to_echo=65029 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 26752, 0, 32768, 128, 2560, 128, 0, 128, 0, 128, 0, 0, 128, 0, 14976, 32768, 19968, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 1536, 32768, 0, 128, 128, 128, 128, 128, 0, 128, 0, 0, 128, 0, 0, 32768, 32768, 31360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 32640, 31488, 128, 0, 128, 128, 128, 32768, 1408, 128, 0, 0, 32768, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32512, 0, 128, 32768, 0, 32128, 32768, 896, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 128, 128, 3328, 0, 128, 29952, 32384, 128, 128, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 32256, 25344, 128, 0, 0, 128, 896, 32768, 7424, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 128, 128, 128, 384, 3328, 128, 128, 128, 10240, 128, 128, 0, 32768, 0, 32768, 12928, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 32768, 0, 128, 128, 0, 23296, 7552, 1408, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 100, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 4096, 0, 0, 4096, 4025, 4068, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 143, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4023, 1, 0, 4087, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[3, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 4096, 0, 0, 4094, 4093, 4028, 4096, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 4096, 0, 4084, 49, 0, 4082, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[41, 1, 0, 4090, 1, 1, 36, 1, 1, 1, 2, 1, 1, 0, 4095, 1, 4081, 0, 0, 0, 0, 4094, 4090, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 4091, 2, 3, 4094, 0, 1, 4088, 4090, 4065, 4095, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 4095, 0, 1, 1, 0, 0, 1, 4071, 4095, 1, 3, 2, 1, 0, 1, 1, 0, 0, 4095, 0, 4065, 4094, 0, 1, 1, 1, 4095, 0, 4095, 0, 0, 66, 1, 1, 0, 1, 1, 0, 0, 4095, 0, 4095, 1, 1, 1, 142, 0, 1, 4031, 0, 0, 0, 1, 0, 0, 4095, 1, 0, 4094, 0, 3975, 21, 1, 4085, 4095, 0, 0, 4089, 0, 8, 4095, 4093, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[41, 1, 0, 4093, 1, 1, 0, 0, 1, 0, 4, 1, 1, 0, 4095, 0, 3958, 0, 0, 0, 0, 4095, 4093, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 4095, 17, 5, 4095, 0, 1, 4076, 4074, 4095, 4095, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 4083, 4095, 1, 3, 0, 1, 0, 1, 1, 0, 0, 4095, 1, 4088, 4090, 1, 1, 0, 0, 4095, 1, 4095, 0, 1, 41, 1, 1, 0, 1, 0, 0, 0, 4095, 0, 4095, 1, 1, 1, 222, 0, 1, 4035, 0, 0, 0, 1, 0, 0, 4095, 1, 1, 4095, 0, 4028, 0, 1, 4095, 4095, 0, 0, 4093, 0, 1, 4095, 4093, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[26, 1, 0, 4086, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 4095, 0, 3951, 0, 1, 0, 0, 4095, 4095, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 4095, 76, 10, 4095, 0, 1, 4092, 4088, 4095, 4095, 0, 0, 0, 3, 0, 1, 0, 1, 0, 1, 0, 4095, 0, 1, 0, 0, 2, 1, 4080, 4095, 1, 3, 0, 1, 0, 1, 1, 0, 0, 4095, 1, 4042, 4092, 1, 1, 0, 1, 4095, 1, 4095, 0, 1, 27, 1, 0, 0, 1, 0, 0, 0, 4095, 0, 4095, 1, 1, 1, 243, 0, 0, 4056, 0, 0, 0, 1, 0, 0, 4095, 1, 1, 4095, 0, 3985, 0, 1, 4095, 4095, 0, 0, 4089, 0, 0, 4095, 4092, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[7, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4074, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 4096, 0, 0, 4096, 4048, 4069, 4096, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 120, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4042, 11, 0, 4095, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[33, 0, 0, 4096, 0, 0, 77, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 4096, 0, 0, 7, 0, 4072, 4041, 0, 0, 0, 4078, 0, 0, 0, 0, 12, 0, 4085, 9, 0, 4090, 0, 0, 4096, 4040, 4032, 4096, 0, 0, 0, 24, 5, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 1, 0, 0, 4082, 4096, 0, 0, 14, 0, 0, 0, 0, 7, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 4096, 0, 0, 0, 153, 0, 0, 4089, 0, 0, 0, 0, 15, 0, 4094, 0, 0, 4082, 0, 4032, 57, 0, 3994, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4077, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 19, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4092, 4091, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 4096, 2, 0, 4095, 0, 0, 4096, 4088, 4017, 4096, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 1, 4096, 0, 0, 0, 63, 0, 0, 4079, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4095, 0, 4058, 56, 0, 4061, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 100, 0, 0, 0, 4096, 0, 4096, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 32, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 40, 0, 0, 4096, 4025, 4068, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3994, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3948, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3940, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4023, 1, 0, 4087, 0, 0, 0, 70, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16435, 16430, 16349, 16390, 16392, 16364, 16339, 16373] avg=16384.0000 max=16435 max_rank=0 min=16339 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=114637 max_load_reduction_pct=0.874611 moved_to_echo=65468 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[41, 1, 0, 4090, 1, 1, 36, 1, 1, 1, 2, 1, 1, 0, 4095, 1, 4095, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 4094, 4090, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 32, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 2, 3, 40, 0, 1, 4088, 4090, 4065, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 4095, 0, 1, 1, 0, 0, 1, 4071, 4095, 1, 3987, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 1, 0, 1, 1, 0, 0, 4095, 0, 4065, 4094, 0, 1, 1, 1, 3946, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 4095, 0, 0, 66, 1, 1, 0, 1, 1, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 142, 0, 1, 4031, 0, 0, 0, 1, 0, 0, 4095, 1, 0, 4095, 3937, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3975, 21, 1, 4085, 0, 0, 0, 70, 0, 8, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 16413, 16381, 16364, 16367, 16429, 16358, 16349] avg=16384.0000 max=16429 max_rank=5 min=16349 min_rank=7 imbalance_ratio=1.002747 max_load_reduction=114643 max_load_reduction_pct=0.874657 moved_to_echo=65503 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 4096, 0, 0, 6, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 4096, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 32, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 40, 0, 0, 4094, 4093, 4028, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3994, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3948, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4096, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 4096, 3941, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4084, 49, 0, 4082, 0, 0, 0, 70, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16343, 16434, 16379, 16390, 16391, 16369, 16289, 16477] avg=16384.0000 max=16477 max_rank=7 min=16289 min_rank=6 imbalance_ratio=1.005676 max_load_reduction=114595 max_load_reduction_pct=0.874290 moved_to_echo=65526 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[7, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 4096, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4074, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 32, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 40, 0, 0, 4096, 4048, 4069, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3994, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3948, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3940, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4042, 11, 0, 4095, 0, 0, 0, 70, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16347, 16414, 16356, 16390, 16392, 16366, 16397, 16410] avg=16384.0000 max=16414 max_rank=1 min=16347 min_rank=0 imbalance_ratio=1.001831 max_load_reduction=114658 max_load_reduction_pct=0.874771 moved_to_echo=65565 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[41, 1, 0, 4093, 1, 1, 0, 0, 1, 0, 4, 1, 1, 0, 4095, 0, 4095, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 0, 0, 0, 0, 4095, 4093, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 32, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 17, 5, 40, 0, 1, 4076, 4074, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 4083, 4095, 1, 3991, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 1, 1, 0, 0, 4095, 1, 4088, 4090, 1, 1, 0, 0, 3947, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 1, 4095, 0, 1, 41, 1, 1, 0, 1, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 222, 0, 1, 4035, 0, 0, 0, 1, 0, 0, 4095, 1, 1, 4095, 3937, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4028, 0, 1, 4095, 0, 0, 0, 70, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16378, 16295, 16403, 16379, 16385, 16405, 16442, 16385] avg=16384.0000 max=16442 max_rank=6 min=16295 min_rank=1 imbalance_ratio=1.003540 max_load_reduction=114630 max_load_reduction_pct=0.874557 moved_to_echo=65617 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[33, 0, 0, 4096, 0, 0, 77, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 4096, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 4072, 4041, 0, 0, 0, 4078, 0, 0, 0, 0, 12, 36, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 9, 0, 32, 0, 0, 4096, 4040, 4032, 0, 0, 0, 0, 24, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 1, 0, 0, 4082, 4096, 0, 4000, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 7, 0, 4095, 0, 4096, 4096, 0, 0, 0, 0, 3946, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 4096, 5, 0, 12, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 4089, 0, 0, 0, 0, 15, 0, 4094, 0, 0, 4096, 3940, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4032, 57, 0, 3994, 0, 0, 0, 60, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16451, 16367, 16323, 16397, 16410, 16384, 16438, 16302] avg=16384.0000 max=16451 max_rank=0 min=16302 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=114621 max_load_reduction_pct=0.874489 moved_to_echo=65534 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[26, 1, 0, 4086, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 4095, 0, 4095, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 0, 1, 0, 0, 4095, 4095, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 32, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 76, 10, 40, 0, 1, 4092, 4088, 4095, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 4095, 0, 1, 0, 0, 2, 1, 4080, 4095, 1, 3987, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 1, 1, 0, 0, 4095, 1, 4042, 4092, 1, 1, 0, 1, 3947, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 1, 4095, 0, 1, 27, 1, 0, 0, 1, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 243, 0, 0, 4056, 0, 0, 0, 1, 0, 0, 4095, 1, 1, 4095, 3936, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3985, 0, 1, 4095, 0, 0, 0, 70, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16353, 16291, 16500, 16373, 16342, 16390, 16482, 16341] avg=16384.0000 max=16500 max_rank=2 min=16291 min_rank=1 imbalance_ratio=1.007080 max_load_reduction=114572 max_load_reduction_pct=0.874115 moved_to_echo=65555 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 19, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 4096, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 4091, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 32, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 40, 0, 0, 4096, 4088, 4017, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 3994, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3947, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 4079, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 3940, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4058, 56, 0, 4061, 0, 0, 0, 70, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16354, 16428, 16381, 16389, 16393, 16365, 16327, 16435] avg=16384.0000 max=16435 max_rank=7 min=16327 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=114637 max_load_reduction_pct=0.874611 moved_to_echo=65520 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32640, 128, 32768, 128, 128, 256, 128, 128, 0, 128, 128, 128, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[256, 128, 0, 32768, 128, 128, 256, 128, 128, 128, 128, 128, 128, 0, 32768, 128, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 0, 32256, 256, 128, 32640, 0, 0, 0, 640, 0, 128, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32512, 0, 128, 128, 0, 32768, 32768, 0, 0, 0, 32768, 128, 0, 0, 0, 128, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 1152, 0, 128, 32512, 0, 0, 0, 128, 128, 0, 32768, 128, 128, 32768, 31616, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 256, 128, 384, 0, 128, 32768, 32640, 32512, 0, 0, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 0, 128, 128, 128, 0, 32768, 128, 32768, 32768, 128, 128, 128, 128, 31616, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 128, 128, 128, 128, 32768, 0, 128, 128, 128, 128, 128, 32768, 32768, 128, 32000, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3936, 4096, 0, 0, 0, 0, 0, 4096, 0, 835, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 235, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 3446, 0, 0, 0, 0, 4096, 0, 1737, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 0, 4096, 929, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 7, 4096, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 2916, 0, 3551, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 2890, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3896, 4094, 1, 1, 0, 0, 0, 4095, 1, 626, 1, 1, 4095, 4094, 1, 0, 1, 0, 0, 4096, 1, 1, 89, 1, 4095, 0, 0, 10, 1, 4058, 1, 1, 1, 1, 1, 3546, 1, 0, 0, 1, 4094, 0, 3649, 1, 0, 0, 9, 0, 0, 0, 0, 1, 4094, 4094, 0, 4095, 234, 1, 0, 0, 1, 0, 1, 4095, 0, 0, 1, 1, 0, 4088, 1, 1, 157, 4065, 0, 3988, 4060, 0, 0, 0, 4080, 0, 1, 0, 1, 4095, 0, 0, 4091, 1, 0, 1, 4095, 0, 2375, 0, 2653, 0, 1, 0, 1, 1, 0, 6, 4094, 0, 4066, 0, 0, 1, 4095, 1, 0, 1, 4095, 3466, 4095, 0, 4095, 21, 0, 1, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 4096, 0, 754, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4076, 0, 0, 466, 0, 4096, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 3805, 0, 0, 0, 0, 4095, 0, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4096, 0, 4096, 422, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 54, 4096, 59, 4096, 4096, 0, 0, 0, 3984, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 3405, 0, 1365, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 2218, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4042, 4096, 0, 0, 0, 0, 0, 4096, 0, 256, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4060, 0, 0, 358, 0, 4096, 0, 0, 0, 0, 4044, 0, 0, 27, 0, 0, 3363, 0, 0, 0, 11, 4096, 15, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 56, 4084, 4096, 0, 4088, 265, 5, 0, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 4066, 0, 0, 0, 4096, 65, 4080, 4090, 15, 0, 0, 4041, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 2415, 0, 3584, 0, 0, 0, 0, 1, 0, 38, 4087, 0, 4096, 0, 2, 9, 4094, 0, 0, 0, 4096, 2457, 4096, 0, 4090, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 4096, 0, 286, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 965, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3389, 0, 0, 0, 0, 4096, 0, 3639, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 4096, 0, 4096, 34, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 0, 0, 1, 4096, 3, 4096, 4096, 0, 0, 0, 4071, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 3573, 0, 2773, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 1742, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3848, 4095, 1, 0, 1, 0, 0, 4094, 0, 777, 0, 1, 4095, 4094, 1, 1, 1, 0, 0, 4095, 1, 1, 76, 1, 4095, 0, 1, 6, 1, 4055, 1, 1, 0, 1, 1, 3581, 1, 0, 4, 0, 4095, 0, 3468, 1, 0, 0, 50, 0, 0, 1, 0, 0, 4095, 4094, 0, 4095, 854, 0, 0, 0, 1, 0, 1, 4095, 0, 0, 0, 1, 0, 4090, 1, 0, 102, 4043, 0, 4066, 4027, 0, 1, 0, 4078, 0, 1, 1, 1, 4095, 0, 0, 4093, 1, 0, 1, 4095, 3, 3353, 0, 1266, 0, 4, 0, 1, 0, 0, 0, 4095, 0, 4068, 0, 0, 0, 4095, 1, 0, 1, 4095, 3321, 4095, 0, 4095, 4, 0, 1, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3759, 4095, 1, 0, 1, 0, 0, 4094, 0, 840, 0, 1, 4095, 4094, 1, 0, 1, 0, 0, 4095, 1, 1, 362, 1, 4095, 0, 0, 2, 1, 4082, 1, 1, 0, 1, 1, 2755, 1, 0, 1, 0, 4095, 0, 3796, 1, 0, 1, 29, 0, 0, 1, 0, 0, 4095, 4093, 1, 4095, 91, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 1, 0, 4087, 1, 0, 89, 3978, 0, 4093, 4022, 0, 1, 0, 4050, 0, 1, 1, 1, 4095, 0, 0, 4095, 1, 1, 1, 4095, 36, 2593, 0, 2819, 0, 3, 0, 1, 0, 0, 0, 4095, 1, 4090, 1, 0, 0, 4095, 1, 0, 1, 4095, 3562, 4095, 0, 4095, 3, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3996, 4096, 0, 0, 0, 0, 0, 4096, 0, 988, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4057, 0, 0, 274, 0, 4096, 0, 0, 0, 0, 4038, 0, 0, 4, 0, 0, 3592, 0, 0, 0, 1, 4096, 1, 3459, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4096, 4096, 0, 4095, 403, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4087, 0, 0, 30, 4096, 26, 4096, 4095, 2, 0, 0, 3949, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 1441, 0, 3611, 0, 0, 0, 0, 0, 0, 19, 4095, 0, 4096, 0, 0, 1, 4095, 0, 0, 0, 4096, 2851, 4096, 0, 4095, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3936, 4096, 0, 0, 0, 0, 0, 4096, 0, 835, 0, 0, 2817, 426, 321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3753, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 235, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 3446, 0, 0, 0, 0, 4096, 0, 1737, 0, 1050, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4096, 4096, 0, 4096, 929, 0, 0, 0, 3670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 7, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 49, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2725, 0, 2916, 0, 3551, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1279, 2890, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16527, 16276, 14712, 16892, 16370, 16454, 17384, 16457] avg=16384.0000 max=17384 max_rank=6 min=14712 min_rank=2 imbalance_ratio=1.061035 max_load_reduction=113688 max_load_reduction_pct=0.867371 moved_to_echo=66665 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3896, 4094, 1, 1, 0, 0, 0, 4095, 1, 626, 1, 1, 2816, 426, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3752, 4094, 1, 0, 1, 0, 0, 4096, 1, 1, 89, 1, 4095, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4058, 1, 1, 1, 1, 1, 3546, 1, 0, 0, 1, 4094, 0, 3649, 1, 1050, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1, 4094, 4094, 0, 4095, 234, 1, 0, 0, 3669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4072, 0, 0, 1, 1, 0, 4088, 1, 1, 157, 4065, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 4080, 0, 1, 0, 1, 4095, 0, 0, 4091, 1, 0, 1, 49, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2725, 0, 2375, 0, 2653, 0, 1, 0, 1, 1, 0, 6, 4094, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 1, 1279, 3466, 0, 0, 4095, 21, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16279, 16141, 16700, 16197, 16376, 16402, 15922, 17055] avg=16384.0000 max=17055 max_rank=7 min=15922 min_rank=6 imbalance_ratio=1.040955 max_load_reduction=114017 max_load_reduction_pct=0.869881 moved_to_echo=65755 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 4096, 0, 754, 0, 0, 2817, 426, 321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3753, 4096, 0, 0, 0, 0, 0, 4076, 0, 0, 466, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 0, 3805, 0, 0, 0, 0, 4095, 0, 3992, 0, 1050, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4096, 0, 4096, 422, 0, 0, 0, 3670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 4096, 0, 0, 54, 4096, 59, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3984, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 49, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2725, 0, 3405, 0, 1365, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1279, 2218, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16603, 16487, 17302, 16388, 16474, 16344, 15689, 15785] avg=16384.0000 max=17302 max_rank=2 min=15689 min_rank=6 imbalance_ratio=1.056030 max_load_reduction=113770 max_load_reduction_pct=0.867996 moved_to_echo=64292 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4042, 4096, 0, 0, 0, 0, 0, 4096, 0, 256, 0, 0, 2818, 427, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3746, 4096, 0, 0, 0, 0, 0, 4060, 0, 0, 358, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 27, 0, 0, 3363, 0, 0, 0, 11, 4096, 15, 3718, 0, 1055, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 4084, 4096, 0, 4088, 265, 5, 0, 0, 3669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 0, 0, 10, 0, 0, 4066, 0, 0, 0, 4096, 65, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 15, 0, 0, 4041, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 54, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2717, 0, 2415, 0, 3584, 0, 0, 0, 0, 1, 0, 38, 4087, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 9, 4094, 0, 0, 0, 1278, 2457, 0, 0, 4090, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16059, 16356, 16625, 16263, 16386, 16419, 16938, 16026] avg=16384.0000 max=16938 max_rank=6 min=16026 min_rank=7 imbalance_ratio=1.033813 max_load_reduction=114134 max_load_reduction_pct=0.870773 moved_to_echo=65769 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4088, 4096, 0, 0, 0, 0, 0, 4096, 0, 286, 0, 0, 2817, 426, 321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3753, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 965, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3389, 0, 0, 0, 0, 4096, 0, 3639, 0, 1050, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4096, 4096, 0, 4096, 34, 0, 0, 0, 3670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 4089, 0, 0, 1, 4096, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4071, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 49, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2725, 0, 3573, 0, 2773, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1279, 1742, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16130, 17006, 16564, 16001, 16358, 16431, 17273, 15309] avg=16384.0000 max=17273 max_rank=6 min=15309 min_rank=7 imbalance_ratio=1.054260 max_load_reduction=113799 max_load_reduction_pct=0.868217 moved_to_echo=65371 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3848, 4095, 1, 0, 1, 0, 0, 4094, 0, 777, 0, 1, 2816, 426, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3752, 4094, 1, 1, 1, 0, 0, 4095, 1, 1, 76, 1, 4095, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4055, 1, 1, 0, 1, 1, 3581, 1, 0, 4, 0, 4095, 0, 3468, 1, 1050, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 1, 0, 0, 4095, 4094, 0, 4095, 854, 0, 0, 0, 3669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4072, 0, 0, 0, 1, 0, 4090, 1, 0, 102, 4043, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 1, 0, 4078, 0, 1, 1, 1, 4095, 0, 0, 4093, 1, 0, 1, 49, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2725, 3, 3353, 0, 1266, 0, 4, 0, 1, 0, 0, 0, 4095, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 1279, 3321, 0, 0, 4095, 4, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16380, 16125, 16554, 16858, 16377, 16371, 15515, 16892] avg=16384.0000 max=16892 max_rank=7 min=15515 min_rank=6 imbalance_ratio=1.031006 max_load_reduction=114180 max_load_reduction_pct=0.871124 moved_to_echo=65155 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3759, 4095, 1, 0, 1, 0, 0, 4094, 0, 840, 0, 1, 2816, 426, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3752, 4094, 1, 0, 1, 0, 0, 4095, 1, 1, 362, 1, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4082, 1, 1, 0, 1, 1, 2755, 1, 0, 1, 0, 4095, 0, 3796, 1, 1050, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 29, 0, 0, 1, 0, 0, 4095, 4093, 1, 4095, 91, 0, 0, 0, 3669, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4072, 0, 0, 0, 1, 0, 4087, 1, 0, 89, 3978, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 1, 0, 4050, 0, 1, 1, 1, 4095, 0, 0, 4095, 1, 1, 1, 49, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2725, 36, 2593, 0, 2819, 0, 3, 0, 1, 0, 0, 0, 4095, 1, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 1279, 3562, 0, 0, 4095, 3, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16354, 16405, 16080, 16075, 16322, 16341, 16364, 17131] avg=16384.0000 max=17131 max_rank=7 min=16075 min_rank=3 imbalance_ratio=1.045593 max_load_reduction=113941 max_load_reduction_pct=0.869301 moved_to_echo=66158 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3996, 4096, 0, 0, 0, 0, 0, 4096, 0, 988, 0, 0, 2817, 426, 321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3753, 4096, 0, 0, 0, 0, 0, 4057, 0, 0, 274, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 4, 0, 0, 3592, 0, 0, 0, 1, 4096, 1, 3459, 0, 1050, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4096, 4096, 0, 4095, 403, 0, 0, 0, 3670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 1, 0, 0, 4087, 0, 0, 30, 4096, 26, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 3949, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 49, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2725, 0, 1441, 0, 3611, 0, 0, 0, 0, 0, 0, 19, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 1279, 2851, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16740, 16276, 16535, 16398, 16409, 16310, 15987, 16417] avg=16384.0000 max=16740 max_rank=0 min=15987 min_rank=6 imbalance_ratio=1.021729 max_load_reduction=114332 max_load_reduction_pct=0.872284 moved_to_echo=65123 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 32768, 128, 0, 128, 10240, 22528, 0, 0, 32768, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[21888, 128, 22144, 0, 21632, 0, 128, 0, 128, 128, 0, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[30080, 32768, 128, 128, 128, 0, 0, 32768, 128, 128, 2944, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 32640, 128, 128, 128, 128, 128, 27520, 128, 0, 128, 128, 32768, 128, 27520, 128, 8448, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32640, 128, 128, 0, 32384, 0, 128, 128, 128, 32768, 0, 0, 32768, 128, 128, 128, 512, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 128, 0, 31744, 32768, 128, 128, 128, 0, 0, 32768, 128, 5376, 128, 128, 22656, 3456, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 0, 128, 0, 128, 32768, 32768, 128, 32768, 3328, 128, 0, 0, 29440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 32640, 0, 0, 128, 128, 0, 32768, 128, 128, 512, 32640, 256, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[3167, 0, 0, 4096, 0, 0, 4093, 1, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 4096, 0, 118, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 4096, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1563, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 51, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4084, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4092, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 3309, 0, 0, 4094, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[4088, 0, 0, 4096, 0, 0, 4096, 93, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 4095, 0, 7, 0, 4096, 4023, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 211, 4092, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 25, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4067, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 3947, 0, 0, 3987, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[4067, 1, 2, 4094, 1, 1, 4094, 1, 0, 0, 1, 0, 0, 0, 1, 4093, 17, 2992, 2, 576, 1, 4094, 4041, 1, 0, 1, 4095, 0, 316, 0, 0, 0, 4094, 0, 0, 4094, 4095, 4095, 0, 1, 1, 1, 0, 1, 15, 0, 0, 1, 1, 0, 1, 0, 13, 1, 0, 1, 4095, 1, 0, 4080, 0, 0, 1, 4095, 1, 0, 0, 0, 1, 1242, 4095, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 4094, 4094, 0, 222, 0, 0, 0, 0, 0, 1, 4095, 4095, 1, 0, 4095, 1, 1, 3474, 1, 3682, 0, 1, 0, 4095, 1, 0, 0, 4093, 0, 4094, 25, 4095, 4095, 4095, 0, 1, 0, 1, 4095, 3903, 0, 1, 4094, 2, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4061, 0, 0, 4088, 0, 0, 4063, 35, 0, 0, 0, 33, 0, 0, 0, 4096, 6, 4090, 0, 2, 0, 4096, 3992, 0, 0, 0, 4082, 0, 0, 1, 0, 0, 4096, 0, 0, 4062, 4091, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 4096, 0, 15, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 685, 4063, 2, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 4096, 4083, 0, 66, 0, 0, 22, 0, 12, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4055, 9, 0, 0, 4096, 0, 0, 0, 4096, 0, 4057, 0, 4095, 4094, 4096, 0, 0, 0, 0, 4079, 3643, 37, 0, 3949, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[3994, 0, 0, 4096, 0, 0, 4095, 113, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 67, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4074, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 241, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 220, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4080, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 3772, 7, 0, 3895, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[4052, 0, 0, 4095, 1, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 1, 4093, 19, 3009, 5, 632, 1, 4095, 4009, 0, 1, 1, 4095, 0, 242, 0, 0, 0, 4094, 1, 1, 4095, 4095, 4095, 0, 0, 0, 1, 0, 1, 52, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 4095, 1, 0, 4049, 0, 1, 1, 4095, 1, 0, 0, 0, 0, 1378, 4095, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 4095, 4095, 1, 397, 0, 0, 0, 0, 0, 1, 4090, 4095, 2, 0, 4095, 1, 1, 3503, 1, 3771, 0, 1, 1, 4094, 1, 0, 3, 4094, 0, 4095, 81, 4095, 4095, 4094, 0, 1, 0, 1, 4095, 3471, 0, 1, 4091, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3988, 0, 0, 4095, 0, 1, 4095, 0, 0, 0, 1, 0, 0, 0, 1, 4094, 23, 3195, 1, 671, 1, 4095, 4053, 0, 1, 1, 4095, 0, 297, 0, 0, 0, 4093, 2, 2, 4095, 4095, 4095, 0, 0, 0, 1, 0, 1, 115, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 4095, 1, 0, 3961, 0, 1, 1, 4095, 1, 0, 0, 0, 0, 1032, 4095, 2, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 4095, 4095, 0, 34, 0, 0, 0, 0, 0, 1, 4092, 4094, 3, 0, 4095, 1, 1, 3647, 0, 3721, 0, 1, 1, 4094, 1, 0, 1, 4094, 0, 4095, 22, 4095, 4095, 4095, 0, 0, 0, 1, 4095, 3902, 0, 1, 4095, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4065, 0, 0, 4096, 0, 0, 4076, 63, 0, 0, 0, 32, 0, 0, 0, 4096, 0, 4095, 0, 95, 0, 4096, 4030, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 4053, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 939, 4074, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 0, 15, 0, 7, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4068, 0, 4094, 4095, 4096, 0, 0, 0, 0, 4095, 3438, 16, 0, 3751, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3167, 0, 0, 4096, 0, 0, 4093, 1, 0, 0, 0, 7, 0, 0, 0, 4096, 84, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 118, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 47, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 4096, 4088, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3542, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1563, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 3185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 4096, 0, 51, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 0, 0, 4096, 0, 4084, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 518, 0, 0, 0, 0, 0, 4096, 3309, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15580, 16557, 16371, 16387, 17037, 16304, 16727, 16109] avg=16384.0000 max=17037 max_rank=4 min=15580 min_rank=0 imbalance_ratio=1.039856 max_load_reduction=114035 max_load_reduction_pct=0.870018 moved_to_echo=66177 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4067, 1, 2, 4094, 1, 1, 4094, 1, 0, 0, 1, 0, 0, 0, 1, 4093, 84, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 2992, 2, 576, 1, 4094, 4041, 1, 0, 1, 4095, 0, 316, 0, 0, 0, 46, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4094, 4095, 4087, 0, 1, 1, 1, 0, 1, 15, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 13, 1, 0, 1, 4095, 1, 0, 4080, 0, 0, 1, 4095, 3541, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1242, 4095, 1, 1, 0, 0, 1, 0, 0, 0, 1, 4095, 4095, 3184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3964, 4094, 0, 222, 0, 0, 0, 0, 0, 1, 4095, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 1, 1, 3474, 1, 3682, 0, 1, 0, 4095, 1, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 25, 0, 518, 0, 0, 1, 0, 1, 4095, 3903, 0, 1, 4094, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16476, 16190, 16390, 16386, 16717, 16474, 15704, 16735] avg=16384.0000 max=16735 max_rank=7 min=15704 min_rank=6 imbalance_ratio=1.021423 max_load_reduction=114337 max_load_reduction_pct=0.872322 moved_to_echo=65630 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4061, 0, 0, 4088, 0, 0, 4063, 35, 0, 0, 0, 33, 0, 0, 0, 4096, 86, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4090, 0, 2, 0, 4096, 3992, 0, 0, 0, 4082, 0, 0, 1, 0, 0, 50, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4062, 4091, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 4096, 0, 15, 4096, 0, 0, 0, 4096, 3546, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 685, 4063, 2, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 4095, 3191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 4083, 0, 66, 0, 0, 22, 0, 12, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 4096, 0, 4055, 9, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 508, 0, 0, 0, 0, 0, 4079, 3643, 37, 0, 3949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16502, 16329, 16335, 16448, 16154, 16335, 16696, 16273] avg=16384.0000 max=16696 max_rank=6 min=16154 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=114376 max_load_reduction_pct=0.872620 moved_to_echo=65458 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3994, 0, 0, 4096, 0, 0, 4095, 113, 0, 0, 0, 1, 0, 0, 0, 4096, 84, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 67, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 47, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4074, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3542, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 4096, 3185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 4096, 0, 220, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 518, 0, 0, 0, 0, 0, 4096, 3772, 7, 0, 3895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16515, 16506, 16354, 16402, 15716, 16473, 16738, 16368] avg=16384.0000 max=16738 max_rank=6 min=15716 min_rank=4 imbalance_ratio=1.021606 max_load_reduction=114334 max_load_reduction_pct=0.872299 moved_to_echo=65295 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4052, 0, 0, 4095, 1, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 1, 4093, 84, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 3009, 5, 632, 1, 4095, 4009, 0, 1, 1, 4095, 0, 242, 0, 0, 0, 46, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 4095, 4095, 4087, 0, 0, 0, 1, 0, 1, 52, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 4095, 1, 0, 4049, 0, 1, 1, 4095, 3541, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1378, 4095, 0, 1, 0, 0, 1, 0, 0, 0, 1, 4094, 4095, 3184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3965, 4095, 1, 397, 0, 0, 0, 0, 0, 1, 4090, 4095, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 1, 1, 3503, 1, 3771, 0, 1, 1, 4094, 1, 0, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 81, 0, 518, 0, 0, 1, 0, 1, 4095, 3471, 0, 1, 4091, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16458, 16163, 16428, 16344, 16850, 16648, 15826, 16355] avg=16384.0000 max=16850 max_rank=4 min=15826 min_rank=6 imbalance_ratio=1.028442 max_load_reduction=114222 max_load_reduction_pct=0.871445 moved_to_echo=65679 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3988, 0, 0, 4095, 0, 1, 4095, 0, 0, 0, 1, 0, 0, 0, 1, 4094, 84, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3195, 1, 671, 1, 4095, 4053, 0, 1, 1, 4095, 0, 297, 0, 0, 0, 46, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2, 2, 4095, 4095, 4087, 0, 0, 0, 1, 0, 1, 115, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 4095, 1, 0, 3961, 0, 1, 1, 4095, 3541, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1032, 4095, 2, 1, 0, 0, 1, 0, 0, 0, 1, 4095, 4095, 3184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3965, 4095, 0, 34, 0, 0, 0, 0, 0, 1, 4092, 4094, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 1, 1, 3647, 0, 3721, 0, 1, 1, 4094, 1, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 22, 0, 518, 0, 0, 0, 0, 1, 4095, 3902, 0, 1, 4095, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16395, 16487, 16493, 16256, 16507, 16286, 15917, 16731] avg=16384.0000 max=16731 max_rank=7 min=15917 min_rank=6 imbalance_ratio=1.021179 max_load_reduction=114341 max_load_reduction_pct=0.872353 moved_to_echo=65441 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4088, 0, 0, 4096, 0, 0, 4096, 93, 0, 0, 0, 9, 0, 0, 0, 4096, 84, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 7, 0, 4096, 4023, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 47, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3542, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 4092, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 4096, 3185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 4096, 0, 25, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 0, 0, 4096, 0, 4083, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 518, 0, 0, 0, 0, 0, 4096, 3947, 0, 0, 3987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16598, 16372, 16368, 16426, 15689, 16278, 16726, 16615] avg=16384.0000 max=16726 max_rank=6 min=15689 min_rank=4 imbalance_ratio=1.020874 max_load_reduction=114346 max_load_reduction_pct=0.872391 moved_to_echo=65308 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4065, 0, 0, 4096, 0, 0, 4076, 63, 0, 0, 0, 32, 0, 0, 0, 4096, 84, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 95, 0, 4096, 4030, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 47, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4053, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 0, 4096, 3541, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 939, 4074, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 4094, 3185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3965, 4095, 0, 0, 0, 0, 15, 0, 7, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 518, 0, 0, 0, 0, 0, 4095, 3438, 16, 0, 3751, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16548, 16468, 16333, 16423, 16402, 16274, 16738, 15886] avg=16384.0000 max=16738 max_rank=6 min=15886 min_rank=7 imbalance_ratio=1.021606 max_load_reduction=114334 max_load_reduction_pct=0.872299 moved_to_echo=65300 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[31488, 128, 128, 32768, 128, 128, 32768, 384, 0, 0, 128, 128, 0, 0, 128, 32768, 768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 0, 128, 7296, 32768, 128, 128, 0, 0, 128, 128, 0, 0, 128, 32768, 32768, 25600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 31744, 32768, 128, 1024, 0, 0, 128, 0, 128, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 2944, 128, 128, 31104, 128, 31616, 128, 128, 128, 32768, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 0, 4224, 0, 0, 128, 0, 128, 32768, 29440, 128, 128, 32000, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 32768, 32768, 32768, 0, 128, 128, 128, 0, 128, 256, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 29696, 128, 2176, 128, 32768, 32384, 128, 128, 128, 32768, 0, 896, 128, 0, 0, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 0, 256, 128, 128, 128, 32768, 128, 128, 32640, 0, 128, 128, 32768, 28416, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4043, 0, 0, 0, 4096, 0, 4095, 33, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 24, 0, 4080, 0, 0, 0, 0, 4071, 162, 3470, 4096, 0, 189, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 818, 0, 0, 0, 0, 0, 0, 4096, 0, 4077, 0, 4089, 0, 0, 0, 1, 4096, 4079, 4096, 40, 2333, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 4075, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 4096, 1238, 4094, 0, 0, 12, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 288, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4061, 0, 0, 0, 4096, 0, 4088, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4072, 123, 2979, 4096, 0, 179, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 630, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4060, 4096, 5, 3229, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 857, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 476, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0, 0, 0, 4096, 0, 4095, 21, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 3967, 1, 3655, 4096, 0, 5, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 36, 4096, 0, 0, 1874, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 0, 0, 4096, 4088, 4096, 6, 1778, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 477, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 1161, 0, 0, 1, 1, 0, 1, 8, 4067, 1, 1, 0, 4093, 0, 0, 4095, 4095, 4095, 0, 0, 1, 4091, 1, 3617, 6, 1, 1, 4095, 1, 43, 0, 1, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 2365, 1, 3892, 4095, 1, 5, 1, 4095, 0, 1, 0, 0, 4094, 1, 1, 4091, 7, 0, 4018, 0, 3, 0, 1, 1, 0, 1, 0, 4090, 0, 31, 7, 0, 0, 0, 1, 1, 0, 4094, 1, 4095, 0, 4086, 1, 21, 1, 0, 4094, 4086, 4095, 0, 3779, 0, 1, 1, 0, 1, 4095, 0, 1, 0, 0, 4089, 1, 4093, 1, 1, 1, 4095, 1, 4094, 1, 0, 0, 0, 0, 32, 4095, 1550, 4094, 1, 0, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 0, 3, 1392, 1, 0, 0, 0, 0, 0, 9, 4044, 1, 1, 0, 4085, 0, 0, 4095, 4095, 4095, 0, 0, 1, 4095, 1, 3541, 3, 0, 0, 4095, 1, 49, 1, 1, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 2267, 0, 4059, 4095, 1, 20, 1, 4095, 0, 1, 0, 0, 4095, 1, 1, 4094, 0, 0, 3988, 0, 4, 0, 1, 1, 0, 1, 0, 4091, 0, 12, 99, 0, 1, 1, 1, 1, 0, 4095, 1, 4095, 0, 4081, 1, 10, 1, 0, 4095, 4082, 4095, 0, 3142, 0, 0, 6, 0, 3, 4095, 0, 1, 0, 0, 4089, 1, 4095, 1, 1, 1, 4095, 1, 4092, 0, 0, 0, 0, 0, 102, 4095, 1871, 4094, 1, 0, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4092, 26, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4070, 76, 2328, 4096, 0, 143, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 583, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 0, 0, 4096, 4041, 4096, 1, 2637, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2449, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1, 1, 0, 7, 1100, 2, 0, 0, 0, 0, 0, 10, 4061, 1, 1, 0, 4086, 0, 0, 4095, 4095, 4095, 0, 0, 1, 4095, 1, 3333, 2, 0, 0, 4095, 1, 35, 1, 1, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 2196, 0, 3736, 4095, 1, 230, 1, 4095, 0, 1, 0, 0, 4095, 1, 1, 4094, 0, 0, 4012, 0, 5, 0, 1, 1, 0, 1, 0, 4090, 0, 80, 7, 0, 0, 0, 1, 1, 0, 4095, 1, 4095, 0, 4065, 1, 0, 1, 0, 4095, 4095, 4095, 0, 3822, 0, 0, 0, 0, 6, 4095, 0, 1, 0, 0, 4090, 1, 4095, 1, 1, 1, 4095, 1, 4094, 1, 0, 0, 0, 0, 44, 4095, 1938, 4095, 0, 1, 0, 4089] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0, 0, 0, 4096, 0, 4096, 9, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0, 0, 4079, 69, 3336, 4096, 0, 64, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 854, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4094, 0, 0, 0, 1, 4096, 4087, 4096, 6, 2440, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 1352, 4096, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4043, 0, 0, 0, 4096, 0, 4095, 33, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 4080, 0, 0, 0, 0, 4071, 162, 3470, 3870, 1193, 341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4021, 0, 189, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 818, 0, 0, 0, 0, 4096, 4096, 2903, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1212, 0, 4077, 0, 4089, 0, 0, 0, 1, 0, 4079, 0, 40, 2333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 4075, 0, 0, 0, 2543, 1489, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 2607, 1238, 4094, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15850, 16589, 17211, 16498, 16582, 15831, 16375, 16136] avg=16384.0000 max=17211 max_rank=2 min=15831 min_rank=5 imbalance_ratio=1.050476 max_load_reduction=113861 max_load_reduction_pct=0.868690 moved_to_echo=64924 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 288, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4061, 0, 0, 0, 4096, 0, 4088, 5, 0, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4072, 123, 2979, 3868, 1192, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 179, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 630, 0, 0, 0, 0, 4096, 4096, 2904, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1219, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4060, 0, 5, 3229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2541, 1486, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2610, 857, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16110, 16574, 16666, 16492, 16384, 16705, 16386, 15755] avg=16384.0000 max=16705 max_rank=5 min=15755 min_rank=7 imbalance_ratio=1.019592 max_load_reduction=114367 max_load_reduction_pct=0.872551 moved_to_echo=65230 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 476, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 4095, 21, 0, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 0, 3967, 1, 3655, 3868, 1192, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 4096, 0, 0, 1874, 0, 0, 0, 0, 4096, 4096, 2904, 562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1219, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 4088, 0, 6, 1778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2541, 1486, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2610, 477, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16298, 16631, 17118, 16318, 17664, 15282, 16386, 15375] avg=16384.0000 max=17664 max_rank=4 min=15282 min_rank=5 imbalance_ratio=1.078125 max_load_reduction=113408 max_load_reduction_pct=0.865234 moved_to_echo=64707 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 1161, 0, 0, 1, 1, 0, 1, 8, 4067, 1, 1, 0, 4095, 4095, 3534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4095, 0, 0, 1, 4091, 1, 3617, 6, 1, 1, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 43, 0, 1, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 2365, 1, 3892, 3867, 1191, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 1, 5, 1, 0, 0, 1, 0, 0, 4094, 1, 1, 4091, 7, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 1, 0, 1, 0, 4090, 0, 31, 7, 0, 0, 0, 1, 4095, 4095, 2903, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1219, 1, 4095, 0, 4086, 1, 21, 1, 0, 0, 4086, 0, 0, 3779, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 4095, 0, 1, 0, 0, 4089, 1, 4093, 1, 1, 1, 2539, 1485, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 0, 0, 0, 0, 32, 2610, 1550, 4094, 1, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16967, 16134, 15794, 16244, 15789, 17290, 16380, 16474] avg=16384.0000 max=17290 max_rank=5 min=15789 min_rank=4 imbalance_ratio=1.055298 max_load_reduction=113782 max_load_reduction_pct=0.868088 moved_to_echo=65933 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3, 1392, 1, 0, 0, 0, 0, 0, 9, 4044, 1, 1, 0, 4095, 4095, 3534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 4095, 0, 0, 1, 4095, 1, 3541, 3, 0, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 49, 1, 1, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 2267, 0, 4059, 3867, 1191, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 1, 20, 1, 0, 0, 1, 0, 0, 4095, 1, 1, 4094, 0, 0, 3988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 1, 0, 1, 0, 4091, 0, 12, 99, 0, 1, 1, 1, 4095, 4095, 2904, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1219, 1, 4095, 0, 4081, 1, 10, 1, 0, 0, 4082, 0, 0, 3142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3, 4095, 0, 1, 0, 0, 4089, 1, 4095, 1, 1, 1, 2540, 1485, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 0, 0, 0, 0, 102, 2610, 1871, 4094, 1, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17176, 16049, 15870, 16226, 15867, 16633, 16389, 16862] avg=16384.0000 max=17176 max_rank=0 min=15867 min_rank=4 imbalance_ratio=1.048340 max_load_reduction=113896 max_load_reduction_pct=0.868958 moved_to_echo=65751 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4092, 26, 0, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4070, 76, 2328, 3868, 1193, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 143, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 583, 0, 0, 0, 0, 4096, 4096, 2903, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1219, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 4041, 0, 1, 2637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 2541, 1486, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2610, 2449, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15851, 16634, 15972, 16457, 16336, 16089, 16386, 17347] avg=16384.0000 max=17347 max_rank=7 min=15851 min_rank=0 imbalance_ratio=1.058777 max_load_reduction=113725 max_load_reduction_pct=0.867653 moved_to_echo=66158 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 7, 1100, 2, 0, 0, 0, 0, 0, 10, 4061, 1, 1, 0, 4095, 4095, 3534, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 4095, 0, 0, 1, 4095, 1, 3333, 2, 0, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 35, 1, 1, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 2196, 0, 3736, 3867, 1192, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 1, 230, 1, 0, 0, 1, 0, 0, 4095, 1, 1, 4094, 0, 0, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 1, 0, 1, 0, 4090, 0, 80, 7, 0, 0, 0, 1, 4095, 4095, 2903, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1219, 1, 4095, 0, 4065, 1, 0, 1, 0, 0, 4095, 0, 0, 3822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4095, 0, 1, 0, 0, 4090, 1, 4095, 1, 1, 1, 2540, 1485, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 1, 0, 0, 0, 0, 44, 2610, 1938, 4095, 0, 1, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16908, 15841, 15463, 16460, 15840, 17300, 16387, 16873] avg=16384.0000 max=17300 max_rank=5 min=15463 min_rank=2 imbalance_ratio=1.055908 max_load_reduction=113772 max_load_reduction_pct=0.868011 moved_to_echo=66400 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 4096, 9, 0, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4095, 0, 0, 0, 0, 4079, 69, 3336, 3868, 1192, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 0, 64, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 854, 0, 0, 0, 0, 4096, 4096, 2904, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1219, 0, 4095, 0, 4094, 0, 0, 0, 1, 0, 4087, 0, 6, 2440, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 2541, 1486, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 2610, 1352, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15912, 16620, 16978, 16377, 16610, 15942, 16383, 16250] avg=16384.0000 max=16978 max_rank=2 min=15912 min_rank=0 imbalance_ratio=1.036255 max_load_reduction=114094 max_load_reduction_pct=0.870468 moved_to_echo=65185 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 0, 0, 0, 32768, 0, 0, 128, 32768, 128, 30976, 128, 128, 128, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0, 256, 20992, 11776, 32768, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 0, 128, 0, 0, 32768, 128, 32768, 128, 128, 128, 20352, 11904, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 9856, 128, 32768, 0, 32768, 128, 128, 128, 128, 0, 32640, 0, 128, 23168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 0, 32768, 0, 128, 128, 128, 27136, 512, 27520, 30976, 9600, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 128, 32768, 0, 128, 4992, 0, 128, 128, 128, 32768, 32768, 23296, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32256, 128, 896, 128, 0, 0, 128, 0, 0, 32768, 128, 128, 32768, 128, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 4608, 128, 0, 128, 128, 0, 128, 128, 32768, 128, 128, 0, 32768, 32768, 28288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[1, 1, 0, 1, 4095, 0, 0, 4061, 89, 14, 1, 4095, 4094, 4080, 1, 4095, 4094, 1, 4093, 1, 0, 0, 0, 4094, 0, 0, 6, 4079, 0, 4009, 1, 0, 4090, 1, 0, 0, 4094, 1, 1, 0, 0, 2175, 1, 1, 1, 15, 2019, 0, 4050, 4044, 1, 1, 1, 1, 4095, 1, 0, 0, 4095, 7, 1, 1, 0, 4086, 1, 4095, 0, 1, 0, 0, 1, 0, 4083, 3892, 0, 1, 0, 0, 0, 3, 4093, 4094, 0, 2, 0, 0, 1, 4086, 0, 0, 1, 0, 1, 4095, 1, 4090, 0, 9, 1, 280, 0, 4095, 4044, 3, 29, 80, 0, 1, 2, 8, 3846, 0, 0, 1, 0, 0, 0, 0, 4095, 2, 1, 1, 1, 14, 133, 4095, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 3724, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3395, 0, 0, 0, 11, 1039, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 4, 0, 0, 19, 0, 4094, 4096, 0, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4075, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4055, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4083, 0, 4052, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3475, 0, 0, 0, 24, 528, 0, 4096, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 23, 0, 0, 0, 4096, 4096, 35, 0, 0, 0, 0, 0, 4096, 4096, 9, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 143, 0, 0, 23, 0, 4041, 4096, 0, 0, 57, 0, 0, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4087, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4094, 0, 4042, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2971, 0, 0, 0, 68, 1102, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 33, 0, 0, 48, 0, 4002, 4096, 0, 0, 42, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4039, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3523, 0, 0, 0, 25, 547, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 11, 0, 4083, 4096, 0, 0, 60, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4095, 0, 0, 4020, 100, 40, 0, 4095, 4094, 4076, 1, 4095, 4088, 1, 4093, 1, 0, 0, 0, 4093, 0, 0, 10, 4091, 0, 3982, 0, 5, 4057, 1, 0, 0, 4095, 1, 1, 0, 0, 3118, 1, 1, 1, 9, 1269, 0, 4061, 4021, 1, 1, 1, 1, 4095, 0, 1, 0, 4095, 15, 1, 1, 0, 4088, 1, 4095, 0, 1, 0, 0, 1, 0, 4088, 3770, 0, 0, 0, 0, 0, 2, 4087, 4095, 0, 1, 0, 0, 1, 4084, 0, 0, 1, 0, 2, 4095, 1, 4092, 0, 6, 1, 322, 0, 4095, 4040, 3, 37, 56, 0, 1, 3, 2, 3832, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 1, 1, 1, 20, 123, 4095, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4095, 1, 0, 4036, 102, 41, 0, 4095, 4094, 4075, 1, 4095, 4095, 1, 4092, 1, 0, 0, 0, 4094, 0, 0, 11, 4092, 0, 3940, 0, 3, 4079, 1, 0, 0, 4095, 1, 1, 0, 0, 1487, 1, 1, 1, 0, 2900, 1, 4053, 4022, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 13, 1, 1, 0, 4078, 1, 4095, 0, 1, 0, 0, 1, 0, 4084, 3833, 0, 0, 0, 0, 0, 0, 4088, 4095, 0, 0, 0, 0, 1, 4084, 0, 0, 1, 0, 1, 4095, 0, 4092, 0, 3, 1, 244, 0, 4095, 4033, 5, 33, 68, 0, 1, 2, 9, 3859, 0, 0, 0, 0, 1, 0, 0, 4095, 1, 1, 1, 1, 19, 135, 4095, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4079, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 4089, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3689, 0, 0, 0, 22, 363, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 2, 0, 0, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 47, 0, 0, 6, 0, 4013, 4096, 0, 0, 79, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 124, 0, 0, 0, 4095, 0, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3395, 0, 0, 0, 11, 1039, 0, 3959, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 42, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 4090, 53, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 19, 0, 4094, 4096, 0, 0, 7, 0, 0, 0, 0, 4096, 0, 4008, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16367, 16135, 16613, 16426, 16478, 16390, 16337, 16326] avg=16384.0000 max=16613 max_rank=2 min=16135 min_rank=1 imbalance_ratio=1.013977 max_load_reduction=114459 max_load_reduction_pct=0.873253 moved_to_echo=65531 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 1, 4095, 0, 0, 4061, 89, 14, 1, 0, 4077, 4080, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 4093, 1, 0, 0, 0, 124, 0, 0, 6, 4079, 0, 4009, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 4094, 1, 1, 0, 0, 2175, 1, 1, 1, 15, 2019, 0, 3957, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 4044, 1, 1, 1, 1, 4095, 1, 0, 0, 42, 7, 1, 1, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 0, 4083, 3892, 0, 1, 0, 0, 0, 3, 4089, 52, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4094, 0, 2, 0, 0, 1, 4086, 0, 0, 1, 0, 1, 6, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 280, 0, 4095, 4044, 3, 29, 80, 0, 1, 2, 8, 3846, 0, 4007, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 2, 1, 1, 1, 14, 133, 4095, 1, 0, 4095, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16421, 16408, 16373, 16331, 16264, 16375, 16418, 16482] avg=16384.0000 max=16482 max_rank=7 min=16264 min_rank=4 imbalance_ratio=1.005981 max_load_reduction=114590 max_load_reduction_pct=0.874252 moved_to_echo=65539 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4075, 0, 0, 4096, 0, 0, 0, 0, 4073, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 4096, 0, 0, 0, 0, 117, 0, 0, 0, 4083, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3475, 0, 0, 0, 24, 528, 0, 3962, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4091, 0, 0, 0, 28, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 23, 0, 0, 0, 4096, 4096, 35, 0, 0, 0, 0, 0, 4091, 54, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 9, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 0, 0, 23, 0, 4041, 4096, 0, 0, 57, 0, 0, 0, 0, 4072, 0, 4015, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16340, 16403, 16204, 16406, 16544, 16398, 16464, 16313] avg=16384.0000 max=16544 max_rank=4 min=16204 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=114528 max_load_reduction_pct=0.873779 moved_to_echo=65719 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 4096, 0, 0, 0, 0, 124, 0, 0, 0, 4094, 0, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 2971, 0, 0, 0, 68, 1102, 0, 3958, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 42, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4090, 53, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 48, 0, 4002, 4096, 0, 0, 42, 0, 0, 0, 0, 4095, 0, 4008, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16367, 16443, 16308, 16419, 16477, 16390, 16337, 16331] avg=16384.0000 max=16477 max_rank=4 min=16308 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=114595 max_load_reduction_pct=0.874290 moved_to_echo=65535 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 124, 0, 0, 0, 4096, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3523, 0, 0, 0, 25, 547, 0, 3959, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 42, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4090, 54, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 11, 0, 4083, 4096, 0, 0, 60, 0, 0, 0, 0, 4096, 0, 4008, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16367, 16451, 16263, 16426, 16478, 16390, 16369, 16328] avg=16384.0000 max=16478 max_rank=4 min=16263 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=114594 max_load_reduction_pct=0.874283 moved_to_echo=65565 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4095, 1, 0, 4036, 102, 41, 0, 0, 4077, 4075, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4092, 1, 0, 0, 0, 124, 0, 0, 11, 4092, 0, 3940, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 1, 0, 0, 4095, 1, 1, 0, 0, 1487, 1, 1, 1, 0, 2900, 1, 3957, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 4022, 1, 1, 1, 1, 4095, 0, 0, 0, 42, 13, 1, 1, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 0, 4084, 3833, 0, 0, 0, 0, 0, 0, 4089, 52, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 4095, 0, 0, 0, 0, 1, 4084, 0, 0, 1, 0, 1, 6, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 244, 0, 4095, 4033, 5, 33, 68, 0, 1, 2, 9, 3859, 0, 4007, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 1, 1, 1, 1, 19, 135, 4095, 1, 0, 4095, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16431, 16359, 16542, 16309, 16202, 16368, 16373, 16488] avg=16384.0000 max=16542 max_rank=2 min=16202 min_rank=4 imbalance_ratio=1.009644 max_load_reduction=114530 max_load_reduction_pct=0.873795 moved_to_echo=65431 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4095, 0, 0, 4020, 100, 40, 0, 0, 4077, 4076, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1, 4093, 1, 0, 0, 0, 124, 0, 0, 10, 4091, 0, 3982, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 1, 0, 0, 4095, 1, 1, 0, 0, 3118, 1, 1, 1, 9, 1269, 0, 3956, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 4021, 1, 1, 1, 1, 4095, 0, 1, 0, 42, 15, 1, 1, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 0, 4088, 3770, 0, 0, 0, 0, 0, 2, 4089, 53, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4095, 0, 1, 0, 0, 1, 4084, 0, 0, 1, 0, 2, 6, 1, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 322, 0, 4095, 4040, 3, 37, 56, 0, 1, 3, 2, 3832, 0, 4007, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 1, 1, 1, 20, 123, 4095, 1, 0, 4095, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16412, 16395, 16527, 16329, 16146, 16370, 16418, 16475] avg=16384.0000 max=16527 max_rank=2 min=16146 min_rank=4 imbalance_ratio=1.008728 max_load_reduction=114545 max_load_reduction_pct=0.873909 moved_to_echo=65409 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4079, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 4096, 0, 0, 0, 0, 124, 0, 0, 0, 4090, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3689, 0, 0, 0, 22, 363, 0, 3959, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 42, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2, 0, 0, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 4090, 53, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 6, 0, 4013, 4096, 0, 0, 79, 0, 0, 0, 0, 4094, 0, 4008, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16367, 16478, 16242, 16426, 16483, 16391, 16356, 16329] avg=16384.0000 max=16483 max_rank=4 min=16242 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=114589 max_load_reduction_pct=0.874245 moved_to_echo=65559 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 32768, 128, 128, 0, 0, 128, 32768, 0, 0, 128, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 32640, 128, 128, 128, 128, 32768, 128, 128, 0, 384, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 0, 0, 32768, 128, 128, 0, 0, 23936, 128, 128, 128, 256, 9856, 128, 31744, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 32640, 384, 128, 128, 0, 32640, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 32768, 128, 0, 0, 0, 1024, 0, 0, 128, 32768, 0, 32000, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 128, 128, 1024, 0, 32640, 32640, 128, 128, 512, 0, 128, 128, 128, 32000, 0, 32128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 128, 0, 32768, 32000, 128, 128, 0, 0, 0, 128, 32768, 512, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 0, 0, 32768, 128, 128, 128, 128, 128, 512, 32768, 128, 0, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4093, 28, 0, 29, 0, 4068, 1, 0, 4095, 1, 3309, 1, 4094, 1, 0, 0, 4086, 2, 0, 1, 1, 4095, 4095, 0, 1, 4095, 4080, 4081, 0, 0, 1, 1, 1, 0, 0, 1, 1, 106, 9, 1, 0, 124, 1, 0, 4001, 0, 0, 1, 4094, 1, 4057, 22, 0, 0, 4076, 4095, 1, 1, 1, 0, 0, 1, 4014, 17, 1, 4, 4074, 4095, 1, 0, 0, 1, 1, 0, 1, 1, 4095, 0, 1, 1, 2, 734, 0, 0, 4093, 4095, 0, 4087, 0, 1, 0, 1, 0, 4039, 1, 4095, 0, 4063, 4095, 0, 0, 16, 4095, 1, 122, 0, 0, 0, 1, 1, 0, 1, 4085, 0, 0, 0, 1, 4095, 4091, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 2, 4092, 31, 0, 27, 0, 4065, 1, 0, 4094, 1, 3124, 1, 4093, 1, 0, 0, 4079, 0, 0, 1, 1, 4095, 4094, 0, 1, 4095, 4091, 4087, 0, 3, 1, 1, 1, 1, 0, 1, 1, 28, 7, 0, 0, 354, 1, 0, 3997, 0, 0, 1, 4093, 1, 4067, 21, 0, 0, 4093, 4095, 0, 1, 1, 0, 0, 1, 4028, 3, 1, 4, 4078, 4095, 1, 1, 0, 0, 1, 0, 0, 1, 4095, 1, 0, 1, 5, 806, 0, 0, 4092, 4094, 1, 4094, 0, 1, 0, 1, 0, 4007, 1, 4095, 0, 4076, 4094, 1, 0, 19, 4095, 1, 87, 0, 0, 1, 1, 1, 0, 1, 4056, 1, 0, 0, 0, 4095, 4093, 0, 2, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 33, 0, 0, 4096, 0, 6, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 4065, 4096, 0, 4096, 0, 0, 55, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 8, 0, 0, 0, 4096, 0, 0, 4096, 0, 4093, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4096, 0, 65, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 4096, 4096, 0, 13, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 4040, 4086, 0, 4096, 0, 0, 48, 0, 0, 4082, 0, 4096, 0, 4096, 4091, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4036, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 13, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 4096, 0, 0, 10, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4094, 20, 0, 22, 0, 4073, 1, 0, 4095, 1, 3361, 1, 4094, 0, 0, 0, 4080, 1, 0, 1, 1, 4095, 4095, 0, 1, 4095, 4069, 4092, 0, 0, 2, 1, 1, 0, 0, 1, 1, 46, 2, 2, 0, 210, 1, 0, 3997, 0, 0, 1, 4094, 1, 4057, 24, 0, 0, 4079, 4095, 0, 1, 1, 0, 0, 2, 4030, 17, 1, 3, 4083, 4095, 2, 0, 0, 1, 1, 0, 0, 1, 4095, 0, 1, 1, 7, 723, 0, 0, 4092, 4095, 0, 4093, 0, 0, 0, 1, 0, 3945, 1, 4095, 0, 4067, 4095, 0, 0, 33, 4095, 1, 105, 0, 1, 0, 1, 1, 0, 1, 4094, 0, 0, 0, 1, 4095, 4092, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 0, 6, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 32, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 4062, 4095, 0, 4095, 0, 0, 5, 0, 0, 4095, 0, 4096, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[310, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 4094, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 3838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3786, 258, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16693, 16399, 16256, 16422, 16403, 16123, 16386, 16390] avg=16384.0000 max=16693 max_rank=0 min=16123 min_rank=5 imbalance_ratio=1.018860 max_load_reduction=114379 max_load_reduction_pct=0.872643 moved_to_echo=65302 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[310, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 33, 0, 0, 4096, 4096, 4094, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 4096, 0, 4096, 0, 0, 0, 4096, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 0, 0, 0, 0, 0, 0, 0, 4065, 4096, 0, 3838, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3786, 258, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16694, 16399, 16290, 16428, 16408, 16076, 16386, 16391] avg=16384.0000 max=16694 max_rank=0 min=16076 min_rank=5 imbalance_ratio=1.018921 max_load_reduction=114378 max_load_reduction_pct=0.872635 moved_to_echo=65261 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[310, 0, 0, 2, 4092, 31, 0, 27, 0, 4065, 1, 0, 4094, 1, 3124, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 4079, 0, 0, 1, 1, 0, 16, 0, 1, 0, 4091, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 1, 1, 1, 0, 1, 1, 28, 7, 0, 0, 354, 1, 0, 4095, 4095, 4093, 3966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 1, 4093, 1, 4067, 21, 0, 0, 4093, 38, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4028, 3, 1, 4, 4078, 4095, 1, 1, 0, 0, 1, 0, 0, 1, 3997, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 5, 806, 0, 0, 4092, 4094, 1, 3837, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 1, 2, 0, 4076, 4094, 1, 0, 19, 4095, 1, 87, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 4056, 1, 0, 0, 0, 4095, 4093, 0, 2, 0, 0, 0, 3785, 257, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15748, 16370, 16648, 16313, 16323, 16934, 16384, 16352] avg=16384.0000 max=16934 max_rank=5 min=15748 min_rank=0 imbalance_ratio=1.033569 max_load_reduction=114138 max_load_reduction_pct=0.870804 moved_to_echo=65993 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[310, 0, 0, 0, 4093, 28, 0, 29, 0, 4068, 1, 0, 4095, 1, 3309, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 4086, 2, 0, 1, 1, 0, 16, 0, 1, 0, 4080, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 106, 9, 1, 0, 124, 1, 0, 4095, 4095, 4093, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 1, 4094, 1, 4057, 22, 0, 0, 4076, 38, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4014, 17, 1, 4, 4074, 4095, 1, 0, 0, 1, 1, 0, 1, 1, 3997, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 2, 734, 0, 0, 4093, 4095, 0, 3830, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 1, 2, 0, 4063, 4095, 0, 0, 16, 4095, 1, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 4085, 0, 0, 0, 1, 4095, 4091, 0, 1, 0, 0, 0, 3785, 257, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15935, 16363, 16496, 16293, 16320, 16853, 16434, 16378] avg=16384.0000 max=16853 max_rank=5 min=15935 min_rank=0 imbalance_ratio=1.028625 max_load_reduction=114219 max_load_reduction_pct=0.871422 moved_to_echo=65985 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[307, 0, 0, 0, 4096, 8, 0, 0, 0, 4096, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4096, 4096, 4096, 3974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 65, 0, 4096, 0, 4096, 0, 0, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 6, 4096, 4096, 0, 13, 0, 0, 0, 0, 0, 0, 4005, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 0, 0, 0, 0, 0, 0, 4040, 4086, 0, 3835, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 4096, 4091, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3789, 261, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16696, 16388, 16280, 16473, 16428, 16048, 16368, 16391] avg=16384.0000 max=16696 max_rank=0 min=16048 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=114376 max_load_reduction_pct=0.872620 moved_to_echo=65235 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[310, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0, 0, 8, 0, 0, 4096, 4096, 4095, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 3838, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3786, 258, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16634, 16399, 16315, 16422, 16412, 16114, 16385, 16391] avg=16384.0000 max=16634 max_rank=0 min=16114 min_rank=5 imbalance_ratio=1.015259 max_load_reduction=114438 max_load_reduction_pct=0.873093 moved_to_echo=65302 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[310, 0, 0, 0, 4094, 20, 0, 22, 0, 4073, 1, 0, 4095, 1, 3361, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4080, 1, 0, 1, 1, 0, 16, 0, 1, 0, 4069, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 0, 1, 1, 46, 2, 2, 0, 210, 1, 0, 4095, 4095, 4095, 3967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3997, 0, 0, 1, 4094, 1, 4057, 24, 0, 0, 4079, 38, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4030, 17, 1, 3, 4083, 4095, 2, 0, 0, 1, 1, 0, 0, 1, 3997, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 7, 723, 0, 0, 4092, 4095, 0, 3836, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 1, 0, 0, 4067, 4095, 0, 0, 33, 4095, 1, 105, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 4094, 0, 0, 0, 1, 4095, 4092, 0, 0, 1, 0, 0, 3785, 257, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15978, 16355, 16519, 16293, 16345, 16851, 16343, 16388] avg=16384.0000 max=16851 max_rank=5 min=15978 min_rank=0 imbalance_ratio=1.028503 max_load_reduction=114221 max_load_reduction_pct=0.871437 moved_to_echo=65927 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[310, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 4096, 4096, 4094, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 4096, 0, 4096, 0, 0, 0, 4096, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 3998, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 4062, 4095, 0, 3837, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 4096, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3786, 258, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16694, 16399, 16268, 16428, 16433, 16073, 16386, 16391] avg=16384.0000 max=16694 max_rank=0 min=16073 min_rank=5 imbalance_ratio=1.018921 max_load_reduction=114378 max_load_reduction_pct=0.872635 moved_to_echo=65283 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32512, 128, 128, 0, 32768, 32768, 128, 0, 128, 32768, 128, 384, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 0, 0, 32768, 128, 0, 128, 128, 0, 128, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32512, 0, 128, 128, 32768, 128, 32768, 128, 0, 0, 32768, 384, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[2560, 0, 0, 128, 32768, 128, 0, 128, 0, 32768, 128, 0, 32768, 128, 30336, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 32640, 128, 128, 128, 32768, 32768, 128, 128, 128, 128, 128, 0, 128, 128, 32000, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0, 128, 32768, 32768, 0, 128, 128, 0, 0, 30336, 2176, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 128, 256, 128, 128, 0, 768, 128, 0, 32768, 32768, 32768, 31872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32640, 128, 128, 128, 128, 2304, 0, 0, 32640, 32768, 128, 30720, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3204, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3936, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1538, 3915, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 32, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2221, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 3893, 4096, 0, 0, 4077, 0, 0, 4096, 0, 0, 0, 4096, 38, 0, 0, 2, 0, 218, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3917, 0, 0, 0, 0, 0, 1287, 410, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3716, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4050, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1991, 3621, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 74, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1289, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 3998, 4096, 0, 0, 4073, 0, 0, 4096, 0, 0, 0, 4096, 27, 0, 0, 1, 0, 268, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 1246, 239, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4048, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2307, 2406, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 7, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3206, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4089, 4096, 0, 0, 4057, 0, 0, 4096, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 77, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3653, 0, 0, 0, 0, 0, 586, 146, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1, 1, 4067, 1, 1, 0, 0, 0, 4094, 1, 1, 3824, 3, 1, 17, 2, 4091, 1, 4095, 0, 0, 0, 0, 4095, 4095, 1, 4084, 1, 1, 8, 4088, 1, 1, 1, 0, 0, 4065, 0, 4, 0, 15, 3452, 7, 2, 4074, 0, 0, 3, 0, 0, 0, 0, 0, 4086, 0, 4094, 4093, 0, 1, 0, 0, 4095, 4095, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 4093, 1, 1, 1, 2, 1452, 4095, 250, 20, 4095, 1, 4094, 1, 0, 4095, 4094, 4080, 1, 0, 3535, 0, 0, 4095, 0, 1, 0, 4066, 0, 0, 24, 0, 1, 1472, 4095, 0, 0, 1, 4095, 206, 0, 1, 47, 0, 1921, 0, 3, 0, 1, 1, 2009, 2362, 4092, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 12, 0, 4096, 0, 0, 3332, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3910, 0, 0, 0, 4090, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 2492, 3523, 0, 0, 4075, 0, 0, 0, 23, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 60, 0, 4096, 4062, 0, 4, 5, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 1670, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4090, 4016, 4096, 0, 0, 4066, 0, 0, 4096, 8, 0, 0, 4096, 62, 0, 0, 25, 0, 442, 4083, 0, 0, 0, 4092, 4, 0, 0, 0, 0, 3846, 0, 0, 0, 0, 0, 922, 343, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 2, 4056, 1, 1, 0, 0, 0, 4094, 1, 1, 3997, 4, 1, 9, 2, 4090, 1, 4095, 0, 0, 0, 0, 4095, 4095, 1, 4085, 1, 1, 15, 4091, 1, 1, 1, 0, 0, 3933, 0, 5, 0, 81, 2051, 4, 0, 4038, 0, 0, 3, 0, 0, 0, 0, 0, 4086, 1, 4095, 4092, 0, 1, 1, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 4095, 1, 1, 1, 0, 2846, 4095, 177, 15, 4095, 1, 4094, 1, 0, 4095, 4090, 4073, 1, 0, 3593, 0, 0, 4095, 0, 1, 0, 4068, 0, 0, 37, 0, 1, 1348, 4095, 0, 0, 1, 4095, 207, 0, 1, 49, 0, 1615, 1, 3, 0, 1, 1, 2599, 2161, 4092, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 3695, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4033, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1377, 3747, 0, 0, 4088, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 67, 0, 4096, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2805, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 3902, 4096, 0, 0, 4033, 0, 0, 4096, 0, 0, 0, 4096, 47, 0, 0, 8, 0, 31, 4095, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 424, 423, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[2, 1, 4070, 1, 1, 1, 1, 0, 4094, 1, 1, 3985, 4, 1, 11, 1, 4092, 1, 4095, 0, 0, 0, 1, 4094, 4095, 0, 4068, 1, 1, 18, 4089, 1, 1, 1, 0, 0, 3982, 0, 10, 0, 123, 2816, 7, 2, 4086, 0, 0, 0, 1, 0, 0, 0, 0, 4088, 0, 4095, 4092, 0, 1, 0, 0, 4095, 4095, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 4088, 1, 1, 1, 3, 3024, 4095, 287, 6, 4095, 2, 4094, 0, 0, 4095, 4085, 4085, 0, 0, 3445, 0, 0, 4095, 1, 1, 0, 4073, 0, 0, 26, 0, 0, 1352, 4092, 1, 0, 1, 4095, 196, 0, 1, 80, 0, 1845, 1, 4, 0, 2, 1, 1089, 2383, 4094, 0, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3204, 0, 0, 0, 0, 3691, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 63, 0, 3936, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1538, 3915, 0, 0, 4081, 0, 0, 0, 3256, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4073, 4096, 0, 0, 32, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2221, 4096, 4096, 1377, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 0, 4096, 3893, 4096, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 38, 0, 0, 2, 0, 218, 4095, 0, 0, 0, 4096, 0, 3318, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3917, 0, 0, 0, 0, 0, 1287, 410, 4096, 0, 0, 0, 4096, 2719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15864, 16287, 17439, 16393, 16291, 16387, 15886, 16525] avg=16384.0000 max=17439 max_rank=2 min=15864 min_rank=0 imbalance_ratio=1.064392 max_load_reduction=113633 max_load_reduction_pct=0.866951 moved_to_echo=65089 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4048, 0, 0, 0, 0, 3691, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 63, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 2307, 2406, 0, 0, 4081, 0, 0, 0, 3256, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4073, 4096, 0, 0, 7, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3206, 4096, 4096, 1377, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 0, 4096, 4089, 4096, 0, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 77, 4096, 0, 0, 0, 4096, 0, 3318, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3653, 0, 0, 0, 0, 0, 586, 146, 4096, 0, 0, 0, 4096, 2719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16708, 16447, 16699, 16368, 17276, 16563, 15715, 15296] avg=16384.0000 max=17276 max_rank=4 min=15296 min_rank=7 imbalance_ratio=1.054443 max_load_reduction=113796 max_load_reduction_pct=0.868195 moved_to_echo=64850 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3716, 0, 0, 0, 0, 3692, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 63, 0, 4050, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1991, 3621, 0, 0, 4095, 0, 0, 0, 3256, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4073, 4096, 0, 0, 74, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1289, 4096, 4096, 1378, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 0, 4096, 3998, 4096, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 27, 0, 0, 1, 0, 268, 4091, 0, 0, 0, 4096, 0, 3318, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 1246, 239, 4096, 0, 0, 0, 4096, 2718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16401, 17612, 16435, 15359, 16488, 15920, 16480] avg=16384.0000 max=17612 max_rank=2 min=15359 min_rank=4 imbalance_ratio=1.074951 max_load_reduction=113460 max_load_reduction_pct=0.865631 moved_to_echo=64247 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 12, 0, 4096, 0, 0, 3332, 0, 0, 0, 0, 3692, 780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 55, 0, 3910, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 0, 2492, 3523, 0, 0, 4075, 0, 0, 0, 3261, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 4096, 0, 4067, 4096, 0, 0, 60, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 1670, 4096, 4096, 1378, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 4090, 4016, 4096, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 62, 0, 0, 25, 0, 442, 4083, 0, 0, 0, 4092, 4, 3323, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 0, 0, 0, 0, 0, 922, 343, 4096, 0, 0, 0, 4096, 2718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16008, 16247, 18009, 16404, 15737, 16482, 16164, 16021] avg=16384.0000 max=18009 max_rank=2 min=15737 min_rank=4 imbalance_ratio=1.099182 max_load_reduction=113063 max_load_reduction_pct=0.862602 moved_to_echo=64404 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[2, 1, 4070, 1, 1, 1, 1, 0, 4094, 1, 1, 3985, 4, 1, 11, 1, 3690, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 1, 4094, 63, 0, 4068, 1, 1, 18, 4089, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 3982, 0, 10, 0, 123, 2816, 7, 2, 4086, 0, 0, 0, 3256, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4088, 0, 4072, 4092, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 4088, 1, 1, 1, 3, 3024, 4095, 4095, 1376, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 6, 0, 2, 224, 0, 0, 4095, 4085, 4085, 0, 0, 3445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4073, 0, 0, 26, 0, 0, 1352, 4092, 1, 0, 1, 4095, 196, 3317, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 80, 0, 1845, 1, 4, 0, 2, 1, 1089, 2383, 4094, 0, 1, 1, 4095, 2719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16641, 16429, 14837, 16349, 17093, 16229, 17178, 16316] avg=16384.0000 max=17178 max_rank=6 min=14837 min_rank=2 imbalance_ratio=1.048462 max_load_reduction=113894 max_load_reduction_pct=0.868942 moved_to_echo=66816 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4056, 1, 1, 0, 0, 0, 4094, 1, 1, 3997, 4, 1, 9, 2, 3691, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 0, 0, 0, 0, 4095, 63, 1, 4085, 1, 1, 15, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 3933, 0, 5, 0, 81, 2051, 4, 0, 4038, 0, 0, 3, 3256, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 4072, 4092, 0, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 4095, 1, 1, 1, 0, 2846, 4095, 4095, 1376, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 15, 0, 1, 224, 1, 0, 4095, 4090, 4073, 1, 0, 3593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4068, 0, 0, 37, 0, 1, 1348, 4095, 0, 0, 1, 4095, 207, 3317, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 49, 0, 1615, 1, 3, 0, 1, 1, 2599, 2161, 4092, 0, 1, 0, 4095, 2719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16636, 16444, 13926, 16348, 16917, 16270, 17193, 17338] avg=16384.0000 max=17338 max_rank=7 min=13926 min_rank=2 imbalance_ratio=1.058228 max_load_reduction=113734 max_load_reduction_pct=0.867722 moved_to_echo=67718 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 4067, 1, 1, 0, 0, 0, 4094, 1, 1, 3824, 3, 1, 17, 2, 3691, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 0, 0, 0, 4095, 63, 1, 4084, 1, 1, 8, 4088, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4065, 0, 4, 0, 15, 3452, 7, 2, 4074, 0, 0, 3, 3256, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 4071, 4093, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 4093, 1, 1, 1, 2, 1452, 4095, 4095, 1377, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 20, 0, 1, 224, 1, 0, 4095, 4094, 4080, 1, 0, 3535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4066, 0, 0, 24, 0, 1, 1472, 4095, 0, 0, 1, 4095, 206, 3317, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 47, 0, 1921, 0, 3, 0, 1, 1, 2009, 2362, 4092, 0, 1, 0, 4095, 2718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16481, 16434, 15433, 16346, 15525, 16301, 17301, 17251] avg=16384.0000 max=17301 max_rank=6 min=15433 min_rank=2 imbalance_ratio=1.055969 max_load_reduction=113771 max_load_reduction_pct=0.868004 moved_to_echo=66378 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 3695, 0, 0, 0, 0, 3692, 777, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 63, 0, 4033, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1377, 3747, 0, 0, 4088, 0, 0, 0, 3256, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4073, 4096, 0, 0, 67, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2805, 4096, 4096, 1377, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 0, 4096, 3902, 4096, 0, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 47, 0, 0, 8, 0, 31, 4095, 0, 0, 0, 4096, 1, 3318, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 424, 423, 4096, 0, 0, 0, 4096, 2719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16357, 16383, 17117, 16429, 16874, 16352, 15715, 15845] avg=16384.0000 max=17117 max_rank=2 min=15715 min_rank=6 imbalance_ratio=1.044739 max_load_reduction=113955 max_load_reduction_pct=0.869408 moved_to_echo=64786 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 128, 32768, 512, 128, 32384, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 0, 32768, 128, 32640, 32768, 0, 128, 256, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 32768, 256, 0, 128, 128, 128, 5248, 32768, 128, 0, 128, 32768, 640, 26624, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 256, 0, 25088, 128, 128, 0, 128, 128, 10240, 8576, 32768, 0, 128, 128, 32768, 21760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 0, 32512, 0, 128, 0, 9984, 25600, 128, 128, 32640, 0, 0, 128, 26112, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 128, 0, 32768, 128, 128, 29824, 128, 128, 128, 128, 29568, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[768, 128, 0, 128, 1792, 128, 0, 32768, 32256, 32768, 128, 0, 30976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 18560, 32768, 32768, 11136, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 861, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 953, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2286, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4093, 0, 0, 1, 9, 0, 4090, 0, 0, 0, 249, 4086, 0, 4091, 0, 4094, 0, 7, 1, 0, 12, 4095, 0, 0, 1, 1, 0, 0, 0, 1, 4083, 92, 4087, 1, 1, 0, 1, 4095, 0, 1, 1, 1, 2, 1, 3, 4094, 2, 0, 2952, 1, 2, 0, 4082, 1, 6, 1, 0, 1, 0, 0, 0, 25, 0, 2, 14, 1, 1, 1, 911, 4095, 1, 0, 4008, 4085, 1, 4095, 0, 3975, 4092, 135, 4095, 0, 0, 70, 0, 1, 4046, 4093, 0, 0, 3991, 0, 1, 0, 1, 0, 4, 0, 4095, 4095, 4063, 1, 4095, 0, 4093, 0, 4095, 4095, 0, 1, 0, 1, 4095, 154, 0, 4095, 0, 1, 1, 0, 0, 0, 1, 4003] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4093, 0, 0, 1, 5, 0, 4092, 0, 0, 0, 289, 4095, 0, 4091, 0, 4094, 0, 5, 0, 0, 11, 4095, 0, 1, 1, 1, 0, 0, 0, 0, 4087, 47, 4087, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 1, 0, 2, 4094, 3, 0, 3462, 1, 2, 0, 4095, 1, 3, 1, 0, 2, 0, 2, 0, 22, 0, 1, 11, 1, 1, 1, 343, 4095, 1, 1, 4016, 4084, 1, 4095, 1, 3975, 4093, 138, 4095, 0, 0, 10, 0, 1, 4041, 4088, 0, 1, 4006, 0, 1, 0, 1, 0, 3, 0, 4095, 4095, 4073, 1, 4095, 0, 4092, 0, 4095, 4095, 0, 1, 0, 1, 4095, 189, 0, 4094, 0, 1, 0, 0, 1, 0, 1, 4055] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 501, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2585, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 4096, 0, 0, 4096, 4096, 0, 4095, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 425, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 735, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2916, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 22, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 814, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1731, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1562, 4085, 0, 0, 4096, 4096, 0, 4074, 16, 4096, 4096, 1, 4096, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 4086, 2, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 0, 1, 7, 0, 4092, 0, 0, 0, 271, 4084, 0, 4091, 0, 4094, 0, 7, 1, 0, 11, 4095, 0, 0, 1, 1, 0, 0, 0, 1, 4079, 67, 4085, 1, 1, 0, 1, 4095, 0, 1, 1, 1, 2, 1, 3, 4095, 2, 0, 3704, 1, 2, 0, 4089, 1, 6, 1, 0, 0, 0, 0, 0, 29, 0, 2, 16, 1, 1, 1, 334, 4095, 1, 0, 4023, 4083, 1, 4095, 0, 3918, 4093, 131, 4095, 0, 0, 41, 0, 1, 4050, 4080, 0, 0, 3982, 0, 1, 0, 1, 0, 5, 0, 4095, 4095, 4059, 1, 4095, 0, 4093, 0, 4095, 4095, 0, 1, 0, 1, 4095, 80, 0, 4095, 0, 1, 1, 0, 1, 0, 1, 3996] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 104, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 1, 4055, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2871, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 4095, 0, 0, 4096, 4096, 0, 4095, 1, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4095, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 861, 4096, 2366, 1293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2376, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2383, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 953, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2286, 2803, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 47, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2384, 1683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16808, 16377, 16367, 14961, 17377, 16431, 16396, 16355] avg=16384.0000 max=17377 max_rank=4 min=14961 min_rank=3 imbalance_ratio=1.060608 max_load_reduction=113695 max_load_reduction_pct=0.867424 moved_to_echo=66559 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4093, 0, 0, 1, 5, 0, 4092, 0, 0, 0, 289, 4095, 2364, 1293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4094, 0, 5, 0, 0, 11, 4095, 0, 1, 1, 1, 0, 0, 2375, 1714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 47, 4087, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 1, 0, 2381, 1699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 3, 0, 3462, 1, 2, 0, 4095, 1, 3, 1, 0, 2, 0, 2, 4095, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 1, 11, 1, 1, 1, 343, 2802, 1, 1, 4016, 4084, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3975, 47, 138, 0, 0, 0, 10, 0, 1, 4041, 4088, 0, 1, 4006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 3, 0, 0, 0, 4073, 1, 12, 0, 4092, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 189, 0, 4094, 0, 1, 0, 0, 1, 0, 1, 4055, 2384, 1682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16234, 16388, 16402, 17483, 15380, 16308, 16373, 16504] avg=16384.0000 max=17483 max_rank=3 min=15380 min_rank=4 imbalance_ratio=1.067078 max_load_reduction=113589 max_load_reduction_pct=0.866615 moved_to_echo=64565 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 501, 4096, 2366, 1293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2377, 1714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2382, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2585, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 2803, 0, 0, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 47, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2384, 1683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16448, 16379, 16362, 16592, 16103, 16431, 16396, 16361] avg=16384.0000 max=16592 max_rank=3 min=16103 min_rank=4 imbalance_ratio=1.012695 max_load_reduction=114480 max_load_reduction_pct=0.873413 moved_to_echo=65291 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 425, 4096, 2366, 1293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2376, 1713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2383, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 735, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2916, 2803, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 47, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 22, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 2384, 1683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 16377, 16371, 14743, 18007, 16431, 16396, 16375] avg=16384.0000 max=18007 max_rank=4 min=14743 min_rank=3 imbalance_ratio=1.099060 max_load_reduction=113065 max_load_reduction_pct=0.862617 moved_to_echo=67209 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 814, 4096, 2369, 1293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 0, 0, 2377, 1714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2382, 1703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1731, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 1719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1562, 2792, 0, 0, 4096, 4096, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 4096, 39, 1, 0, 0, 0, 0, 0, 0, 4088, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 2, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2387, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16764, 16390, 16377, 15736, 16620, 16432, 16388, 16365] avg=16384.0000 max=16764 max_rank=0 min=15736 min_rank=3 imbalance_ratio=1.023193 max_load_reduction=114308 max_load_reduction_pct=0.872101 moved_to_echo=65805 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4093, 0, 0, 1, 9, 0, 4090, 0, 0, 0, 249, 4086, 2364, 1293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4094, 0, 7, 1, 0, 12, 4095, 0, 0, 1, 1, 0, 0, 2376, 1714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4083, 92, 4087, 1, 1, 0, 1, 4095, 0, 1, 1, 1, 2, 1, 2381, 1699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 2, 0, 2952, 1, 2, 0, 4082, 1, 6, 1, 0, 1, 0, 0, 4095, 1719, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 2, 14, 1, 1, 1, 911, 2802, 1, 0, 4008, 4085, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 46, 135, 0, 0, 0, 70, 0, 1, 4046, 4093, 0, 0, 3991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4, 0, 0, 0, 4063, 1, 12, 0, 4093, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 154, 0, 4095, 0, 1, 1, 0, 0, 0, 1, 4003, 2384, 1682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16187, 16392, 16447, 16959, 15947, 16357, 16365, 16418] avg=16384.0000 max=16959 max_rank=3 min=15947 min_rank=4 imbalance_ratio=1.035095 max_load_reduction=114113 max_load_reduction_pct=0.870613 moved_to_echo=65087 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 0, 1, 7, 0, 4092, 0, 0, 0, 271, 4084, 2364, 1293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4094, 0, 7, 1, 0, 11, 4095, 0, 0, 1, 1, 0, 0, 2375, 1714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4079, 67, 4085, 1, 1, 0, 1, 4095, 0, 1, 1, 1, 2, 1, 2381, 1699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 2, 0, 3704, 1, 2, 0, 4089, 1, 6, 1, 0, 0, 0, 0, 4095, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 2, 16, 1, 1, 1, 334, 2802, 1, 0, 4023, 4083, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3918, 47, 131, 0, 0, 0, 41, 0, 1, 4050, 4080, 0, 0, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 5, 0, 0, 0, 4059, 1, 12, 0, 4093, 0, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 80, 0, 4095, 0, 1, 1, 0, 1, 0, 1, 3996, 2384, 1682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16208, 16390, 16416, 17719, 15389, 16250, 16362, 16338] avg=16384.0000 max=17719 max_rank=3 min=15389 min_rank=4 imbalance_ratio=1.081482 max_load_reduction=113353 max_load_reduction_pct=0.864815 moved_to_echo=64339 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 104, 4096, 2366, 1293, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2376, 1714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4055, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2382, 1700, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2871, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1160, 2802, 0, 0, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 47, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 12, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2384, 1683, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16051, 16379, 16330, 16879, 16249, 16432, 16396, 16356] avg=16384.0000 max=16879 max_rank=3 min=16051 min_rank=0 imbalance_ratio=1.030212 max_load_reduction=114193 max_load_reduction_pct=0.871223 moved_to_echo=65433 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 32768, 256, 32768, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 128, 128, 19072, 13696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 32768, 512, 128, 32768, 0, 128, 128, 0, 128, 0, 128, 32640, 19200, 13568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 128, 128, 0, 32768, 0, 0, 0, 3584, 32768, 18944, 10368, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 0, 32768, 0, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 0, 0, 19072, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 0, 19072, 128, 128, 0, 32768, 128, 128, 128, 0, 128, 0, 128, 32768, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 32384, 384, 512, 0, 0, 0, 128, 0, 128, 32640, 32768, 0, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 0, 32768, 128, 128, 0, 32768, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 128, 10624, 22528, 128, 128, 32640, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 86, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 763, 4096, 0, 4096, 0, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 369, 4096, 4096, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 2609, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 238, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 3615, 0, 0, 0, 4096, 4095, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 128, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1332, 4096, 0, 4096, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 178, 4096, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 1386, 0, 4083, 0, 4081, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1377, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 3716, 0, 0, 0, 4096, 4094, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 51, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1893, 4096, 0, 4083, 0, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1275, 4096, 4096, 0, 0, 0, 2, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 226, 0, 4095, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 109, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4059, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 1, 0, 2, 0, 0, 0, 4066, 98, 1, 159, 11, 1, 0, 0, 4095, 4094, 1, 0, 0, 4095, 0, 0, 2, 4091, 7, 1, 2115, 4095, 0, 4085, 6, 218, 1, 1, 25, 1, 0, 42, 0, 1, 0, 4, 0, 1, 1, 1, 4, 0, 0, 0, 7, 0, 4083, 4093, 98, 4094, 4095, 0, 0, 0, 3, 3650, 0, 2, 1, 0, 1, 0, 0, 4087, 11, 1, 0, 0, 1, 1, 4085, 4090, 4079, 0, 1972, 0, 4095, 0, 3942, 0, 4094, 16, 1, 0, 0, 4089, 0, 1, 4094, 4094, 0, 1, 4095, 0, 0, 0, 4094, 1, 1, 1, 0, 237, 1, 4095, 4052, 0, 0, 4095, 2, 1, 1, 3985, 1, 2, 1, 4094, 4095, 4009] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 226, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4, 4096, 0, 14, 0, 4096, 0, 0, 1139, 4096, 0, 4030, 0, 187, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4096, 4096, 369, 4095, 4096, 0, 0, 0, 26, 4065, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 4096, 4096, 4096, 0, 2140, 0, 4095, 0, 4084, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 430, 0, 4096, 4087, 0, 8, 4079, 0, 0, 0, 3765, 0, 0, 0, 4096, 4072, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4070, 82, 1, 145, 13, 1, 0, 0, 4095, 4093, 1, 0, 0, 4095, 0, 0, 2, 4090, 33, 0, 2614, 4095, 0, 4039, 7, 409, 1, 1, 9, 1, 0, 19, 0, 1, 0, 6, 0, 1, 1, 1, 5, 0, 1, 0, 3, 0, 4078, 4095, 793, 4095, 4095, 0, 0, 0, 0, 3690, 0, 1, 1, 15, 1, 0, 0, 4084, 9, 1, 0, 0, 0, 1, 4084, 4089, 4073, 0, 897, 0, 4090, 0, 3637, 1, 4094, 11, 1, 0, 0, 4089, 1, 1, 4095, 4095, 0, 1, 4095, 0, 0, 1, 4095, 1, 1, 1, 0, 214, 1, 4095, 4068, 0, 0, 4095, 1, 0, 1, 4044, 1, 2, 0, 4095, 4095, 4011] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 4080, 99, 1, 141, 12, 1, 0, 0, 4095, 4093, 1, 0, 0, 4095, 0, 0, 1, 4091, 41, 1, 2598, 4095, 0, 4056, 7, 804, 1, 1, 15, 1, 0, 19, 0, 1, 0, 4, 0, 1, 1, 1, 3, 0, 1, 0, 5, 0, 4077, 4095, 126, 4094, 4095, 0, 0, 0, 0, 3647, 0, 2, 1, 23, 1, 0, 0, 4084, 10, 1, 0, 0, 0, 1, 4082, 4089, 4076, 0, 720, 0, 4094, 0, 3731, 1, 4094, 10, 1, 0, 0, 4088, 1, 1, 4095, 4095, 0, 1, 4095, 0, 0, 1, 4095, 0, 1, 1, 0, 512, 1, 4095, 4062, 0, 0, 4095, 1, 0, 1, 4084, 1, 3, 0, 4095, 4095, 4029] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 32, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1990, 4096, 0, 4065, 0, 146, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 4096, 670, 4096, 4096, 0, 0, 0, 12, 4076, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 1462, 0, 4092, 0, 4075, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 74, 0, 4096, 4096, 0, 0, 4094, 0, 0, 0, 3881, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 51, 0, 0, 0, 0, 4096, 4096, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2281, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1893, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3564, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1275, 4096, 3613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3849, 1815, 1288, 532, 483, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 226, 0, 2807, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3697, 0, 0, 0, 4096, 0, 0, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 247, 0, 0, 0, 4059, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16282, 16462, 16587, 17176, 16556, 15321, 16094, 16594] avg=16384.0000 max=17176 max_rank=3 min=15321 min_rank=5 imbalance_ratio=1.048340 max_load_reduction=113896 max_load_reduction_pct=0.868958 moved_to_echo=64565 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 128, 0, 0, 0, 0, 4096, 4096, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2281, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1332, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3565, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 178, 4096, 3613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3849, 1815, 1284, 531, 483, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1386, 0, 2799, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3697, 0, 0, 0, 4096, 0, 0, 0, 0, 1377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 247, 0, 0, 0, 3716, 0, 0, 0, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16359, 15901, 16122, 16079, 16542, 16458, 17362, 16249] avg=16384.0000 max=17362 max_rank=6 min=15901 min_rank=1 imbalance_ratio=1.059692 max_load_reduction=113710 max_load_reduction_pct=0.867538 moved_to_echo=66611 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 86, 0, 0, 0, 0, 4096, 4096, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2281, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 763, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3564, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 369, 4096, 3613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3849, 1815, 1288, 532, 483, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2609, 0, 2808, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3697, 0, 0, 0, 4096, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 247, 0, 0, 0, 3615, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16316, 15332, 16630, 16270, 16447, 17705, 16223, 16149] avg=16384.0000 max=17705 max_rank=5 min=15332 min_rank=1 imbalance_ratio=1.080627 max_load_reduction=113367 max_load_reduction_pct=0.864922 moved_to_echo=66524 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 1, 0, 2, 0, 0, 0, 4066, 98, 1, 159, 11, 1, 4095, 4095, 3941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2280, 4094, 1, 0, 0, 0, 0, 0, 2, 4091, 7, 1, 2115, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 6, 218, 1, 1, 25, 1, 0, 42, 0, 1, 0, 4, 0, 1, 4094, 4090, 3558, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4, 0, 0, 0, 7, 0, 4083, 4093, 98, 4094, 3612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3650, 0, 2, 1, 0, 1, 0, 0, 4087, 11, 1, 0, 0, 1, 1, 3848, 1815, 1288, 531, 483, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 4079, 0, 1972, 0, 2807, 0, 3942, 0, 0, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 4094, 0, 1, 3696, 0, 0, 0, 4094, 1, 1, 1, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4052, 0, 0, 247, 2, 1, 1, 3985, 1, 2, 1, 0, 4095, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16472, 16686, 16280, 15993, 16122, 16902, 16220, 16397] avg=16384.0000 max=16902 max_rank=5 min=15993 min_rank=3 imbalance_ratio=1.031616 max_load_reduction=114170 max_load_reduction_pct=0.871048 moved_to_echo=65641 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 4096, 226, 0, 0, 0, 0, 4096, 4096, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2278, 4096, 0, 0, 4, 0, 0, 14, 0, 4096, 0, 0, 1139, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 0, 187, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3565, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4096, 4096, 369, 4095, 3607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 4065, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 3835, 1818, 1293, 531, 489, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 2140, 0, 2802, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3695, 0, 0, 0, 4096, 0, 0, 0, 0, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 8, 244, 0, 0, 0, 3765, 0, 0, 0, 0, 4072, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16465, 15723, 16138, 16282, 16561, 17218, 16413, 16272] avg=16384.0000 max=17218 max_rank=5 min=15723 min_rank=1 imbalance_ratio=1.050903 max_load_reduction=113854 max_load_reduction_pct=0.868637 moved_to_echo=66464 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4070, 82, 1, 145, 13, 1, 4095, 4095, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2280, 4093, 1, 0, 0, 0, 0, 0, 2, 4090, 33, 0, 2614, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 7, 409, 1, 1, 9, 1, 0, 19, 0, 1, 0, 6, 0, 1, 4094, 4089, 3559, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 5, 0, 1, 0, 3, 0, 4078, 4095, 793, 4095, 3612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3690, 0, 1, 1, 15, 1, 0, 0, 4084, 9, 1, 0, 0, 0, 1, 3848, 1815, 1287, 530, 483, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4073, 0, 897, 0, 2803, 0, 3637, 1, 0, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4095, 0, 1, 3696, 0, 0, 1, 4095, 1, 1, 1, 0, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4068, 0, 0, 247, 1, 0, 1, 4044, 1, 2, 0, 0, 4095, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16446, 17208, 16389, 16684, 16165, 15507, 16202, 16471] avg=16384.0000 max=17208 max_rank=1 min=15507 min_rank=5 imbalance_ratio=1.050293 max_load_reduction=113864 max_load_reduction_pct=0.868713 moved_to_echo=64345 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 4080, 99, 1, 141, 12, 1, 4095, 4095, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2280, 4093, 1, 0, 0, 0, 0, 0, 1, 4091, 41, 1, 2598, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 7, 804, 1, 1, 15, 1, 0, 19, 0, 1, 0, 4, 0, 1, 4094, 4089, 3558, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 3, 0, 1, 0, 5, 0, 4077, 4095, 126, 4094, 3612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3647, 0, 2, 1, 23, 1, 0, 0, 4084, 10, 1, 0, 0, 0, 1, 3848, 1815, 1288, 530, 483, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4076, 0, 720, 0, 2806, 0, 3731, 1, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4095, 0, 1, 3696, 0, 0, 1, 4095, 0, 1, 1, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4062, 0, 0, 247, 1, 0, 1, 4084, 1, 3, 0, 0, 4095, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16469, 17201, 16804, 16015, 16133, 15427, 16499, 16524] avg=16384.0000 max=17201 max_rank=1 min=15427 min_rank=5 imbalance_ratio=1.049866 max_load_reduction=113871 max_load_reduction_pct=0.868767 moved_to_echo=64583 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 32, 0, 0, 0, 0, 4096, 4096, 3943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2281, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1990, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 146, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3565, 153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 4096, 670, 4096, 3613, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4076, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3847, 1815, 1287, 531, 483, 399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1462, 0, 2805, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3697, 0, 0, 0, 4096, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 247, 0, 0, 0, 3881, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16263, 16559, 16122, 16573, 16546, 16534, 16059, 16416] avg=16384.0000 max=16573 max_rank=3 min=16059 min_rank=6 imbalance_ratio=1.011536 max_load_reduction=114499 max_load_reduction_pct=0.873558 moved_to_echo=65555 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 18304, 32768, 128, 0, 128, 0, 0, 128, 128, 32768, 128, 128, 14464, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 0, 128, 0, 0, 0, 32768, 896, 128, 512, 128, 128, 32768, 32768, 31616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 32768, 0, 128, 2048, 128, 128, 128, 31232, 128, 128, 128, 0, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 29696, 0, 0, 128, 32768, 128, 128, 128, 0, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 0, 32768, 0, 11520, 0, 22528, 0, 31744, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32640, 128, 3200, 128, 128, 128, 128, 0, 128, 0, 128, 0, 128, 0, 128, 32768, 32768, 28544, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 0, 32768, 32768, 3968, 32768, 28928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 31360, 0, 128, 128, 128, 128, 0, 0, 32768, 128, 128, 128, 0, 128, 128, 30848, 14592, 10368, 4352, 3968, 3200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2621, 1935, 1546, 0, 572, 0, 1702, 1261, 1248, 2168, 1758, 665, 1048, 2206, 0, 876, 757, 1387, 1138, 830, 32, 788, 155, 2018, 435, 65, 2827, 2473, 1796, 216, 1365, 985, 94, 0, 2176, 946, 466, 2038, 819, 470, 186, 9, 1792, 2644, 0, 1487, 1447, 1253, 954, 669, 848, 1257, 35, 85, 6, 1417, 0, 523, 39, 820, 1374, 2356, 1391, 713, 912, 8, 0, 1584, 1742, 1862, 991, 913, 825, 1045, 1728, 356, 1414, 40, 1587, 2066, 1104, 454, 13, 13, 2197, 528, 1280, 19, 2325, 289, 1148, 639, 308, 1529, 819, 0, 1412, 1002, 6, 1760, 21, 650, 1392, 1313, 308, 1464, 512, 1304, 358, 1283, 2268, 1520, 2197, 1620, 365, 1395, 2187, 1061, 1937, 1448, 943, 2247, 1573, 616, 1145, 29, 788, 23] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2341, 1451, 1194, 3, 377, 0, 1843, 949, 1509, 2653, 1727, 1124, 1009, 2608, 0, 1043, 868, 1072, 1468, 655, 23, 755, 207, 1894, 343, 80, 2913, 1703, 1101, 160, 1283, 1180, 96, 2, 2633, 758, 341, 2538, 908, 289, 329, 5, 1593, 2192, 0, 1532, 1276, 684, 1225, 792, 799, 1641, 40, 63, 16, 1216, 0, 646, 63, 831, 1474, 2154, 1414, 447, 715, 1, 0, 1566, 1661, 2074, 515, 791, 1115, 1742, 1977, 645, 1344, 25, 1648, 1838, 961, 287, 11, 10, 1835, 1006, 1251, 35, 2237, 310, 710, 772, 436, 1288, 663, 1, 1462, 1203, 2, 2110, 12, 823, 1744, 1550, 367, 1893, 183, 1396, 433, 1782, 2504, 1677, 1570, 1264, 594, 1548, 2323, 1308, 2134, 1380, 1052, 2301, 763, 820, 1203, 25, 642, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1727, 1257, 1808, 1, 784, 1, 1813, 1328, 1266, 2361, 1508, 678, 1217, 1852, 0, 774, 955, 1305, 1569, 1591, 12, 684, 134, 1878, 457, 106, 3008, 1895, 1289, 71, 1412, 1147, 156, 2, 1482, 653, 524, 2312, 1590, 855, 305, 0, 1796, 2420, 0, 1546, 1332, 1240, 1401, 397, 603, 1163, 3, 19, 26, 1456, 0, 316, 25, 715, 1122, 2625, 1484, 658, 1076, 6, 0, 1612, 1888, 2070, 1423, 1354, 1381, 1744, 2144, 692, 1735, 7, 1403, 1634, 1046, 404, 18, 22, 2076, 498, 1130, 13, 2183, 247, 1014, 1031, 218, 1638, 928, 0, 695, 530, 11, 1771, 10, 552, 1098, 1078, 347, 1294, 389, 1446, 534, 1029, 2704, 1699, 1946, 1462, 620, 729, 2646, 1396, 1776, 1032, 1119, 2088, 1634, 656, 1218, 26, 779, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[1999, 1597, 2025, 0, 491, 0, 1918, 1613, 1445, 2587, 1185, 521, 1122, 1826, 0, 689, 935, 1317, 1146, 1593, 2, 849, 164, 1672, 641, 68, 3349, 1589, 1586, 67, 1332, 1069, 347, 2, 1858, 409, 177, 1789, 1281, 846, 206, 15, 1518, 2362, 0, 1243, 1477, 1097, 1003, 340, 529, 1278, 23, 30, 40, 1501, 0, 598, 74, 452, 1111, 2668, 833, 717, 1086, 4, 0, 1576, 2020, 1901, 1006, 1612, 1228, 1126, 2402, 630, 1065, 8, 1070, 2015, 1041, 540, 22, 20, 2337, 587, 1298, 10, 2558, 152, 1253, 813, 272, 1357, 1153, 2, 1352, 965, 0, 1889, 30, 822, 1301, 1045, 297, 1066, 306, 1237, 641, 930, 2714, 1833, 1835, 2078, 423, 906, 2812, 1610, 2305, 1369, 1049, 2166, 1209, 989, 900, 29, 561, 18] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2188, 1283, 2087, 0, 566, 0, 1426, 1068, 1171, 1884, 1276, 998, 1426, 1345, 0, 616, 519, 1701, 1487, 922, 10, 769, 150, 2784, 685, 81, 2719, 2104, 1549, 69, 1665, 1404, 419, 15, 2035, 888, 304, 1909, 936, 445, 204, 7, 1858, 2729, 6, 1600, 1283, 918, 1134, 636, 520, 1362, 23, 82, 33, 1515, 0, 632, 82, 1010, 1347, 2566, 1546, 585, 1279, 8, 1, 1943, 1809, 1442, 1265, 1531, 850, 1696, 1360, 728, 1310, 16, 990, 1451, 1289, 766, 19, 26, 2107, 654, 1159, 75, 1683, 350, 1320, 896, 320, 1903, 633, 8, 1421, 937, 3, 2356, 42, 599, 1614, 1016, 281, 1497, 336, 1187, 668, 1256, 2400, 1427, 1746, 1788, 341, 1113, 2102, 1102, 1624, 1722, 1125, 1852, 1602, 798, 896, 81, 586, 86] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2153, 1689, 1571, 0, 658, 1, 2173, 1079, 1434, 2386, 1715, 927, 913, 2163, 0, 858, 757, 1291, 690, 911, 64, 911, 138, 1854, 692, 52, 3073, 1676, 1780, 120, 1350, 852, 99, 0, 2070, 631, 211, 2995, 1547, 876, 305, 8, 1963, 2768, 1, 1319, 941, 1566, 1175, 579, 1046, 948, 9, 131, 8, 1260, 0, 841, 34, 451, 1325, 2099, 1521, 454, 531, 4, 0, 1894, 1995, 2346, 1482, 1241, 1006, 1269, 1745, 273, 1779, 88, 1543, 2144, 1396, 333, 16, 4, 2096, 1049, 1060, 7, 2092, 421, 1696, 641, 340, 1574, 1026, 0, 1321, 582, 24, 1877, 36, 442, 1070, 1065, 189, 1524, 338, 1101, 411, 970, 2835, 1656, 1492, 1560, 533, 1116, 1624, 1409, 2152, 1052, 797, 1779, 1398, 650, 1120, 8, 705, 33] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[2220, 2082, 1441, 0, 517, 0, 1870, 2153, 1608, 2370, 1956, 853, 1267, 2824, 0, 1162, 620, 1514, 1550, 1043, 43, 829, 186, 2144, 470, 311, 2763, 1898, 2068, 481, 1414, 819, 92, 1, 2546, 986, 456, 1893, 1146, 719, 220, 8, 2351, 2549, 0, 1307, 1007, 1218, 1001, 515, 877, 1136, 27, 128, 58, 1633, 1, 459, 57, 518, 1299, 1731, 1154, 385, 811, 6, 0, 1531, 1223, 2416, 1019, 1400, 1249, 1080, 1466, 764, 1757, 37, 1560, 1439, 1259, 812, 10, 46, 2070, 1056, 1878, 11, 2037, 610, 910, 708, 210, 1335, 574, 1, 1681, 953, 6, 2135, 47, 755, 1179, 958, 189, 1138, 122, 1446, 332, 611, 2632, 817, 1244, 1618, 475, 1181, 2531, 1183, 1365, 1230, 981, 1410, 1021, 811, 1307, 36, 362, 77] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1747, 1603, 1476, 0, 515, 0, 2658, 494, 983, 3105, 2185, 439, 821, 2766, 0, 491, 659, 1963, 721, 1217, 14, 1083, 155, 1607, 493, 182, 3097, 1668, 842, 129, 1563, 1241, 257, 10, 2072, 736, 189, 2273, 910, 1039, 181, 2, 1863, 2557, 0, 1633, 1436, 870, 1202, 605, 465, 911, 68, 226, 1, 1564, 0, 367, 18, 1288, 1147, 3132, 1005, 674, 803, 4, 0, 1262, 2305, 1315, 1670, 866, 862, 1444, 2084, 363, 1300, 5, 1434, 1542, 918, 272, 8, 12, 2246, 932, 890, 8, 2143, 211, 1450, 880, 419, 2452, 811, 1, 1223, 681, 5, 1504, 15, 345, 1333, 948, 184, 1508, 365, 885, 842, 1417, 2487, 1946, 2155, 1736, 698, 537, 2345, 1654, 2087, 957, 1020, 2572, 1139, 551, 1562, 37, 796, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2621, 1935, 1546, 0, 572, 0, 1702, 1261, 1248, 0, 1758, 665, 1048, 1592, 0, 876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 1387, 1138, 830, 32, 788, 155, 2018, 435, 65, 2065, 2473, 1796, 216, 1365, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 2176, 946, 466, 2038, 819, 470, 186, 9, 1792, 2644, 0, 1487, 1447, 1253, 437, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 954, 669, 848, 1257, 35, 85, 6, 1417, 0, 523, 39, 820, 1374, 2356, 1391, 713, 2168, 1699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 912, 8, 0, 1584, 1742, 99, 991, 913, 825, 1045, 1728, 356, 1414, 40, 1587, 2066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1104, 454, 13, 13, 2197, 528, 1280, 19, 2325, 289, 1148, 639, 308, 1529, 819, 0, 1763, 762, 488, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1412, 1002, 6, 1760, 21, 650, 1392, 1313, 308, 1464, 512, 1304, 358, 1283, 2268, 1520, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2197, 1620, 365, 1395, 0, 1061, 1937, 1448, 943, 2120, 1573, 616, 1145, 29, 788, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16824, 16505, 16313, 16354, 15310, 15855, 16651, 17260] avg=16384.0000 max=17260 max_rank=7 min=15310 min_rank=4 imbalance_ratio=1.053467 max_load_reduction=113812 max_load_reduction_pct=0.868317 moved_to_echo=65076 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2341, 1451, 1194, 3, 377, 0, 1843, 949, 1509, 0, 1727, 1124, 1009, 1882, 0, 1043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 868, 1072, 1468, 655, 23, 755, 207, 1894, 343, 80, 2128, 1703, 1101, 160, 1283, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 2, 2633, 758, 341, 2538, 908, 289, 329, 5, 1593, 2192, 0, 1532, 1276, 684, 516, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1225, 792, 799, 1641, 40, 63, 16, 1216, 0, 646, 63, 831, 1474, 2154, 1414, 447, 2653, 1805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 715, 1, 0, 1566, 1661, 110, 515, 791, 1115, 1742, 1977, 645, 1344, 25, 1648, 1838, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 961, 287, 11, 10, 1835, 1006, 1251, 35, 2237, 310, 710, 772, 436, 1288, 663, 1, 1964, 785, 518, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1462, 1203, 2, 2110, 12, 823, 1744, 1550, 367, 1893, 183, 1396, 433, 1782, 2504, 1677, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1570, 1264, 594, 1548, 0, 1308, 2134, 1380, 1052, 2171, 763, 820, 1203, 25, 642, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16452, 14920, 15742, 17279, 15693, 15290, 19221, 16475] avg=16384.0000 max=19221 max_rank=6 min=14920 min_rank=1 imbalance_ratio=1.173157 max_load_reduction=111851 max_load_reduction_pct=0.853355 moved_to_echo=66679 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1727, 1257, 1808, 1, 784, 1, 1813, 1328, 1266, 0, 1508, 678, 1217, 1336, 0, 774, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 955, 1305, 1569, 1591, 12, 684, 134, 1878, 457, 106, 2197, 1895, 1289, 71, 1412, 1147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 2, 1482, 653, 524, 2312, 1590, 855, 305, 0, 1796, 2420, 0, 1546, 1332, 1240, 367, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1401, 397, 603, 1163, 3, 19, 26, 1456, 0, 316, 25, 715, 1122, 2625, 1484, 658, 2361, 2057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1076, 6, 0, 1612, 1888, 110, 1423, 1354, 1381, 1744, 2144, 692, 1735, 7, 1403, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1046, 404, 18, 22, 2076, 498, 1130, 13, 2183, 247, 1014, 1031, 218, 1638, 928, 0, 1960, 811, 589, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 695, 530, 11, 1771, 10, 552, 1098, 1078, 347, 1294, 389, 1446, 534, 1029, 2704, 1699, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1946, 1462, 620, 729, 0, 1396, 1776, 1032, 1119, 1971, 1634, 656, 1218, 26, 779, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15498, 16702, 16625, 16431, 18209, 15975, 15259, 16373] avg=16384.0000 max=18209 max_rank=4 min=15259 min_rank=6 imbalance_ratio=1.111389 max_load_reduction=112863 max_load_reduction_pct=0.861076 moved_to_echo=65816 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1999, 1597, 2025, 0, 491, 0, 1918, 1613, 1445, 0, 1185, 521, 1122, 1318, 0, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 935, 1317, 1146, 1593, 2, 849, 164, 1672, 641, 68, 2446, 1589, 1586, 67, 1332, 1069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 347, 2, 1858, 409, 177, 1789, 1281, 846, 206, 15, 1518, 2362, 0, 1243, 1477, 1097, 361, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 340, 529, 1278, 23, 30, 40, 1501, 0, 598, 74, 452, 1111, 2668, 833, 717, 2587, 2186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 4, 0, 1576, 2020, 101, 1006, 1612, 1228, 1126, 2402, 630, 1065, 8, 1070, 2015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 540, 22, 20, 2337, 587, 1298, 10, 2558, 152, 1253, 813, 272, 1357, 1153, 2, 1800, 903, 626, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1352, 965, 0, 1889, 30, 822, 1301, 1045, 297, 1066, 306, 1237, 641, 930, 2714, 1833, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1835, 2078, 423, 906, 0, 1610, 2305, 1369, 1049, 2044, 1209, 989, 900, 29, 561, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15923, 16476, 15035, 15970, 16949, 16891, 16503, 17325] avg=16384.0000 max=17325 max_rank=7 min=15035 min_rank=2 imbalance_ratio=1.057434 max_load_reduction=113747 max_load_reduction_pct=0.867821 moved_to_echo=67668 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2188, 1283, 2087, 0, 566, 0, 1426, 1068, 1171, 0, 1276, 998, 1426, 963, 0, 616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 519, 1701, 1487, 922, 10, 769, 150, 2784, 685, 81, 1981, 2104, 1549, 69, 1665, 1404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 419, 15, 2035, 888, 304, 1909, 936, 445, 204, 7, 1858, 2729, 6, 1600, 1283, 918, 270, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1134, 636, 520, 1362, 23, 82, 33, 1515, 0, 632, 82, 1010, 1347, 2566, 1546, 585, 1884, 1634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1279, 8, 1, 1943, 1809, 72, 1265, 1531, 850, 1696, 1360, 728, 1310, 16, 990, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1289, 766, 19, 26, 2107, 654, 1159, 75, 1683, 350, 1320, 896, 320, 1903, 633, 8, 1370, 738, 468, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1421, 937, 3, 2356, 42, 599, 1614, 1016, 281, 1497, 336, 1187, 668, 1256, 2400, 1427, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1746, 1788, 341, 1113, 0, 1102, 1624, 1722, 1125, 1740, 1602, 798, 896, 81, 586, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15068, 17880, 15870, 16591, 16309, 15896, 17108, 16350] avg=16384.0000 max=17880 max_rank=1 min=15068 min_rank=0 imbalance_ratio=1.091309 max_load_reduction=113192 max_load_reduction_pct=0.863586 moved_to_echo=65663 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2153, 1689, 1571, 0, 658, 1, 2173, 1079, 1434, 0, 1715, 927, 913, 1561, 0, 858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 1291, 690, 911, 64, 911, 138, 1854, 692, 52, 2244, 1676, 1780, 120, 1350, 852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 2070, 631, 211, 2995, 1547, 876, 305, 8, 1963, 2768, 1, 1319, 941, 1566, 428, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1175, 579, 1046, 948, 9, 131, 8, 1260, 0, 841, 34, 451, 1325, 2099, 1521, 454, 2386, 1263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 531, 4, 0, 1894, 1995, 124, 1482, 1241, 1006, 1269, 1745, 273, 1779, 88, 1543, 2144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1396, 333, 16, 4, 2096, 1049, 1060, 7, 2092, 421, 1696, 641, 340, 1574, 1026, 0, 2222, 829, 361, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1321, 582, 24, 1877, 36, 442, 1070, 1065, 189, 1524, 338, 1101, 411, 970, 2835, 1656, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1492, 1560, 533, 1116, 0, 1409, 2152, 1052, 797, 1678, 1398, 650, 1120, 8, 705, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16732, 15382, 17767, 15530, 17118, 17337, 15503, 15703] avg=16384.0000 max=17767 max_rank=2 min=15382 min_rank=1 imbalance_ratio=1.084412 max_load_reduction=113305 max_load_reduction_pct=0.864449 moved_to_echo=65661 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2220, 2082, 1441, 0, 517, 0, 1870, 2153, 1608, 0, 1956, 853, 1267, 2038, 0, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 620, 1514, 1550, 1043, 43, 829, 186, 2144, 470, 311, 2018, 1898, 2068, 481, 1414, 819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 1, 2546, 986, 456, 1893, 1146, 719, 220, 8, 2351, 2549, 0, 1307, 1007, 1218, 559, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1001, 515, 877, 1136, 27, 128, 58, 1633, 1, 459, 57, 518, 1299, 1731, 1154, 385, 2370, 1967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 811, 6, 0, 1531, 1223, 128, 1019, 1400, 1249, 1080, 1466, 764, 1757, 37, 1560, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1259, 812, 10, 46, 2070, 1056, 1878, 11, 2037, 610, 910, 708, 210, 1335, 574, 1, 2288, 745, 564, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1681, 953, 6, 2135, 47, 755, 1179, 958, 189, 1138, 122, 1446, 332, 611, 2632, 817, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1244, 1618, 475, 1181, 0, 1183, 1365, 1230, 981, 1331, 1021, 811, 1307, 36, 362, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[19167, 17408, 17088, 15316, 15470, 17351, 15050, 14222] avg=16384.0000 max=19167 max_rank=0 min=14222 min_rank=7 imbalance_ratio=1.169861 max_load_reduction=111905 max_load_reduction_pct=0.853767 moved_to_echo=62093 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1747, 1603, 1476, 0, 515, 0, 2658, 494, 983, 0, 2185, 439, 821, 1996, 0, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 659, 1963, 721, 1217, 14, 1083, 155, 1607, 493, 182, 2262, 1668, 842, 129, 1563, 1241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 10, 2072, 736, 189, 2273, 910, 1039, 181, 2, 1863, 2557, 0, 1633, 1436, 870, 548, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1202, 605, 465, 911, 68, 226, 1, 1564, 0, 367, 18, 1288, 1147, 3132, 1005, 674, 3105, 1823, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 803, 4, 0, 1262, 2305, 70, 1670, 866, 862, 1444, 2084, 363, 1300, 5, 1434, 1542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 918, 272, 8, 12, 2246, 932, 890, 8, 2143, 211, 1450, 880, 419, 2452, 811, 1, 1245, 835, 522, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1223, 681, 5, 1504, 15, 345, 1333, 948, 184, 1508, 365, 885, 842, 1417, 2487, 1946, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2155, 1736, 698, 537, 0, 1654, 2087, 957, 1020, 2427, 1139, 551, 1562, 37, 796, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15408, 15799, 16632, 17601, 16014, 16477, 15777, 17364] avg=16384.0000 max=17601 max_rank=3 min=15408 min_rank=0 imbalance_ratio=1.074280 max_load_reduction=113471 max_load_reduction_pct=0.865715 moved_to_echo=65632 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[17024, 12928, 13184, 128, 4480, 128, 15488, 9984, 10752, 0, 13312, 6272, 8832, 12800, 0, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[9216, 4608, 5760, 9728, 256, 768, 256, 11648, 128, 4480, 512, 6144, 10240, 19456, 10368, 4736, 19584, 14464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[6144, 11648, 9856, 8832, 256, 6784, 1408, 15872, 4224, 1024, 17408, 15104, 12032, 1408, 11392, 8704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[9088, 3968, 128, 256, 17024, 6400, 9984, 256, 17280, 2688, 9600, 6400, 2560, 13184, 6656, 128, 14720, 6528, 4224, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1664, 128, 16896, 6016, 2688, 17792, 9216, 5632, 2048, 128, 14848, 20224, 128, 11776, 10240, 8960, 3584, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[7296, 128, 128, 13056, 14720, 896, 9472, 9728, 8576, 11264, 14976, 4480, 11776, 256, 11264, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[14208, 13184, 4096, 8576, 0, 10752, 15488, 10240, 8192, 15488, 10368, 6016, 9472, 384, 5248, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[10624, 6912, 128, 15488, 256, 4992, 10752, 9088, 2176, 11392, 2560, 10112, 4224, 9344, 20608, 12672, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 3, 305, 2206, 0, 8, 3, 0, 1, 9, 0, 0, 1, 0, 4093, 0, 0, 4094, 0, 0, 3950, 4, 4019, 0, 0, 2850, 0, 4081, 4096, 1, 2, 0, 4093, 0, 2365, 31, 1294, 1, 0, 4091, 0, 0, 353, 0, 382, 0, 943, 758, 3974, 0, 0, 4096, 5, 2036, 4096, 0, 0, 3536, 907, 0, 0, 4082, 0, 3841, 30, 0, 0, 487, 10, 0, 4090, 0, 4084, 4094, 0, 0, 0, 0, 0, 0, 671, 0, 3227, 0, 0, 2, 578, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 1, 4092, 1368, 119, 4093, 0, 777, 4094, 0, 3444, 4, 0, 0, 4092, 4096, 3, 0, 1, 0, 1071, 7, 0, 4086, 3339, 0, 321, 0, 0, 0, 4089] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 160, 2937, 0, 0, 0, 0, 0, 1, 4, 1, 0, 0, 4092, 1, 0, 4095, 1, 0, 4008, 12, 3995, 3, 0, 3535, 0, 4092, 4088, 45, 1, 0, 4095, 0, 2943, 79, 786, 0, 1, 3965, 0, 0, 11, 0, 316, 1, 3774, 285, 4029, 1, 1, 4095, 23, 366, 4088, 2, 1, 3812, 708, 1, 0, 3923, 1, 3557, 0, 0, 0, 1087, 0, 0, 4095, 0, 4090, 4023, 0, 1, 0, 0, 1, 0, 117, 13, 3351, 0, 1, 0, 669, 0, 1, 1, 5, 1, 4095, 0, 0, 0, 0, 0, 4095, 649, 323, 3905, 1, 1446, 4083, 1, 2909, 0, 0, 5, 4095, 4094, 0, 1, 207, 1, 1536, 2, 1, 3994, 2105, 0, 57, 0, 0, 1, 4074] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 105, 2793, 0, 1, 1, 0, 0, 2, 12, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 4070, 0, 4066, 1, 0, 3743, 0, 4096, 4093, 6, 0, 0, 4096, 0, 2338, 70, 762, 0, 0, 3843, 0, 0, 157, 0, 159, 0, 1055, 1069, 4084, 0, 0, 4096, 0, 1928, 4096, 0, 0, 4078, 946, 0, 0, 4082, 0, 3578, 0, 0, 0, 1163, 0, 0, 4090, 0, 4064, 4039, 0, 0, 0, 0, 0, 0, 607, 0, 3819, 0, 0, 0, 353, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 770, 421, 4086, 0, 974, 4096, 0, 2761, 0, 0, 0, 4096, 4096, 0, 0, 66, 0, 661, 0, 0, 4058, 2376, 0, 575, 0, 0, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 91, 2380, 0, 58, 2, 0, 0, 0, 33, 1, 0, 0, 4096, 0, 0, 4096, 0, 0, 3986, 1, 3822, 0, 0, 2982, 0, 4069, 4096, 6, 0, 0, 4093, 0, 1821, 379, 660, 2, 0, 4092, 0, 0, 45, 0, 399, 0, 2038, 582, 4016, 0, 0, 4095, 4, 2322, 4096, 0, 0, 3725, 565, 0, 0, 4010, 0, 3658, 3, 0, 0, 335, 17, 0, 4095, 0, 4091, 4094, 0, 0, 0, 0, 0, 0, 191, 0, 3616, 0, 0, 0, 613, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4094, 1335, 201, 4095, 0, 1632, 4096, 0, 2936, 0, 0, 0, 4096, 4096, 0, 0, 21, 0, 1466, 5, 0, 3623, 3523, 0, 294, 1, 0, 0, 4085] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 100, 2614, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4072, 8, 3841, 5, 0, 3224, 0, 4096, 4096, 4, 0, 2, 4096, 0, 1972, 26, 2478, 0, 0, 4060, 0, 0, 429, 0, 240, 0, 1473, 712, 4047, 0, 0, 4096, 0, 1425, 4096, 0, 0, 3994, 914, 0, 0, 4018, 0, 3675, 0, 0, 0, 366, 0, 0, 4095, 0, 4045, 4090, 0, 0, 0, 0, 0, 2, 43, 0, 3854, 0, 0, 0, 473, 0, 0, 2, 0, 0, 4092, 0, 1, 0, 0, 0, 4096, 842, 311, 4091, 0, 237, 4096, 0, 3528, 0, 0, 0, 4096, 4096, 0, 0, 22, 0, 1751, 1, 0, 3965, 2687, 0, 88, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 501, 2206, 0, 14, 1, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4095, 0, 1, 3904, 0, 3947, 0, 0, 2989, 0, 4063, 4096, 11, 0, 0, 4095, 2, 1536, 105, 412, 3, 0, 4088, 1, 0, 160, 0, 668, 0, 1448, 225, 4018, 0, 0, 4095, 4, 2357, 4094, 0, 0, 3994, 865, 1, 0, 3813, 1, 3913, 2, 0, 0, 223, 0, 0, 4095, 0, 4092, 4094, 0, 0, 1, 0, 0, 0, 139, 0, 3630, 0, 0, 0, 414, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 1, 0, 4095, 2718, 90, 4094, 0, 376, 4096, 0, 3628, 0, 0, 1, 4096, 4096, 0, 0, 41, 0, 1868, 3, 0, 4010, 2589, 2, 570, 0, 1, 0, 4086] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 266, 2779, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 3960, 0, 4065, 0, 0, 1315, 0, 4087, 4094, 4, 0, 0, 4096, 1, 3922, 24, 653, 2, 0, 4089, 0, 0, 82, 0, 805, 0, 1210, 1474, 4059, 0, 0, 4096, 0, 1283, 4096, 0, 0, 4050, 1330, 0, 0, 3876, 0, 3895, 8, 0, 0, 240, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 170, 1, 4053, 0, 0, 0, 261, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 2705, 194, 4091, 0, 748, 4096, 0, 2261, 0, 0, 0, 4096, 4096, 0, 0, 9, 0, 846, 0, 0, 3863, 2214, 0, 736, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 1, 0, 314, 3132, 0, 1, 5, 1, 0, 0, 2, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 3865, 0, 3856, 0, 0, 3699, 0, 4092, 4096, 9, 0, 3, 4096, 0, 3006, 1, 387, 0, 0, 4068, 0, 0, 491, 0, 1122, 0, 2349, 476, 4071, 0, 0, 4096, 3, 1888, 4096, 0, 0, 3799, 1427, 0, 0, 4093, 0, 3619, 4, 0, 0, 622, 3, 0, 4084, 0, 4054, 4074, 0, 0, 0, 0, 0, 0, 306, 0, 2750, 0, 0, 0, 274, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 1038, 426, 4086, 0, 1236, 4096, 0, 1867, 0, 0, 0, 4096, 4096, 2, 0, 35, 0, 462, 0, 0, 4059, 1885, 0, 877, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 100, 2614, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 4096, 4096, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 8, 3841, 5, 0, 3224, 0, 4096, 1315, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 1972, 26, 2478, 0, 0, 4060, 0, 0, 429, 0, 240, 0, 1473, 1408, 619, 153, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 712, 4047, 0, 0, 0, 0, 1425, 1125, 0, 0, 3994, 914, 0, 0, 4018, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3675, 0, 0, 0, 366, 0, 0, 3942, 0, 4045, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 43, 0, 3854, 0, 0, 0, 473, 0, 0, 2, 0, 0, 4092, 0, 1, 2971, 2881, 2162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2688, 842, 311, 4091, 0, 237, 4096, 0, 3528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 22, 0, 1751, 1, 0, 3965, 2687, 0, 88, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16219, 16565, 17005, 16235, 16118, 16481, 15793, 16656] avg=16384.0000 max=17005 max_rank=2 min=15793 min_rank=6 imbalance_ratio=1.037903 max_load_reduction=114067 max_load_reduction_pct=0.870262 moved_to_echo=65048 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 160, 2937, 0, 0, 0, 0, 0, 1, 4, 1, 0, 0, 4092, 4095, 4095, 1216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4008, 12, 3995, 3, 0, 3535, 0, 4092, 1303, 45, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 2943, 79, 786, 0, 1, 3965, 0, 0, 11, 0, 316, 1, 3774, 1412, 620, 156, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 285, 4029, 1, 1, 0, 23, 366, 1119, 2, 1, 3812, 708, 1, 0, 3923, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3557, 0, 0, 0, 1087, 0, 0, 3939, 0, 4090, 4023, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 13, 3351, 0, 1, 0, 669, 0, 1, 1, 5, 1, 4095, 0, 0, 2969, 2879, 2165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2683, 649, 323, 3905, 1, 1446, 4083, 1, 2909, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 1, 207, 1, 1536, 2, 1, 3994, 2105, 0, 57, 0, 0, 1, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16602, 16996, 18211, 14272, 16698, 16267, 16005, 16021] avg=16384.0000 max=18211 max_rank=2 min=14272 min_rank=3 imbalance_ratio=1.111511 max_load_reduction=112861 max_load_reduction_pct=0.861061 moved_to_echo=64991 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 314, 3132, 0, 1, 5, 1, 0, 0, 2, 0, 0, 0, 4095, 4096, 4096, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 0, 3856, 0, 0, 3699, 0, 4092, 1315, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 3006, 1, 387, 0, 0, 4068, 0, 0, 491, 0, 1122, 0, 2349, 1407, 619, 153, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 476, 4071, 0, 0, 0, 3, 1888, 1125, 0, 0, 3799, 1427, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3619, 4, 0, 0, 622, 3, 0, 3931, 0, 4054, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 0, 2750, 0, 0, 0, 274, 0, 0, 0, 0, 0, 4096, 0, 0, 2971, 2881, 2162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2687, 1038, 426, 4086, 0, 1236, 4096, 0, 1867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 2, 0, 35, 0, 462, 0, 0, 4059, 1885, 0, 877, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16958, 16836, 17751, 16882, 16307, 15440, 15436, 15462] avg=16384.0000 max=17751 max_rank=2 min=15436 min_rank=6 imbalance_ratio=1.083435 max_load_reduction=113321 max_load_reduction_pct=0.864571 moved_to_echo=62645 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 501, 2206, 0, 14, 1, 0, 0, 0, 1, 0, 0, 0, 4096, 4095, 4095, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3904, 0, 3947, 0, 0, 2989, 0, 4063, 1315, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 1536, 105, 412, 3, 0, 4088, 1, 0, 160, 0, 668, 0, 1448, 1408, 619, 153, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 4018, 0, 0, 0, 4, 2357, 1124, 0, 0, 3994, 865, 1, 0, 3813, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 2, 0, 0, 223, 0, 0, 3942, 0, 4092, 4094, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 3630, 0, 0, 0, 414, 0, 0, 0, 0, 0, 4095, 1, 0, 2970, 2881, 2162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 2687, 2718, 90, 4094, 0, 376, 4096, 0, 3628, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 41, 0, 1868, 3, 0, 4010, 2589, 2, 570, 0, 1, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16225, 16230, 14747, 16402, 16267, 16292, 17692, 17217] avg=16384.0000 max=17692 max_rank=6 min=14747 min_rank=2 imbalance_ratio=1.079834 max_load_reduction=113380 max_load_reduction_pct=0.865021 moved_to_echo=67468 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 105, 2793, 0, 1, 1, 0, 0, 2, 12, 0, 0, 0, 4093, 4096, 4096, 1216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 4066, 1, 0, 3743, 0, 4096, 1314, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2338, 70, 762, 0, 0, 3843, 0, 0, 157, 0, 159, 0, 1055, 1408, 618, 153, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1069, 4084, 0, 0, 0, 0, 1928, 1125, 0, 0, 4078, 946, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3578, 0, 0, 0, 1163, 0, 0, 3937, 0, 4064, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 607, 0, 3819, 0, 0, 0, 353, 0, 0, 0, 0, 0, 4096, 0, 0, 2971, 2880, 2161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2688, 770, 421, 4086, 0, 974, 4096, 0, 2761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 66, 0, 661, 0, 0, 4058, 2376, 0, 575, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16415, 17296, 14708, 17312, 16781, 16887, 15796, 15877] avg=16384.0000 max=17312 max_rank=3 min=14708 min_rank=2 imbalance_ratio=1.056641 max_load_reduction=113760 max_load_reduction_pct=0.867920 moved_to_echo=65341 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 91, 2380, 0, 58, 2, 0, 0, 0, 33, 1, 0, 0, 4096, 4095, 4096, 1216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 1, 3822, 0, 0, 2982, 0, 4069, 1315, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1821, 379, 660, 2, 0, 4092, 0, 0, 45, 0, 399, 0, 2038, 1407, 619, 153, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 582, 4016, 0, 0, 0, 4, 2322, 1125, 0, 0, 3725, 565, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3658, 3, 0, 0, 335, 17, 0, 3942, 0, 4091, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 3616, 0, 0, 0, 613, 0, 0, 0, 0, 0, 4096, 1, 0, 2971, 2880, 2162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2687, 1335, 201, 4095, 0, 1632, 4096, 0, 2936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 21, 0, 1466, 5, 0, 3623, 3523, 0, 294, 1, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16068, 16181, 15757, 16349, 16140, 16530, 16982, 17065] avg=16384.0000 max=17065 max_rank=7 min=15757 min_rank=2 imbalance_ratio=1.041565 max_load_reduction=114007 max_load_reduction_pct=0.869804 moved_to_echo=66717 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 305, 2206, 0, 8, 3, 0, 1, 9, 0, 0, 1, 0, 4093, 4096, 4094, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3950, 4, 4019, 0, 0, 2850, 0, 4081, 1315, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 2365, 31, 1294, 1, 0, 4091, 0, 0, 353, 0, 382, 0, 943, 1407, 619, 153, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 758, 3974, 0, 0, 0, 5, 2036, 1125, 0, 0, 3536, 907, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3841, 30, 0, 0, 487, 10, 0, 3937, 0, 4084, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 671, 0, 3227, 0, 0, 2, 578, 0, 0, 0, 0, 0, 4092, 0, 0, 2971, 2877, 2162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2685, 1368, 119, 4093, 0, 777, 4094, 0, 3444, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 3, 0, 1, 0, 1071, 7, 0, 4086, 3339, 0, 321, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16034, 16222, 15781, 16423, 16483, 16580, 16585, 16964] avg=16384.0000 max=16964 max_rank=7 min=15781 min_rank=2 imbalance_ratio=1.035400 max_load_reduction=114108 max_load_reduction_pct=0.870575 moved_to_echo=66612 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 266, 2779, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 4096, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 0, 4065, 0, 0, 1315, 0, 4087, 1315, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 3922, 24, 653, 2, 0, 4089, 0, 0, 82, 0, 805, 0, 1210, 1408, 618, 153, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1474, 4059, 0, 0, 0, 0, 1283, 1125, 0, 0, 4050, 1330, 0, 0, 3876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3895, 8, 0, 0, 240, 0, 0, 3943, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170, 1, 4053, 0, 0, 0, 261, 0, 0, 0, 0, 0, 4096, 0, 1, 2971, 2881, 2161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2688, 2705, 194, 4091, 0, 748, 4096, 0, 2261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 0, 9, 0, 846, 0, 0, 3863, 2214, 0, 736, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16551, 14746, 17112, 17197, 16278, 16595, 16783, 15810] avg=16384.0000 max=17197 max_rank=3 min=14746 min_rank=1 imbalance_ratio=1.049622 max_load_reduction=113875 max_load_reduction_pct=0.868797 moved_to_echo=65466 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 128, 31872, 128, 31616, 128, 0, 24448, 0, 32768, 10624, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 1920, 21120, 0, 128, 128, 128, 128, 128, 128, 128, 128, 0, 32768, 32768, 32768, 9728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[5632, 32384, 128, 128, 0, 128, 13696, 9088, 128, 128, 31104, 7680, 128, 0, 32000, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 2304, 128, 28416, 0, 128, 128, 3712, 0, 128, 128, 128, 128, 32768, 128, 128, 23808, 23040, 17408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[29824, 128, 0, 0, 4608, 128, 0, 31616, 0, 32640, 32640, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32384, 128, 128, 512, 128, 9728, 128, 128, 31744, 20736, 128, 3584, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 21504, 11520, 2176, 32640, 128, 7552, 32768, 128, 23424, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 128, 19968, 768, 7552, 128, 128, 32384, 128, 0, 1792, 0, 4096, 128, 14336, 11392, 4992, 1280, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 2397, 0, 1616, 0, 0, 108, 0, 4094, 912, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 1, 5, 4095, 0, 4095, 4095, 0, 0, 0, 4095, 222, 0, 2, 1, 4096, 1, 0, 4096, 1, 0, 0, 0, 4095, 0, 0, 0, 1, 4090, 4094, 775, 0, 0, 4062, 0, 4095, 4068, 1, 0, 4084, 4032, 0, 0, 3, 4095, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 4096, 3, 0, 4095, 9, 169, 3860, 4095, 0, 0, 4031, 4036, 0, 2211, 0, 188, 0, 0, 2381, 0, 0, 0, 4, 1, 54, 1, 0, 3529, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 8, 4095, 0, 0, 1, 0, 0, 1, 2314, 0, 4096, 1, 4095, 15, 2, 1, 0, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2612, 0, 541, 0, 0, 10, 0, 4096, 1462, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 10, 4094, 0, 4096, 4095, 0, 0, 0, 4096, 11, 0, 2, 0, 4096, 3, 0, 4096, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 4093, 4095, 2244, 0, 0, 4047, 0, 4093, 4085, 0, 0, 4065, 3942, 0, 0, 5, 4096, 0, 0, 4096, 8, 0, 0, 0, 0, 1, 4096, 1, 0, 4096, 1, 681, 3867, 3994, 0, 0, 4087, 4063, 0, 2069, 0, 955, 0, 0, 1795, 0, 0, 0, 24, 0, 257, 0, 0, 2150, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 93, 4096, 0, 0, 0, 0, 0, 0, 2071, 0, 4083, 0, 4095, 4, 0, 0, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 374, 0, 765, 0, 0, 25, 0, 4096, 2264, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 33, 4096, 0, 4096, 4096, 0, 0, 1, 4095, 23, 0, 0, 0, 4096, 4, 1, 4096, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4088, 4095, 1802, 0, 0, 4082, 0, 4096, 4096, 0, 0, 4043, 4057, 0, 0, 0, 4096, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 9, 328, 3172, 4082, 0, 0, 4084, 4071, 0, 2738, 0, 2606, 0, 0, 2357, 0, 0, 0, 123, 0, 127, 0, 0, 1872, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 105, 4096, 0, 0, 3, 0, 0, 0, 1904, 0, 4096, 0, 4093, 8, 0, 1, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 1957, 0, 178, 1, 1, 2, 0, 4093, 2078, 0, 1, 1, 0, 4095, 0, 1, 0, 1, 0, 5, 4094, 1, 4094, 4095, 1, 0, 2, 4092, 30, 1, 7, 0, 4095, 6, 0, 4095, 1, 0, 0, 0, 4095, 1, 44, 1, 0, 4091, 4094, 1000, 1, 0, 3955, 0, 4090, 4089, 0, 1, 4013, 4060, 1, 1, 8, 4095, 0, 1, 4090, 18, 0, 0, 0, 0, 1, 4095, 1, 1, 4095, 0, 696, 2768, 4087, 2, 1, 3931, 4041, 0, 3219, 1, 1630, 0, 0, 3526, 1, 0, 0, 78, 0, 562, 0, 1, 2683, 0, 4095, 0, 1, 0, 1, 0, 1, 0, 49, 4095, 1, 1, 13, 1, 1, 1, 636, 1, 3904, 0, 4033, 24, 0, 4, 0, 12] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2281, 0, 1012, 0, 0, 70, 0, 4094, 2436, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 100, 4094, 0, 4096, 4095, 0, 0, 0, 4095, 42, 0, 1, 0, 4095, 1, 2, 4095, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 4093, 1440, 0, 0, 3865, 0, 4096, 4090, 0, 0, 4077, 4021, 0, 0, 4, 4096, 0, 0, 4094, 3, 0, 0, 0, 0, 1, 4096, 3, 0, 4096, 27, 250, 2097, 4095, 0, 0, 3921, 3873, 0, 2666, 0, 507, 1, 0, 2446, 0, 0, 0, 71, 1, 157, 0, 0, 3762, 0, 4091, 1, 0, 0, 0, 1, 0, 0, 60, 4095, 0, 0, 1, 0, 0, 0, 1759, 0, 4095, 0, 4093, 15, 0, 1, 0, 12] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1748, 0, 62, 0, 0, 1, 0, 4096, 2136, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 31, 0, 0, 0, 4096, 0, 0, 4096, 5, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 4095, 374, 0, 0, 3750, 0, 4096, 4096, 0, 0, 4077, 3970, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3, 743, 2695, 4090, 0, 0, 3998, 4086, 0, 3605, 0, 1865, 0, 0, 2527, 0, 0, 0, 123, 0, 647, 0, 0, 2189, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 0, 0, 0, 0, 0, 0, 2318, 0, 4094, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1908, 0, 616, 0, 0, 9, 0, 4090, 1523, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 6, 25, 4091, 0, 4096, 4088, 1, 0, 0, 4095, 3, 0, 3, 1, 4096, 11, 0, 4096, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 4091, 4089, 1326, 0, 0, 3917, 0, 4096, 3975, 0, 0, 4056, 3829, 0, 0, 10, 4096, 0, 0, 4090, 65, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 9, 358, 3070, 4096, 0, 0, 4034, 4008, 0, 2609, 0, 2003, 0, 0, 2696, 0, 0, 0, 9, 5, 344, 0, 0, 3537, 0, 4088, 0, 0, 0, 0, 6, 0, 0, 88, 4088, 0, 0, 6, 0, 0, 0, 982, 0, 4096, 0, 4092, 38, 0, 5, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 2742, 0, 1167, 0, 0, 88, 0, 4096, 1087, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 3, 11, 4090, 0, 4096, 4096, 0, 0, 0, 4096, 8, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 11, 0, 0, 4078, 4095, 1376, 0, 0, 4032, 0, 4096, 4096, 0, 0, 4064, 3713, 0, 0, 0, 4095, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 4096, 1, 0, 4095, 9, 160, 3791, 4096, 0, 0, 3987, 4064, 0, 2413, 0, 808, 0, 0, 2543, 0, 0, 0, 111, 0, 99, 0, 0, 3694, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1020, 0, 4088, 0, 4070, 0, 0, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 2397, 0, 1616, 0, 0, 108, 0, 4094, 912, 0, 0, 1, 0, 4095, 2357, 764, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 4095, 0, 4028, 4095, 0, 0, 0, 4095, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 4096, 1, 0, 4085, 1, 0, 0, 0, 4095, 0, 0, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 775, 0, 0, 4062, 0, 0, 3107, 1, 0, 4084, 4032, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 4096, 3, 0, 4095, 9, 169, 3860, 405, 67, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1738, 0, 0, 4031, 4036, 0, 2211, 0, 188, 0, 0, 2381, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 54, 1, 0, 3529, 0, 4057, 0, 0, 0, 0, 0, 0, 0, 8, 4095, 3330, 1607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 2314, 0, 4096, 1, 4095, 15, 2, 1, 0, 3, 4095, 2488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16901, 16541, 16372, 16064, 16811, 14589, 16682, 17112] avg=16384.0000 max=17112 max_rank=7 min=14589 min_rank=5 imbalance_ratio=1.044434 max_load_reduction=113960 max_load_reduction_pct=0.869446 moved_to_echo=65194 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 374, 0, 765, 0, 0, 25, 0, 4096, 2264, 0, 0, 0, 0, 4096, 2350, 764, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4096, 0, 4029, 4096, 0, 0, 1, 4095, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 1, 4085, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1802, 0, 0, 4082, 0, 0, 3128, 0, 0, 4043, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 9, 328, 3172, 408, 67, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1732, 0, 0, 4084, 4071, 0, 2738, 0, 2606, 0, 0, 2357, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 1872, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 105, 4096, 3331, 1608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1904, 0, 4096, 0, 4093, 8, 0, 1, 0, 1, 4096, 2488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15294, 16373, 16371, 17112, 16286, 17711, 15235, 16690] avg=16384.0000 max=17711 max_rank=5 min=15235 min_rank=6 imbalance_ratio=1.080994 max_load_reduction=113361 max_load_reduction_pct=0.864876 moved_to_echo=65922 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2612, 0, 541, 0, 0, 10, 0, 4096, 1462, 0, 0, 0, 0, 4096, 2299, 763, 558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4094, 0, 4029, 4095, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 3, 0, 4085, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2244, 0, 0, 4047, 0, 0, 3120, 0, 0, 4065, 3942, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 1, 4096, 1, 0, 4096, 1, 681, 3867, 407, 67, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1695, 0, 0, 4087, 4063, 0, 2069, 0, 955, 0, 0, 1795, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 2150, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 93, 4096, 3332, 1607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2071, 0, 4083, 0, 4095, 4, 0, 0, 0, 5, 4096, 2486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16437, 16335, 16386, 17423, 17332, 14688, 15631, 16840] avg=16384.0000 max=17423 max_rank=3 min=14688 min_rank=5 imbalance_ratio=1.063416 max_load_reduction=113649 max_load_reduction_pct=0.867073 moved_to_echo=64491 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1748, 0, 62, 0, 0, 1, 0, 4096, 2136, 0, 0, 0, 0, 4096, 2354, 764, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4029, 4096, 0, 0, 0, 4096, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 5, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 0, 0, 3750, 0, 0, 3128, 0, 0, 4077, 3970, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 3, 743, 2695, 408, 67, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1736, 0, 0, 3998, 4086, 0, 3605, 0, 1865, 0, 0, 2527, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 647, 0, 0, 2189, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 12, 4096, 3331, 1609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2318, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 4096, 2487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15817, 16349, 16379, 15299, 16217, 17940, 15980, 17091] avg=16384.0000 max=17940 max_rank=5 min=15299 min_rank=3 imbalance_ratio=1.094971 max_load_reduction=113132 max_load_reduction_pct=0.863129 moved_to_echo=67228 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1957, 0, 178, 1, 1, 2, 0, 4093, 2078, 0, 1, 1, 0, 4095, 2357, 764, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 5, 4094, 1, 4022, 4095, 1, 0, 2, 4092, 30, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4095, 6, 0, 4081, 1, 0, 0, 0, 4095, 1, 44, 1, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 1, 0, 3955, 0, 0, 3116, 0, 1, 4013, 4060, 1, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 18, 0, 0, 0, 0, 1, 4095, 1, 1, 4095, 0, 696, 2768, 412, 72, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1730, 2, 1, 3931, 4041, 0, 3219, 1, 1630, 0, 0, 3526, 1, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 562, 0, 1, 2683, 0, 4095, 0, 1, 0, 1, 0, 1, 0, 49, 4095, 3330, 1607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 13, 1, 1, 1, 636, 1, 3904, 0, 4033, 24, 0, 4, 0, 12, 4095, 2483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16090, 16345, 16422, 16156, 16264, 18160, 16425, 15210] avg=16384.0000 max=18160 max_rank=5 min=15210 min_rank=7 imbalance_ratio=1.108398 max_load_reduction=112912 max_load_reduction_pct=0.861450 moved_to_echo=66059 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2281, 0, 1012, 0, 0, 70, 0, 4094, 2436, 0, 0, 0, 0, 4096, 2357, 764, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 4094, 0, 4029, 4095, 0, 0, 0, 4095, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 2, 4084, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1440, 0, 0, 3865, 0, 0, 3124, 0, 0, 4077, 4021, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 1, 4096, 3, 0, 4096, 27, 250, 2097, 407, 67, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1738, 0, 0, 3921, 3873, 0, 2666, 0, 507, 1, 0, 2446, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 157, 0, 0, 3762, 0, 4091, 1, 0, 0, 0, 1, 0, 0, 60, 4095, 3329, 1608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1759, 0, 4095, 0, 4093, 15, 0, 1, 0, 12, 4096, 2488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17669, 16455, 16377, 16531, 15152, 15223, 17105, 16560] avg=16384.0000 max=17669 max_rank=0 min=15152 min_rank=4 imbalance_ratio=1.078430 max_load_reduction=113403 max_load_reduction_pct=0.865196 moved_to_echo=64040 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1908, 0, 616, 0, 0, 9, 0, 4090, 1523, 0, 0, 0, 0, 4096, 2358, 763, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6, 25, 4091, 0, 4029, 4088, 1, 0, 0, 4095, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 4096, 11, 0, 4085, 0, 0, 0, 0, 4096, 0, 15, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1326, 0, 0, 3917, 0, 0, 3036, 0, 0, 4056, 3829, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 65, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 9, 358, 3070, 396, 67, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1738, 0, 0, 4034, 4008, 0, 2609, 0, 2003, 0, 0, 2696, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 344, 0, 0, 3537, 0, 4088, 0, 0, 0, 0, 6, 0, 0, 88, 4088, 3326, 1609, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 982, 0, 4096, 0, 4092, 38, 0, 5, 0, 5, 4096, 2487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15906, 16340, 16398, 16174, 16259, 17097, 17091, 15807] avg=16384.0000 max=17097 max_rank=5 min=15807 min_rank=7 imbalance_ratio=1.043518 max_load_reduction=113975 max_load_reduction_pct=0.869560 moved_to_echo=66254 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 2742, 0, 1167, 0, 0, 88, 0, 4096, 1087, 0, 0, 0, 0, 4096, 2358, 764, 560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 11, 4090, 0, 4029, 4096, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 0, 0, 0, 1, 4096, 0, 11, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1376, 0, 0, 4032, 0, 0, 3128, 0, 0, 4064, 3713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 4096, 1, 0, 4095, 9, 160, 3791, 408, 67, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1738, 0, 0, 3987, 4064, 0, 2413, 0, 808, 0, 0, 2543, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 3694, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3331, 1608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1020, 0, 4088, 0, 4070, 0, 0, 0, 0, 1, 4095, 2488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16958, 16334, 16367, 16313, 16751, 15664, 16923, 15762] avg=16384.0000 max=16958 max_rank=0 min=15664 min_rank=5 imbalance_ratio=1.035034 max_load_reduction=114114 max_load_reduction_pct=0.870621 moved_to_echo=65100 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 32768, 128, 128, 32768, 128, 0, 0, 128, 32768, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 2304, 128, 128, 23424, 0, 32768, 128, 128, 0, 128, 128, 128, 0, 512, 32768, 26752, 12928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 128, 0, 0, 0, 0, 128, 32768, 128, 128, 32768, 128, 3456, 25344, 3328, 640, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[13952, 128, 128, 32128, 32256, 0, 21632, 128, 10624, 128, 0, 20352, 128, 0, 0, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 10368, 128, 0, 31744, 0, 0, 24960, 128, 128, 32512, 31744, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 128, 128, 128, 13056, 128, 32640, 128, 32768, 128, 128, 128, 0, 128, 32768, 19968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 128, 256, 32768, 128, 32256, 32768, 128, 0, 128, 32768, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 16128, 0, 6016, 128, 128, 384, 0, 32768, 13952, 0, 128, 128, 0, 32768, 18816, 6144, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3408, 4096, 0, 0, 0, 4093, 4096, 0, 0, 3760, 3768, 0, 21, 0, 0, 8, 0, 0, 14, 4092, 0, 3881, 3451, 0, 0, 4096, 0, 0, 4096, 4068, 0, 0, 0, 0, 4096, 0, 132, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 1, 0, 0, 4096, 132, 0, 0, 0, 0, 6, 2232, 1, 0, 4096, 0, 0, 0, 4, 4095, 0, 0, 4096, 4087, 0, 0, 0, 3265, 0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 61, 0, 0, 0, 0, 10, 0, 1, 1590, 4089, 0, 0, 4095, 0, 0, 3506, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 15, 4094, 0, 0, 0, 47, 0, 0, 0, 0, 0, 4096, 0, 3505, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3385, 4092, 0, 0, 2, 4095, 4096, 0, 0, 2738, 4015, 0, 1, 0, 0, 0, 0, 0, 11, 4095, 0, 3444, 3590, 0, 0, 4095, 0, 0, 4096, 4089, 0, 1, 0, 0, 4096, 0, 499, 4096, 11, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 0, 4096, 35, 0, 0, 0, 0, 2, 2006, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4079, 0, 0, 0, 3646, 0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 41, 0, 0, 0, 0, 9, 0, 22, 1608, 4083, 0, 0, 4094, 0, 0, 3813, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 68, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 3826, 0, 2, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 3030, 4092, 0, 0, 2, 4095, 4095, 0, 0, 4047, 3569, 1, 0, 0, 1, 45, 1, 1, 13, 4095, 0, 3609, 3282, 0, 1, 4085, 0, 0, 4094, 4093, 0, 67, 1, 0, 4095, 3, 154, 4095, 6, 0, 0, 0, 17, 1, 4095, 4095, 2, 1, 0, 4095, 125, 1, 0, 1, 1, 0, 1335, 0, 0, 4095, 0, 0, 0, 7, 4094, 1, 0, 4088, 4060, 1, 1, 1, 3974, 0, 0, 0, 4095, 0, 4087, 4094, 0, 0, 778, 1, 6, 0, 1, 25, 1, 0, 1790, 4056, 1, 0, 4090, 0, 1, 2841, 1, 0, 1, 1, 4095, 44, 1, 0, 0, 41, 4090, 1, 0, 1, 4, 0, 0, 1, 0, 0, 4095, 1, 4060, 1, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3211, 4095, 0, 0, 0, 4095, 4096, 0, 0, 3420, 4069, 0, 0, 3, 0, 10, 0, 0, 8, 4091, 1, 3468, 3821, 0, 0, 4093, 0, 0, 4094, 4080, 1, 0, 0, 0, 4095, 1, 208, 4094, 1, 1, 0, 0, 1, 0, 4096, 4096, 1, 0, 0, 4096, 57, 1, 0, 0, 0, 9, 487, 18, 0, 4096, 0, 2, 0, 1, 4095, 0, 0, 4096, 4091, 0, 0, 0, 4033, 0, 2, 0, 4096, 0, 4095, 4095, 0, 0, 148, 0, 1, 0, 0, 0, 0, 7, 1951, 4087, 0, 0, 4095, 0, 0, 4020, 0, 1, 0, 0, 4096, 14, 0, 0, 0, 50, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4095, 0, 3786, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2520, 4096, 0, 0, 0, 4096, 4096, 0, 0, 3642, 4032, 0, 3, 1, 0, 15, 0, 0, 2, 4096, 0, 3919, 3259, 0, 0, 4096, 0, 0, 4095, 4079, 0, 0, 0, 0, 4096, 0, 147, 4096, 2, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 62, 0, 0, 0, 0, 0, 1421, 3, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 4085, 0, 0, 0, 4080, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 109, 0, 2, 0, 0, 4, 0, 0, 2551, 4093, 0, 0, 4084, 0, 0, 2766, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 15, 4096, 0, 0, 0, 210, 0, 0, 0, 0, 0, 4096, 0, 4048, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2212, 4094, 1, 0, 48, 4094, 4096, 0, 0, 3474, 4082, 0, 31, 14, 0, 3, 0, 0, 10, 4090, 0, 3674, 3736, 0, 0, 4094, 0, 0, 4094, 4095, 0, 0, 0, 0, 4094, 0, 101, 4096, 4, 25, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 475, 0, 0, 0, 0, 1, 1987, 5, 0, 4096, 0, 0, 0, 1, 4091, 0, 0, 4096, 4080, 0, 0, 0, 4072, 0, 0, 0, 4096, 0, 4090, 4095, 0, 0, 173, 0, 11, 0, 0, 8, 0, 53, 1722, 4085, 0, 0, 4094, 0, 0, 3785, 0, 3, 0, 0, 4096, 1, 0, 0, 0, 104, 4095, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 3001, 0, 8, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1762, 4094, 2, 0, 0, 4077, 4096, 0, 0, 3293, 4035, 0, 0, 0, 0, 20, 0, 0, 49, 4066, 0, 3860, 3728, 0, 0, 4089, 0, 0, 4096, 4063, 0, 0, 0, 0, 4095, 1, 541, 4096, 13, 0, 0, 0, 4, 0, 4096, 4096, 0, 0, 0, 4096, 659, 1, 0, 0, 0, 2, 1931, 11, 0, 4096, 0, 0, 0, 2, 4083, 0, 0, 4094, 4071, 0, 0, 0, 4037, 2, 0, 0, 4096, 0, 4086, 4095, 0, 0, 59, 0, 18, 0, 0, 32, 0, 32, 2334, 4085, 0, 0, 4094, 0, 0, 3817, 0, 6, 0, 0, 4096, 3, 0, 0, 0, 105, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 2549, 0, 16, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1184, 4096, 0, 0, 2, 4096, 4096, 0, 0, 3494, 4089, 0, 1, 3, 0, 0, 0, 0, 7, 4096, 0, 3737, 4034, 0, 0, 4096, 0, 0, 4094, 4096, 0, 3, 0, 0, 4096, 0, 132, 4095, 0, 1, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 72, 0, 0, 0, 0, 1, 2978, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 4096, 4068, 0, 0, 0, 4086, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 158, 0, 2, 0, 0, 0, 0, 1, 880, 4095, 0, 2, 4088, 0, 0, 4021, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 147, 4095, 0, 0, 0, 132, 0, 0, 0, 0, 0, 4096, 0, 3635, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3408, 2256, 0, 0, 0, 4093, 0, 0, 0, 3760, 3768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 8, 0, 0, 14, 4092, 0, 3881, 3451, 0, 0, 4096, 0, 0, 823, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4068, 0, 0, 0, 0, 4096, 0, 132, 3852, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 2089, 132, 0, 0, 0, 0, 6, 2232, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 0, 0, 4096, 4087, 0, 0, 0, 3265, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2089, 0, 4094, 4096, 0, 0, 61, 0, 0, 0, 0, 10, 0, 1, 1590, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3506, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 15, 2007, 1517, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 47, 0, 0, 0, 0, 0, 4096, 0, 3505, 0, 1, 0, 4096, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17285, 16414, 16245, 16749, 15763, 16030, 16257, 16329] avg=16384.0000 max=17285 max_rank=0 min=15763 min_rank=4 imbalance_ratio=1.054993 max_load_reduction=113787 max_load_reduction_pct=0.868126 moved_to_echo=64379 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3385, 2254, 0, 0, 2, 4095, 0, 0, 0, 2738, 4015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 11, 4095, 0, 3444, 3590, 0, 0, 4095, 0, 0, 822, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4089, 0, 1, 0, 0, 4096, 0, 499, 3852, 11, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2089, 35, 0, 0, 0, 0, 2, 2006, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4079, 0, 0, 0, 3646, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2089, 0, 4094, 4096, 0, 0, 41, 0, 0, 0, 0, 9, 0, 22, 1608, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3813, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 68, 2007, 1517, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4096, 0, 3826, 0, 2, 0, 4096, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16489, 16086, 16646, 16422, 16133, 16042, 16637, 16617] avg=16384.0000 max=16646 max_rank=2 min=16042 min_rank=5 imbalance_ratio=1.015991 max_load_reduction=114426 max_load_reduction_pct=0.873001 moved_to_echo=65429 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3211, 2255, 0, 0, 0, 4095, 0, 0, 0, 3420, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 10, 0, 0, 8, 4091, 1, 3468, 3821, 0, 0, 4093, 0, 0, 823, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4080, 1, 0, 0, 0, 4095, 1, 208, 3850, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 2089, 57, 1, 0, 0, 0, 9, 487, 18, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 4095, 0, 0, 4096, 4091, 0, 0, 0, 4033, 0, 2, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2089, 0, 4095, 4095, 0, 0, 148, 0, 1, 0, 0, 0, 0, 7, 1951, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4020, 0, 1, 0, 0, 4096, 14, 0, 0, 0, 50, 2007, 1517, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4095, 0, 3786, 0, 0, 0, 4096, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17050, 16346, 16332, 14950, 16536, 16473, 16817, 16568] avg=16384.0000 max=17050 max_rank=0 min=14950 min_rank=3 imbalance_ratio=1.040649 max_load_reduction=114022 max_load_reduction_pct=0.869919 moved_to_echo=66394 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 3030, 2246, 0, 0, 2, 4095, 0, 0, 0, 4047, 3569, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 45, 1, 1, 13, 4095, 0, 3609, 3282, 0, 1, 4085, 0, 0, 825, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4093, 0, 67, 1, 0, 4095, 3, 154, 3843, 6, 0, 0, 0, 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 2, 1, 0, 2083, 125, 1, 0, 1, 1, 0, 1335, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4094, 1, 0, 4088, 4060, 1, 1, 1, 3974, 0, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2077, 0, 4087, 4094, 0, 0, 778, 1, 6, 0, 1, 25, 1, 0, 1790, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4090, 0, 1, 2841, 1, 0, 1, 1, 4095, 44, 1, 0, 0, 41, 2012, 1521, 1021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 0, 1, 4, 0, 0, 1, 0, 0, 4095, 1, 4060, 1, 0, 1, 4095, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16993, 15993, 16374, 15834, 16444, 16916, 15671, 16847] avg=16384.0000 max=16993 max_rank=0 min=15671 min_rank=6 imbalance_ratio=1.037170 max_load_reduction=114079 max_load_reduction_pct=0.870354 moved_to_echo=65878 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2520, 2256, 0, 0, 0, 4096, 0, 0, 0, 3642, 4032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 15, 0, 0, 2, 4096, 0, 3919, 3259, 0, 0, 4096, 0, 0, 823, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4079, 0, 0, 0, 0, 4096, 0, 147, 3852, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2089, 62, 0, 0, 0, 0, 0, 1421, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 4085, 0, 0, 0, 4080, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2088, 0, 4096, 4096, 0, 0, 109, 0, 2, 0, 0, 4, 0, 0, 2551, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 2766, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 15, 2007, 1516, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 210, 0, 0, 0, 0, 0, 4096, 0, 4048, 0, 0, 0, 4096, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16546, 16242, 16272, 15863, 16572, 17039, 15502, 17036] avg=16384.0000 max=17039 max_rank=5 min=15502 min_rank=6 imbalance_ratio=1.039978 max_load_reduction=114033 max_load_reduction_pct=0.870003 moved_to_echo=66149 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2212, 2254, 1, 0, 48, 4094, 0, 0, 0, 3474, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 14, 0, 3, 0, 0, 10, 4090, 0, 3674, 3736, 0, 0, 4094, 0, 0, 823, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 0, 0, 0, 0, 4094, 0, 101, 3852, 4, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2089, 475, 0, 0, 0, 0, 1, 1987, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 4096, 4080, 0, 0, 0, 4072, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2089, 0, 4090, 4095, 0, 0, 173, 0, 11, 0, 0, 8, 0, 53, 1722, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3785, 0, 3, 0, 0, 4096, 1, 0, 0, 0, 104, 2007, 1517, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 3001, 0, 8, 0, 4096, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16165, 16503, 16265, 16845, 16556, 16326, 16624, 15788] avg=16384.0000 max=16845 max_rank=3 min=15788 min_rank=7 imbalance_ratio=1.028137 max_load_reduction=114227 max_load_reduction_pct=0.871483 moved_to_echo=65294 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1762, 2254, 2, 0, 0, 4077, 0, 0, 0, 3293, 4035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 49, 4066, 0, 3860, 3728, 0, 0, 4089, 0, 0, 823, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4063, 0, 0, 0, 0, 4095, 1, 541, 3852, 13, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2089, 659, 1, 0, 0, 0, 2, 1931, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4083, 0, 0, 4094, 4071, 0, 0, 0, 4037, 2, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2089, 0, 4086, 4095, 0, 0, 59, 0, 18, 0, 0, 32, 0, 32, 2334, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3817, 0, 6, 0, 0, 4096, 3, 0, 0, 0, 105, 2007, 1517, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 2549, 0, 16, 0, 4096, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15423, 16663, 16665, 16981, 16505, 16830, 16662, 15343] avg=16384.0000 max=16981 max_rank=3 min=15343 min_rank=7 imbalance_ratio=1.036438 max_load_reduction=114091 max_load_reduction_pct=0.870445 moved_to_echo=65340 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 1184, 2256, 0, 0, 2, 4096, 0, 0, 0, 3494, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 7, 4096, 0, 3737, 4034, 0, 0, 4096, 0, 0, 823, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 3, 0, 0, 4096, 0, 132, 3851, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2089, 72, 0, 0, 0, 0, 1, 2978, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 4068, 0, 0, 0, 4086, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2088, 0, 4096, 4096, 0, 0, 158, 0, 2, 0, 0, 0, 0, 1, 880, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4088, 0, 0, 4021, 0, 0, 0, 0, 4096, 8, 0, 0, 0, 147, 2007, 1516, 1017, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 132, 0, 0, 0, 0, 0, 4096, 0, 3635, 0, 0, 0, 4096, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15121, 16825, 16273, 17428, 16563, 15416, 16902, 16544] avg=16384.0000 max=17428 max_rank=3 min=15121 min_rank=0 imbalance_ratio=1.063721 max_load_reduction=113644 max_load_reduction_pct=0.867035 moved_to_echo=65425 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 128, 128, 0, 32768, 128, 1920, 30848, 128, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 128, 0, 16768, 1664, 128, 0, 128, 128, 128, 14464, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 32768, 128, 29696, 28928, 0, 128, 32768, 0, 0, 6656, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 128, 20736, 18048, 128, 0, 128, 32768, 0, 0, 0, 27904, 31744, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 0, 32768, 32640, 128, 128, 128, 31232, 128, 128, 0, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 0, 128, 512, 0, 0, 128, 0, 0, 32768, 128, 28416, 128, 128, 128, 32768, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[16768, 0, 32768, 32768, 0, 0, 1536, 128, 128, 0, 128, 128, 128, 128, 14464, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 32768, 0, 128, 28672, 128, 128, 128, 128, 32768, 128, 128, 0, 0, 640, 16128, 12160, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[6, 0, 4066, 0, 0, 2132, 4096, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 4096, 4094, 0, 4096, 0, 4096, 4068, 0, 0, 0, 1, 2839, 4083, 0, 0, 13, 4096, 0, 56, 0, 22, 0, 0, 0, 5, 4095, 0, 0, 0, 4096, 0, 0, 1, 3932, 4096, 1, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 4096, 0, 0, 4096, 2670, 0, 0, 1836, 4096, 4095, 0, 1, 0, 4091, 0, 0, 4096, 0, 0, 5, 6, 2098, 4080, 0, 0, 0, 1, 0, 952, 0, 0, 4015, 8, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[71, 0, 4091, 0, 0, 2835, 4094, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 4096, 0, 4096, 0, 4096, 4030, 1, 0, 0, 0, 2296, 4093, 0, 0, 0, 4096, 0, 1, 0, 9, 0, 0, 0, 17, 4096, 0, 0, 0, 4096, 0, 0, 4, 4037, 4096, 0, 4052, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 125, 0, 0, 4096, 0, 0, 4096, 1477, 0, 0, 1782, 4096, 4096, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 7, 0, 3243, 4096, 0, 0, 0, 6, 0, 608, 0, 0, 4077, 6, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 4094, 0, 0, 3910, 4096, 0, 0, 0, 4096, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4096, 4096, 0, 4096, 0, 4096, 4072, 0, 0, 0, 0, 3409, 4074, 0, 0, 3, 4096, 0, 0, 0, 39, 0, 0, 0, 8, 4096, 0, 0, 0, 4096, 0, 0, 0, 4035, 4096, 0, 4020, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 69, 0, 0, 4095, 0, 0, 4096, 3490, 0, 0, 298, 4096, 4096, 0, 15, 0, 4093, 0, 0, 4096, 0, 0, 27, 0, 516, 4096, 0, 0, 0, 20, 0, 724, 0, 0, 4047, 4, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4077, 0, 1, 2634, 4094, 0, 0, 1, 4095, 12, 1, 1, 1, 1, 0, 1, 9, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 4076, 4095, 4086, 0, 4095, 1, 4095, 3991, 0, 0, 1, 2, 3331, 4068, 1, 0, 17, 4095, 0, 36, 1, 21, 0, 0, 0, 17, 4095, 1, 1, 0, 4095, 0, 0, 5, 4078, 4095, 0, 4011, 4093, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 1, 1, 1, 4090, 1, 0, 458, 0, 1, 4095, 1, 0, 4095, 1336, 0, 1, 1768, 4095, 4095, 1, 46, 0, 4085, 0, 1, 4094, 1, 0, 23, 30, 2718, 4095, 0, 0, 0, 25, 0, 101, 0, 0, 4082, 7, 4095, 0, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 3823, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4096, 4096, 0, 4096, 0, 4096, 4024, 0, 0, 0, 0, 2766, 4079, 0, 0, 0, 4096, 0, 14, 0, 17, 0, 0, 0, 26, 4096, 0, 0, 0, 4096, 0, 0, 0, 3981, 4096, 0, 4027, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 4096, 0, 0, 4096, 3260, 0, 0, 1137, 4096, 4096, 0, 26, 0, 4096, 0, 0, 4096, 0, 0, 11, 3, 1184, 4093, 0, 0, 0, 12, 0, 322, 0, 0, 4078, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 4078, 0, 0, 2709, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4096, 4095, 0, 4096, 0, 4096, 4089, 0, 0, 0, 0, 2578, 4084, 0, 0, 19, 4095, 0, 0, 0, 2, 0, 0, 0, 4, 4095, 0, 0, 0, 4096, 0, 0, 0, 4052, 4096, 1, 4091, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 42, 0, 0, 4096, 0, 1, 4096, 3102, 0, 0, 1657, 4095, 4094, 0, 11, 1, 4094, 0, 0, 4096, 0, 0, 21, 0, 2091, 4090, 0, 0, 0, 13, 0, 181, 0, 0, 4065, 4, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[13, 0, 4093, 0, 0, 3623, 4096, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 4096, 4094, 0, 4096, 0, 4096, 4066, 0, 0, 0, 1, 2300, 4044, 0, 0, 6, 4096, 0, 20, 0, 29, 0, 0, 0, 58, 4091, 0, 0, 0, 4096, 0, 0, 0, 3684, 4096, 2, 4016, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 10, 0, 0, 4096, 0, 0, 4096, 3300, 0, 0, 1325, 4096, 4092, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 25, 0, 2021, 4080, 0, 0, 0, 8, 0, 223, 0, 0, 4040, 16, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[1, 0, 4093, 0, 0, 3410, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 4096, 4096, 0, 4096, 0, 4096, 4045, 0, 0, 0, 0, 2694, 4080, 0, 0, 0, 4096, 0, 1, 0, 1, 0, 0, 0, 29, 4096, 0, 0, 0, 4096, 0, 0, 5, 4028, 4096, 0, 4082, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 4092, 0, 0, 4096, 3423, 0, 0, 1462, 4096, 4096, 0, 4, 0, 4096, 0, 0, 4096, 0, 0, 8, 5, 824, 4096, 0, 0, 0, 9, 0, 690, 0, 0, 4020, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[71, 0, 4091, 0, 0, 2835, 4094, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4096, 4096, 2653, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1381, 4030, 1, 0, 0, 0, 2296, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 17, 4096, 0, 0, 0, 4096, 0, 0, 4, 4037, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4052, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 125, 0, 0, 4096, 0, 0, 4096, 1477, 812, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1782, 2660, 4096, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3243, 4096, 0, 0, 0, 6, 0, 608, 0, 0, 4077, 6, 1816, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16140, 16407, 15897, 16418, 16387, 15139, 16736, 17948] avg=16384.0000 max=17948 max_rank=7 min=15139 min_rank=5 imbalance_ratio=1.095459 max_load_reduction=113124 max_load_reduction_pct=0.863068 moved_to_echo=66210 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 4066, 0, 0, 2132, 4096, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 4096, 4094, 2653, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1381, 4068, 0, 0, 0, 1, 2839, 4083, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 22, 0, 0, 0, 5, 4095, 0, 0, 0, 4096, 0, 0, 1, 3932, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4086, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 4096, 0, 0, 4096, 2670, 812, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1836, 2660, 4095, 0, 1, 0, 4091, 0, 0, 4096, 0, 0, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2098, 4080, 0, 0, 0, 1, 0, 952, 0, 0, 4015, 8, 1816, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15352, 16384, 16481, 16365, 16422, 16212, 16790, 17066] avg=16384.0000 max=17066 max_rank=7 min=15352 min_rank=0 imbalance_ratio=1.041626 max_load_reduction=114006 max_load_reduction_pct=0.869797 moved_to_echo=66490 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4077, 0, 1, 2634, 4094, 0, 0, 1, 4095, 12, 1, 1, 1, 1, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 4076, 4095, 4086, 2658, 1473, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1370, 3991, 0, 0, 1, 2, 3331, 4068, 1, 0, 17, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 1, 21, 0, 0, 0, 17, 4095, 1, 1, 0, 4095, 0, 0, 5, 4078, 4095, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4011, 4093, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 4095, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4090, 1, 0, 458, 0, 1, 4095, 1, 0, 4095, 1336, 812, 439, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1768, 2648, 4095, 1, 46, 0, 4085, 0, 1, 4094, 1, 0, 23, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2718, 4095, 0, 0, 0, 25, 0, 101, 0, 0, 4082, 7, 1810, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15875, 16404, 16877, 16512, 16349, 15331, 16793, 16931] avg=16384.0000 max=16931 max_rank=7 min=15331 min_rank=5 imbalance_ratio=1.033386 max_load_reduction=114141 max_load_reduction_pct=0.870827 moved_to_echo=65404 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 3823, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 4096, 4096, 2653, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1381, 4024, 0, 0, 0, 0, 2766, 4079, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 17, 0, 0, 0, 26, 4096, 0, 0, 0, 4096, 0, 0, 0, 3981, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4027, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 4096, 0, 0, 4096, 3260, 812, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1137, 2660, 4096, 0, 26, 0, 4096, 0, 0, 4096, 0, 0, 11, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1184, 4093, 0, 0, 0, 12, 0, 322, 0, 0, 4078, 0, 1816, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17060, 16388, 16346, 16388, 16362, 16802, 16125, 15601] avg=16384.0000 max=17060 max_rank=0 min=15601 min_rank=7 imbalance_ratio=1.041260 max_load_reduction=114012 max_load_reduction_pct=0.869843 moved_to_echo=64890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4078, 0, 0, 2709, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 1, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 4096, 4095, 2653, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1381, 4089, 0, 0, 0, 0, 2578, 4084, 0, 0, 19, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, 4095, 0, 0, 0, 4096, 0, 0, 0, 4052, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4091, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 42, 0, 0, 4096, 0, 1, 4096, 3102, 812, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1657, 2659, 4094, 0, 11, 1, 4094, 0, 0, 4096, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2091, 4090, 0, 0, 0, 13, 0, 181, 0, 0, 4065, 4, 1816, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15940, 16379, 16246, 16407, 16428, 16683, 16633, 16356] avg=16384.0000 max=16683 max_rank=5 min=15940 min_rank=0 imbalance_ratio=1.018250 max_load_reduction=114389 max_load_reduction_pct=0.872719 moved_to_echo=66100 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4094, 0, 0, 3910, 4096, 0, 0, 0, 4096, 26, 0, 0, 0, 0, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4063, 4096, 4096, 2653, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1381, 4072, 0, 0, 0, 0, 3409, 4074, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 8, 4096, 0, 0, 0, 4096, 0, 0, 0, 4035, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4020, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 69, 0, 0, 4095, 0, 0, 4096, 3490, 812, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 2660, 4096, 0, 15, 0, 4093, 0, 0, 4096, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 4096, 0, 0, 0, 20, 0, 724, 0, 0, 4047, 4, 1816, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17175, 16376, 17035, 16432, 16355, 17095, 15285, 15319] avg=16384.0000 max=17175 max_rank=0 min=15285 min_rank=6 imbalance_ratio=1.048279 max_load_reduction=113897 max_load_reduction_pct=0.868965 moved_to_echo=64054 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[13, 0, 4093, 0, 0, 3623, 4096, 0, 0, 0, 4096, 8, 0, 0, 0, 0, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 4096, 4094, 2653, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1381, 4066, 0, 0, 0, 1, 2300, 4044, 0, 0, 6, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 29, 0, 0, 0, 58, 4091, 0, 0, 0, 4096, 0, 0, 0, 3684, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4016, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 10, 0, 0, 4096, 0, 0, 4096, 3300, 812, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 2660, 4092, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2021, 4080, 0, 0, 0, 8, 0, 223, 0, 0, 4040, 16, 1816, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16881, 16376, 15894, 16136, 16353, 16849, 16283, 16300] avg=16384.0000 max=16881 max_rank=0 min=15894 min_rank=2 imbalance_ratio=1.030334 max_load_reduction=114191 max_load_reduction_pct=0.871208 moved_to_echo=65785 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 4093, 0, 0, 3410, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 4096, 4096, 2653, 1468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1381, 4045, 0, 0, 0, 0, 2694, 4080, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 29, 4096, 0, 0, 0, 4096, 0, 0, 5, 4028, 4096, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4093, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 4092, 0, 0, 4096, 3423, 812, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1462, 2660, 4096, 0, 4, 0, 4096, 0, 0, 4096, 0, 0, 8, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 824, 4096, 0, 0, 0, 9, 0, 690, 0, 0, 4020, 0, 1816, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16649, 16358, 16296, 16414, 16416, 16961, 16427, 15551] avg=16384.0000 max=16961 max_rank=5 min=15551 min_rank=7 imbalance_ratio=1.035217 max_load_reduction=114111 max_load_reduction_pct=0.870598 moved_to_echo=65355 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 32512, 32768, 128, 128, 0, 128, 0, 128, 0, 128, 0, 0, 128, 32768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 0, 128, 128, 128, 0, 0, 0, 128, 128, 32640, 32768, 32768, 21248, 11776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 32768, 0, 128, 25088, 32768, 0, 0, 128, 32768, 128, 128, 128, 128, 128, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[14720, 32768, 0, 0, 0, 128, 0, 3840, 0, 0, 32512, 128, 14592, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 256, 0, 0, 0, 256, 32768, 128, 128, 0, 32768, 0, 0, 128, 31872, 32768, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 11392, 21376, 32768, 128, 128, 128, 32768, 0, 128, 32768, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 128, 128, 768, 0, 128, 32768, 128, 128, 32768, 22144, 6528, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 0, 128, 11136, 32512, 128, 0, 128, 128, 22272, 32640, 128, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3742, 0, 0, 48, 0, 3, 0, 0, 2, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 1, 0, 0, 3094, 4087, 0, 0, 0, 0, 0, 0, 4096, 3245, 54, 0, 0, 0, 0, 4094, 0, 0, 0, 3077, 0, 0, 0, 4052, 0, 4087, 4096, 952, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 4, 3858, 4096, 2386, 0, 0, 0, 4, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 59, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 9, 0, 0, 3, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4095, 3945, 0, 0, 32, 0, 6, 0, 0, 0, 4082, 0, 4095, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 2, 1, 0, 2215, 4092, 0, 0, 0, 0, 0, 0, 4095, 3677, 66, 0, 0, 3, 0, 4096, 0, 0, 0, 3113, 0, 0, 0, 4084, 0, 4095, 4096, 1132, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 4087, 2, 3751, 4052, 2533, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 0, 4096, 59, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 4096, 0, 0, 25, 0, 0, 1, 0, 6, 0, 0, 0, 3, 0, 0, 0, 4095, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3839, 0, 0, 107, 0, 25, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 3617, 4094, 0, 0, 0, 0, 0, 1, 4096, 2764, 35, 0, 0, 0, 0, 4096, 0, 0, 0, 2068, 0, 0, 0, 4078, 0, 4094, 4096, 2059, 0, 11, 0, 0, 0, 1, 0, 0, 0, 4096, 4090, 0, 3685, 4095, 2184, 0, 0, 0, 0, 4093, 0, 4096, 4096, 0, 0, 0, 4096, 68, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 45, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3541, 0, 0, 94, 0, 5, 0, 0, 0, 4075, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 3142, 4094, 0, 0, 0, 0, 0, 3, 4096, 3810, 54, 0, 0, 0, 0, 4096, 0, 0, 0, 1758, 0, 0, 0, 4069, 0, 4096, 4096, 1884, 0, 23, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 1, 3817, 4094, 2385, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 0, 4096, 19, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4095, 3853, 0, 0, 106, 0, 2, 0, 0, 1, 4095, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 2, 0, 0, 3252, 4085, 4, 0, 0, 0, 0, 2, 4096, 3481, 16, 0, 0, 2, 0, 4094, 0, 0, 0, 2687, 0, 0, 0, 4080, 0, 4069, 4096, 844, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 1, 3772, 4072, 2311, 0, 0, 0, 0, 4091, 0, 4096, 4096, 0, 0, 0, 4096, 214, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 13, 0, 0, 1, 0, 6, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 1, 0, 4095, 4095, 3182, 1, 0, 164, 1, 7, 0, 0, 0, 4092, 1, 4088, 1, 0, 1, 0, 4, 1, 4095, 4095, 1, 0, 0, 1, 4095, 4095, 4095, 0, 0, 6, 0, 1, 1405, 4087, 0, 0, 0, 0, 0, 13, 4095, 2900, 9, 0, 1, 0, 1, 4095, 0, 1, 1, 2049, 0, 1, 0, 4054, 0, 4087, 4095, 2703, 1, 160, 1, 1, 0, 0, 1, 0, 1, 4091, 4084, 8, 3874, 4094, 3548, 2, 0, 0, 3, 4089, 1, 4095, 4095, 1, 0, 0, 4095, 514, 0, 1, 0, 1, 1, 0, 0, 4086, 0, 4095, 4083, 0, 1, 11, 0, 0, 12, 0, 0, 1, 0, 1, 11, 1, 0, 0, 4094, 1, 0, 0, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4093, 3954, 0, 0, 98, 0, 20, 0, 0, 0, 4084, 0, 4096, 0, 0, 0, 0, 5, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 5, 0, 0, 2581, 4093, 0, 0, 0, 0, 0, 3, 4096, 4036, 85, 0, 0, 3, 0, 4085, 0, 0, 0, 1503, 0, 0, 0, 4030, 0, 4090, 4096, 553, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4095, 4076, 5, 4004, 4086, 3578, 0, 0, 0, 21, 4074, 0, 4096, 4096, 0, 0, 0, 4096, 185, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 4096, 4095, 0, 0, 1, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3635, 0, 0, 6, 0, 0, 0, 0, 0, 4093, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 2335, 4095, 1, 0, 0, 0, 0, 7, 4096, 4057, 10, 0, 0, 1, 0, 4096, 0, 0, 0, 1976, 0, 0, 0, 4085, 0, 4075, 4096, 1127, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 16, 3772, 4087, 3299, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 264, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4095, 3945, 0, 0, 32, 0, 6, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 4095, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1390, 4096, 0, 0, 2, 1, 0, 2215, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2406, 3677, 66, 0, 0, 3, 0, 4096, 0, 0, 0, 3113, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2675, 1132, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 4087, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3751, 4052, 2533, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 1438, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 4096, 0, 0, 25, 0, 0, 1689, 1278, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 3, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 2706, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16563, 16377, 15892, 17445, 16087, 15927, 16398, 16383] avg=16384.0000 max=17445 max_rank=3 min=15892 min_rank=2 imbalance_ratio=1.064758 max_load_reduction=113627 max_load_reduction_pct=0.866905 moved_to_echo=64795 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3742, 0, 0, 48, 0, 3, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1390, 4096, 0, 0, 1, 0, 0, 3094, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2406, 3245, 54, 0, 0, 0, 0, 4094, 0, 0, 0, 3077, 0, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 2675, 952, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3858, 4096, 2386, 0, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 1438, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 9, 0, 0, 1690, 1278, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2706, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16387, 16764, 16928, 15907, 15937, 16383, 16391] avg=16384.0000 max=16928 max_rank=3 min=15907 min_rank=4 imbalance_ratio=1.033203 max_load_reduction=114144 max_load_reduction_pct=0.870850 moved_to_echo=64618 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3839, 0, 0, 107, 0, 25, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1390, 4096, 0, 0, 0, 0, 0, 3617, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2406, 2764, 35, 0, 0, 0, 0, 4096, 0, 0, 0, 2068, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2675, 2059, 0, 11, 0, 0, 0, 1, 0, 0, 0, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3685, 4095, 2184, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 1438, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 45, 0, 0, 1690, 1278, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2706, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16553, 16390, 17294, 15447, 17026, 15563, 16419, 16380] avg=16384.0000 max=17294 max_rank=2 min=15447 min_rank=3 imbalance_ratio=1.055542 max_load_reduction=113778 max_load_reduction_pct=0.868057 moved_to_echo=65388 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3541, 0, 0, 94, 0, 5, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1390, 4096, 0, 0, 0, 0, 0, 3142, 4094, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2406, 3810, 54, 0, 0, 0, 0, 4096, 0, 0, 0, 1758, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2675, 1884, 0, 23, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3817, 4094, 2385, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1438, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 4096, 0, 0, 1, 0, 0, 1690, 1278, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2706, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16222, 16370, 16821, 16193, 16869, 15848, 16373, 16376] avg=16384.0000 max=16869 max_rank=4 min=15848 min_rank=5 imbalance_ratio=1.029602 max_load_reduction=114203 max_load_reduction_pct=0.871300 moved_to_echo=65466 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 1, 0, 4095, 4095, 3182, 1, 0, 164, 1, 7, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 4088, 1, 0, 1, 0, 4, 1, 4095, 4095, 1, 0, 0, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1388, 4095, 0, 0, 6, 0, 1, 1405, 4087, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2402, 2900, 9, 0, 1, 0, 1, 4095, 0, 1, 1, 2049, 0, 1, 0, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 2662, 2703, 1, 160, 1, 1, 0, 0, 1, 0, 1, 4091, 4084, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3874, 4094, 3548, 2, 0, 0, 3, 4089, 1, 0, 0, 1, 0, 0, 1431, 514, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 4086, 0, 4095, 4083, 0, 1, 11, 0, 0, 1693, 1283, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 1, 0, 1, 11, 1, 0, 0, 4094, 1, 0, 0, 1, 1, 4095, 4095, 2707, 1381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15943, 16387, 15090, 15514, 17800, 17557, 16380, 16401] avg=16384.0000 max=17800 max_rank=4 min=15090 min_rank=2 imbalance_ratio=1.086426 max_load_reduction=113272 max_load_reduction_pct=0.864197 moved_to_echo=68138 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4095, 3853, 0, 0, 106, 0, 2, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1390, 4096, 0, 0, 2, 0, 0, 3252, 4085, 4, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2406, 3481, 16, 0, 0, 2, 0, 4094, 0, 0, 0, 2687, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 2675, 844, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3772, 4072, 2311, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 1438, 214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 13, 0, 0, 1690, 1278, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 2706, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16542, 16392, 16927, 16766, 15778, 15898, 16388, 16381] avg=16384.0000 max=16927 max_rank=2 min=15778 min_rank=4 imbalance_ratio=1.033142 max_load_reduction=114145 max_load_reduction_pct=0.870857 moved_to_echo=64445 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4093, 3954, 0, 0, 98, 0, 20, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4096, 0, 0, 0, 0, 5, 0, 4096, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1390, 4096, 0, 0, 5, 0, 0, 2581, 4093, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2406, 4036, 85, 0, 0, 3, 0, 4085, 0, 0, 0, 1503, 0, 0, 0, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 2675, 553, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4095, 4076, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 4086, 3578, 0, 0, 0, 21, 4074, 0, 0, 0, 0, 0, 0, 1438, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 4096, 4095, 0, 0, 1, 0, 0, 1690, 1278, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2706, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16651, 16384, 16264, 16148, 15505, 17386, 16352, 16382] avg=16384.0000 max=17386 max_rank=5 min=15505 min_rank=4 imbalance_ratio=1.061157 max_load_reduction=113686 max_load_reduction_pct=0.867355 moved_to_echo=65625 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 3635, 0, 0, 6, 0, 0, 0, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1390, 4096, 0, 0, 0, 0, 0, 2335, 4095, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2406, 4057, 10, 0, 0, 1, 0, 4096, 0, 0, 0, 1976, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 2675, 1127, 0, 15, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3772, 4087, 3299, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1438, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 4, 0, 0, 1690, 1278, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2706, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16223, 16385, 16020, 16631, 16100, 16956, 16379, 16378] avg=16384.0000 max=16956 max_rank=5 min=16020 min_rank=2 imbalance_ratio=1.034912 max_load_reduction=114116 max_load_reduction_pct=0.870636 moved_to_echo=65813 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 32768, 128, 0, 128, 0, 128, 128, 32768, 32768, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 128, 0, 128, 0, 32768, 32768, 29696, 128, 0, 768, 128, 128, 0, 0, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[30592, 32768, 22272, 128, 0, 0, 128, 32768, 128, 0, 0, 128, 0, 0, 11520, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 0, 128, 128, 0, 0, 32768, 0, 32768, 32768, 0, 128, 128, 0, 0, 13568, 10240, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[19328, 28032, 384, 0, 128, 128, 128, 32768, 0, 128, 128, 18304, 0, 128, 0, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 11136, 32768, 0, 0, 128, 128, 128, 21760, 32768, 128, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 0, 128, 128, 128, 0, 0, 32768, 128, 0, 0, 128, 128, 32768, 32768, 21760, 11136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 32768, 21504, 11264, 128, 256, 128, 128, 0, 128, 128, 0, 128, 32768, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 3926, 0, 4096, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 980, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4095, 4095, 0, 4093, 4096, 0, 2, 4089, 1727, 0, 0, 4096, 0, 6, 4090, 4096, 0, 0, 0, 0, 0, 4096, 212, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 591, 0, 0, 0, 963, 0, 0, 0, 4096, 0, 4096, 4096, 0, 6, 4096, 4096, 4096, 4096, 10, 0, 3, 0, 1, 0, 3, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 3811, 4087, 0, 0, 0, 0, 4096, 0, 0, 25, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 3833, 0, 4096, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 4093, 0, 0, 4096, 4089, 0, 4093, 4096, 0, 233, 4092, 1443, 0, 4, 4096, 0, 10, 4078, 4096, 0, 0, 0, 0, 0, 4094, 673, 0, 0, 0, 4092, 0, 0, 4096, 0, 0, 0, 0, 1, 267, 0, 0, 0, 1160, 0, 0, 0, 4096, 0, 4096, 4096, 0, 5, 4096, 4096, 4095, 4093, 89, 0, 2, 0, 0, 0, 5, 0, 4096, 4092, 0, 0, 0, 4096, 0, 0, 3933, 4009, 0, 0, 0, 0, 4096, 0, 0, 79, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 4020, 0, 4096, 0, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 912, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4094, 4096, 0, 4096, 4096, 0, 7, 4088, 1195, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 116, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 0, 3, 202, 0, 0, 0, 1663, 0, 0, 0, 4096, 0, 4096, 4096, 0, 1, 4096, 4096, 4096, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 3676, 3465, 0, 0, 0, 0, 4096, 0, 0, 478, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 3996, 0, 4095, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 727, 0, 1, 0, 4096, 0, 0, 4095, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4091, 0, 0, 4096, 4095, 0, 4095, 4096, 0, 177, 4089, 1112, 0, 4, 4095, 0, 5, 4075, 4096, 1, 0, 0, 0, 1, 4095, 1164, 0, 0, 0, 4090, 0, 0, 4096, 0, 0, 0, 0, 6, 280, 1, 0, 0, 766, 0, 0, 0, 4096, 0, 4096, 4096, 0, 10, 4095, 4096, 4095, 4094, 8, 0, 2, 0, 1, 0, 0, 0, 4096, 4093, 0, 0, 0, 4095, 0, 0, 4006, 4060, 0, 0, 0, 0, 4096, 0, 0, 39, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 3782, 0, 4096, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1196, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 2, 0, 0, 4089, 0, 0, 4096, 4094, 0, 4095, 4096, 0, 313, 4093, 1412, 0, 6, 4094, 0, 3, 4089, 4096, 0, 0, 0, 0, 0, 4094, 310, 0, 0, 0, 4081, 0, 0, 4096, 0, 0, 0, 0, 4, 168, 1, 0, 0, 912, 0, 0, 0, 4096, 0, 4096, 4094, 0, 11, 4096, 4096, 4095, 4095, 2, 0, 2, 0, 2, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 4014, 4069, 0, 0, 0, 0, 4096, 0, 0, 174, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1, 3229, 0, 4095, 1, 237, 1, 0, 1, 0, 1, 0, 0, 1, 0, 472, 0, 0, 1, 4095, 1, 1, 4094, 1, 1, 1, 6, 4095, 0, 6, 1, 0, 4085, 0, 0, 3994, 4079, 1, 4091, 4094, 1, 132, 4076, 128, 0, 1, 4095, 0, 19, 4005, 4095, 0, 0, 1, 1, 0, 4095, 1081, 0, 0, 0, 4095, 0, 0, 4095, 1, 1, 0, 0, 89, 56, 0, 1, 1, 2657, 3, 0, 0, 4095, 0, 4095, 3968, 0, 3, 4095, 4095, 4089, 4095, 168, 1, 0, 0, 0, 1, 12, 0, 4095, 4093, 1, 1, 1, 4095, 0, 1, 3893, 4069, 1, 1, 0, 0, 4095, 0, 0, 493, 4095, 0, 1, 1, 0, 0, 1, 2, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[0, 3711, 0, 4096, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2005, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 5, 4096, 0, 8, 0, 0, 4090, 0, 0, 4096, 4093, 0, 4088, 4096, 0, 33, 4077, 1422, 0, 8, 4092, 0, 30, 4071, 4096, 3, 0, 0, 0, 0, 4094, 362, 0, 0, 0, 4067, 0, 0, 4096, 0, 0, 0, 0, 11, 281, 0, 0, 0, 527, 0, 0, 0, 4096, 0, 4096, 4096, 0, 25, 4096, 4096, 4091, 4084, 0, 0, 5, 0, 7, 0, 0, 0, 4096, 4093, 0, 0, 0, 4096, 0, 0, 3878, 4025, 0, 0, 0, 0, 4096, 0, 0, 121, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 3680, 0, 4096, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4094, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 279, 4094, 1490, 0, 0, 4096, 0, 10, 4074, 4096, 0, 0, 0, 0, 0, 4096, 135, 0, 0, 0, 4095, 0, 0, 4096, 0, 1, 0, 0, 6, 271, 0, 0, 0, 951, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 4095, 2, 0, 0, 0, 0, 0, 4, 0, 4096, 4094, 0, 0, 0, 4096, 0, 0, 3999, 3965, 0, 0, 0, 0, 4096, 0, 0, 409, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3833, 0, 4096, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 4096, 3035, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 2213, 1360, 484, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 2729, 0, 0, 0, 0, 233, 4092, 1443, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4078, 3612, 0, 0, 0, 0, 0, 4094, 673, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 267, 0, 0, 0, 1160, 0, 0, 0, 4096, 0, 4093, 1558, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 4065, 0, 4095, 4093, 89, 0, 2, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 3933, 4009, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15973, 16378, 16690, 16559, 16332, 16450, 16455, 16235] avg=16384.0000 max=16690 max_rank=2 min=15973 min_rank=0 imbalance_ratio=1.018677 max_load_reduction=114382 max_load_reduction_pct=0.872665 moved_to_echo=65472 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3926, 0, 4096, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 980, 4096, 3035, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2213, 1362, 484, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 2733, 0, 0, 0, 0, 2, 4089, 1727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4090, 3612, 0, 0, 0, 0, 0, 4096, 212, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 591, 0, 0, 0, 963, 0, 0, 0, 4096, 0, 4093, 1558, 1061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 4065, 0, 4096, 4096, 10, 0, 3, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 3811, 4087, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16418, 16378, 16742, 16105, 16458, 16376, 16414, 16181] avg=16384.0000 max=16742 max_rank=2 min=16105 min_rank=3 imbalance_ratio=1.021851 max_load_reduction=114330 max_load_reduction_pct=0.872269 moved_to_echo=65429 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3996, 0, 4095, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 727, 4096, 3035, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4095, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 2213, 1362, 484, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 2733, 0, 0, 0, 0, 177, 4089, 1112, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4075, 3612, 1, 0, 0, 0, 1, 4095, 1164, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 280, 1, 0, 0, 766, 0, 0, 0, 4096, 0, 4095, 1558, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 4064, 0, 4095, 4094, 8, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4095, 0, 0, 4006, 4060, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4096, 4096, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16244, 16379, 16302, 17043, 15958, 16370, 16579, 16197] avg=16384.0000 max=17043 max_rank=3 min=15958 min_rank=4 imbalance_ratio=1.040222 max_load_reduction=114029 max_load_reduction_pct=0.869972 moved_to_echo=65104 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 4020, 0, 4096, 0, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 912, 4096, 3036, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2213, 1362, 484, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 2734, 0, 0, 0, 0, 7, 4088, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3612, 0, 0, 0, 0, 0, 4096, 116, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3, 202, 0, 0, 0, 1663, 0, 0, 0, 4096, 0, 4096, 1558, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 4065, 0, 4096, 4096, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 3676, 3465, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17018, 16377, 16214, 16014, 16774, 16384, 15657, 16634] avg=16384.0000 max=17018 max_rank=0 min=15657 min_rank=6 imbalance_ratio=1.038696 max_load_reduction=114054 max_load_reduction_pct=0.870163 moved_to_echo=65449 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 3229, 0, 4095, 1, 237, 1, 0, 1, 0, 1, 0, 0, 1, 0, 472, 4094, 3036, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 4094, 1, 1, 1, 6, 4095, 0, 6, 1, 0, 2218, 1360, 491, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 3994, 2719, 1, 0, 0, 1, 132, 4076, 128, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 4005, 3604, 0, 0, 1, 1, 0, 4095, 1081, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 89, 56, 0, 1, 1, 2657, 3, 0, 0, 4095, 0, 4091, 1562, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 3, 4057, 0, 4089, 4095, 168, 1, 0, 0, 0, 1, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 1, 1, 4095, 0, 1, 3893, 4069, 1, 1, 0, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 493, 4095, 0, 1, 1, 0, 0, 1, 2, 1, 0, 0, 0, 0, 0, 4095, 4095, 3867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15397, 16410, 15137, 16901, 17711, 16394, 16471, 16651] avg=16384.0000 max=17711 max_rank=4 min=15137 min_rank=2 imbalance_ratio=1.080994 max_load_reduction=113361 max_load_reduction_pct=0.864876 moved_to_echo=67227 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3782, 0, 4096, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1196, 4096, 3035, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4096, 0, 2, 0, 0, 2213, 1361, 484, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 2733, 0, 0, 0, 0, 313, 4093, 1412, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4089, 3612, 0, 0, 0, 0, 0, 4094, 310, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 168, 1, 0, 0, 912, 0, 0, 0, 4096, 0, 4095, 1558, 1059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 11, 4065, 0, 4095, 4095, 2, 0, 2, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4014, 4069, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16473, 16381, 16742, 16189, 15989, 16367, 16600, 16331] avg=16384.0000 max=16742 max_rank=2 min=15989 min_rank=4 imbalance_ratio=1.021851 max_load_reduction=114330 max_load_reduction_pct=0.872269 moved_to_echo=65287 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3711, 0, 4096, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2005, 4096, 3034, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 5, 4096, 0, 8, 0, 0, 2213, 1361, 484, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 2732, 0, 0, 0, 0, 33, 4077, 1422, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4071, 3612, 3, 0, 0, 0, 0, 4094, 362, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 11, 281, 0, 0, 0, 527, 0, 0, 0, 4096, 0, 4088, 1558, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 25, 4065, 0, 4091, 4084, 0, 0, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 3878, 4025, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17201, 16391, 16458, 16239, 15715, 16373, 16417, 16278] avg=16384.0000 max=17201 max_rank=0 min=15715 min_rank=4 imbalance_ratio=1.049866 max_load_reduction=113871 max_load_reduction_pct=0.868767 moved_to_echo=64783 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3680, 0, 4096, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 4096, 3036, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 2213, 1362, 484, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4096, 2734, 0, 0, 0, 0, 279, 4094, 1490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4074, 3612, 0, 0, 0, 0, 0, 4096, 135, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 6, 271, 0, 0, 0, 951, 0, 0, 0, 4096, 0, 4096, 1558, 1060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4065, 0, 4096, 4095, 2, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 3999, 3965, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 409, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16348, 16378, 16787, 16022, 16135, 16358, 16479, 16565] avg=16384.0000 max=16787 max_rank=2 min=16022 min_rank=3 imbalance_ratio=1.024597 max_load_reduction=114285 max_load_reduction_pct=0.871925 moved_to_echo=65537 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 30208, 0, 32768, 128, 1280, 128, 0, 128, 0, 128, 0, 0, 128, 0, 8192, 32768, 24320, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 32640, 28928, 128, 0, 128, 128, 128, 32768, 4096, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 0, 0, 128, 2176, 128, 128, 128, 9600, 128, 0, 0, 32768, 0, 32768, 12544, 8576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 0, 32768, 21888, 128, 0, 0, 128, 1280, 32768, 9984, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32640, 0, 128, 32512, 0, 32768, 32768, 384, 128, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 32768, 0, 128, 128, 0, 17792, 11008, 3968, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32768, 128, 128, 128, 32768, 0, 128, 31232, 31872, 128, 128, 0, 0, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 1920, 32768, 0, 128, 128, 0, 0, 128, 128, 128, 0, 0, 128, 128, 0, 32768, 32768, 30976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[22, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 248, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4094, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 4096, 0, 0, 4096, 4087, 4090, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 153, 0, 0, 3910, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 4095, 0, 3866, 3, 0, 4084, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[5, 0, 0, 4095, 0, 0, 4, 0, 0, 0, 23, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4093, 4088, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 56, 3, 4096, 0, 0, 4096, 4029, 4071, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 12, 0, 0, 4087, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 4096, 0, 4071, 30, 0, 4092, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[6, 0, 0, 4095, 0, 0, 1, 1, 0, 0, 22, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4094, 4091, 0, 0, 0, 4093, 0, 0, 0, 0, 1, 0, 4095, 59, 0, 4095, 0, 0, 4096, 4054, 4056, 4095, 1, 0, 0, 38, 1, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4095, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 5, 0, 0, 1, 0, 0, 0, 0, 4096, 1, 4094, 0, 0, 0, 101, 1, 0, 4022, 0, 0, 1, 0, 5, 0, 4095, 0, 0, 4095, 0, 4032, 3, 0, 4088, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[27, 1, 0, 4082, 0, 1, 25, 0, 1, 0, 10, 1, 1, 0, 4095, 0, 4006, 0, 1, 0, 0, 4095, 4095, 0, 0, 0, 4092, 1, 2, 0, 0, 0, 0, 4086, 171, 17, 4095, 0, 1, 4082, 4089, 4077, 4095, 0, 0, 0, 6, 0, 1, 2, 1, 0, 1, 0, 4095, 0, 1, 0, 0, 1, 1, 4056, 4095, 1, 0, 0, 1, 0, 1, 1, 0, 0, 4095, 1, 4087, 4094, 1, 1, 1, 0, 4095, 1, 4095, 0, 1, 41, 1, 1, 0, 1, 0, 0, 0, 4095, 0, 4095, 1, 1, 1, 207, 0, 0, 4041, 0, 0, 0, 1, 1, 0, 4095, 1, 1, 4095, 0, 3830, 14, 1, 4091, 4095, 0, 0, 4095, 0, 1, 4095, 4091, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[32, 0, 0, 4096, 0, 0, 20, 0, 0, 0, 27, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 4087, 4053, 0, 0, 0, 4077, 0, 2, 0, 0, 4, 0, 4095, 18, 0, 4095, 0, 0, 4096, 4026, 4055, 4096, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 0, 6, 0, 0, 0, 0, 2, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 0, 175, 0, 0, 4043, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4091, 0, 4012, 42, 0, 4063, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[19, 0, 0, 4096, 0, 0, 16, 0, 0, 0, 291, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4094, 4084, 0, 0, 0, 4094, 0, 1, 0, 0, 1, 0, 4095, 17, 0, 4095, 0, 0, 4096, 4005, 4077, 4096, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 461, 0, 0, 3506, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4095, 0, 4006, 2, 0, 4093, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[98, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 4080, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 4096, 0, 0, 4096, 4094, 4064, 4096, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 36, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4029, 14, 0, 4088, 4096, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[63, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 4096, 0, 4083, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 41, 0, 4096, 0, 0, 4096, 4002, 4082, 4096, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 548, 0, 1, 3835, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3779, 10, 0, 4089, 4096, 0, 0, 4096, 0, 0, 4096, 4095, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 248, 0, 0, 0, 4096, 0, 4096, 3915, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 4093, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 22, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 12, 0, 0, 4096, 4087, 4090, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3920, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 3933, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 3910, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 4096, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3866, 3, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 141, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16536, 16404, 16396, 16389, 16383, 16379, 16300, 16285] avg=16384.0000 max=16536 max_rank=0 min=16285 min_rank=7 imbalance_ratio=1.009277 max_load_reduction=114536 max_load_reduction_pct=0.873840 moved_to_echo=65347 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[6, 0, 0, 4095, 0, 0, 1, 1, 0, 0, 22, 0, 0, 0, 4096, 0, 4096, 3914, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 4091, 0, 0, 0, 4093, 0, 0, 0, 0, 1, 22, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 59, 0, 13, 0, 0, 4096, 4054, 4056, 0, 1, 0, 0, 38, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4095, 4096, 0, 3919, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 4096, 0, 0, 0, 0, 3933, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 1, 0, 5, 0, 0, 1, 0, 0, 0, 0, 4096, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 1, 0, 4022, 0, 0, 1, 0, 5, 0, 4095, 0, 0, 4096, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4032, 3, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 141, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16287, 16403, 16413, 16389, 16383, 16384, 16361, 16452] avg=16384.0000 max=16452 max_rank=7 min=16287 min_rank=0 imbalance_ratio=1.004150 max_load_reduction=114620 max_load_reduction_pct=0.874481 moved_to_echo=65580 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[27, 1, 0, 4082, 0, 1, 25, 0, 1, 0, 10, 1, 1, 0, 4095, 0, 4095, 3914, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 1, 0, 0, 4095, 4095, 0, 0, 0, 4092, 1, 2, 0, 0, 0, 29, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 171, 17, 6, 0, 1, 4082, 4089, 4077, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 0, 1, 0, 4095, 0, 1, 0, 0, 1, 1, 4056, 4095, 1, 3922, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 4095, 1, 4087, 4094, 1, 1, 1, 0, 3930, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1, 4095, 0, 1, 41, 1, 1, 0, 1, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 207, 0, 0, 4041, 0, 0, 0, 1, 1, 0, 4095, 1, 1, 4095, 4039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3830, 14, 1, 4091, 0, 0, 0, 0, 0, 1, 0, 132, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16309, 16332, 16535, 16358, 16380, 16415, 16484, 16259] avg=16384.0000 max=16535 max_rank=2 min=16259 min_rank=7 imbalance_ratio=1.009216 max_load_reduction=114537 max_load_reduction_pct=0.873848 moved_to_echo=65538 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[19, 0, 0, 4096, 0, 0, 16, 0, 0, 0, 291, 0, 0, 0, 4096, 0, 4096, 3915, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 4084, 0, 0, 0, 4094, 0, 1, 0, 0, 1, 22, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 17, 0, 13, 0, 0, 4096, 4005, 4077, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 4091, 4096, 0, 3919, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 3933, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 461, 0, 0, 3506, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4006, 2, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 141, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16585, 16398, 16309, 16384, 16385, 16379, 16201, 16431] avg=16384.0000 max=16585 max_rank=0 min=16201 min_rank=6 imbalance_ratio=1.012268 max_load_reduction=114487 max_load_reduction_pct=0.873466 moved_to_echo=65396 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[5, 0, 0, 4095, 0, 0, 4, 0, 0, 0, 23, 0, 0, 0, 4096, 0, 4096, 3914, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4093, 4088, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 22, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 56, 3, 12, 0, 0, 4096, 4029, 4071, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3920, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3933, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4087, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 4096, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4071, 30, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 141, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16288, 16397, 16369, 16390, 16385, 16378, 16339, 16526] avg=16384.0000 max=16526 max_rank=7 min=16288 min_rank=0 imbalance_ratio=1.008667 max_load_reduction=114546 max_load_reduction_pct=0.873917 moved_to_echo=65628 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[32, 0, 0, 4096, 0, 0, 20, 0, 0, 0, 27, 0, 0, 0, 4096, 0, 4096, 3915, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 4087, 4053, 0, 0, 0, 4077, 0, 2, 0, 0, 4, 22, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 18, 0, 13, 0, 0, 4096, 4026, 4055, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 4090, 4096, 0, 3919, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 2, 0, 4096, 0, 4094, 4096, 0, 0, 0, 0, 3933, 163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 4043, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 4096, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4012, 42, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 141, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16338, 16348, 16335, 16384, 16390, 16381, 16452, 16444] avg=16384.0000 max=16452 max_rank=6 min=16335 min_rank=2 imbalance_ratio=1.004150 max_load_reduction=114620 max_load_reduction_pct=0.874481 moved_to_echo=65667 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[98, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 4096, 3915, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 22, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 12, 0, 0, 4096, 4094, 4064, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3920, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 4096, 0, 0, 0, 0, 3933, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 4051, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4029, 14, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 141, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16363, 16395, 16382, 16389, 16382, 16378, 16319, 16464] avg=16384.0000 max=16464 max_rank=7 min=16319 min_rank=6 imbalance_ratio=1.004883 max_load_reduction=114608 max_load_reduction_pct=0.874390 moved_to_echo=65543 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[63, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 4096, 0, 4096, 3915, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 4096, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 22, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 41, 0, 12, 0, 0, 4096, 4002, 4082, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 3920, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 3932, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 548, 0, 1, 3835, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3779, 10, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 141, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16366, 16395, 16333, 16389, 16384, 16378, 16616, 16211] avg=16384.0000 max=16616 max_rank=6 min=16211 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=114456 max_load_reduction_pct=0.873230 moved_to_echo=65589 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 32768, 128, 128, 128, 128, 128, 128, 128, 0, 0, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 128, 128, 0, 32768, 32768, 0, 0, 0, 32768, 128, 128, 0, 0, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 0, 128, 0, 128, 128, 128, 32768, 32768, 128, 31360, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 0, 128, 128, 128, 0, 32768, 128, 32768, 32768, 128, 128, 128, 0, 31488, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 0, 31744, 128, 128, 32768, 0, 0, 0, 0, 0, 128, 0, 1152, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[384, 128, 0, 32768, 0, 128, 128, 128, 128, 0, 768, 128, 128, 0, 32768, 0, 32768, 31360, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 384, 128, 128, 0, 128, 32768, 32512, 32640, 0, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 1792, 128, 128, 31616, 0, 0, 128, 128, 128, 0, 32768, 128, 128, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3574, 4096, 0, 0, 0, 0, 0, 4096, 0, 532, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 224, 0, 4096, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 3647, 0, 0, 3, 0, 4096, 0, 3864, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4096, 0, 4096, 1255, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 13, 4096, 4096, 0, 0, 0, 3990, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 959, 0, 2868, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 3667, 4096, 0, 4096, 2, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2603, 4096, 0, 0, 0, 0, 0, 4096, 0, 608, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 234, 0, 4096, 0, 0, 0, 0, 4081, 0, 0, 4, 0, 0, 3756, 0, 0, 0, 0, 4096, 0, 3630, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 4096, 0, 4096, 254, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 0, 8, 4096, 29, 4096, 4096, 1, 0, 0, 4065, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 2970, 0, 3342, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 3089, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3747, 4095, 1, 0, 1, 0, 0, 4095, 6, 1824, 2, 1, 4095, 4094, 1, 0, 1, 0, 0, 4095, 1, 1, 35, 1, 4095, 0, 0, 4, 1, 3714, 1, 1, 0, 1, 1, 1615, 1, 0, 15, 0, 4095, 0, 3619, 1, 0, 0, 1, 0, 0, 0, 0, 7, 4095, 4095, 0, 4095, 379, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 2, 1, 0, 4080, 1, 0, 136, 4013, 0, 4090, 4060, 0, 1, 0, 4032, 0, 1, 1, 1, 4095, 0, 0, 4094, 1, 1, 1, 4095, 18, 2774, 0, 2998, 0, 1, 0, 1, 1, 0, 1, 4094, 0, 4076, 1, 0, 0, 4095, 1, 0, 1, 4095, 3895, 4095, 0, 4095, 1, 0, 1, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3126, 4096, 0, 0, 0, 0, 0, 4096, 0, 662, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4073, 0, 0, 443, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 14, 0, 0, 3665, 0, 0, 0, 1, 4096, 4, 3376, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4094, 4095, 0, 4094, 1064, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4087, 0, 0, 3, 4096, 34, 4094, 4094, 6, 0, 0, 4046, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 3319, 0, 3039, 0, 0, 0, 0, 1, 0, 4, 4095, 0, 4096, 0, 0, 2, 4095, 0, 0, 0, 4096, 1773, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 3401, 4095, 0, 0, 0, 0, 1, 4096, 0, 573, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 4081, 0, 0, 323, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 7, 0, 0, 3569, 0, 0, 0, 0, 4096, 3, 3291, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4095, 4092, 0, 4095, 405, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4089, 0, 0, 0, 4096, 15, 4095, 4096, 4, 0, 0, 4042, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 4096, 0, 3416, 1, 3019, 1, 0, 0, 0, 1, 0, 12, 4096, 0, 4096, 0, 1, 1, 4096, 0, 0, 0, 4096, 2513, 4095, 0, 4095, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3869, 4096, 0, 0, 0, 0, 0, 4096, 0, 838, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4082, 0, 0, 128, 0, 4096, 0, 0, 0, 0, 4081, 0, 0, 2, 0, 0, 3317, 0, 0, 0, 0, 4096, 1, 2885, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4095, 4088, 0, 4095, 757, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4090, 0, 0, 5, 4096, 12, 4095, 4096, 3, 0, 0, 4085, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 3785, 0, 2710, 0, 0, 0, 0, 1, 0, 9, 4095, 0, 4096, 0, 0, 1, 4096, 0, 0, 0, 4096, 2196, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3924, 4096, 0, 0, 0, 0, 0, 4096, 0, 795, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 276, 0, 4096, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 3557, 0, 0, 1, 0, 4096, 0, 2588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 1051, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 4096, 25, 4096, 4096, 0, 0, 0, 4059, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 2887, 0, 3210, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 2229, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3743, 4096, 0, 0, 0, 0, 0, 4096, 0, 536, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 63, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 3, 0, 0, 3140, 0, 0, 0, 0, 4096, 1, 3803, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 4096, 0, 4096, 565, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 24, 4096, 14, 4096, 4096, 1, 0, 0, 3952, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 3378, 0, 2166, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 3185, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2603, 4096, 0, 0, 0, 0, 0, 4096, 0, 608, 0, 0, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4092, 0, 0, 234, 0, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 4, 0, 0, 3756, 0, 0, 0, 0, 4096, 0, 3630, 0, 1411, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 4096, 0, 4096, 254, 0, 0, 0, 2818, 351, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 4088, 0, 0, 8, 4096, 29, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4065, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 41, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2334, 0, 2970, 0, 3342, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3089, 4096, 0, 1278, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15299, 16406, 17145, 15921, 16390, 16418, 16838, 16655] avg=16384.0000 max=17145 max_rank=2 min=15299 min_rank=0 imbalance_ratio=1.046448 max_load_reduction=113927 max_load_reduction_pct=0.869194 moved_to_echo=66301 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3574, 4096, 0, 0, 0, 0, 0, 4096, 0, 532, 0, 0, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 224, 0, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 3647, 0, 0, 3, 0, 4096, 0, 3864, 0, 1411, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 4096, 0, 4096, 1255, 0, 0, 0, 2818, 351, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 4096, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3990, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 41, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2334, 0, 959, 0, 2868, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3667, 4096, 0, 1278, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16194, 16399, 17253, 16920, 16376, 16342, 14353, 17235] avg=16384.0000 max=17253 max_rank=2 min=14353 min_rank=6 imbalance_ratio=1.053040 max_load_reduction=113819 max_load_reduction_pct=0.868370 moved_to_echo=64306 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 3401, 4095, 0, 0, 0, 0, 1, 4096, 0, 573, 0, 0, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 4081, 0, 0, 323, 0, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 7, 0, 0, 3569, 0, 0, 0, 0, 4096, 3, 3291, 0, 1411, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4095, 4092, 0, 4095, 405, 1, 0, 0, 2818, 351, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 1, 0, 0, 4089, 0, 0, 0, 4096, 15, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 4042, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 41, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2334, 0, 3416, 1, 3019, 1, 0, 0, 0, 1, 0, 12, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4096, 0, 0, 0, 0, 2513, 4095, 0, 1277, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16064, 16483, 16632, 16072, 16369, 16397, 16976, 16079] avg=16384.0000 max=16976 max_rank=6 min=16064 min_rank=0 imbalance_ratio=1.036133 max_load_reduction=114096 max_load_reduction_pct=0.870483 moved_to_echo=65821 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 3747, 4095, 1, 0, 1, 0, 0, 4095, 6, 1824, 2, 1, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4094, 1, 0, 1, 0, 0, 4095, 1, 1, 35, 1, 3880, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3714, 1, 1, 0, 1, 1, 1615, 1, 0, 15, 0, 4095, 0, 3619, 1, 1416, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 4095, 4095, 0, 4095, 379, 0, 0, 0, 2823, 352, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4071, 0, 0, 2, 1, 0, 4080, 1, 0, 136, 4013, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 1, 0, 4032, 0, 1, 1, 1, 4095, 0, 0, 4094, 1, 1, 1, 45, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2327, 18, 2774, 0, 2998, 0, 1, 0, 1, 1, 0, 1, 4094, 0, 4076, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 3895, 4095, 0, 1272, 1, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17669, 16208, 14651, 16046, 16395, 16357, 16292, 17454] avg=16384.0000 max=17669 max_rank=0 min=14651 min_rank=2 imbalance_ratio=1.078430 max_load_reduction=113403 max_load_reduction_pct=0.865196 moved_to_echo=66498 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3743, 4096, 0, 0, 0, 0, 0, 4096, 0, 536, 0, 0, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 63, 0, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 3, 0, 0, 3140, 0, 0, 0, 0, 4096, 1, 3803, 0, 1411, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 4096, 0, 4096, 565, 0, 0, 0, 2818, 351, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 4093, 0, 0, 24, 4096, 14, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 3952, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 41, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2334, 0, 3378, 0, 2166, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3185, 4096, 0, 1278, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16367, 16238, 16712, 16232, 16396, 16305, 16071, 16751] avg=16384.0000 max=16751 max_rank=7 min=16071 min_rank=6 imbalance_ratio=1.022400 max_load_reduction=114321 max_load_reduction_pct=0.872200 moved_to_echo=65523 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3126, 4096, 0, 0, 0, 0, 0, 4096, 0, 662, 0, 0, 3897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4073, 0, 0, 443, 0, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 14, 0, 0, 3665, 0, 0, 0, 1, 4096, 4, 3376, 0, 1411, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4094, 4095, 0, 4094, 1064, 0, 0, 0, 2818, 351, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 1, 0, 0, 4087, 0, 0, 3, 4096, 34, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 6, 0, 0, 4046, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 41, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2334, 0, 3319, 0, 3039, 0, 0, 0, 0, 1, 0, 4, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 1773, 4096, 0, 1278, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15877, 16596, 16829, 16752, 16388, 16402, 16888, 15340] avg=16384.0000 max=16888 max_rank=6 min=15340 min_rank=7 imbalance_ratio=1.030762 max_load_reduction=114184 max_load_reduction_pct=0.871155 moved_to_echo=65018 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3869, 4096, 0, 0, 0, 0, 0, 4096, 0, 838, 0, 0, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4082, 0, 0, 128, 0, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 2, 0, 0, 3317, 0, 0, 0, 0, 4096, 1, 2885, 0, 1411, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4095, 4088, 0, 4095, 757, 0, 0, 0, 2818, 351, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 1, 0, 0, 4090, 0, 0, 5, 4096, 12, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 4085, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 41, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2334, 0, 3785, 0, 2710, 0, 0, 0, 0, 1, 0, 9, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 2196, 4096, 0, 1278, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16795, 16290, 15960, 16422, 16372, 16440, 17030, 15763] avg=16384.0000 max=17030 max_rank=6 min=15763 min_rank=7 imbalance_ratio=1.039429 max_load_reduction=114042 max_load_reduction_pct=0.870071 moved_to_echo=65605 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3924, 4096, 0, 0, 0, 0, 0, 4096, 0, 795, 0, 0, 3896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 276, 0, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 3557, 0, 0, 1, 0, 4096, 0, 2588, 0, 1411, 167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4096, 1051, 0, 0, 0, 2818, 351, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 4096, 25, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4059, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 41, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2334, 0, 2887, 0, 3210, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2229, 4096, 0, 1278, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16807, 16452, 15890, 16707, 16386, 16411, 16624, 15795] avg=16384.0000 max=16807 max_rank=0 min=15795 min_rank=7 imbalance_ratio=1.025818 max_load_reduction=114265 max_load_reduction_pct=0.871773 moved_to_echo=65216 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[18688, 128, 23552, 128, 23424, 128, 128, 0, 128, 128, 0, 128, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 0, 28032, 32768, 128, 0, 128, 0, 128, 32768, 128, 6400, 128, 128, 31232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 32768, 128, 0, 128, 0, 0, 32768, 128, 128, 1792, 128, 31104, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 0, 32384, 0, 128, 128, 128, 32768, 0, 0, 32768, 128, 128, 128, 384, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 128, 0, 128, 0, 22656, 32768, 0, 10240, 128, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 0, 128, 32640, 0, 0, 128, 128, 0, 32768, 128, 0, 256, 32768, 256, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[0, 0, 128, 0, 0, 0, 0, 128, 32768, 32768, 0, 32768, 5760, 128, 0, 0, 22656, 2816, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 32384, 128, 128, 128, 128, 128, 26368, 128, 0, 128, 128, 32768, 128, 27136, 128, 11392, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[4046, 0, 0, 4096, 0, 0, 4092, 3, 0, 0, 0, 8, 0, 0, 0, 4096, 0, 4096, 0, 208, 0, 4096, 4075, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2011, 4093, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 78, 0, 0, 3, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4092, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4094, 1999, 3, 0, 3964, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4091, 0, 0, 4094, 0, 1, 4095, 0, 0, 0, 1, 0, 0, 0, 1, 4092, 30, 3151, 2, 743, 1, 4094, 4038, 0, 1, 1, 4094, 0, 288, 0, 0, 0, 4094, 0, 1, 4095, 4095, 4095, 0, 1, 0, 1, 0, 1, 107, 0, 0, 1, 1, 0, 1, 0, 2, 1, 1, 1, 4095, 1, 0, 4006, 0, 0, 1, 4095, 1, 0, 0, 0, 3, 1028, 4095, 8, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 4095, 4095, 1, 60, 0, 0, 0, 0, 0, 1, 4087, 4095, 0, 0, 4094, 1, 1, 3490, 1, 3691, 0, 1, 1, 4095, 0, 0, 0, 4093, 1, 4095, 50, 4095, 4095, 4095, 0, 0, 1, 1, 4095, 3891, 0, 1, 4095, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[3634, 0, 0, 4095, 0, 0, 4089, 2, 0, 0, 0, 12, 0, 0, 0, 4096, 1, 4095, 0, 161, 0, 4095, 4082, 0, 0, 0, 4094, 0, 0, 4, 0, 0, 4096, 0, 0, 4088, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1327, 4091, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 17, 0, 0, 10, 0, 2, 0, 4096, 4096, 0, 1, 4096, 0, 0, 4096, 0, 4010, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4085, 0, 4095, 4095, 4096, 0, 0, 0, 0, 4094, 3241, 4, 0, 4004, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[3952, 0, 0, 4096, 0, 0, 4090, 13, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4096, 0, 23, 0, 4096, 4074, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 4077, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1731, 4087, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 25, 0, 0, 2, 0, 3, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4007, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4087, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 2692, 6, 0, 3981, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[3772, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 0, 4096, 0, 17, 0, 4096, 4080, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 912, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 22, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 3993, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4085, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 3635, 0, 0, 4064, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[3636, 0, 0, 4095, 0, 0, 4079, 0, 0, 0, 0, 48, 0, 0, 0, 4096, 1, 4096, 0, 96, 0, 4096, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 4073, 4096, 4096, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 3, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1159, 4092, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 54, 0, 0, 5, 0, 2, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4094, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4082, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4093, 3324, 2, 0, 4003, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3842, 0, 0, 4096, 0, 0, 4069, 6, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 4090, 0, 36, 0, 4096, 4029, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 0, 0, 4047, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 5, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1144, 4070, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 57, 0, 0, 29, 0, 6, 0, 4096, 4096, 0, 0, 4096, 0, 0, 4096, 0, 4094, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 4073, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4092, 3316, 25, 0, 3982, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[3602, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 4095, 0, 29, 0, 4096, 4087, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4096, 0, 0, 4095, 4096, 4096, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1134, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4094, 0, 0, 4095, 0, 4065, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4094, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4096, 3641, 0, 0, 3920, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4046, 0, 0, 4096, 0, 0, 4092, 3, 0, 0, 0, 8, 0, 0, 0, 4096, 162, 58, 45, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 208, 0, 4038, 4075, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4089, 4096, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2961, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2011, 4093, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 4096, 2789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4096, 0, 78, 0, 0, 3, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4093, 0, 0, 0, 166, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 973, 0, 0, 0, 0, 0, 4094, 1999, 3, 0, 3964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16607, 16513, 16377, 16394, 17089, 16420, 16547, 15125] avg=16384.0000 max=17089 max_rank=4 min=15125 min_rank=7 imbalance_ratio=1.043030 max_load_reduction=113983 max_load_reduction_pct=0.869621 moved_to_echo=65181 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3634, 0, 0, 4095, 0, 0, 4089, 2, 0, 0, 0, 12, 0, 0, 0, 4096, 162, 58, 45, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 161, 0, 4037, 4082, 0, 0, 0, 4094, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 4096, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 4, 4096, 0, 0, 0, 4096, 2960, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1327, 4091, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4095, 2789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4095, 0, 17, 0, 0, 10, 0, 2, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 4096, 0, 4010, 1, 0, 0, 166, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 973, 0, 0, 0, 0, 0, 4094, 3241, 4, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16194, 16474, 16376, 16395, 16399, 16367, 16466, 16401] avg=16384.0000 max=16474 max_rank=1 min=16194 min_rank=0 imbalance_ratio=1.005493 max_load_reduction=114598 max_load_reduction_pct=0.874313 moved_to_echo=65633 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4091, 0, 0, 4094, 0, 1, 4095, 0, 0, 0, 1, 0, 0, 0, 1, 4092, 168, 61, 50, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 3151, 2, 743, 1, 4033, 4038, 0, 1, 1, 4094, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 4095, 4095, 4093, 0, 1, 0, 1, 0, 1, 107, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 2, 1, 1, 1, 4095, 1, 0, 4006, 0, 0, 1, 4095, 2961, 1143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 1028, 4095, 8, 1, 0, 0, 1, 0, 0, 0, 1, 4095, 4095, 2790, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4045, 4095, 1, 60, 0, 0, 0, 0, 0, 1, 4087, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 1, 3490, 1, 3691, 0, 1, 1, 162, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 50, 0, 966, 0, 0, 0, 1, 1, 4095, 3891, 0, 1, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16656, 16382, 16489, 16309, 16118, 16386, 15536, 17196] avg=16384.0000 max=17196 max_rank=7 min=15536 min_rank=6 imbalance_ratio=1.049561 max_load_reduction=113876 max_load_reduction_pct=0.868805 moved_to_echo=65236 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3772, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 7, 0, 0, 0, 4096, 162, 58, 45, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 17, 0, 4038, 4080, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2961, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 912, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4096, 0, 22, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3993, 0, 0, 0, 166, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 973, 0, 0, 0, 0, 0, 4096, 3635, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16333, 16327, 16372, 16392, 15987, 16361, 16447, 16853] avg=16384.0000 max=16853 max_rank=7 min=15987 min_rank=4 imbalance_ratio=1.028625 max_load_reduction=114219 max_load_reduction_pct=0.871422 moved_to_echo=65648 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3952, 0, 0, 4096, 0, 0, 4090, 13, 0, 0, 0, 1, 0, 0, 0, 4096, 162, 58, 45, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 4038, 4074, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4077, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2961, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1731, 4087, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 4096, 2789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4096, 0, 25, 0, 0, 2, 0, 3, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4007, 0, 0, 0, 166, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 973, 0, 0, 0, 0, 0, 4096, 2692, 6, 0, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16514, 16328, 16364, 16396, 16805, 16369, 16461, 15835] avg=16384.0000 max=16805 max_rank=4 min=15835 min_rank=7 imbalance_ratio=1.025696 max_load_reduction=114267 max_load_reduction_pct=0.871788 moved_to_echo=65470 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3636, 0, 0, 4095, 0, 0, 4079, 0, 0, 0, 0, 48, 0, 0, 0, 4096, 162, 58, 45, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 96, 0, 4038, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4073, 4096, 4095, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 3, 4092, 0, 0, 0, 4096, 2961, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1159, 4092, 0, 0, 0, 0, 0, 10, 0, 0, 0, 4096, 4096, 2789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4094, 0, 54, 0, 0, 5, 0, 2, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4094, 1, 0, 0, 166, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 973, 0, 0, 0, 0, 0, 4093, 3324, 2, 0, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16220, 16419, 16375, 16392, 16242, 16398, 16549, 16477] avg=16384.0000 max=16549 max_rank=6 min=16220 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=114523 max_load_reduction_pct=0.873741 moved_to_echo=65666 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3842, 0, 0, 4096, 0, 0, 4069, 6, 0, 0, 0, 16, 0, 0, 0, 4096, 162, 58, 45, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 36, 0, 4038, 4029, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4047, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 5, 4096, 0, 0, 0, 4096, 2961, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1144, 4070, 0, 0, 0, 0, 0, 26, 0, 0, 0, 4096, 4096, 2789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4093, 0, 57, 0, 0, 29, 0, 6, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4094, 2, 0, 0, 166, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 0, 0, 973, 0, 0, 0, 0, 0, 4092, 3316, 25, 0, 3982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16391, 16283, 16334, 16404, 16221, 16428, 16550, 16461] avg=16384.0000 max=16550 max_rank=6 min=16221 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=114522 max_load_reduction_pct=0.873734 moved_to_echo=65660 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3602, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 162, 58, 45, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 29, 0, 4038, 4087, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 4096, 4095, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2961, 1141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1134, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 4095, 0, 4065, 0, 0, 0, 166, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 973, 0, 0, 0, 0, 0, 4096, 3641, 0, 0, 3920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16157, 16346, 16385, 16390, 16211, 16343, 16516, 16724] avg=16384.0000 max=16724 max_rank=7 min=16157 min_rank=0 imbalance_ratio=1.020752 max_load_reduction=114348 max_load_reduction_pct=0.872406 moved_to_echo=65794 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 128, 32256, 128, 32128, 128, 128, 128, 1408, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[30592, 0, 0, 32768, 0, 128, 32768, 128, 0, 0, 128, 128, 0, 0, 128, 32768, 1408, 512, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 32512, 32768, 128, 384, 0, 0, 128, 0, 128, 128, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 0, 128, 32768, 32768, 32768, 0, 128, 0, 128, 0, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 0, 128, 10496, 32768, 128, 128, 0, 0, 128, 128, 0, 0, 128, 32768, 32768, 22400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 0, 7808, 0, 0, 0, 128, 128, 32768, 25856, 128, 128, 32128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 31872, 128, 1408, 128, 32384, 32640, 0, 128, 128, 32768, 0, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 0, 128, 128, 128, 128, 32768, 128, 128, 32768, 0, 0, 128, 32768, 23808, 9216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 562, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4094, 4082, 0, 0, 0, 4096, 0, 4096, 19, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 5, 0, 4095, 0, 0, 0, 0, 3998, 57, 3817, 4096, 0, 40, 0, 4096, 1, 0, 0, 0, 4095, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 2516, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 4095, 0, 0, 0, 1, 4095, 4095, 4096, 41, 749, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 4096, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 502, 4095, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 834, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4095, 0, 0, 0, 4096, 0, 4096, 11, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 3905, 42, 3945, 4096, 0, 12, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 849, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 1, 4096, 4086, 4096, 6, 1279, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1414, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 490, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4094, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 3702, 17, 3873, 4096, 0, 40, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 942, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 0, 0, 4096, 4095, 4096, 7, 1752, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1453, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 0, 10, 1094, 1, 1, 0, 1, 0, 0, 10, 4050, 1, 1, 0, 4093, 0, 0, 4095, 4095, 4095, 0, 0, 1, 4093, 1, 3629, 18, 0, 2, 4095, 1, 55, 1, 1, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 2593, 0, 3853, 4095, 1, 17, 1, 4095, 0, 1, 0, 0, 4095, 1, 1, 4094, 2, 0, 4059, 0, 7, 1, 1, 1, 0, 1, 0, 4092, 0, 61, 191, 0, 0, 0, 1, 1, 0, 4093, 1, 4095, 0, 4075, 1, 18, 1, 0, 4095, 4091, 4095, 1, 3724, 0, 0, 2, 0, 0, 4095, 0, 1, 0, 0, 4094, 1, 4093, 1, 1, 1, 4095, 1, 4089, 0, 0, 0, 0, 0, 3, 4095, 1220, 4094, 0, 1, 0, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4093, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4026, 53, 3799, 4096, 0, 69, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1367, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 0, 0, 4096, 4093, 4096, 7, 745, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 2003, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 47, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 9, 0, 4091, 0, 0, 0, 0, 3925, 92, 3874, 4096, 0, 5, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 2474, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4087, 4096, 17, 543, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 1259, 4094, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4091, 0, 0, 0, 4096, 0, 4096, 35, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 4069, 4, 3994, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 16, 4096, 0, 0, 1858, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 4095, 0, 0, 0, 0, 4096, 4079, 4096, 61, 1770, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 315, 4096, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 260, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4094, 0, 0, 0, 4096, 0, 4096, 32, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 3659, 35, 3617, 4096, 0, 4, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 1725, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 12, 985, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1950, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 834, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 41, 0, 4096, 11, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 3905, 42, 3945, 2694, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 12, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 849, 0, 0, 0, 0, 4096, 4096, 2133, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2641, 0, 4096, 0, 4096, 0, 0, 0, 1, 0, 4086, 0, 6, 1279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 1455, 1361, 1262, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2834, 1414, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16758, 16435, 16650, 16377, 15730, 16205, 16381, 16536] avg=16384.0000 max=16758 max_rank=0 min=15730 min_rank=4 imbalance_ratio=1.022827 max_load_reduction=114314 max_load_reduction_pct=0.872147 moved_to_echo=64852 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 562, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4082, 0, 0, 0, 41, 0, 4096, 19, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4095, 0, 0, 0, 0, 3998, 57, 3817, 2694, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 40, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4096, 0, 0, 2516, 0, 0, 0, 0, 4096, 4095, 2133, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2641, 0, 4094, 0, 4095, 0, 0, 0, 1, 0, 4095, 0, 41, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4091, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 1455, 1361, 1262, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2834, 502, 4095, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16486, 16426, 16629, 16407, 17404, 15716, 16379, 15625] avg=16384.0000 max=17404 max_rank=4 min=15625 min_rank=7 imbalance_ratio=1.062256 max_load_reduction=113668 max_load_reduction_pct=0.867218 moved_to_echo=65124 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 490, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 41, 0, 4094, 14, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 3702, 17, 3873, 2694, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 40, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 942, 0, 0, 0, 0, 4096, 4096, 2133, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2641, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 7, 1752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1455, 1361, 1262, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2834, 1453, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16414, 16437, 16347, 16406, 15823, 16686, 16384, 16575] avg=16384.0000 max=16686 max_rank=5 min=15823 min_rank=4 imbalance_ratio=1.018433 max_load_reduction=114386 max_load_reduction_pct=0.872696 moved_to_echo=65468 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 213, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 41, 0, 4093, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 0, 4026, 53, 3799, 2694, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 69, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1367, 0, 0, 0, 0, 4096, 4096, 2133, 461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2641, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 4093, 0, 7, 745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1455, 1361, 1262, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2834, 2003, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16136, 16430, 16636, 16434, 16249, 15677, 16385, 17125] avg=16384.0000 max=17125 max_rank=7 min=15677 min_rank=5 imbalance_ratio=1.045227 max_load_reduction=113947 max_load_reduction_pct=0.869347 moved_to_echo=65436 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 10, 1094, 1, 1, 0, 1, 0, 0, 10, 4050, 1, 1, 0, 4095, 4095, 3636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 1, 27, 1, 3629, 18, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 55, 1, 1, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 2593, 0, 3853, 2700, 1962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 1, 17, 1, 0, 0, 1, 0, 0, 4095, 1, 1, 4094, 2, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1, 1, 1, 0, 1, 0, 4092, 0, 61, 191, 0, 0, 0, 1, 4095, 4095, 2131, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2633, 1, 4095, 0, 4075, 1, 18, 1, 0, 0, 4091, 0, 1, 3724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4095, 0, 1, 0, 0, 4094, 1, 4093, 1, 1, 1, 1460, 1366, 1262, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 3, 2833, 1220, 4094, 0, 1, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16996, 15963, 15264, 16342, 15136, 18641, 16402, 16328] avg=16384.0000 max=18641 max_rank=5 min=15136 min_rank=4 imbalance_ratio=1.137756 max_load_reduction=112431 max_load_reduction_pct=0.857780 moved_to_echo=66507 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4091, 0, 0, 0, 41, 0, 4096, 35, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 0, 0, 4069, 4, 3994, 2694, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 16, 4096, 0, 0, 1858, 0, 0, 0, 0, 4096, 4096, 2133, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2641, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 4079, 0, 61, 1770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 1455, 1361, 1262, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 2834, 315, 4096, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16109, 16455, 16824, 16367, 16757, 16741, 16382, 15437] avg=16384.0000 max=16824 max_rank=2 min=15437 min_rank=7 imbalance_ratio=1.026855 max_load_reduction=114248 max_load_reduction_pct=0.871643 moved_to_echo=65317 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 41, 0, 4096, 47, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4091, 0, 0, 0, 0, 3925, 92, 3874, 2694, 1964, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 5, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 2474, 0, 0, 0, 0, 4096, 4096, 2132, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2641, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4087, 0, 17, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 1455, 1361, 1262, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 2834, 1259, 4094, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15989, 16472, 16649, 16370, 17363, 15480, 16375, 16374] avg=16384.0000 max=17363 max_rank=4 min=15480 min_rank=5 imbalance_ratio=1.059753 max_load_reduction=113709 max_load_reduction_pct=0.867531 moved_to_echo=65592 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 260, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 41, 0, 4096, 32, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 4096, 0, 0, 0, 0, 3659, 35, 3617, 2694, 1963, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 1725, 0, 0, 0, 0, 4096, 4096, 2133, 460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2641, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 12, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1455, 1361, 1262, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2834, 1950, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16184, 16454, 16073, 16369, 16610, 15926, 16384, 17072] avg=16384.0000 max=17072 max_rank=7 min=15926 min_rank=5 imbalance_ratio=1.041992 max_load_reduction=114000 max_load_reduction_pct=0.869751 moved_to_echo=65992 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=10 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32640, 128, 256, 128, 0, 128, 128, 0, 0, 32768, 128, 128, 32768, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 0, 32768, 0, 128, 0, 0, 32768, 128, 32768, 128, 128, 128, 11648, 11008, 10112, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 3712, 128, 128, 0, 128, 0, 0, 128, 32768, 128, 128, 0, 32768, 32768, 29184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 0, 0, 32768, 32768, 0, 0, 128, 384, 128, 32384, 256, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 32768, 0, 128, 0, 0, 0, 128, 22784, 10240, 32768, 0, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 128, 128, 0, 128, 128, 32768, 0, 128, 12032, 0, 0, 0, 128, 32768, 32768, 17152, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 21120, 128, 32768, 0, 32768, 128, 128, 128, 128, 0, 32768, 0, 256, 11648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 0, 0, 128, 0, 32768, 0, 128, 128, 128, 29952, 384, 30848, 21632, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4090, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 4071, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4008, 0, 0, 0, 28, 72, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 15, 0, 0, 9, 0, 4062, 4096, 0, 0, 28, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 4056, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3723, 0, 0, 0, 102, 282, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 22, 0, 0, 7, 0, 4092, 4096, 0, 0, 9, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 4086, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3990, 0, 0, 0, 43, 64, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 3, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 9, 0, 0, 4, 0, 4081, 4096, 0, 0, 16, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4084, 0, 4096, 0, 0, 0, 0, 4096, 2, 0, 0, 4095, 0, 4034, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3824, 0, 0, 0, 12, 315, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 4, 0, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 4095, 4095, 1, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 21, 0, 0, 7, 0, 4085, 4096, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4086, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4076, 0, 4075, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3950, 0, 0, 0, 28, 110, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4092, 0, 0, 7, 0, 0, 0, 4096, 4096, 8, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 55, 0, 0, 18, 0, 4057, 4096, 0, 0, 34, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4091, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 4076, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3230, 0, 0, 0, 46, 833, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 2, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 5, 0, 0, 19, 0, 4093, 4096, 0, 0, 3, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4095, 1, 0, 4018, 166, 32, 0, 4093, 4095, 4067, 1, 4095, 4090, 1, 4095, 1, 0, 0, 0, 4088, 0, 0, 4, 4080, 0, 3980, 1, 10, 4093, 1, 0, 0, 4095, 1, 1, 0, 0, 2704, 1, 1, 1, 10, 1664, 0, 4046, 4003, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 12, 1, 1, 0, 4086, 1, 4095, 0, 1, 0, 0, 1, 0, 4090, 3858, 5, 1, 0, 0, 0, 2, 4094, 4095, 0, 1, 0, 0, 1, 4089, 0, 0, 1, 0, 1, 4095, 1, 4088, 0, 0, 1, 410, 0, 4095, 4048, 0, 34, 68, 0, 1, 2, 0, 3659, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 3, 106, 4095, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 4095, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 4071, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3931, 0, 0, 0, 13, 164, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 14, 0, 0, 3, 0, 4086, 4096, 0, 0, 11, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4085, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 51, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3990, 0, 0, 0, 43, 64, 0, 4037, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 18, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 4, 0, 4081, 4096, 0, 0, 16, 0, 0, 0, 0, 4092, 0, 4063, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 16422, 16337, 16402, 16410, 16386, 16369, 16373] avg=16384.0000 max=16422 max_rank=1 min=16337 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=114650 max_load_reduction_pct=0.874710 moved_to_echo=65538 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4085, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 51, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3723, 0, 0, 0, 102, 282, 0, 4037, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 18, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 4095, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 7, 0, 4092, 4096, 0, 0, 9, 0, 0, 0, 0, 4096, 0, 4063, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 16390, 16347, 16402, 16411, 16385, 16393, 16371] avg=16384.0000 max=16411 max_rank=4 min=16347 min_rank=2 imbalance_ratio=1.001648 max_load_reduction=114661 max_load_reduction_pct=0.874794 moved_to_echo=65560 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4085, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 51, 0, 0, 0, 0, 4096, 0, 0, 0, 4093, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4008, 0, 0, 0, 28, 72, 0, 4037, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 18, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 4094, 15, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 9, 0, 4062, 4096, 0, 0, 28, 0, 0, 0, 0, 4094, 0, 4063, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 16401, 16348, 16402, 16414, 16386, 16375, 16373] avg=16384.0000 max=16414 max_rank=4 min=16348 min_rank=2 imbalance_ratio=1.001831 max_load_reduction=114658 max_load_reduction_pct=0.874771 moved_to_echo=65548 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4085, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 51, 0, 0, 0, 0, 4096, 2, 0, 0, 4095, 0, 4034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3824, 0, 0, 0, 12, 315, 0, 4037, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 18, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4, 0, 0, 0, 4096, 4096, 3, 0, 0, 0, 0, 0, 4094, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 1, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 7, 0, 4085, 4096, 0, 0, 1, 0, 0, 0, 0, 4094, 0, 4063, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 16362, 16391, 16401, 16415, 16386, 16375, 16370] avg=16384.0000 max=16415 max_rank=4 min=16362 min_rank=1 imbalance_ratio=1.001892 max_load_reduction=114657 max_load_reduction_pct=0.874763 moved_to_echo=65546 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4095, 1, 0, 4018, 166, 32, 0, 4082, 0, 4067, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 44, 1, 0, 0, 0, 4088, 0, 0, 4, 4080, 0, 3980, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 4095, 1, 1, 0, 0, 2704, 1, 1, 1, 10, 1664, 0, 4041, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 4003, 1, 1, 1, 1, 4095, 0, 0, 0, 11, 12, 1, 1, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 0, 4090, 3858, 5, 1, 0, 0, 0, 2, 4095, 17, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 0, 1, 0, 0, 1, 4089, 0, 0, 1, 0, 1, 0, 1, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 410, 0, 4095, 4048, 0, 34, 68, 0, 1, 2, 0, 3659, 0, 4067, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 3, 106, 4095, 1, 0, 4095, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16465, 16299, 16624, 16259, 16178, 16371, 16395, 16481] avg=16384.0000 max=16624 max_rank=2 min=16178 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=114448 max_load_reduction_pct=0.873169 moved_to_echo=65425 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 4085, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 51, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3230, 0, 0, 0, 46, 833, 0, 4037, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 18, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 0, 4094, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 19, 0, 4093, 4096, 0, 0, 3, 0, 0, 0, 0, 4092, 0, 4063, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16372, 16406, 16349, 16402, 16412, 16386, 16379, 16366] avg=16384.0000 max=16412 max_rank=4 min=16349 min_rank=2 imbalance_ratio=1.001709 max_load_reduction=114660 max_load_reduction_pct=0.874786 moved_to_echo=65543 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4096, 0, 0, 0, 4085, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 51, 0, 0, 0, 0, 4096, 0, 0, 0, 4076, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3950, 0, 0, 0, 28, 110, 0, 4037, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 18, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 7, 0, 0, 0, 4096, 4096, 8, 0, 0, 0, 0, 0, 4096, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 18, 0, 4057, 4096, 0, 0, 34, 0, 0, 0, 0, 4078, 0, 4063, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16371, 16384, 16328, 16402, 16422, 16386, 16409, 16370] avg=16384.0000 max=16422 max_rank=4 min=16328 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=114650 max_load_reduction_pct=0.874710 moved_to_echo=65587 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4085, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 51, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 3931, 0, 0, 0, 13, 164, 0, 4037, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 18, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4094, 15, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 3, 0, 4086, 4096, 0, 0, 11, 0, 0, 0, 0, 4096, 0, 4063, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16373, 16408, 16348, 16402, 16410, 16386, 16377, 16368] avg=16384.0000 max=16410 max_rank=4 min=16348 min_rank=2 imbalance_ratio=1.001587 max_load_reduction=114662 max_load_reduction_pct=0.874802 moved_to_echo=65541 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 32768, 128, 128, 128, 128, 32768, 0, 0, 0, 256, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 0, 128, 512, 0, 32768, 32768, 0, 128, 256, 0, 128, 128, 0, 32384, 0, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 512, 128, 0, 0, 0, 32768, 128, 0, 128, 32768, 0, 32512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 32768, 128, 128, 0, 128, 128, 32768, 0, 0, 128, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 128, 32768, 128, 0, 32768, 256, 128, 0, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 0, 0, 32768, 128, 128, 0, 0, 29440, 128, 128, 128, 384, 3584, 0, 32384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 0, 0, 0, 128, 32768, 0, 128, 128, 128, 128, 128, 32768, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 128, 0, 32768, 32640, 128, 128, 0, 0, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 4093, 19, 0, 26, 0, 4067, 1, 0, 4095, 1, 3124, 1, 4093, 0, 0, 0, 4082, 0, 0, 1, 1, 4095, 4094, 1, 1, 4095, 4082, 4094, 0, 0, 1, 1, 1, 1, 0, 1, 1, 22, 7, 2, 0, 815, 1, 0, 3985, 0, 0, 1, 4094, 1, 4045, 26, 0, 0, 4095, 4095, 1, 1, 1, 0, 0, 1, 4036, 9, 1, 7, 4087, 4095, 1, 0, 0, 1, 1, 0, 0, 1, 4095, 1, 1, 1, 10, 328, 0, 0, 4085, 4088, 0, 4091, 0, 0, 0, 1, 0, 3996, 1, 4095, 0, 4072, 4095, 0, 0, 41, 4095, 1, 83, 0, 0, 0, 1, 1, 0, 1, 4095, 3, 0, 0, 1, 4095, 4093, 0, 2, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4026, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 49, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4081, 4096, 0, 4096, 0, 0, 14, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4089, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 10, 0, 0, 4096, 0, 10, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 0, 4096, 0, 0, 13, 0, 0, 4095, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4091, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 4096, 0, 0, 3, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 4091, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 4096, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 1, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4095, 0, 0, 16, 0, 0, 4096, 0, 4096, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 1, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 4096, 0, 0, 2, 0, 0, 4095, 0, 4096, 0, 4096, 4095, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4074, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 6, 0, 0, 4096, 0, 23, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 1, 4096, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 4080, 4095, 0, 4096, 0, 0, 12, 0, 0, 4095, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4095, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 0, 0, 8, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[134, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 4096, 4096, 4095, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 4093, 4096, 0, 4066, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3962, 104, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16513, 16389, 16271, 16397, 16391, 16343, 16385, 16383] avg=16384.0000 max=16513 max_rank=0 min=16271 min_rank=2 imbalance_ratio=1.007874 max_load_reduction=114559 max_load_reduction_pct=0.874016 moved_to_echo=65502 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[127, 0, 0, 1, 4093, 19, 0, 26, 0, 4067, 1, 0, 4095, 1, 3124, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4082, 0, 0, 1, 1, 4095, 0, 1, 1, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1, 22, 7, 2, 0, 815, 1, 0, 4095, 4094, 4095, 3975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 0, 0, 1, 4094, 1, 4045, 26, 0, 0, 6, 4095, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4036, 9, 1, 7, 4087, 4095, 1, 0, 0, 1, 1, 0, 0, 1, 3981, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 1, 10, 328, 0, 0, 4085, 4088, 0, 4057, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3996, 1, 0, 0, 4072, 4095, 0, 0, 41, 4095, 1, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 4095, 3, 0, 0, 1, 4095, 4093, 0, 2, 0, 0, 0, 3968, 108, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15555, 16356, 17112, 16256, 16340, 16667, 16384, 16402] avg=16384.0000 max=17112 max_rank=2 min=15555 min_rank=0 imbalance_ratio=1.044434 max_load_reduction=113960 max_load_reduction_pct=0.869446 moved_to_echo=65793 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[134, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4026, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 49, 0, 0, 4096, 4096, 4096, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4081, 4096, 0, 4066, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3962, 104, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16448, 16388, 16330, 16397, 16389, 16352, 16384, 16384] avg=16384.0000 max=16448 max_rank=0 min=16330 min_rank=2 imbalance_ratio=1.003906 max_load_reduction=114624 max_load_reduction_pct=0.874512 moved_to_echo=65509 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[134, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 4096, 4096, 4095, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 4096, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 1, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 3979, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 0, 4065, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3962, 104, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16514, 16388, 16267, 16398, 16389, 16352, 16384, 16380] avg=16384.0000 max=16514 max_rank=0 min=16267 min_rank=2 imbalance_ratio=1.007935 max_load_reduction=114558 max_load_reduction_pct=0.874008 moved_to_echo=65505 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[134, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 10, 0, 0, 4096, 4096, 4095, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 10, 0, 4096, 0, 4096, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 4082, 4096, 0, 4066, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3962, 104, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16511, 16389, 16271, 16407, 16387, 16343, 16384, 16380] avg=16384.0000 max=16511 max_rank=0 min=16271 min_rank=2 imbalance_ratio=1.007751 max_load_reduction=114561 max_load_reduction_pct=0.874031 moved_to_echo=65494 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[134, 0, 0, 0, 4096, 0, 0, 1, 0, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 4096, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 1, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 3979, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 0, 4066, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4095, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3962, 104, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16518, 16389, 16261, 16400, 16390, 16351, 16383, 16380] avg=16384.0000 max=16518 max_rank=0 min=16261 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=114554 max_load_reduction_pct=0.873978 moved_to_echo=65504 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[134, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 6, 0, 0, 4096, 4096, 4096, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 23, 0, 4096, 0, 4096, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 0, 0, 1, 4096, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 3979, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 4080, 4095, 0, 4066, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3962, 104, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16496, 16384, 16293, 16420, 16400, 16317, 16383, 16379] avg=16384.0000 max=16496 max_rank=0 min=16293 min_rank=2 imbalance_ratio=1.006836 max_load_reduction=114576 max_load_reduction_pct=0.874146 moved_to_echo=65479 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[134, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 4096, 4096, 4095, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 13, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4066, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 3962, 104, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16517, 16389, 16267, 16397, 16386, 16347, 16385, 16384] avg=16384.0000 max=16517 max_rank=0 min=16267 min_rank=2 imbalance_ratio=1.008118 max_load_reduction=114555 max_load_reduction_pct=0.873985 moved_to_echo=65502 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[1152, 0, 0, 128, 32768, 128, 128, 128, 0, 32768, 128, 0, 32768, 128, 31744, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 128, 128, 32768, 128, 32768, 128, 0, 0, 128, 32768, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[32768, 0, 0, 0, 32768, 0, 0, 128, 128, 32768, 0, 128, 128, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 32768, 128, 128, 0, 32768, 32768, 0, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 128, 128, 128, 384, 0, 0, 32768, 32768, 0, 32640, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 32768, 32768, 128, 128, 0, 128, 128, 0, 0, 128, 31872, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 0, 128, 32768, 128, 0, 0, 128, 32768, 32768, 0, 128, 0, 0, 0, 31744, 896, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 0, 128, 128, 128, 128, 0, 128, 128, 128, 128, 128, 0, 1024, 128, 0, 32768, 32768, 32768, 31872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 3997, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4070, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 2316, 3182, 0, 0, 4057, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 65, 0, 4096, 4094, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1653, 4095, 0, 0, 4096, 0, 4096, 0, 0, 4095, 4075, 4096, 0, 0, 4089, 0, 0, 4096, 1, 0, 0, 4096, 25, 0, 0, 7, 0, 67, 4093, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3948, 0, 0, 0, 0, 0, 354, 864, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3987, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4040, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 870, 2769, 0, 0, 4039, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 36, 0, 4096, 4091, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2884, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4045, 4096, 0, 0, 4067, 0, 0, 4096, 0, 0, 0, 4096, 21, 0, 0, 5, 0, 199, 4090, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 0, 0, 574, 1265, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3455, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3082, 2330, 0, 0, 4086, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 4096, 4095, 0, 0, 13, 0, 4096, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2380, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4042, 4096, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 4096, 9, 0, 0, 6, 0, 83, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3642, 0, 0, 0, 0, 0, 753, 707, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 1, 4066, 1, 1, 1, 0, 0, 4094, 1, 1, 3770, 4, 1, 12, 2, 4093, 1, 4095, 0, 0, 0, 0, 4095, 4094, 1, 4084, 1, 1, 18, 4087, 1, 1, 1, 0, 0, 4067, 0, 3, 0, 62, 3566, 4, 2, 3922, 0, 0, 2, 0, 0, 0, 0, 1, 4091, 0, 4095, 4092, 0, 1, 1, 0, 4095, 4095, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 4083, 1, 1, 1, 7, 2585, 4095, 276, 4, 4095, 1, 4093, 1, 0, 4095, 4085, 4083, 1, 0, 3315, 0, 0, 4095, 0, 1, 0, 4057, 7, 0, 20, 0, 0, 1267, 4082, 1, 0, 1, 4095, 191, 0, 1, 70, 0, 1992, 0, 6, 0, 1, 1, 1414, 2223, 4093, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 3918, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4025, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1519, 2827, 0, 0, 4083, 0, 0, 0, 11, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 62, 0, 4096, 4087, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 2399, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4022, 4096, 0, 0, 4053, 0, 0, 4096, 0, 0, 0, 4096, 41, 0, 0, 15, 0, 288, 4096, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 1223, 266, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4069, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4090, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1766, 3325, 0, 0, 3999, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 14, 0, 4096, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3057, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4067, 4096, 0, 0, 4066, 0, 0, 4096, 0, 0, 0, 4096, 13, 0, 0, 5, 0, 32, 4096, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 2658, 0, 0, 0, 0, 0, 1178, 429, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3786, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1484, 3627, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 100, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1709, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4073, 4096, 0, 0, 4079, 0, 0, 4096, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 1032, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3893, 0, 0, 0, 0, 0, 441, 356, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3981, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1238, 2368, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 21, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2696, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4041, 4096, 0, 0, 4093, 0, 0, 4096, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 972, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4003, 0, 0, 0, 0, 0, 213, 991, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3987, 0, 0, 0, 0, 3506, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 24, 0, 0, 0, 0, 0, 4096, 0, 4040, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 870, 2769, 0, 0, 4039, 0, 0, 0, 3257, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 36, 0, 4058, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2884, 4096, 4096, 1086, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 121, 4045, 4096, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 21, 0, 0, 5, 0, 199, 4090, 0, 0, 0, 4096, 0, 3526, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 0, 0, 574, 1265, 4096, 0, 0, 0, 4096, 3010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16500, 16352, 15480, 16380, 16849, 16425, 16071, 17015] avg=16384.0000 max=17015 max_rank=7 min=15480 min_rank=2 imbalance_ratio=1.038513 max_load_reduction=114057 max_load_reduction_pct=0.870186 moved_to_echo=66360 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 3997, 0, 0, 0, 0, 3507, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 24, 0, 0, 0, 0, 0, 4096, 0, 4070, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 2316, 3182, 0, 0, 4057, 0, 0, 0, 3257, 448, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 65, 0, 4058, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1653, 4095, 4096, 1087, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 122, 4075, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 25, 0, 0, 7, 0, 67, 4093, 0, 0, 0, 4095, 0, 3525, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3948, 0, 0, 0, 0, 0, 354, 864, 4096, 0, 0, 0, 4096, 3009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16512, 16382, 17357, 16412, 15617, 16478, 15947, 16367] avg=16384.0000 max=17357 max_rank=2 min=15617 min_rank=4 imbalance_ratio=1.059387 max_load_reduction=113715 max_load_reduction_pct=0.867577 moved_to_echo=64409 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3455, 0, 0, 0, 0, 3506, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 24, 0, 0, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3082, 2330, 0, 0, 4086, 0, 0, 0, 3257, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 13, 0, 4058, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2380, 4096, 4096, 1087, 590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 121, 4042, 4096, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 9, 0, 0, 6, 0, 83, 4096, 0, 0, 0, 4096, 0, 3526, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3642, 0, 0, 0, 0, 0, 753, 707, 4096, 0, 0, 0, 4096, 3009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15968, 16399, 17300, 16360, 16346, 16446, 15950, 16303] avg=16384.0000 max=17300 max_rank=2 min=15950 min_rank=6 imbalance_ratio=1.055908 max_load_reduction=113772 max_load_reduction_pct=0.868011 moved_to_echo=65045 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4066, 1, 1, 1, 0, 0, 4094, 1, 1, 3770, 4, 1, 12, 2, 3506, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 21, 0, 0, 0, 0, 0, 4094, 1, 4084, 1, 1, 18, 4087, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4067, 0, 3, 0, 62, 3566, 4, 2, 3922, 0, 0, 2, 3257, 450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4091, 0, 0, 4092, 0, 1, 1, 0, 4054, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 4083, 1, 1, 1, 7, 2585, 4095, 4095, 1087, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 276, 4, 0, 1, 4093, 1, 0, 118, 4085, 4083, 1, 0, 3315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4057, 7, 0, 20, 0, 0, 1267, 4082, 1, 0, 1, 4095, 191, 3527, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 70, 0, 1992, 0, 6, 0, 1, 1, 1414, 2223, 4093, 0, 1, 0, 4095, 3008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16278, 16402, 15337, 16335, 16548, 15977, 17290, 16905] avg=16384.0000 max=17290 max_rank=6 min=15337 min_rank=2 imbalance_ratio=1.055298 max_load_reduction=113782 max_load_reduction_pct=0.868088 moved_to_echo=66720 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4069, 0, 0, 0, 0, 3507, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 24, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1766, 3325, 0, 0, 3999, 0, 0, 0, 3257, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 14, 0, 4058, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 3057, 4096, 4096, 1087, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 121, 4067, 4096, 0, 0, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 13, 0, 0, 5, 0, 32, 4096, 0, 0, 0, 4095, 1, 3526, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2658, 0, 0, 0, 0, 0, 1178, 429, 4096, 0, 0, 0, 4096, 3009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16584, 16401, 16892, 16361, 17020, 16446, 15902, 15466] avg=16384.0000 max=17020 max_rank=4 min=15466 min_rank=7 imbalance_ratio=1.038818 max_load_reduction=114052 max_load_reduction_pct=0.870148 moved_to_echo=64834 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3786, 0, 0, 0, 0, 3507, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 24, 0, 0, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1484, 3627, 0, 0, 4095, 0, 0, 0, 3257, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 100, 0, 4058, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1709, 4096, 4096, 1087, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 121, 4073, 4096, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 1032, 4094, 0, 0, 0, 4096, 0, 3526, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3893, 0, 0, 0, 0, 0, 441, 356, 4096, 0, 0, 0, 4096, 3009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16300, 16399, 17008, 16447, 15673, 16465, 16889, 15891] avg=16384.0000 max=17008 max_rank=2 min=15673 min_rank=4 imbalance_ratio=1.038086 max_load_reduction=114064 max_load_reduction_pct=0.870239 moved_to_echo=64918 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 3918, 0, 0, 0, 0, 3507, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 24, 0, 0, 0, 0, 0, 4096, 0, 4025, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1519, 2827, 0, 0, 4083, 0, 0, 0, 3257, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 62, 0, 4058, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 2399, 4096, 4096, 1087, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 121, 4022, 4096, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 41, 0, 0, 15, 0, 288, 4096, 0, 0, 0, 4095, 1, 3526, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4027, 0, 0, 0, 0, 0, 1223, 266, 4096, 0, 0, 0, 4096, 3009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16435, 16336, 16231, 16410, 16359, 16388, 16196, 16717] avg=16384.0000 max=16717 max_rank=7 min=16196 min_rank=6 imbalance_ratio=1.020325 max_load_reduction=114355 max_load_reduction_pct=0.872459 moved_to_echo=65660 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 3981, 0, 0, 0, 0, 3507, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 24, 0, 0, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1238, 2368, 0, 0, 4059, 0, 0, 0, 3257, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 21, 0, 4058, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2696, 4096, 4096, 1087, 589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 121, 4041, 4096, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 972, 4093, 0, 0, 0, 4096, 0, 3526, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 0, 0, 0, 213, 991, 4096, 0, 0, 0, 4096, 3009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16495, 16401, 15467, 16367, 16660, 16447, 16827, 16408] avg=16384.0000 max=16827 max_rank=6 min=15467 min_rank=2 imbalance_ratio=1.027039 max_load_reduction=114245 max_load_reduction_pct=0.871620 moved_to_echo=66342 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 128, 0, 32768, 128, 128, 30976, 128, 128, 128, 128, 28160, 6528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 256, 0, 0, 0, 0, 0, 32768, 128, 32640, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[128, 0, 0, 0, 128, 32768, 0, 0, 32768, 0, 128, 384, 0, 32512, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 32768, 256, 0, 128, 128, 0, 3968, 32768, 128, 0, 128, 32768, 256, 28288, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 0, 32768, 0, 128, 0, 12416, 24064, 128, 128, 32384, 0, 0, 128, 26112, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 28160, 0, 128, 0, 128, 128, 6272, 7168, 32768, 0, 128, 0, 32768, 24192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[384, 128, 0, 128, 32768, 128, 0, 1024, 32512, 32768, 128, 0, 31872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 0, 128, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 19456, 32768, 32768, 8704, 4736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 102, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 4037, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 775, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3277, 4095, 0, 0, 4096, 4096, 0, 4093, 3, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 4096, 1, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4095, 1, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 84, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 2, 3922, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 956, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3230, 4095, 0, 0, 4096, 4096, 0, 4093, 2, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 497, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 3677, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2219, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1777, 4096, 0, 0, 4096, 4096, 0, 4094, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 22, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 184, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4071, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2217, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1718, 4096, 0, 0, 4096, 4096, 0, 4096, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4092, 0, 0, 1, 6, 0, 4095, 0, 0, 0, 102, 4095, 0, 4092, 0, 4094, 0, 4, 1, 0, 11, 4095, 0, 2, 1, 1, 0, 0, 0, 1, 4095, 68, 4094, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 2, 0, 1, 4093, 2, 0, 3900, 1, 1, 0, 4094, 1, 3, 1, 0, 1, 0, 0, 0, 21, 0, 1, 18, 1, 1, 1, 169, 4095, 1, 1, 4071, 4085, 1, 4095, 0, 3982, 4092, 127, 4095, 0, 0, 7, 0, 1, 4047, 4089, 0, 0, 3992, 0, 1, 0, 1, 0, 4, 0, 4095, 4095, 4071, 1, 4095, 0, 4088, 0, 4095, 4095, 0, 1, 0, 1, 4095, 8, 0, 4095, 0, 1, 1, 0, 1, 0, 1, 4082] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 223, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3972, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2443, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1555, 4096, 0, 0, 4096, 4096, 0, 4094, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 193, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 5, 4018, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2207, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 4091, 0, 0, 4096, 4096, 0, 4086, 10, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 307, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 838, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2965, 4096, 0, 0, 4096, 4096, 0, 4096, 1, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 102, 4096, 2055, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2250, 1842, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4037, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2253, 1945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 775, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3277, 2040, 0, 0, 4096, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 1, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2251, 1841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16273, 16381, 16427, 15215, 17602, 16404, 16389, 16381] avg=16384.0000 max=17602 max_rank=4 min=15215 min_rank=3 imbalance_ratio=1.074341 max_load_reduction=113470 max_load_reduction_pct=0.865707 moved_to_echo=66776 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 497, 4096, 2056, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2250, 1843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 3677, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2253, 1945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2219, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1777, 2040, 0, 0, 4096, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 22, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2251, 1841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16669, 16381, 16069, 16658, 16103, 16401, 16389, 16402] avg=16384.0000 max=16669 max_rank=0 min=16069 min_rank=2 imbalance_ratio=1.017395 max_load_reduction=114403 max_load_reduction_pct=0.872826 moved_to_echo=65295 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 184, 4096, 2056, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2250, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4071, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2252, 1945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2217, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1718, 2040, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2251, 1843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16356, 16382, 16460, 16656, 16046, 16401, 16387, 16384] avg=16384.0000 max=16656 max_rank=3 min=16046 min_rank=4 imbalance_ratio=1.016602 max_load_reduction=114416 max_load_reduction_pct=0.872925 moved_to_echo=65218 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 84, 4096, 2055, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2250, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 3922, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2252, 1945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 956, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3230, 2040, 0, 0, 4096, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2251, 1841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16255, 16383, 16313, 15395, 17555, 16403, 16389, 16379] avg=16384.0000 max=17555 max_rank=4 min=15395 min_rank=3 imbalance_ratio=1.071472 max_load_reduction=113517 max_load_reduction_pct=0.866066 moved_to_echo=66726 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4092, 0, 0, 1, 6, 0, 4095, 0, 0, 0, 102, 4095, 2060, 1830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4094, 0, 4, 1, 0, 11, 4095, 0, 2, 1, 1, 0, 0, 2254, 1843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 68, 4094, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 2, 0, 2252, 1945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 2, 0, 3900, 1, 1, 0, 4094, 1, 3, 1, 0, 1, 0, 0, 4095, 2150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 1, 18, 1, 1, 1, 169, 2035, 1, 1, 4071, 4085, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3982, 6, 127, 0, 0, 0, 7, 0, 1, 4047, 4089, 0, 0, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4, 0, 0, 4095, 4071, 1, 0, 0, 4088, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 8, 0, 4095, 0, 1, 1, 0, 1, 0, 1, 4082, 2256, 1841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16283, 16398, 16557, 18343, 14501, 16251, 16356, 16383] avg=16384.0000 max=18343 max_rank=3 min=14501 min_rank=4 imbalance_ratio=1.119568 max_load_reduction=112729 max_load_reduction_pct=0.860054 moved_to_echo=63491 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 223, 4096, 2056, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2250, 1843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3972, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2253, 1945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2443, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1555, 2040, 0, 0, 4096, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2251, 1841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16395, 16381, 16362, 16882, 15881, 16401, 16389, 16381] avg=16384.0000 max=16882 max_rank=3 min=15881 min_rank=4 imbalance_ratio=1.030396 max_load_reduction=114190 max_load_reduction_pct=0.871201 moved_to_echo=65052 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 193, 4096, 2053, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 2252, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 4018, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2252, 1945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2207, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1775, 2038, 0, 0, 4096, 4096, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2251, 1844, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16362, 16385, 16412, 16646, 16091, 16410, 16384, 16382] avg=16384.0000 max=16646 max_rank=3 min=16091 min_rank=4 imbalance_ratio=1.015991 max_load_reduction=114426 max_load_reduction_pct=0.873001 moved_to_echo=65267 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 307, 4096, 2056, 1828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2250, 1843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4082, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2253, 1945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 838, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2965, 2040, 0, 0, 4096, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 2251, 1841, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16479, 16381, 16472, 15277, 17293, 16401, 16389, 16380] avg=16384.0000 max=17293 max_rank=4 min=15277 min_rank=3 imbalance_ratio=1.055481 max_load_reduction=113779 max_load_reduction_pct=0.868065 moved_to_echo=66463 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 128, 128, 0, 32768, 0, 0, 0, 1792, 32768, 16512, 14720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 32768, 128, 128, 32768, 0, 128, 128, 0, 128, 0, 128, 32768, 18048, 14848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 0, 128, 0, 128, 32768, 32768, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 0, 15616, 128, 128, 128, 32768, 128, 128, 128, 0, 128, 0, 0, 32768, 17280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 32768, 128, 32000, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 128, 0, 18048, 15616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 0, 32768, 0, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 0, 0, 18048, 14848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 0, 128, 128, 128, 128, 128, 16512, 16384, 128, 128, 32768, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 32768, 32768, 128, 0, 128, 32768, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 159, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 2599, 4096, 0, 4068, 0, 111, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4096, 629, 4095, 4096, 0, 0, 0, 5, 4092, 1, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 4096, 4096, 0, 620, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 121, 0, 4096, 4094, 0, 0, 4094, 0, 0, 0, 3972, 0, 0, 0, 4096, 4090, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 57, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 2, 0, 4096, 0, 0, 1578, 4096, 0, 4074, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 4096, 1587, 4096, 4096, 0, 0, 0, 3, 4094, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 4096, 4096, 0, 448, 0, 4096, 0, 4090, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 101, 0, 4096, 4095, 0, 0, 4094, 0, 0, 0, 4086, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 4082, 97, 1, 171, 12, 1, 0, 0, 4095, 4094, 1, 0, 0, 4095, 0, 0, 1, 4092, 27, 1, 2660, 4095, 0, 4009, 5, 690, 1, 1, 33, 1, 0, 5, 0, 1, 0, 4, 0, 1, 1, 1, 4, 0, 1, 0, 8, 0, 4091, 4095, 184, 4093, 4095, 0, 0, 0, 8, 3753, 0, 1, 1, 4, 1, 0, 0, 4091, 4, 1, 0, 0, 2, 1, 4080, 4090, 4067, 0, 994, 0, 4094, 0, 3893, 1, 4094, 13, 1, 0, 0, 4087, 1, 1, 4094, 4094, 0, 1, 4095, 0, 0, 1, 4095, 0, 1, 1, 0, 112, 1, 4095, 4026, 0, 0, 4095, 2, 0, 1, 3968, 1, 1, 1, 4095, 4095, 4062] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 90, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 2030, 4096, 0, 4092, 0, 95, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 302, 4096, 4096, 0, 0, 0, 8, 4084, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 4096, 4096, 0, 704, 0, 4090, 0, 4077, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1162, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 3839, 0, 0, 0, 4096, 4095, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 154, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 0, 564, 4096, 0, 4049, 0, 23, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1567, 4096, 4096, 0, 0, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 1778, 0, 4095, 0, 4092, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 85, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4075, 0, 0, 0, 4096, 4095, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 126, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 2172, 4096, 0, 4089, 0, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 616, 4096, 4096, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 879, 0, 4093, 0, 4095, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 431, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 3663, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 3, 0, 4096, 0, 0, 1832, 4096, 0, 4029, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 4096, 4096, 1009, 4096, 4096, 0, 0, 0, 11, 4070, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 4096, 4096, 4096, 0, 256, 0, 4096, 0, 4075, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 1028, 0, 4096, 4095, 0, 0, 4090, 0, 0, 0, 4029, 0, 0, 0, 4096, 4093, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 281, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1203, 4096, 0, 4068, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 1049, 4096, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 1170, 0, 4096, 0, 4090, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 242, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 4089, 0, 0, 0, 4096, 4096, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 159, 0, 0, 0, 0, 4096, 4096, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2263, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2599, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 111, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3743, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 3226, 629, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4092, 1, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 3949, 1833, 870, 818, 410, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 620, 0, 3278, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 3686, 0, 0, 0, 4096, 0, 0, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 145, 0, 0, 0, 3972, 0, 0, 0, 0, 4090, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16400, 17151, 16263, 16143, 16438, 16186, 16094, 16397] avg=16384.0000 max=17151 max_rank=1 min=16094 min_rank=6 imbalance_ratio=1.046814 max_load_reduction=113921 max_load_reduction_pct=0.869148 moved_to_echo=65115 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 4082, 97, 1, 171, 12, 1, 4095, 4095, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2259, 4094, 1, 0, 0, 0, 0, 0, 1, 4092, 27, 1, 2660, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 5, 690, 1, 1, 33, 1, 0, 5, 0, 1, 0, 4, 0, 1, 4094, 4090, 3735, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4, 0, 1, 0, 8, 0, 4091, 3221, 184, 4093, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3753, 0, 1, 1, 4, 1, 0, 0, 4091, 4, 1, 0, 0, 2, 1, 3954, 1836, 874, 822, 411, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 4067, 0, 994, 0, 3272, 0, 3893, 1, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4094, 4094, 0, 1, 3684, 0, 0, 1, 4095, 0, 1, 1, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4026, 0, 0, 141, 2, 0, 1, 3968, 1, 1, 1, 0, 4095, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16507, 17230, 16815, 15699, 16116, 16321, 16085, 16299] avg=16384.0000 max=17230 max_rank=1 min=15699 min_rank=3 imbalance_ratio=1.051636 max_load_reduction=113842 max_load_reduction_pct=0.868546 moved_to_echo=64821 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 90, 0, 0, 0, 0, 4096, 4096, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2263, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2030, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 95, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3742, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3226, 302, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4084, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 3950, 1833, 870, 817, 410, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 704, 0, 3273, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3686, 0, 0, 0, 4096, 0, 0, 0, 0, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 146, 0, 0, 0, 3839, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16328, 16581, 16269, 15816, 16424, 16246, 17136, 16272] avg=16384.0000 max=17136 max_rank=6 min=15816 min_rank=3 imbalance_ratio=1.045898 max_load_reduction=113936 max_load_reduction_pct=0.869263 moved_to_echo=66078 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 57, 0, 0, 0, 0, 4096, 4096, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2263, 4096, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 1578, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3743, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 3226, 1587, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 3949, 1833, 870, 818, 410, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 448, 0, 3278, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3686, 0, 0, 0, 4096, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 145, 0, 0, 0, 4086, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16295, 16131, 16514, 17103, 16428, 16008, 16075, 16518] avg=16384.0000 max=17103 max_rank=3 min=16008 min_rank=5 imbalance_ratio=1.043884 max_load_reduction=113969 max_load_reduction_pct=0.869514 moved_to_echo=65029 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 154, 0, 0, 0, 0, 4096, 4096, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2263, 4096, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 564, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 0, 23, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3742, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3226, 1567, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3950, 1833, 870, 818, 410, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1778, 0, 3277, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3686, 0, 0, 0, 4096, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 146, 0, 0, 0, 4075, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16392, 15116, 16155, 17081, 16422, 17339, 16059, 16508] avg=16384.0000 max=17339 max_rank=5 min=15116 min_rank=1 imbalance_ratio=1.058289 max_load_reduction=113733 max_load_reduction_pct=0.867714 moved_to_echo=66328 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 126, 0, 0, 0, 0, 4096, 4096, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2263, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2172, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3743, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3226, 616, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3950, 1833, 870, 818, 410, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 879, 0, 3275, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3686, 0, 0, 0, 4096, 0, 0, 0, 0, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 146, 0, 0, 0, 3663, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16364, 16723, 16506, 16130, 16406, 16441, 16405, 16097] avg=16384.0000 max=16723 max_rank=1 min=16097 min_rank=7 imbalance_ratio=1.020691 max_load_reduction=114349 max_load_reduction_pct=0.872414 moved_to_echo=65349 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 0, 4096, 4096, 3951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2263, 4096, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 0, 1832, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4029, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3743, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 4096, 3226, 1009, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4070, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 3945, 1833, 870, 818, 410, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 256, 0, 3278, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3686, 0, 0, 0, 4096, 0, 0, 0, 0, 1028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 145, 0, 0, 0, 4029, 0, 0, 0, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16267, 16386, 16211, 16537, 16410, 15801, 17002, 16458] avg=16384.0000 max=17002 max_rank=6 min=15801 min_rank=5 imbalance_ratio=1.037720 max_load_reduction=114070 max_load_reduction_pct=0.870285 moved_to_echo=65671 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 281, 0, 0, 0, 0, 4096, 4096, 3950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2263, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1203, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3743, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3226, 1049, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3950, 1833, 870, 818, 410, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 1170, 0, 3278, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 3686, 0, 0, 0, 4096, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 146, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16519, 15754, 16339, 16563, 16428, 16730, 16216, 16523] avg=16384.0000 max=16730 max_rank=5 min=15754 min_rank=1 imbalance_ratio=1.021118 max_load_reduction=114342 max_load_reduction_pct=0.872360 moved_to_echo=65897 active_clone_links=13 active_echo_slots_local=13 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 32768, 0, 0, 1280, 128, 0, 128, 31744, 128, 128, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 128, 0, 128, 0, 0, 0, 32768, 1024, 128, 256, 128, 128, 32768, 32768, 31616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 29568, 0, 0, 128, 32768, 0, 128, 128, 0, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 18176, 32768, 128, 0, 0, 0, 0, 128, 128, 32768, 128, 128, 14720, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131584 observed_loss=-512 tokens_per_expert=[32768, 0, 32768, 0, 6912, 0, 26240, 0, 32512, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 128, 32768, 25856, 7040, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32512, 128, 1920, 128, 128, 128, 128, 0, 128, 0, 128, 0, 128, 0, 128, 32768, 32768, 29952, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 32384, 128, 128, 128, 128, 128, 0, 0, 32768, 128, 128, 128, 0, 128, 128, 31616, 14720, 7040, 6656, 3328, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131584 slack_slots=130560 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True + [2026-08-30 05:03:40] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 796.7 | throughput per GPU (TFLOP/s/GPU): 418.5 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.790970E+00 | loss scale: 1.0 | grad norm: 4.948 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[2044, 1577, 1588, 4, 1668, 1, 1200, 1484, 1202, 2078, 1555, 815, 1367, 1083, 1, 872, 593, 1072, 1075, 811, 11, 991, 188, 2242, 791, 154, 2435, 1292, 1299, 316, 1292, 1121, 203, 31, 1895, 741, 394, 1458, 1464, 541, 535, 45, 1147, 1739, 0, 1305, 974, 1151, 1486, 557, 747, 1569, 22, 313, 76, 1840, 0, 936, 229, 836, 1162, 1574, 925, 1063, 954, 70, 0, 1228, 1894, 2213, 1932, 1409, 1434, 1453, 1724, 1053, 1933, 125, 875, 1885, 1485, 730, 37, 17, 1786, 812, 1527, 41, 2184, 335, 1172, 560, 404, 1017, 873, 0, 673, 689, 47, 2548, 80, 830, 1562, 1145, 379, 1287, 592, 817, 1197, 1924, 2331, 1957, 2020, 1212, 372, 1369, 2328, 857, 1656, 1175, 1075, 1689, 1333, 1368, 1065, 53, 1026, 74] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[1958, 1445, 1252, 0, 896, 2, 1600, 1750, 1435, 2632, 1632, 1165, 1072, 2123, 0, 684, 658, 1101, 761, 585, 58, 1037, 188, 2303, 809, 356, 2360, 1861, 1727, 333, 1384, 1049, 100, 5, 2091, 1121, 312, 2080, 932, 903, 521, 23, 1427, 2150, 0, 823, 1275, 941, 1155, 586, 1082, 1247, 41, 151, 18, 1255, 0, 589, 33, 932, 1033, 2015, 1291, 1138, 710, 1, 0, 1404, 1596, 2011, 789, 1063, 760, 959, 1144, 1212, 1537, 70, 1815, 1720, 1274, 823, 35, 3, 2487, 1033, 1198, 11, 2269, 630, 1083, 356, 581, 2034, 1235, 0, 1529, 1412, 8, 1666, 39, 954, 1202, 1444, 272, 1143, 511, 1014, 975, 1124, 2258, 1565, 1929, 1493, 680, 1154, 2080, 1476, 1645, 1267, 900, 1517, 1146, 720, 1473, 20, 1085, 42] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[1715, 1057, 1231, 0, 1672, 0, 1344, 1532, 1398, 1538, 1270, 1120, 1069, 1121, 1, 1144, 785, 1751, 567, 1062, 1, 937, 210, 1982, 630, 101, 2559, 1582, 1273, 122, 1123, 995, 212, 33, 1932, 350, 220, 2376, 1377, 744, 380, 12, 1415, 2560, 5, 1062, 1737, 1498, 1551, 605, 620, 1431, 34, 147, 21, 1386, 0, 717, 115, 1116, 851, 1575, 858, 1753, 1239, 8, 0, 1421, 2711, 2304, 1690, 1201, 1694, 2110, 1793, 993, 1366, 11, 1921, 1318, 1457, 461, 42, 31, 2161, 576, 1219, 60, 1788, 418, 1192, 1160, 534, 1523, 987, 1, 828, 900, 22, 1682, 56, 843, 1132, 1048, 267, 1291, 561, 1008, 1345, 1228, 2078, 2310, 1593, 1019, 345, 860, 2653, 1089, 1576, 1227, 1596, 1334, 1147, 668, 1514, 51, 786, 40] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2125, 1735, 1172, 0, 1399, 1, 1065, 1310, 1057, 2286, 1821, 1393, 1280, 2141, 1, 696, 502, 1431, 548, 711, 7, 1026, 236, 1927, 945, 150, 2150, 1122, 822, 481, 1137, 1382, 328, 14, 2107, 859, 136, 1925, 1125, 1682, 413, 46, 1192, 1417, 1, 1053, 1159, 910, 1218, 482, 930, 1543, 75, 399, 25, 1452, 2, 977, 136, 1139, 979, 1757, 1131, 1066, 647, 7, 0, 1987, 2014, 1201, 1818, 804, 1121, 1413, 1205, 1164, 1524, 29, 1424, 1539, 1166, 464, 76, 27, 1823, 980, 1490, 28, 1796, 425, 1355, 932, 741, 1894, 884, 11, 1533, 1024, 12, 1932, 104, 771, 2092, 1496, 288, 1484, 484, 946, 632, 1658, 2570, 1941, 1608, 1264, 963, 1010, 2104, 1490, 1534, 1015, 1052, 1540, 1735, 970, 1241, 43, 737, 73] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[2416, 1139, 1404, 6, 952, 0, 1645, 2154, 1325, 2493, 2159, 1116, 845, 2082, 0, 620, 821, 1498, 703, 659, 20, 1014, 157, 1844, 749, 258, 2351, 1445, 1124, 143, 1071, 999, 75, 1, 2350, 576, 404, 2730, 1506, 1169, 204, 3, 1425, 2092, 4, 1315, 1848, 1370, 961, 461, 1242, 1490, 28, 312, 30, 1271, 0, 675, 55, 1375, 664, 1767, 966, 1157, 1052, 16, 1, 1443, 1978, 2224, 1789, 1013, 1257, 1269, 1128, 719, 1818, 38, 2180, 1163, 1141, 627, 20, 5, 1746, 1127, 1219, 9, 2065, 771, 1524, 660, 603, 1455, 1269, 0, 895, 427, 17, 1734, 18, 724, 1292, 981, 258, 1101, 541, 1005, 535, 1693, 2367, 1887, 1873, 1276, 800, 1184, 1718, 1125, 1176, 1331, 1075, 1729, 1066, 507, 1876, 31, 695, 68] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1964, 1282, 1404, 1, 914, 0, 1855, 1957, 1539, 1808, 2253, 909, 1457, 1546, 0, 1038, 578, 1308, 887, 803, 13, 1097, 237, 1959, 966, 182, 2233, 1450, 1323, 193, 1659, 1300, 91, 2, 2301, 760, 187, 1284, 1116, 969, 204, 77, 1604, 1984, 0, 1105, 1177, 976, 1000, 361, 1825, 1417, 98, 510, 26, 1442, 0, 682, 174, 1318, 1134, 1748, 916, 1169, 830, 16, 0, 1141, 1207, 2364, 2012, 1676, 1544, 1660, 1495, 812, 2104, 119, 1243, 1864, 1164, 1105, 43, 28, 1674, 562, 989, 34, 1931, 578, 1122, 714, 397, 1925, 895, 2, 1172, 1215, 10, 1877, 43, 633, 1833, 1063, 401, 1259, 501, 731, 713, 719, 1905, 1726, 1904, 1576, 573, 1268, 2182, 999, 1185, 1503, 1117, 1565, 1120, 1253, 989, 61, 843, 111] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[2362, 1668, 1403, 5, 509, 0, 1368, 1729, 1370, 2325, 2392, 633, 817, 2653, 0, 792, 990, 1568, 715, 802, 19, 998, 212, 1964, 690, 393, 2711, 1707, 1049, 544, 1143, 880, 111, 8, 1809, 1205, 371, 2101, 1206, 1197, 433, 59, 1696, 2009, 0, 1065, 1802, 1681, 1393, 735, 1418, 1471, 28, 340, 5, 927, 0, 730, 32, 1592, 979, 2394, 1103, 958, 732, 2, 0, 1111, 1849, 1542, 1238, 886, 935, 1142, 1746, 1065, 2234, 138, 2017, 1674, 1094, 466, 41, 44, 1986, 1070, 940, 15, 1445, 632, 749, 562, 719, 1593, 967, 2, 1221, 806, 16, 1710, 15, 1011, 1013, 1016, 306, 1055, 511, 915, 385, 1051, 2668, 1742, 1483, 1129, 666, 928, 1805, 1903, 1355, 1140, 992, 1496, 1448, 658, 2147, 13, 515, 48] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1610, 911, 1519, 0, 1094, 4, 1151, 1784, 1059, 1545, 1076, 1268, 1214, 1089, 0, 998, 664, 1005, 944, 1025, 21, 823, 257, 1859, 784, 69, 1983, 1191, 1178, 199, 939, 1256, 232, 28, 1997, 397, 637, 1825, 959, 927, 447, 42, 1185, 1595, 1, 1277, 1129, 821, 1417, 618, 452, 1988, 54, 298, 75, 1411, 0, 541, 231, 881, 1020, 2019, 977, 1129, 1393, 10, 0, 1404, 2055, 1714, 1975, 1192, 1093, 1477, 2577, 1150, 1823, 21, 1145, 1323, 1306, 383, 100, 128, 2214, 1048, 1247, 58, 2249, 566, 1109, 1187, 1143, 1371, 946, 33, 928, 827, 45, 2129, 104, 973, 1592, 1286, 367, 1271, 640, 1683, 1474, 1784, 1574, 2123, 1535, 1702, 713, 944, 2473, 1263, 1422, 1459, 1238, 1756, 1186, 919, 1818, 154, 1047, 74] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1573, 1577, 1588, 4, 1668, 1, 1200, 1484, 1202, 0, 1555, 815, 1367, 1083, 1, 872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 593, 1072, 1075, 811, 11, 991, 188, 2242, 791, 154, 2435, 1292, 1299, 316, 1292, 1121, 471, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 31, 1895, 741, 394, 1458, 1464, 541, 535, 45, 1147, 1739, 0, 1305, 974, 1151, 970, 260, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1486, 557, 747, 1569, 22, 313, 76, 1840, 0, 936, 229, 836, 1162, 1574, 925, 1063, 2078, 1262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 954, 70, 0, 1228, 924, 0, 1932, 1409, 1434, 1453, 1724, 1053, 1933, 125, 875, 1885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1485, 730, 37, 17, 1786, 812, 1527, 41, 2184, 335, 1172, 560, 404, 1017, 873, 0, 1856, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 673, 689, 47, 2548, 80, 830, 1562, 1145, 379, 1287, 592, 817, 1197, 1924, 1651, 1957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2020, 1212, 372, 1369, 274, 857, 1656, 1175, 1075, 1689, 1333, 1368, 1065, 53, 1026, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15990, 16574, 15051, 16675, 16999, 15787, 17378, 16618] avg=16384.0000 max=17378 max_rank=6 min=15051 min_rank=2 imbalance_ratio=1.060669 max_load_reduction=113694 max_load_reduction_pct=0.867416 moved_to_echo=66782 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1507, 1445, 1252, 0, 896, 2, 1600, 1750, 1435, 0, 1632, 1165, 1072, 2123, 0, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 658, 1101, 761, 585, 58, 1037, 188, 2303, 809, 356, 2360, 1861, 1727, 333, 1384, 1049, 451, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 5, 2091, 1121, 312, 2080, 932, 903, 521, 23, 1427, 2150, 0, 823, 1275, 941, 817, 252, 177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 586, 1082, 1247, 41, 151, 18, 1255, 0, 589, 33, 932, 1033, 2015, 1291, 1138, 2632, 1147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 710, 1, 0, 1404, 779, 0, 789, 1063, 760, 959, 1144, 1212, 1537, 70, 1815, 1720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1274, 823, 35, 3, 2487, 1033, 1198, 11, 2269, 630, 1083, 356, 581, 2034, 1235, 0, 1659, 864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1529, 1412, 8, 1666, 39, 954, 1202, 1444, 272, 1143, 511, 1014, 975, 1124, 1599, 1565, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1929, 1493, 680, 1154, 244, 1476, 1645, 1267, 900, 1517, 1146, 720, 1473, 20, 1085, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16563, 17428, 15950, 16345, 13963, 17575, 16457, 16791] avg=16384.0000 max=17575 max_rank=5 min=13963 min_rank=4 imbalance_ratio=1.072693 max_load_reduction=113497 max_load_reduction_pct=0.865913 moved_to_echo=64786 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1636, 1735, 1172, 0, 1399, 1, 1065, 1310, 1057, 0, 1821, 1393, 1280, 2141, 1, 696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 502, 1431, 548, 711, 7, 1026, 236, 1927, 945, 150, 2150, 1122, 822, 481, 1137, 1382, 489, 463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 14, 2107, 859, 136, 1925, 1125, 1682, 413, 46, 1192, 1417, 1, 1053, 1159, 910, 1032, 286, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1218, 482, 930, 1543, 75, 399, 25, 1452, 2, 977, 136, 1139, 979, 1757, 1131, 1066, 2286, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 647, 7, 0, 1987, 982, 0, 1818, 804, 1121, 1413, 1205, 1164, 1524, 29, 1424, 1539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1166, 464, 76, 27, 1823, 980, 1490, 28, 1796, 425, 1355, 932, 741, 1894, 884, 11, 1678, 515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1533, 1024, 12, 1932, 104, 771, 2092, 1496, 288, 1484, 484, 946, 632, 1658, 1821, 1941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1608, 1264, 963, 1010, 247, 1490, 1534, 1015, 1052, 1540, 1735, 970, 1241, 43, 737, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16707, 15529, 15864, 16283, 15664, 16285, 18218, 16522] avg=16384.0000 max=18218 max_rank=6 min=15529 min_rank=1 imbalance_ratio=1.111938 max_load_reduction=112854 max_load_reduction_pct=0.861008 moved_to_echo=66689 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1860, 1139, 1404, 6, 952, 0, 1645, 2154, 1325, 0, 2159, 1116, 845, 2082, 0, 620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 821, 1498, 703, 659, 20, 1014, 157, 1844, 749, 258, 2351, 1445, 1124, 143, 1071, 999, 556, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 1, 2350, 576, 404, 2730, 1506, 1169, 204, 3, 1425, 2092, 4, 1315, 1848, 1370, 1013, 264, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 961, 461, 1242, 1490, 28, 312, 30, 1271, 0, 675, 55, 1375, 664, 1767, 966, 1157, 2493, 1269, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 16, 1, 1443, 965, 0, 1789, 1013, 1257, 1269, 1128, 719, 1818, 38, 2180, 1163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1141, 627, 20, 5, 1746, 1127, 1219, 9, 2065, 771, 1524, 660, 603, 1455, 1269, 0, 1370, 955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 895, 427, 17, 1734, 18, 724, 1292, 981, 258, 1101, 541, 1005, 535, 1693, 1676, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1873, 1276, 800, 1184, 202, 1125, 1176, 1331, 1075, 1729, 1066, 507, 1876, 31, 695, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17307, 15839, 18495, 16216, 15851, 16566, 14784, 16014] avg=16384.0000 max=18495 max_rank=2 min=14784 min_rank=6 imbalance_ratio=1.128845 max_load_reduction=112577 max_load_reduction_pct=0.858894 moved_to_echo=63215 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1320, 1057, 1231, 0, 1672, 0, 1344, 1532, 1398, 0, 1270, 1120, 1069, 1121, 1, 1144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 785, 1751, 567, 1062, 1, 937, 210, 1982, 630, 101, 2559, 1582, 1273, 122, 1123, 995, 395, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 33, 1932, 350, 220, 2376, 1377, 744, 380, 12, 1415, 2560, 5, 1062, 1737, 1498, 1389, 232, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1551, 605, 620, 1431, 34, 147, 21, 1386, 0, 717, 115, 1116, 851, 1575, 858, 1753, 1538, 1315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1239, 8, 0, 1421, 1322, 0, 1690, 1201, 1694, 2110, 1793, 993, 1366, 11, 1921, 1318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1457, 461, 42, 31, 2161, 576, 1219, 60, 1788, 418, 1192, 1160, 534, 1523, 987, 1, 2116, 989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 900, 22, 1682, 56, 843, 1132, 1048, 267, 1291, 561, 1008, 1345, 1228, 1472, 2310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1593, 1019, 345, 860, 311, 1089, 1576, 1227, 1596, 1334, 1147, 668, 1514, 51, 786, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15279, 16449, 17760, 15633, 18087, 16715, 15993, 15156] avg=16384.0000 max=18087 max_rank=4 min=15156 min_rank=7 imbalance_ratio=1.103943 max_load_reduction=112985 max_load_reduction_pct=0.862007 moved_to_echo=65951 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1512, 1282, 1404, 1, 914, 0, 1855, 1957, 1539, 0, 2253, 909, 1457, 1546, 0, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 1308, 887, 803, 13, 1097, 237, 1959, 966, 182, 2233, 1450, 1323, 193, 1659, 1300, 452, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 2, 2301, 760, 187, 1284, 1116, 969, 204, 77, 1604, 1984, 0, 1105, 1177, 976, 618, 212, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1000, 361, 1825, 1417, 98, 510, 26, 1442, 0, 682, 174, 1318, 1134, 1748, 916, 1169, 1808, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 830, 16, 0, 1141, 589, 0, 2012, 1676, 1544, 1660, 1495, 812, 2104, 119, 1243, 1864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1164, 1105, 43, 28, 1674, 562, 989, 34, 1931, 578, 1122, 714, 397, 1925, 895, 2, 1740, 1016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1172, 1215, 10, 1877, 43, 633, 1833, 1063, 401, 1259, 501, 731, 713, 719, 1350, 1726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1904, 1576, 573, 1268, 256, 999, 1185, 1503, 1117, 1565, 1120, 1253, 989, 61, 843, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17667, 16983, 14853, 16976, 17105, 15919, 15246, 16323] avg=16384.0000 max=17667 max_rank=0 min=14853 min_rank=2 imbalance_ratio=1.078308 max_load_reduction=113405 max_load_reduction_pct=0.865211 moved_to_echo=64593 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1814, 1668, 1403, 5, 509, 0, 1368, 1729, 1370, 0, 2392, 633, 817, 2653, 0, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 990, 1568, 715, 802, 19, 998, 212, 1964, 690, 393, 2711, 1707, 1049, 544, 1143, 880, 548, 485, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 8, 1809, 1205, 371, 2101, 1206, 1197, 433, 59, 1696, 2009, 0, 1065, 1802, 1681, 951, 301, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1393, 735, 1418, 1471, 28, 340, 5, 927, 0, 730, 32, 1592, 979, 2394, 1103, 958, 2325, 880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 732, 2, 0, 1111, 898, 0, 1238, 886, 935, 1142, 1746, 1065, 2234, 138, 2017, 1674, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 466, 41, 44, 1986, 1070, 940, 15, 1445, 632, 749, 562, 719, 1593, 967, 2, 1442, 662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1221, 806, 16, 1710, 15, 1011, 1013, 1016, 306, 1055, 511, 915, 385, 1051, 1882, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1483, 1129, 666, 928, 206, 1903, 1355, 1140, 992, 1496, 1448, 658, 2147, 13, 515, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17153, 17418, 18162, 17310, 15818, 14429, 14655, 16127] avg=16384.0000 max=18162 max_rank=2 min=14429 min_rank=5 imbalance_ratio=1.108521 max_load_reduction=112910 max_load_reduction_pct=0.861435 moved_to_echo=61029 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1239, 911, 1519, 0, 1094, 4, 1151, 1784, 1059, 0, 1076, 1268, 1214, 1089, 0, 998, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 664, 1005, 944, 1025, 21, 823, 257, 1859, 784, 69, 1983, 1191, 1178, 199, 939, 1256, 371, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 28, 1997, 397, 637, 1825, 959, 927, 447, 42, 1185, 1595, 1, 1277, 1129, 821, 1053, 175, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1417, 618, 452, 1988, 54, 298, 75, 1411, 0, 541, 231, 881, 1020, 2019, 977, 1129, 1545, 978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1393, 10, 0, 1404, 1002, 0, 1975, 1192, 1093, 1477, 2577, 1150, 1823, 21, 1145, 1323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1306, 383, 100, 128, 2214, 1048, 1247, 58, 2249, 566, 1109, 1187, 1143, 1371, 946, 33, 1972, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 928, 827, 45, 2129, 104, 973, 1592, 1286, 367, 1271, 640, 1683, 1474, 1784, 1115, 2123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1535, 1702, 713, 944, 291, 1263, 1422, 1459, 1238, 1756, 1186, 919, 1818, 154, 1047, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14406, 14852, 14937, 15634, 17585, 17796, 18341, 17521] avg=16384.0000 max=18341 max_rank=6 min=14406 min_rank=0 imbalance_ratio=1.119446 max_load_reduction=112731 max_load_reduction_pct=0.860069 moved_to_echo=71243 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[10112, 5120, 512, 384, 16000, 7296, 9856, 256, 15744, 4480, 9344, 6144, 5248, 12928, 8064, 128, 13952, 6784, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1408, 128, 16512, 6016, 2688, 15872, 9728, 8192, 3200, 384, 11136, 15616, 128, 9088, 11136, 9472, 7936, 2048, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[7680, 256, 128, 11264, 7552, 0, 13312, 9344, 9856, 11520, 12928, 8192, 14464, 640, 12672, 12544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[12544, 10880, 11008, 128, 9216, 128, 11264, 13824, 10496, 0, 14208, 8448, 9216, 13952, 128, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[13952, 10752, 5120, 8832, 2048, 10240, 11648, 10240, 9088, 12672, 10240, 7168, 12160, 512, 6784, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[10240, 4480, 8320, 12160, 384, 2560, 384, 11008, 128, 5888, 1024, 9216, 7936, 14976, 8192, 9472, 16768, 8960, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[5632, 10752, 6272, 6528, 256, 7936, 1792, 16128, 6400, 1664, 18816, 11776, 9856, 2432, 9856, 9088, 3840, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[8832, 7424, 256, 15360, 512, 6784, 11776, 9600, 2560, 9984, 4352, 8192, 7296, 11264, 12672, 15360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 3884, 35, 0, 25, 47, 0, 0, 0, 4074, 0, 0, 0, 4096, 0, 0, 4082, 0, 0, 851, 0, 2991, 10, 0, 211, 0, 1296, 3955, 67, 0, 2, 4095, 4, 332, 0, 4092, 3, 0, 4093, 0, 0, 3262, 0, 3249, 0, 3487, 5, 2279, 0, 0, 4087, 0, 3989, 4091, 0, 0, 51, 3593, 0, 0, 309, 0, 3876, 0, 0, 0, 202, 21, 0, 3704, 0, 4094, 3915, 0, 0, 0, 20, 0, 26, 59, 3, 4096, 0, 0, 4, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 496, 3375, 4082, 0, 3982, 4065, 0, 181, 3, 0, 0, 4082, 4092, 0, 0, 15, 0, 2237, 1, 0, 856, 3396, 0, 1240, 0, 7, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[6, 19, 0, 2977, 159, 9, 0, 0, 12, 0, 0, 2585, 5, 0, 3, 4086, 0, 12, 4088, 3, 0, 2710, 0, 4094, 4, 0, 308, 0, 3829, 3979, 0, 0, 27, 4086, 0, 937, 0, 4090, 0, 8, 4049, 1, 3, 1934, 0, 2484, 7, 4089, 12, 3522, 21, 2, 4091, 0, 3800, 4073, 3, 0, 37, 3982, 0, 0, 29, 0, 3717, 13, 0, 7, 616, 37, 8, 3050, 15, 3964, 3547, 0, 7, 3, 0, 0, 368, 116, 0, 4058, 3, 0, 0, 188, 0, 7, 11, 0, 1, 4089, 348, 0, 0, 7, 0, 4076, 58, 4096, 4068, 7, 3507, 2252, 8, 39, 0, 8, 1, 4071, 4067, 0, 7, 250, 0, 2742, 0, 0, 1371, 1507, 0, 497, 0, 0, 0, 4085] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 3997, 533, 0, 9, 3, 0, 0, 0, 3508, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1919, 0, 3970, 6, 0, 613, 0, 1538, 3775, 17, 0, 0, 4095, 1, 841, 44, 4096, 0, 0, 4061, 0, 0, 2608, 0, 3320, 0, 3916, 0, 3037, 0, 0, 4096, 51, 3884, 4096, 0, 0, 11, 3765, 0, 0, 19, 0, 4059, 0, 0, 0, 1692, 1, 0, 2225, 1, 4055, 2917, 0, 0, 0, 0, 0, 228, 408, 12, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 17, 3964, 4095, 0, 3984, 3905, 0, 12, 0, 0, 0, 4095, 4096, 0, 0, 41, 3, 994, 6, 0, 793, 2213, 1, 851, 0, 0, 0, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 4, 0, 3439, 44, 0, 7, 0, 1, 2, 20, 3972, 0, 0, 0, 4095, 0, 3, 4094, 3, 0, 1079, 0, 4065, 0, 0, 21, 0, 3267, 3884, 117, 0, 0, 4087, 0, 904, 18, 4092, 0, 0, 4090, 0, 0, 2792, 1, 2041, 0, 4019, 6, 2191, 0, 0, 4096, 0, 3957, 4093, 3, 0, 419, 3350, 0, 0, 129, 0, 4065, 0, 0, 0, 233, 0, 3, 3744, 1, 4049, 3777, 1, 0, 2, 0, 2, 13, 196, 5, 4093, 0, 1, 0, 4, 0, 0, 0, 0, 0, 4096, 19, 0, 1, 0, 0, 4096, 298, 3329, 4091, 0, 3509, 4083, 2, 396, 10, 3, 3, 4089, 4093, 0, 0, 77, 0, 1894, 0, 4, 162, 3438, 0, 701, 1, 3, 0, 4080] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 2794, 525, 0, 2, 1, 0, 0, 2, 3140, 0, 0, 0, 4091, 0, 0, 4095, 0, 0, 1774, 3, 3816, 1, 0, 747, 0, 3328, 3988, 1, 1, 1, 4096, 0, 288, 12, 4096, 1, 0, 4061, 0, 0, 2585, 0, 2438, 0, 4004, 1, 3112, 0, 0, 4096, 5, 1842, 4095, 0, 0, 6, 3843, 1, 0, 7, 0, 3676, 0, 0, 0, 2019, 0, 0, 2579, 0, 3871, 3462, 0, 0, 0, 1, 0, 326, 303, 17, 4095, 1, 0, 0, 5, 1, 0, 0, 1, 0, 4096, 2, 0, 1, 0, 0, 4095, 8, 3675, 4095, 0, 3899, 4069, 0, 12, 4, 0, 0, 4095, 4096, 0, 0, 651, 1, 2308, 9, 0, 895, 2924, 0, 788, 0, 0, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 4002, 173, 0, 0, 0, 0, 0, 0, 4031, 0, 1, 0, 4083, 0, 1, 4095, 0, 0, 1729, 20, 3854, 0, 0, 25, 0, 2353, 4022, 154, 0, 1, 4096, 0, 348, 7, 4096, 2, 0, 4082, 2, 2, 3373, 0, 2634, 0, 3937, 38, 2568, 1, 0, 4096, 4, 3357, 4096, 0, 0, 35, 3444, 0, 0, 229, 2, 4003, 0, 0, 1, 149, 19, 0, 3135, 0, 4040, 3633, 0, 0, 2, 0, 0, 77, 184, 47, 4089, 0, 0, 0, 20, 0, 0, 9, 0, 0, 4088, 28, 1, 1, 0, 0, 4081, 136, 2897, 4095, 0, 3714, 4047, 0, 62, 6, 0, 0, 4095, 4094, 4, 1, 129, 0, 1509, 0, 0, 182, 3919, 0, 1494, 0, 0, 0, 4088] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[2, 7, 1, 3395, 73, 1, 0, 9, 4, 0, 22, 3461, 2, 2, 0, 4094, 0, 4, 4093, 2, 0, 2214, 0, 4063, 58, 1, 336, 2, 3890, 3958, 12, 0, 0, 4095, 0, 587, 2, 4093, 0, 3, 4086, 0, 2, 2977, 2, 1669, 4, 4043, 2, 3373, 0, 2, 4092, 0, 2373, 4094, 3, 3, 14, 3860, 0, 3, 8, 1, 4013, 0, 0, 4, 780, 8, 2, 2849, 3, 3735, 3035, 0, 2, 2, 2, 2, 107, 102, 17, 4089, 0, 2, 3, 143, 0, 2, 0, 0, 3, 4094, 39, 0, 2, 2, 0, 4094, 25, 3931, 4090, 2, 3756, 3906, 3, 148, 0, 2, 0, 4091, 4088, 0, 2, 1515, 0, 916, 0, 0, 725, 3414, 1, 131, 0, 0, 2, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 6, 0, 3907, 1316, 0, 0, 0, 1, 2, 0, 2290, 0, 0, 0, 4092, 0, 0, 4096, 0, 2, 2952, 8, 4094, 1, 0, 489, 0, 3860, 3743, 34, 0, 11, 4096, 1, 389, 0, 4092, 0, 0, 4068, 0, 0, 2249, 0, 2456, 0, 4090, 11, 2632, 1, 0, 4094, 0, 2248, 4096, 0, 0, 45, 3811, 0, 0, 18, 0, 3925, 1, 0, 0, 2586, 0, 0, 2592, 0, 3075, 2031, 0, 0, 0, 0, 0, 645, 159, 0, 4096, 0, 0, 9, 104, 0, 0, 0, 0, 0, 4096, 58, 0, 0, 0, 0, 4094, 1, 4040, 4096, 0, 4023, 3581, 0, 18, 1, 0, 0, 4096, 4087, 0, 0, 506, 0, 2017, 0, 0, 1798, 1204, 0, 737, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3884, 35, 0, 25, 47, 0, 0, 0, 4074, 0, 0, 0, 4096, 3235, 1191, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3359, 0, 0, 851, 0, 2991, 10, 0, 211, 0, 1296, 3955, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2506, 4, 332, 0, 0, 3, 0, 4093, 0, 0, 3262, 0, 3249, 0, 3487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2279, 0, 0, 2370, 0, 3989, 4091, 0, 0, 51, 3593, 0, 0, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3876, 0, 0, 0, 202, 21, 0, 3704, 0, 4094, 3915, 0, 0, 0, 20, 0, 723, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 59, 3, 4096, 0, 0, 4, 10, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 3554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 496, 3375, 4082, 0, 3982, 4065, 0, 181, 3, 0, 0, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 15, 0, 2237, 1, 0, 856, 3396, 0, 1240, 0, 7, 0, 4096, 1041, 526, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17126, 12740, 16938, 16687, 17103, 15940, 17031, 17507] avg=16384.0000 max=17507 max_rank=7 min=12740 min_rank=1 imbalance_ratio=1.068542 max_load_reduction=113565 max_load_reduction_pct=0.866432 moved_to_echo=67581 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3997, 533, 0, 9, 3, 0, 0, 0, 3508, 0, 0, 0, 4096, 3245, 1194, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3370, 0, 0, 1919, 0, 3970, 6, 0, 613, 0, 1538, 3775, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2506, 1, 841, 44, 0, 0, 0, 4061, 0, 0, 2608, 0, 3320, 0, 3916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3037, 0, 0, 2375, 51, 3884, 4096, 0, 0, 11, 3765, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 1692, 1, 0, 2225, 1, 4055, 2917, 0, 0, 0, 0, 0, 726, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 408, 12, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4096, 1, 0, 4096, 3557, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 3964, 4095, 0, 3984, 3905, 0, 12, 0, 0, 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 41, 3, 994, 6, 0, 793, 2213, 1, 851, 0, 0, 0, 4092, 1041, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17124, 15208, 17297, 17238, 16224, 16496, 16827, 14658] avg=16384.0000 max=17297 max_rank=2 min=14658 min_rank=7 imbalance_ratio=1.055725 max_load_reduction=113775 max_load_reduction_pct=0.868034 moved_to_echo=64205 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2794, 525, 0, 2, 1, 0, 0, 2, 3140, 0, 0, 0, 4091, 3245, 1194, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3370, 0, 0, 1774, 3, 3816, 1, 0, 747, 0, 3328, 3988, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2507, 0, 288, 12, 0, 1, 0, 4061, 0, 0, 2585, 0, 2438, 0, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3112, 0, 0, 2375, 5, 1842, 4095, 0, 0, 6, 3843, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3676, 0, 0, 0, 2019, 0, 0, 2579, 0, 3871, 3462, 0, 0, 0, 1, 0, 725, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 303, 17, 4095, 1, 0, 0, 5, 1, 0, 0, 1, 0, 4096, 2, 0, 4096, 3556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8, 3675, 4095, 0, 3899, 4069, 0, 12, 4, 0, 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 651, 1, 2308, 9, 0, 895, 2924, 0, 788, 0, 0, 0, 4093, 1041, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15533, 17029, 15897, 15287, 16881, 16499, 16613, 17333] avg=16384.0000 max=17333 max_rank=7 min=15287 min_rank=3 imbalance_ratio=1.057922 max_load_reduction=113739 max_load_reduction_pct=0.867760 moved_to_echo=67326 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 3439, 44, 0, 7, 0, 1, 2, 20, 3972, 0, 0, 0, 4095, 3240, 1194, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3369, 3, 0, 1079, 0, 4065, 0, 0, 21, 0, 3267, 3884, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2501, 0, 904, 18, 0, 0, 0, 4090, 0, 0, 2792, 1, 2041, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2191, 0, 0, 2375, 0, 3957, 4093, 3, 0, 419, 3350, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 233, 0, 3, 3744, 1, 4049, 3777, 1, 0, 2, 0, 2, 725, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 196, 5, 4093, 0, 1, 0, 4, 0, 0, 0, 0, 0, 4096, 19, 0, 4092, 3556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 298, 3329, 4091, 0, 3509, 4083, 2, 396, 10, 3, 3, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 77, 0, 1894, 0, 4, 162, 3438, 0, 701, 1, 3, 0, 4080, 1039, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16558, 15808, 16366, 16523, 17149, 16075, 16574, 16019] avg=16384.0000 max=17149 max_rank=4 min=15808 min_rank=1 imbalance_ratio=1.046692 max_load_reduction=113923 max_load_reduction_pct=0.869164 moved_to_echo=65817 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[6, 19, 0, 2977, 159, 9, 0, 0, 12, 0, 0, 2585, 5, 0, 3, 4086, 3226, 1193, 537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3364, 3, 0, 2710, 0, 4094, 4, 0, 308, 0, 3829, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 2500, 0, 937, 0, 0, 0, 8, 4049, 1, 3, 1934, 0, 2484, 7, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 3522, 21, 2, 2371, 0, 3800, 4073, 3, 0, 37, 3982, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3717, 13, 0, 7, 616, 37, 8, 3050, 15, 3964, 3547, 0, 7, 3, 0, 0, 724, 547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 116, 0, 4058, 3, 0, 0, 188, 0, 7, 11, 0, 1, 4089, 348, 0, 4090, 3539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 58, 4096, 4068, 7, 3507, 2252, 8, 39, 0, 8, 1, 845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 7, 250, 0, 2742, 0, 0, 1371, 1507, 0, 497, 0, 0, 0, 4085, 1039, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14817, 18303, 16039, 17852, 16255, 16818, 14896, 16092] avg=16384.0000 max=18303 max_rank=1 min=14817 min_rank=0 imbalance_ratio=1.117126 max_load_reduction=112769 max_load_reduction_pct=0.860359 moved_to_echo=64061 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4002, 173, 0, 0, 0, 0, 0, 0, 4031, 0, 1, 0, 4083, 3248, 1198, 538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3365, 0, 0, 1729, 20, 3854, 0, 0, 25, 0, 2353, 4022, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2495, 0, 348, 7, 0, 2, 0, 4082, 2, 2, 3373, 0, 2634, 0, 3937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 2568, 1, 0, 2367, 4, 3357, 4096, 0, 0, 35, 3444, 0, 0, 229, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 1, 149, 19, 0, 3135, 0, 4040, 3633, 0, 0, 2, 0, 0, 730, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 184, 47, 4089, 0, 0, 0, 20, 0, 0, 9, 0, 0, 4088, 28, 1, 4096, 3543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 136, 2897, 4095, 0, 3714, 4047, 0, 62, 6, 0, 0, 847, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4, 1, 129, 0, 1509, 0, 0, 182, 3919, 0, 1494, 0, 0, 0, 4088, 1047, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17274, 15523, 16883, 16141, 16266, 16182, 15805, 16998] avg=16384.0000 max=17274 max_rank=0 min=15523 min_rank=1 imbalance_ratio=1.054321 max_load_reduction=113798 max_load_reduction_pct=0.868210 moved_to_echo=65251 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[2, 7, 1, 3395, 73, 1, 0, 9, 4, 0, 22, 3461, 2, 2, 0, 4094, 3242, 1193, 540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3368, 2, 0, 2214, 0, 4063, 58, 1, 336, 2, 3890, 3958, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2506, 0, 587, 2, 0, 0, 3, 4086, 0, 2, 2977, 2, 1669, 4, 4043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3373, 0, 2, 2372, 0, 2373, 4094, 3, 3, 14, 3860, 0, 3, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4013, 0, 0, 4, 780, 8, 2, 2849, 3, 3735, 3035, 0, 2, 2, 2, 2, 725, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 102, 17, 4089, 0, 2, 3, 143, 0, 2, 0, 0, 3, 4094, 39, 0, 4093, 3554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 25, 3931, 4090, 2, 3756, 3906, 3, 148, 0, 2, 0, 849, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 2, 1515, 0, 916, 0, 0, 725, 3414, 1, 131, 0, 0, 2, 4091, 1041, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16048, 17908, 15881, 16108, 15710, 16248, 16716, 16453] avg=16384.0000 max=17908 max_rank=1 min=15710 min_rank=4 imbalance_ratio=1.093018 max_load_reduction=113164 max_load_reduction_pct=0.863373 moved_to_echo=65127 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 3907, 1316, 0, 0, 0, 1, 2, 0, 2290, 0, 0, 0, 4092, 3246, 1193, 539, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3370, 0, 2, 2952, 8, 4094, 1, 0, 489, 0, 3860, 3743, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2507, 1, 389, 0, 0, 0, 0, 4068, 0, 0, 2249, 0, 2456, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2632, 1, 0, 2374, 0, 2248, 4096, 0, 0, 45, 3811, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3925, 1, 0, 0, 2586, 0, 0, 2592, 0, 3075, 2031, 0, 0, 0, 0, 0, 726, 548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 645, 159, 0, 4096, 0, 0, 9, 104, 0, 0, 0, 0, 0, 4096, 58, 0, 4092, 3555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4040, 4096, 0, 4023, 3581, 0, 18, 1, 0, 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 506, 0, 2017, 0, 0, 1798, 1204, 0, 737, 0, 0, 0, 4095, 1041, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16592, 18553, 15771, 15236, 15484, 16814, 16610, 16012] avg=16384.0000 max=18553 max_rank=1 min=15236 min_rank=3 imbalance_ratio=1.132385 max_load_reduction=112519 max_load_reduction_pct=0.858452 moved_to_echo=64920 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 20096, 128, 4736, 128, 0, 128, 128, 32640, 128, 128, 21888, 128, 20352, 128, 31616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 27008, 128, 128, 15232, 128, 30976, 128, 128, 2816, 128, 23424, 31360, 512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 22784, 128, 128, 19072, 128, 25472, 32768, 128, 128, 640, 29696, 128, 128, 768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 128, 3200, 128, 14720, 128, 128, 6784, 22016, 128, 6528, 128, 128, 128, 32768, 8448, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[31360, 128, 0, 128, 8320, 128, 128, 23936, 128, 30976, 26368, 128, 128, 128, 128, 128, 5888, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 28416, 2944, 128, 128, 128, 128, 128, 128, 27136, 128, 128, 128, 32768, 25984, 9600, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 0, 1152, 29312, 32768, 128, 30464, 29952, 128, 896, 128, 128, 128, 6912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[1792, 1536, 128, 32768, 128, 128, 128, 512, 128, 128, 128, 128, 128, 32768, 512, 128, 32768, 28416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 1074, 0, 272, 0, 0, 11, 0, 4096, 3900, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 86, 4092, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 4, 0, 0, 0, 3941, 0, 1, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 2207, 4086, 406, 0, 0, 1540, 0, 4095, 98, 0, 0, 3531, 556, 0, 0, 4093, 4096, 0, 0, 4095, 238, 0, 0, 0, 0, 0, 4096, 1044, 0, 4096, 3771, 4096, 372, 4055, 0, 0, 4074, 1932, 0, 3546, 0, 2071, 1, 0, 0, 0, 0, 0, 61, 0, 2239, 0, 0, 4096, 0, 2977, 40, 0, 0, 0, 0, 0, 0, 899, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4088, 0, 4083, 139, 0, 0, 0, 7] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 733, 0, 1637, 0, 0, 2, 0, 4090, 3972, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 259, 4012, 4075, 0, 4096, 4091, 1, 0, 5, 4091, 0, 0, 2, 0, 4086, 34, 54, 4096, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 3735, 4066, 2279, 0, 0, 402, 0, 4096, 49, 0, 0, 3230, 322, 0, 0, 4096, 4096, 0, 0, 4092, 1620, 0, 0, 0, 1, 4, 4096, 1105, 0, 4096, 4039, 4096, 1149, 4016, 3, 0, 4096, 756, 0, 3245, 0, 940, 0, 0, 0, 0, 0, 0, 13, 1, 1069, 0, 0, 4047, 0, 1965, 23, 0, 0, 0, 3, 5, 0, 213, 4084, 0, 0, 25, 0, 0, 0, 6, 0, 4092, 0, 4085, 161, 0, 16, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 305, 1, 120, 0, 0, 4, 0, 4095, 3925, 1, 0, 0, 0, 4095, 1, 0, 0, 0, 67, 4090, 4091, 0, 4096, 4095, 0, 0, 0, 4095, 1, 0, 1, 0, 4086, 3, 0, 4095, 0, 0, 0, 1, 4096, 0, 12, 0, 0, 2359, 4093, 330, 0, 0, 1213, 1, 4096, 3, 0, 0, 3250, 706, 0, 0, 4078, 4095, 0, 0, 4096, 453, 0, 0, 0, 0, 0, 4095, 2402, 0, 4096, 3848, 4096, 273, 4084, 1, 0, 4081, 2859, 1, 3562, 0, 1769, 0, 0, 0, 0, 0, 0, 223, 1, 3178, 0, 0, 4094, 1, 1401, 30, 0, 0, 0, 1, 0, 0, 526, 4095, 0, 0, 1, 0, 0, 0, 8, 0, 4066, 0, 4085, 43, 0, 3, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 263, 0, 706, 0, 0, 30, 17, 4094, 3521, 4, 0, 0, 0, 4091, 6, 0, 0, 0, 36, 4032, 4075, 0, 4096, 4094, 0, 0, 0, 4088, 0, 0, 0, 1, 4052, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 3531, 4081, 2423, 0, 0, 957, 1, 4091, 218, 0, 0, 2839, 512, 0, 0, 4096, 4095, 0, 0, 4085, 1149, 1, 0, 0, 0, 8, 4096, 2569, 0, 4095, 4069, 4096, 2185, 4094, 5, 0, 4032, 1208, 2, 2478, 0, 344, 0, 3, 0, 0, 0, 0, 3, 2, 517, 0, 0, 4078, 0, 2875, 18, 0, 0, 0, 2, 8, 0, 176, 4077, 0, 0, 26, 0, 0, 0, 9, 0, 4051, 0, 4093, 333, 0, 24, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[3, 2, 739, 0, 5, 7, 7, 0, 5, 4081, 4066, 0, 7, 1, 0, 4087, 0, 8, 0, 0, 301, 4073, 4082, 7, 4087, 4089, 16, 0, 8, 4082, 12, 0, 0, 0, 4088, 0, 0, 4089, 31, 8, 5, 8, 4088, 9, 0, 7, 0, 2051, 4091, 56, 8, 4, 1492, 0, 4089, 167, 0, 5, 3117, 582, 5, 7, 4063, 4087, 3, 7, 4079, 72, 0, 6, 0, 0, 1, 4089, 712, 2, 4084, 3807, 4087, 350, 3737, 18, 7, 4076, 2847, 0, 3656, 8, 2842, 0, 0, 39, 8, 2, 1, 278, 0, 3721, 9, 6, 4017, 0, 1772, 5, 8, 0, 6, 0, 0, 9, 563, 4084, 8, 7, 15, 0, 0, 7, 0, 7, 4045, 1, 4027, 0, 0, 0, 0, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 460, 1, 277, 0, 0, 0, 0, 4090, 3724, 3, 0, 0, 0, 4095, 1, 0, 0, 0, 91, 4087, 4092, 0, 4093, 4092, 0, 0, 0, 4092, 19, 0, 0, 3, 4083, 1, 0, 4094, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 2976, 4082, 1039, 0, 1, 472, 3, 4095, 189, 0, 0, 3275, 345, 0, 0, 3923, 4093, 3, 0, 4092, 425, 5, 0, 0, 0, 0, 4096, 3251, 0, 4096, 4096, 4095, 1484, 4089, 0, 0, 4091, 2135, 2, 1609, 0, 955, 0, 0, 3, 0, 0, 0, 2, 0, 3144, 0, 0, 4092, 0, 2736, 8, 0, 0, 0, 3, 0, 0, 248, 4096, 0, 0, 5, 0, 0, 4, 4, 0, 4087, 0, 4081, 26, 0, 4, 0, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 287, 0, 29, 2, 3, 0, 0, 4086, 3140, 0, 3, 2, 0, 4092, 4, 3, 0, 2, 45, 4074, 4094, 3, 4093, 4091, 0, 0, 0, 4091, 10, 0, 12, 0, 4092, 14, 3, 4093, 2, 0, 3, 3, 4093, 3, 40, 3, 3, 2950, 4093, 122, 2, 0, 918, 0, 4089, 60, 0, 2, 3664, 457, 1, 3, 4011, 4093, 0, 3, 4085, 943, 0, 2, 0, 0, 0, 4093, 1267, 3, 4093, 4053, 4093, 829, 4050, 0, 3, 4078, 2972, 0, 3179, 3, 908, 0, 0, 0, 3, 0, 1, 363, 0, 3345, 0, 3, 4053, 1, 3205, 21, 3, 3, 2, 0, 0, 0, 140, 4075, 3, 3, 11, 2, 2, 0, 1, 3, 4060, 0, 4086, 25, 0, 18, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 532, 0, 8, 0, 0, 21, 4, 4091, 3162, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 71, 4070, 4095, 0, 4096, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 4, 2, 4093, 0, 0, 0, 0, 4096, 0, 26, 0, 0, 1860, 4094, 99, 0, 0, 1420, 0, 4096, 167, 0, 0, 3580, 14, 0, 0, 4033, 4096, 0, 0, 4096, 345, 0, 0, 0, 0, 0, 4096, 536, 0, 4096, 3451, 4096, 362, 3840, 7, 0, 4082, 3492, 0, 3982, 0, 2810, 0, 0, 1, 0, 0, 0, 766, 0, 3825, 0, 0, 4083, 0, 2406, 6, 0, 0, 0, 5, 2, 0, 159, 4094, 0, 0, 1, 0, 0, 0, 1, 0, 4092, 0, 4060, 36, 0, 33, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1074, 0, 272, 0, 0, 11, 0, 4096, 3900, 0, 0, 0, 0, 4096, 3494, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 4092, 4096, 0, 0, 4009, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 0, 1, 4096, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 2207, 751, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 406, 0, 0, 1540, 0, 2483, 98, 0, 0, 3531, 556, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 238, 0, 0, 0, 0, 0, 0, 1044, 0, 4096, 3771, 1162, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 4074, 1932, 0, 3546, 0, 2071, 1, 0, 0, 0, 0, 0, 61, 984, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2239, 0, 0, 4096, 0, 2977, 40, 0, 0, 0, 0, 0, 0, 899, 4096, 2183, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4088, 0, 4083, 139, 0, 0, 0, 7, 4096, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17026, 16383, 15721, 16793, 14778, 16811, 17132, 16428] avg=16384.0000 max=17132 max_rank=6 min=14778 min_rank=4 imbalance_ratio=1.045654 max_load_reduction=113940 max_load_reduction_pct=0.869293 moved_to_echo=65149 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 305, 1, 120, 0, 0, 4, 0, 4095, 3925, 1, 0, 0, 0, 4095, 3493, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 67, 4090, 4091, 0, 0, 4008, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4086, 3, 0, 4095, 0, 0, 0, 1, 4096, 0, 12, 0, 0, 2359, 751, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 330, 0, 0, 1213, 1, 2484, 3, 0, 0, 3250, 706, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 453, 0, 0, 0, 0, 0, 0, 2402, 0, 4096, 3848, 1162, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 1, 0, 4081, 2859, 1, 3562, 0, 1769, 0, 0, 0, 0, 0, 0, 223, 984, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3178, 0, 0, 4094, 1, 1401, 30, 0, 0, 0, 1, 0, 0, 526, 4095, 2183, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8, 0, 4066, 0, 4085, 43, 0, 3, 0, 0, 4095, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16122, 16353, 16032, 16158, 16330, 17651, 16112, 16314] avg=16384.0000 max=17651 max_rank=5 min=16032 min_rank=2 imbalance_ratio=1.077332 max_load_reduction=113421 max_load_reduction_pct=0.865334 moved_to_echo=66407 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 733, 0, 1637, 0, 0, 2, 0, 4090, 3972, 0, 0, 0, 0, 4096, 3494, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 4012, 4075, 0, 0, 4005, 1, 0, 5, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4086, 34, 54, 4096, 0, 0, 0, 0, 4096, 0, 11, 0, 0, 3735, 751, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 2279, 0, 0, 402, 0, 2484, 49, 0, 0, 3230, 322, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1620, 0, 0, 0, 1, 4, 0, 1105, 0, 4096, 4039, 1162, 1149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 3, 0, 4096, 756, 0, 3245, 0, 940, 0, 0, 0, 0, 0, 0, 13, 984, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1069, 0, 0, 4047, 0, 1965, 23, 0, 0, 0, 3, 5, 0, 213, 4084, 2183, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 6, 0, 4092, 0, 4085, 161, 0, 16, 0, 0, 4096, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18107, 16448, 17493, 16928, 17268, 14139, 14195, 16494] avg=16384.0000 max=18107 max_rank=0 min=14139 min_rank=5 imbalance_ratio=1.105164 max_load_reduction=112965 max_load_reduction_pct=0.861855 moved_to_echo=62096 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[3, 2, 739, 0, 5, 7, 7, 0, 5, 4081, 4066, 0, 7, 1, 0, 4087, 3487, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 301, 4073, 4082, 7, 0, 4003, 16, 0, 8, 4082, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4089, 31, 8, 5, 8, 4088, 9, 0, 7, 0, 2051, 749, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 56, 8, 4, 1492, 0, 2479, 167, 0, 5, 3117, 582, 5, 7, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7, 4079, 72, 0, 6, 0, 0, 1, 0, 712, 2, 4084, 3807, 1159, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3737, 18, 7, 4076, 2847, 0, 3656, 8, 2842, 0, 0, 39, 8, 2, 1, 278, 983, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3721, 9, 6, 4017, 0, 1772, 5, 8, 0, 6, 0, 0, 9, 563, 4084, 2179, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 7, 15, 0, 0, 7, 0, 7, 4045, 1, 4027, 0, 0, 0, 0, 5, 4089, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16580, 16592, 15760, 16076, 14282, 18588, 16979, 16215] avg=16384.0000 max=18588 max_rank=5 min=14282 min_rank=4 imbalance_ratio=1.134521 max_load_reduction=112484 max_load_reduction_pct=0.858185 moved_to_echo=66064 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 263, 0, 706, 0, 0, 30, 17, 4094, 3521, 4, 0, 0, 0, 4091, 3494, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 36, 4032, 4075, 0, 0, 4003, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4052, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 21, 0, 0, 3531, 754, 629, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 2423, 0, 0, 957, 1, 2475, 218, 0, 0, 2839, 512, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1149, 1, 0, 0, 0, 8, 0, 2569, 0, 4095, 4069, 1153, 2185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 5, 0, 4032, 1208, 2, 2478, 0, 344, 0, 3, 0, 0, 0, 0, 3, 987, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 517, 0, 0, 4078, 0, 2875, 18, 0, 0, 0, 2, 8, 0, 176, 4077, 2189, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 9, 0, 4051, 0, 4093, 333, 0, 24, 0, 0, 4096, 4012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16304, 16240, 17178, 17602, 19314, 13247, 14543, 16644] avg=16384.0000 max=19314 max_rank=4 min=13247 min_rank=5 imbalance_ratio=1.178833 max_load_reduction=111758 max_load_reduction_pct=0.852646 moved_to_echo=63748 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 460, 1, 277, 0, 0, 0, 0, 4090, 3724, 3, 0, 0, 0, 4095, 3491, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 91, 4087, 4092, 0, 0, 4006, 0, 0, 0, 4092, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4083, 1, 0, 4094, 0, 0, 0, 0, 4096, 0, 9, 0, 0, 2976, 751, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 1039, 0, 1, 472, 3, 2483, 189, 0, 0, 3275, 345, 0, 0, 3923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4092, 425, 5, 0, 0, 0, 0, 0, 3251, 0, 4096, 4096, 1161, 1484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4091, 2135, 2, 1609, 0, 955, 0, 0, 3, 0, 0, 0, 2, 984, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3144, 0, 0, 4092, 0, 2736, 8, 0, 0, 0, 3, 0, 0, 248, 4096, 2183, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4, 4, 0, 4087, 0, 4081, 26, 0, 4, 0, 9, 4096, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16223, 16388, 16641, 15812, 18613, 13956, 17112, 16327] avg=16384.0000 max=18613 max_rank=4 min=13956 min_rank=5 imbalance_ratio=1.136047 max_load_reduction=112459 max_load_reduction_pct=0.857994 moved_to_echo=66008 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 287, 0, 29, 2, 3, 0, 0, 4086, 3140, 0, 3, 2, 0, 4092, 3492, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 2, 45, 4074, 4094, 3, 0, 4005, 0, 0, 0, 4091, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4092, 14, 3, 4093, 2, 0, 3, 3, 4093, 3, 40, 3, 3, 2950, 750, 627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 122, 2, 0, 918, 0, 2479, 60, 0, 2, 3664, 457, 1, 3, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4085, 943, 0, 2, 0, 0, 0, 0, 1267, 3, 4093, 4053, 1161, 829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4050, 0, 3, 4078, 2972, 0, 3179, 3, 908, 0, 0, 0, 3, 0, 1, 363, 983, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3345, 0, 3, 4053, 1, 3205, 21, 3, 3, 2, 0, 0, 0, 140, 4075, 2182, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 11, 2, 2, 0, 1, 3, 4060, 0, 4086, 25, 0, 18, 0, 0, 4093, 4010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15219, 16331, 16691, 15812, 16439, 16629, 17634, 16317] avg=16384.0000 max=17634 max_rank=6 min=15219 min_rank=0 imbalance_ratio=1.076294 max_load_reduction=113438 max_load_reduction_pct=0.865463 moved_to_echo=67019 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 532, 0, 8, 0, 0, 21, 4, 4091, 3162, 0, 0, 0, 0, 4096, 3494, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 4070, 4095, 0, 0, 4009, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4, 2, 4093, 0, 0, 0, 0, 4096, 0, 26, 0, 0, 1860, 751, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 99, 0, 0, 1420, 0, 2484, 167, 0, 0, 3580, 14, 0, 0, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 345, 0, 0, 0, 0, 0, 0, 536, 0, 4096, 3451, 1162, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3840, 7, 0, 4082, 3492, 0, 3982, 0, 2810, 0, 0, 1, 0, 0, 0, 766, 984, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3825, 0, 0, 4083, 0, 2406, 6, 0, 0, 0, 5, 2, 0, 159, 4094, 2183, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 4092, 0, 4060, 36, 0, 33, 0, 1, 4096, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15491, 16337, 15556, 15891, 14048, 20051, 17365, 16333] avg=16384.0000 max=20051 max_rank=5 min=14048 min_rank=4 imbalance_ratio=1.223816 max_load_reduction=111021 max_load_reduction_pct=0.847023 moved_to_echo=67797 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 21120, 128, 128, 32640, 128, 19456, 256, 128, 128, 128, 128, 128, 128, 2944, 32768, 17536, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 32640, 128, 128, 32768, 128, 128, 128, 128, 32768, 128, 128, 128, 128, 21760, 6016, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 1024, 32640, 32768, 128, 0, 32128, 128, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 6784, 128, 128, 8448, 128, 19968, 1024, 0, 128, 26496, 3584, 128, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 128, 128, 128, 128, 32640, 128, 32640, 768, 0, 128, 0, 128, 32768, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 4480, 128, 3072, 128, 128, 128, 128, 32768, 29440, 128, 128, 128, 0, 32768, 28032, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32000, 128, 128, 32640, 18304, 128, 25344, 128, 12672, 128, 128, 128, 128, 128, 128, 1792, 7936, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 5248, 128, 128, 0, 128, 128, 0, 12928, 128, 32768, 31232, 9344, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3493, 4094, 1, 0, 0, 3260, 4096, 0, 0, 4084, 4, 0, 0, 5, 0, 3, 0, 0, 1902, 3832, 0, 1839, 1, 0, 0, 4094, 0, 0, 4056, 4089, 1, 659, 0, 0, 4095, 1, 422, 4096, 1, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1230, 1, 1, 0, 0, 0, 10, 4056, 262, 0, 4096, 0, 0, 0, 0, 4080, 0, 0, 4087, 2418, 0, 0, 0, 4094, 1, 68, 13, 4032, 0, 4095, 4095, 0, 0, 4090, 0, 7, 0, 0, 21, 0, 3804, 1659, 4092, 0, 887, 4093, 0, 0, 3747, 0, 2, 0, 0, 4096, 3853, 0, 0, 0, 27, 108, 0, 0, 0, 21, 0, 0, 0, 0, 0, 4095, 0, 3511, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3590, 4072, 0, 0, 5, 2934, 4096, 0, 0, 4059, 0, 0, 1, 16, 0, 0, 0, 0, 1528, 3424, 0, 2627, 0, 0, 0, 4096, 0, 0, 4086, 4056, 0, 266, 0, 0, 4095, 0, 85, 4095, 0, 2, 0, 3, 0, 0, 4096, 4096, 0, 0, 0, 1639, 2, 0, 0, 0, 0, 10, 4013, 1744, 0, 4096, 0, 1, 0, 0, 4081, 0, 0, 4091, 2326, 0, 0, 0, 4094, 0, 56, 10, 3940, 0, 4069, 4096, 0, 0, 4096, 0, 0, 0, 1, 22, 0, 2584, 1768, 4095, 0, 958, 4011, 0, 0, 4037, 0, 1, 0, 0, 4096, 3766, 0, 0, 0, 27, 15, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 3902, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3855, 4082, 2, 0, 10, 3778, 4096, 0, 0, 4065, 14, 0, 0, 2, 0, 0, 0, 0, 1957, 3549, 0, 1175, 7, 0, 0, 4094, 0, 3, 3756, 4095, 3, 130, 0, 0, 4089, 4, 1494, 4096, 8, 4, 0, 0, 12, 0, 4096, 4096, 0, 0, 0, 1138, 0, 1, 0, 0, 0, 15, 4087, 420, 0, 4096, 0, 1, 0, 4, 4087, 0, 0, 4095, 3422, 0, 0, 0, 4087, 2, 65, 2, 3936, 0, 4072, 4092, 0, 0, 3980, 0, 6, 0, 0, 11, 0, 3486, 1127, 4084, 0, 1551, 4088, 0, 0, 4044, 0, 6, 0, 0, 4096, 3476, 0, 0, 0, 59, 4, 0, 0, 0, 81, 0, 0, 0, 0, 0, 4093, 0, 2686, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3903, 4065, 5, 0, 4, 3021, 4096, 0, 0, 4048, 20, 0, 0, 0, 0, 7, 0, 0, 2948, 3410, 0, 367, 25, 0, 0, 4087, 0, 0, 3963, 4095, 1, 78, 0, 0, 4085, 3, 324, 4096, 12, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2999, 0, 1, 0, 0, 0, 20, 4086, 1246, 0, 4096, 0, 0, 0, 1, 4072, 0, 0, 4091, 2650, 0, 0, 0, 4094, 3, 198, 11, 3873, 0, 4066, 4094, 1, 0, 4065, 0, 4, 0, 0, 42, 0, 4034, 1967, 4077, 0, 654, 4057, 0, 0, 4026, 0, 7, 0, 0, 4096, 3012, 0, 0, 0, 54, 68, 0, 0, 0, 30, 0, 0, 0, 0, 0, 4093, 0, 2329, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[3, 0, 3, 3, 3891, 4054, 1, 0, 0, 2198, 4093, 0, 0, 4039, 2, 3, 1, 0, 3, 0, 0, 3, 1430, 3168, 0, 2174, 45, 3, 3, 4089, 0, 0, 3985, 4091, 0, 1568, 3, 1, 4091, 0, 348, 4093, 1, 2, 0, 2, 5, 3, 4093, 4093, 0, 3, 1, 1229, 0, 0, 0, 3, 3, 13, 4059, 208, 0, 4093, 2, 0, 3, 12, 4093, 3, 2, 4073, 1605, 3, 3, 3, 4072, 0, 0, 3, 3979, 2, 4087, 4093, 0, 2, 4048, 0, 0, 0, 1, 36, 0, 3830, 3389, 4078, 3, 1963, 4008, 0, 3, 3229, 3, 0, 2, 3, 4093, 3591, 3, 0, 0, 24, 80, 3, 0, 3, 2, 0, 0, 3, 0, 0, 4093, 0, 3336, 3, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3302, 4088, 5, 0, 0, 3449, 4096, 0, 0, 3921, 36, 0, 0, 0, 0, 6, 0, 0, 3270, 2140, 0, 2409, 6, 0, 0, 4087, 0, 0, 3735, 3995, 0, 105, 0, 0, 4091, 3, 224, 4096, 10, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 1163, 0, 0, 0, 0, 0, 91, 4068, 2221, 0, 4096, 0, 0, 0, 9, 4036, 0, 0, 4092, 3572, 0, 0, 0, 4061, 0, 28, 33, 3947, 0, 4052, 4095, 0, 0, 3996, 0, 11, 0, 0, 149, 0, 3975, 1736, 4085, 0, 1151, 3960, 0, 0, 4086, 0, 9, 0, 0, 4096, 1983, 0, 0, 0, 95, 118, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4092, 0, 2679, 0, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[8, 1, 10, 8, 2020, 4088, 0, 4, 0, 3027, 4087, 0, 2, 4069, 0, 7, 12, 5, 8, 0, 9, 2, 1315, 3920, 0, 3390, 77, 9, 6, 4082, 0, 0, 4074, 3969, 0, 388, 10, 0, 4087, 0, 149, 4087, 1, 1, 15, 0, 14, 7, 4086, 4088, 0, 8, 0, 1217, 0, 0, 0, 9, 3, 0, 4027, 67, 1, 4084, 11, 0, 3, 25, 4086, 8, 0, 4088, 2552, 9, 9, 8, 4081, 0, 0, 0, 4059, 2, 4096, 4087, 0, 1, 4095, 0, 0, 1, 0, 0, 0, 3335, 2550, 4061, 7, 1621, 3866, 0, 8, 3732, 0, 0, 6, 9, 4085, 3863, 8, 7, 3, 12, 0, 8, 0, 2, 205, 0, 0, 8, 0, 8, 4087, 0, 3794, 8, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3970, 4096, 2, 0, 0, 3474, 4096, 0, 0, 4088, 5, 0, 0, 0, 0, 0, 0, 0, 1741, 4012, 0, 3439, 47, 0, 0, 4095, 0, 0, 3968, 4062, 0, 154, 0, 0, 4087, 0, 383, 4096, 1, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 2, 851, 0, 0, 0, 0, 0, 20, 4077, 231, 0, 4096, 0, 1, 0, 0, 4065, 0, 0, 4095, 3024, 0, 0, 0, 4091, 0, 2, 14, 4070, 0, 4080, 4096, 0, 0, 4044, 0, 0, 0, 0, 31, 0, 2170, 1924, 4070, 0, 315, 4074, 0, 0, 2969, 0, 0, 0, 0, 4095, 4044, 0, 0, 0, 18, 93, 0, 0, 0, 227, 0, 0, 0, 0, 0, 4096, 0, 4077, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3590, 4072, 0, 0, 5, 2934, 1592, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 16, 0, 0, 0, 0, 1528, 3424, 0, 2627, 0, 0, 0, 4096, 0, 0, 2219, 1816, 597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4056, 0, 266, 0, 0, 4095, 0, 85, 3421, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1877, 0, 0, 0, 1639, 2, 0, 0, 0, 0, 10, 4013, 1744, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4081, 0, 0, 4091, 2326, 0, 0, 0, 4094, 0, 56, 10, 674, 585, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3940, 0, 2887, 0, 0, 0, 4096, 0, 0, 0, 1, 22, 0, 2584, 1768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 958, 4011, 0, 0, 4037, 0, 1, 0, 0, 3996, 3766, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 3902, 0, 0, 0, 4096, 4095, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16252, 16324, 16014, 17477, 16018, 15298, 16796, 16893] avg=16384.0000 max=17477 max_rank=3 min=15298 min_rank=5 imbalance_ratio=1.066711 max_load_reduction=113595 max_load_reduction_pct=0.866661 moved_to_echo=65005 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3493, 4094, 1, 0, 0, 3260, 1592, 0, 0, 4084, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 3, 0, 0, 1902, 3832, 0, 1839, 1, 0, 0, 4094, 0, 0, 2219, 1816, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4056, 4089, 1, 659, 0, 0, 4095, 1, 422, 3422, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1877, 0, 0, 0, 1230, 1, 1, 0, 0, 0, 10, 4056, 262, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 0, 4087, 2418, 0, 0, 0, 4094, 1, 68, 13, 674, 589, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4032, 0, 2905, 0, 0, 0, 4090, 0, 7, 0, 0, 21, 0, 3804, 1659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 887, 4093, 0, 0, 3747, 0, 2, 0, 0, 3996, 3853, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 21, 0, 0, 0, 0, 0, 4095, 0, 3511, 0, 0, 0, 4095, 4092, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16528, 16312, 16746, 15629, 16124, 16518, 16605, 16610] avg=16384.0000 max=16746 max_rank=2 min=15629 min_rank=3 imbalance_ratio=1.022095 max_load_reduction=114326 max_load_reduction_pct=0.872238 moved_to_echo=65857 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3302, 4088, 5, 0, 0, 3449, 1592, 0, 0, 3921, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 3270, 2140, 0, 2409, 6, 0, 0, 4087, 0, 0, 2219, 1816, 595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3735, 3995, 0, 105, 0, 0, 4091, 3, 224, 3422, 10, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1877, 0, 0, 0, 1163, 0, 0, 0, 0, 0, 91, 4068, 2221, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4036, 0, 0, 4092, 3572, 0, 0, 0, 4061, 0, 28, 33, 674, 583, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 0, 2874, 0, 0, 0, 3996, 0, 11, 0, 0, 149, 0, 3975, 1736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1151, 3960, 0, 0, 4086, 0, 9, 0, 0, 3996, 1983, 0, 0, 0, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 17, 0, 0, 0, 0, 0, 4092, 0, 2679, 0, 3, 0, 4095, 4085, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16393, 16548, 15586, 17612, 17188, 16688, 15280, 15777] avg=16384.0000 max=17612 max_rank=3 min=15280 min_rank=6 imbalance_ratio=1.074951 max_load_reduction=113460 max_load_reduction_pct=0.865631 moved_to_echo=64933 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3903, 4065, 5, 0, 4, 3021, 1587, 0, 0, 4048, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2948, 3410, 0, 367, 25, 0, 0, 4087, 0, 0, 2222, 1820, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 4095, 1, 78, 0, 0, 4085, 3, 324, 3419, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1874, 0, 0, 0, 2999, 0, 1, 0, 0, 0, 20, 4086, 1246, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4072, 0, 0, 4091, 2650, 0, 0, 0, 4094, 3, 198, 11, 677, 587, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3873, 0, 2879, 0, 1, 0, 4065, 0, 4, 0, 0, 42, 0, 4034, 1967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 654, 4057, 0, 0, 4026, 0, 7, 0, 0, 3994, 3012, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 30, 0, 0, 0, 0, 0, 4093, 0, 2329, 0, 0, 0, 4094, 4077, 689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16653, 15486, 15980, 18418, 16486, 16865, 15804, 15380] avg=16384.0000 max=18418 max_rank=3 min=15380 min_rank=7 imbalance_ratio=1.124146 max_load_reduction=112654 max_load_reduction_pct=0.859482 moved_to_echo=64535 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3855, 4082, 2, 0, 10, 3778, 1592, 0, 0, 4065, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1957, 3549, 0, 1175, 7, 0, 0, 4094, 0, 3, 2219, 1816, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3756, 4095, 3, 130, 0, 0, 4089, 4, 1494, 3422, 8, 4, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1877, 0, 0, 0, 1138, 0, 1, 0, 0, 0, 15, 4087, 420, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 4087, 0, 0, 4095, 3422, 0, 0, 0, 4087, 2, 65, 2, 674, 585, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 2889, 0, 0, 0, 3980, 0, 6, 0, 0, 11, 0, 3486, 1127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1551, 4088, 0, 0, 4044, 0, 6, 0, 0, 3996, 3476, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 81, 0, 0, 0, 0, 0, 4093, 0, 2686, 0, 0, 0, 4092, 4084, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17398, 15420, 17017, 15730, 17124, 15435, 17220, 15728] avg=16384.0000 max=17398 max_rank=0 min=15420 min_rank=1 imbalance_ratio=1.061890 max_load_reduction=113674 max_load_reduction_pct=0.867264 moved_to_echo=65507 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[8, 1, 10, 8, 2020, 4088, 0, 4, 0, 3027, 1589, 0, 2, 4069, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 5, 8, 0, 9, 2, 1315, 3920, 0, 3390, 77, 9, 6, 4082, 0, 0, 2215, 1812, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 3969, 0, 388, 10, 0, 4087, 0, 149, 3415, 1, 1, 15, 0, 14, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1873, 0, 8, 0, 1217, 0, 0, 0, 9, 3, 0, 4027, 67, 1, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 3, 25, 4086, 8, 0, 4088, 2552, 9, 9, 8, 4081, 0, 0, 0, 672, 589, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 2, 2906, 0, 0, 1, 4095, 0, 0, 1, 0, 0, 0, 3335, 2550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1621, 3866, 0, 8, 3732, 0, 0, 6, 9, 3985, 3863, 8, 7, 3, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 205, 0, 0, 8, 0, 8, 4087, 0, 3794, 8, 0, 0, 4087, 4061, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14833, 17463, 16130, 15375, 16241, 16949, 17127, 16954] avg=16384.0000 max=17463 max_rank=1 min=14833 min_rank=0 imbalance_ratio=1.065857 max_load_reduction=113609 max_load_reduction_pct=0.866768 moved_to_echo=67271 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 3, 3, 3891, 4054, 1, 0, 0, 2198, 1591, 0, 0, 4039, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 3, 1430, 3168, 0, 2174, 45, 3, 3, 4089, 0, 0, 2217, 1815, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3985, 4091, 0, 1568, 3, 1, 4091, 0, 348, 3420, 1, 2, 0, 2, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1876, 0, 3, 1, 1229, 0, 0, 0, 3, 3, 13, 4059, 208, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 12, 4093, 3, 2, 4073, 1605, 3, 3, 3, 4072, 0, 0, 3, 673, 588, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3979, 2, 2899, 0, 0, 2, 4048, 0, 0, 0, 1, 36, 0, 3830, 3389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1963, 4008, 0, 3, 3229, 3, 0, 2, 3, 3993, 3591, 3, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 3, 0, 3, 2, 0, 0, 3, 0, 0, 4093, 0, 3336, 3, 0, 2, 4093, 4078, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15788, 15551, 17520, 15581, 15238, 18186, 16825, 16383] avg=16384.0000 max=18186 max_rank=5 min=15238 min_rank=4 imbalance_ratio=1.109985 max_load_reduction=112886 max_load_reduction_pct=0.861252 moved_to_echo=66632 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3970, 4096, 2, 0, 0, 3474, 1592, 0, 0, 4088, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1741, 4012, 0, 3439, 47, 0, 0, 4095, 0, 0, 2219, 1816, 599, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 4062, 0, 154, 0, 0, 4087, 0, 383, 3422, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1877, 0, 0, 2, 851, 0, 0, 0, 0, 0, 20, 4077, 231, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4065, 0, 0, 4095, 3024, 0, 0, 0, 4091, 0, 2, 14, 674, 587, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 2894, 0, 0, 0, 4044, 0, 0, 0, 0, 31, 0, 2170, 1924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 4074, 0, 0, 2969, 0, 0, 0, 0, 3995, 4044, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 227, 0, 0, 0, 0, 0, 4096, 0, 4077, 0, 0, 0, 4096, 4070, 688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17227, 17968, 16079, 15250, 16653, 15133, 15415, 17347] avg=16384.0000 max=17968 max_rank=1 min=15133 min_rank=5 imbalance_ratio=1.096680 max_load_reduction=113104 max_load_reduction_pct=0.862915 moved_to_echo=64548 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 28032, 32640, 128, 128, 128, 25216, 12800, 0, 128, 32384, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 15104, 0, 128, 128, 11520, 128, 128, 0, 128, 128, 256, 32512, 6400, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[31744, 32512, 128, 3456, 128, 128, 32768, 128, 3456, 27392, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 9216, 32256, 0, 128, 29952, 128, 128, 128, 128, 32000, 27648, 128, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[512, 128, 0, 128, 640, 128, 0, 128, 0, 128, 32768, 0, 26368, 128, 128, 128, 32768, 32768, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[31872, 128, 23168, 0, 128, 128, 32512, 0, 128, 128, 128, 384, 0, 27264, 16128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 128, 128, 32640, 128, 128, 32768, 21632, 128, 128, 128, 32768, 128, 512, 128, 5504, 4736, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 128, 128, 128, 16128, 27520, 0, 17536, 256, 128, 128, 32768, 0, 128, 17792, 14592, 4864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[2, 0, 4087, 0, 0, 2, 3119, 0, 1, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3916, 4096, 4096, 0, 4096, 0, 4096, 728, 2, 0, 0, 0, 2317, 4005, 0, 0, 4096, 4096, 506, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3106, 4096, 0, 926, 4095, 3885, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4089, 0, 28, 0, 0, 0, 4096, 0, 0, 4086, 0, 0, 172, 0, 0, 4096, 33, 0, 0, 0, 4096, 4096, 0, 0, 0, 3502, 0, 0, 4096, 0, 0, 1, 3510, 3, 3712, 0, 1, 0, 0, 0, 8, 0, 0, 3400, 0, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4086, 0, 0, 0, 2216, 0, 1, 0, 4096, 6, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3671, 4096, 4094, 0, 4096, 0, 4095, 685, 6, 0, 0, 1, 3121, 4026, 0, 0, 4096, 4096, 869, 4094, 0, 7, 0, 0, 0, 2, 4095, 0, 0, 0, 4096, 0, 0, 0, 1721, 4096, 1, 2068, 4096, 3003, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4089, 0, 16, 0, 0, 0, 4096, 0, 1, 4092, 0, 0, 28, 0, 0, 4096, 1, 0, 0, 1, 4095, 4095, 0, 1, 0, 3844, 0, 0, 4096, 0, 0, 17, 4055, 4, 3916, 0, 0, 1, 4, 0, 8, 0, 0, 3583, 3, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 2, 4049, 2, 0, 4, 1511, 11, 0, 8, 4087, 0, 0, 0, 8, 0, 1, 9, 0, 6, 0, 0, 7, 8, 0, 4085, 7, 7, 7, 4088, 4085, 4088, 0, 4087, 8, 4087, 1747, 49, 0, 1, 4, 2557, 4080, 8, 0, 4084, 4086, 1663, 4082, 9, 0, 2, 0, 0, 26, 4088, 10, 8, 0, 4087, 0, 0, 187, 1719, 4086, 0, 470, 4086, 2372, 7, 0, 3, 14, 0, 0, 9, 0, 26, 8, 3929, 0, 11, 9, 8, 7, 4075, 9, 0, 4088, 0, 9, 6, 9, 0, 4085, 0, 3, 9, 56, 4086, 4084, 7, 12, 0, 4049, 7, 10, 4066, 0, 0, 0, 4092, 128, 4064, 0, 8, 0, 86, 0, 0, 0, 0, 4075, 0, 4087, 6, 0, 4052] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[18, 0, 4062, 0, 0, 0, 2238, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 4014, 4096, 4089, 0, 4096, 0, 4096, 317, 15, 0, 0, 3, 3998, 4037, 0, 0, 4096, 4096, 1714, 4087, 0, 22, 0, 0, 0, 54, 4096, 0, 0, 0, 4096, 0, 0, 33, 3500, 4096, 0, 1084, 4095, 3393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 9, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 581, 0, 0, 4096, 242, 0, 0, 1, 4095, 4093, 0, 0, 0, 3955, 0, 0, 4096, 0, 0, 60, 3108, 120, 2724, 0, 0, 0, 16, 0, 5, 0, 0, 1616, 35, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 3, 4052, 0, 2, 0, 2781, 2, 0, 2, 4093, 10, 2, 0, 3, 0, 0, 3, 8, 3, 1, 0, 3, 3, 0, 4075, 0, 3, 3, 4062, 4093, 4091, 2, 4093, 3, 4093, 591, 6, 0, 1, 5, 3238, 3976, 1, 1, 4093, 4093, 863, 4093, 3, 0, 1, 0, 0, 23, 4093, 3, 3, 0, 4093, 0, 0, 14, 1886, 4093, 0, 189, 4085, 3041, 2, 0, 2, 0, 0, 0, 3, 0, 1, 3, 4092, 0, 0, 3, 3, 3, 4092, 3, 0, 4093, 2, 3, 42, 3, 0, 4092, 0, 0, 3, 24, 4093, 4093, 3, 16, 0, 4060, 1, 3, 4091, 3, 0, 4, 4044, 102, 3840, 0, 0, 0, 90, 0, 0, 0, 0, 4001, 0, 4093, 2, 0, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4077, 0, 0, 0, 3291, 0, 0, 0, 4096, 24, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 3742, 4096, 4085, 0, 4096, 0, 4096, 2566, 0, 0, 0, 2, 3209, 4063, 0, 0, 4096, 4096, 1427, 4091, 0, 6, 0, 0, 0, 29, 4095, 0, 0, 0, 4096, 0, 0, 1, 3234, 4096, 2, 1174, 4096, 3246, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 31, 0, 0, 0, 4096, 0, 2, 4075, 0, 0, 392, 0, 0, 4096, 32, 0, 0, 17, 4094, 4094, 0, 0, 0, 2636, 0, 0, 4096, 0, 0, 32, 3660, 30, 3006, 0, 0, 0, 43, 0, 23, 0, 0, 971, 21, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[26, 0, 4083, 0, 0, 0, 1446, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 13, 0, 3, 0, 0, 0, 0, 4091, 0, 0, 0, 3939, 4096, 4090, 0, 4096, 0, 4093, 1010, 0, 0, 0, 4, 3183, 4079, 0, 0, 4096, 4096, 1670, 4091, 0, 9, 0, 0, 0, 9, 4093, 0, 0, 0, 4096, 0, 0, 9, 2369, 4096, 2, 1427, 4096, 3164, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4059, 0, 16, 0, 0, 0, 4096, 0, 2, 4084, 0, 0, 37, 0, 1, 4096, 133, 0, 0, 0, 4096, 4092, 0, 0, 0, 3455, 0, 0, 4096, 0, 1, 5, 3716, 49, 3783, 0, 0, 0, 4, 0, 6, 0, 0, 3368, 12, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 0, 1666, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 24, 0, 1, 0, 0, 0, 0, 4088, 0, 0, 0, 3742, 4096, 4096, 0, 4096, 0, 4096, 998, 3, 0, 0, 0, 2722, 3971, 0, 0, 4096, 4096, 730, 4092, 0, 25, 0, 0, 0, 49, 4096, 0, 0, 0, 4096, 0, 0, 3, 3198, 4096, 0, 13, 4095, 3738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 3, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 62, 0, 0, 4096, 1, 0, 0, 8, 4096, 4096, 0, 1, 0, 3843, 0, 0, 4096, 0, 0, 36, 4069, 48, 3985, 0, 0, 0, 19, 0, 3, 0, 0, 3927, 8, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 4087, 0, 0, 2, 3119, 0, 1, 0, 4096, 0, 0, 0, 0, 1, 4096, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3916, 4096, 4096, 164, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 728, 2, 0, 0, 0, 2317, 4005, 0, 0, 2944, 0, 506, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3106, 1445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 926, 4095, 3885, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 4096, 0, 0, 4086, 0, 0, 172, 0, 0, 4096, 33, 2277, 1589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 709, 4096, 0, 0, 0, 3502, 0, 0, 4096, 0, 0, 1, 3510, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3712, 0, 1, 0, 0, 0, 8, 0, 0, 3400, 0, 4096, 0, 0, 4096, 988, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17223, 16397, 14598, 16840, 17066, 16377, 15914, 16657] avg=16384.0000 max=17223 max_rank=0 min=14598 min_rank=2 imbalance_ratio=1.051208 max_load_reduction=113849 max_load_reduction_pct=0.868599 moved_to_echo=66014 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4086, 0, 0, 0, 2216, 0, 1, 0, 4096, 6, 0, 0, 0, 1, 4096, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 3671, 4096, 4094, 164, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 685, 6, 0, 0, 1, 3121, 4026, 0, 0, 2944, 0, 869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 7, 0, 0, 0, 2, 4095, 0, 0, 0, 4096, 0, 0, 0, 1721, 1445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 1, 2068, 4096, 3003, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 1, 4092, 0, 0, 28, 0, 0, 4096, 1, 2277, 1589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 708, 4095, 0, 1, 0, 3844, 0, 0, 4096, 0, 0, 17, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3916, 0, 0, 1, 4, 0, 8, 0, 0, 3583, 3, 4096, 0, 0, 4096, 988, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16321, 16151, 15747, 15460, 17328, 16196, 16817, 17052] avg=16384.0000 max=17328 max_rank=4 min=15460 min_rank=3 imbalance_ratio=1.057617 max_load_reduction=113744 max_load_reduction_pct=0.867798 moved_to_echo=67393 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 4049, 2, 0, 4, 1511, 11, 0, 8, 4087, 0, 0, 0, 8, 0, 4087, 1814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 0, 6, 0, 0, 7, 8, 0, 4085, 7, 7, 7, 4088, 4085, 4088, 164, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4087, 1747, 49, 0, 1, 4, 2557, 4080, 8, 0, 2935, 0, 1663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 9, 0, 2, 0, 0, 26, 4088, 10, 8, 0, 4087, 0, 0, 187, 1719, 1441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 470, 4086, 2372, 7, 0, 3, 14, 0, 0, 9, 0, 26, 8, 3929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 9, 8, 7, 4075, 9, 0, 4088, 0, 9, 6, 9, 0, 4085, 0, 2272, 1585, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 9, 56, 708, 4084, 7, 12, 0, 4049, 7, 10, 4066, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 4064, 0, 8, 0, 86, 0, 0, 0, 0, 4075, 0, 4087, 6, 0, 4052, 985, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15583, 16591, 17139, 15659, 14981, 16173, 17103, 17843] avg=16384.0000 max=17843 max_rank=7 min=14981 min_rank=4 imbalance_ratio=1.089050 max_load_reduction=113229 max_load_reduction_pct=0.863869 moved_to_echo=66100 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[18, 0, 4062, 0, 0, 0, 2238, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 4096, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 4014, 4096, 4089, 164, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 317, 15, 0, 0, 3, 3998, 4037, 0, 0, 2944, 0, 1714, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 22, 0, 0, 0, 54, 4096, 0, 0, 0, 4096, 0, 0, 33, 3500, 1445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 1084, 4095, 3393, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 581, 0, 0, 4096, 242, 2277, 1589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 708, 4093, 0, 0, 0, 3955, 0, 0, 4096, 0, 0, 60, 3108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 2724, 0, 0, 0, 16, 0, 5, 0, 0, 1616, 35, 4096, 0, 0, 4096, 988, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16341, 16490, 17124, 17333, 16733, 16981, 16021, 14049] avg=16384.0000 max=17333 max_rank=3 min=14049 min_rank=7 imbalance_ratio=1.057922 max_load_reduction=113739 max_load_reduction_pct=0.867760 moved_to_echo=63784 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4077, 0, 0, 0, 3291, 0, 0, 0, 4096, 24, 0, 0, 0, 0, 4096, 1820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 3742, 4096, 4085, 171, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2566, 0, 0, 0, 2, 3209, 4063, 0, 0, 2932, 0, 1427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 6, 0, 0, 0, 29, 4095, 0, 0, 0, 4096, 0, 0, 1, 3234, 1447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 2, 1174, 4096, 3246, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 4096, 0, 2, 4075, 0, 0, 392, 0, 0, 4096, 32, 2276, 1591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 697, 4094, 0, 0, 0, 2636, 0, 0, 4096, 0, 0, 32, 3660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 3006, 0, 0, 0, 43, 0, 23, 0, 0, 971, 21, 4096, 0, 0, 4096, 993, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17404, 16237, 18295, 16999, 16676, 16591, 15232, 13638] avg=16384.0000 max=18295 max_rank=2 min=13638 min_rank=7 imbalance_ratio=1.116638 max_load_reduction=112777 max_load_reduction_pct=0.860420 moved_to_echo=62137 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[26, 0, 4083, 0, 0, 0, 1446, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 3, 0, 0, 0, 0, 4091, 0, 0, 0, 3939, 4096, 4090, 164, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1010, 0, 0, 0, 4, 3183, 4079, 0, 0, 2944, 0, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 9, 0, 0, 0, 9, 4093, 0, 0, 0, 4096, 0, 0, 9, 2369, 1445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 2, 1427, 4096, 3164, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4096, 0, 2, 4084, 0, 0, 37, 0, 1, 4096, 133, 2277, 1589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 709, 4092, 0, 0, 0, 3455, 0, 0, 4096, 0, 1, 5, 3716, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 3783, 0, 0, 0, 4, 0, 6, 0, 0, 3368, 12, 4096, 0, 0, 4096, 988, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15567, 16425, 16983, 16121, 16816, 16331, 16074, 16755] avg=16384.0000 max=16983 max_rank=2 min=15567 min_rank=0 imbalance_ratio=1.036560 max_load_reduction=114089 max_load_reduction_pct=0.870430 moved_to_echo=65976 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 4052, 0, 2, 0, 2781, 2, 0, 2, 4093, 10, 2, 0, 3, 0, 4093, 1818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 3, 1, 0, 3, 3, 0, 4075, 0, 3, 3, 4062, 4093, 4091, 164, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 4093, 591, 6, 0, 1, 5, 3238, 3976, 1, 1, 2942, 0, 863, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 1, 0, 0, 23, 4093, 3, 3, 0, 4093, 0, 0, 14, 1886, 1444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 189, 4085, 3041, 2, 0, 2, 0, 0, 0, 3, 0, 1, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 4092, 3, 0, 4093, 2, 3, 42, 3, 0, 4092, 0, 2275, 1588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 24, 708, 4093, 3, 16, 0, 4060, 1, 3, 4091, 3, 0, 4, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 3840, 0, 0, 0, 90, 0, 0, 0, 0, 4001, 0, 4093, 2, 0, 4087, 987, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16861, 16541, 15722, 15656, 15482, 16202, 17053, 17555] avg=16384.0000 max=17555 max_rank=7 min=15482 min_rank=4 imbalance_ratio=1.071472 max_load_reduction=113517 max_load_reduction_pct=0.866066 moved_to_echo=66292 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 0, 1666, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 4096, 1819, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 1, 0, 0, 0, 0, 4088, 0, 0, 0, 3742, 4096, 4096, 164, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 998, 3, 0, 0, 0, 2722, 3971, 0, 0, 2944, 0, 730, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 25, 0, 0, 0, 49, 4096, 0, 0, 0, 4096, 0, 0, 3, 3198, 1445, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 13, 4095, 3738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 1, 4096, 0, 0, 62, 0, 0, 4096, 1, 2277, 1589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 709, 4096, 0, 1, 0, 3843, 0, 0, 4096, 0, 0, 36, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 3985, 0, 0, 0, 19, 0, 3, 0, 0, 3927, 8, 4096, 0, 0, 4096, 988, 353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15772, 16240, 15464, 17004, 15990, 16221, 16858, 17523] avg=16384.0000 max=17523 max_rank=7 min=15464 min_rank=2 imbalance_ratio=1.069519 max_load_reduction=113549 max_load_reduction_pct=0.866310 moved_to_echo=66592 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 5760, 32768, 128, 128, 0, 29440, 128, 128, 32768, 128, 128, 256, 30336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32768, 128, 128, 128, 0, 0, 256, 32768, 128, 128, 0, 32768, 0, 0, 256, 20736, 11648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 128, 0, 32768, 128, 128, 128, 31232, 32768, 32768, 1408, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 32768, 8704, 128, 0, 128, 128, 24448, 32256, 128, 128, 23552, 0, 9472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 32640, 128, 128, 128, 18304, 128, 128, 128, 32768, 128, 128, 0, 128, 128, 32768, 14592, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 128, 1408, 128, 128, 32768, 512, 18304, 12800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[512, 29056, 0, 128, 128, 384, 0, 128, 0, 0, 24960, 128, 32768, 128, 0, 32768, 7936, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32640, 128, 7424, 32768, 25856, 128, 0, 128, 128, 128, 128, 128, 0, 128, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 2303, 0, 0, 0, 0, 327, 4095, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 4086, 4, 559, 0, 0, 0, 0, 0, 4096, 3183, 216, 0, 0, 32, 0, 4096, 0, 0, 0, 3509, 0, 0, 402, 147, 0, 4093, 4095, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 156, 4096, 4090, 3846, 4069, 3788, 0, 0, 0, 1, 3984, 0, 4096, 4096, 0, 0, 0, 4096, 917, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 15, 0, 0, 5, 0, 2, 0, 0, 0, 1226, 0, 0, 0, 4095, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 2476, 0, 3, 0, 0, 397, 4096, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4095, 4096, 0, 0, 2, 1, 0, 4087, 26, 438, 0, 0, 0, 0, 0, 4096, 1711, 477, 0, 0, 15, 0, 4095, 0, 0, 0, 3593, 0, 0, 393, 1158, 0, 4095, 4096, 4093, 0, 6, 0, 0, 0, 0, 0, 0, 0, 254, 4093, 4086, 3947, 4096, 4032, 0, 0, 0, 0, 3998, 0, 4096, 4096, 0, 0, 0, 4096, 361, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 12, 0, 0, 0, 0, 14, 0, 0, 0, 1291, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3173, 0, 20, 0, 0, 436, 4096, 0, 4096, 0, 0, 0, 0, 3, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 9, 1, 0, 4063, 4, 1349, 0, 0, 0, 0, 0, 4096, 1621, 515, 0, 0, 0, 0, 4094, 0, 0, 0, 4003, 0, 0, 542, 92, 0, 4095, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 4077, 4089, 3439, 4094, 4005, 0, 0, 0, 36, 3850, 0, 4096, 4096, 0, 0, 0, 4096, 503, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 4096, 4096, 0, 0, 96, 0, 0, 5, 0, 0, 0, 0, 0, 705, 0, 0, 0, 4095, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4083, 0, 10, 1, 9, 5, 4087, 4088, 0, 9, 0, 363, 8, 0, 2, 0, 956, 4075, 9, 4035, 10, 13, 8, 0, 0, 8, 4064, 4086, 8, 0, 0, 4, 4087, 4087, 4084, 0, 0, 0, 0, 9, 4087, 154, 68, 5, 0, 1, 0, 4, 4085, 3342, 992, 22, 0, 0, 9, 4083, 0, 9, 9, 2909, 0, 9, 418, 1039, 0, 4081, 4087, 4087, 9, 60, 7, 9, 0, 44, 8, 0, 9, 104, 4096, 4090, 4072, 4079, 4034, 0, 4, 0, 33, 4065, 5, 4087, 4087, 0, 0, 0, 4087, 1243, 0, 9, 0, 9, 6, 1, 0, 4086, 0, 4087, 4077, 0, 0, 68, 1, 1, 44, 5, 0, 9, 0, 8, 649, 0, 0, 0, 4046, 4, 0, 5, 9, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4092, 0, 3, 0, 3, 0, 4093, 4093, 3, 3, 0, 1007, 2, 34, 1, 0, 283, 4091, 3, 4088, 3, 2, 3, 0, 0, 3, 4090, 4093, 3, 0, 0, 0, 4093, 4093, 4093, 0, 0, 6, 0, 3, 4085, 18, 502, 0, 0, 1, 0, 0, 4093, 2693, 423, 0, 2, 0, 3, 4092, 0, 3, 3, 2817, 0, 3, 222, 549, 0, 4093, 4093, 4075, 3, 114, 3, 3, 0, 1, 3, 0, 3, 101, 4084, 4093, 3960, 4093, 3977, 1, 1, 0, 11, 4082, 3, 4093, 4093, 3, 0, 1, 4093, 1313, 0, 3, 0, 3, 2, 0, 0, 4060, 0, 4093, 4088, 0, 1, 24, 0, 0, 79, 0, 2, 3, 0, 0, 2472, 2, 0, 3, 4042, 3, 0, 0, 3, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2703, 0, 18, 0, 0, 118, 4096, 0, 4096, 0, 0, 0, 0, 9, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 16, 0, 0, 4071, 1, 840, 0, 0, 2, 0, 0, 4094, 2757, 69, 0, 0, 22, 0, 4094, 0, 0, 0, 3606, 0, 0, 1092, 114, 0, 4096, 4096, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 4081, 4084, 3170, 4066, 4082, 0, 0, 0, 2, 3982, 0, 4096, 4096, 0, 0, 0, 4096, 634, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4095, 4096, 0, 0, 6, 0, 0, 8, 0, 28, 0, 0, 0, 1026, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4093, 0, 0, 0, 1876, 0, 8, 0, 0, 11, 4096, 0, 4096, 0, 0, 0, 1, 4, 0, 4096, 4096, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 8, 4, 0, 4084, 46, 922, 0, 0, 0, 0, 0, 4092, 2279, 97, 0, 0, 30, 0, 4092, 0, 0, 0, 3975, 0, 0, 1157, 645, 0, 4096, 4096, 4088, 0, 9, 0, 0, 0, 0, 0, 0, 0, 566, 4087, 4076, 3758, 4092, 3427, 0, 0, 0, 0, 4053, 0, 4096, 4096, 0, 0, 0, 4096, 1532, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 4093, 4096, 2, 0, 47, 0, 0, 4, 0, 7, 0, 0, 0, 213, 0, 0, 0, 4086, 0, 4, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 725, 0, 16, 0, 0, 345, 4096, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 4095, 4096, 4096, 0, 0, 7, 0, 0, 4073, 151, 14, 0, 0, 0, 0, 1, 4096, 2282, 1228, 0, 0, 0, 0, 4096, 0, 0, 0, 3287, 0, 0, 969, 649, 0, 4089, 4096, 4082, 0, 1, 0, 0, 0, 0, 0, 0, 0, 124, 4087, 4077, 4024, 4096, 4012, 0, 0, 0, 3, 4052, 0, 4096, 4096, 0, 0, 0, 4096, 1077, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 4096, 4096, 0, 0, 6, 0, 0, 17, 0, 7, 0, 0, 0, 1760, 0, 0, 0, 4090, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 2, 0, 0, 2303, 0, 0, 0, 0, 1178, 625, 365, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 327, 4095, 0, 4096, 0, 0, 0, 1, 0, 0, 4096, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3471, 4096, 0, 0, 0, 0, 0, 4086, 4, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3183, 216, 0, 0, 32, 0, 4096, 0, 0, 0, 3509, 0, 0, 402, 147, 270, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3834, 0, 4096, 0, 0, 0, 0, 1, 0, 0, 0, 0, 156, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3846, 3704, 3788, 0, 0, 0, 1, 3984, 0, 0, 0, 0, 0, 0, 0, 917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4096, 4096, 0, 0, 15, 0, 0, 2917, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 0, 0, 0, 1226, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 4096, 2912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16843, 16359, 16312, 16210, 16273, 16240, 16403, 16432] avg=16384.0000 max=16843 max_rank=0 min=16210 min_rank=3 imbalance_ratio=1.028015 max_load_reduction=114229 max_load_reduction_pct=0.871498 moved_to_echo=65348 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4083, 0, 10, 1, 9, 5, 4087, 4088, 0, 9, 0, 363, 8, 0, 2, 0, 1177, 624, 366, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 956, 4075, 9, 4035, 10, 13, 8, 0, 0, 8, 4064, 3735, 8, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 3463, 4084, 0, 0, 0, 0, 9, 4087, 154, 68, 5, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 3342, 992, 22, 0, 0, 9, 4083, 0, 9, 9, 2909, 0, 9, 418, 1039, 269, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3823, 0, 4087, 9, 60, 7, 9, 0, 44, 8, 0, 9, 104, 4096, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 3713, 4034, 0, 4, 0, 33, 4065, 5, 0, 0, 0, 0, 0, 0, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 9, 6, 1, 0, 4086, 0, 4087, 4077, 0, 0, 68, 1, 1, 2910, 1183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 5, 0, 9, 0, 8, 649, 0, 0, 0, 4046, 4, 0, 5, 9, 8, 4087, 4087, 2904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14914, 16925, 15962, 17453, 16346, 17169, 16438, 15865] avg=16384.0000 max=17453 max_rank=3 min=14914 min_rank=0 imbalance_ratio=1.065247 max_load_reduction=113619 max_load_reduction_pct=0.866844 moved_to_echo=65818 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3173, 0, 20, 0, 0, 1178, 625, 367, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 436, 4096, 0, 4096, 0, 0, 0, 0, 3, 0, 4096, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3471, 4096, 0, 0, 9, 1, 0, 4063, 4, 1349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1621, 515, 0, 0, 0, 0, 4094, 0, 0, 0, 4003, 0, 0, 542, 92, 270, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 4077, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3439, 3727, 4005, 0, 0, 0, 36, 3850, 0, 0, 0, 0, 0, 0, 0, 503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 4096, 4096, 0, 0, 96, 0, 0, 2918, 1184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 705, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 4096, 4096, 2912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17733, 16471, 17089, 15492, 16436, 15560, 16381, 15910] avg=16384.0000 max=17733 max_rank=0 min=15492 min_rank=3 imbalance_ratio=1.082336 max_load_reduction=113339 max_load_reduction_pct=0.864708 moved_to_echo=64287 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4095, 0, 0, 0, 2476, 0, 3, 0, 0, 1179, 625, 367, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 4096, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3470, 4096, 0, 0, 2, 1, 0, 4087, 26, 438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1711, 477, 0, 0, 15, 0, 4095, 0, 0, 0, 3593, 0, 0, 393, 1158, 270, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 0, 4093, 0, 6, 0, 0, 0, 0, 0, 0, 0, 254, 4093, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3947, 3729, 4032, 0, 0, 0, 0, 3998, 0, 0, 0, 0, 0, 0, 0, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 4096, 0, 0, 12, 0, 0, 2917, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 1291, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17019, 16431, 16216, 16067, 16368, 16067, 16400, 16504] avg=16384.0000 max=17019 max_rank=0 min=16067 min_rank=3 imbalance_ratio=1.038757 max_load_reduction=114053 max_load_reduction_pct=0.870155 moved_to_echo=65339 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 2703, 0, 18, 0, 0, 1179, 628, 370, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 4096, 0, 4096, 0, 0, 0, 0, 9, 0, 4096, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3468, 4096, 0, 0, 16, 0, 0, 4071, 1, 840, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2757, 69, 0, 0, 22, 0, 4094, 0, 0, 0, 3606, 0, 0, 1092, 114, 273, 265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3831, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 4081, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3170, 3696, 4082, 0, 0, 0, 2, 3982, 0, 0, 0, 0, 0, 0, 0, 634, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4095, 4096, 0, 0, 6, 0, 0, 2917, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 28, 0, 0, 0, 1026, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17274, 16150, 16590, 16386, 16468, 15566, 16377, 16261] avg=16384.0000 max=17274 max_rank=0 min=15566 min_rank=5 imbalance_ratio=1.054321 max_load_reduction=113798 max_load_reduction_pct=0.868210 moved_to_echo=64672 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4092, 0, 3, 0, 3, 0, 4093, 4093, 3, 3, 0, 1007, 2, 34, 1, 0, 1178, 625, 367, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 4091, 3, 4088, 3, 2, 3, 0, 0, 3, 4090, 3741, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3468, 4093, 0, 0, 6, 0, 3, 4085, 18, 502, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 2693, 423, 0, 2, 0, 3, 4092, 0, 3, 3, 2817, 0, 3, 222, 549, 270, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3834, 0, 4075, 3, 114, 3, 3, 0, 1, 3, 0, 3, 101, 4084, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3960, 3726, 3977, 1, 1, 0, 11, 4082, 3, 0, 0, 3, 0, 1, 0, 1313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 2, 0, 0, 4060, 0, 4093, 4088, 0, 1, 24, 0, 0, 2915, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 2, 3, 0, 0, 2472, 2, 0, 3, 4042, 3, 0, 0, 3, 3, 4093, 4093, 2908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15586, 16310, 16269, 15432, 16317, 17078, 16374, 17706] avg=16384.0000 max=17706 max_rank=7 min=15432 min_rank=3 imbalance_ratio=1.080688 max_load_reduction=113366 max_load_reduction_pct=0.864914 moved_to_echo=67475 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4093, 0, 0, 0, 1876, 0, 8, 0, 0, 1179, 625, 367, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4096, 0, 4096, 0, 0, 0, 1, 4, 0, 4096, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3471, 4096, 0, 0, 8, 4, 0, 4084, 46, 922, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 2279, 97, 0, 0, 30, 0, 4092, 0, 0, 0, 3975, 0, 0, 1157, 645, 270, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3837, 0, 4088, 0, 9, 0, 0, 0, 0, 0, 0, 0, 566, 4087, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3758, 3725, 3427, 0, 0, 0, 0, 4053, 0, 0, 0, 0, 0, 0, 0, 1532, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 4093, 4096, 2, 0, 47, 0, 0, 2917, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7, 0, 0, 0, 213, 0, 0, 0, 4086, 0, 4, 0, 0, 0, 4096, 4096, 2911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16421, 16048, 16726, 16896, 16663, 16495, 16406, 15417] avg=16384.0000 max=16896 max_rank=3 min=15417 min_rank=7 imbalance_ratio=1.031250 max_load_reduction=114176 max_load_reduction_pct=0.871094 moved_to_echo=64981 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 725, 0, 16, 0, 0, 1179, 625, 367, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 4096, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 3744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3471, 4096, 0, 0, 7, 0, 0, 4073, 151, 14, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2282, 1228, 0, 0, 0, 0, 4096, 0, 0, 0, 3287, 0, 0, 969, 649, 270, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3830, 0, 4082, 0, 1, 0, 0, 0, 0, 0, 0, 0, 124, 4087, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 3729, 4012, 0, 0, 0, 3, 4052, 0, 0, 0, 0, 0, 0, 0, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3993, 0, 4096, 4096, 0, 0, 6, 0, 0, 2917, 1185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 7, 0, 0, 0, 1760, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4096, 4096, 2911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15282, 16378, 15908, 17136, 16201, 16897, 16293, 16977] avg=16384.0000 max=17136 max_rank=3 min=15282 min_rank=0 imbalance_ratio=1.045898 max_load_reduction=113936 max_load_reduction_pct=0.869263 moved_to_echo=66368 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 30720, 0, 32768, 128, 256, 128, 128, 128, 128, 128, 0, 128, 2176, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 27776, 32768, 0, 0, 128, 128, 128, 32640, 512, 4736, 128, 0, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 32768, 32768, 128, 128, 0, 14720, 128, 128, 128, 0, 9472, 5120, 2944, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[256, 128, 128, 128, 0, 128, 9344, 128, 0, 128, 32768, 128, 128, 128, 128, 128, 32768, 32768, 23296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[2944, 32768, 128, 32768, 128, 128, 128, 128, 128, 128, 32768, 29952, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 19968, 4096, 128, 128, 128, 128, 32768, 0, 128, 128, 27776, 0, 128, 5248, 4480, 2176, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 0, 128, 128, 128, 0, 32512, 0, 32768, 32768, 128, 128, 384, 128, 128, 23424, 9600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[30336, 29824, 31360, 128, 128, 0, 128, 32128, 128, 0, 0, 128, 0, 128, 0, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 3611, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 11, 0, 4096, 0, 0, 79, 0, 0, 0, 1, 4096, 0, 1, 0, 0, 4095, 0, 0, 52, 4095, 0, 4095, 4096, 0, 4095, 4089, 4095, 0, 7, 4095, 0, 3, 798, 4096, 2, 0, 0, 0, 1, 4094, 3085, 0, 0, 0, 3502, 0, 0, 4096, 0, 4053, 0, 0, 0, 4085, 6, 0, 0, 57, 105, 0, 0, 4096, 0, 4096, 636, 0, 10, 2860, 4096, 0, 4084, 17, 0, 1, 0, 4, 0, 0, 0, 4089, 4095, 0, 0, 0, 4096, 0, 0, 3717, 2062, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[0, 3853, 0, 4096, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 0, 30, 0, 4096, 0, 0, 133, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 206, 4096, 0, 4096, 4096, 0, 4096, 4096, 4096, 0, 15, 4096, 0, 0, 767, 4096, 0, 0, 0, 0, 0, 4096, 1790, 0, 0, 0, 3684, 0, 0, 4096, 0, 4091, 0, 0, 1, 4086, 0, 0, 0, 0, 131, 0, 0, 4096, 0, 4096, 387, 0, 78, 3748, 4096, 1, 4073, 1, 0, 0, 0, 0, 0, 0, 0, 4090, 4095, 0, 0, 0, 4096, 0, 0, 3968, 1928, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 3358, 0, 4096, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3461, 0, 240, 0, 4096, 0, 0, 246, 0, 0, 0, 1, 4096, 0, 4, 0, 0, 4093, 0, 0, 416, 4093, 0, 4077, 4096, 0, 4096, 4055, 4095, 0, 31, 4087, 0, 34, 784, 4096, 4, 0, 0, 0, 13, 4071, 2427, 0, 0, 0, 2921, 0, 0, 4096, 0, 3790, 0, 0, 0, 4091, 6, 0, 0, 110, 16, 0, 0, 4096, 0, 4096, 2176, 0, 81, 1740, 4096, 0, 4090, 0, 0, 1, 0, 12, 0, 0, 0, 3905, 4095, 0, 0, 0, 4096, 0, 0, 3671, 3499, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[9, 1663, 8, 4087, 9, 4042, 9, 0, 0, 0, 9, 0, 1, 8, 0, 3552, 0, 211, 9, 4086, 9, 9, 1037, 11, 9, 9, 0, 4087, 0, 0, 9, 4, 4087, 0, 0, 6, 4086, 9, 4087, 4087, 9, 4086, 4088, 4085, 0, 0, 4088, 21, 15, 182, 4085, 0, 0, 10, 9, 0, 4086, 769, 0, 0, 0, 3958, 0, 230, 4085, 8, 3899, 0, 1, 67, 3857, 0, 7, 2, 839, 2038, 15, 0, 4082, 0, 4084, 628, 0, 0, 2511, 4086, 2, 4087, 1, 0, 0, 7, 0, 9, 0, 0, 4062, 4085, 8, 10, 10, 4087, 0, 9, 4067, 3170, 0, 9, 30, 0, 4085, 14, 0, 0, 4086, 0, 8, 10, 4, 2, 9, 4, 7, 11, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[3, 3046, 3, 4093, 3, 4013, 3, 0, 0, 0, 3, 0, 1, 3, 0, 3767, 0, 142, 3, 4093, 3, 3, 558, 3, 3, 3, 0, 4092, 0, 0, 3, 0, 4092, 2, 0, 1, 4090, 3, 4086, 4093, 3, 4093, 4089, 4093, 0, 0, 4093, 2, 15, 280, 4093, 2, 0, 3, 3, 0, 4093, 3184, 0, 0, 0, 3788, 0, 115, 4093, 3, 4028, 0, 0, 71, 3908, 0, 2, 0, 340, 1425, 2, 0, 4090, 0, 4093, 950, 0, 0, 1583, 4093, 17, 4093, 1, 2, 0, 0, 0, 3, 0, 0, 4093, 4087, 3, 3, 3, 4092, 0, 3, 4009, 1608, 0, 3, 7, 0, 4092, 3, 0, 0, 4093, 0, 3, 3, 1, 1, 3, 1, 3, 1, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[0, 3499, 0, 4096, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 551, 0, 4096, 0, 0, 58, 0, 0, 0, 2, 4096, 0, 9, 0, 0, 4093, 0, 0, 31, 4095, 0, 4079, 4096, 0, 4096, 4068, 4096, 0, 33, 4089, 0, 19, 139, 4096, 3, 0, 0, 0, 1, 4086, 2695, 0, 0, 0, 2887, 0, 0, 4096, 0, 4067, 0, 0, 0, 4096, 25, 0, 0, 132, 723, 0, 0, 4096, 0, 4096, 1604, 0, 110, 3527, 4096, 1, 4066, 0, 0, 5, 0, 13, 0, 0, 0, 4093, 4094, 0, 0, 0, 4096, 0, 0, 3956, 640, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 3925, 0, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 225, 0, 4096, 0, 0, 701, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 48, 4095, 0, 4083, 4096, 0, 4096, 4058, 4096, 0, 1, 4082, 0, 13, 478, 4096, 1, 0, 0, 0, 0, 4095, 2077, 0, 0, 0, 3845, 0, 0, 4096, 0, 4016, 0, 0, 0, 4096, 21, 0, 0, 269, 241, 0, 0, 4096, 0, 4096, 826, 0, 65, 3025, 4096, 0, 4090, 0, 0, 2, 0, 8, 0, 0, 0, 4081, 4095, 0, 0, 0, 4096, 0, 0, 3855, 1060, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 3481, 0, 4092, 0, 4069, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3812, 0, 67, 0, 4096, 0, 0, 54, 0, 0, 0, 4, 4092, 0, 0, 0, 0, 4090, 0, 0, 85, 4092, 0, 4085, 4096, 0, 4096, 4077, 4096, 0, 21, 4092, 0, 10, 1614, 4096, 6, 0, 0, 0, 7, 4086, 2961, 0, 0, 0, 3179, 0, 0, 4096, 0, 3995, 0, 0, 0, 4089, 14, 0, 0, 8, 564, 0, 0, 4096, 0, 4096, 1146, 0, 27, 2354, 4096, 0, 4086, 11, 0, 4, 0, 8, 0, 0, 0, 4089, 4096, 0, 0, 0, 4095, 0, 0, 4033, 1314, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3853, 0, 4096, 0, 3935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4061, 589, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 4096, 0, 0, 133, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 3543, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 206, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 15, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 767, 4096, 0, 0, 0, 0, 0, 4096, 1790, 0, 0, 0, 3684, 0, 0, 1110, 553, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3356, 0, 4091, 0, 0, 1, 4086, 0, 0, 0, 0, 131, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 387, 0, 78, 3748, 4096, 1, 4073, 1, 0, 0, 0, 0, 0, 0, 0, 248, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4095, 0, 0, 0, 0, 0, 0, 3968, 1928, 0, 0, 0, 0, 2396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17026, 15997, 16518, 16096, 15761, 16815, 16477, 16382] avg=16384.0000 max=17026 max_rank=0 min=15761 min_rank=4 imbalance_ratio=1.039185 max_load_reduction=114046 max_load_reduction_pct=0.870102 moved_to_echo=65435 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3611, 0, 4096, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 589, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4096, 0, 0, 79, 0, 0, 0, 1, 4096, 0, 1, 0, 0, 4095, 3543, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 52, 0, 0, 4095, 0, 0, 0, 4089, 0, 0, 7, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 798, 4096, 2, 0, 0, 0, 1, 4094, 3085, 0, 0, 0, 3502, 0, 0, 1111, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3356, 0, 4053, 0, 0, 0, 4085, 6, 0, 0, 57, 105, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 636, 0, 10, 2860, 4096, 0, 4084, 17, 0, 1, 0, 4, 0, 0, 0, 248, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4095, 0, 0, 0, 0, 0, 0, 3717, 2062, 0, 0, 0, 0, 2396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4095, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16920, 15925, 16346, 17244, 15758, 16139, 16359, 16381] avg=16384.0000 max=17244 max_rank=3 min=15758 min_rank=4 imbalance_ratio=1.052490 max_load_reduction=113828 max_load_reduction_pct=0.868439 moved_to_echo=64637 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[9, 1663, 8, 4087, 9, 4042, 9, 0, 0, 0, 9, 0, 1, 8, 0, 3552, 587, 490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 211, 9, 4086, 9, 9, 1037, 11, 9, 9, 0, 4087, 0, 0, 9, 4, 4085, 3536, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 6, 0, 9, 4087, 0, 9, 0, 4088, 0, 0, 0, 4088, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 182, 4085, 0, 0, 10, 9, 0, 4086, 769, 0, 0, 0, 3958, 0, 230, 1108, 550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3347, 8, 3899, 0, 1, 67, 3857, 0, 7, 2, 839, 2038, 15, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 628, 0, 0, 2511, 4086, 2, 4087, 1, 0, 0, 7, 0, 9, 0, 0, 248, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 4085, 8, 10, 10, 0, 0, 9, 4067, 3170, 0, 9, 30, 0, 2390, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 8, 10, 4, 2, 9, 4, 7, 11, 0, 0, 0, 0, 4087, 4086, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14474, 17115, 16308, 15002, 18162, 15750, 17864, 16397] avg=16384.0000 max=18162 max_rank=4 min=14474 min_rank=0 imbalance_ratio=1.108521 max_load_reduction=112910 max_load_reduction_pct=0.861435 moved_to_echo=68173 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3358, 0, 4096, 0, 4027, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3461, 589, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 4096, 0, 0, 246, 0, 0, 0, 1, 4096, 0, 4, 0, 0, 4095, 3541, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 0, 0, 416, 0, 0, 4077, 0, 0, 0, 4055, 0, 0, 31, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 784, 4096, 4, 0, 0, 0, 13, 4071, 2427, 0, 0, 0, 2921, 0, 0, 1111, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3356, 0, 3790, 0, 0, 0, 4091, 6, 0, 0, 110, 16, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2176, 0, 81, 1740, 4096, 0, 4090, 0, 0, 1, 0, 12, 0, 0, 0, 248, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3905, 4095, 0, 0, 0, 0, 0, 0, 3671, 3499, 0, 0, 0, 0, 2396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16023, 16322, 16672, 16013, 15465, 16627, 17566, 16384] avg=16384.0000 max=17566 max_rank=6 min=15465 min_rank=4 imbalance_ratio=1.072144 max_load_reduction=113506 max_load_reduction_pct=0.865982 moved_to_echo=66042 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3499, 0, 4096, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 591, 495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 0, 4096, 0, 0, 58, 0, 0, 0, 2, 4096, 0, 9, 0, 0, 4096, 3544, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 0, 0, 31, 0, 0, 4079, 0, 0, 0, 4068, 0, 0, 33, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 139, 4096, 3, 0, 0, 0, 1, 4086, 2695, 0, 0, 0, 2887, 0, 0, 1113, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3347, 0, 4067, 0, 0, 0, 4096, 25, 0, 0, 132, 723, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1604, 0, 110, 3527, 4096, 1, 4066, 0, 0, 5, 0, 13, 0, 0, 0, 254, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4094, 0, 0, 0, 0, 0, 0, 3956, 640, 0, 0, 0, 0, 2392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16818, 16456, 16300, 15590, 16486, 17865, 15175, 16382] avg=16384.0000 max=17865 max_rank=5 min=15175 min_rank=6 imbalance_ratio=1.090393 max_load_reduction=113207 max_load_reduction_pct=0.863701 moved_to_echo=65908 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[3, 3046, 3, 4093, 3, 4013, 3, 0, 0, 0, 3, 0, 1, 3, 0, 3767, 588, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 142, 3, 4093, 3, 3, 558, 3, 3, 3, 0, 4092, 0, 0, 3, 0, 4093, 3539, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 2, 0, 1, 0, 3, 4086, 0, 3, 0, 4089, 0, 0, 0, 4093, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 280, 4093, 2, 0, 3, 3, 0, 4093, 3184, 0, 0, 0, 3788, 0, 115, 1110, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3354, 3, 4028, 0, 0, 71, 3908, 0, 2, 0, 340, 1425, 2, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 950, 0, 0, 1583, 4093, 17, 4093, 1, 2, 0, 0, 0, 3, 0, 0, 248, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4087, 3, 3, 3, 0, 0, 3, 4009, 1608, 0, 3, 7, 0, 2394, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3, 3, 1, 1, 3, 1, 3, 1, 0, 0, 0, 0, 4093, 4093, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16017, 16542, 16284, 17237, 17223, 15170, 16216, 16383] avg=16384.0000 max=17237 max_rank=3 min=15170 min_rank=5 imbalance_ratio=1.052063 max_load_reduction=113835 max_load_reduction_pct=0.868492 moved_to_echo=64992 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3481, 0, 4092, 0, 4069, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3812, 589, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 4096, 0, 0, 54, 0, 0, 0, 4, 4092, 0, 0, 0, 0, 4096, 3540, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4003, 0, 0, 85, 0, 0, 4085, 0, 0, 0, 4077, 0, 0, 21, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1614, 4096, 6, 0, 0, 0, 7, 4086, 2961, 0, 0, 0, 3179, 0, 0, 1111, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3356, 0, 3995, 0, 0, 0, 4089, 14, 0, 0, 8, 564, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1146, 0, 27, 2354, 4096, 0, 4086, 11, 0, 4, 0, 8, 0, 0, 0, 248, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4096, 0, 0, 0, 0, 0, 0, 4033, 1314, 0, 0, 0, 0, 2396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16538, 15953, 16363, 17622, 16122, 16163, 15928, 16383] avg=16384.0000 max=17622 max_rank=3 min=15928 min_rank=6 imbalance_ratio=1.075562 max_load_reduction=113450 max_load_reduction_pct=0.865555 moved_to_echo=64596 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3925, 0, 4096, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4079, 589, 492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 4096, 0, 0, 701, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 3543, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 0, 48, 0, 0, 4083, 0, 0, 0, 4058, 0, 0, 1, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 478, 4096, 1, 0, 0, 0, 0, 4095, 2077, 0, 0, 0, 3845, 0, 0, 1111, 552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3356, 0, 4016, 0, 0, 0, 4096, 21, 0, 0, 269, 241, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 826, 0, 65, 3025, 4096, 0, 4090, 0, 0, 2, 0, 8, 0, 0, 0, 248, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 4095, 0, 0, 0, 0, 0, 0, 3855, 1060, 0, 0, 0, 0, 2396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17256, 16762, 16281, 16268, 16095, 16543, 15487, 16380] avg=16384.0000 max=17256 max_rank=0 min=15487 min_rank=6 imbalance_ratio=1.053223 max_load_reduction=113816 max_load_reduction_pct=0.868347 moved_to_echo=64505 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32128, 128, 0, 896, 0, 128, 32768, 0, 128, 0, 32640, 0, 0, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32512, 32768, 128, 128, 128, 0, 0, 128, 31360, 15360, 0, 128, 128, 0, 19200, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 32768, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 0, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 5120, 32768, 128, 0, 128, 128, 128, 32768, 19072, 0, 0, 0, 27776, 0, 384, 8960, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 8448, 0, 384, 21376, 32768, 128, 32768, 128, 128, 128, 128, 128, 128, 0, 0, 2048, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[26880, 128, 32000, 0, 128, 256, 32384, 128, 128, 128, 1792, 5248, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 26496, 128, 32768, 128, 32384, 128, 0, 0, 0, 128, 128, 128, 128, 0, 30848, 4736, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 1536, 128, 32768, 128, 128, 2944, 128, 128, 128, 128, 32768, 0, 128, 128, 128, 32768, 28416, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 3729, 0, 0, 0, 0, 6, 0, 4086, 3960, 2, 4095, 0, 0, 4014, 4096, 0, 774, 1, 0, 0, 33, 2, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 3, 0, 0, 4008, 4096, 0, 4094, 11, 0, 0, 0, 0, 2, 0, 4094, 0, 14, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 2, 4095, 0, 0, 0, 4096, 0, 0, 30, 0, 0, 1, 0, 9, 0, 4094, 0, 0, 4091, 0, 34, 4076, 0, 3907, 4096, 0, 0, 4096, 0, 0, 4096, 4084, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 3651, 0, 0, 0, 0, 1, 0, 4093, 3994, 2, 4095, 0, 0, 3587, 4096, 0, 1060, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4042, 4096, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 13, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4096, 1, 4096, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 1, 0, 5, 0, 4095, 0, 0, 4095, 0, 152, 4092, 0, 3978, 4096, 0, 0, 4096, 0, 0, 4096, 4095, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[14, 0, 0, 4096, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3881, 0, 0, 0, 0, 0, 0, 4096, 3944, 8, 4096, 0, 0, 4006, 4096, 0, 486, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4062, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 41, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 45, 4093, 0, 4008, 4096, 0, 0, 4096, 0, 0, 4096, 4067, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[773, 5, 1, 4060, 2, 9, 22, 0, 9, 196, 4026, 9, 10, 0, 4086, 5, 4008, 12, 0, 0, 0, 4088, 4087, 0, 0, 0, 4070, 11, 0, 0, 0, 0, 0, 4087, 4005, 258, 4087, 6, 10, 2832, 4087, 98, 567, 0, 0, 7, 0, 0, 9, 2, 7, 0, 9, 0, 4085, 0, 8, 10, 0, 0, 8, 3705, 4086, 12, 4087, 0, 9, 0, 8, 10, 0, 0, 4088, 0, 1, 3893, 10, 9, 0, 3, 4086, 6, 4087, 0, 5, 0, 9, 2, 3, 10, 8, 0, 0, 4084, 0, 4084, 8, 10, 10, 4086, 0, 95, 1, 0, 0, 0, 9, 0, 0, 4086, 9, 7, 4087, 0, 359, 4011, 9, 4074, 4086, 0, 0, 3987, 0, 48, 4082, 3944, 4088, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[3, 0, 0, 4096, 0, 0, 10, 2, 0, 0, 4089, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4084, 4092, 0, 0, 0, 3898, 0, 0, 0, 0, 5, 0, 4090, 4032, 0, 4093, 0, 0, 3703, 4094, 0, 751, 4, 0, 0, 12, 4, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 2, 0, 0, 4021, 4096, 0, 4092, 5, 0, 0, 0, 0, 0, 0, 4092, 0, 67, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 6, 4089, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 5, 0, 10, 0, 4092, 0, 0, 4092, 0, 32, 4063, 0, 4034, 4096, 0, 0, 4096, 0, 0, 4096, 4040, 4091, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[2, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 4073, 4080, 0, 0, 0, 3542, 0, 0, 0, 0, 6, 0, 4092, 3759, 3, 4096, 0, 0, 4047, 4096, 0, 1668, 0, 0, 0, 40, 1, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 3, 0, 0, 4042, 4096, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 4095, 0, 5, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 34, 0, 0, 3, 0, 44, 0, 4094, 0, 0, 4079, 0, 94, 4096, 0, 3248, 4096, 0, 0, 4096, 0, 0, 4096, 4088, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[42, 2, 0, 4093, 0, 3, 21, 0, 3, 4, 4051, 3, 3, 0, 4093, 2, 4057, 0, 3, 0, 0, 4093, 4093, 0, 2, 0, 4061, 3, 10, 0, 0, 0, 0, 4069, 3722, 119, 4093, 0, 3, 3434, 4093, 8, 1252, 0, 0, 2, 21, 0, 3, 0, 3, 0, 3, 0, 4092, 0, 3, 1, 0, 2, 3, 3735, 4093, 4, 4093, 4, 3, 0, 3, 3, 0, 0, 4093, 0, 11, 4057, 3, 3, 5, 2, 4093, 3, 4093, 0, 3, 0, 3, 0, 0, 3, 3, 0, 0, 4093, 0, 4093, 3, 3, 3, 4093, 0, 3, 32, 0, 7, 0, 3, 0, 1, 4093, 3, 2, 4093, 0, 82, 4086, 3, 4072, 4093, 0, 0, 4074, 0, 9, 4091, 4063, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[9, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4090, 4083, 0, 0, 0, 3640, 0, 0, 0, 0, 1, 0, 4095, 4073, 5, 4096, 0, 0, 3671, 4096, 0, 899, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4057, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 58, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 1, 4096, 0, 0, 0, 4096, 0, 0, 38, 0, 0, 0, 0, 27, 0, 4096, 0, 0, 4095, 0, 46, 4096, 0, 3964, 4096, 0, 0, 4096, 0, 0, 4096, 4095, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4096, 0, 0, 12, 0, 0, 0, 4090, 0, 0, 0, 4096, 0, 3868, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4083, 4096, 0, 0, 0, 3729, 0, 0, 0, 0, 6, 312, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 3960, 2, 3675, 0, 0, 4014, 0, 0, 774, 1, 0, 0, 33, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 3, 0, 0, 4008, 4096, 0, 4096, 83, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 11, 0, 0, 0, 0, 2, 0, 4094, 0, 14, 4096, 0, 0, 0, 0, 4010, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 30, 0, 0, 1, 0, 9, 0, 4094, 0, 0, 4096, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 34, 4076, 0, 3907, 0, 0, 0, 4096, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16270, 16325, 16547, 16432, 16409, 16383, 16374, 16332] avg=16384.0000 max=16547 max_rank=2 min=16270 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=114525 max_load_reduction_pct=0.873756 moved_to_echo=65498 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 3867, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 3651, 0, 0, 0, 0, 1, 309, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3994, 2, 3683, 0, 0, 3587, 0, 0, 1060, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4042, 4096, 0, 4096, 82, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 4095, 0, 13, 4096, 0, 0, 0, 0, 4013, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 1, 0, 5, 0, 4095, 0, 0, 4096, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 152, 4092, 0, 3978, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16262, 16241, 16423, 16461, 16396, 16385, 16346, 16558] avg=16384.0000 max=16558 max_rank=7 min=16241 min_rank=1 imbalance_ratio=1.010620 max_load_reduction=114514 max_load_reduction_pct=0.873672 moved_to_echo=65685 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[14, 0, 0, 4096, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 4096, 0, 3868, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 3881, 0, 0, 0, 0, 0, 309, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3944, 8, 3684, 0, 0, 4006, 0, 0, 486, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4062, 4096, 0, 4096, 82, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 41, 4096, 0, 0, 0, 0, 4014, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 45, 4093, 0, 4008, 0, 0, 0, 4096, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16283, 16480, 16230, 16479, 16426, 16382, 16338, 16454] avg=16384.0000 max=16480 max_rank=1 min=16230 min_rank=2 imbalance_ratio=1.005859 max_load_reduction=114592 max_load_reduction_pct=0.874268 moved_to_echo=65600 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[773, 5, 1, 4060, 2, 9, 22, 0, 9, 196, 4026, 9, 10, 0, 4086, 5, 3860, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 12, 0, 0, 0, 4088, 4087, 0, 0, 0, 4070, 11, 0, 0, 0, 0, 308, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4005, 258, 3676, 6, 10, 2832, 0, 98, 567, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 7, 0, 9, 0, 4085, 0, 8, 10, 0, 0, 8, 3705, 4086, 12, 4082, 83, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 9, 0, 8, 10, 0, 0, 4088, 0, 1, 3893, 10, 9, 0, 3, 4005, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 6, 4085, 0, 5, 0, 9, 2, 3, 10, 8, 0, 0, 4084, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 10, 10, 4086, 0, 95, 1, 0, 0, 0, 9, 0, 0, 4086, 9, 7, 4086, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 359, 4011, 9, 4074, 0, 0, 0, 3987, 0, 48, 0, 3944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17176, 16586, 15546, 16153, 16206, 16382, 16447, 16576] avg=16384.0000 max=17176 max_rank=0 min=15546 min_rank=2 imbalance_ratio=1.048340 max_load_reduction=113896 max_load_reduction_pct=0.868958 moved_to_echo=65611 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 0, 4096, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3852, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4073, 4080, 0, 0, 0, 3542, 0, 0, 0, 0, 6, 309, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 3759, 3, 3684, 0, 0, 4047, 0, 0, 1668, 0, 0, 0, 40, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 3, 0, 0, 4042, 4096, 0, 4096, 82, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 0, 0, 4095, 0, 5, 4096, 0, 0, 0, 0, 4011, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 34, 0, 0, 3, 0, 44, 0, 4094, 0, 0, 4096, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 94, 4096, 0, 3248, 0, 0, 0, 4096, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16256, 16100, 17294, 16466, 16389, 16385, 16416, 15766] avg=16384.0000 max=17294 max_rank=2 min=15766 min_rank=7 imbalance_ratio=1.055542 max_load_reduction=113778 max_load_reduction_pct=0.868057 moved_to_echo=64956 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 4096, 0, 0, 10, 2, 0, 0, 4089, 0, 0, 0, 4096, 0, 3864, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4084, 4092, 0, 0, 0, 3898, 0, 0, 0, 0, 5, 309, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4032, 0, 3681, 0, 0, 3703, 0, 0, 751, 4, 0, 0, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 0, 0, 2, 0, 0, 4021, 4096, 0, 4096, 83, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 5, 0, 0, 0, 0, 0, 0, 4092, 0, 67, 4096, 0, 0, 0, 0, 4008, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4096, 6, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 5, 0, 10, 0, 4092, 0, 0, 4096, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 32, 4063, 0, 4034, 0, 0, 0, 4096, 0, 0, 0, 4040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16263, 16486, 16277, 16444, 16443, 16385, 16364, 16410] avg=16384.0000 max=16486 max_rank=1 min=16263 min_rank=0 imbalance_ratio=1.006226 max_load_reduction=114586 max_load_reduction_pct=0.874222 moved_to_echo=65602 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[42, 2, 0, 4093, 0, 3, 21, 0, 3, 4, 4051, 3, 3, 0, 4093, 2, 3865, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 3, 0, 0, 4093, 4093, 0, 2, 0, 4061, 3, 10, 0, 0, 0, 309, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 3722, 119, 3681, 0, 3, 3434, 0, 8, 1252, 0, 0, 2, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 3, 0, 4092, 0, 3, 1, 0, 2, 3, 3735, 4093, 4, 4091, 83, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 4, 3, 0, 3, 3, 0, 0, 4093, 0, 11, 4057, 3, 3, 5, 2, 4010, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 4091, 0, 3, 0, 3, 0, 0, 3, 3, 0, 0, 4093, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 4093, 0, 3, 32, 0, 7, 0, 3, 0, 1, 4093, 3, 2, 4093, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 82, 4086, 3, 4072, 0, 0, 0, 4074, 0, 9, 0, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16288, 16633, 16311, 16163, 16373, 16385, 16385, 16534] avg=16384.0000 max=16633 max_rank=1 min=16163 min_rank=3 imbalance_ratio=1.015198 max_load_reduction=114439 max_load_reduction_pct=0.873100 moved_to_echo=65677 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[9, 0, 0, 4096, 0, 0, 7, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3867, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 4083, 0, 0, 0, 3640, 0, 0, 0, 0, 1, 309, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4073, 5, 3684, 0, 0, 3671, 0, 0, 899, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4057, 4096, 0, 4096, 82, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 58, 4096, 0, 0, 0, 0, 4014, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 38, 0, 0, 0, 0, 27, 0, 4096, 0, 0, 4096, 4049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 46, 4096, 0, 3964, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16274, 16221, 16444, 16474, 16430, 16385, 16402, 16442] avg=16384.0000 max=16474 max_rank=3 min=16221 min_rank=1 imbalance_ratio=1.005493 max_load_reduction=114598 max_load_reduction_pct=0.874313 moved_to_echo=65659 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 32768, 0, 128, 0, 128, 128, 128, 128, 128, 128, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[896, 128, 128, 32768, 128, 128, 128, 128, 128, 256, 32640, 128, 128, 0, 32768, 128, 30976, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 128, 0, 32768, 0, 256, 32640, 128, 128, 128, 128, 32128, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131456 observed_loss=-384 tokens_per_expert=[1152, 0, 896, 32640, 128, 31360, 0, 0, 0, 32640, 0, 128, 0, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 32768, 0, 128, 256, 0, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 0, 128, 128, 32768, 0, 128, 128, 128, 128, 128, 31744, 32768, 128, 32768, 768, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32640, 128, 128, 0, 0, 32768, 32768, 0, 128, 0, 30592, 128, 128, 0, 0, 128, 2560, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 32768, 31616, 512, 29568, 128, 128, 29312, 0, 128, 7552, 128, 0, 128, 256, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131456 slack_slots=130688 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 821, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 845, 0, 0, 0, 1, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 1060, 0, 6, 4095, 4096, 0, 4096, 0, 1, 3443, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4095, 0, 0, 2568, 12, 4096, 4088, 4095, 6, 0, 0, 3943, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1683, 1471, 0, 13, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 536, 0, 1, 1, 4095, 0, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 10, 0, 819, 4084, 10, 0, 8, 0, 0, 4050, 8, 4086, 5, 9, 4086, 4077, 9, 45, 9, 0, 0, 4091, 4, 9, 3807, 9, 4086, 0, 0, 2, 17, 4080, 17, 10, 2668, 12, 10, 1790, 9, 0, 27, 0, 4079, 0, 1, 12, 0, 0, 281, 0, 0, 205, 0, 0, 4088, 4026, 9, 4087, 80, 0, 1077, 0, 2, 0, 9, 4083, 0, 1, 0, 8, 0, 4061, 9, 0, 3683, 0, 3049, 3570, 3737, 0, 0, 0, 3915, 0, 9, 10, 7, 4086, 3, 0, 4035, 10, 4, 10, 4087, 4054, 3181, 0, 0, 0, 74, 0, 9, 0, 0, 0, 4082, 5, 2, 7, 0, 0, 4087, 32, 0, 10, 4086, 590, 4086, 0, 4085, 311, 0, 10, 0, 4065] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 793, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4069, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 606, 0, 0, 0, 5, 4096, 13, 0, 0, 0, 0, 0, 0, 0, 161, 0, 32, 4094, 4096, 0, 4096, 0, 0, 2898, 0, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 4087, 0, 0, 2425, 10, 4096, 3992, 4091, 71, 0, 0, 3781, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 1667, 3464, 0, 2, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 602, 0, 3, 5, 4092, 0, 0, 0, 4096, 0, 4096, 0, 4095, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 200, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4051, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4074, 0, 0, 4096, 0, 0, 938, 0, 0, 0, 7, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 74, 0, 31, 4086, 4091, 0, 4095, 0, 2, 3438, 0, 0, 0, 0, 4096, 0, 0, 9, 0, 0, 4086, 0, 0, 1867, 63, 4096, 4030, 4094, 34, 0, 0, 3850, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2007, 2505, 0, 29, 0, 0, 0, 0, 2, 0, 0, 4093, 0, 1462, 0, 2, 5, 4094, 0, 0, 0, 4096, 4, 4096, 1, 4094, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 1034, 4096, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 4094, 0, 0, 87, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 719, 0, 1, 4096, 4087, 0, 4096, 0, 0, 2645, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 2732, 21, 4096, 3877, 4096, 1, 0, 0, 3872, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2475, 2478, 0, 29, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 528, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 4096, 0, 4096, 2, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 875, 4091, 3, 0, 3, 0, 0, 4091, 13, 4093, 7, 3, 4093, 4091, 3, 14, 3, 0, 0, 4094, 2, 3, 3982, 3, 4093, 0, 0, 13, 3, 4093, 4, 3, 3185, 4, 3, 1266, 3, 1, 0, 1, 4093, 0, 5, 3, 0, 0, 62, 0, 0, 221, 0, 0, 4093, 4085, 3, 4093, 1, 0, 2402, 0, 0, 0, 3, 4092, 1, 0, 0, 2, 0, 4080, 3, 0, 2603, 0, 3665, 3773, 3883, 0, 1, 0, 3324, 0, 3, 3, 3, 4093, 1, 0, 4076, 3, 2, 3, 4093, 3703, 3341, 0, 22, 0, 5, 0, 3, 0, 0, 0, 4088, 4, 17, 3, 0, 0, 4093, 3, 0, 3, 4093, 275, 4093, 0, 4093, 227, 0, 3, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 17, 0, 0, 976, 4093, 0, 0, 0, 0, 3, 4096, 1, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4067, 0, 0, 4093, 0, 4096, 4, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 587, 0, 0, 0, 0, 4096, 9, 0, 0, 0, 0, 0, 0, 0, 7, 0, 21, 4091, 4081, 0, 4093, 1, 3, 3601, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 4085, 0, 0, 1892, 12, 4096, 4063, 4095, 28, 0, 0, 4011, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3151, 1378, 0, 7, 3, 0, 0, 0, 3, 0, 1, 4093, 0, 776, 0, 3, 4, 4092, 0, 2, 0, 4096, 0, 4094, 0, 4092, 6, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 2156, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4075, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 303, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 0, 0, 0, 95, 0, 19, 4095, 4095, 0, 4096, 0, 0, 3363, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4080, 0, 0, 1218, 18, 4096, 4028, 4096, 3, 0, 0, 3655, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3346, 1288, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1026, 0, 0, 2, 4096, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 821, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2613, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4096, 0, 0, 0, 0, 0, 4090, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 4096, 0, 0, 845, 0, 0, 0, 1, 4096, 1, 0, 0, 3576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1060, 0, 6, 4095, 4096, 0, 4096, 0, 1, 3443, 0, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2067, 0, 0, 1, 0, 0, 4095, 0, 0, 2568, 12, 4096, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 0, 0, 3943, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 183, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1683, 1471, 0, 13, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 536, 0, 1426, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 0, 0, 3920, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16242, 16435, 16701, 17667, 16927, 16595, 14297, 16208] avg=16384.0000 max=17667 max_rank=3 min=14297 min_rank=6 imbalance_ratio=1.078308 max_load_reduction=113405 max_load_reduction_pct=0.865211 moved_to_echo=64027 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 1034, 4096, 0, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 2613, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 4094, 0, 0, 87, 0, 0, 0, 0, 4094, 0, 0, 0, 3576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 719, 0, 1, 4096, 4087, 0, 4096, 0, 0, 2645, 0, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2064, 0, 0, 0, 0, 0, 4096, 0, 0, 2732, 21, 4096, 3877, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 3872, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 183, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2475, 2478, 0, 29, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 528, 0, 1426, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 3920, 0, 4096, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16456, 16438, 15943, 16514, 16886, 16520, 16104, 16211] avg=16384.0000 max=16886 max_rank=4 min=15943 min_rank=2 imbalance_ratio=1.030640 max_load_reduction=114186 max_load_reduction_pct=0.871170 moved_to_echo=65721 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 10, 0, 819, 4084, 10, 0, 8, 0, 0, 4050, 8, 4086, 5, 9, 2607, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4077, 9, 45, 9, 0, 0, 4091, 4, 9, 3807, 9, 4086, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 4080, 17, 10, 2668, 12, 10, 1790, 9, 0, 27, 0, 4079, 0, 1, 12, 3566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 0, 205, 0, 0, 4088, 4026, 9, 4087, 80, 0, 1077, 0, 868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 9, 2059, 0, 1, 0, 8, 0, 4061, 9, 0, 3683, 0, 3049, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3737, 0, 0, 0, 3915, 0, 9, 10, 7, 4086, 3, 0, 4035, 10, 4, 10, 183, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 4054, 3181, 0, 0, 0, 74, 0, 9, 0, 0, 0, 4082, 5, 2, 7, 1422, 973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 32, 0, 10, 0, 590, 3911, 0, 4085, 311, 0, 10, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16216, 16205, 16298, 14721, 16451, 16184, 17896, 17101] avg=16384.0000 max=17896 max_rank=6 min=14721 min_rank=3 imbalance_ratio=1.092285 max_load_reduction=113176 max_load_reduction_pct=0.863464 moved_to_echo=67632 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 793, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2613, 520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4096, 0, 0, 0, 0, 0, 4069, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 4096, 0, 0, 606, 0, 0, 0, 5, 4096, 13, 0, 0, 3576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 32, 4094, 4096, 0, 4096, 0, 0, 2898, 0, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2067, 0, 0, 7, 0, 0, 4087, 0, 0, 2425, 10, 4096, 3992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 71, 0, 0, 3781, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 183, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1667, 3464, 0, 2, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 602, 0, 1426, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 4092, 0, 0, 0, 0, 0, 3920, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16215, 16414, 16481, 16247, 16684, 16494, 16326, 16211] avg=16384.0000 max=16684 max_rank=4 min=16211 min_rank=7 imbalance_ratio=1.018311 max_load_reduction=114388 max_load_reduction_pct=0.872711 moved_to_echo=65715 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 8, 0, 0, 200, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2618, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 4096, 0, 0, 0, 0, 0, 4051, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 0, 0, 4096, 0, 0, 938, 0, 0, 0, 7, 4096, 15, 0, 0, 3575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 31, 4086, 4091, 0, 4095, 0, 2, 3438, 0, 875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2052, 0, 0, 9, 0, 0, 4086, 0, 0, 1867, 63, 4096, 4030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 34, 0, 0, 3850, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 187, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2007, 2505, 0, 29, 0, 0, 0, 0, 2, 0, 0, 4093, 0, 1462, 0, 1428, 982, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 4094, 0, 0, 0, 0, 4, 3918, 1, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15634, 16389, 16801, 16692, 16203, 16535, 16604, 16214] avg=16384.0000 max=16801 max_rank=2 min=15634 min_rank=0 imbalance_ratio=1.025452 max_load_reduction=114271 max_load_reduction_pct=0.871819 moved_to_echo=65556 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 17, 0, 0, 976, 4093, 0, 0, 0, 0, 3, 4096, 1, 4096, 0, 0, 2613, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4096, 0, 0, 0, 0, 0, 4067, 0, 0, 4093, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 4096, 0, 0, 587, 0, 0, 0, 0, 4096, 9, 0, 0, 3575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 21, 4091, 4081, 0, 4093, 1, 3, 3601, 0, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2067, 0, 0, 4, 0, 0, 4085, 0, 0, 1892, 12, 4096, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 28, 0, 0, 4011, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 183, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3151, 1378, 0, 7, 3, 0, 0, 0, 3, 0, 1, 4093, 0, 776, 0, 1426, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 4092, 0, 2, 0, 0, 0, 3918, 0, 4092, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16416, 16413, 16448, 16768, 16219, 16685, 15910, 16213] avg=16384.0000 max=16768 max_rank=3 min=15910 min_rank=6 imbalance_ratio=1.023438 max_load_reduction=114304 max_load_reduction_pct=0.872070 moved_to_echo=65027 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 3, 0, 875, 4091, 3, 0, 3, 0, 0, 4091, 13, 4093, 7, 3, 2611, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4091, 3, 14, 3, 0, 0, 4094, 2, 3, 3982, 3, 4093, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 4, 3, 3185, 4, 3, 1266, 3, 1, 0, 1, 4093, 0, 5, 3, 3572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 221, 0, 0, 4093, 4085, 3, 4093, 1, 0, 2402, 0, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2064, 1, 0, 0, 2, 0, 4080, 3, 0, 2603, 0, 3665, 3773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3883, 0, 1, 0, 3324, 0, 3, 3, 3, 4093, 1, 0, 4076, 3, 2, 3, 183, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3703, 3341, 0, 22, 0, 5, 0, 3, 0, 0, 0, 4088, 4, 17, 3, 1425, 975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3, 0, 3, 0, 275, 3917, 0, 4093, 227, 0, 3, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16314, 16358, 16239, 15830, 16194, 15754, 17679, 16704] avg=16384.0000 max=17679 max_rank=6 min=15754 min_rank=5 imbalance_ratio=1.079041 max_load_reduction=113393 max_load_reduction_pct=0.865120 moved_to_echo=66331 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 2156, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2613, 521, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 4096, 0, 0, 0, 0, 0, 4075, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 4096, 0, 0, 303, 0, 0, 0, 0, 4096, 3, 0, 0, 3575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 19, 4095, 4095, 0, 4096, 0, 0, 3363, 0, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2067, 0, 0, 1, 0, 0, 4080, 0, 0, 1218, 18, 4096, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 3655, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 183, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3346, 1288, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1026, 0, 1426, 976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 3920, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17579, 16420, 16161, 16633, 15508, 16305, 16256, 16210] avg=16384.0000 max=17579 max_rank=0 min=15508 min_rank=4 imbalance_ratio=1.072937 max_load_reduction=113493 max_load_reduction_pct=0.865883 moved_to_echo=64279 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 22144, 19200, 0, 128, 128, 128, 0, 128, 128, 0, 128, 32768, 128, 4992, 128, 11520, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[512, 32768, 128, 128, 128, 0, 0, 32640, 128, 128, 32384, 128, 32768, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32256, 256, 128, 0, 30464, 0, 128, 128, 128, 32768, 128, 0, 32768, 128, 128, 128, 1536, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 128, 0, 7680, 32768, 128, 0, 128, 0, 128, 32768, 128, 32768, 128, 128, 20992, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 32768, 128, 128, 30464, 128, 128, 6528, 128, 128, 128, 128, 32768, 128, 128, 128, 28672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 0, 896, 31360, 128, 32768, 640, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[0, 0, 384, 0, 0, 2560, 0, 128, 32768, 32768, 128, 32768, 128, 128, 22912, 0, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 16512, 128, 128, 128, 128, 0, 32768, 128, 0, 19072, 256, 31360, 31488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[3271, 0, 0, 4095, 0, 0, 4092, 4083, 0, 0, 0, 1905, 0, 0, 0, 4095, 1, 4096, 0, 481, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 3371, 0, 4083, 4095, 1525, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 0, 1, 1494, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4036, 2, 1, 0, 0, 0, 0, 235, 0, 1, 0, 0, 0, 0, 4096, 4095, 0, 4095, 0, 0, 53, 0, 4096, 0, 4096, 4096, 0, 0, 46, 0, 0, 4096, 0, 2, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 4067, 0, 4095, 4095, 4096, 0, 0, 0, 0, 4089, 4096, 5, 0, 4093, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[2572, 0, 0, 4096, 0, 0, 4096, 4045, 0, 0, 0, 2824, 0, 0, 0, 4096, 0, 4096, 0, 321, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3518, 0, 4085, 4096, 1890, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 842, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4067, 1, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 0, 0, 274, 0, 4096, 0, 4096, 4096, 0, 0, 56, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 4095, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1582, 0, 0, 4093, 0, 0, 4075, 3976, 0, 0, 0, 3288, 0, 0, 0, 4095, 4, 4096, 0, 201, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 4096, 3685, 0, 4045, 4094, 3039, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 4096, 0, 4, 475, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 4095, 0, 2, 0, 0, 0, 0, 133, 0, 1, 0, 0, 0, 0, 4096, 4084, 0, 4093, 0, 0, 51, 0, 4096, 0, 4096, 4096, 0, 0, 79, 0, 0, 4096, 0, 1, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 4024, 0, 4096, 4093, 4096, 0, 0, 0, 0, 4064, 4096, 16, 0, 4080, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[2456, 0, 0, 4092, 0, 0, 4083, 4090, 0, 0, 0, 3056, 0, 0, 0, 4096, 3, 4096, 0, 272, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 4, 0, 0, 4096, 3910, 0, 4080, 4094, 2052, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 13, 155, 0, 0, 0, 4096, 0, 0, 0, 0, 4, 4095, 1, 3, 0, 0, 0, 0, 344, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 4090, 0, 0, 51, 0, 4096, 0, 4096, 4096, 0, 4, 64, 0, 0, 4096, 0, 2, 6, 0, 0, 4096, 0, 0, 0, 4095, 0, 4058, 0, 4090, 4093, 4096, 0, 0, 0, 0, 4072, 4096, 12, 0, 4087, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[1925, 0, 0, 4092, 0, 0, 4070, 4063, 0, 0, 0, 3737, 0, 0, 0, 4096, 0, 4096, 0, 699, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 2539, 0, 4053, 4096, 2120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 4096, 0, 6, 859, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 263, 0, 0, 0, 0, 0, 0, 4096, 4090, 0, 4094, 0, 0, 176, 0, 4096, 0, 4096, 4096, 0, 0, 85, 0, 0, 4096, 0, 12, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 4038, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4053, 4096, 31, 0, 4092, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[3932, 1, 0, 4092, 1, 2, 4093, 2362, 3, 2, 3, 124, 0, 0, 3, 4087, 33, 3190, 14, 2159, 0, 4088, 0, 1, 2, 3, 4093, 0, 242, 0, 0, 0, 4088, 3698, 7, 4093, 4093, 3300, 1, 1, 2, 3, 0, 3, 144, 0, 0, 3, 3, 0, 3, 0, 46, 3, 1, 5, 4093, 3, 0, 292, 0, 0, 3, 4093, 3, 0, 0, 1, 1, 4002, 1174, 19, 3, 0, 0, 4, 9, 0, 0, 3, 3, 0, 3, 4093, 4093, 2, 4091, 0, 0, 0, 0, 3869, 3, 4074, 4092, 3, 0, 400, 3, 3, 3833, 3, 0, 0, 3, 2, 4091, 3, 0, 10, 4089, 2, 4089, 26, 4092, 4093, 4091, 0, 0, 0, 3, 4093, 3979, 0, 3, 4093, 9, 3] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[3834, 4, 1, 4084, 7, 0, 4088, 1974, 8, 0, 8, 11, 0, 0, 10, 4083, 1, 2458, 27, 2664, 0, 4078, 0, 0, 5, 10, 4086, 0, 488, 0, 0, 0, 4077, 3586, 11, 4087, 4086, 3756, 1, 15, 3, 9, 0, 9, 264, 0, 0, 11, 10, 0, 9, 0, 149, 9, 8, 15, 4087, 10, 0, 97, 0, 0, 10, 4086, 9, 0, 0, 0, 0, 3974, 1401, 12, 9, 0, 0, 14, 32, 0, 0, 9, 10, 0, 10, 4084, 4087, 2, 4086, 0, 0, 0, 0, 3741, 8, 4060, 4081, 15, 0, 512, 10, 10, 3674, 12, 0, 0, 9, 6, 4072, 8, 0, 70, 4078, 0, 4088, 117, 4086, 4086, 4075, 0, 4, 0, 11, 4087, 3870, 0, 10, 4087, 103, 9] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[4017, 0, 0, 4096, 0, 0, 4082, 4096, 0, 0, 0, 932, 0, 0, 0, 4096, 0, 4096, 0, 908, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 3353, 0, 4068, 4096, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 4096, 0, 2, 2287, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 4096, 0, 0, 8, 0, 4096, 0, 4096, 4096, 0, 0, 122, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4041, 0, 4096, 4096, 4096, 0, 0, 0, 0, 4067, 4095, 14, 0, 4090, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2572, 0, 0, 3777, 0, 0, 4096, 4045, 0, 0, 0, 2824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 321, 0, 4096, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1692, 1341, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3518, 0, 4085, 2404, 1890, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 842, 0, 0, 0, 4096, 4096, 2794, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 1, 0, 0, 0, 0, 0, 79, 0, 0, 0, 4096, 4096, 3665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 274, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 4096, 0, 2700, 1297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 55, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17314, 15962, 15995, 16356, 16004, 16662, 16342, 16437] avg=16384.0000 max=17314 max_rank=0 min=15962 min_rank=1 imbalance_ratio=1.056763 max_load_reduction=113758 max_load_reduction_pct=0.867905 moved_to_echo=65445 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3271, 0, 0, 3776, 0, 0, 4092, 4083, 0, 0, 0, 1905, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 481, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1692, 1341, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3371, 0, 4083, 2403, 1525, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 0, 1, 1494, 0, 0, 0, 4096, 4095, 2794, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4036, 2, 1, 0, 0, 0, 0, 235, 0, 1, 0, 4096, 4095, 3665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 53, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 4096, 0, 2, 1, 0, 0, 4096, 0, 0, 0, 4096, 0, 2700, 1297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 55, 0, 0, 0, 0, 4089, 4096, 5, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17127, 16121, 15480, 17033, 16132, 16440, 16334, 16405] avg=16384.0000 max=17127 max_rank=0 min=15480 min_rank=2 imbalance_ratio=1.045349 max_load_reduction=113945 max_load_reduction_pct=0.869331 moved_to_echo=65311 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1582, 0, 0, 3770, 0, 0, 4075, 3976, 0, 0, 0, 3288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4096, 0, 201, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 1697, 1346, 323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3685, 0, 4045, 2397, 3039, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 4096, 0, 4, 475, 0, 0, 0, 4096, 4095, 2798, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 2, 0, 0, 0, 0, 133, 0, 1, 0, 4096, 4093, 3665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 4093, 0, 0, 51, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 4096, 0, 1, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 2702, 1298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 0, 0, 48, 0, 0, 0, 0, 4064, 4096, 16, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16691, 15858, 17263, 16055, 16086, 16420, 16371, 16328] avg=16384.0000 max=17263 max_rank=2 min=15858 min_rank=1 imbalance_ratio=1.053650 max_load_reduction=113809 max_load_reduction_pct=0.868294 moved_to_echo=65205 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1925, 0, 0, 3773, 0, 0, 4070, 4063, 0, 0, 0, 3737, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 699, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1692, 1341, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2539, 0, 4053, 2404, 2120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 4096, 0, 6, 859, 0, 0, 0, 4096, 4096, 2794, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 263, 0, 0, 0, 4096, 4096, 3666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4094, 0, 0, 176, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 4096, 0, 12, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 2700, 1297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 0, 0, 0, 55, 0, 0, 0, 0, 4053, 4096, 31, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17568, 16338, 15212, 16428, 16219, 16557, 16385, 16365] avg=16384.0000 max=17568 max_rank=0 min=15212 min_rank=2 imbalance_ratio=1.072266 max_load_reduction=113504 max_load_reduction_pct=0.865967 moved_to_echo=65526 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3834, 4, 1, 3766, 7, 0, 4088, 1974, 8, 0, 8, 11, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2458, 27, 2664, 0, 4078, 0, 0, 5, 10, 4086, 0, 488, 0, 0, 0, 1688, 1334, 318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 3586, 11, 4087, 2398, 3756, 1, 15, 3, 9, 0, 9, 264, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 9, 0, 149, 9, 8, 15, 4087, 10, 0, 97, 0, 0, 10, 4086, 4083, 2784, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 3974, 1401, 12, 9, 0, 0, 14, 32, 0, 0, 9, 4084, 4086, 3656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 10, 0, 4087, 2, 4086, 0, 0, 0, 0, 3741, 8, 4060, 5, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 10, 10, 3674, 12, 0, 0, 9, 6, 4072, 8, 0, 70, 4078, 0, 2686, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 117, 0, 0, 55, 0, 4, 0, 11, 4087, 3870, 0, 10, 4087, 103, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[13711, 17157, 18227, 15787, 17286, 16024, 16439, 16441] avg=16384.0000 max=18227 max_rank=2 min=13711 min_rank=0 imbalance_ratio=1.112488 max_load_reduction=112845 max_load_reduction_pct=0.860939 moved_to_echo=66190 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2456, 0, 0, 3773, 0, 0, 4083, 4090, 0, 0, 0, 3056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 272, 0, 4096, 0, 0, 0, 0, 4092, 0, 0, 4, 0, 0, 1691, 1341, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3910, 0, 4080, 2403, 2052, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 13, 155, 0, 0, 0, 4096, 4096, 2794, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 1, 3, 0, 0, 0, 0, 344, 0, 0, 0, 4096, 4093, 3659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4090, 0, 0, 51, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 64, 0, 0, 4096, 0, 2, 6, 0, 0, 4096, 0, 0, 0, 4095, 0, 2700, 1297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4058, 0, 0, 0, 55, 0, 0, 0, 0, 4072, 4096, 12, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17458, 15914, 16542, 15695, 16295, 16428, 16360, 16380] avg=16384.0000 max=17458 max_rank=0 min=15695 min_rank=3 imbalance_ratio=1.065552 max_load_reduction=113614 max_load_reduction_pct=0.866806 moved_to_echo=65463 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3932, 1, 0, 3773, 1, 2, 4093, 2362, 3, 2, 3, 124, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 3190, 14, 2159, 0, 4088, 0, 1, 2, 3, 4093, 0, 242, 0, 0, 0, 1691, 1339, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3698, 7, 4093, 2402, 3300, 1, 1, 2, 3, 0, 3, 144, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 46, 3, 1, 5, 4093, 3, 0, 292, 0, 0, 3, 4093, 4087, 2791, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 1, 4002, 1174, 19, 3, 0, 0, 4, 9, 0, 0, 3, 4093, 4093, 3661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 4093, 2, 4091, 0, 0, 0, 0, 3869, 3, 4074, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 3, 3, 3833, 3, 0, 0, 3, 2, 4091, 3, 0, 10, 4089, 2, 2696, 1296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 26, 0, 0, 56, 0, 0, 0, 3, 4093, 3979, 0, 3, 4093, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14299, 17174, 17745, 15854, 17066, 16146, 16434, 16354] avg=16384.0000 max=17745 max_rank=2 min=14299 min_rank=0 imbalance_ratio=1.083069 max_load_reduction=113327 max_load_reduction_pct=0.864616 moved_to_echo=66000 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4017, 0, 0, 3777, 0, 0, 4082, 4096, 0, 0, 0, 932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 908, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1692, 1341, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3353, 0, 4068, 2404, 687, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 4096, 0, 2, 2287, 0, 0, 0, 4096, 4096, 2794, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 4096, 4096, 3665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4096, 0, 0, 8, 0, 4096, 0, 4096, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 2700, 1297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 55, 0, 0, 0, 0, 4067, 4095, 14, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16904, 16548, 14608, 17864, 15984, 16395, 16407, 16362] avg=16384.0000 max=17864 max_rank=3 min=14608 min_rank=2 imbalance_ratio=1.090332 max_load_reduction=113208 max_load_reduction_pct=0.863708 moved_to_echo=65148 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 0, 128, 128, 32384, 2688, 128, 128, 0, 0, 128, 1280, 0, 128, 128, 32768, 32768, 29312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 1408, 128, 128, 32128, 128, 128, 128, 128, 128, 32768, 128, 0, 128, 32768, 128, 21632, 10496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32512, 256, 0, 0, 512, 0, 128, 0, 128, 32640, 32512, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[23680, 128, 128, 30208, 128, 128, 32768, 28800, 128, 128, 128, 16000, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 32768, 0, 0, 640, 0, 32256, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 27776, 128, 32640, 19328, 18432, 128, 128, 128, 128, 0, 128, 512, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 0, 512, 128, 128, 128, 32768, 128, 128, 6528, 0, 0, 128, 32768, 32768, 22400, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 30336, 128, 7808, 0, 32768, 128, 128, 128, 128, 32768, 0, 768, 128, 0, 0, 13568, 10752, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 753, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4044, 4096, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2257, 0, 4073, 0, 0, 0, 0, 30, 47, 0, 4096, 0, 2794, 0, 4096, 1263, 0, 0, 0, 4096, 0, 0, 4096, 8, 0, 4096, 0, 17, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1173, 0, 0, 0, 0, 0, 0, 4096, 0, 4091, 0, 4094, 0, 0, 0, 0, 4095, 4015, 4096, 8, 0, 0, 0, 63, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 4096, 0, 4081, 0, 0, 0, 0, 0, 4065, 4096, 4086, 4094, 0, 0, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 517, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4094, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3404, 0, 4093, 0, 0, 0, 0, 8, 9, 0, 4096, 0, 2441, 0, 4096, 1731, 0, 0, 0, 4096, 0, 0, 4096, 5, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 113, 0, 0, 0, 0, 0, 0, 4096, 0, 4094, 0, 4096, 0, 0, 0, 0, 4095, 4088, 4096, 14, 0, 0, 0, 2, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 0, 4089, 0, 0, 0, 0, 0, 4092, 4096, 4073, 4095, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 585, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 3, 0, 4096, 3979, 4094, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2631, 0, 4080, 0, 0, 0, 0, 44, 30, 0, 4096, 0, 3402, 0, 4096, 852, 0, 0, 0, 4096, 0, 0, 4096, 11, 0, 4096, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 796, 0, 0, 0, 0, 0, 0, 4096, 0, 4056, 0, 4091, 0, 0, 0, 0, 4094, 4079, 4096, 25, 0, 0, 0, 65, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 0, 0, 4072, 4096, 4065, 4094, 0, 0, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 548, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 0, 4096, 4073, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1531, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 4096, 0, 3382, 0, 4096, 2298, 0, 0, 0, 4096, 0, 0, 4096, 2, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 442, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4093, 4096, 14, 0, 0, 0, 17, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 0, 0, 4096, 4096, 4086, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[10, 7, 0, 83, 3918, 23, 0, 12, 5, 5, 2, 142, 3413, 10, 10, 0, 3945, 0, 0, 4086, 4086, 4086, 0, 0, 8, 4085, 8, 0, 0, 1, 10, 4086, 11, 551, 6, 11, 0, 0, 7, 0, 4087, 0, 13, 10, 1, 0, 0, 238, 4085, 10, 838, 9, 4085, 3007, 10, 0, 0, 4086, 10, 9, 4057, 1, 0, 3247, 0, 16, 0, 10, 9, 0, 12, 0, 4008, 0, 3, 0, 0, 15, 4, 9, 9, 0, 4077, 10, 4086, 0, 4040, 12, 10, 7, 0, 4084, 4080, 4086, 0, 682, 0, 0, 671, 0, 0, 4086, 0, 9, 0, 0, 3904, 10, 4084, 5, 9, 9, 4083, 10, 4053, 15, 0, 0, 0, 0, 4087, 4083, 4087, 4081, 11, 8, 0, 4068] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4072, 4095, 0, 0, 0, 4096, 0, 11, 6, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3207, 0, 4069, 0, 0, 0, 0, 8, 23, 0, 4096, 0, 2298, 0, 4096, 2514, 0, 0, 0, 4093, 0, 0, 4096, 9, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 158, 0, 0, 0, 0, 0, 0, 4096, 0, 4090, 0, 4096, 0, 0, 0, 0, 4091, 4082, 4096, 35, 0, 0, 0, 14, 4, 0, 4096, 1, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 4096, 0, 4087, 0, 0, 2, 0, 0, 4085, 4096, 4027, 4092, 0, 0, 4, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[3, 0, 0, 31, 3398, 6, 0, 2, 1, 1, 0, 48, 3826, 3, 1, 0, 4055, 0, 0, 4093, 4093, 4093, 0, 0, 2, 4093, 3, 0, 0, 1, 7, 4093, 3, 230, 2, 3, 0, 0, 121, 0, 4093, 0, 4, 3, 3, 0, 0, 72, 4093, 3, 1396, 3, 4093, 2934, 3, 0, 0, 4093, 3, 3, 4085, 10, 0, 3794, 0, 13, 0, 3, 3, 0, 4, 0, 4057, 0, 3, 45, 0, 2, 2, 3, 3, 0, 4086, 3, 4093, 0, 4089, 1, 14, 2, 0, 4092, 4092, 4093, 2, 278, 0, 0, 332, 0, 0, 4093, 0, 3, 0, 0, 4032, 3, 4091, 3, 3, 3, 4092, 3, 4075, 5, 0, 0, 0, 0, 4093, 4093, 4093, 4086, 3, 2, 0, 4081] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4088, 4096, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3040, 0, 4074, 0, 0, 0, 0, 40, 68, 0, 4096, 0, 2721, 0, 4096, 1956, 0, 0, 0, 4096, 0, 0, 4096, 9, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 105, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 4091, 0, 0, 0, 0, 4096, 4088, 4096, 5, 0, 0, 0, 3, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 4096, 0, 4087, 0, 0, 0, 0, 0, 4073, 4096, 4086, 4096, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 548, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 2553, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 4073, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1531, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 4096, 4048, 1543, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3382, 0, 0, 2298, 0, 0, 0, 3851, 0, 0, 4096, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 442, 0, 0, 0, 0, 4096, 4096, 3746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 3639, 245, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 4096, 0, 4086, 47, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15739, 16409, 15729, 17725, 16477, 16294, 16288, 16411] avg=16384.0000 max=17725 max_rank=3 min=15729 min_rank=2 imbalance_ratio=1.081848 max_load_reduction=113347 max_load_reduction_pct=0.864769 moved_to_echo=65470 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 753, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 2553, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4044, 4096, 0, 0, 0, 4096, 0, 5, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2257, 0, 4073, 0, 0, 0, 0, 30, 47, 0, 4095, 4049, 1543, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2794, 0, 0, 1263, 0, 0, 0, 3851, 0, 0, 4096, 8, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1173, 0, 0, 0, 0, 4096, 4096, 3746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 4091, 0, 4094, 0, 0, 0, 0, 0, 4015, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 3637, 245, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 4065, 0, 4086, 47, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15944, 16384, 16504, 16108, 17224, 16203, 16328, 16377] avg=16384.0000 max=17224 max_rank=4 min=15944 min_rank=0 imbalance_ratio=1.051270 max_load_reduction=113848 max_load_reduction_pct=0.868591 moved_to_echo=66132 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 517, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 2553, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 4096, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3404, 0, 4093, 0, 0, 0, 0, 8, 9, 0, 4095, 4049, 1543, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2441, 0, 0, 1731, 0, 0, 0, 3851, 0, 0, 4096, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 113, 0, 0, 0, 0, 4096, 4096, 3746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 4094, 0, 4096, 0, 0, 0, 0, 0, 4088, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 3638, 245, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 4092, 0, 4073, 47, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15708, 16430, 17611, 16220, 16148, 16287, 16270, 16398] avg=16384.0000 max=17611 max_rank=2 min=15708 min_rank=0 imbalance_ratio=1.074890 max_load_reduction=113461 max_load_reduction_pct=0.865639 moved_to_echo=65103 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 585, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 2554, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 3979, 4094, 0, 0, 0, 4096, 0, 15, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2631, 0, 4080, 0, 0, 0, 0, 44, 30, 0, 4094, 4050, 1542, 413, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3402, 0, 0, 852, 0, 0, 0, 3848, 0, 0, 4096, 11, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 796, 0, 0, 0, 0, 4096, 4096, 3745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3992, 0, 4056, 0, 4091, 0, 0, 0, 0, 0, 4079, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4086, 0, 0, 0, 3642, 248, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 4072, 0, 4065, 39, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15778, 16329, 16884, 16305, 16835, 16243, 16337, 16361] avg=16384.0000 max=16884 max_rank=2 min=15778 min_rank=0 imbalance_ratio=1.030518 max_load_reduction=114188 max_load_reduction_pct=0.871185 moved_to_echo=65776 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[10, 7, 0, 83, 3918, 23, 0, 12, 5, 5, 2, 142, 3413, 10, 10, 0, 4085, 4083, 2546, 350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3945, 0, 0, 0, 4086, 4086, 0, 0, 8, 4085, 8, 0, 0, 1, 10, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 551, 6, 11, 0, 0, 7, 0, 4087, 0, 13, 10, 1, 0, 0, 238, 4084, 4039, 1537, 408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 838, 9, 0, 3007, 10, 0, 0, 3842, 10, 9, 4057, 1, 0, 3247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 10, 9, 0, 12, 0, 4008, 0, 3, 0, 0, 15, 4, 9, 4086, 4086, 3735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3977, 10, 4086, 0, 4040, 12, 10, 7, 0, 0, 4080, 0, 0, 682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 671, 0, 0, 4086, 0, 9, 0, 0, 3904, 10, 4084, 5, 9, 9, 3626, 244, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4053, 15, 0, 0, 0, 0, 4087, 0, 4087, 47, 11, 8, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18704, 16276, 15003, 15040, 15993, 16913, 16757, 16386] avg=16384.0000 max=18704 max_rank=0 min=15003 min_rank=2 imbalance_ratio=1.141602 max_load_reduction=112368 max_load_reduction_pct=0.857300 moved_to_echo=66049 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 2554, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4072, 4095, 0, 0, 0, 4096, 0, 11, 6, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3207, 0, 4069, 0, 0, 0, 0, 8, 23, 0, 4091, 4049, 1542, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2298, 0, 0, 2514, 0, 0, 0, 3849, 0, 0, 4096, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 158, 0, 0, 0, 0, 4096, 4096, 3745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 4090, 0, 4096, 0, 0, 0, 0, 0, 4082, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 4, 0, 4096, 1, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 3635, 244, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 2, 0, 0, 4085, 0, 4027, 47, 0, 0, 4, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15269, 16423, 17399, 16862, 16192, 16298, 16281, 16348] avg=16384.0000 max=17399 max_rank=2 min=15269 min_rank=0 imbalance_ratio=1.061951 max_load_reduction=113673 max_load_reduction_pct=0.867256 moved_to_echo=65119 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 0, 31, 3398, 6, 0, 2, 1, 1, 0, 48, 3826, 3, 1, 0, 4093, 4093, 2551, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 0, 4093, 4093, 0, 0, 2, 4093, 3, 0, 0, 1, 7, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 230, 2, 3, 0, 0, 121, 0, 4093, 0, 4, 3, 3, 0, 0, 72, 4092, 4046, 1541, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1396, 3, 0, 2934, 3, 0, 0, 3849, 3, 3, 4085, 10, 0, 3794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 3, 3, 0, 4, 0, 4057, 0, 3, 45, 0, 2, 2, 3, 4093, 4093, 3742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3985, 3, 4093, 0, 4089, 1, 14, 2, 0, 0, 4092, 0, 2, 278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 332, 0, 0, 4093, 0, 3, 0, 0, 4032, 3, 4091, 3, 3, 3, 3630, 244, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4075, 5, 0, 0, 0, 0, 4093, 0, 4093, 47, 3, 2, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18408, 16394, 14622, 16083, 16063, 16562, 16538, 16402] avg=16384.0000 max=18408 max_rank=0 min=14622 min_rank=2 imbalance_ratio=1.123535 max_load_reduction=112664 max_load_reduction_pct=0.859558 moved_to_echo=65565 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 2553, 351, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4088, 4096, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3040, 0, 4074, 0, 0, 0, 0, 40, 68, 0, 4096, 4049, 1543, 410, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2721, 0, 0, 1956, 0, 0, 0, 3851, 0, 0, 4096, 9, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 105, 0, 0, 0, 0, 4096, 4096, 3745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3995, 0, 4093, 0, 4091, 0, 0, 0, 0, 0, 4088, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 3639, 245, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 4073, 0, 4086, 47, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15522, 16427, 17320, 16729, 16140, 16272, 16273, 16389] avg=16384.0000 max=17320 max_rank=2 min=15522 min_rank=0 imbalance_ratio=1.057129 max_load_reduction=113752 max_load_reduction_pct=0.867859 moved_to_echo=65074 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 19328, 128, 0, 16640, 128, 0, 0, 30848, 128, 128, 32768, 128, 0, 31744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 0, 32768, 0, 128, 2944, 0, 128, 128, 128, 32768, 32768, 29952, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 32000, 128, 32768, 0, 32768, 128, 128, 128, 0, 0, 32640, 0, 128, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 32768, 128, 0, 128, 0, 0, 32768, 0, 32640, 384, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 0, 128, 10240, 128, 0, 128, 128, 128, 128, 256, 31872, 128, 128, 0, 32768, 32768, 20480, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 896, 128, 128, 0, 0, 16256, 0, 32768, 0, 128, 128, 128, 256, 256, 384, 32768, 32384, 12416, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 0, 1280, 128, 0, 32768, 128, 128, 0, 0, 32512, 128, 32768, 128, 128, 128, 29184, 2048, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32640, 128, 0, 0, 32640, 32768, 0, 0, 128, 32768, 128, 128, 128, 128, 128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 1508, 4096, 0, 4096, 4096, 4096, 0, 4093, 4077, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 17, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 17, 0, 4087, 0, 0, 0, 4096, 338, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 1145, 0, 1220, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 4096, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4096, 1275, 4095, 0, 4096, 4096, 4096, 0, 4096, 4009, 0, 4096, 0, 0, 0, 0, 74, 0, 0, 0, 4095, 0, 15, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 82, 0, 4088, 0, 0, 0, 4096, 262, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 1, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 7, 0, 0, 1440, 0, 1114, 4096, 0, 0, 4, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 4096, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 0, 4096, 1403, 4093, 0, 4096, 4096, 4096, 0, 4088, 4060, 0, 4096, 0, 0, 0, 0, 99, 0, 0, 0, 4087, 0, 92, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 6, 0, 4041, 0, 0, 0, 4096, 566, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 3, 0, 0, 0, 4096, 4096, 38, 0, 0, 0, 0, 0, 2, 4096, 10, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 15, 0, 0, 741, 0, 1306, 4096, 0, 0, 48, 0, 0, 0, 0, 4075, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 4096, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4096, 762, 4092, 0, 4096, 4096, 4096, 0, 4096, 4080, 0, 4096, 0, 0, 0, 0, 80, 0, 0, 0, 4077, 0, 101, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 38, 0, 4052, 0, 0, 0, 4096, 572, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 4096, 82, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 14, 0, 0, 267, 0, 2596, 4096, 0, 0, 50, 0, 0, 0, 0, 4073, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3732, 4094, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[8, 7, 4, 13, 4086, 8, 0, 3949, 2229, 4086, 0, 4079, 4074, 3942, 9, 4083, 4085, 10, 4065, 13, 0, 0, 0, 33, 0, 0, 66, 4062, 0, 8, 0, 0, 4078, 12, 0, 0, 4084, 12, 9, 0, 0, 3419, 10, 10, 10, 3905, 2903, 3, 3598, 3541, 15, 9, 10, 11, 4086, 0, 1, 0, 4084, 177, 10, 11, 10, 4023, 10, 4085, 0, 9, 0, 15, 10, 0, 3844, 2922, 0, 11, 9, 0, 0, 18, 0, 4085, 0, 0, 0, 0, 11, 3936, 0, 0, 10, 0, 16, 4081, 8, 4036, 0, 177, 8, 3207, 0, 1745, 3811, 40, 548, 212, 0, 10, 33, 200, 2547, 0, 0, 0, 0, 0, 0, 0, 333, 61, 11, 10, 10, 1854, 4087, 4080, 12, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 0, 4096, 1600, 4090, 0, 4096, 4096, 4096, 0, 4096, 4077, 0, 4096, 0, 0, 0, 0, 37, 0, 0, 0, 4089, 0, 128, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 31, 0, 4079, 0, 0, 0, 4096, 108, 0, 4096, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 5, 0, 0, 0, 4096, 4096, 13, 0, 0, 0, 0, 0, 3, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 14, 0, 0, 1015, 0, 1336, 4096, 0, 0, 11, 0, 0, 0, 0, 4086, 0, 4, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3952, 4092, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[3, 2, 2, 4, 4093, 3, 0, 4045, 1743, 4093, 0, 4092, 4091, 4038, 3, 4092, 4080, 3, 4084, 4, 1, 0, 0, 49, 0, 0, 16, 4081, 0, 6, 0, 20, 4090, 3, 0, 0, 4092, 4, 3, 0, 0, 3788, 3, 3, 3, 4022, 1987, 0, 3882, 3841, 4, 3, 3, 3, 4093, 0, 0, 0, 4092, 65, 3, 3, 0, 4076, 3, 4093, 0, 3, 0, 0, 3, 0, 4044, 3503, 0, 1, 2, 0, 0, 3, 0, 4093, 0, 1, 0, 0, 3, 4049, 0, 0, 3, 0, 4, 4092, 1, 4072, 0, 46, 3, 2886, 0, 1675, 3982, 0, 260, 94, 0, 3, 13, 20, 3243, 0, 0, 0, 1, 3, 0, 0, 162, 7, 3, 3, 3, 1789, 4093, 4092, 4, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4096, 2136, 4096, 0, 4096, 4096, 4096, 0, 4094, 4062, 0, 4096, 0, 0, 0, 0, 62, 0, 0, 0, 4093, 0, 668, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 8, 0, 4070, 0, 0, 0, 4096, 314, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 8, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 13, 0, 0, 1027, 0, 34, 4096, 0, 0, 18, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 4096, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 1508, 0, 0, 0, 0, 4096, 0, 2562, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4096, 0, 0, 0, 0, 5, 0, 0, 0, 4096, 0, 17, 0, 0, 1706, 1531, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3363, 0, 0, 17, 0, 4087, 0, 0, 0, 4096, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 163, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 2227, 2101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1145, 0, 1220, 4096, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 4096, 1381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 4096, 4096, 0, 0, 2715, 1995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16358, 16232, 15997, 16547, 16616, 16413, 16035, 16874] avg=16384.0000 max=16874 max_rank=7 min=15997 min_rank=2 imbalance_ratio=1.029907 max_load_reduction=114198 max_load_reduction_pct=0.871262 moved_to_echo=65938 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4096, 1275, 0, 0, 0, 0, 4096, 0, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 0, 4096, 0, 0, 0, 0, 74, 0, 0, 0, 4095, 0, 15, 0, 0, 1706, 1532, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3363, 0, 0, 82, 0, 4088, 0, 0, 0, 4096, 262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 163, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 0, 4096, 4096, 1, 0, 0, 0, 0, 0, 2227, 2102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1440, 0, 1114, 4096, 0, 0, 4, 0, 0, 0, 0, 4094, 0, 4096, 1381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 4096, 4096, 0, 0, 2714, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16126, 16231, 15987, 16547, 16618, 16414, 16232, 16917] avg=16384.0000 max=16917 max_rank=7 min=15987 min_rank=2 imbalance_ratio=1.032532 max_load_reduction=114155 max_load_reduction_pct=0.870934 moved_to_echo=66181 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4089, 0, 0, 4096, 1403, 0, 0, 0, 0, 4096, 0, 2554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 0, 4096, 0, 0, 0, 0, 99, 0, 0, 0, 4087, 0, 92, 0, 0, 1712, 1534, 707, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3356, 0, 0, 6, 0, 4041, 0, 0, 0, 4096, 566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 155, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3, 0, 0, 0, 4096, 4096, 38, 0, 0, 0, 0, 0, 2229, 2102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 10, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 741, 0, 1306, 4096, 0, 0, 48, 0, 0, 0, 0, 4075, 0, 4096, 1381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 4096, 4096, 0, 0, 2712, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16238, 16387, 16161, 16538, 16658, 16429, 15758, 16903] avg=16384.0000 max=16903 max_rank=7 min=15758 min_rank=6 imbalance_ratio=1.031677 max_load_reduction=114169 max_load_reduction_pct=0.871040 moved_to_echo=65748 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4096, 762, 0, 0, 0, 0, 4096, 0, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 0, 4096, 0, 0, 0, 0, 80, 0, 0, 0, 4077, 0, 101, 0, 0, 1706, 1532, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3363, 0, 0, 38, 0, 4052, 0, 0, 0, 4096, 572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 163, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 4096, 82, 0, 0, 0, 0, 0, 2227, 2101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 267, 0, 2596, 4096, 0, 0, 50, 0, 0, 0, 0, 4073, 0, 4096, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3732, 4094, 4096, 0, 0, 2712, 1995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15612, 16376, 16217, 16546, 16699, 16419, 16572, 16631] avg=16384.0000 max=16699 max_rank=4 min=15612 min_rank=0 imbalance_ratio=1.019226 max_load_reduction=114373 max_load_reduction_pct=0.872597 moved_to_echo=66321 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[8, 7, 4, 13, 4086, 8, 0, 3949, 2229, 0, 0, 0, 0, 3942, 9, 2556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 10, 4065, 13, 0, 0, 0, 33, 0, 0, 66, 4062, 0, 8, 0, 0, 1701, 1527, 702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 12, 0, 0, 3353, 12, 9, 0, 0, 3419, 10, 10, 10, 3905, 2903, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3598, 3541, 15, 9, 10, 11, 4086, 0, 1, 0, 163, 177, 10, 11, 10, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4085, 0, 9, 0, 15, 10, 0, 3844, 2922, 0, 11, 9, 0, 0, 18, 2220, 2091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 11, 3936, 0, 0, 10, 0, 16, 4081, 8, 4036, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 177, 8, 3207, 0, 1745, 3811, 40, 548, 212, 0, 10, 33, 200, 2547, 0, 4079, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 61, 11, 10, 10, 1854, 4087, 4080, 12, 0, 2707, 1983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16811, 16272, 17724, 15665, 15244, 16212, 17996, 15148] avg=16384.0000 max=17996 max_rank=6 min=15148 min_rank=7 imbalance_ratio=1.098389 max_load_reduction=113076 max_load_reduction_pct=0.862701 moved_to_echo=64600 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4091, 0, 0, 4096, 1600, 0, 0, 0, 0, 4096, 0, 2564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4096, 0, 0, 0, 0, 37, 0, 0, 0, 4089, 0, 128, 0, 0, 1706, 1532, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3363, 0, 0, 31, 0, 4079, 0, 0, 0, 4096, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 163, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 5, 0, 0, 0, 4096, 4096, 13, 0, 0, 0, 0, 0, 2227, 2102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1015, 0, 1336, 4096, 0, 0, 11, 0, 0, 0, 0, 4086, 0, 4096, 1380, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3952, 4092, 4096, 0, 0, 2710, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16447, 16369, 15773, 16544, 16632, 16422, 16034, 16851] avg=16384.0000 max=16851 max_rank=7 min=15773 min_rank=2 imbalance_ratio=1.028503 max_load_reduction=114221 max_load_reduction_pct=0.871437 moved_to_echo=65939 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[3, 2, 2, 4, 4093, 3, 0, 4045, 1743, 0, 0, 0, 0, 4038, 3, 2561, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 3, 4084, 4, 1, 0, 0, 49, 0, 0, 16, 4081, 0, 6, 0, 20, 1705, 1531, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 0, 0, 3360, 4, 3, 0, 0, 3788, 3, 3, 3, 4022, 1987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3882, 3841, 4, 3, 3, 3, 4093, 0, 0, 0, 162, 65, 3, 3, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 3, 0, 0, 3, 0, 4044, 3503, 0, 1, 2, 0, 0, 3, 2225, 2100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 0, 0, 3, 4049, 0, 0, 3, 0, 4, 4092, 1, 4072, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 3, 2886, 0, 1675, 3982, 0, 260, 94, 0, 3, 13, 20, 3243, 0, 4092, 1381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 162, 7, 3, 3, 3, 1789, 4093, 4092, 4, 0, 2712, 1991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16497, 16283, 17266, 16138, 15980, 16347, 17698, 14863] avg=16384.0000 max=17698 max_rank=6 min=14863 min_rank=7 imbalance_ratio=1.080200 max_load_reduction=113374 max_load_reduction_pct=0.864975 moved_to_echo=64888 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4096, 2136, 0, 0, 0, 0, 4096, 0, 2563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 4096, 0, 0, 0, 0, 62, 0, 0, 0, 4093, 0, 668, 0, 0, 1706, 1531, 704, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3363, 0, 0, 8, 0, 4070, 0, 0, 0, 4096, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 163, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 8, 0, 0, 0, 0, 0, 2227, 2102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 1027, 0, 34, 4096, 0, 0, 18, 0, 0, 0, 0, 4082, 0, 4096, 1381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 4096, 4096, 0, 0, 2715, 1994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16983, 16922, 15947, 16547, 16625, 16416, 14747, 16885] avg=16384.0000 max=16983 max_rank=0 min=14747 min_rank=6 imbalance_ratio=1.036560 max_load_reduction=114089 max_load_reduction_pct=0.870430 moved_to_echo=64673 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 32640, 12672, 0, 0, 0, 0, 32640, 128, 20608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32128, 32000, 128, 128, 128, 128, 32768, 0, 128, 0, 1408, 256, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 0, 0, 27008, 128, 128, 256, 0, 31744, 128, 128, 128, 32512, 7168, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 128, 0, 0, 128, 32640, 0, 0, 128, 0, 128, 32768, 128, 32768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32640, 128, 32768, 128, 128, 0, 0, 512, 0, 0, 128, 32768, 0, 1152, 0, 128, 13696, 12288, 5632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 128, 0, 0, 512, 128, 128, 128, 128, 27392, 32768, 32768, 128, 0, 21760, 16000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 256, 128, 11776, 0, 11136, 32384, 128, 896, 512, 0, 128, 128, 256, 30336, 0, 32768, 11136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 0, 128, 128, 128, 128, 0, 32512, 31104, 256, 128, 128, 0, 0, 128, 17920, 16896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 49, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 501, 4096, 0, 0, 0, 0, 3874, 0, 4095, 0, 0, 1, 4096, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 3680, 0, 0, 0, 1, 4078, 0, 0, 1171, 4095, 0, 4096, 0, 0, 4096, 0, 0, 4095, 0, 4096, 0, 4096, 4095, 0, 0, 0, 4096, 0, 24, 0, 0, 0, 0, 0, 0, 0, 3004, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 166, 4096, 0, 0, 0, 0, 4039, 0, 4096, 1, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3661, 0, 0, 0, 0, 4006, 0, 0, 2271, 4096, 0, 4096, 0, 0, 4048, 0, 0, 4096, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 6, 0, 0, 6, 0, 0, 0, 0, 2278, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 3, 0, 0, 4096, 0, 0, 11, 0, 43, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4092, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 9, 0, 4096, 0, 4096, 0, 0, 0, 398, 4096, 0, 0, 0, 0, 4088, 0, 4092, 0, 0, 4, 4096, 4096, 0, 5, 2, 0, 0, 0, 0, 0, 3962, 0, 0, 0, 0, 4057, 0, 0, 3344, 4092, 0, 4096, 0, 0, 4096, 0, 0, 4088, 0, 4096, 0, 4096, 4094, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 478, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 110, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 6, 0, 4096, 0, 4096, 0, 0, 0, 440, 4096, 0, 0, 0, 0, 4054, 0, 4091, 0, 0, 4, 4096, 4096, 0, 6, 1, 0, 0, 0, 0, 0, 3674, 0, 0, 0, 0, 4008, 0, 0, 2889, 4092, 0, 4096, 0, 0, 4094, 0, 0, 4094, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1199, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4079, 0, 0, 30, 4048, 120, 0, 89, 0, 3930, 13, 0, 255, 10, 0, 10, 4060, 1, 0, 0, 4037, 28, 0, 10, 12, 4087, 4083, 0, 12, 4083, 3934, 3977, 0, 0, 22, 9, 12, 6, 0, 10, 11, 242, 51, 4, 0, 4085, 7, 0, 3452, 0, 0, 9, 4064, 11, 3951, 151, 0, 0, 95, 4086, 7, 11, 10, 0, 916, 44, 3734, 303, 13, 42, 3916, 4080, 17, 0, 0, 3, 12, 0, 11, 10, 4086, 4, 3, 9, 104, 4088, 0, 0, 4056, 4081, 0, 4084, 0, 7, 2454, 9, 0, 4074, 9, 4084, 0, 3940, 4085, 0, 0, 134, 4086, 8, 2924, 0, 0, 281, 11, 10, 0, 11, 1975, 0, 0, 0, 10, 4085, 4017, 0, 22, 0, 6, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 57, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 29, 0, 4096, 0, 4096, 0, 0, 0, 413, 4096, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 5, 4096, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 3939, 0, 0, 0, 0, 4067, 0, 0, 3275, 4087, 0, 4096, 0, 0, 4096, 0, 0, 4090, 0, 4096, 0, 4096, 4094, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 4, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4090, 0, 0, 6, 4081, 53, 0, 33, 0, 4020, 4, 0, 141, 3, 0, 3, 4086, 1, 0, 0, 4072, 5, 0, 3, 4, 4093, 4088, 0, 4, 4092, 4052, 4054, 0, 1, 6, 3, 3, 0, 0, 3, 4, 79, 22, 1, 0, 4092, 2, 0, 3724, 0, 0, 3, 4087, 3, 4027, 55, 0, 0, 109, 4093, 4, 3, 3, 0, 1402, 12, 3942, 117, 3, 13, 4008, 4092, 5, 0, 0, 2, 4, 0, 3, 3, 4088, 3, 2, 3, 44, 4093, 0, 0, 3959, 4090, 0, 4092, 0, 2, 3376, 3, 0, 4090, 3, 4093, 0, 4043, 4092, 0, 0, 75, 4093, 3, 1446, 0, 0, 113, 3, 3, 0, 3, 2142, 3, 0, 0, 3, 4093, 4083, 0, 9, 2, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 61, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 26, 0, 4096, 0, 4096, 0, 0, 0, 1266, 4096, 0, 0, 0, 0, 4078, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 3845, 0, 0, 0, 0, 4068, 0, 0, 1120, 4093, 0, 4096, 0, 0, 4096, 0, 0, 4093, 0, 4096, 0, 4096, 4096, 0, 0, 0, 4096, 0, 580, 0, 0, 0, 0, 0, 0, 0, 1340, 5, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 2213, 1614, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 53, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 166, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4039, 0, 4096, 1, 0, 0, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 492, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3661, 0, 0, 0, 0, 4006, 0, 0, 2271, 1883, 0, 0, 0, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3431, 0, 4096, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2278, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 2429, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16289, 16437, 16329, 16243, 17124, 15869, 15731, 17050] avg=16384.0000 max=17124 max_rank=4 min=15731 min_rank=6 imbalance_ratio=1.045166 max_load_reduction=113948 max_load_reduction_pct=0.869354 moved_to_echo=65774 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 49, 0, 2213, 1614, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 53, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 4096, 0, 4096, 0, 0, 0, 501, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3874, 0, 4095, 0, 0, 1, 4096, 4096, 0, 1, 1, 0, 0, 0, 0, 0, 492, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3680, 0, 0, 0, 1, 4078, 0, 0, 1171, 1882, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 3431, 0, 4096, 4095, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3004, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 2429, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16338, 16436, 16329, 16583, 16961, 14908, 15741, 17776] avg=16384.0000 max=17776 max_rank=7 min=14908 min_rank=5 imbalance_ratio=1.084961 max_load_reduction=113296 max_load_reduction_pct=0.864380 moved_to_echo=65386 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 110, 0, 2211, 1614, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 53, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 6, 0, 4096, 0, 4096, 0, 0, 0, 440, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4054, 0, 4091, 0, 0, 4, 4096, 4096, 0, 6, 1, 0, 0, 0, 0, 0, 492, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3674, 0, 0, 0, 0, 4008, 0, 0, 2889, 1881, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3431, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1199, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 2429, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16400, 16437, 16329, 16525, 17145, 16546, 15717, 15973] avg=16384.0000 max=17145 max_rank=4 min=15717 min_rank=6 imbalance_ratio=1.046448 max_load_reduction=113927 max_load_reduction_pct=0.869194 moved_to_echo=65381 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4079, 0, 0, 30, 4048, 120, 0, 89, 0, 3930, 13, 0, 255, 10, 0, 10, 2205, 1609, 172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 1, 0, 0, 4037, 28, 0, 10, 12, 0, 53, 0, 12, 0, 3934, 3977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 9, 12, 6, 0, 10, 11, 242, 51, 4, 0, 4085, 7, 0, 4086, 4084, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3452, 0, 0, 9, 4064, 11, 3951, 151, 0, 0, 95, 3782, 7, 11, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 916, 44, 3734, 303, 13, 42, 3916, 4080, 17, 0, 0, 3, 12, 0, 11, 10, 490, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4, 3, 9, 104, 4088, 0, 0, 4056, 1876, 0, 0, 0, 7, 2454, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4074, 9, 3422, 0, 3940, 4085, 0, 0, 134, 0, 8, 2924, 0, 0, 281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 10, 0, 11, 1975, 0, 0, 0, 10, 4085, 4017, 0, 22, 0, 6, 0, 4087, 2421, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16570, 16124, 16658, 15543, 13895, 16696, 18877, 16709] avg=16384.0000 max=18877 max_rank=6 min=13895 min_rank=4 imbalance_ratio=1.152161 max_load_reduction=112195 max_load_reduction_pct=0.855980 moved_to_echo=66177 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 3, 0, 0, 4096, 0, 0, 11, 0, 43, 0, 2211, 1614, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 53, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 9, 0, 4096, 0, 4096, 0, 0, 0, 398, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4092, 0, 0, 4, 4096, 4096, 0, 5, 2, 0, 0, 0, 0, 0, 492, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 0, 0, 0, 0, 4057, 0, 0, 3344, 1881, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 3431, 0, 4096, 4094, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 478, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4092, 2429, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16343, 16437, 16329, 16486, 17180, 17340, 15711, 15246] avg=16384.0000 max=17340 max_rank=5 min=15246 min_rank=7 imbalance_ratio=1.058350 max_load_reduction=113732 max_load_reduction_pct=0.867706 moved_to_echo=65477 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 57, 0, 2214, 1619, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 46, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 29, 0, 4096, 0, 4096, 0, 0, 0, 413, 3785, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4095, 0, 0, 5, 4096, 4096, 0, 7, 0, 0, 0, 0, 0, 0, 496, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3939, 0, 0, 0, 0, 4067, 0, 0, 3275, 1873, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 3424, 0, 4096, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 509, 4, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 2431, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16356, 16430, 16330, 16515, 17201, 17250, 15705, 15285] avg=16384.0000 max=17250 max_rank=5 min=15285 min_rank=7 imbalance_ratio=1.052856 max_load_reduction=113822 max_load_reduction_pct=0.868393 moved_to_echo=65441 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4090, 0, 0, 6, 4081, 53, 0, 33, 0, 4020, 4, 0, 141, 3, 0, 3, 2210, 1611, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 1, 0, 0, 4072, 5, 0, 3, 4, 0, 53, 0, 4, 0, 4052, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 3, 3, 0, 0, 3, 4, 79, 22, 1, 0, 4092, 2, 0, 4093, 4092, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3724, 0, 0, 3, 4087, 3, 4027, 55, 0, 0, 109, 3788, 4, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1402, 12, 3942, 117, 3, 13, 4008, 4092, 5, 0, 0, 2, 4, 0, 3, 3, 491, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3, 2, 3, 44, 4093, 0, 0, 3959, 1880, 0, 0, 0, 2, 3376, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 3, 3429, 0, 4043, 4092, 0, 0, 75, 0, 3, 1446, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 3, 2142, 3, 0, 0, 3, 4093, 4083, 0, 9, 2, 1, 0, 4093, 2424, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16428, 16334, 16439, 15806, 14402, 17453, 17294, 16916] avg=16384.0000 max=17453 max_rank=5 min=14402 min_rank=4 imbalance_ratio=1.065247 max_load_reduction=113619 max_load_reduction_pct=0.866844 moved_to_echo=66065 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 61, 0, 2212, 1614, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 53, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 26, 0, 4096, 0, 4096, 0, 0, 0, 1266, 3791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4096, 0, 0, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 492, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3845, 0, 0, 0, 0, 4068, 0, 0, 1120, 1881, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 3431, 0, 4096, 4096, 0, 0, 0, 0, 0, 580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1340, 5, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 2429, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16348, 16437, 16329, 17371, 17164, 15010, 16296, 16117] avg=16384.0000 max=17371 max_rank=3 min=15010 min_rank=5 imbalance_ratio=1.060242 max_load_reduction=113701 max_load_reduction_pct=0.867470 moved_to_echo=64587 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 0, 0, 128, 32768, 256, 128, 128, 0, 32640, 128, 0, 512, 128, 384, 128, 17792, 12928, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[31872, 0, 128, 128, 32768, 128, 32640, 256, 0, 0, 3456, 30336, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[30976, 128, 128, 128, 256, 32512, 0, 0, 22144, 15104, 0, 0, 0, 128, 30464, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 0, 32768, 128, 0, 128, 128, 0, 512, 0, 128, 0, 32640, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[26624, 128, 32256, 512, 128, 128, 32512, 32768, 128, 128, 128, 128, 128, 0, 128, 128, 3968, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 128, 128, 128, 0, 128, 128, 384, 128, 128, 0, 32768, 128, 0, 32768, 32768, 32384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 128, 27520, 0, 32640, 32768, 128, 0, 256, 0, 128, 4992, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 0, 128, 12928, 128, 0, 0, 128, 32768, 32768, 0, 128, 128, 128, 0, 32768, 19456, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3812, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3856, 0, 0, 0, 0, 0, 0, 0, 2015, 0, 0, 4, 0, 4096, 0, 4096, 4090, 0, 0, 4096, 0, 4096, 3915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 16, 4096, 3951, 0, 4096, 0, 4096, 0, 0, 4096, 1, 4096, 0, 0, 4009, 0, 0, 4096, 0, 0, 0, 4095, 14, 0, 0, 0, 0, 0, 3051, 0, 0, 0, 4096, 0, 0, 0, 4035, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4043, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 10, 0, 0, 0, 3783, 8, 0, 0, 0, 0, 0, 0, 1217, 0, 0, 2, 0, 4096, 0, 4096, 4088, 0, 0, 4096, 0, 4096, 4079, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2, 4095, 4062, 0, 4096, 0, 4095, 0, 0, 4095, 0, 4096, 0, 0, 4050, 0, 0, 4096, 0, 0, 0, 4096, 31, 0, 0, 2, 0, 0, 3319, 0, 0, 0, 4095, 1, 0, 0, 4081, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 4065, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3907, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 20, 0, 0, 0, 2988, 41, 0, 0, 0, 0, 0, 0, 2197, 0, 0, 0, 0, 4096, 0, 4096, 4084, 0, 0, 4096, 0, 4096, 3972, 0, 21, 3, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 6, 4096, 3834, 0, 4096, 0, 4092, 0, 0, 4096, 0, 4096, 0, 0, 4038, 0, 0, 4096, 0, 0, 0, 4096, 85, 0, 0, 7, 0, 0, 3686, 0, 0, 0, 4096, 4, 0, 0, 3917, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[8, 14, 3774, 11, 11, 10, 0, 0, 4076, 10, 9, 4085, 66, 9, 135, 23, 4049, 12, 4086, 0, 0, 0, 0, 4083, 4072, 8, 3915, 11, 12, 202, 4073, 9, 11, 17, 0, 0, 0, 0, 33, 0, 3265, 0, 85, 12, 0, 0, 0, 52, 915, 0, 0, 210, 3, 3999, 0, 4072, 3852, 0, 9, 3855, 0, 4085, 4082, 0, 0, 0, 8, 0, 0, 11, 0, 10, 10, 4053, 10, 9, 11, 19, 1188, 4084, 4086, 278, 4085, 10, 4073, 8, 0, 4086, 471, 3811, 7, 0, 2538, 0, 0, 4085, 0, 9, 0, 3948, 0, 0, 363, 0, 0, 50, 4059, 0, 0, 8, 4086, 1004, 0, 10, 4083, 0, 2976, 11, 19, 0, 17, 10, 0, 4087, 4042, 0, 9, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 7, 0, 4096, 0, 0, 4077, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3976, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 24, 0, 0, 0, 3784, 44, 0, 0, 0, 0, 0, 0, 1203, 0, 0, 15, 0, 4096, 0, 4096, 4093, 0, 0, 4096, 0, 4096, 4005, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 4094, 3971, 0, 4096, 0, 4089, 0, 0, 4094, 0, 4096, 0, 1, 4037, 0, 0, 4096, 2, 0, 0, 4096, 64, 0, 0, 12, 0, 0, 3686, 0, 0, 0, 4091, 0, 0, 0, 3869, 0, 4096, 0, 0, 0, 0, 0, 6, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 4075, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4024, 0, 0, 0, 4096, 0, 0, 0, 8, 0, 8, 0, 0, 0, 3535, 21, 0, 0, 0, 0, 0, 0, 1335, 0, 0, 6, 0, 4096, 0, 4096, 4095, 0, 0, 4096, 0, 4096, 3885, 0, 11, 4, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 3, 4092, 3963, 0, 4096, 0, 4092, 0, 0, 4093, 0, 4096, 0, 4, 4033, 0, 0, 4096, 1, 0, 0, 4096, 65, 0, 0, 8, 0, 0, 3842, 0, 0, 0, 4092, 0, 0, 0, 3953, 0, 4096, 0, 0, 0, 0, 0, 3, 4096, 4096, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[4, 4, 3916, 3, 4, 0, 0, 0, 4089, 3, 3, 4093, 24, 3, 46, 7, 4089, 4, 4093, 0, 0, 0, 0, 4092, 4091, 3, 4074, 3, 3, 72, 4088, 3, 3, 5, 0, 0, 0, 0, 16, 0, 3714, 0, 14, 5, 0, 1, 0, 8, 590, 0, 0, 153, 3, 4072, 0, 4091, 3978, 0, 3, 4002, 0, 4093, 4065, 0, 0, 0, 1, 0, 1, 3, 0, 3, 3, 4076, 3, 3, 4, 8, 626, 4093, 4093, 115, 4092, 4, 4086, 3, 0, 4093, 127, 3997, 3, 0, 3353, 0, 0, 4093, 0, 3, 0, 4026, 0, 0, 125, 0, 2, 0, 4030, 0, 0, 3, 4093, 537, 0, 3, 4079, 0, 3448, 1, 22, 0, 6, 3, 0, 4093, 4082, 0, 3, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4026, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3820, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 18, 0, 0, 0, 3901, 63, 0, 0, 0, 0, 0, 0, 1720, 0, 0, 2, 0, 4096, 0, 4096, 4093, 0, 0, 4096, 0, 4096, 3968, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 4012, 0, 4096, 0, 4096, 0, 0, 4096, 0, 4096, 0, 0, 4089, 0, 0, 4096, 0, 0, 0, 4096, 71, 0, 0, 6, 0, 0, 2996, 0, 0, 0, 4096, 3, 0, 0, 4072, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 3187, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 122, 4096, 0, 3812, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3856, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2015, 0, 0, 4, 0, 2848, 0, 0, 4090, 0, 0, 4096, 0, 0, 3915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 16, 4096, 4096, 3666, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3951, 0, 0, 0, 4096, 0, 0, 354, 1, 4096, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 14, 0, 0, 0, 0, 0, 3051, 0, 0, 0, 4096, 0, 3775, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4035, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16421, 16222, 16575, 16968, 16169, 16507, 15586, 16624] avg=16384.0000 max=16968 max_rank=3 min=15586 min_rank=6 imbalance_ratio=1.035645 max_load_reduction=114104 max_load_reduction_pct=0.870544 moved_to_echo=64886 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4084, 0, 0, 0, 0, 3187, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 122, 4096, 0, 4043, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 3783, 8, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1217, 0, 0, 2, 0, 2848, 0, 0, 4088, 0, 0, 4096, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 2, 4095, 4096, 3666, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 4095, 0, 0, 353, 0, 4096, 0, 0, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 31, 0, 0, 2, 0, 0, 3319, 0, 0, 0, 4095, 1, 3775, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 4096, 0, 0, 0, 0, 0, 1, 4096, 4096, 0, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16411, 16453, 16520, 16330, 16157, 16656, 15874, 16671] avg=16384.0000 max=16671 max_rank=7 min=15874 min_rank=6 imbalance_ratio=1.017517 max_load_reduction=114401 max_load_reduction_pct=0.872810 moved_to_echo=65358 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[8, 14, 3774, 11, 11, 10, 0, 0, 4076, 10, 9, 4085, 66, 9, 135, 23, 3180, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4049, 12, 0, 0, 0, 0, 0, 122, 4072, 8, 3915, 11, 12, 202, 4073, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 17, 0, 0, 0, 0, 33, 0, 3265, 0, 85, 12, 0, 0, 0, 52, 4086, 4085, 4085, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 915, 0, 0, 210, 3, 2780, 0, 0, 3852, 0, 9, 3855, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 11, 0, 10, 10, 4053, 10, 9, 11, 19, 1188, 4084, 4072, 3655, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 278, 0, 10, 4073, 8, 0, 352, 471, 3811, 7, 0, 2538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3948, 0, 0, 363, 0, 0, 50, 4059, 0, 0, 8, 4086, 1004, 3763, 554, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4083, 0, 2976, 11, 19, 0, 17, 10, 0, 4087, 4042, 0, 9, 0, 294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16346, 16485, 16161, 15706, 17338, 15634, 17844, 15558] avg=16384.0000 max=17844 max_rank=6 min=15558 min_rank=7 imbalance_ratio=1.089111 max_load_reduction=113228 max_load_reduction_pct=0.863861 moved_to_echo=66374 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 4065, 0, 0, 0, 0, 3187, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 122, 4096, 0, 3907, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 20, 0, 0, 0, 2988, 41, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2197, 0, 0, 0, 0, 2848, 0, 0, 4084, 0, 0, 4096, 0, 0, 3972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 3, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 6, 4096, 4096, 3666, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3834, 0, 0, 0, 4092, 0, 0, 354, 0, 4096, 0, 0, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 85, 0, 0, 7, 0, 0, 3686, 0, 0, 0, 4096, 4, 3775, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3917, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16396, 16317, 15768, 17197, 16170, 16414, 16304, 16506] avg=16384.0000 max=17197 max_rank=3 min=15768 min_rank=2 imbalance_ratio=1.049622 max_load_reduction=113875 max_load_reduction_pct=0.868797 moved_to_echo=65394 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 7, 0, 4096, 0, 0, 4077, 0, 0, 0, 0, 3190, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 115, 4096, 0, 3976, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 24, 0, 0, 0, 3784, 44, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1203, 0, 0, 15, 0, 2841, 0, 0, 4093, 0, 0, 4096, 0, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 4094, 4096, 3665, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 0, 4089, 0, 0, 345, 0, 4096, 0, 1, 4037, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 64, 0, 0, 12, 0, 0, 3686, 0, 0, 0, 4091, 0, 3777, 559, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3869, 0, 4096, 0, 0, 0, 0, 0, 6, 4096, 4096, 0, 0, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16417, 16379, 16579, 16253, 16151, 16539, 16287, 16467] avg=16384.0000 max=16579 max_rank=2 min=16151 min_rank=4 imbalance_ratio=1.011902 max_load_reduction=114493 max_load_reduction_pct=0.873512 moved_to_echo=65444 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 5, 0, 4096, 0, 0, 4075, 0, 0, 0, 0, 3185, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 122, 4096, 0, 4024, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8, 0, 0, 0, 3535, 21, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1335, 0, 0, 6, 0, 2848, 0, 0, 4095, 0, 0, 4096, 0, 0, 3885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 3, 4092, 4096, 3665, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 4092, 0, 0, 353, 0, 4096, 0, 4, 4033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 65, 0, 0, 8, 0, 0, 3842, 0, 0, 0, 4092, 0, 3775, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3953, 0, 4096, 0, 0, 0, 0, 0, 3, 4096, 4096, 0, 0, 1, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16404, 16434, 16291, 16265, 16157, 16541, 16434, 16546] avg=16384.0000 max=16546 max_rank=7 min=16157 min_rank=4 imbalance_ratio=1.009888 max_load_reduction=114526 max_load_reduction_pct=0.873764 moved_to_echo=65678 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[4, 4, 3916, 3, 4, 0, 0, 0, 4089, 3, 3, 4093, 24, 3, 46, 7, 3185, 941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 0, 0, 0, 0, 0, 122, 4091, 3, 4074, 3, 3, 72, 4088, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 0, 0, 0, 0, 16, 0, 3714, 0, 14, 5, 0, 1, 0, 8, 4093, 4093, 4093, 431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 590, 0, 0, 153, 3, 2832, 0, 0, 3978, 0, 3, 4002, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 3, 0, 3, 3, 4076, 3, 3, 4, 8, 626, 4093, 4091, 3661, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 115, 0, 4, 4086, 3, 0, 353, 127, 3997, 3, 0, 3353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4026, 0, 0, 125, 0, 2, 0, 4030, 0, 0, 3, 4093, 537, 3771, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4079, 0, 3448, 1, 22, 0, 6, 3, 0, 4093, 4082, 0, 3, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16325, 16552, 16476, 15626, 16775, 16134, 17145, 16039] avg=16384.0000 max=17145 max_rank=6 min=15626 min_rank=3 imbalance_ratio=1.046448 max_load_reduction=113927 max_load_reduction_pct=0.869194 moved_to_echo=66093 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 4026, 0, 0, 0, 0, 3187, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 122, 4096, 0, 3820, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 18, 0, 0, 0, 3901, 63, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1720, 0, 0, 2, 0, 2848, 0, 0, 4093, 0, 0, 4096, 0, 0, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 4096, 3666, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 0, 4096, 0, 0, 354, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 71, 0, 0, 6, 0, 0, 2996, 0, 0, 0, 4096, 3, 3775, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16352, 16230, 16702, 16727, 16155, 16647, 15598, 16661] avg=16384.0000 max=16727 max_rank=3 min=15598 min_rank=6 imbalance_ratio=1.020935 max_load_reduction=114345 max_load_reduction_pct=0.872383 moved_to_echo=65061 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 32384, 128, 128, 128, 128, 0, 32768, 128, 128, 32640, 128, 128, 256, 128, 25600, 7552, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32000, 512, 0, 128, 32768, 128, 0, 2944, 640, 32384, 128, 128, 30208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 0, 1024, 32768, 128, 31616, 128, 128, 384, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 32128, 0, 31104, 128, 128, 0, 128, 128, 128, 32768, 32768, 0, 128, 128, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[11264, 0, 0, 512, 128, 22784, 0, 0, 32384, 0, 128, 32512, 0, 0, 32000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 0, 28928, 256, 128, 128, 0, 128, 0, 128, 32768, 32768, 32768, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 32640, 384, 0, 512, 128, 128, 128, 28672, 0, 0, 128, 32768, 1664, 30208, 4480, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 1920, 32768, 32768, 29312, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 120, 3057, 0, 4096, 0, 4096, 0, 4083, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 2611, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4091, 0, 0, 4096, 4096, 0, 4077, 207, 250, 4096, 11, 4096, 0, 0, 0, 0, 0, 4096, 2488, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 8, 0, 0, 0, 4094, 3564, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 98, 3177, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3483, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 16, 4096, 0, 0, 4096, 4096, 0, 4095, 129, 165, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 1346, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 60, 0, 0, 0, 4096, 3803, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 72, 1846, 0, 4096, 0, 4096, 0, 4093, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 3807, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4095, 0, 0, 4096, 4096, 0, 4088, 157, 305, 4096, 2, 4096, 0, 0, 0, 0, 0, 4096, 2458, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 7, 0, 0, 0, 4094, 3634, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 24, 3499, 0, 4096, 0, 4096, 0, 4088, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 2332, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4081, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4093, 0, 0, 4096, 4096, 0, 4071, 18, 192, 4096, 16, 4096, 0, 0, 0, 0, 0, 4096, 2707, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4095, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 19, 0, 0, 0, 4093, 3509, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 9, 11, 0, 4081, 0, 0, 9, 128, 0, 4049, 7, 0, 0, 1021, 3456, 0, 4026, 0, 4049, 0, 4082, 8, 0, 77, 4085, 0, 0, 11, 10, 0, 0, 0, 1, 3689, 106, 1850, 12, 9, 0, 10, 4079, 0, 10, 11, 9, 23, 0, 34, 4077, 45, 0, 4086, 10, 58, 0, 4009, 14, 15, 10, 6, 12, 0, 4, 0, 180, 7, 25, 207, 11, 10, 9, 0, 4086, 8, 12, 3782, 3973, 9, 4087, 5, 0, 4064, 2484, 4085, 0, 0, 398, 0, 9, 3972, 22, 0, 10, 3888, 0, 13, 0, 8, 0, 69, 0, 4084, 4084, 3880, 11, 4084, 0, 4060, 0, 4084, 4057, 2, 10, 0, 12, 4085, 3746, 0, 4081, 0, 9, 9, 0, 2, 0, 8, 4023] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 3, 3, 0, 4086, 0, 0, 3, 50, 0, 4088, 2, 0, 0, 1361, 3147, 0, 4078, 0, 4087, 0, 4092, 3, 0, 22, 4093, 0, 1, 3, 3, 0, 0, 0, 0, 4038, 33, 2555, 4, 3, 0, 3, 4091, 0, 3, 4, 3, 6, 1, 9, 4090, 11, 0, 4093, 2, 16, 0, 4037, 4, 0, 3, 1, 3, 0, 1, 0, 87, 2, 8, 81, 3, 3, 3, 0, 4093, 3, 3, 3945, 4049, 3, 4093, 2, 53, 4084, 1435, 4093, 0, 0, 190, 0, 3, 4030, 315, 0, 3, 3989, 0, 4, 0, 3, 0, 20, 0, 4093, 4093, 3996, 3, 4093, 0, 4078, 0, 4092, 4092, 5, 3, 0, 3, 4093, 3512, 0, 4092, 0, 3, 3, 0, 5, 0, 3, 4071] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 2335, 0, 4096, 0, 4096, 0, 4086, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 2, 3656, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4092, 0, 0, 4096, 4096, 0, 4081, 29, 393, 4096, 10, 4096, 0, 0, 0, 0, 0, 4096, 1988, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4095, 4096, 48, 0, 0, 0, 4091, 3845, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 2996, 0, 4096, 0, 4096, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3937, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4095, 0, 0, 4096, 4096, 0, 4082, 81, 110, 4096, 4, 4096, 0, 0, 0, 0, 0, 4096, 1549, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 14, 0, 0, 0, 4095, 3603, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 98, 3177, 2997, 1902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3483, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2156, 1938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 16, 4094, 0, 0, 4096, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 165, 4096, 0, 1724, 0, 0, 0, 0, 0, 4096, 1346, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 4096, 3803, 12, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16366, 16392, 16870, 16380, 16399, 15652, 16422, 16591] avg=16384.0000 max=16870 max_rank=2 min=15652 min_rank=5 imbalance_ratio=1.029663 max_load_reduction=114202 max_load_reduction_pct=0.871292 moved_to_echo=65064 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 72, 1846, 2997, 1902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4093, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3807, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2156, 1938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 4093, 0, 0, 4096, 4096, 0, 4088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 305, 4096, 2, 1724, 0, 0, 0, 0, 0, 4096, 2458, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 4094, 3634, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15008, 16390, 17194, 16377, 16384, 16934, 16422, 16363] avg=16384.0000 max=17194 max_rank=2 min=15008 min_rank=0 imbalance_ratio=1.049438 max_load_reduction=113878 max_load_reduction_pct=0.868820 moved_to_echo=66103 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 120, 3057, 2998, 1906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4083, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 12, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2611, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4084, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2160, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 4087, 0, 0, 4096, 4096, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 250, 4096, 11, 1710, 0, 0, 0, 0, 0, 4096, 2488, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 4094, 3564, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16273, 16389, 15997, 16376, 16374, 16954, 16414, 16295] avg=16384.0000 max=16954 max_rank=5 min=15997 min_rank=2 imbalance_ratio=1.034790 max_load_reduction=114118 max_load_reduction_pct=0.870651 moved_to_echo=66037 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 9, 11, 0, 4081, 0, 0, 9, 128, 0, 4049, 7, 0, 0, 1021, 3456, 2988, 1896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4026, 0, 4049, 0, 4082, 8, 0, 77, 4085, 0, 0, 11, 10, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3689, 106, 1850, 12, 9, 0, 10, 4079, 0, 10, 11, 9, 23, 0, 4084, 1096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 4077, 45, 0, 4086, 10, 58, 0, 4009, 14, 15, 10, 6, 12, 0, 4, 2149, 1932, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 7, 25, 207, 11, 10, 9, 0, 4085, 8, 12, 3782, 3973, 9, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4064, 2484, 1720, 0, 0, 398, 0, 9, 3972, 22, 0, 10, 3888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 8, 0, 69, 0, 0, 39, 3880, 11, 0, 0, 4060, 0, 4084, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 10, 0, 12, 4085, 3746, 0, 4081, 0, 9, 9, 0, 2, 0, 8, 4023, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17655, 16355, 14989, 16461, 16405, 16572, 16221, 16414] avg=16384.0000 max=17655 max_rank=0 min=14989 min_rank=2 imbalance_ratio=1.077576 max_load_reduction=113417 max_load_reduction_pct=0.865303 moved_to_echo=65612 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 24, 3499, 2997, 1901, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4088, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2332, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4081, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2155, 1938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4091, 0, 0, 4096, 4096, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 192, 4096, 16, 1724, 0, 0, 0, 0, 0, 4096, 2707, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 4093, 3509, 8, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16614, 16387, 15719, 16366, 16369, 16945, 16423, 16249] avg=16384.0000 max=16945 max_rank=5 min=15719 min_rank=2 imbalance_ratio=1.034241 max_load_reduction=114127 max_load_reduction_pct=0.870720 moved_to_echo=65986 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3, 2335, 2997, 1902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4086, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 3656, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2156, 1938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4091, 0, 0, 4096, 4096, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 393, 4096, 10, 1724, 0, 0, 0, 0, 0, 4096, 1988, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4096, 0, 0, 0, 4096, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 4091, 3845, 5, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15426, 16385, 17045, 16371, 16385, 16432, 16421, 16607] avg=16384.0000 max=17045 max_rank=2 min=15426 min_rank=0 imbalance_ratio=1.040344 max_load_reduction=114027 max_load_reduction_pct=0.869957 moved_to_echo=65845 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 3, 3, 0, 4086, 0, 0, 3, 50, 0, 4088, 2, 0, 0, 1361, 3147, 2996, 1900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 4087, 0, 4092, 3, 0, 22, 4093, 0, 1, 3, 3, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4038, 33, 2555, 4, 3, 0, 3, 4091, 0, 3, 4, 3, 6, 1, 4093, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 4090, 11, 0, 4093, 2, 16, 0, 4037, 4, 0, 3, 1, 3, 0, 1, 2154, 1936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 2, 8, 81, 3, 3, 3, 0, 4091, 3, 3, 3945, 4049, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 53, 4084, 1435, 1723, 0, 0, 190, 0, 3, 4030, 315, 0, 3, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 0, 20, 0, 0, 39, 3996, 3, 0, 0, 4078, 0, 4092, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 0, 3, 4093, 3512, 0, 4092, 0, 3, 3, 0, 5, 0, 3, 4071, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17639, 16390, 15934, 16360, 16374, 15827, 16327, 16221] avg=16384.0000 max=17639 max_rank=0 min=15827 min_rank=5 imbalance_ratio=1.076599 max_load_reduction=113433 max_load_reduction_pct=0.865425 moved_to_echo=64749 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 2996, 2997, 1902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3937, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 2156, 1938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4093, 0, 0, 4096, 4096, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 110, 4096, 4, 1724, 0, 0, 0, 0, 0, 4096, 1549, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 4096, 0, 0, 0, 4096, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4095, 3603, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16091, 16384, 17324, 16381, 16382, 15756, 16422, 16332] avg=16384.0000 max=17324 max_rank=2 min=15756 min_rank=5 imbalance_ratio=1.057373 max_load_reduction=113748 max_load_reduction_pct=0.867828 moved_to_echo=64892 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 32384, 256, 24320, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 128, 128, 32768, 8832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[640, 1536, 32768, 3968, 13824, 0, 0, 640, 0, 128, 32640, 12928, 0, 128, 32512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 384, 32512, 128, 0, 0, 32768, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 32768, 0, 32768, 0, 32768, 128, 0, 128, 32768, 0, 128, 128, 128, 0, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 32768, 128, 0, 32768, 128, 128, 0, 32640, 128, 128, 128, 128, 128, 0, 128, 17280, 15616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[256, 128, 0, 128, 32768, 29312, 128, 32768, 0, 128, 128, 0, 128, 0, 128, 32768, 3456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 128, 128, 256, 0, 32768, 128, 0, 0, 2816, 23552, 24064, 15232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 384, 128, 128, 384, 128, 128, 128, 128, 32768, 128, 128, 32384, 32640, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4073, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 3831, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1208, 0, 0, 1, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 2004, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4095, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 9, 0, 13, 0, 4096, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 57, 0, 1, 4092, 0, 0, 0, 1097, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4048, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4051, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 280, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 3230, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 16, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 6, 0, 1, 4096, 0, 0, 0, 663, 0, 0, 0, 4096, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 4039, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4092, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 93, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 0, 0, 2851, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 22, 0, 0, 0, 4096, 4096, 4095, 0, 31, 0, 51, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 54, 0, 0, 4070, 0, 0, 0, 1104, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 0, 7, 0, 0, 0, 4032, 3887, 3, 37, 16, 3, 0, 0, 4093, 4089, 4, 0, 0, 4092, 0, 1910, 3, 4081, 174, 2, 330, 4092, 0, 0, 25, 418, 3, 3, 34, 3, 0, 4093, 0, 4, 0, 12, 0, 4, 3, 4, 433, 0, 2, 0, 41, 0, 3593, 4092, 0, 4090, 4093, 0, 0, 0, 0, 3863, 0, 5, 3, 25, 3, 0, 1, 4054, 92, 4, 0, 0, 4, 3, 4062, 4041, 4037, 0, 2, 0, 624, 4, 4020, 3, 4089, 12, 3, 0, 0, 4068, 3, 3, 4092, 4092, 0, 4, 4092, 4019, 0, 2, 4092, 0, 4, 4, 0, 4093, 4, 4092, 2412, 442, 0, 4093, 4, 2, 3, 2485, 3, 6, 6, 4092, 732, 3365] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 12, 0, 0, 5, 0, 19, 0, 2, 0, 3964, 3123, 9, 100, 61, 9, 0, 0, 4085, 4070, 13, 0, 0, 4081, 0, 545, 8, 4060, 313, 18, 729, 4083, 0, 18, 81, 1724, 10, 10, 53, 9, 0, 4080, 0, 14, 0, 40, 0, 12, 10, 13, 492, 0, 6, 0, 121, 0, 3002, 4052, 0, 4079, 4085, 0, 0, 0, 0, 3683, 0, 15, 11, 86, 10, 0, 0, 3895, 245, 13, 0, 0, 7, 10, 4022, 3930, 3993, 0, 5, 0, 1352, 7, 3932, 9, 4073, 23, 11, 0, 0, 4023, 7, 10, 4083, 4081, 0, 12, 4084, 3989, 0, 8, 4080, 0, 15, 13, 0, 4086, 15, 4081, 905, 1036, 0, 4086, 13, 0, 10, 3769, 11, 23, 9, 4083, 1531, 2457] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4063, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 229, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 2497, 0, 0, 0, 0, 0, 4096, 4096, 0, 4094, 4096, 0, 0, 0, 8, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 4096, 4094, 0, 24, 0, 28, 0, 4096, 0, 4096, 0, 0, 0, 6, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 52, 0, 2, 4074, 0, 0, 0, 1345, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 4096, 4078, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4004, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 246, 0, 0, 4, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 2438, 0, 0, 0, 0, 0, 4096, 4096, 6, 4093, 4096, 0, 0, 0, 3, 4091, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 4096, 4095, 0, 14, 0, 68, 0, 4096, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4094, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 28, 0, 2, 4083, 0, 0, 0, 1413, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 4077, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 3626, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 1004, 0, 0, 7, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 2716, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 4096, 0, 0, 0, 0, 4085, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 34, 0, 63, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 419, 0, 0, 4082, 0, 0, 0, 352, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 4048, 0, 0, 0, 0, 4096, 3589, 675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4051, 0, 4096, 0, 0, 0, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 4096, 4096, 3421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3230, 0, 0, 0, 0, 0, 4095, 4096, 0, 4096, 2198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3451, 2190, 1898, 487, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3886, 4096, 4096, 0, 0, 0, 16, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 20, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 6, 0, 1, 4096, 0, 0, 0, 663, 0, 0, 0, 1906, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16505, 16984, 15983, 17715, 16428, 16190, 16404, 14863] avg=16384.0000 max=17715 max_rank=3 min=14863 min_rank=7 imbalance_ratio=1.081238 max_load_reduction=113357 max_load_reduction_pct=0.864845 moved_to_echo=63885 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4096, 4073, 0, 0, 0, 0, 4096, 3589, 676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3831, 0, 4096, 0, 0, 0, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1208, 0, 0, 1, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 4096, 3420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2004, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 2198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3451, 2190, 1897, 487, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3886, 4096, 4095, 0, 9, 0, 13, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 20, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 57, 0, 1, 4092, 0, 0, 0, 1097, 0, 0, 0, 1906, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16534, 16764, 16913, 16490, 16426, 16196, 16404, 15345] avg=16384.0000 max=16913 max_rank=2 min=15345 min_rank=7 imbalance_ratio=1.032288 max_load_reduction=114159 max_load_reduction_pct=0.870964 moved_to_echo=64371 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 4096, 4039, 0, 0, 0, 0, 4096, 3589, 675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4092, 0, 4096, 0, 0, 0, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 0, 0, 4096, 4096, 3421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2851, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 2198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4089, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 22, 0, 0, 0, 3451, 2190, 1898, 487, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3886, 4096, 4095, 0, 31, 0, 51, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 20, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 54, 0, 0, 4070, 0, 0, 0, 1104, 0, 0, 0, 1906, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16505, 17025, 15776, 17337, 16444, 16255, 16404, 15326] avg=16384.0000 max=17337 max_rank=3 min=15326 min_rank=7 imbalance_ratio=1.058167 max_load_reduction=113735 max_load_reduction_pct=0.867729 moved_to_echo=64429 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 12, 0, 0, 5, 0, 19, 0, 2, 0, 3964, 3123, 9, 100, 61, 9, 4023, 3578, 673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 13, 0, 0, 4081, 0, 545, 8, 4060, 313, 18, 729, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 81, 1724, 10, 10, 53, 9, 0, 4080, 0, 14, 0, 40, 0, 12, 4086, 4085, 3400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 13, 492, 0, 6, 0, 121, 0, 3002, 4052, 0, 4079, 2192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3683, 0, 15, 11, 86, 10, 0, 0, 3895, 245, 13, 0, 0, 7, 10, 3440, 2183, 1893, 485, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3816, 3930, 3993, 0, 5, 0, 1352, 7, 3932, 9, 0, 23, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 10, 4083, 4081, 0, 12, 21, 3989, 0, 8, 4080, 0, 15, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4081, 905, 1036, 0, 4086, 13, 0, 10, 3769, 11, 23, 9, 1900, 1531, 2457, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15578, 14480, 17622, 13967, 16182, 17078, 16319, 19846] avg=16384.0000 max=19846 max_rank=7 min=13967 min_rank=3 imbalance_ratio=1.211304 max_load_reduction=111226 max_load_reduction_pct=0.848587 moved_to_echo=69425 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4096, 4077, 0, 0, 0, 0, 4096, 3590, 676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4063, 0, 4096, 0, 0, 0, 642, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 4096, 4096, 3420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2497, 0, 0, 0, 0, 0, 4096, 4096, 0, 4094, 2196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 3454, 2197, 1900, 491, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3881, 4096, 4094, 0, 24, 0, 28, 0, 4096, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 15, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 52, 0, 2, 4074, 0, 0, 0, 1345, 0, 0, 0, 1899, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16541, 16993, 15917, 16979, 16454, 16225, 16399, 15564] avg=16384.0000 max=16993 max_rank=1 min=15564 min_rank=7 imbalance_ratio=1.037170 max_load_reduction=114079 max_load_reduction_pct=0.870354 moved_to_echo=64642 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 4096, 4078, 0, 0, 0, 0, 4096, 3589, 676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 4004, 0, 4096, 0, 0, 0, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 4, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 4096, 4096, 3420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2438, 0, 0, 0, 0, 0, 4096, 4096, 6, 4093, 2198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4091, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 3451, 2190, 1898, 487, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3886, 4096, 4095, 0, 14, 0, 68, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 20, 4094, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 28, 0, 2, 4083, 0, 0, 0, 1413, 0, 0, 0, 1906, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16549, 16937, 15948, 16927, 16428, 16257, 16402, 15624] avg=16384.0000 max=16937 max_rank=1 min=15624 min_rank=7 imbalance_ratio=1.033752 max_load_reduction=114135 max_load_reduction_pct=0.870781 moved_to_echo=64711 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 0, 0, 0, 7, 0, 0, 0, 4032, 3887, 3, 37, 16, 3, 4068, 3585, 675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 4, 0, 0, 4092, 0, 1910, 3, 4081, 174, 2, 330, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 418, 3, 3, 34, 3, 0, 4093, 0, 4, 0, 12, 0, 4, 4093, 4093, 3414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 433, 0, 2, 0, 41, 0, 3593, 4092, 0, 4090, 2197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3863, 0, 5, 3, 25, 3, 0, 1, 4054, 92, 4, 0, 0, 4, 3, 3447, 2188, 1896, 486, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3854, 4041, 4037, 0, 2, 0, 624, 4, 4020, 3, 0, 12, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 4092, 4092, 0, 4, 21, 4019, 0, 2, 4092, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4092, 2412, 442, 0, 4093, 4, 2, 3, 2485, 3, 6, 6, 1904, 732, 3365, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16317, 15330, 16199, 14455, 16282, 16600, 16336, 19553] avg=16384.0000 max=19553 max_rank=7 min=14455 min_rank=3 imbalance_ratio=1.193420 max_load_reduction=111519 max_load_reduction_pct=0.850822 moved_to_echo=68771 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 4077, 0, 0, 0, 0, 4096, 3589, 676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 3626, 0, 4096, 0, 0, 0, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1004, 0, 0, 7, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 4096, 4096, 3420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2716, 0, 0, 0, 0, 0, 4096, 4096, 0, 4096, 2198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 11, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3451, 2190, 1898, 487, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3886, 4096, 4096, 0, 34, 0, 63, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 20, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 419, 0, 0, 4082, 0, 0, 0, 352, 0, 0, 0, 1906, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16543, 16559, 16714, 17202, 16428, 16271, 16404, 14951] avg=16384.0000 max=17202 max_rank=3 min=14951 min_rank=7 imbalance_ratio=1.049927 max_load_reduction=113870 max_load_reduction_pct=0.868759 moved_to_echo=64054 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 0, 128, 0, 32640, 31488, 128, 256, 128, 128, 32768, 28800, 5504, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 3968, 1536, 128, 32768, 128, 128, 128, 12288, 128, 128, 128, 15360, 2304, 30464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 32768, 32768, 0, 128, 256, 32640, 0, 128, 32768, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 0, 32768, 128, 0, 0, 32768, 0, 26240, 128, 32768, 512, 128, 1152, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 5248, 128, 128, 128, 128, 0, 32768, 0, 128, 0, 128, 0, 128, 32768, 32768, 27392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[31104, 32640, 32640, 0, 128, 0, 2304, 128, 32640, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 128, 16768, 0, 128, 0, 256, 0, 31232, 32768, 128, 32768, 17664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 32128, 128, 128, 128, 128, 128, 0, 128, 32640, 384, 128, 128, 0, 128, 128, 27648, 17536, 15232, 3968, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[1756, 1442, 1419, 1, 792, 0, 2034, 1617, 1270, 2536, 1913, 983, 1277, 2077, 0, 739, 836, 1443, 1150, 932, 41, 787, 298, 1976, 683, 202, 2433, 1794, 1504, 573, 1057, 1297, 144, 0, 2259, 505, 383, 2479, 773, 961, 409, 17, 1593, 2021, 0, 1168, 2082, 1234, 1283, 676, 1097, 1313, 50, 510, 17, 1333, 1, 932, 45, 1040, 1126, 2021, 921, 998, 1021, 5, 0, 1372, 1397, 1766, 1798, 1142, 1023, 898, 1653, 1237, 1294, 35, 1896, 1465, 1394, 397, 47, 18, 1653, 1023, 1182, 29, 1488, 552, 1171, 860, 853, 1391, 1235, 1, 1097, 873, 21, 1433, 26, 1098, 1345, 1050, 390, 1250, 656, 1135, 585, 963, 2297, 1476, 1962, 1318, 531, 987, 1601, 1954, 1508, 1068, 1065, 1368, 1409, 779, 1460, 26, 754, 38] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[2327, 1541, 762, 0, 776, 0, 1348, 2534, 1393, 2032, 1585, 559, 585, 2471, 0, 1160, 672, 1134, 1300, 725, 33, 1451, 407, 2086, 892, 389, 1951, 1217, 1351, 435, 1977, 851, 56, 49, 2114, 1115, 286, 2161, 1275, 1242, 586, 55, 2339, 1606, 0, 1027, 1392, 1255, 959, 504, 779, 2218, 37, 158, 48, 869, 0, 576, 122, 718, 1782, 1121, 1461, 1325, 786, 3, 1, 2086, 1242, 2951, 827, 1103, 911, 1614, 1557, 1132, 1185, 50, 1504, 1382, 672, 1123, 48, 24, 2060, 994, 1663, 44, 1603, 301, 834, 819, 370, 1665, 917, 2, 1132, 1397, 8, 1817, 117, 913, 1174, 1639, 266, 1720, 443, 1225, 1272, 645, 1658, 1232, 1102, 1011, 857, 1086, 2509, 1167, 2178, 1274, 950, 1688, 1124, 553, 1641, 40, 523, 59] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[2202, 1488, 1123, 7, 1055, 0, 1824, 1052, 1102, 2155, 1699, 1282, 1154, 1997, 1, 934, 567, 1491, 633, 833, 6, 792, 328, 1901, 984, 270, 2277, 1344, 839, 185, 1260, 1240, 179, 49, 1700, 559, 266, 2317, 1219, 999, 375, 61, 1817, 1854, 1, 1513, 1565, 885, 1451, 784, 935, 1639, 25, 366, 35, 1653, 0, 886, 279, 1253, 1330, 1373, 715, 1296, 730, 10, 0, 1573, 2378, 1893, 1651, 1077, 1035, 1332, 1750, 1033, 1448, 36, 1266, 1560, 1369, 410, 93, 16, 2072, 788, 1506, 16, 1720, 317, 985, 1220, 764, 1758, 1121, 5, 1458, 670, 5, 2053, 34, 1132, 1157, 1701, 395, 1408, 385, 1113, 917, 798, 1964, 2248, 1654, 1025, 549, 1069, 1516, 955, 1452, 1429, 1188, 1393, 1421, 734, 1855, 78, 937, 43] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[1941, 1401, 1371, 0, 1059, 1, 1224, 2106, 1473, 2204, 2019, 1378, 1074, 1808, 2, 699, 893, 1106, 941, 1037, 10, 780, 249, 1471, 952, 171, 2808, 1345, 1171, 303, 1232, 1321, 156, 0, 1538, 801, 541, 1993, 1157, 333, 439, 81, 1552, 2763, 0, 1904, 1492, 1010, 1241, 442, 1018, 1466, 138, 79, 25, 1158, 2, 702, 116, 1139, 967, 2145, 1910, 982, 1086, 5, 0, 1326, 2072, 1360, 2155, 1090, 1252, 1049, 1397, 1235, 1455, 61, 1462, 1120, 1538, 412, 63, 5, 1822, 728, 1107, 48, 1884, 342, 1093, 600, 510, 987, 813, 4, 1262, 789, 8, 1940, 174, 757, 1347, 730, 359, 1331, 707, 1068, 1142, 1268, 2856, 1179, 1265, 1069, 992, 1087, 2240, 1463, 1124, 1333, 1335, 1908, 1649, 1634, 1134, 90, 844, 67] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[1611, 852, 2184, 0, 1426, 1, 812, 1419, 682, 1404, 1212, 1325, 1774, 1599, 0, 917, 1124, 1326, 1882, 743, 19, 853, 251, 2142, 460, 22, 2404, 2243, 262, 138, 923, 1321, 281, 0, 2554, 115, 1338, 1834, 1220, 589, 366, 33, 1008, 2312, 0, 1288, 1292, 1539, 1353, 397, 593, 1434, 25, 88, 134, 2092, 1, 750, 283, 745, 960, 2048, 692, 998, 1953, 19, 0, 1114, 2259, 2270, 2535, 1081, 1253, 1607, 1524, 1435, 2061, 9, 1615, 577, 956, 777, 44, 107, 2239, 1600, 793, 64, 1084, 240, 423, 954, 586, 1172, 927, 0, 418, 497, 11, 1772, 33, 904, 1575, 1533, 283, 1629, 1218, 2170, 959, 1840, 1640, 1523, 2549, 1123, 226, 1547, 2589, 739, 387, 1255, 1549, 1039, 1240, 563, 2541, 198, 418, 133] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[2018, 1188, 1665, 0, 803, 0, 1756, 1596, 1203, 2350, 1590, 1332, 1420, 1757, 3, 749, 665, 1343, 827, 663, 9, 824, 271, 1897, 630, 330, 2267, 1730, 1465, 264, 1156, 1394, 47, 2, 2395, 1195, 531, 1819, 715, 943, 475, 3, 1133, 2110, 0, 1374, 1590, 1269, 919, 579, 673, 1293, 53, 440, 23, 901, 7, 357, 91, 582, 1393, 2486, 735, 802, 1131, 3, 0, 1739, 1824, 2157, 1352, 2074, 1057, 1877, 1686, 809, 1724, 60, 1484, 1655, 1720, 415, 39, 9, 2114, 954, 879, 28, 1817, 502, 984, 955, 830, 1547, 853, 0, 987, 1352, 41, 1425, 43, 923, 1424, 1942, 268, 1291, 419, 1356, 1063, 1858, 2266, 1515, 2588, 1164, 404, 816, 1362, 884, 1301, 1120, 1045, 1894, 864, 645, 1601, 28, 777, 28] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[1983, 1492, 1409, 0, 929, 0, 1436, 1861, 1229, 1891, 2054, 1048, 903, 1784, 0, 749, 643, 1364, 872, 862, 57, 1006, 271, 2060, 929, 211, 2786, 1316, 1121, 382, 1213, 767, 147, 4, 1905, 966, 334, 1903, 1154, 528, 340, 33, 1724, 1720, 0, 1242, 1308, 1478, 1098, 686, 1186, 1304, 22, 180, 20, 1458, 1, 641, 122, 1731, 865, 2044, 1028, 1162, 811, 5, 0, 950, 1915, 1604, 1609, 1398, 1192, 1262, 1277, 1332, 1763, 83, 1877, 1538, 1285, 438, 49, 1, 1866, 856, 1307, 16, 2099, 311, 1264, 946, 613, 1740, 993, 3, 1187, 1077, 42, 1350, 26, 1108, 1241, 1103, 398, 1449, 703, 995, 648, 582, 2201, 1858, 2003, 1341, 921, 1159, 2470, 1812, 1258, 1258, 1283, 1879, 1333, 601, 1620, 62, 700, 39] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[1959, 1842, 1398, 0, 697, 0, 1449, 1779, 1889, 2257, 2734, 864, 1021, 2475, 0, 996, 567, 1520, 1188, 442, 53, 1301, 201, 2167, 505, 238, 2512, 1257, 936, 206, 1447, 989, 55, 7, 2174, 876, 156, 1870, 856, 1271, 319, 27, 1278, 2374, 0, 1420, 1461, 456, 999, 476, 1239, 1599, 17, 351, 9, 1168, 2, 1052, 294, 1132, 1060, 2243, 999, 1106, 1161, 18, 0, 1376, 1425, 1777, 741, 1113, 926, 1361, 1344, 330, 1910, 106, 1811, 1792, 977, 688, 23, 13, 1901, 974, 627, 22, 2033, 538, 876, 533, 937, 1776, 1297, 4, 1321, 1026, 9, 1923, 39, 1002, 1716, 1359, 216, 1182, 437, 738, 913, 1806, 2495, 1682, 2053, 789, 599, 1231, 1696, 1277, 1755, 1297, 847, 1521, 932, 978, 1914, 27, 607, 40] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1756, 1442, 1419, 1, 792, 0, 2034, 1617, 1270, 0, 1913, 983, 1277, 1214, 0, 739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 836, 1443, 1150, 932, 41, 787, 298, 1976, 683, 202, 2433, 1794, 1504, 573, 1057, 1297, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 2259, 505, 383, 2479, 773, 961, 409, 17, 1593, 2021, 0, 1168, 2082, 1234, 746, 88, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1283, 676, 1097, 1313, 50, 510, 17, 1333, 1, 932, 45, 1040, 1126, 2021, 921, 998, 2536, 1055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1021, 5, 0, 1372, 1000, 0, 1798, 1142, 1023, 898, 1653, 1237, 1294, 35, 1896, 1465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1394, 397, 47, 18, 1653, 1023, 1182, 29, 1488, 552, 1171, 860, 853, 1391, 1235, 1, 1299, 857, 711, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 873, 21, 1433, 26, 1098, 1345, 1050, 390, 1250, 656, 1135, 585, 963, 1551, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1962, 1318, 531, 987, 302, 1954, 1508, 1068, 1065, 1368, 1409, 779, 1460, 26, 754, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16457, 17315, 16868, 16954, 15839, 16161, 14949, 16529] avg=16384.0000 max=17315 max_rank=1 min=14949 min_rank=6 imbalance_ratio=1.056824 max_load_reduction=113757 max_load_reduction_pct=0.867897 moved_to_echo=63478 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2327, 1541, 762, 0, 776, 0, 1348, 2534, 1393, 0, 1585, 559, 585, 1445, 0, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 1134, 1300, 725, 33, 1451, 407, 2086, 892, 389, 1951, 1217, 1351, 435, 1977, 851, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 49, 2114, 1115, 286, 2161, 1275, 1242, 586, 55, 2339, 1606, 0, 1027, 1392, 1255, 538, 78, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 959, 504, 779, 2218, 37, 158, 48, 869, 0, 576, 122, 718, 1782, 1121, 1461, 1325, 2032, 1765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 786, 3, 1, 2086, 889, 0, 827, 1103, 911, 1614, 1557, 1132, 1185, 50, 1504, 1382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 672, 1123, 48, 24, 2060, 994, 1663, 44, 1603, 301, 834, 819, 370, 1665, 917, 2, 2035, 1019, 1186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 1397, 8, 1817, 117, 913, 1174, 1639, 266, 1720, 443, 1225, 1272, 645, 1120, 1232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1102, 1011, 857, 1086, 474, 1167, 2178, 1274, 950, 1688, 1124, 553, 1641, 40, 523, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16015, 17146, 17181, 16474, 15030, 17379, 16120, 15727] avg=16384.0000 max=17379 max_rank=5 min=15030 min_rank=4 imbalance_ratio=1.060730 max_load_reduction=113693 max_load_reduction_pct=0.867409 moved_to_echo=64256 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2202, 1488, 1123, 7, 1055, 0, 1824, 1052, 1102, 0, 1699, 1282, 1154, 1167, 1, 934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 1491, 633, 833, 6, 792, 328, 1901, 984, 270, 2277, 1344, 839, 185, 1260, 1240, 527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 49, 1700, 559, 266, 2317, 1219, 999, 375, 61, 1817, 1854, 1, 1513, 1565, 885, 638, 150, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1451, 784, 935, 1639, 25, 366, 35, 1653, 0, 886, 279, 1253, 1330, 1373, 715, 1296, 2155, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 730, 10, 0, 1573, 1701, 0, 1651, 1077, 1035, 1332, 1750, 1033, 1448, 36, 1266, 1560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1369, 410, 93, 16, 2072, 788, 1506, 16, 1720, 317, 985, 1220, 764, 1758, 1121, 5, 1230, 824, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1458, 670, 5, 2053, 34, 1132, 1157, 1701, 395, 1408, 385, 1113, 917, 798, 1326, 2248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1654, 1025, 549, 1069, 286, 955, 1452, 1429, 1188, 1393, 1421, 734, 1855, 78, 937, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16090, 15477, 16153, 17306, 16202, 16976, 16800, 16068] avg=16384.0000 max=17306 max_rank=3 min=15477 min_rank=1 imbalance_ratio=1.056274 max_load_reduction=113766 max_load_reduction_pct=0.867966 moved_to_echo=66046 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1941, 1401, 1371, 0, 1059, 1, 1224, 2106, 1473, 0, 2019, 1378, 1074, 1057, 2, 699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 893, 1106, 941, 1037, 10, 780, 249, 1471, 952, 171, 2808, 1345, 1171, 303, 1232, 1321, 459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 156, 0, 1538, 801, 541, 1993, 1157, 333, 439, 81, 1552, 2763, 0, 1904, 1492, 1010, 927, 130, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1241, 442, 1018, 1466, 138, 79, 25, 1158, 2, 702, 116, 1139, 967, 2145, 1910, 982, 2204, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 5, 0, 1326, 1483, 0, 2155, 1090, 1252, 1049, 1397, 1235, 1455, 61, 1462, 1120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1538, 412, 63, 5, 1822, 728, 1107, 48, 1884, 342, 1093, 600, 510, 987, 813, 4, 1817, 746, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 789, 8, 1940, 174, 757, 1347, 730, 359, 1331, 707, 1068, 1142, 1268, 1929, 1179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1265, 1069, 992, 1087, 423, 1463, 1124, 1333, 1335, 1908, 1649, 1634, 1134, 90, 844, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16805, 16249, 16822, 16548, 16176, 15065, 15990, 17417] avg=16384.0000 max=17417 max_rank=7 min=15065 min_rank=5 imbalance_ratio=1.063049 max_load_reduction=113655 max_load_reduction_pct=0.867119 moved_to_echo=64648 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1611, 852, 2184, 0, 1426, 1, 812, 1419, 682, 0, 1212, 1325, 1774, 929, 0, 917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1124, 1326, 1882, 743, 19, 853, 251, 2142, 460, 22, 2404, 2243, 262, 138, 923, 1321, 505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 281, 0, 2554, 115, 1338, 1834, 1220, 589, 366, 33, 1008, 2312, 0, 1288, 1292, 1539, 536, 146, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1353, 397, 593, 1434, 25, 88, 134, 2092, 1, 750, 283, 745, 960, 2048, 692, 998, 1404, 1357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1953, 19, 0, 1114, 1608, 0, 2535, 1081, 1253, 1607, 1524, 1435, 2061, 9, 1615, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 956, 777, 44, 107, 2239, 1600, 793, 64, 1084, 240, 423, 954, 586, 1172, 927, 0, 2103, 662, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 497, 11, 1772, 33, 904, 1575, 1533, 283, 1629, 1218, 2170, 959, 1840, 1104, 1523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2549, 1123, 226, 1547, 486, 739, 387, 1255, 1549, 1039, 1240, 563, 2541, 198, 418, 133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15144, 16618, 16459, 15354, 18391, 15644, 17469, 15993] avg=16384.0000 max=18391 max_rank=4 min=15144 min_rank=0 imbalance_ratio=1.122498 max_load_reduction=112681 max_load_reduction_pct=0.859688 moved_to_echo=67497 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[2018, 1188, 1665, 0, 803, 0, 1756, 1596, 1203, 0, 1590, 1332, 1420, 1027, 3, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 665, 1343, 827, 663, 9, 824, 271, 1897, 630, 330, 2267, 1730, 1465, 264, 1156, 1394, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 2, 2395, 1195, 531, 1819, 715, 943, 475, 3, 1133, 2110, 0, 1374, 1590, 1269, 736, 115, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 919, 579, 673, 1293, 53, 440, 23, 901, 7, 357, 91, 582, 1393, 2486, 735, 802, 2350, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1131, 3, 0, 1739, 1305, 0, 1352, 2074, 1057, 1877, 1686, 809, 1724, 60, 1484, 1655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1720, 415, 39, 9, 2114, 954, 879, 28, 1817, 502, 984, 955, 830, 1547, 853, 0, 1105, 725, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 987, 1352, 41, 1425, 43, 923, 1424, 1942, 268, 1291, 419, 1356, 1063, 1858, 1530, 1515, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2588, 1164, 404, 816, 257, 884, 1301, 1120, 1045, 1894, 864, 645, 1601, 28, 777, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16350, 16139, 16457, 14974, 17956, 16343, 17437, 15416] avg=16384.0000 max=17956 max_rank=4 min=14974 min_rank=3 imbalance_ratio=1.095947 max_load_reduction=113116 max_load_reduction_pct=0.863007 moved_to_echo=67152 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1983, 1492, 1409, 0, 929, 0, 1436, 1861, 1229, 0, 2054, 1048, 903, 1043, 0, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 643, 1364, 872, 862, 57, 1006, 271, 2060, 929, 211, 2786, 1316, 1121, 382, 1213, 767, 424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 4, 1905, 966, 334, 1903, 1154, 528, 340, 33, 1724, 1720, 0, 1242, 1308, 1478, 715, 120, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1098, 686, 1186, 1304, 22, 180, 20, 1458, 1, 641, 122, 1731, 865, 2044, 1028, 1162, 1891, 958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 811, 5, 0, 950, 1371, 0, 1609, 1398, 1192, 1262, 1277, 1332, 1763, 83, 1877, 1538, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1285, 438, 49, 1, 1866, 856, 1307, 16, 2099, 311, 1264, 946, 613, 1740, 993, 3, 2004, 736, 646, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1187, 1077, 42, 1350, 26, 1108, 1241, 1103, 398, 1449, 703, 995, 648, 582, 1486, 1858, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2003, 1341, 921, 1159, 466, 1812, 1258, 1258, 1283, 1879, 1333, 601, 1620, 62, 700, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16136, 16284, 15626, 16397, 16468, 17173, 15253, 17735] avg=16384.0000 max=17735 max_rank=7 min=15253 min_rank=6 imbalance_ratio=1.082458 max_load_reduction=113337 max_load_reduction_pct=0.864693 moved_to_echo=66629 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[1959, 1842, 1398, 0, 697, 0, 1449, 1779, 1889, 0, 2734, 864, 1021, 1447, 0, 996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 1520, 1188, 442, 53, 1301, 201, 2167, 505, 238, 2512, 1257, 936, 206, 1447, 989, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 7, 2174, 876, 156, 1870, 856, 1271, 319, 27, 1278, 2374, 0, 1420, 1461, 456, 810, 89, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 999, 476, 1239, 1599, 17, 351, 9, 1168, 2, 1052, 294, 1132, 1060, 2243, 999, 1106, 2257, 1062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1161, 18, 0, 1376, 1021, 0, 741, 1113, 926, 1361, 1344, 330, 1910, 106, 1811, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 977, 688, 23, 13, 1901, 974, 627, 22, 2033, 538, 876, 533, 937, 1776, 1297, 4, 1376, 1021, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1321, 1026, 9, 1923, 39, 1002, 1716, 1359, 216, 1182, 437, 738, 913, 1806, 1685, 1682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2053, 789, 599, 1231, 320, 1277, 1755, 1297, 847, 1521, 932, 978, 1914, 27, 607, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18075, 15844, 15506, 17065, 15010, 16331, 17054, 16187] avg=16384.0000 max=18075 max_rank=0 min=15010 min_rank=4 imbalance_ratio=1.103210 max_load_reduction=112997 max_load_reduction_pct=0.862099 moved_to_echo=64582 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[8704, 128, 128, 11648, 10496, 0, 12672, 10112, 8704, 11008, 12288, 8576, 12928, 512, 12928, 11136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[15872, 11264, 11392, 128, 7552, 128, 11904, 14080, 10368, 0, 14848, 8832, 9216, 9344, 128, 7040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[15232, 8960, 5120, 9088, 3072, 10368, 11008, 10112, 9344, 12800, 9984, 6528, 13824, 640, 5632, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[8960, 7808, 256, 13824, 512, 7936, 11008, 11136, 2688, 11264, 4992, 9856, 7552, 9856, 11776, 12800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[9344, 4608, 7552, 12288, 384, 2176, 384, 10752, 128, 6016, 1408, 8448, 9600, 15488, 8576, 8704, 16896, 9472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[1152, 128, 16640, 6144, 3840, 16384, 8448, 6912, 3328, 384, 12544, 16768, 128, 11008, 12288, 9216, 5760, 1024, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[9984, 4736, 512, 256, 15744, 7936, 9088, 384, 13824, 3200, 7680, 6912, 5504, 12160, 8192, 128, 13056, 6656, 6400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[6016, 10752, 8832, 6272, 256, 7808, 2304, 15744, 6144, 1920, 19456, 12288, 8704, 2560, 10368, 9216, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 3929, 80, 0, 5, 1, 0, 2, 1, 3790, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 991, 0, 4083, 1, 2, 21, 0, 3124, 4018, 48, 0, 3, 4094, 3, 600, 7, 4092, 1, 0, 4096, 0, 1, 2695, 0, 3414, 0, 3989, 0, 2388, 0, 0, 4096, 0, 4049, 4091, 1, 0, 147, 3476, 0, 0, 133, 2, 3970, 2, 0, 0, 270, 14, 0, 3164, 0, 4059, 3047, 0, 0, 2, 1, 0, 21, 255, 7, 4096, 0, 0, 0, 203, 2, 0, 0, 2, 0, 4096, 17, 2, 2, 0, 0, 4094, 364, 2928, 4093, 0, 3692, 4062, 0, 31, 0, 0, 0, 4093, 4093, 0, 0, 86, 0, 1550, 4, 2, 514, 3975, 0, 499, 3, 1, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 3940, 93, 1, 9, 0, 1, 0, 0, 3774, 1, 0, 1, 4095, 1, 0, 4095, 1, 0, 712, 0, 3974, 0, 0, 178, 0, 1485, 3690, 45, 0, 0, 4095, 0, 1195, 15, 4096, 1, 0, 4089, 0, 1, 2680, 1, 3276, 1, 3883, 4, 2259, 0, 1, 4095, 19, 3962, 4094, 0, 1, 123, 3746, 0, 1, 75, 1, 3876, 0, 1, 1, 666, 19, 1, 3277, 0, 4060, 3528, 1, 1, 0, 11, 1, 110, 162, 66, 4095, 1, 1, 0, 8, 0, 1, 0, 0, 1, 4095, 1, 0, 0, 1, 0, 4095, 112, 3992, 4093, 1, 3938, 4072, 0, 17, 0, 0, 0, 4095, 4095, 0, 1, 138, 0, 1250, 0, 0, 1223, 3485, 1, 577, 0, 1, 1, 4089] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 3903, 439, 0, 7, 1, 0, 2, 0, 3283, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 2183, 0, 3661, 5, 0, 141, 0, 3572, 4061, 53, 0, 5, 4092, 2, 437, 4, 4095, 0, 0, 4037, 0, 0, 3351, 0, 2192, 0, 3976, 35, 2819, 2, 0, 4096, 2, 3196, 4093, 0, 0, 46, 3580, 0, 0, 189, 0, 4047, 12, 1, 0, 826, 1, 0, 2416, 1, 4060, 2468, 2, 0, 0, 1, 0, 650, 81, 3, 4075, 0, 2, 0, 250, 0, 0, 0, 0, 0, 4095, 4, 0, 0, 0, 0, 4095, 109, 4072, 4083, 0, 3806, 3783, 0, 223, 4, 0, 2, 4093, 4096, 0, 0, 409, 3, 1184, 6, 0, 851, 2316, 1, 1101, 0, 0, 2, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 3863, 177, 0, 0, 0, 0, 0, 9, 4019, 0, 0, 0, 4090, 0, 0, 4095, 1, 0, 655, 11, 3930, 0, 1, 198, 1, 3582, 3297, 114, 0, 2, 4095, 0, 525, 32, 4093, 1, 0, 4076, 0, 0, 3082, 1, 2983, 0, 4056, 1, 1442, 1, 1, 4095, 0, 4055, 4095, 0, 1, 9, 3778, 0, 0, 85, 0, 3644, 0, 0, 0, 291, 3, 1, 3354, 2, 3940, 3840, 0, 0, 0, 0, 0, 16, 767, 18, 4094, 1, 0, 1, 34, 0, 0, 4, 0, 1, 4093, 19, 0, 1, 0, 0, 4091, 229, 3413, 4089, 0, 3410, 4060, 0, 3, 10, 0, 1, 4094, 4095, 0, 1, 191, 1, 2448, 0, 0, 344, 3090, 2, 746, 0, 0, 0, 4077] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1, 0, 0, 2595, 574, 0, 0, 0, 0, 0, 0, 3344, 0, 0, 1, 4095, 0, 0, 4095, 0, 0, 3252, 2, 4096, 0, 0, 8, 0, 4041, 3919, 11, 1, 1, 4094, 1, 1393, 0, 4089, 0, 0, 3659, 0, 1, 2261, 0, 1540, 0, 4087, 50, 2737, 1, 0, 4095, 0, 1130, 4095, 0, 0, 3, 3797, 1, 0, 230, 0, 4055, 1, 0, 0, 2965, 6, 1, 3326, 0, 2754, 574, 0, 0, 0, 0, 1, 51, 31, 5, 3621, 1, 0, 0, 960, 0, 1, 0, 0, 0, 4093, 246, 0, 0, 1, 0, 4083, 0, 3385, 4094, 0, 3280, 2988, 1, 31, 0, 0, 0, 4092, 4095, 0, 0, 3102, 0, 3913, 1, 1, 918, 2969, 0, 31, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[0, 4, 0, 3677, 130, 0, 0, 0, 3, 0, 28, 2508, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 1759, 15, 4049, 1, 0, 148, 0, 3922, 3747, 184, 0, 17, 4094, 2, 1959, 0, 4076, 0, 0, 4084, 3, 0, 3332, 0, 2158, 1, 4093, 107, 2397, 1, 0, 4095, 1, 3188, 4096, 1, 0, 0, 3994, 0, 0, 87, 0, 3723, 0, 0, 0, 1678, 2, 0, 2564, 3, 3269, 3695, 1, 0, 0, 0, 0, 983, 163, 6, 4088, 1, 1, 3, 82, 0, 0, 0, 0, 0, 4095, 553, 0, 0, 0, 0, 4096, 56, 3915, 4091, 0, 3305, 3130, 0, 218, 0, 1, 0, 4092, 4057, 0, 0, 775, 1, 1439, 0, 1, 591, 1900, 0, 246, 0, 0, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 3501, 72, 0, 1, 1, 0, 0, 0, 2193, 0, 0, 0, 4095, 1, 0, 4095, 0, 0, 833, 0, 2847, 5, 1, 76, 0, 2980, 3319, 7, 1, 5, 4094, 0, 1354, 0, 4094, 0, 0, 3826, 0, 0, 3520, 1, 2990, 0, 4064, 2, 3052, 1, 0, 4096, 0, 3945, 4095, 0, 1, 2, 3676, 3, 0, 798, 1, 3933, 9, 0, 0, 1217, 1, 0, 2200, 0, 3666, 3497, 0, 0, 0, 0, 0, 1395, 237, 0, 4095, 0, 0, 11, 14, 0, 0, 0, 0, 0, 4095, 15, 2, 0, 0, 0, 4093, 94, 3900, 4093, 0, 3936, 3751, 0, 336, 0, 0, 1, 4095, 4087, 0, 0, 146, 0, 1383, 1, 0, 1018, 2378, 0, 1630, 0, 1, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[0, 0, 0, 3530, 89, 1, 3, 0, 2, 0, 0, 3601, 0, 0, 0, 4095, 0, 0, 4095, 1, 0, 195, 11, 3976, 1, 0, 204, 0, 2784, 3969, 20, 0, 14, 4095, 0, 650, 3, 4067, 0, 0, 4084, 0, 1, 2958, 0, 3073, 0, 4045, 20, 2550, 2, 1, 4096, 0, 3935, 4095, 0, 1, 118, 3328, 1, 0, 161, 0, 3748, 0, 0, 0, 809, 1, 0, 2774, 0, 4014, 3605, 0, 0, 0, 1, 0, 107, 254, 0, 4082, 0, 1, 3, 12, 0, 0, 0, 0, 0, 4094, 14, 0, 0, 1, 2, 4095, 253, 3173, 4096, 0, 2952, 3935, 1, 9, 0, 1, 0, 4096, 4095, 1, 1, 113, 0, 2944, 0, 0, 817, 3198, 0, 1804, 0, 0, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3940, 93, 1, 9, 0, 1, 0, 0, 3774, 1, 0, 1, 4095, 2926, 1306, 912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 1, 0, 712, 0, 3974, 0, 0, 178, 0, 1485, 3690, 45, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1195, 15, 1652, 1, 0, 4089, 0, 1, 2680, 1, 3276, 1, 3883, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2259, 0, 1, 2422, 19, 3962, 4094, 0, 1, 123, 3746, 0, 1, 75, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3876, 0, 1, 1, 666, 19, 1, 3277, 0, 4060, 3528, 1, 1, 0, 11, 1, 1138, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 162, 66, 4095, 1, 1, 0, 8, 0, 1, 0, 0, 1, 4095, 1, 0, 4095, 3183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1169, 112, 3992, 4093, 1, 3938, 4072, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 138, 0, 1250, 0, 0, 1223, 3485, 1, 577, 0, 1, 1, 4089, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17060, 14191, 16794, 16708, 17206, 15819, 17395, 15899] avg=16384.0000 max=17395 max_rank=6 min=14191 min_rank=1 imbalance_ratio=1.061707 max_load_reduction=113677 max_load_reduction_pct=0.867287 moved_to_echo=66319 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3929, 80, 0, 5, 1, 0, 2, 1, 3790, 0, 0, 0, 4096, 2925, 1305, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 991, 0, 4083, 1, 2, 21, 0, 3124, 4018, 48, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 600, 7, 1650, 1, 0, 4096, 0, 1, 2695, 0, 3414, 0, 3989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2388, 0, 0, 2422, 0, 4049, 4091, 1, 0, 147, 3476, 0, 0, 133, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3970, 2, 0, 0, 270, 14, 0, 3164, 0, 4059, 3047, 0, 0, 2, 1, 0, 1137, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 255, 7, 4096, 0, 0, 0, 203, 2, 0, 0, 2, 0, 4096, 17, 2, 4094, 3182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1169, 364, 2928, 4093, 0, 3692, 4062, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 86, 0, 1550, 4, 2, 514, 3975, 0, 499, 3, 1, 0, 4090, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17045, 16394, 16459, 16709, 16291, 15977, 16341, 15856] avg=16384.0000 max=17045 max_rank=0 min=15856 min_rank=7 imbalance_ratio=1.040344 max_load_reduction=114027 max_load_reduction_pct=0.869957 moved_to_echo=64465 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3903, 439, 0, 7, 1, 0, 2, 0, 3283, 0, 0, 0, 4095, 2926, 1306, 912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2183, 0, 3661, 5, 0, 141, 0, 3572, 4061, 53, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 437, 4, 1651, 0, 0, 4037, 0, 0, 3351, 0, 2192, 0, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 2819, 2, 0, 2422, 2, 3196, 4093, 0, 0, 46, 3580, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 12, 1, 0, 826, 1, 0, 2416, 1, 4060, 2468, 2, 0, 0, 1, 0, 1138, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 650, 81, 3, 4075, 0, 2, 0, 250, 0, 0, 0, 0, 0, 4095, 4, 0, 4092, 3181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1169, 109, 4072, 4083, 0, 3806, 3783, 0, 223, 4, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 409, 3, 1184, 6, 0, 851, 2316, 1, 1101, 0, 0, 2, 4087, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16874, 17782, 15655, 16384, 15598, 16433, 17251, 15095] avg=16384.0000 max=17782 max_rank=1 min=15095 min_rank=7 imbalance_ratio=1.085327 max_load_reduction=113290 max_load_reduction_pct=0.864334 moved_to_echo=64377 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 4, 0, 3677, 130, 0, 0, 0, 3, 0, 28, 2508, 0, 0, 0, 4096, 2926, 1300, 911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1759, 15, 4049, 1, 0, 148, 0, 3922, 3747, 184, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 2, 1959, 0, 1643, 0, 0, 4084, 3, 0, 3332, 0, 2158, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 2397, 1, 0, 2422, 1, 3188, 4096, 1, 0, 0, 3994, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3723, 0, 0, 0, 1678, 2, 0, 2564, 3, 3269, 3695, 1, 0, 0, 0, 0, 1133, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 983, 163, 6, 4088, 1, 1, 3, 82, 0, 0, 0, 0, 0, 4095, 553, 0, 4094, 3181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 56, 3915, 4091, 0, 3305, 3130, 0, 218, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 0, 775, 1, 1439, 0, 1, 591, 1900, 0, 246, 0, 0, 0, 4095, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15583, 17931, 17292, 16294, 16693, 17250, 15886, 14143] avg=16384.0000 max=17931 max_rank=1 min=14143 min_rank=7 imbalance_ratio=1.094421 max_load_reduction=113141 max_load_reduction_pct=0.863197 moved_to_echo=63972 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 2595, 574, 0, 0, 0, 0, 0, 0, 3344, 0, 0, 1, 4095, 2920, 1307, 912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3252, 2, 4096, 0, 0, 8, 0, 4041, 3919, 11, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1393, 0, 1641, 0, 0, 3659, 0, 1, 2261, 0, 1540, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 2737, 1, 0, 2411, 0, 1130, 4095, 0, 0, 3, 3797, 1, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 1, 0, 0, 2965, 6, 1, 3326, 0, 2754, 574, 0, 0, 0, 0, 1, 1141, 630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 31, 5, 3621, 1, 0, 0, 960, 0, 1, 0, 0, 0, 4093, 246, 0, 4094, 3180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1163, 0, 3385, 4094, 0, 3280, 2988, 1, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 3102, 0, 3913, 1, 1, 918, 2969, 0, 31, 0, 0, 0, 4095, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15749, 19438, 14584, 14455, 15454, 16283, 14943, 20166] avg=16384.0000 max=20166 max_rank=7 min=14455 min_rank=3 imbalance_ratio=1.230835 max_load_reduction=110906 max_load_reduction_pct=0.846146 moved_to_echo=66846 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3863, 177, 0, 0, 0, 0, 0, 9, 4019, 0, 0, 0, 4090, 2923, 1305, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 655, 11, 3930, 0, 1, 198, 1, 3582, 3297, 114, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 525, 32, 1651, 1, 0, 4076, 0, 0, 3082, 1, 2983, 0, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1442, 1, 1, 2422, 0, 4055, 4095, 0, 1, 9, 3778, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3644, 0, 0, 0, 291, 3, 1, 3354, 2, 3940, 3840, 0, 0, 0, 0, 0, 1137, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 767, 18, 4094, 1, 0, 1, 34, 0, 0, 4, 0, 1, 4093, 19, 0, 4095, 3181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1168, 229, 3413, 4089, 0, 3410, 4060, 0, 3, 10, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 191, 1, 2448, 0, 0, 344, 3090, 2, 746, 0, 0, 0, 4077, 1038, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17300, 15895, 16409, 15890, 16837, 16324, 16384, 16033] avg=16384.0000 max=17300 max_rank=0 min=15890 min_rank=3 imbalance_ratio=1.055908 max_load_reduction=113772 max_load_reduction_pct=0.868011 moved_to_echo=65578 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3501, 72, 0, 1, 1, 0, 0, 0, 2193, 0, 0, 0, 4095, 2924, 1305, 913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 833, 0, 2847, 5, 1, 76, 0, 2980, 3319, 7, 1, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1354, 0, 1651, 0, 0, 3826, 0, 0, 3520, 1, 2990, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3052, 1, 0, 2422, 0, 3945, 4095, 0, 1, 2, 3676, 3, 0, 798, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 9, 0, 0, 1217, 1, 0, 2200, 0, 3666, 3497, 0, 0, 0, 0, 0, 1138, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1395, 237, 0, 4095, 0, 0, 11, 14, 0, 0, 0, 0, 0, 4095, 15, 2, 4094, 3182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1169, 94, 3900, 4093, 0, 3936, 3751, 0, 336, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 146, 0, 1383, 1, 0, 1018, 2378, 0, 1630, 0, 1, 0, 4094, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15005, 14175, 17411, 17998, 16286, 17140, 17280, 15777] avg=16384.0000 max=17998 max_rank=3 min=14175 min_rank=1 imbalance_ratio=1.098511 max_load_reduction=113074 max_load_reduction_pct=0.862686 moved_to_echo=66483 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 3530, 89, 1, 3, 0, 2, 0, 0, 3601, 0, 0, 0, 4095, 2926, 1297, 912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 195, 11, 3976, 1, 0, 204, 0, 2784, 3969, 20, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 650, 3, 1640, 0, 0, 4084, 0, 1, 2958, 0, 3073, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 2550, 2, 1, 2422, 0, 3935, 4095, 0, 1, 118, 3328, 1, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3748, 0, 0, 0, 809, 1, 0, 2774, 0, 4014, 3605, 0, 0, 0, 1, 0, 1130, 625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 254, 0, 4082, 0, 1, 3, 12, 0, 0, 0, 0, 0, 4094, 14, 0, 4095, 3184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1169, 253, 3173, 4096, 0, 2952, 3935, 1, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 113, 0, 2944, 0, 0, 817, 3198, 0, 1804, 0, 0, 0, 4091, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16456, 15266, 16468, 16634, 16707, 15846, 15592, 18103] avg=16384.0000 max=18103 max_rank=7 min=15266 min_rank=1 imbalance_ratio=1.104919 max_load_reduction=112969 max_load_reduction_pct=0.861885 moved_to_echo=66248 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 32768, 128, 0, 10624, 128, 30720, 128, 128, 1024, 128, 25600, 30080, 512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 0, 128, 8192, 128, 13184, 128, 0, 32000, 128, 128, 23936, 128, 21632, 128, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 128, 128, 4992, 128, 16128, 128, 128, 6400, 23424, 128, 6656, 128, 128, 128, 32768, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 29056, 1664, 128, 128, 128, 128, 128, 128, 26624, 128, 0, 128, 32768, 23424, 10496, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[3456, 2048, 128, 32256, 128, 128, 128, 1664, 128, 128, 128, 128, 128, 32768, 896, 128, 32768, 25472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[256, 19712, 128, 128, 19456, 128, 27520, 32768, 128, 128, 512, 29440, 128, 128, 1792, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 9472, 1280, 28800, 32768, 128, 28416, 29824, 128, 896, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132992 observed_loss=-1920 tokens_per_expert=[31104, 128, 128, 128, 8832, 128, 128, 23168, 128, 29824, 24320, 128, 128, 128, 128, 128, 9216, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132992 slack_slots=129152 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 434, 0, 210, 1, 1, 0, 0, 4092, 4030, 0, 1, 1, 0, 4095, 0, 1, 0, 1, 79, 4076, 4095, 1, 4094, 4095, 0, 0, 0, 4095, 1, 0, 0, 0, 4085, 1, 0, 4095, 0, 0, 1, 0, 4095, 1, 11, 1, 0, 3420, 4086, 176, 1, 0, 397, 0, 4095, 254, 0, 1, 3531, 540, 1, 1, 4092, 4094, 0, 1, 4095, 954, 1, 1, 0, 0, 0, 4095, 826, 1, 4095, 4005, 4095, 815, 4060, 1, 1, 4093, 2958, 0, 2762, 1, 2373, 0, 0, 0, 1, 12, 1, 3, 0, 2144, 0, 1, 4094, 0, 2599, 56, 1, 1, 1, 0, 0, 0, 92, 4091, 1, 1, 14, 1, 1, 1, 4, 1, 4086, 0, 4066, 178, 0, 4, 0, 2] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 118, 0, 33, 1, 1, 4, 0, 4094, 2615, 0, 1, 0, 0, 4095, 0, 1, 0, 0, 52, 4093, 4095, 1, 4095, 4095, 0, 0, 2, 4096, 2, 0, 0, 0, 4094, 0, 3, 4095, 0, 1, 1, 0, 4095, 1, 1, 1, 0, 2992, 4094, 3, 1, 0, 1170, 0, 4095, 72, 0, 0, 3198, 175, 0, 1, 3183, 4094, 0, 0, 4095, 92, 0, 0, 0, 0, 0, 4095, 1479, 0, 4095, 3891, 4094, 574, 4090, 0, 1, 4079, 3453, 0, 3968, 1, 2507, 0, 0, 0, 0, 0, 1, 116, 0, 3815, 1, 1, 3998, 0, 3109, 1, 1, 1, 1, 1, 0, 0, 451, 4091, 1, 1, 0, 1, 0, 1, 14, 1, 4086, 0, 3940, 51, 0, 6, 0, 4] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[2, 0, 705, 0, 516, 0, 0, 1, 0, 4093, 4037, 2, 0, 0, 0, 4094, 0, 0, 0, 0, 361, 4077, 4092, 0, 4096, 4094, 0, 0, 0, 4094, 0, 0, 2, 0, 4084, 12, 0, 4095, 1, 0, 0, 0, 4096, 0, 17, 0, 0, 3487, 4093, 1012, 0, 3, 883, 0, 4096, 33, 1, 0, 3146, 958, 0, 0, 4077, 4093, 0, 0, 4092, 518, 0, 0, 0, 2, 2, 4096, 2444, 0, 4095, 4047, 4096, 750, 4093, 0, 0, 4060, 2418, 0, 2446, 0, 214, 0, 0, 0, 0, 0, 0, 31, 2, 2600, 0, 0, 4094, 0, 1959, 19, 0, 0, 0, 5, 0, 0, 217, 4096, 0, 0, 2, 0, 0, 0, 29, 0, 4095, 0, 4043, 43, 0, 3, 0, 8] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[1, 0, 449, 0, 140, 0, 0, 0, 0, 4093, 3157, 1, 0, 0, 0, 4094, 0, 0, 0, 0, 24, 4081, 4094, 0, 4096, 4094, 0, 0, 0, 4094, 6, 0, 2, 0, 4086, 3, 0, 4095, 0, 0, 0, 0, 4096, 0, 27, 0, 0, 2721, 4088, 71, 0, 2, 853, 2, 4094, 314, 0, 0, 2533, 77, 0, 0, 3697, 4096, 0, 0, 4096, 225, 0, 0, 0, 0, 1, 4096, 1635, 0, 4096, 3868, 4096, 743, 3778, 0, 0, 4073, 3217, 1, 3511, 0, 3148, 0, 0, 0, 0, 0, 0, 8, 2, 2876, 0, 0, 4054, 0, 3169, 23, 0, 0, 0, 2, 2, 0, 634, 4085, 0, 0, 2, 0, 0, 0, 0, 0, 4073, 0, 4067, 197, 2, 5, 0, 6] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[0, 1, 667, 0, 75, 0, 0, 0, 1, 4088, 3929, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 55, 4048, 4059, 0, 4095, 4092, 0, 0, 0, 4071, 0, 0, 2, 0, 4094, 70, 0, 4095, 0, 0, 0, 0, 4095, 0, 48, 0, 1, 2544, 4095, 182, 1, 2, 1882, 0, 4096, 0, 0, 0, 3596, 484, 0, 0, 3941, 4096, 0, 1, 4095, 283, 0, 1, 1, 0, 26, 4095, 1282, 0, 4095, 3267, 4095, 780, 3994, 16, 0, 4089, 1850, 0, 3790, 0, 2295, 0, 0, 0, 0, 1, 1, 13, 0, 3139, 0, 0, 3937, 1, 2416, 6, 1, 0, 1, 0, 3, 1, 508, 4064, 1, 1, 33, 0, 0, 1, 0, 1, 4046, 0, 4059, 147, 0, 27, 3, 5] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 106, 0, 610, 1, 1, 0, 0, 4092, 4085, 0, 1, 0, 0, 4095, 0, 0, 0, 0, 401, 4089, 4095, 1, 4095, 4094, 0, 0, 0, 4095, 0, 1, 3, 0, 3935, 0, 0, 4095, 0, 0, 1, 1, 4095, 1, 2, 1, 1, 3670, 4095, 282, 1, 0, 745, 0, 4095, 0, 0, 1, 3769, 696, 0, 1, 3976, 4095, 0, 0, 4091, 802, 0, 0, 0, 0, 0, 4095, 1685, 1, 4095, 4084, 4095, 892, 3920, 0, 1, 4047, 1010, 0, 3329, 1, 2480, 0, 0, 0, 1, 0, 0, 16, 0, 2813, 0, 1, 4048, 0, 1221, 11, 1, 1, 1, 4, 0, 0, 574, 4077, 1, 1, 11, 1, 0, 1, 9, 1, 4095, 0, 4076, 21, 0, 13, 0, 22] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 43, 0, 118, 0, 0, 0, 0, 4093, 4080, 2, 0, 0, 0, 4095, 0, 0, 0, 0, 76, 4067, 4089, 0, 4096, 4088, 0, 0, 0, 4094, 0, 0, 1, 0, 4093, 18, 0, 4095, 15, 0, 0, 0, 4096, 0, 71, 0, 0, 3223, 4088, 1687, 0, 1, 2065, 1, 4096, 138, 1, 0, 3665, 171, 0, 0, 4028, 4093, 3, 0, 4096, 1006, 0, 0, 0, 0, 0, 4096, 432, 0, 4096, 3774, 4096, 754, 3626, 2, 0, 4095, 476, 0, 3938, 0, 3008, 0, 0, 0, 0, 0, 0, 26, 0, 1107, 0, 0, 3959, 0, 3444, 2, 0, 0, 0, 1, 3, 0, 94, 4087, 0, 0, 10, 0, 0, 0, 126, 0, 4057, 0, 3998, 61, 0, 10, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[0, 0, 245, 0, 889, 0, 0, 0, 0, 4095, 4094, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 355, 4069, 4090, 0, 4096, 4095, 0, 0, 3, 4095, 1, 0, 0, 0, 4089, 0, 5, 4095, 0, 0, 0, 0, 4095, 0, 7, 0, 0, 3423, 4096, 1621, 0, 0, 2196, 0, 4094, 2, 0, 0, 3178, 196, 0, 1, 4086, 4096, 1, 0, 4096, 379, 0, 0, 0, 0, 1, 4095, 1938, 0, 4096, 4025, 4096, 935, 4036, 0, 0, 4096, 726, 1, 2323, 0, 1356, 0, 1, 0, 0, 0, 1, 21, 0, 1908, 0, 0, 4072, 1, 2815, 26, 0, 0, 0, 0, 1, 0, 112, 4091, 1, 0, 1, 0, 0, 0, 12, 0, 4094, 0, 4072, 138, 0, 2, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 434, 0, 210, 1, 1, 0, 0, 4092, 4030, 0, 1, 1, 0, 4095, 3688, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 79, 4076, 4095, 1, 0, 3952, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 4095, 0, 0, 1, 0, 4095, 1, 11, 1, 0, 3420, 701, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 176, 1, 0, 397, 0, 0, 254, 0, 1, 3531, 540, 1, 1, 4092, 2661, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 954, 1, 1, 0, 0, 0, 1541, 826, 1, 0, 4005, 4095, 815, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 1, 1, 4093, 2958, 0, 2762, 1, 2373, 0, 0, 0, 1, 12, 1, 3, 732, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2144, 0, 1, 4094, 0, 2599, 56, 1, 1, 1, 0, 0, 0, 92, 4091, 2359, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 14, 1, 1, 1, 4, 1, 4086, 0, 4066, 178, 0, 4, 0, 2, 4094, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16634, 16301, 16606, 15741, 16335, 17141, 15846, 16468] avg=16384.0000 max=17141 max_rank=5 min=15741 min_rank=3 imbalance_ratio=1.046204 max_load_reduction=113931 max_load_reduction_pct=0.869225 moved_to_echo=65790 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 118, 0, 33, 1, 1, 4, 0, 4094, 2615, 0, 1, 0, 0, 4095, 3689, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 52, 4093, 4095, 1, 0, 3950, 0, 0, 2, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 3, 4095, 0, 1, 1, 0, 4095, 1, 1, 1, 0, 2992, 706, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 1, 0, 1170, 0, 0, 72, 0, 0, 3198, 175, 0, 1, 3183, 2650, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 92, 0, 0, 0, 0, 0, 1535, 1479, 0, 0, 3891, 4094, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 1, 4079, 3453, 0, 3968, 1, 2507, 0, 0, 0, 0, 0, 1, 116, 738, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3815, 1, 1, 3998, 0, 3109, 1, 1, 1, 1, 1, 0, 0, 451, 4091, 2364, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 1, 14, 1, 4086, 0, 3940, 51, 0, 6, 0, 4, 4095, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14733, 16292, 16186, 14547, 15760, 19099, 18241, 16214] avg=16384.0000 max=19099 max_rank=5 min=14547 min_rank=3 imbalance_ratio=1.165710 max_load_reduction=111973 max_load_reduction_pct=0.854286 moved_to_echo=69314 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[2, 0, 705, 0, 516, 0, 0, 1, 0, 4093, 4037, 2, 0, 0, 0, 4094, 3689, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 4077, 4092, 0, 0, 3951, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4084, 12, 0, 4095, 1, 0, 0, 0, 4096, 0, 17, 0, 0, 3487, 701, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1012, 0, 3, 883, 0, 0, 33, 1, 0, 3146, 958, 0, 0, 4077, 2660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 518, 0, 0, 0, 2, 2, 1542, 2444, 0, 0, 4047, 4096, 750, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4060, 2418, 0, 2446, 0, 214, 0, 0, 0, 0, 0, 0, 31, 732, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2600, 0, 0, 4094, 0, 1959, 19, 0, 0, 0, 5, 0, 0, 217, 4096, 2359, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 29, 0, 4095, 0, 4043, 43, 0, 3, 0, 8, 4096, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17220, 16575, 16690, 16866, 17493, 14137, 15758, 16333] avg=16384.0000 max=17493 max_rank=4 min=14137 min_rank=5 imbalance_ratio=1.067688 max_load_reduction=113579 max_load_reduction_pct=0.866539 moved_to_echo=63721 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 667, 0, 75, 0, 0, 0, 1, 4088, 3929, 0, 0, 0, 0, 4095, 3689, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 55, 4048, 4059, 0, 0, 3950, 0, 0, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 70, 0, 4095, 0, 0, 0, 0, 4095, 0, 48, 0, 1, 2544, 701, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 182, 1, 2, 1882, 0, 0, 0, 0, 0, 3596, 484, 0, 0, 3941, 2663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 283, 0, 1, 1, 0, 26, 1541, 1282, 0, 0, 3267, 4095, 780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 16, 0, 4089, 1850, 0, 3790, 0, 2295, 0, 0, 0, 0, 1, 1, 13, 732, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3139, 0, 0, 3937, 1, 2416, 6, 1, 0, 1, 0, 3, 1, 508, 4064, 2359, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 33, 0, 0, 1, 0, 1, 4046, 0, 4059, 147, 0, 27, 3, 5, 4095, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16626, 16184, 15845, 16846, 15372, 16923, 16843, 16433] avg=16384.0000 max=16923 max_rank=5 min=15372 min_rank=4 imbalance_ratio=1.032898 max_load_reduction=114149 max_load_reduction_pct=0.870888 moved_to_echo=65571 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 449, 0, 140, 0, 0, 0, 0, 4093, 3157, 1, 0, 0, 0, 4094, 3687, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 4081, 4094, 0, 0, 3951, 0, 0, 0, 4094, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4086, 3, 0, 4095, 0, 0, 0, 0, 4096, 0, 27, 0, 0, 2721, 701, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 71, 0, 2, 853, 2, 0, 314, 0, 0, 2533, 77, 0, 0, 3697, 2663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 225, 0, 0, 0, 0, 1, 1542, 1635, 0, 0, 3868, 4096, 743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3778, 0, 0, 4073, 3217, 1, 3511, 0, 3148, 0, 0, 0, 0, 0, 0, 8, 732, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2876, 0, 0, 4054, 0, 3169, 23, 0, 0, 0, 2, 2, 0, 634, 4085, 2359, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4073, 0, 4067, 197, 2, 5, 0, 6, 4096, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15705, 16250, 15926, 14300, 16206, 18611, 17613, 16461] avg=16384.0000 max=18611 max_rank=5 min=14300 min_rank=3 imbalance_ratio=1.135925 max_load_reduction=112461 max_load_reduction_pct=0.858009 moved_to_echo=68891 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 43, 0, 118, 0, 0, 0, 0, 4093, 4080, 2, 0, 0, 0, 4095, 3690, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 4067, 4089, 0, 0, 3946, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4093, 18, 0, 4095, 15, 0, 0, 0, 4096, 0, 71, 0, 0, 3223, 701, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 1687, 0, 1, 2065, 1, 0, 138, 1, 0, 3665, 171, 0, 0, 4028, 2660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4096, 1006, 0, 0, 0, 0, 0, 1542, 432, 0, 0, 3774, 4096, 754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3626, 2, 0, 4095, 476, 0, 3938, 0, 3008, 0, 0, 0, 0, 0, 0, 26, 732, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1107, 0, 0, 3959, 0, 3444, 2, 0, 0, 0, 1, 3, 0, 94, 4087, 2359, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 126, 0, 4057, 0, 3998, 61, 0, 10, 1, 1, 4096, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16204, 16272, 16508, 18505, 15703, 16045, 15462, 16373] avg=16384.0000 max=18505 max_rank=3 min=15462 min_rank=6 imbalance_ratio=1.129456 max_load_reduction=112567 max_load_reduction_pct=0.858818 moved_to_echo=63583 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 106, 0, 610, 1, 1, 0, 0, 4092, 4085, 0, 1, 0, 0, 4095, 3689, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 4089, 4095, 1, 0, 3951, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3935, 0, 0, 4095, 0, 0, 1, 1, 4095, 1, 2, 1, 1, 3670, 701, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 282, 1, 0, 745, 0, 0, 0, 0, 1, 3769, 696, 0, 1, 3976, 2662, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 802, 0, 0, 0, 0, 0, 1541, 1685, 1, 0, 4084, 4095, 892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3920, 0, 1, 4047, 1010, 0, 3329, 1, 2480, 0, 0, 0, 1, 0, 0, 16, 732, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2813, 0, 1, 4048, 0, 1221, 11, 1, 1, 1, 4, 0, 0, 574, 4077, 2359, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 11, 1, 0, 1, 9, 1, 4095, 0, 4076, 21, 0, 13, 0, 22, 4095, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16762, 16633, 16701, 16228, 17191, 15680, 15517, 16360] avg=16384.0000 max=17191 max_rank=4 min=15517 min_rank=6 imbalance_ratio=1.049255 max_load_reduction=113881 max_load_reduction_pct=0.868843 moved_to_echo=64748 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 245, 0, 889, 0, 0, 0, 0, 4095, 4094, 0, 0, 0, 0, 4095, 3687, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 355, 4069, 4090, 0, 0, 3952, 0, 0, 3, 4095, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 5, 4095, 0, 0, 0, 0, 4095, 0, 7, 0, 0, 3423, 701, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1621, 0, 0, 2196, 0, 0, 2, 0, 0, 3178, 196, 0, 1, 4086, 2663, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 379, 0, 0, 0, 0, 1, 1541, 1938, 0, 0, 4025, 4096, 935, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 4096, 726, 1, 2323, 0, 1356, 0, 1, 0, 0, 0, 1, 21, 732, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1908, 0, 0, 4072, 1, 2815, 26, 0, 0, 0, 0, 1, 0, 112, 4091, 2359, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 12, 0, 4094, 0, 4072, 138, 0, 2, 1, 0, 4096, 4013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17188, 16565, 16610, 18039, 17012, 13436, 15792, 16430] avg=16384.0000 max=18039 max_rank=3 min=13436 min_rank=5 imbalance_ratio=1.101013 max_load_reduction=113033 max_load_reduction_pct=0.862373 moved_to_echo=62670 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 32640, 128, 128, 32768, 128, 128, 128, 128, 32768, 128, 256, 128, 128, 25600, 5632, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[31616, 128, 128, 32640, 16128, 128, 26112, 128, 17408, 0, 128, 0, 128, 128, 128, 256, 5888, 1152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 20480, 128, 128, 32256, 128, 20736, 256, 128, 128, 128, 128, 128, 128, 2688, 32768, 18944, 3328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 128, 256, 128, 32640, 0, 32384, 896, 128, 128, 128, 128, 32768, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 2816, 0, 2688, 128, 128, 128, 128, 32768, 30080, 128, 128, 128, 0, 32768, 29568, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 5120, 128, 128, 10240, 128, 0, 896, 128, 128, 26624, 3328, 128, 128, 31104, 21376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 32768, 4352, 128, 128, 128, 128, 128, 12416, 11776, 128, 0, 30976, 32768, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 128, 128, 1408, 32640, 32768, 128, 0, 31616, 0, 0, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2780, 4093, 3, 0, 25, 2971, 4096, 0, 0, 4093, 3, 0, 0, 1, 0, 3, 0, 0, 2756, 3495, 0, 1836, 13, 0, 0, 4093, 0, 0, 3396, 4062, 1, 383, 0, 0, 4091, 2, 215, 4096, 2, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 0, 1717, 0, 2, 0, 0, 0, 38, 4069, 413, 0, 4096, 0, 0, 0, 0, 4087, 0, 0, 4093, 3395, 0, 0, 0, 4088, 2, 106, 2, 3891, 0, 4089, 4094, 0, 0, 4086, 0, 5, 0, 0, 26, 0, 4058, 2665, 4079, 0, 384, 3843, 0, 0, 4041, 0, 4, 0, 0, 4096, 3344, 0, 0, 0, 25, 9, 0, 0, 0, 32, 0, 0, 0, 0, 0, 4094, 0, 3396, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3830, 4082, 0, 0, 0, 3032, 4095, 0, 0, 4055, 32, 0, 0, 0, 1, 7, 1, 0, 1042, 3532, 1, 2935, 70, 0, 0, 4096, 0, 0, 4080, 4060, 2, 504, 0, 1, 4049, 1, 375, 4096, 10, 2, 0, 0, 0, 0, 4096, 4095, 0, 1, 0, 1427, 0, 0, 0, 1, 0, 18, 4070, 223, 0, 4095, 1, 0, 0, 7, 4072, 0, 0, 4083, 3669, 0, 0, 0, 4095, 0, 0, 12, 4053, 0, 4042, 4095, 1, 0, 3954, 0, 6, 0, 0, 48, 1, 2019, 3311, 4067, 0, 228, 3933, 1, 1, 3252, 0, 0, 0, 1, 4095, 3651, 0, 0, 0, 55, 126, 1, 0, 1, 150, 1, 0, 1, 1, 1, 4093, 0, 3927, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 2993, 4087, 0, 0, 1, 3272, 4095, 0, 0, 4034, 0, 1, 0, 1, 1, 0, 1, 0, 2775, 2572, 0, 3064, 15, 1, 1, 4093, 0, 0, 4066, 3997, 0, 231, 1, 0, 4095, 0, 533, 4094, 0, 5, 3, 0, 2, 1, 4095, 4095, 0, 1, 1, 1297, 0, 0, 0, 1, 1, 12, 4073, 624, 0, 4095, 1, 0, 0, 11, 4094, 1, 1, 4084, 2616, 1, 1, 1, 4093, 0, 147, 9, 3794, 1, 4091, 4095, 0, 1, 4062, 0, 0, 0, 0, 29, 1, 2566, 2381, 4085, 1, 966, 3996, 0, 1, 3954, 0, 0, 1, 1, 4095, 3733, 1, 0, 0, 66, 13, 1, 0, 1, 73, 0, 0, 1, 0, 0, 4095, 0, 3572, 1, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[1, 1, 1, 1, 3850, 4092, 0, 0, 0, 2611, 4095, 0, 0, 4067, 6, 1, 0, 0, 1, 0, 1, 0, 657, 3835, 0, 2264, 79, 0, 1, 4094, 1, 0, 4086, 4042, 0, 481, 1, 0, 4095, 0, 606, 4095, 0, 0, 0, 0, 1, 1, 4095, 4095, 0, 1, 1, 3176, 2, 0, 0, 1, 1, 3, 4094, 263, 0, 4095, 0, 0, 0, 15, 4095, 1, 1, 4091, 3517, 1, 1, 1, 4080, 0, 0, 3, 4072, 0, 4077, 4095, 0, 0, 4063, 0, 0, 0, 0, 26, 0, 3527, 1991, 4074, 1, 654, 3879, 0, 1, 2056, 0, 0, 1, 1, 4095, 3462, 1, 0, 0, 29, 77, 1, 0, 0, 4, 0, 0, 1, 1, 0, 4095, 0, 4086, 1, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3699, 4060, 4, 0, 0, 2586, 4096, 0, 0, 4081, 12, 0, 0, 5, 0, 2, 0, 0, 3421, 3069, 0, 2681, 13, 0, 0, 4089, 0, 0, 4077, 4026, 2, 29, 0, 0, 4093, 3, 1099, 4096, 5, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 1104, 6, 1, 0, 0, 0, 14, 4094, 865, 0, 4096, 0, 0, 0, 0, 3988, 0, 0, 4092, 3010, 0, 0, 0, 4062, 2, 13, 338, 3966, 0, 4071, 4094, 0, 0, 4036, 0, 6, 0, 0, 116, 0, 2636, 1550, 4088, 0, 657, 4078, 0, 0, 3062, 0, 5, 0, 0, 4096, 3435, 0, 0, 0, 33, 70, 0, 0, 0, 29, 0, 0, 0, 0, 0, 4093, 0, 3826, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 2516, 4095, 0, 0, 0, 3345, 4095, 0, 0, 3978, 16, 1, 0, 3, 1, 0, 1, 1, 2626, 3231, 0, 2687, 19, 1, 1, 4087, 0, 0, 4014, 4071, 0, 39, 1, 0, 4095, 3, 360, 4095, 13, 2, 0, 0, 4, 1, 4095, 4095, 0, 1, 0, 576, 0, 0, 0, 1, 1, 13, 4074, 1078, 0, 4095, 1, 0, 0, 15, 4091, 1, 0, 4082, 3188, 1, 1, 1, 4091, 3, 203, 51, 3907, 0, 4080, 4095, 0, 0, 3997, 0, 9, 0, 0, 23, 0, 3924, 1955, 4077, 1, 308, 4046, 0, 1, 3531, 0, 0, 1, 1, 4092, 3683, 1, 0, 0, 107, 97, 1, 0, 1, 25, 0, 0, 1, 0, 0, 4095, 0, 3848, 1, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3980, 4083, 2, 0, 1, 2549, 4096, 0, 0, 4068, 10, 0, 0, 0, 0, 2, 0, 0, 2706, 2405, 0, 3487, 101, 0, 0, 4093, 0, 0, 4066, 3520, 1, 54, 0, 0, 4093, 3, 464, 4096, 5, 0, 0, 0, 1, 0, 4096, 4096, 1, 0, 0, 882, 1, 1, 0, 0, 0, 17, 4025, 785, 0, 4096, 0, 0, 0, 0, 4071, 0, 0, 4093, 3592, 0, 0, 0, 4079, 0, 0, 35, 3885, 0, 4078, 4095, 0, 0, 4016, 0, 5, 0, 0, 20, 0, 3891, 3771, 4075, 0, 199, 3690, 0, 0, 3988, 0, 2, 0, 0, 4096, 2807, 0, 0, 0, 24, 184, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4094, 0, 2385, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3799, 4092, 3, 0, 0, 3282, 4096, 0, 0, 4051, 8, 0, 0, 0, 0, 4, 0, 0, 2232, 2276, 0, 1771, 18, 0, 0, 4094, 0, 0, 3966, 4041, 1, 587, 0, 0, 4094, 2, 409, 4096, 2, 0, 1, 0, 1, 0, 4096, 4096, 0, 0, 0, 2751, 0, 0, 0, 0, 0, 34, 3876, 881, 0, 4096, 0, 0, 0, 4, 4066, 0, 0, 4096, 2494, 0, 0, 0, 4092, 0, 1, 101, 4045, 0, 4082, 4095, 1, 0, 4064, 0, 6, 0, 0, 57, 0, 3830, 2919, 4084, 0, 405, 4094, 1, 0, 4045, 0, 1, 0, 0, 4096, 2468, 0, 0, 1, 48, 92, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4095, 0, 2931, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 2780, 4093, 3, 0, 25, 2971, 1842, 0, 0, 4093, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 2756, 3495, 0, 1836, 13, 0, 0, 4093, 0, 0, 2229, 1761, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3396, 4062, 1, 383, 0, 0, 4091, 2, 215, 3543, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1867, 4096, 0, 0, 0, 1717, 0, 2, 0, 0, 0, 38, 4069, 413, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 4093, 3395, 0, 0, 0, 4088, 2, 106, 2, 553, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3891, 0, 2484, 0, 0, 0, 4086, 0, 5, 0, 0, 26, 0, 4058, 2665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 384, 3843, 0, 0, 4041, 0, 4, 0, 0, 4096, 3344, 0, 0, 0, 25, 1005, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 32, 0, 0, 0, 0, 0, 4094, 0, 3396, 0, 0, 0, 4094, 4079, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15810, 16523, 15697, 16298, 16590, 17215, 16742, 16197] avg=16384.0000 max=17215 max_rank=5 min=15697 min_rank=2 imbalance_ratio=1.050720 max_load_reduction=113857 max_load_reduction_pct=0.868660 moved_to_echo=66744 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 2993, 4087, 0, 0, 1, 3272, 1841, 0, 0, 4034, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 2775, 2572, 0, 3064, 15, 1, 1, 4093, 0, 0, 2228, 1761, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 3997, 0, 231, 1, 0, 4095, 0, 533, 3542, 0, 5, 3, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1867, 4095, 0, 1, 1, 1297, 0, 0, 0, 1, 1, 12, 4073, 624, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 11, 4094, 1, 1, 4084, 2616, 1, 1, 1, 4093, 0, 147, 9, 552, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3794, 1, 2485, 0, 0, 1, 4062, 0, 0, 0, 0, 29, 1, 2566, 2381, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 966, 3996, 0, 1, 3954, 0, 0, 1, 1, 4095, 3733, 1, 0, 0, 66, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, 0, 1, 73, 0, 0, 1, 0, 0, 4095, 0, 3572, 1, 0, 1, 4095, 4085, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16232, 16849, 16476, 16067, 15876, 15320, 17821, 16431] avg=16384.0000 max=17821 max_rank=6 min=15320 min_rank=5 imbalance_ratio=1.087708 max_load_reduction=113251 max_load_reduction_pct=0.864037 moved_to_echo=65448 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3830, 4082, 0, 0, 0, 3032, 1841, 0, 0, 4055, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 1, 0, 1042, 3532, 1, 2935, 70, 0, 0, 4096, 0, 0, 2229, 1761, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 4060, 2, 504, 0, 1, 4049, 1, 375, 3543, 10, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1867, 4095, 0, 1, 0, 1427, 0, 0, 0, 1, 0, 18, 4070, 223, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 4072, 0, 0, 4083, 3669, 0, 0, 0, 4095, 0, 0, 12, 553, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 2455, 0, 1, 0, 3954, 0, 6, 0, 0, 48, 1, 2019, 3311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 3933, 1, 1, 3252, 0, 0, 0, 1, 4095, 3651, 0, 0, 0, 55, 994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 1, 0, 1, 150, 1, 0, 1, 1, 1, 4093, 0, 3927, 0, 0, 0, 4095, 4067, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16872, 16007, 16627, 15797, 16753, 15848, 16211, 16957] avg=16384.0000 max=16957 max_rank=7 min=15797 min_rank=3 imbalance_ratio=1.034973 max_load_reduction=114115 max_load_reduction_pct=0.870628 moved_to_echo=65769 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 1, 1, 3850, 4092, 0, 0, 0, 2611, 1836, 0, 0, 4067, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 657, 3835, 0, 2264, 79, 0, 1, 4094, 1, 0, 2233, 1765, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 4042, 0, 481, 1, 0, 4095, 0, 606, 3540, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1862, 4095, 0, 1, 1, 3176, 2, 0, 0, 1, 1, 3, 4094, 263, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 4095, 1, 1, 4091, 3517, 1, 1, 1, 4080, 0, 0, 3, 555, 268, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4072, 0, 2466, 0, 0, 0, 4063, 0, 0, 0, 0, 26, 0, 3527, 1991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 654, 3879, 0, 1, 2056, 0, 0, 1, 1, 4095, 3462, 1, 0, 0, 29, 1006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 1, 0, 0, 4, 0, 0, 1, 1, 0, 4095, 0, 4086, 1, 0, 1, 4095, 4074, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16467, 15268, 16853, 17594, 16629, 16145, 15186, 16930] avg=16384.0000 max=17594 max_rank=3 min=15186 min_rank=6 imbalance_ratio=1.073853 max_load_reduction=113478 max_load_reduction_pct=0.865768 moved_to_echo=64890 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3699, 4060, 4, 0, 0, 2586, 1842, 0, 0, 4081, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2, 0, 0, 3421, 3069, 0, 2681, 13, 0, 0, 4089, 0, 0, 2229, 1761, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 4026, 2, 29, 0, 0, 4093, 3, 1099, 3543, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1867, 4096, 0, 0, 0, 1104, 6, 1, 0, 0, 0, 14, 4094, 865, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3988, 0, 0, 4092, 3010, 0, 0, 0, 4062, 2, 13, 338, 553, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 2473, 0, 0, 0, 4036, 0, 6, 0, 0, 116, 0, 2636, 1550, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 657, 4078, 0, 0, 3062, 0, 5, 0, 0, 4096, 3435, 0, 0, 0, 33, 1001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 29, 0, 0, 0, 0, 0, 4093, 0, 3826, 0, 0, 0, 4094, 4088, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16284, 17604, 16877, 16143, 16321, 14783, 16367, 16693] avg=16384.0000 max=17604 max_rank=1 min=14783 min_rank=5 imbalance_ratio=1.074463 max_load_reduction=113468 max_load_reduction_pct=0.865692 moved_to_echo=64164 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3980, 4083, 2, 0, 1, 2549, 1842, 0, 0, 4068, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2706, 2405, 0, 3487, 101, 0, 0, 4093, 0, 0, 2229, 1761, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 3520, 1, 54, 0, 0, 4093, 3, 464, 3543, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1867, 4096, 1, 0, 0, 882, 1, 1, 0, 0, 0, 17, 4025, 785, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 4093, 3592, 0, 0, 0, 4079, 0, 0, 35, 553, 263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3885, 0, 2478, 0, 0, 0, 4016, 0, 5, 0, 0, 20, 0, 3891, 3771, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 3690, 0, 0, 3988, 0, 2, 0, 0, 4096, 2807, 0, 0, 0, 24, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4094, 0, 2385, 0, 0, 0, 4095, 4075, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16535, 17119, 15750, 15771, 16686, 18066, 15808, 15337] avg=16384.0000 max=18066 max_rank=5 min=15337 min_rank=7 imbalance_ratio=1.102661 max_load_reduction=113006 max_load_reduction_pct=0.862167 moved_to_echo=65897 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 2516, 4095, 0, 0, 0, 3345, 1841, 0, 0, 3978, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 1, 1, 2626, 3231, 0, 2687, 19, 1, 1, 4087, 0, 0, 2228, 1761, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4014, 4071, 0, 39, 1, 0, 4095, 3, 360, 3542, 13, 2, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1867, 4095, 0, 1, 0, 576, 0, 0, 0, 1, 1, 13, 4074, 1078, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 15, 4091, 1, 0, 4082, 3188, 1, 1, 1, 4091, 3, 203, 51, 553, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3907, 0, 2478, 0, 0, 0, 3997, 0, 9, 0, 0, 23, 0, 3924, 1955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 308, 4046, 0, 1, 3531, 0, 0, 1, 1, 4092, 3683, 1, 0, 0, 107, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 1, 0, 1, 25, 0, 0, 1, 0, 0, 4095, 0, 3848, 1, 1, 0, 4095, 4077, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15795, 16982, 16145, 15801, 16546, 16293, 16775, 16735] avg=16384.0000 max=16982 max_rank=1 min=15795 min_rank=0 imbalance_ratio=1.036499 max_load_reduction=114090 max_load_reduction_pct=0.870438 moved_to_echo=66349 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 3799, 4092, 3, 0, 0, 3282, 1842, 0, 0, 4051, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 2232, 2276, 0, 1771, 18, 0, 0, 4094, 0, 0, 2229, 1761, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 4041, 1, 587, 0, 0, 4094, 2, 409, 3543, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1867, 4096, 0, 0, 0, 2751, 0, 0, 0, 0, 0, 34, 3876, 881, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4066, 0, 0, 4096, 2494, 0, 0, 0, 4092, 0, 1, 101, 553, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 2480, 0, 1, 0, 4064, 0, 6, 0, 0, 57, 0, 3830, 2919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 405, 4094, 1, 0, 4045, 0, 1, 0, 0, 4096, 2468, 0, 0, 1, 48, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4095, 0, 2931, 0, 0, 0, 4095, 4084, 493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17077, 14720, 16647, 17601, 15671, 17402, 16162, 15792] avg=16384.0000 max=17601 max_rank=3 min=14720 min_rank=1 imbalance_ratio=1.074280 max_load_reduction=113471 max_load_reduction_pct=0.865715 moved_to_echo=65027 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 3840, 31616, 128, 128, 28032, 0, 128, 128, 128, 32768, 26624, 128, 0, 128, 512, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[31872, 31872, 128, 2432, 128, 128, 32768, 128, 4096, 28416, 128, 128, 128, 0, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 0, 0, 128, 32640, 128, 128, 32768, 25600, 128, 128, 128, 32768, 128, 512, 640, 4480, 2176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[31616, 128, 19840, 0, 128, 128, 32384, 0, 128, 0, 0, 384, 128, 26496, 20608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[768, 128, 0, 128, 384, 128, 0, 128, 128, 128, 32768, 0, 28032, 128, 128, 128, 32768, 32640, 3968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[14976, 32768, 128, 128, 128, 13056, 128, 128, 0, 128, 128, 256, 32384, 5248, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 128, 27520, 32768, 128, 0, 128, 23680, 14848, 0, 0, 32512, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[0, 128, 128, 128, 128, 128, 18304, 24448, 128, 20736, 384, 128, 128, 32768, 128, 0, 17920, 14208, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 1, 0, 2320, 1, 0, 1, 4095, 8, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 4086, 1, 1, 1, 4095, 4095, 4095, 1, 4095, 1, 4095, 1387, 1, 0, 0, 0, 3240, 3964, 0, 0, 4095, 4095, 1855, 4095, 1, 0, 0, 0, 1, 4, 4095, 1, 1, 0, 4095, 0, 0, 11, 2185, 4095, 0, 325, 4092, 3267, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 4083, 0, 1, 1, 1, 1, 4095, 1, 0, 4095, 1, 1, 133, 1, 0, 4095, 9, 0, 1, 0, 4095, 4094, 1, 1, 0, 3930, 1, 1, 4094, 0, 0, 0, 4028, 41, 3303, 0, 0, 0, 28, 0, 0, 0, 0, 2746, 0, 4095, 1, 0, 4094] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[17, 1, 4061, 0, 1, 0, 1598, 1, 0, 0, 4095, 19, 0, 1, 1, 0, 0, 1, 32, 0, 0, 0, 1, 1, 0, 4050, 0, 0, 0, 3768, 4095, 4088, 0, 4095, 1, 4095, 2130, 0, 0, 0, 2, 2793, 3820, 0, 1, 4095, 4096, 1117, 4095, 1, 1, 0, 1, 0, 44, 4095, 0, 0, 0, 4095, 0, 0, 142, 2153, 4095, 0, 145, 4088, 3202, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4077, 1, 25, 1, 0, 0, 4095, 1, 0, 4094, 1, 1, 212, 1, 0, 4095, 2, 1, 0, 140, 4093, 4092, 0, 0, 0, 3873, 0, 1, 4095, 0, 1, 37, 3874, 151, 3835, 0, 0, 1, 123, 0, 0, 0, 1, 3698, 23, 4095, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[3, 0, 4075, 0, 0, 0, 1735, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 0, 0, 8, 0, 2, 0, 0, 0, 0, 4086, 0, 0, 0, 3528, 4096, 4093, 0, 4096, 0, 4095, 746, 1, 0, 0, 2, 3445, 4076, 0, 0, 4096, 4096, 1198, 4090, 0, 9, 1, 0, 0, 16, 4094, 0, 0, 0, 4096, 0, 1, 6, 3177, 4096, 2, 365, 4094, 3126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 15, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 103, 0, 0, 4096, 94, 0, 0, 2, 4094, 4093, 0, 0, 0, 3979, 0, 0, 4096, 0, 0, 19, 3927, 14, 3872, 0, 0, 2, 2, 0, 25, 0, 0, 3411, 7, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4081, 0, 0, 0, 2854, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 0, 0, 21, 0, 1, 0, 0, 0, 0, 4086, 0, 0, 0, 3916, 4096, 4091, 0, 4096, 0, 4095, 1638, 1, 0, 0, 1, 2527, 3544, 0, 0, 4096, 4096, 885, 4036, 0, 21, 0, 0, 0, 10, 4095, 0, 0, 0, 4096, 0, 0, 0, 3300, 4096, 1, 595, 4096, 3287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 14, 0, 0, 0, 4096, 0, 6, 4094, 0, 0, 1312, 0, 0, 4096, 4, 0, 0, 1, 4095, 4095, 0, 3, 0, 3928, 0, 0, 4096, 0, 0, 81, 2625, 74, 3524, 0, 0, 0, 84, 0, 0, 0, 0, 2688, 6, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 1, 3693, 0, 0, 0, 4095, 3, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 4077, 0, 1, 1, 4064, 4095, 4095, 0, 4095, 1, 4095, 849, 5, 1, 1, 0, 1033, 4086, 1, 0, 4095, 4094, 328, 4095, 1, 0, 0, 0, 0, 9, 4095, 1, 1, 0, 4095, 0, 0, 151, 3622, 4095, 0, 6, 4094, 2883, 1, 0, 1, 0, 2, 0, 1, 0, 1, 1, 3995, 0, 0, 1, 1, 1, 4092, 1, 0, 4095, 0, 1, 115, 1, 0, 4095, 4, 0, 1, 0, 4095, 4095, 0, 0, 0, 3839, 1, 1, 4094, 1, 0, 6, 4066, 21, 4088, 0, 1, 0, 46, 0, 0, 0, 0, 4065, 0, 4095, 0, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 0, 996, 0, 3, 0, 4096, 8, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 3964, 4096, 4091, 0, 4096, 0, 4094, 1903, 18, 0, 0, 3, 3375, 3920, 0, 0, 4096, 4096, 1586, 4093, 0, 7, 0, 0, 0, 17, 4094, 0, 0, 0, 4096, 0, 0, 50, 2432, 4096, 2, 42, 4093, 3583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 192, 0, 0, 4096, 0, 0, 0, 6, 4096, 4094, 0, 0, 0, 3885, 0, 0, 4096, 0, 0, 32, 3869, 28, 3841, 0, 0, 0, 2, 0, 6, 0, 0, 3111, 6, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[0, 0, 4086, 0, 0, 0, 1455, 0, 0, 1, 4095, 23, 1, 0, 1, 0, 0, 1, 9, 1, 0, 0, 1, 1, 0, 4083, 0, 0, 1, 3987, 4095, 4095, 0, 4095, 1, 4095, 449, 0, 1, 0, 0, 3829, 4043, 0, 0, 4095, 4095, 533, 4095, 1, 0, 0, 0, 0, 13, 4095, 1, 1, 0, 4095, 0, 0, 12, 3048, 4095, 0, 998, 4090, 3802, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 4070, 0, 157, 1, 1, 1, 4095, 1, 0, 4093, 1, 1, 137, 1, 0, 4095, 53, 0, 1, 1, 4095, 4094, 1, 0, 0, 3952, 1, 1, 4094, 1, 0, 10, 3275, 92, 3945, 0, 1, 0, 41, 0, 0, 0, 0, 3048, 4, 4095, 0, 0, 4090] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[22, 0, 4071, 0, 0, 0, 2369, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 1, 0, 19, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 3880, 4096, 4093, 0, 4096, 0, 4095, 1023, 0, 0, 0, 1, 2270, 4025, 0, 0, 4095, 4096, 1034, 4095, 0, 4, 0, 0, 0, 24, 4096, 0, 0, 0, 4096, 0, 1, 0, 3345, 4096, 0, 1400, 4096, 3256, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4094, 0, 29, 0, 0, 0, 4096, 0, 2, 4095, 0, 0, 174, 0, 0, 4096, 16, 0, 0, 3, 4095, 4094, 0, 0, 1, 3430, 0, 0, 4096, 0, 0, 26, 3910, 88, 3524, 0, 0, 0, 0, 0, 4, 0, 0, 3021, 4, 4096, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 1, 0, 2320, 1, 0, 1, 4095, 8, 1, 0, 1, 0, 4095, 1965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 4086, 1, 1, 1, 4095, 4095, 4095, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4095, 1387, 1, 0, 0, 0, 3240, 3964, 0, 0, 3203, 0, 1855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 1, 4, 4095, 1, 1, 0, 4095, 0, 0, 11, 2185, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 325, 4092, 3267, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 4083, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 4095, 1, 0, 4095, 1, 1, 133, 1, 0, 4095, 9, 2130, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 4094, 1, 1, 0, 3930, 1, 1, 599, 0, 0, 0, 4028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 3303, 0, 0, 0, 28, 0, 0, 0, 0, 2746, 0, 4095, 1, 0, 4094, 753, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16581, 16579, 17747, 15617, 16198, 16179, 16751, 15420] avg=16384.0000 max=17747 max_rank=2 min=15420 min_rank=7 imbalance_ratio=1.083191 max_load_reduction=113325 max_load_reduction_pct=0.864601 moved_to_echo=64548 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[17, 1, 4061, 0, 1, 0, 1598, 1, 0, 0, 4095, 19, 0, 1, 1, 0, 4095, 1965, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 32, 0, 0, 0, 1, 1, 0, 4050, 0, 0, 0, 3768, 4095, 4088, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 2130, 0, 0, 0, 2, 2793, 3820, 0, 1, 3203, 0, 1117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 1, 0, 44, 4095, 0, 0, 0, 4095, 0, 0, 142, 2153, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 145, 4088, 3202, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 4077, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 25, 1, 0, 0, 4095, 1, 0, 4094, 1, 1, 212, 1, 0, 4095, 2, 2131, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 140, 4093, 4092, 0, 0, 0, 3873, 0, 1, 600, 0, 1, 37, 3874, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 3835, 0, 0, 1, 123, 0, 0, 0, 1, 3698, 23, 4095, 0, 0, 4096, 753, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15855, 16236, 17162, 15755, 15943, 16274, 16712, 17135] avg=16384.0000 max=17162 max_rank=2 min=15755 min_rank=3 imbalance_ratio=1.047485 max_load_reduction=113910 max_load_reduction_pct=0.869064 moved_to_echo=66064 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4081, 0, 0, 0, 2854, 0, 0, 0, 4096, 12, 0, 0, 0, 0, 4096, 1966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 1, 0, 0, 0, 0, 4086, 0, 0, 0, 3916, 4096, 4091, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1638, 1, 0, 0, 1, 2527, 3544, 0, 0, 3204, 0, 885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 21, 0, 0, 0, 10, 4095, 0, 0, 0, 4096, 0, 0, 0, 3300, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 595, 4096, 3287, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 4096, 0, 6, 4094, 0, 0, 1312, 0, 0, 4096, 4, 2130, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 4095, 0, 3, 0, 3928, 0, 0, 601, 0, 0, 81, 2625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 3524, 0, 0, 0, 84, 0, 0, 0, 0, 2688, 6, 4096, 0, 0, 4096, 753, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17105, 16411, 15895, 16686, 16500, 17366, 15429, 15680] avg=16384.0000 max=17366 max_rank=5 min=15429 min_rank=6 imbalance_ratio=1.059937 max_load_reduction=113706 max_load_reduction_pct=0.867508 moved_to_echo=64975 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4093, 0, 0, 1, 3693, 0, 0, 0, 4095, 3, 0, 0, 1, 0, 4095, 1966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 4077, 0, 1, 1, 4064, 4095, 4095, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 849, 5, 1, 1, 0, 1033, 4086, 1, 0, 3193, 0, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 9, 4095, 1, 1, 0, 4095, 0, 0, 151, 3622, 1131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 6, 4094, 2883, 1, 0, 1, 0, 2, 0, 1, 0, 1, 1, 3995, 339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4092, 1, 0, 4095, 0, 1, 115, 1, 0, 4095, 4, 2128, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 4095, 0, 0, 0, 3839, 1, 1, 586, 1, 0, 6, 4066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4088, 0, 1, 0, 46, 0, 0, 0, 0, 4065, 0, 4095, 0, 0, 4093, 759, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17947, 16539, 13593, 17201, 15419, 16153, 16691, 17529] avg=16384.0000 max=17947 max_rank=0 min=13593 min_rank=2 imbalance_ratio=1.095398 max_load_reduction=113125 max_load_reduction_pct=0.863075 moved_to_echo=65792 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[3, 0, 4075, 0, 0, 0, 1735, 0, 0, 0, 4096, 11, 0, 0, 0, 0, 4096, 1966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 2, 0, 0, 0, 0, 4086, 0, 0, 0, 3528, 4096, 4093, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 746, 1, 0, 0, 2, 3445, 4076, 0, 0, 3204, 0, 1198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 9, 1, 0, 0, 16, 4094, 0, 0, 0, 4096, 0, 1, 6, 3177, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 365, 4094, 3126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 4096, 0, 0, 4094, 0, 0, 103, 0, 0, 4096, 94, 2130, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 4093, 0, 0, 0, 3979, 0, 0, 601, 0, 0, 19, 3927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 3872, 0, 0, 2, 2, 0, 25, 0, 0, 3411, 7, 4096, 0, 0, 4096, 753, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15982, 16013, 16767, 16618, 16098, 16242, 16715, 16637] avg=16384.0000 max=16767 max_rank=2 min=15982 min_rank=0 imbalance_ratio=1.023376 max_load_reduction=114305 max_load_reduction_pct=0.872078 moved_to_echo=65692 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4089, 0, 0, 0, 996, 0, 3, 0, 4096, 8, 0, 0, 0, 0, 4096, 1966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 3964, 4096, 4091, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1903, 18, 0, 0, 3, 3375, 3920, 0, 0, 3204, 0, 1586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 7, 0, 0, 0, 17, 4094, 0, 0, 0, 4096, 0, 0, 50, 2432, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 42, 4093, 3583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 4096, 0, 0, 192, 0, 0, 4096, 0, 2130, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4096, 4094, 0, 0, 0, 3885, 0, 0, 601, 0, 0, 32, 3869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 3841, 0, 0, 0, 2, 0, 6, 0, 0, 3111, 6, 4096, 0, 0, 4096, 753, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15254, 16446, 18103, 15917, 16245, 16226, 16583, 16298] avg=16384.0000 max=18103 max_rank=2 min=15254 min_rank=0 imbalance_ratio=1.104919 max_load_reduction=112969 max_load_reduction_pct=0.861885 moved_to_echo=65352 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4086, 0, 0, 0, 1455, 0, 0, 1, 4095, 23, 1, 0, 1, 0, 4095, 1966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 9, 1, 0, 0, 1, 1, 0, 4083, 0, 0, 1, 3987, 4095, 4095, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 449, 0, 1, 0, 0, 3829, 4043, 0, 0, 3203, 0, 533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 13, 4095, 1, 1, 0, 4095, 0, 0, 12, 3048, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 998, 4090, 3802, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 4070, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 1, 1, 1, 4095, 1, 0, 4093, 1, 1, 137, 1, 0, 4095, 53, 2129, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 4094, 1, 0, 0, 3952, 1, 1, 599, 1, 0, 10, 3275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 3945, 0, 1, 0, 41, 0, 0, 0, 0, 3048, 4, 4095, 0, 0, 4090, 753, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15723, 16474, 16154, 16489, 17393, 16380, 16031, 16428] avg=16384.0000 max=17393 max_rank=4 min=15723 min_rank=0 imbalance_ratio=1.061584 max_load_reduction=113679 max_load_reduction_pct=0.867302 moved_to_echo=66232 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[22, 0, 4071, 0, 0, 0, 2369, 0, 0, 0, 4096, 5, 0, 0, 0, 0, 4096, 1966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 19, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 3880, 4096, 4093, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1023, 0, 0, 0, 1, 2270, 4025, 0, 0, 3203, 0, 1034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 4, 0, 0, 0, 24, 4096, 0, 0, 0, 4096, 0, 1, 0, 3345, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1400, 4096, 3256, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4094, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 4096, 0, 2, 4095, 0, 0, 174, 0, 0, 4096, 16, 2130, 1614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4095, 4094, 0, 0, 1, 3430, 0, 0, 601, 0, 0, 26, 3910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 3524, 0, 0, 0, 0, 0, 4, 0, 0, 3021, 4, 4096, 0, 0, 4096, 753, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16625, 16374, 15651, 16789, 17276, 16252, 16160, 15945] avg=16384.0000 max=17276 max_rank=4 min=15651 min_rank=2 imbalance_ratio=1.054443 max_load_reduction=113796 max_load_reduction_pct=0.868195 moved_to_echo=65633 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 32768, 0, 128, 128, 17024, 128, 128, 128, 32768, 128, 128, 128, 128, 0, 32768, 15744, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 256, 128, 128, 128, 32768, 128, 128, 32768, 128, 128, 2432, 128, 0, 32768, 256, 17152, 12928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 128, 128, 128, 128, 256, 32768, 128, 128, 0, 32768, 0, 128, 384, 23296, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 128, 128, 128, 128, 0, 128, 128, 0, 32768, 128, 128, 128, 31232, 32768, 32768, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 256, 32768, 32768, 128, 128, 128, 30848, 128, 128, 4864, 128, 128, 256, 29696, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 3968, 32768, 26496, 128, 0, 128, 0, 128, 0, 128, 0, 128, 128, 32640, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 32768, 10240, 128, 128, 128, 128, 22528, 31488, 128, 128, 25728, 0, 8576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[512, 29952, 0, 128, 128, 384, 0, 128, 0, 128, 25856, 128, 32768, 128, 0, 32768, 6144, 2944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 1, 4095, 4095, 1, 1, 0, 2516, 1, 0, 1, 1, 391, 4095, 1, 4094, 1, 0, 1, 0, 0, 1, 4071, 4095, 1, 0, 0, 0, 4094, 4095, 4095, 0, 0, 0, 0, 1, 4094, 130, 297, 0, 0, 0, 0, 3, 4095, 2974, 551, 0, 1, 2, 1, 4095, 0, 1, 1, 4037, 0, 1, 389, 66, 0, 4093, 4095, 4092, 1, 14, 1, 1, 0, 0, 1, 0, 1, 143, 4095, 4095, 4037, 4085, 3899, 1, 0, 0, 19, 4066, 1, 4095, 4095, 1, 0, 0, 4094, 284, 0, 1, 0, 1, 1, 0, 0, 4080, 0, 4095, 4095, 0, 0, 1, 0, 0, 14, 0, 0, 1, 0, 0, 806, 1, 0, 0, 4082, 1, 0, 0, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4094, 0, 0, 0, 1978, 0, 2, 0, 0, 147, 4096, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 8, 2, 0, 4076, 57, 890, 0, 0, 0, 0, 0, 4095, 3365, 66, 0, 0, 14, 0, 4095, 0, 0, 0, 3811, 0, 0, 711, 697, 0, 4096, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 4089, 4080, 3730, 4042, 3875, 0, 0, 0, 2, 4060, 0, 4096, 4096, 0, 0, 0, 4096, 583, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 4096, 2, 0, 22, 0, 0, 3, 0, 22, 0, 0, 0, 386, 0, 0, 0, 4095, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 1, 1, 1, 0, 4095, 4095, 0, 1, 0, 489, 1, 0, 0, 0, 180, 4095, 1, 4089, 1, 0, 1, 0, 0, 1, 4089, 4095, 1, 0, 0, 0, 4095, 4095, 4095, 0, 0, 0, 0, 1, 4092, 16, 121, 1, 0, 0, 0, 5, 4095, 1295, 2487, 0, 1, 0, 1, 4095, 0, 1, 1, 3316, 0, 1, 488, 1869, 0, 4095, 4095, 4093, 1, 37, 1, 1, 0, 2, 0, 0, 1, 344, 4096, 4095, 4006, 4092, 4080, 0, 0, 0, 6, 4069, 1, 4095, 4095, 1, 0, 0, 4095, 449, 0, 1, 0, 1, 0, 1, 0, 3808, 0, 4095, 4095, 0, 0, 19, 0, 0, 12, 0, 0, 1, 0, 1, 1634, 0, 0, 0, 4049, 0, 0, 0, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 2996, 0, 102, 0, 0, 936, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 4096, 4096, 0, 0, 0, 0, 4096, 4096, 4096, 0, 0, 4, 0, 0, 4087, 25, 90, 0, 0, 1, 0, 0, 4095, 1971, 803, 0, 0, 3, 0, 4096, 0, 0, 0, 3787, 0, 0, 135, 144, 0, 4094, 4096, 4086, 0, 13, 0, 0, 0, 0, 0, 0, 0, 69, 4082, 4081, 3667, 4095, 4091, 0, 0, 0, 0, 3396, 0, 4096, 4096, 0, 0, 0, 4096, 352, 0, 0, 0, 0, 0, 0, 0, 3796, 0, 4095, 4096, 1, 0, 296, 0, 0, 12, 0, 6, 0, 0, 0, 2030, 0, 0, 0, 4093, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 1, 1, 1, 1, 4095, 4096, 0, 1, 0, 1120, 0, 45, 0, 0, 279, 4095, 0, 4095, 1, 0, 0, 1, 3, 1, 4095, 4095, 1, 0, 0, 0, 4095, 4095, 4095, 1, 0, 31, 2, 0, 4045, 12, 125, 0, 1, 0, 0, 0, 4095, 2020, 1545, 0, 0, 1, 1, 4094, 0, 1, 1, 2395, 0, 1, 792, 1598, 0, 4095, 4095, 4057, 1, 39, 1, 0, 0, 0, 0, 1, 0, 122, 4073, 4096, 3919, 4095, 4090, 0, 0, 0, 33, 4028, 0, 4095, 4095, 1, 0, 1, 4095, 988, 0, 1, 0, 1, 0, 0, 0, 3990, 1, 4095, 4093, 0, 0, 0, 0, 0, 0, 0, 24, 1, 0, 1, 1614, 1, 0, 0, 4059, 1, 0, 1, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 2085, 0, 4, 0, 0, 868, 4096, 0, 4096, 0, 0, 0, 1, 6, 0, 4096, 4096, 0, 0, 0, 0, 4090, 4095, 4096, 0, 0, 4, 2, 0, 4085, 1, 352, 0, 0, 2, 0, 0, 4095, 1256, 971, 0, 0, 10, 0, 4095, 0, 0, 0, 3654, 0, 0, 456, 845, 0, 4094, 4096, 4091, 0, 1, 0, 0, 0, 0, 0, 0, 0, 340, 4092, 4090, 3893, 4096, 3920, 1, 0, 0, 0, 3926, 0, 4096, 4096, 0, 0, 0, 4096, 1339, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 4096, 4096, 0, 0, 38, 0, 0, 0, 0, 1, 0, 0, 0, 710, 0, 0, 0, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 1, 1, 4095, 4095, 7, 1, 0, 2255, 1, 3, 0, 0, 47, 4094, 1, 4095, 1, 1, 1, 0, 3, 1, 4091, 4095, 1, 0, 0, 1, 4095, 4095, 4095, 0, 0, 2, 0, 1, 4093, 64, 65, 0, 0, 0, 0, 0, 4095, 2788, 69, 0, 1, 3, 1, 4095, 0, 1, 1, 3975, 0, 1, 864, 263, 0, 4092, 4095, 4091, 1, 18, 1, 1, 0, 0, 1, 0, 1, 179, 4085, 4095, 3571, 4090, 3942, 0, 0, 0, 5, 4035, 0, 4095, 4095, 1, 0, 0, 4094, 601, 0, 1, 0, 1, 0, 0, 0, 4077, 0, 4095, 4095, 0, 1, 33, 0, 0, 0, 0, 0, 1, 0, 1, 1844, 1, 0, 0, 4076, 1, 0, 0, 1, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4095, 1, 0, 0, 2846, 0, 9, 0, 0, 138, 4096, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 4096, 0, 0, 0, 0, 4094, 4096, 4096, 0, 0, 9, 2, 0, 4073, 0, 535, 0, 0, 0, 0, 0, 4095, 2366, 174, 0, 0, 2, 0, 4096, 0, 0, 0, 3485, 0, 0, 533, 198, 0, 4096, 4096, 4087, 0, 1, 0, 0, 0, 0, 0, 0, 0, 398, 4087, 4092, 3836, 4017, 4000, 0, 1, 0, 5, 4037, 0, 4096, 4096, 0, 0, 0, 4096, 1050, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 4095, 4096, 1, 0, 176, 0, 0, 4, 0, 15, 0, 0, 0, 947, 0, 0, 0, 4081, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 0, 1, 4095, 4095, 1, 1, 0, 2516, 1, 0, 1, 1, 1210, 371, 339, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 4095, 1, 4094, 1, 0, 1, 0, 0, 1, 4071, 3724, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4095, 3756, 0, 0, 0, 0, 1, 4094, 130, 297, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2974, 551, 0, 1, 2, 1, 4095, 0, 1, 1, 4037, 0, 1, 389, 66, 180, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3871, 0, 4092, 1, 14, 1, 1, 0, 0, 1, 0, 1, 143, 4095, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 3905, 3899, 1, 0, 0, 19, 4066, 1, 0, 0, 1, 0, 0, 0, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 4080, 0, 4095, 4095, 0, 0, 1, 0, 0, 2884, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 1, 0, 0, 806, 1, 0, 0, 4082, 1, 0, 0, 1, 1, 4095, 4095, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16844, 16380, 16470, 16500, 16315, 16213, 16402, 15948] avg=16384.0000 max=16844 max_rank=0 min=15948 min_rank=7 imbalance_ratio=1.028076 max_load_reduction=114228 max_load_reduction_pct=0.871490 moved_to_echo=64878 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4094, 0, 0, 0, 1978, 0, 2, 0, 0, 1211, 371, 339, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 4096, 0, 4096, 0, 0, 0, 0, 2, 0, 4096, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3757, 0, 0, 8, 2, 0, 4076, 57, 890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3365, 66, 0, 0, 14, 0, 4095, 0, 0, 0, 3811, 0, 0, 711, 697, 178, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3874, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 251, 4089, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3730, 3864, 3875, 0, 0, 0, 2, 4060, 0, 0, 0, 0, 0, 0, 0, 583, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 4094, 4096, 2, 0, 22, 0, 0, 2885, 1245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 22, 0, 0, 0, 386, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 4096, 4096, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16302, 16162, 16982, 17138, 16386, 16114, 16438, 15550] avg=16384.0000 max=17138 max_rank=3 min=15550 min_rank=7 imbalance_ratio=1.046021 max_load_reduction=113934 max_load_reduction_pct=0.869247 moved_to_echo=64488 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 1, 1, 0, 4095, 4095, 0, 1, 0, 489, 1, 0, 0, 0, 1211, 372, 345, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 4095, 1, 4089, 1, 0, 1, 0, 0, 1, 4089, 3723, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 3750, 0, 0, 0, 0, 1, 4092, 16, 121, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1295, 2487, 0, 1, 0, 1, 4095, 0, 1, 1, 3316, 0, 1, 488, 1869, 184, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3866, 0, 4093, 1, 37, 1, 1, 0, 2, 0, 0, 1, 344, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4006, 3908, 4080, 0, 0, 0, 6, 4069, 1, 0, 0, 1, 0, 0, 0, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 3808, 0, 4095, 4095, 0, 0, 19, 0, 0, 2884, 1245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 1, 0, 1, 1634, 0, 0, 0, 4049, 0, 0, 0, 1, 1, 4095, 4095, 2850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14829, 16181, 16176, 17941, 16537, 16520, 16149, 16739] avg=16384.0000 max=17941 max_rank=3 min=14829 min_rank=0 imbalance_ratio=1.095032 max_load_reduction=113131 max_load_reduction_pct=0.863121 moved_to_echo=65945 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 1, 1, 1, 4095, 4096, 0, 1, 0, 1120, 0, 45, 0, 0, 1210, 371, 339, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 279, 4095, 0, 4095, 1, 0, 0, 1, 3, 1, 4095, 3724, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 3756, 1, 0, 31, 2, 0, 4045, 12, 125, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2020, 1545, 0, 0, 1, 1, 4094, 0, 1, 1, 2395, 0, 1, 792, 1598, 180, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3873, 0, 4057, 1, 39, 1, 0, 0, 0, 0, 1, 0, 122, 4073, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3919, 3915, 4090, 0, 0, 0, 33, 4028, 0, 0, 0, 1, 0, 1, 0, 988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 3990, 1, 4095, 4093, 0, 0, 0, 0, 0, 2885, 1244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 1, 0, 1, 1614, 1, 0, 0, 4059, 1, 0, 1, 1, 0, 4095, 4095, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15492, 16295, 16163, 16830, 16263, 16975, 16310, 16744] avg=16384.0000 max=16975 max_rank=5 min=15492 min_rank=0 imbalance_ratio=1.036072 max_load_reduction=114097 max_load_reduction_pct=0.870491 moved_to_echo=66292 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 2085, 0, 4, 0, 0, 1211, 371, 339, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 868, 4096, 0, 4096, 0, 0, 0, 1, 6, 0, 4096, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4095, 3757, 0, 0, 4, 2, 0, 4085, 1, 352, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1256, 971, 0, 0, 10, 0, 4095, 0, 0, 0, 3654, 0, 0, 456, 845, 180, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3872, 0, 4091, 0, 1, 0, 0, 0, 0, 0, 0, 0, 340, 4092, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3893, 3916, 3920, 1, 0, 0, 0, 3926, 0, 0, 0, 0, 0, 0, 0, 1339, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 4096, 4096, 0, 0, 38, 0, 0, 2885, 1245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 710, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 4096, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16413, 16888, 16388, 15668, 16486, 16995, 16384, 15850] avg=16384.0000 max=16995 max_rank=5 min=15668 min_rank=3 imbalance_ratio=1.037292 max_load_reduction=114077 max_load_reduction_pct=0.870338 moved_to_echo=65715 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 1, 0, 1, 1, 4095, 4095, 7, 1, 0, 2255, 1, 3, 0, 0, 1211, 371, 339, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 4094, 1, 4095, 1, 1, 1, 0, 3, 1, 4091, 3724, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 3756, 0, 0, 2, 0, 1, 4093, 64, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2788, 69, 0, 1, 3, 1, 4095, 0, 1, 1, 3975, 0, 1, 864, 263, 180, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3870, 0, 4091, 1, 18, 1, 1, 0, 0, 1, 0, 1, 179, 4085, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3571, 3910, 3942, 0, 0, 0, 5, 4035, 0, 0, 0, 1, 0, 0, 0, 601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4077, 0, 4095, 4095, 0, 1, 33, 0, 0, 2883, 1245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1844, 1, 0, 0, 4076, 1, 0, 0, 1, 1, 4095, 4095, 2850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16592, 16061, 16171, 16443, 16343, 16065, 16431, 16966] avg=16384.0000 max=16966 max_rank=7 min=16061 min_rank=1 imbalance_ratio=1.035522 max_load_reduction=114106 max_load_reduction_pct=0.870560 moved_to_echo=65805 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 2996, 0, 102, 0, 0, 1211, 371, 339, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 936, 4096, 0, 4096, 0, 0, 0, 0, 4, 0, 4096, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 3757, 0, 0, 4, 0, 0, 4087, 25, 90, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1971, 803, 0, 0, 3, 0, 4096, 0, 0, 0, 3787, 0, 0, 135, 144, 180, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3872, 0, 4086, 0, 13, 0, 0, 0, 0, 0, 0, 0, 69, 4082, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3667, 3915, 4091, 0, 0, 0, 0, 3396, 0, 0, 0, 0, 0, 0, 0, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3796, 0, 4095, 4096, 1, 0, 296, 0, 0, 2885, 1245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 6, 0, 0, 0, 2030, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 4096, 4096, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17421, 16953, 16156, 15320, 16203, 15421, 16414, 17184] avg=16384.0000 max=17421 max_rank=0 min=15320 min_rank=3 imbalance_ratio=1.063293 max_load_reduction=113651 max_load_reduction_pct=0.867088 moved_to_echo=65222 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 0, 0, 4095, 4095, 1, 0, 0, 2846, 0, 9, 0, 0, 1211, 371, 339, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 4096, 0, 4096, 0, 0, 0, 0, 1, 0, 4096, 3725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 3757, 0, 0, 9, 2, 0, 4073, 0, 535, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2366, 174, 0, 0, 2, 0, 4096, 0, 0, 0, 3485, 0, 0, 533, 198, 177, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3874, 0, 4087, 0, 1, 0, 0, 0, 0, 0, 0, 0, 398, 4087, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3836, 3840, 4000, 0, 1, 0, 5, 4037, 0, 0, 0, 0, 0, 0, 0, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4046, 0, 4095, 4096, 1, 0, 176, 0, 0, 2885, 1245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 15, 0, 0, 0, 947, 0, 0, 0, 4081, 0, 1, 0, 0, 0, 4096, 4096, 2851, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17179, 16152, 16566, 15232, 16539, 16769, 16544, 16091] avg=16384.0000 max=17179 max_rank=0 min=15232 min_rank=3 imbalance_ratio=1.048523 max_load_reduction=113893 max_load_reduction_pct=0.868935 moved_to_echo=65943 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 18048, 6784, 0, 128, 128, 128, 32768, 0, 128, 128, 28544, 0, 128, 4480, 5760, 1536, 896, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 32768, 30080, 128, 0, 128, 128, 128, 32768, 384, 2560, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 30976, 0, 32768, 128, 128, 128, 128, 0, 128, 128, 128, 128, 1920, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[30720, 31232, 32000, 128, 128, 0, 128, 31744, 128, 0, 0, 128, 0, 128, 0, 5760, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[3072, 32768, 128, 32768, 128, 128, 128, 128, 128, 128, 32768, 29824, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 128, 0, 32000, 128, 32768, 32768, 128, 128, 640, 0, 0, 23168, 9984, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[32768, 0, 128, 128, 128, 128, 32768, 32768, 128, 128, 0, 16384, 128, 256, 128, 128, 9728, 3072, 2816, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 0, 128, 128, 0, 128, 9984, 128, 0, 0, 32640, 128, 128, 128, 128, 128, 32768, 32768, 22912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[0, 3857, 0, 4095, 0, 4091, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4085, 0, 104, 0, 4096, 0, 0, 210, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4093, 0, 0, 174, 4093, 0, 4088, 4096, 0, 4096, 4079, 4096, 0, 6, 4091, 0, 12, 1050, 4096, 3, 0, 0, 0, 2, 4092, 2673, 0, 0, 0, 3688, 0, 0, 4096, 0, 4083, 0, 0, 0, 4079, 18, 0, 0, 37, 491, 0, 0, 4096, 0, 4096, 1232, 0, 31, 2159, 4096, 5, 4072, 0, 0, 2, 0, 6, 0, 0, 0, 4020, 4094, 0, 0, 0, 4096, 0, 0, 3971, 931, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[1, 3236, 0, 4095, 1, 4057, 1, 0, 2, 0, 1, 0, 0, 1, 0, 4001, 0, 49, 1, 4095, 1, 1, 588, 1, 1, 1, 0, 4095, 0, 0, 1, 0, 4095, 1, 0, 61, 4095, 1, 4095, 4095, 1, 4095, 4089, 4094, 0, 0, 4095, 3, 4, 1197, 4095, 0, 1, 1, 1, 0, 4095, 2108, 0, 0, 0, 3603, 0, 2, 4095, 1, 3910, 0, 0, 13, 4068, 0, 1, 0, 229, 495, 0, 0, 4094, 0, 4095, 1512, 0, 0, 1751, 4095, 5, 4091, 0, 0, 0, 0, 0, 1, 0, 0, 4083, 4095, 1, 1, 1, 4094, 0, 1, 4052, 1933, 1, 1, 0, 0, 4095, 1, 0, 0, 4095, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[0, 3008, 1, 4095, 1, 4057, 1, 0, 0, 0, 1, 0, 0, 0, 0, 3771, 0, 69, 0, 4095, 1, 1, 542, 1, 1, 1, 0, 4095, 0, 7, 1, 0, 4092, 0, 0, 33, 4063, 0, 4050, 4095, 0, 4095, 4071, 4095, 0, 0, 4096, 0, 31, 231, 4095, 3, 1, 1, 0, 0, 4092, 1995, 0, 0, 1, 3974, 0, 0, 4095, 0, 3838, 0, 0, 5, 4089, 82, 0, 1, 310, 854, 1, 0, 4095, 0, 4095, 1410, 0, 55, 2314, 4095, 0, 4093, 4, 1, 0, 0, 23, 1, 0, 0, 4092, 4095, 0, 0, 1, 4095, 0, 1, 2826, 3420, 1, 1, 19, 1, 4095, 1, 1, 0, 4095, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[1, 2595, 0, 4095, 1, 4072, 1, 0, 1, 0, 1, 0, 1, 1, 0, 4089, 0, 49, 1, 4095, 1, 1, 147, 1, 1, 1, 0, 4095, 0, 0, 1, 0, 4093, 0, 0, 0, 4087, 1, 4095, 4094, 1, 4095, 4091, 4095, 0, 0, 4095, 3, 12, 838, 4095, 0, 0, 1, 1, 0, 4095, 3414, 0, 0, 0, 4071, 0, 3, 4095, 1, 4049, 0, 0, 1, 4068, 0, 1, 0, 452, 1784, 0, 0, 4095, 0, 4095, 172, 0, 0, 2300, 4095, 0, 4095, 421, 1, 0, 0, 0, 1, 0, 0, 4095, 4095, 1, 1, 1, 4095, 0, 1, 3098, 1210, 1, 1, 19, 0, 4095, 1, 0, 0, 4095, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[0, 3313, 0, 4094, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 0, 298, 0, 4096, 0, 0, 180, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4093, 0, 0, 31, 4094, 0, 4091, 4096, 0, 4095, 4088, 4096, 0, 8, 4092, 0, 5, 1284, 4096, 3, 0, 0, 0, 2, 4093, 3041, 0, 0, 0, 2347, 0, 0, 4096, 0, 3925, 0, 0, 0, 4086, 10, 0, 0, 81, 340, 0, 0, 4096, 0, 4096, 625, 0, 18, 3114, 4096, 0, 4083, 5, 0, 4, 0, 4, 0, 0, 0, 4066, 4092, 0, 0, 0, 4096, 0, 0, 3859, 2302, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1, 3868, 1, 4095, 1, 3794, 1, 0, 0, 0, 1, 0, 0, 1, 0, 3930, 0, 228, 1, 4095, 1, 1, 370, 1, 1, 1, 0, 4094, 0, 0, 1, 0, 4095, 1, 0, 104, 4094, 1, 4094, 4095, 1, 4095, 4085, 4095, 0, 0, 4091, 0, 14, 538, 4095, 2, 1, 1, 1, 0, 4095, 530, 0, 0, 1, 3358, 0, 0, 4095, 1, 3926, 0, 0, 9, 4086, 0, 1, 0, 229, 1059, 0, 0, 4094, 0, 4095, 1581, 0, 3, 2362, 4095, 3, 4092, 0, 0, 0, 0, 2, 1, 0, 0, 4094, 4094, 1, 1, 1, 4095, 0, 1, 3987, 2893, 0, 1, 0, 0, 4095, 1, 0, 0, 4095, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[0, 1960, 0, 4095, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3903, 0, 205, 0, 4096, 0, 0, 178, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4095, 0, 0, 5, 4095, 0, 4085, 4096, 0, 4095, 4072, 4096, 0, 17, 4094, 0, 13, 1040, 4096, 3, 0, 0, 0, 2, 4083, 792, 0, 0, 0, 3127, 0, 0, 4096, 0, 4011, 0, 0, 0, 4088, 3, 0, 0, 437, 104, 0, 0, 4096, 0, 4096, 3435, 0, 32, 2558, 4096, 0, 4094, 74, 0, 1, 0, 3, 0, 0, 0, 4025, 4096, 0, 0, 0, 4096, 0, 0, 3135, 3770, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[0, 3402, 0, 4095, 0, 4068, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3714, 0, 94, 0, 4096, 0, 0, 135, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 4095, 0, 0, 56, 4096, 0, 4089, 4095, 0, 4095, 4078, 4096, 0, 17, 4091, 0, 10, 466, 4096, 1, 0, 0, 0, 2, 4071, 1980, 0, 0, 0, 3612, 0, 0, 4096, 0, 4049, 0, 0, 0, 4084, 19, 0, 0, 60, 112, 0, 0, 4096, 0, 4096, 2898, 0, 67, 2609, 4096, 0, 4094, 5, 0, 1, 0, 10, 0, 1, 0, 4030, 4096, 0, 0, 0, 4096, 0, 0, 3472, 2044, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 3236, 0, 4095, 1, 4057, 1, 0, 2, 0, 1, 0, 0, 1, 0, 4001, 766, 360, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 1, 4095, 1, 1, 588, 1, 1, 1, 0, 4095, 0, 0, 1, 0, 4095, 3662, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 61, 4095, 1, 4095, 0, 1, 0, 4089, 0, 0, 0, 4064, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1197, 4095, 0, 1, 1, 1, 0, 4095, 2108, 0, 0, 0, 3603, 0, 2, 1378, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3329, 1, 3910, 0, 0, 13, 4068, 0, 1, 0, 229, 495, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1512, 0, 0, 1751, 4095, 5, 4091, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 4095, 1, 1, 1, 2356, 0, 1, 4052, 1933, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4095, 4094, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16553, 16592, 16410, 16918, 16140, 15550, 16526, 16383] avg=16384.0000 max=16918 max_rank=3 min=15550 min_rank=5 imbalance_ratio=1.032593 max_load_reduction=114154 max_load_reduction_pct=0.870926 moved_to_echo=64599 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3857, 0, 4095, 0, 4091, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4085, 766, 361, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 4096, 0, 0, 210, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 4096, 3660, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 4093, 0, 4088, 0, 0, 0, 4079, 0, 0, 6, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1050, 4096, 3, 0, 0, 0, 2, 4092, 2673, 0, 0, 0, 3688, 0, 0, 1379, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3330, 0, 4083, 0, 0, 0, 4079, 18, 0, 0, 37, 491, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1232, 0, 31, 2159, 4096, 5, 4072, 0, 0, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4020, 4094, 0, 0, 0, 2356, 0, 0, 3971, 931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17288, 16266, 16500, 17428, 16134, 15699, 15372, 16385] avg=16384.0000 max=17428 max_rank=3 min=15372 min_rank=6 imbalance_ratio=1.063721 max_load_reduction=113644 max_load_reduction_pct=0.867035 moved_to_echo=63590 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 2595, 0, 4095, 1, 4072, 1, 0, 1, 0, 1, 0, 1, 1, 0, 4089, 773, 364, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 1, 4095, 1, 1, 147, 1, 1, 1, 0, 4095, 0, 0, 1, 0, 4095, 3661, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 1, 4095, 0, 1, 0, 4091, 0, 0, 0, 4056, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 838, 4095, 0, 0, 1, 1, 0, 4095, 3414, 0, 0, 0, 4071, 0, 3, 1382, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3322, 1, 4049, 0, 0, 1, 4068, 0, 1, 0, 452, 1784, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 172, 0, 0, 2300, 4095, 0, 4095, 421, 1, 0, 0, 0, 1, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 1, 1, 1, 2349, 0, 1, 3098, 1210, 1, 1, 19, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 4095, 4095, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16027, 16151, 16334, 18344, 17773, 15187, 14873, 16383] avg=16384.0000 max=18344 max_rank=3 min=14873 min_rank=6 imbalance_ratio=1.119629 max_load_reduction=112728 max_load_reduction_pct=0.860046 moved_to_echo=64216 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3008, 1, 4095, 1, 4057, 1, 0, 0, 0, 1, 0, 0, 0, 0, 3771, 766, 360, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 4095, 1, 1, 542, 1, 1, 1, 0, 4095, 0, 7, 1, 0, 4095, 3659, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 4063, 0, 4050, 0, 0, 0, 4071, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 231, 4095, 3, 1, 1, 0, 0, 4092, 1995, 0, 0, 1, 3974, 0, 0, 1378, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3329, 0, 3838, 0, 0, 5, 4089, 82, 0, 1, 310, 854, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1410, 0, 55, 2314, 4095, 0, 4093, 4, 1, 0, 0, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 4095, 0, 0, 1, 2357, 0, 1, 2826, 3420, 1, 1, 19, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 4095, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16092, 16569, 16282, 16235, 16604, 16091, 16815, 16384] avg=16384.0000 max=16815 max_rank=6 min=16091 min_rank=5 imbalance_ratio=1.026306 max_load_reduction=114257 max_load_reduction_pct=0.871712 moved_to_echo=65894 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 1960, 0, 4095, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3903, 766, 361, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 4096, 0, 0, 178, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4095, 3662, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4095, 0, 4085, 0, 0, 0, 4072, 0, 0, 17, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1040, 4096, 3, 0, 0, 0, 2, 4083, 792, 0, 0, 0, 3127, 0, 0, 1379, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3330, 0, 4011, 0, 0, 0, 4088, 3, 0, 0, 437, 104, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3435, 0, 32, 2558, 4096, 0, 4094, 74, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4025, 4096, 0, 0, 0, 2356, 0, 0, 3135, 3770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15209, 16334, 16337, 14968, 16069, 18389, 17382, 16384] avg=16384.0000 max=18389 max_rank=5 min=14968 min_rank=3 imbalance_ratio=1.122375 max_load_reduction=112683 max_load_reduction_pct=0.859703 moved_to_echo=68224 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3313, 0, 4094, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3966, 766, 361, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 0, 4096, 0, 0, 180, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 4095, 3660, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 4094, 0, 4091, 0, 0, 0, 4088, 0, 0, 8, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1284, 4096, 3, 0, 0, 0, 2, 4093, 3041, 0, 0, 0, 2347, 0, 0, 1379, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3330, 0, 3925, 0, 0, 0, 4086, 10, 0, 0, 81, 340, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 625, 0, 18, 3114, 4096, 0, 4083, 5, 0, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 4092, 0, 0, 0, 2356, 0, 0, 3859, 2302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 4096, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16615, 16428, 16373, 16683, 15868, 16045, 16675, 16385] avg=16384.0000 max=16683 max_rank=3 min=15868 min_rank=4 imbalance_ratio=1.018250 max_load_reduction=114389 max_load_reduction_pct=0.872719 moved_to_echo=64973 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1, 3868, 1, 4095, 1, 3794, 1, 0, 0, 0, 1, 0, 0, 1, 0, 3930, 766, 360, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 1, 4095, 1, 1, 370, 1, 1, 1, 0, 4094, 0, 0, 1, 0, 4095, 3663, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 104, 4094, 1, 4094, 0, 1, 0, 4085, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 538, 4095, 2, 1, 1, 1, 0, 4095, 530, 0, 0, 1, 3358, 0, 0, 1378, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3329, 1, 3926, 0, 0, 9, 4086, 0, 1, 0, 229, 1059, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1581, 0, 3, 2362, 4095, 3, 4092, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4094, 1, 1, 1, 2357, 0, 1, 3987, 2893, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4095, 4095, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16850, 16553, 16440, 14446, 16734, 16234, 17431, 16384] avg=16384.0000 max=17431 max_rank=6 min=14446 min_rank=3 imbalance_ratio=1.063904 max_load_reduction=113641 max_load_reduction_pct=0.867012 moved_to_echo=66783 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[0, 3402, 0, 4095, 0, 4068, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3714, 766, 361, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 4096, 0, 0, 135, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 4095, 3662, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 4096, 0, 4089, 0, 0, 0, 4078, 0, 0, 17, 4060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 466, 4096, 1, 0, 0, 0, 2, 4071, 1980, 0, 0, 0, 3612, 0, 0, 1379, 433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3330, 0, 4049, 0, 0, 0, 4084, 19, 0, 0, 60, 112, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2898, 0, 67, 2609, 4096, 0, 4094, 5, 0, 1, 0, 10, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4030, 4096, 0, 0, 0, 2356, 0, 0, 3472, 2044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4096, 4096, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16438, 16179, 16396, 16050, 15750, 17877, 15998, 16384] avg=16384.0000 max=17877 max_rank=5 min=15750 min_rank=4 imbalance_ratio=1.091125 max_load_reduction=113195 max_load_reduction_pct=0.863609 moved_to_echo=66009 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 0, 512, 32768, 128, 32768, 0, 128, 0, 32768, 0, 0, 128, 32512, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 32768, 0, 128, 128, 128, 128, 128, 128, 128, 128, 0, 128, 128, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 6656, 32768, 128, 128, 128, 128, 128, 32768, 16640, 0, 0, 128, 27904, 0, 128, 11136, 3584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 25344, 128, 32768, 128, 32384, 128, 0, 128, 0, 128, 128, 128, 128, 128, 31488, 6144, 2944, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32512, 32768, 128, 128, 128, 18944, 0, 128, 28416, 18560, 128, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 12928, 0, 256, 19200, 32768, 128, 32768, 512, 128, 128, 0, 128, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[26752, 128, 31872, 0, 0, 128, 32768, 256, 128, 128, 1920, 5248, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 1152, 128, 32768, 128, 128, 2432, 128, 128, 128, 128, 32768, 0, 128, 128, 0, 32768, 29312, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 4088, 0, 0, 0, 0, 4087, 4096, 0, 0, 0, 3811, 0, 0, 0, 0, 5, 0, 4090, 4042, 20, 4094, 0, 0, 3986, 4095, 0, 418, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 2, 0, 0, 4089, 4096, 0, 4095, 5, 0, 0, 0, 0, 1, 0, 4094, 0, 11, 4096, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 1, 0, 0, 2, 0, 4096, 3, 4087, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 6, 0, 4094, 0, 0, 4094, 0, 95, 4096, 0, 4030, 4096, 0, 0, 4096, 0, 0, 4096, 4073, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[18, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 4044, 0, 0, 0, 0, 4092, 4060, 0, 0, 0, 3326, 0, 0, 0, 0, 4, 0, 4094, 3472, 2, 4094, 0, 0, 3624, 4096, 0, 2085, 0, 0, 0, 31, 2, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 2, 0, 0, 4084, 4096, 0, 4095, 3, 0, 0, 0, 0, 2, 0, 4094, 0, 53, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 2, 4090, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 2, 0, 23, 0, 4094, 0, 0, 4094, 0, 24, 4096, 0, 3831, 4096, 0, 0, 4096, 0, 0, 4096, 4083, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[29, 0, 0, 4095, 0, 1, 0, 0, 1, 1, 4080, 1, 1, 0, 4095, 0, 4033, 0, 1, 0, 0, 4095, 4093, 0, 1, 0, 3659, 1, 0, 0, 0, 0, 0, 4095, 3666, 30, 4095, 1, 1, 3676, 4095, 0, 2272, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 3883, 4095, 1, 4095, 0, 1, 0, 1, 1, 0, 0, 4095, 1, 6, 4083, 0, 1, 1, 0, 4095, 1, 4095, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 4095, 1, 1, 1, 4095, 0, 1, 6, 0, 0, 0, 1, 0, 1, 4095, 1, 1, 4095, 0, 117, 4083, 1, 3251, 4095, 0, 1, 4084, 0, 9, 4095, 4077, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[25, 0, 0, 4095, 0, 1, 20, 0, 1, 0, 4095, 1, 1, 0, 4095, 1, 4068, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 4064, 1, 0, 0, 0, 0, 0, 4095, 4043, 126, 4095, 1, 1, 3078, 4095, 0, 967, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 3861, 4095, 1, 4095, 0, 1, 0, 1, 1, 0, 0, 4095, 0, 245, 4079, 0, 1, 1, 0, 4095, 1, 4095, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 4095, 0, 4095, 1, 1, 1, 4095, 0, 4, 0, 0, 0, 0, 1, 0, 0, 4095, 1, 1, 4095, 0, 10, 4094, 1, 4092, 4095, 0, 0, 4089, 0, 9, 4095, 4075, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[130, 0, 0, 4096, 0, 0, 9, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 4075, 0, 0, 0, 0, 4086, 4087, 0, 0, 0, 3209, 0, 0, 0, 0, 2, 0, 4088, 3911, 89, 4095, 0, 0, 4012, 4096, 0, 1331, 0, 0, 0, 12, 1, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 1, 0, 0, 4043, 4096, 0, 4094, 3, 0, 0, 0, 0, 1, 0, 4094, 0, 12, 4096, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 1, 4095, 0, 0, 0, 4096, 0, 0, 63, 0, 0, 2, 0, 19, 0, 4096, 0, 0, 4092, 0, 3, 4096, 0, 3687, 4094, 0, 0, 4096, 0, 0, 4096, 4096, 4094, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[14, 1, 0, 4095, 0, 0, 66, 0, 0, 0, 4045, 1, 0, 0, 4095, 1, 4095, 1, 1, 0, 0, 4094, 4095, 0, 0, 0, 3656, 1, 0, 0, 0, 0, 0, 4060, 3801, 85, 4095, 1, 1, 3709, 4095, 0, 1133, 0, 0, 1, 33, 0, 1, 0, 1, 0, 1, 3, 4095, 0, 0, 1, 0, 0, 1, 3951, 4095, 1, 4095, 4, 1, 0, 0, 1, 0, 0, 4096, 1, 33, 4077, 1, 1, 0, 0, 4095, 1, 4095, 0, 0, 0, 1, 1, 0, 1, 0, 7, 0, 4095, 0, 4095, 1, 1, 1, 4095, 0, 1, 27, 0, 1, 0, 1, 40, 0, 4095, 0, 0, 4095, 0, 3, 4094, 1, 4020, 4095, 0, 0, 4095, 0, 2, 4095, 4087, 4096, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[37, 1, 0, 4095, 0, 1, 23, 0, 1, 0, 4092, 1, 1, 0, 4095, 1, 4083, 0, 1, 0, 0, 4092, 4092, 0, 0, 0, 3809, 1, 1, 0, 0, 0, 0, 4095, 3643, 14, 4095, 0, 1, 2865, 4095, 0, 2032, 0, 0, 0, 2, 0, 1, 0, 1, 1, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 3885, 4095, 1, 4095, 1, 1, 0, 1, 1, 0, 0, 4095, 0, 2, 4078, 1, 1, 0, 1, 4095, 1, 4095, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 4095, 0, 4095, 1, 1, 1, 4095, 0, 1, 4, 0, 0, 1, 1, 0, 0, 4095, 1, 1, 4095, 0, 164, 4089, 1, 4030, 4095, 0, 0, 4094, 0, 11, 4095, 4089, 4095, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[4, 0, 0, 4096, 0, 0, 5, 1, 0, 0, 4094, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4087, 4095, 0, 0, 0, 3502, 0, 0, 0, 0, 3, 0, 4094, 3877, 37, 4095, 0, 0, 4020, 4095, 0, 945, 1, 0, 0, 10, 1, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4051, 4096, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 4095, 0, 3, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4093, 0, 0, 0, 4096, 1, 0, 20, 0, 0, 1, 0, 6, 0, 4095, 0, 0, 4094, 0, 113, 4095, 0, 3907, 4095, 1, 0, 4096, 0, 0, 4096, 4092, 4093, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[29, 0, 0, 4095, 0, 1, 0, 0, 1, 1, 4080, 1, 1, 0, 4095, 0, 4014, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4033, 0, 1, 0, 0, 4095, 4093, 0, 1, 0, 3659, 1, 0, 0, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3666, 30, 3406, 1, 1, 3676, 0, 0, 2272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 3883, 4095, 1, 4095, 81, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 1, 1, 0, 0, 4095, 1, 6, 4083, 0, 1, 1, 0, 3865, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4095, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4095, 0, 1, 6, 0, 0, 0, 1, 0, 1, 4095, 1, 1, 4095, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 117, 4083, 1, 3251, 0, 0, 1, 4084, 0, 9, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16360, 16381, 17147, 16286, 16341, 16383, 16363, 15811] avg=16384.0000 max=17147 max_rank=2 min=15811 min_rank=7 imbalance_ratio=1.046570 max_load_reduction=113925 max_load_reduction_pct=0.869179 moved_to_echo=64898 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 4012, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 4087, 4096, 0, 0, 0, 3811, 0, 0, 0, 0, 5, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 4042, 20, 3406, 0, 0, 3986, 0, 0, 418, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 0, 0, 2, 0, 0, 4089, 4096, 0, 4096, 81, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 5, 0, 0, 0, 0, 1, 0, 4094, 0, 11, 4096, 0, 0, 0, 0, 3864, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 2, 0, 0, 0, 0, 1, 0, 0, 2, 0, 4096, 3, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 6, 0, 4094, 0, 0, 4096, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 95, 4096, 0, 4030, 0, 0, 0, 4096, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16343, 16584, 15967, 16494, 16357, 16383, 16366, 16578] avg=16384.0000 max=16584 max_rank=1 min=15967 min_rank=2 imbalance_ratio=1.012207 max_load_reduction=114488 max_load_reduction_pct=0.873474 moved_to_echo=65684 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[18, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 4013, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4044, 0, 0, 0, 0, 4092, 4060, 0, 0, 0, 3326, 0, 0, 0, 0, 4, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3472, 2, 3406, 0, 0, 3624, 0, 0, 2085, 0, 0, 0, 31, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 2, 0, 0, 4084, 4096, 0, 4096, 80, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3, 0, 0, 0, 0, 2, 0, 4094, 0, 53, 4096, 0, 0, 0, 0, 3864, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 2, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 2, 0, 23, 0, 4094, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 24, 4096, 0, 3831, 0, 0, 0, 4096, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16362, 16023, 16716, 16487, 16398, 16382, 16386, 16318] avg=16384.0000 max=16716 max_rank=2 min=16023 min_rank=1 imbalance_ratio=1.020264 max_load_reduction=114356 max_load_reduction_pct=0.872467 moved_to_echo=65484 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[25, 0, 0, 4095, 0, 1, 20, 0, 1, 0, 4095, 1, 1, 0, 4095, 1, 4015, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4068, 0, 0, 0, 0, 4095, 4095, 0, 0, 0, 4064, 1, 0, 0, 0, 0, 500, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4043, 126, 3402, 1, 1, 3078, 0, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 3861, 4095, 1, 4095, 80, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 1, 1, 0, 0, 4095, 0, 245, 4079, 0, 1, 1, 0, 3867, 193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 1, 4095, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4095, 0, 4, 0, 0, 0, 0, 1, 0, 0, 4095, 1, 1, 4095, 4063, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 10, 4094, 1, 4092, 0, 0, 0, 4089, 0, 9, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16397, 16828, 15713, 16265, 16579, 16381, 16358, 16551] avg=16384.0000 max=16828 max_rank=1 min=15713 min_rank=2 imbalance_ratio=1.027100 max_load_reduction=114244 max_load_reduction_pct=0.871613 moved_to_echo=65869 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[130, 0, 0, 4096, 0, 0, 9, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 4014, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 4086, 4087, 0, 0, 0, 3209, 0, 0, 0, 0, 2, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 3911, 89, 3406, 0, 0, 4012, 0, 0, 1331, 0, 0, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 1, 0, 0, 4043, 4096, 0, 4094, 80, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3, 0, 0, 0, 0, 1, 0, 4094, 0, 12, 4096, 0, 0, 0, 0, 3862, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 63, 0, 0, 2, 0, 19, 0, 4096, 0, 0, 4096, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 3, 4096, 0, 3687, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16480, 15957, 16850, 16443, 16353, 16386, 16437, 16166] avg=16384.0000 max=16850 max_rank=2 min=15957 min_rank=1 imbalance_ratio=1.028442 max_load_reduction=114222 max_load_reduction_pct=0.871445 moved_to_echo=65342 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[14, 1, 0, 4095, 0, 0, 66, 0, 0, 0, 4045, 1, 0, 0, 4095, 1, 4015, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 0, 4094, 4095, 0, 0, 0, 3656, 1, 0, 0, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 3801, 85, 3406, 1, 1, 3709, 0, 0, 1133, 0, 0, 1, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 3, 4095, 0, 0, 1, 0, 0, 1, 3951, 4095, 1, 4095, 81, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4, 1, 0, 0, 1, 0, 0, 4096, 1, 33, 4077, 1, 1, 0, 0, 3865, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4095, 0, 0, 0, 1, 1, 0, 1, 0, 7, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4095, 0, 1, 27, 0, 1, 0, 1, 40, 0, 4095, 0, 0, 4095, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 3, 4094, 1, 4020, 0, 0, 0, 4095, 0, 2, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16375, 16441, 16230, 16357, 16366, 16391, 16422, 16490] avg=16384.0000 max=16490 max_rank=7 min=16230 min_rank=2 imbalance_ratio=1.006470 max_load_reduction=114582 max_load_reduction_pct=0.874191 moved_to_echo=65669 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[37, 1, 0, 4095, 0, 1, 23, 0, 1, 0, 4092, 1, 1, 0, 4095, 1, 4015, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 1, 0, 0, 4092, 4092, 0, 0, 0, 3809, 1, 1, 0, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3643, 14, 3406, 0, 1, 2865, 0, 0, 2032, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 4095, 0, 1, 0, 0, 0, 1, 3885, 4095, 1, 4095, 80, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 1, 0, 1, 1, 0, 0, 4095, 0, 2, 4078, 1, 1, 0, 1, 3865, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 4095, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 4095, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 4095, 0, 1, 4, 0, 0, 1, 1, 0, 0, 4095, 1, 1, 4095, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 164, 4089, 1, 4030, 0, 0, 0, 4094, 0, 11, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16405, 16577, 16058, 16288, 16333, 16384, 16361, 16666] avg=16384.0000 max=16666 max_rank=7 min=16058 min_rank=2 imbalance_ratio=1.017212 max_load_reduction=114406 max_load_reduction_pct=0.872849 moved_to_echo=65744 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[4, 0, 0, 4096, 0, 0, 5, 1, 0, 0, 4094, 0, 0, 0, 4096, 0, 4012, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4087, 4095, 0, 0, 0, 3502, 0, 0, 0, 0, 3, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3877, 37, 3406, 0, 0, 4020, 0, 0, 945, 1, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 1, 0, 0, 4051, 4096, 0, 4095, 81, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 0, 4095, 0, 3, 4096, 0, 0, 0, 0, 3864, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 20, 0, 0, 1, 0, 6, 0, 4095, 0, 0, 4096, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 113, 4095, 0, 3907, 0, 1, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16350, 16281, 16391, 16452, 16345, 16382, 16379, 16492] avg=16384.0000 max=16492 max_rank=7 min=16281 min_rank=1 imbalance_ratio=1.006592 max_load_reduction=114580 max_load_reduction_pct=0.874176 moved_to_echo=65598 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[32768, 128, 32768, 128, 128, 0, 128, 128, 128, 128, 128, 128, 0, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 32768, 30464, 512, 27264, 128, 128, 29056, 0, 0, 11264, 128, 0, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32768, 128, 128, 0, 128, 128, 128, 0, 32768, 128, 384, 32768, 128, 128, 128, 128, 30976, 1536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[384, 128, 0, 32768, 0, 128, 256, 128, 128, 128, 32768, 128, 128, 0, 32768, 128, 32128, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 128, 128, 128, 32768, 0, 128, 128, 128, 0, 128, 31872, 32768, 128, 32768, 768, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32640, 128, 128, 0, 0, 32768, 32768, 0, 128, 0, 29056, 128, 128, 0, 0, 128, 4096, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 128, 128, 32768, 128, 128, 256, 0, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[1536, 0, 640, 32768, 128, 30848, 0, 128, 128, 32768, 0, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 696, 4094, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4068, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4083, 0, 0, 4096, 0, 0, 366, 0, 0, 0, 4, 4096, 6, 0, 0, 0, 0, 0, 0, 0, 9, 0, 13, 4093, 4092, 0, 4094, 0, 2, 2450, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 4089, 0, 0, 2918, 2, 4096, 4068, 4094, 25, 0, 0, 3966, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3356, 2070, 0, 68, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 515, 0, 2, 2, 4094, 0, 0, 0, 4096, 2, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1161, 4095, 1, 1, 0, 0, 0, 4095, 24, 4095, 2, 1, 4095, 4095, 1, 4, 1, 0, 0, 4096, 0, 1, 4088, 1, 4095, 0, 0, 34, 1, 4095, 1, 0, 4094, 0, 1, 1033, 1, 1, 0, 24, 4095, 0, 44, 1, 0, 1, 1, 0, 0, 154, 0, 0, 4095, 4095, 0, 4095, 0, 0, 2630, 0, 1, 0, 1, 4095, 0, 0, 1, 0, 0, 4089, 1, 0, 2138, 0, 4056, 4051, 4018, 0, 0, 0, 3528, 0, 1, 0, 0, 4095, 1, 0, 4092, 1, 1, 1, 4095, 3882, 1770, 0, 4, 0, 1, 0, 1, 0, 0, 17, 4089, 0, 61, 0, 0, 0, 4095, 1, 0, 1, 4095, 8, 4095, 0, 4095, 46, 0, 1, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1045, 4094, 1, 0, 1, 0, 0, 4094, 2, 4095, 0, 1, 4095, 4093, 1, 10, 1, 0, 0, 4095, 1, 1, 4061, 1, 4095, 0, 0, 32, 1, 4095, 1, 1, 3868, 0, 1, 920, 1, 0, 0, 0, 4094, 0, 3, 1, 0, 1, 27, 0, 0, 28, 0, 0, 4095, 4084, 0, 4095, 2, 0, 1676, 0, 1, 0, 1, 4095, 0, 0, 0, 1, 0, 4092, 1, 0, 2713, 0, 3828, 3549, 3975, 0, 1, 0, 3979, 0, 1, 1, 1, 4095, 0, 0, 4093, 1, 2, 1, 4095, 3717, 3045, 0, 22, 0, 3, 0, 1, 0, 0, 0, 4095, 0, 169, 1, 0, 0, 4095, 2, 0, 1, 4095, 126, 4095, 0, 4095, 74, 0, 1, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 574, 4094, 1, 1, 1, 0, 0, 4095, 6, 4095, 3, 1, 4095, 4095, 1, 2, 1, 0, 0, 4095, 1, 1, 4087, 1, 4095, 0, 0, 0, 1, 4095, 1, 1, 3929, 1, 1, 443, 1, 0, 0, 6, 4095, 0, 1, 1, 0, 0, 1, 0, 0, 20, 0, 1, 4095, 4075, 1, 4095, 0, 0, 1243, 0, 1, 0, 1, 4095, 0, 0, 3, 0, 0, 4091, 1, 0, 3801, 0, 3894, 3578, 3964, 0, 1, 0, 4031, 0, 1, 0, 1, 4095, 0, 0, 4092, 1, 1, 1, 4095, 3559, 3475, 0, 135, 0, 1, 0, 1, 0, 0, 0, 4094, 1, 33, 0, 0, 0, 4095, 1, 0, 1, 4095, 25, 4095, 0, 4095, 68, 1, 1, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 980, 4095, 1, 0, 1, 0, 0, 4091, 1, 4095, 1, 1, 4095, 4094, 1, 4, 1, 0, 0, 4095, 0, 1, 4072, 1, 4095, 0, 0, 42, 1, 4090, 1, 1, 3831, 1, 1, 666, 1, 0, 0, 0, 4094, 0, 9, 1, 0, 0, 7, 0, 0, 63, 0, 0, 4095, 4093, 1, 4095, 17, 0, 3185, 0, 1, 0, 1, 4095, 0, 0, 0, 0, 0, 4093, 1, 0, 3466, 0, 3907, 4048, 3991, 0, 1, 0, 2877, 0, 1, 0, 1, 4095, 1, 0, 4091, 1, 2, 1, 4095, 4032, 1485, 0, 1, 0, 3, 0, 1, 0, 0, 0, 4095, 0, 3, 1, 0, 0, 4095, 0, 0, 1, 4095, 91, 4095, 0, 4095, 93, 0, 1, 0, 4095] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 1180, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4075, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 789, 0, 0, 0, 1, 4096, 11, 0, 0, 0, 0, 0, 0, 0, 59, 0, 16, 4094, 4091, 0, 4095, 0, 1, 2744, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 4080, 0, 0, 2912, 3, 4096, 4061, 4094, 40, 0, 0, 3319, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3019, 1888, 0, 33, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 445, 0, 1, 2, 4095, 0, 0, 0, 4096, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 574, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4071, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 478, 0, 0, 0, 2, 4096, 5, 2, 0, 0, 0, 0, 0, 0, 460, 0, 10, 4093, 4095, 0, 4094, 0, 2, 3111, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 4092, 0, 0, 2875, 13, 4096, 4079, 4094, 25, 0, 0, 3504, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 2795, 1752, 0, 4, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 823, 0, 2, 2, 4093, 0, 2, 0, 4096, 0, 4096, 0, 4094, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 307, 4095, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4069, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 89, 0, 0, 0, 0, 4096, 4, 0, 0, 0, 0, 0, 0, 0, 113, 0, 10, 4093, 4092, 0, 4095, 0, 0, 3017, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4093, 0, 0, 3045, 18, 4096, 4001, 4096, 23, 0, 0, 3816, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 4096, 3561, 1948, 1, 5, 1, 0, 0, 0, 1, 0, 0, 4095, 0, 562, 0, 1, 1, 4095, 0, 1, 0, 4096, 0, 4095, 1, 4095, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 696, 4094, 0, 0, 0, 0, 0, 4096, 2, 4096, 0, 0, 2747, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 4068, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 4096, 0, 0, 366, 0, 0, 0, 4, 4096, 6, 0, 0, 3568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 13, 4093, 4092, 0, 4094, 0, 2, 2450, 0, 1471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1367, 0, 0, 2, 0, 0, 4089, 0, 0, 2918, 2, 4096, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 25, 0, 0, 3966, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 441, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3356, 2070, 0, 68, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 515, 0, 1342, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4094, 0, 0, 0, 0, 2, 4096, 0, 4096, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16260, 16363, 16219, 16224, 16542, 16785, 16358, 16321] avg=16384.0000 max=16785 max_rank=5 min=16219 min_rank=2 imbalance_ratio=1.024475 max_load_reduction=114287 max_load_reduction_pct=0.871941 moved_to_echo=66006 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1045, 4094, 1, 0, 1, 0, 0, 4094, 2, 4095, 0, 1, 2746, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4093, 1, 10, 1, 0, 0, 4095, 1, 1, 4061, 1, 4095, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 3868, 0, 1, 920, 1, 0, 0, 0, 4094, 0, 3, 1, 3567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 27, 0, 0, 28, 0, 0, 4095, 4084, 0, 4095, 2, 0, 1676, 0, 1470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1367, 0, 0, 0, 1, 0, 4092, 1, 0, 2713, 0, 3828, 3549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 1, 0, 3979, 0, 1, 1, 1, 4095, 0, 0, 4093, 1, 2, 1, 441, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3717, 3045, 0, 22, 0, 3, 0, 1, 0, 0, 0, 4095, 0, 169, 1, 1342, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 1, 0, 126, 4095, 0, 4095, 74, 0, 1, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16608, 16398, 16553, 15478, 15553, 16657, 17307, 16518] avg=16384.0000 max=17307 max_rank=6 min=15478 min_rank=3 imbalance_ratio=1.056335 max_load_reduction=113765 max_load_reduction_pct=0.867958 moved_to_echo=66035 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 1161, 4095, 1, 1, 0, 0, 0, 4095, 24, 4095, 2, 1, 2746, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4095, 1, 4, 1, 0, 0, 4096, 0, 1, 4088, 1, 4095, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 0, 4094, 0, 1, 1033, 1, 1, 0, 24, 4095, 0, 44, 1, 3567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 154, 0, 0, 4095, 4095, 0, 4095, 0, 0, 2630, 0, 1470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1367, 0, 0, 1, 0, 0, 4089, 1, 0, 2138, 0, 4056, 4051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4018, 0, 0, 0, 3528, 0, 1, 0, 0, 4095, 1, 0, 4092, 1, 1, 1, 441, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3882, 1770, 0, 4, 0, 1, 0, 1, 0, 0, 17, 4089, 0, 61, 0, 1342, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 8, 4095, 0, 4095, 46, 0, 1, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16750, 16423, 16958, 16541, 15705, 16245, 16079, 16371] avg=16384.0000 max=16958 max_rank=2 min=15705 min_rank=4 imbalance_ratio=1.035034 max_load_reduction=114114 max_load_reduction_pct=0.870621 moved_to_echo=64400 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 6, 0, 0, 1180, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2747, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 4075, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 4096, 0, 0, 789, 0, 0, 0, 1, 4096, 11, 0, 0, 3568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 16, 4094, 4091, 0, 4095, 0, 1, 2744, 0, 1471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1367, 0, 0, 2, 0, 0, 4080, 0, 0, 2912, 3, 4096, 4061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 40, 0, 0, 3319, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 441, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3019, 1888, 0, 33, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 445, 0, 1342, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 4095, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16748, 16370, 16654, 16571, 16521, 16153, 15736, 16319] avg=16384.0000 max=16748 max_rank=0 min=15736 min_rank=6 imbalance_ratio=1.022217 max_load_reduction=114324 max_load_reduction_pct=0.872223 moved_to_echo=64729 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 980, 4095, 1, 0, 1, 0, 0, 4091, 1, 4095, 1, 1, 2750, 529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 4, 1, 0, 0, 4095, 0, 1, 4072, 1, 4095, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4090, 1, 1, 3831, 1, 1, 666, 1, 0, 0, 0, 4094, 0, 9, 1, 3566, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 63, 0, 0, 4095, 4093, 1, 4095, 17, 0, 3185, 0, 1474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1353, 0, 0, 0, 0, 0, 4093, 1, 0, 3466, 0, 3907, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3991, 0, 1, 0, 2877, 0, 1, 0, 1, 4095, 1, 0, 4091, 1, 2, 1, 446, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 4032, 1485, 0, 1, 0, 3, 0, 1, 0, 0, 0, 4095, 0, 3, 1, 1345, 822, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 91, 4095, 0, 4095, 93, 0, 1, 0, 4025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16546, 16406, 16263, 17030, 16870, 15578, 15883, 16496] avg=16384.0000 max=17030 max_rank=3 min=15578 min_rank=5 imbalance_ratio=1.039429 max_load_reduction=114042 max_load_reduction_pct=0.870071 moved_to_echo=64827 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 5, 0, 0, 574, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2747, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 4071, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 478, 0, 0, 0, 2, 4096, 5, 2, 0, 3568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 10, 4093, 4095, 0, 4094, 0, 2, 3111, 0, 1471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1367, 0, 0, 2, 0, 0, 4092, 0, 0, 2875, 13, 4096, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 25, 0, 0, 3504, 1, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 441, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2795, 1752, 0, 4, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 823, 0, 1342, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4093, 0, 2, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16142, 16366, 16338, 17336, 16524, 16324, 15724, 16318] avg=16384.0000 max=17336 max_rank=3 min=15724 min_rank=6 imbalance_ratio=1.058105 max_load_reduction=113736 max_load_reduction_pct=0.867737 moved_to_echo=64890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 1, 0, 574, 4094, 1, 1, 1, 0, 0, 4095, 6, 4095, 3, 1, 2746, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4095, 1, 2, 1, 0, 0, 4095, 1, 1, 4087, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 1, 3929, 1, 1, 443, 1, 0, 0, 6, 4095, 0, 1, 1, 3567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 20, 0, 1, 4095, 4075, 1, 4095, 0, 0, 1243, 0, 1470, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1367, 0, 0, 3, 0, 0, 4091, 1, 0, 3801, 0, 3894, 3578, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3964, 0, 1, 0, 4031, 0, 1, 0, 1, 4095, 0, 0, 4092, 1, 1, 1, 441, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 3559, 3475, 0, 135, 0, 1, 0, 1, 0, 0, 0, 4094, 1, 33, 0, 1345, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 1, 0, 25, 4095, 0, 4095, 68, 1, 1, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16146, 16383, 16143, 15001, 16737, 16695, 17556, 16411] avg=16384.0000 max=17556 max_rank=6 min=15001 min_rank=3 imbalance_ratio=1.071533 max_load_reduction=113516 max_load_reduction_pct=0.866058 moved_to_echo=67399 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[0, 2, 0, 0, 307, 4095, 0, 0, 0, 0, 1, 4096, 0, 4096, 0, 0, 2747, 528, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4096, 0, 0, 0, 0, 0, 4069, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 89, 0, 0, 0, 0, 4096, 4, 0, 0, 3568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 10, 4093, 4092, 0, 4095, 0, 0, 3017, 0, 1471, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1367, 0, 0, 0, 0, 0, 4093, 0, 0, 3045, 18, 4096, 4001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 23, 0, 0, 3816, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 441, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3561, 1948, 1, 5, 1, 0, 0, 0, 1, 0, 0, 4095, 0, 562, 0, 1342, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 1, 0, 0, 0, 4095, 1, 4095, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15872, 16363, 15944, 16891, 16620, 16635, 16429, 16318] avg=16384.0000 max=16891 max_rank=3 min=15872 min_rank=0 imbalance_ratio=1.030945 max_load_reduction=114181 max_load_reduction_pct=0.871132 moved_to_echo=66002 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[32768, 28032, 17536, 128, 384, 128, 128, 0, 128, 128, 0, 128, 32768, 128, 2688, 128, 10752, 6656, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 32768, 128, 128, 32128, 128, 128, 4864, 128, 128, 0, 128, 32768, 128, 128, 128, 28544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 32768, 128, 128, 128, 0, 256, 32768, 128, 32768, 384, 128, 128, 0, 32256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[32384, 128, 128, 0, 29056, 128, 128, 128, 128, 32768, 128, 0, 32768, 128, 128, 128, 3584, 640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 0, 1024, 0, 128, 32768, 32768, 128, 32768, 128, 128, 20096, 0, 11776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 128, 11008, 0, 0, 128, 128, 0, 32768, 128, 0, 23936, 128, 32128, 31488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 32768, 128, 128, 128, 0, 0, 32768, 128, 128, 32768, 128, 32768, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 128, 0, 6528, 32768, 128, 128, 128, 0, 128, 32768, 128, 32768, 128, 128, 22016, 4352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[2486, 0, 0, 4095, 0, 1, 4095, 3318, 1, 0, 1, 842, 0, 0, 1, 4093, 25, 3688, 3, 1962, 0, 4093, 0, 0, 1, 1, 4095, 0, 74, 0, 0, 0, 4093, 3733, 1, 4095, 4095, 2786, 1, 0, 0, 1, 0, 1, 45, 0, 0, 1, 1, 0, 1, 0, 17, 1, 1, 1, 4095, 1, 0, 744, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 4089, 386, 4, 1, 0, 0, 1, 6, 0, 0, 1, 1, 0, 1, 4095, 4095, 1, 4094, 0, 0, 0, 0, 4024, 1, 4086, 4094, 2, 0, 516, 1, 1, 4040, 0, 0, 0, 1, 1, 4093, 1, 0, 2, 4094, 0, 4095, 0, 4095, 4095, 4094, 0, 0, 0, 1, 4095, 4077, 0, 1, 4095, 5, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[3000, 0, 0, 4093, 0, 0, 4089, 3998, 0, 0, 0, 2839, 0, 0, 0, 4096, 2, 4096, 0, 265, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 4096, 3757, 0, 4082, 4096, 1989, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 0, 4, 71, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 0, 259, 0, 0, 0, 0, 0, 0, 4096, 4092, 0, 4096, 0, 0, 22, 0, 4096, 0, 4096, 4096, 0, 0, 259, 0, 0, 4096, 0, 2, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 4053, 0, 4093, 4094, 4096, 0, 0, 0, 0, 4065, 4096, 7, 0, 4090, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[3100, 0, 0, 4094, 0, 0, 4090, 4070, 0, 0, 0, 2410, 0, 0, 0, 4096, 3, 4096, 0, 97, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 4096, 3676, 0, 4036, 4095, 1839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 4096, 0, 4, 796, 0, 0, 0, 4096, 0, 0, 0, 0, 1, 3858, 0, 2, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 4096, 4093, 0, 4095, 0, 0, 15, 0, 4096, 0, 4096, 4096, 0, 0, 425, 0, 0, 4096, 0, 1, 3, 0, 0, 4096, 0, 0, 0, 4094, 0, 4043, 0, 4092, 4094, 4096, 0, 0, 0, 0, 4088, 4096, 6, 0, 4093, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[3592, 1, 1, 4095, 1, 0, 4095, 3687, 0, 0, 0, 240, 1, 0, 1, 4093, 53, 3929, 10, 676, 0, 4095, 0, 0, 0, 1, 4095, 0, 71, 0, 0, 0, 4094, 2618, 0, 4095, 4095, 3527, 1, 1, 0, 1, 0, 1, 9, 0, 0, 1, 1, 0, 1, 0, 388, 1, 0, 1, 4095, 1, 0, 1039, 0, 0, 1, 4095, 1, 0, 0, 1, 0, 4039, 24, 30, 1, 0, 0, 1, 52, 0, 0, 1, 1, 0, 1, 4095, 4095, 0, 4095, 0, 0, 0, 0, 3966, 1, 4095, 4095, 1, 0, 789, 1, 1, 4051, 1, 0, 0, 1, 0, 4095, 0, 0, 2, 4095, 0, 4093, 0, 4095, 4095, 4095, 0, 0, 0, 1, 4095, 4063, 4, 1, 4095, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[3147, 0, 0, 4094, 0, 0, 4082, 4096, 0, 0, 0, 1248, 0, 0, 0, 4096, 2, 4096, 0, 402, 0, 4095, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 4096, 1518, 0, 4049, 4095, 3051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 4096, 0, 4, 2623, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3961, 0, 1, 0, 0, 0, 0, 221, 0, 1, 0, 0, 0, 0, 4096, 4093, 0, 4096, 0, 0, 12, 0, 4096, 0, 4096, 4096, 0, 0, 191, 0, 0, 4096, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 4004, 0, 4095, 4095, 4096, 0, 0, 0, 0, 4069, 4096, 17, 0, 4089, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[4078, 0, 0, 4095, 1, 0, 4095, 3737, 1, 0, 1, 124, 0, 0, 1, 4094, 23, 3574, 1, 1222, 0, 4092, 0, 0, 0, 1, 4095, 0, 81, 0, 0, 0, 4094, 3682, 0, 4095, 4095, 2620, 0, 1, 0, 1, 0, 1, 41, 0, 0, 1, 1, 0, 1, 0, 106, 1, 1, 1, 4095, 1, 0, 1471, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 3340, 283, 11, 1, 0, 0, 1, 10, 0, 0, 1, 1, 0, 1, 4095, 4095, 1, 4095, 0, 0, 0, 0, 4018, 1, 4087, 4095, 1, 0, 348, 1, 1, 4007, 1, 0, 0, 1, 1, 4094, 1, 0, 3, 4094, 0, 4094, 2, 4095, 4095, 4094, 0, 0, 0, 1, 4095, 4081, 0, 1, 4095, 7, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1583, 1, 0, 4095, 0, 0, 4095, 3462, 1, 0, 1, 1098, 0, 0, 1, 4093, 33, 3745, 12, 1885, 0, 4093, 0, 0, 1, 1, 4095, 0, 64, 0, 0, 0, 4094, 3718, 2, 4095, 4095, 3436, 0, 1, 0, 1, 0, 1, 78, 0, 0, 1, 1, 0, 1, 0, 15, 1, 0, 1, 4095, 1, 0, 614, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 4090, 411, 10, 1, 0, 0, 1, 13, 0, 0, 1, 1, 0, 1, 4095, 4093, 1, 4090, 0, 0, 0, 0, 4019, 1, 4086, 4095, 2, 0, 483, 1, 1, 4018, 1, 0, 0, 1, 1, 4094, 1, 0, 1, 4094, 0, 4095, 1, 4095, 4095, 4095, 0, 0, 2, 1, 4095, 4080, 0, 1, 4095, 2, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[1210, 0, 0, 4095, 0, 0, 4083, 4096, 0, 0, 0, 2763, 0, 0, 0, 4096, 1, 4096, 0, 1623, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 4096, 3195, 0, 4067, 4095, 2341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 4096, 0, 3, 986, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4071, 1, 0, 0, 0, 0, 0, 176, 1, 1, 0, 0, 0, 0, 4096, 4092, 0, 4093, 0, 0, 16, 0, 4096, 0, 4096, 4096, 0, 1, 55, 0, 0, 4096, 0, 8, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 4034, 0, 4095, 4096, 4096, 0, 0, 0, 0, 4065, 4096, 13, 0, 4092, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[2486, 0, 0, 4095, 0, 1, 4095, 3318, 1, 0, 1, 842, 0, 0, 1, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 3688, 3, 1962, 0, 4093, 0, 0, 1, 1, 4095, 0, 74, 0, 0, 0, 1842, 1361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3733, 1, 4095, 2253, 2786, 1, 0, 0, 1, 0, 1, 45, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 17, 1, 1, 1, 4095, 1, 0, 744, 0, 0, 1, 4095, 4052, 2867, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4089, 386, 4, 1, 0, 0, 1, 6, 0, 0, 1, 4095, 4094, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4095, 1, 4094, 0, 0, 0, 0, 4024, 1, 4086, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 516, 1, 1, 4040, 0, 0, 0, 1, 1, 4093, 1, 0, 2, 4094, 0, 2562, 1178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 50, 0, 0, 0, 0, 0, 1, 4095, 4077, 0, 1, 4095, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15010, 17145, 17010, 15969, 16681, 16347, 16490, 16420] avg=16384.0000 max=17145 max_rank=1 min=15010 min_rank=0 imbalance_ratio=1.046448 max_load_reduction=113927 max_load_reduction_pct=0.869194 moved_to_echo=65938 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3000, 0, 0, 4093, 0, 0, 4089, 3998, 0, 0, 0, 2839, 0, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 265, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 1, 0, 0, 1843, 1362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3757, 0, 4082, 2253, 1989, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 4096, 0, 4, 71, 0, 0, 0, 4096, 4054, 2866, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 0, 259, 0, 0, 0, 4096, 4096, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4096, 0, 0, 22, 0, 4096, 0, 4096, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 0, 0, 4096, 0, 2, 3, 0, 0, 4096, 0, 0, 0, 4096, 0, 2564, 1178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 49, 0, 0, 0, 0, 0, 0, 4065, 4096, 7, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[18189, 15758, 16178, 15306, 16543, 16444, 16294, 16360] avg=16384.0000 max=18189 max_rank=0 min=15306 min_rank=3 imbalance_ratio=1.110168 max_load_reduction=112883 max_load_reduction_pct=0.861229 moved_to_echo=65641 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3100, 0, 0, 4094, 0, 0, 4090, 4070, 0, 0, 0, 2410, 0, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4096, 0, 97, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 1842, 1362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3676, 0, 4036, 2253, 1839, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 4096, 0, 4, 796, 0, 0, 0, 4096, 4054, 2865, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3858, 0, 2, 0, 0, 0, 0, 67, 0, 0, 0, 4096, 4096, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4095, 0, 0, 15, 0, 4096, 0, 4096, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 425, 0, 0, 4096, 0, 1, 3, 0, 0, 4096, 0, 0, 0, 4094, 0, 2564, 1177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4043, 0, 50, 0, 0, 0, 0, 0, 0, 4088, 4096, 6, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17934, 15592, 15900, 16255, 16122, 16437, 16456, 16376] avg=16384.0000 max=17934 max_rank=0 min=15592 min_rank=1 imbalance_ratio=1.094604 max_load_reduction=113138 max_load_reduction_pct=0.863174 moved_to_echo=65391 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3592, 1, 1, 4095, 1, 0, 4095, 3687, 0, 0, 0, 240, 1, 0, 1, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 3929, 10, 676, 0, 4095, 0, 0, 0, 1, 4095, 0, 71, 0, 0, 0, 1842, 1361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2618, 0, 4095, 2253, 3527, 1, 1, 0, 1, 0, 1, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 388, 1, 0, 1, 4095, 1, 0, 1039, 0, 0, 1, 4095, 4053, 2867, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 4039, 24, 30, 1, 0, 0, 1, 52, 0, 0, 1, 4095, 4095, 4003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4095, 0, 4095, 0, 0, 0, 0, 3966, 1, 4095, 42, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 789, 1, 1, 4051, 1, 0, 0, 1, 0, 4095, 0, 0, 2, 4095, 0, 2562, 1178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 50, 0, 0, 0, 0, 0, 1, 4095, 4063, 4, 1, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15884, 16133, 16601, 16635, 16343, 16297, 16776, 16403] avg=16384.0000 max=16776 max_rank=6 min=15884 min_rank=0 imbalance_ratio=1.023926 max_load_reduction=114296 max_load_reduction_pct=0.872009 moved_to_echo=65819 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[3147, 0, 0, 4094, 0, 0, 4082, 4096, 0, 0, 0, 1248, 0, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4096, 0, 402, 0, 4095, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 1842, 1362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1518, 0, 4049, 2253, 3051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 0, 0, 0, 4096, 0, 4, 2623, 0, 0, 0, 4096, 4054, 2867, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3961, 0, 1, 0, 0, 0, 0, 221, 0, 1, 0, 4096, 4096, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 4096, 0, 0, 12, 0, 4096, 0, 4096, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 0, 4096, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 2564, 1178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4004, 0, 50, 0, 0, 0, 0, 0, 0, 4069, 4096, 17, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16837, 15894, 14967, 18013, 16378, 16435, 16223, 16325] avg=16384.0000 max=18013 max_rank=3 min=14967 min_rank=2 imbalance_ratio=1.099426 max_load_reduction=113059 max_load_reduction_pct=0.862572 moved_to_echo=65361 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[4078, 0, 0, 4095, 1, 0, 4095, 3737, 1, 0, 1, 124, 0, 0, 1, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 3574, 1, 1222, 0, 4092, 0, 0, 0, 1, 4095, 0, 81, 0, 0, 0, 1847, 1367, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3682, 0, 4095, 2248, 2620, 0, 1, 0, 1, 0, 1, 41, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 106, 1, 1, 1, 4095, 1, 0, 1471, 0, 0, 1, 4095, 4054, 2874, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3340, 283, 11, 1, 0, 0, 1, 10, 0, 0, 1, 4095, 4094, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4095, 1, 4095, 0, 0, 0, 0, 4018, 1, 4087, 41, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 348, 1, 1, 4007, 1, 0, 0, 1, 1, 4094, 1, 0, 3, 4094, 0, 2566, 1183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 38, 0, 0, 0, 0, 0, 1, 4095, 4081, 0, 1, 4095, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16294, 16303, 16784, 16795, 15839, 16341, 16301, 16415] avg=16384.0000 max=16795 max_rank=3 min=15839 min_rank=4 imbalance_ratio=1.025085 max_load_reduction=114277 max_load_reduction_pct=0.871864 moved_to_echo=64896 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1583, 1, 0, 4095, 0, 0, 4095, 3462, 1, 0, 1, 1098, 0, 0, 1, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 3745, 12, 1885, 0, 4093, 0, 0, 1, 1, 4095, 0, 64, 0, 0, 0, 1842, 1361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3718, 2, 4095, 2253, 3436, 0, 1, 0, 1, 0, 1, 78, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 15, 1, 0, 1, 4095, 1, 0, 614, 0, 0, 1, 4095, 4053, 2867, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4090, 411, 10, 1, 0, 0, 1, 13, 0, 0, 1, 4095, 4095, 4002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4093, 1, 4090, 0, 0, 0, 0, 4019, 1, 4086, 42, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 483, 1, 1, 4018, 1, 0, 0, 1, 1, 4094, 1, 0, 1, 4094, 0, 2562, 1178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 50, 0, 0, 0, 0, 2, 1, 4095, 4080, 0, 1, 4095, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[14507, 17132, 17680, 15838, 16720, 16336, 16436, 16423] avg=16384.0000 max=17680 max_rank=2 min=14507 min_rank=0 imbalance_ratio=1.079102 max_load_reduction=113392 max_load_reduction_pct=0.865112 moved_to_echo=65915 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1210, 0, 0, 4095, 0, 0, 4083, 4096, 0, 0, 0, 2763, 0, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 1623, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 0, 1842, 1362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3195, 0, 4067, 2253, 2341, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 4096, 0, 3, 986, 0, 0, 0, 4095, 4054, 2867, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 1, 0, 0, 0, 0, 0, 176, 1, 1, 0, 4096, 4096, 4004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 4093, 0, 0, 16, 0, 4096, 0, 4096, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 55, 0, 0, 4096, 0, 8, 2, 0, 0, 4096, 0, 0, 0, 4096, 0, 2564, 1178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 50, 0, 0, 0, 0, 0, 0, 4065, 4096, 13, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16417, 17115, 15952, 16261, 16446, 16435, 16096, 16350] avg=16384.0000 max=17115 max_rank=1 min=15952 min_rank=2 imbalance_ratio=1.044617 max_load_reduction=113957 max_load_reduction_pct=0.869423 moved_to_echo=65327 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=7 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 0, 128, 128, 31616, 1152, 128, 128, 0, 0, 128, 896, 128, 128, 128, 32768, 32768, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[128, 0, 128, 0, 32768, 128, 32768, 0, 0, 128, 0, 32512, 128, 32768, 384, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 31360, 128, 8192, 0, 32768, 0, 0, 128, 128, 32768, 0, 384, 128, 0, 0, 14848, 11008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 3072, 128, 128, 32512, 128, 128, 128, 128, 128, 32768, 128, 0, 128, 32768, 0, 20608, 9472, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[22272, 128, 128, 32768, 128, 128, 32768, 30464, 128, 0, 128, 11648, 128, 0, 128, 1408, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32512, 128, 512, 0, 0, 0, 0, 128, 128, 32768, 32768, 128, 128, 32768, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[128, 0, 128, 0, 1152, 128, 128, 128, 32768, 128, 128, 8448, 0, 0, 128, 32768, 32512, 23040, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 25984, 128, 32640, 18048, 21632, 128, 128, 0, 128, 0, 128, 256, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 263, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 2, 0, 4096, 4077, 4096, 0, 0, 0, 4096, 0, 5, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2108, 0, 4090, 0, 0, 0, 0, 6, 22, 0, 4096, 0, 3113, 0, 4096, 2157, 0, 0, 0, 4096, 0, 0, 4096, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 377, 0, 0, 0, 0, 0, 0, 4096, 0, 4092, 0, 4086, 0, 0, 0, 0, 4094, 4093, 4096, 49, 0, 0, 0, 209, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 4096, 0, 4094, 0, 0, 2, 0, 0, 4078, 4096, 4038, 4094, 0, 0, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[1, 1, 0, 15, 1220, 2, 0, 1, 0, 0, 0, 17, 4004, 1, 0, 0, 4085, 0, 0, 4095, 4095, 4095, 0, 0, 1, 4095, 1, 0, 0, 0, 0, 4095, 1, 131, 1, 1, 0, 0, 1094, 0, 4095, 0, 1, 1, 1, 0, 0, 14, 4095, 1, 2733, 1, 4095, 2902, 1, 0, 0, 4095, 1, 1, 4094, 0, 0, 3947, 0, 8, 0, 1, 1, 0, 0, 0, 4084, 0, 1, 113, 1, 0, 1, 1, 1, 0, 4093, 1, 4095, 0, 4082, 0, 3, 1, 0, 4095, 4095, 4095, 8, 66, 0, 0, 160, 0, 2, 4095, 0, 1, 0, 0, 4091, 1, 4095, 1, 1, 1, 4095, 1, 4078, 0, 0, 0, 1, 0, 4095, 4095, 4095, 4094, 1, 1, 0, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[1, 1, 0, 16, 498, 2, 0, 1, 0, 0, 0, 25, 3992, 1, 1, 0, 4086, 0, 0, 4095, 4095, 4095, 0, 0, 0, 4095, 1, 0, 0, 1, 0, 4095, 1, 111, 1, 1, 0, 0, 1039, 0, 4095, 0, 1, 1, 1, 0, 0, 18, 4095, 1, 3227, 1, 4095, 3077, 1, 0, 0, 4095, 1, 1, 4093, 272, 0, 3925, 0, 5, 0, 1, 1, 0, 1, 0, 4088, 0, 1, 21, 0, 0, 1, 1, 1, 0, 4095, 1, 4095, 0, 4069, 1, 5, 1, 0, 4095, 4094, 4095, 1, 72, 0, 0, 141, 0, 5, 4095, 0, 1, 0, 0, 4090, 1, 4094, 1, 1, 1, 4095, 1, 4077, 1, 0, 0, 0, 0, 4095, 4095, 4095, 4093, 0, 0, 0, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 813, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4083, 4096, 0, 0, 0, 4096, 0, 4, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1656, 0, 4078, 0, 0, 0, 0, 24, 79, 0, 4096, 0, 3815, 0, 4096, 1110, 0, 0, 0, 4095, 0, 0, 4096, 13, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 724, 0, 0, 0, 0, 0, 0, 4096, 0, 4038, 0, 4094, 0, 0, 0, 2, 4095, 4074, 4096, 13, 0, 0, 0, 136, 1, 0, 4093, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 4096, 0, 4090, 0, 0, 0, 0, 0, 4044, 4096, 4071, 4095, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 965, 2, 0, 1, 1, 0, 1, 19, 4050, 1, 1, 0, 4094, 0, 0, 4095, 4095, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 2, 4095, 1, 107, 0, 1, 0, 0, 1399, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 4095, 1, 2748, 1, 4095, 2959, 1, 0, 0, 4095, 1, 1, 4089, 19, 0, 3904, 0, 16, 0, 0, 1, 0, 0, 0, 4093, 0, 2, 88, 0, 0, 1, 1, 1, 0, 4094, 1, 4095, 0, 4082, 1, 17, 0, 0, 4095, 4001, 4095, 41, 76, 0, 0, 114, 0, 0, 4095, 0, 1, 0, 0, 4094, 1, 4094, 1, 1, 1, 4095, 1, 4079, 1, 0, 0, 0, 0, 4095, 4095, 4095, 4092, 1, 1, 0, 4086] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[1, 0, 0, 15, 1412, 1, 0, 1, 0, 0, 0, 23, 4021, 1, 1, 0, 4091, 0, 0, 4095, 4095, 4095, 0, 0, 0, 4095, 1, 0, 0, 0, 3, 4095, 1, 120, 1, 1, 0, 0, 1770, 0, 4095, 0, 1, 1, 1, 0, 0, 26, 4095, 1, 2982, 1, 4095, 1855, 1, 0, 0, 4095, 1, 1, 4093, 0, 0, 3954, 0, 10, 0, 0, 1, 0, 1, 0, 4091, 0, 1, 15, 0, 1, 1, 1, 1, 0, 4092, 1, 4095, 0, 4071, 1, 9, 1, 2, 4095, 4091, 4095, 4, 19, 0, 0, 219, 0, 0, 4095, 0, 1, 0, 0, 4084, 1, 4095, 1, 1, 1, 4095, 1, 4078, 1, 0, 0, 0, 0, 4095, 4095, 4095, 4091, 0, 0, 0, 4087] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 383, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 4096, 4074, 4096, 0, 0, 0, 4096, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1631, 0, 4093, 0, 0, 0, 0, 7, 33, 0, 4096, 0, 3390, 0, 4096, 2448, 0, 0, 0, 4096, 0, 0, 4096, 5, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 297, 0, 0, 0, 0, 0, 0, 4096, 0, 4071, 0, 4096, 0, 0, 0, 2, 4094, 4087, 4096, 17, 0, 0, 0, 65, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 4096, 0, 4087, 0, 0, 2, 0, 0, 4080, 4096, 4086, 4094, 0, 0, 2, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 489, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 1, 1, 4096, 4093, 4094, 0, 0, 0, 4096, 0, 9, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 2789, 0, 4078, 0, 0, 0, 0, 7, 25, 0, 4096, 0, 2606, 0, 4096, 2062, 0, 0, 0, 4095, 0, 0, 4096, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 260, 0, 0, 0, 0, 0, 0, 4095, 0, 4052, 0, 4095, 0, 0, 0, 1, 4095, 4091, 4096, 16, 0, 0, 0, 48, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 4096, 0, 4093, 0, 1, 1, 0, 0, 4074, 4096, 4072, 4095, 0, 0, 1, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 15, 1220, 2, 0, 1, 0, 0, 0, 17, 4004, 1, 0, 0, 4095, 4095, 3121, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 4095, 4095, 0, 0, 1, 8, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 131, 1, 1, 0, 0, 1094, 0, 4095, 0, 1, 1, 1, 0, 0, 14, 4095, 4087, 1357, 974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2733, 1, 0, 2902, 1, 0, 0, 2836, 1, 1, 4094, 0, 0, 3947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 1, 0, 0, 0, 4084, 0, 1, 113, 1, 0, 1, 1, 4095, 4095, 3855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4087, 1, 4095, 0, 4082, 0, 3, 1, 0, 0, 4095, 0, 8, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 2, 4095, 0, 1, 0, 0, 4091, 1, 4095, 1, 1, 1, 2695, 1259, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4078, 0, 0, 0, 1, 0, 4095, 0, 4095, 42, 1, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16813, 16380, 15853, 16517, 16256, 16439, 16408, 16406] avg=16384.0000 max=16813 max_rank=0 min=15853 min_rank=2 imbalance_ratio=1.026184 max_load_reduction=114259 max_load_reduction_pct=0.871727 moved_to_echo=65509 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 263, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3122, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4077, 4096, 0, 0, 0, 8, 0, 5, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2108, 0, 4090, 0, 0, 0, 0, 6, 22, 0, 4094, 4088, 1357, 974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3113, 0, 0, 2157, 0, 0, 0, 2837, 0, 0, 4096, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 377, 0, 0, 0, 0, 4096, 4096, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4092, 0, 4086, 0, 0, 0, 0, 0, 4093, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 2695, 1259, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 4078, 0, 4038, 42, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15913, 16382, 16739, 16304, 16521, 16409, 16452, 16352] avg=16384.0000 max=16739 max_rank=2 min=15913 min_rank=0 imbalance_ratio=1.021667 max_load_reduction=114333 max_load_reduction_pct=0.872292 moved_to_echo=65734 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 16, 498, 2, 0, 1, 0, 0, 0, 25, 3992, 1, 1, 0, 4095, 4095, 3121, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4095, 4095, 0, 0, 0, 1, 1, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 111, 1, 1, 0, 0, 1039, 0, 4095, 0, 1, 1, 1, 0, 0, 18, 4095, 4094, 1360, 974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3227, 1, 0, 3077, 1, 0, 0, 2830, 1, 1, 4093, 272, 0, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 1, 0, 1, 0, 4088, 0, 1, 21, 0, 0, 1, 1, 4095, 4095, 3854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4085, 1, 4095, 0, 4069, 1, 5, 1, 0, 0, 4094, 0, 1, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 5, 4095, 0, 1, 0, 0, 4090, 1, 4094, 1, 1, 1, 2698, 1265, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4077, 1, 0, 0, 0, 0, 4095, 0, 4095, 35, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16090, 16374, 15792, 17429, 16164, 16425, 16403, 16395] avg=16384.0000 max=17429 max_rank=3 min=15792 min_rank=2 imbalance_ratio=1.063782 max_load_reduction=113643 max_load_reduction_pct=0.867027 moved_to_echo=65387 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 813, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3122, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4083, 4096, 0, 0, 0, 8, 0, 4, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1656, 0, 4078, 0, 0, 0, 0, 24, 79, 0, 4095, 4088, 1357, 974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3815, 0, 0, 1110, 0, 0, 0, 2836, 0, 0, 4096, 13, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 724, 0, 0, 0, 0, 4096, 4096, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4038, 0, 4094, 0, 0, 0, 2, 0, 4074, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 1, 0, 4093, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 0, 2696, 1259, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 4044, 0, 4071, 42, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16463, 16385, 16351, 15966, 16870, 16310, 16383, 16344] avg=16384.0000 max=16870 max_rank=4 min=15966 min_rank=3 imbalance_ratio=1.029663 max_load_reduction=114202 max_load_reduction_pct=0.871292 moved_to_echo=65907 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 965, 2, 0, 1, 1, 0, 1, 19, 4050, 1, 1, 0, 4095, 4095, 3120, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4095, 4095, 0, 0, 0, 8, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 107, 0, 1, 0, 0, 1399, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 4095, 4087, 1356, 975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2748, 1, 0, 2959, 1, 0, 0, 2836, 1, 1, 4089, 19, 0, 3904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 1, 0, 0, 0, 4093, 0, 2, 88, 0, 0, 1, 1, 4095, 4095, 3855, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4088, 1, 4095, 0, 4082, 1, 17, 0, 0, 0, 4001, 0, 41, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 4095, 0, 1, 0, 0, 4094, 1, 4094, 1, 1, 1, 2694, 1259, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4079, 1, 0, 0, 0, 0, 4095, 0, 4095, 42, 1, 1, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16593, 16389, 16118, 16560, 16247, 16403, 16361, 16401] avg=16384.0000 max=16593 max_rank=0 min=16118 min_rank=2 imbalance_ratio=1.012756 max_load_reduction=114479 max_load_reduction_pct=0.873405 moved_to_echo=65412 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 383, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3122, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4074, 4096, 0, 0, 0, 8, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1631, 0, 4093, 0, 0, 0, 0, 7, 33, 0, 4094, 4088, 1357, 974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3390, 0, 0, 2448, 0, 0, 0, 2837, 0, 0, 4096, 5, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 297, 0, 0, 0, 0, 4096, 4096, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 4071, 0, 4096, 0, 0, 0, 2, 0, 4087, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 2, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 2695, 1259, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 2, 0, 0, 4080, 0, 4086, 42, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16033, 16376, 16277, 16872, 16444, 16362, 16313, 16395] avg=16384.0000 max=16872 max_rank=3 min=16033 min_rank=0 imbalance_ratio=1.029785 max_load_reduction=114200 max_load_reduction_pct=0.871277 moved_to_echo=65514 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 15, 1412, 1, 0, 1, 0, 0, 0, 23, 4021, 1, 1, 0, 4095, 4095, 3121, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 4095, 4095, 0, 0, 0, 8, 1, 0, 0, 0, 3, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 120, 1, 1, 0, 0, 1770, 0, 4095, 0, 1, 1, 1, 0, 0, 26, 4095, 4087, 1356, 974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2982, 1, 0, 1855, 1, 0, 0, 2836, 1, 1, 4093, 0, 0, 3954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1, 0, 1, 0, 4091, 0, 1, 15, 0, 1, 1, 1, 4095, 4095, 3854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4086, 1, 4095, 0, 4071, 1, 9, 1, 2, 0, 4091, 0, 4, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 219, 0, 0, 4095, 0, 1, 0, 0, 4084, 1, 4095, 1, 1, 1, 2693, 1259, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4078, 1, 0, 0, 0, 0, 4095, 0, 4095, 42, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17028, 16388, 16529, 15725, 16166, 16381, 16456, 16399] avg=16384.0000 max=17028 max_rank=0 min=15725 min_rank=3 imbalance_ratio=1.039307 max_load_reduction=114044 max_load_reduction_pct=0.870087 moved_to_echo=65402 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 489, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 3122, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 1, 0, 4093, 4094, 0, 0, 0, 8, 0, 9, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2789, 0, 4078, 0, 0, 0, 0, 7, 25, 0, 4095, 4088, 1357, 974, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2606, 0, 0, 2062, 0, 0, 0, 2836, 0, 0, 4096, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 260, 0, 0, 0, 0, 4096, 4096, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4052, 0, 4095, 0, 0, 0, 1, 0, 4091, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 1, 0, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 0, 2696, 1259, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1, 1, 0, 0, 4074, 0, 4072, 42, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16139, 16398, 17413, 15699, 16404, 16343, 16296, 16380] avg=16384.0000 max=17413 max_rank=2 min=15699 min_rank=3 imbalance_ratio=1.062805 max_load_reduction=113659 max_load_reduction_pct=0.867149 moved_to_echo=65423 active_clone_links=14 active_echo_slots_local=14 active_home_experts_offloading=11 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 128, 0, 32768, 32768, 0, 0, 128, 128, 128, 128, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 1152, 128, 128, 32768, 0, 128, 0, 0, 32768, 128, 32768, 128, 128, 128, 21632, 10112, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 0, 32768, 128, 32640, 0, 32768, 128, 128, 128, 128, 0, 32640, 0, 256, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132480 observed_loss=-1408 tokens_per_expert=[0, 128, 32768, 128, 128, 128, 128, 0, 32768, 0, 32768, 384, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[0, 128, 24704, 128, 0, 18688, 128, 0, 0, 22784, 128, 128, 32768, 384, 0, 32128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 512, 128, 128, 0, 0, 13568, 0, 32768, 0, 128, 128, 128, 128, 256, 128, 32768, 32768, 10880, 7808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 128, 6144, 128, 0, 128, 128, 0, 128, 128, 32512, 128, 128, 0, 32768, 32768, 25088, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 0, 128, 128, 0, 128, 0, 32768, 0, 128, 1920, 128, 128, 128, 128, 32768, 32768, 30848, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132480 slack_slots=129664 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4096, 1164, 4096, 0, 4096, 4096, 4096, 0, 4096, 4066, 0, 4096, 0, 0, 0, 0, 30, 0, 0, 0, 4094, 0, 97, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 37, 0, 4074, 0, 0, 0, 4096, 420, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 4091, 0, 0, 3, 0, 0, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 1, 4096, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 7, 0, 0, 1410, 0, 1125, 4096, 0, 0, 20, 0, 0, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 4093, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 1807, 4093, 0, 4096, 4096, 4096, 0, 4096, 4088, 0, 4096, 0, 0, 0, 0, 75, 0, 0, 0, 4093, 0, 188, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 7, 0, 4035, 0, 0, 0, 4096, 40, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 0, 0, 2, 0, 0, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 8, 0, 0, 833, 0, 1492, 4096, 0, 0, 5, 0, 0, 0, 0, 4092, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3808, 4094, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4095, 1, 0, 4068, 934, 4095, 0, 4095, 4094, 4076, 1, 4095, 3994, 1, 4092, 1, 0, 0, 0, 16, 0, 0, 5, 4092, 0, 5, 0, 7, 4092, 1, 0, 0, 4095, 1, 1, 3, 0, 3772, 1, 1, 1, 4080, 2061, 0, 3972, 4013, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 32, 1, 1, 0, 4080, 1, 4095, 0, 1, 0, 0, 1, 0, 4036, 3860, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4079, 0, 0, 1, 0, 1, 4095, 0, 4091, 0, 23, 1, 2370, 0, 1409, 4043, 3, 129, 50, 0, 1, 3, 21, 3594, 0, 0, 0, 0, 1, 0, 0, 88, 3, 1, 1, 1, 2609, 4095, 4095, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[1, 1, 0, 1, 4095, 0, 0, 4082, 2459, 4095, 0, 4095, 4094, 4080, 1, 4094, 4069, 1, 4092, 1, 0, 0, 0, 44, 0, 0, 1, 4070, 0, 62, 0, 7, 4095, 1, 0, 0, 4095, 1, 1, 3, 0, 3965, 1, 1, 1, 4092, 598, 0, 4007, 4004, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 17, 1, 1, 0, 4092, 1, 4095, 0, 1, 0, 0, 1, 0, 4068, 3903, 30, 1, 0, 0, 0, 6, 0, 4095, 0, 0, 0, 0, 1, 4085, 0, 0, 1, 0, 1, 4095, 1, 4088, 0, 24, 0, 3130, 0, 441, 4052, 0, 82, 109, 0, 1, 1, 8, 3753, 0, 0, 1, 0, 0, 0, 0, 48, 3, 1, 1, 1, 2137, 4095, 4095, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4096, 876, 4095, 0, 4096, 4096, 4096, 0, 4096, 4067, 0, 4096, 0, 0, 0, 0, 381, 0, 0, 0, 4091, 0, 226, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 185, 0, 4067, 0, 0, 0, 4096, 1002, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 4096, 11, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 12, 0, 0, 373, 0, 1574, 4096, 0, 0, 66, 0, 0, 0, 0, 4089, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3561, 4091, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4095, 1, 0, 4080, 2232, 4095, 0, 4095, 4094, 4081, 1, 4095, 4071, 1, 4095, 1, 0, 0, 0, 46, 0, 0, 5, 4087, 0, 58, 0, 9, 4091, 1, 0, 0, 4095, 1, 1, 0, 0, 4001, 1, 1, 1, 4081, 379, 0, 4000, 3997, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 19, 1, 1, 0, 4079, 1, 4095, 0, 1, 0, 0, 1, 0, 4044, 3827, 0, 1, 0, 0, 0, 4, 0, 4095, 0, 0, 0, 0, 1, 4080, 0, 0, 1, 0, 1, 4095, 0, 4090, 0, 16, 1, 2916, 0, 512, 4040, 0, 92, 56, 0, 1, 2, 13, 3489, 0, 0, 0, 0, 0, 0, 0, 87, 4, 1, 1, 1, 3124, 4095, 4095, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4095, 1, 0, 4068, 584, 4095, 0, 4095, 4094, 4076, 1, 4095, 4081, 1, 4093, 1, 0, 0, 0, 40, 0, 0, 2, 4084, 0, 33, 0, 12, 4094, 1, 0, 0, 4095, 1, 1, 6, 0, 3832, 1, 1, 1, 4092, 1327, 0, 3978, 4021, 1, 1, 1, 1, 4095, 0, 0, 0, 4095, 24, 1, 1, 0, 4083, 1, 4095, 0, 1, 0, 0, 1, 0, 4059, 3916, 1, 1, 1, 0, 0, 3, 0, 4095, 0, 1, 0, 0, 1, 4082, 0, 0, 1, 0, 1, 4095, 0, 4090, 0, 27, 1, 1506, 0, 2557, 4032, 0, 108, 42, 0, 1, 3, 11, 3658, 0, 0, 1, 0, 0, 0, 0, 92, 2, 1, 1, 1, 3085, 4095, 4095, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4096, 891, 4095, 0, 4096, 4096, 4096, 0, 4096, 4090, 0, 4096, 0, 0, 0, 0, 201, 1, 0, 0, 4094, 0, 27, 0, 1, 4096, 0, 0, 0, 4095, 0, 0, 23, 0, 4069, 0, 0, 0, 4096, 116, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 0, 4095, 6, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 8, 0, 0, 915, 0, 2069, 4096, 0, 0, 27, 0, 0, 0, 0, 4084, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 4092, 4096, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 4096, 1164, 2748, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4066, 0, 4096, 0, 0, 0, 0, 30, 0, 0, 0, 4094, 0, 97, 0, 0, 1943, 1348, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3444, 0, 0, 37, 0, 4074, 0, 0, 0, 4096, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 91, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 3, 0, 0, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 2156, 2062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1410, 0, 1125, 4096, 0, 0, 20, 0, 0, 0, 0, 4094, 0, 4096, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3935, 4093, 4096, 0, 0, 2935, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16198, 16319, 16167, 16473, 16509, 16396, 16009, 17001] avg=16384.0000 max=17001 max_rank=7 min=16009 min_rank=6 imbalance_ratio=1.037659 max_load_reduction=114071 max_load_reduction_pct=0.870293 moved_to_echo=65915 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 1, 1, 4095, 1, 0, 4068, 934, 2747, 0, 0, 0, 4076, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3994, 1, 4092, 1, 0, 0, 0, 16, 0, 0, 5, 4092, 0, 5, 0, 7, 1943, 1348, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 3443, 1, 1, 3, 0, 3772, 1, 1, 1, 4080, 2061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3972, 4013, 1, 1, 1, 1, 4095, 0, 0, 0, 91, 32, 1, 1, 0, 4080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 0, 4036, 3860, 0, 0, 1, 0, 0, 0, 2154, 2061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4079, 0, 0, 1, 0, 1, 4095, 0, 4091, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1, 2370, 0, 1409, 4043, 3, 129, 50, 0, 1, 3, 21, 3594, 0, 4095, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 88, 3, 1, 1, 1, 2609, 4095, 4095, 1, 0, 2934, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15925, 16149, 17457, 16289, 16210, 16370, 16903, 15769] avg=16384.0000 max=17457 max_rank=2 min=15769 min_rank=7 imbalance_ratio=1.065491 max_load_reduction=113615 max_load_reduction_pct=0.866814 moved_to_echo=65252 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 4096, 1807, 2746, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4096, 0, 0, 0, 0, 75, 0, 0, 0, 4093, 0, 188, 0, 0, 1943, 1347, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3444, 0, 0, 7, 0, 4035, 0, 0, 0, 4096, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 91, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2, 0, 0, 0, 4096, 4096, 5, 0, 0, 0, 0, 0, 2156, 2062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 833, 0, 1492, 4096, 0, 0, 5, 0, 0, 0, 0, 4092, 0, 4096, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3808, 4094, 4096, 0, 0, 2935, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16841, 16475, 15718, 16474, 16512, 16394, 15783, 16875] avg=16384.0000 max=16875 max_rank=7 min=15718 min_rank=2 imbalance_ratio=1.029968 max_load_reduction=114197 max_load_reduction_pct=0.871254 moved_to_echo=65564 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 0, 1, 4095, 0, 0, 4082, 2459, 2747, 0, 0, 0, 4080, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4069, 1, 4092, 1, 0, 0, 0, 44, 0, 0, 1, 4070, 0, 62, 0, 7, 1943, 1348, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 3443, 1, 1, 3, 0, 3965, 1, 1, 1, 4092, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4007, 4004, 1, 1, 1, 1, 4095, 0, 0, 0, 91, 17, 1, 1, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 0, 4068, 3903, 30, 1, 0, 0, 0, 6, 2154, 2061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4085, 0, 0, 1, 0, 1, 4095, 1, 4088, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 3130, 0, 441, 4052, 0, 82, 109, 0, 1, 1, 8, 3753, 0, 4095, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 48, 3, 1, 1, 1, 2137, 4095, 4095, 1, 0, 2934, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17467, 16283, 16202, 16312, 16321, 16374, 16856, 15257] avg=16384.0000 max=17467 max_rank=0 min=15257 min_rank=7 imbalance_ratio=1.066101 max_load_reduction=113605 max_load_reduction_pct=0.866737 moved_to_echo=64808 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4092, 0, 0, 4096, 876, 2747, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 4096, 0, 0, 0, 0, 381, 0, 0, 0, 4091, 0, 226, 0, 0, 1943, 1348, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3444, 0, 0, 185, 0, 4067, 0, 0, 0, 4096, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 91, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 4096, 4096, 11, 0, 0, 0, 0, 0, 2156, 2062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 373, 0, 1574, 4096, 0, 0, 66, 0, 0, 0, 0, 4089, 0, 4096, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3561, 4091, 4096, 0, 0, 2935, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15907, 16797, 16890, 16475, 16518, 16394, 15467, 16624] avg=16384.0000 max=16890 max_rank=2 min=15467 min_rank=6 imbalance_ratio=1.030884 max_load_reduction=114182 max_load_reduction_pct=0.871140 moved_to_echo=65003 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4095, 1, 0, 4080, 2232, 2743, 0, 0, 0, 4081, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 1, 4095, 1, 0, 0, 0, 46, 0, 0, 5, 4087, 0, 58, 0, 9, 1948, 1352, 647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 3437, 1, 1, 0, 0, 4001, 1, 1, 1, 4081, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4000, 3997, 1, 1, 1, 1, 4095, 0, 0, 0, 81, 19, 1, 1, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 0, 4044, 3827, 0, 1, 0, 0, 0, 4, 2155, 2066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1, 4080, 0, 0, 1, 0, 1, 4095, 0, 4090, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 2916, 0, 512, 4040, 0, 92, 56, 0, 1, 2, 13, 3489, 0, 4095, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 4, 1, 1, 1, 3124, 4095, 4095, 1, 0, 2933, 1939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17235, 16320, 15995, 16277, 16195, 16374, 16395, 16281] avg=16384.0000 max=17235 max_rank=0 min=15995 min_rank=2 imbalance_ratio=1.051941 max_load_reduction=113837 max_load_reduction_pct=0.868507 moved_to_echo=65245 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 1, 4095, 1, 0, 4068, 584, 2747, 0, 0, 0, 4076, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 1, 4093, 1, 0, 0, 0, 40, 0, 0, 2, 4084, 0, 33, 0, 12, 1943, 1348, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 3443, 1, 1, 6, 0, 3832, 1, 1, 1, 4092, 1327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3978, 4021, 1, 1, 1, 1, 4095, 0, 0, 0, 91, 24, 1, 1, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 1, 0, 0, 1, 0, 4059, 3916, 1, 1, 1, 0, 0, 3, 2155, 2061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 1, 4082, 0, 0, 1, 0, 1, 4095, 0, 4090, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 1, 1506, 0, 2557, 4032, 0, 108, 42, 0, 1, 3, 11, 3658, 0, 4095, 1162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 92, 2, 1, 1, 1, 3085, 4095, 4095, 1, 0, 2933, 1939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15574, 16283, 16800, 16298, 16295, 16373, 17203, 16246] avg=16384.0000 max=17203 max_rank=6 min=15574 min_rank=0 imbalance_ratio=1.049988 max_load_reduction=113869 max_load_reduction_pct=0.868752 moved_to_echo=66117 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4095, 0, 0, 4096, 891, 2747, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 0, 0, 201, 1, 0, 0, 4094, 0, 27, 0, 1, 1943, 1348, 645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3443, 0, 0, 23, 0, 4069, 0, 0, 0, 4096, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 91, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 4096, 4, 0, 0, 0, 0, 0, 2156, 2062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 6, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 915, 0, 2069, 4096, 0, 0, 27, 0, 0, 0, 0, 4084, 0, 4096, 1161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3955, 4092, 4096, 0, 0, 2935, 1940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15925, 16446, 15843, 16474, 16512, 16397, 16456, 17019] avg=16384.0000 max=17019 max_rank=7 min=15843 min_rank=2 imbalance_ratio=1.038757 max_load_reduction=114053 max_load_reduction_pct=0.870155 moved_to_echo=66384 active_clone_links=10 active_echo_slots_local=10 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32768, 128, 0, 0, 27648, 128, 128, 384, 0, 31872, 128, 128, 128, 32768, 6016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 128, 13568, 0, 11264, 32640, 128, 512, 384, 0, 128, 128, 128, 30976, 0, 32768, 9344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32384, 32512, 128, 128, 128, 128, 32768, 0, 0, 0, 768, 128, 128, 128, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 128, 128, 0, 128, 128, 32768, 0, 0, 128, 0, 128, 32768, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 128, 32768, 128, 0, 32768, 11008, 22016, 0, 0, 0, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 32768, 0, 128, 128, 0, 128, 0, 32640, 32000, 128, 128, 128, 0, 0, 128, 17280, 16512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[32640, 128, 32768, 128, 0, 0, 0, 896, 128, 0, 128, 32768, 0, 768, 0, 128, 15616, 10880, 5248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[128, 128, 0, 128, 0, 0, 384, 128, 128, 128, 128, 26240, 32768, 32768, 128, 0, 23552, 15616, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 4093, 22, 0, 13, 0, 4068, 1, 0, 90, 1, 0, 1, 4094, 0, 0, 0, 4092, 3, 0, 1, 1, 4095, 4094, 0, 1, 4095, 4082, 4084, 0, 0, 2, 1, 1, 1, 0, 1, 1, 35, 5, 0, 0, 4095, 1, 0, 3983, 0, 0, 1, 4094, 1, 4069, 24, 0, 1, 58, 4095, 1, 1, 1, 0, 2530, 3, 4041, 51, 1, 4, 4065, 4094, 2, 0, 0, 1, 1, 0, 1, 1, 4037, 0, 1, 1, 12, 4095, 0, 0, 3300, 4095, 0, 4095, 0, 0, 3703, 1, 0, 4091, 1, 4095, 0, 4072, 4095, 0, 0, 27, 4095, 1, 525, 0, 0, 7, 1, 1, 0, 1, 2241, 0, 0, 0, 1, 4095, 4087, 0, 2, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 35, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 4094, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 11, 0, 4096, 0, 4096, 0, 0, 0, 485, 4096, 0, 0, 0, 0, 4041, 0, 4093, 0, 0, 4, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 4048, 0, 0, 1971, 4093, 0, 4096, 0, 0, 4096, 0, 0, 4094, 0, 4096, 0, 4096, 4095, 2, 0, 0, 4096, 0, 19, 0, 0, 0, 0, 0, 0, 0, 1755, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 2, 4091, 41, 0, 11, 0, 4066, 1, 0, 33, 1, 0, 1, 4095, 0, 0, 0, 4081, 0, 0, 1, 1, 4095, 4091, 0, 1, 4095, 4093, 4094, 0, 0, 1, 0, 1, 0, 0, 1, 1, 8, 6, 0, 0, 4095, 1, 0, 3903, 0, 0, 1, 4094, 1, 4086, 14, 0, 1, 275, 4095, 1, 1, 1, 0, 2709, 1, 4027, 4, 1, 6, 4062, 4095, 1, 0, 0, 1, 1, 0, 0, 1, 3464, 3, 0, 1, 28, 4095, 0, 0, 3301, 4069, 0, 4095, 0, 1, 4054, 1, 0, 4085, 1, 4095, 0, 4090, 4095, 0, 0, 27, 4095, 0, 530, 0, 0, 0, 1, 1, 0, 1, 2244, 19, 0, 0, 1, 4095, 4094, 0, 1, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 126, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 4096, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 16, 0, 4096, 0, 4096, 0, 0, 0, 1607, 4096, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 3, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 3050, 0, 0, 0, 0, 4077, 0, 0, 2153, 4094, 0, 4096, 0, 0, 3986, 0, 0, 4096, 0, 4096, 0, 4096, 4095, 1, 0, 1, 4096, 0, 29, 0, 0, 0, 0, 0, 0, 0, 1340, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4091, 22, 0, 11, 0, 4069, 1, 0, 45, 1, 0, 1, 4093, 0, 0, 0, 4087, 3, 0, 1, 1, 4095, 4094, 0, 1, 4095, 4089, 4089, 0, 0, 2, 1, 1, 1, 0, 1, 1, 24, 2, 0, 0, 4095, 1, 0, 3864, 0, 0, 1, 4093, 1, 4074, 21, 0, 0, 412, 4095, 1, 1, 1, 0, 3027, 4, 4034, 38, 1, 5, 4063, 4095, 2, 0, 0, 0, 1, 0, 0, 1, 3931, 0, 0, 1, 11, 4095, 0, 0, 3804, 4095, 0, 4095, 0, 1, 3761, 1, 0, 4095, 1, 4095, 0, 4073, 4095, 0, 0, 15, 4095, 1, 779, 0, 0, 30, 1, 1, 0, 1, 863, 0, 0, 0, 1, 4095, 4085, 0, 2, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 4090, 26, 0, 17, 0, 4068, 1, 0, 83, 1, 0, 1, 4093, 0, 0, 0, 4082, 2, 0, 1, 1, 4095, 4094, 0, 1, 4095, 4092, 4093, 0, 0, 2, 1, 1, 1, 0, 1, 1, 9, 3, 0, 0, 4095, 1, 0, 3958, 0, 0, 1, 4094, 1, 4075, 13, 0, 0, 275, 4095, 1, 1, 1, 0, 2615, 3, 4030, 21, 1, 12, 4066, 4095, 1, 0, 0, 1, 1, 0, 0, 1, 3831, 1, 0, 1, 7, 4095, 0, 0, 3481, 4094, 0, 4095, 0, 1, 3745, 1, 0, 4089, 1, 4095, 0, 4077, 4095, 0, 0, 25, 4095, 1, 476, 0, 0, 105, 1, 1, 0, 1, 1970, 0, 0, 0, 1, 4095, 4085, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 2, 0, 111, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 4094, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 974, 4096, 0, 0, 0, 0, 4088, 0, 4094, 0, 0, 3, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 3406, 0, 0, 0, 0, 4080, 0, 0, 2540, 4094, 0, 4096, 0, 0, 4095, 0, 0, 4094, 0, 4096, 0, 4096, 4094, 1, 0, 0, 4096, 0, 10, 0, 0, 0, 0, 0, 0, 0, 1170, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 1, 0, 88, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 4095, 0, 0, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 0, 14, 0, 4096, 0, 4096, 0, 0, 0, 257, 4096, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 1, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 3865, 0, 0, 0, 0, 4070, 0, 0, 3217, 4094, 0, 4096, 0, 0, 4089, 0, 0, 4096, 0, 4096, 0, 4096, 4095, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 787, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 1, 4093, 22, 0, 13, 0, 4068, 1, 0, 90, 1, 0, 1, 2412, 1546, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 4092, 3, 0, 1, 1, 0, 4094, 0, 1, 0, 4082, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 1, 0, 1, 1, 35, 5, 0, 0, 4095, 1, 0, 4095, 4095, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 1, 4094, 1, 4069, 24, 0, 1, 58, 3632, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2530, 3, 4041, 51, 1, 4, 4065, 4094, 2, 0, 0, 1, 1, 0, 1, 1, 408, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4037, 0, 1, 1, 12, 4095, 0, 0, 3300, 1683, 0, 0, 0, 0, 3703, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 4072, 4095, 0, 0, 27, 3782, 1, 525, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 2241, 0, 0, 0, 1, 4095, 4087, 0, 2, 0, 0, 0, 4095, 2535, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16398, 16371, 16415, 15866, 15516, 16833, 16601, 17072] avg=16384.0000 max=17072 max_rank=7 min=15516 min_rank=4 imbalance_ratio=1.041992 max_load_reduction=114000 max_load_reduction_pct=0.869751 moved_to_echo=66022 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 35, 0, 2411, 1547, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 11, 0, 4096, 0, 4096, 0, 0, 0, 485, 3632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 4093, 0, 0, 4, 4096, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 408, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 4048, 0, 0, 1971, 1682, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 4095, 2, 0, 0, 3783, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1755, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4094, 2537, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16339, 16394, 16371, 16416, 17054, 15816, 16089, 16593] avg=16384.0000 max=17054 max_rank=4 min=15816 min_rank=5 imbalance_ratio=1.040894 max_load_reduction=114018 max_load_reduction_pct=0.869888 moved_to_echo=65552 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 2, 4091, 41, 0, 11, 0, 4066, 1, 0, 33, 1, 0, 1, 2397, 1547, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4081, 0, 0, 1, 1, 0, 4091, 0, 1, 0, 4093, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 8, 6, 0, 0, 4095, 1, 0, 4095, 4095, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3903, 0, 0, 1, 4094, 1, 4086, 14, 0, 1, 275, 3632, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2709, 1, 4027, 4, 1, 6, 4062, 4095, 1, 0, 0, 1, 1, 0, 0, 1, 408, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3464, 3, 0, 1, 28, 4095, 0, 0, 3301, 1672, 0, 0, 0, 1, 4054, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 1, 0, 0, 4090, 4095, 0, 0, 27, 3782, 0, 530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 2244, 19, 0, 0, 1, 4095, 4094, 0, 1, 0, 0, 0, 4095, 2536, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16341, 16374, 16385, 16010, 15630, 16620, 16610, 17102] avg=16384.0000 max=17102 max_rank=7 min=15630 min_rank=4 imbalance_ratio=1.043823 max_load_reduction=113970 max_load_reduction_pct=0.869522 moved_to_echo=65962 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4095, 0, 0, 0, 4091, 22, 0, 11, 0, 4069, 1, 0, 45, 1, 0, 1, 2416, 1549, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4087, 3, 0, 1, 1, 0, 4094, 0, 1, 0, 4089, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 1, 0, 1, 1, 24, 2, 0, 0, 4095, 1, 0, 4095, 4095, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3864, 0, 0, 1, 4093, 1, 4074, 21, 0, 0, 412, 3626, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3027, 4, 4034, 38, 1, 5, 4063, 4095, 2, 0, 0, 0, 1, 0, 0, 1, 410, 314, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3931, 0, 0, 1, 11, 4095, 0, 0, 3804, 1679, 0, 0, 0, 1, 3761, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4073, 4095, 0, 0, 15, 3781, 1, 779, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 863, 0, 0, 0, 1, 4095, 4085, 0, 2, 0, 1, 0, 4095, 2538, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16360, 16371, 16401, 16095, 15995, 17284, 16870, 15696] avg=16384.0000 max=17284 max_rank=5 min=15696 min_rank=7 imbalance_ratio=1.054932 max_load_reduction=113788 max_load_reduction_pct=0.868134 moved_to_echo=65845 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 126, 0, 2412, 1547, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 16, 0, 4096, 0, 4096, 0, 0, 0, 1607, 3632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4093, 0, 0, 3, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 408, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3050, 0, 0, 0, 0, 4077, 0, 0, 2153, 1682, 0, 0, 0, 0, 3986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 1, 0, 1, 3783, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1340, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 2537, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16430, 16395, 16371, 17543, 17107, 14948, 16101, 16177] avg=16384.0000 max=17543 max_rank=3 min=14948 min_rank=5 imbalance_ratio=1.070740 max_load_reduction=113529 max_load_reduction_pct=0.866158 moved_to_echo=64333 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 2, 0, 111, 0, 2412, 1547, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 5, 0, 4096, 0, 4096, 0, 0, 0, 974, 3632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 4094, 0, 0, 3, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 408, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3406, 0, 0, 0, 0, 4080, 0, 0, 2540, 1682, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 4094, 1, 0, 0, 3783, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 2, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 2537, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16417, 16394, 16371, 16899, 17100, 15803, 16078, 16010] avg=16384.0000 max=17100 max_rank=4 min=15803 min_rank=5 imbalance_ratio=1.043701 max_load_reduction=113972 max_load_reduction_pct=0.869537 moved_to_echo=64991 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4094, 0, 0, 0, 4090, 26, 0, 17, 0, 4068, 1, 0, 83, 1, 0, 1, 2412, 1546, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4082, 2, 0, 1, 1, 0, 4094, 0, 1, 0, 4092, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 1, 0, 1, 1, 9, 3, 0, 0, 4095, 1, 0, 4095, 4095, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3958, 0, 0, 1, 4094, 1, 4075, 13, 0, 0, 275, 3632, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2615, 3, 4030, 21, 1, 12, 4066, 4095, 1, 0, 0, 1, 1, 0, 0, 1, 408, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3831, 1, 0, 1, 7, 4095, 0, 0, 3481, 1682, 0, 0, 0, 1, 3745, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 4077, 4095, 0, 0, 25, 3782, 1, 476, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1970, 0, 0, 0, 1, 4095, 4085, 0, 0, 0, 0, 0, 4095, 2535, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16394, 16378, 16387, 16052, 15568, 16845, 16651, 16797] avg=16384.0000 max=16845 max_rank=5 min=15568 min_rank=4 imbalance_ratio=1.028137 max_load_reduction=114227 max_load_reduction_pct=0.871483 moved_to_echo=65861 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[4096, 0, 0, 0, 4096, 0, 1, 0, 0, 4096, 0, 0, 1, 0, 88, 0, 2412, 1547, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 14, 0, 4096, 0, 4096, 0, 0, 0, 257, 3632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4096, 0, 0, 1, 4096, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 408, 313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 0, 0, 0, 0, 4070, 0, 0, 3217, 1682, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4095, 0, 0, 0, 3783, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 787, 1, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 4095, 2537, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16393, 16395, 16371, 16191, 17102, 16923, 16072, 15625] avg=16384.0000 max=17102 max_rank=4 min=15625 min_rank=7 imbalance_ratio=1.043823 max_load_reduction=113970 max_load_reduction_pct=0.869522 moved_to_echo=65722 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 0, 0, 128, 32768, 128, 128, 128, 0, 32768, 128, 0, 256, 128, 384, 128, 19328, 12416, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 128, 0, 0, 32768, 32768, 128, 0, 128, 30336, 128, 2432, 0, 0, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131712 observed_loss=-640 tokens_per_expert=[128, 0, 0, 0, 32768, 128, 0, 128, 128, 0, 32768, 0, 128, 0, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32128, 0, 128, 128, 32768, 128, 32768, 128, 0, 128, 4352, 29056, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 128, 128, 128, 0, 128, 128, 128, 128, 0, 0, 32768, 128, 0, 32768, 32768, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[27264, 128, 32512, 128, 128, 128, 32640, 32768, 128, 128, 0, 128, 128, 0, 128, 128, 3328, 2560, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[29696, 128, 128, 128, 128, 32768, 0, 0, 23808, 13568, 0, 0, 0, 128, 31616, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 128, 12416, 128, 0, 0, 128, 32768, 32768, 0, 128, 0, 128, 0, 32768, 20352, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131712 slack_slots=130432 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[1, 1, 4023, 1, 1, 0, 0, 1, 4094, 1, 1, 4095, 5, 1, 14, 2, 4089, 1, 4095, 0, 0, 0, 0, 4095, 4095, 1, 4024, 1, 1, 29, 4091, 1, 1, 1, 0, 0, 0, 0, 4, 0, 3594, 0, 10, 0, 0, 0, 0, 5, 1135, 0, 0, 73, 0, 4085, 0, 4094, 4068, 0, 1, 4060, 0, 4095, 4052, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 4092, 1, 1, 1, 1, 315, 4095, 4073, 36, 4095, 1, 4093, 1, 0, 4095, 33, 4056, 1, 0, 3589, 0, 0, 4095, 0, 1, 0, 4082, 0, 0, 38, 0, 1, 0, 3936, 0, 0, 1, 4095, 252, 0, 1, 3943, 0, 3808, 0, 2, 0, 2, 1, 0, 4095, 4093, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[0, 1, 4017, 1, 1, 0, 0, 0, 4094, 1, 1, 4095, 8, 1, 20, 2, 4092, 1, 4095, 0, 0, 0, 0, 4095, 4094, 1, 4071, 1, 1, 22, 4093, 1, 1, 1, 0, 0, 0, 0, 7, 0, 3988, 0, 11, 1, 0, 0, 0, 4, 410, 0, 0, 67, 0, 4087, 0, 4093, 4063, 0, 1, 4072, 0, 4095, 4046, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 4093, 1, 1, 1, 3, 150, 4095, 4095, 33, 4095, 1, 4094, 1, 0, 4095, 39, 4057, 1, 0, 3712, 0, 0, 4095, 0, 1, 0, 4072, 0, 0, 42, 0, 0, 0, 4034, 0, 0, 1, 4095, 268, 0, 1, 4093, 0, 3848, 0, 3, 0, 2, 1, 0, 4095, 4092, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 4064, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 4047, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 10, 0, 0, 0, 3422, 26, 0, 0, 0, 0, 0, 0, 2074, 0, 0, 12, 0, 4096, 0, 4096, 4093, 0, 0, 4096, 0, 4096, 3782, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 4094, 4051, 0, 4096, 0, 4094, 0, 0, 4094, 0, 4096, 0, 0, 4083, 0, 0, 4096, 0, 0, 0, 4096, 27, 0, 0, 5, 0, 0, 3654, 0, 0, 0, 4094, 2, 0, 0, 3507, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[1, 2, 4024, 1, 1, 1, 0, 0, 4094, 1, 1, 4095, 4, 1, 20, 2, 4094, 1, 4095, 0, 0, 0, 0, 4095, 4095, 1, 4021, 1, 1, 22, 4091, 1, 1, 1, 0, 0, 0, 0, 5, 0, 3913, 0, 7, 1, 1, 0, 0, 1, 656, 0, 0, 88, 0, 4090, 0, 4094, 4041, 0, 1, 4076, 0, 4095, 4068, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 4085, 1, 1, 1, 10, 286, 4095, 4087, 38, 4095, 1, 4091, 1, 0, 4095, 18, 4068, 1, 0, 3499, 0, 0, 4095, 0, 1, 0, 4079, 4, 0, 48, 0, 0, 0, 4017, 2, 0, 1, 4095, 246, 0, 1, 4069, 0, 3837, 0, 4, 0, 2, 1, 0, 4095, 4093, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[1, 1, 4029, 1, 1, 1, 2, 0, 4094, 1, 1, 4095, 5, 1, 16, 2, 4092, 1, 4095, 0, 0, 0, 1, 4094, 4095, 0, 3973, 1, 1, 31, 4090, 1, 1, 1, 0, 0, 0, 0, 4, 0, 3892, 0, 8, 0, 3, 0, 0, 1, 534, 0, 0, 28, 1, 4090, 0, 4095, 4060, 0, 1, 4086, 0, 4095, 4085, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 4067, 1, 0, 1, 5, 106, 4095, 4091, 1, 4095, 0, 4093, 0, 0, 4095, 5, 4062, 0, 0, 3876, 0, 0, 4095, 0, 1, 0, 4072, 109, 0, 33, 0, 0, 0, 3951, 24, 0, 1, 4095, 247, 0, 1, 4079, 0, 3855, 1, 8, 0, 1, 1, 0, 4095, 4094, 0, 1, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4058, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3325, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 15, 0, 0, 0, 4079, 62, 0, 0, 0, 0, 0, 0, 1032, 0, 0, 6, 0, 4096, 0, 4096, 4090, 0, 0, 4096, 0, 4096, 4031, 0, 24, 2, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4095, 3963, 0, 4096, 0, 4095, 0, 0, 4095, 0, 4096, 0, 1, 4083, 0, 0, 4096, 1, 0, 0, 4096, 81, 0, 0, 3, 0, 0, 3945, 0, 0, 0, 4095, 1, 0, 0, 4059, 0, 4096, 0, 0, 0, 0, 0, 7, 4096, 4096, 0, 0, 0] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 4078, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3579, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 13, 0, 0, 0, 3465, 23, 0, 0, 0, 0, 0, 0, 2770, 0, 0, 6, 0, 4096, 0, 4096, 4091, 0, 0, 4096, 0, 4096, 4089, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 7, 4096, 3986, 0, 4096, 0, 4094, 0, 0, 4094, 0, 4096, 0, 0, 4047, 0, 0, 4096, 0, 0, 0, 4096, 57, 0, 0, 3, 0, 0, 2689, 0, 0, 0, 4093, 0, 0, 0, 3956, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 4096, 0, 3705, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 6, 0, 0, 0, 3459, 53, 0, 0, 0, 0, 0, 0, 2166, 0, 0, 4, 0, 4096, 0, 4096, 4072, 0, 0, 4096, 0, 4096, 3793, 0, 11, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 4095, 3865, 0, 4096, 0, 4095, 0, 0, 4095, 0, 4096, 0, 1, 4073, 0, 0, 4096, 1, 0, 0, 4095, 47, 0, 0, 5, 0, 0, 3825, 0, 0, 0, 4094, 0, 0, 0, 3687, 0, 4096, 0, 0, 0, 0, 0, 1, 4095, 4096, 0, 0, 1] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 4023, 1, 1, 0, 0, 1, 4094, 1, 1, 4095, 5, 1, 14, 2, 3183, 942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 0, 4095, 244, 1, 4024, 1, 1, 29, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4, 0, 3594, 0, 10, 0, 0, 0, 0, 5, 4095, 4095, 4095, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1135, 0, 0, 73, 0, 2847, 0, 0, 4068, 0, 1, 4060, 0, 0, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 4092, 1, 1, 1, 1, 315, 4095, 4094, 3762, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4073, 36, 0, 1, 4093, 1, 0, 290, 33, 4056, 1, 0, 3589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4082, 0, 0, 38, 0, 1, 0, 3936, 0, 0, 1, 4095, 252, 3629, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3943, 0, 3808, 0, 2, 0, 2, 1, 0, 4095, 4093, 0, 1, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16366, 16577, 16233, 16236, 16588, 16173, 16657, 16242] avg=16384.0000 max=16657 max_rank=6 min=16173 min_rank=5 imbalance_ratio=1.016663 max_load_reduction=114415 max_load_reduction_pct=0.872917 moved_to_echo=65660 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 4064, 0, 0, 0, 0, 3182, 945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 244, 0, 4047, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 10, 0, 0, 0, 3422, 26, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2074, 0, 0, 12, 0, 2854, 0, 0, 4093, 0, 0, 4096, 0, 0, 3782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 4094, 4096, 3763, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 4094, 0, 0, 290, 0, 4096, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 27, 0, 0, 5, 0, 0, 3654, 0, 0, 0, 4094, 2, 3630, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3507, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16386, 16579, 16082, 16911, 16276, 16614, 16130, 16094] avg=16384.0000 max=16911 max_rank=3 min=16082 min_rank=2 imbalance_ratio=1.032166 max_load_reduction=114161 max_load_reduction_pct=0.870979 moved_to_echo=65114 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 4017, 1, 1, 0, 0, 0, 4094, 1, 1, 4095, 8, 1, 20, 2, 3188, 947, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 0, 4095, 232, 1, 4071, 1, 1, 22, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 7, 0, 3988, 0, 11, 1, 0, 0, 0, 4, 4095, 4095, 4095, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 410, 0, 0, 67, 0, 2839, 0, 0, 4063, 0, 1, 4072, 0, 0, 4046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 4093, 1, 1, 1, 3, 150, 4095, 4093, 3761, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 33, 0, 1, 4094, 1, 0, 279, 39, 4057, 1, 0, 3712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4072, 0, 0, 42, 0, 0, 0, 4034, 0, 0, 1, 4095, 268, 3633, 628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4093, 0, 3848, 0, 3, 0, 2, 1, 0, 4095, 4092, 0, 1, 0, 301, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16610, 16632, 15498, 16432, 16312, 16774, 16437] avg=16384.0000 max=16774 max_rank=6 min=15498 min_rank=3 imbalance_ratio=1.023804 max_load_reduction=114298 max_load_reduction_pct=0.872025 moved_to_echo=65955 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 1, 0, 4096, 0, 0, 4058, 0, 0, 0, 0, 3183, 945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 244, 0, 3325, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 15, 0, 0, 0, 4079, 62, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1032, 0, 0, 6, 0, 2854, 0, 0, 4090, 0, 0, 4096, 0, 0, 4031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 2, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4095, 4096, 3762, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 4095, 0, 0, 290, 0, 4096, 0, 1, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 81, 0, 0, 3, 0, 0, 3945, 0, 0, 0, 4095, 1, 3630, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 4096, 0, 0, 0, 0, 0, 7, 4096, 4096, 0, 0, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16379, 15857, 16782, 16109, 16292, 16528, 16474, 16651] avg=16384.0000 max=16782 max_rank=2 min=15857 min_rank=1 imbalance_ratio=1.024292 max_load_reduction=114290 max_load_reduction_pct=0.871964 moved_to_echo=65945 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 1, 4029, 1, 1, 1, 2, 0, 4094, 1, 1, 4095, 5, 1, 16, 2, 3183, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 1, 0, 0, 0, 0, 1, 4094, 244, 0, 3973, 1, 1, 31, 4090, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 4, 0, 3892, 0, 8, 0, 3, 0, 0, 1, 4095, 4095, 4095, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 534, 0, 0, 28, 1, 2851, 0, 0, 4060, 0, 1, 4086, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 4067, 1, 0, 1, 5, 106, 4095, 4095, 3762, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 1, 0, 0, 4093, 0, 0, 290, 5, 4062, 0, 0, 3876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4072, 109, 0, 33, 0, 0, 0, 3951, 24, 0, 1, 4095, 247, 3629, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4079, 0, 3855, 1, 8, 0, 1, 1, 0, 4095, 4094, 0, 1, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16377, 16529, 16528, 15646, 16358, 16418, 16784, 16432] avg=16384.0000 max=16784 max_rank=6 min=15646 min_rank=3 imbalance_ratio=1.024414 max_load_reduction=114288 max_load_reduction_pct=0.871948 moved_to_echo=65992 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 3, 0, 4096, 0, 0, 4078, 0, 0, 0, 0, 3182, 945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 244, 0, 3579, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 13, 0, 0, 0, 3465, 23, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2770, 0, 0, 6, 0, 2854, 0, 0, 4091, 0, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 7, 4096, 4096, 3762, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3986, 0, 0, 0, 4094, 0, 0, 290, 0, 4096, 0, 0, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 57, 0, 0, 3, 0, 0, 2689, 0, 0, 0, 4093, 0, 3630, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3956, 0, 4096, 0, 0, 0, 0, 0, 2, 4096, 4096, 0, 0, 1, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16400, 16111, 16125, 17906, 16283, 16513, 15190, 16544] avg=16384.0000 max=17906 max_rank=3 min=15190 min_rank=6 imbalance_ratio=1.092896 max_load_reduction=113166 max_load_reduction_pct=0.863388 moved_to_echo=64530 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1, 2, 4024, 1, 1, 1, 0, 0, 4094, 1, 1, 4095, 4, 1, 20, 2, 3183, 943, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 4095, 244, 1, 4021, 1, 1, 22, 4091, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 5, 0, 3913, 0, 7, 1, 1, 0, 0, 1, 4095, 4095, 4095, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 656, 0, 0, 88, 0, 2851, 0, 0, 4041, 0, 1, 4076, 0, 0, 4068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 4085, 1, 1, 1, 10, 286, 4095, 4094, 3761, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 38, 0, 1, 4091, 1, 0, 290, 18, 4068, 1, 0, 3499, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4079, 4, 0, 48, 0, 0, 0, 4017, 2, 0, 1, 4095, 246, 3629, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4069, 0, 3837, 0, 4, 0, 2, 1, 0, 4095, 4093, 0, 1, 0, 296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16374, 16572, 16549, 15781, 16559, 16094, 16744, 16399] avg=16384.0000 max=16744 max_rank=6 min=15781 min_rank=3 imbalance_ratio=1.021973 max_load_reduction=114328 max_load_reduction_pct=0.872253 moved_to_echo=65796 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0, 4091, 0, 0, 0, 0, 3183, 945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 244, 0, 3705, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6, 0, 0, 0, 3459, 53, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2166, 0, 0, 4, 0, 2854, 0, 0, 4072, 0, 0, 4096, 0, 0, 3793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 2, 4095, 4096, 3763, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3865, 0, 0, 0, 4095, 0, 0, 290, 0, 4096, 0, 1, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 47, 0, 0, 5, 0, 0, 3825, 0, 0, 0, 4094, 0, 3630, 622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3687, 0, 4096, 0, 0, 0, 0, 0, 1, 4095, 4096, 0, 0, 1, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16413, 16237, 16141, 16985, 16284, 16420, 16319, 16273] avg=16384.0000 max=16985 max_rank=3 min=16141 min_rank=2 imbalance_ratio=1.036682 max_load_reduction=114087 max_load_reduction_pct=0.870415 moved_to_echo=65296 active_clone_links=12 active_echo_slots_local=12 active_home_experts_offloading=8 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[32768, 128, 0, 0, 0, 0, 128, 32768, 2048, 128, 30848, 128, 128, 128, 32768, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 0, 32768, 384, 0, 256, 128, 128, 0, 30080, 128, 0, 128, 32768, 1024, 29056, 4992, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[32256, 128, 0, 128, 32768, 128, 0, 2432, 128, 32640, 128, 128, 30976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[128, 128, 128, 0, 128, 0, 128, 0, 29824, 256, 128, 128, 128, 0, 0, 128, 32768, 32768, 32768, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 128, 31488, 0, 31744, 128, 128, 0, 128, 128, 128, 32768, 32768, 0, 128, 128, 2432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132864 observed_loss=-1792 tokens_per_expert=[128, 128, 32512, 128, 128, 128, 128, 128, 32768, 128, 128, 32768, 128, 128, 128, 128, 25472, 7680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 128, 0, 128, 128, 0, 128, 128, 32768, 128, 128, 128, 128, 896, 32768, 32768, 30208, 1792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[10880, 0, 0, 384, 128, 22912, 0, 0, 32640, 0, 128, 32768, 0, 0, 32000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132864 slack_slots=129280 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 2510, 0, 4096, 0, 4096, 0, 4089, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 3564, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4094, 0, 0, 4096, 4096, 0, 4085, 92, 372, 4096, 2, 4096, 0, 0, 0, 0, 0, 4096, 1797, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 146, 0, 0, 0, 4093, 3789, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 0, 1, 15, 0, 4090, 1, 0, 0, 1675, 3010, 0, 4090, 0, 4094, 0, 4095, 1, 0, 8, 4095, 0, 0, 1, 1, 0, 0, 0, 1, 4071, 23, 1606, 1, 1, 0, 1, 4095, 0, 1, 1, 1, 2, 1, 4, 4094, 2, 0, 4095, 0, 5, 0, 4083, 1, 1, 1, 0, 0, 0, 0, 0, 36, 0, 3, 27, 1, 1, 1, 0, 4095, 1, 0, 4062, 4082, 1, 4095, 129, 16, 4092, 963, 4095, 0, 0, 69, 0, 1, 4072, 1344, 0, 1, 4057, 0, 1, 0, 1, 0, 7, 0, 4095, 4095, 4069, 1, 4095, 0, 4092, 0, 4095, 4095, 1, 1, 0, 1, 4095, 3530, 0, 4095, 0, 1, 1, 0, 1, 0, 1, 4091] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 0, 1, 13, 0, 4092, 0, 0, 0, 603, 3320, 0, 4091, 0, 4094, 0, 4095, 1, 0, 8, 4095, 0, 0, 1, 1, 0, 0, 0, 0, 4064, 15, 3753, 1, 1, 0, 1, 4095, 0, 1, 1, 1, 2, 0, 3, 4094, 2, 0, 4095, 1, 4, 0, 4057, 1, 2, 1, 0, 1, 0, 0, 0, 45, 0, 3, 26, 1, 1, 1, 19, 4095, 1, 1, 4029, 4081, 1, 4095, 6, 31, 4093, 645, 4095, 0, 0, 55, 0, 1, 4070, 182, 0, 1, 4047, 0, 1, 0, 1, 0, 6, 0, 4095, 4095, 4073, 1, 4095, 0, 4091, 0, 4095, 4095, 5, 1, 0, 1, 4095, 3799, 0, 4095, 0, 1, 1, 0, 1, 0, 1, 4092] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3579, 0, 4096, 0, 4096, 0, 4083, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 2651, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4074, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 4095, 0, 0, 4096, 4096, 0, 4079, 40, 236, 4096, 3, 4096, 0, 0, 0, 0, 0, 4096, 2045, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 23, 0, 0, 0, 4094, 3693, 3, 4095, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4092, 0, 0, 0, 4, 0, 4095, 0, 0, 0, 611, 3646, 0, 4091, 0, 4094, 0, 4095, 0, 0, 8, 4095, 0, 3, 1, 1, 0, 0, 0, 1, 4095, 40, 2387, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 1, 0, 2, 4093, 3, 0, 4095, 0, 1, 0, 4095, 1, 0, 1, 1, 2, 0, 1, 0, 10, 0, 2, 28, 1, 1, 1, 14, 4095, 0, 1, 4067, 4085, 1, 4095, 18, 341, 4092, 684, 4095, 0, 0, 0, 0, 0, 4063, 777, 0, 1, 4077, 0, 1, 0, 1, 0, 3, 0, 4095, 4095, 4089, 1, 4095, 0, 4093, 0, 4095, 4095, 2, 1, 0, 1, 4095, 3814, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 119, 2757, 0, 4096, 0, 4096, 0, 4092, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 3800, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4094, 0, 0, 4096, 4096, 0, 4089, 140, 128, 4096, 4, 4096, 0, 0, 0, 0, 0, 4096, 1500, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 4096, 0, 4096, 0, 4096, 0, 4096, 4096, 20, 0, 0, 0, 4094, 3813, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4091, 0, 0, 1, 14, 0, 4092, 0, 0, 0, 550, 3712, 0, 4091, 0, 4094, 0, 4095, 1, 0, 8, 4095, 0, 5, 1, 1, 0, 0, 0, 0, 4091, 22, 1746, 1, 1, 0, 1, 4095, 0, 1, 1, 1, 2, 0, 2, 4094, 3, 0, 4095, 1, 4, 0, 4088, 1, 0, 1, 0, 0, 0, 0, 0, 51, 0, 2, 27, 1, 1, 1, 0, 4095, 1, 1, 4064, 4075, 1, 4095, 14, 145, 4093, 959, 4095, 0, 0, 28, 0, 1, 4069, 2208, 0, 1, 4054, 0, 1, 0, 1, 0, 3, 0, 4095, 4095, 4073, 1, 4095, 0, 4090, 0, 4095, 4094, 9, 1, 0, 1, 4095, 2944, 0, 4095, 0, 1, 1, 0, 2, 0, 1, 4093] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 24, 3370, 0, 4096, 0, 4096, 0, 4090, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 2630, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4095, 0, 0, 4096, 4096, 0, 4083, 283, 228, 4096, 2, 4095, 0, 0, 0, 0, 0, 4096, 2332, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 4094, 4096, 0, 4096, 1, 4096, 0, 4096, 4096, 31, 0, 0, 0, 4095, 3400, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 15, 2510, 2774, 1726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4089, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3564, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4080, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2359, 1746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 4074, 0, 0, 4096, 4096, 0, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 372, 4096, 2, 1903, 0, 0, 0, 0, 0, 4096, 1797, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 146, 0, 0, 0, 4093, 3789, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 447, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15215, 16380, 17174, 16377, 16388, 16454, 16395, 16689] avg=16384.0000 max=17174 max_rank=2 min=15215 min_rank=0 imbalance_ratio=1.048218 max_load_reduction=113898 max_load_reduction_pct=0.868973 moved_to_echo=65926 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 0, 1, 15, 0, 4090, 1, 0, 0, 1675, 3010, 2774, 1726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 4094, 0, 4095, 1, 0, 8, 4095, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4071, 23, 1606, 1, 1, 0, 1, 4095, 0, 1, 1, 1, 2, 1, 4095, 1321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4094, 2, 0, 4095, 0, 5, 0, 4083, 1, 1, 1, 0, 0, 0, 0, 2358, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 3, 27, 1, 1, 1, 0, 4074, 1, 0, 4062, 4082, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 16, 4092, 963, 1903, 0, 0, 69, 0, 1, 4072, 1344, 0, 1, 4057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 7, 0, 0, 4095, 4069, 1, 0, 0, 4092, 0, 11, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 4095, 3530, 0, 4095, 0, 1, 1, 0, 1, 0, 1, 4091, 447, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[17388, 16386, 15221, 16389, 16384, 16647, 16372, 16285] avg=16384.0000 max=17388 max_rank=0 min=15221 min_rank=2 imbalance_ratio=1.061279 max_load_reduction=113684 max_load_reduction_pct=0.867340 moved_to_echo=65688 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4094, 0, 0, 1, 13, 0, 4092, 0, 0, 0, 603, 3320, 2775, 1732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4094, 0, 4095, 1, 0, 8, 4095, 0, 0, 1, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 15, 3753, 1, 1, 0, 1, 4095, 0, 1, 1, 1, 2, 0, 4095, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4094, 2, 0, 4095, 1, 4, 0, 4057, 1, 2, 1, 0, 1, 0, 0, 2363, 1748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 3, 26, 1, 1, 1, 19, 4070, 1, 1, 4029, 4081, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 31, 4093, 645, 1895, 0, 0, 55, 0, 1, 4070, 182, 0, 1, 4047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 6, 0, 0, 4095, 4073, 1, 0, 0, 4091, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 1, 4095, 3799, 0, 4095, 0, 1, 1, 0, 1, 0, 1, 4092, 452, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16632, 16389, 17350, 16372, 16374, 15026, 16363, 16566] avg=16384.0000 max=17350 max_rank=2 min=15026 min_rank=5 imbalance_ratio=1.058960 max_load_reduction=113722 max_load_reduction_pct=0.867630 moved_to_echo=64329 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3579, 2775, 1726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4083, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2651, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4074, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2359, 1746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 4074, 0, 0, 4096, 4096, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 236, 4096, 3, 1903, 0, 0, 0, 0, 0, 4096, 2045, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 4094, 3693, 3, 4095, 0, 0, 0, 0, 0, 0, 0, 4096, 447, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16272, 16373, 16260, 16371, 16415, 16515, 16395, 16471] avg=16384.0000 max=16515 max_rank=5 min=16260 min_rank=2 imbalance_ratio=1.007996 max_load_reduction=114557 max_load_reduction_pct=0.874001 moved_to_echo=65796 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4092, 0, 0, 0, 4, 0, 4095, 0, 0, 0, 611, 3646, 2774, 1726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4094, 0, 4095, 0, 0, 8, 4095, 0, 3, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 40, 2387, 1, 1, 0, 1, 4094, 0, 1, 1, 1, 1, 0, 4095, 1321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4093, 3, 0, 4095, 0, 1, 0, 4095, 1, 0, 1, 1, 2, 0, 1, 2358, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 2, 28, 1, 1, 1, 14, 4074, 0, 1, 4067, 4085, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 341, 4092, 684, 1903, 0, 0, 0, 0, 0, 4063, 777, 0, 1, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 3, 0, 0, 4095, 4089, 1, 0, 0, 4093, 0, 11, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 1, 4095, 3814, 0, 4095, 0, 1, 1, 0, 0, 0, 0, 4093, 447, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16950, 16389, 16040, 16398, 16380, 15956, 16389, 16570] avg=16384.0000 max=16950 max_rank=0 min=15956 min_rank=5 imbalance_ratio=1.034546 max_load_reduction=114122 max_load_reduction_pct=0.870682 moved_to_echo=65295 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 119, 2757, 2774, 1726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4092, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3800, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4088, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2359, 1746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 4074, 0, 0, 4096, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 128, 4096, 4, 1903, 0, 0, 0, 0, 0, 4096, 1500, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 4094, 3813, 9, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 447, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[15566, 16383, 17410, 16385, 16376, 15963, 16395, 16594] avg=16384.0000 max=17410 max_rank=2 min=15566 min_rank=0 imbalance_ratio=1.062622 max_load_reduction=113662 max_load_reduction_pct=0.867172 moved_to_echo=65328 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 1, 0, 4091, 0, 0, 1, 14, 0, 4092, 0, 0, 0, 550, 3712, 2775, 1726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 4094, 0, 4095, 1, 0, 8, 4095, 0, 5, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 22, 1746, 1, 1, 0, 1, 4095, 0, 1, 1, 1, 2, 0, 4095, 1320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4094, 3, 0, 4095, 1, 4, 0, 4088, 1, 0, 1, 0, 0, 0, 0, 2358, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 2, 27, 1, 1, 1, 0, 4074, 1, 1, 4064, 4075, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 145, 4093, 959, 1903, 0, 0, 28, 0, 1, 4069, 2208, 0, 1, 4054, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 3, 0, 0, 4095, 4073, 1, 0, 0, 4090, 0, 11, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 1, 4095, 2944, 0, 4095, 0, 1, 1, 0, 2, 0, 1, 4093, 447, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16963, 16392, 15377, 16392, 16394, 17475, 16369, 15710] avg=16384.0000 max=17475 max_rank=5 min=15377 min_rank=2 imbalance_ratio=1.066589 max_load_reduction=113597 max_load_reduction_pct=0.866676 moved_to_echo=65948 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 24, 3370, 2774, 1726, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 4096, 0, 4090, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2630, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 1322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2359, 1745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 4074, 0, 0, 4096, 4096, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 228, 4096, 2, 1903, 0, 0, 0, 0, 0, 4096, 2332, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 4096, 0, 0, 1, 4096, 0, 11, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 4095, 3400, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 447, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16086, 16380, 16240, 16388, 16361, 17036, 16394, 16187] avg=16384.0000 max=17036 max_rank=5 min=16086 min_rank=0 imbalance_ratio=1.039795 max_load_reduction=114036 max_load_reduction_pct=0.870026 moved_to_echo=65978 active_clone_links=9 active_echo_slots_local=9 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[256, 128, 0, 128, 32768, 28800, 128, 32768, 0, 128, 128, 0, 128, 0, 128, 32768, 3584, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 128, 32768, 128, 22144, 128, 128, 0, 128, 32768, 0, 128, 128, 128, 128, 128, 32768, 10624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 0, 128, 0, 128, 0, 0, 32768, 32768, 128, 0, 128, 32768, 0, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 32768, 128, 0, 32768, 128, 128, 0, 32768, 128, 128, 128, 128, 128, 0, 128, 18944, 14080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 32768, 0, 32768, 0, 32768, 128, 0, 128, 32768, 0, 128, 128, 128, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=131968 observed_loss=-896 tokens_per_expert=[0, 128, 128, 0, 32768, 0, 0, 128, 128, 0, 32768, 128, 0, 0, 3712, 25984, 22272, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[768, 1536, 32768, 3328, 15232, 0, 0, 256, 0, 128, 32768, 12288, 0, 128, 32640, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132352 observed_loss=-1280 tokens_per_expert=[0, 256, 0, 128, 128, 128, 128, 128, 256, 32640, 128, 128, 32640, 32768, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=131968 slack_slots=130176 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132352 slack_slots=129792 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 4076, 3931, 1, 5, 7, 1, 0, 0, 4095, 4094, 1, 0, 0, 4095, 0, 3224, 1, 4091, 71, 0, 158, 4095, 0, 0, 7, 100, 1, 1, 11, 1, 0, 4095, 0, 1, 0, 3, 0, 1, 1, 1, 440, 0, 1, 0, 15, 0, 3815, 4080, 0, 4094, 4095, 0, 0, 0, 0, 4024, 0, 3, 1, 23, 1, 0, 0, 4081, 34, 1, 0, 0, 1, 1, 4084, 4074, 4071, 0, 0, 0, 482, 0, 4062, 1, 4094, 4, 1, 0, 0, 4088, 1, 1, 4095, 4095, 0, 1, 4095, 4080, 0, 1, 4095, 0, 1, 1, 0, 4095, 1, 4095, 1751, 231, 0, 4095, 1, 0, 1, 2337, 1, 1, 1, 4095, 325, 3733] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 4096, 4086, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4046, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 559, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 0, 0, 2952, 0, 0, 0, 0, 0, 4096, 4096, 0, 4094, 4096, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 4096, 4096, 4093, 0, 7, 0, 45, 0, 4096, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 1, 4093, 0, 4096, 171, 0, 2, 4090, 0, 0, 0, 443, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4082, 3961, 1, 8, 7, 1, 0, 0, 4095, 4094, 1, 0, 0, 4095, 0, 1048, 1, 4091, 40, 0, 160, 4095, 0, 0, 7, 235, 1, 1, 13, 1, 0, 4094, 0, 1, 0, 4, 0, 1, 1, 1, 1359, 0, 1, 0, 12, 0, 4013, 4095, 0, 4094, 4095, 0, 0, 0, 0, 3962, 0, 3, 1, 15, 1, 0, 0, 4084, 15, 1, 0, 0, 2, 1, 4082, 4082, 4071, 0, 68, 0, 1431, 0, 4063, 1, 4094, 6, 1, 0, 0, 4088, 1, 1, 4095, 4094, 0, 1, 4095, 4082, 0, 1, 4095, 0, 1, 1, 0, 4095, 1, 4095, 1104, 264, 0, 4095, 1, 0, 1, 2887, 1, 2, 0, 4095, 316, 3765] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 4090, 3945, 1, 9, 5, 1, 0, 0, 4095, 4094, 1, 0, 0, 4095, 0, 2485, 2, 4092, 48, 0, 10, 4095, 0, 0, 5, 257, 1, 1, 0, 1, 0, 4095, 0, 1, 0, 2, 0, 1, 1, 1, 320, 0, 0, 0, 8, 0, 4056, 4095, 0, 4095, 4095, 0, 0, 0, 1, 3997, 0, 2, 1, 2, 1, 0, 0, 4091, 13, 1, 0, 0, 1, 1, 4086, 4089, 4059, 0, 172, 0, 636, 0, 4067, 1, 4094, 4, 1, 0, 1, 4085, 0, 1, 4095, 4095, 0, 1, 4095, 4069, 0, 0, 4095, 0, 1, 1, 0, 4095, 1, 4095, 1620, 179, 0, 4095, 1, 1, 1, 3022, 1, 2, 0, 4095, 301, 3598] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 0, 2, 0, 0, 0, 4089, 3919, 1, 13, 7, 1, 0, 0, 4095, 4094, 1, 0, 0, 4095, 0, 3806, 1, 4093, 71, 1, 129, 4095, 0, 0, 6, 272, 1, 1, 2, 1, 0, 4093, 0, 1, 0, 3, 0, 1, 1, 1, 677, 0, 1, 0, 7, 0, 3702, 4084, 0, 4094, 4095, 0, 0, 0, 1, 4034, 0, 1, 1, 3, 1, 0, 0, 4090, 11, 1, 0, 0, 0, 1, 4080, 4085, 4075, 0, 1, 0, 295, 0, 4055, 1, 4094, 7, 1, 0, 0, 4083, 1, 1, 4095, 4095, 0, 1, 4095, 4082, 0, 1, 4095, 0, 1, 1, 0, 4095, 1, 4095, 529, 224, 0, 4095, 1, 0, 1, 2860, 1, 2, 0, 4095, 358, 3768] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 4066, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 4050, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 567, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 2094, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 4096, 0, 0, 0, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 4096, 4096, 4093, 0, 17, 0, 42, 0, 4096, 0, 4096, 0, 0, 0, 2, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 0, 4094, 0, 4096, 222, 0, 3, 4083, 0, 0, 0, 1265, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 4096, 0, 3943, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 297, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 3298, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 4096, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 0, 0, 4096, 4096, 4096, 0, 64, 0, 14, 0, 4096, 0, 4096, 0, 0, 0, 1, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4095, 0, 0, 0, 1, 4095, 0, 4096, 7, 0, 0, 4078, 0, 0, 0, 590, 0, 0, 0, 4096, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 1, 4096, 0, 4047, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 372, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 1819, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 4096, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 4096, 4095, 0, 43, 0, 57, 0, 4096, 0, 4096, 0, 0, 0, 3, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 4096, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 4096, 152, 0, 1, 4084, 0, 0, 0, 1716, 0, 0, 0, 4095, 0, 4096] tokens_per_ep_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] avg=131072.0000 max=131072 max_rank=0 min=131072 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 4076, 3931, 1, 5, 7, 1, 4088, 3846, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 682, 4094, 1, 0, 0, 0, 0, 3224, 1, 4091, 71, 0, 158, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 100, 1, 1, 11, 1, 0, 4095, 0, 1, 0, 3, 0, 1, 4095, 4095, 3762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 440, 0, 1, 0, 15, 0, 3815, 4080, 0, 4094, 2574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4024, 0, 3, 1, 23, 1, 0, 0, 4081, 34, 1, 0, 0, 1, 1, 3413, 2665, 1521, 380, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3704, 4074, 4071, 0, 0, 0, 482, 0, 4062, 1, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 10, 4095, 0, 1, 0, 4080, 0, 1, 4095, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1430, 1751, 231, 0, 4095, 1, 0, 1, 2337, 1, 1, 1, 4095, 325, 3733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16290, 16417, 16173, 15021, 16388, 16399, 16381, 18003] avg=16384.0000 max=18003 max_rank=7 min=15021 min_rank=3 imbalance_ratio=1.098816 max_load_reduction=113069 max_load_reduction_pct=0.862648 moved_to_echo=67171 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 4082, 3961, 1, 8, 7, 1, 4088, 3849, 333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 676, 4094, 1, 0, 0, 0, 0, 1048, 1, 4091, 40, 0, 160, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 235, 1, 1, 13, 1, 0, 4094, 0, 1, 0, 4, 0, 1, 4095, 4095, 3761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1359, 0, 1, 0, 12, 0, 4013, 4095, 0, 4094, 2572, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3962, 0, 3, 1, 15, 1, 0, 0, 4084, 15, 1, 0, 0, 2, 1, 3419, 2668, 1523, 383, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3699, 4082, 4071, 0, 68, 0, 1431, 0, 4063, 1, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 4094, 0, 1, 0, 4082, 0, 1, 4095, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1427, 1104, 264, 0, 4095, 1, 0, 1, 2887, 1, 2, 0, 4095, 316, 3765, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16332, 14206, 16309, 16148, 16324, 17422, 16372, 17959] avg=16384.0000 max=17959 max_rank=7 min=14206 min_rank=1 imbalance_ratio=1.096130 max_load_reduction=113113 max_load_reduction_pct=0.862984 moved_to_echo=68077 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 4, 0, 0, 0, 2, 0, 0, 4096, 4086, 0, 0, 0, 0, 4096, 3847, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 682, 4096, 0, 0, 0, 0, 0, 4046, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 559, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 0, 0, 4096, 4096, 3764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2952, 0, 0, 0, 0, 0, 4096, 4096, 0, 4094, 2575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4093, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 0, 3414, 2665, 1521, 382, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3714, 4096, 4093, 0, 7, 0, 45, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1431, 171, 0, 2, 4090, 0, 0, 0, 443, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16463, 17016, 16593, 17813, 16415, 16051, 16392, 14329] avg=16384.0000 max=17813 max_rank=3 min=14329 min_rank=7 imbalance_ratio=1.087219 max_load_reduction=113259 max_load_reduction_pct=0.864098 moved_to_echo=63187 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 4090, 3945, 1, 9, 5, 1, 4085, 3846, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 682, 4094, 1, 0, 0, 0, 0, 2485, 2, 4092, 48, 0, 10, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 257, 1, 1, 0, 1, 0, 4095, 0, 1, 0, 2, 0, 1, 4095, 4095, 3762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 320, 0, 0, 0, 8, 0, 4056, 4095, 0, 4095, 2574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3997, 0, 2, 1, 2, 1, 0, 0, 4091, 13, 1, 0, 0, 1, 1, 3413, 2665, 1521, 381, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3705, 4089, 4059, 0, 172, 0, 636, 0, 4067, 1, 0, 4, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 4095, 0, 1, 0, 4069, 0, 0, 4095, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1430, 1620, 179, 0, 4095, 1, 1, 1, 3022, 1, 2, 0, 4095, 301, 3598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16317, 15509, 16316, 15150, 16330, 16735, 16368, 18347] avg=16384.0000 max=18347 max_rank=7 min=15150 min_rank=3 imbalance_ratio=1.119812 max_load_reduction=112725 max_load_reduction_pct=0.860023 moved_to_echo=67780 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 4087, 0, 0, 0, 0, 4096, 3847, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 682, 4096, 0, 0, 0, 0, 0, 3943, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 297, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4096, 4096, 3764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3298, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 2575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 15, 0, 0, 0, 3414, 2665, 1521, 382, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3714, 4096, 4096, 0, 64, 0, 14, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1431, 7, 0, 0, 4078, 0, 0, 0, 590, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16460, 16913, 16340, 18160, 16427, 16081, 16393, 14298] avg=16384.0000 max=18160 max_rank=3 min=14298 min_rank=7 imbalance_ratio=1.108398 max_load_reduction=112912 max_load_reduction_pct=0.861450 moved_to_echo=63199 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 4096, 4066, 0, 0, 0, 0, 4096, 3847, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 682, 4096, 0, 0, 0, 0, 0, 4050, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 567, 0, 0, 0, 0, 0, 4065, 0, 0, 0, 0, 0, 0, 4096, 4096, 3764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2094, 0, 0, 0, 0, 0, 4096, 4096, 0, 4095, 2575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4092, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 0, 3414, 2665, 1521, 382, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3714, 4096, 4093, 0, 17, 0, 42, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1431, 222, 0, 3, 4083, 0, 0, 0, 1265, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16446, 17020, 16588, 16956, 16415, 16060, 16391, 15196] avg=16384.0000 max=17020 max_rank=1 min=15196 min_rank=7 imbalance_ratio=1.038818 max_load_reduction=114052 max_load_reduction_pct=0.870148 moved_to_echo=64062 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[0, 1, 0, 0, 1, 0, 2, 0, 0, 0, 4089, 3919, 1, 13, 7, 1, 4083, 3846, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 682, 4094, 1, 0, 0, 0, 0, 3806, 1, 4093, 71, 1, 129, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 272, 1, 1, 2, 1, 0, 4093, 0, 1, 0, 3, 0, 1, 4095, 4095, 3762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 677, 0, 1, 0, 7, 0, 3702, 4084, 0, 4094, 2574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4034, 0, 1, 1, 3, 1, 0, 0, 4090, 11, 1, 0, 0, 0, 1, 3413, 2665, 1521, 380, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 4085, 4075, 0, 1, 0, 295, 0, 4055, 1, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 10, 4095, 0, 1, 0, 4082, 0, 1, 4095, 0, 1, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1430, 529, 224, 0, 4095, 1, 0, 1, 2860, 1, 2, 0, 4095, 358, 3768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16295, 16973, 16333, 15141, 16362, 16220, 16383, 17365] avg=16384.0000 max=17365 max_rank=7 min=15141 min_rank=3 imbalance_ratio=1.059875 max_load_reduction=113707 max_load_reduction_pct=0.867516 moved_to_echo=66330 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 4096, 4094, 0, 0, 0, 0, 4096, 3847, 332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 682, 4096, 0, 0, 1, 0, 0, 4047, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 372, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 4096, 4096, 3764, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1819, 0, 0, 0, 0, 0, 4096, 4096, 1, 4096, 2575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 3414, 2665, 1521, 382, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3714, 4096, 4095, 0, 43, 0, 57, 0, 4096, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1431, 152, 0, 1, 4084, 0, 0, 0, 1716, 0, 0, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[16469, 17018, 16420, 16683, 16411, 16104, 16392, 15575] avg=16384.0000 max=17018 max_rank=1 min=15575 min_rank=7 imbalance_ratio=1.038696 max_load_reduction=114054 max_load_reduction_pct=0.870163 moved_to_echo=64482 active_clone_links=11 active_echo_slots_local=11 active_home_experts_offloading=9 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=32 budget=262144 expected_recv=131072 expected_overflow=0 exact_recv_per_rank=[131072, 131072, 131072, 131072, 131072, 131072, 131072, 131072] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=262144 expected_recv=131072 actual_valid_recv=132608 observed_loss=-1536 tokens_per_expert=[128, 128, 0, 128, 128, 0, 128, 128, 128, 0, 32768, 32128, 128, 128, 128, 128, 32768, 30848, 2688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=262144 expected_recv=131072 actual_valid_recv=132096 observed_loss=-1024 tokens_per_expert=[128, 128, 128, 32768, 0, 128, 0, 32768, 0, 128, 32768, 0, 128, 128, 128, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[0, 0, 5504, 32768, 128, 0, 128, 0, 0, 26752, 128, 32768, 256, 128, 512, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 11520, 5632, 1024, 128, 32768, 128, 128, 128, 15232, 128, 128, 128, 32768, 1408, 31360, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[128, 128, 13056, 0, 128, 0, 128, 0, 32000, 32768, 128, 32768, 20608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=262144 expected_recv=131072 actual_valid_recv=132224 observed_loss=-1152 tokens_per_expert=[0, 0, 128, 2688, 128, 128, 128, 128, 0, 32768, 0, 128, 0, 128, 0, 128, 32768, 32768, 30208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=262144 expected_recv=131072 actual_valid_recv=131840 observed_loss=-768 tokens_per_expert=[29696, 32768, 32768, 0, 384, 0, 3072, 0, 32640, 128, 0, 128, 128, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=262144 expected_recv=131072 actual_valid_recv=132736 observed_loss=-1664 tokens_per_expert=[128, 32512, 128, 128, 128, 128, 128, 0, 0, 32768, 128, 128, 128, 0, 128, 128, 27392, 21376, 12288, 3072, 1920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=262144 valid_num_tokens=132608 slack_slots=129536 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=262144 valid_num_tokens=132096 slack_slots=130048 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=262144 valid_num_tokens=132224 slack_slots=129920 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=262144 valid_num_tokens=131840 slack_slots=130304 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=262144 valid_num_tokens=132736 slack_slots=129408 device_initiated_grouped_gemm=True + [2026-08-30 05:03:40] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 798.8 | throughput per GPU (TFLOP/s/GPU): 417.3 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.674550E+00 | loss scale: 1.0 | grad norm: 4.879 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-30 05:03:40 +[rank0]:[W830 05:03:42.229296748 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 05:03:44.552000 466818 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 05:03:44.553000 466818 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 466911 closing signal SIGTERM +W0830 05:03:44.553000 466818 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 466912 closing signal SIGTERM +W0830 05:03:44.553000 466818 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 466913 closing signal SIGTERM +W0830 05:03:44.554000 466818 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 466915 closing signal SIGTERM +W0830 05:03:44.554000 466818 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 466916 closing signal SIGTERM +W0830 05:03:44.554000 466818 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 466918 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 466818 got signal: 15 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/capacity_attempts.tsv b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/capacity_attempts.tsv index 9714c242d3c4a17cc3caedca625ecff6c99a5a51..97fc35a023530568e7d7f65dbd5767e09630df2b 100644 --- a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/capacity_attempts.tsv +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/capacity_attempts.tsv @@ -6,3 +6,6 @@ G2 echo forced 1.08203125 OK 119267 1 0.0 1.0234375 /workspace/tiera-rerun-fixed G3 echo measure 2.00 OK 136111 1 0.0 1.01953125 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap2p00/rep1 G3 echo validate 1.072265625 OK 122853 1 0.0 1.01953125 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1 G3 echo forced 1.072265625 OK 122899 1 0.0 1.017578125 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal/rep1 +G4 echo measure 2.00 OK 146037 1 0.0 1.0166015625 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1 +G4 echo validate 1.068359375 OK 129621 1 0.0 1.0166015625 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1 +G4 echo forced 1.068359375 OK 129579 1 0.0 1.013671875 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal/rep1 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/capacity_selection.tsv b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/capacity_selection.tsv index d9006e93b451fc939d96f0643bd9bf2f2ebef3f6..85b50135d95801751f45d11ad804da903a366bad 100644 --- a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/capacity_selection.tsv +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/capacity_selection.tsv @@ -2,3 +2,4 @@ model backend status selected_capacity required_capacity drop_rate_pct peak_mib G1 echo REQUIRED_CAPACITY_ABOVE_2.00 2.00 1.90625 0.0 125477 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1 G2 echo SELECTED 1.08203125 1.02734375 0.0 119267 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1 G3 echo SELECTED 1.072265625 1.01953125 0.0 122853 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1 +G4 echo SELECTED 1.068359375 1.0166015625 0.0 129621 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/probe_summary.json b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/probe_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a9687b5618747a3211acfc4b58de21068795d53a --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/probe_summary.json @@ -0,0 +1,882 @@ +[ + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "2.00", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.75, + "cold_expert_fraction_mean": 0.18587239583333334, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 384, + "commit": "b099fd84", + "config": "G1", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_accounting_source": "dispatch", + "drop_expected_assignments": 25165824.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.6113739013671875, + "echo_moved_assignment_fraction_mean": 0.5011123021443685, + "echo_moved_assignment_fraction_min": 0.4014892578125, + "echo_moved_assignment_fraction_n": 384, + "experts_per_rank": "1", + "force_balance": "0", + "global_batch_size": "16", + "global_snapshots": 384, + "hottest_expert_token_fraction_max": 1.0, + "hottest_expert_token_fraction_mean": 0.9049501419067383, + "hottest_expert_token_fraction_min": 0.374786376953125, + "hottest_expert_token_fraction_n": 384, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "8", + "num_layers": "16", + "observed_dropped_tokens": 0, + "post_physical_rank_cv_max": 0.3858312238475252, + "post_physical_rank_cv_mean": 0.04369442624774788, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 384, + "post_physical_rank_imax_max": 1.8909912109375, + "post_physical_rank_imax_mean": 1.0827395121256511, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 384, + "pre_home_rank_cv_max": 1.7320508075688772, + "pre_home_rank_cv_mean": 1.4536835274449291, + "pre_home_rank_cv_min": 0.2541278385706527, + "pre_home_rank_cv_n": 384, + "pre_home_rank_imax_max": 4.0, + "pre_home_rank_imax_mean": 3.619800567626953, + "pre_home_rank_imax_min": 1.4991455078125, + "pre_home_rank_imax_n": 384, + "raw_expert_cv_max": 1.7320508075688772, + "raw_expert_cv_mean": 1.4536835274449291, + "raw_expert_cv_min": 0.2541278385706527, + "raw_expert_cv_n": 384, + "raw_expert_imax_ceiling": 4.0, + "raw_expert_imax_hit_ceiling": 1, + "raw_expert_imax_max": 4.0, + "raw_expert_imax_mean": 3.619800567626953, + "raw_expert_imax_min": 1.4991455078125, + "raw_expert_imax_n": 384, + "records": { + "drop_dispatch": 3072, + "drop_experts": 3072, + "drop_setup": 3072, + "echo_post": 3072, + "echo_pre": 3072 + }, + "required_capacity_max": 1.90625, + "required_capacity_mean": 1.0099385579427083, + "required_capacity_min": 0.53125, + "required_capacity_n": 3072, + "router_balancing": "aux_loss", + "run_dir": "G1__echo__matched__seq4096__gbs16__cap2p00/rep1", + "saturated_dispatch_rows": 0, + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "2", + "valid_max_rank_reduction_max": 0.75, + "valid_max_rank_reduction_mean": 0.6772174606036838, + "valid_max_rank_reduction_min": 0.11090858104318564, + "valid_max_rank_reduction_n": 384, + "zero_drop": 1 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "1.08203125", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.71875, + "cold_expert_fraction_mean": 0.159912109375, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 384, + "commit": "b099fd84", + "config": "G2", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_accounting_source": "dispatch", + "drop_expected_assignments": 100663296.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.5000267028808594, + "echo_moved_assignment_fraction_mean": 0.499999205271403, + "echo_moved_assignment_fraction_min": 0.49983978271484375, + "echo_moved_assignment_fraction_n": 384, + "experts_per_rank": "4", + "force_balance": "0", + "global_batch_size": "16", + "global_snapshots": 384, + "hottest_expert_token_fraction_max": 1.0, + "hottest_expert_token_fraction_mean": 0.9680010477701823, + "hottest_expert_token_fraction_min": 0.613189697265625, + "hottest_expert_token_fraction_n": 384, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "32", + "num_layers": "16", + "observed_dropped_tokens": 0, + "post_physical_rank_cv_max": 0.00213623046875, + "post_physical_rank_cv_mean": 1.5938873823057422e-05, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 384, + "post_physical_rank_imax_max": 1.0042724609375, + "post_physical_rank_imax_mean": 1.0000306765238445, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 384, + "pre_home_rank_cv_max": 1.3175703221722546, + "pre_home_rank_cv_mean": 0.6974670284251169, + "pre_home_rank_cv_min": 0.0007166759378550646, + "pre_home_rank_cv_n": 384, + "pre_home_rank_imax_max": 3.998382568359375, + "pre_home_rank_imax_mean": 2.1703339417775473, + "pre_home_rank_imax_min": 1.001617431640625, + "pre_home_rank_imax_n": 384, + "raw_expert_cv_max": 1.7320419981493513, + "raw_expert_cv_mean": 1.5150670645228896, + "raw_expert_cv_min": 0.6923179293416859, + "raw_expert_cv_n": 384, + "raw_expert_imax_ceiling": 4.0, + "raw_expert_imax_hit_ceiling": 1, + "raw_expert_imax_max": 4.0, + "raw_expert_imax_mean": 3.872004191080729, + "raw_expert_imax_min": 2.4527587890625, + "raw_expert_imax_n": 384, + "records": { + "drop_dispatch": 3072, + "drop_experts": 3072, + "drop_setup": 3072, + "echo_post": 3072, + "echo_pre": 3072 + }, + "required_capacity_max": 1.02734375, + "required_capacity_mean": 1.0103823343912761, + "required_capacity_min": 1.0, + "required_capacity_n": 3072, + "router_balancing": "aux_loss", + "run_dir": "G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1", + "saturated_dispatch_rows": 0, + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.7498988696295957, + "valid_max_rank_reduction_mean": 0.5145680576214848, + "valid_max_rank_reduction_min": 0.0016148197800188904, + "valid_max_rank_reduction_n": 384, + "zero_drop": 1 + }, + { + "arm": "forced", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "1.08203125", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.0, + "cold_expert_fraction_mean": 0.0, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 384, + "commit": "b099fd84", + "config": "G2", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_accounting_source": "dispatch", + "drop_expected_assignments": 100663296.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.5, + "echo_moved_assignment_fraction_mean": 0.4999997019767761, + "echo_moved_assignment_fraction_min": 0.49993896484375, + "echo_moved_assignment_fraction_n": 384, + "experts_per_rank": "4", + "force_balance": "1", + "global_batch_size": "16", + "global_snapshots": 384, + "hottest_expert_token_fraction_max": 0.2593994140625, + "hottest_expert_token_fraction_mean": 0.25501465797424316, + "hottest_expert_token_fraction_min": 0.252777099609375, + "hottest_expert_token_fraction_n": 384, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "32", + "num_layers": "16", + "observed_dropped_tokens": 0, + "post_physical_rank_cv_max": 0.000421485208646641, + "post_physical_rank_cv_mean": 2.2897106263584403e-06, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 384, + "post_physical_rank_imax_max": 1.00067138671875, + "post_physical_rank_imax_mean": 1.0000041325887044, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 384, + "pre_home_rank_cv_max": 0.007852555942790041, + "pre_home_rank_cv_mean": 0.004394259674654735, + "pre_home_rank_cv_min": 0.0016880800477701613, + "pre_home_rank_cv_n": 384, + "pre_home_rank_imax_max": 1.01715087890625, + "pre_home_rank_imax_mean": 1.006885051727295, + "pre_home_rank_imax_min": 1.002349853515625, + "pre_home_rank_imax_n": 384, + "raw_expert_cv_max": 0.013095170649570529, + "raw_expert_cv_mean": 0.009386188814570484, + "raw_expert_cv_min": 0.006191008040561732, + "raw_expert_cv_n": 384, + "raw_expert_imax_ceiling": 4.0, + "raw_expert_imax_hit_ceiling": 0, + "raw_expert_imax_max": 1.03759765625, + "raw_expert_imax_mean": 1.0200586318969727, + "raw_expert_imax_min": 1.0111083984375, + "raw_expert_imax_n": 384, + "records": { + "drop_dispatch": 3072, + "drop_experts": 3072, + "drop_setup": 3072, + "echo_post": 3072, + "echo_pre": 3072 + }, + "required_capacity_max": 1.0234375, + "required_capacity_mean": 1.010022481282552, + "required_capacity_min": 1.00390625, + "required_capacity_n": 3072, + "router_balancing": "aux_loss", + "run_dir": "G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1", + "saturated_dispatch_rows": 0, + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.01686168616861686, + "valid_max_rank_reduction_mean": 0.006827796124259797, + "valid_max_rank_reduction_min": 0.0023443446491094536, + "valid_max_rank_reduction_n": 384, + "zero_drop": 1 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "2.00", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.71875, + "cold_expert_fraction_mean": 0.15087890625, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 384, + "commit": "b099fd84", + "config": "G2", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_accounting_source": "dispatch", + "drop_expected_assignments": 100663296.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.500030517578125, + "echo_moved_assignment_fraction_mean": 0.4999993046124776, + "echo_moved_assignment_fraction_min": 0.4998779296875, + "echo_moved_assignment_fraction_n": 384, + "experts_per_rank": "4", + "force_balance": "0", + "global_batch_size": "16", + "global_snapshots": 384, + "hottest_expert_token_fraction_max": 1.0, + "hottest_expert_token_fraction_mean": 0.9681024551391602, + "hottest_expert_token_fraction_min": 0.620849609375, + "hottest_expert_token_fraction_n": 384, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "32", + "num_layers": "16", + "observed_dropped_tokens": 0, + "post_physical_rank_cv_max": 0.0021209716796875, + "post_physical_rank_cv_mean": 1.466968415560366e-05, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 384, + "post_physical_rank_imax_max": 1.004241943359375, + "post_physical_rank_imax_mean": 1.0000279744466145, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 384, + "pre_home_rank_cv_max": 1.317620465314074, + "pre_home_rank_cv_mean": 0.6968048692022525, + "pre_home_rank_cv_min": 0.0012016720234495348, + "pre_home_rank_cv_n": 384, + "pre_home_rank_imax_max": 3.998260498046875, + "pre_home_rank_imax_mean": 2.1875499884287515, + "pre_home_rank_imax_min": 1.00238037109375, + "pre_home_rank_imax_n": 384, + "raw_expert_cv_max": 1.7320331892227239, + "raw_expert_cv_mean": 1.5175145186167975, + "raw_expert_cv_min": 0.6923179293416859, + "raw_expert_cv_n": 384, + "raw_expert_imax_ceiling": 4.0, + "raw_expert_imax_hit_ceiling": 1, + "raw_expert_imax_max": 4.0, + "raw_expert_imax_mean": 3.8724098205566406, + "raw_expert_imax_min": 2.4833984375, + "raw_expert_imax_n": 384, + "records": { + "drop_dispatch": 3072, + "drop_experts": 3072, + "drop_setup": 3072, + "echo_post": 3072, + "echo_pre": 3072 + }, + "required_capacity_max": 1.02734375, + "required_capacity_mean": 1.0105133056640625, + "required_capacity_min": 1.0, + "required_capacity_n": 3072, + "router_balancing": "aux_loss", + "run_dir": "G2__echo__matched__seq4096__gbs16__cap2p00/rep1", + "saturated_dispatch_rows": 0, + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.7498912338281876, + "valid_max_rank_reduction_mean": 0.5172585999990112, + "valid_max_rank_reduction_min": 0.0023747183827558913, + "valid_max_rank_reduction_n": 384, + "zero_drop": 1 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "1.072265625", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.6875, + "cold_expert_fraction_mean": 0.1546630859375, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 384, + "commit": "b099fd84", + "config": "G3", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_accounting_source": "dispatch", + "drop_expected_assignments": 201326592.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.5, + "echo_moved_assignment_fraction_mean": 0.5, + "echo_moved_assignment_fraction_min": 0.5, + "echo_moved_assignment_fraction_n": 384, + "experts_per_rank": "8", + "force_balance": "0", + "global_batch_size": "16", + "global_snapshots": 384, + "hottest_expert_token_fraction_max": 1.0, + "hottest_expert_token_fraction_mean": 0.9773949782053629, + "hottest_expert_token_fraction_min": 0.634124755859375, + "hottest_expert_token_fraction_n": 384, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "64", + "num_layers": "16", + "observed_dropped_tokens": 0, + "post_physical_rank_cv_max": 0.0, + "post_physical_rank_cv_mean": 0.0, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 384, + "post_physical_rank_imax_max": 1.0, + "post_physical_rank_imax_mean": 1.0, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 384, + "pre_home_rank_cv_max": 0.8292601101580809, + "pre_home_rank_cv_mean": 0.47371518867838985, + "pre_home_rank_cv_min": 0.1091963461644949, + "pre_home_rank_cv_n": 384, + "pre_home_rank_imax_max": 2.9946746826171875, + "pre_home_rank_imax_mean": 1.7838420073191326, + "pre_home_rank_imax_min": 1.153167724609375, + "pre_home_rank_imax_n": 384, + "raw_expert_cv_max": 1.7316460620806386, + "raw_expert_cv_mean": 1.525739653918128, + "raw_expert_cv_min": 0.6814247311375694, + "raw_expert_cv_n": 384, + "raw_expert_imax_ceiling": 4.0, + "raw_expert_imax_hit_ceiling": 1, + "raw_expert_imax_max": 4.0, + "raw_expert_imax_mean": 3.9095799128214517, + "raw_expert_imax_min": 2.5364990234375, + "raw_expert_imax_n": 384, + "records": { + "drop_dispatch": 3072, + "drop_experts": 3072, + "drop_setup": 3072, + "echo_post": 3072, + "echo_pre": 3072 + }, + "required_capacity_max": 1.01953125, + "required_capacity_mean": 1.0093224843343098, + "required_capacity_min": 1.0, + "required_capacity_n": 3072, + "router_balancing": "aux_loss", + "run_dir": "G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1", + "saturated_dispatch_rows": 0, + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "16", + "valid_max_rank_reduction_max": 0.6660739125339475, + "valid_max_rank_reduction_mean": 0.41846288670458703, + "valid_max_rank_reduction_min": 0.1328234578029481, + "valid_max_rank_reduction_n": 384, + "zero_drop": 1 + }, + { + "arm": "forced", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "1.072265625", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.0, + "cold_expert_fraction_mean": 0.0, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 384, + "commit": "b099fd84", + "config": "G3", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_accounting_source": "dispatch", + "drop_expected_assignments": 201326592.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.5, + "echo_moved_assignment_fraction_mean": 0.5, + "echo_moved_assignment_fraction_min": 0.5, + "echo_moved_assignment_fraction_n": 384, + "experts_per_rank": "8", + "force_balance": "1", + "global_batch_size": "16", + "global_snapshots": 384, + "hottest_expert_token_fraction_max": 0.26123046875, + "hottest_expert_token_fraction_mean": 0.2557719548543294, + "hottest_expert_token_fraction_min": 0.253570556640625, + "hottest_expert_token_fraction_n": 384, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "64", + "num_layers": "16", + "observed_dropped_tokens": 0, + "post_physical_rank_cv_max": 0.0, + "post_physical_rank_cv_mean": 0.0, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 384, + "post_physical_rank_imax_max": 1.0, + "post_physical_rank_imax_mean": 1.0, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 384, + "pre_home_rank_cv_max": 0.00610409733861204, + "pre_home_rank_cv_mean": 0.0030369541814070297, + "pre_home_rank_cv_min": 0.0013949279180933694, + "pre_home_rank_cv_n": 384, + "pre_home_rank_imax_max": 1.0114288330078125, + "pre_home_rank_imax_mean": 1.0047780275344849, + "pre_home_rank_imax_min": 1.001708984375, + "pre_home_rank_imax_n": 384, + "raw_expert_cv_max": 0.011936138368762292, + "raw_expert_cv_mean": 0.009561000998578461, + "raw_expert_cv_min": 0.006960450128604076, + "raw_expert_cv_n": 384, + "raw_expert_imax_ceiling": 4.0, + "raw_expert_imax_hit_ceiling": 0, + "raw_expert_imax_max": 1.044921875, + "raw_expert_imax_mean": 1.0230878194173176, + "raw_expert_imax_min": 1.0142822265625, + "raw_expert_imax_n": 384, + "records": { + "drop_dispatch": 3072, + "drop_experts": 3072, + "drop_setup": 3072, + "echo_post": 3072, + "echo_pre": 3072 + }, + "required_capacity_max": 1.017578125, + "required_capacity_mean": 1.0088431040445964, + "required_capacity_min": 1.00390625, + "required_capacity_n": 3072, + "router_balancing": "aux_loss", + "run_dir": "G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal/rep1", + "saturated_dispatch_rows": 0, + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "16", + "valid_max_rank_reduction_max": 0.011299690729425963, + "valid_max_rank_reduction_mean": 0.004752627833504172, + "valid_max_rank_reduction_min": 0.0017060687301974165, + "valid_max_rank_reduction_n": 384, + "zero_drop": 1 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "2.00", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.6875, + "cold_expert_fraction_mean": 0.15531412760416666, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 384, + "commit": "b099fd84", + "config": "G3", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_accounting_source": "dispatch", + "drop_expected_assignments": 201326592.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.5, + "echo_moved_assignment_fraction_mean": 0.5, + "echo_moved_assignment_fraction_min": 0.5, + "echo_moved_assignment_fraction_n": 384, + "experts_per_rank": "8", + "force_balance": "0", + "global_batch_size": "16", + "global_snapshots": 384, + "hottest_expert_token_fraction_max": 1.0, + "hottest_expert_token_fraction_mean": 0.9773902098337809, + "hottest_expert_token_fraction_min": 0.638397216796875, + "hottest_expert_token_fraction_n": 384, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "64", + "num_layers": "16", + "observed_dropped_tokens": 0, + "post_physical_rank_cv_max": 0.0, + "post_physical_rank_cv_mean": 0.0, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 384, + "post_physical_rank_imax_max": 1.0, + "post_physical_rank_imax_mean": 1.0, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 384, + "pre_home_rank_cv_max": 0.8292092994255315, + "pre_home_rank_cv_mean": 0.47732093430435846, + "pre_home_rank_cv_min": 0.1091963461644949, + "pre_home_rank_cv_n": 384, + "pre_home_rank_imax_max": 2.9329681396484375, + "pre_home_rank_imax_mean": 1.7915768225987752, + "pre_home_rank_imax_min": 1.1547698974609375, + "pre_home_rank_imax_n": 384, + "raw_expert_cv_max": 1.7316636929706255, + "raw_expert_cv_mean": 1.5249635478133088, + "raw_expert_cv_min": 0.6839438720259622, + "raw_expert_cv_n": 384, + "raw_expert_imax_ceiling": 4.0, + "raw_expert_imax_hit_ceiling": 1, + "raw_expert_imax_max": 4.0, + "raw_expert_imax_mean": 3.9095608393351235, + "raw_expert_imax_min": 2.5535888671875, + "raw_expert_imax_n": 384, + "records": { + "drop_dispatch": 3072, + "drop_experts": 3072, + "drop_setup": 3072, + "echo_post": 3072, + "echo_pre": 3072 + }, + "required_capacity_max": 1.01953125, + "required_capacity_mean": 1.0093262990315754, + "required_capacity_min": 1.0, + "required_capacity_n": 3072, + "router_balancing": "aux_loss", + "run_dir": "G3__echo__matched__seq4096__gbs16__cap2p00/rep1", + "saturated_dispatch_rows": 0, + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "16", + "valid_max_rank_reduction_max": 0.6590484613583747, + "valid_max_rank_reduction_mean": 0.4215810507718854, + "valid_max_rank_reduction_min": 0.13402661240238375, + "valid_max_rank_reduction_n": 384, + "zero_drop": 1 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "1.068359375", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.6796875, + "cold_expert_fraction_mean": 0.16324869791666666, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 384, + "commit": "b099fd84", + "config": "G4", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_accounting_source": "dispatch", + "drop_expected_assignments": 402653184.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.5, + "echo_moved_assignment_fraction_mean": 0.5, + "echo_moved_assignment_fraction_min": 0.5, + "echo_moved_assignment_fraction_n": 384, + "experts_per_rank": "16", + "force_balance": "0", + "global_batch_size": "16", + "global_snapshots": 384, + "hottest_expert_token_fraction_max": 1.0, + "hottest_expert_token_fraction_mean": 0.9825874964396158, + "hottest_expert_token_fraction_min": 0.5750732421875, + "hottest_expert_token_fraction_n": 384, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "128", + "num_layers": "16", + "observed_dropped_tokens": 0, + "post_physical_rank_cv_max": 0.0, + "post_physical_rank_cv_mean": 0.0, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 384, + "post_physical_rank_imax_max": 1.0, + "post_physical_rank_imax_mean": 1.0, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 384, + "pre_home_rank_cv_max": 0.6262571920835467, + "pre_home_rank_cv_mean": 0.33797223191114484, + "pre_home_rank_cv_min": 0.08738761375151358, + "pre_home_rank_cv_n": 384, + "pre_home_rank_imax_max": 2.2536544799804688, + "pre_home_rank_imax_mean": 1.550683856010437, + "pre_home_rank_imax_min": 1.1217041015625, + "pre_home_rank_imax_n": 384, + "raw_expert_cv_max": 1.7303400196303844, + "raw_expert_cv_mean": 1.5202854761326712, + "raw_expert_cv_min": 0.6197223679268041, + "raw_expert_cv_n": 384, + "raw_expert_imax_ceiling": 4.0, + "raw_expert_imax_hit_ceiling": 1, + "raw_expert_imax_max": 4.0, + "raw_expert_imax_mean": 3.9303499857584634, + "raw_expert_imax_min": 2.30029296875, + "raw_expert_imax_n": 384, + "records": { + "drop_dispatch": 3072, + "drop_experts": 3072, + "drop_setup": 3072, + "echo_post": 3072, + "echo_pre": 3072 + }, + "required_capacity_max": 1.0166015625, + "required_capacity_mean": 1.0086215337117512, + "required_capacity_min": 1.0009765625, + "required_capacity_n": 3072, + "router_balancing": "aux_loss", + "run_dir": "G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1", + "saturated_dispatch_rows": 0, + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "32", + "valid_max_rank_reduction_max": 0.5562762575704744, + "valid_max_rank_reduction_mean": 0.3421405808049787, + "valid_max_rank_reduction_min": 0.10849929263249537, + "valid_max_rank_reduction_n": 384, + "zero_drop": 1 + }, + { + "arm": "forced", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "1.068359375", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.0, + "cold_expert_fraction_mean": 0.0, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 384, + "commit": "b099fd84", + "config": "G4", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_accounting_source": "dispatch", + "drop_expected_assignments": 402653184.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.5, + "echo_moved_assignment_fraction_mean": 0.5, + "echo_moved_assignment_fraction_min": 0.5, + "echo_moved_assignment_fraction_n": 384, + "experts_per_rank": "16", + "force_balance": "1", + "global_batch_size": "16", + "global_snapshots": 384, + "hottest_expert_token_fraction_max": 0.259674072265625, + "hottest_expert_token_fraction_mean": 0.25621263186136883, + "hottest_expert_token_fraction_min": 0.253997802734375, + "hottest_expert_token_fraction_n": 384, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "128", + "num_layers": "16", + "observed_dropped_tokens": 0, + "post_physical_rank_cv_max": 0.0, + "post_physical_rank_cv_mean": 0.0, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 384, + "post_physical_rank_imax_max": 1.0, + "post_physical_rank_imax_mean": 1.0, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 384, + "pre_home_rank_cv_max": 0.0036539583092664103, + "pre_home_rank_cv_mean": 0.002179247890018717, + "pre_home_rank_cv_min": 0.0006899225660317828, + "pre_home_rank_cv_n": 384, + "pre_home_rank_imax_max": 1.0071258544921875, + "pre_home_rank_imax_mean": 1.0034127434094746, + "pre_home_rank_imax_min": 1.0009918212890625, + "pre_home_rank_imax_n": 384, + "raw_expert_cv_max": 0.01168330115931115, + "raw_expert_cv_mean": 0.009563696224675585, + "raw_expert_cv_min": 0.008002983940271118, + "raw_expert_cv_n": 384, + "raw_expert_imax_ceiling": 4.0, + "raw_expert_imax_hit_ceiling": 0, + "raw_expert_imax_max": 1.0386962890625, + "raw_expert_imax_mean": 1.0248505274454753, + "raw_expert_imax_min": 1.0159912109375, + "raw_expert_imax_n": 384, + "records": { + "drop_dispatch": 3072, + "drop_experts": 3072, + "drop_setup": 3072, + "echo_post": 3072, + "echo_pre": 3072 + }, + "required_capacity_max": 1.013671875, + "required_capacity_mean": 1.0081939697265625, + "required_capacity_min": 1.00390625, + "required_capacity_n": 3072, + "router_balancing": "aux_loss", + "run_dir": "G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal/rep1", + "saturated_dispatch_rows": 0, + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "32", + "valid_max_rank_reduction_max": 0.007075435965031893, + "valid_max_rank_reduction_mean": 0.0033999001306743632, + "valid_max_rank_reduction_min": 0.000990838554290331, + "valid_max_rank_reduction_n": 384, + "zero_drop": 1 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "2.00", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.671875, + "cold_expert_fraction_mean": 0.16269938151041666, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 384, + "commit": "b099fd84", + "config": "G4", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_accounting_source": "dispatch", + "drop_expected_assignments": 402653184.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.5, + "echo_moved_assignment_fraction_mean": 0.5, + "echo_moved_assignment_fraction_min": 0.5, + "echo_moved_assignment_fraction_n": 384, + "experts_per_rank": "16", + "force_balance": "0", + "global_batch_size": "16", + "global_snapshots": 384, + "hottest_expert_token_fraction_max": 1.0, + "hottest_expert_token_fraction_mean": 0.9825185934702555, + "hottest_expert_token_fraction_min": 0.57318115234375, + "hottest_expert_token_fraction_n": 384, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "16", + "num_experts": "128", + "num_layers": "16", + "observed_dropped_tokens": 0, + "post_physical_rank_cv_max": 0.0, + "post_physical_rank_cv_mean": 0.0, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 384, + "post_physical_rank_imax_max": 1.0, + "post_physical_rank_imax_mean": 1.0, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 384, + "pre_home_rank_cv_max": 0.6019828931462307, + "pre_home_rank_cv_mean": 0.3383858718614603, + "pre_home_rank_cv_min": 0.08738761375151358, + "pre_home_rank_cv_n": 384, + "pre_home_rank_imax_max": 2.255584716796875, + "pre_home_rank_imax_mean": 1.5542763471603394, + "pre_home_rank_imax_min": 1.1217041015625, + "pre_home_rank_imax_n": 384, + "raw_expert_cv_max": 1.7304659197188927, + "raw_expert_cv_mean": 1.521021036726468, + "raw_expert_cv_min": 0.6188791003597877, + "raw_expert_cv_n": 384, + "raw_expert_imax_ceiling": 4.0, + "raw_expert_imax_hit_ceiling": 1, + "raw_expert_imax_max": 4.0, + "raw_expert_imax_mean": 3.930074373881022, + "raw_expert_imax_min": 2.292724609375, + "raw_expert_imax_n": 384, + "records": { + "drop_dispatch": 3072, + "drop_experts": 3072, + "drop_setup": 3072, + "echo_post": 3072, + "echo_pre": 3072 + }, + "required_capacity_max": 1.0166015625, + "required_capacity_mean": 1.0086164474487305, + "required_capacity_min": 1.0009765625, + "required_capacity_n": 3072, + "router_balancing": "aux_loss", + "run_dir": "G4__echo__matched__seq4096__gbs16__cap2p00/rep1", + "saturated_dispatch_rows": 0, + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "32", + "valid_max_rank_reduction_max": 0.5566559781358661, + "valid_max_rank_reduction_mean": 0.3437128068160493, + "valid_max_rank_reduction_min": 0.10849929263249537, + "valid_max_rank_reduction_n": 384, + "zero_drop": 1 + } +] \ No newline at end of file diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/probe_summary.tsv b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/probe_summary.tsv new file mode 100644 index 0000000000000000000000000000000000000000..c9f47fb62aa650c7e868e2eff6c5b9980095cbc3 --- /dev/null +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/probe_summary.tsv @@ -0,0 +1,11 @@ +run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap assignment_sum_mismatches cold_expert_fraction_max cold_expert_fraction_mean cold_expert_fraction_min cold_expert_fraction_n drop_accounting_source drop_expected_assignments drop_rate_pct_overall echo_logged_load_reduction_valid echo_logged_pre_rank_metric_valid echo_moved_assignment_fraction_max echo_moved_assignment_fraction_mean echo_moved_assignment_fraction_min echo_moved_assignment_fraction_n global_snapshots hottest_expert_token_fraction_max hottest_expert_token_fraction_mean hottest_expert_token_fraction_min hottest_expert_token_fraction_n incomplete_snapshots malformed_records observed_dropped_tokens post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n raw_expert_cv_max raw_expert_cv_mean raw_expert_cv_min raw_expert_cv_n raw_expert_imax_ceiling raw_expert_imax_hit_ceiling raw_expert_imax_max raw_expert_imax_mean raw_expert_imax_min raw_expert_imax_n required_capacity_max required_capacity_mean required_capacity_min required_capacity_n saturated_dispatch_rows valid_max_rank_reduction_max valid_max_rank_reduction_mean valid_max_rank_reduction_min valid_max_rank_reduction_n zero_drop +G1__echo__matched__seq4096__gbs16__cap2p00/rep1 natural G1 echo 8 2 1 16 16 0 4096 16 static 2.00 0 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.75 0.18587239583333334 0.0 384 dispatch 25165824.0 0.0 0 0 0.6113739013671875 0.5011123021443685 0.4014892578125 384 384 1.0 0.9049501419067383 0.374786376953125 384 0 0 0 0.3858312238475252 0.04369442624774788 0.0 384 1.8909912109375 1.0827395121256511 1.0 384 1.7320508075688772 1.4536835274449291 0.2541278385706527 384 4.0 3.619800567626953 1.4991455078125 384 1.7320508075688772 1.4536835274449291 0.2541278385706527 384 4.0 1 4.0 3.619800567626953 1.4991455078125 384 1.90625 1.0099385579427083 0.53125 3072 0 0.75 0.6772174606036838 0.11090858104318564 384 1 +G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1 natural G2 echo 32 8 4 16 16 0 4096 16 static 1.08203125 0 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.71875 0.159912109375 0.0 384 dispatch 100663296.0 0.0 0 0 0.5000267028808594 0.499999205271403 0.49983978271484375 384 384 1.0 0.9680010477701823 0.613189697265625 384 0 0 0 0.00213623046875 1.5938873823057422e-05 0.0 384 1.0042724609375 1.0000306765238445 1.0 384 1.3175703221722546 0.6974670284251169 0.0007166759378550646 384 3.998382568359375 2.1703339417775473 1.001617431640625 384 1.7320419981493513 1.5150670645228896 0.6923179293416859 384 4.0 1 4.0 3.872004191080729 2.4527587890625 384 1.02734375 1.0103823343912761 1.0 3072 0 0.7498988696295957 0.5145680576214848 0.0016148197800188904 384 1 +G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1 forced G2 echo 32 8 4 16 16 0 4096 16 static 1.08203125 1 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.0 0.0 0.0 384 dispatch 100663296.0 0.0 0 0 0.5 0.4999997019767761 0.49993896484375 384 384 0.2593994140625 0.25501465797424316 0.252777099609375 384 0 0 0 0.000421485208646641 2.2897106263584403e-06 0.0 384 1.00067138671875 1.0000041325887044 1.0 384 0.007852555942790041 0.004394259674654735 0.0016880800477701613 384 1.01715087890625 1.006885051727295 1.002349853515625 384 0.013095170649570529 0.009386188814570484 0.006191008040561732 384 4.0 0 1.03759765625 1.0200586318969727 1.0111083984375 384 1.0234375 1.010022481282552 1.00390625 3072 0 0.01686168616861686 0.006827796124259797 0.0023443446491094536 384 1 +G2__echo__matched__seq4096__gbs16__cap2p00/rep1 natural G2 echo 32 8 4 16 16 0 4096 16 static 2.00 0 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.71875 0.15087890625 0.0 384 dispatch 100663296.0 0.0 0 0 0.500030517578125 0.4999993046124776 0.4998779296875 384 384 1.0 0.9681024551391602 0.620849609375 384 0 0 0 0.0021209716796875 1.466968415560366e-05 0.0 384 1.004241943359375 1.0000279744466145 1.0 384 1.317620465314074 0.6968048692022525 0.0012016720234495348 384 3.998260498046875 2.1875499884287515 1.00238037109375 384 1.7320331892227239 1.5175145186167975 0.6923179293416859 384 4.0 1 4.0 3.8724098205566406 2.4833984375 384 1.02734375 1.0105133056640625 1.0 3072 0 0.7498912338281876 0.5172585999990112 0.0023747183827558913 384 1 +G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1 natural G3 echo 64 16 8 16 16 0 4096 16 static 1.072265625 0 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.6875 0.1546630859375 0.0 384 dispatch 201326592.0 0.0 0 0 0.5 0.5 0.5 384 384 1.0 0.9773949782053629 0.634124755859375 384 0 0 0 0.0 0.0 0.0 384 1.0 1.0 1.0 384 0.8292601101580809 0.47371518867838985 0.1091963461644949 384 2.9946746826171875 1.7838420073191326 1.153167724609375 384 1.7316460620806386 1.525739653918128 0.6814247311375694 384 4.0 1 4.0 3.9095799128214517 2.5364990234375 384 1.01953125 1.0093224843343098 1.0 3072 0 0.6660739125339475 0.41846288670458703 0.1328234578029481 384 1 +G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal/rep1 forced G3 echo 64 16 8 16 16 0 4096 16 static 1.072265625 1 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.0 0.0 0.0 384 dispatch 201326592.0 0.0 0 0 0.5 0.5 0.5 384 384 0.26123046875 0.2557719548543294 0.253570556640625 384 0 0 0 0.0 0.0 0.0 384 1.0 1.0 1.0 384 0.00610409733861204 0.0030369541814070297 0.0013949279180933694 384 1.0114288330078125 1.0047780275344849 1.001708984375 384 0.011936138368762292 0.009561000998578461 0.006960450128604076 384 4.0 0 1.044921875 1.0230878194173176 1.0142822265625 384 1.017578125 1.0088431040445964 1.00390625 3072 0 0.011299690729425963 0.004752627833504172 0.0017060687301974165 384 1 +G3__echo__matched__seq4096__gbs16__cap2p00/rep1 natural G3 echo 64 16 8 16 16 0 4096 16 static 2.00 0 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.6875 0.15531412760416666 0.0 384 dispatch 201326592.0 0.0 0 0 0.5 0.5 0.5 384 384 1.0 0.9773902098337809 0.638397216796875 384 0 0 0 0.0 0.0 0.0 384 1.0 1.0 1.0 384 0.8292092994255315 0.47732093430435846 0.1091963461644949 384 2.9329681396484375 1.7915768225987752 1.1547698974609375 384 1.7316636929706255 1.5249635478133088 0.6839438720259622 384 4.0 1 4.0 3.9095608393351235 2.5535888671875 384 1.01953125 1.0093262990315754 1.0 3072 0 0.6590484613583747 0.4215810507718854 0.13402661240238375 384 1 +G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1 natural G4 echo 128 32 16 16 16 0 4096 16 static 1.068359375 0 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.6796875 0.16324869791666666 0.0 384 dispatch 402653184.0 0.0 0 0 0.5 0.5 0.5 384 384 1.0 0.9825874964396158 0.5750732421875 384 0 0 0 0.0 0.0 0.0 384 1.0 1.0 1.0 384 0.6262571920835467 0.33797223191114484 0.08738761375151358 384 2.2536544799804688 1.550683856010437 1.1217041015625 384 1.7303400196303844 1.5202854761326712 0.6197223679268041 384 4.0 1 4.0 3.9303499857584634 2.30029296875 384 1.0166015625 1.0086215337117512 1.0009765625 3072 0 0.5562762575704744 0.3421405808049787 0.10849929263249537 384 1 +G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal/rep1 forced G4 echo 128 32 16 16 16 0 4096 16 static 1.068359375 1 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.0 0.0 0.0 384 dispatch 402653184.0 0.0 0 0 0.5 0.5 0.5 384 384 0.259674072265625 0.25621263186136883 0.253997802734375 384 0 0 0 0.0 0.0 0.0 384 1.0 1.0 1.0 384 0.0036539583092664103 0.002179247890018717 0.0006899225660317828 384 1.0071258544921875 1.0034127434094746 1.0009918212890625 384 0.01168330115931115 0.009563696224675585 0.008002983940271118 384 4.0 0 1.0386962890625 1.0248505274454753 1.0159912109375 384 1.013671875 1.0081939697265625 1.00390625 3072 0 0.007075435965031893 0.0033999001306743632 0.000990838554290331 384 1 +G4__echo__matched__seq4096__gbs16__cap2p00/rep1 natural G4 echo 128 32 16 16 16 0 4096 16 static 2.00 0 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.671875 0.16269938151041666 0.0 384 dispatch 402653184.0 0.0 0 0 0.5 0.5 0.5 384 384 1.0 0.9825185934702555 0.57318115234375 384 0 0 0 0.0 0.0 0.0 384 1.0 1.0 1.0 384 0.6019828931462307 0.3383858718614603 0.08738761375151358 384 2.255584716796875 1.5542763471603394 1.1217041015625 384 1.7304659197188927 1.521021036726468 0.6188791003597877 384 4.0 1 4.0 3.930074373881022 2.292724609375 384 1.0166015625 1.0086164474487305 1.0009765625 3072 0 0.5566559781358661 0.3437128068160493 0.10849929263249537 384 1 diff --git a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/results.tsv b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/results.tsv index 51b7bb1060d2e6628551a2d6059daa0a0d9dccb6..3163eb473c16472a044e3e333e04c72519066ccc 100644 --- a/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/results.tsv +++ b/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/results.tsv @@ -6,3 +6,6 @@ G2 echo matched 4 4096 16 1.08203125 none+forcebal 1 OK 628.07 616.10 119267 /wo G3 echo matched 8 4096 16 2.00 none 1 OK 721.76 713.60 136111 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap2p00/rep1 G3 echo matched 8 4096 16 1.072265625 none 1 OK 681.37 670.20 122853 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1 G3 echo matched 8 4096 16 1.072265625 none+forcebal 1 OK 664.27 659.10 122899 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal/rep1 +G4 echo matched 16 4096 16 2.00 none 1 OK 829.14 790.80 146037 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1 +G4 echo matched 16 4096 16 1.068359375 none 1 OK 775.73 738.90 129621 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1 +G4 echo matched 16 4096 16 1.068359375 none+forcebal 1 OK 768.04 733.40 129579 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal/rep1